1	/*******************************************************************
2	* This file is part of the Emulex Linux Device Driver for *
3	* Fibre Channel Host Bus Adapters. *
4	* Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
5	* “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. *
6	* Copyright (C) 2004-2016 Emulex. All rights reserved. *
7	* EMULEX and SLI are trademarks of Emulex. *
8	* www.broadcom.com *
9	* Portions Copyright (C) 2004-2005 Christoph Hellwig *
10	* *
11	* This program is free software; you can redistribute it and/or *
12	* modify it under the terms of version 2 of the GNU General *
13	* Public License as published by the Free Software Foundation. *
14	* This program is distributed in the hope that it will be useful. *
15	* ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
16	* WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
17	* FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
18	* DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19	* TO BE LEGALLY INVALID. See the GNU General Public License for *
20	* more details, a copy of which can be found in the file COPYING *
21	* included with this package. *
22	*******************************************************************/
23
24	#include <linux/blkdev.h>
25	#include <linux/pci.h>
26	#include <linux/interrupt.h>
27	#include <linux/delay.h>
28	#include <linux/slab.h>
29	#include <linux/lockdep.h>
30
31	#include <scsi/scsi.h>
32	#include <scsi/scsi_cmnd.h>
33	#include <scsi/scsi_device.h>
34	#include <scsi/scsi_host.h>
35	#include <scsi/scsi_transport_fc.h>
36	#include <scsi/fc/fc_fs.h>
37	#include <linux/crash_dump.h>
38	#ifdef CONFIG_X86
39	#include <asm/set_memory.h>
40	#endif
41
42	#include "lpfc_hw4.h"
43	#include "lpfc_hw.h"
44	#include "lpfc_sli.h"
45	#include "lpfc_sli4.h"
46	#include "lpfc_nl.h"
47	#include "lpfc_disc.h"
48	#include "lpfc.h"
49	#include "lpfc_scsi.h"
50	#include "lpfc_nvme.h"
51	#include "lpfc_crtn.h"
52	#include "lpfc_logmsg.h"
53	#include "lpfc_compat.h"
54	#include "lpfc_debugfs.h"
55	#include "lpfc_vport.h"
56	#include "lpfc_version.h"
57
58	/* There are only four IOCB completion types. */
59	typedef enum _lpfc_iocb_type {
60	LPFC_UNKNOWN_IOCB,
61	LPFC_UNSOL_IOCB,
62	LPFC_SOL_IOCB,
63	LPFC_ABORT_IOCB
64	} lpfc_iocb_type;
65
66
67	/* Provide function prototypes local to this module. */
68	static int lpfc_sli_issue_mbox_s4(struct lpfc_hba *, LPFC_MBOXQ_t *,
69	uint32_t);
70	static int lpfc_sli4_read_rev(struct lpfc_hba *, LPFC_MBOXQ_t *,
71	uint8_t *, uint32_t *);
72	static struct lpfc_iocbq *
73	lpfc_sli4_els_preprocess_rspiocbq(struct lpfc_hba *phba,
74	struct lpfc_iocbq *rspiocbq);
75	static void lpfc_sli4_send_seq_to_ulp(struct lpfc_vport *,
76	struct hbq_dmabuf *);
77	static void lpfc_sli4_handle_mds_loopback(struct lpfc_vport *vport,
78	struct hbq_dmabuf *dmabuf);
79	static bool lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba,
80	struct lpfc_queue *cq, struct lpfc_cqe *cqe);
81	static int lpfc_sli4_post_sgl_list(struct lpfc_hba *, struct list_head *,
82	int);
83	static void lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba,
84	struct lpfc_queue *eq,
85	struct lpfc_eqe *eqe,
86	enum lpfc_poll_mode poll_mode);
87	static bool lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba);
88	static bool lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba);
89	static struct lpfc_cqe *lpfc_sli4_cq_get(struct lpfc_queue *q);
90	static void __lpfc_sli4_consume_cqe(struct lpfc_hba *phba,
91	struct lpfc_queue *cq,
92	struct lpfc_cqe *cqe);
93	static uint16_t lpfc_wqe_bpl2sgl(struct lpfc_hba *phba,
94	struct lpfc_iocbq *pwqeq,
95	struct lpfc_sglq *sglq);
96
97	union lpfc_wqe128 lpfc_iread_cmd_template;
98	union lpfc_wqe128 lpfc_iwrite_cmd_template;
99	union lpfc_wqe128 lpfc_icmnd_cmd_template;
100
101	/* Setup WQE templates for IOs */
102	void lpfc_wqe_cmd_template(void)
103	{
104	union lpfc_wqe128 *wqe;
105
106	/* IREAD template */
107	wqe = &lpfc_iread_cmd_template;
108	memset(wqe, 0, sizeof(union lpfc_wqe128));
109
110	/* Word 0, 1, 2 - BDE is variable */
111
112	/* Word 3 - cmd_buff_len, payload_offset_len is zero */
113
114	/* Word 4 - total_xfer_len is variable */
115
116	/* Word 5 - is zero */
117
118	/* Word 6 - ctxt_tag, xri_tag is variable */
119
120	/* Word 7 */
121	bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE);
122	bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK);
123	bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3);
124	bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI);
125
126	/* Word 8 - abort_tag is variable */
127
128	/* Word 9 - reqtag is variable */
129
130	/* Word 10 - dbde, wqes is variable */
131	bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0);
132	bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ);
133	bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4);
134	bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0);
135	bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1);
136
137	/* Word 11 - pbde is variable */
138	bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, COMMAND_DATA_IN);
139	bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
140	bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 0);
141
142	/* Word 12 - is zero */
143
144	/* Word 13, 14, 15 - PBDE is variable */
145
146	/* IWRITE template */
147	wqe = &lpfc_iwrite_cmd_template;
148	memset(wqe, 0, sizeof(union lpfc_wqe128));
149
150	/* Word 0, 1, 2 - BDE is variable */
151
152	/* Word 3 - cmd_buff_len, payload_offset_len is zero */
153
154	/* Word 4 - total_xfer_len is variable */
155
156	/* Word 5 - initial_xfer_len is variable */
157
158	/* Word 6 - ctxt_tag, xri_tag is variable */
159
160	/* Word 7 */
161	bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE);
162	bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK);
163	bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3);
164	bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI);
165
166	/* Word 8 - abort_tag is variable */
167
168	/* Word 9 - reqtag is variable */
169
170	/* Word 10 - dbde, wqes is variable */
171	bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0);
172	bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE);
173	bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4);
174	bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);
175	bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
176
177	/* Word 11 - pbde is variable */
178	bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, COMMAND_DATA_OUT);
179	bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
180	bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 0);
181
182	/* Word 12 - is zero */
183
184	/* Word 13, 14, 15 - PBDE is variable */
185
186	/* ICMND template */
187	wqe = &lpfc_icmnd_cmd_template;
188	memset(wqe, 0, sizeof(union lpfc_wqe128));
189
190	/* Word 0, 1, 2 - BDE is variable */
191
192	/* Word 3 - payload_offset_len is variable */
193
194	/* Word 4, 5 - is zero */
195
196	/* Word 6 - ctxt_tag, xri_tag is variable */
197
198	/* Word 7 */
199	bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE);
200	bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0);
201	bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3);
202	bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI);
203
204	/* Word 8 - abort_tag is variable */
205
206	/* Word 9 - reqtag is variable */
207
208	/* Word 10 - dbde, wqes is variable */
209	bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1);
210	bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE);
211	bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE);
212	bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0);
213	bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
214
215	/* Word 11 */
216	bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, COMMAND_DATA_IN);
217	bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
218	bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0);
219
220	/* Word 12, 13, 14, 15 - is zero */
221	}
222
223	#if defined(CONFIG_64BIT) && defined(__LITTLE_ENDIAN)
224	/**
225	* lpfc_sli4_pcimem_bcopy - SLI4 memory copy function
226	* @srcp: Source memory pointer.
227	* @destp: Destination memory pointer.
228	* @cnt: Number of words required to be copied.
229	* Must be a multiple of sizeof(uint64_t)
230	*
231	* This function is used for copying data between driver memory
232	* and the SLI WQ. This function also changes the endianness
233	* of each word if native endianness is different from SLI
234	* endianness. This function can be called with or without
235	* lock.
236	**/
237	static void
238	lpfc_sli4_pcimem_bcopy(void *srcp, void *destp, uint32_t cnt)
239	{
240	uint64_t *src = srcp;
241	uint64_t *dest = destp;
242	int i;
243
244	for (i = 0; i < (int)cnt; i += sizeof(uint64_t))
245	*dest++ = *src++;
246	}
247	#else
248	#define lpfc_sli4_pcimem_bcopy(a, b, c) lpfc_sli_pcimem_bcopy(a, b, c)
249	#endif
250
251	/**
252	* lpfc_sli4_wq_put - Put a Work Queue Entry on an Work Queue
253	* @q: The Work Queue to operate on.
254	* @wqe: The work Queue Entry to put on the Work queue.
255	*
256	* This routine will copy the contents of @wqe to the next available entry on
257	* the @q. This function will then ring the Work Queue Doorbell to signal the
258	* HBA to start processing the Work Queue Entry. This function returns 0 if
259	* successful. If no entries are available on @q then this function will return
260	* -ENOMEM.
261	* The caller is expected to hold the hbalock when calling this routine.
262	**/
263	static int
264	lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe)
265	{
266	union lpfc_wqe *temp_wqe;
267	struct lpfc_register doorbell;
268	uint32_t host_index;
269	uint32_t idx;
270	uint32_t i = 0;
271	uint8_t *tmp;
272	u32 if_type;
273
274	/* sanity check on queue memory */
275	if (unlikely(!q))
276	return -ENOMEM;
277
278	temp_wqe = lpfc_sli4_qe(q, idx: q->host_index);
279
280	/* If the host has not yet processed the next entry then we are done */
281	idx = ((q->host_index + 1) % q->entry_count);
282	if (idx == q->hba_index) {
283	q->WQ_overflow++;
284	return -EBUSY;
285	}
286	q->WQ_posted++;
287	/* set consumption flag every once in a while */
288	if (!((q->host_index + 1) % q->notify_interval))
289	bf_set(wqe_wqec, &wqe->generic.wqe_com, 1);
290	else
291	bf_set(wqe_wqec, &wqe->generic.wqe_com, 0);
292	if (q->phba->sli3_options & LPFC_SLI4_PHWQ_ENABLED)
293	bf_set(wqe_wqid, &wqe->generic.wqe_com, q->queue_id);
294	lpfc_sli4_pcimem_bcopy(srcp: wqe, destp: temp_wqe, cnt: q->entry_size);
295	if (q->dpp_enable && q->phba->cfg_enable_dpp) {
296	/* write to DPP aperture taking advatage of Combined Writes */
297	tmp = (uint8_t *)temp_wqe;
298	#ifdef __raw_writeq
299	for (i = 0; i < q->entry_size; i += sizeof(uint64_t))
300	__raw_writeq(val: *((uint64_t *)(tmp + i)),
301	addr: q->dpp_regaddr + i);
302	#else
303	for (i = 0; i < q->entry_size; i += sizeof(uint32_t))
304	__raw_writel(*((uint32_t *)(tmp + i)),
305	q->dpp_regaddr + i);
306	#endif
307	}
308	/* ensure WQE bcopy and DPP flushed before doorbell write */
309	wmb();
310
311	/* Update the host index before invoking device */
312	host_index = q->host_index;
313
314	q->host_index = idx;
315
316	/* Ring Doorbell */
317	doorbell.word0 = 0;
318	if (q->db_format == LPFC_DB_LIST_FORMAT) {
319	if (q->dpp_enable && q->phba->cfg_enable_dpp) {
320	bf_set(lpfc_if6_wq_db_list_fm_num_posted, &doorbell, 1);
321	bf_set(lpfc_if6_wq_db_list_fm_dpp, &doorbell, 1);
322	bf_set(lpfc_if6_wq_db_list_fm_dpp_id, &doorbell,
323	q->dpp_id);
324	bf_set(lpfc_if6_wq_db_list_fm_id, &doorbell,
325	q->queue_id);
326	} else {
327	bf_set(lpfc_wq_db_list_fm_num_posted, &doorbell, 1);
328	bf_set(lpfc_wq_db_list_fm_id, &doorbell, q->queue_id);
329
330	/* Leave bits <23:16> clear for if_type 6 dpp */
331	if_type = bf_get(lpfc_sli_intf_if_type,
332	&q->phba->sli4_hba.sli_intf);
333	if (if_type != LPFC_SLI_INTF_IF_TYPE_6)
334	bf_set(lpfc_wq_db_list_fm_index, &doorbell,
335	host_index);
336	}
337	} else if (q->db_format == LPFC_DB_RING_FORMAT) {
338	bf_set(lpfc_wq_db_ring_fm_num_posted, &doorbell, 1);
339	bf_set(lpfc_wq_db_ring_fm_id, &doorbell, q->queue_id);
340	} else {
341	return -EINVAL;
342	}
343	writel(val: doorbell.word0, addr: q->db_regaddr);
344
345	return 0;
346	}
347
348	/**
349	* lpfc_sli4_wq_release - Updates internal hba index for WQ
350	* @q: The Work Queue to operate on.
351	* @index: The index to advance the hba index to.
352	*
353	* This routine will update the HBA index of a queue to reflect consumption of
354	* Work Queue Entries by the HBA. When the HBA indicates that it has consumed
355	* an entry the host calls this function to update the queue's internal
356	* pointers.
357	**/
358	static void
359	lpfc_sli4_wq_release(struct lpfc_queue *q, uint32_t index)
360	{
361	/* sanity check on queue memory */
362	if (unlikely(!q))
363	return;
364
365	q->hba_index = index;
366	}
367
368	/**
369	* lpfc_sli4_mq_put - Put a Mailbox Queue Entry on an Mailbox Queue
370	* @q: The Mailbox Queue to operate on.
371	* @mqe: The Mailbox Queue Entry to put on the Work queue.
372	*
373	* This routine will copy the contents of @mqe to the next available entry on
374	* the @q. This function will then ring the Work Queue Doorbell to signal the
375	* HBA to start processing the Work Queue Entry. This function returns 0 if
376	* successful. If no entries are available on @q then this function will return
377	* -ENOMEM.
378	* The caller is expected to hold the hbalock when calling this routine.
379	**/
380	static uint32_t
381	lpfc_sli4_mq_put(struct lpfc_queue *q, struct lpfc_mqe *mqe)
382	{
383	struct lpfc_mqe *temp_mqe;
384	struct lpfc_register doorbell;
385
386	/* sanity check on queue memory */
387	if (unlikely(!q))
388	return -ENOMEM;
389	temp_mqe = lpfc_sli4_qe(q, idx: q->host_index);
390
391	/* If the host has not yet processed the next entry then we are done */
392	if (((q->host_index + 1) % q->entry_count) == q->hba_index)
393	return -ENOMEM;
394	lpfc_sli4_pcimem_bcopy(srcp: mqe, destp: temp_mqe, cnt: q->entry_size);
395	/* Save off the mailbox pointer for completion */
396	q->phba->mbox = (MAILBOX_t *)temp_mqe;
397
398	/* Update the host index before invoking device */
399	q->host_index = ((q->host_index + 1) % q->entry_count);
400
401	/* Ring Doorbell */
402	doorbell.word0 = 0;
403	bf_set(lpfc_mq_doorbell_num_posted, &doorbell, 1);
404	bf_set(lpfc_mq_doorbell_id, &doorbell, q->queue_id);
405	writel(val: doorbell.word0, addr: q->phba->sli4_hba.MQDBregaddr);
406	return 0;
407	}
408
409	/**
410	* lpfc_sli4_mq_release - Updates internal hba index for MQ
411	* @q: The Mailbox Queue to operate on.
412	*
413	* This routine will update the HBA index of a queue to reflect consumption of
414	* a Mailbox Queue Entry by the HBA. When the HBA indicates that it has consumed
415	* an entry the host calls this function to update the queue's internal
416	* pointers. This routine returns the number of entries that were consumed by
417	* the HBA.
418	**/
419	static uint32_t
420	lpfc_sli4_mq_release(struct lpfc_queue *q)
421	{
422	/* sanity check on queue memory */
423	if (unlikely(!q))
424	return 0;
425
426	/* Clear the mailbox pointer for completion */
427	q->phba->mbox = NULL;
428	q->hba_index = ((q->hba_index + 1) % q->entry_count);
429	return 1;
430	}
431
432	/**
433	* lpfc_sli4_eq_get - Gets the next valid EQE from a EQ
434	* @q: The Event Queue to get the first valid EQE from
435	*
436	* This routine will get the first valid Event Queue Entry from @q, update
437	* the queue's internal hba index, and return the EQE. If no valid EQEs are in
438	* the Queue (no more work to do), or the Queue is full of EQEs that have been
439	* processed, but not popped back to the HBA then this routine will return NULL.
440	**/
441	static struct lpfc_eqe *
442	lpfc_sli4_eq_get(struct lpfc_queue *q)
443	{
444	struct lpfc_eqe *eqe;
445
446	/* sanity check on queue memory */
447	if (unlikely(!q))
448	return NULL;
449	eqe = lpfc_sli4_qe(q, idx: q->host_index);
450
451	/* If the next EQE is not valid then we are done */
452	if (bf_get_le32(lpfc_eqe_valid, eqe) != q->qe_valid)
453	return NULL;
454
455	/*
456	* insert barrier for instruction interlock : data from the hardware
457	* must have the valid bit checked before it can be copied and acted
458	* upon. Speculative instructions were allowing a bcopy at the start
459	* of lpfc_sli4_fp_handle_wcqe(), which is called immediately
460	* after our return, to copy data before the valid bit check above
461	* was done. As such, some of the copied data was stale. The barrier
462	* ensures the check is before any data is copied.
463	*/
464	mb();
465	return eqe;
466	}
467
468	/**
469	* lpfc_sli4_eq_clr_intr - Turn off interrupts from this EQ
470	* @q: The Event Queue to disable interrupts
471	*
472	**/
473	void
474	lpfc_sli4_eq_clr_intr(struct lpfc_queue *q)
475	{
476	struct lpfc_register doorbell;
477
478	doorbell.word0 = 0;
479	bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1);
480	bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT);
481	bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell,
482	(q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT));
483	bf_set(lpfc_eqcq_doorbell_eqid_lo, &doorbell, q->queue_id);
484	writel(val: doorbell.word0, addr: q->phba->sli4_hba.EQDBregaddr);
485	}
486
487	/**
488	* lpfc_sli4_if6_eq_clr_intr - Turn off interrupts from this EQ
489	* @q: The Event Queue to disable interrupts
490	*
491	**/
492	void
493	lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q)
494	{
495	struct lpfc_register doorbell;
496
497	doorbell.word0 = 0;
498	bf_set(lpfc_if6_eq_doorbell_eqid, &doorbell, q->queue_id);
499	writel(val: doorbell.word0, addr: q->phba->sli4_hba.EQDBregaddr);
500	}
501
502	/**
503	* lpfc_sli4_write_eq_db - write EQ DB for eqe's consumed or arm state
504	* @phba: adapter with EQ
505	* @q: The Event Queue that the host has completed processing for.
506	* @count: Number of elements that have been consumed
507	* @arm: Indicates whether the host wants to arms this CQ.
508	*
509	* This routine will notify the HBA, by ringing the doorbell, that count
510	* number of EQEs have been processed. The @arm parameter indicates whether
511	* the queue should be rearmed when ringing the doorbell.
512	**/
513	void
514	lpfc_sli4_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
515	uint32_t count, bool arm)
516	{
517	struct lpfc_register doorbell;
518
519	/* sanity check on queue memory */
520	if (unlikely(!q || (count == 0 && !arm)))
521	return;
522
523	/* ring doorbell for number popped */
524	doorbell.word0 = 0;
525	if (arm) {
526	bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1);
527	bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1);
528	}
529	bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count);
530	bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT);
531	bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell,
532	(q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT));
533	bf_set(lpfc_eqcq_doorbell_eqid_lo, &doorbell, q->queue_id);
534	writel(val: doorbell.word0, addr: q->phba->sli4_hba.EQDBregaddr);
535	/* PCI read to flush PCI pipeline on re-arming for INTx mode */
536	if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM))
537	readl(addr: q->phba->sli4_hba.EQDBregaddr);
538	}
539
540	/**
541	* lpfc_sli4_if6_write_eq_db - write EQ DB for eqe's consumed or arm state
542	* @phba: adapter with EQ
543	* @q: The Event Queue that the host has completed processing for.
544	* @count: Number of elements that have been consumed
545	* @arm: Indicates whether the host wants to arms this CQ.
546	*
547	* This routine will notify the HBA, by ringing the doorbell, that count
548	* number of EQEs have been processed. The @arm parameter indicates whether
549	* the queue should be rearmed when ringing the doorbell.
550	**/
551	void
552	lpfc_sli4_if6_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
553	uint32_t count, bool arm)
554	{
555	struct lpfc_register doorbell;
556
557	/* sanity check on queue memory */
558	if (unlikely(!q || (count == 0 && !arm)))
559	return;
560
561	/* ring doorbell for number popped */
562	doorbell.word0 = 0;
563	if (arm)
564	bf_set(lpfc_if6_eq_doorbell_arm, &doorbell, 1);
565	bf_set(lpfc_if6_eq_doorbell_num_released, &doorbell, count);
566	bf_set(lpfc_if6_eq_doorbell_eqid, &doorbell, q->queue_id);
567	writel(val: doorbell.word0, addr: q->phba->sli4_hba.EQDBregaddr);
568	/* PCI read to flush PCI pipeline on re-arming for INTx mode */
569	if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM))
570	readl(addr: q->phba->sli4_hba.EQDBregaddr);
571	}
572
573	static void
574	__lpfc_sli4_consume_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq,
575	struct lpfc_eqe *eqe)
576	{
577	if (!phba->sli4_hba.pc_sli4_params.eqav)
578	bf_set_le32(lpfc_eqe_valid, eqe, 0);
579
580	eq->host_index = ((eq->host_index + 1) % eq->entry_count);
581
582	/* if the index wrapped around, toggle the valid bit */
583	if (phba->sli4_hba.pc_sli4_params.eqav && !eq->host_index)
584	eq->qe_valid = (eq->qe_valid) ? 0 : 1;
585	}
586
587	static void
588	lpfc_sli4_eqcq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq)
589	{
590	struct lpfc_eqe *eqe = NULL;
591	u32 eq_count = 0, cq_count = 0;
592	struct lpfc_cqe *cqe = NULL;
593	struct lpfc_queue *cq = NULL, *childq = NULL;
594	int cqid = 0;
595
596	/* walk all the EQ entries and drop on the floor */
597	eqe = lpfc_sli4_eq_get(q: eq);
598	while (eqe) {
599	/* Get the reference to the corresponding CQ */
600	cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);
601	cq = NULL;
602
603	list_for_each_entry(childq, &eq->child_list, list) {
604	if (childq->queue_id == cqid) {
605	cq = childq;
606	break;
607	}
608	}
609	/* If CQ is valid, iterate through it and drop all the CQEs */
610	if (cq) {
611	cqe = lpfc_sli4_cq_get(q: cq);
612	while (cqe) {
613	__lpfc_sli4_consume_cqe(phba, cq, cqe);
614	cq_count++;
615	cqe = lpfc_sli4_cq_get(q: cq);
616	}
617	/* Clear and re-arm the CQ */
618	phba->sli4_hba.sli4_write_cq_db(phba, cq, cq_count,
619	LPFC_QUEUE_REARM);
620	cq_count = 0;
621	}
622	__lpfc_sli4_consume_eqe(phba, eq, eqe);
623	eq_count++;
624	eqe = lpfc_sli4_eq_get(q: eq);
625	}
626
627	/* Clear and re-arm the EQ */
628	phba->sli4_hba.sli4_write_eq_db(phba, eq, eq_count, LPFC_QUEUE_REARM);
629	}
630
631	static int
632	lpfc_sli4_process_eq(struct lpfc_hba *phba, struct lpfc_queue *eq,
633	u8 rearm, enum lpfc_poll_mode poll_mode)
634	{
635	struct lpfc_eqe *eqe;
636	int count = 0, consumed = 0;
637
638	if (cmpxchg(&eq->queue_claimed, 0, 1) != 0)
639	goto rearm_and_exit;
640
641	eqe = lpfc_sli4_eq_get(q: eq);
642	while (eqe) {
643	lpfc_sli4_hba_handle_eqe(phba, eq, eqe, poll_mode);
644	__lpfc_sli4_consume_eqe(phba, eq, eqe);
645
646	consumed++;
647	if (!(++count % eq->max_proc_limit))
648	break;
649
650	if (!(count % eq->notify_interval)) {
651	phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed,
652	LPFC_QUEUE_NOARM);
653	consumed = 0;
654	}
655
656	eqe = lpfc_sli4_eq_get(q: eq);
657	}
658	eq->EQ_processed += count;
659
660	/* Track the max number of EQEs processed in 1 intr */
661	if (count > eq->EQ_max_eqe)
662	eq->EQ_max_eqe = count;
663
664	xchg(&eq->queue_claimed, 0);
665
666	rearm_and_exit:
667	/* Always clear the EQ. */
668	phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed, rearm);
669
670	return count;
671	}
672
673	/**
674	* lpfc_sli4_cq_get - Gets the next valid CQE from a CQ
675	* @q: The Completion Queue to get the first valid CQE from
676	*
677	* This routine will get the first valid Completion Queue Entry from @q, update
678	* the queue's internal hba index, and return the CQE. If no valid CQEs are in
679	* the Queue (no more work to do), or the Queue is full of CQEs that have been
680	* processed, but not popped back to the HBA then this routine will return NULL.
681	**/
682	static struct lpfc_cqe *
683	lpfc_sli4_cq_get(struct lpfc_queue *q)
684	{
685	struct lpfc_cqe *cqe;
686
687	/* sanity check on queue memory */
688	if (unlikely(!q))
689	return NULL;
690	cqe = lpfc_sli4_qe(q, idx: q->host_index);
691
692	/* If the next CQE is not valid then we are done */
693	if (bf_get_le32(lpfc_cqe_valid, cqe) != q->qe_valid)
694	return NULL;
695
696	/*
697	* insert barrier for instruction interlock : data from the hardware
698	* must have the valid bit checked before it can be copied and acted
699	* upon. Given what was seen in lpfc_sli4_cq_get() of speculative
700	* instructions allowing action on content before valid bit checked,
701	* add barrier here as well. May not be needed as "content" is a
702	* single 32-bit entity here (vs multi word structure for cq's).
703	*/
704	mb();
705	return cqe;
706	}
707
708	static void
709	__lpfc_sli4_consume_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
710	struct lpfc_cqe *cqe)
711	{
712	if (!phba->sli4_hba.pc_sli4_params.cqav)
713	bf_set_le32(lpfc_cqe_valid, cqe, 0);
714
715	cq->host_index = ((cq->host_index + 1) % cq->entry_count);
716
717	/* if the index wrapped around, toggle the valid bit */
718	if (phba->sli4_hba.pc_sli4_params.cqav && !cq->host_index)
719	cq->qe_valid = (cq->qe_valid) ? 0 : 1;
720	}
721
722	/**
723	* lpfc_sli4_write_cq_db - write cq DB for entries consumed or arm state.
724	* @phba: the adapter with the CQ
725	* @q: The Completion Queue that the host has completed processing for.
726	* @count: the number of elements that were consumed
727	* @arm: Indicates whether the host wants to arms this CQ.
728	*
729	* This routine will notify the HBA, by ringing the doorbell, that the
730	* CQEs have been processed. The @arm parameter specifies whether the
731	* queue should be rearmed when ringing the doorbell.
732	**/
733	void
734	lpfc_sli4_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
735	uint32_t count, bool arm)
736	{
737	struct lpfc_register doorbell;
738
739	/* sanity check on queue memory */
740	if (unlikely(!q || (count == 0 && !arm)))
741	return;
742
743	/* ring doorbell for number popped */
744	doorbell.word0 = 0;
745	if (arm)
746	bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1);
747	bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count);
748	bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_COMPLETION);
749	bf_set(lpfc_eqcq_doorbell_cqid_hi, &doorbell,
750	(q->queue_id >> LPFC_CQID_HI_FIELD_SHIFT));
751	bf_set(lpfc_eqcq_doorbell_cqid_lo, &doorbell, q->queue_id);
752	writel(val: doorbell.word0, addr: q->phba->sli4_hba.CQDBregaddr);
753	}
754
755	/**
756	* lpfc_sli4_if6_write_cq_db - write cq DB for entries consumed or arm state.
757	* @phba: the adapter with the CQ
758	* @q: The Completion Queue that the host has completed processing for.
759	* @count: the number of elements that were consumed
760	* @arm: Indicates whether the host wants to arms this CQ.
761	*
762	* This routine will notify the HBA, by ringing the doorbell, that the
763	* CQEs have been processed. The @arm parameter specifies whether the
764	* queue should be rearmed when ringing the doorbell.
765	**/
766	void
767	lpfc_sli4_if6_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
768	uint32_t count, bool arm)
769	{
770	struct lpfc_register doorbell;
771
772	/* sanity check on queue memory */
773	if (unlikely(!q || (count == 0 && !arm)))
774	return;
775
776	/* ring doorbell for number popped */
777	doorbell.word0 = 0;
778	if (arm)
779	bf_set(lpfc_if6_cq_doorbell_arm, &doorbell, 1);
780	bf_set(lpfc_if6_cq_doorbell_num_released, &doorbell, count);
781	bf_set(lpfc_if6_cq_doorbell_cqid, &doorbell, q->queue_id);
782	writel(val: doorbell.word0, addr: q->phba->sli4_hba.CQDBregaddr);
783	}
784
785	/*
786	* lpfc_sli4_rq_put - Put a Receive Buffer Queue Entry on a Receive Queue
787	*
788	* This routine will copy the contents of @wqe to the next available entry on
789	* the @q. This function will then ring the Receive Queue Doorbell to signal the
790	* HBA to start processing the Receive Queue Entry. This function returns the
791	* index that the rqe was copied to if successful. If no entries are available
792	* on @q then this function will return -ENOMEM.
793	* The caller is expected to hold the hbalock when calling this routine.
794	**/
795	int
796	lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq,
797	struct lpfc_rqe *hrqe, struct lpfc_rqe *drqe)
798	{
799	struct lpfc_rqe *temp_hrqe;
800	struct lpfc_rqe *temp_drqe;
801	struct lpfc_register doorbell;
802	int hq_put_index;
803	int dq_put_index;
804
805	/* sanity check on queue memory */
806	if (unlikely(!hq) || unlikely(!dq))
807	return -ENOMEM;
808	hq_put_index = hq->host_index;
809	dq_put_index = dq->host_index;
810	temp_hrqe = lpfc_sli4_qe(q: hq, idx: hq_put_index);
811	temp_drqe = lpfc_sli4_qe(q: dq, idx: dq_put_index);
812
813	if (hq->type != LPFC_HRQ || dq->type != LPFC_DRQ)
814	return -EINVAL;
815	if (hq_put_index != dq_put_index)
816	return -EINVAL;
817	/* If the host has not yet processed the next entry then we are done */
818	if (((hq_put_index + 1) % hq->entry_count) == hq->hba_index)
819	return -EBUSY;
820	lpfc_sli4_pcimem_bcopy(srcp: hrqe, destp: temp_hrqe, cnt: hq->entry_size);
821	lpfc_sli4_pcimem_bcopy(srcp: drqe, destp: temp_drqe, cnt: dq->entry_size);
822
823	/* Update the host index to point to the next slot */
824	hq->host_index = ((hq_put_index + 1) % hq->entry_count);
825	dq->host_index = ((dq_put_index + 1) % dq->entry_count);
826	hq->RQ_buf_posted++;
827
828	/* Ring The Header Receive Queue Doorbell */
829	if (!(hq->host_index % hq->notify_interval)) {
830	doorbell.word0 = 0;
831	if (hq->db_format == LPFC_DB_RING_FORMAT) {
832	bf_set(lpfc_rq_db_ring_fm_num_posted, &doorbell,
833	hq->notify_interval);
834	bf_set(lpfc_rq_db_ring_fm_id, &doorbell, hq->queue_id);
835	} else if (hq->db_format == LPFC_DB_LIST_FORMAT) {
836	bf_set(lpfc_rq_db_list_fm_num_posted, &doorbell,
837	hq->notify_interval);
838	bf_set(lpfc_rq_db_list_fm_index, &doorbell,
839	hq->host_index);
840	bf_set(lpfc_rq_db_list_fm_id, &doorbell, hq->queue_id);
841	} else {
842	return -EINVAL;
843	}
844	writel(val: doorbell.word0, addr: hq->db_regaddr);
845	}
846	return hq_put_index;
847	}
848
849	/*
850	* lpfc_sli4_rq_release - Updates internal hba index for RQ
851	*
852	* This routine will update the HBA index of a queue to reflect consumption of
853	* one Receive Queue Entry by the HBA. When the HBA indicates that it has
854	* consumed an entry the host calls this function to update the queue's
855	* internal pointers. This routine returns the number of entries that were
856	* consumed by the HBA.
857	**/
858	static uint32_t
859	lpfc_sli4_rq_release(struct lpfc_queue *hq, struct lpfc_queue *dq)
860	{
861	/* sanity check on queue memory */
862	if (unlikely(!hq) || unlikely(!dq))
863	return 0;
864
865	if ((hq->type != LPFC_HRQ) || (dq->type != LPFC_DRQ))
866	return 0;
867	hq->hba_index = ((hq->hba_index + 1) % hq->entry_count);
868	dq->hba_index = ((dq->hba_index + 1) % dq->entry_count);
869	return 1;
870	}
871
872	/**
873	* lpfc_cmd_iocb - Get next command iocb entry in the ring
874	* @phba: Pointer to HBA context object.
875	* @pring: Pointer to driver SLI ring object.
876	*
877	* This function returns pointer to next command iocb entry
878	* in the command ring. The caller must hold hbalock to prevent
879	* other threads consume the next command iocb.
880	* SLI-2/SLI-3 provide different sized iocbs.
881	**/
882	static inline IOCB_t *
883	lpfc_cmd_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
884	{
885	return (IOCB_t *) (((char *) pring->sli.sli3.cmdringaddr) +
886	pring->sli.sli3.cmdidx * phba->iocb_cmd_size);
887	}
888
889	/**
890	* lpfc_resp_iocb - Get next response iocb entry in the ring
891	* @phba: Pointer to HBA context object.
892	* @pring: Pointer to driver SLI ring object.
893	*
894	* This function returns pointer to next response iocb entry
895	* in the response ring. The caller must hold hbalock to make sure
896	* that no other thread consume the next response iocb.
897	* SLI-2/SLI-3 provide different sized iocbs.
898	**/
899	static inline IOCB_t *
900	lpfc_resp_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
901	{
902	return (IOCB_t *) (((char *) pring->sli.sli3.rspringaddr) +
903	pring->sli.sli3.rspidx * phba->iocb_rsp_size);
904	}
905
906	/**
907	* __lpfc_sli_get_iocbq - Allocates an iocb object from iocb pool
908	* @phba: Pointer to HBA context object.
909	*
910	* This function is called with hbalock held. This function
911	* allocates a new driver iocb object from the iocb pool. If the
912	* allocation is successful, it returns pointer to the newly
913	* allocated iocb object else it returns NULL.
914	**/
915	struct lpfc_iocbq *
916	__lpfc_sli_get_iocbq(struct lpfc_hba *phba)
917	{
918	struct list_head *lpfc_iocb_list = &phba->lpfc_iocb_list;
919	struct lpfc_iocbq * iocbq = NULL;
920
921	lockdep_assert_held(&phba->hbalock);
922
923	list_remove_head(lpfc_iocb_list, iocbq, struct lpfc_iocbq, list);
924	if (iocbq)
925	phba->iocb_cnt++;
926	if (phba->iocb_cnt > phba->iocb_max)
927	phba->iocb_max = phba->iocb_cnt;
928	return iocbq;
929	}
930
931	/**
932	* __lpfc_clear_active_sglq - Remove the active sglq for this XRI.
933	* @phba: Pointer to HBA context object.
934	* @xritag: XRI value.
935	*
936	* This function clears the sglq pointer from the array of active
937	* sglq's. The xritag that is passed in is used to index into the
938	* array. Before the xritag can be used it needs to be adjusted
939	* by subtracting the xribase.
940	*
941	* Returns sglq ponter = success, NULL = Failure.
942	**/
943	struct lpfc_sglq *
944	__lpfc_clear_active_sglq(struct lpfc_hba *phba, uint16_t xritag)
945	{
946	struct lpfc_sglq *sglq;
947
948	sglq = phba->sli4_hba.lpfc_sglq_active_list[xritag];
949	phba->sli4_hba.lpfc_sglq_active_list[xritag] = NULL;
950	return sglq;
951	}
952
953	/**
954	* __lpfc_get_active_sglq - Get the active sglq for this XRI.
955	* @phba: Pointer to HBA context object.
956	* @xritag: XRI value.
957	*
958	* This function returns the sglq pointer from the array of active
959	* sglq's. The xritag that is passed in is used to index into the
960	* array. Before the xritag can be used it needs to be adjusted
961	* by subtracting the xribase.
962	*
963	* Returns sglq ponter = success, NULL = Failure.
964	**/
965	struct lpfc_sglq *
966	__lpfc_get_active_sglq(struct lpfc_hba *phba, uint16_t xritag)
967	{
968	struct lpfc_sglq *sglq;
969
970	sglq = phba->sli4_hba.lpfc_sglq_active_list[xritag];
971	return sglq;
972	}
973
974	/**
975	* lpfc_clr_rrq_active - Clears RRQ active bit in xri_bitmap.
976	* @phba: Pointer to HBA context object.
977	* @xritag: xri used in this exchange.
978	* @rrq: The RRQ to be cleared.
979	*
980	**/
981	void
982	lpfc_clr_rrq_active(struct lpfc_hba *phba,
983	uint16_t xritag,
984	struct lpfc_node_rrq *rrq)
985	{
986	struct lpfc_nodelist *ndlp = NULL;
987
988	/* Lookup did to verify if did is still active on this vport */
989	if (rrq->vport)
990	ndlp = lpfc_findnode_did(rrq->vport, rrq->nlp_DID);
991
992	if (!ndlp)
993	goto out;
994
995	if (test_and_clear_bit(nr: xritag, addr: ndlp->active_rrqs_xri_bitmap)) {
996	rrq->send_rrq = 0;
997	rrq->xritag = 0;
998	rrq->rrq_stop_time = 0;
999	}
1000	out:
1001	mempool_free(element: rrq, pool: phba->rrq_pool);
1002	}
1003
1004	/**
1005	* lpfc_handle_rrq_active - Checks if RRQ has waithed RATOV.
1006	* @phba: Pointer to HBA context object.
1007	*
1008	* This function is called with hbalock held. This function
1009	* Checks if stop_time (ratov from setting rrq active) has
1010	* been reached, if it has and the send_rrq flag is set then
1011	* it will call lpfc_send_rrq. If the send_rrq flag is not set
1012	* then it will just call the routine to clear the rrq and
1013	* free the rrq resource.
1014	* The timer is set to the next rrq that is going to expire before
1015	* leaving the routine.
1016	*
1017	**/
1018	void
1019	lpfc_handle_rrq_active(struct lpfc_hba *phba)
1020	{
1021	struct lpfc_node_rrq *rrq;
1022	struct lpfc_node_rrq *nextrrq;
1023	unsigned long next_time;
1024	unsigned long iflags;
1025	LIST_HEAD(send_rrq);
1026
1027	spin_lock_irqsave(&phba->hbalock, iflags);
1028	phba->hba_flag &= ~HBA_RRQ_ACTIVE;
1029	next_time = jiffies + msecs_to_jiffies(m: 1000 * (phba->fc_ratov + 1));
1030	list_for_each_entry_safe(rrq, nextrrq,
1031	&phba->active_rrq_list, list) {
1032	if (time_after(jiffies, rrq->rrq_stop_time))
1033	list_move(list: &rrq->list, head: &send_rrq);
1034	else if (time_before(rrq->rrq_stop_time, next_time))
1035	next_time = rrq->rrq_stop_time;
1036	}
1037	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
1038	if ((!list_empty(head: &phba->active_rrq_list)) &&
1039	(!(phba->pport->load_flag & FC_UNLOADING)))
1040	mod_timer(timer: &phba->rrq_tmr, expires: next_time);
1041	list_for_each_entry_safe(rrq, nextrrq, &send_rrq, list) {
1042	list_del(entry: &rrq->list);
1043	if (!rrq->send_rrq) {
1044	/* this call will free the rrq */
1045	lpfc_clr_rrq_active(phba, xritag: rrq->xritag, rrq);
1046	} else if (lpfc_send_rrq(phba, rrq)) {
1047	/* if we send the rrq then the completion handler
1048	* will clear the bit in the xribitmap.
1049	*/
1050	lpfc_clr_rrq_active(phba, xritag: rrq->xritag,
1051	rrq);
1052	}
1053	}
1054	}
1055
1056	/**
1057	* lpfc_get_active_rrq - Get the active RRQ for this exchange.
1058	* @vport: Pointer to vport context object.
1059	* @xri: The xri used in the exchange.
1060	* @did: The targets DID for this exchange.
1061	*
1062	* returns NULL = rrq not found in the phba->active_rrq_list.
1063	* rrq = rrq for this xri and target.
1064	**/
1065	struct lpfc_node_rrq *
1066	lpfc_get_active_rrq(struct lpfc_vport *vport, uint16_t xri, uint32_t did)
1067	{
1068	struct lpfc_hba *phba = vport->phba;
1069	struct lpfc_node_rrq *rrq;
1070	struct lpfc_node_rrq *nextrrq;
1071	unsigned long iflags;
1072
1073	if (phba->sli_rev != LPFC_SLI_REV4)
1074	return NULL;
1075	spin_lock_irqsave(&phba->hbalock, iflags);
1076	list_for_each_entry_safe(rrq, nextrrq, &phba->active_rrq_list, list) {
1077	if (rrq->vport == vport && rrq->xritag == xri &&
1078	rrq->nlp_DID == did){
1079	list_del(entry: &rrq->list);
1080	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
1081	return rrq;
1082	}
1083	}
1084	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
1085	return NULL;
1086	}
1087
1088	/**
1089	* lpfc_cleanup_vports_rrqs - Remove and clear the active RRQ for this vport.
1090	* @vport: Pointer to vport context object.
1091	* @ndlp: Pointer to the lpfc_node_list structure.
1092	* If ndlp is NULL Remove all active RRQs for this vport from the
1093	* phba->active_rrq_list and clear the rrq.
1094	* If ndlp is not NULL then only remove rrqs for this vport & this ndlp.
1095	**/
1096	void
1097	lpfc_cleanup_vports_rrqs(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
1098
1099	{
1100	struct lpfc_hba *phba = vport->phba;
1101	struct lpfc_node_rrq *rrq;
1102	struct lpfc_node_rrq *nextrrq;
1103	unsigned long iflags;
1104	LIST_HEAD(rrq_list);
1105
1106	if (phba->sli_rev != LPFC_SLI_REV4)
1107	return;
1108	if (!ndlp) {
1109	lpfc_sli4_vport_delete_els_xri_aborted(vport);
1110	lpfc_sli4_vport_delete_fcp_xri_aborted(vport);
1111	}
1112	spin_lock_irqsave(&phba->hbalock, iflags);
1113	list_for_each_entry_safe(rrq, nextrrq, &phba->active_rrq_list, list) {
1114	if (rrq->vport != vport)
1115	continue;
1116
1117	if (!ndlp || ndlp == lpfc_findnode_did(vport, rrq->nlp_DID))
1118	list_move(list: &rrq->list, head: &rrq_list);
1119
1120	}
1121	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
1122
1123	list_for_each_entry_safe(rrq, nextrrq, &rrq_list, list) {
1124	list_del(entry: &rrq->list);
1125	lpfc_clr_rrq_active(phba, xritag: rrq->xritag, rrq);
1126	}
1127	}
1128
1129	/**
1130	* lpfc_test_rrq_active - Test RRQ bit in xri_bitmap.
1131	* @phba: Pointer to HBA context object.
1132	* @ndlp: Targets nodelist pointer for this exchange.
1133	* @xritag: the xri in the bitmap to test.
1134	*
1135	* This function returns:
1136	* 0 = rrq not active for this xri
1137	* 1 = rrq is valid for this xri.
1138	**/
1139	int
1140	lpfc_test_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
1141	uint16_t xritag)
1142	{
1143	if (!ndlp)
1144	return 0;
1145	if (!ndlp->active_rrqs_xri_bitmap)
1146	return 0;
1147	if (test_bit(xritag, ndlp->active_rrqs_xri_bitmap))
1148	return 1;
1149	else
1150	return 0;
1151	}
1152
1153	/**
1154	* lpfc_set_rrq_active - set RRQ active bit in xri_bitmap.
1155	* @phba: Pointer to HBA context object.
1156	* @ndlp: nodelist pointer for this target.
1157	* @xritag: xri used in this exchange.
1158	* @rxid: Remote Exchange ID.
1159	* @send_rrq: Flag used to determine if we should send rrq els cmd.
1160	*
1161	* This function takes the hbalock.
1162	* The active bit is always set in the active rrq xri_bitmap even
1163	* if there is no slot avaiable for the other rrq information.
1164	*
1165	* returns 0 rrq actived for this xri
1166	* < 0 No memory or invalid ndlp.
1167	**/
1168	int
1169	lpfc_set_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
1170	uint16_t xritag, uint16_t rxid, uint16_t send_rrq)
1171	{
1172	unsigned long iflags;
1173	struct lpfc_node_rrq *rrq;
1174	int empty;
1175
1176	if (!ndlp)
1177	return -EINVAL;
1178
1179	if (!phba->cfg_enable_rrq)
1180	return -EINVAL;
1181
1182	spin_lock_irqsave(&phba->hbalock, iflags);
1183	if (phba->pport->load_flag & FC_UNLOADING) {
1184	phba->hba_flag &= ~HBA_RRQ_ACTIVE;
1185	goto out;
1186	}
1187
1188	if (ndlp->vport && (ndlp->vport->load_flag & FC_UNLOADING))
1189	goto out;
1190
1191	if (!ndlp->active_rrqs_xri_bitmap)
1192	goto out;
1193
1194	if (test_and_set_bit(nr: xritag, addr: ndlp->active_rrqs_xri_bitmap))
1195	goto out;
1196
1197	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
1198	rrq = mempool_alloc(pool: phba->rrq_pool, GFP_ATOMIC);
1199	if (!rrq) {
1200	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
1201	"3155 Unable to allocate RRQ xri:0x%x rxid:0x%x"
1202	" DID:0x%x Send:%d\n",
1203	xritag, rxid, ndlp->nlp_DID, send_rrq);
1204	return -EINVAL;
1205	}
1206	if (phba->cfg_enable_rrq == 1)
1207	rrq->send_rrq = send_rrq;
1208	else
1209	rrq->send_rrq = 0;
1210	rrq->xritag = xritag;
1211	rrq->rrq_stop_time = jiffies +
1212	msecs_to_jiffies(m: 1000 * (phba->fc_ratov + 1));
1213	rrq->nlp_DID = ndlp->nlp_DID;
1214	rrq->vport = ndlp->vport;
1215	rrq->rxid = rxid;
1216	spin_lock_irqsave(&phba->hbalock, iflags);
1217	empty = list_empty(head: &phba->active_rrq_list);
1218	list_add_tail(new: &rrq->list, head: &phba->active_rrq_list);
1219	phba->hba_flag |= HBA_RRQ_ACTIVE;
1220	if (empty)
1221	lpfc_worker_wake_up(phba);
1222	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
1223	return 0;
1224	out:
1225	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
1226	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
1227	"2921 Can't set rrq active xri:0x%x rxid:0x%x"
1228	" DID:0x%x Send:%d\n",
1229	xritag, rxid, ndlp->nlp_DID, send_rrq);
1230	return -EINVAL;
1231	}
1232
1233	/**
1234	* __lpfc_sli_get_els_sglq - Allocates an iocb object from sgl pool
1235	* @phba: Pointer to HBA context object.
1236	* @piocbq: Pointer to the iocbq.
1237	*
1238	* The driver calls this function with either the nvme ls ring lock
1239	* or the fc els ring lock held depending on the iocb usage. This function
1240	* gets a new driver sglq object from the sglq list. If the list is not empty
1241	* then it is successful, it returns pointer to the newly allocated sglq
1242	* object else it returns NULL.
1243	**/
1244	static struct lpfc_sglq *
1245	__lpfc_sli_get_els_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq)
1246	{
1247	struct list_head *lpfc_els_sgl_list = &phba->sli4_hba.lpfc_els_sgl_list;
1248	struct lpfc_sglq *sglq = NULL;
1249	struct lpfc_sglq *start_sglq = NULL;
1250	struct lpfc_io_buf *lpfc_cmd;
1251	struct lpfc_nodelist *ndlp;
1252	int found = 0;
1253	u8 cmnd;
1254
1255	cmnd = get_job_cmnd(phba, iocbq: piocbq);
1256
1257	if (piocbq->cmd_flag & LPFC_IO_FCP) {
1258	lpfc_cmd = piocbq->io_buf;
1259	ndlp = lpfc_cmd->rdata->pnode;
1260	} else if ((cmnd == CMD_GEN_REQUEST64_CR) &&
1261	!(piocbq->cmd_flag & LPFC_IO_LIBDFC)) {
1262	ndlp = piocbq->ndlp;
1263	} else if (piocbq->cmd_flag & LPFC_IO_LIBDFC) {
1264	if (piocbq->cmd_flag & LPFC_IO_LOOPBACK)
1265	ndlp = NULL;
1266	else
1267	ndlp = piocbq->ndlp;
1268	} else {
1269	ndlp = piocbq->ndlp;
1270	}
1271
1272	spin_lock(lock: &phba->sli4_hba.sgl_list_lock);
1273	list_remove_head(lpfc_els_sgl_list, sglq, struct lpfc_sglq, list);
1274	start_sglq = sglq;
1275	while (!found) {
1276	if (!sglq)
1277	break;
1278	if (ndlp && ndlp->active_rrqs_xri_bitmap &&
1279	test_bit(sglq->sli4_lxritag,
1280	ndlp->active_rrqs_xri_bitmap)) {
1281	/* This xri has an rrq outstanding for this DID.
1282	* put it back in the list and get another xri.
1283	*/
1284	list_add_tail(new: &sglq->list, head: lpfc_els_sgl_list);
1285	sglq = NULL;
1286	list_remove_head(lpfc_els_sgl_list, sglq,
1287	struct lpfc_sglq, list);
1288	if (sglq == start_sglq) {
1289	list_add_tail(new: &sglq->list, head: lpfc_els_sgl_list);
1290	sglq = NULL;
1291	break;
1292	} else
1293	continue;
1294	}
1295	sglq->ndlp = ndlp;
1296	found = 1;
1297	phba->sli4_hba.lpfc_sglq_active_list[sglq->sli4_lxritag] = sglq;
1298	sglq->state = SGL_ALLOCATED;
1299	}
1300	spin_unlock(lock: &phba->sli4_hba.sgl_list_lock);
1301	return sglq;
1302	}
1303
1304	/**
1305	* __lpfc_sli_get_nvmet_sglq - Allocates an iocb object from sgl pool
1306	* @phba: Pointer to HBA context object.
1307	* @piocbq: Pointer to the iocbq.
1308	*
1309	* This function is called with the sgl_list lock held. This function
1310	* gets a new driver sglq object from the sglq list. If the
1311	* list is not empty then it is successful, it returns pointer to the newly
1312	* allocated sglq object else it returns NULL.
1313	**/
1314	struct lpfc_sglq *
1315	__lpfc_sli_get_nvmet_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq)
1316	{
1317	struct list_head *lpfc_nvmet_sgl_list;
1318	struct lpfc_sglq *sglq = NULL;
1319
1320	lpfc_nvmet_sgl_list = &phba->sli4_hba.lpfc_nvmet_sgl_list;
1321
1322	lockdep_assert_held(&phba->sli4_hba.sgl_list_lock);
1323
1324	list_remove_head(lpfc_nvmet_sgl_list, sglq, struct lpfc_sglq, list);
1325	if (!sglq)
1326	return NULL;
1327	phba->sli4_hba.lpfc_sglq_active_list[sglq->sli4_lxritag] = sglq;
1328	sglq->state = SGL_ALLOCATED;
1329	return sglq;
1330	}
1331
1332	/**
1333	* lpfc_sli_get_iocbq - Allocates an iocb object from iocb pool
1334	* @phba: Pointer to HBA context object.
1335	*
1336	* This function is called with no lock held. This function
1337	* allocates a new driver iocb object from the iocb pool. If the
1338	* allocation is successful, it returns pointer to the newly
1339	* allocated iocb object else it returns NULL.
1340	**/
1341	struct lpfc_iocbq *
1342	lpfc_sli_get_iocbq(struct lpfc_hba *phba)
1343	{
1344	struct lpfc_iocbq * iocbq = NULL;
1345	unsigned long iflags;
1346
1347	spin_lock_irqsave(&phba->hbalock, iflags);
1348	iocbq = __lpfc_sli_get_iocbq(phba);
1349	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
1350	return iocbq;
1351	}
1352
1353	/**
1354	* __lpfc_sli_release_iocbq_s4 - Release iocb to the iocb pool
1355	* @phba: Pointer to HBA context object.
1356	* @iocbq: Pointer to driver iocb object.
1357	*
1358	* This function is called to release the driver iocb object
1359	* to the iocb pool. The iotag in the iocb object
1360	* does not change for each use of the iocb object. This function
1361	* clears all other fields of the iocb object when it is freed.
1362	* The sqlq structure that holds the xritag and phys and virtual
1363	* mappings for the scatter gather list is retrieved from the
1364	* active array of sglq. The get of the sglq pointer also clears
1365	* the entry in the array. If the status of the IO indiactes that
1366	* this IO was aborted then the sglq entry it put on the
1367	* lpfc_abts_els_sgl_list until the CQ_ABORTED_XRI is received. If the
1368	* IO has good status or fails for any other reason then the sglq
1369	* entry is added to the free list (lpfc_els_sgl_list). The hbalock is
1370	* asserted held in the code path calling this routine.
1371	**/
1372	static void
1373	__lpfc_sli_release_iocbq_s4(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
1374	{
1375	struct lpfc_sglq *sglq;
1376	unsigned long iflag = 0;
1377	struct lpfc_sli_ring *pring;
1378
1379	if (iocbq->sli4_xritag == NO_XRI)
1380	sglq = NULL;
1381	else
1382	sglq = __lpfc_clear_active_sglq(phba, xritag: iocbq->sli4_lxritag);
1383
1384
1385	if (sglq) {
1386	if (iocbq->cmd_flag & LPFC_IO_NVMET) {
1387	spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock,
1388	iflag);
1389	sglq->state = SGL_FREED;
1390	sglq->ndlp = NULL;
1391	list_add_tail(new: &sglq->list,
1392	head: &phba->sli4_hba.lpfc_nvmet_sgl_list);
1393	spin_unlock_irqrestore(
1394	lock: &phba->sli4_hba.sgl_list_lock, flags: iflag);
1395	goto out;
1396	}
1397
1398	if ((iocbq->cmd_flag & LPFC_EXCHANGE_BUSY) &&
1399	(!(unlikely(pci_channel_offline(phba->pcidev)))) &&
1400	sglq->state != SGL_XRI_ABORTED) {
1401	spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock,
1402	iflag);
1403
1404	/* Check if we can get a reference on ndlp */
1405	if (sglq->ndlp && !lpfc_nlp_get(sglq->ndlp))
1406	sglq->ndlp = NULL;
1407
1408	list_add(new: &sglq->list,
1409	head: &phba->sli4_hba.lpfc_abts_els_sgl_list);
1410	spin_unlock_irqrestore(
1411	lock: &phba->sli4_hba.sgl_list_lock, flags: iflag);
1412	} else {
1413	spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock,
1414	iflag);
1415	sglq->state = SGL_FREED;
1416	sglq->ndlp = NULL;
1417	list_add_tail(new: &sglq->list,
1418	head: &phba->sli4_hba.lpfc_els_sgl_list);
1419	spin_unlock_irqrestore(
1420	lock: &phba->sli4_hba.sgl_list_lock, flags: iflag);
1421	pring = lpfc_phba_elsring(phba);
1422	/* Check if TXQ queue needs to be serviced */
1423	if (pring && (!list_empty(head: &pring->txq)))
1424	lpfc_worker_wake_up(phba);
1425	}
1426	}
1427
1428	out:
1429	/*
1430	* Clean all volatile data fields, preserve iotag and node struct.
1431	*/
1432	memset_startat(iocbq, 0, wqe);
1433	iocbq->sli4_lxritag = NO_XRI;
1434	iocbq->sli4_xritag = NO_XRI;
1435	iocbq->cmd_flag &= ~(LPFC_IO_NVME | LPFC_IO_NVMET | LPFC_IO_CMF |
1436	LPFC_IO_NVME_LS);
1437	list_add_tail(new: &iocbq->list, head: &phba->lpfc_iocb_list);
1438	}
1439
1440
1441	/**
1442	* __lpfc_sli_release_iocbq_s3 - Release iocb to the iocb pool
1443	* @phba: Pointer to HBA context object.
1444	* @iocbq: Pointer to driver iocb object.
1445	*
1446	* This function is called to release the driver iocb object to the
1447	* iocb pool. The iotag in the iocb object does not change for each
1448	* use of the iocb object. This function clears all other fields of
1449	* the iocb object when it is freed. The hbalock is asserted held in
1450	* the code path calling this routine.
1451	**/
1452	static void
1453	__lpfc_sli_release_iocbq_s3(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
1454	{
1455
1456	/*
1457	* Clean all volatile data fields, preserve iotag and node struct.
1458	*/
1459	memset_startat(iocbq, 0, iocb);
1460	iocbq->sli4_xritag = NO_XRI;
1461	list_add_tail(new: &iocbq->list, head: &phba->lpfc_iocb_list);
1462	}
1463
1464	/**
1465	* __lpfc_sli_release_iocbq - Release iocb to the iocb pool
1466	* @phba: Pointer to HBA context object.
1467	* @iocbq: Pointer to driver iocb object.
1468	*
1469	* This function is called with hbalock held to release driver
1470	* iocb object to the iocb pool. The iotag in the iocb object
1471	* does not change for each use of the iocb object. This function
1472	* clears all other fields of the iocb object when it is freed.
1473	**/
1474	static void
1475	__lpfc_sli_release_iocbq(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
1476	{
1477	lockdep_assert_held(&phba->hbalock);
1478
1479	phba->__lpfc_sli_release_iocbq(phba, iocbq);
1480	phba->iocb_cnt--;
1481	}
1482
1483	/**
1484	* lpfc_sli_release_iocbq - Release iocb to the iocb pool
1485	* @phba: Pointer to HBA context object.
1486	* @iocbq: Pointer to driver iocb object.
1487	*
1488	* This function is called with no lock held to release the iocb to
1489	* iocb pool.
1490	**/
1491	void
1492	lpfc_sli_release_iocbq(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
1493	{
1494	unsigned long iflags;
1495
1496	/*
1497	* Clean all volatile data fields, preserve iotag and node struct.
1498	*/
1499	spin_lock_irqsave(&phba->hbalock, iflags);
1500	__lpfc_sli_release_iocbq(phba, iocbq);
1501	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
1502	}
1503
1504	/**
1505	* lpfc_sli_cancel_iocbs - Cancel all iocbs from a list.
1506	* @phba: Pointer to HBA context object.
1507	* @iocblist: List of IOCBs.
1508	* @ulpstatus: ULP status in IOCB command field.
1509	* @ulpWord4: ULP word-4 in IOCB command field.
1510	*
1511	* This function is called with a list of IOCBs to cancel. It cancels the IOCB
1512	* on the list by invoking the complete callback function associated with the
1513	* IOCB with the provided @ulpstatus and @ulpword4 set to the IOCB commond
1514	* fields.
1515	**/
1516	void
1517	lpfc_sli_cancel_iocbs(struct lpfc_hba *phba, struct list_head *iocblist,
1518	uint32_t ulpstatus, uint32_t ulpWord4)
1519	{
1520	struct lpfc_iocbq *piocb;
1521
1522	while (!list_empty(head: iocblist)) {
1523	list_remove_head(iocblist, piocb, struct lpfc_iocbq, list);
1524	if (piocb->cmd_cmpl) {
1525	if (piocb->cmd_flag & LPFC_IO_NVME) {
1526	lpfc_nvme_cancel_iocb(phba, pwqeIn: piocb,
1527	stat: ulpstatus, param: ulpWord4);
1528	} else {
1529	if (phba->sli_rev == LPFC_SLI_REV4) {
1530	bf_set(lpfc_wcqe_c_status,
1531	&piocb->wcqe_cmpl, ulpstatus);
1532	piocb->wcqe_cmpl.parameter = ulpWord4;
1533	} else {
1534	piocb->iocb.ulpStatus = ulpstatus;
1535	piocb->iocb.un.ulpWord[4] = ulpWord4;
1536	}
1537	(piocb->cmd_cmpl) (phba, piocb, piocb);
1538	}
1539	} else {
1540	lpfc_sli_release_iocbq(phba, iocbq: piocb);
1541	}
1542	}
1543	return;
1544	}
1545
1546	/**
1547	* lpfc_sli_iocb_cmd_type - Get the iocb type
1548	* @iocb_cmnd: iocb command code.
1549	*
1550	* This function is called by ring event handler function to get the iocb type.
1551	* This function translates the iocb command to an iocb command type used to
1552	* decide the final disposition of each completed IOCB.
1553	* The function returns
1554	* LPFC_UNKNOWN_IOCB if it is an unsupported iocb
1555	* LPFC_SOL_IOCB if it is a solicited iocb completion
1556	* LPFC_ABORT_IOCB if it is an abort iocb
1557	* LPFC_UNSOL_IOCB if it is an unsolicited iocb
1558	*
1559	* The caller is not required to hold any lock.
1560	**/
1561	static lpfc_iocb_type
1562	lpfc_sli_iocb_cmd_type(uint8_t iocb_cmnd)
1563	{
1564	lpfc_iocb_type type = LPFC_UNKNOWN_IOCB;
1565
1566	if (iocb_cmnd > CMD_MAX_IOCB_CMD)
1567	return 0;
1568
1569	switch (iocb_cmnd) {
1570	case CMD_XMIT_SEQUENCE_CR:
1571	case CMD_XMIT_SEQUENCE_CX:
1572	case CMD_XMIT_BCAST_CN:
1573	case CMD_XMIT_BCAST_CX:
1574	case CMD_ELS_REQUEST_CR:
1575	case CMD_ELS_REQUEST_CX:
1576	case CMD_CREATE_XRI_CR:
1577	case CMD_CREATE_XRI_CX:
1578	case CMD_GET_RPI_CN:
1579	case CMD_XMIT_ELS_RSP_CX:
1580	case CMD_GET_RPI_CR:
1581	case CMD_FCP_IWRITE_CR:
1582	case CMD_FCP_IWRITE_CX:
1583	case CMD_FCP_IREAD_CR:
1584	case CMD_FCP_IREAD_CX:
1585	case CMD_FCP_ICMND_CR:
1586	case CMD_FCP_ICMND_CX:
1587	case CMD_FCP_TSEND_CX:
1588	case CMD_FCP_TRSP_CX:
1589	case CMD_FCP_TRECEIVE_CX:
1590	case CMD_FCP_AUTO_TRSP_CX:
1591	case CMD_ADAPTER_MSG:
1592	case CMD_ADAPTER_DUMP:
1593	case CMD_XMIT_SEQUENCE64_CR:
1594	case CMD_XMIT_SEQUENCE64_CX:
1595	case CMD_XMIT_BCAST64_CN:
1596	case CMD_XMIT_BCAST64_CX:
1597	case CMD_ELS_REQUEST64_CR:
1598	case CMD_ELS_REQUEST64_CX:
1599	case CMD_FCP_IWRITE64_CR:
1600	case CMD_FCP_IWRITE64_CX:
1601	case CMD_FCP_IREAD64_CR:
1602	case CMD_FCP_IREAD64_CX:
1603	case CMD_FCP_ICMND64_CR:
1604	case CMD_FCP_ICMND64_CX:
1605	case CMD_FCP_TSEND64_CX:
1606	case CMD_FCP_TRSP64_CX:
1607	case CMD_FCP_TRECEIVE64_CX:
1608	case CMD_GEN_REQUEST64_CR:
1609	case CMD_GEN_REQUEST64_CX:
1610	case CMD_XMIT_ELS_RSP64_CX:
1611	case DSSCMD_IWRITE64_CR:
1612	case DSSCMD_IWRITE64_CX:
1613	case DSSCMD_IREAD64_CR:
1614	case DSSCMD_IREAD64_CX:
1615	case CMD_SEND_FRAME:
1616	type = LPFC_SOL_IOCB;
1617	break;
1618	case CMD_ABORT_XRI_CN:
1619	case CMD_ABORT_XRI_CX:
1620	case CMD_CLOSE_XRI_CN:
1621	case CMD_CLOSE_XRI_CX:
1622	case CMD_XRI_ABORTED_CX:
1623	case CMD_ABORT_MXRI64_CN:
1624	case CMD_XMIT_BLS_RSP64_CX:
1625	type = LPFC_ABORT_IOCB;
1626	break;
1627	case CMD_RCV_SEQUENCE_CX:
1628	case CMD_RCV_ELS_REQ_CX:
1629	case CMD_RCV_SEQUENCE64_CX:
1630	case CMD_RCV_ELS_REQ64_CX:
1631	case CMD_ASYNC_STATUS:
1632	case CMD_IOCB_RCV_SEQ64_CX:
1633	case CMD_IOCB_RCV_ELS64_CX:
1634	case CMD_IOCB_RCV_CONT64_CX:
1635	case CMD_IOCB_RET_XRI64_CX:
1636	type = LPFC_UNSOL_IOCB;
1637	break;
1638	case CMD_IOCB_XMIT_MSEQ64_CR:
1639	case CMD_IOCB_XMIT_MSEQ64_CX:
1640	case CMD_IOCB_RCV_SEQ_LIST64_CX:
1641	case CMD_IOCB_RCV_ELS_LIST64_CX:
1642	case CMD_IOCB_CLOSE_EXTENDED_CN:
1643	case CMD_IOCB_ABORT_EXTENDED_CN:
1644	case CMD_IOCB_RET_HBQE64_CN:
1645	case CMD_IOCB_FCP_IBIDIR64_CR:
1646	case CMD_IOCB_FCP_IBIDIR64_CX:
1647	case CMD_IOCB_FCP_ITASKMGT64_CX:
1648	case CMD_IOCB_LOGENTRY_CN:
1649	case CMD_IOCB_LOGENTRY_ASYNC_CN:
1650	printk("%s - Unhandled SLI-3 Command x%x\n",
1651	__func__, iocb_cmnd);
1652	type = LPFC_UNKNOWN_IOCB;
1653	break;
1654	default:
1655	type = LPFC_UNKNOWN_IOCB;
1656	break;
1657	}
1658
1659	return type;
1660	}
1661
1662	/**
1663	* lpfc_sli_ring_map - Issue config_ring mbox for all rings
1664	* @phba: Pointer to HBA context object.
1665	*
1666	* This function is called from SLI initialization code
1667	* to configure every ring of the HBA's SLI interface. The
1668	* caller is not required to hold any lock. This function issues
1669	* a config_ring mailbox command for each ring.
1670	* This function returns zero if successful else returns a negative
1671	* error code.
1672	**/
1673	static int
1674	lpfc_sli_ring_map(struct lpfc_hba *phba)
1675	{
1676	struct lpfc_sli *psli = &phba->sli;
1677	LPFC_MBOXQ_t *pmb;
1678	MAILBOX_t *pmbox;
1679	int i, rc, ret = 0;
1680
1681	pmb = (LPFC_MBOXQ_t *) mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
1682	if (!pmb)
1683	return -ENOMEM;
1684	pmbox = &pmb->u.mb;
1685	phba->link_state = LPFC_INIT_MBX_CMDS;
1686	for (i = 0; i < psli->num_rings; i++) {
1687	lpfc_config_ring(phba, i, pmb);
1688	rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
1689	if (rc != MBX_SUCCESS) {
1690	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1691	"0446 Adapter failed to init (%d), "
1692	"mbxCmd x%x CFG_RING, mbxStatus x%x, "
1693	"ring %d\n",
1694	rc, pmbox->mbxCommand,
1695	pmbox->mbxStatus, i);
1696	phba->link_state = LPFC_HBA_ERROR;
1697	ret = -ENXIO;
1698	break;
1699	}
1700	}
1701	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
1702	return ret;
1703	}
1704
1705	/**
1706	* lpfc_sli_ringtxcmpl_put - Adds new iocb to the txcmplq
1707	* @phba: Pointer to HBA context object.
1708	* @pring: Pointer to driver SLI ring object.
1709	* @piocb: Pointer to the driver iocb object.
1710	*
1711	* The driver calls this function with the hbalock held for SLI3 ports or
1712	* the ring lock held for SLI4 ports. The function adds the
1713	* new iocb to txcmplq of the given ring. This function always returns
1714	* 0. If this function is called for ELS ring, this function checks if
1715	* there is a vport associated with the ELS command. This function also
1716	* starts els_tmofunc timer if this is an ELS command.
1717	**/
1718	static int
1719	lpfc_sli_ringtxcmpl_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
1720	struct lpfc_iocbq *piocb)
1721	{
1722	u32 ulp_command = 0;
1723
1724	BUG_ON(!piocb);
1725	ulp_command = get_job_cmnd(phba, iocbq: piocb);
1726
1727	list_add_tail(new: &piocb->list, head: &pring->txcmplq);
1728	piocb->cmd_flag |= LPFC_IO_ON_TXCMPLQ;
1729	pring->txcmplq_cnt++;
1730	if ((unlikely(pring->ringno == LPFC_ELS_RING)) &&
1731	(ulp_command != CMD_ABORT_XRI_WQE) &&
1732	(ulp_command != CMD_ABORT_XRI_CN) &&
1733	(ulp_command != CMD_CLOSE_XRI_CN)) {
1734	BUG_ON(!piocb->vport);
1735	if (!(piocb->vport->load_flag & FC_UNLOADING))
1736	mod_timer(timer: &piocb->vport->els_tmofunc,
1737	expires: jiffies +
1738	msecs_to_jiffies(m: 1000 * (phba->fc_ratov << 1)));
1739	}
1740
1741	return 0;
1742	}
1743
1744	/**
1745	* lpfc_sli_ringtx_get - Get first element of the txq
1746	* @phba: Pointer to HBA context object.
1747	* @pring: Pointer to driver SLI ring object.
1748	*
1749	* This function is called with hbalock held to get next
1750	* iocb in txq of the given ring. If there is any iocb in
1751	* the txq, the function returns first iocb in the list after
1752	* removing the iocb from the list, else it returns NULL.
1753	**/
1754	struct lpfc_iocbq *
1755	lpfc_sli_ringtx_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
1756	{
1757	struct lpfc_iocbq *cmd_iocb;
1758
1759	lockdep_assert_held(&phba->hbalock);
1760
1761	list_remove_head((&pring->txq), cmd_iocb, struct lpfc_iocbq, list);
1762	return cmd_iocb;
1763	}
1764
1765	/**
1766	* lpfc_cmf_sync_cmpl - Process a CMF_SYNC_WQE cmpl
1767	* @phba: Pointer to HBA context object.
1768	* @cmdiocb: Pointer to driver command iocb object.
1769	* @rspiocb: Pointer to driver response iocb object.
1770	*
1771	* This routine will inform the driver of any BW adjustments we need
1772	* to make. These changes will be picked up during the next CMF
1773	* timer interrupt. In addition, any BW changes will be logged
1774	* with LOG_CGN_MGMT.
1775	**/
1776	static void
1777	lpfc_cmf_sync_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
1778	struct lpfc_iocbq *rspiocb)
1779	{
1780	union lpfc_wqe128 *wqe;
1781	uint32_t status, info;
1782	struct lpfc_wcqe_complete *wcqe = &rspiocb->wcqe_cmpl;
1783	uint64_t bw, bwdif, slop;
1784	uint64_t pcent, bwpcent;
1785	int asig, afpin, sigcnt, fpincnt;
1786	int wsigmax, wfpinmax, cg, tdp;
1787	char *s;
1788
1789	/* First check for error */
1790	status = bf_get(lpfc_wcqe_c_status, wcqe);
1791	if (status) {
1792	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1793	"6211 CMF_SYNC_WQE Error "
1794	"req_tag x%x status x%x hwstatus x%x "
1795	"tdatap x%x parm x%x\n",
1796	bf_get(lpfc_wcqe_c_request_tag, wcqe),
1797	bf_get(lpfc_wcqe_c_status, wcqe),
1798	bf_get(lpfc_wcqe_c_hw_status, wcqe),
1799	wcqe->total_data_placed,
1800	wcqe->parameter);
1801	goto out;
1802	}
1803
1804	/* Gather congestion information on a successful cmpl */
1805	info = wcqe->parameter;
1806	phba->cmf_active_info = info;
1807
1808	/* See if firmware info count is valid or has changed */
1809	if (info > LPFC_MAX_CMF_INFO || phba->cmf_info_per_interval == info)
1810	info = 0;
1811	else
1812	phba->cmf_info_per_interval = info;
1813
1814	tdp = bf_get(lpfc_wcqe_c_cmf_bw, wcqe);
1815	cg = bf_get(lpfc_wcqe_c_cmf_cg, wcqe);
1816
1817	/* Get BW requirement from firmware */
1818	bw = (uint64_t)tdp * LPFC_CMF_BLK_SIZE;
1819	if (!bw) {
1820	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1821	"6212 CMF_SYNC_WQE x%x: NULL bw\n",
1822	bf_get(lpfc_wcqe_c_request_tag, wcqe));
1823	goto out;
1824	}
1825
1826	/* Gather information needed for logging if a BW change is required */
1827	wqe = &cmdiocb->wqe;
1828	asig = bf_get(cmf_sync_asig, &wqe->cmf_sync);
1829	afpin = bf_get(cmf_sync_afpin, &wqe->cmf_sync);
1830	fpincnt = bf_get(cmf_sync_wfpincnt, &wqe->cmf_sync);
1831	sigcnt = bf_get(cmf_sync_wsigcnt, &wqe->cmf_sync);
1832	if (phba->cmf_max_bytes_per_interval != bw ||
1833	(asig || afpin || sigcnt || fpincnt)) {
1834	/* Are we increasing or decreasing BW */
1835	if (phba->cmf_max_bytes_per_interval < bw) {
1836	bwdif = bw - phba->cmf_max_bytes_per_interval;
1837	s = "Increase";
1838	} else {
1839	bwdif = phba->cmf_max_bytes_per_interval - bw;
1840	s = "Decrease";
1841	}
1842
1843	/* What is the change percentage */
1844	slop = div_u64(dividend: phba->cmf_link_byte_count, divisor: 200); /*For rounding*/
1845	pcent = div64_u64(dividend: bwdif * 100 + slop,
1846	divisor: phba->cmf_link_byte_count);
1847	bwpcent = div64_u64(dividend: bw * 100 + slop,
1848	divisor: phba->cmf_link_byte_count);
1849	/* Because of bytes adjustment due to shorter timer in
1850	* lpfc_cmf_timer() the cmf_link_byte_count can be shorter and
1851	* may seem like BW is above 100%.
1852	*/
1853	if (bwpcent > 100)
1854	bwpcent = 100;
1855
1856	if (phba->cmf_max_bytes_per_interval < bw &&
1857	bwpcent > 95)
1858	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1859	"6208 Congestion bandwidth "
1860	"limits removed\n");
1861	else if ((phba->cmf_max_bytes_per_interval > bw) &&
1862	((bwpcent + pcent) <= 100) && ((bwpcent + pcent) > 95))
1863	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1864	"6209 Congestion bandwidth "
1865	"limits in effect\n");
1866
1867	if (asig) {
1868	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1869	"6237 BW Threshold %lld%% (%lld): "
1870	"%lld%% %s: Signal Alarm: cg:%d "
1871	"Info:%u\n",
1872	bwpcent, bw, pcent, s, cg,
1873	phba->cmf_active_info);
1874	} else if (afpin) {
1875	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1876	"6238 BW Threshold %lld%% (%lld): "
1877	"%lld%% %s: FPIN Alarm: cg:%d "
1878	"Info:%u\n",
1879	bwpcent, bw, pcent, s, cg,
1880	phba->cmf_active_info);
1881	} else if (sigcnt) {
1882	wsigmax = bf_get(cmf_sync_wsigmax, &wqe->cmf_sync);
1883	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1884	"6239 BW Threshold %lld%% (%lld): "
1885	"%lld%% %s: Signal Warning: "
1886	"Cnt %d Max %d: cg:%d Info:%u\n",
1887	bwpcent, bw, pcent, s, sigcnt,
1888	wsigmax, cg, phba->cmf_active_info);
1889	} else if (fpincnt) {
1890	wfpinmax = bf_get(cmf_sync_wfpinmax, &wqe->cmf_sync);
1891	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1892	"6240 BW Threshold %lld%% (%lld): "
1893	"%lld%% %s: FPIN Warning: "
1894	"Cnt %d Max %d: cg:%d Info:%u\n",
1895	bwpcent, bw, pcent, s, fpincnt,
1896	wfpinmax, cg, phba->cmf_active_info);
1897	} else {
1898	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1899	"6241 BW Threshold %lld%% (%lld): "
1900	"CMF %lld%% %s: cg:%d Info:%u\n",
1901	bwpcent, bw, pcent, s, cg,
1902	phba->cmf_active_info);
1903	}
1904	} else if (info) {
1905	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1906	"6246 Info Threshold %u\n", info);
1907	}
1908
1909	/* Save BW change to be picked up during next timer interrupt */
1910	phba->cmf_last_sync_bw = bw;
1911	out:
1912	lpfc_sli_release_iocbq(phba, iocbq: cmdiocb);
1913	}
1914
1915	/**
1916	* lpfc_issue_cmf_sync_wqe - Issue a CMF_SYNC_WQE
1917	* @phba: Pointer to HBA context object.
1918	* @ms: ms to set in WQE interval, 0 means use init op
1919	* @total: Total rcv bytes for this interval
1920	*
1921	* This routine is called every CMF timer interrupt. Its purpose is
1922	* to issue a CMF_SYNC_WQE to the firmware to inform it of any events
1923	* that may indicate we have congestion (FPINs or Signals). Upon
1924	* completion, the firmware will indicate any BW restrictions the
1925	* driver may need to take.
1926	**/
1927	int
1928	lpfc_issue_cmf_sync_wqe(struct lpfc_hba *phba, u32 ms, u64 total)
1929	{
1930	union lpfc_wqe128 *wqe;
1931	struct lpfc_iocbq *sync_buf;
1932	unsigned long iflags;
1933	u32 ret_val;
1934	u32 atot, wtot, max;
1935	u8 warn_sync_period = 0;
1936
1937	/* First address any alarm / warning activity */
1938	atot = atomic_xchg(v: &phba->cgn_sync_alarm_cnt, new: 0);
1939	wtot = atomic_xchg(v: &phba->cgn_sync_warn_cnt, new: 0);
1940
1941	/* ONLY Managed mode will send the CMF_SYNC_WQE to the HBA */
1942	if (phba->cmf_active_mode != LPFC_CFG_MANAGED ||
1943	phba->link_state == LPFC_LINK_DOWN)
1944	return 0;
1945
1946	spin_lock_irqsave(&phba->hbalock, iflags);
1947	sync_buf = __lpfc_sli_get_iocbq(phba);
1948	if (!sync_buf) {
1949	lpfc_printf_log(phba, KERN_ERR, LOG_CGN_MGMT,
1950	"6244 No available WQEs for CMF_SYNC_WQE\n");
1951	ret_val = ENOMEM;
1952	goto out_unlock;
1953	}
1954
1955	wqe = &sync_buf->wqe;
1956
1957	/* WQEs are reused. Clear stale data and set key fields to zero */
1958	memset(wqe, 0, sizeof(*wqe));
1959
1960	/* If this is the very first CMF_SYNC_WQE, issue an init operation */
1961	if (!ms) {
1962	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1963	"6441 CMF Init %d - CMF_SYNC_WQE\n",
1964	phba->fc_eventTag);
1965	bf_set(cmf_sync_op, &wqe->cmf_sync, 1); /* 1=init */
1966	bf_set(cmf_sync_interval, &wqe->cmf_sync, LPFC_CMF_INTERVAL);
1967	goto initpath;
1968	}
1969
1970	bf_set(cmf_sync_op, &wqe->cmf_sync, 0); /* 0=recalc */
1971	bf_set(cmf_sync_interval, &wqe->cmf_sync, ms);
1972
1973	/* Check for alarms / warnings */
1974	if (atot) {
1975	if (phba->cgn_reg_signal == EDC_CG_SIG_WARN_ALARM) {
1976	/* We hit an Signal alarm condition */
1977	bf_set(cmf_sync_asig, &wqe->cmf_sync, 1);
1978	} else {
1979	/* We hit a FPIN alarm condition */
1980	bf_set(cmf_sync_afpin, &wqe->cmf_sync, 1);
1981	}
1982	} else if (wtot) {
1983	if (phba->cgn_reg_signal == EDC_CG_SIG_WARN_ONLY ||
1984	phba->cgn_reg_signal == EDC_CG_SIG_WARN_ALARM) {
1985	/* We hit an Signal warning condition */
1986	max = LPFC_SEC_TO_MSEC / lpfc_fabric_cgn_frequency *
1987	lpfc_acqe_cgn_frequency;
1988	bf_set(cmf_sync_wsigmax, &wqe->cmf_sync, max);
1989	bf_set(cmf_sync_wsigcnt, &wqe->cmf_sync, wtot);
1990	warn_sync_period = lpfc_acqe_cgn_frequency;
1991	} else {
1992	/* We hit a FPIN warning condition */
1993	bf_set(cmf_sync_wfpinmax, &wqe->cmf_sync, 1);
1994	bf_set(cmf_sync_wfpincnt, &wqe->cmf_sync, 1);
1995	if (phba->cgn_fpin_frequency != LPFC_FPIN_INIT_FREQ)
1996	warn_sync_period =
1997	LPFC_MSECS_TO_SECS(phba->cgn_fpin_frequency);
1998	}
1999	}
2000
2001	/* Update total read blocks during previous timer interval */
2002	wqe->cmf_sync.read_bytes = (u32)(total / LPFC_CMF_BLK_SIZE);
2003
2004	initpath:
2005	bf_set(cmf_sync_ver, &wqe->cmf_sync, LPFC_CMF_SYNC_VER);
2006	wqe->cmf_sync.event_tag = phba->fc_eventTag;
2007	bf_set(cmf_sync_cmnd, &wqe->cmf_sync, CMD_CMF_SYNC_WQE);
2008
2009	/* Setup reqtag to match the wqe completion. */
2010	bf_set(cmf_sync_reqtag, &wqe->cmf_sync, sync_buf->iotag);
2011
2012	bf_set(cmf_sync_qosd, &wqe->cmf_sync, 1);
2013	bf_set(cmf_sync_period, &wqe->cmf_sync, warn_sync_period);
2014
2015	bf_set(cmf_sync_cmd_type, &wqe->cmf_sync, CMF_SYNC_COMMAND);
2016	bf_set(cmf_sync_wqec, &wqe->cmf_sync, 1);
2017	bf_set(cmf_sync_cqid, &wqe->cmf_sync, LPFC_WQE_CQ_ID_DEFAULT);
2018
2019	sync_buf->vport = phba->pport;
2020	sync_buf->cmd_cmpl = lpfc_cmf_sync_cmpl;
2021	sync_buf->cmd_dmabuf = NULL;
2022	sync_buf->rsp_dmabuf = NULL;
2023	sync_buf->bpl_dmabuf = NULL;
2024	sync_buf->sli4_xritag = NO_XRI;
2025
2026	sync_buf->cmd_flag |= LPFC_IO_CMF;
2027	ret_val = lpfc_sli4_issue_wqe(phba, qp: &phba->sli4_hba.hdwq[0], pwqe: sync_buf);
2028	if (ret_val) {
2029	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
2030	"6214 Cannot issue CMF_SYNC_WQE: x%x\n",
2031	ret_val);
2032	__lpfc_sli_release_iocbq(phba, iocbq: sync_buf);
2033	}
2034	out_unlock:
2035	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
2036	return ret_val;
2037	}
2038
2039	/**
2040	* lpfc_sli_next_iocb_slot - Get next iocb slot in the ring
2041	* @phba: Pointer to HBA context object.
2042	* @pring: Pointer to driver SLI ring object.
2043	*
2044	* This function is called with hbalock held and the caller must post the
2045	* iocb without releasing the lock. If the caller releases the lock,
2046	* iocb slot returned by the function is not guaranteed to be available.
2047	* The function returns pointer to the next available iocb slot if there
2048	* is available slot in the ring, else it returns NULL.
2049	* If the get index of the ring is ahead of the put index, the function
2050	* will post an error attention event to the worker thread to take the
2051	* HBA to offline state.
2052	**/
2053	static IOCB_t *
2054	lpfc_sli_next_iocb_slot (struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
2055	{
2056	struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
2057	uint32_t max_cmd_idx = pring->sli.sli3.numCiocb;
2058
2059	lockdep_assert_held(&phba->hbalock);
2060
2061	if ((pring->sli.sli3.next_cmdidx == pring->sli.sli3.cmdidx) &&
2062	(++pring->sli.sli3.next_cmdidx >= max_cmd_idx))
2063	pring->sli.sli3.next_cmdidx = 0;
2064
2065	if (unlikely(pring->sli.sli3.local_getidx ==
2066	pring->sli.sli3.next_cmdidx)) {
2067
2068	pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);
2069
2070	if (unlikely(pring->sli.sli3.local_getidx >= max_cmd_idx)) {
2071	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2072	"0315 Ring %d issue: portCmdGet %d "
2073	"is bigger than cmd ring %d\n",
2074	pring->ringno,
2075	pring->sli.sli3.local_getidx,
2076	max_cmd_idx);
2077
2078	phba->link_state = LPFC_HBA_ERROR;
2079	/*
2080	* All error attention handlers are posted to
2081	* worker thread
2082	*/
2083	phba->work_ha |= HA_ERATT;
2084	phba->work_hs = HS_FFER3;
2085
2086	lpfc_worker_wake_up(phba);
2087
2088	return NULL;
2089	}
2090
2091	if (pring->sli.sli3.local_getidx == pring->sli.sli3.next_cmdidx)
2092	return NULL;
2093	}
2094
2095	return lpfc_cmd_iocb(phba, pring);
2096	}
2097
2098	/**
2099	* lpfc_sli_next_iotag - Get an iotag for the iocb
2100	* @phba: Pointer to HBA context object.
2101	* @iocbq: Pointer to driver iocb object.
2102	*
2103	* This function gets an iotag for the iocb. If there is no unused iotag and
2104	* the iocbq_lookup_len < 0xffff, this function allocates a bigger iotag_lookup
2105	* array and assigns a new iotag.
2106	* The function returns the allocated iotag if successful, else returns zero.
2107	* Zero is not a valid iotag.
2108	* The caller is not required to hold any lock.
2109	**/
2110	uint16_t
2111	lpfc_sli_next_iotag(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
2112	{
2113	struct lpfc_iocbq **new_arr;
2114	struct lpfc_iocbq **old_arr;
2115	size_t new_len;
2116	struct lpfc_sli *psli = &phba->sli;
2117	uint16_t iotag;
2118
2119	spin_lock_irq(lock: &phba->hbalock);
2120	iotag = psli->last_iotag;
2121	if(++iotag < psli->iocbq_lookup_len) {
2122	psli->last_iotag = iotag;
2123	psli->iocbq_lookup[iotag] = iocbq;
2124	spin_unlock_irq(lock: &phba->hbalock);
2125	iocbq->iotag = iotag;
2126	return iotag;
2127	} else if (psli->iocbq_lookup_len < (0xffff
2128	- LPFC_IOCBQ_LOOKUP_INCREMENT)) {
2129	new_len = psli->iocbq_lookup_len + LPFC_IOCBQ_LOOKUP_INCREMENT;
2130	spin_unlock_irq(lock: &phba->hbalock);
2131	new_arr = kcalloc(n: new_len, size: sizeof(struct lpfc_iocbq *),
2132	GFP_KERNEL);
2133	if (new_arr) {
2134	spin_lock_irq(lock: &phba->hbalock);
2135	old_arr = psli->iocbq_lookup;
2136	if (new_len <= psli->iocbq_lookup_len) {
2137	/* highly unprobable case */
2138	kfree(objp: new_arr);
2139	iotag = psli->last_iotag;
2140	if(++iotag < psli->iocbq_lookup_len) {
2141	psli->last_iotag = iotag;
2142	psli->iocbq_lookup[iotag] = iocbq;
2143	spin_unlock_irq(lock: &phba->hbalock);
2144	iocbq->iotag = iotag;
2145	return iotag;
2146	}
2147	spin_unlock_irq(lock: &phba->hbalock);
2148	return 0;
2149	}
2150	if (psli->iocbq_lookup)
2151	memcpy(new_arr, old_arr,
2152	((psli->last_iotag + 1) *
2153	sizeof (struct lpfc_iocbq *)));
2154	psli->iocbq_lookup = new_arr;
2155	psli->iocbq_lookup_len = new_len;
2156	psli->last_iotag = iotag;
2157	psli->iocbq_lookup[iotag] = iocbq;
2158	spin_unlock_irq(lock: &phba->hbalock);
2159	iocbq->iotag = iotag;
2160	kfree(objp: old_arr);
2161	return iotag;
2162	}
2163	} else
2164	spin_unlock_irq(lock: &phba->hbalock);
2165
2166	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
2167	"0318 Failed to allocate IOTAG.last IOTAG is %d\n",
2168	psli->last_iotag);
2169
2170	return 0;
2171	}
2172
2173	/**
2174	* lpfc_sli_submit_iocb - Submit an iocb to the firmware
2175	* @phba: Pointer to HBA context object.
2176	* @pring: Pointer to driver SLI ring object.
2177	* @iocb: Pointer to iocb slot in the ring.
2178	* @nextiocb: Pointer to driver iocb object which need to be
2179	* posted to firmware.
2180	*
2181	* This function is called to post a new iocb to the firmware. This
2182	* function copies the new iocb to ring iocb slot and updates the
2183	* ring pointers. It adds the new iocb to txcmplq if there is
2184	* a completion call back for this iocb else the function will free the
2185	* iocb object. The hbalock is asserted held in the code path calling
2186	* this routine.
2187	**/
2188	static void
2189	lpfc_sli_submit_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
2190	IOCB_t *iocb, struct lpfc_iocbq *nextiocb)
2191	{
2192	/*
2193	* Set up an iotag
2194	*/
2195	nextiocb->iocb.ulpIoTag = (nextiocb->cmd_cmpl) ? nextiocb->iotag : 0;
2196
2197
2198	if (pring->ringno == LPFC_ELS_RING) {
2199	lpfc_debugfs_slow_ring_trc(phba,
2200	"IOCB cmd ring: wd4:x%08x wd6:x%08x wd7:x%08x",
2201	*(((uint32_t *) &nextiocb->iocb) + 4),
2202	*(((uint32_t *) &nextiocb->iocb) + 6),
2203	*(((uint32_t *) &nextiocb->iocb) + 7));
2204	}
2205
2206	/*
2207	* Issue iocb command to adapter
2208	*/
2209	lpfc_sli_pcimem_bcopy(&nextiocb->iocb, iocb, phba->iocb_cmd_size);
2210	wmb();
2211	pring->stats.iocb_cmd++;
2212
2213	/*
2214	* If there is no completion routine to call, we can release the
2215	* IOCB buffer back right now. For IOCBs, like QUE_RING_BUF,
2216	* that have no rsp ring completion, cmd_cmpl MUST be NULL.
2217	*/
2218	if (nextiocb->cmd_cmpl)
2219	lpfc_sli_ringtxcmpl_put(phba, pring, piocb: nextiocb);
2220	else
2221	__lpfc_sli_release_iocbq(phba, iocbq: nextiocb);
2222
2223	/*
2224	* Let the HBA know what IOCB slot will be the next one the
2225	* driver will put a command into.
2226	*/
2227	pring->sli.sli3.cmdidx = pring->sli.sli3.next_cmdidx;
2228	writel(val: pring->sli.sli3.cmdidx, addr: &phba->host_gp[pring->ringno].cmdPutInx);
2229	}
2230
2231	/**
2232	* lpfc_sli_update_full_ring - Update the chip attention register
2233	* @phba: Pointer to HBA context object.
2234	* @pring: Pointer to driver SLI ring object.
2235	*
2236	* The caller is not required to hold any lock for calling this function.
2237	* This function updates the chip attention bits for the ring to inform firmware
2238	* that there are pending work to be done for this ring and requests an
2239	* interrupt when there is space available in the ring. This function is
2240	* called when the driver is unable to post more iocbs to the ring due
2241	* to unavailability of space in the ring.
2242	**/
2243	static void
2244	lpfc_sli_update_full_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
2245	{
2246	int ringno = pring->ringno;
2247
2248	pring->flag |= LPFC_CALL_RING_AVAILABLE;
2249
2250	wmb();
2251
2252	/*
2253	* Set ring 'ringno' to SET R0CE_REQ in Chip Att register.
2254	* The HBA will tell us when an IOCB entry is available.
2255	*/
2256	writel(val: (CA_R0ATT|CA_R0CE_REQ) << (ringno*4), addr: phba->CAregaddr);
2257	readl(addr: phba->CAregaddr); /* flush */
2258
2259	pring->stats.iocb_cmd_full++;
2260	}
2261
2262	/**
2263	* lpfc_sli_update_ring - Update chip attention register
2264	* @phba: Pointer to HBA context object.
2265	* @pring: Pointer to driver SLI ring object.
2266	*
2267	* This function updates the chip attention register bit for the
2268	* given ring to inform HBA that there is more work to be done
2269	* in this ring. The caller is not required to hold any lock.
2270	**/
2271	static void
2272	lpfc_sli_update_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
2273	{
2274	int ringno = pring->ringno;
2275
2276	/*
2277	* Tell the HBA that there is work to do in this ring.
2278	*/
2279	if (!(phba->sli3_options & LPFC_SLI3_CRP_ENABLED)) {
2280	wmb();
2281	writel(CA_R0ATT << (ringno * 4), addr: phba->CAregaddr);
2282	readl(addr: phba->CAregaddr); /* flush */
2283	}
2284	}
2285
2286	/**
2287	* lpfc_sli_resume_iocb - Process iocbs in the txq
2288	* @phba: Pointer to HBA context object.
2289	* @pring: Pointer to driver SLI ring object.
2290	*
2291	* This function is called with hbalock held to post pending iocbs
2292	* in the txq to the firmware. This function is called when driver
2293	* detects space available in the ring.
2294	**/
2295	static void
2296	lpfc_sli_resume_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
2297	{
2298	IOCB_t *iocb;
2299	struct lpfc_iocbq *nextiocb;
2300
2301	lockdep_assert_held(&phba->hbalock);
2302
2303	/*
2304	* Check to see if:
2305	* (a) there is anything on the txq to send
2306	* (b) link is up
2307	* (c) link attention events can be processed (fcp ring only)
2308	* (d) IOCB processing is not blocked by the outstanding mbox command.
2309	*/
2310
2311	if (lpfc_is_link_up(phba) &&
2312	(!list_empty(head: &pring->txq)) &&
2313	(pring->ringno != LPFC_FCP_RING ||
2314	phba->sli.sli_flag & LPFC_PROCESS_LA)) {
2315
2316	while ((iocb = lpfc_sli_next_iocb_slot(phba, pring)) &&
2317	(nextiocb = lpfc_sli_ringtx_get(phba, pring)))
2318	lpfc_sli_submit_iocb(phba, pring, iocb, nextiocb);
2319
2320	if (iocb)
2321	lpfc_sli_update_ring(phba, pring);
2322	else
2323	lpfc_sli_update_full_ring(phba, pring);
2324	}
2325
2326	return;
2327	}
2328
2329	/**
2330	* lpfc_sli_next_hbq_slot - Get next hbq entry for the HBQ
2331	* @phba: Pointer to HBA context object.
2332	* @hbqno: HBQ number.
2333	*
2334	* This function is called with hbalock held to get the next
2335	* available slot for the given HBQ. If there is free slot
2336	* available for the HBQ it will return pointer to the next available
2337	* HBQ entry else it will return NULL.
2338	**/
2339	static struct lpfc_hbq_entry *
2340	lpfc_sli_next_hbq_slot(struct lpfc_hba *phba, uint32_t hbqno)
2341	{
2342	struct hbq_s *hbqp = &phba->hbqs[hbqno];
2343
2344	lockdep_assert_held(&phba->hbalock);
2345
2346	if (hbqp->next_hbqPutIdx == hbqp->hbqPutIdx &&
2347	++hbqp->next_hbqPutIdx >= hbqp->entry_count)
2348	hbqp->next_hbqPutIdx = 0;
2349
2350	if (unlikely(hbqp->local_hbqGetIdx == hbqp->next_hbqPutIdx)) {
2351	uint32_t raw_index = phba->hbq_get[hbqno];
2352	uint32_t getidx = le32_to_cpu(raw_index);
2353
2354	hbqp->local_hbqGetIdx = getidx;
2355
2356	if (unlikely(hbqp->local_hbqGetIdx >= hbqp->entry_count)) {
2357	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2358	"1802 HBQ %d: local_hbqGetIdx "
2359	"%u is > than hbqp->entry_count %u\n",
2360	hbqno, hbqp->local_hbqGetIdx,
2361	hbqp->entry_count);
2362
2363	phba->link_state = LPFC_HBA_ERROR;
2364	return NULL;
2365	}
2366
2367	if (hbqp->local_hbqGetIdx == hbqp->next_hbqPutIdx)
2368	return NULL;
2369	}
2370
2371	return (struct lpfc_hbq_entry *) phba->hbqs[hbqno].hbq_virt +
2372	hbqp->hbqPutIdx;
2373	}
2374
2375	/**
2376	* lpfc_sli_hbqbuf_free_all - Free all the hbq buffers
2377	* @phba: Pointer to HBA context object.
2378	*
2379	* This function is called with no lock held to free all the
2380	* hbq buffers while uninitializing the SLI interface. It also
2381	* frees the HBQ buffers returned by the firmware but not yet
2382	* processed by the upper layers.
2383	**/
2384	void
2385	lpfc_sli_hbqbuf_free_all(struct lpfc_hba *phba)
2386	{
2387	struct lpfc_dmabuf *dmabuf, *next_dmabuf;
2388	struct hbq_dmabuf *hbq_buf;
2389	unsigned long flags;
2390	int i, hbq_count;
2391
2392	hbq_count = lpfc_sli_hbq_count();
2393	/* Return all memory used by all HBQs */
2394	spin_lock_irqsave(&phba->hbalock, flags);
2395	for (i = 0; i < hbq_count; ++i) {
2396	list_for_each_entry_safe(dmabuf, next_dmabuf,
2397	&phba->hbqs[i].hbq_buffer_list, list) {
2398	hbq_buf = container_of(dmabuf, struct hbq_dmabuf, dbuf);
2399	list_del(entry: &hbq_buf->dbuf.list);
2400	(phba->hbqs[i].hbq_free_buffer)(phba, hbq_buf);
2401	}
2402	phba->hbqs[i].buffer_count = 0;
2403	}
2404
2405	/* Mark the HBQs not in use */
2406	phba->hbq_in_use = 0;
2407	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
2408	}
2409
2410	/**
2411	* lpfc_sli_hbq_to_firmware - Post the hbq buffer to firmware
2412	* @phba: Pointer to HBA context object.
2413	* @hbqno: HBQ number.
2414	* @hbq_buf: Pointer to HBQ buffer.
2415	*
2416	* This function is called with the hbalock held to post a
2417	* hbq buffer to the firmware. If the function finds an empty
2418	* slot in the HBQ, it will post the buffer. The function will return
2419	* pointer to the hbq entry if it successfully post the buffer
2420	* else it will return NULL.
2421	**/
2422	static int
2423	lpfc_sli_hbq_to_firmware(struct lpfc_hba *phba, uint32_t hbqno,
2424	struct hbq_dmabuf *hbq_buf)
2425	{
2426	lockdep_assert_held(&phba->hbalock);
2427	return phba->lpfc_sli_hbq_to_firmware(phba, hbqno, hbq_buf);
2428	}
2429
2430	/**
2431	* lpfc_sli_hbq_to_firmware_s3 - Post the hbq buffer to SLI3 firmware
2432	* @phba: Pointer to HBA context object.
2433	* @hbqno: HBQ number.
2434	* @hbq_buf: Pointer to HBQ buffer.
2435	*
2436	* This function is called with the hbalock held to post a hbq buffer to the
2437	* firmware. If the function finds an empty slot in the HBQ, it will post the
2438	* buffer and place it on the hbq_buffer_list. The function will return zero if
2439	* it successfully post the buffer else it will return an error.
2440	**/
2441	static int
2442	lpfc_sli_hbq_to_firmware_s3(struct lpfc_hba *phba, uint32_t hbqno,
2443	struct hbq_dmabuf *hbq_buf)
2444	{
2445	struct lpfc_hbq_entry *hbqe;
2446	dma_addr_t physaddr = hbq_buf->dbuf.phys;
2447
2448	lockdep_assert_held(&phba->hbalock);
2449	/* Get next HBQ entry slot to use */
2450	hbqe = lpfc_sli_next_hbq_slot(phba, hbqno);
2451	if (hbqe) {
2452	struct hbq_s *hbqp = &phba->hbqs[hbqno];
2453
2454	hbqe->bde.addrHigh = le32_to_cpu(putPaddrHigh(physaddr));
2455	hbqe->bde.addrLow = le32_to_cpu(putPaddrLow(physaddr));
2456	hbqe->bde.tus.f.bdeSize = hbq_buf->total_size;
2457	hbqe->bde.tus.f.bdeFlags = 0;
2458	hbqe->bde.tus.w = le32_to_cpu(hbqe->bde.tus.w);
2459	hbqe->buffer_tag = le32_to_cpu(hbq_buf->tag);
2460	/* Sync SLIM */
2461	hbqp->hbqPutIdx = hbqp->next_hbqPutIdx;
2462	writel(val: hbqp->hbqPutIdx, addr: phba->hbq_put + hbqno);
2463	/* flush */
2464	readl(addr: phba->hbq_put + hbqno);
2465	list_add_tail(new: &hbq_buf->dbuf.list, head: &hbqp->hbq_buffer_list);
2466	return 0;
2467	} else
2468	return -ENOMEM;
2469	}
2470
2471	/**
2472	* lpfc_sli_hbq_to_firmware_s4 - Post the hbq buffer to SLI4 firmware
2473	* @phba: Pointer to HBA context object.
2474	* @hbqno: HBQ number.
2475	* @hbq_buf: Pointer to HBQ buffer.
2476	*
2477	* This function is called with the hbalock held to post an RQE to the SLI4
2478	* firmware. If able to post the RQE to the RQ it will queue the hbq entry to
2479	* the hbq_buffer_list and return zero, otherwise it will return an error.
2480	**/
2481	static int
2482	lpfc_sli_hbq_to_firmware_s4(struct lpfc_hba *phba, uint32_t hbqno,
2483	struct hbq_dmabuf *hbq_buf)
2484	{
2485	int rc;
2486	struct lpfc_rqe hrqe;
2487	struct lpfc_rqe drqe;
2488	struct lpfc_queue *hrq;
2489	struct lpfc_queue *drq;
2490
2491	if (hbqno != LPFC_ELS_HBQ)
2492	return 1;
2493	hrq = phba->sli4_hba.hdr_rq;
2494	drq = phba->sli4_hba.dat_rq;
2495
2496	lockdep_assert_held(&phba->hbalock);
2497	hrqe.address_lo = putPaddrLow(hbq_buf->hbuf.phys);
2498	hrqe.address_hi = putPaddrHigh(hbq_buf->hbuf.phys);
2499	drqe.address_lo = putPaddrLow(hbq_buf->dbuf.phys);
2500	drqe.address_hi = putPaddrHigh(hbq_buf->dbuf.phys);
2501	rc = lpfc_sli4_rq_put(hq: hrq, dq: drq, hrqe: &hrqe, drqe: &drqe);
2502	if (rc < 0)
2503	return rc;
2504	hbq_buf->tag = (rc | (hbqno << 16));
2505	list_add_tail(new: &hbq_buf->dbuf.list, head: &phba->hbqs[hbqno].hbq_buffer_list);
2506	return 0;
2507	}
2508
2509	/* HBQ for ELS and CT traffic. */
2510	static struct lpfc_hbq_init lpfc_els_hbq = {
2511	.rn = 1,
2512	.entry_count = 256,
2513	.mask_count = 0,
2514	.profile = 0,
2515	.ring_mask = (1 << LPFC_ELS_RING),
2516	.buffer_count = 0,
2517	.init_count = 40,
2518	.add_count = 40,
2519	};
2520
2521	/* Array of HBQs */
2522	struct lpfc_hbq_init *lpfc_hbq_defs[] = {
2523	&lpfc_els_hbq,
2524	};
2525
2526	/**
2527	* lpfc_sli_hbqbuf_fill_hbqs - Post more hbq buffers to HBQ
2528	* @phba: Pointer to HBA context object.
2529	* @hbqno: HBQ number.
2530	* @count: Number of HBQ buffers to be posted.
2531	*
2532	* This function is called with no lock held to post more hbq buffers to the
2533	* given HBQ. The function returns the number of HBQ buffers successfully
2534	* posted.
2535	**/
2536	static int
2537	lpfc_sli_hbqbuf_fill_hbqs(struct lpfc_hba *phba, uint32_t hbqno, uint32_t count)
2538	{
2539	uint32_t i, posted = 0;
2540	unsigned long flags;
2541	struct hbq_dmabuf *hbq_buffer;
2542	LIST_HEAD(hbq_buf_list);
2543	if (!phba->hbqs[hbqno].hbq_alloc_buffer)
2544	return 0;
2545
2546	if ((phba->hbqs[hbqno].buffer_count + count) >
2547	lpfc_hbq_defs[hbqno]->entry_count)
2548	count = lpfc_hbq_defs[hbqno]->entry_count -
2549	phba->hbqs[hbqno].buffer_count;
2550	if (!count)
2551	return 0;
2552	/* Allocate HBQ entries */
2553	for (i = 0; i < count; i++) {
2554	hbq_buffer = (phba->hbqs[hbqno].hbq_alloc_buffer)(phba);
2555	if (!hbq_buffer)
2556	break;
2557	list_add_tail(new: &hbq_buffer->dbuf.list, head: &hbq_buf_list);
2558	}
2559	/* Check whether HBQ is still in use */
2560	spin_lock_irqsave(&phba->hbalock, flags);
2561	if (!phba->hbq_in_use)
2562	goto err;
2563	while (!list_empty(head: &hbq_buf_list)) {
2564	list_remove_head(&hbq_buf_list, hbq_buffer, struct hbq_dmabuf,
2565	dbuf.list);
2566	hbq_buffer->tag = (phba->hbqs[hbqno].buffer_count |
2567	(hbqno << 16));
2568	if (!lpfc_sli_hbq_to_firmware(phba, hbqno, hbq_buf: hbq_buffer)) {
2569	phba->hbqs[hbqno].buffer_count++;
2570	posted++;
2571	} else
2572	(phba->hbqs[hbqno].hbq_free_buffer)(phba, hbq_buffer);
2573	}
2574	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
2575	return posted;
2576	err:
2577	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
2578	while (!list_empty(head: &hbq_buf_list)) {
2579	list_remove_head(&hbq_buf_list, hbq_buffer, struct hbq_dmabuf,
2580	dbuf.list);
2581	(phba->hbqs[hbqno].hbq_free_buffer)(phba, hbq_buffer);
2582	}
2583	return 0;
2584	}
2585
2586	/**
2587	* lpfc_sli_hbqbuf_add_hbqs - Post more HBQ buffers to firmware
2588	* @phba: Pointer to HBA context object.
2589	* @qno: HBQ number.
2590	*
2591	* This function posts more buffers to the HBQ. This function
2592	* is called with no lock held. The function returns the number of HBQ entries
2593	* successfully allocated.
2594	**/
2595	int
2596	lpfc_sli_hbqbuf_add_hbqs(struct lpfc_hba *phba, uint32_t qno)
2597	{
2598	if (phba->sli_rev == LPFC_SLI_REV4)
2599	return 0;
2600	else
2601	return lpfc_sli_hbqbuf_fill_hbqs(phba, hbqno: qno,
2602	count: lpfc_hbq_defs[qno]->add_count);
2603	}
2604
2605	/**
2606	* lpfc_sli_hbqbuf_init_hbqs - Post initial buffers to the HBQ
2607	* @phba: Pointer to HBA context object.
2608	* @qno: HBQ queue number.
2609	*
2610	* This function is called from SLI initialization code path with
2611	* no lock held to post initial HBQ buffers to firmware. The
2612	* function returns the number of HBQ entries successfully allocated.
2613	**/
2614	static int
2615	lpfc_sli_hbqbuf_init_hbqs(struct lpfc_hba *phba, uint32_t qno)
2616	{
2617	if (phba->sli_rev == LPFC_SLI_REV4)
2618	return lpfc_sli_hbqbuf_fill_hbqs(phba, hbqno: qno,
2619	count: lpfc_hbq_defs[qno]->entry_count);
2620	else
2621	return lpfc_sli_hbqbuf_fill_hbqs(phba, hbqno: qno,
2622	count: lpfc_hbq_defs[qno]->init_count);
2623	}
2624
2625	/*
2626	* lpfc_sli_hbqbuf_get - Remove the first hbq off of an hbq list
2627	*
2628	* This function removes the first hbq buffer on an hbq list and returns a
2629	* pointer to that buffer. If it finds no buffers on the list it returns NULL.
2630	**/
2631	static struct hbq_dmabuf *
2632	lpfc_sli_hbqbuf_get(struct list_head *rb_list)
2633	{
2634	struct lpfc_dmabuf *d_buf;
2635
2636	list_remove_head(rb_list, d_buf, struct lpfc_dmabuf, list);
2637	if (!d_buf)
2638	return NULL;
2639	return container_of(d_buf, struct hbq_dmabuf, dbuf);
2640	}
2641
2642	/**
2643	* lpfc_sli_rqbuf_get - Remove the first dma buffer off of an RQ list
2644	* @phba: Pointer to HBA context object.
2645	* @hrq: HBQ number.
2646	*
2647	* This function removes the first RQ buffer on an RQ buffer list and returns a
2648	* pointer to that buffer. If it finds no buffers on the list it returns NULL.
2649	**/
2650	static struct rqb_dmabuf *
2651	lpfc_sli_rqbuf_get(struct lpfc_hba *phba, struct lpfc_queue *hrq)
2652	{
2653	struct lpfc_dmabuf *h_buf;
2654	struct lpfc_rqb *rqbp;
2655
2656	rqbp = hrq->rqbp;
2657	list_remove_head(&rqbp->rqb_buffer_list, h_buf,
2658	struct lpfc_dmabuf, list);
2659	if (!h_buf)
2660	return NULL;
2661	rqbp->buffer_count--;
2662	return container_of(h_buf, struct rqb_dmabuf, hbuf);
2663	}
2664
2665	/**
2666	* lpfc_sli_hbqbuf_find - Find the hbq buffer associated with a tag
2667	* @phba: Pointer to HBA context object.
2668	* @tag: Tag of the hbq buffer.
2669	*
2670	* This function searches for the hbq buffer associated with the given tag in
2671	* the hbq buffer list. If it finds the hbq buffer, it returns the hbq_buffer
2672	* otherwise it returns NULL.
2673	**/
2674	static struct hbq_dmabuf *
2675	lpfc_sli_hbqbuf_find(struct lpfc_hba *phba, uint32_t tag)
2676	{
2677	struct lpfc_dmabuf *d_buf;
2678	struct hbq_dmabuf *hbq_buf;
2679	uint32_t hbqno;
2680
2681	hbqno = tag >> 16;
2682	if (hbqno >= LPFC_MAX_HBQS)
2683	return NULL;
2684
2685	spin_lock_irq(lock: &phba->hbalock);
2686	list_for_each_entry(d_buf, &phba->hbqs[hbqno].hbq_buffer_list, list) {
2687	hbq_buf = container_of(d_buf, struct hbq_dmabuf, dbuf);
2688	if (hbq_buf->tag == tag) {
2689	spin_unlock_irq(lock: &phba->hbalock);
2690	return hbq_buf;
2691	}
2692	}
2693	spin_unlock_irq(lock: &phba->hbalock);
2694	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2695	"1803 Bad hbq tag. Data: x%x x%x\n",
2696	tag, phba->hbqs[tag >> 16].buffer_count);
2697	return NULL;
2698	}
2699
2700	/**
2701	* lpfc_sli_free_hbq - Give back the hbq buffer to firmware
2702	* @phba: Pointer to HBA context object.
2703	* @hbq_buffer: Pointer to HBQ buffer.
2704	*
2705	* This function is called with hbalock. This function gives back
2706	* the hbq buffer to firmware. If the HBQ does not have space to
2707	* post the buffer, it will free the buffer.
2708	**/
2709	void
2710	lpfc_sli_free_hbq(struct lpfc_hba *phba, struct hbq_dmabuf *hbq_buffer)
2711	{
2712	uint32_t hbqno;
2713
2714	if (hbq_buffer) {
2715	hbqno = hbq_buffer->tag >> 16;
2716	if (lpfc_sli_hbq_to_firmware(phba, hbqno, hbq_buf: hbq_buffer))
2717	(phba->hbqs[hbqno].hbq_free_buffer)(phba, hbq_buffer);
2718	}
2719	}
2720
2721	/**
2722	* lpfc_sli_chk_mbx_command - Check if the mailbox is a legitimate mailbox
2723	* @mbxCommand: mailbox command code.
2724	*
2725	* This function is called by the mailbox event handler function to verify
2726	* that the completed mailbox command is a legitimate mailbox command. If the
2727	* completed mailbox is not known to the function, it will return MBX_SHUTDOWN
2728	* and the mailbox event handler will take the HBA offline.
2729	**/
2730	static int
2731	lpfc_sli_chk_mbx_command(uint8_t mbxCommand)
2732	{
2733	uint8_t ret;
2734
2735	switch (mbxCommand) {
2736	case MBX_LOAD_SM:
2737	case MBX_READ_NV:
2738	case MBX_WRITE_NV:
2739	case MBX_WRITE_VPARMS:
2740	case MBX_RUN_BIU_DIAG:
2741	case MBX_INIT_LINK:
2742	case MBX_DOWN_LINK:
2743	case MBX_CONFIG_LINK:
2744	case MBX_CONFIG_RING:
2745	case MBX_RESET_RING:
2746	case MBX_READ_CONFIG:
2747	case MBX_READ_RCONFIG:
2748	case MBX_READ_SPARM:
2749	case MBX_READ_STATUS:
2750	case MBX_READ_RPI:
2751	case MBX_READ_XRI:
2752	case MBX_READ_REV:
2753	case MBX_READ_LNK_STAT:
2754	case MBX_REG_LOGIN:
2755	case MBX_UNREG_LOGIN:
2756	case MBX_CLEAR_LA:
2757	case MBX_DUMP_MEMORY:
2758	case MBX_DUMP_CONTEXT:
2759	case MBX_RUN_DIAGS:
2760	case MBX_RESTART:
2761	case MBX_UPDATE_CFG:
2762	case MBX_DOWN_LOAD:
2763	case MBX_DEL_LD_ENTRY:
2764	case MBX_RUN_PROGRAM:
2765	case MBX_SET_MASK:
2766	case MBX_SET_VARIABLE:
2767	case MBX_UNREG_D_ID:
2768	case MBX_KILL_BOARD:
2769	case MBX_CONFIG_FARP:
2770	case MBX_BEACON:
2771	case MBX_LOAD_AREA:
2772	case MBX_RUN_BIU_DIAG64:
2773	case MBX_CONFIG_PORT:
2774	case MBX_READ_SPARM64:
2775	case MBX_READ_RPI64:
2776	case MBX_REG_LOGIN64:
2777	case MBX_READ_TOPOLOGY:
2778	case MBX_WRITE_WWN:
2779	case MBX_SET_DEBUG:
2780	case MBX_LOAD_EXP_ROM:
2781	case MBX_ASYNCEVT_ENABLE:
2782	case MBX_REG_VPI:
2783	case MBX_UNREG_VPI:
2784	case MBX_HEARTBEAT:
2785	case MBX_PORT_CAPABILITIES:
2786	case MBX_PORT_IOV_CONTROL:
2787	case MBX_SLI4_CONFIG:
2788	case MBX_SLI4_REQ_FTRS:
2789	case MBX_REG_FCFI:
2790	case MBX_UNREG_FCFI:
2791	case MBX_REG_VFI:
2792	case MBX_UNREG_VFI:
2793	case MBX_INIT_VPI:
2794	case MBX_INIT_VFI:
2795	case MBX_RESUME_RPI:
2796	case MBX_READ_EVENT_LOG_STATUS:
2797	case MBX_READ_EVENT_LOG:
2798	case MBX_SECURITY_MGMT:
2799	case MBX_AUTH_PORT:
2800	case MBX_ACCESS_VDATA:
2801	ret = mbxCommand;
2802	break;
2803	default:
2804	ret = MBX_SHUTDOWN;
2805	break;
2806	}
2807	return ret;
2808	}
2809
2810	/**
2811	* lpfc_sli_wake_mbox_wait - lpfc_sli_issue_mbox_wait mbox completion handler
2812	* @phba: Pointer to HBA context object.
2813	* @pmboxq: Pointer to mailbox command.
2814	*
2815	* This is completion handler function for mailbox commands issued from
2816	* lpfc_sli_issue_mbox_wait function. This function is called by the
2817	* mailbox event handler function with no lock held. This function
2818	* will wake up thread waiting on the wait queue pointed by context1
2819	* of the mailbox.
2820	**/
2821	void
2822	lpfc_sli_wake_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
2823	{
2824	unsigned long drvr_flag;
2825	struct completion *pmbox_done;
2826
2827	/*
2828	* If pmbox_done is empty, the driver thread gave up waiting and
2829	* continued running.
2830	*/
2831	pmboxq->mbox_flag |= LPFC_MBX_WAKE;
2832	spin_lock_irqsave(&phba->hbalock, drvr_flag);
2833	pmbox_done = (struct completion *)pmboxq->context3;
2834	if (pmbox_done)
2835	complete(pmbox_done);
2836	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
2837	return;
2838	}
2839
2840	static void
2841	__lpfc_sli_rpi_release(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2842	{
2843	unsigned long iflags;
2844
2845	if (ndlp->nlp_flag & NLP_RELEASE_RPI) {
2846	lpfc_sli4_free_rpi(vport->phba, ndlp->nlp_rpi);
2847	spin_lock_irqsave(&ndlp->lock, iflags);
2848	ndlp->nlp_flag &= ~NLP_RELEASE_RPI;
2849	ndlp->nlp_rpi = LPFC_RPI_ALLOC_ERROR;
2850	spin_unlock_irqrestore(lock: &ndlp->lock, flags: iflags);
2851	}
2852	ndlp->nlp_flag &= ~NLP_UNREG_INP;
2853	}
2854
2855	void
2856	lpfc_sli_rpi_release(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2857	{
2858	__lpfc_sli_rpi_release(vport, ndlp);
2859	}
2860
2861	/**
2862	* lpfc_sli_def_mbox_cmpl - Default mailbox completion handler
2863	* @phba: Pointer to HBA context object.
2864	* @pmb: Pointer to mailbox object.
2865	*
2866	* This function is the default mailbox completion handler. It
2867	* frees the memory resources associated with the completed mailbox
2868	* command. If the completed command is a REG_LOGIN mailbox command,
2869	* this function will issue a UREG_LOGIN to re-claim the RPI.
2870	**/
2871	void
2872	lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
2873	{
2874	struct lpfc_vport *vport = pmb->vport;
2875	struct lpfc_dmabuf *mp;
2876	struct lpfc_nodelist *ndlp;
2877	struct Scsi_Host *shost;
2878	uint16_t rpi, vpi;
2879	int rc;
2880
2881	/*
2882	* If a REG_LOGIN succeeded after node is destroyed or node
2883	* is in re-discovery driver need to cleanup the RPI.
2884	*/
2885	if (!(phba->pport->load_flag & FC_UNLOADING) &&
2886	pmb->u.mb.mbxCommand == MBX_REG_LOGIN64 &&
2887	!pmb->u.mb.mbxStatus) {
2888	mp = (struct lpfc_dmabuf *)pmb->ctx_buf;
2889	if (mp) {
2890	pmb->ctx_buf = NULL;
2891	lpfc_mbuf_free(phba, mp->virt, mp->phys);
2892	kfree(objp: mp);
2893	}
2894	rpi = pmb->u.mb.un.varWords[0];
2895	vpi = pmb->u.mb.un.varRegLogin.vpi;
2896	if (phba->sli_rev == LPFC_SLI_REV4)
2897	vpi -= phba->sli4_hba.max_cfg_param.vpi_base;
2898	lpfc_unreg_login(phba, vpi, rpi, pmb);
2899	pmb->vport = vport;
2900	pmb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
2901	rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
2902	if (rc != MBX_NOT_FINISHED)
2903	return;
2904	}
2905
2906	if ((pmb->u.mb.mbxCommand == MBX_REG_VPI) &&
2907	!(phba->pport->load_flag & FC_UNLOADING) &&
2908	!pmb->u.mb.mbxStatus) {
2909	shost = lpfc_shost_from_vport(vport);
2910	spin_lock_irq(lock: shost->host_lock);
2911	vport->vpi_state |= LPFC_VPI_REGISTERED;
2912	vport->fc_flag &= ~FC_VPORT_NEEDS_REG_VPI;
2913	spin_unlock_irq(lock: shost->host_lock);
2914	}
2915
2916	if (pmb->u.mb.mbxCommand == MBX_REG_LOGIN64) {
2917	ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp;
2918	lpfc_nlp_put(ndlp);
2919	}
2920
2921	if (pmb->u.mb.mbxCommand == MBX_UNREG_LOGIN) {
2922	ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp;
2923
2924	/* Check to see if there are any deferred events to process */
2925	if (ndlp) {
2926	lpfc_printf_vlog(
2927	vport,
2928	KERN_INFO, LOG_MBOX | LOG_DISCOVERY,
2929	"1438 UNREG cmpl deferred mbox x%x "
2930	"on NPort x%x Data: x%x x%x x%px x%x x%x\n",
2931	ndlp->nlp_rpi, ndlp->nlp_DID,
2932	ndlp->nlp_flag, ndlp->nlp_defer_did,
2933	ndlp, vport->load_flag, kref_read(&ndlp->kref));
2934
2935	if ((ndlp->nlp_flag & NLP_UNREG_INP) &&
2936	(ndlp->nlp_defer_did != NLP_EVT_NOTHING_PENDING)) {
2937	ndlp->nlp_flag &= ~NLP_UNREG_INP;
2938	ndlp->nlp_defer_did = NLP_EVT_NOTHING_PENDING;
2939	lpfc_issue_els_plogi(vport, ndlp->nlp_DID, 0);
2940	} else {
2941	__lpfc_sli_rpi_release(vport, ndlp);
2942	}
2943
2944	/* The unreg_login mailbox is complete and had a
2945	* reference that has to be released. The PLOGI
2946	* got its own ref.
2947	*/
2948	lpfc_nlp_put(ndlp);
2949	pmb->ctx_ndlp = NULL;
2950	}
2951	}
2952
2953	/* This nlp_put pairs with lpfc_sli4_resume_rpi */
2954	if (pmb->u.mb.mbxCommand == MBX_RESUME_RPI) {
2955	ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp;
2956	lpfc_nlp_put(ndlp);
2957	}
2958
2959	/* Check security permission status on INIT_LINK mailbox command */
2960	if ((pmb->u.mb.mbxCommand == MBX_INIT_LINK) &&
2961	(pmb->u.mb.mbxStatus == MBXERR_SEC_NO_PERMISSION))
2962	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2963	"2860 SLI authentication is required "
2964	"for INIT_LINK but has not done yet\n");
2965
2966	if (bf_get(lpfc_mqe_command, &pmb->u.mqe) == MBX_SLI4_CONFIG)
2967	lpfc_sli4_mbox_cmd_free(phba, pmb);
2968	else
2969	lpfc_mbox_rsrc_cleanup(phba, mbox: pmb, locked: MBOX_THD_UNLOCKED);
2970	}
2971	/**
2972	* lpfc_sli4_unreg_rpi_cmpl_clr - mailbox completion handler
2973	* @phba: Pointer to HBA context object.
2974	* @pmb: Pointer to mailbox object.
2975	*
2976	* This function is the unreg rpi mailbox completion handler. It
2977	* frees the memory resources associated with the completed mailbox
2978	* command. An additional reference is put on the ndlp to prevent
2979	* lpfc_nlp_release from freeing the rpi bit in the bitmask before
2980	* the unreg mailbox command completes, this routine puts the
2981	* reference back.
2982	*
2983	**/
2984	void
2985	lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
2986	{
2987	struct lpfc_vport *vport = pmb->vport;
2988	struct lpfc_nodelist *ndlp;
2989
2990	ndlp = pmb->ctx_ndlp;
2991	if (pmb->u.mb.mbxCommand == MBX_UNREG_LOGIN) {
2992	if (phba->sli_rev == LPFC_SLI_REV4 &&
2993	(bf_get(lpfc_sli_intf_if_type,
2994	&phba->sli4_hba.sli_intf) >=
2995	LPFC_SLI_INTF_IF_TYPE_2)) {
2996	if (ndlp) {
2997	lpfc_printf_vlog(
2998	vport, KERN_INFO, LOG_MBOX | LOG_SLI,
2999	"0010 UNREG_LOGIN vpi:%x "
3000	"rpi:%x DID:%x defer x%x flg x%x "
3001	"x%px\n",
3002	vport->vpi, ndlp->nlp_rpi,
3003	ndlp->nlp_DID, ndlp->nlp_defer_did,
3004	ndlp->nlp_flag,
3005	ndlp);
3006	ndlp->nlp_flag &= ~NLP_LOGO_ACC;
3007
3008	/* Check to see if there are any deferred
3009	* events to process
3010	*/
3011	if ((ndlp->nlp_flag & NLP_UNREG_INP) &&
3012	(ndlp->nlp_defer_did !=
3013	NLP_EVT_NOTHING_PENDING)) {
3014	lpfc_printf_vlog(
3015	vport, KERN_INFO, LOG_DISCOVERY,
3016	"4111 UNREG cmpl deferred "
3017	"clr x%x on "
3018	"NPort x%x Data: x%x x%px\n",
3019	ndlp->nlp_rpi, ndlp->nlp_DID,
3020	ndlp->nlp_defer_did, ndlp);
3021	ndlp->nlp_flag &= ~NLP_UNREG_INP;
3022	ndlp->nlp_defer_did =
3023	NLP_EVT_NOTHING_PENDING;
3024	lpfc_issue_els_plogi(
3025	vport, ndlp->nlp_DID, 0);
3026	} else {
3027	__lpfc_sli_rpi_release(vport, ndlp);
3028	}
3029	lpfc_nlp_put(ndlp);
3030	}
3031	}
3032	}
3033
3034	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
3035	}
3036
3037	/**
3038	* lpfc_sli_handle_mb_event - Handle mailbox completions from firmware
3039	* @phba: Pointer to HBA context object.
3040	*
3041	* This function is called with no lock held. This function processes all
3042	* the completed mailbox commands and gives it to upper layers. The interrupt
3043	* service routine processes mailbox completion interrupt and adds completed
3044	* mailbox commands to the mboxq_cmpl queue and signals the worker thread.
3045	* Worker thread call lpfc_sli_handle_mb_event, which will return the
3046	* completed mailbox commands in mboxq_cmpl queue to the upper layers. This
3047	* function returns the mailbox commands to the upper layer by calling the
3048	* completion handler function of each mailbox.
3049	**/
3050	int
3051	lpfc_sli_handle_mb_event(struct lpfc_hba *phba)
3052	{
3053	MAILBOX_t *pmbox;
3054	LPFC_MBOXQ_t *pmb;
3055	int rc;
3056	LIST_HEAD(cmplq);
3057
3058	phba->sli.slistat.mbox_event++;
3059
3060	/* Get all completed mailboxe buffers into the cmplq */
3061	spin_lock_irq(lock: &phba->hbalock);
3062	list_splice_init(list: &phba->sli.mboxq_cmpl, head: &cmplq);
3063	spin_unlock_irq(lock: &phba->hbalock);
3064
3065	/* Get a Mailbox buffer to setup mailbox commands for callback */
3066	do {
3067	list_remove_head(&cmplq, pmb, LPFC_MBOXQ_t, list);
3068	if (pmb == NULL)
3069	break;
3070
3071	pmbox = &pmb->u.mb;
3072
3073	if (pmbox->mbxCommand != MBX_HEARTBEAT) {
3074	if (pmb->vport) {
3075	lpfc_debugfs_disc_trc(pmb->vport,
3076	LPFC_DISC_TRC_MBOX_VPORT,
3077	"MBOX cmpl vport: cmd:x%x mb:x%x x%x",
3078	(uint32_t)pmbox->mbxCommand,
3079	pmbox->un.varWords[0],
3080	pmbox->un.varWords[1]);
3081	}
3082	else {
3083	lpfc_debugfs_disc_trc(phba->pport,
3084	LPFC_DISC_TRC_MBOX,
3085	"MBOX cmpl: cmd:x%x mb:x%x x%x",
3086	(uint32_t)pmbox->mbxCommand,
3087	pmbox->un.varWords[0],
3088	pmbox->un.varWords[1]);
3089	}
3090	}
3091
3092	/*
3093	* It is a fatal error if unknown mbox command completion.
3094	*/
3095	if (lpfc_sli_chk_mbx_command(mbxCommand: pmbox->mbxCommand) ==
3096	MBX_SHUTDOWN) {
3097	/* Unknown mailbox command compl */
3098	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3099	"(%d):0323 Unknown Mailbox command "
3100	"x%x (x%x/x%x) Cmpl\n",
3101	pmb->vport ? pmb->vport->vpi :
3102	LPFC_VPORT_UNKNOWN,
3103	pmbox->mbxCommand,
3104	lpfc_sli_config_mbox_subsys_get(phba,
3105	pmb),
3106	lpfc_sli_config_mbox_opcode_get(phba,
3107	pmb));
3108	phba->link_state = LPFC_HBA_ERROR;
3109	phba->work_hs = HS_FFER3;
3110	lpfc_handle_eratt(phba);
3111	continue;
3112	}
3113
3114	if (pmbox->mbxStatus) {
3115	phba->sli.slistat.mbox_stat_err++;
3116	if (pmbox->mbxStatus == MBXERR_NO_RESOURCES) {
3117	/* Mbox cmd cmpl error - RETRYing */
3118	lpfc_printf_log(phba, KERN_INFO,
3119	LOG_MBOX | LOG_SLI,
3120	"(%d):0305 Mbox cmd cmpl "
3121	"error - RETRYing Data: x%x "
3122	"(x%x/x%x) x%x x%x x%x\n",
3123	pmb->vport ? pmb->vport->vpi :
3124	LPFC_VPORT_UNKNOWN,
3125	pmbox->mbxCommand,
3126	lpfc_sli_config_mbox_subsys_get(phba,
3127	pmb),
3128	lpfc_sli_config_mbox_opcode_get(phba,
3129	pmb),
3130	pmbox->mbxStatus,
3131	pmbox->un.varWords[0],
3132	pmb->vport ? pmb->vport->port_state :
3133	LPFC_VPORT_UNKNOWN);
3134	pmbox->mbxStatus = 0;
3135	pmbox->mbxOwner = OWN_HOST;
3136	rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
3137	if (rc != MBX_NOT_FINISHED)
3138	continue;
3139	}
3140	}
3141
3142	/* Mailbox cmd <cmd> Cmpl <cmpl> */
3143	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
3144	"(%d):0307 Mailbox cmd x%x (x%x/x%x) Cmpl %ps "
3145	"Data: x%x x%x x%x x%x x%x x%x x%x x%x x%x "
3146	"x%x x%x x%x\n",
3147	pmb->vport ? pmb->vport->vpi : 0,
3148	pmbox->mbxCommand,
3149	lpfc_sli_config_mbox_subsys_get(phba, pmb),
3150	lpfc_sli_config_mbox_opcode_get(phba, pmb),
3151	pmb->mbox_cmpl,
3152	*((uint32_t *) pmbox),
3153	pmbox->un.varWords[0],
3154	pmbox->un.varWords[1],
3155	pmbox->un.varWords[2],
3156	pmbox->un.varWords[3],
3157	pmbox->un.varWords[4],
3158	pmbox->un.varWords[5],
3159	pmbox->un.varWords[6],
3160	pmbox->un.varWords[7],
3161	pmbox->un.varWords[8],
3162	pmbox->un.varWords[9],
3163	pmbox->un.varWords[10]);
3164
3165	if (pmb->mbox_cmpl)
3166	pmb->mbox_cmpl(phba,pmb);
3167	} while (1);
3168	return 0;
3169	}
3170
3171	/**
3172	* lpfc_sli_get_buff - Get the buffer associated with the buffer tag
3173	* @phba: Pointer to HBA context object.
3174	* @pring: Pointer to driver SLI ring object.
3175	* @tag: buffer tag.
3176	*
3177	* This function is called with no lock held. When QUE_BUFTAG_BIT bit
3178	* is set in the tag the buffer is posted for a particular exchange,
3179	* the function will return the buffer without replacing the buffer.
3180	* If the buffer is for unsolicited ELS or CT traffic, this function
3181	* returns the buffer and also posts another buffer to the firmware.
3182	**/
3183	static struct lpfc_dmabuf *
3184	lpfc_sli_get_buff(struct lpfc_hba *phba,
3185	struct lpfc_sli_ring *pring,
3186	uint32_t tag)
3187	{
3188	struct hbq_dmabuf *hbq_entry;
3189
3190	if (tag & QUE_BUFTAG_BIT)
3191	return lpfc_sli_ring_taggedbuf_get(phba, pring, tag);
3192	hbq_entry = lpfc_sli_hbqbuf_find(phba, tag);
3193	if (!hbq_entry)
3194	return NULL;
3195	return &hbq_entry->dbuf;
3196	}
3197
3198	/**
3199	* lpfc_nvme_unsol_ls_handler - Process an unsolicited event data buffer
3200	* containing a NVME LS request.
3201	* @phba: pointer to lpfc hba data structure.
3202	* @piocb: pointer to the iocbq struct representing the sequence starting
3203	* frame.
3204	*
3205	* This routine initially validates the NVME LS, validates there is a login
3206	* with the port that sent the LS, and then calls the appropriate nvme host
3207	* or target LS request handler.
3208	**/
3209	static void
3210	lpfc_nvme_unsol_ls_handler(struct lpfc_hba *phba, struct lpfc_iocbq *piocb)
3211	{
3212	struct lpfc_nodelist *ndlp;
3213	struct lpfc_dmabuf *d_buf;
3214	struct hbq_dmabuf *nvmebuf;
3215	struct fc_frame_header *fc_hdr;
3216	struct lpfc_async_xchg_ctx *axchg = NULL;
3217	char *failwhy = NULL;
3218	uint32_t oxid, sid, did, fctl, size;
3219	int ret = 1;
3220
3221	d_buf = piocb->cmd_dmabuf;
3222
3223	nvmebuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
3224	fc_hdr = nvmebuf->hbuf.virt;
3225	oxid = be16_to_cpu(fc_hdr->fh_ox_id);
3226	sid = sli4_sid_from_fc_hdr(fc_hdr);
3227	did = sli4_did_from_fc_hdr(fc_hdr);
3228	fctl = (fc_hdr->fh_f_ctl[0] << 16 |
3229	fc_hdr->fh_f_ctl[1] << 8 |
3230	fc_hdr->fh_f_ctl[2]);
3231	size = bf_get(lpfc_rcqe_length, &nvmebuf->cq_event.cqe.rcqe_cmpl);
3232
3233	lpfc_nvmeio_data(phba, "NVME LS RCV: xri x%x sz %d from %06x\n",
3234	oxid, size, sid);
3235
3236	if (phba->pport->load_flag & FC_UNLOADING) {
3237	failwhy = "Driver Unloading";
3238	} else if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) {
3239	failwhy = "NVME FC4 Disabled";
3240	} else if (!phba->nvmet_support && !phba->pport->localport) {
3241	failwhy = "No Localport";
3242	} else if (phba->nvmet_support && !phba->targetport) {
3243	failwhy = "No Targetport";
3244	} else if (unlikely(fc_hdr->fh_r_ctl != FC_RCTL_ELS4_REQ)) {
3245	failwhy = "Bad NVME LS R_CTL";
3246	} else if (unlikely((fctl & 0x00FF0000) !=
3247	(FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT))) {
3248	failwhy = "Bad NVME LS F_CTL";
3249	} else {
3250	axchg = kzalloc(size: sizeof(*axchg), GFP_ATOMIC);
3251	if (!axchg)
3252	failwhy = "No CTX memory";
3253	}
3254
3255	if (unlikely(failwhy)) {
3256	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3257	"6154 Drop NVME LS: SID %06X OXID x%X: %s\n",
3258	sid, oxid, failwhy);
3259	goto out_fail;
3260	}
3261
3262	/* validate the source of the LS is logged in */
3263	ndlp = lpfc_findnode_did(phba->pport, sid);
3264	if (!ndlp ||
3265	((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
3266	(ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
3267	lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
3268	"6216 NVME Unsol rcv: No ndlp: "
3269	"NPort_ID x%x oxid x%x\n",
3270	sid, oxid);
3271	goto out_fail;
3272	}
3273
3274	axchg->phba = phba;
3275	axchg->ndlp = ndlp;
3276	axchg->size = size;
3277	axchg->oxid = oxid;
3278	axchg->sid = sid;
3279	axchg->wqeq = NULL;
3280	axchg->state = LPFC_NVME_STE_LS_RCV;
3281	axchg->entry_cnt = 1;
3282	axchg->rqb_buffer = (void *)nvmebuf;
3283	axchg->hdwq = &phba->sli4_hba.hdwq[0];
3284	axchg->payload = nvmebuf->dbuf.virt;
3285	INIT_LIST_HEAD(list: &axchg->list);
3286
3287	if (phba->nvmet_support) {
3288	ret = lpfc_nvmet_handle_lsreq(phba, axchg);
3289	spin_lock_irq(lock: &ndlp->lock);
3290	if (!ret && !(ndlp->fc4_xpt_flags & NLP_XPT_HAS_HH)) {
3291	ndlp->fc4_xpt_flags |= NLP_XPT_HAS_HH;
3292	spin_unlock_irq(lock: &ndlp->lock);
3293
3294	/* This reference is a single occurrence to hold the
3295	* node valid until the nvmet transport calls
3296	* host_release.
3297	*/
3298	if (!lpfc_nlp_get(ndlp))
3299	goto out_fail;
3300
3301	lpfc_printf_log(phba, KERN_ERR, LOG_NODE,
3302	"6206 NVMET unsol ls_req ndlp x%px "
3303	"DID x%x xflags x%x refcnt %d\n",
3304	ndlp, ndlp->nlp_DID,
3305	ndlp->fc4_xpt_flags,
3306	kref_read(&ndlp->kref));
3307	} else {
3308	spin_unlock_irq(lock: &ndlp->lock);
3309	}
3310	} else {
3311	ret = lpfc_nvme_handle_lsreq(phba, axchg);
3312	}
3313
3314	/* if zero, LS was successfully handled. If non-zero, LS not handled */
3315	if (!ret)
3316	return;
3317
3318	out_fail:
3319	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3320	"6155 Drop NVME LS from DID %06X: SID %06X OXID x%X "
3321	"NVMe%s handler failed %d\n",
3322	did, sid, oxid,
3323	(phba->nvmet_support) ? "T" : "I", ret);
3324
3325	/* recycle receive buffer */
3326	lpfc_in_buf_free(phba, &nvmebuf->dbuf);
3327
3328	/* If start of new exchange, abort it */
3329	if (axchg && (fctl & FC_FC_FIRST_SEQ && !(fctl & FC_FC_EX_CTX)))
3330	ret = lpfc_nvme_unsol_ls_issue_abort(phba, ctxp: axchg, sid, xri: oxid);
3331
3332	if (ret)
3333	kfree(objp: axchg);
3334	}
3335
3336	/**
3337	* lpfc_complete_unsol_iocb - Complete an unsolicited sequence
3338	* @phba: Pointer to HBA context object.
3339	* @pring: Pointer to driver SLI ring object.
3340	* @saveq: Pointer to the iocbq struct representing the sequence starting frame.
3341	* @fch_r_ctl: the r_ctl for the first frame of the sequence.
3342	* @fch_type: the type for the first frame of the sequence.
3343	*
3344	* This function is called with no lock held. This function uses the r_ctl and
3345	* type of the received sequence to find the correct callback function to call
3346	* to process the sequence.
3347	**/
3348	static int
3349	lpfc_complete_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
3350	struct lpfc_iocbq *saveq, uint32_t fch_r_ctl,
3351	uint32_t fch_type)
3352	{
3353	int i;
3354
3355	switch (fch_type) {
3356	case FC_TYPE_NVME:
3357	lpfc_nvme_unsol_ls_handler(phba, piocb: saveq);
3358	return 1;
3359	default:
3360	break;
3361	}
3362
3363	/* unSolicited Responses */
3364	if (pring->prt[0].profile) {
3365	if (pring->prt[0].lpfc_sli_rcv_unsol_event)
3366	(pring->prt[0].lpfc_sli_rcv_unsol_event) (phba, pring,
3367	saveq);
3368	return 1;
3369	}
3370	/* We must search, based on rctl / type
3371	for the right routine */
3372	for (i = 0; i < pring->num_mask; i++) {
3373	if ((pring->prt[i].rctl == fch_r_ctl) &&
3374	(pring->prt[i].type == fch_type)) {
3375	if (pring->prt[i].lpfc_sli_rcv_unsol_event)
3376	(pring->prt[i].lpfc_sli_rcv_unsol_event)
3377	(phba, pring, saveq);
3378	return 1;
3379	}
3380	}
3381	return 0;
3382	}
3383
3384	static void
3385	lpfc_sli_prep_unsol_wqe(struct lpfc_hba *phba,
3386	struct lpfc_iocbq *saveq)
3387	{
3388	IOCB_t *irsp;
3389	union lpfc_wqe128 *wqe;
3390	u16 i = 0;
3391
3392	irsp = &saveq->iocb;
3393	wqe = &saveq->wqe;
3394
3395	/* Fill wcqe with the IOCB status fields */
3396	bf_set(lpfc_wcqe_c_status, &saveq->wcqe_cmpl, irsp->ulpStatus);
3397	saveq->wcqe_cmpl.word3 = irsp->ulpBdeCount;
3398	saveq->wcqe_cmpl.parameter = irsp->un.ulpWord[4];
3399	saveq->wcqe_cmpl.total_data_placed = irsp->unsli3.rcvsli3.acc_len;
3400
3401	/* Source ID */
3402	bf_set(els_rsp64_sid, &wqe->xmit_els_rsp, irsp->un.rcvels.parmRo);
3403
3404	/* rx-id of the response frame */
3405	bf_set(wqe_ctxt_tag, &wqe->xmit_els_rsp.wqe_com, irsp->ulpContext);
3406
3407	/* ox-id of the frame */
3408	bf_set(wqe_rcvoxid, &wqe->xmit_els_rsp.wqe_com,
3409	irsp->unsli3.rcvsli3.ox_id);
3410
3411	/* DID */
3412	bf_set(wqe_els_did, &wqe->xmit_els_rsp.wqe_dest,
3413	irsp->un.rcvels.remoteID);
3414
3415	/* unsol data len */
3416	for (i = 0; i < irsp->ulpBdeCount; i++) {
3417	struct lpfc_hbq_entry *hbqe = NULL;
3418
3419	if (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED) {
3420	if (i == 0) {
3421	hbqe = (struct lpfc_hbq_entry *)
3422	&irsp->un.ulpWord[0];
3423	saveq->wqe.gen_req.bde.tus.f.bdeSize =
3424	hbqe->bde.tus.f.bdeSize;
3425	} else if (i == 1) {
3426	hbqe = (struct lpfc_hbq_entry *)
3427	&irsp->unsli3.sli3Words[4];
3428	saveq->unsol_rcv_len = hbqe->bde.tus.f.bdeSize;
3429	}
3430	}
3431	}
3432	}
3433
3434	/**
3435	* lpfc_sli_process_unsol_iocb - Unsolicited iocb handler
3436	* @phba: Pointer to HBA context object.
3437	* @pring: Pointer to driver SLI ring object.
3438	* @saveq: Pointer to the unsolicited iocb.
3439	*
3440	* This function is called with no lock held by the ring event handler
3441	* when there is an unsolicited iocb posted to the response ring by the
3442	* firmware. This function gets the buffer associated with the iocbs
3443	* and calls the event handler for the ring. This function handles both
3444	* qring buffers and hbq buffers.
3445	* When the function returns 1 the caller can free the iocb object otherwise
3446	* upper layer functions will free the iocb objects.
3447	**/
3448	static int
3449	lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
3450	struct lpfc_iocbq *saveq)
3451	{
3452	IOCB_t * irsp;
3453	WORD5 * w5p;
3454	dma_addr_t paddr;
3455	uint32_t Rctl, Type;
3456	struct lpfc_iocbq *iocbq;
3457	struct lpfc_dmabuf *dmzbuf;
3458
3459	irsp = &saveq->iocb;
3460	saveq->vport = phba->pport;
3461
3462	if (irsp->ulpCommand == CMD_ASYNC_STATUS) {
3463	if (pring->lpfc_sli_rcv_async_status)
3464	pring->lpfc_sli_rcv_async_status(phba, pring, saveq);
3465	else
3466	lpfc_printf_log(phba,
3467	KERN_WARNING,
3468	LOG_SLI,
3469	"0316 Ring %d handler: unexpected "
3470	"ASYNC_STATUS iocb received evt_code "
3471	"0x%x\n",
3472	pring->ringno,
3473	irsp->un.asyncstat.evt_code);
3474	return 1;
3475	}
3476
3477	if ((irsp->ulpCommand == CMD_IOCB_RET_XRI64_CX) &&
3478	(phba->sli3_options & LPFC_SLI3_HBQ_ENABLED)) {
3479	if (irsp->ulpBdeCount > 0) {
3480	dmzbuf = lpfc_sli_get_buff(phba, pring,
3481	tag: irsp->un.ulpWord[3]);
3482	lpfc_in_buf_free(phba, dmzbuf);
3483	}
3484
3485	if (irsp->ulpBdeCount > 1) {
3486	dmzbuf = lpfc_sli_get_buff(phba, pring,
3487	tag: irsp->unsli3.sli3Words[3]);
3488	lpfc_in_buf_free(phba, dmzbuf);
3489	}
3490
3491	if (irsp->ulpBdeCount > 2) {
3492	dmzbuf = lpfc_sli_get_buff(phba, pring,
3493	tag: irsp->unsli3.sli3Words[7]);
3494	lpfc_in_buf_free(phba, dmzbuf);
3495	}
3496
3497	return 1;
3498	}
3499
3500	if (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED) {
3501	if (irsp->ulpBdeCount != 0) {
3502	saveq->cmd_dmabuf = lpfc_sli_get_buff(phba, pring,
3503	tag: irsp->un.ulpWord[3]);
3504	if (!saveq->cmd_dmabuf)
3505	lpfc_printf_log(phba,
3506	KERN_ERR,
3507	LOG_SLI,
3508	"0341 Ring %d Cannot find buffer for "
3509	"an unsolicited iocb. tag 0x%x\n",
3510	pring->ringno,
3511	irsp->un.ulpWord[3]);
3512	}
3513	if (irsp->ulpBdeCount == 2) {
3514	saveq->bpl_dmabuf = lpfc_sli_get_buff(phba, pring,
3515	tag: irsp->unsli3.sli3Words[7]);
3516	if (!saveq->bpl_dmabuf)
3517	lpfc_printf_log(phba,
3518	KERN_ERR,
3519	LOG_SLI,
3520	"0342 Ring %d Cannot find buffer for an"
3521	" unsolicited iocb. tag 0x%x\n",
3522	pring->ringno,
3523	irsp->unsli3.sli3Words[7]);
3524	}
3525	list_for_each_entry(iocbq, &saveq->list, list) {
3526	irsp = &iocbq->iocb;
3527	if (irsp->ulpBdeCount != 0) {
3528	iocbq->cmd_dmabuf = lpfc_sli_get_buff(phba,
3529	pring,
3530	tag: irsp->un.ulpWord[3]);
3531	if (!iocbq->cmd_dmabuf)
3532	lpfc_printf_log(phba,
3533	KERN_ERR,
3534	LOG_SLI,
3535	"0343 Ring %d Cannot find "
3536	"buffer for an unsolicited iocb"
3537	". tag 0x%x\n", pring->ringno,
3538	irsp->un.ulpWord[3]);
3539	}
3540	if (irsp->ulpBdeCount == 2) {
3541	iocbq->bpl_dmabuf = lpfc_sli_get_buff(phba,
3542	pring,
3543	tag: irsp->unsli3.sli3Words[7]);
3544	if (!iocbq->bpl_dmabuf)
3545	lpfc_printf_log(phba,
3546	KERN_ERR,
3547	LOG_SLI,
3548	"0344 Ring %d Cannot find "
3549	"buffer for an unsolicited "
3550	"iocb. tag 0x%x\n",
3551	pring->ringno,
3552	irsp->unsli3.sli3Words[7]);
3553	}
3554	}
3555	} else {
3556	paddr = getPaddr(irsp->un.cont64[0].addrHigh,
3557	irsp->un.cont64[0].addrLow);
3558	saveq->cmd_dmabuf = lpfc_sli_ringpostbuf_get(phba, pring,
3559	paddr);
3560	if (irsp->ulpBdeCount == 2) {
3561	paddr = getPaddr(irsp->un.cont64[1].addrHigh,
3562	irsp->un.cont64[1].addrLow);
3563	saveq->bpl_dmabuf = lpfc_sli_ringpostbuf_get(phba,
3564	pring,
3565	paddr);
3566	}
3567	}
3568
3569	if (irsp->ulpBdeCount != 0 &&
3570	(irsp->ulpCommand == CMD_IOCB_RCV_CONT64_CX ||
3571	irsp->ulpStatus == IOSTAT_INTERMED_RSP)) {
3572	int found = 0;
3573
3574	/* search continue save q for same XRI */
3575	list_for_each_entry(iocbq, &pring->iocb_continue_saveq, clist) {
3576	if (iocbq->iocb.unsli3.rcvsli3.ox_id ==
3577	saveq->iocb.unsli3.rcvsli3.ox_id) {
3578	list_add_tail(new: &saveq->list, head: &iocbq->list);
3579	found = 1;
3580	break;
3581	}
3582	}
3583	if (!found)
3584	list_add_tail(new: &saveq->clist,
3585	head: &pring->iocb_continue_saveq);
3586
3587	if (saveq->iocb.ulpStatus != IOSTAT_INTERMED_RSP) {
3588	list_del_init(entry: &iocbq->clist);
3589	saveq = iocbq;
3590	irsp = &saveq->iocb;
3591	} else {
3592	return 0;
3593	}
3594	}
3595	if ((irsp->ulpCommand == CMD_RCV_ELS_REQ64_CX) ||
3596	(irsp->ulpCommand == CMD_RCV_ELS_REQ_CX) ||
3597	(irsp->ulpCommand == CMD_IOCB_RCV_ELS64_CX)) {
3598	Rctl = FC_RCTL_ELS_REQ;
3599	Type = FC_TYPE_ELS;
3600	} else {
3601	w5p = (WORD5 *)&(saveq->iocb.un.ulpWord[5]);
3602	Rctl = w5p->hcsw.Rctl;
3603	Type = w5p->hcsw.Type;
3604
3605	/* Firmware Workaround */
3606	if ((Rctl == 0) && (pring->ringno == LPFC_ELS_RING) &&
3607	(irsp->ulpCommand == CMD_RCV_SEQUENCE64_CX ||
3608	irsp->ulpCommand == CMD_IOCB_RCV_SEQ64_CX)) {
3609	Rctl = FC_RCTL_ELS_REQ;
3610	Type = FC_TYPE_ELS;
3611	w5p->hcsw.Rctl = Rctl;
3612	w5p->hcsw.Type = Type;
3613	}
3614	}
3615
3616	if ((phba->sli3_options & LPFC_SLI3_NPIV_ENABLED) &&
3617	(irsp->ulpCommand == CMD_IOCB_RCV_ELS64_CX ||
3618	irsp->ulpCommand == CMD_IOCB_RCV_SEQ64_CX)) {
3619	if (irsp->unsli3.rcvsli3.vpi == 0xffff)
3620	saveq->vport = phba->pport;
3621	else
3622	saveq->vport = lpfc_find_vport_by_vpid(phba,
3623	irsp->unsli3.rcvsli3.vpi);
3624	}
3625
3626	/* Prepare WQE with Unsol frame */
3627	lpfc_sli_prep_unsol_wqe(phba, saveq);
3628
3629	if (!lpfc_complete_unsol_iocb(phba, pring, saveq, fch_r_ctl: Rctl, fch_type: Type))
3630	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
3631	"0313 Ring %d handler: unexpected Rctl x%x "
3632	"Type x%x received\n",
3633	pring->ringno, Rctl, Type);
3634
3635	return 1;
3636	}
3637
3638	/**
3639	* lpfc_sli_iocbq_lookup - Find command iocb for the given response iocb
3640	* @phba: Pointer to HBA context object.
3641	* @pring: Pointer to driver SLI ring object.
3642	* @prspiocb: Pointer to response iocb object.
3643	*
3644	* This function looks up the iocb_lookup table to get the command iocb
3645	* corresponding to the given response iocb using the iotag of the
3646	* response iocb. The driver calls this function with the hbalock held
3647	* for SLI3 ports or the ring lock held for SLI4 ports.
3648	* This function returns the command iocb object if it finds the command
3649	* iocb else returns NULL.
3650	**/
3651	static struct lpfc_iocbq *
3652	lpfc_sli_iocbq_lookup(struct lpfc_hba *phba,
3653	struct lpfc_sli_ring *pring,
3654	struct lpfc_iocbq *prspiocb)
3655	{
3656	struct lpfc_iocbq *cmd_iocb = NULL;
3657	u16 iotag;
3658
3659	if (phba->sli_rev == LPFC_SLI_REV4)
3660	iotag = get_wqe_reqtag(prspiocb);
3661	else
3662	iotag = prspiocb->iocb.ulpIoTag;
3663
3664	if (iotag != 0 && iotag <= phba->sli.last_iotag) {
3665	cmd_iocb = phba->sli.iocbq_lookup[iotag];
3666	if (cmd_iocb->cmd_flag & LPFC_IO_ON_TXCMPLQ) {
3667	/* remove from txcmpl queue list */
3668	list_del_init(entry: &cmd_iocb->list);
3669	cmd_iocb->cmd_flag &= ~LPFC_IO_ON_TXCMPLQ;
3670	pring->txcmplq_cnt--;
3671	return cmd_iocb;
3672	}
3673	}
3674
3675	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3676	"0317 iotag x%x is out of "
3677	"range: max iotag x%x\n",
3678	iotag, phba->sli.last_iotag);
3679	return NULL;
3680	}
3681
3682	/**
3683	* lpfc_sli_iocbq_lookup_by_tag - Find command iocb for the iotag
3684	* @phba: Pointer to HBA context object.
3685	* @pring: Pointer to driver SLI ring object.
3686	* @iotag: IOCB tag.
3687	*
3688	* This function looks up the iocb_lookup table to get the command iocb
3689	* corresponding to the given iotag. The driver calls this function with
3690	* the ring lock held because this function is an SLI4 port only helper.
3691	* This function returns the command iocb object if it finds the command
3692	* iocb else returns NULL.
3693	**/
3694	static struct lpfc_iocbq *
3695	lpfc_sli_iocbq_lookup_by_tag(struct lpfc_hba *phba,
3696	struct lpfc_sli_ring *pring, uint16_t iotag)
3697	{
3698	struct lpfc_iocbq *cmd_iocb = NULL;
3699
3700	if (iotag != 0 && iotag <= phba->sli.last_iotag) {
3701	cmd_iocb = phba->sli.iocbq_lookup[iotag];
3702	if (cmd_iocb->cmd_flag & LPFC_IO_ON_TXCMPLQ) {
3703	/* remove from txcmpl queue list */
3704	list_del_init(entry: &cmd_iocb->list);
3705	cmd_iocb->cmd_flag &= ~LPFC_IO_ON_TXCMPLQ;
3706	pring->txcmplq_cnt--;
3707	return cmd_iocb;
3708	}
3709	}
3710
3711	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3712	"0372 iotag x%x lookup error: max iotag (x%x) "
3713	"cmd_flag x%x\n",
3714	iotag, phba->sli.last_iotag,
3715	cmd_iocb ? cmd_iocb->cmd_flag : 0xffff);
3716	return NULL;
3717	}
3718
3719	/**
3720	* lpfc_sli_process_sol_iocb - process solicited iocb completion
3721	* @phba: Pointer to HBA context object.
3722	* @pring: Pointer to driver SLI ring object.
3723	* @saveq: Pointer to the response iocb to be processed.
3724	*
3725	* This function is called by the ring event handler for non-fcp
3726	* rings when there is a new response iocb in the response ring.
3727	* The caller is not required to hold any locks. This function
3728	* gets the command iocb associated with the response iocb and
3729	* calls the completion handler for the command iocb. If there
3730	* is no completion handler, the function will free the resources
3731	* associated with command iocb. If the response iocb is for
3732	* an already aborted command iocb, the status of the completion
3733	* is changed to IOSTAT_LOCAL_REJECT/IOERR_SLI_ABORTED.
3734	* This function always returns 1.
3735	**/
3736	static int
3737	lpfc_sli_process_sol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
3738	struct lpfc_iocbq *saveq)
3739	{
3740	struct lpfc_iocbq *cmdiocbp;
3741	unsigned long iflag;
3742	u32 ulp_command, ulp_status, ulp_word4, ulp_context, iotag;
3743
3744	if (phba->sli_rev == LPFC_SLI_REV4)
3745	spin_lock_irqsave(&pring->ring_lock, iflag);
3746	else
3747	spin_lock_irqsave(&phba->hbalock, iflag);
3748	cmdiocbp = lpfc_sli_iocbq_lookup(phba, pring, prspiocb: saveq);
3749	if (phba->sli_rev == LPFC_SLI_REV4)
3750	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflag);
3751	else
3752	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
3753
3754	ulp_command = get_job_cmnd(phba, iocbq: saveq);
3755	ulp_status = get_job_ulpstatus(phba, iocbq: saveq);
3756	ulp_word4 = get_job_word4(phba, iocbq: saveq);
3757	ulp_context = get_job_ulpcontext(phba, iocbq: saveq);
3758	if (phba->sli_rev == LPFC_SLI_REV4)
3759	iotag = get_wqe_reqtag(saveq);
3760	else
3761	iotag = saveq->iocb.ulpIoTag;
3762
3763	if (cmdiocbp) {
3764	ulp_command = get_job_cmnd(phba, iocbq: cmdiocbp);
3765	if (cmdiocbp->cmd_cmpl) {
3766	/*
3767	* If an ELS command failed send an event to mgmt
3768	* application.
3769	*/
3770	if (ulp_status &&
3771	(pring->ringno == LPFC_ELS_RING) &&
3772	(ulp_command == CMD_ELS_REQUEST64_CR))
3773	lpfc_send_els_failure_event(phba,
3774	cmdiocbp, saveq);
3775
3776	/*
3777	* Post all ELS completions to the worker thread.
3778	* All other are passed to the completion callback.
3779	*/
3780	if (pring->ringno == LPFC_ELS_RING) {
3781	if ((phba->sli_rev < LPFC_SLI_REV4) &&
3782	(cmdiocbp->cmd_flag &
3783	LPFC_DRIVER_ABORTED)) {
3784	spin_lock_irqsave(&phba->hbalock,
3785	iflag);
3786	cmdiocbp->cmd_flag &=
3787	~LPFC_DRIVER_ABORTED;
3788	spin_unlock_irqrestore(lock: &phba->hbalock,
3789	flags: iflag);
3790	saveq->iocb.ulpStatus =
3791	IOSTAT_LOCAL_REJECT;
3792	saveq->iocb.un.ulpWord[4] =
3793	IOERR_SLI_ABORTED;
3794
3795	/* Firmware could still be in progress
3796	* of DMAing payload, so don't free data
3797	* buffer till after a hbeat.
3798	*/
3799	spin_lock_irqsave(&phba->hbalock,
3800	iflag);
3801	saveq->cmd_flag |= LPFC_DELAY_MEM_FREE;
3802	spin_unlock_irqrestore(lock: &phba->hbalock,
3803	flags: iflag);
3804	}
3805	if (phba->sli_rev == LPFC_SLI_REV4) {
3806	if (saveq->cmd_flag &
3807	LPFC_EXCHANGE_BUSY) {
3808	/* Set cmdiocb flag for the
3809	* exchange busy so sgl (xri)
3810	* will not be released until
3811	* the abort xri is received
3812	* from hba.
3813	*/
3814	spin_lock_irqsave(
3815	&phba->hbalock, iflag);
3816	cmdiocbp->cmd_flag |=
3817	LPFC_EXCHANGE_BUSY;
3818	spin_unlock_irqrestore(
3819	lock: &phba->hbalock, flags: iflag);
3820	}
3821	if (cmdiocbp->cmd_flag &
3822	LPFC_DRIVER_ABORTED) {
3823	/*
3824	* Clear LPFC_DRIVER_ABORTED
3825	* bit in case it was driver
3826	* initiated abort.
3827	*/
3828	spin_lock_irqsave(
3829	&phba->hbalock, iflag);
3830	cmdiocbp->cmd_flag &=
3831	~LPFC_DRIVER_ABORTED;
3832	spin_unlock_irqrestore(
3833	lock: &phba->hbalock, flags: iflag);
3834	set_job_ulpstatus(cmdiocbp,
3835	IOSTAT_LOCAL_REJECT);
3836	set_job_ulpword4(cmdiocbp,
3837	IOERR_ABORT_REQUESTED);
3838	/*
3839	* For SLI4, irspiocb contains
3840	* NO_XRI in sli_xritag, it
3841	* shall not affect releasing
3842	* sgl (xri) process.
3843	*/
3844	set_job_ulpstatus(saveq,
3845	IOSTAT_LOCAL_REJECT);
3846	set_job_ulpword4(saveq,
3847	IOERR_SLI_ABORTED);
3848	spin_lock_irqsave(
3849	&phba->hbalock, iflag);
3850	saveq->cmd_flag |=
3851	LPFC_DELAY_MEM_FREE;
3852	spin_unlock_irqrestore(
3853	lock: &phba->hbalock, flags: iflag);
3854	}
3855	}
3856	}
3857	cmdiocbp->cmd_cmpl(phba, cmdiocbp, saveq);
3858	} else
3859	lpfc_sli_release_iocbq(phba, iocbq: cmdiocbp);
3860	} else {
3861	/*
3862	* Unknown initiating command based on the response iotag.
3863	* This could be the case on the ELS ring because of
3864	* lpfc_els_abort().
3865	*/
3866	if (pring->ringno != LPFC_ELS_RING) {
3867	/*
3868	* Ring <ringno> handler: unexpected completion IoTag
3869	* <IoTag>
3870	*/
3871	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
3872	"0322 Ring %d handler: "
3873	"unexpected completion IoTag x%x "
3874	"Data: x%x x%x x%x x%x\n",
3875	pring->ringno, iotag, ulp_status,
3876	ulp_word4, ulp_command, ulp_context);
3877	}
3878	}
3879
3880	return 1;
3881	}
3882
3883	/**
3884	* lpfc_sli_rsp_pointers_error - Response ring pointer error handler
3885	* @phba: Pointer to HBA context object.
3886	* @pring: Pointer to driver SLI ring object.
3887	*
3888	* This function is called from the iocb ring event handlers when
3889	* put pointer is ahead of the get pointer for a ring. This function signal
3890	* an error attention condition to the worker thread and the worker
3891	* thread will transition the HBA to offline state.
3892	**/
3893	static void
3894	lpfc_sli_rsp_pointers_error(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
3895	{
3896	struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
3897	/*
3898	* Ring <ringno> handler: portRspPut <portRspPut> is bigger than
3899	* rsp ring <portRspMax>
3900	*/
3901	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3902	"0312 Ring %d handler: portRspPut %d "
3903	"is bigger than rsp ring %d\n",
3904	pring->ringno, le32_to_cpu(pgp->rspPutInx),
3905	pring->sli.sli3.numRiocb);
3906
3907	phba->link_state = LPFC_HBA_ERROR;
3908
3909	/*
3910	* All error attention handlers are posted to
3911	* worker thread
3912	*/
3913	phba->work_ha |= HA_ERATT;
3914	phba->work_hs = HS_FFER3;
3915
3916	lpfc_worker_wake_up(phba);
3917
3918	return;
3919	}
3920
3921	/**
3922	* lpfc_poll_eratt - Error attention polling timer timeout handler
3923	* @t: Context to fetch pointer to address of HBA context object from.
3924	*
3925	* This function is invoked by the Error Attention polling timer when the
3926	* timer times out. It will check the SLI Error Attention register for
3927	* possible attention events. If so, it will post an Error Attention event
3928	* and wake up worker thread to process it. Otherwise, it will set up the
3929	* Error Attention polling timer for the next poll.
3930	**/
3931	void lpfc_poll_eratt(struct timer_list *t)
3932	{
3933	struct lpfc_hba *phba;
3934	uint32_t eratt = 0;
3935	uint64_t sli_intr, cnt;
3936
3937	phba = from_timer(phba, t, eratt_poll);
3938	if (!(phba->hba_flag & HBA_SETUP))
3939	return;
3940
3941	/* Here we will also keep track of interrupts per sec of the hba */
3942	sli_intr = phba->sli.slistat.sli_intr;
3943
3944	if (phba->sli.slistat.sli_prev_intr > sli_intr)
3945	cnt = (((uint64_t)(-1) - phba->sli.slistat.sli_prev_intr) +
3946	sli_intr);
3947	else
3948	cnt = (sli_intr - phba->sli.slistat.sli_prev_intr);
3949
3950	/* 64-bit integer division not supported on 32-bit x86 - use do_div */
3951	do_div(cnt, phba->eratt_poll_interval);
3952	phba->sli.slistat.sli_ips = cnt;
3953
3954	phba->sli.slistat.sli_prev_intr = sli_intr;
3955
3956	/* Check chip HA register for error event */
3957	eratt = lpfc_sli_check_eratt(phba);
3958
3959	if (eratt)
3960	/* Tell the worker thread there is work to do */
3961	lpfc_worker_wake_up(phba);
3962	else
3963	/* Restart the timer for next eratt poll */
3964	mod_timer(timer: &phba->eratt_poll,
3965	expires: jiffies +
3966	msecs_to_jiffies(m: 1000 * phba->eratt_poll_interval));
3967	return;
3968	}
3969
3970
3971	/**
3972	* lpfc_sli_handle_fast_ring_event - Handle ring events on FCP ring
3973	* @phba: Pointer to HBA context object.
3974	* @pring: Pointer to driver SLI ring object.
3975	* @mask: Host attention register mask for this ring.
3976	*
3977	* This function is called from the interrupt context when there is a ring
3978	* event for the fcp ring. The caller does not hold any lock.
3979	* The function processes each response iocb in the response ring until it
3980	* finds an iocb with LE bit set and chains all the iocbs up to the iocb with
3981	* LE bit set. The function will call the completion handler of the command iocb
3982	* if the response iocb indicates a completion for a command iocb or it is
3983	* an abort completion. The function will call lpfc_sli_process_unsol_iocb
3984	* function if this is an unsolicited iocb.
3985	* This routine presumes LPFC_FCP_RING handling and doesn't bother
3986	* to check it explicitly.
3987	*/
3988	int
3989	lpfc_sli_handle_fast_ring_event(struct lpfc_hba *phba,
3990	struct lpfc_sli_ring *pring, uint32_t mask)
3991	{
3992	struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
3993	IOCB_t *irsp = NULL;
3994	IOCB_t *entry = NULL;
3995	struct lpfc_iocbq *cmdiocbq = NULL;
3996	struct lpfc_iocbq rspiocbq;
3997	uint32_t status;
3998	uint32_t portRspPut, portRspMax;
3999	int rc = 1;
4000	lpfc_iocb_type type;
4001	unsigned long iflag;
4002	uint32_t rsp_cmpl = 0;
4003
4004	spin_lock_irqsave(&phba->hbalock, iflag);
4005	pring->stats.iocb_event++;
4006
4007	/*
4008	* The next available response entry should never exceed the maximum
4009	* entries. If it does, treat it as an adapter hardware error.
4010	*/
4011	portRspMax = pring->sli.sli3.numRiocb;
4012	portRspPut = le32_to_cpu(pgp->rspPutInx);
4013	if (unlikely(portRspPut >= portRspMax)) {
4014	lpfc_sli_rsp_pointers_error(phba, pring);
4015	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4016	return 1;
4017	}
4018	if (phba->fcp_ring_in_use) {
4019	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4020	return 1;
4021	} else
4022	phba->fcp_ring_in_use = 1;
4023
4024	rmb();
4025	while (pring->sli.sli3.rspidx != portRspPut) {
4026	/*
4027	* Fetch an entry off the ring and copy it into a local data
4028	* structure. The copy involves a byte-swap since the
4029	* network byte order and pci byte orders are different.
4030	*/
4031	entry = lpfc_resp_iocb(phba, pring);
4032	phba->last_completion_time = jiffies;
4033
4034	if (++pring->sli.sli3.rspidx >= portRspMax)
4035	pring->sli.sli3.rspidx = 0;
4036
4037	lpfc_sli_pcimem_bcopy((uint32_t *) entry,
4038	(uint32_t *) &rspiocbq.iocb,
4039	phba->iocb_rsp_size);
4040	INIT_LIST_HEAD(list: &(rspiocbq.list));
4041	irsp = &rspiocbq.iocb;
4042
4043	type = lpfc_sli_iocb_cmd_type(iocb_cmnd: irsp->ulpCommand & CMD_IOCB_MASK);
4044	pring->stats.iocb_rsp++;
4045	rsp_cmpl++;
4046
4047	if (unlikely(irsp->ulpStatus)) {
4048	/*
4049	* If resource errors reported from HBA, reduce
4050	* queuedepths of the SCSI device.
4051	*/
4052	if ((irsp->ulpStatus == IOSTAT_LOCAL_REJECT) &&
4053	((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) ==
4054	IOERR_NO_RESOURCES)) {
4055	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4056	phba->lpfc_rampdown_queue_depth(phba);
4057	spin_lock_irqsave(&phba->hbalock, iflag);
4058	}
4059
4060	/* Rsp ring <ringno> error: IOCB */
4061	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
4062	"0336 Rsp Ring %d error: IOCB Data: "
4063	"x%x x%x x%x x%x x%x x%x x%x x%x\n",
4064	pring->ringno,
4065	irsp->un.ulpWord[0],
4066	irsp->un.ulpWord[1],
4067	irsp->un.ulpWord[2],
4068	irsp->un.ulpWord[3],
4069	irsp->un.ulpWord[4],
4070	irsp->un.ulpWord[5],
4071	*(uint32_t *)&irsp->un1,
4072	*((uint32_t *)&irsp->un1 + 1));
4073	}
4074
4075	switch (type) {
4076	case LPFC_ABORT_IOCB:
4077	case LPFC_SOL_IOCB:
4078	/*
4079	* Idle exchange closed via ABTS from port. No iocb
4080	* resources need to be recovered.
4081	*/
4082	if (unlikely(irsp->ulpCommand == CMD_XRI_ABORTED_CX)) {
4083	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
4084	"0333 IOCB cmd 0x%x"
4085	" processed. Skipping"
4086	" completion\n",
4087	irsp->ulpCommand);
4088	break;
4089	}
4090
4091	cmdiocbq = lpfc_sli_iocbq_lookup(phba, pring,
4092	prspiocb: &rspiocbq);
4093	if (unlikely(!cmdiocbq))
4094	break;
4095	if (cmdiocbq->cmd_flag & LPFC_DRIVER_ABORTED)
4096	cmdiocbq->cmd_flag &= ~LPFC_DRIVER_ABORTED;
4097	if (cmdiocbq->cmd_cmpl) {
4098	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4099	cmdiocbq->cmd_cmpl(phba, cmdiocbq, &rspiocbq);
4100	spin_lock_irqsave(&phba->hbalock, iflag);
4101	}
4102	break;
4103	case LPFC_UNSOL_IOCB:
4104	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4105	lpfc_sli_process_unsol_iocb(phba, pring, saveq: &rspiocbq);
4106	spin_lock_irqsave(&phba->hbalock, iflag);
4107	break;
4108	default:
4109	if (irsp->ulpCommand == CMD_ADAPTER_MSG) {
4110	char adaptermsg[LPFC_MAX_ADPTMSG];
4111	memset(adaptermsg, 0, LPFC_MAX_ADPTMSG);
4112	memcpy(&adaptermsg[0], (uint8_t *) irsp,
4113	MAX_MSG_DATA);
4114	dev_warn(&((phba->pcidev)->dev),
4115	"lpfc%d: %s\n",
4116	phba->brd_no, adaptermsg);
4117	} else {
4118	/* Unknown IOCB command */
4119	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
4120	"0334 Unknown IOCB command "
4121	"Data: x%x, x%x x%x x%x x%x\n",
4122	type, irsp->ulpCommand,
4123	irsp->ulpStatus,
4124	irsp->ulpIoTag,
4125	irsp->ulpContext);
4126	}
4127	break;
4128	}
4129
4130	/*
4131	* The response IOCB has been processed. Update the ring
4132	* pointer in SLIM. If the port response put pointer has not
4133	* been updated, sync the pgp->rspPutInx and fetch the new port
4134	* response put pointer.
4135	*/
4136	writel(val: pring->sli.sli3.rspidx,
4137	addr: &phba->host_gp[pring->ringno].rspGetInx);
4138
4139	if (pring->sli.sli3.rspidx == portRspPut)
4140	portRspPut = le32_to_cpu(pgp->rspPutInx);
4141	}
4142
4143	if ((rsp_cmpl > 0) && (mask & HA_R0RE_REQ)) {
4144	pring->stats.iocb_rsp_full++;
4145	status = ((CA_R0ATT | CA_R0RE_RSP) << (pring->ringno * 4));
4146	writel(val: status, addr: phba->CAregaddr);
4147	readl(addr: phba->CAregaddr);
4148	}
4149	if ((mask & HA_R0CE_RSP) && (pring->flag & LPFC_CALL_RING_AVAILABLE)) {
4150	pring->flag &= ~LPFC_CALL_RING_AVAILABLE;
4151	pring->stats.iocb_cmd_empty++;
4152
4153	/* Force update of the local copy of cmdGetInx */
4154	pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);
4155	lpfc_sli_resume_iocb(phba, pring);
4156
4157	if ((pring->lpfc_sli_cmd_available))
4158	(pring->lpfc_sli_cmd_available) (phba, pring);
4159
4160	}
4161
4162	phba->fcp_ring_in_use = 0;
4163	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4164	return rc;
4165	}
4166
4167	/**
4168	* lpfc_sli_sp_handle_rspiocb - Handle slow-path response iocb
4169	* @phba: Pointer to HBA context object.
4170	* @pring: Pointer to driver SLI ring object.
4171	* @rspiocbp: Pointer to driver response IOCB object.
4172	*
4173	* This function is called from the worker thread when there is a slow-path
4174	* response IOCB to process. This function chains all the response iocbs until
4175	* seeing the iocb with the LE bit set. The function will call
4176	* lpfc_sli_process_sol_iocb function if the response iocb indicates a
4177	* completion of a command iocb. The function will call the
4178	* lpfc_sli_process_unsol_iocb function if this is an unsolicited iocb.
4179	* The function frees the resources or calls the completion handler if this
4180	* iocb is an abort completion. The function returns NULL when the response
4181	* iocb has the LE bit set and all the chained iocbs are processed, otherwise
4182	* this function shall chain the iocb on to the iocb_continueq and return the
4183	* response iocb passed in.
4184	**/
4185	static struct lpfc_iocbq *
4186	lpfc_sli_sp_handle_rspiocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
4187	struct lpfc_iocbq *rspiocbp)
4188	{
4189	struct lpfc_iocbq *saveq;
4190	struct lpfc_iocbq *cmdiocb;
4191	struct lpfc_iocbq *next_iocb;
4192	IOCB_t *irsp;
4193	uint32_t free_saveq;
4194	u8 cmd_type;
4195	lpfc_iocb_type type;
4196	unsigned long iflag;
4197	u32 ulp_status = get_job_ulpstatus(phba, iocbq: rspiocbp);
4198	u32 ulp_word4 = get_job_word4(phba, iocbq: rspiocbp);
4199	u32 ulp_command = get_job_cmnd(phba, iocbq: rspiocbp);
4200	int rc;
4201
4202	spin_lock_irqsave(&phba->hbalock, iflag);
4203	/* First add the response iocb to the countinueq list */
4204	list_add_tail(new: &rspiocbp->list, head: &pring->iocb_continueq);
4205	pring->iocb_continueq_cnt++;
4206
4207	/*
4208	* By default, the driver expects to free all resources
4209	* associated with this iocb completion.
4210	*/
4211	free_saveq = 1;
4212	saveq = list_get_first(&pring->iocb_continueq,
4213	struct lpfc_iocbq, list);
4214	list_del_init(entry: &pring->iocb_continueq);
4215	pring->iocb_continueq_cnt = 0;
4216
4217	pring->stats.iocb_rsp++;
4218
4219	/*
4220	* If resource errors reported from HBA, reduce
4221	* queuedepths of the SCSI device.
4222	*/
4223	if (ulp_status == IOSTAT_LOCAL_REJECT &&
4224	((ulp_word4 & IOERR_PARAM_MASK) ==
4225	IOERR_NO_RESOURCES)) {
4226	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4227	phba->lpfc_rampdown_queue_depth(phba);
4228	spin_lock_irqsave(&phba->hbalock, iflag);
4229	}
4230
4231	if (ulp_status) {
4232	/* Rsp ring <ringno> error: IOCB */
4233	if (phba->sli_rev < LPFC_SLI_REV4) {
4234	irsp = &rspiocbp->iocb;
4235	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
4236	"0328 Rsp Ring %d error: ulp_status x%x "
4237	"IOCB Data: "
4238	"x%08x x%08x x%08x x%08x "
4239	"x%08x x%08x x%08x x%08x "
4240	"x%08x x%08x x%08x x%08x "
4241	"x%08x x%08x x%08x x%08x\n",
4242	pring->ringno, ulp_status,
4243	get_job_ulpword(rspiocbp, 0),
4244	get_job_ulpword(rspiocbp, 1),
4245	get_job_ulpword(rspiocbp, 2),
4246	get_job_ulpword(rspiocbp, 3),
4247	get_job_ulpword(rspiocbp, 4),
4248	get_job_ulpword(rspiocbp, 5),
4249	*(((uint32_t *)irsp) + 6),
4250	*(((uint32_t *)irsp) + 7),
4251	*(((uint32_t *)irsp) + 8),
4252	*(((uint32_t *)irsp) + 9),
4253	*(((uint32_t *)irsp) + 10),
4254	*(((uint32_t *)irsp) + 11),
4255	*(((uint32_t *)irsp) + 12),
4256	*(((uint32_t *)irsp) + 13),
4257	*(((uint32_t *)irsp) + 14),
4258	*(((uint32_t *)irsp) + 15));
4259	} else {
4260	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
4261	"0321 Rsp Ring %d error: "
4262	"IOCB Data: "
4263	"x%x x%x x%x x%x\n",
4264	pring->ringno,
4265	rspiocbp->wcqe_cmpl.word0,
4266	rspiocbp->wcqe_cmpl.total_data_placed,
4267	rspiocbp->wcqe_cmpl.parameter,
4268	rspiocbp->wcqe_cmpl.word3);
4269	}
4270	}
4271
4272
4273	/*
4274	* Fetch the iocb command type and call the correct completion
4275	* routine. Solicited and Unsolicited IOCBs on the ELS ring
4276	* get freed back to the lpfc_iocb_list by the discovery
4277	* kernel thread.
4278	*/
4279	cmd_type = ulp_command & CMD_IOCB_MASK;
4280	type = lpfc_sli_iocb_cmd_type(iocb_cmnd: cmd_type);
4281	switch (type) {
4282	case LPFC_SOL_IOCB:
4283	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4284	rc = lpfc_sli_process_sol_iocb(phba, pring, saveq);
4285	spin_lock_irqsave(&phba->hbalock, iflag);
4286	break;
4287	case LPFC_UNSOL_IOCB:
4288	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4289	rc = lpfc_sli_process_unsol_iocb(phba, pring, saveq);
4290	spin_lock_irqsave(&phba->hbalock, iflag);
4291	if (!rc)
4292	free_saveq = 0;
4293	break;
4294	case LPFC_ABORT_IOCB:
4295	cmdiocb = NULL;
4296	if (ulp_command != CMD_XRI_ABORTED_CX)
4297	cmdiocb = lpfc_sli_iocbq_lookup(phba, pring,
4298	prspiocb: saveq);
4299	if (cmdiocb) {
4300	/* Call the specified completion routine */
4301	if (cmdiocb->cmd_cmpl) {
4302	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4303	cmdiocb->cmd_cmpl(phba, cmdiocb, saveq);
4304	spin_lock_irqsave(&phba->hbalock, iflag);
4305	} else {
4306	__lpfc_sli_release_iocbq(phba, iocbq: cmdiocb);
4307	}
4308	}
4309	break;
4310	case LPFC_UNKNOWN_IOCB:
4311	if (ulp_command == CMD_ADAPTER_MSG) {
4312	char adaptermsg[LPFC_MAX_ADPTMSG];
4313
4314	memset(adaptermsg, 0, LPFC_MAX_ADPTMSG);
4315	memcpy(&adaptermsg[0], (uint8_t *)&rspiocbp->wqe,
4316	MAX_MSG_DATA);
4317	dev_warn(&((phba->pcidev)->dev),
4318	"lpfc%d: %s\n",
4319	phba->brd_no, adaptermsg);
4320	} else {
4321	/* Unknown command */
4322	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
4323	"0335 Unknown IOCB "
4324	"command Data: x%x "
4325	"x%x x%x x%x\n",
4326	ulp_command,
4327	ulp_status,
4328	get_wqe_reqtag(rspiocbp),
4329	get_job_ulpcontext(phba, rspiocbp));
4330	}
4331	break;
4332	}
4333
4334	if (free_saveq) {
4335	list_for_each_entry_safe(rspiocbp, next_iocb,
4336	&saveq->list, list) {
4337	list_del_init(entry: &rspiocbp->list);
4338	__lpfc_sli_release_iocbq(phba, iocbq: rspiocbp);
4339	}
4340	__lpfc_sli_release_iocbq(phba, iocbq: saveq);
4341	}
4342	rspiocbp = NULL;
4343	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4344	return rspiocbp;
4345	}
4346
4347	/**
4348	* lpfc_sli_handle_slow_ring_event - Wrapper func for handling slow-path iocbs
4349	* @phba: Pointer to HBA context object.
4350	* @pring: Pointer to driver SLI ring object.
4351	* @mask: Host attention register mask for this ring.
4352	*
4353	* This routine wraps the actual slow_ring event process routine from the
4354	* API jump table function pointer from the lpfc_hba struct.
4355	**/
4356	void
4357	lpfc_sli_handle_slow_ring_event(struct lpfc_hba *phba,
4358	struct lpfc_sli_ring *pring, uint32_t mask)
4359	{
4360	phba->lpfc_sli_handle_slow_ring_event(phba, pring, mask);
4361	}
4362
4363	/**
4364	* lpfc_sli_handle_slow_ring_event_s3 - Handle SLI3 ring event for non-FCP rings
4365	* @phba: Pointer to HBA context object.
4366	* @pring: Pointer to driver SLI ring object.
4367	* @mask: Host attention register mask for this ring.
4368	*
4369	* This function is called from the worker thread when there is a ring event
4370	* for non-fcp rings. The caller does not hold any lock. The function will
4371	* remove each response iocb in the response ring and calls the handle
4372	* response iocb routine (lpfc_sli_sp_handle_rspiocb) to process it.
4373	**/
4374	static void
4375	lpfc_sli_handle_slow_ring_event_s3(struct lpfc_hba *phba,
4376	struct lpfc_sli_ring *pring, uint32_t mask)
4377	{
4378	struct lpfc_pgp *pgp;
4379	IOCB_t *entry;
4380	IOCB_t *irsp = NULL;
4381	struct lpfc_iocbq *rspiocbp = NULL;
4382	uint32_t portRspPut, portRspMax;
4383	unsigned long iflag;
4384	uint32_t status;
4385
4386	pgp = &phba->port_gp[pring->ringno];
4387	spin_lock_irqsave(&phba->hbalock, iflag);
4388	pring->stats.iocb_event++;
4389
4390	/*
4391	* The next available response entry should never exceed the maximum
4392	* entries. If it does, treat it as an adapter hardware error.
4393	*/
4394	portRspMax = pring->sli.sli3.numRiocb;
4395	portRspPut = le32_to_cpu(pgp->rspPutInx);
4396	if (portRspPut >= portRspMax) {
4397	/*
4398	* Ring <ringno> handler: portRspPut <portRspPut> is bigger than
4399	* rsp ring <portRspMax>
4400	*/
4401	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
4402	"0303 Ring %d handler: portRspPut %d "
4403	"is bigger than rsp ring %d\n",
4404	pring->ringno, portRspPut, portRspMax);
4405
4406	phba->link_state = LPFC_HBA_ERROR;
4407	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4408
4409	phba->work_hs = HS_FFER3;
4410	lpfc_handle_eratt(phba);
4411
4412	return;
4413	}
4414
4415	rmb();
4416	while (pring->sli.sli3.rspidx != portRspPut) {
4417	/*
4418	* Build a completion list and call the appropriate handler.
4419	* The process is to get the next available response iocb, get
4420	* a free iocb from the list, copy the response data into the
4421	* free iocb, insert to the continuation list, and update the
4422	* next response index to slim. This process makes response
4423	* iocb's in the ring available to DMA as fast as possible but
4424	* pays a penalty for a copy operation. Since the iocb is
4425	* only 32 bytes, this penalty is considered small relative to
4426	* the PCI reads for register values and a slim write. When
4427	* the ulpLe field is set, the entire Command has been
4428	* received.
4429	*/
4430	entry = lpfc_resp_iocb(phba, pring);
4431
4432	phba->last_completion_time = jiffies;
4433	rspiocbp = __lpfc_sli_get_iocbq(phba);
4434	if (rspiocbp == NULL) {
4435	printk(KERN_ERR "%s: out of buffers! Failing "
4436	"completion.\n", __func__);
4437	break;
4438	}
4439
4440	lpfc_sli_pcimem_bcopy(entry, &rspiocbp->iocb,
4441	phba->iocb_rsp_size);
4442	irsp = &rspiocbp->iocb;
4443
4444	if (++pring->sli.sli3.rspidx >= portRspMax)
4445	pring->sli.sli3.rspidx = 0;
4446
4447	if (pring->ringno == LPFC_ELS_RING) {
4448	lpfc_debugfs_slow_ring_trc(phba,
4449	"IOCB rsp ring: wd4:x%08x wd6:x%08x wd7:x%08x",
4450	*(((uint32_t *) irsp) + 4),
4451	*(((uint32_t *) irsp) + 6),
4452	*(((uint32_t *) irsp) + 7));
4453	}
4454
4455	writel(val: pring->sli.sli3.rspidx,
4456	addr: &phba->host_gp[pring->ringno].rspGetInx);
4457
4458	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4459	/* Handle the response IOCB */
4460	rspiocbp = lpfc_sli_sp_handle_rspiocb(phba, pring, rspiocbp);
4461	spin_lock_irqsave(&phba->hbalock, iflag);
4462
4463	/*
4464	* If the port response put pointer has not been updated, sync
4465	* the pgp->rspPutInx in the MAILBOX_tand fetch the new port
4466	* response put pointer.
4467	*/
4468	if (pring->sli.sli3.rspidx == portRspPut) {
4469	portRspPut = le32_to_cpu(pgp->rspPutInx);
4470	}
4471	} /* while (pring->sli.sli3.rspidx != portRspPut) */
4472
4473	if ((rspiocbp != NULL) && (mask & HA_R0RE_REQ)) {
4474	/* At least one response entry has been freed */
4475	pring->stats.iocb_rsp_full++;
4476	/* SET RxRE_RSP in Chip Att register */
4477	status = ((CA_R0ATT | CA_R0RE_RSP) << (pring->ringno * 4));
4478	writel(val: status, addr: phba->CAregaddr);
4479	readl(addr: phba->CAregaddr); /* flush */
4480	}
4481	if ((mask & HA_R0CE_RSP) && (pring->flag & LPFC_CALL_RING_AVAILABLE)) {
4482	pring->flag &= ~LPFC_CALL_RING_AVAILABLE;
4483	pring->stats.iocb_cmd_empty++;
4484
4485	/* Force update of the local copy of cmdGetInx */
4486	pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);
4487	lpfc_sli_resume_iocb(phba, pring);
4488
4489	if ((pring->lpfc_sli_cmd_available))
4490	(pring->lpfc_sli_cmd_available) (phba, pring);
4491
4492	}
4493
4494	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4495	return;
4496	}
4497
4498	/**
4499	* lpfc_sli_handle_slow_ring_event_s4 - Handle SLI4 slow-path els events
4500	* @phba: Pointer to HBA context object.
4501	* @pring: Pointer to driver SLI ring object.
4502	* @mask: Host attention register mask for this ring.
4503	*
4504	* This function is called from the worker thread when there is a pending
4505	* ELS response iocb on the driver internal slow-path response iocb worker
4506	* queue. The caller does not hold any lock. The function will remove each
4507	* response iocb from the response worker queue and calls the handle
4508	* response iocb routine (lpfc_sli_sp_handle_rspiocb) to process it.
4509	**/
4510	static void
4511	lpfc_sli_handle_slow_ring_event_s4(struct lpfc_hba *phba,
4512	struct lpfc_sli_ring *pring, uint32_t mask)
4513	{
4514	struct lpfc_iocbq *irspiocbq;
4515	struct hbq_dmabuf *dmabuf;
4516	struct lpfc_cq_event *cq_event;
4517	unsigned long iflag;
4518	int count = 0;
4519
4520	spin_lock_irqsave(&phba->hbalock, iflag);
4521	phba->hba_flag &= ~HBA_SP_QUEUE_EVT;
4522	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4523	while (!list_empty(head: &phba->sli4_hba.sp_queue_event)) {
4524	/* Get the response iocb from the head of work queue */
4525	spin_lock_irqsave(&phba->hbalock, iflag);
4526	list_remove_head(&phba->sli4_hba.sp_queue_event,
4527	cq_event, struct lpfc_cq_event, list);
4528	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
4529
4530	switch (bf_get(lpfc_wcqe_c_code, &cq_event->cqe.wcqe_cmpl)) {
4531	case CQE_CODE_COMPL_WQE:
4532	irspiocbq = container_of(cq_event, struct lpfc_iocbq,
4533	cq_event);
4534	/* Translate ELS WCQE to response IOCBQ */
4535	irspiocbq = lpfc_sli4_els_preprocess_rspiocbq(phba,
4536	rspiocbq: irspiocbq);
4537	if (irspiocbq)
4538	lpfc_sli_sp_handle_rspiocb(phba, pring,
4539	rspiocbp: irspiocbq);
4540	count++;
4541	break;
4542	case CQE_CODE_RECEIVE:
4543	case CQE_CODE_RECEIVE_V1:
4544	dmabuf = container_of(cq_event, struct hbq_dmabuf,
4545	cq_event);
4546	lpfc_sli4_handle_received_buffer(phba, dmabuf);
4547	count++;
4548	break;
4549	default:
4550	break;
4551	}
4552
4553	/* Limit the number of events to 64 to avoid soft lockups */
4554	if (count == 64)
4555	break;
4556	}
4557	}
4558
4559	/**
4560	* lpfc_sli_abort_iocb_ring - Abort all iocbs in the ring
4561	* @phba: Pointer to HBA context object.
4562	* @pring: Pointer to driver SLI ring object.
4563	*
4564	* This function aborts all iocbs in the given ring and frees all the iocb
4565	* objects in txq. This function issues an abort iocb for all the iocb commands
4566	* in txcmplq. The iocbs in the txcmplq is not guaranteed to complete before
4567	* the return of this function. The caller is not required to hold any locks.
4568	**/
4569	void
4570	lpfc_sli_abort_iocb_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
4571	{
4572	LIST_HEAD(tx_completions);
4573	LIST_HEAD(txcmplq_completions);
4574	struct lpfc_iocbq *iocb, *next_iocb;
4575	int offline;
4576
4577	if (pring->ringno == LPFC_ELS_RING) {
4578	lpfc_fabric_abort_hba(phba);
4579	}
4580	offline = pci_channel_offline(pdev: phba->pcidev);
4581
4582	/* Error everything on txq and txcmplq
4583	* First do the txq.
4584	*/
4585	if (phba->sli_rev >= LPFC_SLI_REV4) {
4586	spin_lock_irq(lock: &pring->ring_lock);
4587	list_splice_init(list: &pring->txq, head: &tx_completions);
4588	pring->txq_cnt = 0;
4589
4590	if (offline) {
4591	list_splice_init(list: &pring->txcmplq,
4592	head: &txcmplq_completions);
4593	} else {
4594	/* Next issue ABTS for everything on the txcmplq */
4595	list_for_each_entry_safe(iocb, next_iocb,
4596	&pring->txcmplq, list)
4597	lpfc_sli_issue_abort_iotag(phba, pring,
4598	iocb, NULL);
4599	}
4600	spin_unlock_irq(lock: &pring->ring_lock);
4601	} else {
4602	spin_lock_irq(lock: &phba->hbalock);
4603	list_splice_init(list: &pring->txq, head: &tx_completions);
4604	pring->txq_cnt = 0;
4605
4606	if (offline) {
4607	list_splice_init(list: &pring->txcmplq, head: &txcmplq_completions);
4608	} else {
4609	/* Next issue ABTS for everything on the txcmplq */
4610	list_for_each_entry_safe(iocb, next_iocb,
4611	&pring->txcmplq, list)
4612	lpfc_sli_issue_abort_iotag(phba, pring,
4613	iocb, NULL);
4614	}
4615	spin_unlock_irq(lock: &phba->hbalock);
4616	}
4617
4618	if (offline) {
4619	/* Cancel all the IOCBs from the completions list */
4620	lpfc_sli_cancel_iocbs(phba, iocblist: &txcmplq_completions,
4621	IOSTAT_LOCAL_REJECT, IOERR_SLI_ABORTED);
4622	} else {
4623	/* Make sure HBA is alive */
4624	lpfc_issue_hb_tmo(phba);
4625	}
4626	/* Cancel all the IOCBs from the completions list */
4627	lpfc_sli_cancel_iocbs(phba, iocblist: &tx_completions, IOSTAT_LOCAL_REJECT,
4628	IOERR_SLI_ABORTED);
4629	}
4630
4631	/**
4632	* lpfc_sli_abort_fcp_rings - Abort all iocbs in all FCP rings
4633	* @phba: Pointer to HBA context object.
4634	*
4635	* This function aborts all iocbs in FCP rings and frees all the iocb
4636	* objects in txq. This function issues an abort iocb for all the iocb commands
4637	* in txcmplq. The iocbs in the txcmplq is not guaranteed to complete before
4638	* the return of this function. The caller is not required to hold any locks.
4639	**/
4640	void
4641	lpfc_sli_abort_fcp_rings(struct lpfc_hba *phba)
4642	{
4643	struct lpfc_sli *psli = &phba->sli;
4644	struct lpfc_sli_ring *pring;
4645	uint32_t i;
4646
4647	/* Look on all the FCP Rings for the iotag */
4648	if (phba->sli_rev >= LPFC_SLI_REV4) {
4649	for (i = 0; i < phba->cfg_hdw_queue; i++) {
4650	pring = phba->sli4_hba.hdwq[i].io_wq->pring;
4651	lpfc_sli_abort_iocb_ring(phba, pring);
4652	}
4653	} else {
4654	pring = &psli->sli3_ring[LPFC_FCP_RING];
4655	lpfc_sli_abort_iocb_ring(phba, pring);
4656	}
4657	}
4658
4659	/**
4660	* lpfc_sli_flush_io_rings - flush all iocbs in the IO ring
4661	* @phba: Pointer to HBA context object.
4662	*
4663	* This function flushes all iocbs in the IO ring and frees all the iocb
4664	* objects in txq and txcmplq. This function will not issue abort iocbs
4665	* for all the iocb commands in txcmplq, they will just be returned with
4666	* IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
4667	* slot has been permanently disabled.
4668	**/
4669	void
4670	lpfc_sli_flush_io_rings(struct lpfc_hba *phba)
4671	{
4672	LIST_HEAD(txq);
4673	LIST_HEAD(txcmplq);
4674	struct lpfc_sli *psli = &phba->sli;
4675	struct lpfc_sli_ring *pring;
4676	uint32_t i;
4677	struct lpfc_iocbq *piocb, *next_iocb;
4678
4679	spin_lock_irq(lock: &phba->hbalock);
4680	/* Indicate the I/O queues are flushed */
4681	phba->hba_flag |= HBA_IOQ_FLUSH;
4682	spin_unlock_irq(lock: &phba->hbalock);
4683
4684	/* Look on all the FCP Rings for the iotag */
4685	if (phba->sli_rev >= LPFC_SLI_REV4) {
4686	for (i = 0; i < phba->cfg_hdw_queue; i++) {
4687	pring = phba->sli4_hba.hdwq[i].io_wq->pring;
4688
4689	spin_lock_irq(lock: &pring->ring_lock);
4690	/* Retrieve everything on txq */
4691	list_splice_init(list: &pring->txq, head: &txq);
4692	list_for_each_entry_safe(piocb, next_iocb,
4693	&pring->txcmplq, list)
4694	piocb->cmd_flag &= ~LPFC_IO_ON_TXCMPLQ;
4695	/* Retrieve everything on the txcmplq */
4696	list_splice_init(list: &pring->txcmplq, head: &txcmplq);
4697	pring->txq_cnt = 0;
4698	pring->txcmplq_cnt = 0;
4699	spin_unlock_irq(lock: &pring->ring_lock);
4700
4701	/* Flush the txq */
4702	lpfc_sli_cancel_iocbs(phba, iocblist: &txq,
4703	IOSTAT_LOCAL_REJECT,
4704	IOERR_SLI_DOWN);
4705	/* Flush the txcmplq */
4706	lpfc_sli_cancel_iocbs(phba, iocblist: &txcmplq,
4707	IOSTAT_LOCAL_REJECT,
4708	IOERR_SLI_DOWN);
4709	if (unlikely(pci_channel_offline(phba->pcidev)))
4710	lpfc_sli4_io_xri_aborted(phba, NULL, idx: 0);
4711	}
4712	} else {
4713	pring = &psli->sli3_ring[LPFC_FCP_RING];
4714
4715	spin_lock_irq(lock: &phba->hbalock);
4716	/* Retrieve everything on txq */
4717	list_splice_init(list: &pring->txq, head: &txq);
4718	list_for_each_entry_safe(piocb, next_iocb,
4719	&pring->txcmplq, list)
4720	piocb->cmd_flag &= ~LPFC_IO_ON_TXCMPLQ;
4721	/* Retrieve everything on the txcmplq */
4722	list_splice_init(list: &pring->txcmplq, head: &txcmplq);
4723	pring->txq_cnt = 0;
4724	pring->txcmplq_cnt = 0;
4725	spin_unlock_irq(lock: &phba->hbalock);
4726
4727	/* Flush the txq */
4728	lpfc_sli_cancel_iocbs(phba, iocblist: &txq, IOSTAT_LOCAL_REJECT,
4729	IOERR_SLI_DOWN);
4730	/* Flush the txcmpq */
4731	lpfc_sli_cancel_iocbs(phba, iocblist: &txcmplq, IOSTAT_LOCAL_REJECT,
4732	IOERR_SLI_DOWN);
4733	}
4734	}
4735
4736	/**
4737	* lpfc_sli_brdready_s3 - Check for sli3 host ready status
4738	* @phba: Pointer to HBA context object.
4739	* @mask: Bit mask to be checked.
4740	*
4741	* This function reads the host status register and compares
4742	* with the provided bit mask to check if HBA completed
4743	* the restart. This function will wait in a loop for the
4744	* HBA to complete restart. If the HBA does not restart within
4745	* 15 iterations, the function will reset the HBA again. The
4746	* function returns 1 when HBA fail to restart otherwise returns
4747	* zero.
4748	**/
4749	static int
4750	lpfc_sli_brdready_s3(struct lpfc_hba *phba, uint32_t mask)
4751	{
4752	uint32_t status;
4753	int i = 0;
4754	int retval = 0;
4755
4756	/* Read the HBA Host Status Register */
4757	if (lpfc_readl(addr: phba->HSregaddr, data: &status))
4758	return 1;
4759
4760	phba->hba_flag |= HBA_NEEDS_CFG_PORT;
4761
4762	/*
4763	* Check status register every 100ms for 5 retries, then every
4764	* 500ms for 5, then every 2.5 sec for 5, then reset board and
4765	* every 2.5 sec for 4.
4766	* Break our of the loop if errors occurred during init.
4767	*/
4768	while (((status & mask) != mask) &&
4769	!(status & HS_FFERM) &&
4770	i++ < 20) {
4771
4772	if (i <= 5)
4773	msleep(msecs: 10);
4774	else if (i <= 10)
4775	msleep(msecs: 500);
4776	else
4777	msleep(msecs: 2500);
4778
4779	if (i == 15) {
4780	/* Do post */
4781	phba->pport->port_state = LPFC_VPORT_UNKNOWN;
4782	lpfc_sli_brdrestart(phba);
4783	}
4784	/* Read the HBA Host Status Register */
4785	if (lpfc_readl(addr: phba->HSregaddr, data: &status)) {
4786	retval = 1;
4787	break;
4788	}
4789	}
4790
4791	/* Check to see if any errors occurred during init */
4792	if ((status & HS_FFERM) || (i >= 20)) {
4793	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
4794	"2751 Adapter failed to restart, "
4795	"status reg x%x, FW Data: A8 x%x AC x%x\n",
4796	status,
4797	readl(phba->MBslimaddr + 0xa8),
4798	readl(phba->MBslimaddr + 0xac));
4799	phba->link_state = LPFC_HBA_ERROR;
4800	retval = 1;
4801	}
4802
4803	return retval;
4804	}
4805
4806	/**
4807	* lpfc_sli_brdready_s4 - Check for sli4 host ready status
4808	* @phba: Pointer to HBA context object.
4809	* @mask: Bit mask to be checked.
4810	*
4811	* This function checks the host status register to check if HBA is
4812	* ready. This function will wait in a loop for the HBA to be ready
4813	* If the HBA is not ready , the function will will reset the HBA PCI
4814	* function again. The function returns 1 when HBA fail to be ready
4815	* otherwise returns zero.
4816	**/
4817	static int
4818	lpfc_sli_brdready_s4(struct lpfc_hba *phba, uint32_t mask)
4819	{
4820	uint32_t status;
4821	int retval = 0;
4822
4823	/* Read the HBA Host Status Register */
4824	status = lpfc_sli4_post_status_check(phba);
4825
4826	if (status) {
4827	phba->pport->port_state = LPFC_VPORT_UNKNOWN;
4828	lpfc_sli_brdrestart(phba);
4829	status = lpfc_sli4_post_status_check(phba);
4830	}
4831
4832	/* Check to see if any errors occurred during init */
4833	if (status) {
4834	phba->link_state = LPFC_HBA_ERROR;
4835	retval = 1;
4836	} else
4837	phba->sli4_hba.intr_enable = 0;
4838
4839	phba->hba_flag &= ~HBA_SETUP;
4840	return retval;
4841	}
4842
4843	/**
4844	* lpfc_sli_brdready - Wrapper func for checking the hba readyness
4845	* @phba: Pointer to HBA context object.
4846	* @mask: Bit mask to be checked.
4847	*
4848	* This routine wraps the actual SLI3 or SLI4 hba readyness check routine
4849	* from the API jump table function pointer from the lpfc_hba struct.
4850	**/
4851	int
4852	lpfc_sli_brdready(struct lpfc_hba *phba, uint32_t mask)
4853	{
4854	return phba->lpfc_sli_brdready(phba, mask);
4855	}
4856
4857	#define BARRIER_TEST_PATTERN (0xdeadbeef)
4858
4859	/**
4860	* lpfc_reset_barrier - Make HBA ready for HBA reset
4861	* @phba: Pointer to HBA context object.
4862	*
4863	* This function is called before resetting an HBA. This function is called
4864	* with hbalock held and requests HBA to quiesce DMAs before a reset.
4865	**/
4866	void lpfc_reset_barrier(struct lpfc_hba *phba)
4867	{
4868	uint32_t __iomem *resp_buf;
4869	uint32_t __iomem *mbox_buf;
4870	volatile struct MAILBOX_word0 mbox;
4871	uint32_t hc_copy, ha_copy, resp_data;
4872	int i;
4873	uint8_t hdrtype;
4874
4875	lockdep_assert_held(&phba->hbalock);
4876
4877	pci_read_config_byte(dev: phba->pcidev, PCI_HEADER_TYPE, val: &hdrtype);
4878	if (hdrtype != 0x80 ||
4879	(FC_JEDEC_ID(phba->vpd.rev.biuRev) != HELIOS_JEDEC_ID &&
4880	FC_JEDEC_ID(phba->vpd.rev.biuRev) != THOR_JEDEC_ID))
4881	return;
4882
4883	/*
4884	* Tell the other part of the chip to suspend temporarily all
4885	* its DMA activity.
4886	*/
4887	resp_buf = phba->MBslimaddr;
4888
4889	/* Disable the error attention */
4890	if (lpfc_readl(addr: phba->HCregaddr, data: &hc_copy))
4891	return;
4892	writel(val: (hc_copy & ~HC_ERINT_ENA), addr: phba->HCregaddr);
4893	readl(addr: phba->HCregaddr); /* flush */
4894	phba->link_flag |= LS_IGNORE_ERATT;
4895
4896	if (lpfc_readl(addr: phba->HAregaddr, data: &ha_copy))
4897	return;
4898	if (ha_copy & HA_ERATT) {
4899	/* Clear Chip error bit */
4900	writel(HA_ERATT, addr: phba->HAregaddr);
4901	phba->pport->stopped = 1;
4902	}
4903
4904	mbox.word0 = 0;
4905	mbox.mbxCommand = MBX_KILL_BOARD;
4906	mbox.mbxOwner = OWN_CHIP;
4907
4908	writel(BARRIER_TEST_PATTERN, addr: (resp_buf + 1));
4909	mbox_buf = phba->MBslimaddr;
4910	writel(val: mbox.word0, addr: mbox_buf);
4911
4912	for (i = 0; i < 50; i++) {
4913	if (lpfc_readl(addr: (resp_buf + 1), data: &resp_data))
4914	return;
4915	if (resp_data != ~(BARRIER_TEST_PATTERN))
4916	mdelay(1);
4917	else
4918	break;
4919	}
4920	resp_data = 0;
4921	if (lpfc_readl(addr: (resp_buf + 1), data: &resp_data))
4922	return;
4923	if (resp_data != ~(BARRIER_TEST_PATTERN)) {
4924	if (phba->sli.sli_flag & LPFC_SLI_ACTIVE ||
4925	phba->pport->stopped)
4926	goto restore_hc;
4927	else
4928	goto clear_errat;
4929	}
4930
4931	mbox.mbxOwner = OWN_HOST;
4932	resp_data = 0;
4933	for (i = 0; i < 500; i++) {
4934	if (lpfc_readl(addr: resp_buf, data: &resp_data))
4935	return;
4936	if (resp_data != mbox.word0)
4937	mdelay(1);
4938	else
4939	break;
4940	}
4941
4942	clear_errat:
4943
4944	while (++i < 500) {
4945	if (lpfc_readl(addr: phba->HAregaddr, data: &ha_copy))
4946	return;
4947	if (!(ha_copy & HA_ERATT))
4948	mdelay(1);
4949	else
4950	break;
4951	}
4952
4953	if (readl(addr: phba->HAregaddr) & HA_ERATT) {
4954	writel(HA_ERATT, addr: phba->HAregaddr);
4955	phba->pport->stopped = 1;
4956	}
4957
4958	restore_hc:
4959	phba->link_flag &= ~LS_IGNORE_ERATT;
4960	writel(val: hc_copy, addr: phba->HCregaddr);
4961	readl(addr: phba->HCregaddr); /* flush */
4962	}
4963
4964	/**
4965	* lpfc_sli_brdkill - Issue a kill_board mailbox command
4966	* @phba: Pointer to HBA context object.
4967	*
4968	* This function issues a kill_board mailbox command and waits for
4969	* the error attention interrupt. This function is called for stopping
4970	* the firmware processing. The caller is not required to hold any
4971	* locks. This function calls lpfc_hba_down_post function to free
4972	* any pending commands after the kill. The function will return 1 when it
4973	* fails to kill the board else will return 0.
4974	**/
4975	int
4976	lpfc_sli_brdkill(struct lpfc_hba *phba)
4977	{
4978	struct lpfc_sli *psli;
4979	LPFC_MBOXQ_t *pmb;
4980	uint32_t status;
4981	uint32_t ha_copy;
4982	int retval;
4983	int i = 0;
4984
4985	psli = &phba->sli;
4986
4987	/* Kill HBA */
4988	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
4989	"0329 Kill HBA Data: x%x x%x\n",
4990	phba->pport->port_state, psli->sli_flag);
4991
4992	pmb = (LPFC_MBOXQ_t *) mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
4993	if (!pmb)
4994	return 1;
4995
4996	/* Disable the error attention */
4997	spin_lock_irq(lock: &phba->hbalock);
4998	if (lpfc_readl(addr: phba->HCregaddr, data: &status)) {
4999	spin_unlock_irq(lock: &phba->hbalock);
5000	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
5001	return 1;
5002	}
5003	status &= ~HC_ERINT_ENA;
5004	writel(val: status, addr: phba->HCregaddr);
5005	readl(addr: phba->HCregaddr); /* flush */
5006	phba->link_flag |= LS_IGNORE_ERATT;
5007	spin_unlock_irq(lock: &phba->hbalock);
5008
5009	lpfc_kill_board(phba, pmb);
5010	pmb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
5011	retval = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
5012
5013	if (retval != MBX_SUCCESS) {
5014	if (retval != MBX_BUSY)
5015	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
5016	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5017	"2752 KILL_BOARD command failed retval %d\n",
5018	retval);
5019	spin_lock_irq(lock: &phba->hbalock);
5020	phba->link_flag &= ~LS_IGNORE_ERATT;
5021	spin_unlock_irq(lock: &phba->hbalock);
5022	return 1;
5023	}
5024
5025	spin_lock_irq(lock: &phba->hbalock);
5026	psli->sli_flag &= ~LPFC_SLI_ACTIVE;
5027	spin_unlock_irq(lock: &phba->hbalock);
5028
5029	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
5030
5031	/* There is no completion for a KILL_BOARD mbox cmd. Check for an error
5032	* attention every 100ms for 3 seconds. If we don't get ERATT after
5033	* 3 seconds we still set HBA_ERROR state because the status of the
5034	* board is now undefined.
5035	*/
5036	if (lpfc_readl(addr: phba->HAregaddr, data: &ha_copy))
5037	return 1;
5038	while ((i++ < 30) && !(ha_copy & HA_ERATT)) {
5039	mdelay(100);
5040	if (lpfc_readl(addr: phba->HAregaddr, data: &ha_copy))
5041	return 1;
5042	}
5043
5044	del_timer_sync(timer: &psli->mbox_tmo);
5045	if (ha_copy & HA_ERATT) {
5046	writel(HA_ERATT, addr: phba->HAregaddr);
5047	phba->pport->stopped = 1;
5048	}
5049	spin_lock_irq(lock: &phba->hbalock);
5050	psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
5051	psli->mbox_active = NULL;
5052	phba->link_flag &= ~LS_IGNORE_ERATT;
5053	spin_unlock_irq(lock: &phba->hbalock);
5054
5055	lpfc_hba_down_post(phba);
5056	phba->link_state = LPFC_HBA_ERROR;
5057
5058	return ha_copy & HA_ERATT ? 0 : 1;
5059	}
5060
5061	/**
5062	* lpfc_sli_brdreset - Reset a sli-2 or sli-3 HBA
5063	* @phba: Pointer to HBA context object.
5064	*
5065	* This function resets the HBA by writing HC_INITFF to the control
5066	* register. After the HBA resets, this function resets all the iocb ring
5067	* indices. This function disables PCI layer parity checking during
5068	* the reset.
5069	* This function returns 0 always.
5070	* The caller is not required to hold any locks.
5071	**/
5072	int
5073	lpfc_sli_brdreset(struct lpfc_hba *phba)
5074	{
5075	struct lpfc_sli *psli;
5076	struct lpfc_sli_ring *pring;
5077	uint16_t cfg_value;
5078	int i;
5079
5080	psli = &phba->sli;
5081
5082	/* Reset HBA */
5083	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
5084	"0325 Reset HBA Data: x%x x%x\n",
5085	(phba->pport) ? phba->pport->port_state : 0,
5086	psli->sli_flag);
5087
5088	/* perform board reset */
5089	phba->fc_eventTag = 0;
5090	phba->link_events = 0;
5091	phba->hba_flag |= HBA_NEEDS_CFG_PORT;
5092	if (phba->pport) {
5093	phba->pport->fc_myDID = 0;
5094	phba->pport->fc_prevDID = 0;
5095	}
5096
5097	/* Turn off parity checking and serr during the physical reset */
5098	if (pci_read_config_word(dev: phba->pcidev, PCI_COMMAND, val: &cfg_value))
5099	return -EIO;
5100
5101	pci_write_config_word(dev: phba->pcidev, PCI_COMMAND,
5102	val: (cfg_value &
5103	~(PCI_COMMAND_PARITY | PCI_COMMAND_SERR)));
5104
5105	psli->sli_flag &= ~(LPFC_SLI_ACTIVE | LPFC_PROCESS_LA);
5106
5107	/* Now toggle INITFF bit in the Host Control Register */
5108	writel(HC_INITFF, addr: phba->HCregaddr);
5109	mdelay(1);
5110	readl(addr: phba->HCregaddr); /* flush */
5111	writel(val: 0, addr: phba->HCregaddr);
5112	readl(addr: phba->HCregaddr); /* flush */
5113
5114	/* Restore PCI cmd register */
5115	pci_write_config_word(dev: phba->pcidev, PCI_COMMAND, val: cfg_value);
5116
5117	/* Initialize relevant SLI info */
5118	for (i = 0; i < psli->num_rings; i++) {
5119	pring = &psli->sli3_ring[i];
5120	pring->flag = 0;
5121	pring->sli.sli3.rspidx = 0;
5122	pring->sli.sli3.next_cmdidx = 0;
5123	pring->sli.sli3.local_getidx = 0;
5124	pring->sli.sli3.cmdidx = 0;
5125	pring->missbufcnt = 0;
5126	}
5127
5128	phba->link_state = LPFC_WARM_START;
5129	return 0;
5130	}
5131
5132	/**
5133	* lpfc_sli4_brdreset - Reset a sli-4 HBA
5134	* @phba: Pointer to HBA context object.
5135	*
5136	* This function resets a SLI4 HBA. This function disables PCI layer parity
5137	* checking during resets the device. The caller is not required to hold
5138	* any locks.
5139	*
5140	* This function returns 0 on success else returns negative error code.
5141	**/
5142	int
5143	lpfc_sli4_brdreset(struct lpfc_hba *phba)
5144	{
5145	struct lpfc_sli *psli = &phba->sli;
5146	uint16_t cfg_value;
5147	int rc = 0;
5148
5149	/* Reset HBA */
5150	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
5151	"0295 Reset HBA Data: x%x x%x x%x\n",
5152	phba->pport->port_state, psli->sli_flag,
5153	phba->hba_flag);
5154
5155	/* perform board reset */
5156	phba->fc_eventTag = 0;
5157	phba->link_events = 0;
5158	phba->pport->fc_myDID = 0;
5159	phba->pport->fc_prevDID = 0;
5160	phba->hba_flag &= ~HBA_SETUP;
5161
5162	spin_lock_irq(lock: &phba->hbalock);
5163	psli->sli_flag &= ~(LPFC_PROCESS_LA);
5164	phba->fcf.fcf_flag = 0;
5165	spin_unlock_irq(lock: &phba->hbalock);
5166
5167	/* Now physically reset the device */
5168	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5169	"0389 Performing PCI function reset!\n");
5170
5171	/* Turn off parity checking and serr during the physical reset */
5172	if (pci_read_config_word(dev: phba->pcidev, PCI_COMMAND, val: &cfg_value)) {
5173	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5174	"3205 PCI read Config failed\n");
5175	return -EIO;
5176	}
5177
5178	pci_write_config_word(dev: phba->pcidev, PCI_COMMAND, val: (cfg_value &
5179	~(PCI_COMMAND_PARITY | PCI_COMMAND_SERR)));
5180
5181	/* Perform FCoE PCI function reset before freeing queue memory */
5182	rc = lpfc_pci_function_reset(phba);
5183
5184	/* Restore PCI cmd register */
5185	pci_write_config_word(dev: phba->pcidev, PCI_COMMAND, val: cfg_value);
5186
5187	return rc;
5188	}
5189
5190	/**
5191	* lpfc_sli_brdrestart_s3 - Restart a sli-3 hba
5192	* @phba: Pointer to HBA context object.
5193	*
5194	* This function is called in the SLI initialization code path to
5195	* restart the HBA. The caller is not required to hold any lock.
5196	* This function writes MBX_RESTART mailbox command to the SLIM and
5197	* resets the HBA. At the end of the function, it calls lpfc_hba_down_post
5198	* function to free any pending commands. The function enables
5199	* POST only during the first initialization. The function returns zero.
5200	* The function does not guarantee completion of MBX_RESTART mailbox
5201	* command before the return of this function.
5202	**/
5203	static int
5204	lpfc_sli_brdrestart_s3(struct lpfc_hba *phba)
5205	{
5206	volatile struct MAILBOX_word0 mb;
5207	struct lpfc_sli *psli;
5208	void __iomem *to_slim;
5209
5210	spin_lock_irq(lock: &phba->hbalock);
5211
5212	psli = &phba->sli;
5213
5214	/* Restart HBA */
5215	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
5216	"0337 Restart HBA Data: x%x x%x\n",
5217	(phba->pport) ? phba->pport->port_state : 0,
5218	psli->sli_flag);
5219
5220	mb.word0 = 0;
5221	mb.mbxCommand = MBX_RESTART;
5222	mb.mbxHc = 1;
5223
5224	lpfc_reset_barrier(phba);
5225
5226	to_slim = phba->MBslimaddr;
5227	writel(val: mb.word0, addr: to_slim);
5228	readl(addr: to_slim); /* flush */
5229
5230	/* Only skip post after fc_ffinit is completed */
5231	if (phba->pport && phba->pport->port_state)
5232	mb.word0 = 1; /* This is really setting up word1 */
5233	else
5234	mb.word0 = 0; /* This is really setting up word1 */
5235	to_slim = phba->MBslimaddr + sizeof (uint32_t);
5236	writel(val: mb.word0, addr: to_slim);
5237	readl(addr: to_slim); /* flush */
5238
5239	lpfc_sli_brdreset(phba);
5240	if (phba->pport)
5241	phba->pport->stopped = 0;
5242	phba->link_state = LPFC_INIT_START;
5243	phba->hba_flag = 0;
5244	spin_unlock_irq(lock: &phba->hbalock);
5245
5246	memset(&psli->lnk_stat_offsets, 0, sizeof(psli->lnk_stat_offsets));
5247	psli->stats_start = ktime_get_seconds();
5248
5249	/* Give the INITFF and Post time to settle. */
5250	mdelay(100);
5251
5252	lpfc_hba_down_post(phba);
5253
5254	return 0;
5255	}
5256
5257	/**
5258	* lpfc_sli_brdrestart_s4 - Restart the sli-4 hba
5259	* @phba: Pointer to HBA context object.
5260	*
5261	* This function is called in the SLI initialization code path to restart
5262	* a SLI4 HBA. The caller is not required to hold any lock.
5263	* At the end of the function, it calls lpfc_hba_down_post function to
5264	* free any pending commands.
5265	**/
5266	static int
5267	lpfc_sli_brdrestart_s4(struct lpfc_hba *phba)
5268	{
5269	struct lpfc_sli *psli = &phba->sli;
5270	int rc;
5271
5272	/* Restart HBA */
5273	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
5274	"0296 Restart HBA Data: x%x x%x\n",
5275	phba->pport->port_state, psli->sli_flag);
5276
5277	rc = lpfc_sli4_brdreset(phba);
5278	if (rc) {
5279	phba->link_state = LPFC_HBA_ERROR;
5280	goto hba_down_queue;
5281	}
5282
5283	spin_lock_irq(lock: &phba->hbalock);
5284	phba->pport->stopped = 0;
5285	phba->link_state = LPFC_INIT_START;
5286	phba->hba_flag = 0;
5287	/* Preserve FA-PWWN expectation */
5288	phba->sli4_hba.fawwpn_flag &= LPFC_FAWWPN_FABRIC;
5289	spin_unlock_irq(lock: &phba->hbalock);
5290
5291	memset(&psli->lnk_stat_offsets, 0, sizeof(psli->lnk_stat_offsets));
5292	psli->stats_start = ktime_get_seconds();
5293
5294	hba_down_queue:
5295	lpfc_hba_down_post(phba);
5296	lpfc_sli4_queue_destroy(phba);
5297
5298	return rc;
5299	}
5300
5301	/**
5302	* lpfc_sli_brdrestart - Wrapper func for restarting hba
5303	* @phba: Pointer to HBA context object.
5304	*
5305	* This routine wraps the actual SLI3 or SLI4 hba restart routine from the
5306	* API jump table function pointer from the lpfc_hba struct.
5307	**/
5308	int
5309	lpfc_sli_brdrestart(struct lpfc_hba *phba)
5310	{
5311	return phba->lpfc_sli_brdrestart(phba);
5312	}
5313
5314	/**
5315	* lpfc_sli_chipset_init - Wait for the restart of the HBA after a restart
5316	* @phba: Pointer to HBA context object.
5317	*
5318	* This function is called after a HBA restart to wait for successful
5319	* restart of the HBA. Successful restart of the HBA is indicated by
5320	* HS_FFRDY and HS_MBRDY bits. If the HBA fails to restart even after 15
5321	* iteration, the function will restart the HBA again. The function returns
5322	* zero if HBA successfully restarted else returns negative error code.
5323	**/
5324	int
5325	lpfc_sli_chipset_init(struct lpfc_hba *phba)
5326	{
5327	uint32_t status, i = 0;
5328
5329	/* Read the HBA Host Status Register */
5330	if (lpfc_readl(addr: phba->HSregaddr, data: &status))
5331	return -EIO;
5332
5333	/* Check status register to see what current state is */
5334	i = 0;
5335	while ((status & (HS_FFRDY | HS_MBRDY)) != (HS_FFRDY | HS_MBRDY)) {
5336
5337	/* Check every 10ms for 10 retries, then every 100ms for 90
5338	* retries, then every 1 sec for 50 retires for a total of
5339	* ~60 seconds before reset the board again and check every
5340	* 1 sec for 50 retries. The up to 60 seconds before the
5341	* board ready is required by the Falcon FIPS zeroization
5342	* complete, and any reset the board in between shall cause
5343	* restart of zeroization, further delay the board ready.
5344	*/
5345	if (i++ >= 200) {
5346	/* Adapter failed to init, timeout, status reg
5347	<status> */
5348	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5349	"0436 Adapter failed to init, "
5350	"timeout, status reg x%x, "
5351	"FW Data: A8 x%x AC x%x\n", status,
5352	readl(phba->MBslimaddr + 0xa8),
5353	readl(phba->MBslimaddr + 0xac));
5354	phba->link_state = LPFC_HBA_ERROR;
5355	return -ETIMEDOUT;
5356	}
5357
5358	/* Check to see if any errors occurred during init */
5359	if (status & HS_FFERM) {
5360	/* ERROR: During chipset initialization */
5361	/* Adapter failed to init, chipset, status reg
5362	<status> */
5363	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5364	"0437 Adapter failed to init, "
5365	"chipset, status reg x%x, "
5366	"FW Data: A8 x%x AC x%x\n", status,
5367	readl(phba->MBslimaddr + 0xa8),
5368	readl(phba->MBslimaddr + 0xac));
5369	phba->link_state = LPFC_HBA_ERROR;
5370	return -EIO;
5371	}
5372
5373	if (i <= 10)
5374	msleep(msecs: 10);
5375	else if (i <= 100)
5376	msleep(msecs: 100);
5377	else
5378	msleep(msecs: 1000);
5379
5380	if (i == 150) {
5381	/* Do post */
5382	phba->pport->port_state = LPFC_VPORT_UNKNOWN;
5383	lpfc_sli_brdrestart(phba);
5384	}
5385	/* Read the HBA Host Status Register */
5386	if (lpfc_readl(addr: phba->HSregaddr, data: &status))
5387	return -EIO;
5388	}
5389
5390	/* Check to see if any errors occurred during init */
5391	if (status & HS_FFERM) {
5392	/* ERROR: During chipset initialization */
5393	/* Adapter failed to init, chipset, status reg <status> */
5394	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5395	"0438 Adapter failed to init, chipset, "
5396	"status reg x%x, "
5397	"FW Data: A8 x%x AC x%x\n", status,
5398	readl(phba->MBslimaddr + 0xa8),
5399	readl(phba->MBslimaddr + 0xac));
5400	phba->link_state = LPFC_HBA_ERROR;
5401	return -EIO;
5402	}
5403
5404	phba->hba_flag |= HBA_NEEDS_CFG_PORT;
5405
5406	/* Clear all interrupt enable conditions */
5407	writel(val: 0, addr: phba->HCregaddr);
5408	readl(addr: phba->HCregaddr); /* flush */
5409
5410	/* setup host attn register */
5411	writel(val: 0xffffffff, addr: phba->HAregaddr);
5412	readl(addr: phba->HAregaddr); /* flush */
5413	return 0;
5414	}
5415
5416	/**
5417	* lpfc_sli_hbq_count - Get the number of HBQs to be configured
5418	*
5419	* This function calculates and returns the number of HBQs required to be
5420	* configured.
5421	**/
5422	int
5423	lpfc_sli_hbq_count(void)
5424	{
5425	return ARRAY_SIZE(lpfc_hbq_defs);
5426	}
5427
5428	/**
5429	* lpfc_sli_hbq_entry_count - Calculate total number of hbq entries
5430	*
5431	* This function adds the number of hbq entries in every HBQ to get
5432	* the total number of hbq entries required for the HBA and returns
5433	* the total count.
5434	**/
5435	static int
5436	lpfc_sli_hbq_entry_count(void)
5437	{
5438	int hbq_count = lpfc_sli_hbq_count();
5439	int count = 0;
5440	int i;
5441
5442	for (i = 0; i < hbq_count; ++i)
5443	count += lpfc_hbq_defs[i]->entry_count;
5444	return count;
5445	}
5446
5447	/**
5448	* lpfc_sli_hbq_size - Calculate memory required for all hbq entries
5449	*
5450	* This function calculates amount of memory required for all hbq entries
5451	* to be configured and returns the total memory required.
5452	**/
5453	int
5454	lpfc_sli_hbq_size(void)
5455	{
5456	return lpfc_sli_hbq_entry_count() * sizeof(struct lpfc_hbq_entry);
5457	}
5458
5459	/**
5460	* lpfc_sli_hbq_setup - configure and initialize HBQs
5461	* @phba: Pointer to HBA context object.
5462	*
5463	* This function is called during the SLI initialization to configure
5464	* all the HBQs and post buffers to the HBQ. The caller is not
5465	* required to hold any locks. This function will return zero if successful
5466	* else it will return negative error code.
5467	**/
5468	static int
5469	lpfc_sli_hbq_setup(struct lpfc_hba *phba)
5470	{
5471	int hbq_count = lpfc_sli_hbq_count();
5472	LPFC_MBOXQ_t *pmb;
5473	MAILBOX_t *pmbox;
5474	uint32_t hbqno;
5475	uint32_t hbq_entry_index;
5476
5477	/* Get a Mailbox buffer to setup mailbox
5478	* commands for HBA initialization
5479	*/
5480	pmb = (LPFC_MBOXQ_t *) mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
5481
5482	if (!pmb)
5483	return -ENOMEM;
5484
5485	pmbox = &pmb->u.mb;
5486
5487	/* Initialize the struct lpfc_sli_hbq structure for each hbq */
5488	phba->link_state = LPFC_INIT_MBX_CMDS;
5489	phba->hbq_in_use = 1;
5490
5491	hbq_entry_index = 0;
5492	for (hbqno = 0; hbqno < hbq_count; ++hbqno) {
5493	phba->hbqs[hbqno].next_hbqPutIdx = 0;
5494	phba->hbqs[hbqno].hbqPutIdx = 0;
5495	phba->hbqs[hbqno].local_hbqGetIdx = 0;
5496	phba->hbqs[hbqno].entry_count =
5497	lpfc_hbq_defs[hbqno]->entry_count;
5498	lpfc_config_hbq(phba, hbqno, lpfc_hbq_defs[hbqno],
5499	hbq_entry_index, pmb);
5500	hbq_entry_index += phba->hbqs[hbqno].entry_count;
5501
5502	if (lpfc_sli_issue_mbox(phba, pmb, MBX_POLL) != MBX_SUCCESS) {
5503	/* Adapter failed to init, mbxCmd <cmd> CFG_RING,
5504	mbxStatus <status>, ring <num> */
5505
5506	lpfc_printf_log(phba, KERN_ERR,
5507	LOG_SLI | LOG_VPORT,
5508	"1805 Adapter failed to init. "
5509	"Data: x%x x%x x%x\n",
5510	pmbox->mbxCommand,
5511	pmbox->mbxStatus, hbqno);
5512
5513	phba->link_state = LPFC_HBA_ERROR;
5514	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
5515	return -ENXIO;
5516	}
5517	}
5518	phba->hbq_count = hbq_count;
5519
5520	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
5521
5522	/* Initially populate or replenish the HBQs */
5523	for (hbqno = 0; hbqno < hbq_count; ++hbqno)
5524	lpfc_sli_hbqbuf_init_hbqs(phba, qno: hbqno);
5525	return 0;
5526	}
5527
5528	/**
5529	* lpfc_sli4_rb_setup - Initialize and post RBs to HBA
5530	* @phba: Pointer to HBA context object.
5531	*
5532	* This function is called during the SLI initialization to configure
5533	* all the HBQs and post buffers to the HBQ. The caller is not
5534	* required to hold any locks. This function will return zero if successful
5535	* else it will return negative error code.
5536	**/
5537	static int
5538	lpfc_sli4_rb_setup(struct lpfc_hba *phba)
5539	{
5540	phba->hbq_in_use = 1;
5541	/**
5542	* Specific case when the MDS diagnostics is enabled and supported.
5543	* The receive buffer count is truncated to manage the incoming
5544	* traffic.
5545	**/
5546	if (phba->cfg_enable_mds_diags && phba->mds_diags_support)
5547	phba->hbqs[LPFC_ELS_HBQ].entry_count =
5548	lpfc_hbq_defs[LPFC_ELS_HBQ]->entry_count >> 1;
5549	else
5550	phba->hbqs[LPFC_ELS_HBQ].entry_count =
5551	lpfc_hbq_defs[LPFC_ELS_HBQ]->entry_count;
5552	phba->hbq_count = 1;
5553	lpfc_sli_hbqbuf_init_hbqs(phba, LPFC_ELS_HBQ);
5554	/* Initially populate or replenish the HBQs */
5555	return 0;
5556	}
5557
5558	/**
5559	* lpfc_sli_config_port - Issue config port mailbox command
5560	* @phba: Pointer to HBA context object.
5561	* @sli_mode: sli mode - 2/3
5562	*
5563	* This function is called by the sli initialization code path
5564	* to issue config_port mailbox command. This function restarts the
5565	* HBA firmware and issues a config_port mailbox command to configure
5566	* the SLI interface in the sli mode specified by sli_mode
5567	* variable. The caller is not required to hold any locks.
5568	* The function returns 0 if successful, else returns negative error
5569	* code.
5570	**/
5571	int
5572	lpfc_sli_config_port(struct lpfc_hba *phba, int sli_mode)
5573	{
5574	LPFC_MBOXQ_t *pmb;
5575	uint32_t resetcount = 0, rc = 0, done = 0;
5576
5577	pmb = (LPFC_MBOXQ_t *) mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
5578	if (!pmb) {
5579	phba->link_state = LPFC_HBA_ERROR;
5580	return -ENOMEM;
5581	}
5582
5583	phba->sli_rev = sli_mode;
5584	while (resetcount < 2 && !done) {
5585	spin_lock_irq(lock: &phba->hbalock);
5586	phba->sli.sli_flag |= LPFC_SLI_MBOX_ACTIVE;
5587	spin_unlock_irq(lock: &phba->hbalock);
5588	phba->pport->port_state = LPFC_VPORT_UNKNOWN;
5589	lpfc_sli_brdrestart(phba);
5590	rc = lpfc_sli_chipset_init(phba);
5591	if (rc)
5592	break;
5593
5594	spin_lock_irq(lock: &phba->hbalock);
5595	phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
5596	spin_unlock_irq(lock: &phba->hbalock);
5597	resetcount++;
5598
5599	/* Call pre CONFIG_PORT mailbox command initialization. A
5600	* value of 0 means the call was successful. Any other
5601	* nonzero value is a failure, but if ERESTART is returned,
5602	* the driver may reset the HBA and try again.
5603	*/
5604	rc = lpfc_config_port_prep(phba);
5605	if (rc == -ERESTART) {
5606	phba->link_state = LPFC_LINK_UNKNOWN;
5607	continue;
5608	} else if (rc)
5609	break;
5610
5611	phba->link_state = LPFC_INIT_MBX_CMDS;
5612	lpfc_config_port(phba, pmb);
5613	rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
5614	phba->sli3_options &= ~(LPFC_SLI3_NPIV_ENABLED |
5615	LPFC_SLI3_HBQ_ENABLED |
5616	LPFC_SLI3_CRP_ENABLED |
5617	LPFC_SLI3_DSS_ENABLED);
5618	if (rc != MBX_SUCCESS) {
5619	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5620	"0442 Adapter failed to init, mbxCmd x%x "
5621	"CONFIG_PORT, mbxStatus x%x Data: x%x\n",
5622	pmb->u.mb.mbxCommand, pmb->u.mb.mbxStatus, 0);
5623	spin_lock_irq(lock: &phba->hbalock);
5624	phba->sli.sli_flag &= ~LPFC_SLI_ACTIVE;
5625	spin_unlock_irq(lock: &phba->hbalock);
5626	rc = -ENXIO;
5627	} else {
5628	/* Allow asynchronous mailbox command to go through */
5629	spin_lock_irq(lock: &phba->hbalock);
5630	phba->sli.sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
5631	spin_unlock_irq(lock: &phba->hbalock);
5632	done = 1;
5633
5634	if ((pmb->u.mb.un.varCfgPort.casabt == 1) &&
5635	(pmb->u.mb.un.varCfgPort.gasabt == 0))
5636	lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
5637	"3110 Port did not grant ASABT\n");
5638	}
5639	}
5640	if (!done) {
5641	rc = -EINVAL;
5642	goto do_prep_failed;
5643	}
5644	if (pmb->u.mb.un.varCfgPort.sli_mode == 3) {
5645	if (!pmb->u.mb.un.varCfgPort.cMA) {
5646	rc = -ENXIO;
5647	goto do_prep_failed;
5648	}
5649	if (phba->max_vpi && pmb->u.mb.un.varCfgPort.gmv) {
5650	phba->sli3_options |= LPFC_SLI3_NPIV_ENABLED;
5651	phba->max_vpi = pmb->u.mb.un.varCfgPort.max_vpi;
5652	phba->max_vports = (phba->max_vpi > phba->max_vports) ?
5653	phba->max_vpi : phba->max_vports;
5654
5655	} else
5656	phba->max_vpi = 0;
5657	if (pmb->u.mb.un.varCfgPort.gerbm)
5658	phba->sli3_options |= LPFC_SLI3_HBQ_ENABLED;
5659	if (pmb->u.mb.un.varCfgPort.gcrp)
5660	phba->sli3_options |= LPFC_SLI3_CRP_ENABLED;
5661
5662	phba->hbq_get = phba->mbox->us.s3_pgp.hbq_get;
5663	phba->port_gp = phba->mbox->us.s3_pgp.port;
5664
5665	if (phba->sli3_options & LPFC_SLI3_BG_ENABLED) {
5666	if (pmb->u.mb.un.varCfgPort.gbg == 0) {
5667	phba->cfg_enable_bg = 0;
5668	phba->sli3_options &= ~LPFC_SLI3_BG_ENABLED;
5669	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5670	"0443 Adapter did not grant "
5671	"BlockGuard\n");
5672	}
5673	}
5674	} else {
5675	phba->hbq_get = NULL;
5676	phba->port_gp = phba->mbox->us.s2.port;
5677	phba->max_vpi = 0;
5678	}
5679	do_prep_failed:
5680	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
5681	return rc;
5682	}
5683
5684
5685	/**
5686	* lpfc_sli_hba_setup - SLI initialization function
5687	* @phba: Pointer to HBA context object.
5688	*
5689	* This function is the main SLI initialization function. This function
5690	* is called by the HBA initialization code, HBA reset code and HBA
5691	* error attention handler code. Caller is not required to hold any
5692	* locks. This function issues config_port mailbox command to configure
5693	* the SLI, setup iocb rings and HBQ rings. In the end the function
5694	* calls the config_port_post function to issue init_link mailbox
5695	* command and to start the discovery. The function will return zero
5696	* if successful, else it will return negative error code.
5697	**/
5698	int
5699	lpfc_sli_hba_setup(struct lpfc_hba *phba)
5700	{
5701	uint32_t rc;
5702	int i;
5703	int longs;
5704
5705	/* Enable ISR already does config_port because of config_msi mbx */
5706	if (phba->hba_flag & HBA_NEEDS_CFG_PORT) {
5707	rc = lpfc_sli_config_port(phba, LPFC_SLI_REV3);
5708	if (rc)
5709	return -EIO;
5710	phba->hba_flag &= ~HBA_NEEDS_CFG_PORT;
5711	}
5712	phba->fcp_embed_io = 0; /* SLI4 FC support only */
5713
5714	if (phba->sli_rev == 3) {
5715	phba->iocb_cmd_size = SLI3_IOCB_CMD_SIZE;
5716	phba->iocb_rsp_size = SLI3_IOCB_RSP_SIZE;
5717	} else {
5718	phba->iocb_cmd_size = SLI2_IOCB_CMD_SIZE;
5719	phba->iocb_rsp_size = SLI2_IOCB_RSP_SIZE;
5720	phba->sli3_options = 0;
5721	}
5722
5723	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5724	"0444 Firmware in SLI %x mode. Max_vpi %d\n",
5725	phba->sli_rev, phba->max_vpi);
5726	rc = lpfc_sli_ring_map(phba);
5727
5728	if (rc)
5729	goto lpfc_sli_hba_setup_error;
5730
5731	/* Initialize VPIs. */
5732	if (phba->sli_rev == LPFC_SLI_REV3) {
5733	/*
5734	* The VPI bitmask and physical ID array are allocated
5735	* and initialized once only - at driver load. A port
5736	* reset doesn't need to reinitialize this memory.
5737	*/
5738	if ((phba->vpi_bmask == NULL) && (phba->vpi_ids == NULL)) {
5739	longs = (phba->max_vpi + BITS_PER_LONG) / BITS_PER_LONG;
5740	phba->vpi_bmask = kcalloc(n: longs,
5741	size: sizeof(unsigned long),
5742	GFP_KERNEL);
5743	if (!phba->vpi_bmask) {
5744	rc = -ENOMEM;
5745	goto lpfc_sli_hba_setup_error;
5746	}
5747
5748	phba->vpi_ids = kcalloc(n: phba->max_vpi + 1,
5749	size: sizeof(uint16_t),
5750	GFP_KERNEL);
5751	if (!phba->vpi_ids) {
5752	kfree(objp: phba->vpi_bmask);
5753	rc = -ENOMEM;
5754	goto lpfc_sli_hba_setup_error;
5755	}
5756	for (i = 0; i < phba->max_vpi; i++)
5757	phba->vpi_ids[i] = i;
5758	}
5759	}
5760
5761	/* Init HBQs */
5762	if (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED) {
5763	rc = lpfc_sli_hbq_setup(phba);
5764	if (rc)
5765	goto lpfc_sli_hba_setup_error;
5766	}
5767	spin_lock_irq(lock: &phba->hbalock);
5768	phba->sli.sli_flag |= LPFC_PROCESS_LA;
5769	spin_unlock_irq(lock: &phba->hbalock);
5770
5771	rc = lpfc_config_port_post(phba);
5772	if (rc)
5773	goto lpfc_sli_hba_setup_error;
5774
5775	return rc;
5776
5777	lpfc_sli_hba_setup_error:
5778	phba->link_state = LPFC_HBA_ERROR;
5779	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5780	"0445 Firmware initialization failed\n");
5781	return rc;
5782	}
5783
5784	/**
5785	* lpfc_sli4_read_fcoe_params - Read fcoe params from conf region
5786	* @phba: Pointer to HBA context object.
5787	*
5788	* This function issue a dump mailbox command to read config region
5789	* 23 and parse the records in the region and populate driver
5790	* data structure.
5791	**/
5792	static int
5793	lpfc_sli4_read_fcoe_params(struct lpfc_hba *phba)
5794	{
5795	LPFC_MBOXQ_t *mboxq;
5796	struct lpfc_dmabuf *mp;
5797	struct lpfc_mqe *mqe;
5798	uint32_t data_length;
5799	int rc;
5800
5801	/* Program the default value of vlan_id and fc_map */
5802	phba->valid_vlan = 0;
5803	phba->fc_map[0] = LPFC_FCOE_FCF_MAP0;
5804	phba->fc_map[1] = LPFC_FCOE_FCF_MAP1;
5805	phba->fc_map[2] = LPFC_FCOE_FCF_MAP2;
5806
5807	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
5808	if (!mboxq)
5809	return -ENOMEM;
5810
5811	mqe = &mboxq->u.mqe;
5812	if (lpfc_sli4_dump_cfg_rg23(phba, mboxq)) {
5813	rc = -ENOMEM;
5814	goto out_free_mboxq;
5815	}
5816
5817	mp = (struct lpfc_dmabuf *)mboxq->ctx_buf;
5818	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
5819
5820	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
5821	"(%d):2571 Mailbox cmd x%x Status x%x "
5822	"Data: x%x x%x x%x x%x x%x x%x x%x x%x x%x "
5823	"x%x x%x x%x x%x x%x x%x x%x x%x x%x "
5824	"CQ: x%x x%x x%x x%x\n",
5825	mboxq->vport ? mboxq->vport->vpi : 0,
5826	bf_get(lpfc_mqe_command, mqe),
5827	bf_get(lpfc_mqe_status, mqe),
5828	mqe->un.mb_words[0], mqe->un.mb_words[1],
5829	mqe->un.mb_words[2], mqe->un.mb_words[3],
5830	mqe->un.mb_words[4], mqe->un.mb_words[5],
5831	mqe->un.mb_words[6], mqe->un.mb_words[7],
5832	mqe->un.mb_words[8], mqe->un.mb_words[9],
5833	mqe->un.mb_words[10], mqe->un.mb_words[11],
5834	mqe->un.mb_words[12], mqe->un.mb_words[13],
5835	mqe->un.mb_words[14], mqe->un.mb_words[15],
5836	mqe->un.mb_words[16], mqe->un.mb_words[50],
5837	mboxq->mcqe.word0,
5838	mboxq->mcqe.mcqe_tag0, mboxq->mcqe.mcqe_tag1,
5839	mboxq->mcqe.trailer);
5840
5841	if (rc) {
5842	rc = -EIO;
5843	goto out_free_mboxq;
5844	}
5845	data_length = mqe->un.mb_words[5];
5846	if (data_length > DMP_RGN23_SIZE) {
5847	rc = -EIO;
5848	goto out_free_mboxq;
5849	}
5850
5851	lpfc_parse_fcoe_conf(phba, mp->virt, data_length);
5852	rc = 0;
5853
5854	out_free_mboxq:
5855	lpfc_mbox_rsrc_cleanup(phba, mbox: mboxq, locked: MBOX_THD_UNLOCKED);
5856	return rc;
5857	}
5858
5859	/**
5860	* lpfc_sli4_read_rev - Issue READ_REV and collect vpd data
5861	* @phba: pointer to lpfc hba data structure.
5862	* @mboxq: pointer to the LPFC_MBOXQ_t structure.
5863	* @vpd: pointer to the memory to hold resulting port vpd data.
5864	* @vpd_size: On input, the number of bytes allocated to @vpd.
5865	* On output, the number of data bytes in @vpd.
5866	*
5867	* This routine executes a READ_REV SLI4 mailbox command. In
5868	* addition, this routine gets the port vpd data.
5869	*
5870	* Return codes
5871	* 0 - successful
5872	* -ENOMEM - could not allocated memory.
5873	**/
5874	static int
5875	lpfc_sli4_read_rev(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq,
5876	uint8_t *vpd, uint32_t *vpd_size)
5877	{
5878	int rc = 0;
5879	uint32_t dma_size;
5880	struct lpfc_dmabuf *dmabuf;
5881	struct lpfc_mqe *mqe;
5882
5883	dmabuf = kzalloc(size: sizeof(struct lpfc_dmabuf), GFP_KERNEL);
5884	if (!dmabuf)
5885	return -ENOMEM;
5886
5887	/*
5888	* Get a DMA buffer for the vpd data resulting from the READ_REV
5889	* mailbox command.
5890	*/
5891	dma_size = *vpd_size;
5892	dmabuf->virt = dma_alloc_coherent(dev: &phba->pcidev->dev, size: dma_size,
5893	dma_handle: &dmabuf->phys, GFP_KERNEL);
5894	if (!dmabuf->virt) {
5895	kfree(objp: dmabuf);
5896	return -ENOMEM;
5897	}
5898
5899	/*
5900	* The SLI4 implementation of READ_REV conflicts at word1,
5901	* bits 31:16 and SLI4 adds vpd functionality not present
5902	* in SLI3. This code corrects the conflicts.
5903	*/
5904	lpfc_read_rev(phba, mboxq);
5905	mqe = &mboxq->u.mqe;
5906	mqe->un.read_rev.vpd_paddr_high = putPaddrHigh(dmabuf->phys);
5907	mqe->un.read_rev.vpd_paddr_low = putPaddrLow(dmabuf->phys);
5908	mqe->un.read_rev.word1 &= 0x0000FFFF;
5909	bf_set(lpfc_mbx_rd_rev_vpd, &mqe->un.read_rev, 1);
5910	bf_set(lpfc_mbx_rd_rev_avail_len, &mqe->un.read_rev, dma_size);
5911
5912	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
5913	if (rc) {
5914	dma_free_coherent(dev: &phba->pcidev->dev, size: dma_size,
5915	cpu_addr: dmabuf->virt, dma_handle: dmabuf->phys);
5916	kfree(objp: dmabuf);
5917	return -EIO;
5918	}
5919
5920	/*
5921	* The available vpd length cannot be bigger than the
5922	* DMA buffer passed to the port. Catch the less than
5923	* case and update the caller's size.
5924	*/
5925	if (mqe->un.read_rev.avail_vpd_len < *vpd_size)
5926	*vpd_size = mqe->un.read_rev.avail_vpd_len;
5927
5928	memcpy(vpd, dmabuf->virt, *vpd_size);
5929
5930	dma_free_coherent(dev: &phba->pcidev->dev, size: dma_size,
5931	cpu_addr: dmabuf->virt, dma_handle: dmabuf->phys);
5932	kfree(objp: dmabuf);
5933	return 0;
5934	}
5935
5936	/**
5937	* lpfc_sli4_get_ctl_attr - Retrieve SLI4 device controller attributes
5938	* @phba: pointer to lpfc hba data structure.
5939	*
5940	* This routine retrieves SLI4 device physical port name this PCI function
5941	* is attached to.
5942	*
5943	* Return codes
5944	* 0 - successful
5945	* otherwise - failed to retrieve controller attributes
5946	**/
5947	static int
5948	lpfc_sli4_get_ctl_attr(struct lpfc_hba *phba)
5949	{
5950	LPFC_MBOXQ_t *mboxq;
5951	struct lpfc_mbx_get_cntl_attributes *mbx_cntl_attr;
5952	struct lpfc_controller_attribute *cntl_attr;
5953	void *virtaddr = NULL;
5954	uint32_t alloclen, reqlen;
5955	uint32_t shdr_status, shdr_add_status;
5956	union lpfc_sli4_cfg_shdr *shdr;
5957	int rc;
5958
5959	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
5960	if (!mboxq)
5961	return -ENOMEM;
5962
5963	/* Send COMMON_GET_CNTL_ATTRIBUTES mbox cmd */
5964	reqlen = sizeof(struct lpfc_mbx_get_cntl_attributes);
5965	alloclen = lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
5966	LPFC_MBOX_OPCODE_GET_CNTL_ATTRIBUTES, reqlen,
5967	LPFC_SLI4_MBX_NEMBED);
5968
5969	if (alloclen < reqlen) {
5970	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5971	"3084 Allocated DMA memory size (%d) is "
5972	"less than the requested DMA memory size "
5973	"(%d)\n", alloclen, reqlen);
5974	rc = -ENOMEM;
5975	goto out_free_mboxq;
5976	}
5977	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
5978	virtaddr = mboxq->sge_array->addr[0];
5979	mbx_cntl_attr = (struct lpfc_mbx_get_cntl_attributes *)virtaddr;
5980	shdr = &mbx_cntl_attr->cfg_shdr;
5981	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
5982	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
5983	if (shdr_status || shdr_add_status || rc) {
5984	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
5985	"3085 Mailbox x%x (x%x/x%x) failed, "
5986	"rc:x%x, status:x%x, add_status:x%x\n",
5987	bf_get(lpfc_mqe_command, &mboxq->u.mqe),
5988	lpfc_sli_config_mbox_subsys_get(phba, mboxq),
5989	lpfc_sli_config_mbox_opcode_get(phba, mboxq),
5990	rc, shdr_status, shdr_add_status);
5991	rc = -ENXIO;
5992	goto out_free_mboxq;
5993	}
5994
5995	cntl_attr = &mbx_cntl_attr->cntl_attr;
5996	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_VAL;
5997	phba->sli4_hba.lnk_info.lnk_tp =
5998	bf_get(lpfc_cntl_attr_lnk_type, cntl_attr);
5999	phba->sli4_hba.lnk_info.lnk_no =
6000	bf_get(lpfc_cntl_attr_lnk_numb, cntl_attr);
6001	phba->sli4_hba.flash_id = bf_get(lpfc_cntl_attr_flash_id, cntl_attr);
6002	phba->sli4_hba.asic_rev = bf_get(lpfc_cntl_attr_asic_rev, cntl_attr);
6003
6004	memset(phba->BIOSVersion, 0, sizeof(phba->BIOSVersion));
6005	strlcat(p: phba->BIOSVersion, q: (char *)cntl_attr->bios_ver_str,
6006	avail: sizeof(phba->BIOSVersion));
6007
6008	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
6009	"3086 lnk_type:%d, lnk_numb:%d, bios_ver:%s, "
6010	"flash_id: x%02x, asic_rev: x%02x\n",
6011	phba->sli4_hba.lnk_info.lnk_tp,
6012	phba->sli4_hba.lnk_info.lnk_no,
6013	phba->BIOSVersion, phba->sli4_hba.flash_id,
6014	phba->sli4_hba.asic_rev);
6015	out_free_mboxq:
6016	if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG)
6017	lpfc_sli4_mbox_cmd_free(phba, mboxq);
6018	else
6019	mempool_free(element: mboxq, pool: phba->mbox_mem_pool);
6020	return rc;
6021	}
6022
6023	/**
6024	* lpfc_sli4_retrieve_pport_name - Retrieve SLI4 device physical port name
6025	* @phba: pointer to lpfc hba data structure.
6026	*
6027	* This routine retrieves SLI4 device physical port name this PCI function
6028	* is attached to.
6029	*
6030	* Return codes
6031	* 0 - successful
6032	* otherwise - failed to retrieve physical port name
6033	**/
6034	static int
6035	lpfc_sli4_retrieve_pport_name(struct lpfc_hba *phba)
6036	{
6037	LPFC_MBOXQ_t *mboxq;
6038	struct lpfc_mbx_get_port_name *get_port_name;
6039	uint32_t shdr_status, shdr_add_status;
6040	union lpfc_sli4_cfg_shdr *shdr;
6041	char cport_name = 0;
6042	int rc;
6043
6044	/* We assume nothing at this point */
6045	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL;
6046	phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_NON;
6047
6048	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
6049	if (!mboxq)
6050	return -ENOMEM;
6051	/* obtain link type and link number via READ_CONFIG */
6052	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL;
6053	lpfc_sli4_read_config(phba);
6054
6055	if (phba->sli4_hba.fawwpn_flag & LPFC_FAWWPN_CONFIG)
6056	phba->sli4_hba.fawwpn_flag |= LPFC_FAWWPN_FABRIC;
6057
6058	if (phba->sli4_hba.lnk_info.lnk_dv == LPFC_LNK_DAT_VAL)
6059	goto retrieve_ppname;
6060
6061	/* obtain link type and link number via COMMON_GET_CNTL_ATTRIBUTES */
6062	rc = lpfc_sli4_get_ctl_attr(phba);
6063	if (rc)
6064	goto out_free_mboxq;
6065
6066	retrieve_ppname:
6067	lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
6068	LPFC_MBOX_OPCODE_GET_PORT_NAME,
6069	sizeof(struct lpfc_mbx_get_port_name) -
6070	sizeof(struct lpfc_sli4_cfg_mhdr),
6071	LPFC_SLI4_MBX_EMBED);
6072	get_port_name = &mboxq->u.mqe.un.get_port_name;
6073	shdr = (union lpfc_sli4_cfg_shdr *)&get_port_name->header.cfg_shdr;
6074	bf_set(lpfc_mbox_hdr_version, &shdr->request, LPFC_OPCODE_VERSION_1);
6075	bf_set(lpfc_mbx_get_port_name_lnk_type, &get_port_name->u.request,
6076	phba->sli4_hba.lnk_info.lnk_tp);
6077	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
6078	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
6079	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
6080	if (shdr_status || shdr_add_status || rc) {
6081	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
6082	"3087 Mailbox x%x (x%x/x%x) failed: "
6083	"rc:x%x, status:x%x, add_status:x%x\n",
6084	bf_get(lpfc_mqe_command, &mboxq->u.mqe),
6085	lpfc_sli_config_mbox_subsys_get(phba, mboxq),
6086	lpfc_sli_config_mbox_opcode_get(phba, mboxq),
6087	rc, shdr_status, shdr_add_status);
6088	rc = -ENXIO;
6089	goto out_free_mboxq;
6090	}
6091	switch (phba->sli4_hba.lnk_info.lnk_no) {
6092	case LPFC_LINK_NUMBER_0:
6093	cport_name = bf_get(lpfc_mbx_get_port_name_name0,
6094	&get_port_name->u.response);
6095	phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_GET;
6096	break;
6097	case LPFC_LINK_NUMBER_1:
6098	cport_name = bf_get(lpfc_mbx_get_port_name_name1,
6099	&get_port_name->u.response);
6100	phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_GET;
6101	break;
6102	case LPFC_LINK_NUMBER_2:
6103	cport_name = bf_get(lpfc_mbx_get_port_name_name2,
6104	&get_port_name->u.response);
6105	phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_GET;
6106	break;
6107	case LPFC_LINK_NUMBER_3:
6108	cport_name = bf_get(lpfc_mbx_get_port_name_name3,
6109	&get_port_name->u.response);
6110	phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_GET;
6111	break;
6112	default:
6113	break;
6114	}
6115
6116	if (phba->sli4_hba.pport_name_sta == LPFC_SLI4_PPNAME_GET) {
6117	phba->Port[0] = cport_name;
6118	phba->Port[1] = '\0';
6119	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
6120	"3091 SLI get port name: %s\n", phba->Port);
6121	}
6122
6123	out_free_mboxq:
6124	if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG)
6125	lpfc_sli4_mbox_cmd_free(phba, mboxq);
6126	else
6127	mempool_free(element: mboxq, pool: phba->mbox_mem_pool);
6128	return rc;
6129	}
6130
6131	/**
6132	* lpfc_sli4_arm_cqeq_intr - Arm sli-4 device completion and event queues
6133	* @phba: pointer to lpfc hba data structure.
6134	*
6135	* This routine is called to explicitly arm the SLI4 device's completion and
6136	* event queues
6137	**/
6138	static void
6139	lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba)
6140	{
6141	int qidx;
6142	struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba;
6143	struct lpfc_sli4_hdw_queue *qp;
6144	struct lpfc_queue *eq;
6145
6146	sli4_hba->sli4_write_cq_db(phba, sli4_hba->mbx_cq, 0, LPFC_QUEUE_REARM);
6147	sli4_hba->sli4_write_cq_db(phba, sli4_hba->els_cq, 0, LPFC_QUEUE_REARM);
6148	if (sli4_hba->nvmels_cq)
6149	sli4_hba->sli4_write_cq_db(phba, sli4_hba->nvmels_cq, 0,
6150	LPFC_QUEUE_REARM);
6151
6152	if (sli4_hba->hdwq) {
6153	/* Loop thru all Hardware Queues */
6154	for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) {
6155	qp = &sli4_hba->hdwq[qidx];
6156	/* ARM the corresponding CQ */
6157	sli4_hba->sli4_write_cq_db(phba, qp->io_cq, 0,
6158	LPFC_QUEUE_REARM);
6159	}
6160
6161	/* Loop thru all IRQ vectors */
6162	for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) {
6163	eq = sli4_hba->hba_eq_hdl[qidx].eq;
6164	/* ARM the corresponding EQ */
6165	sli4_hba->sli4_write_eq_db(phba, eq,
6166	0, LPFC_QUEUE_REARM);
6167	}
6168	}
6169
6170	if (phba->nvmet_support) {
6171	for (qidx = 0; qidx < phba->cfg_nvmet_mrq; qidx++) {
6172	sli4_hba->sli4_write_cq_db(phba,
6173	sli4_hba->nvmet_cqset[qidx], 0,
6174	LPFC_QUEUE_REARM);
6175	}
6176	}
6177	}
6178
6179	/**
6180	* lpfc_sli4_get_avail_extnt_rsrc - Get available resource extent count.
6181	* @phba: Pointer to HBA context object.
6182	* @type: The resource extent type.
6183	* @extnt_count: buffer to hold port available extent count.
6184	* @extnt_size: buffer to hold element count per extent.
6185	*
6186	* This function calls the port and retrievs the number of available
6187	* extents and their size for a particular extent type.
6188	*
6189	* Returns: 0 if successful. Nonzero otherwise.
6190	**/
6191	int
6192	lpfc_sli4_get_avail_extnt_rsrc(struct lpfc_hba *phba, uint16_t type,
6193	uint16_t *extnt_count, uint16_t *extnt_size)
6194	{
6195	int rc = 0;
6196	uint32_t length;
6197	uint32_t mbox_tmo;
6198	struct lpfc_mbx_get_rsrc_extent_info *rsrc_info;
6199	LPFC_MBOXQ_t *mbox;
6200
6201	*extnt_count = 0;
6202	*extnt_size = 0;
6203
6204	mbox = (LPFC_MBOXQ_t *) mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
6205	if (!mbox)
6206	return -ENOMEM;
6207
6208	/* Find out how many extents are available for this resource type */
6209	length = (sizeof(struct lpfc_mbx_get_rsrc_extent_info) -
6210	sizeof(struct lpfc_sli4_cfg_mhdr));
6211	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
6212	LPFC_MBOX_OPCODE_GET_RSRC_EXTENT_INFO,
6213	length, LPFC_SLI4_MBX_EMBED);
6214
6215	/* Send an extents count of 0 - the GET doesn't use it. */
6216	rc = lpfc_sli4_mbox_rsrc_extent(phba, mbox, 0, type,
6217	LPFC_SLI4_MBX_EMBED);
6218	if (unlikely(rc)) {
6219	rc = -EIO;
6220	goto err_exit;
6221	}
6222
6223	if (!phba->sli4_hba.intr_enable)
6224	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
6225	else {
6226	mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
6227	rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
6228	}
6229	if (unlikely(rc)) {
6230	rc = -EIO;
6231	goto err_exit;
6232	}
6233
6234	rsrc_info = &mbox->u.mqe.un.rsrc_extent_info;
6235	if (bf_get(lpfc_mbox_hdr_status,
6236	&rsrc_info->header.cfg_shdr.response)) {
6237	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6238	"2930 Failed to get resource extents "
6239	"Status 0x%x Add'l Status 0x%x\n",
6240	bf_get(lpfc_mbox_hdr_status,
6241	&rsrc_info->header.cfg_shdr.response),
6242	bf_get(lpfc_mbox_hdr_add_status,
6243	&rsrc_info->header.cfg_shdr.response));
6244	rc = -EIO;
6245	goto err_exit;
6246	}
6247
6248	*extnt_count = bf_get(lpfc_mbx_get_rsrc_extent_info_cnt,
6249	&rsrc_info->u.rsp);
6250	*extnt_size = bf_get(lpfc_mbx_get_rsrc_extent_info_size,
6251	&rsrc_info->u.rsp);
6252
6253	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
6254	"3162 Retrieved extents type-%d from port: count:%d, "
6255	"size:%d\n", type, *extnt_count, *extnt_size);
6256
6257	err_exit:
6258	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
6259	return rc;
6260	}
6261
6262	/**
6263	* lpfc_sli4_chk_avail_extnt_rsrc - Check for available SLI4 resource extents.
6264	* @phba: Pointer to HBA context object.
6265	* @type: The extent type to check.
6266	*
6267	* This function reads the current available extents from the port and checks
6268	* if the extent count or extent size has changed since the last access.
6269	* Callers use this routine post port reset to understand if there is a
6270	* extent reprovisioning requirement.
6271	*
6272	* Returns:
6273	* -Error: error indicates problem.
6274	* 1: Extent count or size has changed.
6275	* 0: No changes.
6276	**/
6277	static int
6278	lpfc_sli4_chk_avail_extnt_rsrc(struct lpfc_hba *phba, uint16_t type)
6279	{
6280	uint16_t curr_ext_cnt, rsrc_ext_cnt;
6281	uint16_t size_diff, rsrc_ext_size;
6282	int rc = 0;
6283	struct lpfc_rsrc_blks *rsrc_entry;
6284	struct list_head *rsrc_blk_list = NULL;
6285
6286	size_diff = 0;
6287	curr_ext_cnt = 0;
6288	rc = lpfc_sli4_get_avail_extnt_rsrc(phba, type,
6289	extnt_count: &rsrc_ext_cnt,
6290	extnt_size: &rsrc_ext_size);
6291	if (unlikely(rc))
6292	return -EIO;
6293
6294	switch (type) {
6295	case LPFC_RSC_TYPE_FCOE_RPI:
6296	rsrc_blk_list = &phba->sli4_hba.lpfc_rpi_blk_list;
6297	break;
6298	case LPFC_RSC_TYPE_FCOE_VPI:
6299	rsrc_blk_list = &phba->lpfc_vpi_blk_list;
6300	break;
6301	case LPFC_RSC_TYPE_FCOE_XRI:
6302	rsrc_blk_list = &phba->sli4_hba.lpfc_xri_blk_list;
6303	break;
6304	case LPFC_RSC_TYPE_FCOE_VFI:
6305	rsrc_blk_list = &phba->sli4_hba.lpfc_vfi_blk_list;
6306	break;
6307	default:
6308	break;
6309	}
6310
6311	list_for_each_entry(rsrc_entry, rsrc_blk_list, list) {
6312	curr_ext_cnt++;
6313	if (rsrc_entry->rsrc_size != rsrc_ext_size)
6314	size_diff++;
6315	}
6316
6317	if (curr_ext_cnt != rsrc_ext_cnt || size_diff != 0)
6318	rc = 1;
6319
6320	return rc;
6321	}
6322
6323	/**
6324	* lpfc_sli4_cfg_post_extnts -
6325	* @phba: Pointer to HBA context object.
6326	* @extnt_cnt: number of available extents.
6327	* @type: the extent type (rpi, xri, vfi, vpi).
6328	* @emb: buffer to hold either MBX_EMBED or MBX_NEMBED operation.
6329	* @mbox: pointer to the caller's allocated mailbox structure.
6330	*
6331	* This function executes the extents allocation request. It also
6332	* takes care of the amount of memory needed to allocate or get the
6333	* allocated extents. It is the caller's responsibility to evaluate
6334	* the response.
6335	*
6336	* Returns:
6337	* -Error: Error value describes the condition found.
6338	* 0: if successful
6339	**/
6340	static int
6341	lpfc_sli4_cfg_post_extnts(struct lpfc_hba *phba, uint16_t extnt_cnt,
6342	uint16_t type, bool *emb, LPFC_MBOXQ_t *mbox)
6343	{
6344	int rc = 0;
6345	uint32_t req_len;
6346	uint32_t emb_len;
6347	uint32_t alloc_len, mbox_tmo;
6348
6349	/* Calculate the total requested length of the dma memory */
6350	req_len = extnt_cnt * sizeof(uint16_t);
6351
6352	/*
6353	* Calculate the size of an embedded mailbox. The uint32_t
6354	* accounts for extents-specific word.
6355	*/
6356	emb_len = sizeof(MAILBOX_t) - sizeof(struct mbox_header) -
6357	sizeof(uint32_t);
6358
6359	/*
6360	* Presume the allocation and response will fit into an embedded
6361	* mailbox. If not true, reconfigure to a non-embedded mailbox.
6362	*/
6363	*emb = LPFC_SLI4_MBX_EMBED;
6364	if (req_len > emb_len) {
6365	req_len = extnt_cnt * sizeof(uint16_t) +
6366	sizeof(union lpfc_sli4_cfg_shdr) +
6367	sizeof(uint32_t);
6368	*emb = LPFC_SLI4_MBX_NEMBED;
6369	}
6370
6371	alloc_len = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
6372	LPFC_MBOX_OPCODE_ALLOC_RSRC_EXTENT,
6373	req_len, *emb);
6374	if (alloc_len < req_len) {
6375	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6376	"2982 Allocated DMA memory size (x%x) is "
6377	"less than the requested DMA memory "
6378	"size (x%x)\n", alloc_len, req_len);
6379	return -ENOMEM;
6380	}
6381	rc = lpfc_sli4_mbox_rsrc_extent(phba, mbox, extnt_cnt, type, *emb);
6382	if (unlikely(rc))
6383	return -EIO;
6384
6385	if (!phba->sli4_hba.intr_enable)
6386	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
6387	else {
6388	mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
6389	rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
6390	}
6391
6392	if (unlikely(rc))
6393	rc = -EIO;
6394	return rc;
6395	}
6396
6397	/**
6398	* lpfc_sli4_alloc_extent - Allocate an SLI4 resource extent.
6399	* @phba: Pointer to HBA context object.
6400	* @type: The resource extent type to allocate.
6401	*
6402	* This function allocates the number of elements for the specified
6403	* resource type.
6404	**/
6405	static int
6406	lpfc_sli4_alloc_extent(struct lpfc_hba *phba, uint16_t type)
6407	{
6408	bool emb = false;
6409	uint16_t rsrc_id_cnt, rsrc_cnt, rsrc_size;
6410	uint16_t rsrc_id, rsrc_start, j, k;
6411	uint16_t *ids;
6412	int i, rc;
6413	unsigned long longs;
6414	unsigned long *bmask;
6415	struct lpfc_rsrc_blks *rsrc_blks;
6416	LPFC_MBOXQ_t *mbox;
6417	uint32_t length;
6418	struct lpfc_id_range *id_array = NULL;
6419	void *virtaddr = NULL;
6420	struct lpfc_mbx_nembed_rsrc_extent *n_rsrc;
6421	struct lpfc_mbx_alloc_rsrc_extents *rsrc_ext;
6422	struct list_head *ext_blk_list;
6423
6424	rc = lpfc_sli4_get_avail_extnt_rsrc(phba, type,
6425	extnt_count: &rsrc_cnt,
6426	extnt_size: &rsrc_size);
6427	if (unlikely(rc))
6428	return -EIO;
6429
6430	if ((rsrc_cnt == 0) || (rsrc_size == 0)) {
6431	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6432	"3009 No available Resource Extents "
6433	"for resource type 0x%x: Count: 0x%x, "
6434	"Size 0x%x\n", type, rsrc_cnt,
6435	rsrc_size);
6436	return -ENOMEM;
6437	}
6438
6439	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_INIT | LOG_SLI,
6440	"2903 Post resource extents type-0x%x: "
6441	"count:%d, size %d\n", type, rsrc_cnt, rsrc_size);
6442
6443	mbox = (LPFC_MBOXQ_t *) mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
6444	if (!mbox)
6445	return -ENOMEM;
6446
6447	rc = lpfc_sli4_cfg_post_extnts(phba, extnt_cnt: rsrc_cnt, type, emb: &emb, mbox);
6448	if (unlikely(rc)) {
6449	rc = -EIO;
6450	goto err_exit;
6451	}
6452
6453	/*
6454	* Figure out where the response is located. Then get local pointers
6455	* to the response data. The port does not guarantee to respond to
6456	* all extents counts request so update the local variable with the
6457	* allocated count from the port.
6458	*/
6459	if (emb == LPFC_SLI4_MBX_EMBED) {
6460	rsrc_ext = &mbox->u.mqe.un.alloc_rsrc_extents;
6461	id_array = &rsrc_ext->u.rsp.id[0];
6462	rsrc_cnt = bf_get(lpfc_mbx_rsrc_cnt, &rsrc_ext->u.rsp);
6463	} else {
6464	virtaddr = mbox->sge_array->addr[0];
6465	n_rsrc = (struct lpfc_mbx_nembed_rsrc_extent *) virtaddr;
6466	rsrc_cnt = bf_get(lpfc_mbx_rsrc_cnt, n_rsrc);
6467	id_array = &n_rsrc->id;
6468	}
6469
6470	longs = ((rsrc_cnt * rsrc_size) + BITS_PER_LONG - 1) / BITS_PER_LONG;
6471	rsrc_id_cnt = rsrc_cnt * rsrc_size;
6472
6473	/*
6474	* Based on the resource size and count, correct the base and max
6475	* resource values.
6476	*/
6477	length = sizeof(struct lpfc_rsrc_blks);
6478	switch (type) {
6479	case LPFC_RSC_TYPE_FCOE_RPI:
6480	phba->sli4_hba.rpi_bmask = kcalloc(n: longs,
6481	size: sizeof(unsigned long),
6482	GFP_KERNEL);
6483	if (unlikely(!phba->sli4_hba.rpi_bmask)) {
6484	rc = -ENOMEM;
6485	goto err_exit;
6486	}
6487	phba->sli4_hba.rpi_ids = kcalloc(n: rsrc_id_cnt,
6488	size: sizeof(uint16_t),
6489	GFP_KERNEL);
6490	if (unlikely(!phba->sli4_hba.rpi_ids)) {
6491	kfree(objp: phba->sli4_hba.rpi_bmask);
6492	rc = -ENOMEM;
6493	goto err_exit;
6494	}
6495
6496	/*
6497	* The next_rpi was initialized with the maximum available
6498	* count but the port may allocate a smaller number. Catch
6499	* that case and update the next_rpi.
6500	*/
6501	phba->sli4_hba.next_rpi = rsrc_id_cnt;
6502
6503	/* Initialize local ptrs for common extent processing later. */
6504	bmask = phba->sli4_hba.rpi_bmask;
6505	ids = phba->sli4_hba.rpi_ids;
6506	ext_blk_list = &phba->sli4_hba.lpfc_rpi_blk_list;
6507	break;
6508	case LPFC_RSC_TYPE_FCOE_VPI:
6509	phba->vpi_bmask = kcalloc(n: longs, size: sizeof(unsigned long),
6510	GFP_KERNEL);
6511	if (unlikely(!phba->vpi_bmask)) {
6512	rc = -ENOMEM;
6513	goto err_exit;
6514	}
6515	phba->vpi_ids = kcalloc(n: rsrc_id_cnt, size: sizeof(uint16_t),
6516	GFP_KERNEL);
6517	if (unlikely(!phba->vpi_ids)) {
6518	kfree(objp: phba->vpi_bmask);
6519	rc = -ENOMEM;
6520	goto err_exit;
6521	}
6522
6523	/* Initialize local ptrs for common extent processing later. */
6524	bmask = phba->vpi_bmask;
6525	ids = phba->vpi_ids;
6526	ext_blk_list = &phba->lpfc_vpi_blk_list;
6527	break;
6528	case LPFC_RSC_TYPE_FCOE_XRI:
6529	phba->sli4_hba.xri_bmask = kcalloc(n: longs,
6530	size: sizeof(unsigned long),
6531	GFP_KERNEL);
6532	if (unlikely(!phba->sli4_hba.xri_bmask)) {
6533	rc = -ENOMEM;
6534	goto err_exit;
6535	}
6536	phba->sli4_hba.max_cfg_param.xri_used = 0;
6537	phba->sli4_hba.xri_ids = kcalloc(n: rsrc_id_cnt,
6538	size: sizeof(uint16_t),
6539	GFP_KERNEL);
6540	if (unlikely(!phba->sli4_hba.xri_ids)) {
6541	kfree(objp: phba->sli4_hba.xri_bmask);
6542	rc = -ENOMEM;
6543	goto err_exit;
6544	}
6545
6546	/* Initialize local ptrs for common extent processing later. */
6547	bmask = phba->sli4_hba.xri_bmask;
6548	ids = phba->sli4_hba.xri_ids;
6549	ext_blk_list = &phba->sli4_hba.lpfc_xri_blk_list;
6550	break;
6551	case LPFC_RSC_TYPE_FCOE_VFI:
6552	phba->sli4_hba.vfi_bmask = kcalloc(n: longs,
6553	size: sizeof(unsigned long),
6554	GFP_KERNEL);
6555	if (unlikely(!phba->sli4_hba.vfi_bmask)) {
6556	rc = -ENOMEM;
6557	goto err_exit;
6558	}
6559	phba->sli4_hba.vfi_ids = kcalloc(n: rsrc_id_cnt,
6560	size: sizeof(uint16_t),
6561	GFP_KERNEL);
6562	if (unlikely(!phba->sli4_hba.vfi_ids)) {
6563	kfree(objp: phba->sli4_hba.vfi_bmask);
6564	rc = -ENOMEM;
6565	goto err_exit;
6566	}
6567
6568	/* Initialize local ptrs for common extent processing later. */
6569	bmask = phba->sli4_hba.vfi_bmask;
6570	ids = phba->sli4_hba.vfi_ids;
6571	ext_blk_list = &phba->sli4_hba.lpfc_vfi_blk_list;
6572	break;
6573	default:
6574	/* Unsupported Opcode. Fail call. */
6575	id_array = NULL;
6576	bmask = NULL;
6577	ids = NULL;
6578	ext_blk_list = NULL;
6579	goto err_exit;
6580	}
6581
6582	/*
6583	* Complete initializing the extent configuration with the
6584	* allocated ids assigned to this function. The bitmask serves
6585	* as an index into the array and manages the available ids. The
6586	* array just stores the ids communicated to the port via the wqes.
6587	*/
6588	for (i = 0, j = 0, k = 0; i < rsrc_cnt; i++) {
6589	if ((i % 2) == 0)
6590	rsrc_id = bf_get(lpfc_mbx_rsrc_id_word4_0,
6591	&id_array[k]);
6592	else
6593	rsrc_id = bf_get(lpfc_mbx_rsrc_id_word4_1,
6594	&id_array[k]);
6595
6596	rsrc_blks = kzalloc(size: length, GFP_KERNEL);
6597	if (unlikely(!rsrc_blks)) {
6598	rc = -ENOMEM;
6599	kfree(objp: bmask);
6600	kfree(objp: ids);
6601	goto err_exit;
6602	}
6603	rsrc_blks->rsrc_start = rsrc_id;
6604	rsrc_blks->rsrc_size = rsrc_size;
6605	list_add_tail(new: &rsrc_blks->list, head: ext_blk_list);
6606	rsrc_start = rsrc_id;
6607	if ((type == LPFC_RSC_TYPE_FCOE_XRI) && (j == 0)) {
6608	phba->sli4_hba.io_xri_start = rsrc_start +
6609	lpfc_sli4_get_iocb_cnt(phba);
6610	}
6611
6612	while (rsrc_id < (rsrc_start + rsrc_size)) {
6613	ids[j] = rsrc_id;
6614	rsrc_id++;
6615	j++;
6616	}
6617	/* Entire word processed. Get next word.*/
6618	if ((i % 2) == 1)
6619	k++;
6620	}
6621	err_exit:
6622	lpfc_sli4_mbox_cmd_free(phba, mbox);
6623	return rc;
6624	}
6625
6626
6627
6628	/**
6629	* lpfc_sli4_dealloc_extent - Deallocate an SLI4 resource extent.
6630	* @phba: Pointer to HBA context object.
6631	* @type: the extent's type.
6632	*
6633	* This function deallocates all extents of a particular resource type.
6634	* SLI4 does not allow for deallocating a particular extent range. It
6635	* is the caller's responsibility to release all kernel memory resources.
6636	**/
6637	static int
6638	lpfc_sli4_dealloc_extent(struct lpfc_hba *phba, uint16_t type)
6639	{
6640	int rc;
6641	uint32_t length, mbox_tmo = 0;
6642	LPFC_MBOXQ_t *mbox;
6643	struct lpfc_mbx_dealloc_rsrc_extents *dealloc_rsrc;
6644	struct lpfc_rsrc_blks *rsrc_blk, *rsrc_blk_next;
6645
6646	mbox = (LPFC_MBOXQ_t *) mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
6647	if (!mbox)
6648	return -ENOMEM;
6649
6650	/*
6651	* This function sends an embedded mailbox because it only sends the
6652	* the resource type. All extents of this type are released by the
6653	* port.
6654	*/
6655	length = (sizeof(struct lpfc_mbx_dealloc_rsrc_extents) -
6656	sizeof(struct lpfc_sli4_cfg_mhdr));
6657	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
6658	LPFC_MBOX_OPCODE_DEALLOC_RSRC_EXTENT,
6659	length, LPFC_SLI4_MBX_EMBED);
6660
6661	/* Send an extents count of 0 - the dealloc doesn't use it. */
6662	rc = lpfc_sli4_mbox_rsrc_extent(phba, mbox, 0, type,
6663	LPFC_SLI4_MBX_EMBED);
6664	if (unlikely(rc)) {
6665	rc = -EIO;
6666	goto out_free_mbox;
6667	}
6668	if (!phba->sli4_hba.intr_enable)
6669	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
6670	else {
6671	mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
6672	rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
6673	}
6674	if (unlikely(rc)) {
6675	rc = -EIO;
6676	goto out_free_mbox;
6677	}
6678
6679	dealloc_rsrc = &mbox->u.mqe.un.dealloc_rsrc_extents;
6680	if (bf_get(lpfc_mbox_hdr_status,
6681	&dealloc_rsrc->header.cfg_shdr.response)) {
6682	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6683	"2919 Failed to release resource extents "
6684	"for type %d - Status 0x%x Add'l Status 0x%x. "
6685	"Resource memory not released.\n",
6686	type,
6687	bf_get(lpfc_mbox_hdr_status,
6688	&dealloc_rsrc->header.cfg_shdr.response),
6689	bf_get(lpfc_mbox_hdr_add_status,
6690	&dealloc_rsrc->header.cfg_shdr.response));
6691	rc = -EIO;
6692	goto out_free_mbox;
6693	}
6694
6695	/* Release kernel memory resources for the specific type. */
6696	switch (type) {
6697	case LPFC_RSC_TYPE_FCOE_VPI:
6698	kfree(objp: phba->vpi_bmask);
6699	kfree(objp: phba->vpi_ids);
6700	bf_set(lpfc_vpi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
6701	list_for_each_entry_safe(rsrc_blk, rsrc_blk_next,
6702	&phba->lpfc_vpi_blk_list, list) {
6703	list_del_init(entry: &rsrc_blk->list);
6704	kfree(objp: rsrc_blk);
6705	}
6706	phba->sli4_hba.max_cfg_param.vpi_used = 0;
6707	break;
6708	case LPFC_RSC_TYPE_FCOE_XRI:
6709	kfree(objp: phba->sli4_hba.xri_bmask);
6710	kfree(objp: phba->sli4_hba.xri_ids);
6711	list_for_each_entry_safe(rsrc_blk, rsrc_blk_next,
6712	&phba->sli4_hba.lpfc_xri_blk_list, list) {
6713	list_del_init(entry: &rsrc_blk->list);
6714	kfree(objp: rsrc_blk);
6715	}
6716	break;
6717	case LPFC_RSC_TYPE_FCOE_VFI:
6718	kfree(objp: phba->sli4_hba.vfi_bmask);
6719	kfree(objp: phba->sli4_hba.vfi_ids);
6720	bf_set(lpfc_vfi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
6721	list_for_each_entry_safe(rsrc_blk, rsrc_blk_next,
6722	&phba->sli4_hba.lpfc_vfi_blk_list, list) {
6723	list_del_init(entry: &rsrc_blk->list);
6724	kfree(objp: rsrc_blk);
6725	}
6726	break;
6727	case LPFC_RSC_TYPE_FCOE_RPI:
6728	/* RPI bitmask and physical id array are cleaned up earlier. */
6729	list_for_each_entry_safe(rsrc_blk, rsrc_blk_next,
6730	&phba->sli4_hba.lpfc_rpi_blk_list, list) {
6731	list_del_init(entry: &rsrc_blk->list);
6732	kfree(objp: rsrc_blk);
6733	}
6734	break;
6735	default:
6736	break;
6737	}
6738
6739	bf_set(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
6740
6741	out_free_mbox:
6742	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
6743	return rc;
6744	}
6745
6746	static void
6747	lpfc_set_features(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox,
6748	uint32_t feature)
6749	{
6750	uint32_t len;
6751	u32 sig_freq = 0;
6752
6753	len = sizeof(struct lpfc_mbx_set_feature) -
6754	sizeof(struct lpfc_sli4_cfg_mhdr);
6755	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
6756	LPFC_MBOX_OPCODE_SET_FEATURES, len,
6757	LPFC_SLI4_MBX_EMBED);
6758
6759	switch (feature) {
6760	case LPFC_SET_UE_RECOVERY:
6761	bf_set(lpfc_mbx_set_feature_UER,
6762	&mbox->u.mqe.un.set_feature, 1);
6763	mbox->u.mqe.un.set_feature.feature = LPFC_SET_UE_RECOVERY;
6764	mbox->u.mqe.un.set_feature.param_len = 8;
6765	break;
6766	case LPFC_SET_MDS_DIAGS:
6767	bf_set(lpfc_mbx_set_feature_mds,
6768	&mbox->u.mqe.un.set_feature, 1);
6769	bf_set(lpfc_mbx_set_feature_mds_deep_loopbk,
6770	&mbox->u.mqe.un.set_feature, 1);
6771	mbox->u.mqe.un.set_feature.feature = LPFC_SET_MDS_DIAGS;
6772	mbox->u.mqe.un.set_feature.param_len = 8;
6773	break;
6774	case LPFC_SET_CGN_SIGNAL:
6775	if (phba->cmf_active_mode == LPFC_CFG_OFF)
6776	sig_freq = 0;
6777	else
6778	sig_freq = phba->cgn_sig_freq;
6779
6780	if (phba->cgn_reg_signal == EDC_CG_SIG_WARN_ALARM) {
6781	bf_set(lpfc_mbx_set_feature_CGN_alarm_freq,
6782	&mbox->u.mqe.un.set_feature, sig_freq);
6783	bf_set(lpfc_mbx_set_feature_CGN_warn_freq,
6784	&mbox->u.mqe.un.set_feature, sig_freq);
6785	}
6786
6787	if (phba->cgn_reg_signal == EDC_CG_SIG_WARN_ONLY)
6788	bf_set(lpfc_mbx_set_feature_CGN_warn_freq,
6789	&mbox->u.mqe.un.set_feature, sig_freq);
6790
6791	if (phba->cmf_active_mode == LPFC_CFG_OFF ||
6792	phba->cgn_reg_signal == EDC_CG_SIG_NOTSUPPORTED)
6793	sig_freq = 0;
6794	else
6795	sig_freq = lpfc_acqe_cgn_frequency;
6796
6797	bf_set(lpfc_mbx_set_feature_CGN_acqe_freq,
6798	&mbox->u.mqe.un.set_feature, sig_freq);
6799
6800	mbox->u.mqe.un.set_feature.feature = LPFC_SET_CGN_SIGNAL;
6801	mbox->u.mqe.un.set_feature.param_len = 12;
6802	break;
6803	case LPFC_SET_DUAL_DUMP:
6804	bf_set(lpfc_mbx_set_feature_dd,
6805	&mbox->u.mqe.un.set_feature, LPFC_ENABLE_DUAL_DUMP);
6806	bf_set(lpfc_mbx_set_feature_ddquery,
6807	&mbox->u.mqe.un.set_feature, 0);
6808	mbox->u.mqe.un.set_feature.feature = LPFC_SET_DUAL_DUMP;
6809	mbox->u.mqe.un.set_feature.param_len = 4;
6810	break;
6811	case LPFC_SET_ENABLE_MI:
6812	mbox->u.mqe.un.set_feature.feature = LPFC_SET_ENABLE_MI;
6813	mbox->u.mqe.un.set_feature.param_len = 4;
6814	bf_set(lpfc_mbx_set_feature_milunq, &mbox->u.mqe.un.set_feature,
6815	phba->pport->cfg_lun_queue_depth);
6816	bf_set(lpfc_mbx_set_feature_mi, &mbox->u.mqe.un.set_feature,
6817	phba->sli4_hba.pc_sli4_params.mi_ver);
6818	break;
6819	case LPFC_SET_LD_SIGNAL:
6820	mbox->u.mqe.un.set_feature.feature = LPFC_SET_LD_SIGNAL;
6821	mbox->u.mqe.un.set_feature.param_len = 16;
6822	bf_set(lpfc_mbx_set_feature_lds_qry,
6823	&mbox->u.mqe.un.set_feature, LPFC_QUERY_LDS_OP);
6824	break;
6825	case LPFC_SET_ENABLE_CMF:
6826	mbox->u.mqe.un.set_feature.feature = LPFC_SET_ENABLE_CMF;
6827	mbox->u.mqe.un.set_feature.param_len = 4;
6828	bf_set(lpfc_mbx_set_feature_cmf,
6829	&mbox->u.mqe.un.set_feature, 1);
6830	break;
6831	}
6832	return;
6833	}
6834
6835	/**
6836	* lpfc_ras_stop_fwlog: Disable FW logging by the adapter
6837	* @phba: Pointer to HBA context object.
6838	*
6839	* Disable FW logging into host memory on the adapter. To
6840	* be done before reading logs from the host memory.
6841	**/
6842	void
6843	lpfc_ras_stop_fwlog(struct lpfc_hba *phba)
6844	{
6845	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
6846
6847	spin_lock_irq(lock: &phba->hbalock);
6848	ras_fwlog->state = INACTIVE;
6849	spin_unlock_irq(lock: &phba->hbalock);
6850
6851	/* Disable FW logging to host memory */
6852	writel(LPFC_CTL_PDEV_CTL_DDL_RAS,
6853	addr: phba->sli4_hba.conf_regs_memmap_p + LPFC_CTL_PDEV_CTL_OFFSET);
6854
6855	/* Wait 10ms for firmware to stop using DMA buffer */
6856	usleep_range(min: 10 * 1000, max: 20 * 1000);
6857	}
6858
6859	/**
6860	* lpfc_sli4_ras_dma_free - Free memory allocated for FW logging.
6861	* @phba: Pointer to HBA context object.
6862	*
6863	* This function is called to free memory allocated for RAS FW logging
6864	* support in the driver.
6865	**/
6866	void
6867	lpfc_sli4_ras_dma_free(struct lpfc_hba *phba)
6868	{
6869	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
6870	struct lpfc_dmabuf *dmabuf, *next;
6871
6872	if (!list_empty(head: &ras_fwlog->fwlog_buff_list)) {
6873	list_for_each_entry_safe(dmabuf, next,
6874	&ras_fwlog->fwlog_buff_list,
6875	list) {
6876	list_del(entry: &dmabuf->list);
6877	dma_free_coherent(dev: &phba->pcidev->dev,
6878	LPFC_RAS_MAX_ENTRY_SIZE,
6879	cpu_addr: dmabuf->virt, dma_handle: dmabuf->phys);
6880	kfree(objp: dmabuf);
6881	}
6882	}
6883
6884	if (ras_fwlog->lwpd.virt) {
6885	dma_free_coherent(dev: &phba->pcidev->dev,
6886	size: sizeof(uint32_t) * 2,
6887	cpu_addr: ras_fwlog->lwpd.virt,
6888	dma_handle: ras_fwlog->lwpd.phys);
6889	ras_fwlog->lwpd.virt = NULL;
6890	}
6891
6892	spin_lock_irq(lock: &phba->hbalock);
6893	ras_fwlog->state = INACTIVE;
6894	spin_unlock_irq(lock: &phba->hbalock);
6895	}
6896
6897	/**
6898	* lpfc_sli4_ras_dma_alloc: Allocate memory for FW support
6899	* @phba: Pointer to HBA context object.
6900	* @fwlog_buff_count: Count of buffers to be created.
6901	*
6902	* This routine DMA memory for Log Write Position Data[LPWD] and buffer
6903	* to update FW log is posted to the adapter.
6904	* Buffer count is calculated based on module param ras_fwlog_buffsize
6905	* Size of each buffer posted to FW is 64K.
6906	**/
6907
6908	static int
6909	lpfc_sli4_ras_dma_alloc(struct lpfc_hba *phba,
6910	uint32_t fwlog_buff_count)
6911	{
6912	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
6913	struct lpfc_dmabuf *dmabuf;
6914	int rc = 0, i = 0;
6915
6916	/* Initialize List */
6917	INIT_LIST_HEAD(list: &ras_fwlog->fwlog_buff_list);
6918
6919	/* Allocate memory for the LWPD */
6920	ras_fwlog->lwpd.virt = dma_alloc_coherent(dev: &phba->pcidev->dev,
6921	size: sizeof(uint32_t) * 2,
6922	dma_handle: &ras_fwlog->lwpd.phys,
6923	GFP_KERNEL);
6924	if (!ras_fwlog->lwpd.virt) {
6925	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6926	"6185 LWPD Memory Alloc Failed\n");
6927
6928	return -ENOMEM;
6929	}
6930
6931	ras_fwlog->fw_buffcount = fwlog_buff_count;
6932	for (i = 0; i < ras_fwlog->fw_buffcount; i++) {
6933	dmabuf = kzalloc(size: sizeof(struct lpfc_dmabuf),
6934	GFP_KERNEL);
6935	if (!dmabuf) {
6936	rc = -ENOMEM;
6937	lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
6938	"6186 Memory Alloc failed FW logging");
6939	goto free_mem;
6940	}
6941
6942	dmabuf->virt = dma_alloc_coherent(dev: &phba->pcidev->dev,
6943	LPFC_RAS_MAX_ENTRY_SIZE,
6944	dma_handle: &dmabuf->phys, GFP_KERNEL);
6945	if (!dmabuf->virt) {
6946	kfree(objp: dmabuf);
6947	rc = -ENOMEM;
6948	lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
6949	"6187 DMA Alloc Failed FW logging");
6950	goto free_mem;
6951	}
6952	dmabuf->buffer_tag = i;
6953	list_add_tail(new: &dmabuf->list, head: &ras_fwlog->fwlog_buff_list);
6954	}
6955
6956	free_mem:
6957	if (rc)
6958	lpfc_sli4_ras_dma_free(phba);
6959
6960	return rc;
6961	}
6962
6963	/**
6964	* lpfc_sli4_ras_mbox_cmpl: Completion handler for RAS MBX command
6965	* @phba: pointer to lpfc hba data structure.
6966	* @pmb: pointer to the driver internal queue element for mailbox command.
6967	*
6968	* Completion handler for driver's RAS MBX command to the device.
6969	**/
6970	static void
6971	lpfc_sli4_ras_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
6972	{
6973	MAILBOX_t *mb;
6974	union lpfc_sli4_cfg_shdr *shdr;
6975	uint32_t shdr_status, shdr_add_status;
6976	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
6977
6978	mb = &pmb->u.mb;
6979
6980	shdr = (union lpfc_sli4_cfg_shdr *)
6981	&pmb->u.mqe.un.ras_fwlog.header.cfg_shdr;
6982	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
6983	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
6984
6985	if (mb->mbxStatus != MBX_SUCCESS || shdr_status) {
6986	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6987	"6188 FW LOG mailbox "
6988	"completed with status x%x add_status x%x,"
6989	" mbx status x%x\n",
6990	shdr_status, shdr_add_status, mb->mbxStatus);
6991
6992	ras_fwlog->ras_hwsupport = false;
6993	goto disable_ras;
6994	}
6995
6996	spin_lock_irq(lock: &phba->hbalock);
6997	ras_fwlog->state = ACTIVE;
6998	spin_unlock_irq(lock: &phba->hbalock);
6999	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
7000
7001	return;
7002
7003	disable_ras:
7004	/* Free RAS DMA memory */
7005	lpfc_sli4_ras_dma_free(phba);
7006	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
7007	}
7008
7009	/**
7010	* lpfc_sli4_ras_fwlog_init: Initialize memory and post RAS MBX command
7011	* @phba: pointer to lpfc hba data structure.
7012	* @fwlog_level: Logging verbosity level.
7013	* @fwlog_enable: Enable/Disable logging.
7014	*
7015	* Initialize memory and post mailbox command to enable FW logging in host
7016	* memory.
7017	**/
7018	int
7019	lpfc_sli4_ras_fwlog_init(struct lpfc_hba *phba,
7020	uint32_t fwlog_level,
7021	uint32_t fwlog_enable)
7022	{
7023	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
7024	struct lpfc_mbx_set_ras_fwlog *mbx_fwlog = NULL;
7025	struct lpfc_dmabuf *dmabuf;
7026	LPFC_MBOXQ_t *mbox;
7027	uint32_t len = 0, fwlog_buffsize, fwlog_entry_count;
7028	int rc = 0;
7029
7030	spin_lock_irq(lock: &phba->hbalock);
7031	ras_fwlog->state = INACTIVE;
7032	spin_unlock_irq(lock: &phba->hbalock);
7033
7034	fwlog_buffsize = (LPFC_RAS_MIN_BUFF_POST_SIZE *
7035	phba->cfg_ras_fwlog_buffsize);
7036	fwlog_entry_count = (fwlog_buffsize/LPFC_RAS_MAX_ENTRY_SIZE);
7037
7038	/*
7039	* If re-enabling FW logging support use earlier allocated
7040	* DMA buffers while posting MBX command.
7041	**/
7042	if (!ras_fwlog->lwpd.virt) {
7043	rc = lpfc_sli4_ras_dma_alloc(phba, fwlog_buff_count: fwlog_entry_count);
7044	if (rc) {
7045	lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
7046	"6189 FW Log Memory Allocation Failed");
7047	return rc;
7048	}
7049	}
7050
7051	/* Setup Mailbox command */
7052	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
7053	if (!mbox) {
7054	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7055	"6190 RAS MBX Alloc Failed");
7056	rc = -ENOMEM;
7057	goto mem_free;
7058	}
7059
7060	ras_fwlog->fw_loglevel = fwlog_level;
7061	len = (sizeof(struct lpfc_mbx_set_ras_fwlog) -
7062	sizeof(struct lpfc_sli4_cfg_mhdr));
7063
7064	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_LOWLEVEL,
7065	LPFC_MBOX_OPCODE_SET_DIAG_LOG_OPTION,
7066	len, LPFC_SLI4_MBX_EMBED);
7067
7068	mbx_fwlog = (struct lpfc_mbx_set_ras_fwlog *)&mbox->u.mqe.un.ras_fwlog;
7069	bf_set(lpfc_fwlog_enable, &mbx_fwlog->u.request,
7070	fwlog_enable);
7071	bf_set(lpfc_fwlog_loglvl, &mbx_fwlog->u.request,
7072	ras_fwlog->fw_loglevel);
7073	bf_set(lpfc_fwlog_buffcnt, &mbx_fwlog->u.request,
7074	ras_fwlog->fw_buffcount);
7075	bf_set(lpfc_fwlog_buffsz, &mbx_fwlog->u.request,
7076	LPFC_RAS_MAX_ENTRY_SIZE/SLI4_PAGE_SIZE);
7077
7078	/* Update DMA buffer address */
7079	list_for_each_entry(dmabuf, &ras_fwlog->fwlog_buff_list, list) {
7080	memset(dmabuf->virt, 0, LPFC_RAS_MAX_ENTRY_SIZE);
7081
7082	mbx_fwlog->u.request.buff_fwlog[dmabuf->buffer_tag].addr_lo =
7083	putPaddrLow(dmabuf->phys);
7084
7085	mbx_fwlog->u.request.buff_fwlog[dmabuf->buffer_tag].addr_hi =
7086	putPaddrHigh(dmabuf->phys);
7087	}
7088
7089	/* Update LPWD address */
7090	mbx_fwlog->u.request.lwpd.addr_lo = putPaddrLow(ras_fwlog->lwpd.phys);
7091	mbx_fwlog->u.request.lwpd.addr_hi = putPaddrHigh(ras_fwlog->lwpd.phys);
7092
7093	spin_lock_irq(lock: &phba->hbalock);
7094	ras_fwlog->state = REG_INPROGRESS;
7095	spin_unlock_irq(lock: &phba->hbalock);
7096	mbox->vport = phba->pport;
7097	mbox->mbox_cmpl = lpfc_sli4_ras_mbox_cmpl;
7098
7099	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT);
7100
7101	if (rc == MBX_NOT_FINISHED) {
7102	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7103	"6191 FW-Log Mailbox failed. "
7104	"status %d mbxStatus : x%x", rc,
7105	bf_get(lpfc_mqe_status, &mbox->u.mqe));
7106	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
7107	rc = -EIO;
7108	goto mem_free;
7109	} else
7110	rc = 0;
7111	mem_free:
7112	if (rc)
7113	lpfc_sli4_ras_dma_free(phba);
7114
7115	return rc;
7116	}
7117
7118	/**
7119	* lpfc_sli4_ras_setup - Check if RAS supported on the adapter
7120	* @phba: Pointer to HBA context object.
7121	*
7122	* Check if RAS is supported on the adapter and initialize it.
7123	**/
7124	void
7125	lpfc_sli4_ras_setup(struct lpfc_hba *phba)
7126	{
7127	/* Check RAS FW Log needs to be enabled or not */
7128	if (lpfc_check_fwlog_support(phba))
7129	return;
7130
7131	lpfc_sli4_ras_fwlog_init(phba, fwlog_level: phba->cfg_ras_fwlog_level,
7132	LPFC_RAS_ENABLE_LOGGING);
7133	}
7134
7135	/**
7136	* lpfc_sli4_alloc_resource_identifiers - Allocate all SLI4 resource extents.
7137	* @phba: Pointer to HBA context object.
7138	*
7139	* This function allocates all SLI4 resource identifiers.
7140	**/
7141	int
7142	lpfc_sli4_alloc_resource_identifiers(struct lpfc_hba *phba)
7143	{
7144	int i, rc, error = 0;
7145	uint16_t count, base;
7146	unsigned long longs;
7147
7148	if (!phba->sli4_hba.rpi_hdrs_in_use)
7149	phba->sli4_hba.next_rpi = phba->sli4_hba.max_cfg_param.max_rpi;
7150	if (phba->sli4_hba.extents_in_use) {
7151	/*
7152	* The port supports resource extents. The XRI, VPI, VFI, RPI
7153	* resource extent count must be read and allocated before
7154	* provisioning the resource id arrays.
7155	*/
7156	if (bf_get(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags) ==
7157	LPFC_IDX_RSRC_RDY) {
7158	/*
7159	* Extent-based resources are set - the driver could
7160	* be in a port reset. Figure out if any corrective
7161	* actions need to be taken.
7162	*/
7163	rc = lpfc_sli4_chk_avail_extnt_rsrc(phba,
7164	LPFC_RSC_TYPE_FCOE_VFI);
7165	if (rc != 0)
7166	error++;
7167	rc = lpfc_sli4_chk_avail_extnt_rsrc(phba,
7168	LPFC_RSC_TYPE_FCOE_VPI);
7169	if (rc != 0)
7170	error++;
7171	rc = lpfc_sli4_chk_avail_extnt_rsrc(phba,
7172	LPFC_RSC_TYPE_FCOE_XRI);
7173	if (rc != 0)
7174	error++;
7175	rc = lpfc_sli4_chk_avail_extnt_rsrc(phba,
7176	LPFC_RSC_TYPE_FCOE_RPI);
7177	if (rc != 0)
7178	error++;
7179
7180	/*
7181	* It's possible that the number of resources
7182	* provided to this port instance changed between
7183	* resets. Detect this condition and reallocate
7184	* resources. Otherwise, there is no action.
7185	*/
7186	if (error) {
7187	lpfc_printf_log(phba, KERN_INFO,
7188	LOG_MBOX | LOG_INIT,
7189	"2931 Detected extent resource "
7190	"change. Reallocating all "
7191	"extents.\n");
7192	rc = lpfc_sli4_dealloc_extent(phba,
7193	LPFC_RSC_TYPE_FCOE_VFI);
7194	rc = lpfc_sli4_dealloc_extent(phba,
7195	LPFC_RSC_TYPE_FCOE_VPI);
7196	rc = lpfc_sli4_dealloc_extent(phba,
7197	LPFC_RSC_TYPE_FCOE_XRI);
7198	rc = lpfc_sli4_dealloc_extent(phba,
7199	LPFC_RSC_TYPE_FCOE_RPI);
7200	} else
7201	return 0;
7202	}
7203
7204	rc = lpfc_sli4_alloc_extent(phba, LPFC_RSC_TYPE_FCOE_VFI);
7205	if (unlikely(rc))
7206	goto err_exit;
7207
7208	rc = lpfc_sli4_alloc_extent(phba, LPFC_RSC_TYPE_FCOE_VPI);
7209	if (unlikely(rc))
7210	goto err_exit;
7211
7212	rc = lpfc_sli4_alloc_extent(phba, LPFC_RSC_TYPE_FCOE_RPI);
7213	if (unlikely(rc))
7214	goto err_exit;
7215
7216	rc = lpfc_sli4_alloc_extent(phba, LPFC_RSC_TYPE_FCOE_XRI);
7217	if (unlikely(rc))
7218	goto err_exit;
7219	bf_set(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags,
7220	LPFC_IDX_RSRC_RDY);
7221	return rc;
7222	} else {
7223	/*
7224	* The port does not support resource extents. The XRI, VPI,
7225	* VFI, RPI resource ids were determined from READ_CONFIG.
7226	* Just allocate the bitmasks and provision the resource id
7227	* arrays. If a port reset is active, the resources don't
7228	* need any action - just exit.
7229	*/
7230	if (bf_get(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags) ==
7231	LPFC_IDX_RSRC_RDY) {
7232	lpfc_sli4_dealloc_resource_identifiers(phba);
7233	lpfc_sli4_remove_rpis(phba);
7234	}
7235	/* RPIs. */
7236	count = phba->sli4_hba.max_cfg_param.max_rpi;
7237	if (count <= 0) {
7238	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7239	"3279 Invalid provisioning of "
7240	"rpi:%d\n", count);
7241	rc = -EINVAL;
7242	goto err_exit;
7243	}
7244	base = phba->sli4_hba.max_cfg_param.rpi_base;
7245	longs = (count + BITS_PER_LONG - 1) / BITS_PER_LONG;
7246	phba->sli4_hba.rpi_bmask = kcalloc(n: longs,
7247	size: sizeof(unsigned long),
7248	GFP_KERNEL);
7249	if (unlikely(!phba->sli4_hba.rpi_bmask)) {
7250	rc = -ENOMEM;
7251	goto err_exit;
7252	}
7253	phba->sli4_hba.rpi_ids = kcalloc(n: count, size: sizeof(uint16_t),
7254	GFP_KERNEL);
7255	if (unlikely(!phba->sli4_hba.rpi_ids)) {
7256	rc = -ENOMEM;
7257	goto free_rpi_bmask;
7258	}
7259
7260	for (i = 0; i < count; i++)
7261	phba->sli4_hba.rpi_ids[i] = base + i;
7262
7263	/* VPIs. */
7264	count = phba->sli4_hba.max_cfg_param.max_vpi;
7265	if (count <= 0) {
7266	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7267	"3280 Invalid provisioning of "
7268	"vpi:%d\n", count);
7269	rc = -EINVAL;
7270	goto free_rpi_ids;
7271	}
7272	base = phba->sli4_hba.max_cfg_param.vpi_base;
7273	longs = (count + BITS_PER_LONG - 1) / BITS_PER_LONG;
7274	phba->vpi_bmask = kcalloc(n: longs, size: sizeof(unsigned long),
7275	GFP_KERNEL);
7276	if (unlikely(!phba->vpi_bmask)) {
7277	rc = -ENOMEM;
7278	goto free_rpi_ids;
7279	}
7280	phba->vpi_ids = kcalloc(n: count, size: sizeof(uint16_t),
7281	GFP_KERNEL);
7282	if (unlikely(!phba->vpi_ids)) {
7283	rc = -ENOMEM;
7284	goto free_vpi_bmask;
7285	}
7286
7287	for (i = 0; i < count; i++)
7288	phba->vpi_ids[i] = base + i;
7289
7290	/* XRIs. */
7291	count = phba->sli4_hba.max_cfg_param.max_xri;
7292	if (count <= 0) {
7293	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7294	"3281 Invalid provisioning of "
7295	"xri:%d\n", count);
7296	rc = -EINVAL;
7297	goto free_vpi_ids;
7298	}
7299	base = phba->sli4_hba.max_cfg_param.xri_base;
7300	longs = (count + BITS_PER_LONG - 1) / BITS_PER_LONG;
7301	phba->sli4_hba.xri_bmask = kcalloc(n: longs,
7302	size: sizeof(unsigned long),
7303	GFP_KERNEL);
7304	if (unlikely(!phba->sli4_hba.xri_bmask)) {
7305	rc = -ENOMEM;
7306	goto free_vpi_ids;
7307	}
7308	phba->sli4_hba.max_cfg_param.xri_used = 0;
7309	phba->sli4_hba.xri_ids = kcalloc(n: count, size: sizeof(uint16_t),
7310	GFP_KERNEL);
7311	if (unlikely(!phba->sli4_hba.xri_ids)) {
7312	rc = -ENOMEM;
7313	goto free_xri_bmask;
7314	}
7315
7316	for (i = 0; i < count; i++)
7317	phba->sli4_hba.xri_ids[i] = base + i;
7318
7319	/* VFIs. */
7320	count = phba->sli4_hba.max_cfg_param.max_vfi;
7321	if (count <= 0) {
7322	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7323	"3282 Invalid provisioning of "
7324	"vfi:%d\n", count);
7325	rc = -EINVAL;
7326	goto free_xri_ids;
7327	}
7328	base = phba->sli4_hba.max_cfg_param.vfi_base;
7329	longs = (count + BITS_PER_LONG - 1) / BITS_PER_LONG;
7330	phba->sli4_hba.vfi_bmask = kcalloc(n: longs,
7331	size: sizeof(unsigned long),
7332	GFP_KERNEL);
7333	if (unlikely(!phba->sli4_hba.vfi_bmask)) {
7334	rc = -ENOMEM;
7335	goto free_xri_ids;
7336	}
7337	phba->sli4_hba.vfi_ids = kcalloc(n: count, size: sizeof(uint16_t),
7338	GFP_KERNEL);
7339	if (unlikely(!phba->sli4_hba.vfi_ids)) {
7340	rc = -ENOMEM;
7341	goto free_vfi_bmask;
7342	}
7343
7344	for (i = 0; i < count; i++)
7345	phba->sli4_hba.vfi_ids[i] = base + i;
7346
7347	/*
7348	* Mark all resources ready. An HBA reset doesn't need
7349	* to reset the initialization.
7350	*/
7351	bf_set(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags,
7352	LPFC_IDX_RSRC_RDY);
7353	return 0;
7354	}
7355
7356	free_vfi_bmask:
7357	kfree(objp: phba->sli4_hba.vfi_bmask);
7358	phba->sli4_hba.vfi_bmask = NULL;
7359	free_xri_ids:
7360	kfree(objp: phba->sli4_hba.xri_ids);
7361	phba->sli4_hba.xri_ids = NULL;
7362	free_xri_bmask:
7363	kfree(objp: phba->sli4_hba.xri_bmask);
7364	phba->sli4_hba.xri_bmask = NULL;
7365	free_vpi_ids:
7366	kfree(objp: phba->vpi_ids);
7367	phba->vpi_ids = NULL;
7368	free_vpi_bmask:
7369	kfree(objp: phba->vpi_bmask);
7370	phba->vpi_bmask = NULL;
7371	free_rpi_ids:
7372	kfree(objp: phba->sli4_hba.rpi_ids);
7373	phba->sli4_hba.rpi_ids = NULL;
7374	free_rpi_bmask:
7375	kfree(objp: phba->sli4_hba.rpi_bmask);
7376	phba->sli4_hba.rpi_bmask = NULL;
7377	err_exit:
7378	return rc;
7379	}
7380
7381	/**
7382	* lpfc_sli4_dealloc_resource_identifiers - Deallocate all SLI4 resource extents.
7383	* @phba: Pointer to HBA context object.
7384	*
7385	* This function allocates the number of elements for the specified
7386	* resource type.
7387	**/
7388	int
7389	lpfc_sli4_dealloc_resource_identifiers(struct lpfc_hba *phba)
7390	{
7391	if (phba->sli4_hba.extents_in_use) {
7392	lpfc_sli4_dealloc_extent(phba, LPFC_RSC_TYPE_FCOE_VPI);
7393	lpfc_sli4_dealloc_extent(phba, LPFC_RSC_TYPE_FCOE_RPI);
7394	lpfc_sli4_dealloc_extent(phba, LPFC_RSC_TYPE_FCOE_XRI);
7395	lpfc_sli4_dealloc_extent(phba, LPFC_RSC_TYPE_FCOE_VFI);
7396	} else {
7397	kfree(objp: phba->vpi_bmask);
7398	phba->sli4_hba.max_cfg_param.vpi_used = 0;
7399	kfree(objp: phba->vpi_ids);
7400	bf_set(lpfc_vpi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
7401	kfree(objp: phba->sli4_hba.xri_bmask);
7402	kfree(objp: phba->sli4_hba.xri_ids);
7403	kfree(objp: phba->sli4_hba.vfi_bmask);
7404	kfree(objp: phba->sli4_hba.vfi_ids);
7405	bf_set(lpfc_vfi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
7406	bf_set(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
7407	}
7408
7409	return 0;
7410	}
7411
7412	/**
7413	* lpfc_sli4_get_allocated_extnts - Get the port's allocated extents.
7414	* @phba: Pointer to HBA context object.
7415	* @type: The resource extent type.
7416	* @extnt_cnt: buffer to hold port extent count response
7417	* @extnt_size: buffer to hold port extent size response.
7418	*
7419	* This function calls the port to read the host allocated extents
7420	* for a particular type.
7421	**/
7422	int
7423	lpfc_sli4_get_allocated_extnts(struct lpfc_hba *phba, uint16_t type,
7424	uint16_t *extnt_cnt, uint16_t *extnt_size)
7425	{
7426	bool emb;
7427	int rc = 0;
7428	uint16_t curr_blks = 0;
7429	uint32_t req_len, emb_len;
7430	uint32_t alloc_len, mbox_tmo;
7431	struct list_head *blk_list_head;
7432	struct lpfc_rsrc_blks *rsrc_blk;
7433	LPFC_MBOXQ_t *mbox;
7434	void *virtaddr = NULL;
7435	struct lpfc_mbx_nembed_rsrc_extent *n_rsrc;
7436	struct lpfc_mbx_alloc_rsrc_extents *rsrc_ext;
7437	union lpfc_sli4_cfg_shdr *shdr;
7438
7439	switch (type) {
7440	case LPFC_RSC_TYPE_FCOE_VPI:
7441	blk_list_head = &phba->lpfc_vpi_blk_list;
7442	break;
7443	case LPFC_RSC_TYPE_FCOE_XRI:
7444	blk_list_head = &phba->sli4_hba.lpfc_xri_blk_list;
7445	break;
7446	case LPFC_RSC_TYPE_FCOE_VFI:
7447	blk_list_head = &phba->sli4_hba.lpfc_vfi_blk_list;
7448	break;
7449	case LPFC_RSC_TYPE_FCOE_RPI:
7450	blk_list_head = &phba->sli4_hba.lpfc_rpi_blk_list;
7451	break;
7452	default:
7453	return -EIO;
7454	}
7455
7456	/* Count the number of extents currently allocatd for this type. */
7457	list_for_each_entry(rsrc_blk, blk_list_head, list) {
7458	if (curr_blks == 0) {
7459	/*
7460	* The GET_ALLOCATED mailbox does not return the size,
7461	* just the count. The size should be just the size
7462	* stored in the current allocated block and all sizes
7463	* for an extent type are the same so set the return
7464	* value now.
7465	*/
7466	*extnt_size = rsrc_blk->rsrc_size;
7467	}
7468	curr_blks++;
7469	}
7470
7471	/*
7472	* Calculate the size of an embedded mailbox. The uint32_t
7473	* accounts for extents-specific word.
7474	*/
7475	emb_len = sizeof(MAILBOX_t) - sizeof(struct mbox_header) -
7476	sizeof(uint32_t);
7477
7478	/*
7479	* Presume the allocation and response will fit into an embedded
7480	* mailbox. If not true, reconfigure to a non-embedded mailbox.
7481	*/
7482	emb = LPFC_SLI4_MBX_EMBED;
7483	req_len = emb_len;
7484	if (req_len > emb_len) {
7485	req_len = curr_blks * sizeof(uint16_t) +
7486	sizeof(union lpfc_sli4_cfg_shdr) +
7487	sizeof(uint32_t);
7488	emb = LPFC_SLI4_MBX_NEMBED;
7489	}
7490
7491	mbox = (LPFC_MBOXQ_t *) mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
7492	if (!mbox)
7493	return -ENOMEM;
7494	memset(mbox, 0, sizeof(LPFC_MBOXQ_t));
7495
7496	alloc_len = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
7497	LPFC_MBOX_OPCODE_GET_ALLOC_RSRC_EXTENT,
7498	req_len, emb);
7499	if (alloc_len < req_len) {
7500	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7501	"2983 Allocated DMA memory size (x%x) is "
7502	"less than the requested DMA memory "
7503	"size (x%x)\n", alloc_len, req_len);
7504	rc = -ENOMEM;
7505	goto err_exit;
7506	}
7507	rc = lpfc_sli4_mbox_rsrc_extent(phba, mbox, curr_blks, type, emb);
7508	if (unlikely(rc)) {
7509	rc = -EIO;
7510	goto err_exit;
7511	}
7512
7513	if (!phba->sli4_hba.intr_enable)
7514	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
7515	else {
7516	mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
7517	rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
7518	}
7519
7520	if (unlikely(rc)) {
7521	rc = -EIO;
7522	goto err_exit;
7523	}
7524
7525	/*
7526	* Figure out where the response is located. Then get local pointers
7527	* to the response data. The port does not guarantee to respond to
7528	* all extents counts request so update the local variable with the
7529	* allocated count from the port.
7530	*/
7531	if (emb == LPFC_SLI4_MBX_EMBED) {
7532	rsrc_ext = &mbox->u.mqe.un.alloc_rsrc_extents;
7533	shdr = &rsrc_ext->header.cfg_shdr;
7534	*extnt_cnt = bf_get(lpfc_mbx_rsrc_cnt, &rsrc_ext->u.rsp);
7535	} else {
7536	virtaddr = mbox->sge_array->addr[0];
7537	n_rsrc = (struct lpfc_mbx_nembed_rsrc_extent *) virtaddr;
7538	shdr = &n_rsrc->cfg_shdr;
7539	*extnt_cnt = bf_get(lpfc_mbx_rsrc_cnt, n_rsrc);
7540	}
7541
7542	if (bf_get(lpfc_mbox_hdr_status, &shdr->response)) {
7543	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7544	"2984 Failed to read allocated resources "
7545	"for type %d - Status 0x%x Add'l Status 0x%x.\n",
7546	type,
7547	bf_get(lpfc_mbox_hdr_status, &shdr->response),
7548	bf_get(lpfc_mbox_hdr_add_status, &shdr->response));
7549	rc = -EIO;
7550	goto err_exit;
7551	}
7552	err_exit:
7553	lpfc_sli4_mbox_cmd_free(phba, mbox);
7554	return rc;
7555	}
7556
7557	/**
7558	* lpfc_sli4_repost_sgl_list - Repost the buffers sgl pages as block
7559	* @phba: pointer to lpfc hba data structure.
7560	* @sgl_list: linked link of sgl buffers to post
7561	* @cnt: number of linked list buffers
7562	*
7563	* This routine walks the list of buffers that have been allocated and
7564	* repost them to the port by using SGL block post. This is needed after a
7565	* pci_function_reset/warm_start or start. It attempts to construct blocks
7566	* of buffer sgls which contains contiguous xris and uses the non-embedded
7567	* SGL block post mailbox commands to post them to the port. For single
7568	* buffer sgl with non-contiguous xri, if any, it shall use embedded SGL post
7569	* mailbox command for posting.
7570	*
7571	* Returns: 0 = success, non-zero failure.
7572	**/
7573	static int
7574	lpfc_sli4_repost_sgl_list(struct lpfc_hba *phba,
7575	struct list_head *sgl_list, int cnt)
7576	{
7577	struct lpfc_sglq *sglq_entry = NULL;
7578	struct lpfc_sglq *sglq_entry_next = NULL;
7579	struct lpfc_sglq *sglq_entry_first = NULL;
7580	int status, total_cnt;
7581	int post_cnt = 0, num_posted = 0, block_cnt = 0;
7582	int last_xritag = NO_XRI;
7583	LIST_HEAD(prep_sgl_list);
7584	LIST_HEAD(blck_sgl_list);
7585	LIST_HEAD(allc_sgl_list);
7586	LIST_HEAD(post_sgl_list);
7587	LIST_HEAD(free_sgl_list);
7588
7589	spin_lock_irq(lock: &phba->hbalock);
7590	spin_lock(lock: &phba->sli4_hba.sgl_list_lock);
7591	list_splice_init(list: sgl_list, head: &allc_sgl_list);
7592	spin_unlock(lock: &phba->sli4_hba.sgl_list_lock);
7593	spin_unlock_irq(lock: &phba->hbalock);
7594
7595	total_cnt = cnt;
7596	list_for_each_entry_safe(sglq_entry, sglq_entry_next,
7597	&allc_sgl_list, list) {
7598	list_del_init(entry: &sglq_entry->list);
7599	block_cnt++;
7600	if ((last_xritag != NO_XRI) &&
7601	(sglq_entry->sli4_xritag != last_xritag + 1)) {
7602	/* a hole in xri block, form a sgl posting block */
7603	list_splice_init(list: &prep_sgl_list, head: &blck_sgl_list);
7604	post_cnt = block_cnt - 1;
7605	/* prepare list for next posting block */
7606	list_add_tail(new: &sglq_entry->list, head: &prep_sgl_list);
7607	block_cnt = 1;
7608	} else {
7609	/* prepare list for next posting block */
7610	list_add_tail(new: &sglq_entry->list, head: &prep_sgl_list);
7611	/* enough sgls for non-embed sgl mbox command */
7612	if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) {
7613	list_splice_init(list: &prep_sgl_list,
7614	head: &blck_sgl_list);
7615	post_cnt = block_cnt;
7616	block_cnt = 0;
7617	}
7618	}
7619	num_posted++;
7620
7621	/* keep track of last sgl's xritag */
7622	last_xritag = sglq_entry->sli4_xritag;
7623
7624	/* end of repost sgl list condition for buffers */
7625	if (num_posted == total_cnt) {
7626	if (post_cnt == 0) {
7627	list_splice_init(list: &prep_sgl_list,
7628	head: &blck_sgl_list);
7629	post_cnt = block_cnt;
7630	} else if (block_cnt == 1) {
7631	status = lpfc_sli4_post_sgl(phba,
7632	sglq_entry->phys, 0,
7633	sglq_entry->sli4_xritag);
7634	if (!status) {
7635	/* successful, put sgl to posted list */
7636	list_add_tail(new: &sglq_entry->list,
7637	head: &post_sgl_list);
7638	} else {
7639	/* Failure, put sgl to free list */
7640	lpfc_printf_log(phba, KERN_WARNING,
7641	LOG_SLI,
7642	"3159 Failed to post "
7643	"sgl, xritag:x%x\n",
7644	sglq_entry->sli4_xritag);
7645	list_add_tail(new: &sglq_entry->list,
7646	head: &free_sgl_list);
7647	total_cnt--;
7648	}
7649	}
7650	}
7651
7652	/* continue until a nembed page worth of sgls */
7653	if (post_cnt == 0)
7654	continue;
7655
7656	/* post the buffer list sgls as a block */
7657	status = lpfc_sli4_post_sgl_list(phba, &blck_sgl_list,
7658	post_cnt);
7659
7660	if (!status) {
7661	/* success, put sgl list to posted sgl list */
7662	list_splice_init(list: &blck_sgl_list, head: &post_sgl_list);
7663	} else {
7664	/* Failure, put sgl list to free sgl list */
7665	sglq_entry_first = list_first_entry(&blck_sgl_list,
7666	struct lpfc_sglq,
7667	list);
7668	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
7669	"3160 Failed to post sgl-list, "
7670	"xritag:x%x-x%x\n",
7671	sglq_entry_first->sli4_xritag,
7672	(sglq_entry_first->sli4_xritag +
7673	post_cnt - 1));
7674	list_splice_init(list: &blck_sgl_list, head: &free_sgl_list);
7675	total_cnt -= post_cnt;
7676	}
7677
7678	/* don't reset xirtag due to hole in xri block */
7679	if (block_cnt == 0)
7680	last_xritag = NO_XRI;
7681
7682	/* reset sgl post count for next round of posting */
7683	post_cnt = 0;
7684	}
7685
7686	/* free the sgls failed to post */
7687	lpfc_free_sgl_list(phba, &free_sgl_list);
7688
7689	/* push sgls posted to the available list */
7690	if (!list_empty(head: &post_sgl_list)) {
7691	spin_lock_irq(lock: &phba->hbalock);
7692	spin_lock(lock: &phba->sli4_hba.sgl_list_lock);
7693	list_splice_init(list: &post_sgl_list, head: sgl_list);
7694	spin_unlock(lock: &phba->sli4_hba.sgl_list_lock);
7695	spin_unlock_irq(lock: &phba->hbalock);
7696	} else {
7697	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7698	"3161 Failure to post sgl to port,status %x "
7699	"blkcnt %d totalcnt %d postcnt %d\n",
7700	status, block_cnt, total_cnt, post_cnt);
7701	return -EIO;
7702	}
7703
7704	/* return the number of XRIs actually posted */
7705	return total_cnt;
7706	}
7707
7708	/**
7709	* lpfc_sli4_repost_io_sgl_list - Repost all the allocated nvme buffer sgls
7710	* @phba: pointer to lpfc hba data structure.
7711	*
7712	* This routine walks the list of nvme buffers that have been allocated and
7713	* repost them to the port by using SGL block post. This is needed after a
7714	* pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine
7715	* is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list
7716	* to the lpfc_io_buf_list. If the repost fails, reject all nvme buffers.
7717	*
7718	* Returns: 0 = success, non-zero failure.
7719	**/
7720	static int
7721	lpfc_sli4_repost_io_sgl_list(struct lpfc_hba *phba)
7722	{
7723	LIST_HEAD(post_nblist);
7724	int num_posted, rc = 0;
7725
7726	/* get all NVME buffers need to repost to a local list */
7727	lpfc_io_buf_flush(phba, sglist: &post_nblist);
7728
7729	/* post the list of nvme buffer sgls to port if available */
7730	if (!list_empty(head: &post_nblist)) {
7731	num_posted = lpfc_sli4_post_io_sgl_list(
7732	phba, blist: &post_nblist, xricnt: phba->sli4_hba.io_xri_cnt);
7733	/* failed to post any nvme buffer, return error */
7734	if (num_posted == 0)
7735	rc = -EIO;
7736	}
7737	return rc;
7738	}
7739
7740	static void
7741	lpfc_set_host_data(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox)
7742	{
7743	uint32_t len;
7744
7745	len = sizeof(struct lpfc_mbx_set_host_data) -
7746	sizeof(struct lpfc_sli4_cfg_mhdr);
7747	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
7748	LPFC_MBOX_OPCODE_SET_HOST_DATA, len,
7749	LPFC_SLI4_MBX_EMBED);
7750
7751	mbox->u.mqe.un.set_host_data.param_id = LPFC_SET_HOST_OS_DRIVER_VERSION;
7752	mbox->u.mqe.un.set_host_data.param_len =
7753	LPFC_HOST_OS_DRIVER_VERSION_SIZE;
7754	snprintf(buf: mbox->u.mqe.un.set_host_data.un.data,
7755	LPFC_HOST_OS_DRIVER_VERSION_SIZE,
7756	fmt: "Linux %s v"LPFC_DRIVER_VERSION,
7757	(phba->hba_flag & HBA_FCOE_MODE) ? "FCoE" : "FC");
7758	}
7759
7760	int
7761	lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq,
7762	struct lpfc_queue *drq, int count, int idx)
7763	{
7764	int rc, i;
7765	struct lpfc_rqe hrqe;
7766	struct lpfc_rqe drqe;
7767	struct lpfc_rqb *rqbp;
7768	unsigned long flags;
7769	struct rqb_dmabuf *rqb_buffer;
7770	LIST_HEAD(rqb_buf_list);
7771
7772	rqbp = hrq->rqbp;
7773	for (i = 0; i < count; i++) {
7774	spin_lock_irqsave(&phba->hbalock, flags);
7775	/* IF RQ is already full, don't bother */
7776	if (rqbp->buffer_count + i >= rqbp->entry_count - 1) {
7777	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
7778	break;
7779	}
7780	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
7781
7782	rqb_buffer = rqbp->rqb_alloc_buffer(phba);
7783	if (!rqb_buffer)
7784	break;
7785	rqb_buffer->hrq = hrq;
7786	rqb_buffer->drq = drq;
7787	rqb_buffer->idx = idx;
7788	list_add_tail(new: &rqb_buffer->hbuf.list, head: &rqb_buf_list);
7789	}
7790
7791	spin_lock_irqsave(&phba->hbalock, flags);
7792	while (!list_empty(head: &rqb_buf_list)) {
7793	list_remove_head(&rqb_buf_list, rqb_buffer, struct rqb_dmabuf,
7794	hbuf.list);
7795
7796	hrqe.address_lo = putPaddrLow(rqb_buffer->hbuf.phys);
7797	hrqe.address_hi = putPaddrHigh(rqb_buffer->hbuf.phys);
7798	drqe.address_lo = putPaddrLow(rqb_buffer->dbuf.phys);
7799	drqe.address_hi = putPaddrHigh(rqb_buffer->dbuf.phys);
7800	rc = lpfc_sli4_rq_put(hq: hrq, dq: drq, hrqe: &hrqe, drqe: &drqe);
7801	if (rc < 0) {
7802	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7803	"6421 Cannot post to HRQ %d: %x %x %x "
7804	"DRQ %x %x\n",
7805	hrq->queue_id,
7806	hrq->host_index,
7807	hrq->hba_index,
7808	hrq->entry_count,
7809	drq->host_index,
7810	drq->hba_index);
7811	rqbp->rqb_free_buffer(phba, rqb_buffer);
7812	} else {
7813	list_add_tail(new: &rqb_buffer->hbuf.list,
7814	head: &rqbp->rqb_buffer_list);
7815	rqbp->buffer_count++;
7816	}
7817	}
7818	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
7819	return 1;
7820	}
7821
7822	static void
7823	lpfc_mbx_cmpl_read_lds_params(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
7824	{
7825	union lpfc_sli4_cfg_shdr *shdr;
7826	u32 shdr_status, shdr_add_status;
7827
7828	shdr = (union lpfc_sli4_cfg_shdr *)
7829	&pmb->u.mqe.un.sli4_config.header.cfg_shdr;
7830	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
7831	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
7832	if (shdr_status || shdr_add_status || pmb->u.mb.mbxStatus) {
7833	lpfc_printf_log(phba, KERN_INFO, LOG_LDS_EVENT | LOG_MBOX,
7834	"4622 SET_FEATURE (x%x) mbox failed, "
7835	"status x%x add_status x%x, mbx status x%x\n",
7836	LPFC_SET_LD_SIGNAL, shdr_status,
7837	shdr_add_status, pmb->u.mb.mbxStatus);
7838	phba->degrade_activate_threshold = 0;
7839	phba->degrade_deactivate_threshold = 0;
7840	phba->fec_degrade_interval = 0;
7841	goto out;
7842	}
7843
7844	phba->degrade_activate_threshold = pmb->u.mqe.un.set_feature.word7;
7845	phba->degrade_deactivate_threshold = pmb->u.mqe.un.set_feature.word8;
7846	phba->fec_degrade_interval = pmb->u.mqe.un.set_feature.word10;
7847
7848	lpfc_printf_log(phba, KERN_INFO, LOG_LDS_EVENT,
7849	"4624 Success: da x%x dd x%x interval x%x\n",
7850	phba->degrade_activate_threshold,
7851	phba->degrade_deactivate_threshold,
7852	phba->fec_degrade_interval);
7853	out:
7854	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
7855	}
7856
7857	int
7858	lpfc_read_lds_params(struct lpfc_hba *phba)
7859	{
7860	LPFC_MBOXQ_t *mboxq;
7861	int rc;
7862
7863	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
7864	if (!mboxq)
7865	return -ENOMEM;
7866
7867	lpfc_set_features(phba, mbox: mboxq, LPFC_SET_LD_SIGNAL);
7868	mboxq->vport = phba->pport;
7869	mboxq->mbox_cmpl = lpfc_mbx_cmpl_read_lds_params;
7870	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT);
7871	if (rc == MBX_NOT_FINISHED) {
7872	mempool_free(element: mboxq, pool: phba->mbox_mem_pool);
7873	return -EIO;
7874	}
7875	return 0;
7876	}
7877
7878	static void
7879	lpfc_mbx_cmpl_cgn_set_ftrs(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
7880	{
7881	struct lpfc_vport *vport = pmb->vport;
7882	union lpfc_sli4_cfg_shdr *shdr;
7883	u32 shdr_status, shdr_add_status;
7884	u32 sig, acqe;
7885
7886	/* Two outcomes. (1) Set featurs was successul and EDC negotiation
7887	* is done. (2) Mailbox failed and send FPIN support only.
7888	*/
7889	shdr = (union lpfc_sli4_cfg_shdr *)
7890	&pmb->u.mqe.un.sli4_config.header.cfg_shdr;
7891	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
7892	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
7893	if (shdr_status || shdr_add_status || pmb->u.mb.mbxStatus) {
7894	lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_CGN_MGMT,
7895	"2516 CGN SET_FEATURE mbox failed with "
7896	"status x%x add_status x%x, mbx status x%x "
7897	"Reset Congestion to FPINs only\n",
7898	shdr_status, shdr_add_status,
7899	pmb->u.mb.mbxStatus);
7900	/* If there is a mbox error, move on to RDF */
7901	phba->cgn_reg_signal = EDC_CG_SIG_NOTSUPPORTED;
7902	phba->cgn_reg_fpin = LPFC_CGN_FPIN_WARN | LPFC_CGN_FPIN_ALARM;
7903	goto out;
7904	}
7905
7906	/* Zero out Congestion Signal ACQE counter */
7907	phba->cgn_acqe_cnt = 0;
7908
7909	acqe = bf_get(lpfc_mbx_set_feature_CGN_acqe_freq,
7910	&pmb->u.mqe.un.set_feature);
7911	sig = bf_get(lpfc_mbx_set_feature_CGN_warn_freq,
7912	&pmb->u.mqe.un.set_feature);
7913	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
7914	"4620 SET_FEATURES Success: Freq: %ds %dms "
7915	" Reg: x%x x%x\n", acqe, sig,
7916	phba->cgn_reg_signal, phba->cgn_reg_fpin);
7917	out:
7918	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
7919
7920	/* Register for FPIN events from the fabric now that the
7921	* EDC common_set_features has completed.
7922	*/
7923	lpfc_issue_els_rdf(vport, retry: 0);
7924	}
7925
7926	int
7927	lpfc_config_cgn_signal(struct lpfc_hba *phba)
7928	{
7929	LPFC_MBOXQ_t *mboxq;
7930	u32 rc;
7931
7932	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
7933	if (!mboxq)
7934	goto out_rdf;
7935
7936	lpfc_set_features(phba, mbox: mboxq, LPFC_SET_CGN_SIGNAL);
7937	mboxq->vport = phba->pport;
7938	mboxq->mbox_cmpl = lpfc_mbx_cmpl_cgn_set_ftrs;
7939
7940	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
7941	"4621 SET_FEATURES: FREQ sig x%x acqe x%x: "
7942	"Reg: x%x x%x\n",
7943	phba->cgn_sig_freq, lpfc_acqe_cgn_frequency,
7944	phba->cgn_reg_signal, phba->cgn_reg_fpin);
7945
7946	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT);
7947	if (rc == MBX_NOT_FINISHED)
7948	goto out;
7949	return 0;
7950
7951	out:
7952	mempool_free(element: mboxq, pool: phba->mbox_mem_pool);
7953	out_rdf:
7954	/* If there is a mbox error, move on to RDF */
7955	phba->cgn_reg_fpin = LPFC_CGN_FPIN_WARN | LPFC_CGN_FPIN_ALARM;
7956	phba->cgn_reg_signal = EDC_CG_SIG_NOTSUPPORTED;
7957	lpfc_issue_els_rdf(vport: phba->pport, retry: 0);
7958	return -EIO;
7959	}
7960
7961	/**
7962	* lpfc_init_idle_stat_hb - Initialize idle_stat tracking
7963	* @phba: pointer to lpfc hba data structure.
7964	*
7965	* This routine initializes the per-eq idle_stat to dynamically dictate
7966	* polling decisions.
7967	*
7968	* Return codes:
7969	* None
7970	**/
7971	static void lpfc_init_idle_stat_hb(struct lpfc_hba *phba)
7972	{
7973	int i;
7974	struct lpfc_sli4_hdw_queue *hdwq;
7975	struct lpfc_queue *eq;
7976	struct lpfc_idle_stat *idle_stat;
7977	u64 wall;
7978
7979	for_each_present_cpu(i) {
7980	hdwq = &phba->sli4_hba.hdwq[phba->sli4_hba.cpu_map[i].hdwq];
7981	eq = hdwq->hba_eq;
7982
7983	/* Skip if we've already handled this eq's primary CPU */
7984	if (eq->chann != i)
7985	continue;
7986
7987	idle_stat = &phba->sli4_hba.idle_stat[i];
7988
7989	idle_stat->prev_idle = get_cpu_idle_time(cpu: i, wall: &wall, io_busy: 1);
7990	idle_stat->prev_wall = wall;
7991
7992	if (phba->nvmet_support ||
7993	phba->cmf_active_mode != LPFC_CFG_OFF ||
7994	phba->intr_type != MSIX)
7995	eq->poll_mode = LPFC_QUEUE_WORK;
7996	else
7997	eq->poll_mode = LPFC_THREADED_IRQ;
7998	}
7999
8000	if (!phba->nvmet_support && phba->intr_type == MSIX)
8001	schedule_delayed_work(dwork: &phba->idle_stat_delay_work,
8002	delay: msecs_to_jiffies(LPFC_IDLE_STAT_DELAY));
8003	}
8004
8005	static void lpfc_sli4_dip(struct lpfc_hba *phba)
8006	{
8007	uint32_t if_type;
8008
8009	if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf);
8010	if (if_type == LPFC_SLI_INTF_IF_TYPE_2 ||
8011	if_type == LPFC_SLI_INTF_IF_TYPE_6) {
8012	struct lpfc_register reg_data;
8013
8014	if (lpfc_readl(addr: phba->sli4_hba.u.if_type2.STATUSregaddr,
8015	data: &reg_data.word0))
8016	return;
8017
8018	if (bf_get(lpfc_sliport_status_dip, &reg_data))
8019	lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
8020	"2904 Firmware Dump Image Present"
8021	" on Adapter");
8022	}
8023	}
8024
8025	/**
8026	* lpfc_rx_monitor_create_ring - Initialize ring buffer for rx_monitor
8027	* @rx_monitor: Pointer to lpfc_rx_info_monitor object
8028	* @entries: Number of rx_info_entry objects to allocate in ring
8029	*
8030	* Return:
8031	* 0 - Success
8032	* ENOMEM - Failure to kmalloc
8033	**/
8034	int lpfc_rx_monitor_create_ring(struct lpfc_rx_info_monitor *rx_monitor,
8035	u32 entries)
8036	{
8037	rx_monitor->ring = kmalloc_array(n: entries, size: sizeof(struct rx_info_entry),
8038	GFP_KERNEL);
8039	if (!rx_monitor->ring)
8040	return -ENOMEM;
8041
8042	rx_monitor->head_idx = 0;
8043	rx_monitor->tail_idx = 0;
8044	spin_lock_init(&rx_monitor->lock);
8045	rx_monitor->entries = entries;
8046
8047	return 0;
8048	}
8049
8050	/**
8051	* lpfc_rx_monitor_destroy_ring - Free ring buffer for rx_monitor
8052	* @rx_monitor: Pointer to lpfc_rx_info_monitor object
8053	*
8054	* Called after cancellation of cmf_timer.
8055	**/
8056	void lpfc_rx_monitor_destroy_ring(struct lpfc_rx_info_monitor *rx_monitor)
8057	{
8058	kfree(objp: rx_monitor->ring);
8059	rx_monitor->ring = NULL;
8060	rx_monitor->entries = 0;
8061	rx_monitor->head_idx = 0;
8062	rx_monitor->tail_idx = 0;
8063	}
8064
8065	/**
8066	* lpfc_rx_monitor_record - Insert an entry into rx_monitor's ring
8067	* @rx_monitor: Pointer to lpfc_rx_info_monitor object
8068	* @entry: Pointer to rx_info_entry
8069	*
8070	* Used to insert an rx_info_entry into rx_monitor's ring. Note that this is a
8071	* deep copy of rx_info_entry not a shallow copy of the rx_info_entry ptr.
8072	*
8073	* This is called from lpfc_cmf_timer, which is in timer/softirq context.
8074	*
8075	* In cases of old data overflow, we do a best effort of FIFO order.
8076	**/
8077	void lpfc_rx_monitor_record(struct lpfc_rx_info_monitor *rx_monitor,
8078	struct rx_info_entry *entry)
8079	{
8080	struct rx_info_entry *ring = rx_monitor->ring;
8081	u32 *head_idx = &rx_monitor->head_idx;
8082	u32 *tail_idx = &rx_monitor->tail_idx;
8083	spinlock_t *ring_lock = &rx_monitor->lock;
8084	u32 ring_size = rx_monitor->entries;
8085
8086	spin_lock(lock: ring_lock);
8087	memcpy(&ring[*tail_idx], entry, sizeof(*entry));
8088	*tail_idx = (*tail_idx + 1) % ring_size;
8089
8090	/* Best effort of FIFO saved data */
8091	if (*tail_idx == *head_idx)
8092	*head_idx = (*head_idx + 1) % ring_size;
8093
8094	spin_unlock(lock: ring_lock);
8095	}
8096
8097	/**
8098	* lpfc_rx_monitor_report - Read out rx_monitor's ring
8099	* @phba: Pointer to lpfc_hba object
8100	* @rx_monitor: Pointer to lpfc_rx_info_monitor object
8101	* @buf: Pointer to char buffer that will contain rx monitor info data
8102	* @buf_len: Length buf including null char
8103	* @max_read_entries: Maximum number of entries to read out of ring
8104	*
8105	* Used to dump/read what's in rx_monitor's ring buffer.
8106	*
8107	* If buf is NULL || buf_len == 0, then it is implied that we want to log the
8108	* information to kmsg instead of filling out buf.
8109	*
8110	* Return:
8111	* Number of entries read out of the ring
8112	**/
8113	u32 lpfc_rx_monitor_report(struct lpfc_hba *phba,
8114	struct lpfc_rx_info_monitor *rx_monitor, char *buf,
8115	u32 buf_len, u32 max_read_entries)
8116	{
8117	struct rx_info_entry *ring = rx_monitor->ring;
8118	struct rx_info_entry *entry;
8119	u32 *head_idx = &rx_monitor->head_idx;
8120	u32 *tail_idx = &rx_monitor->tail_idx;
8121	spinlock_t *ring_lock = &rx_monitor->lock;
8122	u32 ring_size = rx_monitor->entries;
8123	u32 cnt = 0;
8124	char tmp[DBG_LOG_STR_SZ] = {0};
8125	bool log_to_kmsg = (!buf || !buf_len) ? true : false;
8126
8127	if (!log_to_kmsg) {
8128	/* clear the buffer to be sure */
8129	memset(buf, 0, buf_len);
8130
8131	scnprintf(buf, size: buf_len, fmt: "\t%-16s%-16s%-16s%-16s%-8s%-8s%-8s"
8132	"%-8s%-8s%-8s%-16s\n",
8133	"MaxBPI", "Tot_Data_CMF",
8134	"Tot_Data_Cmd", "Tot_Data_Cmpl",
8135	"Lat(us)", "Avg_IO", "Max_IO", "Bsy",
8136	"IO_cnt", "Info", "BWutil(ms)");
8137	}
8138
8139	/* Needs to be _irq because record is called from timer interrupt
8140	* context
8141	*/
8142	spin_lock_irq(lock: ring_lock);
8143	while (*head_idx != *tail_idx) {
8144	entry = &ring[*head_idx];
8145
8146	/* Read out this entry's data. */
8147	if (!log_to_kmsg) {
8148	/* If !log_to_kmsg, then store to buf. */
8149	scnprintf(buf: tmp, size: sizeof(tmp),
8150	fmt: "%03d:\t%-16llu%-16llu%-16llu%-16llu%-8llu"
8151	"%-8llu%-8llu%-8u%-8u%-8u%u(%u)\n",
8152	*head_idx, entry->max_bytes_per_interval,
8153	entry->cmf_bytes, entry->total_bytes,
8154	entry->rcv_bytes, entry->avg_io_latency,
8155	entry->avg_io_size, entry->max_read_cnt,
8156	entry->cmf_busy, entry->io_cnt,
8157	entry->cmf_info, entry->timer_utilization,
8158	entry->timer_interval);
8159
8160	/* Check for buffer overflow */
8161	if ((strlen(buf) + strlen(tmp)) >= buf_len)
8162	break;
8163
8164	/* Append entry's data to buffer */
8165	strlcat(p: buf, q: tmp, avail: buf_len);
8166	} else {
8167	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
8168	"4410 %02u: MBPI %llu Xmit %llu "
8169	"Cmpl %llu Lat %llu ASz %llu Info %02u "
8170	"BWUtil %u Int %u slot %u\n",
8171	cnt, entry->max_bytes_per_interval,
8172	entry->total_bytes, entry->rcv_bytes,
8173	entry->avg_io_latency,
8174	entry->avg_io_size, entry->cmf_info,
8175	entry->timer_utilization,
8176	entry->timer_interval, *head_idx);
8177	}
8178
8179	*head_idx = (*head_idx + 1) % ring_size;
8180
8181	/* Don't feed more than max_read_entries */
8182	cnt++;
8183	if (cnt >= max_read_entries)
8184	break;
8185	}
8186	spin_unlock_irq(lock: ring_lock);
8187
8188	return cnt;
8189	}
8190
8191	/**
8192	* lpfc_cmf_setup - Initialize idle_stat tracking
8193	* @phba: Pointer to HBA context object.
8194	*
8195	* This is called from HBA setup during driver load or when the HBA
8196	* comes online. this does all the initialization to support CMF and MI.
8197	**/
8198	static int
8199	lpfc_cmf_setup(struct lpfc_hba *phba)
8200	{
8201	LPFC_MBOXQ_t *mboxq;
8202	struct lpfc_dmabuf *mp;
8203	struct lpfc_pc_sli4_params *sli4_params;
8204	int rc, cmf, mi_ver;
8205
8206	rc = lpfc_sli4_refresh_params(phba);
8207	if (unlikely(rc))
8208	return rc;
8209
8210	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
8211	if (!mboxq)
8212	return -ENOMEM;
8213
8214	sli4_params = &phba->sli4_hba.pc_sli4_params;
8215
8216	/* Always try to enable MI feature if we can */
8217	if (sli4_params->mi_ver) {
8218	lpfc_set_features(phba, mbox: mboxq, LPFC_SET_ENABLE_MI);
8219	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8220	mi_ver = bf_get(lpfc_mbx_set_feature_mi,
8221	&mboxq->u.mqe.un.set_feature);
8222
8223	if (rc == MBX_SUCCESS) {
8224	if (mi_ver) {
8225	lpfc_printf_log(phba,
8226	KERN_WARNING, LOG_CGN_MGMT,
8227	"6215 MI is enabled\n");
8228	sli4_params->mi_ver = mi_ver;
8229	} else {
8230	lpfc_printf_log(phba,
8231	KERN_WARNING, LOG_CGN_MGMT,
8232	"6338 MI is disabled\n");
8233	sli4_params->mi_ver = 0;
8234	}
8235	} else {
8236	/* mi_ver is already set from GET_SLI4_PARAMETERS */
8237	lpfc_printf_log(phba, KERN_INFO,
8238	LOG_CGN_MGMT | LOG_INIT,
8239	"6245 Enable MI Mailbox x%x (x%x/x%x) "
8240	"failed, rc:x%x mi:x%x\n",
8241	bf_get(lpfc_mqe_command, &mboxq->u.mqe),
8242	lpfc_sli_config_mbox_subsys_get
8243	(phba, mboxq),
8244	lpfc_sli_config_mbox_opcode_get
8245	(phba, mboxq),
8246	rc, sli4_params->mi_ver);
8247	}
8248	} else {
8249	lpfc_printf_log(phba, KERN_WARNING, LOG_CGN_MGMT,
8250	"6217 MI is disabled\n");
8251	}
8252
8253	/* Ensure FDMI is enabled for MI if enable_mi is set */
8254	if (sli4_params->mi_ver)
8255	phba->cfg_fdmi_on = LPFC_FDMI_SUPPORT;
8256
8257	/* Always try to enable CMF feature if we can */
8258	if (sli4_params->cmf) {
8259	lpfc_set_features(phba, mbox: mboxq, LPFC_SET_ENABLE_CMF);
8260	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8261	cmf = bf_get(lpfc_mbx_set_feature_cmf,
8262	&mboxq->u.mqe.un.set_feature);
8263	if (rc == MBX_SUCCESS && cmf) {
8264	lpfc_printf_log(phba, KERN_WARNING, LOG_CGN_MGMT,
8265	"6218 CMF is enabled: mode %d\n",
8266	phba->cmf_active_mode);
8267	} else {
8268	lpfc_printf_log(phba, KERN_WARNING,
8269	LOG_CGN_MGMT | LOG_INIT,
8270	"6219 Enable CMF Mailbox x%x (x%x/x%x) "
8271	"failed, rc:x%x dd:x%x\n",
8272	bf_get(lpfc_mqe_command, &mboxq->u.mqe),
8273	lpfc_sli_config_mbox_subsys_get
8274	(phba, mboxq),
8275	lpfc_sli_config_mbox_opcode_get
8276	(phba, mboxq),
8277	rc, cmf);
8278	sli4_params->cmf = 0;
8279	phba->cmf_active_mode = LPFC_CFG_OFF;
8280	goto no_cmf;
8281	}
8282
8283	/* Allocate Congestion Information Buffer */
8284	if (!phba->cgn_i) {
8285	mp = kmalloc(size: sizeof(*mp), GFP_KERNEL);
8286	if (mp)
8287	mp->virt = dma_alloc_coherent
8288	(dev: &phba->pcidev->dev,
8289	size: sizeof(struct lpfc_cgn_info),
8290	dma_handle: &mp->phys, GFP_KERNEL);
8291	if (!mp || !mp->virt) {
8292	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
8293	"2640 Failed to alloc memory "
8294	"for Congestion Info\n");
8295	kfree(objp: mp);
8296	sli4_params->cmf = 0;
8297	phba->cmf_active_mode = LPFC_CFG_OFF;
8298	goto no_cmf;
8299	}
8300	phba->cgn_i = mp;
8301
8302	/* initialize congestion buffer info */
8303	lpfc_init_congestion_buf(phba);
8304	lpfc_init_congestion_stat(phba);
8305
8306	/* Zero out Congestion Signal counters */
8307	atomic64_set(v: &phba->cgn_acqe_stat.alarm, i: 0);
8308	atomic64_set(v: &phba->cgn_acqe_stat.warn, i: 0);
8309	}
8310
8311	rc = lpfc_sli4_cgn_params_read(phba);
8312	if (rc < 0) {
8313	lpfc_printf_log(phba, KERN_ERR, LOG_CGN_MGMT | LOG_INIT,
8314	"6242 Error reading Cgn Params (%d)\n",
8315	rc);
8316	/* Ensure CGN Mode is off */
8317	sli4_params->cmf = 0;
8318	} else if (!rc) {
8319	lpfc_printf_log(phba, KERN_ERR, LOG_CGN_MGMT | LOG_INIT,
8320	"6243 CGN Event empty object.\n");
8321	/* Ensure CGN Mode is off */
8322	sli4_params->cmf = 0;
8323	}
8324	} else {
8325	no_cmf:
8326	lpfc_printf_log(phba, KERN_WARNING, LOG_CGN_MGMT,
8327	"6220 CMF is disabled\n");
8328	}
8329
8330	/* Only register congestion buffer with firmware if BOTH
8331	* CMF and E2E are enabled.
8332	*/
8333	if (sli4_params->cmf && sli4_params->mi_ver) {
8334	rc = lpfc_reg_congestion_buf(phba);
8335	if (rc) {
8336	dma_free_coherent(dev: &phba->pcidev->dev,
8337	size: sizeof(struct lpfc_cgn_info),
8338	cpu_addr: phba->cgn_i->virt, dma_handle: phba->cgn_i->phys);
8339	kfree(objp: phba->cgn_i);
8340	phba->cgn_i = NULL;
8341	/* Ensure CGN Mode is off */
8342	phba->cmf_active_mode = LPFC_CFG_OFF;
8343	sli4_params->cmf = 0;
8344	return 0;
8345	}
8346	}
8347	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
8348	"6470 Setup MI version %d CMF %d mode %d\n",
8349	sli4_params->mi_ver, sli4_params->cmf,
8350	phba->cmf_active_mode);
8351
8352	mempool_free(element: mboxq, pool: phba->mbox_mem_pool);
8353
8354	/* Initialize atomic counters */
8355	atomic_set(v: &phba->cgn_fabric_warn_cnt, i: 0);
8356	atomic_set(v: &phba->cgn_fabric_alarm_cnt, i: 0);
8357	atomic_set(v: &phba->cgn_sync_alarm_cnt, i: 0);
8358	atomic_set(v: &phba->cgn_sync_warn_cnt, i: 0);
8359	atomic_set(v: &phba->cgn_driver_evt_cnt, i: 0);
8360	atomic_set(v: &phba->cgn_latency_evt_cnt, i: 0);
8361	atomic64_set(v: &phba->cgn_latency_evt, i: 0);
8362
8363	phba->cmf_interval_rate = LPFC_CMF_INTERVAL;
8364
8365	/* Allocate RX Monitor Buffer */
8366	if (!phba->rx_monitor) {
8367	phba->rx_monitor = kzalloc(size: sizeof(*phba->rx_monitor),
8368	GFP_KERNEL);
8369
8370	if (!phba->rx_monitor) {
8371	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
8372	"2644 Failed to alloc memory "
8373	"for RX Monitor Buffer\n");
8374	return -ENOMEM;
8375	}
8376
8377	/* Instruct the rx_monitor object to instantiate its ring */
8378	if (lpfc_rx_monitor_create_ring(rx_monitor: phba->rx_monitor,
8379	LPFC_MAX_RXMONITOR_ENTRY)) {
8380	kfree(objp: phba->rx_monitor);
8381	phba->rx_monitor = NULL;
8382	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
8383	"2645 Failed to alloc memory "
8384	"for RX Monitor's Ring\n");
8385	return -ENOMEM;
8386	}
8387	}
8388
8389	return 0;
8390	}
8391
8392	static int
8393	lpfc_set_host_tm(struct lpfc_hba *phba)
8394	{
8395	LPFC_MBOXQ_t *mboxq;
8396	uint32_t len, rc;
8397	struct timespec64 cur_time;
8398	struct tm broken;
8399	uint32_t month, day, year;
8400	uint32_t hour, minute, second;
8401	struct lpfc_mbx_set_host_date_time *tm;
8402
8403	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
8404	if (!mboxq)
8405	return -ENOMEM;
8406
8407	len = sizeof(struct lpfc_mbx_set_host_data) -
8408	sizeof(struct lpfc_sli4_cfg_mhdr);
8409	lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
8410	LPFC_MBOX_OPCODE_SET_HOST_DATA, len,
8411	LPFC_SLI4_MBX_EMBED);
8412
8413	mboxq->u.mqe.un.set_host_data.param_id = LPFC_SET_HOST_DATE_TIME;
8414	mboxq->u.mqe.un.set_host_data.param_len =
8415	sizeof(struct lpfc_mbx_set_host_date_time);
8416	tm = &mboxq->u.mqe.un.set_host_data.un.tm;
8417	ktime_get_real_ts64(tv: &cur_time);
8418	time64_to_tm(totalsecs: cur_time.tv_sec, offset: 0, result: &broken);
8419	month = broken.tm_mon + 1;
8420	day = broken.tm_mday;
8421	year = broken.tm_year - 100;
8422	hour = broken.tm_hour;
8423	minute = broken.tm_min;
8424	second = broken.tm_sec;
8425	bf_set(lpfc_mbx_set_host_month, tm, month);
8426	bf_set(lpfc_mbx_set_host_day, tm, day);
8427	bf_set(lpfc_mbx_set_host_year, tm, year);
8428	bf_set(lpfc_mbx_set_host_hour, tm, hour);
8429	bf_set(lpfc_mbx_set_host_min, tm, minute);
8430	bf_set(lpfc_mbx_set_host_sec, tm, second);
8431
8432	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8433	mempool_free(element: mboxq, pool: phba->mbox_mem_pool);
8434	return rc;
8435	}
8436
8437	/**
8438	* lpfc_sli4_hba_setup - SLI4 device initialization PCI function
8439	* @phba: Pointer to HBA context object.
8440	*
8441	* This function is the main SLI4 device initialization PCI function. This
8442	* function is called by the HBA initialization code, HBA reset code and
8443	* HBA error attention handler code. Caller is not required to hold any
8444	* locks.
8445	**/
8446	int
8447	lpfc_sli4_hba_setup(struct lpfc_hba *phba)
8448	{
8449	int rc, i, cnt, len, dd;
8450	LPFC_MBOXQ_t *mboxq;
8451	struct lpfc_mqe *mqe;
8452	uint8_t *vpd;
8453	uint32_t vpd_size;
8454	uint32_t ftr_rsp = 0;
8455	struct Scsi_Host *shost = lpfc_shost_from_vport(vport: phba->pport);
8456	struct lpfc_vport *vport = phba->pport;
8457	struct lpfc_dmabuf *mp;
8458	struct lpfc_rqb *rqbp;
8459	u32 flg;
8460
8461	/* Perform a PCI function reset to start from clean */
8462	rc = lpfc_pci_function_reset(phba);
8463	if (unlikely(rc))
8464	return -ENODEV;
8465
8466	/* Check the HBA Host Status Register for readyness */
8467	rc = lpfc_sli4_post_status_check(phba);
8468	if (unlikely(rc))
8469	return -ENODEV;
8470	else {
8471	spin_lock_irq(lock: &phba->hbalock);
8472	phba->sli.sli_flag |= LPFC_SLI_ACTIVE;
8473	flg = phba->sli.sli_flag;
8474	spin_unlock_irq(lock: &phba->hbalock);
8475	/* Allow a little time after setting SLI_ACTIVE for any polled
8476	* MBX commands to complete via BSG.
8477	*/
8478	for (i = 0; i < 50 && (flg & LPFC_SLI_MBOX_ACTIVE); i++) {
8479	msleep(msecs: 20);
8480	spin_lock_irq(lock: &phba->hbalock);
8481	flg = phba->sli.sli_flag;
8482	spin_unlock_irq(lock: &phba->hbalock);
8483	}
8484	}
8485	phba->hba_flag &= ~HBA_SETUP;
8486
8487	lpfc_sli4_dip(phba);
8488
8489	/*
8490	* Allocate a single mailbox container for initializing the
8491	* port.
8492	*/
8493	mboxq = (LPFC_MBOXQ_t *) mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
8494	if (!mboxq)
8495	return -ENOMEM;
8496
8497	/* Issue READ_REV to collect vpd and FW information. */
8498	vpd_size = SLI4_PAGE_SIZE;
8499	vpd = kzalloc(size: vpd_size, GFP_KERNEL);
8500	if (!vpd) {
8501	rc = -ENOMEM;
8502	goto out_free_mbox;
8503	}
8504
8505	rc = lpfc_sli4_read_rev(phba, mboxq, vpd, vpd_size: &vpd_size);
8506	if (unlikely(rc)) {
8507	kfree(objp: vpd);
8508	goto out_free_mbox;
8509	}
8510
8511	mqe = &mboxq->u.mqe;
8512	phba->sli_rev = bf_get(lpfc_mbx_rd_rev_sli_lvl, &mqe->un.read_rev);
8513	if (bf_get(lpfc_mbx_rd_rev_fcoe, &mqe->un.read_rev)) {
8514	phba->hba_flag |= HBA_FCOE_MODE;
8515	phba->fcp_embed_io = 0; /* SLI4 FC support only */
8516	} else {
8517	phba->hba_flag &= ~HBA_FCOE_MODE;
8518	}
8519
8520	if (bf_get(lpfc_mbx_rd_rev_cee_ver, &mqe->un.read_rev) ==
8521	LPFC_DCBX_CEE_MODE)
8522	phba->hba_flag |= HBA_FIP_SUPPORT;
8523	else
8524	phba->hba_flag &= ~HBA_FIP_SUPPORT;
8525
8526	phba->hba_flag &= ~HBA_IOQ_FLUSH;
8527
8528	if (phba->sli_rev != LPFC_SLI_REV4) {
8529	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8530	"0376 READ_REV Error. SLI Level %d "
8531	"FCoE enabled %d\n",
8532	phba->sli_rev, phba->hba_flag & HBA_FCOE_MODE);
8533	rc = -EIO;
8534	kfree(objp: vpd);
8535	goto out_free_mbox;
8536	}
8537
8538	rc = lpfc_set_host_tm(phba);
8539	lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT,
8540	"6468 Set host date / time: Status x%x:\n", rc);
8541
8542	/*
8543	* Continue initialization with default values even if driver failed
8544	* to read FCoE param config regions, only read parameters if the
8545	* board is FCoE
8546	*/
8547	if (phba->hba_flag & HBA_FCOE_MODE &&
8548	lpfc_sli4_read_fcoe_params(phba))
8549	lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_INIT,
8550	"2570 Failed to read FCoE parameters\n");
8551
8552	/*
8553	* Retrieve sli4 device physical port name, failure of doing it
8554	* is considered as non-fatal.
8555	*/
8556	rc = lpfc_sli4_retrieve_pport_name(phba);
8557	if (!rc)
8558	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
8559	"3080 Successful retrieving SLI4 device "
8560	"physical port name: %s.\n", phba->Port);
8561
8562	rc = lpfc_sli4_get_ctl_attr(phba);
8563	if (!rc)
8564	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
8565	"8351 Successful retrieving SLI4 device "
8566	"CTL ATTR\n");
8567
8568	/*
8569	* Evaluate the read rev and vpd data. Populate the driver
8570	* state with the results. If this routine fails, the failure
8571	* is not fatal as the driver will use generic values.
8572	*/
8573	rc = lpfc_parse_vpd(phba, vpd, vpd_size);
8574	if (unlikely(!rc))
8575	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8576	"0377 Error %d parsing vpd. "
8577	"Using defaults.\n", rc);
8578	kfree(objp: vpd);
8579
8580	/* Save information as VPD data */
8581	phba->vpd.rev.biuRev = mqe->un.read_rev.first_hw_rev;
8582	phba->vpd.rev.smRev = mqe->un.read_rev.second_hw_rev;
8583
8584	/*
8585	* This is because first G7 ASIC doesn't support the standard
8586	* 0x5a NVME cmd descriptor type/subtype
8587	*/
8588	if ((bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) ==
8589	LPFC_SLI_INTF_IF_TYPE_6) &&
8590	(phba->vpd.rev.biuRev == LPFC_G7_ASIC_1) &&
8591	(phba->vpd.rev.smRev == 0) &&
8592	(phba->cfg_nvme_embed_cmd == 1))
8593	phba->cfg_nvme_embed_cmd = 0;
8594
8595	phba->vpd.rev.endecRev = mqe->un.read_rev.third_hw_rev;
8596	phba->vpd.rev.fcphHigh = bf_get(lpfc_mbx_rd_rev_fcph_high,
8597	&mqe->un.read_rev);
8598	phba->vpd.rev.fcphLow = bf_get(lpfc_mbx_rd_rev_fcph_low,
8599	&mqe->un.read_rev);
8600	phba->vpd.rev.feaLevelHigh = bf_get(lpfc_mbx_rd_rev_ftr_lvl_high,
8601	&mqe->un.read_rev);
8602	phba->vpd.rev.feaLevelLow = bf_get(lpfc_mbx_rd_rev_ftr_lvl_low,
8603	&mqe->un.read_rev);
8604	phba->vpd.rev.sli1FwRev = mqe->un.read_rev.fw_id_rev;
8605	memcpy(phba->vpd.rev.sli1FwName, mqe->un.read_rev.fw_name, 16);
8606	phba->vpd.rev.sli2FwRev = mqe->un.read_rev.ulp_fw_id_rev;
8607	memcpy(phba->vpd.rev.sli2FwName, mqe->un.read_rev.ulp_fw_name, 16);
8608	phba->vpd.rev.opFwRev = mqe->un.read_rev.fw_id_rev;
8609	memcpy(phba->vpd.rev.opFwName, mqe->un.read_rev.fw_name, 16);
8610	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
8611	"(%d):0380 READ_REV Status x%x "
8612	"fw_rev:%s fcphHi:%x fcphLo:%x flHi:%x flLo:%x\n",
8613	mboxq->vport ? mboxq->vport->vpi : 0,
8614	bf_get(lpfc_mqe_status, mqe),
8615	phba->vpd.rev.opFwName,
8616	phba->vpd.rev.fcphHigh, phba->vpd.rev.fcphLow,
8617	phba->vpd.rev.feaLevelHigh, phba->vpd.rev.feaLevelLow);
8618
8619	if (bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) ==
8620	LPFC_SLI_INTF_IF_TYPE_0) {
8621	lpfc_set_features(phba, mbox: mboxq, LPFC_SET_UE_RECOVERY);
8622	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8623	if (rc == MBX_SUCCESS) {
8624	phba->hba_flag |= HBA_RECOVERABLE_UE;
8625	/* Set 1Sec interval to detect UE */
8626	phba->eratt_poll_interval = 1;
8627	phba->sli4_hba.ue_to_sr = bf_get(
8628	lpfc_mbx_set_feature_UESR,
8629	&mboxq->u.mqe.un.set_feature);
8630	phba->sli4_hba.ue_to_rp = bf_get(
8631	lpfc_mbx_set_feature_UERP,
8632	&mboxq->u.mqe.un.set_feature);
8633	}
8634	}
8635
8636	if (phba->cfg_enable_mds_diags && phba->mds_diags_support) {
8637	/* Enable MDS Diagnostics only if the SLI Port supports it */
8638	lpfc_set_features(phba, mbox: mboxq, LPFC_SET_MDS_DIAGS);
8639	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8640	if (rc != MBX_SUCCESS)
8641	phba->mds_diags_support = 0;
8642	}
8643
8644	/*
8645	* Discover the port's supported feature set and match it against the
8646	* hosts requests.
8647	*/
8648	lpfc_request_features(phba, mboxq);
8649	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8650	if (unlikely(rc)) {
8651	rc = -EIO;
8652	goto out_free_mbox;
8653	}
8654
8655	/* Disable VMID if app header is not supported */
8656	if (phba->cfg_vmid_app_header && !(bf_get(lpfc_mbx_rq_ftr_rsp_ashdr,
8657	&mqe->un.req_ftrs))) {
8658	bf_set(lpfc_ftr_ashdr, &phba->sli4_hba.sli4_flags, 0);
8659	phba->cfg_vmid_app_header = 0;
8660	lpfc_printf_log(phba, KERN_DEBUG, LOG_SLI,
8661	"1242 vmid feature not supported\n");
8662	}
8663
8664	/*
8665	* The port must support FCP initiator mode as this is the
8666	* only mode running in the host.
8667	*/
8668	if (!(bf_get(lpfc_mbx_rq_ftr_rsp_fcpi, &mqe->un.req_ftrs))) {
8669	lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
8670	"0378 No support for fcpi mode.\n");
8671	ftr_rsp++;
8672	}
8673
8674	/* Performance Hints are ONLY for FCoE */
8675	if (phba->hba_flag & HBA_FCOE_MODE) {
8676	if (bf_get(lpfc_mbx_rq_ftr_rsp_perfh, &mqe->un.req_ftrs))
8677	phba->sli3_options |= LPFC_SLI4_PERFH_ENABLED;
8678	else
8679	phba->sli3_options &= ~LPFC_SLI4_PERFH_ENABLED;
8680	}
8681
8682	/*
8683	* If the port cannot support the host's requested features
8684	* then turn off the global config parameters to disable the
8685	* feature in the driver. This is not a fatal error.
8686	*/
8687	if (phba->sli3_options & LPFC_SLI3_BG_ENABLED) {
8688	if (!(bf_get(lpfc_mbx_rq_ftr_rsp_dif, &mqe->un.req_ftrs))) {
8689	phba->cfg_enable_bg = 0;
8690	phba->sli3_options &= ~LPFC_SLI3_BG_ENABLED;
8691	ftr_rsp++;
8692	}
8693	}
8694
8695	if (phba->max_vpi && phba->cfg_enable_npiv &&
8696	!(bf_get(lpfc_mbx_rq_ftr_rsp_npiv, &mqe->un.req_ftrs)))
8697	ftr_rsp++;
8698
8699	if (ftr_rsp) {
8700	lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
8701	"0379 Feature Mismatch Data: x%08x %08x "
8702	"x%x x%x x%x\n", mqe->un.req_ftrs.word2,
8703	mqe->un.req_ftrs.word3, phba->cfg_enable_bg,
8704	phba->cfg_enable_npiv, phba->max_vpi);
8705	if (!(bf_get(lpfc_mbx_rq_ftr_rsp_dif, &mqe->un.req_ftrs)))
8706	phba->cfg_enable_bg = 0;
8707	if (!(bf_get(lpfc_mbx_rq_ftr_rsp_npiv, &mqe->un.req_ftrs)))
8708	phba->cfg_enable_npiv = 0;
8709	}
8710
8711	/* These SLI3 features are assumed in SLI4 */
8712	spin_lock_irq(lock: &phba->hbalock);
8713	phba->sli3_options |= (LPFC_SLI3_NPIV_ENABLED | LPFC_SLI3_HBQ_ENABLED);
8714	spin_unlock_irq(lock: &phba->hbalock);
8715
8716	/* Always try to enable dual dump feature if we can */
8717	lpfc_set_features(phba, mbox: mboxq, LPFC_SET_DUAL_DUMP);
8718	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8719	dd = bf_get(lpfc_mbx_set_feature_dd, &mboxq->u.mqe.un.set_feature);
8720	if ((rc == MBX_SUCCESS) && (dd == LPFC_ENABLE_DUAL_DUMP))
8721	lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
8722	"6448 Dual Dump is enabled\n");
8723	else
8724	lpfc_printf_log(phba, KERN_INFO, LOG_SLI | LOG_INIT,
8725	"6447 Dual Dump Mailbox x%x (x%x/x%x) failed, "
8726	"rc:x%x dd:x%x\n",
8727	bf_get(lpfc_mqe_command, &mboxq->u.mqe),
8728	lpfc_sli_config_mbox_subsys_get(
8729	phba, mboxq),
8730	lpfc_sli_config_mbox_opcode_get(
8731	phba, mboxq),
8732	rc, dd);
8733	/*
8734	* Allocate all resources (xri,rpi,vpi,vfi) now. Subsequent
8735	* calls depends on these resources to complete port setup.
8736	*/
8737	rc = lpfc_sli4_alloc_resource_identifiers(phba);
8738	if (rc) {
8739	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8740	"2920 Failed to alloc Resource IDs "
8741	"rc = x%x\n", rc);
8742	goto out_free_mbox;
8743	}
8744
8745	lpfc_set_host_data(phba, mbox: mboxq);
8746
8747	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8748	if (rc) {
8749	lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
8750	"2134 Failed to set host os driver version %x",
8751	rc);
8752	}
8753
8754	/* Read the port's service parameters. */
8755	rc = lpfc_read_sparam(phba, mboxq, vport->vpi);
8756	if (rc) {
8757	phba->link_state = LPFC_HBA_ERROR;
8758	rc = -ENOMEM;
8759	goto out_free_mbox;
8760	}
8761
8762	mboxq->vport = vport;
8763	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8764	mp = (struct lpfc_dmabuf *)mboxq->ctx_buf;
8765	if (rc == MBX_SUCCESS) {
8766	memcpy(&vport->fc_sparam, mp->virt, sizeof(struct serv_parm));
8767	rc = 0;
8768	}
8769
8770	/*
8771	* This memory was allocated by the lpfc_read_sparam routine but is
8772	* no longer needed. It is released and ctx_buf NULLed to prevent
8773	* unintended pointer access as the mbox is reused.
8774	*/
8775	lpfc_mbuf_free(phba, mp->virt, mp->phys);
8776	kfree(objp: mp);
8777	mboxq->ctx_buf = NULL;
8778	if (unlikely(rc)) {
8779	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8780	"0382 READ_SPARAM command failed "
8781	"status %d, mbxStatus x%x\n",
8782	rc, bf_get(lpfc_mqe_status, mqe));
8783	phba->link_state = LPFC_HBA_ERROR;
8784	rc = -EIO;
8785	goto out_free_mbox;
8786	}
8787
8788	lpfc_update_vport_wwn(vport);
8789
8790	/* Update the fc_host data structures with new wwn. */
8791	fc_host_node_name(shost) = wwn_to_u64(wwn: vport->fc_nodename.u.wwn);
8792	fc_host_port_name(shost) = wwn_to_u64(wwn: vport->fc_portname.u.wwn);
8793
8794	/* Create all the SLI4 queues */
8795	rc = lpfc_sli4_queue_create(phba);
8796	if (rc) {
8797	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8798	"3089 Failed to allocate queues\n");
8799	rc = -ENODEV;
8800	goto out_free_mbox;
8801	}
8802	/* Set up all the queues to the device */
8803	rc = lpfc_sli4_queue_setup(phba);
8804	if (unlikely(rc)) {
8805	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8806	"0381 Error %d during queue setup.\n ", rc);
8807	goto out_stop_timers;
8808	}
8809	/* Initialize the driver internal SLI layer lists. */
8810	lpfc_sli4_setup(phba);
8811	lpfc_sli4_queue_init(phba);
8812
8813	/* update host els xri-sgl sizes and mappings */
8814	rc = lpfc_sli4_els_sgl_update(phba);
8815	if (unlikely(rc)) {
8816	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8817	"1400 Failed to update xri-sgl size and "
8818	"mapping: %d\n", rc);
8819	goto out_destroy_queue;
8820	}
8821
8822	/* register the els sgl pool to the port */
8823	rc = lpfc_sli4_repost_sgl_list(phba, sgl_list: &phba->sli4_hba.lpfc_els_sgl_list,
8824	cnt: phba->sli4_hba.els_xri_cnt);
8825	if (unlikely(rc < 0)) {
8826	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8827	"0582 Error %d during els sgl post "
8828	"operation\n", rc);
8829	rc = -ENODEV;
8830	goto out_destroy_queue;
8831	}
8832	phba->sli4_hba.els_xri_cnt = rc;
8833
8834	if (phba->nvmet_support) {
8835	/* update host nvmet xri-sgl sizes and mappings */
8836	rc = lpfc_sli4_nvmet_sgl_update(phba);
8837	if (unlikely(rc)) {
8838	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8839	"6308 Failed to update nvmet-sgl size "
8840	"and mapping: %d\n", rc);
8841	goto out_destroy_queue;
8842	}
8843
8844	/* register the nvmet sgl pool to the port */
8845	rc = lpfc_sli4_repost_sgl_list(
8846	phba,
8847	sgl_list: &phba->sli4_hba.lpfc_nvmet_sgl_list,
8848	cnt: phba->sli4_hba.nvmet_xri_cnt);
8849	if (unlikely(rc < 0)) {
8850	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8851	"3117 Error %d during nvmet "
8852	"sgl post\n", rc);
8853	rc = -ENODEV;
8854	goto out_destroy_queue;
8855	}
8856	phba->sli4_hba.nvmet_xri_cnt = rc;
8857
8858	/* We allocate an iocbq for every receive context SGL.
8859	* The additional allocation is for abort and ls handling.
8860	*/
8861	cnt = phba->sli4_hba.nvmet_xri_cnt +
8862	phba->sli4_hba.max_cfg_param.max_xri;
8863	} else {
8864	/* update host common xri-sgl sizes and mappings */
8865	rc = lpfc_sli4_io_sgl_update(phba);
8866	if (unlikely(rc)) {
8867	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8868	"6082 Failed to update nvme-sgl size "
8869	"and mapping: %d\n", rc);
8870	goto out_destroy_queue;
8871	}
8872
8873	/* register the allocated common sgl pool to the port */
8874	rc = lpfc_sli4_repost_io_sgl_list(phba);
8875	if (unlikely(rc)) {
8876	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8877	"6116 Error %d during nvme sgl post "
8878	"operation\n", rc);
8879	/* Some NVME buffers were moved to abort nvme list */
8880	/* A pci function reset will repost them */
8881	rc = -ENODEV;
8882	goto out_destroy_queue;
8883	}
8884	/* Each lpfc_io_buf job structure has an iocbq element.
8885	* This cnt provides for abort, els, ct and ls requests.
8886	*/
8887	cnt = phba->sli4_hba.max_cfg_param.max_xri;
8888	}
8889
8890	if (!phba->sli.iocbq_lookup) {
8891	/* Initialize and populate the iocb list per host */
8892	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
8893	"2821 initialize iocb list with %d entries\n",
8894	cnt);
8895	rc = lpfc_init_iocb_list(phba, cnt);
8896	if (rc) {
8897	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8898	"1413 Failed to init iocb list.\n");
8899	goto out_destroy_queue;
8900	}
8901	}
8902
8903	if (phba->nvmet_support)
8904	lpfc_nvmet_create_targetport(phba);
8905
8906	if (phba->nvmet_support && phba->cfg_nvmet_mrq) {
8907	/* Post initial buffers to all RQs created */
8908	for (i = 0; i < phba->cfg_nvmet_mrq; i++) {
8909	rqbp = phba->sli4_hba.nvmet_mrq_hdr[i]->rqbp;
8910	INIT_LIST_HEAD(list: &rqbp->rqb_buffer_list);
8911	rqbp->rqb_alloc_buffer = lpfc_sli4_nvmet_alloc;
8912	rqbp->rqb_free_buffer = lpfc_sli4_nvmet_free;
8913	rqbp->entry_count = LPFC_NVMET_RQE_DEF_COUNT;
8914	rqbp->buffer_count = 0;
8915
8916	lpfc_post_rq_buffer(
8917	phba, hrq: phba->sli4_hba.nvmet_mrq_hdr[i],
8918	drq: phba->sli4_hba.nvmet_mrq_data[i],
8919	count: phba->cfg_nvmet_mrq_post, idx: i);
8920	}
8921	}
8922
8923	/* Post the rpi header region to the device. */
8924	rc = lpfc_sli4_post_all_rpi_hdrs(phba);
8925	if (unlikely(rc)) {
8926	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8927	"0393 Error %d during rpi post operation\n",
8928	rc);
8929	rc = -ENODEV;
8930	goto out_free_iocblist;
8931	}
8932	lpfc_sli4_node_prep(phba);
8933
8934	if (!(phba->hba_flag & HBA_FCOE_MODE)) {
8935	if ((phba->nvmet_support == 0) || (phba->cfg_nvmet_mrq == 1)) {
8936	/*
8937	* The FC Port needs to register FCFI (index 0)
8938	*/
8939	lpfc_reg_fcfi(phba, mboxq);
8940	mboxq->vport = phba->pport;
8941	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8942	if (rc != MBX_SUCCESS)
8943	goto out_unset_queue;
8944	rc = 0;
8945	phba->fcf.fcfi = bf_get(lpfc_reg_fcfi_fcfi,
8946	&mboxq->u.mqe.un.reg_fcfi);
8947	} else {
8948	/* We are a NVME Target mode with MRQ > 1 */
8949
8950	/* First register the FCFI */
8951	lpfc_reg_fcfi_mrq(phba, mbox: mboxq, mode: 0);
8952	mboxq->vport = phba->pport;
8953	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8954	if (rc != MBX_SUCCESS)
8955	goto out_unset_queue;
8956	rc = 0;
8957	phba->fcf.fcfi = bf_get(lpfc_reg_fcfi_mrq_fcfi,
8958	&mboxq->u.mqe.un.reg_fcfi_mrq);
8959
8960	/* Next register the MRQs */
8961	lpfc_reg_fcfi_mrq(phba, mbox: mboxq, mode: 1);
8962	mboxq->vport = phba->pport;
8963	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8964	if (rc != MBX_SUCCESS)
8965	goto out_unset_queue;
8966	rc = 0;
8967	}
8968	/* Check if the port is configured to be disabled */
8969	lpfc_sli_read_link_ste(phba);
8970	}
8971
8972	/* Don't post more new bufs if repost already recovered
8973	* the nvme sgls.
8974	*/
8975	if (phba->nvmet_support == 0) {
8976	if (phba->sli4_hba.io_xri_cnt == 0) {
8977	len = lpfc_new_io_buf(
8978	phba, num_to_alloc: phba->sli4_hba.io_xri_max);
8979	if (len == 0) {
8980	rc = -ENOMEM;
8981	goto out_unset_queue;
8982	}
8983
8984	if (phba->cfg_xri_rebalancing)
8985	lpfc_create_multixri_pools(phba);
8986	}
8987	} else {
8988	phba->cfg_xri_rebalancing = 0;
8989	}
8990
8991	/* Allow asynchronous mailbox command to go through */
8992	spin_lock_irq(lock: &phba->hbalock);
8993	phba->sli.sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
8994	spin_unlock_irq(lock: &phba->hbalock);
8995
8996	/* Post receive buffers to the device */
8997	lpfc_sli4_rb_setup(phba);
8998
8999	/* Reset HBA FCF states after HBA reset */
9000	phba->fcf.fcf_flag = 0;
9001	phba->fcf.current_rec.flag = 0;
9002
9003	/* Start the ELS watchdog timer */
9004	mod_timer(timer: &vport->els_tmofunc,
9005	expires: jiffies + msecs_to_jiffies(m: 1000 * (phba->fc_ratov * 2)));
9006
9007	/* Start heart beat timer */
9008	mod_timer(timer: &phba->hb_tmofunc,
9009	expires: jiffies + msecs_to_jiffies(m: 1000 * LPFC_HB_MBOX_INTERVAL));
9010	phba->hba_flag &= ~(HBA_HBEAT_INP | HBA_HBEAT_TMO);
9011	phba->last_completion_time = jiffies;
9012
9013	/* start eq_delay heartbeat */
9014	if (phba->cfg_auto_imax)
9015	queue_delayed_work(wq: phba->wq, dwork: &phba->eq_delay_work,
9016	delay: msecs_to_jiffies(LPFC_EQ_DELAY_MSECS));
9017
9018	/* start per phba idle_stat_delay heartbeat */
9019	lpfc_init_idle_stat_hb(phba);
9020
9021	/* Start error attention (ERATT) polling timer */
9022	mod_timer(timer: &phba->eratt_poll,
9023	expires: jiffies + msecs_to_jiffies(m: 1000 * phba->eratt_poll_interval));
9024
9025	/*
9026	* The port is ready, set the host's link state to LINK_DOWN
9027	* in preparation for link interrupts.
9028	*/
9029	spin_lock_irq(lock: &phba->hbalock);
9030	phba->link_state = LPFC_LINK_DOWN;
9031
9032	/* Check if physical ports are trunked */
9033	if (bf_get(lpfc_conf_trunk_port0, &phba->sli4_hba))
9034	phba->trunk_link.link0.state = LPFC_LINK_DOWN;
9035	if (bf_get(lpfc_conf_trunk_port1, &phba->sli4_hba))
9036	phba->trunk_link.link1.state = LPFC_LINK_DOWN;
9037	if (bf_get(lpfc_conf_trunk_port2, &phba->sli4_hba))
9038	phba->trunk_link.link2.state = LPFC_LINK_DOWN;
9039	if (bf_get(lpfc_conf_trunk_port3, &phba->sli4_hba))
9040	phba->trunk_link.link3.state = LPFC_LINK_DOWN;
9041	spin_unlock_irq(lock: &phba->hbalock);
9042
9043	/* Arm the CQs and then EQs on device */
9044	lpfc_sli4_arm_cqeq_intr(phba);
9045
9046	/* Indicate device interrupt mode */
9047	phba->sli4_hba.intr_enable = 1;
9048
9049	/* Setup CMF after HBA is initialized */
9050	lpfc_cmf_setup(phba);
9051
9052	if (!(phba->hba_flag & HBA_FCOE_MODE) &&
9053	(phba->hba_flag & LINK_DISABLED)) {
9054	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9055	"3103 Adapter Link is disabled.\n");
9056	lpfc_down_link(phba, mboxq);
9057	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
9058	if (rc != MBX_SUCCESS) {
9059	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9060	"3104 Adapter failed to issue "
9061	"DOWN_LINK mbox cmd, rc:x%x\n", rc);
9062	goto out_io_buff_free;
9063	}
9064	} else if (phba->cfg_suppress_link_up == LPFC_INITIALIZE_LINK) {
9065	/* don't perform init_link on SLI4 FC port loopback test */
9066	if (!(phba->link_flag & LS_LOOPBACK_MODE)) {
9067	rc = phba->lpfc_hba_init_link(phba, MBX_NOWAIT);
9068	if (rc)
9069	goto out_io_buff_free;
9070	}
9071	}
9072	mempool_free(element: mboxq, pool: phba->mbox_mem_pool);
9073
9074	/* Enable RAS FW log support */
9075	lpfc_sli4_ras_setup(phba);
9076
9077	phba->hba_flag |= HBA_SETUP;
9078	return rc;
9079
9080	out_io_buff_free:
9081	/* Free allocated IO Buffers */
9082	lpfc_io_free(phba);
9083	out_unset_queue:
9084	/* Unset all the queues set up in this routine when error out */
9085	lpfc_sli4_queue_unset(phba);
9086	out_free_iocblist:
9087	lpfc_free_iocb_list(phba);
9088	out_destroy_queue:
9089	lpfc_sli4_queue_destroy(phba);
9090	out_stop_timers:
9091	lpfc_stop_hba_timers(phba);
9092	out_free_mbox:
9093	mempool_free(element: mboxq, pool: phba->mbox_mem_pool);
9094	return rc;
9095	}
9096
9097	/**
9098	* lpfc_mbox_timeout - Timeout call back function for mbox timer
9099	* @t: Context to fetch pointer to hba structure from.
9100	*
9101	* This is the callback function for mailbox timer. The mailbox
9102	* timer is armed when a new mailbox command is issued and the timer
9103	* is deleted when the mailbox complete. The function is called by
9104	* the kernel timer code when a mailbox does not complete within
9105	* expected time. This function wakes up the worker thread to
9106	* process the mailbox timeout and returns. All the processing is
9107	* done by the worker thread function lpfc_mbox_timeout_handler.
9108	**/
9109	void
9110	lpfc_mbox_timeout(struct timer_list *t)
9111	{
9112	struct lpfc_hba *phba = from_timer(phba, t, sli.mbox_tmo);
9113	unsigned long iflag;
9114	uint32_t tmo_posted;
9115
9116	spin_lock_irqsave(&phba->pport->work_port_lock, iflag);
9117	tmo_posted = phba->pport->work_port_events & WORKER_MBOX_TMO;
9118	if (!tmo_posted)
9119	phba->pport->work_port_events |= WORKER_MBOX_TMO;
9120	spin_unlock_irqrestore(lock: &phba->pport->work_port_lock, flags: iflag);
9121
9122	if (!tmo_posted)
9123	lpfc_worker_wake_up(phba);
9124	return;
9125	}
9126
9127	/**
9128	* lpfc_sli4_mbox_completions_pending - check to see if any mailbox completions
9129	* are pending
9130	* @phba: Pointer to HBA context object.
9131	*
9132	* This function checks if any mailbox completions are present on the mailbox
9133	* completion queue.
9134	**/
9135	static bool
9136	lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba)
9137	{
9138
9139	uint32_t idx;
9140	struct lpfc_queue *mcq;
9141	struct lpfc_mcqe *mcqe;
9142	bool pending_completions = false;
9143	uint8_t qe_valid;
9144
9145	if (unlikely(!phba) || (phba->sli_rev != LPFC_SLI_REV4))
9146	return false;
9147
9148	/* Check for completions on mailbox completion queue */
9149
9150	mcq = phba->sli4_hba.mbx_cq;
9151	idx = mcq->hba_index;
9152	qe_valid = mcq->qe_valid;
9153	while (bf_get_le32(lpfc_cqe_valid,
9154	(struct lpfc_cqe *)lpfc_sli4_qe(mcq, idx)) == qe_valid) {
9155	mcqe = (struct lpfc_mcqe *)(lpfc_sli4_qe(q: mcq, idx));
9156	if (bf_get_le32(lpfc_trailer_completed, mcqe) &&
9157	(!bf_get_le32(lpfc_trailer_async, mcqe))) {
9158	pending_completions = true;
9159	break;
9160	}
9161	idx = (idx + 1) % mcq->entry_count;
9162	if (mcq->hba_index == idx)
9163	break;
9164
9165	/* if the index wrapped around, toggle the valid bit */
9166	if (phba->sli4_hba.pc_sli4_params.cqav && !idx)
9167	qe_valid = (qe_valid) ? 0 : 1;
9168	}
9169	return pending_completions;
9170
9171	}
9172
9173	/**
9174	* lpfc_sli4_process_missed_mbox_completions - process mbox completions
9175	* that were missed.
9176	* @phba: Pointer to HBA context object.
9177	*
9178	* For sli4, it is possible to miss an interrupt. As such mbox completions
9179	* maybe missed causing erroneous mailbox timeouts to occur. This function
9180	* checks to see if mbox completions are on the mailbox completion queue
9181	* and will process all the completions associated with the eq for the
9182	* mailbox completion queue.
9183	**/
9184	static bool
9185	lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba)
9186	{
9187	struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba;
9188	uint32_t eqidx;
9189	struct lpfc_queue *fpeq = NULL;
9190	struct lpfc_queue *eq;
9191	bool mbox_pending;
9192
9193	if (unlikely(!phba) || (phba->sli_rev != LPFC_SLI_REV4))
9194	return false;
9195
9196	/* Find the EQ associated with the mbox CQ */
9197	if (sli4_hba->hdwq) {
9198	for (eqidx = 0; eqidx < phba->cfg_irq_chann; eqidx++) {
9199	eq = phba->sli4_hba.hba_eq_hdl[eqidx].eq;
9200	if (eq && eq->queue_id == sli4_hba->mbx_cq->assoc_qid) {
9201	fpeq = eq;
9202	break;
9203	}
9204	}
9205	}
9206	if (!fpeq)
9207	return false;
9208
9209	/* Turn off interrupts from this EQ */
9210
9211	sli4_hba->sli4_eq_clr_intr(fpeq);
9212
9213	/* Check to see if a mbox completion is pending */
9214
9215	mbox_pending = lpfc_sli4_mbox_completions_pending(phba);
9216
9217	/*
9218	* If a mbox completion is pending, process all the events on EQ
9219	* associated with the mbox completion queue (this could include
9220	* mailbox commands, async events, els commands, receive queue data
9221	* and fcp commands)
9222	*/
9223
9224	if (mbox_pending)
9225	/* process and rearm the EQ */
9226	lpfc_sli4_process_eq(phba, eq: fpeq, LPFC_QUEUE_REARM,
9227	poll_mode: LPFC_QUEUE_WORK);
9228	else
9229	/* Always clear and re-arm the EQ */
9230	sli4_hba->sli4_write_eq_db(phba, fpeq, 0, LPFC_QUEUE_REARM);
9231
9232	return mbox_pending;
9233
9234	}
9235
9236	/**
9237	* lpfc_mbox_timeout_handler - Worker thread function to handle mailbox timeout
9238	* @phba: Pointer to HBA context object.
9239	*
9240	* This function is called from worker thread when a mailbox command times out.
9241	* The caller is not required to hold any locks. This function will reset the
9242	* HBA and recover all the pending commands.
9243	**/
9244	void
9245	lpfc_mbox_timeout_handler(struct lpfc_hba *phba)
9246	{
9247	LPFC_MBOXQ_t *pmbox = phba->sli.mbox_active;
9248	MAILBOX_t *mb = NULL;
9249
9250	struct lpfc_sli *psli = &phba->sli;
9251
9252	/* If the mailbox completed, process the completion */
9253	lpfc_sli4_process_missed_mbox_completions(phba);
9254
9255	if (!(psli->sli_flag & LPFC_SLI_ACTIVE))
9256	return;
9257
9258	if (pmbox != NULL)
9259	mb = &pmbox->u.mb;
9260	/* Check the pmbox pointer first. There is a race condition
9261	* between the mbox timeout handler getting executed in the
9262	* worklist and the mailbox actually completing. When this
9263	* race condition occurs, the mbox_active will be NULL.
9264	*/
9265	spin_lock_irq(lock: &phba->hbalock);
9266	if (pmbox == NULL) {
9267	lpfc_printf_log(phba, KERN_WARNING,
9268	LOG_MBOX | LOG_SLI,
9269	"0353 Active Mailbox cleared - mailbox timeout "
9270	"exiting\n");
9271	spin_unlock_irq(lock: &phba->hbalock);
9272	return;
9273	}
9274
9275	/* Mbox cmd <mbxCommand> timeout */
9276	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9277	"0310 Mailbox command x%x timeout Data: x%x x%x x%px\n",
9278	mb->mbxCommand,
9279	phba->pport->port_state,
9280	phba->sli.sli_flag,
9281	phba->sli.mbox_active);
9282	spin_unlock_irq(lock: &phba->hbalock);
9283
9284	/* Setting state unknown so lpfc_sli_abort_iocb_ring
9285	* would get IOCB_ERROR from lpfc_sli_issue_iocb, allowing
9286	* it to fail all outstanding SCSI IO.
9287	*/
9288	set_bit(nr: MBX_TMO_ERR, addr: &phba->bit_flags);
9289	spin_lock_irq(lock: &phba->pport->work_port_lock);
9290	phba->pport->work_port_events &= ~WORKER_MBOX_TMO;
9291	spin_unlock_irq(lock: &phba->pport->work_port_lock);
9292	spin_lock_irq(lock: &phba->hbalock);
9293	phba->link_state = LPFC_LINK_UNKNOWN;
9294	psli->sli_flag &= ~LPFC_SLI_ACTIVE;
9295	spin_unlock_irq(lock: &phba->hbalock);
9296
9297	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9298	"0345 Resetting board due to mailbox timeout\n");
9299
9300	/* Reset the HBA device */
9301	lpfc_reset_hba(phba);
9302	}
9303
9304	/**
9305	* lpfc_sli_issue_mbox_s3 - Issue an SLI3 mailbox command to firmware
9306	* @phba: Pointer to HBA context object.
9307	* @pmbox: Pointer to mailbox object.
9308	* @flag: Flag indicating how the mailbox need to be processed.
9309	*
9310	* This function is called by discovery code and HBA management code
9311	* to submit a mailbox command to firmware with SLI-3 interface spec. This
9312	* function gets the hbalock to protect the data structures.
9313	* The mailbox command can be submitted in polling mode, in which case
9314	* this function will wait in a polling loop for the completion of the
9315	* mailbox.
9316	* If the mailbox is submitted in no_wait mode (not polling) the
9317	* function will submit the command and returns immediately without waiting
9318	* for the mailbox completion. The no_wait is supported only when HBA
9319	* is in SLI2/SLI3 mode - interrupts are enabled.
9320	* The SLI interface allows only one mailbox pending at a time. If the
9321	* mailbox is issued in polling mode and there is already a mailbox
9322	* pending, then the function will return an error. If the mailbox is issued
9323	* in NO_WAIT mode and there is a mailbox pending already, the function
9324	* will return MBX_BUSY after queuing the mailbox into mailbox queue.
9325	* The sli layer owns the mailbox object until the completion of mailbox
9326	* command if this function return MBX_BUSY or MBX_SUCCESS. For all other
9327	* return codes the caller owns the mailbox command after the return of
9328	* the function.
9329	**/
9330	static int
9331	lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox,
9332	uint32_t flag)
9333	{
9334	MAILBOX_t *mbx;
9335	struct lpfc_sli *psli = &phba->sli;
9336	uint32_t status, evtctr;
9337	uint32_t ha_copy, hc_copy;
9338	int i;
9339	unsigned long timeout;
9340	unsigned long drvr_flag = 0;
9341	uint32_t word0, ldata;
9342	void __iomem *to_slim;
9343	int processing_queue = 0;
9344
9345	spin_lock_irqsave(&phba->hbalock, drvr_flag);
9346	if (!pmbox) {
9347	phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
9348	/* processing mbox queue from intr_handler */
9349	if (unlikely(psli->sli_flag & LPFC_SLI_ASYNC_MBX_BLK)) {
9350	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
9351	return MBX_SUCCESS;
9352	}
9353	processing_queue = 1;
9354	pmbox = lpfc_mbox_get(phba);
9355	if (!pmbox) {
9356	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
9357	return MBX_SUCCESS;
9358	}
9359	}
9360
9361	if (pmbox->mbox_cmpl && pmbox->mbox_cmpl != lpfc_sli_def_mbox_cmpl &&
9362	pmbox->mbox_cmpl != lpfc_sli_wake_mbox_wait) {
9363	if(!pmbox->vport) {
9364	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
9365	lpfc_printf_log(phba, KERN_ERR,
9366	LOG_MBOX | LOG_VPORT,
9367	"1806 Mbox x%x failed. No vport\n",
9368	pmbox->u.mb.mbxCommand);
9369	dump_stack();
9370	goto out_not_finished;
9371	}
9372	}
9373
9374	/* If the PCI channel is in offline state, do not post mbox. */
9375	if (unlikely(pci_channel_offline(phba->pcidev))) {
9376	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
9377	goto out_not_finished;
9378	}
9379
9380	/* If HBA has a deferred error attention, fail the iocb. */
9381	if (unlikely(phba->hba_flag & DEFER_ERATT)) {
9382	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
9383	goto out_not_finished;
9384	}
9385
9386	psli = &phba->sli;
9387
9388	mbx = &pmbox->u.mb;
9389	status = MBX_SUCCESS;
9390
9391	if (phba->link_state == LPFC_HBA_ERROR) {
9392	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
9393
9394	/* Mbox command <mbxCommand> cannot issue */
9395	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9396	"(%d):0311 Mailbox command x%x cannot "
9397	"issue Data: x%x x%x\n",
9398	pmbox->vport ? pmbox->vport->vpi : 0,
9399	pmbox->u.mb.mbxCommand, psli->sli_flag, flag);
9400	goto out_not_finished;
9401	}
9402
9403	if (mbx->mbxCommand != MBX_KILL_BOARD && flag & MBX_NOWAIT) {
9404	if (lpfc_readl(addr: phba->HCregaddr, data: &hc_copy) ||
9405	!(hc_copy & HC_MBINT_ENA)) {
9406	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
9407	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9408	"(%d):2528 Mailbox command x%x cannot "
9409	"issue Data: x%x x%x\n",
9410	pmbox->vport ? pmbox->vport->vpi : 0,
9411	pmbox->u.mb.mbxCommand, psli->sli_flag, flag);
9412	goto out_not_finished;
9413	}
9414	}
9415
9416	if (psli->sli_flag & LPFC_SLI_MBOX_ACTIVE) {
9417	/* Polling for a mbox command when another one is already active
9418	* is not allowed in SLI. Also, the driver must have established
9419	* SLI2 mode to queue and process multiple mbox commands.
9420	*/
9421
9422	if (flag & MBX_POLL) {
9423	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
9424
9425	/* Mbox command <mbxCommand> cannot issue */
9426	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9427	"(%d):2529 Mailbox command x%x "
9428	"cannot issue Data: x%x x%x\n",
9429	pmbox->vport ? pmbox->vport->vpi : 0,
9430	pmbox->u.mb.mbxCommand,
9431	psli->sli_flag, flag);
9432	goto out_not_finished;
9433	}
9434
9435	if (!(psli->sli_flag & LPFC_SLI_ACTIVE)) {
9436	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
9437	/* Mbox command <mbxCommand> cannot issue */
9438	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9439	"(%d):2530 Mailbox command x%x "
9440	"cannot issue Data: x%x x%x\n",
9441	pmbox->vport ? pmbox->vport->vpi : 0,
9442	pmbox->u.mb.mbxCommand,
9443	psli->sli_flag, flag);
9444	goto out_not_finished;
9445	}
9446
9447	/* Another mailbox command is still being processed, queue this
9448	* command to be processed later.
9449	*/
9450	lpfc_mbox_put(phba, pmbox);
9451
9452	/* Mbox cmd issue - BUSY */
9453	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
9454	"(%d):0308 Mbox cmd issue - BUSY Data: "
9455	"x%x x%x x%x x%x\n",
9456	pmbox->vport ? pmbox->vport->vpi : 0xffffff,
9457	mbx->mbxCommand,
9458	phba->pport ? phba->pport->port_state : 0xff,
9459	psli->sli_flag, flag);
9460
9461	psli->slistat.mbox_busy++;
9462	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
9463
9464	if (pmbox->vport) {
9465	lpfc_debugfs_disc_trc(pmbox->vport,
9466	LPFC_DISC_TRC_MBOX_VPORT,
9467	"MBOX Bsy vport: cmd:x%x mb:x%x x%x",
9468	(uint32_t)mbx->mbxCommand,
9469	mbx->un.varWords[0], mbx->un.varWords[1]);
9470	}
9471	else {
9472	lpfc_debugfs_disc_trc(phba->pport,
9473	LPFC_DISC_TRC_MBOX,
9474	"MBOX Bsy: cmd:x%x mb:x%x x%x",
9475	(uint32_t)mbx->mbxCommand,
9476	mbx->un.varWords[0], mbx->un.varWords[1]);
9477	}
9478
9479	return MBX_BUSY;
9480	}
9481
9482	psli->sli_flag |= LPFC_SLI_MBOX_ACTIVE;
9483
9484	/* If we are not polling, we MUST be in SLI2 mode */
9485	if (flag != MBX_POLL) {
9486	if (!(psli->sli_flag & LPFC_SLI_ACTIVE) &&
9487	(mbx->mbxCommand != MBX_KILL_BOARD)) {
9488	psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
9489	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
9490	/* Mbox command <mbxCommand> cannot issue */
9491	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9492	"(%d):2531 Mailbox command x%x "
9493	"cannot issue Data: x%x x%x\n",
9494	pmbox->vport ? pmbox->vport->vpi : 0,
9495	pmbox->u.mb.mbxCommand,
9496	psli->sli_flag, flag);
9497	goto out_not_finished;
9498	}
9499	/* timeout active mbox command */
9500	timeout = msecs_to_jiffies(m: lpfc_mbox_tmo_val(phba, pmbox) *
9501	1000);
9502	mod_timer(timer: &psli->mbox_tmo, expires: jiffies + timeout);
9503	}
9504
9505	/* Mailbox cmd <cmd> issue */
9506	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
9507	"(%d):0309 Mailbox cmd x%x issue Data: x%x x%x "
9508	"x%x\n",
9509	pmbox->vport ? pmbox->vport->vpi : 0,
9510	mbx->mbxCommand,
9511	phba->pport ? phba->pport->port_state : 0xff,
9512	psli->sli_flag, flag);
9513
9514	if (mbx->mbxCommand != MBX_HEARTBEAT) {
9515	if (pmbox->vport) {
9516	lpfc_debugfs_disc_trc(pmbox->vport,
9517	LPFC_DISC_TRC_MBOX_VPORT,
9518	"MBOX Send vport: cmd:x%x mb:x%x x%x",
9519	(uint32_t)mbx->mbxCommand,
9520	mbx->un.varWords[0], mbx->un.varWords[1]);
9521	}
9522	else {
9523	lpfc_debugfs_disc_trc(phba->pport,
9524	LPFC_DISC_TRC_MBOX,
9525	"MBOX Send: cmd:x%x mb:x%x x%x",
9526	(uint32_t)mbx->mbxCommand,
9527	mbx->un.varWords[0], mbx->un.varWords[1]);
9528	}
9529	}
9530
9531	psli->slistat.mbox_cmd++;
9532	evtctr = psli->slistat.mbox_event;
9533
9534	/* next set own bit for the adapter and copy over command word */
9535	mbx->mbxOwner = OWN_CHIP;
9536
9537	if (psli->sli_flag & LPFC_SLI_ACTIVE) {
9538	/* Populate mbox extension offset word. */
9539	if (pmbox->in_ext_byte_len || pmbox->out_ext_byte_len) {
9540	*(((uint32_t *)mbx) + pmbox->mbox_offset_word)
9541	= (uint8_t *)phba->mbox_ext
9542	- (uint8_t *)phba->mbox;
9543	}
9544
9545	/* Copy the mailbox extension data */
9546	if (pmbox->in_ext_byte_len && pmbox->ctx_buf) {
9547	lpfc_sli_pcimem_bcopy(pmbox->ctx_buf,
9548	(uint8_t *)phba->mbox_ext,
9549	pmbox->in_ext_byte_len);
9550	}
9551	/* Copy command data to host SLIM area */
9552	lpfc_sli_pcimem_bcopy(mbx, phba->mbox, MAILBOX_CMD_SIZE);
9553	} else {
9554	/* Populate mbox extension offset word. */
9555	if (pmbox->in_ext_byte_len || pmbox->out_ext_byte_len)
9556	*(((uint32_t *)mbx) + pmbox->mbox_offset_word)
9557	= MAILBOX_HBA_EXT_OFFSET;
9558
9559	/* Copy the mailbox extension data */
9560	if (pmbox->in_ext_byte_len && pmbox->ctx_buf)
9561	lpfc_memcpy_to_slim(dest: phba->MBslimaddr +
9562	MAILBOX_HBA_EXT_OFFSET,
9563	src: pmbox->ctx_buf, bytes: pmbox->in_ext_byte_len);
9564
9565	if (mbx->mbxCommand == MBX_CONFIG_PORT)
9566	/* copy command data into host mbox for cmpl */
9567	lpfc_sli_pcimem_bcopy(mbx, phba->mbox,
9568	MAILBOX_CMD_SIZE);
9569
9570	/* First copy mbox command data to HBA SLIM, skip past first
9571	word */
9572	to_slim = phba->MBslimaddr + sizeof (uint32_t);
9573	lpfc_memcpy_to_slim(dest: to_slim, src: &mbx->un.varWords[0],
9574	MAILBOX_CMD_SIZE - sizeof (uint32_t));
9575
9576	/* Next copy over first word, with mbxOwner set */
9577	ldata = *((uint32_t *)mbx);
9578	to_slim = phba->MBslimaddr;
9579	writel(val: ldata, addr: to_slim);
9580	readl(addr: to_slim); /* flush */
9581
9582	if (mbx->mbxCommand == MBX_CONFIG_PORT)
9583	/* switch over to host mailbox */
9584	psli->sli_flag |= LPFC_SLI_ACTIVE;
9585	}
9586
9587	wmb();
9588
9589	switch (flag) {
9590	case MBX_NOWAIT:
9591	/* Set up reference to mailbox command */
9592	psli->mbox_active = pmbox;
9593	/* Interrupt board to do it */
9594	writel(CA_MBATT, addr: phba->CAregaddr);
9595	readl(addr: phba->CAregaddr); /* flush */
9596	/* Don't wait for it to finish, just return */
9597	break;
9598
9599	case MBX_POLL:
9600	/* Set up null reference to mailbox command */
9601	psli->mbox_active = NULL;
9602	/* Interrupt board to do it */
9603	writel(CA_MBATT, addr: phba->CAregaddr);
9604	readl(addr: phba->CAregaddr); /* flush */
9605
9606	if (psli->sli_flag & LPFC_SLI_ACTIVE) {
9607	/* First read mbox status word */
9608	word0 = *((uint32_t *)phba->mbox);
9609	word0 = le32_to_cpu(word0);
9610	} else {
9611	/* First read mbox status word */
9612	if (lpfc_readl(addr: phba->MBslimaddr, data: &word0)) {
9613	spin_unlock_irqrestore(lock: &phba->hbalock,
9614	flags: drvr_flag);
9615	goto out_not_finished;
9616	}
9617	}
9618
9619	/* Read the HBA Host Attention Register */
9620	if (lpfc_readl(addr: phba->HAregaddr, data: &ha_copy)) {
9621	spin_unlock_irqrestore(lock: &phba->hbalock,
9622	flags: drvr_flag);
9623	goto out_not_finished;
9624	}
9625	timeout = msecs_to_jiffies(m: lpfc_mbox_tmo_val(phba, pmbox) *
9626	1000) + jiffies;
9627	i = 0;
9628	/* Wait for command to complete */
9629	while (((word0 & OWN_CHIP) == OWN_CHIP) ||
9630	(!(ha_copy & HA_MBATT) &&
9631	(phba->link_state > LPFC_WARM_START))) {
9632	if (time_after(jiffies, timeout)) {
9633	psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
9634	spin_unlock_irqrestore(lock: &phba->hbalock,
9635	flags: drvr_flag);
9636	goto out_not_finished;
9637	}
9638
9639	/* Check if we took a mbox interrupt while we were
9640	polling */
9641	if (((word0 & OWN_CHIP) != OWN_CHIP)
9642	&& (evtctr != psli->slistat.mbox_event))
9643	break;
9644
9645	if (i++ > 10) {
9646	spin_unlock_irqrestore(lock: &phba->hbalock,
9647	flags: drvr_flag);
9648	msleep(msecs: 1);
9649	spin_lock_irqsave(&phba->hbalock, drvr_flag);
9650	}
9651
9652	if (psli->sli_flag & LPFC_SLI_ACTIVE) {
9653	/* First copy command data */
9654	word0 = *((uint32_t *)phba->mbox);
9655	word0 = le32_to_cpu(word0);
9656	if (mbx->mbxCommand == MBX_CONFIG_PORT) {
9657	MAILBOX_t *slimmb;
9658	uint32_t slimword0;
9659	/* Check real SLIM for any errors */
9660	slimword0 = readl(addr: phba->MBslimaddr);
9661	slimmb = (MAILBOX_t *) & slimword0;
9662	if (((slimword0 & OWN_CHIP) != OWN_CHIP)
9663	&& slimmb->mbxStatus) {
9664	psli->sli_flag &=
9665	~LPFC_SLI_ACTIVE;
9666	word0 = slimword0;
9667	}
9668	}
9669	} else {
9670	/* First copy command data */
9671	word0 = readl(addr: phba->MBslimaddr);
9672	}
9673	/* Read the HBA Host Attention Register */
9674	if (lpfc_readl(addr: phba->HAregaddr, data: &ha_copy)) {
9675	spin_unlock_irqrestore(lock: &phba->hbalock,
9676	flags: drvr_flag);
9677	goto out_not_finished;
9678	}
9679	}
9680
9681	if (psli->sli_flag & LPFC_SLI_ACTIVE) {
9682	/* copy results back to user */
9683	lpfc_sli_pcimem_bcopy(phba->mbox, mbx,
9684	MAILBOX_CMD_SIZE);
9685	/* Copy the mailbox extension data */
9686	if (pmbox->out_ext_byte_len && pmbox->ctx_buf) {
9687	lpfc_sli_pcimem_bcopy(phba->mbox_ext,
9688	pmbox->ctx_buf,
9689	pmbox->out_ext_byte_len);
9690	}
9691	} else {
9692	/* First copy command data */
9693	lpfc_memcpy_from_slim(dest: mbx, src: phba->MBslimaddr,
9694	MAILBOX_CMD_SIZE);
9695	/* Copy the mailbox extension data */
9696	if (pmbox->out_ext_byte_len && pmbox->ctx_buf) {
9697	lpfc_memcpy_from_slim(
9698	dest: pmbox->ctx_buf,
9699	src: phba->MBslimaddr +
9700	MAILBOX_HBA_EXT_OFFSET,
9701	bytes: pmbox->out_ext_byte_len);
9702	}
9703	}
9704
9705	writel(HA_MBATT, addr: phba->HAregaddr);
9706	readl(addr: phba->HAregaddr); /* flush */
9707
9708	psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
9709	status = mbx->mbxStatus;
9710	}
9711
9712	spin_unlock_irqrestore(lock: &phba->hbalock, flags: drvr_flag);
9713	return status;
9714
9715	out_not_finished:
9716	if (processing_queue) {
9717	pmbox->u.mb.mbxStatus = MBX_NOT_FINISHED;
9718	lpfc_mbox_cmpl_put(phba, pmbox);
9719	}
9720	return MBX_NOT_FINISHED;
9721	}
9722
9723	/**
9724	* lpfc_sli4_async_mbox_block - Block posting SLI4 asynchronous mailbox command
9725	* @phba: Pointer to HBA context object.
9726	*
9727	* The function blocks the posting of SLI4 asynchronous mailbox commands from
9728	* the driver internal pending mailbox queue. It will then try to wait out the
9729	* possible outstanding mailbox command before return.
9730	*
9731	* Returns:
9732	* 0 - the outstanding mailbox command completed; otherwise, the wait for
9733	* the outstanding mailbox command timed out.
9734	**/
9735	static int
9736	lpfc_sli4_async_mbox_block(struct lpfc_hba *phba)
9737	{
9738	struct lpfc_sli *psli = &phba->sli;
9739	LPFC_MBOXQ_t *mboxq;
9740	int rc = 0;
9741	unsigned long timeout = 0;
9742	u32 sli_flag;
9743	u8 cmd, subsys, opcode;
9744
9745	/* Mark the asynchronous mailbox command posting as blocked */
9746	spin_lock_irq(lock: &phba->hbalock);
9747	psli->sli_flag |= LPFC_SLI_ASYNC_MBX_BLK;
9748	/* Determine how long we might wait for the active mailbox
9749	* command to be gracefully completed by firmware.
9750	*/
9751	if (phba->sli.mbox_active)
9752	timeout = msecs_to_jiffies(m: lpfc_mbox_tmo_val(phba,
9753	phba->sli.mbox_active) *
9754	1000) + jiffies;
9755	spin_unlock_irq(lock: &phba->hbalock);
9756
9757	/* Make sure the mailbox is really active */
9758	if (timeout)
9759	lpfc_sli4_process_missed_mbox_completions(phba);
9760
9761	/* Wait for the outstanding mailbox command to complete */
9762	while (phba->sli.mbox_active) {
9763	/* Check active mailbox complete status every 2ms */
9764	msleep(msecs: 2);
9765	if (time_after(jiffies, timeout)) {
9766	/* Timeout, mark the outstanding cmd not complete */
9767
9768	/* Sanity check sli.mbox_active has not completed or
9769	* cancelled from another context during last 2ms sleep,
9770	* so take hbalock to be sure before logging.
9771	*/
9772	spin_lock_irq(lock: &phba->hbalock);
9773	if (phba->sli.mbox_active) {
9774	mboxq = phba->sli.mbox_active;
9775	cmd = mboxq->u.mb.mbxCommand;
9776	subsys = lpfc_sli_config_mbox_subsys_get(phba,
9777	mboxq);
9778	opcode = lpfc_sli_config_mbox_opcode_get(phba,
9779	mboxq);
9780	sli_flag = psli->sli_flag;
9781	spin_unlock_irq(lock: &phba->hbalock);
9782	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9783	"2352 Mailbox command x%x "
9784	"(x%x/x%x) sli_flag x%x could "
9785	"not complete\n",
9786	cmd, subsys, opcode,
9787	sli_flag);
9788	} else {
9789	spin_unlock_irq(lock: &phba->hbalock);
9790	}
9791
9792	rc = 1;
9793	break;
9794	}
9795	}
9796
9797	/* Can not cleanly block async mailbox command, fails it */
9798	if (rc) {
9799	spin_lock_irq(lock: &phba->hbalock);
9800	psli->sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
9801	spin_unlock_irq(lock: &phba->hbalock);
9802	}
9803	return rc;
9804	}
9805
9806	/**
9807	* lpfc_sli4_async_mbox_unblock - Block posting SLI4 async mailbox command
9808	* @phba: Pointer to HBA context object.
9809	*
9810	* The function unblocks and resume posting of SLI4 asynchronous mailbox
9811	* commands from the driver internal pending mailbox queue. It makes sure
9812	* that there is no outstanding mailbox command before resuming posting
9813	* asynchronous mailbox commands. If, for any reason, there is outstanding
9814	* mailbox command, it will try to wait it out before resuming asynchronous
9815	* mailbox command posting.
9816	**/
9817	static void
9818	lpfc_sli4_async_mbox_unblock(struct lpfc_hba *phba)
9819	{
9820	struct lpfc_sli *psli = &phba->sli;
9821
9822	spin_lock_irq(lock: &phba->hbalock);
9823	if (!(psli->sli_flag & LPFC_SLI_ASYNC_MBX_BLK)) {
9824	/* Asynchronous mailbox posting is not blocked, do nothing */
9825	spin_unlock_irq(lock: &phba->hbalock);
9826	return;
9827	}
9828
9829	/* Outstanding synchronous mailbox command is guaranteed to be done,
9830	* successful or timeout, after timing-out the outstanding mailbox
9831	* command shall always be removed, so just unblock posting async
9832	* mailbox command and resume
9833	*/
9834	psli->sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
9835	spin_unlock_irq(lock: &phba->hbalock);
9836
9837	/* wake up worker thread to post asynchronous mailbox command */
9838	lpfc_worker_wake_up(phba);
9839	}
9840
9841	/**
9842	* lpfc_sli4_wait_bmbx_ready - Wait for bootstrap mailbox register ready
9843	* @phba: Pointer to HBA context object.
9844	* @mboxq: Pointer to mailbox object.
9845	*
9846	* The function waits for the bootstrap mailbox register ready bit from
9847	* port for twice the regular mailbox command timeout value.
9848	*
9849	* 0 - no timeout on waiting for bootstrap mailbox register ready.
9850	* MBXERR_ERROR - wait for bootstrap mailbox register timed out or port
9851	* is in an unrecoverable state.
9852	**/
9853	static int
9854	lpfc_sli4_wait_bmbx_ready(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
9855	{
9856	uint32_t db_ready;
9857	unsigned long timeout;
9858	struct lpfc_register bmbx_reg;
9859	struct lpfc_register portstat_reg = {-1};
9860
9861	/* Sanity check - there is no point to wait if the port is in an
9862	* unrecoverable state.
9863	*/
9864	if (bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) >=
9865	LPFC_SLI_INTF_IF_TYPE_2) {
9866	if (lpfc_readl(addr: phba->sli4_hba.u.if_type2.STATUSregaddr,
9867	data: &portstat_reg.word0) ||
9868	lpfc_sli4_unrecoverable_port(portstat_reg: &portstat_reg)) {
9869	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
9870	"3858 Skipping bmbx ready because "
9871	"Port Status x%x\n",
9872	portstat_reg.word0);
9873	return MBXERR_ERROR;
9874	}
9875	}
9876
9877	timeout = msecs_to_jiffies(m: lpfc_mbox_tmo_val(phba, mboxq)
9878	* 1000) + jiffies;
9879
9880	do {
9881	bmbx_reg.word0 = readl(addr: phba->sli4_hba.BMBXregaddr);
9882	db_ready = bf_get(lpfc_bmbx_rdy, &bmbx_reg);
9883	if (!db_ready)
9884	mdelay(2);
9885
9886	if (time_after(jiffies, timeout))
9887	return MBXERR_ERROR;
9888	} while (!db_ready);
9889
9890	return 0;
9891	}
9892
9893	/**
9894	* lpfc_sli4_post_sync_mbox - Post an SLI4 mailbox to the bootstrap mailbox
9895	* @phba: Pointer to HBA context object.
9896	* @mboxq: Pointer to mailbox object.
9897	*
9898	* The function posts a mailbox to the port. The mailbox is expected
9899	* to be comletely filled in and ready for the port to operate on it.
9900	* This routine executes a synchronous completion operation on the
9901	* mailbox by polling for its completion.
9902	*
9903	* The caller must not be holding any locks when calling this routine.
9904	*
9905	* Returns:
9906	* MBX_SUCCESS - mailbox posted successfully
9907	* Any of the MBX error values.
9908	**/
9909	static int
9910	lpfc_sli4_post_sync_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
9911	{
9912	int rc = MBX_SUCCESS;
9913	unsigned long iflag;
9914	uint32_t mcqe_status;
9915	uint32_t mbx_cmnd;
9916	struct lpfc_sli *psli = &phba->sli;
9917	struct lpfc_mqe *mb = &mboxq->u.mqe;
9918	struct lpfc_bmbx_create *mbox_rgn;
9919	struct dma_address *dma_address;
9920
9921	/*
9922	* Only one mailbox can be active to the bootstrap mailbox region
9923	* at a time and there is no queueing provided.
9924	*/
9925	spin_lock_irqsave(&phba->hbalock, iflag);
9926	if (psli->sli_flag & LPFC_SLI_MBOX_ACTIVE) {
9927	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
9928	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9929	"(%d):2532 Mailbox command x%x (x%x/x%x) "
9930	"cannot issue Data: x%x x%x\n",
9931	mboxq->vport ? mboxq->vport->vpi : 0,
9932	mboxq->u.mb.mbxCommand,
9933	lpfc_sli_config_mbox_subsys_get(phba, mboxq),
9934	lpfc_sli_config_mbox_opcode_get(phba, mboxq),
9935	psli->sli_flag, MBX_POLL);
9936	return MBXERR_ERROR;
9937	}
9938	/* The server grabs the token and owns it until release */
9939	psli->sli_flag |= LPFC_SLI_MBOX_ACTIVE;
9940	phba->sli.mbox_active = mboxq;
9941	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
9942
9943	/* wait for bootstrap mbox register for readyness */
9944	rc = lpfc_sli4_wait_bmbx_ready(phba, mboxq);
9945	if (rc)
9946	goto exit;
9947	/*
9948	* Initialize the bootstrap memory region to avoid stale data areas
9949	* in the mailbox post. Then copy the caller's mailbox contents to
9950	* the bmbx mailbox region.
9951	*/
9952	mbx_cmnd = bf_get(lpfc_mqe_command, mb);
9953	memset(phba->sli4_hba.bmbx.avirt, 0, sizeof(struct lpfc_bmbx_create));
9954	lpfc_sli4_pcimem_bcopy(srcp: mb, destp: phba->sli4_hba.bmbx.avirt,
9955	cnt: sizeof(struct lpfc_mqe));
9956
9957	/* Post the high mailbox dma address to the port and wait for ready. */
9958	dma_address = &phba->sli4_hba.bmbx.dma_address;
9959	writel(val: dma_address->addr_hi, addr: phba->sli4_hba.BMBXregaddr);
9960
9961	/* wait for bootstrap mbox register for hi-address write done */
9962	rc = lpfc_sli4_wait_bmbx_ready(phba, mboxq);
9963	if (rc)
9964	goto exit;
9965
9966	/* Post the low mailbox dma address to the port. */
9967	writel(val: dma_address->addr_lo, addr: phba->sli4_hba.BMBXregaddr);
9968
9969	/* wait for bootstrap mbox register for low address write done */
9970	rc = lpfc_sli4_wait_bmbx_ready(phba, mboxq);
9971	if (rc)
9972	goto exit;
9973
9974	/*
9975	* Read the CQ to ensure the mailbox has completed.
9976	* If so, update the mailbox status so that the upper layers
9977	* can complete the request normally.
9978	*/
9979	lpfc_sli4_pcimem_bcopy(srcp: phba->sli4_hba.bmbx.avirt, destp: mb,
9980	cnt: sizeof(struct lpfc_mqe));
9981	mbox_rgn = (struct lpfc_bmbx_create *) phba->sli4_hba.bmbx.avirt;
9982	lpfc_sli4_pcimem_bcopy(srcp: &mbox_rgn->mcqe, destp: &mboxq->mcqe,
9983	cnt: sizeof(struct lpfc_mcqe));
9984	mcqe_status = bf_get(lpfc_mcqe_status, &mbox_rgn->mcqe);
9985	/*
9986	* When the CQE status indicates a failure and the mailbox status
9987	* indicates success then copy the CQE status into the mailbox status
9988	* (and prefix it with x4000).
9989	*/
9990	if (mcqe_status != MB_CQE_STATUS_SUCCESS) {
9991	if (bf_get(lpfc_mqe_status, mb) == MBX_SUCCESS)
9992	bf_set(lpfc_mqe_status, mb,
9993	(LPFC_MBX_ERROR_RANGE | mcqe_status));
9994	rc = MBXERR_ERROR;
9995	} else
9996	lpfc_sli4_swap_str(phba, mboxq);
9997
9998	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
9999	"(%d):0356 Mailbox cmd x%x (x%x/x%x) Status x%x "
10000	"Data: x%x x%x x%x x%x x%x x%x x%x x%x x%x x%x x%x"
10001	" x%x x%x CQ: x%x x%x x%x x%x\n",
10002	mboxq->vport ? mboxq->vport->vpi : 0, mbx_cmnd,
10003	lpfc_sli_config_mbox_subsys_get(phba, mboxq),
10004	lpfc_sli_config_mbox_opcode_get(phba, mboxq),
10005	bf_get(lpfc_mqe_status, mb),
10006	mb->un.mb_words[0], mb->un.mb_words[1],
10007	mb->un.mb_words[2], mb->un.mb_words[3],
10008	mb->un.mb_words[4], mb->un.mb_words[5],
10009	mb->un.mb_words[6], mb->un.mb_words[7],
10010	mb->un.mb_words[8], mb->un.mb_words[9],
10011	mb->un.mb_words[10], mb->un.mb_words[11],
10012	mb->un.mb_words[12], mboxq->mcqe.word0,
10013	mboxq->mcqe.mcqe_tag0, mboxq->mcqe.mcqe_tag1,
10014	mboxq->mcqe.trailer);
10015	exit:
10016	/* We are holding the token, no needed for lock when release */
10017	spin_lock_irqsave(&phba->hbalock, iflag);
10018	psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
10019	phba->sli.mbox_active = NULL;
10020	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
10021	return rc;
10022	}
10023
10024	/**
10025	* lpfc_sli_issue_mbox_s4 - Issue an SLI4 mailbox command to firmware
10026	* @phba: Pointer to HBA context object.
10027	* @mboxq: Pointer to mailbox object.
10028	* @flag: Flag indicating how the mailbox need to be processed.
10029	*
10030	* This function is called by discovery code and HBA management code to submit
10031	* a mailbox command to firmware with SLI-4 interface spec.
10032	*
10033	* Return codes the caller owns the mailbox command after the return of the
10034	* function.
10035	**/
10036	static int
10037	lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq,
10038	uint32_t flag)
10039	{
10040	struct lpfc_sli *psli = &phba->sli;
10041	unsigned long iflags;
10042	int rc;
10043
10044	/* dump from issue mailbox command if setup */
10045	lpfc_idiag_mbxacc_dump_issue_mbox(phba, &mboxq->u.mb);
10046
10047	rc = lpfc_mbox_dev_check(phba);
10048	if (unlikely(rc)) {
10049	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
10050	"(%d):2544 Mailbox command x%x (x%x/x%x) "
10051	"cannot issue Data: x%x x%x\n",
10052	mboxq->vport ? mboxq->vport->vpi : 0,
10053	mboxq->u.mb.mbxCommand,
10054	lpfc_sli_config_mbox_subsys_get(phba, mboxq),
10055	lpfc_sli_config_mbox_opcode_get(phba, mboxq),
10056	psli->sli_flag, flag);
10057	goto out_not_finished;
10058	}
10059
10060	/* Detect polling mode and jump to a handler */
10061	if (!phba->sli4_hba.intr_enable) {
10062	if (flag == MBX_POLL)
10063	rc = lpfc_sli4_post_sync_mbox(phba, mboxq);
10064	else
10065	rc = -EIO;
10066	if (rc != MBX_SUCCESS)
10067	lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
10068	"(%d):2541 Mailbox command x%x "
10069	"(x%x/x%x) failure: "
10070	"mqe_sta: x%x mcqe_sta: x%x/x%x "
10071	"Data: x%x x%x\n",
10072	mboxq->vport ? mboxq->vport->vpi : 0,
10073	mboxq->u.mb.mbxCommand,
10074	lpfc_sli_config_mbox_subsys_get(phba,
10075	mboxq),
10076	lpfc_sli_config_mbox_opcode_get(phba,
10077	mboxq),
10078	bf_get(lpfc_mqe_status, &mboxq->u.mqe),
10079	bf_get(lpfc_mcqe_status, &mboxq->mcqe),
10080	bf_get(lpfc_mcqe_ext_status,
10081	&mboxq->mcqe),
10082	psli->sli_flag, flag);
10083	return rc;
10084	} else if (flag == MBX_POLL) {
10085	lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
10086	"(%d):2542 Try to issue mailbox command "
10087	"x%x (x%x/x%x) synchronously ahead of async "
10088	"mailbox command queue: x%x x%x\n",
10089	mboxq->vport ? mboxq->vport->vpi : 0,
10090	mboxq->u.mb.mbxCommand,
10091	lpfc_sli_config_mbox_subsys_get(phba, mboxq),
10092	lpfc_sli_config_mbox_opcode_get(phba, mboxq),
10093	psli->sli_flag, flag);
10094	/* Try to block the asynchronous mailbox posting */
10095	rc = lpfc_sli4_async_mbox_block(phba);
10096	if (!rc) {
10097	/* Successfully blocked, now issue sync mbox cmd */
10098	rc = lpfc_sli4_post_sync_mbox(phba, mboxq);
10099	if (rc != MBX_SUCCESS)
10100	lpfc_printf_log(phba, KERN_WARNING,
10101	LOG_MBOX | LOG_SLI,
10102	"(%d):2597 Sync Mailbox command "
10103	"x%x (x%x/x%x) failure: "
10104	"mqe_sta: x%x mcqe_sta: x%x/x%x "
10105	"Data: x%x x%x\n",
10106	mboxq->vport ? mboxq->vport->vpi : 0,
10107	mboxq->u.mb.mbxCommand,
10108	lpfc_sli_config_mbox_subsys_get(phba,
10109	mboxq),
10110	lpfc_sli_config_mbox_opcode_get(phba,
10111	mboxq),
10112	bf_get(lpfc_mqe_status, &mboxq->u.mqe),
10113	bf_get(lpfc_mcqe_status, &mboxq->mcqe),
10114	bf_get(lpfc_mcqe_ext_status,
10115	&mboxq->mcqe),
10116	psli->sli_flag, flag);
10117	/* Unblock the async mailbox posting afterward */
10118	lpfc_sli4_async_mbox_unblock(phba);
10119	}
10120	return rc;
10121	}
10122
10123	/* Now, interrupt mode asynchronous mailbox command */
10124	rc = lpfc_mbox_cmd_check(phba, mboxq);
10125	if (rc) {
10126	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
10127	"(%d):2543 Mailbox command x%x (x%x/x%x) "
10128	"cannot issue Data: x%x x%x\n",
10129	mboxq->vport ? mboxq->vport->vpi : 0,
10130	mboxq->u.mb.mbxCommand,
10131	lpfc_sli_config_mbox_subsys_get(phba, mboxq),
10132	lpfc_sli_config_mbox_opcode_get(phba, mboxq),
10133	psli->sli_flag, flag);
10134	goto out_not_finished;
10135	}
10136
10137	/* Put the mailbox command to the driver internal FIFO */
10138	psli->slistat.mbox_busy++;
10139	spin_lock_irqsave(&phba->hbalock, iflags);
10140	lpfc_mbox_put(phba, mboxq);
10141	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
10142	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
10143	"(%d):0354 Mbox cmd issue - Enqueue Data: "
10144	"x%x (x%x/x%x) x%x x%x x%x\n",
10145	mboxq->vport ? mboxq->vport->vpi : 0xffffff,
10146	bf_get(lpfc_mqe_command, &mboxq->u.mqe),
10147	lpfc_sli_config_mbox_subsys_get(phba, mboxq),
10148	lpfc_sli_config_mbox_opcode_get(phba, mboxq),
10149	phba->pport->port_state,
10150	psli->sli_flag, MBX_NOWAIT);
10151	/* Wake up worker thread to transport mailbox command from head */
10152	lpfc_worker_wake_up(phba);
10153
10154	return MBX_BUSY;
10155
10156	out_not_finished:
10157	return MBX_NOT_FINISHED;
10158	}
10159
10160	/**
10161	* lpfc_sli4_post_async_mbox - Post an SLI4 mailbox command to device
10162	* @phba: Pointer to HBA context object.
10163	*
10164	* This function is called by worker thread to send a mailbox command to
10165	* SLI4 HBA firmware.
10166	*
10167	**/
10168	int
10169	lpfc_sli4_post_async_mbox(struct lpfc_hba *phba)
10170	{
10171	struct lpfc_sli *psli = &phba->sli;
10172	LPFC_MBOXQ_t *mboxq;
10173	int rc = MBX_SUCCESS;
10174	unsigned long iflags;
10175	struct lpfc_mqe *mqe;
10176	uint32_t mbx_cmnd;
10177
10178	/* Check interrupt mode before post async mailbox command */
10179	if (unlikely(!phba->sli4_hba.intr_enable))
10180	return MBX_NOT_FINISHED;
10181
10182	/* Check for mailbox command service token */
10183	spin_lock_irqsave(&phba->hbalock, iflags);
10184	if (unlikely(psli->sli_flag & LPFC_SLI_ASYNC_MBX_BLK)) {
10185	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
10186	return MBX_NOT_FINISHED;
10187	}
10188	if (psli->sli_flag & LPFC_SLI_MBOX_ACTIVE) {
10189	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
10190	return MBX_NOT_FINISHED;
10191	}
10192	if (unlikely(phba->sli.mbox_active)) {
10193	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
10194	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
10195	"0384 There is pending active mailbox cmd\n");
10196	return MBX_NOT_FINISHED;
10197	}
10198	/* Take the mailbox command service token */
10199	psli->sli_flag |= LPFC_SLI_MBOX_ACTIVE;
10200
10201	/* Get the next mailbox command from head of queue */
10202	mboxq = lpfc_mbox_get(phba);
10203
10204	/* If no more mailbox command waiting for post, we're done */
10205	if (!mboxq) {
10206	psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
10207	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
10208	return MBX_SUCCESS;
10209	}
10210	phba->sli.mbox_active = mboxq;
10211	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
10212
10213	/* Check device readiness for posting mailbox command */
10214	rc = lpfc_mbox_dev_check(phba);
10215	if (unlikely(rc))
10216	/* Driver clean routine will clean up pending mailbox */
10217	goto out_not_finished;
10218
10219	/* Prepare the mbox command to be posted */
10220	mqe = &mboxq->u.mqe;
10221	mbx_cmnd = bf_get(lpfc_mqe_command, mqe);
10222
10223	/* Start timer for the mbox_tmo and log some mailbox post messages */
10224	mod_timer(timer: &psli->mbox_tmo, expires: (jiffies +
10225	msecs_to_jiffies(m: 1000 * lpfc_mbox_tmo_val(phba, mboxq))));
10226
10227	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
10228	"(%d):0355 Mailbox cmd x%x (x%x/x%x) issue Data: "
10229	"x%x x%x\n",
10230	mboxq->vport ? mboxq->vport->vpi : 0, mbx_cmnd,
10231	lpfc_sli_config_mbox_subsys_get(phba, mboxq),
10232	lpfc_sli_config_mbox_opcode_get(phba, mboxq),
10233	phba->pport->port_state, psli->sli_flag);
10234
10235	if (mbx_cmnd != MBX_HEARTBEAT) {
10236	if (mboxq->vport) {
10237	lpfc_debugfs_disc_trc(mboxq->vport,
10238	LPFC_DISC_TRC_MBOX_VPORT,
10239	"MBOX Send vport: cmd:x%x mb:x%x x%x",
10240	mbx_cmnd, mqe->un.mb_words[0],
10241	mqe->un.mb_words[1]);
10242	} else {
10243	lpfc_debugfs_disc_trc(phba->pport,
10244	LPFC_DISC_TRC_MBOX,
10245	"MBOX Send: cmd:x%x mb:x%x x%x",
10246	mbx_cmnd, mqe->un.mb_words[0],
10247	mqe->un.mb_words[1]);
10248	}
10249	}
10250	psli->slistat.mbox_cmd++;
10251
10252	/* Post the mailbox command to the port */
10253	rc = lpfc_sli4_mq_put(q: phba->sli4_hba.mbx_wq, mqe);
10254	if (rc != MBX_SUCCESS) {
10255	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
10256	"(%d):2533 Mailbox command x%x (x%x/x%x) "
10257	"cannot issue Data: x%x x%x\n",
10258	mboxq->vport ? mboxq->vport->vpi : 0,
10259	mboxq->u.mb.mbxCommand,
10260	lpfc_sli_config_mbox_subsys_get(phba, mboxq),
10261	lpfc_sli_config_mbox_opcode_get(phba, mboxq),
10262	psli->sli_flag, MBX_NOWAIT);
10263	goto out_not_finished;
10264	}
10265
10266	return rc;
10267
10268	out_not_finished:
10269	spin_lock_irqsave(&phba->hbalock, iflags);
10270	if (phba->sli.mbox_active) {
10271	mboxq->u.mb.mbxStatus = MBX_NOT_FINISHED;
10272	__lpfc_mbox_cmpl_put(phba, mboxq);
10273	/* Release the token */
10274	psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
10275	phba->sli.mbox_active = NULL;
10276	}
10277	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
10278
10279	return MBX_NOT_FINISHED;
10280	}
10281
10282	/**
10283	* lpfc_sli_issue_mbox - Wrapper func for issuing mailbox command
10284	* @phba: Pointer to HBA context object.
10285	* @pmbox: Pointer to mailbox object.
10286	* @flag: Flag indicating how the mailbox need to be processed.
10287	*
10288	* This routine wraps the actual SLI3 or SLI4 mailbox issuing routine from
10289	* the API jump table function pointer from the lpfc_hba struct.
10290	*
10291	* Return codes the caller owns the mailbox command after the return of the
10292	* function.
10293	**/
10294	int
10295	lpfc_sli_issue_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, uint32_t flag)
10296	{
10297	return phba->lpfc_sli_issue_mbox(phba, pmbox, flag);
10298	}
10299
10300	/**
10301	* lpfc_mbox_api_table_setup - Set up mbox api function jump table
10302	* @phba: The hba struct for which this call is being executed.
10303	* @dev_grp: The HBA PCI-Device group number.
10304	*
10305	* This routine sets up the mbox interface API function jump table in @phba
10306	* struct.
10307	* Returns: 0 - success, -ENODEV - failure.
10308	**/
10309	int
10310	lpfc_mbox_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp)
10311	{
10312
10313	switch (dev_grp) {
10314	case LPFC_PCI_DEV_LP:
10315	phba->lpfc_sli_issue_mbox = lpfc_sli_issue_mbox_s3;
10316	phba->lpfc_sli_handle_slow_ring_event =
10317	lpfc_sli_handle_slow_ring_event_s3;
10318	phba->lpfc_sli_hbq_to_firmware = lpfc_sli_hbq_to_firmware_s3;
10319	phba->lpfc_sli_brdrestart = lpfc_sli_brdrestart_s3;
10320	phba->lpfc_sli_brdready = lpfc_sli_brdready_s3;
10321	break;
10322	case LPFC_PCI_DEV_OC:
10323	phba->lpfc_sli_issue_mbox = lpfc_sli_issue_mbox_s4;
10324	phba->lpfc_sli_handle_slow_ring_event =
10325	lpfc_sli_handle_slow_ring_event_s4;
10326	phba->lpfc_sli_hbq_to_firmware = lpfc_sli_hbq_to_firmware_s4;
10327	phba->lpfc_sli_brdrestart = lpfc_sli_brdrestart_s4;
10328	phba->lpfc_sli_brdready = lpfc_sli_brdready_s4;
10329	break;
10330	default:
10331	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
10332	"1420 Invalid HBA PCI-device group: 0x%x\n",
10333	dev_grp);
10334	return -ENODEV;
10335	}
10336	return 0;
10337	}
10338
10339	/**
10340	* __lpfc_sli_ringtx_put - Add an iocb to the txq
10341	* @phba: Pointer to HBA context object.
10342	* @pring: Pointer to driver SLI ring object.
10343	* @piocb: Pointer to address of newly added command iocb.
10344	*
10345	* This function is called with hbalock held for SLI3 ports or
10346	* the ring lock held for SLI4 ports to add a command
10347	* iocb to the txq when SLI layer cannot submit the command iocb
10348	* to the ring.
10349	**/
10350	void
10351	__lpfc_sli_ringtx_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
10352	struct lpfc_iocbq *piocb)
10353	{
10354	if (phba->sli_rev == LPFC_SLI_REV4)
10355	lockdep_assert_held(&pring->ring_lock);
10356	else
10357	lockdep_assert_held(&phba->hbalock);
10358	/* Insert the caller's iocb in the txq tail for later processing. */
10359	list_add_tail(new: &piocb->list, head: &pring->txq);
10360	}
10361
10362	/**
10363	* lpfc_sli_next_iocb - Get the next iocb in the txq
10364	* @phba: Pointer to HBA context object.
10365	* @pring: Pointer to driver SLI ring object.
10366	* @piocb: Pointer to address of newly added command iocb.
10367	*
10368	* This function is called with hbalock held before a new
10369	* iocb is submitted to the firmware. This function checks
10370	* txq to flush the iocbs in txq to Firmware before
10371	* submitting new iocbs to the Firmware.
10372	* If there are iocbs in the txq which need to be submitted
10373	* to firmware, lpfc_sli_next_iocb returns the first element
10374	* of the txq after dequeuing it from txq.
10375	* If there is no iocb in the txq then the function will return
10376	* *piocb and *piocb is set to NULL. Caller needs to check
10377	* *piocb to find if there are more commands in the txq.
10378	**/
10379	static struct lpfc_iocbq *
10380	lpfc_sli_next_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
10381	struct lpfc_iocbq **piocb)
10382	{
10383	struct lpfc_iocbq * nextiocb;
10384
10385	lockdep_assert_held(&phba->hbalock);
10386
10387	nextiocb = lpfc_sli_ringtx_get(phba, pring);
10388	if (!nextiocb) {
10389	nextiocb = *piocb;
10390	*piocb = NULL;
10391	}
10392
10393	return nextiocb;
10394	}
10395
10396	/**
10397	* __lpfc_sli_issue_iocb_s3 - SLI3 device lockless ver of lpfc_sli_issue_iocb
10398	* @phba: Pointer to HBA context object.
10399	* @ring_number: SLI ring number to issue iocb on.
10400	* @piocb: Pointer to command iocb.
10401	* @flag: Flag indicating if this command can be put into txq.
10402	*
10403	* __lpfc_sli_issue_iocb_s3 is used by other functions in the driver to issue
10404	* an iocb command to an HBA with SLI-3 interface spec. If the PCI slot is
10405	* recovering from error state, if HBA is resetting or if LPFC_STOP_IOCB_EVENT
10406	* flag is turned on, the function returns IOCB_ERROR. When the link is down,
10407	* this function allows only iocbs for posting buffers. This function finds
10408	* next available slot in the command ring and posts the command to the
10409	* available slot and writes the port attention register to request HBA start
10410	* processing new iocb. If there is no slot available in the ring and
10411	* flag & SLI_IOCB_RET_IOCB is set, the new iocb is added to the txq, otherwise
10412	* the function returns IOCB_BUSY.
10413	*
10414	* This function is called with hbalock held. The function will return success
10415	* after it successfully submit the iocb to firmware or after adding to the
10416	* txq.
10417	**/
10418	static int
10419	__lpfc_sli_issue_iocb_s3(struct lpfc_hba *phba, uint32_t ring_number,
10420	struct lpfc_iocbq *piocb, uint32_t flag)
10421	{
10422	struct lpfc_iocbq *nextiocb;
10423	IOCB_t *iocb;
10424	struct lpfc_sli_ring *pring = &phba->sli.sli3_ring[ring_number];
10425
10426	lockdep_assert_held(&phba->hbalock);
10427
10428	if (piocb->cmd_cmpl && (!piocb->vport) &&
10429	(piocb->iocb.ulpCommand != CMD_ABORT_XRI_CN) &&
10430	(piocb->iocb.ulpCommand != CMD_CLOSE_XRI_CN)) {
10431	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
10432	"1807 IOCB x%x failed. No vport\n",
10433	piocb->iocb.ulpCommand);
10434	dump_stack();
10435	return IOCB_ERROR;
10436	}
10437
10438
10439	/* If the PCI channel is in offline state, do not post iocbs. */
10440	if (unlikely(pci_channel_offline(phba->pcidev)))
10441	return IOCB_ERROR;
10442
10443	/* If HBA has a deferred error attention, fail the iocb. */
10444	if (unlikely(phba->hba_flag & DEFER_ERATT))
10445	return IOCB_ERROR;
10446
10447	/*
10448	* We should never get an IOCB if we are in a < LINK_DOWN state
10449	*/
10450	if (unlikely(phba->link_state < LPFC_LINK_DOWN))
10451	return IOCB_ERROR;
10452
10453	/*
10454	* Check to see if we are blocking IOCB processing because of a
10455	* outstanding event.
10456	*/
10457	if (unlikely(pring->flag & LPFC_STOP_IOCB_EVENT))
10458	goto iocb_busy;
10459
10460	if (unlikely(phba->link_state == LPFC_LINK_DOWN)) {
10461	/*
10462	* Only CREATE_XRI, CLOSE_XRI, and QUE_RING_BUF
10463	* can be issued if the link is not up.
10464	*/
10465	switch (piocb->iocb.ulpCommand) {
10466	case CMD_QUE_RING_BUF_CN:
10467	case CMD_QUE_RING_BUF64_CN:
10468	/*
10469	* For IOCBs, like QUE_RING_BUF, that have no rsp ring
10470	* completion, cmd_cmpl MUST be 0.
10471	*/
10472	if (piocb->cmd_cmpl)
10473	piocb->cmd_cmpl = NULL;
10474	fallthrough;
10475	case CMD_CREATE_XRI_CR:
10476	case CMD_CLOSE_XRI_CN:
10477	case CMD_CLOSE_XRI_CX:
10478	break;
10479	default:
10480	goto iocb_busy;
10481	}
10482
10483	/*
10484	* For FCP commands, we must be in a state where we can process link
10485	* attention events.
10486	*/
10487	} else if (unlikely(pring->ringno == LPFC_FCP_RING &&
10488	!(phba->sli.sli_flag & LPFC_PROCESS_LA))) {
10489	goto iocb_busy;
10490	}
10491
10492	while ((iocb = lpfc_sli_next_iocb_slot(phba, pring)) &&
10493	(nextiocb = lpfc_sli_next_iocb(phba, pring, piocb: &piocb)))
10494	lpfc_sli_submit_iocb(phba, pring, iocb, nextiocb);
10495
10496	if (iocb)
10497	lpfc_sli_update_ring(phba, pring);
10498	else
10499	lpfc_sli_update_full_ring(phba, pring);
10500
10501	if (!piocb)
10502	return IOCB_SUCCESS;
10503
10504	goto out_busy;
10505
10506	iocb_busy:
10507	pring->stats.iocb_cmd_delay++;
10508
10509	out_busy:
10510
10511	if (!(flag & SLI_IOCB_RET_IOCB)) {
10512	__lpfc_sli_ringtx_put(phba, pring, piocb);
10513	return IOCB_SUCCESS;
10514	}
10515
10516	return IOCB_BUSY;
10517	}
10518
10519	/**
10520	* __lpfc_sli_issue_fcp_io_s3 - SLI3 device for sending fcp io iocb
10521	* @phba: Pointer to HBA context object.
10522	* @ring_number: SLI ring number to issue wqe on.
10523	* @piocb: Pointer to command iocb.
10524	* @flag: Flag indicating if this command can be put into txq.
10525	*
10526	* __lpfc_sli_issue_fcp_io_s3 is wrapper function to invoke lockless func to
10527	* send an iocb command to an HBA with SLI-3 interface spec.
10528	*
10529	* This function takes the hbalock before invoking the lockless version.
10530	* The function will return success after it successfully submit the wqe to
10531	* firmware or after adding to the txq.
10532	**/
10533	static int
10534	__lpfc_sli_issue_fcp_io_s3(struct lpfc_hba *phba, uint32_t ring_number,
10535	struct lpfc_iocbq *piocb, uint32_t flag)
10536	{
10537	unsigned long iflags;
10538	int rc;
10539
10540	spin_lock_irqsave(&phba->hbalock, iflags);
10541	rc = __lpfc_sli_issue_iocb_s3(phba, ring_number, piocb, flag);
10542	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
10543
10544	return rc;
10545	}
10546
10547	/**
10548	* __lpfc_sli_issue_fcp_io_s4 - SLI4 device for sending fcp io wqe
10549	* @phba: Pointer to HBA context object.
10550	* @ring_number: SLI ring number to issue wqe on.
10551	* @piocb: Pointer to command iocb.
10552	* @flag: Flag indicating if this command can be put into txq.
10553	*
10554	* __lpfc_sli_issue_fcp_io_s4 is used by other functions in the driver to issue
10555	* an wqe command to an HBA with SLI-4 interface spec.
10556	*
10557	* This function is a lockless version. The function will return success
10558	* after it successfully submit the wqe to firmware or after adding to the
10559	* txq.
10560	**/
10561	static int
10562	__lpfc_sli_issue_fcp_io_s4(struct lpfc_hba *phba, uint32_t ring_number,
10563	struct lpfc_iocbq *piocb, uint32_t flag)
10564	{
10565	struct lpfc_io_buf *lpfc_cmd = piocb->io_buf;
10566
10567	lpfc_prep_embed_io(phba, lpfc_ncmd: lpfc_cmd);
10568	return lpfc_sli4_issue_wqe(phba, qp: lpfc_cmd->hdwq, pwqe: piocb);
10569	}
10570
10571	void
10572	lpfc_prep_embed_io(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
10573	{
10574	struct lpfc_iocbq *piocb = &lpfc_cmd->cur_iocbq;
10575	union lpfc_wqe128 *wqe = &lpfc_cmd->cur_iocbq.wqe;
10576	struct sli4_sge *sgl;
10577
10578	/* 128 byte wqe support here */
10579	sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
10580
10581	if (phba->fcp_embed_io) {
10582	struct fcp_cmnd *fcp_cmnd;
10583	u32 *ptr;
10584
10585	fcp_cmnd = lpfc_cmd->fcp_cmnd;
10586
10587	/* Word 0-2 - FCP_CMND */
10588	wqe->generic.bde.tus.f.bdeFlags =
10589	BUFF_TYPE_BDE_IMMED;
10590	wqe->generic.bde.tus.f.bdeSize = sgl->sge_len;
10591	wqe->generic.bde.addrHigh = 0;
10592	wqe->generic.bde.addrLow = 88; /* Word 22 */
10593
10594	bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
10595	bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);
10596
10597	/* Word 22-29 FCP CMND Payload */
10598	ptr = &wqe->words[22];
10599	memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd));
10600	} else {
10601	/* Word 0-2 - Inline BDE */
10602	wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
10603	wqe->generic.bde.tus.f.bdeSize = sizeof(struct fcp_cmnd);
10604	wqe->generic.bde.addrHigh = sgl->addr_hi;
10605	wqe->generic.bde.addrLow = sgl->addr_lo;
10606
10607	/* Word 10 */
10608	bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
10609	bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
10610	}
10611
10612	/* add the VMID tags as per switch response */
10613	if (unlikely(piocb->cmd_flag & LPFC_IO_VMID)) {
10614	if (phba->pport->vmid_flag & LPFC_VMID_TYPE_PRIO) {
10615	bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1);
10616	bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com,
10617	(piocb->vmid_tag.cs_ctl_vmid));
10618	} else if (phba->cfg_vmid_app_header) {
10619	bf_set(wqe_appid, &wqe->fcp_iwrite.wqe_com, 1);
10620	bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
10621	wqe->words[31] = piocb->vmid_tag.app_id;
10622	}
10623	}
10624	}
10625
10626	/**
10627	* __lpfc_sli_issue_iocb_s4 - SLI4 device lockless ver of lpfc_sli_issue_iocb
10628	* @phba: Pointer to HBA context object.
10629	* @ring_number: SLI ring number to issue iocb on.
10630	* @piocb: Pointer to command iocb.
10631	* @flag: Flag indicating if this command can be put into txq.
10632	*
10633	* __lpfc_sli_issue_iocb_s4 is used by other functions in the driver to issue
10634	* an iocb command to an HBA with SLI-4 interface spec.
10635	*
10636	* This function is called with ringlock held. The function will return success
10637	* after it successfully submit the iocb to firmware or after adding to the
10638	* txq.
10639	**/
10640	static int
10641	__lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number,
10642	struct lpfc_iocbq *piocb, uint32_t flag)
10643	{
10644	struct lpfc_sglq *sglq;
10645	union lpfc_wqe128 *wqe;
10646	struct lpfc_queue *wq;
10647	struct lpfc_sli_ring *pring;
10648	u32 ulp_command = get_job_cmnd(phba, iocbq: piocb);
10649
10650	/* Get the WQ */
10651	if ((piocb->cmd_flag & LPFC_IO_FCP) ||
10652	(piocb->cmd_flag & LPFC_USE_FCPWQIDX)) {
10653	wq = phba->sli4_hba.hdwq[piocb->hba_wqidx].io_wq;
10654	} else {
10655	wq = phba->sli4_hba.els_wq;
10656	}
10657
10658	/* Get corresponding ring */
10659	pring = wq->pring;
10660
10661	/*
10662	* The WQE can be either 64 or 128 bytes,
10663	*/
10664
10665	lockdep_assert_held(&pring->ring_lock);
10666	wqe = &piocb->wqe;
10667	if (piocb->sli4_xritag == NO_XRI) {
10668	if (ulp_command == CMD_ABORT_XRI_CX)
10669	sglq = NULL;
10670	else {
10671	sglq = __lpfc_sli_get_els_sglq(phba, piocbq: piocb);
10672	if (!sglq) {
10673	if (!(flag & SLI_IOCB_RET_IOCB)) {
10674	__lpfc_sli_ringtx_put(phba,
10675	pring,
10676	piocb);
10677	return IOCB_SUCCESS;
10678	} else {
10679	return IOCB_BUSY;
10680	}
10681	}
10682	}
10683	} else if (piocb->cmd_flag & LPFC_IO_FCP) {
10684	/* These IO's already have an XRI and a mapped sgl. */
10685	sglq = NULL;
10686	}
10687	else {
10688	/*
10689	* This is a continuation of a commandi,(CX) so this
10690	* sglq is on the active list
10691	*/
10692	sglq = __lpfc_get_active_sglq(phba, xritag: piocb->sli4_lxritag);
10693	if (!sglq)
10694	return IOCB_ERROR;
10695	}
10696
10697	if (sglq) {
10698	piocb->sli4_lxritag = sglq->sli4_lxritag;
10699	piocb->sli4_xritag = sglq->sli4_xritag;
10700
10701	/* ABTS sent by initiator to CT exchange, the
10702	* RX_ID field will be filled with the newly
10703	* allocated responder XRI.
10704	*/
10705	if (ulp_command == CMD_XMIT_BLS_RSP64_CX &&
10706	piocb->abort_bls == LPFC_ABTS_UNSOL_INT)
10707	bf_set(xmit_bls_rsp64_rxid, &wqe->xmit_bls_rsp,
10708	piocb->sli4_xritag);
10709
10710	bf_set(wqe_xri_tag, &wqe->generic.wqe_com,
10711	piocb->sli4_xritag);
10712
10713	if (lpfc_wqe_bpl2sgl(phba, pwqeq: piocb, sglq) == NO_XRI)
10714	return IOCB_ERROR;
10715	}
10716
10717	if (lpfc_sli4_wq_put(q: wq, wqe))
10718	return IOCB_ERROR;
10719
10720	lpfc_sli_ringtxcmpl_put(phba, pring, piocb);
10721
10722	return 0;
10723	}
10724
10725	/*
10726	* lpfc_sli_issue_fcp_io - Wrapper func for issuing fcp i/o
10727	*
10728	* This routine wraps the actual fcp i/o function for issusing WQE for sli-4
10729	* or IOCB for sli-3 function.
10730	* pointer from the lpfc_hba struct.
10731	*
10732	* Return codes:
10733	* IOCB_ERROR - Error
10734	* IOCB_SUCCESS - Success
10735	* IOCB_BUSY - Busy
10736	**/
10737	int
10738	lpfc_sli_issue_fcp_io(struct lpfc_hba *phba, uint32_t ring_number,
10739	struct lpfc_iocbq *piocb, uint32_t flag)
10740	{
10741	return phba->__lpfc_sli_issue_fcp_io(phba, ring_number, piocb, flag);
10742	}
10743
10744	/*
10745	* __lpfc_sli_issue_iocb - Wrapper func of lockless version for issuing iocb
10746	*
10747	* This routine wraps the actual lockless version for issusing IOCB function
10748	* pointer from the lpfc_hba struct.
10749	*
10750	* Return codes:
10751	* IOCB_ERROR - Error
10752	* IOCB_SUCCESS - Success
10753	* IOCB_BUSY - Busy
10754	**/
10755	int
10756	__lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number,
10757	struct lpfc_iocbq *piocb, uint32_t flag)
10758	{
10759	return phba->__lpfc_sli_issue_iocb(phba, ring_number, piocb, flag);
10760	}
10761
10762	static void
10763	__lpfc_sli_prep_els_req_rsp_s3(struct lpfc_iocbq *cmdiocbq,
10764	struct lpfc_vport *vport,
10765	struct lpfc_dmabuf *bmp, u16 cmd_size, u32 did,
10766	u32 elscmd, u8 tmo, u8 expect_rsp)
10767	{
10768	struct lpfc_hba *phba = vport->phba;
10769	IOCB_t *cmd;
10770
10771	cmd = &cmdiocbq->iocb;
10772	memset(cmd, 0, sizeof(*cmd));
10773
10774	cmd->un.elsreq64.bdl.addrHigh = putPaddrHigh(bmp->phys);
10775	cmd->un.elsreq64.bdl.addrLow = putPaddrLow(bmp->phys);
10776	cmd->un.elsreq64.bdl.bdeFlags = BUFF_TYPE_BLP_64;
10777
10778	if (expect_rsp) {
10779	cmd->un.elsreq64.bdl.bdeSize = (2 * sizeof(struct ulp_bde64));
10780	cmd->un.elsreq64.remoteID = did; /* DID */
10781	cmd->ulpCommand = CMD_ELS_REQUEST64_CR;
10782	cmd->ulpTimeout = tmo;
10783	} else {
10784	cmd->un.elsreq64.bdl.bdeSize = sizeof(struct ulp_bde64);
10785	cmd->un.genreq64.xmit_els_remoteID = did; /* DID */
10786	cmd->ulpCommand = CMD_XMIT_ELS_RSP64_CX;
10787	cmd->ulpPU = PARM_NPIV_DID;
10788	}
10789	cmd->ulpBdeCount = 1;
10790	cmd->ulpLe = 1;
10791	cmd->ulpClass = CLASS3;
10792
10793	/* If we have NPIV enabled, we want to send ELS traffic by VPI. */
10794	if (phba->sli3_options & LPFC_SLI3_NPIV_ENABLED) {
10795	if (expect_rsp) {
10796	cmd->un.elsreq64.myID = vport->fc_myDID;
10797
10798	/* For ELS_REQUEST64_CR, use the VPI by default */
10799	cmd->ulpContext = phba->vpi_ids[vport->vpi];
10800	}
10801
10802	cmd->ulpCt_h = 0;
10803	/* The CT field must be 0=INVALID_RPI for the ECHO cmd */
10804	if (elscmd == ELS_CMD_ECHO)
10805	cmd->ulpCt_l = 0; /* context = invalid RPI */
10806	else
10807	cmd->ulpCt_l = 1; /* context = VPI */
10808	}
10809	}
10810
10811	static void
10812	__lpfc_sli_prep_els_req_rsp_s4(struct lpfc_iocbq *cmdiocbq,
10813	struct lpfc_vport *vport,
10814	struct lpfc_dmabuf *bmp, u16 cmd_size, u32 did,
10815	u32 elscmd, u8 tmo, u8 expect_rsp)
10816	{
10817	struct lpfc_hba *phba = vport->phba;
10818	union lpfc_wqe128 *wqe;
10819	struct ulp_bde64_le *bde;
10820	u8 els_id;
10821
10822	wqe = &cmdiocbq->wqe;
10823	memset(wqe, 0, sizeof(*wqe));
10824
10825	/* Word 0 - 2 BDE */
10826	bde = (struct ulp_bde64_le *)&wqe->generic.bde;
10827	bde->addr_low = cpu_to_le32(putPaddrLow(bmp->phys));
10828	bde->addr_high = cpu_to_le32(putPaddrHigh(bmp->phys));
10829	bde->type_size = cpu_to_le32(cmd_size);
10830	bde->type_size |= cpu_to_le32(ULP_BDE64_TYPE_BDE_64);
10831
10832	if (expect_rsp) {
10833	bf_set(wqe_cmnd, &wqe->els_req.wqe_com, CMD_ELS_REQUEST64_WQE);
10834
10835	/* Transfer length */
10836	wqe->els_req.payload_len = cmd_size;
10837	wqe->els_req.max_response_payload_len = FCELSSIZE;
10838
10839	/* DID */
10840	bf_set(wqe_els_did, &wqe->els_req.wqe_dest, did);
10841
10842	/* Word 11 - ELS_ID */
10843	switch (elscmd) {
10844	case ELS_CMD_PLOGI:
10845	els_id = LPFC_ELS_ID_PLOGI;
10846	break;
10847	case ELS_CMD_FLOGI:
10848	els_id = LPFC_ELS_ID_FLOGI;
10849	break;
10850	case ELS_CMD_LOGO:
10851	els_id = LPFC_ELS_ID_LOGO;
10852	break;
10853	case ELS_CMD_FDISC:
10854	if (!vport->fc_myDID) {
10855	els_id = LPFC_ELS_ID_FDISC;
10856	break;
10857	}
10858	fallthrough;
10859	default:
10860	els_id = LPFC_ELS_ID_DEFAULT;
10861	break;
10862	}
10863
10864	bf_set(wqe_els_id, &wqe->els_req.wqe_com, els_id);
10865	} else {
10866	/* DID */
10867	bf_set(wqe_els_did, &wqe->xmit_els_rsp.wqe_dest, did);
10868
10869	/* Transfer length */
10870	wqe->xmit_els_rsp.response_payload_len = cmd_size;
10871
10872	bf_set(wqe_cmnd, &wqe->xmit_els_rsp.wqe_com,
10873	CMD_XMIT_ELS_RSP64_WQE);
10874	}
10875
10876	bf_set(wqe_tmo, &wqe->generic.wqe_com, tmo);
10877	bf_set(wqe_reqtag, &wqe->generic.wqe_com, cmdiocbq->iotag);
10878	bf_set(wqe_class, &wqe->generic.wqe_com, CLASS3);
10879
10880	/* If we have NPIV enabled, we want to send ELS traffic by VPI.
10881	* For SLI4, since the driver controls VPIs we also want to include
10882	* all ELS pt2pt protocol traffic as well.
10883	*/
10884	if ((phba->sli3_options & LPFC_SLI3_NPIV_ENABLED) ||
10885	(vport->fc_flag & FC_PT2PT)) {
10886	if (expect_rsp) {
10887	bf_set(els_req64_sid, &wqe->els_req, vport->fc_myDID);
10888
10889	/* For ELS_REQUEST64_WQE, use the VPI by default */
10890	bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com,
10891	phba->vpi_ids[vport->vpi]);
10892	}
10893
10894	/* The CT field must be 0=INVALID_RPI for the ECHO cmd */
10895	if (elscmd == ELS_CMD_ECHO)
10896	bf_set(wqe_ct, &wqe->generic.wqe_com, 0);
10897	else
10898	bf_set(wqe_ct, &wqe->generic.wqe_com, 1);
10899	}
10900	}
10901
10902	void
10903	lpfc_sli_prep_els_req_rsp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocbq,
10904	struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
10905	u16 cmd_size, u32 did, u32 elscmd, u8 tmo,
10906	u8 expect_rsp)
10907	{
10908	phba->__lpfc_sli_prep_els_req_rsp(cmdiocbq, vport, bmp, cmd_size, did,
10909	elscmd, tmo, expect_rsp);
10910	}
10911
10912	static void
10913	__lpfc_sli_prep_gen_req_s3(struct lpfc_iocbq *cmdiocbq, struct lpfc_dmabuf *bmp,
10914	u16 rpi, u32 num_entry, u8 tmo)
10915	{
10916	IOCB_t *cmd;
10917
10918	cmd = &cmdiocbq->iocb;
10919	memset(cmd, 0, sizeof(*cmd));
10920
10921	cmd->un.genreq64.bdl.addrHigh = putPaddrHigh(bmp->phys);
10922	cmd->un.genreq64.bdl.addrLow = putPaddrLow(bmp->phys);
10923	cmd->un.genreq64.bdl.bdeFlags = BUFF_TYPE_BLP_64;
10924	cmd->un.genreq64.bdl.bdeSize = num_entry * sizeof(struct ulp_bde64);
10925
10926	cmd->un.genreq64.w5.hcsw.Rctl = FC_RCTL_DD_UNSOL_CTL;
10927	cmd->un.genreq64.w5.hcsw.Type = FC_TYPE_CT;
10928	cmd->un.genreq64.w5.hcsw.Fctl = (SI | LA);
10929
10930	cmd->ulpContext = rpi;
10931	cmd->ulpClass = CLASS3;
10932	cmd->ulpCommand = CMD_GEN_REQUEST64_CR;
10933	cmd->ulpBdeCount = 1;
10934	cmd->ulpLe = 1;
10935	cmd->ulpOwner = OWN_CHIP;
10936	cmd->ulpTimeout = tmo;
10937	}
10938
10939	static void
10940	__lpfc_sli_prep_gen_req_s4(struct lpfc_iocbq *cmdiocbq, struct lpfc_dmabuf *bmp,
10941	u16 rpi, u32 num_entry, u8 tmo)
10942	{
10943	union lpfc_wqe128 *cmdwqe;
10944	struct ulp_bde64_le *bde, *bpl;
10945	u32 xmit_len = 0, total_len = 0, size, type, i;
10946
10947	cmdwqe = &cmdiocbq->wqe;
10948	memset(cmdwqe, 0, sizeof(*cmdwqe));
10949
10950	/* Calculate total_len and xmit_len */
10951	bpl = (struct ulp_bde64_le *)bmp->virt;
10952	for (i = 0; i < num_entry; i++) {
10953	size = le32_to_cpu(bpl[i].type_size) & ULP_BDE64_SIZE_MASK;
10954	total_len += size;
10955	}
10956	for (i = 0; i < num_entry; i++) {
10957	size = le32_to_cpu(bpl[i].type_size) & ULP_BDE64_SIZE_MASK;
10958	type = le32_to_cpu(bpl[i].type_size) & ULP_BDE64_TYPE_MASK;
10959	if (type != ULP_BDE64_TYPE_BDE_64)
10960	break;
10961	xmit_len += size;
10962	}
10963
10964	/* Words 0 - 2 */
10965	bde = (struct ulp_bde64_le *)&cmdwqe->generic.bde;
10966	bde->addr_low = bpl->addr_low;
10967	bde->addr_high = bpl->addr_high;
10968	bde->type_size = cpu_to_le32(xmit_len);
10969	bde->type_size |= cpu_to_le32(ULP_BDE64_TYPE_BDE_64);
10970
10971	/* Word 3 */
10972	cmdwqe->gen_req.request_payload_len = xmit_len;
10973
10974	/* Word 5 */
10975	bf_set(wqe_type, &cmdwqe->gen_req.wge_ctl, FC_TYPE_CT);
10976	bf_set(wqe_rctl, &cmdwqe->gen_req.wge_ctl, FC_RCTL_DD_UNSOL_CTL);
10977	bf_set(wqe_si, &cmdwqe->gen_req.wge_ctl, 1);
10978	bf_set(wqe_la, &cmdwqe->gen_req.wge_ctl, 1);
10979
10980	/* Word 6 */
10981	bf_set(wqe_ctxt_tag, &cmdwqe->gen_req.wqe_com, rpi);
10982
10983	/* Word 7 */
10984	bf_set(wqe_tmo, &cmdwqe->gen_req.wqe_com, tmo);
10985	bf_set(wqe_class, &cmdwqe->gen_req.wqe_com, CLASS3);
10986	bf_set(wqe_cmnd, &cmdwqe->gen_req.wqe_com, CMD_GEN_REQUEST64_CR);
10987	bf_set(wqe_ct, &cmdwqe->gen_req.wqe_com, SLI4_CT_RPI);
10988
10989	/* Word 12 */
10990	cmdwqe->gen_req.max_response_payload_len = total_len - xmit_len;
10991	}
10992
10993	void
10994	lpfc_sli_prep_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocbq,
10995	struct lpfc_dmabuf *bmp, u16 rpi, u32 num_entry, u8 tmo)
10996	{
10997	phba->__lpfc_sli_prep_gen_req(cmdiocbq, bmp, rpi, num_entry, tmo);
10998	}
10999
11000	static void
11001	__lpfc_sli_prep_xmit_seq64_s3(struct lpfc_iocbq *cmdiocbq,
11002	struct lpfc_dmabuf *bmp, u16 rpi, u16 ox_id,
11003	u32 num_entry, u8 rctl, u8 last_seq, u8 cr_cx_cmd)
11004	{
11005	IOCB_t *icmd;
11006
11007	icmd = &cmdiocbq->iocb;
11008	memset(icmd, 0, sizeof(*icmd));
11009
11010	icmd->un.xseq64.bdl.addrHigh = putPaddrHigh(bmp->phys);
11011	icmd->un.xseq64.bdl.addrLow = putPaddrLow(bmp->phys);
11012	icmd->un.xseq64.bdl.bdeFlags = BUFF_TYPE_BLP_64;
11013	icmd->un.xseq64.bdl.bdeSize = (num_entry * sizeof(struct ulp_bde64));
11014	icmd->un.xseq64.w5.hcsw.Fctl = LA;
11015	if (last_seq)
11016	icmd->un.xseq64.w5.hcsw.Fctl |= LS;
11017	icmd->un.xseq64.w5.hcsw.Dfctl = 0;
11018	icmd->un.xseq64.w5.hcsw.Rctl = rctl;
11019	icmd->un.xseq64.w5.hcsw.Type = FC_TYPE_CT;
11020
11021	icmd->ulpBdeCount = 1;
11022	icmd->ulpLe = 1;
11023	icmd->ulpClass = CLASS3;
11024
11025	switch (cr_cx_cmd) {
11026	case CMD_XMIT_SEQUENCE64_CR:
11027	icmd->ulpContext = rpi;
11028	icmd->ulpCommand = CMD_XMIT_SEQUENCE64_CR;
11029	break;
11030	case CMD_XMIT_SEQUENCE64_CX:
11031	icmd->ulpContext = ox_id;
11032	icmd->ulpCommand = CMD_XMIT_SEQUENCE64_CX;
11033	break;
11034	default:
11035	break;
11036	}
11037	}
11038
11039	static void
11040	__lpfc_sli_prep_xmit_seq64_s4(struct lpfc_iocbq *cmdiocbq,
11041	struct lpfc_dmabuf *bmp, u16 rpi, u16 ox_id,
11042	u32 full_size, u8 rctl, u8 last_seq, u8 cr_cx_cmd)
11043	{
11044	union lpfc_wqe128 *wqe;
11045	struct ulp_bde64 *bpl;
11046
11047	wqe = &cmdiocbq->wqe;
11048	memset(wqe, 0, sizeof(*wqe));
11049
11050	/* Words 0 - 2 */
11051	bpl = (struct ulp_bde64 *)bmp->virt;
11052	wqe->xmit_sequence.bde.addrHigh = bpl->addrHigh;
11053	wqe->xmit_sequence.bde.addrLow = bpl->addrLow;
11054	wqe->xmit_sequence.bde.tus.w = bpl->tus.w;
11055
11056	/* Word 5 */
11057	bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, last_seq);
11058	bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 1);
11059	bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0);
11060	bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, rctl);
11061	bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_CT);
11062
11063	/* Word 6 */
11064	bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com, rpi);
11065
11066	bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com,
11067	CMD_XMIT_SEQUENCE64_WQE);
11068
11069	/* Word 7 */
11070	bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3);
11071
11072	/* Word 9 */
11073	bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ox_id);
11074
11075	/* Word 12 */
11076	if (cmdiocbq->cmd_flag & (LPFC_IO_LIBDFC | LPFC_IO_LOOPBACK))
11077	wqe->xmit_sequence.xmit_len = full_size;
11078	else
11079	wqe->xmit_sequence.xmit_len =
11080	wqe->xmit_sequence.bde.tus.f.bdeSize;
11081	}
11082
11083	void
11084	lpfc_sli_prep_xmit_seq64(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocbq,
11085	struct lpfc_dmabuf *bmp, u16 rpi, u16 ox_id,
11086	u32 num_entry, u8 rctl, u8 last_seq, u8 cr_cx_cmd)
11087	{
11088	phba->__lpfc_sli_prep_xmit_seq64(cmdiocbq, bmp, rpi, ox_id, num_entry,
11089	rctl, last_seq, cr_cx_cmd);
11090	}
11091
11092	static void
11093	__lpfc_sli_prep_abort_xri_s3(struct lpfc_iocbq *cmdiocbq, u16 ulp_context,
11094	u16 iotag, u8 ulp_class, u16 cqid, bool ia,
11095	bool wqec)
11096	{
11097	IOCB_t *icmd = NULL;
11098
11099	icmd = &cmdiocbq->iocb;
11100	memset(icmd, 0, sizeof(*icmd));
11101
11102	/* Word 5 */
11103	icmd->un.acxri.abortContextTag = ulp_context;
11104	icmd->un.acxri.abortIoTag = iotag;
11105
11106	if (ia) {
11107	/* Word 7 */
11108	icmd->ulpCommand = CMD_CLOSE_XRI_CN;
11109	} else {
11110	/* Word 3 */
11111	icmd->un.acxri.abortType = ABORT_TYPE_ABTS;
11112
11113	/* Word 7 */
11114	icmd->ulpClass = ulp_class;
11115	icmd->ulpCommand = CMD_ABORT_XRI_CN;
11116	}
11117
11118	/* Word 7 */
11119	icmd->ulpLe = 1;
11120	}
11121
11122	static void
11123	__lpfc_sli_prep_abort_xri_s4(struct lpfc_iocbq *cmdiocbq, u16 ulp_context,
11124	u16 iotag, u8 ulp_class, u16 cqid, bool ia,
11125	bool wqec)
11126	{
11127	union lpfc_wqe128 *wqe;
11128
11129	wqe = &cmdiocbq->wqe;
11130	memset(wqe, 0, sizeof(*wqe));
11131
11132	/* Word 3 */
11133	bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG);
11134	if (ia)
11135	bf_set(abort_cmd_ia, &wqe->abort_cmd, 1);
11136	else
11137	bf_set(abort_cmd_ia, &wqe->abort_cmd, 0);
11138
11139	/* Word 7 */
11140	bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_WQE);
11141
11142	/* Word 8 */
11143	wqe->abort_cmd.wqe_com.abort_tag = ulp_context;
11144
11145	/* Word 9 */
11146	bf_set(wqe_reqtag, &wqe->abort_cmd.wqe_com, iotag);
11147
11148	/* Word 10 */
11149	bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1);
11150
11151	/* Word 11 */
11152	if (wqec)
11153	bf_set(wqe_wqec, &wqe->abort_cmd.wqe_com, 1);
11154	bf_set(wqe_cqid, &wqe->abort_cmd.wqe_com, cqid);
11155	bf_set(wqe_cmd_type, &wqe->abort_cmd.wqe_com, OTHER_COMMAND);
11156	}
11157
11158	void
11159	lpfc_sli_prep_abort_xri(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocbq,
11160	u16 ulp_context, u16 iotag, u8 ulp_class, u16 cqid,
11161	bool ia, bool wqec)
11162	{
11163	phba->__lpfc_sli_prep_abort_xri(cmdiocbq, ulp_context, iotag, ulp_class,
11164	cqid, ia, wqec);
11165	}
11166
11167	/**
11168	* lpfc_sli_api_table_setup - Set up sli api function jump table
11169	* @phba: The hba struct for which this call is being executed.
11170	* @dev_grp: The HBA PCI-Device group number.
11171	*
11172	* This routine sets up the SLI interface API function jump table in @phba
11173	* struct.
11174	* Returns: 0 - success, -ENODEV - failure.
11175	**/
11176	int
11177	lpfc_sli_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp)
11178	{
11179
11180	switch (dev_grp) {
11181	case LPFC_PCI_DEV_LP:
11182	phba->__lpfc_sli_issue_iocb = __lpfc_sli_issue_iocb_s3;
11183	phba->__lpfc_sli_release_iocbq = __lpfc_sli_release_iocbq_s3;
11184	phba->__lpfc_sli_issue_fcp_io = __lpfc_sli_issue_fcp_io_s3;
11185	phba->__lpfc_sli_prep_els_req_rsp = __lpfc_sli_prep_els_req_rsp_s3;
11186	phba->__lpfc_sli_prep_gen_req = __lpfc_sli_prep_gen_req_s3;
11187	phba->__lpfc_sli_prep_xmit_seq64 = __lpfc_sli_prep_xmit_seq64_s3;
11188	phba->__lpfc_sli_prep_abort_xri = __lpfc_sli_prep_abort_xri_s3;
11189	break;
11190	case LPFC_PCI_DEV_OC:
11191	phba->__lpfc_sli_issue_iocb = __lpfc_sli_issue_iocb_s4;
11192	phba->__lpfc_sli_release_iocbq = __lpfc_sli_release_iocbq_s4;
11193	phba->__lpfc_sli_issue_fcp_io = __lpfc_sli_issue_fcp_io_s4;
11194	phba->__lpfc_sli_prep_els_req_rsp = __lpfc_sli_prep_els_req_rsp_s4;
11195	phba->__lpfc_sli_prep_gen_req = __lpfc_sli_prep_gen_req_s4;
11196	phba->__lpfc_sli_prep_xmit_seq64 = __lpfc_sli_prep_xmit_seq64_s4;
11197	phba->__lpfc_sli_prep_abort_xri = __lpfc_sli_prep_abort_xri_s4;
11198	break;
11199	default:
11200	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
11201	"1419 Invalid HBA PCI-device group: 0x%x\n",
11202	dev_grp);
11203	return -ENODEV;
11204	}
11205	return 0;
11206	}
11207
11208	/**
11209	* lpfc_sli4_calc_ring - Calculates which ring to use
11210	* @phba: Pointer to HBA context object.
11211	* @piocb: Pointer to command iocb.
11212	*
11213	* For SLI4 only, FCP IO can deferred to one fo many WQs, based on
11214	* hba_wqidx, thus we need to calculate the corresponding ring.
11215	* Since ABORTS must go on the same WQ of the command they are
11216	* aborting, we use command's hba_wqidx.
11217	*/
11218	struct lpfc_sli_ring *
11219	lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb)
11220	{
11221	struct lpfc_io_buf *lpfc_cmd;
11222
11223	if (piocb->cmd_flag & (LPFC_IO_FCP | LPFC_USE_FCPWQIDX)) {
11224	if (unlikely(!phba->sli4_hba.hdwq))
11225	return NULL;
11226	/*
11227	* for abort iocb hba_wqidx should already
11228	* be setup based on what work queue we used.
11229	*/
11230	if (!(piocb->cmd_flag & LPFC_USE_FCPWQIDX)) {
11231	lpfc_cmd = piocb->io_buf;
11232	piocb->hba_wqidx = lpfc_cmd->hdwq_no;
11233	}
11234	return phba->sli4_hba.hdwq[piocb->hba_wqidx].io_wq->pring;
11235	} else {
11236	if (unlikely(!phba->sli4_hba.els_wq))
11237	return NULL;
11238	piocb->hba_wqidx = 0;
11239	return phba->sli4_hba.els_wq->pring;
11240	}
11241	}
11242
11243	inline void lpfc_sli4_poll_eq(struct lpfc_queue *eq)
11244	{
11245	struct lpfc_hba *phba = eq->phba;
11246
11247	/*
11248	* Unlocking an irq is one of the entry point to check
11249	* for re-schedule, but we are good for io submission
11250	* path as midlayer does a get_cpu to glue us in. Flush
11251	* out the invalidate queue so we can see the updated
11252	* value for flag.
11253	*/
11254	smp_rmb();
11255
11256	if (READ_ONCE(eq->mode) == LPFC_EQ_POLL)
11257	/* We will not likely get the completion for the caller
11258	* during this iteration but i guess that's fine.
11259	* Future io's coming on this eq should be able to
11260	* pick it up. As for the case of single io's, they
11261	* will be handled through a sched from polling timer
11262	* function which is currently triggered every 1msec.
11263	*/
11264	lpfc_sli4_process_eq(phba, eq, LPFC_QUEUE_NOARM,
11265	poll_mode: LPFC_QUEUE_WORK);
11266	}
11267
11268	/**
11269	* lpfc_sli_issue_iocb - Wrapper function for __lpfc_sli_issue_iocb
11270	* @phba: Pointer to HBA context object.
11271	* @ring_number: Ring number
11272	* @piocb: Pointer to command iocb.
11273	* @flag: Flag indicating if this command can be put into txq.
11274	*
11275	* lpfc_sli_issue_iocb is a wrapper around __lpfc_sli_issue_iocb
11276	* function. This function gets the hbalock and calls
11277	* __lpfc_sli_issue_iocb function and will return the error returned
11278	* by __lpfc_sli_issue_iocb function. This wrapper is used by
11279	* functions which do not hold hbalock.
11280	**/
11281	int
11282	lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number,
11283	struct lpfc_iocbq *piocb, uint32_t flag)
11284	{
11285	struct lpfc_sli_ring *pring;
11286	struct lpfc_queue *eq;
11287	unsigned long iflags;
11288	int rc;
11289
11290	/* If the PCI channel is in offline state, do not post iocbs. */
11291	if (unlikely(pci_channel_offline(phba->pcidev)))
11292	return IOCB_ERROR;
11293
11294	if (phba->sli_rev == LPFC_SLI_REV4) {
11295	lpfc_sli_prep_wqe(phba, job: piocb);
11296
11297	eq = phba->sli4_hba.hdwq[piocb->hba_wqidx].hba_eq;
11298
11299	pring = lpfc_sli4_calc_ring(phba, piocb);
11300	if (unlikely(pring == NULL))
11301	return IOCB_ERROR;
11302
11303	spin_lock_irqsave(&pring->ring_lock, iflags);
11304	rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag);
11305	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
11306
11307	lpfc_sli4_poll_eq(eq);
11308	} else {
11309	/* For now, SLI2/3 will still use hbalock */
11310	spin_lock_irqsave(&phba->hbalock, iflags);
11311	rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag);
11312	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
11313	}
11314	return rc;
11315	}
11316
11317	/**
11318	* lpfc_extra_ring_setup - Extra ring setup function
11319	* @phba: Pointer to HBA context object.
11320	*
11321	* This function is called while driver attaches with the
11322	* HBA to setup the extra ring. The extra ring is used
11323	* only when driver needs to support target mode functionality
11324	* or IP over FC functionalities.
11325	*
11326	* This function is called with no lock held. SLI3 only.
11327	**/
11328	static int
11329	lpfc_extra_ring_setup( struct lpfc_hba *phba)
11330	{
11331	struct lpfc_sli *psli;
11332	struct lpfc_sli_ring *pring;
11333
11334	psli = &phba->sli;
11335
11336	/* Adjust cmd/rsp ring iocb entries more evenly */
11337
11338	/* Take some away from the FCP ring */
11339	pring = &psli->sli3_ring[LPFC_FCP_RING];
11340	pring->sli.sli3.numCiocb -= SLI2_IOCB_CMD_R1XTRA_ENTRIES;
11341	pring->sli.sli3.numRiocb -= SLI2_IOCB_RSP_R1XTRA_ENTRIES;
11342	pring->sli.sli3.numCiocb -= SLI2_IOCB_CMD_R3XTRA_ENTRIES;
11343	pring->sli.sli3.numRiocb -= SLI2_IOCB_RSP_R3XTRA_ENTRIES;
11344
11345	/* and give them to the extra ring */
11346	pring = &psli->sli3_ring[LPFC_EXTRA_RING];
11347
11348	pring->sli.sli3.numCiocb += SLI2_IOCB_CMD_R1XTRA_ENTRIES;
11349	pring->sli.sli3.numRiocb += SLI2_IOCB_RSP_R1XTRA_ENTRIES;
11350	pring->sli.sli3.numCiocb += SLI2_IOCB_CMD_R3XTRA_ENTRIES;
11351	pring->sli.sli3.numRiocb += SLI2_IOCB_RSP_R3XTRA_ENTRIES;
11352
11353	/* Setup default profile for this ring */
11354	pring->iotag_max = 4096;
11355	pring->num_mask = 1;
11356	pring->prt[0].profile = 0; /* Mask 0 */
11357	pring->prt[0].rctl = phba->cfg_multi_ring_rctl;
11358	pring->prt[0].type = phba->cfg_multi_ring_type;
11359	pring->prt[0].lpfc_sli_rcv_unsol_event = NULL;
11360	return 0;
11361	}
11362
11363	static void
11364	lpfc_sli_post_recovery_event(struct lpfc_hba *phba,
11365	struct lpfc_nodelist *ndlp)
11366	{
11367	unsigned long iflags;
11368	struct lpfc_work_evt *evtp = &ndlp->recovery_evt;
11369
11370	spin_lock_irqsave(&phba->hbalock, iflags);
11371	if (!list_empty(head: &evtp->evt_listp)) {
11372	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
11373	return;
11374	}
11375
11376	/* Incrementing the reference count until the queued work is done. */
11377	evtp->evt_arg1 = lpfc_nlp_get(ndlp);
11378	if (!evtp->evt_arg1) {
11379	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
11380	return;
11381	}
11382	evtp->evt = LPFC_EVT_RECOVER_PORT;
11383	list_add_tail(new: &evtp->evt_listp, head: &phba->work_list);
11384	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
11385
11386	lpfc_worker_wake_up(phba);
11387	}
11388
11389	/* lpfc_sli_abts_err_handler - handle a failed ABTS request from an SLI3 port.
11390	* @phba: Pointer to HBA context object.
11391	* @iocbq: Pointer to iocb object.
11392	*
11393	* The async_event handler calls this routine when it receives
11394	* an ASYNC_STATUS_CN event from the port. The port generates
11395	* this event when an Abort Sequence request to an rport fails
11396	* twice in succession. The abort could be originated by the
11397	* driver or by the port. The ABTS could have been for an ELS
11398	* or FCP IO. The port only generates this event when an ABTS
11399	* fails to complete after one retry.
11400	*/
11401	static void
11402	lpfc_sli_abts_err_handler(struct lpfc_hba *phba,
11403	struct lpfc_iocbq *iocbq)
11404	{
11405	struct lpfc_nodelist *ndlp = NULL;
11406	uint16_t rpi = 0, vpi = 0;
11407	struct lpfc_vport *vport = NULL;
11408
11409	/* The rpi in the ulpContext is vport-sensitive. */
11410	vpi = iocbq->iocb.un.asyncstat.sub_ctxt_tag;
11411	rpi = iocbq->iocb.ulpContext;
11412
11413	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
11414	"3092 Port generated ABTS async event "
11415	"on vpi %d rpi %d status 0x%x\n",
11416	vpi, rpi, iocbq->iocb.ulpStatus);
11417
11418	vport = lpfc_find_vport_by_vpid(phba, vpi);
11419	if (!vport)
11420	goto err_exit;
11421	ndlp = lpfc_findnode_rpi(vport, rpi);
11422	if (!ndlp)
11423	goto err_exit;
11424
11425	if (iocbq->iocb.ulpStatus == IOSTAT_LOCAL_REJECT)
11426	lpfc_sli_abts_recover_port(vport, ndlp);
11427	return;
11428
11429	err_exit:
11430	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
11431	"3095 Event Context not found, no "
11432	"action on vpi %d rpi %d status 0x%x, reason 0x%x\n",
11433	vpi, rpi, iocbq->iocb.ulpStatus,
11434	iocbq->iocb.ulpContext);
11435	}
11436
11437	/* lpfc_sli4_abts_err_handler - handle a failed ABTS request from an SLI4 port.
11438	* @phba: pointer to HBA context object.
11439	* @ndlp: nodelist pointer for the impacted rport.
11440	* @axri: pointer to the wcqe containing the failed exchange.
11441	*
11442	* The driver calls this routine when it receives an ABORT_XRI_FCP CQE from the
11443	* port. The port generates this event when an abort exchange request to an
11444	* rport fails twice in succession with no reply. The abort could be originated
11445	* by the driver or by the port. The ABTS could have been for an ELS or FCP IO.
11446	*/
11447	void
11448	lpfc_sli4_abts_err_handler(struct lpfc_hba *phba,
11449	struct lpfc_nodelist *ndlp,
11450	struct sli4_wcqe_xri_aborted *axri)
11451	{
11452	uint32_t ext_status = 0;
11453
11454	if (!ndlp) {
11455	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
11456	"3115 Node Context not found, driver "
11457	"ignoring abts err event\n");
11458	return;
11459	}
11460
11461	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
11462	"3116 Port generated FCP XRI ABORT event on "
11463	"vpi %d rpi %d xri x%x status 0x%x parameter x%x\n",
11464	ndlp->vport->vpi, phba->sli4_hba.rpi_ids[ndlp->nlp_rpi],
11465	bf_get(lpfc_wcqe_xa_xri, axri),
11466	bf_get(lpfc_wcqe_xa_status, axri),
11467	axri->parameter);
11468
11469	/*
11470	* Catch the ABTS protocol failure case. Older OCe FW releases returned
11471	* LOCAL_REJECT and 0 for a failed ABTS exchange and later OCe and
11472	* LPe FW releases returned LOCAL_REJECT and SEQUENCE_TIMEOUT.
11473	*/
11474	ext_status = axri->parameter & IOERR_PARAM_MASK;
11475	if ((bf_get(lpfc_wcqe_xa_status, axri) == IOSTAT_LOCAL_REJECT) &&
11476	((ext_status == IOERR_SEQUENCE_TIMEOUT) || (ext_status == 0)))
11477	lpfc_sli_post_recovery_event(phba, ndlp);
11478	}
11479
11480	/**
11481	* lpfc_sli_async_event_handler - ASYNC iocb handler function
11482	* @phba: Pointer to HBA context object.
11483	* @pring: Pointer to driver SLI ring object.
11484	* @iocbq: Pointer to iocb object.
11485	*
11486	* This function is called by the slow ring event handler
11487	* function when there is an ASYNC event iocb in the ring.
11488	* This function is called with no lock held.
11489	* Currently this function handles only temperature related
11490	* ASYNC events. The function decodes the temperature sensor
11491	* event message and posts events for the management applications.
11492	**/
11493	static void
11494	lpfc_sli_async_event_handler(struct lpfc_hba * phba,
11495	struct lpfc_sli_ring * pring, struct lpfc_iocbq * iocbq)
11496	{
11497	IOCB_t *icmd;
11498	uint16_t evt_code;
11499	struct temp_event temp_event_data;
11500	struct Scsi_Host *shost;
11501	uint32_t *iocb_w;
11502
11503	icmd = &iocbq->iocb;
11504	evt_code = icmd->un.asyncstat.evt_code;
11505
11506	switch (evt_code) {
11507	case ASYNC_TEMP_WARN:
11508	case ASYNC_TEMP_SAFE:
11509	temp_event_data.data = (uint32_t) icmd->ulpContext;
11510	temp_event_data.event_type = FC_REG_TEMPERATURE_EVENT;
11511	if (evt_code == ASYNC_TEMP_WARN) {
11512	temp_event_data.event_code = LPFC_THRESHOLD_TEMP;
11513	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
11514	"0347 Adapter is very hot, please take "
11515	"corrective action. temperature : %d Celsius\n",
11516	(uint32_t) icmd->ulpContext);
11517	} else {
11518	temp_event_data.event_code = LPFC_NORMAL_TEMP;
11519	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
11520	"0340 Adapter temperature is OK now. "
11521	"temperature : %d Celsius\n",
11522	(uint32_t) icmd->ulpContext);
11523	}
11524
11525	/* Send temperature change event to applications */
11526	shost = lpfc_shost_from_vport(vport: phba->pport);
11527	fc_host_post_vendor_event(shost, event_number: fc_get_event_number(),
11528	data_len: sizeof(temp_event_data), data_buf: (char *) &temp_event_data,
11529	LPFC_NL_VENDOR_ID);
11530	break;
11531	case ASYNC_STATUS_CN:
11532	lpfc_sli_abts_err_handler(phba, iocbq);
11533	break;
11534	default:
11535	iocb_w = (uint32_t *) icmd;
11536	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
11537	"0346 Ring %d handler: unexpected ASYNC_STATUS"
11538	" evt_code 0x%x\n"
11539	"W0 0x%08x W1 0x%08x W2 0x%08x W3 0x%08x\n"
11540	"W4 0x%08x W5 0x%08x W6 0x%08x W7 0x%08x\n"
11541	"W8 0x%08x W9 0x%08x W10 0x%08x W11 0x%08x\n"
11542	"W12 0x%08x W13 0x%08x W14 0x%08x W15 0x%08x\n",
11543	pring->ringno, icmd->un.asyncstat.evt_code,
11544	iocb_w[0], iocb_w[1], iocb_w[2], iocb_w[3],
11545	iocb_w[4], iocb_w[5], iocb_w[6], iocb_w[7],
11546	iocb_w[8], iocb_w[9], iocb_w[10], iocb_w[11],
11547	iocb_w[12], iocb_w[13], iocb_w[14], iocb_w[15]);
11548
11549	break;
11550	}
11551	}
11552
11553
11554	/**
11555	* lpfc_sli4_setup - SLI ring setup function
11556	* @phba: Pointer to HBA context object.
11557	*
11558	* lpfc_sli_setup sets up rings of the SLI interface with
11559	* number of iocbs per ring and iotags. This function is
11560	* called while driver attach to the HBA and before the
11561	* interrupts are enabled. So there is no need for locking.
11562	*
11563	* This function always returns 0.
11564	**/
11565	int
11566	lpfc_sli4_setup(struct lpfc_hba *phba)
11567	{
11568	struct lpfc_sli_ring *pring;
11569
11570	pring = phba->sli4_hba.els_wq->pring;
11571	pring->num_mask = LPFC_MAX_RING_MASK;
11572	pring->prt[0].profile = 0; /* Mask 0 */
11573	pring->prt[0].rctl = FC_RCTL_ELS_REQ;
11574	pring->prt[0].type = FC_TYPE_ELS;
11575	pring->prt[0].lpfc_sli_rcv_unsol_event =
11576	lpfc_els_unsol_event;
11577	pring->prt[1].profile = 0; /* Mask 1 */
11578	pring->prt[1].rctl = FC_RCTL_ELS_REP;
11579	pring->prt[1].type = FC_TYPE_ELS;
11580	pring->prt[1].lpfc_sli_rcv_unsol_event =
11581	lpfc_els_unsol_event;
11582	pring->prt[2].profile = 0; /* Mask 2 */
11583	/* NameServer Inquiry */
11584	pring->prt[2].rctl = FC_RCTL_DD_UNSOL_CTL;
11585	/* NameServer */
11586	pring->prt[2].type = FC_TYPE_CT;
11587	pring->prt[2].lpfc_sli_rcv_unsol_event =
11588	lpfc_ct_unsol_event;
11589	pring->prt[3].profile = 0; /* Mask 3 */
11590	/* NameServer response */
11591	pring->prt[3].rctl = FC_RCTL_DD_SOL_CTL;
11592	/* NameServer */
11593	pring->prt[3].type = FC_TYPE_CT;
11594	pring->prt[3].lpfc_sli_rcv_unsol_event =
11595	lpfc_ct_unsol_event;
11596	return 0;
11597	}
11598
11599	/**
11600	* lpfc_sli_setup - SLI ring setup function
11601	* @phba: Pointer to HBA context object.
11602	*
11603	* lpfc_sli_setup sets up rings of the SLI interface with
11604	* number of iocbs per ring and iotags. This function is
11605	* called while driver attach to the HBA and before the
11606	* interrupts are enabled. So there is no need for locking.
11607	*
11608	* This function always returns 0. SLI3 only.
11609	**/
11610	int
11611	lpfc_sli_setup(struct lpfc_hba *phba)
11612	{
11613	int i, totiocbsize = 0;
11614	struct lpfc_sli *psli = &phba->sli;
11615	struct lpfc_sli_ring *pring;
11616
11617	psli->num_rings = MAX_SLI3_CONFIGURED_RINGS;
11618	psli->sli_flag = 0;
11619
11620	psli->iocbq_lookup = NULL;
11621	psli->iocbq_lookup_len = 0;
11622	psli->last_iotag = 0;
11623
11624	for (i = 0; i < psli->num_rings; i++) {
11625	pring = &psli->sli3_ring[i];
11626	switch (i) {
11627	case LPFC_FCP_RING: /* ring 0 - FCP */
11628	/* numCiocb and numRiocb are used in config_port */
11629	pring->sli.sli3.numCiocb = SLI2_IOCB_CMD_R0_ENTRIES;
11630	pring->sli.sli3.numRiocb = SLI2_IOCB_RSP_R0_ENTRIES;
11631	pring->sli.sli3.numCiocb +=
11632	SLI2_IOCB_CMD_R1XTRA_ENTRIES;
11633	pring->sli.sli3.numRiocb +=
11634	SLI2_IOCB_RSP_R1XTRA_ENTRIES;
11635	pring->sli.sli3.numCiocb +=
11636	SLI2_IOCB_CMD_R3XTRA_ENTRIES;
11637	pring->sli.sli3.numRiocb +=
11638	SLI2_IOCB_RSP_R3XTRA_ENTRIES;
11639	pring->sli.sli3.sizeCiocb = (phba->sli_rev == 3) ?
11640	SLI3_IOCB_CMD_SIZE :
11641	SLI2_IOCB_CMD_SIZE;
11642	pring->sli.sli3.sizeRiocb = (phba->sli_rev == 3) ?
11643	SLI3_IOCB_RSP_SIZE :
11644	SLI2_IOCB_RSP_SIZE;
11645	pring->iotag_ctr = 0;
11646	pring->iotag_max =
11647	(phba->cfg_hba_queue_depth * 2);
11648	pring->fast_iotag = pring->iotag_max;
11649	pring->num_mask = 0;
11650	break;
11651	case LPFC_EXTRA_RING: /* ring 1 - EXTRA */
11652	/* numCiocb and numRiocb are used in config_port */
11653	pring->sli.sli3.numCiocb = SLI2_IOCB_CMD_R1_ENTRIES;
11654	pring->sli.sli3.numRiocb = SLI2_IOCB_RSP_R1_ENTRIES;
11655	pring->sli.sli3.sizeCiocb = (phba->sli_rev == 3) ?
11656	SLI3_IOCB_CMD_SIZE :
11657	SLI2_IOCB_CMD_SIZE;
11658	pring->sli.sli3.sizeRiocb = (phba->sli_rev == 3) ?
11659	SLI3_IOCB_RSP_SIZE :
11660	SLI2_IOCB_RSP_SIZE;
11661	pring->iotag_max = phba->cfg_hba_queue_depth;
11662	pring->num_mask = 0;
11663	break;
11664	case LPFC_ELS_RING: /* ring 2 - ELS / CT */
11665	/* numCiocb and numRiocb are used in config_port */
11666	pring->sli.sli3.numCiocb = SLI2_IOCB_CMD_R2_ENTRIES;
11667	pring->sli.sli3.numRiocb = SLI2_IOCB_RSP_R2_ENTRIES;
11668	pring->sli.sli3.sizeCiocb = (phba->sli_rev == 3) ?
11669	SLI3_IOCB_CMD_SIZE :
11670	SLI2_IOCB_CMD_SIZE;
11671	pring->sli.sli3.sizeRiocb = (phba->sli_rev == 3) ?
11672	SLI3_IOCB_RSP_SIZE :
11673	SLI2_IOCB_RSP_SIZE;
11674	pring->fast_iotag = 0;
11675	pring->iotag_ctr = 0;
11676	pring->iotag_max = 4096;
11677	pring->lpfc_sli_rcv_async_status =
11678	lpfc_sli_async_event_handler;
11679	pring->num_mask = LPFC_MAX_RING_MASK;
11680	pring->prt[0].profile = 0; /* Mask 0 */
11681	pring->prt[0].rctl = FC_RCTL_ELS_REQ;
11682	pring->prt[0].type = FC_TYPE_ELS;
11683	pring->prt[0].lpfc_sli_rcv_unsol_event =
11684	lpfc_els_unsol_event;
11685	pring->prt[1].profile = 0; /* Mask 1 */
11686	pring->prt[1].rctl = FC_RCTL_ELS_REP;
11687	pring->prt[1].type = FC_TYPE_ELS;
11688	pring->prt[1].lpfc_sli_rcv_unsol_event =
11689	lpfc_els_unsol_event;
11690	pring->prt[2].profile = 0; /* Mask 2 */
11691	/* NameServer Inquiry */
11692	pring->prt[2].rctl = FC_RCTL_DD_UNSOL_CTL;
11693	/* NameServer */
11694	pring->prt[2].type = FC_TYPE_CT;
11695	pring->prt[2].lpfc_sli_rcv_unsol_event =
11696	lpfc_ct_unsol_event;
11697	pring->prt[3].profile = 0; /* Mask 3 */
11698	/* NameServer response */
11699	pring->prt[3].rctl = FC_RCTL_DD_SOL_CTL;
11700	/* NameServer */
11701	pring->prt[3].type = FC_TYPE_CT;
11702	pring->prt[3].lpfc_sli_rcv_unsol_event =
11703	lpfc_ct_unsol_event;
11704	break;
11705	}
11706	totiocbsize += (pring->sli.sli3.numCiocb *
11707	pring->sli.sli3.sizeCiocb) +
11708	(pring->sli.sli3.numRiocb * pring->sli.sli3.sizeRiocb);
11709	}
11710	if (totiocbsize > MAX_SLIM_IOCB_SIZE) {
11711	/* Too many cmd / rsp ring entries in SLI2 SLIM */
11712	printk(KERN_ERR "%d:0462 Too many cmd / rsp ring entries in "
11713	"SLI2 SLIM Data: x%x x%lx\n",
11714	phba->brd_no, totiocbsize,
11715	(unsigned long) MAX_SLIM_IOCB_SIZE);
11716	}
11717	if (phba->cfg_multi_ring_support == 2)
11718	lpfc_extra_ring_setup(phba);
11719
11720	return 0;
11721	}
11722
11723	/**
11724	* lpfc_sli4_queue_init - Queue initialization function
11725	* @phba: Pointer to HBA context object.
11726	*
11727	* lpfc_sli4_queue_init sets up mailbox queues and iocb queues for each
11728	* ring. This function also initializes ring indices of each ring.
11729	* This function is called during the initialization of the SLI
11730	* interface of an HBA.
11731	* This function is called with no lock held and always returns
11732	* 1.
11733	**/
11734	void
11735	lpfc_sli4_queue_init(struct lpfc_hba *phba)
11736	{
11737	struct lpfc_sli *psli;
11738	struct lpfc_sli_ring *pring;
11739	int i;
11740
11741	psli = &phba->sli;
11742	spin_lock_irq(lock: &phba->hbalock);
11743	INIT_LIST_HEAD(list: &psli->mboxq);
11744	INIT_LIST_HEAD(list: &psli->mboxq_cmpl);
11745	/* Initialize list headers for txq and txcmplq as double linked lists */
11746	for (i = 0; i < phba->cfg_hdw_queue; i++) {
11747	pring = phba->sli4_hba.hdwq[i].io_wq->pring;
11748	pring->flag = 0;
11749	pring->ringno = LPFC_FCP_RING;
11750	pring->txcmplq_cnt = 0;
11751	INIT_LIST_HEAD(list: &pring->txq);
11752	INIT_LIST_HEAD(list: &pring->txcmplq);
11753	INIT_LIST_HEAD(list: &pring->iocb_continueq);
11754	spin_lock_init(&pring->ring_lock);
11755	}
11756	pring = phba->sli4_hba.els_wq->pring;
11757	pring->flag = 0;
11758	pring->ringno = LPFC_ELS_RING;
11759	pring->txcmplq_cnt = 0;
11760	INIT_LIST_HEAD(list: &pring->txq);
11761	INIT_LIST_HEAD(list: &pring->txcmplq);
11762	INIT_LIST_HEAD(list: &pring->iocb_continueq);
11763	spin_lock_init(&pring->ring_lock);
11764
11765	if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
11766	pring = phba->sli4_hba.nvmels_wq->pring;
11767	pring->flag = 0;
11768	pring->ringno = LPFC_ELS_RING;
11769	pring->txcmplq_cnt = 0;
11770	INIT_LIST_HEAD(list: &pring->txq);
11771	INIT_LIST_HEAD(list: &pring->txcmplq);
11772	INIT_LIST_HEAD(list: &pring->iocb_continueq);
11773	spin_lock_init(&pring->ring_lock);
11774	}
11775
11776	spin_unlock_irq(lock: &phba->hbalock);
11777	}
11778
11779	/**
11780	* lpfc_sli_queue_init - Queue initialization function
11781	* @phba: Pointer to HBA context object.
11782	*
11783	* lpfc_sli_queue_init sets up mailbox queues and iocb queues for each
11784	* ring. This function also initializes ring indices of each ring.
11785	* This function is called during the initialization of the SLI
11786	* interface of an HBA.
11787	* This function is called with no lock held and always returns
11788	* 1.
11789	**/
11790	void
11791	lpfc_sli_queue_init(struct lpfc_hba *phba)
11792	{
11793	struct lpfc_sli *psli;
11794	struct lpfc_sli_ring *pring;
11795	int i;
11796
11797	psli = &phba->sli;
11798	spin_lock_irq(lock: &phba->hbalock);
11799	INIT_LIST_HEAD(list: &psli->mboxq);
11800	INIT_LIST_HEAD(list: &psli->mboxq_cmpl);
11801	/* Initialize list headers for txq and txcmplq as double linked lists */
11802	for (i = 0; i < psli->num_rings; i++) {
11803	pring = &psli->sli3_ring[i];
11804	pring->ringno = i;
11805	pring->sli.sli3.next_cmdidx = 0;
11806	pring->sli.sli3.local_getidx = 0;
11807	pring->sli.sli3.cmdidx = 0;
11808	INIT_LIST_HEAD(list: &pring->iocb_continueq);
11809	INIT_LIST_HEAD(list: &pring->iocb_continue_saveq);
11810	INIT_LIST_HEAD(list: &pring->postbufq);
11811	pring->flag = 0;
11812	INIT_LIST_HEAD(list: &pring->txq);
11813	INIT_LIST_HEAD(list: &pring->txcmplq);
11814	spin_lock_init(&pring->ring_lock);
11815	}
11816	spin_unlock_irq(lock: &phba->hbalock);
11817	}
11818
11819	/**
11820	* lpfc_sli_mbox_sys_flush - Flush mailbox command sub-system
11821	* @phba: Pointer to HBA context object.
11822	*
11823	* This routine flushes the mailbox command subsystem. It will unconditionally
11824	* flush all the mailbox commands in the three possible stages in the mailbox
11825	* command sub-system: pending mailbox command queue; the outstanding mailbox
11826	* command; and completed mailbox command queue. It is caller's responsibility
11827	* to make sure that the driver is in the proper state to flush the mailbox
11828	* command sub-system. Namely, the posting of mailbox commands into the
11829	* pending mailbox command queue from the various clients must be stopped;
11830	* either the HBA is in a state that it will never works on the outstanding
11831	* mailbox command (such as in EEH or ERATT conditions) or the outstanding
11832	* mailbox command has been completed.
11833	**/
11834	static void
11835	lpfc_sli_mbox_sys_flush(struct lpfc_hba *phba)
11836	{
11837	LIST_HEAD(completions);
11838	struct lpfc_sli *psli = &phba->sli;
11839	LPFC_MBOXQ_t *pmb;
11840	unsigned long iflag;
11841
11842	/* Disable softirqs, including timers from obtaining phba->hbalock */
11843	local_bh_disable();
11844
11845	/* Flush all the mailbox commands in the mbox system */
11846	spin_lock_irqsave(&phba->hbalock, iflag);
11847
11848	/* The pending mailbox command queue */
11849	list_splice_init(list: &phba->sli.mboxq, head: &completions);
11850	/* The outstanding active mailbox command */
11851	if (psli->mbox_active) {
11852	list_add_tail(new: &psli->mbox_active->list, head: &completions);
11853	psli->mbox_active = NULL;
11854	psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
11855	}
11856	/* The completed mailbox command queue */
11857	list_splice_init(list: &phba->sli.mboxq_cmpl, head: &completions);
11858	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
11859
11860	/* Enable softirqs again, done with phba->hbalock */
11861	local_bh_enable();
11862
11863	/* Return all flushed mailbox commands with MBX_NOT_FINISHED status */
11864	while (!list_empty(head: &completions)) {
11865	list_remove_head(&completions, pmb, LPFC_MBOXQ_t, list);
11866	pmb->u.mb.mbxStatus = MBX_NOT_FINISHED;
11867	if (pmb->mbox_cmpl)
11868	pmb->mbox_cmpl(phba, pmb);
11869	}
11870	}
11871
11872	/**
11873	* lpfc_sli_host_down - Vport cleanup function
11874	* @vport: Pointer to virtual port object.
11875	*
11876	* lpfc_sli_host_down is called to clean up the resources
11877	* associated with a vport before destroying virtual
11878	* port data structures.
11879	* This function does following operations:
11880	* - Free discovery resources associated with this virtual
11881	* port.
11882	* - Free iocbs associated with this virtual port in
11883	* the txq.
11884	* - Send abort for all iocb commands associated with this
11885	* vport in txcmplq.
11886	*
11887	* This function is called with no lock held and always returns 1.
11888	**/
11889	int
11890	lpfc_sli_host_down(struct lpfc_vport *vport)
11891	{
11892	LIST_HEAD(completions);
11893	struct lpfc_hba *phba = vport->phba;
11894	struct lpfc_sli *psli = &phba->sli;
11895	struct lpfc_queue *qp = NULL;
11896	struct lpfc_sli_ring *pring;
11897	struct lpfc_iocbq *iocb, *next_iocb;
11898	int i;
11899	unsigned long flags = 0;
11900	uint16_t prev_pring_flag;
11901
11902	lpfc_cleanup_discovery_resources(vport);
11903
11904	spin_lock_irqsave(&phba->hbalock, flags);
11905
11906	/*
11907	* Error everything on the txq since these iocbs
11908	* have not been given to the FW yet.
11909	* Also issue ABTS for everything on the txcmplq
11910	*/
11911	if (phba->sli_rev != LPFC_SLI_REV4) {
11912	for (i = 0; i < psli->num_rings; i++) {
11913	pring = &psli->sli3_ring[i];
11914	prev_pring_flag = pring->flag;
11915	/* Only slow rings */
11916	if (pring->ringno == LPFC_ELS_RING) {
11917	pring->flag |= LPFC_DEFERRED_RING_EVENT;
11918	/* Set the lpfc data pending flag */
11919	set_bit(LPFC_DATA_READY, addr: &phba->data_flags);
11920	}
11921	list_for_each_entry_safe(iocb, next_iocb,
11922	&pring->txq, list) {
11923	if (iocb->vport != vport)
11924	continue;
11925	list_move_tail(list: &iocb->list, head: &completions);
11926	}
11927	list_for_each_entry_safe(iocb, next_iocb,
11928	&pring->txcmplq, list) {
11929	if (iocb->vport != vport)
11930	continue;
11931	lpfc_sli_issue_abort_iotag(phba, pring, iocb,
11932	NULL);
11933	}
11934	pring->flag = prev_pring_flag;
11935	}
11936	} else {
11937	list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
11938	pring = qp->pring;
11939	if (!pring)
11940	continue;
11941	if (pring == phba->sli4_hba.els_wq->pring) {
11942	pring->flag |= LPFC_DEFERRED_RING_EVENT;
11943	/* Set the lpfc data pending flag */
11944	set_bit(LPFC_DATA_READY, addr: &phba->data_flags);
11945	}
11946	prev_pring_flag = pring->flag;
11947	spin_lock(lock: &pring->ring_lock);
11948	list_for_each_entry_safe(iocb, next_iocb,
11949	&pring->txq, list) {
11950	if (iocb->vport != vport)
11951	continue;
11952	list_move_tail(list: &iocb->list, head: &completions);
11953	}
11954	spin_unlock(lock: &pring->ring_lock);
11955	list_for_each_entry_safe(iocb, next_iocb,
11956	&pring->txcmplq, list) {
11957	if (iocb->vport != vport)
11958	continue;
11959	lpfc_sli_issue_abort_iotag(phba, pring, iocb,
11960	NULL);
11961	}
11962	pring->flag = prev_pring_flag;
11963	}
11964	}
11965	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
11966
11967	/* Make sure HBA is alive */
11968	lpfc_issue_hb_tmo(phba);
11969
11970	/* Cancel all the IOCBs from the completions list */
11971	lpfc_sli_cancel_iocbs(phba, iocblist: &completions, IOSTAT_LOCAL_REJECT,
11972	IOERR_SLI_DOWN);
11973	return 1;
11974	}
11975
11976	/**
11977	* lpfc_sli_hba_down - Resource cleanup function for the HBA
11978	* @phba: Pointer to HBA context object.
11979	*
11980	* This function cleans up all iocb, buffers, mailbox commands
11981	* while shutting down the HBA. This function is called with no
11982	* lock held and always returns 1.
11983	* This function does the following to cleanup driver resources:
11984	* - Free discovery resources for each virtual port
11985	* - Cleanup any pending fabric iocbs
11986	* - Iterate through the iocb txq and free each entry
11987	* in the list.
11988	* - Free up any buffer posted to the HBA
11989	* - Free mailbox commands in the mailbox queue.
11990	**/
11991	int
11992	lpfc_sli_hba_down(struct lpfc_hba *phba)
11993	{
11994	LIST_HEAD(completions);
11995	struct lpfc_sli *psli = &phba->sli;
11996	struct lpfc_queue *qp = NULL;
11997	struct lpfc_sli_ring *pring;
11998	struct lpfc_dmabuf *buf_ptr;
11999	unsigned long flags = 0;
12000	int i;
12001
12002	/* Shutdown the mailbox command sub-system */
12003	lpfc_sli_mbox_sys_shutdown(phba, LPFC_MBX_WAIT);
12004
12005	lpfc_hba_down_prep(phba);
12006
12007	/* Disable softirqs, including timers from obtaining phba->hbalock */
12008	local_bh_disable();
12009
12010	lpfc_fabric_abort_hba(phba);
12011
12012	spin_lock_irqsave(&phba->hbalock, flags);
12013
12014	/*
12015	* Error everything on the txq since these iocbs
12016	* have not been given to the FW yet.
12017	*/
12018	if (phba->sli_rev != LPFC_SLI_REV4) {
12019	for (i = 0; i < psli->num_rings; i++) {
12020	pring = &psli->sli3_ring[i];
12021	/* Only slow rings */
12022	if (pring->ringno == LPFC_ELS_RING) {
12023	pring->flag |= LPFC_DEFERRED_RING_EVENT;
12024	/* Set the lpfc data pending flag */
12025	set_bit(LPFC_DATA_READY, addr: &phba->data_flags);
12026	}
12027	list_splice_init(list: &pring->txq, head: &completions);
12028	}
12029	} else {
12030	list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
12031	pring = qp->pring;
12032	if (!pring)
12033	continue;
12034	spin_lock(lock: &pring->ring_lock);
12035	list_splice_init(list: &pring->txq, head: &completions);
12036	spin_unlock(lock: &pring->ring_lock);
12037	if (pring == phba->sli4_hba.els_wq->pring) {
12038	pring->flag |= LPFC_DEFERRED_RING_EVENT;
12039	/* Set the lpfc data pending flag */
12040	set_bit(LPFC_DATA_READY, addr: &phba->data_flags);
12041	}
12042	}
12043	}
12044	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
12045
12046	/* Cancel all the IOCBs from the completions list */
12047	lpfc_sli_cancel_iocbs(phba, iocblist: &completions, IOSTAT_LOCAL_REJECT,
12048	IOERR_SLI_DOWN);
12049
12050	spin_lock_irqsave(&phba->hbalock, flags);
12051	list_splice_init(list: &phba->elsbuf, head: &completions);
12052	phba->elsbuf_cnt = 0;
12053	phba->elsbuf_prev_cnt = 0;
12054	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
12055
12056	while (!list_empty(head: &completions)) {
12057	list_remove_head(&completions, buf_ptr,
12058	struct lpfc_dmabuf, list);
12059	lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
12060	kfree(objp: buf_ptr);
12061	}
12062
12063	/* Enable softirqs again, done with phba->hbalock */
12064	local_bh_enable();
12065
12066	/* Return any active mbox cmds */
12067	del_timer_sync(timer: &psli->mbox_tmo);
12068
12069	spin_lock_irqsave(&phba->pport->work_port_lock, flags);
12070	phba->pport->work_port_events &= ~WORKER_MBOX_TMO;
12071	spin_unlock_irqrestore(lock: &phba->pport->work_port_lock, flags);
12072
12073	return 1;
12074	}
12075
12076	/**
12077	* lpfc_sli_pcimem_bcopy - SLI memory copy function
12078	* @srcp: Source memory pointer.
12079	* @destp: Destination memory pointer.
12080	* @cnt: Number of words required to be copied.
12081	*
12082	* This function is used for copying data between driver memory
12083	* and the SLI memory. This function also changes the endianness
12084	* of each word if native endianness is different from SLI
12085	* endianness. This function can be called with or without
12086	* lock.
12087	**/
12088	void
12089	lpfc_sli_pcimem_bcopy(void *srcp, void *destp, uint32_t cnt)
12090	{
12091	uint32_t *src = srcp;
12092	uint32_t *dest = destp;
12093	uint32_t ldata;
12094	int i;
12095
12096	for (i = 0; i < (int)cnt; i += sizeof (uint32_t)) {
12097	ldata = *src;
12098	ldata = le32_to_cpu(ldata);
12099	*dest = ldata;
12100	src++;
12101	dest++;
12102	}
12103	}
12104
12105
12106	/**
12107	* lpfc_sli_bemem_bcopy - SLI memory copy function
12108	* @srcp: Source memory pointer.
12109	* @destp: Destination memory pointer.
12110	* @cnt: Number of words required to be copied.
12111	*
12112	* This function is used for copying data between a data structure
12113	* with big endian representation to local endianness.
12114	* This function can be called with or without lock.
12115	**/
12116	void
12117	lpfc_sli_bemem_bcopy(void *srcp, void *destp, uint32_t cnt)
12118	{
12119	uint32_t *src = srcp;
12120	uint32_t *dest = destp;
12121	uint32_t ldata;
12122	int i;
12123
12124	for (i = 0; i < (int)cnt; i += sizeof(uint32_t)) {
12125	ldata = *src;
12126	ldata = be32_to_cpu(ldata);
12127	*dest = ldata;
12128	src++;
12129	dest++;
12130	}
12131	}
12132
12133	/**
12134	* lpfc_sli_ringpostbuf_put - Function to add a buffer to postbufq
12135	* @phba: Pointer to HBA context object.
12136	* @pring: Pointer to driver SLI ring object.
12137	* @mp: Pointer to driver buffer object.
12138	*
12139	* This function is called with no lock held.
12140	* It always return zero after adding the buffer to the postbufq
12141	* buffer list.
12142	**/
12143	int
12144	lpfc_sli_ringpostbuf_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
12145	struct lpfc_dmabuf *mp)
12146	{
12147	/* Stick struct lpfc_dmabuf at end of postbufq so driver can look it up
12148	later */
12149	spin_lock_irq(lock: &phba->hbalock);
12150	list_add_tail(new: &mp->list, head: &pring->postbufq);
12151	pring->postbufq_cnt++;
12152	spin_unlock_irq(lock: &phba->hbalock);
12153	return 0;
12154	}
12155
12156	/**
12157	* lpfc_sli_get_buffer_tag - allocates a tag for a CMD_QUE_XRI64_CX buffer
12158	* @phba: Pointer to HBA context object.
12159	*
12160	* When HBQ is enabled, buffers are searched based on tags. This function
12161	* allocates a tag for buffer posted using CMD_QUE_XRI64_CX iocb. The
12162	* tag is bit wise or-ed with QUE_BUFTAG_BIT to make sure that the tag
12163	* does not conflict with tags of buffer posted for unsolicited events.
12164	* The function returns the allocated tag. The function is called with
12165	* no locks held.
12166	**/
12167	uint32_t
12168	lpfc_sli_get_buffer_tag(struct lpfc_hba *phba)
12169	{
12170	spin_lock_irq(lock: &phba->hbalock);
12171	phba->buffer_tag_count++;
12172	/*
12173	* Always set the QUE_BUFTAG_BIT to distiguish between
12174	* a tag assigned by HBQ.
12175	*/
12176	phba->buffer_tag_count |= QUE_BUFTAG_BIT;
12177	spin_unlock_irq(lock: &phba->hbalock);
12178	return phba->buffer_tag_count;
12179	}
12180
12181	/**
12182	* lpfc_sli_ring_taggedbuf_get - find HBQ buffer associated with given tag
12183	* @phba: Pointer to HBA context object.
12184	* @pring: Pointer to driver SLI ring object.
12185	* @tag: Buffer tag.
12186	*
12187	* Buffers posted using CMD_QUE_XRI64_CX iocb are in pring->postbufq
12188	* list. After HBA DMA data to these buffers, CMD_IOCB_RET_XRI64_CX
12189	* iocb is posted to the response ring with the tag of the buffer.
12190	* This function searches the pring->postbufq list using the tag
12191	* to find buffer associated with CMD_IOCB_RET_XRI64_CX
12192	* iocb. If the buffer is found then lpfc_dmabuf object of the
12193	* buffer is returned to the caller else NULL is returned.
12194	* This function is called with no lock held.
12195	**/
12196	struct lpfc_dmabuf *
12197	lpfc_sli_ring_taggedbuf_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
12198	uint32_t tag)
12199	{
12200	struct lpfc_dmabuf *mp, *next_mp;
12201	struct list_head *slp = &pring->postbufq;
12202
12203	/* Search postbufq, from the beginning, looking for a match on tag */
12204	spin_lock_irq(lock: &phba->hbalock);
12205	list_for_each_entry_safe(mp, next_mp, &pring->postbufq, list) {
12206	if (mp->buffer_tag == tag) {
12207	list_del_init(entry: &mp->list);
12208	pring->postbufq_cnt--;
12209	spin_unlock_irq(lock: &phba->hbalock);
12210	return mp;
12211	}
12212	}
12213
12214	spin_unlock_irq(lock: &phba->hbalock);
12215	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
12216	"0402 Cannot find virtual addr for buffer tag on "
12217	"ring %d Data x%lx x%px x%px x%x\n",
12218	pring->ringno, (unsigned long) tag,
12219	slp->next, slp->prev, pring->postbufq_cnt);
12220
12221	return NULL;
12222	}
12223
12224	/**
12225	* lpfc_sli_ringpostbuf_get - search buffers for unsolicited CT and ELS events
12226	* @phba: Pointer to HBA context object.
12227	* @pring: Pointer to driver SLI ring object.
12228	* @phys: DMA address of the buffer.
12229	*
12230	* This function searches the buffer list using the dma_address
12231	* of unsolicited event to find the driver's lpfc_dmabuf object
12232	* corresponding to the dma_address. The function returns the
12233	* lpfc_dmabuf object if a buffer is found else it returns NULL.
12234	* This function is called by the ct and els unsolicited event
12235	* handlers to get the buffer associated with the unsolicited
12236	* event.
12237	*
12238	* This function is called with no lock held.
12239	**/
12240	struct lpfc_dmabuf *
12241	lpfc_sli_ringpostbuf_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
12242	dma_addr_t phys)
12243	{
12244	struct lpfc_dmabuf *mp, *next_mp;
12245	struct list_head *slp = &pring->postbufq;
12246
12247	/* Search postbufq, from the beginning, looking for a match on phys */
12248	spin_lock_irq(lock: &phba->hbalock);
12249	list_for_each_entry_safe(mp, next_mp, &pring->postbufq, list) {
12250	if (mp->phys == phys) {
12251	list_del_init(entry: &mp->list);
12252	pring->postbufq_cnt--;
12253	spin_unlock_irq(lock: &phba->hbalock);
12254	return mp;
12255	}
12256	}
12257
12258	spin_unlock_irq(lock: &phba->hbalock);
12259	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
12260	"0410 Cannot find virtual addr for mapped buf on "
12261	"ring %d Data x%llx x%px x%px x%x\n",
12262	pring->ringno, (unsigned long long)phys,
12263	slp->next, slp->prev, pring->postbufq_cnt);
12264	return NULL;
12265	}
12266
12267	/**
12268	* lpfc_sli_abort_els_cmpl - Completion handler for the els abort iocbs
12269	* @phba: Pointer to HBA context object.
12270	* @cmdiocb: Pointer to driver command iocb object.
12271	* @rspiocb: Pointer to driver response iocb object.
12272	*
12273	* This function is the completion handler for the abort iocbs for
12274	* ELS commands. This function is called from the ELS ring event
12275	* handler with no lock held. This function frees memory resources
12276	* associated with the abort iocb.
12277	**/
12278	static void
12279	lpfc_sli_abort_els_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
12280	struct lpfc_iocbq *rspiocb)
12281	{
12282	u32 ulp_status = get_job_ulpstatus(phba, iocbq: rspiocb);
12283	u32 ulp_word4 = get_job_word4(phba, iocbq: rspiocb);
12284	u8 cmnd = get_job_cmnd(phba, iocbq: cmdiocb);
12285
12286	if (ulp_status) {
12287	/*
12288	* Assume that the port already completed and returned, or
12289	* will return the iocb. Just Log the message.
12290	*/
12291	if (phba->sli_rev < LPFC_SLI_REV4) {
12292	if (cmnd == CMD_ABORT_XRI_CX &&
12293	ulp_status == IOSTAT_LOCAL_REJECT &&
12294	ulp_word4 == IOERR_ABORT_REQUESTED) {
12295	goto release_iocb;
12296	}
12297	}
12298
12299	lpfc_printf_log(phba, KERN_WARNING, LOG_ELS | LOG_SLI,
12300	"0327 Cannot abort els iocb x%px "
12301	"with io cmd xri %x abort tag : x%x, "
12302	"abort status %x abort code %x\n",
12303	cmdiocb, get_job_abtsiotag(phba, cmdiocb),
12304	(phba->sli_rev == LPFC_SLI_REV4) ?
12305	get_wqe_reqtag(cmdiocb) :
12306	cmdiocb->iocb.un.acxri.abortContextTag,
12307	ulp_status, ulp_word4);
12308
12309	}
12310	release_iocb:
12311	lpfc_sli_release_iocbq(phba, iocbq: cmdiocb);
12312	return;
12313	}
12314
12315	/**
12316	* lpfc_ignore_els_cmpl - Completion handler for aborted ELS command
12317	* @phba: Pointer to HBA context object.
12318	* @cmdiocb: Pointer to driver command iocb object.
12319	* @rspiocb: Pointer to driver response iocb object.
12320	*
12321	* The function is called from SLI ring event handler with no
12322	* lock held. This function is the completion handler for ELS commands
12323	* which are aborted. The function frees memory resources used for
12324	* the aborted ELS commands.
12325	**/
12326	void
12327	lpfc_ignore_els_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
12328	struct lpfc_iocbq *rspiocb)
12329	{
12330	struct lpfc_nodelist *ndlp = cmdiocb->ndlp;
12331	IOCB_t *irsp;
12332	LPFC_MBOXQ_t *mbox;
12333	u32 ulp_command, ulp_status, ulp_word4, iotag;
12334
12335	ulp_command = get_job_cmnd(phba, iocbq: cmdiocb);
12336	ulp_status = get_job_ulpstatus(phba, iocbq: rspiocb);
12337	ulp_word4 = get_job_word4(phba, iocbq: rspiocb);
12338
12339	if (phba->sli_rev == LPFC_SLI_REV4) {
12340	iotag = get_wqe_reqtag(cmdiocb);
12341	} else {
12342	irsp = &rspiocb->iocb;
12343	iotag = irsp->ulpIoTag;
12344
12345	/* It is possible a PLOGI_RJT for NPIV ports to get aborted.
12346	* The MBX_REG_LOGIN64 mbox command is freed back to the
12347	* mbox_mem_pool here.
12348	*/
12349	if (cmdiocb->context_un.mbox) {
12350	mbox = cmdiocb->context_un.mbox;
12351	lpfc_mbox_rsrc_cleanup(phba, mbox, locked: MBOX_THD_UNLOCKED);
12352	cmdiocb->context_un.mbox = NULL;
12353	}
12354	}
12355
12356	/* ELS cmd tag <ulpIoTag> completes */
12357	lpfc_printf_log(phba, KERN_INFO, LOG_ELS,
12358	"0139 Ignoring ELS cmd code x%x completion Data: "
12359	"x%x x%x x%x x%px\n",
12360	ulp_command, ulp_status, ulp_word4, iotag,
12361	cmdiocb->ndlp);
12362	/*
12363	* Deref the ndlp after free_iocb. sli_release_iocb will access the ndlp
12364	* if exchange is busy.
12365	*/
12366	if (ulp_command == CMD_GEN_REQUEST64_CR)
12367	lpfc_ct_free_iocb(phba, cmdiocb);
12368	else
12369	lpfc_els_free_iocb(phba, cmdiocb);
12370
12371	lpfc_nlp_put(ndlp);
12372	}
12373
12374	/**
12375	* lpfc_sli_issue_abort_iotag - Abort function for a command iocb
12376	* @phba: Pointer to HBA context object.
12377	* @pring: Pointer to driver SLI ring object.
12378	* @cmdiocb: Pointer to driver command iocb object.
12379	* @cmpl: completion function.
12380	*
12381	* This function issues an abort iocb for the provided command iocb. In case
12382	* of unloading, the abort iocb will not be issued to commands on the ELS
12383	* ring. Instead, the callback function shall be changed to those commands
12384	* so that nothing happens when them finishes. This function is called with
12385	* hbalock held andno ring_lock held (SLI4). The function returns IOCB_SUCCESS
12386	* when the command iocb is an abort request.
12387	*
12388	**/
12389	int
12390	lpfc_sli_issue_abort_iotag(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
12391	struct lpfc_iocbq *cmdiocb, void *cmpl)
12392	{
12393	struct lpfc_vport *vport = cmdiocb->vport;
12394	struct lpfc_iocbq *abtsiocbp;
12395	int retval = IOCB_ERROR;
12396	unsigned long iflags;
12397	struct lpfc_nodelist *ndlp = NULL;
12398	u32 ulp_command = get_job_cmnd(phba, iocbq: cmdiocb);
12399	u16 ulp_context, iotag;
12400	bool ia;
12401
12402	/*
12403	* There are certain command types we don't want to abort. And we
12404	* don't want to abort commands that are already in the process of
12405	* being aborted.
12406	*/
12407	if (ulp_command == CMD_ABORT_XRI_WQE ||
12408	ulp_command == CMD_ABORT_XRI_CN ||
12409	ulp_command == CMD_CLOSE_XRI_CN ||
12410	cmdiocb->cmd_flag & LPFC_DRIVER_ABORTED)
12411	return IOCB_ABORTING;
12412
12413	if (!pring) {
12414	if (cmdiocb->cmd_flag & LPFC_IO_FABRIC)
12415	cmdiocb->fabric_cmd_cmpl = lpfc_ignore_els_cmpl;
12416	else
12417	cmdiocb->cmd_cmpl = lpfc_ignore_els_cmpl;
12418	return retval;
12419	}
12420
12421	/*
12422	* If we're unloading, don't abort iocb on the ELS ring, but change
12423	* the callback so that nothing happens when it finishes.
12424	*/
12425	if ((vport->load_flag & FC_UNLOADING) &&
12426	pring->ringno == LPFC_ELS_RING) {
12427	if (cmdiocb->cmd_flag & LPFC_IO_FABRIC)
12428	cmdiocb->fabric_cmd_cmpl = lpfc_ignore_els_cmpl;
12429	else
12430	cmdiocb->cmd_cmpl = lpfc_ignore_els_cmpl;
12431	return retval;
12432	}
12433
12434	/* issue ABTS for this IOCB based on iotag */
12435	abtsiocbp = __lpfc_sli_get_iocbq(phba);
12436	if (abtsiocbp == NULL)
12437	return IOCB_NORESOURCE;
12438
12439	/* This signals the response to set the correct status
12440	* before calling the completion handler
12441	*/
12442	cmdiocb->cmd_flag |= LPFC_DRIVER_ABORTED;
12443
12444	if (phba->sli_rev == LPFC_SLI_REV4) {
12445	ulp_context = cmdiocb->sli4_xritag;
12446	iotag = abtsiocbp->iotag;
12447	} else {
12448	iotag = cmdiocb->iocb.ulpIoTag;
12449	if (pring->ringno == LPFC_ELS_RING) {
12450	ndlp = cmdiocb->ndlp;
12451	ulp_context = ndlp->nlp_rpi;
12452	} else {
12453	ulp_context = cmdiocb->iocb.ulpContext;
12454	}
12455	}
12456
12457	if (phba->link_state < LPFC_LINK_UP ||
12458	(phba->sli_rev == LPFC_SLI_REV4 &&
12459	phba->sli4_hba.link_state.status == LPFC_FC_LA_TYPE_LINK_DOWN) ||
12460	(phba->link_flag & LS_EXTERNAL_LOOPBACK))
12461	ia = true;
12462	else
12463	ia = false;
12464
12465	lpfc_sli_prep_abort_xri(phba, cmdiocbq: abtsiocbp, ulp_context, iotag,
12466	ulp_class: cmdiocb->iocb.ulpClass,
12467	LPFC_WQE_CQ_ID_DEFAULT, ia, wqec: false);
12468
12469	abtsiocbp->vport = vport;
12470
12471	/* ABTS WQE must go to the same WQ as the WQE to be aborted */
12472	abtsiocbp->hba_wqidx = cmdiocb->hba_wqidx;
12473	if (cmdiocb->cmd_flag & LPFC_IO_FCP)
12474	abtsiocbp->cmd_flag |= (LPFC_IO_FCP | LPFC_USE_FCPWQIDX);
12475
12476	if (cmdiocb->cmd_flag & LPFC_IO_FOF)
12477	abtsiocbp->cmd_flag |= LPFC_IO_FOF;
12478
12479	if (cmpl)
12480	abtsiocbp->cmd_cmpl = cmpl;
12481	else
12482	abtsiocbp->cmd_cmpl = lpfc_sli_abort_els_cmpl;
12483	abtsiocbp->vport = vport;
12484
12485	if (phba->sli_rev == LPFC_SLI_REV4) {
12486	pring = lpfc_sli4_calc_ring(phba, piocb: abtsiocbp);
12487	if (unlikely(pring == NULL))
12488	goto abort_iotag_exit;
12489	/* Note: both hbalock and ring_lock need to be set here */
12490	spin_lock_irqsave(&pring->ring_lock, iflags);
12491	retval = __lpfc_sli_issue_iocb(phba, ring_number: pring->ringno,
12492	piocb: abtsiocbp, flag: 0);
12493	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
12494	} else {
12495	retval = __lpfc_sli_issue_iocb(phba, ring_number: pring->ringno,
12496	piocb: abtsiocbp, flag: 0);
12497	}
12498
12499	abort_iotag_exit:
12500
12501	lpfc_printf_vlog(vport, KERN_INFO, LOG_SLI,
12502	"0339 Abort IO XRI x%x, Original iotag x%x, "
12503	"abort tag x%x Cmdjob : x%px Abortjob : x%px "
12504	"retval x%x\n",
12505	ulp_context, (phba->sli_rev == LPFC_SLI_REV4) ?
12506	cmdiocb->iotag : iotag, iotag, cmdiocb, abtsiocbp,
12507	retval);
12508	if (retval) {
12509	cmdiocb->cmd_flag &= ~LPFC_DRIVER_ABORTED;
12510	__lpfc_sli_release_iocbq(phba, iocbq: abtsiocbp);
12511	}
12512
12513	/*
12514	* Caller to this routine should check for IOCB_ERROR
12515	* and handle it properly. This routine no longer removes
12516	* iocb off txcmplq and call compl in case of IOCB_ERROR.
12517	*/
12518	return retval;
12519	}
12520
12521	/**
12522	* lpfc_sli_hba_iocb_abort - Abort all iocbs to an hba.
12523	* @phba: pointer to lpfc HBA data structure.
12524	*
12525	* This routine will abort all pending and outstanding iocbs to an HBA.
12526	**/
12527	void
12528	lpfc_sli_hba_iocb_abort(struct lpfc_hba *phba)
12529	{
12530	struct lpfc_sli *psli = &phba->sli;
12531	struct lpfc_sli_ring *pring;
12532	struct lpfc_queue *qp = NULL;
12533	int i;
12534
12535	if (phba->sli_rev != LPFC_SLI_REV4) {
12536	for (i = 0; i < psli->num_rings; i++) {
12537	pring = &psli->sli3_ring[i];
12538	lpfc_sli_abort_iocb_ring(phba, pring);
12539	}
12540	return;
12541	}
12542	list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
12543	pring = qp->pring;
12544	if (!pring)
12545	continue;
12546	lpfc_sli_abort_iocb_ring(phba, pring);
12547	}
12548	}
12549
12550	/**
12551	* lpfc_sli_validate_fcp_iocb_for_abort - filter iocbs appropriate for FCP aborts
12552	* @iocbq: Pointer to iocb object.
12553	* @vport: Pointer to driver virtual port object.
12554	*
12555	* This function acts as an iocb filter for functions which abort FCP iocbs.
12556	*
12557	* Return values
12558	* -ENODEV, if a null iocb or vport ptr is encountered
12559	* -EINVAL, if the iocb is not an FCP I/O, not on the TX cmpl queue, premarked as
12560	* driver already started the abort process, or is an abort iocb itself
12561	* 0, passes criteria for aborting the FCP I/O iocb
12562	**/
12563	static int
12564	lpfc_sli_validate_fcp_iocb_for_abort(struct lpfc_iocbq *iocbq,
12565	struct lpfc_vport *vport)
12566	{
12567	u8 ulp_command;
12568
12569	/* No null ptr vports */
12570	if (!iocbq || iocbq->vport != vport)
12571	return -ENODEV;
12572
12573	/* iocb must be for FCP IO, already exists on the TX cmpl queue,
12574	* can't be premarked as driver aborted, nor be an ABORT iocb itself
12575	*/
12576	ulp_command = get_job_cmnd(phba: vport->phba, iocbq);
12577	if (!(iocbq->cmd_flag & LPFC_IO_FCP) ||
12578	!(iocbq->cmd_flag & LPFC_IO_ON_TXCMPLQ) ||
12579	(iocbq->cmd_flag & LPFC_DRIVER_ABORTED) ||
12580	(ulp_command == CMD_ABORT_XRI_CN ||
12581	ulp_command == CMD_CLOSE_XRI_CN ||
12582	ulp_command == CMD_ABORT_XRI_WQE))
12583	return -EINVAL;
12584
12585	return 0;
12586	}
12587
12588	/**
12589	* lpfc_sli_validate_fcp_iocb - validate commands associated with a SCSI target
12590	* @iocbq: Pointer to driver iocb object.
12591	* @vport: Pointer to driver virtual port object.
12592	* @tgt_id: SCSI ID of the target.
12593	* @lun_id: LUN ID of the scsi device.
12594	* @ctx_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST
12595	*
12596	* This function acts as an iocb filter for validating a lun/SCSI target/SCSI
12597	* host.
12598	*
12599	* It will return
12600	* 0 if the filtering criteria is met for the given iocb and will return
12601	* 1 if the filtering criteria is not met.
12602	* If ctx_cmd == LPFC_CTX_LUN, the function returns 0 only if the
12603	* given iocb is for the SCSI device specified by vport, tgt_id and
12604	* lun_id parameter.
12605	* If ctx_cmd == LPFC_CTX_TGT, the function returns 0 only if the
12606	* given iocb is for the SCSI target specified by vport and tgt_id
12607	* parameters.
12608	* If ctx_cmd == LPFC_CTX_HOST, the function returns 0 only if the
12609	* given iocb is for the SCSI host associated with the given vport.
12610	* This function is called with no locks held.
12611	**/
12612	static int
12613	lpfc_sli_validate_fcp_iocb(struct lpfc_iocbq *iocbq, struct lpfc_vport *vport,
12614	uint16_t tgt_id, uint64_t lun_id,
12615	lpfc_ctx_cmd ctx_cmd)
12616	{
12617	struct lpfc_io_buf *lpfc_cmd;
12618	int rc = 1;
12619
12620	lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq);
12621
12622	if (lpfc_cmd->pCmd == NULL)
12623	return rc;
12624
12625	switch (ctx_cmd) {
12626	case LPFC_CTX_LUN:
12627	if ((lpfc_cmd->rdata) && (lpfc_cmd->rdata->pnode) &&
12628	(lpfc_cmd->rdata->pnode->nlp_sid == tgt_id) &&
12629	(scsilun_to_int(&lpfc_cmd->fcp_cmnd->fcp_lun) == lun_id))
12630	rc = 0;
12631	break;
12632	case LPFC_CTX_TGT:
12633	if ((lpfc_cmd->rdata) && (lpfc_cmd->rdata->pnode) &&
12634	(lpfc_cmd->rdata->pnode->nlp_sid == tgt_id))
12635	rc = 0;
12636	break;
12637	case LPFC_CTX_HOST:
12638	rc = 0;
12639	break;
12640	default:
12641	printk(KERN_ERR "%s: Unknown context cmd type, value %d\n",
12642	__func__, ctx_cmd);
12643	break;
12644	}
12645
12646	return rc;
12647	}
12648
12649	/**
12650	* lpfc_sli_sum_iocb - Function to count the number of FCP iocbs pending
12651	* @vport: Pointer to virtual port.
12652	* @tgt_id: SCSI ID of the target.
12653	* @lun_id: LUN ID of the scsi device.
12654	* @ctx_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST.
12655	*
12656	* This function returns number of FCP commands pending for the vport.
12657	* When ctx_cmd == LPFC_CTX_LUN, the function returns number of FCP
12658	* commands pending on the vport associated with SCSI device specified
12659	* by tgt_id and lun_id parameters.
12660	* When ctx_cmd == LPFC_CTX_TGT, the function returns number of FCP
12661	* commands pending on the vport associated with SCSI target specified
12662	* by tgt_id parameter.
12663	* When ctx_cmd == LPFC_CTX_HOST, the function returns number of FCP
12664	* commands pending on the vport.
12665	* This function returns the number of iocbs which satisfy the filter.
12666	* This function is called without any lock held.
12667	**/
12668	int
12669	lpfc_sli_sum_iocb(struct lpfc_vport *vport, uint16_t tgt_id, uint64_t lun_id,
12670	lpfc_ctx_cmd ctx_cmd)
12671	{
12672	struct lpfc_hba *phba = vport->phba;
12673	struct lpfc_iocbq *iocbq;
12674	int sum, i;
12675	unsigned long iflags;
12676	u8 ulp_command;
12677
12678	spin_lock_irqsave(&phba->hbalock, iflags);
12679	for (i = 1, sum = 0; i <= phba->sli.last_iotag; i++) {
12680	iocbq = phba->sli.iocbq_lookup[i];
12681
12682	if (!iocbq || iocbq->vport != vport)
12683	continue;
12684	if (!(iocbq->cmd_flag & LPFC_IO_FCP) ||
12685	!(iocbq->cmd_flag & LPFC_IO_ON_TXCMPLQ))
12686	continue;
12687
12688	/* Include counting outstanding aborts */
12689	ulp_command = get_job_cmnd(phba, iocbq);
12690	if (ulp_command == CMD_ABORT_XRI_CN ||
12691	ulp_command == CMD_CLOSE_XRI_CN ||
12692	ulp_command == CMD_ABORT_XRI_WQE) {
12693	sum++;
12694	continue;
12695	}
12696
12697	if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id,
12698	ctx_cmd) == 0)
12699	sum++;
12700	}
12701	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
12702
12703	return sum;
12704	}
12705
12706	/**
12707	* lpfc_sli_abort_fcp_cmpl - Completion handler function for aborted FCP IOCBs
12708	* @phba: Pointer to HBA context object
12709	* @cmdiocb: Pointer to command iocb object.
12710	* @rspiocb: Pointer to response iocb object.
12711	*
12712	* This function is called when an aborted FCP iocb completes. This
12713	* function is called by the ring event handler with no lock held.
12714	* This function frees the iocb.
12715	**/
12716	void
12717	lpfc_sli_abort_fcp_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
12718	struct lpfc_iocbq *rspiocb)
12719	{
12720	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
12721	"3096 ABORT_XRI_CX completing on rpi x%x "
12722	"original iotag x%x, abort cmd iotag x%x "
12723	"status 0x%x, reason 0x%x\n",
12724	(phba->sli_rev == LPFC_SLI_REV4) ?
12725	cmdiocb->sli4_xritag :
12726	cmdiocb->iocb.un.acxri.abortContextTag,
12727	get_job_abtsiotag(phba, cmdiocb),
12728	cmdiocb->iotag, get_job_ulpstatus(phba, rspiocb),
12729	get_job_word4(phba, rspiocb));
12730	lpfc_sli_release_iocbq(phba, iocbq: cmdiocb);
12731	return;
12732	}
12733
12734	/**
12735	* lpfc_sli_abort_iocb - issue abort for all commands on a host/target/LUN
12736	* @vport: Pointer to virtual port.
12737	* @tgt_id: SCSI ID of the target.
12738	* @lun_id: LUN ID of the scsi device.
12739	* @abort_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST.
12740	*
12741	* This function sends an abort command for every SCSI command
12742	* associated with the given virtual port pending on the ring
12743	* filtered by lpfc_sli_validate_fcp_iocb_for_abort and then
12744	* lpfc_sli_validate_fcp_iocb function. The ordering for validation before
12745	* submitting abort iocbs must be lpfc_sli_validate_fcp_iocb_for_abort
12746	* followed by lpfc_sli_validate_fcp_iocb.
12747	*
12748	* When abort_cmd == LPFC_CTX_LUN, the function sends abort only to the
12749	* FCP iocbs associated with lun specified by tgt_id and lun_id
12750	* parameters
12751	* When abort_cmd == LPFC_CTX_TGT, the function sends abort only to the
12752	* FCP iocbs associated with SCSI target specified by tgt_id parameter.
12753	* When abort_cmd == LPFC_CTX_HOST, the function sends abort to all
12754	* FCP iocbs associated with virtual port.
12755	* The pring used for SLI3 is sli3_ring[LPFC_FCP_RING], for SLI4
12756	* lpfc_sli4_calc_ring is used.
12757	* This function returns number of iocbs it failed to abort.
12758	* This function is called with no locks held.
12759	**/
12760	int
12761	lpfc_sli_abort_iocb(struct lpfc_vport *vport, u16 tgt_id, u64 lun_id,
12762	lpfc_ctx_cmd abort_cmd)
12763	{
12764	struct lpfc_hba *phba = vport->phba;
12765	struct lpfc_sli_ring *pring = NULL;
12766	struct lpfc_iocbq *iocbq;
12767	int errcnt = 0, ret_val = 0;
12768	unsigned long iflags;
12769	int i;
12770
12771	/* all I/Os are in process of being flushed */
12772	if (phba->hba_flag & HBA_IOQ_FLUSH)
12773	return errcnt;
12774
12775	for (i = 1; i <= phba->sli.last_iotag; i++) {
12776	iocbq = phba->sli.iocbq_lookup[i];
12777
12778	if (lpfc_sli_validate_fcp_iocb_for_abort(iocbq, vport))
12779	continue;
12780
12781	if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id,
12782	ctx_cmd: abort_cmd) != 0)
12783	continue;
12784
12785	spin_lock_irqsave(&phba->hbalock, iflags);
12786	if (phba->sli_rev == LPFC_SLI_REV3) {
12787	pring = &phba->sli.sli3_ring[LPFC_FCP_RING];
12788	} else if (phba->sli_rev == LPFC_SLI_REV4) {
12789	pring = lpfc_sli4_calc_ring(phba, piocb: iocbq);
12790	}
12791	ret_val = lpfc_sli_issue_abort_iotag(phba, pring, cmdiocb: iocbq,
12792	cmpl: lpfc_sli_abort_fcp_cmpl);
12793	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
12794	if (ret_val != IOCB_SUCCESS)
12795	errcnt++;
12796	}
12797
12798	return errcnt;
12799	}
12800
12801	/**
12802	* lpfc_sli_abort_taskmgmt - issue abort for all commands on a host/target/LUN
12803	* @vport: Pointer to virtual port.
12804	* @pring: Pointer to driver SLI ring object.
12805	* @tgt_id: SCSI ID of the target.
12806	* @lun_id: LUN ID of the scsi device.
12807	* @cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST.
12808	*
12809	* This function sends an abort command for every SCSI command
12810	* associated with the given virtual port pending on the ring
12811	* filtered by lpfc_sli_validate_fcp_iocb_for_abort and then
12812	* lpfc_sli_validate_fcp_iocb function. The ordering for validation before
12813	* submitting abort iocbs must be lpfc_sli_validate_fcp_iocb_for_abort
12814	* followed by lpfc_sli_validate_fcp_iocb.
12815	*
12816	* When taskmgmt_cmd == LPFC_CTX_LUN, the function sends abort only to the
12817	* FCP iocbs associated with lun specified by tgt_id and lun_id
12818	* parameters
12819	* When taskmgmt_cmd == LPFC_CTX_TGT, the function sends abort only to the
12820	* FCP iocbs associated with SCSI target specified by tgt_id parameter.
12821	* When taskmgmt_cmd == LPFC_CTX_HOST, the function sends abort to all
12822	* FCP iocbs associated with virtual port.
12823	* This function returns number of iocbs it aborted .
12824	* This function is called with no locks held right after a taskmgmt
12825	* command is sent.
12826	**/
12827	int
12828	lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring,
12829	uint16_t tgt_id, uint64_t lun_id, lpfc_ctx_cmd cmd)
12830	{
12831	struct lpfc_hba *phba = vport->phba;
12832	struct lpfc_io_buf *lpfc_cmd;
12833	struct lpfc_iocbq *abtsiocbq;
12834	struct lpfc_nodelist *ndlp = NULL;
12835	struct lpfc_iocbq *iocbq;
12836	int sum, i, ret_val;
12837	unsigned long iflags;
12838	struct lpfc_sli_ring *pring_s4 = NULL;
12839	u16 ulp_context, iotag, cqid = LPFC_WQE_CQ_ID_DEFAULT;
12840	bool ia;
12841
12842	spin_lock_irqsave(&phba->hbalock, iflags);
12843
12844	/* all I/Os are in process of being flushed */
12845	if (phba->hba_flag & HBA_IOQ_FLUSH) {
12846	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
12847	return 0;
12848	}
12849	sum = 0;
12850
12851	for (i = 1; i <= phba->sli.last_iotag; i++) {
12852	iocbq = phba->sli.iocbq_lookup[i];
12853
12854	if (lpfc_sli_validate_fcp_iocb_for_abort(iocbq, vport))
12855	continue;
12856
12857	if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id,
12858	ctx_cmd: cmd) != 0)
12859	continue;
12860
12861	/* Guard against IO completion being called at same time */
12862	lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq);
12863	spin_lock(lock: &lpfc_cmd->buf_lock);
12864
12865	if (!lpfc_cmd->pCmd) {
12866	spin_unlock(lock: &lpfc_cmd->buf_lock);
12867	continue;
12868	}
12869
12870	if (phba->sli_rev == LPFC_SLI_REV4) {
12871	pring_s4 =
12872	phba->sli4_hba.hdwq[iocbq->hba_wqidx].io_wq->pring;
12873	if (!pring_s4) {
12874	spin_unlock(lock: &lpfc_cmd->buf_lock);
12875	continue;
12876	}
12877	/* Note: both hbalock and ring_lock must be set here */
12878	spin_lock(lock: &pring_s4->ring_lock);
12879	}
12880
12881	/*
12882	* If the iocbq is already being aborted, don't take a second
12883	* action, but do count it.
12884	*/
12885	if ((iocbq->cmd_flag & LPFC_DRIVER_ABORTED) ||
12886	!(iocbq->cmd_flag & LPFC_IO_ON_TXCMPLQ)) {
12887	if (phba->sli_rev == LPFC_SLI_REV4)
12888	spin_unlock(lock: &pring_s4->ring_lock);
12889	spin_unlock(lock: &lpfc_cmd->buf_lock);
12890	continue;
12891	}
12892
12893	/* issue ABTS for this IOCB based on iotag */
12894	abtsiocbq = __lpfc_sli_get_iocbq(phba);
12895	if (!abtsiocbq) {
12896	if (phba->sli_rev == LPFC_SLI_REV4)
12897	spin_unlock(lock: &pring_s4->ring_lock);
12898	spin_unlock(lock: &lpfc_cmd->buf_lock);
12899	continue;
12900	}
12901
12902	if (phba->sli_rev == LPFC_SLI_REV4) {
12903	iotag = abtsiocbq->iotag;
12904	ulp_context = iocbq->sli4_xritag;
12905	cqid = lpfc_cmd->hdwq->io_cq_map;
12906	} else {
12907	iotag = iocbq->iocb.ulpIoTag;
12908	if (pring->ringno == LPFC_ELS_RING) {
12909	ndlp = iocbq->ndlp;
12910	ulp_context = ndlp->nlp_rpi;
12911	} else {
12912	ulp_context = iocbq->iocb.ulpContext;
12913	}
12914	}
12915
12916	ndlp = lpfc_cmd->rdata->pnode;
12917
12918	if (lpfc_is_link_up(phba) &&
12919	(ndlp && ndlp->nlp_state == NLP_STE_MAPPED_NODE) &&
12920	!(phba->link_flag & LS_EXTERNAL_LOOPBACK))
12921	ia = false;
12922	else
12923	ia = true;
12924
12925	lpfc_sli_prep_abort_xri(phba, cmdiocbq: abtsiocbq, ulp_context, iotag,
12926	ulp_class: iocbq->iocb.ulpClass, cqid,
12927	ia, wqec: false);
12928
12929	abtsiocbq->vport = vport;
12930
12931	/* ABTS WQE must go to the same WQ as the WQE to be aborted */
12932	abtsiocbq->hba_wqidx = iocbq->hba_wqidx;
12933	if (iocbq->cmd_flag & LPFC_IO_FCP)
12934	abtsiocbq->cmd_flag |= LPFC_USE_FCPWQIDX;
12935	if (iocbq->cmd_flag & LPFC_IO_FOF)
12936	abtsiocbq->cmd_flag |= LPFC_IO_FOF;
12937
12938	/* Setup callback routine and issue the command. */
12939	abtsiocbq->cmd_cmpl = lpfc_sli_abort_fcp_cmpl;
12940
12941	/*
12942	* Indicate the IO is being aborted by the driver and set
12943	* the caller's flag into the aborted IO.
12944	*/
12945	iocbq->cmd_flag |= LPFC_DRIVER_ABORTED;
12946
12947	if (phba->sli_rev == LPFC_SLI_REV4) {
12948	ret_val = __lpfc_sli_issue_iocb(phba, ring_number: pring_s4->ringno,
12949	piocb: abtsiocbq, flag: 0);
12950	spin_unlock(lock: &pring_s4->ring_lock);
12951	} else {
12952	ret_val = __lpfc_sli_issue_iocb(phba, ring_number: pring->ringno,
12953	piocb: abtsiocbq, flag: 0);
12954	}
12955
12956	spin_unlock(lock: &lpfc_cmd->buf_lock);
12957
12958	if (ret_val == IOCB_ERROR)
12959	__lpfc_sli_release_iocbq(phba, iocbq: abtsiocbq);
12960	else
12961	sum++;
12962	}
12963	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
12964	return sum;
12965	}
12966
12967	/**
12968	* lpfc_sli_wake_iocb_wait - lpfc_sli_issue_iocb_wait's completion handler
12969	* @phba: Pointer to HBA context object.
12970	* @cmdiocbq: Pointer to command iocb.
12971	* @rspiocbq: Pointer to response iocb.
12972	*
12973	* This function is the completion handler for iocbs issued using
12974	* lpfc_sli_issue_iocb_wait function. This function is called by the
12975	* ring event handler function without any lock held. This function
12976	* can be called from both worker thread context and interrupt
12977	* context. This function also can be called from other thread which
12978	* cleans up the SLI layer objects.
12979	* This function copy the contents of the response iocb to the
12980	* response iocb memory object provided by the caller of
12981	* lpfc_sli_issue_iocb_wait and then wakes up the thread which
12982	* sleeps for the iocb completion.
12983	**/
12984	static void
12985	lpfc_sli_wake_iocb_wait(struct lpfc_hba *phba,
12986	struct lpfc_iocbq *cmdiocbq,
12987	struct lpfc_iocbq *rspiocbq)
12988	{
12989	wait_queue_head_t *pdone_q;
12990	unsigned long iflags;
12991	struct lpfc_io_buf *lpfc_cmd;
12992	size_t offset = offsetof(struct lpfc_iocbq, wqe);
12993
12994	spin_lock_irqsave(&phba->hbalock, iflags);
12995	if (cmdiocbq->cmd_flag & LPFC_IO_WAKE_TMO) {
12996
12997	/*
12998	* A time out has occurred for the iocb. If a time out
12999	* completion handler has been supplied, call it. Otherwise,
13000	* just free the iocbq.
13001	*/
13002
13003	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
13004	cmdiocbq->cmd_cmpl = cmdiocbq->wait_cmd_cmpl;
13005	cmdiocbq->wait_cmd_cmpl = NULL;
13006	if (cmdiocbq->cmd_cmpl)
13007	cmdiocbq->cmd_cmpl(phba, cmdiocbq, NULL);
13008	else
13009	lpfc_sli_release_iocbq(phba, iocbq: cmdiocbq);
13010	return;
13011	}
13012
13013	/* Copy the contents of the local rspiocb into the caller's buffer. */
13014	cmdiocbq->cmd_flag |= LPFC_IO_WAKE;
13015	if (cmdiocbq->rsp_iocb && rspiocbq)
13016	memcpy((char *)cmdiocbq->rsp_iocb + offset,
13017	(char *)rspiocbq + offset, sizeof(*rspiocbq) - offset);
13018
13019	/* Set the exchange busy flag for task management commands */
13020	if ((cmdiocbq->cmd_flag & LPFC_IO_FCP) &&
13021	!(cmdiocbq->cmd_flag & LPFC_IO_LIBDFC)) {
13022	lpfc_cmd = container_of(cmdiocbq, struct lpfc_io_buf,
13023	cur_iocbq);
13024	if (rspiocbq && (rspiocbq->cmd_flag & LPFC_EXCHANGE_BUSY))
13025	lpfc_cmd->flags |= LPFC_SBUF_XBUSY;
13026	else
13027	lpfc_cmd->flags &= ~LPFC_SBUF_XBUSY;
13028	}
13029
13030	pdone_q = cmdiocbq->context_un.wait_queue;
13031	if (pdone_q)
13032	wake_up(pdone_q);
13033	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
13034	return;
13035	}
13036
13037	/**
13038	* lpfc_chk_iocb_flg - Test IOCB flag with lock held.
13039	* @phba: Pointer to HBA context object..
13040	* @piocbq: Pointer to command iocb.
13041	* @flag: Flag to test.
13042	*
13043	* This routine grabs the hbalock and then test the cmd_flag to
13044	* see if the passed in flag is set.
13045	* Returns:
13046	* 1 if flag is set.
13047	* 0 if flag is not set.
13048	**/
13049	static int
13050	lpfc_chk_iocb_flg(struct lpfc_hba *phba,
13051	struct lpfc_iocbq *piocbq, uint32_t flag)
13052	{
13053	unsigned long iflags;
13054	int ret;
13055
13056	spin_lock_irqsave(&phba->hbalock, iflags);
13057	ret = piocbq->cmd_flag & flag;
13058	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
13059	return ret;
13060
13061	}
13062
13063	/**
13064	* lpfc_sli_issue_iocb_wait - Synchronous function to issue iocb commands
13065	* @phba: Pointer to HBA context object..
13066	* @ring_number: Ring number
13067	* @piocb: Pointer to command iocb.
13068	* @prspiocbq: Pointer to response iocb.
13069	* @timeout: Timeout in number of seconds.
13070	*
13071	* This function issues the iocb to firmware and waits for the
13072	* iocb to complete. The cmd_cmpl field of the shall be used
13073	* to handle iocbs which time out. If the field is NULL, the
13074	* function shall free the iocbq structure. If more clean up is
13075	* needed, the caller is expected to provide a completion function
13076	* that will provide the needed clean up. If the iocb command is
13077	* not completed within timeout seconds, the function will either
13078	* free the iocbq structure (if cmd_cmpl == NULL) or execute the
13079	* completion function set in the cmd_cmpl field and then return
13080	* a status of IOCB_TIMEDOUT. The caller should not free the iocb
13081	* resources if this function returns IOCB_TIMEDOUT.
13082	* The function waits for the iocb completion using an
13083	* non-interruptible wait.
13084	* This function will sleep while waiting for iocb completion.
13085	* So, this function should not be called from any context which
13086	* does not allow sleeping. Due to the same reason, this function
13087	* cannot be called with interrupt disabled.
13088	* This function assumes that the iocb completions occur while
13089	* this function sleep. So, this function cannot be called from
13090	* the thread which process iocb completion for this ring.
13091	* This function clears the cmd_flag of the iocb object before
13092	* issuing the iocb and the iocb completion handler sets this
13093	* flag and wakes this thread when the iocb completes.
13094	* The contents of the response iocb will be copied to prspiocbq
13095	* by the completion handler when the command completes.
13096	* This function returns IOCB_SUCCESS when success.
13097	* This function is called with no lock held.
13098	**/
13099	int
13100	lpfc_sli_issue_iocb_wait(struct lpfc_hba *phba,
13101	uint32_t ring_number,
13102	struct lpfc_iocbq *piocb,
13103	struct lpfc_iocbq *prspiocbq,
13104	uint32_t timeout)
13105	{
13106	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_q);
13107	long timeleft, timeout_req = 0;
13108	int retval = IOCB_SUCCESS;
13109	uint32_t creg_val;
13110	struct lpfc_iocbq *iocb;
13111	int txq_cnt = 0;
13112	int txcmplq_cnt = 0;
13113	struct lpfc_sli_ring *pring;
13114	unsigned long iflags;
13115	bool iocb_completed = true;
13116
13117	if (phba->sli_rev >= LPFC_SLI_REV4) {
13118	lpfc_sli_prep_wqe(phba, job: piocb);
13119
13120	pring = lpfc_sli4_calc_ring(phba, piocb);
13121	} else
13122	pring = &phba->sli.sli3_ring[ring_number];
13123	/*
13124	* If the caller has provided a response iocbq buffer, then rsp_iocb
13125	* is NULL or its an error.
13126	*/
13127	if (prspiocbq) {
13128	if (piocb->rsp_iocb)
13129	return IOCB_ERROR;
13130	piocb->rsp_iocb = prspiocbq;
13131	}
13132
13133	piocb->wait_cmd_cmpl = piocb->cmd_cmpl;
13134	piocb->cmd_cmpl = lpfc_sli_wake_iocb_wait;
13135	piocb->context_un.wait_queue = &done_q;
13136	piocb->cmd_flag &= ~(LPFC_IO_WAKE | LPFC_IO_WAKE_TMO);
13137
13138	if (phba->cfg_poll & DISABLE_FCP_RING_INT) {
13139	if (lpfc_readl(addr: phba->HCregaddr, data: &creg_val))
13140	return IOCB_ERROR;
13141	creg_val |= (HC_R0INT_ENA << LPFC_FCP_RING);
13142	writel(val: creg_val, addr: phba->HCregaddr);
13143	readl(addr: phba->HCregaddr); /* flush */
13144	}
13145
13146	retval = lpfc_sli_issue_iocb(phba, ring_number, piocb,
13147	SLI_IOCB_RET_IOCB);
13148	if (retval == IOCB_SUCCESS) {
13149	timeout_req = msecs_to_jiffies(m: timeout * 1000);
13150	timeleft = wait_event_timeout(done_q,
13151	lpfc_chk_iocb_flg(phba, piocb, LPFC_IO_WAKE),
13152	timeout_req);
13153	spin_lock_irqsave(&phba->hbalock, iflags);
13154	if (!(piocb->cmd_flag & LPFC_IO_WAKE)) {
13155
13156	/*
13157	* IOCB timed out. Inform the wake iocb wait
13158	* completion function and set local status
13159	*/
13160
13161	iocb_completed = false;
13162	piocb->cmd_flag |= LPFC_IO_WAKE_TMO;
13163	}
13164	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
13165	if (iocb_completed) {
13166	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
13167	"0331 IOCB wake signaled\n");
13168	/* Note: we are not indicating if the IOCB has a success
13169	* status or not - that's for the caller to check.
13170	* IOCB_SUCCESS means just that the command was sent and
13171	* completed. Not that it completed successfully.
13172	* */
13173	} else if (timeleft == 0) {
13174	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13175	"0338 IOCB wait timeout error - no "
13176	"wake response Data x%x\n", timeout);
13177	retval = IOCB_TIMEDOUT;
13178	} else {
13179	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13180	"0330 IOCB wake NOT set, "
13181	"Data x%x x%lx\n",
13182	timeout, (timeleft / jiffies));
13183	retval = IOCB_TIMEDOUT;
13184	}
13185	} else if (retval == IOCB_BUSY) {
13186	if (phba->cfg_log_verbose & LOG_SLI) {
13187	list_for_each_entry(iocb, &pring->txq, list) {
13188	txq_cnt++;
13189	}
13190	list_for_each_entry(iocb, &pring->txcmplq, list) {
13191	txcmplq_cnt++;
13192	}
13193	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
13194	"2818 Max IOCBs %d txq cnt %d txcmplq cnt %d\n",
13195	phba->iocb_cnt, txq_cnt, txcmplq_cnt);
13196	}
13197	return retval;
13198	} else {
13199	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
13200	"0332 IOCB wait issue failed, Data x%x\n",
13201	retval);
13202	retval = IOCB_ERROR;
13203	}
13204
13205	if (phba->cfg_poll & DISABLE_FCP_RING_INT) {
13206	if (lpfc_readl(addr: phba->HCregaddr, data: &creg_val))
13207	return IOCB_ERROR;
13208	creg_val &= ~(HC_R0INT_ENA << LPFC_FCP_RING);
13209	writel(val: creg_val, addr: phba->HCregaddr);
13210	readl(addr: phba->HCregaddr); /* flush */
13211	}
13212
13213	if (prspiocbq)
13214	piocb->rsp_iocb = NULL;
13215
13216	piocb->context_un.wait_queue = NULL;
13217	piocb->cmd_cmpl = NULL;
13218	return retval;
13219	}
13220
13221	/**
13222	* lpfc_sli_issue_mbox_wait - Synchronous function to issue mailbox
13223	* @phba: Pointer to HBA context object.
13224	* @pmboxq: Pointer to driver mailbox object.
13225	* @timeout: Timeout in number of seconds.
13226	*
13227	* This function issues the mailbox to firmware and waits for the
13228	* mailbox command to complete. If the mailbox command is not
13229	* completed within timeout seconds, it returns MBX_TIMEOUT.
13230	* The function waits for the mailbox completion using an
13231	* interruptible wait. If the thread is woken up due to a
13232	* signal, MBX_TIMEOUT error is returned to the caller. Caller
13233	* should not free the mailbox resources, if this function returns
13234	* MBX_TIMEOUT.
13235	* This function will sleep while waiting for mailbox completion.
13236	* So, this function should not be called from any context which
13237	* does not allow sleeping. Due to the same reason, this function
13238	* cannot be called with interrupt disabled.
13239	* This function assumes that the mailbox completion occurs while
13240	* this function sleep. So, this function cannot be called from
13241	* the worker thread which processes mailbox completion.
13242	* This function is called in the context of HBA management
13243	* applications.
13244	* This function returns MBX_SUCCESS when successful.
13245	* This function is called with no lock held.
13246	**/
13247	int
13248	lpfc_sli_issue_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq,
13249	uint32_t timeout)
13250	{
13251	struct completion mbox_done;
13252	int retval;
13253	unsigned long flag;
13254
13255	pmboxq->mbox_flag &= ~LPFC_MBX_WAKE;
13256	/* setup wake call as IOCB callback */
13257	pmboxq->mbox_cmpl = lpfc_sli_wake_mbox_wait;
13258
13259	/* setup context3 field to pass wait_queue pointer to wake function */
13260	init_completion(x: &mbox_done);
13261	pmboxq->context3 = &mbox_done;
13262	/* now issue the command */
13263	retval = lpfc_sli_issue_mbox(phba, pmbox: pmboxq, MBX_NOWAIT);
13264	if (retval == MBX_BUSY || retval == MBX_SUCCESS) {
13265	wait_for_completion_timeout(x: &mbox_done,
13266	timeout: msecs_to_jiffies(m: timeout * 1000));
13267
13268	spin_lock_irqsave(&phba->hbalock, flag);
13269	pmboxq->context3 = NULL;
13270	/*
13271	* if LPFC_MBX_WAKE flag is set the mailbox is completed
13272	* else do not free the resources.
13273	*/
13274	if (pmboxq->mbox_flag & LPFC_MBX_WAKE) {
13275	retval = MBX_SUCCESS;
13276	} else {
13277	retval = MBX_TIMEOUT;
13278	pmboxq->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
13279	}
13280	spin_unlock_irqrestore(lock: &phba->hbalock, flags: flag);
13281	}
13282	return retval;
13283	}
13284
13285	/**
13286	* lpfc_sli_mbox_sys_shutdown - shutdown mailbox command sub-system
13287	* @phba: Pointer to HBA context.
13288	* @mbx_action: Mailbox shutdown options.
13289	*
13290	* This function is called to shutdown the driver's mailbox sub-system.
13291	* It first marks the mailbox sub-system is in a block state to prevent
13292	* the asynchronous mailbox command from issued off the pending mailbox
13293	* command queue. If the mailbox command sub-system shutdown is due to
13294	* HBA error conditions such as EEH or ERATT, this routine shall invoke
13295	* the mailbox sub-system flush routine to forcefully bring down the
13296	* mailbox sub-system. Otherwise, if it is due to normal condition (such
13297	* as with offline or HBA function reset), this routine will wait for the
13298	* outstanding mailbox command to complete before invoking the mailbox
13299	* sub-system flush routine to gracefully bring down mailbox sub-system.
13300	**/
13301	void
13302	lpfc_sli_mbox_sys_shutdown(struct lpfc_hba *phba, int mbx_action)
13303	{
13304	struct lpfc_sli *psli = &phba->sli;
13305	unsigned long timeout;
13306
13307	if (mbx_action == LPFC_MBX_NO_WAIT) {
13308	/* delay 100ms for port state */
13309	msleep(msecs: 100);
13310	lpfc_sli_mbox_sys_flush(phba);
13311	return;
13312	}
13313	timeout = msecs_to_jiffies(LPFC_MBOX_TMO * 1000) + jiffies;
13314
13315	/* Disable softirqs, including timers from obtaining phba->hbalock */
13316	local_bh_disable();
13317
13318	spin_lock_irq(lock: &phba->hbalock);
13319	psli->sli_flag |= LPFC_SLI_ASYNC_MBX_BLK;
13320
13321	if (psli->sli_flag & LPFC_SLI_ACTIVE) {
13322	/* Determine how long we might wait for the active mailbox
13323	* command to be gracefully completed by firmware.
13324	*/
13325	if (phba->sli.mbox_active)
13326	timeout = msecs_to_jiffies(m: lpfc_mbox_tmo_val(phba,
13327	phba->sli.mbox_active) *
13328	1000) + jiffies;
13329	spin_unlock_irq(lock: &phba->hbalock);
13330
13331	/* Enable softirqs again, done with phba->hbalock */
13332	local_bh_enable();
13333
13334	while (phba->sli.mbox_active) {
13335	/* Check active mailbox complete status every 2ms */
13336	msleep(msecs: 2);
13337	if (time_after(jiffies, timeout))
13338	/* Timeout, let the mailbox flush routine to
13339	* forcefully release active mailbox command
13340	*/
13341	break;
13342	}
13343	} else {
13344	spin_unlock_irq(lock: &phba->hbalock);
13345
13346	/* Enable softirqs again, done with phba->hbalock */
13347	local_bh_enable();
13348	}
13349
13350	lpfc_sli_mbox_sys_flush(phba);
13351	}
13352
13353	/**
13354	* lpfc_sli_eratt_read - read sli-3 error attention events
13355	* @phba: Pointer to HBA context.
13356	*
13357	* This function is called to read the SLI3 device error attention registers
13358	* for possible error attention events. The caller must hold the hostlock
13359	* with spin_lock_irq().
13360	*
13361	* This function returns 1 when there is Error Attention in the Host Attention
13362	* Register and returns 0 otherwise.
13363	**/
13364	static int
13365	lpfc_sli_eratt_read(struct lpfc_hba *phba)
13366	{
13367	uint32_t ha_copy;
13368
13369	/* Read chip Host Attention (HA) register */
13370	if (lpfc_readl(addr: phba->HAregaddr, data: &ha_copy))
13371	goto unplug_err;
13372
13373	if (ha_copy & HA_ERATT) {
13374	/* Read host status register to retrieve error event */
13375	if (lpfc_sli_read_hs(phba))
13376	goto unplug_err;
13377
13378	/* Check if there is a deferred error condition is active */
13379	if ((HS_FFER1 & phba->work_hs) &&
13380	((HS_FFER2 | HS_FFER3 | HS_FFER4 | HS_FFER5 |
13381	HS_FFER6 | HS_FFER7 | HS_FFER8) & phba->work_hs)) {
13382	phba->hba_flag |= DEFER_ERATT;
13383	/* Clear all interrupt enable conditions */
13384	writel(val: 0, addr: phba->HCregaddr);
13385	readl(addr: phba->HCregaddr);
13386	}
13387
13388	/* Set the driver HA work bitmap */
13389	phba->work_ha |= HA_ERATT;
13390	/* Indicate polling handles this ERATT */
13391	phba->hba_flag |= HBA_ERATT_HANDLED;
13392	return 1;
13393	}
13394	return 0;
13395
13396	unplug_err:
13397	/* Set the driver HS work bitmap */
13398	phba->work_hs |= UNPLUG_ERR;
13399	/* Set the driver HA work bitmap */
13400	phba->work_ha |= HA_ERATT;
13401	/* Indicate polling handles this ERATT */
13402	phba->hba_flag |= HBA_ERATT_HANDLED;
13403	return 1;
13404	}
13405
13406	/**
13407	* lpfc_sli4_eratt_read - read sli-4 error attention events
13408	* @phba: Pointer to HBA context.
13409	*
13410	* This function is called to read the SLI4 device error attention registers
13411	* for possible error attention events. The caller must hold the hostlock
13412	* with spin_lock_irq().
13413	*
13414	* This function returns 1 when there is Error Attention in the Host Attention
13415	* Register and returns 0 otherwise.
13416	**/
13417	static int
13418	lpfc_sli4_eratt_read(struct lpfc_hba *phba)
13419	{
13420	uint32_t uerr_sta_hi, uerr_sta_lo;
13421	uint32_t if_type, portsmphr;
13422	struct lpfc_register portstat_reg;
13423	u32 logmask;
13424
13425	/*
13426	* For now, use the SLI4 device internal unrecoverable error
13427	* registers for error attention. This can be changed later.
13428	*/
13429	if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf);
13430	switch (if_type) {
13431	case LPFC_SLI_INTF_IF_TYPE_0:
13432	if (lpfc_readl(addr: phba->sli4_hba.u.if_type0.UERRLOregaddr,
13433	data: &uerr_sta_lo) ||
13434	lpfc_readl(addr: phba->sli4_hba.u.if_type0.UERRHIregaddr,
13435	data: &uerr_sta_hi)) {
13436	phba->work_hs |= UNPLUG_ERR;
13437	phba->work_ha |= HA_ERATT;
13438	phba->hba_flag |= HBA_ERATT_HANDLED;
13439	return 1;
13440	}
13441	if ((~phba->sli4_hba.ue_mask_lo & uerr_sta_lo) ||
13442	(~phba->sli4_hba.ue_mask_hi & uerr_sta_hi)) {
13443	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13444	"1423 HBA Unrecoverable error: "
13445	"uerr_lo_reg=0x%x, uerr_hi_reg=0x%x, "
13446	"ue_mask_lo_reg=0x%x, "
13447	"ue_mask_hi_reg=0x%x\n",
13448	uerr_sta_lo, uerr_sta_hi,
13449	phba->sli4_hba.ue_mask_lo,
13450	phba->sli4_hba.ue_mask_hi);
13451	phba->work_status[0] = uerr_sta_lo;
13452	phba->work_status[1] = uerr_sta_hi;
13453	phba->work_ha |= HA_ERATT;
13454	phba->hba_flag |= HBA_ERATT_HANDLED;
13455	return 1;
13456	}
13457	break;
13458	case LPFC_SLI_INTF_IF_TYPE_2:
13459	case LPFC_SLI_INTF_IF_TYPE_6:
13460	if (lpfc_readl(addr: phba->sli4_hba.u.if_type2.STATUSregaddr,
13461	data: &portstat_reg.word0) ||
13462	lpfc_readl(addr: phba->sli4_hba.PSMPHRregaddr,
13463	data: &portsmphr)){
13464	phba->work_hs |= UNPLUG_ERR;
13465	phba->work_ha |= HA_ERATT;
13466	phba->hba_flag |= HBA_ERATT_HANDLED;
13467	return 1;
13468	}
13469	if (bf_get(lpfc_sliport_status_err, &portstat_reg)) {
13470	phba->work_status[0] =
13471	readl(addr: phba->sli4_hba.u.if_type2.ERR1regaddr);
13472	phba->work_status[1] =
13473	readl(addr: phba->sli4_hba.u.if_type2.ERR2regaddr);
13474	logmask = LOG_TRACE_EVENT;
13475	if (phba->work_status[0] ==
13476	SLIPORT_ERR1_REG_ERR_CODE_2 &&
13477	phba->work_status[1] == SLIPORT_ERR2_REG_FW_RESTART)
13478	logmask = LOG_SLI;
13479	lpfc_printf_log(phba, KERN_ERR, logmask,
13480	"2885 Port Status Event: "
13481	"port status reg 0x%x, "
13482	"port smphr reg 0x%x, "
13483	"error 1=0x%x, error 2=0x%x\n",
13484	portstat_reg.word0,
13485	portsmphr,
13486	phba->work_status[0],
13487	phba->work_status[1]);
13488	phba->work_ha |= HA_ERATT;
13489	phba->hba_flag |= HBA_ERATT_HANDLED;
13490	return 1;
13491	}
13492	break;
13493	case LPFC_SLI_INTF_IF_TYPE_1:
13494	default:
13495	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13496	"2886 HBA Error Attention on unsupported "
13497	"if type %d.", if_type);
13498	return 1;
13499	}
13500
13501	return 0;
13502	}
13503
13504	/**
13505	* lpfc_sli_check_eratt - check error attention events
13506	* @phba: Pointer to HBA context.
13507	*
13508	* This function is called from timer soft interrupt context to check HBA's
13509	* error attention register bit for error attention events.
13510	*
13511	* This function returns 1 when there is Error Attention in the Host Attention
13512	* Register and returns 0 otherwise.
13513	**/
13514	int
13515	lpfc_sli_check_eratt(struct lpfc_hba *phba)
13516	{
13517	uint32_t ha_copy;
13518
13519	/* If somebody is waiting to handle an eratt, don't process it
13520	* here. The brdkill function will do this.
13521	*/
13522	if (phba->link_flag & LS_IGNORE_ERATT)
13523	return 0;
13524
13525	/* Check if interrupt handler handles this ERATT */
13526	spin_lock_irq(lock: &phba->hbalock);
13527	if (phba->hba_flag & HBA_ERATT_HANDLED) {
13528	/* Interrupt handler has handled ERATT */
13529	spin_unlock_irq(lock: &phba->hbalock);
13530	return 0;
13531	}
13532
13533	/*
13534	* If there is deferred error attention, do not check for error
13535	* attention
13536	*/
13537	if (unlikely(phba->hba_flag & DEFER_ERATT)) {
13538	spin_unlock_irq(lock: &phba->hbalock);
13539	return 0;
13540	}
13541
13542	/* If PCI channel is offline, don't process it */
13543	if (unlikely(pci_channel_offline(phba->pcidev))) {
13544	spin_unlock_irq(lock: &phba->hbalock);
13545	return 0;
13546	}
13547
13548	switch (phba->sli_rev) {
13549	case LPFC_SLI_REV2:
13550	case LPFC_SLI_REV3:
13551	/* Read chip Host Attention (HA) register */
13552	ha_copy = lpfc_sli_eratt_read(phba);
13553	break;
13554	case LPFC_SLI_REV4:
13555	/* Read device Uncoverable Error (UERR) registers */
13556	ha_copy = lpfc_sli4_eratt_read(phba);
13557	break;
13558	default:
13559	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13560	"0299 Invalid SLI revision (%d)\n",
13561	phba->sli_rev);
13562	ha_copy = 0;
13563	break;
13564	}
13565	spin_unlock_irq(lock: &phba->hbalock);
13566
13567	return ha_copy;
13568	}
13569
13570	/**
13571	* lpfc_intr_state_check - Check device state for interrupt handling
13572	* @phba: Pointer to HBA context.
13573	*
13574	* This inline routine checks whether a device or its PCI slot is in a state
13575	* that the interrupt should be handled.
13576	*
13577	* This function returns 0 if the device or the PCI slot is in a state that
13578	* interrupt should be handled, otherwise -EIO.
13579	*/
13580	static inline int
13581	lpfc_intr_state_check(struct lpfc_hba *phba)
13582	{
13583	/* If the pci channel is offline, ignore all the interrupts */
13584	if (unlikely(pci_channel_offline(phba->pcidev)))
13585	return -EIO;
13586
13587	/* Update device level interrupt statistics */
13588	phba->sli.slistat.sli_intr++;
13589
13590	/* Ignore all interrupts during initialization. */
13591	if (unlikely(phba->link_state < LPFC_LINK_DOWN))
13592	return -EIO;
13593
13594	return 0;
13595	}
13596
13597	/**
13598	* lpfc_sli_sp_intr_handler - Slow-path interrupt handler to SLI-3 device
13599	* @irq: Interrupt number.
13600	* @dev_id: The device context pointer.
13601	*
13602	* This function is directly called from the PCI layer as an interrupt
13603	* service routine when device with SLI-3 interface spec is enabled with
13604	* MSI-X multi-message interrupt mode and there are slow-path events in
13605	* the HBA. However, when the device is enabled with either MSI or Pin-IRQ
13606	* interrupt mode, this function is called as part of the device-level
13607	* interrupt handler. When the PCI slot is in error recovery or the HBA
13608	* is undergoing initialization, the interrupt handler will not process
13609	* the interrupt. The link attention and ELS ring attention events are
13610	* handled by the worker thread. The interrupt handler signals the worker
13611	* thread and returns for these events. This function is called without
13612	* any lock held. It gets the hbalock to access and update SLI data
13613	* structures.
13614	*
13615	* This function returns IRQ_HANDLED when interrupt is handled else it
13616	* returns IRQ_NONE.
13617	**/
13618	irqreturn_t
13619	lpfc_sli_sp_intr_handler(int irq, void *dev_id)
13620	{
13621	struct lpfc_hba *phba;
13622	uint32_t ha_copy, hc_copy;
13623	uint32_t work_ha_copy;
13624	unsigned long status;
13625	unsigned long iflag;
13626	uint32_t control;
13627
13628	MAILBOX_t *mbox, *pmbox;
13629	struct lpfc_vport *vport;
13630	struct lpfc_nodelist *ndlp;
13631	struct lpfc_dmabuf *mp;
13632	LPFC_MBOXQ_t *pmb;
13633	int rc;
13634
13635	/*
13636	* Get the driver's phba structure from the dev_id and
13637	* assume the HBA is not interrupting.
13638	*/
13639	phba = (struct lpfc_hba *)dev_id;
13640
13641	if (unlikely(!phba))
13642	return IRQ_NONE;
13643
13644	/*
13645	* Stuff needs to be attented to when this function is invoked as an
13646	* individual interrupt handler in MSI-X multi-message interrupt mode
13647	*/
13648	if (phba->intr_type == MSIX) {
13649	/* Check device state for handling interrupt */
13650	if (lpfc_intr_state_check(phba))
13651	return IRQ_NONE;
13652	/* Need to read HA REG for slow-path events */
13653	spin_lock_irqsave(&phba->hbalock, iflag);
13654	if (lpfc_readl(addr: phba->HAregaddr, data: &ha_copy))
13655	goto unplug_error;
13656	/* If somebody is waiting to handle an eratt don't process it
13657	* here. The brdkill function will do this.
13658	*/
13659	if (phba->link_flag & LS_IGNORE_ERATT)
13660	ha_copy &= ~HA_ERATT;
13661	/* Check the need for handling ERATT in interrupt handler */
13662	if (ha_copy & HA_ERATT) {
13663	if (phba->hba_flag & HBA_ERATT_HANDLED)
13664	/* ERATT polling has handled ERATT */
13665	ha_copy &= ~HA_ERATT;
13666	else
13667	/* Indicate interrupt handler handles ERATT */
13668	phba->hba_flag |= HBA_ERATT_HANDLED;
13669	}
13670
13671	/*
13672	* If there is deferred error attention, do not check for any
13673	* interrupt.
13674	*/
13675	if (unlikely(phba->hba_flag & DEFER_ERATT)) {
13676	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
13677	return IRQ_NONE;
13678	}
13679
13680	/* Clear up only attention source related to slow-path */
13681	if (lpfc_readl(addr: phba->HCregaddr, data: &hc_copy))
13682	goto unplug_error;
13683
13684	writel(val: hc_copy & ~(HC_MBINT_ENA | HC_R2INT_ENA |
13685	HC_LAINT_ENA | HC_ERINT_ENA),
13686	addr: phba->HCregaddr);
13687	writel(val: (ha_copy & (HA_MBATT | HA_R2_CLR_MSK)),
13688	addr: phba->HAregaddr);
13689	writel(val: hc_copy, addr: phba->HCregaddr);
13690	readl(addr: phba->HAregaddr); /* flush */
13691	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
13692	} else
13693	ha_copy = phba->ha_copy;
13694
13695	work_ha_copy = ha_copy & phba->work_ha_mask;
13696
13697	if (work_ha_copy) {
13698	if (work_ha_copy & HA_LATT) {
13699	if (phba->sli.sli_flag & LPFC_PROCESS_LA) {
13700	/*
13701	* Turn off Link Attention interrupts
13702	* until CLEAR_LA done
13703	*/
13704	spin_lock_irqsave(&phba->hbalock, iflag);
13705	phba->sli.sli_flag &= ~LPFC_PROCESS_LA;
13706	if (lpfc_readl(addr: phba->HCregaddr, data: &control))
13707	goto unplug_error;
13708	control &= ~HC_LAINT_ENA;
13709	writel(val: control, addr: phba->HCregaddr);
13710	readl(addr: phba->HCregaddr); /* flush */
13711	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
13712	}
13713	else
13714	work_ha_copy &= ~HA_LATT;
13715	}
13716
13717	if (work_ha_copy & ~(HA_ERATT | HA_MBATT | HA_LATT)) {
13718	/*
13719	* Turn off Slow Rings interrupts, LPFC_ELS_RING is
13720	* the only slow ring.
13721	*/
13722	status = (work_ha_copy &
13723	(HA_RXMASK << (4*LPFC_ELS_RING)));
13724	status >>= (4*LPFC_ELS_RING);
13725	if (status & HA_RXMASK) {
13726	spin_lock_irqsave(&phba->hbalock, iflag);
13727	if (lpfc_readl(addr: phba->HCregaddr, data: &control))
13728	goto unplug_error;
13729
13730	lpfc_debugfs_slow_ring_trc(phba,
13731	"ISR slow ring: ctl:x%x stat:x%x isrcnt:x%x",
13732	control, status,
13733	(uint32_t)phba->sli.slistat.sli_intr);
13734
13735	if (control & (HC_R0INT_ENA << LPFC_ELS_RING)) {
13736	lpfc_debugfs_slow_ring_trc(phba,
13737	"ISR Disable ring:"
13738	"pwork:x%x hawork:x%x wait:x%x",
13739	phba->work_ha, work_ha_copy,
13740	(uint32_t)((unsigned long)
13741	&phba->work_waitq));
13742
13743	control &=
13744	~(HC_R0INT_ENA << LPFC_ELS_RING);
13745	writel(val: control, addr: phba->HCregaddr);
13746	readl(addr: phba->HCregaddr); /* flush */
13747	}
13748	else {
13749	lpfc_debugfs_slow_ring_trc(phba,
13750	"ISR slow ring: pwork:"
13751	"x%x hawork:x%x wait:x%x",
13752	phba->work_ha, work_ha_copy,
13753	(uint32_t)((unsigned long)
13754	&phba->work_waitq));
13755	}
13756	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
13757	}
13758	}
13759	spin_lock_irqsave(&phba->hbalock, iflag);
13760	if (work_ha_copy & HA_ERATT) {
13761	if (lpfc_sli_read_hs(phba))
13762	goto unplug_error;
13763	/*
13764	* Check if there is a deferred error condition
13765	* is active
13766	*/
13767	if ((HS_FFER1 & phba->work_hs) &&
13768	((HS_FFER2 | HS_FFER3 | HS_FFER4 | HS_FFER5 |
13769	HS_FFER6 | HS_FFER7 | HS_FFER8) &
13770	phba->work_hs)) {
13771	phba->hba_flag |= DEFER_ERATT;
13772	/* Clear all interrupt enable conditions */
13773	writel(val: 0, addr: phba->HCregaddr);
13774	readl(addr: phba->HCregaddr);
13775	}
13776	}
13777
13778	if ((work_ha_copy & HA_MBATT) && (phba->sli.mbox_active)) {
13779	pmb = phba->sli.mbox_active;
13780	pmbox = &pmb->u.mb;
13781	mbox = phba->mbox;
13782	vport = pmb->vport;
13783
13784	/* First check out the status word */
13785	lpfc_sli_pcimem_bcopy(srcp: mbox, destp: pmbox, cnt: sizeof(uint32_t));
13786	if (pmbox->mbxOwner != OWN_HOST) {
13787	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
13788	/*
13789	* Stray Mailbox Interrupt, mbxCommand <cmd>
13790	* mbxStatus <status>
13791	*/
13792	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13793	"(%d):0304 Stray Mailbox "
13794	"Interrupt mbxCommand x%x "
13795	"mbxStatus x%x\n",
13796	(vport ? vport->vpi : 0),
13797	pmbox->mbxCommand,
13798	pmbox->mbxStatus);
13799	/* clear mailbox attention bit */
13800	work_ha_copy &= ~HA_MBATT;
13801	} else {
13802	phba->sli.mbox_active = NULL;
13803	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
13804	phba->last_completion_time = jiffies;
13805	del_timer(timer: &phba->sli.mbox_tmo);
13806	if (pmb->mbox_cmpl) {
13807	lpfc_sli_pcimem_bcopy(srcp: mbox, destp: pmbox,
13808	MAILBOX_CMD_SIZE);
13809	if (pmb->out_ext_byte_len &&
13810	pmb->ctx_buf)
13811	lpfc_sli_pcimem_bcopy(
13812	srcp: phba->mbox_ext,
13813	destp: pmb->ctx_buf,
13814	cnt: pmb->out_ext_byte_len);
13815	}
13816	if (pmb->mbox_flag & LPFC_MBX_IMED_UNREG) {
13817	pmb->mbox_flag &= ~LPFC_MBX_IMED_UNREG;
13818
13819	lpfc_debugfs_disc_trc(vport,
13820	LPFC_DISC_TRC_MBOX_VPORT,
13821	"MBOX dflt rpi: : "
13822	"status:x%x rpi:x%x",
13823	(uint32_t)pmbox->mbxStatus,
13824	pmbox->un.varWords[0], 0);
13825
13826	if (!pmbox->mbxStatus) {
13827	mp = (struct lpfc_dmabuf *)
13828	(pmb->ctx_buf);
13829	ndlp = (struct lpfc_nodelist *)
13830	pmb->ctx_ndlp;
13831
13832	/* Reg_LOGIN of dflt RPI was
13833	* successful. new lets get
13834	* rid of the RPI using the
13835	* same mbox buffer.
13836	*/
13837	lpfc_unreg_login(phba,
13838	vport->vpi,
13839	pmbox->un.varWords[0],
13840	pmb);
13841	pmb->mbox_cmpl =
13842	lpfc_mbx_cmpl_dflt_rpi;
13843	pmb->ctx_buf = mp;
13844	pmb->ctx_ndlp = ndlp;
13845	pmb->vport = vport;
13846	rc = lpfc_sli_issue_mbox(phba,
13847	pmbox: pmb,
13848	MBX_NOWAIT);
13849	if (rc != MBX_BUSY)
13850	lpfc_printf_log(phba,
13851	KERN_ERR,
13852	LOG_TRACE_EVENT,
13853	"0350 rc should have"
13854	"been MBX_BUSY\n");
13855	if (rc != MBX_NOT_FINISHED)
13856	goto send_current_mbox;
13857	}
13858	}
13859	spin_lock_irqsave(
13860	&phba->pport->work_port_lock,
13861	iflag);
13862	phba->pport->work_port_events &=
13863	~WORKER_MBOX_TMO;
13864	spin_unlock_irqrestore(
13865	lock: &phba->pport->work_port_lock,
13866	flags: iflag);
13867
13868	/* Do NOT queue MBX_HEARTBEAT to the worker
13869	* thread for processing.
13870	*/
13871	if (pmbox->mbxCommand == MBX_HEARTBEAT) {
13872	/* Process mbox now */
13873	phba->sli.mbox_active = NULL;
13874	phba->sli.sli_flag &=
13875	~LPFC_SLI_MBOX_ACTIVE;
13876	if (pmb->mbox_cmpl)
13877	pmb->mbox_cmpl(phba, pmb);
13878	} else {
13879	/* Queue to worker thread to process */
13880	lpfc_mbox_cmpl_put(phba, pmb);
13881	}
13882	}
13883	} else
13884	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
13885
13886	if ((work_ha_copy & HA_MBATT) &&
13887	(phba->sli.mbox_active == NULL)) {
13888	send_current_mbox:
13889	/* Process next mailbox command if there is one */
13890	do {
13891	rc = lpfc_sli_issue_mbox(phba, NULL,
13892	MBX_NOWAIT);
13893	} while (rc == MBX_NOT_FINISHED);
13894	if (rc != MBX_SUCCESS)
13895	lpfc_printf_log(phba, KERN_ERR,
13896	LOG_TRACE_EVENT,
13897	"0349 rc should be "
13898	"MBX_SUCCESS\n");
13899	}
13900
13901	spin_lock_irqsave(&phba->hbalock, iflag);
13902	phba->work_ha |= work_ha_copy;
13903	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
13904	lpfc_worker_wake_up(phba);
13905	}
13906	return IRQ_HANDLED;
13907	unplug_error:
13908	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
13909	return IRQ_HANDLED;
13910
13911	} /* lpfc_sli_sp_intr_handler */
13912
13913	/**
13914	* lpfc_sli_fp_intr_handler - Fast-path interrupt handler to SLI-3 device.
13915	* @irq: Interrupt number.
13916	* @dev_id: The device context pointer.
13917	*
13918	* This function is directly called from the PCI layer as an interrupt
13919	* service routine when device with SLI-3 interface spec is enabled with
13920	* MSI-X multi-message interrupt mode and there is a fast-path FCP IOCB
13921	* ring event in the HBA. However, when the device is enabled with either
13922	* MSI or Pin-IRQ interrupt mode, this function is called as part of the
13923	* device-level interrupt handler. When the PCI slot is in error recovery
13924	* or the HBA is undergoing initialization, the interrupt handler will not
13925	* process the interrupt. The SCSI FCP fast-path ring event are handled in
13926	* the intrrupt context. This function is called without any lock held.
13927	* It gets the hbalock to access and update SLI data structures.
13928	*
13929	* This function returns IRQ_HANDLED when interrupt is handled else it
13930	* returns IRQ_NONE.
13931	**/
13932	irqreturn_t
13933	lpfc_sli_fp_intr_handler(int irq, void *dev_id)
13934	{
13935	struct lpfc_hba *phba;
13936	uint32_t ha_copy;
13937	unsigned long status;
13938	unsigned long iflag;
13939	struct lpfc_sli_ring *pring;
13940
13941	/* Get the driver's phba structure from the dev_id and
13942	* assume the HBA is not interrupting.
13943	*/
13944	phba = (struct lpfc_hba *) dev_id;
13945
13946	if (unlikely(!phba))
13947	return IRQ_NONE;
13948
13949	/*
13950	* Stuff needs to be attented to when this function is invoked as an
13951	* individual interrupt handler in MSI-X multi-message interrupt mode
13952	*/
13953	if (phba->intr_type == MSIX) {
13954	/* Check device state for handling interrupt */
13955	if (lpfc_intr_state_check(phba))
13956	return IRQ_NONE;
13957	/* Need to read HA REG for FCP ring and other ring events */
13958	if (lpfc_readl(addr: phba->HAregaddr, data: &ha_copy))
13959	return IRQ_HANDLED;
13960	/* Clear up only attention source related to fast-path */
13961	spin_lock_irqsave(&phba->hbalock, iflag);
13962	/*
13963	* If there is deferred error attention, do not check for
13964	* any interrupt.
13965	*/
13966	if (unlikely(phba->hba_flag & DEFER_ERATT)) {
13967	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
13968	return IRQ_NONE;
13969	}
13970	writel(val: (ha_copy & (HA_R0_CLR_MSK | HA_R1_CLR_MSK)),
13971	addr: phba->HAregaddr);
13972	readl(addr: phba->HAregaddr); /* flush */
13973	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
13974	} else
13975	ha_copy = phba->ha_copy;
13976
13977	/*
13978	* Process all events on FCP ring. Take the optimized path for FCP IO.
13979	*/
13980	ha_copy &= ~(phba->work_ha_mask);
13981
13982	status = (ha_copy & (HA_RXMASK << (4*LPFC_FCP_RING)));
13983	status >>= (4*LPFC_FCP_RING);
13984	pring = &phba->sli.sli3_ring[LPFC_FCP_RING];
13985	if (status & HA_RXMASK)
13986	lpfc_sli_handle_fast_ring_event(phba, pring, mask: status);
13987
13988	if (phba->cfg_multi_ring_support == 2) {
13989	/*
13990	* Process all events on extra ring. Take the optimized path
13991	* for extra ring IO.
13992	*/
13993	status = (ha_copy & (HA_RXMASK << (4*LPFC_EXTRA_RING)));
13994	status >>= (4*LPFC_EXTRA_RING);
13995	if (status & HA_RXMASK) {
13996	lpfc_sli_handle_fast_ring_event(phba,
13997	pring: &phba->sli.sli3_ring[LPFC_EXTRA_RING],
13998	mask: status);
13999	}
14000	}
14001	return IRQ_HANDLED;
14002	} /* lpfc_sli_fp_intr_handler */
14003
14004	/**
14005	* lpfc_sli_intr_handler - Device-level interrupt handler to SLI-3 device
14006	* @irq: Interrupt number.
14007	* @dev_id: The device context pointer.
14008	*
14009	* This function is the HBA device-level interrupt handler to device with
14010	* SLI-3 interface spec, called from the PCI layer when either MSI or
14011	* Pin-IRQ interrupt mode is enabled and there is an event in the HBA which
14012	* requires driver attention. This function invokes the slow-path interrupt
14013	* attention handling function and fast-path interrupt attention handling
14014	* function in turn to process the relevant HBA attention events. This
14015	* function is called without any lock held. It gets the hbalock to access
14016	* and update SLI data structures.
14017	*
14018	* This function returns IRQ_HANDLED when interrupt is handled, else it
14019	* returns IRQ_NONE.
14020	**/
14021	irqreturn_t
14022	lpfc_sli_intr_handler(int irq, void *dev_id)
14023	{
14024	struct lpfc_hba *phba;
14025	irqreturn_t sp_irq_rc, fp_irq_rc;
14026	unsigned long status1, status2;
14027	uint32_t hc_copy;
14028
14029	/*
14030	* Get the driver's phba structure from the dev_id and
14031	* assume the HBA is not interrupting.
14032	*/
14033	phba = (struct lpfc_hba *) dev_id;
14034
14035	if (unlikely(!phba))
14036	return IRQ_NONE;
14037
14038	/* Check device state for handling interrupt */
14039	if (lpfc_intr_state_check(phba))
14040	return IRQ_NONE;
14041
14042	spin_lock(lock: &phba->hbalock);
14043	if (lpfc_readl(addr: phba->HAregaddr, data: &phba->ha_copy)) {
14044	spin_unlock(lock: &phba->hbalock);
14045	return IRQ_HANDLED;
14046	}
14047
14048	if (unlikely(!phba->ha_copy)) {
14049	spin_unlock(lock: &phba->hbalock);
14050	return IRQ_NONE;
14051	} else if (phba->ha_copy & HA_ERATT) {
14052	if (phba->hba_flag & HBA_ERATT_HANDLED)
14053	/* ERATT polling has handled ERATT */
14054	phba->ha_copy &= ~HA_ERATT;
14055	else
14056	/* Indicate interrupt handler handles ERATT */
14057	phba->hba_flag |= HBA_ERATT_HANDLED;
14058	}
14059
14060	/*
14061	* If there is deferred error attention, do not check for any interrupt.
14062	*/
14063	if (unlikely(phba->hba_flag & DEFER_ERATT)) {
14064	spin_unlock(lock: &phba->hbalock);
14065	return IRQ_NONE;
14066	}
14067
14068	/* Clear attention sources except link and error attentions */
14069	if (lpfc_readl(addr: phba->HCregaddr, data: &hc_copy)) {
14070	spin_unlock(lock: &phba->hbalock);
14071	return IRQ_HANDLED;
14072	}
14073	writel(val: hc_copy & ~(HC_MBINT_ENA | HC_R0INT_ENA | HC_R1INT_ENA
14074	| HC_R2INT_ENA | HC_LAINT_ENA | HC_ERINT_ENA),
14075	addr: phba->HCregaddr);
14076	writel(val: (phba->ha_copy & ~(HA_LATT | HA_ERATT)), addr: phba->HAregaddr);
14077	writel(val: hc_copy, addr: phba->HCregaddr);
14078	readl(addr: phba->HAregaddr); /* flush */
14079	spin_unlock(lock: &phba->hbalock);
14080
14081	/*
14082	* Invokes slow-path host attention interrupt handling as appropriate.
14083	*/
14084
14085	/* status of events with mailbox and link attention */
14086	status1 = phba->ha_copy & (HA_MBATT | HA_LATT | HA_ERATT);
14087
14088	/* status of events with ELS ring */
14089	status2 = (phba->ha_copy & (HA_RXMASK << (4*LPFC_ELS_RING)));
14090	status2 >>= (4*LPFC_ELS_RING);
14091
14092	if (status1 || (status2 & HA_RXMASK))
14093	sp_irq_rc = lpfc_sli_sp_intr_handler(irq, dev_id);
14094	else
14095	sp_irq_rc = IRQ_NONE;
14096
14097	/*
14098	* Invoke fast-path host attention interrupt handling as appropriate.
14099	*/
14100
14101	/* status of events with FCP ring */
14102	status1 = (phba->ha_copy & (HA_RXMASK << (4*LPFC_FCP_RING)));
14103	status1 >>= (4*LPFC_FCP_RING);
14104
14105	/* status of events with extra ring */
14106	if (phba->cfg_multi_ring_support == 2) {
14107	status2 = (phba->ha_copy & (HA_RXMASK << (4*LPFC_EXTRA_RING)));
14108	status2 >>= (4*LPFC_EXTRA_RING);
14109	} else
14110	status2 = 0;
14111
14112	if ((status1 & HA_RXMASK) || (status2 & HA_RXMASK))
14113	fp_irq_rc = lpfc_sli_fp_intr_handler(irq, dev_id);
14114	else
14115	fp_irq_rc = IRQ_NONE;
14116
14117	/* Return device-level interrupt handling status */
14118	return (sp_irq_rc == IRQ_HANDLED) ? sp_irq_rc : fp_irq_rc;
14119	} /* lpfc_sli_intr_handler */
14120
14121	/**
14122	* lpfc_sli4_els_xri_abort_event_proc - Process els xri abort event
14123	* @phba: pointer to lpfc hba data structure.
14124	*
14125	* This routine is invoked by the worker thread to process all the pending
14126	* SLI4 els abort xri events.
14127	**/
14128	void lpfc_sli4_els_xri_abort_event_proc(struct lpfc_hba *phba)
14129	{
14130	struct lpfc_cq_event *cq_event;
14131	unsigned long iflags;
14132
14133	/* First, declare the els xri abort event has been handled */
14134	spin_lock_irqsave(&phba->hbalock, iflags);
14135	phba->hba_flag &= ~ELS_XRI_ABORT_EVENT;
14136	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14137
14138	/* Now, handle all the els xri abort events */
14139	spin_lock_irqsave(&phba->sli4_hba.els_xri_abrt_list_lock, iflags);
14140	while (!list_empty(head: &phba->sli4_hba.sp_els_xri_aborted_work_queue)) {
14141	/* Get the first event from the head of the event queue */
14142	list_remove_head(&phba->sli4_hba.sp_els_xri_aborted_work_queue,
14143	cq_event, struct lpfc_cq_event, list);
14144	spin_unlock_irqrestore(lock: &phba->sli4_hba.els_xri_abrt_list_lock,
14145	flags: iflags);
14146	/* Notify aborted XRI for ELS work queue */
14147	lpfc_sli4_els_xri_aborted(phba, &cq_event->cqe.wcqe_axri);
14148
14149	/* Free the event processed back to the free pool */
14150	lpfc_sli4_cq_event_release(phba, cq_event);
14151	spin_lock_irqsave(&phba->sli4_hba.els_xri_abrt_list_lock,
14152	iflags);
14153	}
14154	spin_unlock_irqrestore(lock: &phba->sli4_hba.els_xri_abrt_list_lock, flags: iflags);
14155	}
14156
14157	/**
14158	* lpfc_sli4_els_preprocess_rspiocbq - Get response iocbq from els wcqe
14159	* @phba: Pointer to HBA context object.
14160	* @irspiocbq: Pointer to work-queue completion queue entry.
14161	*
14162	* This routine handles an ELS work-queue completion event and construct
14163	* a pseudo response ELS IOCBQ from the SLI4 ELS WCQE for the common
14164	* discovery engine to handle.
14165	*
14166	* Return: Pointer to the receive IOCBQ, NULL otherwise.
14167	**/
14168	static struct lpfc_iocbq *
14169	lpfc_sli4_els_preprocess_rspiocbq(struct lpfc_hba *phba,
14170	struct lpfc_iocbq *irspiocbq)
14171	{
14172	struct lpfc_sli_ring *pring;
14173	struct lpfc_iocbq *cmdiocbq;
14174	struct lpfc_wcqe_complete *wcqe;
14175	unsigned long iflags;
14176
14177	pring = lpfc_phba_elsring(phba);
14178	if (unlikely(!pring))
14179	return NULL;
14180
14181	wcqe = &irspiocbq->cq_event.cqe.wcqe_cmpl;
14182	spin_lock_irqsave(&pring->ring_lock, iflags);
14183	pring->stats.iocb_event++;
14184	/* Look up the ELS command IOCB and create pseudo response IOCB */
14185	cmdiocbq = lpfc_sli_iocbq_lookup_by_tag(phba, pring,
14186	bf_get(lpfc_wcqe_c_request_tag, wcqe));
14187	if (unlikely(!cmdiocbq)) {
14188	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
14189	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
14190	"0386 ELS complete with no corresponding "
14191	"cmdiocb: 0x%x 0x%x 0x%x 0x%x\n",
14192	wcqe->word0, wcqe->total_data_placed,
14193	wcqe->parameter, wcqe->word3);
14194	lpfc_sli_release_iocbq(phba, iocbq: irspiocbq);
14195	return NULL;
14196	}
14197
14198	memcpy(&irspiocbq->wqe, &cmdiocbq->wqe, sizeof(union lpfc_wqe128));
14199	memcpy(&irspiocbq->wcqe_cmpl, wcqe, sizeof(*wcqe));
14200
14201	/* Put the iocb back on the txcmplq */
14202	lpfc_sli_ringtxcmpl_put(phba, pring, piocb: cmdiocbq);
14203	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
14204
14205	if (bf_get(lpfc_wcqe_c_xb, wcqe)) {
14206	spin_lock_irqsave(&phba->hbalock, iflags);
14207	irspiocbq->cmd_flag |= LPFC_EXCHANGE_BUSY;
14208	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14209	}
14210
14211	return irspiocbq;
14212	}
14213
14214	inline struct lpfc_cq_event *
14215	lpfc_cq_event_setup(struct lpfc_hba *phba, void *entry, int size)
14216	{
14217	struct lpfc_cq_event *cq_event;
14218
14219	/* Allocate a new internal CQ_EVENT entry */
14220	cq_event = lpfc_sli4_cq_event_alloc(phba);
14221	if (!cq_event) {
14222	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14223	"0602 Failed to alloc CQ_EVENT entry\n");
14224	return NULL;
14225	}
14226
14227	/* Move the CQE into the event */
14228	memcpy(&cq_event->cqe, entry, size);
14229	return cq_event;
14230	}
14231
14232	/**
14233	* lpfc_sli4_sp_handle_async_event - Handle an asynchronous event
14234	* @phba: Pointer to HBA context object.
14235	* @mcqe: Pointer to mailbox completion queue entry.
14236	*
14237	* This routine process a mailbox completion queue entry with asynchronous
14238	* event.
14239	*
14240	* Return: true if work posted to worker thread, otherwise false.
14241	**/
14242	static bool
14243	lpfc_sli4_sp_handle_async_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe)
14244	{
14245	struct lpfc_cq_event *cq_event;
14246	unsigned long iflags;
14247
14248	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
14249	"0392 Async Event: word0:x%x, word1:x%x, "
14250	"word2:x%x, word3:x%x\n", mcqe->word0,
14251	mcqe->mcqe_tag0, mcqe->mcqe_tag1, mcqe->trailer);
14252
14253	cq_event = lpfc_cq_event_setup(phba, entry: mcqe, size: sizeof(struct lpfc_mcqe));
14254	if (!cq_event)
14255	return false;
14256
14257	spin_lock_irqsave(&phba->sli4_hba.asynce_list_lock, iflags);
14258	list_add_tail(new: &cq_event->list, head: &phba->sli4_hba.sp_asynce_work_queue);
14259	spin_unlock_irqrestore(lock: &phba->sli4_hba.asynce_list_lock, flags: iflags);
14260
14261	/* Set the async event flag */
14262	spin_lock_irqsave(&phba->hbalock, iflags);
14263	phba->hba_flag |= ASYNC_EVENT;
14264	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14265
14266	return true;
14267	}
14268
14269	/**
14270	* lpfc_sli4_sp_handle_mbox_event - Handle a mailbox completion event
14271	* @phba: Pointer to HBA context object.
14272	* @mcqe: Pointer to mailbox completion queue entry.
14273	*
14274	* This routine process a mailbox completion queue entry with mailbox
14275	* completion event.
14276	*
14277	* Return: true if work posted to worker thread, otherwise false.
14278	**/
14279	static bool
14280	lpfc_sli4_sp_handle_mbox_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe)
14281	{
14282	uint32_t mcqe_status;
14283	MAILBOX_t *mbox, *pmbox;
14284	struct lpfc_mqe *mqe;
14285	struct lpfc_vport *vport;
14286	struct lpfc_nodelist *ndlp;
14287	struct lpfc_dmabuf *mp;
14288	unsigned long iflags;
14289	LPFC_MBOXQ_t *pmb;
14290	bool workposted = false;
14291	int rc;
14292
14293	/* If not a mailbox complete MCQE, out by checking mailbox consume */
14294	if (!bf_get(lpfc_trailer_completed, mcqe))
14295	goto out_no_mqe_complete;
14296
14297	/* Get the reference to the active mbox command */
14298	spin_lock_irqsave(&phba->hbalock, iflags);
14299	pmb = phba->sli.mbox_active;
14300	if (unlikely(!pmb)) {
14301	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14302	"1832 No pending MBOX command to handle\n");
14303	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14304	goto out_no_mqe_complete;
14305	}
14306	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14307	mqe = &pmb->u.mqe;
14308	pmbox = (MAILBOX_t *)&pmb->u.mqe;
14309	mbox = phba->mbox;
14310	vport = pmb->vport;
14311
14312	/* Reset heartbeat timer */
14313	phba->last_completion_time = jiffies;
14314	del_timer(timer: &phba->sli.mbox_tmo);
14315
14316	/* Move mbox data to caller's mailbox region, do endian swapping */
14317	if (pmb->mbox_cmpl && mbox)
14318	lpfc_sli4_pcimem_bcopy(srcp: mbox, destp: mqe, cnt: sizeof(struct lpfc_mqe));
14319
14320	/*
14321	* For mcqe errors, conditionally move a modified error code to
14322	* the mbox so that the error will not be missed.
14323	*/
14324	mcqe_status = bf_get(lpfc_mcqe_status, mcqe);
14325	if (mcqe_status != MB_CQE_STATUS_SUCCESS) {
14326	if (bf_get(lpfc_mqe_status, mqe) == MBX_SUCCESS)
14327	bf_set(lpfc_mqe_status, mqe,
14328	(LPFC_MBX_ERROR_RANGE | mcqe_status));
14329	}
14330	if (pmb->mbox_flag & LPFC_MBX_IMED_UNREG) {
14331	pmb->mbox_flag &= ~LPFC_MBX_IMED_UNREG;
14332	lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_MBOX_VPORT,
14333	"MBOX dflt rpi: status:x%x rpi:x%x",
14334	mcqe_status,
14335	pmbox->un.varWords[0], 0);
14336	if (mcqe_status == MB_CQE_STATUS_SUCCESS) {
14337	mp = (struct lpfc_dmabuf *)(pmb->ctx_buf);
14338	ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp;
14339
14340	/* Reg_LOGIN of dflt RPI was successful. Mark the
14341	* node as having an UNREG_LOGIN in progress to stop
14342	* an unsolicited PLOGI from the same NPortId from
14343	* starting another mailbox transaction.
14344	*/
14345	spin_lock_irqsave(&ndlp->lock, iflags);
14346	ndlp->nlp_flag |= NLP_UNREG_INP;
14347	spin_unlock_irqrestore(lock: &ndlp->lock, flags: iflags);
14348	lpfc_unreg_login(phba, vport->vpi,
14349	pmbox->un.varWords[0], pmb);
14350	pmb->mbox_cmpl = lpfc_mbx_cmpl_dflt_rpi;
14351	pmb->ctx_buf = mp;
14352
14353	/* No reference taken here. This is a default
14354	* RPI reg/immediate unreg cycle. The reference was
14355	* taken in the reg rpi path and is released when
14356	* this mailbox completes.
14357	*/
14358	pmb->ctx_ndlp = ndlp;
14359	pmb->vport = vport;
14360	rc = lpfc_sli_issue_mbox(phba, pmbox: pmb, MBX_NOWAIT);
14361	if (rc != MBX_BUSY)
14362	lpfc_printf_log(phba, KERN_ERR,
14363	LOG_TRACE_EVENT,
14364	"0385 rc should "
14365	"have been MBX_BUSY\n");
14366	if (rc != MBX_NOT_FINISHED)
14367	goto send_current_mbox;
14368	}
14369	}
14370	spin_lock_irqsave(&phba->pport->work_port_lock, iflags);
14371	phba->pport->work_port_events &= ~WORKER_MBOX_TMO;
14372	spin_unlock_irqrestore(lock: &phba->pport->work_port_lock, flags: iflags);
14373
14374	/* Do NOT queue MBX_HEARTBEAT to the worker thread for processing. */
14375	if (pmbox->mbxCommand == MBX_HEARTBEAT) {
14376	spin_lock_irqsave(&phba->hbalock, iflags);
14377	/* Release the mailbox command posting token */
14378	phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
14379	phba->sli.mbox_active = NULL;
14380	if (bf_get(lpfc_trailer_consumed, mcqe))
14381	lpfc_sli4_mq_release(q: phba->sli4_hba.mbx_wq);
14382	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14383
14384	/* Post the next mbox command, if there is one */
14385	lpfc_sli4_post_async_mbox(phba);
14386
14387	/* Process cmpl now */
14388	if (pmb->mbox_cmpl)
14389	pmb->mbox_cmpl(phba, pmb);
14390	return false;
14391	}
14392
14393	/* There is mailbox completion work to queue to the worker thread */
14394	spin_lock_irqsave(&phba->hbalock, iflags);
14395	__lpfc_mbox_cmpl_put(phba, pmb);
14396	phba->work_ha |= HA_MBATT;
14397	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14398	workposted = true;
14399
14400	send_current_mbox:
14401	spin_lock_irqsave(&phba->hbalock, iflags);
14402	/* Release the mailbox command posting token */
14403	phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
14404	/* Setting active mailbox pointer need to be in sync to flag clear */
14405	phba->sli.mbox_active = NULL;
14406	if (bf_get(lpfc_trailer_consumed, mcqe))
14407	lpfc_sli4_mq_release(q: phba->sli4_hba.mbx_wq);
14408	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14409	/* Wake up worker thread to post the next pending mailbox command */
14410	lpfc_worker_wake_up(phba);
14411	return workposted;
14412
14413	out_no_mqe_complete:
14414	spin_lock_irqsave(&phba->hbalock, iflags);
14415	if (bf_get(lpfc_trailer_consumed, mcqe))
14416	lpfc_sli4_mq_release(q: phba->sli4_hba.mbx_wq);
14417	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14418	return false;
14419	}
14420
14421	/**
14422	* lpfc_sli4_sp_handle_mcqe - Process a mailbox completion queue entry
14423	* @phba: Pointer to HBA context object.
14424	* @cq: Pointer to associated CQ
14425	* @cqe: Pointer to mailbox completion queue entry.
14426	*
14427	* This routine process a mailbox completion queue entry, it invokes the
14428	* proper mailbox complete handling or asynchronous event handling routine
14429	* according to the MCQE's async bit.
14430	*
14431	* Return: true if work posted to worker thread, otherwise false.
14432	**/
14433	static bool
14434	lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
14435	struct lpfc_cqe *cqe)
14436	{
14437	struct lpfc_mcqe mcqe;
14438	bool workposted;
14439
14440	cq->CQ_mbox++;
14441
14442	/* Copy the mailbox MCQE and convert endian order as needed */
14443	lpfc_sli4_pcimem_bcopy(srcp: cqe, destp: &mcqe, cnt: sizeof(struct lpfc_mcqe));
14444
14445	/* Invoke the proper event handling routine */
14446	if (!bf_get(lpfc_trailer_async, &mcqe))
14447	workposted = lpfc_sli4_sp_handle_mbox_event(phba, mcqe: &mcqe);
14448	else
14449	workposted = lpfc_sli4_sp_handle_async_event(phba, mcqe: &mcqe);
14450	return workposted;
14451	}
14452
14453	/**
14454	* lpfc_sli4_sp_handle_els_wcqe - Handle els work-queue completion event
14455	* @phba: Pointer to HBA context object.
14456	* @cq: Pointer to associated CQ
14457	* @wcqe: Pointer to work-queue completion queue entry.
14458	*
14459	* This routine handles an ELS work-queue completion event.
14460	*
14461	* Return: true if work posted to worker thread, otherwise false.
14462	**/
14463	static bool
14464	lpfc_sli4_sp_handle_els_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
14465	struct lpfc_wcqe_complete *wcqe)
14466	{
14467	struct lpfc_iocbq *irspiocbq;
14468	unsigned long iflags;
14469	struct lpfc_sli_ring *pring = cq->pring;
14470	int txq_cnt = 0;
14471	int txcmplq_cnt = 0;
14472
14473	/* Check for response status */
14474	if (unlikely(bf_get(lpfc_wcqe_c_status, wcqe))) {
14475	/* Log the error status */
14476	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
14477	"0357 ELS CQE error: status=x%x: "
14478	"CQE: %08x %08x %08x %08x\n",
14479	bf_get(lpfc_wcqe_c_status, wcqe),
14480	wcqe->word0, wcqe->total_data_placed,
14481	wcqe->parameter, wcqe->word3);
14482	}
14483
14484	/* Get an irspiocbq for later ELS response processing use */
14485	irspiocbq = lpfc_sli_get_iocbq(phba);
14486	if (!irspiocbq) {
14487	if (!list_empty(head: &pring->txq))
14488	txq_cnt++;
14489	if (!list_empty(head: &pring->txcmplq))
14490	txcmplq_cnt++;
14491	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14492	"0387 NO IOCBQ data: txq_cnt=%d iocb_cnt=%d "
14493	"els_txcmplq_cnt=%d\n",
14494	txq_cnt, phba->iocb_cnt,
14495	txcmplq_cnt);
14496	return false;
14497	}
14498
14499	/* Save off the slow-path queue event for work thread to process */
14500	memcpy(&irspiocbq->cq_event.cqe.wcqe_cmpl, wcqe, sizeof(*wcqe));
14501	spin_lock_irqsave(&phba->hbalock, iflags);
14502	list_add_tail(new: &irspiocbq->cq_event.list,
14503	head: &phba->sli4_hba.sp_queue_event);
14504	phba->hba_flag |= HBA_SP_QUEUE_EVT;
14505	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14506
14507	return true;
14508	}
14509
14510	/**
14511	* lpfc_sli4_sp_handle_rel_wcqe - Handle slow-path WQ entry consumed event
14512	* @phba: Pointer to HBA context object.
14513	* @wcqe: Pointer to work-queue completion queue entry.
14514	*
14515	* This routine handles slow-path WQ entry consumed event by invoking the
14516	* proper WQ release routine to the slow-path WQ.
14517	**/
14518	static void
14519	lpfc_sli4_sp_handle_rel_wcqe(struct lpfc_hba *phba,
14520	struct lpfc_wcqe_release *wcqe)
14521	{
14522	/* sanity check on queue memory */
14523	if (unlikely(!phba->sli4_hba.els_wq))
14524	return;
14525	/* Check for the slow-path ELS work queue */
14526	if (bf_get(lpfc_wcqe_r_wq_id, wcqe) == phba->sli4_hba.els_wq->queue_id)
14527	lpfc_sli4_wq_release(q: phba->sli4_hba.els_wq,
14528	bf_get(lpfc_wcqe_r_wqe_index, wcqe));
14529	else
14530	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
14531	"2579 Slow-path wqe consume event carries "
14532	"miss-matched qid: wcqe-qid=x%x, sp-qid=x%x\n",
14533	bf_get(lpfc_wcqe_r_wqe_index, wcqe),
14534	phba->sli4_hba.els_wq->queue_id);
14535	}
14536
14537	/**
14538	* lpfc_sli4_sp_handle_abort_xri_wcqe - Handle a xri abort event
14539	* @phba: Pointer to HBA context object.
14540	* @cq: Pointer to a WQ completion queue.
14541	* @wcqe: Pointer to work-queue completion queue entry.
14542	*
14543	* This routine handles an XRI abort event.
14544	*
14545	* Return: true if work posted to worker thread, otherwise false.
14546	**/
14547	static bool
14548	lpfc_sli4_sp_handle_abort_xri_wcqe(struct lpfc_hba *phba,
14549	struct lpfc_queue *cq,
14550	struct sli4_wcqe_xri_aborted *wcqe)
14551	{
14552	bool workposted = false;
14553	struct lpfc_cq_event *cq_event;
14554	unsigned long iflags;
14555
14556	switch (cq->subtype) {
14557	case LPFC_IO:
14558	lpfc_sli4_io_xri_aborted(phba, axri: wcqe, idx: cq->hdwq);
14559	if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
14560	/* Notify aborted XRI for NVME work queue */
14561	if (phba->nvmet_support)
14562	lpfc_sli4_nvmet_xri_aborted(phba, axri: wcqe);
14563	}
14564	workposted = false;
14565	break;
14566	case LPFC_NVME_LS: /* NVME LS uses ELS resources */
14567	case LPFC_ELS:
14568	cq_event = lpfc_cq_event_setup(phba, entry: wcqe, size: sizeof(*wcqe));
14569	if (!cq_event) {
14570	workposted = false;
14571	break;
14572	}
14573	cq_event->hdwq = cq->hdwq;
14574	spin_lock_irqsave(&phba->sli4_hba.els_xri_abrt_list_lock,
14575	iflags);
14576	list_add_tail(new: &cq_event->list,
14577	head: &phba->sli4_hba.sp_els_xri_aborted_work_queue);
14578	/* Set the els xri abort event flag */
14579	phba->hba_flag |= ELS_XRI_ABORT_EVENT;
14580	spin_unlock_irqrestore(lock: &phba->sli4_hba.els_xri_abrt_list_lock,
14581	flags: iflags);
14582	workposted = true;
14583	break;
14584	default:
14585	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14586	"0603 Invalid CQ subtype %d: "
14587	"%08x %08x %08x %08x\n",
14588	cq->subtype, wcqe->word0, wcqe->parameter,
14589	wcqe->word2, wcqe->word3);
14590	workposted = false;
14591	break;
14592	}
14593	return workposted;
14594	}
14595
14596	#define FC_RCTL_MDS_DIAGS 0xF4
14597
14598	/**
14599	* lpfc_sli4_sp_handle_rcqe - Process a receive-queue completion queue entry
14600	* @phba: Pointer to HBA context object.
14601	* @rcqe: Pointer to receive-queue completion queue entry.
14602	*
14603	* This routine process a receive-queue completion queue entry.
14604	*
14605	* Return: true if work posted to worker thread, otherwise false.
14606	**/
14607	static bool
14608	lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe)
14609	{
14610	bool workposted = false;
14611	struct fc_frame_header *fc_hdr;
14612	struct lpfc_queue *hrq = phba->sli4_hba.hdr_rq;
14613	struct lpfc_queue *drq = phba->sli4_hba.dat_rq;
14614	struct lpfc_nvmet_tgtport *tgtp;
14615	struct hbq_dmabuf *dma_buf;
14616	uint32_t status, rq_id;
14617	unsigned long iflags;
14618
14619	/* sanity check on queue memory */
14620	if (unlikely(!hrq) || unlikely(!drq))
14621	return workposted;
14622
14623	if (bf_get(lpfc_cqe_code, rcqe) == CQE_CODE_RECEIVE_V1)
14624	rq_id = bf_get(lpfc_rcqe_rq_id_v1, rcqe);
14625	else
14626	rq_id = bf_get(lpfc_rcqe_rq_id, rcqe);
14627	if (rq_id != hrq->queue_id)
14628	goto out;
14629
14630	status = bf_get(lpfc_rcqe_status, rcqe);
14631	switch (status) {
14632	case FC_STATUS_RQ_BUF_LEN_EXCEEDED:
14633	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14634	"2537 Receive Frame Truncated!!\n");
14635	fallthrough;
14636	case FC_STATUS_RQ_SUCCESS:
14637	spin_lock_irqsave(&phba->hbalock, iflags);
14638	lpfc_sli4_rq_release(hq: hrq, dq: drq);
14639	dma_buf = lpfc_sli_hbqbuf_get(rb_list: &phba->hbqs[0].hbq_buffer_list);
14640	if (!dma_buf) {
14641	hrq->RQ_no_buf_found++;
14642	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14643	goto out;
14644	}
14645	hrq->RQ_rcv_buf++;
14646	hrq->RQ_buf_posted--;
14647	memcpy(&dma_buf->cq_event.cqe.rcqe_cmpl, rcqe, sizeof(*rcqe));
14648
14649	fc_hdr = (struct fc_frame_header *)dma_buf->hbuf.virt;
14650
14651	if (fc_hdr->fh_r_ctl == FC_RCTL_MDS_DIAGS ||
14652	fc_hdr->fh_r_ctl == FC_RCTL_DD_UNSOL_DATA) {
14653	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14654	/* Handle MDS Loopback frames */
14655	if (!(phba->pport->load_flag & FC_UNLOADING))
14656	lpfc_sli4_handle_mds_loopback(vport: phba->pport,
14657	dmabuf: dma_buf);
14658	else
14659	lpfc_in_buf_free(phba, &dma_buf->dbuf);
14660	break;
14661	}
14662
14663	/* save off the frame for the work thread to process */
14664	list_add_tail(new: &dma_buf->cq_event.list,
14665	head: &phba->sli4_hba.sp_queue_event);
14666	/* Frame received */
14667	phba->hba_flag |= HBA_SP_QUEUE_EVT;
14668	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14669	workposted = true;
14670	break;
14671	case FC_STATUS_INSUFF_BUF_FRM_DISC:
14672	if (phba->nvmet_support) {
14673	tgtp = phba->targetport->private;
14674	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14675	"6402 RQE Error x%x, posted %d err_cnt "
14676	"%d: %x %x %x\n",
14677	status, hrq->RQ_buf_posted,
14678	hrq->RQ_no_posted_buf,
14679	atomic_read(&tgtp->rcv_fcp_cmd_in),
14680	atomic_read(&tgtp->rcv_fcp_cmd_out),
14681	atomic_read(&tgtp->xmt_fcp_release));
14682	}
14683	fallthrough;
14684
14685	case FC_STATUS_INSUFF_BUF_NEED_BUF:
14686	hrq->RQ_no_posted_buf++;
14687	/* Post more buffers if possible */
14688	spin_lock_irqsave(&phba->hbalock, iflags);
14689	phba->hba_flag |= HBA_POST_RECEIVE_BUFFER;
14690	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14691	workposted = true;
14692	break;
14693	case FC_STATUS_RQ_DMA_FAILURE:
14694	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14695	"2564 RQE DMA Error x%x, x%08x x%08x x%08x "
14696	"x%08x\n",
14697	status, rcqe->word0, rcqe->word1,
14698	rcqe->word2, rcqe->word3);
14699
14700	/* If IV set, no further recovery */
14701	if (bf_get(lpfc_rcqe_iv, rcqe))
14702	break;
14703
14704	/* recycle consumed resource */
14705	spin_lock_irqsave(&phba->hbalock, iflags);
14706	lpfc_sli4_rq_release(hq: hrq, dq: drq);
14707	dma_buf = lpfc_sli_hbqbuf_get(rb_list: &phba->hbqs[0].hbq_buffer_list);
14708	if (!dma_buf) {
14709	hrq->RQ_no_buf_found++;
14710	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14711	break;
14712	}
14713	hrq->RQ_rcv_buf++;
14714	hrq->RQ_buf_posted--;
14715	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
14716	lpfc_in_buf_free(phba, &dma_buf->dbuf);
14717	break;
14718	default:
14719	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14720	"2565 Unexpected RQE Status x%x, w0-3 x%08x "
14721	"x%08x x%08x x%08x\n",
14722	status, rcqe->word0, rcqe->word1,
14723	rcqe->word2, rcqe->word3);
14724	break;
14725	}
14726	out:
14727	return workposted;
14728	}
14729
14730	/**
14731	* lpfc_sli4_sp_handle_cqe - Process a slow path completion queue entry
14732	* @phba: Pointer to HBA context object.
14733	* @cq: Pointer to the completion queue.
14734	* @cqe: Pointer to a completion queue entry.
14735	*
14736	* This routine process a slow-path work-queue or receive queue completion queue
14737	* entry.
14738	*
14739	* Return: true if work posted to worker thread, otherwise false.
14740	**/
14741	static bool
14742	lpfc_sli4_sp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
14743	struct lpfc_cqe *cqe)
14744	{
14745	struct lpfc_cqe cqevt;
14746	bool workposted = false;
14747
14748	/* Copy the work queue CQE and convert endian order if needed */
14749	lpfc_sli4_pcimem_bcopy(srcp: cqe, destp: &cqevt, cnt: sizeof(struct lpfc_cqe));
14750
14751	/* Check and process for different type of WCQE and dispatch */
14752	switch (bf_get(lpfc_cqe_code, &cqevt)) {
14753	case CQE_CODE_COMPL_WQE:
14754	/* Process the WQ/RQ complete event */
14755	phba->last_completion_time = jiffies;
14756	workposted = lpfc_sli4_sp_handle_els_wcqe(phba, cq,
14757	wcqe: (struct lpfc_wcqe_complete *)&cqevt);
14758	break;
14759	case CQE_CODE_RELEASE_WQE:
14760	/* Process the WQ release event */
14761	lpfc_sli4_sp_handle_rel_wcqe(phba,
14762	wcqe: (struct lpfc_wcqe_release *)&cqevt);
14763	break;
14764	case CQE_CODE_XRI_ABORTED:
14765	/* Process the WQ XRI abort event */
14766	phba->last_completion_time = jiffies;
14767	workposted = lpfc_sli4_sp_handle_abort_xri_wcqe(phba, cq,
14768	wcqe: (struct sli4_wcqe_xri_aborted *)&cqevt);
14769	break;
14770	case CQE_CODE_RECEIVE:
14771	case CQE_CODE_RECEIVE_V1:
14772	/* Process the RQ event */
14773	phba->last_completion_time = jiffies;
14774	workposted = lpfc_sli4_sp_handle_rcqe(phba,
14775	rcqe: (struct lpfc_rcqe *)&cqevt);
14776	break;
14777	default:
14778	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14779	"0388 Not a valid WCQE code: x%x\n",
14780	bf_get(lpfc_cqe_code, &cqevt));
14781	break;
14782	}
14783	return workposted;
14784	}
14785
14786	/**
14787	* lpfc_sli4_sp_handle_eqe - Process a slow-path event queue entry
14788	* @phba: Pointer to HBA context object.
14789	* @eqe: Pointer to fast-path event queue entry.
14790	* @speq: Pointer to slow-path event queue.
14791	*
14792	* This routine process a event queue entry from the slow-path event queue.
14793	* It will check the MajorCode and MinorCode to determine this is for a
14794	* completion event on a completion queue, if not, an error shall be logged
14795	* and just return. Otherwise, it will get to the corresponding completion
14796	* queue and process all the entries on that completion queue, rearm the
14797	* completion queue, and then return.
14798	*
14799	**/
14800	static void
14801	lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
14802	struct lpfc_queue *speq)
14803	{
14804	struct lpfc_queue *cq = NULL, *childq;
14805	uint16_t cqid;
14806	int ret = 0;
14807
14808	/* Get the reference to the corresponding CQ */
14809	cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);
14810
14811	list_for_each_entry(childq, &speq->child_list, list) {
14812	if (childq->queue_id == cqid) {
14813	cq = childq;
14814	break;
14815	}
14816	}
14817	if (unlikely(!cq)) {
14818	if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)
14819	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14820	"0365 Slow-path CQ identifier "
14821	"(%d) does not exist\n", cqid);
14822	return;
14823	}
14824
14825	/* Save EQ associated with this CQ */
14826	cq->assoc_qp = speq;
14827
14828	if (is_kdump_kernel())
14829	ret = queue_work(wq: phba->wq, work: &cq->spwork);
14830	else
14831	ret = queue_work_on(cpu: cq->chann, wq: phba->wq, work: &cq->spwork);
14832
14833	if (!ret)
14834	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14835	"0390 Cannot schedule queue work "
14836	"for CQ eqcqid=%d, cqid=%d on CPU %d\n",
14837	cqid, cq->queue_id, raw_smp_processor_id());
14838	}
14839
14840	/**
14841	* __lpfc_sli4_process_cq - Process elements of a CQ
14842	* @phba: Pointer to HBA context object.
14843	* @cq: Pointer to CQ to be processed
14844	* @handler: Routine to process each cqe
14845	* @delay: Pointer to usdelay to set in case of rescheduling of the handler
14846	*
14847	* This routine processes completion queue entries in a CQ. While a valid
14848	* queue element is found, the handler is called. During processing checks
14849	* are made for periodic doorbell writes to let the hardware know of
14850	* element consumption.
14851	*
14852	* If the max limit on cqes to process is hit, or there are no more valid
14853	* entries, the loop stops. If we processed a sufficient number of elements,
14854	* meaning there is sufficient load, rather than rearming and generating
14855	* another interrupt, a cq rescheduling delay will be set. A delay of 0
14856	* indicates no rescheduling.
14857	*
14858	* Returns True if work scheduled, False otherwise.
14859	**/
14860	static bool
14861	__lpfc_sli4_process_cq(struct lpfc_hba *phba, struct lpfc_queue *cq,
14862	bool (*handler)(struct lpfc_hba *, struct lpfc_queue *,
14863	struct lpfc_cqe *), unsigned long *delay)
14864	{
14865	struct lpfc_cqe *cqe;
14866	bool workposted = false;
14867	int count = 0, consumed = 0;
14868	bool arm = true;
14869
14870	/* default - no reschedule */
14871	*delay = 0;
14872
14873	if (cmpxchg(&cq->queue_claimed, 0, 1) != 0)
14874	goto rearm_and_exit;
14875
14876	/* Process all the entries to the CQ */
14877	cq->q_flag = 0;
14878	cqe = lpfc_sli4_cq_get(q: cq);
14879	while (cqe) {
14880	workposted |= handler(phba, cq, cqe);
14881	__lpfc_sli4_consume_cqe(phba, cq, cqe);
14882
14883	consumed++;
14884	if (!(++count % cq->max_proc_limit))
14885	break;
14886
14887	if (!(count % cq->notify_interval)) {
14888	phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed,
14889	LPFC_QUEUE_NOARM);
14890	consumed = 0;
14891	cq->assoc_qp->q_flag |= HBA_EQ_DELAY_CHK;
14892	}
14893
14894	if (count == LPFC_NVMET_CQ_NOTIFY)
14895	cq->q_flag |= HBA_NVMET_CQ_NOTIFY;
14896
14897	cqe = lpfc_sli4_cq_get(q: cq);
14898	}
14899	if (count >= phba->cfg_cq_poll_threshold) {
14900	*delay = 1;
14901	arm = false;
14902	}
14903
14904	/* Track the max number of CQEs processed in 1 EQ */
14905	if (count > cq->CQ_max_cqe)
14906	cq->CQ_max_cqe = count;
14907
14908	cq->assoc_qp->EQ_cqe_cnt += count;
14909
14910	/* Catch the no cq entry condition */
14911	if (unlikely(count == 0))
14912	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
14913	"0369 No entry from completion queue "
14914	"qid=%d\n", cq->queue_id);
14915
14916	xchg(&cq->queue_claimed, 0);
14917
14918	rearm_and_exit:
14919	phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed,
14920	arm ? LPFC_QUEUE_REARM : LPFC_QUEUE_NOARM);
14921
14922	return workposted;
14923	}
14924
14925	/**
14926	* __lpfc_sli4_sp_process_cq - Process a slow-path event queue entry
14927	* @cq: pointer to CQ to process
14928	*
14929	* This routine calls the cq processing routine with a handler specific
14930	* to the type of queue bound to it.
14931	*
14932	* The CQ routine returns two values: the first is the calling status,
14933	* which indicates whether work was queued to the background discovery
14934	* thread. If true, the routine should wakeup the discovery thread;
14935	* the second is the delay parameter. If non-zero, rather than rearming
14936	* the CQ and yet another interrupt, the CQ handler should be queued so
14937	* that it is processed in a subsequent polling action. The value of
14938	* the delay indicates when to reschedule it.
14939	**/
14940	static void
14941	__lpfc_sli4_sp_process_cq(struct lpfc_queue *cq)
14942	{
14943	struct lpfc_hba *phba = cq->phba;
14944	unsigned long delay;
14945	bool workposted = false;
14946	int ret = 0;
14947
14948	/* Process and rearm the CQ */
14949	switch (cq->type) {
14950	case LPFC_MCQ:
14951	workposted |= __lpfc_sli4_process_cq(phba, cq,
14952	handler: lpfc_sli4_sp_handle_mcqe,
14953	delay: &delay);
14954	break;
14955	case LPFC_WCQ:
14956	if (cq->subtype == LPFC_IO)
14957	workposted |= __lpfc_sli4_process_cq(phba, cq,
14958	handler: lpfc_sli4_fp_handle_cqe,
14959	delay: &delay);
14960	else
14961	workposted |= __lpfc_sli4_process_cq(phba, cq,
14962	handler: lpfc_sli4_sp_handle_cqe,
14963	delay: &delay);
14964	break;
14965	default:
14966	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14967	"0370 Invalid completion queue type (%d)\n",
14968	cq->type);
14969	return;
14970	}
14971
14972	if (delay) {
14973	if (is_kdump_kernel())
14974	ret = queue_delayed_work(wq: phba->wq, dwork: &cq->sched_spwork,
14975	delay);
14976	else
14977	ret = queue_delayed_work_on(cpu: cq->chann, wq: phba->wq,
14978	work: &cq->sched_spwork, delay);
14979	if (!ret)
14980	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14981	"0394 Cannot schedule queue work "
14982	"for cqid=%d on CPU %d\n",
14983	cq->queue_id, cq->chann);
14984	}
14985
14986	/* wake up worker thread if there are works to be done */
14987	if (workposted)
14988	lpfc_worker_wake_up(phba);
14989	}
14990
14991	/**
14992	* lpfc_sli4_sp_process_cq - slow-path work handler when started by
14993	* interrupt
14994	* @work: pointer to work element
14995	*
14996	* translates from the work handler and calls the slow-path handler.
14997	**/
14998	static void
14999	lpfc_sli4_sp_process_cq(struct work_struct *work)
15000	{
15001	struct lpfc_queue *cq = container_of(work, struct lpfc_queue, spwork);
15002
15003	__lpfc_sli4_sp_process_cq(cq);
15004	}
15005
15006	/**
15007	* lpfc_sli4_dly_sp_process_cq - slow-path work handler when started by timer
15008	* @work: pointer to work element
15009	*
15010	* translates from the work handler and calls the slow-path handler.
15011	**/
15012	static void
15013	lpfc_sli4_dly_sp_process_cq(struct work_struct *work)
15014	{
15015	struct lpfc_queue *cq = container_of(to_delayed_work(work),
15016	struct lpfc_queue, sched_spwork);
15017
15018	__lpfc_sli4_sp_process_cq(cq);
15019	}
15020
15021	/**
15022	* lpfc_sli4_fp_handle_fcp_wcqe - Process fast-path work queue completion entry
15023	* @phba: Pointer to HBA context object.
15024	* @cq: Pointer to associated CQ
15025	* @wcqe: Pointer to work-queue completion queue entry.
15026	*
15027	* This routine process a fast-path work queue completion entry from fast-path
15028	* event queue for FCP command response completion.
15029	**/
15030	static void
15031	lpfc_sli4_fp_handle_fcp_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
15032	struct lpfc_wcqe_complete *wcqe)
15033	{
15034	struct lpfc_sli_ring *pring = cq->pring;
15035	struct lpfc_iocbq *cmdiocbq;
15036	unsigned long iflags;
15037
15038	/* Check for response status */
15039	if (unlikely(bf_get(lpfc_wcqe_c_status, wcqe))) {
15040	/* If resource errors reported from HBA, reduce queue
15041	* depth of the SCSI device.
15042	*/
15043	if (((bf_get(lpfc_wcqe_c_status, wcqe) ==
15044	IOSTAT_LOCAL_REJECT)) &&
15045	((wcqe->parameter & IOERR_PARAM_MASK) ==
15046	IOERR_NO_RESOURCES))
15047	phba->lpfc_rampdown_queue_depth(phba);
15048
15049	/* Log the cmpl status */
15050	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
15051	"0373 FCP CQE cmpl: status=x%x: "
15052	"CQE: %08x %08x %08x %08x\n",
15053	bf_get(lpfc_wcqe_c_status, wcqe),
15054	wcqe->word0, wcqe->total_data_placed,
15055	wcqe->parameter, wcqe->word3);
15056	}
15057
15058	/* Look up the FCP command IOCB and create pseudo response IOCB */
15059	spin_lock_irqsave(&pring->ring_lock, iflags);
15060	pring->stats.iocb_event++;
15061	cmdiocbq = lpfc_sli_iocbq_lookup_by_tag(phba, pring,
15062	bf_get(lpfc_wcqe_c_request_tag, wcqe));
15063	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
15064	if (unlikely(!cmdiocbq)) {
15065	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
15066	"0374 FCP complete with no corresponding "
15067	"cmdiocb: iotag (%d)\n",
15068	bf_get(lpfc_wcqe_c_request_tag, wcqe));
15069	return;
15070	}
15071	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
15072	cmdiocbq->isr_timestamp = cq->isr_timestamp;
15073	#endif
15074	if (bf_get(lpfc_wcqe_c_xb, wcqe)) {
15075	spin_lock_irqsave(&phba->hbalock, iflags);
15076	cmdiocbq->cmd_flag |= LPFC_EXCHANGE_BUSY;
15077	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
15078	}
15079
15080	if (cmdiocbq->cmd_cmpl) {
15081	/* For FCP the flag is cleared in cmd_cmpl */
15082	if (!(cmdiocbq->cmd_flag & LPFC_IO_FCP) &&
15083	cmdiocbq->cmd_flag & LPFC_DRIVER_ABORTED) {
15084	spin_lock_irqsave(&phba->hbalock, iflags);
15085	cmdiocbq->cmd_flag &= ~LPFC_DRIVER_ABORTED;
15086	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
15087	}
15088
15089	/* Pass the cmd_iocb and the wcqe to the upper layer */
15090	memcpy(&cmdiocbq->wcqe_cmpl, wcqe,
15091	sizeof(struct lpfc_wcqe_complete));
15092	cmdiocbq->cmd_cmpl(phba, cmdiocbq, cmdiocbq);
15093	} else {
15094	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
15095	"0375 FCP cmdiocb not callback function "
15096	"iotag: (%d)\n",
15097	bf_get(lpfc_wcqe_c_request_tag, wcqe));
15098	}
15099	}
15100
15101	/**
15102	* lpfc_sli4_fp_handle_rel_wcqe - Handle fast-path WQ entry consumed event
15103	* @phba: Pointer to HBA context object.
15104	* @cq: Pointer to completion queue.
15105	* @wcqe: Pointer to work-queue completion queue entry.
15106	*
15107	* This routine handles an fast-path WQ entry consumed event by invoking the
15108	* proper WQ release routine to the slow-path WQ.
15109	**/
15110	static void
15111	lpfc_sli4_fp_handle_rel_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
15112	struct lpfc_wcqe_release *wcqe)
15113	{
15114	struct lpfc_queue *childwq;
15115	bool wqid_matched = false;
15116	uint16_t hba_wqid;
15117
15118	/* Check for fast-path FCP work queue release */
15119	hba_wqid = bf_get(lpfc_wcqe_r_wq_id, wcqe);
15120	list_for_each_entry(childwq, &cq->child_list, list) {
15121	if (childwq->queue_id == hba_wqid) {
15122	lpfc_sli4_wq_release(q: childwq,
15123	bf_get(lpfc_wcqe_r_wqe_index, wcqe));
15124	if (childwq->q_flag & HBA_NVMET_WQFULL)
15125	lpfc_nvmet_wqfull_process(phba, wq: childwq);
15126	wqid_matched = true;
15127	break;
15128	}
15129	}
15130	/* Report warning log message if no match found */
15131	if (wqid_matched != true)
15132	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
15133	"2580 Fast-path wqe consume event carries "
15134	"miss-matched qid: wcqe-qid=x%x\n", hba_wqid);
15135	}
15136
15137	/**
15138	* lpfc_sli4_nvmet_handle_rcqe - Process a receive-queue completion queue entry
15139	* @phba: Pointer to HBA context object.
15140	* @cq: Pointer to completion queue.
15141	* @rcqe: Pointer to receive-queue completion queue entry.
15142	*
15143	* This routine process a receive-queue completion queue entry.
15144	*
15145	* Return: true if work posted to worker thread, otherwise false.
15146	**/
15147	static bool
15148	lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
15149	struct lpfc_rcqe *rcqe)
15150	{
15151	bool workposted = false;
15152	struct lpfc_queue *hrq;
15153	struct lpfc_queue *drq;
15154	struct rqb_dmabuf *dma_buf;
15155	struct fc_frame_header *fc_hdr;
15156	struct lpfc_nvmet_tgtport *tgtp;
15157	uint32_t status, rq_id;
15158	unsigned long iflags;
15159	uint32_t fctl, idx;
15160
15161	if ((phba->nvmet_support == 0) ||
15162	(phba->sli4_hba.nvmet_cqset == NULL))
15163	return workposted;
15164
15165	idx = cq->queue_id - phba->sli4_hba.nvmet_cqset[0]->queue_id;
15166	hrq = phba->sli4_hba.nvmet_mrq_hdr[idx];
15167	drq = phba->sli4_hba.nvmet_mrq_data[idx];
15168
15169	/* sanity check on queue memory */
15170	if (unlikely(!hrq) || unlikely(!drq))
15171	return workposted;
15172
15173	if (bf_get(lpfc_cqe_code, rcqe) == CQE_CODE_RECEIVE_V1)
15174	rq_id = bf_get(lpfc_rcqe_rq_id_v1, rcqe);
15175	else
15176	rq_id = bf_get(lpfc_rcqe_rq_id, rcqe);
15177
15178	if ((phba->nvmet_support == 0) ||
15179	(rq_id != hrq->queue_id))
15180	return workposted;
15181
15182	status = bf_get(lpfc_rcqe_status, rcqe);
15183	switch (status) {
15184	case FC_STATUS_RQ_BUF_LEN_EXCEEDED:
15185	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15186	"6126 Receive Frame Truncated!!\n");
15187	fallthrough;
15188	case FC_STATUS_RQ_SUCCESS:
15189	spin_lock_irqsave(&phba->hbalock, iflags);
15190	lpfc_sli4_rq_release(hq: hrq, dq: drq);
15191	dma_buf = lpfc_sli_rqbuf_get(phba, hrq);
15192	if (!dma_buf) {
15193	hrq->RQ_no_buf_found++;
15194	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
15195	goto out;
15196	}
15197	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
15198	hrq->RQ_rcv_buf++;
15199	hrq->RQ_buf_posted--;
15200	fc_hdr = (struct fc_frame_header *)dma_buf->hbuf.virt;
15201
15202	/* Just some basic sanity checks on FCP Command frame */
15203	fctl = (fc_hdr->fh_f_ctl[0] << 16 |
15204	fc_hdr->fh_f_ctl[1] << 8 |
15205	fc_hdr->fh_f_ctl[2]);
15206	if (((fctl &
15207	(FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT)) !=
15208	(FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT)) ||
15209	(fc_hdr->fh_seq_cnt != 0)) /* 0 byte swapped is still 0 */
15210	goto drop;
15211
15212	if (fc_hdr->fh_type == FC_TYPE_FCP) {
15213	dma_buf->bytes_recv = bf_get(lpfc_rcqe_length, rcqe);
15214	lpfc_nvmet_unsol_fcp_event(
15215	phba, idx, nvmebuf: dma_buf, isr_ts: cq->isr_timestamp,
15216	cqflag: cq->q_flag & HBA_NVMET_CQ_NOTIFY);
15217	return false;
15218	}
15219	drop:
15220	lpfc_rq_buf_free(phba, mp: &dma_buf->hbuf);
15221	break;
15222	case FC_STATUS_INSUFF_BUF_FRM_DISC:
15223	if (phba->nvmet_support) {
15224	tgtp = phba->targetport->private;
15225	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15226	"6401 RQE Error x%x, posted %d err_cnt "
15227	"%d: %x %x %x\n",
15228	status, hrq->RQ_buf_posted,
15229	hrq->RQ_no_posted_buf,
15230	atomic_read(&tgtp->rcv_fcp_cmd_in),
15231	atomic_read(&tgtp->rcv_fcp_cmd_out),
15232	atomic_read(&tgtp->xmt_fcp_release));
15233	}
15234	fallthrough;
15235
15236	case FC_STATUS_INSUFF_BUF_NEED_BUF:
15237	hrq->RQ_no_posted_buf++;
15238	/* Post more buffers if possible */
15239	break;
15240	case FC_STATUS_RQ_DMA_FAILURE:
15241	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15242	"2575 RQE DMA Error x%x, x%08x x%08x x%08x "
15243	"x%08x\n",
15244	status, rcqe->word0, rcqe->word1,
15245	rcqe->word2, rcqe->word3);
15246
15247	/* If IV set, no further recovery */
15248	if (bf_get(lpfc_rcqe_iv, rcqe))
15249	break;
15250
15251	/* recycle consumed resource */
15252	spin_lock_irqsave(&phba->hbalock, iflags);
15253	lpfc_sli4_rq_release(hq: hrq, dq: drq);
15254	dma_buf = lpfc_sli_rqbuf_get(phba, hrq);
15255	if (!dma_buf) {
15256	hrq->RQ_no_buf_found++;
15257	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
15258	break;
15259	}
15260	hrq->RQ_rcv_buf++;
15261	hrq->RQ_buf_posted--;
15262	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
15263	lpfc_rq_buf_free(phba, mp: &dma_buf->hbuf);
15264	break;
15265	default:
15266	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15267	"2576 Unexpected RQE Status x%x, w0-3 x%08x "
15268	"x%08x x%08x x%08x\n",
15269	status, rcqe->word0, rcqe->word1,
15270	rcqe->word2, rcqe->word3);
15271	break;
15272	}
15273	out:
15274	return workposted;
15275	}
15276
15277	/**
15278	* lpfc_sli4_fp_handle_cqe - Process fast-path work queue completion entry
15279	* @phba: adapter with cq
15280	* @cq: Pointer to the completion queue.
15281	* @cqe: Pointer to fast-path completion queue entry.
15282	*
15283	* This routine process a fast-path work queue completion entry from fast-path
15284	* event queue for FCP command response completion.
15285	*
15286	* Return: true if work posted to worker thread, otherwise false.
15287	**/
15288	static bool
15289	lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
15290	struct lpfc_cqe *cqe)
15291	{
15292	struct lpfc_wcqe_release wcqe;
15293	bool workposted = false;
15294
15295	/* Copy the work queue CQE and convert endian order if needed */
15296	lpfc_sli4_pcimem_bcopy(srcp: cqe, destp: &wcqe, cnt: sizeof(struct lpfc_cqe));
15297
15298	/* Check and process for different type of WCQE and dispatch */
15299	switch (bf_get(lpfc_wcqe_c_code, &wcqe)) {
15300	case CQE_CODE_COMPL_WQE:
15301	case CQE_CODE_NVME_ERSP:
15302	cq->CQ_wq++;
15303	/* Process the WQ complete event */
15304	phba->last_completion_time = jiffies;
15305	if (cq->subtype == LPFC_IO || cq->subtype == LPFC_NVME_LS)
15306	lpfc_sli4_fp_handle_fcp_wcqe(phba, cq,
15307	wcqe: (struct lpfc_wcqe_complete *)&wcqe);
15308	break;
15309	case CQE_CODE_RELEASE_WQE:
15310	cq->CQ_release_wqe++;
15311	/* Process the WQ release event */
15312	lpfc_sli4_fp_handle_rel_wcqe(phba, cq,
15313	wcqe: (struct lpfc_wcqe_release *)&wcqe);
15314	break;
15315	case CQE_CODE_XRI_ABORTED:
15316	cq->CQ_xri_aborted++;
15317	/* Process the WQ XRI abort event */
15318	phba->last_completion_time = jiffies;
15319	workposted = lpfc_sli4_sp_handle_abort_xri_wcqe(phba, cq,
15320	wcqe: (struct sli4_wcqe_xri_aborted *)&wcqe);
15321	break;
15322	case CQE_CODE_RECEIVE_V1:
15323	case CQE_CODE_RECEIVE:
15324	phba->last_completion_time = jiffies;
15325	if (cq->subtype == LPFC_NVMET) {
15326	workposted = lpfc_sli4_nvmet_handle_rcqe(
15327	phba, cq, rcqe: (struct lpfc_rcqe *)&wcqe);
15328	}
15329	break;
15330	default:
15331	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15332	"0144 Not a valid CQE code: x%x\n",
15333	bf_get(lpfc_wcqe_c_code, &wcqe));
15334	break;
15335	}
15336	return workposted;
15337	}
15338
15339	/**
15340	* __lpfc_sli4_hba_process_cq - Process a fast-path event queue entry
15341	* @cq: Pointer to CQ to be processed
15342	*
15343	* This routine calls the cq processing routine with the handler for
15344	* fast path CQEs.
15345	*
15346	* The CQ routine returns two values: the first is the calling status,
15347	* which indicates whether work was queued to the background discovery
15348	* thread. If true, the routine should wakeup the discovery thread;
15349	* the second is the delay parameter. If non-zero, rather than rearming
15350	* the CQ and yet another interrupt, the CQ handler should be queued so
15351	* that it is processed in a subsequent polling action. The value of
15352	* the delay indicates when to reschedule it.
15353	**/
15354	static void
15355	__lpfc_sli4_hba_process_cq(struct lpfc_queue *cq)
15356	{
15357	struct lpfc_hba *phba = cq->phba;
15358	unsigned long delay;
15359	bool workposted = false;
15360	int ret;
15361
15362	/* process and rearm the CQ */
15363	workposted |= __lpfc_sli4_process_cq(phba, cq, handler: lpfc_sli4_fp_handle_cqe,
15364	delay: &delay);
15365
15366	if (delay) {
15367	if (is_kdump_kernel())
15368	ret = queue_delayed_work(wq: phba->wq, dwork: &cq->sched_irqwork,
15369	delay);
15370	else
15371	ret = queue_delayed_work_on(cpu: cq->chann, wq: phba->wq,
15372	work: &cq->sched_irqwork, delay);
15373	if (!ret)
15374	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15375	"0367 Cannot schedule queue work "
15376	"for cqid=%d on CPU %d\n",
15377	cq->queue_id, cq->chann);
15378	}
15379
15380	/* wake up worker thread if there are works to be done */
15381	if (workposted)
15382	lpfc_worker_wake_up(phba);
15383	}
15384
15385	/**
15386	* lpfc_sli4_hba_process_cq - fast-path work handler when started by
15387	* interrupt
15388	* @work: pointer to work element
15389	*
15390	* translates from the work handler and calls the fast-path handler.
15391	**/
15392	static void
15393	lpfc_sli4_hba_process_cq(struct work_struct *work)
15394	{
15395	struct lpfc_queue *cq = container_of(work, struct lpfc_queue, irqwork);
15396
15397	__lpfc_sli4_hba_process_cq(cq);
15398	}
15399
15400	/**
15401	* lpfc_sli4_hba_handle_eqe - Process a fast-path event queue entry
15402	* @phba: Pointer to HBA context object.
15403	* @eq: Pointer to the queue structure.
15404	* @eqe: Pointer to fast-path event queue entry.
15405	* @poll_mode: poll_mode to execute processing the cq.
15406	*
15407	* This routine process a event queue entry from the fast-path event queue.
15408	* It will check the MajorCode and MinorCode to determine this is for a
15409	* completion event on a completion queue, if not, an error shall be logged
15410	* and just return. Otherwise, it will get to the corresponding completion
15411	* queue and process all the entries on the completion queue, rearm the
15412	* completion queue, and then return.
15413	**/
15414	static void
15415	lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq,
15416	struct lpfc_eqe *eqe, enum lpfc_poll_mode poll_mode)
15417	{
15418	struct lpfc_queue *cq = NULL;
15419	uint32_t qidx = eq->hdwq;
15420	uint16_t cqid, id;
15421	int ret;
15422
15423	if (unlikely(bf_get_le32(lpfc_eqe_major_code, eqe) != 0)) {
15424	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15425	"0366 Not a valid completion "
15426	"event: majorcode=x%x, minorcode=x%x\n",
15427	bf_get_le32(lpfc_eqe_major_code, eqe),
15428	bf_get_le32(lpfc_eqe_minor_code, eqe));
15429	return;
15430	}
15431
15432	/* Get the reference to the corresponding CQ */
15433	cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);
15434
15435	/* Use the fast lookup method first */
15436	if (cqid <= phba->sli4_hba.cq_max) {
15437	cq = phba->sli4_hba.cq_lookup[cqid];
15438	if (cq)
15439	goto work_cq;
15440	}
15441
15442	/* Next check for NVMET completion */
15443	if (phba->cfg_nvmet_mrq && phba->sli4_hba.nvmet_cqset) {
15444	id = phba->sli4_hba.nvmet_cqset[0]->queue_id;
15445	if ((cqid >= id) && (cqid < (id + phba->cfg_nvmet_mrq))) {
15446	/* Process NVMET unsol rcv */
15447	cq = phba->sli4_hba.nvmet_cqset[cqid - id];
15448	goto process_cq;
15449	}
15450	}
15451
15452	if (phba->sli4_hba.nvmels_cq &&
15453	(cqid == phba->sli4_hba.nvmels_cq->queue_id)) {
15454	/* Process NVME unsol rcv */
15455	cq = phba->sli4_hba.nvmels_cq;
15456	}
15457
15458	/* Otherwise this is a Slow path event */
15459	if (cq == NULL) {
15460	lpfc_sli4_sp_handle_eqe(phba, eqe,
15461	speq: phba->sli4_hba.hdwq[qidx].hba_eq);
15462	return;
15463	}
15464
15465	process_cq:
15466	if (unlikely(cqid != cq->queue_id)) {
15467	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15468	"0368 Miss-matched fast-path completion "
15469	"queue identifier: eqcqid=%d, fcpcqid=%d\n",
15470	cqid, cq->queue_id);
15471	return;
15472	}
15473
15474	work_cq:
15475	#if defined(CONFIG_SCSI_LPFC_DEBUG_FS)
15476	if (phba->ktime_on)
15477	cq->isr_timestamp = ktime_get_ns();
15478	else
15479	cq->isr_timestamp = 0;
15480	#endif
15481
15482	switch (poll_mode) {
15483	case LPFC_THREADED_IRQ:
15484	__lpfc_sli4_hba_process_cq(cq);
15485	break;
15486	case LPFC_QUEUE_WORK:
15487	default:
15488	if (is_kdump_kernel())
15489	ret = queue_work(wq: phba->wq, work: &cq->irqwork);
15490	else
15491	ret = queue_work_on(cpu: cq->chann, wq: phba->wq, work: &cq->irqwork);
15492	if (!ret)
15493	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15494	"0383 Cannot schedule queue work "
15495	"for CQ eqcqid=%d, cqid=%d on CPU %d\n",
15496	cqid, cq->queue_id,
15497	raw_smp_processor_id());
15498	break;
15499	}
15500	}
15501
15502	/**
15503	* lpfc_sli4_dly_hba_process_cq - fast-path work handler when started by timer
15504	* @work: pointer to work element
15505	*
15506	* translates from the work handler and calls the fast-path handler.
15507	**/
15508	static void
15509	lpfc_sli4_dly_hba_process_cq(struct work_struct *work)
15510	{
15511	struct lpfc_queue *cq = container_of(to_delayed_work(work),
15512	struct lpfc_queue, sched_irqwork);
15513
15514	__lpfc_sli4_hba_process_cq(cq);
15515	}
15516
15517	/**
15518	* lpfc_sli4_hba_intr_handler - HBA interrupt handler to SLI-4 device
15519	* @irq: Interrupt number.
15520	* @dev_id: The device context pointer.
15521	*
15522	* This function is directly called from the PCI layer as an interrupt
15523	* service routine when device with SLI-4 interface spec is enabled with
15524	* MSI-X multi-message interrupt mode and there is a fast-path FCP IOCB
15525	* ring event in the HBA. However, when the device is enabled with either
15526	* MSI or Pin-IRQ interrupt mode, this function is called as part of the
15527	* device-level interrupt handler. When the PCI slot is in error recovery
15528	* or the HBA is undergoing initialization, the interrupt handler will not
15529	* process the interrupt. The SCSI FCP fast-path ring event are handled in
15530	* the intrrupt context. This function is called without any lock held.
15531	* It gets the hbalock to access and update SLI data structures. Note that,
15532	* the FCP EQ to FCP CQ are one-to-one map such that the FCP EQ index is
15533	* equal to that of FCP CQ index.
15534	*
15535	* The link attention and ELS ring attention events are handled
15536	* by the worker thread. The interrupt handler signals the worker thread
15537	* and returns for these events. This function is called without any lock
15538	* held. It gets the hbalock to access and update SLI data structures.
15539	*
15540	* This function returns IRQ_HANDLED when interrupt is handled, IRQ_WAKE_THREAD
15541	* when interrupt is scheduled to be handled from a threaded irq context, or
15542	* else returns IRQ_NONE.
15543	**/
15544	irqreturn_t
15545	lpfc_sli4_hba_intr_handler(int irq, void *dev_id)
15546	{
15547	struct lpfc_hba *phba;
15548	struct lpfc_hba_eq_hdl *hba_eq_hdl;
15549	struct lpfc_queue *fpeq;
15550	unsigned long iflag;
15551	int hba_eqidx;
15552	int ecount = 0;
15553	struct lpfc_eq_intr_info *eqi;
15554
15555	/* Get the driver's phba structure from the dev_id */
15556	hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id;
15557	phba = hba_eq_hdl->phba;
15558	hba_eqidx = hba_eq_hdl->idx;
15559
15560	if (unlikely(!phba))
15561	return IRQ_NONE;
15562	if (unlikely(!phba->sli4_hba.hdwq))
15563	return IRQ_NONE;
15564
15565	/* Get to the EQ struct associated with this vector */
15566	fpeq = phba->sli4_hba.hba_eq_hdl[hba_eqidx].eq;
15567	if (unlikely(!fpeq))
15568	return IRQ_NONE;
15569
15570	/* Check device state for handling interrupt */
15571	if (unlikely(lpfc_intr_state_check(phba))) {
15572	/* Check again for link_state with lock held */
15573	spin_lock_irqsave(&phba->hbalock, iflag);
15574	if (phba->link_state < LPFC_LINK_DOWN)
15575	/* Flush, clear interrupt, and rearm the EQ */
15576	lpfc_sli4_eqcq_flush(phba, eq: fpeq);
15577	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
15578	return IRQ_NONE;
15579	}
15580
15581	switch (fpeq->poll_mode) {
15582	case LPFC_THREADED_IRQ:
15583	/* CGN mgmt is mutually exclusive from irq processing */
15584	if (phba->cmf_active_mode == LPFC_CFG_OFF)
15585	return IRQ_WAKE_THREAD;
15586	fallthrough;
15587	case LPFC_QUEUE_WORK:
15588	default:
15589	eqi = this_cpu_ptr(phba->sli4_hba.eq_info);
15590	eqi->icnt++;
15591
15592	fpeq->last_cpu = raw_smp_processor_id();
15593
15594	if (eqi->icnt > LPFC_EQD_ISR_TRIGGER &&
15595	fpeq->q_flag & HBA_EQ_DELAY_CHK &&
15596	phba->cfg_auto_imax &&
15597	fpeq->q_mode != LPFC_MAX_AUTO_EQ_DELAY &&
15598	phba->sli.sli_flag & LPFC_SLI_USE_EQDR)
15599	lpfc_sli4_mod_hba_eq_delay(phba, eq: fpeq,
15600	LPFC_MAX_AUTO_EQ_DELAY);
15601
15602	/* process and rearm the EQ */
15603	ecount = lpfc_sli4_process_eq(phba, eq: fpeq, LPFC_QUEUE_REARM,
15604	poll_mode: LPFC_QUEUE_WORK);
15605
15606	if (unlikely(ecount == 0)) {
15607	fpeq->EQ_no_entry++;
15608	if (phba->intr_type == MSIX)
15609	/* MSI-X treated interrupt served as no EQ share INT */
15610	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
15611	"0358 MSI-X interrupt with no EQE\n");
15612	else
15613	/* Non MSI-X treated on interrupt as EQ share INT */
15614	return IRQ_NONE;
15615	}
15616	}
15617
15618	return IRQ_HANDLED;
15619	} /* lpfc_sli4_hba_intr_handler */
15620
15621	/**
15622	* lpfc_sli4_intr_handler - Device-level interrupt handler for SLI-4 device
15623	* @irq: Interrupt number.
15624	* @dev_id: The device context pointer.
15625	*
15626	* This function is the device-level interrupt handler to device with SLI-4
15627	* interface spec, called from the PCI layer when either MSI or Pin-IRQ
15628	* interrupt mode is enabled and there is an event in the HBA which requires
15629	* driver attention. This function invokes the slow-path interrupt attention
15630	* handling function and fast-path interrupt attention handling function in
15631	* turn to process the relevant HBA attention events. This function is called
15632	* without any lock held. It gets the hbalock to access and update SLI data
15633	* structures.
15634	*
15635	* This function returns IRQ_HANDLED when interrupt is handled, else it
15636	* returns IRQ_NONE.
15637	**/
15638	irqreturn_t
15639	lpfc_sli4_intr_handler(int irq, void *dev_id)
15640	{
15641	struct lpfc_hba *phba;
15642	irqreturn_t hba_irq_rc;
15643	bool hba_handled = false;
15644	int qidx;
15645
15646	/* Get the driver's phba structure from the dev_id */
15647	phba = (struct lpfc_hba *)dev_id;
15648
15649	if (unlikely(!phba))
15650	return IRQ_NONE;
15651
15652	/*
15653	* Invoke fast-path host attention interrupt handling as appropriate.
15654	*/
15655	for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) {
15656	hba_irq_rc = lpfc_sli4_hba_intr_handler(irq,
15657	dev_id: &phba->sli4_hba.hba_eq_hdl[qidx]);
15658	if (hba_irq_rc == IRQ_HANDLED)
15659	hba_handled |= true;
15660	}
15661
15662	return (hba_handled == true) ? IRQ_HANDLED : IRQ_NONE;
15663	} /* lpfc_sli4_intr_handler */
15664
15665	void lpfc_sli4_poll_hbtimer(struct timer_list *t)
15666	{
15667	struct lpfc_hba *phba = from_timer(phba, t, cpuhp_poll_timer);
15668	struct lpfc_queue *eq;
15669
15670	rcu_read_lock();
15671
15672	list_for_each_entry_rcu(eq, &phba->poll_list, _poll_list)
15673	lpfc_sli4_poll_eq(eq);
15674	if (!list_empty(head: &phba->poll_list))
15675	mod_timer(timer: &phba->cpuhp_poll_timer,
15676	expires: jiffies + msecs_to_jiffies(LPFC_POLL_HB));
15677
15678	rcu_read_unlock();
15679	}
15680
15681	static inline void lpfc_sli4_add_to_poll_list(struct lpfc_queue *eq)
15682	{
15683	struct lpfc_hba *phba = eq->phba;
15684
15685	/* kickstart slowpath processing if needed */
15686	if (list_empty(head: &phba->poll_list))
15687	mod_timer(timer: &phba->cpuhp_poll_timer,
15688	expires: jiffies + msecs_to_jiffies(LPFC_POLL_HB));
15689
15690	list_add_rcu(new: &eq->_poll_list, head: &phba->poll_list);
15691	synchronize_rcu();
15692	}
15693
15694	static inline void lpfc_sli4_remove_from_poll_list(struct lpfc_queue *eq)
15695	{
15696	struct lpfc_hba *phba = eq->phba;
15697
15698	/* Disable slowpath processing for this eq. Kick start the eq
15699	* by RE-ARMING the eq's ASAP
15700	*/
15701	list_del_rcu(entry: &eq->_poll_list);
15702	synchronize_rcu();
15703
15704	if (list_empty(head: &phba->poll_list))
15705	del_timer_sync(timer: &phba->cpuhp_poll_timer);
15706	}
15707
15708	void lpfc_sli4_cleanup_poll_list(struct lpfc_hba *phba)
15709	{
15710	struct lpfc_queue *eq, *next;
15711
15712	list_for_each_entry_safe(eq, next, &phba->poll_list, _poll_list)
15713	list_del(entry: &eq->_poll_list);
15714
15715	INIT_LIST_HEAD(list: &phba->poll_list);
15716	synchronize_rcu();
15717	}
15718
15719	static inline void
15720	__lpfc_sli4_switch_eqmode(struct lpfc_queue *eq, uint8_t mode)
15721	{
15722	if (mode == eq->mode)
15723	return;
15724	/*
15725	* currently this function is only called during a hotplug
15726	* event and the cpu on which this function is executing
15727	* is going offline. By now the hotplug has instructed
15728	* the scheduler to remove this cpu from cpu active mask.
15729	* So we don't need to work about being put aside by the
15730	* scheduler for a high priority process. Yes, the inte-
15731	* rrupts could come but they are known to retire ASAP.
15732	*/
15733
15734	/* Disable polling in the fastpath */
15735	WRITE_ONCE(eq->mode, mode);
15736	/* flush out the store buffer */
15737	smp_wmb();
15738
15739	/*
15740	* Add this eq to the polling list and start polling. For
15741	* a grace period both interrupt handler and poller will
15742	* try to process the eq _but_ that's fine. We have a
15743	* synchronization mechanism in place (queue_claimed) to
15744	* deal with it. This is just a draining phase for int-
15745	* errupt handler (not eq's) as we have guranteed through
15746	* barrier that all the CPUs have seen the new CQ_POLLED
15747	* state. which will effectively disable the REARMING of
15748	* the EQ. The whole idea is eq's die off eventually as
15749	* we are not rearming EQ's anymore.
15750	*/
15751	mode ? lpfc_sli4_add_to_poll_list(eq) :
15752	lpfc_sli4_remove_from_poll_list(eq);
15753	}
15754
15755	void lpfc_sli4_start_polling(struct lpfc_queue *eq)
15756	{
15757	__lpfc_sli4_switch_eqmode(eq, LPFC_EQ_POLL);
15758	}
15759
15760	void lpfc_sli4_stop_polling(struct lpfc_queue *eq)
15761	{
15762	struct lpfc_hba *phba = eq->phba;
15763
15764	__lpfc_sli4_switch_eqmode(eq, LPFC_EQ_INTERRUPT);
15765
15766	/* Kick start for the pending io's in h/w.
15767	* Once we switch back to interrupt processing on a eq
15768	* the io path completion will only arm eq's when it
15769	* receives a completion. But since eq's are in disa-
15770	* rmed state it doesn't receive a completion. This
15771	* creates a deadlock scenaro.
15772	*/
15773	phba->sli4_hba.sli4_write_eq_db(phba, eq, 0, LPFC_QUEUE_REARM);
15774	}
15775
15776	/**
15777	* lpfc_sli4_queue_free - free a queue structure and associated memory
15778	* @queue: The queue structure to free.
15779	*
15780	* This function frees a queue structure and the DMAable memory used for
15781	* the host resident queue. This function must be called after destroying the
15782	* queue on the HBA.
15783	**/
15784	void
15785	lpfc_sli4_queue_free(struct lpfc_queue *queue)
15786	{
15787	struct lpfc_dmabuf *dmabuf;
15788
15789	if (!queue)
15790	return;
15791
15792	if (!list_empty(head: &queue->wq_list))
15793	list_del(entry: &queue->wq_list);
15794
15795	while (!list_empty(head: &queue->page_list)) {
15796	list_remove_head(&queue->page_list, dmabuf, struct lpfc_dmabuf,
15797	list);
15798	dma_free_coherent(dev: &queue->phba->pcidev->dev, size: queue->page_size,
15799	cpu_addr: dmabuf->virt, dma_handle: dmabuf->phys);
15800	kfree(objp: dmabuf);
15801	}
15802	if (queue->rqbp) {
15803	lpfc_free_rq_buffer(phba: queue->phba, hq: queue);
15804	kfree(objp: queue->rqbp);
15805	}
15806
15807	if (!list_empty(head: &queue->cpu_list))
15808	list_del(entry: &queue->cpu_list);
15809
15810	kfree(objp: queue);
15811	return;
15812	}
15813
15814	/**
15815	* lpfc_sli4_queue_alloc - Allocate and initialize a queue structure
15816	* @phba: The HBA that this queue is being created on.
15817	* @page_size: The size of a queue page
15818	* @entry_size: The size of each queue entry for this queue.
15819	* @entry_count: The number of entries that this queue will handle.
15820	* @cpu: The cpu that will primarily utilize this queue.
15821	*
15822	* This function allocates a queue structure and the DMAable memory used for
15823	* the host resident queue. This function must be called before creating the
15824	* queue on the HBA.
15825	**/
15826	struct lpfc_queue *
15827	lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size,
15828	uint32_t entry_size, uint32_t entry_count, int cpu)
15829	{
15830	struct lpfc_queue *queue;
15831	struct lpfc_dmabuf *dmabuf;
15832	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
15833	uint16_t x, pgcnt;
15834
15835	if (!phba->sli4_hba.pc_sli4_params.supported)
15836	hw_page_size = page_size;
15837
15838	pgcnt = ALIGN(entry_size * entry_count, hw_page_size) / hw_page_size;
15839
15840	/* If needed, Adjust page count to match the max the adapter supports */
15841	if (pgcnt > phba->sli4_hba.pc_sli4_params.wqpcnt)
15842	pgcnt = phba->sli4_hba.pc_sli4_params.wqpcnt;
15843
15844	queue = kzalloc_node(size: sizeof(*queue) + (sizeof(void *) * pgcnt),
15845	GFP_KERNEL, cpu_to_node(cpu));
15846	if (!queue)
15847	return NULL;
15848
15849	INIT_LIST_HEAD(list: &queue->list);
15850	INIT_LIST_HEAD(list: &queue->_poll_list);
15851	INIT_LIST_HEAD(list: &queue->wq_list);
15852	INIT_LIST_HEAD(list: &queue->wqfull_list);
15853	INIT_LIST_HEAD(list: &queue->page_list);
15854	INIT_LIST_HEAD(list: &queue->child_list);
15855	INIT_LIST_HEAD(list: &queue->cpu_list);
15856
15857	/* Set queue parameters now. If the system cannot provide memory
15858	* resources, the free routine needs to know what was allocated.
15859	*/
15860	queue->page_count = pgcnt;
15861	queue->q_pgs = (void **)&queue[1];
15862	queue->entry_cnt_per_pg = hw_page_size / entry_size;
15863	queue->entry_size = entry_size;
15864	queue->entry_count = entry_count;
15865	queue->page_size = hw_page_size;
15866	queue->phba = phba;
15867
15868	for (x = 0; x < queue->page_count; x++) {
15869	dmabuf = kzalloc_node(size: sizeof(*dmabuf), GFP_KERNEL,
15870	node: dev_to_node(dev: &phba->pcidev->dev));
15871	if (!dmabuf)
15872	goto out_fail;
15873	dmabuf->virt = dma_alloc_coherent(dev: &phba->pcidev->dev,
15874	size: hw_page_size, dma_handle: &dmabuf->phys,
15875	GFP_KERNEL);
15876	if (!dmabuf->virt) {
15877	kfree(objp: dmabuf);
15878	goto out_fail;
15879	}
15880	dmabuf->buffer_tag = x;
15881	list_add_tail(new: &dmabuf->list, head: &queue->page_list);
15882	/* use lpfc_sli4_qe to index a paritcular entry in this page */
15883	queue->q_pgs[x] = dmabuf->virt;
15884	}
15885	INIT_WORK(&queue->irqwork, lpfc_sli4_hba_process_cq);
15886	INIT_WORK(&queue->spwork, lpfc_sli4_sp_process_cq);
15887	INIT_DELAYED_WORK(&queue->sched_irqwork, lpfc_sli4_dly_hba_process_cq);
15888	INIT_DELAYED_WORK(&queue->sched_spwork, lpfc_sli4_dly_sp_process_cq);
15889
15890	/* notify_interval will be set during q creation */
15891
15892	return queue;
15893	out_fail:
15894	lpfc_sli4_queue_free(queue);
15895	return NULL;
15896	}
15897
15898	/**
15899	* lpfc_dual_chute_pci_bar_map - Map pci base address register to host memory
15900	* @phba: HBA structure that indicates port to create a queue on.
15901	* @pci_barset: PCI BAR set flag.
15902	*
15903	* This function shall perform iomap of the specified PCI BAR address to host
15904	* memory address if not already done so and return it. The returned host
15905	* memory address can be NULL.
15906	*/
15907	static void __iomem *
15908	lpfc_dual_chute_pci_bar_map(struct lpfc_hba *phba, uint16_t pci_barset)
15909	{
15910	if (!phba->pcidev)
15911	return NULL;
15912
15913	switch (pci_barset) {
15914	case WQ_PCI_BAR_0_AND_1:
15915	return phba->pci_bar0_memmap_p;
15916	case WQ_PCI_BAR_2_AND_3:
15917	return phba->pci_bar2_memmap_p;
15918	case WQ_PCI_BAR_4_AND_5:
15919	return phba->pci_bar4_memmap_p;
15920	default:
15921	break;
15922	}
15923	return NULL;
15924	}
15925
15926	/**
15927	* lpfc_modify_hba_eq_delay - Modify Delay Multiplier on EQs
15928	* @phba: HBA structure that EQs are on.
15929	* @startq: The starting EQ index to modify
15930	* @numq: The number of EQs (consecutive indexes) to modify
15931	* @usdelay: amount of delay
15932	*
15933	* This function revises the EQ delay on 1 or more EQs. The EQ delay
15934	* is set either by writing to a register (if supported by the SLI Port)
15935	* or by mailbox command. The mailbox command allows several EQs to be
15936	* updated at once.
15937	*
15938	* The @phba struct is used to send a mailbox command to HBA. The @startq
15939	* is used to get the starting EQ index to change. The @numq value is
15940	* used to specify how many consecutive EQ indexes, starting at EQ index,
15941	* are to be changed. This function is asynchronous and will wait for any
15942	* mailbox commands to finish before returning.
15943	*
15944	* On success this function will return a zero. If unable to allocate
15945	* enough memory this function will return -ENOMEM. If a mailbox command
15946	* fails this function will return -ENXIO. Note: on ENXIO, some EQs may
15947	* have had their delay multipler changed.
15948	**/
15949	void
15950	lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq,
15951	uint32_t numq, uint32_t usdelay)
15952	{
15953	struct lpfc_mbx_modify_eq_delay *eq_delay;
15954	LPFC_MBOXQ_t *mbox;
15955	struct lpfc_queue *eq;
15956	int cnt = 0, rc, length;
15957	uint32_t shdr_status, shdr_add_status;
15958	uint32_t dmult;
15959	int qidx;
15960	union lpfc_sli4_cfg_shdr *shdr;
15961
15962	if (startq >= phba->cfg_irq_chann)
15963	return;
15964
15965	if (usdelay > 0xFFFF) {
15966	lpfc_printf_log(phba, KERN_INFO, LOG_INIT | LOG_FCP | LOG_NVME,
15967	"6429 usdelay %d too large. Scaled down to "
15968	"0xFFFF.\n", usdelay);
15969	usdelay = 0xFFFF;
15970	}
15971
15972	/* set values by EQ_DELAY register if supported */
15973	if (phba->sli.sli_flag & LPFC_SLI_USE_EQDR) {
15974	for (qidx = startq; qidx < phba->cfg_irq_chann; qidx++) {
15975	eq = phba->sli4_hba.hba_eq_hdl[qidx].eq;
15976	if (!eq)
15977	continue;
15978
15979	lpfc_sli4_mod_hba_eq_delay(phba, eq, delay: usdelay);
15980
15981	if (++cnt >= numq)
15982	break;
15983	}
15984	return;
15985	}
15986
15987	/* Otherwise, set values by mailbox cmd */
15988
15989	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
15990	if (!mbox) {
15991	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15992	"6428 Failed allocating mailbox cmd buffer."
15993	" EQ delay was not set.\n");
15994	return;
15995	}
15996	length = (sizeof(struct lpfc_mbx_modify_eq_delay) -
15997	sizeof(struct lpfc_sli4_cfg_mhdr));
15998	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
15999	LPFC_MBOX_OPCODE_MODIFY_EQ_DELAY,
16000	length, LPFC_SLI4_MBX_EMBED);
16001	eq_delay = &mbox->u.mqe.un.eq_delay;
16002
16003	/* Calculate delay multiper from maximum interrupt per second */
16004	dmult = (usdelay * LPFC_DMULT_CONST) / LPFC_SEC_TO_USEC;
16005	if (dmult)
16006	dmult--;
16007	if (dmult > LPFC_DMULT_MAX)
16008	dmult = LPFC_DMULT_MAX;
16009
16010	for (qidx = startq; qidx < phba->cfg_irq_chann; qidx++) {
16011	eq = phba->sli4_hba.hba_eq_hdl[qidx].eq;
16012	if (!eq)
16013	continue;
16014	eq->q_mode = usdelay;
16015	eq_delay->u.request.eq[cnt].eq_id = eq->queue_id;
16016	eq_delay->u.request.eq[cnt].phase = 0;
16017	eq_delay->u.request.eq[cnt].delay_multi = dmult;
16018
16019	if (++cnt >= numq)
16020	break;
16021	}
16022	eq_delay->u.request.num_eq = cnt;
16023
16024	mbox->vport = phba->pport;
16025	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
16026	mbox->ctx_ndlp = NULL;
16027	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
16028	shdr = (union lpfc_sli4_cfg_shdr *) &eq_delay->header.cfg_shdr;
16029	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
16030	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
16031	if (shdr_status || shdr_add_status || rc) {
16032	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16033	"2512 MODIFY_EQ_DELAY mailbox failed with "
16034	"status x%x add_status x%x, mbx status x%x\n",
16035	shdr_status, shdr_add_status, rc);
16036	}
16037	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
16038	return;
16039	}
16040
16041	/**
16042	* lpfc_eq_create - Create an Event Queue on the HBA
16043	* @phba: HBA structure that indicates port to create a queue on.
16044	* @eq: The queue structure to use to create the event queue.
16045	* @imax: The maximum interrupt per second limit.
16046	*
16047	* This function creates an event queue, as detailed in @eq, on a port,
16048	* described by @phba by sending an EQ_CREATE mailbox command to the HBA.
16049	*
16050	* The @phba struct is used to send mailbox command to HBA. The @eq struct
16051	* is used to get the entry count and entry size that are necessary to
16052	* determine the number of pages to allocate and use for this queue. This
16053	* function will send the EQ_CREATE mailbox command to the HBA to setup the
16054	* event queue. This function is asynchronous and will wait for the mailbox
16055	* command to finish before continuing.
16056	*
16057	* On success this function will return a zero. If unable to allocate enough
16058	* memory this function will return -ENOMEM. If the queue create mailbox command
16059	* fails this function will return -ENXIO.
16060	**/
16061	int
16062	lpfc_eq_create(struct lpfc_hba *phba, struct lpfc_queue *eq, uint32_t imax)
16063	{
16064	struct lpfc_mbx_eq_create *eq_create;
16065	LPFC_MBOXQ_t *mbox;
16066	int rc, length, status = 0;
16067	struct lpfc_dmabuf *dmabuf;
16068	uint32_t shdr_status, shdr_add_status;
16069	union lpfc_sli4_cfg_shdr *shdr;
16070	uint16_t dmult;
16071	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
16072
16073	/* sanity check on queue memory */
16074	if (!eq)
16075	return -ENODEV;
16076	if (!phba->sli4_hba.pc_sli4_params.supported)
16077	hw_page_size = SLI4_PAGE_SIZE;
16078
16079	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
16080	if (!mbox)
16081	return -ENOMEM;
16082	length = (sizeof(struct lpfc_mbx_eq_create) -
16083	sizeof(struct lpfc_sli4_cfg_mhdr));
16084	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
16085	LPFC_MBOX_OPCODE_EQ_CREATE,
16086	length, LPFC_SLI4_MBX_EMBED);
16087	eq_create = &mbox->u.mqe.un.eq_create;
16088	shdr = (union lpfc_sli4_cfg_shdr *) &eq_create->header.cfg_shdr;
16089	bf_set(lpfc_mbx_eq_create_num_pages, &eq_create->u.request,
16090	eq->page_count);
16091	bf_set(lpfc_eq_context_size, &eq_create->u.request.context,
16092	LPFC_EQE_SIZE);
16093	bf_set(lpfc_eq_context_valid, &eq_create->u.request.context, 1);
16094
16095	/* Use version 2 of CREATE_EQ if eqav is set */
16096	if (phba->sli4_hba.pc_sli4_params.eqav) {
16097	bf_set(lpfc_mbox_hdr_version, &shdr->request,
16098	LPFC_Q_CREATE_VERSION_2);
16099	bf_set(lpfc_eq_context_autovalid, &eq_create->u.request.context,
16100	phba->sli4_hba.pc_sli4_params.eqav);
16101	}
16102
16103	/* don't setup delay multiplier using EQ_CREATE */
16104	dmult = 0;
16105	bf_set(lpfc_eq_context_delay_multi, &eq_create->u.request.context,
16106	dmult);
16107	switch (eq->entry_count) {
16108	default:
16109	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16110	"0360 Unsupported EQ count. (%d)\n",
16111	eq->entry_count);
16112	if (eq->entry_count < 256) {
16113	status = -EINVAL;
16114	goto out;
16115	}
16116	fallthrough; /* otherwise default to smallest count */
16117	case 256:
16118	bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
16119	LPFC_EQ_CNT_256);
16120	break;
16121	case 512:
16122	bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
16123	LPFC_EQ_CNT_512);
16124	break;
16125	case 1024:
16126	bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
16127	LPFC_EQ_CNT_1024);
16128	break;
16129	case 2048:
16130	bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
16131	LPFC_EQ_CNT_2048);
16132	break;
16133	case 4096:
16134	bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
16135	LPFC_EQ_CNT_4096);
16136	break;
16137	}
16138	list_for_each_entry(dmabuf, &eq->page_list, list) {
16139	memset(dmabuf->virt, 0, hw_page_size);
16140	eq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
16141	putPaddrLow(dmabuf->phys);
16142	eq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
16143	putPaddrHigh(dmabuf->phys);
16144	}
16145	mbox->vport = phba->pport;
16146	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
16147	mbox->ctx_buf = NULL;
16148	mbox->ctx_ndlp = NULL;
16149	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
16150	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
16151	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
16152	if (shdr_status || shdr_add_status || rc) {
16153	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16154	"2500 EQ_CREATE mailbox failed with "
16155	"status x%x add_status x%x, mbx status x%x\n",
16156	shdr_status, shdr_add_status, rc);
16157	status = -ENXIO;
16158	}
16159	eq->type = LPFC_EQ;
16160	eq->subtype = LPFC_NONE;
16161	eq->queue_id = bf_get(lpfc_mbx_eq_create_q_id, &eq_create->u.response);
16162	if (eq->queue_id == 0xFFFF)
16163	status = -ENXIO;
16164	eq->host_index = 0;
16165	eq->notify_interval = LPFC_EQ_NOTIFY_INTRVL;
16166	eq->max_proc_limit = LPFC_EQ_MAX_PROC_LIMIT;
16167	out:
16168	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
16169	return status;
16170	}
16171
16172	/**
16173	* lpfc_sli4_hba_intr_handler_th - SLI4 HBA threaded interrupt handler
16174	* @irq: Interrupt number.
16175	* @dev_id: The device context pointer.
16176	*
16177	* This routine is a mirror of lpfc_sli4_hba_intr_handler, but executed within
16178	* threaded irq context.
16179	*
16180	* Returns
16181	* IRQ_HANDLED - interrupt is handled
16182	* IRQ_NONE - otherwise
16183	**/
16184	irqreturn_t lpfc_sli4_hba_intr_handler_th(int irq, void *dev_id)
16185	{
16186	struct lpfc_hba *phba;
16187	struct lpfc_hba_eq_hdl *hba_eq_hdl;
16188	struct lpfc_queue *fpeq;
16189	int ecount = 0;
16190	int hba_eqidx;
16191	struct lpfc_eq_intr_info *eqi;
16192
16193	/* Get the driver's phba structure from the dev_id */
16194	hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id;
16195	phba = hba_eq_hdl->phba;
16196	hba_eqidx = hba_eq_hdl->idx;
16197
16198	if (unlikely(!phba))
16199	return IRQ_NONE;
16200	if (unlikely(!phba->sli4_hba.hdwq))
16201	return IRQ_NONE;
16202
16203	/* Get to the EQ struct associated with this vector */
16204	fpeq = phba->sli4_hba.hba_eq_hdl[hba_eqidx].eq;
16205	if (unlikely(!fpeq))
16206	return IRQ_NONE;
16207
16208	eqi = per_cpu_ptr(phba->sli4_hba.eq_info, raw_smp_processor_id());
16209	eqi->icnt++;
16210
16211	fpeq->last_cpu = raw_smp_processor_id();
16212
16213	if (eqi->icnt > LPFC_EQD_ISR_TRIGGER &&
16214	fpeq->q_flag & HBA_EQ_DELAY_CHK &&
16215	phba->cfg_auto_imax &&
16216	fpeq->q_mode != LPFC_MAX_AUTO_EQ_DELAY &&
16217	phba->sli.sli_flag & LPFC_SLI_USE_EQDR)
16218	lpfc_sli4_mod_hba_eq_delay(phba, eq: fpeq, LPFC_MAX_AUTO_EQ_DELAY);
16219
16220	/* process and rearm the EQ */
16221	ecount = lpfc_sli4_process_eq(phba, eq: fpeq, LPFC_QUEUE_REARM,
16222	poll_mode: LPFC_THREADED_IRQ);
16223
16224	if (unlikely(ecount == 0)) {
16225	fpeq->EQ_no_entry++;
16226	if (phba->intr_type == MSIX)
16227	/* MSI-X treated interrupt served as no EQ share INT */
16228	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
16229	"3358 MSI-X interrupt with no EQE\n");
16230	else
16231	/* Non MSI-X treated on interrupt as EQ share INT */
16232	return IRQ_NONE;
16233	}
16234	return IRQ_HANDLED;
16235	}
16236
16237	/**
16238	* lpfc_cq_create - Create a Completion Queue on the HBA
16239	* @phba: HBA structure that indicates port to create a queue on.
16240	* @cq: The queue structure to use to create the completion queue.
16241	* @eq: The event queue to bind this completion queue to.
16242	* @type: Type of queue (EQ, GCQ, MCQ, WCQ, etc).
16243	* @subtype: Functional purpose of the queue (MBOX, IO, ELS, NVMET, etc).
16244	*
16245	* This function creates a completion queue, as detailed in @wq, on a port,
16246	* described by @phba by sending a CQ_CREATE mailbox command to the HBA.
16247	*
16248	* The @phba struct is used to send mailbox command to HBA. The @cq struct
16249	* is used to get the entry count and entry size that are necessary to
16250	* determine the number of pages to allocate and use for this queue. The @eq
16251	* is used to indicate which event queue to bind this completion queue to. This
16252	* function will send the CQ_CREATE mailbox command to the HBA to setup the
16253	* completion queue. This function is asynchronous and will wait for the mailbox
16254	* command to finish before continuing.
16255	*
16256	* On success this function will return a zero. If unable to allocate enough
16257	* memory this function will return -ENOMEM. If the queue create mailbox command
16258	* fails this function will return -ENXIO.
16259	**/
16260	int
16261	lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq,
16262	struct lpfc_queue *eq, uint32_t type, uint32_t subtype)
16263	{
16264	struct lpfc_mbx_cq_create *cq_create;
16265	struct lpfc_dmabuf *dmabuf;
16266	LPFC_MBOXQ_t *mbox;
16267	int rc, length, status = 0;
16268	uint32_t shdr_status, shdr_add_status;
16269	union lpfc_sli4_cfg_shdr *shdr;
16270
16271	/* sanity check on queue memory */
16272	if (!cq || !eq)
16273	return -ENODEV;
16274
16275	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
16276	if (!mbox)
16277	return -ENOMEM;
16278	length = (sizeof(struct lpfc_mbx_cq_create) -
16279	sizeof(struct lpfc_sli4_cfg_mhdr));
16280	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
16281	LPFC_MBOX_OPCODE_CQ_CREATE,
16282	length, LPFC_SLI4_MBX_EMBED);
16283	cq_create = &mbox->u.mqe.un.cq_create;
16284	shdr = (union lpfc_sli4_cfg_shdr *) &cq_create->header.cfg_shdr;
16285	bf_set(lpfc_mbx_cq_create_num_pages, &cq_create->u.request,
16286	cq->page_count);
16287	bf_set(lpfc_cq_context_event, &cq_create->u.request.context, 1);
16288	bf_set(lpfc_cq_context_valid, &cq_create->u.request.context, 1);
16289	bf_set(lpfc_mbox_hdr_version, &shdr->request,
16290	phba->sli4_hba.pc_sli4_params.cqv);
16291	if (phba->sli4_hba.pc_sli4_params.cqv == LPFC_Q_CREATE_VERSION_2) {
16292	bf_set(lpfc_mbx_cq_create_page_size, &cq_create->u.request,
16293	(cq->page_size / SLI4_PAGE_SIZE));
16294	bf_set(lpfc_cq_eq_id_2, &cq_create->u.request.context,
16295	eq->queue_id);
16296	bf_set(lpfc_cq_context_autovalid, &cq_create->u.request.context,
16297	phba->sli4_hba.pc_sli4_params.cqav);
16298	} else {
16299	bf_set(lpfc_cq_eq_id, &cq_create->u.request.context,
16300	eq->queue_id);
16301	}
16302	switch (cq->entry_count) {
16303	case 2048:
16304	case 4096:
16305	if (phba->sli4_hba.pc_sli4_params.cqv ==
16306	LPFC_Q_CREATE_VERSION_2) {
16307	cq_create->u.request.context.lpfc_cq_context_count =
16308	cq->entry_count;
16309	bf_set(lpfc_cq_context_count,
16310	&cq_create->u.request.context,
16311	LPFC_CQ_CNT_WORD7);
16312	break;
16313	}
16314	fallthrough;
16315	default:
16316	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16317	"0361 Unsupported CQ count: "
16318	"entry cnt %d sz %d pg cnt %d\n",
16319	cq->entry_count, cq->entry_size,
16320	cq->page_count);
16321	if (cq->entry_count < 256) {
16322	status = -EINVAL;
16323	goto out;
16324	}
16325	fallthrough; /* otherwise default to smallest count */
16326	case 256:
16327	bf_set(lpfc_cq_context_count, &cq_create->u.request.context,
16328	LPFC_CQ_CNT_256);
16329	break;
16330	case 512:
16331	bf_set(lpfc_cq_context_count, &cq_create->u.request.context,
16332	LPFC_CQ_CNT_512);
16333	break;
16334	case 1024:
16335	bf_set(lpfc_cq_context_count, &cq_create->u.request.context,
16336	LPFC_CQ_CNT_1024);
16337	break;
16338	}
16339	list_for_each_entry(dmabuf, &cq->page_list, list) {
16340	memset(dmabuf->virt, 0, cq->page_size);
16341	cq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
16342	putPaddrLow(dmabuf->phys);
16343	cq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
16344	putPaddrHigh(dmabuf->phys);
16345	}
16346	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
16347
16348	/* The IOCTL status is embedded in the mailbox subheader. */
16349	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
16350	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
16351	if (shdr_status || shdr_add_status || rc) {
16352	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16353	"2501 CQ_CREATE mailbox failed with "
16354	"status x%x add_status x%x, mbx status x%x\n",
16355	shdr_status, shdr_add_status, rc);
16356	status = -ENXIO;
16357	goto out;
16358	}
16359	cq->queue_id = bf_get(lpfc_mbx_cq_create_q_id, &cq_create->u.response);
16360	if (cq->queue_id == 0xFFFF) {
16361	status = -ENXIO;
16362	goto out;
16363	}
16364	/* link the cq onto the parent eq child list */
16365	list_add_tail(new: &cq->list, head: &eq->child_list);
16366	/* Set up completion queue's type and subtype */
16367	cq->type = type;
16368	cq->subtype = subtype;
16369	cq->queue_id = bf_get(lpfc_mbx_cq_create_q_id, &cq_create->u.response);
16370	cq->assoc_qid = eq->queue_id;
16371	cq->assoc_qp = eq;
16372	cq->host_index = 0;
16373	cq->notify_interval = LPFC_CQ_NOTIFY_INTRVL;
16374	cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit, cq->entry_count);
16375
16376	if (cq->queue_id > phba->sli4_hba.cq_max)
16377	phba->sli4_hba.cq_max = cq->queue_id;
16378	out:
16379	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
16380	return status;
16381	}
16382
16383	/**
16384	* lpfc_cq_create_set - Create a set of Completion Queues on the HBA for MRQ
16385	* @phba: HBA structure that indicates port to create a queue on.
16386	* @cqp: The queue structure array to use to create the completion queues.
16387	* @hdwq: The hardware queue array with the EQ to bind completion queues to.
16388	* @type: Type of queue (EQ, GCQ, MCQ, WCQ, etc).
16389	* @subtype: Functional purpose of the queue (MBOX, IO, ELS, NVMET, etc).
16390	*
16391	* This function creates a set of completion queue, s to support MRQ
16392	* as detailed in @cqp, on a port,
16393	* described by @phba by sending a CREATE_CQ_SET mailbox command to the HBA.
16394	*
16395	* The @phba struct is used to send mailbox command to HBA. The @cq struct
16396	* is used to get the entry count and entry size that are necessary to
16397	* determine the number of pages to allocate and use for this queue. The @eq
16398	* is used to indicate which event queue to bind this completion queue to. This
16399	* function will send the CREATE_CQ_SET mailbox command to the HBA to setup the
16400	* completion queue. This function is asynchronous and will wait for the mailbox
16401	* command to finish before continuing.
16402	*
16403	* On success this function will return a zero. If unable to allocate enough
16404	* memory this function will return -ENOMEM. If the queue create mailbox command
16405	* fails this function will return -ENXIO.
16406	**/
16407	int
16408	lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp,
16409	struct lpfc_sli4_hdw_queue *hdwq, uint32_t type,
16410	uint32_t subtype)
16411	{
16412	struct lpfc_queue *cq;
16413	struct lpfc_queue *eq;
16414	struct lpfc_mbx_cq_create_set *cq_set;
16415	struct lpfc_dmabuf *dmabuf;
16416	LPFC_MBOXQ_t *mbox;
16417	int rc, length, alloclen, status = 0;
16418	int cnt, idx, numcq, page_idx = 0;
16419	uint32_t shdr_status, shdr_add_status;
16420	union lpfc_sli4_cfg_shdr *shdr;
16421	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
16422
16423	/* sanity check on queue memory */
16424	numcq = phba->cfg_nvmet_mrq;
16425	if (!cqp || !hdwq || !numcq)
16426	return -ENODEV;
16427
16428	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
16429	if (!mbox)
16430	return -ENOMEM;
16431
16432	length = sizeof(struct lpfc_mbx_cq_create_set);
16433	length += ((numcq * cqp[0]->page_count) *
16434	sizeof(struct dma_address));
16435	alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
16436	LPFC_MBOX_OPCODE_FCOE_CQ_CREATE_SET, length,
16437	LPFC_SLI4_MBX_NEMBED);
16438	if (alloclen < length) {
16439	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16440	"3098 Allocated DMA memory size (%d) is "
16441	"less than the requested DMA memory size "
16442	"(%d)\n", alloclen, length);
16443	status = -ENOMEM;
16444	goto out;
16445	}
16446	cq_set = mbox->sge_array->addr[0];
16447	shdr = (union lpfc_sli4_cfg_shdr *)&cq_set->cfg_shdr;
16448	bf_set(lpfc_mbox_hdr_version, &shdr->request, 0);
16449
16450	for (idx = 0; idx < numcq; idx++) {
16451	cq = cqp[idx];
16452	eq = hdwq[idx].hba_eq;
16453	if (!cq || !eq) {
16454	status = -ENOMEM;
16455	goto out;
16456	}
16457	if (!phba->sli4_hba.pc_sli4_params.supported)
16458	hw_page_size = cq->page_size;
16459
16460	switch (idx) {
16461	case 0:
16462	bf_set(lpfc_mbx_cq_create_set_page_size,
16463	&cq_set->u.request,
16464	(hw_page_size / SLI4_PAGE_SIZE));
16465	bf_set(lpfc_mbx_cq_create_set_num_pages,
16466	&cq_set->u.request, cq->page_count);
16467	bf_set(lpfc_mbx_cq_create_set_evt,
16468	&cq_set->u.request, 1);
16469	bf_set(lpfc_mbx_cq_create_set_valid,
16470	&cq_set->u.request, 1);
16471	bf_set(lpfc_mbx_cq_create_set_cqe_size,
16472	&cq_set->u.request, 0);
16473	bf_set(lpfc_mbx_cq_create_set_num_cq,
16474	&cq_set->u.request, numcq);
16475	bf_set(lpfc_mbx_cq_create_set_autovalid,
16476	&cq_set->u.request,
16477	phba->sli4_hba.pc_sli4_params.cqav);
16478	switch (cq->entry_count) {
16479	case 2048:
16480	case 4096:
16481	if (phba->sli4_hba.pc_sli4_params.cqv ==
16482	LPFC_Q_CREATE_VERSION_2) {
16483	bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
16484	&cq_set->u.request,
16485	cq->entry_count);
16486	bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
16487	&cq_set->u.request,
16488	LPFC_CQ_CNT_WORD7);
16489	break;
16490	}
16491	fallthrough;
16492	default:
16493	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16494	"3118 Bad CQ count. (%d)\n",
16495	cq->entry_count);
16496	if (cq->entry_count < 256) {
16497	status = -EINVAL;
16498	goto out;
16499	}
16500	fallthrough; /* otherwise default to smallest */
16501	case 256:
16502	bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
16503	&cq_set->u.request, LPFC_CQ_CNT_256);
16504	break;
16505	case 512:
16506	bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
16507	&cq_set->u.request, LPFC_CQ_CNT_512);
16508	break;
16509	case 1024:
16510	bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
16511	&cq_set->u.request, LPFC_CQ_CNT_1024);
16512	break;
16513	}
16514	bf_set(lpfc_mbx_cq_create_set_eq_id0,
16515	&cq_set->u.request, eq->queue_id);
16516	break;
16517	case 1:
16518	bf_set(lpfc_mbx_cq_create_set_eq_id1,
16519	&cq_set->u.request, eq->queue_id);
16520	break;
16521	case 2:
16522	bf_set(lpfc_mbx_cq_create_set_eq_id2,
16523	&cq_set->u.request, eq->queue_id);
16524	break;
16525	case 3:
16526	bf_set(lpfc_mbx_cq_create_set_eq_id3,
16527	&cq_set->u.request, eq->queue_id);
16528	break;
16529	case 4:
16530	bf_set(lpfc_mbx_cq_create_set_eq_id4,
16531	&cq_set->u.request, eq->queue_id);
16532	break;
16533	case 5:
16534	bf_set(lpfc_mbx_cq_create_set_eq_id5,
16535	&cq_set->u.request, eq->queue_id);
16536	break;
16537	case 6:
16538	bf_set(lpfc_mbx_cq_create_set_eq_id6,
16539	&cq_set->u.request, eq->queue_id);
16540	break;
16541	case 7:
16542	bf_set(lpfc_mbx_cq_create_set_eq_id7,
16543	&cq_set->u.request, eq->queue_id);
16544	break;
16545	case 8:
16546	bf_set(lpfc_mbx_cq_create_set_eq_id8,
16547	&cq_set->u.request, eq->queue_id);
16548	break;
16549	case 9:
16550	bf_set(lpfc_mbx_cq_create_set_eq_id9,
16551	&cq_set->u.request, eq->queue_id);
16552	break;
16553	case 10:
16554	bf_set(lpfc_mbx_cq_create_set_eq_id10,
16555	&cq_set->u.request, eq->queue_id);
16556	break;
16557	case 11:
16558	bf_set(lpfc_mbx_cq_create_set_eq_id11,
16559	&cq_set->u.request, eq->queue_id);
16560	break;
16561	case 12:
16562	bf_set(lpfc_mbx_cq_create_set_eq_id12,
16563	&cq_set->u.request, eq->queue_id);
16564	break;
16565	case 13:
16566	bf_set(lpfc_mbx_cq_create_set_eq_id13,
16567	&cq_set->u.request, eq->queue_id);
16568	break;
16569	case 14:
16570	bf_set(lpfc_mbx_cq_create_set_eq_id14,
16571	&cq_set->u.request, eq->queue_id);
16572	break;
16573	case 15:
16574	bf_set(lpfc_mbx_cq_create_set_eq_id15,
16575	&cq_set->u.request, eq->queue_id);
16576	break;
16577	}
16578
16579	/* link the cq onto the parent eq child list */
16580	list_add_tail(new: &cq->list, head: &eq->child_list);
16581	/* Set up completion queue's type and subtype */
16582	cq->type = type;
16583	cq->subtype = subtype;
16584	cq->assoc_qid = eq->queue_id;
16585	cq->assoc_qp = eq;
16586	cq->host_index = 0;
16587	cq->notify_interval = LPFC_CQ_NOTIFY_INTRVL;
16588	cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit,
16589	cq->entry_count);
16590	cq->chann = idx;
16591
16592	rc = 0;
16593	list_for_each_entry(dmabuf, &cq->page_list, list) {
16594	memset(dmabuf->virt, 0, hw_page_size);
16595	cnt = page_idx + dmabuf->buffer_tag;
16596	cq_set->u.request.page[cnt].addr_lo =
16597	putPaddrLow(dmabuf->phys);
16598	cq_set->u.request.page[cnt].addr_hi =
16599	putPaddrHigh(dmabuf->phys);
16600	rc++;
16601	}
16602	page_idx += rc;
16603	}
16604
16605	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
16606
16607	/* The IOCTL status is embedded in the mailbox subheader. */
16608	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
16609	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
16610	if (shdr_status || shdr_add_status || rc) {
16611	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16612	"3119 CQ_CREATE_SET mailbox failed with "
16613	"status x%x add_status x%x, mbx status x%x\n",
16614	shdr_status, shdr_add_status, rc);
16615	status = -ENXIO;
16616	goto out;
16617	}
16618	rc = bf_get(lpfc_mbx_cq_create_set_base_id, &cq_set->u.response);
16619	if (rc == 0xFFFF) {
16620	status = -ENXIO;
16621	goto out;
16622	}
16623
16624	for (idx = 0; idx < numcq; idx++) {
16625	cq = cqp[idx];
16626	cq->queue_id = rc + idx;
16627	if (cq->queue_id > phba->sli4_hba.cq_max)
16628	phba->sli4_hba.cq_max = cq->queue_id;
16629	}
16630
16631	out:
16632	lpfc_sli4_mbox_cmd_free(phba, mbox);
16633	return status;
16634	}
16635
16636	/**
16637	* lpfc_mq_create_fb_init - Send MCC_CREATE without async events registration
16638	* @phba: HBA structure that indicates port to create a queue on.
16639	* @mq: The queue structure to use to create the mailbox queue.
16640	* @mbox: An allocated pointer to type LPFC_MBOXQ_t
16641	* @cq: The completion queue to associate with this cq.
16642	*
16643	* This function provides failback (fb) functionality when the
16644	* mq_create_ext fails on older FW generations. It's purpose is identical
16645	* to mq_create_ext otherwise.
16646	*
16647	* This routine cannot fail as all attributes were previously accessed and
16648	* initialized in mq_create_ext.
16649	**/
16650	static void
16651	lpfc_mq_create_fb_init(struct lpfc_hba *phba, struct lpfc_queue *mq,
16652	LPFC_MBOXQ_t *mbox, struct lpfc_queue *cq)
16653	{
16654	struct lpfc_mbx_mq_create *mq_create;
16655	struct lpfc_dmabuf *dmabuf;
16656	int length;
16657
16658	length = (sizeof(struct lpfc_mbx_mq_create) -
16659	sizeof(struct lpfc_sli4_cfg_mhdr));
16660	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
16661	LPFC_MBOX_OPCODE_MQ_CREATE,
16662	length, LPFC_SLI4_MBX_EMBED);
16663	mq_create = &mbox->u.mqe.un.mq_create;
16664	bf_set(lpfc_mbx_mq_create_num_pages, &mq_create->u.request,
16665	mq->page_count);
16666	bf_set(lpfc_mq_context_cq_id, &mq_create->u.request.context,
16667	cq->queue_id);
16668	bf_set(lpfc_mq_context_valid, &mq_create->u.request.context, 1);
16669	switch (mq->entry_count) {
16670	case 16:
16671	bf_set(lpfc_mq_context_ring_size, &mq_create->u.request.context,
16672	LPFC_MQ_RING_SIZE_16);
16673	break;
16674	case 32:
16675	bf_set(lpfc_mq_context_ring_size, &mq_create->u.request.context,
16676	LPFC_MQ_RING_SIZE_32);
16677	break;
16678	case 64:
16679	bf_set(lpfc_mq_context_ring_size, &mq_create->u.request.context,
16680	LPFC_MQ_RING_SIZE_64);
16681	break;
16682	case 128:
16683	bf_set(lpfc_mq_context_ring_size, &mq_create->u.request.context,
16684	LPFC_MQ_RING_SIZE_128);
16685	break;
16686	}
16687	list_for_each_entry(dmabuf, &mq->page_list, list) {
16688	mq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
16689	putPaddrLow(dmabuf->phys);
16690	mq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
16691	putPaddrHigh(dmabuf->phys);
16692	}
16693	}
16694
16695	/**
16696	* lpfc_mq_create - Create a mailbox Queue on the HBA
16697	* @phba: HBA structure that indicates port to create a queue on.
16698	* @mq: The queue structure to use to create the mailbox queue.
16699	* @cq: The completion queue to associate with this cq.
16700	* @subtype: The queue's subtype.
16701	*
16702	* This function creates a mailbox queue, as detailed in @mq, on a port,
16703	* described by @phba by sending a MQ_CREATE mailbox command to the HBA.
16704	*
16705	* The @phba struct is used to send mailbox command to HBA. The @cq struct
16706	* is used to get the entry count and entry size that are necessary to
16707	* determine the number of pages to allocate and use for this queue. This
16708	* function will send the MQ_CREATE mailbox command to the HBA to setup the
16709	* mailbox queue. This function is asynchronous and will wait for the mailbox
16710	* command to finish before continuing.
16711	*
16712	* On success this function will return a zero. If unable to allocate enough
16713	* memory this function will return -ENOMEM. If the queue create mailbox command
16714	* fails this function will return -ENXIO.
16715	**/
16716	int32_t
16717	lpfc_mq_create(struct lpfc_hba *phba, struct lpfc_queue *mq,
16718	struct lpfc_queue *cq, uint32_t subtype)
16719	{
16720	struct lpfc_mbx_mq_create *mq_create;
16721	struct lpfc_mbx_mq_create_ext *mq_create_ext;
16722	struct lpfc_dmabuf *dmabuf;
16723	LPFC_MBOXQ_t *mbox;
16724	int rc, length, status = 0;
16725	uint32_t shdr_status, shdr_add_status;
16726	union lpfc_sli4_cfg_shdr *shdr;
16727	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
16728
16729	/* sanity check on queue memory */
16730	if (!mq || !cq)
16731	return -ENODEV;
16732	if (!phba->sli4_hba.pc_sli4_params.supported)
16733	hw_page_size = SLI4_PAGE_SIZE;
16734
16735	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
16736	if (!mbox)
16737	return -ENOMEM;
16738	length = (sizeof(struct lpfc_mbx_mq_create_ext) -
16739	sizeof(struct lpfc_sli4_cfg_mhdr));
16740	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
16741	LPFC_MBOX_OPCODE_MQ_CREATE_EXT,
16742	length, LPFC_SLI4_MBX_EMBED);
16743
16744	mq_create_ext = &mbox->u.mqe.un.mq_create_ext;
16745	shdr = (union lpfc_sli4_cfg_shdr *) &mq_create_ext->header.cfg_shdr;
16746	bf_set(lpfc_mbx_mq_create_ext_num_pages,
16747	&mq_create_ext->u.request, mq->page_count);
16748	bf_set(lpfc_mbx_mq_create_ext_async_evt_link,
16749	&mq_create_ext->u.request, 1);
16750	bf_set(lpfc_mbx_mq_create_ext_async_evt_fip,
16751	&mq_create_ext->u.request, 1);
16752	bf_set(lpfc_mbx_mq_create_ext_async_evt_group5,
16753	&mq_create_ext->u.request, 1);
16754	bf_set(lpfc_mbx_mq_create_ext_async_evt_fc,
16755	&mq_create_ext->u.request, 1);
16756	bf_set(lpfc_mbx_mq_create_ext_async_evt_sli,
16757	&mq_create_ext->u.request, 1);
16758	bf_set(lpfc_mq_context_valid, &mq_create_ext->u.request.context, 1);
16759	bf_set(lpfc_mbox_hdr_version, &shdr->request,
16760	phba->sli4_hba.pc_sli4_params.mqv);
16761	if (phba->sli4_hba.pc_sli4_params.mqv == LPFC_Q_CREATE_VERSION_1)
16762	bf_set(lpfc_mbx_mq_create_ext_cq_id, &mq_create_ext->u.request,
16763	cq->queue_id);
16764	else
16765	bf_set(lpfc_mq_context_cq_id, &mq_create_ext->u.request.context,
16766	cq->queue_id);
16767	switch (mq->entry_count) {
16768	default:
16769	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16770	"0362 Unsupported MQ count. (%d)\n",
16771	mq->entry_count);
16772	if (mq->entry_count < 16) {
16773	status = -EINVAL;
16774	goto out;
16775	}
16776	fallthrough; /* otherwise default to smallest count */
16777	case 16:
16778	bf_set(lpfc_mq_context_ring_size,
16779	&mq_create_ext->u.request.context,
16780	LPFC_MQ_RING_SIZE_16);
16781	break;
16782	case 32:
16783	bf_set(lpfc_mq_context_ring_size,
16784	&mq_create_ext->u.request.context,
16785	LPFC_MQ_RING_SIZE_32);
16786	break;
16787	case 64:
16788	bf_set(lpfc_mq_context_ring_size,
16789	&mq_create_ext->u.request.context,
16790	LPFC_MQ_RING_SIZE_64);
16791	break;
16792	case 128:
16793	bf_set(lpfc_mq_context_ring_size,
16794	&mq_create_ext->u.request.context,
16795	LPFC_MQ_RING_SIZE_128);
16796	break;
16797	}
16798	list_for_each_entry(dmabuf, &mq->page_list, list) {
16799	memset(dmabuf->virt, 0, hw_page_size);
16800	mq_create_ext->u.request.page[dmabuf->buffer_tag].addr_lo =
16801	putPaddrLow(dmabuf->phys);
16802	mq_create_ext->u.request.page[dmabuf->buffer_tag].addr_hi =
16803	putPaddrHigh(dmabuf->phys);
16804	}
16805	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
16806	mq->queue_id = bf_get(lpfc_mbx_mq_create_q_id,
16807	&mq_create_ext->u.response);
16808	if (rc != MBX_SUCCESS) {
16809	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
16810	"2795 MQ_CREATE_EXT failed with "
16811	"status x%x. Failback to MQ_CREATE.\n",
16812	rc);
16813	lpfc_mq_create_fb_init(phba, mq, mbox, cq);
16814	mq_create = &mbox->u.mqe.un.mq_create;
16815	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
16816	shdr = (union lpfc_sli4_cfg_shdr *) &mq_create->header.cfg_shdr;
16817	mq->queue_id = bf_get(lpfc_mbx_mq_create_q_id,
16818	&mq_create->u.response);
16819	}
16820
16821	/* The IOCTL status is embedded in the mailbox subheader. */
16822	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
16823	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
16824	if (shdr_status || shdr_add_status || rc) {
16825	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16826	"2502 MQ_CREATE mailbox failed with "
16827	"status x%x add_status x%x, mbx status x%x\n",
16828	shdr_status, shdr_add_status, rc);
16829	status = -ENXIO;
16830	goto out;
16831	}
16832	if (mq->queue_id == 0xFFFF) {
16833	status = -ENXIO;
16834	goto out;
16835	}
16836	mq->type = LPFC_MQ;
16837	mq->assoc_qid = cq->queue_id;
16838	mq->subtype = subtype;
16839	mq->host_index = 0;
16840	mq->hba_index = 0;
16841
16842	/* link the mq onto the parent cq child list */
16843	list_add_tail(new: &mq->list, head: &cq->child_list);
16844	out:
16845	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
16846	return status;
16847	}
16848
16849	/**
16850	* lpfc_wq_create - Create a Work Queue on the HBA
16851	* @phba: HBA structure that indicates port to create a queue on.
16852	* @wq: The queue structure to use to create the work queue.
16853	* @cq: The completion queue to bind this work queue to.
16854	* @subtype: The subtype of the work queue indicating its functionality.
16855	*
16856	* This function creates a work queue, as detailed in @wq, on a port, described
16857	* by @phba by sending a WQ_CREATE mailbox command to the HBA.
16858	*
16859	* The @phba struct is used to send mailbox command to HBA. The @wq struct
16860	* is used to get the entry count and entry size that are necessary to
16861	* determine the number of pages to allocate and use for this queue. The @cq
16862	* is used to indicate which completion queue to bind this work queue to. This
16863	* function will send the WQ_CREATE mailbox command to the HBA to setup the
16864	* work queue. This function is asynchronous and will wait for the mailbox
16865	* command to finish before continuing.
16866	*
16867	* On success this function will return a zero. If unable to allocate enough
16868	* memory this function will return -ENOMEM. If the queue create mailbox command
16869	* fails this function will return -ENXIO.
16870	**/
16871	int
16872	lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq,
16873	struct lpfc_queue *cq, uint32_t subtype)
16874	{
16875	struct lpfc_mbx_wq_create *wq_create;
16876	struct lpfc_dmabuf *dmabuf;
16877	LPFC_MBOXQ_t *mbox;
16878	int rc, length, status = 0;
16879	uint32_t shdr_status, shdr_add_status;
16880	union lpfc_sli4_cfg_shdr *shdr;
16881	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
16882	struct dma_address *page;
16883	void __iomem *bar_memmap_p;
16884	uint32_t db_offset;
16885	uint16_t pci_barset;
16886	uint8_t dpp_barset;
16887	uint32_t dpp_offset;
16888	uint8_t wq_create_version;
16889	#ifdef CONFIG_X86
16890	unsigned long pg_addr;
16891	#endif
16892
16893	/* sanity check on queue memory */
16894	if (!wq || !cq)
16895	return -ENODEV;
16896	if (!phba->sli4_hba.pc_sli4_params.supported)
16897	hw_page_size = wq->page_size;
16898
16899	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
16900	if (!mbox)
16901	return -ENOMEM;
16902	length = (sizeof(struct lpfc_mbx_wq_create) -
16903	sizeof(struct lpfc_sli4_cfg_mhdr));
16904	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
16905	LPFC_MBOX_OPCODE_FCOE_WQ_CREATE,
16906	length, LPFC_SLI4_MBX_EMBED);
16907	wq_create = &mbox->u.mqe.un.wq_create;
16908	shdr = (union lpfc_sli4_cfg_shdr *) &wq_create->header.cfg_shdr;
16909	bf_set(lpfc_mbx_wq_create_num_pages, &wq_create->u.request,
16910	wq->page_count);
16911	bf_set(lpfc_mbx_wq_create_cq_id, &wq_create->u.request,
16912	cq->queue_id);
16913
16914	/* wqv is the earliest version supported, NOT the latest */
16915	bf_set(lpfc_mbox_hdr_version, &shdr->request,
16916	phba->sli4_hba.pc_sli4_params.wqv);
16917
16918	if ((phba->sli4_hba.pc_sli4_params.wqsize & LPFC_WQ_SZ128_SUPPORT) ||
16919	(wq->page_size > SLI4_PAGE_SIZE))
16920	wq_create_version = LPFC_Q_CREATE_VERSION_1;
16921	else
16922	wq_create_version = LPFC_Q_CREATE_VERSION_0;
16923
16924	switch (wq_create_version) {
16925	case LPFC_Q_CREATE_VERSION_1:
16926	bf_set(lpfc_mbx_wq_create_wqe_count, &wq_create->u.request_1,
16927	wq->entry_count);
16928	bf_set(lpfc_mbox_hdr_version, &shdr->request,
16929	LPFC_Q_CREATE_VERSION_1);
16930
16931	switch (wq->entry_size) {
16932	default:
16933	case 64:
16934	bf_set(lpfc_mbx_wq_create_wqe_size,
16935	&wq_create->u.request_1,
16936	LPFC_WQ_WQE_SIZE_64);
16937	break;
16938	case 128:
16939	bf_set(lpfc_mbx_wq_create_wqe_size,
16940	&wq_create->u.request_1,
16941	LPFC_WQ_WQE_SIZE_128);
16942	break;
16943	}
16944	/* Request DPP by default */
16945	bf_set(lpfc_mbx_wq_create_dpp_req, &wq_create->u.request_1, 1);
16946	bf_set(lpfc_mbx_wq_create_page_size,
16947	&wq_create->u.request_1,
16948	(wq->page_size / SLI4_PAGE_SIZE));
16949	page = wq_create->u.request_1.page;
16950	break;
16951	default:
16952	page = wq_create->u.request.page;
16953	break;
16954	}
16955
16956	list_for_each_entry(dmabuf, &wq->page_list, list) {
16957	memset(dmabuf->virt, 0, hw_page_size);
16958	page[dmabuf->buffer_tag].addr_lo = putPaddrLow(dmabuf->phys);
16959	page[dmabuf->buffer_tag].addr_hi = putPaddrHigh(dmabuf->phys);
16960	}
16961
16962	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE)
16963	bf_set(lpfc_mbx_wq_create_dua, &wq_create->u.request, 1);
16964
16965	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
16966	/* The IOCTL status is embedded in the mailbox subheader. */
16967	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
16968	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
16969	if (shdr_status || shdr_add_status || rc) {
16970	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16971	"2503 WQ_CREATE mailbox failed with "
16972	"status x%x add_status x%x, mbx status x%x\n",
16973	shdr_status, shdr_add_status, rc);
16974	status = -ENXIO;
16975	goto out;
16976	}
16977
16978	if (wq_create_version == LPFC_Q_CREATE_VERSION_0)
16979	wq->queue_id = bf_get(lpfc_mbx_wq_create_q_id,
16980	&wq_create->u.response);
16981	else
16982	wq->queue_id = bf_get(lpfc_mbx_wq_create_v1_q_id,
16983	&wq_create->u.response_1);
16984
16985	if (wq->queue_id == 0xFFFF) {
16986	status = -ENXIO;
16987	goto out;
16988	}
16989
16990	wq->db_format = LPFC_DB_LIST_FORMAT;
16991	if (wq_create_version == LPFC_Q_CREATE_VERSION_0) {
16992	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE) {
16993	wq->db_format = bf_get(lpfc_mbx_wq_create_db_format,
16994	&wq_create->u.response);
16995	if ((wq->db_format != LPFC_DB_LIST_FORMAT) &&
16996	(wq->db_format != LPFC_DB_RING_FORMAT)) {
16997	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16998	"3265 WQ[%d] doorbell format "
16999	"not supported: x%x\n",
17000	wq->queue_id, wq->db_format);
17001	status = -EINVAL;
17002	goto out;
17003	}
17004	pci_barset = bf_get(lpfc_mbx_wq_create_bar_set,
17005	&wq_create->u.response);
17006	bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
17007	pci_barset);
17008	if (!bar_memmap_p) {
17009	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17010	"3263 WQ[%d] failed to memmap "
17011	"pci barset:x%x\n",
17012	wq->queue_id, pci_barset);
17013	status = -ENOMEM;
17014	goto out;
17015	}
17016	db_offset = wq_create->u.response.doorbell_offset;
17017	if ((db_offset != LPFC_ULP0_WQ_DOORBELL) &&
17018	(db_offset != LPFC_ULP1_WQ_DOORBELL)) {
17019	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17020	"3252 WQ[%d] doorbell offset "
17021	"not supported: x%x\n",
17022	wq->queue_id, db_offset);
17023	status = -EINVAL;
17024	goto out;
17025	}
17026	wq->db_regaddr = bar_memmap_p + db_offset;
17027	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
17028	"3264 WQ[%d]: barset:x%x, offset:x%x, "
17029	"format:x%x\n", wq->queue_id,
17030	pci_barset, db_offset, wq->db_format);
17031	} else
17032	wq->db_regaddr = phba->sli4_hba.WQDBregaddr;
17033	} else {
17034	/* Check if DPP was honored by the firmware */
17035	wq->dpp_enable = bf_get(lpfc_mbx_wq_create_dpp_rsp,
17036	&wq_create->u.response_1);
17037	if (wq->dpp_enable) {
17038	pci_barset = bf_get(lpfc_mbx_wq_create_v1_bar_set,
17039	&wq_create->u.response_1);
17040	bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
17041	pci_barset);
17042	if (!bar_memmap_p) {
17043	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17044	"3267 WQ[%d] failed to memmap "
17045	"pci barset:x%x\n",
17046	wq->queue_id, pci_barset);
17047	status = -ENOMEM;
17048	goto out;
17049	}
17050	db_offset = wq_create->u.response_1.doorbell_offset;
17051	wq->db_regaddr = bar_memmap_p + db_offset;
17052	wq->dpp_id = bf_get(lpfc_mbx_wq_create_dpp_id,
17053	&wq_create->u.response_1);
17054	dpp_barset = bf_get(lpfc_mbx_wq_create_dpp_bar,
17055	&wq_create->u.response_1);
17056	bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
17057	pci_barset: dpp_barset);
17058	if (!bar_memmap_p) {
17059	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17060	"3268 WQ[%d] failed to memmap "
17061	"pci barset:x%x\n",
17062	wq->queue_id, dpp_barset);
17063	status = -ENOMEM;
17064	goto out;
17065	}
17066	dpp_offset = wq_create->u.response_1.dpp_offset;
17067	wq->dpp_regaddr = bar_memmap_p + dpp_offset;
17068	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
17069	"3271 WQ[%d]: barset:x%x, offset:x%x, "
17070	"dpp_id:x%x dpp_barset:x%x "
17071	"dpp_offset:x%x\n",
17072	wq->queue_id, pci_barset, db_offset,
17073	wq->dpp_id, dpp_barset, dpp_offset);
17074
17075	#ifdef CONFIG_X86
17076	/* Enable combined writes for DPP aperture */
17077	pg_addr = (unsigned long)(wq->dpp_regaddr) & PAGE_MASK;
17078	rc = set_memory_wc(addr: pg_addr, numpages: 1);
17079	if (rc) {
17080	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
17081	"3272 Cannot setup Combined "
17082	"Write on WQ[%d] - disable DPP\n",
17083	wq->queue_id);
17084	phba->cfg_enable_dpp = 0;
17085	}
17086	#else
17087	phba->cfg_enable_dpp = 0;
17088	#endif
17089	} else
17090	wq->db_regaddr = phba->sli4_hba.WQDBregaddr;
17091	}
17092	wq->pring = kzalloc(size: sizeof(struct lpfc_sli_ring), GFP_KERNEL);
17093	if (wq->pring == NULL) {
17094	status = -ENOMEM;
17095	goto out;
17096	}
17097	wq->type = LPFC_WQ;
17098	wq->assoc_qid = cq->queue_id;
17099	wq->subtype = subtype;
17100	wq->host_index = 0;
17101	wq->hba_index = 0;
17102	wq->notify_interval = LPFC_WQ_NOTIFY_INTRVL;
17103
17104	/* link the wq onto the parent cq child list */
17105	list_add_tail(new: &wq->list, head: &cq->child_list);
17106	out:
17107	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
17108	return status;
17109	}
17110
17111	/**
17112	* lpfc_rq_create - Create a Receive Queue on the HBA
17113	* @phba: HBA structure that indicates port to create a queue on.
17114	* @hrq: The queue structure to use to create the header receive queue.
17115	* @drq: The queue structure to use to create the data receive queue.
17116	* @cq: The completion queue to bind this work queue to.
17117	* @subtype: The subtype of the work queue indicating its functionality.
17118	*
17119	* This function creates a receive buffer queue pair , as detailed in @hrq and
17120	* @drq, on a port, described by @phba by sending a RQ_CREATE mailbox command
17121	* to the HBA.
17122	*
17123	* The @phba struct is used to send mailbox command to HBA. The @drq and @hrq
17124	* struct is used to get the entry count that is necessary to determine the
17125	* number of pages to use for this queue. The @cq is used to indicate which
17126	* completion queue to bind received buffers that are posted to these queues to.
17127	* This function will send the RQ_CREATE mailbox command to the HBA to setup the
17128	* receive queue pair. This function is asynchronous and will wait for the
17129	* mailbox command to finish before continuing.
17130	*
17131	* On success this function will return a zero. If unable to allocate enough
17132	* memory this function will return -ENOMEM. If the queue create mailbox command
17133	* fails this function will return -ENXIO.
17134	**/
17135	int
17136	lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq,
17137	struct lpfc_queue *drq, struct lpfc_queue *cq, uint32_t subtype)
17138	{
17139	struct lpfc_mbx_rq_create *rq_create;
17140	struct lpfc_dmabuf *dmabuf;
17141	LPFC_MBOXQ_t *mbox;
17142	int rc, length, status = 0;
17143	uint32_t shdr_status, shdr_add_status;
17144	union lpfc_sli4_cfg_shdr *shdr;
17145	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
17146	void __iomem *bar_memmap_p;
17147	uint32_t db_offset;
17148	uint16_t pci_barset;
17149
17150	/* sanity check on queue memory */
17151	if (!hrq || !drq || !cq)
17152	return -ENODEV;
17153	if (!phba->sli4_hba.pc_sli4_params.supported)
17154	hw_page_size = SLI4_PAGE_SIZE;
17155
17156	if (hrq->entry_count != drq->entry_count)
17157	return -EINVAL;
17158	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
17159	if (!mbox)
17160	return -ENOMEM;
17161	length = (sizeof(struct lpfc_mbx_rq_create) -
17162	sizeof(struct lpfc_sli4_cfg_mhdr));
17163	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
17164	LPFC_MBOX_OPCODE_FCOE_RQ_CREATE,
17165	length, LPFC_SLI4_MBX_EMBED);
17166	rq_create = &mbox->u.mqe.un.rq_create;
17167	shdr = (union lpfc_sli4_cfg_shdr *) &rq_create->header.cfg_shdr;
17168	bf_set(lpfc_mbox_hdr_version, &shdr->request,
17169	phba->sli4_hba.pc_sli4_params.rqv);
17170	if (phba->sli4_hba.pc_sli4_params.rqv == LPFC_Q_CREATE_VERSION_1) {
17171	bf_set(lpfc_rq_context_rqe_count_1,
17172	&rq_create->u.request.context,
17173	hrq->entry_count);
17174	rq_create->u.request.context.buffer_size = LPFC_HDR_BUF_SIZE;
17175	bf_set(lpfc_rq_context_rqe_size,
17176	&rq_create->u.request.context,
17177	LPFC_RQE_SIZE_8);
17178	bf_set(lpfc_rq_context_page_size,
17179	&rq_create->u.request.context,
17180	LPFC_RQ_PAGE_SIZE_4096);
17181	} else {
17182	switch (hrq->entry_count) {
17183	default:
17184	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17185	"2535 Unsupported RQ count. (%d)\n",
17186	hrq->entry_count);
17187	if (hrq->entry_count < 512) {
17188	status = -EINVAL;
17189	goto out;
17190	}
17191	fallthrough; /* otherwise default to smallest count */
17192	case 512:
17193	bf_set(lpfc_rq_context_rqe_count,
17194	&rq_create->u.request.context,
17195	LPFC_RQ_RING_SIZE_512);
17196	break;
17197	case 1024:
17198	bf_set(lpfc_rq_context_rqe_count,
17199	&rq_create->u.request.context,
17200	LPFC_RQ_RING_SIZE_1024);
17201	break;
17202	case 2048:
17203	bf_set(lpfc_rq_context_rqe_count,
17204	&rq_create->u.request.context,
17205	LPFC_RQ_RING_SIZE_2048);
17206	break;
17207	case 4096:
17208	bf_set(lpfc_rq_context_rqe_count,
17209	&rq_create->u.request.context,
17210	LPFC_RQ_RING_SIZE_4096);
17211	break;
17212	}
17213	bf_set(lpfc_rq_context_buf_size, &rq_create->u.request.context,
17214	LPFC_HDR_BUF_SIZE);
17215	}
17216	bf_set(lpfc_rq_context_cq_id, &rq_create->u.request.context,
17217	cq->queue_id);
17218	bf_set(lpfc_mbx_rq_create_num_pages, &rq_create->u.request,
17219	hrq->page_count);
17220	list_for_each_entry(dmabuf, &hrq->page_list, list) {
17221	memset(dmabuf->virt, 0, hw_page_size);
17222	rq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
17223	putPaddrLow(dmabuf->phys);
17224	rq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
17225	putPaddrHigh(dmabuf->phys);
17226	}
17227	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE)
17228	bf_set(lpfc_mbx_rq_create_dua, &rq_create->u.request, 1);
17229
17230	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
17231	/* The IOCTL status is embedded in the mailbox subheader. */
17232	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17233	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17234	if (shdr_status || shdr_add_status || rc) {
17235	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17236	"2504 RQ_CREATE mailbox failed with "
17237	"status x%x add_status x%x, mbx status x%x\n",
17238	shdr_status, shdr_add_status, rc);
17239	status = -ENXIO;
17240	goto out;
17241	}
17242	hrq->queue_id = bf_get(lpfc_mbx_rq_create_q_id, &rq_create->u.response);
17243	if (hrq->queue_id == 0xFFFF) {
17244	status = -ENXIO;
17245	goto out;
17246	}
17247
17248	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE) {
17249	hrq->db_format = bf_get(lpfc_mbx_rq_create_db_format,
17250	&rq_create->u.response);
17251	if ((hrq->db_format != LPFC_DB_LIST_FORMAT) &&
17252	(hrq->db_format != LPFC_DB_RING_FORMAT)) {
17253	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17254	"3262 RQ [%d] doorbell format not "
17255	"supported: x%x\n", hrq->queue_id,
17256	hrq->db_format);
17257	status = -EINVAL;
17258	goto out;
17259	}
17260
17261	pci_barset = bf_get(lpfc_mbx_rq_create_bar_set,
17262	&rq_create->u.response);
17263	bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba, pci_barset);
17264	if (!bar_memmap_p) {
17265	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17266	"3269 RQ[%d] failed to memmap pci "
17267	"barset:x%x\n", hrq->queue_id,
17268	pci_barset);
17269	status = -ENOMEM;
17270	goto out;
17271	}
17272
17273	db_offset = rq_create->u.response.doorbell_offset;
17274	if ((db_offset != LPFC_ULP0_RQ_DOORBELL) &&
17275	(db_offset != LPFC_ULP1_RQ_DOORBELL)) {
17276	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17277	"3270 RQ[%d] doorbell offset not "
17278	"supported: x%x\n", hrq->queue_id,
17279	db_offset);
17280	status = -EINVAL;
17281	goto out;
17282	}
17283	hrq->db_regaddr = bar_memmap_p + db_offset;
17284	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
17285	"3266 RQ[qid:%d]: barset:x%x, offset:x%x, "
17286	"format:x%x\n", hrq->queue_id, pci_barset,
17287	db_offset, hrq->db_format);
17288	} else {
17289	hrq->db_format = LPFC_DB_RING_FORMAT;
17290	hrq->db_regaddr = phba->sli4_hba.RQDBregaddr;
17291	}
17292	hrq->type = LPFC_HRQ;
17293	hrq->assoc_qid = cq->queue_id;
17294	hrq->subtype = subtype;
17295	hrq->host_index = 0;
17296	hrq->hba_index = 0;
17297	hrq->notify_interval = LPFC_RQ_NOTIFY_INTRVL;
17298
17299	/* now create the data queue */
17300	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
17301	LPFC_MBOX_OPCODE_FCOE_RQ_CREATE,
17302	length, LPFC_SLI4_MBX_EMBED);
17303	bf_set(lpfc_mbox_hdr_version, &shdr->request,
17304	phba->sli4_hba.pc_sli4_params.rqv);
17305	if (phba->sli4_hba.pc_sli4_params.rqv == LPFC_Q_CREATE_VERSION_1) {
17306	bf_set(lpfc_rq_context_rqe_count_1,
17307	&rq_create->u.request.context, hrq->entry_count);
17308	if (subtype == LPFC_NVMET)
17309	rq_create->u.request.context.buffer_size =
17310	LPFC_NVMET_DATA_BUF_SIZE;
17311	else
17312	rq_create->u.request.context.buffer_size =
17313	LPFC_DATA_BUF_SIZE;
17314	bf_set(lpfc_rq_context_rqe_size, &rq_create->u.request.context,
17315	LPFC_RQE_SIZE_8);
17316	bf_set(lpfc_rq_context_page_size, &rq_create->u.request.context,
17317	(PAGE_SIZE/SLI4_PAGE_SIZE));
17318	} else {
17319	switch (drq->entry_count) {
17320	default:
17321	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17322	"2536 Unsupported RQ count. (%d)\n",
17323	drq->entry_count);
17324	if (drq->entry_count < 512) {
17325	status = -EINVAL;
17326	goto out;
17327	}
17328	fallthrough; /* otherwise default to smallest count */
17329	case 512:
17330	bf_set(lpfc_rq_context_rqe_count,
17331	&rq_create->u.request.context,
17332	LPFC_RQ_RING_SIZE_512);
17333	break;
17334	case 1024:
17335	bf_set(lpfc_rq_context_rqe_count,
17336	&rq_create->u.request.context,
17337	LPFC_RQ_RING_SIZE_1024);
17338	break;
17339	case 2048:
17340	bf_set(lpfc_rq_context_rqe_count,
17341	&rq_create->u.request.context,
17342	LPFC_RQ_RING_SIZE_2048);
17343	break;
17344	case 4096:
17345	bf_set(lpfc_rq_context_rqe_count,
17346	&rq_create->u.request.context,
17347	LPFC_RQ_RING_SIZE_4096);
17348	break;
17349	}
17350	if (subtype == LPFC_NVMET)
17351	bf_set(lpfc_rq_context_buf_size,
17352	&rq_create->u.request.context,
17353	LPFC_NVMET_DATA_BUF_SIZE);
17354	else
17355	bf_set(lpfc_rq_context_buf_size,
17356	&rq_create->u.request.context,
17357	LPFC_DATA_BUF_SIZE);
17358	}
17359	bf_set(lpfc_rq_context_cq_id, &rq_create->u.request.context,
17360	cq->queue_id);
17361	bf_set(lpfc_mbx_rq_create_num_pages, &rq_create->u.request,
17362	drq->page_count);
17363	list_for_each_entry(dmabuf, &drq->page_list, list) {
17364	rq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
17365	putPaddrLow(dmabuf->phys);
17366	rq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
17367	putPaddrHigh(dmabuf->phys);
17368	}
17369	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE)
17370	bf_set(lpfc_mbx_rq_create_dua, &rq_create->u.request, 1);
17371	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
17372	/* The IOCTL status is embedded in the mailbox subheader. */
17373	shdr = (union lpfc_sli4_cfg_shdr *) &rq_create->header.cfg_shdr;
17374	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17375	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17376	if (shdr_status || shdr_add_status || rc) {
17377	status = -ENXIO;
17378	goto out;
17379	}
17380	drq->queue_id = bf_get(lpfc_mbx_rq_create_q_id, &rq_create->u.response);
17381	if (drq->queue_id == 0xFFFF) {
17382	status = -ENXIO;
17383	goto out;
17384	}
17385	drq->type = LPFC_DRQ;
17386	drq->assoc_qid = cq->queue_id;
17387	drq->subtype = subtype;
17388	drq->host_index = 0;
17389	drq->hba_index = 0;
17390	drq->notify_interval = LPFC_RQ_NOTIFY_INTRVL;
17391
17392	/* link the header and data RQs onto the parent cq child list */
17393	list_add_tail(new: &hrq->list, head: &cq->child_list);
17394	list_add_tail(new: &drq->list, head: &cq->child_list);
17395
17396	out:
17397	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
17398	return status;
17399	}
17400
17401	/**
17402	* lpfc_mrq_create - Create MRQ Receive Queues on the HBA
17403	* @phba: HBA structure that indicates port to create a queue on.
17404	* @hrqp: The queue structure array to use to create the header receive queues.
17405	* @drqp: The queue structure array to use to create the data receive queues.
17406	* @cqp: The completion queue array to bind these receive queues to.
17407	* @subtype: Functional purpose of the queue (MBOX, IO, ELS, NVMET, etc).
17408	*
17409	* This function creates a receive buffer queue pair , as detailed in @hrq and
17410	* @drq, on a port, described by @phba by sending a RQ_CREATE mailbox command
17411	* to the HBA.
17412	*
17413	* The @phba struct is used to send mailbox command to HBA. The @drq and @hrq
17414	* struct is used to get the entry count that is necessary to determine the
17415	* number of pages to use for this queue. The @cq is used to indicate which
17416	* completion queue to bind received buffers that are posted to these queues to.
17417	* This function will send the RQ_CREATE mailbox command to the HBA to setup the
17418	* receive queue pair. This function is asynchronous and will wait for the
17419	* mailbox command to finish before continuing.
17420	*
17421	* On success this function will return a zero. If unable to allocate enough
17422	* memory this function will return -ENOMEM. If the queue create mailbox command
17423	* fails this function will return -ENXIO.
17424	**/
17425	int
17426	lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp,
17427	struct lpfc_queue **drqp, struct lpfc_queue **cqp,
17428	uint32_t subtype)
17429	{
17430	struct lpfc_queue *hrq, *drq, *cq;
17431	struct lpfc_mbx_rq_create_v2 *rq_create;
17432	struct lpfc_dmabuf *dmabuf;
17433	LPFC_MBOXQ_t *mbox;
17434	int rc, length, alloclen, status = 0;
17435	int cnt, idx, numrq, page_idx = 0;
17436	uint32_t shdr_status, shdr_add_status;
17437	union lpfc_sli4_cfg_shdr *shdr;
17438	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
17439
17440	numrq = phba->cfg_nvmet_mrq;
17441	/* sanity check on array memory */
17442	if (!hrqp || !drqp || !cqp || !numrq)
17443	return -ENODEV;
17444	if (!phba->sli4_hba.pc_sli4_params.supported)
17445	hw_page_size = SLI4_PAGE_SIZE;
17446
17447	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
17448	if (!mbox)
17449	return -ENOMEM;
17450
17451	length = sizeof(struct lpfc_mbx_rq_create_v2);
17452	length += ((2 * numrq * hrqp[0]->page_count) *
17453	sizeof(struct dma_address));
17454
17455	alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
17456	LPFC_MBOX_OPCODE_FCOE_RQ_CREATE, length,
17457	LPFC_SLI4_MBX_NEMBED);
17458	if (alloclen < length) {
17459	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17460	"3099 Allocated DMA memory size (%d) is "
17461	"less than the requested DMA memory size "
17462	"(%d)\n", alloclen, length);
17463	status = -ENOMEM;
17464	goto out;
17465	}
17466
17467
17468
17469	rq_create = mbox->sge_array->addr[0];
17470	shdr = (union lpfc_sli4_cfg_shdr *)&rq_create->cfg_shdr;
17471
17472	bf_set(lpfc_mbox_hdr_version, &shdr->request, LPFC_Q_CREATE_VERSION_2);
17473	cnt = 0;
17474
17475	for (idx = 0; idx < numrq; idx++) {
17476	hrq = hrqp[idx];
17477	drq = drqp[idx];
17478	cq = cqp[idx];
17479
17480	/* sanity check on queue memory */
17481	if (!hrq || !drq || !cq) {
17482	status = -ENODEV;
17483	goto out;
17484	}
17485
17486	if (hrq->entry_count != drq->entry_count) {
17487	status = -EINVAL;
17488	goto out;
17489	}
17490
17491	if (idx == 0) {
17492	bf_set(lpfc_mbx_rq_create_num_pages,
17493	&rq_create->u.request,
17494	hrq->page_count);
17495	bf_set(lpfc_mbx_rq_create_rq_cnt,
17496	&rq_create->u.request, (numrq * 2));
17497	bf_set(lpfc_mbx_rq_create_dnb, &rq_create->u.request,
17498	1);
17499	bf_set(lpfc_rq_context_base_cq,
17500	&rq_create->u.request.context,
17501	cq->queue_id);
17502	bf_set(lpfc_rq_context_data_size,
17503	&rq_create->u.request.context,
17504	LPFC_NVMET_DATA_BUF_SIZE);
17505	bf_set(lpfc_rq_context_hdr_size,
17506	&rq_create->u.request.context,
17507	LPFC_HDR_BUF_SIZE);
17508	bf_set(lpfc_rq_context_rqe_count_1,
17509	&rq_create->u.request.context,
17510	hrq->entry_count);
17511	bf_set(lpfc_rq_context_rqe_size,
17512	&rq_create->u.request.context,
17513	LPFC_RQE_SIZE_8);
17514	bf_set(lpfc_rq_context_page_size,
17515	&rq_create->u.request.context,
17516	(PAGE_SIZE/SLI4_PAGE_SIZE));
17517	}
17518	rc = 0;
17519	list_for_each_entry(dmabuf, &hrq->page_list, list) {
17520	memset(dmabuf->virt, 0, hw_page_size);
17521	cnt = page_idx + dmabuf->buffer_tag;
17522	rq_create->u.request.page[cnt].addr_lo =
17523	putPaddrLow(dmabuf->phys);
17524	rq_create->u.request.page[cnt].addr_hi =
17525	putPaddrHigh(dmabuf->phys);
17526	rc++;
17527	}
17528	page_idx += rc;
17529
17530	rc = 0;
17531	list_for_each_entry(dmabuf, &drq->page_list, list) {
17532	memset(dmabuf->virt, 0, hw_page_size);
17533	cnt = page_idx + dmabuf->buffer_tag;
17534	rq_create->u.request.page[cnt].addr_lo =
17535	putPaddrLow(dmabuf->phys);
17536	rq_create->u.request.page[cnt].addr_hi =
17537	putPaddrHigh(dmabuf->phys);
17538	rc++;
17539	}
17540	page_idx += rc;
17541
17542	hrq->db_format = LPFC_DB_RING_FORMAT;
17543	hrq->db_regaddr = phba->sli4_hba.RQDBregaddr;
17544	hrq->type = LPFC_HRQ;
17545	hrq->assoc_qid = cq->queue_id;
17546	hrq->subtype = subtype;
17547	hrq->host_index = 0;
17548	hrq->hba_index = 0;
17549	hrq->notify_interval = LPFC_RQ_NOTIFY_INTRVL;
17550
17551	drq->db_format = LPFC_DB_RING_FORMAT;
17552	drq->db_regaddr = phba->sli4_hba.RQDBregaddr;
17553	drq->type = LPFC_DRQ;
17554	drq->assoc_qid = cq->queue_id;
17555	drq->subtype = subtype;
17556	drq->host_index = 0;
17557	drq->hba_index = 0;
17558	drq->notify_interval = LPFC_RQ_NOTIFY_INTRVL;
17559
17560	list_add_tail(new: &hrq->list, head: &cq->child_list);
17561	list_add_tail(new: &drq->list, head: &cq->child_list);
17562	}
17563
17564	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
17565	/* The IOCTL status is embedded in the mailbox subheader. */
17566	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17567	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17568	if (shdr_status || shdr_add_status || rc) {
17569	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17570	"3120 RQ_CREATE mailbox failed with "
17571	"status x%x add_status x%x, mbx status x%x\n",
17572	shdr_status, shdr_add_status, rc);
17573	status = -ENXIO;
17574	goto out;
17575	}
17576	rc = bf_get(lpfc_mbx_rq_create_q_id, &rq_create->u.response);
17577	if (rc == 0xFFFF) {
17578	status = -ENXIO;
17579	goto out;
17580	}
17581
17582	/* Initialize all RQs with associated queue id */
17583	for (idx = 0; idx < numrq; idx++) {
17584	hrq = hrqp[idx];
17585	hrq->queue_id = rc + (2 * idx);
17586	drq = drqp[idx];
17587	drq->queue_id = rc + (2 * idx) + 1;
17588	}
17589
17590	out:
17591	lpfc_sli4_mbox_cmd_free(phba, mbox);
17592	return status;
17593	}
17594
17595	/**
17596	* lpfc_eq_destroy - Destroy an event Queue on the HBA
17597	* @phba: HBA structure that indicates port to destroy a queue on.
17598	* @eq: The queue structure associated with the queue to destroy.
17599	*
17600	* This function destroys a queue, as detailed in @eq by sending an mailbox
17601	* command, specific to the type of queue, to the HBA.
17602	*
17603	* The @eq struct is used to get the queue ID of the queue to destroy.
17604	*
17605	* On success this function will return a zero. If the queue destroy mailbox
17606	* command fails this function will return -ENXIO.
17607	**/
17608	int
17609	lpfc_eq_destroy(struct lpfc_hba *phba, struct lpfc_queue *eq)
17610	{
17611	LPFC_MBOXQ_t *mbox;
17612	int rc, length, status = 0;
17613	uint32_t shdr_status, shdr_add_status;
17614	union lpfc_sli4_cfg_shdr *shdr;
17615
17616	/* sanity check on queue memory */
17617	if (!eq)
17618	return -ENODEV;
17619
17620	mbox = mempool_alloc(pool: eq->phba->mbox_mem_pool, GFP_KERNEL);
17621	if (!mbox)
17622	return -ENOMEM;
17623	length = (sizeof(struct lpfc_mbx_eq_destroy) -
17624	sizeof(struct lpfc_sli4_cfg_mhdr));
17625	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
17626	LPFC_MBOX_OPCODE_EQ_DESTROY,
17627	length, LPFC_SLI4_MBX_EMBED);
17628	bf_set(lpfc_mbx_eq_destroy_q_id, &mbox->u.mqe.un.eq_destroy.u.request,
17629	eq->queue_id);
17630	mbox->vport = eq->phba->pport;
17631	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
17632
17633	rc = lpfc_sli_issue_mbox(phba: eq->phba, pmbox: mbox, MBX_POLL);
17634	/* The IOCTL status is embedded in the mailbox subheader. */
17635	shdr = (union lpfc_sli4_cfg_shdr *)
17636	&mbox->u.mqe.un.eq_destroy.header.cfg_shdr;
17637	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17638	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17639	if (shdr_status || shdr_add_status || rc) {
17640	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17641	"2505 EQ_DESTROY mailbox failed with "
17642	"status x%x add_status x%x, mbx status x%x\n",
17643	shdr_status, shdr_add_status, rc);
17644	status = -ENXIO;
17645	}
17646
17647	/* Remove eq from any list */
17648	list_del_init(entry: &eq->list);
17649	mempool_free(element: mbox, pool: eq->phba->mbox_mem_pool);
17650	return status;
17651	}
17652
17653	/**
17654	* lpfc_cq_destroy - Destroy a Completion Queue on the HBA
17655	* @phba: HBA structure that indicates port to destroy a queue on.
17656	* @cq: The queue structure associated with the queue to destroy.
17657	*
17658	* This function destroys a queue, as detailed in @cq by sending an mailbox
17659	* command, specific to the type of queue, to the HBA.
17660	*
17661	* The @cq struct is used to get the queue ID of the queue to destroy.
17662	*
17663	* On success this function will return a zero. If the queue destroy mailbox
17664	* command fails this function will return -ENXIO.
17665	**/
17666	int
17667	lpfc_cq_destroy(struct lpfc_hba *phba, struct lpfc_queue *cq)
17668	{
17669	LPFC_MBOXQ_t *mbox;
17670	int rc, length, status = 0;
17671	uint32_t shdr_status, shdr_add_status;
17672	union lpfc_sli4_cfg_shdr *shdr;
17673
17674	/* sanity check on queue memory */
17675	if (!cq)
17676	return -ENODEV;
17677	mbox = mempool_alloc(pool: cq->phba->mbox_mem_pool, GFP_KERNEL);
17678	if (!mbox)
17679	return -ENOMEM;
17680	length = (sizeof(struct lpfc_mbx_cq_destroy) -
17681	sizeof(struct lpfc_sli4_cfg_mhdr));
17682	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
17683	LPFC_MBOX_OPCODE_CQ_DESTROY,
17684	length, LPFC_SLI4_MBX_EMBED);
17685	bf_set(lpfc_mbx_cq_destroy_q_id, &mbox->u.mqe.un.cq_destroy.u.request,
17686	cq->queue_id);
17687	mbox->vport = cq->phba->pport;
17688	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
17689	rc = lpfc_sli_issue_mbox(phba: cq->phba, pmbox: mbox, MBX_POLL);
17690	/* The IOCTL status is embedded in the mailbox subheader. */
17691	shdr = (union lpfc_sli4_cfg_shdr *)
17692	&mbox->u.mqe.un.wq_create.header.cfg_shdr;
17693	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17694	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17695	if (shdr_status || shdr_add_status || rc) {
17696	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17697	"2506 CQ_DESTROY mailbox failed with "
17698	"status x%x add_status x%x, mbx status x%x\n",
17699	shdr_status, shdr_add_status, rc);
17700	status = -ENXIO;
17701	}
17702	/* Remove cq from any list */
17703	list_del_init(entry: &cq->list);
17704	mempool_free(element: mbox, pool: cq->phba->mbox_mem_pool);
17705	return status;
17706	}
17707
17708	/**
17709	* lpfc_mq_destroy - Destroy a Mailbox Queue on the HBA
17710	* @phba: HBA structure that indicates port to destroy a queue on.
17711	* @mq: The queue structure associated with the queue to destroy.
17712	*
17713	* This function destroys a queue, as detailed in @mq by sending an mailbox
17714	* command, specific to the type of queue, to the HBA.
17715	*
17716	* The @mq struct is used to get the queue ID of the queue to destroy.
17717	*
17718	* On success this function will return a zero. If the queue destroy mailbox
17719	* command fails this function will return -ENXIO.
17720	**/
17721	int
17722	lpfc_mq_destroy(struct lpfc_hba *phba, struct lpfc_queue *mq)
17723	{
17724	LPFC_MBOXQ_t *mbox;
17725	int rc, length, status = 0;
17726	uint32_t shdr_status, shdr_add_status;
17727	union lpfc_sli4_cfg_shdr *shdr;
17728
17729	/* sanity check on queue memory */
17730	if (!mq)
17731	return -ENODEV;
17732	mbox = mempool_alloc(pool: mq->phba->mbox_mem_pool, GFP_KERNEL);
17733	if (!mbox)
17734	return -ENOMEM;
17735	length = (sizeof(struct lpfc_mbx_mq_destroy) -
17736	sizeof(struct lpfc_sli4_cfg_mhdr));
17737	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
17738	LPFC_MBOX_OPCODE_MQ_DESTROY,
17739	length, LPFC_SLI4_MBX_EMBED);
17740	bf_set(lpfc_mbx_mq_destroy_q_id, &mbox->u.mqe.un.mq_destroy.u.request,
17741	mq->queue_id);
17742	mbox->vport = mq->phba->pport;
17743	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
17744	rc = lpfc_sli_issue_mbox(phba: mq->phba, pmbox: mbox, MBX_POLL);
17745	/* The IOCTL status is embedded in the mailbox subheader. */
17746	shdr = (union lpfc_sli4_cfg_shdr *)
17747	&mbox->u.mqe.un.mq_destroy.header.cfg_shdr;
17748	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17749	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17750	if (shdr_status || shdr_add_status || rc) {
17751	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17752	"2507 MQ_DESTROY mailbox failed with "
17753	"status x%x add_status x%x, mbx status x%x\n",
17754	shdr_status, shdr_add_status, rc);
17755	status = -ENXIO;
17756	}
17757	/* Remove mq from any list */
17758	list_del_init(entry: &mq->list);
17759	mempool_free(element: mbox, pool: mq->phba->mbox_mem_pool);
17760	return status;
17761	}
17762
17763	/**
17764	* lpfc_wq_destroy - Destroy a Work Queue on the HBA
17765	* @phba: HBA structure that indicates port to destroy a queue on.
17766	* @wq: The queue structure associated with the queue to destroy.
17767	*
17768	* This function destroys a queue, as detailed in @wq by sending an mailbox
17769	* command, specific to the type of queue, to the HBA.
17770	*
17771	* The @wq struct is used to get the queue ID of the queue to destroy.
17772	*
17773	* On success this function will return a zero. If the queue destroy mailbox
17774	* command fails this function will return -ENXIO.
17775	**/
17776	int
17777	lpfc_wq_destroy(struct lpfc_hba *phba, struct lpfc_queue *wq)
17778	{
17779	LPFC_MBOXQ_t *mbox;
17780	int rc, length, status = 0;
17781	uint32_t shdr_status, shdr_add_status;
17782	union lpfc_sli4_cfg_shdr *shdr;
17783
17784	/* sanity check on queue memory */
17785	if (!wq)
17786	return -ENODEV;
17787	mbox = mempool_alloc(pool: wq->phba->mbox_mem_pool, GFP_KERNEL);
17788	if (!mbox)
17789	return -ENOMEM;
17790	length = (sizeof(struct lpfc_mbx_wq_destroy) -
17791	sizeof(struct lpfc_sli4_cfg_mhdr));
17792	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
17793	LPFC_MBOX_OPCODE_FCOE_WQ_DESTROY,
17794	length, LPFC_SLI4_MBX_EMBED);
17795	bf_set(lpfc_mbx_wq_destroy_q_id, &mbox->u.mqe.un.wq_destroy.u.request,
17796	wq->queue_id);
17797	mbox->vport = wq->phba->pport;
17798	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
17799	rc = lpfc_sli_issue_mbox(phba: wq->phba, pmbox: mbox, MBX_POLL);
17800	shdr = (union lpfc_sli4_cfg_shdr *)
17801	&mbox->u.mqe.un.wq_destroy.header.cfg_shdr;
17802	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17803	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17804	if (shdr_status || shdr_add_status || rc) {
17805	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17806	"2508 WQ_DESTROY mailbox failed with "
17807	"status x%x add_status x%x, mbx status x%x\n",
17808	shdr_status, shdr_add_status, rc);
17809	status = -ENXIO;
17810	}
17811	/* Remove wq from any list */
17812	list_del_init(entry: &wq->list);
17813	kfree(objp: wq->pring);
17814	wq->pring = NULL;
17815	mempool_free(element: mbox, pool: wq->phba->mbox_mem_pool);
17816	return status;
17817	}
17818
17819	/**
17820	* lpfc_rq_destroy - Destroy a Receive Queue on the HBA
17821	* @phba: HBA structure that indicates port to destroy a queue on.
17822	* @hrq: The queue structure associated with the queue to destroy.
17823	* @drq: The queue structure associated with the queue to destroy.
17824	*
17825	* This function destroys a queue, as detailed in @rq by sending an mailbox
17826	* command, specific to the type of queue, to the HBA.
17827	*
17828	* The @rq struct is used to get the queue ID of the queue to destroy.
17829	*
17830	* On success this function will return a zero. If the queue destroy mailbox
17831	* command fails this function will return -ENXIO.
17832	**/
17833	int
17834	lpfc_rq_destroy(struct lpfc_hba *phba, struct lpfc_queue *hrq,
17835	struct lpfc_queue *drq)
17836	{
17837	LPFC_MBOXQ_t *mbox;
17838	int rc, length, status = 0;
17839	uint32_t shdr_status, shdr_add_status;
17840	union lpfc_sli4_cfg_shdr *shdr;
17841
17842	/* sanity check on queue memory */
17843	if (!hrq || !drq)
17844	return -ENODEV;
17845	mbox = mempool_alloc(pool: hrq->phba->mbox_mem_pool, GFP_KERNEL);
17846	if (!mbox)
17847	return -ENOMEM;
17848	length = (sizeof(struct lpfc_mbx_rq_destroy) -
17849	sizeof(struct lpfc_sli4_cfg_mhdr));
17850	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
17851	LPFC_MBOX_OPCODE_FCOE_RQ_DESTROY,
17852	length, LPFC_SLI4_MBX_EMBED);
17853	bf_set(lpfc_mbx_rq_destroy_q_id, &mbox->u.mqe.un.rq_destroy.u.request,
17854	hrq->queue_id);
17855	mbox->vport = hrq->phba->pport;
17856	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
17857	rc = lpfc_sli_issue_mbox(phba: hrq->phba, pmbox: mbox, MBX_POLL);
17858	/* The IOCTL status is embedded in the mailbox subheader. */
17859	shdr = (union lpfc_sli4_cfg_shdr *)
17860	&mbox->u.mqe.un.rq_destroy.header.cfg_shdr;
17861	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17862	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17863	if (shdr_status || shdr_add_status || rc) {
17864	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17865	"2509 RQ_DESTROY mailbox failed with "
17866	"status x%x add_status x%x, mbx status x%x\n",
17867	shdr_status, shdr_add_status, rc);
17868	mempool_free(element: mbox, pool: hrq->phba->mbox_mem_pool);
17869	return -ENXIO;
17870	}
17871	bf_set(lpfc_mbx_rq_destroy_q_id, &mbox->u.mqe.un.rq_destroy.u.request,
17872	drq->queue_id);
17873	rc = lpfc_sli_issue_mbox(phba: drq->phba, pmbox: mbox, MBX_POLL);
17874	shdr = (union lpfc_sli4_cfg_shdr *)
17875	&mbox->u.mqe.un.rq_destroy.header.cfg_shdr;
17876	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17877	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17878	if (shdr_status || shdr_add_status || rc) {
17879	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17880	"2510 RQ_DESTROY mailbox failed with "
17881	"status x%x add_status x%x, mbx status x%x\n",
17882	shdr_status, shdr_add_status, rc);
17883	status = -ENXIO;
17884	}
17885	list_del_init(entry: &hrq->list);
17886	list_del_init(entry: &drq->list);
17887	mempool_free(element: mbox, pool: hrq->phba->mbox_mem_pool);
17888	return status;
17889	}
17890
17891	/**
17892	* lpfc_sli4_post_sgl - Post scatter gather list for an XRI to HBA
17893	* @phba: The virtual port for which this call being executed.
17894	* @pdma_phys_addr0: Physical address of the 1st SGL page.
17895	* @pdma_phys_addr1: Physical address of the 2nd SGL page.
17896	* @xritag: the xritag that ties this io to the SGL pages.
17897	*
17898	* This routine will post the sgl pages for the IO that has the xritag
17899	* that is in the iocbq structure. The xritag is assigned during iocbq
17900	* creation and persists for as long as the driver is loaded.
17901	* if the caller has fewer than 256 scatter gather segments to map then
17902	* pdma_phys_addr1 should be 0.
17903	* If the caller needs to map more than 256 scatter gather segment then
17904	* pdma_phys_addr1 should be a valid physical address.
17905	* physical address for SGLs must be 64 byte aligned.
17906	* If you are going to map 2 SGL's then the first one must have 256 entries
17907	* the second sgl can have between 1 and 256 entries.
17908	*
17909	* Return codes:
17910	* 0 - Success
17911	* -ENXIO, -ENOMEM - Failure
17912	**/
17913	int
17914	lpfc_sli4_post_sgl(struct lpfc_hba *phba,
17915	dma_addr_t pdma_phys_addr0,
17916	dma_addr_t pdma_phys_addr1,
17917	uint16_t xritag)
17918	{
17919	struct lpfc_mbx_post_sgl_pages *post_sgl_pages;
17920	LPFC_MBOXQ_t *mbox;
17921	int rc;
17922	uint32_t shdr_status, shdr_add_status;
17923	uint32_t mbox_tmo;
17924	union lpfc_sli4_cfg_shdr *shdr;
17925
17926	if (xritag == NO_XRI) {
17927	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17928	"0364 Invalid param:\n");
17929	return -EINVAL;
17930	}
17931
17932	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
17933	if (!mbox)
17934	return -ENOMEM;
17935
17936	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
17937	LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES,
17938	sizeof(struct lpfc_mbx_post_sgl_pages) -
17939	sizeof(struct lpfc_sli4_cfg_mhdr), LPFC_SLI4_MBX_EMBED);
17940
17941	post_sgl_pages = (struct lpfc_mbx_post_sgl_pages *)
17942	&mbox->u.mqe.un.post_sgl_pages;
17943	bf_set(lpfc_post_sgl_pages_xri, post_sgl_pages, xritag);
17944	bf_set(lpfc_post_sgl_pages_xricnt, post_sgl_pages, 1);
17945
17946	post_sgl_pages->sgl_pg_pairs[0].sgl_pg0_addr_lo =
17947	cpu_to_le32(putPaddrLow(pdma_phys_addr0));
17948	post_sgl_pages->sgl_pg_pairs[0].sgl_pg0_addr_hi =
17949	cpu_to_le32(putPaddrHigh(pdma_phys_addr0));
17950
17951	post_sgl_pages->sgl_pg_pairs[0].sgl_pg1_addr_lo =
17952	cpu_to_le32(putPaddrLow(pdma_phys_addr1));
17953	post_sgl_pages->sgl_pg_pairs[0].sgl_pg1_addr_hi =
17954	cpu_to_le32(putPaddrHigh(pdma_phys_addr1));
17955	if (!phba->sli4_hba.intr_enable)
17956	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
17957	else {
17958	mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
17959	rc = lpfc_sli_issue_mbox_wait(phba, pmboxq: mbox, timeout: mbox_tmo);
17960	}
17961	/* The IOCTL status is embedded in the mailbox subheader. */
17962	shdr = (union lpfc_sli4_cfg_shdr *) &post_sgl_pages->header.cfg_shdr;
17963	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17964	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17965	if (!phba->sli4_hba.intr_enable)
17966	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
17967	else if (rc != MBX_TIMEOUT)
17968	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
17969	if (shdr_status || shdr_add_status || rc) {
17970	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17971	"2511 POST_SGL mailbox failed with "
17972	"status x%x add_status x%x, mbx status x%x\n",
17973	shdr_status, shdr_add_status, rc);
17974	}
17975	return 0;
17976	}
17977
17978	/**
17979	* lpfc_sli4_alloc_xri - Get an available rpi in the device's range
17980	* @phba: pointer to lpfc hba data structure.
17981	*
17982	* This routine is invoked to post rpi header templates to the
17983	* HBA consistent with the SLI-4 interface spec. This routine
17984	* posts a SLI4_PAGE_SIZE memory region to the port to hold up to
17985	* SLI4_PAGE_SIZE modulo 64 rpi context headers.
17986	*
17987	* Returns
17988	* A nonzero rpi defined as rpi_base <= rpi < max_rpi if successful
17989	* LPFC_RPI_ALLOC_ERROR if no rpis are available.
17990	**/
17991	static uint16_t
17992	lpfc_sli4_alloc_xri(struct lpfc_hba *phba)
17993	{
17994	unsigned long xri;
17995
17996	/*
17997	* Fetch the next logical xri. Because this index is logical,
17998	* the driver starts at 0 each time.
17999	*/
18000	spin_lock_irq(lock: &phba->hbalock);
18001	xri = find_first_zero_bit(addr: phba->sli4_hba.xri_bmask,
18002	size: phba->sli4_hba.max_cfg_param.max_xri);
18003	if (xri >= phba->sli4_hba.max_cfg_param.max_xri) {
18004	spin_unlock_irq(lock: &phba->hbalock);
18005	return NO_XRI;
18006	} else {
18007	set_bit(nr: xri, addr: phba->sli4_hba.xri_bmask);
18008	phba->sli4_hba.max_cfg_param.xri_used++;
18009	}
18010	spin_unlock_irq(lock: &phba->hbalock);
18011	return xri;
18012	}
18013
18014	/**
18015	* __lpfc_sli4_free_xri - Release an xri for reuse.
18016	* @phba: pointer to lpfc hba data structure.
18017	* @xri: xri to release.
18018	*
18019	* This routine is invoked to release an xri to the pool of
18020	* available rpis maintained by the driver.
18021	**/
18022	static void
18023	__lpfc_sli4_free_xri(struct lpfc_hba *phba, int xri)
18024	{
18025	if (test_and_clear_bit(nr: xri, addr: phba->sli4_hba.xri_bmask)) {
18026	phba->sli4_hba.max_cfg_param.xri_used--;
18027	}
18028	}
18029
18030	/**
18031	* lpfc_sli4_free_xri - Release an xri for reuse.
18032	* @phba: pointer to lpfc hba data structure.
18033	* @xri: xri to release.
18034	*
18035	* This routine is invoked to release an xri to the pool of
18036	* available rpis maintained by the driver.
18037	**/
18038	void
18039	lpfc_sli4_free_xri(struct lpfc_hba *phba, int xri)
18040	{
18041	spin_lock_irq(lock: &phba->hbalock);
18042	__lpfc_sli4_free_xri(phba, xri);
18043	spin_unlock_irq(lock: &phba->hbalock);
18044	}
18045
18046	/**
18047	* lpfc_sli4_next_xritag - Get an xritag for the io
18048	* @phba: Pointer to HBA context object.
18049	*
18050	* This function gets an xritag for the iocb. If there is no unused xritag
18051	* it will return 0xffff.
18052	* The function returns the allocated xritag if successful, else returns zero.
18053	* Zero is not a valid xritag.
18054	* The caller is not required to hold any lock.
18055	**/
18056	uint16_t
18057	lpfc_sli4_next_xritag(struct lpfc_hba *phba)
18058	{
18059	uint16_t xri_index;
18060
18061	xri_index = lpfc_sli4_alloc_xri(phba);
18062	if (xri_index == NO_XRI)
18063	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
18064	"2004 Failed to allocate XRI.last XRITAG is %d"
18065	" Max XRI is %d, Used XRI is %d\n",
18066	xri_index,
18067	phba->sli4_hba.max_cfg_param.max_xri,
18068	phba->sli4_hba.max_cfg_param.xri_used);
18069	return xri_index;
18070	}
18071
18072	/**
18073	* lpfc_sli4_post_sgl_list - post a block of ELS sgls to the port.
18074	* @phba: pointer to lpfc hba data structure.
18075	* @post_sgl_list: pointer to els sgl entry list.
18076	* @post_cnt: number of els sgl entries on the list.
18077	*
18078	* This routine is invoked to post a block of driver's sgl pages to the
18079	* HBA using non-embedded mailbox command. No Lock is held. This routine
18080	* is only called when the driver is loading and after all IO has been
18081	* stopped.
18082	**/
18083	static int
18084	lpfc_sli4_post_sgl_list(struct lpfc_hba *phba,
18085	struct list_head *post_sgl_list,
18086	int post_cnt)
18087	{
18088	struct lpfc_sglq *sglq_entry = NULL, *sglq_next = NULL;
18089	struct lpfc_mbx_post_uembed_sgl_page1 *sgl;
18090	struct sgl_page_pairs *sgl_pg_pairs;
18091	void *viraddr;
18092	LPFC_MBOXQ_t *mbox;
18093	uint32_t reqlen, alloclen, pg_pairs;
18094	uint32_t mbox_tmo;
18095	uint16_t xritag_start = 0;
18096	int rc = 0;
18097	uint32_t shdr_status, shdr_add_status;
18098	union lpfc_sli4_cfg_shdr *shdr;
18099
18100	reqlen = post_cnt * sizeof(struct sgl_page_pairs) +
18101	sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t);
18102	if (reqlen > SLI4_PAGE_SIZE) {
18103	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18104	"2559 Block sgl registration required DMA "
18105	"size (%d) great than a page\n", reqlen);
18106	return -ENOMEM;
18107	}
18108
18109	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
18110	if (!mbox)
18111	return -ENOMEM;
18112
18113	/* Allocate DMA memory and set up the non-embedded mailbox command */
18114	alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
18115	LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES, reqlen,
18116	LPFC_SLI4_MBX_NEMBED);
18117
18118	if (alloclen < reqlen) {
18119	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18120	"0285 Allocated DMA memory size (%d) is "
18121	"less than the requested DMA memory "
18122	"size (%d)\n", alloclen, reqlen);
18123	lpfc_sli4_mbox_cmd_free(phba, mbox);
18124	return -ENOMEM;
18125	}
18126	/* Set up the SGL pages in the non-embedded DMA pages */
18127	viraddr = mbox->sge_array->addr[0];
18128	sgl = (struct lpfc_mbx_post_uembed_sgl_page1 *)viraddr;
18129	sgl_pg_pairs = &sgl->sgl_pg_pairs;
18130
18131	pg_pairs = 0;
18132	list_for_each_entry_safe(sglq_entry, sglq_next, post_sgl_list, list) {
18133	/* Set up the sge entry */
18134	sgl_pg_pairs->sgl_pg0_addr_lo =
18135	cpu_to_le32(putPaddrLow(sglq_entry->phys));
18136	sgl_pg_pairs->sgl_pg0_addr_hi =
18137	cpu_to_le32(putPaddrHigh(sglq_entry->phys));
18138	sgl_pg_pairs->sgl_pg1_addr_lo =
18139	cpu_to_le32(putPaddrLow(0));
18140	sgl_pg_pairs->sgl_pg1_addr_hi =
18141	cpu_to_le32(putPaddrHigh(0));
18142
18143	/* Keep the first xritag on the list */
18144	if (pg_pairs == 0)
18145	xritag_start = sglq_entry->sli4_xritag;
18146	sgl_pg_pairs++;
18147	pg_pairs++;
18148	}
18149
18150	/* Complete initialization and perform endian conversion. */
18151	bf_set(lpfc_post_sgl_pages_xri, sgl, xritag_start);
18152	bf_set(lpfc_post_sgl_pages_xricnt, sgl, post_cnt);
18153	sgl->word0 = cpu_to_le32(sgl->word0);
18154
18155	if (!phba->sli4_hba.intr_enable)
18156	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
18157	else {
18158	mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
18159	rc = lpfc_sli_issue_mbox_wait(phba, pmboxq: mbox, timeout: mbox_tmo);
18160	}
18161	shdr = (union lpfc_sli4_cfg_shdr *) &sgl->cfg_shdr;
18162	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
18163	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
18164	if (!phba->sli4_hba.intr_enable)
18165	lpfc_sli4_mbox_cmd_free(phba, mbox);
18166	else if (rc != MBX_TIMEOUT)
18167	lpfc_sli4_mbox_cmd_free(phba, mbox);
18168	if (shdr_status || shdr_add_status || rc) {
18169	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18170	"2513 POST_SGL_BLOCK mailbox command failed "
18171	"status x%x add_status x%x mbx status x%x\n",
18172	shdr_status, shdr_add_status, rc);
18173	rc = -ENXIO;
18174	}
18175	return rc;
18176	}
18177
18178	/**
18179	* lpfc_sli4_post_io_sgl_block - post a block of nvme sgl list to firmware
18180	* @phba: pointer to lpfc hba data structure.
18181	* @nblist: pointer to nvme buffer list.
18182	* @count: number of scsi buffers on the list.
18183	*
18184	* This routine is invoked to post a block of @count scsi sgl pages from a
18185	* SCSI buffer list @nblist to the HBA using non-embedded mailbox command.
18186	* No Lock is held.
18187	*
18188	**/
18189	static int
18190	lpfc_sli4_post_io_sgl_block(struct lpfc_hba *phba, struct list_head *nblist,
18191	int count)
18192	{
18193	struct lpfc_io_buf *lpfc_ncmd;
18194	struct lpfc_mbx_post_uembed_sgl_page1 *sgl;
18195	struct sgl_page_pairs *sgl_pg_pairs;
18196	void *viraddr;
18197	LPFC_MBOXQ_t *mbox;
18198	uint32_t reqlen, alloclen, pg_pairs;
18199	uint32_t mbox_tmo;
18200	uint16_t xritag_start = 0;
18201	int rc = 0;
18202	uint32_t shdr_status, shdr_add_status;
18203	dma_addr_t pdma_phys_bpl1;
18204	union lpfc_sli4_cfg_shdr *shdr;
18205
18206	/* Calculate the requested length of the dma memory */
18207	reqlen = count * sizeof(struct sgl_page_pairs) +
18208	sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t);
18209	if (reqlen > SLI4_PAGE_SIZE) {
18210	lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
18211	"6118 Block sgl registration required DMA "
18212	"size (%d) great than a page\n", reqlen);
18213	return -ENOMEM;
18214	}
18215	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
18216	if (!mbox) {
18217	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18218	"6119 Failed to allocate mbox cmd memory\n");
18219	return -ENOMEM;
18220	}
18221
18222	/* Allocate DMA memory and set up the non-embedded mailbox command */
18223	alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
18224	LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES,
18225	reqlen, LPFC_SLI4_MBX_NEMBED);
18226
18227	if (alloclen < reqlen) {
18228	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18229	"6120 Allocated DMA memory size (%d) is "
18230	"less than the requested DMA memory "
18231	"size (%d)\n", alloclen, reqlen);
18232	lpfc_sli4_mbox_cmd_free(phba, mbox);
18233	return -ENOMEM;
18234	}
18235
18236	/* Get the first SGE entry from the non-embedded DMA memory */
18237	viraddr = mbox->sge_array->addr[0];
18238
18239	/* Set up the SGL pages in the non-embedded DMA pages */
18240	sgl = (struct lpfc_mbx_post_uembed_sgl_page1 *)viraddr;
18241	sgl_pg_pairs = &sgl->sgl_pg_pairs;
18242
18243	pg_pairs = 0;
18244	list_for_each_entry(lpfc_ncmd, nblist, list) {
18245	/* Set up the sge entry */
18246	sgl_pg_pairs->sgl_pg0_addr_lo =
18247	cpu_to_le32(putPaddrLow(lpfc_ncmd->dma_phys_sgl));
18248	sgl_pg_pairs->sgl_pg0_addr_hi =
18249	cpu_to_le32(putPaddrHigh(lpfc_ncmd->dma_phys_sgl));
18250	if (phba->cfg_sg_dma_buf_size > SGL_PAGE_SIZE)
18251	pdma_phys_bpl1 = lpfc_ncmd->dma_phys_sgl +
18252	SGL_PAGE_SIZE;
18253	else
18254	pdma_phys_bpl1 = 0;
18255	sgl_pg_pairs->sgl_pg1_addr_lo =
18256	cpu_to_le32(putPaddrLow(pdma_phys_bpl1));
18257	sgl_pg_pairs->sgl_pg1_addr_hi =
18258	cpu_to_le32(putPaddrHigh(pdma_phys_bpl1));
18259	/* Keep the first xritag on the list */
18260	if (pg_pairs == 0)
18261	xritag_start = lpfc_ncmd->cur_iocbq.sli4_xritag;
18262	sgl_pg_pairs++;
18263	pg_pairs++;
18264	}
18265	bf_set(lpfc_post_sgl_pages_xri, sgl, xritag_start);
18266	bf_set(lpfc_post_sgl_pages_xricnt, sgl, pg_pairs);
18267	/* Perform endian conversion if necessary */
18268	sgl->word0 = cpu_to_le32(sgl->word0);
18269
18270	if (!phba->sli4_hba.intr_enable) {
18271	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
18272	} else {
18273	mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
18274	rc = lpfc_sli_issue_mbox_wait(phba, pmboxq: mbox, timeout: mbox_tmo);
18275	}
18276	shdr = (union lpfc_sli4_cfg_shdr *)&sgl->cfg_shdr;
18277	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
18278	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
18279	if (!phba->sli4_hba.intr_enable)
18280	lpfc_sli4_mbox_cmd_free(phba, mbox);
18281	else if (rc != MBX_TIMEOUT)
18282	lpfc_sli4_mbox_cmd_free(phba, mbox);
18283	if (shdr_status || shdr_add_status || rc) {
18284	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18285	"6125 POST_SGL_BLOCK mailbox command failed "
18286	"status x%x add_status x%x mbx status x%x\n",
18287	shdr_status, shdr_add_status, rc);
18288	rc = -ENXIO;
18289	}
18290	return rc;
18291	}
18292
18293	/**
18294	* lpfc_sli4_post_io_sgl_list - Post blocks of nvme buffer sgls from a list
18295	* @phba: pointer to lpfc hba data structure.
18296	* @post_nblist: pointer to the nvme buffer list.
18297	* @sb_count: number of nvme buffers.
18298	*
18299	* This routine walks a list of nvme buffers that was passed in. It attempts
18300	* to construct blocks of nvme buffer sgls which contains contiguous xris and
18301	* uses the non-embedded SGL block post mailbox commands to post to the port.
18302	* For single NVME buffer sgl with non-contiguous xri, if any, it shall use
18303	* embedded SGL post mailbox command for posting. The @post_nblist passed in
18304	* must be local list, thus no lock is needed when manipulate the list.
18305	*
18306	* Returns: 0 = failure, non-zero number of successfully posted buffers.
18307	**/
18308	int
18309	lpfc_sli4_post_io_sgl_list(struct lpfc_hba *phba,
18310	struct list_head *post_nblist, int sb_count)
18311	{
18312	struct lpfc_io_buf *lpfc_ncmd, *lpfc_ncmd_next;
18313	int status, sgl_size;
18314	int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0;
18315	dma_addr_t pdma_phys_sgl1;
18316	int last_xritag = NO_XRI;
18317	int cur_xritag;
18318	LIST_HEAD(prep_nblist);
18319	LIST_HEAD(blck_nblist);
18320	LIST_HEAD(nvme_nblist);
18321
18322	/* sanity check */
18323	if (sb_count <= 0)
18324	return -EINVAL;
18325
18326	sgl_size = phba->cfg_sg_dma_buf_size;
18327	list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, post_nblist, list) {
18328	list_del_init(entry: &lpfc_ncmd->list);
18329	block_cnt++;
18330	if ((last_xritag != NO_XRI) &&
18331	(lpfc_ncmd->cur_iocbq.sli4_xritag != last_xritag + 1)) {
18332	/* a hole in xri block, form a sgl posting block */
18333	list_splice_init(list: &prep_nblist, head: &blck_nblist);
18334	post_cnt = block_cnt - 1;
18335	/* prepare list for next posting block */
18336	list_add_tail(new: &lpfc_ncmd->list, head: &prep_nblist);
18337	block_cnt = 1;
18338	} else {
18339	/* prepare list for next posting block */
18340	list_add_tail(new: &lpfc_ncmd->list, head: &prep_nblist);
18341	/* enough sgls for non-embed sgl mbox command */
18342	if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) {
18343	list_splice_init(list: &prep_nblist, head: &blck_nblist);
18344	post_cnt = block_cnt;
18345	block_cnt = 0;
18346	}
18347	}
18348	num_posting++;
18349	last_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag;
18350
18351	/* end of repost sgl list condition for NVME buffers */
18352	if (num_posting == sb_count) {
18353	if (post_cnt == 0) {
18354	/* last sgl posting block */
18355	list_splice_init(list: &prep_nblist, head: &blck_nblist);
18356	post_cnt = block_cnt;
18357	} else if (block_cnt == 1) {
18358	/* last single sgl with non-contiguous xri */
18359	if (sgl_size > SGL_PAGE_SIZE)
18360	pdma_phys_sgl1 =
18361	lpfc_ncmd->dma_phys_sgl +
18362	SGL_PAGE_SIZE;
18363	else
18364	pdma_phys_sgl1 = 0;
18365	cur_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag;
18366	status = lpfc_sli4_post_sgl(
18367	phba, pdma_phys_addr0: lpfc_ncmd->dma_phys_sgl,
18368	pdma_phys_addr1: pdma_phys_sgl1, xritag: cur_xritag);
18369	if (status) {
18370	/* Post error. Buffer unavailable. */
18371	lpfc_ncmd->flags |=
18372	LPFC_SBUF_NOT_POSTED;
18373	} else {
18374	/* Post success. Bffer available. */
18375	lpfc_ncmd->flags &=
18376	~LPFC_SBUF_NOT_POSTED;
18377	lpfc_ncmd->status = IOSTAT_SUCCESS;
18378	num_posted++;
18379	}
18380	/* success, put on NVME buffer sgl list */
18381	list_add_tail(new: &lpfc_ncmd->list, head: &nvme_nblist);
18382	}
18383	}
18384
18385	/* continue until a nembed page worth of sgls */
18386	if (post_cnt == 0)
18387	continue;
18388
18389	/* post block of NVME buffer list sgls */
18390	status = lpfc_sli4_post_io_sgl_block(phba, nblist: &blck_nblist,
18391	count: post_cnt);
18392
18393	/* don't reset xirtag due to hole in xri block */
18394	if (block_cnt == 0)
18395	last_xritag = NO_XRI;
18396
18397	/* reset NVME buffer post count for next round of posting */
18398	post_cnt = 0;
18399
18400	/* put posted NVME buffer-sgl posted on NVME buffer sgl list */
18401	while (!list_empty(head: &blck_nblist)) {
18402	list_remove_head(&blck_nblist, lpfc_ncmd,
18403	struct lpfc_io_buf, list);
18404	if (status) {
18405	/* Post error. Mark buffer unavailable. */
18406	lpfc_ncmd->flags |= LPFC_SBUF_NOT_POSTED;
18407	} else {
18408	/* Post success, Mark buffer available. */
18409	lpfc_ncmd->flags &= ~LPFC_SBUF_NOT_POSTED;
18410	lpfc_ncmd->status = IOSTAT_SUCCESS;
18411	num_posted++;
18412	}
18413	list_add_tail(new: &lpfc_ncmd->list, head: &nvme_nblist);
18414	}
18415	}
18416	/* Push NVME buffers with sgl posted to the available list */
18417	lpfc_io_buf_replenish(phba, cbuf: &nvme_nblist);
18418
18419	return num_posted;
18420	}
18421
18422	/**
18423	* lpfc_fc_frame_check - Check that this frame is a valid frame to handle
18424	* @phba: pointer to lpfc_hba struct that the frame was received on
18425	* @fc_hdr: A pointer to the FC Header data (In Big Endian Format)
18426	*
18427	* This function checks the fields in the @fc_hdr to see if the FC frame is a
18428	* valid type of frame that the LPFC driver will handle. This function will
18429	* return a zero if the frame is a valid frame or a non zero value when the
18430	* frame does not pass the check.
18431	**/
18432	static int
18433	lpfc_fc_frame_check(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr)
18434	{
18435	/* make rctl_names static to save stack space */
18436	struct fc_vft_header *fc_vft_hdr;
18437	uint32_t *header = (uint32_t *) fc_hdr;
18438
18439	#define FC_RCTL_MDS_DIAGS 0xF4
18440
18441	switch (fc_hdr->fh_r_ctl) {
18442	case FC_RCTL_DD_UNCAT: /* uncategorized information */
18443	case FC_RCTL_DD_SOL_DATA: /* solicited data */
18444	case FC_RCTL_DD_UNSOL_CTL: /* unsolicited control */
18445	case FC_RCTL_DD_SOL_CTL: /* solicited control or reply */
18446	case FC_RCTL_DD_UNSOL_DATA: /* unsolicited data */
18447	case FC_RCTL_DD_DATA_DESC: /* data descriptor */
18448	case FC_RCTL_DD_UNSOL_CMD: /* unsolicited command */
18449	case FC_RCTL_DD_CMD_STATUS: /* command status */
18450	case FC_RCTL_ELS_REQ: /* extended link services request */
18451	case FC_RCTL_ELS_REP: /* extended link services reply */
18452	case FC_RCTL_ELS4_REQ: /* FC-4 ELS request */
18453	case FC_RCTL_ELS4_REP: /* FC-4 ELS reply */
18454	case FC_RCTL_BA_ABTS: /* basic link service abort */
18455	case FC_RCTL_BA_RMC: /* remove connection */
18456	case FC_RCTL_BA_ACC: /* basic accept */
18457	case FC_RCTL_BA_RJT: /* basic reject */
18458	case FC_RCTL_BA_PRMT:
18459	case FC_RCTL_ACK_1: /* acknowledge_1 */
18460	case FC_RCTL_ACK_0: /* acknowledge_0 */
18461	case FC_RCTL_P_RJT: /* port reject */
18462	case FC_RCTL_F_RJT: /* fabric reject */
18463	case FC_RCTL_P_BSY: /* port busy */
18464	case FC_RCTL_F_BSY: /* fabric busy to data frame */
18465	case FC_RCTL_F_BSYL: /* fabric busy to link control frame */
18466	case FC_RCTL_LCR: /* link credit reset */
18467	case FC_RCTL_MDS_DIAGS: /* MDS Diagnostics */
18468	case FC_RCTL_END: /* end */
18469	break;
18470	case FC_RCTL_VFTH: /* Virtual Fabric tagging Header */
18471	fc_vft_hdr = (struct fc_vft_header *)fc_hdr;
18472	fc_hdr = &((struct fc_frame_header *)fc_vft_hdr)[1];
18473	return lpfc_fc_frame_check(phba, fc_hdr);
18474	case FC_RCTL_BA_NOP: /* basic link service NOP */
18475	default:
18476	goto drop;
18477	}
18478
18479	switch (fc_hdr->fh_type) {
18480	case FC_TYPE_BLS:
18481	case FC_TYPE_ELS:
18482	case FC_TYPE_FCP:
18483	case FC_TYPE_CT:
18484	case FC_TYPE_NVME:
18485	break;
18486	case FC_TYPE_IP:
18487	case FC_TYPE_ILS:
18488	default:
18489	goto drop;
18490	}
18491
18492	lpfc_printf_log(phba, KERN_INFO, LOG_ELS,
18493	"2538 Received frame rctl:x%x, type:x%x, "
18494	"frame Data:%08x %08x %08x %08x %08x %08x %08x\n",
18495	fc_hdr->fh_r_ctl, fc_hdr->fh_type,
18496	be32_to_cpu(header[0]), be32_to_cpu(header[1]),
18497	be32_to_cpu(header[2]), be32_to_cpu(header[3]),
18498	be32_to_cpu(header[4]), be32_to_cpu(header[5]),
18499	be32_to_cpu(header[6]));
18500	return 0;
18501	drop:
18502	lpfc_printf_log(phba, KERN_WARNING, LOG_ELS,
18503	"2539 Dropped frame rctl:x%x type:x%x\n",
18504	fc_hdr->fh_r_ctl, fc_hdr->fh_type);
18505	return 1;
18506	}
18507
18508	/**
18509	* lpfc_fc_hdr_get_vfi - Get the VFI from an FC frame
18510	* @fc_hdr: A pointer to the FC Header data (In Big Endian Format)
18511	*
18512	* This function processes the FC header to retrieve the VFI from the VF
18513	* header, if one exists. This function will return the VFI if one exists
18514	* or 0 if no VSAN Header exists.
18515	**/
18516	static uint32_t
18517	lpfc_fc_hdr_get_vfi(struct fc_frame_header *fc_hdr)
18518	{
18519	struct fc_vft_header *fc_vft_hdr = (struct fc_vft_header *)fc_hdr;
18520
18521	if (fc_hdr->fh_r_ctl != FC_RCTL_VFTH)
18522	return 0;
18523	return bf_get(fc_vft_hdr_vf_id, fc_vft_hdr);
18524	}
18525
18526	/**
18527	* lpfc_fc_frame_to_vport - Finds the vport that a frame is destined to
18528	* @phba: Pointer to the HBA structure to search for the vport on
18529	* @fc_hdr: A pointer to the FC Header data (In Big Endian Format)
18530	* @fcfi: The FC Fabric ID that the frame came from
18531	* @did: Destination ID to match against
18532	*
18533	* This function searches the @phba for a vport that matches the content of the
18534	* @fc_hdr passed in and the @fcfi. This function uses the @fc_hdr to fetch the
18535	* VFI, if the Virtual Fabric Tagging Header exists, and the DID. This function
18536	* returns the matching vport pointer or NULL if unable to match frame to a
18537	* vport.
18538	**/
18539	static struct lpfc_vport *
18540	lpfc_fc_frame_to_vport(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr,
18541	uint16_t fcfi, uint32_t did)
18542	{
18543	struct lpfc_vport **vports;
18544	struct lpfc_vport *vport = NULL;
18545	int i;
18546
18547	if (did == Fabric_DID)
18548	return phba->pport;
18549	if ((phba->pport->fc_flag & FC_PT2PT) &&
18550	!(phba->link_state == LPFC_HBA_READY))
18551	return phba->pport;
18552
18553	vports = lpfc_create_vport_work_array(phba);
18554	if (vports != NULL) {
18555	for (i = 0; i <= phba->max_vpi && vports[i] != NULL; i++) {
18556	if (phba->fcf.fcfi == fcfi &&
18557	vports[i]->vfi == lpfc_fc_hdr_get_vfi(fc_hdr) &&
18558	vports[i]->fc_myDID == did) {
18559	vport = vports[i];
18560	break;
18561	}
18562	}
18563	}
18564	lpfc_destroy_vport_work_array(phba, vports);
18565	return vport;
18566	}
18567
18568	/**
18569	* lpfc_update_rcv_time_stamp - Update vport's rcv seq time stamp
18570	* @vport: The vport to work on.
18571	*
18572	* This function updates the receive sequence time stamp for this vport. The
18573	* receive sequence time stamp indicates the time that the last frame of the
18574	* the sequence that has been idle for the longest amount of time was received.
18575	* the driver uses this time stamp to indicate if any received sequences have
18576	* timed out.
18577	**/
18578	static void
18579	lpfc_update_rcv_time_stamp(struct lpfc_vport *vport)
18580	{
18581	struct lpfc_dmabuf *h_buf;
18582	struct hbq_dmabuf *dmabuf = NULL;
18583
18584	/* get the oldest sequence on the rcv list */
18585	h_buf = list_get_first(&vport->rcv_buffer_list,
18586	struct lpfc_dmabuf, list);
18587	if (!h_buf)
18588	return;
18589	dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
18590	vport->rcv_buffer_time_stamp = dmabuf->time_stamp;
18591	}
18592
18593	/**
18594	* lpfc_cleanup_rcv_buffers - Cleans up all outstanding receive sequences.
18595	* @vport: The vport that the received sequences were sent to.
18596	*
18597	* This function cleans up all outstanding received sequences. This is called
18598	* by the driver when a link event or user action invalidates all the received
18599	* sequences.
18600	**/
18601	void
18602	lpfc_cleanup_rcv_buffers(struct lpfc_vport *vport)
18603	{
18604	struct lpfc_dmabuf *h_buf, *hnext;
18605	struct lpfc_dmabuf *d_buf, *dnext;
18606	struct hbq_dmabuf *dmabuf = NULL;
18607
18608	/* start with the oldest sequence on the rcv list */
18609	list_for_each_entry_safe(h_buf, hnext, &vport->rcv_buffer_list, list) {
18610	dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
18611	list_del_init(entry: &dmabuf->hbuf.list);
18612	list_for_each_entry_safe(d_buf, dnext,
18613	&dmabuf->dbuf.list, list) {
18614	list_del_init(entry: &d_buf->list);
18615	lpfc_in_buf_free(vport->phba, d_buf);
18616	}
18617	lpfc_in_buf_free(vport->phba, &dmabuf->dbuf);
18618	}
18619	}
18620
18621	/**
18622	* lpfc_rcv_seq_check_edtov - Cleans up timed out receive sequences.
18623	* @vport: The vport that the received sequences were sent to.
18624	*
18625	* This function determines whether any received sequences have timed out by
18626	* first checking the vport's rcv_buffer_time_stamp. If this time_stamp
18627	* indicates that there is at least one timed out sequence this routine will
18628	* go through the received sequences one at a time from most inactive to most
18629	* active to determine which ones need to be cleaned up. Once it has determined
18630	* that a sequence needs to be cleaned up it will simply free up the resources
18631	* without sending an abort.
18632	**/
18633	void
18634	lpfc_rcv_seq_check_edtov(struct lpfc_vport *vport)
18635	{
18636	struct lpfc_dmabuf *h_buf, *hnext;
18637	struct lpfc_dmabuf *d_buf, *dnext;
18638	struct hbq_dmabuf *dmabuf = NULL;
18639	unsigned long timeout;
18640	int abort_count = 0;
18641
18642	timeout = (msecs_to_jiffies(m: vport->phba->fc_edtov) +
18643	vport->rcv_buffer_time_stamp);
18644	if (list_empty(head: &vport->rcv_buffer_list) ||
18645	time_before(jiffies, timeout))
18646	return;
18647	/* start with the oldest sequence on the rcv list */
18648	list_for_each_entry_safe(h_buf, hnext, &vport->rcv_buffer_list, list) {
18649	dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
18650	timeout = (msecs_to_jiffies(m: vport->phba->fc_edtov) +
18651	dmabuf->time_stamp);
18652	if (time_before(jiffies, timeout))
18653	break;
18654	abort_count++;
18655	list_del_init(entry: &dmabuf->hbuf.list);
18656	list_for_each_entry_safe(d_buf, dnext,
18657	&dmabuf->dbuf.list, list) {
18658	list_del_init(entry: &d_buf->list);
18659	lpfc_in_buf_free(vport->phba, d_buf);
18660	}
18661	lpfc_in_buf_free(vport->phba, &dmabuf->dbuf);
18662	}
18663	if (abort_count)
18664	lpfc_update_rcv_time_stamp(vport);
18665	}
18666
18667	/**
18668	* lpfc_fc_frame_add - Adds a frame to the vport's list of received sequences
18669	* @vport: pointer to a vitural port
18670	* @dmabuf: pointer to a dmabuf that describes the hdr and data of the FC frame
18671	*
18672	* This function searches through the existing incomplete sequences that have
18673	* been sent to this @vport. If the frame matches one of the incomplete
18674	* sequences then the dbuf in the @dmabuf is added to the list of frames that
18675	* make up that sequence. If no sequence is found that matches this frame then
18676	* the function will add the hbuf in the @dmabuf to the @vport's rcv_buffer_list
18677	* This function returns a pointer to the first dmabuf in the sequence list that
18678	* the frame was linked to.
18679	**/
18680	static struct hbq_dmabuf *
18681	lpfc_fc_frame_add(struct lpfc_vport *vport, struct hbq_dmabuf *dmabuf)
18682	{
18683	struct fc_frame_header *new_hdr;
18684	struct fc_frame_header *temp_hdr;
18685	struct lpfc_dmabuf *d_buf;
18686	struct lpfc_dmabuf *h_buf;
18687	struct hbq_dmabuf *seq_dmabuf = NULL;
18688	struct hbq_dmabuf *temp_dmabuf = NULL;
18689	uint8_t found = 0;
18690
18691	INIT_LIST_HEAD(list: &dmabuf->dbuf.list);
18692	dmabuf->time_stamp = jiffies;
18693	new_hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
18694
18695	/* Use the hdr_buf to find the sequence that this frame belongs to */
18696	list_for_each_entry(h_buf, &vport->rcv_buffer_list, list) {
18697	temp_hdr = (struct fc_frame_header *)h_buf->virt;
18698	if ((temp_hdr->fh_seq_id != new_hdr->fh_seq_id) ||
18699	(temp_hdr->fh_ox_id != new_hdr->fh_ox_id) ||
18700	(memcmp(p: &temp_hdr->fh_s_id, q: &new_hdr->fh_s_id, size: 3)))
18701	continue;
18702	/* found a pending sequence that matches this frame */
18703	seq_dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
18704	break;
18705	}
18706	if (!seq_dmabuf) {
18707	/*
18708	* This indicates first frame received for this sequence.
18709	* Queue the buffer on the vport's rcv_buffer_list.
18710	*/
18711	list_add_tail(new: &dmabuf->hbuf.list, head: &vport->rcv_buffer_list);
18712	lpfc_update_rcv_time_stamp(vport);
18713	return dmabuf;
18714	}
18715	temp_hdr = seq_dmabuf->hbuf.virt;
18716	if (be16_to_cpu(new_hdr->fh_seq_cnt) <
18717	be16_to_cpu(temp_hdr->fh_seq_cnt)) {
18718	list_del_init(entry: &seq_dmabuf->hbuf.list);
18719	list_add_tail(new: &dmabuf->hbuf.list, head: &vport->rcv_buffer_list);
18720	list_add_tail(new: &dmabuf->dbuf.list, head: &seq_dmabuf->dbuf.list);
18721	lpfc_update_rcv_time_stamp(vport);
18722	return dmabuf;
18723	}
18724	/* move this sequence to the tail to indicate a young sequence */
18725	list_move_tail(list: &seq_dmabuf->hbuf.list, head: &vport->rcv_buffer_list);
18726	seq_dmabuf->time_stamp = jiffies;
18727	lpfc_update_rcv_time_stamp(vport);
18728	if (list_empty(head: &seq_dmabuf->dbuf.list)) {
18729	list_add_tail(new: &dmabuf->dbuf.list, head: &seq_dmabuf->dbuf.list);
18730	return seq_dmabuf;
18731	}
18732	/* find the correct place in the sequence to insert this frame */
18733	d_buf = list_entry(seq_dmabuf->dbuf.list.prev, typeof(*d_buf), list);
18734	while (!found) {
18735	temp_dmabuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
18736	temp_hdr = (struct fc_frame_header *)temp_dmabuf->hbuf.virt;
18737	/*
18738	* If the frame's sequence count is greater than the frame on
18739	* the list then insert the frame right after this frame
18740	*/
18741	if (be16_to_cpu(new_hdr->fh_seq_cnt) >
18742	be16_to_cpu(temp_hdr->fh_seq_cnt)) {
18743	list_add(new: &dmabuf->dbuf.list, head: &temp_dmabuf->dbuf.list);
18744	found = 1;
18745	break;
18746	}
18747
18748	if (&d_buf->list == &seq_dmabuf->dbuf.list)
18749	break;
18750	d_buf = list_entry(d_buf->list.prev, typeof(*d_buf), list);
18751	}
18752
18753	if (found)
18754	return seq_dmabuf;
18755	return NULL;
18756	}
18757
18758	/**
18759	* lpfc_sli4_abort_partial_seq - Abort partially assembled unsol sequence
18760	* @vport: pointer to a vitural port
18761	* @dmabuf: pointer to a dmabuf that describes the FC sequence
18762	*
18763	* This function tries to abort from the partially assembed sequence, described
18764	* by the information from basic abbort @dmabuf. It checks to see whether such
18765	* partially assembled sequence held by the driver. If so, it shall free up all
18766	* the frames from the partially assembled sequence.
18767	*
18768	* Return
18769	* true -- if there is matching partially assembled sequence present and all
18770	* the frames freed with the sequence;
18771	* false -- if there is no matching partially assembled sequence present so
18772	* nothing got aborted in the lower layer driver
18773	**/
18774	static bool
18775	lpfc_sli4_abort_partial_seq(struct lpfc_vport *vport,
18776	struct hbq_dmabuf *dmabuf)
18777	{
18778	struct fc_frame_header *new_hdr;
18779	struct fc_frame_header *temp_hdr;
18780	struct lpfc_dmabuf *d_buf, *n_buf, *h_buf;
18781	struct hbq_dmabuf *seq_dmabuf = NULL;
18782
18783	/* Use the hdr_buf to find the sequence that matches this frame */
18784	INIT_LIST_HEAD(list: &dmabuf->dbuf.list);
18785	INIT_LIST_HEAD(list: &dmabuf->hbuf.list);
18786	new_hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
18787	list_for_each_entry(h_buf, &vport->rcv_buffer_list, list) {
18788	temp_hdr = (struct fc_frame_header *)h_buf->virt;
18789	if ((temp_hdr->fh_seq_id != new_hdr->fh_seq_id) ||
18790	(temp_hdr->fh_ox_id != new_hdr->fh_ox_id) ||
18791	(memcmp(p: &temp_hdr->fh_s_id, q: &new_hdr->fh_s_id, size: 3)))
18792	continue;
18793	/* found a pending sequence that matches this frame */
18794	seq_dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
18795	break;
18796	}
18797
18798	/* Free up all the frames from the partially assembled sequence */
18799	if (seq_dmabuf) {
18800	list_for_each_entry_safe(d_buf, n_buf,
18801	&seq_dmabuf->dbuf.list, list) {
18802	list_del_init(entry: &d_buf->list);
18803	lpfc_in_buf_free(vport->phba, d_buf);
18804	}
18805	return true;
18806	}
18807	return false;
18808	}
18809
18810	/**
18811	* lpfc_sli4_abort_ulp_seq - Abort assembled unsol sequence from ulp
18812	* @vport: pointer to a vitural port
18813	* @dmabuf: pointer to a dmabuf that describes the FC sequence
18814	*
18815	* This function tries to abort from the assembed sequence from upper level
18816	* protocol, described by the information from basic abbort @dmabuf. It
18817	* checks to see whether such pending context exists at upper level protocol.
18818	* If so, it shall clean up the pending context.
18819	*
18820	* Return
18821	* true -- if there is matching pending context of the sequence cleaned
18822	* at ulp;
18823	* false -- if there is no matching pending context of the sequence present
18824	* at ulp.
18825	**/
18826	static bool
18827	lpfc_sli4_abort_ulp_seq(struct lpfc_vport *vport, struct hbq_dmabuf *dmabuf)
18828	{
18829	struct lpfc_hba *phba = vport->phba;
18830	int handled;
18831
18832	/* Accepting abort at ulp with SLI4 only */
18833	if (phba->sli_rev < LPFC_SLI_REV4)
18834	return false;
18835
18836	/* Register all caring upper level protocols to attend abort */
18837	handled = lpfc_ct_handle_unsol_abort(phba, dmabuf);
18838	if (handled)
18839	return true;
18840
18841	return false;
18842	}
18843
18844	/**
18845	* lpfc_sli4_seq_abort_rsp_cmpl - BLS ABORT RSP seq abort iocb complete handler
18846	* @phba: Pointer to HBA context object.
18847	* @cmd_iocbq: pointer to the command iocbq structure.
18848	* @rsp_iocbq: pointer to the response iocbq structure.
18849	*
18850	* This function handles the sequence abort response iocb command complete
18851	* event. It properly releases the memory allocated to the sequence abort
18852	* accept iocb.
18853	**/
18854	static void
18855	lpfc_sli4_seq_abort_rsp_cmpl(struct lpfc_hba *phba,
18856	struct lpfc_iocbq *cmd_iocbq,
18857	struct lpfc_iocbq *rsp_iocbq)
18858	{
18859	if (cmd_iocbq) {
18860	lpfc_nlp_put(cmd_iocbq->ndlp);
18861	lpfc_sli_release_iocbq(phba, iocbq: cmd_iocbq);
18862	}
18863
18864	/* Failure means BLS ABORT RSP did not get delivered to remote node*/
18865	if (rsp_iocbq && rsp_iocbq->iocb.ulpStatus)
18866	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18867	"3154 BLS ABORT RSP failed, data: x%x/x%x\n",
18868	get_job_ulpstatus(phba, rsp_iocbq),
18869	get_job_word4(phba, rsp_iocbq));
18870	}
18871
18872	/**
18873	* lpfc_sli4_xri_inrange - check xri is in range of xris owned by driver.
18874	* @phba: Pointer to HBA context object.
18875	* @xri: xri id in transaction.
18876	*
18877	* This function validates the xri maps to the known range of XRIs allocated an
18878	* used by the driver.
18879	**/
18880	uint16_t
18881	lpfc_sli4_xri_inrange(struct lpfc_hba *phba,
18882	uint16_t xri)
18883	{
18884	uint16_t i;
18885
18886	for (i = 0; i < phba->sli4_hba.max_cfg_param.max_xri; i++) {
18887	if (xri == phba->sli4_hba.xri_ids[i])
18888	return i;
18889	}
18890	return NO_XRI;
18891	}
18892
18893	/**
18894	* lpfc_sli4_seq_abort_rsp - bls rsp to sequence abort
18895	* @vport: pointer to a virtual port.
18896	* @fc_hdr: pointer to a FC frame header.
18897	* @aborted: was the partially assembled receive sequence successfully aborted
18898	*
18899	* This function sends a basic response to a previous unsol sequence abort
18900	* event after aborting the sequence handling.
18901	**/
18902	void
18903	lpfc_sli4_seq_abort_rsp(struct lpfc_vport *vport,
18904	struct fc_frame_header *fc_hdr, bool aborted)
18905	{
18906	struct lpfc_hba *phba = vport->phba;
18907	struct lpfc_iocbq *ctiocb = NULL;
18908	struct lpfc_nodelist *ndlp;
18909	uint16_t oxid, rxid, xri, lxri;
18910	uint32_t sid, fctl;
18911	union lpfc_wqe128 *icmd;
18912	int rc;
18913
18914	if (!lpfc_is_link_up(phba))
18915	return;
18916
18917	sid = sli4_sid_from_fc_hdr(fc_hdr);
18918	oxid = be16_to_cpu(fc_hdr->fh_ox_id);
18919	rxid = be16_to_cpu(fc_hdr->fh_rx_id);
18920
18921	ndlp = lpfc_findnode_did(vport, sid);
18922	if (!ndlp) {
18923	ndlp = lpfc_nlp_init(vport, did: sid);
18924	if (!ndlp) {
18925	lpfc_printf_vlog(vport, KERN_WARNING, LOG_ELS,
18926	"1268 Failed to allocate ndlp for "
18927	"oxid:x%x SID:x%x\n", oxid, sid);
18928	return;
18929	}
18930	/* Put ndlp onto pport node list */
18931	lpfc_enqueue_node(vport, ndlp);
18932	}
18933
18934	/* Allocate buffer for rsp iocb */
18935	ctiocb = lpfc_sli_get_iocbq(phba);
18936	if (!ctiocb)
18937	return;
18938
18939	icmd = &ctiocb->wqe;
18940
18941	/* Extract the F_CTL field from FC_HDR */
18942	fctl = sli4_fctl_from_fc_hdr(fc_hdr);
18943
18944	ctiocb->ndlp = lpfc_nlp_get(ndlp);
18945	if (!ctiocb->ndlp) {
18946	lpfc_sli_release_iocbq(phba, iocbq: ctiocb);
18947	return;
18948	}
18949
18950	ctiocb->vport = phba->pport;
18951	ctiocb->cmd_cmpl = lpfc_sli4_seq_abort_rsp_cmpl;
18952	ctiocb->sli4_lxritag = NO_XRI;
18953	ctiocb->sli4_xritag = NO_XRI;
18954	ctiocb->abort_rctl = FC_RCTL_BA_ACC;
18955
18956	if (fctl & FC_FC_EX_CTX)
18957	/* Exchange responder sent the abort so we
18958	* own the oxid.
18959	*/
18960	xri = oxid;
18961	else
18962	xri = rxid;
18963	lxri = lpfc_sli4_xri_inrange(phba, xri);
18964	if (lxri != NO_XRI)
18965	lpfc_set_rrq_active(phba, ndlp, xritag: lxri,
18966	rxid: (xri == oxid) ? rxid : oxid, send_rrq: 0);
18967	/* For BA_ABTS from exchange responder, if the logical xri with
18968	* the oxid maps to the FCP XRI range, the port no longer has
18969	* that exchange context, send a BLS_RJT. Override the IOCB for
18970	* a BA_RJT.
18971	*/
18972	if ((fctl & FC_FC_EX_CTX) &&
18973	(lxri > lpfc_sli4_get_iocb_cnt(phba))) {
18974	ctiocb->abort_rctl = FC_RCTL_BA_RJT;
18975	bf_set(xmit_bls_rsp64_rjt_vspec, &icmd->xmit_bls_rsp, 0);
18976	bf_set(xmit_bls_rsp64_rjt_expc, &icmd->xmit_bls_rsp,
18977	FC_BA_RJT_INV_XID);
18978	bf_set(xmit_bls_rsp64_rjt_rsnc, &icmd->xmit_bls_rsp,
18979	FC_BA_RJT_UNABLE);
18980	}
18981
18982	/* If BA_ABTS failed to abort a partially assembled receive sequence,
18983	* the driver no longer has that exchange, send a BLS_RJT. Override
18984	* the IOCB for a BA_RJT.
18985	*/
18986	if (aborted == false) {
18987	ctiocb->abort_rctl = FC_RCTL_BA_RJT;
18988	bf_set(xmit_bls_rsp64_rjt_vspec, &icmd->xmit_bls_rsp, 0);
18989	bf_set(xmit_bls_rsp64_rjt_expc, &icmd->xmit_bls_rsp,
18990	FC_BA_RJT_INV_XID);
18991	bf_set(xmit_bls_rsp64_rjt_rsnc, &icmd->xmit_bls_rsp,
18992	FC_BA_RJT_UNABLE);
18993	}
18994
18995	if (fctl & FC_FC_EX_CTX) {
18996	/* ABTS sent by responder to CT exchange, construction
18997	* of BA_ACC will use OX_ID from ABTS for the XRI_TAG
18998	* field and RX_ID from ABTS for RX_ID field.
18999	*/
19000	ctiocb->abort_bls = LPFC_ABTS_UNSOL_RSP;
19001	bf_set(xmit_bls_rsp64_rxid, &icmd->xmit_bls_rsp, rxid);
19002	} else {
19003	/* ABTS sent by initiator to CT exchange, construction
19004	* of BA_ACC will need to allocate a new XRI as for the
19005	* XRI_TAG field.
19006	*/
19007	ctiocb->abort_bls = LPFC_ABTS_UNSOL_INT;
19008	}
19009
19010	/* OX_ID is invariable to who sent ABTS to CT exchange */
19011	bf_set(xmit_bls_rsp64_oxid, &icmd->xmit_bls_rsp, oxid);
19012	bf_set(xmit_bls_rsp64_oxid, &icmd->xmit_bls_rsp, rxid);
19013
19014	/* Use CT=VPI */
19015	bf_set(wqe_els_did, &icmd->xmit_bls_rsp.wqe_dest,
19016	ndlp->nlp_DID);
19017	bf_set(xmit_bls_rsp64_temprpi, &icmd->xmit_bls_rsp,
19018	phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
19019	bf_set(wqe_cmnd, &icmd->generic.wqe_com, CMD_XMIT_BLS_RSP64_CX);
19020
19021	/* Xmit CT abts response on exchange <xid> */
19022	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
19023	"1200 Send BLS cmd x%x on oxid x%x Data: x%x\n",
19024	ctiocb->abort_rctl, oxid, phba->link_state);
19025
19026	rc = lpfc_sli_issue_iocb(phba, LPFC_ELS_RING, piocb: ctiocb, flag: 0);
19027	if (rc == IOCB_ERROR) {
19028	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
19029	"2925 Failed to issue CT ABTS RSP x%x on "
19030	"xri x%x, Data x%x\n",
19031	ctiocb->abort_rctl, oxid,
19032	phba->link_state);
19033	lpfc_nlp_put(ndlp);
19034	ctiocb->ndlp = NULL;
19035	lpfc_sli_release_iocbq(phba, iocbq: ctiocb);
19036	}
19037	}
19038
19039	/**
19040	* lpfc_sli4_handle_unsol_abort - Handle sli-4 unsolicited abort event
19041	* @vport: Pointer to the vport on which this sequence was received
19042	* @dmabuf: pointer to a dmabuf that describes the FC sequence
19043	*
19044	* This function handles an SLI-4 unsolicited abort event. If the unsolicited
19045	* receive sequence is only partially assembed by the driver, it shall abort
19046	* the partially assembled frames for the sequence. Otherwise, if the
19047	* unsolicited receive sequence has been completely assembled and passed to
19048	* the Upper Layer Protocol (ULP), it then mark the per oxid status for the
19049	* unsolicited sequence has been aborted. After that, it will issue a basic
19050	* accept to accept the abort.
19051	**/
19052	static void
19053	lpfc_sli4_handle_unsol_abort(struct lpfc_vport *vport,
19054	struct hbq_dmabuf *dmabuf)
19055	{
19056	struct lpfc_hba *phba = vport->phba;
19057	struct fc_frame_header fc_hdr;
19058	uint32_t fctl;
19059	bool aborted;
19060
19061	/* Make a copy of fc_hdr before the dmabuf being released */
19062	memcpy(&fc_hdr, dmabuf->hbuf.virt, sizeof(struct fc_frame_header));
19063	fctl = sli4_fctl_from_fc_hdr(&fc_hdr);
19064
19065	if (fctl & FC_FC_EX_CTX) {
19066	/* ABTS by responder to exchange, no cleanup needed */
19067	aborted = true;
19068	} else {
19069	/* ABTS by initiator to exchange, need to do cleanup */
19070	aborted = lpfc_sli4_abort_partial_seq(vport, dmabuf);
19071	if (aborted == false)
19072	aborted = lpfc_sli4_abort_ulp_seq(vport, dmabuf);
19073	}
19074	lpfc_in_buf_free(phba, &dmabuf->dbuf);
19075
19076	if (phba->nvmet_support) {
19077	lpfc_nvmet_rcv_unsol_abort(vport, fc_hdr: &fc_hdr);
19078	return;
19079	}
19080
19081	/* Respond with BA_ACC or BA_RJT accordingly */
19082	lpfc_sli4_seq_abort_rsp(vport, fc_hdr: &fc_hdr, aborted);
19083	}
19084
19085	/**
19086	* lpfc_seq_complete - Indicates if a sequence is complete
19087	* @dmabuf: pointer to a dmabuf that describes the FC sequence
19088	*
19089	* This function checks the sequence, starting with the frame described by
19090	* @dmabuf, to see if all the frames associated with this sequence are present.
19091	* the frames associated with this sequence are linked to the @dmabuf using the
19092	* dbuf list. This function looks for two major things. 1) That the first frame
19093	* has a sequence count of zero. 2) There is a frame with last frame of sequence
19094	* set. 3) That there are no holes in the sequence count. The function will
19095	* return 1 when the sequence is complete, otherwise it will return 0.
19096	**/
19097	static int
19098	lpfc_seq_complete(struct hbq_dmabuf *dmabuf)
19099	{
19100	struct fc_frame_header *hdr;
19101	struct lpfc_dmabuf *d_buf;
19102	struct hbq_dmabuf *seq_dmabuf;
19103	uint32_t fctl;
19104	int seq_count = 0;
19105
19106	hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
19107	/* make sure first fame of sequence has a sequence count of zero */
19108	if (hdr->fh_seq_cnt != seq_count)
19109	return 0;
19110	fctl = (hdr->fh_f_ctl[0] << 16 |
19111	hdr->fh_f_ctl[1] << 8 |
19112	hdr->fh_f_ctl[2]);
19113	/* If last frame of sequence we can return success. */
19114	if (fctl & FC_FC_END_SEQ)
19115	return 1;
19116	list_for_each_entry(d_buf, &dmabuf->dbuf.list, list) {
19117	seq_dmabuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
19118	hdr = (struct fc_frame_header *)seq_dmabuf->hbuf.virt;
19119	/* If there is a hole in the sequence count then fail. */
19120	if (++seq_count != be16_to_cpu(hdr->fh_seq_cnt))
19121	return 0;
19122	fctl = (hdr->fh_f_ctl[0] << 16 |
19123	hdr->fh_f_ctl[1] << 8 |
19124	hdr->fh_f_ctl[2]);
19125	/* If last frame of sequence we can return success. */
19126	if (fctl & FC_FC_END_SEQ)
19127	return 1;
19128	}
19129	return 0;
19130	}
19131
19132	/**
19133	* lpfc_prep_seq - Prep sequence for ULP processing
19134	* @vport: Pointer to the vport on which this sequence was received
19135	* @seq_dmabuf: pointer to a dmabuf that describes the FC sequence
19136	*
19137	* This function takes a sequence, described by a list of frames, and creates
19138	* a list of iocbq structures to describe the sequence. This iocbq list will be
19139	* used to issue to the generic unsolicited sequence handler. This routine
19140	* returns a pointer to the first iocbq in the list. If the function is unable
19141	* to allocate an iocbq then it throw out the received frames that were not
19142	* able to be described and return a pointer to the first iocbq. If unable to
19143	* allocate any iocbqs (including the first) this function will return NULL.
19144	**/
19145	static struct lpfc_iocbq *
19146	lpfc_prep_seq(struct lpfc_vport *vport, struct hbq_dmabuf *seq_dmabuf)
19147	{
19148	struct hbq_dmabuf *hbq_buf;
19149	struct lpfc_dmabuf *d_buf, *n_buf;
19150	struct lpfc_iocbq *first_iocbq, *iocbq;
19151	struct fc_frame_header *fc_hdr;
19152	uint32_t sid;
19153	uint32_t len, tot_len;
19154
19155	fc_hdr = (struct fc_frame_header *)seq_dmabuf->hbuf.virt;
19156	/* remove from receive buffer list */
19157	list_del_init(entry: &seq_dmabuf->hbuf.list);
19158	lpfc_update_rcv_time_stamp(vport);
19159	/* get the Remote Port's SID */
19160	sid = sli4_sid_from_fc_hdr(fc_hdr);
19161	tot_len = 0;
19162	/* Get an iocbq struct to fill in. */
19163	first_iocbq = lpfc_sli_get_iocbq(phba: vport->phba);
19164	if (first_iocbq) {
19165	/* Initialize the first IOCB. */
19166	first_iocbq->wcqe_cmpl.total_data_placed = 0;
19167	bf_set(lpfc_wcqe_c_status, &first_iocbq->wcqe_cmpl,
19168	IOSTAT_SUCCESS);
19169	first_iocbq->vport = vport;
19170
19171	/* Check FC Header to see what TYPE of frame we are rcv'ing */
19172	if (sli4_type_from_fc_hdr(fc_hdr) == FC_TYPE_ELS) {
19173	bf_set(els_rsp64_sid, &first_iocbq->wqe.xmit_els_rsp,
19174	sli4_did_from_fc_hdr(fc_hdr));
19175	}
19176
19177	bf_set(wqe_ctxt_tag, &first_iocbq->wqe.xmit_els_rsp.wqe_com,
19178	NO_XRI);
19179	bf_set(wqe_rcvoxid, &first_iocbq->wqe.xmit_els_rsp.wqe_com,
19180	be16_to_cpu(fc_hdr->fh_ox_id));
19181
19182	/* put the first buffer into the first iocb */
19183	tot_len = bf_get(lpfc_rcqe_length,
19184	&seq_dmabuf->cq_event.cqe.rcqe_cmpl);
19185
19186	first_iocbq->cmd_dmabuf = &seq_dmabuf->dbuf;
19187	first_iocbq->bpl_dmabuf = NULL;
19188	/* Keep track of the BDE count */
19189	first_iocbq->wcqe_cmpl.word3 = 1;
19190
19191	if (tot_len > LPFC_DATA_BUF_SIZE)
19192	first_iocbq->wqe.gen_req.bde.tus.f.bdeSize =
19193	LPFC_DATA_BUF_SIZE;
19194	else
19195	first_iocbq->wqe.gen_req.bde.tus.f.bdeSize = tot_len;
19196
19197	first_iocbq->wcqe_cmpl.total_data_placed = tot_len;
19198	bf_set(wqe_els_did, &first_iocbq->wqe.xmit_els_rsp.wqe_dest,
19199	sid);
19200	}
19201	iocbq = first_iocbq;
19202	/*
19203	* Each IOCBq can have two Buffers assigned, so go through the list
19204	* of buffers for this sequence and save two buffers in each IOCBq
19205	*/
19206	list_for_each_entry_safe(d_buf, n_buf, &seq_dmabuf->dbuf.list, list) {
19207	if (!iocbq) {
19208	lpfc_in_buf_free(vport->phba, d_buf);
19209	continue;
19210	}
19211	if (!iocbq->bpl_dmabuf) {
19212	iocbq->bpl_dmabuf = d_buf;
19213	iocbq->wcqe_cmpl.word3++;
19214	/* We need to get the size out of the right CQE */
19215	hbq_buf = container_of(d_buf, struct hbq_dmabuf, dbuf);
19216	len = bf_get(lpfc_rcqe_length,
19217	&hbq_buf->cq_event.cqe.rcqe_cmpl);
19218	iocbq->unsol_rcv_len = len;
19219	iocbq->wcqe_cmpl.total_data_placed += len;
19220	tot_len += len;
19221	} else {
19222	iocbq = lpfc_sli_get_iocbq(phba: vport->phba);
19223	if (!iocbq) {
19224	if (first_iocbq) {
19225	bf_set(lpfc_wcqe_c_status,
19226	&first_iocbq->wcqe_cmpl,
19227	IOSTAT_SUCCESS);
19228	first_iocbq->wcqe_cmpl.parameter =
19229	IOERR_NO_RESOURCES;
19230	}
19231	lpfc_in_buf_free(vport->phba, d_buf);
19232	continue;
19233	}
19234	/* We need to get the size out of the right CQE */
19235	hbq_buf = container_of(d_buf, struct hbq_dmabuf, dbuf);
19236	len = bf_get(lpfc_rcqe_length,
19237	&hbq_buf->cq_event.cqe.rcqe_cmpl);
19238	iocbq->cmd_dmabuf = d_buf;
19239	iocbq->bpl_dmabuf = NULL;
19240	iocbq->wcqe_cmpl.word3 = 1;
19241
19242	if (len > LPFC_DATA_BUF_SIZE)
19243	iocbq->wqe.xmit_els_rsp.bde.tus.f.bdeSize =
19244	LPFC_DATA_BUF_SIZE;
19245	else
19246	iocbq->wqe.xmit_els_rsp.bde.tus.f.bdeSize =
19247	len;
19248
19249	tot_len += len;
19250	iocbq->wcqe_cmpl.total_data_placed = tot_len;
19251	bf_set(wqe_els_did, &iocbq->wqe.xmit_els_rsp.wqe_dest,
19252	sid);
19253	list_add_tail(new: &iocbq->list, head: &first_iocbq->list);
19254	}
19255	}
19256	/* Free the sequence's header buffer */
19257	if (!first_iocbq)
19258	lpfc_in_buf_free(vport->phba, &seq_dmabuf->dbuf);
19259
19260	return first_iocbq;
19261	}
19262
19263	static void
19264	lpfc_sli4_send_seq_to_ulp(struct lpfc_vport *vport,
19265	struct hbq_dmabuf *seq_dmabuf)
19266	{
19267	struct fc_frame_header *fc_hdr;
19268	struct lpfc_iocbq *iocbq, *curr_iocb, *next_iocb;
19269	struct lpfc_hba *phba = vport->phba;
19270
19271	fc_hdr = (struct fc_frame_header *)seq_dmabuf->hbuf.virt;
19272	iocbq = lpfc_prep_seq(vport, seq_dmabuf);
19273	if (!iocbq) {
19274	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19275	"2707 Ring %d handler: Failed to allocate "
19276	"iocb Rctl x%x Type x%x received\n",
19277	LPFC_ELS_RING,
19278	fc_hdr->fh_r_ctl, fc_hdr->fh_type);
19279	return;
19280	}
19281	if (!lpfc_complete_unsol_iocb(phba,
19282	pring: phba->sli4_hba.els_wq->pring,
19283	saveq: iocbq, fch_r_ctl: fc_hdr->fh_r_ctl,
19284	fch_type: fc_hdr->fh_type)) {
19285	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19286	"2540 Ring %d handler: unexpected Rctl "
19287	"x%x Type x%x received\n",
19288	LPFC_ELS_RING,
19289	fc_hdr->fh_r_ctl, fc_hdr->fh_type);
19290	lpfc_in_buf_free(phba, &seq_dmabuf->dbuf);
19291	}
19292
19293	/* Free iocb created in lpfc_prep_seq */
19294	list_for_each_entry_safe(curr_iocb, next_iocb,
19295	&iocbq->list, list) {
19296	list_del_init(entry: &curr_iocb->list);
19297	lpfc_sli_release_iocbq(phba, iocbq: curr_iocb);
19298	}
19299	lpfc_sli_release_iocbq(phba, iocbq);
19300	}
19301
19302	static void
19303	lpfc_sli4_mds_loopback_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
19304	struct lpfc_iocbq *rspiocb)
19305	{
19306	struct lpfc_dmabuf *pcmd = cmdiocb->cmd_dmabuf;
19307
19308	if (pcmd && pcmd->virt)
19309	dma_pool_free(pool: phba->lpfc_drb_pool, vaddr: pcmd->virt, addr: pcmd->phys);
19310	kfree(objp: pcmd);
19311	lpfc_sli_release_iocbq(phba, iocbq: cmdiocb);
19312	lpfc_drain_txq(phba);
19313	}
19314
19315	static void
19316	lpfc_sli4_handle_mds_loopback(struct lpfc_vport *vport,
19317	struct hbq_dmabuf *dmabuf)
19318	{
19319	struct fc_frame_header *fc_hdr;
19320	struct lpfc_hba *phba = vport->phba;
19321	struct lpfc_iocbq *iocbq = NULL;
19322	union lpfc_wqe128 *pwqe;
19323	struct lpfc_dmabuf *pcmd = NULL;
19324	uint32_t frame_len;
19325	int rc;
19326	unsigned long iflags;
19327
19328	fc_hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
19329	frame_len = bf_get(lpfc_rcqe_length, &dmabuf->cq_event.cqe.rcqe_cmpl);
19330
19331	/* Send the received frame back */
19332	iocbq = lpfc_sli_get_iocbq(phba);
19333	if (!iocbq) {
19334	/* Queue cq event and wakeup worker thread to process it */
19335	spin_lock_irqsave(&phba->hbalock, iflags);
19336	list_add_tail(new: &dmabuf->cq_event.list,
19337	head: &phba->sli4_hba.sp_queue_event);
19338	phba->hba_flag |= HBA_SP_QUEUE_EVT;
19339	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflags);
19340	lpfc_worker_wake_up(phba);
19341	return;
19342	}
19343
19344	/* Allocate buffer for command payload */
19345	pcmd = kmalloc(size: sizeof(struct lpfc_dmabuf), GFP_KERNEL);
19346	if (pcmd)
19347	pcmd->virt = dma_pool_alloc(pool: phba->lpfc_drb_pool, GFP_KERNEL,
19348	handle: &pcmd->phys);
19349	if (!pcmd || !pcmd->virt)
19350	goto exit;
19351
19352	INIT_LIST_HEAD(list: &pcmd->list);
19353
19354	/* copyin the payload */
19355	memcpy(pcmd->virt, dmabuf->dbuf.virt, frame_len);
19356
19357	iocbq->cmd_dmabuf = pcmd;
19358	iocbq->vport = vport;
19359	iocbq->cmd_flag &= ~LPFC_FIP_ELS_ID_MASK;
19360	iocbq->cmd_flag |= LPFC_USE_FCPWQIDX;
19361	iocbq->num_bdes = 0;
19362
19363	pwqe = &iocbq->wqe;
19364	/* fill in BDE's for command */
19365	pwqe->gen_req.bde.addrHigh = putPaddrHigh(pcmd->phys);
19366	pwqe->gen_req.bde.addrLow = putPaddrLow(pcmd->phys);
19367	pwqe->gen_req.bde.tus.f.bdeSize = frame_len;
19368	pwqe->gen_req.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
19369
19370	pwqe->send_frame.frame_len = frame_len;
19371	pwqe->send_frame.fc_hdr_wd0 = be32_to_cpu(*((__be32 *)fc_hdr));
19372	pwqe->send_frame.fc_hdr_wd1 = be32_to_cpu(*((__be32 *)fc_hdr + 1));
19373	pwqe->send_frame.fc_hdr_wd2 = be32_to_cpu(*((__be32 *)fc_hdr + 2));
19374	pwqe->send_frame.fc_hdr_wd3 = be32_to_cpu(*((__be32 *)fc_hdr + 3));
19375	pwqe->send_frame.fc_hdr_wd4 = be32_to_cpu(*((__be32 *)fc_hdr + 4));
19376	pwqe->send_frame.fc_hdr_wd5 = be32_to_cpu(*((__be32 *)fc_hdr + 5));
19377
19378	pwqe->generic.wqe_com.word7 = 0;
19379	pwqe->generic.wqe_com.word10 = 0;
19380
19381	bf_set(wqe_cmnd, &pwqe->generic.wqe_com, CMD_SEND_FRAME);
19382	bf_set(wqe_sof, &pwqe->generic.wqe_com, 0x2E); /* SOF byte */
19383	bf_set(wqe_eof, &pwqe->generic.wqe_com, 0x41); /* EOF byte */
19384	bf_set(wqe_lenloc, &pwqe->generic.wqe_com, 1);
19385	bf_set(wqe_xbl, &pwqe->generic.wqe_com, 1);
19386	bf_set(wqe_dbde, &pwqe->generic.wqe_com, 1);
19387	bf_set(wqe_xc, &pwqe->generic.wqe_com, 1);
19388	bf_set(wqe_cmd_type, &pwqe->generic.wqe_com, 0xA);
19389	bf_set(wqe_cqid, &pwqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
19390	bf_set(wqe_xri_tag, &pwqe->generic.wqe_com, iocbq->sli4_xritag);
19391	bf_set(wqe_reqtag, &pwqe->generic.wqe_com, iocbq->iotag);
19392	bf_set(wqe_class, &pwqe->generic.wqe_com, CLASS3);
19393	pwqe->generic.wqe_com.abort_tag = iocbq->iotag;
19394
19395	iocbq->cmd_cmpl = lpfc_sli4_mds_loopback_cmpl;
19396
19397	rc = lpfc_sli_issue_iocb(phba, LPFC_ELS_RING, piocb: iocbq, flag: 0);
19398	if (rc == IOCB_ERROR)
19399	goto exit;
19400
19401	lpfc_in_buf_free(phba, &dmabuf->dbuf);
19402	return;
19403
19404	exit:
19405	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
19406	"2023 Unable to process MDS loopback frame\n");
19407	if (pcmd && pcmd->virt)
19408	dma_pool_free(pool: phba->lpfc_drb_pool, vaddr: pcmd->virt, addr: pcmd->phys);
19409	kfree(objp: pcmd);
19410	if (iocbq)
19411	lpfc_sli_release_iocbq(phba, iocbq);
19412	lpfc_in_buf_free(phba, &dmabuf->dbuf);
19413	}
19414
19415	/**
19416	* lpfc_sli4_handle_received_buffer - Handle received buffers from firmware
19417	* @phba: Pointer to HBA context object.
19418	* @dmabuf: Pointer to a dmabuf that describes the FC sequence.
19419	*
19420	* This function is called with no lock held. This function processes all
19421	* the received buffers and gives it to upper layers when a received buffer
19422	* indicates that it is the final frame in the sequence. The interrupt
19423	* service routine processes received buffers at interrupt contexts.
19424	* Worker thread calls lpfc_sli4_handle_received_buffer, which will call the
19425	* appropriate receive function when the final frame in a sequence is received.
19426	**/
19427	void
19428	lpfc_sli4_handle_received_buffer(struct lpfc_hba *phba,
19429	struct hbq_dmabuf *dmabuf)
19430	{
19431	struct hbq_dmabuf *seq_dmabuf;
19432	struct fc_frame_header *fc_hdr;
19433	struct lpfc_vport *vport;
19434	uint32_t fcfi;
19435	uint32_t did;
19436
19437	/* Process each received buffer */
19438	fc_hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
19439
19440	if (fc_hdr->fh_r_ctl == FC_RCTL_MDS_DIAGS ||
19441	fc_hdr->fh_r_ctl == FC_RCTL_DD_UNSOL_DATA) {
19442	vport = phba->pport;
19443	/* Handle MDS Loopback frames */
19444	if (!(phba->pport->load_flag & FC_UNLOADING))
19445	lpfc_sli4_handle_mds_loopback(vport, dmabuf);
19446	else
19447	lpfc_in_buf_free(phba, &dmabuf->dbuf);
19448	return;
19449	}
19450
19451	/* check to see if this a valid type of frame */
19452	if (lpfc_fc_frame_check(phba, fc_hdr)) {
19453	lpfc_in_buf_free(phba, &dmabuf->dbuf);
19454	return;
19455	}
19456
19457	if ((bf_get(lpfc_cqe_code,
19458	&dmabuf->cq_event.cqe.rcqe_cmpl) == CQE_CODE_RECEIVE_V1))
19459	fcfi = bf_get(lpfc_rcqe_fcf_id_v1,
19460	&dmabuf->cq_event.cqe.rcqe_cmpl);
19461	else
19462	fcfi = bf_get(lpfc_rcqe_fcf_id,
19463	&dmabuf->cq_event.cqe.rcqe_cmpl);
19464
19465	if (fc_hdr->fh_r_ctl == 0xF4 && fc_hdr->fh_type == 0xFF) {
19466	vport = phba->pport;
19467	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
19468	"2023 MDS Loopback %d bytes\n",
19469	bf_get(lpfc_rcqe_length,
19470	&dmabuf->cq_event.cqe.rcqe_cmpl));
19471	/* Handle MDS Loopback frames */
19472	lpfc_sli4_handle_mds_loopback(vport, dmabuf);
19473	return;
19474	}
19475
19476	/* d_id this frame is directed to */
19477	did = sli4_did_from_fc_hdr(fc_hdr);
19478
19479	vport = lpfc_fc_frame_to_vport(phba, fc_hdr, fcfi, did);
19480	if (!vport) {
19481	/* throw out the frame */
19482	lpfc_in_buf_free(phba, &dmabuf->dbuf);
19483	return;
19484	}
19485
19486	/* vport is registered unless we rcv a FLOGI directed to Fabric_DID */
19487	if (!(vport->vpi_state & LPFC_VPI_REGISTERED) &&
19488	(did != Fabric_DID)) {
19489	/*
19490	* Throw out the frame if we are not pt2pt.
19491	* The pt2pt protocol allows for discovery frames
19492	* to be received without a registered VPI.
19493	*/
19494	if (!(vport->fc_flag & FC_PT2PT) ||
19495	(phba->link_state == LPFC_HBA_READY)) {
19496	lpfc_in_buf_free(phba, &dmabuf->dbuf);
19497	return;
19498	}
19499	}
19500
19501	/* Handle the basic abort sequence (BA_ABTS) event */
19502	if (fc_hdr->fh_r_ctl == FC_RCTL_BA_ABTS) {
19503	lpfc_sli4_handle_unsol_abort(vport, dmabuf);
19504	return;
19505	}
19506
19507	/* Link this frame */
19508	seq_dmabuf = lpfc_fc_frame_add(vport, dmabuf);
19509	if (!seq_dmabuf) {
19510	/* unable to add frame to vport - throw it out */
19511	lpfc_in_buf_free(phba, &dmabuf->dbuf);
19512	return;
19513	}
19514	/* If not last frame in sequence continue processing frames. */
19515	if (!lpfc_seq_complete(dmabuf: seq_dmabuf))
19516	return;
19517
19518	/* Send the complete sequence to the upper layer protocol */
19519	lpfc_sli4_send_seq_to_ulp(vport, seq_dmabuf);
19520	}
19521
19522	/**
19523	* lpfc_sli4_post_all_rpi_hdrs - Post the rpi header memory region to the port
19524	* @phba: pointer to lpfc hba data structure.
19525	*
19526	* This routine is invoked to post rpi header templates to the
19527	* HBA consistent with the SLI-4 interface spec. This routine
19528	* posts a SLI4_PAGE_SIZE memory region to the port to hold up to
19529	* SLI4_PAGE_SIZE modulo 64 rpi context headers.
19530	*
19531	* This routine does not require any locks. It's usage is expected
19532	* to be driver load or reset recovery when the driver is
19533	* sequential.
19534	*
19535	* Return codes
19536	* 0 - successful
19537	* -EIO - The mailbox failed to complete successfully.
19538	* When this error occurs, the driver is not guaranteed
19539	* to have any rpi regions posted to the device and
19540	* must either attempt to repost the regions or take a
19541	* fatal error.
19542	**/
19543	int
19544	lpfc_sli4_post_all_rpi_hdrs(struct lpfc_hba *phba)
19545	{
19546	struct lpfc_rpi_hdr *rpi_page;
19547	uint32_t rc = 0;
19548	uint16_t lrpi = 0;
19549
19550	/* SLI4 ports that support extents do not require RPI headers. */
19551	if (!phba->sli4_hba.rpi_hdrs_in_use)
19552	goto exit;
19553	if (phba->sli4_hba.extents_in_use)
19554	return -EIO;
19555
19556	list_for_each_entry(rpi_page, &phba->sli4_hba.lpfc_rpi_hdr_list, list) {
19557	/*
19558	* Assign the rpi headers a physical rpi only if the driver
19559	* has not initialized those resources. A port reset only
19560	* needs the headers posted.
19561	*/
19562	if (bf_get(lpfc_rpi_rsrc_rdy, &phba->sli4_hba.sli4_flags) !=
19563	LPFC_RPI_RSRC_RDY)
19564	rpi_page->start_rpi = phba->sli4_hba.rpi_ids[lrpi];
19565
19566	rc = lpfc_sli4_post_rpi_hdr(phba, rpi_page);
19567	if (rc != MBX_SUCCESS) {
19568	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19569	"2008 Error %d posting all rpi "
19570	"headers\n", rc);
19571	rc = -EIO;
19572	break;
19573	}
19574	}
19575
19576	exit:
19577	bf_set(lpfc_rpi_rsrc_rdy, &phba->sli4_hba.sli4_flags,
19578	LPFC_RPI_RSRC_RDY);
19579	return rc;
19580	}
19581
19582	/**
19583	* lpfc_sli4_post_rpi_hdr - Post an rpi header memory region to the port
19584	* @phba: pointer to lpfc hba data structure.
19585	* @rpi_page: pointer to the rpi memory region.
19586	*
19587	* This routine is invoked to post a single rpi header to the
19588	* HBA consistent with the SLI-4 interface spec. This memory region
19589	* maps up to 64 rpi context regions.
19590	*
19591	* Return codes
19592	* 0 - successful
19593	* -ENOMEM - No available memory
19594	* -EIO - The mailbox failed to complete successfully.
19595	**/
19596	int
19597	lpfc_sli4_post_rpi_hdr(struct lpfc_hba *phba, struct lpfc_rpi_hdr *rpi_page)
19598	{
19599	LPFC_MBOXQ_t *mboxq;
19600	struct lpfc_mbx_post_hdr_tmpl *hdr_tmpl;
19601	uint32_t rc = 0;
19602	uint32_t shdr_status, shdr_add_status;
19603	union lpfc_sli4_cfg_shdr *shdr;
19604
19605	/* SLI4 ports that support extents do not require RPI headers. */
19606	if (!phba->sli4_hba.rpi_hdrs_in_use)
19607	return rc;
19608	if (phba->sli4_hba.extents_in_use)
19609	return -EIO;
19610
19611	/* The port is notified of the header region via a mailbox command. */
19612	mboxq = (LPFC_MBOXQ_t *) mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
19613	if (!mboxq) {
19614	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19615	"2001 Unable to allocate memory for issuing "
19616	"SLI_CONFIG_SPECIAL mailbox command\n");
19617	return -ENOMEM;
19618	}
19619
19620	/* Post all rpi memory regions to the port. */
19621	hdr_tmpl = &mboxq->u.mqe.un.hdr_tmpl;
19622	lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_FCOE,
19623	LPFC_MBOX_OPCODE_FCOE_POST_HDR_TEMPLATE,
19624	sizeof(struct lpfc_mbx_post_hdr_tmpl) -
19625	sizeof(struct lpfc_sli4_cfg_mhdr),
19626	LPFC_SLI4_MBX_EMBED);
19627
19628
19629	/* Post the physical rpi to the port for this rpi header. */
19630	bf_set(lpfc_mbx_post_hdr_tmpl_rpi_offset, hdr_tmpl,
19631	rpi_page->start_rpi);
19632	bf_set(lpfc_mbx_post_hdr_tmpl_page_cnt,
19633	hdr_tmpl, rpi_page->page_count);
19634
19635	hdr_tmpl->rpi_paddr_lo = putPaddrLow(rpi_page->dmabuf->phys);
19636	hdr_tmpl->rpi_paddr_hi = putPaddrHigh(rpi_page->dmabuf->phys);
19637	rc = lpfc_sli_issue_mbox(phba, pmbox: mboxq, MBX_POLL);
19638	shdr = (union lpfc_sli4_cfg_shdr *) &hdr_tmpl->header.cfg_shdr;
19639	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
19640	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
19641	mempool_free(element: mboxq, pool: phba->mbox_mem_pool);
19642	if (shdr_status || shdr_add_status || rc) {
19643	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19644	"2514 POST_RPI_HDR mailbox failed with "
19645	"status x%x add_status x%x, mbx status x%x\n",
19646	shdr_status, shdr_add_status, rc);
19647	rc = -ENXIO;
19648	} else {
19649	/*
19650	* The next_rpi stores the next logical module-64 rpi value used
19651	* to post physical rpis in subsequent rpi postings.
19652	*/
19653	spin_lock_irq(lock: &phba->hbalock);
19654	phba->sli4_hba.next_rpi = rpi_page->next_rpi;
19655	spin_unlock_irq(lock: &phba->hbalock);
19656	}
19657	return rc;
19658	}
19659
19660	/**
19661	* lpfc_sli4_alloc_rpi - Get an available rpi in the device's range
19662	* @phba: pointer to lpfc hba data structure.
19663	*
19664	* This routine is invoked to post rpi header templates to the
19665	* HBA consistent with the SLI-4 interface spec. This routine
19666	* posts a SLI4_PAGE_SIZE memory region to the port to hold up to
19667	* SLI4_PAGE_SIZE modulo 64 rpi context headers.
19668	*
19669	* Returns
19670	* A nonzero rpi defined as rpi_base <= rpi < max_rpi if successful
19671	* LPFC_RPI_ALLOC_ERROR if no rpis are available.
19672	**/
19673	int
19674	lpfc_sli4_alloc_rpi(struct lpfc_hba *phba)
19675	{
19676	unsigned long rpi;
19677	uint16_t max_rpi, rpi_limit;
19678	uint16_t rpi_remaining, lrpi = 0;
19679	struct lpfc_rpi_hdr *rpi_hdr;
19680	unsigned long iflag;
19681
19682	/*
19683	* Fetch the next logical rpi. Because this index is logical,
19684	* the driver starts at 0 each time.
19685	*/
19686	spin_lock_irqsave(&phba->hbalock, iflag);
19687	max_rpi = phba->sli4_hba.max_cfg_param.max_rpi;
19688	rpi_limit = phba->sli4_hba.next_rpi;
19689
19690	rpi = find_first_zero_bit(addr: phba->sli4_hba.rpi_bmask, size: rpi_limit);
19691	if (rpi >= rpi_limit)
19692	rpi = LPFC_RPI_ALLOC_ERROR;
19693	else {
19694	set_bit(nr: rpi, addr: phba->sli4_hba.rpi_bmask);
19695	phba->sli4_hba.max_cfg_param.rpi_used++;
19696	phba->sli4_hba.rpi_count++;
19697	}
19698	lpfc_printf_log(phba, KERN_INFO,
19699	LOG_NODE | LOG_DISCOVERY,
19700	"0001 Allocated rpi:x%x max:x%x lim:x%x\n",
19701	(int) rpi, max_rpi, rpi_limit);
19702
19703	/*
19704	* Don't try to allocate more rpi header regions if the device limit
19705	* has been exhausted.
19706	*/
19707	if ((rpi == LPFC_RPI_ALLOC_ERROR) &&
19708	(phba->sli4_hba.rpi_count >= max_rpi)) {
19709	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
19710	return rpi;
19711	}
19712
19713	/*
19714	* RPI header postings are not required for SLI4 ports capable of
19715	* extents.
19716	*/
19717	if (!phba->sli4_hba.rpi_hdrs_in_use) {
19718	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
19719	return rpi;
19720	}
19721
19722	/*
19723	* If the driver is running low on rpi resources, allocate another
19724	* page now. Note that the next_rpi value is used because
19725	* it represents how many are actually in use whereas max_rpi notes
19726	* how many are supported max by the device.
19727	*/
19728	rpi_remaining = phba->sli4_hba.next_rpi - phba->sli4_hba.rpi_count;
19729	spin_unlock_irqrestore(lock: &phba->hbalock, flags: iflag);
19730	if (rpi_remaining < LPFC_RPI_LOW_WATER_MARK) {
19731	rpi_hdr = lpfc_sli4_create_rpi_hdr(phba);
19732	if (!rpi_hdr) {
19733	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19734	"2002 Error Could not grow rpi "
19735	"count\n");
19736	} else {
19737	lrpi = rpi_hdr->start_rpi;
19738	rpi_hdr->start_rpi = phba->sli4_hba.rpi_ids[lrpi];
19739	lpfc_sli4_post_rpi_hdr(phba, rpi_page: rpi_hdr);
19740	}
19741	}
19742
19743	return rpi;
19744	}
19745
19746	/**
19747	* __lpfc_sli4_free_rpi - Release an rpi for reuse.
19748	* @phba: pointer to lpfc hba data structure.
19749	* @rpi: rpi to free
19750	*
19751	* This routine is invoked to release an rpi to the pool of
19752	* available rpis maintained by the driver.
19753	**/
19754	static void
19755	__lpfc_sli4_free_rpi(struct lpfc_hba *phba, int rpi)
19756	{
19757	/*
19758	* if the rpi value indicates a prior unreg has already
19759	* been done, skip the unreg.
19760	*/
19761	if (rpi == LPFC_RPI_ALLOC_ERROR)
19762	return;
19763
19764	if (test_and_clear_bit(nr: rpi, addr: phba->sli4_hba.rpi_bmask)) {
19765	phba->sli4_hba.rpi_count--;
19766	phba->sli4_hba.max_cfg_param.rpi_used--;
19767	} else {
19768	lpfc_printf_log(phba, KERN_INFO,
19769	LOG_NODE | LOG_DISCOVERY,
19770	"2016 rpi %x not inuse\n",
19771	rpi);
19772	}
19773	}
19774
19775	/**
19776	* lpfc_sli4_free_rpi - Release an rpi for reuse.
19777	* @phba: pointer to lpfc hba data structure.
19778	* @rpi: rpi to free
19779	*
19780	* This routine is invoked to release an rpi to the pool of
19781	* available rpis maintained by the driver.
19782	**/
19783	void
19784	lpfc_sli4_free_rpi(struct lpfc_hba *phba, int rpi)
19785	{
19786	spin_lock_irq(lock: &phba->hbalock);
19787	__lpfc_sli4_free_rpi(phba, rpi);
19788	spin_unlock_irq(lock: &phba->hbalock);
19789	}
19790
19791	/**
19792	* lpfc_sli4_remove_rpis - Remove the rpi bitmask region
19793	* @phba: pointer to lpfc hba data structure.
19794	*
19795	* This routine is invoked to remove the memory region that
19796	* provided rpi via a bitmask.
19797	**/
19798	void
19799	lpfc_sli4_remove_rpis(struct lpfc_hba *phba)
19800	{
19801	kfree(objp: phba->sli4_hba.rpi_bmask);
19802	kfree(objp: phba->sli4_hba.rpi_ids);
19803	bf_set(lpfc_rpi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
19804	}
19805
19806	/**
19807	* lpfc_sli4_resume_rpi - Remove the rpi bitmask region
19808	* @ndlp: pointer to lpfc nodelist data structure.
19809	* @cmpl: completion call-back.
19810	* @arg: data to load as MBox 'caller buffer information'
19811	*
19812	* This routine is invoked to remove the memory region that
19813	* provided rpi via a bitmask.
19814	**/
19815	int
19816	lpfc_sli4_resume_rpi(struct lpfc_nodelist *ndlp,
19817	void (*cmpl)(struct lpfc_hba *, LPFC_MBOXQ_t *), void *arg)
19818	{
19819	LPFC_MBOXQ_t *mboxq;
19820	struct lpfc_hba *phba = ndlp->phba;
19821	int rc;
19822
19823	/* The port is notified of the header region via a mailbox command. */
19824	mboxq = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
19825	if (!mboxq)
19826	return -ENOMEM;
19827
19828	/* If cmpl assigned, then this nlp_get pairs with
19829	* lpfc_mbx_cmpl_resume_rpi.
19830	*
19831	* Else cmpl is NULL, then this nlp_get pairs with
19832	* lpfc_sli_def_mbox_cmpl.
19833	*/
19834	if (!lpfc_nlp_get(ndlp)) {
19835	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19836	"2122 %s: Failed to get nlp ref\n",
19837	__func__);
19838	mempool_free(element: mboxq, pool: phba->mbox_mem_pool);
19839	return -EIO;
19840	}
19841
19842	/* Post all rpi memory regions to the port. */
19843	lpfc_resume_rpi(mboxq, ndlp);
19844	if (cmpl) {
19845	mboxq->mbox_cmpl = cmpl;
19846	mboxq->ctx_buf = arg;
19847	} else
19848	mboxq->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
19849	mboxq->ctx_ndlp = ndlp;
19850	mboxq->vport = ndlp->vport;
19851	rc = lpfc_sli_issue_mbox(phba, pmbox: mboxq, MBX_NOWAIT);
19852	if (rc == MBX_NOT_FINISHED) {
19853	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19854	"2010 Resume RPI Mailbox failed "
19855	"status %d, mbxStatus x%x\n", rc,
19856	bf_get(lpfc_mqe_status, &mboxq->u.mqe));
19857	lpfc_nlp_put(ndlp);
19858	mempool_free(element: mboxq, pool: phba->mbox_mem_pool);
19859	return -EIO;
19860	}
19861	return 0;
19862	}
19863
19864	/**
19865	* lpfc_sli4_init_vpi - Initialize a vpi with the port
19866	* @vport: Pointer to the vport for which the vpi is being initialized
19867	*
19868	* This routine is invoked to activate a vpi with the port.
19869	*
19870	* Returns:
19871	* 0 success
19872	* -Evalue otherwise
19873	**/
19874	int
19875	lpfc_sli4_init_vpi(struct lpfc_vport *vport)
19876	{
19877	LPFC_MBOXQ_t *mboxq;
19878	int rc = 0;
19879	int retval = MBX_SUCCESS;
19880	uint32_t mbox_tmo;
19881	struct lpfc_hba *phba = vport->phba;
19882	mboxq = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
19883	if (!mboxq)
19884	return -ENOMEM;
19885	lpfc_init_vpi(phba, mboxq, vport->vpi);
19886	mbox_tmo = lpfc_mbox_tmo_val(phba, mboxq);
19887	rc = lpfc_sli_issue_mbox_wait(phba, pmboxq: mboxq, timeout: mbox_tmo);
19888	if (rc != MBX_SUCCESS) {
19889	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
19890	"2022 INIT VPI Mailbox failed "
19891	"status %d, mbxStatus x%x\n", rc,
19892	bf_get(lpfc_mqe_status, &mboxq->u.mqe));
19893	retval = -EIO;
19894	}
19895	if (rc != MBX_TIMEOUT)
19896	mempool_free(element: mboxq, pool: vport->phba->mbox_mem_pool);
19897
19898	return retval;
19899	}
19900
19901	/**
19902	* lpfc_mbx_cmpl_add_fcf_record - add fcf mbox completion handler.
19903	* @phba: pointer to lpfc hba data structure.
19904	* @mboxq: Pointer to mailbox object.
19905	*
19906	* This routine is invoked to manually add a single FCF record. The caller
19907	* must pass a completely initialized FCF_Record. This routine takes
19908	* care of the nonembedded mailbox operations.
19909	**/
19910	static void
19911	lpfc_mbx_cmpl_add_fcf_record(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
19912	{
19913	void *virt_addr;
19914	union lpfc_sli4_cfg_shdr *shdr;
19915	uint32_t shdr_status, shdr_add_status;
19916
19917	virt_addr = mboxq->sge_array->addr[0];
19918	/* The IOCTL status is embedded in the mailbox subheader. */
19919	shdr = (union lpfc_sli4_cfg_shdr *) virt_addr;
19920	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
19921	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
19922
19923	if ((shdr_status || shdr_add_status) &&
19924	(shdr_status != STATUS_FCF_IN_USE))
19925	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19926	"2558 ADD_FCF_RECORD mailbox failed with "
19927	"status x%x add_status x%x\n",
19928	shdr_status, shdr_add_status);
19929
19930	lpfc_sli4_mbox_cmd_free(phba, mboxq);
19931	}
19932
19933	/**
19934	* lpfc_sli4_add_fcf_record - Manually add an FCF Record.
19935	* @phba: pointer to lpfc hba data structure.
19936	* @fcf_record: pointer to the initialized fcf record to add.
19937	*
19938	* This routine is invoked to manually add a single FCF record. The caller
19939	* must pass a completely initialized FCF_Record. This routine takes
19940	* care of the nonembedded mailbox operations.
19941	**/
19942	int
19943	lpfc_sli4_add_fcf_record(struct lpfc_hba *phba, struct fcf_record *fcf_record)
19944	{
19945	int rc = 0;
19946	LPFC_MBOXQ_t *mboxq;
19947	uint8_t *bytep;
19948	void *virt_addr;
19949	struct lpfc_mbx_sge sge;
19950	uint32_t alloc_len, req_len;
19951	uint32_t fcfindex;
19952
19953	mboxq = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
19954	if (!mboxq) {
19955	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19956	"2009 Failed to allocate mbox for ADD_FCF cmd\n");
19957	return -ENOMEM;
19958	}
19959
19960	req_len = sizeof(struct fcf_record) + sizeof(union lpfc_sli4_cfg_shdr) +
19961	sizeof(uint32_t);
19962
19963	/* Allocate DMA memory and set up the non-embedded mailbox command */
19964	alloc_len = lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_FCOE,
19965	LPFC_MBOX_OPCODE_FCOE_ADD_FCF,
19966	req_len, LPFC_SLI4_MBX_NEMBED);
19967	if (alloc_len < req_len) {
19968	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19969	"2523 Allocated DMA memory size (x%x) is "
19970	"less than the requested DMA memory "
19971	"size (x%x)\n", alloc_len, req_len);
19972	lpfc_sli4_mbox_cmd_free(phba, mboxq);
19973	return -ENOMEM;
19974	}
19975
19976	/*
19977	* Get the first SGE entry from the non-embedded DMA memory. This
19978	* routine only uses a single SGE.
19979	*/
19980	lpfc_sli4_mbx_sge_get(mboxq, 0, &sge);
19981	virt_addr = mboxq->sge_array->addr[0];
19982	/*
19983	* Configure the FCF record for FCFI 0. This is the driver's
19984	* hardcoded default and gets used in nonFIP mode.
19985	*/
19986	fcfindex = bf_get(lpfc_fcf_record_fcf_index, fcf_record);
19987	bytep = virt_addr + sizeof(union lpfc_sli4_cfg_shdr);
19988	lpfc_sli_pcimem_bcopy(srcp: &fcfindex, destp: bytep, cnt: sizeof(uint32_t));
19989
19990	/*
19991	* Copy the fcf_index and the FCF Record Data. The data starts after
19992	* the FCoE header plus word10. The data copy needs to be endian
19993	* correct.
19994	*/
19995	bytep += sizeof(uint32_t);
19996	lpfc_sli_pcimem_bcopy(srcp: fcf_record, destp: bytep, cnt: sizeof(struct fcf_record));
19997	mboxq->vport = phba->pport;
19998	mboxq->mbox_cmpl = lpfc_mbx_cmpl_add_fcf_record;
19999	rc = lpfc_sli_issue_mbox(phba, pmbox: mboxq, MBX_NOWAIT);
20000	if (rc == MBX_NOT_FINISHED) {
20001	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20002	"2515 ADD_FCF_RECORD mailbox failed with "
20003	"status 0x%x\n", rc);
20004	lpfc_sli4_mbox_cmd_free(phba, mboxq);
20005	rc = -EIO;
20006	} else
20007	rc = 0;
20008
20009	return rc;
20010	}
20011
20012	/**
20013	* lpfc_sli4_build_dflt_fcf_record - Build the driver's default FCF Record.
20014	* @phba: pointer to lpfc hba data structure.
20015	* @fcf_record: pointer to the fcf record to write the default data.
20016	* @fcf_index: FCF table entry index.
20017	*
20018	* This routine is invoked to build the driver's default FCF record. The
20019	* values used are hardcoded. This routine handles memory initialization.
20020	*
20021	**/
20022	void
20023	lpfc_sli4_build_dflt_fcf_record(struct lpfc_hba *phba,
20024	struct fcf_record *fcf_record,
20025	uint16_t fcf_index)
20026	{
20027	memset(fcf_record, 0, sizeof(struct fcf_record));
20028	fcf_record->max_rcv_size = LPFC_FCOE_MAX_RCV_SIZE;
20029	fcf_record->fka_adv_period = LPFC_FCOE_FKA_ADV_PER;
20030	fcf_record->fip_priority = LPFC_FCOE_FIP_PRIORITY;
20031	bf_set(lpfc_fcf_record_mac_0, fcf_record, phba->fc_map[0]);
20032	bf_set(lpfc_fcf_record_mac_1, fcf_record, phba->fc_map[1]);
20033	bf_set(lpfc_fcf_record_mac_2, fcf_record, phba->fc_map[2]);
20034	bf_set(lpfc_fcf_record_mac_3, fcf_record, LPFC_FCOE_FCF_MAC3);
20035	bf_set(lpfc_fcf_record_mac_4, fcf_record, LPFC_FCOE_FCF_MAC4);
20036	bf_set(lpfc_fcf_record_mac_5, fcf_record, LPFC_FCOE_FCF_MAC5);
20037	bf_set(lpfc_fcf_record_fc_map_0, fcf_record, phba->fc_map[0]);
20038	bf_set(lpfc_fcf_record_fc_map_1, fcf_record, phba->fc_map[1]);
20039	bf_set(lpfc_fcf_record_fc_map_2, fcf_record, phba->fc_map[2]);
20040	bf_set(lpfc_fcf_record_fcf_valid, fcf_record, 1);
20041	bf_set(lpfc_fcf_record_fcf_avail, fcf_record, 1);
20042	bf_set(lpfc_fcf_record_fcf_index, fcf_record, fcf_index);
20043	bf_set(lpfc_fcf_record_mac_addr_prov, fcf_record,
20044	LPFC_FCF_FPMA | LPFC_FCF_SPMA);
20045	/* Set the VLAN bit map */
20046	if (phba->valid_vlan) {
20047	fcf_record->vlan_bitmap[phba->vlan_id / 8]
20048	= 1 << (phba->vlan_id % 8);
20049	}
20050	}
20051
20052	/**
20053	* lpfc_sli4_fcf_scan_read_fcf_rec - Read hba fcf record for fcf scan.
20054	* @phba: pointer to lpfc hba data structure.
20055	* @fcf_index: FCF table entry offset.
20056	*
20057	* This routine is invoked to scan the entire FCF table by reading FCF
20058	* record and processing it one at a time starting from the @fcf_index
20059	* for initial FCF discovery or fast FCF failover rediscovery.
20060	*
20061	* Return 0 if the mailbox command is submitted successfully, none 0
20062	* otherwise.
20063	**/
20064	int
20065	lpfc_sli4_fcf_scan_read_fcf_rec(struct lpfc_hba *phba, uint16_t fcf_index)
20066	{
20067	int rc = 0, error;
20068	LPFC_MBOXQ_t *mboxq;
20069
20070	phba->fcoe_eventtag_at_fcf_scan = phba->fcoe_eventtag;
20071	phba->fcoe_cvl_eventtag_attn = phba->fcoe_cvl_eventtag;
20072	mboxq = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
20073	if (!mboxq) {
20074	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20075	"2000 Failed to allocate mbox for "
20076	"READ_FCF cmd\n");
20077	error = -ENOMEM;
20078	goto fail_fcf_scan;
20079	}
20080	/* Construct the read FCF record mailbox command */
20081	rc = lpfc_sli4_mbx_read_fcf_rec(phba, mboxq, fcf_index);
20082	if (rc) {
20083	error = -EINVAL;
20084	goto fail_fcf_scan;
20085	}
20086	/* Issue the mailbox command asynchronously */
20087	mboxq->vport = phba->pport;
20088	mboxq->mbox_cmpl = lpfc_mbx_cmpl_fcf_scan_read_fcf_rec;
20089
20090	spin_lock_irq(lock: &phba->hbalock);
20091	phba->hba_flag |= FCF_TS_INPROG;
20092	spin_unlock_irq(lock: &phba->hbalock);
20093
20094	rc = lpfc_sli_issue_mbox(phba, pmbox: mboxq, MBX_NOWAIT);
20095	if (rc == MBX_NOT_FINISHED)
20096	error = -EIO;
20097	else {
20098	/* Reset eligible FCF count for new scan */
20099	if (fcf_index == LPFC_FCOE_FCF_GET_FIRST)
20100	phba->fcf.eligible_fcf_cnt = 0;
20101	error = 0;
20102	}
20103	fail_fcf_scan:
20104	if (error) {
20105	if (mboxq)
20106	lpfc_sli4_mbox_cmd_free(phba, mboxq);
20107	/* FCF scan failed, clear FCF_TS_INPROG flag */
20108	spin_lock_irq(lock: &phba->hbalock);
20109	phba->hba_flag &= ~FCF_TS_INPROG;
20110	spin_unlock_irq(lock: &phba->hbalock);
20111	}
20112	return error;
20113	}
20114
20115	/**
20116	* lpfc_sli4_fcf_rr_read_fcf_rec - Read hba fcf record for roundrobin fcf.
20117	* @phba: pointer to lpfc hba data structure.
20118	* @fcf_index: FCF table entry offset.
20119	*
20120	* This routine is invoked to read an FCF record indicated by @fcf_index
20121	* and to use it for FLOGI roundrobin FCF failover.
20122	*
20123	* Return 0 if the mailbox command is submitted successfully, none 0
20124	* otherwise.
20125	**/
20126	int
20127	lpfc_sli4_fcf_rr_read_fcf_rec(struct lpfc_hba *phba, uint16_t fcf_index)
20128	{
20129	int rc = 0, error;
20130	LPFC_MBOXQ_t *mboxq;
20131
20132	mboxq = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
20133	if (!mboxq) {
20134	lpfc_printf_log(phba, KERN_ERR, LOG_FIP | LOG_INIT,
20135	"2763 Failed to allocate mbox for "
20136	"READ_FCF cmd\n");
20137	error = -ENOMEM;
20138	goto fail_fcf_read;
20139	}
20140	/* Construct the read FCF record mailbox command */
20141	rc = lpfc_sli4_mbx_read_fcf_rec(phba, mboxq, fcf_index);
20142	if (rc) {
20143	error = -EINVAL;
20144	goto fail_fcf_read;
20145	}
20146	/* Issue the mailbox command asynchronously */
20147	mboxq->vport = phba->pport;
20148	mboxq->mbox_cmpl = lpfc_mbx_cmpl_fcf_rr_read_fcf_rec;
20149	rc = lpfc_sli_issue_mbox(phba, pmbox: mboxq, MBX_NOWAIT);
20150	if (rc == MBX_NOT_FINISHED)
20151	error = -EIO;
20152	else
20153	error = 0;
20154
20155	fail_fcf_read:
20156	if (error && mboxq)
20157	lpfc_sli4_mbox_cmd_free(phba, mboxq);
20158	return error;
20159	}
20160
20161	/**
20162	* lpfc_sli4_read_fcf_rec - Read hba fcf record for update eligible fcf bmask.
20163	* @phba: pointer to lpfc hba data structure.
20164	* @fcf_index: FCF table entry offset.
20165	*
20166	* This routine is invoked to read an FCF record indicated by @fcf_index to
20167	* determine whether it's eligible for FLOGI roundrobin failover list.
20168	*
20169	* Return 0 if the mailbox command is submitted successfully, none 0
20170	* otherwise.
20171	**/
20172	int
20173	lpfc_sli4_read_fcf_rec(struct lpfc_hba *phba, uint16_t fcf_index)
20174	{
20175	int rc = 0, error;
20176	LPFC_MBOXQ_t *mboxq;
20177
20178	mboxq = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
20179	if (!mboxq) {
20180	lpfc_printf_log(phba, KERN_ERR, LOG_FIP | LOG_INIT,
20181	"2758 Failed to allocate mbox for "
20182	"READ_FCF cmd\n");
20183	error = -ENOMEM;
20184	goto fail_fcf_read;
20185	}
20186	/* Construct the read FCF record mailbox command */
20187	rc = lpfc_sli4_mbx_read_fcf_rec(phba, mboxq, fcf_index);
20188	if (rc) {
20189	error = -EINVAL;
20190	goto fail_fcf_read;
20191	}
20192	/* Issue the mailbox command asynchronously */
20193	mboxq->vport = phba->pport;
20194	mboxq->mbox_cmpl = lpfc_mbx_cmpl_read_fcf_rec;
20195	rc = lpfc_sli_issue_mbox(phba, pmbox: mboxq, MBX_NOWAIT);
20196	if (rc == MBX_NOT_FINISHED)
20197	error = -EIO;
20198	else
20199	error = 0;
20200
20201	fail_fcf_read:
20202	if (error && mboxq)
20203	lpfc_sli4_mbox_cmd_free(phba, mboxq);
20204	return error;
20205	}
20206
20207	/**
20208	* lpfc_check_next_fcf_pri_level
20209	* @phba: pointer to the lpfc_hba struct for this port.
20210	* This routine is called from the lpfc_sli4_fcf_rr_next_index_get
20211	* routine when the rr_bmask is empty. The FCF indecies are put into the
20212	* rr_bmask based on their priority level. Starting from the highest priority
20213	* to the lowest. The most likely FCF candidate will be in the highest
20214	* priority group. When this routine is called it searches the fcf_pri list for
20215	* next lowest priority group and repopulates the rr_bmask with only those
20216	* fcf_indexes.
20217	* returns:
20218	* 1=success 0=failure
20219	**/
20220	static int
20221	lpfc_check_next_fcf_pri_level(struct lpfc_hba *phba)
20222	{
20223	uint16_t next_fcf_pri;
20224	uint16_t last_index;
20225	struct lpfc_fcf_pri *fcf_pri;
20226	int rc;
20227	int ret = 0;
20228
20229	last_index = find_first_bit(addr: phba->fcf.fcf_rr_bmask,
20230	LPFC_SLI4_FCF_TBL_INDX_MAX);
20231	lpfc_printf_log(phba, KERN_INFO, LOG_FIP,
20232	"3060 Last IDX %d\n", last_index);
20233
20234	/* Verify the priority list has 2 or more entries */
20235	spin_lock_irq(lock: &phba->hbalock);
20236	if (list_empty(head: &phba->fcf.fcf_pri_list) ||
20237	list_is_singular(head: &phba->fcf.fcf_pri_list)) {
20238	spin_unlock_irq(lock: &phba->hbalock);
20239	lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
20240	"3061 Last IDX %d\n", last_index);
20241	return 0; /* Empty rr list */
20242	}
20243	spin_unlock_irq(lock: &phba->hbalock);
20244
20245	next_fcf_pri = 0;
20246	/*
20247	* Clear the rr_bmask and set all of the bits that are at this
20248	* priority.
20249	*/
20250	memset(phba->fcf.fcf_rr_bmask, 0,
20251	sizeof(*phba->fcf.fcf_rr_bmask));
20252	spin_lock_irq(lock: &phba->hbalock);
20253	list_for_each_entry(fcf_pri, &phba->fcf.fcf_pri_list, list) {
20254	if (fcf_pri->fcf_rec.flag & LPFC_FCF_FLOGI_FAILED)
20255	continue;
20256	/*
20257	* the 1st priority that has not FLOGI failed
20258	* will be the highest.
20259	*/
20260	if (!next_fcf_pri)
20261	next_fcf_pri = fcf_pri->fcf_rec.priority;
20262	spin_unlock_irq(lock: &phba->hbalock);
20263	if (fcf_pri->fcf_rec.priority == next_fcf_pri) {
20264	rc = lpfc_sli4_fcf_rr_index_set(phba,
20265	fcf_pri->fcf_rec.fcf_index);
20266	if (rc)
20267	return 0;
20268	}
20269	spin_lock_irq(lock: &phba->hbalock);
20270	}
20271	/*
20272	* if next_fcf_pri was not set above and the list is not empty then
20273	* we have failed flogis on all of them. So reset flogi failed
20274	* and start at the beginning.
20275	*/
20276	if (!next_fcf_pri && !list_empty(head: &phba->fcf.fcf_pri_list)) {
20277	list_for_each_entry(fcf_pri, &phba->fcf.fcf_pri_list, list) {
20278	fcf_pri->fcf_rec.flag &= ~LPFC_FCF_FLOGI_FAILED;
20279	/*
20280	* the 1st priority that has not FLOGI failed
20281	* will be the highest.
20282	*/
20283	if (!next_fcf_pri)
20284	next_fcf_pri = fcf_pri->fcf_rec.priority;
20285	spin_unlock_irq(lock: &phba->hbalock);
20286	if (fcf_pri->fcf_rec.priority == next_fcf_pri) {
20287	rc = lpfc_sli4_fcf_rr_index_set(phba,
20288	fcf_pri->fcf_rec.fcf_index);
20289	if (rc)
20290	return 0;
20291	}
20292	spin_lock_irq(lock: &phba->hbalock);
20293	}
20294	} else
20295	ret = 1;
20296	spin_unlock_irq(lock: &phba->hbalock);
20297
20298	return ret;
20299	}
20300	/**
20301	* lpfc_sli4_fcf_rr_next_index_get - Get next eligible fcf record index
20302	* @phba: pointer to lpfc hba data structure.
20303	*
20304	* This routine is to get the next eligible FCF record index in a round
20305	* robin fashion. If the next eligible FCF record index equals to the
20306	* initial roundrobin FCF record index, LPFC_FCOE_FCF_NEXT_NONE (0xFFFF)
20307	* shall be returned, otherwise, the next eligible FCF record's index
20308	* shall be returned.
20309	**/
20310	uint16_t
20311	lpfc_sli4_fcf_rr_next_index_get(struct lpfc_hba *phba)
20312	{
20313	uint16_t next_fcf_index;
20314
20315	initial_priority:
20316	/* Search start from next bit of currently registered FCF index */
20317	next_fcf_index = phba->fcf.current_rec.fcf_indx;
20318
20319	next_priority:
20320	/* Determine the next fcf index to check */
20321	next_fcf_index = (next_fcf_index + 1) % LPFC_SLI4_FCF_TBL_INDX_MAX;
20322	next_fcf_index = find_next_bit(addr: phba->fcf.fcf_rr_bmask,
20323	LPFC_SLI4_FCF_TBL_INDX_MAX,
20324	offset: next_fcf_index);
20325
20326	/* Wrap around condition on phba->fcf.fcf_rr_bmask */
20327	if (next_fcf_index >= LPFC_SLI4_FCF_TBL_INDX_MAX) {
20328	/*
20329	* If we have wrapped then we need to clear the bits that
20330	* have been tested so that we can detect when we should
20331	* change the priority level.
20332	*/
20333	next_fcf_index = find_first_bit(addr: phba->fcf.fcf_rr_bmask,
20334	LPFC_SLI4_FCF_TBL_INDX_MAX);
20335	}
20336
20337
20338	/* Check roundrobin failover list empty condition */
20339	if (next_fcf_index >= LPFC_SLI4_FCF_TBL_INDX_MAX ||
20340	next_fcf_index == phba->fcf.current_rec.fcf_indx) {
20341	/*
20342	* If next fcf index is not found check if there are lower
20343	* Priority level fcf's in the fcf_priority list.
20344	* Set up the rr_bmask with all of the avaiable fcf bits
20345	* at that level and continue the selection process.
20346	*/
20347	if (lpfc_check_next_fcf_pri_level(phba))
20348	goto initial_priority;
20349	lpfc_printf_log(phba, KERN_WARNING, LOG_FIP,
20350	"2844 No roundrobin failover FCF available\n");
20351
20352	return LPFC_FCOE_FCF_NEXT_NONE;
20353	}
20354
20355	if (next_fcf_index < LPFC_SLI4_FCF_TBL_INDX_MAX &&
20356	phba->fcf.fcf_pri[next_fcf_index].fcf_rec.flag &
20357	LPFC_FCF_FLOGI_FAILED) {
20358	if (list_is_singular(head: &phba->fcf.fcf_pri_list))
20359	return LPFC_FCOE_FCF_NEXT_NONE;
20360
20361	goto next_priority;
20362	}
20363
20364	lpfc_printf_log(phba, KERN_INFO, LOG_FIP,
20365	"2845 Get next roundrobin failover FCF (x%x)\n",
20366	next_fcf_index);
20367
20368	return next_fcf_index;
20369	}
20370
20371	/**
20372	* lpfc_sli4_fcf_rr_index_set - Set bmask with eligible fcf record index
20373	* @phba: pointer to lpfc hba data structure.
20374	* @fcf_index: index into the FCF table to 'set'
20375	*
20376	* This routine sets the FCF record index in to the eligible bmask for
20377	* roundrobin failover search. It checks to make sure that the index
20378	* does not go beyond the range of the driver allocated bmask dimension
20379	* before setting the bit.
20380	*
20381	* Returns 0 if the index bit successfully set, otherwise, it returns
20382	* -EINVAL.
20383	**/
20384	int
20385	lpfc_sli4_fcf_rr_index_set(struct lpfc_hba *phba, uint16_t fcf_index)
20386	{
20387	if (fcf_index >= LPFC_SLI4_FCF_TBL_INDX_MAX) {
20388	lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
20389	"2610 FCF (x%x) reached driver's book "
20390	"keeping dimension:x%x\n",
20391	fcf_index, LPFC_SLI4_FCF_TBL_INDX_MAX);
20392	return -EINVAL;
20393	}
20394	/* Set the eligible FCF record index bmask */
20395	set_bit(nr: fcf_index, addr: phba->fcf.fcf_rr_bmask);
20396
20397	lpfc_printf_log(phba, KERN_INFO, LOG_FIP,
20398	"2790 Set FCF (x%x) to roundrobin FCF failover "
20399	"bmask\n", fcf_index);
20400
20401	return 0;
20402	}
20403
20404	/**
20405	* lpfc_sli4_fcf_rr_index_clear - Clear bmask from eligible fcf record index
20406	* @phba: pointer to lpfc hba data structure.
20407	* @fcf_index: index into the FCF table to 'clear'
20408	*
20409	* This routine clears the FCF record index from the eligible bmask for
20410	* roundrobin failover search. It checks to make sure that the index
20411	* does not go beyond the range of the driver allocated bmask dimension
20412	* before clearing the bit.
20413	**/
20414	void
20415	lpfc_sli4_fcf_rr_index_clear(struct lpfc_hba *phba, uint16_t fcf_index)
20416	{
20417	struct lpfc_fcf_pri *fcf_pri, *fcf_pri_next;
20418	if (fcf_index >= LPFC_SLI4_FCF_TBL_INDX_MAX) {
20419	lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
20420	"2762 FCF (x%x) reached driver's book "
20421	"keeping dimension:x%x\n",
20422	fcf_index, LPFC_SLI4_FCF_TBL_INDX_MAX);
20423	return;
20424	}
20425	/* Clear the eligible FCF record index bmask */
20426	spin_lock_irq(lock: &phba->hbalock);
20427	list_for_each_entry_safe(fcf_pri, fcf_pri_next, &phba->fcf.fcf_pri_list,
20428	list) {
20429	if (fcf_pri->fcf_rec.fcf_index == fcf_index) {
20430	list_del_init(entry: &fcf_pri->list);
20431	break;
20432	}
20433	}
20434	spin_unlock_irq(lock: &phba->hbalock);
20435	clear_bit(nr: fcf_index, addr: phba->fcf.fcf_rr_bmask);
20436
20437	lpfc_printf_log(phba, KERN_INFO, LOG_FIP,
20438	"2791 Clear FCF (x%x) from roundrobin failover "
20439	"bmask\n", fcf_index);
20440	}
20441
20442	/**
20443	* lpfc_mbx_cmpl_redisc_fcf_table - completion routine for rediscover FCF table
20444	* @phba: pointer to lpfc hba data structure.
20445	* @mbox: An allocated pointer to type LPFC_MBOXQ_t
20446	*
20447	* This routine is the completion routine for the rediscover FCF table mailbox
20448	* command. If the mailbox command returned failure, it will try to stop the
20449	* FCF rediscover wait timer.
20450	**/
20451	static void
20452	lpfc_mbx_cmpl_redisc_fcf_table(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox)
20453	{
20454	struct lpfc_mbx_redisc_fcf_tbl *redisc_fcf;
20455	uint32_t shdr_status, shdr_add_status;
20456
20457	redisc_fcf = &mbox->u.mqe.un.redisc_fcf_tbl;
20458
20459	shdr_status = bf_get(lpfc_mbox_hdr_status,
20460	&redisc_fcf->header.cfg_shdr.response);
20461	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status,
20462	&redisc_fcf->header.cfg_shdr.response);
20463	if (shdr_status || shdr_add_status) {
20464	lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
20465	"2746 Requesting for FCF rediscovery failed "
20466	"status x%x add_status x%x\n",
20467	shdr_status, shdr_add_status);
20468	if (phba->fcf.fcf_flag & FCF_ACVL_DISC) {
20469	spin_lock_irq(lock: &phba->hbalock);
20470	phba->fcf.fcf_flag &= ~FCF_ACVL_DISC;
20471	spin_unlock_irq(lock: &phba->hbalock);
20472	/*
20473	* CVL event triggered FCF rediscover request failed,
20474	* last resort to re-try current registered FCF entry.
20475	*/
20476	lpfc_retry_pport_discovery(phba);
20477	} else {
20478	spin_lock_irq(lock: &phba->hbalock);
20479	phba->fcf.fcf_flag &= ~FCF_DEAD_DISC;
20480	spin_unlock_irq(lock: &phba->hbalock);
20481	/*
20482	* DEAD FCF event triggered FCF rediscover request
20483	* failed, last resort to fail over as a link down
20484	* to FCF registration.
20485	*/
20486	lpfc_sli4_fcf_dead_failthrough(phba);
20487	}
20488	} else {
20489	lpfc_printf_log(phba, KERN_INFO, LOG_FIP,
20490	"2775 Start FCF rediscover quiescent timer\n");
20491	/*
20492	* Start FCF rediscovery wait timer for pending FCF
20493	* before rescan FCF record table.
20494	*/
20495	lpfc_fcf_redisc_wait_start_timer(phba);
20496	}
20497
20498	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
20499	}
20500
20501	/**
20502	* lpfc_sli4_redisc_fcf_table - Request to rediscover entire FCF table by port.
20503	* @phba: pointer to lpfc hba data structure.
20504	*
20505	* This routine is invoked to request for rediscovery of the entire FCF table
20506	* by the port.
20507	**/
20508	int
20509	lpfc_sli4_redisc_fcf_table(struct lpfc_hba *phba)
20510	{
20511	LPFC_MBOXQ_t *mbox;
20512	struct lpfc_mbx_redisc_fcf_tbl *redisc_fcf;
20513	int rc, length;
20514
20515	/* Cancel retry delay timers to all vports before FCF rediscover */
20516	lpfc_cancel_all_vport_retry_delay_timer(phba);
20517
20518	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
20519	if (!mbox) {
20520	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20521	"2745 Failed to allocate mbox for "
20522	"requesting FCF rediscover.\n");
20523	return -ENOMEM;
20524	}
20525
20526	length = (sizeof(struct lpfc_mbx_redisc_fcf_tbl) -
20527	sizeof(struct lpfc_sli4_cfg_mhdr));
20528	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
20529	LPFC_MBOX_OPCODE_FCOE_REDISCOVER_FCF,
20530	length, LPFC_SLI4_MBX_EMBED);
20531
20532	redisc_fcf = &mbox->u.mqe.un.redisc_fcf_tbl;
20533	/* Set count to 0 for invalidating the entire FCF database */
20534	bf_set(lpfc_mbx_redisc_fcf_count, redisc_fcf, 0);
20535
20536	/* Issue the mailbox command asynchronously */
20537	mbox->vport = phba->pport;
20538	mbox->mbox_cmpl = lpfc_mbx_cmpl_redisc_fcf_table;
20539	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_NOWAIT);
20540
20541	if (rc == MBX_NOT_FINISHED) {
20542	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
20543	return -EIO;
20544	}
20545	return 0;
20546	}
20547
20548	/**
20549	* lpfc_sli4_fcf_dead_failthrough - Failthrough routine to fcf dead event
20550	* @phba: pointer to lpfc hba data structure.
20551	*
20552	* This function is the failover routine as a last resort to the FCF DEAD
20553	* event when driver failed to perform fast FCF failover.
20554	**/
20555	void
20556	lpfc_sli4_fcf_dead_failthrough(struct lpfc_hba *phba)
20557	{
20558	uint32_t link_state;
20559
20560	/*
20561	* Last resort as FCF DEAD event failover will treat this as
20562	* a link down, but save the link state because we don't want
20563	* it to be changed to Link Down unless it is already down.
20564	*/
20565	link_state = phba->link_state;
20566	lpfc_linkdown(phba);
20567	phba->link_state = link_state;
20568
20569	/* Unregister FCF if no devices connected to it */
20570	lpfc_unregister_unused_fcf(phba);
20571	}
20572
20573	/**
20574	* lpfc_sli_get_config_region23 - Get sli3 port region 23 data.
20575	* @phba: pointer to lpfc hba data structure.
20576	* @rgn23_data: pointer to configure region 23 data.
20577	*
20578	* This function gets SLI3 port configure region 23 data through memory dump
20579	* mailbox command. When it successfully retrieves data, the size of the data
20580	* will be returned, otherwise, 0 will be returned.
20581	**/
20582	static uint32_t
20583	lpfc_sli_get_config_region23(struct lpfc_hba *phba, char *rgn23_data)
20584	{
20585	LPFC_MBOXQ_t *pmb = NULL;
20586	MAILBOX_t *mb;
20587	uint32_t offset = 0;
20588	int rc;
20589
20590	if (!rgn23_data)
20591	return 0;
20592
20593	pmb = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
20594	if (!pmb) {
20595	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20596	"2600 failed to allocate mailbox memory\n");
20597	return 0;
20598	}
20599	mb = &pmb->u.mb;
20600
20601	do {
20602	lpfc_dump_mem(phba, pmb, offset, DMP_REGION_23);
20603	rc = lpfc_sli_issue_mbox(phba, pmbox: pmb, MBX_POLL);
20604
20605	if (rc != MBX_SUCCESS) {
20606	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
20607	"2601 failed to read config "
20608	"region 23, rc 0x%x Status 0x%x\n",
20609	rc, mb->mbxStatus);
20610	mb->un.varDmp.word_cnt = 0;
20611	}
20612	/*
20613	* dump mem may return a zero when finished or we got a
20614	* mailbox error, either way we are done.
20615	*/
20616	if (mb->un.varDmp.word_cnt == 0)
20617	break;
20618
20619	if (mb->un.varDmp.word_cnt > DMP_RGN23_SIZE - offset)
20620	mb->un.varDmp.word_cnt = DMP_RGN23_SIZE - offset;
20621
20622	lpfc_sli_pcimem_bcopy(srcp: ((uint8_t *)mb) + DMP_RSP_OFFSET,
20623	destp: rgn23_data + offset,
20624	cnt: mb->un.varDmp.word_cnt);
20625	offset += mb->un.varDmp.word_cnt;
20626	} while (mb->un.varDmp.word_cnt && offset < DMP_RGN23_SIZE);
20627
20628	mempool_free(element: pmb, pool: phba->mbox_mem_pool);
20629	return offset;
20630	}
20631
20632	/**
20633	* lpfc_sli4_get_config_region23 - Get sli4 port region 23 data.
20634	* @phba: pointer to lpfc hba data structure.
20635	* @rgn23_data: pointer to configure region 23 data.
20636	*
20637	* This function gets SLI4 port configure region 23 data through memory dump
20638	* mailbox command. When it successfully retrieves data, the size of the data
20639	* will be returned, otherwise, 0 will be returned.
20640	**/
20641	static uint32_t
20642	lpfc_sli4_get_config_region23(struct lpfc_hba *phba, char *rgn23_data)
20643	{
20644	LPFC_MBOXQ_t *mboxq = NULL;
20645	struct lpfc_dmabuf *mp = NULL;
20646	struct lpfc_mqe *mqe;
20647	uint32_t data_length = 0;
20648	int rc;
20649
20650	if (!rgn23_data)
20651	return 0;
20652
20653	mboxq = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
20654	if (!mboxq) {
20655	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20656	"3105 failed to allocate mailbox memory\n");
20657	return 0;
20658	}
20659
20660	if (lpfc_sli4_dump_cfg_rg23(phba, mboxq))
20661	goto out;
20662	mqe = &mboxq->u.mqe;
20663	mp = (struct lpfc_dmabuf *)mboxq->ctx_buf;
20664	rc = lpfc_sli_issue_mbox(phba, pmbox: mboxq, MBX_POLL);
20665	if (rc)
20666	goto out;
20667	data_length = mqe->un.mb_words[5];
20668	if (data_length == 0)
20669	goto out;
20670	if (data_length > DMP_RGN23_SIZE) {
20671	data_length = 0;
20672	goto out;
20673	}
20674	lpfc_sli_pcimem_bcopy(srcp: (char *)mp->virt, destp: rgn23_data, cnt: data_length);
20675	out:
20676	lpfc_mbox_rsrc_cleanup(phba, mbox: mboxq, locked: MBOX_THD_UNLOCKED);
20677	return data_length;
20678	}
20679
20680	/**
20681	* lpfc_sli_read_link_ste - Read region 23 to decide if link is disabled.
20682	* @phba: pointer to lpfc hba data structure.
20683	*
20684	* This function read region 23 and parse TLV for port status to
20685	* decide if the user disaled the port. If the TLV indicates the
20686	* port is disabled, the hba_flag is set accordingly.
20687	**/
20688	void
20689	lpfc_sli_read_link_ste(struct lpfc_hba *phba)
20690	{
20691	uint8_t *rgn23_data = NULL;
20692	uint32_t if_type, data_size, sub_tlv_len, tlv_offset;
20693	uint32_t offset = 0;
20694
20695	/* Get adapter Region 23 data */
20696	rgn23_data = kzalloc(DMP_RGN23_SIZE, GFP_KERNEL);
20697	if (!rgn23_data)
20698	goto out;
20699
20700	if (phba->sli_rev < LPFC_SLI_REV4)
20701	data_size = lpfc_sli_get_config_region23(phba, rgn23_data);
20702	else {
20703	if_type = bf_get(lpfc_sli_intf_if_type,
20704	&phba->sli4_hba.sli_intf);
20705	if (if_type == LPFC_SLI_INTF_IF_TYPE_0)
20706	goto out;
20707	data_size = lpfc_sli4_get_config_region23(phba, rgn23_data);
20708	}
20709
20710	if (!data_size)
20711	goto out;
20712
20713	/* Check the region signature first */
20714	if (memcmp(p: &rgn23_data[offset], LPFC_REGION23_SIGNATURE, size: 4)) {
20715	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20716	"2619 Config region 23 has bad signature\n");
20717	goto out;
20718	}
20719	offset += 4;
20720
20721	/* Check the data structure version */
20722	if (rgn23_data[offset] != LPFC_REGION23_VERSION) {
20723	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20724	"2620 Config region 23 has bad version\n");
20725	goto out;
20726	}
20727	offset += 4;
20728
20729	/* Parse TLV entries in the region */
20730	while (offset < data_size) {
20731	if (rgn23_data[offset] == LPFC_REGION23_LAST_REC)
20732	break;
20733	/*
20734	* If the TLV is not driver specific TLV or driver id is
20735	* not linux driver id, skip the record.
20736	*/
20737	if ((rgn23_data[offset] != DRIVER_SPECIFIC_TYPE) ||
20738	(rgn23_data[offset + 2] != LINUX_DRIVER_ID) ||
20739	(rgn23_data[offset + 3] != 0)) {
20740	offset += rgn23_data[offset + 1] * 4 + 4;
20741	continue;
20742	}
20743
20744	/* Driver found a driver specific TLV in the config region */
20745	sub_tlv_len = rgn23_data[offset + 1] * 4;
20746	offset += 4;
20747	tlv_offset = 0;
20748
20749	/*
20750	* Search for configured port state sub-TLV.
20751	*/
20752	while ((offset < data_size) &&
20753	(tlv_offset < sub_tlv_len)) {
20754	if (rgn23_data[offset] == LPFC_REGION23_LAST_REC) {
20755	offset += 4;
20756	tlv_offset += 4;
20757	break;
20758	}
20759	if (rgn23_data[offset] != PORT_STE_TYPE) {
20760	offset += rgn23_data[offset + 1] * 4 + 4;
20761	tlv_offset += rgn23_data[offset + 1] * 4 + 4;
20762	continue;
20763	}
20764
20765	/* This HBA contains PORT_STE configured */
20766	if (!rgn23_data[offset + 2])
20767	phba->hba_flag |= LINK_DISABLED;
20768
20769	goto out;
20770	}
20771	}
20772
20773	out:
20774	kfree(objp: rgn23_data);
20775	return;
20776	}
20777
20778	/**
20779	* lpfc_log_fw_write_cmpl - logs firmware write completion status
20780	* @phba: pointer to lpfc hba data structure
20781	* @shdr_status: wr_object rsp's status field
20782	* @shdr_add_status: wr_object rsp's add_status field
20783	* @shdr_add_status_2: wr_object rsp's add_status_2 field
20784	* @shdr_change_status: wr_object rsp's change_status field
20785	* @shdr_csf: wr_object rsp's csf bit
20786	*
20787	* This routine is intended to be called after a firmware write completes.
20788	* It will log next action items to be performed by the user to instantiate
20789	* the newly downloaded firmware or reason for incompatibility.
20790	**/
20791	static void
20792	lpfc_log_fw_write_cmpl(struct lpfc_hba *phba, u32 shdr_status,
20793	u32 shdr_add_status, u32 shdr_add_status_2,
20794	u32 shdr_change_status, u32 shdr_csf)
20795	{
20796	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
20797	"4198 %s: flash_id x%02x, asic_rev x%02x, "
20798	"status x%02x, add_status x%02x, add_status_2 x%02x, "
20799	"change_status x%02x, csf %01x\n", __func__,
20800	phba->sli4_hba.flash_id, phba->sli4_hba.asic_rev,
20801	shdr_status, shdr_add_status, shdr_add_status_2,
20802	shdr_change_status, shdr_csf);
20803
20804	if (shdr_add_status == LPFC_ADD_STATUS_INCOMPAT_OBJ) {
20805	switch (shdr_add_status_2) {
20806	case LPFC_ADD_STATUS_2_INCOMPAT_FLASH:
20807	lpfc_log_msg(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
20808	"4199 Firmware write failed: "
20809	"image incompatible with flash x%02x\n",
20810	phba->sli4_hba.flash_id);
20811	break;
20812	case LPFC_ADD_STATUS_2_INCORRECT_ASIC:
20813	lpfc_log_msg(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
20814	"4200 Firmware write failed: "
20815	"image incompatible with ASIC "
20816	"architecture x%02x\n",
20817	phba->sli4_hba.asic_rev);
20818	break;
20819	default:
20820	lpfc_log_msg(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
20821	"4210 Firmware write failed: "
20822	"add_status_2 x%02x\n",
20823	shdr_add_status_2);
20824	break;
20825	}
20826	} else if (!shdr_status && !shdr_add_status) {
20827	if (shdr_change_status == LPFC_CHANGE_STATUS_FW_RESET ||
20828	shdr_change_status == LPFC_CHANGE_STATUS_PORT_MIGRATION) {
20829	if (shdr_csf)
20830	shdr_change_status =
20831	LPFC_CHANGE_STATUS_PCI_RESET;
20832	}
20833
20834	switch (shdr_change_status) {
20835	case (LPFC_CHANGE_STATUS_PHYS_DEV_RESET):
20836	lpfc_log_msg(phba, KERN_NOTICE, LOG_MBOX | LOG_SLI,
20837	"3198 Firmware write complete: System "
20838	"reboot required to instantiate\n");
20839	break;
20840	case (LPFC_CHANGE_STATUS_FW_RESET):
20841	lpfc_log_msg(phba, KERN_NOTICE, LOG_MBOX | LOG_SLI,
20842	"3199 Firmware write complete: "
20843	"Firmware reset required to "
20844	"instantiate\n");
20845	break;
20846	case (LPFC_CHANGE_STATUS_PORT_MIGRATION):
20847	lpfc_log_msg(phba, KERN_NOTICE, LOG_MBOX | LOG_SLI,
20848	"3200 Firmware write complete: Port "
20849	"Migration or PCI Reset required to "
20850	"instantiate\n");
20851	break;
20852	case (LPFC_CHANGE_STATUS_PCI_RESET):
20853	lpfc_log_msg(phba, KERN_NOTICE, LOG_MBOX | LOG_SLI,
20854	"3201 Firmware write complete: PCI "
20855	"Reset required to instantiate\n");
20856	break;
20857	default:
20858	break;
20859	}
20860	}
20861	}
20862
20863	/**
20864	* lpfc_wr_object - write an object to the firmware
20865	* @phba: HBA structure that indicates port to create a queue on.
20866	* @dmabuf_list: list of dmabufs to write to the port.
20867	* @size: the total byte value of the objects to write to the port.
20868	* @offset: the current offset to be used to start the transfer.
20869	*
20870	* This routine will create a wr_object mailbox command to send to the port.
20871	* the mailbox command will be constructed using the dma buffers described in
20872	* @dmabuf_list to create a list of BDEs. This routine will fill in as many
20873	* BDEs that the imbedded mailbox can support. The @offset variable will be
20874	* used to indicate the starting offset of the transfer and will also return
20875	* the offset after the write object mailbox has completed. @size is used to
20876	* determine the end of the object and whether the eof bit should be set.
20877	*
20878	* Return 0 is successful and offset will contain the new offset to use
20879	* for the next write.
20880	* Return negative value for error cases.
20881	**/
20882	int
20883	lpfc_wr_object(struct lpfc_hba *phba, struct list_head *dmabuf_list,
20884	uint32_t size, uint32_t *offset)
20885	{
20886	struct lpfc_mbx_wr_object *wr_object;
20887	LPFC_MBOXQ_t *mbox;
20888	int rc = 0, i = 0;
20889	int mbox_status = 0;
20890	uint32_t shdr_status, shdr_add_status, shdr_add_status_2;
20891	uint32_t shdr_change_status = 0, shdr_csf = 0;
20892	uint32_t mbox_tmo;
20893	struct lpfc_dmabuf *dmabuf;
20894	uint32_t written = 0;
20895	bool check_change_status = false;
20896
20897	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
20898	if (!mbox)
20899	return -ENOMEM;
20900
20901	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
20902	LPFC_MBOX_OPCODE_WRITE_OBJECT,
20903	sizeof(struct lpfc_mbx_wr_object) -
20904	sizeof(struct lpfc_sli4_cfg_mhdr), LPFC_SLI4_MBX_EMBED);
20905
20906	wr_object = (struct lpfc_mbx_wr_object *)&mbox->u.mqe.un.wr_object;
20907	wr_object->u.request.write_offset = *offset;
20908	sprintf(buf: (uint8_t *)wr_object->u.request.object_name, fmt: "/");
20909	wr_object->u.request.object_name[0] =
20910	cpu_to_le32(wr_object->u.request.object_name[0]);
20911	bf_set(lpfc_wr_object_eof, &wr_object->u.request, 0);
20912	list_for_each_entry(dmabuf, dmabuf_list, list) {
20913	if (i >= LPFC_MBX_WR_CONFIG_MAX_BDE || written >= size)
20914	break;
20915	wr_object->u.request.bde[i].addrLow = putPaddrLow(dmabuf->phys);
20916	wr_object->u.request.bde[i].addrHigh =
20917	putPaddrHigh(dmabuf->phys);
20918	if (written + SLI4_PAGE_SIZE >= size) {
20919	wr_object->u.request.bde[i].tus.f.bdeSize =
20920	(size - written);
20921	written += (size - written);
20922	bf_set(lpfc_wr_object_eof, &wr_object->u.request, 1);
20923	bf_set(lpfc_wr_object_eas, &wr_object->u.request, 1);
20924	check_change_status = true;
20925	} else {
20926	wr_object->u.request.bde[i].tus.f.bdeSize =
20927	SLI4_PAGE_SIZE;
20928	written += SLI4_PAGE_SIZE;
20929	}
20930	i++;
20931	}
20932	wr_object->u.request.bde_count = i;
20933	bf_set(lpfc_wr_object_write_length, &wr_object->u.request, written);
20934	if (!phba->sli4_hba.intr_enable)
20935	mbox_status = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
20936	else {
20937	mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
20938	mbox_status = lpfc_sli_issue_mbox_wait(phba, pmboxq: mbox, timeout: mbox_tmo);
20939	}
20940
20941	/* The mbox status needs to be maintained to detect MBOX_TIMEOUT. */
20942	rc = mbox_status;
20943
20944	/* The IOCTL status is embedded in the mailbox subheader. */
20945	shdr_status = bf_get(lpfc_mbox_hdr_status,
20946	&wr_object->header.cfg_shdr.response);
20947	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status,
20948	&wr_object->header.cfg_shdr.response);
20949	shdr_add_status_2 = bf_get(lpfc_mbox_hdr_add_status_2,
20950	&wr_object->header.cfg_shdr.response);
20951	if (check_change_status) {
20952	shdr_change_status = bf_get(lpfc_wr_object_change_status,
20953	&wr_object->u.response);
20954	shdr_csf = bf_get(lpfc_wr_object_csf,
20955	&wr_object->u.response);
20956	}
20957
20958	if (shdr_status || shdr_add_status || shdr_add_status_2 || rc) {
20959	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20960	"3025 Write Object mailbox failed with "
20961	"status x%x add_status x%x, add_status_2 x%x, "
20962	"mbx status x%x\n",
20963	shdr_status, shdr_add_status, shdr_add_status_2,
20964	rc);
20965	rc = -ENXIO;
20966	*offset = shdr_add_status;
20967	} else {
20968	*offset += wr_object->u.response.actual_write_length;
20969	}
20970
20971	if (rc || check_change_status)
20972	lpfc_log_fw_write_cmpl(phba, shdr_status, shdr_add_status,
20973	shdr_add_status_2, shdr_change_status,
20974	shdr_csf);
20975
20976	if (!phba->sli4_hba.intr_enable)
20977	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
20978	else if (mbox_status != MBX_TIMEOUT)
20979	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
20980
20981	return rc;
20982	}
20983
20984	/**
20985	* lpfc_cleanup_pending_mbox - Free up vport discovery mailbox commands.
20986	* @vport: pointer to vport data structure.
20987	*
20988	* This function iterate through the mailboxq and clean up all REG_LOGIN
20989	* and REG_VPI mailbox commands associated with the vport. This function
20990	* is called when driver want to restart discovery of the vport due to
20991	* a Clear Virtual Link event.
20992	**/
20993	void
20994	lpfc_cleanup_pending_mbox(struct lpfc_vport *vport)
20995	{
20996	struct lpfc_hba *phba = vport->phba;
20997	LPFC_MBOXQ_t *mb, *nextmb;
20998	struct lpfc_nodelist *ndlp;
20999	struct lpfc_nodelist *act_mbx_ndlp = NULL;
21000	LIST_HEAD(mbox_cmd_list);
21001	uint8_t restart_loop;
21002
21003	/* Clean up internally queued mailbox commands with the vport */
21004	spin_lock_irq(lock: &phba->hbalock);
21005	list_for_each_entry_safe(mb, nextmb, &phba->sli.mboxq, list) {
21006	if (mb->vport != vport)
21007	continue;
21008
21009	if ((mb->u.mb.mbxCommand != MBX_REG_LOGIN64) &&
21010	(mb->u.mb.mbxCommand != MBX_REG_VPI))
21011	continue;
21012
21013	list_move_tail(list: &mb->list, head: &mbox_cmd_list);
21014	}
21015	/* Clean up active mailbox command with the vport */
21016	mb = phba->sli.mbox_active;
21017	if (mb && (mb->vport == vport)) {
21018	if ((mb->u.mb.mbxCommand == MBX_REG_LOGIN64) ||
21019	(mb->u.mb.mbxCommand == MBX_REG_VPI))
21020	mb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
21021	if (mb->u.mb.mbxCommand == MBX_REG_LOGIN64) {
21022	act_mbx_ndlp = (struct lpfc_nodelist *)mb->ctx_ndlp;
21023
21024	/* This reference is local to this routine. The
21025	* reference is removed at routine exit.
21026	*/
21027	act_mbx_ndlp = lpfc_nlp_get(act_mbx_ndlp);
21028
21029	/* Unregister the RPI when mailbox complete */
21030	mb->mbox_flag |= LPFC_MBX_IMED_UNREG;
21031	}
21032	}
21033	/* Cleanup any mailbox completions which are not yet processed */
21034	do {
21035	restart_loop = 0;
21036	list_for_each_entry(mb, &phba->sli.mboxq_cmpl, list) {
21037	/*
21038	* If this mailox is already processed or it is
21039	* for another vport ignore it.
21040	*/
21041	if ((mb->vport != vport) ||
21042	(mb->mbox_flag & LPFC_MBX_IMED_UNREG))
21043	continue;
21044
21045	if ((mb->u.mb.mbxCommand != MBX_REG_LOGIN64) &&
21046	(mb->u.mb.mbxCommand != MBX_REG_VPI))
21047	continue;
21048
21049	mb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
21050	if (mb->u.mb.mbxCommand == MBX_REG_LOGIN64) {
21051	ndlp = (struct lpfc_nodelist *)mb->ctx_ndlp;
21052	/* Unregister the RPI when mailbox complete */
21053	mb->mbox_flag |= LPFC_MBX_IMED_UNREG;
21054	restart_loop = 1;
21055	spin_unlock_irq(lock: &phba->hbalock);
21056	spin_lock(lock: &ndlp->lock);
21057	ndlp->nlp_flag &= ~NLP_IGNR_REG_CMPL;
21058	spin_unlock(lock: &ndlp->lock);
21059	spin_lock_irq(lock: &phba->hbalock);
21060	break;
21061	}
21062	}
21063	} while (restart_loop);
21064
21065	spin_unlock_irq(lock: &phba->hbalock);
21066
21067	/* Release the cleaned-up mailbox commands */
21068	while (!list_empty(head: &mbox_cmd_list)) {
21069	list_remove_head(&mbox_cmd_list, mb, LPFC_MBOXQ_t, list);
21070	if (mb->u.mb.mbxCommand == MBX_REG_LOGIN64) {
21071	ndlp = (struct lpfc_nodelist *)mb->ctx_ndlp;
21072	mb->ctx_ndlp = NULL;
21073	if (ndlp) {
21074	spin_lock(lock: &ndlp->lock);
21075	ndlp->nlp_flag &= ~NLP_IGNR_REG_CMPL;
21076	spin_unlock(lock: &ndlp->lock);
21077	lpfc_nlp_put(ndlp);
21078	}
21079	}
21080	lpfc_mbox_rsrc_cleanup(phba, mbox: mb, locked: MBOX_THD_UNLOCKED);
21081	}
21082
21083	/* Release the ndlp with the cleaned-up active mailbox command */
21084	if (act_mbx_ndlp) {
21085	spin_lock(lock: &act_mbx_ndlp->lock);
21086	act_mbx_ndlp->nlp_flag &= ~NLP_IGNR_REG_CMPL;
21087	spin_unlock(lock: &act_mbx_ndlp->lock);
21088	lpfc_nlp_put(act_mbx_ndlp);
21089	}
21090	}
21091
21092	/**
21093	* lpfc_drain_txq - Drain the txq
21094	* @phba: Pointer to HBA context object.
21095	*
21096	* This function attempt to submit IOCBs on the txq
21097	* to the adapter. For SLI4 adapters, the txq contains
21098	* ELS IOCBs that have been deferred because the there
21099	* are no SGLs. This congestion can occur with large
21100	* vport counts during node discovery.
21101	**/
21102
21103	uint32_t
21104	lpfc_drain_txq(struct lpfc_hba *phba)
21105	{
21106	LIST_HEAD(completions);
21107	struct lpfc_sli_ring *pring;
21108	struct lpfc_iocbq *piocbq = NULL;
21109	unsigned long iflags = 0;
21110	char *fail_msg = NULL;
21111	uint32_t txq_cnt = 0;
21112	struct lpfc_queue *wq;
21113	int ret = 0;
21114
21115	if (phba->link_flag & LS_MDS_LOOPBACK) {
21116	/* MDS WQE are posted only to first WQ*/
21117	wq = phba->sli4_hba.hdwq[0].io_wq;
21118	if (unlikely(!wq))
21119	return 0;
21120	pring = wq->pring;
21121	} else {
21122	wq = phba->sli4_hba.els_wq;
21123	if (unlikely(!wq))
21124	return 0;
21125	pring = lpfc_phba_elsring(phba);
21126	}
21127
21128	if (unlikely(!pring) || list_empty(head: &pring->txq))
21129	return 0;
21130
21131	spin_lock_irqsave(&pring->ring_lock, iflags);
21132	list_for_each_entry(piocbq, &pring->txq, list) {
21133	txq_cnt++;
21134	}
21135
21136	if (txq_cnt > pring->txq_max)
21137	pring->txq_max = txq_cnt;
21138
21139	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
21140
21141	while (!list_empty(head: &pring->txq)) {
21142	spin_lock_irqsave(&pring->ring_lock, iflags);
21143
21144	piocbq = lpfc_sli_ringtx_get(phba, pring);
21145	if (!piocbq) {
21146	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
21147	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
21148	"2823 txq empty and txq_cnt is %d\n ",
21149	txq_cnt);
21150	break;
21151	}
21152	txq_cnt--;
21153
21154	ret = __lpfc_sli_issue_iocb(phba, ring_number: pring->ringno, piocb: piocbq, flag: 0);
21155
21156	if (ret && ret != IOCB_BUSY) {
21157	fail_msg = " - Cannot send IO ";
21158	piocbq->cmd_flag &= ~LPFC_DRIVER_ABORTED;
21159	}
21160	if (fail_msg) {
21161	piocbq->cmd_flag |= LPFC_DRIVER_ABORTED;
21162	/* Failed means we can't issue and need to cancel */
21163	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
21164	"2822 IOCB failed %s iotag 0x%x "
21165	"xri 0x%x %d flg x%x\n",
21166	fail_msg, piocbq->iotag,
21167	piocbq->sli4_xritag, ret,
21168	piocbq->cmd_flag);
21169	list_add_tail(new: &piocbq->list, head: &completions);
21170	fail_msg = NULL;
21171	}
21172	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
21173	if (txq_cnt == 0 || ret == IOCB_BUSY)
21174	break;
21175	}
21176	/* Cancel all the IOCBs that cannot be issued */
21177	lpfc_sli_cancel_iocbs(phba, iocblist: &completions, IOSTAT_LOCAL_REJECT,
21178	IOERR_SLI_ABORTED);
21179
21180	return txq_cnt;
21181	}
21182
21183	/**
21184	* lpfc_wqe_bpl2sgl - Convert the bpl/bde to a sgl.
21185	* @phba: Pointer to HBA context object.
21186	* @pwqeq: Pointer to command WQE.
21187	* @sglq: Pointer to the scatter gather queue object.
21188	*
21189	* This routine converts the bpl or bde that is in the WQE
21190	* to a sgl list for the sli4 hardware. The physical address
21191	* of the bpl/bde is converted back to a virtual address.
21192	* If the WQE contains a BPL then the list of BDE's is
21193	* converted to sli4_sge's. If the WQE contains a single
21194	* BDE then it is converted to a single sli_sge.
21195	* The WQE is still in cpu endianness so the contents of
21196	* the bpl can be used without byte swapping.
21197	*
21198	* Returns valid XRI = Success, NO_XRI = Failure.
21199	*/
21200	static uint16_t
21201	lpfc_wqe_bpl2sgl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeq,
21202	struct lpfc_sglq *sglq)
21203	{
21204	uint16_t xritag = NO_XRI;
21205	struct ulp_bde64 *bpl = NULL;
21206	struct ulp_bde64 bde;
21207	struct sli4_sge *sgl = NULL;
21208	struct lpfc_dmabuf *dmabuf;
21209	union lpfc_wqe128 *wqe;
21210	int numBdes = 0;
21211	int i = 0;
21212	uint32_t offset = 0; /* accumulated offset in the sg request list */
21213	int inbound = 0; /* number of sg reply entries inbound from firmware */
21214	uint32_t cmd;
21215
21216	if (!pwqeq || !sglq)
21217	return xritag;
21218
21219	sgl = (struct sli4_sge *)sglq->sgl;
21220	wqe = &pwqeq->wqe;
21221	pwqeq->iocb.ulpIoTag = pwqeq->iotag;
21222
21223	cmd = bf_get(wqe_cmnd, &wqe->generic.wqe_com);
21224	if (cmd == CMD_XMIT_BLS_RSP64_WQE)
21225	return sglq->sli4_xritag;
21226	numBdes = pwqeq->num_bdes;
21227	if (numBdes) {
21228	/* The addrHigh and addrLow fields within the WQE
21229	* have not been byteswapped yet so there is no
21230	* need to swap them back.
21231	*/
21232	if (pwqeq->bpl_dmabuf)
21233	dmabuf = pwqeq->bpl_dmabuf;
21234	else
21235	return xritag;
21236
21237	bpl = (struct ulp_bde64 *)dmabuf->virt;
21238	if (!bpl)
21239	return xritag;
21240
21241	for (i = 0; i < numBdes; i++) {
21242	/* Should already be byte swapped. */
21243	sgl->addr_hi = bpl->addrHigh;
21244	sgl->addr_lo = bpl->addrLow;
21245
21246	sgl->word2 = le32_to_cpu(sgl->word2);
21247	if ((i+1) == numBdes)
21248	bf_set(lpfc_sli4_sge_last, sgl, 1);
21249	else
21250	bf_set(lpfc_sli4_sge_last, sgl, 0);
21251	/* swap the size field back to the cpu so we
21252	* can assign it to the sgl.
21253	*/
21254	bde.tus.w = le32_to_cpu(bpl->tus.w);
21255	sgl->sge_len = cpu_to_le32(bde.tus.f.bdeSize);
21256	/* The offsets in the sgl need to be accumulated
21257	* separately for the request and reply lists.
21258	* The request is always first, the reply follows.
21259	*/
21260	switch (cmd) {
21261	case CMD_GEN_REQUEST64_WQE:
21262	/* add up the reply sg entries */
21263	if (bpl->tus.f.bdeFlags == BUFF_TYPE_BDE_64I)
21264	inbound++;
21265	/* first inbound? reset the offset */
21266	if (inbound == 1)
21267	offset = 0;
21268	bf_set(lpfc_sli4_sge_offset, sgl, offset);
21269	bf_set(lpfc_sli4_sge_type, sgl,
21270	LPFC_SGE_TYPE_DATA);
21271	offset += bde.tus.f.bdeSize;
21272	break;
21273	case CMD_FCP_TRSP64_WQE:
21274	bf_set(lpfc_sli4_sge_offset, sgl, 0);
21275	bf_set(lpfc_sli4_sge_type, sgl,
21276	LPFC_SGE_TYPE_DATA);
21277	break;
21278	case CMD_FCP_TSEND64_WQE:
21279	case CMD_FCP_TRECEIVE64_WQE:
21280	bf_set(lpfc_sli4_sge_type, sgl,
21281	bpl->tus.f.bdeFlags);
21282	if (i < 3)
21283	offset = 0;
21284	else
21285	offset += bde.tus.f.bdeSize;
21286	bf_set(lpfc_sli4_sge_offset, sgl, offset);
21287	break;
21288	}
21289	sgl->word2 = cpu_to_le32(sgl->word2);
21290	bpl++;
21291	sgl++;
21292	}
21293	} else if (wqe->gen_req.bde.tus.f.bdeFlags == BUFF_TYPE_BDE_64) {
21294	/* The addrHigh and addrLow fields of the BDE have not
21295	* been byteswapped yet so they need to be swapped
21296	* before putting them in the sgl.
21297	*/
21298	sgl->addr_hi = cpu_to_le32(wqe->gen_req.bde.addrHigh);
21299	sgl->addr_lo = cpu_to_le32(wqe->gen_req.bde.addrLow);
21300	sgl->word2 = le32_to_cpu(sgl->word2);
21301	bf_set(lpfc_sli4_sge_last, sgl, 1);
21302	sgl->word2 = cpu_to_le32(sgl->word2);
21303	sgl->sge_len = cpu_to_le32(wqe->gen_req.bde.tus.f.bdeSize);
21304	}
21305	return sglq->sli4_xritag;
21306	}
21307
21308	/**
21309	* lpfc_sli4_issue_wqe - Issue an SLI4 Work Queue Entry (WQE)
21310	* @phba: Pointer to HBA context object.
21311	* @qp: Pointer to HDW queue.
21312	* @pwqe: Pointer to command WQE.
21313	**/
21314	int
21315	lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp,
21316	struct lpfc_iocbq *pwqe)
21317	{
21318	union lpfc_wqe128 *wqe = &pwqe->wqe;
21319	struct lpfc_async_xchg_ctx *ctxp;
21320	struct lpfc_queue *wq;
21321	struct lpfc_sglq *sglq;
21322	struct lpfc_sli_ring *pring;
21323	unsigned long iflags;
21324	uint32_t ret = 0;
21325
21326	/* NVME_LS and NVME_LS ABTS requests. */
21327	if (pwqe->cmd_flag & LPFC_IO_NVME_LS) {
21328	pring = phba->sli4_hba.nvmels_wq->pring;
21329	lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags,
21330	qp, wq_access);
21331	sglq = __lpfc_sli_get_els_sglq(phba, piocbq: pwqe);
21332	if (!sglq) {
21333	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
21334	return WQE_BUSY;
21335	}
21336	pwqe->sli4_lxritag = sglq->sli4_lxritag;
21337	pwqe->sli4_xritag = sglq->sli4_xritag;
21338	if (lpfc_wqe_bpl2sgl(phba, pwqeq: pwqe, sglq) == NO_XRI) {
21339	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
21340	return WQE_ERROR;
21341	}
21342	bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com,
21343	pwqe->sli4_xritag);
21344	ret = lpfc_sli4_wq_put(q: phba->sli4_hba.nvmels_wq, wqe);
21345	if (ret) {
21346	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
21347	return ret;
21348	}
21349
21350	lpfc_sli_ringtxcmpl_put(phba, pring, piocb: pwqe);
21351	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
21352
21353	lpfc_sli4_poll_eq(eq: qp->hba_eq);
21354	return 0;
21355	}
21356
21357	/* NVME_FCREQ and NVME_ABTS requests */
21358	if (pwqe->cmd_flag & (LPFC_IO_NVME | LPFC_IO_FCP | LPFC_IO_CMF)) {
21359	/* Get the IO distribution (hba_wqidx) for WQ assignment. */
21360	wq = qp->io_wq;
21361	pring = wq->pring;
21362
21363	bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->io_cq_map);
21364
21365	lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags,
21366	qp, wq_access);
21367	ret = lpfc_sli4_wq_put(q: wq, wqe);
21368	if (ret) {
21369	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
21370	return ret;
21371	}
21372	lpfc_sli_ringtxcmpl_put(phba, pring, piocb: pwqe);
21373	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
21374
21375	lpfc_sli4_poll_eq(eq: qp->hba_eq);
21376	return 0;
21377	}
21378
21379	/* NVMET requests */
21380	if (pwqe->cmd_flag & LPFC_IO_NVMET) {
21381	/* Get the IO distribution (hba_wqidx) for WQ assignment. */
21382	wq = qp->io_wq;
21383	pring = wq->pring;
21384
21385	ctxp = pwqe->context_un.axchg;
21386	sglq = ctxp->ctxbuf->sglq;
21387	if (pwqe->sli4_xritag == NO_XRI) {
21388	pwqe->sli4_lxritag = sglq->sli4_lxritag;
21389	pwqe->sli4_xritag = sglq->sli4_xritag;
21390	}
21391	bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com,
21392	pwqe->sli4_xritag);
21393	bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->io_cq_map);
21394
21395	lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags,
21396	qp, wq_access);
21397	ret = lpfc_sli4_wq_put(q: wq, wqe);
21398	if (ret) {
21399	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
21400	return ret;
21401	}
21402	lpfc_sli_ringtxcmpl_put(phba, pring, piocb: pwqe);
21403	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
21404
21405	lpfc_sli4_poll_eq(eq: qp->hba_eq);
21406	return 0;
21407	}
21408	return WQE_ERROR;
21409	}
21410
21411	/**
21412	* lpfc_sli4_issue_abort_iotag - SLI-4 WQE init & issue for the Abort
21413	* @phba: Pointer to HBA context object.
21414	* @cmdiocb: Pointer to driver command iocb object.
21415	* @cmpl: completion function.
21416	*
21417	* Fill the appropriate fields for the abort WQE and call
21418	* internal routine lpfc_sli4_issue_wqe to send the WQE
21419	* This function is called with hbalock held and no ring_lock held.
21420	*
21421	* RETURNS 0 - SUCCESS
21422	**/
21423
21424	int
21425	lpfc_sli4_issue_abort_iotag(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
21426	void *cmpl)
21427	{
21428	struct lpfc_vport *vport = cmdiocb->vport;
21429	struct lpfc_iocbq *abtsiocb = NULL;
21430	union lpfc_wqe128 *abtswqe;
21431	struct lpfc_io_buf *lpfc_cmd;
21432	int retval = IOCB_ERROR;
21433	u16 xritag = cmdiocb->sli4_xritag;
21434
21435	/*
21436	* The scsi command can not be in txq and it is in flight because the
21437	* pCmd is still pointing at the SCSI command we have to abort. There
21438	* is no need to search the txcmplq. Just send an abort to the FW.
21439	*/
21440
21441	abtsiocb = __lpfc_sli_get_iocbq(phba);
21442	if (!abtsiocb)
21443	return WQE_NORESOURCE;
21444
21445	/* Indicate the IO is being aborted by the driver. */
21446	cmdiocb->cmd_flag |= LPFC_DRIVER_ABORTED;
21447
21448	abtswqe = &abtsiocb->wqe;
21449	memset(abtswqe, 0, sizeof(*abtswqe));
21450
21451	if (!lpfc_is_link_up(phba) || (phba->link_flag & LS_EXTERNAL_LOOPBACK))
21452	bf_set(abort_cmd_ia, &abtswqe->abort_cmd, 1);
21453	bf_set(abort_cmd_criteria, &abtswqe->abort_cmd, T_XRI_TAG);
21454	abtswqe->abort_cmd.rsrvd5 = 0;
21455	abtswqe->abort_cmd.wqe_com.abort_tag = xritag;
21456	bf_set(wqe_reqtag, &abtswqe->abort_cmd.wqe_com, abtsiocb->iotag);
21457	bf_set(wqe_cmnd, &abtswqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
21458	bf_set(wqe_xri_tag, &abtswqe->generic.wqe_com, 0);
21459	bf_set(wqe_qosd, &abtswqe->abort_cmd.wqe_com, 1);
21460	bf_set(wqe_lenloc, &abtswqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
21461	bf_set(wqe_cmd_type, &abtswqe->abort_cmd.wqe_com, OTHER_COMMAND);
21462
21463	/* ABTS WQE must go to the same WQ as the WQE to be aborted */
21464	abtsiocb->hba_wqidx = cmdiocb->hba_wqidx;
21465	abtsiocb->cmd_flag |= LPFC_USE_FCPWQIDX;
21466	if (cmdiocb->cmd_flag & LPFC_IO_FCP)
21467	abtsiocb->cmd_flag |= LPFC_IO_FCP;
21468	if (cmdiocb->cmd_flag & LPFC_IO_NVME)
21469	abtsiocb->cmd_flag |= LPFC_IO_NVME;
21470	if (cmdiocb->cmd_flag & LPFC_IO_FOF)
21471	abtsiocb->cmd_flag |= LPFC_IO_FOF;
21472	abtsiocb->vport = vport;
21473	abtsiocb->cmd_cmpl = cmpl;
21474
21475	lpfc_cmd = container_of(cmdiocb, struct lpfc_io_buf, cur_iocbq);
21476	retval = lpfc_sli4_issue_wqe(phba, qp: lpfc_cmd->hdwq, pwqe: abtsiocb);
21477
21478	lpfc_printf_vlog(vport, KERN_INFO, LOG_SLI | LOG_NVME_ABTS | LOG_FCP,
21479	"0359 Abort xri x%x, original iotag x%x, "
21480	"abort cmd iotag x%x retval x%x\n",
21481	xritag, cmdiocb->iotag, abtsiocb->iotag, retval);
21482
21483	if (retval) {
21484	cmdiocb->cmd_flag &= ~LPFC_DRIVER_ABORTED;
21485	__lpfc_sli_release_iocbq(phba, iocbq: abtsiocb);
21486	}
21487
21488	return retval;
21489	}
21490
21491	#ifdef LPFC_MXP_STAT
21492	/**
21493	* lpfc_snapshot_mxp - Snapshot pbl, pvt and busy count
21494	* @phba: pointer to lpfc hba data structure.
21495	* @hwqid: belong to which HWQ.
21496	*
21497	* The purpose of this routine is to take a snapshot of pbl, pvt and busy count
21498	* 15 seconds after a test case is running.
21499	*
21500	* The user should call lpfc_debugfs_multixripools_write before running a test
21501	* case to clear stat_snapshot_taken. Then the user starts a test case. During
21502	* test case is running, stat_snapshot_taken is incremented by 1 every time when
21503	* this routine is called from heartbeat timer. When stat_snapshot_taken is
21504	* equal to LPFC_MXP_SNAPSHOT_TAKEN, a snapshot is taken.
21505	**/
21506	void lpfc_snapshot_mxp(struct lpfc_hba *phba, u32 hwqid)
21507	{
21508	struct lpfc_sli4_hdw_queue *qp;
21509	struct lpfc_multixri_pool *multixri_pool;
21510	struct lpfc_pvt_pool *pvt_pool;
21511	struct lpfc_pbl_pool *pbl_pool;
21512	u32 txcmplq_cnt;
21513
21514	qp = &phba->sli4_hba.hdwq[hwqid];
21515	multixri_pool = qp->p_multixri_pool;
21516	if (!multixri_pool)
21517	return;
21518
21519	if (multixri_pool->stat_snapshot_taken == LPFC_MXP_SNAPSHOT_TAKEN) {
21520	pvt_pool = &qp->p_multixri_pool->pvt_pool;
21521	pbl_pool = &qp->p_multixri_pool->pbl_pool;
21522	txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt;
21523
21524	multixri_pool->stat_pbl_count = pbl_pool->count;
21525	multixri_pool->stat_pvt_count = pvt_pool->count;
21526	multixri_pool->stat_busy_count = txcmplq_cnt;
21527	}
21528
21529	multixri_pool->stat_snapshot_taken++;
21530	}
21531	#endif
21532
21533	/**
21534	* lpfc_adjust_pvt_pool_count - Adjust private pool count
21535	* @phba: pointer to lpfc hba data structure.
21536	* @hwqid: belong to which HWQ.
21537	*
21538	* This routine moves some XRIs from private to public pool when private pool
21539	* is not busy.
21540	**/
21541	void lpfc_adjust_pvt_pool_count(struct lpfc_hba *phba, u32 hwqid)
21542	{
21543	struct lpfc_multixri_pool *multixri_pool;
21544	u32 io_req_count;
21545	u32 prev_io_req_count;
21546
21547	multixri_pool = phba->sli4_hba.hdwq[hwqid].p_multixri_pool;
21548	if (!multixri_pool)
21549	return;
21550	io_req_count = multixri_pool->io_req_count;
21551	prev_io_req_count = multixri_pool->prev_io_req_count;
21552
21553	if (prev_io_req_count != io_req_count) {
21554	/* Private pool is busy */
21555	multixri_pool->prev_io_req_count = io_req_count;
21556	} else {
21557	/* Private pool is not busy.
21558	* Move XRIs from private to public pool.
21559	*/
21560	lpfc_move_xri_pvt_to_pbl(phba, hwqid);
21561	}
21562	}
21563
21564	/**
21565	* lpfc_adjust_high_watermark - Adjust high watermark
21566	* @phba: pointer to lpfc hba data structure.
21567	* @hwqid: belong to which HWQ.
21568	*
21569	* This routine sets high watermark as number of outstanding XRIs,
21570	* but make sure the new value is between xri_limit/2 and xri_limit.
21571	**/
21572	void lpfc_adjust_high_watermark(struct lpfc_hba *phba, u32 hwqid)
21573	{
21574	u32 new_watermark;
21575	u32 watermark_max;
21576	u32 watermark_min;
21577	u32 xri_limit;
21578	u32 txcmplq_cnt;
21579	u32 abts_io_bufs;
21580	struct lpfc_multixri_pool *multixri_pool;
21581	struct lpfc_sli4_hdw_queue *qp;
21582
21583	qp = &phba->sli4_hba.hdwq[hwqid];
21584	multixri_pool = qp->p_multixri_pool;
21585	if (!multixri_pool)
21586	return;
21587	xri_limit = multixri_pool->xri_limit;
21588
21589	watermark_max = xri_limit;
21590	watermark_min = xri_limit / 2;
21591
21592	txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt;
21593	abts_io_bufs = qp->abts_scsi_io_bufs;
21594	abts_io_bufs += qp->abts_nvme_io_bufs;
21595
21596	new_watermark = txcmplq_cnt + abts_io_bufs;
21597	new_watermark = min(watermark_max, new_watermark);
21598	new_watermark = max(watermark_min, new_watermark);
21599	multixri_pool->pvt_pool.high_watermark = new_watermark;
21600
21601	#ifdef LPFC_MXP_STAT
21602	multixri_pool->stat_max_hwm = max(multixri_pool->stat_max_hwm,
21603	new_watermark);
21604	#endif
21605	}
21606
21607	/**
21608	* lpfc_move_xri_pvt_to_pbl - Move some XRIs from private to public pool
21609	* @phba: pointer to lpfc hba data structure.
21610	* @hwqid: belong to which HWQ.
21611	*
21612	* This routine is called from hearbeat timer when pvt_pool is idle.
21613	* All free XRIs are moved from private to public pool on hwqid with 2 steps.
21614	* The first step moves (all - low_watermark) amount of XRIs.
21615	* The second step moves the rest of XRIs.
21616	**/
21617	void lpfc_move_xri_pvt_to_pbl(struct lpfc_hba *phba, u32 hwqid)
21618	{
21619	struct lpfc_pbl_pool *pbl_pool;
21620	struct lpfc_pvt_pool *pvt_pool;
21621	struct lpfc_sli4_hdw_queue *qp;
21622	struct lpfc_io_buf *lpfc_ncmd;
21623	struct lpfc_io_buf *lpfc_ncmd_next;
21624	unsigned long iflag;
21625	struct list_head tmp_list;
21626	u32 tmp_count;
21627
21628	qp = &phba->sli4_hba.hdwq[hwqid];
21629	pbl_pool = &qp->p_multixri_pool->pbl_pool;
21630	pvt_pool = &qp->p_multixri_pool->pvt_pool;
21631	tmp_count = 0;
21632
21633	lpfc_qp_spin_lock_irqsave(&pbl_pool->lock, iflag, qp, mv_to_pub_pool);
21634	lpfc_qp_spin_lock(&pvt_pool->lock, qp, mv_from_pvt_pool);
21635
21636	if (pvt_pool->count > pvt_pool->low_watermark) {
21637	/* Step 1: move (all - low_watermark) from pvt_pool
21638	* to pbl_pool
21639	*/
21640
21641	/* Move low watermark of bufs from pvt_pool to tmp_list */
21642	INIT_LIST_HEAD(list: &tmp_list);
21643	list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,
21644	&pvt_pool->list, list) {
21645	list_move_tail(list: &lpfc_ncmd->list, head: &tmp_list);
21646	tmp_count++;
21647	if (tmp_count >= pvt_pool->low_watermark)
21648	break;
21649	}
21650
21651	/* Move all bufs from pvt_pool to pbl_pool */
21652	list_splice_init(list: &pvt_pool->list, head: &pbl_pool->list);
21653
21654	/* Move all bufs from tmp_list to pvt_pool */
21655	list_splice(list: &tmp_list, head: &pvt_pool->list);
21656
21657	pbl_pool->count += (pvt_pool->count - tmp_count);
21658	pvt_pool->count = tmp_count;
21659	} else {
21660	/* Step 2: move the rest from pvt_pool to pbl_pool */
21661	list_splice_init(list: &pvt_pool->list, head: &pbl_pool->list);
21662	pbl_pool->count += pvt_pool->count;
21663	pvt_pool->count = 0;
21664	}
21665
21666	spin_unlock(lock: &pvt_pool->lock);
21667	spin_unlock_irqrestore(lock: &pbl_pool->lock, flags: iflag);
21668	}
21669
21670	/**
21671	* _lpfc_move_xri_pbl_to_pvt - Move some XRIs from public to private pool
21672	* @phba: pointer to lpfc hba data structure
21673	* @qp: pointer to HDW queue
21674	* @pbl_pool: specified public free XRI pool
21675	* @pvt_pool: specified private free XRI pool
21676	* @count: number of XRIs to move
21677	*
21678	* This routine tries to move some free common bufs from the specified pbl_pool
21679	* to the specified pvt_pool. It might move less than count XRIs if there's not
21680	* enough in public pool.
21681	*
21682	* Return:
21683	* true - if XRIs are successfully moved from the specified pbl_pool to the
21684	* specified pvt_pool
21685	* false - if the specified pbl_pool is empty or locked by someone else
21686	**/
21687	static bool
21688	_lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp,
21689	struct lpfc_pbl_pool *pbl_pool,
21690	struct lpfc_pvt_pool *pvt_pool, u32 count)
21691	{
21692	struct lpfc_io_buf *lpfc_ncmd;
21693	struct lpfc_io_buf *lpfc_ncmd_next;
21694	unsigned long iflag;
21695	int ret;
21696
21697	ret = spin_trylock_irqsave(&pbl_pool->lock, iflag);
21698	if (ret) {
21699	if (pbl_pool->count) {
21700	/* Move a batch of XRIs from public to private pool */
21701	lpfc_qp_spin_lock(&pvt_pool->lock, qp, mv_to_pvt_pool);
21702	list_for_each_entry_safe(lpfc_ncmd,
21703	lpfc_ncmd_next,
21704	&pbl_pool->list,
21705	list) {
21706	list_move_tail(list: &lpfc_ncmd->list,
21707	head: &pvt_pool->list);
21708	pvt_pool->count++;
21709	pbl_pool->count--;
21710	count--;
21711	if (count == 0)
21712	break;
21713	}
21714
21715	spin_unlock(lock: &pvt_pool->lock);
21716	spin_unlock_irqrestore(lock: &pbl_pool->lock, flags: iflag);
21717	return true;
21718	}
21719	spin_unlock_irqrestore(lock: &pbl_pool->lock, flags: iflag);
21720	}
21721
21722	return false;
21723	}
21724
21725	/**
21726	* lpfc_move_xri_pbl_to_pvt - Move some XRIs from public to private pool
21727	* @phba: pointer to lpfc hba data structure.
21728	* @hwqid: belong to which HWQ.
21729	* @count: number of XRIs to move
21730	*
21731	* This routine tries to find some free common bufs in one of public pools with
21732	* Round Robin method. The search always starts from local hwqid, then the next
21733	* HWQ which was found last time (rrb_next_hwqid). Once a public pool is found,
21734	* a batch of free common bufs are moved to private pool on hwqid.
21735	* It might move less than count XRIs if there's not enough in public pool.
21736	**/
21737	void lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, u32 hwqid, u32 count)
21738	{
21739	struct lpfc_multixri_pool *multixri_pool;
21740	struct lpfc_multixri_pool *next_multixri_pool;
21741	struct lpfc_pvt_pool *pvt_pool;
21742	struct lpfc_pbl_pool *pbl_pool;
21743	struct lpfc_sli4_hdw_queue *qp;
21744	u32 next_hwqid;
21745	u32 hwq_count;
21746	int ret;
21747
21748	qp = &phba->sli4_hba.hdwq[hwqid];
21749	multixri_pool = qp->p_multixri_pool;
21750	pvt_pool = &multixri_pool->pvt_pool;
21751	pbl_pool = &multixri_pool->pbl_pool;
21752
21753	/* Check if local pbl_pool is available */
21754	ret = _lpfc_move_xri_pbl_to_pvt(phba, qp, pbl_pool, pvt_pool, count);
21755	if (ret) {
21756	#ifdef LPFC_MXP_STAT
21757	multixri_pool->local_pbl_hit_count++;
21758	#endif
21759	return;
21760	}
21761
21762	hwq_count = phba->cfg_hdw_queue;
21763
21764	/* Get the next hwqid which was found last time */
21765	next_hwqid = multixri_pool->rrb_next_hwqid;
21766
21767	do {
21768	/* Go to next hwq */
21769	next_hwqid = (next_hwqid + 1) % hwq_count;
21770
21771	next_multixri_pool =
21772	phba->sli4_hba.hdwq[next_hwqid].p_multixri_pool;
21773	pbl_pool = &next_multixri_pool->pbl_pool;
21774
21775	/* Check if the public free xri pool is available */
21776	ret = _lpfc_move_xri_pbl_to_pvt(
21777	phba, qp, pbl_pool, pvt_pool, count);
21778
21779	/* Exit while-loop if success or all hwqid are checked */
21780	} while (!ret && next_hwqid != multixri_pool->rrb_next_hwqid);
21781
21782	/* Starting point for the next time */
21783	multixri_pool->rrb_next_hwqid = next_hwqid;
21784
21785	if (!ret) {
21786	/* stats: all public pools are empty*/
21787	multixri_pool->pbl_empty_count++;
21788	}
21789
21790	#ifdef LPFC_MXP_STAT
21791	if (ret) {
21792	if (next_hwqid == hwqid)
21793	multixri_pool->local_pbl_hit_count++;
21794	else
21795	multixri_pool->other_pbl_hit_count++;
21796	}
21797	#endif
21798	}
21799
21800	/**
21801	* lpfc_keep_pvt_pool_above_lowwm - Keep pvt_pool above low watermark
21802	* @phba: pointer to lpfc hba data structure.
21803	* @hwqid: belong to which HWQ.
21804	*
21805	* This routine get a batch of XRIs from pbl_pool if pvt_pool is less than
21806	* low watermark.
21807	**/
21808	void lpfc_keep_pvt_pool_above_lowwm(struct lpfc_hba *phba, u32 hwqid)
21809	{
21810	struct lpfc_multixri_pool *multixri_pool;
21811	struct lpfc_pvt_pool *pvt_pool;
21812
21813	multixri_pool = phba->sli4_hba.hdwq[hwqid].p_multixri_pool;
21814	pvt_pool = &multixri_pool->pvt_pool;
21815
21816	if (pvt_pool->count < pvt_pool->low_watermark)
21817	lpfc_move_xri_pbl_to_pvt(phba, hwqid, XRI_BATCH);
21818	}
21819
21820	/**
21821	* lpfc_release_io_buf - Return one IO buf back to free pool
21822	* @phba: pointer to lpfc hba data structure.
21823	* @lpfc_ncmd: IO buf to be returned.
21824	* @qp: belong to which HWQ.
21825	*
21826	* This routine returns one IO buf back to free pool. If this is an urgent IO,
21827	* the IO buf is returned to expedite pool. If cfg_xri_rebalancing==1,
21828	* the IO buf is returned to pbl_pool or pvt_pool based on watermark and
21829	* xri_limit. If cfg_xri_rebalancing==0, the IO buf is returned to
21830	* lpfc_io_buf_list_put.
21831	**/
21832	void lpfc_release_io_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd,
21833	struct lpfc_sli4_hdw_queue *qp)
21834	{
21835	unsigned long iflag;
21836	struct lpfc_pbl_pool *pbl_pool;
21837	struct lpfc_pvt_pool *pvt_pool;
21838	struct lpfc_epd_pool *epd_pool;
21839	u32 txcmplq_cnt;
21840	u32 xri_owned;
21841	u32 xri_limit;
21842	u32 abts_io_bufs;
21843
21844	/* MUST zero fields if buffer is reused by another protocol */
21845	lpfc_ncmd->nvmeCmd = NULL;
21846	lpfc_ncmd->cur_iocbq.cmd_cmpl = NULL;
21847
21848	if (phba->cfg_xpsgl && !phba->nvmet_support &&
21849	!list_empty(head: &lpfc_ncmd->dma_sgl_xtra_list))
21850	lpfc_put_sgl_per_hdwq(phba, buf: lpfc_ncmd);
21851
21852	if (!list_empty(head: &lpfc_ncmd->dma_cmd_rsp_list))
21853	lpfc_put_cmd_rsp_buf_per_hdwq(phba, buf: lpfc_ncmd);
21854
21855	if (phba->cfg_xri_rebalancing) {
21856	if (lpfc_ncmd->expedite) {
21857	/* Return to expedite pool */
21858	epd_pool = &phba->epd_pool;
21859	spin_lock_irqsave(&epd_pool->lock, iflag);
21860	list_add_tail(new: &lpfc_ncmd->list, head: &epd_pool->list);
21861	epd_pool->count++;
21862	spin_unlock_irqrestore(lock: &epd_pool->lock, flags: iflag);
21863	return;
21864	}
21865
21866	/* Avoid invalid access if an IO sneaks in and is being rejected
21867	* just _after_ xri pools are destroyed in lpfc_offline.
21868	* Nothing much can be done at this point.
21869	*/
21870	if (!qp->p_multixri_pool)
21871	return;
21872
21873	pbl_pool = &qp->p_multixri_pool->pbl_pool;
21874	pvt_pool = &qp->p_multixri_pool->pvt_pool;
21875
21876	txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt;
21877	abts_io_bufs = qp->abts_scsi_io_bufs;
21878	abts_io_bufs += qp->abts_nvme_io_bufs;
21879
21880	xri_owned = pvt_pool->count + txcmplq_cnt + abts_io_bufs;
21881	xri_limit = qp->p_multixri_pool->xri_limit;
21882
21883	#ifdef LPFC_MXP_STAT
21884	if (xri_owned <= xri_limit)
21885	qp->p_multixri_pool->below_limit_count++;
21886	else
21887	qp->p_multixri_pool->above_limit_count++;
21888	#endif
21889
21890	/* XRI goes to either public or private free xri pool
21891	* based on watermark and xri_limit
21892	*/
21893	if ((pvt_pool->count < pvt_pool->low_watermark) ||
21894	(xri_owned < xri_limit &&
21895	pvt_pool->count < pvt_pool->high_watermark)) {
21896	lpfc_qp_spin_lock_irqsave(&pvt_pool->lock, iflag,
21897	qp, free_pvt_pool);
21898	list_add_tail(new: &lpfc_ncmd->list,
21899	head: &pvt_pool->list);
21900	pvt_pool->count++;
21901	spin_unlock_irqrestore(lock: &pvt_pool->lock, flags: iflag);
21902	} else {
21903	lpfc_qp_spin_lock_irqsave(&pbl_pool->lock, iflag,
21904	qp, free_pub_pool);
21905	list_add_tail(new: &lpfc_ncmd->list,
21906	head: &pbl_pool->list);
21907	pbl_pool->count++;
21908	spin_unlock_irqrestore(lock: &pbl_pool->lock, flags: iflag);
21909	}
21910	} else {
21911	lpfc_qp_spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag,
21912	qp, free_xri);
21913	list_add_tail(new: &lpfc_ncmd->list,
21914	head: &qp->lpfc_io_buf_list_put);
21915	qp->put_io_bufs++;
21916	spin_unlock_irqrestore(lock: &qp->io_buf_list_put_lock,
21917	flags: iflag);
21918	}
21919	}
21920
21921	/**
21922	* lpfc_get_io_buf_from_private_pool - Get one free IO buf from private pool
21923	* @phba: pointer to lpfc hba data structure.
21924	* @qp: pointer to HDW queue
21925	* @pvt_pool: pointer to private pool data structure.
21926	* @ndlp: pointer to lpfc nodelist data structure.
21927	*
21928	* This routine tries to get one free IO buf from private pool.
21929	*
21930	* Return:
21931	* pointer to one free IO buf - if private pool is not empty
21932	* NULL - if private pool is empty
21933	**/
21934	static struct lpfc_io_buf *
21935	lpfc_get_io_buf_from_private_pool(struct lpfc_hba *phba,
21936	struct lpfc_sli4_hdw_queue *qp,
21937	struct lpfc_pvt_pool *pvt_pool,
21938	struct lpfc_nodelist *ndlp)
21939	{
21940	struct lpfc_io_buf *lpfc_ncmd;
21941	struct lpfc_io_buf *lpfc_ncmd_next;
21942	unsigned long iflag;
21943
21944	lpfc_qp_spin_lock_irqsave(&pvt_pool->lock, iflag, qp, alloc_pvt_pool);
21945	list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,
21946	&pvt_pool->list, list) {
21947	if (lpfc_test_rrq_active(
21948	phba, ndlp, xritag: lpfc_ncmd->cur_iocbq.sli4_lxritag))
21949	continue;
21950	list_del(entry: &lpfc_ncmd->list);
21951	pvt_pool->count--;
21952	spin_unlock_irqrestore(lock: &pvt_pool->lock, flags: iflag);
21953	return lpfc_ncmd;
21954	}
21955	spin_unlock_irqrestore(lock: &pvt_pool->lock, flags: iflag);
21956
21957	return NULL;
21958	}
21959
21960	/**
21961	* lpfc_get_io_buf_from_expedite_pool - Get one free IO buf from expedite pool
21962	* @phba: pointer to lpfc hba data structure.
21963	*
21964	* This routine tries to get one free IO buf from expedite pool.
21965	*
21966	* Return:
21967	* pointer to one free IO buf - if expedite pool is not empty
21968	* NULL - if expedite pool is empty
21969	**/
21970	static struct lpfc_io_buf *
21971	lpfc_get_io_buf_from_expedite_pool(struct lpfc_hba *phba)
21972	{
21973	struct lpfc_io_buf *lpfc_ncmd = NULL, *iter;
21974	struct lpfc_io_buf *lpfc_ncmd_next;
21975	unsigned long iflag;
21976	struct lpfc_epd_pool *epd_pool;
21977
21978	epd_pool = &phba->epd_pool;
21979
21980	spin_lock_irqsave(&epd_pool->lock, iflag);
21981	if (epd_pool->count > 0) {
21982	list_for_each_entry_safe(iter, lpfc_ncmd_next,
21983	&epd_pool->list, list) {
21984	list_del(entry: &iter->list);
21985	epd_pool->count--;
21986	lpfc_ncmd = iter;
21987	break;
21988	}
21989	}
21990	spin_unlock_irqrestore(lock: &epd_pool->lock, flags: iflag);
21991
21992	return lpfc_ncmd;
21993	}
21994
21995	/**
21996	* lpfc_get_io_buf_from_multixri_pools - Get one free IO bufs
21997	* @phba: pointer to lpfc hba data structure.
21998	* @ndlp: pointer to lpfc nodelist data structure.
21999	* @hwqid: belong to which HWQ
22000	* @expedite: 1 means this request is urgent.
22001	*
22002	* This routine will do the following actions and then return a pointer to
22003	* one free IO buf.
22004	*
22005	* 1. If private free xri count is empty, move some XRIs from public to
22006	* private pool.
22007	* 2. Get one XRI from private free xri pool.
22008	* 3. If we fail to get one from pvt_pool and this is an expedite request,
22009	* get one free xri from expedite pool.
22010	*
22011	* Note: ndlp is only used on SCSI side for RRQ testing.
22012	* The caller should pass NULL for ndlp on NVME side.
22013	*
22014	* Return:
22015	* pointer to one free IO buf - if private pool is not empty
22016	* NULL - if private pool is empty
22017	**/
22018	static struct lpfc_io_buf *
22019	lpfc_get_io_buf_from_multixri_pools(struct lpfc_hba *phba,
22020	struct lpfc_nodelist *ndlp,
22021	int hwqid, int expedite)
22022	{
22023	struct lpfc_sli4_hdw_queue *qp;
22024	struct lpfc_multixri_pool *multixri_pool;
22025	struct lpfc_pvt_pool *pvt_pool;
22026	struct lpfc_io_buf *lpfc_ncmd;
22027
22028	qp = &phba->sli4_hba.hdwq[hwqid];
22029	lpfc_ncmd = NULL;
22030	if (!qp) {
22031	lpfc_printf_log(phba, KERN_INFO,
22032	LOG_SLI | LOG_NVME_ABTS | LOG_FCP,
22033	"5556 NULL qp for hwqid x%x\n", hwqid);
22034	return lpfc_ncmd;
22035	}
22036	multixri_pool = qp->p_multixri_pool;
22037	if (!multixri_pool) {
22038	lpfc_printf_log(phba, KERN_INFO,
22039	LOG_SLI | LOG_NVME_ABTS | LOG_FCP,
22040	"5557 NULL multixri for hwqid x%x\n", hwqid);
22041	return lpfc_ncmd;
22042	}
22043	pvt_pool = &multixri_pool->pvt_pool;
22044	if (!pvt_pool) {
22045	lpfc_printf_log(phba, KERN_INFO,
22046	LOG_SLI | LOG_NVME_ABTS | LOG_FCP,
22047	"5558 NULL pvt_pool for hwqid x%x\n", hwqid);
22048	return lpfc_ncmd;
22049	}
22050	multixri_pool->io_req_count++;
22051
22052	/* If pvt_pool is empty, move some XRIs from public to private pool */
22053	if (pvt_pool->count == 0)
22054	lpfc_move_xri_pbl_to_pvt(phba, hwqid, XRI_BATCH);
22055
22056	/* Get one XRI from private free xri pool */
22057	lpfc_ncmd = lpfc_get_io_buf_from_private_pool(phba, qp, pvt_pool, ndlp);
22058
22059	if (lpfc_ncmd) {
22060	lpfc_ncmd->hdwq = qp;
22061	lpfc_ncmd->hdwq_no = hwqid;
22062	} else if (expedite) {
22063	/* If we fail to get one from pvt_pool and this is an expedite
22064	* request, get one free xri from expedite pool.
22065	*/
22066	lpfc_ncmd = lpfc_get_io_buf_from_expedite_pool(phba);
22067	}
22068
22069	return lpfc_ncmd;
22070	}
22071
22072	static inline struct lpfc_io_buf *
22073	lpfc_io_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, int idx)
22074	{
22075	struct lpfc_sli4_hdw_queue *qp;
22076	struct lpfc_io_buf *lpfc_cmd, *lpfc_cmd_next;
22077
22078	qp = &phba->sli4_hba.hdwq[idx];
22079	list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next,
22080	&qp->lpfc_io_buf_list_get, list) {
22081	if (lpfc_test_rrq_active(phba, ndlp,
22082	xritag: lpfc_cmd->cur_iocbq.sli4_lxritag))
22083	continue;
22084
22085	if (lpfc_cmd->flags & LPFC_SBUF_NOT_POSTED)
22086	continue;
22087
22088	list_del_init(entry: &lpfc_cmd->list);
22089	qp->get_io_bufs--;
22090	lpfc_cmd->hdwq = qp;
22091	lpfc_cmd->hdwq_no = idx;
22092	return lpfc_cmd;
22093	}
22094	return NULL;
22095	}
22096
22097	/**
22098	* lpfc_get_io_buf - Get one IO buffer from free pool
22099	* @phba: The HBA for which this call is being executed.
22100	* @ndlp: pointer to lpfc nodelist data structure.
22101	* @hwqid: belong to which HWQ
22102	* @expedite: 1 means this request is urgent.
22103	*
22104	* This routine gets one IO buffer from free pool. If cfg_xri_rebalancing==1,
22105	* removes a IO buffer from multiXRI pools. If cfg_xri_rebalancing==0, removes
22106	* a IO buffer from head of @hdwq io_buf_list and returns to caller.
22107	*
22108	* Note: ndlp is only used on SCSI side for RRQ testing.
22109	* The caller should pass NULL for ndlp on NVME side.
22110	*
22111	* Return codes:
22112	* NULL - Error
22113	* Pointer to lpfc_io_buf - Success
22114	**/
22115	struct lpfc_io_buf *lpfc_get_io_buf(struct lpfc_hba *phba,
22116	struct lpfc_nodelist *ndlp,
22117	u32 hwqid, int expedite)
22118	{
22119	struct lpfc_sli4_hdw_queue *qp;
22120	unsigned long iflag;
22121	struct lpfc_io_buf *lpfc_cmd;
22122
22123	qp = &phba->sli4_hba.hdwq[hwqid];
22124	lpfc_cmd = NULL;
22125	if (!qp) {
22126	lpfc_printf_log(phba, KERN_WARNING,
22127	LOG_SLI | LOG_NVME_ABTS | LOG_FCP,
22128	"5555 NULL qp for hwqid x%x\n", hwqid);
22129	return lpfc_cmd;
22130	}
22131
22132	if (phba->cfg_xri_rebalancing)
22133	lpfc_cmd = lpfc_get_io_buf_from_multixri_pools(
22134	phba, ndlp, hwqid, expedite);
22135	else {
22136	lpfc_qp_spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag,
22137	qp, alloc_xri_get);
22138	if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite)
22139	lpfc_cmd = lpfc_io_buf(phba, ndlp, idx: hwqid);
22140	if (!lpfc_cmd) {
22141	lpfc_qp_spin_lock(&qp->io_buf_list_put_lock,
22142	qp, alloc_xri_put);
22143	list_splice(list: &qp->lpfc_io_buf_list_put,
22144	head: &qp->lpfc_io_buf_list_get);
22145	qp->get_io_bufs += qp->put_io_bufs;
22146	INIT_LIST_HEAD(list: &qp->lpfc_io_buf_list_put);
22147	qp->put_io_bufs = 0;
22148	spin_unlock(lock: &qp->io_buf_list_put_lock);
22149	if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT ||
22150	expedite)
22151	lpfc_cmd = lpfc_io_buf(phba, ndlp, idx: hwqid);
22152	}
22153	spin_unlock_irqrestore(lock: &qp->io_buf_list_get_lock, flags: iflag);
22154	}
22155
22156	return lpfc_cmd;
22157	}
22158
22159	/**
22160	* lpfc_read_object - Retrieve object data from HBA
22161	* @phba: The HBA for which this call is being executed.
22162	* @rdobject: Pathname of object data we want to read.
22163	* @datap: Pointer to where data will be copied to.
22164	* @datasz: size of data area
22165	*
22166	* This routine is limited to object sizes of LPFC_BPL_SIZE (1024) or less.
22167	* The data will be truncated if datasz is not large enough.
22168	* Version 1 is not supported with Embedded mbox cmd, so we must use version 0.
22169	* Returns the actual bytes read from the object.
22170	*/
22171	int
22172	lpfc_read_object(struct lpfc_hba *phba, char *rdobject, uint32_t *datap,
22173	uint32_t datasz)
22174	{
22175	struct lpfc_mbx_read_object *read_object;
22176	LPFC_MBOXQ_t *mbox;
22177	int rc, length, eof, j, byte_cnt = 0;
22178	uint32_t shdr_status, shdr_add_status;
22179	union lpfc_sli4_cfg_shdr *shdr;
22180	struct lpfc_dmabuf *pcmd;
22181	u32 rd_object_name[LPFC_MBX_OBJECT_NAME_LEN_DW] = {0};
22182
22183	mbox = mempool_alloc(pool: phba->mbox_mem_pool, GFP_KERNEL);
22184	if (!mbox)
22185	return -ENOMEM;
22186	length = (sizeof(struct lpfc_mbx_read_object) -
22187	sizeof(struct lpfc_sli4_cfg_mhdr));
22188	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
22189	LPFC_MBOX_OPCODE_READ_OBJECT,
22190	length, LPFC_SLI4_MBX_EMBED);
22191	read_object = &mbox->u.mqe.un.read_object;
22192	shdr = (union lpfc_sli4_cfg_shdr *)&read_object->header.cfg_shdr;
22193
22194	bf_set(lpfc_mbox_hdr_version, &shdr->request, LPFC_Q_CREATE_VERSION_0);
22195	bf_set(lpfc_mbx_rd_object_rlen, &read_object->u.request, datasz);
22196	read_object->u.request.rd_object_offset = 0;
22197	read_object->u.request.rd_object_cnt = 1;
22198
22199	memset((void *)read_object->u.request.rd_object_name, 0,
22200	LPFC_OBJ_NAME_SZ);
22201	scnprintf(buf: (char *)rd_object_name, size: sizeof(rd_object_name), fmt: rdobject);
22202	for (j = 0; j < strlen(rdobject); j++)
22203	read_object->u.request.rd_object_name[j] =
22204	cpu_to_le32(rd_object_name[j]);
22205
22206	pcmd = kmalloc(size: sizeof(*pcmd), GFP_KERNEL);
22207	if (pcmd)
22208	pcmd->virt = lpfc_mbuf_alloc(phba, MEM_PRI, &pcmd->phys);
22209	if (!pcmd || !pcmd->virt) {
22210	kfree(objp: pcmd);
22211	mempool_free(element: mbox, pool: phba->mbox_mem_pool);
22212	return -ENOMEM;
22213	}
22214	memset((void *)pcmd->virt, 0, LPFC_BPL_SIZE);
22215	read_object->u.request.rd_object_hbuf[0].pa_lo =
22216	putPaddrLow(pcmd->phys);
22217	read_object->u.request.rd_object_hbuf[0].pa_hi =
22218	putPaddrHigh(pcmd->phys);
22219	read_object->u.request.rd_object_hbuf[0].length = LPFC_BPL_SIZE;
22220
22221	mbox->vport = phba->pport;
22222	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
22223	mbox->ctx_ndlp = NULL;
22224
22225	rc = lpfc_sli_issue_mbox(phba, pmbox: mbox, MBX_POLL);
22226	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
22227	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
22228
22229	if (shdr_status == STATUS_FAILED &&
22230	shdr_add_status == ADD_STATUS_INVALID_OBJECT_NAME) {
22231	lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_CGN_MGMT,
22232	"4674 No port cfg file in FW.\n");
22233	byte_cnt = -ENOENT;
22234	} else if (shdr_status || shdr_add_status || rc) {
22235	lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_CGN_MGMT,
22236	"2625 READ_OBJECT mailbox failed with "
22237	"status x%x add_status x%x, mbx status x%x\n",
22238	shdr_status, shdr_add_status, rc);
22239	byte_cnt = -ENXIO;
22240	} else {
22241	/* Success */
22242	length = read_object->u.response.rd_object_actual_rlen;
22243	eof = bf_get(lpfc_mbx_rd_object_eof, &read_object->u.response);
22244	lpfc_printf_log(phba, KERN_INFO, LOG_INIT | LOG_CGN_MGMT,
22245	"2626 READ_OBJECT Success len %d:%d, EOF %d\n",
22246	length, datasz, eof);
22247
22248	/* Detect the port config file exists but is empty */
22249	if (!length && eof) {
22250	byte_cnt = 0;
22251	goto exit;
22252	}
22253
22254	byte_cnt = length;
22255	lpfc_sli_pcimem_bcopy(srcp: pcmd->virt, destp: datap, cnt: byte_cnt);
22256	}
22257
22258	exit:
22259	/* This is an embedded SLI4 mailbox with an external buffer allocated.
22260	* Free the pcmd and then cleanup with the correct routine.
22261	*/
22262	lpfc_mbuf_free(phba, pcmd->virt, pcmd->phys);
22263	kfree(objp: pcmd);
22264	lpfc_sli4_mbox_cmd_free(phba, mbox);
22265	return byte_cnt;
22266	}
22267
22268	/**
22269	* lpfc_get_sgl_per_hdwq - Get one SGL chunk from hdwq's pool
22270	* @phba: The HBA for which this call is being executed.
22271	* @lpfc_buf: IO buf structure to append the SGL chunk
22272	*
22273	* This routine gets one SGL chunk buffer from hdwq's SGL chunk pool,
22274	* and will allocate an SGL chunk if the pool is empty.
22275	*
22276	* Return codes:
22277	* NULL - Error
22278	* Pointer to sli4_hybrid_sgl - Success
22279	**/
22280	struct sli4_hybrid_sgl *
22281	lpfc_get_sgl_per_hdwq(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_buf)
22282	{
22283	struct sli4_hybrid_sgl *list_entry = NULL;
22284	struct sli4_hybrid_sgl *tmp = NULL;
22285	struct sli4_hybrid_sgl *allocated_sgl = NULL;
22286	struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq;
22287	struct list_head *buf_list = &hdwq->sgl_list;
22288	unsigned long iflags;
22289
22290	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22291
22292	if (likely(!list_empty(buf_list))) {
22293	/* break off 1 chunk from the sgl_list */
22294	list_for_each_entry_safe(list_entry, tmp,
22295	buf_list, list_node) {
22296	list_move_tail(list: &list_entry->list_node,
22297	head: &lpfc_buf->dma_sgl_xtra_list);
22298	break;
22299	}
22300	} else {
22301	/* allocate more */
22302	spin_unlock_irqrestore(lock: &hdwq->hdwq_lock, flags: iflags);
22303	tmp = kmalloc_node(size: sizeof(*tmp), GFP_ATOMIC,
22304	cpu_to_node(cpu: hdwq->io_wq->chann));
22305	if (!tmp) {
22306	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
22307	"8353 error kmalloc memory for HDWQ "
22308	"%d %s\n",
22309	lpfc_buf->hdwq_no, __func__);
22310	return NULL;
22311	}
22312
22313	tmp->dma_sgl = dma_pool_alloc(pool: phba->lpfc_sg_dma_buf_pool,
22314	GFP_ATOMIC, handle: &tmp->dma_phys_sgl);
22315	if (!tmp->dma_sgl) {
22316	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
22317	"8354 error pool_alloc memory for HDWQ "
22318	"%d %s\n",
22319	lpfc_buf->hdwq_no, __func__);
22320	kfree(objp: tmp);
22321	return NULL;
22322	}
22323
22324	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22325	list_add_tail(new: &tmp->list_node, head: &lpfc_buf->dma_sgl_xtra_list);
22326	}
22327
22328	allocated_sgl = list_last_entry(&lpfc_buf->dma_sgl_xtra_list,
22329	struct sli4_hybrid_sgl,
22330	list_node);
22331
22332	spin_unlock_irqrestore(lock: &hdwq->hdwq_lock, flags: iflags);
22333
22334	return allocated_sgl;
22335	}
22336
22337	/**
22338	* lpfc_put_sgl_per_hdwq - Put one SGL chunk into hdwq pool
22339	* @phba: The HBA for which this call is being executed.
22340	* @lpfc_buf: IO buf structure with the SGL chunk
22341	*
22342	* This routine puts one SGL chunk buffer into hdwq's SGL chunk pool.
22343	*
22344	* Return codes:
22345	* 0 - Success
22346	* -EINVAL - Error
22347	**/
22348	int
22349	lpfc_put_sgl_per_hdwq(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_buf)
22350	{
22351	int rc = 0;
22352	struct sli4_hybrid_sgl *list_entry = NULL;
22353	struct sli4_hybrid_sgl *tmp = NULL;
22354	struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq;
22355	struct list_head *buf_list = &hdwq->sgl_list;
22356	unsigned long iflags;
22357
22358	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22359
22360	if (likely(!list_empty(&lpfc_buf->dma_sgl_xtra_list))) {
22361	list_for_each_entry_safe(list_entry, tmp,
22362	&lpfc_buf->dma_sgl_xtra_list,
22363	list_node) {
22364	list_move_tail(list: &list_entry->list_node,
22365	head: buf_list);
22366	}
22367	} else {
22368	rc = -EINVAL;
22369	}
22370
22371	spin_unlock_irqrestore(lock: &hdwq->hdwq_lock, flags: iflags);
22372	return rc;
22373	}
22374
22375	/**
22376	* lpfc_free_sgl_per_hdwq - Free all SGL chunks of hdwq pool
22377	* @phba: phba object
22378	* @hdwq: hdwq to cleanup sgl buff resources on
22379	*
22380	* This routine frees all SGL chunks of hdwq SGL chunk pool.
22381	*
22382	* Return codes:
22383	* None
22384	**/
22385	void
22386	lpfc_free_sgl_per_hdwq(struct lpfc_hba *phba,
22387	struct lpfc_sli4_hdw_queue *hdwq)
22388	{
22389	struct list_head *buf_list = &hdwq->sgl_list;
22390	struct sli4_hybrid_sgl *list_entry = NULL;
22391	struct sli4_hybrid_sgl *tmp = NULL;
22392	unsigned long iflags;
22393
22394	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22395
22396	/* Free sgl pool */
22397	list_for_each_entry_safe(list_entry, tmp,
22398	buf_list, list_node) {
22399	list_del(entry: &list_entry->list_node);
22400	dma_pool_free(pool: phba->lpfc_sg_dma_buf_pool,
22401	vaddr: list_entry->dma_sgl,
22402	addr: list_entry->dma_phys_sgl);
22403	kfree(objp: list_entry);
22404	}
22405
22406	spin_unlock_irqrestore(lock: &hdwq->hdwq_lock, flags: iflags);
22407	}
22408
22409	/**
22410	* lpfc_get_cmd_rsp_buf_per_hdwq - Get one CMD/RSP buffer from hdwq
22411	* @phba: The HBA for which this call is being executed.
22412	* @lpfc_buf: IO buf structure to attach the CMD/RSP buffer
22413	*
22414	* This routine gets one CMD/RSP buffer from hdwq's CMD/RSP pool,
22415	* and will allocate an CMD/RSP buffer if the pool is empty.
22416	*
22417	* Return codes:
22418	* NULL - Error
22419	* Pointer to fcp_cmd_rsp_buf - Success
22420	**/
22421	struct fcp_cmd_rsp_buf *
22422	lpfc_get_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba,
22423	struct lpfc_io_buf *lpfc_buf)
22424	{
22425	struct fcp_cmd_rsp_buf *list_entry = NULL;
22426	struct fcp_cmd_rsp_buf *tmp = NULL;
22427	struct fcp_cmd_rsp_buf *allocated_buf = NULL;
22428	struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq;
22429	struct list_head *buf_list = &hdwq->cmd_rsp_buf_list;
22430	unsigned long iflags;
22431
22432	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22433
22434	if (likely(!list_empty(buf_list))) {
22435	/* break off 1 chunk from the list */
22436	list_for_each_entry_safe(list_entry, tmp,
22437	buf_list,
22438	list_node) {
22439	list_move_tail(list: &list_entry->list_node,
22440	head: &lpfc_buf->dma_cmd_rsp_list);
22441	break;
22442	}
22443	} else {
22444	/* allocate more */
22445	spin_unlock_irqrestore(lock: &hdwq->hdwq_lock, flags: iflags);
22446	tmp = kmalloc_node(size: sizeof(*tmp), GFP_ATOMIC,
22447	cpu_to_node(cpu: hdwq->io_wq->chann));
22448	if (!tmp) {
22449	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
22450	"8355 error kmalloc memory for HDWQ "
22451	"%d %s\n",
22452	lpfc_buf->hdwq_no, __func__);
22453	return NULL;
22454	}
22455
22456	tmp->fcp_cmnd = dma_pool_zalloc(pool: phba->lpfc_cmd_rsp_buf_pool,
22457	GFP_ATOMIC,
22458	handle: &tmp->fcp_cmd_rsp_dma_handle);
22459
22460	if (!tmp->fcp_cmnd) {
22461	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
22462	"8356 error pool_alloc memory for HDWQ "
22463	"%d %s\n",
22464	lpfc_buf->hdwq_no, __func__);
22465	kfree(objp: tmp);
22466	return NULL;
22467	}
22468
22469	tmp->fcp_rsp = (struct fcp_rsp *)((uint8_t *)tmp->fcp_cmnd +
22470	sizeof(struct fcp_cmnd));
22471
22472	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22473	list_add_tail(new: &tmp->list_node, head: &lpfc_buf->dma_cmd_rsp_list);
22474	}
22475
22476	allocated_buf = list_last_entry(&lpfc_buf->dma_cmd_rsp_list,
22477	struct fcp_cmd_rsp_buf,
22478	list_node);
22479
22480	spin_unlock_irqrestore(lock: &hdwq->hdwq_lock, flags: iflags);
22481
22482	return allocated_buf;
22483	}
22484
22485	/**
22486	* lpfc_put_cmd_rsp_buf_per_hdwq - Put one CMD/RSP buffer into hdwq pool
22487	* @phba: The HBA for which this call is being executed.
22488	* @lpfc_buf: IO buf structure with the CMD/RSP buf
22489	*
22490	* This routine puts one CMD/RSP buffer into executing CPU's CMD/RSP pool.
22491	*
22492	* Return codes:
22493	* 0 - Success
22494	* -EINVAL - Error
22495	**/
22496	int
22497	lpfc_put_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba,
22498	struct lpfc_io_buf *lpfc_buf)
22499	{
22500	int rc = 0;
22501	struct fcp_cmd_rsp_buf *list_entry = NULL;
22502	struct fcp_cmd_rsp_buf *tmp = NULL;
22503	struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq;
22504	struct list_head *buf_list = &hdwq->cmd_rsp_buf_list;
22505	unsigned long iflags;
22506
22507	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22508
22509	if (likely(!list_empty(&lpfc_buf->dma_cmd_rsp_list))) {
22510	list_for_each_entry_safe(list_entry, tmp,
22511	&lpfc_buf->dma_cmd_rsp_list,
22512	list_node) {
22513	list_move_tail(list: &list_entry->list_node,
22514	head: buf_list);
22515	}
22516	} else {
22517	rc = -EINVAL;
22518	}
22519
22520	spin_unlock_irqrestore(lock: &hdwq->hdwq_lock, flags: iflags);
22521	return rc;
22522	}
22523
22524	/**
22525	* lpfc_free_cmd_rsp_buf_per_hdwq - Free all CMD/RSP chunks of hdwq pool
22526	* @phba: phba object
22527	* @hdwq: hdwq to cleanup cmd rsp buff resources on
22528	*
22529	* This routine frees all CMD/RSP buffers of hdwq's CMD/RSP buf pool.
22530	*
22531	* Return codes:
22532	* None
22533	**/
22534	void
22535	lpfc_free_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba,
22536	struct lpfc_sli4_hdw_queue *hdwq)
22537	{
22538	struct list_head *buf_list = &hdwq->cmd_rsp_buf_list;
22539	struct fcp_cmd_rsp_buf *list_entry = NULL;
22540	struct fcp_cmd_rsp_buf *tmp = NULL;
22541	unsigned long iflags;
22542
22543	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22544
22545	/* Free cmd_rsp buf pool */
22546	list_for_each_entry_safe(list_entry, tmp,
22547	buf_list,
22548	list_node) {
22549	list_del(entry: &list_entry->list_node);
22550	dma_pool_free(pool: phba->lpfc_cmd_rsp_buf_pool,
22551	vaddr: list_entry->fcp_cmnd,
22552	addr: list_entry->fcp_cmd_rsp_dma_handle);
22553	kfree(objp: list_entry);
22554	}
22555
22556	spin_unlock_irqrestore(lock: &hdwq->hdwq_lock, flags: iflags);
22557	}
22558
22559	/**
22560	* lpfc_sli_prep_wqe - Prepare WQE for the command to be posted
22561	* @phba: phba object
22562	* @job: job entry of the command to be posted.
22563	*
22564	* Fill the common fields of the wqe for each of the command.
22565	*
22566	* Return codes:
22567	* None
22568	**/
22569	void
22570	lpfc_sli_prep_wqe(struct lpfc_hba *phba, struct lpfc_iocbq *job)
22571	{
22572	u8 cmnd;
22573	u32 *pcmd;
22574	u32 if_type = 0;
22575	u32 fip, abort_tag;
22576	struct lpfc_nodelist *ndlp = NULL;
22577	union lpfc_wqe128 *wqe = &job->wqe;
22578	u8 command_type = ELS_COMMAND_NON_FIP;
22579
22580	fip = phba->hba_flag & HBA_FIP_SUPPORT;
22581	/* The fcp commands will set command type */
22582	if (job->cmd_flag & LPFC_IO_FCP)
22583	command_type = FCP_COMMAND;
22584	else if (fip && (job->cmd_flag & LPFC_FIP_ELS_ID_MASK))
22585	command_type = ELS_COMMAND_FIP;
22586	else
22587	command_type = ELS_COMMAND_NON_FIP;
22588
22589	abort_tag = job->iotag;
22590	cmnd = bf_get(wqe_cmnd, &wqe->els_req.wqe_com);
22591
22592	switch (cmnd) {
22593	case CMD_ELS_REQUEST64_WQE:
22594	ndlp = job->ndlp;
22595
22596	if_type = bf_get(lpfc_sli_intf_if_type,
22597	&phba->sli4_hba.sli_intf);
22598	if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) {
22599	pcmd = (u32 *)job->cmd_dmabuf->virt;
22600	if (pcmd && (*pcmd == ELS_CMD_FLOGI ||
22601	*pcmd == ELS_CMD_SCR ||
22602	*pcmd == ELS_CMD_RDF ||
22603	*pcmd == ELS_CMD_EDC ||
22604	*pcmd == ELS_CMD_RSCN_XMT ||
22605	*pcmd == ELS_CMD_FDISC ||
22606	*pcmd == ELS_CMD_LOGO ||
22607	*pcmd == ELS_CMD_QFPA ||
22608	*pcmd == ELS_CMD_UVEM ||
22609	*pcmd == ELS_CMD_PLOGI)) {
22610	bf_set(els_req64_sp, &wqe->els_req, 1);
22611	bf_set(els_req64_sid, &wqe->els_req,
22612	job->vport->fc_myDID);
22613
22614	if ((*pcmd == ELS_CMD_FLOGI) &&
22615	!(phba->fc_topology ==
22616	LPFC_TOPOLOGY_LOOP))
22617	bf_set(els_req64_sid, &wqe->els_req, 0);
22618
22619	bf_set(wqe_ct, &wqe->els_req.wqe_com, 1);
22620	bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com,
22621	phba->vpi_ids[job->vport->vpi]);
22622	} else if (pcmd) {
22623	bf_set(wqe_ct, &wqe->els_req.wqe_com, 0);
22624	bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com,
22625	phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
22626	}
22627	}
22628
22629	bf_set(wqe_temp_rpi, &wqe->els_req.wqe_com,
22630	phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
22631
22632	bf_set(wqe_dbde, &wqe->els_req.wqe_com, 1);
22633	bf_set(wqe_iod, &wqe->els_req.wqe_com, LPFC_WQE_IOD_READ);
22634	bf_set(wqe_qosd, &wqe->els_req.wqe_com, 1);
22635	bf_set(wqe_lenloc, &wqe->els_req.wqe_com, LPFC_WQE_LENLOC_NONE);
22636	bf_set(wqe_ebde_cnt, &wqe->els_req.wqe_com, 0);
22637	break;
22638	case CMD_XMIT_ELS_RSP64_WQE:
22639	ndlp = job->ndlp;
22640
22641	/* word4 */
22642	wqe->xmit_els_rsp.word4 = 0;
22643
22644	if_type = bf_get(lpfc_sli_intf_if_type,
22645	&phba->sli4_hba.sli_intf);
22646	if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) {
22647	if (job->vport->fc_flag & FC_PT2PT) {
22648	bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1);
22649	bf_set(els_rsp64_sid, &wqe->xmit_els_rsp,
22650	job->vport->fc_myDID);
22651	if (job->vport->fc_myDID == Fabric_DID) {
22652	bf_set(wqe_els_did,
22653	&wqe->xmit_els_rsp.wqe_dest, 0);
22654	}
22655	}
22656	}
22657
22658	bf_set(wqe_dbde, &wqe->xmit_els_rsp.wqe_com, 1);
22659	bf_set(wqe_iod, &wqe->xmit_els_rsp.wqe_com, LPFC_WQE_IOD_WRITE);
22660	bf_set(wqe_qosd, &wqe->xmit_els_rsp.wqe_com, 1);
22661	bf_set(wqe_lenloc, &wqe->xmit_els_rsp.wqe_com,
22662	LPFC_WQE_LENLOC_WORD3);
22663	bf_set(wqe_ebde_cnt, &wqe->xmit_els_rsp.wqe_com, 0);
22664
22665	if (phba->fc_topology == LPFC_TOPOLOGY_LOOP) {
22666	bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1);
22667	bf_set(els_rsp64_sid, &wqe->xmit_els_rsp,
22668	job->vport->fc_myDID);
22669	bf_set(wqe_ct, &wqe->xmit_els_rsp.wqe_com, 1);
22670	}
22671
22672	if (phba->sli_rev == LPFC_SLI_REV4) {
22673	bf_set(wqe_rsp_temp_rpi, &wqe->xmit_els_rsp,
22674	phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
22675
22676	if (bf_get(wqe_ct, &wqe->xmit_els_rsp.wqe_com))
22677	bf_set(wqe_ctxt_tag, &wqe->xmit_els_rsp.wqe_com,
22678	phba->vpi_ids[job->vport->vpi]);
22679	}
22680	command_type = OTHER_COMMAND;
22681	break;
22682	case CMD_GEN_REQUEST64_WQE:
22683	/* Word 10 */
22684	bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
22685	bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
22686	bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
22687	bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
22688	bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
22689	command_type = OTHER_COMMAND;
22690	break;
22691	case CMD_XMIT_SEQUENCE64_WQE:
22692	if (phba->link_flag & LS_LOOPBACK_MODE)
22693	bf_set(wqe_xo, &wqe->xmit_sequence.wge_ctl, 1);
22694
22695	wqe->xmit_sequence.rsvd3 = 0;
22696	bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0);
22697	bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1);
22698	bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com,
22699	LPFC_WQE_IOD_WRITE);
22700	bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com,
22701	LPFC_WQE_LENLOC_WORD12);
22702	bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0);
22703	command_type = OTHER_COMMAND;
22704	break;
22705	case CMD_XMIT_BLS_RSP64_WQE:
22706	bf_set(xmit_bls_rsp64_seqcnthi, &wqe->xmit_bls_rsp, 0xffff);
22707	bf_set(wqe_xmit_bls_pt, &wqe->xmit_bls_rsp.wqe_dest, 0x1);
22708	bf_set(wqe_ct, &wqe->xmit_bls_rsp.wqe_com, 1);
22709	bf_set(wqe_ctxt_tag, &wqe->xmit_bls_rsp.wqe_com,
22710	phba->vpi_ids[phba->pport->vpi]);
22711	bf_set(wqe_qosd, &wqe->xmit_bls_rsp.wqe_com, 1);
22712	bf_set(wqe_lenloc, &wqe->xmit_bls_rsp.wqe_com,
22713	LPFC_WQE_LENLOC_NONE);
22714	/* Overwrite the pre-set comnd type with OTHER_COMMAND */
22715	command_type = OTHER_COMMAND;
22716	break;
22717	case CMD_FCP_ICMND64_WQE: /* task mgmt commands */
22718	case CMD_ABORT_XRI_WQE: /* abort iotag */
22719	case CMD_SEND_FRAME: /* mds loopback */
22720	/* cases already formatted for sli4 wqe - no chgs necessary */
22721	return;
22722	default:
22723	dump_stack();
22724	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
22725	"6207 Invalid command 0x%x\n",
22726	cmnd);
22727	break;
22728	}
22729
22730	wqe->generic.wqe_com.abort_tag = abort_tag;
22731	bf_set(wqe_reqtag, &wqe->generic.wqe_com, job->iotag);
22732	bf_set(wqe_cmd_type, &wqe->generic.wqe_com, command_type);
22733	bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
22734	}
22735