1	/*******************************************************************
2	* This file is part of the Emulex Linux Device Driver for *
3	* Fibre Channel Host Bus Adapters. *
4	* Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
5	* “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. *
6	* Copyright (C) 2004-2016 Emulex. All rights reserved. *
7	* EMULEX and SLI are trademarks of Emulex. *
8	* www.broadcom.com *
9	* Portions Copyright (C) 2004-2005 Christoph Hellwig *
10	* *
11	* This program is free software; you can redistribute it and/or *
12	* modify it under the terms of version 2 of the GNU General *
13	* Public License as published by the Free Software Foundation. *
14	* This program is distributed in the hope that it will be useful. *
15	* ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
16	* WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
17	* FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
18	* DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19	* TO BE LEGALLY INVALID. See the GNU General Public License for *
20	* more details, a copy of which can be found in the file COPYING *
21	* included with this package. *
22	********************************************************************/
23	#include <linux/pci.h>
24	#include <linux/slab.h>
25	#include <linux/interrupt.h>
26	#include <linux/delay.h>
27	#include <asm/unaligned.h>
28	#include <linux/crc-t10dif.h>
29	#include <net/checksum.h>
30
31	#include <scsi/scsi.h>
32	#include <scsi/scsi_device.h>
33	#include <scsi/scsi_eh.h>
34	#include <scsi/scsi_host.h>
35	#include <scsi/scsi_tcq.h>
36	#include <scsi/scsi_transport_fc.h>
37	#include <scsi/fc/fc_fs.h>
38
39	#include "lpfc_version.h"
40	#include "lpfc_hw4.h"
41	#include "lpfc_hw.h"
42	#include "lpfc_sli.h"
43	#include "lpfc_sli4.h"
44	#include "lpfc_nl.h"
45	#include "lpfc_disc.h"
46	#include "lpfc.h"
47	#include "lpfc_nvme.h"
48	#include "lpfc_scsi.h"
49	#include "lpfc_logmsg.h"
50	#include "lpfc_crtn.h"
51	#include "lpfc_vport.h"
52	#include "lpfc_debugfs.h"
53
54	/* NVME initiator-based functions */
55
56	static struct lpfc_io_buf *
57	lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
58	int idx, int expedite);
59
60	static void
61	lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *);
62
63	static struct nvme_fc_port_template lpfc_nvme_template;
64
65	/**
66	* lpfc_nvme_create_queue -
67	* @pnvme_lport: Transport localport that LS is to be issued from
68	* @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
69	* @qsize: Size of the queue in bytes
70	* @handle: An opaque driver handle used in follow-up calls.
71	*
72	* Driver registers this routine to preallocate and initialize any
73	* internal data structures to bind the @qidx to its internal IO queues.
74	* A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
75	*
76	* Return value :
77	* 0 - Success
78	* -EINVAL - Unsupported input value.
79	* -ENOMEM - Could not alloc necessary memory
80	**/
81	static int
82	lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
83	unsigned int qidx, u16 qsize,
84	void **handle)
85	{
86	struct lpfc_nvme_lport *lport;
87	struct lpfc_vport *vport;
88	struct lpfc_nvme_qhandle *qhandle;
89	char *str;
90
91	if (!pnvme_lport->private)
92	return -ENOMEM;
93
94	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
95	vport = lport->vport;
96
97	if (!vport || vport->load_flag & FC_UNLOADING ||
98	vport->phba->hba_flag & HBA_IOQ_FLUSH)
99	return -ENODEV;
100
101	qhandle = kzalloc(size: sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
102	if (qhandle == NULL)
103	return -ENOMEM;
104
105	qhandle->cpu_id = raw_smp_processor_id();
106	qhandle->qidx = qidx;
107	/*
108	* NVME qidx == 0 is the admin queue, so both admin queue
109	* and first IO queue will use MSI-X vector and associated
110	* EQ/CQ/WQ at index 0. After that they are sequentially assigned.
111	*/
112	if (qidx) {
113	str = "IO "; /* IO queue */
114	qhandle->index = ((qidx - 1) %
115	lpfc_nvme_template.max_hw_queues);
116	} else {
117	str = "ADM"; /* Admin queue */
118	qhandle->index = qidx;
119	}
120
121	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
122	"6073 Binding %s HdwQueue %d (cpu %d) to "
123	"hdw_queue %d qhandle x%px\n", str,
124	qidx, qhandle->cpu_id, qhandle->index, qhandle);
125	*handle = (void *)qhandle;
126	return 0;
127	}
128
129	/**
130	* lpfc_nvme_delete_queue -
131	* @pnvme_lport: Transport localport that LS is to be issued from
132	* @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
133	* @handle: An opaque driver handle from lpfc_nvme_create_queue
134	*
135	* Driver registers this routine to free
136	* any internal data structures to bind the @qidx to its internal
137	* IO queues.
138	*
139	* Return value :
140	* 0 - Success
141	* TODO: What are the failure codes.
142	**/
143	static void
144	lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,
145	unsigned int qidx,
146	void *handle)
147	{
148	struct lpfc_nvme_lport *lport;
149	struct lpfc_vport *vport;
150
151	if (!pnvme_lport->private)
152	return;
153
154	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
155	vport = lport->vport;
156
157	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
158	"6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n",
159	lport, qidx, handle);
160	kfree(objp: handle);
161	}
162
163	static void
164	lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
165	{
166	struct lpfc_nvme_lport *lport = localport->private;
167
168	lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME,
169	"6173 localport x%px delete complete\n",
170	lport);
171
172	/* release any threads waiting for the unreg to complete */
173	if (lport->vport->localport)
174	complete(lport->lport_unreg_cmp);
175	}
176
177	/* lpfc_nvme_remoteport_delete
178	*
179	* @remoteport: Pointer to an nvme transport remoteport instance.
180	*
181	* This is a template downcall. NVME transport calls this function
182	* when it has completed the unregistration of a previously
183	* registered remoteport.
184	*
185	* Return value :
186	* None
187	*/
188	static void
189	lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
190	{
191	struct lpfc_nvme_rport *rport = remoteport->private;
192	struct lpfc_vport *vport;
193	struct lpfc_nodelist *ndlp;
194	u32 fc4_xpt_flags;
195
196	ndlp = rport->ndlp;
197	if (!ndlp) {
198	pr_err("**** %s: NULL ndlp on rport x%px remoteport x%px\n",
199	__func__, rport, remoteport);
200	goto rport_err;
201	}
202
203	vport = ndlp->vport;
204	if (!vport) {
205	pr_err("**** %s: Null vport on ndlp x%px, ste x%x rport x%px\n",
206	__func__, ndlp, ndlp->nlp_state, rport);
207	goto rport_err;
208	}
209
210	fc4_xpt_flags = NVME_XPT_REGD | SCSI_XPT_REGD;
211
212	/* Remove this rport from the lport's list - memory is owned by the
213	* transport. Remove the ndlp reference for the NVME transport before
214	* calling state machine to remove the node.
215	*/
216	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
217	"6146 remoteport delete of remoteport x%px, ndlp x%px "
218	"DID x%x xflags x%x\n",
219	remoteport, ndlp, ndlp->nlp_DID, ndlp->fc4_xpt_flags);
220	spin_lock_irq(lock: &ndlp->lock);
221
222	/* The register rebind might have occurred before the delete
223	* downcall. Guard against this race.
224	*/
225	if (ndlp->fc4_xpt_flags & NVME_XPT_UNREG_WAIT)
226	ndlp->fc4_xpt_flags &= ~(NVME_XPT_UNREG_WAIT | NVME_XPT_REGD);
227
228	spin_unlock_irq(lock: &ndlp->lock);
229
230	/* On a devloss timeout event, one more put is executed provided the
231	* NVME and SCSI rport unregister requests are complete.
232	*/
233	if (!(ndlp->fc4_xpt_flags & fc4_xpt_flags))
234	lpfc_disc_state_machine(vport, ndlp, NULL, NLP_EVT_DEVICE_RM);
235
236	rport_err:
237	return;
238	}
239
240	/**
241	* lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request
242	* @phba: pointer to lpfc hba data structure.
243	* @axchg: pointer to exchange context for the NVME LS request
244	*
245	* This routine is used for processing an asychronously received NVME LS
246	* request. Any remaining validation is done and the LS is then forwarded
247	* to the nvme-fc transport via nvme_fc_rcv_ls_req().
248	*
249	* The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing)
250	* -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done.
251	* __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg.
252	*
253	* Returns 0 if LS was handled and delivered to the transport
254	* Returns 1 if LS failed to be handled and should be dropped
255	*/
256	int
257	lpfc_nvme_handle_lsreq(struct lpfc_hba *phba,
258	struct lpfc_async_xchg_ctx *axchg)
259	{
260	#if (IS_ENABLED(CONFIG_NVME_FC))
261	struct lpfc_vport *vport;
262	struct lpfc_nvme_rport *lpfc_rport;
263	struct nvme_fc_remote_port *remoteport;
264	struct lpfc_nvme_lport *lport;
265	uint32_t *payload = axchg->payload;
266	int rc;
267
268	vport = axchg->ndlp->vport;
269	lpfc_rport = axchg->ndlp->nrport;
270	if (!lpfc_rport)
271	return -EINVAL;
272
273	remoteport = lpfc_rport->remoteport;
274	if (!vport->localport ||
275	vport->phba->hba_flag & HBA_IOQ_FLUSH)
276	return -EINVAL;
277
278	lport = vport->localport->private;
279	if (!lport)
280	return -EINVAL;
281
282	rc = nvme_fc_rcv_ls_req(remoteport, lsrsp: &axchg->ls_rsp, lsreqbuf: axchg->payload,
283	lsreqbuf_len: axchg->size);
284
285	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
286	"6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x "
287	"%08x %08x %08x\n",
288	axchg->size, rc,
289	*payload, *(payload+1), *(payload+2),
290	*(payload+3), *(payload+4), *(payload+5));
291
292	if (!rc)
293	return 0;
294	#endif
295	return 1;
296	}
297
298	/**
299	* __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME
300	* LS request.
301	* @phba: Pointer to HBA context object
302	* @vport: The local port that issued the LS
303	* @cmdwqe: Pointer to driver command WQE object.
304	* @wcqe: Pointer to driver response CQE object.
305	*
306	* This function is the generic completion handler for NVME LS requests.
307	* The function updates any states and statistics, calls the transport
308	* ls_req done() routine, then tears down the command and buffers used
309	* for the LS request.
310	**/
311	void
312	__lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport,
313	struct lpfc_iocbq *cmdwqe,
314	struct lpfc_wcqe_complete *wcqe)
315	{
316	struct nvmefc_ls_req *pnvme_lsreq;
317	struct lpfc_dmabuf *buf_ptr;
318	struct lpfc_nodelist *ndlp;
319	int status;
320
321	pnvme_lsreq = cmdwqe->context_un.nvme_lsreq;
322	ndlp = cmdwqe->ndlp;
323	buf_ptr = cmdwqe->bpl_dmabuf;
324
325	status = bf_get(lpfc_wcqe_c_status, wcqe);
326
327	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
328	"6047 NVMEx LS REQ x%px cmpl DID %x Xri: %x "
329	"status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px "
330	"ndlp:x%px\n",
331	pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
332	cmdwqe->sli4_xritag, status,
333	(wcqe->parameter & 0xffff),
334	cmdwqe, pnvme_lsreq, cmdwqe->bpl_dmabuf,
335	ndlp);
336
337	lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n",
338	cmdwqe->sli4_xritag, status, wcqe->parameter);
339
340	if (buf_ptr) {
341	lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
342	kfree(objp: buf_ptr);
343	cmdwqe->bpl_dmabuf = NULL;
344	}
345	if (pnvme_lsreq->done) {
346	if (status != CQE_STATUS_SUCCESS)
347	status = -ENXIO;
348	pnvme_lsreq->done(pnvme_lsreq, status);
349	} else {
350	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
351	"6046 NVMEx cmpl without done call back? "
352	"Data x%px DID %x Xri: %x status %x\n",
353	pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
354	cmdwqe->sli4_xritag, status);
355	}
356	if (ndlp) {
357	lpfc_nlp_put(ndlp);
358	cmdwqe->ndlp = NULL;
359	}
360	lpfc_sli_release_iocbq(phba, cmdwqe);
361	}
362
363	static void
364	lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
365	struct lpfc_iocbq *rspwqe)
366	{
367	struct lpfc_vport *vport = cmdwqe->vport;
368	struct lpfc_nvme_lport *lport;
369	uint32_t status;
370	struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
371
372	status = bf_get(lpfc_wcqe_c_status, wcqe);
373
374	if (vport->localport) {
375	lport = (struct lpfc_nvme_lport *)vport->localport->private;
376	if (lport) {
377	atomic_inc(v: &lport->fc4NvmeLsCmpls);
378	if (status) {
379	if (bf_get(lpfc_wcqe_c_xb, wcqe))
380	atomic_inc(v: &lport->cmpl_ls_xb);
381	atomic_inc(v: &lport->cmpl_ls_err);
382	}
383	}
384	}
385
386	__lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe);
387	}
388
389	static int
390	lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
391	struct lpfc_dmabuf *inp,
392	struct nvmefc_ls_req *pnvme_lsreq,
393	void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
394	struct lpfc_iocbq *),
395	struct lpfc_nodelist *ndlp, uint32_t num_entry,
396	uint32_t tmo, uint8_t retry)
397	{
398	struct lpfc_hba *phba = vport->phba;
399	union lpfc_wqe128 *wqe;
400	struct lpfc_iocbq *genwqe;
401	struct ulp_bde64 *bpl;
402	struct ulp_bde64 bde;
403	int i, rc, xmit_len, first_len;
404
405	/* Allocate buffer for command WQE */
406	genwqe = lpfc_sli_get_iocbq(phba);
407	if (genwqe == NULL)
408	return 1;
409
410	wqe = &genwqe->wqe;
411	/* Initialize only 64 bytes */
412	memset(wqe, 0, sizeof(union lpfc_wqe));
413
414	genwqe->bpl_dmabuf = bmp;
415	genwqe->cmd_flag |= LPFC_IO_NVME_LS;
416
417	/* Save for completion so we can release these resources */
418	genwqe->ndlp = lpfc_nlp_get(ndlp);
419	if (!genwqe->ndlp) {
420	dev_warn(&phba->pcidev->dev,
421	"Warning: Failed node ref, not sending LS_REQ\n");
422	lpfc_sli_release_iocbq(phba, genwqe);
423	return 1;
424	}
425
426	genwqe->context_un.nvme_lsreq = pnvme_lsreq;
427	/* Fill in payload, bp points to frame payload */
428
429	if (!tmo)
430	/* FC spec states we need 3 * ratov for CT requests */
431	tmo = (3 * phba->fc_ratov);
432
433	/* For this command calculate the xmit length of the request bde. */
434	xmit_len = 0;
435	first_len = 0;
436	bpl = (struct ulp_bde64 *)bmp->virt;
437	for (i = 0; i < num_entry; i++) {
438	bde.tus.w = bpl[i].tus.w;
439	if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
440	break;
441	xmit_len += bde.tus.f.bdeSize;
442	if (i == 0)
443	first_len = xmit_len;
444	}
445
446	genwqe->num_bdes = num_entry;
447	genwqe->hba_wqidx = 0;
448
449	/* Words 0 - 2 */
450	wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
451	wqe->generic.bde.tus.f.bdeSize = first_len;
452	wqe->generic.bde.addrLow = bpl[0].addrLow;
453	wqe->generic.bde.addrHigh = bpl[0].addrHigh;
454
455	/* Word 3 */
456	wqe->gen_req.request_payload_len = first_len;
457
458	/* Word 4 */
459
460	/* Word 5 */
461	bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);
462	bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);
463	bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);
464	bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ);
465	bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);
466
467	/* Word 6 */
468	bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,
469	phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
470	bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);
471
472	/* Word 7 */
473	bf_set(wqe_tmo, &wqe->gen_req.wqe_com, tmo);
474	bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);
475	bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);
476	bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);
477
478	/* Word 8 */
479	wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;
480
481	/* Word 9 */
482	bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);
483
484	/* Word 10 */
485	bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
486	bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
487	bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
488	bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
489	bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
490
491	/* Word 11 */
492	bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
493	bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);
494
495
496	/* Issue GEN REQ WQE for NPORT <did> */
497	genwqe->cmd_cmpl = cmpl;
498	genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;
499	genwqe->vport = vport;
500	genwqe->retry = retry;
501
502	lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n",
503	genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
504
505	rc = lpfc_sli4_issue_wqe(phba, qp: &phba->sli4_hba.hdwq[0], pwqe: genwqe);
506	if (rc) {
507	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
508	"6045 Issue GEN REQ WQE to NPORT x%x "
509	"Data: x%x x%x rc x%x\n",
510	ndlp->nlp_DID, genwqe->iotag,
511	vport->port_state, rc);
512	lpfc_nlp_put(ndlp);
513	lpfc_sli_release_iocbq(phba, genwqe);
514	return 1;
515	}
516
517	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS,
518	"6050 Issue GEN REQ WQE to NPORT x%x "
519	"Data: oxid: x%x state: x%x wq:x%px lsreq:x%px "
520	"bmp:x%px xmit:%d 1st:%d\n",
521	ndlp->nlp_DID, genwqe->sli4_xritag,
522	vport->port_state,
523	genwqe, pnvme_lsreq, bmp, xmit_len, first_len);
524	return 0;
525	}
526
527
528	/**
529	* __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request
530	* @vport: The local port issuing the LS
531	* @ndlp: The remote port to send the LS to
532	* @pnvme_lsreq: Pointer to LS request structure from the transport
533	* @gen_req_cmp: Completion call-back
534	*
535	* Routine validates the ndlp, builds buffers and sends a GEN_REQUEST
536	* WQE to perform the LS operation.
537	*
538	* Return value :
539	* 0 - Success
540	* non-zero: various error codes, in form of -Exxx
541	**/
542	int
543	__lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
544	struct nvmefc_ls_req *pnvme_lsreq,
545	void (*gen_req_cmp)(struct lpfc_hba *phba,
546	struct lpfc_iocbq *cmdwqe,
547	struct lpfc_iocbq *rspwqe))
548	{
549	struct lpfc_dmabuf *bmp;
550	struct ulp_bde64 *bpl;
551	int ret;
552	uint16_t ntype, nstate;
553
554	if (!ndlp) {
555	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
556	"6051 NVMEx LS REQ: Bad NDLP x%px, Failing "
557	"LS Req\n",
558	ndlp);
559	return -ENODEV;
560	}
561
562	ntype = ndlp->nlp_type;
563	nstate = ndlp->nlp_state;
564	if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||
565	(ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {
566	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
567	"6088 NVMEx LS REQ: Fail DID x%06x not "
568	"ready for IO. Type x%x, State x%x\n",
569	ndlp->nlp_DID, ntype, nstate);
570	return -ENODEV;
571	}
572	if (vport->phba->hba_flag & HBA_IOQ_FLUSH)
573	return -ENODEV;
574
575	if (!vport->phba->sli4_hba.nvmels_wq)
576	return -ENOMEM;
577
578	/*
579	* there are two dma buf in the request, actually there is one and
580	* the second one is just the start address + cmd size.
581	* Before calling lpfc_nvme_gen_req these buffers need to be wrapped
582	* in a lpfc_dmabuf struct. When freeing we just free the wrapper
583	* because the nvem layer owns the data bufs.
584	* We do not have to break these packets open, we don't care what is
585	* in them. And we do not have to look at the resonse data, we only
586	* care that we got a response. All of the caring is going to happen
587	* in the nvme-fc layer.
588	*/
589
590	bmp = kmalloc(size: sizeof(*bmp), GFP_KERNEL);
591	if (!bmp) {
592	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
593	"6044 NVMEx LS REQ: Could not alloc LS buf "
594	"for DID %x\n",
595	ndlp->nlp_DID);
596	return -ENOMEM;
597	}
598
599	bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
600	if (!bmp->virt) {
601	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
602	"6042 NVMEx LS REQ: Could not alloc mbuf "
603	"for DID %x\n",
604	ndlp->nlp_DID);
605	kfree(objp: bmp);
606	return -ENOMEM;
607	}
608
609	INIT_LIST_HEAD(list: &bmp->list);
610
611	bpl = (struct ulp_bde64 *)bmp->virt;
612	bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));
613	bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));
614	bpl->tus.f.bdeFlags = 0;
615	bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;
616	bpl->tus.w = le32_to_cpu(bpl->tus.w);
617	bpl++;
618
619	bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));
620	bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));
621	bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
622	bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;
623	bpl->tus.w = le32_to_cpu(bpl->tus.w);
624
625	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
626	"6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, "
627	"rqstlen:%d rsplen:%d %pad %pad\n",
628	ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen,
629	pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
630	&pnvme_lsreq->rspdma);
631
632	ret = lpfc_nvme_gen_req(vport, bmp, inp: pnvme_lsreq->rqstaddr,
633	pnvme_lsreq, cmpl: gen_req_cmp, ndlp, num_entry: 2,
634	tmo: pnvme_lsreq->timeout, retry: 0);
635	if (ret != WQE_SUCCESS) {
636	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
637	"6052 NVMEx REQ: EXIT. issue ls wqe failed "
638	"lsreq x%px Status %x DID %x\n",
639	pnvme_lsreq, ret, ndlp->nlp_DID);
640	lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);
641	kfree(objp: bmp);
642	return -EIO;
643	}
644
645	return 0;
646	}
647
648	/**
649	* lpfc_nvme_ls_req - Issue an NVME Link Service request
650	* @pnvme_lport: Transport localport that LS is to be issued from.
651	* @pnvme_rport: Transport remoteport that LS is to be sent to.
652	* @pnvme_lsreq: the transport nvme_ls_req structure for the LS
653	*
654	* Driver registers this routine to handle any link service request
655	* from the nvme_fc transport to a remote nvme-aware port.
656	*
657	* Return value :
658	* 0 - Success
659	* non-zero: various error codes, in form of -Exxx
660	**/
661	static int
662	lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
663	struct nvme_fc_remote_port *pnvme_rport,
664	struct nvmefc_ls_req *pnvme_lsreq)
665	{
666	struct lpfc_nvme_lport *lport;
667	struct lpfc_nvme_rport *rport;
668	struct lpfc_vport *vport;
669	int ret;
670
671	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
672	rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
673	if (unlikely(!lport) || unlikely(!rport))
674	return -EINVAL;
675
676	vport = lport->vport;
677	if (vport->load_flag & FC_UNLOADING ||
678	vport->phba->hba_flag & HBA_IOQ_FLUSH)
679	return -ENODEV;
680
681	atomic_inc(v: &lport->fc4NvmeLsRequests);
682
683	ret = __lpfc_nvme_ls_req(vport, ndlp: rport->ndlp, pnvme_lsreq,
684	gen_req_cmp: lpfc_nvme_ls_req_cmp);
685	if (ret)
686	atomic_inc(v: &lport->xmt_ls_err);
687
688	return ret;
689	}
690
691	/**
692	* __lpfc_nvme_ls_abort - Generic service routine to abort a prior
693	* NVME LS request
694	* @vport: The local port that issued the LS
695	* @ndlp: The remote port the LS was sent to
696	* @pnvme_lsreq: Pointer to LS request structure from the transport
697	*
698	* The driver validates the ndlp, looks for the LS, and aborts the
699	* LS if found.
700	*
701	* Returns:
702	* 0 : if LS found and aborted
703	* non-zero: various error conditions in form -Exxx
704	**/
705	int
706	__lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
707	struct nvmefc_ls_req *pnvme_lsreq)
708	{
709	struct lpfc_hba *phba = vport->phba;
710	struct lpfc_sli_ring *pring;
711	struct lpfc_iocbq *wqe, *next_wqe;
712	bool foundit = false;
713
714	if (!ndlp) {
715	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
716	"6049 NVMEx LS REQ Abort: Bad NDLP x%px DID "
717	"x%06x, Failing LS Req\n",
718	ndlp, ndlp ? ndlp->nlp_DID : 0);
719	return -EINVAL;
720	}
721
722	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
723	"6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq "
724	"x%px rqstlen:%d rsplen:%d %pad %pad\n",
725	pnvme_lsreq, pnvme_lsreq->rqstlen,
726	pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
727	&pnvme_lsreq->rspdma);
728
729	/*
730	* Lock the ELS ring txcmplq and look for the wqe that matches
731	* this ELS. If found, issue an abort on the wqe.
732	*/
733	pring = phba->sli4_hba.nvmels_wq->pring;
734	spin_lock_irq(lock: &phba->hbalock);
735	spin_lock(lock: &pring->ring_lock);
736	list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {
737	if (wqe->context_un.nvme_lsreq == pnvme_lsreq) {
738	wqe->cmd_flag |= LPFC_DRIVER_ABORTED;
739	foundit = true;
740	break;
741	}
742	}
743	spin_unlock(lock: &pring->ring_lock);
744
745	if (foundit)
746	lpfc_sli_issue_abort_iotag(phba, pring, wqe, NULL);
747	spin_unlock_irq(lock: &phba->hbalock);
748
749	if (foundit)
750	return 0;
751
752	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
753	"6213 NVMEx LS REQ Abort: Unable to locate req x%px\n",
754	pnvme_lsreq);
755	return -EINVAL;
756	}
757
758	static int
759	lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport,
760	struct nvme_fc_remote_port *remoteport,
761	struct nvmefc_ls_rsp *ls_rsp)
762	{
763	struct lpfc_async_xchg_ctx *axchg =
764	container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);
765	struct lpfc_nvme_lport *lport;
766	int rc;
767
768	if (axchg->phba->pport->load_flag & FC_UNLOADING)
769	return -ENODEV;
770
771	lport = (struct lpfc_nvme_lport *)localport->private;
772
773	rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, xmt_ls_rsp_cmp: __lpfc_nvme_xmt_ls_rsp_cmp);
774
775	if (rc) {
776	/*
777	* unless the failure is due to having already sent
778	* the response, an abort will be generated for the
779	* exchange if the rsp can't be sent.
780	*/
781	if (rc != -EALREADY)
782	atomic_inc(v: &lport->xmt_ls_abort);
783	return rc;
784	}
785
786	return 0;
787	}
788
789	/**
790	* lpfc_nvme_ls_abort - Abort a prior NVME LS request
791	* @pnvme_lport: Transport localport that LS is to be issued from.
792	* @pnvme_rport: Transport remoteport that LS is to be sent to.
793	* @pnvme_lsreq: the transport nvme_ls_req structure for the LS
794	*
795	* Driver registers this routine to abort a NVME LS request that is
796	* in progress (from the transports perspective).
797	**/
798	static void
799	lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
800	struct nvme_fc_remote_port *pnvme_rport,
801	struct nvmefc_ls_req *pnvme_lsreq)
802	{
803	struct lpfc_nvme_lport *lport;
804	struct lpfc_vport *vport;
805	struct lpfc_nodelist *ndlp;
806	int ret;
807
808	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
809	if (unlikely(!lport))
810	return;
811	vport = lport->vport;
812
813	if (vport->load_flag & FC_UNLOADING)
814	return;
815
816	ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
817
818	ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq);
819	if (!ret)
820	atomic_inc(v: &lport->xmt_ls_abort);
821	}
822
823	/* Fix up the existing sgls for NVME IO. */
824	static inline void
825	lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
826	struct lpfc_io_buf *lpfc_ncmd,
827	struct nvmefc_fcp_req *nCmd)
828	{
829	struct lpfc_hba *phba = vport->phba;
830	struct sli4_sge *sgl;
831	union lpfc_wqe128 *wqe;
832	uint32_t *wptr, *dptr;
833
834	/*
835	* Get a local pointer to the built-in wqe and correct
836	* the cmd size to match NVME's 96 bytes and fix
837	* the dma address.
838	*/
839
840	wqe = &lpfc_ncmd->cur_iocbq.wqe;
841
842	/*
843	* Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
844	* match NVME. NVME sends 96 bytes. Also, use the
845	* nvme commands command and response dma addresses
846	* rather than the virtual memory to ease the restore
847	* operation.
848	*/
849	sgl = lpfc_ncmd->dma_sgl;
850	sgl->sge_len = cpu_to_le32(nCmd->cmdlen);
851	if (phba->cfg_nvme_embed_cmd) {
852	sgl->addr_hi = 0;
853	sgl->addr_lo = 0;
854
855	/* Word 0-2 - NVME CMND IU (embedded payload) */
856	wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;
857	wqe->generic.bde.tus.f.bdeSize = 56;
858	wqe->generic.bde.addrHigh = 0;
859	wqe->generic.bde.addrLow = 64; /* Word 16 */
860
861	/* Word 10 - dbde is 0, wqes is 1 in template */
862
863	/*
864	* Embed the payload in the last half of the WQE
865	* WQE words 16-30 get the NVME CMD IU payload
866	*
867	* WQE words 16-19 get payload Words 1-4
868	* WQE words 20-21 get payload Words 6-7
869	* WQE words 22-29 get payload Words 16-23
870	*/
871	wptr = &wqe->words[16]; /* WQE ptr */
872	dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */
873	dptr++; /* Skip Word 0 in payload */
874
875	*wptr++ = *dptr++; /* Word 1 */
876	*wptr++ = *dptr++; /* Word 2 */
877	*wptr++ = *dptr++; /* Word 3 */
878	*wptr++ = *dptr++; /* Word 4 */
879	dptr++; /* Skip Word 5 in payload */
880	*wptr++ = *dptr++; /* Word 6 */
881	*wptr++ = *dptr++; /* Word 7 */
882	dptr += 8; /* Skip Words 8-15 in payload */
883	*wptr++ = *dptr++; /* Word 16 */
884	*wptr++ = *dptr++; /* Word 17 */
885	*wptr++ = *dptr++; /* Word 18 */
886	*wptr++ = *dptr++; /* Word 19 */
887	*wptr++ = *dptr++; /* Word 20 */
888	*wptr++ = *dptr++; /* Word 21 */
889	*wptr++ = *dptr++; /* Word 22 */
890	*wptr = *dptr; /* Word 23 */
891	} else {
892	sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
893	sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
894
895	/* Word 0-2 - NVME CMND IU Inline BDE */
896	wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
897	wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
898	wqe->generic.bde.addrHigh = sgl->addr_hi;
899	wqe->generic.bde.addrLow = sgl->addr_lo;
900
901	/* Word 10 */
902	bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
903	bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
904	}
905
906	sgl++;
907
908	/* Setup the physical region for the FCP RSP */
909	sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
910	sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
911	sgl->word2 = le32_to_cpu(sgl->word2);
912	if (nCmd->sg_cnt)
913	bf_set(lpfc_sli4_sge_last, sgl, 0);
914	else
915	bf_set(lpfc_sli4_sge_last, sgl, 1);
916	sgl->word2 = cpu_to_le32(sgl->word2);
917	sgl->sge_len = cpu_to_le32(nCmd->rsplen);
918	}
919
920
921	/*
922	* lpfc_nvme_io_cmd_cmpl - Complete an NVME-over-FCP IO
923	*
924	* Driver registers this routine as it io request handler. This
925	* routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
926	* data structure to the rport indicated in @lpfc_nvme_rport.
927	*
928	* Return value :
929	* 0 - Success
930	* TODO: What are the failure codes.
931	**/
932	static void
933	lpfc_nvme_io_cmd_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
934	struct lpfc_iocbq *pwqeOut)
935	{
936	struct lpfc_io_buf *lpfc_ncmd = pwqeIn->io_buf;
937	struct lpfc_wcqe_complete *wcqe = &pwqeOut->wcqe_cmpl;
938	struct lpfc_vport *vport = pwqeIn->vport;
939	struct nvmefc_fcp_req *nCmd;
940	struct nvme_fc_ersp_iu *ep;
941	struct nvme_fc_cmd_iu *cp;
942	struct lpfc_nodelist *ndlp;
943	struct lpfc_nvme_fcpreq_priv *freqpriv;
944	struct lpfc_nvme_lport *lport;
945	uint32_t code, status, idx;
946	uint16_t cid, sqhd, data;
947	uint32_t *ptr;
948	uint32_t lat;
949	bool call_done = false;
950	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
951	int cpu;
952	#endif
953	bool offline = false;
954
955	/* Sanity check on return of outstanding command */
956	if (!lpfc_ncmd) {
957	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
958	"6071 Null lpfc_ncmd pointer. No "
959	"release, skip completion\n");
960	return;
961	}
962
963	/* Guard against abort handler being called at same time */
964	spin_lock(lock: &lpfc_ncmd->buf_lock);
965
966	if (!lpfc_ncmd->nvmeCmd) {
967	spin_unlock(lock: &lpfc_ncmd->buf_lock);
968	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
969	"6066 Missing cmpl ptrs: lpfc_ncmd x%px, "
970	"nvmeCmd x%px\n",
971	lpfc_ncmd, lpfc_ncmd->nvmeCmd);
972
973	/* Release the lpfc_ncmd regardless of the missing elements. */
974	lpfc_release_nvme_buf(phba, lpfc_ncmd);
975	return;
976	}
977	nCmd = lpfc_ncmd->nvmeCmd;
978	status = bf_get(lpfc_wcqe_c_status, wcqe);
979
980	idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
981	phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++;
982
983	if (unlikely(status && vport->localport)) {
984	lport = (struct lpfc_nvme_lport *)vport->localport->private;
985	if (lport) {
986	if (bf_get(lpfc_wcqe_c_xb, wcqe))
987	atomic_inc(v: &lport->cmpl_fcp_xb);
988	atomic_inc(v: &lport->cmpl_fcp_err);
989	}
990	}
991
992	lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
993	lpfc_ncmd->cur_iocbq.sli4_xritag,
994	status, wcqe->parameter);
995	/*
996	* Catch race where our node has transitioned, but the
997	* transport is still transitioning.
998	*/
999	ndlp = lpfc_ncmd->ndlp;
1000	if (!ndlp) {
1001	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1002	"6062 Ignoring NVME cmpl. No ndlp\n");
1003	goto out_err;
1004	}
1005
1006	code = bf_get(lpfc_wcqe_c_code, wcqe);
1007	if (code == CQE_CODE_NVME_ERSP) {
1008	/* For this type of CQE, we need to rebuild the rsp */
1009	ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1010
1011	/*
1012	* Get Command Id from cmd to plug into response. This
1013	* code is not needed in the next NVME Transport drop.
1014	*/
1015	cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1016	cid = cp->sqe.common.command_id;
1017
1018	/*
1019	* RSN is in CQE word 2
1020	* SQHD is in CQE Word 3 bits 15:0
1021	* Cmd Specific info is in CQE Word 1
1022	* and in CQE Word 0 bits 15:0
1023	*/
1024	sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);
1025
1026	/* Now lets build the NVME ERSP IU */
1027	ep->iu_len = cpu_to_be16(8);
1028	ep->rsn = wcqe->parameter;
1029	ep->xfrd_len = cpu_to_be32(nCmd->payload_length);
1030	ep->rsvd12 = 0;
1031	ptr = (uint32_t *)&ep->cqe.result.u64;
1032	*ptr++ = wcqe->total_data_placed;
1033	data = bf_get(lpfc_wcqe_c_ersp0, wcqe);
1034	*ptr = (uint32_t)data;
1035	ep->cqe.sq_head = sqhd;
1036	ep->cqe.sq_id = nCmd->sqid;
1037	ep->cqe.command_id = cid;
1038	ep->cqe.status = 0;
1039
1040	lpfc_ncmd->status = IOSTAT_SUCCESS;
1041	lpfc_ncmd->result = 0;
1042	nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;
1043	nCmd->transferred_length = nCmd->payload_length;
1044	} else {
1045	lpfc_ncmd->status = status;
1046	lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
1047
1048	/* For NVME, the only failure path that results in an
1049	* IO error is when the adapter rejects it. All other
1050	* conditions are a success case and resolved by the
1051	* transport.
1052	* IOSTAT_FCP_RSP_ERROR means:
1053	* 1. Length of data received doesn't match total
1054	* transfer length in WQE
1055	* 2. If the RSP payload does NOT match these cases:
1056	* a. RSP length 12/24 bytes and all zeros
1057	* b. NVME ERSP
1058	*/
1059	switch (lpfc_ncmd->status) {
1060	case IOSTAT_SUCCESS:
1061	nCmd->transferred_length = wcqe->total_data_placed;
1062	nCmd->rcv_rsplen = 0;
1063	nCmd->status = 0;
1064	break;
1065	case IOSTAT_FCP_RSP_ERROR:
1066	nCmd->transferred_length = wcqe->total_data_placed;
1067	nCmd->rcv_rsplen = wcqe->parameter;
1068	nCmd->status = 0;
1069
1070	/* Get the NVME cmd details for this unique error. */
1071	cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1072	ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1073
1074	/* Check if this is really an ERSP */
1075	if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) {
1076	lpfc_ncmd->status = IOSTAT_SUCCESS;
1077	lpfc_ncmd->result = 0;
1078
1079	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
1080	"6084 NVME FCP_ERR ERSP: "
1081	"xri %x placed x%x opcode x%x cmd_id "
1082	"x%x cqe_status x%x\n",
1083	lpfc_ncmd->cur_iocbq.sli4_xritag,
1084	wcqe->total_data_placed,
1085	cp->sqe.common.opcode,
1086	cp->sqe.common.command_id,
1087	ep->cqe.status);
1088	break;
1089	}
1090	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1091	"6081 NVME Completion Protocol Error: "
1092	"xri %x status x%x result x%x "
1093	"placed x%x opcode x%x cmd_id x%x, "
1094	"cqe_status x%x\n",
1095	lpfc_ncmd->cur_iocbq.sli4_xritag,
1096	lpfc_ncmd->status, lpfc_ncmd->result,
1097	wcqe->total_data_placed,
1098	cp->sqe.common.opcode,
1099	cp->sqe.common.command_id,
1100	ep->cqe.status);
1101	break;
1102	case IOSTAT_LOCAL_REJECT:
1103	/* Let fall through to set command final state. */
1104	if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)
1105	lpfc_printf_vlog(vport, KERN_INFO,
1106	LOG_NVME_IOERR,
1107	"6032 Delay Aborted cmd x%px "
1108	"nvme cmd x%px, xri x%x, "
1109	"xb %d\n",
1110	lpfc_ncmd, nCmd,
1111	lpfc_ncmd->cur_iocbq.sli4_xritag,
1112	bf_get(lpfc_wcqe_c_xb, wcqe));
1113	fallthrough;
1114	default:
1115	out_err:
1116	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1117	"6072 NVME Completion Error: xri %x "
1118	"status x%x result x%x [x%x] "
1119	"placed x%x\n",
1120	lpfc_ncmd->cur_iocbq.sli4_xritag,
1121	lpfc_ncmd->status, lpfc_ncmd->result,
1122	wcqe->parameter,
1123	wcqe->total_data_placed);
1124	nCmd->transferred_length = 0;
1125	nCmd->rcv_rsplen = 0;
1126	nCmd->status = NVME_SC_INTERNAL;
1127	if (pci_channel_offline(pdev: vport->phba->pcidev) ||
1128	lpfc_ncmd->result == IOERR_SLI_DOWN)
1129	offline = true;
1130	}
1131	}
1132
1133	/* pick up SLI4 exhange busy condition */
1134	if (bf_get(lpfc_wcqe_c_xb, wcqe) && !offline)
1135	lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;
1136	else
1137	lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;
1138
1139	/* Update stats and complete the IO. There is
1140	* no need for dma unprep because the nvme_transport
1141	* owns the dma address.
1142	*/
1143	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1144	if (lpfc_ncmd->ts_cmd_start) {
1145	lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;
1146	lpfc_ncmd->ts_data_io = ktime_get_ns();
1147	phba->ktime_last_cmd = lpfc_ncmd->ts_data_io;
1148	lpfc_io_ktime(phba, ncmd: lpfc_ncmd);
1149	}
1150	if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) {
1151	cpu = raw_smp_processor_id();
1152	this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
1153	if (lpfc_ncmd->cpu != cpu)
1154	lpfc_printf_vlog(vport,
1155	KERN_INFO, LOG_NVME_IOERR,
1156	"6701 CPU Check cmpl: "
1157	"cpu %d expect %d\n",
1158	cpu, lpfc_ncmd->cpu);
1159	}
1160	#endif
1161
1162	/* NVME targets need completion held off until the abort exchange
1163	* completes unless the NVME Rport is getting unregistered.
1164	*/
1165
1166	if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
1167	freqpriv = nCmd->private;
1168	freqpriv->nvme_buf = NULL;
1169	lpfc_ncmd->nvmeCmd = NULL;
1170	call_done = true;
1171	}
1172	spin_unlock(lock: &lpfc_ncmd->buf_lock);
1173
1174	/* Check if IO qualified for CMF */
1175	if (phba->cmf_active_mode != LPFC_CFG_OFF &&
1176	nCmd->io_dir == NVMEFC_FCP_READ &&
1177	nCmd->payload_length) {
1178	/* Used when calculating average latency */
1179	lat = ktime_get_ns() - lpfc_ncmd->rx_cmd_start;
1180	lpfc_update_cmf_cmpl(phba, val: lat, sz: nCmd->payload_length, NULL);
1181	}
1182
1183	if (call_done)
1184	nCmd->done(nCmd);
1185
1186	/* Call release with XB=1 to queue the IO into the abort list. */
1187	lpfc_release_nvme_buf(phba, lpfc_ncmd);
1188	}
1189
1190
1191	/**
1192	* lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
1193	* @vport: pointer to a host virtual N_Port data structure
1194	* @lpfc_ncmd: Pointer to lpfc scsi command
1195	* @pnode: pointer to a node-list data structure
1196	* @cstat: pointer to the control status structure
1197	*
1198	* Driver registers this routine as it io request handler. This
1199	* routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1200	* data structure to the rport indicated in @lpfc_nvme_rport.
1201	*
1202	* Return value :
1203	* 0 - Success
1204	* TODO: What are the failure codes.
1205	**/
1206	static int
1207	lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
1208	struct lpfc_io_buf *lpfc_ncmd,
1209	struct lpfc_nodelist *pnode,
1210	struct lpfc_fc4_ctrl_stat *cstat)
1211	{
1212	struct lpfc_hba *phba = vport->phba;
1213	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1214	struct nvme_common_command *sqe;
1215	struct lpfc_iocbq *pwqeq = &lpfc_ncmd->cur_iocbq;
1216	union lpfc_wqe128 *wqe = &pwqeq->wqe;
1217	uint32_t req_len;
1218
1219	/*
1220	* There are three possibilities here - use scatter-gather segment, use
1221	* the single mapping, or neither.
1222	*/
1223	if (nCmd->sg_cnt) {
1224	if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
1225	/* From the iwrite template, initialize words 7 - 11 */
1226	memcpy(&wqe->words[7],
1227	&lpfc_iwrite_cmd_template.words[7],
1228	sizeof(uint32_t) * 5);
1229
1230	/* Word 4 */
1231	wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;
1232
1233	/* Word 5 */
1234	if ((phba->cfg_nvme_enable_fb) &&
1235	(pnode->nlp_flag & NLP_FIRSTBURST)) {
1236	req_len = lpfc_ncmd->nvmeCmd->payload_length;
1237	if (req_len < pnode->nvme_fb_size)
1238	wqe->fcp_iwrite.initial_xfer_len =
1239	req_len;
1240	else
1241	wqe->fcp_iwrite.initial_xfer_len =
1242	pnode->nvme_fb_size;
1243	} else {
1244	wqe->fcp_iwrite.initial_xfer_len = 0;
1245	}
1246	cstat->output_requests++;
1247	} else {
1248	/* From the iread template, initialize words 7 - 11 */
1249	memcpy(&wqe->words[7],
1250	&lpfc_iread_cmd_template.words[7],
1251	sizeof(uint32_t) * 5);
1252
1253	/* Word 4 */
1254	wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
1255
1256	/* Word 5 */
1257	wqe->fcp_iread.rsrvd5 = 0;
1258
1259	/* For a CMF Managed port, iod must be zero'ed */
1260	if (phba->cmf_active_mode == LPFC_CFG_MANAGED)
1261	bf_set(wqe_iod, &wqe->fcp_iread.wqe_com,
1262	LPFC_WQE_IOD_NONE);
1263	cstat->input_requests++;
1264	}
1265	} else {
1266	/* From the icmnd template, initialize words 4 - 11 */
1267	memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
1268	sizeof(uint32_t) * 8);
1269	cstat->control_requests++;
1270	}
1271
1272	if (pnode->nlp_nvme_info & NLP_NVME_NSLER) {
1273	bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
1274	sqe = &((struct nvme_fc_cmd_iu *)
1275	nCmd->cmdaddr)->sqe.common;
1276	if (sqe->opcode == nvme_admin_async_event)
1277	bf_set(wqe_ffrq, &wqe->generic.wqe_com, 1);
1278	}
1279
1280	/*
1281	* Finish initializing those WQE fields that are independent
1282	* of the nvme_cmnd request_buffer
1283	*/
1284
1285	/* Word 3 */
1286	bf_set(payload_offset_len, &wqe->fcp_icmd,
1287	(nCmd->rsplen + nCmd->cmdlen));
1288
1289	/* Word 6 */
1290	bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
1291	phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
1292	bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
1293
1294	/* Word 8 */
1295	wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
1296
1297	/* Word 9 */
1298	bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
1299
1300	/* Word 10 */
1301	bf_set(wqe_xchg, &wqe->fcp_iwrite.wqe_com, LPFC_NVME_XCHG);
1302
1303	/* Words 13 14 15 are for PBDE support */
1304
1305	/* add the VMID tags as per switch response */
1306	if (unlikely(lpfc_ncmd->cur_iocbq.cmd_flag & LPFC_IO_VMID)) {
1307	if (phba->pport->vmid_priority_tagging) {
1308	bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1);
1309	bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com,
1310	lpfc_ncmd->cur_iocbq.vmid_tag.cs_ctl_vmid);
1311	} else {
1312	bf_set(wqe_appid, &wqe->fcp_iwrite.wqe_com, 1);
1313	bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
1314	wqe->words[31] = lpfc_ncmd->cur_iocbq.vmid_tag.app_id;
1315	}
1316	}
1317
1318	pwqeq->vport = vport;
1319	return 0;
1320	}
1321
1322
1323	/**
1324	* lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
1325	* @vport: pointer to a host virtual N_Port data structure
1326	* @lpfc_ncmd: Pointer to lpfc scsi command
1327	*
1328	* Driver registers this routine as it io request handler. This
1329	* routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1330	* data structure to the rport indicated in @lpfc_nvme_rport.
1331	*
1332	* Return value :
1333	* 0 - Success
1334	* TODO: What are the failure codes.
1335	**/
1336	static int
1337	lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
1338	struct lpfc_io_buf *lpfc_ncmd)
1339	{
1340	struct lpfc_hba *phba = vport->phba;
1341	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1342	union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;
1343	struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;
1344	struct sli4_hybrid_sgl *sgl_xtra = NULL;
1345	struct scatterlist *data_sg;
1346	struct sli4_sge *first_data_sgl;
1347	struct ulp_bde64 *bde;
1348	dma_addr_t physaddr = 0;
1349	uint32_t dma_len = 0;
1350	uint32_t dma_offset = 0;
1351	int nseg, i, j;
1352	bool lsp_just_set = false;
1353
1354	/* Fix up the command and response DMA stuff. */
1355	lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);
1356
1357	/*
1358	* There are three possibilities here - use scatter-gather segment, use
1359	* the single mapping, or neither.
1360	*/
1361	if (nCmd->sg_cnt) {
1362	/*
1363	* Jump over the cmd and rsp SGEs. The fix routine
1364	* has already adjusted for this.
1365	*/
1366	sgl += 2;
1367
1368	first_data_sgl = sgl;
1369	lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
1370	if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {
1371	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1372	"6058 Too many sg segments from "
1373	"NVME Transport. Max %d, "
1374	"nvmeIO sg_cnt %d\n",
1375	phba->cfg_nvme_seg_cnt + 1,
1376	lpfc_ncmd->seg_cnt);
1377	lpfc_ncmd->seg_cnt = 0;
1378	return 1;
1379	}
1380
1381	/*
1382	* The driver established a maximum scatter-gather segment count
1383	* during probe that limits the number of sg elements in any
1384	* single nvme command. Just run through the seg_cnt and format
1385	* the sge's.
1386	*/
1387	nseg = nCmd->sg_cnt;
1388	data_sg = nCmd->first_sgl;
1389
1390	/* for tracking the segment boundaries */
1391	j = 2;
1392	for (i = 0; i < nseg; i++) {
1393	if (data_sg == NULL) {
1394	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1395	"6059 dptr err %d, nseg %d\n",
1396	i, nseg);
1397	lpfc_ncmd->seg_cnt = 0;
1398	return 1;
1399	}
1400
1401	sgl->word2 = 0;
1402	if (nseg == 1) {
1403	bf_set(lpfc_sli4_sge_last, sgl, 1);
1404	bf_set(lpfc_sli4_sge_type, sgl,
1405	LPFC_SGE_TYPE_DATA);
1406	} else {
1407	bf_set(lpfc_sli4_sge_last, sgl, 0);
1408
1409	/* expand the segment */
1410	if (!lsp_just_set &&
1411	!((j + 1) % phba->border_sge_num) &&
1412	((nseg - 1) != i)) {
1413	/* set LSP type */
1414	bf_set(lpfc_sli4_sge_type, sgl,
1415	LPFC_SGE_TYPE_LSP);
1416
1417	sgl_xtra = lpfc_get_sgl_per_hdwq(
1418	phba, buf: lpfc_ncmd);
1419
1420	if (unlikely(!sgl_xtra)) {
1421	lpfc_ncmd->seg_cnt = 0;
1422	return 1;
1423	}
1424	sgl->addr_lo = cpu_to_le32(putPaddrLow(
1425	sgl_xtra->dma_phys_sgl));
1426	sgl->addr_hi = cpu_to_le32(putPaddrHigh(
1427	sgl_xtra->dma_phys_sgl));
1428
1429	} else {
1430	bf_set(lpfc_sli4_sge_type, sgl,
1431	LPFC_SGE_TYPE_DATA);
1432	}
1433	}
1434
1435	if (!(bf_get(lpfc_sli4_sge_type, sgl) &
1436	LPFC_SGE_TYPE_LSP)) {
1437	if ((nseg - 1) == i)
1438	bf_set(lpfc_sli4_sge_last, sgl, 1);
1439
1440	physaddr = sg_dma_address(data_sg);
1441	dma_len = sg_dma_len(data_sg);
1442	sgl->addr_lo = cpu_to_le32(
1443	putPaddrLow(physaddr));
1444	sgl->addr_hi = cpu_to_le32(
1445	putPaddrHigh(physaddr));
1446
1447	bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
1448	sgl->word2 = cpu_to_le32(sgl->word2);
1449	sgl->sge_len = cpu_to_le32(dma_len);
1450
1451	dma_offset += dma_len;
1452	data_sg = sg_next(data_sg);
1453
1454	sgl++;
1455
1456	lsp_just_set = false;
1457	} else {
1458	sgl->word2 = cpu_to_le32(sgl->word2);
1459
1460	sgl->sge_len = cpu_to_le32(
1461	phba->cfg_sg_dma_buf_size);
1462
1463	sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
1464	i = i - 1;
1465
1466	lsp_just_set = true;
1467	}
1468
1469	j++;
1470	}
1471
1472	/* PBDE support for first data SGE only */
1473	if (nseg == 1 && phba->cfg_enable_pbde) {
1474	/* Words 13-15 */
1475	bde = (struct ulp_bde64 *)
1476	&wqe->words[13];
1477	bde->addrLow = first_data_sgl->addr_lo;
1478	bde->addrHigh = first_data_sgl->addr_hi;
1479	bde->tus.f.bdeSize =
1480	le32_to_cpu(first_data_sgl->sge_len);
1481	bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1482	bde->tus.w = cpu_to_le32(bde->tus.w);
1483
1484	/* Word 11 - set PBDE bit */
1485	bf_set(wqe_pbde, &wqe->generic.wqe_com, 1);
1486	} else {
1487	memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
1488	/* Word 11 - PBDE bit disabled by default template */
1489	}
1490
1491	} else {
1492	lpfc_ncmd->seg_cnt = 0;
1493
1494	/* For this clause to be valid, the payload_length
1495	* and sg_cnt must zero.
1496	*/
1497	if (nCmd->payload_length != 0) {
1498	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1499	"6063 NVME DMA Prep Err: sg_cnt %d "
1500	"payload_length x%x\n",
1501	nCmd->sg_cnt, nCmd->payload_length);
1502	return 1;
1503	}
1504	}
1505	return 0;
1506	}
1507
1508	/**
1509	* lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
1510	* @pnvme_lport: Pointer to the driver's local port data
1511	* @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1512	* @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1513	* @pnvme_fcreq: IO request from nvme fc to driver.
1514	*
1515	* Driver registers this routine as it io request handler. This
1516	* routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1517	* data structure to the rport indicated in @lpfc_nvme_rport.
1518	*
1519	* Return value :
1520	* 0 - Success
1521	* TODO: What are the failure codes.
1522	**/
1523	static int
1524	lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
1525	struct nvme_fc_remote_port *pnvme_rport,
1526	void *hw_queue_handle,
1527	struct nvmefc_fcp_req *pnvme_fcreq)
1528	{
1529	int ret = 0;
1530	int expedite = 0;
1531	int idx, cpu;
1532	struct lpfc_nvme_lport *lport;
1533	struct lpfc_fc4_ctrl_stat *cstat;
1534	struct lpfc_vport *vport;
1535	struct lpfc_hba *phba;
1536	struct lpfc_nodelist *ndlp;
1537	struct lpfc_io_buf *lpfc_ncmd;
1538	struct lpfc_nvme_rport *rport;
1539	struct lpfc_nvme_qhandle *lpfc_queue_info;
1540	struct lpfc_nvme_fcpreq_priv *freqpriv;
1541	struct nvme_common_command *sqe;
1542	uint64_t start = 0;
1543	#if (IS_ENABLED(CONFIG_NVME_FC))
1544	u8 *uuid = NULL;
1545	int err;
1546	enum dma_data_direction iodir;
1547	#endif
1548
1549	/* Validate pointers. LLDD fault handling with transport does
1550	* have timing races.
1551	*/
1552	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1553	if (unlikely(!lport)) {
1554	ret = -EINVAL;
1555	goto out_fail;
1556	}
1557
1558	vport = lport->vport;
1559
1560	if (unlikely(!hw_queue_handle)) {
1561	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1562	"6117 Fail IO, NULL hw_queue_handle\n");
1563	atomic_inc(v: &lport->xmt_fcp_err);
1564	ret = -EBUSY;
1565	goto out_fail;
1566	}
1567
1568	phba = vport->phba;
1569
1570	if ((unlikely(vport->load_flag & FC_UNLOADING)) ||
1571	phba->hba_flag & HBA_IOQ_FLUSH) {
1572	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1573	"6124 Fail IO, Driver unload\n");
1574	atomic_inc(v: &lport->xmt_fcp_err);
1575	ret = -ENODEV;
1576	goto out_fail;
1577	}
1578
1579	freqpriv = pnvme_fcreq->private;
1580	if (unlikely(!freqpriv)) {
1581	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1582	"6158 Fail IO, NULL request data\n");
1583	atomic_inc(v: &lport->xmt_fcp_err);
1584	ret = -EINVAL;
1585	goto out_fail;
1586	}
1587
1588	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1589	if (phba->ktime_on)
1590	start = ktime_get_ns();
1591	#endif
1592	rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
1593	lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;
1594
1595	/*
1596	* Catch race where our node has transitioned, but the
1597	* transport is still transitioning.
1598	*/
1599	ndlp = rport->ndlp;
1600	if (!ndlp) {
1601	lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1602	"6053 Busy IO, ndlp not ready: rport x%px "
1603	"ndlp x%px, DID x%06x\n",
1604	rport, ndlp, pnvme_rport->port_id);
1605	atomic_inc(v: &lport->xmt_fcp_err);
1606	ret = -EBUSY;
1607	goto out_fail;
1608	}
1609
1610	/* The remote node has to be a mapped target or it's an error. */
1611	if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
1612	(ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
1613	lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1614	"6036 Fail IO, DID x%06x not ready for "
1615	"IO. State x%x, Type x%x Flg x%x\n",
1616	pnvme_rport->port_id,
1617	ndlp->nlp_state, ndlp->nlp_type,
1618	ndlp->fc4_xpt_flags);
1619	atomic_inc(v: &lport->xmt_fcp_bad_ndlp);
1620	ret = -EBUSY;
1621	goto out_fail;
1622
1623	}
1624
1625	/* Currently only NVME Keep alive commands should be expedited
1626	* if the driver runs out of a resource. These should only be
1627	* issued on the admin queue, qidx 0
1628	*/
1629	if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) {
1630	sqe = &((struct nvme_fc_cmd_iu *)
1631	pnvme_fcreq->cmdaddr)->sqe.common;
1632	if (sqe->opcode == nvme_admin_keep_alive)
1633	expedite = 1;
1634	}
1635
1636	/* Check if IO qualifies for CMF */
1637	if (phba->cmf_active_mode != LPFC_CFG_OFF &&
1638	pnvme_fcreq->io_dir == NVMEFC_FCP_READ &&
1639	pnvme_fcreq->payload_length) {
1640	ret = lpfc_update_cmf_cmd(phba, sz: pnvme_fcreq->payload_length);
1641	if (ret) {
1642	ret = -EBUSY;
1643	goto out_fail;
1644	}
1645	/* Get start time for IO latency */
1646	start = ktime_get_ns();
1647	}
1648
1649	/* The node is shared with FCP IO, make sure the IO pending count does
1650	* not exceed the programmed depth.
1651	*/
1652	if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
1653	if ((atomic_read(v: &ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
1654	!expedite) {
1655	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1656	"6174 Fail IO, ndlp qdepth exceeded: "
1657	"idx %d DID %x pend %d qdepth %d\n",
1658	lpfc_queue_info->index, ndlp->nlp_DID,
1659	atomic_read(&ndlp->cmd_pending),
1660	ndlp->cmd_qdepth);
1661	atomic_inc(v: &lport->xmt_fcp_qdepth);
1662	ret = -EBUSY;
1663	goto out_fail1;
1664	}
1665	}
1666
1667	/* Lookup Hardware Queue index based on fcp_io_sched module parameter */
1668	if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
1669	idx = lpfc_queue_info->index;
1670	} else {
1671	cpu = raw_smp_processor_id();
1672	idx = phba->sli4_hba.cpu_map[cpu].hdwq;
1673	}
1674
1675	lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);
1676	if (lpfc_ncmd == NULL) {
1677	atomic_inc(v: &lport->xmt_fcp_noxri);
1678	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1679	"6065 Fail IO, driver buffer pool is empty: "
1680	"idx %d DID %x\n",
1681	lpfc_queue_info->index, ndlp->nlp_DID);
1682	ret = -EBUSY;
1683	goto out_fail1;
1684	}
1685	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1686	if (start) {
1687	lpfc_ncmd->ts_cmd_start = start;
1688	lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;
1689	} else {
1690	lpfc_ncmd->ts_cmd_start = 0;
1691	}
1692	#endif
1693	lpfc_ncmd->rx_cmd_start = start;
1694
1695	/*
1696	* Store the data needed by the driver to issue, abort, and complete
1697	* an IO.
1698	* Do not let the IO hang out forever. There is no midlayer issuing
1699	* an abort so inform the FW of the maximum IO pending time.
1700	*/
1701	freqpriv->nvme_buf = lpfc_ncmd;
1702	lpfc_ncmd->nvmeCmd = pnvme_fcreq;
1703	lpfc_ncmd->ndlp = ndlp;
1704	lpfc_ncmd->qidx = lpfc_queue_info->qidx;
1705
1706	#if (IS_ENABLED(CONFIG_NVME_FC))
1707	/* check the necessary and sufficient condition to support VMID */
1708	if (lpfc_is_vmid_enabled(phba) &&
1709	(ndlp->vmid_support ||
1710	phba->pport->vmid_priority_tagging ==
1711	LPFC_VMID_PRIO_TAG_ALL_TARGETS)) {
1712	/* is the I/O generated by a VM, get the associated virtual */
1713	/* entity id */
1714	uuid = nvme_fc_io_getuuid(req: pnvme_fcreq);
1715
1716	if (uuid) {
1717	if (pnvme_fcreq->io_dir == NVMEFC_FCP_WRITE)
1718	iodir = DMA_TO_DEVICE;
1719	else if (pnvme_fcreq->io_dir == NVMEFC_FCP_READ)
1720	iodir = DMA_FROM_DEVICE;
1721	else
1722	iodir = DMA_NONE;
1723
1724	err = lpfc_vmid_get_appid(vport, uuid, iodir,
1725	tag: (union lpfc_vmid_io_tag *)
1726	&lpfc_ncmd->cur_iocbq.vmid_tag);
1727	if (!err)
1728	lpfc_ncmd->cur_iocbq.cmd_flag |= LPFC_IO_VMID;
1729	}
1730	}
1731	#endif
1732
1733	/*
1734	* Issue the IO on the WQ indicated by index in the hw_queue_handle.
1735	* This identfier was create in our hardware queue create callback
1736	* routine. The driver now is dependent on the IO queue steering from
1737	* the transport. We are trusting the upper NVME layers know which
1738	* index to use and that they have affinitized a CPU to this hardware
1739	* queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
1740	*/
1741	lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
1742	cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat;
1743
1744	lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, pnode: ndlp, cstat);
1745	ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
1746	if (ret) {
1747	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1748	"6175 Fail IO, Prep DMA: "
1749	"idx %d DID %x\n",
1750	lpfc_queue_info->index, ndlp->nlp_DID);
1751	atomic_inc(v: &lport->xmt_fcp_err);
1752	ret = -ENOMEM;
1753	goto out_free_nvme_buf;
1754	}
1755
1756	lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1757	lpfc_ncmd->cur_iocbq.sli4_xritag,
1758	lpfc_queue_info->index, ndlp->nlp_DID);
1759
1760	ret = lpfc_sli4_issue_wqe(phba, qp: lpfc_ncmd->hdwq, pwqe: &lpfc_ncmd->cur_iocbq);
1761	if (ret) {
1762	atomic_inc(v: &lport->xmt_fcp_wqerr);
1763	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1764	"6113 Fail IO, Could not issue WQE err %x "
1765	"sid: x%x did: x%x oxid: x%x\n",
1766	ret, vport->fc_myDID, ndlp->nlp_DID,
1767	lpfc_ncmd->cur_iocbq.sli4_xritag);
1768	goto out_free_nvme_buf;
1769	}
1770
1771	if (phba->cfg_xri_rebalancing)
1772	lpfc_keep_pvt_pool_above_lowwm(phba, hwqid: lpfc_ncmd->hdwq_no);
1773
1774	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1775	if (lpfc_ncmd->ts_cmd_start)
1776	lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
1777
1778	if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) {
1779	cpu = raw_smp_processor_id();
1780	this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
1781	lpfc_ncmd->cpu = cpu;
1782	if (idx != cpu)
1783	lpfc_printf_vlog(vport,
1784	KERN_INFO, LOG_NVME_IOERR,
1785	"6702 CPU Check cmd: "
1786	"cpu %d wq %d\n",
1787	lpfc_ncmd->cpu,
1788	lpfc_queue_info->index);
1789	}
1790	#endif
1791	return 0;
1792
1793	out_free_nvme_buf:
1794	if (lpfc_ncmd->nvmeCmd->sg_cnt) {
1795	if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
1796	cstat->output_requests--;
1797	else
1798	cstat->input_requests--;
1799	} else
1800	cstat->control_requests--;
1801	lpfc_release_nvme_buf(phba, lpfc_ncmd);
1802	out_fail1:
1803	lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT,
1804	sz: pnvme_fcreq->payload_length, NULL);
1805	out_fail:
1806	return ret;
1807	}
1808
1809	/**
1810	* lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
1811	* @phba: Pointer to HBA context object
1812	* @cmdiocb: Pointer to command iocb object.
1813	* @rspiocb: Pointer to response iocb object.
1814	*
1815	* This is the callback function for any NVME FCP IO that was aborted.
1816	*
1817	* Return value:
1818	* None
1819	**/
1820	void
1821	lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
1822	struct lpfc_iocbq *rspiocb)
1823	{
1824	struct lpfc_wcqe_complete *abts_cmpl = &rspiocb->wcqe_cmpl;
1825
1826	lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
1827	"6145 ABORT_XRI_CN completing on rpi x%x "
1828	"original iotag x%x, abort cmd iotag x%x "
1829	"req_tag x%x, status x%x, hwstatus x%x\n",
1830	bf_get(wqe_ctxt_tag, &cmdiocb->wqe.generic.wqe_com),
1831	get_job_abtsiotag(phba, cmdiocb), cmdiocb->iotag,
1832	bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),
1833	bf_get(lpfc_wcqe_c_status, abts_cmpl),
1834	bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));
1835	lpfc_sli_release_iocbq(phba, cmdiocb);
1836	}
1837
1838	/**
1839	* lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
1840	* @pnvme_lport: Pointer to the driver's local port data
1841	* @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1842	* @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1843	* @pnvme_fcreq: IO request from nvme fc to driver.
1844	*
1845	* Driver registers this routine as its nvme request io abort handler. This
1846	* routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
1847	* data structure to the rport indicated in @lpfc_nvme_rport. This routine
1848	* is executed asynchronously - one the target is validated as "MAPPED" and
1849	* ready for IO, the driver issues the abort request and returns.
1850	*
1851	* Return value:
1852	* None
1853	**/
1854	static void
1855	lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
1856	struct nvme_fc_remote_port *pnvme_rport,
1857	void *hw_queue_handle,
1858	struct nvmefc_fcp_req *pnvme_fcreq)
1859	{
1860	struct lpfc_nvme_lport *lport;
1861	struct lpfc_vport *vport;
1862	struct lpfc_hba *phba;
1863	struct lpfc_io_buf *lpfc_nbuf;
1864	struct lpfc_iocbq *nvmereq_wqe;
1865	struct lpfc_nvme_fcpreq_priv *freqpriv;
1866	unsigned long flags;
1867	int ret_val;
1868
1869	/* Validate pointers. LLDD fault handling with transport does
1870	* have timing races.
1871	*/
1872	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1873	if (unlikely(!lport))
1874	return;
1875
1876	vport = lport->vport;
1877
1878	if (unlikely(!hw_queue_handle)) {
1879	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1880	"6129 Fail Abort, HW Queue Handle NULL.\n");
1881	return;
1882	}
1883
1884	phba = vport->phba;
1885	freqpriv = pnvme_fcreq->private;
1886
1887	if (unlikely(!freqpriv))
1888	return;
1889	if (vport->load_flag & FC_UNLOADING)
1890	return;
1891
1892	/* Announce entry to new IO submit field. */
1893	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1894	"6002 Abort Request to rport DID x%06x "
1895	"for nvme_fc_req x%px\n",
1896	pnvme_rport->port_id,
1897	pnvme_fcreq);
1898
1899	lpfc_nbuf = freqpriv->nvme_buf;
1900	if (!lpfc_nbuf) {
1901	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1902	"6140 NVME IO req has no matching lpfc nvme "
1903	"io buffer. Skipping abort req.\n");
1904	return;
1905	} else if (!lpfc_nbuf->nvmeCmd) {
1906	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1907	"6141 lpfc NVME IO req has no nvme_fcreq "
1908	"io buffer. Skipping abort req.\n");
1909	return;
1910	}
1911
1912	/* Guard against IO completion being called at same time */
1913	spin_lock_irqsave(&lpfc_nbuf->buf_lock, flags);
1914
1915	/* If the hba is getting reset, this flag is set. It is
1916	* cleared when the reset is complete and rings reestablished.
1917	*/
1918	spin_lock(lock: &phba->hbalock);
1919	/* driver queued commands are in process of being flushed */
1920	if (phba->hba_flag & HBA_IOQ_FLUSH) {
1921	spin_unlock(lock: &phba->hbalock);
1922	spin_unlock_irqrestore(lock: &lpfc_nbuf->buf_lock, flags);
1923	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1924	"6139 Driver in reset cleanup - flushing "
1925	"NVME Req now. hba_flag x%x\n",
1926	phba->hba_flag);
1927	return;
1928	}
1929
1930	nvmereq_wqe = &lpfc_nbuf->cur_iocbq;
1931
1932	/*
1933	* The lpfc_nbuf and the mapped nvme_fcreq in the driver's
1934	* state must match the nvme_fcreq passed by the nvme
1935	* transport. If they don't match, it is likely the driver
1936	* has already completed the NVME IO and the nvme transport
1937	* has not seen it yet.
1938	*/
1939	if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
1940	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1941	"6143 NVME req mismatch: "
1942	"lpfc_nbuf x%px nvmeCmd x%px, "
1943	"pnvme_fcreq x%px. Skipping Abort xri x%x\n",
1944	lpfc_nbuf, lpfc_nbuf->nvmeCmd,
1945	pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1946	goto out_unlock;
1947	}
1948
1949	/* Don't abort IOs no longer on the pending queue. */
1950	if (!(nvmereq_wqe->cmd_flag & LPFC_IO_ON_TXCMPLQ)) {
1951	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1952	"6142 NVME IO req x%px not queued - skipping "
1953	"abort req xri x%x\n",
1954	pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1955	goto out_unlock;
1956	}
1957
1958	atomic_inc(v: &lport->xmt_fcp_abort);
1959	lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1960	nvmereq_wqe->sli4_xritag,
1961	nvmereq_wqe->hba_wqidx, pnvme_rport->port_id);
1962
1963	/* Outstanding abort is in progress */
1964	if (nvmereq_wqe->cmd_flag & LPFC_DRIVER_ABORTED) {
1965	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1966	"6144 Outstanding NVME I/O Abort Request "
1967	"still pending on nvme_fcreq x%px, "
1968	"lpfc_ncmd x%px xri x%x\n",
1969	pnvme_fcreq, lpfc_nbuf,
1970	nvmereq_wqe->sli4_xritag);
1971	goto out_unlock;
1972	}
1973
1974	ret_val = lpfc_sli4_issue_abort_iotag(phba, cmdiocb: nvmereq_wqe,
1975	cmpl: lpfc_nvme_abort_fcreq_cmpl);
1976
1977	spin_unlock(lock: &phba->hbalock);
1978	spin_unlock_irqrestore(lock: &lpfc_nbuf->buf_lock, flags);
1979
1980	/* Make sure HBA is alive */
1981	lpfc_issue_hb_tmo(phba);
1982
1983	if (ret_val != WQE_SUCCESS) {
1984	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1985	"6137 Failed abts issue_wqe with status x%x "
1986	"for nvme_fcreq x%px.\n",
1987	ret_val, pnvme_fcreq);
1988	return;
1989	}
1990
1991	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1992	"6138 Transport Abort NVME Request Issued for "
1993	"ox_id x%x\n",
1994	nvmereq_wqe->sli4_xritag);
1995	return;
1996
1997	out_unlock:
1998	spin_unlock(lock: &phba->hbalock);
1999	spin_unlock_irqrestore(lock: &lpfc_nbuf->buf_lock, flags);
2000	return;
2001	}
2002
2003	/* Declare and initialization an instance of the FC NVME template. */
2004	static struct nvme_fc_port_template lpfc_nvme_template = {
2005	/* initiator-based functions */
2006	.localport_delete = lpfc_nvme_localport_delete,
2007	.remoteport_delete = lpfc_nvme_remoteport_delete,
2008	.create_queue = lpfc_nvme_create_queue,
2009	.delete_queue = lpfc_nvme_delete_queue,
2010	.ls_req = lpfc_nvme_ls_req,
2011	.fcp_io = lpfc_nvme_fcp_io_submit,
2012	.ls_abort = lpfc_nvme_ls_abort,
2013	.fcp_abort = lpfc_nvme_fcp_abort,
2014	.xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp,
2015
2016	.max_hw_queues = 1,
2017	.max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2018	.max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2019	.dma_boundary = 0xFFFFFFFF,
2020
2021	/* Sizes of additional private data for data structures.
2022	* No use for the last two sizes at this time.
2023	*/
2024	.local_priv_sz = sizeof(struct lpfc_nvme_lport),
2025	.remote_priv_sz = sizeof(struct lpfc_nvme_rport),
2026	.lsrqst_priv_sz = 0,
2027	.fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv),
2028	};
2029
2030	/*
2031	* lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA
2032	*
2033	* This routine removes a nvme buffer from head of @hdwq io_buf_list
2034	* and returns to caller.
2035	*
2036	* Return codes:
2037	* NULL - Error
2038	* Pointer to lpfc_nvme_buf - Success
2039	**/
2040	static struct lpfc_io_buf *
2041	lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
2042	int idx, int expedite)
2043	{
2044	struct lpfc_io_buf *lpfc_ncmd;
2045	struct lpfc_sli4_hdw_queue *qp;
2046	struct sli4_sge *sgl;
2047	struct lpfc_iocbq *pwqeq;
2048	union lpfc_wqe128 *wqe;
2049
2050	lpfc_ncmd = lpfc_get_io_buf(phba, NULL, hwqid: idx, expedite);
2051
2052	if (lpfc_ncmd) {
2053	pwqeq = &(lpfc_ncmd->cur_iocbq);
2054	wqe = &pwqeq->wqe;
2055
2056	/* Setup key fields in buffer that may have been changed
2057	* if other protocols used this buffer.
2058	*/
2059	pwqeq->cmd_flag = LPFC_IO_NVME;
2060	pwqeq->cmd_cmpl = lpfc_nvme_io_cmd_cmpl;
2061	lpfc_ncmd->start_time = jiffies;
2062	lpfc_ncmd->flags = 0;
2063
2064	/* Rsp SGE will be filled in when we rcv an IO
2065	* from the NVME Layer to be sent.
2066	* The cmd is going to be embedded so we need a SKIP SGE.
2067	*/
2068	sgl = lpfc_ncmd->dma_sgl;
2069	bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2070	bf_set(lpfc_sli4_sge_last, sgl, 0);
2071	sgl->word2 = cpu_to_le32(sgl->word2);
2072	/* Fill in word 3 / sgl_len during cmd submission */
2073
2074	/* Initialize 64 bytes only */
2075	memset(wqe, 0, sizeof(union lpfc_wqe));
2076
2077	if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
2078	atomic_inc(v: &ndlp->cmd_pending);
2079	lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
2080	}
2081
2082	} else {
2083	qp = &phba->sli4_hba.hdwq[idx];
2084	qp->empty_io_bufs++;
2085	}
2086
2087	return lpfc_ncmd;
2088	}
2089
2090	/**
2091	* lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
2092	* @phba: The Hba for which this call is being executed.
2093	* @lpfc_ncmd: The nvme buffer which is being released.
2094	*
2095	* This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
2096	* lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer
2097	* and cannot be reused for at least RA_TOV amount of time if it was
2098	* aborted.
2099	**/
2100	static void
2101	lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)
2102	{
2103	struct lpfc_sli4_hdw_queue *qp;
2104	unsigned long iflag = 0;
2105
2106	if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)
2107	atomic_dec(v: &lpfc_ncmd->ndlp->cmd_pending);
2108
2109	lpfc_ncmd->ndlp = NULL;
2110	lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
2111
2112	qp = lpfc_ncmd->hdwq;
2113	if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
2114	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2115	"6310 XB release deferred for "
2116	"ox_id x%x on reqtag x%x\n",
2117	lpfc_ncmd->cur_iocbq.sli4_xritag,
2118	lpfc_ncmd->cur_iocbq.iotag);
2119
2120	spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
2121	list_add_tail(new: &lpfc_ncmd->list,
2122	head: &qp->lpfc_abts_io_buf_list);
2123	qp->abts_nvme_io_bufs++;
2124	spin_unlock_irqrestore(lock: &qp->abts_io_buf_list_lock, flags: iflag);
2125	} else
2126	lpfc_release_io_buf(phba, ncmd: (struct lpfc_io_buf *)lpfc_ncmd, qp);
2127	}
2128
2129	/**
2130	* lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2131	* @vport: the lpfc_vport instance requesting a localport.
2132	*
2133	* This routine is invoked to create an nvme localport instance to bind
2134	* to the nvme_fc_transport. It is called once during driver load
2135	* like lpfc_create_shost after all other services are initialized.
2136	* It requires a vport, vpi, and wwns at call time. Other localport
2137	* parameters are modified as the driver's FCID and the Fabric WWN
2138	* are established.
2139	*
2140	* Return codes
2141	* 0 - successful
2142	* -ENOMEM - no heap memory available
2143	* other values - from nvme registration upcall
2144	**/
2145	int
2146	lpfc_nvme_create_localport(struct lpfc_vport *vport)
2147	{
2148	int ret = 0;
2149	struct lpfc_hba *phba = vport->phba;
2150	struct nvme_fc_port_info nfcp_info;
2151	struct nvme_fc_local_port *localport;
2152	struct lpfc_nvme_lport *lport;
2153
2154	/* Initialize this localport instance. The vport wwn usage ensures
2155	* that NPIV is accounted for.
2156	*/
2157	memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));
2158	nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;
2159	nfcp_info.node_name = wwn_to_u64(wwn: vport->fc_nodename.u.wwn);
2160	nfcp_info.port_name = wwn_to_u64(wwn: vport->fc_portname.u.wwn);
2161
2162	/* We need to tell the transport layer + 1 because it takes page
2163	* alignment into account. When space for the SGL is allocated we
2164	* allocate + 3, one for cmd, one for rsp and one for this alignment
2165	*/
2166	lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
2167
2168	/* Advertise how many hw queues we support based on cfg_hdw_queue,
2169	* which will not exceed cpu count.
2170	*/
2171	lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue;
2172
2173	if (!IS_ENABLED(CONFIG_NVME_FC))
2174	return ret;
2175
2176	/* localport is allocated from the stack, but the registration
2177	* call allocates heap memory as well as the private area.
2178	*/
2179
2180	ret = nvme_fc_register_localport(pinfo: &nfcp_info, template: &lpfc_nvme_template,
2181	dev: &vport->phba->pcidev->dev, lport_p: &localport);
2182	if (!ret) {
2183	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
2184	"6005 Successfully registered local "
2185	"NVME port num %d, localP x%px, private "
2186	"x%px, sg_seg %d\n",
2187	localport->port_num, localport,
2188	localport->private,
2189	lpfc_nvme_template.max_sgl_segments);
2190
2191	/* Private is our lport size declared in the template. */
2192	lport = (struct lpfc_nvme_lport *)localport->private;
2193	vport->localport = localport;
2194	lport->vport = vport;
2195	vport->nvmei_support = 1;
2196
2197	atomic_set(v: &lport->xmt_fcp_noxri, i: 0);
2198	atomic_set(v: &lport->xmt_fcp_bad_ndlp, i: 0);
2199	atomic_set(v: &lport->xmt_fcp_qdepth, i: 0);
2200	atomic_set(v: &lport->xmt_fcp_err, i: 0);
2201	atomic_set(v: &lport->xmt_fcp_wqerr, i: 0);
2202	atomic_set(v: &lport->xmt_fcp_abort, i: 0);
2203	atomic_set(v: &lport->xmt_ls_abort, i: 0);
2204	atomic_set(v: &lport->xmt_ls_err, i: 0);
2205	atomic_set(v: &lport->cmpl_fcp_xb, i: 0);
2206	atomic_set(v: &lport->cmpl_fcp_err, i: 0);
2207	atomic_set(v: &lport->cmpl_ls_xb, i: 0);
2208	atomic_set(v: &lport->cmpl_ls_err, i: 0);
2209
2210	atomic_set(v: &lport->fc4NvmeLsRequests, i: 0);
2211	atomic_set(v: &lport->fc4NvmeLsCmpls, i: 0);
2212	}
2213
2214	return ret;
2215	}
2216
2217	#if (IS_ENABLED(CONFIG_NVME_FC))
2218	/* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.
2219	*
2220	* The driver has to wait for the host nvme transport to callback
2221	* indicating the localport has successfully unregistered all
2222	* resources. Since this is an uninterruptible wait, loop every ten
2223	* seconds and print a message indicating no progress.
2224	*
2225	* An uninterruptible wait is used because of the risk of transport-to-
2226	* driver state mismatch.
2227	*/
2228	static void
2229	lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
2230	struct lpfc_nvme_lport *lport,
2231	struct completion *lport_unreg_cmp)
2232	{
2233	u32 wait_tmo;
2234	int ret, i, pending = 0;
2235	struct lpfc_sli_ring *pring;
2236	struct lpfc_hba *phba = vport->phba;
2237	struct lpfc_sli4_hdw_queue *qp;
2238	int abts_scsi, abts_nvme;
2239
2240	/* Host transport has to clean up and confirm requiring an indefinite
2241	* wait. Print a message if a 10 second wait expires and renew the
2242	* wait. This is unexpected.
2243	*/
2244	wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000);
2245	while (true) {
2246	ret = wait_for_completion_timeout(x: lport_unreg_cmp, timeout: wait_tmo);
2247	if (unlikely(!ret)) {
2248	pending = 0;
2249	abts_scsi = 0;
2250	abts_nvme = 0;
2251	for (i = 0; i < phba->cfg_hdw_queue; i++) {
2252	qp = &phba->sli4_hba.hdwq[i];
2253	if (!vport->localport || !qp || !qp->io_wq)
2254	return;
2255
2256	pring = qp->io_wq->pring;
2257	if (!pring)
2258	continue;
2259	pending += pring->txcmplq_cnt;
2260	abts_scsi += qp->abts_scsi_io_bufs;
2261	abts_nvme += qp->abts_nvme_io_bufs;
2262	}
2263	if (!vport->localport ||
2264	test_bit(HBA_PCI_ERR, &vport->phba->bit_flags) ||
2265	phba->link_state == LPFC_HBA_ERROR ||
2266	vport->load_flag & FC_UNLOADING)
2267	return;
2268
2269	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2270	"6176 Lport x%px Localport x%px wait "
2271	"timed out. Pending %d [%d:%d]. "
2272	"Renewing.\n",
2273	lport, vport->localport, pending,
2274	abts_scsi, abts_nvme);
2275	continue;
2276	}
2277	break;
2278	}
2279	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
2280	"6177 Lport x%px Localport x%px Complete Success\n",
2281	lport, vport->localport);
2282	}
2283	#endif
2284
2285	/**
2286	* lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
2287	* @vport: pointer to a host virtual N_Port data structure
2288	*
2289	* This routine is invoked to destroy all lports bound to the phba.
2290	* The lport memory was allocated by the nvme fc transport and is
2291	* released there. This routine ensures all rports bound to the
2292	* lport have been disconnected.
2293	*
2294	**/
2295	void
2296	lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
2297	{
2298	#if (IS_ENABLED(CONFIG_NVME_FC))
2299	struct nvme_fc_local_port *localport;
2300	struct lpfc_nvme_lport *lport;
2301	int ret;
2302	DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp);
2303
2304	if (vport->nvmei_support == 0)
2305	return;
2306
2307	localport = vport->localport;
2308	if (!localport)
2309	return;
2310	lport = (struct lpfc_nvme_lport *)localport->private;
2311
2312	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2313	"6011 Destroying NVME localport x%px\n",
2314	localport);
2315
2316	/* lport's rport list is clear. Unregister
2317	* lport and release resources.
2318	*/
2319	lport->lport_unreg_cmp = &lport_unreg_cmp;
2320	ret = nvme_fc_unregister_localport(localport);
2321
2322	/* Wait for completion. This either blocks
2323	* indefinitely or succeeds
2324	*/
2325	lpfc_nvme_lport_unreg_wait(vport, lport, lport_unreg_cmp: &lport_unreg_cmp);
2326	vport->localport = NULL;
2327
2328	/* Regardless of the unregister upcall response, clear
2329	* nvmei_support. All rports are unregistered and the
2330	* driver will clean up.
2331	*/
2332	vport->nvmei_support = 0;
2333	if (ret == 0) {
2334	lpfc_printf_vlog(vport,
2335	KERN_INFO, LOG_NVME_DISC,
2336	"6009 Unregistered lport Success\n");
2337	} else {
2338	lpfc_printf_vlog(vport,
2339	KERN_INFO, LOG_NVME_DISC,
2340	"6010 Unregistered lport "
2341	"Failed, status x%x\n",
2342	ret);
2343	}
2344	#endif
2345	}
2346
2347	void
2348	lpfc_nvme_update_localport(struct lpfc_vport *vport)
2349	{
2350	#if (IS_ENABLED(CONFIG_NVME_FC))
2351	struct nvme_fc_local_port *localport;
2352	struct lpfc_nvme_lport *lport;
2353
2354	localport = vport->localport;
2355	if (!localport) {
2356	lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2357	"6710 Update NVME fail. No localport\n");
2358	return;
2359	}
2360	lport = (struct lpfc_nvme_lport *)localport->private;
2361	if (!lport) {
2362	lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2363	"6171 Update NVME fail. localP x%px, No lport\n",
2364	localport);
2365	return;
2366	}
2367	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2368	"6012 Update NVME lport x%px did x%x\n",
2369	localport, vport->fc_myDID);
2370
2371	localport->port_id = vport->fc_myDID;
2372	if (localport->port_id == 0)
2373	localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;
2374	else
2375	localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;
2376
2377	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2378	"6030 bound lport x%px to DID x%06x\n",
2379	lport, localport->port_id);
2380	#endif
2381	}
2382
2383	int
2384	lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2385	{
2386	#if (IS_ENABLED(CONFIG_NVME_FC))
2387	int ret = 0;
2388	struct nvme_fc_local_port *localport;
2389	struct lpfc_nvme_lport *lport;
2390	struct lpfc_nvme_rport *rport;
2391	struct lpfc_nvme_rport *oldrport;
2392	struct nvme_fc_remote_port *remote_port;
2393	struct nvme_fc_port_info rpinfo;
2394	struct lpfc_nodelist *prev_ndlp = NULL;
2395	struct fc_rport *srport = ndlp->rport;
2396
2397	lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
2398	"6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2399	ndlp->nlp_DID, ndlp->nlp_type);
2400
2401	localport = vport->localport;
2402	if (!localport)
2403	return 0;
2404
2405	lport = (struct lpfc_nvme_lport *)localport->private;
2406
2407	/* NVME rports are not preserved across devloss.
2408	* Just register this instance. Note, rpinfo->dev_loss_tmo
2409	* is left 0 to indicate accept transport defaults. The
2410	* driver communicates port role capabilities consistent
2411	* with the PRLI response data.
2412	*/
2413	memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info));
2414	rpinfo.port_id = ndlp->nlp_DID;
2415	if (ndlp->nlp_type & NLP_NVME_TARGET)
2416	rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;
2417	if (ndlp->nlp_type & NLP_NVME_INITIATOR)
2418	rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;
2419
2420	if (ndlp->nlp_type & NLP_NVME_DISCOVERY)
2421	rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;
2422
2423	rpinfo.port_name = wwn_to_u64(wwn: ndlp->nlp_portname.u.wwn);
2424	rpinfo.node_name = wwn_to_u64(wwn: ndlp->nlp_nodename.u.wwn);
2425	if (srport)
2426	rpinfo.dev_loss_tmo = srport->dev_loss_tmo;
2427	else
2428	rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo;
2429
2430	spin_lock_irq(lock: &ndlp->lock);
2431
2432	/* If an oldrport exists, so does the ndlp reference. If not
2433	* a new reference is needed because either the node has never
2434	* been registered or it's been unregistered and getting deleted.
2435	*/
2436	oldrport = lpfc_ndlp_get_nrport(ndlp);
2437	if (oldrport) {
2438	prev_ndlp = oldrport->ndlp;
2439	spin_unlock_irq(lock: &ndlp->lock);
2440	} else {
2441	spin_unlock_irq(lock: &ndlp->lock);
2442	if (!lpfc_nlp_get(ndlp)) {
2443	dev_warn(&vport->phba->pcidev->dev,
2444	"Warning - No node ref - exit register\n");
2445	return 0;
2446	}
2447	}
2448
2449	ret = nvme_fc_register_remoteport(localport, pinfo: &rpinfo, rport_p: &remote_port);
2450	if (!ret) {
2451	/* If the ndlp already has an nrport, this is just
2452	* a resume of the existing rport. Else this is a
2453	* new rport.
2454	*/
2455	/* Guard against an unregister/reregister
2456	* race that leaves the WAIT flag set.
2457	*/
2458	spin_lock_irq(lock: &ndlp->lock);
2459	ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;
2460	ndlp->fc4_xpt_flags |= NVME_XPT_REGD;
2461	spin_unlock_irq(lock: &ndlp->lock);
2462	rport = remote_port->private;
2463	if (oldrport) {
2464
2465	/* Sever the ndlp<->rport association
2466	* before dropping the ndlp ref from
2467	* register.
2468	*/
2469	spin_lock_irq(lock: &ndlp->lock);
2470	ndlp->nrport = NULL;
2471	ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;
2472	spin_unlock_irq(lock: &ndlp->lock);
2473	rport->ndlp = NULL;
2474	rport->remoteport = NULL;
2475
2476	/* Reference only removed if previous NDLP is no longer
2477	* active. It might be just a swap and removing the
2478	* reference would cause a premature cleanup.
2479	*/
2480	if (prev_ndlp && prev_ndlp != ndlp) {
2481	if (!prev_ndlp->nrport)
2482	lpfc_nlp_put(prev_ndlp);
2483	}
2484	}
2485
2486	/* Clean bind the rport to the ndlp. */
2487	rport->remoteport = remote_port;
2488	rport->lport = lport;
2489	rport->ndlp = ndlp;
2490	spin_lock_irq(lock: &ndlp->lock);
2491	ndlp->nrport = rport;
2492	spin_unlock_irq(lock: &ndlp->lock);
2493	lpfc_printf_vlog(vport, KERN_INFO,
2494	LOG_NVME_DISC | LOG_NODE,
2495	"6022 Bind lport x%px to remoteport x%px "
2496	"rport x%px WWNN 0x%llx, "
2497	"Rport WWPN 0x%llx DID "
2498	"x%06x Role x%x, ndlp %p prev_ndlp x%px\n",
2499	lport, remote_port, rport,
2500	rpinfo.node_name, rpinfo.port_name,
2501	rpinfo.port_id, rpinfo.port_role,
2502	ndlp, prev_ndlp);
2503	} else {
2504	lpfc_printf_vlog(vport, KERN_ERR,
2505	LOG_TRACE_EVENT,
2506	"6031 RemotePort Registration failed "
2507	"err: %d, DID x%06x ref %u\n",
2508	ret, ndlp->nlp_DID, kref_read(&ndlp->kref));
2509	lpfc_nlp_put(ndlp);
2510	}
2511
2512	return ret;
2513	#else
2514	return 0;
2515	#endif
2516	}
2517
2518	/*
2519	* lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport
2520	*
2521	* If the ndlp represents an NVME Target, that we are logged into,
2522	* ping the NVME FC Transport layer to initiate a device rescan
2523	* on this remote NPort.
2524	*/
2525	void
2526	lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2527	{
2528	#if (IS_ENABLED(CONFIG_NVME_FC))
2529	struct lpfc_nvme_rport *nrport;
2530	struct nvme_fc_remote_port *remoteport = NULL;
2531
2532	spin_lock_irq(lock: &ndlp->lock);
2533	nrport = lpfc_ndlp_get_nrport(ndlp);
2534	if (nrport)
2535	remoteport = nrport->remoteport;
2536	spin_unlock_irq(lock: &ndlp->lock);
2537
2538	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2539	"6170 Rescan NPort DID x%06x type x%x "
2540	"state x%x nrport x%px remoteport x%px\n",
2541	ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state,
2542	nrport, remoteport);
2543
2544	if (!nrport || !remoteport)
2545	goto rescan_exit;
2546
2547	/* Rescan an NVME target in MAPPED state with DISCOVERY role set */
2548	if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY &&
2549	ndlp->nlp_state == NLP_STE_MAPPED_NODE) {
2550	nvme_fc_rescan_remoteport(remoteport);
2551
2552	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2553	"6172 NVME rescanned DID x%06x "
2554	"port_state x%x\n",
2555	ndlp->nlp_DID, remoteport->port_state);
2556	}
2557	return;
2558	rescan_exit:
2559	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2560	"6169 Skip NVME Rport Rescan, NVME remoteport "
2561	"unregistered\n");
2562	#endif
2563	}
2564
2565	/* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
2566	*
2567	* There is no notion of Devloss or rport recovery from the current
2568	* nvme_transport perspective. Loss of an rport just means IO cannot
2569	* be sent and recovery is completely up to the initator.
2570	* For now, the driver just unbinds the DID and port_role so that
2571	* no further IO can be issued.
2572	*/
2573	void
2574	lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2575	{
2576	#if (IS_ENABLED(CONFIG_NVME_FC))
2577	int ret;
2578	struct nvme_fc_local_port *localport;
2579	struct lpfc_nvme_lport *lport;
2580	struct lpfc_nvme_rport *rport;
2581	struct nvme_fc_remote_port *remoteport = NULL;
2582
2583	localport = vport->localport;
2584
2585	/* This is fundamental error. The localport is always
2586	* available until driver unload. Just exit.
2587	*/
2588	if (!localport)
2589	return;
2590
2591	lport = (struct lpfc_nvme_lport *)localport->private;
2592	if (!lport)
2593	goto input_err;
2594
2595	spin_lock_irq(lock: &ndlp->lock);
2596	rport = lpfc_ndlp_get_nrport(ndlp);
2597	if (rport)
2598	remoteport = rport->remoteport;
2599	spin_unlock_irq(lock: &ndlp->lock);
2600	if (!remoteport)
2601	goto input_err;
2602
2603	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2604	"6033 Unreg nvme remoteport x%px, portname x%llx, "
2605	"port_id x%06x, portstate x%x port type x%x "
2606	"refcnt %d\n",
2607	remoteport, remoteport->port_name,
2608	remoteport->port_id, remoteport->port_state,
2609	ndlp->nlp_type, kref_read(&ndlp->kref));
2610
2611	/* Sanity check ndlp type. Only call for NVME ports. Don't
2612	* clear any rport state until the transport calls back.
2613	*/
2614
2615	if (ndlp->nlp_type & NLP_NVME_TARGET) {
2616	/* No concern about the role change on the nvme remoteport.
2617	* The transport will update it.
2618	*/
2619	spin_lock_irq(lock: &vport->phba->hbalock);
2620	ndlp->fc4_xpt_flags |= NVME_XPT_UNREG_WAIT;
2621	spin_unlock_irq(lock: &vport->phba->hbalock);
2622
2623	/* Don't let the host nvme transport keep sending keep-alives
2624	* on this remoteport. Vport is unloading, no recovery. The
2625	* return values is ignored. The upcall is a courtesy to the
2626	* transport.
2627	*/
2628	if (vport->load_flag & FC_UNLOADING ||
2629	unlikely(vport->phba->link_state == LPFC_HBA_ERROR))
2630	(void)nvme_fc_set_remoteport_devloss(remoteport, dev_loss_tmo: 0);
2631
2632	ret = nvme_fc_unregister_remoteport(remoteport);
2633
2634	/* The driver no longer knows if the nrport memory is valid.
2635	* because the controller teardown process has begun and
2636	* is asynchronous. Break the binding in the ndlp. Also
2637	* remove the register ndlp reference to setup node release.
2638	*/
2639	ndlp->nrport = NULL;
2640	lpfc_nlp_put(ndlp);
2641	if (ret != 0) {
2642	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2643	"6167 NVME unregister failed %d "
2644	"port_state x%x\n",
2645	ret, remoteport->port_state);
2646
2647	if (vport->load_flag & FC_UNLOADING) {
2648	/* Only 1 thread can drop the initial node
2649	* reference. Check if another thread has set
2650	* NLP_DROPPED.
2651	*/
2652	spin_lock_irq(lock: &ndlp->lock);
2653	if (!(ndlp->nlp_flag & NLP_DROPPED)) {
2654	ndlp->nlp_flag |= NLP_DROPPED;
2655	spin_unlock_irq(lock: &ndlp->lock);
2656	lpfc_nlp_put(ndlp);
2657	return;
2658	}
2659	spin_unlock_irq(lock: &ndlp->lock);
2660	}
2661	}
2662	}
2663	return;
2664
2665	input_err:
2666	#endif
2667	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2668	"6168 State error: lport x%px, rport x%px FCID x%06x\n",
2669	vport->localport, ndlp->rport, ndlp->nlp_DID);
2670	}
2671
2672	/**
2673	* lpfc_sli4_nvme_pci_offline_aborted - Fast-path process of NVME xri abort
2674	* @phba: pointer to lpfc hba data structure.
2675	* @lpfc_ncmd: The nvme job structure for the request being aborted.
2676	*
2677	* This routine is invoked by the worker thread to process a SLI4 fast-path
2678	* NVME aborted xri. Aborted NVME IO commands are completed to the transport
2679	* here.
2680	**/
2681	void
2682	lpfc_sli4_nvme_pci_offline_aborted(struct lpfc_hba *phba,
2683	struct lpfc_io_buf *lpfc_ncmd)
2684	{
2685	struct nvmefc_fcp_req *nvme_cmd = NULL;
2686
2687	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2688	"6533 %s nvme_cmd %p tag x%x abort complete and "
2689	"xri released\n", __func__,
2690	lpfc_ncmd->nvmeCmd,
2691	lpfc_ncmd->cur_iocbq.iotag);
2692
2693	/* Aborted NVME commands are required to not complete
2694	* before the abort exchange command fully completes.
2695	* Once completed, it is available via the put list.
2696	*/
2697	if (lpfc_ncmd->nvmeCmd) {
2698	nvme_cmd = lpfc_ncmd->nvmeCmd;
2699	nvme_cmd->transferred_length = 0;
2700	nvme_cmd->rcv_rsplen = 0;
2701	nvme_cmd->status = NVME_SC_INTERNAL;
2702	nvme_cmd->done(nvme_cmd);
2703	lpfc_ncmd->nvmeCmd = NULL;
2704	}
2705	lpfc_release_nvme_buf(phba, lpfc_ncmd);
2706	}
2707
2708	/**
2709	* lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2710	* @phba: pointer to lpfc hba data structure.
2711	* @axri: pointer to the fcp xri abort wcqe structure.
2712	* @lpfc_ncmd: The nvme job structure for the request being aborted.
2713	*
2714	* This routine is invoked by the worker thread to process a SLI4 fast-path
2715	* NVME aborted xri. Aborted NVME IO commands are completed to the transport
2716	* here.
2717	**/
2718	void
2719	lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba,
2720	struct sli4_wcqe_xri_aborted *axri,
2721	struct lpfc_io_buf *lpfc_ncmd)
2722	{
2723	uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
2724	struct nvmefc_fcp_req *nvme_cmd = NULL;
2725	struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;
2726
2727
2728	if (ndlp)
2729	lpfc_sli4_abts_err_handler(phba, ndlp, axri);
2730
2731	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2732	"6311 nvme_cmd %p xri x%x tag x%x abort complete and "
2733	"xri released\n",
2734	lpfc_ncmd->nvmeCmd, xri,
2735	lpfc_ncmd->cur_iocbq.iotag);
2736
2737	/* Aborted NVME commands are required to not complete
2738	* before the abort exchange command fully completes.
2739	* Once completed, it is available via the put list.
2740	*/
2741	if (lpfc_ncmd->nvmeCmd) {
2742	nvme_cmd = lpfc_ncmd->nvmeCmd;
2743	nvme_cmd->done(nvme_cmd);
2744	lpfc_ncmd->nvmeCmd = NULL;
2745	}
2746	lpfc_release_nvme_buf(phba, lpfc_ncmd);
2747	}
2748
2749	/**
2750	* lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete
2751	* @phba: Pointer to HBA context object.
2752	*
2753	* This function flushes all wqes in the nvme rings and frees all resources
2754	* in the txcmplq. This function does not issue abort wqes for the IO
2755	* commands in txcmplq, they will just be returned with
2756	* IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
2757	* slot has been permanently disabled.
2758	**/
2759	void
2760	lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba)
2761	{
2762	struct lpfc_sli_ring *pring;
2763	u32 i, wait_cnt = 0;
2764
2765	if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq)
2766	return;
2767
2768	/* Cycle through all IO rings and make sure all outstanding
2769	* WQEs have been removed from the txcmplqs.
2770	*/
2771	for (i = 0; i < phba->cfg_hdw_queue; i++) {
2772	if (!phba->sli4_hba.hdwq[i].io_wq)
2773	continue;
2774	pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2775
2776	if (!pring)
2777	continue;
2778
2779	/* Retrieve everything on the txcmplq */
2780	while (!list_empty(head: &pring->txcmplq)) {
2781	msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);
2782	wait_cnt++;
2783
2784	/* The sleep is 10mS. Every ten seconds,
2785	* dump a message. Something is wrong.
2786	*/
2787	if ((wait_cnt % 1000) == 0) {
2788	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2789	"6178 NVME IO not empty, "
2790	"cnt %d\n", wait_cnt);
2791	}
2792	}
2793	}
2794
2795	/* Make sure HBA is alive */
2796	lpfc_issue_hb_tmo(phba);
2797
2798	}
2799
2800	void
2801	lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
2802	uint32_t stat, uint32_t param)
2803	{
2804	#if (IS_ENABLED(CONFIG_NVME_FC))
2805	struct lpfc_io_buf *lpfc_ncmd;
2806	struct nvmefc_fcp_req *nCmd;
2807	struct lpfc_wcqe_complete wcqe;
2808	struct lpfc_wcqe_complete *wcqep = &wcqe;
2809
2810	lpfc_ncmd = pwqeIn->io_buf;
2811	if (!lpfc_ncmd) {
2812	lpfc_sli_release_iocbq(phba, pwqeIn);
2813	return;
2814	}
2815	/* For abort iocb just return, IO iocb will do a done call */
2816	if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) ==
2817	CMD_ABORT_XRI_CX) {
2818	lpfc_sli_release_iocbq(phba, pwqeIn);
2819	return;
2820	}
2821
2822	spin_lock(lock: &lpfc_ncmd->buf_lock);
2823	nCmd = lpfc_ncmd->nvmeCmd;
2824	if (!nCmd) {
2825	spin_unlock(lock: &lpfc_ncmd->buf_lock);
2826	lpfc_release_nvme_buf(phba, lpfc_ncmd);
2827	return;
2828	}
2829	spin_unlock(lock: &lpfc_ncmd->buf_lock);
2830
2831	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2832	"6194 NVME Cancel xri %x\n",
2833	lpfc_ncmd->cur_iocbq.sli4_xritag);
2834
2835	wcqep->word0 = 0;
2836	bf_set(lpfc_wcqe_c_status, wcqep, stat);
2837	wcqep->parameter = param;
2838	wcqep->total_data_placed = 0;
2839	wcqep->word3 = 0; /* xb is 0 */
2840
2841	/* Call release with XB=1 to queue the IO into the abort list. */
2842	if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)
2843	bf_set(lpfc_wcqe_c_xb, wcqep, 1);
2844
2845	memcpy(&pwqeIn->wcqe_cmpl, wcqep, sizeof(*wcqep));
2846	(pwqeIn->cmd_cmpl)(phba, pwqeIn, pwqeIn);
2847	#endif
2848	}
2849