1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*******************************************************************************
3	* IBM Virtual SCSI Target Driver
4	* Copyright (C) 2003-2005 Dave Boutcher (boutcher@us.ibm.com) IBM Corp.
5	* Santiago Leon (santil@us.ibm.com) IBM Corp.
6	* Linda Xie (lxie@us.ibm.com) IBM Corp.
7	*
8	* Copyright (C) 2005-2011 FUJITA Tomonori <tomof@acm.org>
9	* Copyright (C) 2010 Nicholas A. Bellinger <nab@kernel.org>
10	*
11	* Authors: Bryant G. Ly <bryantly@linux.vnet.ibm.com>
12	* Authors: Michael Cyr <mikecyr@linux.vnet.ibm.com>
13	*
14	****************************************************************************/
15
16	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
17
18	#include <linux/module.h>
19	#include <linux/kernel.h>
20	#include <linux/slab.h>
21	#include <linux/types.h>
22	#include <linux/list.h>
23	#include <linux/string.h>
24	#include <linux/delay.h>
25	#include <linux/of.h>
26
27	#include <target/target_core_base.h>
28	#include <target/target_core_fabric.h>
29
30	#include <asm/hvcall.h>
31	#include <asm/vio.h>
32
33	#include <scsi/viosrp.h>
34
35	#include "ibmvscsi_tgt.h"
36
37	#define IBMVSCSIS_VERSION "v0.2"
38
39	#define INITIAL_SRP_LIMIT 1024
40	#define DEFAULT_MAX_SECTORS 256
41	#define MAX_TXU 1024 * 1024
42
43	static uint max_vdma_size = MAX_H_COPY_RDMA;
44
45	static char system_id[SYS_ID_NAME_LEN] = "";
46	static char partition_name[PARTITION_NAMELEN] = "UNKNOWN";
47	static uint partition_number = -1;
48
49	/* Adapter list and lock to control it */
50	static DEFINE_SPINLOCK(ibmvscsis_dev_lock);
51	static LIST_HEAD(ibmvscsis_dev_list);
52
53	static long ibmvscsis_parse_command(struct scsi_info *vscsi,
54	struct viosrp_crq *crq);
55
56	static void ibmvscsis_adapter_idle(struct scsi_info *vscsi);
57
58	static void ibmvscsis_determine_resid(struct se_cmd *se_cmd,
59	struct srp_rsp *rsp)
60	{
61	u32 residual_count = se_cmd->residual_count;
62
63	if (!residual_count)
64	return;
65
66	if (se_cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
67	if (se_cmd->data_direction == DMA_TO_DEVICE) {
68	/* residual data from an underflow write */
69	rsp->flags = SRP_RSP_FLAG_DOUNDER;
70	rsp->data_out_res_cnt = cpu_to_be32(residual_count);
71	} else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
72	/* residual data from an underflow read */
73	rsp->flags = SRP_RSP_FLAG_DIUNDER;
74	rsp->data_in_res_cnt = cpu_to_be32(residual_count);
75	}
76	} else if (se_cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
77	if (se_cmd->data_direction == DMA_TO_DEVICE) {
78	/* residual data from an overflow write */
79	rsp->flags = SRP_RSP_FLAG_DOOVER;
80	rsp->data_out_res_cnt = cpu_to_be32(residual_count);
81	} else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
82	/* residual data from an overflow read */
83	rsp->flags = SRP_RSP_FLAG_DIOVER;
84	rsp->data_in_res_cnt = cpu_to_be32(residual_count);
85	}
86	}
87	}
88
89	/**
90	* connection_broken() - Determine if the connection to the client is good
91	* @vscsi: Pointer to our adapter structure
92	*
93	* This function attempts to send a ping MAD to the client. If the call to
94	* queue the request returns H_CLOSED then the connection has been broken
95	* and the function returns TRUE.
96	*
97	* EXECUTION ENVIRONMENT:
98	* Interrupt or Process environment
99	*/
100	static bool connection_broken(struct scsi_info *vscsi)
101	{
102	struct viosrp_crq *crq;
103	u64 buffer[2] = { 0, 0 };
104	long h_return_code;
105	bool rc = false;
106
107	/* create a PING crq */
108	crq = (struct viosrp_crq *)&buffer;
109	crq->valid = VALID_CMD_RESP_EL;
110	crq->format = MESSAGE_IN_CRQ;
111	crq->status = PING;
112
113	h_return_code = h_send_crq(vscsi->dds.unit_id,
114	cpu_to_be64(buffer[MSG_HI]),
115	cpu_to_be64(buffer[MSG_LOW]));
116
117	dev_dbg(&vscsi->dev, "Connection_broken: rc %ld\n", h_return_code);
118
119	if (h_return_code == H_CLOSED)
120	rc = true;
121
122	return rc;
123	}
124
125	/**
126	* ibmvscsis_unregister_command_q() - Helper Function-Unregister Command Queue
127	* @vscsi: Pointer to our adapter structure
128	*
129	* This function calls h_free_q then frees the interrupt bit etc.
130	* It must release the lock before doing so because of the time it can take
131	* for h_free_crq in PHYP
132	* NOTE: * the caller must make sure that state and or flags will prevent
133	* interrupt handler from scheduling work.
134	* * anyone calling this function may need to set the CRQ_CLOSED flag
135	* we can't do it here, because we don't have the lock
136	*
137	* EXECUTION ENVIRONMENT:
138	* Process level
139	*/
140	static long ibmvscsis_unregister_command_q(struct scsi_info *vscsi)
141	{
142	long qrc;
143	long rc = ADAPT_SUCCESS;
144	int ticks = 0;
145
146	do {
147	qrc = h_free_crq(vscsi->dds.unit_id);
148	switch (qrc) {
149	case H_SUCCESS:
150	spin_lock_bh(lock: &vscsi->intr_lock);
151	vscsi->flags &= ~PREP_FOR_SUSPEND_FLAGS;
152	spin_unlock_bh(lock: &vscsi->intr_lock);
153	break;
154
155	case H_HARDWARE:
156	case H_PARAMETER:
157	dev_err(&vscsi->dev, "unregister_command_q: error from h_free_crq %ld\n",
158	qrc);
159	rc = ERROR;
160	break;
161
162	case H_BUSY:
163	case H_LONG_BUSY_ORDER_1_MSEC:
164	/* msleep not good for small values */
165	usleep_range(min: 1000, max: 2000);
166	ticks += 1;
167	break;
168	case H_LONG_BUSY_ORDER_10_MSEC:
169	usleep_range(min: 10000, max: 20000);
170	ticks += 10;
171	break;
172	case H_LONG_BUSY_ORDER_100_MSEC:
173	msleep(msecs: 100);
174	ticks += 100;
175	break;
176	case H_LONG_BUSY_ORDER_1_SEC:
177	ssleep(seconds: 1);
178	ticks += 1000;
179	break;
180	case H_LONG_BUSY_ORDER_10_SEC:
181	ssleep(seconds: 10);
182	ticks += 10000;
183	break;
184	case H_LONG_BUSY_ORDER_100_SEC:
185	ssleep(seconds: 100);
186	ticks += 100000;
187	break;
188	default:
189	dev_err(&vscsi->dev, "unregister_command_q: unknown error %ld from h_free_crq\n",
190	qrc);
191	rc = ERROR;
192	break;
193	}
194
195	/*
196	* dont wait more then 300 seconds
197	* ticks are in milliseconds more or less
198	*/
199	if (ticks > 300000 && qrc != H_SUCCESS) {
200	rc = ERROR;
201	dev_err(&vscsi->dev, "Excessive wait for h_free_crq\n");
202	}
203	} while (qrc != H_SUCCESS && rc == ADAPT_SUCCESS);
204
205	dev_dbg(&vscsi->dev, "Freeing CRQ: phyp rc %ld, rc %ld\n", qrc, rc);
206
207	return rc;
208	}
209
210	/**
211	* ibmvscsis_delete_client_info() - Helper function to Delete Client Info
212	* @vscsi: Pointer to our adapter structure
213	* @client_closed: True if client closed its queue
214	*
215	* Deletes information specific to the client when the client goes away
216	*
217	* EXECUTION ENVIRONMENT:
218	* Interrupt or Process
219	*/
220	static void ibmvscsis_delete_client_info(struct scsi_info *vscsi,
221	bool client_closed)
222	{
223	vscsi->client_cap = 0;
224
225	/*
226	* Some things we don't want to clear if we're closing the queue,
227	* because some clients don't resend the host handshake when they
228	* get a transport event.
229	*/
230	if (client_closed)
231	vscsi->client_data.os_type = 0;
232	}
233
234	/**
235	* ibmvscsis_free_command_q() - Free Command Queue
236	* @vscsi: Pointer to our adapter structure
237	*
238	* This function calls unregister_command_q, then clears interrupts and
239	* any pending interrupt acknowledgments associated with the command q.
240	* It also clears memory if there is no error.
241	*
242	* PHYP did not meet the PAPR architecture so that we must give up the
243	* lock. This causes a timing hole regarding state change. To close the
244	* hole this routine does accounting on any change that occurred during
245	* the time the lock is not held.
246	* NOTE: must give up and then acquire the interrupt lock, the caller must
247	* make sure that state and or flags will prevent interrupt handler from
248	* scheduling work.
249	*
250	* EXECUTION ENVIRONMENT:
251	* Process level, interrupt lock is held
252	*/
253	static long ibmvscsis_free_command_q(struct scsi_info *vscsi)
254	{
255	int bytes;
256	u32 flags_under_lock;
257	u16 state_under_lock;
258	long rc = ADAPT_SUCCESS;
259
260	if (!(vscsi->flags & CRQ_CLOSED)) {
261	vio_disable_interrupts(vscsi->dma_dev);
262
263	state_under_lock = vscsi->new_state;
264	flags_under_lock = vscsi->flags;
265	vscsi->phyp_acr_state = 0;
266	vscsi->phyp_acr_flags = 0;
267
268	spin_unlock_bh(lock: &vscsi->intr_lock);
269	rc = ibmvscsis_unregister_command_q(vscsi);
270	spin_lock_bh(lock: &vscsi->intr_lock);
271
272	if (state_under_lock != vscsi->new_state)
273	vscsi->phyp_acr_state = vscsi->new_state;
274
275	vscsi->phyp_acr_flags = ((~flags_under_lock) & vscsi->flags);
276
277	if (rc == ADAPT_SUCCESS) {
278	bytes = vscsi->cmd_q.size * PAGE_SIZE;
279	memset(vscsi->cmd_q.base_addr, 0, bytes);
280	vscsi->cmd_q.index = 0;
281	vscsi->flags |= CRQ_CLOSED;
282
283	ibmvscsis_delete_client_info(vscsi, client_closed: false);
284	}
285
286	dev_dbg(&vscsi->dev, "free_command_q: flags 0x%x, state 0x%hx, acr_flags 0x%x, acr_state 0x%hx\n",
287	vscsi->flags, vscsi->state, vscsi->phyp_acr_flags,
288	vscsi->phyp_acr_state);
289	}
290	return rc;
291	}
292
293	/**
294	* ibmvscsis_cmd_q_dequeue() - Get valid Command element
295	* @mask: Mask to use in case index wraps
296	* @current_index: Current index into command queue
297	* @base_addr: Pointer to start of command queue
298	*
299	* Returns a pointer to a valid command element or NULL, if the command
300	* queue is empty
301	*
302	* EXECUTION ENVIRONMENT:
303	* Interrupt environment, interrupt lock held
304	*/
305	static struct viosrp_crq *ibmvscsis_cmd_q_dequeue(uint mask,
306	uint *current_index,
307	struct viosrp_crq *base_addr)
308	{
309	struct viosrp_crq *ptr;
310
311	ptr = base_addr + *current_index;
312
313	if (ptr->valid) {
314	*current_index = (*current_index + 1) & mask;
315	dma_rmb();
316	} else {
317	ptr = NULL;
318	}
319
320	return ptr;
321	}
322
323	/**
324	* ibmvscsis_send_init_message() - send initialize message to the client
325	* @vscsi: Pointer to our adapter structure
326	* @format: Which Init Message format to send
327	*
328	* EXECUTION ENVIRONMENT:
329	* Interrupt environment interrupt lock held
330	*/
331	static long ibmvscsis_send_init_message(struct scsi_info *vscsi, u8 format)
332	{
333	struct viosrp_crq *crq;
334	u64 buffer[2] = { 0, 0 };
335	long rc;
336
337	crq = (struct viosrp_crq *)&buffer;
338	crq->valid = VALID_INIT_MSG;
339	crq->format = format;
340	rc = h_send_crq(vscsi->dds.unit_id, cpu_to_be64(buffer[MSG_HI]),
341	cpu_to_be64(buffer[MSG_LOW]));
342
343	return rc;
344	}
345
346	/**
347	* ibmvscsis_check_init_msg() - Check init message valid
348	* @vscsi: Pointer to our adapter structure
349	* @format: Pointer to return format of Init Message, if any.
350	* Set to UNUSED_FORMAT if no Init Message in queue.
351	*
352	* Checks if an initialize message was queued by the initiatior
353	* after the queue was created and before the interrupt was enabled.
354	*
355	* EXECUTION ENVIRONMENT:
356	* Process level only, interrupt lock held
357	*/
358	static long ibmvscsis_check_init_msg(struct scsi_info *vscsi, uint *format)
359	{
360	struct viosrp_crq *crq;
361	long rc = ADAPT_SUCCESS;
362
363	crq = ibmvscsis_cmd_q_dequeue(mask: vscsi->cmd_q.mask, current_index: &vscsi->cmd_q.index,
364	base_addr: vscsi->cmd_q.base_addr);
365	if (!crq) {
366	*format = (uint)UNUSED_FORMAT;
367	} else if (crq->valid == VALID_INIT_MSG && crq->format == INIT_MSG) {
368	*format = (uint)INIT_MSG;
369	crq->valid = INVALIDATE_CMD_RESP_EL;
370	dma_rmb();
371
372	/*
373	* the caller has ensured no initialize message was
374	* sent after the queue was
375	* created so there should be no other message on the queue.
376	*/
377	crq = ibmvscsis_cmd_q_dequeue(mask: vscsi->cmd_q.mask,
378	current_index: &vscsi->cmd_q.index,
379	base_addr: vscsi->cmd_q.base_addr);
380	if (crq) {
381	*format = (uint)(crq->format);
382	rc = ERROR;
383	crq->valid = INVALIDATE_CMD_RESP_EL;
384	dma_rmb();
385	}
386	} else {
387	*format = (uint)(crq->format);
388	rc = ERROR;
389	crq->valid = INVALIDATE_CMD_RESP_EL;
390	dma_rmb();
391	}
392
393	return rc;
394	}
395
396	/**
397	* ibmvscsis_disconnect() - Helper function to disconnect
398	* @work: Pointer to work_struct, gives access to our adapter structure
399	*
400	* An error has occurred or the driver received a Transport event,
401	* and the driver is requesting that the command queue be de-registered
402	* in a safe manner. If there is no outstanding I/O then we can stop the
403	* queue. If we are restarting the queue it will be reflected in the
404	* the state of the adapter.
405	*
406	* EXECUTION ENVIRONMENT:
407	* Process environment
408	*/
409	static void ibmvscsis_disconnect(struct work_struct *work)
410	{
411	struct scsi_info *vscsi = container_of(work, struct scsi_info,
412	proc_work);
413	u16 new_state;
414	bool wait_idle = false;
415
416	spin_lock_bh(lock: &vscsi->intr_lock);
417	new_state = vscsi->new_state;
418	vscsi->new_state = 0;
419
420	vscsi->flags |= DISCONNECT_SCHEDULED;
421	vscsi->flags &= ~SCHEDULE_DISCONNECT;
422
423	dev_dbg(&vscsi->dev, "disconnect: flags 0x%x, state 0x%hx\n",
424	vscsi->flags, vscsi->state);
425
426	/*
427	* check which state we are in and see if we
428	* should transitition to the new state
429	*/
430	switch (vscsi->state) {
431	/* Should never be called while in this state. */
432	case NO_QUEUE:
433	/*
434	* Can never transition from this state;
435	* igonore errors and logout.
436	*/
437	case UNCONFIGURING:
438	break;
439
440	/* can transition from this state to UNCONFIGURING */
441	case ERR_DISCONNECT:
442	if (new_state == UNCONFIGURING)
443	vscsi->state = new_state;
444	break;
445
446	/*
447	* Can transition from this state to unconfiguring
448	* or err disconnect.
449	*/
450	case ERR_DISCONNECT_RECONNECT:
451	switch (new_state) {
452	case UNCONFIGURING:
453	case ERR_DISCONNECT:
454	vscsi->state = new_state;
455	break;
456
457	case WAIT_IDLE:
458	break;
459	default:
460	break;
461	}
462	break;
463
464	/* can transition from this state to UNCONFIGURING */
465	case ERR_DISCONNECTED:
466	if (new_state == UNCONFIGURING)
467	vscsi->state = new_state;
468	break;
469
470	case WAIT_ENABLED:
471	switch (new_state) {
472	case UNCONFIGURING:
473	vscsi->state = new_state;
474	vscsi->flags |= RESPONSE_Q_DOWN;
475	vscsi->flags &= ~(SCHEDULE_DISCONNECT |
476	DISCONNECT_SCHEDULED);
477	dma_rmb();
478	if (vscsi->flags & CFG_SLEEPING) {
479	vscsi->flags &= ~CFG_SLEEPING;
480	complete(&vscsi->unconfig);
481	}
482	break;
483
484	/* should never happen */
485	case ERR_DISCONNECT:
486	case ERR_DISCONNECT_RECONNECT:
487	case WAIT_IDLE:
488	dev_err(&vscsi->dev, "disconnect: invalid state %d for WAIT_IDLE\n",
489	vscsi->state);
490	break;
491	}
492	break;
493
494	case WAIT_IDLE:
495	switch (new_state) {
496	case UNCONFIGURING:
497	vscsi->flags |= RESPONSE_Q_DOWN;
498	vscsi->state = new_state;
499	vscsi->flags &= ~(SCHEDULE_DISCONNECT |
500	DISCONNECT_SCHEDULED);
501	ibmvscsis_free_command_q(vscsi);
502	break;
503	case ERR_DISCONNECT:
504	case ERR_DISCONNECT_RECONNECT:
505	vscsi->state = new_state;
506	break;
507	}
508	break;
509
510	/*
511	* Initiator has not done a successful srp login
512	* or has done a successful srp logout ( adapter was not
513	* busy). In the first case there can be responses queued
514	* waiting for space on the initiators response queue (MAD)
515	* The second case the adapter is idle. Assume the worse case,
516	* i.e. the second case.
517	*/
518	case WAIT_CONNECTION:
519	case CONNECTED:
520	case SRP_PROCESSING:
521	wait_idle = true;
522	vscsi->state = new_state;
523	break;
524
525	/* can transition from this state to UNCONFIGURING */
526	case UNDEFINED:
527	if (new_state == UNCONFIGURING)
528	vscsi->state = new_state;
529	break;
530	default:
531	break;
532	}
533
534	if (wait_idle) {
535	dev_dbg(&vscsi->dev, "disconnect start wait, active %d, sched %d\n",
536	(int)list_empty(head: &vscsi->active_q),
537	(int)list_empty(head: &vscsi->schedule_q));
538	if (!list_empty(head: &vscsi->active_q) ||
539	!list_empty(head: &vscsi->schedule_q)) {
540	vscsi->flags |= WAIT_FOR_IDLE;
541	dev_dbg(&vscsi->dev, "disconnect flags 0x%x\n",
542	vscsi->flags);
543	/*
544	* This routine is can not be called with the interrupt
545	* lock held.
546	*/
547	spin_unlock_bh(lock: &vscsi->intr_lock);
548	wait_for_completion(&vscsi->wait_idle);
549	spin_lock_bh(lock: &vscsi->intr_lock);
550	}
551	dev_dbg(&vscsi->dev, "disconnect stop wait\n");
552
553	ibmvscsis_adapter_idle(vscsi);
554	}
555
556	spin_unlock_bh(lock: &vscsi->intr_lock);
557	}
558
559	/**
560	* ibmvscsis_post_disconnect() - Schedule the disconnect
561	* @vscsi: Pointer to our adapter structure
562	* @new_state: State to move to after disconnecting
563	* @flag_bits: Flags to turn on in adapter structure
564	*
565	* If it's already been scheduled, then see if we need to "upgrade"
566	* the new state (if the one passed in is more "severe" than the
567	* previous one).
568	*
569	* PRECONDITION:
570	* interrupt lock is held
571	*/
572	static void ibmvscsis_post_disconnect(struct scsi_info *vscsi, uint new_state,
573	uint flag_bits)
574	{
575	uint state;
576
577	/* check the validity of the new state */
578	switch (new_state) {
579	case UNCONFIGURING:
580	case ERR_DISCONNECT:
581	case ERR_DISCONNECT_RECONNECT:
582	case WAIT_IDLE:
583	break;
584
585	default:
586	dev_err(&vscsi->dev, "post_disconnect: Invalid new state %d\n",
587	new_state);
588	return;
589	}
590
591	vscsi->flags |= flag_bits;
592
593	dev_dbg(&vscsi->dev, "post_disconnect: new_state 0x%x, flag_bits 0x%x, vscsi->flags 0x%x, state %hx\n",
594	new_state, flag_bits, vscsi->flags, vscsi->state);
595
596	if (!(vscsi->flags & (DISCONNECT_SCHEDULED | SCHEDULE_DISCONNECT))) {
597	vscsi->flags |= SCHEDULE_DISCONNECT;
598	vscsi->new_state = new_state;
599
600	INIT_WORK(&vscsi->proc_work, ibmvscsis_disconnect);
601	(void)queue_work(wq: vscsi->work_q, work: &vscsi->proc_work);
602	} else {
603	if (vscsi->new_state)
604	state = vscsi->new_state;
605	else
606	state = vscsi->state;
607
608	switch (state) {
609	case NO_QUEUE:
610	case UNCONFIGURING:
611	break;
612
613	case ERR_DISCONNECTED:
614	case ERR_DISCONNECT:
615	case UNDEFINED:
616	if (new_state == UNCONFIGURING)
617	vscsi->new_state = new_state;
618	break;
619
620	case ERR_DISCONNECT_RECONNECT:
621	switch (new_state) {
622	case UNCONFIGURING:
623	case ERR_DISCONNECT:
624	vscsi->new_state = new_state;
625	break;
626	default:
627	break;
628	}
629	break;
630
631	case WAIT_ENABLED:
632	case WAIT_IDLE:
633	case WAIT_CONNECTION:
634	case CONNECTED:
635	case SRP_PROCESSING:
636	vscsi->new_state = new_state;
637	break;
638
639	default:
640	break;
641	}
642	}
643
644	dev_dbg(&vscsi->dev, "Leaving post_disconnect: flags 0x%x, new_state 0x%x\n",
645	vscsi->flags, vscsi->new_state);
646	}
647
648	/**
649	* ibmvscsis_handle_init_compl_msg() - Respond to an Init Complete Message
650	* @vscsi: Pointer to our adapter structure
651	*
652	* Must be called with interrupt lock held.
653	*/
654	static long ibmvscsis_handle_init_compl_msg(struct scsi_info *vscsi)
655	{
656	long rc = ADAPT_SUCCESS;
657
658	switch (vscsi->state) {
659	case NO_QUEUE:
660	case ERR_DISCONNECT:
661	case ERR_DISCONNECT_RECONNECT:
662	case ERR_DISCONNECTED:
663	case UNCONFIGURING:
664	case UNDEFINED:
665	rc = ERROR;
666	break;
667
668	case WAIT_CONNECTION:
669	vscsi->state = CONNECTED;
670	break;
671
672	case WAIT_IDLE:
673	case SRP_PROCESSING:
674	case CONNECTED:
675	case WAIT_ENABLED:
676	default:
677	rc = ERROR;
678	dev_err(&vscsi->dev, "init_msg: invalid state %d to get init compl msg\n",
679	vscsi->state);
680	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
681	break;
682	}
683
684	return rc;
685	}
686
687	/**
688	* ibmvscsis_handle_init_msg() - Respond to an Init Message
689	* @vscsi: Pointer to our adapter structure
690	*
691	* Must be called with interrupt lock held.
692	*/
693	static long ibmvscsis_handle_init_msg(struct scsi_info *vscsi)
694	{
695	long rc = ADAPT_SUCCESS;
696
697	switch (vscsi->state) {
698	case WAIT_CONNECTION:
699	rc = ibmvscsis_send_init_message(vscsi, format: INIT_COMPLETE_MSG);
700	switch (rc) {
701	case H_SUCCESS:
702	vscsi->state = CONNECTED;
703	break;
704
705	case H_PARAMETER:
706	dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
707	rc);
708	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, flag_bits: 0);
709	break;
710
711	case H_DROPPED:
712	dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
713	rc);
714	rc = ERROR;
715	ibmvscsis_post_disconnect(vscsi,
716	ERR_DISCONNECT_RECONNECT, flag_bits: 0);
717	break;
718
719	case H_CLOSED:
720	dev_warn(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
721	rc);
722	rc = 0;
723	break;
724	}
725	break;
726
727	case UNDEFINED:
728	rc = ERROR;
729	break;
730
731	case UNCONFIGURING:
732	break;
733
734	case WAIT_ENABLED:
735	case CONNECTED:
736	case SRP_PROCESSING:
737	case WAIT_IDLE:
738	case NO_QUEUE:
739	case ERR_DISCONNECT:
740	case ERR_DISCONNECT_RECONNECT:
741	case ERR_DISCONNECTED:
742	default:
743	rc = ERROR;
744	dev_err(&vscsi->dev, "init_msg: invalid state %d to get init msg\n",
745	vscsi->state);
746	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
747	break;
748	}
749
750	return rc;
751	}
752
753	/**
754	* ibmvscsis_init_msg() - Respond to an init message
755	* @vscsi: Pointer to our adapter structure
756	* @crq: Pointer to CRQ element containing the Init Message
757	*
758	* EXECUTION ENVIRONMENT:
759	* Interrupt, interrupt lock held
760	*/
761	static long ibmvscsis_init_msg(struct scsi_info *vscsi, struct viosrp_crq *crq)
762	{
763	long rc = ADAPT_SUCCESS;
764
765	dev_dbg(&vscsi->dev, "init_msg: state 0x%hx\n", vscsi->state);
766
767	rc = h_vioctl(vscsi->dds.unit_id, H_GET_PARTNER_INFO,
768	(u64)vscsi->map_ioba | ((u64)PAGE_SIZE << 32), 0, 0, 0,
769	0);
770	if (rc == H_SUCCESS) {
771	vscsi->client_data.partition_number =
772	be64_to_cpu(*(u64 *)vscsi->map_buf);
773	dev_dbg(&vscsi->dev, "init_msg, part num %d\n",
774	vscsi->client_data.partition_number);
775	} else {
776	dev_dbg(&vscsi->dev, "init_msg h_vioctl rc %ld\n", rc);
777	rc = ADAPT_SUCCESS;
778	}
779
780	if (crq->format == INIT_MSG) {
781	rc = ibmvscsis_handle_init_msg(vscsi);
782	} else if (crq->format == INIT_COMPLETE_MSG) {
783	rc = ibmvscsis_handle_init_compl_msg(vscsi);
784	} else {
785	rc = ERROR;
786	dev_err(&vscsi->dev, "init_msg: invalid format %d\n",
787	(uint)crq->format);
788	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
789	}
790
791	return rc;
792	}
793
794	/**
795	* ibmvscsis_establish_new_q() - Establish new CRQ queue
796	* @vscsi: Pointer to our adapter structure
797	*
798	* Must be called with interrupt lock held.
799	*/
800	static long ibmvscsis_establish_new_q(struct scsi_info *vscsi)
801	{
802	long rc = ADAPT_SUCCESS;
803	uint format;
804
805	rc = h_vioctl(vscsi->dds.unit_id, H_ENABLE_PREPARE_FOR_SUSPEND, 30000,
806	0, 0, 0, 0);
807	if (rc == H_SUCCESS)
808	vscsi->flags |= PREP_FOR_SUSPEND_ENABLED;
809	else if (rc != H_NOT_FOUND)
810	dev_err(&vscsi->dev, "Error from Enable Prepare for Suspend: %ld\n",
811	rc);
812
813	vscsi->flags &= PRESERVE_FLAG_FIELDS;
814	vscsi->rsp_q_timer.timer_pops = 0;
815	vscsi->debit = 0;
816	vscsi->credit = 0;
817
818	rc = vio_enable_interrupts(vscsi->dma_dev);
819	if (rc) {
820	dev_warn(&vscsi->dev, "establish_new_q: failed to enable interrupts, rc %ld\n",
821	rc);
822	return rc;
823	}
824
825	rc = ibmvscsis_check_init_msg(vscsi, format: &format);
826	if (rc) {
827	dev_err(&vscsi->dev, "establish_new_q: check_init_msg failed, rc %ld\n",
828	rc);
829	return rc;
830	}
831
832	if (format == UNUSED_FORMAT) {
833	rc = ibmvscsis_send_init_message(vscsi, format: INIT_MSG);
834	switch (rc) {
835	case H_SUCCESS:
836	case H_DROPPED:
837	case H_CLOSED:
838	rc = ADAPT_SUCCESS;
839	break;
840
841	case H_PARAMETER:
842	case H_HARDWARE:
843	break;
844
845	default:
846	vscsi->state = UNDEFINED;
847	rc = H_HARDWARE;
848	break;
849	}
850	} else if (format == INIT_MSG) {
851	rc = ibmvscsis_handle_init_msg(vscsi);
852	}
853
854	return rc;
855	}
856
857	/**
858	* ibmvscsis_reset_queue() - Reset CRQ Queue
859	* @vscsi: Pointer to our adapter structure
860	*
861	* This function calls h_free_q and then calls h_reg_q and does all
862	* of the bookkeeping to get us back to where we can communicate.
863	*
864	* Actually, we don't always call h_free_crq. A problem was discovered
865	* where one partition would close and reopen his queue, which would
866	* cause his partner to get a transport event, which would cause him to
867	* close and reopen his queue, which would cause the original partition
868	* to get a transport event, etc., etc. To prevent this, we don't
869	* actually close our queue if the client initiated the reset, (i.e.
870	* either we got a transport event or we have detected that the client's
871	* queue is gone)
872	*
873	* EXECUTION ENVIRONMENT:
874	* Process environment, called with interrupt lock held
875	*/
876	static void ibmvscsis_reset_queue(struct scsi_info *vscsi)
877	{
878	int bytes;
879	long rc = ADAPT_SUCCESS;
880
881	dev_dbg(&vscsi->dev, "reset_queue: flags 0x%x\n", vscsi->flags);
882
883	/* don't reset, the client did it for us */
884	if (vscsi->flags & (CLIENT_FAILED | TRANS_EVENT)) {
885	vscsi->flags &= PRESERVE_FLAG_FIELDS;
886	vscsi->rsp_q_timer.timer_pops = 0;
887	vscsi->debit = 0;
888	vscsi->credit = 0;
889	vscsi->state = WAIT_CONNECTION;
890	vio_enable_interrupts(vscsi->dma_dev);
891	} else {
892	rc = ibmvscsis_free_command_q(vscsi);
893	if (rc == ADAPT_SUCCESS) {
894	vscsi->state = WAIT_CONNECTION;
895
896	bytes = vscsi->cmd_q.size * PAGE_SIZE;
897	rc = h_reg_crq(vscsi->dds.unit_id,
898	vscsi->cmd_q.crq_token, bytes);
899	if (rc == H_CLOSED || rc == H_SUCCESS) {
900	rc = ibmvscsis_establish_new_q(vscsi);
901	}
902
903	if (rc != ADAPT_SUCCESS) {
904	dev_dbg(&vscsi->dev, "reset_queue: reg_crq rc %ld\n",
905	rc);
906
907	vscsi->state = ERR_DISCONNECTED;
908	vscsi->flags |= RESPONSE_Q_DOWN;
909	ibmvscsis_free_command_q(vscsi);
910	}
911	} else {
912	vscsi->state = ERR_DISCONNECTED;
913	vscsi->flags |= RESPONSE_Q_DOWN;
914	}
915	}
916	}
917
918	/**
919	* ibmvscsis_free_cmd_resources() - Free command resources
920	* @vscsi: Pointer to our adapter structure
921	* @cmd: Command which is not longer in use
922	*
923	* Must be called with interrupt lock held.
924	*/
925	static void ibmvscsis_free_cmd_resources(struct scsi_info *vscsi,
926	struct ibmvscsis_cmd *cmd)
927	{
928	struct iu_entry *iue = cmd->iue;
929
930	switch (cmd->type) {
931	case TASK_MANAGEMENT:
932	case SCSI_CDB:
933	/*
934	* When the queue goes down this value is cleared, so it
935	* cannot be cleared in this general purpose function.
936	*/
937	if (vscsi->debit)
938	vscsi->debit -= 1;
939	break;
940	case ADAPTER_MAD:
941	vscsi->flags &= ~PROCESSING_MAD;
942	break;
943	case UNSET_TYPE:
944	break;
945	default:
946	dev_err(&vscsi->dev, "free_cmd_resources unknown type %d\n",
947	cmd->type);
948	break;
949	}
950
951	cmd->iue = NULL;
952	list_add_tail(new: &cmd->list, head: &vscsi->free_cmd);
953	srp_iu_put(iue);
954
955	if (list_empty(head: &vscsi->active_q) && list_empty(head: &vscsi->schedule_q) &&
956	list_empty(head: &vscsi->waiting_rsp) && (vscsi->flags & WAIT_FOR_IDLE)) {
957	vscsi->flags &= ~WAIT_FOR_IDLE;
958	complete(&vscsi->wait_idle);
959	}
960	}
961
962	/**
963	* ibmvscsis_ready_for_suspend() - Helper function to call VIOCTL
964	* @vscsi: Pointer to our adapter structure
965	* @idle: Indicates whether we were called from adapter_idle. This
966	* is important to know if we need to do a disconnect, since if
967	* we're called from adapter_idle, we're still processing the
968	* current disconnect, so we can't just call post_disconnect.
969	*
970	* This function is called when the adapter is idle when phyp has sent
971	* us a Prepare for Suspend Transport Event.
972	*
973	* EXECUTION ENVIRONMENT:
974	* Process or interrupt environment called with interrupt lock held
975	*/
976	static long ibmvscsis_ready_for_suspend(struct scsi_info *vscsi, bool idle)
977	{
978	long rc = 0;
979	struct viosrp_crq *crq;
980
981	/* See if there is a Resume event in the queue */
982	crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
983
984	dev_dbg(&vscsi->dev, "ready_suspend: flags 0x%x, state 0x%hx crq_valid:%x\n",
985	vscsi->flags, vscsi->state, (int)crq->valid);
986
987	if (!(vscsi->flags & PREP_FOR_SUSPEND_ABORTED) && !(crq->valid)) {
988	rc = h_vioctl(vscsi->dds.unit_id, H_READY_FOR_SUSPEND, 0, 0, 0,
989	0, 0);
990	if (rc) {
991	dev_err(&vscsi->dev, "Ready for Suspend Vioctl failed: %ld\n",
992	rc);
993	rc = 0;
994	}
995	} else if (((vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE) &&
996	(vscsi->flags & PREP_FOR_SUSPEND_ABORTED)) ||
997	((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) ||
998	(crq->format != RESUME_FROM_SUSP)))) {
999	if (idle) {
1000	vscsi->state = ERR_DISCONNECT_RECONNECT;
1001	ibmvscsis_reset_queue(vscsi);
1002	rc = -1;
1003	} else if (vscsi->state == CONNECTED) {
1004	ibmvscsis_post_disconnect(vscsi,
1005	ERR_DISCONNECT_RECONNECT, flag_bits: 0);
1006	}
1007
1008	vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
1009
1010	if ((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) ||
1011	(crq->format != RESUME_FROM_SUSP)))
1012	dev_err(&vscsi->dev, "Invalid element in CRQ after Prepare for Suspend");
1013	}
1014
1015	vscsi->flags &= ~(PREP_FOR_SUSPEND_PENDING | PREP_FOR_SUSPEND_ABORTED);
1016
1017	return rc;
1018	}
1019
1020	/**
1021	* ibmvscsis_trans_event() - Handle a Transport Event
1022	* @vscsi: Pointer to our adapter structure
1023	* @crq: Pointer to CRQ entry containing the Transport Event
1024	*
1025	* Do the logic to close the I_T nexus. This function may not
1026	* behave to specification.
1027	*
1028	* EXECUTION ENVIRONMENT:
1029	* Interrupt, interrupt lock held
1030	*/
1031	static long ibmvscsis_trans_event(struct scsi_info *vscsi,
1032	struct viosrp_crq *crq)
1033	{
1034	long rc = ADAPT_SUCCESS;
1035
1036	dev_dbg(&vscsi->dev, "trans_event: format %d, flags 0x%x, state 0x%hx\n",
1037	(int)crq->format, vscsi->flags, vscsi->state);
1038
1039	switch (crq->format) {
1040	case MIGRATED:
1041	case PARTNER_FAILED:
1042	case PARTNER_DEREGISTER:
1043	ibmvscsis_delete_client_info(vscsi, client_closed: true);
1044	if (crq->format == MIGRATED)
1045	vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
1046	switch (vscsi->state) {
1047	case NO_QUEUE:
1048	case ERR_DISCONNECTED:
1049	case UNDEFINED:
1050	break;
1051
1052	case UNCONFIGURING:
1053	vscsi->flags |= (RESPONSE_Q_DOWN | TRANS_EVENT);
1054	break;
1055
1056	case WAIT_ENABLED:
1057	break;
1058
1059	case WAIT_CONNECTION:
1060	break;
1061
1062	case CONNECTED:
1063	ibmvscsis_post_disconnect(vscsi, WAIT_IDLE,
1064	flag_bits: (RESPONSE_Q_DOWN |
1065	TRANS_EVENT));
1066	break;
1067
1068	case SRP_PROCESSING:
1069	if ((vscsi->debit > 0) ||
1070	!list_empty(head: &vscsi->schedule_q) ||
1071	!list_empty(head: &vscsi->waiting_rsp) ||
1072	!list_empty(head: &vscsi->active_q)) {
1073	dev_dbg(&vscsi->dev, "debit %d, sched %d, wait %d, active %d\n",
1074	vscsi->debit,
1075	(int)list_empty(head: &vscsi->schedule_q),
1076	(int)list_empty(head: &vscsi->waiting_rsp),
1077	(int)list_empty(head: &vscsi->active_q));
1078	dev_warn(&vscsi->dev, "connection lost with outstanding work\n");
1079	} else {
1080	dev_dbg(&vscsi->dev, "trans_event: SRP Processing, but no outstanding work\n");
1081	}
1082
1083	ibmvscsis_post_disconnect(vscsi, WAIT_IDLE,
1084	flag_bits: (RESPONSE_Q_DOWN |
1085	TRANS_EVENT));
1086	break;
1087
1088	case ERR_DISCONNECT:
1089	case ERR_DISCONNECT_RECONNECT:
1090	case WAIT_IDLE:
1091	vscsi->flags |= (RESPONSE_Q_DOWN | TRANS_EVENT);
1092	break;
1093	}
1094	break;
1095
1096	case PREPARE_FOR_SUSPEND:
1097	dev_dbg(&vscsi->dev, "Prep for Suspend, crq status = 0x%x\n",
1098	(int)crq->status);
1099	switch (vscsi->state) {
1100	case ERR_DISCONNECTED:
1101	case WAIT_CONNECTION:
1102	case CONNECTED:
1103	ibmvscsis_ready_for_suspend(vscsi, idle: false);
1104	break;
1105	case SRP_PROCESSING:
1106	vscsi->resume_state = vscsi->state;
1107	vscsi->flags |= PREP_FOR_SUSPEND_PENDING;
1108	if (crq->status == CRQ_ENTRY_OVERWRITTEN)
1109	vscsi->flags |= PREP_FOR_SUSPEND_OVERWRITE;
1110	ibmvscsis_post_disconnect(vscsi, WAIT_IDLE, flag_bits: 0);
1111	break;
1112	case NO_QUEUE:
1113	case UNDEFINED:
1114	case UNCONFIGURING:
1115	case WAIT_ENABLED:
1116	case ERR_DISCONNECT:
1117	case ERR_DISCONNECT_RECONNECT:
1118	case WAIT_IDLE:
1119	dev_err(&vscsi->dev, "Invalid state for Prepare for Suspend Trans Event: 0x%x\n",
1120	vscsi->state);
1121	break;
1122	}
1123	break;
1124
1125	case RESUME_FROM_SUSP:
1126	dev_dbg(&vscsi->dev, "Resume from Suspend, crq status = 0x%x\n",
1127	(int)crq->status);
1128	if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) {
1129	vscsi->flags |= PREP_FOR_SUSPEND_ABORTED;
1130	} else {
1131	if ((crq->status == CRQ_ENTRY_OVERWRITTEN) ||
1132	(vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE)) {
1133	ibmvscsis_post_disconnect(vscsi,
1134	ERR_DISCONNECT_RECONNECT,
1135	flag_bits: 0);
1136	vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
1137	}
1138	}
1139	break;
1140
1141	default:
1142	rc = ERROR;
1143	dev_err(&vscsi->dev, "trans_event: invalid format %d\n",
1144	(uint)crq->format);
1145	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT,
1146	RESPONSE_Q_DOWN);
1147	break;
1148	}
1149
1150	rc = vscsi->flags & SCHEDULE_DISCONNECT;
1151
1152	dev_dbg(&vscsi->dev, "Leaving trans_event: flags 0x%x, state 0x%hx, rc %ld\n",
1153	vscsi->flags, vscsi->state, rc);
1154
1155	return rc;
1156	}
1157
1158	/**
1159	* ibmvscsis_poll_cmd_q() - Poll Command Queue
1160	* @vscsi: Pointer to our adapter structure
1161	*
1162	* Called to handle command elements that may have arrived while
1163	* interrupts were disabled.
1164	*
1165	* EXECUTION ENVIRONMENT:
1166	* intr_lock must be held
1167	*/
1168	static void ibmvscsis_poll_cmd_q(struct scsi_info *vscsi)
1169	{
1170	struct viosrp_crq *crq;
1171	long rc;
1172	bool ack = true;
1173	volatile u8 valid;
1174
1175	dev_dbg(&vscsi->dev, "poll_cmd_q: flags 0x%x, state 0x%hx, q index %ud\n",
1176	vscsi->flags, vscsi->state, vscsi->cmd_q.index);
1177
1178	rc = vscsi->flags & SCHEDULE_DISCONNECT;
1179	crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
1180	valid = crq->valid;
1181	dma_rmb();
1182
1183	while (valid) {
1184	poll_work:
1185	vscsi->cmd_q.index =
1186	(vscsi->cmd_q.index + 1) & vscsi->cmd_q.mask;
1187
1188	if (!rc) {
1189	rc = ibmvscsis_parse_command(vscsi, crq);
1190	} else {
1191	if ((uint)crq->valid == VALID_TRANS_EVENT) {
1192	/*
1193	* must service the transport layer events even
1194	* in an error state, dont break out until all
1195	* the consecutive transport events have been
1196	* processed
1197	*/
1198	rc = ibmvscsis_trans_event(vscsi, crq);
1199	} else if (vscsi->flags & TRANS_EVENT) {
1200	/*
1201	* if a tranport event has occurred leave
1202	* everything but transport events on the queue
1203	*/
1204	dev_dbg(&vscsi->dev, "poll_cmd_q, ignoring\n");
1205
1206	/*
1207	* need to decrement the queue index so we can
1208	* look at the elment again
1209	*/
1210	if (vscsi->cmd_q.index)
1211	vscsi->cmd_q.index -= 1;
1212	else
1213	/*
1214	* index is at 0 it just wrapped.
1215	* have it index last element in q
1216	*/
1217	vscsi->cmd_q.index = vscsi->cmd_q.mask;
1218	break;
1219	}
1220	}
1221
1222	crq->valid = INVALIDATE_CMD_RESP_EL;
1223
1224	crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
1225	valid = crq->valid;
1226	dma_rmb();
1227	}
1228
1229	if (!rc) {
1230	if (ack) {
1231	vio_enable_interrupts(vscsi->dma_dev);
1232	ack = false;
1233	dev_dbg(&vscsi->dev, "poll_cmd_q, reenabling interrupts\n");
1234	}
1235	valid = crq->valid;
1236	dma_rmb();
1237	if (valid)
1238	goto poll_work;
1239	}
1240
1241	dev_dbg(&vscsi->dev, "Leaving poll_cmd_q: rc %ld\n", rc);
1242	}
1243
1244	/**
1245	* ibmvscsis_free_cmd_qs() - Free elements in queue
1246	* @vscsi: Pointer to our adapter structure
1247	*
1248	* Free all of the elements on all queues that are waiting for
1249	* whatever reason.
1250	*
1251	* PRECONDITION:
1252	* Called with interrupt lock held
1253	*/
1254	static void ibmvscsis_free_cmd_qs(struct scsi_info *vscsi)
1255	{
1256	struct ibmvscsis_cmd *cmd, *nxt;
1257
1258	dev_dbg(&vscsi->dev, "free_cmd_qs: waiting_rsp empty %d, timer starter %d\n",
1259	(int)list_empty(head: &vscsi->waiting_rsp),
1260	vscsi->rsp_q_timer.started);
1261
1262	list_for_each_entry_safe(cmd, nxt, &vscsi->waiting_rsp, list) {
1263	list_del(entry: &cmd->list);
1264	ibmvscsis_free_cmd_resources(vscsi, cmd);
1265	}
1266	}
1267
1268	/**
1269	* ibmvscsis_get_free_cmd() - Get free command from list
1270	* @vscsi: Pointer to our adapter structure
1271	*
1272	* Must be called with interrupt lock held.
1273	*/
1274	static struct ibmvscsis_cmd *ibmvscsis_get_free_cmd(struct scsi_info *vscsi)
1275	{
1276	struct ibmvscsis_cmd *cmd = NULL;
1277	struct iu_entry *iue;
1278
1279	iue = srp_iu_get(&vscsi->target);
1280	if (iue) {
1281	cmd = list_first_entry_or_null(&vscsi->free_cmd,
1282	struct ibmvscsis_cmd, list);
1283	if (cmd) {
1284	if (cmd->abort_cmd)
1285	cmd->abort_cmd = NULL;
1286	cmd->flags &= ~(DELAY_SEND);
1287	list_del(entry: &cmd->list);
1288	cmd->iue = iue;
1289	cmd->type = UNSET_TYPE;
1290	memset(&cmd->se_cmd, 0, sizeof(cmd->se_cmd));
1291	} else {
1292	srp_iu_put(iue);
1293	}
1294	}
1295
1296	return cmd;
1297	}
1298
1299	/**
1300	* ibmvscsis_adapter_idle() - Helper function to handle idle adapter
1301	* @vscsi: Pointer to our adapter structure
1302	*
1303	* This function is called when the adapter is idle when the driver
1304	* is attempting to clear an error condition.
1305	* The adapter is considered busy if any of its cmd queues
1306	* are non-empty. This function can be invoked
1307	* from the off level disconnect function.
1308	*
1309	* EXECUTION ENVIRONMENT:
1310	* Process environment called with interrupt lock held
1311	*/
1312	static void ibmvscsis_adapter_idle(struct scsi_info *vscsi)
1313	{
1314	int free_qs = false;
1315	long rc = 0;
1316
1317	dev_dbg(&vscsi->dev, "adapter_idle: flags 0x%x, state 0x%hx\n",
1318	vscsi->flags, vscsi->state);
1319
1320	/* Only need to free qs if we're disconnecting from client */
1321	if (vscsi->state != WAIT_CONNECTION || vscsi->flags & TRANS_EVENT)
1322	free_qs = true;
1323
1324	switch (vscsi->state) {
1325	case UNCONFIGURING:
1326	ibmvscsis_free_command_q(vscsi);
1327	dma_rmb();
1328	isync();
1329	if (vscsi->flags & CFG_SLEEPING) {
1330	vscsi->flags &= ~CFG_SLEEPING;
1331	complete(&vscsi->unconfig);
1332	}
1333	break;
1334	case ERR_DISCONNECT_RECONNECT:
1335	ibmvscsis_reset_queue(vscsi);
1336	dev_dbg(&vscsi->dev, "adapter_idle, disc_rec: flags 0x%x\n",
1337	vscsi->flags);
1338	break;
1339
1340	case ERR_DISCONNECT:
1341	ibmvscsis_free_command_q(vscsi);
1342	vscsi->flags &= ~(SCHEDULE_DISCONNECT | DISCONNECT_SCHEDULED);
1343	vscsi->flags |= RESPONSE_Q_DOWN;
1344	if (vscsi->tport.enabled)
1345	vscsi->state = ERR_DISCONNECTED;
1346	else
1347	vscsi->state = WAIT_ENABLED;
1348	dev_dbg(&vscsi->dev, "adapter_idle, disc: flags 0x%x, state 0x%hx\n",
1349	vscsi->flags, vscsi->state);
1350	break;
1351
1352	case WAIT_IDLE:
1353	vscsi->rsp_q_timer.timer_pops = 0;
1354	vscsi->debit = 0;
1355	vscsi->credit = 0;
1356	if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) {
1357	vscsi->state = vscsi->resume_state;
1358	vscsi->resume_state = 0;
1359	rc = ibmvscsis_ready_for_suspend(vscsi, idle: true);
1360	vscsi->flags &= ~DISCONNECT_SCHEDULED;
1361	if (rc)
1362	break;
1363	} else if (vscsi->flags & TRANS_EVENT) {
1364	vscsi->state = WAIT_CONNECTION;
1365	vscsi->flags &= PRESERVE_FLAG_FIELDS;
1366	} else {
1367	vscsi->state = CONNECTED;
1368	vscsi->flags &= ~DISCONNECT_SCHEDULED;
1369	}
1370
1371	dev_dbg(&vscsi->dev, "adapter_idle, wait: flags 0x%x, state 0x%hx\n",
1372	vscsi->flags, vscsi->state);
1373	ibmvscsis_poll_cmd_q(vscsi);
1374	break;
1375
1376	case ERR_DISCONNECTED:
1377	vscsi->flags &= ~DISCONNECT_SCHEDULED;
1378	dev_dbg(&vscsi->dev, "adapter_idle, disconnected: flags 0x%x, state 0x%hx\n",
1379	vscsi->flags, vscsi->state);
1380	break;
1381
1382	default:
1383	dev_err(&vscsi->dev, "adapter_idle: in invalid state %d\n",
1384	vscsi->state);
1385	break;
1386	}
1387
1388	if (free_qs)
1389	ibmvscsis_free_cmd_qs(vscsi);
1390
1391	/*
1392	* There is a timing window where we could lose a disconnect request.
1393	* The known path to this window occurs during the DISCONNECT_RECONNECT
1394	* case above: reset_queue calls free_command_q, which will release the
1395	* interrupt lock. During that time, a new post_disconnect call can be
1396	* made with a "more severe" state (DISCONNECT or UNCONFIGURING).
1397	* Because the DISCONNECT_SCHEDULED flag is already set, post_disconnect
1398	* will only set the new_state. Now free_command_q reacquires the intr
1399	* lock and clears the DISCONNECT_SCHEDULED flag (using PRESERVE_FLAG_
1400	* FIELDS), and the disconnect is lost. This is particularly bad when
1401	* the new disconnect was for UNCONFIGURING, since the unconfigure hangs
1402	* forever.
1403	* Fix is that free command queue sets acr state and acr flags if there
1404	* is a change under the lock
1405	* note free command queue writes to this state it clears it
1406	* before releasing the lock, different drivers call the free command
1407	* queue different times so dont initialize above
1408	*/
1409	if (vscsi->phyp_acr_state != 0) {
1410	/*
1411	* set any bits in flags that may have been cleared by
1412	* a call to free command queue in switch statement
1413	* or reset queue
1414	*/
1415	vscsi->flags |= vscsi->phyp_acr_flags;
1416	ibmvscsis_post_disconnect(vscsi, new_state: vscsi->phyp_acr_state, flag_bits: 0);
1417	vscsi->phyp_acr_state = 0;
1418	vscsi->phyp_acr_flags = 0;
1419
1420	dev_dbg(&vscsi->dev, "adapter_idle: flags 0x%x, state 0x%hx, acr_flags 0x%x, acr_state 0x%hx\n",
1421	vscsi->flags, vscsi->state, vscsi->phyp_acr_flags,
1422	vscsi->phyp_acr_state);
1423	}
1424
1425	dev_dbg(&vscsi->dev, "Leaving adapter_idle: flags 0x%x, state 0x%hx, new_state 0x%x\n",
1426	vscsi->flags, vscsi->state, vscsi->new_state);
1427	}
1428
1429	/**
1430	* ibmvscsis_copy_crq_packet() - Copy CRQ Packet
1431	* @vscsi: Pointer to our adapter structure
1432	* @cmd: Pointer to command element to use to process the request
1433	* @crq: Pointer to CRQ entry containing the request
1434	*
1435	* Copy the srp information unit from the hosted
1436	* partition using remote dma
1437	*
1438	* EXECUTION ENVIRONMENT:
1439	* Interrupt, interrupt lock held
1440	*/
1441	static long ibmvscsis_copy_crq_packet(struct scsi_info *vscsi,
1442	struct ibmvscsis_cmd *cmd,
1443	struct viosrp_crq *crq)
1444	{
1445	struct iu_entry *iue = cmd->iue;
1446	long rc = 0;
1447	u16 len;
1448
1449	len = be16_to_cpu(crq->IU_length);
1450	if ((len > SRP_MAX_IU_LEN) || (len == 0)) {
1451	dev_err(&vscsi->dev, "copy_crq: Invalid len %d passed", len);
1452	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
1453	return SRP_VIOLATION;
1454	}
1455
1456	rc = h_copy_rdma(len, vscsi->dds.window[REMOTE].liobn,
1457	be64_to_cpu(crq->IU_data_ptr),
1458	vscsi->dds.window[LOCAL].liobn, iue->sbuf->dma);
1459
1460	switch (rc) {
1461	case H_SUCCESS:
1462	cmd->init_time = mftb();
1463	iue->remote_token = crq->IU_data_ptr;
1464	iue->iu_len = len;
1465	dev_dbg(&vscsi->dev, "copy_crq: ioba 0x%llx, init_time 0x%llx\n",
1466	be64_to_cpu(crq->IU_data_ptr), cmd->init_time);
1467	break;
1468	case H_PERMISSION:
1469	if (connection_broken(vscsi))
1470	ibmvscsis_post_disconnect(vscsi,
1471	ERR_DISCONNECT_RECONNECT,
1472	flag_bits: (RESPONSE_Q_DOWN |
1473	CLIENT_FAILED));
1474	else
1475	ibmvscsis_post_disconnect(vscsi,
1476	ERR_DISCONNECT_RECONNECT, flag_bits: 0);
1477
1478	dev_err(&vscsi->dev, "copy_crq: h_copy_rdma failed, rc %ld\n",
1479	rc);
1480	break;
1481	case H_DEST_PARM:
1482	case H_SOURCE_PARM:
1483	default:
1484	dev_err(&vscsi->dev, "copy_crq: h_copy_rdma failed, rc %ld\n",
1485	rc);
1486	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
1487	break;
1488	}
1489
1490	return rc;
1491	}
1492
1493	/**
1494	* ibmvscsis_adapter_info - Service an Adapter Info MAnagement Data gram
1495	* @vscsi: Pointer to our adapter structure
1496	* @iue: Information Unit containing the Adapter Info MAD request
1497	*
1498	* EXECUTION ENVIRONMENT:
1499	* Interrupt adapter lock is held
1500	*/
1501	static long ibmvscsis_adapter_info(struct scsi_info *vscsi,
1502	struct iu_entry *iue)
1503	{
1504	struct viosrp_adapter_info *mad = &vio_iu(iue)->mad.adapter_info;
1505	struct mad_adapter_info_data *info;
1506	uint flag_bits = 0;
1507	dma_addr_t token;
1508	long rc;
1509
1510	mad->common.status = cpu_to_be16(VIOSRP_MAD_SUCCESS);
1511
1512	if (be16_to_cpu(mad->common.length) > sizeof(*info)) {
1513	mad->common.status = cpu_to_be16(VIOSRP_MAD_FAILED);
1514	return 0;
1515	}
1516
1517	info = dma_alloc_coherent(&vscsi->dma_dev->dev, sizeof(*info), &token,
1518	GFP_ATOMIC);
1519	if (!info) {
1520	dev_err(&vscsi->dev, "bad dma_alloc_coherent %p\n",
1521	iue->target);
1522	mad->common.status = cpu_to_be16(VIOSRP_MAD_FAILED);
1523	return 0;
1524	}
1525
1526	/* Get remote info */
1527	rc = h_copy_rdma(be16_to_cpu(mad->common.length),
1528	vscsi->dds.window[REMOTE].liobn,
1529	be64_to_cpu(mad->buffer),
1530	vscsi->dds.window[LOCAL].liobn, token);
1531
1532	if (rc != H_SUCCESS) {
1533	if (rc == H_PERMISSION) {
1534	if (connection_broken(vscsi))
1535	flag_bits = (RESPONSE_Q_DOWN | CLIENT_FAILED);
1536	}
1537	dev_warn(&vscsi->dev, "adapter_info: h_copy_rdma from client failed, rc %ld\n",
1538	rc);
1539	dev_dbg(&vscsi->dev, "adapter_info: ioba 0x%llx, flags 0x%x, flag_bits 0x%x\n",
1540	be64_to_cpu(mad->buffer), vscsi->flags, flag_bits);
1541	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
1542	flag_bits);
1543	goto free_dma;
1544	}
1545
1546	/*
1547	* Copy client info, but ignore partition number, which we
1548	* already got from phyp - unless we failed to get it from
1549	* phyp (e.g. if we're running on a p5 system).
1550	*/
1551	if (vscsi->client_data.partition_number == 0)
1552	vscsi->client_data.partition_number =
1553	be32_to_cpu(info->partition_number);
1554	strncpy(p: vscsi->client_data.srp_version, q: info->srp_version,
1555	size: sizeof(vscsi->client_data.srp_version));
1556	strncpy(p: vscsi->client_data.partition_name, q: info->partition_name,
1557	size: sizeof(vscsi->client_data.partition_name));
1558	vscsi->client_data.mad_version = be32_to_cpu(info->mad_version);
1559	vscsi->client_data.os_type = be32_to_cpu(info->os_type);
1560
1561	/* Copy our info */
1562	strncpy(p: info->srp_version, SRP_VERSION,
1563	size: sizeof(info->srp_version));
1564	strncpy(p: info->partition_name, q: vscsi->dds.partition_name,
1565	size: sizeof(info->partition_name));
1566	info->partition_number = cpu_to_be32(vscsi->dds.partition_num);
1567	info->mad_version = cpu_to_be32(MAD_VERSION_1);
1568	info->os_type = cpu_to_be32(LINUX);
1569	memset(&info->port_max_txu[0], 0, sizeof(info->port_max_txu));
1570	info->port_max_txu[0] = cpu_to_be32(MAX_TXU);
1571
1572	dma_wmb();
1573	rc = h_copy_rdma(sizeof(*info), vscsi->dds.window[LOCAL].liobn,
1574	token, vscsi->dds.window[REMOTE].liobn,
1575	be64_to_cpu(mad->buffer));
1576	switch (rc) {
1577	case H_SUCCESS:
1578	break;
1579
1580	case H_SOURCE_PARM:
1581	case H_DEST_PARM:
1582	case H_PERMISSION:
1583	if (connection_broken(vscsi))
1584	flag_bits = (RESPONSE_Q_DOWN | CLIENT_FAILED);
1585	fallthrough;
1586	default:
1587	dev_err(&vscsi->dev, "adapter_info: h_copy_rdma to client failed, rc %ld\n",
1588	rc);
1589	ibmvscsis_post_disconnect(vscsi,
1590	ERR_DISCONNECT_RECONNECT,
1591	flag_bits);
1592	break;
1593	}
1594
1595	free_dma:
1596	dma_free_coherent(&vscsi->dma_dev->dev, sizeof(*info), info, token);
1597	dev_dbg(&vscsi->dev, "Leaving adapter_info, rc %ld\n", rc);
1598
1599	return rc;
1600	}
1601
1602	/**
1603	* ibmvscsis_cap_mad() - Service a Capabilities MAnagement Data gram
1604	* @vscsi: Pointer to our adapter structure
1605	* @iue: Information Unit containing the Capabilities MAD request
1606	*
1607	* NOTE: if you return an error from this routine you must be
1608	* disconnecting or you will cause a hang
1609	*
1610	* EXECUTION ENVIRONMENT:
1611	* Interrupt called with adapter lock held
1612	*/
1613	static int ibmvscsis_cap_mad(struct scsi_info *vscsi, struct iu_entry *iue)
1614	{
1615	struct viosrp_capabilities *mad = &vio_iu(iue)->mad.capabilities;
1616	struct capabilities *cap;
1617	struct mad_capability_common *common;
1618	dma_addr_t token;
1619	u16 olen, len, status, min_len, cap_len;
1620	u32 flag;
1621	uint flag_bits = 0;
1622	long rc = 0;
1623
1624	olen = be16_to_cpu(mad->common.length);
1625	/*
1626	* struct capabilities hardcodes a couple capabilities after the
1627	* header, but the capabilities can actually be in any order.
1628	*/
1629	min_len = offsetof(struct capabilities, migration);
1630	if ((olen < min_len) || (olen > PAGE_SIZE)) {
1631	dev_warn(&vscsi->dev, "cap_mad: invalid len %d\n", olen);
1632	mad->common.status = cpu_to_be16(VIOSRP_MAD_FAILED);
1633	return 0;
1634	}
1635
1636	cap = dma_alloc_coherent(&vscsi->dma_dev->dev, olen, &token,
1637	GFP_ATOMIC);
1638	if (!cap) {
1639	dev_err(&vscsi->dev, "bad dma_alloc_coherent %p\n",
1640	iue->target);
1641	mad->common.status = cpu_to_be16(VIOSRP_MAD_FAILED);
1642	return 0;
1643	}
1644	rc = h_copy_rdma(olen, vscsi->dds.window[REMOTE].liobn,
1645	be64_to_cpu(mad->buffer),
1646	vscsi->dds.window[LOCAL].liobn, token);
1647	if (rc == H_SUCCESS) {
1648	strncpy(cap->name, dev_name(&vscsi->dma_dev->dev),
1649	SRP_MAX_LOC_LEN);
1650
1651	len = olen - min_len;
1652	status = VIOSRP_MAD_SUCCESS;
1653	common = (struct mad_capability_common *)&cap->migration;
1654
1655	while ((len > 0) && (status == VIOSRP_MAD_SUCCESS) && !rc) {
1656	dev_dbg(&vscsi->dev, "cap_mad: len left %hd, cap type %d, cap len %hd\n",
1657	len, be32_to_cpu(common->cap_type),
1658	be16_to_cpu(common->length));
1659
1660	cap_len = be16_to_cpu(common->length);
1661	if (cap_len > len) {
1662	dev_err(&vscsi->dev, "cap_mad: cap len mismatch with total len\n");
1663	status = VIOSRP_MAD_FAILED;
1664	break;
1665	}
1666
1667	if (cap_len == 0) {
1668	dev_err(&vscsi->dev, "cap_mad: cap len is 0\n");
1669	status = VIOSRP_MAD_FAILED;
1670	break;
1671	}
1672
1673	switch (common->cap_type) {
1674	default:
1675	dev_dbg(&vscsi->dev, "cap_mad: unsupported capability\n");
1676	common->server_support = 0;
1677	flag = cpu_to_be32((u32)CAP_LIST_SUPPORTED);
1678	cap->flags &= ~flag;
1679	break;
1680	}
1681
1682	len = len - cap_len;
1683	common = (struct mad_capability_common *)
1684	((char *)common + cap_len);
1685	}
1686
1687	mad->common.status = cpu_to_be16(status);
1688
1689	dma_wmb();
1690	rc = h_copy_rdma(olen, vscsi->dds.window[LOCAL].liobn, token,
1691	vscsi->dds.window[REMOTE].liobn,
1692	be64_to_cpu(mad->buffer));
1693
1694	if (rc != H_SUCCESS) {
1695	dev_dbg(&vscsi->dev, "cap_mad: failed to copy to client, rc %ld\n",
1696	rc);
1697
1698	if (rc == H_PERMISSION) {
1699	if (connection_broken(vscsi))
1700	flag_bits = (RESPONSE_Q_DOWN |
1701	CLIENT_FAILED);
1702	}
1703
1704	dev_warn(&vscsi->dev, "cap_mad: error copying data to client, rc %ld\n",
1705	rc);
1706	ibmvscsis_post_disconnect(vscsi,
1707	ERR_DISCONNECT_RECONNECT,
1708	flag_bits);
1709	}
1710	}
1711
1712	dma_free_coherent(&vscsi->dma_dev->dev, olen, cap, token);
1713
1714	dev_dbg(&vscsi->dev, "Leaving cap_mad, rc %ld, client_cap 0x%x\n",
1715	rc, vscsi->client_cap);
1716
1717	return rc;
1718	}
1719
1720	/**
1721	* ibmvscsis_process_mad() - Service a MAnagement Data gram
1722	* @vscsi: Pointer to our adapter structure
1723	* @iue: Information Unit containing the MAD request
1724	*
1725	* Must be called with interrupt lock held.
1726	*/
1727	static long ibmvscsis_process_mad(struct scsi_info *vscsi, struct iu_entry *iue)
1728	{
1729	struct mad_common *mad = (struct mad_common *)&vio_iu(iue)->mad;
1730	struct viosrp_empty_iu *empty;
1731	long rc = ADAPT_SUCCESS;
1732
1733	switch (be32_to_cpu(mad->type)) {
1734	case VIOSRP_EMPTY_IU_TYPE:
1735	empty = &vio_iu(iue)->mad.empty_iu;
1736	vscsi->empty_iu_id = be64_to_cpu(empty->buffer);
1737	vscsi->empty_iu_tag = be64_to_cpu(empty->common.tag);
1738	mad->status = cpu_to_be16(VIOSRP_MAD_SUCCESS);
1739	break;
1740	case VIOSRP_ADAPTER_INFO_TYPE:
1741	rc = ibmvscsis_adapter_info(vscsi, iue);
1742	break;
1743	case VIOSRP_CAPABILITIES_TYPE:
1744	rc = ibmvscsis_cap_mad(vscsi, iue);
1745	break;
1746	case VIOSRP_ENABLE_FAST_FAIL:
1747	if (vscsi->state == CONNECTED) {
1748	vscsi->fast_fail = true;
1749	mad->status = cpu_to_be16(VIOSRP_MAD_SUCCESS);
1750	} else {
1751	dev_warn(&vscsi->dev, "fast fail mad sent after login\n");
1752	mad->status = cpu_to_be16(VIOSRP_MAD_FAILED);
1753	}
1754	break;
1755	default:
1756	mad->status = cpu_to_be16(VIOSRP_MAD_NOT_SUPPORTED);
1757	break;
1758	}
1759
1760	return rc;
1761	}
1762
1763	/**
1764	* srp_snd_msg_failed() - Handle an error when sending a response
1765	* @vscsi: Pointer to our adapter structure
1766	* @rc: The return code from the h_send_crq command
1767	*
1768	* Must be called with interrupt lock held.
1769	*/
1770	static void srp_snd_msg_failed(struct scsi_info *vscsi, long rc)
1771	{
1772	ktime_t kt;
1773
1774	if (rc != H_DROPPED) {
1775	ibmvscsis_free_cmd_qs(vscsi);
1776
1777	if (rc == H_CLOSED)
1778	vscsi->flags |= CLIENT_FAILED;
1779
1780	/* don't flag the same problem multiple times */
1781	if (!(vscsi->flags & RESPONSE_Q_DOWN)) {
1782	vscsi->flags |= RESPONSE_Q_DOWN;
1783	if (!(vscsi->state & (ERR_DISCONNECT |
1784	ERR_DISCONNECT_RECONNECT |
1785	ERR_DISCONNECTED | UNDEFINED))) {
1786	dev_err(&vscsi->dev, "snd_msg_failed: setting RESPONSE_Q_DOWN, state 0x%hx, flags 0x%x, rc %ld\n",
1787	vscsi->state, vscsi->flags, rc);
1788	}
1789	ibmvscsis_post_disconnect(vscsi,
1790	ERR_DISCONNECT_RECONNECT, flag_bits: 0);
1791	}
1792	return;
1793	}
1794
1795	/*
1796	* The response queue is full.
1797	* If the server is processing SRP requests, i.e.
1798	* the client has successfully done an
1799	* SRP_LOGIN, then it will wait forever for room in
1800	* the queue. However if the system admin
1801	* is attempting to unconfigure the server then one
1802	* or more children will be in a state where
1803	* they are being removed. So if there is even one
1804	* child being removed then the driver assumes
1805	* the system admin is attempting to break the
1806	* connection with the client and MAX_TIMER_POPS
1807	* is honored.
1808	*/
1809	if ((vscsi->rsp_q_timer.timer_pops < MAX_TIMER_POPS) ||
1810	(vscsi->state == SRP_PROCESSING)) {
1811	dev_dbg(&vscsi->dev, "snd_msg_failed: response queue full, flags 0x%x, timer started %d, pops %d\n",
1812	vscsi->flags, (int)vscsi->rsp_q_timer.started,
1813	vscsi->rsp_q_timer.timer_pops);
1814
1815	/*
1816	* Check if the timer is running; if it
1817	* is not then start it up.
1818	*/
1819	if (!vscsi->rsp_q_timer.started) {
1820	if (vscsi->rsp_q_timer.timer_pops <
1821	MAX_TIMER_POPS) {
1822	kt = WAIT_NANO_SECONDS;
1823	} else {
1824	/*
1825	* slide the timeslice if the maximum
1826	* timer pops have already happened
1827	*/
1828	kt = ktime_set(WAIT_SECONDS, nsecs: 0);
1829	}
1830
1831	vscsi->rsp_q_timer.started = true;
1832	hrtimer_start(timer: &vscsi->rsp_q_timer.timer, tim: kt,
1833	mode: HRTIMER_MODE_REL);
1834	}
1835	} else {
1836	/*
1837	* TBD: Do we need to worry about this? Need to get
1838	* remove working.
1839	*/
1840	/*
1841	* waited a long time and it appears the system admin
1842	* is bring this driver down
1843	*/
1844	vscsi->flags |= RESPONSE_Q_DOWN;
1845	ibmvscsis_free_cmd_qs(vscsi);
1846	/*
1847	* if the driver is already attempting to disconnect
1848	* from the client and has already logged an error
1849	* trace this event but don't put it in the error log
1850	*/
1851	if (!(vscsi->state & (ERR_DISCONNECT |
1852	ERR_DISCONNECT_RECONNECT |
1853	ERR_DISCONNECTED | UNDEFINED))) {
1854	dev_err(&vscsi->dev, "client crq full too long\n");
1855	ibmvscsis_post_disconnect(vscsi,
1856	ERR_DISCONNECT_RECONNECT,
1857	flag_bits: 0);
1858	}
1859	}
1860	}
1861
1862	/**
1863	* ibmvscsis_send_messages() - Send a Response
1864	* @vscsi: Pointer to our adapter structure
1865	*
1866	* Send a response, first checking the waiting queue. Responses are
1867	* sent in order they are received. If the response cannot be sent,
1868	* because the client queue is full, it stays on the waiting queue.
1869	*
1870	* PRECONDITION:
1871	* Called with interrupt lock held
1872	*/
1873	static void ibmvscsis_send_messages(struct scsi_info *vscsi)
1874	{
1875	struct viosrp_crq empty_crq = { };
1876	struct viosrp_crq *crq = &empty_crq;
1877	struct ibmvscsis_cmd *cmd, *nxt;
1878	long rc = ADAPT_SUCCESS;
1879	bool retry = false;
1880
1881	if (!(vscsi->flags & RESPONSE_Q_DOWN)) {
1882	do {
1883	retry = false;
1884	list_for_each_entry_safe(cmd, nxt, &vscsi->waiting_rsp,
1885	list) {
1886	/*
1887	* Check to make sure abort cmd gets processed
1888	* prior to the abort tmr cmd
1889	*/
1890	if (cmd->flags & DELAY_SEND)
1891	continue;
1892
1893	if (cmd->abort_cmd) {
1894	retry = true;
1895	cmd->abort_cmd->flags &= ~(DELAY_SEND);
1896	cmd->abort_cmd = NULL;
1897	}
1898
1899	/*
1900	* If CMD_T_ABORTED w/o CMD_T_TAS scenarios and
1901	* the case where LIO issued a
1902	* ABORT_TASK: Sending TMR_TASK_DOES_NOT_EXIST
1903	* case then we dont send a response, since it
1904	* was already done.
1905	*/
1906	if (cmd->se_cmd.transport_state & CMD_T_ABORTED &&
1907	!(cmd->se_cmd.transport_state & CMD_T_TAS)) {
1908	list_del(entry: &cmd->list);
1909	ibmvscsis_free_cmd_resources(vscsi,
1910	cmd);
1911	/*
1912	* With a successfully aborted op
1913	* through LIO we want to increment the
1914	* the vscsi credit so that when we dont
1915	* send a rsp to the original scsi abort
1916	* op (h_send_crq), but the tm rsp to
1917	* the abort is sent, the credit is
1918	* correctly sent with the abort tm rsp.
1919	* We would need 1 for the abort tm rsp
1920	* and 1 credit for the aborted scsi op.
1921	* Thus we need to increment here.
1922	* Also we want to increment the credit
1923	* here because we want to make sure
1924	* cmd is actually released first
1925	* otherwise the client will think it
1926	* it can send a new cmd, and we could
1927	* find ourselves short of cmd elements.
1928	*/
1929	vscsi->credit += 1;
1930	} else {
1931	crq->valid = VALID_CMD_RESP_EL;
1932	crq->format = cmd->rsp.format;
1933
1934	if (cmd->flags & CMD_FAST_FAIL)
1935	crq->status = VIOSRP_ADAPTER_FAIL;
1936
1937	crq->IU_length = cpu_to_be16(cmd->rsp.len);
1938
1939	rc = h_send_crq(vscsi->dma_dev->unit_address,
1940	be64_to_cpu(crq->high),
1941	be64_to_cpu(cmd->rsp.tag));
1942
1943	dev_dbg(&vscsi->dev, "send_messages: cmd %p, tag 0x%llx, rc %ld\n",
1944	cmd, be64_to_cpu(cmd->rsp.tag),
1945	rc);
1946
1947	/* if all ok free up the command
1948	* element resources
1949	*/
1950	if (rc == H_SUCCESS) {
1951	/* some movement has occurred */
1952	vscsi->rsp_q_timer.timer_pops = 0;
1953	list_del(entry: &cmd->list);
1954
1955	ibmvscsis_free_cmd_resources(vscsi,
1956	cmd);
1957	} else {
1958	srp_snd_msg_failed(vscsi, rc);
1959	break;
1960	}
1961	}
1962	}
1963	} while (retry);
1964
1965	if (!rc) {
1966	/*
1967	* The timer could pop with the queue empty. If
1968	* this happens, rc will always indicate a
1969	* success; clear the pop count.
1970	*/
1971	vscsi->rsp_q_timer.timer_pops = 0;
1972	}
1973	} else {
1974	ibmvscsis_free_cmd_qs(vscsi);
1975	}
1976	}
1977
1978	/* Called with intr lock held */
1979	static void ibmvscsis_send_mad_resp(struct scsi_info *vscsi,
1980	struct ibmvscsis_cmd *cmd,
1981	struct viosrp_crq *crq)
1982	{
1983	struct iu_entry *iue = cmd->iue;
1984	struct mad_common *mad = (struct mad_common *)&vio_iu(iue)->mad;
1985	uint flag_bits = 0;
1986	long rc;
1987
1988	dma_wmb();
1989	rc = h_copy_rdma(sizeof(struct mad_common),
1990	vscsi->dds.window[LOCAL].liobn, iue->sbuf->dma,
1991	vscsi->dds.window[REMOTE].liobn,
1992	be64_to_cpu(crq->IU_data_ptr));
1993	if (!rc) {
1994	cmd->rsp.format = VIOSRP_MAD_FORMAT;
1995	cmd->rsp.len = sizeof(struct mad_common);
1996	cmd->rsp.tag = mad->tag;
1997	list_add_tail(new: &cmd->list, head: &vscsi->waiting_rsp);
1998	ibmvscsis_send_messages(vscsi);
1999	} else {
2000	dev_dbg(&vscsi->dev, "Error sending mad response, rc %ld\n",
2001	rc);
2002	if (rc == H_PERMISSION) {
2003	if (connection_broken(vscsi))
2004	flag_bits = (RESPONSE_Q_DOWN | CLIENT_FAILED);
2005	}
2006	dev_err(&vscsi->dev, "mad: failed to copy to client, rc %ld\n",
2007	rc);
2008
2009	ibmvscsis_free_cmd_resources(vscsi, cmd);
2010	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
2011	flag_bits);
2012	}
2013	}
2014
2015	/**
2016	* ibmvscsis_mad() - Service a MAnagement Data gram.
2017	* @vscsi: Pointer to our adapter structure
2018	* @crq: Pointer to the CRQ entry containing the MAD request
2019	*
2020	* EXECUTION ENVIRONMENT:
2021	* Interrupt, called with adapter lock held
2022	*/
2023	static long ibmvscsis_mad(struct scsi_info *vscsi, struct viosrp_crq *crq)
2024	{
2025	struct iu_entry *iue;
2026	struct ibmvscsis_cmd *cmd;
2027	struct mad_common *mad;
2028	long rc = ADAPT_SUCCESS;
2029
2030	switch (vscsi->state) {
2031	/*
2032	* We have not exchanged Init Msgs yet, so this MAD was sent
2033	* before the last Transport Event; client will not be
2034	* expecting a response.
2035	*/
2036	case WAIT_CONNECTION:
2037	dev_dbg(&vscsi->dev, "mad: in Wait Connection state, ignoring MAD, flags %d\n",
2038	vscsi->flags);
2039	return ADAPT_SUCCESS;
2040
2041	case SRP_PROCESSING:
2042	case CONNECTED:
2043	break;
2044
2045	/*
2046	* We should never get here while we're in these states.
2047	* Just log an error and get out.
2048	*/
2049	case UNCONFIGURING:
2050	case WAIT_IDLE:
2051	case ERR_DISCONNECT:
2052	case ERR_DISCONNECT_RECONNECT:
2053	default:
2054	dev_err(&vscsi->dev, "mad: invalid adapter state %d for mad\n",
2055	vscsi->state);
2056	return ADAPT_SUCCESS;
2057	}
2058
2059	cmd = ibmvscsis_get_free_cmd(vscsi);
2060	if (!cmd) {
2061	dev_err(&vscsi->dev, "mad: failed to get cmd, debit %d\n",
2062	vscsi->debit);
2063	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
2064	return ERROR;
2065	}
2066	iue = cmd->iue;
2067	cmd->type = ADAPTER_MAD;
2068
2069	rc = ibmvscsis_copy_crq_packet(vscsi, cmd, crq);
2070	if (!rc) {
2071	mad = (struct mad_common *)&vio_iu(iue)->mad;
2072
2073	dev_dbg(&vscsi->dev, "mad: type %d\n", be32_to_cpu(mad->type));
2074
2075	rc = ibmvscsis_process_mad(vscsi, iue);
2076
2077	dev_dbg(&vscsi->dev, "mad: status %hd, rc %ld\n",
2078	be16_to_cpu(mad->status), rc);
2079
2080	if (!rc)
2081	ibmvscsis_send_mad_resp(vscsi, cmd, crq);
2082	} else {
2083	ibmvscsis_free_cmd_resources(vscsi, cmd);
2084	}
2085
2086	dev_dbg(&vscsi->dev, "Leaving mad, rc %ld\n", rc);
2087	return rc;
2088	}
2089
2090	/**
2091	* ibmvscsis_login_rsp() - Create/copy a login response notice to the client
2092	* @vscsi: Pointer to our adapter structure
2093	* @cmd: Pointer to the command for the SRP Login request
2094	*
2095	* EXECUTION ENVIRONMENT:
2096	* Interrupt, interrupt lock held
2097	*/
2098	static long ibmvscsis_login_rsp(struct scsi_info *vscsi,
2099	struct ibmvscsis_cmd *cmd)
2100	{
2101	struct iu_entry *iue = cmd->iue;
2102	struct srp_login_rsp *rsp = &vio_iu(iue)->srp.login_rsp;
2103	struct format_code *fmt;
2104	uint flag_bits = 0;
2105	long rc = ADAPT_SUCCESS;
2106
2107	memset(rsp, 0, sizeof(struct srp_login_rsp));
2108
2109	rsp->opcode = SRP_LOGIN_RSP;
2110	rsp->req_lim_delta = cpu_to_be32(vscsi->request_limit);
2111	rsp->tag = cmd->rsp.tag;
2112	rsp->max_it_iu_len = cpu_to_be32(SRP_MAX_IU_LEN);
2113	rsp->max_ti_iu_len = cpu_to_be32(SRP_MAX_IU_LEN);
2114	fmt = (struct format_code *)&rsp->buf_fmt;
2115	fmt->buffers = SUPPORTED_FORMATS;
2116	vscsi->credit = 0;
2117
2118	cmd->rsp.len = sizeof(struct srp_login_rsp);
2119
2120	dma_wmb();
2121	rc = h_copy_rdma(cmd->rsp.len, vscsi->dds.window[LOCAL].liobn,
2122	iue->sbuf->dma, vscsi->dds.window[REMOTE].liobn,
2123	be64_to_cpu(iue->remote_token));
2124
2125	switch (rc) {
2126	case H_SUCCESS:
2127	break;
2128
2129	case H_PERMISSION:
2130	if (connection_broken(vscsi))
2131	flag_bits = RESPONSE_Q_DOWN | CLIENT_FAILED;
2132	dev_err(&vscsi->dev, "login_rsp: error copying to client, rc %ld\n",
2133	rc);
2134	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
2135	flag_bits);
2136	break;
2137	case H_SOURCE_PARM:
2138	case H_DEST_PARM:
2139	default:
2140	dev_err(&vscsi->dev, "login_rsp: error copying to client, rc %ld\n",
2141	rc);
2142	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
2143	break;
2144	}
2145
2146	return rc;
2147	}
2148
2149	/**
2150	* ibmvscsis_srp_login_rej() - Create/copy a login rejection notice to client
2151	* @vscsi: Pointer to our adapter structure
2152	* @cmd: Pointer to the command for the SRP Login request
2153	* @reason: The reason the SRP Login is being rejected, per SRP protocol
2154	*
2155	* EXECUTION ENVIRONMENT:
2156	* Interrupt, interrupt lock held
2157	*/
2158	static long ibmvscsis_srp_login_rej(struct scsi_info *vscsi,
2159	struct ibmvscsis_cmd *cmd, u32 reason)
2160	{
2161	struct iu_entry *iue = cmd->iue;
2162	struct srp_login_rej *rej = &vio_iu(iue)->srp.login_rej;
2163	struct format_code *fmt;
2164	uint flag_bits = 0;
2165	long rc = ADAPT_SUCCESS;
2166
2167	memset(rej, 0, sizeof(*rej));
2168
2169	rej->opcode = SRP_LOGIN_REJ;
2170	rej->reason = cpu_to_be32(reason);
2171	rej->tag = cmd->rsp.tag;
2172	fmt = (struct format_code *)&rej->buf_fmt;
2173	fmt->buffers = SUPPORTED_FORMATS;
2174
2175	cmd->rsp.len = sizeof(*rej);
2176
2177	dma_wmb();
2178	rc = h_copy_rdma(cmd->rsp.len, vscsi->dds.window[LOCAL].liobn,
2179	iue->sbuf->dma, vscsi->dds.window[REMOTE].liobn,
2180	be64_to_cpu(iue->remote_token));
2181
2182	switch (rc) {
2183	case H_SUCCESS:
2184	break;
2185	case H_PERMISSION:
2186	if (connection_broken(vscsi))
2187	flag_bits = RESPONSE_Q_DOWN | CLIENT_FAILED;
2188	dev_err(&vscsi->dev, "login_rej: error copying to client, rc %ld\n",
2189	rc);
2190	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
2191	flag_bits);
2192	break;
2193	case H_SOURCE_PARM:
2194	case H_DEST_PARM:
2195	default:
2196	dev_err(&vscsi->dev, "login_rej: error copying to client, rc %ld\n",
2197	rc);
2198	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
2199	break;
2200	}
2201
2202	return rc;
2203	}
2204
2205	static int ibmvscsis_make_nexus(struct ibmvscsis_tport *tport)
2206	{
2207	char *name = tport->tport_name;
2208	struct ibmvscsis_nexus *nexus;
2209	struct scsi_info *vscsi = container_of(tport, struct scsi_info, tport);
2210	int rc;
2211
2212	if (tport->ibmv_nexus) {
2213	dev_dbg(&vscsi->dev, "tport->ibmv_nexus already exists\n");
2214	return 0;
2215	}
2216
2217	nexus = kzalloc(size: sizeof(*nexus), GFP_KERNEL);
2218	if (!nexus) {
2219	dev_err(&vscsi->dev, "Unable to allocate struct ibmvscsis_nexus\n");
2220	return -ENOMEM;
2221	}
2222
2223	nexus->se_sess = target_setup_session(&tport->se_tpg, 0, 0,
2224	prot_op: TARGET_PROT_NORMAL, name, nexus,
2225	NULL);
2226	if (IS_ERR(ptr: nexus->se_sess)) {
2227	rc = PTR_ERR(ptr: nexus->se_sess);
2228	goto transport_init_fail;
2229	}
2230
2231	tport->ibmv_nexus = nexus;
2232
2233	return 0;
2234
2235	transport_init_fail:
2236	kfree(objp: nexus);
2237	return rc;
2238	}
2239
2240	static int ibmvscsis_drop_nexus(struct ibmvscsis_tport *tport)
2241	{
2242	struct se_session *se_sess;
2243	struct ibmvscsis_nexus *nexus;
2244
2245	nexus = tport->ibmv_nexus;
2246	if (!nexus)
2247	return -ENODEV;
2248
2249	se_sess = nexus->se_sess;
2250	if (!se_sess)
2251	return -ENODEV;
2252
2253	/*
2254	* Release the SCSI I_T Nexus to the emulated ibmvscsis Target Port
2255	*/
2256	target_remove_session(se_sess);
2257	tport->ibmv_nexus = NULL;
2258	kfree(objp: nexus);
2259
2260	return 0;
2261	}
2262
2263	/**
2264	* ibmvscsis_srp_login() - Process an SRP Login Request
2265	* @vscsi: Pointer to our adapter structure
2266	* @cmd: Command element to use to process the SRP Login request
2267	* @crq: Pointer to CRQ entry containing the SRP Login request
2268	*
2269	* EXECUTION ENVIRONMENT:
2270	* Interrupt, called with interrupt lock held
2271	*/
2272	static long ibmvscsis_srp_login(struct scsi_info *vscsi,
2273	struct ibmvscsis_cmd *cmd,
2274	struct viosrp_crq *crq)
2275	{
2276	struct iu_entry *iue = cmd->iue;
2277	struct srp_login_req *req = &vio_iu(iue)->srp.login_req;
2278	struct port_id {
2279	__be64 id_extension;
2280	__be64 io_guid;
2281	} *iport, *tport;
2282	struct format_code *fmt;
2283	u32 reason = 0x0;
2284	long rc = ADAPT_SUCCESS;
2285
2286	iport = (struct port_id *)req->initiator_port_id;
2287	tport = (struct port_id *)req->target_port_id;
2288	fmt = (struct format_code *)&req->req_buf_fmt;
2289	if (be32_to_cpu(req->req_it_iu_len) > SRP_MAX_IU_LEN)
2290	reason = SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE;
2291	else if (be32_to_cpu(req->req_it_iu_len) < 64)
2292	reason = SRP_LOGIN_REJ_UNABLE_ESTABLISH_CHANNEL;
2293	else if ((be64_to_cpu(iport->id_extension) > (MAX_NUM_PORTS - 1)) ||
2294	(be64_to_cpu(tport->id_extension) > (MAX_NUM_PORTS - 1)))
2295	reason = SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL;
2296	else if (req->req_flags & SRP_MULTICHAN_MULTI)
2297	reason = SRP_LOGIN_REJ_MULTI_CHANNEL_UNSUPPORTED;
2298	else if (fmt->buffers & (~SUPPORTED_FORMATS))
2299	reason = SRP_LOGIN_REJ_UNSUPPORTED_DESCRIPTOR_FMT;
2300	else if ((fmt->buffers & SUPPORTED_FORMATS) == 0)
2301	reason = SRP_LOGIN_REJ_UNSUPPORTED_DESCRIPTOR_FMT;
2302
2303	if (vscsi->state == SRP_PROCESSING)
2304	reason = SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED;
2305
2306	rc = ibmvscsis_make_nexus(tport: &vscsi->tport);
2307	if (rc)
2308	reason = SRP_LOGIN_REJ_UNABLE_ESTABLISH_CHANNEL;
2309
2310	cmd->rsp.format = VIOSRP_SRP_FORMAT;
2311	cmd->rsp.tag = req->tag;
2312
2313	dev_dbg(&vscsi->dev, "srp_login: reason 0x%x\n", reason);
2314
2315	if (reason)
2316	rc = ibmvscsis_srp_login_rej(vscsi, cmd, reason);
2317	else
2318	rc = ibmvscsis_login_rsp(vscsi, cmd);
2319
2320	if (!rc) {
2321	if (!reason)
2322	vscsi->state = SRP_PROCESSING;
2323
2324	list_add_tail(new: &cmd->list, head: &vscsi->waiting_rsp);
2325	ibmvscsis_send_messages(vscsi);
2326	} else {
2327	ibmvscsis_free_cmd_resources(vscsi, cmd);
2328	}
2329
2330	dev_dbg(&vscsi->dev, "Leaving srp_login, rc %ld\n", rc);
2331	return rc;
2332	}
2333
2334	/**
2335	* ibmvscsis_srp_i_logout() - Helper Function to close I_T Nexus
2336	* @vscsi: Pointer to our adapter structure
2337	* @cmd: Command element to use to process the Implicit Logout request
2338	* @crq: Pointer to CRQ entry containing the Implicit Logout request
2339	*
2340	* Do the logic to close the I_T nexus. This function may not
2341	* behave to specification.
2342	*
2343	* EXECUTION ENVIRONMENT:
2344	* Interrupt, interrupt lock held
2345	*/
2346	static long ibmvscsis_srp_i_logout(struct scsi_info *vscsi,
2347	struct ibmvscsis_cmd *cmd,
2348	struct viosrp_crq *crq)
2349	{
2350	struct iu_entry *iue = cmd->iue;
2351	struct srp_i_logout *log_out = &vio_iu(iue)->srp.i_logout;
2352
2353	if ((vscsi->debit > 0) || !list_empty(head: &vscsi->schedule_q) ||
2354	!list_empty(head: &vscsi->waiting_rsp)) {
2355	dev_err(&vscsi->dev, "i_logout: outstanding work\n");
2356	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, flag_bits: 0);
2357	} else {
2358	cmd->rsp.format = SRP_FORMAT;
2359	cmd->rsp.tag = log_out->tag;
2360	cmd->rsp.len = sizeof(struct mad_common);
2361	list_add_tail(new: &cmd->list, head: &vscsi->waiting_rsp);
2362	ibmvscsis_send_messages(vscsi);
2363
2364	ibmvscsis_post_disconnect(vscsi, WAIT_IDLE, flag_bits: 0);
2365	}
2366
2367	return ADAPT_SUCCESS;
2368	}
2369
2370	/* Called with intr lock held */
2371	static void ibmvscsis_srp_cmd(struct scsi_info *vscsi, struct viosrp_crq *crq)
2372	{
2373	struct ibmvscsis_cmd *cmd;
2374	struct iu_entry *iue;
2375	struct srp_cmd *srp;
2376	struct srp_tsk_mgmt *tsk;
2377	long rc;
2378
2379	if (vscsi->request_limit - vscsi->debit <= 0) {
2380	/* Client has exceeded request limit */
2381	dev_err(&vscsi->dev, "Client exceeded the request limit (%d), debit %d\n",
2382	vscsi->request_limit, vscsi->debit);
2383	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
2384	return;
2385	}
2386
2387	cmd = ibmvscsis_get_free_cmd(vscsi);
2388	if (!cmd) {
2389	dev_err(&vscsi->dev, "srp_cmd failed to get cmd, debit %d\n",
2390	vscsi->debit);
2391	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
2392	return;
2393	}
2394	iue = cmd->iue;
2395	srp = &vio_iu(iue)->srp.cmd;
2396
2397	rc = ibmvscsis_copy_crq_packet(vscsi, cmd, crq);
2398	if (rc) {
2399	ibmvscsis_free_cmd_resources(vscsi, cmd);
2400	return;
2401	}
2402
2403	if (vscsi->state == SRP_PROCESSING) {
2404	switch (srp->opcode) {
2405	case SRP_LOGIN_REQ:
2406	rc = ibmvscsis_srp_login(vscsi, cmd, crq);
2407	break;
2408
2409	case SRP_TSK_MGMT:
2410	tsk = &vio_iu(iue)->srp.tsk_mgmt;
2411	dev_dbg(&vscsi->dev, "tsk_mgmt tag: %llu (0x%llx)\n",
2412	tsk->tag, tsk->tag);
2413	cmd->rsp.tag = tsk->tag;
2414	vscsi->debit += 1;
2415	cmd->type = TASK_MANAGEMENT;
2416	list_add_tail(new: &cmd->list, head: &vscsi->schedule_q);
2417	queue_work(wq: vscsi->work_q, work: &cmd->work);
2418	break;
2419
2420	case SRP_CMD:
2421	dev_dbg(&vscsi->dev, "srp_cmd tag: %llu (0x%llx)\n",
2422	srp->tag, srp->tag);
2423	cmd->rsp.tag = srp->tag;
2424	vscsi->debit += 1;
2425	cmd->type = SCSI_CDB;
2426	/*
2427	* We want to keep track of work waiting for
2428	* the workqueue.
2429	*/
2430	list_add_tail(new: &cmd->list, head: &vscsi->schedule_q);
2431	queue_work(wq: vscsi->work_q, work: &cmd->work);
2432	break;
2433
2434	case SRP_I_LOGOUT:
2435	rc = ibmvscsis_srp_i_logout(vscsi, cmd, crq);
2436	break;
2437
2438	case SRP_CRED_RSP:
2439	case SRP_AER_RSP:
2440	default:
2441	ibmvscsis_free_cmd_resources(vscsi, cmd);
2442	dev_err(&vscsi->dev, "invalid srp cmd, opcode %d\n",
2443	(uint)srp->opcode);
2444	ibmvscsis_post_disconnect(vscsi,
2445	ERR_DISCONNECT_RECONNECT, flag_bits: 0);
2446	break;
2447	}
2448	} else if (srp->opcode == SRP_LOGIN_REQ && vscsi->state == CONNECTED) {
2449	rc = ibmvscsis_srp_login(vscsi, cmd, crq);
2450	} else {
2451	ibmvscsis_free_cmd_resources(vscsi, cmd);
2452	dev_err(&vscsi->dev, "Invalid state %d to handle srp cmd\n",
2453	vscsi->state);
2454	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
2455	}
2456	}
2457
2458	/**
2459	* ibmvscsis_ping_response() - Respond to a ping request
2460	* @vscsi: Pointer to our adapter structure
2461	*
2462	* Let the client know that the server is alive and waiting on
2463	* its native I/O stack.
2464	* If any type of error occurs from the call to queue a ping
2465	* response then the client is either not accepting or receiving
2466	* interrupts. Disconnect with an error.
2467	*
2468	* EXECUTION ENVIRONMENT:
2469	* Interrupt, interrupt lock held
2470	*/
2471	static long ibmvscsis_ping_response(struct scsi_info *vscsi)
2472	{
2473	struct viosrp_crq *crq;
2474	u64 buffer[2] = { 0, 0 };
2475	long rc;
2476
2477	crq = (struct viosrp_crq *)&buffer;
2478	crq->valid = VALID_CMD_RESP_EL;
2479	crq->format = (u8)MESSAGE_IN_CRQ;
2480	crq->status = PING_RESPONSE;
2481
2482	rc = h_send_crq(vscsi->dds.unit_id, cpu_to_be64(buffer[MSG_HI]),
2483	cpu_to_be64(buffer[MSG_LOW]));
2484
2485	switch (rc) {
2486	case H_SUCCESS:
2487	break;
2488	case H_CLOSED:
2489	vscsi->flags |= CLIENT_FAILED;
2490	fallthrough;
2491	case H_DROPPED:
2492	vscsi->flags |= RESPONSE_Q_DOWN;
2493	fallthrough;
2494	case H_REMOTE_PARM:
2495	dev_err(&vscsi->dev, "ping_response: h_send_crq failed, rc %ld\n",
2496	rc);
2497	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
2498	break;
2499	default:
2500	dev_err(&vscsi->dev, "ping_response: h_send_crq returned unknown rc %ld\n",
2501	rc);
2502	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, flag_bits: 0);
2503	break;
2504	}
2505
2506	return rc;
2507	}
2508
2509	/**
2510	* ibmvscsis_parse_command() - Parse an element taken from the cmd rsp queue.
2511	* @vscsi: Pointer to our adapter structure
2512	* @crq: Pointer to CRQ element containing the SRP request
2513	*
2514	* This function will return success if the command queue element is valid
2515	* and the srp iu or MAD request it pointed to was also valid. That does
2516	* not mean that an error was not returned to the client.
2517	*
2518	* EXECUTION ENVIRONMENT:
2519	* Interrupt, intr lock held
2520	*/
2521	static long ibmvscsis_parse_command(struct scsi_info *vscsi,
2522	struct viosrp_crq *crq)
2523	{
2524	long rc = ADAPT_SUCCESS;
2525
2526	switch (crq->valid) {
2527	case VALID_CMD_RESP_EL:
2528	switch (crq->format) {
2529	case OS400_FORMAT:
2530	case AIX_FORMAT:
2531	case LINUX_FORMAT:
2532	case MAD_FORMAT:
2533	if (vscsi->flags & PROCESSING_MAD) {
2534	rc = ERROR;
2535	dev_err(&vscsi->dev, "parse_command: already processing mad\n");
2536	ibmvscsis_post_disconnect(vscsi,
2537	ERR_DISCONNECT_RECONNECT,
2538	flag_bits: 0);
2539	} else {
2540	vscsi->flags |= PROCESSING_MAD;
2541	rc = ibmvscsis_mad(vscsi, crq);
2542	}
2543	break;
2544
2545	case SRP_FORMAT:
2546	ibmvscsis_srp_cmd(vscsi, crq);
2547	break;
2548
2549	case MESSAGE_IN_CRQ:
2550	if (crq->status == PING)
2551	ibmvscsis_ping_response(vscsi);
2552	break;
2553
2554	default:
2555	dev_err(&vscsi->dev, "parse_command: invalid format %d\n",
2556	(uint)crq->format);
2557	ibmvscsis_post_disconnect(vscsi,
2558	ERR_DISCONNECT_RECONNECT, flag_bits: 0);
2559	break;
2560	}
2561	break;
2562
2563	case VALID_TRANS_EVENT:
2564	rc = ibmvscsis_trans_event(vscsi, crq);
2565	break;
2566
2567	case VALID_INIT_MSG:
2568	rc = ibmvscsis_init_msg(vscsi, crq);
2569	break;
2570
2571	default:
2572	dev_err(&vscsi->dev, "parse_command: invalid valid field %d\n",
2573	(uint)crq->valid);
2574	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
2575	break;
2576	}
2577
2578	/*
2579	* Return only what the interrupt handler cares
2580	* about. Most errors we keep right on trucking.
2581	*/
2582	rc = vscsi->flags & SCHEDULE_DISCONNECT;
2583
2584	return rc;
2585	}
2586
2587	static int read_dma_window(struct scsi_info *vscsi)
2588	{
2589	struct vio_dev *vdev = vscsi->dma_dev;
2590	const __be32 *dma_window;
2591	const __be32 *prop;
2592
2593	/* TODO Using of_parse_dma_window would be better, but it doesn't give
2594	* a way to read multiple windows without already knowing the size of
2595	* a window or the number of windows.
2596	*/
2597	dma_window = (const __be32 *)vio_get_attribute(vdev,
2598	"ibm,my-dma-window",
2599	NULL);
2600	if (!dma_window) {
2601	dev_err(&vscsi->dev, "Couldn't find ibm,my-dma-window property\n");
2602	return -1;
2603	}
2604
2605	vscsi->dds.window[LOCAL].liobn = be32_to_cpu(*dma_window);
2606	dma_window++;
2607
2608	prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-address-cells",
2609	NULL);
2610	if (!prop) {
2611	dev_warn(&vscsi->dev, "Couldn't find ibm,#dma-address-cells property\n");
2612	dma_window++;
2613	} else {
2614	dma_window += be32_to_cpu(*prop);
2615	}
2616
2617	prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-size-cells",
2618	NULL);
2619	if (!prop) {
2620	dev_warn(&vscsi->dev, "Couldn't find ibm,#dma-size-cells property\n");
2621	dma_window++;
2622	} else {
2623	dma_window += be32_to_cpu(*prop);
2624	}
2625
2626	/* dma_window should point to the second window now */
2627	vscsi->dds.window[REMOTE].liobn = be32_to_cpu(*dma_window);
2628
2629	return 0;
2630	}
2631
2632	static struct ibmvscsis_tport *ibmvscsis_lookup_port(const char *name)
2633	{
2634	struct ibmvscsis_tport *tport = NULL;
2635	struct vio_dev *vdev;
2636	struct scsi_info *vscsi;
2637
2638	spin_lock_bh(lock: &ibmvscsis_dev_lock);
2639	list_for_each_entry(vscsi, &ibmvscsis_dev_list, list) {
2640	vdev = vscsi->dma_dev;
2641	if (!strcmp(dev_name(&vdev->dev), name)) {
2642	tport = &vscsi->tport;
2643	break;
2644	}
2645	}
2646	spin_unlock_bh(lock: &ibmvscsis_dev_lock);
2647
2648	return tport;
2649	}
2650
2651	/**
2652	* ibmvscsis_parse_cmd() - Parse SRP Command
2653	* @vscsi: Pointer to our adapter structure
2654	* @cmd: Pointer to command element with SRP command
2655	*
2656	* Parse the srp command; if it is valid then submit it to tcm.
2657	* Note: The return code does not reflect the status of the SCSI CDB.
2658	*
2659	* EXECUTION ENVIRONMENT:
2660	* Process level
2661	*/
2662	static void ibmvscsis_parse_cmd(struct scsi_info *vscsi,
2663	struct ibmvscsis_cmd *cmd)
2664	{
2665	struct iu_entry *iue = cmd->iue;
2666	struct srp_cmd *srp = (struct srp_cmd *)iue->sbuf->buf;
2667	struct ibmvscsis_nexus *nexus;
2668	u64 data_len = 0;
2669	enum dma_data_direction dir;
2670	int attr = 0;
2671
2672	nexus = vscsi->tport.ibmv_nexus;
2673	/*
2674	* additional length in bytes. Note that the SRP spec says that
2675	* additional length is in 4-byte words, but technically the
2676	* additional length field is only the upper 6 bits of the byte.
2677	* The lower 2 bits are reserved. If the lower 2 bits are 0 (as
2678	* all reserved fields should be), then interpreting the byte as
2679	* an int will yield the length in bytes.
2680	*/
2681	if (srp->add_cdb_len & 0x03) {
2682	dev_err(&vscsi->dev, "parse_cmd: reserved bits set in IU\n");
2683	spin_lock_bh(lock: &vscsi->intr_lock);
2684	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
2685	ibmvscsis_free_cmd_resources(vscsi, cmd);
2686	spin_unlock_bh(lock: &vscsi->intr_lock);
2687	return;
2688	}
2689
2690	if (srp_get_desc_table(srp_cmd: srp, dir: &dir, data_len: &data_len)) {
2691	dev_err(&vscsi->dev, "0x%llx: parsing SRP descriptor table failed.\n",
2692	srp->tag);
2693	goto fail;
2694	}
2695
2696	cmd->rsp.sol_not = srp->sol_not;
2697
2698	switch (srp->task_attr) {
2699	case SRP_SIMPLE_TASK:
2700	attr = TCM_SIMPLE_TAG;
2701	break;
2702	case SRP_ORDERED_TASK:
2703	attr = TCM_ORDERED_TAG;
2704	break;
2705	case SRP_HEAD_TASK:
2706	attr = TCM_HEAD_TAG;
2707	break;
2708	case SRP_ACA_TASK:
2709	attr = TCM_ACA_TAG;
2710	break;
2711	default:
2712	dev_err(&vscsi->dev, "Invalid task attribute %d\n",
2713	srp->task_attr);
2714	goto fail;
2715	}
2716
2717	cmd->se_cmd.tag = be64_to_cpu(srp->tag);
2718
2719	spin_lock_bh(lock: &vscsi->intr_lock);
2720	list_add_tail(new: &cmd->list, head: &vscsi->active_q);
2721	spin_unlock_bh(lock: &vscsi->intr_lock);
2722
2723	srp->lun.scsi_lun[0] &= 0x3f;
2724
2725	target_submit_cmd(&cmd->se_cmd, nexus->se_sess, srp->cdb,
2726	cmd->sense_buf, scsilun_to_int(&srp->lun),
2727	data_len, attr, dir, 0);
2728	return;
2729
2730	fail:
2731	spin_lock_bh(lock: &vscsi->intr_lock);
2732	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, flag_bits: 0);
2733	spin_unlock_bh(lock: &vscsi->intr_lock);
2734	}
2735
2736	/**
2737	* ibmvscsis_parse_task() - Parse SRP Task Management Request
2738	* @vscsi: Pointer to our adapter structure
2739	* @cmd: Pointer to command element with SRP task management request
2740	*
2741	* Parse the srp task management request; if it is valid then submit it to tcm.
2742	* Note: The return code does not reflect the status of the task management
2743	* request.
2744	*
2745	* EXECUTION ENVIRONMENT:
2746	* Processor level
2747	*/
2748	static void ibmvscsis_parse_task(struct scsi_info *vscsi,
2749	struct ibmvscsis_cmd *cmd)
2750	{
2751	struct iu_entry *iue = cmd->iue;
2752	struct srp_tsk_mgmt *srp_tsk = &vio_iu(iue)->srp.tsk_mgmt;
2753	int tcm_type;
2754	u64 tag_to_abort = 0;
2755	int rc = 0;
2756	struct ibmvscsis_nexus *nexus;
2757
2758	nexus = vscsi->tport.ibmv_nexus;
2759
2760	cmd->rsp.sol_not = srp_tsk->sol_not;
2761
2762	switch (srp_tsk->tsk_mgmt_func) {
2763	case SRP_TSK_ABORT_TASK:
2764	tcm_type = TMR_ABORT_TASK;
2765	tag_to_abort = be64_to_cpu(srp_tsk->task_tag);
2766	break;
2767	case SRP_TSK_ABORT_TASK_SET:
2768	tcm_type = TMR_ABORT_TASK_SET;
2769	break;
2770	case SRP_TSK_CLEAR_TASK_SET:
2771	tcm_type = TMR_CLEAR_TASK_SET;
2772	break;
2773	case SRP_TSK_LUN_RESET:
2774	tcm_type = TMR_LUN_RESET;
2775	break;
2776	case SRP_TSK_CLEAR_ACA:
2777	tcm_type = TMR_CLEAR_ACA;
2778	break;
2779	default:
2780	dev_err(&vscsi->dev, "unknown task mgmt func %d\n",
2781	srp_tsk->tsk_mgmt_func);
2782	cmd->se_cmd.se_tmr_req->response =
2783	TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
2784	rc = -1;
2785	break;
2786	}
2787
2788	if (!rc) {
2789	cmd->se_cmd.tag = be64_to_cpu(srp_tsk->tag);
2790
2791	spin_lock_bh(lock: &vscsi->intr_lock);
2792	list_add_tail(new: &cmd->list, head: &vscsi->active_q);
2793	spin_unlock_bh(lock: &vscsi->intr_lock);
2794
2795	srp_tsk->lun.scsi_lun[0] &= 0x3f;
2796
2797	dev_dbg(&vscsi->dev, "calling submit_tmr, func %d\n",
2798	srp_tsk->tsk_mgmt_func);
2799	rc = target_submit_tmr(se_cmd: &cmd->se_cmd, se_sess: nexus->se_sess, NULL,
2800	unpacked_lun: scsilun_to_int(&srp_tsk->lun), fabric_tmr_ptr: srp_tsk,
2801	tm_type: tcm_type, GFP_KERNEL, tag_to_abort, 0);
2802	if (rc) {
2803	dev_err(&vscsi->dev, "target_submit_tmr failed, rc %d\n",
2804	rc);
2805	spin_lock_bh(lock: &vscsi->intr_lock);
2806	list_del(entry: &cmd->list);
2807	spin_unlock_bh(lock: &vscsi->intr_lock);
2808	cmd->se_cmd.se_tmr_req->response =
2809	TMR_FUNCTION_REJECTED;
2810	}
2811	}
2812
2813	if (rc)
2814	transport_send_check_condition_and_sense(&cmd->se_cmd, 0, 0);
2815	}
2816
2817	static void ibmvscsis_scheduler(struct work_struct *work)
2818	{
2819	struct ibmvscsis_cmd *cmd = container_of(work, struct ibmvscsis_cmd,
2820	work);
2821	struct scsi_info *vscsi = cmd->adapter;
2822
2823	spin_lock_bh(lock: &vscsi->intr_lock);
2824
2825	/* Remove from schedule_q */
2826	list_del(entry: &cmd->list);
2827
2828	/* Don't submit cmd if we're disconnecting */
2829	if (vscsi->flags & (SCHEDULE_DISCONNECT | DISCONNECT_SCHEDULED)) {
2830	ibmvscsis_free_cmd_resources(vscsi, cmd);
2831
2832	/* ibmvscsis_disconnect might be waiting for us */
2833	if (list_empty(head: &vscsi->active_q) &&
2834	list_empty(head: &vscsi->schedule_q) &&
2835	(vscsi->flags & WAIT_FOR_IDLE)) {
2836	vscsi->flags &= ~WAIT_FOR_IDLE;
2837	complete(&vscsi->wait_idle);
2838	}
2839
2840	spin_unlock_bh(lock: &vscsi->intr_lock);
2841	return;
2842	}
2843
2844	spin_unlock_bh(lock: &vscsi->intr_lock);
2845
2846	switch (cmd->type) {
2847	case SCSI_CDB:
2848	ibmvscsis_parse_cmd(vscsi, cmd);
2849	break;
2850	case TASK_MANAGEMENT:
2851	ibmvscsis_parse_task(vscsi, cmd);
2852	break;
2853	default:
2854	dev_err(&vscsi->dev, "scheduler, invalid cmd type %d\n",
2855	cmd->type);
2856	spin_lock_bh(lock: &vscsi->intr_lock);
2857	ibmvscsis_free_cmd_resources(vscsi, cmd);
2858	spin_unlock_bh(lock: &vscsi->intr_lock);
2859	break;
2860	}
2861	}
2862
2863	static int ibmvscsis_alloc_cmds(struct scsi_info *vscsi, int num)
2864	{
2865	struct ibmvscsis_cmd *cmd;
2866	int i;
2867
2868	INIT_LIST_HEAD(list: &vscsi->free_cmd);
2869	vscsi->cmd_pool = kcalloc(n: num, size: sizeof(struct ibmvscsis_cmd),
2870	GFP_KERNEL);
2871	if (!vscsi->cmd_pool)
2872	return -ENOMEM;
2873
2874	for (i = 0, cmd = (struct ibmvscsis_cmd *)vscsi->cmd_pool; i < num;
2875	i++, cmd++) {
2876	cmd->abort_cmd = NULL;
2877	cmd->adapter = vscsi;
2878	INIT_WORK(&cmd->work, ibmvscsis_scheduler);
2879	list_add_tail(new: &cmd->list, head: &vscsi->free_cmd);
2880	}
2881
2882	return 0;
2883	}
2884
2885	static void ibmvscsis_free_cmds(struct scsi_info *vscsi)
2886	{
2887	kfree(objp: vscsi->cmd_pool);
2888	vscsi->cmd_pool = NULL;
2889	INIT_LIST_HEAD(list: &vscsi->free_cmd);
2890	}
2891
2892	/**
2893	* ibmvscsis_service_wait_q() - Service Waiting Queue
2894	* @timer: Pointer to timer which has expired
2895	*
2896	* This routine is called when the timer pops to service the waiting
2897	* queue. Elements on the queue have completed, their responses have been
2898	* copied to the client, but the client's response queue was full so
2899	* the queue message could not be sent. The routine grabs the proper locks
2900	* and calls send messages.
2901	*
2902	* EXECUTION ENVIRONMENT:
2903	* called at interrupt level
2904	*/
2905	static enum hrtimer_restart ibmvscsis_service_wait_q(struct hrtimer *timer)
2906	{
2907	struct timer_cb *p_timer = container_of(timer, struct timer_cb, timer);
2908	struct scsi_info *vscsi = container_of(p_timer, struct scsi_info,
2909	rsp_q_timer);
2910
2911	spin_lock_bh(lock: &vscsi->intr_lock);
2912	p_timer->timer_pops += 1;
2913	p_timer->started = false;
2914	ibmvscsis_send_messages(vscsi);
2915	spin_unlock_bh(lock: &vscsi->intr_lock);
2916
2917	return HRTIMER_NORESTART;
2918	}
2919
2920	static long ibmvscsis_alloctimer(struct scsi_info *vscsi)
2921	{
2922	struct timer_cb *p_timer;
2923
2924	p_timer = &vscsi->rsp_q_timer;
2925	hrtimer_init(timer: &p_timer->timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
2926
2927	p_timer->timer.function = ibmvscsis_service_wait_q;
2928	p_timer->started = false;
2929	p_timer->timer_pops = 0;
2930
2931	return ADAPT_SUCCESS;
2932	}
2933
2934	static void ibmvscsis_freetimer(struct scsi_info *vscsi)
2935	{
2936	struct timer_cb *p_timer;
2937
2938	p_timer = &vscsi->rsp_q_timer;
2939
2940	(void)hrtimer_cancel(timer: &p_timer->timer);
2941
2942	p_timer->started = false;
2943	p_timer->timer_pops = 0;
2944	}
2945
2946	static irqreturn_t ibmvscsis_interrupt(int dummy, void *data)
2947	{
2948	struct scsi_info *vscsi = data;
2949
2950	vio_disable_interrupts(vscsi->dma_dev);
2951	tasklet_schedule(t: &vscsi->work_task);
2952
2953	return IRQ_HANDLED;
2954	}
2955
2956	/**
2957	* ibmvscsis_enable_change_state() - Set new state based on enabled status
2958	* @vscsi: Pointer to our adapter structure
2959	*
2960	* This function determines our new state now that we are enabled. This
2961	* may involve sending an Init Complete message to the client.
2962	*
2963	* Must be called with interrupt lock held.
2964	*/
2965	static long ibmvscsis_enable_change_state(struct scsi_info *vscsi)
2966	{
2967	int bytes;
2968	long rc = ADAPT_SUCCESS;
2969
2970	bytes = vscsi->cmd_q.size * PAGE_SIZE;
2971	rc = h_reg_crq(vscsi->dds.unit_id, vscsi->cmd_q.crq_token, bytes);
2972	if (rc == H_CLOSED || rc == H_SUCCESS) {
2973	vscsi->state = WAIT_CONNECTION;
2974	rc = ibmvscsis_establish_new_q(vscsi);
2975	}
2976
2977	if (rc != ADAPT_SUCCESS) {
2978	vscsi->state = ERR_DISCONNECTED;
2979	vscsi->flags |= RESPONSE_Q_DOWN;
2980	}
2981
2982	return rc;
2983	}
2984
2985	/**
2986	* ibmvscsis_create_command_q() - Create Command Queue
2987	* @vscsi: Pointer to our adapter structure
2988	* @num_cmds: Currently unused. In the future, may be used to determine
2989	* the size of the CRQ.
2990	*
2991	* Allocates memory for command queue maps remote memory into an ioba
2992	* initializes the command response queue
2993	*
2994	* EXECUTION ENVIRONMENT:
2995	* Process level only
2996	*/
2997	static long ibmvscsis_create_command_q(struct scsi_info *vscsi, int num_cmds)
2998	{
2999	int pages;
3000	struct vio_dev *vdev = vscsi->dma_dev;
3001
3002	/* We might support multiple pages in the future, but just 1 for now */
3003	pages = 1;
3004
3005	vscsi->cmd_q.size = pages;
3006
3007	vscsi->cmd_q.base_addr =
3008	(struct viosrp_crq *)get_zeroed_page(GFP_KERNEL);
3009	if (!vscsi->cmd_q.base_addr)
3010	return -ENOMEM;
3011
3012	vscsi->cmd_q.mask = ((uint)pages * CRQ_PER_PAGE) - 1;
3013
3014	vscsi->cmd_q.crq_token = dma_map_single(&vdev->dev,
3015	vscsi->cmd_q.base_addr,
3016	PAGE_SIZE, DMA_BIDIRECTIONAL);
3017	if (dma_mapping_error(&vdev->dev, vscsi->cmd_q.crq_token)) {
3018	free_page((unsigned long)vscsi->cmd_q.base_addr);
3019	return -ENOMEM;
3020	}
3021
3022	return 0;
3023	}
3024
3025	/**
3026	* ibmvscsis_destroy_command_q - Destroy Command Queue
3027	* @vscsi: Pointer to our adapter structure
3028	*
3029	* Releases memory for command queue and unmaps mapped remote memory.
3030	*
3031	* EXECUTION ENVIRONMENT:
3032	* Process level only
3033	*/
3034	static void ibmvscsis_destroy_command_q(struct scsi_info *vscsi)
3035	{
3036	dma_unmap_single(&vscsi->dma_dev->dev, vscsi->cmd_q.crq_token,
3037	PAGE_SIZE, DMA_BIDIRECTIONAL);
3038	free_page((unsigned long)vscsi->cmd_q.base_addr);
3039	vscsi->cmd_q.base_addr = NULL;
3040	vscsi->state = NO_QUEUE;
3041	}
3042
3043	static u8 ibmvscsis_fast_fail(struct scsi_info *vscsi,
3044	struct ibmvscsis_cmd *cmd)
3045	{
3046	struct iu_entry *iue = cmd->iue;
3047	struct se_cmd *se_cmd = &cmd->se_cmd;
3048	struct srp_cmd *srp = (struct srp_cmd *)iue->sbuf->buf;
3049	struct scsi_sense_hdr sshdr;
3050	u8 rc = se_cmd->scsi_status;
3051
3052	if (vscsi->fast_fail && (READ_CMD(srp->cdb) || WRITE_CMD(srp->cdb)))
3053	if (scsi_normalize_sense(sense_buffer: se_cmd->sense_buffer,
3054	sb_len: se_cmd->scsi_sense_length, sshdr: &sshdr))
3055	if (sshdr.sense_key == HARDWARE_ERROR &&
3056	(se_cmd->residual_count == 0 ||
3057	se_cmd->residual_count == se_cmd->data_length)) {
3058	rc = NO_SENSE;
3059	cmd->flags |= CMD_FAST_FAIL;
3060	}
3061
3062	return rc;
3063	}
3064
3065	/**
3066	* srp_build_response() - Build an SRP response buffer
3067	* @vscsi: Pointer to our adapter structure
3068	* @cmd: Pointer to command for which to send the response
3069	* @len_p: Where to return the length of the IU response sent. This
3070	* is needed to construct the CRQ response.
3071	*
3072	* Build the SRP response buffer and copy it to the client's memory space.
3073	*/
3074	static long srp_build_response(struct scsi_info *vscsi,
3075	struct ibmvscsis_cmd *cmd, uint *len_p)
3076	{
3077	struct iu_entry *iue = cmd->iue;
3078	struct se_cmd *se_cmd = &cmd->se_cmd;
3079	struct srp_rsp *rsp;
3080	uint len;
3081	u32 rsp_code;
3082	char *data;
3083	u32 *tsk_status;
3084	long rc = ADAPT_SUCCESS;
3085
3086	spin_lock_bh(lock: &vscsi->intr_lock);
3087
3088	rsp = &vio_iu(iue)->srp.rsp;
3089	len = sizeof(*rsp);
3090	memset(rsp, 0, len);
3091	data = rsp->data;
3092
3093	rsp->opcode = SRP_RSP;
3094
3095	rsp->req_lim_delta = cpu_to_be32(1 + vscsi->credit);
3096	rsp->tag = cmd->rsp.tag;
3097	rsp->flags = 0;
3098
3099	if (cmd->type == SCSI_CDB) {
3100	rsp->status = ibmvscsis_fast_fail(vscsi, cmd);
3101	if (rsp->status) {
3102	dev_dbg(&vscsi->dev, "build_resp: cmd %p, scsi status %d\n",
3103	cmd, (int)rsp->status);
3104	ibmvscsis_determine_resid(se_cmd, rsp);
3105	if (se_cmd->scsi_sense_length && se_cmd->sense_buffer) {
3106	rsp->sense_data_len =
3107	cpu_to_be32(se_cmd->scsi_sense_length);
3108	rsp->flags |= SRP_RSP_FLAG_SNSVALID;
3109	len += se_cmd->scsi_sense_length;
3110	memcpy(data, se_cmd->sense_buffer,
3111	se_cmd->scsi_sense_length);
3112	}
3113	rsp->sol_not = (cmd->rsp.sol_not & UCSOLNT) >>
3114	UCSOLNT_RESP_SHIFT;
3115	} else if (cmd->flags & CMD_FAST_FAIL) {
3116	dev_dbg(&vscsi->dev, "build_resp: cmd %p, fast fail\n",
3117	cmd);
3118	rsp->sol_not = (cmd->rsp.sol_not & UCSOLNT) >>
3119	UCSOLNT_RESP_SHIFT;
3120	} else {
3121	rsp->sol_not = (cmd->rsp.sol_not & SCSOLNT) >>
3122	SCSOLNT_RESP_SHIFT;
3123	}
3124	} else {
3125	/* this is task management */
3126	rsp->status = 0;
3127	rsp->resp_data_len = cpu_to_be32(4);
3128	rsp->flags |= SRP_RSP_FLAG_RSPVALID;
3129
3130	switch (se_cmd->se_tmr_req->response) {
3131	case TMR_FUNCTION_COMPLETE:
3132	case TMR_TASK_DOES_NOT_EXIST:
3133	rsp_code = SRP_TASK_MANAGEMENT_FUNCTION_COMPLETE;
3134	rsp->sol_not = (cmd->rsp.sol_not & SCSOLNT) >>
3135	SCSOLNT_RESP_SHIFT;
3136	break;
3137	case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED:
3138	case TMR_LUN_DOES_NOT_EXIST:
3139	rsp_code = SRP_TASK_MANAGEMENT_FUNCTION_NOT_SUPPORTED;
3140	rsp->sol_not = (cmd->rsp.sol_not & UCSOLNT) >>
3141	UCSOLNT_RESP_SHIFT;
3142	break;
3143	case TMR_FUNCTION_FAILED:
3144	case TMR_FUNCTION_REJECTED:
3145	default:
3146	rsp_code = SRP_TASK_MANAGEMENT_FUNCTION_FAILED;
3147	rsp->sol_not = (cmd->rsp.sol_not & UCSOLNT) >>
3148	UCSOLNT_RESP_SHIFT;
3149	break;
3150	}
3151
3152	tsk_status = (u32 *)data;
3153	*tsk_status = cpu_to_be32(rsp_code);
3154	data = (char *)(tsk_status + 1);
3155	len += 4;
3156	}
3157
3158	dma_wmb();
3159	rc = h_copy_rdma(len, vscsi->dds.window[LOCAL].liobn, iue->sbuf->dma,
3160	vscsi->dds.window[REMOTE].liobn,
3161	be64_to_cpu(iue->remote_token));
3162
3163	switch (rc) {
3164	case H_SUCCESS:
3165	vscsi->credit = 0;
3166	*len_p = len;
3167	break;
3168	case H_PERMISSION:
3169	if (connection_broken(vscsi))
3170	vscsi->flags |= RESPONSE_Q_DOWN | CLIENT_FAILED;
3171
3172	dev_err(&vscsi->dev, "build_response: error copying to client, rc %ld, flags 0x%x, state 0x%hx\n",
3173	rc, vscsi->flags, vscsi->state);
3174	break;
3175	case H_SOURCE_PARM:
3176	case H_DEST_PARM:
3177	default:
3178	dev_err(&vscsi->dev, "build_response: error copying to client, rc %ld\n",
3179	rc);
3180	break;
3181	}
3182
3183	spin_unlock_bh(lock: &vscsi->intr_lock);
3184
3185	return rc;
3186	}
3187
3188	static int ibmvscsis_rdma(struct ibmvscsis_cmd *cmd, struct scatterlist *sg,
3189	int nsg, struct srp_direct_buf *md, int nmd,
3190	enum dma_data_direction dir, unsigned int bytes)
3191	{
3192	struct iu_entry *iue = cmd->iue;
3193	struct srp_target *target = iue->target;
3194	struct scsi_info *vscsi = target->ldata;
3195	struct scatterlist *sgp;
3196	dma_addr_t client_ioba, server_ioba;
3197	ulong buf_len;
3198	ulong client_len, server_len;
3199	int md_idx;
3200	long tx_len;
3201	long rc = 0;
3202
3203	if (bytes == 0)
3204	return 0;
3205
3206	sgp = sg;
3207	client_len = 0;
3208	server_len = 0;
3209	md_idx = 0;
3210	tx_len = bytes;
3211
3212	do {
3213	if (client_len == 0) {
3214	if (md_idx >= nmd) {
3215	dev_err(&vscsi->dev, "rdma: ran out of client memory descriptors\n");
3216	rc = -EIO;
3217	break;
3218	}
3219	client_ioba = be64_to_cpu(md[md_idx].va);
3220	client_len = be32_to_cpu(md[md_idx].len);
3221	}
3222	if (server_len == 0) {
3223	if (!sgp) {
3224	dev_err(&vscsi->dev, "rdma: ran out of scatter/gather list\n");
3225	rc = -EIO;
3226	break;
3227	}
3228	server_ioba = sg_dma_address(sgp);
3229	server_len = sg_dma_len(sgp);
3230	}
3231
3232	buf_len = tx_len;
3233
3234	if (buf_len > client_len)
3235	buf_len = client_len;
3236
3237	if (buf_len > server_len)
3238	buf_len = server_len;
3239
3240	if (buf_len > max_vdma_size)
3241	buf_len = max_vdma_size;
3242
3243	if (dir == DMA_TO_DEVICE) {
3244	/* read from client */
3245	rc = h_copy_rdma(buf_len,
3246	vscsi->dds.window[REMOTE].liobn,
3247	client_ioba,
3248	vscsi->dds.window[LOCAL].liobn,
3249	server_ioba);
3250	} else {
3251	/* The h_copy_rdma will cause phyp, running in another
3252	* partition, to read memory, so we need to make sure
3253	* the data has been written out, hence these syncs.
3254	*/
3255	/* ensure that everything is in memory */
3256	isync();
3257	/* ensure that memory has been made visible */
3258	dma_wmb();
3259	rc = h_copy_rdma(buf_len,
3260	vscsi->dds.window[LOCAL].liobn,
3261	server_ioba,
3262	vscsi->dds.window[REMOTE].liobn,
3263	client_ioba);
3264	}
3265	switch (rc) {
3266	case H_SUCCESS:
3267	break;
3268	case H_PERMISSION:
3269	case H_SOURCE_PARM:
3270	case H_DEST_PARM:
3271	if (connection_broken(vscsi)) {
3272	spin_lock_bh(lock: &vscsi->intr_lock);
3273	vscsi->flags |=
3274	(RESPONSE_Q_DOWN | CLIENT_FAILED);
3275	spin_unlock_bh(lock: &vscsi->intr_lock);
3276	}
3277	dev_err(&vscsi->dev, "rdma: h_copy_rdma failed, rc %ld\n",
3278	rc);
3279	break;
3280
3281	default:
3282	dev_err(&vscsi->dev, "rdma: unknown error %ld from h_copy_rdma\n",
3283	rc);
3284	break;
3285	}
3286
3287	if (!rc) {
3288	tx_len -= buf_len;
3289	if (tx_len) {
3290	client_len -= buf_len;
3291	if (client_len == 0)
3292	md_idx++;
3293	else
3294	client_ioba += buf_len;
3295
3296	server_len -= buf_len;
3297	if (server_len == 0)
3298	sgp = sg_next(sgp);
3299	else
3300	server_ioba += buf_len;
3301	} else {
3302	break;
3303	}
3304	}
3305	} while (!rc);
3306
3307	return rc;
3308	}
3309
3310	/**
3311	* ibmvscsis_handle_crq() - Handle CRQ
3312	* @data: Pointer to our adapter structure
3313	*
3314	* Read the command elements from the command queue and copy the payloads
3315	* associated with the command elements to local memory and execute the
3316	* SRP requests.
3317	*
3318	* Note: this is an edge triggered interrupt. It can not be shared.
3319	*/
3320	static void ibmvscsis_handle_crq(unsigned long data)
3321	{
3322	struct scsi_info *vscsi = (struct scsi_info *)data;
3323	struct viosrp_crq *crq;
3324	long rc;
3325	bool ack = true;
3326	volatile u8 valid;
3327
3328	spin_lock_bh(lock: &vscsi->intr_lock);
3329
3330	dev_dbg(&vscsi->dev, "got interrupt\n");
3331
3332	/*
3333	* if we are in a path where we are waiting for all pending commands
3334	* to complete because we received a transport event and anything in
3335	* the command queue is for a new connection, do nothing
3336	*/
3337	if (TARGET_STOP(vscsi)) {
3338	vio_enable_interrupts(vscsi->dma_dev);
3339
3340	dev_dbg(&vscsi->dev, "handle_crq, don't process: flags 0x%x, state 0x%hx\n",
3341	vscsi->flags, vscsi->state);
3342	spin_unlock_bh(lock: &vscsi->intr_lock);
3343	return;
3344	}
3345
3346	rc = vscsi->flags & SCHEDULE_DISCONNECT;
3347	crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
3348	valid = crq->valid;
3349	dma_rmb();
3350
3351	while (valid) {
3352	/*
3353	* These are edege triggered interrupts. After dropping out of
3354	* the while loop, the code must check for work since an
3355	* interrupt could be lost, and an elment be left on the queue,
3356	* hence the label.
3357	*/
3358	cmd_work:
3359	vscsi->cmd_q.index =
3360	(vscsi->cmd_q.index + 1) & vscsi->cmd_q.mask;
3361
3362	if (!rc) {
3363	rc = ibmvscsis_parse_command(vscsi, crq);
3364	} else {
3365	if ((uint)crq->valid == VALID_TRANS_EVENT) {
3366	/*
3367	* must service the transport layer events even
3368	* in an error state, dont break out until all
3369	* the consecutive transport events have been
3370	* processed
3371	*/
3372	rc = ibmvscsis_trans_event(vscsi, crq);
3373	} else if (vscsi->flags & TRANS_EVENT) {
3374	/*
3375	* if a transport event has occurred leave
3376	* everything but transport events on the queue
3377	*
3378	* need to decrement the queue index so we can
3379	* look at the element again
3380	*/
3381	if (vscsi->cmd_q.index)
3382	vscsi->cmd_q.index -= 1;
3383	else
3384	/*
3385	* index is at 0 it just wrapped.
3386	* have it index last element in q
3387	*/
3388	vscsi->cmd_q.index = vscsi->cmd_q.mask;
3389	break;
3390	}
3391	}
3392
3393	crq->valid = INVALIDATE_CMD_RESP_EL;
3394
3395	crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
3396	valid = crq->valid;
3397	dma_rmb();
3398	}
3399
3400	if (!rc) {
3401	if (ack) {
3402	vio_enable_interrupts(vscsi->dma_dev);
3403	ack = false;
3404	dev_dbg(&vscsi->dev, "handle_crq, reenabling interrupts\n");
3405	}
3406	valid = crq->valid;
3407	dma_rmb();
3408	if (valid)
3409	goto cmd_work;
3410	} else {
3411	dev_dbg(&vscsi->dev, "handle_crq, error: flags 0x%x, state 0x%hx, crq index 0x%x\n",
3412	vscsi->flags, vscsi->state, vscsi->cmd_q.index);
3413	}
3414
3415	dev_dbg(&vscsi->dev, "Leaving handle_crq: schedule_q empty %d, flags 0x%x, state 0x%hx\n",
3416	(int)list_empty(head: &vscsi->schedule_q), vscsi->flags,
3417	vscsi->state);
3418
3419	spin_unlock_bh(lock: &vscsi->intr_lock);
3420	}
3421
3422	static int ibmvscsis_probe(struct vio_dev *vdev,
3423	const struct vio_device_id *id)
3424	{
3425	struct scsi_info *vscsi;
3426	int rc = 0;
3427	long hrc = 0;
3428	char wq_name[24];
3429
3430	vscsi = kzalloc(size: sizeof(*vscsi), GFP_KERNEL);
3431	if (!vscsi) {
3432	rc = -ENOMEM;
3433	dev_err(&vdev->dev, "probe: allocation of adapter failed\n");
3434	return rc;
3435	}
3436
3437	vscsi->dma_dev = vdev;
3438	vscsi->dev = vdev->dev;
3439	INIT_LIST_HEAD(list: &vscsi->schedule_q);
3440	INIT_LIST_HEAD(list: &vscsi->waiting_rsp);
3441	INIT_LIST_HEAD(list: &vscsi->active_q);
3442
3443	snprintf(buf: vscsi->tport.tport_name, IBMVSCSIS_NAMELEN, fmt: "%s",
3444	dev_name(&vdev->dev));
3445
3446	dev_dbg(&vscsi->dev, "probe tport_name: %s\n", vscsi->tport.tport_name);
3447
3448	rc = read_dma_window(vscsi);
3449	if (rc)
3450	goto free_adapter;
3451	dev_dbg(&vscsi->dev, "Probe: liobn 0x%x, riobn 0x%x\n",
3452	vscsi->dds.window[LOCAL].liobn,
3453	vscsi->dds.window[REMOTE].liobn);
3454
3455	snprintf(buf: vscsi->eye, size: sizeof(vscsi->eye), fmt: "VSCSI %s", vdev->name);
3456
3457	vscsi->dds.unit_id = vdev->unit_address;
3458	strscpy(p: vscsi->dds.partition_name, q: partition_name,
3459	size: sizeof(vscsi->dds.partition_name));
3460	vscsi->dds.partition_num = partition_number;
3461
3462	spin_lock_bh(lock: &ibmvscsis_dev_lock);
3463	list_add_tail(new: &vscsi->list, head: &ibmvscsis_dev_list);
3464	spin_unlock_bh(lock: &ibmvscsis_dev_lock);
3465
3466	/*
3467	* TBD: How do we determine # of cmds to request? Do we know how
3468	* many "children" we have?
3469	*/
3470	vscsi->request_limit = INITIAL_SRP_LIMIT;
3471	rc = srp_target_alloc(&vscsi->target, &vdev->dev, vscsi->request_limit,
3472	SRP_MAX_IU_LEN);
3473	if (rc)
3474	goto rem_list;
3475
3476	vscsi->target.ldata = vscsi;
3477
3478	rc = ibmvscsis_alloc_cmds(vscsi, num: vscsi->request_limit);
3479	if (rc) {
3480	dev_err(&vscsi->dev, "alloc_cmds failed, rc %d, num %d\n",
3481	rc, vscsi->request_limit);
3482	goto free_target;
3483	}
3484
3485	/*
3486	* Note: the lock is used in freeing timers, so must initialize
3487	* first so that ordering in case of error is correct.
3488	*/
3489	spin_lock_init(&vscsi->intr_lock);
3490
3491	rc = ibmvscsis_alloctimer(vscsi);
3492	if (rc) {
3493	dev_err(&vscsi->dev, "probe: alloctimer failed, rc %d\n", rc);
3494	goto free_cmds;
3495	}
3496
3497	rc = ibmvscsis_create_command_q(vscsi, num_cmds: 256);
3498	if (rc) {
3499	dev_err(&vscsi->dev, "probe: create_command_q failed, rc %d\n",
3500	rc);
3501	goto free_timer;
3502	}
3503
3504	vscsi->map_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
3505	if (!vscsi->map_buf) {
3506	rc = -ENOMEM;
3507	dev_err(&vscsi->dev, "probe: allocating cmd buffer failed\n");
3508	goto destroy_queue;
3509	}
3510
3511	vscsi->map_ioba = dma_map_single(&vdev->dev, vscsi->map_buf, PAGE_SIZE,
3512	DMA_BIDIRECTIONAL);
3513	if (dma_mapping_error(&vdev->dev, vscsi->map_ioba)) {
3514	rc = -ENOMEM;
3515	dev_err(&vscsi->dev, "probe: error mapping command buffer\n");
3516	goto free_buf;
3517	}
3518
3519	hrc = h_vioctl(vscsi->dds.unit_id, H_GET_PARTNER_INFO,
3520	(u64)vscsi->map_ioba | ((u64)PAGE_SIZE << 32), 0, 0, 0,
3521	0);
3522	if (hrc == H_SUCCESS)
3523	vscsi->client_data.partition_number =
3524	be64_to_cpu(*(u64 *)vscsi->map_buf);
3525	/*
3526	* We expect the VIOCTL to fail if we're configured as "any
3527	* client can connect" and the client isn't activated yet.
3528	* We'll make the call again when he sends an init msg.
3529	*/
3530	dev_dbg(&vscsi->dev, "probe hrc %ld, client partition num %d\n",
3531	hrc, vscsi->client_data.partition_number);
3532
3533	tasklet_init(t: &vscsi->work_task, func: ibmvscsis_handle_crq,
3534	data: (unsigned long)vscsi);
3535
3536	init_completion(x: &vscsi->wait_idle);
3537	init_completion(x: &vscsi->unconfig);
3538
3539	snprintf(buf: wq_name, size: 24, fmt: "ibmvscsis%s", dev_name(&vdev->dev));
3540	vscsi->work_q = create_workqueue(wq_name);
3541	if (!vscsi->work_q) {
3542	rc = -ENOMEM;
3543	dev_err(&vscsi->dev, "create_workqueue failed\n");
3544	goto unmap_buf;
3545	}
3546
3547	rc = request_irq(irq: vdev->irq, handler: ibmvscsis_interrupt, flags: 0, name: "ibmvscsis", dev: vscsi);
3548	if (rc) {
3549	rc = -EPERM;
3550	dev_err(&vscsi->dev, "probe: request_irq failed, rc %d\n", rc);
3551	goto destroy_WQ;
3552	}
3553
3554	vscsi->state = WAIT_ENABLED;
3555
3556	dev_set_drvdata(&vdev->dev, vscsi);
3557
3558	return 0;
3559
3560	destroy_WQ:
3561	destroy_workqueue(wq: vscsi->work_q);
3562	unmap_buf:
3563	dma_unmap_single(&vdev->dev, vscsi->map_ioba, PAGE_SIZE,
3564	DMA_BIDIRECTIONAL);
3565	free_buf:
3566	kfree(objp: vscsi->map_buf);
3567	destroy_queue:
3568	tasklet_kill(t: &vscsi->work_task);
3569	ibmvscsis_unregister_command_q(vscsi);
3570	ibmvscsis_destroy_command_q(vscsi);
3571	free_timer:
3572	ibmvscsis_freetimer(vscsi);
3573	free_cmds:
3574	ibmvscsis_free_cmds(vscsi);
3575	free_target:
3576	srp_target_free(&vscsi->target);
3577	rem_list:
3578	spin_lock_bh(lock: &ibmvscsis_dev_lock);
3579	list_del(entry: &vscsi->list);
3580	spin_unlock_bh(lock: &ibmvscsis_dev_lock);
3581	free_adapter:
3582	kfree(objp: vscsi);
3583
3584	return rc;
3585	}
3586
3587	static void ibmvscsis_remove(struct vio_dev *vdev)
3588	{
3589	struct scsi_info *vscsi = dev_get_drvdata(&vdev->dev);
3590
3591	dev_dbg(&vscsi->dev, "remove (%s)\n", dev_name(&vscsi->dma_dev->dev));
3592
3593	spin_lock_bh(lock: &vscsi->intr_lock);
3594	ibmvscsis_post_disconnect(vscsi, UNCONFIGURING, flag_bits: 0);
3595	vscsi->flags |= CFG_SLEEPING;
3596	spin_unlock_bh(lock: &vscsi->intr_lock);
3597	wait_for_completion(&vscsi->unconfig);
3598
3599	vio_disable_interrupts(vdev);
3600	free_irq(vdev->irq, vscsi);
3601	destroy_workqueue(wq: vscsi->work_q);
3602	dma_unmap_single(&vdev->dev, vscsi->map_ioba, PAGE_SIZE,
3603	DMA_BIDIRECTIONAL);
3604	kfree(objp: vscsi->map_buf);
3605	tasklet_kill(t: &vscsi->work_task);
3606	ibmvscsis_destroy_command_q(vscsi);
3607	ibmvscsis_freetimer(vscsi);
3608	ibmvscsis_free_cmds(vscsi);
3609	srp_target_free(&vscsi->target);
3610	spin_lock_bh(lock: &ibmvscsis_dev_lock);
3611	list_del(entry: &vscsi->list);
3612	spin_unlock_bh(lock: &ibmvscsis_dev_lock);
3613	kfree(objp: vscsi);
3614	}
3615
3616	static ssize_t system_id_show(struct device *dev,
3617	struct device_attribute *attr, char *buf)
3618	{
3619	return snprintf(buf, PAGE_SIZE, fmt: "%s\n", system_id);
3620	}
3621
3622	static ssize_t partition_number_show(struct device *dev,
3623	struct device_attribute *attr, char *buf)
3624	{
3625	return snprintf(buf, PAGE_SIZE, fmt: "%x\n", partition_number);
3626	}
3627
3628	static ssize_t unit_address_show(struct device *dev,
3629	struct device_attribute *attr, char *buf)
3630	{
3631	struct scsi_info *vscsi = container_of(dev, struct scsi_info, dev);
3632
3633	return snprintf(buf, PAGE_SIZE, fmt: "%x\n", vscsi->dma_dev->unit_address);
3634	}
3635
3636	static int ibmvscsis_get_system_info(void)
3637	{
3638	struct device_node *rootdn, *vdevdn;
3639	const char *id, *model, *name;
3640	const uint *num;
3641
3642	rootdn = of_find_node_by_path(path: "/");
3643	if (!rootdn)
3644	return -ENOENT;
3645
3646	model = of_get_property(node: rootdn, name: "model", NULL);
3647	id = of_get_property(node: rootdn, name: "system-id", NULL);
3648	if (model && id)
3649	snprintf(buf: system_id, size: sizeof(system_id), fmt: "%s-%s", model, id);
3650
3651	name = of_get_property(node: rootdn, name: "ibm,partition-name", NULL);
3652	if (name)
3653	strncpy(p: partition_name, q: name, size: sizeof(partition_name));
3654
3655	num = of_get_property(node: rootdn, name: "ibm,partition-no", NULL);
3656	if (num)
3657	partition_number = of_read_number(cell: num, size: 1);
3658
3659	of_node_put(node: rootdn);
3660
3661	vdevdn = of_find_node_by_path(path: "/vdevice");
3662	if (vdevdn) {
3663	const uint *mvds;
3664
3665	mvds = of_get_property(node: vdevdn, name: "ibm,max-virtual-dma-size",
3666	NULL);
3667	if (mvds)
3668	max_vdma_size = *mvds;
3669	of_node_put(node: vdevdn);
3670	}
3671
3672	return 0;
3673	}
3674
3675	static char *ibmvscsis_get_fabric_wwn(struct se_portal_group *se_tpg)
3676	{
3677	struct ibmvscsis_tport *tport =
3678	container_of(se_tpg, struct ibmvscsis_tport, se_tpg);
3679
3680	return tport->tport_name;
3681	}
3682
3683	static u16 ibmvscsis_get_tag(struct se_portal_group *se_tpg)
3684	{
3685	struct ibmvscsis_tport *tport =
3686	container_of(se_tpg, struct ibmvscsis_tport, se_tpg);
3687
3688	return tport->tport_tpgt;
3689	}
3690
3691	static u32 ibmvscsis_get_default_depth(struct se_portal_group *se_tpg)
3692	{
3693	return 1;
3694	}
3695
3696	static int ibmvscsis_check_true(struct se_portal_group *se_tpg)
3697	{
3698	return 1;
3699	}
3700
3701	static int ibmvscsis_check_stop_free(struct se_cmd *se_cmd)
3702	{
3703	return target_put_sess_cmd(se_cmd);
3704	}
3705
3706	static void ibmvscsis_release_cmd(struct se_cmd *se_cmd)
3707	{
3708	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3709	se_cmd);
3710	struct scsi_info *vscsi = cmd->adapter;
3711
3712	spin_lock_bh(lock: &vscsi->intr_lock);
3713	/* Remove from active_q */
3714	list_move_tail(list: &cmd->list, head: &vscsi->waiting_rsp);
3715	ibmvscsis_send_messages(vscsi);
3716	spin_unlock_bh(lock: &vscsi->intr_lock);
3717	}
3718
3719	static int ibmvscsis_write_pending(struct se_cmd *se_cmd)
3720	{
3721	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3722	se_cmd);
3723	struct scsi_info *vscsi = cmd->adapter;
3724	struct iu_entry *iue = cmd->iue;
3725	int rc;
3726
3727	/*
3728	* If CLIENT_FAILED OR RESPONSE_Q_DOWN, then just return success
3729	* since LIO can't do anything about it, and we dont want to
3730	* attempt an srp_transfer_data.
3731	*/
3732	if ((vscsi->flags & (CLIENT_FAILED | RESPONSE_Q_DOWN))) {
3733	dev_err(&vscsi->dev, "write_pending failed since: %d\n",
3734	vscsi->flags);
3735	return -EIO;
3736
3737	}
3738
3739	rc = srp_transfer_data(cmd, &vio_iu(iue)->srp.cmd, ibmvscsis_rdma,
3740	1, 1);
3741	if (rc) {
3742	dev_err(&vscsi->dev, "srp_transfer_data() failed: %d\n", rc);
3743	return -EIO;
3744	}
3745	/*
3746	* We now tell TCM to add this WRITE CDB directly into the TCM storage
3747	* object execution queue.
3748	*/
3749	target_execute_cmd(cmd: se_cmd);
3750	return 0;
3751	}
3752
3753	static int ibmvscsis_queue_data_in(struct se_cmd *se_cmd)
3754	{
3755	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3756	se_cmd);
3757	struct iu_entry *iue = cmd->iue;
3758	struct scsi_info *vscsi = cmd->adapter;
3759	uint len = 0;
3760	int rc;
3761
3762	rc = srp_transfer_data(cmd, &vio_iu(iue)->srp.cmd, ibmvscsis_rdma, 1,
3763	1);
3764	if (rc) {
3765	dev_err(&vscsi->dev, "srp_transfer_data failed: %d\n", rc);
3766	se_cmd->scsi_sense_length = 18;
3767	memset(se_cmd->sense_buffer, 0, se_cmd->scsi_sense_length);
3768	/* Logical Unit Communication Time-out asc/ascq = 0x0801 */
3769	scsi_build_sense_buffer(desc: 0, buf: se_cmd->sense_buffer, MEDIUM_ERROR,
3770	asc: 0x08, ascq: 0x01);
3771	}
3772
3773	srp_build_response(vscsi, cmd, len_p: &len);
3774	cmd->rsp.format = SRP_FORMAT;
3775	cmd->rsp.len = len;
3776
3777	return 0;
3778	}
3779
3780	static int ibmvscsis_queue_status(struct se_cmd *se_cmd)
3781	{
3782	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3783	se_cmd);
3784	struct scsi_info *vscsi = cmd->adapter;
3785	uint len;
3786
3787	dev_dbg(&vscsi->dev, "queue_status %p\n", se_cmd);
3788
3789	srp_build_response(vscsi, cmd, len_p: &len);
3790	cmd->rsp.format = SRP_FORMAT;
3791	cmd->rsp.len = len;
3792
3793	return 0;
3794	}
3795
3796	static void ibmvscsis_queue_tm_rsp(struct se_cmd *se_cmd)
3797	{
3798	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3799	se_cmd);
3800	struct scsi_info *vscsi = cmd->adapter;
3801	struct ibmvscsis_cmd *cmd_itr;
3802	struct iu_entry *iue = iue = cmd->iue;
3803	struct srp_tsk_mgmt *srp_tsk = &vio_iu(iue)->srp.tsk_mgmt;
3804	u64 tag_to_abort = be64_to_cpu(srp_tsk->task_tag);
3805	uint len;
3806
3807	dev_dbg(&vscsi->dev, "queue_tm_rsp %p, status %d\n",
3808	se_cmd, (int)se_cmd->se_tmr_req->response);
3809
3810	if (srp_tsk->tsk_mgmt_func == SRP_TSK_ABORT_TASK &&
3811	cmd->se_cmd.se_tmr_req->response == TMR_TASK_DOES_NOT_EXIST) {
3812	spin_lock_bh(lock: &vscsi->intr_lock);
3813	list_for_each_entry(cmd_itr, &vscsi->active_q, list) {
3814	if (tag_to_abort == cmd_itr->se_cmd.tag) {
3815	cmd_itr->abort_cmd = cmd;
3816	cmd->flags |= DELAY_SEND;
3817	break;
3818	}
3819	}
3820	spin_unlock_bh(lock: &vscsi->intr_lock);
3821	}
3822
3823	srp_build_response(vscsi, cmd, len_p: &len);
3824	cmd->rsp.format = SRP_FORMAT;
3825	cmd->rsp.len = len;
3826	}
3827
3828	static void ibmvscsis_aborted_task(struct se_cmd *se_cmd)
3829	{
3830	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3831	se_cmd);
3832	struct scsi_info *vscsi = cmd->adapter;
3833
3834	dev_dbg(&vscsi->dev, "ibmvscsis_aborted_task %p task_tag: %llu\n",
3835	se_cmd, se_cmd->tag);
3836	}
3837
3838	static struct se_wwn *ibmvscsis_make_tport(struct target_fabric_configfs *tf,
3839	struct config_group *group,
3840	const char *name)
3841	{
3842	struct ibmvscsis_tport *tport;
3843	struct scsi_info *vscsi;
3844
3845	tport = ibmvscsis_lookup_port(name);
3846	if (tport) {
3847	vscsi = container_of(tport, struct scsi_info, tport);
3848	tport->tport_proto_id = SCSI_PROTOCOL_SRP;
3849	dev_dbg(&vscsi->dev, "make_tport(%s), pointer:%p, tport_id:%x\n",
3850	name, tport, tport->tport_proto_id);
3851	return &tport->tport_wwn;
3852	}
3853
3854	return ERR_PTR(error: -EINVAL);
3855	}
3856
3857	static void ibmvscsis_drop_tport(struct se_wwn *wwn)
3858	{
3859	struct ibmvscsis_tport *tport = container_of(wwn,
3860	struct ibmvscsis_tport,
3861	tport_wwn);
3862	struct scsi_info *vscsi = container_of(tport, struct scsi_info, tport);
3863
3864	dev_dbg(&vscsi->dev, "drop_tport(%s)\n",
3865	config_item_name(item: &tport->tport_wwn.wwn_group.cg_item));
3866	}
3867
3868	static struct se_portal_group *ibmvscsis_make_tpg(struct se_wwn *wwn,
3869	const char *name)
3870	{
3871	struct ibmvscsis_tport *tport =
3872	container_of(wwn, struct ibmvscsis_tport, tport_wwn);
3873	u16 tpgt;
3874	int rc;
3875
3876	if (strstr(name, "tpgt_") != name)
3877	return ERR_PTR(error: -EINVAL);
3878	rc = kstrtou16(s: name + 5, base: 0, res: &tpgt);
3879	if (rc)
3880	return ERR_PTR(error: rc);
3881	tport->tport_tpgt = tpgt;
3882
3883	tport->releasing = false;
3884
3885	rc = core_tpg_register(&tport->tport_wwn, &tport->se_tpg,
3886	tport->tport_proto_id);
3887	if (rc)
3888	return ERR_PTR(error: rc);
3889
3890	return &tport->se_tpg;
3891	}
3892
3893	static void ibmvscsis_drop_tpg(struct se_portal_group *se_tpg)
3894	{
3895	struct ibmvscsis_tport *tport = container_of(se_tpg,
3896	struct ibmvscsis_tport,
3897	se_tpg);
3898
3899	tport->releasing = true;
3900	tport->enabled = false;
3901
3902	/*
3903	* Release the virtual I_T Nexus for this ibmvscsis TPG
3904	*/
3905	ibmvscsis_drop_nexus(tport);
3906	/*
3907	* Deregister the se_tpg from TCM..
3908	*/
3909	core_tpg_deregister(se_tpg);
3910	}
3911
3912	static ssize_t ibmvscsis_wwn_version_show(struct config_item *item,
3913	char *page)
3914	{
3915	return scnprintf(buf: page, PAGE_SIZE, fmt: "%s\n", IBMVSCSIS_VERSION);
3916	}
3917	CONFIGFS_ATTR_RO(ibmvscsis_wwn_, version);
3918
3919	static struct configfs_attribute *ibmvscsis_wwn_attrs[] = {
3920	&ibmvscsis_wwn_attr_version,
3921	NULL,
3922	};
3923
3924
3925	static int ibmvscsis_enable_tpg(struct se_portal_group *se_tpg, bool enable)
3926	{
3927	struct ibmvscsis_tport *tport = container_of(se_tpg,
3928	struct ibmvscsis_tport,
3929	se_tpg);
3930	struct scsi_info *vscsi = container_of(tport, struct scsi_info, tport);
3931	long lrc;
3932
3933	if (enable) {
3934	spin_lock_bh(lock: &vscsi->intr_lock);
3935	tport->enabled = true;
3936	lrc = ibmvscsis_enable_change_state(vscsi);
3937	if (lrc)
3938	dev_err(&vscsi->dev, "enable_change_state failed, rc %ld state %d\n",
3939	lrc, vscsi->state);
3940	spin_unlock_bh(lock: &vscsi->intr_lock);
3941	} else {
3942	spin_lock_bh(lock: &vscsi->intr_lock);
3943	tport->enabled = false;
3944	/* This simulates the server going down */
3945	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, flag_bits: 0);
3946	spin_unlock_bh(lock: &vscsi->intr_lock);
3947	}
3948
3949	return 0;
3950	}
3951
3952	static const struct target_core_fabric_ops ibmvscsis_ops = {
3953	.module = THIS_MODULE,
3954	.fabric_name = "ibmvscsis",
3955	.max_data_sg_nents = MAX_TXU / PAGE_SIZE,
3956	.tpg_get_wwn = ibmvscsis_get_fabric_wwn,
3957	.tpg_get_tag = ibmvscsis_get_tag,
3958	.tpg_get_default_depth = ibmvscsis_get_default_depth,
3959	.tpg_check_demo_mode = ibmvscsis_check_true,
3960	.tpg_check_demo_mode_cache = ibmvscsis_check_true,
3961	.check_stop_free = ibmvscsis_check_stop_free,
3962	.release_cmd = ibmvscsis_release_cmd,
3963	.write_pending = ibmvscsis_write_pending,
3964	.queue_data_in = ibmvscsis_queue_data_in,
3965	.queue_status = ibmvscsis_queue_status,
3966	.queue_tm_rsp = ibmvscsis_queue_tm_rsp,
3967	.aborted_task = ibmvscsis_aborted_task,
3968	/*
3969	* Setup function pointers for logic in target_core_fabric_configfs.c
3970	*/
3971	.fabric_make_wwn = ibmvscsis_make_tport,
3972	.fabric_drop_wwn = ibmvscsis_drop_tport,
3973	.fabric_make_tpg = ibmvscsis_make_tpg,
3974	.fabric_enable_tpg = ibmvscsis_enable_tpg,
3975	.fabric_drop_tpg = ibmvscsis_drop_tpg,
3976
3977	.tfc_wwn_attrs = ibmvscsis_wwn_attrs,
3978
3979	.default_submit_type = TARGET_DIRECT_SUBMIT,
3980	.direct_submit_supp = 1,
3981	};
3982
3983	static void ibmvscsis_dev_release(struct device *dev) {};
3984
3985	static struct device_attribute dev_attr_system_id =
3986	__ATTR(system_id, S_IRUGO, system_id_show, NULL);
3987
3988	static struct device_attribute dev_attr_partition_number =
3989	__ATTR(partition_number, S_IRUGO, partition_number_show, NULL);
3990
3991	static struct device_attribute dev_attr_unit_address =
3992	__ATTR(unit_address, S_IRUGO, unit_address_show, NULL);
3993
3994	static struct attribute *ibmvscsis_dev_attrs[] = {
3995	&dev_attr_system_id.attr,
3996	&dev_attr_partition_number.attr,
3997	&dev_attr_unit_address.attr,
3998	};
3999	ATTRIBUTE_GROUPS(ibmvscsis_dev);
4000
4001	static struct class ibmvscsis_class = {
4002	.name = "ibmvscsis",
4003	.dev_release = ibmvscsis_dev_release,
4004	.dev_groups = ibmvscsis_dev_groups,
4005	};
4006
4007	static const struct vio_device_id ibmvscsis_device_table[] = {
4008	{ .type: "v-scsi-host", .compat: "IBM,v-scsi-host" },
4009	{ "", "" }
4010	};
4011	MODULE_DEVICE_TABLE(vio, ibmvscsis_device_table);
4012
4013	static struct vio_driver ibmvscsis_driver = {
4014	.name = "ibmvscsis",
4015	.id_table = ibmvscsis_device_table,
4016	.probe = ibmvscsis_probe,
4017	.remove = ibmvscsis_remove,
4018	};
4019
4020	/*
4021	* ibmvscsis_init() - Kernel Module initialization
4022	*
4023	* Note: vio_register_driver() registers callback functions, and at least one
4024	* of those callback functions calls TCM - Linux IO Target Subsystem, thus
4025	* the SCSI Target template must be registered before vio_register_driver()
4026	* is called.
4027	*/
4028	static int __init ibmvscsis_init(void)
4029	{
4030	int rc = 0;
4031
4032	rc = ibmvscsis_get_system_info();
4033	if (rc) {
4034	pr_err("rc %d from get_system_info\n", rc);
4035	goto out;
4036	}
4037
4038	rc = class_register(&ibmvscsis_class);
4039	if (rc) {
4040	pr_err("failed class register\n");
4041	goto out;
4042	}
4043
4044	rc = target_register_template(fo: &ibmvscsis_ops);
4045	if (rc) {
4046	pr_err("rc %d from target_register_template\n", rc);
4047	goto unregister_class;
4048	}
4049
4050	rc = vio_register_driver(&ibmvscsis_driver);
4051	if (rc) {
4052	pr_err("rc %d from vio_register_driver\n", rc);
4053	goto unregister_target;
4054	}
4055
4056	return 0;
4057
4058	unregister_target:
4059	target_unregister_template(fo: &ibmvscsis_ops);
4060	unregister_class:
4061	class_unregister(&ibmvscsis_class);
4062	out:
4063	return rc;
4064	}
4065
4066	static void __exit ibmvscsis_exit(void)
4067	{
4068	pr_info("Unregister IBM virtual SCSI host driver\n");
4069	vio_unregister_driver(&ibmvscsis_driver);
4070	target_unregister_template(fo: &ibmvscsis_ops);
4071	class_unregister(&ibmvscsis_class);
4072	}
4073
4074	MODULE_DESCRIPTION("IBMVSCSIS fabric driver");
4075	MODULE_AUTHOR("Bryant G. Ly and Michael Cyr");
4076	MODULE_LICENSE("GPL");
4077	MODULE_VERSION(IBMVSCSIS_VERSION);
4078	module_init(ibmvscsis_init);
4079	module_exit(ibmvscsis_exit);
4080