1	// SPDX-License-Identifier: GPL-2.0+
2	/*******************************************************************************
3	* Vhost kernel TCM fabric driver for virtio SCSI initiators
4	*
5	* (C) Copyright 2010-2013 Datera, Inc.
6	* (C) Copyright 2010-2012 IBM Corp.
7	*
8	* Authors: Nicholas A. Bellinger <nab@daterainc.com>
9	* Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
10	****************************************************************************/
11
12	#include <linux/module.h>
13	#include <linux/moduleparam.h>
14	#include <generated/utsrelease.h>
15	#include <linux/utsname.h>
16	#include <linux/init.h>
17	#include <linux/slab.h>
18	#include <linux/kthread.h>
19	#include <linux/types.h>
20	#include <linux/string.h>
21	#include <linux/configfs.h>
22	#include <linux/ctype.h>
23	#include <linux/compat.h>
24	#include <linux/eventfd.h>
25	#include <linux/fs.h>
26	#include <linux/vmalloc.h>
27	#include <linux/miscdevice.h>
28	#include <linux/blk_types.h>
29	#include <linux/bio.h>
30	#include <asm/unaligned.h>
31	#include <scsi/scsi_common.h>
32	#include <scsi/scsi_proto.h>
33	#include <target/target_core_base.h>
34	#include <target/target_core_fabric.h>
35	#include <linux/vhost.h>
36	#include <linux/virtio_scsi.h>
37	#include <linux/llist.h>
38	#include <linux/bitmap.h>
39
40	#include "vhost.h"
41
42	#define VHOST_SCSI_VERSION "v0.1"
43	#define VHOST_SCSI_NAMELEN 256
44	#define VHOST_SCSI_MAX_CDB_SIZE 32
45	#define VHOST_SCSI_PREALLOC_SGLS 2048
46	#define VHOST_SCSI_PREALLOC_UPAGES 2048
47	#define VHOST_SCSI_PREALLOC_PROT_SGLS 2048
48
49	/* Max number of requests before requeueing the job.
50	* Using this limit prevents one virtqueue from starving others with
51	* request.
52	*/
53	#define VHOST_SCSI_WEIGHT 256
54
55	struct vhost_scsi_inflight {
56	/* Wait for the flush operation to finish */
57	struct completion comp;
58	/* Refcount for the inflight reqs */
59	struct kref kref;
60	};
61
62	struct vhost_scsi_cmd {
63	/* Descriptor from vhost_get_vq_desc() for virt_queue segment */
64	int tvc_vq_desc;
65	/* virtio-scsi initiator task attribute */
66	int tvc_task_attr;
67	/* virtio-scsi response incoming iovecs */
68	int tvc_in_iovs;
69	/* virtio-scsi initiator data direction */
70	enum dma_data_direction tvc_data_direction;
71	/* Expected data transfer length from virtio-scsi header */
72	u32 tvc_exp_data_len;
73	/* The Tag from include/linux/virtio_scsi.h:struct virtio_scsi_cmd_req */
74	u64 tvc_tag;
75	/* The number of scatterlists associated with this cmd */
76	u32 tvc_sgl_count;
77	u32 tvc_prot_sgl_count;
78	/* Saved unpacked SCSI LUN for vhost_scsi_target_queue_cmd() */
79	u32 tvc_lun;
80	u32 copied_iov:1;
81	const void *saved_iter_addr;
82	struct iov_iter saved_iter;
83	/* Pointer to the SGL formatted memory from virtio-scsi */
84	struct scatterlist *tvc_sgl;
85	struct scatterlist *tvc_prot_sgl;
86	struct page **tvc_upages;
87	/* Pointer to response header iovec */
88	struct iovec *tvc_resp_iov;
89	/* Pointer to vhost_scsi for our device */
90	struct vhost_scsi *tvc_vhost;
91	/* Pointer to vhost_virtqueue for the cmd */
92	struct vhost_virtqueue *tvc_vq;
93	/* Pointer to vhost nexus memory */
94	struct vhost_scsi_nexus *tvc_nexus;
95	/* The TCM I/O descriptor that is accessed via container_of() */
96	struct se_cmd tvc_se_cmd;
97	/* Copy of the incoming SCSI command descriptor block (CDB) */
98	unsigned char tvc_cdb[VHOST_SCSI_MAX_CDB_SIZE];
99	/* Sense buffer that will be mapped into outgoing status */
100	unsigned char tvc_sense_buf[TRANSPORT_SENSE_BUFFER];
101	/* Completed commands list, serviced from vhost worker thread */
102	struct llist_node tvc_completion_list;
103	/* Used to track inflight cmd */
104	struct vhost_scsi_inflight *inflight;
105	};
106
107	struct vhost_scsi_nexus {
108	/* Pointer to TCM session for I_T Nexus */
109	struct se_session *tvn_se_sess;
110	};
111
112	struct vhost_scsi_tpg {
113	/* Vhost port target portal group tag for TCM */
114	u16 tport_tpgt;
115	/* Used to track number of TPG Port/Lun Links wrt to explict I_T Nexus shutdown */
116	int tv_tpg_port_count;
117	/* Used for vhost_scsi device reference to tpg_nexus, protected by tv_tpg_mutex */
118	int tv_tpg_vhost_count;
119	/* Used for enabling T10-PI with legacy devices */
120	int tv_fabric_prot_type;
121	/* list for vhost_scsi_list */
122	struct list_head tv_tpg_list;
123	/* Used to protect access for tpg_nexus */
124	struct mutex tv_tpg_mutex;
125	/* Pointer to the TCM VHost I_T Nexus for this TPG endpoint */
126	struct vhost_scsi_nexus *tpg_nexus;
127	/* Pointer back to vhost_scsi_tport */
128	struct vhost_scsi_tport *tport;
129	/* Returned by vhost_scsi_make_tpg() */
130	struct se_portal_group se_tpg;
131	/* Pointer back to vhost_scsi, protected by tv_tpg_mutex */
132	struct vhost_scsi *vhost_scsi;
133	};
134
135	struct vhost_scsi_tport {
136	/* SCSI protocol the tport is providing */
137	u8 tport_proto_id;
138	/* Binary World Wide unique Port Name for Vhost Target port */
139	u64 tport_wwpn;
140	/* ASCII formatted WWPN for Vhost Target port */
141	char tport_name[VHOST_SCSI_NAMELEN];
142	/* Returned by vhost_scsi_make_tport() */
143	struct se_wwn tport_wwn;
144	};
145
146	struct vhost_scsi_evt {
147	/* event to be sent to guest */
148	struct virtio_scsi_event event;
149	/* event list, serviced from vhost worker thread */
150	struct llist_node list;
151	};
152
153	enum {
154	VHOST_SCSI_VQ_CTL = 0,
155	VHOST_SCSI_VQ_EVT = 1,
156	VHOST_SCSI_VQ_IO = 2,
157	};
158
159	/* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */
160	enum {
161	VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG) |
162	(1ULL << VIRTIO_SCSI_F_T10_PI)
163	};
164
165	#define VHOST_SCSI_MAX_TARGET 256
166	#define VHOST_SCSI_MAX_IO_VQ 1024
167	#define VHOST_SCSI_MAX_EVENT 128
168
169	static unsigned vhost_scsi_max_io_vqs = 128;
170	module_param_named(max_io_vqs, vhost_scsi_max_io_vqs, uint, 0644);
171	MODULE_PARM_DESC(max_io_vqs, "Set the max number of IO virtqueues a vhost scsi device can support. The default is 128. The max is 1024.");
172
173	struct vhost_scsi_virtqueue {
174	struct vhost_virtqueue vq;
175	struct vhost_scsi *vs;
176	/*
177	* Reference counting for inflight reqs, used for flush operation. At
178	* each time, one reference tracks new commands submitted, while we
179	* wait for another one to reach 0.
180	*/
181	struct vhost_scsi_inflight inflights[2];
182	/*
183	* Indicate current inflight in use, protected by vq->mutex.
184	* Writers must also take dev mutex and flush under it.
185	*/
186	int inflight_idx;
187	struct vhost_scsi_cmd *scsi_cmds;
188	struct sbitmap scsi_tags;
189	int max_cmds;
190
191	struct vhost_work completion_work;
192	struct llist_head completion_list;
193	};
194
195	struct vhost_scsi {
196	/* Protected by vhost_scsi->dev.mutex */
197	struct vhost_scsi_tpg **vs_tpg;
198	char vs_vhost_wwpn[TRANSPORT_IQN_LEN];
199
200	struct vhost_dev dev;
201	struct vhost_scsi_virtqueue *vqs;
202	struct vhost_scsi_inflight **old_inflight;
203
204	struct vhost_work vs_event_work; /* evt injection work item */
205	struct llist_head vs_event_list; /* evt injection queue */
206
207	bool vs_events_missed; /* any missed events, protected by vq->mutex */
208	int vs_events_nr; /* num of pending events, protected by vq->mutex */
209	};
210
211	struct vhost_scsi_tmf {
212	struct vhost_work vwork;
213	struct vhost_scsi *vhost;
214	struct vhost_scsi_virtqueue *svq;
215
216	struct se_cmd se_cmd;
217	u8 scsi_resp;
218	struct vhost_scsi_inflight *inflight;
219	struct iovec resp_iov;
220	int in_iovs;
221	int vq_desc;
222	};
223
224	/*
225	* Context for processing request and control queue operations.
226	*/
227	struct vhost_scsi_ctx {
228	int head;
229	unsigned int out, in;
230	size_t req_size, rsp_size;
231	size_t out_size, in_size;
232	u8 *target, *lunp;
233	void *req;
234	struct iov_iter out_iter;
235	};
236
237	/*
238	* Global mutex to protect vhost_scsi TPG list for vhost IOCTLs and LIO
239	* configfs management operations.
240	*/
241	static DEFINE_MUTEX(vhost_scsi_mutex);
242	static LIST_HEAD(vhost_scsi_list);
243
244	static void vhost_scsi_done_inflight(struct kref *kref)
245	{
246	struct vhost_scsi_inflight *inflight;
247
248	inflight = container_of(kref, struct vhost_scsi_inflight, kref);
249	complete(&inflight->comp);
250	}
251
252	static void vhost_scsi_init_inflight(struct vhost_scsi *vs,
253	struct vhost_scsi_inflight *old_inflight[])
254	{
255	struct vhost_scsi_inflight *new_inflight;
256	struct vhost_virtqueue *vq;
257	int idx, i;
258
259	for (i = 0; i < vs->dev.nvqs; i++) {
260	vq = &vs->vqs[i].vq;
261
262	mutex_lock(&vq->mutex);
263
264	/* store old infight */
265	idx = vs->vqs[i].inflight_idx;
266	if (old_inflight)
267	old_inflight[i] = &vs->vqs[i].inflights[idx];
268
269	/* setup new infight */
270	vs->vqs[i].inflight_idx = idx ^ 1;
271	new_inflight = &vs->vqs[i].inflights[idx ^ 1];
272	kref_init(kref: &new_inflight->kref);
273	init_completion(x: &new_inflight->comp);
274
275	mutex_unlock(lock: &vq->mutex);
276	}
277	}
278
279	static struct vhost_scsi_inflight *
280	vhost_scsi_get_inflight(struct vhost_virtqueue *vq)
281	{
282	struct vhost_scsi_inflight *inflight;
283	struct vhost_scsi_virtqueue *svq;
284
285	svq = container_of(vq, struct vhost_scsi_virtqueue, vq);
286	inflight = &svq->inflights[svq->inflight_idx];
287	kref_get(kref: &inflight->kref);
288
289	return inflight;
290	}
291
292	static void vhost_scsi_put_inflight(struct vhost_scsi_inflight *inflight)
293	{
294	kref_put(kref: &inflight->kref, release: vhost_scsi_done_inflight);
295	}
296
297	static int vhost_scsi_check_true(struct se_portal_group *se_tpg)
298	{
299	return 1;
300	}
301
302	static char *vhost_scsi_get_fabric_wwn(struct se_portal_group *se_tpg)
303	{
304	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
305	struct vhost_scsi_tpg, se_tpg);
306	struct vhost_scsi_tport *tport = tpg->tport;
307
308	return &tport->tport_name[0];
309	}
310
311	static u16 vhost_scsi_get_tpgt(struct se_portal_group *se_tpg)
312	{
313	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
314	struct vhost_scsi_tpg, se_tpg);
315	return tpg->tport_tpgt;
316	}
317
318	static int vhost_scsi_check_prot_fabric_only(struct se_portal_group *se_tpg)
319	{
320	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
321	struct vhost_scsi_tpg, se_tpg);
322
323	return tpg->tv_fabric_prot_type;
324	}
325
326	static void vhost_scsi_release_cmd_res(struct se_cmd *se_cmd)
327	{
328	struct vhost_scsi_cmd *tv_cmd = container_of(se_cmd,
329	struct vhost_scsi_cmd, tvc_se_cmd);
330	struct vhost_scsi_virtqueue *svq = container_of(tv_cmd->tvc_vq,
331	struct vhost_scsi_virtqueue, vq);
332	struct vhost_scsi_inflight *inflight = tv_cmd->inflight;
333	int i;
334
335	if (tv_cmd->tvc_sgl_count) {
336	for (i = 0; i < tv_cmd->tvc_sgl_count; i++) {
337	if (tv_cmd->copied_iov)
338	__free_page(sg_page(&tv_cmd->tvc_sgl[i]));
339	else
340	put_page(page: sg_page(sg: &tv_cmd->tvc_sgl[i]));
341	}
342	kfree(objp: tv_cmd->saved_iter_addr);
343	}
344	if (tv_cmd->tvc_prot_sgl_count) {
345	for (i = 0; i < tv_cmd->tvc_prot_sgl_count; i++)
346	put_page(page: sg_page(sg: &tv_cmd->tvc_prot_sgl[i]));
347	}
348
349	sbitmap_clear_bit(sb: &svq->scsi_tags, bitnr: se_cmd->map_tag);
350	vhost_scsi_put_inflight(inflight);
351	}
352
353	static void vhost_scsi_release_tmf_res(struct vhost_scsi_tmf *tmf)
354	{
355	struct vhost_scsi_inflight *inflight = tmf->inflight;
356
357	kfree(objp: tmf);
358	vhost_scsi_put_inflight(inflight);
359	}
360
361	static void vhost_scsi_release_cmd(struct se_cmd *se_cmd)
362	{
363	if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) {
364	struct vhost_scsi_tmf *tmf = container_of(se_cmd,
365	struct vhost_scsi_tmf, se_cmd);
366	struct vhost_virtqueue *vq = &tmf->svq->vq;
367
368	vhost_vq_work_queue(vq, work: &tmf->vwork);
369	} else {
370	struct vhost_scsi_cmd *cmd = container_of(se_cmd,
371	struct vhost_scsi_cmd, tvc_se_cmd);
372	struct vhost_scsi_virtqueue *svq = container_of(cmd->tvc_vq,
373	struct vhost_scsi_virtqueue, vq);
374
375	llist_add(new: &cmd->tvc_completion_list, head: &svq->completion_list);
376	vhost_vq_work_queue(vq: &svq->vq, work: &svq->completion_work);
377	}
378	}
379
380	static int vhost_scsi_write_pending(struct se_cmd *se_cmd)
381	{
382	/* Go ahead and process the write immediately */
383	target_execute_cmd(cmd: se_cmd);
384	return 0;
385	}
386
387	static int vhost_scsi_queue_data_in(struct se_cmd *se_cmd)
388	{
389	transport_generic_free_cmd(se_cmd, 0);
390	return 0;
391	}
392
393	static int vhost_scsi_queue_status(struct se_cmd *se_cmd)
394	{
395	transport_generic_free_cmd(se_cmd, 0);
396	return 0;
397	}
398
399	static void vhost_scsi_queue_tm_rsp(struct se_cmd *se_cmd)
400	{
401	struct vhost_scsi_tmf *tmf = container_of(se_cmd, struct vhost_scsi_tmf,
402	se_cmd);
403
404	tmf->scsi_resp = se_cmd->se_tmr_req->response;
405	transport_generic_free_cmd(&tmf->se_cmd, 0);
406	}
407
408	static void vhost_scsi_aborted_task(struct se_cmd *se_cmd)
409	{
410	return;
411	}
412
413	static void vhost_scsi_free_evt(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)
414	{
415	vs->vs_events_nr--;
416	kfree(objp: evt);
417	}
418
419	static struct vhost_scsi_evt *
420	vhost_scsi_allocate_evt(struct vhost_scsi *vs,
421	u32 event, u32 reason)
422	{
423	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
424	struct vhost_scsi_evt *evt;
425
426	if (vs->vs_events_nr > VHOST_SCSI_MAX_EVENT) {
427	vs->vs_events_missed = true;
428	return NULL;
429	}
430
431	evt = kzalloc(size: sizeof(*evt), GFP_KERNEL);
432	if (!evt) {
433	vq_err(vq, "Failed to allocate vhost_scsi_evt\n");
434	vs->vs_events_missed = true;
435	return NULL;
436	}
437
438	evt->event.event = cpu_to_vhost32(vq, val: event);
439	evt->event.reason = cpu_to_vhost32(vq, val: reason);
440	vs->vs_events_nr++;
441
442	return evt;
443	}
444
445	static int vhost_scsi_check_stop_free(struct se_cmd *se_cmd)
446	{
447	return target_put_sess_cmd(se_cmd);
448	}
449
450	static void
451	vhost_scsi_do_evt_work(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)
452	{
453	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
454	struct virtio_scsi_event *event = &evt->event;
455	struct virtio_scsi_event __user *eventp;
456	unsigned out, in;
457	int head, ret;
458
459	if (!vhost_vq_get_backend(vq)) {
460	vs->vs_events_missed = true;
461	return;
462	}
463
464	again:
465	vhost_disable_notify(&vs->dev, vq);
466	head = vhost_get_vq_desc(vq, iov: vq->iov,
467	ARRAY_SIZE(vq->iov), out_num: &out, in_num: &in,
468	NULL, NULL);
469	if (head < 0) {
470	vs->vs_events_missed = true;
471	return;
472	}
473	if (head == vq->num) {
474	if (vhost_enable_notify(&vs->dev, vq))
475	goto again;
476	vs->vs_events_missed = true;
477	return;
478	}
479
480	if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {
481	vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",
482	vq->iov[out].iov_len);
483	vs->vs_events_missed = true;
484	return;
485	}
486
487	if (vs->vs_events_missed) {
488	event->event |= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED);
489	vs->vs_events_missed = false;
490	}
491
492	eventp = vq->iov[out].iov_base;
493	ret = __copy_to_user(to: eventp, from: event, n: sizeof(*event));
494	if (!ret)
495	vhost_add_used_and_signal(&vs->dev, vq, id: head, len: 0);
496	else
497	vq_err(vq, "Faulted on vhost_scsi_send_event\n");
498	}
499
500	static void vhost_scsi_evt_work(struct vhost_work *work)
501	{
502	struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
503	vs_event_work);
504	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
505	struct vhost_scsi_evt *evt, *t;
506	struct llist_node *llnode;
507
508	mutex_lock(&vq->mutex);
509	llnode = llist_del_all(head: &vs->vs_event_list);
510	llist_for_each_entry_safe(evt, t, llnode, list) {
511	vhost_scsi_do_evt_work(vs, evt);
512	vhost_scsi_free_evt(vs, evt);
513	}
514	mutex_unlock(lock: &vq->mutex);
515	}
516
517	static int vhost_scsi_copy_sgl_to_iov(struct vhost_scsi_cmd *cmd)
518	{
519	struct iov_iter *iter = &cmd->saved_iter;
520	struct scatterlist *sg = cmd->tvc_sgl;
521	struct page *page;
522	size_t len;
523	int i;
524
525	for (i = 0; i < cmd->tvc_sgl_count; i++) {
526	page = sg_page(sg: &sg[i]);
527	len = sg[i].length;
528
529	if (copy_page_to_iter(page, offset: 0, bytes: len, i: iter) != len) {
530	pr_err("Could not copy data while handling misaligned cmd. Error %zu\n",
531	len);
532	return -1;
533	}
534	}
535
536	return 0;
537	}
538
539	/* Fill in status and signal that we are done processing this command
540	*
541	* This is scheduled in the vhost work queue so we are called with the owner
542	* process mm and can access the vring.
543	*/
544	static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
545	{
546	struct vhost_scsi_virtqueue *svq = container_of(work,
547	struct vhost_scsi_virtqueue, completion_work);
548	struct virtio_scsi_cmd_resp v_rsp;
549	struct vhost_scsi_cmd *cmd, *t;
550	struct llist_node *llnode;
551	struct se_cmd *se_cmd;
552	struct iov_iter iov_iter;
553	bool signal = false;
554	int ret;
555
556	llnode = llist_del_all(head: &svq->completion_list);
557	llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) {
558	se_cmd = &cmd->tvc_se_cmd;
559
560	pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__,
561	cmd, se_cmd->residual_count, se_cmd->scsi_status);
562	memset(&v_rsp, 0, sizeof(v_rsp));
563
564	if (cmd->saved_iter_addr && vhost_scsi_copy_sgl_to_iov(cmd)) {
565	v_rsp.response = VIRTIO_SCSI_S_BAD_TARGET;
566	} else {
567	v_rsp.resid = cpu_to_vhost32(vq: cmd->tvc_vq,
568	val: se_cmd->residual_count);
569	/* TODO is status_qualifier field needed? */
570	v_rsp.status = se_cmd->scsi_status;
571	v_rsp.sense_len = cpu_to_vhost32(vq: cmd->tvc_vq,
572	val: se_cmd->scsi_sense_length);
573	memcpy(v_rsp.sense, cmd->tvc_sense_buf,
574	se_cmd->scsi_sense_length);
575	}
576
577	iov_iter_init(i: &iov_iter, ITER_DEST, iov: cmd->tvc_resp_iov,
578	nr_segs: cmd->tvc_in_iovs, count: sizeof(v_rsp));
579	ret = copy_to_iter(addr: &v_rsp, bytes: sizeof(v_rsp), i: &iov_iter);
580	if (likely(ret == sizeof(v_rsp))) {
581	signal = true;
582
583	vhost_add_used(cmd->tvc_vq, head: cmd->tvc_vq_desc, len: 0);
584	} else
585	pr_err("Faulted on virtio_scsi_cmd_resp\n");
586
587	vhost_scsi_release_cmd_res(se_cmd);
588	}
589
590	if (signal)
591	vhost_signal(&svq->vs->dev, &svq->vq);
592	}
593
594	static struct vhost_scsi_cmd *
595	vhost_scsi_get_cmd(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg,
596	unsigned char *cdb, u64 scsi_tag, u16 lun, u8 task_attr,
597	u32 exp_data_len, int data_direction)
598	{
599	struct vhost_scsi_virtqueue *svq = container_of(vq,
600	struct vhost_scsi_virtqueue, vq);
601	struct vhost_scsi_cmd *cmd;
602	struct vhost_scsi_nexus *tv_nexus;
603	struct scatterlist *sg, *prot_sg;
604	struct iovec *tvc_resp_iov;
605	struct page **pages;
606	int tag;
607
608	tv_nexus = tpg->tpg_nexus;
609	if (!tv_nexus) {
610	pr_err("Unable to locate active struct vhost_scsi_nexus\n");
611	return ERR_PTR(error: -EIO);
612	}
613
614	tag = sbitmap_get(sb: &svq->scsi_tags);
615	if (tag < 0) {
616	pr_err("Unable to obtain tag for vhost_scsi_cmd\n");
617	return ERR_PTR(error: -ENOMEM);
618	}
619
620	cmd = &svq->scsi_cmds[tag];
621	sg = cmd->tvc_sgl;
622	prot_sg = cmd->tvc_prot_sgl;
623	pages = cmd->tvc_upages;
624	tvc_resp_iov = cmd->tvc_resp_iov;
625	memset(cmd, 0, sizeof(*cmd));
626	cmd->tvc_sgl = sg;
627	cmd->tvc_prot_sgl = prot_sg;
628	cmd->tvc_upages = pages;
629	cmd->tvc_se_cmd.map_tag = tag;
630	cmd->tvc_tag = scsi_tag;
631	cmd->tvc_lun = lun;
632	cmd->tvc_task_attr = task_attr;
633	cmd->tvc_exp_data_len = exp_data_len;
634	cmd->tvc_data_direction = data_direction;
635	cmd->tvc_nexus = tv_nexus;
636	cmd->inflight = vhost_scsi_get_inflight(vq);
637	cmd->tvc_resp_iov = tvc_resp_iov;
638
639	memcpy(cmd->tvc_cdb, cdb, VHOST_SCSI_MAX_CDB_SIZE);
640
641	return cmd;
642	}
643
644	/*
645	* Map a user memory range into a scatterlist
646	*
647	* Returns the number of scatterlist entries used or -errno on error.
648	*/
649	static int
650	vhost_scsi_map_to_sgl(struct vhost_scsi_cmd *cmd,
651	struct iov_iter *iter,
652	struct scatterlist *sgl,
653	bool is_prot)
654	{
655	struct page **pages = cmd->tvc_upages;
656	struct scatterlist *sg = sgl;
657	ssize_t bytes, mapped_bytes;
658	size_t offset, mapped_offset;
659	unsigned int npages = 0;
660
661	bytes = iov_iter_get_pages2(i: iter, pages, LONG_MAX,
662	VHOST_SCSI_PREALLOC_UPAGES, start: &offset);
663	/* No pages were pinned */
664	if (bytes <= 0)
665	return bytes < 0 ? bytes : -EFAULT;
666
667	mapped_bytes = bytes;
668	mapped_offset = offset;
669
670	while (bytes) {
671	unsigned n = min_t(unsigned, PAGE_SIZE - offset, bytes);
672	/*
673	* The block layer requires bios/requests to be a multiple of
674	* 512 bytes, but Windows can send us vecs that are misaligned.
675	* This can result in bios and later requests with misaligned
676	* sizes if we have to break up a cmd/scatterlist into multiple
677	* bios.
678	*
679	* We currently only break up a command into multiple bios if
680	* we hit the vec/seg limit, so check if our sgl_count is
681	* greater than the max and if a vec in the cmd has a
682	* misaligned offset/size.
683	*/
684	if (!is_prot &&
685	(offset & (SECTOR_SIZE - 1) || n & (SECTOR_SIZE - 1)) &&
686	cmd->tvc_sgl_count > BIO_MAX_VECS) {
687	WARN_ONCE(true,
688	"vhost-scsi detected misaligned IO. Performance may be degraded.");
689	goto revert_iter_get_pages;
690	}
691
692	sg_set_page(sg: sg++, page: pages[npages++], len: n, offset);
693	bytes -= n;
694	offset = 0;
695	}
696
697	return npages;
698
699	revert_iter_get_pages:
700	iov_iter_revert(i: iter, bytes: mapped_bytes);
701
702	npages = 0;
703	while (mapped_bytes) {
704	unsigned int n = min_t(unsigned int, PAGE_SIZE - mapped_offset,
705	mapped_bytes);
706
707	put_page(page: pages[npages++]);
708
709	mapped_bytes -= n;
710	mapped_offset = 0;
711	}
712
713	return -EINVAL;
714	}
715
716	static int
717	vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls)
718	{
719	int sgl_count = 0;
720
721	if (!iter || !iter_iov(iter)) {
722	pr_err("%s: iter->iov is NULL, but expected bytes: %zu"
723	" present\n", __func__, bytes);
724	return -EINVAL;
725	}
726
727	sgl_count = iov_iter_npages(i: iter, maxpages: 0xffff);
728	if (sgl_count > max_sgls) {
729	pr_err("%s: requested sgl_count: %d exceeds pre-allocated"
730	" max_sgls: %d\n", __func__, sgl_count, max_sgls);
731	return -EINVAL;
732	}
733	return sgl_count;
734	}
735
736	static int
737	vhost_scsi_copy_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter,
738	struct scatterlist *sg, int sg_count)
739	{
740	size_t len = iov_iter_count(i: iter);
741	unsigned int nbytes = 0;
742	struct page *page;
743	int i;
744
745	if (cmd->tvc_data_direction == DMA_FROM_DEVICE) {
746	cmd->saved_iter_addr = dup_iter(new: &cmd->saved_iter, old: iter,
747	GFP_KERNEL);
748	if (!cmd->saved_iter_addr)
749	return -ENOMEM;
750	}
751
752	for (i = 0; i < sg_count; i++) {
753	page = alloc_page(GFP_KERNEL);
754	if (!page) {
755	i--;
756	goto err;
757	}
758
759	nbytes = min_t(unsigned int, PAGE_SIZE, len);
760	sg_set_page(sg: &sg[i], page, len: nbytes, offset: 0);
761
762	if (cmd->tvc_data_direction == DMA_TO_DEVICE &&
763	copy_page_from_iter(page, offset: 0, bytes: nbytes, i: iter) != nbytes)
764	goto err;
765
766	len -= nbytes;
767	}
768
769	cmd->copied_iov = 1;
770	return 0;
771
772	err:
773	pr_err("Could not read %u bytes while handling misaligned cmd\n",
774	nbytes);
775
776	for (; i >= 0; i--)
777	__free_page(sg_page(&sg[i]));
778	kfree(objp: cmd->saved_iter_addr);
779	return -ENOMEM;
780	}
781
782	static int
783	vhost_scsi_map_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter,
784	struct scatterlist *sg, int sg_count, bool is_prot)
785	{
786	struct scatterlist *p = sg;
787	size_t revert_bytes;
788	int ret;
789
790	while (iov_iter_count(i: iter)) {
791	ret = vhost_scsi_map_to_sgl(cmd, iter, sgl: sg, is_prot);
792	if (ret < 0) {
793	revert_bytes = 0;
794
795	while (p < sg) {
796	struct page *page = sg_page(sg: p);
797
798	if (page) {
799	put_page(page);
800	revert_bytes += p->length;
801	}
802	p++;
803	}
804
805	iov_iter_revert(i: iter, bytes: revert_bytes);
806	return ret;
807	}
808	sg += ret;
809	}
810
811	return 0;
812	}
813
814	static int
815	vhost_scsi_mapal(struct vhost_scsi_cmd *cmd,
816	size_t prot_bytes, struct iov_iter *prot_iter,
817	size_t data_bytes, struct iov_iter *data_iter)
818	{
819	int sgl_count, ret;
820
821	if (prot_bytes) {
822	sgl_count = vhost_scsi_calc_sgls(iter: prot_iter, bytes: prot_bytes,
823	VHOST_SCSI_PREALLOC_PROT_SGLS);
824	if (sgl_count < 0)
825	return sgl_count;
826
827	sg_init_table(cmd->tvc_prot_sgl, sgl_count);
828	cmd->tvc_prot_sgl_count = sgl_count;
829	pr_debug("%s prot_sg %p prot_sgl_count %u\n", __func__,
830	cmd->tvc_prot_sgl, cmd->tvc_prot_sgl_count);
831
832	ret = vhost_scsi_map_iov_to_sgl(cmd, iter: prot_iter,
833	sg: cmd->tvc_prot_sgl,
834	sg_count: cmd->tvc_prot_sgl_count, is_prot: true);
835	if (ret < 0) {
836	cmd->tvc_prot_sgl_count = 0;
837	return ret;
838	}
839	}
840	sgl_count = vhost_scsi_calc_sgls(iter: data_iter, bytes: data_bytes,
841	VHOST_SCSI_PREALLOC_SGLS);
842	if (sgl_count < 0)
843	return sgl_count;
844
845	sg_init_table(cmd->tvc_sgl, sgl_count);
846	cmd->tvc_sgl_count = sgl_count;
847	pr_debug("%s data_sg %p data_sgl_count %u\n", __func__,
848	cmd->tvc_sgl, cmd->tvc_sgl_count);
849
850	ret = vhost_scsi_map_iov_to_sgl(cmd, iter: data_iter, sg: cmd->tvc_sgl,
851	sg_count: cmd->tvc_sgl_count, is_prot: false);
852	if (ret == -EINVAL) {
853	sg_init_table(cmd->tvc_sgl, cmd->tvc_sgl_count);
854	ret = vhost_scsi_copy_iov_to_sgl(cmd, iter: data_iter, sg: cmd->tvc_sgl,
855	sg_count: cmd->tvc_sgl_count);
856	}
857
858	if (ret < 0) {
859	cmd->tvc_sgl_count = 0;
860	return ret;
861	}
862	return 0;
863	}
864
865	static int vhost_scsi_to_tcm_attr(int attr)
866	{
867	switch (attr) {
868	case VIRTIO_SCSI_S_SIMPLE:
869	return TCM_SIMPLE_TAG;
870	case VIRTIO_SCSI_S_ORDERED:
871	return TCM_ORDERED_TAG;
872	case VIRTIO_SCSI_S_HEAD:
873	return TCM_HEAD_TAG;
874	case VIRTIO_SCSI_S_ACA:
875	return TCM_ACA_TAG;
876	default:
877	break;
878	}
879	return TCM_SIMPLE_TAG;
880	}
881
882	static void vhost_scsi_target_queue_cmd(struct vhost_scsi_cmd *cmd)
883	{
884	struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
885	struct vhost_scsi_nexus *tv_nexus;
886	struct scatterlist *sg_ptr, *sg_prot_ptr = NULL;
887
888	/* FIXME: BIDI operation */
889	if (cmd->tvc_sgl_count) {
890	sg_ptr = cmd->tvc_sgl;
891
892	if (cmd->tvc_prot_sgl_count)
893	sg_prot_ptr = cmd->tvc_prot_sgl;
894	else
895	se_cmd->prot_pto = true;
896	} else {
897	sg_ptr = NULL;
898	}
899	tv_nexus = cmd->tvc_nexus;
900
901	se_cmd->tag = 0;
902	target_init_cmd(se_cmd, se_sess: tv_nexus->tvn_se_sess, sense: &cmd->tvc_sense_buf[0],
903	unpacked_lun: cmd->tvc_lun, data_length: cmd->tvc_exp_data_len,
904	task_attr: vhost_scsi_to_tcm_attr(attr: cmd->tvc_task_attr),
905	data_dir: cmd->tvc_data_direction, flags: TARGET_SCF_ACK_KREF);
906
907	if (target_submit_prep(se_cmd, cdb: cmd->tvc_cdb, sgl: sg_ptr,
908	sgl_count: cmd->tvc_sgl_count, NULL, sgl_bidi_count: 0, sgl_prot: sg_prot_ptr,
909	sgl_prot_count: cmd->tvc_prot_sgl_count, GFP_KERNEL))
910	return;
911
912	target_submit(se_cmd);
913	}
914
915	static void
916	vhost_scsi_send_bad_target(struct vhost_scsi *vs,
917	struct vhost_virtqueue *vq,
918	int head, unsigned out)
919	{
920	struct virtio_scsi_cmd_resp __user *resp;
921	struct virtio_scsi_cmd_resp rsp;
922	int ret;
923
924	memset(&rsp, 0, sizeof(rsp));
925	rsp.response = VIRTIO_SCSI_S_BAD_TARGET;
926	resp = vq->iov[out].iov_base;
927	ret = __copy_to_user(to: resp, from: &rsp, n: sizeof(rsp));
928	if (!ret)
929	vhost_add_used_and_signal(&vs->dev, vq, id: head, len: 0);
930	else
931	pr_err("Faulted on virtio_scsi_cmd_resp\n");
932	}
933
934	static int
935	vhost_scsi_get_desc(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
936	struct vhost_scsi_ctx *vc)
937	{
938	int ret = -ENXIO;
939
940	vc->head = vhost_get_vq_desc(vq, iov: vq->iov,
941	ARRAY_SIZE(vq->iov), out_num: &vc->out, in_num: &vc->in,
942	NULL, NULL);
943
944	pr_debug("vhost_get_vq_desc: head: %d, out: %u in: %u\n",
945	vc->head, vc->out, vc->in);
946
947	/* On error, stop handling until the next kick. */
948	if (unlikely(vc->head < 0))
949	goto done;
950
951	/* Nothing new? Wait for eventfd to tell us they refilled. */
952	if (vc->head == vq->num) {
953	if (unlikely(vhost_enable_notify(&vs->dev, vq))) {
954	vhost_disable_notify(&vs->dev, vq);
955	ret = -EAGAIN;
956	}
957	goto done;
958	}
959
960	/*
961	* Get the size of request and response buffers.
962	* FIXME: Not correct for BIDI operation
963	*/
964	vc->out_size = iov_length(iov: vq->iov, nr_segs: vc->out);
965	vc->in_size = iov_length(iov: &vq->iov[vc->out], nr_segs: vc->in);
966
967	/*
968	* Copy over the virtio-scsi request header, which for a
969	* ANY_LAYOUT enabled guest may span multiple iovecs, or a
970	* single iovec may contain both the header + outgoing
971	* WRITE payloads.
972	*
973	* copy_from_iter() will advance out_iter, so that it will
974	* point at the start of the outgoing WRITE payload, if
975	* DMA_TO_DEVICE is set.
976	*/
977	iov_iter_init(i: &vc->out_iter, ITER_SOURCE, iov: vq->iov, nr_segs: vc->out, count: vc->out_size);
978	ret = 0;
979
980	done:
981	return ret;
982	}
983
984	static int
985	vhost_scsi_chk_size(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc)
986	{
987	if (unlikely(vc->in_size < vc->rsp_size)) {
988	vq_err(vq,
989	"Response buf too small, need min %zu bytes got %zu",
990	vc->rsp_size, vc->in_size);
991	return -EINVAL;
992	} else if (unlikely(vc->out_size < vc->req_size)) {
993	vq_err(vq,
994	"Request buf too small, need min %zu bytes got %zu",
995	vc->req_size, vc->out_size);
996	return -EIO;
997	}
998
999	return 0;
1000	}
1001
1002	static int
1003	vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc,
1004	struct vhost_scsi_tpg **tpgp)
1005	{
1006	int ret = -EIO;
1007
1008	if (unlikely(!copy_from_iter_full(vc->req, vc->req_size,
1009	&vc->out_iter))) {
1010	vq_err(vq, "Faulted on copy_from_iter_full\n");
1011	} else if (unlikely(*vc->lunp != 1)) {
1012	/* virtio-scsi spec requires byte 0 of the lun to be 1 */
1013	vq_err(vq, "Illegal virtio-scsi lun: %u\n", *vc->lunp);
1014	} else {
1015	struct vhost_scsi_tpg **vs_tpg, *tpg;
1016
1017	vs_tpg = vhost_vq_get_backend(vq); /* validated at handler entry */
1018
1019	tpg = READ_ONCE(vs_tpg[*vc->target]);
1020	if (unlikely(!tpg)) {
1021	vq_err(vq, "Target 0x%x does not exist\n", *vc->target);
1022	} else {
1023	if (tpgp)
1024	*tpgp = tpg;
1025	ret = 0;
1026	}
1027	}
1028
1029	return ret;
1030	}
1031
1032	static u16 vhost_buf_to_lun(u8 *lun_buf)
1033	{
1034	return ((lun_buf[2] << 8) | lun_buf[3]) & 0x3FFF;
1035	}
1036
1037	static void
1038	vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
1039	{
1040	struct vhost_scsi_tpg **vs_tpg, *tpg;
1041	struct virtio_scsi_cmd_req v_req;
1042	struct virtio_scsi_cmd_req_pi v_req_pi;
1043	struct vhost_scsi_ctx vc;
1044	struct vhost_scsi_cmd *cmd;
1045	struct iov_iter in_iter, prot_iter, data_iter;
1046	u64 tag;
1047	u32 exp_data_len, data_direction;
1048	int ret, prot_bytes, i, c = 0;
1049	u16 lun;
1050	u8 task_attr;
1051	bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI);
1052	void *cdb;
1053
1054	mutex_lock(&vq->mutex);
1055	/*
1056	* We can handle the vq only after the endpoint is setup by calling the
1057	* VHOST_SCSI_SET_ENDPOINT ioctl.
1058	*/
1059	vs_tpg = vhost_vq_get_backend(vq);
1060	if (!vs_tpg)
1061	goto out;
1062
1063	memset(&vc, 0, sizeof(vc));
1064	vc.rsp_size = sizeof(struct virtio_scsi_cmd_resp);
1065
1066	vhost_disable_notify(&vs->dev, vq);
1067
1068	do {
1069	ret = vhost_scsi_get_desc(vs, vq, vc: &vc);
1070	if (ret)
1071	goto err;
1072
1073	/*
1074	* Setup pointers and values based upon different virtio-scsi
1075	* request header if T10_PI is enabled in KVM guest.
1076	*/
1077	if (t10_pi) {
1078	vc.req = &v_req_pi;
1079	vc.req_size = sizeof(v_req_pi);
1080	vc.lunp = &v_req_pi.lun[0];
1081	vc.target = &v_req_pi.lun[1];
1082	} else {
1083	vc.req = &v_req;
1084	vc.req_size = sizeof(v_req);
1085	vc.lunp = &v_req.lun[0];
1086	vc.target = &v_req.lun[1];
1087	}
1088
1089	/*
1090	* Validate the size of request and response buffers.
1091	* Check for a sane response buffer so we can report
1092	* early errors back to the guest.
1093	*/
1094	ret = vhost_scsi_chk_size(vq, vc: &vc);
1095	if (ret)
1096	goto err;
1097
1098	ret = vhost_scsi_get_req(vq, vc: &vc, tpgp: &tpg);
1099	if (ret)
1100	goto err;
1101
1102	ret = -EIO; /* bad target on any error from here on */
1103
1104	/*
1105	* Determine data_direction by calculating the total outgoing
1106	* iovec sizes + incoming iovec sizes vs. virtio-scsi request +
1107	* response headers respectively.
1108	*
1109	* For DMA_TO_DEVICE this is out_iter, which is already pointing
1110	* to the right place.
1111	*
1112	* For DMA_FROM_DEVICE, the iovec will be just past the end
1113	* of the virtio-scsi response header in either the same
1114	* or immediately following iovec.
1115	*
1116	* Any associated T10_PI bytes for the outgoing / incoming
1117	* payloads are included in calculation of exp_data_len here.
1118	*/
1119	prot_bytes = 0;
1120
1121	if (vc.out_size > vc.req_size) {
1122	data_direction = DMA_TO_DEVICE;
1123	exp_data_len = vc.out_size - vc.req_size;
1124	data_iter = vc.out_iter;
1125	} else if (vc.in_size > vc.rsp_size) {
1126	data_direction = DMA_FROM_DEVICE;
1127	exp_data_len = vc.in_size - vc.rsp_size;
1128
1129	iov_iter_init(i: &in_iter, ITER_DEST, iov: &vq->iov[vc.out], nr_segs: vc.in,
1130	count: vc.rsp_size + exp_data_len);
1131	iov_iter_advance(i: &in_iter, bytes: vc.rsp_size);
1132	data_iter = in_iter;
1133	} else {
1134	data_direction = DMA_NONE;
1135	exp_data_len = 0;
1136	}
1137	/*
1138	* If T10_PI header + payload is present, setup prot_iter values
1139	* and recalculate data_iter for vhost_scsi_mapal() mapping to
1140	* host scatterlists via get_user_pages_fast().
1141	*/
1142	if (t10_pi) {
1143	if (v_req_pi.pi_bytesout) {
1144	if (data_direction != DMA_TO_DEVICE) {
1145	vq_err(vq, "Received non zero pi_bytesout,"
1146	" but wrong data_direction\n");
1147	goto err;
1148	}
1149	prot_bytes = vhost32_to_cpu(vq, val: v_req_pi.pi_bytesout);
1150	} else if (v_req_pi.pi_bytesin) {
1151	if (data_direction != DMA_FROM_DEVICE) {
1152	vq_err(vq, "Received non zero pi_bytesin,"
1153	" but wrong data_direction\n");
1154	goto err;
1155	}
1156	prot_bytes = vhost32_to_cpu(vq, val: v_req_pi.pi_bytesin);
1157	}
1158	/*
1159	* Set prot_iter to data_iter and truncate it to
1160	* prot_bytes, and advance data_iter past any
1161	* preceding prot_bytes that may be present.
1162	*
1163	* Also fix up the exp_data_len to reflect only the
1164	* actual data payload length.
1165	*/
1166	if (prot_bytes) {
1167	exp_data_len -= prot_bytes;
1168	prot_iter = data_iter;
1169	iov_iter_truncate(i: &prot_iter, count: prot_bytes);
1170	iov_iter_advance(i: &data_iter, bytes: prot_bytes);
1171	}
1172	tag = vhost64_to_cpu(vq, val: v_req_pi.tag);
1173	task_attr = v_req_pi.task_attr;
1174	cdb = &v_req_pi.cdb[0];
1175	lun = vhost_buf_to_lun(lun_buf: v_req_pi.lun);
1176	} else {
1177	tag = vhost64_to_cpu(vq, val: v_req.tag);
1178	task_attr = v_req.task_attr;
1179	cdb = &v_req.cdb[0];
1180	lun = vhost_buf_to_lun(lun_buf: v_req.lun);
1181	}
1182	/*
1183	* Check that the received CDB size does not exceeded our
1184	* hardcoded max for vhost-scsi, then get a pre-allocated
1185	* cmd descriptor for the new virtio-scsi tag.
1186	*
1187	* TODO what if cdb was too small for varlen cdb header?
1188	*/
1189	if (unlikely(scsi_command_size(cdb) > VHOST_SCSI_MAX_CDB_SIZE)) {
1190	vq_err(vq, "Received SCSI CDB with command_size: %d that"
1191	" exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1192	scsi_command_size(cdb), VHOST_SCSI_MAX_CDB_SIZE);
1193	goto err;
1194	}
1195	cmd = vhost_scsi_get_cmd(vq, tpg, cdb, scsi_tag: tag, lun, task_attr,
1196	exp_data_len: exp_data_len + prot_bytes,
1197	data_direction);
1198	if (IS_ERR(ptr: cmd)) {
1199	vq_err(vq, "vhost_scsi_get_cmd failed %ld\n",
1200	PTR_ERR(cmd));
1201	goto err;
1202	}
1203	cmd->tvc_vhost = vs;
1204	cmd->tvc_vq = vq;
1205	for (i = 0; i < vc.in ; i++)
1206	cmd->tvc_resp_iov[i] = vq->iov[vc.out + i];
1207	cmd->tvc_in_iovs = vc.in;
1208
1209	pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n",
1210	cmd->tvc_cdb[0], cmd->tvc_lun);
1211	pr_debug("cmd: %p exp_data_len: %d, prot_bytes: %d data_direction:"
1212	" %d\n", cmd, exp_data_len, prot_bytes, data_direction);
1213
1214	if (data_direction != DMA_NONE) {
1215	if (unlikely(vhost_scsi_mapal(cmd, prot_bytes,
1216	&prot_iter, exp_data_len,
1217	&data_iter))) {
1218	vq_err(vq, "Failed to map iov to sgl\n");
1219	vhost_scsi_release_cmd_res(se_cmd: &cmd->tvc_se_cmd);
1220	goto err;
1221	}
1222	}
1223	/*
1224	* Save the descriptor from vhost_get_vq_desc() to be used to
1225	* complete the virtio-scsi request in TCM callback context via
1226	* vhost_scsi_queue_data_in() and vhost_scsi_queue_status()
1227	*/
1228	cmd->tvc_vq_desc = vc.head;
1229	vhost_scsi_target_queue_cmd(cmd);
1230	ret = 0;
1231	err:
1232	/*
1233	* ENXIO: No more requests, or read error, wait for next kick
1234	* EINVAL: Invalid response buffer, drop the request
1235	* EIO: Respond with bad target
1236	* EAGAIN: Pending request
1237	*/
1238	if (ret == -ENXIO)
1239	break;
1240	else if (ret == -EIO)
1241	vhost_scsi_send_bad_target(vs, vq, head: vc.head, out: vc.out);
1242	} while (likely(!vhost_exceeds_weight(vq, ++c, 0)));
1243	out:
1244	mutex_unlock(lock: &vq->mutex);
1245	}
1246
1247	static void
1248	vhost_scsi_send_tmf_resp(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
1249	int in_iovs, int vq_desc, struct iovec *resp_iov,
1250	int tmf_resp_code)
1251	{
1252	struct virtio_scsi_ctrl_tmf_resp rsp;
1253	struct iov_iter iov_iter;
1254	int ret;
1255
1256	pr_debug("%s\n", __func__);
1257	memset(&rsp, 0, sizeof(rsp));
1258	rsp.response = tmf_resp_code;
1259
1260	iov_iter_init(i: &iov_iter, ITER_DEST, iov: resp_iov, nr_segs: in_iovs, count: sizeof(rsp));
1261
1262	ret = copy_to_iter(addr: &rsp, bytes: sizeof(rsp), i: &iov_iter);
1263	if (likely(ret == sizeof(rsp)))
1264	vhost_add_used_and_signal(&vs->dev, vq, id: vq_desc, len: 0);
1265	else
1266	pr_err("Faulted on virtio_scsi_ctrl_tmf_resp\n");
1267	}
1268
1269	static void vhost_scsi_tmf_resp_work(struct vhost_work *work)
1270	{
1271	struct vhost_scsi_tmf *tmf = container_of(work, struct vhost_scsi_tmf,
1272	vwork);
1273	struct vhost_virtqueue *ctl_vq, *vq;
1274	int resp_code, i;
1275
1276	if (tmf->scsi_resp == TMR_FUNCTION_COMPLETE) {
1277	/*
1278	* Flush IO vqs that don't share a worker with the ctl to make
1279	* sure they have sent their responses before us.
1280	*/
1281	ctl_vq = &tmf->vhost->vqs[VHOST_SCSI_VQ_CTL].vq;
1282	for (i = VHOST_SCSI_VQ_IO; i < tmf->vhost->dev.nvqs; i++) {
1283	vq = &tmf->vhost->vqs[i].vq;
1284
1285	if (vhost_vq_is_setup(vq) &&
1286	vq->worker != ctl_vq->worker)
1287	vhost_vq_flush(vq);
1288	}
1289
1290	resp_code = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED;
1291	} else {
1292	resp_code = VIRTIO_SCSI_S_FUNCTION_REJECTED;
1293	}
1294
1295	vhost_scsi_send_tmf_resp(vs: tmf->vhost, vq: &tmf->svq->vq, in_iovs: tmf->in_iovs,
1296	vq_desc: tmf->vq_desc, resp_iov: &tmf->resp_iov, tmf_resp_code: resp_code);
1297	vhost_scsi_release_tmf_res(tmf);
1298	}
1299
1300	static void
1301	vhost_scsi_handle_tmf(struct vhost_scsi *vs, struct vhost_scsi_tpg *tpg,
1302	struct vhost_virtqueue *vq,
1303	struct virtio_scsi_ctrl_tmf_req *vtmf,
1304	struct vhost_scsi_ctx *vc)
1305	{
1306	struct vhost_scsi_virtqueue *svq = container_of(vq,
1307	struct vhost_scsi_virtqueue, vq);
1308	struct vhost_scsi_tmf *tmf;
1309
1310	if (vhost32_to_cpu(vq, val: vtmf->subtype) !=
1311	VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET)
1312	goto send_reject;
1313
1314	if (!tpg->tpg_nexus || !tpg->tpg_nexus->tvn_se_sess) {
1315	pr_err("Unable to locate active struct vhost_scsi_nexus for LUN RESET.\n");
1316	goto send_reject;
1317	}
1318
1319	tmf = kzalloc(size: sizeof(*tmf), GFP_KERNEL);
1320	if (!tmf)
1321	goto send_reject;
1322
1323	vhost_work_init(work: &tmf->vwork, fn: vhost_scsi_tmf_resp_work);
1324	tmf->vhost = vs;
1325	tmf->svq = svq;
1326	tmf->resp_iov = vq->iov[vc->out];
1327	tmf->vq_desc = vc->head;
1328	tmf->in_iovs = vc->in;
1329	tmf->inflight = vhost_scsi_get_inflight(vq);
1330
1331	if (target_submit_tmr(se_cmd: &tmf->se_cmd, se_sess: tpg->tpg_nexus->tvn_se_sess, NULL,
1332	unpacked_lun: vhost_buf_to_lun(lun_buf: vtmf->lun), NULL,
1333	tm_type: TMR_LUN_RESET, GFP_KERNEL, 0,
1334	TARGET_SCF_ACK_KREF) < 0) {
1335	vhost_scsi_release_tmf_res(tmf);
1336	goto send_reject;
1337	}
1338
1339	return;
1340
1341	send_reject:
1342	vhost_scsi_send_tmf_resp(vs, vq, in_iovs: vc->in, vq_desc: vc->head, resp_iov: &vq->iov[vc->out],
1343	VIRTIO_SCSI_S_FUNCTION_REJECTED);
1344	}
1345
1346	static void
1347	vhost_scsi_send_an_resp(struct vhost_scsi *vs,
1348	struct vhost_virtqueue *vq,
1349	struct vhost_scsi_ctx *vc)
1350	{
1351	struct virtio_scsi_ctrl_an_resp rsp;
1352	struct iov_iter iov_iter;
1353	int ret;
1354
1355	pr_debug("%s\n", __func__);
1356	memset(&rsp, 0, sizeof(rsp)); /* event_actual = 0 */
1357	rsp.response = VIRTIO_SCSI_S_OK;
1358
1359	iov_iter_init(i: &iov_iter, ITER_DEST, iov: &vq->iov[vc->out], nr_segs: vc->in, count: sizeof(rsp));
1360
1361	ret = copy_to_iter(addr: &rsp, bytes: sizeof(rsp), i: &iov_iter);
1362	if (likely(ret == sizeof(rsp)))
1363	vhost_add_used_and_signal(&vs->dev, vq, id: vc->head, len: 0);
1364	else
1365	pr_err("Faulted on virtio_scsi_ctrl_an_resp\n");
1366	}
1367
1368	static void
1369	vhost_scsi_ctl_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
1370	{
1371	struct vhost_scsi_tpg *tpg;
1372	union {
1373	__virtio32 type;
1374	struct virtio_scsi_ctrl_an_req an;
1375	struct virtio_scsi_ctrl_tmf_req tmf;
1376	} v_req;
1377	struct vhost_scsi_ctx vc;
1378	size_t typ_size;
1379	int ret, c = 0;
1380
1381	mutex_lock(&vq->mutex);
1382	/*
1383	* We can handle the vq only after the endpoint is setup by calling the
1384	* VHOST_SCSI_SET_ENDPOINT ioctl.
1385	*/
1386	if (!vhost_vq_get_backend(vq))
1387	goto out;
1388
1389	memset(&vc, 0, sizeof(vc));
1390
1391	vhost_disable_notify(&vs->dev, vq);
1392
1393	do {
1394	ret = vhost_scsi_get_desc(vs, vq, vc: &vc);
1395	if (ret)
1396	goto err;
1397
1398	/*
1399	* Get the request type first in order to setup
1400	* other parameters dependent on the type.
1401	*/
1402	vc.req = &v_req.type;
1403	typ_size = sizeof(v_req.type);
1404
1405	if (unlikely(!copy_from_iter_full(vc.req, typ_size,
1406	&vc.out_iter))) {
1407	vq_err(vq, "Faulted on copy_from_iter tmf type\n");
1408	/*
1409	* The size of the response buffer depends on the
1410	* request type and must be validated against it.
1411	* Since the request type is not known, don't send
1412	* a response.
1413	*/
1414	continue;
1415	}
1416
1417	switch (vhost32_to_cpu(vq, val: v_req.type)) {
1418	case VIRTIO_SCSI_T_TMF:
1419	vc.req = &v_req.tmf;
1420	vc.req_size = sizeof(struct virtio_scsi_ctrl_tmf_req);
1421	vc.rsp_size = sizeof(struct virtio_scsi_ctrl_tmf_resp);
1422	vc.lunp = &v_req.tmf.lun[0];
1423	vc.target = &v_req.tmf.lun[1];
1424	break;
1425	case VIRTIO_SCSI_T_AN_QUERY:
1426	case VIRTIO_SCSI_T_AN_SUBSCRIBE:
1427	vc.req = &v_req.an;
1428	vc.req_size = sizeof(struct virtio_scsi_ctrl_an_req);
1429	vc.rsp_size = sizeof(struct virtio_scsi_ctrl_an_resp);
1430	vc.lunp = &v_req.an.lun[0];
1431	vc.target = NULL;
1432	break;
1433	default:
1434	vq_err(vq, "Unknown control request %d", v_req.type);
1435	continue;
1436	}
1437
1438	/*
1439	* Validate the size of request and response buffers.
1440	* Check for a sane response buffer so we can report
1441	* early errors back to the guest.
1442	*/
1443	ret = vhost_scsi_chk_size(vq, vc: &vc);
1444	if (ret)
1445	goto err;
1446
1447	/*
1448	* Get the rest of the request now that its size is known.
1449	*/
1450	vc.req += typ_size;
1451	vc.req_size -= typ_size;
1452
1453	ret = vhost_scsi_get_req(vq, vc: &vc, tpgp: &tpg);
1454	if (ret)
1455	goto err;
1456
1457	if (v_req.type == VIRTIO_SCSI_T_TMF)
1458	vhost_scsi_handle_tmf(vs, tpg, vq, vtmf: &v_req.tmf, vc: &vc);
1459	else
1460	vhost_scsi_send_an_resp(vs, vq, vc: &vc);
1461	err:
1462	/*
1463	* ENXIO: No more requests, or read error, wait for next kick
1464	* EINVAL: Invalid response buffer, drop the request
1465	* EIO: Respond with bad target
1466	* EAGAIN: Pending request
1467	*/
1468	if (ret == -ENXIO)
1469	break;
1470	else if (ret == -EIO)
1471	vhost_scsi_send_bad_target(vs, vq, head: vc.head, out: vc.out);
1472	} while (likely(!vhost_exceeds_weight(vq, ++c, 0)));
1473	out:
1474	mutex_unlock(lock: &vq->mutex);
1475	}
1476
1477	static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)
1478	{
1479	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1480	poll.work);
1481	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1482
1483	pr_debug("%s: The handling func for control queue.\n", __func__);
1484	vhost_scsi_ctl_handle_vq(vs, vq);
1485	}
1486
1487	static void
1488	vhost_scsi_send_evt(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
1489	struct vhost_scsi_tpg *tpg, struct se_lun *lun,
1490	u32 event, u32 reason)
1491	{
1492	struct vhost_scsi_evt *evt;
1493
1494	evt = vhost_scsi_allocate_evt(vs, event, reason);
1495	if (!evt)
1496	return;
1497
1498	if (tpg && lun) {
1499	/* TODO: share lun setup code with virtio-scsi.ko */
1500	/*
1501	* Note: evt->event is zeroed when we allocate it and
1502	* lun[4-7] need to be zero according to virtio-scsi spec.
1503	*/
1504	evt->event.lun[0] = 0x01;
1505	evt->event.lun[1] = tpg->tport_tpgt;
1506	if (lun->unpacked_lun >= 256)
1507	evt->event.lun[2] = lun->unpacked_lun >> 8 | 0x40 ;
1508	evt->event.lun[3] = lun->unpacked_lun & 0xFF;
1509	}
1510
1511	llist_add(new: &evt->list, head: &vs->vs_event_list);
1512	vhost_vq_work_queue(vq, work: &vs->vs_event_work);
1513	}
1514
1515	static void vhost_scsi_evt_handle_kick(struct vhost_work *work)
1516	{
1517	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1518	poll.work);
1519	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1520
1521	mutex_lock(&vq->mutex);
1522	if (!vhost_vq_get_backend(vq))
1523	goto out;
1524
1525	if (vs->vs_events_missed)
1526	vhost_scsi_send_evt(vs, vq, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT,
1527	reason: 0);
1528	out:
1529	mutex_unlock(lock: &vq->mutex);
1530	}
1531
1532	static void vhost_scsi_handle_kick(struct vhost_work *work)
1533	{
1534	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1535	poll.work);
1536	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1537
1538	vhost_scsi_handle_vq(vs, vq);
1539	}
1540
1541	/* Callers must hold dev mutex */
1542	static void vhost_scsi_flush(struct vhost_scsi *vs)
1543	{
1544	int i;
1545
1546	/* Init new inflight and remember the old inflight */
1547	vhost_scsi_init_inflight(vs, old_inflight: vs->old_inflight);
1548
1549	/*
1550	* The inflight->kref was initialized to 1. We decrement it here to
1551	* indicate the start of the flush operation so that it will reach 0
1552	* when all the reqs are finished.
1553	*/
1554	for (i = 0; i < vs->dev.nvqs; i++)
1555	kref_put(kref: &vs->old_inflight[i]->kref, release: vhost_scsi_done_inflight);
1556
1557	/* Flush both the vhost poll and vhost work */
1558	vhost_dev_flush(dev: &vs->dev);
1559
1560	/* Wait for all reqs issued before the flush to be finished */
1561	for (i = 0; i < vs->dev.nvqs; i++)
1562	wait_for_completion(&vs->old_inflight[i]->comp);
1563	}
1564
1565	static void vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue *vq)
1566	{
1567	struct vhost_scsi_virtqueue *svq = container_of(vq,
1568	struct vhost_scsi_virtqueue, vq);
1569	struct vhost_scsi_cmd *tv_cmd;
1570	unsigned int i;
1571
1572	if (!svq->scsi_cmds)
1573	return;
1574
1575	for (i = 0; i < svq->max_cmds; i++) {
1576	tv_cmd = &svq->scsi_cmds[i];
1577
1578	kfree(objp: tv_cmd->tvc_sgl);
1579	kfree(objp: tv_cmd->tvc_prot_sgl);
1580	kfree(objp: tv_cmd->tvc_upages);
1581	kfree(objp: tv_cmd->tvc_resp_iov);
1582	}
1583
1584	sbitmap_free(sb: &svq->scsi_tags);
1585	kfree(objp: svq->scsi_cmds);
1586	svq->scsi_cmds = NULL;
1587	}
1588
1589	static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds)
1590	{
1591	struct vhost_scsi_virtqueue *svq = container_of(vq,
1592	struct vhost_scsi_virtqueue, vq);
1593	struct vhost_scsi_cmd *tv_cmd;
1594	unsigned int i;
1595
1596	if (svq->scsi_cmds)
1597	return 0;
1598
1599	if (sbitmap_init_node(sb: &svq->scsi_tags, depth: max_cmds, shift: -1, GFP_KERNEL,
1600	NUMA_NO_NODE, round_robin: false, alloc_hint: true))
1601	return -ENOMEM;
1602	svq->max_cmds = max_cmds;
1603
1604	svq->scsi_cmds = kcalloc(n: max_cmds, size: sizeof(*tv_cmd), GFP_KERNEL);
1605	if (!svq->scsi_cmds) {
1606	sbitmap_free(sb: &svq->scsi_tags);
1607	return -ENOMEM;
1608	}
1609
1610	for (i = 0; i < max_cmds; i++) {
1611	tv_cmd = &svq->scsi_cmds[i];
1612
1613	tv_cmd->tvc_sgl = kcalloc(VHOST_SCSI_PREALLOC_SGLS,
1614	size: sizeof(struct scatterlist),
1615	GFP_KERNEL);
1616	if (!tv_cmd->tvc_sgl) {
1617	pr_err("Unable to allocate tv_cmd->tvc_sgl\n");
1618	goto out;
1619	}
1620
1621	tv_cmd->tvc_upages = kcalloc(VHOST_SCSI_PREALLOC_UPAGES,
1622	size: sizeof(struct page *),
1623	GFP_KERNEL);
1624	if (!tv_cmd->tvc_upages) {
1625	pr_err("Unable to allocate tv_cmd->tvc_upages\n");
1626	goto out;
1627	}
1628
1629	tv_cmd->tvc_resp_iov = kcalloc(UIO_MAXIOV,
1630	size: sizeof(struct iovec),
1631	GFP_KERNEL);
1632	if (!tv_cmd->tvc_resp_iov) {
1633	pr_err("Unable to allocate tv_cmd->tvc_resp_iov\n");
1634	goto out;
1635	}
1636
1637	tv_cmd->tvc_prot_sgl = kcalloc(VHOST_SCSI_PREALLOC_PROT_SGLS,
1638	size: sizeof(struct scatterlist),
1639	GFP_KERNEL);
1640	if (!tv_cmd->tvc_prot_sgl) {
1641	pr_err("Unable to allocate tv_cmd->tvc_prot_sgl\n");
1642	goto out;
1643	}
1644	}
1645	return 0;
1646	out:
1647	vhost_scsi_destroy_vq_cmds(vq);
1648	return -ENOMEM;
1649	}
1650
1651	/*
1652	* Called from vhost_scsi_ioctl() context to walk the list of available
1653	* vhost_scsi_tpg with an active struct vhost_scsi_nexus
1654	*
1655	* The lock nesting rule is:
1656	* vs->dev.mutex -> vhost_scsi_mutex -> tpg->tv_tpg_mutex -> vq->mutex
1657	*/
1658	static int
1659	vhost_scsi_set_endpoint(struct vhost_scsi *vs,
1660	struct vhost_scsi_target *t)
1661	{
1662	struct se_portal_group *se_tpg;
1663	struct vhost_scsi_tport *tv_tport;
1664	struct vhost_scsi_tpg *tpg;
1665	struct vhost_scsi_tpg **vs_tpg;
1666	struct vhost_virtqueue *vq;
1667	int index, ret, i, len;
1668	bool match = false;
1669
1670	mutex_lock(&vs->dev.mutex);
1671
1672	/* Verify that ring has been setup correctly. */
1673	for (index = 0; index < vs->dev.nvqs; ++index) {
1674	/* Verify that ring has been setup correctly. */
1675	if (!vhost_vq_access_ok(vq: &vs->vqs[index].vq)) {
1676	ret = -EFAULT;
1677	goto out;
1678	}
1679	}
1680
1681	len = sizeof(vs_tpg[0]) * VHOST_SCSI_MAX_TARGET;
1682	vs_tpg = kzalloc(size: len, GFP_KERNEL);
1683	if (!vs_tpg) {
1684	ret = -ENOMEM;
1685	goto out;
1686	}
1687	if (vs->vs_tpg)
1688	memcpy(vs_tpg, vs->vs_tpg, len);
1689
1690	mutex_lock(&vhost_scsi_mutex);
1691	list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) {
1692	mutex_lock(&tpg->tv_tpg_mutex);
1693	if (!tpg->tpg_nexus) {
1694	mutex_unlock(lock: &tpg->tv_tpg_mutex);
1695	continue;
1696	}
1697	if (tpg->tv_tpg_vhost_count != 0) {
1698	mutex_unlock(lock: &tpg->tv_tpg_mutex);
1699	continue;
1700	}
1701	tv_tport = tpg->tport;
1702
1703	if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1704	if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) {
1705	mutex_unlock(lock: &tpg->tv_tpg_mutex);
1706	mutex_unlock(lock: &vhost_scsi_mutex);
1707	ret = -EEXIST;
1708	goto undepend;
1709	}
1710	/*
1711	* In order to ensure individual vhost-scsi configfs
1712	* groups cannot be removed while in use by vhost ioctl,
1713	* go ahead and take an explicit se_tpg->tpg_group.cg_item
1714	* dependency now.
1715	*/
1716	se_tpg = &tpg->se_tpg;
1717	ret = target_depend_item(item: &se_tpg->tpg_group.cg_item);
1718	if (ret) {
1719	pr_warn("target_depend_item() failed: %d\n", ret);
1720	mutex_unlock(lock: &tpg->tv_tpg_mutex);
1721	mutex_unlock(lock: &vhost_scsi_mutex);
1722	goto undepend;
1723	}
1724	tpg->tv_tpg_vhost_count++;
1725	tpg->vhost_scsi = vs;
1726	vs_tpg[tpg->tport_tpgt] = tpg;
1727	match = true;
1728	}
1729	mutex_unlock(lock: &tpg->tv_tpg_mutex);
1730	}
1731	mutex_unlock(lock: &vhost_scsi_mutex);
1732
1733	if (match) {
1734	memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
1735	sizeof(vs->vs_vhost_wwpn));
1736
1737	for (i = VHOST_SCSI_VQ_IO; i < vs->dev.nvqs; i++) {
1738	vq = &vs->vqs[i].vq;
1739	if (!vhost_vq_is_setup(vq))
1740	continue;
1741
1742	ret = vhost_scsi_setup_vq_cmds(vq, max_cmds: vq->num);
1743	if (ret)
1744	goto destroy_vq_cmds;
1745	}
1746
1747	for (i = 0; i < vs->dev.nvqs; i++) {
1748	vq = &vs->vqs[i].vq;
1749	mutex_lock(&vq->mutex);
1750	vhost_vq_set_backend(vq, private_data: vs_tpg);
1751	vhost_vq_init_access(vq);
1752	mutex_unlock(lock: &vq->mutex);
1753	}
1754	ret = 0;
1755	} else {
1756	ret = -EEXIST;
1757	}
1758
1759	/*
1760	* Act as synchronize_rcu to make sure access to
1761	* old vs->vs_tpg is finished.
1762	*/
1763	vhost_scsi_flush(vs);
1764	kfree(objp: vs->vs_tpg);
1765	vs->vs_tpg = vs_tpg;
1766	goto out;
1767
1768	destroy_vq_cmds:
1769	for (i--; i >= VHOST_SCSI_VQ_IO; i--) {
1770	if (!vhost_vq_get_backend(vq: &vs->vqs[i].vq))
1771	vhost_scsi_destroy_vq_cmds(vq: &vs->vqs[i].vq);
1772	}
1773	undepend:
1774	for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1775	tpg = vs_tpg[i];
1776	if (tpg) {
1777	mutex_lock(&tpg->tv_tpg_mutex);
1778	tpg->vhost_scsi = NULL;
1779	tpg->tv_tpg_vhost_count--;
1780	mutex_unlock(lock: &tpg->tv_tpg_mutex);
1781	target_undepend_item(item: &tpg->se_tpg.tpg_group.cg_item);
1782	}
1783	}
1784	kfree(objp: vs_tpg);
1785	out:
1786	mutex_unlock(lock: &vs->dev.mutex);
1787	return ret;
1788	}
1789
1790	static int
1791	vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
1792	struct vhost_scsi_target *t)
1793	{
1794	struct se_portal_group *se_tpg;
1795	struct vhost_scsi_tport *tv_tport;
1796	struct vhost_scsi_tpg *tpg;
1797	struct vhost_virtqueue *vq;
1798	bool match = false;
1799	int index, ret, i;
1800	u8 target;
1801
1802	mutex_lock(&vs->dev.mutex);
1803	/* Verify that ring has been setup correctly. */
1804	for (index = 0; index < vs->dev.nvqs; ++index) {
1805	if (!vhost_vq_access_ok(vq: &vs->vqs[index].vq)) {
1806	ret = -EFAULT;
1807	goto err_dev;
1808	}
1809	}
1810
1811	if (!vs->vs_tpg) {
1812	ret = 0;
1813	goto err_dev;
1814	}
1815
1816	for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1817	target = i;
1818	tpg = vs->vs_tpg[target];
1819	if (!tpg)
1820	continue;
1821
1822	tv_tport = tpg->tport;
1823	if (!tv_tport) {
1824	ret = -ENODEV;
1825	goto err_dev;
1826	}
1827
1828	if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1829	pr_warn("tv_tport->tport_name: %s, tpg->tport_tpgt: %hu"
1830	" does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n",
1831	tv_tport->tport_name, tpg->tport_tpgt,
1832	t->vhost_wwpn, t->vhost_tpgt);
1833	ret = -EINVAL;
1834	goto err_dev;
1835	}
1836	match = true;
1837	}
1838	if (!match)
1839	goto free_vs_tpg;
1840
1841	/* Prevent new cmds from starting and accessing the tpgs/sessions */
1842	for (i = 0; i < vs->dev.nvqs; i++) {
1843	vq = &vs->vqs[i].vq;
1844	mutex_lock(&vq->mutex);
1845	vhost_vq_set_backend(vq, NULL);
1846	mutex_unlock(lock: &vq->mutex);
1847	}
1848	/* Make sure cmds are not running before tearing them down. */
1849	vhost_scsi_flush(vs);
1850
1851	for (i = 0; i < vs->dev.nvqs; i++) {
1852	vq = &vs->vqs[i].vq;
1853	vhost_scsi_destroy_vq_cmds(vq);
1854	}
1855
1856	/*
1857	* We can now release our hold on the tpg and sessions and userspace
1858	* can free them after this point.
1859	*/
1860	for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1861	target = i;
1862	tpg = vs->vs_tpg[target];
1863	if (!tpg)
1864	continue;
1865
1866	mutex_lock(&tpg->tv_tpg_mutex);
1867
1868	tpg->tv_tpg_vhost_count--;
1869	tpg->vhost_scsi = NULL;
1870	vs->vs_tpg[target] = NULL;
1871
1872	mutex_unlock(lock: &tpg->tv_tpg_mutex);
1873
1874	se_tpg = &tpg->se_tpg;
1875	target_undepend_item(item: &se_tpg->tpg_group.cg_item);
1876	}
1877
1878	free_vs_tpg:
1879	/*
1880	* Act as synchronize_rcu to make sure access to
1881	* old vs->vs_tpg is finished.
1882	*/
1883	vhost_scsi_flush(vs);
1884	kfree(objp: vs->vs_tpg);
1885	vs->vs_tpg = NULL;
1886	WARN_ON(vs->vs_events_nr);
1887	mutex_unlock(lock: &vs->dev.mutex);
1888	return 0;
1889
1890	err_dev:
1891	mutex_unlock(lock: &vs->dev.mutex);
1892	return ret;
1893	}
1894
1895	static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features)
1896	{
1897	struct vhost_virtqueue *vq;
1898	int i;
1899
1900	if (features & ~VHOST_SCSI_FEATURES)
1901	return -EOPNOTSUPP;
1902
1903	mutex_lock(&vs->dev.mutex);
1904	if ((features & (1 << VHOST_F_LOG_ALL)) &&
1905	!vhost_log_access_ok(&vs->dev)) {
1906	mutex_unlock(lock: &vs->dev.mutex);
1907	return -EFAULT;
1908	}
1909
1910	for (i = 0; i < vs->dev.nvqs; i++) {
1911	vq = &vs->vqs[i].vq;
1912	mutex_lock(&vq->mutex);
1913	vq->acked_features = features;
1914	mutex_unlock(lock: &vq->mutex);
1915	}
1916	mutex_unlock(lock: &vs->dev.mutex);
1917	return 0;
1918	}
1919
1920	static int vhost_scsi_open(struct inode *inode, struct file *f)
1921	{
1922	struct vhost_scsi_virtqueue *svq;
1923	struct vhost_scsi *vs;
1924	struct vhost_virtqueue **vqs;
1925	int r = -ENOMEM, i, nvqs = vhost_scsi_max_io_vqs;
1926
1927	vs = kvzalloc(size: sizeof(*vs), GFP_KERNEL);
1928	if (!vs)
1929	goto err_vs;
1930
1931	if (nvqs > VHOST_SCSI_MAX_IO_VQ) {
1932	pr_err("Invalid max_io_vqs of %d. Using %d.\n", nvqs,
1933	VHOST_SCSI_MAX_IO_VQ);
1934	nvqs = VHOST_SCSI_MAX_IO_VQ;
1935	} else if (nvqs == 0) {
1936	pr_err("Invalid max_io_vqs of %d. Using 1.\n", nvqs);
1937	nvqs = 1;
1938	}
1939	nvqs += VHOST_SCSI_VQ_IO;
1940
1941	vs->old_inflight = kmalloc_array(n: nvqs, size: sizeof(*vs->old_inflight),
1942	GFP_KERNEL | __GFP_ZERO);
1943	if (!vs->old_inflight)
1944	goto err_inflight;
1945
1946	vs->vqs = kmalloc_array(n: nvqs, size: sizeof(*vs->vqs),
1947	GFP_KERNEL | __GFP_ZERO);
1948	if (!vs->vqs)
1949	goto err_vqs;
1950
1951	vqs = kmalloc_array(n: nvqs, size: sizeof(*vqs), GFP_KERNEL);
1952	if (!vqs)
1953	goto err_local_vqs;
1954
1955	vhost_work_init(work: &vs->vs_event_work, fn: vhost_scsi_evt_work);
1956
1957	vs->vs_events_nr = 0;
1958	vs->vs_events_missed = false;
1959
1960	vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;
1961	vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1962	vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;
1963	vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;
1964	for (i = VHOST_SCSI_VQ_IO; i < nvqs; i++) {
1965	svq = &vs->vqs[i];
1966
1967	vqs[i] = &svq->vq;
1968	svq->vs = vs;
1969	init_llist_head(list: &svq->completion_list);
1970	vhost_work_init(work: &svq->completion_work,
1971	fn: vhost_scsi_complete_cmd_work);
1972	svq->vq.handle_kick = vhost_scsi_handle_kick;
1973	}
1974	vhost_dev_init(&vs->dev, vqs, nvqs, UIO_MAXIOV,
1975	VHOST_SCSI_WEIGHT, byte_weight: 0, use_worker: true, NULL);
1976
1977	vhost_scsi_init_inflight(vs, NULL);
1978
1979	f->private_data = vs;
1980	return 0;
1981
1982	err_local_vqs:
1983	kfree(objp: vs->vqs);
1984	err_vqs:
1985	kfree(objp: vs->old_inflight);
1986	err_inflight:
1987	kvfree(addr: vs);
1988	err_vs:
1989	return r;
1990	}
1991
1992	static int vhost_scsi_release(struct inode *inode, struct file *f)
1993	{
1994	struct vhost_scsi *vs = f->private_data;
1995	struct vhost_scsi_target t;
1996
1997	mutex_lock(&vs->dev.mutex);
1998	memcpy(t.vhost_wwpn, vs->vs_vhost_wwpn, sizeof(t.vhost_wwpn));
1999	mutex_unlock(lock: &vs->dev.mutex);
2000	vhost_scsi_clear_endpoint(vs, t: &t);
2001	vhost_dev_stop(&vs->dev);
2002	vhost_dev_cleanup(&vs->dev);
2003	kfree(objp: vs->dev.vqs);
2004	kfree(objp: vs->vqs);
2005	kfree(objp: vs->old_inflight);
2006	kvfree(addr: vs);
2007	return 0;
2008	}
2009
2010	static long
2011	vhost_scsi_ioctl(struct file *f,
2012	unsigned int ioctl,
2013	unsigned long arg)
2014	{
2015	struct vhost_scsi *vs = f->private_data;
2016	struct vhost_scsi_target backend;
2017	void __user *argp = (void __user *)arg;
2018	u64 __user *featurep = argp;
2019	u32 __user *eventsp = argp;
2020	u32 events_missed;
2021	u64 features;
2022	int r, abi_version = VHOST_SCSI_ABI_VERSION;
2023	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
2024
2025	switch (ioctl) {
2026	case VHOST_SCSI_SET_ENDPOINT:
2027	if (copy_from_user(to: &backend, from: argp, n: sizeof backend))
2028	return -EFAULT;
2029	if (backend.reserved != 0)
2030	return -EOPNOTSUPP;
2031
2032	return vhost_scsi_set_endpoint(vs, t: &backend);
2033	case VHOST_SCSI_CLEAR_ENDPOINT:
2034	if (copy_from_user(to: &backend, from: argp, n: sizeof backend))
2035	return -EFAULT;
2036	if (backend.reserved != 0)
2037	return -EOPNOTSUPP;
2038
2039	return vhost_scsi_clear_endpoint(vs, t: &backend);
2040	case VHOST_SCSI_GET_ABI_VERSION:
2041	if (copy_to_user(to: argp, from: &abi_version, n: sizeof abi_version))
2042	return -EFAULT;
2043	return 0;
2044	case VHOST_SCSI_SET_EVENTS_MISSED:
2045	if (get_user(events_missed, eventsp))
2046	return -EFAULT;
2047	mutex_lock(&vq->mutex);
2048	vs->vs_events_missed = events_missed;
2049	mutex_unlock(lock: &vq->mutex);
2050	return 0;
2051	case VHOST_SCSI_GET_EVENTS_MISSED:
2052	mutex_lock(&vq->mutex);
2053	events_missed = vs->vs_events_missed;
2054	mutex_unlock(lock: &vq->mutex);
2055	if (put_user(events_missed, eventsp))
2056	return -EFAULT;
2057	return 0;
2058	case VHOST_GET_FEATURES:
2059	features = VHOST_SCSI_FEATURES;
2060	if (copy_to_user(to: featurep, from: &features, n: sizeof features))
2061	return -EFAULT;
2062	return 0;
2063	case VHOST_SET_FEATURES:
2064	if (copy_from_user(to: &features, from: featurep, n: sizeof features))
2065	return -EFAULT;
2066	return vhost_scsi_set_features(vs, features);
2067	case VHOST_NEW_WORKER:
2068	case VHOST_FREE_WORKER:
2069	case VHOST_ATTACH_VRING_WORKER:
2070	case VHOST_GET_VRING_WORKER:
2071	mutex_lock(&vs->dev.mutex);
2072	r = vhost_worker_ioctl(dev: &vs->dev, ioctl, argp);
2073	mutex_unlock(lock: &vs->dev.mutex);
2074	return r;
2075	default:
2076	mutex_lock(&vs->dev.mutex);
2077	r = vhost_dev_ioctl(&vs->dev, ioctl, argp);
2078	/* TODO: flush backend after dev ioctl. */
2079	if (r == -ENOIOCTLCMD)
2080	r = vhost_vring_ioctl(d: &vs->dev, ioctl, argp);
2081	mutex_unlock(lock: &vs->dev.mutex);
2082	return r;
2083	}
2084	}
2085
2086	static const struct file_operations vhost_scsi_fops = {
2087	.owner = THIS_MODULE,
2088	.release = vhost_scsi_release,
2089	.unlocked_ioctl = vhost_scsi_ioctl,
2090	.compat_ioctl = compat_ptr_ioctl,
2091	.open = vhost_scsi_open,
2092	.llseek = noop_llseek,
2093	};
2094
2095	static struct miscdevice vhost_scsi_misc = {
2096	MISC_DYNAMIC_MINOR,
2097	"vhost-scsi",
2098	&vhost_scsi_fops,
2099	};
2100
2101	static int __init vhost_scsi_register(void)
2102	{
2103	return misc_register(misc: &vhost_scsi_misc);
2104	}
2105
2106	static void vhost_scsi_deregister(void)
2107	{
2108	misc_deregister(misc: &vhost_scsi_misc);
2109	}
2110
2111	static char *vhost_scsi_dump_proto_id(struct vhost_scsi_tport *tport)
2112	{
2113	switch (tport->tport_proto_id) {
2114	case SCSI_PROTOCOL_SAS:
2115	return "SAS";
2116	case SCSI_PROTOCOL_FCP:
2117	return "FCP";
2118	case SCSI_PROTOCOL_ISCSI:
2119	return "iSCSI";
2120	default:
2121	break;
2122	}
2123
2124	return "Unknown";
2125	}
2126
2127	static void
2128	vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
2129	struct se_lun *lun, bool plug)
2130	{
2131
2132	struct vhost_scsi *vs = tpg->vhost_scsi;
2133	struct vhost_virtqueue *vq;
2134	u32 reason;
2135
2136	if (!vs)
2137	return;
2138
2139	if (plug)
2140	reason = VIRTIO_SCSI_EVT_RESET_RESCAN;
2141	else
2142	reason = VIRTIO_SCSI_EVT_RESET_REMOVED;
2143
2144	vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
2145	mutex_lock(&vq->mutex);
2146	/*
2147	* We can't queue events if the backend has been cleared, because
2148	* we could end up queueing an event after the flush.
2149	*/
2150	if (!vhost_vq_get_backend(vq))
2151	goto unlock;
2152
2153	if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))
2154	vhost_scsi_send_evt(vs, vq, tpg, lun,
2155	VIRTIO_SCSI_T_TRANSPORT_RESET, reason);
2156	unlock:
2157	mutex_unlock(lock: &vq->mutex);
2158	}
2159
2160	static void vhost_scsi_hotplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
2161	{
2162	vhost_scsi_do_plug(tpg, lun, plug: true);
2163	}
2164
2165	static void vhost_scsi_hotunplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
2166	{
2167	vhost_scsi_do_plug(tpg, lun, plug: false);
2168	}
2169
2170	static int vhost_scsi_port_link(struct se_portal_group *se_tpg,
2171	struct se_lun *lun)
2172	{
2173	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2174	struct vhost_scsi_tpg, se_tpg);
2175
2176	mutex_lock(&tpg->tv_tpg_mutex);
2177	tpg->tv_tpg_port_count++;
2178	vhost_scsi_hotplug(tpg, lun);
2179	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2180
2181	return 0;
2182	}
2183
2184	static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg,
2185	struct se_lun *lun)
2186	{
2187	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2188	struct vhost_scsi_tpg, se_tpg);
2189
2190	mutex_lock(&tpg->tv_tpg_mutex);
2191	tpg->tv_tpg_port_count--;
2192	vhost_scsi_hotunplug(tpg, lun);
2193	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2194	}
2195
2196	static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store(
2197	struct config_item *item, const char *page, size_t count)
2198	{
2199	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2200	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2201	struct vhost_scsi_tpg, se_tpg);
2202	unsigned long val;
2203	int ret = kstrtoul(s: page, base: 0, res: &val);
2204
2205	if (ret) {
2206	pr_err("kstrtoul() returned %d for fabric_prot_type\n", ret);
2207	return ret;
2208	}
2209	if (val != 0 && val != 1 && val != 3) {
2210	pr_err("Invalid vhost_scsi fabric_prot_type: %lu\n", val);
2211	return -EINVAL;
2212	}
2213	tpg->tv_fabric_prot_type = val;
2214
2215	return count;
2216	}
2217
2218	static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_show(
2219	struct config_item *item, char *page)
2220	{
2221	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2222	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2223	struct vhost_scsi_tpg, se_tpg);
2224
2225	return sysfs_emit(buf: page, fmt: "%d\n", tpg->tv_fabric_prot_type);
2226	}
2227
2228	CONFIGFS_ATTR(vhost_scsi_tpg_attrib_, fabric_prot_type);
2229
2230	static struct configfs_attribute *vhost_scsi_tpg_attrib_attrs[] = {
2231	&vhost_scsi_tpg_attrib_attr_fabric_prot_type,
2232	NULL,
2233	};
2234
2235	static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg,
2236	const char *name)
2237	{
2238	struct vhost_scsi_nexus *tv_nexus;
2239
2240	mutex_lock(&tpg->tv_tpg_mutex);
2241	if (tpg->tpg_nexus) {
2242	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2243	pr_debug("tpg->tpg_nexus already exists\n");
2244	return -EEXIST;
2245	}
2246
2247	tv_nexus = kzalloc(size: sizeof(*tv_nexus), GFP_KERNEL);
2248	if (!tv_nexus) {
2249	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2250	pr_err("Unable to allocate struct vhost_scsi_nexus\n");
2251	return -ENOMEM;
2252	}
2253	/*
2254	* Since we are running in 'demo mode' this call with generate a
2255	* struct se_node_acl for the vhost_scsi struct se_portal_group with
2256	* the SCSI Initiator port name of the passed configfs group 'name'.
2257	*/
2258	tv_nexus->tvn_se_sess = target_setup_session(&tpg->se_tpg, 0, 0,
2259	prot_op: TARGET_PROT_DIN_PASS | TARGET_PROT_DOUT_PASS,
2260	(unsigned char *)name, tv_nexus, NULL);
2261	if (IS_ERR(ptr: tv_nexus->tvn_se_sess)) {
2262	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2263	kfree(objp: tv_nexus);
2264	return -ENOMEM;
2265	}
2266	tpg->tpg_nexus = tv_nexus;
2267
2268	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2269	return 0;
2270	}
2271
2272	static int vhost_scsi_drop_nexus(struct vhost_scsi_tpg *tpg)
2273	{
2274	struct se_session *se_sess;
2275	struct vhost_scsi_nexus *tv_nexus;
2276
2277	mutex_lock(&tpg->tv_tpg_mutex);
2278	tv_nexus = tpg->tpg_nexus;
2279	if (!tv_nexus) {
2280	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2281	return -ENODEV;
2282	}
2283
2284	se_sess = tv_nexus->tvn_se_sess;
2285	if (!se_sess) {
2286	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2287	return -ENODEV;
2288	}
2289
2290	if (tpg->tv_tpg_port_count != 0) {
2291	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2292	pr_err("Unable to remove TCM_vhost I_T Nexus with"
2293	" active TPG port count: %d\n",
2294	tpg->tv_tpg_port_count);
2295	return -EBUSY;
2296	}
2297
2298	if (tpg->tv_tpg_vhost_count != 0) {
2299	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2300	pr_err("Unable to remove TCM_vhost I_T Nexus with"
2301	" active TPG vhost count: %d\n",
2302	tpg->tv_tpg_vhost_count);
2303	return -EBUSY;
2304	}
2305
2306	pr_debug("TCM_vhost_ConfigFS: Removing I_T Nexus to emulated"
2307	" %s Initiator Port: %s\n", vhost_scsi_dump_proto_id(tpg->tport),
2308	tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2309
2310	/*
2311	* Release the SCSI I_T Nexus to the emulated vhost Target Port
2312	*/
2313	target_remove_session(se_sess);
2314	tpg->tpg_nexus = NULL;
2315	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2316
2317	kfree(objp: tv_nexus);
2318	return 0;
2319	}
2320
2321	static ssize_t vhost_scsi_tpg_nexus_show(struct config_item *item, char *page)
2322	{
2323	struct se_portal_group *se_tpg = to_tpg(item);
2324	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2325	struct vhost_scsi_tpg, se_tpg);
2326	struct vhost_scsi_nexus *tv_nexus;
2327	ssize_t ret;
2328
2329	mutex_lock(&tpg->tv_tpg_mutex);
2330	tv_nexus = tpg->tpg_nexus;
2331	if (!tv_nexus) {
2332	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2333	return -ENODEV;
2334	}
2335	ret = sysfs_emit(buf: page, fmt: "%s\n",
2336	tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2337	mutex_unlock(lock: &tpg->tv_tpg_mutex);
2338
2339	return ret;
2340	}
2341
2342	static ssize_t vhost_scsi_tpg_nexus_store(struct config_item *item,
2343	const char *page, size_t count)
2344	{
2345	struct se_portal_group *se_tpg = to_tpg(item);
2346	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2347	struct vhost_scsi_tpg, se_tpg);
2348	struct vhost_scsi_tport *tport_wwn = tpg->tport;
2349	unsigned char i_port[VHOST_SCSI_NAMELEN], *ptr, *port_ptr;
2350	int ret;
2351	/*
2352	* Shutdown the active I_T nexus if 'NULL' is passed..
2353	*/
2354	if (!strncmp(page, "NULL", 4)) {
2355	ret = vhost_scsi_drop_nexus(tpg);
2356	return (!ret) ? count : ret;
2357	}
2358	/*
2359	* Otherwise make sure the passed virtual Initiator port WWN matches
2360	* the fabric protocol_id set in vhost_scsi_make_tport(), and call
2361	* vhost_scsi_make_nexus().
2362	*/
2363	if (strlen(page) >= VHOST_SCSI_NAMELEN) {
2364	pr_err("Emulated NAA Sas Address: %s, exceeds"
2365	" max: %d\n", page, VHOST_SCSI_NAMELEN);
2366	return -EINVAL;
2367	}
2368	snprintf(buf: &i_port[0], VHOST_SCSI_NAMELEN, fmt: "%s", page);
2369
2370	ptr = strstr(i_port, "naa.");
2371	if (ptr) {
2372	if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_SAS) {
2373	pr_err("Passed SAS Initiator Port %s does not"
2374	" match target port protoid: %s\n", i_port,
2375	vhost_scsi_dump_proto_id(tport_wwn));
2376	return -EINVAL;
2377	}
2378	port_ptr = &i_port[0];
2379	goto check_newline;
2380	}
2381	ptr = strstr(i_port, "fc.");
2382	if (ptr) {
2383	if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_FCP) {
2384	pr_err("Passed FCP Initiator Port %s does not"
2385	" match target port protoid: %s\n", i_port,
2386	vhost_scsi_dump_proto_id(tport_wwn));
2387	return -EINVAL;
2388	}
2389	port_ptr = &i_port[3]; /* Skip over "fc." */
2390	goto check_newline;
2391	}
2392	ptr = strstr(i_port, "iqn.");
2393	if (ptr) {
2394	if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_ISCSI) {
2395	pr_err("Passed iSCSI Initiator Port %s does not"
2396	" match target port protoid: %s\n", i_port,
2397	vhost_scsi_dump_proto_id(tport_wwn));
2398	return -EINVAL;
2399	}
2400	port_ptr = &i_port[0];
2401	goto check_newline;
2402	}
2403	pr_err("Unable to locate prefix for emulated Initiator Port:"
2404	" %s\n", i_port);
2405	return -EINVAL;
2406	/*
2407	* Clear any trailing newline for the NAA WWN
2408	*/
2409	check_newline:
2410	if (i_port[strlen(i_port)-1] == '\n')
2411	i_port[strlen(i_port)-1] = '\0';
2412
2413	ret = vhost_scsi_make_nexus(tpg, name: port_ptr);
2414	if (ret < 0)
2415	return ret;
2416
2417	return count;
2418	}
2419
2420	CONFIGFS_ATTR(vhost_scsi_tpg_, nexus);
2421
2422	static struct configfs_attribute *vhost_scsi_tpg_attrs[] = {
2423	&vhost_scsi_tpg_attr_nexus,
2424	NULL,
2425	};
2426
2427	static struct se_portal_group *
2428	vhost_scsi_make_tpg(struct se_wwn *wwn, const char *name)
2429	{
2430	struct vhost_scsi_tport *tport = container_of(wwn,
2431	struct vhost_scsi_tport, tport_wwn);
2432
2433	struct vhost_scsi_tpg *tpg;
2434	u16 tpgt;
2435	int ret;
2436
2437	if (strstr(name, "tpgt_") != name)
2438	return ERR_PTR(error: -EINVAL);
2439	if (kstrtou16(s: name + 5, base: 10, res: &tpgt) || tpgt >= VHOST_SCSI_MAX_TARGET)
2440	return ERR_PTR(error: -EINVAL);
2441
2442	tpg = kzalloc(size: sizeof(*tpg), GFP_KERNEL);
2443	if (!tpg) {
2444	pr_err("Unable to allocate struct vhost_scsi_tpg");
2445	return ERR_PTR(error: -ENOMEM);
2446	}
2447	mutex_init(&tpg->tv_tpg_mutex);
2448	INIT_LIST_HEAD(list: &tpg->tv_tpg_list);
2449	tpg->tport = tport;
2450	tpg->tport_tpgt = tpgt;
2451
2452	ret = core_tpg_register(wwn, &tpg->se_tpg, tport->tport_proto_id);
2453	if (ret < 0) {
2454	kfree(objp: tpg);
2455	return NULL;
2456	}
2457	mutex_lock(&vhost_scsi_mutex);
2458	list_add_tail(new: &tpg->tv_tpg_list, head: &vhost_scsi_list);
2459	mutex_unlock(lock: &vhost_scsi_mutex);
2460
2461	return &tpg->se_tpg;
2462	}
2463
2464	static void vhost_scsi_drop_tpg(struct se_portal_group *se_tpg)
2465	{
2466	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2467	struct vhost_scsi_tpg, se_tpg);
2468
2469	mutex_lock(&vhost_scsi_mutex);
2470	list_del(entry: &tpg->tv_tpg_list);
2471	mutex_unlock(lock: &vhost_scsi_mutex);
2472	/*
2473	* Release the virtual I_T Nexus for this vhost TPG
2474	*/
2475	vhost_scsi_drop_nexus(tpg);
2476	/*
2477	* Deregister the se_tpg from TCM..
2478	*/
2479	core_tpg_deregister(se_tpg);
2480	kfree(objp: tpg);
2481	}
2482
2483	static struct se_wwn *
2484	vhost_scsi_make_tport(struct target_fabric_configfs *tf,
2485	struct config_group *group,
2486	const char *name)
2487	{
2488	struct vhost_scsi_tport *tport;
2489	char *ptr;
2490	u64 wwpn = 0;
2491	int off = 0;
2492
2493	/* if (vhost_scsi_parse_wwn(name, &wwpn, 1) < 0)
2494	return ERR_PTR(-EINVAL); */
2495
2496	tport = kzalloc(size: sizeof(*tport), GFP_KERNEL);
2497	if (!tport) {
2498	pr_err("Unable to allocate struct vhost_scsi_tport");
2499	return ERR_PTR(error: -ENOMEM);
2500	}
2501	tport->tport_wwpn = wwpn;
2502	/*
2503	* Determine the emulated Protocol Identifier and Target Port Name
2504	* based on the incoming configfs directory name.
2505	*/
2506	ptr = strstr(name, "naa.");
2507	if (ptr) {
2508	tport->tport_proto_id = SCSI_PROTOCOL_SAS;
2509	goto check_len;
2510	}
2511	ptr = strstr(name, "fc.");
2512	if (ptr) {
2513	tport->tport_proto_id = SCSI_PROTOCOL_FCP;
2514	off = 3; /* Skip over "fc." */
2515	goto check_len;
2516	}
2517	ptr = strstr(name, "iqn.");
2518	if (ptr) {
2519	tport->tport_proto_id = SCSI_PROTOCOL_ISCSI;
2520	goto check_len;
2521	}
2522
2523	pr_err("Unable to locate prefix for emulated Target Port:"
2524	" %s\n", name);
2525	kfree(objp: tport);
2526	return ERR_PTR(error: -EINVAL);
2527
2528	check_len:
2529	if (strlen(name) >= VHOST_SCSI_NAMELEN) {
2530	pr_err("Emulated %s Address: %s, exceeds"
2531	" max: %d\n", name, vhost_scsi_dump_proto_id(tport),
2532	VHOST_SCSI_NAMELEN);
2533	kfree(objp: tport);
2534	return ERR_PTR(error: -EINVAL);
2535	}
2536	snprintf(buf: &tport->tport_name[0], VHOST_SCSI_NAMELEN, fmt: "%s", &name[off]);
2537
2538	pr_debug("TCM_VHost_ConfigFS: Allocated emulated Target"
2539	" %s Address: %s\n", vhost_scsi_dump_proto_id(tport), name);
2540
2541	return &tport->tport_wwn;
2542	}
2543
2544	static void vhost_scsi_drop_tport(struct se_wwn *wwn)
2545	{
2546	struct vhost_scsi_tport *tport = container_of(wwn,
2547	struct vhost_scsi_tport, tport_wwn);
2548
2549	pr_debug("TCM_VHost_ConfigFS: Deallocating emulated Target"
2550	" %s Address: %s\n", vhost_scsi_dump_proto_id(tport),
2551	tport->tport_name);
2552
2553	kfree(objp: tport);
2554	}
2555
2556	static ssize_t
2557	vhost_scsi_wwn_version_show(struct config_item *item, char *page)
2558	{
2559	return sysfs_emit(buf: page, fmt: "TCM_VHOST fabric module %s on %s/%s"
2560	"on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2561	utsname()->machine);
2562	}
2563
2564	CONFIGFS_ATTR_RO(vhost_scsi_wwn_, version);
2565
2566	static struct configfs_attribute *vhost_scsi_wwn_attrs[] = {
2567	&vhost_scsi_wwn_attr_version,
2568	NULL,
2569	};
2570
2571	static const struct target_core_fabric_ops vhost_scsi_ops = {
2572	.module = THIS_MODULE,
2573	.fabric_name = "vhost",
2574	.max_data_sg_nents = VHOST_SCSI_PREALLOC_SGLS,
2575	.tpg_get_wwn = vhost_scsi_get_fabric_wwn,
2576	.tpg_get_tag = vhost_scsi_get_tpgt,
2577	.tpg_check_demo_mode = vhost_scsi_check_true,
2578	.tpg_check_demo_mode_cache = vhost_scsi_check_true,
2579	.tpg_check_prot_fabric_only = vhost_scsi_check_prot_fabric_only,
2580	.release_cmd = vhost_scsi_release_cmd,
2581	.check_stop_free = vhost_scsi_check_stop_free,
2582	.sess_get_initiator_sid = NULL,
2583	.write_pending = vhost_scsi_write_pending,
2584	.queue_data_in = vhost_scsi_queue_data_in,
2585	.queue_status = vhost_scsi_queue_status,
2586	.queue_tm_rsp = vhost_scsi_queue_tm_rsp,
2587	.aborted_task = vhost_scsi_aborted_task,
2588	/*
2589	* Setup callers for generic logic in target_core_fabric_configfs.c
2590	*/
2591	.fabric_make_wwn = vhost_scsi_make_tport,
2592	.fabric_drop_wwn = vhost_scsi_drop_tport,
2593	.fabric_make_tpg = vhost_scsi_make_tpg,
2594	.fabric_drop_tpg = vhost_scsi_drop_tpg,
2595	.fabric_post_link = vhost_scsi_port_link,
2596	.fabric_pre_unlink = vhost_scsi_port_unlink,
2597
2598	.tfc_wwn_attrs = vhost_scsi_wwn_attrs,
2599	.tfc_tpg_base_attrs = vhost_scsi_tpg_attrs,
2600	.tfc_tpg_attrib_attrs = vhost_scsi_tpg_attrib_attrs,
2601
2602	.default_submit_type = TARGET_QUEUE_SUBMIT,
2603	.direct_submit_supp = 1,
2604	};
2605
2606	static int __init vhost_scsi_init(void)
2607	{
2608	int ret = -ENOMEM;
2609
2610	pr_debug("TCM_VHOST fabric module %s on %s/%s"
2611	" on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2612	utsname()->machine);
2613
2614	ret = vhost_scsi_register();
2615	if (ret < 0)
2616	goto out;
2617
2618	ret = target_register_template(fo: &vhost_scsi_ops);
2619	if (ret < 0)
2620	goto out_vhost_scsi_deregister;
2621
2622	return 0;
2623
2624	out_vhost_scsi_deregister:
2625	vhost_scsi_deregister();
2626	out:
2627	return ret;
2628	};
2629
2630	static void vhost_scsi_exit(void)
2631	{
2632	target_unregister_template(fo: &vhost_scsi_ops);
2633	vhost_scsi_deregister();
2634	};
2635
2636	MODULE_DESCRIPTION("VHOST_SCSI series fabric driver");
2637	MODULE_ALIAS("tcm_vhost");
2638	MODULE_LICENSE("GPL");
2639	module_init(vhost_scsi_init);
2640	module_exit(vhost_scsi_exit);
2641