1// SPDX-License-Identifier: GPL-2.0
2/*
3 * NVMe over Fabrics loopback device.
4 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7#include <linux/scatterlist.h>
8#include <linux/blk-mq.h>
9#include <linux/nvme.h>
10#include <linux/module.h>
11#include <linux/parser.h>
12#include "nvmet.h"
13#include "../host/nvme.h"
14#include "../host/fabrics.h"
15
16#define NVME_LOOP_MAX_SEGMENTS 256
17
18struct nvme_loop_iod {
19 struct nvme_request nvme_req;
20 struct nvme_command cmd;
21 struct nvme_completion cqe;
22 struct nvmet_req req;
23 struct nvme_loop_queue *queue;
24 struct work_struct work;
25 struct sg_table sg_table;
26 struct scatterlist first_sgl[];
27};
28
29struct nvme_loop_ctrl {
30 struct nvme_loop_queue *queues;
31
32 struct blk_mq_tag_set admin_tag_set;
33
34 struct list_head list;
35 struct blk_mq_tag_set tag_set;
36 struct nvme_loop_iod async_event_iod;
37 struct nvme_ctrl ctrl;
38
39 struct nvmet_port *port;
40};
41
42static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl)
43{
44 return container_of(ctrl, struct nvme_loop_ctrl, ctrl);
45}
46
47enum nvme_loop_queue_flags {
48 NVME_LOOP_Q_LIVE = 0,
49};
50
51struct nvme_loop_queue {
52 struct nvmet_cq nvme_cq;
53 struct nvmet_sq nvme_sq;
54 struct nvme_loop_ctrl *ctrl;
55 unsigned long flags;
56};
57
58static LIST_HEAD(nvme_loop_ports);
59static DEFINE_MUTEX(nvme_loop_ports_mutex);
60
61static LIST_HEAD(nvme_loop_ctrl_list);
62static DEFINE_MUTEX(nvme_loop_ctrl_mutex);
63
64static void nvme_loop_queue_response(struct nvmet_req *nvme_req);
65static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl);
66
67static const struct nvmet_fabrics_ops nvme_loop_ops;
68
69static inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue)
70{
71 return queue - queue->ctrl->queues;
72}
73
74static void nvme_loop_complete_rq(struct request *req)
75{
76 struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(rq: req);
77
78 sg_free_table_chained(table: &iod->sg_table, NVME_INLINE_SG_CNT);
79 nvme_complete_rq(req);
80}
81
82static struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue)
83{
84 u32 queue_idx = nvme_loop_queue_idx(queue);
85
86 if (queue_idx == 0)
87 return queue->ctrl->admin_tag_set.tags[queue_idx];
88 return queue->ctrl->tag_set.tags[queue_idx - 1];
89}
90
91static void nvme_loop_queue_response(struct nvmet_req *req)
92{
93 struct nvme_loop_queue *queue =
94 container_of(req->sq, struct nvme_loop_queue, nvme_sq);
95 struct nvme_completion *cqe = req->cqe;
96
97 /*
98 * AEN requests are special as they don't time out and can
99 * survive any kind of queue freeze and often don't respond to
100 * aborts. We don't even bother to allocate a struct request
101 * for them but rather special case them here.
102 */
103 if (unlikely(nvme_is_aen_req(nvme_loop_queue_idx(queue),
104 cqe->command_id))) {
105 nvme_complete_async_event(ctrl: &queue->ctrl->ctrl, status: cqe->status,
106 res: &cqe->result);
107 } else {
108 struct request *rq;
109
110 rq = nvme_find_rq(tags: nvme_loop_tagset(queue), command_id: cqe->command_id);
111 if (!rq) {
112 dev_err(queue->ctrl->ctrl.device,
113 "got bad command_id %#x on queue %d\n",
114 cqe->command_id, nvme_loop_queue_idx(queue));
115 return;
116 }
117
118 if (!nvme_try_complete_req(req: rq, status: cqe->status, result: cqe->result))
119 nvme_loop_complete_rq(req: rq);
120 }
121}
122
123static void nvme_loop_execute_work(struct work_struct *work)
124{
125 struct nvme_loop_iod *iod =
126 container_of(work, struct nvme_loop_iod, work);
127
128 iod->req.execute(&iod->req);
129}
130
131static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
132 const struct blk_mq_queue_data *bd)
133{
134 struct nvme_ns *ns = hctx->queue->queuedata;
135 struct nvme_loop_queue *queue = hctx->driver_data;
136 struct request *req = bd->rq;
137 struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(rq: req);
138 bool queue_ready = test_bit(NVME_LOOP_Q_LIVE, &queue->flags);
139 blk_status_t ret;
140
141 if (!nvme_check_ready(ctrl: &queue->ctrl->ctrl, rq: req, queue_live: queue_ready))
142 return nvme_fail_nonready_command(ctrl: &queue->ctrl->ctrl, req);
143
144 ret = nvme_setup_cmd(ns, req);
145 if (ret)
146 return ret;
147
148 nvme_start_request(rq: req);
149 iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
150 iod->req.port = queue->ctrl->port;
151 if (!nvmet_req_init(req: &iod->req, cq: &queue->nvme_cq,
152 sq: &queue->nvme_sq, ops: &nvme_loop_ops))
153 return BLK_STS_OK;
154
155 if (blk_rq_nr_phys_segments(rq: req)) {
156 iod->sg_table.sgl = iod->first_sgl;
157 if (sg_alloc_table_chained(table: &iod->sg_table,
158 nents: blk_rq_nr_phys_segments(rq: req),
159 first_chunk: iod->sg_table.sgl, NVME_INLINE_SG_CNT)) {
160 nvme_cleanup_cmd(req);
161 return BLK_STS_RESOURCE;
162 }
163
164 iod->req.sg = iod->sg_table.sgl;
165 iod->req.sg_cnt = blk_rq_map_sg(q: req->q, rq: req, sglist: iod->sg_table.sgl);
166 iod->req.transfer_len = blk_rq_payload_bytes(rq: req);
167 }
168
169 queue_work(wq: nvmet_wq, work: &iod->work);
170 return BLK_STS_OK;
171}
172
173static void nvme_loop_submit_async_event(struct nvme_ctrl *arg)
174{
175 struct nvme_loop_ctrl *ctrl = to_loop_ctrl(ctrl: arg);
176 struct nvme_loop_queue *queue = &ctrl->queues[0];
177 struct nvme_loop_iod *iod = &ctrl->async_event_iod;
178
179 memset(&iod->cmd, 0, sizeof(iod->cmd));
180 iod->cmd.common.opcode = nvme_admin_async_event;
181 iod->cmd.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
182 iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
183
184 if (!nvmet_req_init(req: &iod->req, cq: &queue->nvme_cq, sq: &queue->nvme_sq,
185 ops: &nvme_loop_ops)) {
186 dev_err(ctrl->ctrl.device, "failed async event work\n");
187 return;
188 }
189
190 queue_work(wq: nvmet_wq, work: &iod->work);
191}
192
193static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl,
194 struct nvme_loop_iod *iod, unsigned int queue_idx)
195{
196 iod->req.cmd = &iod->cmd;
197 iod->req.cqe = &iod->cqe;
198 iod->queue = &ctrl->queues[queue_idx];
199 INIT_WORK(&iod->work, nvme_loop_execute_work);
200 return 0;
201}
202
203static int nvme_loop_init_request(struct blk_mq_tag_set *set,
204 struct request *req, unsigned int hctx_idx,
205 unsigned int numa_node)
206{
207 struct nvme_loop_ctrl *ctrl = to_loop_ctrl(ctrl: set->driver_data);
208 struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(rq: req);
209
210 nvme_req(req)->ctrl = &ctrl->ctrl;
211 nvme_req(req)->cmd = &iod->cmd;
212 return nvme_loop_init_iod(ctrl, iod: blk_mq_rq_to_pdu(rq: req),
213 queue_idx: (set == &ctrl->tag_set) ? hctx_idx + 1 : 0);
214}
215
216static struct lock_class_key loop_hctx_fq_lock_key;
217
218static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
219 unsigned int hctx_idx)
220{
221 struct nvme_loop_ctrl *ctrl = to_loop_ctrl(ctrl: data);
222 struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1];
223
224 BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);
225
226 /*
227 * flush_end_io() can be called recursively for us, so use our own
228 * lock class key for avoiding lockdep possible recursive locking,
229 * then we can remove the dynamically allocated lock class for each
230 * flush queue, that way may cause horrible boot delay.
231 */
232 blk_mq_hctx_set_fq_lock_class(hctx, key: &loop_hctx_fq_lock_key);
233
234 hctx->driver_data = queue;
235 return 0;
236}
237
238static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
239 unsigned int hctx_idx)
240{
241 struct nvme_loop_ctrl *ctrl = to_loop_ctrl(ctrl: data);
242 struct nvme_loop_queue *queue = &ctrl->queues[0];
243
244 BUG_ON(hctx_idx != 0);
245
246 hctx->driver_data = queue;
247 return 0;
248}
249
250static const struct blk_mq_ops nvme_loop_mq_ops = {
251 .queue_rq = nvme_loop_queue_rq,
252 .complete = nvme_loop_complete_rq,
253 .init_request = nvme_loop_init_request,
254 .init_hctx = nvme_loop_init_hctx,
255};
256
257static const struct blk_mq_ops nvme_loop_admin_mq_ops = {
258 .queue_rq = nvme_loop_queue_rq,
259 .complete = nvme_loop_complete_rq,
260 .init_request = nvme_loop_init_request,
261 .init_hctx = nvme_loop_init_admin_hctx,
262};
263
264static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl)
265{
266 if (!test_and_clear_bit(nr: NVME_LOOP_Q_LIVE, addr: &ctrl->queues[0].flags))
267 return;
268 nvmet_sq_destroy(sq: &ctrl->queues[0].nvme_sq);
269 nvme_remove_admin_tag_set(ctrl: &ctrl->ctrl);
270}
271
272static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl)
273{
274 struct nvme_loop_ctrl *ctrl = to_loop_ctrl(ctrl: nctrl);
275
276 if (list_empty(head: &ctrl->list))
277 goto free_ctrl;
278
279 mutex_lock(&nvme_loop_ctrl_mutex);
280 list_del(entry: &ctrl->list);
281 mutex_unlock(lock: &nvme_loop_ctrl_mutex);
282
283 if (nctrl->tagset)
284 nvme_remove_io_tag_set(ctrl: nctrl);
285 kfree(objp: ctrl->queues);
286 nvmf_free_options(opts: nctrl->opts);
287free_ctrl:
288 kfree(objp: ctrl);
289}
290
291static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl)
292{
293 int i;
294
295 for (i = 1; i < ctrl->ctrl.queue_count; i++) {
296 clear_bit(nr: NVME_LOOP_Q_LIVE, addr: &ctrl->queues[i].flags);
297 nvmet_sq_destroy(sq: &ctrl->queues[i].nvme_sq);
298 }
299 ctrl->ctrl.queue_count = 1;
300}
301
302static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl)
303{
304 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
305 unsigned int nr_io_queues;
306 int ret, i;
307
308 nr_io_queues = min(opts->nr_io_queues, num_online_cpus());
309 ret = nvme_set_queue_count(ctrl: &ctrl->ctrl, count: &nr_io_queues);
310 if (ret || !nr_io_queues)
311 return ret;
312
313 dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues);
314
315 for (i = 1; i <= nr_io_queues; i++) {
316 ctrl->queues[i].ctrl = ctrl;
317 ret = nvmet_sq_init(sq: &ctrl->queues[i].nvme_sq);
318 if (ret)
319 goto out_destroy_queues;
320
321 ctrl->ctrl.queue_count++;
322 }
323
324 return 0;
325
326out_destroy_queues:
327 nvme_loop_destroy_io_queues(ctrl);
328 return ret;
329}
330
331static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl)
332{
333 int i, ret;
334
335 for (i = 1; i < ctrl->ctrl.queue_count; i++) {
336 ret = nvmf_connect_io_queue(ctrl: &ctrl->ctrl, qid: i);
337 if (ret)
338 return ret;
339 set_bit(nr: NVME_LOOP_Q_LIVE, addr: &ctrl->queues[i].flags);
340 }
341
342 return 0;
343}
344
345static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)
346{
347 int error;
348
349 ctrl->queues[0].ctrl = ctrl;
350 error = nvmet_sq_init(sq: &ctrl->queues[0].nvme_sq);
351 if (error)
352 return error;
353 ctrl->ctrl.queue_count = 1;
354
355 error = nvme_alloc_admin_tag_set(ctrl: &ctrl->ctrl, set: &ctrl->admin_tag_set,
356 ops: &nvme_loop_admin_mq_ops,
357 cmd_size: sizeof(struct nvme_loop_iod) +
358 NVME_INLINE_SG_CNT * sizeof(struct scatterlist));
359 if (error)
360 goto out_free_sq;
361
362 /* reset stopped state for the fresh admin queue */
363 clear_bit(nr: NVME_CTRL_ADMIN_Q_STOPPED, addr: &ctrl->ctrl.flags);
364
365 error = nvmf_connect_admin_queue(ctrl: &ctrl->ctrl);
366 if (error)
367 goto out_cleanup_tagset;
368
369 set_bit(nr: NVME_LOOP_Q_LIVE, addr: &ctrl->queues[0].flags);
370
371 error = nvme_enable_ctrl(ctrl: &ctrl->ctrl);
372 if (error)
373 goto out_cleanup_tagset;
374
375 ctrl->ctrl.max_hw_sectors =
376 (NVME_LOOP_MAX_SEGMENTS - 1) << PAGE_SECTORS_SHIFT;
377
378 nvme_unquiesce_admin_queue(ctrl: &ctrl->ctrl);
379
380 error = nvme_init_ctrl_finish(ctrl: &ctrl->ctrl, was_suspended: false);
381 if (error)
382 goto out_cleanup_tagset;
383
384 return 0;
385
386out_cleanup_tagset:
387 clear_bit(nr: NVME_LOOP_Q_LIVE, addr: &ctrl->queues[0].flags);
388 nvme_remove_admin_tag_set(ctrl: &ctrl->ctrl);
389out_free_sq:
390 nvmet_sq_destroy(sq: &ctrl->queues[0].nvme_sq);
391 return error;
392}
393
394static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)
395{
396 if (ctrl->ctrl.queue_count > 1) {
397 nvme_quiesce_io_queues(ctrl: &ctrl->ctrl);
398 nvme_cancel_tagset(ctrl: &ctrl->ctrl);
399 nvme_loop_destroy_io_queues(ctrl);
400 }
401
402 nvme_quiesce_admin_queue(ctrl: &ctrl->ctrl);
403 if (ctrl->ctrl.state == NVME_CTRL_LIVE)
404 nvme_disable_ctrl(ctrl: &ctrl->ctrl, shutdown: true);
405
406 nvme_cancel_admin_tagset(ctrl: &ctrl->ctrl);
407 nvme_loop_destroy_admin_queue(ctrl);
408}
409
410static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl)
411{
412 nvme_loop_shutdown_ctrl(ctrl: to_loop_ctrl(ctrl));
413}
414
415static void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl)
416{
417 struct nvme_loop_ctrl *ctrl;
418
419 mutex_lock(&nvme_loop_ctrl_mutex);
420 list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) {
421 if (ctrl->ctrl.cntlid == nctrl->cntlid)
422 nvme_delete_ctrl(ctrl: &ctrl->ctrl);
423 }
424 mutex_unlock(lock: &nvme_loop_ctrl_mutex);
425}
426
427static void nvme_loop_reset_ctrl_work(struct work_struct *work)
428{
429 struct nvme_loop_ctrl *ctrl =
430 container_of(work, struct nvme_loop_ctrl, ctrl.reset_work);
431 int ret;
432
433 nvme_stop_ctrl(ctrl: &ctrl->ctrl);
434 nvme_loop_shutdown_ctrl(ctrl);
435
436 if (!nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_CONNECTING)) {
437 if (ctrl->ctrl.state != NVME_CTRL_DELETING &&
438 ctrl->ctrl.state != NVME_CTRL_DELETING_NOIO)
439 /* state change failure for non-deleted ctrl? */
440 WARN_ON_ONCE(1);
441 return;
442 }
443
444 ret = nvme_loop_configure_admin_queue(ctrl);
445 if (ret)
446 goto out_disable;
447
448 ret = nvme_loop_init_io_queues(ctrl);
449 if (ret)
450 goto out_destroy_admin;
451
452 ret = nvme_loop_connect_io_queues(ctrl);
453 if (ret)
454 goto out_destroy_io;
455
456 blk_mq_update_nr_hw_queues(set: &ctrl->tag_set,
457 nr_hw_queues: ctrl->ctrl.queue_count - 1);
458
459 if (!nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_LIVE))
460 WARN_ON_ONCE(1);
461
462 nvme_start_ctrl(ctrl: &ctrl->ctrl);
463
464 return;
465
466out_destroy_io:
467 nvme_loop_destroy_io_queues(ctrl);
468out_destroy_admin:
469 nvme_loop_destroy_admin_queue(ctrl);
470out_disable:
471 dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
472 nvme_uninit_ctrl(ctrl: &ctrl->ctrl);
473}
474
475static const struct nvme_ctrl_ops nvme_loop_ctrl_ops = {
476 .name = "loop",
477 .module = THIS_MODULE,
478 .flags = NVME_F_FABRICS,
479 .reg_read32 = nvmf_reg_read32,
480 .reg_read64 = nvmf_reg_read64,
481 .reg_write32 = nvmf_reg_write32,
482 .free_ctrl = nvme_loop_free_ctrl,
483 .submit_async_event = nvme_loop_submit_async_event,
484 .delete_ctrl = nvme_loop_delete_ctrl_host,
485 .get_address = nvmf_get_address,
486};
487
488static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)
489{
490 int ret;
491
492 ret = nvme_loop_init_io_queues(ctrl);
493 if (ret)
494 return ret;
495
496 ret = nvme_alloc_io_tag_set(ctrl: &ctrl->ctrl, set: &ctrl->tag_set,
497 ops: &nvme_loop_mq_ops, nr_maps: 1,
498 cmd_size: sizeof(struct nvme_loop_iod) +
499 NVME_INLINE_SG_CNT * sizeof(struct scatterlist));
500 if (ret)
501 goto out_destroy_queues;
502
503 ret = nvme_loop_connect_io_queues(ctrl);
504 if (ret)
505 goto out_cleanup_tagset;
506
507 return 0;
508
509out_cleanup_tagset:
510 nvme_remove_io_tag_set(ctrl: &ctrl->ctrl);
511out_destroy_queues:
512 nvme_loop_destroy_io_queues(ctrl);
513 return ret;
514}
515
516static struct nvmet_port *nvme_loop_find_port(struct nvme_ctrl *ctrl)
517{
518 struct nvmet_port *p, *found = NULL;
519
520 mutex_lock(&nvme_loop_ports_mutex);
521 list_for_each_entry(p, &nvme_loop_ports, entry) {
522 /* if no transport address is specified use the first port */
523 if ((ctrl->opts->mask & NVMF_OPT_TRADDR) &&
524 strcmp(ctrl->opts->traddr, p->disc_addr.traddr))
525 continue;
526 found = p;
527 break;
528 }
529 mutex_unlock(lock: &nvme_loop_ports_mutex);
530 return found;
531}
532
533static struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev,
534 struct nvmf_ctrl_options *opts)
535{
536 struct nvme_loop_ctrl *ctrl;
537 int ret;
538
539 ctrl = kzalloc(size: sizeof(*ctrl), GFP_KERNEL);
540 if (!ctrl)
541 return ERR_PTR(error: -ENOMEM);
542 ctrl->ctrl.opts = opts;
543 INIT_LIST_HEAD(list: &ctrl->list);
544
545 INIT_WORK(&ctrl->ctrl.reset_work, nvme_loop_reset_ctrl_work);
546
547 ret = nvme_init_ctrl(ctrl: &ctrl->ctrl, dev, ops: &nvme_loop_ctrl_ops,
548 quirks: 0 /* no quirks, we're perfect! */);
549 if (ret) {
550 kfree(objp: ctrl);
551 goto out;
552 }
553
554 if (!nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_CONNECTING))
555 WARN_ON_ONCE(1);
556
557 ret = -ENOMEM;
558
559 ctrl->ctrl.kato = opts->kato;
560 ctrl->port = nvme_loop_find_port(ctrl: &ctrl->ctrl);
561
562 ctrl->queues = kcalloc(n: opts->nr_io_queues + 1, size: sizeof(*ctrl->queues),
563 GFP_KERNEL);
564 if (!ctrl->queues)
565 goto out_uninit_ctrl;
566
567 ret = nvme_loop_configure_admin_queue(ctrl);
568 if (ret)
569 goto out_free_queues;
570
571 if (opts->queue_size > ctrl->ctrl.maxcmd) {
572 /* warn if maxcmd is lower than queue_size */
573 dev_warn(ctrl->ctrl.device,
574 "queue_size %zu > ctrl maxcmd %u, clamping down\n",
575 opts->queue_size, ctrl->ctrl.maxcmd);
576 opts->queue_size = ctrl->ctrl.maxcmd;
577 }
578 ctrl->ctrl.sqsize = opts->queue_size - 1;
579
580 if (opts->nr_io_queues) {
581 ret = nvme_loop_create_io_queues(ctrl);
582 if (ret)
583 goto out_remove_admin_queue;
584 }
585
586 nvme_loop_init_iod(ctrl, iod: &ctrl->async_event_iod, queue_idx: 0);
587
588 dev_info(ctrl->ctrl.device,
589 "new ctrl: \"%s\"\n", ctrl->ctrl.opts->subsysnqn);
590
591 if (!nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_LIVE))
592 WARN_ON_ONCE(1);
593
594 mutex_lock(&nvme_loop_ctrl_mutex);
595 list_add_tail(new: &ctrl->list, head: &nvme_loop_ctrl_list);
596 mutex_unlock(lock: &nvme_loop_ctrl_mutex);
597
598 nvme_start_ctrl(ctrl: &ctrl->ctrl);
599
600 return &ctrl->ctrl;
601
602out_remove_admin_queue:
603 nvme_loop_destroy_admin_queue(ctrl);
604out_free_queues:
605 kfree(objp: ctrl->queues);
606out_uninit_ctrl:
607 nvme_uninit_ctrl(ctrl: &ctrl->ctrl);
608 nvme_put_ctrl(ctrl: &ctrl->ctrl);
609out:
610 if (ret > 0)
611 ret = -EIO;
612 return ERR_PTR(error: ret);
613}
614
615static int nvme_loop_add_port(struct nvmet_port *port)
616{
617 mutex_lock(&nvme_loop_ports_mutex);
618 list_add_tail(new: &port->entry, head: &nvme_loop_ports);
619 mutex_unlock(lock: &nvme_loop_ports_mutex);
620 return 0;
621}
622
623static void nvme_loop_remove_port(struct nvmet_port *port)
624{
625 mutex_lock(&nvme_loop_ports_mutex);
626 list_del_init(entry: &port->entry);
627 mutex_unlock(lock: &nvme_loop_ports_mutex);
628
629 /*
630 * Ensure any ctrls that are in the process of being
631 * deleted are in fact deleted before we return
632 * and free the port. This is to prevent active
633 * ctrls from using a port after it's freed.
634 */
635 flush_workqueue(nvme_delete_wq);
636}
637
638static const struct nvmet_fabrics_ops nvme_loop_ops = {
639 .owner = THIS_MODULE,
640 .type = NVMF_TRTYPE_LOOP,
641 .add_port = nvme_loop_add_port,
642 .remove_port = nvme_loop_remove_port,
643 .queue_response = nvme_loop_queue_response,
644 .delete_ctrl = nvme_loop_delete_ctrl,
645};
646
647static struct nvmf_transport_ops nvme_loop_transport = {
648 .name = "loop",
649 .module = THIS_MODULE,
650 .create_ctrl = nvme_loop_create_ctrl,
651 .allowed_opts = NVMF_OPT_TRADDR,
652};
653
654static int __init nvme_loop_init_module(void)
655{
656 int ret;
657
658 ret = nvmet_register_transport(ops: &nvme_loop_ops);
659 if (ret)
660 return ret;
661
662 ret = nvmf_register_transport(ops: &nvme_loop_transport);
663 if (ret)
664 nvmet_unregister_transport(ops: &nvme_loop_ops);
665
666 return ret;
667}
668
669static void __exit nvme_loop_cleanup_module(void)
670{
671 struct nvme_loop_ctrl *ctrl, *next;
672
673 nvmf_unregister_transport(ops: &nvme_loop_transport);
674 nvmet_unregister_transport(ops: &nvme_loop_ops);
675
676 mutex_lock(&nvme_loop_ctrl_mutex);
677 list_for_each_entry_safe(ctrl, next, &nvme_loop_ctrl_list, list)
678 nvme_delete_ctrl(ctrl: &ctrl->ctrl);
679 mutex_unlock(lock: &nvme_loop_ctrl_mutex);
680
681 flush_workqueue(nvme_delete_wq);
682}
683
684module_init(nvme_loop_init_module);
685module_exit(nvme_loop_cleanup_module);
686
687MODULE_LICENSE("GPL v2");
688MODULE_ALIAS("nvmet-transport-254"); /* 254 == NVMF_TRTYPE_LOOP */
689

source code of linux/drivers/nvme/target/loop.c