1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * NVMe over Fabrics common host code. |
4 | * Copyright (c) 2015-2016 HGST, a Western Digital Company. |
5 | */ |
6 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
7 | #include <linux/init.h> |
8 | #include <linux/miscdevice.h> |
9 | #include <linux/module.h> |
10 | #include <linux/mutex.h> |
11 | #include <linux/parser.h> |
12 | #include <linux/seq_file.h> |
13 | #include "nvme.h" |
14 | #include "fabrics.h" |
15 | #include <linux/nvme-keyring.h> |
16 | |
17 | static LIST_HEAD(nvmf_transports); |
18 | static DECLARE_RWSEM(nvmf_transports_rwsem); |
19 | |
20 | static LIST_HEAD(nvmf_hosts); |
21 | static DEFINE_MUTEX(nvmf_hosts_mutex); |
22 | |
23 | static struct nvmf_host *nvmf_default_host; |
24 | |
25 | static struct nvmf_host *nvmf_host_alloc(const char *hostnqn, uuid_t *id) |
26 | { |
27 | struct nvmf_host *host; |
28 | |
29 | host = kmalloc(size: sizeof(*host), GFP_KERNEL); |
30 | if (!host) |
31 | return NULL; |
32 | |
33 | kref_init(kref: &host->ref); |
34 | uuid_copy(dst: &host->id, src: id); |
35 | strscpy(p: host->nqn, q: hostnqn, NVMF_NQN_SIZE); |
36 | |
37 | return host; |
38 | } |
39 | |
40 | static struct nvmf_host *nvmf_host_add(const char *hostnqn, uuid_t *id) |
41 | { |
42 | struct nvmf_host *host; |
43 | |
44 | mutex_lock(&nvmf_hosts_mutex); |
45 | |
46 | /* |
47 | * We have defined a host as how it is perceived by the target. |
48 | * Therefore, we don't allow different Host NQNs with the same Host ID. |
49 | * Similarly, we do not allow the usage of the same Host NQN with |
50 | * different Host IDs. This'll maintain unambiguous host identification. |
51 | */ |
52 | list_for_each_entry(host, &nvmf_hosts, list) { |
53 | bool same_hostnqn = !strcmp(host->nqn, hostnqn); |
54 | bool same_hostid = uuid_equal(u1: &host->id, u2: id); |
55 | |
56 | if (same_hostnqn && same_hostid) { |
57 | kref_get(kref: &host->ref); |
58 | goto out_unlock; |
59 | } |
60 | if (same_hostnqn) { |
61 | pr_err("found same hostnqn %s but different hostid %pUb\n" , |
62 | hostnqn, id); |
63 | host = ERR_PTR(error: -EINVAL); |
64 | goto out_unlock; |
65 | } |
66 | if (same_hostid) { |
67 | pr_err("found same hostid %pUb but different hostnqn %s\n" , |
68 | id, hostnqn); |
69 | host = ERR_PTR(error: -EINVAL); |
70 | goto out_unlock; |
71 | } |
72 | } |
73 | |
74 | host = nvmf_host_alloc(hostnqn, id); |
75 | if (!host) { |
76 | host = ERR_PTR(error: -ENOMEM); |
77 | goto out_unlock; |
78 | } |
79 | |
80 | list_add_tail(new: &host->list, head: &nvmf_hosts); |
81 | out_unlock: |
82 | mutex_unlock(lock: &nvmf_hosts_mutex); |
83 | return host; |
84 | } |
85 | |
86 | static struct nvmf_host *nvmf_host_default(void) |
87 | { |
88 | struct nvmf_host *host; |
89 | char nqn[NVMF_NQN_SIZE]; |
90 | uuid_t id; |
91 | |
92 | uuid_gen(u: &id); |
93 | snprintf(buf: nqn, NVMF_NQN_SIZE, |
94 | fmt: "nqn.2014-08.org.nvmexpress:uuid:%pUb" , &id); |
95 | |
96 | host = nvmf_host_alloc(hostnqn: nqn, id: &id); |
97 | if (!host) |
98 | return NULL; |
99 | |
100 | mutex_lock(&nvmf_hosts_mutex); |
101 | list_add_tail(new: &host->list, head: &nvmf_hosts); |
102 | mutex_unlock(lock: &nvmf_hosts_mutex); |
103 | |
104 | return host; |
105 | } |
106 | |
107 | static void nvmf_host_destroy(struct kref *ref) |
108 | { |
109 | struct nvmf_host *host = container_of(ref, struct nvmf_host, ref); |
110 | |
111 | mutex_lock(&nvmf_hosts_mutex); |
112 | list_del(entry: &host->list); |
113 | mutex_unlock(lock: &nvmf_hosts_mutex); |
114 | |
115 | kfree(objp: host); |
116 | } |
117 | |
118 | static void nvmf_host_put(struct nvmf_host *host) |
119 | { |
120 | if (host) |
121 | kref_put(kref: &host->ref, release: nvmf_host_destroy); |
122 | } |
123 | |
124 | /** |
125 | * nvmf_get_address() - Get address/port |
126 | * @ctrl: Host NVMe controller instance which we got the address |
127 | * @buf: OUTPUT parameter that will contain the address/port |
128 | * @size: buffer size |
129 | */ |
130 | int nvmf_get_address(struct nvme_ctrl *ctrl, char *buf, int size) |
131 | { |
132 | int len = 0; |
133 | |
134 | if (ctrl->opts->mask & NVMF_OPT_TRADDR) |
135 | len += scnprintf(buf, size, fmt: "traddr=%s" , ctrl->opts->traddr); |
136 | if (ctrl->opts->mask & NVMF_OPT_TRSVCID) |
137 | len += scnprintf(buf: buf + len, size: size - len, fmt: "%strsvcid=%s" , |
138 | (len) ? "," : "" , ctrl->opts->trsvcid); |
139 | if (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR) |
140 | len += scnprintf(buf: buf + len, size: size - len, fmt: "%shost_traddr=%s" , |
141 | (len) ? "," : "" , ctrl->opts->host_traddr); |
142 | if (ctrl->opts->mask & NVMF_OPT_HOST_IFACE) |
143 | len += scnprintf(buf: buf + len, size: size - len, fmt: "%shost_iface=%s" , |
144 | (len) ? "," : "" , ctrl->opts->host_iface); |
145 | len += scnprintf(buf: buf + len, size: size - len, fmt: "\n" ); |
146 | |
147 | return len; |
148 | } |
149 | EXPORT_SYMBOL_GPL(nvmf_get_address); |
150 | |
151 | /** |
152 | * nvmf_reg_read32() - NVMe Fabrics "Property Get" API function. |
153 | * @ctrl: Host NVMe controller instance maintaining the admin |
154 | * queue used to submit the property read command to |
155 | * the allocated NVMe controller resource on the target system. |
156 | * @off: Starting offset value of the targeted property |
157 | * register (see the fabrics section of the NVMe standard). |
158 | * @val: OUTPUT parameter that will contain the value of |
159 | * the property after a successful read. |
160 | * |
161 | * Used by the host system to retrieve a 32-bit capsule property value |
162 | * from an NVMe controller on the target system. |
163 | * |
164 | * ("Capsule property" is an "PCIe register concept" applied to the |
165 | * NVMe fabrics space.) |
166 | * |
167 | * Return: |
168 | * 0: successful read |
169 | * > 0: NVMe error status code |
170 | * < 0: Linux errno error code |
171 | */ |
172 | int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val) |
173 | { |
174 | struct nvme_command cmd = { }; |
175 | union nvme_result res; |
176 | int ret; |
177 | |
178 | cmd.prop_get.opcode = nvme_fabrics_command; |
179 | cmd.prop_get.fctype = nvme_fabrics_type_property_get; |
180 | cmd.prop_get.offset = cpu_to_le32(off); |
181 | |
182 | ret = __nvme_submit_sync_cmd(q: ctrl->fabrics_q, cmd: &cmd, result: &res, NULL, bufflen: 0, |
183 | NVME_QID_ANY, at_head: 0, flags: 0); |
184 | |
185 | if (ret >= 0) |
186 | *val = le64_to_cpu(res.u64); |
187 | if (unlikely(ret != 0)) |
188 | dev_err(ctrl->device, |
189 | "Property Get error: %d, offset %#x\n" , |
190 | ret > 0 ? ret & ~NVME_SC_DNR : ret, off); |
191 | |
192 | return ret; |
193 | } |
194 | EXPORT_SYMBOL_GPL(nvmf_reg_read32); |
195 | |
196 | /** |
197 | * nvmf_reg_read64() - NVMe Fabrics "Property Get" API function. |
198 | * @ctrl: Host NVMe controller instance maintaining the admin |
199 | * queue used to submit the property read command to |
200 | * the allocated controller resource on the target system. |
201 | * @off: Starting offset value of the targeted property |
202 | * register (see the fabrics section of the NVMe standard). |
203 | * @val: OUTPUT parameter that will contain the value of |
204 | * the property after a successful read. |
205 | * |
206 | * Used by the host system to retrieve a 64-bit capsule property value |
207 | * from an NVMe controller on the target system. |
208 | * |
209 | * ("Capsule property" is an "PCIe register concept" applied to the |
210 | * NVMe fabrics space.) |
211 | * |
212 | * Return: |
213 | * 0: successful read |
214 | * > 0: NVMe error status code |
215 | * < 0: Linux errno error code |
216 | */ |
217 | int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val) |
218 | { |
219 | struct nvme_command cmd = { }; |
220 | union nvme_result res; |
221 | int ret; |
222 | |
223 | cmd.prop_get.opcode = nvme_fabrics_command; |
224 | cmd.prop_get.fctype = nvme_fabrics_type_property_get; |
225 | cmd.prop_get.attrib = 1; |
226 | cmd.prop_get.offset = cpu_to_le32(off); |
227 | |
228 | ret = __nvme_submit_sync_cmd(q: ctrl->fabrics_q, cmd: &cmd, result: &res, NULL, bufflen: 0, |
229 | NVME_QID_ANY, at_head: 0, flags: 0); |
230 | |
231 | if (ret >= 0) |
232 | *val = le64_to_cpu(res.u64); |
233 | if (unlikely(ret != 0)) |
234 | dev_err(ctrl->device, |
235 | "Property Get error: %d, offset %#x\n" , |
236 | ret > 0 ? ret & ~NVME_SC_DNR : ret, off); |
237 | return ret; |
238 | } |
239 | EXPORT_SYMBOL_GPL(nvmf_reg_read64); |
240 | |
241 | /** |
242 | * nvmf_reg_write32() - NVMe Fabrics "Property Write" API function. |
243 | * @ctrl: Host NVMe controller instance maintaining the admin |
244 | * queue used to submit the property read command to |
245 | * the allocated NVMe controller resource on the target system. |
246 | * @off: Starting offset value of the targeted property |
247 | * register (see the fabrics section of the NVMe standard). |
248 | * @val: Input parameter that contains the value to be |
249 | * written to the property. |
250 | * |
251 | * Used by the NVMe host system to write a 32-bit capsule property value |
252 | * to an NVMe controller on the target system. |
253 | * |
254 | * ("Capsule property" is an "PCIe register concept" applied to the |
255 | * NVMe fabrics space.) |
256 | * |
257 | * Return: |
258 | * 0: successful write |
259 | * > 0: NVMe error status code |
260 | * < 0: Linux errno error code |
261 | */ |
262 | int nvmf_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val) |
263 | { |
264 | struct nvme_command cmd = { }; |
265 | int ret; |
266 | |
267 | cmd.prop_set.opcode = nvme_fabrics_command; |
268 | cmd.prop_set.fctype = nvme_fabrics_type_property_set; |
269 | cmd.prop_set.attrib = 0; |
270 | cmd.prop_set.offset = cpu_to_le32(off); |
271 | cmd.prop_set.value = cpu_to_le64(val); |
272 | |
273 | ret = __nvme_submit_sync_cmd(q: ctrl->fabrics_q, cmd: &cmd, NULL, NULL, bufflen: 0, |
274 | NVME_QID_ANY, at_head: 0, flags: 0); |
275 | if (unlikely(ret)) |
276 | dev_err(ctrl->device, |
277 | "Property Set error: %d, offset %#x\n" , |
278 | ret > 0 ? ret & ~NVME_SC_DNR : ret, off); |
279 | return ret; |
280 | } |
281 | EXPORT_SYMBOL_GPL(nvmf_reg_write32); |
282 | |
283 | /** |
284 | * nvmf_log_connect_error() - Error-parsing-diagnostic print out function for |
285 | * connect() errors. |
286 | * @ctrl: The specific /dev/nvmeX device that had the error. |
287 | * @errval: Error code to be decoded in a more human-friendly |
288 | * printout. |
289 | * @offset: For use with the NVMe error code |
290 | * NVME_SC_CONNECT_INVALID_PARAM. |
291 | * @cmd: This is the SQE portion of a submission capsule. |
292 | * @data: This is the "Data" portion of a submission capsule. |
293 | */ |
294 | static void nvmf_log_connect_error(struct nvme_ctrl *ctrl, |
295 | int errval, int offset, struct nvme_command *cmd, |
296 | struct nvmf_connect_data *data) |
297 | { |
298 | int err_sctype = errval & ~NVME_SC_DNR; |
299 | |
300 | if (errval < 0) { |
301 | dev_err(ctrl->device, |
302 | "Connect command failed, errno: %d\n" , errval); |
303 | return; |
304 | } |
305 | |
306 | switch (err_sctype) { |
307 | case NVME_SC_CONNECT_INVALID_PARAM: |
308 | if (offset >> 16) { |
309 | char *inv_data = "Connect Invalid Data Parameter" ; |
310 | |
311 | switch (offset & 0xffff) { |
312 | case (offsetof(struct nvmf_connect_data, cntlid)): |
313 | dev_err(ctrl->device, |
314 | "%s, cntlid: %d\n" , |
315 | inv_data, data->cntlid); |
316 | break; |
317 | case (offsetof(struct nvmf_connect_data, hostnqn)): |
318 | dev_err(ctrl->device, |
319 | "%s, hostnqn \"%s\"\n" , |
320 | inv_data, data->hostnqn); |
321 | break; |
322 | case (offsetof(struct nvmf_connect_data, subsysnqn)): |
323 | dev_err(ctrl->device, |
324 | "%s, subsysnqn \"%s\"\n" , |
325 | inv_data, data->subsysnqn); |
326 | break; |
327 | default: |
328 | dev_err(ctrl->device, |
329 | "%s, starting byte offset: %d\n" , |
330 | inv_data, offset & 0xffff); |
331 | break; |
332 | } |
333 | } else { |
334 | char *inv_sqe = "Connect Invalid SQE Parameter" ; |
335 | |
336 | switch (offset) { |
337 | case (offsetof(struct nvmf_connect_command, qid)): |
338 | dev_err(ctrl->device, |
339 | "%s, qid %d\n" , |
340 | inv_sqe, cmd->connect.qid); |
341 | break; |
342 | default: |
343 | dev_err(ctrl->device, |
344 | "%s, starting byte offset: %d\n" , |
345 | inv_sqe, offset); |
346 | } |
347 | } |
348 | break; |
349 | case NVME_SC_CONNECT_INVALID_HOST: |
350 | dev_err(ctrl->device, |
351 | "Connect for subsystem %s is not allowed, hostnqn: %s\n" , |
352 | data->subsysnqn, data->hostnqn); |
353 | break; |
354 | case NVME_SC_CONNECT_CTRL_BUSY: |
355 | dev_err(ctrl->device, |
356 | "Connect command failed: controller is busy or not available\n" ); |
357 | break; |
358 | case NVME_SC_CONNECT_FORMAT: |
359 | dev_err(ctrl->device, |
360 | "Connect incompatible format: %d" , |
361 | cmd->connect.recfmt); |
362 | break; |
363 | case NVME_SC_HOST_PATH_ERROR: |
364 | dev_err(ctrl->device, |
365 | "Connect command failed: host path error\n" ); |
366 | break; |
367 | case NVME_SC_AUTH_REQUIRED: |
368 | dev_err(ctrl->device, |
369 | "Connect command failed: authentication required\n" ); |
370 | break; |
371 | default: |
372 | dev_err(ctrl->device, |
373 | "Connect command failed, error wo/DNR bit: %d\n" , |
374 | err_sctype); |
375 | break; |
376 | } |
377 | } |
378 | |
379 | static struct nvmf_connect_data *nvmf_connect_data_prep(struct nvme_ctrl *ctrl, |
380 | u16 cntlid) |
381 | { |
382 | struct nvmf_connect_data *data; |
383 | |
384 | data = kzalloc(size: sizeof(*data), GFP_KERNEL); |
385 | if (!data) |
386 | return NULL; |
387 | |
388 | uuid_copy(dst: &data->hostid, src: &ctrl->opts->host->id); |
389 | data->cntlid = cpu_to_le16(cntlid); |
390 | strncpy(p: data->subsysnqn, q: ctrl->opts->subsysnqn, NVMF_NQN_SIZE); |
391 | strncpy(p: data->hostnqn, q: ctrl->opts->host->nqn, NVMF_NQN_SIZE); |
392 | |
393 | return data; |
394 | } |
395 | |
396 | static void nvmf_connect_cmd_prep(struct nvme_ctrl *ctrl, u16 qid, |
397 | struct nvme_command *cmd) |
398 | { |
399 | cmd->connect.opcode = nvme_fabrics_command; |
400 | cmd->connect.fctype = nvme_fabrics_type_connect; |
401 | cmd->connect.qid = cpu_to_le16(qid); |
402 | |
403 | if (qid) { |
404 | cmd->connect.sqsize = cpu_to_le16(ctrl->sqsize); |
405 | } else { |
406 | cmd->connect.sqsize = cpu_to_le16(NVME_AQ_DEPTH - 1); |
407 | |
408 | /* |
409 | * set keep-alive timeout in seconds granularity (ms * 1000) |
410 | */ |
411 | cmd->connect.kato = cpu_to_le32(ctrl->kato * 1000); |
412 | } |
413 | |
414 | if (ctrl->opts->disable_sqflow) |
415 | cmd->connect.cattr |= NVME_CONNECT_DISABLE_SQFLOW; |
416 | } |
417 | |
418 | /** |
419 | * nvmf_connect_admin_queue() - NVMe Fabrics Admin Queue "Connect" |
420 | * API function. |
421 | * @ctrl: Host nvme controller instance used to request |
422 | * a new NVMe controller allocation on the target |
423 | * system and establish an NVMe Admin connection to |
424 | * that controller. |
425 | * |
426 | * This function enables an NVMe host device to request a new allocation of |
427 | * an NVMe controller resource on a target system as well establish a |
428 | * fabrics-protocol connection of the NVMe Admin queue between the |
429 | * host system device and the allocated NVMe controller on the |
430 | * target system via a NVMe Fabrics "Connect" command. |
431 | * |
432 | * Return: |
433 | * 0: success |
434 | * > 0: NVMe error status code |
435 | * < 0: Linux errno error code |
436 | * |
437 | */ |
438 | int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl) |
439 | { |
440 | struct nvme_command cmd = { }; |
441 | union nvme_result res; |
442 | struct nvmf_connect_data *data; |
443 | int ret; |
444 | u32 result; |
445 | |
446 | nvmf_connect_cmd_prep(ctrl, qid: 0, cmd: &cmd); |
447 | |
448 | data = nvmf_connect_data_prep(ctrl, cntlid: 0xffff); |
449 | if (!data) |
450 | return -ENOMEM; |
451 | |
452 | ret = __nvme_submit_sync_cmd(q: ctrl->fabrics_q, cmd: &cmd, result: &res, |
453 | buffer: data, bufflen: sizeof(*data), NVME_QID_ANY, at_head: 1, |
454 | flags: BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT); |
455 | if (ret) { |
456 | nvmf_log_connect_error(ctrl, errval: ret, le32_to_cpu(res.u32), |
457 | cmd: &cmd, data); |
458 | goto out_free_data; |
459 | } |
460 | |
461 | result = le32_to_cpu(res.u32); |
462 | ctrl->cntlid = result & 0xFFFF; |
463 | if (result & (NVME_CONNECT_AUTHREQ_ATR | NVME_CONNECT_AUTHREQ_ASCR)) { |
464 | /* Secure concatenation is not implemented */ |
465 | if (result & NVME_CONNECT_AUTHREQ_ASCR) { |
466 | dev_warn(ctrl->device, |
467 | "qid 0: secure concatenation is not supported\n" ); |
468 | ret = NVME_SC_AUTH_REQUIRED; |
469 | goto out_free_data; |
470 | } |
471 | /* Authentication required */ |
472 | ret = nvme_auth_negotiate(ctrl, qid: 0); |
473 | if (ret) { |
474 | dev_warn(ctrl->device, |
475 | "qid 0: authentication setup failed\n" ); |
476 | ret = NVME_SC_AUTH_REQUIRED; |
477 | goto out_free_data; |
478 | } |
479 | ret = nvme_auth_wait(ctrl, qid: 0); |
480 | if (ret) |
481 | dev_warn(ctrl->device, |
482 | "qid 0: authentication failed\n" ); |
483 | else |
484 | dev_info(ctrl->device, |
485 | "qid 0: authenticated\n" ); |
486 | } |
487 | out_free_data: |
488 | kfree(objp: data); |
489 | return ret; |
490 | } |
491 | EXPORT_SYMBOL_GPL(nvmf_connect_admin_queue); |
492 | |
493 | /** |
494 | * nvmf_connect_io_queue() - NVMe Fabrics I/O Queue "Connect" |
495 | * API function. |
496 | * @ctrl: Host nvme controller instance used to establish an |
497 | * NVMe I/O queue connection to the already allocated NVMe |
498 | * controller on the target system. |
499 | * @qid: NVMe I/O queue number for the new I/O connection between |
500 | * host and target (note qid == 0 is illegal as this is |
501 | * the Admin queue, per NVMe standard). |
502 | * |
503 | * This function issues a fabrics-protocol connection |
504 | * of a NVMe I/O queue (via NVMe Fabrics "Connect" command) |
505 | * between the host system device and the allocated NVMe controller |
506 | * on the target system. |
507 | * |
508 | * Return: |
509 | * 0: success |
510 | * > 0: NVMe error status code |
511 | * < 0: Linux errno error code |
512 | */ |
513 | int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid) |
514 | { |
515 | struct nvme_command cmd = { }; |
516 | struct nvmf_connect_data *data; |
517 | union nvme_result res; |
518 | int ret; |
519 | u32 result; |
520 | |
521 | nvmf_connect_cmd_prep(ctrl, qid, cmd: &cmd); |
522 | |
523 | data = nvmf_connect_data_prep(ctrl, cntlid: ctrl->cntlid); |
524 | if (!data) |
525 | return -ENOMEM; |
526 | |
527 | ret = __nvme_submit_sync_cmd(q: ctrl->connect_q, cmd: &cmd, result: &res, |
528 | buffer: data, bufflen: sizeof(*data), qid, at_head: 1, |
529 | flags: BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT); |
530 | if (ret) { |
531 | nvmf_log_connect_error(ctrl, errval: ret, le32_to_cpu(res.u32), |
532 | cmd: &cmd, data); |
533 | } |
534 | result = le32_to_cpu(res.u32); |
535 | if (result & (NVME_CONNECT_AUTHREQ_ATR | NVME_CONNECT_AUTHREQ_ASCR)) { |
536 | /* Secure concatenation is not implemented */ |
537 | if (result & NVME_CONNECT_AUTHREQ_ASCR) { |
538 | dev_warn(ctrl->device, |
539 | "qid 0: secure concatenation is not supported\n" ); |
540 | ret = NVME_SC_AUTH_REQUIRED; |
541 | goto out_free_data; |
542 | } |
543 | /* Authentication required */ |
544 | ret = nvme_auth_negotiate(ctrl, qid); |
545 | if (ret) { |
546 | dev_warn(ctrl->device, |
547 | "qid %d: authentication setup failed\n" , qid); |
548 | ret = NVME_SC_AUTH_REQUIRED; |
549 | } else { |
550 | ret = nvme_auth_wait(ctrl, qid); |
551 | if (ret) |
552 | dev_warn(ctrl->device, |
553 | "qid %u: authentication failed\n" , qid); |
554 | } |
555 | } |
556 | out_free_data: |
557 | kfree(objp: data); |
558 | return ret; |
559 | } |
560 | EXPORT_SYMBOL_GPL(nvmf_connect_io_queue); |
561 | |
562 | bool nvmf_should_reconnect(struct nvme_ctrl *ctrl) |
563 | { |
564 | if (ctrl->opts->max_reconnects == -1 || |
565 | ctrl->nr_reconnects < ctrl->opts->max_reconnects) |
566 | return true; |
567 | |
568 | return false; |
569 | } |
570 | EXPORT_SYMBOL_GPL(nvmf_should_reconnect); |
571 | |
572 | /** |
573 | * nvmf_register_transport() - NVMe Fabrics Library registration function. |
574 | * @ops: Transport ops instance to be registered to the |
575 | * common fabrics library. |
576 | * |
577 | * API function that registers the type of specific transport fabric |
578 | * being implemented to the common NVMe fabrics library. Part of |
579 | * the overall init sequence of starting up a fabrics driver. |
580 | */ |
581 | int nvmf_register_transport(struct nvmf_transport_ops *ops) |
582 | { |
583 | if (!ops->create_ctrl) |
584 | return -EINVAL; |
585 | |
586 | down_write(sem: &nvmf_transports_rwsem); |
587 | list_add_tail(new: &ops->entry, head: &nvmf_transports); |
588 | up_write(sem: &nvmf_transports_rwsem); |
589 | |
590 | return 0; |
591 | } |
592 | EXPORT_SYMBOL_GPL(nvmf_register_transport); |
593 | |
594 | /** |
595 | * nvmf_unregister_transport() - NVMe Fabrics Library unregistration function. |
596 | * @ops: Transport ops instance to be unregistered from the |
597 | * common fabrics library. |
598 | * |
599 | * Fabrics API function that unregisters the type of specific transport |
600 | * fabric being implemented from the common NVMe fabrics library. |
601 | * Part of the overall exit sequence of unloading the implemented driver. |
602 | */ |
603 | void nvmf_unregister_transport(struct nvmf_transport_ops *ops) |
604 | { |
605 | down_write(sem: &nvmf_transports_rwsem); |
606 | list_del(entry: &ops->entry); |
607 | up_write(sem: &nvmf_transports_rwsem); |
608 | } |
609 | EXPORT_SYMBOL_GPL(nvmf_unregister_transport); |
610 | |
611 | static struct nvmf_transport_ops *nvmf_lookup_transport( |
612 | struct nvmf_ctrl_options *opts) |
613 | { |
614 | struct nvmf_transport_ops *ops; |
615 | |
616 | lockdep_assert_held(&nvmf_transports_rwsem); |
617 | |
618 | list_for_each_entry(ops, &nvmf_transports, entry) { |
619 | if (strcmp(ops->name, opts->transport) == 0) |
620 | return ops; |
621 | } |
622 | |
623 | return NULL; |
624 | } |
625 | |
626 | static struct key *nvmf_parse_key(int key_id) |
627 | { |
628 | struct key *key; |
629 | |
630 | if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) { |
631 | pr_err("TLS is not supported\n" ); |
632 | return ERR_PTR(error: -EINVAL); |
633 | } |
634 | |
635 | key = key_lookup(id: key_id); |
636 | if (!IS_ERR(ptr: key)) |
637 | pr_err("key id %08x not found\n" , key_id); |
638 | else |
639 | pr_debug("Using key id %08x\n" , key_id); |
640 | return key; |
641 | } |
642 | |
643 | static const match_table_t opt_tokens = { |
644 | { NVMF_OPT_TRANSPORT, "transport=%s" }, |
645 | { NVMF_OPT_TRADDR, "traddr=%s" }, |
646 | { NVMF_OPT_TRSVCID, "trsvcid=%s" }, |
647 | { NVMF_OPT_NQN, "nqn=%s" }, |
648 | { NVMF_OPT_QUEUE_SIZE, "queue_size=%d" }, |
649 | { NVMF_OPT_NR_IO_QUEUES, "nr_io_queues=%d" }, |
650 | { NVMF_OPT_RECONNECT_DELAY, "reconnect_delay=%d" }, |
651 | { NVMF_OPT_CTRL_LOSS_TMO, "ctrl_loss_tmo=%d" }, |
652 | { NVMF_OPT_KATO, "keep_alive_tmo=%d" }, |
653 | { NVMF_OPT_HOSTNQN, "hostnqn=%s" }, |
654 | { NVMF_OPT_HOST_TRADDR, "host_traddr=%s" }, |
655 | { NVMF_OPT_HOST_IFACE, "host_iface=%s" }, |
656 | { NVMF_OPT_HOST_ID, "hostid=%s" }, |
657 | { NVMF_OPT_DUP_CONNECT, "duplicate_connect" }, |
658 | { NVMF_OPT_DISABLE_SQFLOW, "disable_sqflow" }, |
659 | { NVMF_OPT_HDR_DIGEST, "hdr_digest" }, |
660 | { NVMF_OPT_DATA_DIGEST, "data_digest" }, |
661 | { NVMF_OPT_NR_WRITE_QUEUES, "nr_write_queues=%d" }, |
662 | { NVMF_OPT_NR_POLL_QUEUES, "nr_poll_queues=%d" }, |
663 | { NVMF_OPT_TOS, "tos=%d" }, |
664 | #ifdef CONFIG_NVME_TCP_TLS |
665 | { NVMF_OPT_KEYRING, "keyring=%d" }, |
666 | { NVMF_OPT_TLS_KEY, "tls_key=%d" }, |
667 | #endif |
668 | { NVMF_OPT_FAIL_FAST_TMO, "fast_io_fail_tmo=%d" }, |
669 | { NVMF_OPT_DISCOVERY, "discovery" }, |
670 | { NVMF_OPT_DHCHAP_SECRET, "dhchap_secret=%s" }, |
671 | { NVMF_OPT_DHCHAP_CTRL_SECRET, "dhchap_ctrl_secret=%s" }, |
672 | #ifdef CONFIG_NVME_TCP_TLS |
673 | { NVMF_OPT_TLS, "tls" }, |
674 | #endif |
675 | { NVMF_OPT_ERR, NULL } |
676 | }; |
677 | |
678 | static int nvmf_parse_options(struct nvmf_ctrl_options *opts, |
679 | const char *buf) |
680 | { |
681 | substring_t args[MAX_OPT_ARGS]; |
682 | char *options, *o, *p; |
683 | int token, ret = 0; |
684 | size_t nqnlen = 0; |
685 | int ctrl_loss_tmo = NVMF_DEF_CTRL_LOSS_TMO, key_id; |
686 | uuid_t hostid; |
687 | char hostnqn[NVMF_NQN_SIZE]; |
688 | struct key *key; |
689 | |
690 | /* Set defaults */ |
691 | opts->queue_size = NVMF_DEF_QUEUE_SIZE; |
692 | opts->nr_io_queues = num_online_cpus(); |
693 | opts->reconnect_delay = NVMF_DEF_RECONNECT_DELAY; |
694 | opts->kato = 0; |
695 | opts->duplicate_connect = false; |
696 | opts->fast_io_fail_tmo = NVMF_DEF_FAIL_FAST_TMO; |
697 | opts->hdr_digest = false; |
698 | opts->data_digest = false; |
699 | opts->tos = -1; /* < 0 == use transport default */ |
700 | opts->tls = false; |
701 | opts->tls_key = NULL; |
702 | opts->keyring = NULL; |
703 | |
704 | options = o = kstrdup(s: buf, GFP_KERNEL); |
705 | if (!options) |
706 | return -ENOMEM; |
707 | |
708 | /* use default host if not given by user space */ |
709 | uuid_copy(dst: &hostid, src: &nvmf_default_host->id); |
710 | strscpy(p: hostnqn, q: nvmf_default_host->nqn, NVMF_NQN_SIZE); |
711 | |
712 | while ((p = strsep(&o, ",\n" )) != NULL) { |
713 | if (!*p) |
714 | continue; |
715 | |
716 | token = match_token(p, table: opt_tokens, args); |
717 | opts->mask |= token; |
718 | switch (token) { |
719 | case NVMF_OPT_TRANSPORT: |
720 | p = match_strdup(args); |
721 | if (!p) { |
722 | ret = -ENOMEM; |
723 | goto out; |
724 | } |
725 | kfree(objp: opts->transport); |
726 | opts->transport = p; |
727 | break; |
728 | case NVMF_OPT_NQN: |
729 | p = match_strdup(args); |
730 | if (!p) { |
731 | ret = -ENOMEM; |
732 | goto out; |
733 | } |
734 | kfree(objp: opts->subsysnqn); |
735 | opts->subsysnqn = p; |
736 | nqnlen = strlen(opts->subsysnqn); |
737 | if (nqnlen >= NVMF_NQN_SIZE) { |
738 | pr_err("%s needs to be < %d bytes\n" , |
739 | opts->subsysnqn, NVMF_NQN_SIZE); |
740 | ret = -EINVAL; |
741 | goto out; |
742 | } |
743 | opts->discovery_nqn = |
744 | !(strcmp(opts->subsysnqn, |
745 | NVME_DISC_SUBSYS_NAME)); |
746 | break; |
747 | case NVMF_OPT_TRADDR: |
748 | p = match_strdup(args); |
749 | if (!p) { |
750 | ret = -ENOMEM; |
751 | goto out; |
752 | } |
753 | kfree(objp: opts->traddr); |
754 | opts->traddr = p; |
755 | break; |
756 | case NVMF_OPT_TRSVCID: |
757 | p = match_strdup(args); |
758 | if (!p) { |
759 | ret = -ENOMEM; |
760 | goto out; |
761 | } |
762 | kfree(objp: opts->trsvcid); |
763 | opts->trsvcid = p; |
764 | break; |
765 | case NVMF_OPT_QUEUE_SIZE: |
766 | if (match_int(args, result: &token)) { |
767 | ret = -EINVAL; |
768 | goto out; |
769 | } |
770 | if (token < NVMF_MIN_QUEUE_SIZE || |
771 | token > NVMF_MAX_QUEUE_SIZE) { |
772 | pr_err("Invalid queue_size %d\n" , token); |
773 | ret = -EINVAL; |
774 | goto out; |
775 | } |
776 | opts->queue_size = token; |
777 | break; |
778 | case NVMF_OPT_NR_IO_QUEUES: |
779 | if (match_int(args, result: &token)) { |
780 | ret = -EINVAL; |
781 | goto out; |
782 | } |
783 | if (token <= 0) { |
784 | pr_err("Invalid number of IOQs %d\n" , token); |
785 | ret = -EINVAL; |
786 | goto out; |
787 | } |
788 | if (opts->discovery_nqn) { |
789 | pr_debug("Ignoring nr_io_queues value for discovery controller\n" ); |
790 | break; |
791 | } |
792 | |
793 | opts->nr_io_queues = min_t(unsigned int, |
794 | num_online_cpus(), token); |
795 | break; |
796 | case NVMF_OPT_KATO: |
797 | if (match_int(args, result: &token)) { |
798 | ret = -EINVAL; |
799 | goto out; |
800 | } |
801 | |
802 | if (token < 0) { |
803 | pr_err("Invalid keep_alive_tmo %d\n" , token); |
804 | ret = -EINVAL; |
805 | goto out; |
806 | } else if (token == 0 && !opts->discovery_nqn) { |
807 | /* Allowed for debug */ |
808 | pr_warn("keep_alive_tmo 0 won't execute keep alives!!!\n" ); |
809 | } |
810 | opts->kato = token; |
811 | break; |
812 | case NVMF_OPT_CTRL_LOSS_TMO: |
813 | if (match_int(args, result: &token)) { |
814 | ret = -EINVAL; |
815 | goto out; |
816 | } |
817 | |
818 | if (token < 0) |
819 | pr_warn("ctrl_loss_tmo < 0 will reconnect forever\n" ); |
820 | ctrl_loss_tmo = token; |
821 | break; |
822 | case NVMF_OPT_FAIL_FAST_TMO: |
823 | if (match_int(args, result: &token)) { |
824 | ret = -EINVAL; |
825 | goto out; |
826 | } |
827 | |
828 | if (token >= 0) |
829 | pr_warn("I/O fail on reconnect controller after %d sec\n" , |
830 | token); |
831 | else |
832 | token = -1; |
833 | |
834 | opts->fast_io_fail_tmo = token; |
835 | break; |
836 | case NVMF_OPT_HOSTNQN: |
837 | if (opts->host) { |
838 | pr_err("hostnqn already user-assigned: %s\n" , |
839 | opts->host->nqn); |
840 | ret = -EADDRINUSE; |
841 | goto out; |
842 | } |
843 | p = match_strdup(args); |
844 | if (!p) { |
845 | ret = -ENOMEM; |
846 | goto out; |
847 | } |
848 | nqnlen = strlen(p); |
849 | if (nqnlen >= NVMF_NQN_SIZE) { |
850 | pr_err("%s needs to be < %d bytes\n" , |
851 | p, NVMF_NQN_SIZE); |
852 | kfree(objp: p); |
853 | ret = -EINVAL; |
854 | goto out; |
855 | } |
856 | strscpy(p: hostnqn, q: p, NVMF_NQN_SIZE); |
857 | kfree(objp: p); |
858 | break; |
859 | case NVMF_OPT_RECONNECT_DELAY: |
860 | if (match_int(args, result: &token)) { |
861 | ret = -EINVAL; |
862 | goto out; |
863 | } |
864 | if (token <= 0) { |
865 | pr_err("Invalid reconnect_delay %d\n" , token); |
866 | ret = -EINVAL; |
867 | goto out; |
868 | } |
869 | opts->reconnect_delay = token; |
870 | break; |
871 | case NVMF_OPT_HOST_TRADDR: |
872 | p = match_strdup(args); |
873 | if (!p) { |
874 | ret = -ENOMEM; |
875 | goto out; |
876 | } |
877 | kfree(objp: opts->host_traddr); |
878 | opts->host_traddr = p; |
879 | break; |
880 | case NVMF_OPT_HOST_IFACE: |
881 | p = match_strdup(args); |
882 | if (!p) { |
883 | ret = -ENOMEM; |
884 | goto out; |
885 | } |
886 | kfree(objp: opts->host_iface); |
887 | opts->host_iface = p; |
888 | break; |
889 | case NVMF_OPT_HOST_ID: |
890 | p = match_strdup(args); |
891 | if (!p) { |
892 | ret = -ENOMEM; |
893 | goto out; |
894 | } |
895 | ret = uuid_parse(uuid: p, u: &hostid); |
896 | if (ret) { |
897 | pr_err("Invalid hostid %s\n" , p); |
898 | ret = -EINVAL; |
899 | kfree(objp: p); |
900 | goto out; |
901 | } |
902 | kfree(objp: p); |
903 | break; |
904 | case NVMF_OPT_DUP_CONNECT: |
905 | opts->duplicate_connect = true; |
906 | break; |
907 | case NVMF_OPT_DISABLE_SQFLOW: |
908 | opts->disable_sqflow = true; |
909 | break; |
910 | case NVMF_OPT_HDR_DIGEST: |
911 | opts->hdr_digest = true; |
912 | break; |
913 | case NVMF_OPT_DATA_DIGEST: |
914 | opts->data_digest = true; |
915 | break; |
916 | case NVMF_OPT_NR_WRITE_QUEUES: |
917 | if (match_int(args, result: &token)) { |
918 | ret = -EINVAL; |
919 | goto out; |
920 | } |
921 | if (token <= 0) { |
922 | pr_err("Invalid nr_write_queues %d\n" , token); |
923 | ret = -EINVAL; |
924 | goto out; |
925 | } |
926 | opts->nr_write_queues = token; |
927 | break; |
928 | case NVMF_OPT_NR_POLL_QUEUES: |
929 | if (match_int(args, result: &token)) { |
930 | ret = -EINVAL; |
931 | goto out; |
932 | } |
933 | if (token <= 0) { |
934 | pr_err("Invalid nr_poll_queues %d\n" , token); |
935 | ret = -EINVAL; |
936 | goto out; |
937 | } |
938 | opts->nr_poll_queues = token; |
939 | break; |
940 | case NVMF_OPT_TOS: |
941 | if (match_int(args, result: &token)) { |
942 | ret = -EINVAL; |
943 | goto out; |
944 | } |
945 | if (token < 0) { |
946 | pr_err("Invalid type of service %d\n" , token); |
947 | ret = -EINVAL; |
948 | goto out; |
949 | } |
950 | if (token > 255) { |
951 | pr_warn("Clamping type of service to 255\n" ); |
952 | token = 255; |
953 | } |
954 | opts->tos = token; |
955 | break; |
956 | case NVMF_OPT_KEYRING: |
957 | if (match_int(args, result: &key_id) || key_id <= 0) { |
958 | ret = -EINVAL; |
959 | goto out; |
960 | } |
961 | key = nvmf_parse_key(key_id); |
962 | if (IS_ERR(ptr: key)) { |
963 | ret = PTR_ERR(ptr: key); |
964 | goto out; |
965 | } |
966 | key_put(key: opts->keyring); |
967 | opts->keyring = key; |
968 | break; |
969 | case NVMF_OPT_TLS_KEY: |
970 | if (match_int(args, result: &key_id) || key_id <= 0) { |
971 | ret = -EINVAL; |
972 | goto out; |
973 | } |
974 | key = nvmf_parse_key(key_id); |
975 | if (IS_ERR(ptr: key)) { |
976 | ret = PTR_ERR(ptr: key); |
977 | goto out; |
978 | } |
979 | key_put(key: opts->tls_key); |
980 | opts->tls_key = key; |
981 | break; |
982 | case NVMF_OPT_DISCOVERY: |
983 | opts->discovery_nqn = true; |
984 | break; |
985 | case NVMF_OPT_DHCHAP_SECRET: |
986 | p = match_strdup(args); |
987 | if (!p) { |
988 | ret = -ENOMEM; |
989 | goto out; |
990 | } |
991 | if (strlen(p) < 11 || strncmp(p, "DHHC-1:" , 7)) { |
992 | pr_err("Invalid DH-CHAP secret %s\n" , p); |
993 | ret = -EINVAL; |
994 | goto out; |
995 | } |
996 | kfree(objp: opts->dhchap_secret); |
997 | opts->dhchap_secret = p; |
998 | break; |
999 | case NVMF_OPT_DHCHAP_CTRL_SECRET: |
1000 | p = match_strdup(args); |
1001 | if (!p) { |
1002 | ret = -ENOMEM; |
1003 | goto out; |
1004 | } |
1005 | if (strlen(p) < 11 || strncmp(p, "DHHC-1:" , 7)) { |
1006 | pr_err("Invalid DH-CHAP secret %s\n" , p); |
1007 | ret = -EINVAL; |
1008 | goto out; |
1009 | } |
1010 | kfree(objp: opts->dhchap_ctrl_secret); |
1011 | opts->dhchap_ctrl_secret = p; |
1012 | break; |
1013 | case NVMF_OPT_TLS: |
1014 | if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) { |
1015 | pr_err("TLS is not supported\n" ); |
1016 | ret = -EINVAL; |
1017 | goto out; |
1018 | } |
1019 | opts->tls = true; |
1020 | break; |
1021 | default: |
1022 | pr_warn("unknown parameter or missing value '%s' in ctrl creation request\n" , |
1023 | p); |
1024 | ret = -EINVAL; |
1025 | goto out; |
1026 | } |
1027 | } |
1028 | |
1029 | if (opts->discovery_nqn) { |
1030 | opts->nr_io_queues = 0; |
1031 | opts->nr_write_queues = 0; |
1032 | opts->nr_poll_queues = 0; |
1033 | opts->duplicate_connect = true; |
1034 | } else { |
1035 | if (!opts->kato) |
1036 | opts->kato = NVME_DEFAULT_KATO; |
1037 | } |
1038 | if (ctrl_loss_tmo < 0) { |
1039 | opts->max_reconnects = -1; |
1040 | } else { |
1041 | opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo, |
1042 | opts->reconnect_delay); |
1043 | if (ctrl_loss_tmo < opts->fast_io_fail_tmo) |
1044 | pr_warn("failfast tmo (%d) larger than controller loss tmo (%d)\n" , |
1045 | opts->fast_io_fail_tmo, ctrl_loss_tmo); |
1046 | } |
1047 | |
1048 | opts->host = nvmf_host_add(hostnqn, id: &hostid); |
1049 | if (IS_ERR(ptr: opts->host)) { |
1050 | ret = PTR_ERR(ptr: opts->host); |
1051 | opts->host = NULL; |
1052 | goto out; |
1053 | } |
1054 | |
1055 | out: |
1056 | kfree(objp: options); |
1057 | return ret; |
1058 | } |
1059 | |
1060 | void nvmf_set_io_queues(struct nvmf_ctrl_options *opts, u32 nr_io_queues, |
1061 | u32 io_queues[HCTX_MAX_TYPES]) |
1062 | { |
1063 | if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) { |
1064 | /* |
1065 | * separate read/write queues |
1066 | * hand out dedicated default queues only after we have |
1067 | * sufficient read queues. |
1068 | */ |
1069 | io_queues[HCTX_TYPE_READ] = opts->nr_io_queues; |
1070 | nr_io_queues -= io_queues[HCTX_TYPE_READ]; |
1071 | io_queues[HCTX_TYPE_DEFAULT] = |
1072 | min(opts->nr_write_queues, nr_io_queues); |
1073 | nr_io_queues -= io_queues[HCTX_TYPE_DEFAULT]; |
1074 | } else { |
1075 | /* |
1076 | * shared read/write queues |
1077 | * either no write queues were requested, or we don't have |
1078 | * sufficient queue count to have dedicated default queues. |
1079 | */ |
1080 | io_queues[HCTX_TYPE_DEFAULT] = |
1081 | min(opts->nr_io_queues, nr_io_queues); |
1082 | nr_io_queues -= io_queues[HCTX_TYPE_DEFAULT]; |
1083 | } |
1084 | |
1085 | if (opts->nr_poll_queues && nr_io_queues) { |
1086 | /* map dedicated poll queues only if we have queues left */ |
1087 | io_queues[HCTX_TYPE_POLL] = |
1088 | min(opts->nr_poll_queues, nr_io_queues); |
1089 | } |
1090 | } |
1091 | EXPORT_SYMBOL_GPL(nvmf_set_io_queues); |
1092 | |
1093 | void nvmf_map_queues(struct blk_mq_tag_set *set, struct nvme_ctrl *ctrl, |
1094 | u32 io_queues[HCTX_MAX_TYPES]) |
1095 | { |
1096 | struct nvmf_ctrl_options *opts = ctrl->opts; |
1097 | |
1098 | if (opts->nr_write_queues && io_queues[HCTX_TYPE_READ]) { |
1099 | /* separate read/write queues */ |
1100 | set->map[HCTX_TYPE_DEFAULT].nr_queues = |
1101 | io_queues[HCTX_TYPE_DEFAULT]; |
1102 | set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; |
1103 | set->map[HCTX_TYPE_READ].nr_queues = |
1104 | io_queues[HCTX_TYPE_READ]; |
1105 | set->map[HCTX_TYPE_READ].queue_offset = |
1106 | io_queues[HCTX_TYPE_DEFAULT]; |
1107 | } else { |
1108 | /* shared read/write queues */ |
1109 | set->map[HCTX_TYPE_DEFAULT].nr_queues = |
1110 | io_queues[HCTX_TYPE_DEFAULT]; |
1111 | set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; |
1112 | set->map[HCTX_TYPE_READ].nr_queues = |
1113 | io_queues[HCTX_TYPE_DEFAULT]; |
1114 | set->map[HCTX_TYPE_READ].queue_offset = 0; |
1115 | } |
1116 | |
1117 | blk_mq_map_queues(qmap: &set->map[HCTX_TYPE_DEFAULT]); |
1118 | blk_mq_map_queues(qmap: &set->map[HCTX_TYPE_READ]); |
1119 | if (opts->nr_poll_queues && io_queues[HCTX_TYPE_POLL]) { |
1120 | /* map dedicated poll queues only if we have queues left */ |
1121 | set->map[HCTX_TYPE_POLL].nr_queues = io_queues[HCTX_TYPE_POLL]; |
1122 | set->map[HCTX_TYPE_POLL].queue_offset = |
1123 | io_queues[HCTX_TYPE_DEFAULT] + |
1124 | io_queues[HCTX_TYPE_READ]; |
1125 | blk_mq_map_queues(qmap: &set->map[HCTX_TYPE_POLL]); |
1126 | } |
1127 | |
1128 | dev_info(ctrl->device, |
1129 | "mapped %d/%d/%d default/read/poll queues.\n" , |
1130 | io_queues[HCTX_TYPE_DEFAULT], |
1131 | io_queues[HCTX_TYPE_READ], |
1132 | io_queues[HCTX_TYPE_POLL]); |
1133 | } |
1134 | EXPORT_SYMBOL_GPL(nvmf_map_queues); |
1135 | |
1136 | static int nvmf_check_required_opts(struct nvmf_ctrl_options *opts, |
1137 | unsigned int required_opts) |
1138 | { |
1139 | if ((opts->mask & required_opts) != required_opts) { |
1140 | unsigned int i; |
1141 | |
1142 | for (i = 0; i < ARRAY_SIZE(opt_tokens); i++) { |
1143 | if ((opt_tokens[i].token & required_opts) && |
1144 | !(opt_tokens[i].token & opts->mask)) { |
1145 | pr_warn("missing parameter '%s'\n" , |
1146 | opt_tokens[i].pattern); |
1147 | } |
1148 | } |
1149 | |
1150 | return -EINVAL; |
1151 | } |
1152 | |
1153 | return 0; |
1154 | } |
1155 | |
1156 | bool nvmf_ip_options_match(struct nvme_ctrl *ctrl, |
1157 | struct nvmf_ctrl_options *opts) |
1158 | { |
1159 | if (!nvmf_ctlr_matches_baseopts(ctrl, opts) || |
1160 | strcmp(opts->traddr, ctrl->opts->traddr) || |
1161 | strcmp(opts->trsvcid, ctrl->opts->trsvcid)) |
1162 | return false; |
1163 | |
1164 | /* |
1165 | * Checking the local address or host interfaces is rough. |
1166 | * |
1167 | * In most cases, none is specified and the host port or |
1168 | * host interface is selected by the stack. |
1169 | * |
1170 | * Assume no match if: |
1171 | * - local address or host interface is specified and address |
1172 | * or host interface is not the same |
1173 | * - local address or host interface is not specified but |
1174 | * remote is, or vice versa (admin using specific |
1175 | * host_traddr/host_iface when it matters). |
1176 | */ |
1177 | if ((opts->mask & NVMF_OPT_HOST_TRADDR) && |
1178 | (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) { |
1179 | if (strcmp(opts->host_traddr, ctrl->opts->host_traddr)) |
1180 | return false; |
1181 | } else if ((opts->mask & NVMF_OPT_HOST_TRADDR) || |
1182 | (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) { |
1183 | return false; |
1184 | } |
1185 | |
1186 | if ((opts->mask & NVMF_OPT_HOST_IFACE) && |
1187 | (ctrl->opts->mask & NVMF_OPT_HOST_IFACE)) { |
1188 | if (strcmp(opts->host_iface, ctrl->opts->host_iface)) |
1189 | return false; |
1190 | } else if ((opts->mask & NVMF_OPT_HOST_IFACE) || |
1191 | (ctrl->opts->mask & NVMF_OPT_HOST_IFACE)) { |
1192 | return false; |
1193 | } |
1194 | |
1195 | return true; |
1196 | } |
1197 | EXPORT_SYMBOL_GPL(nvmf_ip_options_match); |
1198 | |
1199 | static int nvmf_check_allowed_opts(struct nvmf_ctrl_options *opts, |
1200 | unsigned int allowed_opts) |
1201 | { |
1202 | if (opts->mask & ~allowed_opts) { |
1203 | unsigned int i; |
1204 | |
1205 | for (i = 0; i < ARRAY_SIZE(opt_tokens); i++) { |
1206 | if ((opt_tokens[i].token & opts->mask) && |
1207 | (opt_tokens[i].token & ~allowed_opts)) { |
1208 | pr_warn("invalid parameter '%s'\n" , |
1209 | opt_tokens[i].pattern); |
1210 | } |
1211 | } |
1212 | |
1213 | return -EINVAL; |
1214 | } |
1215 | |
1216 | return 0; |
1217 | } |
1218 | |
1219 | void nvmf_free_options(struct nvmf_ctrl_options *opts) |
1220 | { |
1221 | nvmf_host_put(host: opts->host); |
1222 | key_put(key: opts->keyring); |
1223 | key_put(key: opts->tls_key); |
1224 | kfree(objp: opts->transport); |
1225 | kfree(objp: opts->traddr); |
1226 | kfree(objp: opts->trsvcid); |
1227 | kfree(objp: opts->subsysnqn); |
1228 | kfree(objp: opts->host_traddr); |
1229 | kfree(objp: opts->host_iface); |
1230 | kfree(objp: opts->dhchap_secret); |
1231 | kfree(objp: opts->dhchap_ctrl_secret); |
1232 | kfree(objp: opts); |
1233 | } |
1234 | EXPORT_SYMBOL_GPL(nvmf_free_options); |
1235 | |
1236 | #define NVMF_REQUIRED_OPTS (NVMF_OPT_TRANSPORT | NVMF_OPT_NQN) |
1237 | #define NVMF_ALLOWED_OPTS (NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \ |
1238 | NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \ |
1239 | NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\ |
1240 | NVMF_OPT_DISABLE_SQFLOW | NVMF_OPT_DISCOVERY |\ |
1241 | NVMF_OPT_FAIL_FAST_TMO | NVMF_OPT_DHCHAP_SECRET |\ |
1242 | NVMF_OPT_DHCHAP_CTRL_SECRET) |
1243 | |
1244 | static struct nvme_ctrl * |
1245 | nvmf_create_ctrl(struct device *dev, const char *buf) |
1246 | { |
1247 | struct nvmf_ctrl_options *opts; |
1248 | struct nvmf_transport_ops *ops; |
1249 | struct nvme_ctrl *ctrl; |
1250 | int ret; |
1251 | |
1252 | opts = kzalloc(size: sizeof(*opts), GFP_KERNEL); |
1253 | if (!opts) |
1254 | return ERR_PTR(error: -ENOMEM); |
1255 | |
1256 | ret = nvmf_parse_options(opts, buf); |
1257 | if (ret) |
1258 | goto out_free_opts; |
1259 | |
1260 | |
1261 | request_module("nvme-%s" , opts->transport); |
1262 | |
1263 | /* |
1264 | * Check the generic options first as we need a valid transport for |
1265 | * the lookup below. Then clear the generic flags so that transport |
1266 | * drivers don't have to care about them. |
1267 | */ |
1268 | ret = nvmf_check_required_opts(opts, NVMF_REQUIRED_OPTS); |
1269 | if (ret) |
1270 | goto out_free_opts; |
1271 | opts->mask &= ~NVMF_REQUIRED_OPTS; |
1272 | |
1273 | down_read(sem: &nvmf_transports_rwsem); |
1274 | ops = nvmf_lookup_transport(opts); |
1275 | if (!ops) { |
1276 | pr_info("no handler found for transport %s.\n" , |
1277 | opts->transport); |
1278 | ret = -EINVAL; |
1279 | goto out_unlock; |
1280 | } |
1281 | |
1282 | if (!try_module_get(module: ops->module)) { |
1283 | ret = -EBUSY; |
1284 | goto out_unlock; |
1285 | } |
1286 | up_read(sem: &nvmf_transports_rwsem); |
1287 | |
1288 | ret = nvmf_check_required_opts(opts, required_opts: ops->required_opts); |
1289 | if (ret) |
1290 | goto out_module_put; |
1291 | ret = nvmf_check_allowed_opts(opts, NVMF_ALLOWED_OPTS | |
1292 | ops->allowed_opts | ops->required_opts); |
1293 | if (ret) |
1294 | goto out_module_put; |
1295 | |
1296 | ctrl = ops->create_ctrl(dev, opts); |
1297 | if (IS_ERR(ptr: ctrl)) { |
1298 | ret = PTR_ERR(ptr: ctrl); |
1299 | goto out_module_put; |
1300 | } |
1301 | |
1302 | module_put(module: ops->module); |
1303 | return ctrl; |
1304 | |
1305 | out_module_put: |
1306 | module_put(module: ops->module); |
1307 | goto out_free_opts; |
1308 | out_unlock: |
1309 | up_read(sem: &nvmf_transports_rwsem); |
1310 | out_free_opts: |
1311 | nvmf_free_options(opts); |
1312 | return ERR_PTR(error: ret); |
1313 | } |
1314 | |
1315 | static struct class *nvmf_class; |
1316 | static struct device *nvmf_device; |
1317 | static DEFINE_MUTEX(nvmf_dev_mutex); |
1318 | |
1319 | static ssize_t nvmf_dev_write(struct file *file, const char __user *ubuf, |
1320 | size_t count, loff_t *pos) |
1321 | { |
1322 | struct seq_file *seq_file = file->private_data; |
1323 | struct nvme_ctrl *ctrl; |
1324 | const char *buf; |
1325 | int ret = 0; |
1326 | |
1327 | if (count > PAGE_SIZE) |
1328 | return -ENOMEM; |
1329 | |
1330 | buf = memdup_user_nul(ubuf, count); |
1331 | if (IS_ERR(ptr: buf)) |
1332 | return PTR_ERR(ptr: buf); |
1333 | |
1334 | mutex_lock(&nvmf_dev_mutex); |
1335 | if (seq_file->private) { |
1336 | ret = -EINVAL; |
1337 | goto out_unlock; |
1338 | } |
1339 | |
1340 | ctrl = nvmf_create_ctrl(dev: nvmf_device, buf); |
1341 | if (IS_ERR(ptr: ctrl)) { |
1342 | ret = PTR_ERR(ptr: ctrl); |
1343 | goto out_unlock; |
1344 | } |
1345 | |
1346 | seq_file->private = ctrl; |
1347 | |
1348 | out_unlock: |
1349 | mutex_unlock(lock: &nvmf_dev_mutex); |
1350 | kfree(objp: buf); |
1351 | return ret ? ret : count; |
1352 | } |
1353 | |
1354 | static void __nvmf_concat_opt_tokens(struct seq_file *seq_file) |
1355 | { |
1356 | const struct match_token *tok; |
1357 | int idx; |
1358 | |
1359 | /* |
1360 | * Add dummy entries for instance and cntlid to |
1361 | * signal an invalid/non-existing controller |
1362 | */ |
1363 | seq_puts(m: seq_file, s: "instance=-1,cntlid=-1" ); |
1364 | for (idx = 0; idx < ARRAY_SIZE(opt_tokens); idx++) { |
1365 | tok = &opt_tokens[idx]; |
1366 | if (tok->token == NVMF_OPT_ERR) |
1367 | continue; |
1368 | seq_puts(m: seq_file, s: "," ); |
1369 | seq_puts(m: seq_file, s: tok->pattern); |
1370 | } |
1371 | seq_puts(m: seq_file, s: "\n" ); |
1372 | } |
1373 | |
1374 | static int nvmf_dev_show(struct seq_file *seq_file, void *private) |
1375 | { |
1376 | struct nvme_ctrl *ctrl; |
1377 | |
1378 | mutex_lock(&nvmf_dev_mutex); |
1379 | ctrl = seq_file->private; |
1380 | if (!ctrl) { |
1381 | __nvmf_concat_opt_tokens(seq_file); |
1382 | goto out_unlock; |
1383 | } |
1384 | |
1385 | seq_printf(m: seq_file, fmt: "instance=%d,cntlid=%d\n" , |
1386 | ctrl->instance, ctrl->cntlid); |
1387 | |
1388 | out_unlock: |
1389 | mutex_unlock(lock: &nvmf_dev_mutex); |
1390 | return 0; |
1391 | } |
1392 | |
1393 | static int nvmf_dev_open(struct inode *inode, struct file *file) |
1394 | { |
1395 | /* |
1396 | * The miscdevice code initializes file->private_data, but doesn't |
1397 | * make use of it later. |
1398 | */ |
1399 | file->private_data = NULL; |
1400 | return single_open(file, nvmf_dev_show, NULL); |
1401 | } |
1402 | |
1403 | static int nvmf_dev_release(struct inode *inode, struct file *file) |
1404 | { |
1405 | struct seq_file *seq_file = file->private_data; |
1406 | struct nvme_ctrl *ctrl = seq_file->private; |
1407 | |
1408 | if (ctrl) |
1409 | nvme_put_ctrl(ctrl); |
1410 | return single_release(inode, file); |
1411 | } |
1412 | |
1413 | static const struct file_operations nvmf_dev_fops = { |
1414 | .owner = THIS_MODULE, |
1415 | .write = nvmf_dev_write, |
1416 | .read = seq_read, |
1417 | .open = nvmf_dev_open, |
1418 | .release = nvmf_dev_release, |
1419 | }; |
1420 | |
1421 | static struct miscdevice nvmf_misc = { |
1422 | .minor = MISC_DYNAMIC_MINOR, |
1423 | .name = "nvme-fabrics" , |
1424 | .fops = &nvmf_dev_fops, |
1425 | }; |
1426 | |
1427 | static int __init nvmf_init(void) |
1428 | { |
1429 | int ret; |
1430 | |
1431 | nvmf_default_host = nvmf_host_default(); |
1432 | if (!nvmf_default_host) |
1433 | return -ENOMEM; |
1434 | |
1435 | nvmf_class = class_create(name: "nvme-fabrics" ); |
1436 | if (IS_ERR(ptr: nvmf_class)) { |
1437 | pr_err("couldn't register class nvme-fabrics\n" ); |
1438 | ret = PTR_ERR(ptr: nvmf_class); |
1439 | goto out_free_host; |
1440 | } |
1441 | |
1442 | nvmf_device = |
1443 | device_create(cls: nvmf_class, NULL, MKDEV(0, 0), NULL, fmt: "ctl" ); |
1444 | if (IS_ERR(ptr: nvmf_device)) { |
1445 | pr_err("couldn't create nvme-fabrics device!\n" ); |
1446 | ret = PTR_ERR(ptr: nvmf_device); |
1447 | goto out_destroy_class; |
1448 | } |
1449 | |
1450 | ret = misc_register(misc: &nvmf_misc); |
1451 | if (ret) { |
1452 | pr_err("couldn't register misc device: %d\n" , ret); |
1453 | goto out_destroy_device; |
1454 | } |
1455 | |
1456 | return 0; |
1457 | |
1458 | out_destroy_device: |
1459 | device_destroy(cls: nvmf_class, MKDEV(0, 0)); |
1460 | out_destroy_class: |
1461 | class_destroy(cls: nvmf_class); |
1462 | out_free_host: |
1463 | nvmf_host_put(host: nvmf_default_host); |
1464 | return ret; |
1465 | } |
1466 | |
1467 | static void __exit nvmf_exit(void) |
1468 | { |
1469 | misc_deregister(misc: &nvmf_misc); |
1470 | device_destroy(cls: nvmf_class, MKDEV(0, 0)); |
1471 | class_destroy(cls: nvmf_class); |
1472 | nvmf_host_put(host: nvmf_default_host); |
1473 | |
1474 | BUILD_BUG_ON(sizeof(struct nvmf_common_command) != 64); |
1475 | BUILD_BUG_ON(sizeof(struct nvmf_connect_command) != 64); |
1476 | BUILD_BUG_ON(sizeof(struct nvmf_property_get_command) != 64); |
1477 | BUILD_BUG_ON(sizeof(struct nvmf_property_set_command) != 64); |
1478 | BUILD_BUG_ON(sizeof(struct nvmf_auth_send_command) != 64); |
1479 | BUILD_BUG_ON(sizeof(struct nvmf_auth_receive_command) != 64); |
1480 | BUILD_BUG_ON(sizeof(struct nvmf_connect_data) != 1024); |
1481 | BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_negotiate_data) != 8); |
1482 | BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_challenge_data) != 16); |
1483 | BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_reply_data) != 16); |
1484 | BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_success1_data) != 16); |
1485 | BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_success2_data) != 16); |
1486 | } |
1487 | |
1488 | MODULE_LICENSE("GPL v2" ); |
1489 | |
1490 | module_init(nvmf_init); |
1491 | module_exit(nvmf_exit); |
1492 | |