1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2020 Hannes Reinecke, SUSE Linux |
4 | */ |
5 | |
6 | #include <linux/crc32.h> |
7 | #include <linux/base64.h> |
8 | #include <linux/prandom.h> |
9 | #include <asm/unaligned.h> |
10 | #include <crypto/hash.h> |
11 | #include <crypto/dh.h> |
12 | #include "nvme.h" |
13 | #include "fabrics.h" |
14 | #include <linux/nvme-auth.h> |
15 | |
16 | #define CHAP_BUF_SIZE 4096 |
17 | static struct kmem_cache *nvme_chap_buf_cache; |
18 | static mempool_t *nvme_chap_buf_pool; |
19 | |
20 | struct nvme_dhchap_queue_context { |
21 | struct list_head entry; |
22 | struct work_struct auth_work; |
23 | struct nvme_ctrl *ctrl; |
24 | struct crypto_shash *shash_tfm; |
25 | struct crypto_kpp *dh_tfm; |
26 | struct nvme_dhchap_key *transformed_key; |
27 | void *buf; |
28 | int qid; |
29 | int error; |
30 | u32 s1; |
31 | u32 s2; |
32 | bool bi_directional; |
33 | u16 transaction; |
34 | u8 status; |
35 | u8 dhgroup_id; |
36 | u8 hash_id; |
37 | size_t hash_len; |
38 | u8 c1[64]; |
39 | u8 c2[64]; |
40 | u8 response[64]; |
41 | u8 *ctrl_key; |
42 | u8 *host_key; |
43 | u8 *sess_key; |
44 | int ctrl_key_len; |
45 | int host_key_len; |
46 | int sess_key_len; |
47 | }; |
48 | |
49 | static struct workqueue_struct *nvme_auth_wq; |
50 | |
51 | static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl) |
52 | { |
53 | return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues + |
54 | ctrl->opts->nr_poll_queues + 1; |
55 | } |
56 | |
57 | static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid, |
58 | void *data, size_t data_len, bool auth_send) |
59 | { |
60 | struct nvme_command cmd = {}; |
61 | nvme_submit_flags_t flags = NVME_SUBMIT_RETRY; |
62 | struct request_queue *q = ctrl->fabrics_q; |
63 | int ret; |
64 | |
65 | if (qid != 0) { |
66 | flags |= NVME_SUBMIT_NOWAIT | NVME_SUBMIT_RESERVED; |
67 | q = ctrl->connect_q; |
68 | } |
69 | |
70 | cmd.auth_common.opcode = nvme_fabrics_command; |
71 | cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER; |
72 | cmd.auth_common.spsp0 = 0x01; |
73 | cmd.auth_common.spsp1 = 0x01; |
74 | if (auth_send) { |
75 | cmd.auth_send.fctype = nvme_fabrics_type_auth_send; |
76 | cmd.auth_send.tl = cpu_to_le32(data_len); |
77 | } else { |
78 | cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive; |
79 | cmd.auth_receive.al = cpu_to_le32(data_len); |
80 | } |
81 | |
82 | ret = __nvme_submit_sync_cmd(q, cmd: &cmd, NULL, buffer: data, bufflen: data_len, |
83 | qid: qid == 0 ? NVME_QID_ANY : qid, flags); |
84 | if (ret > 0) |
85 | dev_warn(ctrl->device, |
86 | "qid %d auth_send failed with status %d\n" , qid, ret); |
87 | else if (ret < 0) |
88 | dev_err(ctrl->device, |
89 | "qid %d auth_send failed with error %d\n" , qid, ret); |
90 | return ret; |
91 | } |
92 | |
93 | static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid, |
94 | struct nvmf_auth_dhchap_failure_data *data, |
95 | u16 transaction, u8 expected_msg) |
96 | { |
97 | dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n" , |
98 | __func__, qid, data->auth_type, data->auth_id); |
99 | |
100 | if (data->auth_type == NVME_AUTH_COMMON_MESSAGES && |
101 | data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) { |
102 | return data->rescode_exp; |
103 | } |
104 | if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES || |
105 | data->auth_id != expected_msg) { |
106 | dev_warn(ctrl->device, |
107 | "qid %d invalid message %02x/%02x\n" , |
108 | qid, data->auth_type, data->auth_id); |
109 | return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; |
110 | } |
111 | if (le16_to_cpu(data->t_id) != transaction) { |
112 | dev_warn(ctrl->device, |
113 | "qid %d invalid transaction ID %d\n" , |
114 | qid, le16_to_cpu(data->t_id)); |
115 | return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; |
116 | } |
117 | return 0; |
118 | } |
119 | |
120 | static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl, |
121 | struct nvme_dhchap_queue_context *chap) |
122 | { |
123 | struct nvmf_auth_dhchap_negotiate_data *data = chap->buf; |
124 | size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol); |
125 | |
126 | if (size > CHAP_BUF_SIZE) { |
127 | chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; |
128 | return -EINVAL; |
129 | } |
130 | memset((u8 *)chap->buf, 0, size); |
131 | data->auth_type = NVME_AUTH_COMMON_MESSAGES; |
132 | data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; |
133 | data->t_id = cpu_to_le16(chap->transaction); |
134 | data->sc_c = 0; /* No secure channel concatenation */ |
135 | data->napd = 1; |
136 | data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID; |
137 | data->auth_protocol[0].dhchap.halen = 3; |
138 | data->auth_protocol[0].dhchap.dhlen = 6; |
139 | data->auth_protocol[0].dhchap.idlist[0] = NVME_AUTH_HASH_SHA256; |
140 | data->auth_protocol[0].dhchap.idlist[1] = NVME_AUTH_HASH_SHA384; |
141 | data->auth_protocol[0].dhchap.idlist[2] = NVME_AUTH_HASH_SHA512; |
142 | data->auth_protocol[0].dhchap.idlist[30] = NVME_AUTH_DHGROUP_NULL; |
143 | data->auth_protocol[0].dhchap.idlist[31] = NVME_AUTH_DHGROUP_2048; |
144 | data->auth_protocol[0].dhchap.idlist[32] = NVME_AUTH_DHGROUP_3072; |
145 | data->auth_protocol[0].dhchap.idlist[33] = NVME_AUTH_DHGROUP_4096; |
146 | data->auth_protocol[0].dhchap.idlist[34] = NVME_AUTH_DHGROUP_6144; |
147 | data->auth_protocol[0].dhchap.idlist[35] = NVME_AUTH_DHGROUP_8192; |
148 | |
149 | return size; |
150 | } |
151 | |
152 | static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl, |
153 | struct nvme_dhchap_queue_context *chap) |
154 | { |
155 | struct nvmf_auth_dhchap_challenge_data *data = chap->buf; |
156 | u16 dhvlen = le16_to_cpu(data->dhvlen); |
157 | size_t size = sizeof(*data) + data->hl + dhvlen; |
158 | const char *gid_name = nvme_auth_dhgroup_name(dhgroup_id: data->dhgid); |
159 | const char *hmac_name, *kpp_name; |
160 | |
161 | if (size > CHAP_BUF_SIZE) { |
162 | chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; |
163 | return -EINVAL; |
164 | } |
165 | |
166 | hmac_name = nvme_auth_hmac_name(hmac_id: data->hashid); |
167 | if (!hmac_name) { |
168 | dev_warn(ctrl->device, |
169 | "qid %d: invalid HASH ID %d\n" , |
170 | chap->qid, data->hashid); |
171 | chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; |
172 | return -EPROTO; |
173 | } |
174 | |
175 | if (chap->hash_id == data->hashid && chap->shash_tfm && |
176 | !strcmp(crypto_shash_alg_name(tfm: chap->shash_tfm), hmac_name) && |
177 | crypto_shash_digestsize(tfm: chap->shash_tfm) == data->hl) { |
178 | dev_dbg(ctrl->device, |
179 | "qid %d: reuse existing hash %s\n" , |
180 | chap->qid, hmac_name); |
181 | goto select_kpp; |
182 | } |
183 | |
184 | /* Reset if hash cannot be reused */ |
185 | if (chap->shash_tfm) { |
186 | crypto_free_shash(tfm: chap->shash_tfm); |
187 | chap->hash_id = 0; |
188 | chap->hash_len = 0; |
189 | } |
190 | chap->shash_tfm = crypto_alloc_shash(alg_name: hmac_name, type: 0, |
191 | CRYPTO_ALG_ALLOCATES_MEMORY); |
192 | if (IS_ERR(ptr: chap->shash_tfm)) { |
193 | dev_warn(ctrl->device, |
194 | "qid %d: failed to allocate hash %s, error %ld\n" , |
195 | chap->qid, hmac_name, PTR_ERR(chap->shash_tfm)); |
196 | chap->shash_tfm = NULL; |
197 | chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; |
198 | return -ENOMEM; |
199 | } |
200 | |
201 | if (crypto_shash_digestsize(tfm: chap->shash_tfm) != data->hl) { |
202 | dev_warn(ctrl->device, |
203 | "qid %d: invalid hash length %d\n" , |
204 | chap->qid, data->hl); |
205 | crypto_free_shash(tfm: chap->shash_tfm); |
206 | chap->shash_tfm = NULL; |
207 | chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; |
208 | return -EPROTO; |
209 | } |
210 | |
211 | chap->hash_id = data->hashid; |
212 | chap->hash_len = data->hl; |
213 | dev_dbg(ctrl->device, "qid %d: selected hash %s\n" , |
214 | chap->qid, hmac_name); |
215 | |
216 | select_kpp: |
217 | kpp_name = nvme_auth_dhgroup_kpp(dhgroup_id: data->dhgid); |
218 | if (!kpp_name) { |
219 | dev_warn(ctrl->device, |
220 | "qid %d: invalid DH group id %d\n" , |
221 | chap->qid, data->dhgid); |
222 | chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; |
223 | /* Leave previous dh_tfm intact */ |
224 | return -EPROTO; |
225 | } |
226 | |
227 | if (chap->dhgroup_id == data->dhgid && |
228 | (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) { |
229 | dev_dbg(ctrl->device, |
230 | "qid %d: reuse existing DH group %s\n" , |
231 | chap->qid, gid_name); |
232 | goto skip_kpp; |
233 | } |
234 | |
235 | /* Reset dh_tfm if it can't be reused */ |
236 | if (chap->dh_tfm) { |
237 | crypto_free_kpp(tfm: chap->dh_tfm); |
238 | chap->dh_tfm = NULL; |
239 | } |
240 | |
241 | if (data->dhgid != NVME_AUTH_DHGROUP_NULL) { |
242 | if (dhvlen == 0) { |
243 | dev_warn(ctrl->device, |
244 | "qid %d: empty DH value\n" , |
245 | chap->qid); |
246 | chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; |
247 | return -EPROTO; |
248 | } |
249 | |
250 | chap->dh_tfm = crypto_alloc_kpp(alg_name: kpp_name, type: 0, mask: 0); |
251 | if (IS_ERR(ptr: chap->dh_tfm)) { |
252 | int ret = PTR_ERR(ptr: chap->dh_tfm); |
253 | |
254 | dev_warn(ctrl->device, |
255 | "qid %d: error %d initializing DH group %s\n" , |
256 | chap->qid, ret, gid_name); |
257 | chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; |
258 | chap->dh_tfm = NULL; |
259 | return ret; |
260 | } |
261 | dev_dbg(ctrl->device, "qid %d: selected DH group %s\n" , |
262 | chap->qid, gid_name); |
263 | } else if (dhvlen != 0) { |
264 | dev_warn(ctrl->device, |
265 | "qid %d: invalid DH value for NULL DH\n" , |
266 | chap->qid); |
267 | chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; |
268 | return -EPROTO; |
269 | } |
270 | chap->dhgroup_id = data->dhgid; |
271 | |
272 | skip_kpp: |
273 | chap->s1 = le32_to_cpu(data->seqnum); |
274 | memcpy(chap->c1, data->cval, chap->hash_len); |
275 | if (dhvlen) { |
276 | chap->ctrl_key = kmalloc(size: dhvlen, GFP_KERNEL); |
277 | if (!chap->ctrl_key) { |
278 | chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; |
279 | return -ENOMEM; |
280 | } |
281 | chap->ctrl_key_len = dhvlen; |
282 | memcpy(chap->ctrl_key, data->cval + chap->hash_len, |
283 | dhvlen); |
284 | dev_dbg(ctrl->device, "ctrl public key %*ph\n" , |
285 | (int)chap->ctrl_key_len, chap->ctrl_key); |
286 | } |
287 | |
288 | return 0; |
289 | } |
290 | |
291 | static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl, |
292 | struct nvme_dhchap_queue_context *chap) |
293 | { |
294 | struct nvmf_auth_dhchap_reply_data *data = chap->buf; |
295 | size_t size = sizeof(*data); |
296 | |
297 | size += 2 * chap->hash_len; |
298 | |
299 | if (chap->host_key_len) |
300 | size += chap->host_key_len; |
301 | |
302 | if (size > CHAP_BUF_SIZE) { |
303 | chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; |
304 | return -EINVAL; |
305 | } |
306 | |
307 | memset(chap->buf, 0, size); |
308 | data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; |
309 | data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY; |
310 | data->t_id = cpu_to_le16(chap->transaction); |
311 | data->hl = chap->hash_len; |
312 | data->dhvlen = cpu_to_le16(chap->host_key_len); |
313 | memcpy(data->rval, chap->response, chap->hash_len); |
314 | if (ctrl->ctrl_key) { |
315 | chap->bi_directional = true; |
316 | get_random_bytes(buf: chap->c2, len: chap->hash_len); |
317 | data->cvalid = 1; |
318 | memcpy(data->rval + chap->hash_len, chap->c2, |
319 | chap->hash_len); |
320 | dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n" , |
321 | __func__, chap->qid, (int)chap->hash_len, chap->c2); |
322 | } else { |
323 | memset(chap->c2, 0, chap->hash_len); |
324 | } |
325 | chap->s2 = nvme_auth_get_seqnum(); |
326 | data->seqnum = cpu_to_le32(chap->s2); |
327 | if (chap->host_key_len) { |
328 | dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n" , |
329 | __func__, chap->qid, |
330 | chap->host_key_len, chap->host_key); |
331 | memcpy(data->rval + 2 * chap->hash_len, chap->host_key, |
332 | chap->host_key_len); |
333 | } |
334 | |
335 | return size; |
336 | } |
337 | |
338 | static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl, |
339 | struct nvme_dhchap_queue_context *chap) |
340 | { |
341 | struct nvmf_auth_dhchap_success1_data *data = chap->buf; |
342 | size_t size = sizeof(*data) + chap->hash_len; |
343 | |
344 | if (size > CHAP_BUF_SIZE) { |
345 | chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; |
346 | return -EINVAL; |
347 | } |
348 | |
349 | if (data->hl != chap->hash_len) { |
350 | dev_warn(ctrl->device, |
351 | "qid %d: invalid hash length %u\n" , |
352 | chap->qid, data->hl); |
353 | chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; |
354 | return -EPROTO; |
355 | } |
356 | |
357 | /* Just print out information for the admin queue */ |
358 | if (chap->qid == 0) |
359 | dev_info(ctrl->device, |
360 | "qid 0: authenticated with hash %s dhgroup %s\n" , |
361 | nvme_auth_hmac_name(chap->hash_id), |
362 | nvme_auth_dhgroup_name(chap->dhgroup_id)); |
363 | |
364 | if (!data->rvalid) |
365 | return 0; |
366 | |
367 | /* Validate controller response */ |
368 | if (memcmp(p: chap->response, q: data->rval, size: data->hl)) { |
369 | dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n" , |
370 | __func__, chap->qid, (int)chap->hash_len, data->rval); |
371 | dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n" , |
372 | __func__, chap->qid, (int)chap->hash_len, |
373 | chap->response); |
374 | dev_warn(ctrl->device, |
375 | "qid %d: controller authentication failed\n" , |
376 | chap->qid); |
377 | chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; |
378 | return -ECONNREFUSED; |
379 | } |
380 | |
381 | /* Just print out information for the admin queue */ |
382 | if (chap->qid == 0) |
383 | dev_info(ctrl->device, |
384 | "qid 0: controller authenticated\n" ); |
385 | return 0; |
386 | } |
387 | |
388 | static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl, |
389 | struct nvme_dhchap_queue_context *chap) |
390 | { |
391 | struct nvmf_auth_dhchap_success2_data *data = chap->buf; |
392 | size_t size = sizeof(*data); |
393 | |
394 | memset(chap->buf, 0, size); |
395 | data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; |
396 | data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2; |
397 | data->t_id = cpu_to_le16(chap->transaction); |
398 | |
399 | return size; |
400 | } |
401 | |
402 | static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl, |
403 | struct nvme_dhchap_queue_context *chap) |
404 | { |
405 | struct nvmf_auth_dhchap_failure_data *data = chap->buf; |
406 | size_t size = sizeof(*data); |
407 | |
408 | memset(chap->buf, 0, size); |
409 | data->auth_type = NVME_AUTH_COMMON_MESSAGES; |
410 | data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2; |
411 | data->t_id = cpu_to_le16(chap->transaction); |
412 | data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED; |
413 | data->rescode_exp = chap->status; |
414 | |
415 | return size; |
416 | } |
417 | |
418 | static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl, |
419 | struct nvme_dhchap_queue_context *chap) |
420 | { |
421 | SHASH_DESC_ON_STACK(shash, chap->shash_tfm); |
422 | u8 buf[4], *challenge = chap->c1; |
423 | int ret; |
424 | |
425 | dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n" , |
426 | __func__, chap->qid, chap->s1, chap->transaction); |
427 | |
428 | if (!chap->transformed_key) { |
429 | chap->transformed_key = nvme_auth_transform_key(key: ctrl->host_key, |
430 | nqn: ctrl->opts->host->nqn); |
431 | if (IS_ERR(ptr: chap->transformed_key)) { |
432 | ret = PTR_ERR(ptr: chap->transformed_key); |
433 | chap->transformed_key = NULL; |
434 | return ret; |
435 | } |
436 | } else { |
437 | dev_dbg(ctrl->device, "%s: qid %d re-using host response\n" , |
438 | __func__, chap->qid); |
439 | } |
440 | |
441 | ret = crypto_shash_setkey(tfm: chap->shash_tfm, |
442 | key: chap->transformed_key->key, keylen: chap->transformed_key->len); |
443 | if (ret) { |
444 | dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n" , |
445 | chap->qid, ret); |
446 | goto out; |
447 | } |
448 | |
449 | if (chap->dh_tfm) { |
450 | challenge = kmalloc(size: chap->hash_len, GFP_KERNEL); |
451 | if (!challenge) { |
452 | ret = -ENOMEM; |
453 | goto out; |
454 | } |
455 | ret = nvme_auth_augmented_challenge(hmac_id: chap->hash_id, |
456 | skey: chap->sess_key, |
457 | skey_len: chap->sess_key_len, |
458 | challenge: chap->c1, aug: challenge, |
459 | hlen: chap->hash_len); |
460 | if (ret) |
461 | goto out; |
462 | } |
463 | |
464 | shash->tfm = chap->shash_tfm; |
465 | ret = crypto_shash_init(desc: shash); |
466 | if (ret) |
467 | goto out; |
468 | ret = crypto_shash_update(desc: shash, data: challenge, len: chap->hash_len); |
469 | if (ret) |
470 | goto out; |
471 | put_unaligned_le32(val: chap->s1, p: buf); |
472 | ret = crypto_shash_update(desc: shash, data: buf, len: 4); |
473 | if (ret) |
474 | goto out; |
475 | put_unaligned_le16(val: chap->transaction, p: buf); |
476 | ret = crypto_shash_update(desc: shash, data: buf, len: 2); |
477 | if (ret) |
478 | goto out; |
479 | memset(buf, 0, sizeof(buf)); |
480 | ret = crypto_shash_update(desc: shash, data: buf, len: 1); |
481 | if (ret) |
482 | goto out; |
483 | ret = crypto_shash_update(desc: shash, data: "HostHost" , len: 8); |
484 | if (ret) |
485 | goto out; |
486 | ret = crypto_shash_update(desc: shash, data: ctrl->opts->host->nqn, |
487 | strlen(ctrl->opts->host->nqn)); |
488 | if (ret) |
489 | goto out; |
490 | ret = crypto_shash_update(desc: shash, data: buf, len: 1); |
491 | if (ret) |
492 | goto out; |
493 | ret = crypto_shash_update(desc: shash, data: ctrl->opts->subsysnqn, |
494 | strlen(ctrl->opts->subsysnqn)); |
495 | if (ret) |
496 | goto out; |
497 | ret = crypto_shash_final(desc: shash, out: chap->response); |
498 | out: |
499 | if (challenge != chap->c1) |
500 | kfree(objp: challenge); |
501 | return ret; |
502 | } |
503 | |
504 | static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl, |
505 | struct nvme_dhchap_queue_context *chap) |
506 | { |
507 | SHASH_DESC_ON_STACK(shash, chap->shash_tfm); |
508 | struct nvme_dhchap_key *transformed_key; |
509 | u8 buf[4], *challenge = chap->c2; |
510 | int ret; |
511 | |
512 | transformed_key = nvme_auth_transform_key(key: ctrl->ctrl_key, |
513 | nqn: ctrl->opts->subsysnqn); |
514 | if (IS_ERR(ptr: transformed_key)) { |
515 | ret = PTR_ERR(ptr: transformed_key); |
516 | return ret; |
517 | } |
518 | |
519 | ret = crypto_shash_setkey(tfm: chap->shash_tfm, |
520 | key: transformed_key->key, keylen: transformed_key->len); |
521 | if (ret) { |
522 | dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n" , |
523 | chap->qid, ret); |
524 | goto out; |
525 | } |
526 | |
527 | if (chap->dh_tfm) { |
528 | challenge = kmalloc(size: chap->hash_len, GFP_KERNEL); |
529 | if (!challenge) { |
530 | ret = -ENOMEM; |
531 | goto out; |
532 | } |
533 | ret = nvme_auth_augmented_challenge(hmac_id: chap->hash_id, |
534 | skey: chap->sess_key, |
535 | skey_len: chap->sess_key_len, |
536 | challenge: chap->c2, aug: challenge, |
537 | hlen: chap->hash_len); |
538 | if (ret) |
539 | goto out; |
540 | } |
541 | dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n" , |
542 | __func__, chap->qid, chap->s2, chap->transaction); |
543 | dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n" , |
544 | __func__, chap->qid, (int)chap->hash_len, challenge); |
545 | dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n" , |
546 | __func__, chap->qid, ctrl->opts->subsysnqn); |
547 | dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n" , |
548 | __func__, chap->qid, ctrl->opts->host->nqn); |
549 | shash->tfm = chap->shash_tfm; |
550 | ret = crypto_shash_init(desc: shash); |
551 | if (ret) |
552 | goto out; |
553 | ret = crypto_shash_update(desc: shash, data: challenge, len: chap->hash_len); |
554 | if (ret) |
555 | goto out; |
556 | put_unaligned_le32(val: chap->s2, p: buf); |
557 | ret = crypto_shash_update(desc: shash, data: buf, len: 4); |
558 | if (ret) |
559 | goto out; |
560 | put_unaligned_le16(val: chap->transaction, p: buf); |
561 | ret = crypto_shash_update(desc: shash, data: buf, len: 2); |
562 | if (ret) |
563 | goto out; |
564 | memset(buf, 0, 4); |
565 | ret = crypto_shash_update(desc: shash, data: buf, len: 1); |
566 | if (ret) |
567 | goto out; |
568 | ret = crypto_shash_update(desc: shash, data: "Controller" , len: 10); |
569 | if (ret) |
570 | goto out; |
571 | ret = crypto_shash_update(desc: shash, data: ctrl->opts->subsysnqn, |
572 | strlen(ctrl->opts->subsysnqn)); |
573 | if (ret) |
574 | goto out; |
575 | ret = crypto_shash_update(desc: shash, data: buf, len: 1); |
576 | if (ret) |
577 | goto out; |
578 | ret = crypto_shash_update(desc: shash, data: ctrl->opts->host->nqn, |
579 | strlen(ctrl->opts->host->nqn)); |
580 | if (ret) |
581 | goto out; |
582 | ret = crypto_shash_final(desc: shash, out: chap->response); |
583 | out: |
584 | if (challenge != chap->c2) |
585 | kfree(objp: challenge); |
586 | nvme_auth_free_key(key: transformed_key); |
587 | return ret; |
588 | } |
589 | |
590 | static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl, |
591 | struct nvme_dhchap_queue_context *chap) |
592 | { |
593 | int ret; |
594 | |
595 | if (chap->host_key && chap->host_key_len) { |
596 | dev_dbg(ctrl->device, |
597 | "qid %d: reusing host key\n" , chap->qid); |
598 | goto gen_sesskey; |
599 | } |
600 | ret = nvme_auth_gen_privkey(dh_tfm: chap->dh_tfm, dh_gid: chap->dhgroup_id); |
601 | if (ret < 0) { |
602 | chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; |
603 | return ret; |
604 | } |
605 | |
606 | chap->host_key_len = crypto_kpp_maxsize(tfm: chap->dh_tfm); |
607 | |
608 | chap->host_key = kzalloc(size: chap->host_key_len, GFP_KERNEL); |
609 | if (!chap->host_key) { |
610 | chap->host_key_len = 0; |
611 | chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; |
612 | return -ENOMEM; |
613 | } |
614 | ret = nvme_auth_gen_pubkey(dh_tfm: chap->dh_tfm, |
615 | host_key: chap->host_key, host_key_len: chap->host_key_len); |
616 | if (ret) { |
617 | dev_dbg(ctrl->device, |
618 | "failed to generate public key, error %d\n" , ret); |
619 | chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; |
620 | return ret; |
621 | } |
622 | |
623 | gen_sesskey: |
624 | chap->sess_key_len = chap->host_key_len; |
625 | chap->sess_key = kmalloc(size: chap->sess_key_len, GFP_KERNEL); |
626 | if (!chap->sess_key) { |
627 | chap->sess_key_len = 0; |
628 | chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; |
629 | return -ENOMEM; |
630 | } |
631 | |
632 | ret = nvme_auth_gen_shared_secret(dh_tfm: chap->dh_tfm, |
633 | ctrl_key: chap->ctrl_key, ctrl_key_len: chap->ctrl_key_len, |
634 | sess_key: chap->sess_key, sess_key_len: chap->sess_key_len); |
635 | if (ret) { |
636 | dev_dbg(ctrl->device, |
637 | "failed to generate shared secret, error %d\n" , ret); |
638 | chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; |
639 | return ret; |
640 | } |
641 | dev_dbg(ctrl->device, "shared secret %*ph\n" , |
642 | (int)chap->sess_key_len, chap->sess_key); |
643 | return 0; |
644 | } |
645 | |
646 | static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap) |
647 | { |
648 | nvme_auth_free_key(key: chap->transformed_key); |
649 | chap->transformed_key = NULL; |
650 | kfree_sensitive(objp: chap->host_key); |
651 | chap->host_key = NULL; |
652 | chap->host_key_len = 0; |
653 | kfree_sensitive(objp: chap->ctrl_key); |
654 | chap->ctrl_key = NULL; |
655 | chap->ctrl_key_len = 0; |
656 | kfree_sensitive(objp: chap->sess_key); |
657 | chap->sess_key = NULL; |
658 | chap->sess_key_len = 0; |
659 | chap->status = 0; |
660 | chap->error = 0; |
661 | chap->s1 = 0; |
662 | chap->s2 = 0; |
663 | chap->bi_directional = false; |
664 | chap->transaction = 0; |
665 | memset(chap->c1, 0, sizeof(chap->c1)); |
666 | memset(chap->c2, 0, sizeof(chap->c2)); |
667 | mempool_free(element: chap->buf, pool: nvme_chap_buf_pool); |
668 | chap->buf = NULL; |
669 | } |
670 | |
671 | static void nvme_auth_free_dhchap(struct nvme_dhchap_queue_context *chap) |
672 | { |
673 | nvme_auth_reset_dhchap(chap); |
674 | if (chap->shash_tfm) |
675 | crypto_free_shash(tfm: chap->shash_tfm); |
676 | if (chap->dh_tfm) |
677 | crypto_free_kpp(tfm: chap->dh_tfm); |
678 | } |
679 | |
680 | static void nvme_queue_auth_work(struct work_struct *work) |
681 | { |
682 | struct nvme_dhchap_queue_context *chap = |
683 | container_of(work, struct nvme_dhchap_queue_context, auth_work); |
684 | struct nvme_ctrl *ctrl = chap->ctrl; |
685 | size_t tl; |
686 | int ret = 0; |
687 | |
688 | /* |
689 | * Allocate a large enough buffer for the entire negotiation: |
690 | * 4k is enough to ffdhe8192. |
691 | */ |
692 | chap->buf = mempool_alloc(pool: nvme_chap_buf_pool, GFP_KERNEL); |
693 | if (!chap->buf) { |
694 | chap->error = -ENOMEM; |
695 | return; |
696 | } |
697 | |
698 | chap->transaction = ctrl->transaction++; |
699 | |
700 | /* DH-HMAC-CHAP Step 1: send negotiate */ |
701 | dev_dbg(ctrl->device, "%s: qid %d send negotiate\n" , |
702 | __func__, chap->qid); |
703 | ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap); |
704 | if (ret < 0) { |
705 | chap->error = ret; |
706 | return; |
707 | } |
708 | tl = ret; |
709 | ret = nvme_auth_submit(ctrl, qid: chap->qid, data: chap->buf, data_len: tl, auth_send: true); |
710 | if (ret) { |
711 | chap->error = ret; |
712 | return; |
713 | } |
714 | |
715 | /* DH-HMAC-CHAP Step 2: receive challenge */ |
716 | dev_dbg(ctrl->device, "%s: qid %d receive challenge\n" , |
717 | __func__, chap->qid); |
718 | |
719 | memset(chap->buf, 0, CHAP_BUF_SIZE); |
720 | ret = nvme_auth_submit(ctrl, qid: chap->qid, data: chap->buf, CHAP_BUF_SIZE, |
721 | auth_send: false); |
722 | if (ret) { |
723 | dev_warn(ctrl->device, |
724 | "qid %d failed to receive challenge, %s %d\n" , |
725 | chap->qid, ret < 0 ? "error" : "nvme status" , ret); |
726 | chap->error = ret; |
727 | return; |
728 | } |
729 | ret = nvme_auth_receive_validate(ctrl, qid: chap->qid, data: chap->buf, transaction: chap->transaction, |
730 | expected_msg: NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE); |
731 | if (ret) { |
732 | chap->status = ret; |
733 | chap->error = -ECONNREFUSED; |
734 | return; |
735 | } |
736 | |
737 | ret = nvme_auth_process_dhchap_challenge(ctrl, chap); |
738 | if (ret) { |
739 | /* Invalid challenge parameters */ |
740 | chap->error = ret; |
741 | goto fail2; |
742 | } |
743 | |
744 | if (chap->ctrl_key_len) { |
745 | dev_dbg(ctrl->device, |
746 | "%s: qid %d DH exponential\n" , |
747 | __func__, chap->qid); |
748 | ret = nvme_auth_dhchap_exponential(ctrl, chap); |
749 | if (ret) { |
750 | chap->error = ret; |
751 | goto fail2; |
752 | } |
753 | } |
754 | |
755 | dev_dbg(ctrl->device, "%s: qid %d host response\n" , |
756 | __func__, chap->qid); |
757 | mutex_lock(&ctrl->dhchap_auth_mutex); |
758 | ret = nvme_auth_dhchap_setup_host_response(ctrl, chap); |
759 | mutex_unlock(lock: &ctrl->dhchap_auth_mutex); |
760 | if (ret) { |
761 | chap->error = ret; |
762 | goto fail2; |
763 | } |
764 | |
765 | /* DH-HMAC-CHAP Step 3: send reply */ |
766 | dev_dbg(ctrl->device, "%s: qid %d send reply\n" , |
767 | __func__, chap->qid); |
768 | ret = nvme_auth_set_dhchap_reply_data(ctrl, chap); |
769 | if (ret < 0) { |
770 | chap->error = ret; |
771 | goto fail2; |
772 | } |
773 | |
774 | tl = ret; |
775 | ret = nvme_auth_submit(ctrl, qid: chap->qid, data: chap->buf, data_len: tl, auth_send: true); |
776 | if (ret) { |
777 | chap->error = ret; |
778 | goto fail2; |
779 | } |
780 | |
781 | /* DH-HMAC-CHAP Step 4: receive success1 */ |
782 | dev_dbg(ctrl->device, "%s: qid %d receive success1\n" , |
783 | __func__, chap->qid); |
784 | |
785 | memset(chap->buf, 0, CHAP_BUF_SIZE); |
786 | ret = nvme_auth_submit(ctrl, qid: chap->qid, data: chap->buf, CHAP_BUF_SIZE, |
787 | auth_send: false); |
788 | if (ret) { |
789 | dev_warn(ctrl->device, |
790 | "qid %d failed to receive success1, %s %d\n" , |
791 | chap->qid, ret < 0 ? "error" : "nvme status" , ret); |
792 | chap->error = ret; |
793 | return; |
794 | } |
795 | ret = nvme_auth_receive_validate(ctrl, qid: chap->qid, |
796 | data: chap->buf, transaction: chap->transaction, |
797 | expected_msg: NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1); |
798 | if (ret) { |
799 | chap->status = ret; |
800 | chap->error = -ECONNREFUSED; |
801 | return; |
802 | } |
803 | |
804 | mutex_lock(&ctrl->dhchap_auth_mutex); |
805 | if (ctrl->ctrl_key) { |
806 | dev_dbg(ctrl->device, |
807 | "%s: qid %d controller response\n" , |
808 | __func__, chap->qid); |
809 | ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap); |
810 | if (ret) { |
811 | mutex_unlock(lock: &ctrl->dhchap_auth_mutex); |
812 | chap->error = ret; |
813 | goto fail2; |
814 | } |
815 | } |
816 | mutex_unlock(lock: &ctrl->dhchap_auth_mutex); |
817 | |
818 | ret = nvme_auth_process_dhchap_success1(ctrl, chap); |
819 | if (ret) { |
820 | /* Controller authentication failed */ |
821 | chap->error = -ECONNREFUSED; |
822 | goto fail2; |
823 | } |
824 | |
825 | if (chap->bi_directional) { |
826 | /* DH-HMAC-CHAP Step 5: send success2 */ |
827 | dev_dbg(ctrl->device, "%s: qid %d send success2\n" , |
828 | __func__, chap->qid); |
829 | tl = nvme_auth_set_dhchap_success2_data(ctrl, chap); |
830 | ret = nvme_auth_submit(ctrl, qid: chap->qid, data: chap->buf, data_len: tl, auth_send: true); |
831 | if (ret) |
832 | chap->error = ret; |
833 | } |
834 | if (!ret) { |
835 | chap->error = 0; |
836 | return; |
837 | } |
838 | |
839 | fail2: |
840 | if (chap->status == 0) |
841 | chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; |
842 | dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n" , |
843 | __func__, chap->qid, chap->status); |
844 | tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap); |
845 | ret = nvme_auth_submit(ctrl, qid: chap->qid, data: chap->buf, data_len: tl, auth_send: true); |
846 | /* |
847 | * only update error if send failure2 failed and no other |
848 | * error had been set during authentication. |
849 | */ |
850 | if (ret && !chap->error) |
851 | chap->error = ret; |
852 | } |
853 | |
854 | int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid) |
855 | { |
856 | struct nvme_dhchap_queue_context *chap; |
857 | |
858 | if (!ctrl->host_key) { |
859 | dev_warn(ctrl->device, "qid %d: no key\n" , qid); |
860 | return -ENOKEY; |
861 | } |
862 | |
863 | if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) { |
864 | dev_warn(ctrl->device, "qid %d: invalid ctrl key\n" , qid); |
865 | return -ENOKEY; |
866 | } |
867 | |
868 | chap = &ctrl->dhchap_ctxs[qid]; |
869 | cancel_work_sync(work: &chap->auth_work); |
870 | queue_work(wq: nvme_auth_wq, work: &chap->auth_work); |
871 | return 0; |
872 | } |
873 | EXPORT_SYMBOL_GPL(nvme_auth_negotiate); |
874 | |
875 | int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid) |
876 | { |
877 | struct nvme_dhchap_queue_context *chap; |
878 | int ret; |
879 | |
880 | chap = &ctrl->dhchap_ctxs[qid]; |
881 | flush_work(work: &chap->auth_work); |
882 | ret = chap->error; |
883 | /* clear sensitive info */ |
884 | nvme_auth_reset_dhchap(chap); |
885 | return ret; |
886 | } |
887 | EXPORT_SYMBOL_GPL(nvme_auth_wait); |
888 | |
889 | static void nvme_ctrl_auth_work(struct work_struct *work) |
890 | { |
891 | struct nvme_ctrl *ctrl = |
892 | container_of(work, struct nvme_ctrl, dhchap_auth_work); |
893 | int ret, q; |
894 | |
895 | /* |
896 | * If the ctrl is no connected, bail as reconnect will handle |
897 | * authentication. |
898 | */ |
899 | if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) |
900 | return; |
901 | |
902 | /* Authenticate admin queue first */ |
903 | ret = nvme_auth_negotiate(ctrl, 0); |
904 | if (ret) { |
905 | dev_warn(ctrl->device, |
906 | "qid 0: error %d setting up authentication\n" , ret); |
907 | return; |
908 | } |
909 | ret = nvme_auth_wait(ctrl, 0); |
910 | if (ret) { |
911 | dev_warn(ctrl->device, |
912 | "qid 0: authentication failed\n" ); |
913 | return; |
914 | } |
915 | |
916 | for (q = 1; q < ctrl->queue_count; q++) { |
917 | ret = nvme_auth_negotiate(ctrl, q); |
918 | if (ret) { |
919 | dev_warn(ctrl->device, |
920 | "qid %d: error %d setting up authentication\n" , |
921 | q, ret); |
922 | break; |
923 | } |
924 | } |
925 | |
926 | /* |
927 | * Failure is a soft-state; credentials remain valid until |
928 | * the controller terminates the connection. |
929 | */ |
930 | for (q = 1; q < ctrl->queue_count; q++) { |
931 | ret = nvme_auth_wait(ctrl, q); |
932 | if (ret) |
933 | dev_warn(ctrl->device, |
934 | "qid %d: authentication failed\n" , q); |
935 | } |
936 | } |
937 | |
938 | int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl) |
939 | { |
940 | struct nvme_dhchap_queue_context *chap; |
941 | int i, ret; |
942 | |
943 | mutex_init(&ctrl->dhchap_auth_mutex); |
944 | INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work); |
945 | if (!ctrl->opts) |
946 | return 0; |
947 | ret = nvme_auth_generate_key(secret: ctrl->opts->dhchap_secret, |
948 | ret_key: &ctrl->host_key); |
949 | if (ret) |
950 | return ret; |
951 | ret = nvme_auth_generate_key(secret: ctrl->opts->dhchap_ctrl_secret, |
952 | ret_key: &ctrl->ctrl_key); |
953 | if (ret) |
954 | goto err_free_dhchap_secret; |
955 | |
956 | if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret) |
957 | return 0; |
958 | |
959 | ctrl->dhchap_ctxs = kvcalloc(n: ctrl_max_dhchaps(ctrl), |
960 | size: sizeof(*chap), GFP_KERNEL); |
961 | if (!ctrl->dhchap_ctxs) { |
962 | ret = -ENOMEM; |
963 | goto err_free_dhchap_ctrl_secret; |
964 | } |
965 | |
966 | for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) { |
967 | chap = &ctrl->dhchap_ctxs[i]; |
968 | chap->qid = i; |
969 | chap->ctrl = ctrl; |
970 | INIT_WORK(&chap->auth_work, nvme_queue_auth_work); |
971 | } |
972 | |
973 | return 0; |
974 | err_free_dhchap_ctrl_secret: |
975 | nvme_auth_free_key(key: ctrl->ctrl_key); |
976 | ctrl->ctrl_key = NULL; |
977 | err_free_dhchap_secret: |
978 | nvme_auth_free_key(key: ctrl->host_key); |
979 | ctrl->host_key = NULL; |
980 | return ret; |
981 | } |
982 | EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl); |
983 | |
984 | void nvme_auth_stop(struct nvme_ctrl *ctrl) |
985 | { |
986 | cancel_work_sync(work: &ctrl->dhchap_auth_work); |
987 | } |
988 | EXPORT_SYMBOL_GPL(nvme_auth_stop); |
989 | |
990 | void nvme_auth_free(struct nvme_ctrl *ctrl) |
991 | { |
992 | int i; |
993 | |
994 | if (ctrl->dhchap_ctxs) { |
995 | for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) |
996 | nvme_auth_free_dhchap(chap: &ctrl->dhchap_ctxs[i]); |
997 | kfree(objp: ctrl->dhchap_ctxs); |
998 | } |
999 | if (ctrl->host_key) { |
1000 | nvme_auth_free_key(key: ctrl->host_key); |
1001 | ctrl->host_key = NULL; |
1002 | } |
1003 | if (ctrl->ctrl_key) { |
1004 | nvme_auth_free_key(key: ctrl->ctrl_key); |
1005 | ctrl->ctrl_key = NULL; |
1006 | } |
1007 | } |
1008 | EXPORT_SYMBOL_GPL(nvme_auth_free); |
1009 | |
1010 | int __init nvme_init_auth(void) |
1011 | { |
1012 | nvme_auth_wq = alloc_workqueue(fmt: "nvme-auth-wq" , |
1013 | flags: WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, max_active: 0); |
1014 | if (!nvme_auth_wq) |
1015 | return -ENOMEM; |
1016 | |
1017 | nvme_chap_buf_cache = kmem_cache_create(name: "nvme-chap-buf-cache" , |
1018 | CHAP_BUF_SIZE, align: 0, SLAB_HWCACHE_ALIGN, NULL); |
1019 | if (!nvme_chap_buf_cache) |
1020 | goto err_destroy_workqueue; |
1021 | |
1022 | nvme_chap_buf_pool = mempool_create(min_nr: 16, alloc_fn: mempool_alloc_slab, |
1023 | free_fn: mempool_free_slab, pool_data: nvme_chap_buf_cache); |
1024 | if (!nvme_chap_buf_pool) |
1025 | goto err_destroy_chap_buf_cache; |
1026 | |
1027 | return 0; |
1028 | err_destroy_chap_buf_cache: |
1029 | kmem_cache_destroy(s: nvme_chap_buf_cache); |
1030 | err_destroy_workqueue: |
1031 | destroy_workqueue(wq: nvme_auth_wq); |
1032 | return -ENOMEM; |
1033 | } |
1034 | |
1035 | void __exit nvme_exit_auth(void) |
1036 | { |
1037 | mempool_destroy(pool: nvme_chap_buf_pool); |
1038 | kmem_cache_destroy(s: nvme_chap_buf_cache); |
1039 | destroy_workqueue(wq: nvme_auth_wq); |
1040 | } |
1041 | |