1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* NVMe over Fabrics TCP host.
4	* Copyright (c) 2018 Lightbits Labs. All rights reserved.
5	*/
6	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7	#include <linux/module.h>
8	#include <linux/init.h>
9	#include <linux/slab.h>
10	#include <linux/err.h>
11	#include <linux/key.h>
12	#include <linux/nvme-tcp.h>
13	#include <linux/nvme-keyring.h>
14	#include <net/sock.h>
15	#include <net/tcp.h>
16	#include <net/tls.h>
17	#include <net/tls_prot.h>
18	#include <net/handshake.h>
19	#include <linux/blk-mq.h>
20	#include <crypto/hash.h>
21	#include <net/busy_poll.h>
22	#include <trace/events/sock.h>
23
24	#include "nvme.h"
25	#include "fabrics.h"
26
27	struct nvme_tcp_queue;
28
29	/* Define the socket priority to use for connections were it is desirable
30	* that the NIC consider performing optimized packet processing or filtering.
31	* A non-zero value being sufficient to indicate general consideration of any
32	* possible optimization. Making it a module param allows for alternative
33	* values that may be unique for some NIC implementations.
34	*/
35	static int so_priority;
36	module_param(so_priority, int, 0644);
37	MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority");
38
39	#ifdef CONFIG_NVME_TCP_TLS
40	/*
41	* TLS handshake timeout
42	*/
43	static int tls_handshake_timeout = 10;
44	module_param(tls_handshake_timeout, int, 0644);
45	MODULE_PARM_DESC(tls_handshake_timeout,
46	"nvme TLS handshake timeout in seconds (default 10)");
47	#endif
48
49	#ifdef CONFIG_DEBUG_LOCK_ALLOC
50	/* lockdep can detect a circular dependency of the form
51	* sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock
52	* because dependencies are tracked for both nvme-tcp and user contexts. Using
53	* a separate class prevents lockdep from conflating nvme-tcp socket use with
54	* user-space socket API use.
55	*/
56	static struct lock_class_key nvme_tcp_sk_key[2];
57	static struct lock_class_key nvme_tcp_slock_key[2];
58
59	static void nvme_tcp_reclassify_socket(struct socket *sock)
60	{
61	struct sock *sk = sock->sk;
62
63	if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
64	return;
65
66	switch (sk->sk_family) {
67	case AF_INET:
68	sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME",
69	&nvme_tcp_slock_key[0],
70	"sk_lock-AF_INET-NVME",
71	&nvme_tcp_sk_key[0]);
72	break;
73	case AF_INET6:
74	sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME",
75	&nvme_tcp_slock_key[1],
76	"sk_lock-AF_INET6-NVME",
77	&nvme_tcp_sk_key[1]);
78	break;
79	default:
80	WARN_ON_ONCE(1);
81	}
82	}
83	#else
84	static void nvme_tcp_reclassify_socket(struct socket *sock) { }
85	#endif
86
87	enum nvme_tcp_send_state {
88	NVME_TCP_SEND_CMD_PDU = 0,
89	NVME_TCP_SEND_H2C_PDU,
90	NVME_TCP_SEND_DATA,
91	NVME_TCP_SEND_DDGST,
92	};
93
94	struct nvme_tcp_request {
95	struct nvme_request req;
96	void *pdu;
97	struct nvme_tcp_queue *queue;
98	u32 data_len;
99	u32 pdu_len;
100	u32 pdu_sent;
101	u32 h2cdata_left;
102	u32 h2cdata_offset;
103	u16 ttag;
104	__le16 status;
105	struct list_head entry;
106	struct llist_node lentry;
107	__le32 ddgst;
108
109	struct bio *curr_bio;
110	struct iov_iter iter;
111
112	/* send state */
113	size_t offset;
114	size_t data_sent;
115	enum nvme_tcp_send_state state;
116	};
117
118	enum nvme_tcp_queue_flags {
119	NVME_TCP_Q_ALLOCATED = 0,
120	NVME_TCP_Q_LIVE = 1,
121	NVME_TCP_Q_POLLING = 2,
122	};
123
124	enum nvme_tcp_recv_state {
125	NVME_TCP_RECV_PDU = 0,
126	NVME_TCP_RECV_DATA,
127	NVME_TCP_RECV_DDGST,
128	};
129
130	struct nvme_tcp_ctrl;
131	struct nvme_tcp_queue {
132	struct socket *sock;
133	struct work_struct io_work;
134	int io_cpu;
135
136	struct mutex queue_lock;
137	struct mutex send_mutex;
138	struct llist_head req_list;
139	struct list_head send_list;
140
141	/* recv state */
142	void *pdu;
143	int pdu_remaining;
144	int pdu_offset;
145	size_t data_remaining;
146	size_t ddgst_remaining;
147	unsigned int nr_cqe;
148
149	/* send state */
150	struct nvme_tcp_request *request;
151
152	u32 maxh2cdata;
153	size_t cmnd_capsule_len;
154	struct nvme_tcp_ctrl *ctrl;
155	unsigned long flags;
156	bool rd_enabled;
157
158	bool hdr_digest;
159	bool data_digest;
160	struct ahash_request *rcv_hash;
161	struct ahash_request *snd_hash;
162	__le32 exp_ddgst;
163	__le32 recv_ddgst;
164	#ifdef CONFIG_NVME_TCP_TLS
165	struct completion tls_complete;
166	int tls_err;
167	#endif
168	struct page_frag_cache pf_cache;
169
170	void (*state_change)(struct sock *);
171	void (*data_ready)(struct sock *);
172	void (*write_space)(struct sock *);
173	};
174
175	struct nvme_tcp_ctrl {
176	/* read only in the hot path */
177	struct nvme_tcp_queue *queues;
178	struct blk_mq_tag_set tag_set;
179
180	/* other member variables */
181	struct list_head list;
182	struct blk_mq_tag_set admin_tag_set;
183	struct sockaddr_storage addr;
184	struct sockaddr_storage src_addr;
185	struct nvme_ctrl ctrl;
186
187	struct work_struct err_work;
188	struct delayed_work connect_work;
189	struct nvme_tcp_request async_req;
190	u32 io_queues[HCTX_MAX_TYPES];
191	};
192
193	static LIST_HEAD(nvme_tcp_ctrl_list);
194	static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
195	static struct workqueue_struct *nvme_tcp_wq;
196	static const struct blk_mq_ops nvme_tcp_mq_ops;
197	static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
198	static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
199
200	static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
201	{
202	return container_of(ctrl, struct nvme_tcp_ctrl, ctrl);
203	}
204
205	static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
206	{
207	return queue - queue->ctrl->queues;
208	}
209
210	static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)
211	{
212	u32 queue_idx = nvme_tcp_queue_id(queue);
213
214	if (queue_idx == 0)
215	return queue->ctrl->admin_tag_set.tags[queue_idx];
216	return queue->ctrl->tag_set.tags[queue_idx - 1];
217	}
218
219	static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)
220	{
221	return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
222	}
223
224	static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)
225	{
226	return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
227	}
228
229	static inline void *nvme_tcp_req_cmd_pdu(struct nvme_tcp_request *req)
230	{
231	return req->pdu;
232	}
233
234	static inline void *nvme_tcp_req_data_pdu(struct nvme_tcp_request *req)
235	{
236	/* use the pdu space in the back for the data pdu */
237	return req->pdu + sizeof(struct nvme_tcp_cmd_pdu) -
238	sizeof(struct nvme_tcp_data_pdu);
239	}
240
241	static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_request *req)
242	{
243	if (nvme_is_fabrics(cmd: req->req.cmd))
244	return NVME_TCP_ADMIN_CCSZ;
245	return req->queue->cmnd_capsule_len - sizeof(struct nvme_command);
246	}
247
248	static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req)
249	{
250	return req == &req->queue->ctrl->async_req;
251	}
252
253	static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req)
254	{
255	struct request *rq;
256
257	if (unlikely(nvme_tcp_async_req(req)))
258	return false; /* async events don't have a request */
259
260	rq = blk_mq_rq_from_pdu(pdu: req);
261
262	return rq_data_dir(rq) == WRITE && req->data_len &&
263	req->data_len <= nvme_tcp_inline_data_size(req);
264	}
265
266	static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req)
267	{
268	return req->iter.bvec->bv_page;
269	}
270
271	static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req)
272	{
273	return req->iter.bvec->bv_offset + req->iter.iov_offset;
274	}
275
276	static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req)
277	{
278	return min_t(size_t, iov_iter_single_seg_count(&req->iter),
279	req->pdu_len - req->pdu_sent);
280	}
281
282	static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req)
283	{
284	return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ?
285	req->pdu_len - req->pdu_sent : 0;
286	}
287
288	static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
289	int len)
290	{
291	return nvme_tcp_pdu_data_left(req) <= len;
292	}
293
294	static void nvme_tcp_init_iter(struct nvme_tcp_request *req,
295	unsigned int dir)
296	{
297	struct request *rq = blk_mq_rq_from_pdu(pdu: req);
298	struct bio_vec *vec;
299	unsigned int size;
300	int nr_bvec;
301	size_t offset;
302
303	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
304	vec = &rq->special_vec;
305	nr_bvec = 1;
306	size = blk_rq_payload_bytes(rq);
307	offset = 0;
308	} else {
309	struct bio *bio = req->curr_bio;
310	struct bvec_iter bi;
311	struct bio_vec bv;
312
313	vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
314	nr_bvec = 0;
315	bio_for_each_bvec(bv, bio, bi) {
316	nr_bvec++;
317	}
318	size = bio->bi_iter.bi_size;
319	offset = bio->bi_iter.bi_bvec_done;
320	}
321
322	iov_iter_bvec(i: &req->iter, direction: dir, bvec: vec, nr_segs: nr_bvec, count: size);
323	req->iter.iov_offset = offset;
324	}
325
326	static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
327	int len)
328	{
329	req->data_sent += len;
330	req->pdu_sent += len;
331	iov_iter_advance(i: &req->iter, bytes: len);
332	if (!iov_iter_count(i: &req->iter) &&
333	req->data_sent < req->data_len) {
334	req->curr_bio = req->curr_bio->bi_next;
335	nvme_tcp_init_iter(req, ITER_SOURCE);
336	}
337	}
338
339	static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue)
340	{
341	int ret;
342
343	/* drain the send queue as much as we can... */
344	do {
345	ret = nvme_tcp_try_send(queue);
346	} while (ret > 0);
347	}
348
349	static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue)
350	{
351	return !list_empty(head: &queue->send_list) ||
352	!llist_empty(head: &queue->req_list);
353	}
354
355	static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
356	bool sync, bool last)
357	{
358	struct nvme_tcp_queue *queue = req->queue;
359	bool empty;
360
361	empty = llist_add(new: &req->lentry, head: &queue->req_list) &&
362	list_empty(head: &queue->send_list) && !queue->request;
363
364	/*
365	* if we're the first on the send_list and we can try to send
366	* directly, otherwise queue io_work. Also, only do that if we
367	* are on the same cpu, so we don't introduce contention.
368	*/
369	if (queue->io_cpu == raw_smp_processor_id() &&
370	sync && empty && mutex_trylock(lock: &queue->send_mutex)) {
371	nvme_tcp_send_all(queue);
372	mutex_unlock(lock: &queue->send_mutex);
373	}
374
375	if (last && nvme_tcp_queue_more(queue))
376	queue_work_on(cpu: queue->io_cpu, wq: nvme_tcp_wq, work: &queue->io_work);
377	}
378
379	static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)
380	{
381	struct nvme_tcp_request *req;
382	struct llist_node *node;
383
384	for (node = llist_del_all(head: &queue->req_list); node; node = node->next) {
385	req = llist_entry(node, struct nvme_tcp_request, lentry);
386	list_add(new: &req->entry, head: &queue->send_list);
387	}
388	}
389
390	static inline struct nvme_tcp_request *
391	nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
392	{
393	struct nvme_tcp_request *req;
394
395	req = list_first_entry_or_null(&queue->send_list,
396	struct nvme_tcp_request, entry);
397	if (!req) {
398	nvme_tcp_process_req_list(queue);
399	req = list_first_entry_or_null(&queue->send_list,
400	struct nvme_tcp_request, entry);
401	if (unlikely(!req))
402	return NULL;
403	}
404
405	list_del(entry: &req->entry);
406	return req;
407	}
408
409	static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
410	__le32 *dgst)
411	{
412	ahash_request_set_crypt(req: hash, NULL, result: (u8 *)dgst, nbytes: 0);
413	crypto_ahash_final(req: hash);
414	}
415
416	static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
417	struct page *page, off_t off, size_t len)
418	{
419	struct scatterlist sg;
420
421	sg_init_table(&sg, 1);
422	sg_set_page(sg: &sg, page, len, offset: off);
423	ahash_request_set_crypt(req: hash, src: &sg, NULL, nbytes: len);
424	crypto_ahash_update(req: hash);
425	}
426
427	static inline void nvme_tcp_hdgst(struct ahash_request *hash,
428	void *pdu, size_t len)
429	{
430	struct scatterlist sg;
431
432	sg_init_one(&sg, pdu, len);
433	ahash_request_set_crypt(req: hash, src: &sg, result: pdu + len, nbytes: len);
434	crypto_ahash_digest(req: hash);
435	}
436
437	static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
438	void *pdu, size_t pdu_len)
439	{
440	struct nvme_tcp_hdr *hdr = pdu;
441	__le32 recv_digest;
442	__le32 exp_digest;
443
444	if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
445	dev_err(queue->ctrl->ctrl.device,
446	"queue %d: header digest flag is cleared\n",
447	nvme_tcp_queue_id(queue));
448	return -EPROTO;
449	}
450
451	recv_digest = *(__le32 *)(pdu + hdr->hlen);
452	nvme_tcp_hdgst(hash: queue->rcv_hash, pdu, len: pdu_len);
453	exp_digest = *(__le32 *)(pdu + hdr->hlen);
454	if (recv_digest != exp_digest) {
455	dev_err(queue->ctrl->ctrl.device,
456	"header digest error: recv %#x expected %#x\n",
457	le32_to_cpu(recv_digest), le32_to_cpu(exp_digest));
458	return -EIO;
459	}
460
461	return 0;
462	}
463
464	static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
465	{
466	struct nvme_tcp_hdr *hdr = pdu;
467	u8 digest_len = nvme_tcp_hdgst_len(queue);
468	u32 len;
469
470	len = le32_to_cpu(hdr->plen) - hdr->hlen -
471	((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0);
472
473	if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
474	dev_err(queue->ctrl->ctrl.device,
475	"queue %d: data digest flag is cleared\n",
476	nvme_tcp_queue_id(queue));
477	return -EPROTO;
478	}
479	crypto_ahash_init(req: queue->rcv_hash);
480
481	return 0;
482	}
483
484	static void nvme_tcp_exit_request(struct blk_mq_tag_set *set,
485	struct request *rq, unsigned int hctx_idx)
486	{
487	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
488
489	page_frag_free(addr: req->pdu);
490	}
491
492	static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
493	struct request *rq, unsigned int hctx_idx,
494	unsigned int numa_node)
495	{
496	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(ctrl: set->driver_data);
497	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
498	struct nvme_tcp_cmd_pdu *pdu;
499	int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
500	struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];
501	u8 hdgst = nvme_tcp_hdgst_len(queue);
502
503	req->pdu = page_frag_alloc(nc: &queue->pf_cache,
504	fragsz: sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
505	GFP_KERNEL | __GFP_ZERO);
506	if (!req->pdu)
507	return -ENOMEM;
508
509	pdu = req->pdu;
510	req->queue = queue;
511	nvme_req(req: rq)->ctrl = &ctrl->ctrl;
512	nvme_req(req: rq)->cmd = &pdu->cmd;
513
514	return 0;
515	}
516
517	static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
518	unsigned int hctx_idx)
519	{
520	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(ctrl: data);
521	struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];
522
523	hctx->driver_data = queue;
524	return 0;
525	}
526
527	static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
528	unsigned int hctx_idx)
529	{
530	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(ctrl: data);
531	struct nvme_tcp_queue *queue = &ctrl->queues[0];
532
533	hctx->driver_data = queue;
534	return 0;
535	}
536
537	static enum nvme_tcp_recv_state
538	nvme_tcp_recv_state(struct nvme_tcp_queue *queue)
539	{
540	return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :
541	(queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :
542	NVME_TCP_RECV_DATA;
543	}
544
545	static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
546	{
547	queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +
548	nvme_tcp_hdgst_len(queue);
549	queue->pdu_offset = 0;
550	queue->data_remaining = -1;
551	queue->ddgst_remaining = 0;
552	}
553
554	static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
555	{
556	if (!nvme_change_ctrl_state(ctrl, new_state: NVME_CTRL_RESETTING))
557	return;
558
559	dev_warn(ctrl->device, "starting error recovery\n");
560	queue_work(wq: nvme_reset_wq, work: &to_tcp_ctrl(ctrl)->err_work);
561	}
562
563	static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
564	struct nvme_completion *cqe)
565	{
566	struct nvme_tcp_request *req;
567	struct request *rq;
568
569	rq = nvme_find_rq(tags: nvme_tcp_tagset(queue), command_id: cqe->command_id);
570	if (!rq) {
571	dev_err(queue->ctrl->ctrl.device,
572	"got bad cqe.command_id %#x on queue %d\n",
573	cqe->command_id, nvme_tcp_queue_id(queue));
574	nvme_tcp_error_recovery(ctrl: &queue->ctrl->ctrl);
575	return -EINVAL;
576	}
577
578	req = blk_mq_rq_to_pdu(rq);
579	if (req->status == cpu_to_le16(NVME_SC_SUCCESS))
580	req->status = cqe->status;
581
582	if (!nvme_try_complete_req(req: rq, status: req->status, result: cqe->result))
583	nvme_complete_rq(req: rq);
584	queue->nr_cqe++;
585
586	return 0;
587	}
588
589	static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,
590	struct nvme_tcp_data_pdu *pdu)
591	{
592	struct request *rq;
593
594	rq = nvme_find_rq(tags: nvme_tcp_tagset(queue), command_id: pdu->command_id);
595	if (!rq) {
596	dev_err(queue->ctrl->ctrl.device,
597	"got bad c2hdata.command_id %#x on queue %d\n",
598	pdu->command_id, nvme_tcp_queue_id(queue));
599	return -ENOENT;
600	}
601
602	if (!blk_rq_payload_bytes(rq)) {
603	dev_err(queue->ctrl->ctrl.device,
604	"queue %d tag %#x unexpected data\n",
605	nvme_tcp_queue_id(queue), rq->tag);
606	return -EIO;
607	}
608
609	queue->data_remaining = le32_to_cpu(pdu->data_length);
610
611	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
612	unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
613	dev_err(queue->ctrl->ctrl.device,
614	"queue %d tag %#x SUCCESS set but not last PDU\n",
615	nvme_tcp_queue_id(queue), rq->tag);
616	nvme_tcp_error_recovery(ctrl: &queue->ctrl->ctrl);
617	return -EPROTO;
618	}
619
620	return 0;
621	}
622
623	static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,
624	struct nvme_tcp_rsp_pdu *pdu)
625	{
626	struct nvme_completion *cqe = &pdu->cqe;
627	int ret = 0;
628
629	/*
630	* AEN requests are special as they don't time out and can
631	* survive any kind of queue freeze and often don't respond to
632	* aborts. We don't even bother to allocate a struct request
633	* for them but rather special case them here.
634	*/
635	if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue),
636	cqe->command_id)))
637	nvme_complete_async_event(ctrl: &queue->ctrl->ctrl, status: cqe->status,
638	res: &cqe->result);
639	else
640	ret = nvme_tcp_process_nvme_cqe(queue, cqe);
641
642	return ret;
643	}
644
645	static void nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req)
646	{
647	struct nvme_tcp_data_pdu *data = nvme_tcp_req_data_pdu(req);
648	struct nvme_tcp_queue *queue = req->queue;
649	struct request *rq = blk_mq_rq_from_pdu(pdu: req);
650	u32 h2cdata_sent = req->pdu_len;
651	u8 hdgst = nvme_tcp_hdgst_len(queue);
652	u8 ddgst = nvme_tcp_ddgst_len(queue);
653
654	req->state = NVME_TCP_SEND_H2C_PDU;
655	req->offset = 0;
656	req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata);
657	req->pdu_sent = 0;
658	req->h2cdata_left -= req->pdu_len;
659	req->h2cdata_offset += h2cdata_sent;
660
661	memset(data, 0, sizeof(*data));
662	data->hdr.type = nvme_tcp_h2c_data;
663	if (!req->h2cdata_left)
664	data->hdr.flags = NVME_TCP_F_DATA_LAST;
665	if (queue->hdr_digest)
666	data->hdr.flags |= NVME_TCP_F_HDGST;
667	if (queue->data_digest)
668	data->hdr.flags |= NVME_TCP_F_DDGST;
669	data->hdr.hlen = sizeof(*data);
670	data->hdr.pdo = data->hdr.hlen + hdgst;
671	data->hdr.plen =
672	cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst);
673	data->ttag = req->ttag;
674	data->command_id = nvme_cid(rq);
675	data->data_offset = cpu_to_le32(req->h2cdata_offset);
676	data->data_length = cpu_to_le32(req->pdu_len);
677	}
678
679	static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
680	struct nvme_tcp_r2t_pdu *pdu)
681	{
682	struct nvme_tcp_request *req;
683	struct request *rq;
684	u32 r2t_length = le32_to_cpu(pdu->r2t_length);
685	u32 r2t_offset = le32_to_cpu(pdu->r2t_offset);
686
687	rq = nvme_find_rq(tags: nvme_tcp_tagset(queue), command_id: pdu->command_id);
688	if (!rq) {
689	dev_err(queue->ctrl->ctrl.device,
690	"got bad r2t.command_id %#x on queue %d\n",
691	pdu->command_id, nvme_tcp_queue_id(queue));
692	return -ENOENT;
693	}
694	req = blk_mq_rq_to_pdu(rq);
695
696	if (unlikely(!r2t_length)) {
697	dev_err(queue->ctrl->ctrl.device,
698	"req %d r2t len is %u, probably a bug...\n",
699	rq->tag, r2t_length);
700	return -EPROTO;
701	}
702
703	if (unlikely(req->data_sent + r2t_length > req->data_len)) {
704	dev_err(queue->ctrl->ctrl.device,
705	"req %d r2t len %u exceeded data len %u (%zu sent)\n",
706	rq->tag, r2t_length, req->data_len, req->data_sent);
707	return -EPROTO;
708	}
709
710	if (unlikely(r2t_offset < req->data_sent)) {
711	dev_err(queue->ctrl->ctrl.device,
712	"req %d unexpected r2t offset %u (expected %zu)\n",
713	rq->tag, r2t_offset, req->data_sent);
714	return -EPROTO;
715	}
716
717	req->pdu_len = 0;
718	req->h2cdata_left = r2t_length;
719	req->h2cdata_offset = r2t_offset;
720	req->ttag = pdu->ttag;
721
722	nvme_tcp_setup_h2c_data_pdu(req);
723	nvme_tcp_queue_request(req, sync: false, last: true);
724
725	return 0;
726	}
727
728	static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,
729	unsigned int *offset, size_t *len)
730	{
731	struct nvme_tcp_hdr *hdr;
732	char *pdu = queue->pdu;
733	size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);
734	int ret;
735
736	ret = skb_copy_bits(skb, offset: *offset,
737	to: &pdu[queue->pdu_offset], len: rcv_len);
738	if (unlikely(ret))
739	return ret;
740
741	queue->pdu_remaining -= rcv_len;
742	queue->pdu_offset += rcv_len;
743	*offset += rcv_len;
744	*len -= rcv_len;
745	if (queue->pdu_remaining)
746	return 0;
747
748	hdr = queue->pdu;
749	if (queue->hdr_digest) {
750	ret = nvme_tcp_verify_hdgst(queue, pdu: queue->pdu, pdu_len: hdr->hlen);
751	if (unlikely(ret))
752	return ret;
753	}
754
755
756	if (queue->data_digest) {
757	ret = nvme_tcp_check_ddgst(queue, pdu: queue->pdu);
758	if (unlikely(ret))
759	return ret;
760	}
761
762	switch (hdr->type) {
763	case nvme_tcp_c2h_data:
764	return nvme_tcp_handle_c2h_data(queue, pdu: (void *)queue->pdu);
765	case nvme_tcp_rsp:
766	nvme_tcp_init_recv_ctx(queue);
767	return nvme_tcp_handle_comp(queue, pdu: (void *)queue->pdu);
768	case nvme_tcp_r2t:
769	nvme_tcp_init_recv_ctx(queue);
770	return nvme_tcp_handle_r2t(queue, pdu: (void *)queue->pdu);
771	default:
772	dev_err(queue->ctrl->ctrl.device,
773	"unsupported pdu type (%d)\n", hdr->type);
774	return -EINVAL;
775	}
776	}
777
778	static inline void nvme_tcp_end_request(struct request *rq, u16 status)
779	{
780	union nvme_result res = {};
781
782	if (!nvme_try_complete_req(req: rq, cpu_to_le16(status << 1), result: res))
783	nvme_complete_rq(req: rq);
784	}
785
786	static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
787	unsigned int *offset, size_t *len)
788	{
789	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
790	struct request *rq =
791	nvme_cid_to_rq(tags: nvme_tcp_tagset(queue), command_id: pdu->command_id);
792	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
793
794	while (true) {
795	int recv_len, ret;
796
797	recv_len = min_t(size_t, *len, queue->data_remaining);
798	if (!recv_len)
799	break;
800
801	if (!iov_iter_count(i: &req->iter)) {
802	req->curr_bio = req->curr_bio->bi_next;
803
804	/*
805	* If we don`t have any bios it means that controller
806	* sent more data than we requested, hence error
807	*/
808	if (!req->curr_bio) {
809	dev_err(queue->ctrl->ctrl.device,
810	"queue %d no space in request %#x",
811	nvme_tcp_queue_id(queue), rq->tag);
812	nvme_tcp_init_recv_ctx(queue);
813	return -EIO;
814	}
815	nvme_tcp_init_iter(req, ITER_DEST);
816	}
817
818	/* we can read only from what is left in this bio */
819	recv_len = min_t(size_t, recv_len,
820	iov_iter_count(&req->iter));
821
822	if (queue->data_digest)
823	ret = skb_copy_and_hash_datagram_iter(skb, offset: *offset,
824	to: &req->iter, len: recv_len, hash: queue->rcv_hash);
825	else
826	ret = skb_copy_datagram_iter(from: skb, offset: *offset,
827	to: &req->iter, size: recv_len);
828	if (ret) {
829	dev_err(queue->ctrl->ctrl.device,
830	"queue %d failed to copy request %#x data",
831	nvme_tcp_queue_id(queue), rq->tag);
832	return ret;
833	}
834
835	*len -= recv_len;
836	*offset += recv_len;
837	queue->data_remaining -= recv_len;
838	}
839
840	if (!queue->data_remaining) {
841	if (queue->data_digest) {
842	nvme_tcp_ddgst_final(hash: queue->rcv_hash, dgst: &queue->exp_ddgst);
843	queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
844	} else {
845	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
846	nvme_tcp_end_request(rq,
847	le16_to_cpu(req->status));
848	queue->nr_cqe++;
849	}
850	nvme_tcp_init_recv_ctx(queue);
851	}
852	}
853
854	return 0;
855	}
856
857	static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,
858	struct sk_buff *skb, unsigned int *offset, size_t *len)
859	{
860	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
861	char *ddgst = (char *)&queue->recv_ddgst;
862	size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
863	off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
864	int ret;
865
866	ret = skb_copy_bits(skb, offset: *offset, to: &ddgst[off], len: recv_len);
867	if (unlikely(ret))
868	return ret;
869
870	queue->ddgst_remaining -= recv_len;
871	*offset += recv_len;
872	*len -= recv_len;
873	if (queue->ddgst_remaining)
874	return 0;
875
876	if (queue->recv_ddgst != queue->exp_ddgst) {
877	struct request *rq = nvme_cid_to_rq(tags: nvme_tcp_tagset(queue),
878	command_id: pdu->command_id);
879	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
880
881	req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR);
882
883	dev_err(queue->ctrl->ctrl.device,
884	"data digest error: recv %#x expected %#x\n",
885	le32_to_cpu(queue->recv_ddgst),
886	le32_to_cpu(queue->exp_ddgst));
887	}
888
889	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
890	struct request *rq = nvme_cid_to_rq(tags: nvme_tcp_tagset(queue),
891	command_id: pdu->command_id);
892	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
893
894	nvme_tcp_end_request(rq, le16_to_cpu(req->status));
895	queue->nr_cqe++;
896	}
897
898	nvme_tcp_init_recv_ctx(queue);
899	return 0;
900	}
901
902	static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
903	unsigned int offset, size_t len)
904	{
905	struct nvme_tcp_queue *queue = desc->arg.data;
906	size_t consumed = len;
907	int result;
908
909	if (unlikely(!queue->rd_enabled))
910	return -EFAULT;
911
912	while (len) {
913	switch (nvme_tcp_recv_state(queue)) {
914	case NVME_TCP_RECV_PDU:
915	result = nvme_tcp_recv_pdu(queue, skb, offset: &offset, len: &len);
916	break;
917	case NVME_TCP_RECV_DATA:
918	result = nvme_tcp_recv_data(queue, skb, offset: &offset, len: &len);
919	break;
920	case NVME_TCP_RECV_DDGST:
921	result = nvme_tcp_recv_ddgst(queue, skb, offset: &offset, len: &len);
922	break;
923	default:
924	result = -EFAULT;
925	}
926	if (result) {
927	dev_err(queue->ctrl->ctrl.device,
928	"receive failed: %d\n", result);
929	queue->rd_enabled = false;
930	nvme_tcp_error_recovery(ctrl: &queue->ctrl->ctrl);
931	return result;
932	}
933	}
934
935	return consumed;
936	}
937
938	static void nvme_tcp_data_ready(struct sock *sk)
939	{
940	struct nvme_tcp_queue *queue;
941
942	trace_sk_data_ready(sk);
943
944	read_lock_bh(&sk->sk_callback_lock);
945	queue = sk->sk_user_data;
946	if (likely(queue && queue->rd_enabled) &&
947	!test_bit(NVME_TCP_Q_POLLING, &queue->flags))
948	queue_work_on(cpu: queue->io_cpu, wq: nvme_tcp_wq, work: &queue->io_work);
949	read_unlock_bh(&sk->sk_callback_lock);
950	}
951
952	static void nvme_tcp_write_space(struct sock *sk)
953	{
954	struct nvme_tcp_queue *queue;
955
956	read_lock_bh(&sk->sk_callback_lock);
957	queue = sk->sk_user_data;
958	if (likely(queue && sk_stream_is_writeable(sk))) {
959	clear_bit(SOCK_NOSPACE, addr: &sk->sk_socket->flags);
960	queue_work_on(cpu: queue->io_cpu, wq: nvme_tcp_wq, work: &queue->io_work);
961	}
962	read_unlock_bh(&sk->sk_callback_lock);
963	}
964
965	static void nvme_tcp_state_change(struct sock *sk)
966	{
967	struct nvme_tcp_queue *queue;
968
969	read_lock_bh(&sk->sk_callback_lock);
970	queue = sk->sk_user_data;
971	if (!queue)
972	goto done;
973
974	switch (sk->sk_state) {
975	case TCP_CLOSE:
976	case TCP_CLOSE_WAIT:
977	case TCP_LAST_ACK:
978	case TCP_FIN_WAIT1:
979	case TCP_FIN_WAIT2:
980	nvme_tcp_error_recovery(ctrl: &queue->ctrl->ctrl);
981	break;
982	default:
983	dev_info(queue->ctrl->ctrl.device,
984	"queue %d socket state %d\n",
985	nvme_tcp_queue_id(queue), sk->sk_state);
986	}
987
988	queue->state_change(sk);
989	done:
990	read_unlock_bh(&sk->sk_callback_lock);
991	}
992
993	static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
994	{
995	queue->request = NULL;
996	}
997
998	static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
999	{
1000	if (nvme_tcp_async_req(req)) {
1001	union nvme_result res = {};
1002
1003	nvme_complete_async_event(ctrl: &req->queue->ctrl->ctrl,
1004	cpu_to_le16(NVME_SC_HOST_PATH_ERROR), res: &res);
1005	} else {
1006	nvme_tcp_end_request(rq: blk_mq_rq_from_pdu(pdu: req),
1007	status: NVME_SC_HOST_PATH_ERROR);
1008	}
1009	}
1010
1011	static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
1012	{
1013	struct nvme_tcp_queue *queue = req->queue;
1014	int req_data_len = req->data_len;
1015	u32 h2cdata_left = req->h2cdata_left;
1016
1017	while (true) {
1018	struct bio_vec bvec;
1019	struct msghdr msg = {
1020	.msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES,
1021	};
1022	struct page *page = nvme_tcp_req_cur_page(req);
1023	size_t offset = nvme_tcp_req_cur_offset(req);
1024	size_t len = nvme_tcp_req_cur_length(req);
1025	bool last = nvme_tcp_pdu_last_send(req, len);
1026	int req_data_sent = req->data_sent;
1027	int ret;
1028
1029	if (last && !queue->data_digest && !nvme_tcp_queue_more(queue))
1030	msg.msg_flags |= MSG_EOR;
1031	else
1032	msg.msg_flags |= MSG_MORE;
1033
1034	if (!sendpage_ok(page))
1035	msg.msg_flags &= ~MSG_SPLICE_PAGES;
1036
1037	bvec_set_page(bv: &bvec, page, len, offset);
1038	iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: &bvec, nr_segs: 1, count: len);
1039	ret = sock_sendmsg(sock: queue->sock, msg: &msg);
1040	if (ret <= 0)
1041	return ret;
1042
1043	if (queue->data_digest)
1044	nvme_tcp_ddgst_update(hash: queue->snd_hash, page,
1045	off: offset, len: ret);
1046
1047	/*
1048	* update the request iterator except for the last payload send
1049	* in the request where we don't want to modify it as we may
1050	* compete with the RX path completing the request.
1051	*/
1052	if (req_data_sent + ret < req_data_len)
1053	nvme_tcp_advance_req(req, len: ret);
1054
1055	/* fully successful last send in current PDU */
1056	if (last && ret == len) {
1057	if (queue->data_digest) {
1058	nvme_tcp_ddgst_final(hash: queue->snd_hash,
1059	dgst: &req->ddgst);
1060	req->state = NVME_TCP_SEND_DDGST;
1061	req->offset = 0;
1062	} else {
1063	if (h2cdata_left)
1064	nvme_tcp_setup_h2c_data_pdu(req);
1065	else
1066	nvme_tcp_done_send_req(queue);
1067	}
1068	return 1;
1069	}
1070	}
1071	return -EAGAIN;
1072	}
1073
1074	static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
1075	{
1076	struct nvme_tcp_queue *queue = req->queue;
1077	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
1078	struct bio_vec bvec;
1079	struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, };
1080	bool inline_data = nvme_tcp_has_inline_data(req);
1081	u8 hdgst = nvme_tcp_hdgst_len(queue);
1082	int len = sizeof(*pdu) + hdgst - req->offset;
1083	int ret;
1084
1085	if (inline_data || nvme_tcp_queue_more(queue))
1086	msg.msg_flags |= MSG_MORE;
1087	else
1088	msg.msg_flags |= MSG_EOR;
1089
1090	if (queue->hdr_digest && !req->offset)
1091	nvme_tcp_hdgst(hash: queue->snd_hash, pdu, len: sizeof(*pdu));
1092
1093	bvec_set_virt(bv: &bvec, vaddr: (void *)pdu + req->offset, len);
1094	iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: &bvec, nr_segs: 1, count: len);
1095	ret = sock_sendmsg(sock: queue->sock, msg: &msg);
1096	if (unlikely(ret <= 0))
1097	return ret;
1098
1099	len -= ret;
1100	if (!len) {
1101	if (inline_data) {
1102	req->state = NVME_TCP_SEND_DATA;
1103	if (queue->data_digest)
1104	crypto_ahash_init(req: queue->snd_hash);
1105	} else {
1106	nvme_tcp_done_send_req(queue);
1107	}
1108	return 1;
1109	}
1110	req->offset += ret;
1111
1112	return -EAGAIN;
1113	}
1114
1115	static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
1116	{
1117	struct nvme_tcp_queue *queue = req->queue;
1118	struct nvme_tcp_data_pdu *pdu = nvme_tcp_req_data_pdu(req);
1119	struct bio_vec bvec;
1120	struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_MORE, };
1121	u8 hdgst = nvme_tcp_hdgst_len(queue);
1122	int len = sizeof(*pdu) - req->offset + hdgst;
1123	int ret;
1124
1125	if (queue->hdr_digest && !req->offset)
1126	nvme_tcp_hdgst(hash: queue->snd_hash, pdu, len: sizeof(*pdu));
1127
1128	if (!req->h2cdata_left)
1129	msg.msg_flags |= MSG_SPLICE_PAGES;
1130
1131	bvec_set_virt(bv: &bvec, vaddr: (void *)pdu + req->offset, len);
1132	iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: &bvec, nr_segs: 1, count: len);
1133	ret = sock_sendmsg(sock: queue->sock, msg: &msg);
1134	if (unlikely(ret <= 0))
1135	return ret;
1136
1137	len -= ret;
1138	if (!len) {
1139	req->state = NVME_TCP_SEND_DATA;
1140	if (queue->data_digest)
1141	crypto_ahash_init(req: queue->snd_hash);
1142	return 1;
1143	}
1144	req->offset += ret;
1145
1146	return -EAGAIN;
1147	}
1148
1149	static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req)
1150	{
1151	struct nvme_tcp_queue *queue = req->queue;
1152	size_t offset = req->offset;
1153	u32 h2cdata_left = req->h2cdata_left;
1154	int ret;
1155	struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1156	struct kvec iov = {
1157	.iov_base = (u8 *)&req->ddgst + req->offset,
1158	.iov_len = NVME_TCP_DIGEST_LENGTH - req->offset
1159	};
1160
1161	if (nvme_tcp_queue_more(queue))
1162	msg.msg_flags |= MSG_MORE;
1163	else
1164	msg.msg_flags |= MSG_EOR;
1165
1166	ret = kernel_sendmsg(sock: queue->sock, msg: &msg, vec: &iov, num: 1, len: iov.iov_len);
1167	if (unlikely(ret <= 0))
1168	return ret;
1169
1170	if (offset + ret == NVME_TCP_DIGEST_LENGTH) {
1171	if (h2cdata_left)
1172	nvme_tcp_setup_h2c_data_pdu(req);
1173	else
1174	nvme_tcp_done_send_req(queue);
1175	return 1;
1176	}
1177
1178	req->offset += ret;
1179	return -EAGAIN;
1180	}
1181
1182	static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
1183	{
1184	struct nvme_tcp_request *req;
1185	unsigned int noreclaim_flag;
1186	int ret = 1;
1187
1188	if (!queue->request) {
1189	queue->request = nvme_tcp_fetch_request(queue);
1190	if (!queue->request)
1191	return 0;
1192	}
1193	req = queue->request;
1194
1195	noreclaim_flag = memalloc_noreclaim_save();
1196	if (req->state == NVME_TCP_SEND_CMD_PDU) {
1197	ret = nvme_tcp_try_send_cmd_pdu(req);
1198	if (ret <= 0)
1199	goto done;
1200	if (!nvme_tcp_has_inline_data(req))
1201	goto out;
1202	}
1203
1204	if (req->state == NVME_TCP_SEND_H2C_PDU) {
1205	ret = nvme_tcp_try_send_data_pdu(req);
1206	if (ret <= 0)
1207	goto done;
1208	}
1209
1210	if (req->state == NVME_TCP_SEND_DATA) {
1211	ret = nvme_tcp_try_send_data(req);
1212	if (ret <= 0)
1213	goto done;
1214	}
1215
1216	if (req->state == NVME_TCP_SEND_DDGST)
1217	ret = nvme_tcp_try_send_ddgst(req);
1218	done:
1219	if (ret == -EAGAIN) {
1220	ret = 0;
1221	} else if (ret < 0) {
1222	dev_err(queue->ctrl->ctrl.device,
1223	"failed to send request %d\n", ret);
1224	nvme_tcp_fail_request(req: queue->request);
1225	nvme_tcp_done_send_req(queue);
1226	}
1227	out:
1228	memalloc_noreclaim_restore(flags: noreclaim_flag);
1229	return ret;
1230	}
1231
1232	static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
1233	{
1234	struct socket *sock = queue->sock;
1235	struct sock *sk = sock->sk;
1236	read_descriptor_t rd_desc;
1237	int consumed;
1238
1239	rd_desc.arg.data = queue;
1240	rd_desc.count = 1;
1241	lock_sock(sk);
1242	queue->nr_cqe = 0;
1243	consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
1244	release_sock(sk);
1245	return consumed;
1246	}
1247
1248	static void nvme_tcp_io_work(struct work_struct *w)
1249	{
1250	struct nvme_tcp_queue *queue =
1251	container_of(w, struct nvme_tcp_queue, io_work);
1252	unsigned long deadline = jiffies + msecs_to_jiffies(m: 1);
1253
1254	do {
1255	bool pending = false;
1256	int result;
1257
1258	if (mutex_trylock(lock: &queue->send_mutex)) {
1259	result = nvme_tcp_try_send(queue);
1260	mutex_unlock(lock: &queue->send_mutex);
1261	if (result > 0)
1262	pending = true;
1263	else if (unlikely(result < 0))
1264	break;
1265	}
1266
1267	result = nvme_tcp_try_recv(queue);
1268	if (result > 0)
1269	pending = true;
1270	else if (unlikely(result < 0))
1271	return;
1272
1273	if (!pending || !queue->rd_enabled)
1274	return;
1275
1276	} while (!time_after(jiffies, deadline)); /* quota is exhausted */
1277
1278	queue_work_on(cpu: queue->io_cpu, wq: nvme_tcp_wq, work: &queue->io_work);
1279	}
1280
1281	static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
1282	{
1283	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: queue->rcv_hash);
1284
1285	ahash_request_free(req: queue->rcv_hash);
1286	ahash_request_free(req: queue->snd_hash);
1287	crypto_free_ahash(tfm);
1288	}
1289
1290	static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
1291	{
1292	struct crypto_ahash *tfm;
1293
1294	tfm = crypto_alloc_ahash(alg_name: "crc32c", type: 0, CRYPTO_ALG_ASYNC);
1295	if (IS_ERR(ptr: tfm))
1296	return PTR_ERR(ptr: tfm);
1297
1298	queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1299	if (!queue->snd_hash)
1300	goto free_tfm;
1301	ahash_request_set_callback(req: queue->snd_hash, flags: 0, NULL, NULL);
1302
1303	queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1304	if (!queue->rcv_hash)
1305	goto free_snd_hash;
1306	ahash_request_set_callback(req: queue->rcv_hash, flags: 0, NULL, NULL);
1307
1308	return 0;
1309	free_snd_hash:
1310	ahash_request_free(req: queue->snd_hash);
1311	free_tfm:
1312	crypto_free_ahash(tfm);
1313	return -ENOMEM;
1314	}
1315
1316	static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
1317	{
1318	struct nvme_tcp_request *async = &ctrl->async_req;
1319
1320	page_frag_free(addr: async->pdu);
1321	}
1322
1323	static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
1324	{
1325	struct nvme_tcp_queue *queue = &ctrl->queues[0];
1326	struct nvme_tcp_request *async = &ctrl->async_req;
1327	u8 hdgst = nvme_tcp_hdgst_len(queue);
1328
1329	async->pdu = page_frag_alloc(nc: &queue->pf_cache,
1330	fragsz: sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
1331	GFP_KERNEL | __GFP_ZERO);
1332	if (!async->pdu)
1333	return -ENOMEM;
1334
1335	async->queue = &ctrl->queues[0];
1336	return 0;
1337	}
1338
1339	static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
1340	{
1341	struct page *page;
1342	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(ctrl: nctrl);
1343	struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1344	unsigned int noreclaim_flag;
1345
1346	if (!test_and_clear_bit(nr: NVME_TCP_Q_ALLOCATED, addr: &queue->flags))
1347	return;
1348
1349	if (queue->hdr_digest || queue->data_digest)
1350	nvme_tcp_free_crypto(queue);
1351
1352	if (queue->pf_cache.va) {
1353	page = virt_to_head_page(x: queue->pf_cache.va);
1354	__page_frag_cache_drain(page, count: queue->pf_cache.pagecnt_bias);
1355	queue->pf_cache.va = NULL;
1356	}
1357
1358	noreclaim_flag = memalloc_noreclaim_save();
1359	/* ->sock will be released by fput() */
1360	fput(queue->sock->file);
1361	queue->sock = NULL;
1362	memalloc_noreclaim_restore(flags: noreclaim_flag);
1363
1364	kfree(objp: queue->pdu);
1365	mutex_destroy(lock: &queue->send_mutex);
1366	mutex_destroy(lock: &queue->queue_lock);
1367	}
1368
1369	static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
1370	{
1371	struct nvme_tcp_icreq_pdu *icreq;
1372	struct nvme_tcp_icresp_pdu *icresp;
1373	char cbuf[CMSG_LEN(sizeof(char))] = {};
1374	u8 ctype;
1375	struct msghdr msg = {};
1376	struct kvec iov;
1377	bool ctrl_hdgst, ctrl_ddgst;
1378	u32 maxh2cdata;
1379	int ret;
1380
1381	icreq = kzalloc(size: sizeof(*icreq), GFP_KERNEL);
1382	if (!icreq)
1383	return -ENOMEM;
1384
1385	icresp = kzalloc(size: sizeof(*icresp), GFP_KERNEL);
1386	if (!icresp) {
1387	ret = -ENOMEM;
1388	goto free_icreq;
1389	}
1390
1391	icreq->hdr.type = nvme_tcp_icreq;
1392	icreq->hdr.hlen = sizeof(*icreq);
1393	icreq->hdr.pdo = 0;
1394	icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
1395	icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
1396	icreq->maxr2t = 0; /* single inflight r2t supported */
1397	icreq->hpda = 0; /* no alignment constraint */
1398	if (queue->hdr_digest)
1399	icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
1400	if (queue->data_digest)
1401	icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
1402
1403	iov.iov_base = icreq;
1404	iov.iov_len = sizeof(*icreq);
1405	ret = kernel_sendmsg(sock: queue->sock, msg: &msg, vec: &iov, num: 1, len: iov.iov_len);
1406	if (ret < 0) {
1407	pr_warn("queue %d: failed to send icreq, error %d\n",
1408	nvme_tcp_queue_id(queue), ret);
1409	goto free_icresp;
1410	}
1411
1412	memset(&msg, 0, sizeof(msg));
1413	iov.iov_base = icresp;
1414	iov.iov_len = sizeof(*icresp);
1415	if (queue->ctrl->ctrl.opts->tls) {
1416	msg.msg_control = cbuf;
1417	msg.msg_controllen = sizeof(cbuf);
1418	}
1419	ret = kernel_recvmsg(sock: queue->sock, msg: &msg, vec: &iov, num: 1,
1420	len: iov.iov_len, flags: msg.msg_flags);
1421	if (ret < 0) {
1422	pr_warn("queue %d: failed to receive icresp, error %d\n",
1423	nvme_tcp_queue_id(queue), ret);
1424	goto free_icresp;
1425	}
1426	if (queue->ctrl->ctrl.opts->tls) {
1427	ctype = tls_get_record_type(sk: queue->sock->sk,
1428	msg: (struct cmsghdr *)cbuf);
1429	if (ctype != TLS_RECORD_TYPE_DATA) {
1430	pr_err("queue %d: unhandled TLS record %d\n",
1431	nvme_tcp_queue_id(queue), ctype);
1432	return -ENOTCONN;
1433	}
1434	}
1435	ret = -EINVAL;
1436	if (icresp->hdr.type != nvme_tcp_icresp) {
1437	pr_err("queue %d: bad type returned %d\n",
1438	nvme_tcp_queue_id(queue), icresp->hdr.type);
1439	goto free_icresp;
1440	}
1441
1442	if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) {
1443	pr_err("queue %d: bad pdu length returned %d\n",
1444	nvme_tcp_queue_id(queue), icresp->hdr.plen);
1445	goto free_icresp;
1446	}
1447
1448	if (icresp->pfv != NVME_TCP_PFV_1_0) {
1449	pr_err("queue %d: bad pfv returned %d\n",
1450	nvme_tcp_queue_id(queue), icresp->pfv);
1451	goto free_icresp;
1452	}
1453
1454	ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE);
1455	if ((queue->data_digest && !ctrl_ddgst) ||
1456	(!queue->data_digest && ctrl_ddgst)) {
1457	pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n",
1458	nvme_tcp_queue_id(queue),
1459	queue->data_digest ? "enabled" : "disabled",
1460	ctrl_ddgst ? "enabled" : "disabled");
1461	goto free_icresp;
1462	}
1463
1464	ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE);
1465	if ((queue->hdr_digest && !ctrl_hdgst) ||
1466	(!queue->hdr_digest && ctrl_hdgst)) {
1467	pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n",
1468	nvme_tcp_queue_id(queue),
1469	queue->hdr_digest ? "enabled" : "disabled",
1470	ctrl_hdgst ? "enabled" : "disabled");
1471	goto free_icresp;
1472	}
1473
1474	if (icresp->cpda != 0) {
1475	pr_err("queue %d: unsupported cpda returned %d\n",
1476	nvme_tcp_queue_id(queue), icresp->cpda);
1477	goto free_icresp;
1478	}
1479
1480	maxh2cdata = le32_to_cpu(icresp->maxdata);
1481	if ((maxh2cdata % 4) || (maxh2cdata < NVME_TCP_MIN_MAXH2CDATA)) {
1482	pr_err("queue %d: invalid maxh2cdata returned %u\n",
1483	nvme_tcp_queue_id(queue), maxh2cdata);
1484	goto free_icresp;
1485	}
1486	queue->maxh2cdata = maxh2cdata;
1487
1488	ret = 0;
1489	free_icresp:
1490	kfree(objp: icresp);
1491	free_icreq:
1492	kfree(objp: icreq);
1493	return ret;
1494	}
1495
1496	static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue)
1497	{
1498	return nvme_tcp_queue_id(queue) == 0;
1499	}
1500
1501	static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue)
1502	{
1503	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1504	int qid = nvme_tcp_queue_id(queue);
1505
1506	return !nvme_tcp_admin_queue(queue) &&
1507	qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT];
1508	}
1509
1510	static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue)
1511	{
1512	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1513	int qid = nvme_tcp_queue_id(queue);
1514
1515	return !nvme_tcp_admin_queue(queue) &&
1516	!nvme_tcp_default_queue(queue) &&
1517	qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1518	ctrl->io_queues[HCTX_TYPE_READ];
1519	}
1520
1521	static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)
1522	{
1523	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1524	int qid = nvme_tcp_queue_id(queue);
1525
1526	return !nvme_tcp_admin_queue(queue) &&
1527	!nvme_tcp_default_queue(queue) &&
1528	!nvme_tcp_read_queue(queue) &&
1529	qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1530	ctrl->io_queues[HCTX_TYPE_READ] +
1531	ctrl->io_queues[HCTX_TYPE_POLL];
1532	}
1533
1534	static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
1535	{
1536	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1537	int qid = nvme_tcp_queue_id(queue);
1538	int n = 0;
1539
1540	if (nvme_tcp_default_queue(queue))
1541	n = qid - 1;
1542	else if (nvme_tcp_read_queue(queue))
1543	n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1;
1544	else if (nvme_tcp_poll_queue(queue))
1545	n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] -
1546	ctrl->io_queues[HCTX_TYPE_READ] - 1;
1547	queue->io_cpu = cpumask_next_wrap(n: n - 1, cpu_online_mask, start: -1, wrap: false);
1548	}
1549
1550	#ifdef CONFIG_NVME_TCP_TLS
1551	static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid)
1552	{
1553	struct nvme_tcp_queue *queue = data;
1554	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1555	int qid = nvme_tcp_queue_id(queue);
1556	struct key *tls_key;
1557
1558	dev_dbg(ctrl->ctrl.device, "queue %d: TLS handshake done, key %x, status %d\n",
1559	qid, pskid, status);
1560
1561	if (status) {
1562	queue->tls_err = -status;
1563	goto out_complete;
1564	}
1565
1566	tls_key = key_lookup(id: pskid);
1567	if (IS_ERR(ptr: tls_key)) {
1568	dev_warn(ctrl->ctrl.device, "queue %d: Invalid key %x\n",
1569	qid, pskid);
1570	queue->tls_err = -ENOKEY;
1571	} else {
1572	ctrl->ctrl.tls_key = tls_key;
1573	queue->tls_err = 0;
1574	}
1575
1576	out_complete:
1577	complete(&queue->tls_complete);
1578	}
1579
1580	static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
1581	struct nvme_tcp_queue *queue,
1582	key_serial_t pskid)
1583	{
1584	int qid = nvme_tcp_queue_id(queue);
1585	int ret;
1586	struct tls_handshake_args args;
1587	unsigned long tmo = tls_handshake_timeout * HZ;
1588	key_serial_t keyring = nvme_keyring_id();
1589
1590	dev_dbg(nctrl->device, "queue %d: start TLS with key %x\n",
1591	qid, pskid);
1592	memset(&args, 0, sizeof(args));
1593	args.ta_sock = queue->sock;
1594	args.ta_done = nvme_tcp_tls_done;
1595	args.ta_data = queue;
1596	args.ta_my_peerids[0] = pskid;
1597	args.ta_num_peerids = 1;
1598	if (nctrl->opts->keyring)
1599	keyring = key_serial(key: nctrl->opts->keyring);
1600	args.ta_keyring = keyring;
1601	args.ta_timeout_ms = tls_handshake_timeout * 1000;
1602	queue->tls_err = -EOPNOTSUPP;
1603	init_completion(x: &queue->tls_complete);
1604	ret = tls_client_hello_psk(args: &args, GFP_KERNEL);
1605	if (ret) {
1606	dev_err(nctrl->device, "queue %d: failed to start TLS: %d\n",
1607	qid, ret);
1608	return ret;
1609	}
1610	ret = wait_for_completion_interruptible_timeout(x: &queue->tls_complete, timeout: tmo);
1611	if (ret <= 0) {
1612	if (ret == 0)
1613	ret = -ETIMEDOUT;
1614
1615	dev_err(nctrl->device,
1616	"queue %d: TLS handshake failed, error %d\n",
1617	qid, ret);
1618	tls_handshake_cancel(sk: queue->sock->sk);
1619	} else {
1620	dev_dbg(nctrl->device,
1621	"queue %d: TLS handshake complete, error %d\n",
1622	qid, queue->tls_err);
1623	ret = queue->tls_err;
1624	}
1625	return ret;
1626	}
1627	#else
1628	static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
1629	struct nvme_tcp_queue *queue,
1630	key_serial_t pskid)
1631	{
1632	return -EPROTONOSUPPORT;
1633	}
1634	#endif
1635
1636	static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
1637	key_serial_t pskid)
1638	{
1639	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(ctrl: nctrl);
1640	struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1641	int ret, rcv_pdu_size;
1642	struct file *sock_file;
1643
1644	mutex_init(&queue->queue_lock);
1645	queue->ctrl = ctrl;
1646	init_llist_head(list: &queue->req_list);
1647	INIT_LIST_HEAD(list: &queue->send_list);
1648	mutex_init(&queue->send_mutex);
1649	INIT_WORK(&queue->io_work, nvme_tcp_io_work);
1650
1651	if (qid > 0)
1652	queue->cmnd_capsule_len = nctrl->ioccsz * 16;
1653	else
1654	queue->cmnd_capsule_len = sizeof(struct nvme_command) +
1655	NVME_TCP_ADMIN_CCSZ;
1656
1657	ret = sock_create(family: ctrl->addr.ss_family, type: SOCK_STREAM,
1658	IPPROTO_TCP, res: &queue->sock);
1659	if (ret) {
1660	dev_err(nctrl->device,
1661	"failed to create socket: %d\n", ret);
1662	goto err_destroy_mutex;
1663	}
1664
1665	sock_file = sock_alloc_file(sock: queue->sock, O_CLOEXEC, NULL);
1666	if (IS_ERR(ptr: sock_file)) {
1667	ret = PTR_ERR(ptr: sock_file);
1668	goto err_destroy_mutex;
1669	}
1670	nvme_tcp_reclassify_socket(sock: queue->sock);
1671
1672	/* Single syn retry */
1673	tcp_sock_set_syncnt(sk: queue->sock->sk, val: 1);
1674
1675	/* Set TCP no delay */
1676	tcp_sock_set_nodelay(sk: queue->sock->sk);
1677
1678	/*
1679	* Cleanup whatever is sitting in the TCP transmit queue on socket
1680	* close. This is done to prevent stale data from being sent should
1681	* the network connection be restored before TCP times out.
1682	*/
1683	sock_no_linger(sk: queue->sock->sk);
1684
1685	if (so_priority > 0)
1686	sock_set_priority(sk: queue->sock->sk, priority: so_priority);
1687
1688	/* Set socket type of service */
1689	if (nctrl->opts->tos >= 0)
1690	ip_sock_set_tos(sk: queue->sock->sk, val: nctrl->opts->tos);
1691
1692	/* Set 10 seconds timeout for icresp recvmsg */
1693	queue->sock->sk->sk_rcvtimeo = 10 * HZ;
1694
1695	queue->sock->sk->sk_allocation = GFP_ATOMIC;
1696	queue->sock->sk->sk_use_task_frag = false;
1697	nvme_tcp_set_queue_io_cpu(queue);
1698	queue->request = NULL;
1699	queue->data_remaining = 0;
1700	queue->ddgst_remaining = 0;
1701	queue->pdu_remaining = 0;
1702	queue->pdu_offset = 0;
1703	sk_set_memalloc(sk: queue->sock->sk);
1704
1705	if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) {
1706	ret = kernel_bind(sock: queue->sock, addr: (struct sockaddr *)&ctrl->src_addr,
1707	addrlen: sizeof(ctrl->src_addr));
1708	if (ret) {
1709	dev_err(nctrl->device,
1710	"failed to bind queue %d socket %d\n",
1711	qid, ret);
1712	goto err_sock;
1713	}
1714	}
1715
1716	if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) {
1717	char *iface = nctrl->opts->host_iface;
1718	sockptr_t optval = KERNEL_SOCKPTR(p: iface);
1719
1720	ret = sock_setsockopt(sock: queue->sock, SOL_SOCKET, SO_BINDTODEVICE,
1721	optval, strlen(iface));
1722	if (ret) {
1723	dev_err(nctrl->device,
1724	"failed to bind to interface %s queue %d err %d\n",
1725	iface, qid, ret);
1726	goto err_sock;
1727	}
1728	}
1729
1730	queue->hdr_digest = nctrl->opts->hdr_digest;
1731	queue->data_digest = nctrl->opts->data_digest;
1732	if (queue->hdr_digest || queue->data_digest) {
1733	ret = nvme_tcp_alloc_crypto(queue);
1734	if (ret) {
1735	dev_err(nctrl->device,
1736	"failed to allocate queue %d crypto\n", qid);
1737	goto err_sock;
1738	}
1739	}
1740
1741	rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
1742	nvme_tcp_hdgst_len(queue);
1743	queue->pdu = kmalloc(size: rcv_pdu_size, GFP_KERNEL);
1744	if (!queue->pdu) {
1745	ret = -ENOMEM;
1746	goto err_crypto;
1747	}
1748
1749	dev_dbg(nctrl->device, "connecting queue %d\n",
1750	nvme_tcp_queue_id(queue));
1751
1752	ret = kernel_connect(sock: queue->sock, addr: (struct sockaddr *)&ctrl->addr,
1753	addrlen: sizeof(ctrl->addr), flags: 0);
1754	if (ret) {
1755	dev_err(nctrl->device,
1756	"failed to connect socket: %d\n", ret);
1757	goto err_rcv_pdu;
1758	}
1759
1760	/* If PSKs are configured try to start TLS */
1761	if (pskid) {
1762	ret = nvme_tcp_start_tls(nctrl, queue, pskid);
1763	if (ret)
1764	goto err_init_connect;
1765	}
1766
1767	ret = nvme_tcp_init_connection(queue);
1768	if (ret)
1769	goto err_init_connect;
1770
1771	set_bit(nr: NVME_TCP_Q_ALLOCATED, addr: &queue->flags);
1772
1773	return 0;
1774
1775	err_init_connect:
1776	kernel_sock_shutdown(sock: queue->sock, how: SHUT_RDWR);
1777	err_rcv_pdu:
1778	kfree(objp: queue->pdu);
1779	err_crypto:
1780	if (queue->hdr_digest || queue->data_digest)
1781	nvme_tcp_free_crypto(queue);
1782	err_sock:
1783	/* ->sock will be released by fput() */
1784	fput(queue->sock->file);
1785	queue->sock = NULL;
1786	err_destroy_mutex:
1787	mutex_destroy(lock: &queue->send_mutex);
1788	mutex_destroy(lock: &queue->queue_lock);
1789	return ret;
1790	}
1791
1792	static void nvme_tcp_restore_sock_ops(struct nvme_tcp_queue *queue)
1793	{
1794	struct socket *sock = queue->sock;
1795
1796	write_lock_bh(&sock->sk->sk_callback_lock);
1797	sock->sk->sk_user_data = NULL;
1798	sock->sk->sk_data_ready = queue->data_ready;
1799	sock->sk->sk_state_change = queue->state_change;
1800	sock->sk->sk_write_space = queue->write_space;
1801	write_unlock_bh(&sock->sk->sk_callback_lock);
1802	}
1803
1804	static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)
1805	{
1806	kernel_sock_shutdown(sock: queue->sock, how: SHUT_RDWR);
1807	nvme_tcp_restore_sock_ops(queue);
1808	cancel_work_sync(work: &queue->io_work);
1809	}
1810
1811	static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)
1812	{
1813	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(ctrl: nctrl);
1814	struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1815
1816	if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1817	return;
1818
1819	mutex_lock(&queue->queue_lock);
1820	if (test_and_clear_bit(nr: NVME_TCP_Q_LIVE, addr: &queue->flags))
1821	__nvme_tcp_stop_queue(queue);
1822	mutex_unlock(lock: &queue->queue_lock);
1823	}
1824
1825	static void nvme_tcp_setup_sock_ops(struct nvme_tcp_queue *queue)
1826	{
1827	write_lock_bh(&queue->sock->sk->sk_callback_lock);
1828	queue->sock->sk->sk_user_data = queue;
1829	queue->state_change = queue->sock->sk->sk_state_change;
1830	queue->data_ready = queue->sock->sk->sk_data_ready;
1831	queue->write_space = queue->sock->sk->sk_write_space;
1832	queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;
1833	queue->sock->sk->sk_state_change = nvme_tcp_state_change;
1834	queue->sock->sk->sk_write_space = nvme_tcp_write_space;
1835	#ifdef CONFIG_NET_RX_BUSY_POLL
1836	queue->sock->sk->sk_ll_usec = 1;
1837	#endif
1838	write_unlock_bh(&queue->sock->sk->sk_callback_lock);
1839	}
1840
1841	static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
1842	{
1843	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(ctrl: nctrl);
1844	struct nvme_tcp_queue *queue = &ctrl->queues[idx];
1845	int ret;
1846
1847	queue->rd_enabled = true;
1848	nvme_tcp_init_recv_ctx(queue);
1849	nvme_tcp_setup_sock_ops(queue);
1850
1851	if (idx)
1852	ret = nvmf_connect_io_queue(ctrl: nctrl, qid: idx);
1853	else
1854	ret = nvmf_connect_admin_queue(ctrl: nctrl);
1855
1856	if (!ret) {
1857	set_bit(nr: NVME_TCP_Q_LIVE, addr: &queue->flags);
1858	} else {
1859	if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1860	__nvme_tcp_stop_queue(queue);
1861	dev_err(nctrl->device,
1862	"failed to connect queue: %d ret=%d\n", idx, ret);
1863	}
1864	return ret;
1865	}
1866
1867	static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl)
1868	{
1869	if (to_tcp_ctrl(ctrl)->async_req.pdu) {
1870	cancel_work_sync(work: &ctrl->async_event_work);
1871	nvme_tcp_free_async_req(ctrl: to_tcp_ctrl(ctrl));
1872	to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
1873	}
1874
1875	nvme_tcp_free_queue(nctrl: ctrl, qid: 0);
1876	}
1877
1878	static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl)
1879	{
1880	int i;
1881
1882	for (i = 1; i < ctrl->queue_count; i++)
1883	nvme_tcp_free_queue(nctrl: ctrl, qid: i);
1884	}
1885
1886	static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
1887	{
1888	int i;
1889
1890	for (i = 1; i < ctrl->queue_count; i++)
1891	nvme_tcp_stop_queue(nctrl: ctrl, qid: i);
1892	}
1893
1894	static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl,
1895	int first, int last)
1896	{
1897	int i, ret;
1898
1899	for (i = first; i < last; i++) {
1900	ret = nvme_tcp_start_queue(nctrl: ctrl, idx: i);
1901	if (ret)
1902	goto out_stop_queues;
1903	}
1904
1905	return 0;
1906
1907	out_stop_queues:
1908	for (i--; i >= first; i--)
1909	nvme_tcp_stop_queue(nctrl: ctrl, qid: i);
1910	return ret;
1911	}
1912
1913	static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
1914	{
1915	int ret;
1916	key_serial_t pskid = 0;
1917
1918	if (ctrl->opts->tls) {
1919	if (ctrl->opts->tls_key)
1920	pskid = key_serial(key: ctrl->opts->tls_key);
1921	else
1922	pskid = nvme_tls_psk_default(keyring: ctrl->opts->keyring,
1923	hostnqn: ctrl->opts->host->nqn,
1924	subnqn: ctrl->opts->subsysnqn);
1925	if (!pskid) {
1926	dev_err(ctrl->device, "no valid PSK found\n");
1927	ret = -ENOKEY;
1928	goto out_free_queue;
1929	}
1930	}
1931
1932	ret = nvme_tcp_alloc_queue(nctrl: ctrl, qid: 0, pskid);
1933	if (ret)
1934	goto out_free_queue;
1935
1936	ret = nvme_tcp_alloc_async_req(ctrl: to_tcp_ctrl(ctrl));
1937	if (ret)
1938	goto out_free_queue;
1939
1940	return 0;
1941
1942	out_free_queue:
1943	nvme_tcp_free_queue(nctrl: ctrl, qid: 0);
1944	return ret;
1945	}
1946
1947	static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1948	{
1949	int i, ret;
1950
1951	if (ctrl->opts->tls && !ctrl->tls_key) {
1952	dev_err(ctrl->device, "no PSK negotiated\n");
1953	return -ENOKEY;
1954	}
1955	for (i = 1; i < ctrl->queue_count; i++) {
1956	ret = nvme_tcp_alloc_queue(nctrl: ctrl, qid: i,
1957	pskid: key_serial(key: ctrl->tls_key));
1958	if (ret)
1959	goto out_free_queues;
1960	}
1961
1962	return 0;
1963
1964	out_free_queues:
1965	for (i--; i >= 1; i--)
1966	nvme_tcp_free_queue(nctrl: ctrl, qid: i);
1967
1968	return ret;
1969	}
1970
1971	static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1972	{
1973	unsigned int nr_io_queues;
1974	int ret;
1975
1976	nr_io_queues = nvmf_nr_io_queues(opts: ctrl->opts);
1977	ret = nvme_set_queue_count(ctrl, count: &nr_io_queues);
1978	if (ret)
1979	return ret;
1980
1981	if (nr_io_queues == 0) {
1982	dev_err(ctrl->device,
1983	"unable to set any I/O queues\n");
1984	return -ENOMEM;
1985	}
1986
1987	ctrl->queue_count = nr_io_queues + 1;
1988	dev_info(ctrl->device,
1989	"creating %d I/O queues.\n", nr_io_queues);
1990
1991	nvmf_set_io_queues(opts: ctrl->opts, nr_io_queues,
1992	io_queues: to_tcp_ctrl(ctrl)->io_queues);
1993	return __nvme_tcp_alloc_io_queues(ctrl);
1994	}
1995
1996	static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)
1997	{
1998	nvme_tcp_stop_io_queues(ctrl);
1999	if (remove)
2000	nvme_remove_io_tag_set(ctrl);
2001	nvme_tcp_free_io_queues(ctrl);
2002	}
2003
2004	static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
2005	{
2006	int ret, nr_queues;
2007
2008	ret = nvme_tcp_alloc_io_queues(ctrl);
2009	if (ret)
2010	return ret;
2011
2012	if (new) {
2013	ret = nvme_alloc_io_tag_set(ctrl, set: &to_tcp_ctrl(ctrl)->tag_set,
2014	ops: &nvme_tcp_mq_ops,
2015	nr_maps: ctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2,
2016	cmd_size: sizeof(struct nvme_tcp_request));
2017	if (ret)
2018	goto out_free_io_queues;
2019	}
2020
2021	/*
2022	* Only start IO queues for which we have allocated the tagset
2023	* and limitted it to the available queues. On reconnects, the
2024	* queue number might have changed.
2025	*/
2026	nr_queues = min(ctrl->tagset->nr_hw_queues + 1, ctrl->queue_count);
2027	ret = nvme_tcp_start_io_queues(ctrl, first: 1, last: nr_queues);
2028	if (ret)
2029	goto out_cleanup_connect_q;
2030
2031	if (!new) {
2032	nvme_start_freeze(ctrl);
2033	nvme_unquiesce_io_queues(ctrl);
2034	if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) {
2035	/*
2036	* If we timed out waiting for freeze we are likely to
2037	* be stuck. Fail the controller initialization just
2038	* to be safe.
2039	*/
2040	ret = -ENODEV;
2041	nvme_unfreeze(ctrl);
2042	goto out_wait_freeze_timed_out;
2043	}
2044	blk_mq_update_nr_hw_queues(set: ctrl->tagset,
2045	nr_hw_queues: ctrl->queue_count - 1);
2046	nvme_unfreeze(ctrl);
2047	}
2048
2049	/*
2050	* If the number of queues has increased (reconnect case)
2051	* start all new queues now.
2052	*/
2053	ret = nvme_tcp_start_io_queues(ctrl, first: nr_queues,
2054	last: ctrl->tagset->nr_hw_queues + 1);
2055	if (ret)
2056	goto out_wait_freeze_timed_out;
2057
2058	return 0;
2059
2060	out_wait_freeze_timed_out:
2061	nvme_quiesce_io_queues(ctrl);
2062	nvme_sync_io_queues(ctrl);
2063	nvme_tcp_stop_io_queues(ctrl);
2064	out_cleanup_connect_q:
2065	nvme_cancel_tagset(ctrl);
2066	if (new)
2067	nvme_remove_io_tag_set(ctrl);
2068	out_free_io_queues:
2069	nvme_tcp_free_io_queues(ctrl);
2070	return ret;
2071	}
2072
2073	static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
2074	{
2075	nvme_tcp_stop_queue(nctrl: ctrl, qid: 0);
2076	if (remove)
2077	nvme_remove_admin_tag_set(ctrl);
2078	nvme_tcp_free_admin_queue(ctrl);
2079	}
2080
2081	static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
2082	{
2083	int error;
2084
2085	error = nvme_tcp_alloc_admin_queue(ctrl);
2086	if (error)
2087	return error;
2088
2089	if (new) {
2090	error = nvme_alloc_admin_tag_set(ctrl,
2091	set: &to_tcp_ctrl(ctrl)->admin_tag_set,
2092	ops: &nvme_tcp_admin_mq_ops,
2093	cmd_size: sizeof(struct nvme_tcp_request));
2094	if (error)
2095	goto out_free_queue;
2096	}
2097
2098	error = nvme_tcp_start_queue(nctrl: ctrl, idx: 0);
2099	if (error)
2100	goto out_cleanup_tagset;
2101
2102	error = nvme_enable_ctrl(ctrl);
2103	if (error)
2104	goto out_stop_queue;
2105
2106	nvme_unquiesce_admin_queue(ctrl);
2107
2108	error = nvme_init_ctrl_finish(ctrl, was_suspended: false);
2109	if (error)
2110	goto out_quiesce_queue;
2111
2112	return 0;
2113
2114	out_quiesce_queue:
2115	nvme_quiesce_admin_queue(ctrl);
2116	blk_sync_queue(q: ctrl->admin_q);
2117	out_stop_queue:
2118	nvme_tcp_stop_queue(nctrl: ctrl, qid: 0);
2119	nvme_cancel_admin_tagset(ctrl);
2120	out_cleanup_tagset:
2121	if (new)
2122	nvme_remove_admin_tag_set(ctrl);
2123	out_free_queue:
2124	nvme_tcp_free_admin_queue(ctrl);
2125	return error;
2126	}
2127
2128	static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,
2129	bool remove)
2130	{
2131	nvme_quiesce_admin_queue(ctrl);
2132	blk_sync_queue(q: ctrl->admin_q);
2133	nvme_tcp_stop_queue(nctrl: ctrl, qid: 0);
2134	nvme_cancel_admin_tagset(ctrl);
2135	if (remove)
2136	nvme_unquiesce_admin_queue(ctrl);
2137	nvme_tcp_destroy_admin_queue(ctrl, remove);
2138	}
2139
2140	static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
2141	bool remove)
2142	{
2143	if (ctrl->queue_count <= 1)
2144	return;
2145	nvme_quiesce_admin_queue(ctrl);
2146	nvme_quiesce_io_queues(ctrl);
2147	nvme_sync_io_queues(ctrl);
2148	nvme_tcp_stop_io_queues(ctrl);
2149	nvme_cancel_tagset(ctrl);
2150	if (remove)
2151	nvme_unquiesce_io_queues(ctrl);
2152	nvme_tcp_destroy_io_queues(ctrl, remove);
2153	}
2154
2155	static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
2156	{
2157	/* If we are resetting/deleting then do nothing */
2158	if (ctrl->state != NVME_CTRL_CONNECTING) {
2159	WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW ||
2160	ctrl->state == NVME_CTRL_LIVE);
2161	return;
2162	}
2163
2164	if (nvmf_should_reconnect(ctrl)) {
2165	dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
2166	ctrl->opts->reconnect_delay);
2167	queue_delayed_work(wq: nvme_wq, dwork: &to_tcp_ctrl(ctrl)->connect_work,
2168	delay: ctrl->opts->reconnect_delay * HZ);
2169	} else {
2170	dev_info(ctrl->device, "Removing controller...\n");
2171	nvme_delete_ctrl(ctrl);
2172	}
2173	}
2174
2175	static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
2176	{
2177	struct nvmf_ctrl_options *opts = ctrl->opts;
2178	int ret;
2179
2180	ret = nvme_tcp_configure_admin_queue(ctrl, new);
2181	if (ret)
2182	return ret;
2183
2184	if (ctrl->icdoff) {
2185	ret = -EOPNOTSUPP;
2186	dev_err(ctrl->device, "icdoff is not supported!\n");
2187	goto destroy_admin;
2188	}
2189
2190	if (!nvme_ctrl_sgl_supported(ctrl)) {
2191	ret = -EOPNOTSUPP;
2192	dev_err(ctrl->device, "Mandatory sgls are not supported!\n");
2193	goto destroy_admin;
2194	}
2195
2196	if (opts->queue_size > ctrl->sqsize + 1)
2197	dev_warn(ctrl->device,
2198	"queue_size %zu > ctrl sqsize %u, clamping down\n",
2199	opts->queue_size, ctrl->sqsize + 1);
2200
2201	if (ctrl->sqsize + 1 > ctrl->maxcmd) {
2202	dev_warn(ctrl->device,
2203	"sqsize %u > ctrl maxcmd %u, clamping down\n",
2204	ctrl->sqsize + 1, ctrl->maxcmd);
2205	ctrl->sqsize = ctrl->maxcmd - 1;
2206	}
2207
2208	if (ctrl->queue_count > 1) {
2209	ret = nvme_tcp_configure_io_queues(ctrl, new);
2210	if (ret)
2211	goto destroy_admin;
2212	}
2213
2214	if (!nvme_change_ctrl_state(ctrl, new_state: NVME_CTRL_LIVE)) {
2215	/*
2216	* state change failure is ok if we started ctrl delete,
2217	* unless we're during creation of a new controller to
2218	* avoid races with teardown flow.
2219	*/
2220	WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2221	ctrl->state != NVME_CTRL_DELETING_NOIO);
2222	WARN_ON_ONCE(new);
2223	ret = -EINVAL;
2224	goto destroy_io;
2225	}
2226
2227	nvme_start_ctrl(ctrl);
2228	return 0;
2229
2230	destroy_io:
2231	if (ctrl->queue_count > 1) {
2232	nvme_quiesce_io_queues(ctrl);
2233	nvme_sync_io_queues(ctrl);
2234	nvme_tcp_stop_io_queues(ctrl);
2235	nvme_cancel_tagset(ctrl);
2236	nvme_tcp_destroy_io_queues(ctrl, remove: new);
2237	}
2238	destroy_admin:
2239	nvme_quiesce_admin_queue(ctrl);
2240	blk_sync_queue(q: ctrl->admin_q);
2241	nvme_tcp_stop_queue(nctrl: ctrl, qid: 0);
2242	nvme_cancel_admin_tagset(ctrl);
2243	nvme_tcp_destroy_admin_queue(ctrl, remove: new);
2244	return ret;
2245	}
2246
2247	static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
2248	{
2249	struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
2250	struct nvme_tcp_ctrl, connect_work);
2251	struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2252
2253	++ctrl->nr_reconnects;
2254
2255	if (nvme_tcp_setup_ctrl(ctrl, new: false))
2256	goto requeue;
2257
2258	dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
2259	ctrl->nr_reconnects);
2260
2261	ctrl->nr_reconnects = 0;
2262
2263	return;
2264
2265	requeue:
2266	dev_info(ctrl->device, "Failed reconnect attempt %d\n",
2267	ctrl->nr_reconnects);
2268	nvme_tcp_reconnect_or_remove(ctrl);
2269	}
2270
2271	static void nvme_tcp_error_recovery_work(struct work_struct *work)
2272	{
2273	struct nvme_tcp_ctrl *tcp_ctrl = container_of(work,
2274	struct nvme_tcp_ctrl, err_work);
2275	struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2276
2277	nvme_stop_keep_alive(ctrl);
2278	flush_work(work: &ctrl->async_event_work);
2279	nvme_tcp_teardown_io_queues(ctrl, remove: false);
2280	/* unquiesce to fail fast pending requests */
2281	nvme_unquiesce_io_queues(ctrl);
2282	nvme_tcp_teardown_admin_queue(ctrl, remove: false);
2283	nvme_unquiesce_admin_queue(ctrl);
2284	nvme_auth_stop(ctrl);
2285
2286	if (!nvme_change_ctrl_state(ctrl, new_state: NVME_CTRL_CONNECTING)) {
2287	/* state change failure is ok if we started ctrl delete */
2288	WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2289	ctrl->state != NVME_CTRL_DELETING_NOIO);
2290	return;
2291	}
2292
2293	nvme_tcp_reconnect_or_remove(ctrl);
2294	}
2295
2296	static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2297	{
2298	nvme_tcp_teardown_io_queues(ctrl, remove: shutdown);
2299	nvme_quiesce_admin_queue(ctrl);
2300	nvme_disable_ctrl(ctrl, shutdown);
2301	nvme_tcp_teardown_admin_queue(ctrl, remove: shutdown);
2302	}
2303
2304	static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl)
2305	{
2306	nvme_tcp_teardown_ctrl(ctrl, shutdown: true);
2307	}
2308
2309	static void nvme_reset_ctrl_work(struct work_struct *work)
2310	{
2311	struct nvme_ctrl *ctrl =
2312	container_of(work, struct nvme_ctrl, reset_work);
2313
2314	nvme_stop_ctrl(ctrl);
2315	nvme_tcp_teardown_ctrl(ctrl, shutdown: false);
2316
2317	if (!nvme_change_ctrl_state(ctrl, new_state: NVME_CTRL_CONNECTING)) {
2318	/* state change failure is ok if we started ctrl delete */
2319	WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2320	ctrl->state != NVME_CTRL_DELETING_NOIO);
2321	return;
2322	}
2323
2324	if (nvme_tcp_setup_ctrl(ctrl, new: false))
2325	goto out_fail;
2326
2327	return;
2328
2329	out_fail:
2330	++ctrl->nr_reconnects;
2331	nvme_tcp_reconnect_or_remove(ctrl);
2332	}
2333
2334	static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
2335	{
2336	flush_work(work: &to_tcp_ctrl(ctrl)->err_work);
2337	cancel_delayed_work_sync(dwork: &to_tcp_ctrl(ctrl)->connect_work);
2338	}
2339
2340	static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
2341	{
2342	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(ctrl: nctrl);
2343
2344	if (list_empty(head: &ctrl->list))
2345	goto free_ctrl;
2346
2347	mutex_lock(&nvme_tcp_ctrl_mutex);
2348	list_del(entry: &ctrl->list);
2349	mutex_unlock(lock: &nvme_tcp_ctrl_mutex);
2350
2351	nvmf_free_options(opts: nctrl->opts);
2352	free_ctrl:
2353	kfree(objp: ctrl->queues);
2354	kfree(objp: ctrl);
2355	}
2356
2357	static void nvme_tcp_set_sg_null(struct nvme_command *c)
2358	{
2359	struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2360
2361	sg->addr = 0;
2362	sg->length = 0;
2363	sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2364	NVME_SGL_FMT_TRANSPORT_A;
2365	}
2366
2367	static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,
2368	struct nvme_command *c, u32 data_len)
2369	{
2370	struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2371
2372	sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
2373	sg->length = cpu_to_le32(data_len);
2374	sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
2375	}
2376
2377	static void nvme_tcp_set_sg_host_data(struct nvme_command *c,
2378	u32 data_len)
2379	{
2380	struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2381
2382	sg->addr = 0;
2383	sg->length = cpu_to_le32(data_len);
2384	sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2385	NVME_SGL_FMT_TRANSPORT_A;
2386	}
2387
2388	static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
2389	{
2390	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(ctrl: arg);
2391	struct nvme_tcp_queue *queue = &ctrl->queues[0];
2392	struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
2393	struct nvme_command *cmd = &pdu->cmd;
2394	u8 hdgst = nvme_tcp_hdgst_len(queue);
2395
2396	memset(pdu, 0, sizeof(*pdu));
2397	pdu->hdr.type = nvme_tcp_cmd;
2398	if (queue->hdr_digest)
2399	pdu->hdr.flags |= NVME_TCP_F_HDGST;
2400	pdu->hdr.hlen = sizeof(*pdu);
2401	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
2402
2403	cmd->common.opcode = nvme_admin_async_event;
2404	cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
2405	cmd->common.flags |= NVME_CMD_SGL_METABUF;
2406	nvme_tcp_set_sg_null(c: cmd);
2407
2408	ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU;
2409	ctrl->async_req.offset = 0;
2410	ctrl->async_req.curr_bio = NULL;
2411	ctrl->async_req.data_len = 0;
2412
2413	nvme_tcp_queue_request(req: &ctrl->async_req, sync: true, last: true);
2414	}
2415
2416	static void nvme_tcp_complete_timed_out(struct request *rq)
2417	{
2418	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2419	struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2420
2421	nvme_tcp_stop_queue(nctrl: ctrl, qid: nvme_tcp_queue_id(queue: req->queue));
2422	nvmf_complete_timed_out_request(rq);
2423	}
2424
2425	static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq)
2426	{
2427	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2428	struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2429	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2430	u8 opc = pdu->cmd.common.opcode, fctype = pdu->cmd.fabrics.fctype;
2431	int qid = nvme_tcp_queue_id(queue: req->queue);
2432
2433	dev_warn(ctrl->device,
2434	"queue %d: timeout cid %#x type %d opcode %#x (%s)\n",
2435	nvme_tcp_queue_id(req->queue), nvme_cid(rq), pdu->hdr.type,
2436	opc, nvme_opcode_str(qid, opc, fctype));
2437
2438	if (ctrl->state != NVME_CTRL_LIVE) {
2439	/*
2440	* If we are resetting, connecting or deleting we should
2441	* complete immediately because we may block controller
2442	* teardown or setup sequence
2443	* - ctrl disable/shutdown fabrics requests
2444	* - connect requests
2445	* - initialization admin requests
2446	* - I/O requests that entered after unquiescing and
2447	* the controller stopped responding
2448	*
2449	* All other requests should be cancelled by the error
2450	* recovery work, so it's fine that we fail it here.
2451	*/
2452	nvme_tcp_complete_timed_out(rq);
2453	return BLK_EH_DONE;
2454	}
2455
2456	/*
2457	* LIVE state should trigger the normal error recovery which will
2458	* handle completing this request.
2459	*/
2460	nvme_tcp_error_recovery(ctrl);
2461	return BLK_EH_RESET_TIMER;
2462	}
2463
2464	static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,
2465	struct request *rq)
2466	{
2467	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2468	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2469	struct nvme_command *c = &pdu->cmd;
2470
2471	c->common.flags |= NVME_CMD_SGL_METABUF;
2472
2473	if (!blk_rq_nr_phys_segments(rq))
2474	nvme_tcp_set_sg_null(c);
2475	else if (rq_data_dir(rq) == WRITE &&
2476	req->data_len <= nvme_tcp_inline_data_size(req))
2477	nvme_tcp_set_sg_inline(queue, c, data_len: req->data_len);
2478	else
2479	nvme_tcp_set_sg_host_data(c, data_len: req->data_len);
2480
2481	return 0;
2482	}
2483
2484	static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
2485	struct request *rq)
2486	{
2487	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2488	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2489	struct nvme_tcp_queue *queue = req->queue;
2490	u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;
2491	blk_status_t ret;
2492
2493	ret = nvme_setup_cmd(ns, req: rq);
2494	if (ret)
2495	return ret;
2496
2497	req->state = NVME_TCP_SEND_CMD_PDU;
2498	req->status = cpu_to_le16(NVME_SC_SUCCESS);
2499	req->offset = 0;
2500	req->data_sent = 0;
2501	req->pdu_len = 0;
2502	req->pdu_sent = 0;
2503	req->h2cdata_left = 0;
2504	req->data_len = blk_rq_nr_phys_segments(rq) ?
2505	blk_rq_payload_bytes(rq) : 0;
2506	req->curr_bio = rq->bio;
2507	if (req->curr_bio && req->data_len)
2508	nvme_tcp_init_iter(req, rq_data_dir(rq));
2509
2510	if (rq_data_dir(rq) == WRITE &&
2511	req->data_len <= nvme_tcp_inline_data_size(req))
2512	req->pdu_len = req->data_len;
2513
2514	pdu->hdr.type = nvme_tcp_cmd;
2515	pdu->hdr.flags = 0;
2516	if (queue->hdr_digest)
2517	pdu->hdr.flags |= NVME_TCP_F_HDGST;
2518	if (queue->data_digest && req->pdu_len) {
2519	pdu->hdr.flags |= NVME_TCP_F_DDGST;
2520	ddgst = nvme_tcp_ddgst_len(queue);
2521	}
2522	pdu->hdr.hlen = sizeof(*pdu);
2523	pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
2524	pdu->hdr.plen =
2525	cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);
2526
2527	ret = nvme_tcp_map_data(queue, rq);
2528	if (unlikely(ret)) {
2529	nvme_cleanup_cmd(req: rq);
2530	dev_err(queue->ctrl->ctrl.device,
2531	"Failed to map data (%d)\n", ret);
2532	return ret;
2533	}
2534
2535	return 0;
2536	}
2537
2538	static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)
2539	{
2540	struct nvme_tcp_queue *queue = hctx->driver_data;
2541
2542	if (!llist_empty(head: &queue->req_list))
2543	queue_work_on(cpu: queue->io_cpu, wq: nvme_tcp_wq, work: &queue->io_work);
2544	}
2545
2546	static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
2547	const struct blk_mq_queue_data *bd)
2548	{
2549	struct nvme_ns *ns = hctx->queue->queuedata;
2550	struct nvme_tcp_queue *queue = hctx->driver_data;
2551	struct request *rq = bd->rq;
2552	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2553	bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);
2554	blk_status_t ret;
2555
2556	if (!nvme_check_ready(ctrl: &queue->ctrl->ctrl, rq, queue_live: queue_ready))
2557	return nvme_fail_nonready_command(ctrl: &queue->ctrl->ctrl, req: rq);
2558
2559	ret = nvme_tcp_setup_cmd_pdu(ns, rq);
2560	if (unlikely(ret))
2561	return ret;
2562
2563	nvme_start_request(rq);
2564
2565	nvme_tcp_queue_request(req, sync: true, last: bd->last);
2566
2567	return BLK_STS_OK;
2568	}
2569
2570	static void nvme_tcp_map_queues(struct blk_mq_tag_set *set)
2571	{
2572	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(ctrl: set->driver_data);
2573
2574	nvmf_map_queues(set, ctrl: &ctrl->ctrl, io_queues: ctrl->io_queues);
2575	}
2576
2577	static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
2578	{
2579	struct nvme_tcp_queue *queue = hctx->driver_data;
2580	struct sock *sk = queue->sock->sk;
2581
2582	if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2583	return 0;
2584
2585	set_bit(nr: NVME_TCP_Q_POLLING, addr: &queue->flags);
2586	if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(list: &sk->sk_receive_queue))
2587	sk_busy_loop(sk, nonblock: true);
2588	nvme_tcp_try_recv(queue);
2589	clear_bit(nr: NVME_TCP_Q_POLLING, addr: &queue->flags);
2590	return queue->nr_cqe;
2591	}
2592
2593	static int nvme_tcp_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
2594	{
2595	struct nvme_tcp_queue *queue = &to_tcp_ctrl(ctrl)->queues[0];
2596	struct sockaddr_storage src_addr;
2597	int ret, len;
2598
2599	len = nvmf_get_address(ctrl, buf, size);
2600
2601	mutex_lock(&queue->queue_lock);
2602
2603	if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2604	goto done;
2605	ret = kernel_getsockname(sock: queue->sock, addr: (struct sockaddr *)&src_addr);
2606	if (ret > 0) {
2607	if (len > 0)
2608	len--; /* strip trailing newline */
2609	len += scnprintf(buf: buf + len, size: size - len, fmt: "%ssrc_addr=%pISc\n",
2610	(len) ? "," : "", &src_addr);
2611	}
2612	done:
2613	mutex_unlock(lock: &queue->queue_lock);
2614
2615	return len;
2616	}
2617
2618	static const struct blk_mq_ops nvme_tcp_mq_ops = {
2619	.queue_rq = nvme_tcp_queue_rq,
2620	.commit_rqs = nvme_tcp_commit_rqs,
2621	.complete = nvme_complete_rq,
2622	.init_request = nvme_tcp_init_request,
2623	.exit_request = nvme_tcp_exit_request,
2624	.init_hctx = nvme_tcp_init_hctx,
2625	.timeout = nvme_tcp_timeout,
2626	.map_queues = nvme_tcp_map_queues,
2627	.poll = nvme_tcp_poll,
2628	};
2629
2630	static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
2631	.queue_rq = nvme_tcp_queue_rq,
2632	.complete = nvme_complete_rq,
2633	.init_request = nvme_tcp_init_request,
2634	.exit_request = nvme_tcp_exit_request,
2635	.init_hctx = nvme_tcp_init_admin_hctx,
2636	.timeout = nvme_tcp_timeout,
2637	};
2638
2639	static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
2640	.name = "tcp",
2641	.module = THIS_MODULE,
2642	.flags = NVME_F_FABRICS | NVME_F_BLOCKING,
2643	.reg_read32 = nvmf_reg_read32,
2644	.reg_read64 = nvmf_reg_read64,
2645	.reg_write32 = nvmf_reg_write32,
2646	.free_ctrl = nvme_tcp_free_ctrl,
2647	.submit_async_event = nvme_tcp_submit_async_event,
2648	.delete_ctrl = nvme_tcp_delete_ctrl,
2649	.get_address = nvme_tcp_get_address,
2650	.stop_ctrl = nvme_tcp_stop_ctrl,
2651	};
2652
2653	static bool
2654	nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts)
2655	{
2656	struct nvme_tcp_ctrl *ctrl;
2657	bool found = false;
2658
2659	mutex_lock(&nvme_tcp_ctrl_mutex);
2660	list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) {
2661	found = nvmf_ip_options_match(ctrl: &ctrl->ctrl, opts);
2662	if (found)
2663	break;
2664	}
2665	mutex_unlock(lock: &nvme_tcp_ctrl_mutex);
2666
2667	return found;
2668	}
2669
2670	static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
2671	struct nvmf_ctrl_options *opts)
2672	{
2673	struct nvme_tcp_ctrl *ctrl;
2674	int ret;
2675
2676	ctrl = kzalloc(size: sizeof(*ctrl), GFP_KERNEL);
2677	if (!ctrl)
2678	return ERR_PTR(error: -ENOMEM);
2679
2680	INIT_LIST_HEAD(list: &ctrl->list);
2681	ctrl->ctrl.opts = opts;
2682	ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
2683	opts->nr_poll_queues + 1;
2684	ctrl->ctrl.sqsize = opts->queue_size - 1;
2685	ctrl->ctrl.kato = opts->kato;
2686
2687	INIT_DELAYED_WORK(&ctrl->connect_work,
2688	nvme_tcp_reconnect_ctrl_work);
2689	INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);
2690	INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);
2691
2692	if (!(opts->mask & NVMF_OPT_TRSVCID)) {
2693	opts->trsvcid =
2694	kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
2695	if (!opts->trsvcid) {
2696	ret = -ENOMEM;
2697	goto out_free_ctrl;
2698	}
2699	opts->mask |= NVMF_OPT_TRSVCID;
2700	}
2701
2702	ret = inet_pton_with_scope(net: &init_net, AF_UNSPEC,
2703	src: opts->traddr, port: opts->trsvcid, addr: &ctrl->addr);
2704	if (ret) {
2705	pr_err("malformed address passed: %s:%s\n",
2706	opts->traddr, opts->trsvcid);
2707	goto out_free_ctrl;
2708	}
2709
2710	if (opts->mask & NVMF_OPT_HOST_TRADDR) {
2711	ret = inet_pton_with_scope(net: &init_net, AF_UNSPEC,
2712	src: opts->host_traddr, NULL, addr: &ctrl->src_addr);
2713	if (ret) {
2714	pr_err("malformed src address passed: %s\n",
2715	opts->host_traddr);
2716	goto out_free_ctrl;
2717	}
2718	}
2719
2720	if (opts->mask & NVMF_OPT_HOST_IFACE) {
2721	if (!__dev_get_by_name(net: &init_net, name: opts->host_iface)) {
2722	pr_err("invalid interface passed: %s\n",
2723	opts->host_iface);
2724	ret = -ENODEV;
2725	goto out_free_ctrl;
2726	}
2727	}
2728
2729	if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) {
2730	ret = -EALREADY;
2731	goto out_free_ctrl;
2732	}
2733
2734	ctrl->queues = kcalloc(n: ctrl->ctrl.queue_count, size: sizeof(*ctrl->queues),
2735	GFP_KERNEL);
2736	if (!ctrl->queues) {
2737	ret = -ENOMEM;
2738	goto out_free_ctrl;
2739	}
2740
2741	ret = nvme_init_ctrl(ctrl: &ctrl->ctrl, dev, ops: &nvme_tcp_ctrl_ops, quirks: 0);
2742	if (ret)
2743	goto out_kfree_queues;
2744
2745	if (!nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_CONNECTING)) {
2746	WARN_ON_ONCE(1);
2747	ret = -EINTR;
2748	goto out_uninit_ctrl;
2749	}
2750
2751	ret = nvme_tcp_setup_ctrl(ctrl: &ctrl->ctrl, new: true);
2752	if (ret)
2753	goto out_uninit_ctrl;
2754
2755	dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
2756	nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr);
2757
2758	mutex_lock(&nvme_tcp_ctrl_mutex);
2759	list_add_tail(new: &ctrl->list, head: &nvme_tcp_ctrl_list);
2760	mutex_unlock(lock: &nvme_tcp_ctrl_mutex);
2761
2762	return &ctrl->ctrl;
2763
2764	out_uninit_ctrl:
2765	nvme_uninit_ctrl(ctrl: &ctrl->ctrl);
2766	nvme_put_ctrl(ctrl: &ctrl->ctrl);
2767	if (ret > 0)
2768	ret = -EIO;
2769	return ERR_PTR(error: ret);
2770	out_kfree_queues:
2771	kfree(objp: ctrl->queues);
2772	out_free_ctrl:
2773	kfree(objp: ctrl);
2774	return ERR_PTR(error: ret);
2775	}
2776
2777	static struct nvmf_transport_ops nvme_tcp_transport = {
2778	.name = "tcp",
2779	.module = THIS_MODULE,
2780	.required_opts = NVMF_OPT_TRADDR,
2781	.allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2782	NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
2783	NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST |
2784	NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES |
2785	NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE | NVMF_OPT_TLS |
2786	NVMF_OPT_KEYRING | NVMF_OPT_TLS_KEY,
2787	.create_ctrl = nvme_tcp_create_ctrl,
2788	};
2789
2790	static int __init nvme_tcp_init_module(void)
2791	{
2792	BUILD_BUG_ON(sizeof(struct nvme_tcp_hdr) != 8);
2793	BUILD_BUG_ON(sizeof(struct nvme_tcp_cmd_pdu) != 72);
2794	BUILD_BUG_ON(sizeof(struct nvme_tcp_data_pdu) != 24);
2795	BUILD_BUG_ON(sizeof(struct nvme_tcp_rsp_pdu) != 24);
2796	BUILD_BUG_ON(sizeof(struct nvme_tcp_r2t_pdu) != 24);
2797	BUILD_BUG_ON(sizeof(struct nvme_tcp_icreq_pdu) != 128);
2798	BUILD_BUG_ON(sizeof(struct nvme_tcp_icresp_pdu) != 128);
2799	BUILD_BUG_ON(sizeof(struct nvme_tcp_term_pdu) != 24);
2800
2801	nvme_tcp_wq = alloc_workqueue(fmt: "nvme_tcp_wq",
2802	flags: WQ_MEM_RECLAIM | WQ_HIGHPRI, max_active: 0);
2803	if (!nvme_tcp_wq)
2804	return -ENOMEM;
2805
2806	nvmf_register_transport(ops: &nvme_tcp_transport);
2807	return 0;
2808	}
2809
2810	static void __exit nvme_tcp_cleanup_module(void)
2811	{
2812	struct nvme_tcp_ctrl *ctrl;
2813
2814	nvmf_unregister_transport(ops: &nvme_tcp_transport);
2815
2816	mutex_lock(&nvme_tcp_ctrl_mutex);
2817	list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list)
2818	nvme_delete_ctrl(ctrl: &ctrl->ctrl);
2819	mutex_unlock(lock: &nvme_tcp_ctrl_mutex);
2820	flush_workqueue(nvme_delete_wq);
2821
2822	destroy_workqueue(wq: nvme_tcp_wq);
2823	}
2824
2825	module_init(nvme_tcp_init_module);
2826	module_exit(nvme_tcp_cleanup_module);
2827
2828	MODULE_LICENSE("GPL v2");
2829