1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (c) 2016 Chelsio Communications, Inc.
4	*/
5
6	#include <linux/workqueue.h>
7	#include <linux/kthread.h>
8	#include <linux/sched/signal.h>
9
10	#include <asm/unaligned.h>
11	#include <net/tcp.h>
12	#include <target/target_core_base.h>
13	#include <target/target_core_fabric.h>
14	#include "cxgbit.h"
15
16	struct sge_opaque_hdr {
17	void *dev;
18	dma_addr_t addr[MAX_SKB_FRAGS + 1];
19	};
20
21	static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
22
23	#define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
24	sizeof(struct fw_ofld_tx_data_wr))
25
26	static struct sk_buff *
27	__cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
28	{
29	struct sk_buff *skb = NULL;
30	u8 submode = 0;
31	int errcode;
32	static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
33
34	if (len) {
35	skb = alloc_skb_with_frags(header_len: hdr_len, data_len: len,
36	max_page_order: 0, errcode: &errcode,
37	GFP_KERNEL);
38	if (!skb)
39	return NULL;
40
41	skb_reserve(skb, TX_HDR_LEN);
42	skb_reset_transport_header(skb);
43	__skb_put(skb, ISCSI_HDR_LEN);
44	skb->data_len = len;
45	skb->len += len;
46	submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
47
48	} else {
49	u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
50
51	skb = alloc_skb(size: hdr_len + iso_len, GFP_KERNEL);
52	if (!skb)
53	return NULL;
54
55	skb_reserve(skb, TX_HDR_LEN + iso_len);
56	skb_reset_transport_header(skb);
57	__skb_put(skb, ISCSI_HDR_LEN);
58	}
59
60	submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
61	cxgbit_skcb_submode(skb) = submode;
62	cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
63	cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
64	return skb;
65	}
66
67	static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
68	{
69	return __cxgbit_alloc_skb(csk, len, iso: false);
70	}
71
72	/*
73	* cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
74	* @skb: the packet
75	*
76	* Returns true if a packet can be sent as an offload WR with immediate
77	* data. We currently use the same limit as for Ethernet packets.
78	*/
79	static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
80	{
81	int length = skb->len;
82
83	if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
84	length += sizeof(struct fw_ofld_tx_data_wr);
85
86	if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
87	length += sizeof(struct cpl_tx_data_iso);
88
89	return length <= MAX_IMM_OFLD_TX_DATA_WR_LEN;
90	}
91
92	/*
93	* cxgbit_sgl_len - calculates the size of an SGL of the given capacity
94	* @n: the number of SGL entries
95	* Calculates the number of flits needed for a scatter/gather list that
96	* can hold the given number of entries.
97	*/
98	static inline unsigned int cxgbit_sgl_len(unsigned int n)
99	{
100	n--;
101	return (3 * n) / 2 + (n & 1) + 2;
102	}
103
104	/*
105	* cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
106	* @skb: the packet
107	*
108	* Returns the number of flits needed for the given offload packet.
109	* These packets are already fully constructed and no additional headers
110	* will be added.
111	*/
112	static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
113	{
114	unsigned int flits, cnt;
115
116	if (cxgbit_is_ofld_imm(skb))
117	return DIV_ROUND_UP(skb->len, 8);
118	flits = skb_transport_offset(skb) / 8;
119	cnt = skb_shinfo(skb)->nr_frags;
120	if (skb_tail_pointer(skb) != skb_transport_header(skb))
121	cnt++;
122	return flits + cxgbit_sgl_len(n: cnt);
123	}
124
125	#define CXGBIT_ISO_FSLICE 0x1
126	#define CXGBIT_ISO_LSLICE 0x2
127	static void
128	cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
129	{
130	struct cpl_tx_data_iso *cpl;
131	unsigned int submode = cxgbit_skcb_submode(skb);
132	unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
133	unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
134
135	cpl = __skb_push(skb, sizeof(*cpl));
136
137	cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
138	CPL_TX_DATA_ISO_FIRST_V(fslice) |
139	CPL_TX_DATA_ISO_LAST_V(lslice) |
140	CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
141	CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
142	CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
143	CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
144	CPL_TX_DATA_ISO_SCSI_V(2));
145
146	cpl->ahs_len = 0;
147	cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
148	cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
149	cpl->len = htonl(iso_info->len);
150	cpl->reserved2_seglen_offset = htonl(0);
151	cpl->datasn_offset = htonl(0);
152	cpl->buffer_offset = htonl(0);
153	cpl->reserved3 = 0;
154
155	__skb_pull(skb, sizeof(*cpl));
156	}
157
158	static void
159	cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
160	u32 len, u32 credits, u32 compl)
161	{
162	struct fw_ofld_tx_data_wr *req;
163	const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
164	u32 submode = cxgbit_skcb_submode(skb);
165	u32 wr_ulp_mode = 0;
166	u32 hdr_size = sizeof(*req);
167	u32 opcode = FW_OFLD_TX_DATA_WR;
168	u32 immlen = 0;
169	u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :
170	T6_TX_FORCE_F;
171
172	if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
173	opcode = FW_ISCSI_TX_DATA_WR;
174	immlen += sizeof(struct cpl_tx_data_iso);
175	hdr_size += sizeof(struct cpl_tx_data_iso);
176	submode |= 8;
177	}
178
179	if (cxgbit_is_ofld_imm(skb))
180	immlen += dlen;
181
182	req = __skb_push(skb, len: hdr_size);
183	req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
184	FW_WR_COMPL_V(compl) |
185	FW_WR_IMMDLEN_V(immlen));
186	req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
187	FW_WR_LEN16_V(credits));
188	req->plen = htonl(len);
189	wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
190	FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
191
192	req->tunnel_to_proxy = htonl(wr_ulp_mode | force |
193	FW_OFLD_TX_DATA_WR_SHOVE_F);
194	}
195
196	static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
197	{
198	kfree_skb(skb);
199	}
200
201	void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
202	{
203	struct sk_buff *skb;
204
205	while (csk->wr_cred && ((skb = skb_peek(list_: &csk->txq)) != NULL)) {
206	u32 dlen = skb->len;
207	u32 len = skb->len;
208	u32 credits_needed;
209	u32 compl = 0;
210	u32 flowclen16 = 0;
211	u32 iso_cpl_len = 0;
212
213	if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
214	iso_cpl_len = sizeof(struct cpl_tx_data_iso);
215
216	if (cxgbit_is_ofld_imm(skb))
217	credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
218	else
219	credits_needed = DIV_ROUND_UP((8 *
220	cxgbit_calc_tx_flits_ofld(skb)) +
221	iso_cpl_len, 16);
222
223	if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
224	credits_needed += DIV_ROUND_UP(
225	sizeof(struct fw_ofld_tx_data_wr), 16);
226	/*
227	* Assumes the initial credits is large enough to support
228	* fw_flowc_wr plus largest possible first payload
229	*/
230
231	if (!test_and_set_bit(nr: CSK_TX_DATA_SENT, addr: &csk->com.flags)) {
232	flowclen16 = cxgbit_send_tx_flowc_wr(csk);
233	csk->wr_cred -= flowclen16;
234	csk->wr_una_cred += flowclen16;
235	}
236
237	if (csk->wr_cred < credits_needed) {
238	pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
239	csk, skb->len, skb->data_len,
240	credits_needed, csk->wr_cred);
241	break;
242	}
243	__skb_unlink(skb, list: &csk->txq);
244	set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
245	skb->csum = (__force __wsum)(credits_needed + flowclen16);
246	csk->wr_cred -= credits_needed;
247	csk->wr_una_cred += credits_needed;
248
249	pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
250	csk, skb->len, skb->data_len, credits_needed,
251	csk->wr_cred, csk->wr_una_cred);
252
253	if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
254	len += cxgbit_skcb_tx_extralen(skb);
255
256	if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
257	(!before(seq1: csk->write_seq,
258	seq2: csk->snd_una + csk->snd_win))) {
259	compl = 1;
260	csk->wr_una_cred = 0;
261	}
262
263	cxgbit_tx_data_wr(csk, skb, dlen, len, credits: credits_needed,
264	compl);
265	csk->snd_nxt += len;
266
267	} else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
268	(csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
269	struct cpl_close_con_req *req =
270	(struct cpl_close_con_req *)skb->data;
271	req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
272	csk->wr_una_cred = 0;
273	}
274
275	cxgbit_sock_enqueue_wr(csk, skb);
276	t4_set_arp_err_handler(skb, csk,
277	cxgbit_arp_failure_skb_discard);
278
279	pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
280	csk, csk->tid, skb, len);
281
282	cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
283	}
284	}
285
286	static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
287	{
288	struct sk_buff_head backlogq;
289	struct sk_buff *skb;
290	void (*fn)(struct cxgbit_sock *, struct sk_buff *);
291
292	skb_queue_head_init(list: &backlogq);
293
294	spin_lock_bh(lock: &csk->lock);
295	while (skb_queue_len(list_: &csk->backlogq)) {
296	skb_queue_splice_init(list: &csk->backlogq, head: &backlogq);
297	spin_unlock_bh(lock: &csk->lock);
298
299	while ((skb = __skb_dequeue(list: &backlogq))) {
300	fn = cxgbit_skcb_rx_backlog_fn(skb);
301	fn(csk, skb);
302	}
303
304	spin_lock_bh(lock: &csk->lock);
305	}
306
307	csk->lock_owner = false;
308	spin_unlock_bh(lock: &csk->lock);
309	}
310
311	static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
312	{
313	int ret = 0;
314
315	spin_lock_bh(lock: &csk->lock);
316	csk->lock_owner = true;
317	spin_unlock_bh(lock: &csk->lock);
318
319	if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
320	signal_pending(current))) {
321	__kfree_skb(skb);
322	__skb_queue_purge(list: &csk->ppodq);
323	ret = -1;
324	goto unlock;
325	}
326
327	csk->write_seq += skb->len +
328	cxgbit_skcb_tx_extralen(skb);
329
330	skb_queue_splice_tail_init(list: &csk->ppodq, head: &csk->txq);
331	__skb_queue_tail(list: &csk->txq, newsk: skb);
332	cxgbit_push_tx_frames(csk);
333
334	unlock:
335	cxgbit_unlock_sock(csk);
336	return ret;
337	}
338
339	static int
340	cxgbit_map_skb(struct iscsit_cmd *cmd, struct sk_buff *skb, u32 data_offset,
341	u32 data_length)
342	{
343	u32 i = 0, nr_frags = MAX_SKB_FRAGS;
344	u32 padding = ((-data_length) & 3);
345	struct scatterlist *sg;
346	struct page *page;
347	unsigned int page_off;
348
349	if (padding)
350	nr_frags--;
351
352	/*
353	* We know each entry in t_data_sg contains a page.
354	*/
355	sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
356	page_off = (data_offset % PAGE_SIZE);
357
358	while (data_length && (i < nr_frags)) {
359	u32 cur_len = min_t(u32, data_length, sg->length - page_off);
360
361	page = sg_page(sg);
362
363	get_page(page);
364	skb_fill_page_desc(skb, i, page, off: sg->offset + page_off,
365	size: cur_len);
366	skb->data_len += cur_len;
367	skb->len += cur_len;
368	skb->truesize += cur_len;
369
370	data_length -= cur_len;
371	page_off = 0;
372	sg = sg_next(sg);
373	i++;
374	}
375
376	if (data_length)
377	return -1;
378
379	if (padding) {
380	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
381	if (!page)
382	return -1;
383	skb_fill_page_desc(skb, i, page, off: 0, size: padding);
384	skb->data_len += padding;
385	skb->len += padding;
386	skb->truesize += padding;
387	}
388
389	return 0;
390	}
391
392	static int
393	cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsit_cmd *cmd,
394	struct iscsi_datain_req *dr)
395	{
396	struct iscsit_conn *conn = csk->conn;
397	struct sk_buff *skb;
398	struct iscsi_datain datain;
399	struct cxgbit_iso_info iso_info;
400	u32 data_length = cmd->se_cmd.data_length;
401	u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
402	u32 num_pdu, plen, tx_data = 0;
403	bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
404	SCF_TRANSPORT_TASK_SENSE);
405	bool set_statsn = false;
406	int ret = -1;
407
408	while (data_length) {
409	num_pdu = (data_length + mrdsl - 1) / mrdsl;
410	if (num_pdu > csk->max_iso_npdu)
411	num_pdu = csk->max_iso_npdu;
412
413	plen = num_pdu * mrdsl;
414	if (plen > data_length)
415	plen = data_length;
416
417	skb = __cxgbit_alloc_skb(csk, len: 0, iso: true);
418	if (unlikely(!skb))
419	return -ENOMEM;
420
421	memset(skb->data, 0, ISCSI_HDR_LEN);
422	cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
423	cxgbit_skcb_submode(skb) |= (csk->submode &
424	CXGBIT_SUBMODE_DCRC);
425	cxgbit_skcb_tx_extralen(skb) = (num_pdu *
426	cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
427	((num_pdu - 1) * ISCSI_HDR_LEN);
428
429	memset(&datain, 0, sizeof(struct iscsi_datain));
430	memset(&iso_info, 0, sizeof(iso_info));
431
432	if (!tx_data)
433	iso_info.flags |= CXGBIT_ISO_FSLICE;
434
435	if (!(data_length - plen)) {
436	iso_info.flags |= CXGBIT_ISO_LSLICE;
437	if (!task_sense) {
438	datain.flags = ISCSI_FLAG_DATA_STATUS;
439	iscsit_increment_maxcmdsn(cmd, conn->sess);
440	cmd->stat_sn = conn->stat_sn++;
441	set_statsn = true;
442	}
443	}
444
445	iso_info.burst_len = num_pdu * mrdsl;
446	iso_info.mpdu = mrdsl;
447	iso_info.len = ISCSI_HDR_LEN + plen;
448
449	cxgbit_cpl_tx_data_iso(skb, iso_info: &iso_info);
450
451	datain.offset = tx_data;
452	datain.data_sn = cmd->data_sn - 1;
453
454	iscsit_build_datain_pdu(cmd, conn, &datain,
455	(struct iscsi_data_rsp *)skb->data,
456	set_statsn);
457
458	ret = cxgbit_map_skb(cmd, skb, data_offset: tx_data, data_length: plen);
459	if (unlikely(ret)) {
460	__kfree_skb(skb);
461	goto out;
462	}
463
464	ret = cxgbit_queue_skb(csk, skb);
465	if (unlikely(ret))
466	goto out;
467
468	tx_data += plen;
469	data_length -= plen;
470
471	cmd->read_data_done += plen;
472	cmd->data_sn += num_pdu;
473	}
474
475	dr->dr_complete = DATAIN_COMPLETE_NORMAL;
476
477	return 0;
478
479	out:
480	return ret;
481	}
482
483	static int
484	cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsit_cmd *cmd,
485	const struct iscsi_datain *datain)
486	{
487	struct sk_buff *skb;
488	int ret = 0;
489
490	skb = cxgbit_alloc_skb(csk, len: 0);
491	if (unlikely(!skb))
492	return -ENOMEM;
493
494	memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
495
496	if (datain->length) {
497	cxgbit_skcb_submode(skb) |= (csk->submode &
498	CXGBIT_SUBMODE_DCRC);
499	cxgbit_skcb_tx_extralen(skb) =
500	cxgbit_digest_len[cxgbit_skcb_submode(skb)];
501	}
502
503	ret = cxgbit_map_skb(cmd, skb, data_offset: datain->offset, data_length: datain->length);
504	if (ret < 0) {
505	__kfree_skb(skb);
506	return ret;
507	}
508
509	return cxgbit_queue_skb(csk, skb);
510	}
511
512	static int
513	cxgbit_xmit_datain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
514	struct iscsi_datain_req *dr,
515	const struct iscsi_datain *datain)
516	{
517	struct cxgbit_sock *csk = conn->context;
518	u32 data_length = cmd->se_cmd.data_length;
519	u32 padding = ((-data_length) & 3);
520	u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
521
522	if ((data_length > mrdsl) && (!dr->recovery) &&
523	(!padding) && (!datain->offset) && csk->max_iso_npdu) {
524	atomic_long_add(i: data_length - datain->length,
525	v: &conn->sess->tx_data_octets);
526	return cxgbit_tx_datain_iso(csk, cmd, dr);
527	}
528
529	return cxgbit_tx_datain(csk, cmd, datain);
530	}
531
532	static int
533	cxgbit_xmit_nondatain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
534	const void *data_buf, u32 data_buf_len)
535	{
536	struct cxgbit_sock *csk = conn->context;
537	struct sk_buff *skb;
538	u32 padding = ((-data_buf_len) & 3);
539
540	skb = cxgbit_alloc_skb(csk, len: data_buf_len + padding);
541	if (unlikely(!skb))
542	return -ENOMEM;
543
544	memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
545
546	if (data_buf_len) {
547	u32 pad_bytes = 0;
548
549	skb_store_bits(skb, ISCSI_HDR_LEN, from: data_buf, len: data_buf_len);
550
551	if (padding)
552	skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
553	from: &pad_bytes, len: padding);
554	}
555
556	cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
557	cxgbit_skcb_submode(skb)];
558
559	return cxgbit_queue_skb(csk, skb);
560	}
561
562	int
563	cxgbit_xmit_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
564	struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
565	{
566	if (dr)
567	return cxgbit_xmit_datain_pdu(conn, cmd, dr, datain: buf);
568	else
569	return cxgbit_xmit_nondatain_pdu(conn, cmd, data_buf: buf, data_buf_len: buf_len);
570	}
571
572	int cxgbit_validate_params(struct iscsit_conn *conn)
573	{
574	struct cxgbit_sock *csk = conn->context;
575	struct cxgbit_device *cdev = csk->com.cdev;
576	struct iscsi_param *param;
577	u32 max_xmitdsl;
578
579	param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
580	conn->param_list);
581	if (!param)
582	return -1;
583
584	if (kstrtou32(s: param->value, base: 0, res: &max_xmitdsl) < 0)
585	return -1;
586
587	if (max_xmitdsl > cdev->mdsl) {
588	if (iscsi_change_param_sprintf(
589	conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
590	return -1;
591	}
592
593	return 0;
594	}
595
596	static int cxgbit_set_digest(struct cxgbit_sock *csk)
597	{
598	struct iscsit_conn *conn = csk->conn;
599	struct iscsi_param *param;
600
601	param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
602	if (!param) {
603	pr_err("param not found key %s\n", HEADERDIGEST);
604	return -1;
605	}
606
607	if (!strcmp(param->value, CRC32C))
608	csk->submode |= CXGBIT_SUBMODE_HCRC;
609
610	param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
611	if (!param) {
612	csk->submode = 0;
613	pr_err("param not found key %s\n", DATADIGEST);
614	return -1;
615	}
616
617	if (!strcmp(param->value, CRC32C))
618	csk->submode |= CXGBIT_SUBMODE_DCRC;
619
620	if (cxgbit_setup_conn_digest(csk)) {
621	csk->submode = 0;
622	return -1;
623	}
624
625	return 0;
626	}
627
628	static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
629	{
630	struct iscsit_conn *conn = csk->conn;
631	struct iscsi_conn_ops *conn_ops = conn->conn_ops;
632	struct iscsi_param *param;
633	u32 mrdsl, mbl;
634	u32 max_npdu, max_iso_npdu;
635	u32 max_iso_payload;
636
637	if (conn->login->leading_connection) {
638	param = iscsi_find_param_from_key(MAXBURSTLENGTH,
639	conn->param_list);
640	if (!param) {
641	pr_err("param not found key %s\n", MAXBURSTLENGTH);
642	return -1;
643	}
644
645	if (kstrtou32(s: param->value, base: 0, res: &mbl) < 0)
646	return -1;
647	} else {
648	mbl = conn->sess->sess_ops->MaxBurstLength;
649	}
650
651	mrdsl = conn_ops->MaxRecvDataSegmentLength;
652	max_npdu = mbl / mrdsl;
653
654	max_iso_payload = rounddown(CXGBIT_MAX_ISO_PAYLOAD, csk->emss);
655
656	max_iso_npdu = max_iso_payload /
657	(ISCSI_HDR_LEN + mrdsl +
658	cxgbit_digest_len[csk->submode]);
659
660	csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
661
662	if (csk->max_iso_npdu <= 1)
663	csk->max_iso_npdu = 0;
664
665	return 0;
666	}
667
668	/*
669	* cxgbit_seq_pdu_inorder()
670	* @csk: pointer to cxgbit socket structure
671	*
672	* This function checks whether data sequence and data
673	* pdu are in order.
674	*
675	* Return: returns -1 on error, 0 if data sequence and
676	* data pdu are in order, 1 if data sequence or data pdu
677	* is not in order.
678	*/
679	static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk)
680	{
681	struct iscsit_conn *conn = csk->conn;
682	struct iscsi_param *param;
683
684	if (conn->login->leading_connection) {
685	param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
686	conn->param_list);
687	if (!param) {
688	pr_err("param not found key %s\n", DATASEQUENCEINORDER);
689	return -1;
690	}
691
692	if (strcmp(param->value, YES))
693	return 1;
694
695	param = iscsi_find_param_from_key(DATAPDUINORDER,
696	conn->param_list);
697	if (!param) {
698	pr_err("param not found key %s\n", DATAPDUINORDER);
699	return -1;
700	}
701
702	if (strcmp(param->value, YES))
703	return 1;
704
705	} else {
706	if (!conn->sess->sess_ops->DataSequenceInOrder)
707	return 1;
708	if (!conn->sess->sess_ops->DataPDUInOrder)
709	return 1;
710	}
711
712	return 0;
713	}
714
715	static int cxgbit_set_params(struct iscsit_conn *conn)
716	{
717	struct cxgbit_sock *csk = conn->context;
718	struct cxgbit_device *cdev = csk->com.cdev;
719	struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
720	struct iscsi_conn_ops *conn_ops = conn->conn_ops;
721	struct iscsi_param *param;
722	u8 erl;
723
724	if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
725	conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
726
727	if (cxgbit_set_digest(csk))
728	return -1;
729
730	if (conn->login->leading_connection) {
731	param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
732	conn->param_list);
733	if (!param) {
734	pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
735	return -1;
736	}
737	if (kstrtou8(s: param->value, base: 0, res: &erl) < 0)
738	return -1;
739	} else {
740	erl = conn->sess->sess_ops->ErrorRecoveryLevel;
741	}
742
743	if (!erl) {
744	int ret;
745
746	ret = cxgbit_seq_pdu_inorder(csk);
747	if (ret < 0) {
748	return -1;
749	} else if (ret > 0) {
750	if (is_t5(cdev->lldi.adapter_type))
751	goto enable_ddp;
752	else
753	return 0;
754	}
755
756	if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
757	if (cxgbit_set_iso_npdu(csk))
758	return -1;
759	}
760
761	enable_ddp:
762	if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
763	if (cxgbit_setup_conn_pgidx(csk,
764	ppm->tformat.pgsz_idx_dflt))
765	return -1;
766	set_bit(nr: CSK_DDP_ENABLE, addr: &csk->com.flags);
767	}
768	}
769
770	return 0;
771	}
772
773	int
774	cxgbit_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login,
775	u32 length)
776	{
777	struct cxgbit_sock *csk = conn->context;
778	struct sk_buff *skb;
779	u32 padding_buf = 0;
780	u8 padding = ((-length) & 3);
781
782	skb = cxgbit_alloc_skb(csk, len: length + padding);
783	if (!skb)
784	return -ENOMEM;
785	skb_store_bits(skb, offset: 0, from: login->rsp, ISCSI_HDR_LEN);
786	skb_store_bits(skb, ISCSI_HDR_LEN, from: login->rsp_buf, len: length);
787
788	if (padding)
789	skb_store_bits(skb, ISCSI_HDR_LEN + length,
790	from: &padding_buf, len: padding);
791
792	if (login->login_complete) {
793	if (cxgbit_set_params(conn)) {
794	kfree_skb(skb);
795	return -1;
796	}
797
798	set_bit(nr: CSK_LOGIN_DONE, addr: &csk->com.flags);
799	}
800
801	if (cxgbit_queue_skb(csk, skb))
802	return -1;
803
804	if ((!login->login_complete) && (!login->login_failed))
805	schedule_delayed_work(dwork: &conn->login_work, delay: 0);
806
807	return 0;
808	}
809
810	static void
811	cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
812	unsigned int nents, u32 skip)
813	{
814	struct skb_seq_state st;
815	const u8 *buf;
816	unsigned int consumed = 0, buf_len;
817	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
818
819	skb_prepare_seq_read(skb, from: pdu_cb->doffset,
820	to: pdu_cb->doffset + pdu_cb->dlen,
821	st: &st);
822
823	while (true) {
824	buf_len = skb_seq_read(consumed, data: &buf, st: &st);
825	if (!buf_len) {
826	skb_abort_seq_read(st: &st);
827	break;
828	}
829
830	consumed += sg_pcopy_from_buffer(sgl: sg, nents, buf: (void *)buf,
831	buflen: buf_len, skip: skip + consumed);
832	}
833	}
834
835	static struct iscsit_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
836	{
837	struct iscsit_conn *conn = csk->conn;
838	struct cxgbi_ppm *ppm = cdev2ppm(cdev: csk->com.cdev);
839	struct cxgbit_cmd *ccmd;
840	struct iscsit_cmd *cmd;
841
842	cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
843	if (!cmd) {
844	pr_err("Unable to allocate iscsit_cmd + cxgbit_cmd\n");
845	return NULL;
846	}
847
848	ccmd = iscsit_priv_cmd(cmd);
849	ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
850	ccmd->setup_ddp = true;
851
852	return cmd;
853	}
854
855	static int
856	cxgbit_handle_immediate_data(struct iscsit_cmd *cmd, struct iscsi_scsi_req *hdr,
857	u32 length)
858	{
859	struct iscsit_conn *conn = cmd->conn;
860	struct cxgbit_sock *csk = conn->context;
861	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
862
863	if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
864	pr_err("ImmediateData CRC32C DataDigest error\n");
865	if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
866	pr_err("Unable to recover from"
867	" Immediate Data digest failure while"
868	" in ERL=0.\n");
869	iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
870	(unsigned char *)hdr);
871	return IMMEDIATE_DATA_CANNOT_RECOVER;
872	}
873
874	iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
875	(unsigned char *)hdr);
876	return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
877	}
878
879	if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
880	struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
881	struct skb_shared_info *ssi = skb_shinfo(csk->skb);
882	skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
883
884	sg_init_table(&ccmd->sg, 1);
885	sg_set_page(sg: &ccmd->sg, page: skb_frag_page(frag: dfrag),
886	len: skb_frag_size(frag: dfrag), offset: skb_frag_off(frag: dfrag));
887	get_page(page: skb_frag_page(frag: dfrag));
888
889	cmd->se_cmd.t_data_sg = &ccmd->sg;
890	cmd->se_cmd.t_data_nents = 1;
891
892	ccmd->release = true;
893	} else {
894	struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
895	u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
896
897	cxgbit_skb_copy_to_sg(skb: csk->skb, sg, nents: sg_nents, skip: 0);
898	}
899
900	cmd->write_data_done += pdu_cb->dlen;
901
902	if (cmd->write_data_done == cmd->se_cmd.data_length) {
903	spin_lock_bh(lock: &cmd->istate_lock);
904	cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
905	cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
906	spin_unlock_bh(lock: &cmd->istate_lock);
907	}
908
909	return IMMEDIATE_DATA_NORMAL_OPERATION;
910	}
911
912	static int
913	cxgbit_get_immediate_data(struct iscsit_cmd *cmd, struct iscsi_scsi_req *hdr,
914	bool dump_payload)
915	{
916	struct iscsit_conn *conn = cmd->conn;
917	int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
918	/*
919	* Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
920	*/
921	if (dump_payload)
922	goto after_immediate_data;
923
924	immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
925	length: cmd->first_burst_len);
926	after_immediate_data:
927	if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
928	/*
929	* A PDU/CmdSN carrying Immediate Data passed
930	* DataCRC, check against ExpCmdSN/MaxCmdSN if
931	* Immediate Bit is not set.
932	*/
933	cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
934	(unsigned char *)hdr,
935	hdr->cmdsn);
936	if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
937	return -1;
938
939	if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
940	target_put_sess_cmd(&cmd->se_cmd);
941	return 0;
942	} else if (cmd->unsolicited_data) {
943	iscsit_set_unsolicited_dataout(cmd);
944	}
945
946	} else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
947	/*
948	* Immediate Data failed DataCRC and ERL>=1,
949	* silently drop this PDU and let the initiator
950	* plug the CmdSN gap.
951	*
952	* FIXME: Send Unsolicited NOPIN with reserved
953	* TTT here to help the initiator figure out
954	* the missing CmdSN, although they should be
955	* intelligent enough to determine the missing
956	* CmdSN and issue a retry to plug the sequence.
957	*/
958	cmd->i_state = ISTATE_REMOVE;
959	iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
960	} else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
961	return -1;
962
963	return 0;
964	}
965
966	static int
967	cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsit_cmd *cmd)
968	{
969	struct iscsit_conn *conn = csk->conn;
970	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
971	struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
972	int rc;
973	bool dump_payload = false;
974
975	rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
976	if (rc < 0)
977	return rc;
978
979	if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
980	(pdu_cb->nr_dfrags == 1))
981	cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
982
983	rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
984	if (rc < 0)
985	return 0;
986	else if (rc > 0)
987	dump_payload = true;
988
989	if (!pdu_cb->dlen)
990	return 0;
991
992	return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
993	}
994
995	static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
996	{
997	struct scatterlist *sg_start;
998	struct iscsit_conn *conn = csk->conn;
999	struct iscsit_cmd *cmd = NULL;
1000	struct cxgbit_cmd *ccmd;
1001	struct cxgbi_task_tag_info *ttinfo;
1002	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1003	struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
1004	u32 data_offset = be32_to_cpu(hdr->offset);
1005	u32 data_len = ntoh24(hdr->dlength);
1006	int rc, sg_nents, sg_off;
1007	bool dcrc_err = false;
1008
1009	if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) {
1010	u32 offset = be32_to_cpu(hdr->offset);
1011	u32 ddp_data_len;
1012	bool success = false;
1013
1014	cmd = iscsit_find_cmd_from_itt_or_dump(conn, init_task_tag: hdr->itt, length: 0);
1015	if (!cmd)
1016	return 0;
1017
1018	ddp_data_len = offset - cmd->write_data_done;
1019	atomic_long_add(i: ddp_data_len, v: &conn->sess->rx_data_octets);
1020
1021	cmd->write_data_done = offset;
1022	cmd->next_burst_len = ddp_data_len;
1023	cmd->data_sn = be32_to_cpu(hdr->datasn);
1024
1025	rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr,
1026	cmd, data_len, &success);
1027	if (rc < 0)
1028	return rc;
1029	else if (!success)
1030	return 0;
1031	} else {
1032	rc = iscsit_check_dataout_hdr(conn, buf: (unsigned char *)hdr, out_cmd: &cmd);
1033	if (rc < 0)
1034	return rc;
1035	else if (!cmd)
1036	return 0;
1037	}
1038
1039	if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1040	pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1041	" DataSN: 0x%08x\n",
1042	hdr->itt, hdr->offset, data_len,
1043	hdr->datasn);
1044
1045	dcrc_err = true;
1046	goto check_payload;
1047	}
1048
1049	pr_debug("DataOut data_len: %u, "
1050	"write_data_done: %u, data_length: %u\n",
1051	data_len, cmd->write_data_done,
1052	cmd->se_cmd.data_length);
1053
1054	if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1055	u32 skip = data_offset % PAGE_SIZE;
1056
1057	sg_off = data_offset / PAGE_SIZE;
1058	sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1059	sg_nents = max(1UL, DIV_ROUND_UP(skip + data_len, PAGE_SIZE));
1060
1061	cxgbit_skb_copy_to_sg(skb: csk->skb, sg: sg_start, nents: sg_nents, skip);
1062	}
1063
1064	ccmd = iscsit_priv_cmd(cmd);
1065	ttinfo = &ccmd->ttinfo;
1066
1067	if (ccmd->release && ttinfo->sgl &&
1068	(cmd->se_cmd.data_length == (cmd->write_data_done + data_len))) {
1069	struct cxgbit_device *cdev = csk->com.cdev;
1070	struct cxgbi_ppm *ppm = cdev2ppm(cdev);
1071
1072	dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl, ttinfo->nents,
1073	DMA_FROM_DEVICE);
1074	ttinfo->nents = 0;
1075	ttinfo->sgl = NULL;
1076	}
1077
1078	check_payload:
1079
1080	rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1081	if (rc < 0)
1082	return rc;
1083
1084	return 0;
1085	}
1086
1087	static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsit_cmd *cmd)
1088	{
1089	struct iscsit_conn *conn = csk->conn;
1090	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1091	struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1092	unsigned char *ping_data = NULL;
1093	u32 payload_length = pdu_cb->dlen;
1094	int ret;
1095
1096	ret = iscsit_setup_nop_out(conn, cmd, hdr);
1097	if (ret < 0)
1098	return 0;
1099
1100	if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1101	if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1102	pr_err("Unable to recover from"
1103	" NOPOUT Ping DataCRC failure while in"
1104	" ERL=0.\n");
1105	ret = -1;
1106	goto out;
1107	} else {
1108	/*
1109	* drop this PDU and let the
1110	* initiator plug the CmdSN gap.
1111	*/
1112	pr_info("Dropping NOPOUT"
1113	" Command CmdSN: 0x%08x due to"
1114	" DataCRC error.\n", hdr->cmdsn);
1115	ret = 0;
1116	goto out;
1117	}
1118	}
1119
1120	/*
1121	* Handle NOP-OUT payload for traditional iSCSI sockets
1122	*/
1123	if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1124	ping_data = kzalloc(size: payload_length + 1, GFP_KERNEL);
1125	if (!ping_data) {
1126	pr_err("Unable to allocate memory for"
1127	" NOPOUT ping data.\n");
1128	ret = -1;
1129	goto out;
1130	}
1131
1132	skb_copy_bits(skb: csk->skb, offset: pdu_cb->doffset,
1133	to: ping_data, len: payload_length);
1134
1135	ping_data[payload_length] = '\0';
1136	/*
1137	* Attach ping data to struct iscsit_cmd->buf_ptr.
1138	*/
1139	cmd->buf_ptr = ping_data;
1140	cmd->buf_ptr_size = payload_length;
1141
1142	pr_debug("Got %u bytes of NOPOUT ping"
1143	" data.\n", payload_length);
1144	pr_debug("Ping Data: \"%s\"\n", ping_data);
1145	}
1146
1147	return iscsit_process_nop_out(conn, cmd, hdr);
1148	out:
1149	if (cmd)
1150	iscsit_free_cmd(cmd, false);
1151	return ret;
1152	}
1153
1154	static int
1155	cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsit_cmd *cmd)
1156	{
1157	struct iscsit_conn *conn = csk->conn;
1158	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1159	struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1160	u32 payload_length = pdu_cb->dlen;
1161	int rc;
1162	unsigned char *text_in = NULL;
1163
1164	rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1165	if (rc < 0)
1166	return rc;
1167
1168	if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1169	if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1170	pr_err("Unable to recover from"
1171	" Text Data digest failure while in"
1172	" ERL=0.\n");
1173	goto reject;
1174	} else {
1175	/*
1176	* drop this PDU and let the
1177	* initiator plug the CmdSN gap.
1178	*/
1179	pr_info("Dropping Text"
1180	" Command CmdSN: 0x%08x due to"
1181	" DataCRC error.\n", hdr->cmdsn);
1182	return 0;
1183	}
1184	}
1185
1186	if (payload_length) {
1187	text_in = kzalloc(size: payload_length, GFP_KERNEL);
1188	if (!text_in) {
1189	pr_err("Unable to allocate text_in of payload_length: %u\n",
1190	payload_length);
1191	return -ENOMEM;
1192	}
1193	skb_copy_bits(skb: csk->skb, offset: pdu_cb->doffset,
1194	to: text_in, len: payload_length);
1195
1196	text_in[payload_length - 1] = '\0';
1197
1198	cmd->text_in_ptr = text_in;
1199	}
1200
1201	return iscsit_process_text_cmd(conn, cmd, hdr);
1202
1203	reject:
1204	return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1205	pdu_cb->hdr);
1206	}
1207
1208	static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1209	{
1210	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1211	struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1212	struct iscsit_conn *conn = csk->conn;
1213	struct iscsit_cmd *cmd = NULL;
1214	u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1215	int ret = -EINVAL;
1216
1217	switch (opcode) {
1218	case ISCSI_OP_SCSI_CMD:
1219	cmd = cxgbit_allocate_cmd(csk);
1220	if (!cmd)
1221	goto reject;
1222
1223	ret = cxgbit_handle_scsi_cmd(csk, cmd);
1224	break;
1225	case ISCSI_OP_SCSI_DATA_OUT:
1226	ret = cxgbit_handle_iscsi_dataout(csk);
1227	break;
1228	case ISCSI_OP_NOOP_OUT:
1229	if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1230	cmd = cxgbit_allocate_cmd(csk);
1231	if (!cmd)
1232	goto reject;
1233	}
1234
1235	ret = cxgbit_handle_nop_out(csk, cmd);
1236	break;
1237	case ISCSI_OP_SCSI_TMFUNC:
1238	cmd = cxgbit_allocate_cmd(csk);
1239	if (!cmd)
1240	goto reject;
1241
1242	ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1243	(unsigned char *)hdr);
1244	break;
1245	case ISCSI_OP_TEXT:
1246	if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1247	cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1248	if (!cmd)
1249	goto reject;
1250	} else {
1251	cmd = cxgbit_allocate_cmd(csk);
1252	if (!cmd)
1253	goto reject;
1254	}
1255
1256	ret = cxgbit_handle_text_cmd(csk, cmd);
1257	break;
1258	case ISCSI_OP_LOGOUT:
1259	cmd = cxgbit_allocate_cmd(csk);
1260	if (!cmd)
1261	goto reject;
1262
1263	ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1264	if (ret > 0)
1265	wait_for_completion_timeout(x: &conn->conn_logout_comp,
1266	SECONDS_FOR_LOGOUT_COMP
1267	* HZ);
1268	break;
1269	case ISCSI_OP_SNACK:
1270	ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1271	break;
1272	default:
1273	pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1274	dump_stack();
1275	break;
1276	}
1277
1278	return ret;
1279
1280	reject:
1281	return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1282	(unsigned char *)hdr);
1283	return ret;
1284	}
1285
1286	static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1287	{
1288	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1289	struct iscsit_conn *conn = csk->conn;
1290	struct iscsi_hdr *hdr = pdu_cb->hdr;
1291	u8 opcode;
1292
1293	if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1294	atomic_long_inc(v: &conn->sess->conn_digest_errors);
1295	goto transport_err;
1296	}
1297
1298	if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1299	goto transport_err;
1300
1301	opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1302
1303	if (conn->sess->sess_ops->SessionType &&
1304	((!(opcode & ISCSI_OP_TEXT)) ||
1305	(!(opcode & ISCSI_OP_LOGOUT)))) {
1306	pr_err("Received illegal iSCSI Opcode: 0x%02x"
1307	" while in Discovery Session, rejecting.\n", opcode);
1308	iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1309	(unsigned char *)hdr);
1310	goto transport_err;
1311	}
1312
1313	if (cxgbit_target_rx_opcode(csk) < 0)
1314	goto transport_err;
1315
1316	return 0;
1317
1318	transport_err:
1319	return -1;
1320	}
1321
1322	static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1323	{
1324	struct iscsit_conn *conn = csk->conn;
1325	struct iscsi_login *login = conn->login;
1326	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1327	struct iscsi_login_req *login_req;
1328
1329	login_req = (struct iscsi_login_req *)login->req;
1330	memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1331
1332	pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1333	" CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1334	login_req->flags, login_req->itt, login_req->cmdsn,
1335	login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1336	/*
1337	* Setup the initial iscsi_login values from the leading
1338	* login request PDU.
1339	*/
1340	if (login->first_request) {
1341	login_req = (struct iscsi_login_req *)login->req;
1342	login->leading_connection = (!login_req->tsih) ? 1 : 0;
1343	login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(
1344	login_req->flags);
1345	login->version_min = login_req->min_version;
1346	login->version_max = login_req->max_version;
1347	memcpy(login->isid, login_req->isid, 6);
1348	login->cmd_sn = be32_to_cpu(login_req->cmdsn);
1349	login->init_task_tag = login_req->itt;
1350	login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1351	login->cid = be16_to_cpu(login_req->cid);
1352	login->tsih = be16_to_cpu(login_req->tsih);
1353	}
1354
1355	if (iscsi_target_check_login_request(conn, login) < 0)
1356	return -1;
1357
1358	memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1359	skb_copy_bits(skb: csk->skb, offset: pdu_cb->doffset, to: login->req_buf, len: pdu_cb->dlen);
1360
1361	return 0;
1362	}
1363
1364	static int
1365	cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1366	{
1367	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1368	int ret;
1369
1370	cxgbit_rx_pdu_cb(skb) = pdu_cb;
1371
1372	csk->skb = skb;
1373
1374	if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1375	ret = cxgbit_rx_login_pdu(csk);
1376	set_bit(nr: CSK_LOGIN_PDU_DONE, addr: &csk->com.flags);
1377	} else {
1378	ret = cxgbit_rx_opcode(csk);
1379	}
1380
1381	return ret;
1382	}
1383
1384	static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1385	{
1386	struct skb_shared_info *ssi = skb_shinfo(skb);
1387	struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1388	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1389	u8 i;
1390
1391	pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1392	skb, skb->head, skb->data, skb->len, skb->data_len,
1393	ssi->nr_frags);
1394	pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1395	skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1396
1397	for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1398	pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1399	"frags %u.\n",
1400	skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1401	pdu_cb->ddigest, pdu_cb->frags);
1402	for (i = 0; i < ssi->nr_frags; i++)
1403	pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1404	skb, i, skb_frag_off(&ssi->frags[i]),
1405	skb_frag_size(&ssi->frags[i]));
1406	}
1407
1408	static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1409	{
1410	struct sk_buff *skb = csk->lro_hskb;
1411	struct skb_shared_info *ssi = skb_shinfo(skb);
1412	u8 i;
1413
1414	memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1415	for (i = 0; i < ssi->nr_frags; i++)
1416	put_page(page: skb_frag_page(frag: &ssi->frags[i]));
1417	ssi->nr_frags = 0;
1418	skb->data_len = 0;
1419	skb->truesize -= skb->len;
1420	skb->len = 0;
1421	}
1422
1423	static void
1424	cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1425	{
1426	struct sk_buff *hskb = csk->lro_hskb;
1427	struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1428	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1429	struct skb_shared_info *hssi = skb_shinfo(hskb);
1430	struct skb_shared_info *ssi = skb_shinfo(skb);
1431	unsigned int len = 0;
1432
1433	if (pdu_cb->flags & PDUCBF_RX_HDR) {
1434	u8 hfrag_idx = hssi->nr_frags;
1435
1436	hpdu_cb->flags |= pdu_cb->flags;
1437	hpdu_cb->seq = pdu_cb->seq;
1438	hpdu_cb->hdr = pdu_cb->hdr;
1439	hpdu_cb->hlen = pdu_cb->hlen;
1440
1441	memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx],
1442	sizeof(skb_frag_t));
1443
1444	get_page(page: skb_frag_page(frag: &hssi->frags[hfrag_idx]));
1445	hssi->nr_frags++;
1446	hpdu_cb->frags++;
1447	hpdu_cb->hfrag_idx = hfrag_idx;
1448
1449	len = skb_frag_size(frag: &hssi->frags[hfrag_idx]);
1450	hskb->len += len;
1451	hskb->data_len += len;
1452	hskb->truesize += len;
1453	}
1454
1455	if (pdu_cb->flags & PDUCBF_RX_DATA) {
1456	u8 dfrag_idx = hssi->nr_frags, i;
1457
1458	hpdu_cb->flags |= pdu_cb->flags;
1459	hpdu_cb->dfrag_idx = dfrag_idx;
1460
1461	len = 0;
1462	for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) {
1463	memcpy(&hssi->frags[dfrag_idx],
1464	&ssi->frags[pdu_cb->dfrag_idx + i],
1465	sizeof(skb_frag_t));
1466
1467	get_page(page: skb_frag_page(frag: &hssi->frags[dfrag_idx]));
1468
1469	len += skb_frag_size(frag: &hssi->frags[dfrag_idx]);
1470
1471	hssi->nr_frags++;
1472	hpdu_cb->frags++;
1473	}
1474
1475	hpdu_cb->dlen = pdu_cb->dlen;
1476	hpdu_cb->doffset = hpdu_cb->hlen;
1477	hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1478	hskb->len += len;
1479	hskb->data_len += len;
1480	hskb->truesize += len;
1481	}
1482
1483	if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1484	hpdu_cb->flags |= pdu_cb->flags;
1485
1486	if (hpdu_cb->flags & PDUCBF_RX_DATA)
1487	hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1488
1489	hpdu_cb->ddigest = pdu_cb->ddigest;
1490	hpdu_cb->pdulen = pdu_cb->pdulen;
1491	}
1492	}
1493
1494	static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1495	{
1496	struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1497	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1498	u8 pdu_idx = 0, last_idx = 0;
1499	int ret = 0;
1500
1501	if (!pdu_cb->complete) {
1502	cxgbit_lro_skb_merge(csk, skb, pdu_idx: 0);
1503
1504	if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1505	struct sk_buff *hskb = csk->lro_hskb;
1506
1507	ret = cxgbit_process_iscsi_pdu(csk, skb: hskb, idx: 0);
1508
1509	cxgbit_lro_hskb_reset(csk);
1510
1511	if (ret < 0)
1512	goto out;
1513	}
1514
1515	pdu_idx = 1;
1516	}
1517
1518	if (lro_cb->pdu_idx)
1519	last_idx = lro_cb->pdu_idx - 1;
1520
1521	for (; pdu_idx <= last_idx; pdu_idx++) {
1522	ret = cxgbit_process_iscsi_pdu(csk, skb, idx: pdu_idx);
1523	if (ret < 0)
1524	goto out;
1525	}
1526
1527	if ((!lro_cb->complete) && lro_cb->pdu_idx)
1528	cxgbit_lro_skb_merge(csk, skb, pdu_idx: lro_cb->pdu_idx);
1529
1530	out:
1531	return ret;
1532	}
1533
1534	static int cxgbit_t5_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1535	{
1536	struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1537	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1538	int ret = -1;
1539
1540	if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1541	(pdu_cb->seq != csk->rcv_nxt)) {
1542	pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1543	csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1544	cxgbit_lro_skb_dump(skb);
1545	return ret;
1546	}
1547
1548	csk->rcv_nxt += lro_cb->pdu_totallen;
1549
1550	ret = cxgbit_process_lro_skb(csk, skb);
1551
1552	csk->rx_credits += lro_cb->pdu_totallen;
1553
1554	if (csk->rx_credits >= (csk->rcv_win / 4))
1555	cxgbit_rx_data_ack(csk);
1556
1557	return ret;
1558	}
1559
1560	static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1561	{
1562	struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1563	int ret;
1564
1565	ret = cxgbit_process_lro_skb(csk, skb);
1566	if (ret)
1567	return ret;
1568
1569	csk->rx_credits += lro_cb->pdu_totallen;
1570	if (csk->rx_credits >= csk->rcv_win) {
1571	csk->rx_credits = 0;
1572	cxgbit_rx_data_ack(csk);
1573	}
1574
1575	return 0;
1576	}
1577
1578	static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1579	{
1580	struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
1581	int ret = -1;
1582
1583	if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) {
1584	if (is_t5(lldi->adapter_type))
1585	ret = cxgbit_t5_rx_lro_skb(csk, skb);
1586	else
1587	ret = cxgbit_rx_lro_skb(csk, skb);
1588	}
1589
1590	__kfree_skb(skb);
1591	return ret;
1592	}
1593
1594	static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1595	{
1596	spin_lock_bh(lock: &csk->rxq.lock);
1597	if (skb_queue_len(list_: &csk->rxq)) {
1598	skb_queue_splice_init(list: &csk->rxq, head: rxq);
1599	spin_unlock_bh(lock: &csk->rxq.lock);
1600	return true;
1601	}
1602	spin_unlock_bh(lock: &csk->rxq.lock);
1603	return false;
1604	}
1605
1606	static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1607	{
1608	struct sk_buff *skb;
1609	struct sk_buff_head rxq;
1610
1611	skb_queue_head_init(list: &rxq);
1612
1613	wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1614
1615	if (signal_pending(current))
1616	goto out;
1617
1618	while ((skb = __skb_dequeue(list: &rxq))) {
1619	if (cxgbit_rx_skb(csk, skb))
1620	goto out;
1621	}
1622
1623	return 0;
1624	out:
1625	__skb_queue_purge(list: &rxq);
1626	return -1;
1627	}
1628
1629	int cxgbit_get_login_rx(struct iscsit_conn *conn, struct iscsi_login *login)
1630	{
1631	struct cxgbit_sock *csk = conn->context;
1632	int ret = -1;
1633
1634	while (!test_and_clear_bit(nr: CSK_LOGIN_PDU_DONE, addr: &csk->com.flags)) {
1635	ret = cxgbit_wait_rxq(csk);
1636	if (ret) {
1637	clear_bit(nr: CSK_LOGIN_PDU_DONE, addr: &csk->com.flags);
1638	break;
1639	}
1640	}
1641
1642	return ret;
1643	}
1644
1645	void cxgbit_get_rx_pdu(struct iscsit_conn *conn)
1646	{
1647	struct cxgbit_sock *csk = conn->context;
1648
1649	while (!kthread_should_stop()) {
1650	iscsit_thread_check_cpumask(conn, current, mode: 0);
1651	if (cxgbit_wait_rxq(csk))
1652	return;
1653	}
1654	}
1655