1	/*
2	* cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
3	*
4	* Copyright (C) 2003-2015 Chelsio Communications. All rights reserved.
5	*
6	* This program is distributed in the hope that it will be useful, but WITHOUT
7	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
8	* FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
9	* release for licensing terms and conditions.
10	*
11	* Written by: Dimitris Michailidis (dm@chelsio.com)
12	* Karen Xie (kxie@chelsio.com)
13	*/
14
15	#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
16
17	#include <linux/module.h>
18	#include <linux/moduleparam.h>
19	#include <scsi/scsi_host.h>
20
21	#include "common.h"
22	#include "t3_cpl.h"
23	#include "t3cdev.h"
24	#include "cxgb3_defs.h"
25	#include "cxgb3_ctl_defs.h"
26	#include "cxgb3_offload.h"
27	#include "firmware_exports.h"
28	#include "cxgb3i.h"
29
30	static unsigned int dbg_level;
31	#include "../libcxgbi.h"
32
33	#define DRV_MODULE_NAME "cxgb3i"
34	#define DRV_MODULE_DESC "Chelsio T3 iSCSI Driver"
35	#define DRV_MODULE_VERSION "2.0.1-ko"
36	#define DRV_MODULE_RELDATE "Apr. 2015"
37
38	static char version[] =
39	DRV_MODULE_DESC " " DRV_MODULE_NAME
40	" v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
41
42	MODULE_AUTHOR("Chelsio Communications, Inc.");
43	MODULE_DESCRIPTION(DRV_MODULE_DESC);
44	MODULE_VERSION(DRV_MODULE_VERSION);
45	MODULE_LICENSE("GPL");
46
47	module_param(dbg_level, uint, 0644);
48	MODULE_PARM_DESC(dbg_level, "debug flag (default=0)");
49
50	static int cxgb3i_rcv_win = 256 * 1024;
51	module_param(cxgb3i_rcv_win, int, 0644);
52	MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)");
53
54	static int cxgb3i_snd_win = 128 * 1024;
55	module_param(cxgb3i_snd_win, int, 0644);
56	MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)");
57
58	static int cxgb3i_rx_credit_thres = 10 * 1024;
59	module_param(cxgb3i_rx_credit_thres, int, 0644);
60	MODULE_PARM_DESC(cxgb3i_rx_credit_thres,
61	"RX credits return threshold in bytes (default=10KB)");
62
63	static unsigned int cxgb3i_max_connect = 8 * 1024;
64	module_param(cxgb3i_max_connect, uint, 0644);
65	MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)");
66
67	static unsigned int cxgb3i_sport_base = 20000;
68	module_param(cxgb3i_sport_base, uint, 0644);
69	MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)");
70
71	static void cxgb3i_dev_open(struct t3cdev *);
72	static void cxgb3i_dev_close(struct t3cdev *);
73	static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32);
74
75	static struct cxgb3_client t3_client = {
76	.name = DRV_MODULE_NAME,
77	.handlers = cxgb3i_cpl_handlers,
78	.add = cxgb3i_dev_open,
79	.remove = cxgb3i_dev_close,
80	.event_handler = cxgb3i_dev_event_handler,
81	};
82
83	static const struct scsi_host_template cxgb3i_host_template = {
84	.module = THIS_MODULE,
85	.name = DRV_MODULE_NAME,
86	.proc_name = DRV_MODULE_NAME,
87	.can_queue = CXGB3I_SCSI_HOST_QDEPTH,
88	.queuecommand = iscsi_queuecommand,
89	.change_queue_depth = scsi_change_queue_depth,
90	.sg_tablesize = SG_ALL,
91	.max_sectors = 0xFFFF,
92	.cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
93	.eh_timed_out = iscsi_eh_cmd_timed_out,
94	.eh_abort_handler = iscsi_eh_abort,
95	.eh_device_reset_handler = iscsi_eh_device_reset,
96	.eh_target_reset_handler = iscsi_eh_recover_target,
97	.target_alloc = iscsi_target_alloc,
98	.dma_boundary = PAGE_SIZE - 1,
99	.this_id = -1,
100	.track_queue_depth = 1,
101	.cmd_size = sizeof(struct iscsi_cmd),
102	};
103
104	static struct iscsi_transport cxgb3i_iscsi_transport = {
105	.owner = THIS_MODULE,
106	.name = DRV_MODULE_NAME,
107	/* owner and name should be set already */
108	.caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
109	| CAP_DATADGST | CAP_DIGEST_OFFLOAD |
110	CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,
111	.attr_is_visible = cxgbi_attr_is_visible,
112	.get_host_param = cxgbi_get_host_param,
113	.set_host_param = cxgbi_set_host_param,
114	/* session management */
115	.create_session = cxgbi_create_session,
116	.destroy_session = cxgbi_destroy_session,
117	.get_session_param = iscsi_session_get_param,
118	/* connection management */
119	.create_conn = cxgbi_create_conn,
120	.bind_conn = cxgbi_bind_conn,
121	.unbind_conn = iscsi_conn_unbind,
122	.destroy_conn = iscsi_tcp_conn_teardown,
123	.start_conn = iscsi_conn_start,
124	.stop_conn = iscsi_conn_stop,
125	.get_conn_param = iscsi_conn_get_param,
126	.set_param = cxgbi_set_conn_param,
127	.get_stats = cxgbi_get_conn_stats,
128	/* pdu xmit req from user space */
129	.send_pdu = iscsi_conn_send_pdu,
130	/* task */
131	.init_task = iscsi_tcp_task_init,
132	.xmit_task = iscsi_tcp_task_xmit,
133	.cleanup_task = cxgbi_cleanup_task,
134	/* pdu */
135	.alloc_pdu = cxgbi_conn_alloc_pdu,
136	.init_pdu = cxgbi_conn_init_pdu,
137	.xmit_pdu = cxgbi_conn_xmit_pdu,
138	.parse_pdu_itt = cxgbi_parse_pdu_itt,
139	/* TCP connect/disconnect */
140	.get_ep_param = cxgbi_get_ep_param,
141	.ep_connect = cxgbi_ep_connect,
142	.ep_poll = cxgbi_ep_poll,
143	.ep_disconnect = cxgbi_ep_disconnect,
144	/* Error recovery timeout call */
145	.session_recovery_timedout = iscsi_session_recovery_timedout,
146	};
147
148	static struct scsi_transport_template *cxgb3i_stt;
149
150	/*
151	* CPL (Chelsio Protocol Language) defines a message passing interface between
152	* the host driver and Chelsio asic.
153	* The section below implments CPLs that related to iscsi tcp connection
154	* open/close/abort and data send/receive.
155	*/
156
157	static int push_tx_frames(struct cxgbi_sock *csk, int req_completion);
158
159	static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
160	const struct l2t_entry *e)
161	{
162	unsigned int wscale = cxgbi_sock_compute_wscale(win: csk->rcv_win);
163	struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head;
164
165	skb->priority = CPL_PRIORITY_SETUP;
166
167	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
168	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid));
169	req->local_port = csk->saddr.sin_port;
170	req->peer_port = csk->daddr.sin_port;
171	req->local_ip = csk->saddr.sin_addr.s_addr;
172	req->peer_ip = csk->daddr.sin_addr.s_addr;
173
174	req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS |
175	V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) |
176	V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx));
177	req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) |
178	V_RCV_BUFSIZ(csk->rcv_win >> 10));
179
180	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
181	"csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n",
182	csk, csk->state, csk->flags, csk->atid,
183	&req->local_ip, ntohs(req->local_port),
184	&req->peer_ip, ntohs(req->peer_port),
185	csk->mss_idx, e->idx, e->smt_idx);
186
187	l2t_send(dev: csk->cdev->lldev, skb, e: csk->l2t);
188	}
189
190	static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
191	{
192	cxgbi_sock_act_open_req_arp_failure(NULL, skb);
193	}
194
195	/*
196	* CPL connection close request: host ->
197	*
198	* Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
199	* the write queue (i.e., after any unsent txt data).
200	*/
201	static void send_close_req(struct cxgbi_sock *csk)
202	{
203	struct sk_buff *skb = csk->cpl_close;
204	struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
205	unsigned int tid = csk->tid;
206
207	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
208	"csk 0x%p,%u,0x%lx,%u.\n",
209	csk, csk->state, csk->flags, csk->tid);
210
211	csk->cpl_close = NULL;
212	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
213	req->wr.wr_lo = htonl(V_WR_TID(tid));
214	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
215	req->rsvd = htonl(csk->write_seq);
216
217	cxgbi_sock_skb_entail(csk, skb);
218	if (csk->state >= CTP_ESTABLISHED)
219	push_tx_frames(csk, req_completion: 1);
220	}
221
222	/*
223	* CPL connection abort request: host ->
224	*
225	* Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
226	* for the same connection and also that we do not try to send a message
227	* after the connection has closed.
228	*/
229	static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb)
230	{
231	struct cpl_abort_req *req = cplhdr(skb);
232
233	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
234	"t3dev 0x%p, tid %u, skb 0x%p.\n",
235	tdev, GET_TID(req), skb);
236	req->cmd = CPL_ABORT_NO_RST;
237	cxgb3_ofld_send(dev: tdev, skb);
238	}
239
240	static void send_abort_req(struct cxgbi_sock *csk)
241	{
242	struct sk_buff *skb = csk->cpl_abort_req;
243	struct cpl_abort_req *req;
244
245	if (unlikely(csk->state == CTP_ABORTING || !skb))
246	return;
247	cxgbi_sock_set_state(csk, state: CTP_ABORTING);
248	cxgbi_sock_set_flag(csk, flag: CTPF_ABORT_RPL_PENDING);
249	/* Purge the send queue so we don't send anything after an abort. */
250	cxgbi_sock_purge_write_queue(csk);
251
252	csk->cpl_abort_req = NULL;
253	req = (struct cpl_abort_req *)skb->head;
254	skb->priority = CPL_PRIORITY_DATA;
255	set_arp_failure_handler(skb, hnd: abort_arp_failure);
256	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
257	req->wr.wr_lo = htonl(V_WR_TID(csk->tid));
258	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));
259	req->rsvd0 = htonl(csk->snd_nxt);
260	req->rsvd1 = !cxgbi_sock_flag(csk, flag: CTPF_TX_DATA_SENT);
261	req->cmd = CPL_ABORT_SEND_RST;
262
263	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
264	"csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
265	csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,
266	req->rsvd1);
267
268	l2t_send(dev: csk->cdev->lldev, skb, e: csk->l2t);
269	}
270
271	/*
272	* CPL connection abort reply: host ->
273	*
274	* Send an ABORT_RPL message in response of the ABORT_REQ received.
275	*/
276	static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)
277	{
278	struct sk_buff *skb = csk->cpl_abort_rpl;
279	struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
280
281	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
282	"csk 0x%p,%u,0x%lx,%u, status %d.\n",
283	csk, csk->state, csk->flags, csk->tid, rst_status);
284
285	csk->cpl_abort_rpl = NULL;
286	skb->priority = CPL_PRIORITY_DATA;
287	rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
288	rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid));
289	OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));
290	rpl->cmd = rst_status;
291	cxgb3_ofld_send(dev: csk->cdev->lldev, skb);
292	}
293
294	/*
295	* CPL connection rx data ack: host ->
296	* Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
297	* credits sent.
298	*/
299	static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)
300	{
301	struct sk_buff *skb;
302	struct cpl_rx_data_ack *req;
303	u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
304
305	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
306	"csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n",
307	csk, csk->state, csk->flags, csk->tid, credits, dack);
308
309	skb = alloc_wr(wrlen: sizeof(*req), dlen: 0, GFP_ATOMIC);
310	if (!skb) {
311	pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);
312	return 0;
313	}
314	req = (struct cpl_rx_data_ack *)skb->head;
315	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
316	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid));
317	req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) |
318	V_RX_CREDITS(credits));
319	skb->priority = CPL_PRIORITY_ACK;
320	cxgb3_ofld_send(dev: csk->cdev->lldev, skb);
321	return credits;
322	}
323
324	/*
325	* CPL connection tx data: host ->
326	*
327	* Send iscsi PDU via TX_DATA CPL message. Returns the number of
328	* credits sent.
329	* Each TX_DATA consumes work request credit (wrs), so we need to keep track of
330	* how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
331	*/
332
333	static unsigned int wrlen __read_mostly;
334	static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
335
336	static void init_wr_tab(unsigned int wr_len)
337	{
338	int i;
339
340	if (skb_wrs[1]) /* already initialized */
341	return;
342	for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
343	int sgl_len = (3 * i) / 2 + (i & 1);
344
345	sgl_len += 3;
346	skb_wrs[i] = (sgl_len <= wr_len
347	? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
348	}
349	wrlen = wr_len * 8;
350	}
351
352	static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,
353	int len, int req_completion)
354	{
355	struct tx_data_wr *req;
356	struct l2t_entry *l2t = csk->l2t;
357
358	skb_reset_transport_header(skb);
359	req = __skb_push(skb, len: sizeof(*req));
360	req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) |
361	(req_completion ? F_WR_COMPL : 0));
362	req->wr_lo = htonl(V_WR_TID(csk->tid));
363	/* len includes the length of any HW ULP additions */
364	req->len = htonl(len);
365	/* V_TX_ULP_SUBMODE sets both the mode and submode */
366	req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_tx_ulp_mode(skb)) |
367	V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1)));
368	req->sndseq = htonl(csk->snd_nxt);
369	req->param = htonl(V_TX_PORT(l2t->smt_idx));
370
371	if (!cxgbi_sock_flag(csk, flag: CTPF_TX_DATA_SENT)) {
372	req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
373	V_TX_CPU_IDX(csk->rss_qid));
374	/* sendbuffer is in units of 32KB. */
375	req->param |= htonl(V_TX_SNDBUF(csk->snd_win >> 15));
376	cxgbi_sock_set_flag(csk, flag: CTPF_TX_DATA_SENT);
377	}
378	}
379
380	/*
381	* push_tx_frames -- start transmit
382	*
383	* Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
384	* connection's send queue and sends them on to T3. Must be called with the
385	* connection's lock held. Returns the amount of send buffer space that was
386	* freed as a result of sending queued data to T3.
387	*/
388
389	static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb)
390	{
391	kfree_skb(skb);
392	}
393
394	static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)
395	{
396	int total_size = 0;
397	struct sk_buff *skb;
398
399	if (unlikely(csk->state < CTP_ESTABLISHED ||
400	csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
401	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
402	"csk 0x%p,%u,0x%lx,%u, in closing state.\n",
403	csk, csk->state, csk->flags, csk->tid);
404	return 0;
405	}
406
407	while (csk->wr_cred && (skb = skb_peek(list_: &csk->write_queue)) != NULL) {
408	int len = skb->len; /* length before skb_push */
409	int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
410	int wrs_needed = skb_wrs[frags];
411
412	if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
413	wrs_needed = 1;
414
415	WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
416
417	if (csk->wr_cred < wrs_needed) {
418	log_debug(1 << CXGBI_DBG_PDU_TX,
419	"csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n",
420	csk, skb->len, skb->data_len, frags,
421	wrs_needed, csk->wr_cred);
422	break;
423	}
424
425	__skb_unlink(skb, list: &csk->write_queue);
426	skb->priority = CPL_PRIORITY_DATA;
427	skb->csum = wrs_needed; /* remember this until the WR_ACK */
428	csk->wr_cred -= wrs_needed;
429	csk->wr_una_cred += wrs_needed;
430	cxgbi_sock_enqueue_wr(csk, skb);
431
432	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
433	"csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, "
434	"left %u, unack %u.\n",
435	csk, skb->len, skb->data_len, frags, skb->csum,
436	csk->wr_cred, csk->wr_una_cred);
437
438	if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
439	if ((req_completion &&
440	csk->wr_una_cred == wrs_needed) ||
441	csk->wr_una_cred >= csk->wr_max_cred / 2) {
442	req_completion = 1;
443	csk->wr_una_cred = 0;
444	}
445	len += cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb));
446	make_tx_data_wr(csk, skb, len, req_completion);
447	csk->snd_nxt += len;
448	cxgbi_skcb_clear_flag(skb, flag: SKCBF_TX_NEED_HDR);
449	}
450	total_size += skb->truesize;
451	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
452	"csk 0x%p, tid 0x%x, send skb 0x%p.\n",
453	csk, csk->tid, skb);
454	set_arp_failure_handler(skb, hnd: arp_failure_skb_discard);
455	l2t_send(dev: csk->cdev->lldev, skb, e: csk->l2t);
456	}
457	return total_size;
458	}
459
460	/*
461	* Process a CPL_ACT_ESTABLISH message: -> host
462	* Updates connection state from an active establish CPL message. Runs with
463	* the connection lock held.
464	*/
465
466	static inline void free_atid(struct cxgbi_sock *csk)
467	{
468	if (cxgbi_sock_flag(csk, flag: CTPF_HAS_ATID)) {
469	cxgb3_free_atid(dev: csk->cdev->lldev, atid: csk->atid);
470	cxgbi_sock_clear_flag(csk, flag: CTPF_HAS_ATID);
471	cxgbi_sock_put(csk);
472	}
473	}
474
475	static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
476	{
477	struct cxgbi_sock *csk = ctx;
478	struct cpl_act_establish *req = cplhdr(skb);
479	unsigned int tid = GET_TID(req);
480	unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
481	u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */
482
483	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
484	"atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
485	atid, atid, csk, csk->state, csk->flags, rcv_isn);
486
487	cxgbi_sock_get(csk);
488	cxgbi_sock_set_flag(csk, flag: CTPF_HAS_TID);
489	csk->tid = tid;
490	cxgb3_insert_tid(dev: csk->cdev->lldev, client: &t3_client, ctx: csk, tid);
491
492	free_atid(csk);
493
494	csk->rss_qid = G_QNUM(ntohs(skb->csum));
495
496	spin_lock_bh(lock: &csk->lock);
497	if (csk->retry_timer.function) {
498	del_timer(timer: &csk->retry_timer);
499	csk->retry_timer.function = NULL;
500	}
501
502	if (unlikely(csk->state != CTP_ACTIVE_OPEN))
503	pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
504	csk, csk->state, csk->flags, csk->tid);
505
506	csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;
507	if (csk->rcv_win > (M_RCV_BUFSIZ << 10))
508	csk->rcv_wup -= csk->rcv_win - (M_RCV_BUFSIZ << 10);
509
510	cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
511
512	if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))
513	/* upper layer has requested closing */
514	send_abort_req(csk);
515	else {
516	if (skb_queue_len(list_: &csk->write_queue))
517	push_tx_frames(csk, req_completion: 1);
518	cxgbi_conn_tx_open(csk);
519	}
520
521	spin_unlock_bh(lock: &csk->lock);
522	__kfree_skb(skb);
523	return 0;
524	}
525
526	/*
527	* Process a CPL_ACT_OPEN_RPL message: -> host
528	* Handle active open failures.
529	*/
530	static int act_open_rpl_status_to_errno(int status)
531	{
532	switch (status) {
533	case CPL_ERR_CONN_RESET:
534	return -ECONNREFUSED;
535	case CPL_ERR_ARP_MISS:
536	return -EHOSTUNREACH;
537	case CPL_ERR_CONN_TIMEDOUT:
538	return -ETIMEDOUT;
539	case CPL_ERR_TCAM_FULL:
540	return -ENOMEM;
541	case CPL_ERR_CONN_EXIST:
542	return -EADDRINUSE;
543	default:
544	return -EIO;
545	}
546	}
547
548	static void act_open_retry_timer(struct timer_list *t)
549	{
550	struct cxgbi_sock *csk = from_timer(csk, t, retry_timer);
551	struct sk_buff *skb;
552
553	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
554	"csk 0x%p,%u,0x%lx,%u.\n",
555	csk, csk->state, csk->flags, csk->tid);
556
557	cxgbi_sock_get(csk);
558	spin_lock_bh(lock: &csk->lock);
559	skb = alloc_wr(wrlen: sizeof(struct cpl_act_open_req), dlen: 0, GFP_ATOMIC);
560	if (!skb)
561	cxgbi_sock_fail_act_open(csk, -ENOMEM);
562	else {
563	skb->sk = (struct sock *)csk;
564	set_arp_failure_handler(skb, hnd: act_open_arp_failure);
565	send_act_open_req(csk, skb, e: csk->l2t);
566	}
567	spin_unlock_bh(lock: &csk->lock);
568	cxgbi_sock_put(csk);
569	}
570
571	static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
572	{
573	struct cxgbi_sock *csk = ctx;
574	struct cpl_act_open_rpl *rpl = cplhdr(skb);
575
576	pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n",
577	csk, csk->state, csk->flags, csk->atid, rpl->status,
578	&csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
579	&csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
580
581	if (rpl->status != CPL_ERR_TCAM_FULL &&
582	rpl->status != CPL_ERR_CONN_EXIST &&
583	rpl->status != CPL_ERR_ARP_MISS)
584	cxgb3_queue_tid_release(dev: tdev, GET_TID(rpl));
585
586	cxgbi_sock_get(csk);
587	spin_lock_bh(lock: &csk->lock);
588	if (rpl->status == CPL_ERR_CONN_EXIST &&
589	csk->retry_timer.function != act_open_retry_timer) {
590	csk->retry_timer.function = act_open_retry_timer;
591	mod_timer(timer: &csk->retry_timer, expires: jiffies + HZ / 2);
592	} else
593	cxgbi_sock_fail_act_open(csk,
594	act_open_rpl_status_to_errno(status: rpl->status));
595
596	spin_unlock_bh(lock: &csk->lock);
597	cxgbi_sock_put(csk);
598	__kfree_skb(skb);
599	return 0;
600	}
601
602	/*
603	* Process PEER_CLOSE CPL messages: -> host
604	* Handle peer FIN.
605	*/
606	static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
607	{
608	struct cxgbi_sock *csk = ctx;
609
610	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
611	"csk 0x%p,%u,0x%lx,%u.\n",
612	csk, csk->state, csk->flags, csk->tid);
613
614	cxgbi_sock_rcv_peer_close(csk);
615	__kfree_skb(skb);
616	return 0;
617	}
618
619	/*
620	* Process CLOSE_CONN_RPL CPL message: -> host
621	* Process a peer ACK to our FIN.
622	*/
623	static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
624	void *ctx)
625	{
626	struct cxgbi_sock *csk = ctx;
627	struct cpl_close_con_rpl *rpl = cplhdr(skb);
628
629	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
630	"csk 0x%p,%u,0x%lx,%u, snxt %u.\n",
631	csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt));
632
633	cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));
634	__kfree_skb(skb);
635	return 0;
636	}
637
638	/*
639	* Process ABORT_REQ_RSS CPL message: -> host
640	* Process abort requests. If we are waiting for an ABORT_RPL we ignore this
641	* request except that we need to reply to it.
642	*/
643
644	static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,
645	int *need_rst)
646	{
647	switch (abort_reason) {
648	case CPL_ERR_BAD_SYN:
649	case CPL_ERR_CONN_RESET:
650	return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET;
651	case CPL_ERR_XMIT_TIMEDOUT:
652	case CPL_ERR_PERSIST_TIMEDOUT:
653	case CPL_ERR_FINWAIT2_TIMEDOUT:
654	case CPL_ERR_KEEPALIVE_TIMEDOUT:
655	return -ETIMEDOUT;
656	default:
657	return -EIO;
658	}
659	}
660
661	static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
662	{
663	const struct cpl_abort_req_rss *req = cplhdr(skb);
664	struct cxgbi_sock *csk = ctx;
665	int rst_status = CPL_ABORT_NO_RST;
666
667	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
668	"csk 0x%p,%u,0x%lx,%u.\n",
669	csk, csk->state, csk->flags, csk->tid);
670
671	if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
672	req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
673	goto done;
674	}
675
676	cxgbi_sock_get(csk);
677	spin_lock_bh(lock: &csk->lock);
678
679	if (!cxgbi_sock_flag(csk, flag: CTPF_ABORT_REQ_RCVD)) {
680	cxgbi_sock_set_flag(csk, flag: CTPF_ABORT_REQ_RCVD);
681	cxgbi_sock_set_state(csk, state: CTP_ABORTING);
682	goto out;
683	}
684
685	cxgbi_sock_clear_flag(csk, flag: CTPF_ABORT_REQ_RCVD);
686	send_abort_rpl(csk, rst_status);
687
688	if (!cxgbi_sock_flag(csk, flag: CTPF_ABORT_RPL_PENDING)) {
689	csk->err = abort_status_to_errno(csk, abort_reason: req->status, need_rst: &rst_status);
690	cxgbi_sock_closed(csk);
691	}
692
693	out:
694	spin_unlock_bh(lock: &csk->lock);
695	cxgbi_sock_put(csk);
696	done:
697	__kfree_skb(skb);
698	return 0;
699	}
700
701	/*
702	* Process ABORT_RPL_RSS CPL message: -> host
703	* Process abort replies. We only process these messages if we anticipate
704	* them as the coordination between SW and HW in this area is somewhat lacking
705	* and sometimes we get ABORT_RPLs after we are done with the connection that
706	* originated the ABORT_REQ.
707	*/
708	static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
709	{
710	struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
711	struct cxgbi_sock *csk = ctx;
712
713	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
714	"status 0x%x, csk 0x%p, s %u, 0x%lx.\n",
715	rpl->status, csk, csk ? csk->state : 0,
716	csk ? csk->flags : 0UL);
717	/*
718	* Ignore replies to post-close aborts indicating that the abort was
719	* requested too late. These connections are terminated when we get
720	* PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
721	* arrives the TID is either no longer used or it has been recycled.
722	*/
723	if (rpl->status == CPL_ERR_ABORT_FAILED)
724	goto rel_skb;
725	/*
726	* Sometimes we've already closed the connection, e.g., a post-close
727	* abort races with ABORT_REQ_RSS, the latter frees the connection
728	* expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
729	* but FW turns the ABORT_REQ into a regular one and so we get
730	* ABORT_RPL_RSS with status 0 and no connection.
731	*/
732	if (csk)
733	cxgbi_sock_rcv_abort_rpl(csk);
734	rel_skb:
735	__kfree_skb(skb);
736	return 0;
737	}
738
739	/*
740	* Process RX_ISCSI_HDR CPL message: -> host
741	* Handle received PDUs, the payload could be DDP'ed. If not, the payload
742	* follow after the bhs.
743	*/
744	static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
745	{
746	struct cxgbi_sock *csk = ctx;
747	struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
748	struct cpl_iscsi_hdr_norss data_cpl;
749	struct cpl_rx_data_ddp_norss ddp_cpl;
750	unsigned int hdr_len, data_len, status;
751	unsigned int len;
752	int err;
753
754	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
755	"csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n",
756	csk, csk->state, csk->flags, csk->tid, skb, skb->len);
757
758	spin_lock_bh(lock: &csk->lock);
759
760	if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
761	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
762	"csk 0x%p,%u,0x%lx,%u, bad state.\n",
763	csk, csk->state, csk->flags, csk->tid);
764	if (csk->state != CTP_ABORTING)
765	goto abort_conn;
766	else
767	goto discard;
768	}
769
770	cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq);
771	cxgbi_skcb_flags(skb) = 0;
772
773	skb_reset_transport_header(skb);
774	__skb_pull(skb, len: sizeof(struct cpl_iscsi_hdr));
775
776	len = hdr_len = ntohs(hdr_cpl->len);
777	/* msg coalesce is off or not enough data received */
778	if (skb->len <= hdr_len) {
779	pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n",
780	csk->cdev->ports[csk->port_id]->name, csk->tid,
781	skb->len, hdr_len);
782	goto abort_conn;
783	}
784	cxgbi_skcb_set_flag(skb, flag: SKCBF_RX_COALESCED);
785
786	err = skb_copy_bits(skb, offset: skb->len - sizeof(ddp_cpl), to: &ddp_cpl,
787	len: sizeof(ddp_cpl));
788	if (err < 0) {
789	pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n",
790	csk->cdev->ports[csk->port_id]->name, csk->tid,
791	skb->len, sizeof(ddp_cpl), err);
792	goto abort_conn;
793	}
794
795	cxgbi_skcb_set_flag(skb, flag: SKCBF_RX_STATUS);
796	cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len);
797	cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
798	status = ntohl(ddp_cpl.ddp_status);
799
800	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
801	"csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n",
802	csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status);
803
804	if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT))
805	cxgbi_skcb_set_flag(skb, flag: SKCBF_RX_HCRC_ERR);
806	if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT))
807	cxgbi_skcb_set_flag(skb, flag: SKCBF_RX_DCRC_ERR);
808	if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT))
809	cxgbi_skcb_set_flag(skb, flag: SKCBF_RX_PAD_ERR);
810
811	if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
812	err = skb_copy_bits(skb, offset: hdr_len, to: &data_cpl, len: sizeof(data_cpl));
813	if (err < 0) {
814	pr_err("%s: tid %u, cp %zu/%u failed %d.\n",
815	csk->cdev->ports[csk->port_id]->name,
816	csk->tid, sizeof(data_cpl), skb->len, err);
817	goto abort_conn;
818	}
819	data_len = ntohs(data_cpl.len);
820	log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX,
821	"skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n",
822	skb, data_len, cxgbi_skcb_rx_pdulen(skb), status);
823	len += sizeof(data_cpl) + data_len;
824	} else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT))
825	cxgbi_skcb_set_flag(skb, flag: SKCBF_RX_DATA_DDPD);
826
827	csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb);
828	__pskb_trim(skb, len);
829	__skb_queue_tail(list: &csk->receive_queue, newsk: skb);
830	cxgbi_conn_pdu_ready(csk);
831
832	spin_unlock_bh(lock: &csk->lock);
833	return 0;
834
835	abort_conn:
836	send_abort_req(csk);
837	discard:
838	spin_unlock_bh(lock: &csk->lock);
839	__kfree_skb(skb);
840	return 0;
841	}
842
843	/*
844	* Process TX_DATA_ACK CPL messages: -> host
845	* Process an acknowledgment of WR completion. Advance snd_una and send the
846	* next batch of work requests from the write queue.
847	*/
848	static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
849	{
850	struct cxgbi_sock *csk = ctx;
851	struct cpl_wr_ack *hdr = cplhdr(skb);
852
853	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
854	"csk 0x%p,%u,0x%lx,%u, cr %u.\n",
855	csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits));
856
857	cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1);
858	__kfree_skb(skb);
859	return 0;
860	}
861
862	/*
863	* for each connection, pre-allocate skbs needed for close/abort requests. So
864	* that we can service the request right away.
865	*/
866	static int alloc_cpls(struct cxgbi_sock *csk)
867	{
868	csk->cpl_close = alloc_wr(wrlen: sizeof(struct cpl_close_con_req), dlen: 0,
869	GFP_KERNEL);
870	if (!csk->cpl_close)
871	return -ENOMEM;
872	csk->cpl_abort_req = alloc_wr(wrlen: sizeof(struct cpl_abort_req), dlen: 0,
873	GFP_KERNEL);
874	if (!csk->cpl_abort_req)
875	goto free_cpl_skbs;
876
877	csk->cpl_abort_rpl = alloc_wr(wrlen: sizeof(struct cpl_abort_rpl), dlen: 0,
878	GFP_KERNEL);
879	if (!csk->cpl_abort_rpl)
880	goto free_cpl_skbs;
881
882	return 0;
883
884	free_cpl_skbs:
885	cxgbi_sock_free_cpl_skbs(csk);
886	return -ENOMEM;
887	}
888
889	static void l2t_put(struct cxgbi_sock *csk)
890	{
891	struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
892
893	if (csk->l2t) {
894	l2t_release(t: t3dev, e: csk->l2t);
895	csk->l2t = NULL;
896	cxgbi_sock_put(csk);
897	}
898	}
899
900	/*
901	* release_offload_resources - release offload resource
902	* Release resources held by an offload connection (TID, L2T entry, etc.)
903	*/
904	static void release_offload_resources(struct cxgbi_sock *csk)
905	{
906	struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
907
908	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
909	"csk 0x%p,%u,0x%lx,%u.\n",
910	csk, csk->state, csk->flags, csk->tid);
911
912	csk->rss_qid = 0;
913	cxgbi_sock_free_cpl_skbs(csk);
914
915	if (csk->wr_cred != csk->wr_max_cred) {
916	cxgbi_sock_purge_wr_queue(csk);
917	cxgbi_sock_reset_wr_list(csk);
918	}
919	l2t_put(csk);
920	if (cxgbi_sock_flag(csk, flag: CTPF_HAS_ATID))
921	free_atid(csk);
922	else if (cxgbi_sock_flag(csk, flag: CTPF_HAS_TID)) {
923	cxgb3_remove_tid(dev: t3dev, ctx: (void *)csk, tid: csk->tid);
924	cxgbi_sock_clear_flag(csk, flag: CTPF_HAS_TID);
925	cxgbi_sock_put(csk);
926	}
927	csk->dst = NULL;
928	csk->cdev = NULL;
929	}
930
931	static void update_address(struct cxgbi_hba *chba)
932	{
933	if (chba->ipv4addr) {
934	if (chba->vdev &&
935	chba->ipv4addr != cxgb3i_get_private_ipv4addr(ndev: chba->vdev)) {
936	cxgb3i_set_private_ipv4addr(ndev: chba->vdev, addr: chba->ipv4addr);
937	cxgb3i_set_private_ipv4addr(ndev: chba->ndev, addr: 0);
938	pr_info("%s set %pI4.\n",
939	chba->vdev->name, &chba->ipv4addr);
940	} else if (chba->ipv4addr !=
941	cxgb3i_get_private_ipv4addr(ndev: chba->ndev)) {
942	cxgb3i_set_private_ipv4addr(ndev: chba->ndev, addr: chba->ipv4addr);
943	pr_info("%s set %pI4.\n",
944	chba->ndev->name, &chba->ipv4addr);
945	}
946	} else if (cxgb3i_get_private_ipv4addr(ndev: chba->ndev)) {
947	if (chba->vdev)
948	cxgb3i_set_private_ipv4addr(ndev: chba->vdev, addr: 0);
949	cxgb3i_set_private_ipv4addr(ndev: chba->ndev, addr: 0);
950	}
951	}
952
953	static int init_act_open(struct cxgbi_sock *csk)
954	{
955	struct dst_entry *dst = csk->dst;
956	struct cxgbi_device *cdev = csk->cdev;
957	struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
958	struct net_device *ndev = cdev->ports[csk->port_id];
959	struct cxgbi_hba *chba = cdev->hbas[csk->port_id];
960	struct sk_buff *skb = NULL;
961	int ret;
962
963	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
964	"csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags);
965
966	update_address(chba);
967	if (chba->ipv4addr)
968	csk->saddr.sin_addr.s_addr = chba->ipv4addr;
969
970	csk->rss_qid = 0;
971	csk->l2t = t3_l2t_get(cdev: t3dev, dst, dev: ndev,
972	daddr: &csk->daddr.sin_addr.s_addr);
973	if (!csk->l2t) {
974	pr_err("NO l2t available.\n");
975	return -EINVAL;
976	}
977	cxgbi_sock_get(csk);
978
979	csk->atid = cxgb3_alloc_atid(dev: t3dev, client: &t3_client, ctx: csk);
980	if (csk->atid < 0) {
981	pr_err("NO atid available.\n");
982	ret = -EINVAL;
983	goto put_sock;
984	}
985	cxgbi_sock_set_flag(csk, flag: CTPF_HAS_ATID);
986	cxgbi_sock_get(csk);
987
988	skb = alloc_wr(wrlen: sizeof(struct cpl_act_open_req), dlen: 0, GFP_KERNEL);
989	if (!skb) {
990	ret = -ENOMEM;
991	goto free_atid;
992	}
993	skb->sk = (struct sock *)csk;
994	set_arp_failure_handler(skb, hnd: act_open_arp_failure);
995	csk->snd_win = cxgb3i_snd_win;
996	csk->rcv_win = cxgb3i_rcv_win;
997
998	csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1;
999	csk->wr_una_cred = 0;
1000	csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst));
1001	cxgbi_sock_reset_wr_list(csk);
1002	csk->err = 0;
1003
1004	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1005	"csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n",
1006	csk, csk->state, csk->flags,
1007	&csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
1008	&csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
1009
1010	cxgbi_sock_set_state(csk, state: CTP_ACTIVE_OPEN);
1011	send_act_open_req(csk, skb, e: csk->l2t);
1012	return 0;
1013
1014	free_atid:
1015	cxgb3_free_atid(dev: t3dev, atid: csk->atid);
1016	put_sock:
1017	cxgbi_sock_put(csk);
1018	l2t_release(t: t3dev, e: csk->l2t);
1019	csk->l2t = NULL;
1020
1021	return ret;
1022	}
1023
1024	cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = {
1025	[CPL_ACT_ESTABLISH] = do_act_establish,
1026	[CPL_ACT_OPEN_RPL] = do_act_open_rpl,
1027	[CPL_PEER_CLOSE] = do_peer_close,
1028	[CPL_ABORT_REQ_RSS] = do_abort_req,
1029	[CPL_ABORT_RPL_RSS] = do_abort_rpl,
1030	[CPL_CLOSE_CON_RPL] = do_close_con_rpl,
1031	[CPL_TX_DMA_ACK] = do_wr_ack,
1032	[CPL_ISCSI_HDR] = do_iscsi_hdr,
1033	};
1034
1035	/**
1036	* cxgb3i_ofld_init - allocate and initialize resources for each adapter found
1037	* @cdev: cxgbi adapter
1038	*/
1039	static int cxgb3i_ofld_init(struct cxgbi_device *cdev)
1040	{
1041	struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
1042	struct adap_ports port;
1043	struct ofld_page_info rx_page_info;
1044	unsigned int wr_len;
1045	int rc;
1046
1047	if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 ||
1048	t3dev->ctl(t3dev, GET_PORTS, &port) < 0 ||
1049	t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) {
1050	pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev);
1051	return -EINVAL;
1052	}
1053
1054	if (cxgb3i_max_connect > CXGBI_MAX_CONN)
1055	cxgb3i_max_connect = CXGBI_MAX_CONN;
1056
1057	rc = cxgbi_device_portmap_create(cdev, base: cxgb3i_sport_base,
1058	max_conn: cxgb3i_max_connect);
1059	if (rc < 0)
1060	return rc;
1061
1062	init_wr_tab(wr_len);
1063	cdev->csk_release_offload_resources = release_offload_resources;
1064	cdev->csk_push_tx_frames = push_tx_frames;
1065	cdev->csk_send_abort_req = send_abort_req;
1066	cdev->csk_send_close_req = send_close_req;
1067	cdev->csk_send_rx_credits = send_rx_credits;
1068	cdev->csk_alloc_cpls = alloc_cpls;
1069	cdev->csk_init_act_open = init_act_open;
1070
1071	pr_info("cdev 0x%p, offload up, added.\n", cdev);
1072	return 0;
1073	}
1074
1075	/*
1076	* functions to program the pagepod in h/w
1077	*/
1078	static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
1079	{
1080	struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;
1081
1082	memset(req, 0, sizeof(*req));
1083
1084	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
1085	req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |
1086	V_ULPTX_CMD(ULP_MEM_WRITE));
1087	req->len = htonl(V_ULP_MEMIO_DATA_LEN(IPPOD_SIZE >> 5) |
1088	V_ULPTX_NFLITS((IPPOD_SIZE >> 3) + 1));
1089	}
1090
1091	static struct cxgbi_ppm *cdev2ppm(struct cxgbi_device *cdev)
1092	{
1093	return ((struct t3cdev *)cdev->lldev)->ulp_iscsi;
1094	}
1095
1096	static int ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk,
1097	struct cxgbi_task_tag_info *ttinfo)
1098	{
1099	unsigned int idx = ttinfo->idx;
1100	unsigned int npods = ttinfo->npods;
1101	struct scatterlist *sg = ttinfo->sgl;
1102	struct cxgbi_pagepod *ppod;
1103	struct ulp_mem_io *req;
1104	unsigned int sg_off;
1105	unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
1106	int i;
1107
1108	for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) {
1109	struct sk_buff *skb = alloc_wr(wrlen: sizeof(struct ulp_mem_io) +
1110	IPPOD_SIZE, dlen: 0, GFP_ATOMIC);
1111
1112	if (!skb)
1113	return -ENOMEM;
1114	ulp_mem_io_set_hdr(skb, addr: pm_addr);
1115	req = (struct ulp_mem_io *)skb->head;
1116	ppod = (struct cxgbi_pagepod *)(req + 1);
1117	sg_off = i * PPOD_PAGES_MAX;
1118	cxgbi_ddp_set_one_ppod(ppod, ttinfo, sg_pp: &sg,
1119	sg_off: &sg_off);
1120	skb->priority = CPL_PRIORITY_CONTROL;
1121	cxgb3_ofld_send(dev: ppm->lldev, skb);
1122	}
1123	return 0;
1124	}
1125
1126	static void ddp_clear_map(struct cxgbi_device *cdev, struct cxgbi_ppm *ppm,
1127	struct cxgbi_task_tag_info *ttinfo)
1128	{
1129	unsigned int idx = ttinfo->idx;
1130	unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
1131	unsigned int npods = ttinfo->npods;
1132	int i;
1133
1134	log_debug(1 << CXGBI_DBG_DDP,
1135	"cdev 0x%p, clear idx %u, npods %u.\n",
1136	cdev, idx, npods);
1137
1138	for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) {
1139	struct sk_buff *skb = alloc_wr(wrlen: sizeof(struct ulp_mem_io) +
1140	IPPOD_SIZE, dlen: 0, GFP_ATOMIC);
1141
1142	if (!skb) {
1143	pr_err("cdev 0x%p, clear ddp, %u,%d/%u, skb OOM.\n",
1144	cdev, idx, i, npods);
1145	continue;
1146	}
1147	ulp_mem_io_set_hdr(skb, addr: pm_addr);
1148	skb->priority = CPL_PRIORITY_CONTROL;
1149	cxgb3_ofld_send(dev: ppm->lldev, skb);
1150	}
1151	}
1152
1153	static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk,
1154	unsigned int tid, int pg_idx)
1155	{
1156	struct sk_buff *skb = alloc_wr(wrlen: sizeof(struct cpl_set_tcb_field), dlen: 0,
1157	GFP_KERNEL);
1158	struct cpl_set_tcb_field *req;
1159	u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0;
1160
1161	log_debug(1 << CXGBI_DBG_DDP,
1162	"csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx);
1163	if (!skb)
1164	return -ENOMEM;
1165
1166	/* set up ulp submode and page size */
1167	req = (struct cpl_set_tcb_field *)skb->head;
1168	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1169	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1170	req->reply = V_NO_REPLY(1);
1171	req->cpu_idx = 0;
1172	req->word = htons(31);
1173	req->mask = cpu_to_be64(0xF0000000);
1174	req->val = cpu_to_be64(val << 28);
1175	skb->priority = CPL_PRIORITY_CONTROL;
1176
1177	cxgb3_ofld_send(dev: csk->cdev->lldev, skb);
1178	return 0;
1179	}
1180
1181	/**
1182	* ddp_setup_conn_digest - setup conn. digest setting
1183	* @csk: cxgb tcp socket
1184	* @tid: connection id
1185	* @hcrc: header digest enabled
1186	* @dcrc: data digest enabled
1187	* set up the iscsi digest settings for a connection identified by tid
1188	*/
1189	static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,
1190	int hcrc, int dcrc)
1191	{
1192	struct sk_buff *skb = alloc_wr(wrlen: sizeof(struct cpl_set_tcb_field), dlen: 0,
1193	GFP_KERNEL);
1194	struct cpl_set_tcb_field *req;
1195	u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
1196
1197	log_debug(1 << CXGBI_DBG_DDP,
1198	"csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc);
1199	if (!skb)
1200	return -ENOMEM;
1201
1202	/* set up ulp submode and page size */
1203	req = (struct cpl_set_tcb_field *)skb->head;
1204	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1205	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1206	req->reply = V_NO_REPLY(1);
1207	req->cpu_idx = 0;
1208	req->word = htons(31);
1209	req->mask = cpu_to_be64(0x0F000000);
1210	req->val = cpu_to_be64(val << 24);
1211	skb->priority = CPL_PRIORITY_CONTROL;
1212
1213	cxgb3_ofld_send(dev: csk->cdev->lldev, skb);
1214	return 0;
1215	}
1216
1217	/**
1218	* cxgb3i_ddp_init - initialize the cxgb3 adapter's ddp resource
1219	* @cdev: cxgb3i adapter
1220	* initialize the ddp pagepod manager for a given adapter
1221	*/
1222	static int cxgb3i_ddp_init(struct cxgbi_device *cdev)
1223	{
1224	struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;
1225	struct net_device *ndev = cdev->ports[0];
1226	struct cxgbi_tag_format tformat;
1227	unsigned int ppmax, tagmask = 0;
1228	struct ulp_iscsi_info uinfo;
1229	int i, err;
1230
1231	err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
1232	if (err < 0) {
1233	pr_err("%s, failed to get iscsi param %d.\n",
1234	ndev->name, err);
1235	return err;
1236	}
1237	if (uinfo.llimit >= uinfo.ulimit) {
1238	pr_warn("T3 %s, iscsi NOT enabled %u ~ %u!\n",
1239	ndev->name, uinfo.llimit, uinfo.ulimit);
1240	return -EACCES;
1241	}
1242
1243	ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;
1244	tagmask = cxgbi_tagmask_set(ppmax);
1245
1246	pr_info("T3 %s: 0x%x~0x%x, 0x%x, tagmask 0x%x -> 0x%x.\n",
1247	ndev->name, uinfo.llimit, uinfo.ulimit, ppmax, uinfo.tagmask,
1248	tagmask);
1249
1250	memset(&tformat, 0, sizeof(struct cxgbi_tag_format));
1251	for (i = 0; i < 4; i++)
1252	tformat.pgsz_order[i] = uinfo.pgsz_factor[i];
1253	cxgbi_tagmask_check(tagmask, &tformat);
1254
1255	err = cxgbi_ddp_ppm_setup(ppm_pp: &tdev->ulp_iscsi, cdev, tformat: &tformat,
1256	iscsi_size: (uinfo.ulimit - uinfo.llimit + 1),
1257	llimit: uinfo.llimit, start: uinfo.llimit, rsvd_factor: 0, edram_start: 0, edram_size: 0);
1258	if (err)
1259	return err;
1260
1261	if (!(cdev->flags & CXGBI_FLAG_DDP_OFF)) {
1262	uinfo.tagmask = tagmask;
1263	uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT);
1264
1265	err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
1266	if (err < 0) {
1267	pr_err("T3 %s fail to set iscsi param %d.\n",
1268	ndev->name, err);
1269	cdev->flags |= CXGBI_FLAG_DDP_OFF;
1270	}
1271	err = 0;
1272	}
1273
1274	cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;
1275	cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;
1276	cdev->csk_ddp_set_map = ddp_set_map;
1277	cdev->csk_ddp_clear_map = ddp_clear_map;
1278	cdev->cdev2ppm = cdev2ppm;
1279	cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
1280	uinfo.max_txsz - ISCSI_PDU_NONPAYLOAD_LEN);
1281	cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
1282	uinfo.max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN);
1283
1284	return 0;
1285	}
1286
1287	static void cxgb3i_dev_close(struct t3cdev *t3dev)
1288	{
1289	struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1290
1291	if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) {
1292	pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0);
1293	return;
1294	}
1295
1296	cxgbi_device_unregister(cdev);
1297	}
1298
1299	/**
1300	* cxgb3i_dev_open - init a t3 adapter structure and any h/w settings
1301	* @t3dev: t3cdev adapter
1302	*/
1303	static void cxgb3i_dev_open(struct t3cdev *t3dev)
1304	{
1305	struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1306	struct adapter *adapter = tdev2adap(t3dev);
1307	int i, err;
1308
1309	if (cdev) {
1310	pr_info("0x%p, updating.\n", cdev);
1311	return;
1312	}
1313
1314	cdev = cxgbi_device_register(0, adapter->params.nports);
1315	if (!cdev) {
1316	pr_warn("device 0x%p register failed.\n", t3dev);
1317	return;
1318	}
1319
1320	cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET;
1321	cdev->lldev = t3dev;
1322	cdev->pdev = adapter->pdev;
1323	cdev->ports = adapter->port;
1324	cdev->nports = adapter->params.nports;
1325	cdev->mtus = adapter->params.mtus;
1326	cdev->nmtus = NMTUS;
1327	cdev->rx_credit_thres = cxgb3i_rx_credit_thres;
1328	cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN;
1329	cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss);
1330	cdev->itp = &cxgb3i_iscsi_transport;
1331
1332	err = cxgb3i_ddp_init(cdev);
1333	if (err) {
1334	pr_info("0x%p ddp init failed %d\n", cdev, err);
1335	goto err_out;
1336	}
1337
1338	err = cxgb3i_ofld_init(cdev);
1339	if (err) {
1340	pr_info("0x%p offload init failed\n", cdev);
1341	goto err_out;
1342	}
1343
1344	err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN,
1345	&cxgb3i_host_template, cxgb3i_stt);
1346	if (err)
1347	goto err_out;
1348
1349	for (i = 0; i < cdev->nports; i++)
1350	cdev->hbas[i]->ipv4addr =
1351	cxgb3i_get_private_ipv4addr(ndev: cdev->ports[i]);
1352
1353	pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n",
1354	cdev, cdev ? cdev->flags : 0, t3dev, err);
1355	return;
1356
1357	err_out:
1358	cxgbi_device_unregister(cdev);
1359	}
1360
1361	static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port)
1362	{
1363	struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1364
1365	log_debug(1 << CXGBI_DBG_TOE,
1366	"0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n",
1367	t3dev, cdev, event, port);
1368	if (!cdev)
1369	return;
1370
1371	switch (event) {
1372	case OFFLOAD_STATUS_DOWN:
1373	cdev->flags |= CXGBI_FLAG_ADAPTER_RESET;
1374	break;
1375	case OFFLOAD_STATUS_UP:
1376	cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET;
1377	break;
1378	}
1379	}
1380
1381	/**
1382	* cxgb3i_init_module - module init entry point
1383	*
1384	* initialize any driver wide global data structures and register itself
1385	* with the cxgb3 module
1386	*/
1387	static int __init cxgb3i_init_module(void)
1388	{
1389	int rc;
1390
1391	printk(KERN_INFO "%s", version);
1392
1393	rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt);
1394	if (rc < 0)
1395	return rc;
1396
1397	cxgb3_register_client(client: &t3_client);
1398	return 0;
1399	}
1400
1401	/**
1402	* cxgb3i_exit_module - module cleanup/exit entry point
1403	*
1404	* go through the driver hba list and for each hba, release any resource held.
1405	* and unregisters iscsi transport and the cxgb3 module
1406	*/
1407	static void __exit cxgb3i_exit_module(void)
1408	{
1409	cxgb3_unregister_client(client: &t3_client);
1410	cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3);
1411	cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt);
1412	}
1413
1414	module_init(cxgb3i_init_module);
1415	module_exit(cxgb3i_exit_module);
1416