1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Shared Memory Communications over RDMA (SMC-R) and RoCE
4	*
5	* AF_SMC protocol family socket handler keeping the AF_INET sock address type
6	* applies to SOCK_STREAM sockets only
7	* offers an alternative communication option for TCP-protocol sockets
8	* applicable with RoCE-cards only
9	*
10	* Initial restrictions:
11	* - support for alternate links postponed
12	*
13	* Copyright IBM Corp. 2016, 2018
14	*
15	* Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
16	* based on prototype from Frank Blaschka
17	*/
18
19	#define KMSG_COMPONENT "smc"
20	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
21
22	#include <linux/module.h>
23	#include <linux/socket.h>
24	#include <linux/workqueue.h>
25	#include <linux/in.h>
26	#include <linux/sched/signal.h>
27	#include <linux/if_vlan.h>
28	#include <linux/rcupdate_wait.h>
29	#include <linux/ctype.h>
30	#include <linux/splice.h>
31
32	#include <net/sock.h>
33	#include <net/tcp.h>
34	#include <net/smc.h>
35	#include <asm/ioctls.h>
36
37	#include <net/net_namespace.h>
38	#include <net/netns/generic.h>
39	#include "smc_netns.h"
40
41	#include "smc.h"
42	#include "smc_clc.h"
43	#include "smc_llc.h"
44	#include "smc_cdc.h"
45	#include "smc_core.h"
46	#include "smc_ib.h"
47	#include "smc_ism.h"
48	#include "smc_pnet.h"
49	#include "smc_netlink.h"
50	#include "smc_tx.h"
51	#include "smc_rx.h"
52	#include "smc_close.h"
53	#include "smc_stats.h"
54	#include "smc_tracepoint.h"
55	#include "smc_sysctl.h"
56
57	static DEFINE_MUTEX(smc_server_lgr_pending); /* serialize link group
58	* creation on server
59	*/
60	static DEFINE_MUTEX(smc_client_lgr_pending); /* serialize link group
61	* creation on client
62	*/
63
64	static struct workqueue_struct *smc_tcp_ls_wq; /* wq for tcp listen work */
65	struct workqueue_struct *smc_hs_wq; /* wq for handshake work */
66	struct workqueue_struct *smc_close_wq; /* wq for close work */
67
68	static void smc_tcp_listen_work(struct work_struct *);
69	static void smc_connect_work(struct work_struct *);
70
71	int smc_nl_dump_hs_limitation(struct sk_buff *skb, struct netlink_callback *cb)
72	{
73	struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(c: cb);
74	void *hdr;
75
76	if (cb_ctx->pos[0])
77	goto out;
78
79	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, seq: cb->nlh->nlmsg_seq,
80	family: &smc_gen_nl_family, NLM_F_MULTI,
81	cmd: SMC_NETLINK_DUMP_HS_LIMITATION);
82	if (!hdr)
83	return -ENOMEM;
84
85	if (nla_put_u8(skb, attrtype: SMC_NLA_HS_LIMITATION_ENABLED,
86	value: sock_net(sk: skb->sk)->smc.limit_smc_hs))
87	goto err;
88
89	genlmsg_end(skb, hdr);
90	cb_ctx->pos[0] = 1;
91	out:
92	return skb->len;
93	err:
94	genlmsg_cancel(skb, hdr);
95	return -EMSGSIZE;
96	}
97
98	int smc_nl_enable_hs_limitation(struct sk_buff *skb, struct genl_info *info)
99	{
100	sock_net(sk: skb->sk)->smc.limit_smc_hs = true;
101	return 0;
102	}
103
104	int smc_nl_disable_hs_limitation(struct sk_buff *skb, struct genl_info *info)
105	{
106	sock_net(sk: skb->sk)->smc.limit_smc_hs = false;
107	return 0;
108	}
109
110	static void smc_set_keepalive(struct sock *sk, int val)
111	{
112	struct smc_sock *smc = smc_sk(sk);
113
114	smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
115	}
116
117	static struct sock *smc_tcp_syn_recv_sock(const struct sock *sk,
118	struct sk_buff *skb,
119	struct request_sock *req,
120	struct dst_entry *dst,
121	struct request_sock *req_unhash,
122	bool *own_req)
123	{
124	struct smc_sock *smc;
125	struct sock *child;
126
127	smc = smc_clcsock_user_data(clcsk: sk);
128
129	if (READ_ONCE(sk->sk_ack_backlog) + atomic_read(v: &smc->queued_smc_hs) >
130	sk->sk_max_ack_backlog)
131	goto drop;
132
133	if (sk_acceptq_is_full(sk: &smc->sk)) {
134	NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
135	goto drop;
136	}
137
138	/* passthrough to original syn recv sock fct */
139	child = smc->ori_af_ops->syn_recv_sock(sk, skb, req, dst, req_unhash,
140	own_req);
141	/* child must not inherit smc or its ops */
142	if (child) {
143	rcu_assign_sk_user_data(child, NULL);
144
145	/* v4-mapped sockets don't inherit parent ops. Don't restore. */
146	if (inet_csk(sk: child)->icsk_af_ops == inet_csk(sk)->icsk_af_ops)
147	inet_csk(sk: child)->icsk_af_ops = smc->ori_af_ops;
148	}
149	return child;
150
151	drop:
152	dst_release(dst);
153	tcp_listendrop(sk);
154	return NULL;
155	}
156
157	static bool smc_hs_congested(const struct sock *sk)
158	{
159	const struct smc_sock *smc;
160
161	smc = smc_clcsock_user_data(clcsk: sk);
162
163	if (!smc)
164	return true;
165
166	if (workqueue_congested(cpu: WORK_CPU_UNBOUND, wq: smc_hs_wq))
167	return true;
168
169	return false;
170	}
171
172	static struct smc_hashinfo smc_v4_hashinfo = {
173	.lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
174	};
175
176	static struct smc_hashinfo smc_v6_hashinfo = {
177	.lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
178	};
179
180	int smc_hash_sk(struct sock *sk)
181	{
182	struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
183	struct hlist_head *head;
184
185	head = &h->ht;
186
187	write_lock_bh(&h->lock);
188	sk_add_node(sk, list: head);
189	write_unlock_bh(&h->lock);
190	sock_prot_inuse_add(net: sock_net(sk), prot: sk->sk_prot, val: 1);
191
192	return 0;
193	}
194	EXPORT_SYMBOL_GPL(smc_hash_sk);
195
196	void smc_unhash_sk(struct sock *sk)
197	{
198	struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
199
200	write_lock_bh(&h->lock);
201	if (sk_del_node_init(sk))
202	sock_prot_inuse_add(net: sock_net(sk), prot: sk->sk_prot, val: -1);
203	write_unlock_bh(&h->lock);
204	}
205	EXPORT_SYMBOL_GPL(smc_unhash_sk);
206
207	/* This will be called before user really release sock_lock. So do the
208	* work which we didn't do because of user hold the sock_lock in the
209	* BH context
210	*/
211	static void smc_release_cb(struct sock *sk)
212	{
213	struct smc_sock *smc = smc_sk(sk);
214
215	if (smc->conn.tx_in_release_sock) {
216	smc_tx_pending(conn: &smc->conn);
217	smc->conn.tx_in_release_sock = false;
218	}
219	}
220
221	struct proto smc_proto = {
222	.name = "SMC",
223	.owner = THIS_MODULE,
224	.keepalive = smc_set_keepalive,
225	.hash = smc_hash_sk,
226	.unhash = smc_unhash_sk,
227	.release_cb = smc_release_cb,
228	.obj_size = sizeof(struct smc_sock),
229	.h.smc_hash = &smc_v4_hashinfo,
230	.slab_flags = SLAB_TYPESAFE_BY_RCU,
231	};
232	EXPORT_SYMBOL_GPL(smc_proto);
233
234	struct proto smc_proto6 = {
235	.name = "SMC6",
236	.owner = THIS_MODULE,
237	.keepalive = smc_set_keepalive,
238	.hash = smc_hash_sk,
239	.unhash = smc_unhash_sk,
240	.release_cb = smc_release_cb,
241	.obj_size = sizeof(struct smc_sock),
242	.h.smc_hash = &smc_v6_hashinfo,
243	.slab_flags = SLAB_TYPESAFE_BY_RCU,
244	};
245	EXPORT_SYMBOL_GPL(smc_proto6);
246
247	static void smc_fback_restore_callbacks(struct smc_sock *smc)
248	{
249	struct sock *clcsk = smc->clcsock->sk;
250
251	write_lock_bh(&clcsk->sk_callback_lock);
252	clcsk->sk_user_data = NULL;
253
254	smc_clcsock_restore_cb(target_cb: &clcsk->sk_state_change, saved_cb: &smc->clcsk_state_change);
255	smc_clcsock_restore_cb(target_cb: &clcsk->sk_data_ready, saved_cb: &smc->clcsk_data_ready);
256	smc_clcsock_restore_cb(target_cb: &clcsk->sk_write_space, saved_cb: &smc->clcsk_write_space);
257	smc_clcsock_restore_cb(target_cb: &clcsk->sk_error_report, saved_cb: &smc->clcsk_error_report);
258
259	write_unlock_bh(&clcsk->sk_callback_lock);
260	}
261
262	static void smc_restore_fallback_changes(struct smc_sock *smc)
263	{
264	if (smc->clcsock->file) { /* non-accepted sockets have no file yet */
265	smc->clcsock->file->private_data = smc->sk.sk_socket;
266	smc->clcsock->file = NULL;
267	smc_fback_restore_callbacks(smc);
268	}
269	}
270
271	static int __smc_release(struct smc_sock *smc)
272	{
273	struct sock *sk = &smc->sk;
274	int rc = 0;
275
276	if (!smc->use_fallback) {
277	rc = smc_close_active(smc);
278	sock_set_flag(sk, flag: SOCK_DEAD);
279	sk->sk_shutdown |= SHUTDOWN_MASK;
280	} else {
281	if (sk->sk_state != SMC_CLOSED) {
282	if (sk->sk_state != SMC_LISTEN &&
283	sk->sk_state != SMC_INIT)
284	sock_put(sk); /* passive closing */
285	if (sk->sk_state == SMC_LISTEN) {
286	/* wake up clcsock accept */
287	rc = kernel_sock_shutdown(sock: smc->clcsock,
288	how: SHUT_RDWR);
289	}
290	sk->sk_state = SMC_CLOSED;
291	sk->sk_state_change(sk);
292	}
293	smc_restore_fallback_changes(smc);
294	}
295
296	sk->sk_prot->unhash(sk);
297
298	if (sk->sk_state == SMC_CLOSED) {
299	if (smc->clcsock) {
300	release_sock(sk);
301	smc_clcsock_release(smc);
302	lock_sock(sk);
303	}
304	if (!smc->use_fallback)
305	smc_conn_free(conn: &smc->conn);
306	}
307
308	return rc;
309	}
310
311	static int smc_release(struct socket *sock)
312	{
313	struct sock *sk = sock->sk;
314	struct smc_sock *smc;
315	int old_state, rc = 0;
316
317	if (!sk)
318	goto out;
319
320	sock_hold(sk); /* sock_put below */
321	smc = smc_sk(sk);
322
323	old_state = sk->sk_state;
324
325	/* cleanup for a dangling non-blocking connect */
326	if (smc->connect_nonblock && old_state == SMC_INIT)
327	tcp_abort(sk: smc->clcsock->sk, ECONNABORTED);
328
329	if (cancel_work_sync(work: &smc->connect_work))
330	sock_put(sk: &smc->sk); /* sock_hold in smc_connect for passive closing */
331
332	if (sk->sk_state == SMC_LISTEN)
333	/* smc_close_non_accepted() is called and acquires
334	* sock lock for child sockets again
335	*/
336	lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
337	else
338	lock_sock(sk);
339
340	if (old_state == SMC_INIT && sk->sk_state == SMC_ACTIVE &&
341	!smc->use_fallback)
342	smc_close_active_abort(smc);
343
344	rc = __smc_release(smc);
345
346	/* detach socket */
347	sock_orphan(sk);
348	sock->sk = NULL;
349	release_sock(sk);
350
351	sock_put(sk); /* sock_hold above */
352	sock_put(sk); /* final sock_put */
353	out:
354	return rc;
355	}
356
357	static void smc_destruct(struct sock *sk)
358	{
359	if (sk->sk_state != SMC_CLOSED)
360	return;
361	if (!sock_flag(sk, flag: SOCK_DEAD))
362	return;
363	}
364
365	static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
366	int protocol)
367	{
368	struct smc_sock *smc;
369	struct proto *prot;
370	struct sock *sk;
371
372	prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
373	sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, kern: 0);
374	if (!sk)
375	return NULL;
376
377	sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
378	sk->sk_state = SMC_INIT;
379	sk->sk_destruct = smc_destruct;
380	sk->sk_protocol = protocol;
381	WRITE_ONCE(sk->sk_sndbuf, 2 * READ_ONCE(net->smc.sysctl_wmem));
382	WRITE_ONCE(sk->sk_rcvbuf, 2 * READ_ONCE(net->smc.sysctl_rmem));
383	smc = smc_sk(sk);
384	INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
385	INIT_WORK(&smc->connect_work, smc_connect_work);
386	INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
387	INIT_LIST_HEAD(list: &smc->accept_q);
388	spin_lock_init(&smc->accept_q_lock);
389	spin_lock_init(&smc->conn.send_lock);
390	sk->sk_prot->hash(sk);
391	mutex_init(&smc->clcsock_release_lock);
392	smc_init_saved_callbacks(smc);
393
394	return sk;
395	}
396
397	static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
398	int addr_len)
399	{
400	struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
401	struct sock *sk = sock->sk;
402	struct smc_sock *smc;
403	int rc;
404
405	smc = smc_sk(sk);
406
407	/* replicate tests from inet_bind(), to be safe wrt. future changes */
408	rc = -EINVAL;
409	if (addr_len < sizeof(struct sockaddr_in))
410	goto out;
411
412	rc = -EAFNOSUPPORT;
413	if (addr->sin_family != AF_INET &&
414	addr->sin_family != AF_INET6 &&
415	addr->sin_family != AF_UNSPEC)
416	goto out;
417	/* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
418	if (addr->sin_family == AF_UNSPEC &&
419	addr->sin_addr.s_addr != htonl(INADDR_ANY))
420	goto out;
421
422	lock_sock(sk);
423
424	/* Check if socket is already active */
425	rc = -EINVAL;
426	if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
427	goto out_rel;
428
429	smc->clcsock->sk->sk_reuse = sk->sk_reuse;
430	smc->clcsock->sk->sk_reuseport = sk->sk_reuseport;
431	rc = kernel_bind(sock: smc->clcsock, addr: uaddr, addrlen: addr_len);
432
433	out_rel:
434	release_sock(sk);
435	out:
436	return rc;
437	}
438
439	/* copy only relevant settings and flags of SOL_SOCKET level from smc to
440	* clc socket (since smc is not called for these options from net/core)
441	*/
442
443	#define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
444	(1UL << SOCK_KEEPOPEN) | \
445	(1UL << SOCK_LINGER) | \
446	(1UL << SOCK_BROADCAST) | \
447	(1UL << SOCK_TIMESTAMP) | \
448	(1UL << SOCK_DBG) | \
449	(1UL << SOCK_RCVTSTAMP) | \
450	(1UL << SOCK_RCVTSTAMPNS) | \
451	(1UL << SOCK_LOCALROUTE) | \
452	(1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
453	(1UL << SOCK_RXQ_OVFL) | \
454	(1UL << SOCK_WIFI_STATUS) | \
455	(1UL << SOCK_NOFCS) | \
456	(1UL << SOCK_FILTER_LOCKED) | \
457	(1UL << SOCK_TSTAMP_NEW))
458
459	/* if set, use value set by setsockopt() - else use IPv4 or SMC sysctl value */
460	static void smc_adjust_sock_bufsizes(struct sock *nsk, struct sock *osk,
461	unsigned long mask)
462	{
463	struct net *nnet = sock_net(sk: nsk);
464
465	nsk->sk_userlocks = osk->sk_userlocks;
466	if (osk->sk_userlocks & SOCK_SNDBUF_LOCK) {
467	nsk->sk_sndbuf = osk->sk_sndbuf;
468	} else {
469	if (mask == SK_FLAGS_SMC_TO_CLC)
470	WRITE_ONCE(nsk->sk_sndbuf,
471	READ_ONCE(nnet->ipv4.sysctl_tcp_wmem[1]));
472	else
473	WRITE_ONCE(nsk->sk_sndbuf,
474	2 * READ_ONCE(nnet->smc.sysctl_wmem));
475	}
476	if (osk->sk_userlocks & SOCK_RCVBUF_LOCK) {
477	nsk->sk_rcvbuf = osk->sk_rcvbuf;
478	} else {
479	if (mask == SK_FLAGS_SMC_TO_CLC)
480	WRITE_ONCE(nsk->sk_rcvbuf,
481	READ_ONCE(nnet->ipv4.sysctl_tcp_rmem[1]));
482	else
483	WRITE_ONCE(nsk->sk_rcvbuf,
484	2 * READ_ONCE(nnet->smc.sysctl_rmem));
485	}
486	}
487
488	static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
489	unsigned long mask)
490	{
491	/* options we don't get control via setsockopt for */
492	nsk->sk_type = osk->sk_type;
493	nsk->sk_sndtimeo = osk->sk_sndtimeo;
494	nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
495	nsk->sk_mark = READ_ONCE(osk->sk_mark);
496	nsk->sk_priority = READ_ONCE(osk->sk_priority);
497	nsk->sk_rcvlowat = osk->sk_rcvlowat;
498	nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
499	nsk->sk_err = osk->sk_err;
500
501	nsk->sk_flags &= ~mask;
502	nsk->sk_flags |= osk->sk_flags & mask;
503
504	smc_adjust_sock_bufsizes(nsk, osk, mask);
505	}
506
507	static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
508	{
509	smc_copy_sock_settings(nsk: smc->clcsock->sk, osk: &smc->sk, SK_FLAGS_SMC_TO_CLC);
510	}
511
512	#define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
513	(1UL << SOCK_KEEPOPEN) | \
514	(1UL << SOCK_LINGER) | \
515	(1UL << SOCK_DBG))
516	/* copy only settings and flags relevant for smc from clc to smc socket */
517	static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
518	{
519	smc_copy_sock_settings(nsk: &smc->sk, osk: smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
520	}
521
522	/* register the new vzalloced sndbuf on all links */
523	static int smcr_lgr_reg_sndbufs(struct smc_link *link,
524	struct smc_buf_desc *snd_desc)
525	{
526	struct smc_link_group *lgr = link->lgr;
527	int i, rc = 0;
528
529	if (!snd_desc->is_vm)
530	return -EINVAL;
531
532	/* protect against parallel smcr_link_reg_buf() */
533	down_write(sem: &lgr->llc_conf_mutex);
534	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
535	if (!smc_link_active(lnk: &lgr->lnk[i]))
536	continue;
537	rc = smcr_link_reg_buf(link: &lgr->lnk[i], rmb_desc: snd_desc);
538	if (rc)
539	break;
540	}
541	up_write(sem: &lgr->llc_conf_mutex);
542	return rc;
543	}
544
545	/* register the new rmb on all links */
546	static int smcr_lgr_reg_rmbs(struct smc_link *link,
547	struct smc_buf_desc *rmb_desc)
548	{
549	struct smc_link_group *lgr = link->lgr;
550	bool do_slow = false;
551	int i, rc = 0;
552
553	rc = smc_llc_flow_initiate(lgr, type: SMC_LLC_FLOW_RKEY);
554	if (rc)
555	return rc;
556
557	down_read(sem: &lgr->llc_conf_mutex);
558	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
559	if (!smc_link_active(lnk: &lgr->lnk[i]))
560	continue;
561	if (!rmb_desc->is_reg_mr[link->link_idx]) {
562	up_read(sem: &lgr->llc_conf_mutex);
563	goto slow_path;
564	}
565	}
566	/* mr register already */
567	goto fast_path;
568	slow_path:
569	do_slow = true;
570	/* protect against parallel smc_llc_cli_rkey_exchange() and
571	* parallel smcr_link_reg_buf()
572	*/
573	down_write(sem: &lgr->llc_conf_mutex);
574	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
575	if (!smc_link_active(lnk: &lgr->lnk[i]))
576	continue;
577	rc = smcr_link_reg_buf(link: &lgr->lnk[i], rmb_desc);
578	if (rc)
579	goto out;
580	}
581	fast_path:
582	/* exchange confirm_rkey msg with peer */
583	rc = smc_llc_do_confirm_rkey(send_link: link, rmb_desc);
584	if (rc) {
585	rc = -EFAULT;
586	goto out;
587	}
588	rmb_desc->is_conf_rkey = true;
589	out:
590	do_slow ? up_write(sem: &lgr->llc_conf_mutex) : up_read(sem: &lgr->llc_conf_mutex);
591	smc_llc_flow_stop(lgr, flow: &lgr->llc_flow_lcl);
592	return rc;
593	}
594
595	static int smcr_clnt_conf_first_link(struct smc_sock *smc)
596	{
597	struct smc_link *link = smc->conn.lnk;
598	struct smc_llc_qentry *qentry;
599	int rc;
600
601	/* receive CONFIRM LINK request from server over RoCE fabric */
602	qentry = smc_llc_wait(lgr: link->lgr, NULL, SMC_LLC_WAIT_TIME,
603	exp_msg: SMC_LLC_CONFIRM_LINK);
604	if (!qentry) {
605	struct smc_clc_msg_decline dclc;
606
607	rc = smc_clc_wait_msg(smc, buf: &dclc, buflen: sizeof(dclc),
608	SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
609	return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
610	}
611	smc_llc_save_peer_uid(qentry);
612	rc = smc_llc_eval_conf_link(qentry, type: SMC_LLC_REQ);
613	smc_llc_flow_qentry_del(flow: &link->lgr->llc_flow_lcl);
614	if (rc)
615	return SMC_CLC_DECL_RMBE_EC;
616
617	rc = smc_ib_modify_qp_rts(lnk: link);
618	if (rc)
619	return SMC_CLC_DECL_ERR_RDYLNK;
620
621	smc_wr_remember_qp_attr(lnk: link);
622
623	/* reg the sndbuf if it was vzalloced */
624	if (smc->conn.sndbuf_desc->is_vm) {
625	if (smcr_link_reg_buf(link, rmb_desc: smc->conn.sndbuf_desc))
626	return SMC_CLC_DECL_ERR_REGBUF;
627	}
628
629	/* reg the rmb */
630	if (smcr_link_reg_buf(link, rmb_desc: smc->conn.rmb_desc))
631	return SMC_CLC_DECL_ERR_REGBUF;
632
633	/* confirm_rkey is implicit on 1st contact */
634	smc->conn.rmb_desc->is_conf_rkey = true;
635
636	/* send CONFIRM LINK response over RoCE fabric */
637	rc = smc_llc_send_confirm_link(lnk: link, reqresp: SMC_LLC_RESP);
638	if (rc < 0)
639	return SMC_CLC_DECL_TIMEOUT_CL;
640
641	smc_llc_link_active(link);
642	smcr_lgr_set_type(lgr: link->lgr, new_type: SMC_LGR_SINGLE);
643
644	if (link->lgr->max_links > 1) {
645	/* optional 2nd link, receive ADD LINK request from server */
646	qentry = smc_llc_wait(lgr: link->lgr, NULL, SMC_LLC_WAIT_TIME,
647	exp_msg: SMC_LLC_ADD_LINK);
648	if (!qentry) {
649	struct smc_clc_msg_decline dclc;
650
651	rc = smc_clc_wait_msg(smc, buf: &dclc, buflen: sizeof(dclc),
652	SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
653	if (rc == -EAGAIN)
654	rc = 0; /* no DECLINE received, go with one link */
655	return rc;
656	}
657	smc_llc_flow_qentry_clr(flow: &link->lgr->llc_flow_lcl);
658	smc_llc_cli_add_link(link, qentry);
659	}
660	return 0;
661	}
662
663	static bool smc_isascii(char *hostname)
664	{
665	int i;
666
667	for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++)
668	if (!isascii(hostname[i]))
669	return false;
670	return true;
671	}
672
673	static void smc_conn_save_peer_info_fce(struct smc_sock *smc,
674	struct smc_clc_msg_accept_confirm *clc)
675	{
676	struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
677	(struct smc_clc_msg_accept_confirm_v2 *)clc;
678	struct smc_clc_first_contact_ext *fce;
679	int clc_v2_len;
680
681	if (clc->hdr.version == SMC_V1 ||
682	!(clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK))
683	return;
684
685	if (smc->conn.lgr->is_smcd) {
686	memcpy(smc->conn.lgr->negotiated_eid, clc_v2->d1.eid,
687	SMC_MAX_EID_LEN);
688	clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2,
689	d1);
690	} else {
691	memcpy(smc->conn.lgr->negotiated_eid, clc_v2->r1.eid,
692	SMC_MAX_EID_LEN);
693	clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2,
694	r1);
695	}
696	fce = (struct smc_clc_first_contact_ext *)(((u8 *)clc_v2) + clc_v2_len);
697	smc->conn.lgr->peer_os = fce->os_type;
698	smc->conn.lgr->peer_smc_release = fce->release;
699	if (smc_isascii(hostname: fce->hostname))
700	memcpy(smc->conn.lgr->peer_hostname, fce->hostname,
701	SMC_MAX_HOSTNAME_LEN);
702	}
703
704	static void smcr_conn_save_peer_info(struct smc_sock *smc,
705	struct smc_clc_msg_accept_confirm *clc)
706	{
707	int bufsize = smc_uncompress_bufsize(compressed: clc->r0.rmbe_size);
708
709	smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx;
710	smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token);
711	smc->conn.peer_rmbe_size = bufsize;
712	atomic_set(v: &smc->conn.peer_rmbe_space, i: smc->conn.peer_rmbe_size);
713	smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
714	}
715
716	static void smcd_conn_save_peer_info(struct smc_sock *smc,
717	struct smc_clc_msg_accept_confirm *clc)
718	{
719	int bufsize = smc_uncompress_bufsize(compressed: clc->d0.dmbe_size);
720
721	smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx;
722	smc->conn.peer_token = clc->d0.token;
723	/* msg header takes up space in the buffer */
724	smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
725	atomic_set(v: &smc->conn.peer_rmbe_space, i: smc->conn.peer_rmbe_size);
726	smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
727	}
728
729	static void smc_conn_save_peer_info(struct smc_sock *smc,
730	struct smc_clc_msg_accept_confirm *clc)
731	{
732	if (smc->conn.lgr->is_smcd)
733	smcd_conn_save_peer_info(smc, clc);
734	else
735	smcr_conn_save_peer_info(smc, clc);
736	smc_conn_save_peer_info_fce(smc, clc);
737	}
738
739	static void smc_link_save_peer_info(struct smc_link *link,
740	struct smc_clc_msg_accept_confirm *clc,
741	struct smc_init_info *ini)
742	{
743	link->peer_qpn = ntoh24(net: clc->r0.qpn);
744	memcpy(link->peer_gid, ini->peer_gid, SMC_GID_SIZE);
745	memcpy(link->peer_mac, ini->peer_mac, sizeof(link->peer_mac));
746	link->peer_psn = ntoh24(net: clc->r0.psn);
747	link->peer_mtu = clc->r0.qp_mtu;
748	}
749
750	static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc,
751	struct smc_stats_fback *fback_arr)
752	{
753	int cnt;
754
755	for (cnt = 0; cnt < SMC_MAX_FBACK_RSN_CNT; cnt++) {
756	if (fback_arr[cnt].fback_code == smc->fallback_rsn) {
757	fback_arr[cnt].count++;
758	break;
759	}
760	if (!fback_arr[cnt].fback_code) {
761	fback_arr[cnt].fback_code = smc->fallback_rsn;
762	fback_arr[cnt].count++;
763	break;
764	}
765	}
766	}
767
768	static void smc_stat_fallback(struct smc_sock *smc)
769	{
770	struct net *net = sock_net(sk: &smc->sk);
771
772	mutex_lock(&net->smc.mutex_fback_rsn);
773	if (smc->listen_smc) {
774	smc_stat_inc_fback_rsn_cnt(smc, fback_arr: net->smc.fback_rsn->srv);
775	net->smc.fback_rsn->srv_fback_cnt++;
776	} else {
777	smc_stat_inc_fback_rsn_cnt(smc, fback_arr: net->smc.fback_rsn->clnt);
778	net->smc.fback_rsn->clnt_fback_cnt++;
779	}
780	mutex_unlock(lock: &net->smc.mutex_fback_rsn);
781	}
782
783	/* must be called under rcu read lock */
784	static void smc_fback_wakeup_waitqueue(struct smc_sock *smc, void *key)
785	{
786	struct socket_wq *wq;
787	__poll_t flags;
788
789	wq = rcu_dereference(smc->sk.sk_wq);
790	if (!skwq_has_sleeper(wq))
791	return;
792
793	/* wake up smc sk->sk_wq */
794	if (!key) {
795	/* sk_state_change */
796	wake_up_interruptible_all(&wq->wait);
797	} else {
798	flags = key_to_poll(key);
799	if (flags & (EPOLLIN | EPOLLOUT))
800	/* sk_data_ready or sk_write_space */
801	wake_up_interruptible_sync_poll(&wq->wait, flags);
802	else if (flags & EPOLLERR)
803	/* sk_error_report */
804	wake_up_interruptible_poll(&wq->wait, flags);
805	}
806	}
807
808	static int smc_fback_mark_woken(wait_queue_entry_t *wait,
809	unsigned int mode, int sync, void *key)
810	{
811	struct smc_mark_woken *mark =
812	container_of(wait, struct smc_mark_woken, wait_entry);
813
814	mark->woken = true;
815	mark->key = key;
816	return 0;
817	}
818
819	static void smc_fback_forward_wakeup(struct smc_sock *smc, struct sock *clcsk,
820	void (*clcsock_callback)(struct sock *sk))
821	{
822	struct smc_mark_woken mark = { .woken = false };
823	struct socket_wq *wq;
824
825	init_waitqueue_func_entry(wq_entry: &mark.wait_entry,
826	func: smc_fback_mark_woken);
827	rcu_read_lock();
828	wq = rcu_dereference(clcsk->sk_wq);
829	if (!wq)
830	goto out;
831	add_wait_queue(wq_head: sk_sleep(sk: clcsk), wq_entry: &mark.wait_entry);
832	clcsock_callback(clcsk);
833	remove_wait_queue(wq_head: sk_sleep(sk: clcsk), wq_entry: &mark.wait_entry);
834
835	if (mark.woken)
836	smc_fback_wakeup_waitqueue(smc, key: mark.key);
837	out:
838	rcu_read_unlock();
839	}
840
841	static void smc_fback_state_change(struct sock *clcsk)
842	{
843	struct smc_sock *smc;
844
845	read_lock_bh(&clcsk->sk_callback_lock);
846	smc = smc_clcsock_user_data(clcsk);
847	if (smc)
848	smc_fback_forward_wakeup(smc, clcsk,
849	clcsock_callback: smc->clcsk_state_change);
850	read_unlock_bh(&clcsk->sk_callback_lock);
851	}
852
853	static void smc_fback_data_ready(struct sock *clcsk)
854	{
855	struct smc_sock *smc;
856
857	read_lock_bh(&clcsk->sk_callback_lock);
858	smc = smc_clcsock_user_data(clcsk);
859	if (smc)
860	smc_fback_forward_wakeup(smc, clcsk,
861	clcsock_callback: smc->clcsk_data_ready);
862	read_unlock_bh(&clcsk->sk_callback_lock);
863	}
864
865	static void smc_fback_write_space(struct sock *clcsk)
866	{
867	struct smc_sock *smc;
868
869	read_lock_bh(&clcsk->sk_callback_lock);
870	smc = smc_clcsock_user_data(clcsk);
871	if (smc)
872	smc_fback_forward_wakeup(smc, clcsk,
873	clcsock_callback: smc->clcsk_write_space);
874	read_unlock_bh(&clcsk->sk_callback_lock);
875	}
876
877	static void smc_fback_error_report(struct sock *clcsk)
878	{
879	struct smc_sock *smc;
880
881	read_lock_bh(&clcsk->sk_callback_lock);
882	smc = smc_clcsock_user_data(clcsk);
883	if (smc)
884	smc_fback_forward_wakeup(smc, clcsk,
885	clcsock_callback: smc->clcsk_error_report);
886	read_unlock_bh(&clcsk->sk_callback_lock);
887	}
888
889	static void smc_fback_replace_callbacks(struct smc_sock *smc)
890	{
891	struct sock *clcsk = smc->clcsock->sk;
892
893	write_lock_bh(&clcsk->sk_callback_lock);
894	clcsk->sk_user_data = (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
895
896	smc_clcsock_replace_cb(target_cb: &clcsk->sk_state_change, new_cb: smc_fback_state_change,
897	saved_cb: &smc->clcsk_state_change);
898	smc_clcsock_replace_cb(target_cb: &clcsk->sk_data_ready, new_cb: smc_fback_data_ready,
899	saved_cb: &smc->clcsk_data_ready);
900	smc_clcsock_replace_cb(target_cb: &clcsk->sk_write_space, new_cb: smc_fback_write_space,
901	saved_cb: &smc->clcsk_write_space);
902	smc_clcsock_replace_cb(target_cb: &clcsk->sk_error_report, new_cb: smc_fback_error_report,
903	saved_cb: &smc->clcsk_error_report);
904
905	write_unlock_bh(&clcsk->sk_callback_lock);
906	}
907
908	static int smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
909	{
910	int rc = 0;
911
912	mutex_lock(&smc->clcsock_release_lock);
913	if (!smc->clcsock) {
914	rc = -EBADF;
915	goto out;
916	}
917
918	smc->use_fallback = true;
919	smc->fallback_rsn = reason_code;
920	smc_stat_fallback(smc);
921	trace_smc_switch_to_fallback(smc, fallback_rsn: reason_code);
922	if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
923	smc->clcsock->file = smc->sk.sk_socket->file;
924	smc->clcsock->file->private_data = smc->clcsock;
925	smc->clcsock->wq.fasync_list =
926	smc->sk.sk_socket->wq.fasync_list;
927
928	/* There might be some wait entries remaining
929	* in smc sk->sk_wq and they should be woken up
930	* as clcsock's wait queue is woken up.
931	*/
932	smc_fback_replace_callbacks(smc);
933	}
934	out:
935	mutex_unlock(lock: &smc->clcsock_release_lock);
936	return rc;
937	}
938
939	/* fall back during connect */
940	static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
941	{
942	struct net *net = sock_net(sk: &smc->sk);
943	int rc = 0;
944
945	rc = smc_switch_to_fallback(smc, reason_code);
946	if (rc) { /* fallback fails */
947	this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
948	if (smc->sk.sk_state == SMC_INIT)
949	sock_put(sk: &smc->sk); /* passive closing */
950	return rc;
951	}
952	smc_copy_sock_settings_to_clc(smc);
953	smc->connect_nonblock = 0;
954	if (smc->sk.sk_state == SMC_INIT)
955	smc->sk.sk_state = SMC_ACTIVE;
956	return 0;
957	}
958
959	/* decline and fall back during connect */
960	static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
961	u8 version)
962	{
963	struct net *net = sock_net(sk: &smc->sk);
964	int rc;
965
966	if (reason_code < 0) { /* error, fallback is not possible */
967	this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
968	if (smc->sk.sk_state == SMC_INIT)
969	sock_put(sk: &smc->sk); /* passive closing */
970	return reason_code;
971	}
972	if (reason_code != SMC_CLC_DECL_PEERDECL) {
973	rc = smc_clc_send_decline(smc, peer_diag_info: reason_code, version);
974	if (rc < 0) {
975	this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
976	if (smc->sk.sk_state == SMC_INIT)
977	sock_put(sk: &smc->sk); /* passive closing */
978	return rc;
979	}
980	}
981	return smc_connect_fallback(smc, reason_code);
982	}
983
984	static void smc_conn_abort(struct smc_sock *smc, int local_first)
985	{
986	struct smc_connection *conn = &smc->conn;
987	struct smc_link_group *lgr = conn->lgr;
988	bool lgr_valid = false;
989
990	if (smc_conn_lgr_valid(conn))
991	lgr_valid = true;
992
993	smc_conn_free(conn);
994	if (local_first && lgr_valid)
995	smc_lgr_cleanup_early(lgr);
996	}
997
998	/* check if there is a rdma device available for this connection. */
999	/* called for connect and listen */
1000	static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
1001	{
1002	/* PNET table look up: search active ib_device and port
1003	* within same PNETID that also contains the ethernet device
1004	* used for the internal TCP socket
1005	*/
1006	smc_pnet_find_roce_resource(sk: smc->clcsock->sk, ini);
1007	if (!ini->check_smcrv2 && !ini->ib_dev)
1008	return SMC_CLC_DECL_NOSMCRDEV;
1009	if (ini->check_smcrv2 && !ini->smcrv2.ib_dev_v2)
1010	return SMC_CLC_DECL_NOSMCRDEV;
1011	return 0;
1012	}
1013
1014	/* check if there is an ISM device available for this connection. */
1015	/* called for connect and listen */
1016	static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
1017	{
1018	/* Find ISM device with same PNETID as connecting interface */
1019	smc_pnet_find_ism_resource(sk: smc->clcsock->sk, ini);
1020	if (!ini->ism_dev[0])
1021	return SMC_CLC_DECL_NOSMCDDEV;
1022	else
1023	ini->ism_chid[0] = smc_ism_get_chid(dev: ini->ism_dev[0]);
1024	return 0;
1025	}
1026
1027	/* is chid unique for the ism devices that are already determined? */
1028	static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini,
1029	int cnt)
1030	{
1031	int i = (!ini->ism_dev[0]) ? 1 : 0;
1032
1033	for (; i < cnt; i++)
1034	if (ini->ism_chid[i] == chid)
1035	return false;
1036	return true;
1037	}
1038
1039	/* determine possible V2 ISM devices (either without PNETID or with PNETID plus
1040	* PNETID matching net_device)
1041	*/
1042	static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
1043	struct smc_init_info *ini)
1044	{
1045	int rc = SMC_CLC_DECL_NOSMCDDEV;
1046	struct smcd_dev *smcd;
1047	int i = 1;
1048	u16 chid;
1049
1050	if (smcd_indicated(smc_type: ini->smc_type_v1))
1051	rc = 0; /* already initialized for V1 */
1052	mutex_lock(&smcd_dev_list.mutex);
1053	list_for_each_entry(smcd, &smcd_dev_list.list, list) {
1054	if (smcd->going_away || smcd == ini->ism_dev[0])
1055	continue;
1056	chid = smc_ism_get_chid(dev: smcd);
1057	if (!smc_find_ism_v2_is_unique_chid(chid, ini, cnt: i))
1058	continue;
1059	if (!smc_pnet_is_pnetid_set(pnetid: smcd->pnetid) ||
1060	smc_pnet_is_ndev_pnetid(net: sock_net(sk: &smc->sk), pnetid: smcd->pnetid)) {
1061	ini->ism_dev[i] = smcd;
1062	ini->ism_chid[i] = chid;
1063	ini->is_smcd = true;
1064	rc = 0;
1065	i++;
1066	if (i > SMC_MAX_ISM_DEVS)
1067	break;
1068	}
1069	}
1070	mutex_unlock(lock: &smcd_dev_list.mutex);
1071	ini->ism_offered_cnt = i - 1;
1072	if (!ini->ism_dev[0] && !ini->ism_dev[1])
1073	ini->smcd_version = 0;
1074
1075	return rc;
1076	}
1077
1078	/* Check for VLAN ID and register it on ISM device just for CLC handshake */
1079	static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
1080	struct smc_init_info *ini)
1081	{
1082	if (ini->vlan_id && smc_ism_get_vlan(dev: ini->ism_dev[0], vlan_id: ini->vlan_id))
1083	return SMC_CLC_DECL_ISMVLANERR;
1084	return 0;
1085	}
1086
1087	static int smc_find_proposal_devices(struct smc_sock *smc,
1088	struct smc_init_info *ini)
1089	{
1090	int rc = 0;
1091
1092	/* check if there is an ism device available */
1093	if (!(ini->smcd_version & SMC_V1) ||
1094	smc_find_ism_device(smc, ini) ||
1095	smc_connect_ism_vlan_setup(smc, ini))
1096	ini->smcd_version &= ~SMC_V1;
1097	/* else ISM V1 is supported for this connection */
1098
1099	/* check if there is an rdma device available */
1100	if (!(ini->smcr_version & SMC_V1) ||
1101	smc_find_rdma_device(smc, ini))
1102	ini->smcr_version &= ~SMC_V1;
1103	/* else RDMA is supported for this connection */
1104
1105	ini->smc_type_v1 = smc_indicated_type(is_smcd: ini->smcd_version & SMC_V1,
1106	is_smcr: ini->smcr_version & SMC_V1);
1107
1108	/* check if there is an ism v2 device available */
1109	if (!(ini->smcd_version & SMC_V2) ||
1110	!smc_ism_is_v2_capable() ||
1111	smc_find_ism_v2_device_clnt(smc, ini))
1112	ini->smcd_version &= ~SMC_V2;
1113
1114	/* check if there is an rdma v2 device available */
1115	ini->check_smcrv2 = true;
1116	ini->smcrv2.saddr = smc->clcsock->sk->sk_rcv_saddr;
1117	if (!(ini->smcr_version & SMC_V2) ||
1118	smc->clcsock->sk->sk_family != AF_INET ||
1119	!smc_clc_ueid_count() ||
1120	smc_find_rdma_device(smc, ini))
1121	ini->smcr_version &= ~SMC_V2;
1122	ini->check_smcrv2 = false;
1123
1124	ini->smc_type_v2 = smc_indicated_type(is_smcd: ini->smcd_version & SMC_V2,
1125	is_smcr: ini->smcr_version & SMC_V2);
1126
1127	/* if neither ISM nor RDMA are supported, fallback */
1128	if (ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N)
1129	rc = SMC_CLC_DECL_NOSMCDEV;
1130
1131	return rc;
1132	}
1133
1134	/* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
1135	* used, the VLAN ID will be registered again during the connection setup.
1136	*/
1137	static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc,
1138	struct smc_init_info *ini)
1139	{
1140	if (!smcd_indicated(smc_type: ini->smc_type_v1))
1141	return 0;
1142	if (ini->vlan_id && smc_ism_put_vlan(dev: ini->ism_dev[0], vlan_id: ini->vlan_id))
1143	return SMC_CLC_DECL_CNFERR;
1144	return 0;
1145	}
1146
1147	#define SMC_CLC_MAX_ACCEPT_LEN \
1148	(sizeof(struct smc_clc_msg_accept_confirm_v2) + \
1149	sizeof(struct smc_clc_first_contact_ext_v2x) + \
1150	sizeof(struct smc_clc_msg_trail))
1151
1152	/* CLC handshake during connect */
1153	static int smc_connect_clc(struct smc_sock *smc,
1154	struct smc_clc_msg_accept_confirm_v2 *aclc2,
1155	struct smc_init_info *ini)
1156	{
1157	int rc = 0;
1158
1159	/* do inband token exchange */
1160	rc = smc_clc_send_proposal(smc, ini);
1161	if (rc)
1162	return rc;
1163	/* receive SMC Accept CLC message */
1164	return smc_clc_wait_msg(smc, buf: aclc2, SMC_CLC_MAX_ACCEPT_LEN,
1165	SMC_CLC_ACCEPT, CLC_WAIT_TIME);
1166	}
1167
1168	void smc_fill_gid_list(struct smc_link_group *lgr,
1169	struct smc_gidlist *gidlist,
1170	struct smc_ib_device *known_dev, u8 *known_gid)
1171	{
1172	struct smc_init_info *alt_ini = NULL;
1173
1174	memset(gidlist, 0, sizeof(*gidlist));
1175	memcpy(gidlist->list[gidlist->len++], known_gid, SMC_GID_SIZE);
1176
1177	alt_ini = kzalloc(size: sizeof(*alt_ini), GFP_KERNEL);
1178	if (!alt_ini)
1179	goto out;
1180
1181	alt_ini->vlan_id = lgr->vlan_id;
1182	alt_ini->check_smcrv2 = true;
1183	alt_ini->smcrv2.saddr = lgr->saddr;
1184	smc_pnet_find_alt_roce(lgr, ini: alt_ini, known_dev);
1185
1186	if (!alt_ini->smcrv2.ib_dev_v2)
1187	goto out;
1188
1189	memcpy(gidlist->list[gidlist->len++], alt_ini->smcrv2.ib_gid_v2,
1190	SMC_GID_SIZE);
1191
1192	out:
1193	kfree(objp: alt_ini);
1194	}
1195
1196	static int smc_connect_rdma_v2_prepare(struct smc_sock *smc,
1197	struct smc_clc_msg_accept_confirm *aclc,
1198	struct smc_init_info *ini)
1199	{
1200	struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1201	(struct smc_clc_msg_accept_confirm_v2 *)aclc;
1202	struct smc_clc_first_contact_ext *fce =
1203	smc_get_clc_first_contact_ext(clc_v2, is_smcd: false);
1204	struct net *net = sock_net(sk: &smc->sk);
1205	int rc;
1206
1207	if (!ini->first_contact_peer || aclc->hdr.version == SMC_V1)
1208	return 0;
1209
1210	if (fce->v2_direct) {
1211	memcpy(ini->smcrv2.nexthop_mac, &aclc->r0.lcl.mac, ETH_ALEN);
1212	ini->smcrv2.uses_gateway = false;
1213	} else {
1214	if (smc_ib_find_route(net, saddr: smc->clcsock->sk->sk_rcv_saddr,
1215	daddr: smc_ib_gid_to_ipv4(gid: aclc->r0.lcl.gid),
1216	nexthop_mac: ini->smcrv2.nexthop_mac,
1217	uses_gateway: &ini->smcrv2.uses_gateway))
1218	return SMC_CLC_DECL_NOROUTE;
1219	if (!ini->smcrv2.uses_gateway) {
1220	/* mismatch: peer claims indirect, but its direct */
1221	return SMC_CLC_DECL_NOINDIRECT;
1222	}
1223	}
1224
1225	ini->release_nr = fce->release;
1226	rc = smc_clc_clnt_v2x_features_validate(fce, ini);
1227	if (rc)
1228	return rc;
1229
1230	return 0;
1231	}
1232
1233	/* setup for RDMA connection of client */
1234	static int smc_connect_rdma(struct smc_sock *smc,
1235	struct smc_clc_msg_accept_confirm *aclc,
1236	struct smc_init_info *ini)
1237	{
1238	int i, reason_code = 0;
1239	struct smc_link *link;
1240	u8 *eid = NULL;
1241
1242	ini->is_smcd = false;
1243	ini->ib_clcqpn = ntoh24(net: aclc->r0.qpn);
1244	ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
1245	memcpy(ini->peer_systemid, aclc->r0.lcl.id_for_peer, SMC_SYSTEMID_LEN);
1246	memcpy(ini->peer_gid, aclc->r0.lcl.gid, SMC_GID_SIZE);
1247	memcpy(ini->peer_mac, aclc->r0.lcl.mac, ETH_ALEN);
1248	ini->max_conns = SMC_CONN_PER_LGR_MAX;
1249	ini->max_links = SMC_LINKS_ADD_LNK_MAX;
1250
1251	reason_code = smc_connect_rdma_v2_prepare(smc, aclc, ini);
1252	if (reason_code)
1253	return reason_code;
1254
1255	mutex_lock(&smc_client_lgr_pending);
1256	reason_code = smc_conn_create(smc, ini);
1257	if (reason_code) {
1258	mutex_unlock(lock: &smc_client_lgr_pending);
1259	return reason_code;
1260	}
1261
1262	smc_conn_save_peer_info(smc, clc: aclc);
1263
1264	if (ini->first_contact_local) {
1265	link = smc->conn.lnk;
1266	} else {
1267	/* set link that was assigned by server */
1268	link = NULL;
1269	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1270	struct smc_link *l = &smc->conn.lgr->lnk[i];
1271
1272	if (l->peer_qpn == ntoh24(net: aclc->r0.qpn) &&
1273	!memcmp(p: l->peer_gid, q: &aclc->r0.lcl.gid,
1274	SMC_GID_SIZE) &&
1275	(aclc->hdr.version > SMC_V1 ||
1276	!memcmp(p: l->peer_mac, q: &aclc->r0.lcl.mac,
1277	size: sizeof(l->peer_mac)))) {
1278	link = l;
1279	break;
1280	}
1281	}
1282	if (!link) {
1283	reason_code = SMC_CLC_DECL_NOSRVLINK;
1284	goto connect_abort;
1285	}
1286	smc_switch_link_and_count(conn: &smc->conn, to_lnk: link);
1287	}
1288
1289	/* create send buffer and rmb */
1290	if (smc_buf_create(smc, is_smcd: false)) {
1291	reason_code = SMC_CLC_DECL_MEM;
1292	goto connect_abort;
1293	}
1294
1295	if (ini->first_contact_local)
1296	smc_link_save_peer_info(link, clc: aclc, ini);
1297
1298	if (smc_rmb_rtoken_handling(conn: &smc->conn, link, clc: aclc)) {
1299	reason_code = SMC_CLC_DECL_ERR_RTOK;
1300	goto connect_abort;
1301	}
1302
1303	smc_close_init(smc);
1304	smc_rx_init(smc);
1305
1306	if (ini->first_contact_local) {
1307	if (smc_ib_ready_link(lnk: link)) {
1308	reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1309	goto connect_abort;
1310	}
1311	} else {
1312	/* reg sendbufs if they were vzalloced */
1313	if (smc->conn.sndbuf_desc->is_vm) {
1314	if (smcr_lgr_reg_sndbufs(link, snd_desc: smc->conn.sndbuf_desc)) {
1315	reason_code = SMC_CLC_DECL_ERR_REGBUF;
1316	goto connect_abort;
1317	}
1318	}
1319	if (smcr_lgr_reg_rmbs(link, rmb_desc: smc->conn.rmb_desc)) {
1320	reason_code = SMC_CLC_DECL_ERR_REGBUF;
1321	goto connect_abort;
1322	}
1323	}
1324
1325	if (aclc->hdr.version > SMC_V1) {
1326	struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1327	(struct smc_clc_msg_accept_confirm_v2 *)aclc;
1328
1329	eid = clc_v2->r1.eid;
1330	if (ini->first_contact_local)
1331	smc_fill_gid_list(lgr: link->lgr, gidlist: &ini->smcrv2.gidlist,
1332	known_dev: link->smcibdev, known_gid: link->gid);
1333	}
1334
1335	reason_code = smc_clc_send_confirm(smc, clnt_first_contact: ini->first_contact_local,
1336	version: aclc->hdr.version, eid, ini);
1337	if (reason_code)
1338	goto connect_abort;
1339
1340	smc_tx_init(smc);
1341
1342	if (ini->first_contact_local) {
1343	/* QP confirmation over RoCE fabric */
1344	smc_llc_flow_initiate(lgr: link->lgr, type: SMC_LLC_FLOW_ADD_LINK);
1345	reason_code = smcr_clnt_conf_first_link(smc);
1346	smc_llc_flow_stop(lgr: link->lgr, flow: &link->lgr->llc_flow_lcl);
1347	if (reason_code)
1348	goto connect_abort;
1349	}
1350	mutex_unlock(lock: &smc_client_lgr_pending);
1351
1352	smc_copy_sock_settings_to_clc(smc);
1353	smc->connect_nonblock = 0;
1354	if (smc->sk.sk_state == SMC_INIT)
1355	smc->sk.sk_state = SMC_ACTIVE;
1356
1357	return 0;
1358	connect_abort:
1359	smc_conn_abort(smc, local_first: ini->first_contact_local);
1360	mutex_unlock(lock: &smc_client_lgr_pending);
1361	smc->connect_nonblock = 0;
1362
1363	return reason_code;
1364	}
1365
1366	/* The server has chosen one of the proposed ISM devices for the communication.
1367	* Determine from the CHID of the received CLC ACCEPT the ISM device chosen.
1368	*/
1369	static int
1370	smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc,
1371	struct smc_init_info *ini)
1372	{
1373	int i;
1374
1375	for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
1376	if (ini->ism_chid[i] == ntohs(aclc->d1.chid)) {
1377	ini->ism_selected = i;
1378	return 0;
1379	}
1380	}
1381
1382	return -EPROTO;
1383	}
1384
1385	/* setup for ISM connection of client */
1386	static int smc_connect_ism(struct smc_sock *smc,
1387	struct smc_clc_msg_accept_confirm *aclc,
1388	struct smc_init_info *ini)
1389	{
1390	u8 *eid = NULL;
1391	int rc = 0;
1392
1393	ini->is_smcd = true;
1394	ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
1395
1396	if (aclc->hdr.version == SMC_V2) {
1397	struct smc_clc_msg_accept_confirm_v2 *aclc_v2 =
1398	(struct smc_clc_msg_accept_confirm_v2 *)aclc;
1399
1400	if (ini->first_contact_peer) {
1401	struct smc_clc_first_contact_ext *fce =
1402	smc_get_clc_first_contact_ext(clc_v2: aclc_v2, is_smcd: true);
1403
1404	ini->release_nr = fce->release;
1405	rc = smc_clc_clnt_v2x_features_validate(fce, ini);
1406	if (rc)
1407	return rc;
1408	}
1409
1410	rc = smc_v2_determine_accepted_chid(aclc: aclc_v2, ini);
1411	if (rc)
1412	return rc;
1413	}
1414	ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid;
1415
1416	/* there is only one lgr role for SMC-D; use server lock */
1417	mutex_lock(&smc_server_lgr_pending);
1418	rc = smc_conn_create(smc, ini);
1419	if (rc) {
1420	mutex_unlock(lock: &smc_server_lgr_pending);
1421	return rc;
1422	}
1423
1424	/* Create send and receive buffers */
1425	rc = smc_buf_create(smc, is_smcd: true);
1426	if (rc) {
1427	rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM;
1428	goto connect_abort;
1429	}
1430
1431	smc_conn_save_peer_info(smc, clc: aclc);
1432	smc_close_init(smc);
1433	smc_rx_init(smc);
1434	smc_tx_init(smc);
1435
1436	if (aclc->hdr.version > SMC_V1) {
1437	struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1438	(struct smc_clc_msg_accept_confirm_v2 *)aclc;
1439
1440	eid = clc_v2->d1.eid;
1441	}
1442
1443	rc = smc_clc_send_confirm(smc, clnt_first_contact: ini->first_contact_local,
1444	version: aclc->hdr.version, eid, ini);
1445	if (rc)
1446	goto connect_abort;
1447	mutex_unlock(lock: &smc_server_lgr_pending);
1448
1449	smc_copy_sock_settings_to_clc(smc);
1450	smc->connect_nonblock = 0;
1451	if (smc->sk.sk_state == SMC_INIT)
1452	smc->sk.sk_state = SMC_ACTIVE;
1453
1454	return 0;
1455	connect_abort:
1456	smc_conn_abort(smc, local_first: ini->first_contact_local);
1457	mutex_unlock(lock: &smc_server_lgr_pending);
1458	smc->connect_nonblock = 0;
1459
1460	return rc;
1461	}
1462
1463	/* check if received accept type and version matches a proposed one */
1464	static int smc_connect_check_aclc(struct smc_init_info *ini,
1465	struct smc_clc_msg_accept_confirm *aclc)
1466	{
1467	if (aclc->hdr.typev1 != SMC_TYPE_R &&
1468	aclc->hdr.typev1 != SMC_TYPE_D)
1469	return SMC_CLC_DECL_MODEUNSUPP;
1470
1471	if (aclc->hdr.version >= SMC_V2) {
1472	if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1473	!smcr_indicated(smc_type: ini->smc_type_v2)) ||
1474	(aclc->hdr.typev1 == SMC_TYPE_D &&
1475	!smcd_indicated(smc_type: ini->smc_type_v2)))
1476	return SMC_CLC_DECL_MODEUNSUPP;
1477	} else {
1478	if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1479	!smcr_indicated(smc_type: ini->smc_type_v1)) ||
1480	(aclc->hdr.typev1 == SMC_TYPE_D &&
1481	!smcd_indicated(smc_type: ini->smc_type_v1)))
1482	return SMC_CLC_DECL_MODEUNSUPP;
1483	}
1484
1485	return 0;
1486	}
1487
1488	/* perform steps before actually connecting */
1489	static int __smc_connect(struct smc_sock *smc)
1490	{
1491	u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1;
1492	struct smc_clc_msg_accept_confirm_v2 *aclc2;
1493	struct smc_clc_msg_accept_confirm *aclc;
1494	struct smc_init_info *ini = NULL;
1495	u8 *buf = NULL;
1496	int rc = 0;
1497
1498	if (smc->use_fallback)
1499	return smc_connect_fallback(smc, reason_code: smc->fallback_rsn);
1500
1501	/* if peer has not signalled SMC-capability, fall back */
1502	if (!tcp_sk(smc->clcsock->sk)->syn_smc)
1503	return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
1504
1505	/* IPSec connections opt out of SMC optimizations */
1506	if (using_ipsec(smc))
1507	return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC,
1508	version);
1509
1510	ini = kzalloc(size: sizeof(*ini), GFP_KERNEL);
1511	if (!ini)
1512	return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM,
1513	version);
1514
1515	ini->smcd_version = SMC_V1 | SMC_V2;
1516	ini->smcr_version = SMC_V1 | SMC_V2;
1517	ini->smc_type_v1 = SMC_TYPE_B;
1518	ini->smc_type_v2 = SMC_TYPE_B;
1519
1520	/* get vlan id from IP device */
1521	if (smc_vlan_by_tcpsk(clcsock: smc->clcsock, ini)) {
1522	ini->smcd_version &= ~SMC_V1;
1523	ini->smcr_version = 0;
1524	ini->smc_type_v1 = SMC_TYPE_N;
1525	if (!ini->smcd_version) {
1526	rc = SMC_CLC_DECL_GETVLANERR;
1527	goto fallback;
1528	}
1529	}
1530
1531	rc = smc_find_proposal_devices(smc, ini);
1532	if (rc)
1533	goto fallback;
1534
1535	buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL);
1536	if (!buf) {
1537	rc = SMC_CLC_DECL_MEM;
1538	goto fallback;
1539	}
1540	aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf;
1541	aclc = (struct smc_clc_msg_accept_confirm *)aclc2;
1542
1543	/* perform CLC handshake */
1544	rc = smc_connect_clc(smc, aclc2, ini);
1545	if (rc) {
1546	/* -EAGAIN on timeout, see tcp_recvmsg() */
1547	if (rc == -EAGAIN) {
1548	rc = -ETIMEDOUT;
1549	smc->sk.sk_err = ETIMEDOUT;
1550	}
1551	goto vlan_cleanup;
1552	}
1553
1554	/* check if smc modes and versions of CLC proposal and accept match */
1555	rc = smc_connect_check_aclc(ini, aclc);
1556	version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2;
1557	if (rc)
1558	goto vlan_cleanup;
1559
1560	/* depending on previous steps, connect using rdma or ism */
1561	if (aclc->hdr.typev1 == SMC_TYPE_R) {
1562	ini->smcr_version = version;
1563	rc = smc_connect_rdma(smc, aclc, ini);
1564	} else if (aclc->hdr.typev1 == SMC_TYPE_D) {
1565	ini->smcd_version = version;
1566	rc = smc_connect_ism(smc, aclc, ini);
1567	}
1568	if (rc)
1569	goto vlan_cleanup;
1570
1571	SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc);
1572	smc_connect_ism_vlan_cleanup(smc, ini);
1573	kfree(objp: buf);
1574	kfree(objp: ini);
1575	return 0;
1576
1577	vlan_cleanup:
1578	smc_connect_ism_vlan_cleanup(smc, ini);
1579	kfree(objp: buf);
1580	fallback:
1581	kfree(objp: ini);
1582	return smc_connect_decline_fallback(smc, reason_code: rc, version);
1583	}
1584
1585	static void smc_connect_work(struct work_struct *work)
1586	{
1587	struct smc_sock *smc = container_of(work, struct smc_sock,
1588	connect_work);
1589	long timeo = smc->sk.sk_sndtimeo;
1590	int rc = 0;
1591
1592	if (!timeo)
1593	timeo = MAX_SCHEDULE_TIMEOUT;
1594	lock_sock(sk: smc->clcsock->sk);
1595	if (smc->clcsock->sk->sk_err) {
1596	smc->sk.sk_err = smc->clcsock->sk->sk_err;
1597	} else if ((1 << smc->clcsock->sk->sk_state) &
1598	(TCPF_SYN_SENT | TCPF_SYN_RECV)) {
1599	rc = sk_stream_wait_connect(sk: smc->clcsock->sk, timeo_p: &timeo);
1600	if ((rc == -EPIPE) &&
1601	((1 << smc->clcsock->sk->sk_state) &
1602	(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
1603	rc = 0;
1604	}
1605	release_sock(sk: smc->clcsock->sk);
1606	lock_sock(sk: &smc->sk);
1607	if (rc != 0 || smc->sk.sk_err) {
1608	smc->sk.sk_state = SMC_CLOSED;
1609	if (rc == -EPIPE || rc == -EAGAIN)
1610	smc->sk.sk_err = EPIPE;
1611	else if (rc == -ECONNREFUSED)
1612	smc->sk.sk_err = ECONNREFUSED;
1613	else if (signal_pending(current))
1614	smc->sk.sk_err = -sock_intr_errno(timeo);
1615	sock_put(sk: &smc->sk); /* passive closing */
1616	goto out;
1617	}
1618
1619	rc = __smc_connect(smc);
1620	if (rc < 0)
1621	smc->sk.sk_err = -rc;
1622
1623	out:
1624	if (!sock_flag(sk: &smc->sk, flag: SOCK_DEAD)) {
1625	if (smc->sk.sk_err) {
1626	smc->sk.sk_state_change(&smc->sk);
1627	} else { /* allow polling before and after fallback decision */
1628	smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
1629	smc->sk.sk_write_space(&smc->sk);
1630	}
1631	}
1632	release_sock(sk: &smc->sk);
1633	}
1634
1635	static int smc_connect(struct socket *sock, struct sockaddr *addr,
1636	int alen, int flags)
1637	{
1638	struct sock *sk = sock->sk;
1639	struct smc_sock *smc;
1640	int rc = -EINVAL;
1641
1642	smc = smc_sk(sk);
1643
1644	/* separate smc parameter checking to be safe */
1645	if (alen < sizeof(addr->sa_family))
1646	goto out_err;
1647	if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
1648	goto out_err;
1649
1650	lock_sock(sk);
1651	switch (sock->state) {
1652	default:
1653	rc = -EINVAL;
1654	goto out;
1655	case SS_CONNECTED:
1656	rc = sk->sk_state == SMC_ACTIVE ? -EISCONN : -EINVAL;
1657	goto out;
1658	case SS_CONNECTING:
1659	if (sk->sk_state == SMC_ACTIVE)
1660	goto connected;
1661	break;
1662	case SS_UNCONNECTED:
1663	sock->state = SS_CONNECTING;
1664	break;
1665	}
1666
1667	switch (sk->sk_state) {
1668	default:
1669	goto out;
1670	case SMC_CLOSED:
1671	rc = sock_error(sk) ? : -ECONNABORTED;
1672	sock->state = SS_UNCONNECTED;
1673	goto out;
1674	case SMC_ACTIVE:
1675	rc = -EISCONN;
1676	goto out;
1677	case SMC_INIT:
1678	break;
1679	}
1680
1681	smc_copy_sock_settings_to_clc(smc);
1682	tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1683	if (smc->connect_nonblock) {
1684	rc = -EALREADY;
1685	goto out;
1686	}
1687	rc = kernel_connect(sock: smc->clcsock, addr, addrlen: alen, flags);
1688	if (rc && rc != -EINPROGRESS)
1689	goto out;
1690
1691	if (smc->use_fallback) {
1692	sock->state = rc ? SS_CONNECTING : SS_CONNECTED;
1693	goto out;
1694	}
1695	sock_hold(sk: &smc->sk); /* sock put in passive closing */
1696	if (flags & O_NONBLOCK) {
1697	if (queue_work(wq: smc_hs_wq, work: &smc->connect_work))
1698	smc->connect_nonblock = 1;
1699	rc = -EINPROGRESS;
1700	goto out;
1701	} else {
1702	rc = __smc_connect(smc);
1703	if (rc < 0)
1704	goto out;
1705	}
1706
1707	connected:
1708	rc = 0;
1709	sock->state = SS_CONNECTED;
1710	out:
1711	release_sock(sk);
1712	out_err:
1713	return rc;
1714	}
1715
1716	static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
1717	{
1718	struct socket *new_clcsock = NULL;
1719	struct sock *lsk = &lsmc->sk;
1720	struct sock *new_sk;
1721	int rc = -EINVAL;
1722
1723	release_sock(sk: lsk);
1724	new_sk = smc_sock_alloc(net: sock_net(sk: lsk), NULL, protocol: lsk->sk_protocol);
1725	if (!new_sk) {
1726	rc = -ENOMEM;
1727	lsk->sk_err = ENOMEM;
1728	*new_smc = NULL;
1729	lock_sock(sk: lsk);
1730	goto out;
1731	}
1732	*new_smc = smc_sk(new_sk);
1733
1734	mutex_lock(&lsmc->clcsock_release_lock);
1735	if (lsmc->clcsock)
1736	rc = kernel_accept(sock: lsmc->clcsock, newsock: &new_clcsock, SOCK_NONBLOCK);
1737	mutex_unlock(lock: &lsmc->clcsock_release_lock);
1738	lock_sock(sk: lsk);
1739	if (rc < 0 && rc != -EAGAIN)
1740	lsk->sk_err = -rc;
1741	if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
1742	new_sk->sk_prot->unhash(new_sk);
1743	if (new_clcsock)
1744	sock_release(sock: new_clcsock);
1745	new_sk->sk_state = SMC_CLOSED;
1746	sock_set_flag(sk: new_sk, flag: SOCK_DEAD);
1747	sock_put(sk: new_sk); /* final */
1748	*new_smc = NULL;
1749	goto out;
1750	}
1751
1752	/* new clcsock has inherited the smc listen-specific sk_data_ready
1753	* function; switch it back to the original sk_data_ready function
1754	*/
1755	new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready;
1756
1757	/* if new clcsock has also inherited the fallback-specific callback
1758	* functions, switch them back to the original ones.
1759	*/
1760	if (lsmc->use_fallback) {
1761	if (lsmc->clcsk_state_change)
1762	new_clcsock->sk->sk_state_change = lsmc->clcsk_state_change;
1763	if (lsmc->clcsk_write_space)
1764	new_clcsock->sk->sk_write_space = lsmc->clcsk_write_space;
1765	if (lsmc->clcsk_error_report)
1766	new_clcsock->sk->sk_error_report = lsmc->clcsk_error_report;
1767	}
1768
1769	(*new_smc)->clcsock = new_clcsock;
1770	out:
1771	return rc;
1772	}
1773
1774	/* add a just created sock to the accept queue of the listen sock as
1775	* candidate for a following socket accept call from user space
1776	*/
1777	static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
1778	{
1779	struct smc_sock *par = smc_sk(parent);
1780
1781	sock_hold(sk); /* sock_put in smc_accept_unlink () */
1782	spin_lock(lock: &par->accept_q_lock);
1783	list_add_tail(new: &smc_sk(sk)->accept_q, head: &par->accept_q);
1784	spin_unlock(lock: &par->accept_q_lock);
1785	sk_acceptq_added(sk: parent);
1786	}
1787
1788	/* remove a socket from the accept queue of its parental listening socket */
1789	static void smc_accept_unlink(struct sock *sk)
1790	{
1791	struct smc_sock *par = smc_sk(sk)->listen_smc;
1792
1793	spin_lock(lock: &par->accept_q_lock);
1794	list_del_init(entry: &smc_sk(sk)->accept_q);
1795	spin_unlock(lock: &par->accept_q_lock);
1796	sk_acceptq_removed(sk: &smc_sk(sk)->listen_smc->sk);
1797	sock_put(sk); /* sock_hold in smc_accept_enqueue */
1798	}
1799
1800	/* remove a sock from the accept queue to bind it to a new socket created
1801	* for a socket accept call from user space
1802	*/
1803	struct sock *smc_accept_dequeue(struct sock *parent,
1804	struct socket *new_sock)
1805	{
1806	struct smc_sock *isk, *n;
1807	struct sock *new_sk;
1808
1809	list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
1810	new_sk = (struct sock *)isk;
1811
1812	smc_accept_unlink(sk: new_sk);
1813	if (new_sk->sk_state == SMC_CLOSED) {
1814	new_sk->sk_prot->unhash(new_sk);
1815	if (isk->clcsock) {
1816	sock_release(sock: isk->clcsock);
1817	isk->clcsock = NULL;
1818	}
1819	sock_put(sk: new_sk); /* final */
1820	continue;
1821	}
1822	if (new_sock) {
1823	sock_graft(sk: new_sk, parent: new_sock);
1824	new_sock->state = SS_CONNECTED;
1825	if (isk->use_fallback) {
1826	smc_sk(new_sk)->clcsock->file = new_sock->file;
1827	isk->clcsock->file->private_data = isk->clcsock;
1828	}
1829	}
1830	return new_sk;
1831	}
1832	return NULL;
1833	}
1834
1835	/* clean up for a created but never accepted sock */
1836	void smc_close_non_accepted(struct sock *sk)
1837	{
1838	struct smc_sock *smc = smc_sk(sk);
1839
1840	sock_hold(sk); /* sock_put below */
1841	lock_sock(sk);
1842	if (!sk->sk_lingertime)
1843	/* wait for peer closing */
1844	WRITE_ONCE(sk->sk_lingertime, SMC_MAX_STREAM_WAIT_TIMEOUT);
1845	__smc_release(smc);
1846	release_sock(sk);
1847	sock_put(sk); /* sock_hold above */
1848	sock_put(sk); /* final sock_put */
1849	}
1850
1851	static int smcr_serv_conf_first_link(struct smc_sock *smc)
1852	{
1853	struct smc_link *link = smc->conn.lnk;
1854	struct smc_llc_qentry *qentry;
1855	int rc;
1856
1857	/* reg the sndbuf if it was vzalloced*/
1858	if (smc->conn.sndbuf_desc->is_vm) {
1859	if (smcr_link_reg_buf(link, rmb_desc: smc->conn.sndbuf_desc))
1860	return SMC_CLC_DECL_ERR_REGBUF;
1861	}
1862
1863	/* reg the rmb */
1864	if (smcr_link_reg_buf(link, rmb_desc: smc->conn.rmb_desc))
1865	return SMC_CLC_DECL_ERR_REGBUF;
1866
1867	/* send CONFIRM LINK request to client over the RoCE fabric */
1868	rc = smc_llc_send_confirm_link(lnk: link, reqresp: SMC_LLC_REQ);
1869	if (rc < 0)
1870	return SMC_CLC_DECL_TIMEOUT_CL;
1871
1872	/* receive CONFIRM LINK response from client over the RoCE fabric */
1873	qentry = smc_llc_wait(lgr: link->lgr, lnk: link, SMC_LLC_WAIT_TIME,
1874	exp_msg: SMC_LLC_CONFIRM_LINK);
1875	if (!qentry) {
1876	struct smc_clc_msg_decline dclc;
1877
1878	rc = smc_clc_wait_msg(smc, buf: &dclc, buflen: sizeof(dclc),
1879	SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1880	return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1881	}
1882	smc_llc_save_peer_uid(qentry);
1883	rc = smc_llc_eval_conf_link(qentry, type: SMC_LLC_RESP);
1884	smc_llc_flow_qentry_del(flow: &link->lgr->llc_flow_lcl);
1885	if (rc)
1886	return SMC_CLC_DECL_RMBE_EC;
1887
1888	/* confirm_rkey is implicit on 1st contact */
1889	smc->conn.rmb_desc->is_conf_rkey = true;
1890
1891	smc_llc_link_active(link);
1892	smcr_lgr_set_type(lgr: link->lgr, new_type: SMC_LGR_SINGLE);
1893
1894	if (link->lgr->max_links > 1) {
1895	down_write(sem: &link->lgr->llc_conf_mutex);
1896	/* initial contact - try to establish second link */
1897	smc_llc_srv_add_link(link, NULL);
1898	up_write(sem: &link->lgr->llc_conf_mutex);
1899	}
1900	return 0;
1901	}
1902
1903	/* listen worker: finish */
1904	static void smc_listen_out(struct smc_sock *new_smc)
1905	{
1906	struct smc_sock *lsmc = new_smc->listen_smc;
1907	struct sock *newsmcsk = &new_smc->sk;
1908
1909	if (tcp_sk(new_smc->clcsock->sk)->syn_smc)
1910	atomic_dec(v: &lsmc->queued_smc_hs);
1911
1912	if (lsmc->sk.sk_state == SMC_LISTEN) {
1913	lock_sock_nested(sk: &lsmc->sk, SINGLE_DEPTH_NESTING);
1914	smc_accept_enqueue(parent: &lsmc->sk, sk: newsmcsk);
1915	release_sock(sk: &lsmc->sk);
1916	} else { /* no longer listening */
1917	smc_close_non_accepted(sk: newsmcsk);
1918	}
1919
1920	/* Wake up accept */
1921	lsmc->sk.sk_data_ready(&lsmc->sk);
1922	sock_put(sk: &lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1923	}
1924
1925	/* listen worker: finish in state connected */
1926	static void smc_listen_out_connected(struct smc_sock *new_smc)
1927	{
1928	struct sock *newsmcsk = &new_smc->sk;
1929
1930	if (newsmcsk->sk_state == SMC_INIT)
1931	newsmcsk->sk_state = SMC_ACTIVE;
1932
1933	smc_listen_out(new_smc);
1934	}
1935
1936	/* listen worker: finish in error state */
1937	static void smc_listen_out_err(struct smc_sock *new_smc)
1938	{
1939	struct sock *newsmcsk = &new_smc->sk;
1940	struct net *net = sock_net(sk: newsmcsk);
1941
1942	this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt);
1943	if (newsmcsk->sk_state == SMC_INIT)
1944	sock_put(sk: &new_smc->sk); /* passive closing */
1945	newsmcsk->sk_state = SMC_CLOSED;
1946
1947	smc_listen_out(new_smc);
1948	}
1949
1950	/* listen worker: decline and fall back if possible */
1951	static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1952	int local_first, u8 version)
1953	{
1954	/* RDMA setup failed, switch back to TCP */
1955	smc_conn_abort(smc: new_smc, local_first);
1956	if (reason_code < 0 ||
1957	smc_switch_to_fallback(smc: new_smc, reason_code)) {
1958	/* error, no fallback possible */
1959	smc_listen_out_err(new_smc);
1960	return;
1961	}
1962	if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1963	if (smc_clc_send_decline(smc: new_smc, peer_diag_info: reason_code, version) < 0) {
1964	smc_listen_out_err(new_smc);
1965	return;
1966	}
1967	}
1968	smc_listen_out_connected(new_smc);
1969	}
1970
1971	/* listen worker: version checking */
1972	static int smc_listen_v2_check(struct smc_sock *new_smc,
1973	struct smc_clc_msg_proposal *pclc,
1974	struct smc_init_info *ini)
1975	{
1976	struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext;
1977	struct smc_clc_v2_extension *pclc_v2_ext;
1978	int rc = SMC_CLC_DECL_PEERNOSMC;
1979
1980	ini->smc_type_v1 = pclc->hdr.typev1;
1981	ini->smc_type_v2 = pclc->hdr.typev2;
1982	ini->smcd_version = smcd_indicated(smc_type: ini->smc_type_v1) ? SMC_V1 : 0;
1983	ini->smcr_version = smcr_indicated(smc_type: ini->smc_type_v1) ? SMC_V1 : 0;
1984	if (pclc->hdr.version > SMC_V1) {
1985	if (smcd_indicated(smc_type: ini->smc_type_v2))
1986	ini->smcd_version |= SMC_V2;
1987	if (smcr_indicated(smc_type: ini->smc_type_v2))
1988	ini->smcr_version |= SMC_V2;
1989	}
1990	if (!(ini->smcd_version & SMC_V2) && !(ini->smcr_version & SMC_V2)) {
1991	rc = SMC_CLC_DECL_PEERNOSMC;
1992	goto out;
1993	}
1994	pclc_v2_ext = smc_get_clc_v2_ext(prop: pclc);
1995	if (!pclc_v2_ext) {
1996	ini->smcd_version &= ~SMC_V2;
1997	ini->smcr_version &= ~SMC_V2;
1998	rc = SMC_CLC_DECL_NOV2EXT;
1999	goto out;
2000	}
2001	pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(prop_v2ext: pclc_v2_ext);
2002	if (ini->smcd_version & SMC_V2) {
2003	if (!smc_ism_is_v2_capable()) {
2004	ini->smcd_version &= ~SMC_V2;
2005	rc = SMC_CLC_DECL_NOISM2SUPP;
2006	} else if (!pclc_smcd_v2_ext) {
2007	ini->smcd_version &= ~SMC_V2;
2008	rc = SMC_CLC_DECL_NOV2DEXT;
2009	} else if (!pclc_v2_ext->hdr.eid_cnt &&
2010	!pclc_v2_ext->hdr.flag.seid) {
2011	ini->smcd_version &= ~SMC_V2;
2012	rc = SMC_CLC_DECL_NOUEID;
2013	}
2014	}
2015	if (ini->smcr_version & SMC_V2) {
2016	if (!pclc_v2_ext->hdr.eid_cnt) {
2017	ini->smcr_version &= ~SMC_V2;
2018	rc = SMC_CLC_DECL_NOUEID;
2019	}
2020	}
2021
2022	ini->release_nr = pclc_v2_ext->hdr.flag.release;
2023	if (pclc_v2_ext->hdr.flag.release > SMC_RELEASE)
2024	ini->release_nr = SMC_RELEASE;
2025
2026	out:
2027	if (!ini->smcd_version && !ini->smcr_version)
2028	return rc;
2029
2030	return 0;
2031	}
2032
2033	/* listen worker: check prefixes */
2034	static int smc_listen_prfx_check(struct smc_sock *new_smc,
2035	struct smc_clc_msg_proposal *pclc)
2036	{
2037	struct smc_clc_msg_proposal_prefix *pclc_prfx;
2038	struct socket *newclcsock = new_smc->clcsock;
2039
2040	if (pclc->hdr.typev1 == SMC_TYPE_N)
2041	return 0;
2042	pclc_prfx = smc_clc_proposal_get_prefix(pclc);
2043	if (smc_clc_prfx_match(clcsock: newclcsock, prop: pclc_prfx))
2044	return SMC_CLC_DECL_DIFFPREFIX;
2045
2046	return 0;
2047	}
2048
2049	/* listen worker: initialize connection and buffers */
2050	static int smc_listen_rdma_init(struct smc_sock *new_smc,
2051	struct smc_init_info *ini)
2052	{
2053	int rc;
2054
2055	/* allocate connection / link group */
2056	rc = smc_conn_create(smc: new_smc, ini);
2057	if (rc)
2058	return rc;
2059
2060	/* create send buffer and rmb */
2061	if (smc_buf_create(smc: new_smc, is_smcd: false)) {
2062	smc_conn_abort(smc: new_smc, local_first: ini->first_contact_local);
2063	return SMC_CLC_DECL_MEM;
2064	}
2065
2066	return 0;
2067	}
2068
2069	/* listen worker: initialize connection and buffers for SMC-D */
2070	static int smc_listen_ism_init(struct smc_sock *new_smc,
2071	struct smc_init_info *ini)
2072	{
2073	int rc;
2074
2075	rc = smc_conn_create(smc: new_smc, ini);
2076	if (rc)
2077	return rc;
2078
2079	/* Create send and receive buffers */
2080	rc = smc_buf_create(smc: new_smc, is_smcd: true);
2081	if (rc) {
2082	smc_conn_abort(smc: new_smc, local_first: ini->first_contact_local);
2083	return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB :
2084	SMC_CLC_DECL_MEM;
2085	}
2086
2087	return 0;
2088	}
2089
2090	static bool smc_is_already_selected(struct smcd_dev *smcd,
2091	struct smc_init_info *ini,
2092	int matches)
2093	{
2094	int i;
2095
2096	for (i = 0; i < matches; i++)
2097	if (smcd == ini->ism_dev[i])
2098	return true;
2099
2100	return false;
2101	}
2102
2103	/* check for ISM devices matching proposed ISM devices */
2104	static void smc_check_ism_v2_match(struct smc_init_info *ini,
2105	u16 proposed_chid, u64 proposed_gid,
2106	unsigned int *matches)
2107	{
2108	struct smcd_dev *smcd;
2109
2110	list_for_each_entry(smcd, &smcd_dev_list.list, list) {
2111	if (smcd->going_away)
2112	continue;
2113	if (smc_is_already_selected(smcd, ini, matches: *matches))
2114	continue;
2115	if (smc_ism_get_chid(dev: smcd) == proposed_chid &&
2116	!smc_ism_cantalk(peer_gid: proposed_gid, ISM_RESERVED_VLANID, dev: smcd)) {
2117	ini->ism_peer_gid[*matches] = proposed_gid;
2118	ini->ism_dev[*matches] = smcd;
2119	(*matches)++;
2120	break;
2121	}
2122	}
2123	}
2124
2125	static void smc_find_ism_store_rc(u32 rc, struct smc_init_info *ini)
2126	{
2127	if (!ini->rc)
2128	ini->rc = rc;
2129	}
2130
2131	static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
2132	struct smc_clc_msg_proposal *pclc,
2133	struct smc_init_info *ini)
2134	{
2135	struct smc_clc_smcd_v2_extension *smcd_v2_ext;
2136	struct smc_clc_v2_extension *smc_v2_ext;
2137	struct smc_clc_msg_smcd *pclc_smcd;
2138	unsigned int matches = 0;
2139	u8 smcd_version;
2140	u8 *eid = NULL;
2141	int i, rc;
2142
2143	if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(smc_type: ini->smc_type_v2))
2144	goto not_found;
2145
2146	pclc_smcd = smc_get_clc_msg_smcd(prop: pclc);
2147	smc_v2_ext = smc_get_clc_v2_ext(prop: pclc);
2148	smcd_v2_ext = smc_get_clc_smcd_v2_ext(prop_v2ext: smc_v2_ext);
2149
2150	mutex_lock(&smcd_dev_list.mutex);
2151	if (pclc_smcd->ism.chid)
2152	/* check for ISM device matching proposed native ISM device */
2153	smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
2154	ntohll(pclc_smcd->ism.gid), matches: &matches);
2155	for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) {
2156	/* check for ISM devices matching proposed non-native ISM
2157	* devices
2158	*/
2159	smc_check_ism_v2_match(ini,
2160	ntohs(smcd_v2_ext->gidchid[i - 1].chid),
2161	ntohll(smcd_v2_ext->gidchid[i - 1].gid),
2162	matches: &matches);
2163	}
2164	mutex_unlock(lock: &smcd_dev_list.mutex);
2165
2166	if (!ini->ism_dev[0]) {
2167	smc_find_ism_store_rc(SMC_CLC_DECL_NOSMCD2DEV, ini);
2168	goto not_found;
2169	}
2170
2171	smc_ism_get_system_eid(eid: &eid);
2172	if (!smc_clc_match_eid(negotiated_eid: ini->negotiated_eid, smc_v2_ext,
2173	peer_eid: smcd_v2_ext->system_eid, local_eid: eid))
2174	goto not_found;
2175
2176	/* separate - outside the smcd_dev_list.lock */
2177	smcd_version = ini->smcd_version;
2178	for (i = 0; i < matches; i++) {
2179	ini->smcd_version = SMC_V2;
2180	ini->is_smcd = true;
2181	ini->ism_selected = i;
2182	rc = smc_listen_ism_init(new_smc, ini);
2183	if (rc) {
2184	smc_find_ism_store_rc(rc, ini);
2185	/* try next active ISM device */
2186	continue;
2187	}
2188	return; /* matching and usable V2 ISM device found */
2189	}
2190	/* no V2 ISM device could be initialized */
2191	ini->smcd_version = smcd_version; /* restore original value */
2192	ini->negotiated_eid[0] = 0;
2193
2194	not_found:
2195	ini->smcd_version &= ~SMC_V2;
2196	ini->ism_dev[0] = NULL;
2197	ini->is_smcd = false;
2198	}
2199
2200	static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
2201	struct smc_clc_msg_proposal *pclc,
2202	struct smc_init_info *ini)
2203	{
2204	struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(prop: pclc);
2205	int rc = 0;
2206
2207	/* check if ISM V1 is available */
2208	if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(smc_type: ini->smc_type_v1))
2209	goto not_found;
2210	ini->is_smcd = true; /* prepare ISM check */
2211	ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid);
2212	rc = smc_find_ism_device(smc: new_smc, ini);
2213	if (rc)
2214	goto not_found;
2215	ini->ism_selected = 0;
2216	rc = smc_listen_ism_init(new_smc, ini);
2217	if (!rc)
2218	return; /* V1 ISM device found */
2219
2220	not_found:
2221	smc_find_ism_store_rc(rc, ini);
2222	ini->smcd_version &= ~SMC_V1;
2223	ini->ism_dev[0] = NULL;
2224	ini->is_smcd = false;
2225	}
2226
2227	/* listen worker: register buffers */
2228	static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first)
2229	{
2230	struct smc_connection *conn = &new_smc->conn;
2231
2232	if (!local_first) {
2233	/* reg sendbufs if they were vzalloced */
2234	if (conn->sndbuf_desc->is_vm) {
2235	if (smcr_lgr_reg_sndbufs(link: conn->lnk,
2236	snd_desc: conn->sndbuf_desc))
2237	return SMC_CLC_DECL_ERR_REGBUF;
2238	}
2239	if (smcr_lgr_reg_rmbs(link: conn->lnk, rmb_desc: conn->rmb_desc))
2240	return SMC_CLC_DECL_ERR_REGBUF;
2241	}
2242
2243	return 0;
2244	}
2245
2246	static void smc_find_rdma_v2_device_serv(struct smc_sock *new_smc,
2247	struct smc_clc_msg_proposal *pclc,
2248	struct smc_init_info *ini)
2249	{
2250	struct smc_clc_v2_extension *smc_v2_ext;
2251	u8 smcr_version;
2252	int rc;
2253
2254	if (!(ini->smcr_version & SMC_V2) || !smcr_indicated(smc_type: ini->smc_type_v2))
2255	goto not_found;
2256
2257	smc_v2_ext = smc_get_clc_v2_ext(prop: pclc);
2258	if (!smc_clc_match_eid(negotiated_eid: ini->negotiated_eid, smc_v2_ext, NULL, NULL))
2259	goto not_found;
2260
2261	/* prepare RDMA check */
2262	memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
2263	memcpy(ini->peer_gid, smc_v2_ext->roce, SMC_GID_SIZE);
2264	memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
2265	ini->check_smcrv2 = true;
2266	ini->smcrv2.clc_sk = new_smc->clcsock->sk;
2267	ini->smcrv2.saddr = new_smc->clcsock->sk->sk_rcv_saddr;
2268	ini->smcrv2.daddr = smc_ib_gid_to_ipv4(gid: smc_v2_ext->roce);
2269	rc = smc_find_rdma_device(smc: new_smc, ini);
2270	if (rc) {
2271	smc_find_ism_store_rc(rc, ini);
2272	goto not_found;
2273	}
2274	if (!ini->smcrv2.uses_gateway)
2275	memcpy(ini->smcrv2.nexthop_mac, pclc->lcl.mac, ETH_ALEN);
2276
2277	smcr_version = ini->smcr_version;
2278	ini->smcr_version = SMC_V2;
2279	rc = smc_listen_rdma_init(new_smc, ini);
2280	if (!rc) {
2281	rc = smc_listen_rdma_reg(new_smc, local_first: ini->first_contact_local);
2282	if (rc)
2283	smc_conn_abort(smc: new_smc, local_first: ini->first_contact_local);
2284	}
2285	if (!rc)
2286	return;
2287	ini->smcr_version = smcr_version;
2288	smc_find_ism_store_rc(rc, ini);
2289
2290	not_found:
2291	ini->smcr_version &= ~SMC_V2;
2292	ini->smcrv2.ib_dev_v2 = NULL;
2293	ini->check_smcrv2 = false;
2294	}
2295
2296	static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc,
2297	struct smc_clc_msg_proposal *pclc,
2298	struct smc_init_info *ini)
2299	{
2300	int rc;
2301
2302	if (!(ini->smcr_version & SMC_V1) || !smcr_indicated(smc_type: ini->smc_type_v1))
2303	return SMC_CLC_DECL_NOSMCDEV;
2304
2305	/* prepare RDMA check */
2306	memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
2307	memcpy(ini->peer_gid, pclc->lcl.gid, SMC_GID_SIZE);
2308	memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
2309	rc = smc_find_rdma_device(smc: new_smc, ini);
2310	if (rc) {
2311	/* no RDMA device found */
2312	return SMC_CLC_DECL_NOSMCDEV;
2313	}
2314	rc = smc_listen_rdma_init(new_smc, ini);
2315	if (rc)
2316	return rc;
2317	return smc_listen_rdma_reg(new_smc, local_first: ini->first_contact_local);
2318	}
2319
2320	/* determine the local device matching to proposal */
2321	static int smc_listen_find_device(struct smc_sock *new_smc,
2322	struct smc_clc_msg_proposal *pclc,
2323	struct smc_init_info *ini)
2324	{
2325	int prfx_rc;
2326
2327	/* check for ISM device matching V2 proposed device */
2328	smc_find_ism_v2_device_serv(new_smc, pclc, ini);
2329	if (ini->ism_dev[0])
2330	return 0;
2331
2332	/* check for matching IP prefix and subnet length (V1) */
2333	prfx_rc = smc_listen_prfx_check(new_smc, pclc);
2334	if (prfx_rc)
2335	smc_find_ism_store_rc(rc: prfx_rc, ini);
2336
2337	/* get vlan id from IP device */
2338	if (smc_vlan_by_tcpsk(clcsock: new_smc->clcsock, ini))
2339	return ini->rc ?: SMC_CLC_DECL_GETVLANERR;
2340
2341	/* check for ISM device matching V1 proposed device */
2342	if (!prfx_rc)
2343	smc_find_ism_v1_device_serv(new_smc, pclc, ini);
2344	if (ini->ism_dev[0])
2345	return 0;
2346
2347	if (!smcr_indicated(smc_type: pclc->hdr.typev1) &&
2348	!smcr_indicated(smc_type: pclc->hdr.typev2))
2349	/* skip RDMA and decline */
2350	return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV;
2351
2352	/* check if RDMA V2 is available */
2353	smc_find_rdma_v2_device_serv(new_smc, pclc, ini);
2354	if (ini->smcrv2.ib_dev_v2)
2355	return 0;
2356
2357	/* check if RDMA V1 is available */
2358	if (!prfx_rc) {
2359	int rc;
2360
2361	rc = smc_find_rdma_v1_device_serv(new_smc, pclc, ini);
2362	smc_find_ism_store_rc(rc, ini);
2363	return (!rc) ? 0 : ini->rc;
2364	}
2365	return prfx_rc;
2366	}
2367
2368	/* listen worker: finish RDMA setup */
2369	static int smc_listen_rdma_finish(struct smc_sock *new_smc,
2370	struct smc_clc_msg_accept_confirm *cclc,
2371	bool local_first,
2372	struct smc_init_info *ini)
2373	{
2374	struct smc_link *link = new_smc->conn.lnk;
2375	int reason_code = 0;
2376
2377	if (local_first)
2378	smc_link_save_peer_info(link, clc: cclc, ini);
2379
2380	if (smc_rmb_rtoken_handling(conn: &new_smc->conn, link, clc: cclc))
2381	return SMC_CLC_DECL_ERR_RTOK;
2382
2383	if (local_first) {
2384	if (smc_ib_ready_link(lnk: link))
2385	return SMC_CLC_DECL_ERR_RDYLNK;
2386	/* QP confirmation over RoCE fabric */
2387	smc_llc_flow_initiate(lgr: link->lgr, type: SMC_LLC_FLOW_ADD_LINK);
2388	reason_code = smcr_serv_conf_first_link(smc: new_smc);
2389	smc_llc_flow_stop(lgr: link->lgr, flow: &link->lgr->llc_flow_lcl);
2390	}
2391	return reason_code;
2392	}
2393
2394	/* setup for connection of server */
2395	static void smc_listen_work(struct work_struct *work)
2396	{
2397	struct smc_sock *new_smc = container_of(work, struct smc_sock,
2398	smc_listen_work);
2399	struct socket *newclcsock = new_smc->clcsock;
2400	struct smc_clc_msg_accept_confirm *cclc;
2401	struct smc_clc_msg_proposal_area *buf;
2402	struct smc_clc_msg_proposal *pclc;
2403	struct smc_init_info *ini = NULL;
2404	u8 proposal_version = SMC_V1;
2405	u8 accept_version;
2406	int rc = 0;
2407
2408	if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
2409	return smc_listen_out_err(new_smc);
2410
2411	if (new_smc->use_fallback) {
2412	smc_listen_out_connected(new_smc);
2413	return;
2414	}
2415
2416	/* check if peer is smc capable */
2417	if (!tcp_sk(newclcsock->sk)->syn_smc) {
2418	rc = smc_switch_to_fallback(smc: new_smc, SMC_CLC_DECL_PEERNOSMC);
2419	if (rc)
2420	smc_listen_out_err(new_smc);
2421	else
2422	smc_listen_out_connected(new_smc);
2423	return;
2424	}
2425
2426	/* do inband token exchange -
2427	* wait for and receive SMC Proposal CLC message
2428	*/
2429	buf = kzalloc(size: sizeof(*buf), GFP_KERNEL);
2430	if (!buf) {
2431	rc = SMC_CLC_DECL_MEM;
2432	goto out_decl;
2433	}
2434	pclc = (struct smc_clc_msg_proposal *)buf;
2435	rc = smc_clc_wait_msg(smc: new_smc, buf: pclc, buflen: sizeof(*buf),
2436	SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
2437	if (rc)
2438	goto out_decl;
2439
2440	if (pclc->hdr.version > SMC_V1)
2441	proposal_version = SMC_V2;
2442
2443	/* IPSec connections opt out of SMC optimizations */
2444	if (using_ipsec(smc: new_smc)) {
2445	rc = SMC_CLC_DECL_IPSEC;
2446	goto out_decl;
2447	}
2448
2449	ini = kzalloc(size: sizeof(*ini), GFP_KERNEL);
2450	if (!ini) {
2451	rc = SMC_CLC_DECL_MEM;
2452	goto out_decl;
2453	}
2454
2455	/* initial version checking */
2456	rc = smc_listen_v2_check(new_smc, pclc, ini);
2457	if (rc)
2458	goto out_decl;
2459
2460	rc = smc_clc_srv_v2x_features_validate(pclc, ini);
2461	if (rc)
2462	goto out_decl;
2463
2464	mutex_lock(&smc_server_lgr_pending);
2465	smc_close_init(smc: new_smc);
2466	smc_rx_init(smc: new_smc);
2467	smc_tx_init(smc: new_smc);
2468
2469	/* determine ISM or RoCE device used for connection */
2470	rc = smc_listen_find_device(new_smc, pclc, ini);
2471	if (rc)
2472	goto out_unlock;
2473
2474	/* send SMC Accept CLC message */
2475	accept_version = ini->is_smcd ? ini->smcd_version : ini->smcr_version;
2476	rc = smc_clc_send_accept(smc: new_smc, srv_first_contact: ini->first_contact_local,
2477	version: accept_version, negotiated_eid: ini->negotiated_eid, ini);
2478	if (rc)
2479	goto out_unlock;
2480
2481	/* SMC-D does not need this lock any more */
2482	if (ini->is_smcd)
2483	mutex_unlock(lock: &smc_server_lgr_pending);
2484
2485	/* receive SMC Confirm CLC message */
2486	memset(buf, 0, sizeof(*buf));
2487	cclc = (struct smc_clc_msg_accept_confirm *)buf;
2488	rc = smc_clc_wait_msg(smc: new_smc, buf: cclc, buflen: sizeof(*buf),
2489	SMC_CLC_CONFIRM, CLC_WAIT_TIME);
2490	if (rc) {
2491	if (!ini->is_smcd)
2492	goto out_unlock;
2493	goto out_decl;
2494	}
2495
2496	rc = smc_clc_v2x_features_confirm_check(cclc, ini);
2497	if (rc) {
2498	if (!ini->is_smcd)
2499	goto out_unlock;
2500	goto out_decl;
2501	}
2502
2503	/* fce smc release version is needed in smc_listen_rdma_finish,
2504	* so save fce info here.
2505	*/
2506	smc_conn_save_peer_info_fce(smc: new_smc, clc: cclc);
2507
2508	/* finish worker */
2509	if (!ini->is_smcd) {
2510	rc = smc_listen_rdma_finish(new_smc, cclc,
2511	local_first: ini->first_contact_local, ini);
2512	if (rc)
2513	goto out_unlock;
2514	mutex_unlock(lock: &smc_server_lgr_pending);
2515	}
2516	smc_conn_save_peer_info(smc: new_smc, clc: cclc);
2517	smc_listen_out_connected(new_smc);
2518	SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
2519	goto out_free;
2520
2521	out_unlock:
2522	mutex_unlock(lock: &smc_server_lgr_pending);
2523	out_decl:
2524	smc_listen_decline(new_smc, reason_code: rc, local_first: ini ? ini->first_contact_local : 0,
2525	version: proposal_version);
2526	out_free:
2527	kfree(objp: ini);
2528	kfree(objp: buf);
2529	}
2530
2531	static void smc_tcp_listen_work(struct work_struct *work)
2532	{
2533	struct smc_sock *lsmc = container_of(work, struct smc_sock,
2534	tcp_listen_work);
2535	struct sock *lsk = &lsmc->sk;
2536	struct smc_sock *new_smc;
2537	int rc = 0;
2538
2539	lock_sock(sk: lsk);
2540	while (lsk->sk_state == SMC_LISTEN) {
2541	rc = smc_clcsock_accept(lsmc, new_smc: &new_smc);
2542	if (rc) /* clcsock accept queue empty or error */
2543	goto out;
2544	if (!new_smc)
2545	continue;
2546
2547	if (tcp_sk(new_smc->clcsock->sk)->syn_smc)
2548	atomic_inc(v: &lsmc->queued_smc_hs);
2549
2550	new_smc->listen_smc = lsmc;
2551	new_smc->use_fallback = lsmc->use_fallback;
2552	new_smc->fallback_rsn = lsmc->fallback_rsn;
2553	sock_hold(sk: lsk); /* sock_put in smc_listen_work */
2554	INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
2555	smc_copy_sock_settings_to_smc(smc: new_smc);
2556	sock_hold(sk: &new_smc->sk); /* sock_put in passive closing */
2557	if (!queue_work(wq: smc_hs_wq, work: &new_smc->smc_listen_work))
2558	sock_put(sk: &new_smc->sk);
2559	}
2560
2561	out:
2562	release_sock(sk: lsk);
2563	sock_put(sk: &lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */
2564	}
2565
2566	static void smc_clcsock_data_ready(struct sock *listen_clcsock)
2567	{
2568	struct smc_sock *lsmc;
2569
2570	read_lock_bh(&listen_clcsock->sk_callback_lock);
2571	lsmc = smc_clcsock_user_data(clcsk: listen_clcsock);
2572	if (!lsmc)
2573	goto out;
2574	lsmc->clcsk_data_ready(listen_clcsock);
2575	if (lsmc->sk.sk_state == SMC_LISTEN) {
2576	sock_hold(sk: &lsmc->sk); /* sock_put in smc_tcp_listen_work() */
2577	if (!queue_work(wq: smc_tcp_ls_wq, work: &lsmc->tcp_listen_work))
2578	sock_put(sk: &lsmc->sk);
2579	}
2580	out:
2581	read_unlock_bh(&listen_clcsock->sk_callback_lock);
2582	}
2583
2584	static int smc_listen(struct socket *sock, int backlog)
2585	{
2586	struct sock *sk = sock->sk;
2587	struct smc_sock *smc;
2588	int rc;
2589
2590	smc = smc_sk(sk);
2591	lock_sock(sk);
2592
2593	rc = -EINVAL;
2594	if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
2595	smc->connect_nonblock || sock->state != SS_UNCONNECTED)
2596	goto out;
2597
2598	rc = 0;
2599	if (sk->sk_state == SMC_LISTEN) {
2600	sk->sk_max_ack_backlog = backlog;
2601	goto out;
2602	}
2603	/* some socket options are handled in core, so we could not apply
2604	* them to the clc socket -- copy smc socket options to clc socket
2605	*/
2606	smc_copy_sock_settings_to_clc(smc);
2607	if (!smc->use_fallback)
2608	tcp_sk(smc->clcsock->sk)->syn_smc = 1;
2609
2610	/* save original sk_data_ready function and establish
2611	* smc-specific sk_data_ready function
2612	*/
2613	write_lock_bh(&smc->clcsock->sk->sk_callback_lock);
2614	smc->clcsock->sk->sk_user_data =
2615	(void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
2616	smc_clcsock_replace_cb(target_cb: &smc->clcsock->sk->sk_data_ready,
2617	new_cb: smc_clcsock_data_ready, saved_cb: &smc->clcsk_data_ready);
2618	write_unlock_bh(&smc->clcsock->sk->sk_callback_lock);
2619
2620	/* save original ops */
2621	smc->ori_af_ops = inet_csk(sk: smc->clcsock->sk)->icsk_af_ops;
2622
2623	smc->af_ops = *smc->ori_af_ops;
2624	smc->af_ops.syn_recv_sock = smc_tcp_syn_recv_sock;
2625
2626	inet_csk(sk: smc->clcsock->sk)->icsk_af_ops = &smc->af_ops;
2627
2628	if (smc->limit_smc_hs)
2629	tcp_sk(smc->clcsock->sk)->smc_hs_congested = smc_hs_congested;
2630
2631	rc = kernel_listen(sock: smc->clcsock, backlog);
2632	if (rc) {
2633	write_lock_bh(&smc->clcsock->sk->sk_callback_lock);
2634	smc_clcsock_restore_cb(target_cb: &smc->clcsock->sk->sk_data_ready,
2635	saved_cb: &smc->clcsk_data_ready);
2636	smc->clcsock->sk->sk_user_data = NULL;
2637	write_unlock_bh(&smc->clcsock->sk->sk_callback_lock);
2638	goto out;
2639	}
2640	sk->sk_max_ack_backlog = backlog;
2641	sk->sk_ack_backlog = 0;
2642	sk->sk_state = SMC_LISTEN;
2643
2644	out:
2645	release_sock(sk);
2646	return rc;
2647	}
2648
2649	static int smc_accept(struct socket *sock, struct socket *new_sock,
2650	int flags, bool kern)
2651	{
2652	struct sock *sk = sock->sk, *nsk;
2653	DECLARE_WAITQUEUE(wait, current);
2654	struct smc_sock *lsmc;
2655	long timeo;
2656	int rc = 0;
2657
2658	lsmc = smc_sk(sk);
2659	sock_hold(sk); /* sock_put below */
2660	lock_sock(sk);
2661
2662	if (lsmc->sk.sk_state != SMC_LISTEN) {
2663	rc = -EINVAL;
2664	release_sock(sk);
2665	goto out;
2666	}
2667
2668	/* Wait for an incoming connection */
2669	timeo = sock_rcvtimeo(sk, noblock: flags & O_NONBLOCK);
2670	add_wait_queue_exclusive(wq_head: sk_sleep(sk), wq_entry: &wait);
2671	while (!(nsk = smc_accept_dequeue(parent: sk, new_sock))) {
2672	set_current_state(TASK_INTERRUPTIBLE);
2673	if (!timeo) {
2674	rc = -EAGAIN;
2675	break;
2676	}
2677	release_sock(sk);
2678	timeo = schedule_timeout(timeout: timeo);
2679	/* wakeup by sk_data_ready in smc_listen_work() */
2680	sched_annotate_sleep();
2681	lock_sock(sk);
2682	if (signal_pending(current)) {
2683	rc = sock_intr_errno(timeo);
2684	break;
2685	}
2686	}
2687	set_current_state(TASK_RUNNING);
2688	remove_wait_queue(wq_head: sk_sleep(sk), wq_entry: &wait);
2689
2690	if (!rc)
2691	rc = sock_error(sk: nsk);
2692	release_sock(sk);
2693	if (rc)
2694	goto out;
2695
2696	if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
2697	/* wait till data arrives on the socket */
2698	timeo = msecs_to_jiffies(m: lsmc->sockopt_defer_accept *
2699	MSEC_PER_SEC);
2700	if (smc_sk(nsk)->use_fallback) {
2701	struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
2702
2703	lock_sock(sk: clcsk);
2704	if (skb_queue_empty(list: &clcsk->sk_receive_queue))
2705	sk_wait_data(sk: clcsk, timeo: &timeo, NULL);
2706	release_sock(sk: clcsk);
2707	} else if (!atomic_read(v: &smc_sk(nsk)->conn.bytes_to_rcv)) {
2708	lock_sock(sk: nsk);
2709	smc_rx_wait(smc_sk(nsk), timeo: &timeo, fcrit: smc_rx_data_available);
2710	release_sock(sk: nsk);
2711	}
2712	}
2713
2714	out:
2715	sock_put(sk); /* sock_hold above */
2716	return rc;
2717	}
2718
2719	static int smc_getname(struct socket *sock, struct sockaddr *addr,
2720	int peer)
2721	{
2722	struct smc_sock *smc;
2723
2724	if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
2725	(sock->sk->sk_state != SMC_APPCLOSEWAIT1))
2726	return -ENOTCONN;
2727
2728	smc = smc_sk(sock->sk);
2729
2730	return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
2731	}
2732
2733	static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
2734	{
2735	struct sock *sk = sock->sk;
2736	struct smc_sock *smc;
2737	int rc;
2738
2739	smc = smc_sk(sk);
2740	lock_sock(sk);
2741
2742	/* SMC does not support connect with fastopen */
2743	if (msg->msg_flags & MSG_FASTOPEN) {
2744	/* not connected yet, fallback */
2745	if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2746	rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2747	if (rc)
2748	goto out;
2749	} else {
2750	rc = -EINVAL;
2751	goto out;
2752	}
2753	} else if ((sk->sk_state != SMC_ACTIVE) &&
2754	(sk->sk_state != SMC_APPCLOSEWAIT1) &&
2755	(sk->sk_state != SMC_INIT)) {
2756	rc = -EPIPE;
2757	goto out;
2758	}
2759
2760	if (smc->use_fallback) {
2761	rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
2762	} else {
2763	rc = smc_tx_sendmsg(smc, msg, len);
2764	SMC_STAT_TX_PAYLOAD(smc, len, rc);
2765	}
2766	out:
2767	release_sock(sk);
2768	return rc;
2769	}
2770
2771	static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2772	int flags)
2773	{
2774	struct sock *sk = sock->sk;
2775	struct smc_sock *smc;
2776	int rc = -ENOTCONN;
2777
2778	smc = smc_sk(sk);
2779	lock_sock(sk);
2780	if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2781	/* socket was connected before, no more data to read */
2782	rc = 0;
2783	goto out;
2784	}
2785	if ((sk->sk_state == SMC_INIT) ||
2786	(sk->sk_state == SMC_LISTEN) ||
2787	(sk->sk_state == SMC_CLOSED))
2788	goto out;
2789
2790	if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2791	rc = 0;
2792	goto out;
2793	}
2794
2795	if (smc->use_fallback) {
2796	rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
2797	} else {
2798	msg->msg_namelen = 0;
2799	rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
2800	SMC_STAT_RX_PAYLOAD(smc, rc, rc);
2801	}
2802
2803	out:
2804	release_sock(sk);
2805	return rc;
2806	}
2807
2808	static __poll_t smc_accept_poll(struct sock *parent)
2809	{
2810	struct smc_sock *isk = smc_sk(parent);
2811	__poll_t mask = 0;
2812
2813	spin_lock(lock: &isk->accept_q_lock);
2814	if (!list_empty(head: &isk->accept_q))
2815	mask = EPOLLIN | EPOLLRDNORM;
2816	spin_unlock(lock: &isk->accept_q_lock);
2817
2818	return mask;
2819	}
2820
2821	static __poll_t smc_poll(struct file *file, struct socket *sock,
2822	poll_table *wait)
2823	{
2824	struct sock *sk = sock->sk;
2825	struct smc_sock *smc;
2826	__poll_t mask = 0;
2827
2828	if (!sk)
2829	return EPOLLNVAL;
2830
2831	smc = smc_sk(sock->sk);
2832	if (smc->use_fallback) {
2833	/* delegate to CLC child sock */
2834	mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
2835	sk->sk_err = smc->clcsock->sk->sk_err;
2836	} else {
2837	if (sk->sk_state != SMC_CLOSED)
2838	sock_poll_wait(filp: file, sock, p: wait);
2839	if (sk->sk_err)
2840	mask |= EPOLLERR;
2841	if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
2842	(sk->sk_state == SMC_CLOSED))
2843	mask |= EPOLLHUP;
2844	if (sk->sk_state == SMC_LISTEN) {
2845	/* woken up by sk_data_ready in smc_listen_work() */
2846	mask |= smc_accept_poll(parent: sk);
2847	} else if (smc->use_fallback) { /* as result of connect_work()*/
2848	mask |= smc->clcsock->ops->poll(file, smc->clcsock,
2849	wait);
2850	sk->sk_err = smc->clcsock->sk->sk_err;
2851	} else {
2852	if ((sk->sk_state != SMC_INIT &&
2853	atomic_read(v: &smc->conn.sndbuf_space)) ||
2854	sk->sk_shutdown & SEND_SHUTDOWN) {
2855	mask |= EPOLLOUT | EPOLLWRNORM;
2856	} else {
2857	sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2858	set_bit(SOCK_NOSPACE, addr: &sk->sk_socket->flags);
2859	}
2860	if (atomic_read(v: &smc->conn.bytes_to_rcv))
2861	mask |= EPOLLIN | EPOLLRDNORM;
2862	if (sk->sk_shutdown & RCV_SHUTDOWN)
2863	mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2864	if (sk->sk_state == SMC_APPCLOSEWAIT1)
2865	mask |= EPOLLIN;
2866	if (smc->conn.urg_state == SMC_URG_VALID)
2867	mask |= EPOLLPRI;
2868	}
2869	}
2870
2871	return mask;
2872	}
2873
2874	static int smc_shutdown(struct socket *sock, int how)
2875	{
2876	struct sock *sk = sock->sk;
2877	bool do_shutdown = true;
2878	struct smc_sock *smc;
2879	int rc = -EINVAL;
2880	int old_state;
2881	int rc1 = 0;
2882
2883	smc = smc_sk(sk);
2884
2885	if ((how < SHUT_RD) || (how > SHUT_RDWR))
2886	return rc;
2887
2888	lock_sock(sk);
2889
2890	if (sock->state == SS_CONNECTING) {
2891	if (sk->sk_state == SMC_ACTIVE)
2892	sock->state = SS_CONNECTED;
2893	else if (sk->sk_state == SMC_PEERCLOSEWAIT1 ||
2894	sk->sk_state == SMC_PEERCLOSEWAIT2 ||
2895	sk->sk_state == SMC_APPCLOSEWAIT1 ||
2896	sk->sk_state == SMC_APPCLOSEWAIT2 ||
2897	sk->sk_state == SMC_APPFINCLOSEWAIT)
2898	sock->state = SS_DISCONNECTING;
2899	}
2900
2901	rc = -ENOTCONN;
2902	if ((sk->sk_state != SMC_ACTIVE) &&
2903	(sk->sk_state != SMC_PEERCLOSEWAIT1) &&
2904	(sk->sk_state != SMC_PEERCLOSEWAIT2) &&
2905	(sk->sk_state != SMC_APPCLOSEWAIT1) &&
2906	(sk->sk_state != SMC_APPCLOSEWAIT2) &&
2907	(sk->sk_state != SMC_APPFINCLOSEWAIT))
2908	goto out;
2909	if (smc->use_fallback) {
2910	rc = kernel_sock_shutdown(sock: smc->clcsock, how);
2911	sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
2912	if (sk->sk_shutdown == SHUTDOWN_MASK) {
2913	sk->sk_state = SMC_CLOSED;
2914	sk->sk_socket->state = SS_UNCONNECTED;
2915	sock_put(sk);
2916	}
2917	goto out;
2918	}
2919	switch (how) {
2920	case SHUT_RDWR: /* shutdown in both directions */
2921	old_state = sk->sk_state;
2922	rc = smc_close_active(smc);
2923	if (old_state == SMC_ACTIVE &&
2924	sk->sk_state == SMC_PEERCLOSEWAIT1)
2925	do_shutdown = false;
2926	break;
2927	case SHUT_WR:
2928	rc = smc_close_shutdown_write(smc);
2929	break;
2930	case SHUT_RD:
2931	rc = 0;
2932	/* nothing more to do because peer is not involved */
2933	break;
2934	}
2935	if (do_shutdown && smc->clcsock)
2936	rc1 = kernel_sock_shutdown(sock: smc->clcsock, how);
2937	/* map sock_shutdown_cmd constants to sk_shutdown value range */
2938	sk->sk_shutdown |= how + 1;
2939
2940	if (sk->sk_state == SMC_CLOSED)
2941	sock->state = SS_UNCONNECTED;
2942	else
2943	sock->state = SS_DISCONNECTING;
2944	out:
2945	release_sock(sk);
2946	return rc ? rc : rc1;
2947	}
2948
2949	static int __smc_getsockopt(struct socket *sock, int level, int optname,
2950	char __user *optval, int __user *optlen)
2951	{
2952	struct smc_sock *smc;
2953	int val, len;
2954
2955	smc = smc_sk(sock->sk);
2956
2957	if (get_user(len, optlen))
2958	return -EFAULT;
2959
2960	len = min_t(int, len, sizeof(int));
2961
2962	if (len < 0)
2963	return -EINVAL;
2964
2965	switch (optname) {
2966	case SMC_LIMIT_HS:
2967	val = smc->limit_smc_hs;
2968	break;
2969	default:
2970	return -EOPNOTSUPP;
2971	}
2972
2973	if (put_user(len, optlen))
2974	return -EFAULT;
2975	if (copy_to_user(to: optval, from: &val, n: len))
2976	return -EFAULT;
2977
2978	return 0;
2979	}
2980
2981	static int __smc_setsockopt(struct socket *sock, int level, int optname,
2982	sockptr_t optval, unsigned int optlen)
2983	{
2984	struct sock *sk = sock->sk;
2985	struct smc_sock *smc;
2986	int val, rc;
2987
2988	smc = smc_sk(sk);
2989
2990	lock_sock(sk);
2991	switch (optname) {
2992	case SMC_LIMIT_HS:
2993	if (optlen < sizeof(int)) {
2994	rc = -EINVAL;
2995	break;
2996	}
2997	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(int))) {
2998	rc = -EFAULT;
2999	break;
3000	}
3001
3002	smc->limit_smc_hs = !!val;
3003	rc = 0;
3004	break;
3005	default:
3006	rc = -EOPNOTSUPP;
3007	break;
3008	}
3009	release_sock(sk);
3010
3011	return rc;
3012	}
3013
3014	static int smc_setsockopt(struct socket *sock, int level, int optname,
3015	sockptr_t optval, unsigned int optlen)
3016	{
3017	struct sock *sk = sock->sk;
3018	struct smc_sock *smc;
3019	int val, rc;
3020
3021	if (level == SOL_TCP && optname == TCP_ULP)
3022	return -EOPNOTSUPP;
3023	else if (level == SOL_SMC)
3024	return __smc_setsockopt(sock, level, optname, optval, optlen);
3025
3026	smc = smc_sk(sk);
3027
3028	/* generic setsockopts reaching us here always apply to the
3029	* CLC socket
3030	*/
3031	mutex_lock(&smc->clcsock_release_lock);
3032	if (!smc->clcsock) {
3033	mutex_unlock(lock: &smc->clcsock_release_lock);
3034	return -EBADF;
3035	}
3036	if (unlikely(!smc->clcsock->ops->setsockopt))
3037	rc = -EOPNOTSUPP;
3038	else
3039	rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
3040	optval, optlen);
3041	if (smc->clcsock->sk->sk_err) {
3042	sk->sk_err = smc->clcsock->sk->sk_err;
3043	sk_error_report(sk);
3044	}
3045	mutex_unlock(lock: &smc->clcsock_release_lock);
3046
3047	if (optlen < sizeof(int))
3048	return -EINVAL;
3049	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(int)))
3050	return -EFAULT;
3051
3052	lock_sock(sk);
3053	if (rc || smc->use_fallback)
3054	goto out;
3055	switch (optname) {
3056	case TCP_FASTOPEN:
3057	case TCP_FASTOPEN_CONNECT:
3058	case TCP_FASTOPEN_KEY:
3059	case TCP_FASTOPEN_NO_COOKIE:
3060	/* option not supported by SMC */
3061	if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
3062	rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
3063	} else {
3064	rc = -EINVAL;
3065	}
3066	break;
3067	case TCP_NODELAY:
3068	if (sk->sk_state != SMC_INIT &&
3069	sk->sk_state != SMC_LISTEN &&
3070	sk->sk_state != SMC_CLOSED) {
3071	if (val) {
3072	SMC_STAT_INC(smc, ndly_cnt);
3073	smc_tx_pending(conn: &smc->conn);
3074	cancel_delayed_work(dwork: &smc->conn.tx_work);
3075	}
3076	}
3077	break;
3078	case TCP_CORK:
3079	if (sk->sk_state != SMC_INIT &&
3080	sk->sk_state != SMC_LISTEN &&
3081	sk->sk_state != SMC_CLOSED) {
3082	if (!val) {
3083	SMC_STAT_INC(smc, cork_cnt);
3084	smc_tx_pending(conn: &smc->conn);
3085	cancel_delayed_work(dwork: &smc->conn.tx_work);
3086	}
3087	}
3088	break;
3089	case TCP_DEFER_ACCEPT:
3090	smc->sockopt_defer_accept = val;
3091	break;
3092	default:
3093	break;
3094	}
3095	out:
3096	release_sock(sk);
3097
3098	return rc;
3099	}
3100
3101	static int smc_getsockopt(struct socket *sock, int level, int optname,
3102	char __user *optval, int __user *optlen)
3103	{
3104	struct smc_sock *smc;
3105	int rc;
3106
3107	if (level == SOL_SMC)
3108	return __smc_getsockopt(sock, level, optname, optval, optlen);
3109
3110	smc = smc_sk(sock->sk);
3111	mutex_lock(&smc->clcsock_release_lock);
3112	if (!smc->clcsock) {
3113	mutex_unlock(lock: &smc->clcsock_release_lock);
3114	return -EBADF;
3115	}
3116	/* socket options apply to the CLC socket */
3117	if (unlikely(!smc->clcsock->ops->getsockopt)) {
3118	mutex_unlock(lock: &smc->clcsock_release_lock);
3119	return -EOPNOTSUPP;
3120	}
3121	rc = smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
3122	optval, optlen);
3123	mutex_unlock(lock: &smc->clcsock_release_lock);
3124	return rc;
3125	}
3126
3127	static int smc_ioctl(struct socket *sock, unsigned int cmd,
3128	unsigned long arg)
3129	{
3130	union smc_host_cursor cons, urg;
3131	struct smc_connection *conn;
3132	struct smc_sock *smc;
3133	int answ;
3134
3135	smc = smc_sk(sock->sk);
3136	conn = &smc->conn;
3137	lock_sock(sk: &smc->sk);
3138	if (smc->use_fallback) {
3139	if (!smc->clcsock) {
3140	release_sock(sk: &smc->sk);
3141	return -EBADF;
3142	}
3143	answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
3144	release_sock(sk: &smc->sk);
3145	return answ;
3146	}
3147	switch (cmd) {
3148	case SIOCINQ: /* same as FIONREAD */
3149	if (smc->sk.sk_state == SMC_LISTEN) {
3150	release_sock(sk: &smc->sk);
3151	return -EINVAL;
3152	}
3153	if (smc->sk.sk_state == SMC_INIT ||
3154	smc->sk.sk_state == SMC_CLOSED)
3155	answ = 0;
3156	else
3157	answ = atomic_read(v: &smc->conn.bytes_to_rcv);
3158	break;
3159	case SIOCOUTQ:
3160	/* output queue size (not send + not acked) */
3161	if (smc->sk.sk_state == SMC_LISTEN) {
3162	release_sock(sk: &smc->sk);
3163	return -EINVAL;
3164	}
3165	if (smc->sk.sk_state == SMC_INIT ||
3166	smc->sk.sk_state == SMC_CLOSED)
3167	answ = 0;
3168	else
3169	answ = smc->conn.sndbuf_desc->len -
3170	atomic_read(v: &smc->conn.sndbuf_space);
3171	break;
3172	case SIOCOUTQNSD:
3173	/* output queue size (not send only) */
3174	if (smc->sk.sk_state == SMC_LISTEN) {
3175	release_sock(sk: &smc->sk);
3176	return -EINVAL;
3177	}
3178	if (smc->sk.sk_state == SMC_INIT ||
3179	smc->sk.sk_state == SMC_CLOSED)
3180	answ = 0;
3181	else
3182	answ = smc_tx_prepared_sends(conn: &smc->conn);
3183	break;
3184	case SIOCATMARK:
3185	if (smc->sk.sk_state == SMC_LISTEN) {
3186	release_sock(sk: &smc->sk);
3187	return -EINVAL;
3188	}
3189	if (smc->sk.sk_state == SMC_INIT ||
3190	smc->sk.sk_state == SMC_CLOSED) {
3191	answ = 0;
3192	} else {
3193	smc_curs_copy(tgt: &cons, src: &conn->local_tx_ctrl.cons, conn);
3194	smc_curs_copy(tgt: &urg, src: &conn->urg_curs, conn);
3195	answ = smc_curs_diff(size: conn->rmb_desc->len,
3196	old: &cons, new: &urg) == 1;
3197	}
3198	break;
3199	default:
3200	release_sock(sk: &smc->sk);
3201	return -ENOIOCTLCMD;
3202	}
3203	release_sock(sk: &smc->sk);
3204
3205	return put_user(answ, (int __user *)arg);
3206	}
3207
3208	/* Map the affected portions of the rmbe into an spd, note the number of bytes
3209	* to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
3210	* updates till whenever a respective page has been fully processed.
3211	* Note that subsequent recv() calls have to wait till all splice() processing
3212	* completed.
3213	*/
3214	static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
3215	struct pipe_inode_info *pipe, size_t len,
3216	unsigned int flags)
3217	{
3218	struct sock *sk = sock->sk;
3219	struct smc_sock *smc;
3220	int rc = -ENOTCONN;
3221
3222	smc = smc_sk(sk);
3223	lock_sock(sk);
3224	if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
3225	/* socket was connected before, no more data to read */
3226	rc = 0;
3227	goto out;
3228	}
3229	if (sk->sk_state == SMC_INIT ||
3230	sk->sk_state == SMC_LISTEN ||
3231	sk->sk_state == SMC_CLOSED)
3232	goto out;
3233
3234	if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
3235	rc = 0;
3236	goto out;
3237	}
3238
3239	if (smc->use_fallback) {
3240	rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
3241	pipe, len, flags);
3242	} else {
3243	if (*ppos) {
3244	rc = -ESPIPE;
3245	goto out;
3246	}
3247	if (flags & SPLICE_F_NONBLOCK)
3248	flags = MSG_DONTWAIT;
3249	else
3250	flags = 0;
3251	SMC_STAT_INC(smc, splice_cnt);
3252	rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
3253	}
3254	out:
3255	release_sock(sk);
3256
3257	return rc;
3258	}
3259
3260	/* must look like tcp */
3261	static const struct proto_ops smc_sock_ops = {
3262	.family = PF_SMC,
3263	.owner = THIS_MODULE,
3264	.release = smc_release,
3265	.bind = smc_bind,
3266	.connect = smc_connect,
3267	.socketpair = sock_no_socketpair,
3268	.accept = smc_accept,
3269	.getname = smc_getname,
3270	.poll = smc_poll,
3271	.ioctl = smc_ioctl,
3272	.listen = smc_listen,
3273	.shutdown = smc_shutdown,
3274	.setsockopt = smc_setsockopt,
3275	.getsockopt = smc_getsockopt,
3276	.sendmsg = smc_sendmsg,
3277	.recvmsg = smc_recvmsg,
3278	.mmap = sock_no_mmap,
3279	.splice_read = smc_splice_read,
3280	};
3281
3282	static int __smc_create(struct net *net, struct socket *sock, int protocol,
3283	int kern, struct socket *clcsock)
3284	{
3285	int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
3286	struct smc_sock *smc;
3287	struct sock *sk;
3288	int rc;
3289
3290	rc = -ESOCKTNOSUPPORT;
3291	if (sock->type != SOCK_STREAM)
3292	goto out;
3293
3294	rc = -EPROTONOSUPPORT;
3295	if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
3296	goto out;
3297
3298	rc = -ENOBUFS;
3299	sock->ops = &smc_sock_ops;
3300	sock->state = SS_UNCONNECTED;
3301	sk = smc_sock_alloc(net, sock, protocol);
3302	if (!sk)
3303	goto out;
3304
3305	/* create internal TCP socket for CLC handshake and fallback */
3306	smc = smc_sk(sk);
3307	smc->use_fallback = false; /* assume rdma capability first */
3308	smc->fallback_rsn = 0;
3309
3310	/* default behavior from limit_smc_hs in every net namespace */
3311	smc->limit_smc_hs = net->smc.limit_smc_hs;
3312
3313	rc = 0;
3314	if (!clcsock) {
3315	rc = sock_create_kern(net, family, type: SOCK_STREAM, IPPROTO_TCP,
3316	res: &smc->clcsock);
3317	if (rc) {
3318	sk_common_release(sk);
3319	goto out;
3320	}
3321
3322	/* smc_clcsock_release() does not wait smc->clcsock->sk's
3323	* destruction; its sk_state might not be TCP_CLOSE after
3324	* smc->sk is close()d, and TCP timers can be fired later,
3325	* which need net ref.
3326	*/
3327	sk = smc->clcsock->sk;
3328	__netns_tracker_free(net, tracker: &sk->ns_tracker, refcounted: false);
3329	sk->sk_net_refcnt = 1;
3330	get_net_track(net, tracker: &sk->ns_tracker, GFP_KERNEL);
3331	sock_inuse_add(net, val: 1);
3332	} else {
3333	smc->clcsock = clcsock;
3334	}
3335
3336	out:
3337	return rc;
3338	}
3339
3340	static int smc_create(struct net *net, struct socket *sock, int protocol,
3341	int kern)
3342	{
3343	return __smc_create(net, sock, protocol, kern, NULL);
3344	}
3345
3346	static const struct net_proto_family smc_sock_family_ops = {
3347	.family = PF_SMC,
3348	.owner = THIS_MODULE,
3349	.create = smc_create,
3350	};
3351
3352	static int smc_ulp_init(struct sock *sk)
3353	{
3354	struct socket *tcp = sk->sk_socket;
3355	struct net *net = sock_net(sk);
3356	struct socket *smcsock;
3357	int protocol, ret;
3358
3359	/* only TCP can be replaced */
3360	if (tcp->type != SOCK_STREAM || sk->sk_protocol != IPPROTO_TCP ||
3361	(sk->sk_family != AF_INET && sk->sk_family != AF_INET6))
3362	return -ESOCKTNOSUPPORT;
3363	/* don't handle wq now */
3364	if (tcp->state != SS_UNCONNECTED || !tcp->file || tcp->wq.fasync_list)
3365	return -ENOTCONN;
3366
3367	if (sk->sk_family == AF_INET)
3368	protocol = SMCPROTO_SMC;
3369	else
3370	protocol = SMCPROTO_SMC6;
3371
3372	smcsock = sock_alloc();
3373	if (!smcsock)
3374	return -ENFILE;
3375
3376	smcsock->type = SOCK_STREAM;
3377	__module_get(THIS_MODULE); /* tried in __tcp_ulp_find_autoload */
3378	ret = __smc_create(net, sock: smcsock, protocol, kern: 1, clcsock: tcp);
3379	if (ret) {
3380	sock_release(sock: smcsock); /* module_put() which ops won't be NULL */
3381	return ret;
3382	}
3383
3384	/* replace tcp socket to smc */
3385	smcsock->file = tcp->file;
3386	smcsock->file->private_data = smcsock;
3387	smcsock->file->f_inode = SOCK_INODE(socket: smcsock); /* replace inode when sock_close */
3388	smcsock->file->f_path.dentry->d_inode = SOCK_INODE(socket: smcsock); /* dput() in __fput */
3389	tcp->file = NULL;
3390
3391	return ret;
3392	}
3393
3394	static void smc_ulp_clone(const struct request_sock *req, struct sock *newsk,
3395	const gfp_t priority)
3396	{
3397	struct inet_connection_sock *icsk = inet_csk(sk: newsk);
3398
3399	/* don't inherit ulp ops to child when listen */
3400	icsk->icsk_ulp_ops = NULL;
3401	}
3402
3403	static struct tcp_ulp_ops smc_ulp_ops __read_mostly = {
3404	.name = "smc",
3405	.owner = THIS_MODULE,
3406	.init = smc_ulp_init,
3407	.clone = smc_ulp_clone,
3408	};
3409
3410	unsigned int smc_net_id;
3411
3412	static __net_init int smc_net_init(struct net *net)
3413	{
3414	int rc;
3415
3416	rc = smc_sysctl_net_init(net);
3417	if (rc)
3418	return rc;
3419	return smc_pnet_net_init(net);
3420	}
3421
3422	static void __net_exit smc_net_exit(struct net *net)
3423	{
3424	smc_sysctl_net_exit(net);
3425	smc_pnet_net_exit(net);
3426	}
3427
3428	static __net_init int smc_net_stat_init(struct net *net)
3429	{
3430	return smc_stats_init(net);
3431	}
3432
3433	static void __net_exit smc_net_stat_exit(struct net *net)
3434	{
3435	smc_stats_exit(net);
3436	}
3437
3438	static struct pernet_operations smc_net_ops = {
3439	.init = smc_net_init,
3440	.exit = smc_net_exit,
3441	.id = &smc_net_id,
3442	.size = sizeof(struct smc_net),
3443	};
3444
3445	static struct pernet_operations smc_net_stat_ops = {
3446	.init = smc_net_stat_init,
3447	.exit = smc_net_stat_exit,
3448	};
3449
3450	static int __init smc_init(void)
3451	{
3452	int rc;
3453
3454	rc = register_pernet_subsys(&smc_net_ops);
3455	if (rc)
3456	return rc;
3457
3458	rc = register_pernet_subsys(&smc_net_stat_ops);
3459	if (rc)
3460	goto out_pernet_subsys;
3461
3462	rc = smc_ism_init();
3463	if (rc)
3464	goto out_pernet_subsys_stat;
3465	smc_clc_init();
3466
3467	rc = smc_nl_init();
3468	if (rc)
3469	goto out_ism;
3470
3471	rc = smc_pnet_init();
3472	if (rc)
3473	goto out_nl;
3474
3475	rc = -ENOMEM;
3476
3477	smc_tcp_ls_wq = alloc_workqueue(fmt: "smc_tcp_ls_wq", flags: 0, max_active: 0);
3478	if (!smc_tcp_ls_wq)
3479	goto out_pnet;
3480
3481	smc_hs_wq = alloc_workqueue(fmt: "smc_hs_wq", flags: 0, max_active: 0);
3482	if (!smc_hs_wq)
3483	goto out_alloc_tcp_ls_wq;
3484
3485	smc_close_wq = alloc_workqueue(fmt: "smc_close_wq", flags: 0, max_active: 0);
3486	if (!smc_close_wq)
3487	goto out_alloc_hs_wq;
3488
3489	rc = smc_core_init();
3490	if (rc) {
3491	pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
3492	goto out_alloc_wqs;
3493	}
3494
3495	rc = smc_llc_init();
3496	if (rc) {
3497	pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
3498	goto out_core;
3499	}
3500
3501	rc = smc_cdc_init();
3502	if (rc) {
3503	pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
3504	goto out_core;
3505	}
3506
3507	rc = proto_register(prot: &smc_proto, alloc_slab: 1);
3508	if (rc) {
3509	pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
3510	goto out_core;
3511	}
3512
3513	rc = proto_register(prot: &smc_proto6, alloc_slab: 1);
3514	if (rc) {
3515	pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
3516	goto out_proto;
3517	}
3518
3519	rc = sock_register(fam: &smc_sock_family_ops);
3520	if (rc) {
3521	pr_err("%s: sock_register fails with %d\n", __func__, rc);
3522	goto out_proto6;
3523	}
3524	INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
3525	INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
3526
3527	rc = smc_ib_register_client();
3528	if (rc) {
3529	pr_err("%s: ib_register fails with %d\n", __func__, rc);
3530	goto out_sock;
3531	}
3532
3533	rc = tcp_register_ulp(type: &smc_ulp_ops);
3534	if (rc) {
3535	pr_err("%s: tcp_ulp_register fails with %d\n", __func__, rc);
3536	goto out_ib;
3537	}
3538
3539	static_branch_enable(&tcp_have_smc);
3540	return 0;
3541
3542	out_ib:
3543	smc_ib_unregister_client();
3544	out_sock:
3545	sock_unregister(PF_SMC);
3546	out_proto6:
3547	proto_unregister(prot: &smc_proto6);
3548	out_proto:
3549	proto_unregister(prot: &smc_proto);
3550	out_core:
3551	smc_core_exit();
3552	out_alloc_wqs:
3553	destroy_workqueue(wq: smc_close_wq);
3554	out_alloc_hs_wq:
3555	destroy_workqueue(wq: smc_hs_wq);
3556	out_alloc_tcp_ls_wq:
3557	destroy_workqueue(wq: smc_tcp_ls_wq);
3558	out_pnet:
3559	smc_pnet_exit();
3560	out_nl:
3561	smc_nl_exit();
3562	out_ism:
3563	smc_clc_exit();
3564	smc_ism_exit();
3565	out_pernet_subsys_stat:
3566	unregister_pernet_subsys(&smc_net_stat_ops);
3567	out_pernet_subsys:
3568	unregister_pernet_subsys(&smc_net_ops);
3569
3570	return rc;
3571	}
3572
3573	static void __exit smc_exit(void)
3574	{
3575	static_branch_disable(&tcp_have_smc);
3576	tcp_unregister_ulp(type: &smc_ulp_ops);
3577	sock_unregister(PF_SMC);
3578	smc_core_exit();
3579	smc_ib_unregister_client();
3580	smc_ism_exit();
3581	destroy_workqueue(wq: smc_close_wq);
3582	destroy_workqueue(wq: smc_tcp_ls_wq);
3583	destroy_workqueue(wq: smc_hs_wq);
3584	proto_unregister(prot: &smc_proto6);
3585	proto_unregister(prot: &smc_proto);
3586	smc_pnet_exit();
3587	smc_nl_exit();
3588	smc_clc_exit();
3589	unregister_pernet_subsys(&smc_net_stat_ops);
3590	unregister_pernet_subsys(&smc_net_ops);
3591	rcu_barrier();
3592	}
3593
3594	module_init(smc_init);
3595	module_exit(smc_exit);
3596
3597	MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
3598	MODULE_DESCRIPTION("smc socket address family");
3599	MODULE_LICENSE("GPL");
3600	MODULE_ALIAS_NETPROTO(PF_SMC);
3601	MODULE_ALIAS_TCP_ULP("smc");
3602	MODULE_ALIAS_GENL_FAMILY(SMC_GENL_FAMILY_NAME);
3603