1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Shared Memory Communications over RDMA (SMC-R) and RoCE
4	*
5	* Link Layer Control (LLC)
6	*
7	* Copyright IBM Corp. 2016
8	*
9	* Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com>
10	* Ursula Braun <ubraun@linux.vnet.ibm.com>
11	*/
12
13	#include <net/tcp.h>
14	#include <rdma/ib_verbs.h>
15
16	#include "smc.h"
17	#include "smc_core.h"
18	#include "smc_clc.h"
19	#include "smc_llc.h"
20	#include "smc_pnet.h"
21
22	#define SMC_LLC_DATA_LEN 40
23
24	struct smc_llc_hdr {
25	struct smc_wr_rx_hdr common;
26	union {
27	struct {
28	u8 length; /* 44 */
29	#if defined(__BIG_ENDIAN_BITFIELD)
30	u8 reserved:4,
31	add_link_rej_rsn:4;
32	#elif defined(__LITTLE_ENDIAN_BITFIELD)
33	u8 add_link_rej_rsn:4,
34	reserved:4;
35	#endif
36	};
37	u16 length_v2; /* 44 - 8192*/
38	};
39	u8 flags;
40	} __packed; /* format defined in
41	* IBM Shared Memory Communications Version 2
42	* (https://www.ibm.com/support/pages/node/6326337)
43	*/
44
45	#define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03
46
47	struct smc_llc_msg_confirm_link { /* type 0x01 */
48	struct smc_llc_hdr hd;
49	u8 sender_mac[ETH_ALEN];
50	u8 sender_gid[SMC_GID_SIZE];
51	u8 sender_qp_num[3];
52	u8 link_num;
53	u8 link_uid[SMC_LGR_ID_SIZE];
54	u8 max_links;
55	u8 max_conns;
56	u8 reserved[8];
57	};
58
59	#define SMC_LLC_FLAG_ADD_LNK_REJ 0x40
60	#define SMC_LLC_REJ_RSN_NO_ALT_PATH 1
61
62	struct smc_llc_msg_add_link { /* type 0x02 */
63	struct smc_llc_hdr hd;
64	u8 sender_mac[ETH_ALEN];
65	u8 reserved2[2];
66	u8 sender_gid[SMC_GID_SIZE];
67	u8 sender_qp_num[3];
68	u8 link_num;
69	#if defined(__BIG_ENDIAN_BITFIELD)
70	u8 reserved3 : 4,
71	qp_mtu : 4;
72	#elif defined(__LITTLE_ENDIAN_BITFIELD)
73	u8 qp_mtu : 4,
74	reserved3 : 4;
75	#endif
76	u8 initial_psn[3];
77	u8 reserved[8];
78	};
79
80	struct smc_llc_msg_add_link_cont_rt {
81	__be32 rmb_key;
82	__be32 rmb_key_new;
83	__be64 rmb_vaddr_new;
84	};
85
86	struct smc_llc_msg_add_link_v2_ext {
87	#if defined(__BIG_ENDIAN_BITFIELD)
88	u8 v2_direct : 1,
89	reserved : 7;
90	#elif defined(__LITTLE_ENDIAN_BITFIELD)
91	u8 reserved : 7,
92	v2_direct : 1;
93	#endif
94	u8 reserved2;
95	u8 client_target_gid[SMC_GID_SIZE];
96	u8 reserved3[8];
97	u16 num_rkeys;
98	struct smc_llc_msg_add_link_cont_rt rt[];
99	} __packed; /* format defined in
100	* IBM Shared Memory Communications Version 2
101	* (https://www.ibm.com/support/pages/node/6326337)
102	*/
103
104	struct smc_llc_msg_req_add_link_v2 {
105	struct smc_llc_hdr hd;
106	u8 reserved[20];
107	u8 gid_cnt;
108	u8 reserved2[3];
109	u8 gid[][SMC_GID_SIZE];
110	};
111
112	#define SMC_LLC_RKEYS_PER_CONT_MSG 2
113
114	struct smc_llc_msg_add_link_cont { /* type 0x03 */
115	struct smc_llc_hdr hd;
116	u8 link_num;
117	u8 num_rkeys;
118	u8 reserved2[2];
119	struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG];
120	u8 reserved[4];
121	} __packed; /* format defined in RFC7609 */
122
123	#define SMC_LLC_FLAG_DEL_LINK_ALL 0x40
124	#define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20
125
126	struct smc_llc_msg_del_link { /* type 0x04 */
127	struct smc_llc_hdr hd;
128	u8 link_num;
129	__be32 reason;
130	u8 reserved[35];
131	} __packed; /* format defined in RFC7609 */
132
133	struct smc_llc_msg_test_link { /* type 0x07 */
134	struct smc_llc_hdr hd;
135	u8 user_data[16];
136	u8 reserved[24];
137	};
138
139	struct smc_rmb_rtoken {
140	union {
141	u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */
142	/* is actually the num of rtokens, first */
143	/* rtoken is always for the current link */
144	u8 link_id; /* link id of the rtoken */
145	};
146	__be32 rmb_key;
147	__be64 rmb_vaddr;
148	} __packed; /* format defined in RFC7609 */
149
150	#define SMC_LLC_RKEYS_PER_MSG 3
151	#define SMC_LLC_RKEYS_PER_MSG_V2 255
152
153	struct smc_llc_msg_confirm_rkey { /* type 0x06 */
154	struct smc_llc_hdr hd;
155	struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
156	u8 reserved;
157	};
158
159	#define SMC_LLC_DEL_RKEY_MAX 8
160	#define SMC_LLC_FLAG_RKEY_RETRY 0x10
161	#define SMC_LLC_FLAG_RKEY_NEG 0x20
162
163	struct smc_llc_msg_delete_rkey { /* type 0x09 */
164	struct smc_llc_hdr hd;
165	u8 num_rkeys;
166	u8 err_mask;
167	u8 reserved[2];
168	__be32 rkey[8];
169	u8 reserved2[4];
170	};
171
172	struct smc_llc_msg_delete_rkey_v2 { /* type 0x29 */
173	struct smc_llc_hdr hd;
174	u8 num_rkeys;
175	u8 num_inval_rkeys;
176	u8 reserved[2];
177	__be32 rkey[];
178	};
179
180	union smc_llc_msg {
181	struct smc_llc_msg_confirm_link confirm_link;
182	struct smc_llc_msg_add_link add_link;
183	struct smc_llc_msg_req_add_link_v2 req_add_link;
184	struct smc_llc_msg_add_link_cont add_link_cont;
185	struct smc_llc_msg_del_link delete_link;
186
187	struct smc_llc_msg_confirm_rkey confirm_rkey;
188	struct smc_llc_msg_delete_rkey delete_rkey;
189
190	struct smc_llc_msg_test_link test_link;
191	struct {
192	struct smc_llc_hdr hdr;
193	u8 data[SMC_LLC_DATA_LEN];
194	} raw;
195	};
196
197	#define SMC_LLC_FLAG_RESP 0x80
198
199	struct smc_llc_qentry {
200	struct list_head list;
201	struct smc_link *link;
202	union smc_llc_msg msg;
203	};
204
205	static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc);
206
207	struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
208	{
209	struct smc_llc_qentry *qentry = flow->qentry;
210
211	flow->qentry = NULL;
212	return qentry;
213	}
214
215	void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
216	{
217	struct smc_llc_qentry *qentry;
218
219	if (flow->qentry) {
220	qentry = flow->qentry;
221	flow->qentry = NULL;
222	kfree(objp: qentry);
223	}
224	}
225
226	static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
227	struct smc_llc_qentry *qentry)
228	{
229	flow->qentry = qentry;
230	}
231
232	static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type,
233	struct smc_llc_qentry *qentry)
234	{
235	u8 msg_type = qentry->msg.raw.hdr.common.llc_type;
236
237	if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) &&
238	flow_type != msg_type && !lgr->delayed_event) {
239	lgr->delayed_event = qentry;
240	return;
241	}
242	/* drop parallel or already-in-progress llc requests */
243	if (flow_type != msg_type)
244	pr_warn_once("smc: SMC-R lg %*phN net %llu dropped parallel "
245	"LLC msg: msg %d flow %d role %d\n",
246	SMC_LGR_ID_SIZE, &lgr->id,
247	lgr->net->net_cookie,
248	qentry->msg.raw.hdr.common.type,
249	flow_type, lgr->role);
250	kfree(objp: qentry);
251	}
252
253	/* try to start a new llc flow, initiated by an incoming llc msg */
254	static bool smc_llc_flow_start(struct smc_llc_flow *flow,
255	struct smc_llc_qentry *qentry)
256	{
257	struct smc_link_group *lgr = qentry->link->lgr;
258
259	spin_lock_bh(lock: &lgr->llc_flow_lock);
260	if (flow->type) {
261	/* a flow is already active */
262	smc_llc_flow_parallel(lgr, flow_type: flow->type, qentry);
263	spin_unlock_bh(lock: &lgr->llc_flow_lock);
264	return false;
265	}
266	switch (qentry->msg.raw.hdr.common.llc_type) {
267	case SMC_LLC_ADD_LINK:
268	flow->type = SMC_LLC_FLOW_ADD_LINK;
269	break;
270	case SMC_LLC_DELETE_LINK:
271	flow->type = SMC_LLC_FLOW_DEL_LINK;
272	break;
273	case SMC_LLC_CONFIRM_RKEY:
274	case SMC_LLC_DELETE_RKEY:
275	flow->type = SMC_LLC_FLOW_RKEY;
276	break;
277	default:
278	flow->type = SMC_LLC_FLOW_NONE;
279	}
280	smc_llc_flow_qentry_set(flow, qentry);
281	spin_unlock_bh(lock: &lgr->llc_flow_lock);
282	return true;
283	}
284
285	/* start a new local llc flow, wait till current flow finished */
286	int smc_llc_flow_initiate(struct smc_link_group *lgr,
287	enum smc_llc_flowtype type)
288	{
289	enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
290	int rc;
291
292	/* all flows except confirm_rkey and delete_rkey are exclusive,
293	* confirm/delete rkey flows can run concurrently (local and remote)
294	*/
295	if (type == SMC_LLC_FLOW_RKEY)
296	allowed_remote = SMC_LLC_FLOW_RKEY;
297	again:
298	if (list_empty(head: &lgr->list))
299	return -ENODEV;
300	spin_lock_bh(lock: &lgr->llc_flow_lock);
301	if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
302	(lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
303	lgr->llc_flow_rmt.type == allowed_remote)) {
304	lgr->llc_flow_lcl.type = type;
305	spin_unlock_bh(lock: &lgr->llc_flow_lock);
306	return 0;
307	}
308	spin_unlock_bh(lock: &lgr->llc_flow_lock);
309	rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) ||
310	(lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
311	(lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
312	lgr->llc_flow_rmt.type == allowed_remote))),
313	SMC_LLC_WAIT_TIME * 10);
314	if (!rc)
315	return -ETIMEDOUT;
316	goto again;
317	}
318
319	/* finish the current llc flow */
320	void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
321	{
322	spin_lock_bh(lock: &lgr->llc_flow_lock);
323	memset(flow, 0, sizeof(*flow));
324	flow->type = SMC_LLC_FLOW_NONE;
325	spin_unlock_bh(lock: &lgr->llc_flow_lock);
326	if (!list_empty(head: &lgr->list) && lgr->delayed_event &&
327	flow == &lgr->llc_flow_lcl)
328	schedule_work(work: &lgr->llc_event_work);
329	else
330	wake_up(&lgr->llc_flow_waiter);
331	}
332
333	/* lnk is optional and used for early wakeup when link goes down, useful in
334	* cases where we wait for a response on the link after we sent a request
335	*/
336	struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
337	struct smc_link *lnk,
338	int time_out, u8 exp_msg)
339	{
340	struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
341	u8 rcv_msg;
342
343	wait_event_timeout(lgr->llc_msg_waiter,
344	(flow->qentry ||
345	(lnk && !smc_link_usable(lnk)) ||
346	list_empty(&lgr->list)),
347	time_out);
348	if (!flow->qentry ||
349	(lnk && !smc_link_usable(lnk)) || list_empty(head: &lgr->list)) {
350	smc_llc_flow_qentry_del(flow);
351	goto out;
352	}
353	rcv_msg = flow->qentry->msg.raw.hdr.common.llc_type;
354	if (exp_msg && rcv_msg != exp_msg) {
355	if (exp_msg == SMC_LLC_ADD_LINK &&
356	rcv_msg == SMC_LLC_DELETE_LINK) {
357	/* flow_start will delay the unexpected msg */
358	smc_llc_flow_start(flow: &lgr->llc_flow_lcl,
359	qentry: smc_llc_flow_qentry_clr(flow));
360	return NULL;
361	}
362	pr_warn_once("smc: SMC-R lg %*phN net %llu dropped unexpected LLC msg: "
363	"msg %d exp %d flow %d role %d flags %x\n",
364	SMC_LGR_ID_SIZE, &lgr->id, lgr->net->net_cookie,
365	rcv_msg, exp_msg,
366	flow->type, lgr->role,
367	flow->qentry->msg.raw.hdr.flags);
368	smc_llc_flow_qentry_del(flow);
369	}
370	out:
371	return flow->qentry;
372	}
373
374	/********************************** send *************************************/
375
376	struct smc_llc_tx_pend {
377	};
378
379	/* handler for send/transmission completion of an LLC msg */
380	static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
381	struct smc_link *link,
382	enum ib_wc_status wc_status)
383	{
384	/* future work: handle wc_status error for recovery and failover */
385	}
386
387	/**
388	* smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
389	* @link: Pointer to SMC link used for sending LLC control message.
390	* @wr_buf: Out variable returning pointer to work request payload buffer.
391	* @pend: Out variable returning pointer to private pending WR tracking.
392	* It's the context the transmit complete handler will get.
393	*
394	* Reserves and pre-fills an entry for a pending work request send/tx.
395	* Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
396	* Can sleep due to smc_get_ctrl_buf (if not in softirq context).
397	*
398	* Return: 0 on success, otherwise an error value.
399	*/
400	static int smc_llc_add_pending_send(struct smc_link *link,
401	struct smc_wr_buf **wr_buf,
402	struct smc_wr_tx_pend_priv **pend)
403	{
404	int rc;
405
406	rc = smc_wr_tx_get_free_slot(link, handler: smc_llc_tx_handler, wr_buf, NULL,
407	wr_pend_priv: pend);
408	if (rc < 0)
409	return rc;
410	BUILD_BUG_ON_MSG(
411	sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
412	"must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
413	BUILD_BUG_ON_MSG(
414	sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
415	"must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
416	BUILD_BUG_ON_MSG(
417	sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
418	"must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
419	return 0;
420	}
421
422	static int smc_llc_add_pending_send_v2(struct smc_link *link,
423	struct smc_wr_v2_buf **wr_buf,
424	struct smc_wr_tx_pend_priv **pend)
425	{
426	int rc;
427
428	rc = smc_wr_tx_get_v2_slot(link, handler: smc_llc_tx_handler, wr_buf, wr_pend_priv: pend);
429	if (rc < 0)
430	return rc;
431	return 0;
432	}
433
434	static void smc_llc_init_msg_hdr(struct smc_llc_hdr *hdr,
435	struct smc_link_group *lgr, size_t len)
436	{
437	if (lgr->smc_version == SMC_V2) {
438	hdr->common.llc_version = SMC_V2;
439	hdr->length_v2 = len;
440	} else {
441	hdr->common.llc_version = 0;
442	hdr->length = len;
443	}
444	}
445
446	/* high-level API to send LLC confirm link */
447	int smc_llc_send_confirm_link(struct smc_link *link,
448	enum smc_llc_reqresp reqresp)
449	{
450	struct smc_llc_msg_confirm_link *confllc;
451	struct smc_wr_tx_pend_priv *pend;
452	struct smc_wr_buf *wr_buf;
453	int rc;
454
455	if (!smc_wr_tx_link_hold(link))
456	return -ENOLINK;
457	rc = smc_llc_add_pending_send(link, wr_buf: &wr_buf, pend: &pend);
458	if (rc)
459	goto put_out;
460	confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
461	memset(confllc, 0, sizeof(*confllc));
462	confllc->hd.common.llc_type = SMC_LLC_CONFIRM_LINK;
463	smc_llc_init_msg_hdr(hdr: &confllc->hd, lgr: link->lgr, len: sizeof(*confllc));
464	confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
465	if (reqresp == SMC_LLC_RESP)
466	confllc->hd.flags |= SMC_LLC_FLAG_RESP;
467	memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
468	ETH_ALEN);
469	memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
470	hton24(net: confllc->sender_qp_num, host: link->roce_qp->qp_num);
471	confllc->link_num = link->link_id;
472	memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE);
473	confllc->max_links = SMC_LINKS_ADD_LNK_MAX;
474	if (link->lgr->smc_version == SMC_V2 &&
475	link->lgr->peer_smc_release >= SMC_RELEASE_1) {
476	confllc->max_conns = link->lgr->max_conns;
477	confllc->max_links = link->lgr->max_links;
478	}
479	/* send llc message */
480	rc = smc_wr_tx_send(link, wr_pend_priv: pend);
481	put_out:
482	smc_wr_tx_link_put(link);
483	return rc;
484	}
485
486	/* send LLC confirm rkey request */
487	static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
488	struct smc_buf_desc *rmb_desc)
489	{
490	struct smc_llc_msg_confirm_rkey *rkeyllc;
491	struct smc_wr_tx_pend_priv *pend;
492	struct smc_wr_buf *wr_buf;
493	struct smc_link *link;
494	int i, rc, rtok_ix;
495
496	if (!smc_wr_tx_link_hold(link: send_link))
497	return -ENOLINK;
498	rc = smc_llc_add_pending_send(link: send_link, wr_buf: &wr_buf, pend: &pend);
499	if (rc)
500	goto put_out;
501	rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
502	memset(rkeyllc, 0, sizeof(*rkeyllc));
503	rkeyllc->hd.common.llc_type = SMC_LLC_CONFIRM_RKEY;
504	smc_llc_init_msg_hdr(hdr: &rkeyllc->hd, lgr: send_link->lgr, len: sizeof(*rkeyllc));
505
506	rtok_ix = 1;
507	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
508	link = &send_link->lgr->lnk[i];
509	if (smc_link_active(lnk: link) && link != send_link) {
510	rkeyllc->rtoken[rtok_ix].link_id = link->link_id;
511	rkeyllc->rtoken[rtok_ix].rmb_key =
512	htonl(rmb_desc->mr[link->link_idx]->rkey);
513	rkeyllc->rtoken[rtok_ix].rmb_vaddr = rmb_desc->is_vm ?
514	cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) :
515	cpu_to_be64((u64)sg_dma_address
516	(rmb_desc->sgt[link->link_idx].sgl));
517	rtok_ix++;
518	}
519	}
520	/* rkey of send_link is in rtoken[0] */
521	rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1;
522	rkeyllc->rtoken[0].rmb_key =
523	htonl(rmb_desc->mr[send_link->link_idx]->rkey);
524	rkeyllc->rtoken[0].rmb_vaddr = rmb_desc->is_vm ?
525	cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) :
526	cpu_to_be64((u64)sg_dma_address
527	(rmb_desc->sgt[send_link->link_idx].sgl));
528	/* send llc message */
529	rc = smc_wr_tx_send(link: send_link, wr_pend_priv: pend);
530	put_out:
531	smc_wr_tx_link_put(link: send_link);
532	return rc;
533	}
534
535	/* send LLC delete rkey request */
536	static int smc_llc_send_delete_rkey(struct smc_link *link,
537	struct smc_buf_desc *rmb_desc)
538	{
539	struct smc_llc_msg_delete_rkey *rkeyllc;
540	struct smc_wr_tx_pend_priv *pend;
541	struct smc_wr_buf *wr_buf;
542	int rc;
543
544	if (!smc_wr_tx_link_hold(link))
545	return -ENOLINK;
546	rc = smc_llc_add_pending_send(link, wr_buf: &wr_buf, pend: &pend);
547	if (rc)
548	goto put_out;
549	rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
550	memset(rkeyllc, 0, sizeof(*rkeyllc));
551	rkeyllc->hd.common.llc_type = SMC_LLC_DELETE_RKEY;
552	smc_llc_init_msg_hdr(hdr: &rkeyllc->hd, lgr: link->lgr, len: sizeof(*rkeyllc));
553	rkeyllc->num_rkeys = 1;
554	rkeyllc->rkey[0] = htonl(rmb_desc->mr[link->link_idx]->rkey);
555	/* send llc message */
556	rc = smc_wr_tx_send(link, wr_pend_priv: pend);
557	put_out:
558	smc_wr_tx_link_put(link);
559	return rc;
560	}
561
562	/* return first buffer from any of the next buf lists */
563	static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr,
564	int *buf_lst)
565	{
566	struct smc_buf_desc *buf_pos;
567
568	while (*buf_lst < SMC_RMBE_SIZES) {
569	buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst],
570	struct smc_buf_desc, list);
571	if (buf_pos)
572	return buf_pos;
573	(*buf_lst)++;
574	}
575	return NULL;
576	}
577
578	/* return next rmb from buffer lists */
579	static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr,
580	int *buf_lst,
581	struct smc_buf_desc *buf_pos)
582	{
583	struct smc_buf_desc *buf_next;
584
585	if (!buf_pos)
586	return _smc_llc_get_next_rmb(lgr, buf_lst);
587
588	if (list_is_last(list: &buf_pos->list, head: &lgr->rmbs[*buf_lst])) {
589	(*buf_lst)++;
590	return _smc_llc_get_next_rmb(lgr, buf_lst);
591	}
592	buf_next = list_next_entry(buf_pos, list);
593	return buf_next;
594	}
595
596	static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr,
597	int *buf_lst)
598	{
599	*buf_lst = 0;
600	return smc_llc_get_next_rmb(lgr, buf_lst, NULL);
601	}
602
603	static int smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext *ext,
604	struct smc_link *link, struct smc_link *link_new)
605	{
606	struct smc_link_group *lgr = link->lgr;
607	struct smc_buf_desc *buf_pos;
608	int prim_lnk_idx, lnk_idx, i;
609	struct smc_buf_desc *rmb;
610	int len = sizeof(*ext);
611	int buf_lst;
612
613	ext->v2_direct = !lgr->uses_gateway;
614	memcpy(ext->client_target_gid, link_new->gid, SMC_GID_SIZE);
615
616	prim_lnk_idx = link->link_idx;
617	lnk_idx = link_new->link_idx;
618	down_write(sem: &lgr->rmbs_lock);
619	ext->num_rkeys = lgr->conns_num;
620	if (!ext->num_rkeys)
621	goto out;
622	buf_pos = smc_llc_get_first_rmb(lgr, buf_lst: &buf_lst);
623	for (i = 0; i < ext->num_rkeys; i++) {
624	while (buf_pos && !(buf_pos)->used)
625	buf_pos = smc_llc_get_next_rmb(lgr, buf_lst: &buf_lst, buf_pos);
626	if (!buf_pos)
627	break;
628	rmb = buf_pos;
629	ext->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey);
630	ext->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey);
631	ext->rt[i].rmb_vaddr_new = rmb->is_vm ?
632	cpu_to_be64((uintptr_t)rmb->cpu_addr) :
633	cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
634	buf_pos = smc_llc_get_next_rmb(lgr, buf_lst: &buf_lst, buf_pos);
635	}
636	len += i * sizeof(ext->rt[0]);
637	out:
638	up_write(sem: &lgr->rmbs_lock);
639	return len;
640	}
641
642	/* send ADD LINK request or response */
643	int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
644	struct smc_link *link_new,
645	enum smc_llc_reqresp reqresp)
646	{
647	struct smc_llc_msg_add_link_v2_ext *ext = NULL;
648	struct smc_llc_msg_add_link *addllc;
649	struct smc_wr_tx_pend_priv *pend;
650	int len = sizeof(*addllc);
651	int rc;
652
653	if (!smc_wr_tx_link_hold(link))
654	return -ENOLINK;
655	if (link->lgr->smc_version == SMC_V2) {
656	struct smc_wr_v2_buf *wr_buf;
657
658	rc = smc_llc_add_pending_send_v2(link, wr_buf: &wr_buf, pend: &pend);
659	if (rc)
660	goto put_out;
661	addllc = (struct smc_llc_msg_add_link *)wr_buf;
662	ext = (struct smc_llc_msg_add_link_v2_ext *)
663	&wr_buf->raw[sizeof(*addllc)];
664	memset(ext, 0, SMC_WR_TX_SIZE);
665	} else {
666	struct smc_wr_buf *wr_buf;
667
668	rc = smc_llc_add_pending_send(link, wr_buf: &wr_buf, pend: &pend);
669	if (rc)
670	goto put_out;
671	addllc = (struct smc_llc_msg_add_link *)wr_buf;
672	}
673
674	memset(addllc, 0, sizeof(*addllc));
675	addllc->hd.common.llc_type = SMC_LLC_ADD_LINK;
676	if (reqresp == SMC_LLC_RESP)
677	addllc->hd.flags |= SMC_LLC_FLAG_RESP;
678	memcpy(addllc->sender_mac, mac, ETH_ALEN);
679	memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
680	if (link_new) {
681	addllc->link_num = link_new->link_id;
682	hton24(net: addllc->sender_qp_num, host: link_new->roce_qp->qp_num);
683	hton24(net: addllc->initial_psn, host: link_new->psn_initial);
684	if (reqresp == SMC_LLC_REQ)
685	addllc->qp_mtu = link_new->path_mtu;
686	else
687	addllc->qp_mtu = min(link_new->path_mtu,
688	link_new->peer_mtu);
689	}
690	if (ext && link_new)
691	len += smc_llc_fill_ext_v2(ext, link, link_new);
692	smc_llc_init_msg_hdr(hdr: &addllc->hd, lgr: link->lgr, len);
693	/* send llc message */
694	if (link->lgr->smc_version == SMC_V2)
695	rc = smc_wr_tx_v2_send(link, priv: pend, len);
696	else
697	rc = smc_wr_tx_send(link, wr_pend_priv: pend);
698	put_out:
699	smc_wr_tx_link_put(link);
700	return rc;
701	}
702
703	/* send DELETE LINK request or response */
704	int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
705	enum smc_llc_reqresp reqresp, bool orderly,
706	u32 reason)
707	{
708	struct smc_llc_msg_del_link *delllc;
709	struct smc_wr_tx_pend_priv *pend;
710	struct smc_wr_buf *wr_buf;
711	int rc;
712
713	if (!smc_wr_tx_link_hold(link))
714	return -ENOLINK;
715	rc = smc_llc_add_pending_send(link, wr_buf: &wr_buf, pend: &pend);
716	if (rc)
717	goto put_out;
718	delllc = (struct smc_llc_msg_del_link *)wr_buf;
719
720	memset(delllc, 0, sizeof(*delllc));
721	delllc->hd.common.llc_type = SMC_LLC_DELETE_LINK;
722	smc_llc_init_msg_hdr(hdr: &delllc->hd, lgr: link->lgr, len: sizeof(*delllc));
723	if (reqresp == SMC_LLC_RESP)
724	delllc->hd.flags |= SMC_LLC_FLAG_RESP;
725	if (orderly)
726	delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
727	if (link_del_id)
728	delllc->link_num = link_del_id;
729	else
730	delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
731	delllc->reason = htonl(reason);
732	/* send llc message */
733	rc = smc_wr_tx_send(link, wr_pend_priv: pend);
734	put_out:
735	smc_wr_tx_link_put(link);
736	return rc;
737	}
738
739	/* send LLC test link request */
740	static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
741	{
742	struct smc_llc_msg_test_link *testllc;
743	struct smc_wr_tx_pend_priv *pend;
744	struct smc_wr_buf *wr_buf;
745	int rc;
746
747	if (!smc_wr_tx_link_hold(link))
748	return -ENOLINK;
749	rc = smc_llc_add_pending_send(link, wr_buf: &wr_buf, pend: &pend);
750	if (rc)
751	goto put_out;
752	testllc = (struct smc_llc_msg_test_link *)wr_buf;
753	memset(testllc, 0, sizeof(*testllc));
754	testllc->hd.common.llc_type = SMC_LLC_TEST_LINK;
755	smc_llc_init_msg_hdr(hdr: &testllc->hd, lgr: link->lgr, len: sizeof(*testllc));
756	memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
757	/* send llc message */
758	rc = smc_wr_tx_send(link, wr_pend_priv: pend);
759	put_out:
760	smc_wr_tx_link_put(link);
761	return rc;
762	}
763
764	/* schedule an llc send on link, may wait for buffers */
765	static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
766	{
767	struct smc_wr_tx_pend_priv *pend;
768	struct smc_wr_buf *wr_buf;
769	int rc;
770
771	if (!smc_wr_tx_link_hold(link))
772	return -ENOLINK;
773	rc = smc_llc_add_pending_send(link, wr_buf: &wr_buf, pend: &pend);
774	if (rc)
775	goto put_out;
776	memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
777	rc = smc_wr_tx_send(link, wr_pend_priv: pend);
778	put_out:
779	smc_wr_tx_link_put(link);
780	return rc;
781	}
782
783	/* schedule an llc send on link, may wait for buffers,
784	* and wait for send completion notification.
785	* @return 0 on success
786	*/
787	static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf)
788	{
789	struct smc_wr_tx_pend_priv *pend;
790	struct smc_wr_buf *wr_buf;
791	int rc;
792
793	if (!smc_wr_tx_link_hold(link))
794	return -ENOLINK;
795	rc = smc_llc_add_pending_send(link, wr_buf: &wr_buf, pend: &pend);
796	if (rc)
797	goto put_out;
798	memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
799	rc = smc_wr_tx_send_wait(link, priv: pend, SMC_LLC_WAIT_TIME);
800	put_out:
801	smc_wr_tx_link_put(link);
802	return rc;
803	}
804
805	/********************************* receive ***********************************/
806
807	static int smc_llc_alloc_alt_link(struct smc_link_group *lgr,
808	enum smc_lgr_type lgr_new_t)
809	{
810	int i;
811
812	if (lgr->type == SMC_LGR_SYMMETRIC ||
813	(lgr->type != SMC_LGR_SINGLE &&
814	(lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
815	lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)))
816	return -EMLINK;
817
818	if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
819	lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) {
820	for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--)
821	if (lgr->lnk[i].state == SMC_LNK_UNUSED)
822	return i;
823	} else {
824	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
825	if (lgr->lnk[i].state == SMC_LNK_UNUSED)
826	return i;
827	}
828	return -EMLINK;
829	}
830
831	/* send one add_link_continue msg */
832	static int smc_llc_add_link_cont(struct smc_link *link,
833	struct smc_link *link_new, u8 *num_rkeys_todo,
834	int *buf_lst, struct smc_buf_desc **buf_pos)
835	{
836	struct smc_llc_msg_add_link_cont *addc_llc;
837	struct smc_link_group *lgr = link->lgr;
838	int prim_lnk_idx, lnk_idx, i, rc;
839	struct smc_wr_tx_pend_priv *pend;
840	struct smc_wr_buf *wr_buf;
841	struct smc_buf_desc *rmb;
842	u8 n;
843
844	if (!smc_wr_tx_link_hold(link))
845	return -ENOLINK;
846	rc = smc_llc_add_pending_send(link, wr_buf: &wr_buf, pend: &pend);
847	if (rc)
848	goto put_out;
849	addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf;
850	memset(addc_llc, 0, sizeof(*addc_llc));
851
852	prim_lnk_idx = link->link_idx;
853	lnk_idx = link_new->link_idx;
854	addc_llc->link_num = link_new->link_id;
855	addc_llc->num_rkeys = *num_rkeys_todo;
856	n = *num_rkeys_todo;
857	for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) {
858	while (*buf_pos && !(*buf_pos)->used)
859	*buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, buf_pos: *buf_pos);
860	if (!*buf_pos) {
861	addc_llc->num_rkeys = addc_llc->num_rkeys -
862	*num_rkeys_todo;
863	*num_rkeys_todo = 0;
864	break;
865	}
866	rmb = *buf_pos;
867
868	addc_llc->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey);
869	addc_llc->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey);
870	addc_llc->rt[i].rmb_vaddr_new = rmb->is_vm ?
871	cpu_to_be64((uintptr_t)rmb->cpu_addr) :
872	cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
873
874	(*num_rkeys_todo)--;
875	*buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, buf_pos: *buf_pos);
876	}
877	addc_llc->hd.common.llc_type = SMC_LLC_ADD_LINK_CONT;
878	addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont);
879	if (lgr->role == SMC_CLNT)
880	addc_llc->hd.flags |= SMC_LLC_FLAG_RESP;
881	rc = smc_wr_tx_send(link, wr_pend_priv: pend);
882	put_out:
883	smc_wr_tx_link_put(link);
884	return rc;
885	}
886
887	static int smc_llc_cli_rkey_exchange(struct smc_link *link,
888	struct smc_link *link_new)
889	{
890	struct smc_llc_msg_add_link_cont *addc_llc;
891	struct smc_link_group *lgr = link->lgr;
892	u8 max, num_rkeys_send, num_rkeys_recv;
893	struct smc_llc_qentry *qentry;
894	struct smc_buf_desc *buf_pos;
895	int buf_lst;
896	int rc = 0;
897	int i;
898
899	down_write(sem: &lgr->rmbs_lock);
900	num_rkeys_send = lgr->conns_num;
901	buf_pos = smc_llc_get_first_rmb(lgr, buf_lst: &buf_lst);
902	do {
903	qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME,
904	exp_msg: SMC_LLC_ADD_LINK_CONT);
905	if (!qentry) {
906	rc = -ETIMEDOUT;
907	break;
908	}
909	addc_llc = &qentry->msg.add_link_cont;
910	num_rkeys_recv = addc_llc->num_rkeys;
911	max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
912	for (i = 0; i < max; i++) {
913	smc_rtoken_set(lgr, link_idx: link->link_idx, link_idx_new: link_new->link_idx,
914	nw_rkey_known: addc_llc->rt[i].rmb_key,
915	nw_vaddr: addc_llc->rt[i].rmb_vaddr_new,
916	nw_rkey: addc_llc->rt[i].rmb_key_new);
917	num_rkeys_recv--;
918	}
919	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
920	rc = smc_llc_add_link_cont(link, link_new, num_rkeys_todo: &num_rkeys_send,
921	buf_lst: &buf_lst, buf_pos: &buf_pos);
922	if (rc)
923	break;
924	} while (num_rkeys_send || num_rkeys_recv);
925
926	up_write(sem: &lgr->rmbs_lock);
927	return rc;
928	}
929
930	/* prepare and send an add link reject response */
931	static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry)
932	{
933	qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
934	qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
935	qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
936	smc_llc_init_msg_hdr(hdr: &qentry->msg.raw.hdr, lgr: qentry->link->lgr,
937	len: sizeof(qentry->msg));
938	return smc_llc_send_message(link: qentry->link, llcbuf: &qentry->msg);
939	}
940
941	static int smc_llc_cli_conf_link(struct smc_link *link,
942	struct smc_init_info *ini,
943	struct smc_link *link_new,
944	enum smc_lgr_type lgr_new_t)
945	{
946	struct smc_link_group *lgr = link->lgr;
947	struct smc_llc_qentry *qentry = NULL;
948	int rc = 0;
949
950	/* receive CONFIRM LINK request over RoCE fabric */
951	qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, exp_msg: 0);
952	if (!qentry) {
953	rc = smc_llc_send_delete_link(link, link_del_id: link_new->link_id,
954	reqresp: SMC_LLC_REQ, orderly: false,
955	SMC_LLC_DEL_LOST_PATH);
956	return -ENOLINK;
957	}
958	if (qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
959	/* received DELETE_LINK instead */
960	qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
961	smc_llc_send_message(link, llcbuf: &qentry->msg);
962	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
963	return -ENOLINK;
964	}
965	smc_llc_save_peer_uid(qentry);
966	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
967
968	rc = smc_ib_modify_qp_rts(lnk: link_new);
969	if (rc) {
970	smc_llc_send_delete_link(link, link_del_id: link_new->link_id, reqresp: SMC_LLC_REQ,
971	orderly: false, SMC_LLC_DEL_LOST_PATH);
972	return -ENOLINK;
973	}
974	smc_wr_remember_qp_attr(lnk: link_new);
975
976	rc = smcr_buf_reg_lgr(lnk: link_new);
977	if (rc) {
978	smc_llc_send_delete_link(link, link_del_id: link_new->link_id, reqresp: SMC_LLC_REQ,
979	orderly: false, SMC_LLC_DEL_LOST_PATH);
980	return -ENOLINK;
981	}
982
983	/* send CONFIRM LINK response over RoCE fabric */
984	rc = smc_llc_send_confirm_link(link: link_new, reqresp: SMC_LLC_RESP);
985	if (rc) {
986	smc_llc_send_delete_link(link, link_del_id: link_new->link_id, reqresp: SMC_LLC_REQ,
987	orderly: false, SMC_LLC_DEL_LOST_PATH);
988	return -ENOLINK;
989	}
990	smc_llc_link_active(link: link_new);
991	if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
992	lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
993	smcr_lgr_set_type_asym(lgr, new_type: lgr_new_t, asym_lnk_idx: link_new->link_idx);
994	else
995	smcr_lgr_set_type(lgr, new_type: lgr_new_t);
996	return 0;
997	}
998
999	static void smc_llc_save_add_link_rkeys(struct smc_link *link,
1000	struct smc_link *link_new)
1001	{
1002	struct smc_llc_msg_add_link_v2_ext *ext;
1003	struct smc_link_group *lgr = link->lgr;
1004	int max, i;
1005
1006	ext = (struct smc_llc_msg_add_link_v2_ext *)((u8 *)lgr->wr_rx_buf_v2 +
1007	SMC_WR_TX_SIZE);
1008	max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
1009	down_write(sem: &lgr->rmbs_lock);
1010	for (i = 0; i < max; i++) {
1011	smc_rtoken_set(lgr, link_idx: link->link_idx, link_idx_new: link_new->link_idx,
1012	nw_rkey_known: ext->rt[i].rmb_key,
1013	nw_vaddr: ext->rt[i].rmb_vaddr_new,
1014	nw_rkey: ext->rt[i].rmb_key_new);
1015	}
1016	up_write(sem: &lgr->rmbs_lock);
1017	}
1018
1019	static void smc_llc_save_add_link_info(struct smc_link *link,
1020	struct smc_llc_msg_add_link *add_llc)
1021	{
1022	link->peer_qpn = ntoh24(net: add_llc->sender_qp_num);
1023	memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE);
1024	memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN);
1025	link->peer_psn = ntoh24(net: add_llc->initial_psn);
1026	link->peer_mtu = add_llc->qp_mtu;
1027	}
1028
1029	/* as an SMC client, process an add link request */
1030	int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry)
1031	{
1032	struct smc_llc_msg_add_link *llc = &qentry->msg.add_link;
1033	enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1034	struct smc_link_group *lgr = smc_get_lgr(link);
1035	struct smc_init_info *ini = NULL;
1036	struct smc_link *lnk_new = NULL;
1037	int lnk_idx, rc = 0;
1038
1039	if (!llc->qp_mtu)
1040	goto out_reject;
1041
1042	ini = kzalloc(size: sizeof(*ini), GFP_KERNEL);
1043	if (!ini) {
1044	rc = -ENOMEM;
1045	goto out_reject;
1046	}
1047
1048	if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1) {
1049	rc = 0;
1050	goto out_reject;
1051	}
1052
1053	ini->vlan_id = lgr->vlan_id;
1054	if (lgr->smc_version == SMC_V2) {
1055	ini->check_smcrv2 = true;
1056	ini->smcrv2.saddr = lgr->saddr;
1057	ini->smcrv2.daddr = smc_ib_gid_to_ipv4(gid: llc->sender_gid);
1058	}
1059	smc_pnet_find_alt_roce(lgr, ini, known_dev: link->smcibdev);
1060	if (!memcmp(p: llc->sender_gid, q: link->peer_gid, SMC_GID_SIZE) &&
1061	(lgr->smc_version == SMC_V2 ||
1062	!memcmp(p: llc->sender_mac, q: link->peer_mac, ETH_ALEN))) {
1063	if (!ini->ib_dev && !ini->smcrv2.ib_dev_v2)
1064	goto out_reject;
1065	lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1066	}
1067	if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1068	lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1069	ini->smcrv2.ib_dev_v2 = link->smcibdev;
1070	ini->smcrv2.ib_port_v2 = link->ibport;
1071	} else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
1072	lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1073	ini->ib_dev = link->smcibdev;
1074	ini->ib_port = link->ibport;
1075	}
1076	lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1077	if (lnk_idx < 0)
1078	goto out_reject;
1079	lnk_new = &lgr->lnk[lnk_idx];
1080	rc = smcr_link_init(lgr, lnk: lnk_new, link_idx: lnk_idx, ini);
1081	if (rc)
1082	goto out_reject;
1083	smc_llc_save_add_link_info(link: lnk_new, add_llc: llc);
1084	lnk_new->link_id = llc->link_num; /* SMC server assigns link id */
1085	smc_llc_link_set_uid(link: lnk_new);
1086
1087	rc = smc_ib_ready_link(lnk: lnk_new);
1088	if (rc)
1089	goto out_clear_lnk;
1090
1091	rc = smcr_buf_map_lgr(lnk: lnk_new);
1092	if (rc)
1093	goto out_clear_lnk;
1094
1095	rc = smc_llc_send_add_link(link,
1096	mac: lnk_new->smcibdev->mac[lnk_new->ibport - 1],
1097	gid: lnk_new->gid, link_new: lnk_new, reqresp: SMC_LLC_RESP);
1098	if (rc)
1099	goto out_clear_lnk;
1100	if (lgr->smc_version == SMC_V2) {
1101	smc_llc_save_add_link_rkeys(link, link_new: lnk_new);
1102	} else {
1103	rc = smc_llc_cli_rkey_exchange(link, link_new: lnk_new);
1104	if (rc) {
1105	rc = 0;
1106	goto out_clear_lnk;
1107	}
1108	}
1109	rc = smc_llc_cli_conf_link(link, ini, link_new: lnk_new, lgr_new_t);
1110	if (!rc)
1111	goto out;
1112	out_clear_lnk:
1113	lnk_new->state = SMC_LNK_INACTIVE;
1114	smcr_link_clear(lnk: lnk_new, log: false);
1115	out_reject:
1116	smc_llc_cli_add_link_reject(qentry);
1117	out:
1118	kfree(objp: ini);
1119	kfree(objp: qentry);
1120	return rc;
1121	}
1122
1123	static void smc_llc_send_request_add_link(struct smc_link *link)
1124	{
1125	struct smc_llc_msg_req_add_link_v2 *llc;
1126	struct smc_wr_tx_pend_priv *pend;
1127	struct smc_wr_v2_buf *wr_buf;
1128	struct smc_gidlist gidlist;
1129	int rc, len, i;
1130
1131	if (!smc_wr_tx_link_hold(link))
1132	return;
1133	if (link->lgr->type == SMC_LGR_SYMMETRIC ||
1134	link->lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1135	goto put_out;
1136
1137	smc_fill_gid_list(lgr: link->lgr, gidlist: &gidlist, known_dev: link->smcibdev, known_gid: link->gid);
1138	if (gidlist.len <= 1)
1139	goto put_out;
1140
1141	rc = smc_llc_add_pending_send_v2(link, wr_buf: &wr_buf, pend: &pend);
1142	if (rc)
1143	goto put_out;
1144	llc = (struct smc_llc_msg_req_add_link_v2 *)wr_buf;
1145	memset(llc, 0, SMC_WR_TX_SIZE);
1146
1147	llc->hd.common.llc_type = SMC_LLC_REQ_ADD_LINK;
1148	for (i = 0; i < gidlist.len; i++)
1149	memcpy(llc->gid[i], gidlist.list[i], sizeof(gidlist.list[0]));
1150	llc->gid_cnt = gidlist.len;
1151	len = sizeof(*llc) + (gidlist.len * sizeof(gidlist.list[0]));
1152	smc_llc_init_msg_hdr(hdr: &llc->hd, lgr: link->lgr, len);
1153	rc = smc_wr_tx_v2_send(link, priv: pend, len);
1154	if (!rc)
1155	/* set REQ_ADD_LINK flow and wait for response from peer */
1156	link->lgr->llc_flow_lcl.type = SMC_LLC_FLOW_REQ_ADD_LINK;
1157	put_out:
1158	smc_wr_tx_link_put(link);
1159	}
1160
1161	/* as an SMC client, invite server to start the add_link processing */
1162	static void smc_llc_cli_add_link_invite(struct smc_link *link,
1163	struct smc_llc_qentry *qentry)
1164	{
1165	struct smc_link_group *lgr = smc_get_lgr(link);
1166	struct smc_init_info *ini = NULL;
1167
1168	if (lgr->smc_version == SMC_V2) {
1169	smc_llc_send_request_add_link(link);
1170	goto out;
1171	}
1172
1173	if (lgr->type == SMC_LGR_SYMMETRIC ||
1174	lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1175	goto out;
1176
1177	if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1)
1178	goto out;
1179
1180	ini = kzalloc(size: sizeof(*ini), GFP_KERNEL);
1181	if (!ini)
1182	goto out;
1183
1184	ini->vlan_id = lgr->vlan_id;
1185	smc_pnet_find_alt_roce(lgr, ini, known_dev: link->smcibdev);
1186	if (!ini->ib_dev)
1187	goto out;
1188
1189	smc_llc_send_add_link(link, mac: ini->ib_dev->mac[ini->ib_port - 1],
1190	gid: ini->ib_gid, NULL, reqresp: SMC_LLC_REQ);
1191	out:
1192	kfree(objp: ini);
1193	kfree(objp: qentry);
1194	}
1195
1196	static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc)
1197	{
1198	int i;
1199
1200	for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++)
1201	if (llc->raw.data[i])
1202	return false;
1203	return true;
1204	}
1205
1206	static bool smc_llc_is_local_add_link(union smc_llc_msg *llc)
1207	{
1208	if (llc->raw.hdr.common.llc_type == SMC_LLC_ADD_LINK &&
1209	smc_llc_is_empty_llc_message(llc))
1210	return true;
1211	return false;
1212	}
1213
1214	static void smc_llc_process_cli_add_link(struct smc_link_group *lgr)
1215	{
1216	struct smc_llc_qentry *qentry;
1217
1218	qentry = smc_llc_flow_qentry_clr(flow: &lgr->llc_flow_lcl);
1219
1220	down_write(sem: &lgr->llc_conf_mutex);
1221	if (smc_llc_is_local_add_link(llc: &qentry->msg))
1222	smc_llc_cli_add_link_invite(link: qentry->link, qentry);
1223	else
1224	smc_llc_cli_add_link(link: qentry->link, qentry);
1225	up_write(sem: &lgr->llc_conf_mutex);
1226	}
1227
1228	static int smc_llc_active_link_count(struct smc_link_group *lgr)
1229	{
1230	int i, link_count = 0;
1231
1232	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1233	if (!smc_link_active(lnk: &lgr->lnk[i]))
1234	continue;
1235	link_count++;
1236	}
1237	return link_count;
1238	}
1239
1240	/* find the asymmetric link when 3 links are established */
1241	static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr)
1242	{
1243	int asym_idx = -ENOENT;
1244	int i, j, k;
1245	bool found;
1246
1247	/* determine asymmetric link */
1248	found = false;
1249	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1250	for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) {
1251	if (!smc_link_usable(lnk: &lgr->lnk[i]) ||
1252	!smc_link_usable(lnk: &lgr->lnk[j]))
1253	continue;
1254	if (!memcmp(p: lgr->lnk[i].gid, q: lgr->lnk[j].gid,
1255	SMC_GID_SIZE)) {
1256	found = true; /* asym_lnk is i or j */
1257	break;
1258	}
1259	}
1260	if (found)
1261	break;
1262	}
1263	if (!found)
1264	goto out; /* no asymmetric link */
1265	for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) {
1266	if (!smc_link_usable(lnk: &lgr->lnk[k]))
1267	continue;
1268	if (k != i &&
1269	!memcmp(p: lgr->lnk[i].peer_gid, q: lgr->lnk[k].peer_gid,
1270	SMC_GID_SIZE)) {
1271	asym_idx = i;
1272	break;
1273	}
1274	if (k != j &&
1275	!memcmp(p: lgr->lnk[j].peer_gid, q: lgr->lnk[k].peer_gid,
1276	SMC_GID_SIZE)) {
1277	asym_idx = j;
1278	break;
1279	}
1280	}
1281	out:
1282	return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx];
1283	}
1284
1285	static void smc_llc_delete_asym_link(struct smc_link_group *lgr)
1286	{
1287	struct smc_link *lnk_new = NULL, *lnk_asym;
1288	struct smc_llc_qentry *qentry;
1289	int rc;
1290
1291	lnk_asym = smc_llc_find_asym_link(lgr);
1292	if (!lnk_asym)
1293	return; /* no asymmetric link */
1294	if (!smc_link_downing(&lnk_asym->state))
1295	return;
1296	lnk_new = smc_switch_conns(lgr, from_lnk: lnk_asym, is_dev_err: false);
1297	smc_wr_tx_wait_no_pending_sends(link: lnk_asym);
1298	if (!lnk_new)
1299	goto out_free;
1300	/* change flow type from ADD_LINK into DEL_LINK */
1301	lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK;
1302	rc = smc_llc_send_delete_link(link: lnk_new, link_del_id: lnk_asym->link_id, reqresp: SMC_LLC_REQ,
1303	orderly: true, SMC_LLC_DEL_NO_ASYM_NEEDED);
1304	if (rc) {
1305	smcr_link_down_cond(lnk: lnk_new);
1306	goto out_free;
1307	}
1308	qentry = smc_llc_wait(lgr, lnk: lnk_new, SMC_LLC_WAIT_TIME,
1309	exp_msg: SMC_LLC_DELETE_LINK);
1310	if (!qentry) {
1311	smcr_link_down_cond(lnk: lnk_new);
1312	goto out_free;
1313	}
1314	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
1315	out_free:
1316	smcr_link_clear(lnk: lnk_asym, log: true);
1317	}
1318
1319	static int smc_llc_srv_rkey_exchange(struct smc_link *link,
1320	struct smc_link *link_new)
1321	{
1322	struct smc_llc_msg_add_link_cont *addc_llc;
1323	struct smc_link_group *lgr = link->lgr;
1324	u8 max, num_rkeys_send, num_rkeys_recv;
1325	struct smc_llc_qentry *qentry = NULL;
1326	struct smc_buf_desc *buf_pos;
1327	int buf_lst;
1328	int rc = 0;
1329	int i;
1330
1331	down_write(sem: &lgr->rmbs_lock);
1332	num_rkeys_send = lgr->conns_num;
1333	buf_pos = smc_llc_get_first_rmb(lgr, buf_lst: &buf_lst);
1334	do {
1335	smc_llc_add_link_cont(link, link_new, num_rkeys_todo: &num_rkeys_send,
1336	buf_lst: &buf_lst, buf_pos: &buf_pos);
1337	qentry = smc_llc_wait(lgr, lnk: link, SMC_LLC_WAIT_TIME,
1338	exp_msg: SMC_LLC_ADD_LINK_CONT);
1339	if (!qentry) {
1340	rc = -ETIMEDOUT;
1341	goto out;
1342	}
1343	addc_llc = &qentry->msg.add_link_cont;
1344	num_rkeys_recv = addc_llc->num_rkeys;
1345	max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
1346	for (i = 0; i < max; i++) {
1347	smc_rtoken_set(lgr, link_idx: link->link_idx, link_idx_new: link_new->link_idx,
1348	nw_rkey_known: addc_llc->rt[i].rmb_key,
1349	nw_vaddr: addc_llc->rt[i].rmb_vaddr_new,
1350	nw_rkey: addc_llc->rt[i].rmb_key_new);
1351	num_rkeys_recv--;
1352	}
1353	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
1354	} while (num_rkeys_send || num_rkeys_recv);
1355	out:
1356	up_write(sem: &lgr->rmbs_lock);
1357	return rc;
1358	}
1359
1360	static int smc_llc_srv_conf_link(struct smc_link *link,
1361	struct smc_link *link_new,
1362	enum smc_lgr_type lgr_new_t)
1363	{
1364	struct smc_link_group *lgr = link->lgr;
1365	struct smc_llc_qentry *qentry = NULL;
1366	int rc;
1367
1368	/* send CONFIRM LINK request over the RoCE fabric */
1369	rc = smc_llc_send_confirm_link(link: link_new, reqresp: SMC_LLC_REQ);
1370	if (rc)
1371	return -ENOLINK;
1372	/* receive CONFIRM LINK response over the RoCE fabric */
1373	qentry = smc_llc_wait(lgr, lnk: link, SMC_LLC_WAIT_FIRST_TIME, exp_msg: 0);
1374	if (!qentry ||
1375	qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
1376	/* send DELETE LINK */
1377	smc_llc_send_delete_link(link, link_del_id: link_new->link_id, reqresp: SMC_LLC_REQ,
1378	orderly: false, SMC_LLC_DEL_LOST_PATH);
1379	if (qentry)
1380	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
1381	return -ENOLINK;
1382	}
1383	smc_llc_save_peer_uid(qentry);
1384	smc_llc_link_active(link: link_new);
1385	if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
1386	lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
1387	smcr_lgr_set_type_asym(lgr, new_type: lgr_new_t, asym_lnk_idx: link_new->link_idx);
1388	else
1389	smcr_lgr_set_type(lgr, new_type: lgr_new_t);
1390	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
1391	return 0;
1392	}
1393
1394	static void smc_llc_send_req_add_link_response(struct smc_llc_qentry *qentry)
1395	{
1396	qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
1397	smc_llc_init_msg_hdr(hdr: &qentry->msg.raw.hdr, lgr: qentry->link->lgr,
1398	len: sizeof(qentry->msg));
1399	memset(&qentry->msg.raw.data, 0, sizeof(qentry->msg.raw.data));
1400	smc_llc_send_message(link: qentry->link, llcbuf: &qentry->msg);
1401	}
1402
1403	int smc_llc_srv_add_link(struct smc_link *link,
1404	struct smc_llc_qentry *req_qentry)
1405	{
1406	enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1407	struct smc_link_group *lgr = link->lgr;
1408	struct smc_llc_msg_add_link *add_llc;
1409	struct smc_llc_qentry *qentry = NULL;
1410	bool send_req_add_link_resp = false;
1411	struct smc_link *link_new = NULL;
1412	struct smc_init_info *ini = NULL;
1413	int lnk_idx, rc = 0;
1414
1415	if (req_qentry &&
1416	req_qentry->msg.raw.hdr.common.llc_type == SMC_LLC_REQ_ADD_LINK)
1417	send_req_add_link_resp = true;
1418
1419	ini = kzalloc(size: sizeof(*ini), GFP_KERNEL);
1420	if (!ini) {
1421	rc = -ENOMEM;
1422	goto out;
1423	}
1424
1425	if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1) {
1426	rc = 0;
1427	goto out;
1428	}
1429
1430	/* ignore client add link recommendation, start new flow */
1431	ini->vlan_id = lgr->vlan_id;
1432	if (lgr->smc_version == SMC_V2) {
1433	ini->check_smcrv2 = true;
1434	ini->smcrv2.saddr = lgr->saddr;
1435	if (send_req_add_link_resp) {
1436	struct smc_llc_msg_req_add_link_v2 *req_add =
1437	&req_qentry->msg.req_add_link;
1438
1439	ini->smcrv2.daddr = smc_ib_gid_to_ipv4(gid: req_add->gid[0]);
1440	}
1441	}
1442	smc_pnet_find_alt_roce(lgr, ini, known_dev: link->smcibdev);
1443	if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1444	lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1445	ini->smcrv2.ib_dev_v2 = link->smcibdev;
1446	ini->smcrv2.ib_port_v2 = link->ibport;
1447	} else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
1448	lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1449	ini->ib_dev = link->smcibdev;
1450	ini->ib_port = link->ibport;
1451	}
1452	lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1453	if (lnk_idx < 0) {
1454	rc = 0;
1455	goto out;
1456	}
1457
1458	rc = smcr_link_init(lgr, lnk: &lgr->lnk[lnk_idx], link_idx: lnk_idx, ini);
1459	if (rc)
1460	goto out;
1461	link_new = &lgr->lnk[lnk_idx];
1462
1463	rc = smcr_buf_map_lgr(lnk: link_new);
1464	if (rc)
1465	goto out_err;
1466
1467	rc = smc_llc_send_add_link(link,
1468	mac: link_new->smcibdev->mac[link_new->ibport-1],
1469	gid: link_new->gid, link_new, reqresp: SMC_LLC_REQ);
1470	if (rc)
1471	goto out_err;
1472	send_req_add_link_resp = false;
1473	/* receive ADD LINK response over the RoCE fabric */
1474	qentry = smc_llc_wait(lgr, lnk: link, SMC_LLC_WAIT_TIME, exp_msg: SMC_LLC_ADD_LINK);
1475	if (!qentry) {
1476	rc = -ETIMEDOUT;
1477	goto out_err;
1478	}
1479	add_llc = &qentry->msg.add_link;
1480	if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) {
1481	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
1482	rc = -ENOLINK;
1483	goto out_err;
1484	}
1485	if (lgr->type == SMC_LGR_SINGLE &&
1486	(!memcmp(p: add_llc->sender_gid, q: link->peer_gid, SMC_GID_SIZE) &&
1487	(lgr->smc_version == SMC_V2 ||
1488	!memcmp(p: add_llc->sender_mac, q: link->peer_mac, ETH_ALEN)))) {
1489	lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1490	}
1491	smc_llc_save_add_link_info(link: link_new, add_llc);
1492	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
1493
1494	rc = smc_ib_ready_link(lnk: link_new);
1495	if (rc)
1496	goto out_err;
1497	rc = smcr_buf_reg_lgr(lnk: link_new);
1498	if (rc)
1499	goto out_err;
1500	if (lgr->smc_version == SMC_V2) {
1501	smc_llc_save_add_link_rkeys(link, link_new);
1502	} else {
1503	rc = smc_llc_srv_rkey_exchange(link, link_new);
1504	if (rc)
1505	goto out_err;
1506	}
1507	rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t);
1508	if (rc)
1509	goto out_err;
1510	kfree(objp: ini);
1511	return 0;
1512	out_err:
1513	if (link_new) {
1514	link_new->state = SMC_LNK_INACTIVE;
1515	smcr_link_clear(lnk: link_new, log: false);
1516	}
1517	out:
1518	kfree(objp: ini);
1519	if (send_req_add_link_resp)
1520	smc_llc_send_req_add_link_response(qentry: req_qentry);
1521	return rc;
1522	}
1523
1524	static void smc_llc_process_srv_add_link(struct smc_link_group *lgr)
1525	{
1526	struct smc_link *link = lgr->llc_flow_lcl.qentry->link;
1527	struct smc_llc_qentry *qentry;
1528	int rc;
1529
1530	qentry = smc_llc_flow_qentry_clr(flow: &lgr->llc_flow_lcl);
1531
1532	down_write(sem: &lgr->llc_conf_mutex);
1533	rc = smc_llc_srv_add_link(link, req_qentry: qentry);
1534	if (!rc && lgr->type == SMC_LGR_SYMMETRIC) {
1535	/* delete any asymmetric link */
1536	smc_llc_delete_asym_link(lgr);
1537	}
1538	up_write(sem: &lgr->llc_conf_mutex);
1539	kfree(objp: qentry);
1540	}
1541
1542	/* enqueue a local add_link req to trigger a new add_link flow */
1543	void smc_llc_add_link_local(struct smc_link *link)
1544	{
1545	struct smc_llc_msg_add_link add_llc = {};
1546
1547	add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK;
1548	smc_llc_init_msg_hdr(hdr: &add_llc.hd, lgr: link->lgr, len: sizeof(add_llc));
1549	/* no dev and port needed */
1550	smc_llc_enqueue(link, llc: (union smc_llc_msg *)&add_llc);
1551	}
1552
1553	/* worker to process an add link message */
1554	static void smc_llc_add_link_work(struct work_struct *work)
1555	{
1556	struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1557	llc_add_link_work);
1558
1559	if (list_empty(head: &lgr->list)) {
1560	/* link group is terminating */
1561	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
1562	goto out;
1563	}
1564
1565	if (lgr->role == SMC_CLNT)
1566	smc_llc_process_cli_add_link(lgr);
1567	else
1568	smc_llc_process_srv_add_link(lgr);
1569	out:
1570	if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_REQ_ADD_LINK)
1571	smc_llc_flow_stop(lgr, flow: &lgr->llc_flow_lcl);
1572	}
1573
1574	/* enqueue a local del_link msg to trigger a new del_link flow,
1575	* called only for role SMC_SERV
1576	*/
1577	void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id)
1578	{
1579	struct smc_llc_msg_del_link del_llc = {};
1580
1581	del_llc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1582	smc_llc_init_msg_hdr(hdr: &del_llc.hd, lgr: link->lgr, len: sizeof(del_llc));
1583	del_llc.link_num = del_link_id;
1584	del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH);
1585	del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1586	smc_llc_enqueue(link, llc: (union smc_llc_msg *)&del_llc);
1587	}
1588
1589	static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)
1590	{
1591	struct smc_link *lnk_del = NULL, *lnk_asym, *lnk;
1592	struct smc_llc_msg_del_link *del_llc;
1593	struct smc_llc_qentry *qentry;
1594	int active_links;
1595	int lnk_idx;
1596
1597	qentry = smc_llc_flow_qentry_clr(flow: &lgr->llc_flow_lcl);
1598	lnk = qentry->link;
1599	del_llc = &qentry->msg.delete_link;
1600
1601	if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1602	smc_lgr_terminate_sched(lgr);
1603	goto out;
1604	}
1605	down_write(sem: &lgr->llc_conf_mutex);
1606	/* delete single link */
1607	for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) {
1608	if (lgr->lnk[lnk_idx].link_id != del_llc->link_num)
1609	continue;
1610	lnk_del = &lgr->lnk[lnk_idx];
1611	break;
1612	}
1613	del_llc->hd.flags |= SMC_LLC_FLAG_RESP;
1614	if (!lnk_del) {
1615	/* link was not found */
1616	del_llc->reason = htonl(SMC_LLC_DEL_NOLNK);
1617	smc_llc_send_message(link: lnk, llcbuf: &qentry->msg);
1618	goto out_unlock;
1619	}
1620	lnk_asym = smc_llc_find_asym_link(lgr);
1621
1622	del_llc->reason = 0;
1623	smc_llc_send_message(link: lnk, llcbuf: &qentry->msg); /* response */
1624
1625	if (smc_link_downing(&lnk_del->state))
1626	smc_switch_conns(lgr, from_lnk: lnk_del, is_dev_err: false);
1627	smcr_link_clear(lnk: lnk_del, log: true);
1628
1629	active_links = smc_llc_active_link_count(lgr);
1630	if (lnk_del == lnk_asym) {
1631	/* expected deletion of asym link, don't change lgr state */
1632	} else if (active_links == 1) {
1633	smcr_lgr_set_type(lgr, new_type: SMC_LGR_SINGLE);
1634	} else if (!active_links) {
1635	smcr_lgr_set_type(lgr, new_type: SMC_LGR_NONE);
1636	smc_lgr_terminate_sched(lgr);
1637	}
1638	out_unlock:
1639	up_write(sem: &lgr->llc_conf_mutex);
1640	out:
1641	kfree(objp: qentry);
1642	}
1643
1644	/* try to send a DELETE LINK ALL request on any active link,
1645	* waiting for send completion
1646	*/
1647	void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)
1648	{
1649	struct smc_llc_msg_del_link delllc = {};
1650	int i;
1651
1652	delllc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1653	smc_llc_init_msg_hdr(hdr: &delllc.hd, lgr, len: sizeof(delllc));
1654	if (ord)
1655	delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1656	delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
1657	delllc.reason = htonl(rsn);
1658
1659	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1660	if (!smc_link_sendable(lnk: &lgr->lnk[i]))
1661	continue;
1662	if (!smc_llc_send_message_wait(link: &lgr->lnk[i], llcbuf: &delllc))
1663	break;
1664	}
1665	}
1666
1667	static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)
1668	{
1669	struct smc_llc_msg_del_link *del_llc;
1670	struct smc_link *lnk, *lnk_del;
1671	struct smc_llc_qentry *qentry;
1672	int active_links;
1673	int i;
1674
1675	down_write(sem: &lgr->llc_conf_mutex);
1676	qentry = smc_llc_flow_qentry_clr(flow: &lgr->llc_flow_lcl);
1677	lnk = qentry->link;
1678	del_llc = &qentry->msg.delete_link;
1679
1680	if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1681	/* delete entire lgr */
1682	smc_llc_send_link_delete_all(lgr, ord: true, ntohl(
1683	qentry->msg.delete_link.reason));
1684	smc_lgr_terminate_sched(lgr);
1685	goto out;
1686	}
1687	/* delete single link */
1688	lnk_del = NULL;
1689	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1690	if (lgr->lnk[i].link_id == del_llc->link_num) {
1691	lnk_del = &lgr->lnk[i];
1692	break;
1693	}
1694	}
1695	if (!lnk_del)
1696	goto out; /* asymmetric link already deleted */
1697
1698	if (smc_link_downing(&lnk_del->state)) {
1699	if (smc_switch_conns(lgr, from_lnk: lnk_del, is_dev_err: false))
1700	smc_wr_tx_wait_no_pending_sends(link: lnk_del);
1701	}
1702	if (!list_empty(head: &lgr->list)) {
1703	/* qentry is either a request from peer (send it back to
1704	* initiate the DELETE_LINK processing), or a locally
1705	* enqueued DELETE_LINK request (forward it)
1706	*/
1707	if (!smc_llc_send_message(link: lnk, llcbuf: &qentry->msg)) {
1708	struct smc_llc_qentry *qentry2;
1709
1710	qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME,
1711	exp_msg: SMC_LLC_DELETE_LINK);
1712	if (qentry2)
1713	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
1714	}
1715	}
1716	smcr_link_clear(lnk: lnk_del, log: true);
1717
1718	active_links = smc_llc_active_link_count(lgr);
1719	if (active_links == 1) {
1720	smcr_lgr_set_type(lgr, new_type: SMC_LGR_SINGLE);
1721	} else if (!active_links) {
1722	smcr_lgr_set_type(lgr, new_type: SMC_LGR_NONE);
1723	smc_lgr_terminate_sched(lgr);
1724	}
1725
1726	if (lgr->type == SMC_LGR_SINGLE && !list_empty(head: &lgr->list)) {
1727	/* trigger setup of asymm alt link */
1728	smc_llc_add_link_local(link: lnk);
1729	}
1730	out:
1731	up_write(sem: &lgr->llc_conf_mutex);
1732	kfree(objp: qentry);
1733	}
1734
1735	static void smc_llc_delete_link_work(struct work_struct *work)
1736	{
1737	struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1738	llc_del_link_work);
1739
1740	if (list_empty(head: &lgr->list)) {
1741	/* link group is terminating */
1742	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
1743	goto out;
1744	}
1745
1746	if (lgr->role == SMC_CLNT)
1747	smc_llc_process_cli_delete_link(lgr);
1748	else
1749	smc_llc_process_srv_delete_link(lgr);
1750	out:
1751	smc_llc_flow_stop(lgr, flow: &lgr->llc_flow_lcl);
1752	}
1753
1754	/* process a confirm_rkey request from peer, remote flow */
1755	static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)
1756	{
1757	struct smc_llc_msg_confirm_rkey *llc;
1758	struct smc_llc_qentry *qentry;
1759	struct smc_link *link;
1760	int num_entries;
1761	int rk_idx;
1762	int i;
1763
1764	qentry = lgr->llc_flow_rmt.qentry;
1765	llc = &qentry->msg.confirm_rkey;
1766	link = qentry->link;
1767
1768	num_entries = llc->rtoken[0].num_rkeys;
1769	if (num_entries > SMC_LLC_RKEYS_PER_MSG)
1770	goto out_err;
1771	/* first rkey entry is for receiving link */
1772	rk_idx = smc_rtoken_add(lnk: link,
1773	nw_vaddr: llc->rtoken[0].rmb_vaddr,
1774	nw_rkey: llc->rtoken[0].rmb_key);
1775	if (rk_idx < 0)
1776	goto out_err;
1777
1778	for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++)
1779	smc_rtoken_set2(lgr, rtok_idx: rk_idx, link_id: llc->rtoken[i].link_id,
1780	nw_vaddr: llc->rtoken[i].rmb_vaddr,
1781	nw_rkey: llc->rtoken[i].rmb_key);
1782	/* max links is 3 so there is no need to support conf_rkey_cont msgs */
1783	goto out;
1784	out_err:
1785	llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1786	llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY;
1787	out:
1788	llc->hd.flags |= SMC_LLC_FLAG_RESP;
1789	smc_llc_init_msg_hdr(hdr: &llc->hd, lgr: link->lgr, len: sizeof(*llc));
1790	smc_llc_send_message(link, llcbuf: &qentry->msg);
1791	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_rmt);
1792	}
1793
1794	/* process a delete_rkey request from peer, remote flow */
1795	static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)
1796	{
1797	struct smc_llc_msg_delete_rkey *llc;
1798	struct smc_llc_qentry *qentry;
1799	struct smc_link *link;
1800	u8 err_mask = 0;
1801	int i, max;
1802
1803	qentry = lgr->llc_flow_rmt.qentry;
1804	llc = &qentry->msg.delete_rkey;
1805	link = qentry->link;
1806
1807	if (lgr->smc_version == SMC_V2) {
1808	struct smc_llc_msg_delete_rkey_v2 *llcv2;
1809
1810	memcpy(lgr->wr_rx_buf_v2, llc, sizeof(*llc));
1811	llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)lgr->wr_rx_buf_v2;
1812	llcv2->num_inval_rkeys = 0;
1813
1814	max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
1815	for (i = 0; i < max; i++) {
1816	if (smc_rtoken_delete(lnk: link, nw_rkey: llcv2->rkey[i]))
1817	llcv2->num_inval_rkeys++;
1818	}
1819	memset(&llc->rkey[0], 0, sizeof(llc->rkey));
1820	memset(&llc->reserved2, 0, sizeof(llc->reserved2));
1821	smc_llc_init_msg_hdr(hdr: &llc->hd, lgr: link->lgr, len: sizeof(*llc));
1822	if (llcv2->num_inval_rkeys) {
1823	llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1824	llc->err_mask = llcv2->num_inval_rkeys;
1825	}
1826	goto finish;
1827	}
1828
1829	max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
1830	for (i = 0; i < max; i++) {
1831	if (smc_rtoken_delete(lnk: link, nw_rkey: llc->rkey[i]))
1832	err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
1833	}
1834	if (err_mask) {
1835	llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1836	llc->err_mask = err_mask;
1837	}
1838	finish:
1839	llc->hd.flags |= SMC_LLC_FLAG_RESP;
1840	smc_llc_send_message(link, llcbuf: &qentry->msg);
1841	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_rmt);
1842	}
1843
1844	static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type)
1845	{
1846	pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu LLC protocol violation: "
1847	"llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id,
1848	lgr->net->net_cookie, type);
1849	smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL);
1850	smc_lgr_terminate_sched(lgr);
1851	}
1852
1853	/* flush the llc event queue */
1854	static void smc_llc_event_flush(struct smc_link_group *lgr)
1855	{
1856	struct smc_llc_qentry *qentry, *q;
1857
1858	spin_lock_bh(lock: &lgr->llc_event_q_lock);
1859	list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
1860	list_del_init(entry: &qentry->list);
1861	kfree(objp: qentry);
1862	}
1863	spin_unlock_bh(lock: &lgr->llc_event_q_lock);
1864	}
1865
1866	static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
1867	{
1868	union smc_llc_msg *llc = &qentry->msg;
1869	struct smc_link *link = qentry->link;
1870	struct smc_link_group *lgr = link->lgr;
1871
1872	if (!smc_link_usable(lnk: link))
1873	goto out;
1874
1875	switch (llc->raw.hdr.common.llc_type) {
1876	case SMC_LLC_TEST_LINK:
1877	llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP;
1878	smc_llc_send_message(link, llcbuf: llc);
1879	break;
1880	case SMC_LLC_ADD_LINK:
1881	if (list_empty(head: &lgr->list))
1882	goto out; /* lgr is terminating */
1883	if (lgr->role == SMC_CLNT) {
1884	if (smc_llc_is_local_add_link(llc)) {
1885	if (lgr->llc_flow_lcl.type ==
1886	SMC_LLC_FLOW_ADD_LINK)
1887	break; /* add_link in progress */
1888	if (smc_llc_flow_start(flow: &lgr->llc_flow_lcl,
1889	qentry)) {
1890	schedule_work(work: &lgr->llc_add_link_work);
1891	}
1892	return;
1893	}
1894	if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1895	!lgr->llc_flow_lcl.qentry) {
1896	/* a flow is waiting for this message */
1897	smc_llc_flow_qentry_set(flow: &lgr->llc_flow_lcl,
1898	qentry);
1899	wake_up(&lgr->llc_msg_waiter);
1900	return;
1901	}
1902	if (lgr->llc_flow_lcl.type ==
1903	SMC_LLC_FLOW_REQ_ADD_LINK) {
1904	/* server started add_link processing */
1905	lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1906	smc_llc_flow_qentry_set(flow: &lgr->llc_flow_lcl,
1907	qentry);
1908	schedule_work(work: &lgr->llc_add_link_work);
1909	return;
1910	}
1911	if (smc_llc_flow_start(flow: &lgr->llc_flow_lcl, qentry)) {
1912	schedule_work(work: &lgr->llc_add_link_work);
1913	}
1914	} else if (smc_llc_flow_start(flow: &lgr->llc_flow_lcl, qentry)) {
1915	/* as smc server, handle client suggestion */
1916	schedule_work(work: &lgr->llc_add_link_work);
1917	}
1918	return;
1919	case SMC_LLC_CONFIRM_LINK:
1920	case SMC_LLC_ADD_LINK_CONT:
1921	if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
1922	/* a flow is waiting for this message */
1923	smc_llc_flow_qentry_set(flow: &lgr->llc_flow_lcl, qentry);
1924	wake_up(&lgr->llc_msg_waiter);
1925	return;
1926	}
1927	break;
1928	case SMC_LLC_DELETE_LINK:
1929	if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1930	!lgr->llc_flow_lcl.qentry) {
1931	/* DEL LINK REQ during ADD LINK SEQ */
1932	smc_llc_flow_qentry_set(flow: &lgr->llc_flow_lcl, qentry);
1933	wake_up(&lgr->llc_msg_waiter);
1934	} else if (smc_llc_flow_start(flow: &lgr->llc_flow_lcl, qentry)) {
1935	schedule_work(work: &lgr->llc_del_link_work);
1936	}
1937	return;
1938	case SMC_LLC_CONFIRM_RKEY:
1939	/* new request from remote, assign to remote flow */
1940	if (smc_llc_flow_start(flow: &lgr->llc_flow_rmt, qentry)) {
1941	/* process here, does not wait for more llc msgs */
1942	smc_llc_rmt_conf_rkey(lgr);
1943	smc_llc_flow_stop(lgr, flow: &lgr->llc_flow_rmt);
1944	}
1945	return;
1946	case SMC_LLC_CONFIRM_RKEY_CONT:
1947	/* not used because max links is 3, and 3 rkeys fit into
1948	* one CONFIRM_RKEY message
1949	*/
1950	break;
1951	case SMC_LLC_DELETE_RKEY:
1952	/* new request from remote, assign to remote flow */
1953	if (smc_llc_flow_start(flow: &lgr->llc_flow_rmt, qentry)) {
1954	/* process here, does not wait for more llc msgs */
1955	smc_llc_rmt_delete_rkey(lgr);
1956	smc_llc_flow_stop(lgr, flow: &lgr->llc_flow_rmt);
1957	}
1958	return;
1959	case SMC_LLC_REQ_ADD_LINK:
1960	/* handle response here, smc_llc_flow_stop() cannot be called
1961	* in tasklet context
1962	*/
1963	if (lgr->role == SMC_CLNT &&
1964	lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK &&
1965	(llc->raw.hdr.flags & SMC_LLC_FLAG_RESP)) {
1966	smc_llc_flow_stop(lgr: link->lgr, flow: &lgr->llc_flow_lcl);
1967	} else if (lgr->role == SMC_SERV) {
1968	if (smc_llc_flow_start(flow: &lgr->llc_flow_lcl, qentry)) {
1969	/* as smc server, handle client suggestion */
1970	lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1971	schedule_work(work: &lgr->llc_add_link_work);
1972	}
1973	return;
1974	}
1975	break;
1976	default:
1977	smc_llc_protocol_violation(lgr, type: llc->raw.hdr.common.type);
1978	break;
1979	}
1980	out:
1981	kfree(objp: qentry);
1982	}
1983
1984	/* worker to process llc messages on the event queue */
1985	static void smc_llc_event_work(struct work_struct *work)
1986	{
1987	struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1988	llc_event_work);
1989	struct smc_llc_qentry *qentry;
1990
1991	if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
1992	qentry = lgr->delayed_event;
1993	lgr->delayed_event = NULL;
1994	if (smc_link_usable(lnk: qentry->link))
1995	smc_llc_event_handler(qentry);
1996	else
1997	kfree(objp: qentry);
1998	}
1999
2000	again:
2001	spin_lock_bh(lock: &lgr->llc_event_q_lock);
2002	if (!list_empty(head: &lgr->llc_event_q)) {
2003	qentry = list_first_entry(&lgr->llc_event_q,
2004	struct smc_llc_qentry, list);
2005	list_del_init(entry: &qentry->list);
2006	spin_unlock_bh(lock: &lgr->llc_event_q_lock);
2007	smc_llc_event_handler(qentry);
2008	goto again;
2009	}
2010	spin_unlock_bh(lock: &lgr->llc_event_q_lock);
2011	}
2012
2013	/* process llc responses in tasklet context */
2014	static void smc_llc_rx_response(struct smc_link *link,
2015	struct smc_llc_qentry *qentry)
2016	{
2017	enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type;
2018	struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl;
2019	u8 llc_type = qentry->msg.raw.hdr.common.llc_type;
2020
2021	switch (llc_type) {
2022	case SMC_LLC_TEST_LINK:
2023	if (smc_link_active(lnk: link))
2024	complete(&link->llc_testlink_resp);
2025	break;
2026	case SMC_LLC_ADD_LINK:
2027	case SMC_LLC_ADD_LINK_CONT:
2028	case SMC_LLC_CONFIRM_LINK:
2029	if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry)
2030	break; /* drop out-of-flow response */
2031	goto assign;
2032	case SMC_LLC_DELETE_LINK:
2033	if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry)
2034	break; /* drop out-of-flow response */
2035	goto assign;
2036	case SMC_LLC_CONFIRM_RKEY:
2037	case SMC_LLC_DELETE_RKEY:
2038	if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry)
2039	break; /* drop out-of-flow response */
2040	goto assign;
2041	case SMC_LLC_CONFIRM_RKEY_CONT:
2042	/* not used because max links is 3 */
2043	break;
2044	default:
2045	smc_llc_protocol_violation(lgr: link->lgr,
2046	type: qentry->msg.raw.hdr.common.type);
2047	break;
2048	}
2049	kfree(objp: qentry);
2050	return;
2051	assign:
2052	/* assign responses to the local flow, we requested them */
2053	smc_llc_flow_qentry_set(flow: &link->lgr->llc_flow_lcl, qentry);
2054	wake_up(&link->lgr->llc_msg_waiter);
2055	}
2056
2057	static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc)
2058	{
2059	struct smc_link_group *lgr = link->lgr;
2060	struct smc_llc_qentry *qentry;
2061	unsigned long flags;
2062
2063	qentry = kmalloc(size: sizeof(*qentry), GFP_ATOMIC);
2064	if (!qentry)
2065	return;
2066	qentry->link = link;
2067	INIT_LIST_HEAD(list: &qentry->list);
2068	memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
2069
2070	/* process responses immediately */
2071	if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) &&
2072	llc->raw.hdr.common.llc_type != SMC_LLC_REQ_ADD_LINK) {
2073	smc_llc_rx_response(link, qentry);
2074	return;
2075	}
2076
2077	/* add requests to event queue */
2078	spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
2079	list_add_tail(new: &qentry->list, head: &lgr->llc_event_q);
2080	spin_unlock_irqrestore(lock: &lgr->llc_event_q_lock, flags);
2081	queue_work(wq: system_highpri_wq, work: &lgr->llc_event_work);
2082	}
2083
2084	/* copy received msg and add it to the event queue */
2085	static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
2086	{
2087	struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
2088	union smc_llc_msg *llc = buf;
2089
2090	if (wc->byte_len < sizeof(*llc))
2091	return; /* short message */
2092	if (!llc->raw.hdr.common.llc_version) {
2093	if (llc->raw.hdr.length != sizeof(*llc))
2094	return; /* invalid message */
2095	} else {
2096	if (llc->raw.hdr.length_v2 < sizeof(*llc))
2097	return; /* invalid message */
2098	}
2099
2100	smc_llc_enqueue(link, llc);
2101	}
2102
2103	/***************************** worker, utils *********************************/
2104
2105	static void smc_llc_testlink_work(struct work_struct *work)
2106	{
2107	struct smc_link *link = container_of(to_delayed_work(work),
2108	struct smc_link, llc_testlink_wrk);
2109	unsigned long next_interval;
2110	unsigned long expire_time;
2111	u8 user_data[16] = { 0 };
2112	int rc;
2113
2114	if (!smc_link_active(lnk: link))
2115	return; /* don't reschedule worker */
2116	expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
2117	if (time_is_after_jiffies(expire_time)) {
2118	next_interval = expire_time - jiffies;
2119	goto out;
2120	}
2121	reinit_completion(x: &link->llc_testlink_resp);
2122	smc_llc_send_test_link(link, user_data);
2123	/* receive TEST LINK response over RoCE fabric */
2124	rc = wait_for_completion_interruptible_timeout(x: &link->llc_testlink_resp,
2125	SMC_LLC_WAIT_TIME);
2126	if (!smc_link_active(lnk: link))
2127	return; /* link state changed */
2128	if (rc <= 0) {
2129	smcr_link_down_cond_sched(lnk: link);
2130	return;
2131	}
2132	next_interval = link->llc_testlink_time;
2133	out:
2134	schedule_delayed_work(dwork: &link->llc_testlink_wrk, delay: next_interval);
2135	}
2136
2137	void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
2138	{
2139	struct net *net = sock_net(sk: smc->clcsock->sk);
2140
2141	INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
2142	INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work);
2143	INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work);
2144	INIT_LIST_HEAD(list: &lgr->llc_event_q);
2145	spin_lock_init(&lgr->llc_event_q_lock);
2146	spin_lock_init(&lgr->llc_flow_lock);
2147	init_waitqueue_head(&lgr->llc_flow_waiter);
2148	init_waitqueue_head(&lgr->llc_msg_waiter);
2149	init_rwsem(&lgr->llc_conf_mutex);
2150	lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time);
2151	}
2152
2153	/* called after lgr was removed from lgr_list */
2154	void smc_llc_lgr_clear(struct smc_link_group *lgr)
2155	{
2156	smc_llc_event_flush(lgr);
2157	wake_up_all(&lgr->llc_flow_waiter);
2158	wake_up_all(&lgr->llc_msg_waiter);
2159	cancel_work_sync(work: &lgr->llc_event_work);
2160	cancel_work_sync(work: &lgr->llc_add_link_work);
2161	cancel_work_sync(work: &lgr->llc_del_link_work);
2162	if (lgr->delayed_event) {
2163	kfree(objp: lgr->delayed_event);
2164	lgr->delayed_event = NULL;
2165	}
2166	}
2167
2168	int smc_llc_link_init(struct smc_link *link)
2169	{
2170	init_completion(x: &link->llc_testlink_resp);
2171	INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
2172	return 0;
2173	}
2174
2175	void smc_llc_link_active(struct smc_link *link)
2176	{
2177	pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link added: id %*phN, "
2178	"peerid %*phN, ibdev %s, ibport %d\n",
2179	SMC_LGR_ID_SIZE, &link->lgr->id,
2180	link->lgr->net->net_cookie,
2181	SMC_LGR_ID_SIZE, &link->link_uid,
2182	SMC_LGR_ID_SIZE, &link->peer_link_uid,
2183	link->smcibdev->ibdev->name, link->ibport);
2184	link->state = SMC_LNK_ACTIVE;
2185	if (link->lgr->llc_testlink_time) {
2186	link->llc_testlink_time = link->lgr->llc_testlink_time;
2187	schedule_delayed_work(dwork: &link->llc_testlink_wrk,
2188	delay: link->llc_testlink_time);
2189	}
2190	}
2191
2192	/* called in worker context */
2193	void smc_llc_link_clear(struct smc_link *link, bool log)
2194	{
2195	if (log)
2196	pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link removed: id %*phN"
2197	", peerid %*phN, ibdev %s, ibport %d\n",
2198	SMC_LGR_ID_SIZE, &link->lgr->id,
2199	link->lgr->net->net_cookie,
2200	SMC_LGR_ID_SIZE, &link->link_uid,
2201	SMC_LGR_ID_SIZE, &link->peer_link_uid,
2202	link->smcibdev->ibdev->name, link->ibport);
2203	complete(&link->llc_testlink_resp);
2204	cancel_delayed_work_sync(dwork: &link->llc_testlink_wrk);
2205	}
2206
2207	/* register a new rtoken at the remote peer (for all links) */
2208	int smc_llc_do_confirm_rkey(struct smc_link *send_link,
2209	struct smc_buf_desc *rmb_desc)
2210	{
2211	struct smc_link_group *lgr = send_link->lgr;
2212	struct smc_llc_qentry *qentry = NULL;
2213	int rc = 0;
2214
2215	rc = smc_llc_send_confirm_rkey(send_link, rmb_desc);
2216	if (rc)
2217	goto out;
2218	/* receive CONFIRM RKEY response from server over RoCE fabric */
2219	qentry = smc_llc_wait(lgr, lnk: send_link, SMC_LLC_WAIT_TIME,
2220	exp_msg: SMC_LLC_CONFIRM_RKEY);
2221	if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2222	rc = -EFAULT;
2223	out:
2224	if (qentry)
2225	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
2226	return rc;
2227	}
2228
2229	/* unregister an rtoken at the remote peer */
2230	int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
2231	struct smc_buf_desc *rmb_desc)
2232	{
2233	struct smc_llc_qentry *qentry = NULL;
2234	struct smc_link *send_link;
2235	int rc = 0;
2236
2237	send_link = smc_llc_usable_link(lgr);
2238	if (!send_link)
2239	return -ENOLINK;
2240
2241	/* protected by llc_flow control */
2242	rc = smc_llc_send_delete_rkey(link: send_link, rmb_desc);
2243	if (rc)
2244	goto out;
2245	/* receive DELETE RKEY response from server over RoCE fabric */
2246	qentry = smc_llc_wait(lgr, lnk: send_link, SMC_LLC_WAIT_TIME,
2247	exp_msg: SMC_LLC_DELETE_RKEY);
2248	if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2249	rc = -EFAULT;
2250	out:
2251	if (qentry)
2252	smc_llc_flow_qentry_del(flow: &lgr->llc_flow_lcl);
2253	return rc;
2254	}
2255
2256	void smc_llc_link_set_uid(struct smc_link *link)
2257	{
2258	__be32 link_uid;
2259
2260	link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id);
2261	memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE);
2262	}
2263
2264	/* save peers link user id, used for debug purposes */
2265	void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry)
2266	{
2267	memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid,
2268	SMC_LGR_ID_SIZE);
2269	}
2270
2271	/* evaluate confirm link request or response */
2272	int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
2273	enum smc_llc_reqresp type)
2274	{
2275	if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */
2276	qentry->link->link_id = qentry->msg.confirm_link.link_num;
2277	smc_llc_link_set_uid(link: qentry->link);
2278	}
2279	if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
2280	return -ENOTSUPP;
2281	return 0;
2282	}
2283
2284	/***************************** init, exit, misc ******************************/
2285
2286	static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
2287	{
2288	.handler = smc_llc_rx_handler,
2289	.type = SMC_LLC_CONFIRM_LINK
2290	},
2291	{
2292	.handler = smc_llc_rx_handler,
2293	.type = SMC_LLC_TEST_LINK
2294	},
2295	{
2296	.handler = smc_llc_rx_handler,
2297	.type = SMC_LLC_ADD_LINK
2298	},
2299	{
2300	.handler = smc_llc_rx_handler,
2301	.type = SMC_LLC_ADD_LINK_CONT
2302	},
2303	{
2304	.handler = smc_llc_rx_handler,
2305	.type = SMC_LLC_DELETE_LINK
2306	},
2307	{
2308	.handler = smc_llc_rx_handler,
2309	.type = SMC_LLC_CONFIRM_RKEY
2310	},
2311	{
2312	.handler = smc_llc_rx_handler,
2313	.type = SMC_LLC_CONFIRM_RKEY_CONT
2314	},
2315	{
2316	.handler = smc_llc_rx_handler,
2317	.type = SMC_LLC_DELETE_RKEY
2318	},
2319	/* V2 types */
2320	{
2321	.handler = smc_llc_rx_handler,
2322	.type = SMC_LLC_CONFIRM_LINK_V2
2323	},
2324	{
2325	.handler = smc_llc_rx_handler,
2326	.type = SMC_LLC_TEST_LINK_V2
2327	},
2328	{
2329	.handler = smc_llc_rx_handler,
2330	.type = SMC_LLC_ADD_LINK_V2
2331	},
2332	{
2333	.handler = smc_llc_rx_handler,
2334	.type = SMC_LLC_DELETE_LINK_V2
2335	},
2336	{
2337	.handler = smc_llc_rx_handler,
2338	.type = SMC_LLC_REQ_ADD_LINK_V2
2339	},
2340	{
2341	.handler = smc_llc_rx_handler,
2342	.type = SMC_LLC_CONFIRM_RKEY_V2
2343	},
2344	{
2345	.handler = smc_llc_rx_handler,
2346	.type = SMC_LLC_DELETE_RKEY_V2
2347	},
2348	{
2349	.handler = NULL,
2350	}
2351	};
2352
2353	int __init smc_llc_init(void)
2354	{
2355	struct smc_wr_rx_handler *handler;
2356	int rc = 0;
2357
2358	for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
2359	INIT_HLIST_NODE(h: &handler->list);
2360	rc = smc_wr_rx_register_handler(handler);
2361	if (rc)
2362	break;
2363	}
2364	return rc;
2365	}
2366