1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2	/*
3	* Copyright (c) 2004-2007 Intel Corporation. All rights reserved.
4	* Copyright (c) 2004 Topspin Corporation. All rights reserved.
5	* Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved.
6	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
7	* Copyright (c) 2019, Mellanox Technologies inc. All rights reserved.
8	*/
9
10	#include <linux/completion.h>
11	#include <linux/dma-mapping.h>
12	#include <linux/device.h>
13	#include <linux/module.h>
14	#include <linux/err.h>
15	#include <linux/idr.h>
16	#include <linux/interrupt.h>
17	#include <linux/random.h>
18	#include <linux/rbtree.h>
19	#include <linux/spinlock.h>
20	#include <linux/slab.h>
21	#include <linux/sysfs.h>
22	#include <linux/workqueue.h>
23	#include <linux/kdev_t.h>
24	#include <linux/etherdevice.h>
25
26	#include <rdma/ib_cache.h>
27	#include <rdma/ib_cm.h>
28	#include <rdma/ib_sysfs.h>
29	#include "cm_msgs.h"
30	#include "core_priv.h"
31	#include "cm_trace.h"
32
33	MODULE_AUTHOR("Sean Hefty");
34	MODULE_DESCRIPTION("InfiniBand CM");
35	MODULE_LICENSE("Dual BSD/GPL");
36
37	#define CM_DESTROY_ID_WAIT_TIMEOUT 10000 /* msecs */
38	static const char * const ibcm_rej_reason_strs[] = {
39	[IB_CM_REJ_NO_QP] = "no QP",
40	[IB_CM_REJ_NO_EEC] = "no EEC",
41	[IB_CM_REJ_NO_RESOURCES] = "no resources",
42	[IB_CM_REJ_TIMEOUT] = "timeout",
43	[IB_CM_REJ_UNSUPPORTED] = "unsupported",
44	[IB_CM_REJ_INVALID_COMM_ID] = "invalid comm ID",
45	[IB_CM_REJ_INVALID_COMM_INSTANCE] = "invalid comm instance",
46	[IB_CM_REJ_INVALID_SERVICE_ID] = "invalid service ID",
47	[IB_CM_REJ_INVALID_TRANSPORT_TYPE] = "invalid transport type",
48	[IB_CM_REJ_STALE_CONN] = "stale conn",
49	[IB_CM_REJ_RDC_NOT_EXIST] = "RDC not exist",
50	[IB_CM_REJ_INVALID_GID] = "invalid GID",
51	[IB_CM_REJ_INVALID_LID] = "invalid LID",
52	[IB_CM_REJ_INVALID_SL] = "invalid SL",
53	[IB_CM_REJ_INVALID_TRAFFIC_CLASS] = "invalid traffic class",
54	[IB_CM_REJ_INVALID_HOP_LIMIT] = "invalid hop limit",
55	[IB_CM_REJ_INVALID_PACKET_RATE] = "invalid packet rate",
56	[IB_CM_REJ_INVALID_ALT_GID] = "invalid alt GID",
57	[IB_CM_REJ_INVALID_ALT_LID] = "invalid alt LID",
58	[IB_CM_REJ_INVALID_ALT_SL] = "invalid alt SL",
59	[IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS] = "invalid alt traffic class",
60	[IB_CM_REJ_INVALID_ALT_HOP_LIMIT] = "invalid alt hop limit",
61	[IB_CM_REJ_INVALID_ALT_PACKET_RATE] = "invalid alt packet rate",
62	[IB_CM_REJ_PORT_CM_REDIRECT] = "port CM redirect",
63	[IB_CM_REJ_PORT_REDIRECT] = "port redirect",
64	[IB_CM_REJ_INVALID_MTU] = "invalid MTU",
65	[IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources",
66	[IB_CM_REJ_CONSUMER_DEFINED] = "consumer defined",
67	[IB_CM_REJ_INVALID_RNR_RETRY] = "invalid RNR retry",
68	[IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID] = "duplicate local comm ID",
69	[IB_CM_REJ_INVALID_CLASS_VERSION] = "invalid class version",
70	[IB_CM_REJ_INVALID_FLOW_LABEL] = "invalid flow label",
71	[IB_CM_REJ_INVALID_ALT_FLOW_LABEL] = "invalid alt flow label",
72	[IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED] =
73	"vendor option is not supported",
74	};
75
76	const char *__attribute_const__ ibcm_reject_msg(int reason)
77	{
78	size_t index = reason;
79
80	if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
81	ibcm_rej_reason_strs[index])
82	return ibcm_rej_reason_strs[index];
83	else
84	return "unrecognized reason";
85	}
86	EXPORT_SYMBOL(ibcm_reject_msg);
87
88	struct cm_id_private;
89	struct cm_work;
90	static int cm_add_one(struct ib_device *device);
91	static void cm_remove_one(struct ib_device *device, void *client_data);
92	static void cm_process_work(struct cm_id_private *cm_id_priv,
93	struct cm_work *work);
94	static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
95	struct ib_cm_sidr_rep_param *param);
96	static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
97	const void *private_data, u8 private_data_len);
98	static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
99	void *private_data, u8 private_data_len);
100	static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
101	enum ib_cm_rej_reason reason, void *ari,
102	u8 ari_length, const void *private_data,
103	u8 private_data_len);
104
105	static struct ib_client cm_client = {
106	.name = "cm",
107	.add = cm_add_one,
108	.remove = cm_remove_one
109	};
110
111	static struct ib_cm {
112	spinlock_t lock;
113	struct list_head device_list;
114	rwlock_t device_lock;
115	struct rb_root listen_service_table;
116	u64 listen_service_id;
117	/* struct rb_root peer_service_table; todo: fix peer to peer */
118	struct rb_root remote_qp_table;
119	struct rb_root remote_id_table;
120	struct rb_root remote_sidr_table;
121	struct xarray local_id_table;
122	u32 local_id_next;
123	__be32 random_id_operand;
124	struct list_head timewait_list;
125	struct workqueue_struct *wq;
126	} cm;
127
128	/* Counter indexes ordered by attribute ID */
129	enum {
130	CM_REQ_COUNTER,
131	CM_MRA_COUNTER,
132	CM_REJ_COUNTER,
133	CM_REP_COUNTER,
134	CM_RTU_COUNTER,
135	CM_DREQ_COUNTER,
136	CM_DREP_COUNTER,
137	CM_SIDR_REQ_COUNTER,
138	CM_SIDR_REP_COUNTER,
139	CM_LAP_COUNTER,
140	CM_APR_COUNTER,
141	CM_ATTR_COUNT,
142	CM_ATTR_ID_OFFSET = 0x0010,
143	};
144
145	enum {
146	CM_XMIT,
147	CM_XMIT_RETRIES,
148	CM_RECV,
149	CM_RECV_DUPLICATES,
150	CM_COUNTER_GROUPS
151	};
152
153	struct cm_counter_attribute {
154	struct ib_port_attribute attr;
155	unsigned short group;
156	unsigned short index;
157	};
158
159	struct cm_port {
160	struct cm_device *cm_dev;
161	struct ib_mad_agent *mad_agent;
162	u32 port_num;
163	atomic_long_t counters[CM_COUNTER_GROUPS][CM_ATTR_COUNT];
164	};
165
166	struct cm_device {
167	struct kref kref;
168	struct list_head list;
169	spinlock_t mad_agent_lock;
170	struct ib_device *ib_device;
171	u8 ack_delay;
172	int going_down;
173	struct cm_port *port[];
174	};
175
176	struct cm_av {
177	struct cm_port *port;
178	struct rdma_ah_attr ah_attr;
179	u16 dlid_datapath;
180	u16 pkey_index;
181	u8 timeout;
182	};
183
184	struct cm_work {
185	struct delayed_work work;
186	struct list_head list;
187	struct cm_port *port;
188	struct ib_mad_recv_wc *mad_recv_wc; /* Received MADs */
189	__be32 local_id; /* Established / timewait */
190	__be32 remote_id;
191	struct ib_cm_event cm_event;
192	struct sa_path_rec path[];
193	};
194
195	struct cm_timewait_info {
196	struct cm_work work;
197	struct list_head list;
198	struct rb_node remote_qp_node;
199	struct rb_node remote_id_node;
200	__be64 remote_ca_guid;
201	__be32 remote_qpn;
202	u8 inserted_remote_qp;
203	u8 inserted_remote_id;
204	};
205
206	struct cm_id_private {
207	struct ib_cm_id id;
208
209	struct rb_node service_node;
210	struct rb_node sidr_id_node;
211	u32 sidr_slid;
212	spinlock_t lock; /* Do not acquire inside cm.lock */
213	struct completion comp;
214	refcount_t refcount;
215	/* Number of clients sharing this ib_cm_id. Only valid for listeners.
216	* Protected by the cm.lock spinlock.
217	*/
218	int listen_sharecount;
219	struct rcu_head rcu;
220
221	struct ib_mad_send_buf *msg;
222	struct cm_timewait_info *timewait_info;
223	/* todo: use alternate port on send failure */
224	struct cm_av av;
225	struct cm_av alt_av;
226
227	void *private_data;
228	__be64 tid;
229	__be32 local_qpn;
230	__be32 remote_qpn;
231	enum ib_qp_type qp_type;
232	__be32 sq_psn;
233	__be32 rq_psn;
234	int timeout_ms;
235	enum ib_mtu path_mtu;
236	__be16 pkey;
237	u8 private_data_len;
238	u8 max_cm_retries;
239	u8 responder_resources;
240	u8 initiator_depth;
241	u8 retry_count;
242	u8 rnr_retry_count;
243	u8 service_timeout;
244	u8 target_ack_delay;
245
246	struct list_head work_list;
247	atomic_t work_count;
248
249	struct rdma_ucm_ece ece;
250	};
251
252	static void cm_dev_release(struct kref *kref)
253	{
254	struct cm_device *cm_dev = container_of(kref, struct cm_device, kref);
255	u32 i;
256
257	rdma_for_each_port(cm_dev->ib_device, i)
258	kfree(objp: cm_dev->port[i - 1]);
259
260	kfree(objp: cm_dev);
261	}
262
263	static void cm_device_put(struct cm_device *cm_dev)
264	{
265	kref_put(kref: &cm_dev->kref, release: cm_dev_release);
266	}
267
268	static void cm_work_handler(struct work_struct *work);
269
270	static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
271	{
272	if (refcount_dec_and_test(r: &cm_id_priv->refcount))
273	complete(&cm_id_priv->comp);
274	}
275
276	static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv)
277	{
278	struct ib_mad_agent *mad_agent;
279	struct ib_mad_send_buf *m;
280	struct ib_ah *ah;
281
282	lockdep_assert_held(&cm_id_priv->lock);
283
284	if (!cm_id_priv->av.port)
285	return ERR_PTR(error: -EINVAL);
286
287	spin_lock(lock: &cm_id_priv->av.port->cm_dev->mad_agent_lock);
288	mad_agent = cm_id_priv->av.port->mad_agent;
289	if (!mad_agent) {
290	m = ERR_PTR(error: -EINVAL);
291	goto out;
292	}
293
294	ah = rdma_create_ah(pd: mad_agent->qp->pd, ah_attr: &cm_id_priv->av.ah_attr, flags: 0);
295	if (IS_ERR(ptr: ah)) {
296	m = ERR_CAST(ptr: ah);
297	goto out;
298	}
299
300	m = ib_create_send_mad(mad_agent, remote_qpn: cm_id_priv->id.remote_cm_qpn,
301	pkey_index: cm_id_priv->av.pkey_index,
302	rmpp_active: 0, hdr_len: IB_MGMT_MAD_HDR, data_len: IB_MGMT_MAD_DATA,
303	GFP_ATOMIC,
304	IB_MGMT_BASE_VERSION);
305	if (IS_ERR(ptr: m)) {
306	rdma_destroy_ah(ah, flags: 0);
307	goto out;
308	}
309
310	/* Timeout set by caller if response is expected. */
311	m->ah = ah;
312	m->retries = cm_id_priv->max_cm_retries;
313
314	refcount_inc(r: &cm_id_priv->refcount);
315	m->context[0] = cm_id_priv;
316
317	out:
318	spin_unlock(lock: &cm_id_priv->av.port->cm_dev->mad_agent_lock);
319	return m;
320	}
321
322	static void cm_free_msg(struct ib_mad_send_buf *msg)
323	{
324	struct cm_id_private *cm_id_priv = msg->context[0];
325
326	if (msg->ah)
327	rdma_destroy_ah(ah: msg->ah, flags: 0);
328	cm_deref_id(cm_id_priv);
329	ib_free_send_mad(send_buf: msg);
330	}
331
332	static struct ib_mad_send_buf *
333	cm_alloc_priv_msg(struct cm_id_private *cm_id_priv)
334	{
335	struct ib_mad_send_buf *msg;
336
337	lockdep_assert_held(&cm_id_priv->lock);
338
339	msg = cm_alloc_msg(cm_id_priv);
340	if (IS_ERR(ptr: msg))
341	return msg;
342	cm_id_priv->msg = msg;
343	return msg;
344	}
345
346	static void cm_free_priv_msg(struct ib_mad_send_buf *msg)
347	{
348	struct cm_id_private *cm_id_priv = msg->context[0];
349
350	lockdep_assert_held(&cm_id_priv->lock);
351
352	if (!WARN_ON(cm_id_priv->msg != msg))
353	cm_id_priv->msg = NULL;
354
355	if (msg->ah)
356	rdma_destroy_ah(ah: msg->ah, flags: 0);
357	cm_deref_id(cm_id_priv);
358	ib_free_send_mad(send_buf: msg);
359	}
360
361	static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,
362	struct ib_mad_recv_wc *mad_recv_wc)
363	{
364	return ib_create_send_mad(mad_agent: port->mad_agent, remote_qpn: 1, pkey_index: mad_recv_wc->wc->pkey_index,
365	rmpp_active: 0, hdr_len: IB_MGMT_MAD_HDR, data_len: IB_MGMT_MAD_DATA,
366	GFP_ATOMIC,
367	IB_MGMT_BASE_VERSION);
368	}
369
370	static int cm_create_response_msg_ah(struct cm_port *port,
371	struct ib_mad_recv_wc *mad_recv_wc,
372	struct ib_mad_send_buf *msg)
373	{
374	struct ib_ah *ah;
375
376	ah = ib_create_ah_from_wc(pd: port->mad_agent->qp->pd, wc: mad_recv_wc->wc,
377	grh: mad_recv_wc->recv_buf.grh, port_num: port->port_num);
378	if (IS_ERR(ptr: ah))
379	return PTR_ERR(ptr: ah);
380
381	msg->ah = ah;
382	return 0;
383	}
384
385	static int cm_alloc_response_msg(struct cm_port *port,
386	struct ib_mad_recv_wc *mad_recv_wc,
387	struct ib_mad_send_buf **msg)
388	{
389	struct ib_mad_send_buf *m;
390	int ret;
391
392	m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);
393	if (IS_ERR(ptr: m))
394	return PTR_ERR(ptr: m);
395
396	ret = cm_create_response_msg_ah(port, mad_recv_wc, msg: m);
397	if (ret) {
398	ib_free_send_mad(send_buf: m);
399	return ret;
400	}
401
402	*msg = m;
403	return 0;
404	}
405
406	static void cm_free_response_msg(struct ib_mad_send_buf *msg)
407	{
408	if (msg->ah)
409	rdma_destroy_ah(ah: msg->ah, flags: 0);
410	ib_free_send_mad(send_buf: msg);
411	}
412
413	static void *cm_copy_private_data(const void *private_data, u8 private_data_len)
414	{
415	void *data;
416
417	if (!private_data || !private_data_len)
418	return NULL;
419
420	data = kmemdup(p: private_data, size: private_data_len, GFP_KERNEL);
421	if (!data)
422	return ERR_PTR(error: -ENOMEM);
423
424	return data;
425	}
426
427	static void cm_set_private_data(struct cm_id_private *cm_id_priv,
428	void *private_data, u8 private_data_len)
429	{
430	if (cm_id_priv->private_data && cm_id_priv->private_data_len)
431	kfree(objp: cm_id_priv->private_data);
432
433	cm_id_priv->private_data = private_data;
434	cm_id_priv->private_data_len = private_data_len;
435	}
436
437	static void cm_set_av_port(struct cm_av *av, struct cm_port *port)
438	{
439	struct cm_port *old_port = av->port;
440
441	if (old_port == port)
442	return;
443
444	av->port = port;
445	if (old_port)
446	cm_device_put(cm_dev: old_port->cm_dev);
447	if (port)
448	kref_get(kref: &port->cm_dev->kref);
449	}
450
451	static void cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
452	struct rdma_ah_attr *ah_attr, struct cm_av *av)
453	{
454	cm_set_av_port(av, port);
455	av->pkey_index = wc->pkey_index;
456	rdma_move_ah_attr(dest: &av->ah_attr, src: ah_attr);
457	}
458
459	static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
460	struct ib_grh *grh, struct cm_av *av)
461	{
462	cm_set_av_port(av, port);
463	av->pkey_index = wc->pkey_index;
464	return ib_init_ah_attr_from_wc(device: port->cm_dev->ib_device,
465	port_num: port->port_num, wc,
466	grh, ah_attr: &av->ah_attr);
467	}
468
469	static struct cm_port *
470	get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)
471	{
472	struct cm_device *cm_dev;
473	struct cm_port *port = NULL;
474	unsigned long flags;
475
476	if (attr) {
477	read_lock_irqsave(&cm.device_lock, flags);
478	list_for_each_entry(cm_dev, &cm.device_list, list) {
479	if (cm_dev->ib_device == attr->device) {
480	port = cm_dev->port[attr->port_num - 1];
481	break;
482	}
483	}
484	read_unlock_irqrestore(&cm.device_lock, flags);
485	} else {
486	/* SGID attribute can be NULL in following
487	* conditions.
488	* (a) Alternative path
489	* (b) IB link layer without GRH
490	* (c) LAP send messages
491	*/
492	read_lock_irqsave(&cm.device_lock, flags);
493	list_for_each_entry(cm_dev, &cm.device_list, list) {
494	attr = rdma_find_gid(device: cm_dev->ib_device,
495	gid: &path->sgid,
496	gid_type: sa_conv_pathrec_to_gid_type(rec: path),
497	NULL);
498	if (!IS_ERR(ptr: attr)) {
499	port = cm_dev->port[attr->port_num - 1];
500	break;
501	}
502	}
503	read_unlock_irqrestore(&cm.device_lock, flags);
504	if (port)
505	rdma_put_gid_attr(attr);
506	}
507	return port;
508	}
509
510	static int cm_init_av_by_path(struct sa_path_rec *path,
511	const struct ib_gid_attr *sgid_attr,
512	struct cm_av *av)
513	{
514	struct rdma_ah_attr new_ah_attr;
515	struct cm_device *cm_dev;
516	struct cm_port *port;
517	int ret;
518
519	port = get_cm_port_from_path(path, attr: sgid_attr);
520	if (!port)
521	return -EINVAL;
522	cm_dev = port->cm_dev;
523
524	ret = ib_find_cached_pkey(device: cm_dev->ib_device, port_num: port->port_num,
525	be16_to_cpu(path->pkey), index: &av->pkey_index);
526	if (ret)
527	return ret;
528
529	cm_set_av_port(av, port);
530
531	/*
532	* av->ah_attr might be initialized based on wc or during
533	* request processing time which might have reference to sgid_attr.
534	* So initialize a new ah_attr on stack.
535	* If initialization fails, old ah_attr is used for sending any
536	* responses. If initialization is successful, than new ah_attr
537	* is used by overwriting the old one. So that right ah_attr
538	* can be used to return an error response.
539	*/
540	ret = ib_init_ah_attr_from_path(device: cm_dev->ib_device, port_num: port->port_num, rec: path,
541	ah_attr: &new_ah_attr, sgid_attr);
542	if (ret)
543	return ret;
544
545	av->timeout = path->packet_life_time + 1;
546	rdma_move_ah_attr(dest: &av->ah_attr, src: &new_ah_attr);
547	return 0;
548	}
549
550	/* Move av created by cm_init_av_by_path(), so av.dgid is not moved */
551	static void cm_move_av_from_path(struct cm_av *dest, struct cm_av *src)
552	{
553	cm_set_av_port(av: dest, port: src->port);
554	cm_set_av_port(av: src, NULL);
555	dest->pkey_index = src->pkey_index;
556	rdma_move_ah_attr(dest: &dest->ah_attr, src: &src->ah_attr);
557	dest->timeout = src->timeout;
558	}
559
560	static void cm_destroy_av(struct cm_av *av)
561	{
562	rdma_destroy_ah_attr(ah_attr: &av->ah_attr);
563	cm_set_av_port(av, NULL);
564	}
565
566	static u32 cm_local_id(__be32 local_id)
567	{
568	return (__force u32) (local_id ^ cm.random_id_operand);
569	}
570
571	static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)
572	{
573	struct cm_id_private *cm_id_priv;
574
575	rcu_read_lock();
576	cm_id_priv = xa_load(&cm.local_id_table, index: cm_local_id(local_id));
577	if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id ||
578	!refcount_inc_not_zero(r: &cm_id_priv->refcount))
579	cm_id_priv = NULL;
580	rcu_read_unlock();
581
582	return cm_id_priv;
583	}
584
585	/*
586	* Trivial helpers to strip endian annotation and compare; the
587	* endianness doesn't actually matter since we just need a stable
588	* order for the RB tree.
589	*/
590	static int be32_lt(__be32 a, __be32 b)
591	{
592	return (__force u32) a < (__force u32) b;
593	}
594
595	static int be32_gt(__be32 a, __be32 b)
596	{
597	return (__force u32) a > (__force u32) b;
598	}
599
600	static int be64_lt(__be64 a, __be64 b)
601	{
602	return (__force u64) a < (__force u64) b;
603	}
604
605	static int be64_gt(__be64 a, __be64 b)
606	{
607	return (__force u64) a > (__force u64) b;
608	}
609
610	/*
611	* Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv
612	* if the new ID was inserted, NULL if it could not be inserted due to a
613	* collision, or the existing cm_id_priv ready for shared usage.
614	*/
615	static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv,
616	ib_cm_handler shared_handler)
617	{
618	struct rb_node **link = &cm.listen_service_table.rb_node;
619	struct rb_node *parent = NULL;
620	struct cm_id_private *cur_cm_id_priv;
621	__be64 service_id = cm_id_priv->id.service_id;
622	unsigned long flags;
623
624	spin_lock_irqsave(&cm.lock, flags);
625	while (*link) {
626	parent = *link;
627	cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
628	service_node);
629
630	if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
631	link = &(*link)->rb_left;
632	else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
633	link = &(*link)->rb_right;
634	else if (be64_lt(a: service_id, b: cur_cm_id_priv->id.service_id))
635	link = &(*link)->rb_left;
636	else if (be64_gt(a: service_id, b: cur_cm_id_priv->id.service_id))
637	link = &(*link)->rb_right;
638	else {
639	/*
640	* Sharing an ib_cm_id with different handlers is not
641	* supported
642	*/
643	if (cur_cm_id_priv->id.cm_handler != shared_handler ||
644	cur_cm_id_priv->id.context ||
645	WARN_ON(!cur_cm_id_priv->id.cm_handler)) {
646	spin_unlock_irqrestore(lock: &cm.lock, flags);
647	return NULL;
648	}
649	refcount_inc(r: &cur_cm_id_priv->refcount);
650	cur_cm_id_priv->listen_sharecount++;
651	spin_unlock_irqrestore(lock: &cm.lock, flags);
652	return cur_cm_id_priv;
653	}
654	}
655	cm_id_priv->listen_sharecount++;
656	rb_link_node(node: &cm_id_priv->service_node, parent, rb_link: link);
657	rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
658	spin_unlock_irqrestore(lock: &cm.lock, flags);
659	return cm_id_priv;
660	}
661
662	static struct cm_id_private *cm_find_listen(struct ib_device *device,
663	__be64 service_id)
664	{
665	struct rb_node *node = cm.listen_service_table.rb_node;
666	struct cm_id_private *cm_id_priv;
667
668	while (node) {
669	cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
670
671	if (device < cm_id_priv->id.device)
672	node = node->rb_left;
673	else if (device > cm_id_priv->id.device)
674	node = node->rb_right;
675	else if (be64_lt(a: service_id, b: cm_id_priv->id.service_id))
676	node = node->rb_left;
677	else if (be64_gt(a: service_id, b: cm_id_priv->id.service_id))
678	node = node->rb_right;
679	else {
680	refcount_inc(r: &cm_id_priv->refcount);
681	return cm_id_priv;
682	}
683	}
684	return NULL;
685	}
686
687	static struct cm_timewait_info *
688	cm_insert_remote_id(struct cm_timewait_info *timewait_info)
689	{
690	struct rb_node **link = &cm.remote_id_table.rb_node;
691	struct rb_node *parent = NULL;
692	struct cm_timewait_info *cur_timewait_info;
693	__be64 remote_ca_guid = timewait_info->remote_ca_guid;
694	__be32 remote_id = timewait_info->work.remote_id;
695
696	while (*link) {
697	parent = *link;
698	cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
699	remote_id_node);
700	if (be32_lt(a: remote_id, b: cur_timewait_info->work.remote_id))
701	link = &(*link)->rb_left;
702	else if (be32_gt(a: remote_id, b: cur_timewait_info->work.remote_id))
703	link = &(*link)->rb_right;
704	else if (be64_lt(a: remote_ca_guid, b: cur_timewait_info->remote_ca_guid))
705	link = &(*link)->rb_left;
706	else if (be64_gt(a: remote_ca_guid, b: cur_timewait_info->remote_ca_guid))
707	link = &(*link)->rb_right;
708	else
709	return cur_timewait_info;
710	}
711	timewait_info->inserted_remote_id = 1;
712	rb_link_node(node: &timewait_info->remote_id_node, parent, rb_link: link);
713	rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
714	return NULL;
715	}
716
717	static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid,
718	__be32 remote_id)
719	{
720	struct rb_node *node = cm.remote_id_table.rb_node;
721	struct cm_timewait_info *timewait_info;
722	struct cm_id_private *res = NULL;
723
724	spin_lock_irq(lock: &cm.lock);
725	while (node) {
726	timewait_info = rb_entry(node, struct cm_timewait_info,
727	remote_id_node);
728	if (be32_lt(a: remote_id, b: timewait_info->work.remote_id))
729	node = node->rb_left;
730	else if (be32_gt(a: remote_id, b: timewait_info->work.remote_id))
731	node = node->rb_right;
732	else if (be64_lt(a: remote_ca_guid, b: timewait_info->remote_ca_guid))
733	node = node->rb_left;
734	else if (be64_gt(a: remote_ca_guid, b: timewait_info->remote_ca_guid))
735	node = node->rb_right;
736	else {
737	res = cm_acquire_id(local_id: timewait_info->work.local_id,
738	remote_id: timewait_info->work.remote_id);
739	break;
740	}
741	}
742	spin_unlock_irq(lock: &cm.lock);
743	return res;
744	}
745
746	static struct cm_timewait_info *
747	cm_insert_remote_qpn(struct cm_timewait_info *timewait_info)
748	{
749	struct rb_node **link = &cm.remote_qp_table.rb_node;
750	struct rb_node *parent = NULL;
751	struct cm_timewait_info *cur_timewait_info;
752	__be64 remote_ca_guid = timewait_info->remote_ca_guid;
753	__be32 remote_qpn = timewait_info->remote_qpn;
754
755	while (*link) {
756	parent = *link;
757	cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
758	remote_qp_node);
759	if (be32_lt(a: remote_qpn, b: cur_timewait_info->remote_qpn))
760	link = &(*link)->rb_left;
761	else if (be32_gt(a: remote_qpn, b: cur_timewait_info->remote_qpn))
762	link = &(*link)->rb_right;
763	else if (be64_lt(a: remote_ca_guid, b: cur_timewait_info->remote_ca_guid))
764	link = &(*link)->rb_left;
765	else if (be64_gt(a: remote_ca_guid, b: cur_timewait_info->remote_ca_guid))
766	link = &(*link)->rb_right;
767	else
768	return cur_timewait_info;
769	}
770	timewait_info->inserted_remote_qp = 1;
771	rb_link_node(node: &timewait_info->remote_qp_node, parent, rb_link: link);
772	rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
773	return NULL;
774	}
775
776	static struct cm_id_private *
777	cm_insert_remote_sidr(struct cm_id_private *cm_id_priv)
778	{
779	struct rb_node **link = &cm.remote_sidr_table.rb_node;
780	struct rb_node *parent = NULL;
781	struct cm_id_private *cur_cm_id_priv;
782	__be32 remote_id = cm_id_priv->id.remote_id;
783
784	while (*link) {
785	parent = *link;
786	cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
787	sidr_id_node);
788	if (be32_lt(a: remote_id, b: cur_cm_id_priv->id.remote_id))
789	link = &(*link)->rb_left;
790	else if (be32_gt(a: remote_id, b: cur_cm_id_priv->id.remote_id))
791	link = &(*link)->rb_right;
792	else {
793	if (cur_cm_id_priv->sidr_slid < cm_id_priv->sidr_slid)
794	link = &(*link)->rb_left;
795	else if (cur_cm_id_priv->sidr_slid > cm_id_priv->sidr_slid)
796	link = &(*link)->rb_right;
797	else
798	return cur_cm_id_priv;
799	}
800	}
801	rb_link_node(node: &cm_id_priv->sidr_id_node, parent, rb_link: link);
802	rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
803	return NULL;
804	}
805
806	static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device,
807	ib_cm_handler cm_handler,
808	void *context)
809	{
810	struct cm_id_private *cm_id_priv;
811	u32 id;
812	int ret;
813
814	cm_id_priv = kzalloc(size: sizeof *cm_id_priv, GFP_KERNEL);
815	if (!cm_id_priv)
816	return ERR_PTR(error: -ENOMEM);
817
818	cm_id_priv->id.state = IB_CM_IDLE;
819	cm_id_priv->id.device = device;
820	cm_id_priv->id.cm_handler = cm_handler;
821	cm_id_priv->id.context = context;
822	cm_id_priv->id.remote_cm_qpn = 1;
823
824	RB_CLEAR_NODE(&cm_id_priv->service_node);
825	RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
826	spin_lock_init(&cm_id_priv->lock);
827	init_completion(x: &cm_id_priv->comp);
828	INIT_LIST_HEAD(list: &cm_id_priv->work_list);
829	atomic_set(v: &cm_id_priv->work_count, i: -1);
830	refcount_set(r: &cm_id_priv->refcount, n: 1);
831
832	ret = xa_alloc_cyclic(xa: &cm.local_id_table, id: &id, NULL, xa_limit_32b,
833	next: &cm.local_id_next, GFP_KERNEL);
834	if (ret < 0)
835	goto error;
836	cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
837
838	return cm_id_priv;
839
840	error:
841	kfree(objp: cm_id_priv);
842	return ERR_PTR(error: ret);
843	}
844
845	/*
846	* Make the ID visible to the MAD handlers and other threads that use the
847	* xarray.
848	*/
849	static void cm_finalize_id(struct cm_id_private *cm_id_priv)
850	{
851	xa_store(&cm.local_id_table, index: cm_local_id(local_id: cm_id_priv->id.local_id),
852	entry: cm_id_priv, GFP_ATOMIC);
853	}
854
855	struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
856	ib_cm_handler cm_handler,
857	void *context)
858	{
859	struct cm_id_private *cm_id_priv;
860
861	cm_id_priv = cm_alloc_id_priv(device, cm_handler, context);
862	if (IS_ERR(ptr: cm_id_priv))
863	return ERR_CAST(ptr: cm_id_priv);
864
865	cm_finalize_id(cm_id_priv);
866	return &cm_id_priv->id;
867	}
868	EXPORT_SYMBOL(ib_create_cm_id);
869
870	static struct cm_work *cm_dequeue_work(struct cm_id_private *cm_id_priv)
871	{
872	struct cm_work *work;
873
874	if (list_empty(head: &cm_id_priv->work_list))
875	return NULL;
876
877	work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
878	list_del(entry: &work->list);
879	return work;
880	}
881
882	static void cm_free_work(struct cm_work *work)
883	{
884	if (work->mad_recv_wc)
885	ib_free_recv_mad(mad_recv_wc: work->mad_recv_wc);
886	kfree(objp: work);
887	}
888
889	static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv,
890	struct cm_work *work)
891	__releases(&cm_id_priv->lock)
892	{
893	bool immediate;
894
895	/*
896	* To deliver the event to the user callback we have the drop the
897	* spinlock, however, we need to ensure that the user callback is single
898	* threaded and receives events in the temporal order. If there are
899	* already events being processed then thread new events onto a list,
900	* the thread currently processing will pick them up.
901	*/
902	immediate = atomic_inc_and_test(v: &cm_id_priv->work_count);
903	if (!immediate) {
904	list_add_tail(new: &work->list, head: &cm_id_priv->work_list);
905	/*
906	* This routine always consumes incoming reference. Once queued
907	* to the work_list then a reference is held by the thread
908	* currently running cm_process_work() and this reference is not
909	* needed.
910	*/
911	cm_deref_id(cm_id_priv);
912	}
913	spin_unlock_irq(lock: &cm_id_priv->lock);
914
915	if (immediate)
916	cm_process_work(cm_id_priv, work);
917	}
918
919	static inline int cm_convert_to_ms(int iba_time)
920	{
921	/* approximate conversion to ms from 4.096us x 2^iba_time */
922	return 1 << max(iba_time - 8, 0);
923	}
924
925	/*
926	* calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
927	* Because of how ack_timeout is stored, adding one doubles the timeout.
928	* To avoid large timeouts, select the max(ack_delay, life_time + 1), and
929	* increment it (round up) only if the other is within 50%.
930	*/
931	static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
932	{
933	int ack_timeout = packet_life_time + 1;
934
935	if (ack_timeout >= ca_ack_delay)
936	ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
937	else
938	ack_timeout = ca_ack_delay +
939	(ack_timeout >= (ca_ack_delay - 1));
940
941	return min(31, ack_timeout);
942	}
943
944	static void cm_remove_remote(struct cm_id_private *cm_id_priv)
945	{
946	struct cm_timewait_info *timewait_info = cm_id_priv->timewait_info;
947
948	if (timewait_info->inserted_remote_id) {
949	rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
950	timewait_info->inserted_remote_id = 0;
951	}
952
953	if (timewait_info->inserted_remote_qp) {
954	rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
955	timewait_info->inserted_remote_qp = 0;
956	}
957	}
958
959	static struct cm_timewait_info *cm_create_timewait_info(__be32 local_id)
960	{
961	struct cm_timewait_info *timewait_info;
962
963	timewait_info = kzalloc(size: sizeof *timewait_info, GFP_KERNEL);
964	if (!timewait_info)
965	return ERR_PTR(error: -ENOMEM);
966
967	timewait_info->work.local_id = local_id;
968	INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
969	timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
970	return timewait_info;
971	}
972
973	static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
974	{
975	int wait_time;
976	unsigned long flags;
977	struct cm_device *cm_dev;
978
979	lockdep_assert_held(&cm_id_priv->lock);
980
981	cm_dev = ib_get_client_data(device: cm_id_priv->id.device, client: &cm_client);
982	if (!cm_dev)
983	return;
984
985	spin_lock_irqsave(&cm.lock, flags);
986	cm_remove_remote(cm_id_priv);
987	list_add_tail(new: &cm_id_priv->timewait_info->list, head: &cm.timewait_list);
988	spin_unlock_irqrestore(lock: &cm.lock, flags);
989
990	/*
991	* The cm_id could be destroyed by the user before we exit timewait.
992	* To protect against this, we search for the cm_id after exiting
993	* timewait before notifying the user that we've exited timewait.
994	*/
995	cm_id_priv->id.state = IB_CM_TIMEWAIT;
996	wait_time = cm_convert_to_ms(iba_time: cm_id_priv->av.timeout);
997
998	/* Check if the device started its remove_one */
999	spin_lock_irqsave(&cm.lock, flags);
1000	if (!cm_dev->going_down)
1001	queue_delayed_work(wq: cm.wq, dwork: &cm_id_priv->timewait_info->work.work,
1002	delay: msecs_to_jiffies(m: wait_time));
1003	spin_unlock_irqrestore(lock: &cm.lock, flags);
1004
1005	/*
1006	* The timewait_info is converted into a work and gets freed during
1007	* cm_free_work() in cm_timewait_handler().
1008	*/
1009	BUILD_BUG_ON(offsetof(struct cm_timewait_info, work) != 0);
1010	cm_id_priv->timewait_info = NULL;
1011	}
1012
1013	static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
1014	{
1015	unsigned long flags;
1016
1017	lockdep_assert_held(&cm_id_priv->lock);
1018
1019	cm_id_priv->id.state = IB_CM_IDLE;
1020	if (cm_id_priv->timewait_info) {
1021	spin_lock_irqsave(&cm.lock, flags);
1022	cm_remove_remote(cm_id_priv);
1023	spin_unlock_irqrestore(lock: &cm.lock, flags);
1024	kfree(objp: cm_id_priv->timewait_info);
1025	cm_id_priv->timewait_info = NULL;
1026	}
1027	}
1028
1029	static noinline void cm_destroy_id_wait_timeout(struct ib_cm_id *cm_id,
1030	enum ib_cm_state old_state)
1031	{
1032	struct cm_id_private *cm_id_priv;
1033
1034	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1035	pr_err("%s: cm_id=%p timed out. state %d -> %d, refcnt=%d\n", __func__,
1036	cm_id, old_state, cm_id->state, refcount_read(&cm_id_priv->refcount));
1037	}
1038
1039	static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
1040	{
1041	struct cm_id_private *cm_id_priv;
1042	enum ib_cm_state old_state;
1043	struct cm_work *work;
1044	int ret;
1045
1046	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1047	spin_lock_irq(lock: &cm_id_priv->lock);
1048	old_state = cm_id->state;
1049	retest:
1050	switch (cm_id->state) {
1051	case IB_CM_LISTEN:
1052	spin_lock(lock: &cm.lock);
1053	if (--cm_id_priv->listen_sharecount > 0) {
1054	/* The id is still shared. */
1055	WARN_ON(refcount_read(&cm_id_priv->refcount) == 1);
1056	spin_unlock(lock: &cm.lock);
1057	spin_unlock_irq(lock: &cm_id_priv->lock);
1058	cm_deref_id(cm_id_priv);
1059	return;
1060	}
1061	cm_id->state = IB_CM_IDLE;
1062	rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
1063	RB_CLEAR_NODE(&cm_id_priv->service_node);
1064	spin_unlock(lock: &cm.lock);
1065	break;
1066	case IB_CM_SIDR_REQ_SENT:
1067	cm_id->state = IB_CM_IDLE;
1068	ib_cancel_mad(send_buf: cm_id_priv->msg);
1069	break;
1070	case IB_CM_SIDR_REQ_RCVD:
1071	cm_send_sidr_rep_locked(cm_id_priv,
1072	param: &(struct ib_cm_sidr_rep_param){
1073	.status = IB_SIDR_REJECT });
1074	/* cm_send_sidr_rep_locked will not move to IDLE if it fails */
1075	cm_id->state = IB_CM_IDLE;
1076	break;
1077	case IB_CM_REQ_SENT:
1078	case IB_CM_MRA_REQ_RCVD:
1079	ib_cancel_mad(send_buf: cm_id_priv->msg);
1080	cm_send_rej_locked(cm_id_priv, reason: IB_CM_REJ_TIMEOUT,
1081	ari: &cm_id_priv->id.device->node_guid,
1082	ari_length: sizeof(cm_id_priv->id.device->node_guid),
1083	NULL, private_data_len: 0);
1084	break;
1085	case IB_CM_REQ_RCVD:
1086	if (err == -ENOMEM) {
1087	/* Do not reject to allow future retries. */
1088	cm_reset_to_idle(cm_id_priv);
1089	} else {
1090	cm_send_rej_locked(cm_id_priv,
1091	reason: IB_CM_REJ_CONSUMER_DEFINED, NULL, ari_length: 0,
1092	NULL, private_data_len: 0);
1093	}
1094	break;
1095	case IB_CM_REP_SENT:
1096	case IB_CM_MRA_REP_RCVD:
1097	ib_cancel_mad(send_buf: cm_id_priv->msg);
1098	cm_send_rej_locked(cm_id_priv, reason: IB_CM_REJ_CONSUMER_DEFINED, NULL,
1099	ari_length: 0, NULL, private_data_len: 0);
1100	goto retest;
1101	case IB_CM_MRA_REQ_SENT:
1102	case IB_CM_REP_RCVD:
1103	case IB_CM_MRA_REP_SENT:
1104	cm_send_rej_locked(cm_id_priv, reason: IB_CM_REJ_CONSUMER_DEFINED, NULL,
1105	ari_length: 0, NULL, private_data_len: 0);
1106	break;
1107	case IB_CM_ESTABLISHED:
1108	if (cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
1109	cm_id->state = IB_CM_IDLE;
1110	break;
1111	}
1112	cm_send_dreq_locked(cm_id_priv, NULL, private_data_len: 0);
1113	goto retest;
1114	case IB_CM_DREQ_SENT:
1115	ib_cancel_mad(send_buf: cm_id_priv->msg);
1116	cm_enter_timewait(cm_id_priv);
1117	goto retest;
1118	case IB_CM_DREQ_RCVD:
1119	cm_send_drep_locked(cm_id_priv, NULL, private_data_len: 0);
1120	WARN_ON(cm_id->state != IB_CM_TIMEWAIT);
1121	goto retest;
1122	case IB_CM_TIMEWAIT:
1123	/*
1124	* The cm_acquire_id in cm_timewait_handler will stop working
1125	* once we do xa_erase below, so just move to idle here for
1126	* consistency.
1127	*/
1128	cm_id->state = IB_CM_IDLE;
1129	break;
1130	case IB_CM_IDLE:
1131	break;
1132	}
1133	WARN_ON(cm_id->state != IB_CM_IDLE);
1134
1135	spin_lock(lock: &cm.lock);
1136	/* Required for cleanup paths related cm_req_handler() */
1137	if (cm_id_priv->timewait_info) {
1138	cm_remove_remote(cm_id_priv);
1139	kfree(objp: cm_id_priv->timewait_info);
1140	cm_id_priv->timewait_info = NULL;
1141	}
1142
1143	WARN_ON(cm_id_priv->listen_sharecount);
1144	WARN_ON(!RB_EMPTY_NODE(&cm_id_priv->service_node));
1145	if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1146	rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
1147	spin_unlock(lock: &cm.lock);
1148	spin_unlock_irq(lock: &cm_id_priv->lock);
1149
1150	xa_erase(&cm.local_id_table, index: cm_local_id(local_id: cm_id->local_id));
1151	cm_deref_id(cm_id_priv);
1152	do {
1153	ret = wait_for_completion_timeout(x: &cm_id_priv->comp,
1154	timeout: msecs_to_jiffies(
1155	CM_DESTROY_ID_WAIT_TIMEOUT));
1156	if (!ret) /* timeout happened */
1157	cm_destroy_id_wait_timeout(cm_id, old_state);
1158	} while (!ret);
1159
1160	while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1161	cm_free_work(work);
1162
1163	cm_destroy_av(av: &cm_id_priv->av);
1164	cm_destroy_av(av: &cm_id_priv->alt_av);
1165	kfree(objp: cm_id_priv->private_data);
1166	kfree_rcu(cm_id_priv, rcu);
1167	}
1168
1169	void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1170	{
1171	cm_destroy_id(cm_id, err: 0);
1172	}
1173	EXPORT_SYMBOL(ib_destroy_cm_id);
1174
1175	static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id)
1176	{
1177	if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1178	(service_id != IB_CM_ASSIGN_SERVICE_ID))
1179	return -EINVAL;
1180
1181	if (service_id == IB_CM_ASSIGN_SERVICE_ID)
1182	cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++);
1183	else
1184	cm_id_priv->id.service_id = service_id;
1185
1186	return 0;
1187	}
1188
1189	/**
1190	* ib_cm_listen - Initiates listening on the specified service ID for
1191	* connection and service ID resolution requests.
1192	* @cm_id: Connection identifier associated with the listen request.
1193	* @service_id: Service identifier matched against incoming connection
1194	* and service ID resolution requests. The service ID should be specified
1195	* network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1196	* assign a service ID to the caller.
1197	*/
1198	int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id)
1199	{
1200	struct cm_id_private *cm_id_priv =
1201	container_of(cm_id, struct cm_id_private, id);
1202	unsigned long flags;
1203	int ret;
1204
1205	spin_lock_irqsave(&cm_id_priv->lock, flags);
1206	if (cm_id_priv->id.state != IB_CM_IDLE) {
1207	ret = -EINVAL;
1208	goto out;
1209	}
1210
1211	ret = cm_init_listen(cm_id_priv, service_id);
1212	if (ret)
1213	goto out;
1214
1215	if (!cm_insert_listen(cm_id_priv, NULL)) {
1216	ret = -EBUSY;
1217	goto out;
1218	}
1219
1220	cm_id_priv->id.state = IB_CM_LISTEN;
1221	ret = 0;
1222
1223	out:
1224	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
1225	return ret;
1226	}
1227	EXPORT_SYMBOL(ib_cm_listen);
1228
1229	/**
1230	* ib_cm_insert_listen - Create a new listening ib_cm_id and listen on
1231	* the given service ID.
1232	*
1233	* If there's an existing ID listening on that same device and service ID,
1234	* return it.
1235	*
1236	* @device: Device associated with the cm_id. All related communication will
1237	* be associated with the specified device.
1238	* @cm_handler: Callback invoked to notify the user of CM events.
1239	* @service_id: Service identifier matched against incoming connection
1240	* and service ID resolution requests. The service ID should be specified
1241	* network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1242	* assign a service ID to the caller.
1243	*
1244	* Callers should call ib_destroy_cm_id when done with the listener ID.
1245	*/
1246	struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1247	ib_cm_handler cm_handler,
1248	__be64 service_id)
1249	{
1250	struct cm_id_private *listen_id_priv;
1251	struct cm_id_private *cm_id_priv;
1252	int err = 0;
1253
1254	/* Create an ID in advance, since the creation may sleep */
1255	cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL);
1256	if (IS_ERR(ptr: cm_id_priv))
1257	return ERR_CAST(ptr: cm_id_priv);
1258
1259	err = cm_init_listen(cm_id_priv, service_id);
1260	if (err) {
1261	ib_destroy_cm_id(&cm_id_priv->id);
1262	return ERR_PTR(error: err);
1263	}
1264
1265	spin_lock_irq(lock: &cm_id_priv->lock);
1266	listen_id_priv = cm_insert_listen(cm_id_priv, shared_handler: cm_handler);
1267	if (listen_id_priv != cm_id_priv) {
1268	spin_unlock_irq(lock: &cm_id_priv->lock);
1269	ib_destroy_cm_id(&cm_id_priv->id);
1270	if (!listen_id_priv)
1271	return ERR_PTR(error: -EINVAL);
1272	return &listen_id_priv->id;
1273	}
1274	cm_id_priv->id.state = IB_CM_LISTEN;
1275	spin_unlock_irq(lock: &cm_id_priv->lock);
1276
1277	/*
1278	* A listen ID does not need to be in the xarray since it does not
1279	* receive mads, is not placed in the remote_id or remote_qpn rbtree,
1280	* and does not enter timewait.
1281	*/
1282
1283	return &cm_id_priv->id;
1284	}
1285	EXPORT_SYMBOL(ib_cm_insert_listen);
1286
1287	static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)
1288	{
1289	u64 hi_tid = 0, low_tid;
1290
1291	lockdep_assert_held(&cm_id_priv->lock);
1292
1293	low_tid = (u64)cm_id_priv->id.local_id;
1294	if (!cm_id_priv->av.port)
1295	return cpu_to_be64(low_tid);
1296
1297	spin_lock(lock: &cm_id_priv->av.port->cm_dev->mad_agent_lock);
1298	if (cm_id_priv->av.port->mad_agent)
1299	hi_tid = ((u64)cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1300	spin_unlock(lock: &cm_id_priv->av.port->cm_dev->mad_agent_lock);
1301	return cpu_to_be64(hi_tid | low_tid);
1302	}
1303
1304	static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1305	__be16 attr_id, __be64 tid)
1306	{
1307	hdr->base_version = IB_MGMT_BASE_VERSION;
1308	hdr->mgmt_class = IB_MGMT_CLASS_CM;
1309	hdr->class_version = IB_CM_CLASS_VERSION;
1310	hdr->method = IB_MGMT_METHOD_SEND;
1311	hdr->attr_id = attr_id;
1312	hdr->tid = tid;
1313	}
1314
1315	static void cm_format_mad_ece_hdr(struct ib_mad_hdr *hdr, __be16 attr_id,
1316	__be64 tid, u32 attr_mod)
1317	{
1318	cm_format_mad_hdr(hdr, attr_id, tid);
1319	hdr->attr_mod = cpu_to_be32(attr_mod);
1320	}
1321
1322	static void cm_format_req(struct cm_req_msg *req_msg,
1323	struct cm_id_private *cm_id_priv,
1324	struct ib_cm_req_param *param)
1325	{
1326	struct sa_path_rec *pri_path = param->primary_path;
1327	struct sa_path_rec *alt_path = param->alternate_path;
1328	bool pri_ext = false;
1329	__be16 lid;
1330
1331	if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1332	pri_ext = opa_is_extended_lid(dlid: pri_path->opa.dlid,
1333	slid: pri_path->opa.slid);
1334
1335	cm_format_mad_ece_hdr(hdr: &req_msg->hdr, CM_REQ_ATTR_ID,
1336	tid: cm_form_tid(cm_id_priv), attr_mod: param->ece.attr_mod);
1337
1338	IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,
1339	be32_to_cpu(cm_id_priv->id.local_id));
1340	IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id));
1341	IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg,
1342	be64_to_cpu(cm_id_priv->id.device->node_guid));
1343	IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num);
1344	IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth);
1345	IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg,
1346	param->remote_cm_response_timeout);
1347	cm_req_set_qp_type(req_msg, qp_type: param->qp_type);
1348	IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control);
1349	IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn);
1350	IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg,
1351	param->local_cm_response_timeout);
1352	IBA_SET(CM_REQ_PARTITION_KEY, req_msg,
1353	be16_to_cpu(param->primary_path->pkey));
1354	IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg,
1355	param->primary_path->mtu);
1356	IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries);
1357
1358	if (param->qp_type != IB_QPT_XRC_INI) {
1359	IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg,
1360	param->responder_resources);
1361	IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count);
1362	IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg,
1363	param->rnr_retry_count);
1364	IBA_SET(CM_REQ_SRQ, req_msg, param->srq);
1365	}
1366
1367	*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) =
1368	pri_path->sgid;
1369	*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) =
1370	pri_path->dgid;
1371	if (pri_ext) {
1372	IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)
1373	->global.interface_id =
1374	OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1375	IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)
1376	->global.interface_id =
1377	OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1378	}
1379	if (pri_path->hop_limit <= 1) {
1380	IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1381	be16_to_cpu(pri_ext ? 0 :
1382	htons(ntohl(sa_path_get_slid(
1383	pri_path)))));
1384	IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1385	be16_to_cpu(pri_ext ? 0 :
1386	htons(ntohl(sa_path_get_dlid(
1387	pri_path)))));
1388	} else {
1389
1390	if (param->primary_path_inbound) {
1391	lid = param->primary_path_inbound->ib.dlid;
1392	IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1393	be16_to_cpu(lid));
1394	} else
1395	IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1396	be16_to_cpu(IB_LID_PERMISSIVE));
1397
1398	/* Work-around until there's a way to obtain remote LID info */
1399	IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1400	be16_to_cpu(IB_LID_PERMISSIVE));
1401	}
1402	IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg,
1403	be32_to_cpu(pri_path->flow_label));
1404	IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate);
1405	IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class);
1406	IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit);
1407	IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl);
1408	IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg,
1409	(pri_path->hop_limit <= 1));
1410	IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg,
1411	cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1412	pri_path->packet_life_time));
1413
1414	if (alt_path) {
1415	bool alt_ext = false;
1416
1417	if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1418	alt_ext = opa_is_extended_lid(dlid: alt_path->opa.dlid,
1419	slid: alt_path->opa.slid);
1420
1421	*IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) =
1422	alt_path->sgid;
1423	*IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) =
1424	alt_path->dgid;
1425	if (alt_ext) {
1426	IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1427	req_msg)
1428	->global.interface_id =
1429	OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1430	IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID,
1431	req_msg)
1432	->global.interface_id =
1433	OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1434	}
1435	if (alt_path->hop_limit <= 1) {
1436	IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1437	be16_to_cpu(
1438	alt_ext ? 0 :
1439	htons(ntohl(sa_path_get_slid(
1440	alt_path)))));
1441	IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1442	be16_to_cpu(
1443	alt_ext ? 0 :
1444	htons(ntohl(sa_path_get_dlid(
1445	alt_path)))));
1446	} else {
1447	IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1448	be16_to_cpu(IB_LID_PERMISSIVE));
1449	IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1450	be16_to_cpu(IB_LID_PERMISSIVE));
1451	}
1452	IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg,
1453	be32_to_cpu(alt_path->flow_label));
1454	IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate);
1455	IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg,
1456	alt_path->traffic_class);
1457	IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg,
1458	alt_path->hop_limit);
1459	IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl);
1460	IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg,
1461	(alt_path->hop_limit <= 1));
1462	IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg,
1463	cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1464	alt_path->packet_life_time));
1465	}
1466	IBA_SET(CM_REQ_VENDOR_ID, req_msg, param->ece.vendor_id);
1467
1468	if (param->private_data && param->private_data_len)
1469	IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data,
1470	param->private_data_len);
1471	}
1472
1473	static int cm_validate_req_param(struct ib_cm_req_param *param)
1474	{
1475	if (!param->primary_path)
1476	return -EINVAL;
1477
1478	if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1479	param->qp_type != IB_QPT_XRC_INI)
1480	return -EINVAL;
1481
1482	if (param->private_data &&
1483	param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1484	return -EINVAL;
1485
1486	if (param->alternate_path &&
1487	(param->alternate_path->pkey != param->primary_path->pkey ||
1488	param->alternate_path->mtu != param->primary_path->mtu))
1489	return -EINVAL;
1490
1491	return 0;
1492	}
1493
1494	int ib_send_cm_req(struct ib_cm_id *cm_id,
1495	struct ib_cm_req_param *param)
1496	{
1497	struct cm_av av = {}, alt_av = {};
1498	struct cm_id_private *cm_id_priv;
1499	struct ib_mad_send_buf *msg;
1500	struct cm_req_msg *req_msg;
1501	unsigned long flags;
1502	int ret;
1503
1504	ret = cm_validate_req_param(param);
1505	if (ret)
1506	return ret;
1507
1508	/* Verify that we're not in timewait. */
1509	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1510	spin_lock_irqsave(&cm_id_priv->lock, flags);
1511	if (cm_id->state != IB_CM_IDLE || WARN_ON(cm_id_priv->timewait_info)) {
1512	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
1513	return -EINVAL;
1514	}
1515	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
1516
1517	cm_id_priv->timewait_info = cm_create_timewait_info(local_id: cm_id_priv->
1518	id.local_id);
1519	if (IS_ERR(ptr: cm_id_priv->timewait_info)) {
1520	ret = PTR_ERR(ptr: cm_id_priv->timewait_info);
1521	cm_id_priv->timewait_info = NULL;
1522	return ret;
1523	}
1524
1525	ret = cm_init_av_by_path(path: param->primary_path,
1526	sgid_attr: param->ppath_sgid_attr, av: &av);
1527	if (ret)
1528	return ret;
1529	if (param->alternate_path) {
1530	ret = cm_init_av_by_path(path: param->alternate_path, NULL,
1531	av: &alt_av);
1532	if (ret) {
1533	cm_destroy_av(av: &av);
1534	return ret;
1535	}
1536	}
1537	cm_id->service_id = param->service_id;
1538	cm_id_priv->timeout_ms = cm_convert_to_ms(
1539	iba_time: param->primary_path->packet_life_time) * 2 +
1540	cm_convert_to_ms(
1541	iba_time: param->remote_cm_response_timeout);
1542	cm_id_priv->max_cm_retries = param->max_cm_retries;
1543	cm_id_priv->initiator_depth = param->initiator_depth;
1544	cm_id_priv->responder_resources = param->responder_resources;
1545	cm_id_priv->retry_count = param->retry_count;
1546	cm_id_priv->path_mtu = param->primary_path->mtu;
1547	cm_id_priv->pkey = param->primary_path->pkey;
1548	cm_id_priv->qp_type = param->qp_type;
1549
1550	spin_lock_irqsave(&cm_id_priv->lock, flags);
1551
1552	cm_move_av_from_path(dest: &cm_id_priv->av, src: &av);
1553	if (param->primary_path_outbound)
1554	cm_id_priv->av.dlid_datapath =
1555	be16_to_cpu(param->primary_path_outbound->ib.dlid);
1556
1557	if (param->alternate_path)
1558	cm_move_av_from_path(dest: &cm_id_priv->alt_av, src: &alt_av);
1559
1560	msg = cm_alloc_priv_msg(cm_id_priv);
1561	if (IS_ERR(ptr: msg)) {
1562	ret = PTR_ERR(ptr: msg);
1563	goto out_unlock;
1564	}
1565
1566	req_msg = (struct cm_req_msg *)msg->mad;
1567	cm_format_req(req_msg, cm_id_priv, param);
1568	cm_id_priv->tid = req_msg->hdr.tid;
1569	msg->timeout_ms = cm_id_priv->timeout_ms;
1570	msg->context[1] = (void *)(unsigned long)IB_CM_REQ_SENT;
1571
1572	cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
1573	cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
1574
1575	trace_icm_send_req(cm_id: &cm_id_priv->id);
1576	ret = ib_post_send_mad(send_buf: msg, NULL);
1577	if (ret)
1578	goto out_free;
1579	BUG_ON(cm_id->state != IB_CM_IDLE);
1580	cm_id->state = IB_CM_REQ_SENT;
1581	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
1582	return 0;
1583	out_free:
1584	cm_free_priv_msg(msg);
1585	out_unlock:
1586	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
1587	return ret;
1588	}
1589	EXPORT_SYMBOL(ib_send_cm_req);
1590
1591	static int cm_issue_rej(struct cm_port *port,
1592	struct ib_mad_recv_wc *mad_recv_wc,
1593	enum ib_cm_rej_reason reason,
1594	enum cm_msg_response msg_rejected,
1595	void *ari, u8 ari_length)
1596	{
1597	struct ib_mad_send_buf *msg = NULL;
1598	struct cm_rej_msg *rej_msg, *rcv_msg;
1599	int ret;
1600
1601	ret = cm_alloc_response_msg(port, mad_recv_wc, msg: &msg);
1602	if (ret)
1603	return ret;
1604
1605	/* We just need common CM header information. Cast to any message. */
1606	rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1607	rej_msg = (struct cm_rej_msg *) msg->mad;
1608
1609	cm_format_mad_hdr(hdr: &rej_msg->hdr, CM_REJ_ATTR_ID, tid: rcv_msg->hdr.tid);
1610	IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1611	IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg));
1612	IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1613	IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1614	IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected);
1615	IBA_SET(CM_REJ_REASON, rej_msg, reason);
1616
1617	if (ari && ari_length) {
1618	IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1619	IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1620	}
1621
1622	trace_icm_issue_rej(
1623	IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg),
1624	IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1625	ret = ib_post_send_mad(send_buf: msg, NULL);
1626	if (ret)
1627	cm_free_response_msg(msg);
1628
1629	return ret;
1630	}
1631
1632	static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1633	{
1634	return ((cpu_to_be16(
1635	IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) ||
1636	(ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1637	req_msg))));
1638	}
1639
1640	static void cm_path_set_rec_type(struct ib_device *ib_device, u32 port_num,
1641	struct sa_path_rec *path, union ib_gid *gid)
1642	{
1643	if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(device: ib_device, port_num))
1644	path->rec_type = SA_PATH_REC_TYPE_OPA;
1645	else
1646	path->rec_type = SA_PATH_REC_TYPE_IB;
1647	}
1648
1649	static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1650	struct sa_path_rec *primary_path,
1651	struct sa_path_rec *alt_path,
1652	struct ib_wc *wc)
1653	{
1654	u32 lid;
1655
1656	if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1657	sa_path_set_dlid(rec: primary_path, dlid: wc->slid);
1658	sa_path_set_slid(rec: primary_path,
1659	IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
1660	req_msg));
1661	} else {
1662	lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1663	CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg));
1664	sa_path_set_dlid(rec: primary_path, dlid: lid);
1665
1666	lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1667	CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg));
1668	sa_path_set_slid(rec: primary_path, slid: lid);
1669	}
1670
1671	if (!cm_req_has_alt_path(req_msg))
1672	return;
1673
1674	if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1675	sa_path_set_dlid(rec: alt_path,
1676	IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
1677	req_msg));
1678	sa_path_set_slid(rec: alt_path,
1679	IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
1680	req_msg));
1681	} else {
1682	lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1683	CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg));
1684	sa_path_set_dlid(rec: alt_path, dlid: lid);
1685
1686	lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1687	CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg));
1688	sa_path_set_slid(rec: alt_path, slid: lid);
1689	}
1690	}
1691
1692	static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1693	struct sa_path_rec *primary_path,
1694	struct sa_path_rec *alt_path,
1695	struct ib_wc *wc)
1696	{
1697	primary_path->dgid =
1698	*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg);
1699	primary_path->sgid =
1700	*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg);
1701	primary_path->flow_label =
1702	cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg));
1703	primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg);
1704	primary_path->traffic_class =
1705	IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg);
1706	primary_path->reversible = 1;
1707	primary_path->pkey =
1708	cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1709	primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg);
1710	primary_path->mtu_selector = IB_SA_EQ;
1711	primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1712	primary_path->rate_selector = IB_SA_EQ;
1713	primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg);
1714	primary_path->packet_life_time_selector = IB_SA_EQ;
1715	primary_path->packet_life_time =
1716	IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg);
1717	primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1718	primary_path->service_id =
1719	cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1720	if (sa_path_is_roce(rec: primary_path))
1721	primary_path->roce.route_resolved = false;
1722
1723	if (cm_req_has_alt_path(req_msg)) {
1724	alt_path->dgid = *IBA_GET_MEM_PTR(
1725	CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg);
1726	alt_path->sgid = *IBA_GET_MEM_PTR(
1727	CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg);
1728	alt_path->flow_label = cpu_to_be32(
1729	IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg));
1730	alt_path->hop_limit =
1731	IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg);
1732	alt_path->traffic_class =
1733	IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg);
1734	alt_path->reversible = 1;
1735	alt_path->pkey =
1736	cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1737	alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg);
1738	alt_path->mtu_selector = IB_SA_EQ;
1739	alt_path->mtu =
1740	IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1741	alt_path->rate_selector = IB_SA_EQ;
1742	alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg);
1743	alt_path->packet_life_time_selector = IB_SA_EQ;
1744	alt_path->packet_life_time =
1745	IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg);
1746	alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1747	alt_path->service_id =
1748	cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1749
1750	if (sa_path_is_roce(rec: alt_path))
1751	alt_path->roce.route_resolved = false;
1752	}
1753	cm_format_path_lid_from_req(req_msg, primary_path, alt_path, wc);
1754	}
1755
1756	static u16 cm_get_bth_pkey(struct cm_work *work)
1757	{
1758	struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1759	u32 port_num = work->port->port_num;
1760	u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1761	u16 pkey;
1762	int ret;
1763
1764	ret = ib_get_cached_pkey(device_handle: ib_dev, port_num, index: pkey_index, pkey: &pkey);
1765	if (ret) {
1766	dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %u, pkey index %u). %d\n",
1767	port_num, pkey_index, ret);
1768	return 0;
1769	}
1770
1771	return pkey;
1772	}
1773
1774	/**
1775	* cm_opa_to_ib_sgid - Convert OPA SGID to IB SGID
1776	* ULPs (such as IPoIB) do not understand OPA GIDs and will
1777	* reject them as the local_gid will not match the sgid. Therefore,
1778	* change the pathrec's SGID to an IB SGID.
1779	*
1780	* @work: Work completion
1781	* @path: Path record
1782	*/
1783	static void cm_opa_to_ib_sgid(struct cm_work *work,
1784	struct sa_path_rec *path)
1785	{
1786	struct ib_device *dev = work->port->cm_dev->ib_device;
1787	u32 port_num = work->port->port_num;
1788
1789	if (rdma_cap_opa_ah(device: dev, port_num) &&
1790	(ib_is_opa_gid(gid: &path->sgid))) {
1791	union ib_gid sgid;
1792
1793	if (rdma_query_gid(device: dev, port_num, index: 0, gid: &sgid)) {
1794	dev_warn(&dev->dev,
1795	"Error updating sgid in CM request\n");
1796	return;
1797	}
1798
1799	path->sgid = sgid;
1800	}
1801	}
1802
1803	static void cm_format_req_event(struct cm_work *work,
1804	struct cm_id_private *cm_id_priv,
1805	struct ib_cm_id *listen_id)
1806	{
1807	struct cm_req_msg *req_msg;
1808	struct ib_cm_req_event_param *param;
1809
1810	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1811	param = &work->cm_event.param.req_rcvd;
1812	param->listen_id = listen_id;
1813	param->bth_pkey = cm_get_bth_pkey(work);
1814	param->port = cm_id_priv->av.port->port_num;
1815	param->primary_path = &work->path[0];
1816	cm_opa_to_ib_sgid(work, path: param->primary_path);
1817	if (cm_req_has_alt_path(req_msg)) {
1818	param->alternate_path = &work->path[1];
1819	cm_opa_to_ib_sgid(work, path: param->alternate_path);
1820	} else {
1821	param->alternate_path = NULL;
1822	}
1823	param->remote_ca_guid =
1824	cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
1825	param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg);
1826	param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg);
1827	param->qp_type = cm_req_get_qp_type(req_msg);
1828	param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg);
1829	param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
1830	param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
1831	param->local_cm_response_timeout =
1832	IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg);
1833	param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg);
1834	param->remote_cm_response_timeout =
1835	IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg);
1836	param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
1837	param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
1838	param->srq = IBA_GET(CM_REQ_SRQ, req_msg);
1839	param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
1840	param->ece.vendor_id = IBA_GET(CM_REQ_VENDOR_ID, req_msg);
1841	param->ece.attr_mod = be32_to_cpu(req_msg->hdr.attr_mod);
1842
1843	work->cm_event.private_data =
1844	IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);
1845	}
1846
1847	static void cm_process_work(struct cm_id_private *cm_id_priv,
1848	struct cm_work *work)
1849	{
1850	int ret;
1851
1852	/* We will typically only have the current event to report. */
1853	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1854	cm_free_work(work);
1855
1856	while (!ret && !atomic_add_negative(i: -1, v: &cm_id_priv->work_count)) {
1857	spin_lock_irq(lock: &cm_id_priv->lock);
1858	work = cm_dequeue_work(cm_id_priv);
1859	spin_unlock_irq(lock: &cm_id_priv->lock);
1860	if (!work)
1861	return;
1862
1863	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1864	&work->cm_event);
1865	cm_free_work(work);
1866	}
1867	cm_deref_id(cm_id_priv);
1868	if (ret)
1869	cm_destroy_id(cm_id: &cm_id_priv->id, err: ret);
1870	}
1871
1872	static void cm_format_mra(struct cm_mra_msg *mra_msg,
1873	struct cm_id_private *cm_id_priv,
1874	enum cm_msg_response msg_mraed, u8 service_timeout,
1875	const void *private_data, u8 private_data_len)
1876	{
1877	cm_format_mad_hdr(hdr: &mra_msg->hdr, CM_MRA_ATTR_ID, tid: cm_id_priv->tid);
1878	IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed);
1879	IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg,
1880	be32_to_cpu(cm_id_priv->id.local_id));
1881	IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg,
1882	be32_to_cpu(cm_id_priv->id.remote_id));
1883	IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout);
1884
1885	if (private_data && private_data_len)
1886	IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data,
1887	private_data_len);
1888	}
1889
1890	static void cm_format_rej(struct cm_rej_msg *rej_msg,
1891	struct cm_id_private *cm_id_priv,
1892	enum ib_cm_rej_reason reason, void *ari,
1893	u8 ari_length, const void *private_data,
1894	u8 private_data_len, enum ib_cm_state state)
1895	{
1896	lockdep_assert_held(&cm_id_priv->lock);
1897
1898	cm_format_mad_hdr(hdr: &rej_msg->hdr, CM_REJ_ATTR_ID, tid: cm_id_priv->tid);
1899	IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1900	be32_to_cpu(cm_id_priv->id.remote_id));
1901
1902	switch (state) {
1903	case IB_CM_REQ_RCVD:
1904	IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0));
1905	IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1906	break;
1907	case IB_CM_MRA_REQ_SENT:
1908	IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1909	be32_to_cpu(cm_id_priv->id.local_id));
1910	IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1911	break;
1912	case IB_CM_REP_RCVD:
1913	case IB_CM_MRA_REP_SENT:
1914	IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1915	be32_to_cpu(cm_id_priv->id.local_id));
1916	IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP);
1917	break;
1918	default:
1919	IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1920	be32_to_cpu(cm_id_priv->id.local_id));
1921	IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg,
1922	CM_MSG_RESPONSE_OTHER);
1923	break;
1924	}
1925
1926	IBA_SET(CM_REJ_REASON, rej_msg, reason);
1927	if (ari && ari_length) {
1928	IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1929	IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1930	}
1931
1932	if (private_data && private_data_len)
1933	IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data,
1934	private_data_len);
1935	}
1936
1937	static void cm_dup_req_handler(struct cm_work *work,
1938	struct cm_id_private *cm_id_priv)
1939	{
1940	struct ib_mad_send_buf *msg = NULL;
1941	int ret;
1942
1943	atomic_long_inc(
1944	v: &work->port->counters[CM_RECV_DUPLICATES][CM_REQ_COUNTER]);
1945
1946	/* Quick state check to discard duplicate REQs. */
1947	spin_lock_irq(lock: &cm_id_priv->lock);
1948	if (cm_id_priv->id.state == IB_CM_REQ_RCVD) {
1949	spin_unlock_irq(lock: &cm_id_priv->lock);
1950	return;
1951	}
1952	spin_unlock_irq(lock: &cm_id_priv->lock);
1953
1954	ret = cm_alloc_response_msg(port: work->port, mad_recv_wc: work->mad_recv_wc, msg: &msg);
1955	if (ret)
1956	return;
1957
1958	spin_lock_irq(lock: &cm_id_priv->lock);
1959	switch (cm_id_priv->id.state) {
1960	case IB_CM_MRA_REQ_SENT:
1961	cm_format_mra(mra_msg: (struct cm_mra_msg *) msg->mad, cm_id_priv,
1962	msg_mraed: CM_MSG_RESPONSE_REQ, service_timeout: cm_id_priv->service_timeout,
1963	private_data: cm_id_priv->private_data,
1964	private_data_len: cm_id_priv->private_data_len);
1965	break;
1966	case IB_CM_TIMEWAIT:
1967	cm_format_rej(rej_msg: (struct cm_rej_msg *)msg->mad, cm_id_priv,
1968	reason: IB_CM_REJ_STALE_CONN, NULL, ari_length: 0, NULL, private_data_len: 0,
1969	state: IB_CM_TIMEWAIT);
1970	break;
1971	default:
1972	goto unlock;
1973	}
1974	spin_unlock_irq(lock: &cm_id_priv->lock);
1975
1976	trace_icm_send_dup_req(cm_id: &cm_id_priv->id);
1977	ret = ib_post_send_mad(send_buf: msg, NULL);
1978	if (ret)
1979	goto free;
1980	return;
1981
1982	unlock: spin_unlock_irq(lock: &cm_id_priv->lock);
1983	free: cm_free_response_msg(msg);
1984	}
1985
1986	static struct cm_id_private *cm_match_req(struct cm_work *work,
1987	struct cm_id_private *cm_id_priv)
1988	{
1989	struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1990	struct cm_timewait_info *timewait_info;
1991	struct cm_req_msg *req_msg;
1992
1993	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1994
1995	/* Check for possible duplicate REQ. */
1996	spin_lock_irq(lock: &cm.lock);
1997	timewait_info = cm_insert_remote_id(timewait_info: cm_id_priv->timewait_info);
1998	if (timewait_info) {
1999	cur_cm_id_priv = cm_acquire_id(local_id: timewait_info->work.local_id,
2000	remote_id: timewait_info->work.remote_id);
2001	spin_unlock_irq(lock: &cm.lock);
2002	if (cur_cm_id_priv) {
2003	cm_dup_req_handler(work, cm_id_priv: cur_cm_id_priv);
2004	cm_deref_id(cm_id_priv: cur_cm_id_priv);
2005	}
2006	return NULL;
2007	}
2008
2009	/* Check for stale connections. */
2010	timewait_info = cm_insert_remote_qpn(timewait_info: cm_id_priv->timewait_info);
2011	if (timewait_info) {
2012	cm_remove_remote(cm_id_priv);
2013	cur_cm_id_priv = cm_acquire_id(local_id: timewait_info->work.local_id,
2014	remote_id: timewait_info->work.remote_id);
2015
2016	spin_unlock_irq(lock: &cm.lock);
2017	cm_issue_rej(port: work->port, mad_recv_wc: work->mad_recv_wc,
2018	reason: IB_CM_REJ_STALE_CONN, msg_rejected: CM_MSG_RESPONSE_REQ,
2019	NULL, ari_length: 0);
2020	if (cur_cm_id_priv) {
2021	ib_send_cm_dreq(cm_id: &cur_cm_id_priv->id, NULL, private_data_len: 0);
2022	cm_deref_id(cm_id_priv: cur_cm_id_priv);
2023	}
2024	return NULL;
2025	}
2026
2027	/* Find matching listen request. */
2028	listen_cm_id_priv = cm_find_listen(
2029	device: cm_id_priv->id.device,
2030	cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));
2031	if (!listen_cm_id_priv) {
2032	cm_remove_remote(cm_id_priv);
2033	spin_unlock_irq(lock: &cm.lock);
2034	cm_issue_rej(port: work->port, mad_recv_wc: work->mad_recv_wc,
2035	reason: IB_CM_REJ_INVALID_SERVICE_ID, msg_rejected: CM_MSG_RESPONSE_REQ,
2036	NULL, ari_length: 0);
2037	return NULL;
2038	}
2039	spin_unlock_irq(lock: &cm.lock);
2040	return listen_cm_id_priv;
2041	}
2042
2043	/*
2044	* Work-around for inter-subnet connections. If the LIDs are permissive,
2045	* we need to override the LID/SL data in the REQ with the LID information
2046	* in the work completion.
2047	*/
2048	static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
2049	{
2050	if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) {
2051	if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
2052	req_msg)) == IB_LID_PERMISSIVE) {
2053	IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
2054	be16_to_cpu(ib_lid_be16(wc->slid)));
2055	IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl);
2056	}
2057
2058	if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
2059	req_msg)) == IB_LID_PERMISSIVE)
2060	IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
2061	wc->dlid_path_bits);
2062	}
2063
2064	if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) {
2065	if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
2066	req_msg)) == IB_LID_PERMISSIVE) {
2067	IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
2068	be16_to_cpu(ib_lid_be16(wc->slid)));
2069	IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl);
2070	}
2071
2072	if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
2073	req_msg)) == IB_LID_PERMISSIVE)
2074	IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
2075	wc->dlid_path_bits);
2076	}
2077	}
2078
2079	static int cm_req_handler(struct cm_work *work)
2080	{
2081	struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
2082	struct cm_req_msg *req_msg;
2083	const struct ib_global_route *grh;
2084	const struct ib_gid_attr *gid_attr;
2085	int ret;
2086
2087	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
2088
2089	cm_id_priv =
2090	cm_alloc_id_priv(device: work->port->cm_dev->ib_device, NULL, NULL);
2091	if (IS_ERR(ptr: cm_id_priv))
2092	return PTR_ERR(ptr: cm_id_priv);
2093
2094	cm_id_priv->id.remote_id =
2095	cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));
2096	cm_id_priv->id.service_id =
2097	cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
2098	cm_id_priv->tid = req_msg->hdr.tid;
2099	cm_id_priv->timeout_ms = cm_convert_to_ms(
2100	IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg));
2101	cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg);
2102	cm_id_priv->remote_qpn =
2103	cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
2104	cm_id_priv->initiator_depth =
2105	IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
2106	cm_id_priv->responder_resources =
2107	IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
2108	cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
2109	cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
2110	cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
2111	cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
2112	cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
2113	cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2114
2115	ret = cm_init_av_for_response(port: work->port, wc: work->mad_recv_wc->wc,
2116	grh: work->mad_recv_wc->recv_buf.grh,
2117	av: &cm_id_priv->av);
2118	if (ret)
2119	goto destroy;
2120	cm_id_priv->timewait_info = cm_create_timewait_info(local_id: cm_id_priv->
2121	id.local_id);
2122	if (IS_ERR(ptr: cm_id_priv->timewait_info)) {
2123	ret = PTR_ERR(ptr: cm_id_priv->timewait_info);
2124	cm_id_priv->timewait_info = NULL;
2125	goto destroy;
2126	}
2127	cm_id_priv->timewait_info->work.remote_id = cm_id_priv->id.remote_id;
2128	cm_id_priv->timewait_info->remote_ca_guid =
2129	cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
2130	cm_id_priv->timewait_info->remote_qpn = cm_id_priv->remote_qpn;
2131
2132	/*
2133	* Note that the ID pointer is not in the xarray at this point,
2134	* so this set is only visible to the local thread.
2135	*/
2136	cm_id_priv->id.state = IB_CM_REQ_RCVD;
2137
2138	listen_cm_id_priv = cm_match_req(work, cm_id_priv);
2139	if (!listen_cm_id_priv) {
2140	trace_icm_no_listener_err(cm_id: &cm_id_priv->id);
2141	cm_id_priv->id.state = IB_CM_IDLE;
2142	ret = -EINVAL;
2143	goto destroy;
2144	}
2145
2146	memset(&work->path[0], 0, sizeof(work->path[0]));
2147	if (cm_req_has_alt_path(req_msg))
2148	memset(&work->path[1], 0, sizeof(work->path[1]));
2149	grh = rdma_ah_read_grh(attr: &cm_id_priv->av.ah_attr);
2150	gid_attr = grh->sgid_attr;
2151
2152	if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) {
2153	work->path[0].rec_type =
2154	sa_conv_gid_to_pathrec_type(type: gid_attr->gid_type);
2155	} else {
2156	cm_process_routed_req(req_msg, wc: work->mad_recv_wc->wc);
2157	cm_path_set_rec_type(
2158	ib_device: work->port->cm_dev->ib_device, port_num: work->port->port_num,
2159	path: &work->path[0],
2160	IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID,
2161	req_msg));
2162	}
2163	if (cm_req_has_alt_path(req_msg))
2164	work->path[1].rec_type = work->path[0].rec_type;
2165	cm_format_paths_from_req(req_msg, primary_path: &work->path[0],
2166	alt_path: &work->path[1], wc: work->mad_recv_wc->wc);
2167	if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
2168	sa_path_set_dmac(rec: &work->path[0],
2169	dmac: cm_id_priv->av.ah_attr.roce.dmac);
2170	work->path[0].hop_limit = grh->hop_limit;
2171
2172	/* This destroy call is needed to pair with cm_init_av_for_response */
2173	cm_destroy_av(av: &cm_id_priv->av);
2174	ret = cm_init_av_by_path(path: &work->path[0], sgid_attr: gid_attr, av: &cm_id_priv->av);
2175	if (ret) {
2176	int err;
2177
2178	err = rdma_query_gid(device: work->port->cm_dev->ib_device,
2179	port_num: work->port->port_num, index: 0,
2180	gid: &work->path[0].sgid);
2181	if (err)
2182	ib_send_cm_rej(cm_id: &cm_id_priv->id, reason: IB_CM_REJ_INVALID_GID,
2183	NULL, ari_length: 0, NULL, private_data_len: 0);
2184	else
2185	ib_send_cm_rej(cm_id: &cm_id_priv->id, reason: IB_CM_REJ_INVALID_GID,
2186	ari: &work->path[0].sgid,
2187	ari_length: sizeof(work->path[0].sgid),
2188	NULL, private_data_len: 0);
2189	goto rejected;
2190	}
2191	if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_IB)
2192	cm_id_priv->av.dlid_datapath =
2193	IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg);
2194
2195	if (cm_req_has_alt_path(req_msg)) {
2196	ret = cm_init_av_by_path(path: &work->path[1], NULL,
2197	av: &cm_id_priv->alt_av);
2198	if (ret) {
2199	ib_send_cm_rej(cm_id: &cm_id_priv->id,
2200	reason: IB_CM_REJ_INVALID_ALT_GID,
2201	ari: &work->path[0].sgid,
2202	ari_length: sizeof(work->path[0].sgid), NULL, private_data_len: 0);
2203	goto rejected;
2204	}
2205	}
2206
2207	cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
2208	cm_id_priv->id.context = listen_cm_id_priv->id.context;
2209	cm_format_req_event(work, cm_id_priv, listen_id: &listen_cm_id_priv->id);
2210
2211	/* Now MAD handlers can see the new ID */
2212	spin_lock_irq(lock: &cm_id_priv->lock);
2213	cm_finalize_id(cm_id_priv);
2214
2215	/* Refcount belongs to the event, pairs with cm_process_work() */
2216	refcount_inc(r: &cm_id_priv->refcount);
2217	cm_queue_work_unlock(cm_id_priv, work);
2218	/*
2219	* Since this ID was just created and was not made visible to other MAD
2220	* handlers until the cm_finalize_id() above we know that the
2221	* cm_process_work() will deliver the event and the listen_cm_id
2222	* embedded in the event can be derefed here.
2223	*/
2224	cm_deref_id(cm_id_priv: listen_cm_id_priv);
2225	return 0;
2226
2227	rejected:
2228	cm_deref_id(cm_id_priv: listen_cm_id_priv);
2229	destroy:
2230	ib_destroy_cm_id(&cm_id_priv->id);
2231	return ret;
2232	}
2233
2234	static void cm_format_rep(struct cm_rep_msg *rep_msg,
2235	struct cm_id_private *cm_id_priv,
2236	struct ib_cm_rep_param *param)
2237	{
2238	cm_format_mad_ece_hdr(hdr: &rep_msg->hdr, CM_REP_ATTR_ID, tid: cm_id_priv->tid,
2239	attr_mod: param->ece.attr_mod);
2240	IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,
2241	be32_to_cpu(cm_id_priv->id.local_id));
2242	IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,
2243	be32_to_cpu(cm_id_priv->id.remote_id));
2244	IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn);
2245	IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg,
2246	param->responder_resources);
2247	IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg,
2248	cm_id_priv->av.port->cm_dev->ack_delay);
2249	IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted);
2250	IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count);
2251	IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg,
2252	be64_to_cpu(cm_id_priv->id.device->node_guid));
2253
2254	if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2255	IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg,
2256	param->initiator_depth);
2257	IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg,
2258	param->flow_control);
2259	IBA_SET(CM_REP_SRQ, rep_msg, param->srq);
2260	IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num);
2261	} else {
2262	IBA_SET(CM_REP_SRQ, rep_msg, 1);
2263	IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num);
2264	}
2265
2266	IBA_SET(CM_REP_VENDOR_ID_L, rep_msg, param->ece.vendor_id);
2267	IBA_SET(CM_REP_VENDOR_ID_M, rep_msg, param->ece.vendor_id >> 8);
2268	IBA_SET(CM_REP_VENDOR_ID_H, rep_msg, param->ece.vendor_id >> 16);
2269
2270	if (param->private_data && param->private_data_len)
2271	IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data,
2272	param->private_data_len);
2273	}
2274
2275	int ib_send_cm_rep(struct ib_cm_id *cm_id,
2276	struct ib_cm_rep_param *param)
2277	{
2278	struct cm_id_private *cm_id_priv;
2279	struct ib_mad_send_buf *msg;
2280	struct cm_rep_msg *rep_msg;
2281	unsigned long flags;
2282	int ret;
2283
2284	if (param->private_data &&
2285	param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2286	return -EINVAL;
2287
2288	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2289	spin_lock_irqsave(&cm_id_priv->lock, flags);
2290	if (cm_id->state != IB_CM_REQ_RCVD &&
2291	cm_id->state != IB_CM_MRA_REQ_SENT) {
2292	trace_icm_send_rep_err(local_id: cm_id_priv->id.local_id, state: cm_id->state);
2293	ret = -EINVAL;
2294	goto out;
2295	}
2296
2297	msg = cm_alloc_priv_msg(cm_id_priv);
2298	if (IS_ERR(ptr: msg)) {
2299	ret = PTR_ERR(ptr: msg);
2300	goto out;
2301	}
2302
2303	rep_msg = (struct cm_rep_msg *) msg->mad;
2304	cm_format_rep(rep_msg, cm_id_priv, param);
2305	msg->timeout_ms = cm_id_priv->timeout_ms;
2306	msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
2307
2308	trace_icm_send_rep(cm_id);
2309	ret = ib_post_send_mad(send_buf: msg, NULL);
2310	if (ret)
2311	goto out_free;
2312
2313	cm_id->state = IB_CM_REP_SENT;
2314	cm_id_priv->initiator_depth = param->initiator_depth;
2315	cm_id_priv->responder_resources = param->responder_resources;
2316	cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2317	WARN_ONCE(param->qp_num & 0xFF000000,
2318	"IBTA declares QPN to be 24 bits, but it is 0x%X\n",
2319	param->qp_num);
2320	cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2321	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
2322	return 0;
2323
2324	out_free:
2325	cm_free_priv_msg(msg);
2326	out:
2327	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
2328	return ret;
2329	}
2330	EXPORT_SYMBOL(ib_send_cm_rep);
2331
2332	static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2333	struct cm_id_private *cm_id_priv,
2334	const void *private_data,
2335	u8 private_data_len)
2336	{
2337	cm_format_mad_hdr(hdr: &rtu_msg->hdr, CM_RTU_ATTR_ID, tid: cm_id_priv->tid);
2338	IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg,
2339	be32_to_cpu(cm_id_priv->id.local_id));
2340	IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg,
2341	be32_to_cpu(cm_id_priv->id.remote_id));
2342
2343	if (private_data && private_data_len)
2344	IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data,
2345	private_data_len);
2346	}
2347
2348	int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2349	const void *private_data,
2350	u8 private_data_len)
2351	{
2352	struct cm_id_private *cm_id_priv;
2353	struct ib_mad_send_buf *msg;
2354	unsigned long flags;
2355	void *data;
2356	int ret;
2357
2358	if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2359	return -EINVAL;
2360
2361	data = cm_copy_private_data(private_data, private_data_len);
2362	if (IS_ERR(ptr: data))
2363	return PTR_ERR(ptr: data);
2364
2365	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2366	spin_lock_irqsave(&cm_id_priv->lock, flags);
2367	if (cm_id->state != IB_CM_REP_RCVD &&
2368	cm_id->state != IB_CM_MRA_REP_SENT) {
2369	trace_icm_send_cm_rtu_err(cm_id);
2370	ret = -EINVAL;
2371	goto error;
2372	}
2373
2374	msg = cm_alloc_msg(cm_id_priv);
2375	if (IS_ERR(ptr: msg)) {
2376	ret = PTR_ERR(ptr: msg);
2377	goto error;
2378	}
2379
2380	cm_format_rtu(rtu_msg: (struct cm_rtu_msg *) msg->mad, cm_id_priv,
2381	private_data, private_data_len);
2382
2383	trace_icm_send_rtu(cm_id);
2384	ret = ib_post_send_mad(send_buf: msg, NULL);
2385	if (ret) {
2386	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
2387	cm_free_msg(msg);
2388	kfree(objp: data);
2389	return ret;
2390	}
2391
2392	cm_id->state = IB_CM_ESTABLISHED;
2393	cm_set_private_data(cm_id_priv, private_data: data, private_data_len);
2394	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
2395	return 0;
2396
2397	error: spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
2398	kfree(objp: data);
2399	return ret;
2400	}
2401	EXPORT_SYMBOL(ib_send_cm_rtu);
2402
2403	static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2404	{
2405	struct cm_rep_msg *rep_msg;
2406	struct ib_cm_rep_event_param *param;
2407
2408	rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2409	param = &work->cm_event.param.rep_rcvd;
2410	param->remote_ca_guid =
2411	cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2412	param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg);
2413	param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2414	param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg);
2415	param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2416	param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2417	param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2418	param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg);
2419	param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);
2420	param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2421	param->srq = IBA_GET(CM_REP_SRQ, rep_msg);
2422	param->ece.vendor_id = IBA_GET(CM_REP_VENDOR_ID_H, rep_msg) << 16;
2423	param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_M, rep_msg) << 8;
2424	param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_L, rep_msg);
2425	param->ece.attr_mod = be32_to_cpu(rep_msg->hdr.attr_mod);
2426
2427	work->cm_event.private_data =
2428	IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);
2429	}
2430
2431	static void cm_dup_rep_handler(struct cm_work *work)
2432	{
2433	struct cm_id_private *cm_id_priv;
2434	struct cm_rep_msg *rep_msg;
2435	struct ib_mad_send_buf *msg = NULL;
2436	int ret;
2437
2438	rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2439	cm_id_priv = cm_acquire_id(
2440	cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)),
2441	cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)));
2442	if (!cm_id_priv)
2443	return;
2444
2445	atomic_long_inc(
2446	v: &work->port->counters[CM_RECV_DUPLICATES][CM_REP_COUNTER]);
2447	ret = cm_alloc_response_msg(port: work->port, mad_recv_wc: work->mad_recv_wc, msg: &msg);
2448	if (ret)
2449	goto deref;
2450
2451	spin_lock_irq(lock: &cm_id_priv->lock);
2452	if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2453	cm_format_rtu(rtu_msg: (struct cm_rtu_msg *) msg->mad, cm_id_priv,
2454	private_data: cm_id_priv->private_data,
2455	private_data_len: cm_id_priv->private_data_len);
2456	else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2457	cm_format_mra(mra_msg: (struct cm_mra_msg *) msg->mad, cm_id_priv,
2458	msg_mraed: CM_MSG_RESPONSE_REP, service_timeout: cm_id_priv->service_timeout,
2459	private_data: cm_id_priv->private_data,
2460	private_data_len: cm_id_priv->private_data_len);
2461	else
2462	goto unlock;
2463	spin_unlock_irq(lock: &cm_id_priv->lock);
2464
2465	trace_icm_send_dup_rep(cm_id: &cm_id_priv->id);
2466	ret = ib_post_send_mad(send_buf: msg, NULL);
2467	if (ret)
2468	goto free;
2469	goto deref;
2470
2471	unlock: spin_unlock_irq(lock: &cm_id_priv->lock);
2472	free: cm_free_response_msg(msg);
2473	deref: cm_deref_id(cm_id_priv);
2474	}
2475
2476	static int cm_rep_handler(struct cm_work *work)
2477	{
2478	struct cm_id_private *cm_id_priv;
2479	struct cm_rep_msg *rep_msg;
2480	int ret;
2481	struct cm_id_private *cur_cm_id_priv;
2482	struct cm_timewait_info *timewait_info;
2483
2484	rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2485	cm_id_priv = cm_acquire_id(
2486	cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), remote_id: 0);
2487	if (!cm_id_priv) {
2488	cm_dup_rep_handler(work);
2489	trace_icm_remote_no_priv_err(
2490	IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2491	return -EINVAL;
2492	}
2493
2494	cm_format_rep_event(work, qp_type: cm_id_priv->qp_type);
2495
2496	spin_lock_irq(lock: &cm_id_priv->lock);
2497	switch (cm_id_priv->id.state) {
2498	case IB_CM_REQ_SENT:
2499	case IB_CM_MRA_REQ_RCVD:
2500	break;
2501	default:
2502	ret = -EINVAL;
2503	trace_icm_rep_unknown_err(
2504	IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2505	IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg),
2506	state: cm_id_priv->id.state);
2507	spin_unlock_irq(lock: &cm_id_priv->lock);
2508	goto error;
2509	}
2510
2511	cm_id_priv->timewait_info->work.remote_id =
2512	cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2513	cm_id_priv->timewait_info->remote_ca_guid =
2514	cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2515	cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, qp_type: cm_id_priv->qp_type);
2516
2517	spin_lock(lock: &cm.lock);
2518	/* Check for duplicate REP. */
2519	if (cm_insert_remote_id(timewait_info: cm_id_priv->timewait_info)) {
2520	spin_unlock(lock: &cm.lock);
2521	spin_unlock_irq(lock: &cm_id_priv->lock);
2522	ret = -EINVAL;
2523	trace_icm_insert_failed_err(
2524	IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2525	goto error;
2526	}
2527	/* Check for a stale connection. */
2528	timewait_info = cm_insert_remote_qpn(timewait_info: cm_id_priv->timewait_info);
2529	if (timewait_info) {
2530	cm_remove_remote(cm_id_priv);
2531	cur_cm_id_priv = cm_acquire_id(local_id: timewait_info->work.local_id,
2532	remote_id: timewait_info->work.remote_id);
2533
2534	spin_unlock(lock: &cm.lock);
2535	spin_unlock_irq(lock: &cm_id_priv->lock);
2536	cm_issue_rej(port: work->port, mad_recv_wc: work->mad_recv_wc,
2537	reason: IB_CM_REJ_STALE_CONN, msg_rejected: CM_MSG_RESPONSE_REP,
2538	NULL, ari_length: 0);
2539	ret = -EINVAL;
2540	trace_icm_staleconn_err(
2541	IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2542	IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2543
2544	if (cur_cm_id_priv) {
2545	ib_send_cm_dreq(cm_id: &cur_cm_id_priv->id, NULL, private_data_len: 0);
2546	cm_deref_id(cm_id_priv: cur_cm_id_priv);
2547	}
2548
2549	goto error;
2550	}
2551	spin_unlock(lock: &cm.lock);
2552
2553	cm_id_priv->id.state = IB_CM_REP_RCVD;
2554	cm_id_priv->id.remote_id =
2555	cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2556	cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, qp_type: cm_id_priv->qp_type);
2557	cm_id_priv->initiator_depth =
2558	IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2559	cm_id_priv->responder_resources =
2560	IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2561	cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2562	cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2563	cm_id_priv->target_ack_delay =
2564	IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2565	cm_id_priv->av.timeout =
2566	cm_ack_timeout(ca_ack_delay: cm_id_priv->target_ack_delay,
2567	packet_life_time: cm_id_priv->av.timeout - 1);
2568	cm_id_priv->alt_av.timeout =
2569	cm_ack_timeout(ca_ack_delay: cm_id_priv->target_ack_delay,
2570	packet_life_time: cm_id_priv->alt_av.timeout - 1);
2571
2572	ib_cancel_mad(send_buf: cm_id_priv->msg);
2573	cm_queue_work_unlock(cm_id_priv, work);
2574	return 0;
2575
2576	error:
2577	cm_deref_id(cm_id_priv);
2578	return ret;
2579	}
2580
2581	static int cm_establish_handler(struct cm_work *work)
2582	{
2583	struct cm_id_private *cm_id_priv;
2584
2585	/* See comment in cm_establish about lookup. */
2586	cm_id_priv = cm_acquire_id(local_id: work->local_id, remote_id: work->remote_id);
2587	if (!cm_id_priv)
2588	return -EINVAL;
2589
2590	spin_lock_irq(lock: &cm_id_priv->lock);
2591	if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2592	spin_unlock_irq(lock: &cm_id_priv->lock);
2593	goto out;
2594	}
2595
2596	ib_cancel_mad(send_buf: cm_id_priv->msg);
2597	cm_queue_work_unlock(cm_id_priv, work);
2598	return 0;
2599	out:
2600	cm_deref_id(cm_id_priv);
2601	return -EINVAL;
2602	}
2603
2604	static int cm_rtu_handler(struct cm_work *work)
2605	{
2606	struct cm_id_private *cm_id_priv;
2607	struct cm_rtu_msg *rtu_msg;
2608
2609	rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2610	cm_id_priv = cm_acquire_id(
2611	cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)),
2612	cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg)));
2613	if (!cm_id_priv)
2614	return -EINVAL;
2615
2616	work->cm_event.private_data =
2617	IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg);
2618
2619	spin_lock_irq(lock: &cm_id_priv->lock);
2620	if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2621	cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2622	spin_unlock_irq(lock: &cm_id_priv->lock);
2623	atomic_long_inc(v: &work->port->counters[CM_RECV_DUPLICATES]
2624	[CM_RTU_COUNTER]);
2625	goto out;
2626	}
2627	cm_id_priv->id.state = IB_CM_ESTABLISHED;
2628
2629	ib_cancel_mad(send_buf: cm_id_priv->msg);
2630	cm_queue_work_unlock(cm_id_priv, work);
2631	return 0;
2632	out:
2633	cm_deref_id(cm_id_priv);
2634	return -EINVAL;
2635	}
2636
2637	static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2638	struct cm_id_private *cm_id_priv,
2639	const void *private_data,
2640	u8 private_data_len)
2641	{
2642	cm_format_mad_hdr(hdr: &dreq_msg->hdr, CM_DREQ_ATTR_ID,
2643	tid: cm_form_tid(cm_id_priv));
2644	IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg,
2645	be32_to_cpu(cm_id_priv->id.local_id));
2646	IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg,
2647	be32_to_cpu(cm_id_priv->id.remote_id));
2648	IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg,
2649	be32_to_cpu(cm_id_priv->remote_qpn));
2650
2651	if (private_data && private_data_len)
2652	IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data,
2653	private_data_len);
2654	}
2655
2656	static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
2657	const void *private_data, u8 private_data_len)
2658	{
2659	struct ib_mad_send_buf *msg;
2660	int ret;
2661
2662	lockdep_assert_held(&cm_id_priv->lock);
2663
2664	if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2665	return -EINVAL;
2666
2667	if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2668	trace_icm_dreq_skipped(cm_id: &cm_id_priv->id);
2669	return -EINVAL;
2670	}
2671
2672	if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2673	cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2674	ib_cancel_mad(send_buf: cm_id_priv->msg);
2675
2676	msg = cm_alloc_priv_msg(cm_id_priv);
2677	if (IS_ERR(ptr: msg)) {
2678	cm_enter_timewait(cm_id_priv);
2679	return PTR_ERR(ptr: msg);
2680	}
2681
2682	cm_format_dreq(dreq_msg: (struct cm_dreq_msg *) msg->mad, cm_id_priv,
2683	private_data, private_data_len);
2684	msg->timeout_ms = cm_id_priv->timeout_ms;
2685	msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2686
2687	trace_icm_send_dreq(cm_id: &cm_id_priv->id);
2688	ret = ib_post_send_mad(send_buf: msg, NULL);
2689	if (ret) {
2690	cm_enter_timewait(cm_id_priv);
2691	cm_free_priv_msg(msg);
2692	return ret;
2693	}
2694
2695	cm_id_priv->id.state = IB_CM_DREQ_SENT;
2696	return 0;
2697	}
2698
2699	int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data,
2700	u8 private_data_len)
2701	{
2702	struct cm_id_private *cm_id_priv =
2703	container_of(cm_id, struct cm_id_private, id);
2704	unsigned long flags;
2705	int ret;
2706
2707	spin_lock_irqsave(&cm_id_priv->lock, flags);
2708	ret = cm_send_dreq_locked(cm_id_priv, private_data, private_data_len);
2709	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
2710	return ret;
2711	}
2712	EXPORT_SYMBOL(ib_send_cm_dreq);
2713
2714	static void cm_format_drep(struct cm_drep_msg *drep_msg,
2715	struct cm_id_private *cm_id_priv,
2716	const void *private_data,
2717	u8 private_data_len)
2718	{
2719	cm_format_mad_hdr(hdr: &drep_msg->hdr, CM_DREP_ATTR_ID, tid: cm_id_priv->tid);
2720	IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2721	be32_to_cpu(cm_id_priv->id.local_id));
2722	IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2723	be32_to_cpu(cm_id_priv->id.remote_id));
2724
2725	if (private_data && private_data_len)
2726	IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data,
2727	private_data_len);
2728	}
2729
2730	static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
2731	void *private_data, u8 private_data_len)
2732	{
2733	struct ib_mad_send_buf *msg;
2734	int ret;
2735
2736	lockdep_assert_held(&cm_id_priv->lock);
2737
2738	if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2739	return -EINVAL;
2740
2741	if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2742	trace_icm_send_drep_err(cm_id: &cm_id_priv->id);
2743	kfree(objp: private_data);
2744	return -EINVAL;
2745	}
2746
2747	cm_set_private_data(cm_id_priv, private_data, private_data_len);
2748	cm_enter_timewait(cm_id_priv);
2749
2750	msg = cm_alloc_msg(cm_id_priv);
2751	if (IS_ERR(ptr: msg))
2752	return PTR_ERR(ptr: msg);
2753
2754	cm_format_drep(drep_msg: (struct cm_drep_msg *) msg->mad, cm_id_priv,
2755	private_data, private_data_len);
2756
2757	trace_icm_send_drep(cm_id: &cm_id_priv->id);
2758	ret = ib_post_send_mad(send_buf: msg, NULL);
2759	if (ret) {
2760	cm_free_msg(msg);
2761	return ret;
2762	}
2763	return 0;
2764	}
2765
2766	int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data,
2767	u8 private_data_len)
2768	{
2769	struct cm_id_private *cm_id_priv =
2770	container_of(cm_id, struct cm_id_private, id);
2771	unsigned long flags;
2772	void *data;
2773	int ret;
2774
2775	data = cm_copy_private_data(private_data, private_data_len);
2776	if (IS_ERR(ptr: data))
2777	return PTR_ERR(ptr: data);
2778
2779	spin_lock_irqsave(&cm_id_priv->lock, flags);
2780	ret = cm_send_drep_locked(cm_id_priv, private_data: data, private_data_len);
2781	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
2782	return ret;
2783	}
2784	EXPORT_SYMBOL(ib_send_cm_drep);
2785
2786	static int cm_issue_drep(struct cm_port *port,
2787	struct ib_mad_recv_wc *mad_recv_wc)
2788	{
2789	struct ib_mad_send_buf *msg = NULL;
2790	struct cm_dreq_msg *dreq_msg;
2791	struct cm_drep_msg *drep_msg;
2792	int ret;
2793
2794	ret = cm_alloc_response_msg(port, mad_recv_wc, msg: &msg);
2795	if (ret)
2796	return ret;
2797
2798	dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2799	drep_msg = (struct cm_drep_msg *) msg->mad;
2800
2801	cm_format_mad_hdr(hdr: &drep_msg->hdr, CM_DREP_ATTR_ID, tid: dreq_msg->hdr.tid);
2802	IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2803	IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg));
2804	IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2805	IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2806
2807	trace_icm_issue_drep(
2808	IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2809	IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2810	ret = ib_post_send_mad(send_buf: msg, NULL);
2811	if (ret)
2812	cm_free_response_msg(msg);
2813
2814	return ret;
2815	}
2816
2817	static int cm_dreq_handler(struct cm_work *work)
2818	{
2819	struct cm_id_private *cm_id_priv;
2820	struct cm_dreq_msg *dreq_msg;
2821	struct ib_mad_send_buf *msg = NULL;
2822
2823	dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2824	cm_id_priv = cm_acquire_id(
2825	cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)),
2826	cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)));
2827	if (!cm_id_priv) {
2828	atomic_long_inc(v: &work->port->counters[CM_RECV_DUPLICATES]
2829	[CM_DREQ_COUNTER]);
2830	cm_issue_drep(port: work->port, mad_recv_wc: work->mad_recv_wc);
2831	trace_icm_no_priv_err(
2832	IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2833	IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2834	return -EINVAL;
2835	}
2836
2837	work->cm_event.private_data =
2838	IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg);
2839
2840	spin_lock_irq(lock: &cm_id_priv->lock);
2841	if (cm_id_priv->local_qpn !=
2842	cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg)))
2843	goto unlock;
2844
2845	switch (cm_id_priv->id.state) {
2846	case IB_CM_REP_SENT:
2847	case IB_CM_DREQ_SENT:
2848	case IB_CM_MRA_REP_RCVD:
2849	ib_cancel_mad(send_buf: cm_id_priv->msg);
2850	break;
2851	case IB_CM_ESTABLISHED:
2852	if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2853	cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2854	ib_cancel_mad(send_buf: cm_id_priv->msg);
2855	break;
2856	case IB_CM_TIMEWAIT:
2857	atomic_long_inc(v: &work->port->counters[CM_RECV_DUPLICATES]
2858	[CM_DREQ_COUNTER]);
2859	msg = cm_alloc_response_msg_no_ah(port: work->port, mad_recv_wc: work->mad_recv_wc);
2860	if (IS_ERR(ptr: msg))
2861	goto unlock;
2862
2863	cm_format_drep(drep_msg: (struct cm_drep_msg *) msg->mad, cm_id_priv,
2864	private_data: cm_id_priv->private_data,
2865	private_data_len: cm_id_priv->private_data_len);
2866	spin_unlock_irq(lock: &cm_id_priv->lock);
2867
2868	if (cm_create_response_msg_ah(port: work->port, mad_recv_wc: work->mad_recv_wc, msg) ||
2869	ib_post_send_mad(send_buf: msg, NULL))
2870	cm_free_response_msg(msg);
2871	goto deref;
2872	case IB_CM_DREQ_RCVD:
2873	atomic_long_inc(v: &work->port->counters[CM_RECV_DUPLICATES]
2874	[CM_DREQ_COUNTER]);
2875	goto unlock;
2876	default:
2877	trace_icm_dreq_unknown_err(cm_id: &cm_id_priv->id);
2878	goto unlock;
2879	}
2880	cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2881	cm_id_priv->tid = dreq_msg->hdr.tid;
2882	cm_queue_work_unlock(cm_id_priv, work);
2883	return 0;
2884
2885	unlock: spin_unlock_irq(lock: &cm_id_priv->lock);
2886	deref: cm_deref_id(cm_id_priv);
2887	return -EINVAL;
2888	}
2889
2890	static int cm_drep_handler(struct cm_work *work)
2891	{
2892	struct cm_id_private *cm_id_priv;
2893	struct cm_drep_msg *drep_msg;
2894
2895	drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2896	cm_id_priv = cm_acquire_id(
2897	cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)),
2898	cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg)));
2899	if (!cm_id_priv)
2900	return -EINVAL;
2901
2902	work->cm_event.private_data =
2903	IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg);
2904
2905	spin_lock_irq(lock: &cm_id_priv->lock);
2906	if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2907	cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2908	spin_unlock_irq(lock: &cm_id_priv->lock);
2909	goto out;
2910	}
2911	cm_enter_timewait(cm_id_priv);
2912
2913	ib_cancel_mad(send_buf: cm_id_priv->msg);
2914	cm_queue_work_unlock(cm_id_priv, work);
2915	return 0;
2916	out:
2917	cm_deref_id(cm_id_priv);
2918	return -EINVAL;
2919	}
2920
2921	static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
2922	enum ib_cm_rej_reason reason, void *ari,
2923	u8 ari_length, const void *private_data,
2924	u8 private_data_len)
2925	{
2926	enum ib_cm_state state = cm_id_priv->id.state;
2927	struct ib_mad_send_buf *msg;
2928	int ret;
2929
2930	lockdep_assert_held(&cm_id_priv->lock);
2931
2932	if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2933	(ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2934	return -EINVAL;
2935
2936	trace_icm_send_rej(cm_id: &cm_id_priv->id, reason);
2937
2938	switch (state) {
2939	case IB_CM_REQ_SENT:
2940	case IB_CM_MRA_REQ_RCVD:
2941	case IB_CM_REQ_RCVD:
2942	case IB_CM_MRA_REQ_SENT:
2943	case IB_CM_REP_RCVD:
2944	case IB_CM_MRA_REP_SENT:
2945	cm_reset_to_idle(cm_id_priv);
2946	msg = cm_alloc_msg(cm_id_priv);
2947	if (IS_ERR(ptr: msg))
2948	return PTR_ERR(ptr: msg);
2949	cm_format_rej(rej_msg: (struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2950	ari, ari_length, private_data, private_data_len,
2951	state);
2952	break;
2953	case IB_CM_REP_SENT:
2954	case IB_CM_MRA_REP_RCVD:
2955	cm_enter_timewait(cm_id_priv);
2956	msg = cm_alloc_msg(cm_id_priv);
2957	if (IS_ERR(ptr: msg))
2958	return PTR_ERR(ptr: msg);
2959	cm_format_rej(rej_msg: (struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2960	ari, ari_length, private_data, private_data_len,
2961	state);
2962	break;
2963	default:
2964	trace_icm_send_unknown_rej_err(cm_id: &cm_id_priv->id);
2965	return -EINVAL;
2966	}
2967
2968	ret = ib_post_send_mad(send_buf: msg, NULL);
2969	if (ret) {
2970	cm_free_msg(msg);
2971	return ret;
2972	}
2973
2974	return 0;
2975	}
2976
2977	int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason,
2978	void *ari, u8 ari_length, const void *private_data,
2979	u8 private_data_len)
2980	{
2981	struct cm_id_private *cm_id_priv =
2982	container_of(cm_id, struct cm_id_private, id);
2983	unsigned long flags;
2984	int ret;
2985
2986	spin_lock_irqsave(&cm_id_priv->lock, flags);
2987	ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length,
2988	private_data, private_data_len);
2989	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
2990	return ret;
2991	}
2992	EXPORT_SYMBOL(ib_send_cm_rej);
2993
2994	static void cm_format_rej_event(struct cm_work *work)
2995	{
2996	struct cm_rej_msg *rej_msg;
2997	struct ib_cm_rej_event_param *param;
2998
2999	rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3000	param = &work->cm_event.param.rej_rcvd;
3001	param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg);
3002	param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg);
3003	param->reason = IBA_GET(CM_REJ_REASON, rej_msg);
3004	work->cm_event.private_data =
3005	IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg);
3006	}
3007
3008	static struct cm_id_private *cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
3009	{
3010	struct cm_id_private *cm_id_priv;
3011	__be32 remote_id;
3012
3013	remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));
3014
3015	if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {
3016	cm_id_priv = cm_find_remote_id(
3017	remote_ca_guid: *((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),
3018	remote_id);
3019	} else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==
3020	CM_MSG_RESPONSE_REQ)
3021	cm_id_priv = cm_acquire_id(
3022	cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3023	remote_id: 0);
3024	else
3025	cm_id_priv = cm_acquire_id(
3026	cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3027	remote_id);
3028
3029	return cm_id_priv;
3030	}
3031
3032	static int cm_rej_handler(struct cm_work *work)
3033	{
3034	struct cm_id_private *cm_id_priv;
3035	struct cm_rej_msg *rej_msg;
3036
3037	rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3038	cm_id_priv = cm_acquire_rejected_id(rej_msg);
3039	if (!cm_id_priv)
3040	return -EINVAL;
3041
3042	cm_format_rej_event(work);
3043
3044	spin_lock_irq(lock: &cm_id_priv->lock);
3045	switch (cm_id_priv->id.state) {
3046	case IB_CM_REQ_SENT:
3047	case IB_CM_MRA_REQ_RCVD:
3048	case IB_CM_REP_SENT:
3049	case IB_CM_MRA_REP_RCVD:
3050	ib_cancel_mad(send_buf: cm_id_priv->msg);
3051	fallthrough;
3052	case IB_CM_REQ_RCVD:
3053	case IB_CM_MRA_REQ_SENT:
3054	if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN)
3055	cm_enter_timewait(cm_id_priv);
3056	else
3057	cm_reset_to_idle(cm_id_priv);
3058	break;
3059	case IB_CM_DREQ_SENT:
3060	ib_cancel_mad(send_buf: cm_id_priv->msg);
3061	fallthrough;
3062	case IB_CM_REP_RCVD:
3063	case IB_CM_MRA_REP_SENT:
3064	cm_enter_timewait(cm_id_priv);
3065	break;
3066	case IB_CM_ESTABLISHED:
3067	if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
3068	cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
3069	if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
3070	ib_cancel_mad(send_buf: cm_id_priv->msg);
3071	cm_enter_timewait(cm_id_priv);
3072	break;
3073	}
3074	fallthrough;
3075	default:
3076	trace_icm_rej_unknown_err(cm_id: &cm_id_priv->id);
3077	spin_unlock_irq(lock: &cm_id_priv->lock);
3078	goto out;
3079	}
3080
3081	cm_queue_work_unlock(cm_id_priv, work);
3082	return 0;
3083	out:
3084	cm_deref_id(cm_id_priv);
3085	return -EINVAL;
3086	}
3087
3088	int ib_send_cm_mra(struct ib_cm_id *cm_id,
3089	u8 service_timeout,
3090	const void *private_data,
3091	u8 private_data_len)
3092	{
3093	struct cm_id_private *cm_id_priv;
3094	struct ib_mad_send_buf *msg;
3095	enum ib_cm_state cm_state;
3096	enum ib_cm_lap_state lap_state;
3097	enum cm_msg_response msg_response;
3098	void *data;
3099	unsigned long flags;
3100	int ret;
3101
3102	if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
3103	return -EINVAL;
3104
3105	data = cm_copy_private_data(private_data, private_data_len);
3106	if (IS_ERR(ptr: data))
3107	return PTR_ERR(ptr: data);
3108
3109	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3110
3111	spin_lock_irqsave(&cm_id_priv->lock, flags);
3112	switch (cm_id_priv->id.state) {
3113	case IB_CM_REQ_RCVD:
3114	cm_state = IB_CM_MRA_REQ_SENT;
3115	lap_state = cm_id->lap_state;
3116	msg_response = CM_MSG_RESPONSE_REQ;
3117	break;
3118	case IB_CM_REP_RCVD:
3119	cm_state = IB_CM_MRA_REP_SENT;
3120	lap_state = cm_id->lap_state;
3121	msg_response = CM_MSG_RESPONSE_REP;
3122	break;
3123	case IB_CM_ESTABLISHED:
3124	if (cm_id->lap_state == IB_CM_LAP_RCVD) {
3125	cm_state = cm_id->state;
3126	lap_state = IB_CM_MRA_LAP_SENT;
3127	msg_response = CM_MSG_RESPONSE_OTHER;
3128	break;
3129	}
3130	fallthrough;
3131	default:
3132	trace_icm_send_mra_unknown_err(cm_id: &cm_id_priv->id);
3133	ret = -EINVAL;
3134	goto error_unlock;
3135	}
3136
3137	if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
3138	msg = cm_alloc_msg(cm_id_priv);
3139	if (IS_ERR(ptr: msg)) {
3140	ret = PTR_ERR(ptr: msg);
3141	goto error_unlock;
3142	}
3143
3144	cm_format_mra(mra_msg: (struct cm_mra_msg *) msg->mad, cm_id_priv,
3145	msg_mraed: msg_response, service_timeout,
3146	private_data, private_data_len);
3147	trace_icm_send_mra(cm_id);
3148	ret = ib_post_send_mad(send_buf: msg, NULL);
3149	if (ret)
3150	goto error_free_msg;
3151	}
3152
3153	cm_id->state = cm_state;
3154	cm_id->lap_state = lap_state;
3155	cm_id_priv->service_timeout = service_timeout;
3156	cm_set_private_data(cm_id_priv, private_data: data, private_data_len);
3157	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
3158	return 0;
3159
3160	error_free_msg:
3161	cm_free_msg(msg);
3162	error_unlock:
3163	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
3164	kfree(objp: data);
3165	return ret;
3166	}
3167	EXPORT_SYMBOL(ib_send_cm_mra);
3168
3169	static struct cm_id_private *cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
3170	{
3171	switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) {
3172	case CM_MSG_RESPONSE_REQ:
3173	return cm_acquire_id(
3174	cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3175	remote_id: 0);
3176	case CM_MSG_RESPONSE_REP:
3177	case CM_MSG_RESPONSE_OTHER:
3178	return cm_acquire_id(
3179	cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3180	cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg)));
3181	default:
3182	return NULL;
3183	}
3184	}
3185
3186	static int cm_mra_handler(struct cm_work *work)
3187	{
3188	struct cm_id_private *cm_id_priv;
3189	struct cm_mra_msg *mra_msg;
3190	int timeout;
3191
3192	mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
3193	cm_id_priv = cm_acquire_mraed_id(mra_msg);
3194	if (!cm_id_priv)
3195	return -EINVAL;
3196
3197	work->cm_event.private_data =
3198	IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg);
3199	work->cm_event.param.mra_rcvd.service_timeout =
3200	IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg);
3201	timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) +
3202	cm_convert_to_ms(iba_time: cm_id_priv->av.timeout);
3203
3204	spin_lock_irq(lock: &cm_id_priv->lock);
3205	switch (cm_id_priv->id.state) {
3206	case IB_CM_REQ_SENT:
3207	if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3208	CM_MSG_RESPONSE_REQ ||
3209	ib_modify_mad(send_buf: cm_id_priv->msg, timeout_ms: timeout))
3210	goto out;
3211	cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
3212	break;
3213	case IB_CM_REP_SENT:
3214	if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3215	CM_MSG_RESPONSE_REP ||
3216	ib_modify_mad(send_buf: cm_id_priv->msg, timeout_ms: timeout))
3217	goto out;
3218	cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3219	break;
3220	case IB_CM_ESTABLISHED:
3221	if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3222	CM_MSG_RESPONSE_OTHER ||
3223	cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3224	ib_modify_mad(send_buf: cm_id_priv->msg, timeout_ms: timeout)) {
3225	if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3226	atomic_long_inc(
3227	v: &work->port->counters[CM_RECV_DUPLICATES]
3228	[CM_MRA_COUNTER]);
3229	goto out;
3230	}
3231	cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3232	break;
3233	case IB_CM_MRA_REQ_RCVD:
3234	case IB_CM_MRA_REP_RCVD:
3235	atomic_long_inc(v: &work->port->counters[CM_RECV_DUPLICATES]
3236	[CM_MRA_COUNTER]);
3237	fallthrough;
3238	default:
3239	trace_icm_mra_unknown_err(cm_id: &cm_id_priv->id);
3240	goto out;
3241	}
3242
3243	cm_id_priv->msg->context[1] = (void *) (unsigned long)
3244	cm_id_priv->id.state;
3245	cm_queue_work_unlock(cm_id_priv, work);
3246	return 0;
3247	out:
3248	spin_unlock_irq(lock: &cm_id_priv->lock);
3249	cm_deref_id(cm_id_priv);
3250	return -EINVAL;
3251	}
3252
3253	static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3254	struct sa_path_rec *path)
3255	{
3256	u32 lid;
3257
3258	if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3259	sa_path_set_dlid(rec: path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID,
3260	lap_msg));
3261	sa_path_set_slid(rec: path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID,
3262	lap_msg));
3263	} else {
3264	lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3265	CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg));
3266	sa_path_set_dlid(rec: path, dlid: lid);
3267
3268	lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3269	CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg));
3270	sa_path_set_slid(rec: path, slid: lid);
3271	}
3272	}
3273
3274	static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3275	struct sa_path_rec *path,
3276	struct cm_lap_msg *lap_msg)
3277	{
3278	path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg);
3279	path->sgid =
3280	*IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg);
3281	path->flow_label =
3282	cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg));
3283	path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg);
3284	path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg);
3285	path->reversible = 1;
3286	path->pkey = cm_id_priv->pkey;
3287	path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg);
3288	path->mtu_selector = IB_SA_EQ;
3289	path->mtu = cm_id_priv->path_mtu;
3290	path->rate_selector = IB_SA_EQ;
3291	path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg);
3292	path->packet_life_time_selector = IB_SA_EQ;
3293	path->packet_life_time =
3294	IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg);
3295	path->packet_life_time -= (path->packet_life_time > 0);
3296	cm_format_path_lid_from_lap(lap_msg, path);
3297	}
3298
3299	static int cm_lap_handler(struct cm_work *work)
3300	{
3301	struct cm_id_private *cm_id_priv;
3302	struct cm_lap_msg *lap_msg;
3303	struct ib_cm_lap_event_param *param;
3304	struct ib_mad_send_buf *msg = NULL;
3305	struct rdma_ah_attr ah_attr;
3306	struct cm_av alt_av = {};
3307	int ret;
3308
3309	/* Currently Alternate path messages are not supported for
3310	* RoCE link layer.
3311	*/
3312	if (rdma_protocol_roce(device: work->port->cm_dev->ib_device,
3313	port_num: work->port->port_num))
3314	return -EINVAL;
3315
3316	/* todo: verify LAP request and send reject APR if invalid. */
3317	lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3318	cm_id_priv = cm_acquire_id(
3319	cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)),
3320	cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg)));
3321	if (!cm_id_priv)
3322	return -EINVAL;
3323
3324	param = &work->cm_event.param.lap_rcvd;
3325	memset(&work->path[0], 0, sizeof(work->path[1]));
3326	cm_path_set_rec_type(ib_device: work->port->cm_dev->ib_device,
3327	port_num: work->port->port_num, path: &work->path[0],
3328	IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID,
3329	lap_msg));
3330	param->alternate_path = &work->path[0];
3331	cm_format_path_from_lap(cm_id_priv, path: param->alternate_path, lap_msg);
3332	work->cm_event.private_data =
3333	IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg);
3334
3335	ret = ib_init_ah_attr_from_wc(device: work->port->cm_dev->ib_device,
3336	port_num: work->port->port_num,
3337	wc: work->mad_recv_wc->wc,
3338	grh: work->mad_recv_wc->recv_buf.grh,
3339	ah_attr: &ah_attr);
3340	if (ret)
3341	goto deref;
3342
3343	ret = cm_init_av_by_path(path: param->alternate_path, NULL, av: &alt_av);
3344	if (ret) {
3345	rdma_destroy_ah_attr(ah_attr: &ah_attr);
3346	goto deref;
3347	}
3348
3349	spin_lock_irq(lock: &cm_id_priv->lock);
3350	cm_init_av_for_lap(port: work->port, wc: work->mad_recv_wc->wc,
3351	ah_attr: &ah_attr, av: &cm_id_priv->av);
3352	cm_move_av_from_path(dest: &cm_id_priv->alt_av, src: &alt_av);
3353
3354	if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3355	goto unlock;
3356
3357	switch (cm_id_priv->id.lap_state) {
3358	case IB_CM_LAP_UNINIT:
3359	case IB_CM_LAP_IDLE:
3360	break;
3361	case IB_CM_MRA_LAP_SENT:
3362	atomic_long_inc(v: &work->port->counters[CM_RECV_DUPLICATES]
3363	[CM_LAP_COUNTER]);
3364	msg = cm_alloc_response_msg_no_ah(port: work->port, mad_recv_wc: work->mad_recv_wc);
3365	if (IS_ERR(ptr: msg))
3366	goto unlock;
3367
3368	cm_format_mra(mra_msg: (struct cm_mra_msg *) msg->mad, cm_id_priv,
3369	msg_mraed: CM_MSG_RESPONSE_OTHER,
3370	service_timeout: cm_id_priv->service_timeout,
3371	private_data: cm_id_priv->private_data,
3372	private_data_len: cm_id_priv->private_data_len);
3373	spin_unlock_irq(lock: &cm_id_priv->lock);
3374
3375	if (cm_create_response_msg_ah(port: work->port, mad_recv_wc: work->mad_recv_wc, msg) ||
3376	ib_post_send_mad(send_buf: msg, NULL))
3377	cm_free_response_msg(msg);
3378	goto deref;
3379	case IB_CM_LAP_RCVD:
3380	atomic_long_inc(v: &work->port->counters[CM_RECV_DUPLICATES]
3381	[CM_LAP_COUNTER]);
3382	goto unlock;
3383	default:
3384	goto unlock;
3385	}
3386
3387	cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3388	cm_id_priv->tid = lap_msg->hdr.tid;
3389	cm_queue_work_unlock(cm_id_priv, work);
3390	return 0;
3391
3392	unlock: spin_unlock_irq(lock: &cm_id_priv->lock);
3393	deref: cm_deref_id(cm_id_priv);
3394	return -EINVAL;
3395	}
3396
3397	static int cm_apr_handler(struct cm_work *work)
3398	{
3399	struct cm_id_private *cm_id_priv;
3400	struct cm_apr_msg *apr_msg;
3401
3402	/* Currently Alternate path messages are not supported for
3403	* RoCE link layer.
3404	*/
3405	if (rdma_protocol_roce(device: work->port->cm_dev->ib_device,
3406	port_num: work->port->port_num))
3407	return -EINVAL;
3408
3409	apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3410	cm_id_priv = cm_acquire_id(
3411	cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)),
3412	cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg)));
3413	if (!cm_id_priv)
3414	return -EINVAL; /* Unmatched reply. */
3415
3416	work->cm_event.param.apr_rcvd.ap_status =
3417	IBA_GET(CM_APR_AR_STATUS, apr_msg);
3418	work->cm_event.param.apr_rcvd.apr_info =
3419	IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg);
3420	work->cm_event.param.apr_rcvd.info_len =
3421	IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg);
3422	work->cm_event.private_data =
3423	IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg);
3424
3425	spin_lock_irq(lock: &cm_id_priv->lock);
3426	if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3427	(cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3428	cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3429	spin_unlock_irq(lock: &cm_id_priv->lock);
3430	goto out;
3431	}
3432	cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3433	ib_cancel_mad(send_buf: cm_id_priv->msg);
3434	cm_queue_work_unlock(cm_id_priv, work);
3435	return 0;
3436	out:
3437	cm_deref_id(cm_id_priv);
3438	return -EINVAL;
3439	}
3440
3441	static int cm_timewait_handler(struct cm_work *work)
3442	{
3443	struct cm_timewait_info *timewait_info;
3444	struct cm_id_private *cm_id_priv;
3445
3446	timewait_info = container_of(work, struct cm_timewait_info, work);
3447	spin_lock_irq(lock: &cm.lock);
3448	list_del(entry: &timewait_info->list);
3449	spin_unlock_irq(lock: &cm.lock);
3450
3451	cm_id_priv = cm_acquire_id(local_id: timewait_info->work.local_id,
3452	remote_id: timewait_info->work.remote_id);
3453	if (!cm_id_priv)
3454	return -EINVAL;
3455
3456	spin_lock_irq(lock: &cm_id_priv->lock);
3457	if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3458	cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3459	spin_unlock_irq(lock: &cm_id_priv->lock);
3460	goto out;
3461	}
3462	cm_id_priv->id.state = IB_CM_IDLE;
3463	cm_queue_work_unlock(cm_id_priv, work);
3464	return 0;
3465	out:
3466	cm_deref_id(cm_id_priv);
3467	return -EINVAL;
3468	}
3469
3470	static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3471	struct cm_id_private *cm_id_priv,
3472	struct ib_cm_sidr_req_param *param)
3473	{
3474	cm_format_mad_hdr(hdr: &sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3475	tid: cm_form_tid(cm_id_priv));
3476	IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg,
3477	be32_to_cpu(cm_id_priv->id.local_id));
3478	IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg,
3479	be16_to_cpu(param->path->pkey));
3480	IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg,
3481	be64_to_cpu(param->service_id));
3482
3483	if (param->private_data && param->private_data_len)
3484	IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg,
3485	param->private_data, param->private_data_len);
3486	}
3487
3488	int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3489	struct ib_cm_sidr_req_param *param)
3490	{
3491	struct cm_id_private *cm_id_priv;
3492	struct ib_mad_send_buf *msg;
3493	struct cm_av av = {};
3494	unsigned long flags;
3495	int ret;
3496
3497	if (!param->path || (param->private_data &&
3498	param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3499	return -EINVAL;
3500
3501	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3502	ret = cm_init_av_by_path(path: param->path, sgid_attr: param->sgid_attr, av: &av);
3503	if (ret)
3504	return ret;
3505
3506	spin_lock_irqsave(&cm_id_priv->lock, flags);
3507	cm_move_av_from_path(dest: &cm_id_priv->av, src: &av);
3508	cm_id->service_id = param->service_id;
3509	cm_id_priv->timeout_ms = param->timeout_ms;
3510	cm_id_priv->max_cm_retries = param->max_cm_retries;
3511	if (cm_id->state != IB_CM_IDLE) {
3512	ret = -EINVAL;
3513	goto out_unlock;
3514	}
3515
3516	msg = cm_alloc_priv_msg(cm_id_priv);
3517	if (IS_ERR(ptr: msg)) {
3518	ret = PTR_ERR(ptr: msg);
3519	goto out_unlock;
3520	}
3521
3522	cm_format_sidr_req(sidr_req_msg: (struct cm_sidr_req_msg *)msg->mad, cm_id_priv,
3523	param);
3524	msg->timeout_ms = cm_id_priv->timeout_ms;
3525	msg->context[1] = (void *)(unsigned long)IB_CM_SIDR_REQ_SENT;
3526
3527	trace_icm_send_sidr_req(cm_id: &cm_id_priv->id);
3528	ret = ib_post_send_mad(send_buf: msg, NULL);
3529	if (ret)
3530	goto out_free;
3531	cm_id->state = IB_CM_SIDR_REQ_SENT;
3532	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
3533	return 0;
3534	out_free:
3535	cm_free_priv_msg(msg);
3536	out_unlock:
3537	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
3538	return ret;
3539	}
3540	EXPORT_SYMBOL(ib_send_cm_sidr_req);
3541
3542	static void cm_format_sidr_req_event(struct cm_work *work,
3543	const struct cm_id_private *rx_cm_id,
3544	struct ib_cm_id *listen_id)
3545	{
3546	struct cm_sidr_req_msg *sidr_req_msg;
3547	struct ib_cm_sidr_req_event_param *param;
3548
3549	sidr_req_msg = (struct cm_sidr_req_msg *)
3550	work->mad_recv_wc->recv_buf.mad;
3551	param = &work->cm_event.param.sidr_req_rcvd;
3552	param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg);
3553	param->listen_id = listen_id;
3554	param->service_id =
3555	cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3556	param->bth_pkey = cm_get_bth_pkey(work);
3557	param->port = work->port->port_num;
3558	param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr;
3559	work->cm_event.private_data =
3560	IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg);
3561	}
3562
3563	static int cm_sidr_req_handler(struct cm_work *work)
3564	{
3565	struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
3566	struct cm_sidr_req_msg *sidr_req_msg;
3567	struct ib_wc *wc;
3568	int ret;
3569
3570	cm_id_priv =
3571	cm_alloc_id_priv(device: work->port->cm_dev->ib_device, NULL, NULL);
3572	if (IS_ERR(ptr: cm_id_priv))
3573	return PTR_ERR(ptr: cm_id_priv);
3574
3575	/* Record SGID/SLID and request ID for lookup. */
3576	sidr_req_msg = (struct cm_sidr_req_msg *)
3577	work->mad_recv_wc->recv_buf.mad;
3578
3579	cm_id_priv->id.remote_id =
3580	cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg));
3581	cm_id_priv->id.service_id =
3582	cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3583	cm_id_priv->tid = sidr_req_msg->hdr.tid;
3584
3585	wc = work->mad_recv_wc->wc;
3586	cm_id_priv->sidr_slid = wc->slid;
3587	ret = cm_init_av_for_response(port: work->port, wc: work->mad_recv_wc->wc,
3588	grh: work->mad_recv_wc->recv_buf.grh,
3589	av: &cm_id_priv->av);
3590	if (ret)
3591	goto out;
3592
3593	spin_lock_irq(lock: &cm.lock);
3594	listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3595	if (listen_cm_id_priv) {
3596	spin_unlock_irq(lock: &cm.lock);
3597	atomic_long_inc(v: &work->port->counters[CM_RECV_DUPLICATES]
3598	[CM_SIDR_REQ_COUNTER]);
3599	goto out; /* Duplicate message. */
3600	}
3601	cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3602	listen_cm_id_priv = cm_find_listen(device: cm_id_priv->id.device,
3603	service_id: cm_id_priv->id.service_id);
3604	if (!listen_cm_id_priv) {
3605	spin_unlock_irq(lock: &cm.lock);
3606	ib_send_cm_sidr_rep(cm_id: &cm_id_priv->id,
3607	param: &(struct ib_cm_sidr_rep_param){
3608	.status = IB_SIDR_UNSUPPORTED });
3609	goto out; /* No match. */
3610	}
3611	spin_unlock_irq(lock: &cm.lock);
3612
3613	cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
3614	cm_id_priv->id.context = listen_cm_id_priv->id.context;
3615
3616	/*
3617	* A SIDR ID does not need to be in the xarray since it does not receive
3618	* mads, is not placed in the remote_id or remote_qpn rbtree, and does
3619	* not enter timewait.
3620	*/
3621
3622	cm_format_sidr_req_event(work, rx_cm_id: cm_id_priv, listen_id: &listen_cm_id_priv->id);
3623	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
3624	cm_free_work(work);
3625	/*
3626	* A pointer to the listen_cm_id is held in the event, so this deref
3627	* must be after the event is delivered above.
3628	*/
3629	cm_deref_id(cm_id_priv: listen_cm_id_priv);
3630	if (ret)
3631	cm_destroy_id(cm_id: &cm_id_priv->id, err: ret);
3632	return 0;
3633	out:
3634	ib_destroy_cm_id(&cm_id_priv->id);
3635	return -EINVAL;
3636	}
3637
3638	static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3639	struct cm_id_private *cm_id_priv,
3640	struct ib_cm_sidr_rep_param *param)
3641	{
3642	cm_format_mad_ece_hdr(hdr: &sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3643	tid: cm_id_priv->tid, attr_mod: param->ece.attr_mod);
3644	IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,
3645	be32_to_cpu(cm_id_priv->id.remote_id));
3646	IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status);
3647	IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num);
3648	IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,
3649	be64_to_cpu(cm_id_priv->id.service_id));
3650	IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey);
3651	IBA_SET(CM_SIDR_REP_VENDOR_ID_L, sidr_rep_msg,
3652	param->ece.vendor_id & 0xFF);
3653	IBA_SET(CM_SIDR_REP_VENDOR_ID_H, sidr_rep_msg,
3654	(param->ece.vendor_id >> 8) & 0xFF);
3655
3656	if (param->info && param->info_length)
3657	IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,
3658	param->info, param->info_length);
3659
3660	if (param->private_data && param->private_data_len)
3661	IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg,
3662	param->private_data, param->private_data_len);
3663	}
3664
3665	static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
3666	struct ib_cm_sidr_rep_param *param)
3667	{
3668	struct ib_mad_send_buf *msg;
3669	unsigned long flags;
3670	int ret;
3671
3672	lockdep_assert_held(&cm_id_priv->lock);
3673
3674	if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3675	(param->private_data &&
3676	param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3677	return -EINVAL;
3678
3679	if (cm_id_priv->id.state != IB_CM_SIDR_REQ_RCVD)
3680	return -EINVAL;
3681
3682	msg = cm_alloc_msg(cm_id_priv);
3683	if (IS_ERR(ptr: msg))
3684	return PTR_ERR(ptr: msg);
3685
3686	cm_format_sidr_rep(sidr_rep_msg: (struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3687	param);
3688	trace_icm_send_sidr_rep(cm_id: &cm_id_priv->id);
3689	ret = ib_post_send_mad(send_buf: msg, NULL);
3690	if (ret) {
3691	cm_free_msg(msg);
3692	return ret;
3693	}
3694	cm_id_priv->id.state = IB_CM_IDLE;
3695	spin_lock_irqsave(&cm.lock, flags);
3696	if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3697	rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3698	RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3699	}
3700	spin_unlock_irqrestore(lock: &cm.lock, flags);
3701	return 0;
3702	}
3703
3704	int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3705	struct ib_cm_sidr_rep_param *param)
3706	{
3707	struct cm_id_private *cm_id_priv =
3708	container_of(cm_id, struct cm_id_private, id);
3709	unsigned long flags;
3710	int ret;
3711
3712	spin_lock_irqsave(&cm_id_priv->lock, flags);
3713	ret = cm_send_sidr_rep_locked(cm_id_priv, param);
3714	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
3715	return ret;
3716	}
3717	EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3718
3719	static void cm_format_sidr_rep_event(struct cm_work *work,
3720	const struct cm_id_private *cm_id_priv)
3721	{
3722	struct cm_sidr_rep_msg *sidr_rep_msg;
3723	struct ib_cm_sidr_rep_event_param *param;
3724
3725	sidr_rep_msg = (struct cm_sidr_rep_msg *)
3726	work->mad_recv_wc->recv_buf.mad;
3727	param = &work->cm_event.param.sidr_rep_rcvd;
3728	param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg);
3729	param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg);
3730	param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg);
3731	param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION,
3732	sidr_rep_msg);
3733	param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH,
3734	sidr_rep_msg);
3735	param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
3736	work->cm_event.private_data =
3737	IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg);
3738	}
3739
3740	static int cm_sidr_rep_handler(struct cm_work *work)
3741	{
3742	struct cm_sidr_rep_msg *sidr_rep_msg;
3743	struct cm_id_private *cm_id_priv;
3744
3745	sidr_rep_msg = (struct cm_sidr_rep_msg *)
3746	work->mad_recv_wc->recv_buf.mad;
3747	cm_id_priv = cm_acquire_id(
3748	cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), remote_id: 0);
3749	if (!cm_id_priv)
3750	return -EINVAL; /* Unmatched reply. */
3751
3752	spin_lock_irq(lock: &cm_id_priv->lock);
3753	if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3754	spin_unlock_irq(lock: &cm_id_priv->lock);
3755	goto out;
3756	}
3757	cm_id_priv->id.state = IB_CM_IDLE;
3758	ib_cancel_mad(send_buf: cm_id_priv->msg);
3759	spin_unlock_irq(lock: &cm_id_priv->lock);
3760
3761	cm_format_sidr_rep_event(work, cm_id_priv);
3762	cm_process_work(cm_id_priv, work);
3763	return 0;
3764	out:
3765	cm_deref_id(cm_id_priv);
3766	return -EINVAL;
3767	}
3768
3769	static void cm_process_send_error(struct cm_id_private *cm_id_priv,
3770	struct ib_mad_send_buf *msg,
3771	enum ib_cm_state state,
3772	enum ib_wc_status wc_status)
3773	{
3774	struct ib_cm_event cm_event = {};
3775	int ret;
3776
3777	/* Discard old sends or ones without a response. */
3778	spin_lock_irq(lock: &cm_id_priv->lock);
3779	if (msg != cm_id_priv->msg) {
3780	spin_unlock_irq(lock: &cm_id_priv->lock);
3781	cm_free_msg(msg);
3782	return;
3783	}
3784	cm_free_priv_msg(msg);
3785
3786	if (state != cm_id_priv->id.state || wc_status == IB_WC_SUCCESS ||
3787	wc_status == IB_WC_WR_FLUSH_ERR)
3788	goto out_unlock;
3789
3790	trace_icm_mad_send_err(state, wc_status);
3791	switch (state) {
3792	case IB_CM_REQ_SENT:
3793	case IB_CM_MRA_REQ_RCVD:
3794	cm_reset_to_idle(cm_id_priv);
3795	cm_event.event = IB_CM_REQ_ERROR;
3796	break;
3797	case IB_CM_REP_SENT:
3798	case IB_CM_MRA_REP_RCVD:
3799	cm_reset_to_idle(cm_id_priv);
3800	cm_event.event = IB_CM_REP_ERROR;
3801	break;
3802	case IB_CM_DREQ_SENT:
3803	cm_enter_timewait(cm_id_priv);
3804	cm_event.event = IB_CM_DREQ_ERROR;
3805	break;
3806	case IB_CM_SIDR_REQ_SENT:
3807	cm_id_priv->id.state = IB_CM_IDLE;
3808	cm_event.event = IB_CM_SIDR_REQ_ERROR;
3809	break;
3810	default:
3811	goto out_unlock;
3812	}
3813	spin_unlock_irq(lock: &cm_id_priv->lock);
3814	cm_event.param.send_status = wc_status;
3815
3816	/* No other events can occur on the cm_id at this point. */
3817	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3818	if (ret)
3819	ib_destroy_cm_id(&cm_id_priv->id);
3820	return;
3821	out_unlock:
3822	spin_unlock_irq(lock: &cm_id_priv->lock);
3823	}
3824
3825	static void cm_send_handler(struct ib_mad_agent *mad_agent,
3826	struct ib_mad_send_wc *mad_send_wc)
3827	{
3828	struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3829	struct cm_id_private *cm_id_priv = msg->context[0];
3830	enum ib_cm_state state =
3831	(enum ib_cm_state)(unsigned long)msg->context[1];
3832	struct cm_port *port;
3833	u16 attr_index;
3834
3835	port = mad_agent->context;
3836	attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3837	msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3838
3839	/*
3840	* If the send was in response to a received message (context[0] is not
3841	* set to a cm_id), and is not a REJ, then it is a send that was
3842	* manually retried.
3843	*/
3844	if (!cm_id_priv && (attr_index != CM_REJ_COUNTER))
3845	msg->retries = 1;
3846
3847	atomic_long_add(i: 1 + msg->retries, v: &port->counters[CM_XMIT][attr_index]);
3848	if (msg->retries)
3849	atomic_long_add(i: msg->retries,
3850	v: &port->counters[CM_XMIT_RETRIES][attr_index]);
3851
3852	if (cm_id_priv)
3853	cm_process_send_error(cm_id_priv, msg, state,
3854	wc_status: mad_send_wc->status);
3855	else
3856	cm_free_response_msg(msg);
3857	}
3858
3859	static void cm_work_handler(struct work_struct *_work)
3860	{
3861	struct cm_work *work = container_of(_work, struct cm_work, work.work);
3862	int ret;
3863
3864	switch (work->cm_event.event) {
3865	case IB_CM_REQ_RECEIVED:
3866	ret = cm_req_handler(work);
3867	break;
3868	case IB_CM_MRA_RECEIVED:
3869	ret = cm_mra_handler(work);
3870	break;
3871	case IB_CM_REJ_RECEIVED:
3872	ret = cm_rej_handler(work);
3873	break;
3874	case IB_CM_REP_RECEIVED:
3875	ret = cm_rep_handler(work);
3876	break;
3877	case IB_CM_RTU_RECEIVED:
3878	ret = cm_rtu_handler(work);
3879	break;
3880	case IB_CM_USER_ESTABLISHED:
3881	ret = cm_establish_handler(work);
3882	break;
3883	case IB_CM_DREQ_RECEIVED:
3884	ret = cm_dreq_handler(work);
3885	break;
3886	case IB_CM_DREP_RECEIVED:
3887	ret = cm_drep_handler(work);
3888	break;
3889	case IB_CM_SIDR_REQ_RECEIVED:
3890	ret = cm_sidr_req_handler(work);
3891	break;
3892	case IB_CM_SIDR_REP_RECEIVED:
3893	ret = cm_sidr_rep_handler(work);
3894	break;
3895	case IB_CM_LAP_RECEIVED:
3896	ret = cm_lap_handler(work);
3897	break;
3898	case IB_CM_APR_RECEIVED:
3899	ret = cm_apr_handler(work);
3900	break;
3901	case IB_CM_TIMEWAIT_EXIT:
3902	ret = cm_timewait_handler(work);
3903	break;
3904	default:
3905	trace_icm_handler_err(event: work->cm_event.event);
3906	ret = -EINVAL;
3907	break;
3908	}
3909	if (ret)
3910	cm_free_work(work);
3911	}
3912
3913	static int cm_establish(struct ib_cm_id *cm_id)
3914	{
3915	struct cm_id_private *cm_id_priv;
3916	struct cm_work *work;
3917	unsigned long flags;
3918	int ret = 0;
3919	struct cm_device *cm_dev;
3920
3921	cm_dev = ib_get_client_data(device: cm_id->device, client: &cm_client);
3922	if (!cm_dev)
3923	return -ENODEV;
3924
3925	work = kmalloc(size: sizeof *work, GFP_ATOMIC);
3926	if (!work)
3927	return -ENOMEM;
3928
3929	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3930	spin_lock_irqsave(&cm_id_priv->lock, flags);
3931	switch (cm_id->state) {
3932	case IB_CM_REP_SENT:
3933	case IB_CM_MRA_REP_RCVD:
3934	cm_id->state = IB_CM_ESTABLISHED;
3935	break;
3936	case IB_CM_ESTABLISHED:
3937	ret = -EISCONN;
3938	break;
3939	default:
3940	trace_icm_establish_err(cm_id);
3941	ret = -EINVAL;
3942	break;
3943	}
3944	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
3945
3946	if (ret) {
3947	kfree(objp: work);
3948	goto out;
3949	}
3950
3951	/*
3952	* The CM worker thread may try to destroy the cm_id before it
3953	* can execute this work item. To prevent potential deadlock,
3954	* we need to find the cm_id once we're in the context of the
3955	* worker thread, rather than holding a reference on it.
3956	*/
3957	INIT_DELAYED_WORK(&work->work, cm_work_handler);
3958	work->local_id = cm_id->local_id;
3959	work->remote_id = cm_id->remote_id;
3960	work->mad_recv_wc = NULL;
3961	work->cm_event.event = IB_CM_USER_ESTABLISHED;
3962
3963	/* Check if the device started its remove_one */
3964	spin_lock_irqsave(&cm.lock, flags);
3965	if (!cm_dev->going_down) {
3966	queue_delayed_work(wq: cm.wq, dwork: &work->work, delay: 0);
3967	} else {
3968	kfree(objp: work);
3969	ret = -ENODEV;
3970	}
3971	spin_unlock_irqrestore(lock: &cm.lock, flags);
3972
3973	out:
3974	return ret;
3975	}
3976
3977	static int cm_migrate(struct ib_cm_id *cm_id)
3978	{
3979	struct cm_id_private *cm_id_priv;
3980	unsigned long flags;
3981	int ret = 0;
3982
3983	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3984	spin_lock_irqsave(&cm_id_priv->lock, flags);
3985	if (cm_id->state == IB_CM_ESTABLISHED &&
3986	(cm_id->lap_state == IB_CM_LAP_UNINIT ||
3987	cm_id->lap_state == IB_CM_LAP_IDLE)) {
3988	cm_id->lap_state = IB_CM_LAP_IDLE;
3989	cm_id_priv->av = cm_id_priv->alt_av;
3990	} else
3991	ret = -EINVAL;
3992	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
3993
3994	return ret;
3995	}
3996
3997	int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3998	{
3999	int ret;
4000
4001	switch (event) {
4002	case IB_EVENT_COMM_EST:
4003	ret = cm_establish(cm_id);
4004	break;
4005	case IB_EVENT_PATH_MIG:
4006	ret = cm_migrate(cm_id);
4007	break;
4008	default:
4009	ret = -EINVAL;
4010	}
4011	return ret;
4012	}
4013	EXPORT_SYMBOL(ib_cm_notify);
4014
4015	static void cm_recv_handler(struct ib_mad_agent *mad_agent,
4016	struct ib_mad_send_buf *send_buf,
4017	struct ib_mad_recv_wc *mad_recv_wc)
4018	{
4019	struct cm_port *port = mad_agent->context;
4020	struct cm_work *work;
4021	enum ib_cm_event_type event;
4022	bool alt_path = false;
4023	u16 attr_id;
4024	int paths = 0;
4025	int going_down = 0;
4026
4027	switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
4028	case CM_REQ_ATTR_ID:
4029	alt_path = cm_req_has_alt_path(req_msg: (struct cm_req_msg *)
4030	mad_recv_wc->recv_buf.mad);
4031	paths = 1 + (alt_path != 0);
4032	event = IB_CM_REQ_RECEIVED;
4033	break;
4034	case CM_MRA_ATTR_ID:
4035	event = IB_CM_MRA_RECEIVED;
4036	break;
4037	case CM_REJ_ATTR_ID:
4038	event = IB_CM_REJ_RECEIVED;
4039	break;
4040	case CM_REP_ATTR_ID:
4041	event = IB_CM_REP_RECEIVED;
4042	break;
4043	case CM_RTU_ATTR_ID:
4044	event = IB_CM_RTU_RECEIVED;
4045	break;
4046	case CM_DREQ_ATTR_ID:
4047	event = IB_CM_DREQ_RECEIVED;
4048	break;
4049	case CM_DREP_ATTR_ID:
4050	event = IB_CM_DREP_RECEIVED;
4051	break;
4052	case CM_SIDR_REQ_ATTR_ID:
4053	event = IB_CM_SIDR_REQ_RECEIVED;
4054	break;
4055	case CM_SIDR_REP_ATTR_ID:
4056	event = IB_CM_SIDR_REP_RECEIVED;
4057	break;
4058	case CM_LAP_ATTR_ID:
4059	paths = 1;
4060	event = IB_CM_LAP_RECEIVED;
4061	break;
4062	case CM_APR_ATTR_ID:
4063	event = IB_CM_APR_RECEIVED;
4064	break;
4065	default:
4066	ib_free_recv_mad(mad_recv_wc);
4067	return;
4068	}
4069
4070	attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4071	atomic_long_inc(v: &port->counters[CM_RECV][attr_id - CM_ATTR_ID_OFFSET]);
4072
4073	work = kmalloc(struct_size(work, path, paths), GFP_KERNEL);
4074	if (!work) {
4075	ib_free_recv_mad(mad_recv_wc);
4076	return;
4077	}
4078
4079	INIT_DELAYED_WORK(&work->work, cm_work_handler);
4080	work->cm_event.event = event;
4081	work->mad_recv_wc = mad_recv_wc;
4082	work->port = port;
4083
4084	/* Check if the device started its remove_one */
4085	spin_lock_irq(lock: &cm.lock);
4086	if (!port->cm_dev->going_down)
4087	queue_delayed_work(wq: cm.wq, dwork: &work->work, delay: 0);
4088	else
4089	going_down = 1;
4090	spin_unlock_irq(lock: &cm.lock);
4091
4092	if (going_down) {
4093	kfree(objp: work);
4094	ib_free_recv_mad(mad_recv_wc);
4095	}
4096	}
4097
4098	static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4099	struct ib_qp_attr *qp_attr,
4100	int *qp_attr_mask)
4101	{
4102	unsigned long flags;
4103	int ret;
4104
4105	spin_lock_irqsave(&cm_id_priv->lock, flags);
4106	switch (cm_id_priv->id.state) {
4107	case IB_CM_REQ_SENT:
4108	case IB_CM_MRA_REQ_RCVD:
4109	case IB_CM_REQ_RCVD:
4110	case IB_CM_MRA_REQ_SENT:
4111	case IB_CM_REP_RCVD:
4112	case IB_CM_MRA_REP_SENT:
4113	case IB_CM_REP_SENT:
4114	case IB_CM_MRA_REP_RCVD:
4115	case IB_CM_ESTABLISHED:
4116	*qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4117	IB_QP_PKEY_INDEX | IB_QP_PORT;
4118	qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4119	if (cm_id_priv->responder_resources) {
4120	struct ib_device *ib_dev = cm_id_priv->id.device;
4121	u64 support_flush = ib_dev->attrs.device_cap_flags &
4122	(IB_DEVICE_FLUSH_GLOBAL | IB_DEVICE_FLUSH_PERSISTENT);
4123	u32 flushable = support_flush ?
4124	(IB_ACCESS_FLUSH_GLOBAL |
4125	IB_ACCESS_FLUSH_PERSISTENT) : 0;
4126
4127	qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4128	IB_ACCESS_REMOTE_ATOMIC |
4129	flushable;
4130	}
4131	qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4132	if (cm_id_priv->av.port)
4133	qp_attr->port_num = cm_id_priv->av.port->port_num;
4134	ret = 0;
4135	break;
4136	default:
4137	trace_icm_qp_init_err(cm_id: &cm_id_priv->id);
4138	ret = -EINVAL;
4139	break;
4140	}
4141	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
4142	return ret;
4143	}
4144
4145	static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4146	struct ib_qp_attr *qp_attr,
4147	int *qp_attr_mask)
4148	{
4149	unsigned long flags;
4150	int ret;
4151
4152	spin_lock_irqsave(&cm_id_priv->lock, flags);
4153	switch (cm_id_priv->id.state) {
4154	case IB_CM_REQ_RCVD:
4155	case IB_CM_MRA_REQ_SENT:
4156	case IB_CM_REP_RCVD:
4157	case IB_CM_MRA_REP_SENT:
4158	case IB_CM_REP_SENT:
4159	case IB_CM_MRA_REP_RCVD:
4160	case IB_CM_ESTABLISHED:
4161	*qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4162	IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4163	qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4164	if ((qp_attr->ah_attr.type == RDMA_AH_ATTR_TYPE_IB) &&
4165	cm_id_priv->av.dlid_datapath &&
4166	(cm_id_priv->av.dlid_datapath != 0xffff))
4167	qp_attr->ah_attr.ib.dlid = cm_id_priv->av.dlid_datapath;
4168	qp_attr->path_mtu = cm_id_priv->path_mtu;
4169	qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4170	qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4171	if (cm_id_priv->qp_type == IB_QPT_RC ||
4172	cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4173	*qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4174	IB_QP_MIN_RNR_TIMER;
4175	qp_attr->max_dest_rd_atomic =
4176	cm_id_priv->responder_resources;
4177	qp_attr->min_rnr_timer = 0;
4178	}
4179	if (rdma_ah_get_dlid(attr: &cm_id_priv->alt_av.ah_attr) &&
4180	cm_id_priv->alt_av.port) {
4181	*qp_attr_mask |= IB_QP_ALT_PATH;
4182	qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4183	qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4184	qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4185	qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4186	}
4187	ret = 0;
4188	break;
4189	default:
4190	trace_icm_qp_rtr_err(cm_id: &cm_id_priv->id);
4191	ret = -EINVAL;
4192	break;
4193	}
4194	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
4195	return ret;
4196	}
4197
4198	static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4199	struct ib_qp_attr *qp_attr,
4200	int *qp_attr_mask)
4201	{
4202	unsigned long flags;
4203	int ret;
4204
4205	spin_lock_irqsave(&cm_id_priv->lock, flags);
4206	switch (cm_id_priv->id.state) {
4207	/* Allow transition to RTS before sending REP */
4208	case IB_CM_REQ_RCVD:
4209	case IB_CM_MRA_REQ_SENT:
4210
4211	case IB_CM_REP_RCVD:
4212	case IB_CM_MRA_REP_SENT:
4213	case IB_CM_REP_SENT:
4214	case IB_CM_MRA_REP_RCVD:
4215	case IB_CM_ESTABLISHED:
4216	if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4217	*qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4218	qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4219	switch (cm_id_priv->qp_type) {
4220	case IB_QPT_RC:
4221	case IB_QPT_XRC_INI:
4222	*qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4223	IB_QP_MAX_QP_RD_ATOMIC;
4224	qp_attr->retry_cnt = cm_id_priv->retry_count;
4225	qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4226	qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4227	fallthrough;
4228	case IB_QPT_XRC_TGT:
4229	*qp_attr_mask |= IB_QP_TIMEOUT;
4230	qp_attr->timeout = cm_id_priv->av.timeout;
4231	break;
4232	default:
4233	break;
4234	}
4235	if (rdma_ah_get_dlid(attr: &cm_id_priv->alt_av.ah_attr)) {
4236	*qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4237	qp_attr->path_mig_state = IB_MIG_REARM;
4238	}
4239	} else {
4240	*qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4241	if (cm_id_priv->alt_av.port)
4242	qp_attr->alt_port_num =
4243	cm_id_priv->alt_av.port->port_num;
4244	qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4245	qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4246	qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4247	qp_attr->path_mig_state = IB_MIG_REARM;
4248	}
4249	ret = 0;
4250	break;
4251	default:
4252	trace_icm_qp_rts_err(cm_id: &cm_id_priv->id);
4253	ret = -EINVAL;
4254	break;
4255	}
4256	spin_unlock_irqrestore(lock: &cm_id_priv->lock, flags);
4257	return ret;
4258	}
4259
4260	int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4261	struct ib_qp_attr *qp_attr,
4262	int *qp_attr_mask)
4263	{
4264	struct cm_id_private *cm_id_priv;
4265	int ret;
4266
4267	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4268	switch (qp_attr->qp_state) {
4269	case IB_QPS_INIT:
4270	ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4271	break;
4272	case IB_QPS_RTR:
4273	ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4274	break;
4275	case IB_QPS_RTS:
4276	ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4277	break;
4278	default:
4279	ret = -EINVAL;
4280	break;
4281	}
4282	return ret;
4283	}
4284	EXPORT_SYMBOL(ib_cm_init_qp_attr);
4285
4286	static ssize_t cm_show_counter(struct ib_device *ibdev, u32 port_num,
4287	struct ib_port_attribute *attr, char *buf)
4288	{
4289	struct cm_counter_attribute *cm_attr =
4290	container_of(attr, struct cm_counter_attribute, attr);
4291	struct cm_device *cm_dev = ib_get_client_data(device: ibdev, client: &cm_client);
4292
4293	if (WARN_ON(!cm_dev))
4294	return -EINVAL;
4295
4296	return sysfs_emit(
4297	buf, fmt: "%ld\n",
4298	atomic_long_read(
4299	v: &cm_dev->port[port_num - 1]
4300	->counters[cm_attr->group][cm_attr->index]));
4301	}
4302
4303	#define CM_COUNTER_ATTR(_name, _group, _index) \
4304	{ \
4305	.attr = __ATTR(_name, 0444, cm_show_counter, NULL), \
4306	.group = _group, .index = _index \
4307	}
4308
4309	#define CM_COUNTER_GROUP(_group, _name) \
4310	static struct cm_counter_attribute cm_counter_attr_##_group[] = { \
4311	CM_COUNTER_ATTR(req, _group, CM_REQ_COUNTER), \
4312	CM_COUNTER_ATTR(mra, _group, CM_MRA_COUNTER), \
4313	CM_COUNTER_ATTR(rej, _group, CM_REJ_COUNTER), \
4314	CM_COUNTER_ATTR(rep, _group, CM_REP_COUNTER), \
4315	CM_COUNTER_ATTR(rtu, _group, CM_RTU_COUNTER), \
4316	CM_COUNTER_ATTR(dreq, _group, CM_DREQ_COUNTER), \
4317	CM_COUNTER_ATTR(drep, _group, CM_DREP_COUNTER), \
4318	CM_COUNTER_ATTR(sidr_req, _group, CM_SIDR_REQ_COUNTER), \
4319	CM_COUNTER_ATTR(sidr_rep, _group, CM_SIDR_REP_COUNTER), \
4320	CM_COUNTER_ATTR(lap, _group, CM_LAP_COUNTER), \
4321	CM_COUNTER_ATTR(apr, _group, CM_APR_COUNTER), \
4322	}; \
4323	static struct attribute *cm_counter_attrs_##_group[] = { \
4324	&cm_counter_attr_##_group[0].attr.attr, \
4325	&cm_counter_attr_##_group[1].attr.attr, \
4326	&cm_counter_attr_##_group[2].attr.attr, \
4327	&cm_counter_attr_##_group[3].attr.attr, \
4328	&cm_counter_attr_##_group[4].attr.attr, \
4329	&cm_counter_attr_##_group[5].attr.attr, \
4330	&cm_counter_attr_##_group[6].attr.attr, \
4331	&cm_counter_attr_##_group[7].attr.attr, \
4332	&cm_counter_attr_##_group[8].attr.attr, \
4333	&cm_counter_attr_##_group[9].attr.attr, \
4334	&cm_counter_attr_##_group[10].attr.attr, \
4335	NULL, \
4336	}; \
4337	static const struct attribute_group cm_counter_group_##_group = { \
4338	.name = _name, \
4339	.attrs = cm_counter_attrs_##_group, \
4340	};
4341
4342	CM_COUNTER_GROUP(CM_XMIT, "cm_tx_msgs")
4343	CM_COUNTER_GROUP(CM_XMIT_RETRIES, "cm_tx_retries")
4344	CM_COUNTER_GROUP(CM_RECV, "cm_rx_msgs")
4345	CM_COUNTER_GROUP(CM_RECV_DUPLICATES, "cm_rx_duplicates")
4346
4347	static const struct attribute_group *cm_counter_groups[] = {
4348	&cm_counter_group_CM_XMIT,
4349	&cm_counter_group_CM_XMIT_RETRIES,
4350	&cm_counter_group_CM_RECV,
4351	&cm_counter_group_CM_RECV_DUPLICATES,
4352	NULL,
4353	};
4354
4355	static int cm_add_one(struct ib_device *ib_device)
4356	{
4357	struct cm_device *cm_dev;
4358	struct cm_port *port;
4359	struct ib_mad_reg_req reg_req = {
4360	.mgmt_class = IB_MGMT_CLASS_CM,
4361	.mgmt_class_version = IB_CM_CLASS_VERSION,
4362	};
4363	struct ib_port_modify port_modify = {
4364	.set_port_cap_mask = IB_PORT_CM_SUP
4365	};
4366	unsigned long flags;
4367	int ret;
4368	int count = 0;
4369	u32 i;
4370
4371	cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
4372	GFP_KERNEL);
4373	if (!cm_dev)
4374	return -ENOMEM;
4375
4376	kref_init(kref: &cm_dev->kref);
4377	spin_lock_init(&cm_dev->mad_agent_lock);
4378	cm_dev->ib_device = ib_device;
4379	cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4380	cm_dev->going_down = 0;
4381
4382	ib_set_client_data(device: ib_device, client: &cm_client, data: cm_dev);
4383
4384	set_bit(IB_MGMT_METHOD_SEND, addr: reg_req.method_mask);
4385	rdma_for_each_port (ib_device, i) {
4386	if (!rdma_cap_ib_cm(device: ib_device, port_num: i))
4387	continue;
4388
4389	port = kzalloc(size: sizeof *port, GFP_KERNEL);
4390	if (!port) {
4391	ret = -ENOMEM;
4392	goto error1;
4393	}
4394
4395	cm_dev->port[i-1] = port;
4396	port->cm_dev = cm_dev;
4397	port->port_num = i;
4398
4399	ret = ib_port_register_client_groups(ibdev: ib_device, port_num: i,
4400	groups: cm_counter_groups);
4401	if (ret)
4402	goto error1;
4403
4404	port->mad_agent = ib_register_mad_agent(device: ib_device, port_num: i,
4405	qp_type: IB_QPT_GSI,
4406	mad_reg_req: &reg_req,
4407	rmpp_version: 0,
4408	send_handler: cm_send_handler,
4409	recv_handler: cm_recv_handler,
4410	context: port,
4411	registration_flags: 0);
4412	if (IS_ERR(ptr: port->mad_agent)) {
4413	ret = PTR_ERR(ptr: port->mad_agent);
4414	goto error2;
4415	}
4416
4417	ret = ib_modify_port(device: ib_device, port_num: i, port_modify_mask: 0, port_modify: &port_modify);
4418	if (ret)
4419	goto error3;
4420
4421	count++;
4422	}
4423
4424	if (!count) {
4425	ret = -EOPNOTSUPP;
4426	goto free;
4427	}
4428
4429	write_lock_irqsave(&cm.device_lock, flags);
4430	list_add_tail(new: &cm_dev->list, head: &cm.device_list);
4431	write_unlock_irqrestore(&cm.device_lock, flags);
4432	return 0;
4433
4434	error3:
4435	ib_unregister_mad_agent(mad_agent: port->mad_agent);
4436	error2:
4437	ib_port_unregister_client_groups(ibdev: ib_device, port_num: i, groups: cm_counter_groups);
4438	error1:
4439	port_modify.set_port_cap_mask = 0;
4440	port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4441	while (--i) {
4442	if (!rdma_cap_ib_cm(device: ib_device, port_num: i))
4443	continue;
4444
4445	port = cm_dev->port[i-1];
4446	ib_modify_port(device: ib_device, port_num: port->port_num, port_modify_mask: 0, port_modify: &port_modify);
4447	ib_unregister_mad_agent(mad_agent: port->mad_agent);
4448	ib_port_unregister_client_groups(ibdev: ib_device, port_num: i,
4449	groups: cm_counter_groups);
4450	}
4451	free:
4452	cm_device_put(cm_dev);
4453	return ret;
4454	}
4455
4456	static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4457	{
4458	struct cm_device *cm_dev = client_data;
4459	struct cm_port *port;
4460	struct ib_port_modify port_modify = {
4461	.clr_port_cap_mask = IB_PORT_CM_SUP
4462	};
4463	unsigned long flags;
4464	u32 i;
4465
4466	write_lock_irqsave(&cm.device_lock, flags);
4467	list_del(entry: &cm_dev->list);
4468	write_unlock_irqrestore(&cm.device_lock, flags);
4469
4470	spin_lock_irq(lock: &cm.lock);
4471	cm_dev->going_down = 1;
4472	spin_unlock_irq(lock: &cm.lock);
4473
4474	rdma_for_each_port (ib_device, i) {
4475	struct ib_mad_agent *mad_agent;
4476
4477	if (!rdma_cap_ib_cm(device: ib_device, port_num: i))
4478	continue;
4479
4480	port = cm_dev->port[i-1];
4481	mad_agent = port->mad_agent;
4482	ib_modify_port(device: ib_device, port_num: port->port_num, port_modify_mask: 0, port_modify: &port_modify);
4483	/*
4484	* We flush the queue here after the going_down set, this
4485	* verify that no new works will be queued in the recv handler,
4486	* after that we can call the unregister_mad_agent
4487	*/
4488	flush_workqueue(cm.wq);
4489	/*
4490	* The above ensures no call paths from the work are running,
4491	* the remaining paths all take the mad_agent_lock.
4492	*/
4493	spin_lock(lock: &cm_dev->mad_agent_lock);
4494	port->mad_agent = NULL;
4495	spin_unlock(lock: &cm_dev->mad_agent_lock);
4496	ib_unregister_mad_agent(mad_agent);
4497	ib_port_unregister_client_groups(ibdev: ib_device, port_num: i,
4498	groups: cm_counter_groups);
4499	}
4500
4501	cm_device_put(cm_dev);
4502	}
4503
4504	static int __init ib_cm_init(void)
4505	{
4506	int ret;
4507
4508	INIT_LIST_HEAD(list: &cm.device_list);
4509	rwlock_init(&cm.device_lock);
4510	spin_lock_init(&cm.lock);
4511	cm.listen_service_table = RB_ROOT;
4512	cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4513	cm.remote_id_table = RB_ROOT;
4514	cm.remote_qp_table = RB_ROOT;
4515	cm.remote_sidr_table = RB_ROOT;
4516	xa_init_flags(xa: &cm.local_id_table, XA_FLAGS_ALLOC);
4517	get_random_bytes(buf: &cm.random_id_operand, len: sizeof cm.random_id_operand);
4518	INIT_LIST_HEAD(list: &cm.timewait_list);
4519
4520	cm.wq = alloc_workqueue(fmt: "ib_cm", flags: 0, max_active: 1);
4521	if (!cm.wq) {
4522	ret = -ENOMEM;
4523	goto error2;
4524	}
4525
4526	ret = ib_register_client(client: &cm_client);
4527	if (ret)
4528	goto error3;
4529
4530	return 0;
4531	error3:
4532	destroy_workqueue(wq: cm.wq);
4533	error2:
4534	return ret;
4535	}
4536
4537	static void __exit ib_cm_cleanup(void)
4538	{
4539	struct cm_timewait_info *timewait_info, *tmp;
4540
4541	spin_lock_irq(lock: &cm.lock);
4542	list_for_each_entry(timewait_info, &cm.timewait_list, list)
4543	cancel_delayed_work(dwork: &timewait_info->work.work);
4544	spin_unlock_irq(lock: &cm.lock);
4545
4546	ib_unregister_client(client: &cm_client);
4547	destroy_workqueue(wq: cm.wq);
4548
4549	list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4550	list_del(entry: &timewait_info->list);
4551	kfree(objp: timewait_info);
4552	}
4553
4554	WARN_ON(!xa_empty(&cm.local_id_table));
4555	}
4556
4557	module_init(ib_cm_init);
4558	module_exit(ib_cm_cleanup);
4559