1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2	/*
3	* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
4	* Copyright (c) 2004 Infinicon Corporation. All rights reserved.
5	* Copyright (c) 2004, 2020 Intel Corporation. All rights reserved.
6	* Copyright (c) 2004 Topspin Corporation. All rights reserved.
7	* Copyright (c) 2004 Voltaire Corporation. All rights reserved.
8	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
9	* Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
10	*/
11
12	#ifndef IB_VERBS_H
13	#define IB_VERBS_H
14
15	#include <linux/ethtool.h>
16	#include <linux/types.h>
17	#include <linux/device.h>
18	#include <linux/dma-mapping.h>
19	#include <linux/kref.h>
20	#include <linux/list.h>
21	#include <linux/rwsem.h>
22	#include <linux/workqueue.h>
23	#include <linux/irq_poll.h>
24	#include <uapi/linux/if_ether.h>
25	#include <net/ipv6.h>
26	#include <net/ip.h>
27	#include <linux/string.h>
28	#include <linux/slab.h>
29	#include <linux/netdevice.h>
30	#include <linux/refcount.h>
31	#include <linux/if_link.h>
32	#include <linux/atomic.h>
33	#include <linux/mmu_notifier.h>
34	#include <linux/uaccess.h>
35	#include <linux/cgroup_rdma.h>
36	#include <linux/irqflags.h>
37	#include <linux/preempt.h>
38	#include <linux/dim.h>
39	#include <uapi/rdma/ib_user_verbs.h>
40	#include <rdma/rdma_counter.h>
41	#include <rdma/restrack.h>
42	#include <rdma/signature.h>
43	#include <uapi/rdma/rdma_user_ioctl.h>
44	#include <uapi/rdma/ib_user_ioctl_verbs.h>
45
46	#define IB_FW_VERSION_NAME_MAX ETHTOOL_FWVERS_LEN
47
48	struct ib_umem_odp;
49	struct ib_uqp_object;
50	struct ib_usrq_object;
51	struct ib_uwq_object;
52	struct rdma_cm_id;
53	struct ib_port;
54	struct hw_stats_device_data;
55
56	extern struct workqueue_struct *ib_wq;
57	extern struct workqueue_struct *ib_comp_wq;
58	extern struct workqueue_struct *ib_comp_unbound_wq;
59
60	struct ib_ucq_object;
61
62	__printf(3, 4) __cold
63	void ibdev_printk(const char *level, const struct ib_device *ibdev,
64	const char *format, ...);
65	__printf(2, 3) __cold
66	void ibdev_emerg(const struct ib_device *ibdev, const char *format, ...);
67	__printf(2, 3) __cold
68	void ibdev_alert(const struct ib_device *ibdev, const char *format, ...);
69	__printf(2, 3) __cold
70	void ibdev_crit(const struct ib_device *ibdev, const char *format, ...);
71	__printf(2, 3) __cold
72	void ibdev_err(const struct ib_device *ibdev, const char *format, ...);
73	__printf(2, 3) __cold
74	void ibdev_warn(const struct ib_device *ibdev, const char *format, ...);
75	__printf(2, 3) __cold
76	void ibdev_notice(const struct ib_device *ibdev, const char *format, ...);
77	__printf(2, 3) __cold
78	void ibdev_info(const struct ib_device *ibdev, const char *format, ...);
79
80	#if defined(CONFIG_DYNAMIC_DEBUG) || \
81	(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
82	#define ibdev_dbg(__dev, format, args...) \
83	dynamic_ibdev_dbg(__dev, format, ##args)
84	#else
85	__printf(2, 3) __cold
86	static inline
87	void ibdev_dbg(const struct ib_device *ibdev, const char *format, ...) {}
88	#endif
89
90	#define ibdev_level_ratelimited(ibdev_level, ibdev, fmt, ...) \
91	do { \
92	static DEFINE_RATELIMIT_STATE(_rs, \
93	DEFAULT_RATELIMIT_INTERVAL, \
94	DEFAULT_RATELIMIT_BURST); \
95	if (__ratelimit(&_rs)) \
96	ibdev_level(ibdev, fmt, ##__VA_ARGS__); \
97	} while (0)
98
99	#define ibdev_emerg_ratelimited(ibdev, fmt, ...) \
100	ibdev_level_ratelimited(ibdev_emerg, ibdev, fmt, ##__VA_ARGS__)
101	#define ibdev_alert_ratelimited(ibdev, fmt, ...) \
102	ibdev_level_ratelimited(ibdev_alert, ibdev, fmt, ##__VA_ARGS__)
103	#define ibdev_crit_ratelimited(ibdev, fmt, ...) \
104	ibdev_level_ratelimited(ibdev_crit, ibdev, fmt, ##__VA_ARGS__)
105	#define ibdev_err_ratelimited(ibdev, fmt, ...) \
106	ibdev_level_ratelimited(ibdev_err, ibdev, fmt, ##__VA_ARGS__)
107	#define ibdev_warn_ratelimited(ibdev, fmt, ...) \
108	ibdev_level_ratelimited(ibdev_warn, ibdev, fmt, ##__VA_ARGS__)
109	#define ibdev_notice_ratelimited(ibdev, fmt, ...) \
110	ibdev_level_ratelimited(ibdev_notice, ibdev, fmt, ##__VA_ARGS__)
111	#define ibdev_info_ratelimited(ibdev, fmt, ...) \
112	ibdev_level_ratelimited(ibdev_info, ibdev, fmt, ##__VA_ARGS__)
113
114	#if defined(CONFIG_DYNAMIC_DEBUG) || \
115	(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
116	/* descriptor check is first to prevent flooding with "callbacks suppressed" */
117	#define ibdev_dbg_ratelimited(ibdev, fmt, ...) \
118	do { \
119	static DEFINE_RATELIMIT_STATE(_rs, \
120	DEFAULT_RATELIMIT_INTERVAL, \
121	DEFAULT_RATELIMIT_BURST); \
122	DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, fmt); \
123	if (DYNAMIC_DEBUG_BRANCH(descriptor) && __ratelimit(&_rs)) \
124	__dynamic_ibdev_dbg(&descriptor, ibdev, fmt, \
125	##__VA_ARGS__); \
126	} while (0)
127	#else
128	__printf(2, 3) __cold
129	static inline
130	void ibdev_dbg_ratelimited(const struct ib_device *ibdev, const char *format, ...) {}
131	#endif
132
133	union ib_gid {
134	u8 raw[16];
135	struct {
136	__be64 subnet_prefix;
137	__be64 interface_id;
138	} global;
139	};
140
141	extern union ib_gid zgid;
142
143	enum ib_gid_type {
144	IB_GID_TYPE_IB = IB_UVERBS_GID_TYPE_IB,
145	IB_GID_TYPE_ROCE = IB_UVERBS_GID_TYPE_ROCE_V1,
146	IB_GID_TYPE_ROCE_UDP_ENCAP = IB_UVERBS_GID_TYPE_ROCE_V2,
147	IB_GID_TYPE_SIZE
148	};
149
150	#define ROCE_V2_UDP_DPORT 4791
151	struct ib_gid_attr {
152	struct net_device __rcu *ndev;
153	struct ib_device *device;
154	union ib_gid gid;
155	enum ib_gid_type gid_type;
156	u16 index;
157	u32 port_num;
158	};
159
160	enum {
161	/* set the local administered indication */
162	IB_SA_WELL_KNOWN_GUID = BIT_ULL(57) | 2,
163	};
164
165	enum rdma_transport_type {
166	RDMA_TRANSPORT_IB,
167	RDMA_TRANSPORT_IWARP,
168	RDMA_TRANSPORT_USNIC,
169	RDMA_TRANSPORT_USNIC_UDP,
170	RDMA_TRANSPORT_UNSPECIFIED,
171	};
172
173	enum rdma_protocol_type {
174	RDMA_PROTOCOL_IB,
175	RDMA_PROTOCOL_IBOE,
176	RDMA_PROTOCOL_IWARP,
177	RDMA_PROTOCOL_USNIC_UDP
178	};
179
180	__attribute_const__ enum rdma_transport_type
181	rdma_node_get_transport(unsigned int node_type);
182
183	enum rdma_network_type {
184	RDMA_NETWORK_IB,
185	RDMA_NETWORK_ROCE_V1,
186	RDMA_NETWORK_IPV4,
187	RDMA_NETWORK_IPV6
188	};
189
190	static inline enum ib_gid_type ib_network_to_gid_type(enum rdma_network_type network_type)
191	{
192	if (network_type == RDMA_NETWORK_IPV4 ||
193	network_type == RDMA_NETWORK_IPV6)
194	return IB_GID_TYPE_ROCE_UDP_ENCAP;
195	else if (network_type == RDMA_NETWORK_ROCE_V1)
196	return IB_GID_TYPE_ROCE;
197	else
198	return IB_GID_TYPE_IB;
199	}
200
201	static inline enum rdma_network_type
202	rdma_gid_attr_network_type(const struct ib_gid_attr *attr)
203	{
204	if (attr->gid_type == IB_GID_TYPE_IB)
205	return RDMA_NETWORK_IB;
206
207	if (attr->gid_type == IB_GID_TYPE_ROCE)
208	return RDMA_NETWORK_ROCE_V1;
209
210	if (ipv6_addr_v4mapped(a: (struct in6_addr *)&attr->gid))
211	return RDMA_NETWORK_IPV4;
212	else
213	return RDMA_NETWORK_IPV6;
214	}
215
216	enum rdma_link_layer {
217	IB_LINK_LAYER_UNSPECIFIED,
218	IB_LINK_LAYER_INFINIBAND,
219	IB_LINK_LAYER_ETHERNET,
220	};
221
222	enum ib_device_cap_flags {
223	IB_DEVICE_RESIZE_MAX_WR = IB_UVERBS_DEVICE_RESIZE_MAX_WR,
224	IB_DEVICE_BAD_PKEY_CNTR = IB_UVERBS_DEVICE_BAD_PKEY_CNTR,
225	IB_DEVICE_BAD_QKEY_CNTR = IB_UVERBS_DEVICE_BAD_QKEY_CNTR,
226	IB_DEVICE_RAW_MULTI = IB_UVERBS_DEVICE_RAW_MULTI,
227	IB_DEVICE_AUTO_PATH_MIG = IB_UVERBS_DEVICE_AUTO_PATH_MIG,
228	IB_DEVICE_CHANGE_PHY_PORT = IB_UVERBS_DEVICE_CHANGE_PHY_PORT,
229	IB_DEVICE_UD_AV_PORT_ENFORCE = IB_UVERBS_DEVICE_UD_AV_PORT_ENFORCE,
230	IB_DEVICE_CURR_QP_STATE_MOD = IB_UVERBS_DEVICE_CURR_QP_STATE_MOD,
231	IB_DEVICE_SHUTDOWN_PORT = IB_UVERBS_DEVICE_SHUTDOWN_PORT,
232	/* IB_DEVICE_INIT_TYPE = IB_UVERBS_DEVICE_INIT_TYPE, (not in use) */
233	IB_DEVICE_PORT_ACTIVE_EVENT = IB_UVERBS_DEVICE_PORT_ACTIVE_EVENT,
234	IB_DEVICE_SYS_IMAGE_GUID = IB_UVERBS_DEVICE_SYS_IMAGE_GUID,
235	IB_DEVICE_RC_RNR_NAK_GEN = IB_UVERBS_DEVICE_RC_RNR_NAK_GEN,
236	IB_DEVICE_SRQ_RESIZE = IB_UVERBS_DEVICE_SRQ_RESIZE,
237	IB_DEVICE_N_NOTIFY_CQ = IB_UVERBS_DEVICE_N_NOTIFY_CQ,
238
239	/* Reserved, old SEND_W_INV = 1 << 16,*/
240	IB_DEVICE_MEM_WINDOW = IB_UVERBS_DEVICE_MEM_WINDOW,
241	/*
242	* Devices should set IB_DEVICE_UD_IP_SUM if they support
243	* insertion of UDP and TCP checksum on outgoing UD IPoIB
244	* messages and can verify the validity of checksum for
245	* incoming messages. Setting this flag implies that the
246	* IPoIB driver may set NETIF_F_IP_CSUM for datagram mode.
247	*/
248	IB_DEVICE_UD_IP_CSUM = IB_UVERBS_DEVICE_UD_IP_CSUM,
249	IB_DEVICE_XRC = IB_UVERBS_DEVICE_XRC,
250
251	/*
252	* This device supports the IB "base memory management extension",
253	* which includes support for fast registrations (IB_WR_REG_MR,
254	* IB_WR_LOCAL_INV and IB_WR_SEND_WITH_INV verbs). This flag should
255	* also be set by any iWarp device which must support FRs to comply
256	* to the iWarp verbs spec. iWarp devices also support the
257	* IB_WR_RDMA_READ_WITH_INV verb for RDMA READs that invalidate the
258	* stag.
259	*/
260	IB_DEVICE_MEM_MGT_EXTENSIONS = IB_UVERBS_DEVICE_MEM_MGT_EXTENSIONS,
261	IB_DEVICE_MEM_WINDOW_TYPE_2A = IB_UVERBS_DEVICE_MEM_WINDOW_TYPE_2A,
262	IB_DEVICE_MEM_WINDOW_TYPE_2B = IB_UVERBS_DEVICE_MEM_WINDOW_TYPE_2B,
263	IB_DEVICE_RC_IP_CSUM = IB_UVERBS_DEVICE_RC_IP_CSUM,
264	/* Deprecated. Please use IB_RAW_PACKET_CAP_IP_CSUM. */
265	IB_DEVICE_RAW_IP_CSUM = IB_UVERBS_DEVICE_RAW_IP_CSUM,
266	IB_DEVICE_MANAGED_FLOW_STEERING =
267	IB_UVERBS_DEVICE_MANAGED_FLOW_STEERING,
268	/* Deprecated. Please use IB_RAW_PACKET_CAP_SCATTER_FCS. */
269	IB_DEVICE_RAW_SCATTER_FCS = IB_UVERBS_DEVICE_RAW_SCATTER_FCS,
270	/* The device supports padding incoming writes to cacheline. */
271	IB_DEVICE_PCI_WRITE_END_PADDING =
272	IB_UVERBS_DEVICE_PCI_WRITE_END_PADDING,
273	/* Placement type attributes */
274	IB_DEVICE_FLUSH_GLOBAL = IB_UVERBS_DEVICE_FLUSH_GLOBAL,
275	IB_DEVICE_FLUSH_PERSISTENT = IB_UVERBS_DEVICE_FLUSH_PERSISTENT,
276	IB_DEVICE_ATOMIC_WRITE = IB_UVERBS_DEVICE_ATOMIC_WRITE,
277	};
278
279	enum ib_kernel_cap_flags {
280	/*
281	* This device supports a per-device lkey or stag that can be
282	* used without performing a memory registration for the local
283	* memory. Note that ULPs should never check this flag, but
284	* instead of use the local_dma_lkey flag in the ib_pd structure,
285	* which will always contain a usable lkey.
286	*/
287	IBK_LOCAL_DMA_LKEY = 1 << 0,
288	/* IB_QP_CREATE_INTEGRITY_EN is supported to implement T10-PI */
289	IBK_INTEGRITY_HANDOVER = 1 << 1,
290	/* IB_ACCESS_ON_DEMAND is supported during reg_user_mr() */
291	IBK_ON_DEMAND_PAGING = 1 << 2,
292	/* IB_MR_TYPE_SG_GAPS is supported */
293	IBK_SG_GAPS_REG = 1 << 3,
294	/* Driver supports RDMA_NLDEV_CMD_DELLINK */
295	IBK_ALLOW_USER_UNREG = 1 << 4,
296
297	/* ipoib will use IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK */
298	IBK_BLOCK_MULTICAST_LOOPBACK = 1 << 5,
299	/* iopib will use IB_QP_CREATE_IPOIB_UD_LSO for its QPs */
300	IBK_UD_TSO = 1 << 6,
301	/* iopib will use the device ops:
302	* get_vf_config
303	* get_vf_guid
304	* get_vf_stats
305	* set_vf_guid
306	* set_vf_link_state
307	*/
308	IBK_VIRTUAL_FUNCTION = 1 << 7,
309	/* ipoib will use IB_QP_CREATE_NETDEV_USE for its QPs */
310	IBK_RDMA_NETDEV_OPA = 1 << 8,
311	};
312
313	enum ib_atomic_cap {
314	IB_ATOMIC_NONE,
315	IB_ATOMIC_HCA,
316	IB_ATOMIC_GLOB
317	};
318
319	enum ib_odp_general_cap_bits {
320	IB_ODP_SUPPORT = 1 << 0,
321	IB_ODP_SUPPORT_IMPLICIT = 1 << 1,
322	};
323
324	enum ib_odp_transport_cap_bits {
325	IB_ODP_SUPPORT_SEND = 1 << 0,
326	IB_ODP_SUPPORT_RECV = 1 << 1,
327	IB_ODP_SUPPORT_WRITE = 1 << 2,
328	IB_ODP_SUPPORT_READ = 1 << 3,
329	IB_ODP_SUPPORT_ATOMIC = 1 << 4,
330	IB_ODP_SUPPORT_SRQ_RECV = 1 << 5,
331	};
332
333	struct ib_odp_caps {
334	uint64_t general_caps;
335	struct {
336	uint32_t rc_odp_caps;
337	uint32_t uc_odp_caps;
338	uint32_t ud_odp_caps;
339	uint32_t xrc_odp_caps;
340	} per_transport_caps;
341	};
342
343	struct ib_rss_caps {
344	/* Corresponding bit will be set if qp type from
345	* 'enum ib_qp_type' is supported, e.g.
346	* supported_qpts |= 1 << IB_QPT_UD
347	*/
348	u32 supported_qpts;
349	u32 max_rwq_indirection_tables;
350	u32 max_rwq_indirection_table_size;
351	};
352
353	enum ib_tm_cap_flags {
354	/* Support tag matching with rendezvous offload for RC transport */
355	IB_TM_CAP_RNDV_RC = 1 << 0,
356	};
357
358	struct ib_tm_caps {
359	/* Max size of RNDV header */
360	u32 max_rndv_hdr_size;
361	/* Max number of entries in tag matching list */
362	u32 max_num_tags;
363	/* From enum ib_tm_cap_flags */
364	u32 flags;
365	/* Max number of outstanding list operations */
366	u32 max_ops;
367	/* Max number of SGE in tag matching entry */
368	u32 max_sge;
369	};
370
371	struct ib_cq_init_attr {
372	unsigned int cqe;
373	u32 comp_vector;
374	u32 flags;
375	};
376
377	enum ib_cq_attr_mask {
378	IB_CQ_MODERATE = 1 << 0,
379	};
380
381	struct ib_cq_caps {
382	u16 max_cq_moderation_count;
383	u16 max_cq_moderation_period;
384	};
385
386	struct ib_dm_mr_attr {
387	u64 length;
388	u64 offset;
389	u32 access_flags;
390	};
391
392	struct ib_dm_alloc_attr {
393	u64 length;
394	u32 alignment;
395	u32 flags;
396	};
397
398	struct ib_device_attr {
399	u64 fw_ver;
400	__be64 sys_image_guid;
401	u64 max_mr_size;
402	u64 page_size_cap;
403	u32 vendor_id;
404	u32 vendor_part_id;
405	u32 hw_ver;
406	int max_qp;
407	int max_qp_wr;
408	u64 device_cap_flags;
409	u64 kernel_cap_flags;
410	int max_send_sge;
411	int max_recv_sge;
412	int max_sge_rd;
413	int max_cq;
414	int max_cqe;
415	int max_mr;
416	int max_pd;
417	int max_qp_rd_atom;
418	int max_ee_rd_atom;
419	int max_res_rd_atom;
420	int max_qp_init_rd_atom;
421	int max_ee_init_rd_atom;
422	enum ib_atomic_cap atomic_cap;
423	enum ib_atomic_cap masked_atomic_cap;
424	int max_ee;
425	int max_rdd;
426	int max_mw;
427	int max_raw_ipv6_qp;
428	int max_raw_ethy_qp;
429	int max_mcast_grp;
430	int max_mcast_qp_attach;
431	int max_total_mcast_qp_attach;
432	int max_ah;
433	int max_srq;
434	int max_srq_wr;
435	int max_srq_sge;
436	unsigned int max_fast_reg_page_list_len;
437	unsigned int max_pi_fast_reg_page_list_len;
438	u16 max_pkeys;
439	u8 local_ca_ack_delay;
440	int sig_prot_cap;
441	int sig_guard_cap;
442	struct ib_odp_caps odp_caps;
443	uint64_t timestamp_mask;
444	uint64_t hca_core_clock; /* in KHZ */
445	struct ib_rss_caps rss_caps;
446	u32 max_wq_type_rq;
447	u32 raw_packet_caps; /* Use ib_raw_packet_caps enum */
448	struct ib_tm_caps tm_caps;
449	struct ib_cq_caps cq_caps;
450	u64 max_dm_size;
451	/* Max entries for sgl for optimized performance per READ */
452	u32 max_sgl_rd;
453	};
454
455	enum ib_mtu {
456	IB_MTU_256 = 1,
457	IB_MTU_512 = 2,
458	IB_MTU_1024 = 3,
459	IB_MTU_2048 = 4,
460	IB_MTU_4096 = 5
461	};
462
463	enum opa_mtu {
464	OPA_MTU_8192 = 6,
465	OPA_MTU_10240 = 7
466	};
467
468	static inline int ib_mtu_enum_to_int(enum ib_mtu mtu)
469	{
470	switch (mtu) {
471	case IB_MTU_256: return 256;
472	case IB_MTU_512: return 512;
473	case IB_MTU_1024: return 1024;
474	case IB_MTU_2048: return 2048;
475	case IB_MTU_4096: return 4096;
476	default: return -1;
477	}
478	}
479
480	static inline enum ib_mtu ib_mtu_int_to_enum(int mtu)
481	{
482	if (mtu >= 4096)
483	return IB_MTU_4096;
484	else if (mtu >= 2048)
485	return IB_MTU_2048;
486	else if (mtu >= 1024)
487	return IB_MTU_1024;
488	else if (mtu >= 512)
489	return IB_MTU_512;
490	else
491	return IB_MTU_256;
492	}
493
494	static inline int opa_mtu_enum_to_int(enum opa_mtu mtu)
495	{
496	switch (mtu) {
497	case OPA_MTU_8192:
498	return 8192;
499	case OPA_MTU_10240:
500	return 10240;
501	default:
502	return(ib_mtu_enum_to_int(mtu: (enum ib_mtu)mtu));
503	}
504	}
505
506	static inline enum opa_mtu opa_mtu_int_to_enum(int mtu)
507	{
508	if (mtu >= 10240)
509	return OPA_MTU_10240;
510	else if (mtu >= 8192)
511	return OPA_MTU_8192;
512	else
513	return ((enum opa_mtu)ib_mtu_int_to_enum(mtu));
514	}
515
516	enum ib_port_state {
517	IB_PORT_NOP = 0,
518	IB_PORT_DOWN = 1,
519	IB_PORT_INIT = 2,
520	IB_PORT_ARMED = 3,
521	IB_PORT_ACTIVE = 4,
522	IB_PORT_ACTIVE_DEFER = 5
523	};
524
525	enum ib_port_phys_state {
526	IB_PORT_PHYS_STATE_SLEEP = 1,
527	IB_PORT_PHYS_STATE_POLLING = 2,
528	IB_PORT_PHYS_STATE_DISABLED = 3,
529	IB_PORT_PHYS_STATE_PORT_CONFIGURATION_TRAINING = 4,
530	IB_PORT_PHYS_STATE_LINK_UP = 5,
531	IB_PORT_PHYS_STATE_LINK_ERROR_RECOVERY = 6,
532	IB_PORT_PHYS_STATE_PHY_TEST = 7,
533	};
534
535	enum ib_port_width {
536	IB_WIDTH_1X = 1,
537	IB_WIDTH_2X = 16,
538	IB_WIDTH_4X = 2,
539	IB_WIDTH_8X = 4,
540	IB_WIDTH_12X = 8
541	};
542
543	static inline int ib_width_enum_to_int(enum ib_port_width width)
544	{
545	switch (width) {
546	case IB_WIDTH_1X: return 1;
547	case IB_WIDTH_2X: return 2;
548	case IB_WIDTH_4X: return 4;
549	case IB_WIDTH_8X: return 8;
550	case IB_WIDTH_12X: return 12;
551	default: return -1;
552	}
553	}
554
555	enum ib_port_speed {
556	IB_SPEED_SDR = 1,
557	IB_SPEED_DDR = 2,
558	IB_SPEED_QDR = 4,
559	IB_SPEED_FDR10 = 8,
560	IB_SPEED_FDR = 16,
561	IB_SPEED_EDR = 32,
562	IB_SPEED_HDR = 64,
563	IB_SPEED_NDR = 128,
564	IB_SPEED_XDR = 256,
565	};
566
567	enum ib_stat_flag {
568	IB_STAT_FLAG_OPTIONAL = 1 << 0,
569	};
570
571	/**
572	* struct rdma_stat_desc
573	* @name - The name of the counter
574	* @flags - Flags of the counter; For example, IB_STAT_FLAG_OPTIONAL
575	* @priv - Driver private information; Core code should not use
576	*/
577	struct rdma_stat_desc {
578	const char *name;
579	unsigned int flags;
580	const void *priv;
581	};
582
583	/**
584	* struct rdma_hw_stats
585	* @lock - Mutex to protect parallel write access to lifespan and values
586	* of counters, which are 64bits and not guaranteed to be written
587	* atomicaly on 32bits systems.
588	* @timestamp - Used by the core code to track when the last update was
589	* @lifespan - Used by the core code to determine how old the counters
590	* should be before being updated again. Stored in jiffies, defaults
591	* to 10 milliseconds, drivers can override the default be specifying
592	* their own value during their allocation routine.
593	* @descs - Array of pointers to static descriptors used for the counters
594	* in directory.
595	* @is_disabled - A bitmap to indicate each counter is currently disabled
596	* or not.
597	* @num_counters - How many hardware counters there are. If name is
598	* shorter than this number, a kernel oops will result. Driver authors
599	* are encouraged to leave BUILD_BUG_ON(ARRAY_SIZE(@name) < num_counters)
600	* in their code to prevent this.
601	* @value - Array of u64 counters that are accessed by the sysfs code and
602	* filled in by the drivers get_stats routine
603	*/
604	struct rdma_hw_stats {
605	struct mutex lock; /* Protect lifespan and values[] */
606	unsigned long timestamp;
607	unsigned long lifespan;
608	const struct rdma_stat_desc *descs;
609	unsigned long *is_disabled;
610	int num_counters;
611	u64 value[] __counted_by(num_counters);
612	};
613
614	#define RDMA_HW_STATS_DEFAULT_LIFESPAN 10
615
616	struct rdma_hw_stats *rdma_alloc_hw_stats_struct(
617	const struct rdma_stat_desc *descs, int num_counters,
618	unsigned long lifespan);
619
620	void rdma_free_hw_stats_struct(struct rdma_hw_stats *stats);
621
622	/* Define bits for the various functionality this port needs to be supported by
623	* the core.
624	*/
625	/* Management 0x00000FFF */
626	#define RDMA_CORE_CAP_IB_MAD 0x00000001
627	#define RDMA_CORE_CAP_IB_SMI 0x00000002
628	#define RDMA_CORE_CAP_IB_CM 0x00000004
629	#define RDMA_CORE_CAP_IW_CM 0x00000008
630	#define RDMA_CORE_CAP_IB_SA 0x00000010
631	#define RDMA_CORE_CAP_OPA_MAD 0x00000020
632
633	/* Address format 0x000FF000 */
634	#define RDMA_CORE_CAP_AF_IB 0x00001000
635	#define RDMA_CORE_CAP_ETH_AH 0x00002000
636	#define RDMA_CORE_CAP_OPA_AH 0x00004000
637	#define RDMA_CORE_CAP_IB_GRH_REQUIRED 0x00008000
638
639	/* Protocol 0xFFF00000 */
640	#define RDMA_CORE_CAP_PROT_IB 0x00100000
641	#define RDMA_CORE_CAP_PROT_ROCE 0x00200000
642	#define RDMA_CORE_CAP_PROT_IWARP 0x00400000
643	#define RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP 0x00800000
644	#define RDMA_CORE_CAP_PROT_RAW_PACKET 0x01000000
645	#define RDMA_CORE_CAP_PROT_USNIC 0x02000000
646
647	#define RDMA_CORE_PORT_IB_GRH_REQUIRED (RDMA_CORE_CAP_IB_GRH_REQUIRED \
648	| RDMA_CORE_CAP_PROT_ROCE \
649	| RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP)
650
651	#define RDMA_CORE_PORT_IBA_IB (RDMA_CORE_CAP_PROT_IB \
652	| RDMA_CORE_CAP_IB_MAD \
653	| RDMA_CORE_CAP_IB_SMI \
654	| RDMA_CORE_CAP_IB_CM \
655	| RDMA_CORE_CAP_IB_SA \
656	| RDMA_CORE_CAP_AF_IB)
657	#define RDMA_CORE_PORT_IBA_ROCE (RDMA_CORE_CAP_PROT_ROCE \
658	| RDMA_CORE_CAP_IB_MAD \
659	| RDMA_CORE_CAP_IB_CM \
660	| RDMA_CORE_CAP_AF_IB \
661	| RDMA_CORE_CAP_ETH_AH)
662	#define RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP \
663	(RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP \
664	| RDMA_CORE_CAP_IB_MAD \
665	| RDMA_CORE_CAP_IB_CM \
666	| RDMA_CORE_CAP_AF_IB \
667	| RDMA_CORE_CAP_ETH_AH)
668	#define RDMA_CORE_PORT_IWARP (RDMA_CORE_CAP_PROT_IWARP \
669	| RDMA_CORE_CAP_IW_CM)
670	#define RDMA_CORE_PORT_INTEL_OPA (RDMA_CORE_PORT_IBA_IB \
671	| RDMA_CORE_CAP_OPA_MAD)
672
673	#define RDMA_CORE_PORT_RAW_PACKET (RDMA_CORE_CAP_PROT_RAW_PACKET)
674
675	#define RDMA_CORE_PORT_USNIC (RDMA_CORE_CAP_PROT_USNIC)
676
677	struct ib_port_attr {
678	u64 subnet_prefix;
679	enum ib_port_state state;
680	enum ib_mtu max_mtu;
681	enum ib_mtu active_mtu;
682	u32 phys_mtu;
683	int gid_tbl_len;
684	unsigned int ip_gids:1;
685	/* This is the value from PortInfo CapabilityMask, defined by IBA */
686	u32 port_cap_flags;
687	u32 max_msg_sz;
688	u32 bad_pkey_cntr;
689	u32 qkey_viol_cntr;
690	u16 pkey_tbl_len;
691	u32 sm_lid;
692	u32 lid;
693	u8 lmc;
694	u8 max_vl_num;
695	u8 sm_sl;
696	u8 subnet_timeout;
697	u8 init_type_reply;
698	u8 active_width;
699	u16 active_speed;
700	u8 phys_state;
701	u16 port_cap_flags2;
702	};
703
704	enum ib_device_modify_flags {
705	IB_DEVICE_MODIFY_SYS_IMAGE_GUID = 1 << 0,
706	IB_DEVICE_MODIFY_NODE_DESC = 1 << 1
707	};
708
709	#define IB_DEVICE_NODE_DESC_MAX 64
710
711	struct ib_device_modify {
712	u64 sys_image_guid;
713	char node_desc[IB_DEVICE_NODE_DESC_MAX];
714	};
715
716	enum ib_port_modify_flags {
717	IB_PORT_SHUTDOWN = 1,
718	IB_PORT_INIT_TYPE = (1<<2),
719	IB_PORT_RESET_QKEY_CNTR = (1<<3),
720	IB_PORT_OPA_MASK_CHG = (1<<4)
721	};
722
723	struct ib_port_modify {
724	u32 set_port_cap_mask;
725	u32 clr_port_cap_mask;
726	u8 init_type;
727	};
728
729	enum ib_event_type {
730	IB_EVENT_CQ_ERR,
731	IB_EVENT_QP_FATAL,
732	IB_EVENT_QP_REQ_ERR,
733	IB_EVENT_QP_ACCESS_ERR,
734	IB_EVENT_COMM_EST,
735	IB_EVENT_SQ_DRAINED,
736	IB_EVENT_PATH_MIG,
737	IB_EVENT_PATH_MIG_ERR,
738	IB_EVENT_DEVICE_FATAL,
739	IB_EVENT_PORT_ACTIVE,
740	IB_EVENT_PORT_ERR,
741	IB_EVENT_LID_CHANGE,
742	IB_EVENT_PKEY_CHANGE,
743	IB_EVENT_SM_CHANGE,
744	IB_EVENT_SRQ_ERR,
745	IB_EVENT_SRQ_LIMIT_REACHED,
746	IB_EVENT_QP_LAST_WQE_REACHED,
747	IB_EVENT_CLIENT_REREGISTER,
748	IB_EVENT_GID_CHANGE,
749	IB_EVENT_WQ_FATAL,
750	};
751
752	const char *__attribute_const__ ib_event_msg(enum ib_event_type event);
753
754	struct ib_event {
755	struct ib_device *device;
756	union {
757	struct ib_cq *cq;
758	struct ib_qp *qp;
759	struct ib_srq *srq;
760	struct ib_wq *wq;
761	u32 port_num;
762	} element;
763	enum ib_event_type event;
764	};
765
766	struct ib_event_handler {
767	struct ib_device *device;
768	void (*handler)(struct ib_event_handler *, struct ib_event *);
769	struct list_head list;
770	};
771
772	#define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler) \
773	do { \
774	(_ptr)->device = _device; \
775	(_ptr)->handler = _handler; \
776	INIT_LIST_HEAD(&(_ptr)->list); \
777	} while (0)
778
779	struct ib_global_route {
780	const struct ib_gid_attr *sgid_attr;
781	union ib_gid dgid;
782	u32 flow_label;
783	u8 sgid_index;
784	u8 hop_limit;
785	u8 traffic_class;
786	};
787
788	struct ib_grh {
789	__be32 version_tclass_flow;
790	__be16 paylen;
791	u8 next_hdr;
792	u8 hop_limit;
793	union ib_gid sgid;
794	union ib_gid dgid;
795	};
796
797	union rdma_network_hdr {
798	struct ib_grh ibgrh;
799	struct {
800	/* The IB spec states that if it's IPv4, the header
801	* is located in the last 20 bytes of the header.
802	*/
803	u8 reserved[20];
804	struct iphdr roce4grh;
805	};
806	};
807
808	#define IB_QPN_MASK 0xFFFFFF
809
810	enum {
811	IB_MULTICAST_QPN = 0xffffff
812	};
813
814	#define IB_LID_PERMISSIVE cpu_to_be16(0xFFFF)
815	#define IB_MULTICAST_LID_BASE cpu_to_be16(0xC000)
816
817	enum ib_ah_flags {
818	IB_AH_GRH = 1
819	};
820
821	enum ib_rate {
822	IB_RATE_PORT_CURRENT = 0,
823	IB_RATE_2_5_GBPS = 2,
824	IB_RATE_5_GBPS = 5,
825	IB_RATE_10_GBPS = 3,
826	IB_RATE_20_GBPS = 6,
827	IB_RATE_30_GBPS = 4,
828	IB_RATE_40_GBPS = 7,
829	IB_RATE_60_GBPS = 8,
830	IB_RATE_80_GBPS = 9,
831	IB_RATE_120_GBPS = 10,
832	IB_RATE_14_GBPS = 11,
833	IB_RATE_56_GBPS = 12,
834	IB_RATE_112_GBPS = 13,
835	IB_RATE_168_GBPS = 14,
836	IB_RATE_25_GBPS = 15,
837	IB_RATE_100_GBPS = 16,
838	IB_RATE_200_GBPS = 17,
839	IB_RATE_300_GBPS = 18,
840	IB_RATE_28_GBPS = 19,
841	IB_RATE_50_GBPS = 20,
842	IB_RATE_400_GBPS = 21,
843	IB_RATE_600_GBPS = 22,
844	IB_RATE_800_GBPS = 23,
845	};
846
847	/**
848	* ib_rate_to_mult - Convert the IB rate enum to a multiple of the
849	* base rate of 2.5 Gbit/sec. For example, IB_RATE_5_GBPS will be
850	* converted to 2, since 5 Gbit/sec is 2 * 2.5 Gbit/sec.
851	* @rate: rate to convert.
852	*/
853	__attribute_const__ int ib_rate_to_mult(enum ib_rate rate);
854
855	/**
856	* ib_rate_to_mbps - Convert the IB rate enum to Mbps.
857	* For example, IB_RATE_2_5_GBPS will be converted to 2500.
858	* @rate: rate to convert.
859	*/
860	__attribute_const__ int ib_rate_to_mbps(enum ib_rate rate);
861
862
863	/**
864	* enum ib_mr_type - memory region type
865	* @IB_MR_TYPE_MEM_REG: memory region that is used for
866	* normal registration
867	* @IB_MR_TYPE_SG_GAPS: memory region that is capable to
868	* register any arbitrary sg lists (without
869	* the normal mr constraints - see
870	* ib_map_mr_sg)
871	* @IB_MR_TYPE_DM: memory region that is used for device
872	* memory registration
873	* @IB_MR_TYPE_USER: memory region that is used for the user-space
874	* application
875	* @IB_MR_TYPE_DMA: memory region that is used for DMA operations
876	* without address translations (VA=PA)
877	* @IB_MR_TYPE_INTEGRITY: memory region that is used for
878	* data integrity operations
879	*/
880	enum ib_mr_type {
881	IB_MR_TYPE_MEM_REG,
882	IB_MR_TYPE_SG_GAPS,
883	IB_MR_TYPE_DM,
884	IB_MR_TYPE_USER,
885	IB_MR_TYPE_DMA,
886	IB_MR_TYPE_INTEGRITY,
887	};
888
889	enum ib_mr_status_check {
890	IB_MR_CHECK_SIG_STATUS = 1,
891	};
892
893	/**
894	* struct ib_mr_status - Memory region status container
895	*
896	* @fail_status: Bitmask of MR checks status. For each
897	* failed check a corresponding status bit is set.
898	* @sig_err: Additional info for IB_MR_CEHCK_SIG_STATUS
899	* failure.
900	*/
901	struct ib_mr_status {
902	u32 fail_status;
903	struct ib_sig_err sig_err;
904	};
905
906	/**
907	* mult_to_ib_rate - Convert a multiple of 2.5 Gbit/sec to an IB rate
908	* enum.
909	* @mult: multiple to convert.
910	*/
911	__attribute_const__ enum ib_rate mult_to_ib_rate(int mult);
912
913	struct rdma_ah_init_attr {
914	struct rdma_ah_attr *ah_attr;
915	u32 flags;
916	struct net_device *xmit_slave;
917	};
918
919	enum rdma_ah_attr_type {
920	RDMA_AH_ATTR_TYPE_UNDEFINED,
921	RDMA_AH_ATTR_TYPE_IB,
922	RDMA_AH_ATTR_TYPE_ROCE,
923	RDMA_AH_ATTR_TYPE_OPA,
924	};
925
926	struct ib_ah_attr {
927	u16 dlid;
928	u8 src_path_bits;
929	};
930
931	struct roce_ah_attr {
932	u8 dmac[ETH_ALEN];
933	};
934
935	struct opa_ah_attr {
936	u32 dlid;
937	u8 src_path_bits;
938	bool make_grd;
939	};
940
941	struct rdma_ah_attr {
942	struct ib_global_route grh;
943	u8 sl;
944	u8 static_rate;
945	u32 port_num;
946	u8 ah_flags;
947	enum rdma_ah_attr_type type;
948	union {
949	struct ib_ah_attr ib;
950	struct roce_ah_attr roce;
951	struct opa_ah_attr opa;
952	};
953	};
954
955	enum ib_wc_status {
956	IB_WC_SUCCESS,
957	IB_WC_LOC_LEN_ERR,
958	IB_WC_LOC_QP_OP_ERR,
959	IB_WC_LOC_EEC_OP_ERR,
960	IB_WC_LOC_PROT_ERR,
961	IB_WC_WR_FLUSH_ERR,
962	IB_WC_MW_BIND_ERR,
963	IB_WC_BAD_RESP_ERR,
964	IB_WC_LOC_ACCESS_ERR,
965	IB_WC_REM_INV_REQ_ERR,
966	IB_WC_REM_ACCESS_ERR,
967	IB_WC_REM_OP_ERR,
968	IB_WC_RETRY_EXC_ERR,
969	IB_WC_RNR_RETRY_EXC_ERR,
970	IB_WC_LOC_RDD_VIOL_ERR,
971	IB_WC_REM_INV_RD_REQ_ERR,
972	IB_WC_REM_ABORT_ERR,
973	IB_WC_INV_EECN_ERR,
974	IB_WC_INV_EEC_STATE_ERR,
975	IB_WC_FATAL_ERR,
976	IB_WC_RESP_TIMEOUT_ERR,
977	IB_WC_GENERAL_ERR
978	};
979
980	const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status);
981
982	enum ib_wc_opcode {
983	IB_WC_SEND = IB_UVERBS_WC_SEND,
984	IB_WC_RDMA_WRITE = IB_UVERBS_WC_RDMA_WRITE,
985	IB_WC_RDMA_READ = IB_UVERBS_WC_RDMA_READ,
986	IB_WC_COMP_SWAP = IB_UVERBS_WC_COMP_SWAP,
987	IB_WC_FETCH_ADD = IB_UVERBS_WC_FETCH_ADD,
988	IB_WC_BIND_MW = IB_UVERBS_WC_BIND_MW,
989	IB_WC_LOCAL_INV = IB_UVERBS_WC_LOCAL_INV,
990	IB_WC_LSO = IB_UVERBS_WC_TSO,
991	IB_WC_ATOMIC_WRITE = IB_UVERBS_WC_ATOMIC_WRITE,
992	IB_WC_REG_MR,
993	IB_WC_MASKED_COMP_SWAP,
994	IB_WC_MASKED_FETCH_ADD,
995	IB_WC_FLUSH = IB_UVERBS_WC_FLUSH,
996	/*
997	* Set value of IB_WC_RECV so consumers can test if a completion is a
998	* receive by testing (opcode & IB_WC_RECV).
999	*/
1000	IB_WC_RECV = 1 << 7,
1001	IB_WC_RECV_RDMA_WITH_IMM
1002	};
1003
1004	enum ib_wc_flags {
1005	IB_WC_GRH = 1,
1006	IB_WC_WITH_IMM = (1<<1),
1007	IB_WC_WITH_INVALIDATE = (1<<2),
1008	IB_WC_IP_CSUM_OK = (1<<3),
1009	IB_WC_WITH_SMAC = (1<<4),
1010	IB_WC_WITH_VLAN = (1<<5),
1011	IB_WC_WITH_NETWORK_HDR_TYPE = (1<<6),
1012	};
1013
1014	struct ib_wc {
1015	union {
1016	u64 wr_id;
1017	struct ib_cqe *wr_cqe;
1018	};
1019	enum ib_wc_status status;
1020	enum ib_wc_opcode opcode;
1021	u32 vendor_err;
1022	u32 byte_len;
1023	struct ib_qp *qp;
1024	union {
1025	__be32 imm_data;
1026	u32 invalidate_rkey;
1027	} ex;
1028	u32 src_qp;
1029	u32 slid;
1030	int wc_flags;
1031	u16 pkey_index;
1032	u8 sl;
1033	u8 dlid_path_bits;
1034	u32 port_num; /* valid only for DR SMPs on switches */
1035	u8 smac[ETH_ALEN];
1036	u16 vlan_id;
1037	u8 network_hdr_type;
1038	};
1039
1040	enum ib_cq_notify_flags {
1041	IB_CQ_SOLICITED = 1 << 0,
1042	IB_CQ_NEXT_COMP = 1 << 1,
1043	IB_CQ_SOLICITED_MASK = IB_CQ_SOLICITED | IB_CQ_NEXT_COMP,
1044	IB_CQ_REPORT_MISSED_EVENTS = 1 << 2,
1045	};
1046
1047	enum ib_srq_type {
1048	IB_SRQT_BASIC = IB_UVERBS_SRQT_BASIC,
1049	IB_SRQT_XRC = IB_UVERBS_SRQT_XRC,
1050	IB_SRQT_TM = IB_UVERBS_SRQT_TM,
1051	};
1052
1053	static inline bool ib_srq_has_cq(enum ib_srq_type srq_type)
1054	{
1055	return srq_type == IB_SRQT_XRC ||
1056	srq_type == IB_SRQT_TM;
1057	}
1058
1059	enum ib_srq_attr_mask {
1060	IB_SRQ_MAX_WR = 1 << 0,
1061	IB_SRQ_LIMIT = 1 << 1,
1062	};
1063
1064	struct ib_srq_attr {
1065	u32 max_wr;
1066	u32 max_sge;
1067	u32 srq_limit;
1068	};
1069
1070	struct ib_srq_init_attr {
1071	void (*event_handler)(struct ib_event *, void *);
1072	void *srq_context;
1073	struct ib_srq_attr attr;
1074	enum ib_srq_type srq_type;
1075
1076	struct {
1077	struct ib_cq *cq;
1078	union {
1079	struct {
1080	struct ib_xrcd *xrcd;
1081	} xrc;
1082
1083	struct {
1084	u32 max_num_tags;
1085	} tag_matching;
1086	};
1087	} ext;
1088	};
1089
1090	struct ib_qp_cap {
1091	u32 max_send_wr;
1092	u32 max_recv_wr;
1093	u32 max_send_sge;
1094	u32 max_recv_sge;
1095	u32 max_inline_data;
1096
1097	/*
1098	* Maximum number of rdma_rw_ctx structures in flight at a time.
1099	* ib_create_qp() will calculate the right amount of needed WRs
1100	* and MRs based on this.
1101	*/
1102	u32 max_rdma_ctxs;
1103	};
1104
1105	enum ib_sig_type {
1106	IB_SIGNAL_ALL_WR,
1107	IB_SIGNAL_REQ_WR
1108	};
1109
1110	enum ib_qp_type {
1111	/*
1112	* IB_QPT_SMI and IB_QPT_GSI have to be the first two entries
1113	* here (and in that order) since the MAD layer uses them as
1114	* indices into a 2-entry table.
1115	*/
1116	IB_QPT_SMI,
1117	IB_QPT_GSI,
1118
1119	IB_QPT_RC = IB_UVERBS_QPT_RC,
1120	IB_QPT_UC = IB_UVERBS_QPT_UC,
1121	IB_QPT_UD = IB_UVERBS_QPT_UD,
1122	IB_QPT_RAW_IPV6,
1123	IB_QPT_RAW_ETHERTYPE,
1124	IB_QPT_RAW_PACKET = IB_UVERBS_QPT_RAW_PACKET,
1125	IB_QPT_XRC_INI = IB_UVERBS_QPT_XRC_INI,
1126	IB_QPT_XRC_TGT = IB_UVERBS_QPT_XRC_TGT,
1127	IB_QPT_MAX,
1128	IB_QPT_DRIVER = IB_UVERBS_QPT_DRIVER,
1129	/* Reserve a range for qp types internal to the low level driver.
1130	* These qp types will not be visible at the IB core layer, so the
1131	* IB_QPT_MAX usages should not be affected in the core layer
1132	*/
1133	IB_QPT_RESERVED1 = 0x1000,
1134	IB_QPT_RESERVED2,
1135	IB_QPT_RESERVED3,
1136	IB_QPT_RESERVED4,
1137	IB_QPT_RESERVED5,
1138	IB_QPT_RESERVED6,
1139	IB_QPT_RESERVED7,
1140	IB_QPT_RESERVED8,
1141	IB_QPT_RESERVED9,
1142	IB_QPT_RESERVED10,
1143	};
1144
1145	enum ib_qp_create_flags {
1146	IB_QP_CREATE_IPOIB_UD_LSO = 1 << 0,
1147	IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK =
1148	IB_UVERBS_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
1149	IB_QP_CREATE_CROSS_CHANNEL = 1 << 2,
1150	IB_QP_CREATE_MANAGED_SEND = 1 << 3,
1151	IB_QP_CREATE_MANAGED_RECV = 1 << 4,
1152	IB_QP_CREATE_NETIF_QP = 1 << 5,
1153	IB_QP_CREATE_INTEGRITY_EN = 1 << 6,
1154	IB_QP_CREATE_NETDEV_USE = 1 << 7,
1155	IB_QP_CREATE_SCATTER_FCS =
1156	IB_UVERBS_QP_CREATE_SCATTER_FCS,
1157	IB_QP_CREATE_CVLAN_STRIPPING =
1158	IB_UVERBS_QP_CREATE_CVLAN_STRIPPING,
1159	IB_QP_CREATE_SOURCE_QPN = 1 << 10,
1160	IB_QP_CREATE_PCI_WRITE_END_PADDING =
1161	IB_UVERBS_QP_CREATE_PCI_WRITE_END_PADDING,
1162	/* reserve bits 26-31 for low level drivers' internal use */
1163	IB_QP_CREATE_RESERVED_START = 1 << 26,
1164	IB_QP_CREATE_RESERVED_END = 1 << 31,
1165	};
1166
1167	/*
1168	* Note: users may not call ib_close_qp or ib_destroy_qp from the event_handler
1169	* callback to destroy the passed in QP.
1170	*/
1171
1172	struct ib_qp_init_attr {
1173	/* This callback occurs in workqueue context */
1174	void (*event_handler)(struct ib_event *, void *);
1175
1176	void *qp_context;
1177	struct ib_cq *send_cq;
1178	struct ib_cq *recv_cq;
1179	struct ib_srq *srq;
1180	struct ib_xrcd *xrcd; /* XRC TGT QPs only */
1181	struct ib_qp_cap cap;
1182	enum ib_sig_type sq_sig_type;
1183	enum ib_qp_type qp_type;
1184	u32 create_flags;
1185
1186	/*
1187	* Only needed for special QP types, or when using the RW API.
1188	*/
1189	u32 port_num;
1190	struct ib_rwq_ind_table *rwq_ind_tbl;
1191	u32 source_qpn;
1192	};
1193
1194	struct ib_qp_open_attr {
1195	void (*event_handler)(struct ib_event *, void *);
1196	void *qp_context;
1197	u32 qp_num;
1198	enum ib_qp_type qp_type;
1199	};
1200
1201	enum ib_rnr_timeout {
1202	IB_RNR_TIMER_655_36 = 0,
1203	IB_RNR_TIMER_000_01 = 1,
1204	IB_RNR_TIMER_000_02 = 2,
1205	IB_RNR_TIMER_000_03 = 3,
1206	IB_RNR_TIMER_000_04 = 4,
1207	IB_RNR_TIMER_000_06 = 5,
1208	IB_RNR_TIMER_000_08 = 6,
1209	IB_RNR_TIMER_000_12 = 7,
1210	IB_RNR_TIMER_000_16 = 8,
1211	IB_RNR_TIMER_000_24 = 9,
1212	IB_RNR_TIMER_000_32 = 10,
1213	IB_RNR_TIMER_000_48 = 11,
1214	IB_RNR_TIMER_000_64 = 12,
1215	IB_RNR_TIMER_000_96 = 13,
1216	IB_RNR_TIMER_001_28 = 14,
1217	IB_RNR_TIMER_001_92 = 15,
1218	IB_RNR_TIMER_002_56 = 16,
1219	IB_RNR_TIMER_003_84 = 17,
1220	IB_RNR_TIMER_005_12 = 18,
1221	IB_RNR_TIMER_007_68 = 19,
1222	IB_RNR_TIMER_010_24 = 20,
1223	IB_RNR_TIMER_015_36 = 21,
1224	IB_RNR_TIMER_020_48 = 22,
1225	IB_RNR_TIMER_030_72 = 23,
1226	IB_RNR_TIMER_040_96 = 24,
1227	IB_RNR_TIMER_061_44 = 25,
1228	IB_RNR_TIMER_081_92 = 26,
1229	IB_RNR_TIMER_122_88 = 27,
1230	IB_RNR_TIMER_163_84 = 28,
1231	IB_RNR_TIMER_245_76 = 29,
1232	IB_RNR_TIMER_327_68 = 30,
1233	IB_RNR_TIMER_491_52 = 31
1234	};
1235
1236	enum ib_qp_attr_mask {
1237	IB_QP_STATE = 1,
1238	IB_QP_CUR_STATE = (1<<1),
1239	IB_QP_EN_SQD_ASYNC_NOTIFY = (1<<2),
1240	IB_QP_ACCESS_FLAGS = (1<<3),
1241	IB_QP_PKEY_INDEX = (1<<4),
1242	IB_QP_PORT = (1<<5),
1243	IB_QP_QKEY = (1<<6),
1244	IB_QP_AV = (1<<7),
1245	IB_QP_PATH_MTU = (1<<8),
1246	IB_QP_TIMEOUT = (1<<9),
1247	IB_QP_RETRY_CNT = (1<<10),
1248	IB_QP_RNR_RETRY = (1<<11),
1249	IB_QP_RQ_PSN = (1<<12),
1250	IB_QP_MAX_QP_RD_ATOMIC = (1<<13),
1251	IB_QP_ALT_PATH = (1<<14),
1252	IB_QP_MIN_RNR_TIMER = (1<<15),
1253	IB_QP_SQ_PSN = (1<<16),
1254	IB_QP_MAX_DEST_RD_ATOMIC = (1<<17),
1255	IB_QP_PATH_MIG_STATE = (1<<18),
1256	IB_QP_CAP = (1<<19),
1257	IB_QP_DEST_QPN = (1<<20),
1258	IB_QP_RESERVED1 = (1<<21),
1259	IB_QP_RESERVED2 = (1<<22),
1260	IB_QP_RESERVED3 = (1<<23),
1261	IB_QP_RESERVED4 = (1<<24),
1262	IB_QP_RATE_LIMIT = (1<<25),
1263
1264	IB_QP_ATTR_STANDARD_BITS = GENMASK(20, 0),
1265	};
1266
1267	enum ib_qp_state {
1268	IB_QPS_RESET,
1269	IB_QPS_INIT,
1270	IB_QPS_RTR,
1271	IB_QPS_RTS,
1272	IB_QPS_SQD,
1273	IB_QPS_SQE,
1274	IB_QPS_ERR
1275	};
1276
1277	enum ib_mig_state {
1278	IB_MIG_MIGRATED,
1279	IB_MIG_REARM,
1280	IB_MIG_ARMED
1281	};
1282
1283	enum ib_mw_type {
1284	IB_MW_TYPE_1 = 1,
1285	IB_MW_TYPE_2 = 2
1286	};
1287
1288	struct ib_qp_attr {
1289	enum ib_qp_state qp_state;
1290	enum ib_qp_state cur_qp_state;
1291	enum ib_mtu path_mtu;
1292	enum ib_mig_state path_mig_state;
1293	u32 qkey;
1294	u32 rq_psn;
1295	u32 sq_psn;
1296	u32 dest_qp_num;
1297	int qp_access_flags;
1298	struct ib_qp_cap cap;
1299	struct rdma_ah_attr ah_attr;
1300	struct rdma_ah_attr alt_ah_attr;
1301	u16 pkey_index;
1302	u16 alt_pkey_index;
1303	u8 en_sqd_async_notify;
1304	u8 sq_draining;
1305	u8 max_rd_atomic;
1306	u8 max_dest_rd_atomic;
1307	u8 min_rnr_timer;
1308	u32 port_num;
1309	u8 timeout;
1310	u8 retry_cnt;
1311	u8 rnr_retry;
1312	u32 alt_port_num;
1313	u8 alt_timeout;
1314	u32 rate_limit;
1315	struct net_device *xmit_slave;
1316	};
1317
1318	enum ib_wr_opcode {
1319	/* These are shared with userspace */
1320	IB_WR_RDMA_WRITE = IB_UVERBS_WR_RDMA_WRITE,
1321	IB_WR_RDMA_WRITE_WITH_IMM = IB_UVERBS_WR_RDMA_WRITE_WITH_IMM,
1322	IB_WR_SEND = IB_UVERBS_WR_SEND,
1323	IB_WR_SEND_WITH_IMM = IB_UVERBS_WR_SEND_WITH_IMM,
1324	IB_WR_RDMA_READ = IB_UVERBS_WR_RDMA_READ,
1325	IB_WR_ATOMIC_CMP_AND_SWP = IB_UVERBS_WR_ATOMIC_CMP_AND_SWP,
1326	IB_WR_ATOMIC_FETCH_AND_ADD = IB_UVERBS_WR_ATOMIC_FETCH_AND_ADD,
1327	IB_WR_BIND_MW = IB_UVERBS_WR_BIND_MW,
1328	IB_WR_LSO = IB_UVERBS_WR_TSO,
1329	IB_WR_SEND_WITH_INV = IB_UVERBS_WR_SEND_WITH_INV,
1330	IB_WR_RDMA_READ_WITH_INV = IB_UVERBS_WR_RDMA_READ_WITH_INV,
1331	IB_WR_LOCAL_INV = IB_UVERBS_WR_LOCAL_INV,
1332	IB_WR_MASKED_ATOMIC_CMP_AND_SWP =
1333	IB_UVERBS_WR_MASKED_ATOMIC_CMP_AND_SWP,
1334	IB_WR_MASKED_ATOMIC_FETCH_AND_ADD =
1335	IB_UVERBS_WR_MASKED_ATOMIC_FETCH_AND_ADD,
1336	IB_WR_FLUSH = IB_UVERBS_WR_FLUSH,
1337	IB_WR_ATOMIC_WRITE = IB_UVERBS_WR_ATOMIC_WRITE,
1338
1339	/* These are kernel only and can not be issued by userspace */
1340	IB_WR_REG_MR = 0x20,
1341	IB_WR_REG_MR_INTEGRITY,
1342
1343	/* reserve values for low level drivers' internal use.
1344	* These values will not be used at all in the ib core layer.
1345	*/
1346	IB_WR_RESERVED1 = 0xf0,
1347	IB_WR_RESERVED2,
1348	IB_WR_RESERVED3,
1349	IB_WR_RESERVED4,
1350	IB_WR_RESERVED5,
1351	IB_WR_RESERVED6,
1352	IB_WR_RESERVED7,
1353	IB_WR_RESERVED8,
1354	IB_WR_RESERVED9,
1355	IB_WR_RESERVED10,
1356	};
1357
1358	enum ib_send_flags {
1359	IB_SEND_FENCE = 1,
1360	IB_SEND_SIGNALED = (1<<1),
1361	IB_SEND_SOLICITED = (1<<2),
1362	IB_SEND_INLINE = (1<<3),
1363	IB_SEND_IP_CSUM = (1<<4),
1364
1365	/* reserve bits 26-31 for low level drivers' internal use */
1366	IB_SEND_RESERVED_START = (1 << 26),
1367	IB_SEND_RESERVED_END = (1 << 31),
1368	};
1369
1370	struct ib_sge {
1371	u64 addr;
1372	u32 length;
1373	u32 lkey;
1374	};
1375
1376	struct ib_cqe {
1377	void (*done)(struct ib_cq *cq, struct ib_wc *wc);
1378	};
1379
1380	struct ib_send_wr {
1381	struct ib_send_wr *next;
1382	union {
1383	u64 wr_id;
1384	struct ib_cqe *wr_cqe;
1385	};
1386	struct ib_sge *sg_list;
1387	int num_sge;
1388	enum ib_wr_opcode opcode;
1389	int send_flags;
1390	union {
1391	__be32 imm_data;
1392	u32 invalidate_rkey;
1393	} ex;
1394	};
1395
1396	struct ib_rdma_wr {
1397	struct ib_send_wr wr;
1398	u64 remote_addr;
1399	u32 rkey;
1400	};
1401
1402	static inline const struct ib_rdma_wr *rdma_wr(const struct ib_send_wr *wr)
1403	{
1404	return container_of(wr, struct ib_rdma_wr, wr);
1405	}
1406
1407	struct ib_atomic_wr {
1408	struct ib_send_wr wr;
1409	u64 remote_addr;
1410	u64 compare_add;
1411	u64 swap;
1412	u64 compare_add_mask;
1413	u64 swap_mask;
1414	u32 rkey;
1415	};
1416
1417	static inline const struct ib_atomic_wr *atomic_wr(const struct ib_send_wr *wr)
1418	{
1419	return container_of(wr, struct ib_atomic_wr, wr);
1420	}
1421
1422	struct ib_ud_wr {
1423	struct ib_send_wr wr;
1424	struct ib_ah *ah;
1425	void *header;
1426	int hlen;
1427	int mss;
1428	u32 remote_qpn;
1429	u32 remote_qkey;
1430	u16 pkey_index; /* valid for GSI only */
1431	u32 port_num; /* valid for DR SMPs on switch only */
1432	};
1433
1434	static inline const struct ib_ud_wr *ud_wr(const struct ib_send_wr *wr)
1435	{
1436	return container_of(wr, struct ib_ud_wr, wr);
1437	}
1438
1439	struct ib_reg_wr {
1440	struct ib_send_wr wr;
1441	struct ib_mr *mr;
1442	u32 key;
1443	int access;
1444	};
1445
1446	static inline const struct ib_reg_wr *reg_wr(const struct ib_send_wr *wr)
1447	{
1448	return container_of(wr, struct ib_reg_wr, wr);
1449	}
1450
1451	struct ib_recv_wr {
1452	struct ib_recv_wr *next;
1453	union {
1454	u64 wr_id;
1455	struct ib_cqe *wr_cqe;
1456	};
1457	struct ib_sge *sg_list;
1458	int num_sge;
1459	};
1460
1461	enum ib_access_flags {
1462	IB_ACCESS_LOCAL_WRITE = IB_UVERBS_ACCESS_LOCAL_WRITE,
1463	IB_ACCESS_REMOTE_WRITE = IB_UVERBS_ACCESS_REMOTE_WRITE,
1464	IB_ACCESS_REMOTE_READ = IB_UVERBS_ACCESS_REMOTE_READ,
1465	IB_ACCESS_REMOTE_ATOMIC = IB_UVERBS_ACCESS_REMOTE_ATOMIC,
1466	IB_ACCESS_MW_BIND = IB_UVERBS_ACCESS_MW_BIND,
1467	IB_ZERO_BASED = IB_UVERBS_ACCESS_ZERO_BASED,
1468	IB_ACCESS_ON_DEMAND = IB_UVERBS_ACCESS_ON_DEMAND,
1469	IB_ACCESS_HUGETLB = IB_UVERBS_ACCESS_HUGETLB,
1470	IB_ACCESS_RELAXED_ORDERING = IB_UVERBS_ACCESS_RELAXED_ORDERING,
1471	IB_ACCESS_FLUSH_GLOBAL = IB_UVERBS_ACCESS_FLUSH_GLOBAL,
1472	IB_ACCESS_FLUSH_PERSISTENT = IB_UVERBS_ACCESS_FLUSH_PERSISTENT,
1473
1474	IB_ACCESS_OPTIONAL = IB_UVERBS_ACCESS_OPTIONAL_RANGE,
1475	IB_ACCESS_SUPPORTED =
1476	((IB_ACCESS_FLUSH_PERSISTENT << 1) - 1) | IB_ACCESS_OPTIONAL,
1477	};
1478
1479	/*
1480	* XXX: these are apparently used for ->rereg_user_mr, no idea why they
1481	* are hidden here instead of a uapi header!
1482	*/
1483	enum ib_mr_rereg_flags {
1484	IB_MR_REREG_TRANS = 1,
1485	IB_MR_REREG_PD = (1<<1),
1486	IB_MR_REREG_ACCESS = (1<<2),
1487	IB_MR_REREG_SUPPORTED = ((IB_MR_REREG_ACCESS << 1) - 1)
1488	};
1489
1490	struct ib_umem;
1491
1492	enum rdma_remove_reason {
1493	/*
1494	* Userspace requested uobject deletion or initial try
1495	* to remove uobject via cleanup. Call could fail
1496	*/
1497	RDMA_REMOVE_DESTROY,
1498	/* Context deletion. This call should delete the actual object itself */
1499	RDMA_REMOVE_CLOSE,
1500	/* Driver is being hot-unplugged. This call should delete the actual object itself */
1501	RDMA_REMOVE_DRIVER_REMOVE,
1502	/* uobj is being cleaned-up before being committed */
1503	RDMA_REMOVE_ABORT,
1504	/* The driver failed to destroy the uobject and is being disconnected */
1505	RDMA_REMOVE_DRIVER_FAILURE,
1506	};
1507
1508	struct ib_rdmacg_object {
1509	#ifdef CONFIG_CGROUP_RDMA
1510	struct rdma_cgroup *cg; /* owner rdma cgroup */
1511	#endif
1512	};
1513
1514	struct ib_ucontext {
1515	struct ib_device *device;
1516	struct ib_uverbs_file *ufile;
1517
1518	struct ib_rdmacg_object cg_obj;
1519	/*
1520	* Implementation details of the RDMA core, don't use in drivers:
1521	*/
1522	struct rdma_restrack_entry res;
1523	struct xarray mmap_xa;
1524	};
1525
1526	struct ib_uobject {
1527	u64 user_handle; /* handle given to us by userspace */
1528	/* ufile & ucontext owning this object */
1529	struct ib_uverbs_file *ufile;
1530	/* FIXME, save memory: ufile->context == context */
1531	struct ib_ucontext *context; /* associated user context */
1532	void *object; /* containing object */
1533	struct list_head list; /* link to context's list */
1534	struct ib_rdmacg_object cg_obj; /* rdmacg object */
1535	int id; /* index into kernel idr */
1536	struct kref ref;
1537	atomic_t usecnt; /* protects exclusive access */
1538	struct rcu_head rcu; /* kfree_rcu() overhead */
1539
1540	const struct uverbs_api_object *uapi_object;
1541	};
1542
1543	struct ib_udata {
1544	const void __user *inbuf;
1545	void __user *outbuf;
1546	size_t inlen;
1547	size_t outlen;
1548	};
1549
1550	struct ib_pd {
1551	u32 local_dma_lkey;
1552	u32 flags;
1553	struct ib_device *device;
1554	struct ib_uobject *uobject;
1555	atomic_t usecnt; /* count all resources */
1556
1557	u32 unsafe_global_rkey;
1558
1559	/*
1560	* Implementation details of the RDMA core, don't use in drivers:
1561	*/
1562	struct ib_mr *__internal_mr;
1563	struct rdma_restrack_entry res;
1564	};
1565
1566	struct ib_xrcd {
1567	struct ib_device *device;
1568	atomic_t usecnt; /* count all exposed resources */
1569	struct inode *inode;
1570	struct rw_semaphore tgt_qps_rwsem;
1571	struct xarray tgt_qps;
1572	};
1573
1574	struct ib_ah {
1575	struct ib_device *device;
1576	struct ib_pd *pd;
1577	struct ib_uobject *uobject;
1578	const struct ib_gid_attr *sgid_attr;
1579	enum rdma_ah_attr_type type;
1580	};
1581
1582	typedef void (*ib_comp_handler)(struct ib_cq *cq, void *cq_context);
1583
1584	enum ib_poll_context {
1585	IB_POLL_SOFTIRQ, /* poll from softirq context */
1586	IB_POLL_WORKQUEUE, /* poll from workqueue */
1587	IB_POLL_UNBOUND_WORKQUEUE, /* poll from unbound workqueue */
1588	IB_POLL_LAST_POOL_TYPE = IB_POLL_UNBOUND_WORKQUEUE,
1589
1590	IB_POLL_DIRECT, /* caller context, no hw completions */
1591	};
1592
1593	struct ib_cq {
1594	struct ib_device *device;
1595	struct ib_ucq_object *uobject;
1596	ib_comp_handler comp_handler;
1597	void (*event_handler)(struct ib_event *, void *);
1598	void *cq_context;
1599	int cqe;
1600	unsigned int cqe_used;
1601	atomic_t usecnt; /* count number of work queues */
1602	enum ib_poll_context poll_ctx;
1603	struct ib_wc *wc;
1604	struct list_head pool_entry;
1605	union {
1606	struct irq_poll iop;
1607	struct work_struct work;
1608	};
1609	struct workqueue_struct *comp_wq;
1610	struct dim *dim;
1611
1612	/* updated only by trace points */
1613	ktime_t timestamp;
1614	u8 interrupt:1;
1615	u8 shared:1;
1616	unsigned int comp_vector;
1617
1618	/*
1619	* Implementation details of the RDMA core, don't use in drivers:
1620	*/
1621	struct rdma_restrack_entry res;
1622	};
1623
1624	struct ib_srq {
1625	struct ib_device *device;
1626	struct ib_pd *pd;
1627	struct ib_usrq_object *uobject;
1628	void (*event_handler)(struct ib_event *, void *);
1629	void *srq_context;
1630	enum ib_srq_type srq_type;
1631	atomic_t usecnt;
1632
1633	struct {
1634	struct ib_cq *cq;
1635	union {
1636	struct {
1637	struct ib_xrcd *xrcd;
1638	u32 srq_num;
1639	} xrc;
1640	};
1641	} ext;
1642
1643	/*
1644	* Implementation details of the RDMA core, don't use in drivers:
1645	*/
1646	struct rdma_restrack_entry res;
1647	};
1648
1649	enum ib_raw_packet_caps {
1650	/*
1651	* Strip cvlan from incoming packet and report it in the matching work
1652	* completion is supported.
1653	*/
1654	IB_RAW_PACKET_CAP_CVLAN_STRIPPING =
1655	IB_UVERBS_RAW_PACKET_CAP_CVLAN_STRIPPING,
1656	/*
1657	* Scatter FCS field of an incoming packet to host memory is supported.
1658	*/
1659	IB_RAW_PACKET_CAP_SCATTER_FCS = IB_UVERBS_RAW_PACKET_CAP_SCATTER_FCS,
1660	/* Checksum offloads are supported (for both send and receive). */
1661	IB_RAW_PACKET_CAP_IP_CSUM = IB_UVERBS_RAW_PACKET_CAP_IP_CSUM,
1662	/*
1663	* When a packet is received for an RQ with no receive WQEs, the
1664	* packet processing is delayed.
1665	*/
1666	IB_RAW_PACKET_CAP_DELAY_DROP = IB_UVERBS_RAW_PACKET_CAP_DELAY_DROP,
1667	};
1668
1669	enum ib_wq_type {
1670	IB_WQT_RQ = IB_UVERBS_WQT_RQ,
1671	};
1672
1673	enum ib_wq_state {
1674	IB_WQS_RESET,
1675	IB_WQS_RDY,
1676	IB_WQS_ERR
1677	};
1678
1679	struct ib_wq {
1680	struct ib_device *device;
1681	struct ib_uwq_object *uobject;
1682	void *wq_context;
1683	void (*event_handler)(struct ib_event *, void *);
1684	struct ib_pd *pd;
1685	struct ib_cq *cq;
1686	u32 wq_num;
1687	enum ib_wq_state state;
1688	enum ib_wq_type wq_type;
1689	atomic_t usecnt;
1690	};
1691
1692	enum ib_wq_flags {
1693	IB_WQ_FLAGS_CVLAN_STRIPPING = IB_UVERBS_WQ_FLAGS_CVLAN_STRIPPING,
1694	IB_WQ_FLAGS_SCATTER_FCS = IB_UVERBS_WQ_FLAGS_SCATTER_FCS,
1695	IB_WQ_FLAGS_DELAY_DROP = IB_UVERBS_WQ_FLAGS_DELAY_DROP,
1696	IB_WQ_FLAGS_PCI_WRITE_END_PADDING =
1697	IB_UVERBS_WQ_FLAGS_PCI_WRITE_END_PADDING,
1698	};
1699
1700	struct ib_wq_init_attr {
1701	void *wq_context;
1702	enum ib_wq_type wq_type;
1703	u32 max_wr;
1704	u32 max_sge;
1705	struct ib_cq *cq;
1706	void (*event_handler)(struct ib_event *, void *);
1707	u32 create_flags; /* Use enum ib_wq_flags */
1708	};
1709
1710	enum ib_wq_attr_mask {
1711	IB_WQ_STATE = 1 << 0,
1712	IB_WQ_CUR_STATE = 1 << 1,
1713	IB_WQ_FLAGS = 1 << 2,
1714	};
1715
1716	struct ib_wq_attr {
1717	enum ib_wq_state wq_state;
1718	enum ib_wq_state curr_wq_state;
1719	u32 flags; /* Use enum ib_wq_flags */
1720	u32 flags_mask; /* Use enum ib_wq_flags */
1721	};
1722
1723	struct ib_rwq_ind_table {
1724	struct ib_device *device;
1725	struct ib_uobject *uobject;
1726	atomic_t usecnt;
1727	u32 ind_tbl_num;
1728	u32 log_ind_tbl_size;
1729	struct ib_wq **ind_tbl;
1730	};
1731
1732	struct ib_rwq_ind_table_init_attr {
1733	u32 log_ind_tbl_size;
1734	/* Each entry is a pointer to Receive Work Queue */
1735	struct ib_wq **ind_tbl;
1736	};
1737
1738	enum port_pkey_state {
1739	IB_PORT_PKEY_NOT_VALID = 0,
1740	IB_PORT_PKEY_VALID = 1,
1741	IB_PORT_PKEY_LISTED = 2,
1742	};
1743
1744	struct ib_qp_security;
1745
1746	struct ib_port_pkey {
1747	enum port_pkey_state state;
1748	u16 pkey_index;
1749	u32 port_num;
1750	struct list_head qp_list;
1751	struct list_head to_error_list;
1752	struct ib_qp_security *sec;
1753	};
1754
1755	struct ib_ports_pkeys {
1756	struct ib_port_pkey main;
1757	struct ib_port_pkey alt;
1758	};
1759
1760	struct ib_qp_security {
1761	struct ib_qp *qp;
1762	struct ib_device *dev;
1763	/* Hold this mutex when changing port and pkey settings. */
1764	struct mutex mutex;
1765	struct ib_ports_pkeys *ports_pkeys;
1766	/* A list of all open shared QP handles. Required to enforce security
1767	* properly for all users of a shared QP.
1768	*/
1769	struct list_head shared_qp_list;
1770	void *security;
1771	bool destroying;
1772	atomic_t error_list_count;
1773	struct completion error_complete;
1774	int error_comps_pending;
1775	};
1776
1777	/*
1778	* @max_write_sge: Maximum SGE elements per RDMA WRITE request.
1779	* @max_read_sge: Maximum SGE elements per RDMA READ request.
1780	*/
1781	struct ib_qp {
1782	struct ib_device *device;
1783	struct ib_pd *pd;
1784	struct ib_cq *send_cq;
1785	struct ib_cq *recv_cq;
1786	spinlock_t mr_lock;
1787	int mrs_used;
1788	struct list_head rdma_mrs;
1789	struct list_head sig_mrs;
1790	struct ib_srq *srq;
1791	struct ib_xrcd *xrcd; /* XRC TGT QPs only */
1792	struct list_head xrcd_list;
1793
1794	/* count times opened, mcast attaches, flow attaches */
1795	atomic_t usecnt;
1796	struct list_head open_list;
1797	struct ib_qp *real_qp;
1798	struct ib_uqp_object *uobject;
1799	void (*event_handler)(struct ib_event *, void *);
1800	void *qp_context;
1801	/* sgid_attrs associated with the AV's */
1802	const struct ib_gid_attr *av_sgid_attr;
1803	const struct ib_gid_attr *alt_path_sgid_attr;
1804	u32 qp_num;
1805	u32 max_write_sge;
1806	u32 max_read_sge;
1807	enum ib_qp_type qp_type;
1808	struct ib_rwq_ind_table *rwq_ind_tbl;
1809	struct ib_qp_security *qp_sec;
1810	u32 port;
1811
1812	bool integrity_en;
1813	/*
1814	* Implementation details of the RDMA core, don't use in drivers:
1815	*/
1816	struct rdma_restrack_entry res;
1817
1818	/* The counter the qp is bind to */
1819	struct rdma_counter *counter;
1820	};
1821
1822	struct ib_dm {
1823	struct ib_device *device;
1824	u32 length;
1825	u32 flags;
1826	struct ib_uobject *uobject;
1827	atomic_t usecnt;
1828	};
1829
1830	struct ib_mr {
1831	struct ib_device *device;
1832	struct ib_pd *pd;
1833	u32 lkey;
1834	u32 rkey;
1835	u64 iova;
1836	u64 length;
1837	unsigned int page_size;
1838	enum ib_mr_type type;
1839	bool need_inval;
1840	union {
1841	struct ib_uobject *uobject; /* user */
1842	struct list_head qp_entry; /* FR */
1843	};
1844
1845	struct ib_dm *dm;
1846	struct ib_sig_attrs *sig_attrs; /* only for IB_MR_TYPE_INTEGRITY MRs */
1847	/*
1848	* Implementation details of the RDMA core, don't use in drivers:
1849	*/
1850	struct rdma_restrack_entry res;
1851	};
1852
1853	struct ib_mw {
1854	struct ib_device *device;
1855	struct ib_pd *pd;
1856	struct ib_uobject *uobject;
1857	u32 rkey;
1858	enum ib_mw_type type;
1859	};
1860
1861	/* Supported steering options */
1862	enum ib_flow_attr_type {
1863	/* steering according to rule specifications */
1864	IB_FLOW_ATTR_NORMAL = 0x0,
1865	/* default unicast and multicast rule -
1866	* receive all Eth traffic which isn't steered to any QP
1867	*/
1868	IB_FLOW_ATTR_ALL_DEFAULT = 0x1,
1869	/* default multicast rule -
1870	* receive all Eth multicast traffic which isn't steered to any QP
1871	*/
1872	IB_FLOW_ATTR_MC_DEFAULT = 0x2,
1873	/* sniffer rule - receive all port traffic */
1874	IB_FLOW_ATTR_SNIFFER = 0x3
1875	};
1876
1877	/* Supported steering header types */
1878	enum ib_flow_spec_type {
1879	/* L2 headers*/
1880	IB_FLOW_SPEC_ETH = 0x20,
1881	IB_FLOW_SPEC_IB = 0x22,
1882	/* L3 header*/
1883	IB_FLOW_SPEC_IPV4 = 0x30,
1884	IB_FLOW_SPEC_IPV6 = 0x31,
1885	IB_FLOW_SPEC_ESP = 0x34,
1886	/* L4 headers*/
1887	IB_FLOW_SPEC_TCP = 0x40,
1888	IB_FLOW_SPEC_UDP = 0x41,
1889	IB_FLOW_SPEC_VXLAN_TUNNEL = 0x50,
1890	IB_FLOW_SPEC_GRE = 0x51,
1891	IB_FLOW_SPEC_MPLS = 0x60,
1892	IB_FLOW_SPEC_INNER = 0x100,
1893	/* Actions */
1894	IB_FLOW_SPEC_ACTION_TAG = 0x1000,
1895	IB_FLOW_SPEC_ACTION_DROP = 0x1001,
1896	IB_FLOW_SPEC_ACTION_HANDLE = 0x1002,
1897	IB_FLOW_SPEC_ACTION_COUNT = 0x1003,
1898	};
1899	#define IB_FLOW_SPEC_LAYER_MASK 0xF0
1900	#define IB_FLOW_SPEC_SUPPORT_LAYERS 10
1901
1902	enum ib_flow_flags {
1903	IB_FLOW_ATTR_FLAGS_DONT_TRAP = 1UL << 1, /* Continue match, no steal */
1904	IB_FLOW_ATTR_FLAGS_EGRESS = 1UL << 2, /* Egress flow */
1905	IB_FLOW_ATTR_FLAGS_RESERVED = 1UL << 3 /* Must be last */
1906	};
1907
1908	struct ib_flow_eth_filter {
1909	u8 dst_mac[6];
1910	u8 src_mac[6];
1911	__be16 ether_type;
1912	__be16 vlan_tag;
1913	};
1914
1915	struct ib_flow_spec_eth {
1916	u32 type;
1917	u16 size;
1918	struct ib_flow_eth_filter val;
1919	struct ib_flow_eth_filter mask;
1920	};
1921
1922	struct ib_flow_ib_filter {
1923	__be16 dlid;
1924	__u8 sl;
1925	};
1926
1927	struct ib_flow_spec_ib {
1928	u32 type;
1929	u16 size;
1930	struct ib_flow_ib_filter val;
1931	struct ib_flow_ib_filter mask;
1932	};
1933
1934	/* IPv4 header flags */
1935	enum ib_ipv4_flags {
1936	IB_IPV4_DONT_FRAG = 0x2, /* Don't enable packet fragmentation */
1937	IB_IPV4_MORE_FRAG = 0X4 /* For All fragmented packets except the
1938	last have this flag set */
1939	};
1940
1941	struct ib_flow_ipv4_filter {
1942	__be32 src_ip;
1943	__be32 dst_ip;
1944	u8 proto;
1945	u8 tos;
1946	u8 ttl;
1947	u8 flags;
1948	};
1949
1950	struct ib_flow_spec_ipv4 {
1951	u32 type;
1952	u16 size;
1953	struct ib_flow_ipv4_filter val;
1954	struct ib_flow_ipv4_filter mask;
1955	};
1956
1957	struct ib_flow_ipv6_filter {
1958	u8 src_ip[16];
1959	u8 dst_ip[16];
1960	__be32 flow_label;
1961	u8 next_hdr;
1962	u8 traffic_class;
1963	u8 hop_limit;
1964	} __packed;
1965
1966	struct ib_flow_spec_ipv6 {
1967	u32 type;
1968	u16 size;
1969	struct ib_flow_ipv6_filter val;
1970	struct ib_flow_ipv6_filter mask;
1971	};
1972
1973	struct ib_flow_tcp_udp_filter {
1974	__be16 dst_port;
1975	__be16 src_port;
1976	};
1977
1978	struct ib_flow_spec_tcp_udp {
1979	u32 type;
1980	u16 size;
1981	struct ib_flow_tcp_udp_filter val;
1982	struct ib_flow_tcp_udp_filter mask;
1983	};
1984
1985	struct ib_flow_tunnel_filter {
1986	__be32 tunnel_id;
1987	};
1988
1989	/* ib_flow_spec_tunnel describes the Vxlan tunnel
1990	* the tunnel_id from val has the vni value
1991	*/
1992	struct ib_flow_spec_tunnel {
1993	u32 type;
1994	u16 size;
1995	struct ib_flow_tunnel_filter val;
1996	struct ib_flow_tunnel_filter mask;
1997	};
1998
1999	struct ib_flow_esp_filter {
2000	__be32 spi;
2001	__be32 seq;
2002	};
2003
2004	struct ib_flow_spec_esp {
2005	u32 type;
2006	u16 size;
2007	struct ib_flow_esp_filter val;
2008	struct ib_flow_esp_filter mask;
2009	};
2010
2011	struct ib_flow_gre_filter {
2012	__be16 c_ks_res0_ver;
2013	__be16 protocol;
2014	__be32 key;
2015	};
2016
2017	struct ib_flow_spec_gre {
2018	u32 type;
2019	u16 size;
2020	struct ib_flow_gre_filter val;
2021	struct ib_flow_gre_filter mask;
2022	};
2023
2024	struct ib_flow_mpls_filter {
2025	__be32 tag;
2026	};
2027
2028	struct ib_flow_spec_mpls {
2029	u32 type;
2030	u16 size;
2031	struct ib_flow_mpls_filter val;
2032	struct ib_flow_mpls_filter mask;
2033	};
2034
2035	struct ib_flow_spec_action_tag {
2036	enum ib_flow_spec_type type;
2037	u16 size;
2038	u32 tag_id;
2039	};
2040
2041	struct ib_flow_spec_action_drop {
2042	enum ib_flow_spec_type type;
2043	u16 size;
2044	};
2045
2046	struct ib_flow_spec_action_handle {
2047	enum ib_flow_spec_type type;
2048	u16 size;
2049	struct ib_flow_action *act;
2050	};
2051
2052	enum ib_counters_description {
2053	IB_COUNTER_PACKETS,
2054	IB_COUNTER_BYTES,
2055	};
2056
2057	struct ib_flow_spec_action_count {
2058	enum ib_flow_spec_type type;
2059	u16 size;
2060	struct ib_counters *counters;
2061	};
2062
2063	union ib_flow_spec {
2064	struct {
2065	u32 type;
2066	u16 size;
2067	};
2068	struct ib_flow_spec_eth eth;
2069	struct ib_flow_spec_ib ib;
2070	struct ib_flow_spec_ipv4 ipv4;
2071	struct ib_flow_spec_tcp_udp tcp_udp;
2072	struct ib_flow_spec_ipv6 ipv6;
2073	struct ib_flow_spec_tunnel tunnel;
2074	struct ib_flow_spec_esp esp;
2075	struct ib_flow_spec_gre gre;
2076	struct ib_flow_spec_mpls mpls;
2077	struct ib_flow_spec_action_tag flow_tag;
2078	struct ib_flow_spec_action_drop drop;
2079	struct ib_flow_spec_action_handle action;
2080	struct ib_flow_spec_action_count flow_count;
2081	};
2082
2083	struct ib_flow_attr {
2084	enum ib_flow_attr_type type;
2085	u16 size;
2086	u16 priority;
2087	u32 flags;
2088	u8 num_of_specs;
2089	u32 port;
2090	union ib_flow_spec flows[];
2091	};
2092
2093	struct ib_flow {
2094	struct ib_qp *qp;
2095	struct ib_device *device;
2096	struct ib_uobject *uobject;
2097	};
2098
2099	enum ib_flow_action_type {
2100	IB_FLOW_ACTION_UNSPECIFIED,
2101	IB_FLOW_ACTION_ESP = 1,
2102	};
2103
2104	struct ib_flow_action_attrs_esp_keymats {
2105	enum ib_uverbs_flow_action_esp_keymat protocol;
2106	union {
2107	struct ib_uverbs_flow_action_esp_keymat_aes_gcm aes_gcm;
2108	} keymat;
2109	};
2110
2111	struct ib_flow_action_attrs_esp_replays {
2112	enum ib_uverbs_flow_action_esp_replay protocol;
2113	union {
2114	struct ib_uverbs_flow_action_esp_replay_bmp bmp;
2115	} replay;
2116	};
2117
2118	enum ib_flow_action_attrs_esp_flags {
2119	/* All user-space flags at the top: Use enum ib_uverbs_flow_action_esp_flags
2120	* This is done in order to share the same flags between user-space and
2121	* kernel and spare an unnecessary translation.
2122	*/
2123
2124	/* Kernel flags */
2125	IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED = 1ULL << 32,
2126	IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS = 1ULL << 33,
2127	};
2128
2129	struct ib_flow_spec_list {
2130	struct ib_flow_spec_list *next;
2131	union ib_flow_spec spec;
2132	};
2133
2134	struct ib_flow_action_attrs_esp {
2135	struct ib_flow_action_attrs_esp_keymats *keymat;
2136	struct ib_flow_action_attrs_esp_replays *replay;
2137	struct ib_flow_spec_list *encap;
2138	/* Used only if IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED is enabled.
2139	* Value of 0 is a valid value.
2140	*/
2141	u32 esn;
2142	u32 spi;
2143	u32 seq;
2144	u32 tfc_pad;
2145	/* Use enum ib_flow_action_attrs_esp_flags */
2146	u64 flags;
2147	u64 hard_limit_pkts;
2148	};
2149
2150	struct ib_flow_action {
2151	struct ib_device *device;
2152	struct ib_uobject *uobject;
2153	enum ib_flow_action_type type;
2154	atomic_t usecnt;
2155	};
2156
2157	struct ib_mad;
2158
2159	enum ib_process_mad_flags {
2160	IB_MAD_IGNORE_MKEY = 1,
2161	IB_MAD_IGNORE_BKEY = 2,
2162	IB_MAD_IGNORE_ALL = IB_MAD_IGNORE_MKEY | IB_MAD_IGNORE_BKEY
2163	};
2164
2165	enum ib_mad_result {
2166	IB_MAD_RESULT_FAILURE = 0, /* (!SUCCESS is the important flag) */
2167	IB_MAD_RESULT_SUCCESS = 1 << 0, /* MAD was successfully processed */
2168	IB_MAD_RESULT_REPLY = 1 << 1, /* Reply packet needs to be sent */
2169	IB_MAD_RESULT_CONSUMED = 1 << 2 /* Packet consumed: stop processing */
2170	};
2171
2172	struct ib_port_cache {
2173	u64 subnet_prefix;
2174	struct ib_pkey_cache *pkey;
2175	struct ib_gid_table *gid;
2176	u8 lmc;
2177	enum ib_port_state port_state;
2178	};
2179
2180	struct ib_port_immutable {
2181	int pkey_tbl_len;
2182	int gid_tbl_len;
2183	u32 core_cap_flags;
2184	u32 max_mad_size;
2185	};
2186
2187	struct ib_port_data {
2188	struct ib_device *ib_dev;
2189
2190	struct ib_port_immutable immutable;
2191
2192	spinlock_t pkey_list_lock;
2193
2194	spinlock_t netdev_lock;
2195
2196	struct list_head pkey_list;
2197
2198	struct ib_port_cache cache;
2199
2200	struct net_device __rcu *netdev;
2201	netdevice_tracker netdev_tracker;
2202	struct hlist_node ndev_hash_link;
2203	struct rdma_port_counter port_counter;
2204	struct ib_port *sysfs;
2205	};
2206
2207	/* rdma netdev type - specifies protocol type */
2208	enum rdma_netdev_t {
2209	RDMA_NETDEV_OPA_VNIC,
2210	RDMA_NETDEV_IPOIB,
2211	};
2212
2213	/**
2214	* struct rdma_netdev - rdma netdev
2215	* For cases where netstack interfacing is required.
2216	*/
2217	struct rdma_netdev {
2218	void *clnt_priv;
2219	struct ib_device *hca;
2220	u32 port_num;
2221	int mtu;
2222
2223	/*
2224	* cleanup function must be specified.
2225	* FIXME: This is only used for OPA_VNIC and that usage should be
2226	* removed too.
2227	*/
2228	void (*free_rdma_netdev)(struct net_device *netdev);
2229
2230	/* control functions */
2231	void (*set_id)(struct net_device *netdev, int id);
2232	/* send packet */
2233	int (*send)(struct net_device *dev, struct sk_buff *skb,
2234	struct ib_ah *address, u32 dqpn);
2235	/* multicast */
2236	int (*attach_mcast)(struct net_device *dev, struct ib_device *hca,
2237	union ib_gid *gid, u16 mlid,
2238	int set_qkey, u32 qkey);
2239	int (*detach_mcast)(struct net_device *dev, struct ib_device *hca,
2240	union ib_gid *gid, u16 mlid);
2241	/* timeout */
2242	void (*tx_timeout)(struct net_device *dev, unsigned int txqueue);
2243	};
2244
2245	struct rdma_netdev_alloc_params {
2246	size_t sizeof_priv;
2247	unsigned int txqs;
2248	unsigned int rxqs;
2249	void *param;
2250
2251	int (*initialize_rdma_netdev)(struct ib_device *device, u32 port_num,
2252	struct net_device *netdev, void *param);
2253	};
2254
2255	struct ib_odp_counters {
2256	atomic64_t faults;
2257	atomic64_t invalidations;
2258	atomic64_t prefetch;
2259	};
2260
2261	struct ib_counters {
2262	struct ib_device *device;
2263	struct ib_uobject *uobject;
2264	/* num of objects attached */
2265	atomic_t usecnt;
2266	};
2267
2268	struct ib_counters_read_attr {
2269	u64 *counters_buff;
2270	u32 ncounters;
2271	u32 flags; /* use enum ib_read_counters_flags */
2272	};
2273
2274	struct uverbs_attr_bundle;
2275	struct iw_cm_id;
2276	struct iw_cm_conn_param;
2277
2278	#define INIT_RDMA_OBJ_SIZE(ib_struct, drv_struct, member) \
2279	.size_##ib_struct = \
2280	(sizeof(struct drv_struct) + \
2281	BUILD_BUG_ON_ZERO(offsetof(struct drv_struct, member)) + \
2282	BUILD_BUG_ON_ZERO( \
2283	!__same_type(((struct drv_struct *)NULL)->member, \
2284	struct ib_struct)))
2285
2286	#define rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, gfp) \
2287	((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \
2288	gfp, false))
2289
2290	#define rdma_zalloc_drv_obj_numa(ib_dev, ib_type) \
2291	((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \
2292	GFP_KERNEL, true))
2293
2294	#define rdma_zalloc_drv_obj(ib_dev, ib_type) \
2295	rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, GFP_KERNEL)
2296
2297	#define DECLARE_RDMA_OBJ_SIZE(ib_struct) size_t size_##ib_struct
2298
2299	struct rdma_user_mmap_entry {
2300	struct kref ref;
2301	struct ib_ucontext *ucontext;
2302	unsigned long start_pgoff;
2303	size_t npages;
2304	bool driver_removed;
2305	};
2306
2307	/* Return the offset (in bytes) the user should pass to libc's mmap() */
2308	static inline u64
2309	rdma_user_mmap_get_offset(const struct rdma_user_mmap_entry *entry)
2310	{
2311	return (u64)entry->start_pgoff << PAGE_SHIFT;
2312	}
2313
2314	/**
2315	* struct ib_device_ops - InfiniBand device operations
2316	* This structure defines all the InfiniBand device operations, providers will
2317	* need to define the supported operations, otherwise they will be set to null.
2318	*/
2319	struct ib_device_ops {
2320	struct module *owner;
2321	enum rdma_driver_id driver_id;
2322	u32 uverbs_abi_ver;
2323	unsigned int uverbs_no_driver_id_binding:1;
2324
2325	/*
2326	* NOTE: New drivers should not make use of device_group; instead new
2327	* device parameter should be exposed via netlink command. This
2328	* mechanism exists only for existing drivers.
2329	*/
2330	const struct attribute_group *device_group;
2331	const struct attribute_group **port_groups;
2332
2333	int (*post_send)(struct ib_qp *qp, const struct ib_send_wr *send_wr,
2334	const struct ib_send_wr **bad_send_wr);
2335	int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
2336	const struct ib_recv_wr **bad_recv_wr);
2337	void (*drain_rq)(struct ib_qp *qp);
2338	void (*drain_sq)(struct ib_qp *qp);
2339	int (*poll_cq)(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
2340	int (*peek_cq)(struct ib_cq *cq, int wc_cnt);
2341	int (*req_notify_cq)(struct ib_cq *cq, enum ib_cq_notify_flags flags);
2342	int (*post_srq_recv)(struct ib_srq *srq,
2343	const struct ib_recv_wr *recv_wr,
2344	const struct ib_recv_wr **bad_recv_wr);
2345	int (*process_mad)(struct ib_device *device, int process_mad_flags,
2346	u32 port_num, const struct ib_wc *in_wc,
2347	const struct ib_grh *in_grh,
2348	const struct ib_mad *in_mad, struct ib_mad *out_mad,
2349	size_t *out_mad_size, u16 *out_mad_pkey_index);
2350	int (*query_device)(struct ib_device *device,
2351	struct ib_device_attr *device_attr,
2352	struct ib_udata *udata);
2353	int (*modify_device)(struct ib_device *device, int device_modify_mask,
2354	struct ib_device_modify *device_modify);
2355	void (*get_dev_fw_str)(struct ib_device *device, char *str);
2356	const struct cpumask *(*get_vector_affinity)(struct ib_device *ibdev,
2357	int comp_vector);
2358	int (*query_port)(struct ib_device *device, u32 port_num,
2359	struct ib_port_attr *port_attr);
2360	int (*modify_port)(struct ib_device *device, u32 port_num,
2361	int port_modify_mask,
2362	struct ib_port_modify *port_modify);
2363	/**
2364	* The following mandatory functions are used only at device
2365	* registration. Keep functions such as these at the end of this
2366	* structure to avoid cache line misses when accessing struct ib_device
2367	* in fast paths.
2368	*/
2369	int (*get_port_immutable)(struct ib_device *device, u32 port_num,
2370	struct ib_port_immutable *immutable);
2371	enum rdma_link_layer (*get_link_layer)(struct ib_device *device,
2372	u32 port_num);
2373	/**
2374	* When calling get_netdev, the HW vendor's driver should return the
2375	* net device of device @device at port @port_num or NULL if such
2376	* a net device doesn't exist. The vendor driver should call dev_hold
2377	* on this net device. The HW vendor's device driver must guarantee
2378	* that this function returns NULL before the net device has finished
2379	* NETDEV_UNREGISTER state.
2380	*/
2381	struct net_device *(*get_netdev)(struct ib_device *device,
2382	u32 port_num);
2383	/**
2384	* rdma netdev operation
2385	*
2386	* Driver implementing alloc_rdma_netdev or rdma_netdev_get_params
2387	* must return -EOPNOTSUPP if it doesn't support the specified type.
2388	*/
2389	struct net_device *(*alloc_rdma_netdev)(
2390	struct ib_device *device, u32 port_num, enum rdma_netdev_t type,
2391	const char *name, unsigned char name_assign_type,
2392	void (*setup)(struct net_device *));
2393
2394	int (*rdma_netdev_get_params)(struct ib_device *device, u32 port_num,
2395	enum rdma_netdev_t type,
2396	struct rdma_netdev_alloc_params *params);
2397	/**
2398	* query_gid should be return GID value for @device, when @port_num
2399	* link layer is either IB or iWarp. It is no-op if @port_num port
2400	* is RoCE link layer.
2401	*/
2402	int (*query_gid)(struct ib_device *device, u32 port_num, int index,
2403	union ib_gid *gid);
2404	/**
2405	* When calling add_gid, the HW vendor's driver should add the gid
2406	* of device of port at gid index available at @attr. Meta-info of
2407	* that gid (for example, the network device related to this gid) is
2408	* available at @attr. @context allows the HW vendor driver to store
2409	* extra information together with a GID entry. The HW vendor driver may
2410	* allocate memory to contain this information and store it in @context
2411	* when a new GID entry is written to. Params are consistent until the
2412	* next call of add_gid or delete_gid. The function should return 0 on
2413	* success or error otherwise. The function could be called
2414	* concurrently for different ports. This function is only called when
2415	* roce_gid_table is used.
2416	*/
2417	int (*add_gid)(const struct ib_gid_attr *attr, void **context);
2418	/**
2419	* When calling del_gid, the HW vendor's driver should delete the
2420	* gid of device @device at gid index gid_index of port port_num
2421	* available in @attr.
2422	* Upon the deletion of a GID entry, the HW vendor must free any
2423	* allocated memory. The caller will clear @context afterwards.
2424	* This function is only called when roce_gid_table is used.
2425	*/
2426	int (*del_gid)(const struct ib_gid_attr *attr, void **context);
2427	int (*query_pkey)(struct ib_device *device, u32 port_num, u16 index,
2428	u16 *pkey);
2429	int (*alloc_ucontext)(struct ib_ucontext *context,
2430	struct ib_udata *udata);
2431	void (*dealloc_ucontext)(struct ib_ucontext *context);
2432	int (*mmap)(struct ib_ucontext *context, struct vm_area_struct *vma);
2433	/**
2434	* This will be called once refcount of an entry in mmap_xa reaches
2435	* zero. The type of the memory that was mapped may differ between
2436	* entries and is opaque to the rdma_user_mmap interface.
2437	* Therefore needs to be implemented by the driver in mmap_free.
2438	*/
2439	void (*mmap_free)(struct rdma_user_mmap_entry *entry);
2440	void (*disassociate_ucontext)(struct ib_ucontext *ibcontext);
2441	int (*alloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
2442	int (*dealloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
2443	int (*create_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,
2444	struct ib_udata *udata);
2445	int (*create_user_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,
2446	struct ib_udata *udata);
2447	int (*modify_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
2448	int (*query_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
2449	int (*destroy_ah)(struct ib_ah *ah, u32 flags);
2450	int (*create_srq)(struct ib_srq *srq,
2451	struct ib_srq_init_attr *srq_init_attr,
2452	struct ib_udata *udata);
2453	int (*modify_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr,
2454	enum ib_srq_attr_mask srq_attr_mask,
2455	struct ib_udata *udata);
2456	int (*query_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
2457	int (*destroy_srq)(struct ib_srq *srq, struct ib_udata *udata);
2458	int (*create_qp)(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr,
2459	struct ib_udata *udata);
2460	int (*modify_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
2461	int qp_attr_mask, struct ib_udata *udata);
2462	int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
2463	int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
2464	int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata);
2465	int (*create_cq)(struct ib_cq *cq, const struct ib_cq_init_attr *attr,
2466	struct ib_udata *udata);
2467	int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
2468	int (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata);
2469	int (*resize_cq)(struct ib_cq *cq, int cqe, struct ib_udata *udata);
2470	struct ib_mr *(*get_dma_mr)(struct ib_pd *pd, int mr_access_flags);
2471	struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length,
2472	u64 virt_addr, int mr_access_flags,
2473	struct ib_udata *udata);
2474	struct ib_mr *(*reg_user_mr_dmabuf)(struct ib_pd *pd, u64 offset,
2475	u64 length, u64 virt_addr, int fd,
2476	int mr_access_flags,
2477	struct ib_udata *udata);
2478	struct ib_mr *(*rereg_user_mr)(struct ib_mr *mr, int flags, u64 start,
2479	u64 length, u64 virt_addr,
2480	int mr_access_flags, struct ib_pd *pd,
2481	struct ib_udata *udata);
2482	int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata);
2483	struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,
2484	u32 max_num_sg);
2485	struct ib_mr *(*alloc_mr_integrity)(struct ib_pd *pd,
2486	u32 max_num_data_sg,
2487	u32 max_num_meta_sg);
2488	int (*advise_mr)(struct ib_pd *pd,
2489	enum ib_uverbs_advise_mr_advice advice, u32 flags,
2490	struct ib_sge *sg_list, u32 num_sge,
2491	struct uverbs_attr_bundle *attrs);
2492
2493	/*
2494	* Kernel users should universally support relaxed ordering (RO), as
2495	* they are designed to read data only after observing the CQE and use
2496	* the DMA API correctly.
2497	*
2498	* Some drivers implicitly enable RO if platform supports it.
2499	*/
2500	int (*map_mr_sg)(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
2501	unsigned int *sg_offset);
2502	int (*check_mr_status)(struct ib_mr *mr, u32 check_mask,
2503	struct ib_mr_status *mr_status);
2504	int (*alloc_mw)(struct ib_mw *mw, struct ib_udata *udata);
2505	int (*dealloc_mw)(struct ib_mw *mw);
2506	int (*attach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
2507	int (*detach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
2508	int (*alloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
2509	int (*dealloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
2510	struct ib_flow *(*create_flow)(struct ib_qp *qp,
2511	struct ib_flow_attr *flow_attr,
2512	struct ib_udata *udata);
2513	int (*destroy_flow)(struct ib_flow *flow_id);
2514	int (*destroy_flow_action)(struct ib_flow_action *action);
2515	int (*set_vf_link_state)(struct ib_device *device, int vf, u32 port,
2516	int state);
2517	int (*get_vf_config)(struct ib_device *device, int vf, u32 port,
2518	struct ifla_vf_info *ivf);
2519	int (*get_vf_stats)(struct ib_device *device, int vf, u32 port,
2520	struct ifla_vf_stats *stats);
2521	int (*get_vf_guid)(struct ib_device *device, int vf, u32 port,
2522	struct ifla_vf_guid *node_guid,
2523	struct ifla_vf_guid *port_guid);
2524	int (*set_vf_guid)(struct ib_device *device, int vf, u32 port, u64 guid,
2525	int type);
2526	struct ib_wq *(*create_wq)(struct ib_pd *pd,
2527	struct ib_wq_init_attr *init_attr,
2528	struct ib_udata *udata);
2529	int (*destroy_wq)(struct ib_wq *wq, struct ib_udata *udata);
2530	int (*modify_wq)(struct ib_wq *wq, struct ib_wq_attr *attr,
2531	u32 wq_attr_mask, struct ib_udata *udata);
2532	int (*create_rwq_ind_table)(struct ib_rwq_ind_table *ib_rwq_ind_table,
2533	struct ib_rwq_ind_table_init_attr *init_attr,
2534	struct ib_udata *udata);
2535	int (*destroy_rwq_ind_table)(struct ib_rwq_ind_table *wq_ind_table);
2536	struct ib_dm *(*alloc_dm)(struct ib_device *device,
2537	struct ib_ucontext *context,
2538	struct ib_dm_alloc_attr *attr,
2539	struct uverbs_attr_bundle *attrs);
2540	int (*dealloc_dm)(struct ib_dm *dm, struct uverbs_attr_bundle *attrs);
2541	struct ib_mr *(*reg_dm_mr)(struct ib_pd *pd, struct ib_dm *dm,
2542	struct ib_dm_mr_attr *attr,
2543	struct uverbs_attr_bundle *attrs);
2544	int (*create_counters)(struct ib_counters *counters,
2545	struct uverbs_attr_bundle *attrs);
2546	int (*destroy_counters)(struct ib_counters *counters);
2547	int (*read_counters)(struct ib_counters *counters,
2548	struct ib_counters_read_attr *counters_read_attr,
2549	struct uverbs_attr_bundle *attrs);
2550	int (*map_mr_sg_pi)(struct ib_mr *mr, struct scatterlist *data_sg,
2551	int data_sg_nents, unsigned int *data_sg_offset,
2552	struct scatterlist *meta_sg, int meta_sg_nents,
2553	unsigned int *meta_sg_offset);
2554
2555	/**
2556	* alloc_hw_[device,port]_stats - Allocate a struct rdma_hw_stats and
2557	* fill in the driver initialized data. The struct is kfree()'ed by
2558	* the sysfs core when the device is removed. A lifespan of -1 in the
2559	* return struct tells the core to set a default lifespan.
2560	*/
2561	struct rdma_hw_stats *(*alloc_hw_device_stats)(struct ib_device *device);
2562	struct rdma_hw_stats *(*alloc_hw_port_stats)(struct ib_device *device,
2563	u32 port_num);
2564	/**
2565	* get_hw_stats - Fill in the counter value(s) in the stats struct.
2566	* @index - The index in the value array we wish to have updated, or
2567	* num_counters if we want all stats updated
2568	* Return codes -
2569	* < 0 - Error, no counters updated
2570	* index - Updated the single counter pointed to by index
2571	* num_counters - Updated all counters (will reset the timestamp
2572	* and prevent further calls for lifespan milliseconds)
2573	* Drivers are allowed to update all counters in leiu of just the
2574	* one given in index at their option
2575	*/
2576	int (*get_hw_stats)(struct ib_device *device,
2577	struct rdma_hw_stats *stats, u32 port, int index);
2578
2579	/**
2580	* modify_hw_stat - Modify the counter configuration
2581	* @enable: true/false when enable/disable a counter
2582	* Return codes - 0 on success or error code otherwise.
2583	*/
2584	int (*modify_hw_stat)(struct ib_device *device, u32 port,
2585	unsigned int counter_index, bool enable);
2586	/**
2587	* Allows rdma drivers to add their own restrack attributes.
2588	*/
2589	int (*fill_res_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);
2590	int (*fill_res_mr_entry_raw)(struct sk_buff *msg, struct ib_mr *ibmr);
2591	int (*fill_res_cq_entry)(struct sk_buff *msg, struct ib_cq *ibcq);
2592	int (*fill_res_cq_entry_raw)(struct sk_buff *msg, struct ib_cq *ibcq);
2593	int (*fill_res_qp_entry)(struct sk_buff *msg, struct ib_qp *ibqp);
2594	int (*fill_res_qp_entry_raw)(struct sk_buff *msg, struct ib_qp *ibqp);
2595	int (*fill_res_cm_id_entry)(struct sk_buff *msg, struct rdma_cm_id *id);
2596	int (*fill_res_srq_entry)(struct sk_buff *msg, struct ib_srq *ib_srq);
2597	int (*fill_res_srq_entry_raw)(struct sk_buff *msg, struct ib_srq *ib_srq);
2598
2599	/* Device lifecycle callbacks */
2600	/*
2601	* Called after the device becomes registered, before clients are
2602	* attached
2603	*/
2604	int (*enable_driver)(struct ib_device *dev);
2605	/*
2606	* This is called as part of ib_dealloc_device().
2607	*/
2608	void (*dealloc_driver)(struct ib_device *dev);
2609
2610	/* iWarp CM callbacks */
2611	void (*iw_add_ref)(struct ib_qp *qp);
2612	void (*iw_rem_ref)(struct ib_qp *qp);
2613	struct ib_qp *(*iw_get_qp)(struct ib_device *device, int qpn);
2614	int (*iw_connect)(struct iw_cm_id *cm_id,
2615	struct iw_cm_conn_param *conn_param);
2616	int (*iw_accept)(struct iw_cm_id *cm_id,
2617	struct iw_cm_conn_param *conn_param);
2618	int (*iw_reject)(struct iw_cm_id *cm_id, const void *pdata,
2619	u8 pdata_len);
2620	int (*iw_create_listen)(struct iw_cm_id *cm_id, int backlog);
2621	int (*iw_destroy_listen)(struct iw_cm_id *cm_id);
2622	/**
2623	* counter_bind_qp - Bind a QP to a counter.
2624	* @counter - The counter to be bound. If counter->id is zero then
2625	* the driver needs to allocate a new counter and set counter->id
2626	*/
2627	int (*counter_bind_qp)(struct rdma_counter *counter, struct ib_qp *qp);
2628	/**
2629	* counter_unbind_qp - Unbind the qp from the dynamically-allocated
2630	* counter and bind it onto the default one
2631	*/
2632	int (*counter_unbind_qp)(struct ib_qp *qp);
2633	/**
2634	* counter_dealloc -De-allocate the hw counter
2635	*/
2636	int (*counter_dealloc)(struct rdma_counter *counter);
2637	/**
2638	* counter_alloc_stats - Allocate a struct rdma_hw_stats and fill in
2639	* the driver initialized data.
2640	*/
2641	struct rdma_hw_stats *(*counter_alloc_stats)(
2642	struct rdma_counter *counter);
2643	/**
2644	* counter_update_stats - Query the stats value of this counter
2645	*/
2646	int (*counter_update_stats)(struct rdma_counter *counter);
2647
2648	/**
2649	* Allows rdma drivers to add their own restrack attributes
2650	* dumped via 'rdma stat' iproute2 command.
2651	*/
2652	int (*fill_stat_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);
2653
2654	/* query driver for its ucontext properties */
2655	int (*query_ucontext)(struct ib_ucontext *context,
2656	struct uverbs_attr_bundle *attrs);
2657
2658	/*
2659	* Provide NUMA node. This API exists for rdmavt/hfi1 only.
2660	* Everyone else relies on Linux memory management model.
2661	*/
2662	int (*get_numa_node)(struct ib_device *dev);
2663
2664	DECLARE_RDMA_OBJ_SIZE(ib_ah);
2665	DECLARE_RDMA_OBJ_SIZE(ib_counters);
2666	DECLARE_RDMA_OBJ_SIZE(ib_cq);
2667	DECLARE_RDMA_OBJ_SIZE(ib_mw);
2668	DECLARE_RDMA_OBJ_SIZE(ib_pd);
2669	DECLARE_RDMA_OBJ_SIZE(ib_qp);
2670	DECLARE_RDMA_OBJ_SIZE(ib_rwq_ind_table);
2671	DECLARE_RDMA_OBJ_SIZE(ib_srq);
2672	DECLARE_RDMA_OBJ_SIZE(ib_ucontext);
2673	DECLARE_RDMA_OBJ_SIZE(ib_xrcd);
2674	};
2675
2676	struct ib_core_device {
2677	/* device must be the first element in structure until,
2678	* union of ib_core_device and device exists in ib_device.
2679	*/
2680	struct device dev;
2681	possible_net_t rdma_net;
2682	struct kobject *ports_kobj;
2683	struct list_head port_list;
2684	struct ib_device *owner; /* reach back to owner ib_device */
2685	};
2686
2687	struct rdma_restrack_root;
2688	struct ib_device {
2689	/* Do not access @dma_device directly from ULP nor from HW drivers. */
2690	struct device *dma_device;
2691	struct ib_device_ops ops;
2692	char name[IB_DEVICE_NAME_MAX];
2693	struct rcu_head rcu_head;
2694
2695	struct list_head event_handler_list;
2696	/* Protects event_handler_list */
2697	struct rw_semaphore event_handler_rwsem;
2698
2699	/* Protects QP's event_handler calls and open_qp list */
2700	spinlock_t qp_open_list_lock;
2701
2702	struct rw_semaphore client_data_rwsem;
2703	struct xarray client_data;
2704	struct mutex unregistration_lock;
2705
2706	/* Synchronize GID, Pkey cache entries, subnet prefix, LMC */
2707	rwlock_t cache_lock;
2708	/**
2709	* port_data is indexed by port number
2710	*/
2711	struct ib_port_data *port_data;
2712
2713	int num_comp_vectors;
2714
2715	union {
2716	struct device dev;
2717	struct ib_core_device coredev;
2718	};
2719
2720	/* First group is for device attributes,
2721	* Second group is for driver provided attributes (optional).
2722	* Third group is for the hw_stats
2723	* It is a NULL terminated array.
2724	*/
2725	const struct attribute_group *groups[4];
2726
2727	u64 uverbs_cmd_mask;
2728
2729	char node_desc[IB_DEVICE_NODE_DESC_MAX];
2730	__be64 node_guid;
2731	u32 local_dma_lkey;
2732	u16 is_switch:1;
2733	/* Indicates kernel verbs support, should not be used in drivers */
2734	u16 kverbs_provider:1;
2735	/* CQ adaptive moderation (RDMA DIM) */
2736	u16 use_cq_dim:1;
2737	u8 node_type;
2738	u32 phys_port_cnt;
2739	struct ib_device_attr attrs;
2740	struct hw_stats_device_data *hw_stats_data;
2741
2742	#ifdef CONFIG_CGROUP_RDMA
2743	struct rdmacg_device cg_device;
2744	#endif
2745
2746	u32 index;
2747
2748	spinlock_t cq_pools_lock;
2749	struct list_head cq_pools[IB_POLL_LAST_POOL_TYPE + 1];
2750
2751	struct rdma_restrack_root *res;
2752
2753	const struct uapi_definition *driver_def;
2754
2755	/*
2756	* Positive refcount indicates that the device is currently
2757	* registered and cannot be unregistered.
2758	*/
2759	refcount_t refcount;
2760	struct completion unreg_completion;
2761	struct work_struct unregistration_work;
2762
2763	const struct rdma_link_ops *link_ops;
2764
2765	/* Protects compat_devs xarray modifications */
2766	struct mutex compat_devs_mutex;
2767	/* Maintains compat devices for each net namespace */
2768	struct xarray compat_devs;
2769
2770	/* Used by iWarp CM */
2771	char iw_ifname[IFNAMSIZ];
2772	u32 iw_driver_flags;
2773	u32 lag_flags;
2774	};
2775
2776	static inline void *rdma_zalloc_obj(struct ib_device *dev, size_t size,
2777	gfp_t gfp, bool is_numa_aware)
2778	{
2779	if (is_numa_aware && dev->ops.get_numa_node)
2780	return kzalloc_node(size, flags: gfp, node: dev->ops.get_numa_node(dev));
2781
2782	return kzalloc(size, flags: gfp);
2783	}
2784
2785	struct ib_client_nl_info;
2786	struct ib_client {
2787	const char *name;
2788	int (*add)(struct ib_device *ibdev);
2789	void (*remove)(struct ib_device *, void *client_data);
2790	void (*rename)(struct ib_device *dev, void *client_data);
2791	int (*get_nl_info)(struct ib_device *ibdev, void *client_data,
2792	struct ib_client_nl_info *res);
2793	int (*get_global_nl_info)(struct ib_client_nl_info *res);
2794
2795	/* Returns the net_dev belonging to this ib_client and matching the
2796	* given parameters.
2797	* @dev: An RDMA device that the net_dev use for communication.
2798	* @port: A physical port number on the RDMA device.
2799	* @pkey: P_Key that the net_dev uses if applicable.
2800	* @gid: A GID that the net_dev uses to communicate.
2801	* @addr: An IP address the net_dev is configured with.
2802	* @client_data: The device's client data set by ib_set_client_data().
2803	*
2804	* An ib_client that implements a net_dev on top of RDMA devices
2805	* (such as IP over IB) should implement this callback, allowing the
2806	* rdma_cm module to find the right net_dev for a given request.
2807	*
2808	* The caller is responsible for calling dev_put on the returned
2809	* netdev. */
2810	struct net_device *(*get_net_dev_by_params)(
2811	struct ib_device *dev,
2812	u32 port,
2813	u16 pkey,
2814	const union ib_gid *gid,
2815	const struct sockaddr *addr,
2816	void *client_data);
2817
2818	refcount_t uses;
2819	struct completion uses_zero;
2820	u32 client_id;
2821
2822	/* kverbs are not required by the client */
2823	u8 no_kverbs_req:1;
2824	};
2825
2826	/*
2827	* IB block DMA iterator
2828	*
2829	* Iterates the DMA-mapped SGL in contiguous memory blocks aligned
2830	* to a HW supported page size.
2831	*/
2832	struct ib_block_iter {
2833	/* internal states */
2834	struct scatterlist *__sg; /* sg holding the current aligned block */
2835	dma_addr_t __dma_addr; /* unaligned DMA address of this block */
2836	size_t __sg_numblocks; /* ib_umem_num_dma_blocks() */
2837	unsigned int __sg_nents; /* number of SG entries */
2838	unsigned int __sg_advance; /* number of bytes to advance in sg in next step */
2839	unsigned int __pg_bit; /* alignment of current block */
2840	};
2841
2842	struct ib_device *_ib_alloc_device(size_t size);
2843	#define ib_alloc_device(drv_struct, member) \
2844	container_of(_ib_alloc_device(sizeof(struct drv_struct) + \
2845	BUILD_BUG_ON_ZERO(offsetof( \
2846	struct drv_struct, member))), \
2847	struct drv_struct, member)
2848
2849	void ib_dealloc_device(struct ib_device *device);
2850
2851	void ib_get_device_fw_str(struct ib_device *device, char *str);
2852
2853	int ib_register_device(struct ib_device *device, const char *name,
2854	struct device *dma_device);
2855	void ib_unregister_device(struct ib_device *device);
2856	void ib_unregister_driver(enum rdma_driver_id driver_id);
2857	void ib_unregister_device_and_put(struct ib_device *device);
2858	void ib_unregister_device_queued(struct ib_device *ib_dev);
2859
2860	int ib_register_client (struct ib_client *client);
2861	void ib_unregister_client(struct ib_client *client);
2862
2863	void __rdma_block_iter_start(struct ib_block_iter *biter,
2864	struct scatterlist *sglist,
2865	unsigned int nents,
2866	unsigned long pgsz);
2867	bool __rdma_block_iter_next(struct ib_block_iter *biter);
2868
2869	/**
2870	* rdma_block_iter_dma_address - get the aligned dma address of the current
2871	* block held by the block iterator.
2872	* @biter: block iterator holding the memory block
2873	*/
2874	static inline dma_addr_t
2875	rdma_block_iter_dma_address(struct ib_block_iter *biter)
2876	{
2877	return biter->__dma_addr & ~(BIT_ULL(biter->__pg_bit) - 1);
2878	}
2879
2880	/**
2881	* rdma_for_each_block - iterate over contiguous memory blocks of the sg list
2882	* @sglist: sglist to iterate over
2883	* @biter: block iterator holding the memory block
2884	* @nents: maximum number of sg entries to iterate over
2885	* @pgsz: best HW supported page size to use
2886	*
2887	* Callers may use rdma_block_iter_dma_address() to get each
2888	* blocks aligned DMA address.
2889	*/
2890	#define rdma_for_each_block(sglist, biter, nents, pgsz) \
2891	for (__rdma_block_iter_start(biter, sglist, nents, \
2892	pgsz); \
2893	__rdma_block_iter_next(biter);)
2894
2895	/**
2896	* ib_get_client_data - Get IB client context
2897	* @device:Device to get context for
2898	* @client:Client to get context for
2899	*
2900	* ib_get_client_data() returns the client context data set with
2901	* ib_set_client_data(). This can only be called while the client is
2902	* registered to the device, once the ib_client remove() callback returns this
2903	* cannot be called.
2904	*/
2905	static inline void *ib_get_client_data(struct ib_device *device,
2906	struct ib_client *client)
2907	{
2908	return xa_load(&device->client_data, index: client->client_id);
2909	}
2910	void ib_set_client_data(struct ib_device *device, struct ib_client *client,
2911	void *data);
2912	void ib_set_device_ops(struct ib_device *device,
2913	const struct ib_device_ops *ops);
2914
2915	int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
2916	unsigned long pfn, unsigned long size, pgprot_t prot,
2917	struct rdma_user_mmap_entry *entry);
2918	int rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext,
2919	struct rdma_user_mmap_entry *entry,
2920	size_t length);
2921	int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext,
2922	struct rdma_user_mmap_entry *entry,
2923	size_t length, u32 min_pgoff,
2924	u32 max_pgoff);
2925
2926	static inline int
2927	rdma_user_mmap_entry_insert_exact(struct ib_ucontext *ucontext,
2928	struct rdma_user_mmap_entry *entry,
2929	size_t length, u32 pgoff)
2930	{
2931	return rdma_user_mmap_entry_insert_range(ucontext, entry, length, min_pgoff: pgoff,
2932	max_pgoff: pgoff);
2933	}
2934
2935	struct rdma_user_mmap_entry *
2936	rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext,
2937	unsigned long pgoff);
2938	struct rdma_user_mmap_entry *
2939	rdma_user_mmap_entry_get(struct ib_ucontext *ucontext,
2940	struct vm_area_struct *vma);
2941	void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry);
2942
2943	void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry);
2944
2945	static inline int ib_copy_from_udata(void *dest, struct ib_udata *udata, size_t len)
2946	{
2947	return copy_from_user(to: dest, from: udata->inbuf, n: len) ? -EFAULT : 0;
2948	}
2949
2950	static inline int ib_copy_to_udata(struct ib_udata *udata, void *src, size_t len)
2951	{
2952	return copy_to_user(to: udata->outbuf, from: src, n: len) ? -EFAULT : 0;
2953	}
2954
2955	static inline bool ib_is_buffer_cleared(const void __user *p,
2956	size_t len)
2957	{
2958	bool ret;
2959	u8 *buf;
2960
2961	if (len > USHRT_MAX)
2962	return false;
2963
2964	buf = memdup_user(p, len);
2965	if (IS_ERR(ptr: buf))
2966	return false;
2967
2968	ret = !memchr_inv(p: buf, c: 0, size: len);
2969	kfree(objp: buf);
2970	return ret;
2971	}
2972
2973	static inline bool ib_is_udata_cleared(struct ib_udata *udata,
2974	size_t offset,
2975	size_t len)
2976	{
2977	return ib_is_buffer_cleared(p: udata->inbuf + offset, len);
2978	}
2979
2980	/**
2981	* ib_modify_qp_is_ok - Check that the supplied attribute mask
2982	* contains all required attributes and no attributes not allowed for
2983	* the given QP state transition.
2984	* @cur_state: Current QP state
2985	* @next_state: Next QP state
2986	* @type: QP type
2987	* @mask: Mask of supplied QP attributes
2988	*
2989	* This function is a helper function that a low-level driver's
2990	* modify_qp method can use to validate the consumer's input. It
2991	* checks that cur_state and next_state are valid QP states, that a
2992	* transition from cur_state to next_state is allowed by the IB spec,
2993	* and that the attribute mask supplied is allowed for the transition.
2994	*/
2995	bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
2996	enum ib_qp_type type, enum ib_qp_attr_mask mask);
2997
2998	void ib_register_event_handler(struct ib_event_handler *event_handler);
2999	void ib_unregister_event_handler(struct ib_event_handler *event_handler);
3000	void ib_dispatch_event(const struct ib_event *event);
3001
3002	int ib_query_port(struct ib_device *device,
3003	u32 port_num, struct ib_port_attr *port_attr);
3004
3005	enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device,
3006	u32 port_num);
3007
3008	/**
3009	* rdma_cap_ib_switch - Check if the device is IB switch
3010	* @device: Device to check
3011	*
3012	* Device driver is responsible for setting is_switch bit on
3013	* in ib_device structure at init time.
3014	*
3015	* Return: true if the device is IB switch.
3016	*/
3017	static inline bool rdma_cap_ib_switch(const struct ib_device *device)
3018	{
3019	return device->is_switch;
3020	}
3021
3022	/**
3023	* rdma_start_port - Return the first valid port number for the device
3024	* specified
3025	*
3026	* @device: Device to be checked
3027	*
3028	* Return start port number
3029	*/
3030	static inline u32 rdma_start_port(const struct ib_device *device)
3031	{
3032	return rdma_cap_ib_switch(device) ? 0 : 1;
3033	}
3034
3035	/**
3036	* rdma_for_each_port - Iterate over all valid port numbers of the IB device
3037	* @device - The struct ib_device * to iterate over
3038	* @iter - The unsigned int to store the port number
3039	*/
3040	#define rdma_for_each_port(device, iter) \
3041	for (iter = rdma_start_port(device + \
3042	BUILD_BUG_ON_ZERO(!__same_type(u32, \
3043	iter))); \
3044	iter <= rdma_end_port(device); iter++)
3045
3046	/**
3047	* rdma_end_port - Return the last valid port number for the device
3048	* specified
3049	*
3050	* @device: Device to be checked
3051	*
3052	* Return last port number
3053	*/
3054	static inline u32 rdma_end_port(const struct ib_device *device)
3055	{
3056	return rdma_cap_ib_switch(device) ? 0 : device->phys_port_cnt;
3057	}
3058
3059	static inline int rdma_is_port_valid(const struct ib_device *device,
3060	unsigned int port)
3061	{
3062	return (port >= rdma_start_port(device) &&
3063	port <= rdma_end_port(device));
3064	}
3065
3066	static inline bool rdma_is_grh_required(const struct ib_device *device,
3067	u32 port_num)
3068	{
3069	return device->port_data[port_num].immutable.core_cap_flags &
3070	RDMA_CORE_PORT_IB_GRH_REQUIRED;
3071	}
3072
3073	static inline bool rdma_protocol_ib(const struct ib_device *device,
3074	u32 port_num)
3075	{
3076	return device->port_data[port_num].immutable.core_cap_flags &
3077	RDMA_CORE_CAP_PROT_IB;
3078	}
3079
3080	static inline bool rdma_protocol_roce(const struct ib_device *device,
3081	u32 port_num)
3082	{
3083	return device->port_data[port_num].immutable.core_cap_flags &
3084	(RDMA_CORE_CAP_PROT_ROCE | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP);
3085	}
3086
3087	static inline bool rdma_protocol_roce_udp_encap(const struct ib_device *device,
3088	u32 port_num)
3089	{
3090	return device->port_data[port_num].immutable.core_cap_flags &
3091	RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
3092	}
3093
3094	static inline bool rdma_protocol_roce_eth_encap(const struct ib_device *device,
3095	u32 port_num)
3096	{
3097	return device->port_data[port_num].immutable.core_cap_flags &
3098	RDMA_CORE_CAP_PROT_ROCE;
3099	}
3100
3101	static inline bool rdma_protocol_iwarp(const struct ib_device *device,
3102	u32 port_num)
3103	{
3104	return device->port_data[port_num].immutable.core_cap_flags &
3105	RDMA_CORE_CAP_PROT_IWARP;
3106	}
3107
3108	static inline bool rdma_ib_or_roce(const struct ib_device *device,
3109	u32 port_num)
3110	{
3111	return rdma_protocol_ib(device, port_num) ||
3112	rdma_protocol_roce(device, port_num);
3113	}
3114
3115	static inline bool rdma_protocol_raw_packet(const struct ib_device *device,
3116	u32 port_num)
3117	{
3118	return device->port_data[port_num].immutable.core_cap_flags &
3119	RDMA_CORE_CAP_PROT_RAW_PACKET;
3120	}
3121
3122	static inline bool rdma_protocol_usnic(const struct ib_device *device,
3123	u32 port_num)
3124	{
3125	return device->port_data[port_num].immutable.core_cap_flags &
3126	RDMA_CORE_CAP_PROT_USNIC;
3127	}
3128
3129	/**
3130	* rdma_cap_ib_mad - Check if the port of a device supports Infiniband
3131	* Management Datagrams.
3132	* @device: Device to check
3133	* @port_num: Port number to check
3134	*
3135	* Management Datagrams (MAD) are a required part of the InfiniBand
3136	* specification and are supported on all InfiniBand devices. A slightly
3137	* extended version are also supported on OPA interfaces.
3138	*
3139	* Return: true if the port supports sending/receiving of MAD packets.
3140	*/
3141	static inline bool rdma_cap_ib_mad(const struct ib_device *device, u32 port_num)
3142	{
3143	return device->port_data[port_num].immutable.core_cap_flags &
3144	RDMA_CORE_CAP_IB_MAD;
3145	}
3146
3147	/**
3148	* rdma_cap_opa_mad - Check if the port of device provides support for OPA
3149	* Management Datagrams.
3150	* @device: Device to check
3151	* @port_num: Port number to check
3152	*
3153	* Intel OmniPath devices extend and/or replace the InfiniBand Management
3154	* datagrams with their own versions. These OPA MADs share many but not all of
3155	* the characteristics of InfiniBand MADs.
3156	*
3157	* OPA MADs differ in the following ways:
3158	*
3159	* 1) MADs are variable size up to 2K
3160	* IBTA defined MADs remain fixed at 256 bytes
3161	* 2) OPA SMPs must carry valid PKeys
3162	* 3) OPA SMP packets are a different format
3163	*
3164	* Return: true if the port supports OPA MAD packet formats.
3165	*/
3166	static inline bool rdma_cap_opa_mad(struct ib_device *device, u32 port_num)
3167	{
3168	return device->port_data[port_num].immutable.core_cap_flags &
3169	RDMA_CORE_CAP_OPA_MAD;
3170	}
3171
3172	/**
3173	* rdma_cap_ib_smi - Check if the port of a device provides an Infiniband
3174	* Subnet Management Agent (SMA) on the Subnet Management Interface (SMI).
3175	* @device: Device to check
3176	* @port_num: Port number to check
3177	*
3178	* Each InfiniBand node is required to provide a Subnet Management Agent
3179	* that the subnet manager can access. Prior to the fabric being fully
3180	* configured by the subnet manager, the SMA is accessed via a well known
3181	* interface called the Subnet Management Interface (SMI). This interface
3182	* uses directed route packets to communicate with the SM to get around the
3183	* chicken and egg problem of the SM needing to know what's on the fabric
3184	* in order to configure the fabric, and needing to configure the fabric in
3185	* order to send packets to the devices on the fabric. These directed
3186	* route packets do not need the fabric fully configured in order to reach
3187	* their destination. The SMI is the only method allowed to send
3188	* directed route packets on an InfiniBand fabric.
3189	*
3190	* Return: true if the port provides an SMI.
3191	*/
3192	static inline bool rdma_cap_ib_smi(const struct ib_device *device, u32 port_num)
3193	{
3194	return device->port_data[port_num].immutable.core_cap_flags &
3195	RDMA_CORE_CAP_IB_SMI;
3196	}
3197
3198	/**
3199	* rdma_cap_ib_cm - Check if the port of device has the capability Infiniband
3200	* Communication Manager.
3201	* @device: Device to check
3202	* @port_num: Port number to check
3203	*
3204	* The InfiniBand Communication Manager is one of many pre-defined General
3205	* Service Agents (GSA) that are accessed via the General Service
3206	* Interface (GSI). It's role is to facilitate establishment of connections
3207	* between nodes as well as other management related tasks for established
3208	* connections.
3209	*
3210	* Return: true if the port supports an IB CM (this does not guarantee that
3211	* a CM is actually running however).
3212	*/
3213	static inline bool rdma_cap_ib_cm(const struct ib_device *device, u32 port_num)
3214	{
3215	return device->port_data[port_num].immutable.core_cap_flags &
3216	RDMA_CORE_CAP_IB_CM;
3217	}
3218
3219	/**
3220	* rdma_cap_iw_cm - Check if the port of device has the capability IWARP
3221	* Communication Manager.
3222	* @device: Device to check
3223	* @port_num: Port number to check
3224	*
3225	* Similar to above, but specific to iWARP connections which have a different
3226	* managment protocol than InfiniBand.
3227	*
3228	* Return: true if the port supports an iWARP CM (this does not guarantee that
3229	* a CM is actually running however).
3230	*/
3231	static inline bool rdma_cap_iw_cm(const struct ib_device *device, u32 port_num)
3232	{
3233	return device->port_data[port_num].immutable.core_cap_flags &
3234	RDMA_CORE_CAP_IW_CM;
3235	}
3236
3237	/**
3238	* rdma_cap_ib_sa - Check if the port of device has the capability Infiniband
3239	* Subnet Administration.
3240	* @device: Device to check
3241	* @port_num: Port number to check
3242	*
3243	* An InfiniBand Subnet Administration (SA) service is a pre-defined General
3244	* Service Agent (GSA) provided by the Subnet Manager (SM). On InfiniBand
3245	* fabrics, devices should resolve routes to other hosts by contacting the
3246	* SA to query the proper route.
3247	*
3248	* Return: true if the port should act as a client to the fabric Subnet
3249	* Administration interface. This does not imply that the SA service is
3250	* running locally.
3251	*/
3252	static inline bool rdma_cap_ib_sa(const struct ib_device *device, u32 port_num)
3253	{
3254	return device->port_data[port_num].immutable.core_cap_flags &
3255	RDMA_CORE_CAP_IB_SA;
3256	}
3257
3258	/**
3259	* rdma_cap_ib_mcast - Check if the port of device has the capability Infiniband
3260	* Multicast.
3261	* @device: Device to check
3262	* @port_num: Port number to check
3263	*
3264	* InfiniBand multicast registration is more complex than normal IPv4 or
3265	* IPv6 multicast registration. Each Host Channel Adapter must register
3266	* with the Subnet Manager when it wishes to join a multicast group. It
3267	* should do so only once regardless of how many queue pairs it subscribes
3268	* to this group. And it should leave the group only after all queue pairs
3269	* attached to the group have been detached.
3270	*
3271	* Return: true if the port must undertake the additional adminstrative
3272	* overhead of registering/unregistering with the SM and tracking of the
3273	* total number of queue pairs attached to the multicast group.
3274	*/
3275	static inline bool rdma_cap_ib_mcast(const struct ib_device *device,
3276	u32 port_num)
3277	{
3278	return rdma_cap_ib_sa(device, port_num);
3279	}
3280
3281	/**
3282	* rdma_cap_af_ib - Check if the port of device has the capability
3283	* Native Infiniband Address.
3284	* @device: Device to check
3285	* @port_num: Port number to check
3286	*
3287	* InfiniBand addressing uses a port's GUID + Subnet Prefix to make a default
3288	* GID. RoCE uses a different mechanism, but still generates a GID via
3289	* a prescribed mechanism and port specific data.
3290	*
3291	* Return: true if the port uses a GID address to identify devices on the
3292	* network.
3293	*/
3294	static inline bool rdma_cap_af_ib(const struct ib_device *device, u32 port_num)
3295	{
3296	return device->port_data[port_num].immutable.core_cap_flags &
3297	RDMA_CORE_CAP_AF_IB;
3298	}
3299
3300	/**
3301	* rdma_cap_eth_ah - Check if the port of device has the capability
3302	* Ethernet Address Handle.
3303	* @device: Device to check
3304	* @port_num: Port number to check
3305	*
3306	* RoCE is InfiniBand over Ethernet, and it uses a well defined technique
3307	* to fabricate GIDs over Ethernet/IP specific addresses native to the
3308	* port. Normally, packet headers are generated by the sending host
3309	* adapter, but when sending connectionless datagrams, we must manually
3310	* inject the proper headers for the fabric we are communicating over.
3311	*
3312	* Return: true if we are running as a RoCE port and must force the
3313	* addition of a Global Route Header built from our Ethernet Address
3314	* Handle into our header list for connectionless packets.
3315	*/
3316	static inline bool rdma_cap_eth_ah(const struct ib_device *device, u32 port_num)
3317	{
3318	return device->port_data[port_num].immutable.core_cap_flags &
3319	RDMA_CORE_CAP_ETH_AH;
3320	}
3321
3322	/**
3323	* rdma_cap_opa_ah - Check if the port of device supports
3324	* OPA Address handles
3325	* @device: Device to check
3326	* @port_num: Port number to check
3327	*
3328	* Return: true if we are running on an OPA device which supports
3329	* the extended OPA addressing.
3330	*/
3331	static inline bool rdma_cap_opa_ah(struct ib_device *device, u32 port_num)
3332	{
3333	return (device->port_data[port_num].immutable.core_cap_flags &
3334	RDMA_CORE_CAP_OPA_AH) == RDMA_CORE_CAP_OPA_AH;
3335	}
3336
3337	/**
3338	* rdma_max_mad_size - Return the max MAD size required by this RDMA Port.
3339	*
3340	* @device: Device
3341	* @port_num: Port number
3342	*
3343	* This MAD size includes the MAD headers and MAD payload. No other headers
3344	* are included.
3345	*
3346	* Return the max MAD size required by the Port. Will return 0 if the port
3347	* does not support MADs
3348	*/
3349	static inline size_t rdma_max_mad_size(const struct ib_device *device,
3350	u32 port_num)
3351	{
3352	return device->port_data[port_num].immutable.max_mad_size;
3353	}
3354
3355	/**
3356	* rdma_cap_roce_gid_table - Check if the port of device uses roce_gid_table
3357	* @device: Device to check
3358	* @port_num: Port number to check
3359	*
3360	* RoCE GID table mechanism manages the various GIDs for a device.
3361	*
3362	* NOTE: if allocating the port's GID table has failed, this call will still
3363	* return true, but any RoCE GID table API will fail.
3364	*
3365	* Return: true if the port uses RoCE GID table mechanism in order to manage
3366	* its GIDs.
3367	*/
3368	static inline bool rdma_cap_roce_gid_table(const struct ib_device *device,
3369	u32 port_num)
3370	{
3371	return rdma_protocol_roce(device, port_num) &&
3372	device->ops.add_gid && device->ops.del_gid;
3373	}
3374
3375	/*
3376	* Check if the device supports READ W/ INVALIDATE.
3377	*/
3378	static inline bool rdma_cap_read_inv(struct ib_device *dev, u32 port_num)
3379	{
3380	/*
3381	* iWarp drivers must support READ W/ INVALIDATE. No other protocol
3382	* has support for it yet.
3383	*/
3384	return rdma_protocol_iwarp(device: dev, port_num);
3385	}
3386
3387	/**
3388	* rdma_core_cap_opa_port - Return whether the RDMA Port is OPA or not.
3389	* @device: Device
3390	* @port_num: 1 based Port number
3391	*
3392	* Return true if port is an Intel OPA port , false if not
3393	*/
3394	static inline bool rdma_core_cap_opa_port(struct ib_device *device,
3395	u32 port_num)
3396	{
3397	return (device->port_data[port_num].immutable.core_cap_flags &
3398	RDMA_CORE_PORT_INTEL_OPA) == RDMA_CORE_PORT_INTEL_OPA;
3399	}
3400
3401	/**
3402	* rdma_mtu_enum_to_int - Return the mtu of the port as an integer value.
3403	* @device: Device
3404	* @port_num: Port number
3405	* @mtu: enum value of MTU
3406	*
3407	* Return the MTU size supported by the port as an integer value. Will return
3408	* -1 if enum value of mtu is not supported.
3409	*/
3410	static inline int rdma_mtu_enum_to_int(struct ib_device *device, u32 port,
3411	int mtu)
3412	{
3413	if (rdma_core_cap_opa_port(device, port_num: port))
3414	return opa_mtu_enum_to_int(mtu: (enum opa_mtu)mtu);
3415	else
3416	return ib_mtu_enum_to_int(mtu: (enum ib_mtu)mtu);
3417	}
3418
3419	/**
3420	* rdma_mtu_from_attr - Return the mtu of the port from the port attribute.
3421	* @device: Device
3422	* @port_num: Port number
3423	* @attr: port attribute
3424	*
3425	* Return the MTU size supported by the port as an integer value.
3426	*/
3427	static inline int rdma_mtu_from_attr(struct ib_device *device, u32 port,
3428	struct ib_port_attr *attr)
3429	{
3430	if (rdma_core_cap_opa_port(device, port_num: port))
3431	return attr->phys_mtu;
3432	else
3433	return ib_mtu_enum_to_int(mtu: attr->max_mtu);
3434	}
3435
3436	int ib_set_vf_link_state(struct ib_device *device, int vf, u32 port,
3437	int state);
3438	int ib_get_vf_config(struct ib_device *device, int vf, u32 port,
3439	struct ifla_vf_info *info);
3440	int ib_get_vf_stats(struct ib_device *device, int vf, u32 port,
3441	struct ifla_vf_stats *stats);
3442	int ib_get_vf_guid(struct ib_device *device, int vf, u32 port,
3443	struct ifla_vf_guid *node_guid,
3444	struct ifla_vf_guid *port_guid);
3445	int ib_set_vf_guid(struct ib_device *device, int vf, u32 port, u64 guid,
3446	int type);
3447
3448	int ib_query_pkey(struct ib_device *device,
3449	u32 port_num, u16 index, u16 *pkey);
3450
3451	int ib_modify_device(struct ib_device *device,
3452	int device_modify_mask,
3453	struct ib_device_modify *device_modify);
3454
3455	int ib_modify_port(struct ib_device *device,
3456	u32 port_num, int port_modify_mask,
3457	struct ib_port_modify *port_modify);
3458
3459	int ib_find_gid(struct ib_device *device, union ib_gid *gid,
3460	u32 *port_num, u16 *index);
3461
3462	int ib_find_pkey(struct ib_device *device,
3463	u32 port_num, u16 pkey, u16 *index);
3464
3465	enum ib_pd_flags {
3466	/*
3467	* Create a memory registration for all memory in the system and place
3468	* the rkey for it into pd->unsafe_global_rkey. This can be used by
3469	* ULPs to avoid the overhead of dynamic MRs.
3470	*
3471	* This flag is generally considered unsafe and must only be used in
3472	* extremly trusted environments. Every use of it will log a warning
3473	* in the kernel log.
3474	*/
3475	IB_PD_UNSAFE_GLOBAL_RKEY = 0x01,
3476	};
3477
3478	struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
3479	const char *caller);
3480
3481	/**
3482	* ib_alloc_pd - Allocates an unused protection domain.
3483	* @device: The device on which to allocate the protection domain.
3484	* @flags: protection domain flags
3485	*
3486	* A protection domain object provides an association between QPs, shared
3487	* receive queues, address handles, memory regions, and memory windows.
3488	*
3489	* Every PD has a local_dma_lkey which can be used as the lkey value for local
3490	* memory operations.
3491	*/
3492	#define ib_alloc_pd(device, flags) \
3493	__ib_alloc_pd((device), (flags), KBUILD_MODNAME)
3494
3495	int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata);
3496
3497	/**
3498	* ib_dealloc_pd - Deallocate kernel PD
3499	* @pd: The protection domain
3500	*
3501	* NOTE: for user PD use ib_dealloc_pd_user with valid udata!
3502	*/
3503	static inline void ib_dealloc_pd(struct ib_pd *pd)
3504	{
3505	int ret = ib_dealloc_pd_user(pd, NULL);
3506
3507	WARN_ONCE(ret, "Destroy of kernel PD shouldn't fail");
3508	}
3509
3510	enum rdma_create_ah_flags {
3511	/* In a sleepable context */
3512	RDMA_CREATE_AH_SLEEPABLE = BIT(0),
3513	};
3514
3515	/**
3516	* rdma_create_ah - Creates an address handle for the given address vector.
3517	* @pd: The protection domain associated with the address handle.
3518	* @ah_attr: The attributes of the address vector.
3519	* @flags: Create address handle flags (see enum rdma_create_ah_flags).
3520	*
3521	* The address handle is used to reference a local or global destination
3522	* in all UD QP post sends.
3523	*/
3524	struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
3525	u32 flags);
3526
3527	/**
3528	* rdma_create_user_ah - Creates an address handle for the given address vector.
3529	* It resolves destination mac address for ah attribute of RoCE type.
3530	* @pd: The protection domain associated with the address handle.
3531	* @ah_attr: The attributes of the address vector.
3532	* @udata: pointer to user's input output buffer information need by
3533	* provider driver.
3534	*
3535	* It returns 0 on success and returns appropriate error code on error.
3536	* The address handle is used to reference a local or global destination
3537	* in all UD QP post sends.
3538	*/
3539	struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,
3540	struct rdma_ah_attr *ah_attr,
3541	struct ib_udata *udata);
3542	/**
3543	* ib_get_gids_from_rdma_hdr - Get sgid and dgid from GRH or IPv4 header
3544	* work completion.
3545	* @hdr: the L3 header to parse
3546	* @net_type: type of header to parse
3547	* @sgid: place to store source gid
3548	* @dgid: place to store destination gid
3549	*/
3550	int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
3551	enum rdma_network_type net_type,
3552	union ib_gid *sgid, union ib_gid *dgid);
3553
3554	/**
3555	* ib_get_rdma_header_version - Get the header version
3556	* @hdr: the L3 header to parse
3557	*/
3558	int ib_get_rdma_header_version(const union rdma_network_hdr *hdr);
3559
3560	/**
3561	* ib_init_ah_attr_from_wc - Initializes address handle attributes from a
3562	* work completion.
3563	* @device: Device on which the received message arrived.
3564	* @port_num: Port on which the received message arrived.
3565	* @wc: Work completion associated with the received message.
3566	* @grh: References the received global route header. This parameter is
3567	* ignored unless the work completion indicates that the GRH is valid.
3568	* @ah_attr: Returned attributes that can be used when creating an address
3569	* handle for replying to the message.
3570	* When ib_init_ah_attr_from_wc() returns success,
3571	* (a) for IB link layer it optionally contains a reference to SGID attribute
3572	* when GRH is present for IB link layer.
3573	* (b) for RoCE link layer it contains a reference to SGID attribute.
3574	* User must invoke rdma_cleanup_ah_attr_gid_attr() to release reference to SGID
3575	* attributes which are initialized using ib_init_ah_attr_from_wc().
3576	*
3577	*/
3578	int ib_init_ah_attr_from_wc(struct ib_device *device, u32 port_num,
3579	const struct ib_wc *wc, const struct ib_grh *grh,
3580	struct rdma_ah_attr *ah_attr);
3581
3582	/**
3583	* ib_create_ah_from_wc - Creates an address handle associated with the
3584	* sender of the specified work completion.
3585	* @pd: The protection domain associated with the address handle.
3586	* @wc: Work completion information associated with a received message.
3587	* @grh: References the received global route header. This parameter is
3588	* ignored unless the work completion indicates that the GRH is valid.
3589	* @port_num: The outbound port number to associate with the address.
3590	*
3591	* The address handle is used to reference a local or global destination
3592	* in all UD QP post sends.
3593	*/
3594	struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
3595	const struct ib_grh *grh, u32 port_num);
3596
3597	/**
3598	* rdma_modify_ah - Modifies the address vector associated with an address
3599	* handle.
3600	* @ah: The address handle to modify.
3601	* @ah_attr: The new address vector attributes to associate with the
3602	* address handle.
3603	*/
3604	int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
3605
3606	/**
3607	* rdma_query_ah - Queries the address vector associated with an address
3608	* handle.
3609	* @ah: The address handle to query.
3610	* @ah_attr: The address vector attributes associated with the address
3611	* handle.
3612	*/
3613	int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
3614
3615	enum rdma_destroy_ah_flags {
3616	/* In a sleepable context */
3617	RDMA_DESTROY_AH_SLEEPABLE = BIT(0),
3618	};
3619
3620	/**
3621	* rdma_destroy_ah_user - Destroys an address handle.
3622	* @ah: The address handle to destroy.
3623	* @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).
3624	* @udata: Valid user data or NULL for kernel objects
3625	*/
3626	int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata);
3627
3628	/**
3629	* rdma_destroy_ah - Destroys an kernel address handle.
3630	* @ah: The address handle to destroy.
3631	* @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).
3632	*
3633	* NOTE: for user ah use rdma_destroy_ah_user with valid udata!
3634	*/
3635	static inline void rdma_destroy_ah(struct ib_ah *ah, u32 flags)
3636	{
3637	int ret = rdma_destroy_ah_user(ah, flags, NULL);
3638
3639	WARN_ONCE(ret, "Destroy of kernel AH shouldn't fail");
3640	}
3641
3642	struct ib_srq *ib_create_srq_user(struct ib_pd *pd,
3643	struct ib_srq_init_attr *srq_init_attr,
3644	struct ib_usrq_object *uobject,
3645	struct ib_udata *udata);
3646	static inline struct ib_srq *
3647	ib_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *srq_init_attr)
3648	{
3649	if (!pd->device->ops.create_srq)
3650	return ERR_PTR(error: -EOPNOTSUPP);
3651
3652	return ib_create_srq_user(pd, srq_init_attr, NULL, NULL);
3653	}
3654
3655	/**
3656	* ib_modify_srq - Modifies the attributes for the specified SRQ.
3657	* @srq: The SRQ to modify.
3658	* @srq_attr: On input, specifies the SRQ attributes to modify. On output,
3659	* the current values of selected SRQ attributes are returned.
3660	* @srq_attr_mask: A bit-mask used to specify which attributes of the SRQ
3661	* are being modified.
3662	*
3663	* The mask may contain IB_SRQ_MAX_WR to resize the SRQ and/or
3664	* IB_SRQ_LIMIT to set the SRQ's limit and request notification when
3665	* the number of receives queued drops below the limit.
3666	*/
3667	int ib_modify_srq(struct ib_srq *srq,
3668	struct ib_srq_attr *srq_attr,
3669	enum ib_srq_attr_mask srq_attr_mask);
3670
3671	/**
3672	* ib_query_srq - Returns the attribute list and current values for the
3673	* specified SRQ.
3674	* @srq: The SRQ to query.
3675	* @srq_attr: The attributes of the specified SRQ.
3676	*/
3677	int ib_query_srq(struct ib_srq *srq,
3678	struct ib_srq_attr *srq_attr);
3679
3680	/**
3681	* ib_destroy_srq_user - Destroys the specified SRQ.
3682	* @srq: The SRQ to destroy.
3683	* @udata: Valid user data or NULL for kernel objects
3684	*/
3685	int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata);
3686
3687	/**
3688	* ib_destroy_srq - Destroys the specified kernel SRQ.
3689	* @srq: The SRQ to destroy.
3690	*
3691	* NOTE: for user srq use ib_destroy_srq_user with valid udata!
3692	*/
3693	static inline void ib_destroy_srq(struct ib_srq *srq)
3694	{
3695	int ret = ib_destroy_srq_user(srq, NULL);
3696
3697	WARN_ONCE(ret, "Destroy of kernel SRQ shouldn't fail");
3698	}
3699
3700	/**
3701	* ib_post_srq_recv - Posts a list of work requests to the specified SRQ.
3702	* @srq: The SRQ to post the work request on.
3703	* @recv_wr: A list of work requests to post on the receive queue.
3704	* @bad_recv_wr: On an immediate failure, this parameter will reference
3705	* the work request that failed to be posted on the QP.
3706	*/
3707	static inline int ib_post_srq_recv(struct ib_srq *srq,
3708	const struct ib_recv_wr *recv_wr,
3709	const struct ib_recv_wr **bad_recv_wr)
3710	{
3711	const struct ib_recv_wr *dummy;
3712
3713	return srq->device->ops.post_srq_recv(srq, recv_wr,
3714	bad_recv_wr ? : &dummy);
3715	}
3716
3717	struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd,
3718	struct ib_qp_init_attr *qp_init_attr,
3719	const char *caller);
3720	/**
3721	* ib_create_qp - Creates a kernel QP associated with the specific protection
3722	* domain.
3723	* @pd: The protection domain associated with the QP.
3724	* @init_attr: A list of initial attributes required to create the
3725	* QP. If QP creation succeeds, then the attributes are updated to
3726	* the actual capabilities of the created QP.
3727	*/
3728	static inline struct ib_qp *ib_create_qp(struct ib_pd *pd,
3729	struct ib_qp_init_attr *init_attr)
3730	{
3731	return ib_create_qp_kernel(pd, qp_init_attr: init_attr, KBUILD_MODNAME);
3732	}
3733
3734	/**
3735	* ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
3736	* @qp: The QP to modify.
3737	* @attr: On input, specifies the QP attributes to modify. On output,
3738	* the current values of selected QP attributes are returned.
3739	* @attr_mask: A bit-mask used to specify which attributes of the QP
3740	* are being modified.
3741	* @udata: pointer to user's input output buffer information
3742	* are being modified.
3743	* It returns 0 on success and returns appropriate error code on error.
3744	*/
3745	int ib_modify_qp_with_udata(struct ib_qp *qp,
3746	struct ib_qp_attr *attr,
3747	int attr_mask,
3748	struct ib_udata *udata);
3749
3750	/**
3751	* ib_modify_qp - Modifies the attributes for the specified QP and then
3752	* transitions the QP to the given state.
3753	* @qp: The QP to modify.
3754	* @qp_attr: On input, specifies the QP attributes to modify. On output,
3755	* the current values of selected QP attributes are returned.
3756	* @qp_attr_mask: A bit-mask used to specify which attributes of the QP
3757	* are being modified.
3758	*/
3759	int ib_modify_qp(struct ib_qp *qp,
3760	struct ib_qp_attr *qp_attr,
3761	int qp_attr_mask);
3762
3763	/**
3764	* ib_query_qp - Returns the attribute list and current values for the
3765	* specified QP.
3766	* @qp: The QP to query.
3767	* @qp_attr: The attributes of the specified QP.
3768	* @qp_attr_mask: A bit-mask used to select specific attributes to query.
3769	* @qp_init_attr: Additional attributes of the selected QP.
3770	*
3771	* The qp_attr_mask may be used to limit the query to gathering only the
3772	* selected attributes.
3773	*/
3774	int ib_query_qp(struct ib_qp *qp,
3775	struct ib_qp_attr *qp_attr,
3776	int qp_attr_mask,
3777	struct ib_qp_init_attr *qp_init_attr);
3778
3779	/**
3780	* ib_destroy_qp - Destroys the specified QP.
3781	* @qp: The QP to destroy.
3782	* @udata: Valid udata or NULL for kernel objects
3783	*/
3784	int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata);
3785
3786	/**
3787	* ib_destroy_qp - Destroys the specified kernel QP.
3788	* @qp: The QP to destroy.
3789	*
3790	* NOTE: for user qp use ib_destroy_qp_user with valid udata!
3791	*/
3792	static inline int ib_destroy_qp(struct ib_qp *qp)
3793	{
3794	return ib_destroy_qp_user(qp, NULL);
3795	}
3796
3797	/**
3798	* ib_open_qp - Obtain a reference to an existing sharable QP.
3799	* @xrcd - XRC domain
3800	* @qp_open_attr: Attributes identifying the QP to open.
3801	*
3802	* Returns a reference to a sharable QP.
3803	*/
3804	struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
3805	struct ib_qp_open_attr *qp_open_attr);
3806
3807	/**
3808	* ib_close_qp - Release an external reference to a QP.
3809	* @qp: The QP handle to release
3810	*
3811	* The opened QP handle is released by the caller. The underlying
3812	* shared QP is not destroyed until all internal references are released.
3813	*/
3814	int ib_close_qp(struct ib_qp *qp);
3815
3816	/**
3817	* ib_post_send - Posts a list of work requests to the send queue of
3818	* the specified QP.
3819	* @qp: The QP to post the work request on.
3820	* @send_wr: A list of work requests to post on the send queue.
3821	* @bad_send_wr: On an immediate failure, this parameter will reference
3822	* the work request that failed to be posted on the QP.
3823	*
3824	* While IBA Vol. 1 section 11.4.1.1 specifies that if an immediate
3825	* error is returned, the QP state shall not be affected,
3826	* ib_post_send() will return an immediate error after queueing any
3827	* earlier work requests in the list.
3828	*/
3829	static inline int ib_post_send(struct ib_qp *qp,
3830	const struct ib_send_wr *send_wr,
3831	const struct ib_send_wr **bad_send_wr)
3832	{
3833	const struct ib_send_wr *dummy;
3834
3835	return qp->device->ops.post_send(qp, send_wr, bad_send_wr ? : &dummy);
3836	}
3837
3838	/**
3839	* ib_post_recv - Posts a list of work requests to the receive queue of
3840	* the specified QP.
3841	* @qp: The QP to post the work request on.
3842	* @recv_wr: A list of work requests to post on the receive queue.
3843	* @bad_recv_wr: On an immediate failure, this parameter will reference
3844	* the work request that failed to be posted on the QP.
3845	*/
3846	static inline int ib_post_recv(struct ib_qp *qp,
3847	const struct ib_recv_wr *recv_wr,
3848	const struct ib_recv_wr **bad_recv_wr)
3849	{
3850	const struct ib_recv_wr *dummy;
3851
3852	return qp->device->ops.post_recv(qp, recv_wr, bad_recv_wr ? : &dummy);
3853	}
3854
3855	struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe,
3856	int comp_vector, enum ib_poll_context poll_ctx,
3857	const char *caller);
3858	static inline struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private,
3859	int nr_cqe, int comp_vector,
3860	enum ib_poll_context poll_ctx)
3861	{
3862	return __ib_alloc_cq(dev, private, nr_cqe, comp_vector, poll_ctx,
3863	KBUILD_MODNAME);
3864	}
3865
3866	struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private,
3867	int nr_cqe, enum ib_poll_context poll_ctx,
3868	const char *caller);
3869
3870	/**
3871	* ib_alloc_cq_any: Allocate kernel CQ
3872	* @dev: The IB device
3873	* @private: Private data attached to the CQE
3874	* @nr_cqe: Number of CQEs in the CQ
3875	* @poll_ctx: Context used for polling the CQ
3876	*/
3877	static inline struct ib_cq *ib_alloc_cq_any(struct ib_device *dev,
3878	void *private, int nr_cqe,
3879	enum ib_poll_context poll_ctx)
3880	{
3881	return __ib_alloc_cq_any(dev, private, nr_cqe, poll_ctx,
3882	KBUILD_MODNAME);
3883	}
3884
3885	void ib_free_cq(struct ib_cq *cq);
3886	int ib_process_cq_direct(struct ib_cq *cq, int budget);
3887
3888	/**
3889	* ib_create_cq - Creates a CQ on the specified device.
3890	* @device: The device on which to create the CQ.
3891	* @comp_handler: A user-specified callback that is invoked when a
3892	* completion event occurs on the CQ.
3893	* @event_handler: A user-specified callback that is invoked when an
3894	* asynchronous event not associated with a completion occurs on the CQ.
3895	* @cq_context: Context associated with the CQ returned to the user via
3896	* the associated completion and event handlers.
3897	* @cq_attr: The attributes the CQ should be created upon.
3898	*
3899	* Users can examine the cq structure to determine the actual CQ size.
3900	*/
3901	struct ib_cq *__ib_create_cq(struct ib_device *device,
3902	ib_comp_handler comp_handler,
3903	void (*event_handler)(struct ib_event *, void *),
3904	void *cq_context,
3905	const struct ib_cq_init_attr *cq_attr,
3906	const char *caller);
3907	#define ib_create_cq(device, cmp_hndlr, evt_hndlr, cq_ctxt, cq_attr) \
3908	__ib_create_cq((device), (cmp_hndlr), (evt_hndlr), (cq_ctxt), (cq_attr), KBUILD_MODNAME)
3909
3910	/**
3911	* ib_resize_cq - Modifies the capacity of the CQ.
3912	* @cq: The CQ to resize.
3913	* @cqe: The minimum size of the CQ.
3914	*
3915	* Users can examine the cq structure to determine the actual CQ size.
3916	*/
3917	int ib_resize_cq(struct ib_cq *cq, int cqe);
3918
3919	/**
3920	* rdma_set_cq_moderation - Modifies moderation params of the CQ
3921	* @cq: The CQ to modify.
3922	* @cq_count: number of CQEs that will trigger an event
3923	* @cq_period: max period of time in usec before triggering an event
3924	*
3925	*/
3926	int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period);
3927
3928	/**
3929	* ib_destroy_cq_user - Destroys the specified CQ.
3930	* @cq: The CQ to destroy.
3931	* @udata: Valid user data or NULL for kernel objects
3932	*/
3933	int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata);
3934
3935	/**
3936	* ib_destroy_cq - Destroys the specified kernel CQ.
3937	* @cq: The CQ to destroy.
3938	*
3939	* NOTE: for user cq use ib_destroy_cq_user with valid udata!
3940	*/
3941	static inline void ib_destroy_cq(struct ib_cq *cq)
3942	{
3943	int ret = ib_destroy_cq_user(cq, NULL);
3944
3945	WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail");
3946	}
3947
3948	/**
3949	* ib_poll_cq - poll a CQ for completion(s)
3950	* @cq:the CQ being polled
3951	* @num_entries:maximum number of completions to return
3952	* @wc:array of at least @num_entries &struct ib_wc where completions
3953	* will be returned
3954	*
3955	* Poll a CQ for (possibly multiple) completions. If the return value
3956	* is < 0, an error occurred. If the return value is >= 0, it is the
3957	* number of completions returned. If the return value is
3958	* non-negative and < num_entries, then the CQ was emptied.
3959	*/
3960	static inline int ib_poll_cq(struct ib_cq *cq, int num_entries,
3961	struct ib_wc *wc)
3962	{
3963	return cq->device->ops.poll_cq(cq, num_entries, wc);
3964	}
3965
3966	/**
3967	* ib_req_notify_cq - Request completion notification on a CQ.
3968	* @cq: The CQ to generate an event for.
3969	* @flags:
3970	* Must contain exactly one of %IB_CQ_SOLICITED or %IB_CQ_NEXT_COMP
3971	* to request an event on the next solicited event or next work
3972	* completion at any type, respectively. %IB_CQ_REPORT_MISSED_EVENTS
3973	* may also be |ed in to request a hint about missed events, as
3974	* described below.
3975	*
3976	* Return Value:
3977	* < 0 means an error occurred while requesting notification
3978	* == 0 means notification was requested successfully, and if
3979	* IB_CQ_REPORT_MISSED_EVENTS was passed in, then no events
3980	* were missed and it is safe to wait for another event. In
3981	* this case is it guaranteed that any work completions added
3982	* to the CQ since the last CQ poll will trigger a completion
3983	* notification event.
3984	* > 0 is only returned if IB_CQ_REPORT_MISSED_EVENTS was passed
3985	* in. It means that the consumer must poll the CQ again to
3986	* make sure it is empty to avoid missing an event because of a
3987	* race between requesting notification and an entry being
3988	* added to the CQ. This return value means it is possible
3989	* (but not guaranteed) that a work completion has been added
3990	* to the CQ since the last poll without triggering a
3991	* completion notification event.
3992	*/
3993	static inline int ib_req_notify_cq(struct ib_cq *cq,
3994	enum ib_cq_notify_flags flags)
3995	{
3996	return cq->device->ops.req_notify_cq(cq, flags);
3997	}
3998
3999	struct ib_cq *ib_cq_pool_get(struct ib_device *dev, unsigned int nr_cqe,
4000	int comp_vector_hint,
4001	enum ib_poll_context poll_ctx);
4002
4003	void ib_cq_pool_put(struct ib_cq *cq, unsigned int nr_cqe);
4004
4005	/*
4006	* Drivers that don't need a DMA mapping at the RDMA layer, set dma_device to
4007	* NULL. This causes the ib_dma* helpers to just stash the kernel virtual
4008	* address into the dma address.
4009	*/
4010	static inline bool ib_uses_virt_dma(struct ib_device *dev)
4011	{
4012	return IS_ENABLED(CONFIG_INFINIBAND_VIRT_DMA) && !dev->dma_device;
4013	}
4014
4015	/*
4016	* Check if a IB device's underlying DMA mapping supports P2PDMA transfers.
4017	*/
4018	static inline bool ib_dma_pci_p2p_dma_supported(struct ib_device *dev)
4019	{
4020	if (ib_uses_virt_dma(dev))
4021	return false;
4022
4023	return dma_pci_p2pdma_supported(dev: dev->dma_device);
4024	}
4025
4026	/**
4027	* ib_virt_dma_to_ptr - Convert a dma_addr to a kernel pointer
4028	* @dma_addr: The DMA address
4029	*
4030	* Used by ib_uses_virt_dma() devices to get back to the kernel pointer after
4031	* going through the dma_addr marshalling.
4032	*/
4033	static inline void *ib_virt_dma_to_ptr(u64 dma_addr)
4034	{
4035	/* virt_dma mode maps the kvs's directly into the dma addr */
4036	return (void *)(uintptr_t)dma_addr;
4037	}
4038
4039	/**
4040	* ib_virt_dma_to_page - Convert a dma_addr to a struct page
4041	* @dma_addr: The DMA address
4042	*
4043	* Used by ib_uses_virt_dma() device to get back to the struct page after going
4044	* through the dma_addr marshalling.
4045	*/
4046	static inline struct page *ib_virt_dma_to_page(u64 dma_addr)
4047	{
4048	return virt_to_page(ib_virt_dma_to_ptr(dma_addr));
4049	}
4050
4051	/**
4052	* ib_dma_mapping_error - check a DMA addr for error
4053	* @dev: The device for which the dma_addr was created
4054	* @dma_addr: The DMA address to check
4055	*/
4056	static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
4057	{
4058	if (ib_uses_virt_dma(dev))
4059	return 0;
4060	return dma_mapping_error(dev: dev->dma_device, dma_addr);
4061	}
4062
4063	/**
4064	* ib_dma_map_single - Map a kernel virtual address to DMA address
4065	* @dev: The device for which the dma_addr is to be created
4066	* @cpu_addr: The kernel virtual address
4067	* @size: The size of the region in bytes
4068	* @direction: The direction of the DMA
4069	*/
4070	static inline u64 ib_dma_map_single(struct ib_device *dev,
4071	void *cpu_addr, size_t size,
4072	enum dma_data_direction direction)
4073	{
4074	if (ib_uses_virt_dma(dev))
4075	return (uintptr_t)cpu_addr;
4076	return dma_map_single(dev->dma_device, cpu_addr, size, direction);
4077	}
4078
4079	/**
4080	* ib_dma_unmap_single - Destroy a mapping created by ib_dma_map_single()
4081	* @dev: The device for which the DMA address was created
4082	* @addr: The DMA address
4083	* @size: The size of the region in bytes
4084	* @direction: The direction of the DMA
4085	*/
4086	static inline void ib_dma_unmap_single(struct ib_device *dev,
4087	u64 addr, size_t size,
4088	enum dma_data_direction direction)
4089	{
4090	if (!ib_uses_virt_dma(dev))
4091	dma_unmap_single(dev->dma_device, addr, size, direction);
4092	}
4093
4094	/**
4095	* ib_dma_map_page - Map a physical page to DMA address
4096	* @dev: The device for which the dma_addr is to be created
4097	* @page: The page to be mapped
4098	* @offset: The offset within the page
4099	* @size: The size of the region in bytes
4100	* @direction: The direction of the DMA
4101	*/
4102	static inline u64 ib_dma_map_page(struct ib_device *dev,
4103	struct page *page,
4104	unsigned long offset,
4105	size_t size,
4106	enum dma_data_direction direction)
4107	{
4108	if (ib_uses_virt_dma(dev))
4109	return (uintptr_t)(page_address(page) + offset);
4110	return dma_map_page(dev->dma_device, page, offset, size, direction);
4111	}
4112
4113	/**
4114	* ib_dma_unmap_page - Destroy a mapping created by ib_dma_map_page()
4115	* @dev: The device for which the DMA address was created
4116	* @addr: The DMA address
4117	* @size: The size of the region in bytes
4118	* @direction: The direction of the DMA
4119	*/
4120	static inline void ib_dma_unmap_page(struct ib_device *dev,
4121	u64 addr, size_t size,
4122	enum dma_data_direction direction)
4123	{
4124	if (!ib_uses_virt_dma(dev))
4125	dma_unmap_page(dev->dma_device, addr, size, direction);
4126	}
4127
4128	int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents);
4129	static inline int ib_dma_map_sg_attrs(struct ib_device *dev,
4130	struct scatterlist *sg, int nents,
4131	enum dma_data_direction direction,
4132	unsigned long dma_attrs)
4133	{
4134	if (ib_uses_virt_dma(dev))
4135	return ib_dma_virt_map_sg(dev, sg, nents);
4136	return dma_map_sg_attrs(dev: dev->dma_device, sg, nents, dir: direction,
4137	attrs: dma_attrs);
4138	}
4139
4140	static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev,
4141	struct scatterlist *sg, int nents,
4142	enum dma_data_direction direction,
4143	unsigned long dma_attrs)
4144	{
4145	if (!ib_uses_virt_dma(dev))
4146	dma_unmap_sg_attrs(dev: dev->dma_device, sg, nents, dir: direction,
4147	attrs: dma_attrs);
4148	}
4149
4150	/**
4151	* ib_dma_map_sgtable_attrs - Map a scatter/gather table to DMA addresses
4152	* @dev: The device for which the DMA addresses are to be created
4153	* @sg: The sg_table object describing the buffer
4154	* @direction: The direction of the DMA
4155	* @attrs: Optional DMA attributes for the map operation
4156	*/
4157	static inline int ib_dma_map_sgtable_attrs(struct ib_device *dev,
4158	struct sg_table *sgt,
4159	enum dma_data_direction direction,
4160	unsigned long dma_attrs)
4161	{
4162	int nents;
4163
4164	if (ib_uses_virt_dma(dev)) {
4165	nents = ib_dma_virt_map_sg(dev, sg: sgt->sgl, nents: sgt->orig_nents);
4166	if (!nents)
4167	return -EIO;
4168	sgt->nents = nents;
4169	return 0;
4170	}
4171	return dma_map_sgtable(dev: dev->dma_device, sgt, dir: direction, attrs: dma_attrs);
4172	}
4173
4174	static inline void ib_dma_unmap_sgtable_attrs(struct ib_device *dev,
4175	struct sg_table *sgt,
4176	enum dma_data_direction direction,
4177	unsigned long dma_attrs)
4178	{
4179	if (!ib_uses_virt_dma(dev))
4180	dma_unmap_sgtable(dev: dev->dma_device, sgt, dir: direction, attrs: dma_attrs);
4181	}
4182
4183	/**
4184	* ib_dma_map_sg - Map a scatter/gather list to DMA addresses
4185	* @dev: The device for which the DMA addresses are to be created
4186	* @sg: The array of scatter/gather entries
4187	* @nents: The number of scatter/gather entries
4188	* @direction: The direction of the DMA
4189	*/
4190	static inline int ib_dma_map_sg(struct ib_device *dev,
4191	struct scatterlist *sg, int nents,
4192	enum dma_data_direction direction)
4193	{
4194	return ib_dma_map_sg_attrs(dev, sg, nents, direction, dma_attrs: 0);
4195	}
4196
4197	/**
4198	* ib_dma_unmap_sg - Unmap a scatter/gather list of DMA addresses
4199	* @dev: The device for which the DMA addresses were created
4200	* @sg: The array of scatter/gather entries
4201	* @nents: The number of scatter/gather entries
4202	* @direction: The direction of the DMA
4203	*/
4204	static inline void ib_dma_unmap_sg(struct ib_device *dev,
4205	struct scatterlist *sg, int nents,
4206	enum dma_data_direction direction)
4207	{
4208	ib_dma_unmap_sg_attrs(dev, sg, nents, direction, dma_attrs: 0);
4209	}
4210
4211	/**
4212	* ib_dma_max_seg_size - Return the size limit of a single DMA transfer
4213	* @dev: The device to query
4214	*
4215	* The returned value represents a size in bytes.
4216	*/
4217	static inline unsigned int ib_dma_max_seg_size(struct ib_device *dev)
4218	{
4219	if (ib_uses_virt_dma(dev))
4220	return UINT_MAX;
4221	return dma_get_max_seg_size(dev: dev->dma_device);
4222	}
4223
4224	/**
4225	* ib_dma_sync_single_for_cpu - Prepare DMA region to be accessed by CPU
4226	* @dev: The device for which the DMA address was created
4227	* @addr: The DMA address
4228	* @size: The size of the region in bytes
4229	* @dir: The direction of the DMA
4230	*/
4231	static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev,
4232	u64 addr,
4233	size_t size,
4234	enum dma_data_direction dir)
4235	{
4236	if (!ib_uses_virt_dma(dev))
4237	dma_sync_single_for_cpu(dev: dev->dma_device, addr, size, dir);
4238	}
4239
4240	/**
4241	* ib_dma_sync_single_for_device - Prepare DMA region to be accessed by device
4242	* @dev: The device for which the DMA address was created
4243	* @addr: The DMA address
4244	* @size: The size of the region in bytes
4245	* @dir: The direction of the DMA
4246	*/
4247	static inline void ib_dma_sync_single_for_device(struct ib_device *dev,
4248	u64 addr,
4249	size_t size,
4250	enum dma_data_direction dir)
4251	{
4252	if (!ib_uses_virt_dma(dev))
4253	dma_sync_single_for_device(dev: dev->dma_device, addr, size, dir);
4254	}
4255
4256	/* ib_reg_user_mr - register a memory region for virtual addresses from kernel
4257	* space. This function should be called when 'current' is the owning MM.
4258	*/
4259	struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
4260	u64 virt_addr, int mr_access_flags);
4261
4262	/* ib_advise_mr - give an advice about an address range in a memory region */
4263	int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
4264	u32 flags, struct ib_sge *sg_list, u32 num_sge);
4265	/**
4266	* ib_dereg_mr_user - Deregisters a memory region and removes it from the
4267	* HCA translation table.
4268	* @mr: The memory region to deregister.
4269	* @udata: Valid user data or NULL for kernel object
4270	*
4271	* This function can fail, if the memory region has memory windows bound to it.
4272	*/
4273	int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata);
4274
4275	/**
4276	* ib_dereg_mr - Deregisters a kernel memory region and removes it from the
4277	* HCA translation table.
4278	* @mr: The memory region to deregister.
4279	*
4280	* This function can fail, if the memory region has memory windows bound to it.
4281	*
4282	* NOTE: for user mr use ib_dereg_mr_user with valid udata!
4283	*/
4284	static inline int ib_dereg_mr(struct ib_mr *mr)
4285	{
4286	return ib_dereg_mr_user(mr, NULL);
4287	}
4288
4289	struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
4290	u32 max_num_sg);
4291
4292	struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
4293	u32 max_num_data_sg,
4294	u32 max_num_meta_sg);
4295
4296	/**
4297	* ib_update_fast_reg_key - updates the key portion of the fast_reg MR
4298	* R_Key and L_Key.
4299	* @mr - struct ib_mr pointer to be updated.
4300	* @newkey - new key to be used.
4301	*/
4302	static inline void ib_update_fast_reg_key(struct ib_mr *mr, u8 newkey)
4303	{
4304	mr->lkey = (mr->lkey & 0xffffff00) | newkey;
4305	mr->rkey = (mr->rkey & 0xffffff00) | newkey;
4306	}
4307
4308	/**
4309	* ib_inc_rkey - increments the key portion of the given rkey. Can be used
4310	* for calculating a new rkey for type 2 memory windows.
4311	* @rkey - the rkey to increment.
4312	*/
4313	static inline u32 ib_inc_rkey(u32 rkey)
4314	{
4315	const u32 mask = 0x000000ff;
4316	return ((rkey + 1) & mask) | (rkey & ~mask);
4317	}
4318
4319	/**
4320	* ib_attach_mcast - Attaches the specified QP to a multicast group.
4321	* @qp: QP to attach to the multicast group. The QP must be type
4322	* IB_QPT_UD.
4323	* @gid: Multicast group GID.
4324	* @lid: Multicast group LID in host byte order.
4325	*
4326	* In order to send and receive multicast packets, subnet
4327	* administration must have created the multicast group and configured
4328	* the fabric appropriately. The port associated with the specified
4329	* QP must also be a member of the multicast group.
4330	*/
4331	int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
4332
4333	/**
4334	* ib_detach_mcast - Detaches the specified QP from a multicast group.
4335	* @qp: QP to detach from the multicast group.
4336	* @gid: Multicast group GID.
4337	* @lid: Multicast group LID in host byte order.
4338	*/
4339	int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
4340
4341	struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,
4342	struct inode *inode, struct ib_udata *udata);
4343	int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata);
4344
4345	static inline int ib_check_mr_access(struct ib_device *ib_dev,
4346	unsigned int flags)
4347	{
4348	u64 device_cap = ib_dev->attrs.device_cap_flags;
4349
4350	/*
4351	* Local write permission is required if remote write or
4352	* remote atomic permission is also requested.
4353	*/
4354	if (flags & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE) &&
4355	!(flags & IB_ACCESS_LOCAL_WRITE))
4356	return -EINVAL;
4357
4358	if (flags & ~IB_ACCESS_SUPPORTED)
4359	return -EINVAL;
4360
4361	if (flags & IB_ACCESS_ON_DEMAND &&
4362	!(ib_dev->attrs.kernel_cap_flags & IBK_ON_DEMAND_PAGING))
4363	return -EOPNOTSUPP;
4364
4365	if ((flags & IB_ACCESS_FLUSH_GLOBAL &&
4366	!(device_cap & IB_DEVICE_FLUSH_GLOBAL)) ||
4367	(flags & IB_ACCESS_FLUSH_PERSISTENT &&
4368	!(device_cap & IB_DEVICE_FLUSH_PERSISTENT)))
4369	return -EOPNOTSUPP;
4370
4371	return 0;
4372	}
4373
4374	static inline bool ib_access_writable(int access_flags)
4375	{
4376	/*
4377	* We have writable memory backing the MR if any of the following
4378	* access flags are set. "Local write" and "remote write" obviously
4379	* require write access. "Remote atomic" can do things like fetch and
4380	* add, which will modify memory, and "MW bind" can change permissions
4381	* by binding a window.
4382	*/
4383	return access_flags &
4384	(IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE |
4385	IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_MW_BIND);
4386	}
4387
4388	/**
4389	* ib_check_mr_status: lightweight check of MR status.
4390	* This routine may provide status checks on a selected
4391	* ib_mr. first use is for signature status check.
4392	*
4393	* @mr: A memory region.
4394	* @check_mask: Bitmask of which checks to perform from
4395	* ib_mr_status_check enumeration.
4396	* @mr_status: The container of relevant status checks.
4397	* failed checks will be indicated in the status bitmask
4398	* and the relevant info shall be in the error item.
4399	*/
4400	int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
4401	struct ib_mr_status *mr_status);
4402
4403	/**
4404	* ib_device_try_get: Hold a registration lock
4405	* device: The device to lock
4406	*
4407	* A device under an active registration lock cannot become unregistered. It
4408	* is only possible to obtain a registration lock on a device that is fully
4409	* registered, otherwise this function returns false.
4410	*
4411	* The registration lock is only necessary for actions which require the
4412	* device to still be registered. Uses that only require the device pointer to
4413	* be valid should use get_device(&ibdev->dev) to hold the memory.
4414	*
4415	*/
4416	static inline bool ib_device_try_get(struct ib_device *dev)
4417	{
4418	return refcount_inc_not_zero(r: &dev->refcount);
4419	}
4420
4421	void ib_device_put(struct ib_device *device);
4422	struct ib_device *ib_device_get_by_netdev(struct net_device *ndev,
4423	enum rdma_driver_id driver_id);
4424	struct ib_device *ib_device_get_by_name(const char *name,
4425	enum rdma_driver_id driver_id);
4426	struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, u32 port,
4427	u16 pkey, const union ib_gid *gid,
4428	const struct sockaddr *addr);
4429	int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev,
4430	unsigned int port);
4431	struct ib_wq *ib_create_wq(struct ib_pd *pd,
4432	struct ib_wq_init_attr *init_attr);
4433	int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata);
4434
4435	int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
4436	unsigned int *sg_offset, unsigned int page_size);
4437	int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
4438	int data_sg_nents, unsigned int *data_sg_offset,
4439	struct scatterlist *meta_sg, int meta_sg_nents,
4440	unsigned int *meta_sg_offset, unsigned int page_size);
4441
4442	static inline int
4443	ib_map_mr_sg_zbva(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
4444	unsigned int *sg_offset, unsigned int page_size)
4445	{
4446	int n;
4447
4448	n = ib_map_mr_sg(mr, sg, sg_nents, sg_offset, page_size);
4449	mr->iova = 0;
4450
4451	return n;
4452	}
4453
4454	int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
4455	unsigned int *sg_offset, int (*set_page)(struct ib_mr *, u64));
4456
4457	void ib_drain_rq(struct ib_qp *qp);
4458	void ib_drain_sq(struct ib_qp *qp);
4459	void ib_drain_qp(struct ib_qp *qp);
4460
4461	int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed,
4462	u8 *width);
4463
4464	static inline u8 *rdma_ah_retrieve_dmac(struct rdma_ah_attr *attr)
4465	{
4466	if (attr->type == RDMA_AH_ATTR_TYPE_ROCE)
4467	return attr->roce.dmac;
4468	return NULL;
4469	}
4470
4471	static inline void rdma_ah_set_dlid(struct rdma_ah_attr *attr, u32 dlid)
4472	{
4473	if (attr->type == RDMA_AH_ATTR_TYPE_IB)
4474	attr->ib.dlid = (u16)dlid;
4475	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4476	attr->opa.dlid = dlid;
4477	}
4478
4479	static inline u32 rdma_ah_get_dlid(const struct rdma_ah_attr *attr)
4480	{
4481	if (attr->type == RDMA_AH_ATTR_TYPE_IB)
4482	return attr->ib.dlid;
4483	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4484	return attr->opa.dlid;
4485	return 0;
4486	}
4487
4488	static inline void rdma_ah_set_sl(struct rdma_ah_attr *attr, u8 sl)
4489	{
4490	attr->sl = sl;
4491	}
4492
4493	static inline u8 rdma_ah_get_sl(const struct rdma_ah_attr *attr)
4494	{
4495	return attr->sl;
4496	}
4497
4498	static inline void rdma_ah_set_path_bits(struct rdma_ah_attr *attr,
4499	u8 src_path_bits)
4500	{
4501	if (attr->type == RDMA_AH_ATTR_TYPE_IB)
4502	attr->ib.src_path_bits = src_path_bits;
4503	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4504	attr->opa.src_path_bits = src_path_bits;
4505	}
4506
4507	static inline u8 rdma_ah_get_path_bits(const struct rdma_ah_attr *attr)
4508	{
4509	if (attr->type == RDMA_AH_ATTR_TYPE_IB)
4510	return attr->ib.src_path_bits;
4511	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4512	return attr->opa.src_path_bits;
4513	return 0;
4514	}
4515
4516	static inline void rdma_ah_set_make_grd(struct rdma_ah_attr *attr,
4517	bool make_grd)
4518	{
4519	if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4520	attr->opa.make_grd = make_grd;
4521	}
4522
4523	static inline bool rdma_ah_get_make_grd(const struct rdma_ah_attr *attr)
4524	{
4525	if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4526	return attr->opa.make_grd;
4527	return false;
4528	}
4529
4530	static inline void rdma_ah_set_port_num(struct rdma_ah_attr *attr, u32 port_num)
4531	{
4532	attr->port_num = port_num;
4533	}
4534
4535	static inline u32 rdma_ah_get_port_num(const struct rdma_ah_attr *attr)
4536	{
4537	return attr->port_num;
4538	}
4539
4540	static inline void rdma_ah_set_static_rate(struct rdma_ah_attr *attr,
4541	u8 static_rate)
4542	{
4543	attr->static_rate = static_rate;
4544	}
4545
4546	static inline u8 rdma_ah_get_static_rate(const struct rdma_ah_attr *attr)
4547	{
4548	return attr->static_rate;
4549	}
4550
4551	static inline void rdma_ah_set_ah_flags(struct rdma_ah_attr *attr,
4552	enum ib_ah_flags flag)
4553	{
4554	attr->ah_flags = flag;
4555	}
4556
4557	static inline enum ib_ah_flags
4558	rdma_ah_get_ah_flags(const struct rdma_ah_attr *attr)
4559	{
4560	return attr->ah_flags;
4561	}
4562
4563	static inline const struct ib_global_route
4564	*rdma_ah_read_grh(const struct rdma_ah_attr *attr)
4565	{
4566	return &attr->grh;
4567	}
4568
4569	/*To retrieve and modify the grh */
4570	static inline struct ib_global_route
4571	*rdma_ah_retrieve_grh(struct rdma_ah_attr *attr)
4572	{
4573	return &attr->grh;
4574	}
4575
4576	static inline void rdma_ah_set_dgid_raw(struct rdma_ah_attr *attr, void *dgid)
4577	{
4578	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
4579
4580	memcpy(grh->dgid.raw, dgid, sizeof(grh->dgid));
4581	}
4582
4583	static inline void rdma_ah_set_subnet_prefix(struct rdma_ah_attr *attr,
4584	__be64 prefix)
4585	{
4586	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
4587
4588	grh->dgid.global.subnet_prefix = prefix;
4589	}
4590
4591	static inline void rdma_ah_set_interface_id(struct rdma_ah_attr *attr,
4592	__be64 if_id)
4593	{
4594	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
4595
4596	grh->dgid.global.interface_id = if_id;
4597	}
4598
4599	static inline void rdma_ah_set_grh(struct rdma_ah_attr *attr,
4600	union ib_gid *dgid, u32 flow_label,
4601	u8 sgid_index, u8 hop_limit,
4602	u8 traffic_class)
4603	{
4604	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
4605
4606	attr->ah_flags = IB_AH_GRH;
4607	if (dgid)
4608	grh->dgid = *dgid;
4609	grh->flow_label = flow_label;
4610	grh->sgid_index = sgid_index;
4611	grh->hop_limit = hop_limit;
4612	grh->traffic_class = traffic_class;
4613	grh->sgid_attr = NULL;
4614	}
4615
4616	void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr);
4617	void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid,
4618	u32 flow_label, u8 hop_limit, u8 traffic_class,
4619	const struct ib_gid_attr *sgid_attr);
4620	void rdma_copy_ah_attr(struct rdma_ah_attr *dest,
4621	const struct rdma_ah_attr *src);
4622	void rdma_replace_ah_attr(struct rdma_ah_attr *old,
4623	const struct rdma_ah_attr *new);
4624	void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src);
4625
4626	/**
4627	* rdma_ah_find_type - Return address handle type.
4628	*
4629	* @dev: Device to be checked
4630	* @port_num: Port number
4631	*/
4632	static inline enum rdma_ah_attr_type rdma_ah_find_type(struct ib_device *dev,
4633	u32 port_num)
4634	{
4635	if (rdma_protocol_roce(device: dev, port_num))
4636	return RDMA_AH_ATTR_TYPE_ROCE;
4637	if (rdma_protocol_ib(device: dev, port_num)) {
4638	if (rdma_cap_opa_ah(device: dev, port_num))
4639	return RDMA_AH_ATTR_TYPE_OPA;
4640	return RDMA_AH_ATTR_TYPE_IB;
4641	}
4642
4643	return RDMA_AH_ATTR_TYPE_UNDEFINED;
4644	}
4645
4646	/**
4647	* ib_lid_cpu16 - Return lid in 16bit CPU encoding.
4648	* In the current implementation the only way to
4649	* get the 32bit lid is from other sources for OPA.
4650	* For IB, lids will always be 16bits so cast the
4651	* value accordingly.
4652	*
4653	* @lid: A 32bit LID
4654	*/
4655	static inline u16 ib_lid_cpu16(u32 lid)
4656	{
4657	WARN_ON_ONCE(lid & 0xFFFF0000);
4658	return (u16)lid;
4659	}
4660
4661	/**
4662	* ib_lid_be16 - Return lid in 16bit BE encoding.
4663	*
4664	* @lid: A 32bit LID
4665	*/
4666	static inline __be16 ib_lid_be16(u32 lid)
4667	{
4668	WARN_ON_ONCE(lid & 0xFFFF0000);
4669	return cpu_to_be16((u16)lid);
4670	}
4671
4672	/**
4673	* ib_get_vector_affinity - Get the affinity mappings of a given completion
4674	* vector
4675	* @device: the rdma device
4676	* @comp_vector: index of completion vector
4677	*
4678	* Returns NULL on failure, otherwise a corresponding cpu map of the
4679	* completion vector (returns all-cpus map if the device driver doesn't
4680	* implement get_vector_affinity).
4681	*/
4682	static inline const struct cpumask *
4683	ib_get_vector_affinity(struct ib_device *device, int comp_vector)
4684	{
4685	if (comp_vector < 0 || comp_vector >= device->num_comp_vectors ||
4686	!device->ops.get_vector_affinity)
4687	return NULL;
4688
4689	return device->ops.get_vector_affinity(device, comp_vector);
4690
4691	}
4692
4693	/**
4694	* rdma_roce_rescan_device - Rescan all of the network devices in the system
4695	* and add their gids, as needed, to the relevant RoCE devices.
4696	*
4697	* @device: the rdma device
4698	*/
4699	void rdma_roce_rescan_device(struct ib_device *ibdev);
4700
4701	struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile);
4702
4703	int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs);
4704
4705	struct net_device *rdma_alloc_netdev(struct ib_device *device, u32 port_num,
4706	enum rdma_netdev_t type, const char *name,
4707	unsigned char name_assign_type,
4708	void (*setup)(struct net_device *));
4709
4710	int rdma_init_netdev(struct ib_device *device, u32 port_num,
4711	enum rdma_netdev_t type, const char *name,
4712	unsigned char name_assign_type,
4713	void (*setup)(struct net_device *),
4714	struct net_device *netdev);
4715
4716	/**
4717	* rdma_device_to_ibdev - Get ib_device pointer from device pointer
4718	*
4719	* @device: device pointer for which ib_device pointer to retrieve
4720	*
4721	* rdma_device_to_ibdev() retrieves ib_device pointer from device.
4722	*
4723	*/
4724	static inline struct ib_device *rdma_device_to_ibdev(struct device *device)
4725	{
4726	struct ib_core_device *coredev =
4727	container_of(device, struct ib_core_device, dev);
4728
4729	return coredev->owner;
4730	}
4731
4732	/**
4733	* ibdev_to_node - return the NUMA node for a given ib_device
4734	* @dev: device to get the NUMA node for.
4735	*/
4736	static inline int ibdev_to_node(struct ib_device *ibdev)
4737	{
4738	struct device *parent = ibdev->dev.parent;
4739
4740	if (!parent)
4741	return NUMA_NO_NODE;
4742	return dev_to_node(dev: parent);
4743	}
4744
4745	/**
4746	* rdma_device_to_drv_device - Helper macro to reach back to driver's
4747	* ib_device holder structure from device pointer.
4748	*
4749	* NOTE: New drivers should not make use of this API; This API is only for
4750	* existing drivers who have exposed sysfs entries using
4751	* ops->device_group.
4752	*/
4753	#define rdma_device_to_drv_device(dev, drv_dev_struct, ibdev_member) \
4754	container_of(rdma_device_to_ibdev(dev), drv_dev_struct, ibdev_member)
4755
4756	bool rdma_dev_access_netns(const struct ib_device *device,
4757	const struct net *net);
4758
4759	#define IB_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000)
4760	#define IB_ROCE_UDP_ENCAP_VALID_PORT_MAX (0xFFFF)
4761	#define IB_GRH_FLOWLABEL_MASK (0x000FFFFF)
4762
4763	/**
4764	* rdma_flow_label_to_udp_sport - generate a RoCE v2 UDP src port value based
4765	* on the flow_label
4766	*
4767	* This function will convert the 20 bit flow_label input to a valid RoCE v2
4768	* UDP src port 14 bit value. All RoCE V2 drivers should use this same
4769	* convention.
4770	*/
4771	static inline u16 rdma_flow_label_to_udp_sport(u32 fl)
4772	{
4773	u32 fl_low = fl & 0x03fff, fl_high = fl & 0xFC000;
4774
4775	fl_low ^= fl_high >> 14;
4776	return (u16)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN);
4777	}
4778
4779	/**
4780	* rdma_calc_flow_label - generate a RDMA symmetric flow label value based on
4781	* local and remote qpn values
4782	*
4783	* This function folded the multiplication results of two qpns, 24 bit each,
4784	* fields, and converts it to a 20 bit results.
4785	*
4786	* This function will create symmetric flow_label value based on the local
4787	* and remote qpn values. this will allow both the requester and responder
4788	* to calculate the same flow_label for a given connection.
4789	*
4790	* This helper function should be used by driver in case the upper layer
4791	* provide a zero flow_label value. This is to improve entropy of RDMA
4792	* traffic in the network.
4793	*/
4794	static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn)
4795	{
4796	u64 v = (u64)lqpn * rqpn;
4797
4798	v ^= v >> 20;
4799	v ^= v >> 40;
4800
4801	return (u32)(v & IB_GRH_FLOWLABEL_MASK);
4802	}
4803
4804	/**
4805	* rdma_get_udp_sport - Calculate and set UDP source port based on the flow
4806	* label. If flow label is not defined in GRH then
4807	* calculate it based on lqpn/rqpn.
4808	*
4809	* @fl: flow label from GRH
4810	* @lqpn: local qp number
4811	* @rqpn: remote qp number
4812	*/
4813	static inline u16 rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
4814	{
4815	if (!fl)
4816	fl = rdma_calc_flow_label(lqpn, rqpn);
4817
4818	return rdma_flow_label_to_udp_sport(fl);
4819	}
4820
4821	const struct ib_port_immutable*
4822	ib_port_immutable_read(struct ib_device *dev, unsigned int port);
4823	#endif /* IB_VERBS_H */
4824