1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2	/*
3	* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
4	* Copyright (c) 2004 Infinicon Corporation. All rights reserved.
5	* Copyright (c) 2004, 2020 Intel Corporation. All rights reserved.
6	* Copyright (c) 2004 Topspin Corporation. All rights reserved.
7	* Copyright (c) 2004 Voltaire Corporation. All rights reserved.
8	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
9	* Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
10	*/
11
12	#ifndef IB_VERBS_H
13	#define IB_VERBS_H
14
15	#include <linux/ethtool.h>
16	#include <linux/types.h>
17	#include <linux/device.h>
18	#include <linux/dma-mapping.h>
19	#include <linux/kref.h>
20	#include <linux/list.h>
21	#include <linux/rwsem.h>
22	#include <linux/workqueue.h>
23	#include <linux/irq_poll.h>
24	#include <uapi/linux/if_ether.h>
25	#include <net/ipv6.h>
26	#include <net/ip.h>
27	#include <linux/string.h>
28	#include <linux/slab.h>
29	#include <linux/netdevice.h>
30	#include <linux/refcount.h>
31	#include <linux/if_link.h>
32	#include <linux/atomic.h>
33	#include <linux/mmu_notifier.h>
34	#include <linux/uaccess.h>
35	#include <linux/cgroup_rdma.h>
36	#include <linux/irqflags.h>
37	#include <linux/preempt.h>
38	#include <linux/dim.h>
39	#include <uapi/rdma/ib_user_verbs.h>
40	#include <rdma/rdma_counter.h>
41	#include <rdma/restrack.h>
42	#include <rdma/signature.h>
43	#include <uapi/rdma/rdma_user_ioctl.h>
44	#include <uapi/rdma/ib_user_ioctl_verbs.h>
45
46	#define IB_FW_VERSION_NAME_MAX ETHTOOL_FWVERS_LEN
47
48	struct ib_umem_odp;
49	struct ib_uqp_object;
50	struct ib_usrq_object;
51	struct ib_uwq_object;
52	struct rdma_cm_id;
53	struct ib_port;
54	struct hw_stats_device_data;
55
56	extern struct workqueue_struct *ib_wq;
57	extern struct workqueue_struct *ib_comp_wq;
58	extern struct workqueue_struct *ib_comp_unbound_wq;
59
60	struct ib_ucq_object;
61
62	__printf(3, 4) __cold
63	void ibdev_printk(const char *level, const struct ib_device *ibdev,
64	const char *format, ...);
65	__printf(2, 3) __cold
66	void ibdev_emerg(const struct ib_device *ibdev, const char *format, ...);
67	__printf(2, 3) __cold
68	void ibdev_alert(const struct ib_device *ibdev, const char *format, ...);
69	__printf(2, 3) __cold
70	void ibdev_crit(const struct ib_device *ibdev, const char *format, ...);
71	__printf(2, 3) __cold
72	void ibdev_err(const struct ib_device *ibdev, const char *format, ...);
73	__printf(2, 3) __cold
74	void ibdev_warn(const struct ib_device *ibdev, const char *format, ...);
75	__printf(2, 3) __cold
76	void ibdev_notice(const struct ib_device *ibdev, const char *format, ...);
77	__printf(2, 3) __cold
78	void ibdev_info(const struct ib_device *ibdev, const char *format, ...);
79
80	#if defined(CONFIG_DYNAMIC_DEBUG) || \
81	(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
82	#define ibdev_dbg(__dev, format, args...) \
83	dynamic_ibdev_dbg(__dev, format, ##args)
84	#else
85	__printf(2, 3) __cold
86	static inline
87	void ibdev_dbg(const struct ib_device *ibdev, const char *format, ...) {}
88	#endif
89
90	#define ibdev_level_ratelimited(ibdev_level, ibdev, fmt, ...) \
91	do { \
92	static DEFINE_RATELIMIT_STATE(_rs, \
93	DEFAULT_RATELIMIT_INTERVAL, \
94	DEFAULT_RATELIMIT_BURST); \
95	if (__ratelimit(&_rs)) \
96	ibdev_level(ibdev, fmt, ##__VA_ARGS__); \
97	} while (0)
98
99	#define ibdev_emerg_ratelimited(ibdev, fmt, ...) \
100	ibdev_level_ratelimited(ibdev_emerg, ibdev, fmt, ##__VA_ARGS__)
101	#define ibdev_alert_ratelimited(ibdev, fmt, ...) \
102	ibdev_level_ratelimited(ibdev_alert, ibdev, fmt, ##__VA_ARGS__)
103	#define ibdev_crit_ratelimited(ibdev, fmt, ...) \
104	ibdev_level_ratelimited(ibdev_crit, ibdev, fmt, ##__VA_ARGS__)
105	#define ibdev_err_ratelimited(ibdev, fmt, ...) \
106	ibdev_level_ratelimited(ibdev_err, ibdev, fmt, ##__VA_ARGS__)
107	#define ibdev_warn_ratelimited(ibdev, fmt, ...) \
108	ibdev_level_ratelimited(ibdev_warn, ibdev, fmt, ##__VA_ARGS__)
109	#define ibdev_notice_ratelimited(ibdev, fmt, ...) \
110	ibdev_level_ratelimited(ibdev_notice, ibdev, fmt, ##__VA_ARGS__)
111	#define ibdev_info_ratelimited(ibdev, fmt, ...) \
112	ibdev_level_ratelimited(ibdev_info, ibdev, fmt, ##__VA_ARGS__)
113
114	#if defined(CONFIG_DYNAMIC_DEBUG) || \
115	(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
116	/* descriptor check is first to prevent flooding with "callbacks suppressed" */
117	#define ibdev_dbg_ratelimited(ibdev, fmt, ...) \
118	do { \
119	static DEFINE_RATELIMIT_STATE(_rs, \
120	DEFAULT_RATELIMIT_INTERVAL, \
121	DEFAULT_RATELIMIT_BURST); \
122	DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, fmt); \
123	if (DYNAMIC_DEBUG_BRANCH(descriptor) && __ratelimit(&_rs)) \
124	__dynamic_ibdev_dbg(&descriptor, ibdev, fmt, \
125	##__VA_ARGS__); \
126	} while (0)
127	#else
128	__printf(2, 3) __cold
129	static inline
130	void ibdev_dbg_ratelimited(const struct ib_device *ibdev, const char *format, ...) {}
131	#endif
132
133	union ib_gid {
134	u8 raw[16];
135	struct {
136	__be64 subnet_prefix;
137	__be64 interface_id;
138	} global;
139	};
140
141	extern union ib_gid zgid;
142
143	enum ib_gid_type {
144	IB_GID_TYPE_IB = IB_UVERBS_GID_TYPE_IB,
145	IB_GID_TYPE_ROCE = IB_UVERBS_GID_TYPE_ROCE_V1,
146	IB_GID_TYPE_ROCE_UDP_ENCAP = IB_UVERBS_GID_TYPE_ROCE_V2,
147	IB_GID_TYPE_SIZE
148	};
149
150	#define ROCE_V2_UDP_DPORT 4791
151	struct ib_gid_attr {
152	struct net_device __rcu *ndev;
153	struct ib_device *device;
154	union ib_gid gid;
155	enum ib_gid_type gid_type;
156	u16 index;
157	u32 port_num;
158	};
159
160	enum {
161	/* set the local administered indication */
162	IB_SA_WELL_KNOWN_GUID = BIT_ULL(57) | 2,
163	};
164
165	enum rdma_transport_type {
166	RDMA_TRANSPORT_IB,
167	RDMA_TRANSPORT_IWARP,
168	RDMA_TRANSPORT_USNIC,
169	RDMA_TRANSPORT_USNIC_UDP,
170	RDMA_TRANSPORT_UNSPECIFIED,
171	};
172
173	enum rdma_protocol_type {
174	RDMA_PROTOCOL_IB,
175	RDMA_PROTOCOL_IBOE,
176	RDMA_PROTOCOL_IWARP,
177	RDMA_PROTOCOL_USNIC_UDP
178	};
179
180	__attribute_const__ enum rdma_transport_type
181	rdma_node_get_transport(unsigned int node_type);
182
183	enum rdma_network_type {
184	RDMA_NETWORK_IB,
185	RDMA_NETWORK_ROCE_V1,
186	RDMA_NETWORK_IPV4,
187	RDMA_NETWORK_IPV6
188	};
189
190	static inline enum ib_gid_type ib_network_to_gid_type(enum rdma_network_type network_type)
191	{
192	if (network_type == RDMA_NETWORK_IPV4 ||
193	network_type == RDMA_NETWORK_IPV6)
194	return IB_GID_TYPE_ROCE_UDP_ENCAP;
195	else if (network_type == RDMA_NETWORK_ROCE_V1)
196	return IB_GID_TYPE_ROCE;
197	else
198	return IB_GID_TYPE_IB;
199	}
200
201	static inline enum rdma_network_type
202	rdma_gid_attr_network_type(const struct ib_gid_attr *attr)
203	{
204	if (attr->gid_type == IB_GID_TYPE_IB)
205	return RDMA_NETWORK_IB;
206
207	if (attr->gid_type == IB_GID_TYPE_ROCE)
208	return RDMA_NETWORK_ROCE_V1;
209
210	if (ipv6_addr_v4mapped(a: (struct in6_addr *)&attr->gid))
211	return RDMA_NETWORK_IPV4;
212	else
213	return RDMA_NETWORK_IPV6;
214	}
215
216	enum rdma_link_layer {
217	IB_LINK_LAYER_UNSPECIFIED,
218	IB_LINK_LAYER_INFINIBAND,
219	IB_LINK_LAYER_ETHERNET,
220	};
221
222	enum ib_device_cap_flags {
223	IB_DEVICE_RESIZE_MAX_WR = IB_UVERBS_DEVICE_RESIZE_MAX_WR,
224	IB_DEVICE_BAD_PKEY_CNTR = IB_UVERBS_DEVICE_BAD_PKEY_CNTR,
225	IB_DEVICE_BAD_QKEY_CNTR = IB_UVERBS_DEVICE_BAD_QKEY_CNTR,
226	IB_DEVICE_RAW_MULTI = IB_UVERBS_DEVICE_RAW_MULTI,
227	IB_DEVICE_AUTO_PATH_MIG = IB_UVERBS_DEVICE_AUTO_PATH_MIG,
228	IB_DEVICE_CHANGE_PHY_PORT = IB_UVERBS_DEVICE_CHANGE_PHY_PORT,
229	IB_DEVICE_UD_AV_PORT_ENFORCE = IB_UVERBS_DEVICE_UD_AV_PORT_ENFORCE,
230	IB_DEVICE_CURR_QP_STATE_MOD = IB_UVERBS_DEVICE_CURR_QP_STATE_MOD,
231	IB_DEVICE_SHUTDOWN_PORT = IB_UVERBS_DEVICE_SHUTDOWN_PORT,
232	/* IB_DEVICE_INIT_TYPE = IB_UVERBS_DEVICE_INIT_TYPE, (not in use) */
233	IB_DEVICE_PORT_ACTIVE_EVENT = IB_UVERBS_DEVICE_PORT_ACTIVE_EVENT,
234	IB_DEVICE_SYS_IMAGE_GUID = IB_UVERBS_DEVICE_SYS_IMAGE_GUID,
235	IB_DEVICE_RC_RNR_NAK_GEN = IB_UVERBS_DEVICE_RC_RNR_NAK_GEN,
236	IB_DEVICE_SRQ_RESIZE = IB_UVERBS_DEVICE_SRQ_RESIZE,
237	IB_DEVICE_N_NOTIFY_CQ = IB_UVERBS_DEVICE_N_NOTIFY_CQ,
238
239	/* Reserved, old SEND_W_INV = 1 << 16,*/
240	IB_DEVICE_MEM_WINDOW = IB_UVERBS_DEVICE_MEM_WINDOW,
241	/*
242	* Devices should set IB_DEVICE_UD_IP_SUM if they support
243	* insertion of UDP and TCP checksum on outgoing UD IPoIB
244	* messages and can verify the validity of checksum for
245	* incoming messages. Setting this flag implies that the
246	* IPoIB driver may set NETIF_F_IP_CSUM for datagram mode.
247	*/
248	IB_DEVICE_UD_IP_CSUM = IB_UVERBS_DEVICE_UD_IP_CSUM,
249	IB_DEVICE_XRC = IB_UVERBS_DEVICE_XRC,
250
251	/*
252	* This device supports the IB "base memory management extension",
253	* which includes support for fast registrations (IB_WR_REG_MR,
254	* IB_WR_LOCAL_INV and IB_WR_SEND_WITH_INV verbs). This flag should
255	* also be set by any iWarp device which must support FRs to comply
256	* to the iWarp verbs spec. iWarp devices also support the
257	* IB_WR_RDMA_READ_WITH_INV verb for RDMA READs that invalidate the
258	* stag.
259	*/
260	IB_DEVICE_MEM_MGT_EXTENSIONS = IB_UVERBS_DEVICE_MEM_MGT_EXTENSIONS,
261	IB_DEVICE_MEM_WINDOW_TYPE_2A = IB_UVERBS_DEVICE_MEM_WINDOW_TYPE_2A,
262	IB_DEVICE_MEM_WINDOW_TYPE_2B = IB_UVERBS_DEVICE_MEM_WINDOW_TYPE_2B,
263	IB_DEVICE_RC_IP_CSUM = IB_UVERBS_DEVICE_RC_IP_CSUM,
264	/* Deprecated. Please use IB_RAW_PACKET_CAP_IP_CSUM. */
265	IB_DEVICE_RAW_IP_CSUM = IB_UVERBS_DEVICE_RAW_IP_CSUM,
266	IB_DEVICE_MANAGED_FLOW_STEERING =
267	IB_UVERBS_DEVICE_MANAGED_FLOW_STEERING,
268	/* Deprecated. Please use IB_RAW_PACKET_CAP_SCATTER_FCS. */
269	IB_DEVICE_RAW_SCATTER_FCS = IB_UVERBS_DEVICE_RAW_SCATTER_FCS,
270	/* The device supports padding incoming writes to cacheline. */
271	IB_DEVICE_PCI_WRITE_END_PADDING =
272	IB_UVERBS_DEVICE_PCI_WRITE_END_PADDING,
273	/* Placement type attributes */
274	IB_DEVICE_FLUSH_GLOBAL = IB_UVERBS_DEVICE_FLUSH_GLOBAL,
275	IB_DEVICE_FLUSH_PERSISTENT = IB_UVERBS_DEVICE_FLUSH_PERSISTENT,
276	IB_DEVICE_ATOMIC_WRITE = IB_UVERBS_DEVICE_ATOMIC_WRITE,
277	};
278
279	enum ib_kernel_cap_flags {
280	/*
281	* This device supports a per-device lkey or stag that can be
282	* used without performing a memory registration for the local
283	* memory. Note that ULPs should never check this flag, but
284	* instead of use the local_dma_lkey flag in the ib_pd structure,
285	* which will always contain a usable lkey.
286	*/
287	IBK_LOCAL_DMA_LKEY = 1 << 0,
288	/* IB_QP_CREATE_INTEGRITY_EN is supported to implement T10-PI */
289	IBK_INTEGRITY_HANDOVER = 1 << 1,
290	/* IB_ACCESS_ON_DEMAND is supported during reg_user_mr() */
291	IBK_ON_DEMAND_PAGING = 1 << 2,
292	/* IB_MR_TYPE_SG_GAPS is supported */
293	IBK_SG_GAPS_REG = 1 << 3,
294	/* Driver supports RDMA_NLDEV_CMD_DELLINK */
295	IBK_ALLOW_USER_UNREG = 1 << 4,
296
297	/* ipoib will use IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK */
298	IBK_BLOCK_MULTICAST_LOOPBACK = 1 << 5,
299	/* iopib will use IB_QP_CREATE_IPOIB_UD_LSO for its QPs */
300	IBK_UD_TSO = 1 << 6,
301	/* iopib will use the device ops:
302	* get_vf_config
303	* get_vf_guid
304	* get_vf_stats
305	* set_vf_guid
306	* set_vf_link_state
307	*/
308	IBK_VIRTUAL_FUNCTION = 1 << 7,
309	/* ipoib will use IB_QP_CREATE_NETDEV_USE for its QPs */
310	IBK_RDMA_NETDEV_OPA = 1 << 8,
311	};
312
313	enum ib_atomic_cap {
314	IB_ATOMIC_NONE,
315	IB_ATOMIC_HCA,
316	IB_ATOMIC_GLOB
317	};
318
319	enum ib_odp_general_cap_bits {
320	IB_ODP_SUPPORT = 1 << 0,
321	IB_ODP_SUPPORT_IMPLICIT = 1 << 1,
322	};
323
324	enum ib_odp_transport_cap_bits {
325	IB_ODP_SUPPORT_SEND = 1 << 0,
326	IB_ODP_SUPPORT_RECV = 1 << 1,
327	IB_ODP_SUPPORT_WRITE = 1 << 2,
328	IB_ODP_SUPPORT_READ = 1 << 3,
329	IB_ODP_SUPPORT_ATOMIC = 1 << 4,
330	IB_ODP_SUPPORT_SRQ_RECV = 1 << 5,
331	};
332
333	struct ib_odp_caps {
334	uint64_t general_caps;
335	struct {
336	uint32_t rc_odp_caps;
337	uint32_t uc_odp_caps;
338	uint32_t ud_odp_caps;
339	uint32_t xrc_odp_caps;
340	} per_transport_caps;
341	};
342
343	struct ib_rss_caps {
344	/* Corresponding bit will be set if qp type from
345	* 'enum ib_qp_type' is supported, e.g.
346	* supported_qpts |= 1 << IB_QPT_UD
347	*/
348	u32 supported_qpts;
349	u32 max_rwq_indirection_tables;
350	u32 max_rwq_indirection_table_size;
351	};
352
353	enum ib_tm_cap_flags {
354	/* Support tag matching with rendezvous offload for RC transport */
355	IB_TM_CAP_RNDV_RC = 1 << 0,
356	};
357
358	struct ib_tm_caps {
359	/* Max size of RNDV header */
360	u32 max_rndv_hdr_size;
361	/* Max number of entries in tag matching list */
362	u32 max_num_tags;
363	/* From enum ib_tm_cap_flags */
364	u32 flags;
365	/* Max number of outstanding list operations */
366	u32 max_ops;
367	/* Max number of SGE in tag matching entry */
368	u32 max_sge;
369	};
370
371	struct ib_cq_init_attr {
372	unsigned int cqe;
373	u32 comp_vector;
374	u32 flags;
375	};
376
377	enum ib_cq_attr_mask {
378	IB_CQ_MODERATE = 1 << 0,
379	};
380
381	struct ib_cq_caps {
382	u16 max_cq_moderation_count;
383	u16 max_cq_moderation_period;
384	};
385
386	struct ib_dm_mr_attr {
387	u64 length;
388	u64 offset;
389	u32 access_flags;
390	};
391
392	struct ib_dm_alloc_attr {
393	u64 length;
394	u32 alignment;
395	u32 flags;
396	};
397
398	struct ib_device_attr {
399	u64 fw_ver;
400	__be64 sys_image_guid;
401	u64 max_mr_size;
402	u64 page_size_cap;
403	u32 vendor_id;
404	u32 vendor_part_id;
405	u32 hw_ver;
406	int max_qp;
407	int max_qp_wr;
408	u64 device_cap_flags;
409	u64 kernel_cap_flags;
410	int max_send_sge;
411	int max_recv_sge;
412	int max_sge_rd;
413	int max_cq;
414	int max_cqe;
415	int max_mr;
416	int max_pd;
417	int max_qp_rd_atom;
418	int max_ee_rd_atom;
419	int max_res_rd_atom;
420	int max_qp_init_rd_atom;
421	int max_ee_init_rd_atom;
422	enum ib_atomic_cap atomic_cap;
423	enum ib_atomic_cap masked_atomic_cap;
424	int max_ee;
425	int max_rdd;
426	int max_mw;
427	int max_raw_ipv6_qp;
428	int max_raw_ethy_qp;
429	int max_mcast_grp;
430	int max_mcast_qp_attach;
431	int max_total_mcast_qp_attach;
432	int max_ah;
433	int max_srq;
434	int max_srq_wr;
435	int max_srq_sge;
436	unsigned int max_fast_reg_page_list_len;
437	unsigned int max_pi_fast_reg_page_list_len;
438	u16 max_pkeys;
439	u8 local_ca_ack_delay;
440	int sig_prot_cap;
441	int sig_guard_cap;
442	struct ib_odp_caps odp_caps;
443	uint64_t timestamp_mask;
444	uint64_t hca_core_clock; /* in KHZ */
445	struct ib_rss_caps rss_caps;
446	u32 max_wq_type_rq;
447	u32 raw_packet_caps; /* Use ib_raw_packet_caps enum */
448	struct ib_tm_caps tm_caps;
449	struct ib_cq_caps cq_caps;
450	u64 max_dm_size;
451	/* Max entries for sgl for optimized performance per READ */
452	u32 max_sgl_rd;
453	};
454
455	enum ib_mtu {
456	IB_MTU_256 = 1,
457	IB_MTU_512 = 2,
458	IB_MTU_1024 = 3,
459	IB_MTU_2048 = 4,
460	IB_MTU_4096 = 5
461	};
462
463	enum opa_mtu {
464	OPA_MTU_8192 = 6,
465	OPA_MTU_10240 = 7
466	};
467
468	static inline int ib_mtu_enum_to_int(enum ib_mtu mtu)
469	{
470	switch (mtu) {
471	case IB_MTU_256: return 256;
472	case IB_MTU_512: return 512;
473	case IB_MTU_1024: return 1024;
474	case IB_MTU_2048: return 2048;
475	case IB_MTU_4096: return 4096;
476	default: return -1;
477	}
478	}
479
480	static inline enum ib_mtu ib_mtu_int_to_enum(int mtu)
481	{
482	if (mtu >= 4096)
483	return IB_MTU_4096;
484	else if (mtu >= 2048)
485	return IB_MTU_2048;
486	else if (mtu >= 1024)
487	return IB_MTU_1024;
488	else if (mtu >= 512)
489	return IB_MTU_512;
490	else
491	return IB_MTU_256;
492	}
493
494	static inline int opa_mtu_enum_to_int(enum opa_mtu mtu)
495	{
496	switch (mtu) {
497	case OPA_MTU_8192:
498	return 8192;
499	case OPA_MTU_10240:
500	return 10240;
501	default:
502	return(ib_mtu_enum_to_int(mtu: (enum ib_mtu)mtu));
503	}
504	}
505
506	static inline enum opa_mtu opa_mtu_int_to_enum(int mtu)
507	{
508	if (mtu >= 10240)
509	return OPA_MTU_10240;
510	else if (mtu >= 8192)
511	return OPA_MTU_8192;
512	else
513	return ((enum opa_mtu)ib_mtu_int_to_enum(mtu));
514	}
515
516	enum ib_port_state {
517	IB_PORT_NOP = 0,
518	IB_PORT_DOWN = 1,
519	IB_PORT_INIT = 2,
520	IB_PORT_ARMED = 3,
521	IB_PORT_ACTIVE = 4,
522	IB_PORT_ACTIVE_DEFER = 5
523	};
524
525	enum ib_port_phys_state {
526	IB_PORT_PHYS_STATE_SLEEP = 1,
527	IB_PORT_PHYS_STATE_POLLING = 2,
528	IB_PORT_PHYS_STATE_DISABLED = 3,
529	IB_PORT_PHYS_STATE_PORT_CONFIGURATION_TRAINING = 4,
530	IB_PORT_PHYS_STATE_LINK_UP = 5,
531	IB_PORT_PHYS_STATE_LINK_ERROR_RECOVERY = 6,
532	IB_PORT_PHYS_STATE_PHY_TEST = 7,
533	};
534
535	enum ib_port_width {
536	IB_WIDTH_1X = 1,
537	IB_WIDTH_2X = 16,
538	IB_WIDTH_4X = 2,
539	IB_WIDTH_8X = 4,
540	IB_WIDTH_12X = 8
541	};
542
543	static inline int ib_width_enum_to_int(enum ib_port_width width)
544	{
545	switch (width) {
546	case IB_WIDTH_1X: return 1;
547	case IB_WIDTH_2X: return 2;
548	case IB_WIDTH_4X: return 4;
549	case IB_WIDTH_8X: return 8;
550	case IB_WIDTH_12X: return 12;
551	default: return -1;
552	}
553	}
554
555	enum ib_port_speed {
556	IB_SPEED_SDR = 1,
557	IB_SPEED_DDR = 2,
558	IB_SPEED_QDR = 4,
559	IB_SPEED_FDR10 = 8,
560	IB_SPEED_FDR = 16,
561	IB_SPEED_EDR = 32,
562	IB_SPEED_HDR = 64,
563	IB_SPEED_NDR = 128,
564	IB_SPEED_XDR = 256,
565	};
566
567	enum ib_stat_flag {
568	IB_STAT_FLAG_OPTIONAL = 1 << 0,
569	};
570
571	/**
572	* struct rdma_stat_desc
573	* @name - The name of the counter
574	* @flags - Flags of the counter; For example, IB_STAT_FLAG_OPTIONAL
575	* @priv - Driver private information; Core code should not use
576	*/
577	struct rdma_stat_desc {
578	const char *name;
579	unsigned int flags;
580	const void *priv;
581	};
582
583	/**
584	* struct rdma_hw_stats
585	* @lock - Mutex to protect parallel write access to lifespan and values
586	* of counters, which are 64bits and not guaranteed to be written
587	* atomicaly on 32bits systems.
588	* @timestamp - Used by the core code to track when the last update was
589	* @lifespan - Used by the core code to determine how old the counters
590	* should be before being updated again. Stored in jiffies, defaults
591	* to 10 milliseconds, drivers can override the default be specifying
592	* their own value during their allocation routine.
593	* @descs - Array of pointers to static descriptors used for the counters
594	* in directory.
595	* @is_disabled - A bitmap to indicate each counter is currently disabled
596	* or not.
597	* @num_counters - How many hardware counters there are. If name is
598	* shorter than this number, a kernel oops will result. Driver authors
599	* are encouraged to leave BUILD_BUG_ON(ARRAY_SIZE(@name) < num_counters)
600	* in their code to prevent this.
601	* @value - Array of u64 counters that are accessed by the sysfs code and
602	* filled in by the drivers get_stats routine
603	*/
604	struct rdma_hw_stats {
605	struct mutex lock; /* Protect lifespan and values[] */
606	unsigned long timestamp;
607	unsigned long lifespan;
608	const struct rdma_stat_desc *descs;
609	unsigned long *is_disabled;
610	int num_counters;
611	u64 value[] __counted_by(num_counters);
612	};
613
614	#define RDMA_HW_STATS_DEFAULT_LIFESPAN 10
615
616	struct rdma_hw_stats *rdma_alloc_hw_stats_struct(
617	const struct rdma_stat_desc *descs, int num_counters,
618	unsigned long lifespan);
619
620	void rdma_free_hw_stats_struct(struct rdma_hw_stats *stats);
621
622	/* Define bits for the various functionality this port needs to be supported by
623	* the core.
624	*/
625	/* Management 0x00000FFF */
626	#define RDMA_CORE_CAP_IB_MAD 0x00000001
627	#define RDMA_CORE_CAP_IB_SMI 0x00000002
628	#define RDMA_CORE_CAP_IB_CM 0x00000004
629	#define RDMA_CORE_CAP_IW_CM 0x00000008
630	#define RDMA_CORE_CAP_IB_SA 0x00000010
631	#define RDMA_CORE_CAP_OPA_MAD 0x00000020
632
633	/* Address format 0x000FF000 */
634	#define RDMA_CORE_CAP_AF_IB 0x00001000
635	#define RDMA_CORE_CAP_ETH_AH 0x00002000
636	#define RDMA_CORE_CAP_OPA_AH 0x00004000
637	#define RDMA_CORE_CAP_IB_GRH_REQUIRED 0x00008000
638
639	/* Protocol 0xFFF00000 */
640	#define RDMA_CORE_CAP_PROT_IB 0x00100000
641	#define RDMA_CORE_CAP_PROT_ROCE 0x00200000
642	#define RDMA_CORE_CAP_PROT_IWARP 0x00400000
643	#define RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP 0x00800000
644	#define RDMA_CORE_CAP_PROT_RAW_PACKET 0x01000000
645	#define RDMA_CORE_CAP_PROT_USNIC 0x02000000
646
647	#define RDMA_CORE_PORT_IB_GRH_REQUIRED (RDMA_CORE_CAP_IB_GRH_REQUIRED \
648	| RDMA_CORE_CAP_PROT_ROCE \
649	| RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP)
650
651	#define RDMA_CORE_PORT_IBA_IB (RDMA_CORE_CAP_PROT_IB \
652	| RDMA_CORE_CAP_IB_MAD \
653	| RDMA_CORE_CAP_IB_SMI \
654	| RDMA_CORE_CAP_IB_CM \
655	| RDMA_CORE_CAP_IB_SA \
656	| RDMA_CORE_CAP_AF_IB)
657	#define RDMA_CORE_PORT_IBA_ROCE (RDMA_CORE_CAP_PROT_ROCE \
658	| RDMA_CORE_CAP_IB_MAD \
659	| RDMA_CORE_CAP_IB_CM \
660	| RDMA_CORE_CAP_AF_IB \
661	| RDMA_CORE_CAP_ETH_AH)
662	#define RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP \
663	(RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP \
664	| RDMA_CORE_CAP_IB_MAD \
665	| RDMA_CORE_CAP_IB_CM \
666	| RDMA_CORE_CAP_AF_IB \
667	| RDMA_CORE_CAP_ETH_AH)
668	#define RDMA_CORE_PORT_IWARP (RDMA_CORE_CAP_PROT_IWARP \
669	| RDMA_CORE_CAP_IW_CM)
670	#define RDMA_CORE_PORT_INTEL_OPA (RDMA_CORE_PORT_IBA_IB \
671	| RDMA_CORE_CAP_OPA_MAD)
672
673	#define RDMA_CORE_PORT_RAW_PACKET (RDMA_CORE_CAP_PROT_RAW_PACKET)
674
675	#define RDMA_CORE_PORT_USNIC (RDMA_CORE_CAP_PROT_USNIC)
676
677	struct ib_port_attr {
678	u64 subnet_prefix;
679	enum ib_port_state state;
680	enum ib_mtu max_mtu;
681	enum ib_mtu active_mtu;
682	u32 phys_mtu;
683	int gid_tbl_len;
684	unsigned int ip_gids:1;
685	/* This is the value from PortInfo CapabilityMask, defined by IBA */
686	u32 port_cap_flags;
687	u32 max_msg_sz;
688	u32 bad_pkey_cntr;
689	u32 qkey_viol_cntr;
690	u16 pkey_tbl_len;
691	u32 sm_lid;
692	u32 lid;
693	u8 lmc;
694	u8 max_vl_num;
695	u8 sm_sl;
696	u8 subnet_timeout;
697	u8 init_type_reply;
698	u8 active_width;
699	u16 active_speed;
700	u8 phys_state;
701	u16 port_cap_flags2;
702	};
703
704	enum ib_device_modify_flags {
705	IB_DEVICE_MODIFY_SYS_IMAGE_GUID = 1 << 0,
706	IB_DEVICE_MODIFY_NODE_DESC = 1 << 1
707	};
708
709	#define IB_DEVICE_NODE_DESC_MAX 64
710
711	struct ib_device_modify {
712	u64 sys_image_guid;
713	char node_desc[IB_DEVICE_NODE_DESC_MAX];
714	};
715
716	enum ib_port_modify_flags {
717	IB_PORT_SHUTDOWN = 1,
718	IB_PORT_INIT_TYPE = (1<<2),
719	IB_PORT_RESET_QKEY_CNTR = (1<<3),
720	IB_PORT_OPA_MASK_CHG = (1<<4)
721	};
722
723	struct ib_port_modify {
724	u32 set_port_cap_mask;
725	u32 clr_port_cap_mask;
726	u8 init_type;
727	};
728
729	enum ib_event_type {
730	IB_EVENT_CQ_ERR,
731	IB_EVENT_QP_FATAL,
732	IB_EVENT_QP_REQ_ERR,
733	IB_EVENT_QP_ACCESS_ERR,
734	IB_EVENT_COMM_EST,
735	IB_EVENT_SQ_DRAINED,
736	IB_EVENT_PATH_MIG,
737	IB_EVENT_PATH_MIG_ERR,
738	IB_EVENT_DEVICE_FATAL,
739	IB_EVENT_PORT_ACTIVE,
740	IB_EVENT_PORT_ERR,
741	IB_EVENT_LID_CHANGE,
742	IB_EVENT_PKEY_CHANGE,
743	IB_EVENT_SM_CHANGE,
744	IB_EVENT_SRQ_ERR,
745	IB_EVENT_SRQ_LIMIT_REACHED,
746	IB_EVENT_QP_LAST_WQE_REACHED,
747	IB_EVENT_CLIENT_REREGISTER,
748	IB_EVENT_GID_CHANGE,
749	IB_EVENT_WQ_FATAL,
750	};
751
752	const char *__attribute_const__ ib_event_msg(enum ib_event_type event);
753
754	struct ib_event {
755	struct ib_device *device;
756	union {
757	struct ib_cq *cq;
758	struct ib_qp *qp;
759	struct ib_srq *srq;
760	struct ib_wq *wq;
761	u32 port_num;
762	} element;
763	enum ib_event_type event;
764	};
765
766	struct ib_event_handler {
767	struct ib_device *device;
768	void (*handler)(struct ib_event_handler *, struct ib_event *);
769	struct list_head list;
770	};
771
772	#define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler) \
773	do { \
774	(_ptr)->device = _device; \
775	(_ptr)->handler = _handler; \
776	INIT_LIST_HEAD(&(_ptr)->list); \
777	} while (0)
778
779	struct ib_global_route {
780	const struct ib_gid_attr *sgid_attr;
781	union ib_gid dgid;
782	u32 flow_label;
783	u8 sgid_index;
784	u8 hop_limit;
785	u8 traffic_class;
786	};
787
788	struct ib_grh {
789	__be32 version_tclass_flow;
790	__be16 paylen;
791	u8 next_hdr;
792	u8 hop_limit;
793	union ib_gid sgid;
794	union ib_gid dgid;
795	};
796
797	union rdma_network_hdr {
798	struct ib_grh ibgrh;
799	struct {
800	/* The IB spec states that if it's IPv4, the header
801	* is located in the last 20 bytes of the header.
802	*/
803	u8 reserved[20];
804	struct iphdr roce4grh;
805	};
806	};
807
808	#define IB_QPN_MASK 0xFFFFFF
809
810	enum {
811	IB_MULTICAST_QPN = 0xffffff
812	};
813
814	#define IB_LID_PERMISSIVE cpu_to_be16(0xFFFF)
815	#define IB_MULTICAST_LID_BASE cpu_to_be16(0xC000)
816
817	enum ib_ah_flags {
818	IB_AH_GRH = 1
819	};
820
821	enum ib_rate {
822	IB_RATE_PORT_CURRENT = 0,
823	IB_RATE_2_5_GBPS = 2,
824	IB_RATE_5_GBPS = 5,
825	IB_RATE_10_GBPS = 3,
826	IB_RATE_20_GBPS = 6,
827	IB_RATE_30_GBPS = 4,
828	IB_RATE_40_GBPS = 7,
829	IB_RATE_60_GBPS = 8,
830	IB_RATE_80_GBPS = 9,
831	IB_RATE_120_GBPS = 10,
832	IB_RATE_14_GBPS = 11,
833	IB_RATE_56_GBPS = 12,
834	IB_RATE_112_GBPS = 13,
835	IB_RATE_168_GBPS = 14,
836	IB_RATE_25_GBPS = 15,
837	IB_RATE_100_GBPS = 16,
838	IB_RATE_200_GBPS = 17,
839	IB_RATE_300_GBPS = 18,
840	IB_RATE_28_GBPS = 19,
841	IB_RATE_50_GBPS = 20,
842	IB_RATE_400_GBPS = 21,
843	IB_RATE_600_GBPS = 22,
844	IB_RATE_800_GBPS = 23,
845	};
846
847	/**
848	* ib_rate_to_mult - Convert the IB rate enum to a multiple of the
849	* base rate of 2.5 Gbit/sec. For example, IB_RATE_5_GBPS will be
850	* converted to 2, since 5 Gbit/sec is 2 * 2.5 Gbit/sec.
851	* @rate: rate to convert.
852	*/
853	__attribute_const__ int ib_rate_to_mult(enum ib_rate rate);
854
855	/**
856	* ib_rate_to_mbps - Convert the IB rate enum to Mbps.
857	* For example, IB_RATE_2_5_GBPS will be converted to 2500.
858	* @rate: rate to convert.
859	*/
860	__attribute_const__ int ib_rate_to_mbps(enum ib_rate rate);
861
862
863	/**
864	* enum ib_mr_type - memory region type
865	* @IB_MR_TYPE_MEM_REG: memory region that is used for
866	* normal registration
867	* @IB_MR_TYPE_SG_GAPS: memory region that is capable to
868	* register any arbitrary sg lists (without
869	* the normal mr constraints - see
870	* ib_map_mr_sg)
871	* @IB_MR_TYPE_DM: memory region that is used for device
872	* memory registration
873	* @IB_MR_TYPE_USER: memory region that is used for the user-space
874	* application
875	* @IB_MR_TYPE_DMA: memory region that is used for DMA operations
876	* without address translations (VA=PA)
877	* @IB_MR_TYPE_INTEGRITY: memory region that is used for
878	* data integrity operations
879	*/
880	enum ib_mr_type {
881	IB_MR_TYPE_MEM_REG,
882	IB_MR_TYPE_SG_GAPS,
883	IB_MR_TYPE_DM,
884	IB_MR_TYPE_USER,
885	IB_MR_TYPE_DMA,
886	IB_MR_TYPE_INTEGRITY,
887	};
888
889	enum ib_mr_status_check {
890	IB_MR_CHECK_SIG_STATUS = 1,
891	};
892
893	/**
894	* struct ib_mr_status - Memory region status container
895	*
896	* @fail_status: Bitmask of MR checks status. For each
897	* failed check a corresponding status bit is set.
898	* @sig_err: Additional info for IB_MR_CEHCK_SIG_STATUS
899	* failure.
900	*/
901	struct ib_mr_status {
902	u32 fail_status;
903	struct ib_sig_err sig_err;
904	};
905
906	/**
907	* mult_to_ib_rate - Convert a multiple of 2.5 Gbit/sec to an IB rate
908	* enum.
909	* @mult: multiple to convert.
910	*/
911	__attribute_const__ enum ib_rate mult_to_ib_rate(int mult);
912
913	struct rdma_ah_init_attr {
914	struct rdma_ah_attr *ah_attr;
915	u32 flags;
916	struct net_device *xmit_slave;
917	};
918
919	enum rdma_ah_attr_type {
920	RDMA_AH_ATTR_TYPE_UNDEFINED,
921	RDMA_AH_ATTR_TYPE_IB,
922	RDMA_AH_ATTR_TYPE_ROCE,
923	RDMA_AH_ATTR_TYPE_OPA,
924	};
925
926	struct ib_ah_attr {
927	u16 dlid;
928	u8 src_path_bits;
929	};
930
931	struct roce_ah_attr {
932	u8 dmac[ETH_ALEN];
933	};
934
935	struct opa_ah_attr {
936	u32 dlid;
937	u8 src_path_bits;
938	bool make_grd;
939	};
940
941	struct rdma_ah_attr {
942	struct ib_global_route grh;
943	u8 sl;
944	u8 static_rate;
945	u32 port_num;
946	u8 ah_flags;
947	enum rdma_ah_attr_type type;
948	union {
949	struct ib_ah_attr ib;
950	struct roce_ah_attr roce;
951	struct opa_ah_attr opa;
952	};
953	};
954
955	enum ib_wc_status {
956	IB_WC_SUCCESS,
957	IB_WC_LOC_LEN_ERR,
958	IB_WC_LOC_QP_OP_ERR,
959	IB_WC_LOC_EEC_OP_ERR,
960	IB_WC_LOC_PROT_ERR,
961	IB_WC_WR_FLUSH_ERR,
962	IB_WC_MW_BIND_ERR,
963	IB_WC_BAD_RESP_ERR,
964	IB_WC_LOC_ACCESS_ERR,
965	IB_WC_REM_INV_REQ_ERR,
966	IB_WC_REM_ACCESS_ERR,
967	IB_WC_REM_OP_ERR,
968	IB_WC_RETRY_EXC_ERR,
969	IB_WC_RNR_RETRY_EXC_ERR,
970	IB_WC_LOC_RDD_VIOL_ERR,
971	IB_WC_REM_INV_RD_REQ_ERR,
972	IB_WC_REM_ABORT_ERR,
973	IB_WC_INV_EECN_ERR,
974	IB_WC_INV_EEC_STATE_ERR,
975	IB_WC_FATAL_ERR,
976	IB_WC_RESP_TIMEOUT_ERR,
977	IB_WC_GENERAL_ERR
978	};
979
980	const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status);
981
982	enum ib_wc_opcode {
983	IB_WC_SEND = IB_UVERBS_WC_SEND,
984	IB_WC_RDMA_WRITE = IB_UVERBS_WC_RDMA_WRITE,
985	IB_WC_RDMA_READ = IB_UVERBS_WC_RDMA_READ,
986	IB_WC_COMP_SWAP = IB_UVERBS_WC_COMP_SWAP,
987	IB_WC_FETCH_ADD = IB_UVERBS_WC_FETCH_ADD,
988	IB_WC_BIND_MW = IB_UVERBS_WC_BIND_MW,
989	IB_WC_LOCAL_INV = IB_UVERBS_WC_LOCAL_INV,
990	IB_WC_LSO = IB_UVERBS_WC_TSO,
991	IB_WC_ATOMIC_WRITE = IB_UVERBS_WC_ATOMIC_WRITE,
992	IB_WC_REG_MR,
993	IB_WC_MASKED_COMP_SWAP,
994	IB_WC_MASKED_FETCH_ADD,
995	IB_WC_FLUSH = IB_UVERBS_WC_FLUSH,
996	/*
997	* Set value of IB_WC_RECV so consumers can test if a completion is a
998	* receive by testing (opcode & IB_WC_RECV).
999	*/
1000	IB_WC_RECV = 1 << 7,
1001	IB_WC_RECV_RDMA_WITH_IMM
1002	};
1003
1004	enum ib_wc_flags {
1005	IB_WC_GRH = 1,
1006	IB_WC_WITH_IMM = (1<<1),
1007	IB_WC_WITH_INVALIDATE = (1<<2),
1008	IB_WC_IP_CSUM_OK = (1<<3),
1009	IB_WC_WITH_SMAC = (1<<4),
1010	IB_WC_WITH_VLAN = (1<<5),
1011	IB_WC_WITH_NETWORK_HDR_TYPE = (1<<6),
1012	};
1013
1014	struct ib_wc {
1015	union {
1016	u64 wr_id;
1017	struct ib_cqe *wr_cqe;
1018	};
1019	enum ib_wc_status status;
1020	enum ib_wc_opcode opcode;
1021	u32 vendor_err;
1022	u32 byte_len;
1023	struct ib_qp *qp;
1024	union {
1025	__be32 imm_data;
1026	u32 invalidate_rkey;
1027	} ex;
1028	u32 src_qp;
1029	u32 slid;
1030	int wc_flags;
1031	u16 pkey_index;
1032	u8 sl;
1033	u8 dlid_path_bits;
1034	u32 port_num; /* valid only for DR SMPs on switches */
1035	u8 smac[ETH_ALEN];
1036	u16 vlan_id;
1037	u8 network_hdr_type;
1038	};
1039
1040	enum ib_cq_notify_flags {
1041	IB_CQ_SOLICITED = 1 << 0,
1042	IB_CQ_NEXT_COMP = 1 << 1,
1043	IB_CQ_SOLICITED_MASK = IB_CQ_SOLICITED | IB_CQ_NEXT_COMP,
1044	IB_CQ_REPORT_MISSED_EVENTS = 1 << 2,
1045	};
1046
1047	enum ib_srq_type {
1048	IB_SRQT_BASIC = IB_UVERBS_SRQT_BASIC,
1049	IB_SRQT_XRC = IB_UVERBS_SRQT_XRC,
1050	IB_SRQT_TM = IB_UVERBS_SRQT_TM,
1051	};
1052
1053	static inline bool ib_srq_has_cq(enum ib_srq_type srq_type)
1054	{
1055	return srq_type == IB_SRQT_XRC ||
1056	srq_type == IB_SRQT_TM;
1057	}
1058
1059	enum ib_srq_attr_mask {
1060	IB_SRQ_MAX_WR = 1 << 0,
1061	IB_SRQ_LIMIT = 1 << 1,
1062	};
1063
1064	struct ib_srq_attr {
1065	u32 max_wr;
1066	u32 max_sge;
1067	u32 srq_limit;
1068	};
1069
1070	struct ib_srq_init_attr {
1071	void (*event_handler)(struct ib_event *, void *);
1072	void *srq_context;
1073	struct ib_srq_attr attr;
1074	enum ib_srq_type srq_type;
1075
1076	struct {
1077	struct ib_cq *cq;
1078	union {
1079	struct {
1080	struct ib_xrcd *xrcd;
1081	} xrc;
1082
1083	struct {
1084	u32 max_num_tags;
1085	} tag_matching;
1086	};
1087	} ext;
1088	};
1089
1090	struct ib_qp_cap {
1091	u32 max_send_wr;
1092	u32 max_recv_wr;
1093	u32 max_send_sge;
1094	u32 max_recv_sge;
1095	u32 max_inline_data;
1096
1097	/*
1098	* Maximum number of rdma_rw_ctx structures in flight at a time.
1099	* ib_create_qp() will calculate the right amount of needed WRs
1100	* and MRs based on this.
1101	*/
1102	u32 max_rdma_ctxs;
1103	};
1104
1105	enum ib_sig_type {
1106	IB_SIGNAL_ALL_WR,
1107	IB_SIGNAL_REQ_WR
1108	};
1109
1110	enum ib_qp_type {
1111	/*
1112	* IB_QPT_SMI and IB_QPT_GSI have to be the first two entries
1113	* here (and in that order) since the MAD layer uses them as
1114	* indices into a 2-entry table.
1115	*/
1116	IB_QPT_SMI,
1117	IB_QPT_GSI,
1118
1119	IB_QPT_RC = IB_UVERBS_QPT_RC,
1120	IB_QPT_UC = IB_UVERBS_QPT_UC,
1121	IB_QPT_UD = IB_UVERBS_QPT_UD,
1122	IB_QPT_RAW_IPV6,
1123	IB_QPT_RAW_ETHERTYPE,
1124	IB_QPT_RAW_PACKET = IB_UVERBS_QPT_RAW_PACKET,
1125	IB_QPT_XRC_INI = IB_UVERBS_QPT_XRC_INI,
1126	IB_QPT_XRC_TGT = IB_UVERBS_QPT_XRC_TGT,
1127	IB_QPT_MAX,
1128	IB_QPT_DRIVER = IB_UVERBS_QPT_DRIVER,
1129	/* Reserve a range for qp types internal to the low level driver.
1130	* These qp types will not be visible at the IB core layer, so the
1131	* IB_QPT_MAX usages should not be affected in the core layer
1132	*/
1133	IB_QPT_RESERVED1 = 0x1000,
1134	IB_QPT_RESERVED2,
1135	IB_QPT_RESERVED3,
1136	IB_QPT_RESERVED4,
1137	IB_QPT_RESERVED5,
1138	IB_QPT_RESERVED6,
1139	IB_QPT_RESERVED7,
1140	IB_QPT_RESERVED8,
1141	IB_QPT_RESERVED9,
1142	IB_QPT_RESERVED10,
1143	};
1144
1145	enum ib_qp_create_flags {
1146	IB_QP_CREATE_IPOIB_UD_LSO = 1 << 0,
1147	IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK =
1148	IB_UVERBS_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
1149	IB_QP_CREATE_CROSS_CHANNEL = 1 << 2,
1150	IB_QP_CREATE_MANAGED_SEND = 1 << 3,
1151	IB_QP_CREATE_MANAGED_RECV = 1 << 4,
1152	IB_QP_CREATE_NETIF_QP = 1 << 5,
1153	IB_QP_CREATE_INTEGRITY_EN = 1 << 6,
1154	IB_QP_CREATE_NETDEV_USE = 1 << 7,
1155	IB_QP_CREATE_SCATTER_FCS =
1156	IB_UVERBS_QP_CREATE_SCATTER_FCS,
1157	IB_QP_CREATE_CVLAN_STRIPPING =
1158	IB_UVERBS_QP_CREATE_CVLAN_STRIPPING,
1159	IB_QP_CREATE_SOURCE_QPN = 1 << 10,
1160	IB_QP_CREATE_PCI_WRITE_END_PADDING =
1161	IB_UVERBS_QP_CREATE_PCI_WRITE_END_PADDING,
1162	/* reserve bits 26-31 for low level drivers' internal use */
1163	IB_QP_CREATE_RESERVED_START = 1 << 26,
1164	IB_QP_CREATE_RESERVED_END = 1 << 31,
1165	};
1166
1167	/*
1168	* Note: users may not call ib_close_qp or ib_destroy_qp from the event_handler
1169	* callback to destroy the passed in QP.
1170	*/
1171
1172	struct ib_qp_init_attr {
1173	/* This callback occurs in workqueue context */
1174	void (*event_handler)(struct ib_event *, void *);
1175
1176	void *qp_context;
1177	struct ib_cq *send_cq;
1178	struct ib_cq *recv_cq;
1179	struct ib_srq *srq;
1180	struct ib_xrcd *xrcd; /* XRC TGT QPs only */
1181	struct ib_qp_cap cap;
1182	enum ib_sig_type sq_sig_type;
1183	enum ib_qp_type qp_type;
1184	u32 create_flags;
1185
1186	/*
1187	* Only needed for special QP types, or when using the RW API.
1188	*/
1189	u32 port_num;
1190	struct ib_rwq_ind_table *rwq_ind_tbl;
1191	u32 source_qpn;
1192	};
1193
1194	struct ib_qp_open_attr {
1195	void (*event_handler)(struct ib_event *, void *);
1196	void *qp_context;
1197	u32 qp_num;
1198	enum ib_qp_type qp_type;
1199	};
1200
1201	enum ib_rnr_timeout {
1202	IB_RNR_TIMER_655_36 = 0,
1203	IB_RNR_TIMER_000_01 = 1,
1204	IB_RNR_TIMER_000_02 = 2,
1205	IB_RNR_TIMER_000_03 = 3,
1206	IB_RNR_TIMER_000_04 = 4,
1207	IB_RNR_TIMER_000_06 = 5,
1208	IB_RNR_TIMER_000_08 = 6,
1209	IB_RNR_TIMER_000_12 = 7,
1210	IB_RNR_TIMER_000_16 = 8,
1211	IB_RNR_TIMER_000_24 = 9,
1212	IB_RNR_TIMER_000_32 = 10,
1213	IB_RNR_TIMER_000_48 = 11,
1214	IB_RNR_TIMER_000_64 = 12,
1215	IB_RNR_TIMER_000_96 = 13,
1216	IB_RNR_TIMER_001_28 = 14,
1217	IB_RNR_TIMER_001_92 = 15,
1218	IB_RNR_TIMER_002_56 = 16,
1219	IB_RNR_TIMER_003_84 = 17,
1220	IB_RNR_TIMER_005_12 = 18,
1221	IB_RNR_TIMER_007_68 = 19,
1222	IB_RNR_TIMER_010_24 = 20,
1223	IB_RNR_TIMER_015_36 = 21,
1224	IB_RNR_TIMER_020_48 = 22,
1225	IB_RNR_TIMER_030_72 = 23,
1226	IB_RNR_TIMER_040_96 = 24,
1227	IB_RNR_TIMER_061_44 = 25,
1228	IB_RNR_TIMER_081_92 = 26,
1229	IB_RNR_TIMER_122_88 = 27,
1230	IB_RNR_TIMER_163_84 = 28,
1231	IB_RNR_TIMER_245_76 = 29,
1232	IB_RNR_TIMER_327_68 = 30,
1233	IB_RNR_TIMER_491_52 = 31
1234	};
1235
1236	enum ib_qp_attr_mask {
1237	IB_QP_STATE = 1,
1238	IB_QP_CUR_STATE = (1<<1),
1239	IB_QP_EN_SQD_ASYNC_NOTIFY = (1<<2),
1240	IB_QP_ACCESS_FLAGS = (1<<3),
1241	IB_QP_PKEY_INDEX = (1<<4),
1242	IB_QP_PORT = (1<<5),
1243	IB_QP_QKEY = (1<<6),
1244	IB_QP_AV = (1<<7),
1245	IB_QP_PATH_MTU = (1<<8),
1246	IB_QP_TIMEOUT = (1<<9),
1247	IB_QP_RETRY_CNT = (1<<10),
1248	IB_QP_RNR_RETRY = (1<<11),
1249	IB_QP_RQ_PSN = (1<<12),
1250	IB_QP_MAX_QP_RD_ATOMIC = (1<<13),
1251	IB_QP_ALT_PATH = (1<<14),
1252	IB_QP_MIN_RNR_TIMER = (1<<15),
1253	IB_QP_SQ_PSN = (1<<16),
1254	IB_QP_MAX_DEST_RD_ATOMIC = (1<<17),
1255	IB_QP_PATH_MIG_STATE = (1<<18),
1256	IB_QP_CAP = (1<<19),
1257	IB_QP_DEST_QPN = (1<<20),
1258	IB_QP_RESERVED1 = (1<<21),
1259	IB_QP_RESERVED2 = (1<<22),
1260	IB_QP_RESERVED3 = (1<<23),
1261	IB_QP_RESERVED4 = (1<<24),
1262	IB_QP_RATE_LIMIT = (1<<25),
1263
1264	IB_QP_ATTR_STANDARD_BITS = GENMASK(20, 0),
1265	};
1266
1267	enum ib_qp_state {
1268	IB_QPS_RESET,
1269	IB_QPS_INIT,
1270	IB_QPS_RTR,
1271	IB_QPS_RTS,
1272	IB_QPS_SQD,
1273	IB_QPS_SQE,
1274	IB_QPS_ERR
1275	};
1276
1277	enum ib_mig_state {
1278	IB_MIG_MIGRATED,
1279	IB_MIG_REARM,
1280	IB_MIG_ARMED
1281	};
1282
1283	enum ib_mw_type {
1284	IB_MW_TYPE_1 = 1,
1285	IB_MW_TYPE_2 = 2
1286	};
1287
1288	struct ib_qp_attr {
1289	enum ib_qp_state qp_state;
1290	enum ib_qp_state cur_qp_state;
1291	enum ib_mtu path_mtu;
1292	enum ib_mig_state path_mig_state;
1293	u32 qkey;
1294	u32 rq_psn;
1295	u32 sq_psn;
1296	u32 dest_qp_num;
1297	int qp_access_flags;
1298	struct ib_qp_cap cap;
1299	struct rdma_ah_attr ah_attr;
1300	struct rdma_ah_attr alt_ah_attr;
1301	u16 pkey_index;
1302	u16 alt_pkey_index;
1303	u8 en_sqd_async_notify;
1304	u8 sq_draining;
1305	u8 max_rd_atomic;
1306	u8 max_dest_rd_atomic;
1307	u8 min_rnr_timer;
1308	u32 port_num;
1309	u8 timeout;
1310	u8 retry_cnt;
1311	u8 rnr_retry;
1312	u32 alt_port_num;
1313	u8 alt_timeout;
1314	u32 rate_limit;
1315	struct net_device *xmit_slave;
1316	};
1317
1318	enum ib_wr_opcode {
1319	/* These are shared with userspace */
1320	IB_WR_RDMA_WRITE = IB_UVERBS_WR_RDMA_WRITE,
1321	IB_WR_RDMA_WRITE_WITH_IMM = IB_UVERBS_WR_RDMA_WRITE_WITH_IMM,
1322	IB_WR_SEND = IB_UVERBS_WR_SEND,
1323	IB_WR_SEND_WITH_IMM = IB_UVERBS_WR_SEND_WITH_IMM,
1324	IB_WR_RDMA_READ = IB_UVERBS_WR_RDMA_READ,
1325	IB_WR_ATOMIC_CMP_AND_SWP = IB_UVERBS_WR_ATOMIC_CMP_AND_SWP,
1326	IB_WR_ATOMIC_FETCH_AND_ADD = IB_UVERBS_WR_ATOMIC_FETCH_AND_ADD,
1327	IB_WR_BIND_MW = IB_UVERBS_WR_BIND_MW,
1328	IB_WR_LSO = IB_UVERBS_WR_TSO,
1329	IB_WR_SEND_WITH_INV = IB_UVERBS_WR_SEND_WITH_INV,
1330	IB_WR_RDMA_READ_WITH_INV = IB_UVERBS_WR_RDMA_READ_WITH_INV,
1331	IB_WR_LOCAL_INV = IB_UVERBS_WR_LOCAL_INV,
1332	IB_WR_MASKED_ATOMIC_CMP_AND_SWP =
1333	IB_UVERBS_WR_MASKED_ATOMIC_CMP_AND_SWP,
1334	IB_WR_MASKED_ATOMIC_FETCH_AND_ADD =
1335	IB_UVERBS_WR_MASKED_ATOMIC_FETCH_AND_ADD,
1336	IB_WR_FLUSH = IB_UVERBS_WR_FLUSH,
1337	IB_WR_ATOMIC_WRITE = IB_UVERBS_WR_ATOMIC_WRITE,
1338
1339	/* These are kernel only and can not be issued by userspace */
1340	IB_WR_REG_MR = 0x20,
1341	IB_WR_REG_MR_INTEGRITY,
1342
1343	/* reserve values for low level drivers' internal use.
1344	* These values will not be used at all in the ib core layer.
1345	*/
1346	IB_WR_RESERVED1 = 0xf0,
1347	IB_WR_RESERVED2,
1348	IB_WR_RESERVED3,
1349	IB_WR_RESERVED4,
1350	IB_WR_RESERVED5,
1351	IB_WR_RESERVED6,
1352	IB_WR_RESERVED7,
1353	IB_WR_RESERVED8,
1354	IB_WR_RESERVED9,
1355	IB_WR_RESERVED10,
1356	};
1357
1358	enum ib_send_flags {
1359	IB_SEND_FENCE = 1,
1360	IB_SEND_SIGNALED = (1<<1),
1361	IB_SEND_SOLICITED = (1<<2),
1362	IB_SEND_INLINE = (1<<3),
1363	IB_SEND_IP_CSUM = (1<<4),
1364
1365	/* reserve bits 26-31 for low level drivers' internal use */
1366	IB_SEND_RESERVED_START = (1 << 26),
1367	IB_SEND_RESERVED_END = (1 << 31),
1368	};
1369
1370	struct ib_sge {
1371	u64 addr;
1372	u32 length;
1373	u32 lkey;
1374	};
1375
1376	struct ib_cqe {
1377	void (*done)(struct ib_cq *cq, struct ib_wc *wc);
1378	};
1379
1380	struct ib_send_wr {
1381	struct ib_send_wr *next;
1382	union {
1383	u64 wr_id;
1384	struct ib_cqe *wr_cqe;
1385	};
1386	struct ib_sge *sg_list;
1387	int num_sge;
1388	enum ib_wr_opcode opcode;
1389	int send_flags;
1390	union {
1391	__be32 imm_data;
1392	u32 invalidate_rkey;
1393	} ex;
1394	};
1395
1396	struct ib_rdma_wr {
1397	struct ib_send_wr wr;
1398	u64 remote_addr;
1399	u32 rkey;
1400	};
1401
1402	static inline const struct ib_rdma_wr *rdma_wr(const struct ib_send_wr *wr)
1403	{
1404	return container_of(wr, struct ib_rdma_wr, wr);
1405	}
1406
1407	struct ib_atomic_wr {
1408	struct ib_send_wr wr;
1409	u64 remote_addr;
1410	u64 compare_add;
1411	u64 swap;
1412	u64 compare_add_mask;
1413	u64 swap_mask;
1414	u32 rkey;
1415	};
1416
1417	static inline const struct ib_atomic_wr *atomic_wr(const struct ib_send_wr *wr)
1418	{
1419	return container_of(wr, struct ib_atomic_wr, wr);
1420	}
1421
1422	struct ib_ud_wr {
1423	struct ib_send_wr wr;
1424	struct ib_ah *ah;
1425	void *header;
1426	int hlen;
1427	int mss;
1428	u32 remote_qpn;
1429	u32 remote_qkey;
1430	u16 pkey_index; /* valid for GSI only */
1431	u32 port_num; /* valid for DR SMPs on switch only */
1432	};
1433
1434	static inline const struct ib_ud_wr *ud_wr(const struct ib_send_wr *wr)
1435	{
1436	return container_of(wr, struct ib_ud_wr, wr);
1437	}
1438
1439	struct ib_reg_wr {
1440	struct ib_send_wr wr;
1441	struct ib_mr *mr;
1442	u32 key;
1443	int access;
1444	};
1445
1446	static inline const struct ib_reg_wr *reg_wr(const struct ib_send_wr *wr)
1447	{
1448	return container_of(wr, struct ib_reg_wr, wr);
1449	}
1450
1451	struct ib_recv_wr {
1452	struct ib_recv_wr *next;
1453	union {
1454	u64 wr_id;
1455	struct ib_cqe *wr_cqe;
1456	};
1457	struct ib_sge *sg_list;
1458	int num_sge;
1459	};
1460
1461	enum ib_access_flags {
1462	IB_ACCESS_LOCAL_WRITE = IB_UVERBS_ACCESS_LOCAL_WRITE,
1463	IB_ACCESS_REMOTE_WRITE = IB_UVERBS_ACCESS_REMOTE_WRITE,
1464	IB_ACCESS_REMOTE_READ = IB_UVERBS_ACCESS_REMOTE_READ,
1465	IB_ACCESS_REMOTE_ATOMIC = IB_UVERBS_ACCESS_REMOTE_ATOMIC,
1466	IB_ACCESS_MW_BIND = IB_UVERBS_ACCESS_MW_BIND,
1467	IB_ZERO_BASED = IB_UVERBS_ACCESS_ZERO_BASED,
1468	IB_ACCESS_ON_DEMAND = IB_UVERBS_ACCESS_ON_DEMAND,
1469	IB_ACCESS_HUGETLB = IB_UVERBS_ACCESS_HUGETLB,
1470	IB_ACCESS_RELAXED_ORDERING = IB_UVERBS_ACCESS_RELAXED_ORDERING,
1471	IB_ACCESS_FLUSH_GLOBAL = IB_UVERBS_ACCESS_FLUSH_GLOBAL,
1472	IB_ACCESS_FLUSH_PERSISTENT = IB_UVERBS_ACCESS_FLUSH_PERSISTENT,
1473
1474	IB_ACCESS_OPTIONAL = IB_UVERBS_ACCESS_OPTIONAL_RANGE,
1475	IB_ACCESS_SUPPORTED =
1476	((IB_ACCESS_FLUSH_PERSISTENT << 1) - 1) | IB_ACCESS_OPTIONAL,
1477	};
1478
1479	/*
1480	* XXX: these are apparently used for ->rereg_user_mr, no idea why they
1481	* are hidden here instead of a uapi header!
1482	*/
1483	enum ib_mr_rereg_flags {
1484	IB_MR_REREG_TRANS = 1,
1485	IB_MR_REREG_PD = (1<<1),
1486	IB_MR_REREG_ACCESS = (1<<2),
1487	IB_MR_REREG_SUPPORTED = ((IB_MR_REREG_ACCESS << 1) - 1)
1488	};
1489
1490	struct ib_umem;
1491
1492	enum rdma_remove_reason {
1493	/*
1494	* Userspace requested uobject deletion or initial try
1495	* to remove uobject via cleanup. Call could fail
1496	*/
1497	RDMA_REMOVE_DESTROY,
1498	/* Context deletion. This call should delete the actual object itself */
1499	RDMA_REMOVE_CLOSE,
1500	/* Driver is being hot-unplugged. This call should delete the actual object itself */
1501	RDMA_REMOVE_DRIVER_REMOVE,
1502	/* uobj is being cleaned-up before being committed */
1503	RDMA_REMOVE_ABORT,
1504	/* The driver failed to destroy the uobject and is being disconnected */
1505	RDMA_REMOVE_DRIVER_FAILURE,
1506	};
1507
1508	struct ib_rdmacg_object {
1509	#ifdef CONFIG_CGROUP_RDMA
1510	struct rdma_cgroup *cg; /* owner rdma cgroup */
1511	#endif
1512	};
1513
1514	struct ib_ucontext {
1515	struct ib_device *device;
1516	struct ib_uverbs_file *ufile;
1517
1518	struct ib_rdmacg_object cg_obj;
1519	/*
1520	* Implementation details of the RDMA core, don't use in drivers:
1521	*/
1522	struct rdma_restrack_entry res;
1523	struct xarray mmap_xa;
1524	};
1525
1526	struct ib_uobject {
1527	u64 user_handle; /* handle given to us by userspace */
1528	/* ufile & ucontext owning this object */
1529	struct ib_uverbs_file *ufile;
1530	/* FIXME, save memory: ufile->context == context */
1531	struct ib_ucontext *context; /* associated user context */
1532	void *object; /* containing object */
1533	struct list_head list; /* link to context's list */
1534	struct ib_rdmacg_object cg_obj; /* rdmacg object */
1535	int id; /* index into kernel idr */
1536	struct kref ref;
1537	atomic_t usecnt; /* protects exclusive access */
1538	struct rcu_head rcu; /* kfree_rcu() overhead */
1539
1540	const struct uverbs_api_object *uapi_object;
1541	};
1542
1543	struct ib_udata {
1544	const void __user *inbuf;
1545	void __user *outbuf;
1546	size_t inlen;
1547	size_t outlen;
1548	};
1549
1550	struct ib_pd {
1551	u32 local_dma_lkey;
1552	u32 flags;
1553	struct ib_device *device;
1554	struct ib_uobject *uobject;
1555	atomic_t usecnt; /* count all resources */
1556
1557	u32 unsafe_global_rkey;
1558
1559	/*
1560	* Implementation details of the RDMA core, don't use in drivers:
1561	*/
1562	struct ib_mr *__internal_mr;
1563	struct rdma_restrack_entry res;
1564	};
1565
1566	struct ib_xrcd {
1567	struct ib_device *device;
1568	atomic_t usecnt; /* count all exposed resources */
1569	struct inode *inode;
1570	struct rw_semaphore tgt_qps_rwsem;
1571	struct xarray tgt_qps;
1572	};
1573
1574	struct ib_ah {
1575	struct ib_device *device;
1576	struct ib_pd *pd;
1577	struct ib_uobject *uobject;
1578	const struct ib_gid_attr *sgid_attr;
1579	enum rdma_ah_attr_type type;
1580	};
1581
1582	typedef void (*ib_comp_handler)(struct ib_cq *cq, void *cq_context);
1583
1584	enum ib_poll_context {
1585	IB_POLL_SOFTIRQ, /* poll from softirq context */
1586	IB_POLL_WORKQUEUE, /* poll from workqueue */
1587	IB_POLL_UNBOUND_WORKQUEUE, /* poll from unbound workqueue */
1588	IB_POLL_LAST_POOL_TYPE = IB_POLL_UNBOUND_WORKQUEUE,
1589
1590	IB_POLL_DIRECT, /* caller context, no hw completions */
1591	};
1592
1593	struct ib_cq {
1594	struct ib_device *device;
1595	struct ib_ucq_object *uobject;
1596	ib_comp_handler comp_handler;
1597	void (*event_handler)(struct ib_event *, void *);
1598	void *cq_context;
1599	int cqe;
1600	unsigned int cqe_used;
1601	atomic_t usecnt; /* count number of work queues */
1602	enum ib_poll_context poll_ctx;
1603	struct ib_wc *wc;
1604	struct list_head pool_entry;
1605	union {
1606	struct irq_poll iop;
1607	struct work_struct work;
1608	};
1609	struct workqueue_struct *comp_wq;
1610	struct dim *dim;
1611
1612	/* updated only by trace points */
1613	ktime_t timestamp;
1614	u8 interrupt:1;
1615	u8 shared:1;
1616	unsigned int comp_vector;
1617
1618	/*
1619	* Implementation details of the RDMA core, don't use in drivers:
1620	*/
1621	struct rdma_restrack_entry res;
1622	};
1623
1624	struct ib_srq {
1625	struct ib_device *device;
1626	struct ib_pd *pd;
1627	struct ib_usrq_object *uobject;
1628	void (*event_handler)(struct ib_event *, void *);
1629	void *srq_context;
1630	enum ib_srq_type srq_type;
1631	atomic_t usecnt;
1632
1633	struct {
1634	struct ib_cq *cq;
1635	union {
1636	struct {
1637	struct ib_xrcd *xrcd;
1638	u32 srq_num;
1639	} xrc;
1640	};
1641	} ext;
1642
1643	/*
1644	* Implementation details of the RDMA core, don't use in drivers:
1645	*/
1646	struct rdma_restrack_entry res;
1647	};
1648
1649	enum ib_raw_packet_caps {
1650	/*
1651	* Strip cvlan from incoming packet and report it in the matching work
1652	* completion is supported.
1653	*/
1654	IB_RAW_PACKET_CAP_CVLAN_STRIPPING =
1655	IB_UVERBS_RAW_PACKET_CAP_CVLAN_STRIPPING,
1656	/*
1657	* Scatter FCS field of an incoming packet to host memory is supported.
1658	*/
1659	IB_RAW_PACKET_CAP_SCATTER_FCS = IB_UVERBS_RAW_PACKET_CAP_SCATTER_FCS,
1660	/* Checksum offloads are supported (for both send and receive). */
1661	IB_RAW_PACKET_CAP_IP_CSUM = IB_UVERBS_RAW_PACKET_CAP_IP_CSUM,
1662	/*
1663	* When a packet is received for an RQ with no receive WQEs, the
1664	* packet processing is delayed.
1665	*/
1666	IB_RAW_PACKET_CAP_DELAY_DROP = IB_UVERBS_RAW_PACKET_CAP_DELAY_DROP,
1667	};
1668
1669	enum ib_wq_type {
1670	IB_WQT_RQ = IB_UVERBS_WQT_RQ,
1671	};
1672
1673	enum ib_wq_state {
1674	IB_WQS_RESET,
1675	IB_WQS_RDY,
1676	IB_WQS_ERR
1677	};
1678
1679	struct ib_wq {
1680	struct ib_device *device;
1681	struct ib_uwq_object *uobject;
1682	void *wq_context;
1683	void (*event_handler)(struct ib_event *, void *);
1684	struct ib_pd *pd;
1685	struct ib_cq *cq;
1686	u32 wq_num;
1687	enum ib_wq_state state;
1688	enum ib_wq_type wq_type;
1689	atomic_t usecnt;
1690	};
1691
1692	enum ib_wq_flags {
1693	IB_WQ_FLAGS_CVLAN_STRIPPING = IB_UVERBS_WQ_FLAGS_CVLAN_STRIPPING,
1694	IB_WQ_FLAGS_SCATTER_FCS = IB_UVERBS_WQ_FLAGS_SCATTER_FCS,
1695	IB_WQ_FLAGS_DELAY_DROP = IB_UVERBS_WQ_FLAGS_DELAY_DROP,
1696	IB_WQ_FLAGS_PCI_WRITE_END_PADDING =
1697	IB_UVERBS_WQ_FLAGS_PCI_WRITE_END_PADDING,
1698	};
1699
1700	struct ib_wq_init_attr {
1701	void *wq_context;
1702	enum ib_wq_type wq_type;
1703	u32 max_wr;
1704	u32 max_sge;
1705	struct ib_cq *cq;
1706	void (*event_handler)(struct ib_event *, void *);
1707	u32 create_flags; /* Use enum ib_wq_flags */
1708	};
1709
1710	enum ib_wq_attr_mask {
1711	IB_WQ_STATE = 1 << 0,
1712	IB_WQ_CUR_STATE = 1 << 1,
1713	IB_WQ_FLAGS = 1 << 2,
1714	};
1715
1716	struct ib_wq_attr {
1717	enum ib_wq_state wq_state;
1718	enum ib_wq_state curr_wq_state;
1719	u32 flags; /* Use enum ib_wq_flags */
1720	u32 flags_mask; /* Use enum ib_wq_flags */
1721	};
1722
1723	struct ib_rwq_ind_table {
1724	struct ib_device *device;
1725	struct ib_uobject *uobject;
1726	atomic_t usecnt;
1727	u32 ind_tbl_num;
1728	u32 log_ind_tbl_size;
1729	struct ib_wq **ind_tbl;
1730	};
1731
1732	struct ib_rwq_ind_table_init_attr {
1733	u32 log_ind_tbl_size;
1734	/* Each entry is a pointer to Receive Work Queue */
1735	struct ib_wq **ind_tbl;
1736	};
1737
1738	enum port_pkey_state {
1739	IB_PORT_PKEY_NOT_VALID = 0,
1740	IB_PORT_PKEY_VALID = 1,
1741	IB_PORT_PKEY_LISTED = 2,
1742	};
1743
1744	struct ib_qp_security;
1745
1746	struct ib_port_pkey {
1747	enum port_pkey_state state;
1748	u16 pkey_index;
1749	u32 port_num;
1750	struct list_head qp_list;
1751	struct list_head to_error_list;
1752	struct ib_qp_security *sec;
1753	};
1754
1755	struct ib_ports_pkeys {
1756	struct ib_port_pkey main;
1757	struct ib_port_pkey alt;
1758	};
1759
1760	struct ib_qp_security {
1761	struct ib_qp *qp;
1762	struct ib_device *dev;
1763	/* Hold this mutex when changing port and pkey settings. */
1764	struct mutex mutex;
1765	struct ib_ports_pkeys *ports_pkeys;
1766	/* A list of all open shared QP handles. Required to enforce security
1767	* properly for all users of a shared QP.
1768	*/
1769	struct list_head shared_qp_list;
1770	void *security;
1771	bool destroying;
1772	atomic_t error_list_count;
1773	struct completion error_complete;
1774	int error_comps_pending;
1775	};
1776
1777	/*
1778	* @max_write_sge: Maximum SGE elements per RDMA WRITE request.
1779	* @max_read_sge: Maximum SGE elements per RDMA READ request.
1780	*/
1781	struct ib_qp {
1782	struct ib_device *device;
1783	struct ib_pd *pd;
1784	struct ib_cq *send_cq;
1785	struct ib_cq *recv_cq;
1786	spinlock_t mr_lock;
1787	int mrs_used;
1788	struct list_head rdma_mrs;
1789	struct list_head sig_mrs;
1790	struct ib_srq *srq;
1791	struct ib_xrcd *xrcd; /* XRC TGT QPs only */
1792	struct list_head xrcd_list;
1793
1794	/* count times opened, mcast attaches, flow attaches */
1795	atomic_t usecnt;
1796	struct list_head open_list;
1797	struct ib_qp *real_qp;
1798	struct ib_uqp_object *uobject;
1799	void (*event_handler)(struct ib_event *, void *);
1800	void *qp_context;
1801	/* sgid_attrs associated with the AV's */
1802	const struct ib_gid_attr *av_sgid_attr;
1803	const struct ib_gid_attr *alt_path_sgid_attr;
1804	u32 qp_num;
1805	u32 max_write_sge;
1806	u32 max_read_sge;
1807	enum ib_qp_type qp_type;
1808	struct ib_rwq_ind_table *rwq_ind_tbl;
1809	struct ib_qp_security *qp_sec;
1810	u32 port;
1811
1812	bool integrity_en;
1813	/*
1814	* Implementation details of the RDMA core, don't use in drivers:
1815	*/
1816	struct rdma_restrack_entry res;
1817
1818	/* The counter the qp is bind to */
1819	struct rdma_counter *counter;
1820	};
1821
1822	struct ib_dm {
1823	struct ib_device *device;
1824	u32 length;
1825	u32 flags;
1826	struct ib_uobject *uobject;
1827	atomic_t usecnt;
1828	};
1829
1830	struct ib_mr {
1831	struct ib_device *device;
1832	struct ib_pd *pd;
1833	u32 lkey;
1834	u32 rkey;
1835	u64 iova;
1836	u64 length;
1837	unsigned int page_size;
1838	enum ib_mr_type type;
1839	bool need_inval;
1840	union {
1841	struct ib_uobject *uobject; /* user */
1842	struct list_head qp_entry; /* FR */
1843	};
1844
1845	struct ib_dm *dm;
1846	struct ib_sig_attrs *sig_attrs; /* only for IB_MR_TYPE_INTEGRITY MRs */
1847	/*
1848	* Implementation details of the RDMA core, don't use in drivers:
1849	*/
1850	struct rdma_restrack_entry res;
1851	};
1852
1853	struct ib_mw {
1854	struct ib_device *device;
1855	struct ib_pd *pd;
1856	struct ib_uobject *uobject;
1857	u32 rkey;
1858	enum ib_mw_type type;
1859	};
1860
1861	/* Supported steering options */
1862	enum ib_flow_attr_type {
1863	/* steering according to rule specifications */
1864	IB_FLOW_ATTR_NORMAL = 0x0,
1865	/* default unicast and multicast rule -
1866	* receive all Eth traffic which isn't steered to any QP
1867	*/
1868	IB_FLOW_ATTR_ALL_DEFAULT = 0x1,
1869	/* default multicast rule -
1870	* receive all Eth multicast traffic which isn't steered to any QP
1871	*/
1872	IB_FLOW_ATTR_MC_DEFAULT = 0x2,
1873	/* sniffer rule - receive all port traffic */
1874	IB_FLOW_ATTR_SNIFFER = 0x3
1875	};
1876
1877	/* Supported steering header types */
1878	enum ib_flow_spec_type {
1879	/* L2 headers*/
1880	IB_FLOW_SPEC_ETH = 0x20,
1881	IB_FLOW_SPEC_IB = 0x22,
1882	/* L3 header*/
1883	IB_FLOW_SPEC_IPV4 = 0x30,
1884	IB_FLOW_SPEC_IPV6 = 0x31,
1885	IB_FLOW_SPEC_ESP = 0x34,
1886	/* L4 headers*/
1887	IB_FLOW_SPEC_TCP = 0x40,
1888	IB_FLOW_SPEC_UDP = 0x41,
1889	IB_FLOW_SPEC_VXLAN_TUNNEL = 0x50,
1890	IB_FLOW_SPEC_GRE = 0x51,
1891	IB_FLOW_SPEC_MPLS = 0x60,
1892	IB_FLOW_SPEC_INNER = 0x100,
1893	/* Actions */
1894	IB_FLOW_SPEC_ACTION_TAG = 0x1000,
1895	IB_FLOW_SPEC_ACTION_DROP = 0x1001,
1896	IB_FLOW_SPEC_ACTION_HANDLE = 0x1002,
1897	IB_FLOW_SPEC_ACTION_COUNT = 0x1003,
1898	};
1899	#define IB_FLOW_SPEC_LAYER_MASK 0xF0
1900	#define IB_FLOW_SPEC_SUPPORT_LAYERS 10
1901
1902	enum ib_flow_flags {
1903	IB_FLOW_ATTR_FLAGS_DONT_TRAP = 1UL << 1, /* Continue match, no steal */
1904	IB_FLOW_ATTR_FLAGS_EGRESS = 1UL << 2, /* Egress flow */
1905	IB_FLOW_ATTR_FLAGS_RESERVED = 1UL << 3 /* Must be last */
1906	};
1907
1908	struct ib_flow_eth_filter {
1909	u8 dst_mac[6];
1910	u8 src_mac[6];
1911	__be16 ether_type;
1912	__be16 vlan_tag;
1913	/* Must be last */
1914	u8 real_sz[];
1915	};
1916
1917	struct ib_flow_spec_eth {
1918	u32 type;
1919	u16 size;
1920	struct ib_flow_eth_filter val;
1921	struct ib_flow_eth_filter mask;
1922	};
1923
1924	struct ib_flow_ib_filter {
1925	__be16 dlid;
1926	__u8 sl;
1927	/* Must be last */
1928	u8 real_sz[];
1929	};
1930
1931	struct ib_flow_spec_ib {
1932	u32 type;
1933	u16 size;
1934	struct ib_flow_ib_filter val;
1935	struct ib_flow_ib_filter mask;
1936	};
1937
1938	/* IPv4 header flags */
1939	enum ib_ipv4_flags {
1940	IB_IPV4_DONT_FRAG = 0x2, /* Don't enable packet fragmentation */
1941	IB_IPV4_MORE_FRAG = 0X4 /* For All fragmented packets except the
1942	last have this flag set */
1943	};
1944
1945	struct ib_flow_ipv4_filter {
1946	__be32 src_ip;
1947	__be32 dst_ip;
1948	u8 proto;
1949	u8 tos;
1950	u8 ttl;
1951	u8 flags;
1952	/* Must be last */
1953	u8 real_sz[];
1954	};
1955
1956	struct ib_flow_spec_ipv4 {
1957	u32 type;
1958	u16 size;
1959	struct ib_flow_ipv4_filter val;
1960	struct ib_flow_ipv4_filter mask;
1961	};
1962
1963	struct ib_flow_ipv6_filter {
1964	u8 src_ip[16];
1965	u8 dst_ip[16];
1966	__be32 flow_label;
1967	u8 next_hdr;
1968	u8 traffic_class;
1969	u8 hop_limit;
1970	/* Must be last */
1971	u8 real_sz[];
1972	};
1973
1974	struct ib_flow_spec_ipv6 {
1975	u32 type;
1976	u16 size;
1977	struct ib_flow_ipv6_filter val;
1978	struct ib_flow_ipv6_filter mask;
1979	};
1980
1981	struct ib_flow_tcp_udp_filter {
1982	__be16 dst_port;
1983	__be16 src_port;
1984	/* Must be last */
1985	u8 real_sz[];
1986	};
1987
1988	struct ib_flow_spec_tcp_udp {
1989	u32 type;
1990	u16 size;
1991	struct ib_flow_tcp_udp_filter val;
1992	struct ib_flow_tcp_udp_filter mask;
1993	};
1994
1995	struct ib_flow_tunnel_filter {
1996	__be32 tunnel_id;
1997	u8 real_sz[];
1998	};
1999
2000	/* ib_flow_spec_tunnel describes the Vxlan tunnel
2001	* the tunnel_id from val has the vni value
2002	*/
2003	struct ib_flow_spec_tunnel {
2004	u32 type;
2005	u16 size;
2006	struct ib_flow_tunnel_filter val;
2007	struct ib_flow_tunnel_filter mask;
2008	};
2009
2010	struct ib_flow_esp_filter {
2011	__be32 spi;
2012	__be32 seq;
2013	/* Must be last */
2014	u8 real_sz[];
2015	};
2016
2017	struct ib_flow_spec_esp {
2018	u32 type;
2019	u16 size;
2020	struct ib_flow_esp_filter val;
2021	struct ib_flow_esp_filter mask;
2022	};
2023
2024	struct ib_flow_gre_filter {
2025	__be16 c_ks_res0_ver;
2026	__be16 protocol;
2027	__be32 key;
2028	/* Must be last */
2029	u8 real_sz[];
2030	};
2031
2032	struct ib_flow_spec_gre {
2033	u32 type;
2034	u16 size;
2035	struct ib_flow_gre_filter val;
2036	struct ib_flow_gre_filter mask;
2037	};
2038
2039	struct ib_flow_mpls_filter {
2040	__be32 tag;
2041	/* Must be last */
2042	u8 real_sz[];
2043	};
2044
2045	struct ib_flow_spec_mpls {
2046	u32 type;
2047	u16 size;
2048	struct ib_flow_mpls_filter val;
2049	struct ib_flow_mpls_filter mask;
2050	};
2051
2052	struct ib_flow_spec_action_tag {
2053	enum ib_flow_spec_type type;
2054	u16 size;
2055	u32 tag_id;
2056	};
2057
2058	struct ib_flow_spec_action_drop {
2059	enum ib_flow_spec_type type;
2060	u16 size;
2061	};
2062
2063	struct ib_flow_spec_action_handle {
2064	enum ib_flow_spec_type type;
2065	u16 size;
2066	struct ib_flow_action *act;
2067	};
2068
2069	enum ib_counters_description {
2070	IB_COUNTER_PACKETS,
2071	IB_COUNTER_BYTES,
2072	};
2073
2074	struct ib_flow_spec_action_count {
2075	enum ib_flow_spec_type type;
2076	u16 size;
2077	struct ib_counters *counters;
2078	};
2079
2080	union ib_flow_spec {
2081	struct {
2082	u32 type;
2083	u16 size;
2084	};
2085	struct ib_flow_spec_eth eth;
2086	struct ib_flow_spec_ib ib;
2087	struct ib_flow_spec_ipv4 ipv4;
2088	struct ib_flow_spec_tcp_udp tcp_udp;
2089	struct ib_flow_spec_ipv6 ipv6;
2090	struct ib_flow_spec_tunnel tunnel;
2091	struct ib_flow_spec_esp esp;
2092	struct ib_flow_spec_gre gre;
2093	struct ib_flow_spec_mpls mpls;
2094	struct ib_flow_spec_action_tag flow_tag;
2095	struct ib_flow_spec_action_drop drop;
2096	struct ib_flow_spec_action_handle action;
2097	struct ib_flow_spec_action_count flow_count;
2098	};
2099
2100	struct ib_flow_attr {
2101	enum ib_flow_attr_type type;
2102	u16 size;
2103	u16 priority;
2104	u32 flags;
2105	u8 num_of_specs;
2106	u32 port;
2107	union ib_flow_spec flows[];
2108	};
2109
2110	struct ib_flow {
2111	struct ib_qp *qp;
2112	struct ib_device *device;
2113	struct ib_uobject *uobject;
2114	};
2115
2116	enum ib_flow_action_type {
2117	IB_FLOW_ACTION_UNSPECIFIED,
2118	IB_FLOW_ACTION_ESP = 1,
2119	};
2120
2121	struct ib_flow_action_attrs_esp_keymats {
2122	enum ib_uverbs_flow_action_esp_keymat protocol;
2123	union {
2124	struct ib_uverbs_flow_action_esp_keymat_aes_gcm aes_gcm;
2125	} keymat;
2126	};
2127
2128	struct ib_flow_action_attrs_esp_replays {
2129	enum ib_uverbs_flow_action_esp_replay protocol;
2130	union {
2131	struct ib_uverbs_flow_action_esp_replay_bmp bmp;
2132	} replay;
2133	};
2134
2135	enum ib_flow_action_attrs_esp_flags {
2136	/* All user-space flags at the top: Use enum ib_uverbs_flow_action_esp_flags
2137	* This is done in order to share the same flags between user-space and
2138	* kernel and spare an unnecessary translation.
2139	*/
2140
2141	/* Kernel flags */
2142	IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED = 1ULL << 32,
2143	IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS = 1ULL << 33,
2144	};
2145
2146	struct ib_flow_spec_list {
2147	struct ib_flow_spec_list *next;
2148	union ib_flow_spec spec;
2149	};
2150
2151	struct ib_flow_action_attrs_esp {
2152	struct ib_flow_action_attrs_esp_keymats *keymat;
2153	struct ib_flow_action_attrs_esp_replays *replay;
2154	struct ib_flow_spec_list *encap;
2155	/* Used only if IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED is enabled.
2156	* Value of 0 is a valid value.
2157	*/
2158	u32 esn;
2159	u32 spi;
2160	u32 seq;
2161	u32 tfc_pad;
2162	/* Use enum ib_flow_action_attrs_esp_flags */
2163	u64 flags;
2164	u64 hard_limit_pkts;
2165	};
2166
2167	struct ib_flow_action {
2168	struct ib_device *device;
2169	struct ib_uobject *uobject;
2170	enum ib_flow_action_type type;
2171	atomic_t usecnt;
2172	};
2173
2174	struct ib_mad;
2175
2176	enum ib_process_mad_flags {
2177	IB_MAD_IGNORE_MKEY = 1,
2178	IB_MAD_IGNORE_BKEY = 2,
2179	IB_MAD_IGNORE_ALL = IB_MAD_IGNORE_MKEY | IB_MAD_IGNORE_BKEY
2180	};
2181
2182	enum ib_mad_result {
2183	IB_MAD_RESULT_FAILURE = 0, /* (!SUCCESS is the important flag) */
2184	IB_MAD_RESULT_SUCCESS = 1 << 0, /* MAD was successfully processed */
2185	IB_MAD_RESULT_REPLY = 1 << 1, /* Reply packet needs to be sent */
2186	IB_MAD_RESULT_CONSUMED = 1 << 2 /* Packet consumed: stop processing */
2187	};
2188
2189	struct ib_port_cache {
2190	u64 subnet_prefix;
2191	struct ib_pkey_cache *pkey;
2192	struct ib_gid_table *gid;
2193	u8 lmc;
2194	enum ib_port_state port_state;
2195	};
2196
2197	struct ib_port_immutable {
2198	int pkey_tbl_len;
2199	int gid_tbl_len;
2200	u32 core_cap_flags;
2201	u32 max_mad_size;
2202	};
2203
2204	struct ib_port_data {
2205	struct ib_device *ib_dev;
2206
2207	struct ib_port_immutable immutable;
2208
2209	spinlock_t pkey_list_lock;
2210
2211	spinlock_t netdev_lock;
2212
2213	struct list_head pkey_list;
2214
2215	struct ib_port_cache cache;
2216
2217	struct net_device __rcu *netdev;
2218	netdevice_tracker netdev_tracker;
2219	struct hlist_node ndev_hash_link;
2220	struct rdma_port_counter port_counter;
2221	struct ib_port *sysfs;
2222	};
2223
2224	/* rdma netdev type - specifies protocol type */
2225	enum rdma_netdev_t {
2226	RDMA_NETDEV_OPA_VNIC,
2227	RDMA_NETDEV_IPOIB,
2228	};
2229
2230	/**
2231	* struct rdma_netdev - rdma netdev
2232	* For cases where netstack interfacing is required.
2233	*/
2234	struct rdma_netdev {
2235	void *clnt_priv;
2236	struct ib_device *hca;
2237	u32 port_num;
2238	int mtu;
2239
2240	/*
2241	* cleanup function must be specified.
2242	* FIXME: This is only used for OPA_VNIC and that usage should be
2243	* removed too.
2244	*/
2245	void (*free_rdma_netdev)(struct net_device *netdev);
2246
2247	/* control functions */
2248	void (*set_id)(struct net_device *netdev, int id);
2249	/* send packet */
2250	int (*send)(struct net_device *dev, struct sk_buff *skb,
2251	struct ib_ah *address, u32 dqpn);
2252	/* multicast */
2253	int (*attach_mcast)(struct net_device *dev, struct ib_device *hca,
2254	union ib_gid *gid, u16 mlid,
2255	int set_qkey, u32 qkey);
2256	int (*detach_mcast)(struct net_device *dev, struct ib_device *hca,
2257	union ib_gid *gid, u16 mlid);
2258	/* timeout */
2259	void (*tx_timeout)(struct net_device *dev, unsigned int txqueue);
2260	};
2261
2262	struct rdma_netdev_alloc_params {
2263	size_t sizeof_priv;
2264	unsigned int txqs;
2265	unsigned int rxqs;
2266	void *param;
2267
2268	int (*initialize_rdma_netdev)(struct ib_device *device, u32 port_num,
2269	struct net_device *netdev, void *param);
2270	};
2271
2272	struct ib_odp_counters {
2273	atomic64_t faults;
2274	atomic64_t invalidations;
2275	atomic64_t prefetch;
2276	};
2277
2278	struct ib_counters {
2279	struct ib_device *device;
2280	struct ib_uobject *uobject;
2281	/* num of objects attached */
2282	atomic_t usecnt;
2283	};
2284
2285	struct ib_counters_read_attr {
2286	u64 *counters_buff;
2287	u32 ncounters;
2288	u32 flags; /* use enum ib_read_counters_flags */
2289	};
2290
2291	struct uverbs_attr_bundle;
2292	struct iw_cm_id;
2293	struct iw_cm_conn_param;
2294
2295	#define INIT_RDMA_OBJ_SIZE(ib_struct, drv_struct, member) \
2296	.size_##ib_struct = \
2297	(sizeof(struct drv_struct) + \
2298	BUILD_BUG_ON_ZERO(offsetof(struct drv_struct, member)) + \
2299	BUILD_BUG_ON_ZERO( \
2300	!__same_type(((struct drv_struct *)NULL)->member, \
2301	struct ib_struct)))
2302
2303	#define rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, gfp) \
2304	((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \
2305	gfp, false))
2306
2307	#define rdma_zalloc_drv_obj_numa(ib_dev, ib_type) \
2308	((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \
2309	GFP_KERNEL, true))
2310
2311	#define rdma_zalloc_drv_obj(ib_dev, ib_type) \
2312	rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, GFP_KERNEL)
2313
2314	#define DECLARE_RDMA_OBJ_SIZE(ib_struct) size_t size_##ib_struct
2315
2316	struct rdma_user_mmap_entry {
2317	struct kref ref;
2318	struct ib_ucontext *ucontext;
2319	unsigned long start_pgoff;
2320	size_t npages;
2321	bool driver_removed;
2322	};
2323
2324	/* Return the offset (in bytes) the user should pass to libc's mmap() */
2325	static inline u64
2326	rdma_user_mmap_get_offset(const struct rdma_user_mmap_entry *entry)
2327	{
2328	return (u64)entry->start_pgoff << PAGE_SHIFT;
2329	}
2330
2331	/**
2332	* struct ib_device_ops - InfiniBand device operations
2333	* This structure defines all the InfiniBand device operations, providers will
2334	* need to define the supported operations, otherwise they will be set to null.
2335	*/
2336	struct ib_device_ops {
2337	struct module *owner;
2338	enum rdma_driver_id driver_id;
2339	u32 uverbs_abi_ver;
2340	unsigned int uverbs_no_driver_id_binding:1;
2341
2342	/*
2343	* NOTE: New drivers should not make use of device_group; instead new
2344	* device parameter should be exposed via netlink command. This
2345	* mechanism exists only for existing drivers.
2346	*/
2347	const struct attribute_group *device_group;
2348	const struct attribute_group **port_groups;
2349
2350	int (*post_send)(struct ib_qp *qp, const struct ib_send_wr *send_wr,
2351	const struct ib_send_wr **bad_send_wr);
2352	int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
2353	const struct ib_recv_wr **bad_recv_wr);
2354	void (*drain_rq)(struct ib_qp *qp);
2355	void (*drain_sq)(struct ib_qp *qp);
2356	int (*poll_cq)(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
2357	int (*peek_cq)(struct ib_cq *cq, int wc_cnt);
2358	int (*req_notify_cq)(struct ib_cq *cq, enum ib_cq_notify_flags flags);
2359	int (*post_srq_recv)(struct ib_srq *srq,
2360	const struct ib_recv_wr *recv_wr,
2361	const struct ib_recv_wr **bad_recv_wr);
2362	int (*process_mad)(struct ib_device *device, int process_mad_flags,
2363	u32 port_num, const struct ib_wc *in_wc,
2364	const struct ib_grh *in_grh,
2365	const struct ib_mad *in_mad, struct ib_mad *out_mad,
2366	size_t *out_mad_size, u16 *out_mad_pkey_index);
2367	int (*query_device)(struct ib_device *device,
2368	struct ib_device_attr *device_attr,
2369	struct ib_udata *udata);
2370	int (*modify_device)(struct ib_device *device, int device_modify_mask,
2371	struct ib_device_modify *device_modify);
2372	void (*get_dev_fw_str)(struct ib_device *device, char *str);
2373	const struct cpumask *(*get_vector_affinity)(struct ib_device *ibdev,
2374	int comp_vector);
2375	int (*query_port)(struct ib_device *device, u32 port_num,
2376	struct ib_port_attr *port_attr);
2377	int (*modify_port)(struct ib_device *device, u32 port_num,
2378	int port_modify_mask,
2379	struct ib_port_modify *port_modify);
2380	/**
2381	* The following mandatory functions are used only at device
2382	* registration. Keep functions such as these at the end of this
2383	* structure to avoid cache line misses when accessing struct ib_device
2384	* in fast paths.
2385	*/
2386	int (*get_port_immutable)(struct ib_device *device, u32 port_num,
2387	struct ib_port_immutable *immutable);
2388	enum rdma_link_layer (*get_link_layer)(struct ib_device *device,
2389	u32 port_num);
2390	/**
2391	* When calling get_netdev, the HW vendor's driver should return the
2392	* net device of device @device at port @port_num or NULL if such
2393	* a net device doesn't exist. The vendor driver should call dev_hold
2394	* on this net device. The HW vendor's device driver must guarantee
2395	* that this function returns NULL before the net device has finished
2396	* NETDEV_UNREGISTER state.
2397	*/
2398	struct net_device *(*get_netdev)(struct ib_device *device,
2399	u32 port_num);
2400	/**
2401	* rdma netdev operation
2402	*
2403	* Driver implementing alloc_rdma_netdev or rdma_netdev_get_params
2404	* must return -EOPNOTSUPP if it doesn't support the specified type.
2405	*/
2406	struct net_device *(*alloc_rdma_netdev)(
2407	struct ib_device *device, u32 port_num, enum rdma_netdev_t type,
2408	const char *name, unsigned char name_assign_type,
2409	void (*setup)(struct net_device *));
2410
2411	int (*rdma_netdev_get_params)(struct ib_device *device, u32 port_num,
2412	enum rdma_netdev_t type,
2413	struct rdma_netdev_alloc_params *params);
2414	/**
2415	* query_gid should be return GID value for @device, when @port_num
2416	* link layer is either IB or iWarp. It is no-op if @port_num port
2417	* is RoCE link layer.
2418	*/
2419	int (*query_gid)(struct ib_device *device, u32 port_num, int index,
2420	union ib_gid *gid);
2421	/**
2422	* When calling add_gid, the HW vendor's driver should add the gid
2423	* of device of port at gid index available at @attr. Meta-info of
2424	* that gid (for example, the network device related to this gid) is
2425	* available at @attr. @context allows the HW vendor driver to store
2426	* extra information together with a GID entry. The HW vendor driver may
2427	* allocate memory to contain this information and store it in @context
2428	* when a new GID entry is written to. Params are consistent until the
2429	* next call of add_gid or delete_gid. The function should return 0 on
2430	* success or error otherwise. The function could be called
2431	* concurrently for different ports. This function is only called when
2432	* roce_gid_table is used.
2433	*/
2434	int (*add_gid)(const struct ib_gid_attr *attr, void **context);
2435	/**
2436	* When calling del_gid, the HW vendor's driver should delete the
2437	* gid of device @device at gid index gid_index of port port_num
2438	* available in @attr.
2439	* Upon the deletion of a GID entry, the HW vendor must free any
2440	* allocated memory. The caller will clear @context afterwards.
2441	* This function is only called when roce_gid_table is used.
2442	*/
2443	int (*del_gid)(const struct ib_gid_attr *attr, void **context);
2444	int (*query_pkey)(struct ib_device *device, u32 port_num, u16 index,
2445	u16 *pkey);
2446	int (*alloc_ucontext)(struct ib_ucontext *context,
2447	struct ib_udata *udata);
2448	void (*dealloc_ucontext)(struct ib_ucontext *context);
2449	int (*mmap)(struct ib_ucontext *context, struct vm_area_struct *vma);
2450	/**
2451	* This will be called once refcount of an entry in mmap_xa reaches
2452	* zero. The type of the memory that was mapped may differ between
2453	* entries and is opaque to the rdma_user_mmap interface.
2454	* Therefore needs to be implemented by the driver in mmap_free.
2455	*/
2456	void (*mmap_free)(struct rdma_user_mmap_entry *entry);
2457	void (*disassociate_ucontext)(struct ib_ucontext *ibcontext);
2458	int (*alloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
2459	int (*dealloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
2460	int (*create_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,
2461	struct ib_udata *udata);
2462	int (*create_user_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,
2463	struct ib_udata *udata);
2464	int (*modify_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
2465	int (*query_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
2466	int (*destroy_ah)(struct ib_ah *ah, u32 flags);
2467	int (*create_srq)(struct ib_srq *srq,
2468	struct ib_srq_init_attr *srq_init_attr,
2469	struct ib_udata *udata);
2470	int (*modify_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr,
2471	enum ib_srq_attr_mask srq_attr_mask,
2472	struct ib_udata *udata);
2473	int (*query_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
2474	int (*destroy_srq)(struct ib_srq *srq, struct ib_udata *udata);
2475	int (*create_qp)(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr,
2476	struct ib_udata *udata);
2477	int (*modify_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
2478	int qp_attr_mask, struct ib_udata *udata);
2479	int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
2480	int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
2481	int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata);
2482	int (*create_cq)(struct ib_cq *cq, const struct ib_cq_init_attr *attr,
2483	struct ib_udata *udata);
2484	int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
2485	int (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata);
2486	int (*resize_cq)(struct ib_cq *cq, int cqe, struct ib_udata *udata);
2487	struct ib_mr *(*get_dma_mr)(struct ib_pd *pd, int mr_access_flags);
2488	struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length,
2489	u64 virt_addr, int mr_access_flags,
2490	struct ib_udata *udata);
2491	struct ib_mr *(*reg_user_mr_dmabuf)(struct ib_pd *pd, u64 offset,
2492	u64 length, u64 virt_addr, int fd,
2493	int mr_access_flags,
2494	struct ib_udata *udata);
2495	struct ib_mr *(*rereg_user_mr)(struct ib_mr *mr, int flags, u64 start,
2496	u64 length, u64 virt_addr,
2497	int mr_access_flags, struct ib_pd *pd,
2498	struct ib_udata *udata);
2499	int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata);
2500	struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,
2501	u32 max_num_sg);
2502	struct ib_mr *(*alloc_mr_integrity)(struct ib_pd *pd,
2503	u32 max_num_data_sg,
2504	u32 max_num_meta_sg);
2505	int (*advise_mr)(struct ib_pd *pd,
2506	enum ib_uverbs_advise_mr_advice advice, u32 flags,
2507	struct ib_sge *sg_list, u32 num_sge,
2508	struct uverbs_attr_bundle *attrs);
2509
2510	/*
2511	* Kernel users should universally support relaxed ordering (RO), as
2512	* they are designed to read data only after observing the CQE and use
2513	* the DMA API correctly.
2514	*
2515	* Some drivers implicitly enable RO if platform supports it.
2516	*/
2517	int (*map_mr_sg)(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
2518	unsigned int *sg_offset);
2519	int (*check_mr_status)(struct ib_mr *mr, u32 check_mask,
2520	struct ib_mr_status *mr_status);
2521	int (*alloc_mw)(struct ib_mw *mw, struct ib_udata *udata);
2522	int (*dealloc_mw)(struct ib_mw *mw);
2523	int (*attach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
2524	int (*detach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
2525	int (*alloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
2526	int (*dealloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
2527	struct ib_flow *(*create_flow)(struct ib_qp *qp,
2528	struct ib_flow_attr *flow_attr,
2529	struct ib_udata *udata);
2530	int (*destroy_flow)(struct ib_flow *flow_id);
2531	int (*destroy_flow_action)(struct ib_flow_action *action);
2532	int (*set_vf_link_state)(struct ib_device *device, int vf, u32 port,
2533	int state);
2534	int (*get_vf_config)(struct ib_device *device, int vf, u32 port,
2535	struct ifla_vf_info *ivf);
2536	int (*get_vf_stats)(struct ib_device *device, int vf, u32 port,
2537	struct ifla_vf_stats *stats);
2538	int (*get_vf_guid)(struct ib_device *device, int vf, u32 port,
2539	struct ifla_vf_guid *node_guid,
2540	struct ifla_vf_guid *port_guid);
2541	int (*set_vf_guid)(struct ib_device *device, int vf, u32 port, u64 guid,
2542	int type);
2543	struct ib_wq *(*create_wq)(struct ib_pd *pd,
2544	struct ib_wq_init_attr *init_attr,
2545	struct ib_udata *udata);
2546	int (*destroy_wq)(struct ib_wq *wq, struct ib_udata *udata);
2547	int (*modify_wq)(struct ib_wq *wq, struct ib_wq_attr *attr,
2548	u32 wq_attr_mask, struct ib_udata *udata);
2549	int (*create_rwq_ind_table)(struct ib_rwq_ind_table *ib_rwq_ind_table,
2550	struct ib_rwq_ind_table_init_attr *init_attr,
2551	struct ib_udata *udata);
2552	int (*destroy_rwq_ind_table)(struct ib_rwq_ind_table *wq_ind_table);
2553	struct ib_dm *(*alloc_dm)(struct ib_device *device,
2554	struct ib_ucontext *context,
2555	struct ib_dm_alloc_attr *attr,
2556	struct uverbs_attr_bundle *attrs);
2557	int (*dealloc_dm)(struct ib_dm *dm, struct uverbs_attr_bundle *attrs);
2558	struct ib_mr *(*reg_dm_mr)(struct ib_pd *pd, struct ib_dm *dm,
2559	struct ib_dm_mr_attr *attr,
2560	struct uverbs_attr_bundle *attrs);
2561	int (*create_counters)(struct ib_counters *counters,
2562	struct uverbs_attr_bundle *attrs);
2563	int (*destroy_counters)(struct ib_counters *counters);
2564	int (*read_counters)(struct ib_counters *counters,
2565	struct ib_counters_read_attr *counters_read_attr,
2566	struct uverbs_attr_bundle *attrs);
2567	int (*map_mr_sg_pi)(struct ib_mr *mr, struct scatterlist *data_sg,
2568	int data_sg_nents, unsigned int *data_sg_offset,
2569	struct scatterlist *meta_sg, int meta_sg_nents,
2570	unsigned int *meta_sg_offset);
2571
2572	/**
2573	* alloc_hw_[device,port]_stats - Allocate a struct rdma_hw_stats and
2574	* fill in the driver initialized data. The struct is kfree()'ed by
2575	* the sysfs core when the device is removed. A lifespan of -1 in the
2576	* return struct tells the core to set a default lifespan.
2577	*/
2578	struct rdma_hw_stats *(*alloc_hw_device_stats)(struct ib_device *device);
2579	struct rdma_hw_stats *(*alloc_hw_port_stats)(struct ib_device *device,
2580	u32 port_num);
2581	/**
2582	* get_hw_stats - Fill in the counter value(s) in the stats struct.
2583	* @index - The index in the value array we wish to have updated, or
2584	* num_counters if we want all stats updated
2585	* Return codes -
2586	* < 0 - Error, no counters updated
2587	* index - Updated the single counter pointed to by index
2588	* num_counters - Updated all counters (will reset the timestamp
2589	* and prevent further calls for lifespan milliseconds)
2590	* Drivers are allowed to update all counters in leiu of just the
2591	* one given in index at their option
2592	*/
2593	int (*get_hw_stats)(struct ib_device *device,
2594	struct rdma_hw_stats *stats, u32 port, int index);
2595
2596	/**
2597	* modify_hw_stat - Modify the counter configuration
2598	* @enable: true/false when enable/disable a counter
2599	* Return codes - 0 on success or error code otherwise.
2600	*/
2601	int (*modify_hw_stat)(struct ib_device *device, u32 port,
2602	unsigned int counter_index, bool enable);
2603	/**
2604	* Allows rdma drivers to add their own restrack attributes.
2605	*/
2606	int (*fill_res_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);
2607	int (*fill_res_mr_entry_raw)(struct sk_buff *msg, struct ib_mr *ibmr);
2608	int (*fill_res_cq_entry)(struct sk_buff *msg, struct ib_cq *ibcq);
2609	int (*fill_res_cq_entry_raw)(struct sk_buff *msg, struct ib_cq *ibcq);
2610	int (*fill_res_qp_entry)(struct sk_buff *msg, struct ib_qp *ibqp);
2611	int (*fill_res_qp_entry_raw)(struct sk_buff *msg, struct ib_qp *ibqp);
2612	int (*fill_res_cm_id_entry)(struct sk_buff *msg, struct rdma_cm_id *id);
2613	int (*fill_res_srq_entry)(struct sk_buff *msg, struct ib_srq *ib_srq);
2614	int (*fill_res_srq_entry_raw)(struct sk_buff *msg, struct ib_srq *ib_srq);
2615
2616	/* Device lifecycle callbacks */
2617	/*
2618	* Called after the device becomes registered, before clients are
2619	* attached
2620	*/
2621	int (*enable_driver)(struct ib_device *dev);
2622	/*
2623	* This is called as part of ib_dealloc_device().
2624	*/
2625	void (*dealloc_driver)(struct ib_device *dev);
2626
2627	/* iWarp CM callbacks */
2628	void (*iw_add_ref)(struct ib_qp *qp);
2629	void (*iw_rem_ref)(struct ib_qp *qp);
2630	struct ib_qp *(*iw_get_qp)(struct ib_device *device, int qpn);
2631	int (*iw_connect)(struct iw_cm_id *cm_id,
2632	struct iw_cm_conn_param *conn_param);
2633	int (*iw_accept)(struct iw_cm_id *cm_id,
2634	struct iw_cm_conn_param *conn_param);
2635	int (*iw_reject)(struct iw_cm_id *cm_id, const void *pdata,
2636	u8 pdata_len);
2637	int (*iw_create_listen)(struct iw_cm_id *cm_id, int backlog);
2638	int (*iw_destroy_listen)(struct iw_cm_id *cm_id);
2639	/**
2640	* counter_bind_qp - Bind a QP to a counter.
2641	* @counter - The counter to be bound. If counter->id is zero then
2642	* the driver needs to allocate a new counter and set counter->id
2643	*/
2644	int (*counter_bind_qp)(struct rdma_counter *counter, struct ib_qp *qp);
2645	/**
2646	* counter_unbind_qp - Unbind the qp from the dynamically-allocated
2647	* counter and bind it onto the default one
2648	*/
2649	int (*counter_unbind_qp)(struct ib_qp *qp);
2650	/**
2651	* counter_dealloc -De-allocate the hw counter
2652	*/
2653	int (*counter_dealloc)(struct rdma_counter *counter);
2654	/**
2655	* counter_alloc_stats - Allocate a struct rdma_hw_stats and fill in
2656	* the driver initialized data.
2657	*/
2658	struct rdma_hw_stats *(*counter_alloc_stats)(
2659	struct rdma_counter *counter);
2660	/**
2661	* counter_update_stats - Query the stats value of this counter
2662	*/
2663	int (*counter_update_stats)(struct rdma_counter *counter);
2664
2665	/**
2666	* Allows rdma drivers to add their own restrack attributes
2667	* dumped via 'rdma stat' iproute2 command.
2668	*/
2669	int (*fill_stat_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);
2670
2671	/* query driver for its ucontext properties */
2672	int (*query_ucontext)(struct ib_ucontext *context,
2673	struct uverbs_attr_bundle *attrs);
2674
2675	/*
2676	* Provide NUMA node. This API exists for rdmavt/hfi1 only.
2677	* Everyone else relies on Linux memory management model.
2678	*/
2679	int (*get_numa_node)(struct ib_device *dev);
2680
2681	DECLARE_RDMA_OBJ_SIZE(ib_ah);
2682	DECLARE_RDMA_OBJ_SIZE(ib_counters);
2683	DECLARE_RDMA_OBJ_SIZE(ib_cq);
2684	DECLARE_RDMA_OBJ_SIZE(ib_mw);
2685	DECLARE_RDMA_OBJ_SIZE(ib_pd);
2686	DECLARE_RDMA_OBJ_SIZE(ib_qp);
2687	DECLARE_RDMA_OBJ_SIZE(ib_rwq_ind_table);
2688	DECLARE_RDMA_OBJ_SIZE(ib_srq);
2689	DECLARE_RDMA_OBJ_SIZE(ib_ucontext);
2690	DECLARE_RDMA_OBJ_SIZE(ib_xrcd);
2691	};
2692
2693	struct ib_core_device {
2694	/* device must be the first element in structure until,
2695	* union of ib_core_device and device exists in ib_device.
2696	*/
2697	struct device dev;
2698	possible_net_t rdma_net;
2699	struct kobject *ports_kobj;
2700	struct list_head port_list;
2701	struct ib_device *owner; /* reach back to owner ib_device */
2702	};
2703
2704	struct rdma_restrack_root;
2705	struct ib_device {
2706	/* Do not access @dma_device directly from ULP nor from HW drivers. */
2707	struct device *dma_device;
2708	struct ib_device_ops ops;
2709	char name[IB_DEVICE_NAME_MAX];
2710	struct rcu_head rcu_head;
2711
2712	struct list_head event_handler_list;
2713	/* Protects event_handler_list */
2714	struct rw_semaphore event_handler_rwsem;
2715
2716	/* Protects QP's event_handler calls and open_qp list */
2717	spinlock_t qp_open_list_lock;
2718
2719	struct rw_semaphore client_data_rwsem;
2720	struct xarray client_data;
2721	struct mutex unregistration_lock;
2722
2723	/* Synchronize GID, Pkey cache entries, subnet prefix, LMC */
2724	rwlock_t cache_lock;
2725	/**
2726	* port_data is indexed by port number
2727	*/
2728	struct ib_port_data *port_data;
2729
2730	int num_comp_vectors;
2731
2732	union {
2733	struct device dev;
2734	struct ib_core_device coredev;
2735	};
2736
2737	/* First group is for device attributes,
2738	* Second group is for driver provided attributes (optional).
2739	* Third group is for the hw_stats
2740	* It is a NULL terminated array.
2741	*/
2742	const struct attribute_group *groups[4];
2743
2744	u64 uverbs_cmd_mask;
2745
2746	char node_desc[IB_DEVICE_NODE_DESC_MAX];
2747	__be64 node_guid;
2748	u32 local_dma_lkey;
2749	u16 is_switch:1;
2750	/* Indicates kernel verbs support, should not be used in drivers */
2751	u16 kverbs_provider:1;
2752	/* CQ adaptive moderation (RDMA DIM) */
2753	u16 use_cq_dim:1;
2754	u8 node_type;
2755	u32 phys_port_cnt;
2756	struct ib_device_attr attrs;
2757	struct hw_stats_device_data *hw_stats_data;
2758
2759	#ifdef CONFIG_CGROUP_RDMA
2760	struct rdmacg_device cg_device;
2761	#endif
2762
2763	u32 index;
2764
2765	spinlock_t cq_pools_lock;
2766	struct list_head cq_pools[IB_POLL_LAST_POOL_TYPE + 1];
2767
2768	struct rdma_restrack_root *res;
2769
2770	const struct uapi_definition *driver_def;
2771
2772	/*
2773	* Positive refcount indicates that the device is currently
2774	* registered and cannot be unregistered.
2775	*/
2776	refcount_t refcount;
2777	struct completion unreg_completion;
2778	struct work_struct unregistration_work;
2779
2780	const struct rdma_link_ops *link_ops;
2781
2782	/* Protects compat_devs xarray modifications */
2783	struct mutex compat_devs_mutex;
2784	/* Maintains compat devices for each net namespace */
2785	struct xarray compat_devs;
2786
2787	/* Used by iWarp CM */
2788	char iw_ifname[IFNAMSIZ];
2789	u32 iw_driver_flags;
2790	u32 lag_flags;
2791	};
2792
2793	static inline void *rdma_zalloc_obj(struct ib_device *dev, size_t size,
2794	gfp_t gfp, bool is_numa_aware)
2795	{
2796	if (is_numa_aware && dev->ops.get_numa_node)
2797	return kzalloc_node(size, flags: gfp, node: dev->ops.get_numa_node(dev));
2798
2799	return kzalloc(size, flags: gfp);
2800	}
2801
2802	struct ib_client_nl_info;
2803	struct ib_client {
2804	const char *name;
2805	int (*add)(struct ib_device *ibdev);
2806	void (*remove)(struct ib_device *, void *client_data);
2807	void (*rename)(struct ib_device *dev, void *client_data);
2808	int (*get_nl_info)(struct ib_device *ibdev, void *client_data,
2809	struct ib_client_nl_info *res);
2810	int (*get_global_nl_info)(struct ib_client_nl_info *res);
2811
2812	/* Returns the net_dev belonging to this ib_client and matching the
2813	* given parameters.
2814	* @dev: An RDMA device that the net_dev use for communication.
2815	* @port: A physical port number on the RDMA device.
2816	* @pkey: P_Key that the net_dev uses if applicable.
2817	* @gid: A GID that the net_dev uses to communicate.
2818	* @addr: An IP address the net_dev is configured with.
2819	* @client_data: The device's client data set by ib_set_client_data().
2820	*
2821	* An ib_client that implements a net_dev on top of RDMA devices
2822	* (such as IP over IB) should implement this callback, allowing the
2823	* rdma_cm module to find the right net_dev for a given request.
2824	*
2825	* The caller is responsible for calling dev_put on the returned
2826	* netdev. */
2827	struct net_device *(*get_net_dev_by_params)(
2828	struct ib_device *dev,
2829	u32 port,
2830	u16 pkey,
2831	const union ib_gid *gid,
2832	const struct sockaddr *addr,
2833	void *client_data);
2834
2835	refcount_t uses;
2836	struct completion uses_zero;
2837	u32 client_id;
2838
2839	/* kverbs are not required by the client */
2840	u8 no_kverbs_req:1;
2841	};
2842
2843	/*
2844	* IB block DMA iterator
2845	*
2846	* Iterates the DMA-mapped SGL in contiguous memory blocks aligned
2847	* to a HW supported page size.
2848	*/
2849	struct ib_block_iter {
2850	/* internal states */
2851	struct scatterlist *__sg; /* sg holding the current aligned block */
2852	dma_addr_t __dma_addr; /* unaligned DMA address of this block */
2853	unsigned int __sg_nents; /* number of SG entries */
2854	unsigned int __sg_advance; /* number of bytes to advance in sg in next step */
2855	unsigned int __pg_bit; /* alignment of current block */
2856	};
2857
2858	struct ib_device *_ib_alloc_device(size_t size);
2859	#define ib_alloc_device(drv_struct, member) \
2860	container_of(_ib_alloc_device(sizeof(struct drv_struct) + \
2861	BUILD_BUG_ON_ZERO(offsetof( \
2862	struct drv_struct, member))), \
2863	struct drv_struct, member)
2864
2865	void ib_dealloc_device(struct ib_device *device);
2866
2867	void ib_get_device_fw_str(struct ib_device *device, char *str);
2868
2869	int ib_register_device(struct ib_device *device, const char *name,
2870	struct device *dma_device);
2871	void ib_unregister_device(struct ib_device *device);
2872	void ib_unregister_driver(enum rdma_driver_id driver_id);
2873	void ib_unregister_device_and_put(struct ib_device *device);
2874	void ib_unregister_device_queued(struct ib_device *ib_dev);
2875
2876	int ib_register_client (struct ib_client *client);
2877	void ib_unregister_client(struct ib_client *client);
2878
2879	void __rdma_block_iter_start(struct ib_block_iter *biter,
2880	struct scatterlist *sglist,
2881	unsigned int nents,
2882	unsigned long pgsz);
2883	bool __rdma_block_iter_next(struct ib_block_iter *biter);
2884
2885	/**
2886	* rdma_block_iter_dma_address - get the aligned dma address of the current
2887	* block held by the block iterator.
2888	* @biter: block iterator holding the memory block
2889	*/
2890	static inline dma_addr_t
2891	rdma_block_iter_dma_address(struct ib_block_iter *biter)
2892	{
2893	return biter->__dma_addr & ~(BIT_ULL(biter->__pg_bit) - 1);
2894	}
2895
2896	/**
2897	* rdma_for_each_block - iterate over contiguous memory blocks of the sg list
2898	* @sglist: sglist to iterate over
2899	* @biter: block iterator holding the memory block
2900	* @nents: maximum number of sg entries to iterate over
2901	* @pgsz: best HW supported page size to use
2902	*
2903	* Callers may use rdma_block_iter_dma_address() to get each
2904	* blocks aligned DMA address.
2905	*/
2906	#define rdma_for_each_block(sglist, biter, nents, pgsz) \
2907	for (__rdma_block_iter_start(biter, sglist, nents, \
2908	pgsz); \
2909	__rdma_block_iter_next(biter);)
2910
2911	/**
2912	* ib_get_client_data - Get IB client context
2913	* @device:Device to get context for
2914	* @client:Client to get context for
2915	*
2916	* ib_get_client_data() returns the client context data set with
2917	* ib_set_client_data(). This can only be called while the client is
2918	* registered to the device, once the ib_client remove() callback returns this
2919	* cannot be called.
2920	*/
2921	static inline void *ib_get_client_data(struct ib_device *device,
2922	struct ib_client *client)
2923	{
2924	return xa_load(&device->client_data, index: client->client_id);
2925	}
2926	void ib_set_client_data(struct ib_device *device, struct ib_client *client,
2927	void *data);
2928	void ib_set_device_ops(struct ib_device *device,
2929	const struct ib_device_ops *ops);
2930
2931	int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
2932	unsigned long pfn, unsigned long size, pgprot_t prot,
2933	struct rdma_user_mmap_entry *entry);
2934	int rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext,
2935	struct rdma_user_mmap_entry *entry,
2936	size_t length);
2937	int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext,
2938	struct rdma_user_mmap_entry *entry,
2939	size_t length, u32 min_pgoff,
2940	u32 max_pgoff);
2941
2942	static inline int
2943	rdma_user_mmap_entry_insert_exact(struct ib_ucontext *ucontext,
2944	struct rdma_user_mmap_entry *entry,
2945	size_t length, u32 pgoff)
2946	{
2947	return rdma_user_mmap_entry_insert_range(ucontext, entry, length, min_pgoff: pgoff,
2948	max_pgoff: pgoff);
2949	}
2950
2951	struct rdma_user_mmap_entry *
2952	rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext,
2953	unsigned long pgoff);
2954	struct rdma_user_mmap_entry *
2955	rdma_user_mmap_entry_get(struct ib_ucontext *ucontext,
2956	struct vm_area_struct *vma);
2957	void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry);
2958
2959	void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry);
2960
2961	static inline int ib_copy_from_udata(void *dest, struct ib_udata *udata, size_t len)
2962	{
2963	return copy_from_user(to: dest, from: udata->inbuf, n: len) ? -EFAULT : 0;
2964	}
2965
2966	static inline int ib_copy_to_udata(struct ib_udata *udata, void *src, size_t len)
2967	{
2968	return copy_to_user(to: udata->outbuf, from: src, n: len) ? -EFAULT : 0;
2969	}
2970
2971	static inline bool ib_is_buffer_cleared(const void __user *p,
2972	size_t len)
2973	{
2974	bool ret;
2975	u8 *buf;
2976
2977	if (len > USHRT_MAX)
2978	return false;
2979
2980	buf = memdup_user(p, len);
2981	if (IS_ERR(ptr: buf))
2982	return false;
2983
2984	ret = !memchr_inv(p: buf, c: 0, size: len);
2985	kfree(objp: buf);
2986	return ret;
2987	}
2988
2989	static inline bool ib_is_udata_cleared(struct ib_udata *udata,
2990	size_t offset,
2991	size_t len)
2992	{
2993	return ib_is_buffer_cleared(p: udata->inbuf + offset, len);
2994	}
2995
2996	/**
2997	* ib_modify_qp_is_ok - Check that the supplied attribute mask
2998	* contains all required attributes and no attributes not allowed for
2999	* the given QP state transition.
3000	* @cur_state: Current QP state
3001	* @next_state: Next QP state
3002	* @type: QP type
3003	* @mask: Mask of supplied QP attributes
3004	*
3005	* This function is a helper function that a low-level driver's
3006	* modify_qp method can use to validate the consumer's input. It
3007	* checks that cur_state and next_state are valid QP states, that a
3008	* transition from cur_state to next_state is allowed by the IB spec,
3009	* and that the attribute mask supplied is allowed for the transition.
3010	*/
3011	bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
3012	enum ib_qp_type type, enum ib_qp_attr_mask mask);
3013
3014	void ib_register_event_handler(struct ib_event_handler *event_handler);
3015	void ib_unregister_event_handler(struct ib_event_handler *event_handler);
3016	void ib_dispatch_event(const struct ib_event *event);
3017
3018	int ib_query_port(struct ib_device *device,
3019	u32 port_num, struct ib_port_attr *port_attr);
3020
3021	enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device,
3022	u32 port_num);
3023
3024	/**
3025	* rdma_cap_ib_switch - Check if the device is IB switch
3026	* @device: Device to check
3027	*
3028	* Device driver is responsible for setting is_switch bit on
3029	* in ib_device structure at init time.
3030	*
3031	* Return: true if the device is IB switch.
3032	*/
3033	static inline bool rdma_cap_ib_switch(const struct ib_device *device)
3034	{
3035	return device->is_switch;
3036	}
3037
3038	/**
3039	* rdma_start_port - Return the first valid port number for the device
3040	* specified
3041	*
3042	* @device: Device to be checked
3043	*
3044	* Return start port number
3045	*/
3046	static inline u32 rdma_start_port(const struct ib_device *device)
3047	{
3048	return rdma_cap_ib_switch(device) ? 0 : 1;
3049	}
3050
3051	/**
3052	* rdma_for_each_port - Iterate over all valid port numbers of the IB device
3053	* @device - The struct ib_device * to iterate over
3054	* @iter - The unsigned int to store the port number
3055	*/
3056	#define rdma_for_each_port(device, iter) \
3057	for (iter = rdma_start_port(device + \
3058	BUILD_BUG_ON_ZERO(!__same_type(u32, \
3059	iter))); \
3060	iter <= rdma_end_port(device); iter++)
3061
3062	/**
3063	* rdma_end_port - Return the last valid port number for the device
3064	* specified
3065	*
3066	* @device: Device to be checked
3067	*
3068	* Return last port number
3069	*/
3070	static inline u32 rdma_end_port(const struct ib_device *device)
3071	{
3072	return rdma_cap_ib_switch(device) ? 0 : device->phys_port_cnt;
3073	}
3074
3075	static inline int rdma_is_port_valid(const struct ib_device *device,
3076	unsigned int port)
3077	{
3078	return (port >= rdma_start_port(device) &&
3079	port <= rdma_end_port(device));
3080	}
3081
3082	static inline bool rdma_is_grh_required(const struct ib_device *device,
3083	u32 port_num)
3084	{
3085	return device->port_data[port_num].immutable.core_cap_flags &
3086	RDMA_CORE_PORT_IB_GRH_REQUIRED;
3087	}
3088
3089	static inline bool rdma_protocol_ib(const struct ib_device *device,
3090	u32 port_num)
3091	{
3092	return device->port_data[port_num].immutable.core_cap_flags &
3093	RDMA_CORE_CAP_PROT_IB;
3094	}
3095
3096	static inline bool rdma_protocol_roce(const struct ib_device *device,
3097	u32 port_num)
3098	{
3099	return device->port_data[port_num].immutable.core_cap_flags &
3100	(RDMA_CORE_CAP_PROT_ROCE | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP);
3101	}
3102
3103	static inline bool rdma_protocol_roce_udp_encap(const struct ib_device *device,
3104	u32 port_num)
3105	{
3106	return device->port_data[port_num].immutable.core_cap_flags &
3107	RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
3108	}
3109
3110	static inline bool rdma_protocol_roce_eth_encap(const struct ib_device *device,
3111	u32 port_num)
3112	{
3113	return device->port_data[port_num].immutable.core_cap_flags &
3114	RDMA_CORE_CAP_PROT_ROCE;
3115	}
3116
3117	static inline bool rdma_protocol_iwarp(const struct ib_device *device,
3118	u32 port_num)
3119	{
3120	return device->port_data[port_num].immutable.core_cap_flags &
3121	RDMA_CORE_CAP_PROT_IWARP;
3122	}
3123
3124	static inline bool rdma_ib_or_roce(const struct ib_device *device,
3125	u32 port_num)
3126	{
3127	return rdma_protocol_ib(device, port_num) ||
3128	rdma_protocol_roce(device, port_num);
3129	}
3130
3131	static inline bool rdma_protocol_raw_packet(const struct ib_device *device,
3132	u32 port_num)
3133	{
3134	return device->port_data[port_num].immutable.core_cap_flags &
3135	RDMA_CORE_CAP_PROT_RAW_PACKET;
3136	}
3137
3138	static inline bool rdma_protocol_usnic(const struct ib_device *device,
3139	u32 port_num)
3140	{
3141	return device->port_data[port_num].immutable.core_cap_flags &
3142	RDMA_CORE_CAP_PROT_USNIC;
3143	}
3144
3145	/**
3146	* rdma_cap_ib_mad - Check if the port of a device supports Infiniband
3147	* Management Datagrams.
3148	* @device: Device to check
3149	* @port_num: Port number to check
3150	*
3151	* Management Datagrams (MAD) are a required part of the InfiniBand
3152	* specification and are supported on all InfiniBand devices. A slightly
3153	* extended version are also supported on OPA interfaces.
3154	*
3155	* Return: true if the port supports sending/receiving of MAD packets.
3156	*/
3157	static inline bool rdma_cap_ib_mad(const struct ib_device *device, u32 port_num)
3158	{
3159	return device->port_data[port_num].immutable.core_cap_flags &
3160	RDMA_CORE_CAP_IB_MAD;
3161	}
3162
3163	/**
3164	* rdma_cap_opa_mad - Check if the port of device provides support for OPA
3165	* Management Datagrams.
3166	* @device: Device to check
3167	* @port_num: Port number to check
3168	*
3169	* Intel OmniPath devices extend and/or replace the InfiniBand Management
3170	* datagrams with their own versions. These OPA MADs share many but not all of
3171	* the characteristics of InfiniBand MADs.
3172	*
3173	* OPA MADs differ in the following ways:
3174	*
3175	* 1) MADs are variable size up to 2K
3176	* IBTA defined MADs remain fixed at 256 bytes
3177	* 2) OPA SMPs must carry valid PKeys
3178	* 3) OPA SMP packets are a different format
3179	*
3180	* Return: true if the port supports OPA MAD packet formats.
3181	*/
3182	static inline bool rdma_cap_opa_mad(struct ib_device *device, u32 port_num)
3183	{
3184	return device->port_data[port_num].immutable.core_cap_flags &
3185	RDMA_CORE_CAP_OPA_MAD;
3186	}
3187
3188	/**
3189	* rdma_cap_ib_smi - Check if the port of a device provides an Infiniband
3190	* Subnet Management Agent (SMA) on the Subnet Management Interface (SMI).
3191	* @device: Device to check
3192	* @port_num: Port number to check
3193	*
3194	* Each InfiniBand node is required to provide a Subnet Management Agent
3195	* that the subnet manager can access. Prior to the fabric being fully
3196	* configured by the subnet manager, the SMA is accessed via a well known
3197	* interface called the Subnet Management Interface (SMI). This interface
3198	* uses directed route packets to communicate with the SM to get around the
3199	* chicken and egg problem of the SM needing to know what's on the fabric
3200	* in order to configure the fabric, and needing to configure the fabric in
3201	* order to send packets to the devices on the fabric. These directed
3202	* route packets do not need the fabric fully configured in order to reach
3203	* their destination. The SMI is the only method allowed to send
3204	* directed route packets on an InfiniBand fabric.
3205	*
3206	* Return: true if the port provides an SMI.
3207	*/
3208	static inline bool rdma_cap_ib_smi(const struct ib_device *device, u32 port_num)
3209	{
3210	return device->port_data[port_num].immutable.core_cap_flags &
3211	RDMA_CORE_CAP_IB_SMI;
3212	}
3213
3214	/**
3215	* rdma_cap_ib_cm - Check if the port of device has the capability Infiniband
3216	* Communication Manager.
3217	* @device: Device to check
3218	* @port_num: Port number to check
3219	*
3220	* The InfiniBand Communication Manager is one of many pre-defined General
3221	* Service Agents (GSA) that are accessed via the General Service
3222	* Interface (GSI). It's role is to facilitate establishment of connections
3223	* between nodes as well as other management related tasks for established
3224	* connections.
3225	*
3226	* Return: true if the port supports an IB CM (this does not guarantee that
3227	* a CM is actually running however).
3228	*/
3229	static inline bool rdma_cap_ib_cm(const struct ib_device *device, u32 port_num)
3230	{
3231	return device->port_data[port_num].immutable.core_cap_flags &
3232	RDMA_CORE_CAP_IB_CM;
3233	}
3234
3235	/**
3236	* rdma_cap_iw_cm - Check if the port of device has the capability IWARP
3237	* Communication Manager.
3238	* @device: Device to check
3239	* @port_num: Port number to check
3240	*
3241	* Similar to above, but specific to iWARP connections which have a different
3242	* managment protocol than InfiniBand.
3243	*
3244	* Return: true if the port supports an iWARP CM (this does not guarantee that
3245	* a CM is actually running however).
3246	*/
3247	static inline bool rdma_cap_iw_cm(const struct ib_device *device, u32 port_num)
3248	{
3249	return device->port_data[port_num].immutable.core_cap_flags &
3250	RDMA_CORE_CAP_IW_CM;
3251	}
3252
3253	/**
3254	* rdma_cap_ib_sa - Check if the port of device has the capability Infiniband
3255	* Subnet Administration.
3256	* @device: Device to check
3257	* @port_num: Port number to check
3258	*
3259	* An InfiniBand Subnet Administration (SA) service is a pre-defined General
3260	* Service Agent (GSA) provided by the Subnet Manager (SM). On InfiniBand
3261	* fabrics, devices should resolve routes to other hosts by contacting the
3262	* SA to query the proper route.
3263	*
3264	* Return: true if the port should act as a client to the fabric Subnet
3265	* Administration interface. This does not imply that the SA service is
3266	* running locally.
3267	*/
3268	static inline bool rdma_cap_ib_sa(const struct ib_device *device, u32 port_num)
3269	{
3270	return device->port_data[port_num].immutable.core_cap_flags &
3271	RDMA_CORE_CAP_IB_SA;
3272	}
3273
3274	/**
3275	* rdma_cap_ib_mcast - Check if the port of device has the capability Infiniband
3276	* Multicast.
3277	* @device: Device to check
3278	* @port_num: Port number to check
3279	*
3280	* InfiniBand multicast registration is more complex than normal IPv4 or
3281	* IPv6 multicast registration. Each Host Channel Adapter must register
3282	* with the Subnet Manager when it wishes to join a multicast group. It
3283	* should do so only once regardless of how many queue pairs it subscribes
3284	* to this group. And it should leave the group only after all queue pairs
3285	* attached to the group have been detached.
3286	*
3287	* Return: true if the port must undertake the additional adminstrative
3288	* overhead of registering/unregistering with the SM and tracking of the
3289	* total number of queue pairs attached to the multicast group.
3290	*/
3291	static inline bool rdma_cap_ib_mcast(const struct ib_device *device,
3292	u32 port_num)
3293	{
3294	return rdma_cap_ib_sa(device, port_num);
3295	}
3296
3297	/**
3298	* rdma_cap_af_ib - Check if the port of device has the capability
3299	* Native Infiniband Address.
3300	* @device: Device to check
3301	* @port_num: Port number to check
3302	*
3303	* InfiniBand addressing uses a port's GUID + Subnet Prefix to make a default
3304	* GID. RoCE uses a different mechanism, but still generates a GID via
3305	* a prescribed mechanism and port specific data.
3306	*
3307	* Return: true if the port uses a GID address to identify devices on the
3308	* network.
3309	*/
3310	static inline bool rdma_cap_af_ib(const struct ib_device *device, u32 port_num)
3311	{
3312	return device->port_data[port_num].immutable.core_cap_flags &
3313	RDMA_CORE_CAP_AF_IB;
3314	}
3315
3316	/**
3317	* rdma_cap_eth_ah - Check if the port of device has the capability
3318	* Ethernet Address Handle.
3319	* @device: Device to check
3320	* @port_num: Port number to check
3321	*
3322	* RoCE is InfiniBand over Ethernet, and it uses a well defined technique
3323	* to fabricate GIDs over Ethernet/IP specific addresses native to the
3324	* port. Normally, packet headers are generated by the sending host
3325	* adapter, but when sending connectionless datagrams, we must manually
3326	* inject the proper headers for the fabric we are communicating over.
3327	*
3328	* Return: true if we are running as a RoCE port and must force the
3329	* addition of a Global Route Header built from our Ethernet Address
3330	* Handle into our header list for connectionless packets.
3331	*/
3332	static inline bool rdma_cap_eth_ah(const struct ib_device *device, u32 port_num)
3333	{
3334	return device->port_data[port_num].immutable.core_cap_flags &
3335	RDMA_CORE_CAP_ETH_AH;
3336	}
3337
3338	/**
3339	* rdma_cap_opa_ah - Check if the port of device supports
3340	* OPA Address handles
3341	* @device: Device to check
3342	* @port_num: Port number to check
3343	*
3344	* Return: true if we are running on an OPA device which supports
3345	* the extended OPA addressing.
3346	*/
3347	static inline bool rdma_cap_opa_ah(struct ib_device *device, u32 port_num)
3348	{
3349	return (device->port_data[port_num].immutable.core_cap_flags &
3350	RDMA_CORE_CAP_OPA_AH) == RDMA_CORE_CAP_OPA_AH;
3351	}
3352
3353	/**
3354	* rdma_max_mad_size - Return the max MAD size required by this RDMA Port.
3355	*
3356	* @device: Device
3357	* @port_num: Port number
3358	*
3359	* This MAD size includes the MAD headers and MAD payload. No other headers
3360	* are included.
3361	*
3362	* Return the max MAD size required by the Port. Will return 0 if the port
3363	* does not support MADs
3364	*/
3365	static inline size_t rdma_max_mad_size(const struct ib_device *device,
3366	u32 port_num)
3367	{
3368	return device->port_data[port_num].immutable.max_mad_size;
3369	}
3370
3371	/**
3372	* rdma_cap_roce_gid_table - Check if the port of device uses roce_gid_table
3373	* @device: Device to check
3374	* @port_num: Port number to check
3375	*
3376	* RoCE GID table mechanism manages the various GIDs for a device.
3377	*
3378	* NOTE: if allocating the port's GID table has failed, this call will still
3379	* return true, but any RoCE GID table API will fail.
3380	*
3381	* Return: true if the port uses RoCE GID table mechanism in order to manage
3382	* its GIDs.
3383	*/
3384	static inline bool rdma_cap_roce_gid_table(const struct ib_device *device,
3385	u32 port_num)
3386	{
3387	return rdma_protocol_roce(device, port_num) &&
3388	device->ops.add_gid && device->ops.del_gid;
3389	}
3390
3391	/*
3392	* Check if the device supports READ W/ INVALIDATE.
3393	*/
3394	static inline bool rdma_cap_read_inv(struct ib_device *dev, u32 port_num)
3395	{
3396	/*
3397	* iWarp drivers must support READ W/ INVALIDATE. No other protocol
3398	* has support for it yet.
3399	*/
3400	return rdma_protocol_iwarp(device: dev, port_num);
3401	}
3402
3403	/**
3404	* rdma_core_cap_opa_port - Return whether the RDMA Port is OPA or not.
3405	* @device: Device
3406	* @port_num: 1 based Port number
3407	*
3408	* Return true if port is an Intel OPA port , false if not
3409	*/
3410	static inline bool rdma_core_cap_opa_port(struct ib_device *device,
3411	u32 port_num)
3412	{
3413	return (device->port_data[port_num].immutable.core_cap_flags &
3414	RDMA_CORE_PORT_INTEL_OPA) == RDMA_CORE_PORT_INTEL_OPA;
3415	}
3416
3417	/**
3418	* rdma_mtu_enum_to_int - Return the mtu of the port as an integer value.
3419	* @device: Device
3420	* @port_num: Port number
3421	* @mtu: enum value of MTU
3422	*
3423	* Return the MTU size supported by the port as an integer value. Will return
3424	* -1 if enum value of mtu is not supported.
3425	*/
3426	static inline int rdma_mtu_enum_to_int(struct ib_device *device, u32 port,
3427	int mtu)
3428	{
3429	if (rdma_core_cap_opa_port(device, port_num: port))
3430	return opa_mtu_enum_to_int(mtu: (enum opa_mtu)mtu);
3431	else
3432	return ib_mtu_enum_to_int(mtu: (enum ib_mtu)mtu);
3433	}
3434
3435	/**
3436	* rdma_mtu_from_attr - Return the mtu of the port from the port attribute.
3437	* @device: Device
3438	* @port_num: Port number
3439	* @attr: port attribute
3440	*
3441	* Return the MTU size supported by the port as an integer value.
3442	*/
3443	static inline int rdma_mtu_from_attr(struct ib_device *device, u32 port,
3444	struct ib_port_attr *attr)
3445	{
3446	if (rdma_core_cap_opa_port(device, port_num: port))
3447	return attr->phys_mtu;
3448	else
3449	return ib_mtu_enum_to_int(mtu: attr->max_mtu);
3450	}
3451
3452	int ib_set_vf_link_state(struct ib_device *device, int vf, u32 port,
3453	int state);
3454	int ib_get_vf_config(struct ib_device *device, int vf, u32 port,
3455	struct ifla_vf_info *info);
3456	int ib_get_vf_stats(struct ib_device *device, int vf, u32 port,
3457	struct ifla_vf_stats *stats);
3458	int ib_get_vf_guid(struct ib_device *device, int vf, u32 port,
3459	struct ifla_vf_guid *node_guid,
3460	struct ifla_vf_guid *port_guid);
3461	int ib_set_vf_guid(struct ib_device *device, int vf, u32 port, u64 guid,
3462	int type);
3463
3464	int ib_query_pkey(struct ib_device *device,
3465	u32 port_num, u16 index, u16 *pkey);
3466
3467	int ib_modify_device(struct ib_device *device,
3468	int device_modify_mask,
3469	struct ib_device_modify *device_modify);
3470
3471	int ib_modify_port(struct ib_device *device,
3472	u32 port_num, int port_modify_mask,
3473	struct ib_port_modify *port_modify);
3474
3475	int ib_find_gid(struct ib_device *device, union ib_gid *gid,
3476	u32 *port_num, u16 *index);
3477
3478	int ib_find_pkey(struct ib_device *device,
3479	u32 port_num, u16 pkey, u16 *index);
3480
3481	enum ib_pd_flags {
3482	/*
3483	* Create a memory registration for all memory in the system and place
3484	* the rkey for it into pd->unsafe_global_rkey. This can be used by
3485	* ULPs to avoid the overhead of dynamic MRs.
3486	*
3487	* This flag is generally considered unsafe and must only be used in
3488	* extremly trusted environments. Every use of it will log a warning
3489	* in the kernel log.
3490	*/
3491	IB_PD_UNSAFE_GLOBAL_RKEY = 0x01,
3492	};
3493
3494	struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
3495	const char *caller);
3496
3497	/**
3498	* ib_alloc_pd - Allocates an unused protection domain.
3499	* @device: The device on which to allocate the protection domain.
3500	* @flags: protection domain flags
3501	*
3502	* A protection domain object provides an association between QPs, shared
3503	* receive queues, address handles, memory regions, and memory windows.
3504	*
3505	* Every PD has a local_dma_lkey which can be used as the lkey value for local
3506	* memory operations.
3507	*/
3508	#define ib_alloc_pd(device, flags) \
3509	__ib_alloc_pd((device), (flags), KBUILD_MODNAME)
3510
3511	int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata);
3512
3513	/**
3514	* ib_dealloc_pd - Deallocate kernel PD
3515	* @pd: The protection domain
3516	*
3517	* NOTE: for user PD use ib_dealloc_pd_user with valid udata!
3518	*/
3519	static inline void ib_dealloc_pd(struct ib_pd *pd)
3520	{
3521	int ret = ib_dealloc_pd_user(pd, NULL);
3522
3523	WARN_ONCE(ret, "Destroy of kernel PD shouldn't fail");
3524	}
3525
3526	enum rdma_create_ah_flags {
3527	/* In a sleepable context */
3528	RDMA_CREATE_AH_SLEEPABLE = BIT(0),
3529	};
3530
3531	/**
3532	* rdma_create_ah - Creates an address handle for the given address vector.
3533	* @pd: The protection domain associated with the address handle.
3534	* @ah_attr: The attributes of the address vector.
3535	* @flags: Create address handle flags (see enum rdma_create_ah_flags).
3536	*
3537	* The address handle is used to reference a local or global destination
3538	* in all UD QP post sends.
3539	*/
3540	struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
3541	u32 flags);
3542
3543	/**
3544	* rdma_create_user_ah - Creates an address handle for the given address vector.
3545	* It resolves destination mac address for ah attribute of RoCE type.
3546	* @pd: The protection domain associated with the address handle.
3547	* @ah_attr: The attributes of the address vector.
3548	* @udata: pointer to user's input output buffer information need by
3549	* provider driver.
3550	*
3551	* It returns 0 on success and returns appropriate error code on error.
3552	* The address handle is used to reference a local or global destination
3553	* in all UD QP post sends.
3554	*/
3555	struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,
3556	struct rdma_ah_attr *ah_attr,
3557	struct ib_udata *udata);
3558	/**
3559	* ib_get_gids_from_rdma_hdr - Get sgid and dgid from GRH or IPv4 header
3560	* work completion.
3561	* @hdr: the L3 header to parse
3562	* @net_type: type of header to parse
3563	* @sgid: place to store source gid
3564	* @dgid: place to store destination gid
3565	*/
3566	int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
3567	enum rdma_network_type net_type,
3568	union ib_gid *sgid, union ib_gid *dgid);
3569
3570	/**
3571	* ib_get_rdma_header_version - Get the header version
3572	* @hdr: the L3 header to parse
3573	*/
3574	int ib_get_rdma_header_version(const union rdma_network_hdr *hdr);
3575
3576	/**
3577	* ib_init_ah_attr_from_wc - Initializes address handle attributes from a
3578	* work completion.
3579	* @device: Device on which the received message arrived.
3580	* @port_num: Port on which the received message arrived.
3581	* @wc: Work completion associated with the received message.
3582	* @grh: References the received global route header. This parameter is
3583	* ignored unless the work completion indicates that the GRH is valid.
3584	* @ah_attr: Returned attributes that can be used when creating an address
3585	* handle for replying to the message.
3586	* When ib_init_ah_attr_from_wc() returns success,
3587	* (a) for IB link layer it optionally contains a reference to SGID attribute
3588	* when GRH is present for IB link layer.
3589	* (b) for RoCE link layer it contains a reference to SGID attribute.
3590	* User must invoke rdma_cleanup_ah_attr_gid_attr() to release reference to SGID
3591	* attributes which are initialized using ib_init_ah_attr_from_wc().
3592	*
3593	*/
3594	int ib_init_ah_attr_from_wc(struct ib_device *device, u32 port_num,
3595	const struct ib_wc *wc, const struct ib_grh *grh,
3596	struct rdma_ah_attr *ah_attr);
3597
3598	/**
3599	* ib_create_ah_from_wc - Creates an address handle associated with the
3600	* sender of the specified work completion.
3601	* @pd: The protection domain associated with the address handle.
3602	* @wc: Work completion information associated with a received message.
3603	* @grh: References the received global route header. This parameter is
3604	* ignored unless the work completion indicates that the GRH is valid.
3605	* @port_num: The outbound port number to associate with the address.
3606	*
3607	* The address handle is used to reference a local or global destination
3608	* in all UD QP post sends.
3609	*/
3610	struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
3611	const struct ib_grh *grh, u32 port_num);
3612
3613	/**
3614	* rdma_modify_ah - Modifies the address vector associated with an address
3615	* handle.
3616	* @ah: The address handle to modify.
3617	* @ah_attr: The new address vector attributes to associate with the
3618	* address handle.
3619	*/
3620	int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
3621
3622	/**
3623	* rdma_query_ah - Queries the address vector associated with an address
3624	* handle.
3625	* @ah: The address handle to query.
3626	* @ah_attr: The address vector attributes associated with the address
3627	* handle.
3628	*/
3629	int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
3630
3631	enum rdma_destroy_ah_flags {
3632	/* In a sleepable context */
3633	RDMA_DESTROY_AH_SLEEPABLE = BIT(0),
3634	};
3635
3636	/**
3637	* rdma_destroy_ah_user - Destroys an address handle.
3638	* @ah: The address handle to destroy.
3639	* @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).
3640	* @udata: Valid user data or NULL for kernel objects
3641	*/
3642	int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata);
3643
3644	/**
3645	* rdma_destroy_ah - Destroys an kernel address handle.
3646	* @ah: The address handle to destroy.
3647	* @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).
3648	*
3649	* NOTE: for user ah use rdma_destroy_ah_user with valid udata!
3650	*/
3651	static inline void rdma_destroy_ah(struct ib_ah *ah, u32 flags)
3652	{
3653	int ret = rdma_destroy_ah_user(ah, flags, NULL);
3654
3655	WARN_ONCE(ret, "Destroy of kernel AH shouldn't fail");
3656	}
3657
3658	struct ib_srq *ib_create_srq_user(struct ib_pd *pd,
3659	struct ib_srq_init_attr *srq_init_attr,
3660	struct ib_usrq_object *uobject,
3661	struct ib_udata *udata);
3662	static inline struct ib_srq *
3663	ib_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *srq_init_attr)
3664	{
3665	if (!pd->device->ops.create_srq)
3666	return ERR_PTR(error: -EOPNOTSUPP);
3667
3668	return ib_create_srq_user(pd, srq_init_attr, NULL, NULL);
3669	}
3670
3671	/**
3672	* ib_modify_srq - Modifies the attributes for the specified SRQ.
3673	* @srq: The SRQ to modify.
3674	* @srq_attr: On input, specifies the SRQ attributes to modify. On output,
3675	* the current values of selected SRQ attributes are returned.
3676	* @srq_attr_mask: A bit-mask used to specify which attributes of the SRQ
3677	* are being modified.
3678	*
3679	* The mask may contain IB_SRQ_MAX_WR to resize the SRQ and/or
3680	* IB_SRQ_LIMIT to set the SRQ's limit and request notification when
3681	* the number of receives queued drops below the limit.
3682	*/
3683	int ib_modify_srq(struct ib_srq *srq,
3684	struct ib_srq_attr *srq_attr,
3685	enum ib_srq_attr_mask srq_attr_mask);
3686
3687	/**
3688	* ib_query_srq - Returns the attribute list and current values for the
3689	* specified SRQ.
3690	* @srq: The SRQ to query.
3691	* @srq_attr: The attributes of the specified SRQ.
3692	*/
3693	int ib_query_srq(struct ib_srq *srq,
3694	struct ib_srq_attr *srq_attr);
3695
3696	/**
3697	* ib_destroy_srq_user - Destroys the specified SRQ.
3698	* @srq: The SRQ to destroy.
3699	* @udata: Valid user data or NULL for kernel objects
3700	*/
3701	int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata);
3702
3703	/**
3704	* ib_destroy_srq - Destroys the specified kernel SRQ.
3705	* @srq: The SRQ to destroy.
3706	*
3707	* NOTE: for user srq use ib_destroy_srq_user with valid udata!
3708	*/
3709	static inline void ib_destroy_srq(struct ib_srq *srq)
3710	{
3711	int ret = ib_destroy_srq_user(srq, NULL);
3712
3713	WARN_ONCE(ret, "Destroy of kernel SRQ shouldn't fail");
3714	}
3715
3716	/**
3717	* ib_post_srq_recv - Posts a list of work requests to the specified SRQ.
3718	* @srq: The SRQ to post the work request on.
3719	* @recv_wr: A list of work requests to post on the receive queue.
3720	* @bad_recv_wr: On an immediate failure, this parameter will reference
3721	* the work request that failed to be posted on the QP.
3722	*/
3723	static inline int ib_post_srq_recv(struct ib_srq *srq,
3724	const struct ib_recv_wr *recv_wr,
3725	const struct ib_recv_wr **bad_recv_wr)
3726	{
3727	const struct ib_recv_wr *dummy;
3728
3729	return srq->device->ops.post_srq_recv(srq, recv_wr,
3730	bad_recv_wr ? : &dummy);
3731	}
3732
3733	struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd,
3734	struct ib_qp_init_attr *qp_init_attr,
3735	const char *caller);
3736	/**
3737	* ib_create_qp - Creates a kernel QP associated with the specific protection
3738	* domain.
3739	* @pd: The protection domain associated with the QP.
3740	* @init_attr: A list of initial attributes required to create the
3741	* QP. If QP creation succeeds, then the attributes are updated to
3742	* the actual capabilities of the created QP.
3743	*/
3744	static inline struct ib_qp *ib_create_qp(struct ib_pd *pd,
3745	struct ib_qp_init_attr *init_attr)
3746	{
3747	return ib_create_qp_kernel(pd, qp_init_attr: init_attr, KBUILD_MODNAME);
3748	}
3749
3750	/**
3751	* ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
3752	* @qp: The QP to modify.
3753	* @attr: On input, specifies the QP attributes to modify. On output,
3754	* the current values of selected QP attributes are returned.
3755	* @attr_mask: A bit-mask used to specify which attributes of the QP
3756	* are being modified.
3757	* @udata: pointer to user's input output buffer information
3758	* are being modified.
3759	* It returns 0 on success and returns appropriate error code on error.
3760	*/
3761	int ib_modify_qp_with_udata(struct ib_qp *qp,
3762	struct ib_qp_attr *attr,
3763	int attr_mask,
3764	struct ib_udata *udata);
3765
3766	/**
3767	* ib_modify_qp - Modifies the attributes for the specified QP and then
3768	* transitions the QP to the given state.
3769	* @qp: The QP to modify.
3770	* @qp_attr: On input, specifies the QP attributes to modify. On output,
3771	* the current values of selected QP attributes are returned.
3772	* @qp_attr_mask: A bit-mask used to specify which attributes of the QP
3773	* are being modified.
3774	*/
3775	int ib_modify_qp(struct ib_qp *qp,
3776	struct ib_qp_attr *qp_attr,
3777	int qp_attr_mask);
3778
3779	/**
3780	* ib_query_qp - Returns the attribute list and current values for the
3781	* specified QP.
3782	* @qp: The QP to query.
3783	* @qp_attr: The attributes of the specified QP.
3784	* @qp_attr_mask: A bit-mask used to select specific attributes to query.
3785	* @qp_init_attr: Additional attributes of the selected QP.
3786	*
3787	* The qp_attr_mask may be used to limit the query to gathering only the
3788	* selected attributes.
3789	*/
3790	int ib_query_qp(struct ib_qp *qp,
3791	struct ib_qp_attr *qp_attr,
3792	int qp_attr_mask,
3793	struct ib_qp_init_attr *qp_init_attr);
3794
3795	/**
3796	* ib_destroy_qp - Destroys the specified QP.
3797	* @qp: The QP to destroy.
3798	* @udata: Valid udata or NULL for kernel objects
3799	*/
3800	int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata);
3801
3802	/**
3803	* ib_destroy_qp - Destroys the specified kernel QP.
3804	* @qp: The QP to destroy.
3805	*
3806	* NOTE: for user qp use ib_destroy_qp_user with valid udata!
3807	*/
3808	static inline int ib_destroy_qp(struct ib_qp *qp)
3809	{
3810	return ib_destroy_qp_user(qp, NULL);
3811	}
3812
3813	/**
3814	* ib_open_qp - Obtain a reference to an existing sharable QP.
3815	* @xrcd - XRC domain
3816	* @qp_open_attr: Attributes identifying the QP to open.
3817	*
3818	* Returns a reference to a sharable QP.
3819	*/
3820	struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
3821	struct ib_qp_open_attr *qp_open_attr);
3822
3823	/**
3824	* ib_close_qp - Release an external reference to a QP.
3825	* @qp: The QP handle to release
3826	*
3827	* The opened QP handle is released by the caller. The underlying
3828	* shared QP is not destroyed until all internal references are released.
3829	*/
3830	int ib_close_qp(struct ib_qp *qp);
3831
3832	/**
3833	* ib_post_send - Posts a list of work requests to the send queue of
3834	* the specified QP.
3835	* @qp: The QP to post the work request on.
3836	* @send_wr: A list of work requests to post on the send queue.
3837	* @bad_send_wr: On an immediate failure, this parameter will reference
3838	* the work request that failed to be posted on the QP.
3839	*
3840	* While IBA Vol. 1 section 11.4.1.1 specifies that if an immediate
3841	* error is returned, the QP state shall not be affected,
3842	* ib_post_send() will return an immediate error after queueing any
3843	* earlier work requests in the list.
3844	*/
3845	static inline int ib_post_send(struct ib_qp *qp,
3846	const struct ib_send_wr *send_wr,
3847	const struct ib_send_wr **bad_send_wr)
3848	{
3849	const struct ib_send_wr *dummy;
3850
3851	return qp->device->ops.post_send(qp, send_wr, bad_send_wr ? : &dummy);
3852	}
3853
3854	/**
3855	* ib_post_recv - Posts a list of work requests to the receive queue of
3856	* the specified QP.
3857	* @qp: The QP to post the work request on.
3858	* @recv_wr: A list of work requests to post on the receive queue.
3859	* @bad_recv_wr: On an immediate failure, this parameter will reference
3860	* the work request that failed to be posted on the QP.
3861	*/
3862	static inline int ib_post_recv(struct ib_qp *qp,
3863	const struct ib_recv_wr *recv_wr,
3864	const struct ib_recv_wr **bad_recv_wr)
3865	{
3866	const struct ib_recv_wr *dummy;
3867
3868	return qp->device->ops.post_recv(qp, recv_wr, bad_recv_wr ? : &dummy);
3869	}
3870
3871	struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe,
3872	int comp_vector, enum ib_poll_context poll_ctx,
3873	const char *caller);
3874	static inline struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private,
3875	int nr_cqe, int comp_vector,
3876	enum ib_poll_context poll_ctx)
3877	{
3878	return __ib_alloc_cq(dev, private, nr_cqe, comp_vector, poll_ctx,
3879	KBUILD_MODNAME);
3880	}
3881
3882	struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private,
3883	int nr_cqe, enum ib_poll_context poll_ctx,
3884	const char *caller);
3885
3886	/**
3887	* ib_alloc_cq_any: Allocate kernel CQ
3888	* @dev: The IB device
3889	* @private: Private data attached to the CQE
3890	* @nr_cqe: Number of CQEs in the CQ
3891	* @poll_ctx: Context used for polling the CQ
3892	*/
3893	static inline struct ib_cq *ib_alloc_cq_any(struct ib_device *dev,
3894	void *private, int nr_cqe,
3895	enum ib_poll_context poll_ctx)
3896	{
3897	return __ib_alloc_cq_any(dev, private, nr_cqe, poll_ctx,
3898	KBUILD_MODNAME);
3899	}
3900
3901	void ib_free_cq(struct ib_cq *cq);
3902	int ib_process_cq_direct(struct ib_cq *cq, int budget);
3903
3904	/**
3905	* ib_create_cq - Creates a CQ on the specified device.
3906	* @device: The device on which to create the CQ.
3907	* @comp_handler: A user-specified callback that is invoked when a
3908	* completion event occurs on the CQ.
3909	* @event_handler: A user-specified callback that is invoked when an
3910	* asynchronous event not associated with a completion occurs on the CQ.
3911	* @cq_context: Context associated with the CQ returned to the user via
3912	* the associated completion and event handlers.
3913	* @cq_attr: The attributes the CQ should be created upon.
3914	*
3915	* Users can examine the cq structure to determine the actual CQ size.
3916	*/
3917	struct ib_cq *__ib_create_cq(struct ib_device *device,
3918	ib_comp_handler comp_handler,
3919	void (*event_handler)(struct ib_event *, void *),
3920	void *cq_context,
3921	const struct ib_cq_init_attr *cq_attr,
3922	const char *caller);
3923	#define ib_create_cq(device, cmp_hndlr, evt_hndlr, cq_ctxt, cq_attr) \
3924	__ib_create_cq((device), (cmp_hndlr), (evt_hndlr), (cq_ctxt), (cq_attr), KBUILD_MODNAME)
3925
3926	/**
3927	* ib_resize_cq - Modifies the capacity of the CQ.
3928	* @cq: The CQ to resize.
3929	* @cqe: The minimum size of the CQ.
3930	*
3931	* Users can examine the cq structure to determine the actual CQ size.
3932	*/
3933	int ib_resize_cq(struct ib_cq *cq, int cqe);
3934
3935	/**
3936	* rdma_set_cq_moderation - Modifies moderation params of the CQ
3937	* @cq: The CQ to modify.
3938	* @cq_count: number of CQEs that will trigger an event
3939	* @cq_period: max period of time in usec before triggering an event
3940	*
3941	*/
3942	int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period);
3943
3944	/**
3945	* ib_destroy_cq_user - Destroys the specified CQ.
3946	* @cq: The CQ to destroy.
3947	* @udata: Valid user data or NULL for kernel objects
3948	*/
3949	int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata);
3950
3951	/**
3952	* ib_destroy_cq - Destroys the specified kernel CQ.
3953	* @cq: The CQ to destroy.
3954	*
3955	* NOTE: for user cq use ib_destroy_cq_user with valid udata!
3956	*/
3957	static inline void ib_destroy_cq(struct ib_cq *cq)
3958	{
3959	int ret = ib_destroy_cq_user(cq, NULL);
3960
3961	WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail");
3962	}
3963
3964	/**
3965	* ib_poll_cq - poll a CQ for completion(s)
3966	* @cq:the CQ being polled
3967	* @num_entries:maximum number of completions to return
3968	* @wc:array of at least @num_entries &struct ib_wc where completions
3969	* will be returned
3970	*
3971	* Poll a CQ for (possibly multiple) completions. If the return value
3972	* is < 0, an error occurred. If the return value is >= 0, it is the
3973	* number of completions returned. If the return value is
3974	* non-negative and < num_entries, then the CQ was emptied.
3975	*/
3976	static inline int ib_poll_cq(struct ib_cq *cq, int num_entries,
3977	struct ib_wc *wc)
3978	{
3979	return cq->device->ops.poll_cq(cq, num_entries, wc);
3980	}
3981
3982	/**
3983	* ib_req_notify_cq - Request completion notification on a CQ.
3984	* @cq: The CQ to generate an event for.
3985	* @flags:
3986	* Must contain exactly one of %IB_CQ_SOLICITED or %IB_CQ_NEXT_COMP
3987	* to request an event on the next solicited event or next work
3988	* completion at any type, respectively. %IB_CQ_REPORT_MISSED_EVENTS
3989	* may also be |ed in to request a hint about missed events, as
3990	* described below.
3991	*
3992	* Return Value:
3993	* < 0 means an error occurred while requesting notification
3994	* == 0 means notification was requested successfully, and if
3995	* IB_CQ_REPORT_MISSED_EVENTS was passed in, then no events
3996	* were missed and it is safe to wait for another event. In
3997	* this case is it guaranteed that any work completions added
3998	* to the CQ since the last CQ poll will trigger a completion
3999	* notification event.
4000	* > 0 is only returned if IB_CQ_REPORT_MISSED_EVENTS was passed
4001	* in. It means that the consumer must poll the CQ again to
4002	* make sure it is empty to avoid missing an event because of a
4003	* race between requesting notification and an entry being
4004	* added to the CQ. This return value means it is possible
4005	* (but not guaranteed) that a work completion has been added
4006	* to the CQ since the last poll without triggering a
4007	* completion notification event.
4008	*/
4009	static inline int ib_req_notify_cq(struct ib_cq *cq,
4010	enum ib_cq_notify_flags flags)
4011	{
4012	return cq->device->ops.req_notify_cq(cq, flags);
4013	}
4014
4015	struct ib_cq *ib_cq_pool_get(struct ib_device *dev, unsigned int nr_cqe,
4016	int comp_vector_hint,
4017	enum ib_poll_context poll_ctx);
4018
4019	void ib_cq_pool_put(struct ib_cq *cq, unsigned int nr_cqe);
4020
4021	/*
4022	* Drivers that don't need a DMA mapping at the RDMA layer, set dma_device to
4023	* NULL. This causes the ib_dma* helpers to just stash the kernel virtual
4024	* address into the dma address.
4025	*/
4026	static inline bool ib_uses_virt_dma(struct ib_device *dev)
4027	{
4028	return IS_ENABLED(CONFIG_INFINIBAND_VIRT_DMA) && !dev->dma_device;
4029	}
4030
4031	/*
4032	* Check if a IB device's underlying DMA mapping supports P2PDMA transfers.
4033	*/
4034	static inline bool ib_dma_pci_p2p_dma_supported(struct ib_device *dev)
4035	{
4036	if (ib_uses_virt_dma(dev))
4037	return false;
4038
4039	return dma_pci_p2pdma_supported(dev: dev->dma_device);
4040	}
4041
4042	/**
4043	* ib_virt_dma_to_ptr - Convert a dma_addr to a kernel pointer
4044	* @dma_addr: The DMA address
4045	*
4046	* Used by ib_uses_virt_dma() devices to get back to the kernel pointer after
4047	* going through the dma_addr marshalling.
4048	*/
4049	static inline void *ib_virt_dma_to_ptr(u64 dma_addr)
4050	{
4051	/* virt_dma mode maps the kvs's directly into the dma addr */
4052	return (void *)(uintptr_t)dma_addr;
4053	}
4054
4055	/**
4056	* ib_virt_dma_to_page - Convert a dma_addr to a struct page
4057	* @dma_addr: The DMA address
4058	*
4059	* Used by ib_uses_virt_dma() device to get back to the struct page after going
4060	* through the dma_addr marshalling.
4061	*/
4062	static inline struct page *ib_virt_dma_to_page(u64 dma_addr)
4063	{
4064	return virt_to_page(ib_virt_dma_to_ptr(dma_addr));
4065	}
4066
4067	/**
4068	* ib_dma_mapping_error - check a DMA addr for error
4069	* @dev: The device for which the dma_addr was created
4070	* @dma_addr: The DMA address to check
4071	*/
4072	static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
4073	{
4074	if (ib_uses_virt_dma(dev))
4075	return 0;
4076	return dma_mapping_error(dev: dev->dma_device, dma_addr);
4077	}
4078
4079	/**
4080	* ib_dma_map_single - Map a kernel virtual address to DMA address
4081	* @dev: The device for which the dma_addr is to be created
4082	* @cpu_addr: The kernel virtual address
4083	* @size: The size of the region in bytes
4084	* @direction: The direction of the DMA
4085	*/
4086	static inline u64 ib_dma_map_single(struct ib_device *dev,
4087	void *cpu_addr, size_t size,
4088	enum dma_data_direction direction)
4089	{
4090	if (ib_uses_virt_dma(dev))
4091	return (uintptr_t)cpu_addr;
4092	return dma_map_single(dev->dma_device, cpu_addr, size, direction);
4093	}
4094
4095	/**
4096	* ib_dma_unmap_single - Destroy a mapping created by ib_dma_map_single()
4097	* @dev: The device for which the DMA address was created
4098	* @addr: The DMA address
4099	* @size: The size of the region in bytes
4100	* @direction: The direction of the DMA
4101	*/
4102	static inline void ib_dma_unmap_single(struct ib_device *dev,
4103	u64 addr, size_t size,
4104	enum dma_data_direction direction)
4105	{
4106	if (!ib_uses_virt_dma(dev))
4107	dma_unmap_single(dev->dma_device, addr, size, direction);
4108	}
4109
4110	/**
4111	* ib_dma_map_page - Map a physical page to DMA address
4112	* @dev: The device for which the dma_addr is to be created
4113	* @page: The page to be mapped
4114	* @offset: The offset within the page
4115	* @size: The size of the region in bytes
4116	* @direction: The direction of the DMA
4117	*/
4118	static inline u64 ib_dma_map_page(struct ib_device *dev,
4119	struct page *page,
4120	unsigned long offset,
4121	size_t size,
4122	enum dma_data_direction direction)
4123	{
4124	if (ib_uses_virt_dma(dev))
4125	return (uintptr_t)(page_address(page) + offset);
4126	return dma_map_page(dev->dma_device, page, offset, size, direction);
4127	}
4128
4129	/**
4130	* ib_dma_unmap_page - Destroy a mapping created by ib_dma_map_page()
4131	* @dev: The device for which the DMA address was created
4132	* @addr: The DMA address
4133	* @size: The size of the region in bytes
4134	* @direction: The direction of the DMA
4135	*/
4136	static inline void ib_dma_unmap_page(struct ib_device *dev,
4137	u64 addr, size_t size,
4138	enum dma_data_direction direction)
4139	{
4140	if (!ib_uses_virt_dma(dev))
4141	dma_unmap_page(dev->dma_device, addr, size, direction);
4142	}
4143
4144	int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents);
4145	static inline int ib_dma_map_sg_attrs(struct ib_device *dev,
4146	struct scatterlist *sg, int nents,
4147	enum dma_data_direction direction,
4148	unsigned long dma_attrs)
4149	{
4150	if (ib_uses_virt_dma(dev))
4151	return ib_dma_virt_map_sg(dev, sg, nents);
4152	return dma_map_sg_attrs(dev: dev->dma_device, sg, nents, dir: direction,
4153	attrs: dma_attrs);
4154	}
4155
4156	static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev,
4157	struct scatterlist *sg, int nents,
4158	enum dma_data_direction direction,
4159	unsigned long dma_attrs)
4160	{
4161	if (!ib_uses_virt_dma(dev))
4162	dma_unmap_sg_attrs(dev: dev->dma_device, sg, nents, dir: direction,
4163	attrs: dma_attrs);
4164	}
4165
4166	/**
4167	* ib_dma_map_sgtable_attrs - Map a scatter/gather table to DMA addresses
4168	* @dev: The device for which the DMA addresses are to be created
4169	* @sg: The sg_table object describing the buffer
4170	* @direction: The direction of the DMA
4171	* @attrs: Optional DMA attributes for the map operation
4172	*/
4173	static inline int ib_dma_map_sgtable_attrs(struct ib_device *dev,
4174	struct sg_table *sgt,
4175	enum dma_data_direction direction,
4176	unsigned long dma_attrs)
4177	{
4178	int nents;
4179
4180	if (ib_uses_virt_dma(dev)) {
4181	nents = ib_dma_virt_map_sg(dev, sg: sgt->sgl, nents: sgt->orig_nents);
4182	if (!nents)
4183	return -EIO;
4184	sgt->nents = nents;
4185	return 0;
4186	}
4187	return dma_map_sgtable(dev: dev->dma_device, sgt, dir: direction, attrs: dma_attrs);
4188	}
4189
4190	static inline void ib_dma_unmap_sgtable_attrs(struct ib_device *dev,
4191	struct sg_table *sgt,
4192	enum dma_data_direction direction,
4193	unsigned long dma_attrs)
4194	{
4195	if (!ib_uses_virt_dma(dev))
4196	dma_unmap_sgtable(dev: dev->dma_device, sgt, dir: direction, attrs: dma_attrs);
4197	}
4198
4199	/**
4200	* ib_dma_map_sg - Map a scatter/gather list to DMA addresses
4201	* @dev: The device for which the DMA addresses are to be created
4202	* @sg: The array of scatter/gather entries
4203	* @nents: The number of scatter/gather entries
4204	* @direction: The direction of the DMA
4205	*/
4206	static inline int ib_dma_map_sg(struct ib_device *dev,
4207	struct scatterlist *sg, int nents,
4208	enum dma_data_direction direction)
4209	{
4210	return ib_dma_map_sg_attrs(dev, sg, nents, direction, dma_attrs: 0);
4211	}
4212
4213	/**
4214	* ib_dma_unmap_sg - Unmap a scatter/gather list of DMA addresses
4215	* @dev: The device for which the DMA addresses were created
4216	* @sg: The array of scatter/gather entries
4217	* @nents: The number of scatter/gather entries
4218	* @direction: The direction of the DMA
4219	*/
4220	static inline void ib_dma_unmap_sg(struct ib_device *dev,
4221	struct scatterlist *sg, int nents,
4222	enum dma_data_direction direction)
4223	{
4224	ib_dma_unmap_sg_attrs(dev, sg, nents, direction, dma_attrs: 0);
4225	}
4226
4227	/**
4228	* ib_dma_max_seg_size - Return the size limit of a single DMA transfer
4229	* @dev: The device to query
4230	*
4231	* The returned value represents a size in bytes.
4232	*/
4233	static inline unsigned int ib_dma_max_seg_size(struct ib_device *dev)
4234	{
4235	if (ib_uses_virt_dma(dev))
4236	return UINT_MAX;
4237	return dma_get_max_seg_size(dev: dev->dma_device);
4238	}
4239
4240	/**
4241	* ib_dma_sync_single_for_cpu - Prepare DMA region to be accessed by CPU
4242	* @dev: The device for which the DMA address was created
4243	* @addr: The DMA address
4244	* @size: The size of the region in bytes
4245	* @dir: The direction of the DMA
4246	*/
4247	static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev,
4248	u64 addr,
4249	size_t size,
4250	enum dma_data_direction dir)
4251	{
4252	if (!ib_uses_virt_dma(dev))
4253	dma_sync_single_for_cpu(dev: dev->dma_device, addr, size, dir);
4254	}
4255
4256	/**
4257	* ib_dma_sync_single_for_device - Prepare DMA region to be accessed by device
4258	* @dev: The device for which the DMA address was created
4259	* @addr: The DMA address
4260	* @size: The size of the region in bytes
4261	* @dir: The direction of the DMA
4262	*/
4263	static inline void ib_dma_sync_single_for_device(struct ib_device *dev,
4264	u64 addr,
4265	size_t size,
4266	enum dma_data_direction dir)
4267	{
4268	if (!ib_uses_virt_dma(dev))
4269	dma_sync_single_for_device(dev: dev->dma_device, addr, size, dir);
4270	}
4271
4272	/* ib_reg_user_mr - register a memory region for virtual addresses from kernel
4273	* space. This function should be called when 'current' is the owning MM.
4274	*/
4275	struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
4276	u64 virt_addr, int mr_access_flags);
4277
4278	/* ib_advise_mr - give an advice about an address range in a memory region */
4279	int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
4280	u32 flags, struct ib_sge *sg_list, u32 num_sge);
4281	/**
4282	* ib_dereg_mr_user - Deregisters a memory region and removes it from the
4283	* HCA translation table.
4284	* @mr: The memory region to deregister.
4285	* @udata: Valid user data or NULL for kernel object
4286	*
4287	* This function can fail, if the memory region has memory windows bound to it.
4288	*/
4289	int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata);
4290
4291	/**
4292	* ib_dereg_mr - Deregisters a kernel memory region and removes it from the
4293	* HCA translation table.
4294	* @mr: The memory region to deregister.
4295	*
4296	* This function can fail, if the memory region has memory windows bound to it.
4297	*
4298	* NOTE: for user mr use ib_dereg_mr_user with valid udata!
4299	*/
4300	static inline int ib_dereg_mr(struct ib_mr *mr)
4301	{
4302	return ib_dereg_mr_user(mr, NULL);
4303	}
4304
4305	struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
4306	u32 max_num_sg);
4307
4308	struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
4309	u32 max_num_data_sg,
4310	u32 max_num_meta_sg);
4311
4312	/**
4313	* ib_update_fast_reg_key - updates the key portion of the fast_reg MR
4314	* R_Key and L_Key.
4315	* @mr - struct ib_mr pointer to be updated.
4316	* @newkey - new key to be used.
4317	*/
4318	static inline void ib_update_fast_reg_key(struct ib_mr *mr, u8 newkey)
4319	{
4320	mr->lkey = (mr->lkey & 0xffffff00) | newkey;
4321	mr->rkey = (mr->rkey & 0xffffff00) | newkey;
4322	}
4323
4324	/**
4325	* ib_inc_rkey - increments the key portion of the given rkey. Can be used
4326	* for calculating a new rkey for type 2 memory windows.
4327	* @rkey - the rkey to increment.
4328	*/
4329	static inline u32 ib_inc_rkey(u32 rkey)
4330	{
4331	const u32 mask = 0x000000ff;
4332	return ((rkey + 1) & mask) | (rkey & ~mask);
4333	}
4334
4335	/**
4336	* ib_attach_mcast - Attaches the specified QP to a multicast group.
4337	* @qp: QP to attach to the multicast group. The QP must be type
4338	* IB_QPT_UD.
4339	* @gid: Multicast group GID.
4340	* @lid: Multicast group LID in host byte order.
4341	*
4342	* In order to send and receive multicast packets, subnet
4343	* administration must have created the multicast group and configured
4344	* the fabric appropriately. The port associated with the specified
4345	* QP must also be a member of the multicast group.
4346	*/
4347	int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
4348
4349	/**
4350	* ib_detach_mcast - Detaches the specified QP from a multicast group.
4351	* @qp: QP to detach from the multicast group.
4352	* @gid: Multicast group GID.
4353	* @lid: Multicast group LID in host byte order.
4354	*/
4355	int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
4356
4357	struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,
4358	struct inode *inode, struct ib_udata *udata);
4359	int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata);
4360
4361	static inline int ib_check_mr_access(struct ib_device *ib_dev,
4362	unsigned int flags)
4363	{
4364	u64 device_cap = ib_dev->attrs.device_cap_flags;
4365
4366	/*
4367	* Local write permission is required if remote write or
4368	* remote atomic permission is also requested.
4369	*/
4370	if (flags & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE) &&
4371	!(flags & IB_ACCESS_LOCAL_WRITE))
4372	return -EINVAL;
4373
4374	if (flags & ~IB_ACCESS_SUPPORTED)
4375	return -EINVAL;
4376
4377	if (flags & IB_ACCESS_ON_DEMAND &&
4378	!(ib_dev->attrs.kernel_cap_flags & IBK_ON_DEMAND_PAGING))
4379	return -EOPNOTSUPP;
4380
4381	if ((flags & IB_ACCESS_FLUSH_GLOBAL &&
4382	!(device_cap & IB_DEVICE_FLUSH_GLOBAL)) ||
4383	(flags & IB_ACCESS_FLUSH_PERSISTENT &&
4384	!(device_cap & IB_DEVICE_FLUSH_PERSISTENT)))
4385	return -EOPNOTSUPP;
4386
4387	return 0;
4388	}
4389
4390	static inline bool ib_access_writable(int access_flags)
4391	{
4392	/*
4393	* We have writable memory backing the MR if any of the following
4394	* access flags are set. "Local write" and "remote write" obviously
4395	* require write access. "Remote atomic" can do things like fetch and
4396	* add, which will modify memory, and "MW bind" can change permissions
4397	* by binding a window.
4398	*/
4399	return access_flags &
4400	(IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE |
4401	IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_MW_BIND);
4402	}
4403
4404	/**
4405	* ib_check_mr_status: lightweight check of MR status.
4406	* This routine may provide status checks on a selected
4407	* ib_mr. first use is for signature status check.
4408	*
4409	* @mr: A memory region.
4410	* @check_mask: Bitmask of which checks to perform from
4411	* ib_mr_status_check enumeration.
4412	* @mr_status: The container of relevant status checks.
4413	* failed checks will be indicated in the status bitmask
4414	* and the relevant info shall be in the error item.
4415	*/
4416	int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
4417	struct ib_mr_status *mr_status);
4418
4419	/**
4420	* ib_device_try_get: Hold a registration lock
4421	* device: The device to lock
4422	*
4423	* A device under an active registration lock cannot become unregistered. It
4424	* is only possible to obtain a registration lock on a device that is fully
4425	* registered, otherwise this function returns false.
4426	*
4427	* The registration lock is only necessary for actions which require the
4428	* device to still be registered. Uses that only require the device pointer to
4429	* be valid should use get_device(&ibdev->dev) to hold the memory.
4430	*
4431	*/
4432	static inline bool ib_device_try_get(struct ib_device *dev)
4433	{
4434	return refcount_inc_not_zero(r: &dev->refcount);
4435	}
4436
4437	void ib_device_put(struct ib_device *device);
4438	struct ib_device *ib_device_get_by_netdev(struct net_device *ndev,
4439	enum rdma_driver_id driver_id);
4440	struct ib_device *ib_device_get_by_name(const char *name,
4441	enum rdma_driver_id driver_id);
4442	struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, u32 port,
4443	u16 pkey, const union ib_gid *gid,
4444	const struct sockaddr *addr);
4445	int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev,
4446	unsigned int port);
4447	struct ib_wq *ib_create_wq(struct ib_pd *pd,
4448	struct ib_wq_init_attr *init_attr);
4449	int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata);
4450
4451	int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
4452	unsigned int *sg_offset, unsigned int page_size);
4453	int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
4454	int data_sg_nents, unsigned int *data_sg_offset,
4455	struct scatterlist *meta_sg, int meta_sg_nents,
4456	unsigned int *meta_sg_offset, unsigned int page_size);
4457
4458	static inline int
4459	ib_map_mr_sg_zbva(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
4460	unsigned int *sg_offset, unsigned int page_size)
4461	{
4462	int n;
4463
4464	n = ib_map_mr_sg(mr, sg, sg_nents, sg_offset, page_size);
4465	mr->iova = 0;
4466
4467	return n;
4468	}
4469
4470	int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
4471	unsigned int *sg_offset, int (*set_page)(struct ib_mr *, u64));
4472
4473	void ib_drain_rq(struct ib_qp *qp);
4474	void ib_drain_sq(struct ib_qp *qp);
4475	void ib_drain_qp(struct ib_qp *qp);
4476
4477	int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed,
4478	u8 *width);
4479
4480	static inline u8 *rdma_ah_retrieve_dmac(struct rdma_ah_attr *attr)
4481	{
4482	if (attr->type == RDMA_AH_ATTR_TYPE_ROCE)
4483	return attr->roce.dmac;
4484	return NULL;
4485	}
4486
4487	static inline void rdma_ah_set_dlid(struct rdma_ah_attr *attr, u32 dlid)
4488	{
4489	if (attr->type == RDMA_AH_ATTR_TYPE_IB)
4490	attr->ib.dlid = (u16)dlid;
4491	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4492	attr->opa.dlid = dlid;
4493	}
4494
4495	static inline u32 rdma_ah_get_dlid(const struct rdma_ah_attr *attr)
4496	{
4497	if (attr->type == RDMA_AH_ATTR_TYPE_IB)
4498	return attr->ib.dlid;
4499	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4500	return attr->opa.dlid;
4501	return 0;
4502	}
4503
4504	static inline void rdma_ah_set_sl(struct rdma_ah_attr *attr, u8 sl)
4505	{
4506	attr->sl = sl;
4507	}
4508
4509	static inline u8 rdma_ah_get_sl(const struct rdma_ah_attr *attr)
4510	{
4511	return attr->sl;
4512	}
4513
4514	static inline void rdma_ah_set_path_bits(struct rdma_ah_attr *attr,
4515	u8 src_path_bits)
4516	{
4517	if (attr->type == RDMA_AH_ATTR_TYPE_IB)
4518	attr->ib.src_path_bits = src_path_bits;
4519	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4520	attr->opa.src_path_bits = src_path_bits;
4521	}
4522
4523	static inline u8 rdma_ah_get_path_bits(const struct rdma_ah_attr *attr)
4524	{
4525	if (attr->type == RDMA_AH_ATTR_TYPE_IB)
4526	return attr->ib.src_path_bits;
4527	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4528	return attr->opa.src_path_bits;
4529	return 0;
4530	}
4531
4532	static inline void rdma_ah_set_make_grd(struct rdma_ah_attr *attr,
4533	bool make_grd)
4534	{
4535	if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4536	attr->opa.make_grd = make_grd;
4537	}
4538
4539	static inline bool rdma_ah_get_make_grd(const struct rdma_ah_attr *attr)
4540	{
4541	if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4542	return attr->opa.make_grd;
4543	return false;
4544	}
4545
4546	static inline void rdma_ah_set_port_num(struct rdma_ah_attr *attr, u32 port_num)
4547	{
4548	attr->port_num = port_num;
4549	}
4550
4551	static inline u32 rdma_ah_get_port_num(const struct rdma_ah_attr *attr)
4552	{
4553	return attr->port_num;
4554	}
4555
4556	static inline void rdma_ah_set_static_rate(struct rdma_ah_attr *attr,
4557	u8 static_rate)
4558	{
4559	attr->static_rate = static_rate;
4560	}
4561
4562	static inline u8 rdma_ah_get_static_rate(const struct rdma_ah_attr *attr)
4563	{
4564	return attr->static_rate;
4565	}
4566
4567	static inline void rdma_ah_set_ah_flags(struct rdma_ah_attr *attr,
4568	enum ib_ah_flags flag)
4569	{
4570	attr->ah_flags = flag;
4571	}
4572
4573	static inline enum ib_ah_flags
4574	rdma_ah_get_ah_flags(const struct rdma_ah_attr *attr)
4575	{
4576	return attr->ah_flags;
4577	}
4578
4579	static inline const struct ib_global_route
4580	*rdma_ah_read_grh(const struct rdma_ah_attr *attr)
4581	{
4582	return &attr->grh;
4583	}
4584
4585	/*To retrieve and modify the grh */
4586	static inline struct ib_global_route
4587	*rdma_ah_retrieve_grh(struct rdma_ah_attr *attr)
4588	{
4589	return &attr->grh;
4590	}
4591
4592	static inline void rdma_ah_set_dgid_raw(struct rdma_ah_attr *attr, void *dgid)
4593	{
4594	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
4595
4596	memcpy(grh->dgid.raw, dgid, sizeof(grh->dgid));
4597	}
4598
4599	static inline void rdma_ah_set_subnet_prefix(struct rdma_ah_attr *attr,
4600	__be64 prefix)
4601	{
4602	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
4603
4604	grh->dgid.global.subnet_prefix = prefix;
4605	}
4606
4607	static inline void rdma_ah_set_interface_id(struct rdma_ah_attr *attr,
4608	__be64 if_id)
4609	{
4610	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
4611
4612	grh->dgid.global.interface_id = if_id;
4613	}
4614
4615	static inline void rdma_ah_set_grh(struct rdma_ah_attr *attr,
4616	union ib_gid *dgid, u32 flow_label,
4617	u8 sgid_index, u8 hop_limit,
4618	u8 traffic_class)
4619	{
4620	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
4621
4622	attr->ah_flags = IB_AH_GRH;
4623	if (dgid)
4624	grh->dgid = *dgid;
4625	grh->flow_label = flow_label;
4626	grh->sgid_index = sgid_index;
4627	grh->hop_limit = hop_limit;
4628	grh->traffic_class = traffic_class;
4629	grh->sgid_attr = NULL;
4630	}
4631
4632	void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr);
4633	void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid,
4634	u32 flow_label, u8 hop_limit, u8 traffic_class,
4635	const struct ib_gid_attr *sgid_attr);
4636	void rdma_copy_ah_attr(struct rdma_ah_attr *dest,
4637	const struct rdma_ah_attr *src);
4638	void rdma_replace_ah_attr(struct rdma_ah_attr *old,
4639	const struct rdma_ah_attr *new);
4640	void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src);
4641
4642	/**
4643	* rdma_ah_find_type - Return address handle type.
4644	*
4645	* @dev: Device to be checked
4646	* @port_num: Port number
4647	*/
4648	static inline enum rdma_ah_attr_type rdma_ah_find_type(struct ib_device *dev,
4649	u32 port_num)
4650	{
4651	if (rdma_protocol_roce(device: dev, port_num))
4652	return RDMA_AH_ATTR_TYPE_ROCE;
4653	if (rdma_protocol_ib(device: dev, port_num)) {
4654	if (rdma_cap_opa_ah(device: dev, port_num))
4655	return RDMA_AH_ATTR_TYPE_OPA;
4656	return RDMA_AH_ATTR_TYPE_IB;
4657	}
4658
4659	return RDMA_AH_ATTR_TYPE_UNDEFINED;
4660	}
4661
4662	/**
4663	* ib_lid_cpu16 - Return lid in 16bit CPU encoding.
4664	* In the current implementation the only way to
4665	* get the 32bit lid is from other sources for OPA.
4666	* For IB, lids will always be 16bits so cast the
4667	* value accordingly.
4668	*
4669	* @lid: A 32bit LID
4670	*/
4671	static inline u16 ib_lid_cpu16(u32 lid)
4672	{
4673	WARN_ON_ONCE(lid & 0xFFFF0000);
4674	return (u16)lid;
4675	}
4676
4677	/**
4678	* ib_lid_be16 - Return lid in 16bit BE encoding.
4679	*
4680	* @lid: A 32bit LID
4681	*/
4682	static inline __be16 ib_lid_be16(u32 lid)
4683	{
4684	WARN_ON_ONCE(lid & 0xFFFF0000);
4685	return cpu_to_be16((u16)lid);
4686	}
4687
4688	/**
4689	* ib_get_vector_affinity - Get the affinity mappings of a given completion
4690	* vector
4691	* @device: the rdma device
4692	* @comp_vector: index of completion vector
4693	*
4694	* Returns NULL on failure, otherwise a corresponding cpu map of the
4695	* completion vector (returns all-cpus map if the device driver doesn't
4696	* implement get_vector_affinity).
4697	*/
4698	static inline const struct cpumask *
4699	ib_get_vector_affinity(struct ib_device *device, int comp_vector)
4700	{
4701	if (comp_vector < 0 || comp_vector >= device->num_comp_vectors ||
4702	!device->ops.get_vector_affinity)
4703	return NULL;
4704
4705	return device->ops.get_vector_affinity(device, comp_vector);
4706
4707	}
4708
4709	/**
4710	* rdma_roce_rescan_device - Rescan all of the network devices in the system
4711	* and add their gids, as needed, to the relevant RoCE devices.
4712	*
4713	* @device: the rdma device
4714	*/
4715	void rdma_roce_rescan_device(struct ib_device *ibdev);
4716
4717	struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile);
4718
4719	int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs);
4720
4721	struct net_device *rdma_alloc_netdev(struct ib_device *device, u32 port_num,
4722	enum rdma_netdev_t type, const char *name,
4723	unsigned char name_assign_type,
4724	void (*setup)(struct net_device *));
4725
4726	int rdma_init_netdev(struct ib_device *device, u32 port_num,
4727	enum rdma_netdev_t type, const char *name,
4728	unsigned char name_assign_type,
4729	void (*setup)(struct net_device *),
4730	struct net_device *netdev);
4731
4732	/**
4733	* rdma_device_to_ibdev - Get ib_device pointer from device pointer
4734	*
4735	* @device: device pointer for which ib_device pointer to retrieve
4736	*
4737	* rdma_device_to_ibdev() retrieves ib_device pointer from device.
4738	*
4739	*/
4740	static inline struct ib_device *rdma_device_to_ibdev(struct device *device)
4741	{
4742	struct ib_core_device *coredev =
4743	container_of(device, struct ib_core_device, dev);
4744
4745	return coredev->owner;
4746	}
4747
4748	/**
4749	* ibdev_to_node - return the NUMA node for a given ib_device
4750	* @dev: device to get the NUMA node for.
4751	*/
4752	static inline int ibdev_to_node(struct ib_device *ibdev)
4753	{
4754	struct device *parent = ibdev->dev.parent;
4755
4756	if (!parent)
4757	return NUMA_NO_NODE;
4758	return dev_to_node(dev: parent);
4759	}
4760
4761	/**
4762	* rdma_device_to_drv_device - Helper macro to reach back to driver's
4763	* ib_device holder structure from device pointer.
4764	*
4765	* NOTE: New drivers should not make use of this API; This API is only for
4766	* existing drivers who have exposed sysfs entries using
4767	* ops->device_group.
4768	*/
4769	#define rdma_device_to_drv_device(dev, drv_dev_struct, ibdev_member) \
4770	container_of(rdma_device_to_ibdev(dev), drv_dev_struct, ibdev_member)
4771
4772	bool rdma_dev_access_netns(const struct ib_device *device,
4773	const struct net *net);
4774
4775	#define IB_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000)
4776	#define IB_ROCE_UDP_ENCAP_VALID_PORT_MAX (0xFFFF)
4777	#define IB_GRH_FLOWLABEL_MASK (0x000FFFFF)
4778
4779	/**
4780	* rdma_flow_label_to_udp_sport - generate a RoCE v2 UDP src port value based
4781	* on the flow_label
4782	*
4783	* This function will convert the 20 bit flow_label input to a valid RoCE v2
4784	* UDP src port 14 bit value. All RoCE V2 drivers should use this same
4785	* convention.
4786	*/
4787	static inline u16 rdma_flow_label_to_udp_sport(u32 fl)
4788	{
4789	u32 fl_low = fl & 0x03fff, fl_high = fl & 0xFC000;
4790
4791	fl_low ^= fl_high >> 14;
4792	return (u16)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN);
4793	}
4794
4795	/**
4796	* rdma_calc_flow_label - generate a RDMA symmetric flow label value based on
4797	* local and remote qpn values
4798	*
4799	* This function folded the multiplication results of two qpns, 24 bit each,
4800	* fields, and converts it to a 20 bit results.
4801	*
4802	* This function will create symmetric flow_label value based on the local
4803	* and remote qpn values. this will allow both the requester and responder
4804	* to calculate the same flow_label for a given connection.
4805	*
4806	* This helper function should be used by driver in case the upper layer
4807	* provide a zero flow_label value. This is to improve entropy of RDMA
4808	* traffic in the network.
4809	*/
4810	static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn)
4811	{
4812	u64 v = (u64)lqpn * rqpn;
4813
4814	v ^= v >> 20;
4815	v ^= v >> 40;
4816
4817	return (u32)(v & IB_GRH_FLOWLABEL_MASK);
4818	}
4819
4820	/**
4821	* rdma_get_udp_sport - Calculate and set UDP source port based on the flow
4822	* label. If flow label is not defined in GRH then
4823	* calculate it based on lqpn/rqpn.
4824	*
4825	* @fl: flow label from GRH
4826	* @lqpn: local qp number
4827	* @rqpn: remote qp number
4828	*/
4829	static inline u16 rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
4830	{
4831	if (!fl)
4832	fl = rdma_calc_flow_label(lqpn, rqpn);
4833
4834	return rdma_flow_label_to_udp_sport(fl);
4835	}
4836
4837	const struct ib_port_immutable*
4838	ib_port_immutable_read(struct ib_device *dev, unsigned int port);
4839	#endif /* IB_VERBS_H */
4840