1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2/*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7#ifndef RXE_VERBS_H
8#define RXE_VERBS_H
9
10#include <linux/interrupt.h>
11#include <linux/workqueue.h>
12#include "rxe_pool.h"
13#include "rxe_task.h"
14#include "rxe_hw_counters.h"
15
16static inline int pkey_match(u16 key1, u16 key2)
17{
18 return (((key1 & 0x7fff) != 0) &&
19 ((key1 & 0x7fff) == (key2 & 0x7fff)) &&
20 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0;
21}
22
23/* Return >0 if psn_a > psn_b
24 * 0 if psn_a == psn_b
25 * <0 if psn_a < psn_b
26 */
27static inline int psn_compare(u32 psn_a, u32 psn_b)
28{
29 s32 diff;
30
31 diff = (psn_a - psn_b) << 8;
32 return diff;
33}
34
35struct rxe_ucontext {
36 struct ib_ucontext ibuc;
37 struct rxe_pool_elem elem;
38};
39
40struct rxe_pd {
41 struct ib_pd ibpd;
42 struct rxe_pool_elem elem;
43};
44
45struct rxe_ah {
46 struct ib_ah ibah;
47 struct rxe_pool_elem elem;
48 struct rxe_av av;
49 bool is_user;
50 int ah_num;
51};
52
53struct rxe_cqe {
54 union {
55 struct ib_wc ibwc;
56 struct ib_uverbs_wc uibwc;
57 };
58};
59
60struct rxe_cq {
61 struct ib_cq ibcq;
62 struct rxe_pool_elem elem;
63 struct rxe_queue *queue;
64 spinlock_t cq_lock;
65 u8 notify;
66 bool is_user;
67 atomic_t num_wq;
68};
69
70enum wqe_state {
71 wqe_state_posted,
72 wqe_state_processing,
73 wqe_state_pending,
74 wqe_state_done,
75 wqe_state_error,
76};
77
78struct rxe_sq {
79 int max_wr;
80 int max_sge;
81 int max_inline;
82 spinlock_t sq_lock; /* guard queue */
83 struct rxe_queue *queue;
84};
85
86struct rxe_rq {
87 int max_wr;
88 int max_sge;
89 spinlock_t producer_lock; /* guard queue producer */
90 spinlock_t consumer_lock; /* guard queue consumer */
91 struct rxe_queue *queue;
92};
93
94struct rxe_srq {
95 struct ib_srq ibsrq;
96 struct rxe_pool_elem elem;
97 struct rxe_pd *pd;
98 struct rxe_rq rq;
99 u32 srq_num;
100
101 int limit;
102 int error;
103};
104
105struct rxe_req_info {
106 int wqe_index;
107 u32 psn;
108 int opcode;
109 atomic_t rd_atomic;
110 int wait_fence;
111 int need_rd_atomic;
112 int wait_psn;
113 int need_retry;
114 int wait_for_rnr_timer;
115 int noack_pkts;
116 struct rxe_task task;
117};
118
119struct rxe_comp_info {
120 u32 psn;
121 int opcode;
122 int timeout;
123 int timeout_retry;
124 int started_retry;
125 u32 retry_cnt;
126 u32 rnr_retry;
127 struct rxe_task task;
128};
129
130enum rdatm_res_state {
131 rdatm_res_state_next,
132 rdatm_res_state_new,
133 rdatm_res_state_replay,
134};
135
136struct resp_res {
137 int type;
138 int replay;
139 u32 first_psn;
140 u32 last_psn;
141 u32 cur_psn;
142 enum rdatm_res_state state;
143
144 union {
145 struct {
146 u64 orig_val;
147 } atomic;
148 struct {
149 u64 va_org;
150 u32 rkey;
151 u32 length;
152 u64 va;
153 u32 resid;
154 } read;
155 struct {
156 u32 length;
157 u64 va;
158 u8 type;
159 u8 level;
160 } flush;
161 };
162};
163
164struct rxe_resp_info {
165 u32 msn;
166 u32 psn;
167 u32 ack_psn;
168 int opcode;
169 int drop_msg;
170 int goto_error;
171 int sent_psn_nak;
172 enum ib_wc_status status;
173 u8 aeth_syndrome;
174
175 /* Receive only */
176 struct rxe_recv_wqe *wqe;
177
178 /* RDMA read / atomic only */
179 u64 va;
180 u64 offset;
181 struct rxe_mr *mr;
182 u32 resid;
183 u32 rkey;
184 u32 length;
185
186 /* SRQ only */
187 struct {
188 struct rxe_recv_wqe wqe;
189 struct ib_sge sge[RXE_MAX_SGE];
190 } srq_wqe;
191
192 /* Responder resources. It's a circular list where the oldest
193 * resource is dropped first.
194 */
195 struct resp_res *resources;
196 unsigned int res_head;
197 unsigned int res_tail;
198 struct resp_res *res;
199 struct rxe_task task;
200};
201
202struct rxe_qp {
203 struct ib_qp ibqp;
204 struct rxe_pool_elem elem;
205 struct ib_qp_attr attr;
206 unsigned int valid;
207 unsigned int mtu;
208 bool is_user;
209
210 struct rxe_pd *pd;
211 struct rxe_srq *srq;
212 struct rxe_cq *scq;
213 struct rxe_cq *rcq;
214
215 enum ib_sig_type sq_sig_type;
216
217 struct rxe_sq sq;
218 struct rxe_rq rq;
219
220 struct socket *sk;
221 u32 dst_cookie;
222 u16 src_port;
223
224 struct rxe_av pri_av;
225 struct rxe_av alt_av;
226
227 atomic_t mcg_num;
228
229 struct sk_buff_head req_pkts;
230 struct sk_buff_head resp_pkts;
231
232 struct rxe_req_info req;
233 struct rxe_comp_info comp;
234 struct rxe_resp_info resp;
235
236 atomic_t ssn;
237 atomic_t skb_out;
238 int need_req_skb;
239
240 /* Timer for retranmitting packet when ACKs have been lost. RC
241 * only. The requester sets it when it is not already
242 * started. The responder resets it whenever an ack is
243 * received.
244 */
245 struct timer_list retrans_timer;
246 u64 qp_timeout_jiffies;
247
248 /* Timer for handling RNR NAKS. */
249 struct timer_list rnr_nak_timer;
250
251 spinlock_t state_lock; /* guard requester and completer */
252
253 struct execute_work cleanup_work;
254};
255
256enum {
257 RXE_ACCESS_REMOTE = IB_ACCESS_REMOTE_READ
258 | IB_ACCESS_REMOTE_WRITE
259 | IB_ACCESS_REMOTE_ATOMIC,
260 RXE_ACCESS_SUPPORTED_MR = RXE_ACCESS_REMOTE
261 | IB_ACCESS_LOCAL_WRITE
262 | IB_ACCESS_MW_BIND
263 | IB_ACCESS_ON_DEMAND
264 | IB_ACCESS_FLUSH_GLOBAL
265 | IB_ACCESS_FLUSH_PERSISTENT
266 | IB_ACCESS_OPTIONAL,
267 RXE_ACCESS_SUPPORTED_QP = RXE_ACCESS_SUPPORTED_MR,
268 RXE_ACCESS_SUPPORTED_MW = RXE_ACCESS_SUPPORTED_MR
269 | IB_ZERO_BASED,
270};
271
272enum rxe_mr_state {
273 RXE_MR_STATE_INVALID,
274 RXE_MR_STATE_FREE,
275 RXE_MR_STATE_VALID,
276};
277
278enum rxe_mr_copy_dir {
279 RXE_TO_MR_OBJ,
280 RXE_FROM_MR_OBJ,
281};
282
283enum rxe_mr_lookup_type {
284 RXE_LOOKUP_LOCAL,
285 RXE_LOOKUP_REMOTE,
286};
287
288enum rxe_rereg {
289 RXE_MR_REREG_SUPPORTED = IB_MR_REREG_PD
290 | IB_MR_REREG_ACCESS,
291};
292
293static inline int rkey_is_mw(u32 rkey)
294{
295 u32 index = rkey >> 8;
296
297 return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX);
298}
299
300struct rxe_mr {
301 struct rxe_pool_elem elem;
302 struct ib_mr ibmr;
303
304 struct ib_umem *umem;
305
306 u32 lkey;
307 u32 rkey;
308 enum rxe_mr_state state;
309 int access;
310 atomic_t num_mw;
311
312 unsigned int page_offset;
313 unsigned int page_shift;
314 u64 page_mask;
315
316 u32 num_buf;
317 u32 nbuf;
318
319 struct xarray page_list;
320};
321
322static inline unsigned int mr_page_size(struct rxe_mr *mr)
323{
324 return mr ? mr->ibmr.page_size : PAGE_SIZE;
325}
326
327enum rxe_mw_state {
328 RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID,
329 RXE_MW_STATE_FREE = RXE_MR_STATE_FREE,
330 RXE_MW_STATE_VALID = RXE_MR_STATE_VALID,
331};
332
333struct rxe_mw {
334 struct ib_mw ibmw;
335 struct rxe_pool_elem elem;
336 spinlock_t lock;
337 enum rxe_mw_state state;
338 struct rxe_qp *qp; /* Type 2 only */
339 struct rxe_mr *mr;
340 u32 rkey;
341 int access;
342 u64 addr;
343 u64 length;
344};
345
346struct rxe_mcg {
347 struct rb_node node;
348 struct kref ref_cnt;
349 struct rxe_dev *rxe;
350 struct list_head qp_list;
351 union ib_gid mgid;
352 atomic_t qp_num;
353 u32 qkey;
354 u16 pkey;
355};
356
357struct rxe_mca {
358 struct list_head qp_list;
359 struct rxe_qp *qp;
360};
361
362struct rxe_port {
363 struct ib_port_attr attr;
364 __be64 port_guid;
365 __be64 subnet_prefix;
366 spinlock_t port_lock; /* guard port */
367 unsigned int mtu_cap;
368 /* special QPs */
369 u32 qp_gsi_index;
370};
371
372struct rxe_dev {
373 struct ib_device ib_dev;
374 struct ib_device_attr attr;
375 int max_ucontext;
376 int max_inline_data;
377 struct mutex usdev_lock;
378
379 struct net_device *ndev;
380
381 struct rxe_pool uc_pool;
382 struct rxe_pool pd_pool;
383 struct rxe_pool ah_pool;
384 struct rxe_pool srq_pool;
385 struct rxe_pool qp_pool;
386 struct rxe_pool cq_pool;
387 struct rxe_pool mr_pool;
388 struct rxe_pool mw_pool;
389
390 /* multicast support */
391 spinlock_t mcg_lock;
392 struct rb_root mcg_tree;
393 atomic_t mcg_num;
394 atomic_t mcg_attach;
395
396 spinlock_t pending_lock; /* guard pending_mmaps */
397 struct list_head pending_mmaps;
398
399 spinlock_t mmap_offset_lock; /* guard mmap_offset */
400 u64 mmap_offset;
401
402 atomic64_t stats_counters[RXE_NUM_OF_COUNTERS];
403
404 struct rxe_port port;
405 struct crypto_shash *tfm;
406};
407
408static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
409{
410 atomic64_inc(v: &rxe->stats_counters[index]);
411}
412
413static inline struct rxe_dev *to_rdev(struct ib_device *dev)
414{
415 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
416}
417
418static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc)
419{
420 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL;
421}
422
423static inline struct rxe_pd *to_rpd(struct ib_pd *pd)
424{
425 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;
426}
427
428static inline struct rxe_ah *to_rah(struct ib_ah *ah)
429{
430 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;
431}
432
433static inline struct rxe_srq *to_rsrq(struct ib_srq *srq)
434{
435 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL;
436}
437
438static inline struct rxe_qp *to_rqp(struct ib_qp *qp)
439{
440 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL;
441}
442
443static inline struct rxe_cq *to_rcq(struct ib_cq *cq)
444{
445 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL;
446}
447
448static inline struct rxe_mr *to_rmr(struct ib_mr *mr)
449{
450 return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL;
451}
452
453static inline struct rxe_mw *to_rmw(struct ib_mw *mw)
454{
455 return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL;
456}
457
458static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah)
459{
460 return to_rpd(pd: ah->ibah.pd);
461}
462
463static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)
464{
465 return to_rpd(pd: mr->ibmr.pd);
466}
467
468static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
469{
470 return to_rpd(pd: mw->ibmw.pd);
471}
472
473int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
474
475#endif /* RXE_VERBS_H */
476

source code of linux/drivers/infiniband/sw/rxe/rxe_verbs.h