1 | /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ |
2 | /* |
3 | * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. |
4 | * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. |
5 | */ |
6 | |
7 | #ifndef RXE_VERBS_H |
8 | #define RXE_VERBS_H |
9 | |
10 | #include <linux/interrupt.h> |
11 | #include <linux/workqueue.h> |
12 | #include "rxe_pool.h" |
13 | #include "rxe_task.h" |
14 | #include "rxe_hw_counters.h" |
15 | |
16 | static inline int pkey_match(u16 key1, u16 key2) |
17 | { |
18 | return (((key1 & 0x7fff) != 0) && |
19 | ((key1 & 0x7fff) == (key2 & 0x7fff)) && |
20 | ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0; |
21 | } |
22 | |
23 | /* Return >0 if psn_a > psn_b |
24 | * 0 if psn_a == psn_b |
25 | * <0 if psn_a < psn_b |
26 | */ |
27 | static inline int psn_compare(u32 psn_a, u32 psn_b) |
28 | { |
29 | s32 diff; |
30 | |
31 | diff = (psn_a - psn_b) << 8; |
32 | return diff; |
33 | } |
34 | |
35 | struct rxe_ucontext { |
36 | struct ib_ucontext ibuc; |
37 | struct rxe_pool_elem elem; |
38 | }; |
39 | |
40 | struct rxe_pd { |
41 | struct ib_pd ibpd; |
42 | struct rxe_pool_elem elem; |
43 | }; |
44 | |
45 | struct rxe_ah { |
46 | struct ib_ah ibah; |
47 | struct rxe_pool_elem elem; |
48 | struct rxe_av av; |
49 | bool is_user; |
50 | int ah_num; |
51 | }; |
52 | |
53 | struct rxe_cqe { |
54 | union { |
55 | struct ib_wc ibwc; |
56 | struct ib_uverbs_wc uibwc; |
57 | }; |
58 | }; |
59 | |
60 | struct rxe_cq { |
61 | struct ib_cq ibcq; |
62 | struct rxe_pool_elem elem; |
63 | struct rxe_queue *queue; |
64 | spinlock_t cq_lock; |
65 | u8 notify; |
66 | bool is_user; |
67 | atomic_t num_wq; |
68 | }; |
69 | |
70 | enum wqe_state { |
71 | wqe_state_posted, |
72 | wqe_state_processing, |
73 | wqe_state_pending, |
74 | wqe_state_done, |
75 | wqe_state_error, |
76 | }; |
77 | |
78 | struct rxe_sq { |
79 | int max_wr; |
80 | int max_sge; |
81 | int max_inline; |
82 | spinlock_t sq_lock; /* guard queue */ |
83 | struct rxe_queue *queue; |
84 | }; |
85 | |
86 | struct rxe_rq { |
87 | int max_wr; |
88 | int max_sge; |
89 | spinlock_t producer_lock; /* guard queue producer */ |
90 | spinlock_t consumer_lock; /* guard queue consumer */ |
91 | struct rxe_queue *queue; |
92 | }; |
93 | |
94 | struct rxe_srq { |
95 | struct ib_srq ibsrq; |
96 | struct rxe_pool_elem elem; |
97 | struct rxe_pd *pd; |
98 | struct rxe_rq rq; |
99 | u32 srq_num; |
100 | |
101 | int limit; |
102 | int error; |
103 | }; |
104 | |
105 | struct rxe_req_info { |
106 | int wqe_index; |
107 | u32 psn; |
108 | int opcode; |
109 | atomic_t rd_atomic; |
110 | int wait_fence; |
111 | int need_rd_atomic; |
112 | int wait_psn; |
113 | int need_retry; |
114 | int wait_for_rnr_timer; |
115 | int noack_pkts; |
116 | struct rxe_task task; |
117 | }; |
118 | |
119 | struct rxe_comp_info { |
120 | u32 psn; |
121 | int opcode; |
122 | int timeout; |
123 | int timeout_retry; |
124 | int started_retry; |
125 | u32 retry_cnt; |
126 | u32 rnr_retry; |
127 | struct rxe_task task; |
128 | }; |
129 | |
130 | enum rdatm_res_state { |
131 | rdatm_res_state_next, |
132 | rdatm_res_state_new, |
133 | rdatm_res_state_replay, |
134 | }; |
135 | |
136 | struct resp_res { |
137 | int type; |
138 | int replay; |
139 | u32 first_psn; |
140 | u32 last_psn; |
141 | u32 cur_psn; |
142 | enum rdatm_res_state state; |
143 | |
144 | union { |
145 | struct { |
146 | u64 orig_val; |
147 | } atomic; |
148 | struct { |
149 | u64 va_org; |
150 | u32 rkey; |
151 | u32 length; |
152 | u64 va; |
153 | u32 resid; |
154 | } read; |
155 | struct { |
156 | u32 length; |
157 | u64 va; |
158 | u8 type; |
159 | u8 level; |
160 | } flush; |
161 | }; |
162 | }; |
163 | |
164 | struct rxe_resp_info { |
165 | u32 msn; |
166 | u32 psn; |
167 | u32 ack_psn; |
168 | int opcode; |
169 | int drop_msg; |
170 | int goto_error; |
171 | int sent_psn_nak; |
172 | enum ib_wc_status status; |
173 | u8 aeth_syndrome; |
174 | |
175 | /* Receive only */ |
176 | struct rxe_recv_wqe *wqe; |
177 | |
178 | /* RDMA read / atomic only */ |
179 | u64 va; |
180 | u64 offset; |
181 | struct rxe_mr *mr; |
182 | u32 resid; |
183 | u32 rkey; |
184 | u32 length; |
185 | |
186 | /* SRQ only */ |
187 | struct { |
188 | struct rxe_recv_wqe wqe; |
189 | struct ib_sge sge[RXE_MAX_SGE]; |
190 | } srq_wqe; |
191 | |
192 | /* Responder resources. It's a circular list where the oldest |
193 | * resource is dropped first. |
194 | */ |
195 | struct resp_res *resources; |
196 | unsigned int res_head; |
197 | unsigned int res_tail; |
198 | struct resp_res *res; |
199 | struct rxe_task task; |
200 | }; |
201 | |
202 | struct rxe_qp { |
203 | struct ib_qp ibqp; |
204 | struct rxe_pool_elem elem; |
205 | struct ib_qp_attr attr; |
206 | unsigned int valid; |
207 | unsigned int mtu; |
208 | bool is_user; |
209 | |
210 | struct rxe_pd *pd; |
211 | struct rxe_srq *srq; |
212 | struct rxe_cq *scq; |
213 | struct rxe_cq *rcq; |
214 | |
215 | enum ib_sig_type sq_sig_type; |
216 | |
217 | struct rxe_sq sq; |
218 | struct rxe_rq rq; |
219 | |
220 | struct socket *sk; |
221 | u32 dst_cookie; |
222 | u16 src_port; |
223 | |
224 | struct rxe_av pri_av; |
225 | struct rxe_av alt_av; |
226 | |
227 | atomic_t mcg_num; |
228 | |
229 | struct sk_buff_head req_pkts; |
230 | struct sk_buff_head resp_pkts; |
231 | |
232 | struct rxe_req_info req; |
233 | struct rxe_comp_info comp; |
234 | struct rxe_resp_info resp; |
235 | |
236 | atomic_t ssn; |
237 | atomic_t skb_out; |
238 | int need_req_skb; |
239 | |
240 | /* Timer for retranmitting packet when ACKs have been lost. RC |
241 | * only. The requester sets it when it is not already |
242 | * started. The responder resets it whenever an ack is |
243 | * received. |
244 | */ |
245 | struct timer_list retrans_timer; |
246 | u64 qp_timeout_jiffies; |
247 | |
248 | /* Timer for handling RNR NAKS. */ |
249 | struct timer_list rnr_nak_timer; |
250 | |
251 | spinlock_t state_lock; /* guard requester and completer */ |
252 | |
253 | struct execute_work cleanup_work; |
254 | }; |
255 | |
256 | enum { |
257 | RXE_ACCESS_REMOTE = IB_ACCESS_REMOTE_READ |
258 | | IB_ACCESS_REMOTE_WRITE |
259 | | IB_ACCESS_REMOTE_ATOMIC, |
260 | RXE_ACCESS_SUPPORTED_MR = RXE_ACCESS_REMOTE |
261 | | IB_ACCESS_LOCAL_WRITE |
262 | | IB_ACCESS_MW_BIND |
263 | | IB_ACCESS_ON_DEMAND |
264 | | IB_ACCESS_FLUSH_GLOBAL |
265 | | IB_ACCESS_FLUSH_PERSISTENT |
266 | | IB_ACCESS_OPTIONAL, |
267 | RXE_ACCESS_SUPPORTED_QP = RXE_ACCESS_SUPPORTED_MR, |
268 | RXE_ACCESS_SUPPORTED_MW = RXE_ACCESS_SUPPORTED_MR |
269 | | IB_ZERO_BASED, |
270 | }; |
271 | |
272 | enum rxe_mr_state { |
273 | RXE_MR_STATE_INVALID, |
274 | RXE_MR_STATE_FREE, |
275 | RXE_MR_STATE_VALID, |
276 | }; |
277 | |
278 | enum rxe_mr_copy_dir { |
279 | RXE_TO_MR_OBJ, |
280 | RXE_FROM_MR_OBJ, |
281 | }; |
282 | |
283 | enum rxe_mr_lookup_type { |
284 | RXE_LOOKUP_LOCAL, |
285 | RXE_LOOKUP_REMOTE, |
286 | }; |
287 | |
288 | enum rxe_rereg { |
289 | RXE_MR_REREG_SUPPORTED = IB_MR_REREG_PD |
290 | | IB_MR_REREG_ACCESS, |
291 | }; |
292 | |
293 | static inline int rkey_is_mw(u32 rkey) |
294 | { |
295 | u32 index = rkey >> 8; |
296 | |
297 | return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX); |
298 | } |
299 | |
300 | struct rxe_mr { |
301 | struct rxe_pool_elem elem; |
302 | struct ib_mr ibmr; |
303 | |
304 | struct ib_umem *umem; |
305 | |
306 | u32 lkey; |
307 | u32 rkey; |
308 | enum rxe_mr_state state; |
309 | int access; |
310 | atomic_t num_mw; |
311 | |
312 | unsigned int page_offset; |
313 | unsigned int page_shift; |
314 | u64 page_mask; |
315 | |
316 | u32 num_buf; |
317 | u32 nbuf; |
318 | |
319 | struct xarray page_list; |
320 | }; |
321 | |
322 | static inline unsigned int mr_page_size(struct rxe_mr *mr) |
323 | { |
324 | return mr ? mr->ibmr.page_size : PAGE_SIZE; |
325 | } |
326 | |
327 | enum rxe_mw_state { |
328 | RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID, |
329 | RXE_MW_STATE_FREE = RXE_MR_STATE_FREE, |
330 | RXE_MW_STATE_VALID = RXE_MR_STATE_VALID, |
331 | }; |
332 | |
333 | struct rxe_mw { |
334 | struct ib_mw ibmw; |
335 | struct rxe_pool_elem elem; |
336 | spinlock_t lock; |
337 | enum rxe_mw_state state; |
338 | struct rxe_qp *qp; /* Type 2 only */ |
339 | struct rxe_mr *mr; |
340 | u32 rkey; |
341 | int access; |
342 | u64 addr; |
343 | u64 length; |
344 | }; |
345 | |
346 | struct rxe_mcg { |
347 | struct rb_node node; |
348 | struct kref ref_cnt; |
349 | struct rxe_dev *rxe; |
350 | struct list_head qp_list; |
351 | union ib_gid mgid; |
352 | atomic_t qp_num; |
353 | u32 qkey; |
354 | u16 pkey; |
355 | }; |
356 | |
357 | struct rxe_mca { |
358 | struct list_head qp_list; |
359 | struct rxe_qp *qp; |
360 | }; |
361 | |
362 | struct rxe_port { |
363 | struct ib_port_attr attr; |
364 | __be64 port_guid; |
365 | __be64 subnet_prefix; |
366 | spinlock_t port_lock; /* guard port */ |
367 | unsigned int mtu_cap; |
368 | /* special QPs */ |
369 | u32 qp_gsi_index; |
370 | }; |
371 | |
372 | struct rxe_dev { |
373 | struct ib_device ib_dev; |
374 | struct ib_device_attr attr; |
375 | int max_ucontext; |
376 | int max_inline_data; |
377 | struct mutex usdev_lock; |
378 | |
379 | struct net_device *ndev; |
380 | |
381 | struct rxe_pool uc_pool; |
382 | struct rxe_pool pd_pool; |
383 | struct rxe_pool ah_pool; |
384 | struct rxe_pool srq_pool; |
385 | struct rxe_pool qp_pool; |
386 | struct rxe_pool cq_pool; |
387 | struct rxe_pool mr_pool; |
388 | struct rxe_pool mw_pool; |
389 | |
390 | /* multicast support */ |
391 | spinlock_t mcg_lock; |
392 | struct rb_root mcg_tree; |
393 | atomic_t mcg_num; |
394 | atomic_t mcg_attach; |
395 | |
396 | spinlock_t pending_lock; /* guard pending_mmaps */ |
397 | struct list_head pending_mmaps; |
398 | |
399 | spinlock_t mmap_offset_lock; /* guard mmap_offset */ |
400 | u64 mmap_offset; |
401 | |
402 | atomic64_t stats_counters[RXE_NUM_OF_COUNTERS]; |
403 | |
404 | struct rxe_port port; |
405 | struct crypto_shash *tfm; |
406 | }; |
407 | |
408 | static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index) |
409 | { |
410 | atomic64_inc(v: &rxe->stats_counters[index]); |
411 | } |
412 | |
413 | static inline struct rxe_dev *to_rdev(struct ib_device *dev) |
414 | { |
415 | return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL; |
416 | } |
417 | |
418 | static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc) |
419 | { |
420 | return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL; |
421 | } |
422 | |
423 | static inline struct rxe_pd *to_rpd(struct ib_pd *pd) |
424 | { |
425 | return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL; |
426 | } |
427 | |
428 | static inline struct rxe_ah *to_rah(struct ib_ah *ah) |
429 | { |
430 | return ah ? container_of(ah, struct rxe_ah, ibah) : NULL; |
431 | } |
432 | |
433 | static inline struct rxe_srq *to_rsrq(struct ib_srq *srq) |
434 | { |
435 | return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL; |
436 | } |
437 | |
438 | static inline struct rxe_qp *to_rqp(struct ib_qp *qp) |
439 | { |
440 | return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL; |
441 | } |
442 | |
443 | static inline struct rxe_cq *to_rcq(struct ib_cq *cq) |
444 | { |
445 | return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL; |
446 | } |
447 | |
448 | static inline struct rxe_mr *to_rmr(struct ib_mr *mr) |
449 | { |
450 | return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL; |
451 | } |
452 | |
453 | static inline struct rxe_mw *to_rmw(struct ib_mw *mw) |
454 | { |
455 | return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL; |
456 | } |
457 | |
458 | static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah) |
459 | { |
460 | return to_rpd(pd: ah->ibah.pd); |
461 | } |
462 | |
463 | static inline struct rxe_pd *mr_pd(struct rxe_mr *mr) |
464 | { |
465 | return to_rpd(pd: mr->ibmr.pd); |
466 | } |
467 | |
468 | static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw) |
469 | { |
470 | return to_rpd(pd: mw->ibmw.pd); |
471 | } |
472 | |
473 | int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name); |
474 | |
475 | #endif /* RXE_VERBS_H */ |
476 | |