1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* |
3 | * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. |
4 | * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. |
5 | */ |
6 | |
7 | #include "rxe.h" |
8 | |
9 | #define RXE_POOL_TIMEOUT (200) |
10 | #define RXE_POOL_ALIGN (16) |
11 | |
12 | static const struct rxe_type_info { |
13 | const char *name; |
14 | size_t size; |
15 | size_t elem_offset; |
16 | void (*cleanup)(struct rxe_pool_elem *elem); |
17 | u32 min_index; |
18 | u32 max_index; |
19 | u32 max_elem; |
20 | } rxe_type_info[RXE_NUM_TYPES] = { |
21 | [RXE_TYPE_UC] = { |
22 | .name = "uc" , |
23 | .size = sizeof(struct rxe_ucontext), |
24 | .elem_offset = offsetof(struct rxe_ucontext, elem), |
25 | .min_index = 1, |
26 | .max_index = RXE_MAX_UCONTEXT, |
27 | .max_elem = RXE_MAX_UCONTEXT, |
28 | }, |
29 | [RXE_TYPE_PD] = { |
30 | .name = "pd" , |
31 | .size = sizeof(struct rxe_pd), |
32 | .elem_offset = offsetof(struct rxe_pd, elem), |
33 | .min_index = 1, |
34 | .max_index = RXE_MAX_PD, |
35 | .max_elem = RXE_MAX_PD, |
36 | }, |
37 | [RXE_TYPE_AH] = { |
38 | .name = "ah" , |
39 | .size = sizeof(struct rxe_ah), |
40 | .elem_offset = offsetof(struct rxe_ah, elem), |
41 | .min_index = RXE_MIN_AH_INDEX, |
42 | .max_index = RXE_MAX_AH_INDEX, |
43 | .max_elem = RXE_MAX_AH, |
44 | }, |
45 | [RXE_TYPE_SRQ] = { |
46 | .name = "srq" , |
47 | .size = sizeof(struct rxe_srq), |
48 | .elem_offset = offsetof(struct rxe_srq, elem), |
49 | .cleanup = rxe_srq_cleanup, |
50 | .min_index = RXE_MIN_SRQ_INDEX, |
51 | .max_index = RXE_MAX_SRQ_INDEX, |
52 | .max_elem = RXE_MAX_SRQ, |
53 | }, |
54 | [RXE_TYPE_QP] = { |
55 | .name = "qp" , |
56 | .size = sizeof(struct rxe_qp), |
57 | .elem_offset = offsetof(struct rxe_qp, elem), |
58 | .cleanup = rxe_qp_cleanup, |
59 | .min_index = RXE_MIN_QP_INDEX, |
60 | .max_index = RXE_MAX_QP_INDEX, |
61 | .max_elem = RXE_MAX_QP, |
62 | }, |
63 | [RXE_TYPE_CQ] = { |
64 | .name = "cq" , |
65 | .size = sizeof(struct rxe_cq), |
66 | .elem_offset = offsetof(struct rxe_cq, elem), |
67 | .cleanup = rxe_cq_cleanup, |
68 | .min_index = 1, |
69 | .max_index = RXE_MAX_CQ, |
70 | .max_elem = RXE_MAX_CQ, |
71 | }, |
72 | [RXE_TYPE_MR] = { |
73 | .name = "mr" , |
74 | .size = sizeof(struct rxe_mr), |
75 | .elem_offset = offsetof(struct rxe_mr, elem), |
76 | .cleanup = rxe_mr_cleanup, |
77 | .min_index = RXE_MIN_MR_INDEX, |
78 | .max_index = RXE_MAX_MR_INDEX, |
79 | .max_elem = RXE_MAX_MR, |
80 | }, |
81 | [RXE_TYPE_MW] = { |
82 | .name = "mw" , |
83 | .size = sizeof(struct rxe_mw), |
84 | .elem_offset = offsetof(struct rxe_mw, elem), |
85 | .cleanup = rxe_mw_cleanup, |
86 | .min_index = RXE_MIN_MW_INDEX, |
87 | .max_index = RXE_MAX_MW_INDEX, |
88 | .max_elem = RXE_MAX_MW, |
89 | }, |
90 | }; |
91 | |
92 | void rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool, |
93 | enum rxe_elem_type type) |
94 | { |
95 | const struct rxe_type_info *info = &rxe_type_info[type]; |
96 | |
97 | memset(pool, 0, sizeof(*pool)); |
98 | |
99 | pool->rxe = rxe; |
100 | pool->name = info->name; |
101 | pool->type = type; |
102 | pool->max_elem = info->max_elem; |
103 | pool->elem_size = ALIGN(info->size, RXE_POOL_ALIGN); |
104 | pool->elem_offset = info->elem_offset; |
105 | pool->cleanup = info->cleanup; |
106 | |
107 | atomic_set(v: &pool->num_elem, i: 0); |
108 | |
109 | xa_init_flags(xa: &pool->xa, XA_FLAGS_ALLOC); |
110 | pool->limit.min = info->min_index; |
111 | pool->limit.max = info->max_index; |
112 | } |
113 | |
114 | void rxe_pool_cleanup(struct rxe_pool *pool) |
115 | { |
116 | WARN_ON(!xa_empty(&pool->xa)); |
117 | } |
118 | |
119 | int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem, |
120 | bool sleepable) |
121 | { |
122 | int err; |
123 | gfp_t gfp_flags; |
124 | |
125 | if (atomic_inc_return(v: &pool->num_elem) > pool->max_elem) |
126 | goto err_cnt; |
127 | |
128 | elem->pool = pool; |
129 | elem->obj = (u8 *)elem - pool->elem_offset; |
130 | kref_init(kref: &elem->ref_cnt); |
131 | init_completion(x: &elem->complete); |
132 | |
133 | /* AH objects are unique in that the create_ah verb |
134 | * can be called in atomic context. If the create_ah |
135 | * call is not sleepable use GFP_ATOMIC. |
136 | */ |
137 | gfp_flags = sleepable ? GFP_KERNEL : GFP_ATOMIC; |
138 | |
139 | if (sleepable) |
140 | might_sleep(); |
141 | err = xa_alloc_cyclic(xa: &pool->xa, id: &elem->index, NULL, limit: pool->limit, |
142 | next: &pool->next, gfp: gfp_flags); |
143 | if (err < 0) |
144 | goto err_cnt; |
145 | |
146 | return 0; |
147 | |
148 | err_cnt: |
149 | atomic_dec(v: &pool->num_elem); |
150 | return -EINVAL; |
151 | } |
152 | |
153 | void *rxe_pool_get_index(struct rxe_pool *pool, u32 index) |
154 | { |
155 | struct rxe_pool_elem *elem; |
156 | struct xarray *xa = &pool->xa; |
157 | void *obj; |
158 | |
159 | rcu_read_lock(); |
160 | elem = xa_load(xa, index); |
161 | if (elem && kref_get_unless_zero(kref: &elem->ref_cnt)) |
162 | obj = elem->obj; |
163 | else |
164 | obj = NULL; |
165 | rcu_read_unlock(); |
166 | |
167 | return obj; |
168 | } |
169 | |
170 | static void rxe_elem_release(struct kref *kref) |
171 | { |
172 | struct rxe_pool_elem *elem = container_of(kref, typeof(*elem), ref_cnt); |
173 | |
174 | complete(&elem->complete); |
175 | } |
176 | |
177 | int __rxe_cleanup(struct rxe_pool_elem *elem, bool sleepable) |
178 | { |
179 | struct rxe_pool *pool = elem->pool; |
180 | struct xarray *xa = &pool->xa; |
181 | static int timeout = RXE_POOL_TIMEOUT; |
182 | int ret, err = 0; |
183 | void *xa_ret; |
184 | |
185 | if (sleepable) |
186 | might_sleep(); |
187 | |
188 | /* erase xarray entry to prevent looking up |
189 | * the pool elem from its index |
190 | */ |
191 | xa_ret = xa_erase(xa, index: elem->index); |
192 | WARN_ON(xa_err(xa_ret)); |
193 | |
194 | /* if this is the last call to rxe_put complete the |
195 | * object. It is safe to touch obj->elem after this since |
196 | * it is freed below |
197 | */ |
198 | __rxe_put(elem); |
199 | |
200 | /* wait until all references to the object have been |
201 | * dropped before final object specific cleanup and |
202 | * return to rdma-core |
203 | */ |
204 | if (sleepable) { |
205 | if (!completion_done(x: &elem->complete) && timeout) { |
206 | ret = wait_for_completion_timeout(x: &elem->complete, |
207 | timeout); |
208 | |
209 | /* Shouldn't happen. There are still references to |
210 | * the object but, rather than deadlock, free the |
211 | * object or pass back to rdma-core. |
212 | */ |
213 | if (WARN_ON(!ret)) |
214 | err = -EINVAL; |
215 | } |
216 | } else { |
217 | unsigned long until = jiffies + timeout; |
218 | |
219 | /* AH objects are unique in that the destroy_ah verb |
220 | * can be called in atomic context. This delay |
221 | * replaces the wait_for_completion call above |
222 | * when the destroy_ah call is not sleepable |
223 | */ |
224 | while (!completion_done(x: &elem->complete) && |
225 | time_before(jiffies, until)) |
226 | mdelay(1); |
227 | |
228 | if (WARN_ON(!completion_done(&elem->complete))) |
229 | err = -EINVAL; |
230 | } |
231 | |
232 | if (pool->cleanup) |
233 | pool->cleanup(elem); |
234 | |
235 | atomic_dec(v: &pool->num_elem); |
236 | |
237 | return err; |
238 | } |
239 | |
240 | int __rxe_get(struct rxe_pool_elem *elem) |
241 | { |
242 | return kref_get_unless_zero(kref: &elem->ref_cnt); |
243 | } |
244 | |
245 | int __rxe_put(struct rxe_pool_elem *elem) |
246 | { |
247 | return kref_put(kref: &elem->ref_cnt, release: rxe_elem_release); |
248 | } |
249 | |
250 | void __rxe_finalize(struct rxe_pool_elem *elem) |
251 | { |
252 | void *xa_ret; |
253 | |
254 | xa_ret = xa_store(&elem->pool->xa, index: elem->index, entry: elem, GFP_KERNEL); |
255 | WARN_ON(xa_err(xa_ret)); |
256 | } |
257 | |