1 | // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause |
2 | /* |
3 | * Copyright(c) 2016 Intel Corporation. |
4 | */ |
5 | |
6 | #include <linux/err.h> |
7 | #include <linux/slab.h> |
8 | #include <linux/vmalloc.h> |
9 | #include <rdma/uverbs_ioctl.h> |
10 | |
11 | #include "srq.h" |
12 | #include "vt.h" |
13 | #include "qp.h" |
14 | /** |
15 | * rvt_driver_srq_init - init srq resources on a per driver basis |
16 | * @rdi: rvt dev structure |
17 | * |
18 | * Do any initialization needed when a driver registers with rdmavt. |
19 | */ |
20 | void rvt_driver_srq_init(struct rvt_dev_info *rdi) |
21 | { |
22 | spin_lock_init(&rdi->n_srqs_lock); |
23 | rdi->n_srqs_allocated = 0; |
24 | } |
25 | |
26 | /** |
27 | * rvt_create_srq - create a shared receive queue |
28 | * @ibsrq: the protection domain of the SRQ to create |
29 | * @srq_init_attr: the attributes of the SRQ |
30 | * @udata: data from libibverbs when creating a user SRQ |
31 | * |
32 | * Return: 0 on success |
33 | */ |
34 | int rvt_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *srq_init_attr, |
35 | struct ib_udata *udata) |
36 | { |
37 | struct rvt_dev_info *dev = ib_to_rvt(ibdev: ibsrq->device); |
38 | struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); |
39 | u32 sz; |
40 | int ret; |
41 | |
42 | if (srq_init_attr->srq_type != IB_SRQT_BASIC) |
43 | return -EOPNOTSUPP; |
44 | |
45 | if (srq_init_attr->attr.max_sge == 0 || |
46 | srq_init_attr->attr.max_sge > dev->dparms.props.max_srq_sge || |
47 | srq_init_attr->attr.max_wr == 0 || |
48 | srq_init_attr->attr.max_wr > dev->dparms.props.max_srq_wr) |
49 | return -EINVAL; |
50 | |
51 | /* |
52 | * Need to use vmalloc() if we want to support large #s of entries. |
53 | */ |
54 | srq->rq.size = srq_init_attr->attr.max_wr + 1; |
55 | srq->rq.max_sge = srq_init_attr->attr.max_sge; |
56 | sz = sizeof(struct ib_sge) * srq->rq.max_sge + |
57 | sizeof(struct rvt_rwqe); |
58 | if (rvt_alloc_rq(rq: &srq->rq, size: srq->rq.size * sz, |
59 | node: dev->dparms.node, udata)) { |
60 | ret = -ENOMEM; |
61 | goto bail_srq; |
62 | } |
63 | |
64 | /* |
65 | * Return the address of the RWQ as the offset to mmap. |
66 | * See rvt_mmap() for details. |
67 | */ |
68 | if (udata && udata->outlen >= sizeof(__u64)) { |
69 | u32 s = sizeof(struct rvt_rwq) + srq->rq.size * sz; |
70 | |
71 | srq->ip = rvt_create_mmap_info(rdi: dev, size: s, udata, obj: srq->rq.wq); |
72 | if (IS_ERR(ptr: srq->ip)) { |
73 | ret = PTR_ERR(ptr: srq->ip); |
74 | goto bail_wq; |
75 | } |
76 | |
77 | ret = ib_copy_to_udata(udata, src: &srq->ip->offset, |
78 | len: sizeof(srq->ip->offset)); |
79 | if (ret) |
80 | goto bail_ip; |
81 | } |
82 | |
83 | /* |
84 | * ib_create_srq() will initialize srq->ibsrq. |
85 | */ |
86 | spin_lock_init(&srq->rq.lock); |
87 | srq->limit = srq_init_attr->attr.srq_limit; |
88 | |
89 | spin_lock(lock: &dev->n_srqs_lock); |
90 | if (dev->n_srqs_allocated == dev->dparms.props.max_srq) { |
91 | spin_unlock(lock: &dev->n_srqs_lock); |
92 | ret = -ENOMEM; |
93 | goto bail_ip; |
94 | } |
95 | |
96 | dev->n_srqs_allocated++; |
97 | spin_unlock(lock: &dev->n_srqs_lock); |
98 | |
99 | if (srq->ip) { |
100 | spin_lock_irq(lock: &dev->pending_lock); |
101 | list_add(new: &srq->ip->pending_mmaps, head: &dev->pending_mmaps); |
102 | spin_unlock_irq(lock: &dev->pending_lock); |
103 | } |
104 | |
105 | return 0; |
106 | |
107 | bail_ip: |
108 | kfree(objp: srq->ip); |
109 | bail_wq: |
110 | rvt_free_rq(rq: &srq->rq); |
111 | bail_srq: |
112 | return ret; |
113 | } |
114 | |
115 | /** |
116 | * rvt_modify_srq - modify a shared receive queue |
117 | * @ibsrq: the SRQ to modify |
118 | * @attr: the new attributes of the SRQ |
119 | * @attr_mask: indicates which attributes to modify |
120 | * @udata: user data for libibverbs.so |
121 | * |
122 | * Return: 0 on success |
123 | */ |
124 | int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, |
125 | enum ib_srq_attr_mask attr_mask, |
126 | struct ib_udata *udata) |
127 | { |
128 | struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); |
129 | struct rvt_dev_info *dev = ib_to_rvt(ibdev: ibsrq->device); |
130 | struct rvt_rq tmp_rq = {}; |
131 | int ret = 0; |
132 | |
133 | if (attr_mask & IB_SRQ_MAX_WR) { |
134 | struct rvt_krwq *okwq = NULL; |
135 | struct rvt_rwq *owq = NULL; |
136 | struct rvt_rwqe *p; |
137 | u32 sz, size, n, head, tail; |
138 | |
139 | /* Check that the requested sizes are below the limits. */ |
140 | if ((attr->max_wr > dev->dparms.props.max_srq_wr) || |
141 | ((attr_mask & IB_SRQ_LIMIT) ? |
142 | attr->srq_limit : srq->limit) > attr->max_wr) |
143 | return -EINVAL; |
144 | sz = sizeof(struct rvt_rwqe) + |
145 | srq->rq.max_sge * sizeof(struct ib_sge); |
146 | size = attr->max_wr + 1; |
147 | if (rvt_alloc_rq(rq: &tmp_rq, size: size * sz, node: dev->dparms.node, |
148 | udata)) |
149 | return -ENOMEM; |
150 | /* Check that we can write the offset to mmap. */ |
151 | if (udata && udata->inlen >= sizeof(__u64)) { |
152 | __u64 offset_addr; |
153 | __u64 offset = 0; |
154 | |
155 | ret = ib_copy_from_udata(dest: &offset_addr, udata, |
156 | len: sizeof(offset_addr)); |
157 | if (ret) |
158 | goto bail_free; |
159 | udata->outbuf = (void __user *) |
160 | (unsigned long)offset_addr; |
161 | ret = ib_copy_to_udata(udata, src: &offset, |
162 | len: sizeof(offset)); |
163 | if (ret) |
164 | goto bail_free; |
165 | } |
166 | |
167 | spin_lock_irq(lock: &srq->rq.kwq->c_lock); |
168 | /* |
169 | * validate head and tail pointer values and compute |
170 | * the number of remaining WQEs. |
171 | */ |
172 | if (udata) { |
173 | owq = srq->rq.wq; |
174 | head = RDMA_READ_UAPI_ATOMIC(owq->head); |
175 | tail = RDMA_READ_UAPI_ATOMIC(owq->tail); |
176 | } else { |
177 | okwq = srq->rq.kwq; |
178 | head = okwq->head; |
179 | tail = okwq->tail; |
180 | } |
181 | if (head >= srq->rq.size || tail >= srq->rq.size) { |
182 | ret = -EINVAL; |
183 | goto bail_unlock; |
184 | } |
185 | n = head; |
186 | if (n < tail) |
187 | n += srq->rq.size - tail; |
188 | else |
189 | n -= tail; |
190 | if (size <= n) { |
191 | ret = -EINVAL; |
192 | goto bail_unlock; |
193 | } |
194 | n = 0; |
195 | p = tmp_rq.kwq->curr_wq; |
196 | while (tail != head) { |
197 | struct rvt_rwqe *wqe; |
198 | int i; |
199 | |
200 | wqe = rvt_get_rwqe_ptr(rq: &srq->rq, n: tail); |
201 | p->wr_id = wqe->wr_id; |
202 | p->num_sge = wqe->num_sge; |
203 | for (i = 0; i < wqe->num_sge; i++) |
204 | p->sg_list[i] = wqe->sg_list[i]; |
205 | n++; |
206 | p = (struct rvt_rwqe *)((char *)p + sz); |
207 | if (++tail >= srq->rq.size) |
208 | tail = 0; |
209 | } |
210 | srq->rq.kwq = tmp_rq.kwq; |
211 | if (udata) { |
212 | srq->rq.wq = tmp_rq.wq; |
213 | RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->head, n); |
214 | RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->tail, 0); |
215 | } else { |
216 | tmp_rq.kwq->head = n; |
217 | tmp_rq.kwq->tail = 0; |
218 | } |
219 | srq->rq.size = size; |
220 | if (attr_mask & IB_SRQ_LIMIT) |
221 | srq->limit = attr->srq_limit; |
222 | spin_unlock_irq(lock: &srq->rq.kwq->c_lock); |
223 | |
224 | vfree(addr: owq); |
225 | kvfree(addr: okwq); |
226 | |
227 | if (srq->ip) { |
228 | struct rvt_mmap_info *ip = srq->ip; |
229 | struct rvt_dev_info *dev = ib_to_rvt(ibdev: srq->ibsrq.device); |
230 | u32 s = sizeof(struct rvt_rwq) + size * sz; |
231 | |
232 | rvt_update_mmap_info(rdi: dev, ip, size: s, obj: tmp_rq.wq); |
233 | |
234 | /* |
235 | * Return the offset to mmap. |
236 | * See rvt_mmap() for details. |
237 | */ |
238 | if (udata && udata->inlen >= sizeof(__u64)) { |
239 | ret = ib_copy_to_udata(udata, src: &ip->offset, |
240 | len: sizeof(ip->offset)); |
241 | if (ret) |
242 | return ret; |
243 | } |
244 | |
245 | /* |
246 | * Put user mapping info onto the pending list |
247 | * unless it already is on the list. |
248 | */ |
249 | spin_lock_irq(lock: &dev->pending_lock); |
250 | if (list_empty(head: &ip->pending_mmaps)) |
251 | list_add(new: &ip->pending_mmaps, |
252 | head: &dev->pending_mmaps); |
253 | spin_unlock_irq(lock: &dev->pending_lock); |
254 | } |
255 | } else if (attr_mask & IB_SRQ_LIMIT) { |
256 | spin_lock_irq(lock: &srq->rq.kwq->c_lock); |
257 | if (attr->srq_limit >= srq->rq.size) |
258 | ret = -EINVAL; |
259 | else |
260 | srq->limit = attr->srq_limit; |
261 | spin_unlock_irq(lock: &srq->rq.kwq->c_lock); |
262 | } |
263 | return ret; |
264 | |
265 | bail_unlock: |
266 | spin_unlock_irq(lock: &srq->rq.kwq->c_lock); |
267 | bail_free: |
268 | rvt_free_rq(rq: &tmp_rq); |
269 | return ret; |
270 | } |
271 | |
272 | /** |
273 | * rvt_query_srq - query srq data |
274 | * @ibsrq: srq to query |
275 | * @attr: return info in attr |
276 | * |
277 | * Return: always 0 |
278 | */ |
279 | int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) |
280 | { |
281 | struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); |
282 | |
283 | attr->max_wr = srq->rq.size - 1; |
284 | attr->max_sge = srq->rq.max_sge; |
285 | attr->srq_limit = srq->limit; |
286 | return 0; |
287 | } |
288 | |
289 | /** |
290 | * rvt_destroy_srq - destory an srq |
291 | * @ibsrq: srq object to destroy |
292 | * @udata: user data for libibverbs.so |
293 | */ |
294 | int rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) |
295 | { |
296 | struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); |
297 | struct rvt_dev_info *dev = ib_to_rvt(ibdev: ibsrq->device); |
298 | |
299 | spin_lock(lock: &dev->n_srqs_lock); |
300 | dev->n_srqs_allocated--; |
301 | spin_unlock(lock: &dev->n_srqs_lock); |
302 | if (srq->ip) |
303 | kref_put(kref: &srq->ip->ref, release: rvt_release_mmap_info); |
304 | kvfree(addr: srq->rq.kwq); |
305 | return 0; |
306 | } |
307 | |