1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* |
3 | * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. |
4 | * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. |
5 | */ |
6 | |
7 | #include <linux/libnvdimm.h> |
8 | |
9 | #include "rxe.h" |
10 | #include "rxe_loc.h" |
11 | |
12 | /* Return a random 8 bit key value that is |
13 | * different than the last_key. Set last_key to -1 |
14 | * if this is the first key for an MR or MW |
15 | */ |
16 | u8 rxe_get_next_key(u32 last_key) |
17 | { |
18 | u8 key; |
19 | |
20 | do { |
21 | get_random_bytes(buf: &key, len: 1); |
22 | } while (key == last_key); |
23 | |
24 | return key; |
25 | } |
26 | |
27 | int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length) |
28 | { |
29 | switch (mr->ibmr.type) { |
30 | case IB_MR_TYPE_DMA: |
31 | return 0; |
32 | |
33 | case IB_MR_TYPE_USER: |
34 | case IB_MR_TYPE_MEM_REG: |
35 | if (iova < mr->ibmr.iova || |
36 | iova + length > mr->ibmr.iova + mr->ibmr.length) { |
37 | rxe_dbg_mr(mr, "iova/length out of range\n" ); |
38 | return -EINVAL; |
39 | } |
40 | return 0; |
41 | |
42 | default: |
43 | rxe_dbg_mr(mr, "mr type not supported\n" ); |
44 | return -EINVAL; |
45 | } |
46 | } |
47 | |
48 | static void rxe_mr_init(int access, struct rxe_mr *mr) |
49 | { |
50 | u32 key = mr->elem.index << 8 | rxe_get_next_key(last_key: -1); |
51 | |
52 | /* set ibmr->l/rkey and also copy into private l/rkey |
53 | * for user MRs these will always be the same |
54 | * for cases where caller 'owns' the key portion |
55 | * they may be different until REG_MR WQE is executed. |
56 | */ |
57 | mr->lkey = mr->ibmr.lkey = key; |
58 | mr->rkey = mr->ibmr.rkey = key; |
59 | |
60 | mr->access = access; |
61 | mr->ibmr.page_size = PAGE_SIZE; |
62 | mr->page_mask = PAGE_MASK; |
63 | mr->page_shift = PAGE_SHIFT; |
64 | mr->state = RXE_MR_STATE_INVALID; |
65 | } |
66 | |
67 | void rxe_mr_init_dma(int access, struct rxe_mr *mr) |
68 | { |
69 | rxe_mr_init(access, mr); |
70 | |
71 | mr->state = RXE_MR_STATE_VALID; |
72 | mr->ibmr.type = IB_MR_TYPE_DMA; |
73 | } |
74 | |
75 | static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova) |
76 | { |
77 | return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift); |
78 | } |
79 | |
80 | static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova) |
81 | { |
82 | return iova & (mr_page_size(mr) - 1); |
83 | } |
84 | |
85 | static bool is_pmem_page(struct page *pg) |
86 | { |
87 | unsigned long paddr = page_to_phys(pg); |
88 | |
89 | return REGION_INTERSECTS == |
90 | region_intersects(offset: paddr, PAGE_SIZE, IORESOURCE_MEM, |
91 | desc: IORES_DESC_PERSISTENT_MEMORY); |
92 | } |
93 | |
94 | static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt) |
95 | { |
96 | XA_STATE(xas, &mr->page_list, 0); |
97 | struct sg_page_iter sg_iter; |
98 | struct page *page; |
99 | bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT); |
100 | |
101 | __sg_page_iter_start(piter: &sg_iter, sglist: sgt->sgl, nents: sgt->orig_nents, pgoffset: 0); |
102 | if (!__sg_page_iter_next(piter: &sg_iter)) |
103 | return 0; |
104 | |
105 | do { |
106 | xas_lock(&xas); |
107 | while (true) { |
108 | page = sg_page_iter_page(piter: &sg_iter); |
109 | |
110 | if (persistent && !is_pmem_page(pg: page)) { |
111 | rxe_dbg_mr(mr, "Page can't be persistent\n" ); |
112 | xas_set_err(xas: &xas, err: -EINVAL); |
113 | break; |
114 | } |
115 | |
116 | xas_store(&xas, entry: page); |
117 | if (xas_error(xas: &xas)) |
118 | break; |
119 | xas_next(xas: &xas); |
120 | if (!__sg_page_iter_next(piter: &sg_iter)) |
121 | break; |
122 | } |
123 | xas_unlock(&xas); |
124 | } while (xas_nomem(&xas, GFP_KERNEL)); |
125 | |
126 | return xas_error(xas: &xas); |
127 | } |
128 | |
129 | int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, |
130 | int access, struct rxe_mr *mr) |
131 | { |
132 | struct ib_umem *umem; |
133 | int err; |
134 | |
135 | rxe_mr_init(access, mr); |
136 | |
137 | xa_init(xa: &mr->page_list); |
138 | |
139 | umem = ib_umem_get(device: &rxe->ib_dev, addr: start, size: length, access); |
140 | if (IS_ERR(ptr: umem)) { |
141 | rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n" , |
142 | (int)PTR_ERR(umem)); |
143 | return PTR_ERR(ptr: umem); |
144 | } |
145 | |
146 | err = rxe_mr_fill_pages_from_sgt(mr, sgt: &umem->sgt_append.sgt); |
147 | if (err) { |
148 | ib_umem_release(umem); |
149 | return err; |
150 | } |
151 | |
152 | mr->umem = umem; |
153 | mr->ibmr.type = IB_MR_TYPE_USER; |
154 | mr->state = RXE_MR_STATE_VALID; |
155 | |
156 | return 0; |
157 | } |
158 | |
159 | static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf) |
160 | { |
161 | XA_STATE(xas, &mr->page_list, 0); |
162 | int i = 0; |
163 | int err; |
164 | |
165 | xa_init(xa: &mr->page_list); |
166 | |
167 | do { |
168 | xas_lock(&xas); |
169 | while (i != num_buf) { |
170 | xas_store(&xas, XA_ZERO_ENTRY); |
171 | if (xas_error(xas: &xas)) |
172 | break; |
173 | xas_next(xas: &xas); |
174 | i++; |
175 | } |
176 | xas_unlock(&xas); |
177 | } while (xas_nomem(&xas, GFP_KERNEL)); |
178 | |
179 | err = xas_error(xas: &xas); |
180 | if (err) |
181 | return err; |
182 | |
183 | mr->num_buf = num_buf; |
184 | |
185 | return 0; |
186 | } |
187 | |
188 | int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) |
189 | { |
190 | int err; |
191 | |
192 | /* always allow remote access for FMRs */ |
193 | rxe_mr_init(access: RXE_ACCESS_REMOTE, mr); |
194 | |
195 | err = rxe_mr_alloc(mr, num_buf: max_pages); |
196 | if (err) |
197 | goto err1; |
198 | |
199 | mr->state = RXE_MR_STATE_FREE; |
200 | mr->ibmr.type = IB_MR_TYPE_MEM_REG; |
201 | |
202 | return 0; |
203 | |
204 | err1: |
205 | return err; |
206 | } |
207 | |
208 | static int rxe_set_page(struct ib_mr *ibmr, u64 dma_addr) |
209 | { |
210 | struct rxe_mr *mr = to_rmr(mr: ibmr); |
211 | struct page *page = ib_virt_dma_to_page(dma_addr); |
212 | bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT); |
213 | int err; |
214 | |
215 | if (persistent && !is_pmem_page(pg: page)) { |
216 | rxe_dbg_mr(mr, "Page cannot be persistent\n" ); |
217 | return -EINVAL; |
218 | } |
219 | |
220 | if (unlikely(mr->nbuf == mr->num_buf)) |
221 | return -ENOMEM; |
222 | |
223 | err = xa_err(entry: xa_store(&mr->page_list, index: mr->nbuf, entry: page, GFP_KERNEL)); |
224 | if (err) |
225 | return err; |
226 | |
227 | mr->nbuf++; |
228 | return 0; |
229 | } |
230 | |
231 | int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl, |
232 | int sg_nents, unsigned int *sg_offset) |
233 | { |
234 | struct rxe_mr *mr = to_rmr(mr: ibmr); |
235 | unsigned int page_size = mr_page_size(mr); |
236 | |
237 | mr->nbuf = 0; |
238 | mr->page_shift = ilog2(page_size); |
239 | mr->page_mask = ~((u64)page_size - 1); |
240 | mr->page_offset = mr->ibmr.iova & (page_size - 1); |
241 | |
242 | return ib_sg_to_pages(mr: ibmr, sgl, sg_nents, sg_offset, set_page: rxe_set_page); |
243 | } |
244 | |
245 | static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr, |
246 | unsigned int length, enum rxe_mr_copy_dir dir) |
247 | { |
248 | unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); |
249 | unsigned long index = rxe_mr_iova_to_index(mr, iova); |
250 | unsigned int bytes; |
251 | struct page *page; |
252 | void *va; |
253 | |
254 | while (length) { |
255 | page = xa_load(&mr->page_list, index); |
256 | if (!page) |
257 | return -EFAULT; |
258 | |
259 | bytes = min_t(unsigned int, length, |
260 | mr_page_size(mr) - page_offset); |
261 | va = kmap_local_page(page); |
262 | if (dir == RXE_FROM_MR_OBJ) |
263 | memcpy(addr, va + page_offset, bytes); |
264 | else |
265 | memcpy(va + page_offset, addr, bytes); |
266 | kunmap_local(va); |
267 | |
268 | page_offset = 0; |
269 | addr += bytes; |
270 | length -= bytes; |
271 | index++; |
272 | } |
273 | |
274 | return 0; |
275 | } |
276 | |
277 | static void rxe_mr_copy_dma(struct rxe_mr *mr, u64 dma_addr, void *addr, |
278 | unsigned int length, enum rxe_mr_copy_dir dir) |
279 | { |
280 | unsigned int page_offset = dma_addr & (PAGE_SIZE - 1); |
281 | unsigned int bytes; |
282 | struct page *page; |
283 | u8 *va; |
284 | |
285 | while (length) { |
286 | page = ib_virt_dma_to_page(dma_addr); |
287 | bytes = min_t(unsigned int, length, |
288 | PAGE_SIZE - page_offset); |
289 | va = kmap_local_page(page); |
290 | |
291 | if (dir == RXE_TO_MR_OBJ) |
292 | memcpy(va + page_offset, addr, bytes); |
293 | else |
294 | memcpy(addr, va + page_offset, bytes); |
295 | |
296 | kunmap_local(va); |
297 | page_offset = 0; |
298 | dma_addr += bytes; |
299 | addr += bytes; |
300 | length -= bytes; |
301 | } |
302 | } |
303 | |
304 | int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, |
305 | unsigned int length, enum rxe_mr_copy_dir dir) |
306 | { |
307 | int err; |
308 | |
309 | if (length == 0) |
310 | return 0; |
311 | |
312 | if (WARN_ON(!mr)) |
313 | return -EINVAL; |
314 | |
315 | if (mr->ibmr.type == IB_MR_TYPE_DMA) { |
316 | rxe_mr_copy_dma(mr, dma_addr: iova, addr, length, dir); |
317 | return 0; |
318 | } |
319 | |
320 | err = mr_check_range(mr, iova, length); |
321 | if (unlikely(err)) { |
322 | rxe_dbg_mr(mr, "iova out of range\n" ); |
323 | return err; |
324 | } |
325 | |
326 | return rxe_mr_copy_xarray(mr, iova, addr, length, dir); |
327 | } |
328 | |
329 | /* copy data in or out of a wqe, i.e. sg list |
330 | * under the control of a dma descriptor |
331 | */ |
332 | int copy_data( |
333 | struct rxe_pd *pd, |
334 | int access, |
335 | struct rxe_dma_info *dma, |
336 | void *addr, |
337 | int length, |
338 | enum rxe_mr_copy_dir dir) |
339 | { |
340 | int bytes; |
341 | struct rxe_sge *sge = &dma->sge[dma->cur_sge]; |
342 | int offset = dma->sge_offset; |
343 | int resid = dma->resid; |
344 | struct rxe_mr *mr = NULL; |
345 | u64 iova; |
346 | int err; |
347 | |
348 | if (length == 0) |
349 | return 0; |
350 | |
351 | if (length > resid) { |
352 | err = -EINVAL; |
353 | goto err2; |
354 | } |
355 | |
356 | if (sge->length && (offset < sge->length)) { |
357 | mr = lookup_mr(pd, access, key: sge->lkey, type: RXE_LOOKUP_LOCAL); |
358 | if (!mr) { |
359 | err = -EINVAL; |
360 | goto err1; |
361 | } |
362 | } |
363 | |
364 | while (length > 0) { |
365 | bytes = length; |
366 | |
367 | if (offset >= sge->length) { |
368 | if (mr) { |
369 | rxe_put(mr); |
370 | mr = NULL; |
371 | } |
372 | sge++; |
373 | dma->cur_sge++; |
374 | offset = 0; |
375 | |
376 | if (dma->cur_sge >= dma->num_sge) { |
377 | err = -ENOSPC; |
378 | goto err2; |
379 | } |
380 | |
381 | if (sge->length) { |
382 | mr = lookup_mr(pd, access, key: sge->lkey, |
383 | type: RXE_LOOKUP_LOCAL); |
384 | if (!mr) { |
385 | err = -EINVAL; |
386 | goto err1; |
387 | } |
388 | } else { |
389 | continue; |
390 | } |
391 | } |
392 | |
393 | if (bytes > sge->length - offset) |
394 | bytes = sge->length - offset; |
395 | |
396 | if (bytes > 0) { |
397 | iova = sge->addr + offset; |
398 | err = rxe_mr_copy(mr, iova, addr, length: bytes, dir); |
399 | if (err) |
400 | goto err2; |
401 | |
402 | offset += bytes; |
403 | resid -= bytes; |
404 | length -= bytes; |
405 | addr += bytes; |
406 | } |
407 | } |
408 | |
409 | dma->sge_offset = offset; |
410 | dma->resid = resid; |
411 | |
412 | if (mr) |
413 | rxe_put(mr); |
414 | |
415 | return 0; |
416 | |
417 | err2: |
418 | if (mr) |
419 | rxe_put(mr); |
420 | err1: |
421 | return err; |
422 | } |
423 | |
424 | int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) |
425 | { |
426 | unsigned int page_offset; |
427 | unsigned long index; |
428 | struct page *page; |
429 | unsigned int bytes; |
430 | int err; |
431 | u8 *va; |
432 | |
433 | /* mr must be valid even if length is zero */ |
434 | if (WARN_ON(!mr)) |
435 | return -EINVAL; |
436 | |
437 | if (length == 0) |
438 | return 0; |
439 | |
440 | if (mr->ibmr.type == IB_MR_TYPE_DMA) |
441 | return -EFAULT; |
442 | |
443 | err = mr_check_range(mr, iova, length); |
444 | if (err) |
445 | return err; |
446 | |
447 | while (length > 0) { |
448 | index = rxe_mr_iova_to_index(mr, iova); |
449 | page = xa_load(&mr->page_list, index); |
450 | page_offset = rxe_mr_iova_to_page_offset(mr, iova); |
451 | if (!page) |
452 | return -EFAULT; |
453 | bytes = min_t(unsigned int, length, |
454 | mr_page_size(mr) - page_offset); |
455 | |
456 | va = kmap_local_page(page); |
457 | arch_wb_cache_pmem(addr: va + page_offset, size: bytes); |
458 | kunmap_local(va); |
459 | |
460 | length -= bytes; |
461 | iova += bytes; |
462 | page_offset = 0; |
463 | } |
464 | |
465 | return 0; |
466 | } |
467 | |
468 | /* Guarantee atomicity of atomic operations at the machine level. */ |
469 | static DEFINE_SPINLOCK(atomic_ops_lock); |
470 | |
471 | int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, |
472 | u64 compare, u64 swap_add, u64 *orig_val) |
473 | { |
474 | unsigned int page_offset; |
475 | struct page *page; |
476 | u64 value; |
477 | u64 *va; |
478 | |
479 | if (unlikely(mr->state != RXE_MR_STATE_VALID)) { |
480 | rxe_dbg_mr(mr, "mr not in valid state\n" ); |
481 | return RESPST_ERR_RKEY_VIOLATION; |
482 | } |
483 | |
484 | if (mr->ibmr.type == IB_MR_TYPE_DMA) { |
485 | page_offset = iova & (PAGE_SIZE - 1); |
486 | page = ib_virt_dma_to_page(dma_addr: iova); |
487 | } else { |
488 | unsigned long index; |
489 | int err; |
490 | |
491 | err = mr_check_range(mr, iova, length: sizeof(value)); |
492 | if (err) { |
493 | rxe_dbg_mr(mr, "iova out of range\n" ); |
494 | return RESPST_ERR_RKEY_VIOLATION; |
495 | } |
496 | page_offset = rxe_mr_iova_to_page_offset(mr, iova); |
497 | index = rxe_mr_iova_to_index(mr, iova); |
498 | page = xa_load(&mr->page_list, index); |
499 | if (!page) |
500 | return RESPST_ERR_RKEY_VIOLATION; |
501 | } |
502 | |
503 | if (unlikely(page_offset & 0x7)) { |
504 | rxe_dbg_mr(mr, "iova not aligned\n" ); |
505 | return RESPST_ERR_MISALIGNED_ATOMIC; |
506 | } |
507 | |
508 | va = kmap_local_page(page); |
509 | |
510 | spin_lock_bh(lock: &atomic_ops_lock); |
511 | value = *orig_val = va[page_offset >> 3]; |
512 | |
513 | if (opcode == IB_OPCODE_RC_COMPARE_SWAP) { |
514 | if (value == compare) |
515 | va[page_offset >> 3] = swap_add; |
516 | } else { |
517 | value += swap_add; |
518 | va[page_offset >> 3] = value; |
519 | } |
520 | spin_unlock_bh(lock: &atomic_ops_lock); |
521 | |
522 | kunmap_local(va); |
523 | |
524 | return 0; |
525 | } |
526 | |
527 | #if defined CONFIG_64BIT |
528 | /* only implemented or called for 64 bit architectures */ |
529 | int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) |
530 | { |
531 | unsigned int page_offset; |
532 | struct page *page; |
533 | u64 *va; |
534 | |
535 | /* See IBA oA19-28 */ |
536 | if (unlikely(mr->state != RXE_MR_STATE_VALID)) { |
537 | rxe_dbg_mr(mr, "mr not in valid state\n" ); |
538 | return RESPST_ERR_RKEY_VIOLATION; |
539 | } |
540 | |
541 | if (mr->ibmr.type == IB_MR_TYPE_DMA) { |
542 | page_offset = iova & (PAGE_SIZE - 1); |
543 | page = ib_virt_dma_to_page(dma_addr: iova); |
544 | } else { |
545 | unsigned long index; |
546 | int err; |
547 | |
548 | /* See IBA oA19-28 */ |
549 | err = mr_check_range(mr, iova, length: sizeof(value)); |
550 | if (unlikely(err)) { |
551 | rxe_dbg_mr(mr, "iova out of range\n" ); |
552 | return RESPST_ERR_RKEY_VIOLATION; |
553 | } |
554 | page_offset = rxe_mr_iova_to_page_offset(mr, iova); |
555 | index = rxe_mr_iova_to_index(mr, iova); |
556 | page = xa_load(&mr->page_list, index); |
557 | if (!page) |
558 | return RESPST_ERR_RKEY_VIOLATION; |
559 | } |
560 | |
561 | /* See IBA A19.4.2 */ |
562 | if (unlikely(page_offset & 0x7)) { |
563 | rxe_dbg_mr(mr, "misaligned address\n" ); |
564 | return RESPST_ERR_MISALIGNED_ATOMIC; |
565 | } |
566 | |
567 | va = kmap_local_page(page); |
568 | |
569 | /* Do atomic write after all prior operations have completed */ |
570 | smp_store_release(&va[page_offset >> 3], value); |
571 | |
572 | kunmap_local(va); |
573 | |
574 | return 0; |
575 | } |
576 | #else |
577 | int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) |
578 | { |
579 | return RESPST_ERR_UNSUPPORTED_OPCODE; |
580 | } |
581 | #endif |
582 | |
583 | int advance_dma_data(struct rxe_dma_info *dma, unsigned int length) |
584 | { |
585 | struct rxe_sge *sge = &dma->sge[dma->cur_sge]; |
586 | int offset = dma->sge_offset; |
587 | int resid = dma->resid; |
588 | |
589 | while (length) { |
590 | unsigned int bytes; |
591 | |
592 | if (offset >= sge->length) { |
593 | sge++; |
594 | dma->cur_sge++; |
595 | offset = 0; |
596 | if (dma->cur_sge >= dma->num_sge) |
597 | return -ENOSPC; |
598 | } |
599 | |
600 | bytes = length; |
601 | |
602 | if (bytes > sge->length - offset) |
603 | bytes = sge->length - offset; |
604 | |
605 | offset += bytes; |
606 | resid -= bytes; |
607 | length -= bytes; |
608 | } |
609 | |
610 | dma->sge_offset = offset; |
611 | dma->resid = resid; |
612 | |
613 | return 0; |
614 | } |
615 | |
616 | struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key, |
617 | enum rxe_mr_lookup_type type) |
618 | { |
619 | struct rxe_mr *mr; |
620 | struct rxe_dev *rxe = to_rdev(dev: pd->ibpd.device); |
621 | int index = key >> 8; |
622 | |
623 | mr = rxe_pool_get_index(pool: &rxe->mr_pool, index); |
624 | if (!mr) |
625 | return NULL; |
626 | |
627 | if (unlikely((type == RXE_LOOKUP_LOCAL && mr->lkey != key) || |
628 | (type == RXE_LOOKUP_REMOTE && mr->rkey != key) || |
629 | mr_pd(mr) != pd || ((access & mr->access) != access) || |
630 | mr->state != RXE_MR_STATE_VALID)) { |
631 | rxe_put(mr); |
632 | mr = NULL; |
633 | } |
634 | |
635 | return mr; |
636 | } |
637 | |
638 | int rxe_invalidate_mr(struct rxe_qp *qp, u32 key) |
639 | { |
640 | struct rxe_dev *rxe = to_rdev(dev: qp->ibqp.device); |
641 | struct rxe_mr *mr; |
642 | int remote; |
643 | int ret; |
644 | |
645 | mr = rxe_pool_get_index(pool: &rxe->mr_pool, index: key >> 8); |
646 | if (!mr) { |
647 | rxe_dbg_qp(qp, "No MR for key %#x\n" , key); |
648 | ret = -EINVAL; |
649 | goto err; |
650 | } |
651 | |
652 | remote = mr->access & RXE_ACCESS_REMOTE; |
653 | if (remote ? (key != mr->rkey) : (key != mr->lkey)) { |
654 | rxe_dbg_mr(mr, "wr key (%#x) doesn't match mr key (%#x)\n" , |
655 | key, (remote ? mr->rkey : mr->lkey)); |
656 | ret = -EINVAL; |
657 | goto err_drop_ref; |
658 | } |
659 | |
660 | if (atomic_read(v: &mr->num_mw) > 0) { |
661 | rxe_dbg_mr(mr, "Attempt to invalidate an MR while bound to MWs\n" ); |
662 | ret = -EINVAL; |
663 | goto err_drop_ref; |
664 | } |
665 | |
666 | if (unlikely(mr->ibmr.type != IB_MR_TYPE_MEM_REG)) { |
667 | rxe_dbg_mr(mr, "Type (%d) is wrong\n" , mr->ibmr.type); |
668 | ret = -EINVAL; |
669 | goto err_drop_ref; |
670 | } |
671 | |
672 | mr->state = RXE_MR_STATE_FREE; |
673 | ret = 0; |
674 | |
675 | err_drop_ref: |
676 | rxe_put(mr); |
677 | err: |
678 | return ret; |
679 | } |
680 | |
681 | /* user can (re)register fast MR by executing a REG_MR WQE. |
682 | * user is expected to hold a reference on the ib mr until the |
683 | * WQE completes. |
684 | * Once a fast MR is created this is the only way to change the |
685 | * private keys. It is the responsibility of the user to maintain |
686 | * the ib mr keys in sync with rxe mr keys. |
687 | */ |
688 | int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe) |
689 | { |
690 | struct rxe_mr *mr = to_rmr(mr: wqe->wr.wr.reg.mr); |
691 | u32 key = wqe->wr.wr.reg.key; |
692 | u32 access = wqe->wr.wr.reg.access; |
693 | |
694 | /* user can only register MR in free state */ |
695 | if (unlikely(mr->state != RXE_MR_STATE_FREE)) { |
696 | rxe_dbg_mr(mr, "mr->lkey = 0x%x not free\n" , mr->lkey); |
697 | return -EINVAL; |
698 | } |
699 | |
700 | /* user can only register mr with qp in same protection domain */ |
701 | if (unlikely(qp->ibqp.pd != mr->ibmr.pd)) { |
702 | rxe_dbg_mr(mr, "qp->pd and mr->pd don't match\n" ); |
703 | return -EINVAL; |
704 | } |
705 | |
706 | /* user is only allowed to change key portion of l/rkey */ |
707 | if (unlikely((mr->lkey & ~0xff) != (key & ~0xff))) { |
708 | rxe_dbg_mr(mr, "key = 0x%x has wrong index mr->lkey = 0x%x\n" , |
709 | key, mr->lkey); |
710 | return -EINVAL; |
711 | } |
712 | |
713 | mr->access = access; |
714 | mr->lkey = key; |
715 | mr->rkey = key; |
716 | mr->ibmr.iova = wqe->wr.wr.reg.mr->iova; |
717 | mr->state = RXE_MR_STATE_VALID; |
718 | |
719 | return 0; |
720 | } |
721 | |
722 | void rxe_mr_cleanup(struct rxe_pool_elem *elem) |
723 | { |
724 | struct rxe_mr *mr = container_of(elem, typeof(*mr), elem); |
725 | |
726 | rxe_put(mr_pd(mr)); |
727 | ib_umem_release(umem: mr->umem); |
728 | |
729 | if (mr->ibmr.type != IB_MR_TYPE_DMA) |
730 | xa_destroy(&mr->page_list); |
731 | } |
732 | |