1 | // SPDX-License-Identifier: MIT |
2 | |
3 | /* |
4 | * Locking: |
5 | * |
6 | * The uvmm mutex protects any operations on the GPU VA space provided by the |
7 | * DRM GPU VA manager. |
8 | * |
9 | * The GEMs dma_resv lock protects the GEMs GPUVA list, hence link/unlink of a |
10 | * mapping to it's backing GEM must be performed under this lock. |
11 | * |
12 | * Actual map/unmap operations within the fence signalling critical path are |
13 | * protected by installing DMA fences to the corresponding GEMs DMA |
14 | * reservations, such that concurrent BO moves, which itself walk the GEMs GPUVA |
15 | * list in order to map/unmap it's entries, can't occur concurrently. |
16 | * |
17 | * Accessing the DRM_GPUVA_INVALIDATED flag doesn't need any separate |
18 | * protection, since there are no accesses other than from BO move callbacks |
19 | * and from the fence signalling critical path, which are already protected by |
20 | * the corresponding GEMs DMA reservation fence. |
21 | */ |
22 | |
23 | #include "nouveau_drv.h" |
24 | #include "nouveau_gem.h" |
25 | #include "nouveau_mem.h" |
26 | #include "nouveau_uvmm.h" |
27 | |
28 | #include <nvif/vmm.h> |
29 | #include <nvif/mem.h> |
30 | |
31 | #include <nvif/class.h> |
32 | #include <nvif/if000c.h> |
33 | #include <nvif/if900d.h> |
34 | |
35 | #define NOUVEAU_VA_SPACE_BITS 47 /* FIXME */ |
36 | #define NOUVEAU_VA_SPACE_START 0x0 |
37 | #define NOUVEAU_VA_SPACE_END (1ULL << NOUVEAU_VA_SPACE_BITS) |
38 | |
39 | #define list_last_op(_ops) list_last_entry(_ops, struct bind_job_op, entry) |
40 | #define list_prev_op(_op) list_prev_entry(_op, entry) |
41 | #define list_for_each_op(_op, _ops) list_for_each_entry(_op, _ops, entry) |
42 | #define list_for_each_op_from_reverse(_op, _ops) \ |
43 | list_for_each_entry_from_reverse(_op, _ops, entry) |
44 | #define list_for_each_op_safe(_op, _n, _ops) list_for_each_entry_safe(_op, _n, _ops, entry) |
45 | |
46 | enum vm_bind_op { |
47 | OP_MAP = DRM_NOUVEAU_VM_BIND_OP_MAP, |
48 | OP_UNMAP = DRM_NOUVEAU_VM_BIND_OP_UNMAP, |
49 | OP_MAP_SPARSE, |
50 | OP_UNMAP_SPARSE, |
51 | }; |
52 | |
53 | struct nouveau_uvma_prealloc { |
54 | struct nouveau_uvma *map; |
55 | struct nouveau_uvma *prev; |
56 | struct nouveau_uvma *next; |
57 | }; |
58 | |
59 | struct bind_job_op { |
60 | struct list_head entry; |
61 | |
62 | enum vm_bind_op op; |
63 | u32 flags; |
64 | |
65 | struct drm_gpuvm_bo *vm_bo; |
66 | |
67 | struct { |
68 | u64 addr; |
69 | u64 range; |
70 | } va; |
71 | |
72 | struct { |
73 | u32 handle; |
74 | u64 offset; |
75 | struct drm_gem_object *obj; |
76 | } gem; |
77 | |
78 | struct nouveau_uvma_region *reg; |
79 | struct nouveau_uvma_prealloc new; |
80 | struct drm_gpuva_ops *ops; |
81 | }; |
82 | |
83 | struct uvmm_map_args { |
84 | struct nouveau_uvma_region *region; |
85 | u64 addr; |
86 | u64 range; |
87 | u8 kind; |
88 | }; |
89 | |
90 | static int |
91 | nouveau_uvmm_vmm_sparse_ref(struct nouveau_uvmm *uvmm, |
92 | u64 addr, u64 range) |
93 | { |
94 | struct nvif_vmm *vmm = &uvmm->vmm.vmm; |
95 | |
96 | return nvif_vmm_raw_sparse(vmm, addr, range, true); |
97 | } |
98 | |
99 | static int |
100 | nouveau_uvmm_vmm_sparse_unref(struct nouveau_uvmm *uvmm, |
101 | u64 addr, u64 range) |
102 | { |
103 | struct nvif_vmm *vmm = &uvmm->vmm.vmm; |
104 | |
105 | return nvif_vmm_raw_sparse(vmm, addr, range, false); |
106 | } |
107 | |
108 | static int |
109 | nouveau_uvmm_vmm_get(struct nouveau_uvmm *uvmm, |
110 | u64 addr, u64 range) |
111 | { |
112 | struct nvif_vmm *vmm = &uvmm->vmm.vmm; |
113 | |
114 | return nvif_vmm_raw_get(vmm, addr, range, PAGE_SHIFT); |
115 | } |
116 | |
117 | static int |
118 | nouveau_uvmm_vmm_put(struct nouveau_uvmm *uvmm, |
119 | u64 addr, u64 range) |
120 | { |
121 | struct nvif_vmm *vmm = &uvmm->vmm.vmm; |
122 | |
123 | return nvif_vmm_raw_put(vmm, addr, range, PAGE_SHIFT); |
124 | } |
125 | |
126 | static int |
127 | nouveau_uvmm_vmm_unmap(struct nouveau_uvmm *uvmm, |
128 | u64 addr, u64 range, bool sparse) |
129 | { |
130 | struct nvif_vmm *vmm = &uvmm->vmm.vmm; |
131 | |
132 | return nvif_vmm_raw_unmap(vmm, addr, range, PAGE_SHIFT, sparse); |
133 | } |
134 | |
135 | static int |
136 | nouveau_uvmm_vmm_map(struct nouveau_uvmm *uvmm, |
137 | u64 addr, u64 range, |
138 | u64 bo_offset, u8 kind, |
139 | struct nouveau_mem *mem) |
140 | { |
141 | struct nvif_vmm *vmm = &uvmm->vmm.vmm; |
142 | union { |
143 | struct gf100_vmm_map_v0 gf100; |
144 | } args; |
145 | u32 argc = 0; |
146 | |
147 | switch (vmm->object.oclass) { |
148 | case NVIF_CLASS_VMM_GF100: |
149 | case NVIF_CLASS_VMM_GM200: |
150 | case NVIF_CLASS_VMM_GP100: |
151 | args.gf100.version = 0; |
152 | if (mem->mem.type & NVIF_MEM_VRAM) |
153 | args.gf100.vol = 0; |
154 | else |
155 | args.gf100.vol = 1; |
156 | args.gf100.ro = 0; |
157 | args.gf100.priv = 0; |
158 | args.gf100.kind = kind; |
159 | argc = sizeof(args.gf100); |
160 | break; |
161 | default: |
162 | WARN_ON(1); |
163 | return -ENOSYS; |
164 | } |
165 | |
166 | return nvif_vmm_raw_map(vmm, addr, range, PAGE_SHIFT, |
167 | &args, argc, |
168 | &mem->mem, bo_offset); |
169 | } |
170 | |
171 | static int |
172 | nouveau_uvma_region_sparse_unref(struct nouveau_uvma_region *reg) |
173 | { |
174 | u64 addr = reg->va.addr; |
175 | u64 range = reg->va.range; |
176 | |
177 | return nouveau_uvmm_vmm_sparse_unref(uvmm: reg->uvmm, addr, range); |
178 | } |
179 | |
180 | static int |
181 | nouveau_uvma_vmm_put(struct nouveau_uvma *uvma) |
182 | { |
183 | u64 addr = uvma->va.va.addr; |
184 | u64 range = uvma->va.va.range; |
185 | |
186 | return nouveau_uvmm_vmm_put(to_uvmm(uvma), addr, range); |
187 | } |
188 | |
189 | static int |
190 | nouveau_uvma_map(struct nouveau_uvma *uvma, |
191 | struct nouveau_mem *mem) |
192 | { |
193 | u64 addr = uvma->va.va.addr; |
194 | u64 offset = uvma->va.gem.offset; |
195 | u64 range = uvma->va.va.range; |
196 | |
197 | return nouveau_uvmm_vmm_map(to_uvmm(uvma), addr, range, |
198 | bo_offset: offset, kind: uvma->kind, mem); |
199 | } |
200 | |
201 | static int |
202 | nouveau_uvma_unmap(struct nouveau_uvma *uvma) |
203 | { |
204 | u64 addr = uvma->va.va.addr; |
205 | u64 range = uvma->va.va.range; |
206 | bool sparse = !!uvma->region; |
207 | |
208 | if (drm_gpuva_invalidated(va: &uvma->va)) |
209 | return 0; |
210 | |
211 | return nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, sparse); |
212 | } |
213 | |
214 | static int |
215 | nouveau_uvma_alloc(struct nouveau_uvma **puvma) |
216 | { |
217 | *puvma = kzalloc(size: sizeof(**puvma), GFP_KERNEL); |
218 | if (!*puvma) |
219 | return -ENOMEM; |
220 | |
221 | return 0; |
222 | } |
223 | |
224 | static void |
225 | nouveau_uvma_free(struct nouveau_uvma *uvma) |
226 | { |
227 | kfree(objp: uvma); |
228 | } |
229 | |
230 | static void |
231 | nouveau_uvma_gem_get(struct nouveau_uvma *uvma) |
232 | { |
233 | drm_gem_object_get(obj: uvma->va.gem.obj); |
234 | } |
235 | |
236 | static void |
237 | nouveau_uvma_gem_put(struct nouveau_uvma *uvma) |
238 | { |
239 | drm_gem_object_put(obj: uvma->va.gem.obj); |
240 | } |
241 | |
242 | static int |
243 | nouveau_uvma_region_alloc(struct nouveau_uvma_region **preg) |
244 | { |
245 | *preg = kzalloc(size: sizeof(**preg), GFP_KERNEL); |
246 | if (!*preg) |
247 | return -ENOMEM; |
248 | |
249 | kref_init(kref: &(*preg)->kref); |
250 | |
251 | return 0; |
252 | } |
253 | |
254 | static void |
255 | nouveau_uvma_region_free(struct kref *kref) |
256 | { |
257 | struct nouveau_uvma_region *reg = |
258 | container_of(kref, struct nouveau_uvma_region, kref); |
259 | |
260 | kfree(objp: reg); |
261 | } |
262 | |
263 | static void |
264 | nouveau_uvma_region_get(struct nouveau_uvma_region *reg) |
265 | { |
266 | kref_get(kref: ®->kref); |
267 | } |
268 | |
269 | static void |
270 | nouveau_uvma_region_put(struct nouveau_uvma_region *reg) |
271 | { |
272 | kref_put(kref: ®->kref, release: nouveau_uvma_region_free); |
273 | } |
274 | |
275 | static int |
276 | __nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm, |
277 | struct nouveau_uvma_region *reg) |
278 | { |
279 | u64 addr = reg->va.addr; |
280 | u64 range = reg->va.range; |
281 | u64 last = addr + range - 1; |
282 | MA_STATE(mas, &uvmm->region_mt, addr, addr); |
283 | |
284 | if (unlikely(mas_walk(&mas))) |
285 | return -EEXIST; |
286 | |
287 | if (unlikely(mas.last < last)) |
288 | return -EEXIST; |
289 | |
290 | mas.index = addr; |
291 | mas.last = last; |
292 | |
293 | mas_store_gfp(mas: &mas, entry: reg, GFP_KERNEL); |
294 | |
295 | reg->uvmm = uvmm; |
296 | |
297 | return 0; |
298 | } |
299 | |
300 | static int |
301 | nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm, |
302 | struct nouveau_uvma_region *reg, |
303 | u64 addr, u64 range) |
304 | { |
305 | int ret; |
306 | |
307 | reg->uvmm = uvmm; |
308 | reg->va.addr = addr; |
309 | reg->va.range = range; |
310 | |
311 | ret = __nouveau_uvma_region_insert(uvmm, reg); |
312 | if (ret) |
313 | return ret; |
314 | |
315 | return 0; |
316 | } |
317 | |
318 | static void |
319 | nouveau_uvma_region_remove(struct nouveau_uvma_region *reg) |
320 | { |
321 | struct nouveau_uvmm *uvmm = reg->uvmm; |
322 | MA_STATE(mas, &uvmm->region_mt, reg->va.addr, 0); |
323 | |
324 | mas_erase(mas: &mas); |
325 | } |
326 | |
327 | static int |
328 | nouveau_uvma_region_create(struct nouveau_uvmm *uvmm, |
329 | u64 addr, u64 range) |
330 | { |
331 | struct nouveau_uvma_region *reg; |
332 | int ret; |
333 | |
334 | if (!drm_gpuvm_interval_empty(gpuvm: &uvmm->base, addr, range)) |
335 | return -ENOSPC; |
336 | |
337 | ret = nouveau_uvma_region_alloc(preg: ®); |
338 | if (ret) |
339 | return ret; |
340 | |
341 | ret = nouveau_uvma_region_insert(uvmm, reg, addr, range); |
342 | if (ret) |
343 | goto err_free_region; |
344 | |
345 | ret = nouveau_uvmm_vmm_sparse_ref(uvmm, addr, range); |
346 | if (ret) |
347 | goto err_region_remove; |
348 | |
349 | return 0; |
350 | |
351 | err_region_remove: |
352 | nouveau_uvma_region_remove(reg); |
353 | err_free_region: |
354 | nouveau_uvma_region_put(reg); |
355 | return ret; |
356 | } |
357 | |
358 | static struct nouveau_uvma_region * |
359 | nouveau_uvma_region_find_first(struct nouveau_uvmm *uvmm, |
360 | u64 addr, u64 range) |
361 | { |
362 | MA_STATE(mas, &uvmm->region_mt, addr, 0); |
363 | |
364 | return mas_find(mas: &mas, max: addr + range - 1); |
365 | } |
366 | |
367 | static struct nouveau_uvma_region * |
368 | nouveau_uvma_region_find(struct nouveau_uvmm *uvmm, |
369 | u64 addr, u64 range) |
370 | { |
371 | struct nouveau_uvma_region *reg; |
372 | |
373 | reg = nouveau_uvma_region_find_first(uvmm, addr, range); |
374 | if (!reg) |
375 | return NULL; |
376 | |
377 | if (reg->va.addr != addr || |
378 | reg->va.range != range) |
379 | return NULL; |
380 | |
381 | return reg; |
382 | } |
383 | |
384 | static bool |
385 | nouveau_uvma_region_empty(struct nouveau_uvma_region *reg) |
386 | { |
387 | struct nouveau_uvmm *uvmm = reg->uvmm; |
388 | |
389 | return drm_gpuvm_interval_empty(gpuvm: &uvmm->base, |
390 | addr: reg->va.addr, |
391 | range: reg->va.range); |
392 | } |
393 | |
394 | static int |
395 | __nouveau_uvma_region_destroy(struct nouveau_uvma_region *reg) |
396 | { |
397 | struct nouveau_uvmm *uvmm = reg->uvmm; |
398 | u64 addr = reg->va.addr; |
399 | u64 range = reg->va.range; |
400 | |
401 | if (!nouveau_uvma_region_empty(reg)) |
402 | return -EBUSY; |
403 | |
404 | nouveau_uvma_region_remove(reg); |
405 | nouveau_uvmm_vmm_sparse_unref(uvmm, addr, range); |
406 | nouveau_uvma_region_put(reg); |
407 | |
408 | return 0; |
409 | } |
410 | |
411 | static int |
412 | nouveau_uvma_region_destroy(struct nouveau_uvmm *uvmm, |
413 | u64 addr, u64 range) |
414 | { |
415 | struct nouveau_uvma_region *reg; |
416 | |
417 | reg = nouveau_uvma_region_find(uvmm, addr, range); |
418 | if (!reg) |
419 | return -ENOENT; |
420 | |
421 | return __nouveau_uvma_region_destroy(reg); |
422 | } |
423 | |
424 | static void |
425 | nouveau_uvma_region_dirty(struct nouveau_uvma_region *reg) |
426 | { |
427 | |
428 | init_completion(x: ®->complete); |
429 | reg->dirty = true; |
430 | } |
431 | |
432 | static void |
433 | nouveau_uvma_region_complete(struct nouveau_uvma_region *reg) |
434 | { |
435 | complete_all(®->complete); |
436 | } |
437 | |
438 | static void |
439 | op_map_prepare_unwind(struct nouveau_uvma *uvma) |
440 | { |
441 | struct drm_gpuva *va = &uvma->va; |
442 | nouveau_uvma_gem_put(uvma); |
443 | drm_gpuva_remove(va); |
444 | nouveau_uvma_free(uvma); |
445 | } |
446 | |
447 | static void |
448 | op_unmap_prepare_unwind(struct drm_gpuva *va) |
449 | { |
450 | drm_gpuva_insert(gpuvm: va->vm, va); |
451 | } |
452 | |
453 | static void |
454 | nouveau_uvmm_sm_prepare_unwind(struct nouveau_uvmm *uvmm, |
455 | struct nouveau_uvma_prealloc *new, |
456 | struct drm_gpuva_ops *ops, |
457 | struct drm_gpuva_op *last, |
458 | struct uvmm_map_args *args) |
459 | { |
460 | struct drm_gpuva_op *op = last; |
461 | u64 vmm_get_start = args ? args->addr : 0; |
462 | u64 vmm_get_end = args ? args->addr + args->range : 0; |
463 | |
464 | /* Unwind GPUVA space. */ |
465 | drm_gpuva_for_each_op_from_reverse(op, ops) { |
466 | switch (op->op) { |
467 | case DRM_GPUVA_OP_MAP: |
468 | op_map_prepare_unwind(uvma: new->map); |
469 | break; |
470 | case DRM_GPUVA_OP_REMAP: { |
471 | struct drm_gpuva_op_remap *r = &op->remap; |
472 | struct drm_gpuva *va = r->unmap->va; |
473 | |
474 | if (r->next) |
475 | op_map_prepare_unwind(uvma: new->next); |
476 | |
477 | if (r->prev) |
478 | op_map_prepare_unwind(uvma: new->prev); |
479 | |
480 | op_unmap_prepare_unwind(va); |
481 | break; |
482 | } |
483 | case DRM_GPUVA_OP_UNMAP: |
484 | op_unmap_prepare_unwind(va: op->unmap.va); |
485 | break; |
486 | default: |
487 | break; |
488 | } |
489 | } |
490 | |
491 | /* Unmap operation don't allocate page tables, hence skip the following |
492 | * page table unwind. |
493 | */ |
494 | if (!args) |
495 | return; |
496 | |
497 | drm_gpuva_for_each_op(op, ops) { |
498 | switch (op->op) { |
499 | case DRM_GPUVA_OP_MAP: { |
500 | u64 vmm_get_range = vmm_get_end - vmm_get_start; |
501 | |
502 | if (vmm_get_range) |
503 | nouveau_uvmm_vmm_put(uvmm, addr: vmm_get_start, |
504 | range: vmm_get_range); |
505 | break; |
506 | } |
507 | case DRM_GPUVA_OP_REMAP: { |
508 | struct drm_gpuva_op_remap *r = &op->remap; |
509 | struct drm_gpuva *va = r->unmap->va; |
510 | u64 ustart = va->va.addr; |
511 | u64 urange = va->va.range; |
512 | u64 uend = ustart + urange; |
513 | |
514 | if (r->prev) |
515 | vmm_get_start = uend; |
516 | |
517 | if (r->next) |
518 | vmm_get_end = ustart; |
519 | |
520 | if (r->prev && r->next) |
521 | vmm_get_start = vmm_get_end = 0; |
522 | |
523 | break; |
524 | } |
525 | case DRM_GPUVA_OP_UNMAP: { |
526 | struct drm_gpuva_op_unmap *u = &op->unmap; |
527 | struct drm_gpuva *va = u->va; |
528 | u64 ustart = va->va.addr; |
529 | u64 urange = va->va.range; |
530 | u64 uend = ustart + urange; |
531 | |
532 | /* Nothing to do for mappings we merge with. */ |
533 | if (uend == vmm_get_start || |
534 | ustart == vmm_get_end) |
535 | break; |
536 | |
537 | if (ustart > vmm_get_start) { |
538 | u64 vmm_get_range = ustart - vmm_get_start; |
539 | |
540 | nouveau_uvmm_vmm_put(uvmm, addr: vmm_get_start, |
541 | range: vmm_get_range); |
542 | } |
543 | vmm_get_start = uend; |
544 | break; |
545 | } |
546 | default: |
547 | break; |
548 | } |
549 | |
550 | if (op == last) |
551 | break; |
552 | } |
553 | } |
554 | |
555 | static void |
556 | nouveau_uvmm_sm_map_prepare_unwind(struct nouveau_uvmm *uvmm, |
557 | struct nouveau_uvma_prealloc *new, |
558 | struct drm_gpuva_ops *ops, |
559 | u64 addr, u64 range) |
560 | { |
561 | struct drm_gpuva_op *last = drm_gpuva_last_op(ops); |
562 | struct uvmm_map_args args = { |
563 | .addr = addr, |
564 | .range = range, |
565 | }; |
566 | |
567 | nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, args: &args); |
568 | } |
569 | |
570 | static void |
571 | nouveau_uvmm_sm_unmap_prepare_unwind(struct nouveau_uvmm *uvmm, |
572 | struct nouveau_uvma_prealloc *new, |
573 | struct drm_gpuva_ops *ops) |
574 | { |
575 | struct drm_gpuva_op *last = drm_gpuva_last_op(ops); |
576 | |
577 | nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, NULL); |
578 | } |
579 | |
580 | static int |
581 | op_map_prepare(struct nouveau_uvmm *uvmm, |
582 | struct nouveau_uvma **puvma, |
583 | struct drm_gpuva_op_map *op, |
584 | struct uvmm_map_args *args) |
585 | { |
586 | struct nouveau_uvma *uvma; |
587 | int ret; |
588 | |
589 | ret = nouveau_uvma_alloc(puvma: &uvma); |
590 | if (ret) |
591 | return ret; |
592 | |
593 | uvma->region = args->region; |
594 | uvma->kind = args->kind; |
595 | |
596 | drm_gpuva_map(gpuvm: &uvmm->base, va: &uvma->va, op); |
597 | |
598 | /* Keep a reference until this uvma is destroyed. */ |
599 | nouveau_uvma_gem_get(uvma); |
600 | |
601 | *puvma = uvma; |
602 | return 0; |
603 | } |
604 | |
605 | static void |
606 | op_unmap_prepare(struct drm_gpuva_op_unmap *u) |
607 | { |
608 | drm_gpuva_unmap(op: u); |
609 | } |
610 | |
611 | /* |
612 | * Note: @args should not be NULL when calling for a map operation. |
613 | */ |
614 | static int |
615 | nouveau_uvmm_sm_prepare(struct nouveau_uvmm *uvmm, |
616 | struct nouveau_uvma_prealloc *new, |
617 | struct drm_gpuva_ops *ops, |
618 | struct uvmm_map_args *args) |
619 | { |
620 | struct drm_gpuva_op *op; |
621 | u64 vmm_get_start = args ? args->addr : 0; |
622 | u64 vmm_get_end = args ? args->addr + args->range : 0; |
623 | int ret; |
624 | |
625 | drm_gpuva_for_each_op(op, ops) { |
626 | switch (op->op) { |
627 | case DRM_GPUVA_OP_MAP: { |
628 | u64 vmm_get_range = vmm_get_end - vmm_get_start; |
629 | |
630 | ret = op_map_prepare(uvmm, puvma: &new->map, op: &op->map, args); |
631 | if (ret) |
632 | goto unwind; |
633 | |
634 | if (vmm_get_range) { |
635 | ret = nouveau_uvmm_vmm_get(uvmm, addr: vmm_get_start, |
636 | range: vmm_get_range); |
637 | if (ret) { |
638 | op_map_prepare_unwind(uvma: new->map); |
639 | goto unwind; |
640 | } |
641 | } |
642 | |
643 | break; |
644 | } |
645 | case DRM_GPUVA_OP_REMAP: { |
646 | struct drm_gpuva_op_remap *r = &op->remap; |
647 | struct drm_gpuva *va = r->unmap->va; |
648 | struct uvmm_map_args remap_args = { |
649 | .kind = uvma_from_va(va)->kind, |
650 | .region = uvma_from_va(va)->region, |
651 | }; |
652 | u64 ustart = va->va.addr; |
653 | u64 urange = va->va.range; |
654 | u64 uend = ustart + urange; |
655 | |
656 | op_unmap_prepare(u: r->unmap); |
657 | |
658 | if (r->prev) { |
659 | ret = op_map_prepare(uvmm, puvma: &new->prev, op: r->prev, |
660 | args: &remap_args); |
661 | if (ret) |
662 | goto unwind; |
663 | |
664 | if (args) |
665 | vmm_get_start = uend; |
666 | } |
667 | |
668 | if (r->next) { |
669 | ret = op_map_prepare(uvmm, puvma: &new->next, op: r->next, |
670 | args: &remap_args); |
671 | if (ret) { |
672 | if (r->prev) |
673 | op_map_prepare_unwind(uvma: new->prev); |
674 | goto unwind; |
675 | } |
676 | |
677 | if (args) |
678 | vmm_get_end = ustart; |
679 | } |
680 | |
681 | if (args && (r->prev && r->next)) |
682 | vmm_get_start = vmm_get_end = 0; |
683 | |
684 | break; |
685 | } |
686 | case DRM_GPUVA_OP_UNMAP: { |
687 | struct drm_gpuva_op_unmap *u = &op->unmap; |
688 | struct drm_gpuva *va = u->va; |
689 | u64 ustart = va->va.addr; |
690 | u64 urange = va->va.range; |
691 | u64 uend = ustart + urange; |
692 | |
693 | op_unmap_prepare(u); |
694 | |
695 | if (!args) |
696 | break; |
697 | |
698 | /* Nothing to do for mappings we merge with. */ |
699 | if (uend == vmm_get_start || |
700 | ustart == vmm_get_end) |
701 | break; |
702 | |
703 | if (ustart > vmm_get_start) { |
704 | u64 vmm_get_range = ustart - vmm_get_start; |
705 | |
706 | ret = nouveau_uvmm_vmm_get(uvmm, addr: vmm_get_start, |
707 | range: vmm_get_range); |
708 | if (ret) { |
709 | op_unmap_prepare_unwind(va); |
710 | goto unwind; |
711 | } |
712 | } |
713 | vmm_get_start = uend; |
714 | |
715 | break; |
716 | } |
717 | default: |
718 | ret = -EINVAL; |
719 | goto unwind; |
720 | } |
721 | } |
722 | |
723 | return 0; |
724 | |
725 | unwind: |
726 | if (op != drm_gpuva_first_op(ops)) |
727 | nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, |
728 | drm_gpuva_prev_op(op), |
729 | args); |
730 | return ret; |
731 | } |
732 | |
733 | static int |
734 | nouveau_uvmm_sm_map_prepare(struct nouveau_uvmm *uvmm, |
735 | struct nouveau_uvma_prealloc *new, |
736 | struct nouveau_uvma_region *region, |
737 | struct drm_gpuva_ops *ops, |
738 | u64 addr, u64 range, u8 kind) |
739 | { |
740 | struct uvmm_map_args args = { |
741 | .region = region, |
742 | .addr = addr, |
743 | .range = range, |
744 | .kind = kind, |
745 | }; |
746 | |
747 | return nouveau_uvmm_sm_prepare(uvmm, new, ops, args: &args); |
748 | } |
749 | |
750 | static int |
751 | nouveau_uvmm_sm_unmap_prepare(struct nouveau_uvmm *uvmm, |
752 | struct nouveau_uvma_prealloc *new, |
753 | struct drm_gpuva_ops *ops) |
754 | { |
755 | return nouveau_uvmm_sm_prepare(uvmm, new, ops, NULL); |
756 | } |
757 | |
758 | static struct drm_gem_object * |
759 | op_gem_obj(struct drm_gpuva_op *op) |
760 | { |
761 | switch (op->op) { |
762 | case DRM_GPUVA_OP_MAP: |
763 | return op->map.gem.obj; |
764 | case DRM_GPUVA_OP_REMAP: |
765 | /* Actually, we're looking for the GEMs backing remap.prev and |
766 | * remap.next, but since this is a remap they're identical to |
767 | * the GEM backing the unmapped GPUVA. |
768 | */ |
769 | return op->remap.unmap->va->gem.obj; |
770 | case DRM_GPUVA_OP_UNMAP: |
771 | return op->unmap.va->gem.obj; |
772 | default: |
773 | WARN(1, "Unknown operation.\n" ); |
774 | return NULL; |
775 | } |
776 | } |
777 | |
778 | static void |
779 | op_map(struct nouveau_uvma *uvma) |
780 | { |
781 | struct nouveau_bo *nvbo = nouveau_gem_object(gem: uvma->va.gem.obj); |
782 | |
783 | nouveau_uvma_map(uvma, mem: nouveau_mem(reg: nvbo->bo.resource)); |
784 | } |
785 | |
786 | static void |
787 | op_unmap(struct drm_gpuva_op_unmap *u) |
788 | { |
789 | struct drm_gpuva *va = u->va; |
790 | struct nouveau_uvma *uvma = uvma_from_va(va); |
791 | |
792 | /* nouveau_uvma_unmap() does not unmap if backing BO is evicted. */ |
793 | if (!u->keep) |
794 | nouveau_uvma_unmap(uvma); |
795 | } |
796 | |
797 | static void |
798 | op_unmap_range(struct drm_gpuva_op_unmap *u, |
799 | u64 addr, u64 range) |
800 | { |
801 | struct nouveau_uvma *uvma = uvma_from_va(u->va); |
802 | bool sparse = !!uvma->region; |
803 | |
804 | if (!drm_gpuva_invalidated(va: u->va)) |
805 | nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, sparse); |
806 | } |
807 | |
808 | static void |
809 | op_remap(struct drm_gpuva_op_remap *r, |
810 | struct nouveau_uvma_prealloc *new) |
811 | { |
812 | struct drm_gpuva_op_unmap *u = r->unmap; |
813 | struct nouveau_uvma *uvma = uvma_from_va(u->va); |
814 | u64 addr = uvma->va.va.addr; |
815 | u64 end = uvma->va.va.addr + uvma->va.va.range; |
816 | |
817 | if (r->prev) |
818 | addr = r->prev->va.addr + r->prev->va.range; |
819 | |
820 | if (r->next) |
821 | end = r->next->va.addr; |
822 | |
823 | op_unmap_range(u, addr, range: end - addr); |
824 | } |
825 | |
826 | static int |
827 | nouveau_uvmm_sm(struct nouveau_uvmm *uvmm, |
828 | struct nouveau_uvma_prealloc *new, |
829 | struct drm_gpuva_ops *ops) |
830 | { |
831 | struct drm_gpuva_op *op; |
832 | |
833 | drm_gpuva_for_each_op(op, ops) { |
834 | switch (op->op) { |
835 | case DRM_GPUVA_OP_MAP: |
836 | op_map(uvma: new->map); |
837 | break; |
838 | case DRM_GPUVA_OP_REMAP: |
839 | op_remap(r: &op->remap, new); |
840 | break; |
841 | case DRM_GPUVA_OP_UNMAP: |
842 | op_unmap(u: &op->unmap); |
843 | break; |
844 | default: |
845 | break; |
846 | } |
847 | } |
848 | |
849 | return 0; |
850 | } |
851 | |
852 | static int |
853 | nouveau_uvmm_sm_map(struct nouveau_uvmm *uvmm, |
854 | struct nouveau_uvma_prealloc *new, |
855 | struct drm_gpuva_ops *ops) |
856 | { |
857 | return nouveau_uvmm_sm(uvmm, new, ops); |
858 | } |
859 | |
860 | static int |
861 | nouveau_uvmm_sm_unmap(struct nouveau_uvmm *uvmm, |
862 | struct nouveau_uvma_prealloc *new, |
863 | struct drm_gpuva_ops *ops) |
864 | { |
865 | return nouveau_uvmm_sm(uvmm, new, ops); |
866 | } |
867 | |
868 | static void |
869 | nouveau_uvmm_sm_cleanup(struct nouveau_uvmm *uvmm, |
870 | struct nouveau_uvma_prealloc *new, |
871 | struct drm_gpuva_ops *ops, bool unmap) |
872 | { |
873 | struct drm_gpuva_op *op; |
874 | |
875 | drm_gpuva_for_each_op(op, ops) { |
876 | switch (op->op) { |
877 | case DRM_GPUVA_OP_MAP: |
878 | break; |
879 | case DRM_GPUVA_OP_REMAP: { |
880 | struct drm_gpuva_op_remap *r = &op->remap; |
881 | struct drm_gpuva_op_map *p = r->prev; |
882 | struct drm_gpuva_op_map *n = r->next; |
883 | struct drm_gpuva *va = r->unmap->va; |
884 | struct nouveau_uvma *uvma = uvma_from_va(va); |
885 | |
886 | if (unmap) { |
887 | u64 addr = va->va.addr; |
888 | u64 end = addr + va->va.range; |
889 | |
890 | if (p) |
891 | addr = p->va.addr + p->va.range; |
892 | |
893 | if (n) |
894 | end = n->va.addr; |
895 | |
896 | nouveau_uvmm_vmm_put(uvmm, addr, range: end - addr); |
897 | } |
898 | |
899 | nouveau_uvma_gem_put(uvma); |
900 | nouveau_uvma_free(uvma); |
901 | break; |
902 | } |
903 | case DRM_GPUVA_OP_UNMAP: { |
904 | struct drm_gpuva_op_unmap *u = &op->unmap; |
905 | struct drm_gpuva *va = u->va; |
906 | struct nouveau_uvma *uvma = uvma_from_va(va); |
907 | |
908 | if (unmap) |
909 | nouveau_uvma_vmm_put(uvma); |
910 | |
911 | nouveau_uvma_gem_put(uvma); |
912 | nouveau_uvma_free(uvma); |
913 | break; |
914 | } |
915 | default: |
916 | break; |
917 | } |
918 | } |
919 | } |
920 | |
921 | static void |
922 | nouveau_uvmm_sm_map_cleanup(struct nouveau_uvmm *uvmm, |
923 | struct nouveau_uvma_prealloc *new, |
924 | struct drm_gpuva_ops *ops) |
925 | { |
926 | nouveau_uvmm_sm_cleanup(uvmm, new, ops, unmap: false); |
927 | } |
928 | |
929 | static void |
930 | nouveau_uvmm_sm_unmap_cleanup(struct nouveau_uvmm *uvmm, |
931 | struct nouveau_uvma_prealloc *new, |
932 | struct drm_gpuva_ops *ops) |
933 | { |
934 | nouveau_uvmm_sm_cleanup(uvmm, new, ops, unmap: true); |
935 | } |
936 | |
937 | static int |
938 | nouveau_uvmm_validate_range(struct nouveau_uvmm *uvmm, u64 addr, u64 range) |
939 | { |
940 | if (addr & ~PAGE_MASK) |
941 | return -EINVAL; |
942 | |
943 | if (range & ~PAGE_MASK) |
944 | return -EINVAL; |
945 | |
946 | if (!drm_gpuvm_range_valid(gpuvm: &uvmm->base, addr, range)) |
947 | return -EINVAL; |
948 | |
949 | return 0; |
950 | } |
951 | |
952 | static int |
953 | nouveau_uvmm_bind_job_alloc(struct nouveau_uvmm_bind_job **pjob) |
954 | { |
955 | *pjob = kzalloc(size: sizeof(**pjob), GFP_KERNEL); |
956 | if (!*pjob) |
957 | return -ENOMEM; |
958 | |
959 | kref_init(kref: &(*pjob)->kref); |
960 | |
961 | return 0; |
962 | } |
963 | |
964 | static void |
965 | nouveau_uvmm_bind_job_free(struct kref *kref) |
966 | { |
967 | struct nouveau_uvmm_bind_job *job = |
968 | container_of(kref, struct nouveau_uvmm_bind_job, kref); |
969 | struct bind_job_op *op, *next; |
970 | |
971 | list_for_each_op_safe(op, next, &job->ops) { |
972 | list_del(entry: &op->entry); |
973 | kfree(objp: op); |
974 | } |
975 | |
976 | nouveau_job_free(job: &job->base); |
977 | kfree(objp: job); |
978 | } |
979 | |
980 | static void |
981 | nouveau_uvmm_bind_job_get(struct nouveau_uvmm_bind_job *job) |
982 | { |
983 | kref_get(kref: &job->kref); |
984 | } |
985 | |
986 | static void |
987 | nouveau_uvmm_bind_job_put(struct nouveau_uvmm_bind_job *job) |
988 | { |
989 | kref_put(kref: &job->kref, release: nouveau_uvmm_bind_job_free); |
990 | } |
991 | |
992 | static int |
993 | bind_validate_op(struct nouveau_job *job, |
994 | struct bind_job_op *op) |
995 | { |
996 | struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli: job->cli); |
997 | struct drm_gem_object *obj = op->gem.obj; |
998 | |
999 | if (op->op == OP_MAP) { |
1000 | if (op->gem.offset & ~PAGE_MASK) |
1001 | return -EINVAL; |
1002 | |
1003 | if (obj->size <= op->gem.offset) |
1004 | return -EINVAL; |
1005 | |
1006 | if (op->va.range > (obj->size - op->gem.offset)) |
1007 | return -EINVAL; |
1008 | } |
1009 | |
1010 | return nouveau_uvmm_validate_range(uvmm, addr: op->va.addr, range: op->va.range); |
1011 | } |
1012 | |
1013 | static void |
1014 | bind_validate_map_sparse(struct nouveau_job *job, u64 addr, u64 range) |
1015 | { |
1016 | struct nouveau_sched *sched = job->sched; |
1017 | struct nouveau_job *__job; |
1018 | struct bind_job_op *op; |
1019 | u64 end = addr + range; |
1020 | |
1021 | again: |
1022 | spin_lock(lock: &sched->job.list.lock); |
1023 | list_for_each_entry(__job, &sched->job.list.head, entry) { |
1024 | struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(__job); |
1025 | |
1026 | list_for_each_op(op, &bind_job->ops) { |
1027 | if (op->op == OP_UNMAP) { |
1028 | u64 op_addr = op->va.addr; |
1029 | u64 op_end = op_addr + op->va.range; |
1030 | |
1031 | if (!(end <= op_addr || addr >= op_end)) { |
1032 | nouveau_uvmm_bind_job_get(job: bind_job); |
1033 | spin_unlock(lock: &sched->job.list.lock); |
1034 | wait_for_completion(&bind_job->complete); |
1035 | nouveau_uvmm_bind_job_put(job: bind_job); |
1036 | goto again; |
1037 | } |
1038 | } |
1039 | } |
1040 | } |
1041 | spin_unlock(lock: &sched->job.list.lock); |
1042 | } |
1043 | |
1044 | static int |
1045 | bind_validate_map_common(struct nouveau_job *job, u64 addr, u64 range, |
1046 | bool sparse) |
1047 | { |
1048 | struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli: job->cli); |
1049 | struct nouveau_uvma_region *reg; |
1050 | u64 reg_addr, reg_end; |
1051 | u64 end = addr + range; |
1052 | |
1053 | again: |
1054 | nouveau_uvmm_lock(uvmm); |
1055 | reg = nouveau_uvma_region_find_first(uvmm, addr, range); |
1056 | if (!reg) { |
1057 | nouveau_uvmm_unlock(uvmm); |
1058 | return 0; |
1059 | } |
1060 | |
1061 | /* Generally, job submits are serialized, hence only |
1062 | * dirty regions can be modified concurrently. |
1063 | */ |
1064 | if (reg->dirty) { |
1065 | nouveau_uvma_region_get(reg); |
1066 | nouveau_uvmm_unlock(uvmm); |
1067 | wait_for_completion(®->complete); |
1068 | nouveau_uvma_region_put(reg); |
1069 | goto again; |
1070 | } |
1071 | nouveau_uvmm_unlock(uvmm); |
1072 | |
1073 | if (sparse) |
1074 | return -ENOSPC; |
1075 | |
1076 | reg_addr = reg->va.addr; |
1077 | reg_end = reg_addr + reg->va.range; |
1078 | |
1079 | /* Make sure the mapping is either outside of a |
1080 | * region or fully enclosed by a region. |
1081 | */ |
1082 | if (reg_addr > addr || reg_end < end) |
1083 | return -ENOSPC; |
1084 | |
1085 | return 0; |
1086 | } |
1087 | |
1088 | static int |
1089 | bind_validate_region(struct nouveau_job *job) |
1090 | { |
1091 | struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); |
1092 | struct bind_job_op *op; |
1093 | int ret; |
1094 | |
1095 | list_for_each_op(op, &bind_job->ops) { |
1096 | u64 op_addr = op->va.addr; |
1097 | u64 op_range = op->va.range; |
1098 | bool sparse = false; |
1099 | |
1100 | switch (op->op) { |
1101 | case OP_MAP_SPARSE: |
1102 | sparse = true; |
1103 | bind_validate_map_sparse(job, addr: op_addr, range: op_range); |
1104 | fallthrough; |
1105 | case OP_MAP: |
1106 | ret = bind_validate_map_common(job, addr: op_addr, range: op_range, |
1107 | sparse); |
1108 | if (ret) |
1109 | return ret; |
1110 | break; |
1111 | default: |
1112 | break; |
1113 | } |
1114 | } |
1115 | |
1116 | return 0; |
1117 | } |
1118 | |
1119 | static void |
1120 | bind_link_gpuvas(struct bind_job_op *bop) |
1121 | { |
1122 | struct nouveau_uvma_prealloc *new = &bop->new; |
1123 | struct drm_gpuvm_bo *vm_bo = bop->vm_bo; |
1124 | struct drm_gpuva_ops *ops = bop->ops; |
1125 | struct drm_gpuva_op *op; |
1126 | |
1127 | drm_gpuva_for_each_op(op, ops) { |
1128 | switch (op->op) { |
1129 | case DRM_GPUVA_OP_MAP: |
1130 | drm_gpuva_link(va: &new->map->va, vm_bo); |
1131 | break; |
1132 | case DRM_GPUVA_OP_REMAP: { |
1133 | struct drm_gpuva *va = op->remap.unmap->va; |
1134 | |
1135 | if (op->remap.prev) |
1136 | drm_gpuva_link(va: &new->prev->va, vm_bo: va->vm_bo); |
1137 | if (op->remap.next) |
1138 | drm_gpuva_link(va: &new->next->va, vm_bo: va->vm_bo); |
1139 | drm_gpuva_unlink(va); |
1140 | break; |
1141 | } |
1142 | case DRM_GPUVA_OP_UNMAP: |
1143 | drm_gpuva_unlink(va: op->unmap.va); |
1144 | break; |
1145 | default: |
1146 | break; |
1147 | } |
1148 | } |
1149 | } |
1150 | |
1151 | static int |
1152 | bind_lock_validate(struct nouveau_job *job, struct drm_exec *exec, |
1153 | unsigned int num_fences) |
1154 | { |
1155 | struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); |
1156 | struct bind_job_op *op; |
1157 | int ret; |
1158 | |
1159 | list_for_each_op(op, &bind_job->ops) { |
1160 | struct drm_gpuva_op *va_op; |
1161 | |
1162 | if (!op->ops) |
1163 | continue; |
1164 | |
1165 | drm_gpuva_for_each_op(va_op, op->ops) { |
1166 | struct drm_gem_object *obj = op_gem_obj(op: va_op); |
1167 | |
1168 | if (unlikely(!obj)) |
1169 | continue; |
1170 | |
1171 | ret = drm_exec_prepare_obj(exec, obj, num_fences); |
1172 | if (ret) |
1173 | return ret; |
1174 | |
1175 | /* Don't validate GEMs backing mappings we're about to |
1176 | * unmap, it's not worth the effort. |
1177 | */ |
1178 | if (va_op->op == DRM_GPUVA_OP_UNMAP) |
1179 | continue; |
1180 | |
1181 | ret = nouveau_bo_validate(nouveau_gem_object(gem: obj), |
1182 | interruptible: true, no_wait_gpu: false); |
1183 | if (ret) |
1184 | return ret; |
1185 | } |
1186 | } |
1187 | |
1188 | return 0; |
1189 | } |
1190 | |
1191 | static int |
1192 | nouveau_uvmm_bind_job_submit(struct nouveau_job *job, |
1193 | struct drm_gpuvm_exec *vme) |
1194 | { |
1195 | struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli: job->cli); |
1196 | struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); |
1197 | struct drm_exec *exec = &vme->exec; |
1198 | struct bind_job_op *op; |
1199 | int ret; |
1200 | |
1201 | list_for_each_op(op, &bind_job->ops) { |
1202 | if (op->op == OP_MAP) { |
1203 | struct drm_gem_object *obj = op->gem.obj = |
1204 | drm_gem_object_lookup(filp: job->file_priv, |
1205 | handle: op->gem.handle); |
1206 | if (!obj) |
1207 | return -ENOENT; |
1208 | |
1209 | dma_resv_lock(obj: obj->resv, NULL); |
1210 | op->vm_bo = drm_gpuvm_bo_obtain(gpuvm: &uvmm->base, obj); |
1211 | dma_resv_unlock(obj: obj->resv); |
1212 | if (IS_ERR(ptr: op->vm_bo)) |
1213 | return PTR_ERR(ptr: op->vm_bo); |
1214 | |
1215 | drm_gpuvm_bo_extobj_add(vm_bo: op->vm_bo); |
1216 | } |
1217 | |
1218 | ret = bind_validate_op(job, op); |
1219 | if (ret) |
1220 | return ret; |
1221 | } |
1222 | |
1223 | /* If a sparse region or mapping overlaps a dirty region, we need to |
1224 | * wait for the region to complete the unbind process. This is due to |
1225 | * how page table management is currently implemented. A future |
1226 | * implementation might change this. |
1227 | */ |
1228 | ret = bind_validate_region(job); |
1229 | if (ret) |
1230 | return ret; |
1231 | |
1232 | /* Once we start modifying the GPU VA space we need to keep holding the |
1233 | * uvmm lock until we can't fail anymore. This is due to the set of GPU |
1234 | * VA space changes must appear atomically and we need to be able to |
1235 | * unwind all GPU VA space changes on failure. |
1236 | */ |
1237 | nouveau_uvmm_lock(uvmm); |
1238 | |
1239 | list_for_each_op(op, &bind_job->ops) { |
1240 | switch (op->op) { |
1241 | case OP_MAP_SPARSE: |
1242 | ret = nouveau_uvma_region_create(uvmm, |
1243 | addr: op->va.addr, |
1244 | range: op->va.range); |
1245 | if (ret) |
1246 | goto unwind_continue; |
1247 | |
1248 | break; |
1249 | case OP_UNMAP_SPARSE: |
1250 | op->reg = nouveau_uvma_region_find(uvmm, addr: op->va.addr, |
1251 | range: op->va.range); |
1252 | if (!op->reg || op->reg->dirty) { |
1253 | ret = -ENOENT; |
1254 | goto unwind_continue; |
1255 | } |
1256 | |
1257 | op->ops = drm_gpuvm_sm_unmap_ops_create(gpuvm: &uvmm->base, |
1258 | addr: op->va.addr, |
1259 | range: op->va.range); |
1260 | if (IS_ERR(ptr: op->ops)) { |
1261 | ret = PTR_ERR(ptr: op->ops); |
1262 | goto unwind_continue; |
1263 | } |
1264 | |
1265 | ret = nouveau_uvmm_sm_unmap_prepare(uvmm, new: &op->new, |
1266 | ops: op->ops); |
1267 | if (ret) { |
1268 | drm_gpuva_ops_free(gpuvm: &uvmm->base, ops: op->ops); |
1269 | op->ops = NULL; |
1270 | op->reg = NULL; |
1271 | goto unwind_continue; |
1272 | } |
1273 | |
1274 | nouveau_uvma_region_dirty(reg: op->reg); |
1275 | |
1276 | break; |
1277 | case OP_MAP: { |
1278 | struct nouveau_uvma_region *reg; |
1279 | |
1280 | reg = nouveau_uvma_region_find_first(uvmm, |
1281 | addr: op->va.addr, |
1282 | range: op->va.range); |
1283 | if (reg) { |
1284 | u64 reg_addr = reg->va.addr; |
1285 | u64 reg_end = reg_addr + reg->va.range; |
1286 | u64 op_addr = op->va.addr; |
1287 | u64 op_end = op_addr + op->va.range; |
1288 | |
1289 | if (unlikely(reg->dirty)) { |
1290 | ret = -EINVAL; |
1291 | goto unwind_continue; |
1292 | } |
1293 | |
1294 | /* Make sure the mapping is either outside of a |
1295 | * region or fully enclosed by a region. |
1296 | */ |
1297 | if (reg_addr > op_addr || reg_end < op_end) { |
1298 | ret = -ENOSPC; |
1299 | goto unwind_continue; |
1300 | } |
1301 | } |
1302 | |
1303 | op->ops = drm_gpuvm_sm_map_ops_create(gpuvm: &uvmm->base, |
1304 | addr: op->va.addr, |
1305 | range: op->va.range, |
1306 | obj: op->gem.obj, |
1307 | offset: op->gem.offset); |
1308 | if (IS_ERR(ptr: op->ops)) { |
1309 | ret = PTR_ERR(ptr: op->ops); |
1310 | goto unwind_continue; |
1311 | } |
1312 | |
1313 | ret = nouveau_uvmm_sm_map_prepare(uvmm, new: &op->new, |
1314 | region: reg, ops: op->ops, |
1315 | addr: op->va.addr, |
1316 | range: op->va.range, |
1317 | kind: op->flags & 0xff); |
1318 | if (ret) { |
1319 | drm_gpuva_ops_free(gpuvm: &uvmm->base, ops: op->ops); |
1320 | op->ops = NULL; |
1321 | goto unwind_continue; |
1322 | } |
1323 | |
1324 | break; |
1325 | } |
1326 | case OP_UNMAP: |
1327 | op->ops = drm_gpuvm_sm_unmap_ops_create(gpuvm: &uvmm->base, |
1328 | addr: op->va.addr, |
1329 | range: op->va.range); |
1330 | if (IS_ERR(ptr: op->ops)) { |
1331 | ret = PTR_ERR(ptr: op->ops); |
1332 | goto unwind_continue; |
1333 | } |
1334 | |
1335 | ret = nouveau_uvmm_sm_unmap_prepare(uvmm, new: &op->new, |
1336 | ops: op->ops); |
1337 | if (ret) { |
1338 | drm_gpuva_ops_free(gpuvm: &uvmm->base, ops: op->ops); |
1339 | op->ops = NULL; |
1340 | goto unwind_continue; |
1341 | } |
1342 | |
1343 | break; |
1344 | default: |
1345 | ret = -EINVAL; |
1346 | goto unwind_continue; |
1347 | } |
1348 | } |
1349 | |
1350 | drm_exec_init(exec, flags: vme->flags, nr: 0); |
1351 | drm_exec_until_all_locked(exec) { |
1352 | ret = bind_lock_validate(job, exec, num_fences: vme->num_fences); |
1353 | drm_exec_retry_on_contention(exec); |
1354 | if (ret) { |
1355 | op = list_last_op(&bind_job->ops); |
1356 | goto unwind; |
1357 | } |
1358 | } |
1359 | |
1360 | /* Link and unlink GPUVAs while holding the dma_resv lock. |
1361 | * |
1362 | * As long as we validate() all GEMs and add fences to all GEMs DMA |
1363 | * reservations backing map and remap operations we can be sure there |
1364 | * won't be any concurrent (in)validations during job execution, hence |
1365 | * we're safe to check drm_gpuva_invalidated() within the fence |
1366 | * signalling critical path without holding a separate lock. |
1367 | * |
1368 | * GPUVAs about to be unmapped are safe as well, since they're unlinked |
1369 | * already. |
1370 | * |
1371 | * GEMs from map and remap operations must be validated before linking |
1372 | * their corresponding mappings to prevent the actual PT update to |
1373 | * happen right away in validate() rather than asynchronously as |
1374 | * intended. |
1375 | * |
1376 | * Note that after linking and unlinking the GPUVAs in this loop this |
1377 | * function cannot fail anymore, hence there is no need for an unwind |
1378 | * path. |
1379 | */ |
1380 | list_for_each_op(op, &bind_job->ops) { |
1381 | switch (op->op) { |
1382 | case OP_UNMAP_SPARSE: |
1383 | case OP_MAP: |
1384 | case OP_UNMAP: |
1385 | bind_link_gpuvas(bop: op); |
1386 | break; |
1387 | default: |
1388 | break; |
1389 | } |
1390 | } |
1391 | nouveau_uvmm_unlock(uvmm); |
1392 | |
1393 | return 0; |
1394 | |
1395 | unwind_continue: |
1396 | op = list_prev_op(op); |
1397 | unwind: |
1398 | list_for_each_op_from_reverse(op, &bind_job->ops) { |
1399 | switch (op->op) { |
1400 | case OP_MAP_SPARSE: |
1401 | nouveau_uvma_region_destroy(uvmm, addr: op->va.addr, |
1402 | range: op->va.range); |
1403 | break; |
1404 | case OP_UNMAP_SPARSE: |
1405 | __nouveau_uvma_region_insert(uvmm, reg: op->reg); |
1406 | nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, new: &op->new, |
1407 | ops: op->ops); |
1408 | break; |
1409 | case OP_MAP: |
1410 | nouveau_uvmm_sm_map_prepare_unwind(uvmm, new: &op->new, |
1411 | ops: op->ops, |
1412 | addr: op->va.addr, |
1413 | range: op->va.range); |
1414 | break; |
1415 | case OP_UNMAP: |
1416 | nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, new: &op->new, |
1417 | ops: op->ops); |
1418 | break; |
1419 | } |
1420 | |
1421 | drm_gpuva_ops_free(gpuvm: &uvmm->base, ops: op->ops); |
1422 | op->ops = NULL; |
1423 | op->reg = NULL; |
1424 | } |
1425 | |
1426 | nouveau_uvmm_unlock(uvmm); |
1427 | drm_gpuvm_exec_unlock(vm_exec: vme); |
1428 | return ret; |
1429 | } |
1430 | |
1431 | static void |
1432 | nouveau_uvmm_bind_job_armed_submit(struct nouveau_job *job, |
1433 | struct drm_gpuvm_exec *vme) |
1434 | { |
1435 | drm_gpuvm_exec_resv_add_fence(vm_exec: vme, fence: job->done_fence, |
1436 | private_usage: job->resv_usage, extobj_usage: job->resv_usage); |
1437 | drm_gpuvm_exec_unlock(vm_exec: vme); |
1438 | } |
1439 | |
1440 | static struct dma_fence * |
1441 | nouveau_uvmm_bind_job_run(struct nouveau_job *job) |
1442 | { |
1443 | struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); |
1444 | struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli: job->cli); |
1445 | struct bind_job_op *op; |
1446 | int ret = 0; |
1447 | |
1448 | list_for_each_op(op, &bind_job->ops) { |
1449 | switch (op->op) { |
1450 | case OP_MAP_SPARSE: |
1451 | /* noop */ |
1452 | break; |
1453 | case OP_MAP: |
1454 | ret = nouveau_uvmm_sm_map(uvmm, new: &op->new, ops: op->ops); |
1455 | if (ret) |
1456 | goto out; |
1457 | break; |
1458 | case OP_UNMAP_SPARSE: |
1459 | fallthrough; |
1460 | case OP_UNMAP: |
1461 | ret = nouveau_uvmm_sm_unmap(uvmm, new: &op->new, ops: op->ops); |
1462 | if (ret) |
1463 | goto out; |
1464 | break; |
1465 | } |
1466 | } |
1467 | |
1468 | out: |
1469 | if (ret) |
1470 | NV_PRINTK(err, job->cli, "bind job failed: %d\n" , ret); |
1471 | return ERR_PTR(error: ret); |
1472 | } |
1473 | |
1474 | static void |
1475 | nouveau_uvmm_bind_job_cleanup(struct nouveau_job *job) |
1476 | { |
1477 | struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); |
1478 | struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli: job->cli); |
1479 | struct bind_job_op *op; |
1480 | |
1481 | list_for_each_op(op, &bind_job->ops) { |
1482 | struct drm_gem_object *obj = op->gem.obj; |
1483 | |
1484 | /* When nouveau_uvmm_bind_job_submit() fails op->ops and op->reg |
1485 | * will be NULL, hence skip the cleanup. |
1486 | */ |
1487 | switch (op->op) { |
1488 | case OP_MAP_SPARSE: |
1489 | /* noop */ |
1490 | break; |
1491 | case OP_UNMAP_SPARSE: |
1492 | if (!IS_ERR_OR_NULL(ptr: op->ops)) |
1493 | nouveau_uvmm_sm_unmap_cleanup(uvmm, new: &op->new, |
1494 | ops: op->ops); |
1495 | |
1496 | if (op->reg) { |
1497 | nouveau_uvma_region_sparse_unref(reg: op->reg); |
1498 | nouveau_uvmm_lock(uvmm); |
1499 | nouveau_uvma_region_remove(reg: op->reg); |
1500 | nouveau_uvmm_unlock(uvmm); |
1501 | nouveau_uvma_region_complete(reg: op->reg); |
1502 | nouveau_uvma_region_put(reg: op->reg); |
1503 | } |
1504 | |
1505 | break; |
1506 | case OP_MAP: |
1507 | if (!IS_ERR_OR_NULL(ptr: op->ops)) |
1508 | nouveau_uvmm_sm_map_cleanup(uvmm, new: &op->new, |
1509 | ops: op->ops); |
1510 | break; |
1511 | case OP_UNMAP: |
1512 | if (!IS_ERR_OR_NULL(ptr: op->ops)) |
1513 | nouveau_uvmm_sm_unmap_cleanup(uvmm, new: &op->new, |
1514 | ops: op->ops); |
1515 | break; |
1516 | } |
1517 | |
1518 | if (!IS_ERR_OR_NULL(ptr: op->ops)) |
1519 | drm_gpuva_ops_free(gpuvm: &uvmm->base, ops: op->ops); |
1520 | |
1521 | if (!IS_ERR_OR_NULL(ptr: op->vm_bo)) { |
1522 | dma_resv_lock(obj: obj->resv, NULL); |
1523 | drm_gpuvm_bo_put(vm_bo: op->vm_bo); |
1524 | dma_resv_unlock(obj: obj->resv); |
1525 | } |
1526 | |
1527 | if (obj) |
1528 | drm_gem_object_put(obj); |
1529 | } |
1530 | |
1531 | nouveau_job_done(job); |
1532 | complete_all(&bind_job->complete); |
1533 | |
1534 | nouveau_uvmm_bind_job_put(job: bind_job); |
1535 | } |
1536 | |
1537 | static struct nouveau_job_ops nouveau_bind_job_ops = { |
1538 | .submit = nouveau_uvmm_bind_job_submit, |
1539 | .armed_submit = nouveau_uvmm_bind_job_armed_submit, |
1540 | .run = nouveau_uvmm_bind_job_run, |
1541 | .free = nouveau_uvmm_bind_job_cleanup, |
1542 | }; |
1543 | |
1544 | static int |
1545 | bind_job_op_from_uop(struct bind_job_op **pop, |
1546 | struct drm_nouveau_vm_bind_op *uop) |
1547 | { |
1548 | struct bind_job_op *op; |
1549 | |
1550 | op = *pop = kzalloc(size: sizeof(*op), GFP_KERNEL); |
1551 | if (!op) |
1552 | return -ENOMEM; |
1553 | |
1554 | switch (uop->op) { |
1555 | case OP_MAP: |
1556 | op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ? |
1557 | OP_MAP_SPARSE : OP_MAP; |
1558 | break; |
1559 | case OP_UNMAP: |
1560 | op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ? |
1561 | OP_UNMAP_SPARSE : OP_UNMAP; |
1562 | break; |
1563 | default: |
1564 | op->op = uop->op; |
1565 | break; |
1566 | } |
1567 | |
1568 | op->flags = uop->flags; |
1569 | op->va.addr = uop->addr; |
1570 | op->va.range = uop->range; |
1571 | op->gem.handle = uop->handle; |
1572 | op->gem.offset = uop->bo_offset; |
1573 | |
1574 | return 0; |
1575 | } |
1576 | |
1577 | static void |
1578 | bind_job_ops_free(struct list_head *ops) |
1579 | { |
1580 | struct bind_job_op *op, *next; |
1581 | |
1582 | list_for_each_op_safe(op, next, ops) { |
1583 | list_del(entry: &op->entry); |
1584 | kfree(objp: op); |
1585 | } |
1586 | } |
1587 | |
1588 | static int |
1589 | nouveau_uvmm_bind_job_init(struct nouveau_uvmm_bind_job **pjob, |
1590 | struct nouveau_uvmm_bind_job_args *__args) |
1591 | { |
1592 | struct nouveau_uvmm_bind_job *job; |
1593 | struct nouveau_job_args args = {}; |
1594 | struct bind_job_op *op; |
1595 | int i, ret; |
1596 | |
1597 | ret = nouveau_uvmm_bind_job_alloc(pjob: &job); |
1598 | if (ret) |
1599 | return ret; |
1600 | |
1601 | INIT_LIST_HEAD(list: &job->ops); |
1602 | |
1603 | for (i = 0; i < __args->op.count; i++) { |
1604 | ret = bind_job_op_from_uop(pop: &op, uop: &__args->op.s[i]); |
1605 | if (ret) |
1606 | goto err_free; |
1607 | |
1608 | list_add_tail(new: &op->entry, head: &job->ops); |
1609 | } |
1610 | |
1611 | init_completion(x: &job->complete); |
1612 | |
1613 | args.file_priv = __args->file_priv; |
1614 | |
1615 | args.sched = __args->sched; |
1616 | args.credits = 1; |
1617 | |
1618 | args.in_sync.count = __args->in_sync.count; |
1619 | args.in_sync.s = __args->in_sync.s; |
1620 | |
1621 | args.out_sync.count = __args->out_sync.count; |
1622 | args.out_sync.s = __args->out_sync.s; |
1623 | |
1624 | args.sync = !(__args->flags & DRM_NOUVEAU_VM_BIND_RUN_ASYNC); |
1625 | args.ops = &nouveau_bind_job_ops; |
1626 | args.resv_usage = DMA_RESV_USAGE_BOOKKEEP; |
1627 | |
1628 | ret = nouveau_job_init(job: &job->base, args: &args); |
1629 | if (ret) |
1630 | goto err_free; |
1631 | |
1632 | *pjob = job; |
1633 | return 0; |
1634 | |
1635 | err_free: |
1636 | bind_job_ops_free(ops: &job->ops); |
1637 | kfree(objp: job); |
1638 | *pjob = NULL; |
1639 | |
1640 | return ret; |
1641 | } |
1642 | |
1643 | static int |
1644 | nouveau_uvmm_vm_bind(struct nouveau_uvmm_bind_job_args *args) |
1645 | { |
1646 | struct nouveau_uvmm_bind_job *job; |
1647 | int ret; |
1648 | |
1649 | ret = nouveau_uvmm_bind_job_init(pjob: &job, args: args); |
1650 | if (ret) |
1651 | return ret; |
1652 | |
1653 | ret = nouveau_job_submit(job: &job->base); |
1654 | if (ret) |
1655 | goto err_job_fini; |
1656 | |
1657 | return 0; |
1658 | |
1659 | err_job_fini: |
1660 | nouveau_job_fini(job: &job->base); |
1661 | return ret; |
1662 | } |
1663 | |
1664 | static int |
1665 | nouveau_uvmm_vm_bind_ucopy(struct nouveau_uvmm_bind_job_args *args, |
1666 | struct drm_nouveau_vm_bind *req) |
1667 | { |
1668 | struct drm_nouveau_sync **s; |
1669 | u32 inc = req->wait_count; |
1670 | u64 ins = req->wait_ptr; |
1671 | u32 outc = req->sig_count; |
1672 | u64 outs = req->sig_ptr; |
1673 | u32 opc = req->op_count; |
1674 | u64 ops = req->op_ptr; |
1675 | int ret; |
1676 | |
1677 | args->flags = req->flags; |
1678 | |
1679 | if (opc) { |
1680 | args->op.count = opc; |
1681 | args->op.s = u_memcpya(user: ops, nmemb: opc, |
1682 | size: sizeof(*args->op.s)); |
1683 | if (IS_ERR(ptr: args->op.s)) |
1684 | return PTR_ERR(ptr: args->op.s); |
1685 | } |
1686 | |
1687 | if (inc) { |
1688 | s = &args->in_sync.s; |
1689 | |
1690 | args->in_sync.count = inc; |
1691 | *s = u_memcpya(user: ins, nmemb: inc, size: sizeof(**s)); |
1692 | if (IS_ERR(ptr: *s)) { |
1693 | ret = PTR_ERR(ptr: *s); |
1694 | goto err_free_ops; |
1695 | } |
1696 | } |
1697 | |
1698 | if (outc) { |
1699 | s = &args->out_sync.s; |
1700 | |
1701 | args->out_sync.count = outc; |
1702 | *s = u_memcpya(user: outs, nmemb: outc, size: sizeof(**s)); |
1703 | if (IS_ERR(ptr: *s)) { |
1704 | ret = PTR_ERR(ptr: *s); |
1705 | goto err_free_ins; |
1706 | } |
1707 | } |
1708 | |
1709 | return 0; |
1710 | |
1711 | err_free_ops: |
1712 | u_free(addr: args->op.s); |
1713 | err_free_ins: |
1714 | u_free(addr: args->in_sync.s); |
1715 | return ret; |
1716 | } |
1717 | |
1718 | static void |
1719 | nouveau_uvmm_vm_bind_ufree(struct nouveau_uvmm_bind_job_args *args) |
1720 | { |
1721 | u_free(addr: args->op.s); |
1722 | u_free(addr: args->in_sync.s); |
1723 | u_free(addr: args->out_sync.s); |
1724 | } |
1725 | |
1726 | int |
1727 | nouveau_uvmm_ioctl_vm_bind(struct drm_device *dev, |
1728 | void *data, |
1729 | struct drm_file *file_priv) |
1730 | { |
1731 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
1732 | struct nouveau_uvmm_bind_job_args args = {}; |
1733 | struct drm_nouveau_vm_bind *req = data; |
1734 | int ret = 0; |
1735 | |
1736 | if (unlikely(!nouveau_cli_uvmm_locked(cli))) |
1737 | return -ENOSYS; |
1738 | |
1739 | ret = nouveau_uvmm_vm_bind_ucopy(args: &args, req); |
1740 | if (ret) |
1741 | return ret; |
1742 | |
1743 | args.sched = cli->sched; |
1744 | args.file_priv = file_priv; |
1745 | |
1746 | ret = nouveau_uvmm_vm_bind(args: &args); |
1747 | if (ret) |
1748 | goto out_free_args; |
1749 | |
1750 | out_free_args: |
1751 | nouveau_uvmm_vm_bind_ufree(args: &args); |
1752 | return ret; |
1753 | } |
1754 | |
1755 | void |
1756 | nouveau_uvmm_bo_map_all(struct nouveau_bo *nvbo, struct nouveau_mem *mem) |
1757 | { |
1758 | struct drm_gem_object *obj = &nvbo->bo.base; |
1759 | struct drm_gpuvm_bo *vm_bo; |
1760 | struct drm_gpuva *va; |
1761 | |
1762 | dma_resv_assert_held(obj->resv); |
1763 | |
1764 | drm_gem_for_each_gpuvm_bo(vm_bo, obj) { |
1765 | drm_gpuvm_bo_for_each_va(va, vm_bo) { |
1766 | struct nouveau_uvma *uvma = uvma_from_va(va); |
1767 | |
1768 | nouveau_uvma_map(uvma, mem); |
1769 | drm_gpuva_invalidate(va, invalidate: false); |
1770 | } |
1771 | } |
1772 | } |
1773 | |
1774 | void |
1775 | nouveau_uvmm_bo_unmap_all(struct nouveau_bo *nvbo) |
1776 | { |
1777 | struct drm_gem_object *obj = &nvbo->bo.base; |
1778 | struct drm_gpuvm_bo *vm_bo; |
1779 | struct drm_gpuva *va; |
1780 | |
1781 | dma_resv_assert_held(obj->resv); |
1782 | |
1783 | drm_gem_for_each_gpuvm_bo(vm_bo, obj) { |
1784 | drm_gpuvm_bo_for_each_va(va, vm_bo) { |
1785 | struct nouveau_uvma *uvma = uvma_from_va(va); |
1786 | |
1787 | nouveau_uvma_unmap(uvma); |
1788 | drm_gpuva_invalidate(va, invalidate: true); |
1789 | } |
1790 | } |
1791 | } |
1792 | |
1793 | static void |
1794 | nouveau_uvmm_free(struct drm_gpuvm *gpuvm) |
1795 | { |
1796 | struct nouveau_uvmm *uvmm = uvmm_from_gpuvm(gpuvm); |
1797 | |
1798 | kfree(objp: uvmm); |
1799 | } |
1800 | |
1801 | static int |
1802 | nouveau_uvmm_bo_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec) |
1803 | { |
1804 | struct nouveau_bo *nvbo = nouveau_gem_object(gem: vm_bo->obj); |
1805 | |
1806 | return nouveau_bo_validate(nvbo, interruptible: true, no_wait_gpu: false); |
1807 | } |
1808 | |
1809 | static const struct drm_gpuvm_ops gpuvm_ops = { |
1810 | .vm_free = nouveau_uvmm_free, |
1811 | .vm_bo_validate = nouveau_uvmm_bo_validate, |
1812 | }; |
1813 | |
1814 | int |
1815 | nouveau_uvmm_ioctl_vm_init(struct drm_device *dev, |
1816 | void *data, |
1817 | struct drm_file *file_priv) |
1818 | { |
1819 | struct nouveau_uvmm *uvmm; |
1820 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
1821 | struct drm_device *drm = cli->drm->dev; |
1822 | struct drm_gem_object *r_obj; |
1823 | struct drm_nouveau_vm_init *init = data; |
1824 | u64 kernel_managed_end; |
1825 | int ret; |
1826 | |
1827 | if (check_add_overflow(init->kernel_managed_addr, |
1828 | init->kernel_managed_size, |
1829 | &kernel_managed_end)) |
1830 | return -EINVAL; |
1831 | |
1832 | if (kernel_managed_end > NOUVEAU_VA_SPACE_END) |
1833 | return -EINVAL; |
1834 | |
1835 | mutex_lock(&cli->mutex); |
1836 | |
1837 | if (unlikely(cli->uvmm.disabled)) { |
1838 | ret = -ENOSYS; |
1839 | goto out_unlock; |
1840 | } |
1841 | |
1842 | uvmm = kzalloc(size: sizeof(*uvmm), GFP_KERNEL); |
1843 | if (!uvmm) { |
1844 | ret = -ENOMEM; |
1845 | goto out_unlock; |
1846 | } |
1847 | |
1848 | r_obj = drm_gpuvm_resv_object_alloc(drm); |
1849 | if (!r_obj) { |
1850 | kfree(objp: uvmm); |
1851 | ret = -ENOMEM; |
1852 | goto out_unlock; |
1853 | } |
1854 | |
1855 | mutex_init(&uvmm->mutex); |
1856 | mt_init_flags(mt: &uvmm->region_mt, MT_FLAGS_LOCK_EXTERN); |
1857 | mt_set_external_lock(&uvmm->region_mt, &uvmm->mutex); |
1858 | |
1859 | drm_gpuvm_init(gpuvm: &uvmm->base, name: cli->name, flags: 0, drm, r_obj, |
1860 | NOUVEAU_VA_SPACE_START, |
1861 | NOUVEAU_VA_SPACE_END, |
1862 | reserve_offset: init->kernel_managed_addr, |
1863 | reserve_range: init->kernel_managed_size, |
1864 | ops: &gpuvm_ops); |
1865 | /* GPUVM takes care from here on. */ |
1866 | drm_gem_object_put(obj: r_obj); |
1867 | |
1868 | ret = nvif_vmm_ctor(&cli->mmu, "uvmm" , |
1869 | cli->vmm.vmm.object.oclass, RAW, |
1870 | init->kernel_managed_addr, |
1871 | init->kernel_managed_size, |
1872 | NULL, 0, &uvmm->vmm.vmm); |
1873 | if (ret) |
1874 | goto out_gpuvm_fini; |
1875 | |
1876 | uvmm->vmm.cli = cli; |
1877 | cli->uvmm.ptr = uvmm; |
1878 | mutex_unlock(lock: &cli->mutex); |
1879 | |
1880 | return 0; |
1881 | |
1882 | out_gpuvm_fini: |
1883 | drm_gpuvm_put(gpuvm: &uvmm->base); |
1884 | out_unlock: |
1885 | mutex_unlock(lock: &cli->mutex); |
1886 | return ret; |
1887 | } |
1888 | |
1889 | void |
1890 | nouveau_uvmm_fini(struct nouveau_uvmm *uvmm) |
1891 | { |
1892 | MA_STATE(mas, &uvmm->region_mt, 0, 0); |
1893 | struct nouveau_uvma_region *reg; |
1894 | struct nouveau_cli *cli = uvmm->vmm.cli; |
1895 | struct drm_gpuva *va, *next; |
1896 | |
1897 | nouveau_uvmm_lock(uvmm); |
1898 | drm_gpuvm_for_each_va_safe(va, next, &uvmm->base) { |
1899 | struct nouveau_uvma *uvma = uvma_from_va(va); |
1900 | struct drm_gem_object *obj = va->gem.obj; |
1901 | |
1902 | if (unlikely(va == &uvmm->base.kernel_alloc_node)) |
1903 | continue; |
1904 | |
1905 | drm_gpuva_remove(va); |
1906 | |
1907 | dma_resv_lock(obj: obj->resv, NULL); |
1908 | drm_gpuva_unlink(va); |
1909 | dma_resv_unlock(obj: obj->resv); |
1910 | |
1911 | nouveau_uvma_unmap(uvma); |
1912 | nouveau_uvma_vmm_put(uvma); |
1913 | |
1914 | nouveau_uvma_gem_put(uvma); |
1915 | nouveau_uvma_free(uvma); |
1916 | } |
1917 | |
1918 | mas_for_each(&mas, reg, ULONG_MAX) { |
1919 | mas_erase(mas: &mas); |
1920 | nouveau_uvma_region_sparse_unref(reg); |
1921 | nouveau_uvma_region_put(reg); |
1922 | } |
1923 | |
1924 | WARN(!mtree_empty(&uvmm->region_mt), |
1925 | "nouveau_uvma_region tree not empty, potentially leaking memory." ); |
1926 | __mt_destroy(mt: &uvmm->region_mt); |
1927 | nouveau_uvmm_unlock(uvmm); |
1928 | |
1929 | mutex_lock(&cli->mutex); |
1930 | nouveau_vmm_fini(&uvmm->vmm); |
1931 | drm_gpuvm_put(gpuvm: &uvmm->base); |
1932 | mutex_unlock(lock: &cli->mutex); |
1933 | } |
1934 | |