1 | /* |
2 | * Copyright (C) 2008 Ben Skeggs. |
3 | * All Rights Reserved. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining |
6 | * a copy of this software and associated documentation files (the |
7 | * "Software"), to deal in the Software without restriction, including |
8 | * without limitation the rights to use, copy, modify, merge, publish, |
9 | * distribute, sublicense, and/or sell copies of the Software, and to |
10 | * permit persons to whom the Software is furnished to do so, subject to |
11 | * the following conditions: |
12 | * |
13 | * The above copyright notice and this permission notice (including the |
14 | * next paragraph) shall be included in all copies or substantial |
15 | * portions of the Software. |
16 | * |
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. |
20 | * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE |
21 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
22 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
24 | * |
25 | */ |
26 | |
27 | #include <drm/drm_gem_ttm_helper.h> |
28 | |
29 | #include "nouveau_drv.h" |
30 | #include "nouveau_dma.h" |
31 | #include "nouveau_fence.h" |
32 | #include "nouveau_abi16.h" |
33 | |
34 | #include "nouveau_ttm.h" |
35 | #include "nouveau_gem.h" |
36 | #include "nouveau_mem.h" |
37 | #include "nouveau_vmm.h" |
38 | |
39 | #include <nvif/class.h> |
40 | #include <nvif/push206e.h> |
41 | |
42 | static vm_fault_t nouveau_ttm_fault(struct vm_fault *vmf) |
43 | { |
44 | struct vm_area_struct *vma = vmf->vma; |
45 | struct ttm_buffer_object *bo = vma->vm_private_data; |
46 | pgprot_t prot; |
47 | vm_fault_t ret; |
48 | |
49 | ret = ttm_bo_vm_reserve(bo, vmf); |
50 | if (ret) |
51 | return ret; |
52 | |
53 | ret = nouveau_ttm_fault_reserve_notify(bo); |
54 | if (ret) |
55 | goto error_unlock; |
56 | |
57 | nouveau_bo_del_io_reserve_lru(bo); |
58 | prot = vm_get_page_prot(vm_flags: vma->vm_flags); |
59 | ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT); |
60 | nouveau_bo_add_io_reserve_lru(bo); |
61 | if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) |
62 | return ret; |
63 | |
64 | error_unlock: |
65 | dma_resv_unlock(obj: bo->base.resv); |
66 | return ret; |
67 | } |
68 | |
69 | static const struct vm_operations_struct nouveau_ttm_vm_ops = { |
70 | .fault = nouveau_ttm_fault, |
71 | .open = ttm_bo_vm_open, |
72 | .close = ttm_bo_vm_close, |
73 | .access = ttm_bo_vm_access |
74 | }; |
75 | |
76 | void |
77 | nouveau_gem_object_del(struct drm_gem_object *gem) |
78 | { |
79 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
80 | struct nouveau_drm *drm = nouveau_bdev(bd: nvbo->bo.bdev); |
81 | struct device *dev = drm->dev->dev; |
82 | int ret; |
83 | |
84 | ret = pm_runtime_get_sync(dev); |
85 | if (WARN_ON(ret < 0 && ret != -EACCES)) { |
86 | pm_runtime_put_autosuspend(dev); |
87 | return; |
88 | } |
89 | |
90 | if (gem->import_attach) |
91 | drm_prime_gem_destroy(obj: gem, sg: nvbo->bo.sg); |
92 | |
93 | ttm_bo_put(bo: &nvbo->bo); |
94 | |
95 | pm_runtime_mark_last_busy(dev); |
96 | pm_runtime_put_autosuspend(dev); |
97 | } |
98 | |
99 | int |
100 | nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) |
101 | { |
102 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
103 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
104 | struct nouveau_drm *drm = nouveau_bdev(bd: nvbo->bo.bdev); |
105 | struct device *dev = drm->dev->dev; |
106 | struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli); |
107 | struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); |
108 | struct nouveau_vma *vma; |
109 | int ret; |
110 | |
111 | if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) |
112 | return 0; |
113 | |
114 | if (nvbo->no_share && uvmm && |
115 | drm_gpuvm_resv(&uvmm->base) != nvbo->bo.base.resv) |
116 | return -EPERM; |
117 | |
118 | ret = ttm_bo_reserve(bo: &nvbo->bo, interruptible: false, no_wait: false, NULL); |
119 | if (ret) |
120 | return ret; |
121 | |
122 | ret = pm_runtime_get_sync(dev); |
123 | if (ret < 0 && ret != -EACCES) { |
124 | pm_runtime_put_autosuspend(dev); |
125 | goto out; |
126 | } |
127 | |
128 | /* only create a VMA on binding */ |
129 | if (!nouveau_cli_uvmm(cli)) |
130 | ret = nouveau_vma_new(nvbo, vmm, &vma); |
131 | else |
132 | ret = 0; |
133 | pm_runtime_mark_last_busy(dev); |
134 | pm_runtime_put_autosuspend(dev); |
135 | out: |
136 | ttm_bo_unreserve(bo: &nvbo->bo); |
137 | return ret; |
138 | } |
139 | |
140 | struct nouveau_gem_object_unmap { |
141 | struct nouveau_cli_work work; |
142 | struct nouveau_vma *vma; |
143 | }; |
144 | |
145 | static void |
146 | nouveau_gem_object_delete(struct nouveau_vma *vma) |
147 | { |
148 | nouveau_fence_unref(&vma->fence); |
149 | nouveau_vma_del(&vma); |
150 | } |
151 | |
152 | static void |
153 | nouveau_gem_object_delete_work(struct nouveau_cli_work *w) |
154 | { |
155 | struct nouveau_gem_object_unmap *work = |
156 | container_of(w, typeof(*work), work); |
157 | nouveau_gem_object_delete(vma: work->vma); |
158 | kfree(objp: work); |
159 | } |
160 | |
161 | static void |
162 | nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) |
163 | { |
164 | struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; |
165 | struct nouveau_gem_object_unmap *work; |
166 | |
167 | list_del_init(entry: &vma->head); |
168 | |
169 | if (!fence) { |
170 | nouveau_gem_object_delete(vma); |
171 | return; |
172 | } |
173 | |
174 | if (!(work = kmalloc(size: sizeof(*work), GFP_KERNEL))) { |
175 | WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); |
176 | nouveau_gem_object_delete(vma); |
177 | return; |
178 | } |
179 | |
180 | work->work.func = nouveau_gem_object_delete_work; |
181 | work->vma = vma; |
182 | nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); |
183 | } |
184 | |
185 | void |
186 | nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) |
187 | { |
188 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
189 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
190 | struct nouveau_drm *drm = nouveau_bdev(bd: nvbo->bo.bdev); |
191 | struct device *dev = drm->dev->dev; |
192 | struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); |
193 | struct nouveau_vma *vma; |
194 | int ret; |
195 | |
196 | if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) |
197 | return; |
198 | |
199 | if (nouveau_cli_uvmm(cli)) |
200 | return; |
201 | |
202 | ret = ttm_bo_reserve(bo: &nvbo->bo, interruptible: false, no_wait: false, NULL); |
203 | if (ret) |
204 | return; |
205 | |
206 | vma = nouveau_vma_find(nvbo, vmm); |
207 | if (vma) { |
208 | if (--vma->refs == 0) { |
209 | ret = pm_runtime_get_sync(dev); |
210 | if (!WARN_ON(ret < 0 && ret != -EACCES)) { |
211 | nouveau_gem_object_unmap(nvbo, vma); |
212 | pm_runtime_mark_last_busy(dev); |
213 | } |
214 | pm_runtime_put_autosuspend(dev); |
215 | } |
216 | } |
217 | ttm_bo_unreserve(bo: &nvbo->bo); |
218 | } |
219 | |
220 | const struct drm_gem_object_funcs nouveau_gem_object_funcs = { |
221 | .free = nouveau_gem_object_del, |
222 | .open = nouveau_gem_object_open, |
223 | .close = nouveau_gem_object_close, |
224 | .export = nouveau_gem_prime_export, |
225 | .pin = nouveau_gem_prime_pin, |
226 | .unpin = nouveau_gem_prime_unpin, |
227 | .get_sg_table = nouveau_gem_prime_get_sg_table, |
228 | .vmap = drm_gem_ttm_vmap, |
229 | .vunmap = drm_gem_ttm_vunmap, |
230 | .mmap = drm_gem_ttm_mmap, |
231 | .vm_ops = &nouveau_ttm_vm_ops, |
232 | }; |
233 | |
234 | int |
235 | nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, |
236 | uint32_t tile_mode, uint32_t tile_flags, |
237 | struct nouveau_bo **pnvbo) |
238 | { |
239 | struct nouveau_drm *drm = cli->drm; |
240 | struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli); |
241 | struct dma_resv *resv = NULL; |
242 | struct nouveau_bo *nvbo; |
243 | int ret; |
244 | |
245 | if (domain & NOUVEAU_GEM_DOMAIN_NO_SHARE) { |
246 | if (unlikely(!uvmm)) |
247 | return -EINVAL; |
248 | |
249 | resv = drm_gpuvm_resv(&uvmm->base); |
250 | } |
251 | |
252 | if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART))) |
253 | domain |= NOUVEAU_GEM_DOMAIN_CPU; |
254 | |
255 | nvbo = nouveau_bo_alloc(cli, size: &size, align: &align, domain, tile_mode, |
256 | tile_flags, internal: false); |
257 | if (IS_ERR(ptr: nvbo)) |
258 | return PTR_ERR(ptr: nvbo); |
259 | |
260 | nvbo->bo.base.funcs = &nouveau_gem_object_funcs; |
261 | nvbo->no_share = domain & NOUVEAU_GEM_DOMAIN_NO_SHARE; |
262 | |
263 | /* Initialize the embedded gem-object. We return a single gem-reference |
264 | * to the caller, instead of a normal nouveau_bo ttm reference. */ |
265 | ret = drm_gem_object_init(dev: drm->dev, obj: &nvbo->bo.base, size); |
266 | if (ret) { |
267 | drm_gem_object_release(obj: &nvbo->bo.base); |
268 | kfree(objp: nvbo); |
269 | return ret; |
270 | } |
271 | |
272 | if (resv) |
273 | dma_resv_lock(obj: resv, NULL); |
274 | |
275 | ret = nouveau_bo_init(nvbo, size, align, domain, NULL, robj: resv); |
276 | |
277 | if (resv) |
278 | dma_resv_unlock(obj: resv); |
279 | |
280 | if (ret) |
281 | return ret; |
282 | |
283 | /* we restrict allowed domains on nv50+ to only the types |
284 | * that were requested at creation time. not possibly on |
285 | * earlier chips without busting the ABI. |
286 | */ |
287 | nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | |
288 | NOUVEAU_GEM_DOMAIN_GART; |
289 | if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) |
290 | nvbo->valid_domains &= domain; |
291 | |
292 | if (nvbo->no_share) { |
293 | nvbo->r_obj = drm_gpuvm_resv_obj(&uvmm->base); |
294 | drm_gem_object_get(obj: nvbo->r_obj); |
295 | } |
296 | |
297 | *pnvbo = nvbo; |
298 | return 0; |
299 | } |
300 | |
301 | static int |
302 | nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, |
303 | struct drm_nouveau_gem_info *rep) |
304 | { |
305 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
306 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
307 | struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); |
308 | struct nouveau_vma *vma; |
309 | |
310 | if (is_power_of_2(n: nvbo->valid_domains)) |
311 | rep->domain = nvbo->valid_domains; |
312 | else if (nvbo->bo.resource->mem_type == TTM_PL_TT) |
313 | rep->domain = NOUVEAU_GEM_DOMAIN_GART; |
314 | else |
315 | rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; |
316 | rep->offset = nvbo->offset; |
317 | if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50 && |
318 | !nouveau_cli_uvmm(cli)) { |
319 | vma = nouveau_vma_find(nvbo, vmm); |
320 | if (!vma) |
321 | return -EINVAL; |
322 | |
323 | rep->offset = vma->addr; |
324 | } else |
325 | rep->offset = 0; |
326 | |
327 | rep->size = nvbo->bo.base.size; |
328 | rep->map_handle = drm_vma_node_offset_addr(node: &nvbo->bo.base.vma_node); |
329 | rep->tile_mode = nvbo->mode; |
330 | rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; |
331 | if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) |
332 | rep->tile_flags |= nvbo->kind << 8; |
333 | else |
334 | if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) |
335 | rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; |
336 | else |
337 | rep->tile_flags |= nvbo->zeta; |
338 | return 0; |
339 | } |
340 | |
341 | int |
342 | nouveau_gem_ioctl_new(struct drm_device *dev, void *data, |
343 | struct drm_file *file_priv) |
344 | { |
345 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
346 | struct drm_nouveau_gem_new *req = data; |
347 | struct nouveau_bo *nvbo = NULL; |
348 | int ret = 0; |
349 | |
350 | /* If uvmm wasn't initialized until now disable it completely to prevent |
351 | * userspace from mixing up UAPIs. |
352 | */ |
353 | nouveau_cli_disable_uvmm_noinit(cli); |
354 | |
355 | ret = nouveau_gem_new(cli, size: req->info.size, align: req->align, |
356 | domain: req->info.domain, tile_mode: req->info.tile_mode, |
357 | tile_flags: req->info.tile_flags, pnvbo: &nvbo); |
358 | if (ret) |
359 | return ret; |
360 | |
361 | ret = drm_gem_handle_create(file_priv, obj: &nvbo->bo.base, |
362 | handlep: &req->info.handle); |
363 | if (ret == 0) { |
364 | ret = nouveau_gem_info(file_priv, gem: &nvbo->bo.base, rep: &req->info); |
365 | if (ret) |
366 | drm_gem_handle_delete(filp: file_priv, handle: req->info.handle); |
367 | } |
368 | |
369 | /* drop reference from allocate - handle holds it now */ |
370 | drm_gem_object_put(obj: &nvbo->bo.base); |
371 | return ret; |
372 | } |
373 | |
374 | static int |
375 | nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, |
376 | uint32_t write_domains, uint32_t valid_domains) |
377 | { |
378 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
379 | struct ttm_buffer_object *bo = &nvbo->bo; |
380 | uint32_t domains = valid_domains & nvbo->valid_domains & |
381 | (write_domains ? write_domains : read_domains); |
382 | uint32_t pref_domains = 0; |
383 | |
384 | if (!domains) |
385 | return -EINVAL; |
386 | |
387 | valid_domains &= ~(NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART); |
388 | |
389 | if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && |
390 | bo->resource->mem_type == TTM_PL_VRAM) |
391 | pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; |
392 | |
393 | else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && |
394 | bo->resource->mem_type == TTM_PL_TT) |
395 | pref_domains |= NOUVEAU_GEM_DOMAIN_GART; |
396 | |
397 | else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) |
398 | pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; |
399 | |
400 | else |
401 | pref_domains |= NOUVEAU_GEM_DOMAIN_GART; |
402 | |
403 | nouveau_bo_placement_set(nvbo, type: pref_domains, busy: valid_domains); |
404 | |
405 | return 0; |
406 | } |
407 | |
408 | struct validate_op { |
409 | struct list_head list; |
410 | struct ww_acquire_ctx ticket; |
411 | }; |
412 | |
413 | static void |
414 | validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, |
415 | struct nouveau_fence *fence, |
416 | struct drm_nouveau_gem_pushbuf_bo *pbbo) |
417 | { |
418 | struct nouveau_bo *nvbo; |
419 | struct drm_nouveau_gem_pushbuf_bo *b; |
420 | |
421 | while (!list_empty(head: &op->list)) { |
422 | nvbo = list_entry(op->list.next, struct nouveau_bo, entry); |
423 | b = &pbbo[nvbo->pbbo_index]; |
424 | |
425 | if (likely(fence)) { |
426 | nouveau_bo_fence(nvbo, fence, exclusive: !!b->write_domains); |
427 | |
428 | if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { |
429 | struct nouveau_vma *vma = |
430 | (void *)(unsigned long)b->user_priv; |
431 | nouveau_fence_unref(&vma->fence); |
432 | dma_fence_get(fence: &fence->base); |
433 | vma->fence = fence; |
434 | } |
435 | } |
436 | |
437 | if (unlikely(nvbo->validate_mapped)) { |
438 | ttm_bo_kunmap(map: &nvbo->kmap); |
439 | nvbo->validate_mapped = false; |
440 | } |
441 | |
442 | list_del(entry: &nvbo->entry); |
443 | nvbo->reserved_by = NULL; |
444 | ttm_bo_unreserve(bo: &nvbo->bo); |
445 | drm_gem_object_put(obj: &nvbo->bo.base); |
446 | } |
447 | } |
448 | |
449 | static void |
450 | validate_fini(struct validate_op *op, struct nouveau_channel *chan, |
451 | struct nouveau_fence *fence, |
452 | struct drm_nouveau_gem_pushbuf_bo *pbbo) |
453 | { |
454 | validate_fini_no_ticket(op, chan, fence, pbbo); |
455 | ww_acquire_fini(ctx: &op->ticket); |
456 | } |
457 | |
458 | static int |
459 | validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, |
460 | struct drm_nouveau_gem_pushbuf_bo *pbbo, |
461 | int nr_buffers, struct validate_op *op) |
462 | { |
463 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
464 | int trycnt = 0; |
465 | int ret = -EINVAL, i; |
466 | struct nouveau_bo *res_bo = NULL; |
467 | LIST_HEAD(gart_list); |
468 | LIST_HEAD(vram_list); |
469 | LIST_HEAD(both_list); |
470 | |
471 | ww_acquire_init(ctx: &op->ticket, ww_class: &reservation_ww_class); |
472 | retry: |
473 | if (++trycnt > 100000) { |
474 | NV_PRINTK(err, cli, "%s failed and gave up.\n" , __func__); |
475 | return -EINVAL; |
476 | } |
477 | |
478 | for (i = 0; i < nr_buffers; i++) { |
479 | struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; |
480 | struct drm_gem_object *gem; |
481 | struct nouveau_bo *nvbo; |
482 | |
483 | gem = drm_gem_object_lookup(filp: file_priv, handle: b->handle); |
484 | if (!gem) { |
485 | NV_PRINTK(err, cli, "Unknown handle 0x%08x\n" , b->handle); |
486 | ret = -ENOENT; |
487 | break; |
488 | } |
489 | nvbo = nouveau_gem_object(gem); |
490 | if (nvbo == res_bo) { |
491 | res_bo = NULL; |
492 | drm_gem_object_put(obj: gem); |
493 | continue; |
494 | } |
495 | |
496 | if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { |
497 | NV_PRINTK(err, cli, "multiple instances of buffer %d on " |
498 | "validation list\n" , b->handle); |
499 | drm_gem_object_put(obj: gem); |
500 | ret = -EINVAL; |
501 | break; |
502 | } |
503 | |
504 | ret = ttm_bo_reserve(bo: &nvbo->bo, interruptible: true, no_wait: false, ticket: &op->ticket); |
505 | if (ret) { |
506 | list_splice_tail_init(list: &vram_list, head: &op->list); |
507 | list_splice_tail_init(list: &gart_list, head: &op->list); |
508 | list_splice_tail_init(list: &both_list, head: &op->list); |
509 | validate_fini_no_ticket(op, chan, NULL, NULL); |
510 | if (unlikely(ret == -EDEADLK)) { |
511 | ret = ttm_bo_reserve_slowpath(bo: &nvbo->bo, interruptible: true, |
512 | ticket: &op->ticket); |
513 | if (!ret) |
514 | res_bo = nvbo; |
515 | } |
516 | if (unlikely(ret)) { |
517 | if (ret != -ERESTARTSYS) |
518 | NV_PRINTK(err, cli, "fail reserve\n" ); |
519 | break; |
520 | } |
521 | } |
522 | |
523 | if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { |
524 | struct nouveau_vmm *vmm = chan->vmm; |
525 | struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); |
526 | if (!vma) { |
527 | NV_PRINTK(err, cli, "vma not found!\n" ); |
528 | ret = -EINVAL; |
529 | break; |
530 | } |
531 | |
532 | b->user_priv = (uint64_t)(unsigned long)vma; |
533 | } else { |
534 | b->user_priv = (uint64_t)(unsigned long)nvbo; |
535 | } |
536 | |
537 | nvbo->reserved_by = file_priv; |
538 | nvbo->pbbo_index = i; |
539 | if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && |
540 | (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) |
541 | list_add_tail(new: &nvbo->entry, head: &both_list); |
542 | else |
543 | if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) |
544 | list_add_tail(new: &nvbo->entry, head: &vram_list); |
545 | else |
546 | if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) |
547 | list_add_tail(new: &nvbo->entry, head: &gart_list); |
548 | else { |
549 | NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n" , |
550 | b->valid_domains); |
551 | list_add_tail(new: &nvbo->entry, head: &both_list); |
552 | ret = -EINVAL; |
553 | break; |
554 | } |
555 | if (nvbo == res_bo) |
556 | goto retry; |
557 | } |
558 | |
559 | ww_acquire_done(ctx: &op->ticket); |
560 | list_splice_tail(list: &vram_list, head: &op->list); |
561 | list_splice_tail(list: &gart_list, head: &op->list); |
562 | list_splice_tail(list: &both_list, head: &op->list); |
563 | if (ret) |
564 | validate_fini(op, chan, NULL, NULL); |
565 | return ret; |
566 | |
567 | } |
568 | |
569 | static int |
570 | validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, |
571 | struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) |
572 | { |
573 | struct nouveau_drm *drm = chan->drm; |
574 | struct nouveau_bo *nvbo; |
575 | int ret, relocs = 0; |
576 | |
577 | list_for_each_entry(nvbo, list, entry) { |
578 | struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; |
579 | |
580 | ret = nouveau_gem_set_domain(gem: &nvbo->bo.base, read_domains: b->read_domains, |
581 | write_domains: b->write_domains, |
582 | valid_domains: b->valid_domains); |
583 | if (unlikely(ret)) { |
584 | NV_PRINTK(err, cli, "fail set_domain\n" ); |
585 | return ret; |
586 | } |
587 | |
588 | ret = nouveau_bo_validate(nvbo, interruptible: true, no_wait_gpu: false); |
589 | if (unlikely(ret)) { |
590 | if (ret != -ERESTARTSYS) |
591 | NV_PRINTK(err, cli, "fail ttm_validate\n" ); |
592 | return ret; |
593 | } |
594 | |
595 | ret = nouveau_fence_sync(nvbo, chan, exclusive: !!b->write_domains, intr: true); |
596 | if (unlikely(ret)) { |
597 | if (ret != -ERESTARTSYS) |
598 | NV_PRINTK(err, cli, "fail post-validate sync\n" ); |
599 | return ret; |
600 | } |
601 | |
602 | if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { |
603 | if (nvbo->offset == b->presumed.offset && |
604 | ((nvbo->bo.resource->mem_type == TTM_PL_VRAM && |
605 | b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || |
606 | (nvbo->bo.resource->mem_type == TTM_PL_TT && |
607 | b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) |
608 | continue; |
609 | |
610 | if (nvbo->bo.resource->mem_type == TTM_PL_TT) |
611 | b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; |
612 | else |
613 | b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; |
614 | b->presumed.offset = nvbo->offset; |
615 | b->presumed.valid = 0; |
616 | relocs++; |
617 | } |
618 | } |
619 | |
620 | return relocs; |
621 | } |
622 | |
623 | static int |
624 | nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, |
625 | struct drm_file *file_priv, |
626 | struct drm_nouveau_gem_pushbuf_bo *pbbo, |
627 | int nr_buffers, |
628 | struct validate_op *op, bool *apply_relocs) |
629 | { |
630 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
631 | int ret; |
632 | |
633 | INIT_LIST_HEAD(list: &op->list); |
634 | |
635 | if (nr_buffers == 0) |
636 | return 0; |
637 | |
638 | ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); |
639 | if (unlikely(ret)) { |
640 | if (ret != -ERESTARTSYS) |
641 | NV_PRINTK(err, cli, "validate_init\n" ); |
642 | return ret; |
643 | } |
644 | |
645 | ret = validate_list(chan, cli, list: &op->list, pbbo); |
646 | if (unlikely(ret < 0)) { |
647 | if (ret != -ERESTARTSYS) |
648 | NV_PRINTK(err, cli, "validating bo list\n" ); |
649 | validate_fini(op, chan, NULL, NULL); |
650 | return ret; |
651 | } else if (ret > 0) { |
652 | *apply_relocs = true; |
653 | } |
654 | |
655 | return 0; |
656 | } |
657 | |
658 | static int |
659 | nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, |
660 | struct drm_nouveau_gem_pushbuf *req, |
661 | struct drm_nouveau_gem_pushbuf_reloc *reloc, |
662 | struct drm_nouveau_gem_pushbuf_bo *bo) |
663 | { |
664 | int ret = 0; |
665 | unsigned i; |
666 | |
667 | for (i = 0; i < req->nr_relocs; i++) { |
668 | struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; |
669 | struct drm_nouveau_gem_pushbuf_bo *b; |
670 | struct nouveau_bo *nvbo; |
671 | uint32_t data; |
672 | long lret; |
673 | |
674 | if (unlikely(r->bo_index >= req->nr_buffers)) { |
675 | NV_PRINTK(err, cli, "reloc bo index invalid\n" ); |
676 | ret = -EINVAL; |
677 | break; |
678 | } |
679 | |
680 | b = &bo[r->bo_index]; |
681 | if (b->presumed.valid) |
682 | continue; |
683 | |
684 | if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { |
685 | NV_PRINTK(err, cli, "reloc container bo index invalid\n" ); |
686 | ret = -EINVAL; |
687 | break; |
688 | } |
689 | nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; |
690 | |
691 | if (unlikely(r->reloc_bo_offset + 4 > |
692 | nvbo->bo.base.size)) { |
693 | NV_PRINTK(err, cli, "reloc outside of bo\n" ); |
694 | ret = -EINVAL; |
695 | break; |
696 | } |
697 | |
698 | if (!nvbo->kmap.virtual) { |
699 | ret = ttm_bo_kmap(bo: &nvbo->bo, start_page: 0, PFN_UP(nvbo->bo.base.size), |
700 | map: &nvbo->kmap); |
701 | if (ret) { |
702 | NV_PRINTK(err, cli, "failed kmap for reloc\n" ); |
703 | break; |
704 | } |
705 | nvbo->validate_mapped = true; |
706 | } |
707 | |
708 | if (r->flags & NOUVEAU_GEM_RELOC_LOW) |
709 | data = b->presumed.offset + r->data; |
710 | else |
711 | if (r->flags & NOUVEAU_GEM_RELOC_HIGH) |
712 | data = (b->presumed.offset + r->data) >> 32; |
713 | else |
714 | data = r->data; |
715 | |
716 | if (r->flags & NOUVEAU_GEM_RELOC_OR) { |
717 | if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) |
718 | data |= r->tor; |
719 | else |
720 | data |= r->vor; |
721 | } |
722 | |
723 | lret = dma_resv_wait_timeout(obj: nvbo->bo.base.resv, |
724 | usage: DMA_RESV_USAGE_BOOKKEEP, |
725 | intr: false, timeout: 15 * HZ); |
726 | if (!lret) |
727 | ret = -EBUSY; |
728 | else if (lret > 0) |
729 | ret = 0; |
730 | else |
731 | ret = lret; |
732 | |
733 | if (ret) { |
734 | NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n" , |
735 | ret); |
736 | break; |
737 | } |
738 | |
739 | nouveau_bo_wr32(nvbo, index: r->reloc_bo_offset >> 2, val: data); |
740 | } |
741 | |
742 | return ret; |
743 | } |
744 | |
745 | int |
746 | nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, |
747 | struct drm_file *file_priv) |
748 | { |
749 | struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); |
750 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
751 | struct nouveau_abi16_chan *temp; |
752 | struct nouveau_drm *drm = nouveau_drm(dev); |
753 | struct drm_nouveau_gem_pushbuf *req = data; |
754 | struct drm_nouveau_gem_pushbuf_push *push; |
755 | struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; |
756 | struct drm_nouveau_gem_pushbuf_bo *bo; |
757 | struct nouveau_channel *chan = NULL; |
758 | struct validate_op op; |
759 | struct nouveau_fence *fence = NULL; |
760 | int i, j, ret = 0; |
761 | bool do_reloc = false, sync = false; |
762 | |
763 | if (unlikely(!abi16)) |
764 | return -ENOMEM; |
765 | |
766 | if (unlikely(nouveau_cli_uvmm(cli))) |
767 | return nouveau_abi16_put(abi16, -ENOSYS); |
768 | |
769 | list_for_each_entry(temp, &abi16->channels, head) { |
770 | if (temp->chan->chid == req->channel) { |
771 | chan = temp->chan; |
772 | break; |
773 | } |
774 | } |
775 | |
776 | if (!chan) |
777 | return nouveau_abi16_put(abi16, -ENOENT); |
778 | if (unlikely(atomic_read(&chan->killed))) |
779 | return nouveau_abi16_put(abi16, -ENODEV); |
780 | |
781 | sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; |
782 | |
783 | req->vram_available = drm->gem.vram_available; |
784 | req->gart_available = drm->gem.gart_available; |
785 | if (unlikely(req->nr_push == 0)) |
786 | goto out_next; |
787 | |
788 | if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { |
789 | NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n" , |
790 | req->nr_push, NOUVEAU_GEM_MAX_PUSH); |
791 | return nouveau_abi16_put(abi16, -EINVAL); |
792 | } |
793 | |
794 | if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { |
795 | NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n" , |
796 | req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); |
797 | return nouveau_abi16_put(abi16, -EINVAL); |
798 | } |
799 | |
800 | if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { |
801 | NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n" , |
802 | req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); |
803 | return nouveau_abi16_put(abi16, -EINVAL); |
804 | } |
805 | |
806 | push = u_memcpya(user: req->push, nmemb: req->nr_push, size: sizeof(*push)); |
807 | if (IS_ERR(ptr: push)) |
808 | return nouveau_abi16_put(abi16, PTR_ERR(ptr: push)); |
809 | |
810 | bo = u_memcpya(user: req->buffers, nmemb: req->nr_buffers, size: sizeof(*bo)); |
811 | if (IS_ERR(ptr: bo)) { |
812 | u_free(addr: push); |
813 | return nouveau_abi16_put(abi16, PTR_ERR(ptr: bo)); |
814 | } |
815 | |
816 | /* Ensure all push buffers are on validate list */ |
817 | for (i = 0; i < req->nr_push; i++) { |
818 | if (push[i].bo_index >= req->nr_buffers) { |
819 | NV_PRINTK(err, cli, "push %d buffer not in list\n" , i); |
820 | ret = -EINVAL; |
821 | goto out_prevalid; |
822 | } |
823 | } |
824 | |
825 | /* Validate buffer list */ |
826 | revalidate: |
827 | ret = nouveau_gem_pushbuf_validate(chan, file_priv, pbbo: bo, |
828 | nr_buffers: req->nr_buffers, op: &op, apply_relocs: &do_reloc); |
829 | if (ret) { |
830 | if (ret != -ERESTARTSYS) |
831 | NV_PRINTK(err, cli, "validate: %d\n" , ret); |
832 | goto out_prevalid; |
833 | } |
834 | |
835 | /* Apply any relocations that are required */ |
836 | if (do_reloc) { |
837 | if (!reloc) { |
838 | validate_fini(op: &op, chan, NULL, pbbo: bo); |
839 | reloc = u_memcpya(user: req->relocs, nmemb: req->nr_relocs, size: sizeof(*reloc)); |
840 | if (IS_ERR(ptr: reloc)) { |
841 | ret = PTR_ERR(ptr: reloc); |
842 | goto out_prevalid; |
843 | } |
844 | |
845 | goto revalidate; |
846 | } |
847 | |
848 | ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); |
849 | if (ret) { |
850 | NV_PRINTK(err, cli, "reloc apply: %d\n" , ret); |
851 | goto out; |
852 | } |
853 | } |
854 | |
855 | if (chan->dma.ib_max) { |
856 | ret = nouveau_dma_wait(chan, slots: req->nr_push + 1, size: 16); |
857 | if (ret) { |
858 | NV_PRINTK(err, cli, "nv50cal_space: %d\n" , ret); |
859 | goto out; |
860 | } |
861 | |
862 | for (i = 0; i < req->nr_push; i++) { |
863 | struct nouveau_vma *vma = (void *)(unsigned long) |
864 | bo[push[i].bo_index].user_priv; |
865 | u64 addr = vma->addr + push[i].offset; |
866 | u32 length = push[i].length & ~NOUVEAU_GEM_PUSHBUF_NO_PREFETCH; |
867 | bool no_prefetch = push[i].length & NOUVEAU_GEM_PUSHBUF_NO_PREFETCH; |
868 | |
869 | nv50_dma_push(chan, addr, length, no_prefetch); |
870 | } |
871 | } else |
872 | if (drm->client.device.info.chipset >= 0x25) { |
873 | ret = PUSH_WAIT(chan->chan.push, req->nr_push * 2); |
874 | if (ret) { |
875 | NV_PRINTK(err, cli, "cal_space: %d\n" , ret); |
876 | goto out; |
877 | } |
878 | |
879 | for (i = 0; i < req->nr_push; i++) { |
880 | struct nouveau_bo *nvbo = (void *)(unsigned long) |
881 | bo[push[i].bo_index].user_priv; |
882 | |
883 | PUSH_CALL(chan->chan.push, nvbo->offset + push[i].offset); |
884 | PUSH_DATA(chan->chan.push, 0); |
885 | } |
886 | } else { |
887 | ret = PUSH_WAIT(chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); |
888 | if (ret) { |
889 | NV_PRINTK(err, cli, "jmp_space: %d\n" , ret); |
890 | goto out; |
891 | } |
892 | |
893 | for (i = 0; i < req->nr_push; i++) { |
894 | struct nouveau_bo *nvbo = (void *)(unsigned long) |
895 | bo[push[i].bo_index].user_priv; |
896 | uint32_t cmd; |
897 | |
898 | cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); |
899 | cmd |= 0x20000000; |
900 | if (unlikely(cmd != req->suffix0)) { |
901 | if (!nvbo->kmap.virtual) { |
902 | ret = ttm_bo_kmap(bo: &nvbo->bo, start_page: 0, |
903 | PFN_UP(nvbo->bo.base.size), |
904 | map: &nvbo->kmap); |
905 | if (ret) { |
906 | WIND_RING(chan); |
907 | goto out; |
908 | } |
909 | nvbo->validate_mapped = true; |
910 | } |
911 | |
912 | nouveau_bo_wr32(nvbo, index: (push[i].offset + |
913 | push[i].length - 8) / 4, val: cmd); |
914 | } |
915 | |
916 | PUSH_JUMP(chan->chan.push, nvbo->offset + push[i].offset); |
917 | PUSH_DATA(chan->chan.push, 0); |
918 | for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) |
919 | PUSH_DATA(chan->chan.push, 0); |
920 | } |
921 | } |
922 | |
923 | ret = nouveau_fence_new(&fence, chan); |
924 | if (ret) { |
925 | NV_PRINTK(err, cli, "error fencing pushbuf: %d\n" , ret); |
926 | WIND_RING(chan); |
927 | goto out; |
928 | } |
929 | |
930 | if (sync) { |
931 | if (!(ret = nouveau_fence_wait(fence, lazy: false, intr: false))) { |
932 | if ((ret = dma_fence_get_status(fence: &fence->base)) == 1) |
933 | ret = 0; |
934 | } |
935 | } |
936 | |
937 | out: |
938 | validate_fini(op: &op, chan, fence, pbbo: bo); |
939 | nouveau_fence_unref(&fence); |
940 | |
941 | if (do_reloc) { |
942 | struct drm_nouveau_gem_pushbuf_bo __user *upbbo = |
943 | u64_to_user_ptr(req->buffers); |
944 | |
945 | for (i = 0; i < req->nr_buffers; i++) { |
946 | if (bo[i].presumed.valid) |
947 | continue; |
948 | |
949 | if (copy_to_user(to: &upbbo[i].presumed, from: &bo[i].presumed, |
950 | n: sizeof(bo[i].presumed))) { |
951 | ret = -EFAULT; |
952 | break; |
953 | } |
954 | } |
955 | } |
956 | out_prevalid: |
957 | if (!IS_ERR(ptr: reloc)) |
958 | u_free(addr: reloc); |
959 | u_free(addr: bo); |
960 | u_free(addr: push); |
961 | |
962 | out_next: |
963 | if (chan->dma.ib_max) { |
964 | req->suffix0 = 0x00000000; |
965 | req->suffix1 = 0x00000000; |
966 | } else |
967 | if (drm->client.device.info.chipset >= 0x25) { |
968 | req->suffix0 = 0x00020000; |
969 | req->suffix1 = 0x00000000; |
970 | } else { |
971 | req->suffix0 = 0x20000000 | |
972 | (chan->push.addr + ((chan->dma.cur + 2) << 2)); |
973 | req->suffix1 = 0x00000000; |
974 | } |
975 | |
976 | return nouveau_abi16_put(abi16, ret); |
977 | } |
978 | |
979 | int |
980 | nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, |
981 | struct drm_file *file_priv) |
982 | { |
983 | struct drm_nouveau_gem_cpu_prep *req = data; |
984 | struct drm_gem_object *gem; |
985 | struct nouveau_bo *nvbo; |
986 | bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); |
987 | bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); |
988 | long lret; |
989 | int ret; |
990 | |
991 | gem = drm_gem_object_lookup(filp: file_priv, handle: req->handle); |
992 | if (!gem) |
993 | return -ENOENT; |
994 | nvbo = nouveau_gem_object(gem); |
995 | |
996 | lret = dma_resv_wait_timeout(obj: nvbo->bo.base.resv, |
997 | usage: dma_resv_usage_rw(write), intr: true, |
998 | timeout: no_wait ? 0 : 30 * HZ); |
999 | if (!lret) |
1000 | ret = -EBUSY; |
1001 | else if (lret > 0) |
1002 | ret = 0; |
1003 | else |
1004 | ret = lret; |
1005 | |
1006 | nouveau_bo_sync_for_cpu(nvbo); |
1007 | drm_gem_object_put(obj: gem); |
1008 | |
1009 | return ret; |
1010 | } |
1011 | |
1012 | int |
1013 | nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, |
1014 | struct drm_file *file_priv) |
1015 | { |
1016 | struct drm_nouveau_gem_cpu_fini *req = data; |
1017 | struct drm_gem_object *gem; |
1018 | struct nouveau_bo *nvbo; |
1019 | |
1020 | gem = drm_gem_object_lookup(filp: file_priv, handle: req->handle); |
1021 | if (!gem) |
1022 | return -ENOENT; |
1023 | nvbo = nouveau_gem_object(gem); |
1024 | |
1025 | nouveau_bo_sync_for_device(nvbo); |
1026 | drm_gem_object_put(obj: gem); |
1027 | return 0; |
1028 | } |
1029 | |
1030 | int |
1031 | nouveau_gem_ioctl_info(struct drm_device *dev, void *data, |
1032 | struct drm_file *file_priv) |
1033 | { |
1034 | struct drm_nouveau_gem_info *req = data; |
1035 | struct drm_gem_object *gem; |
1036 | int ret; |
1037 | |
1038 | gem = drm_gem_object_lookup(filp: file_priv, handle: req->handle); |
1039 | if (!gem) |
1040 | return -ENOENT; |
1041 | |
1042 | ret = nouveau_gem_info(file_priv, gem, rep: req); |
1043 | drm_gem_object_put(obj: gem); |
1044 | return ret; |
1045 | } |
1046 | |
1047 | |