1 | /* |
2 | * Copyright 2009 Jerome Glisse. |
3 | * All Rights Reserved. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * copy of this software and associated documentation files (the |
7 | * "Software"), to deal in the Software without restriction, including |
8 | * without limitation the rights to use, copy, modify, merge, publish, |
9 | * distribute, sub license, and/or sell copies of the Software, and to |
10 | * permit persons to whom the Software is furnished to do so, subject to |
11 | * the following conditions: |
12 | * |
13 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
14 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
15 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
16 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
17 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
18 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
19 | * USE OR OTHER DEALINGS IN THE SOFTWARE. |
20 | * |
21 | * The above copyright notice and this permission notice (including the |
22 | * next paragraph) shall be included in all copies or substantial portions |
23 | * of the Software. |
24 | * |
25 | */ |
26 | /* |
27 | * Authors: |
28 | * Jerome Glisse <glisse@freedesktop.org> |
29 | * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> |
30 | * Dave Airlie |
31 | */ |
32 | |
33 | #include <linux/dma-mapping.h> |
34 | #include <linux/pagemap.h> |
35 | #include <linux/pci.h> |
36 | #include <linux/seq_file.h> |
37 | #include <linux/slab.h> |
38 | #include <linux/swap.h> |
39 | |
40 | #include <drm/drm_device.h> |
41 | #include <drm/drm_file.h> |
42 | #include <drm/drm_prime.h> |
43 | #include <drm/radeon_drm.h> |
44 | #include <drm/ttm/ttm_bo.h> |
45 | #include <drm/ttm/ttm_placement.h> |
46 | #include <drm/ttm/ttm_range_manager.h> |
47 | #include <drm/ttm/ttm_tt.h> |
48 | |
49 | #include "radeon_reg.h" |
50 | #include "radeon.h" |
51 | #include "radeon_ttm.h" |
52 | |
53 | static void radeon_ttm_debugfs_init(struct radeon_device *rdev); |
54 | |
55 | static int radeon_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, |
56 | struct ttm_resource *bo_mem); |
57 | static void radeon_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm); |
58 | |
59 | struct radeon_device *radeon_get_rdev(struct ttm_device *bdev) |
60 | { |
61 | struct radeon_mman *mman; |
62 | struct radeon_device *rdev; |
63 | |
64 | mman = container_of(bdev, struct radeon_mman, bdev); |
65 | rdev = container_of(mman, struct radeon_device, mman); |
66 | return rdev; |
67 | } |
68 | |
69 | static int radeon_ttm_init_vram(struct radeon_device *rdev) |
70 | { |
71 | return ttm_range_man_init(bdev: &rdev->mman.bdev, TTM_PL_VRAM, |
72 | use_tt: false, p_size: rdev->mc.real_vram_size >> PAGE_SHIFT); |
73 | } |
74 | |
75 | static int radeon_ttm_init_gtt(struct radeon_device *rdev) |
76 | { |
77 | return ttm_range_man_init(bdev: &rdev->mman.bdev, TTM_PL_TT, |
78 | use_tt: true, p_size: rdev->mc.gtt_size >> PAGE_SHIFT); |
79 | } |
80 | |
81 | static void radeon_evict_flags(struct ttm_buffer_object *bo, |
82 | struct ttm_placement *placement) |
83 | { |
84 | static const struct ttm_place placements = { |
85 | .fpfn = 0, |
86 | .lpfn = 0, |
87 | .mem_type = TTM_PL_SYSTEM, |
88 | .flags = 0 |
89 | }; |
90 | |
91 | struct radeon_bo *rbo; |
92 | |
93 | if (!radeon_ttm_bo_is_radeon_bo(bo)) { |
94 | placement->placement = &placements; |
95 | placement->busy_placement = &placements; |
96 | placement->num_placement = 1; |
97 | placement->num_busy_placement = 1; |
98 | return; |
99 | } |
100 | rbo = container_of(bo, struct radeon_bo, tbo); |
101 | switch (bo->resource->mem_type) { |
102 | case TTM_PL_VRAM: |
103 | if (rbo->rdev->ring[radeon_copy_ring_index(rbo->rdev)].ready == false) |
104 | radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); |
105 | else if (rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size && |
106 | bo->resource->start < (rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT)) { |
107 | unsigned fpfn = rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; |
108 | int i; |
109 | |
110 | /* Try evicting to the CPU inaccessible part of VRAM |
111 | * first, but only set GTT as busy placement, so this |
112 | * BO will be evicted to GTT rather than causing other |
113 | * BOs to be evicted from VRAM |
114 | */ |
115 | radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM | |
116 | RADEON_GEM_DOMAIN_GTT); |
117 | rbo->placement.num_busy_placement = 0; |
118 | for (i = 0; i < rbo->placement.num_placement; i++) { |
119 | if (rbo->placements[i].mem_type == TTM_PL_VRAM) { |
120 | if (rbo->placements[i].fpfn < fpfn) |
121 | rbo->placements[i].fpfn = fpfn; |
122 | } else { |
123 | rbo->placement.busy_placement = |
124 | &rbo->placements[i]; |
125 | rbo->placement.num_busy_placement = 1; |
126 | } |
127 | } |
128 | } else |
129 | radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); |
130 | break; |
131 | case TTM_PL_TT: |
132 | default: |
133 | radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); |
134 | } |
135 | *placement = rbo->placement; |
136 | } |
137 | |
138 | static int radeon_move_blit(struct ttm_buffer_object *bo, |
139 | bool evict, |
140 | struct ttm_resource *new_mem, |
141 | struct ttm_resource *old_mem) |
142 | { |
143 | struct radeon_device *rdev; |
144 | uint64_t old_start, new_start; |
145 | struct radeon_fence *fence; |
146 | unsigned num_pages; |
147 | int r, ridx; |
148 | |
149 | rdev = radeon_get_rdev(bdev: bo->bdev); |
150 | ridx = radeon_copy_ring_index(rdev); |
151 | old_start = (u64)old_mem->start << PAGE_SHIFT; |
152 | new_start = (u64)new_mem->start << PAGE_SHIFT; |
153 | |
154 | switch (old_mem->mem_type) { |
155 | case TTM_PL_VRAM: |
156 | old_start += rdev->mc.vram_start; |
157 | break; |
158 | case TTM_PL_TT: |
159 | old_start += rdev->mc.gtt_start; |
160 | break; |
161 | default: |
162 | DRM_ERROR("Unknown placement %d\n" , old_mem->mem_type); |
163 | return -EINVAL; |
164 | } |
165 | switch (new_mem->mem_type) { |
166 | case TTM_PL_VRAM: |
167 | new_start += rdev->mc.vram_start; |
168 | break; |
169 | case TTM_PL_TT: |
170 | new_start += rdev->mc.gtt_start; |
171 | break; |
172 | default: |
173 | DRM_ERROR("Unknown placement %d\n" , old_mem->mem_type); |
174 | return -EINVAL; |
175 | } |
176 | if (!rdev->ring[ridx].ready) { |
177 | DRM_ERROR("Trying to move memory with ring turned off.\n" ); |
178 | return -EINVAL; |
179 | } |
180 | |
181 | BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); |
182 | |
183 | num_pages = PFN_UP(new_mem->size) * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); |
184 | fence = radeon_copy(rdev, old_start, new_start, num_pages, bo->base.resv); |
185 | if (IS_ERR(ptr: fence)) |
186 | return PTR_ERR(ptr: fence); |
187 | |
188 | r = ttm_bo_move_accel_cleanup(bo, fence: &fence->base, evict, pipeline: false, new_mem); |
189 | radeon_fence_unref(fence: &fence); |
190 | return r; |
191 | } |
192 | |
193 | static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, |
194 | struct ttm_operation_ctx *ctx, |
195 | struct ttm_resource *new_mem, |
196 | struct ttm_place *hop) |
197 | { |
198 | struct ttm_resource *old_mem = bo->resource; |
199 | struct radeon_device *rdev; |
200 | int r; |
201 | |
202 | if (new_mem->mem_type == TTM_PL_TT) { |
203 | r = radeon_ttm_tt_bind(bdev: bo->bdev, ttm: bo->ttm, bo_mem: new_mem); |
204 | if (r) |
205 | return r; |
206 | } |
207 | |
208 | r = ttm_bo_wait_ctx(bo, ctx); |
209 | if (r) |
210 | return r; |
211 | |
212 | rdev = radeon_get_rdev(bdev: bo->bdev); |
213 | if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM && |
214 | bo->ttm == NULL)) { |
215 | ttm_bo_move_null(bo, new_mem); |
216 | goto out; |
217 | } |
218 | if (old_mem->mem_type == TTM_PL_SYSTEM && |
219 | new_mem->mem_type == TTM_PL_TT) { |
220 | ttm_bo_move_null(bo, new_mem); |
221 | goto out; |
222 | } |
223 | |
224 | if (old_mem->mem_type == TTM_PL_TT && |
225 | new_mem->mem_type == TTM_PL_SYSTEM) { |
226 | radeon_ttm_tt_unbind(bdev: bo->bdev, ttm: bo->ttm); |
227 | ttm_resource_free(bo, res: &bo->resource); |
228 | ttm_bo_assign_mem(bo, new_mem); |
229 | goto out; |
230 | } |
231 | if (rdev->ring[radeon_copy_ring_index(rdev)].ready && |
232 | rdev->asic->copy.copy != NULL) { |
233 | if ((old_mem->mem_type == TTM_PL_SYSTEM && |
234 | new_mem->mem_type == TTM_PL_VRAM) || |
235 | (old_mem->mem_type == TTM_PL_VRAM && |
236 | new_mem->mem_type == TTM_PL_SYSTEM)) { |
237 | hop->fpfn = 0; |
238 | hop->lpfn = 0; |
239 | hop->mem_type = TTM_PL_TT; |
240 | hop->flags = 0; |
241 | return -EMULTIHOP; |
242 | } |
243 | |
244 | r = radeon_move_blit(bo, evict, new_mem, old_mem); |
245 | } else { |
246 | r = -ENODEV; |
247 | } |
248 | |
249 | if (r) { |
250 | r = ttm_bo_move_memcpy(bo, ctx, new_mem); |
251 | if (r) |
252 | return r; |
253 | } |
254 | |
255 | out: |
256 | /* update statistics */ |
257 | atomic64_add(i: bo->base.size, v: &rdev->num_bytes_moved); |
258 | radeon_bo_move_notify(bo); |
259 | return 0; |
260 | } |
261 | |
262 | static int radeon_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem) |
263 | { |
264 | struct radeon_device *rdev = radeon_get_rdev(bdev); |
265 | size_t bus_size = (size_t)mem->size; |
266 | |
267 | switch (mem->mem_type) { |
268 | case TTM_PL_SYSTEM: |
269 | /* system memory */ |
270 | return 0; |
271 | case TTM_PL_TT: |
272 | #if IS_ENABLED(CONFIG_AGP) |
273 | if (rdev->flags & RADEON_IS_AGP) { |
274 | /* RADEON_IS_AGP is set only if AGP is active */ |
275 | mem->bus.offset = (mem->start << PAGE_SHIFT) + |
276 | rdev->mc.agp_base; |
277 | mem->bus.is_iomem = !rdev->agp->cant_use_aperture; |
278 | mem->bus.caching = ttm_write_combined; |
279 | } |
280 | #endif |
281 | break; |
282 | case TTM_PL_VRAM: |
283 | mem->bus.offset = mem->start << PAGE_SHIFT; |
284 | /* check if it's visible */ |
285 | if ((mem->bus.offset + bus_size) > rdev->mc.visible_vram_size) |
286 | return -EINVAL; |
287 | mem->bus.offset += rdev->mc.aper_base; |
288 | mem->bus.is_iomem = true; |
289 | mem->bus.caching = ttm_write_combined; |
290 | #ifdef __alpha__ |
291 | /* |
292 | * Alpha: use bus.addr to hold the ioremap() return, |
293 | * so we can modify bus.base below. |
294 | */ |
295 | mem->bus.addr = ioremap_wc(mem->bus.offset, bus_size); |
296 | if (!mem->bus.addr) |
297 | return -ENOMEM; |
298 | |
299 | /* |
300 | * Alpha: Use just the bus offset plus |
301 | * the hose/domain memory base for bus.base. |
302 | * It then can be used to build PTEs for VRAM |
303 | * access, as done in ttm_bo_vm_fault(). |
304 | */ |
305 | mem->bus.offset = (mem->bus.offset & 0x0ffffffffUL) + |
306 | rdev->hose->dense_mem_base; |
307 | #endif |
308 | break; |
309 | default: |
310 | return -EINVAL; |
311 | } |
312 | return 0; |
313 | } |
314 | |
315 | /* |
316 | * TTM backend functions. |
317 | */ |
318 | struct radeon_ttm_tt { |
319 | struct ttm_tt ttm; |
320 | u64 offset; |
321 | |
322 | uint64_t userptr; |
323 | struct mm_struct *usermm; |
324 | uint32_t userflags; |
325 | bool bound; |
326 | }; |
327 | |
328 | /* prepare the sg table with the user pages */ |
329 | static int radeon_ttm_tt_pin_userptr(struct ttm_device *bdev, struct ttm_tt *ttm) |
330 | { |
331 | struct radeon_device *rdev = radeon_get_rdev(bdev); |
332 | struct radeon_ttm_tt *gtt = (void *)ttm; |
333 | unsigned pinned = 0; |
334 | int r; |
335 | |
336 | int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); |
337 | enum dma_data_direction direction = write ? |
338 | DMA_BIDIRECTIONAL : DMA_TO_DEVICE; |
339 | |
340 | if (current->mm != gtt->usermm) |
341 | return -EPERM; |
342 | |
343 | if (gtt->userflags & RADEON_GEM_USERPTR_ANONONLY) { |
344 | /* check that we only pin down anonymous memory |
345 | to prevent problems with writeback */ |
346 | unsigned long end = gtt->userptr + (u64)ttm->num_pages * PAGE_SIZE; |
347 | struct vm_area_struct *vma; |
348 | vma = find_vma(mm: gtt->usermm, addr: gtt->userptr); |
349 | if (!vma || vma->vm_file || vma->vm_end < end) |
350 | return -EPERM; |
351 | } |
352 | |
353 | do { |
354 | unsigned num_pages = ttm->num_pages - pinned; |
355 | uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE; |
356 | struct page **pages = ttm->pages + pinned; |
357 | |
358 | r = get_user_pages(start: userptr, nr_pages: num_pages, gup_flags: write ? FOLL_WRITE : 0, |
359 | pages); |
360 | if (r < 0) |
361 | goto release_pages; |
362 | |
363 | pinned += r; |
364 | |
365 | } while (pinned < ttm->num_pages); |
366 | |
367 | r = sg_alloc_table_from_pages(sgt: ttm->sg, pages: ttm->pages, n_pages: ttm->num_pages, offset: 0, |
368 | size: (u64)ttm->num_pages << PAGE_SHIFT, |
369 | GFP_KERNEL); |
370 | if (r) |
371 | goto release_sg; |
372 | |
373 | r = dma_map_sgtable(dev: rdev->dev, sgt: ttm->sg, dir: direction, attrs: 0); |
374 | if (r) |
375 | goto release_sg; |
376 | |
377 | drm_prime_sg_to_dma_addr_array(sgt: ttm->sg, addrs: gtt->ttm.dma_address, |
378 | max_pages: ttm->num_pages); |
379 | |
380 | return 0; |
381 | |
382 | release_sg: |
383 | kfree(objp: ttm->sg); |
384 | |
385 | release_pages: |
386 | release_pages(ttm->pages, nr: pinned); |
387 | return r; |
388 | } |
389 | |
390 | static void radeon_ttm_tt_unpin_userptr(struct ttm_device *bdev, struct ttm_tt *ttm) |
391 | { |
392 | struct radeon_device *rdev = radeon_get_rdev(bdev); |
393 | struct radeon_ttm_tt *gtt = (void *)ttm; |
394 | struct sg_page_iter sg_iter; |
395 | |
396 | int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); |
397 | enum dma_data_direction direction = write ? |
398 | DMA_BIDIRECTIONAL : DMA_TO_DEVICE; |
399 | |
400 | /* double check that we don't free the table twice */ |
401 | if (!ttm->sg || !ttm->sg->sgl) |
402 | return; |
403 | |
404 | /* free the sg table and pages again */ |
405 | dma_unmap_sgtable(dev: rdev->dev, sgt: ttm->sg, dir: direction, attrs: 0); |
406 | |
407 | for_each_sgtable_page(ttm->sg, &sg_iter, 0) { |
408 | struct page *page = sg_page_iter_page(piter: &sg_iter); |
409 | if (!(gtt->userflags & RADEON_GEM_USERPTR_READONLY)) |
410 | set_page_dirty(page); |
411 | |
412 | mark_page_accessed(page); |
413 | put_page(page); |
414 | } |
415 | |
416 | sg_free_table(ttm->sg); |
417 | } |
418 | |
419 | static bool radeon_ttm_backend_is_bound(struct ttm_tt *ttm) |
420 | { |
421 | struct radeon_ttm_tt *gtt = (void*)ttm; |
422 | |
423 | return (gtt->bound); |
424 | } |
425 | |
426 | static int radeon_ttm_backend_bind(struct ttm_device *bdev, |
427 | struct ttm_tt *ttm, |
428 | struct ttm_resource *bo_mem) |
429 | { |
430 | struct radeon_ttm_tt *gtt = (void*)ttm; |
431 | struct radeon_device *rdev = radeon_get_rdev(bdev); |
432 | uint32_t flags = RADEON_GART_PAGE_VALID | RADEON_GART_PAGE_READ | |
433 | RADEON_GART_PAGE_WRITE; |
434 | int r; |
435 | |
436 | if (gtt->bound) |
437 | return 0; |
438 | |
439 | if (gtt->userptr) { |
440 | radeon_ttm_tt_pin_userptr(bdev, ttm); |
441 | flags &= ~RADEON_GART_PAGE_WRITE; |
442 | } |
443 | |
444 | gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); |
445 | if (!ttm->num_pages) { |
446 | WARN(1, "nothing to bind %u pages for mreg %p back %p!\n" , |
447 | ttm->num_pages, bo_mem, ttm); |
448 | } |
449 | if (ttm->caching == ttm_cached) |
450 | flags |= RADEON_GART_PAGE_SNOOP; |
451 | r = radeon_gart_bind(rdev, offset: gtt->offset, pages: ttm->num_pages, |
452 | pagelist: ttm->pages, dma_addr: gtt->ttm.dma_address, flags); |
453 | if (r) { |
454 | DRM_ERROR("failed to bind %u pages at 0x%08X\n" , |
455 | ttm->num_pages, (unsigned)gtt->offset); |
456 | return r; |
457 | } |
458 | gtt->bound = true; |
459 | return 0; |
460 | } |
461 | |
462 | static void radeon_ttm_backend_unbind(struct ttm_device *bdev, struct ttm_tt *ttm) |
463 | { |
464 | struct radeon_ttm_tt *gtt = (void *)ttm; |
465 | struct radeon_device *rdev = radeon_get_rdev(bdev); |
466 | |
467 | if (gtt->userptr) |
468 | radeon_ttm_tt_unpin_userptr(bdev, ttm); |
469 | |
470 | if (!gtt->bound) |
471 | return; |
472 | |
473 | radeon_gart_unbind(rdev, offset: gtt->offset, pages: ttm->num_pages); |
474 | |
475 | gtt->bound = false; |
476 | } |
477 | |
478 | static void radeon_ttm_backend_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) |
479 | { |
480 | struct radeon_ttm_tt *gtt = (void *)ttm; |
481 | |
482 | ttm_tt_fini(ttm: >t->ttm); |
483 | kfree(objp: gtt); |
484 | } |
485 | |
486 | static struct ttm_tt *radeon_ttm_tt_create(struct ttm_buffer_object *bo, |
487 | uint32_t page_flags) |
488 | { |
489 | struct radeon_ttm_tt *gtt; |
490 | enum ttm_caching caching; |
491 | struct radeon_bo *rbo; |
492 | #if IS_ENABLED(CONFIG_AGP) |
493 | struct radeon_device *rdev = radeon_get_rdev(bdev: bo->bdev); |
494 | |
495 | if (rdev->flags & RADEON_IS_AGP) { |
496 | return ttm_agp_tt_create(bo, bridge: rdev->agp->bridge, page_flags); |
497 | } |
498 | #endif |
499 | rbo = container_of(bo, struct radeon_bo, tbo); |
500 | |
501 | gtt = kzalloc(size: sizeof(struct radeon_ttm_tt), GFP_KERNEL); |
502 | if (gtt == NULL) { |
503 | return NULL; |
504 | } |
505 | |
506 | if (rbo->flags & RADEON_GEM_GTT_UC) |
507 | caching = ttm_uncached; |
508 | else if (rbo->flags & RADEON_GEM_GTT_WC) |
509 | caching = ttm_write_combined; |
510 | else |
511 | caching = ttm_cached; |
512 | |
513 | if (ttm_sg_tt_init(ttm_dma: >t->ttm, bo, page_flags, caching)) { |
514 | kfree(objp: gtt); |
515 | return NULL; |
516 | } |
517 | return >t->ttm; |
518 | } |
519 | |
520 | static struct radeon_ttm_tt *radeon_ttm_tt_to_gtt(struct radeon_device *rdev, |
521 | struct ttm_tt *ttm) |
522 | { |
523 | #if IS_ENABLED(CONFIG_AGP) |
524 | if (rdev->flags & RADEON_IS_AGP) |
525 | return NULL; |
526 | #endif |
527 | |
528 | if (!ttm) |
529 | return NULL; |
530 | return container_of(ttm, struct radeon_ttm_tt, ttm); |
531 | } |
532 | |
533 | static int radeon_ttm_tt_populate(struct ttm_device *bdev, |
534 | struct ttm_tt *ttm, |
535 | struct ttm_operation_ctx *ctx) |
536 | { |
537 | struct radeon_device *rdev = radeon_get_rdev(bdev); |
538 | struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); |
539 | bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); |
540 | |
541 | if (gtt && gtt->userptr) { |
542 | ttm->sg = kzalloc(size: sizeof(struct sg_table), GFP_KERNEL); |
543 | if (!ttm->sg) |
544 | return -ENOMEM; |
545 | |
546 | ttm->page_flags |= TTM_TT_FLAG_EXTERNAL; |
547 | return 0; |
548 | } |
549 | |
550 | if (slave && ttm->sg) { |
551 | drm_prime_sg_to_dma_addr_array(sgt: ttm->sg, addrs: gtt->ttm.dma_address, |
552 | max_pages: ttm->num_pages); |
553 | return 0; |
554 | } |
555 | |
556 | return ttm_pool_alloc(pool: &rdev->mman.bdev.pool, tt: ttm, ctx); |
557 | } |
558 | |
559 | static void radeon_ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm) |
560 | { |
561 | struct radeon_device *rdev = radeon_get_rdev(bdev); |
562 | struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); |
563 | bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); |
564 | |
565 | radeon_ttm_tt_unbind(bdev, ttm); |
566 | |
567 | if (gtt && gtt->userptr) { |
568 | kfree(objp: ttm->sg); |
569 | ttm->page_flags &= ~TTM_TT_FLAG_EXTERNAL; |
570 | return; |
571 | } |
572 | |
573 | if (slave) |
574 | return; |
575 | |
576 | return ttm_pool_free(pool: &rdev->mman.bdev.pool, tt: ttm); |
577 | } |
578 | |
579 | int radeon_ttm_tt_set_userptr(struct radeon_device *rdev, |
580 | struct ttm_tt *ttm, uint64_t addr, |
581 | uint32_t flags) |
582 | { |
583 | struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); |
584 | |
585 | if (gtt == NULL) |
586 | return -EINVAL; |
587 | |
588 | gtt->userptr = addr; |
589 | gtt->usermm = current->mm; |
590 | gtt->userflags = flags; |
591 | return 0; |
592 | } |
593 | |
594 | bool radeon_ttm_tt_is_bound(struct ttm_device *bdev, |
595 | struct ttm_tt *ttm) |
596 | { |
597 | #if IS_ENABLED(CONFIG_AGP) |
598 | struct radeon_device *rdev = radeon_get_rdev(bdev); |
599 | if (rdev->flags & RADEON_IS_AGP) |
600 | return ttm_agp_is_bound(ttm); |
601 | #endif |
602 | return radeon_ttm_backend_is_bound(ttm); |
603 | } |
604 | |
605 | static int radeon_ttm_tt_bind(struct ttm_device *bdev, |
606 | struct ttm_tt *ttm, |
607 | struct ttm_resource *bo_mem) |
608 | { |
609 | #if IS_ENABLED(CONFIG_AGP) |
610 | struct radeon_device *rdev = radeon_get_rdev(bdev); |
611 | #endif |
612 | |
613 | if (!bo_mem) |
614 | return -EINVAL; |
615 | #if IS_ENABLED(CONFIG_AGP) |
616 | if (rdev->flags & RADEON_IS_AGP) |
617 | return ttm_agp_bind(ttm, bo_mem); |
618 | #endif |
619 | |
620 | return radeon_ttm_backend_bind(bdev, ttm, bo_mem); |
621 | } |
622 | |
623 | static void radeon_ttm_tt_unbind(struct ttm_device *bdev, |
624 | struct ttm_tt *ttm) |
625 | { |
626 | #if IS_ENABLED(CONFIG_AGP) |
627 | struct radeon_device *rdev = radeon_get_rdev(bdev); |
628 | |
629 | if (rdev->flags & RADEON_IS_AGP) { |
630 | ttm_agp_unbind(ttm); |
631 | return; |
632 | } |
633 | #endif |
634 | radeon_ttm_backend_unbind(bdev, ttm); |
635 | } |
636 | |
637 | static void radeon_ttm_tt_destroy(struct ttm_device *bdev, |
638 | struct ttm_tt *ttm) |
639 | { |
640 | #if IS_ENABLED(CONFIG_AGP) |
641 | struct radeon_device *rdev = radeon_get_rdev(bdev); |
642 | |
643 | if (rdev->flags & RADEON_IS_AGP) { |
644 | ttm_agp_destroy(ttm); |
645 | return; |
646 | } |
647 | #endif |
648 | radeon_ttm_backend_destroy(bdev, ttm); |
649 | } |
650 | |
651 | bool radeon_ttm_tt_has_userptr(struct radeon_device *rdev, |
652 | struct ttm_tt *ttm) |
653 | { |
654 | struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); |
655 | |
656 | if (gtt == NULL) |
657 | return false; |
658 | |
659 | return !!gtt->userptr; |
660 | } |
661 | |
662 | bool radeon_ttm_tt_is_readonly(struct radeon_device *rdev, |
663 | struct ttm_tt *ttm) |
664 | { |
665 | struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); |
666 | |
667 | if (gtt == NULL) |
668 | return false; |
669 | |
670 | return !!(gtt->userflags & RADEON_GEM_USERPTR_READONLY); |
671 | } |
672 | |
673 | static struct ttm_device_funcs radeon_bo_driver = { |
674 | .ttm_tt_create = &radeon_ttm_tt_create, |
675 | .ttm_tt_populate = &radeon_ttm_tt_populate, |
676 | .ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate, |
677 | .ttm_tt_destroy = &radeon_ttm_tt_destroy, |
678 | .eviction_valuable = ttm_bo_eviction_valuable, |
679 | .evict_flags = &radeon_evict_flags, |
680 | .move = &radeon_bo_move, |
681 | .io_mem_reserve = &radeon_ttm_io_mem_reserve, |
682 | }; |
683 | |
684 | int radeon_ttm_init(struct radeon_device *rdev) |
685 | { |
686 | int r; |
687 | |
688 | /* No others user of address space so set it to 0 */ |
689 | r = ttm_device_init(bdev: &rdev->mman.bdev, funcs: &radeon_bo_driver, dev: rdev->dev, |
690 | mapping: rdev->ddev->anon_inode->i_mapping, |
691 | vma_manager: rdev->ddev->vma_offset_manager, |
692 | use_dma_alloc: rdev->need_swiotlb, |
693 | use_dma32: dma_addressing_limited(dev: &rdev->pdev->dev)); |
694 | if (r) { |
695 | DRM_ERROR("failed initializing buffer object driver(%d).\n" , r); |
696 | return r; |
697 | } |
698 | rdev->mman.initialized = true; |
699 | |
700 | r = radeon_ttm_init_vram(rdev); |
701 | if (r) { |
702 | DRM_ERROR("Failed initializing VRAM heap.\n" ); |
703 | return r; |
704 | } |
705 | /* Change the size here instead of the init above so only lpfn is affected */ |
706 | radeon_ttm_set_active_vram_size(rdev, size: rdev->mc.visible_vram_size); |
707 | |
708 | r = radeon_bo_create(rdev, size: 256 * 1024, PAGE_SIZE, kernel: true, |
709 | RADEON_GEM_DOMAIN_VRAM, flags: 0, NULL, |
710 | NULL, bo_ptr: &rdev->stolen_vga_memory); |
711 | if (r) { |
712 | return r; |
713 | } |
714 | r = radeon_bo_reserve(bo: rdev->stolen_vga_memory, no_intr: false); |
715 | if (r) |
716 | return r; |
717 | r = radeon_bo_pin(bo: rdev->stolen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL); |
718 | radeon_bo_unreserve(bo: rdev->stolen_vga_memory); |
719 | if (r) { |
720 | radeon_bo_unref(bo: &rdev->stolen_vga_memory); |
721 | return r; |
722 | } |
723 | DRM_INFO("radeon: %uM of VRAM memory ready\n" , |
724 | (unsigned) (rdev->mc.real_vram_size / (1024 * 1024))); |
725 | |
726 | r = radeon_ttm_init_gtt(rdev); |
727 | if (r) { |
728 | DRM_ERROR("Failed initializing GTT heap.\n" ); |
729 | return r; |
730 | } |
731 | DRM_INFO("radeon: %uM of GTT memory ready.\n" , |
732 | (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); |
733 | |
734 | radeon_ttm_debugfs_init(rdev); |
735 | |
736 | return 0; |
737 | } |
738 | |
739 | void radeon_ttm_fini(struct radeon_device *rdev) |
740 | { |
741 | int r; |
742 | |
743 | if (!rdev->mman.initialized) |
744 | return; |
745 | |
746 | if (rdev->stolen_vga_memory) { |
747 | r = radeon_bo_reserve(bo: rdev->stolen_vga_memory, no_intr: false); |
748 | if (r == 0) { |
749 | radeon_bo_unpin(bo: rdev->stolen_vga_memory); |
750 | radeon_bo_unreserve(bo: rdev->stolen_vga_memory); |
751 | } |
752 | radeon_bo_unref(bo: &rdev->stolen_vga_memory); |
753 | } |
754 | ttm_range_man_fini(bdev: &rdev->mman.bdev, TTM_PL_VRAM); |
755 | ttm_range_man_fini(bdev: &rdev->mman.bdev, TTM_PL_TT); |
756 | ttm_device_fini(bdev: &rdev->mman.bdev); |
757 | radeon_gart_fini(rdev); |
758 | rdev->mman.initialized = false; |
759 | DRM_INFO("radeon: ttm finalized\n" ); |
760 | } |
761 | |
762 | /* this should only be called at bootup or when userspace |
763 | * isn't running */ |
764 | void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size) |
765 | { |
766 | struct ttm_resource_manager *man; |
767 | |
768 | if (!rdev->mman.initialized) |
769 | return; |
770 | |
771 | man = ttm_manager_type(bdev: &rdev->mman.bdev, TTM_PL_VRAM); |
772 | /* this just adjusts TTM size idea, which sets lpfn to the correct value */ |
773 | man->size = size >> PAGE_SHIFT; |
774 | } |
775 | |
776 | #if defined(CONFIG_DEBUG_FS) |
777 | |
778 | static int radeon_ttm_page_pool_show(struct seq_file *m, void *data) |
779 | { |
780 | struct radeon_device *rdev = m->private; |
781 | |
782 | return ttm_pool_debugfs(pool: &rdev->mman.bdev.pool, m); |
783 | } |
784 | |
785 | DEFINE_SHOW_ATTRIBUTE(radeon_ttm_page_pool); |
786 | |
787 | static int radeon_ttm_vram_open(struct inode *inode, struct file *filep) |
788 | { |
789 | struct radeon_device *rdev = inode->i_private; |
790 | i_size_write(inode, i_size: rdev->mc.mc_vram_size); |
791 | filep->private_data = inode->i_private; |
792 | return 0; |
793 | } |
794 | |
795 | static ssize_t radeon_ttm_vram_read(struct file *f, char __user *buf, |
796 | size_t size, loff_t *pos) |
797 | { |
798 | struct radeon_device *rdev = f->private_data; |
799 | ssize_t result = 0; |
800 | int r; |
801 | |
802 | if (size & 0x3 || *pos & 0x3) |
803 | return -EINVAL; |
804 | |
805 | while (size) { |
806 | unsigned long flags; |
807 | uint32_t value; |
808 | |
809 | if (*pos >= rdev->mc.mc_vram_size) |
810 | return result; |
811 | |
812 | spin_lock_irqsave(&rdev->mmio_idx_lock, flags); |
813 | WREG32(RADEON_MM_INDEX, ((uint32_t)*pos) | 0x80000000); |
814 | if (rdev->family >= CHIP_CEDAR) |
815 | WREG32(EVERGREEN_MM_INDEX_HI, *pos >> 31); |
816 | value = RREG32(RADEON_MM_DATA); |
817 | spin_unlock_irqrestore(lock: &rdev->mmio_idx_lock, flags); |
818 | |
819 | r = put_user(value, (uint32_t __user *)buf); |
820 | if (r) |
821 | return r; |
822 | |
823 | result += 4; |
824 | buf += 4; |
825 | *pos += 4; |
826 | size -= 4; |
827 | } |
828 | |
829 | return result; |
830 | } |
831 | |
832 | static const struct file_operations radeon_ttm_vram_fops = { |
833 | .owner = THIS_MODULE, |
834 | .open = radeon_ttm_vram_open, |
835 | .read = radeon_ttm_vram_read, |
836 | .llseek = default_llseek |
837 | }; |
838 | |
839 | static int radeon_ttm_gtt_open(struct inode *inode, struct file *filep) |
840 | { |
841 | struct radeon_device *rdev = inode->i_private; |
842 | i_size_write(inode, i_size: rdev->mc.gtt_size); |
843 | filep->private_data = inode->i_private; |
844 | return 0; |
845 | } |
846 | |
847 | static ssize_t radeon_ttm_gtt_read(struct file *f, char __user *buf, |
848 | size_t size, loff_t *pos) |
849 | { |
850 | struct radeon_device *rdev = f->private_data; |
851 | ssize_t result = 0; |
852 | int r; |
853 | |
854 | while (size) { |
855 | loff_t p = *pos / PAGE_SIZE; |
856 | unsigned off = *pos & ~PAGE_MASK; |
857 | size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); |
858 | struct page *page; |
859 | void *ptr; |
860 | |
861 | if (p >= rdev->gart.num_cpu_pages) |
862 | return result; |
863 | |
864 | page = rdev->gart.pages[p]; |
865 | if (page) { |
866 | ptr = kmap_local_page(page); |
867 | ptr += off; |
868 | |
869 | r = copy_to_user(to: buf, from: ptr, n: cur_size); |
870 | kunmap_local(ptr); |
871 | } else |
872 | r = clear_user(to: buf, n: cur_size); |
873 | |
874 | if (r) |
875 | return -EFAULT; |
876 | |
877 | result += cur_size; |
878 | buf += cur_size; |
879 | *pos += cur_size; |
880 | size -= cur_size; |
881 | } |
882 | |
883 | return result; |
884 | } |
885 | |
886 | static const struct file_operations radeon_ttm_gtt_fops = { |
887 | .owner = THIS_MODULE, |
888 | .open = radeon_ttm_gtt_open, |
889 | .read = radeon_ttm_gtt_read, |
890 | .llseek = default_llseek |
891 | }; |
892 | |
893 | #endif |
894 | |
895 | static void radeon_ttm_debugfs_init(struct radeon_device *rdev) |
896 | { |
897 | #if defined(CONFIG_DEBUG_FS) |
898 | struct drm_minor *minor = rdev->ddev->primary; |
899 | struct dentry *root = minor->debugfs_root; |
900 | |
901 | debugfs_create_file(name: "radeon_vram" , mode: 0444, parent: root, data: rdev, |
902 | fops: &radeon_ttm_vram_fops); |
903 | debugfs_create_file(name: "radeon_gtt" , mode: 0444, parent: root, data: rdev, |
904 | fops: &radeon_ttm_gtt_fops); |
905 | debugfs_create_file(name: "ttm_page_pool" , mode: 0444, parent: root, data: rdev, |
906 | fops: &radeon_ttm_page_pool_fops); |
907 | ttm_resource_manager_create_debugfs(man: ttm_manager_type(bdev: &rdev->mman.bdev, |
908 | TTM_PL_VRAM), |
909 | parent: root, name: "radeon_vram_mm" ); |
910 | ttm_resource_manager_create_debugfs(man: ttm_manager_type(bdev: &rdev->mman.bdev, |
911 | TTM_PL_TT), |
912 | parent: root, name: "radeon_gtt_mm" ); |
913 | #endif |
914 | } |
915 | |