1 | /* |
2 | * Copyright 2016 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | * Authors: Christian König |
23 | */ |
24 | |
25 | #include <linux/dma-mapping.h> |
26 | #include <drm/ttm/ttm_range_manager.h> |
27 | |
28 | #include "amdgpu.h" |
29 | #include "amdgpu_vm.h" |
30 | #include "amdgpu_res_cursor.h" |
31 | #include "amdgpu_atomfirmware.h" |
32 | #include "atom.h" |
33 | |
34 | struct amdgpu_vram_reservation { |
35 | u64 start; |
36 | u64 size; |
37 | struct list_head allocated; |
38 | struct list_head blocks; |
39 | }; |
40 | |
41 | static inline struct amdgpu_vram_mgr * |
42 | to_vram_mgr(struct ttm_resource_manager *man) |
43 | { |
44 | return container_of(man, struct amdgpu_vram_mgr, manager); |
45 | } |
46 | |
47 | static inline struct amdgpu_device * |
48 | to_amdgpu_device(struct amdgpu_vram_mgr *mgr) |
49 | { |
50 | return container_of(mgr, struct amdgpu_device, mman.vram_mgr); |
51 | } |
52 | |
53 | static inline struct drm_buddy_block * |
54 | amdgpu_vram_mgr_first_block(struct list_head *list) |
55 | { |
56 | return list_first_entry_or_null(list, struct drm_buddy_block, link); |
57 | } |
58 | |
59 | static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head) |
60 | { |
61 | struct drm_buddy_block *block; |
62 | u64 start, size; |
63 | |
64 | block = amdgpu_vram_mgr_first_block(list: head); |
65 | if (!block) |
66 | return false; |
67 | |
68 | while (head != block->link.next) { |
69 | start = amdgpu_vram_mgr_block_start(block); |
70 | size = amdgpu_vram_mgr_block_size(block); |
71 | |
72 | block = list_entry(block->link.next, struct drm_buddy_block, link); |
73 | if (start + size != amdgpu_vram_mgr_block_start(block)) |
74 | return false; |
75 | } |
76 | |
77 | return true; |
78 | } |
79 | |
80 | static inline u64 amdgpu_vram_mgr_blocks_size(struct list_head *head) |
81 | { |
82 | struct drm_buddy_block *block; |
83 | u64 size = 0; |
84 | |
85 | list_for_each_entry(block, head, link) |
86 | size += amdgpu_vram_mgr_block_size(block); |
87 | |
88 | return size; |
89 | } |
90 | |
91 | /** |
92 | * DOC: mem_info_vram_total |
93 | * |
94 | * The amdgpu driver provides a sysfs API for reporting current total VRAM |
95 | * available on the device |
96 | * The file mem_info_vram_total is used for this and returns the total |
97 | * amount of VRAM in bytes |
98 | */ |
99 | static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev, |
100 | struct device_attribute *attr, char *buf) |
101 | { |
102 | struct drm_device *ddev = dev_get_drvdata(dev); |
103 | struct amdgpu_device *adev = drm_to_adev(ddev); |
104 | |
105 | return sysfs_emit(buf, fmt: "%llu\n" , adev->gmc.real_vram_size); |
106 | } |
107 | |
108 | /** |
109 | * DOC: mem_info_vis_vram_total |
110 | * |
111 | * The amdgpu driver provides a sysfs API for reporting current total |
112 | * visible VRAM available on the device |
113 | * The file mem_info_vis_vram_total is used for this and returns the total |
114 | * amount of visible VRAM in bytes |
115 | */ |
116 | static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev, |
117 | struct device_attribute *attr, char *buf) |
118 | { |
119 | struct drm_device *ddev = dev_get_drvdata(dev); |
120 | struct amdgpu_device *adev = drm_to_adev(ddev); |
121 | |
122 | return sysfs_emit(buf, fmt: "%llu\n" , adev->gmc.visible_vram_size); |
123 | } |
124 | |
125 | /** |
126 | * DOC: mem_info_vram_used |
127 | * |
128 | * The amdgpu driver provides a sysfs API for reporting current total VRAM |
129 | * available on the device |
130 | * The file mem_info_vram_used is used for this and returns the total |
131 | * amount of currently used VRAM in bytes |
132 | */ |
133 | static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev, |
134 | struct device_attribute *attr, |
135 | char *buf) |
136 | { |
137 | struct drm_device *ddev = dev_get_drvdata(dev); |
138 | struct amdgpu_device *adev = drm_to_adev(ddev); |
139 | struct ttm_resource_manager *man = &adev->mman.vram_mgr.manager; |
140 | |
141 | return sysfs_emit(buf, fmt: "%llu\n" , ttm_resource_manager_usage(man)); |
142 | } |
143 | |
144 | /** |
145 | * DOC: mem_info_vis_vram_used |
146 | * |
147 | * The amdgpu driver provides a sysfs API for reporting current total of |
148 | * used visible VRAM |
149 | * The file mem_info_vis_vram_used is used for this and returns the total |
150 | * amount of currently used visible VRAM in bytes |
151 | */ |
152 | static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev, |
153 | struct device_attribute *attr, |
154 | char *buf) |
155 | { |
156 | struct drm_device *ddev = dev_get_drvdata(dev); |
157 | struct amdgpu_device *adev = drm_to_adev(ddev); |
158 | |
159 | return sysfs_emit(buf, fmt: "%llu\n" , |
160 | amdgpu_vram_mgr_vis_usage(mgr: &adev->mman.vram_mgr)); |
161 | } |
162 | |
163 | /** |
164 | * DOC: mem_info_vram_vendor |
165 | * |
166 | * The amdgpu driver provides a sysfs API for reporting the vendor of the |
167 | * installed VRAM |
168 | * The file mem_info_vram_vendor is used for this and returns the name of the |
169 | * vendor. |
170 | */ |
171 | static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev, |
172 | struct device_attribute *attr, |
173 | char *buf) |
174 | { |
175 | struct drm_device *ddev = dev_get_drvdata(dev); |
176 | struct amdgpu_device *adev = drm_to_adev(ddev); |
177 | |
178 | switch (adev->gmc.vram_vendor) { |
179 | case SAMSUNG: |
180 | return sysfs_emit(buf, fmt: "samsung\n" ); |
181 | case INFINEON: |
182 | return sysfs_emit(buf, fmt: "infineon\n" ); |
183 | case ELPIDA: |
184 | return sysfs_emit(buf, fmt: "elpida\n" ); |
185 | case ETRON: |
186 | return sysfs_emit(buf, fmt: "etron\n" ); |
187 | case NANYA: |
188 | return sysfs_emit(buf, fmt: "nanya\n" ); |
189 | case HYNIX: |
190 | return sysfs_emit(buf, fmt: "hynix\n" ); |
191 | case MOSEL: |
192 | return sysfs_emit(buf, fmt: "mosel\n" ); |
193 | case WINBOND: |
194 | return sysfs_emit(buf, fmt: "winbond\n" ); |
195 | case ESMT: |
196 | return sysfs_emit(buf, fmt: "esmt\n" ); |
197 | case MICRON: |
198 | return sysfs_emit(buf, fmt: "micron\n" ); |
199 | default: |
200 | return sysfs_emit(buf, fmt: "unknown\n" ); |
201 | } |
202 | } |
203 | |
204 | static DEVICE_ATTR(mem_info_vram_total, S_IRUGO, |
205 | amdgpu_mem_info_vram_total_show, NULL); |
206 | static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO, |
207 | amdgpu_mem_info_vis_vram_total_show,NULL); |
208 | static DEVICE_ATTR(mem_info_vram_used, S_IRUGO, |
209 | amdgpu_mem_info_vram_used_show, NULL); |
210 | static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO, |
211 | amdgpu_mem_info_vis_vram_used_show, NULL); |
212 | static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO, |
213 | amdgpu_mem_info_vram_vendor, NULL); |
214 | |
215 | static struct attribute *amdgpu_vram_mgr_attributes[] = { |
216 | &dev_attr_mem_info_vram_total.attr, |
217 | &dev_attr_mem_info_vis_vram_total.attr, |
218 | &dev_attr_mem_info_vram_used.attr, |
219 | &dev_attr_mem_info_vis_vram_used.attr, |
220 | &dev_attr_mem_info_vram_vendor.attr, |
221 | NULL |
222 | }; |
223 | |
224 | const struct attribute_group amdgpu_vram_mgr_attr_group = { |
225 | .attrs = amdgpu_vram_mgr_attributes |
226 | }; |
227 | |
228 | /** |
229 | * amdgpu_vram_mgr_vis_size - Calculate visible block size |
230 | * |
231 | * @adev: amdgpu_device pointer |
232 | * @block: DRM BUDDY block structure |
233 | * |
234 | * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM |
235 | */ |
236 | static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, |
237 | struct drm_buddy_block *block) |
238 | { |
239 | u64 start = amdgpu_vram_mgr_block_start(block); |
240 | u64 end = start + amdgpu_vram_mgr_block_size(block); |
241 | |
242 | if (start >= adev->gmc.visible_vram_size) |
243 | return 0; |
244 | |
245 | return (end > adev->gmc.visible_vram_size ? |
246 | adev->gmc.visible_vram_size : end) - start; |
247 | } |
248 | |
249 | /** |
250 | * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size |
251 | * |
252 | * @bo: &amdgpu_bo buffer object (must be in VRAM) |
253 | * |
254 | * Returns: |
255 | * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM. |
256 | */ |
257 | u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) |
258 | { |
259 | struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev); |
260 | struct ttm_resource *res = bo->tbo.resource; |
261 | struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); |
262 | struct drm_buddy_block *block; |
263 | u64 usage = 0; |
264 | |
265 | if (amdgpu_gmc_vram_full_visible(gmc: &adev->gmc)) |
266 | return amdgpu_bo_size(bo); |
267 | |
268 | if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT) |
269 | return 0; |
270 | |
271 | list_for_each_entry(block, &vres->blocks, link) |
272 | usage += amdgpu_vram_mgr_vis_size(adev, block); |
273 | |
274 | return usage; |
275 | } |
276 | |
277 | /* Commit the reservation of VRAM pages */ |
278 | static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man) |
279 | { |
280 | struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); |
281 | struct amdgpu_device *adev = to_amdgpu_device(mgr); |
282 | struct drm_buddy *mm = &mgr->mm; |
283 | struct amdgpu_vram_reservation *rsv, *temp; |
284 | struct drm_buddy_block *block; |
285 | uint64_t vis_usage; |
286 | |
287 | list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) { |
288 | if (drm_buddy_alloc_blocks(mm, start: rsv->start, end: rsv->start + rsv->size, |
289 | size: rsv->size, min_page_size: mm->chunk_size, blocks: &rsv->allocated, |
290 | DRM_BUDDY_RANGE_ALLOCATION)) |
291 | continue; |
292 | |
293 | block = amdgpu_vram_mgr_first_block(list: &rsv->allocated); |
294 | if (!block) |
295 | continue; |
296 | |
297 | dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n" , |
298 | rsv->start, rsv->size); |
299 | |
300 | vis_usage = amdgpu_vram_mgr_vis_size(adev, block); |
301 | atomic64_add(i: vis_usage, v: &mgr->vis_usage); |
302 | spin_lock(lock: &man->bdev->lru_lock); |
303 | man->usage += rsv->size; |
304 | spin_unlock(lock: &man->bdev->lru_lock); |
305 | list_move(list: &rsv->blocks, head: &mgr->reserved_pages); |
306 | } |
307 | } |
308 | |
309 | /** |
310 | * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM |
311 | * |
312 | * @mgr: amdgpu_vram_mgr pointer |
313 | * @start: start address of the range in VRAM |
314 | * @size: size of the range |
315 | * |
316 | * Reserve memory from start address with the specified size in VRAM |
317 | */ |
318 | int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr, |
319 | uint64_t start, uint64_t size) |
320 | { |
321 | struct amdgpu_vram_reservation *rsv; |
322 | |
323 | rsv = kzalloc(size: sizeof(*rsv), GFP_KERNEL); |
324 | if (!rsv) |
325 | return -ENOMEM; |
326 | |
327 | INIT_LIST_HEAD(list: &rsv->allocated); |
328 | INIT_LIST_HEAD(list: &rsv->blocks); |
329 | |
330 | rsv->start = start; |
331 | rsv->size = size; |
332 | |
333 | mutex_lock(&mgr->lock); |
334 | list_add_tail(new: &rsv->blocks, head: &mgr->reservations_pending); |
335 | amdgpu_vram_mgr_do_reserve(man: &mgr->manager); |
336 | mutex_unlock(lock: &mgr->lock); |
337 | |
338 | return 0; |
339 | } |
340 | |
341 | /** |
342 | * amdgpu_vram_mgr_query_page_status - query the reservation status |
343 | * |
344 | * @mgr: amdgpu_vram_mgr pointer |
345 | * @start: start address of a page in VRAM |
346 | * |
347 | * Returns: |
348 | * -EBUSY: the page is still hold and in pending list |
349 | * 0: the page has been reserved |
350 | * -ENOENT: the input page is not a reservation |
351 | */ |
352 | int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr, |
353 | uint64_t start) |
354 | { |
355 | struct amdgpu_vram_reservation *rsv; |
356 | int ret; |
357 | |
358 | mutex_lock(&mgr->lock); |
359 | |
360 | list_for_each_entry(rsv, &mgr->reservations_pending, blocks) { |
361 | if (rsv->start <= start && |
362 | (start < (rsv->start + rsv->size))) { |
363 | ret = -EBUSY; |
364 | goto out; |
365 | } |
366 | } |
367 | |
368 | list_for_each_entry(rsv, &mgr->reserved_pages, blocks) { |
369 | if (rsv->start <= start && |
370 | (start < (rsv->start + rsv->size))) { |
371 | ret = 0; |
372 | goto out; |
373 | } |
374 | } |
375 | |
376 | ret = -ENOENT; |
377 | out: |
378 | mutex_unlock(lock: &mgr->lock); |
379 | return ret; |
380 | } |
381 | |
382 | static void amdgpu_dummy_vram_mgr_debug(struct ttm_resource_manager *man, |
383 | struct drm_printer *printer) |
384 | { |
385 | DRM_DEBUG_DRIVER("Dummy vram mgr debug\n" ); |
386 | } |
387 | |
388 | static bool amdgpu_dummy_vram_mgr_compatible(struct ttm_resource_manager *man, |
389 | struct ttm_resource *res, |
390 | const struct ttm_place *place, |
391 | size_t size) |
392 | { |
393 | DRM_DEBUG_DRIVER("Dummy vram mgr compatible\n" ); |
394 | return false; |
395 | } |
396 | |
397 | static bool amdgpu_dummy_vram_mgr_intersects(struct ttm_resource_manager *man, |
398 | struct ttm_resource *res, |
399 | const struct ttm_place *place, |
400 | size_t size) |
401 | { |
402 | DRM_DEBUG_DRIVER("Dummy vram mgr intersects\n" ); |
403 | return true; |
404 | } |
405 | |
406 | static void amdgpu_dummy_vram_mgr_del(struct ttm_resource_manager *man, |
407 | struct ttm_resource *res) |
408 | { |
409 | DRM_DEBUG_DRIVER("Dummy vram mgr deleted\n" ); |
410 | } |
411 | |
412 | static int amdgpu_dummy_vram_mgr_new(struct ttm_resource_manager *man, |
413 | struct ttm_buffer_object *tbo, |
414 | const struct ttm_place *place, |
415 | struct ttm_resource **res) |
416 | { |
417 | DRM_DEBUG_DRIVER("Dummy vram mgr new\n" ); |
418 | return -ENOSPC; |
419 | } |
420 | |
421 | /** |
422 | * amdgpu_vram_mgr_new - allocate new ranges |
423 | * |
424 | * @man: TTM memory type manager |
425 | * @tbo: TTM BO we need this range for |
426 | * @place: placement flags and restrictions |
427 | * @res: the resulting mem object |
428 | * |
429 | * Allocate VRAM for the given BO. |
430 | */ |
431 | static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, |
432 | struct ttm_buffer_object *tbo, |
433 | const struct ttm_place *place, |
434 | struct ttm_resource **res) |
435 | { |
436 | struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); |
437 | struct amdgpu_device *adev = to_amdgpu_device(mgr); |
438 | u64 vis_usage = 0, max_bytes, min_block_size; |
439 | struct amdgpu_vram_mgr_resource *vres; |
440 | u64 size, remaining_size, lpfn, fpfn; |
441 | struct drm_buddy *mm = &mgr->mm; |
442 | struct drm_buddy_block *block; |
443 | unsigned long pages_per_block; |
444 | int r; |
445 | |
446 | lpfn = (u64)place->lpfn << PAGE_SHIFT; |
447 | if (!lpfn) |
448 | lpfn = man->size; |
449 | |
450 | fpfn = (u64)place->fpfn << PAGE_SHIFT; |
451 | |
452 | max_bytes = adev->gmc.mc_vram_size; |
453 | if (tbo->type != ttm_bo_type_kernel) |
454 | max_bytes -= AMDGPU_VM_RESERVED_VRAM; |
455 | |
456 | if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { |
457 | pages_per_block = ~0ul; |
458 | } else { |
459 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
460 | pages_per_block = HPAGE_PMD_NR; |
461 | #else |
462 | /* default to 2MB */ |
463 | pages_per_block = 2UL << (20UL - PAGE_SHIFT); |
464 | #endif |
465 | pages_per_block = max_t(uint32_t, pages_per_block, |
466 | tbo->page_alignment); |
467 | } |
468 | |
469 | vres = kzalloc(size: sizeof(*vres), GFP_KERNEL); |
470 | if (!vres) |
471 | return -ENOMEM; |
472 | |
473 | ttm_resource_init(bo: tbo, place, res: &vres->base); |
474 | |
475 | /* bail out quickly if there's likely not enough VRAM for this BO */ |
476 | if (ttm_resource_manager_usage(man) > max_bytes) { |
477 | r = -ENOSPC; |
478 | goto error_fini; |
479 | } |
480 | |
481 | INIT_LIST_HEAD(list: &vres->blocks); |
482 | |
483 | if (place->flags & TTM_PL_FLAG_TOPDOWN) |
484 | vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; |
485 | |
486 | if (place->flags & TTM_PL_FLAG_CONTIGUOUS) |
487 | vres->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION; |
488 | |
489 | if (fpfn || lpfn != mgr->mm.size) |
490 | /* Allocate blocks in desired range */ |
491 | vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; |
492 | |
493 | remaining_size = (u64)vres->base.size; |
494 | |
495 | mutex_lock(&mgr->lock); |
496 | while (remaining_size) { |
497 | if (tbo->page_alignment) |
498 | min_block_size = (u64)tbo->page_alignment << PAGE_SHIFT; |
499 | else |
500 | min_block_size = mgr->default_page_size; |
501 | |
502 | BUG_ON(min_block_size < mm->chunk_size); |
503 | |
504 | /* Limit maximum size to 2GiB due to SG table limitations */ |
505 | size = min(remaining_size, 2ULL << 30); |
506 | |
507 | if ((size >= (u64)pages_per_block << PAGE_SHIFT) && |
508 | !(size & (((u64)pages_per_block << PAGE_SHIFT) - 1))) |
509 | min_block_size = (u64)pages_per_block << PAGE_SHIFT; |
510 | |
511 | r = drm_buddy_alloc_blocks(mm, start: fpfn, |
512 | end: lpfn, |
513 | size, |
514 | min_page_size: min_block_size, |
515 | blocks: &vres->blocks, |
516 | flags: vres->flags); |
517 | if (unlikely(r)) |
518 | goto error_free_blocks; |
519 | |
520 | if (size > remaining_size) |
521 | remaining_size = 0; |
522 | else |
523 | remaining_size -= size; |
524 | } |
525 | mutex_unlock(lock: &mgr->lock); |
526 | |
527 | vres->base.start = 0; |
528 | size = max_t(u64, amdgpu_vram_mgr_blocks_size(&vres->blocks), |
529 | vres->base.size); |
530 | list_for_each_entry(block, &vres->blocks, link) { |
531 | unsigned long start; |
532 | |
533 | start = amdgpu_vram_mgr_block_start(block) + |
534 | amdgpu_vram_mgr_block_size(block); |
535 | start >>= PAGE_SHIFT; |
536 | |
537 | if (start > PFN_UP(size)) |
538 | start -= PFN_UP(size); |
539 | else |
540 | start = 0; |
541 | vres->base.start = max(vres->base.start, start); |
542 | |
543 | vis_usage += amdgpu_vram_mgr_vis_size(adev, block); |
544 | } |
545 | |
546 | if (amdgpu_is_vram_mgr_blocks_contiguous(head: &vres->blocks)) |
547 | vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; |
548 | |
549 | if (adev->gmc.xgmi.connected_to_cpu) |
550 | vres->base.bus.caching = ttm_cached; |
551 | else |
552 | vres->base.bus.caching = ttm_write_combined; |
553 | |
554 | atomic64_add(i: vis_usage, v: &mgr->vis_usage); |
555 | *res = &vres->base; |
556 | return 0; |
557 | |
558 | error_free_blocks: |
559 | drm_buddy_free_list(mm, objects: &vres->blocks); |
560 | mutex_unlock(lock: &mgr->lock); |
561 | error_fini: |
562 | ttm_resource_fini(man, res: &vres->base); |
563 | kfree(objp: vres); |
564 | |
565 | return r; |
566 | } |
567 | |
568 | /** |
569 | * amdgpu_vram_mgr_del - free ranges |
570 | * |
571 | * @man: TTM memory type manager |
572 | * @res: TTM memory object |
573 | * |
574 | * Free the allocated VRAM again. |
575 | */ |
576 | static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, |
577 | struct ttm_resource *res) |
578 | { |
579 | struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); |
580 | struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); |
581 | struct amdgpu_device *adev = to_amdgpu_device(mgr); |
582 | struct drm_buddy *mm = &mgr->mm; |
583 | struct drm_buddy_block *block; |
584 | uint64_t vis_usage = 0; |
585 | |
586 | mutex_lock(&mgr->lock); |
587 | list_for_each_entry(block, &vres->blocks, link) |
588 | vis_usage += amdgpu_vram_mgr_vis_size(adev, block); |
589 | |
590 | amdgpu_vram_mgr_do_reserve(man); |
591 | |
592 | drm_buddy_free_list(mm, objects: &vres->blocks); |
593 | mutex_unlock(lock: &mgr->lock); |
594 | |
595 | atomic64_sub(i: vis_usage, v: &mgr->vis_usage); |
596 | |
597 | ttm_resource_fini(man, res); |
598 | kfree(objp: vres); |
599 | } |
600 | |
601 | /** |
602 | * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table |
603 | * |
604 | * @adev: amdgpu device pointer |
605 | * @res: TTM memory object |
606 | * @offset: byte offset from the base of VRAM BO |
607 | * @length: number of bytes to export in sg_table |
608 | * @dev: the other device |
609 | * @dir: dma direction |
610 | * @sgt: resulting sg table |
611 | * |
612 | * Allocate and fill a sg table from a VRAM allocation. |
613 | */ |
614 | int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, |
615 | struct ttm_resource *res, |
616 | u64 offset, u64 length, |
617 | struct device *dev, |
618 | enum dma_data_direction dir, |
619 | struct sg_table **sgt) |
620 | { |
621 | struct amdgpu_res_cursor cursor; |
622 | struct scatterlist *sg; |
623 | int num_entries = 0; |
624 | int i, r; |
625 | |
626 | *sgt = kmalloc(size: sizeof(**sgt), GFP_KERNEL); |
627 | if (!*sgt) |
628 | return -ENOMEM; |
629 | |
630 | /* Determine the number of DRM_BUDDY blocks to export */ |
631 | amdgpu_res_first(res, start: offset, size: length, cur: &cursor); |
632 | while (cursor.remaining) { |
633 | num_entries++; |
634 | amdgpu_res_next(cur: &cursor, size: cursor.size); |
635 | } |
636 | |
637 | r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); |
638 | if (r) |
639 | goto error_free; |
640 | |
641 | /* Initialize scatterlist nodes of sg_table */ |
642 | for_each_sgtable_sg((*sgt), sg, i) |
643 | sg->length = 0; |
644 | |
645 | /* |
646 | * Walk down DRM_BUDDY blocks to populate scatterlist nodes |
647 | * @note: Use iterator api to get first the DRM_BUDDY block |
648 | * and the number of bytes from it. Access the following |
649 | * DRM_BUDDY block(s) if more buffer needs to exported |
650 | */ |
651 | amdgpu_res_first(res, start: offset, size: length, cur: &cursor); |
652 | for_each_sgtable_sg((*sgt), sg, i) { |
653 | phys_addr_t phys = cursor.start + adev->gmc.aper_base; |
654 | size_t size = cursor.size; |
655 | dma_addr_t addr; |
656 | |
657 | addr = dma_map_resource(dev, phys_addr: phys, size, dir, |
658 | DMA_ATTR_SKIP_CPU_SYNC); |
659 | r = dma_mapping_error(dev, dma_addr: addr); |
660 | if (r) |
661 | goto error_unmap; |
662 | |
663 | sg_set_page(sg, NULL, len: size, offset: 0); |
664 | sg_dma_address(sg) = addr; |
665 | sg_dma_len(sg) = size; |
666 | |
667 | amdgpu_res_next(cur: &cursor, size: cursor.size); |
668 | } |
669 | |
670 | return 0; |
671 | |
672 | error_unmap: |
673 | for_each_sgtable_sg((*sgt), sg, i) { |
674 | if (!sg->length) |
675 | continue; |
676 | |
677 | dma_unmap_resource(dev, addr: sg->dma_address, |
678 | size: sg->length, dir, |
679 | DMA_ATTR_SKIP_CPU_SYNC); |
680 | } |
681 | sg_free_table(*sgt); |
682 | |
683 | error_free: |
684 | kfree(objp: *sgt); |
685 | return r; |
686 | } |
687 | |
688 | /** |
689 | * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table |
690 | * |
691 | * @dev: device pointer |
692 | * @dir: data direction of resource to unmap |
693 | * @sgt: sg table to free |
694 | * |
695 | * Free a previously allocate sg table. |
696 | */ |
697 | void amdgpu_vram_mgr_free_sgt(struct device *dev, |
698 | enum dma_data_direction dir, |
699 | struct sg_table *sgt) |
700 | { |
701 | struct scatterlist *sg; |
702 | int i; |
703 | |
704 | for_each_sgtable_sg(sgt, sg, i) |
705 | dma_unmap_resource(dev, addr: sg->dma_address, |
706 | size: sg->length, dir, |
707 | DMA_ATTR_SKIP_CPU_SYNC); |
708 | sg_free_table(sgt); |
709 | kfree(objp: sgt); |
710 | } |
711 | |
712 | /** |
713 | * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part |
714 | * |
715 | * @mgr: amdgpu_vram_mgr pointer |
716 | * |
717 | * Returns how many bytes are used in the visible part of VRAM |
718 | */ |
719 | uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr) |
720 | { |
721 | return atomic64_read(v: &mgr->vis_usage); |
722 | } |
723 | |
724 | /** |
725 | * amdgpu_vram_mgr_intersects - test each drm buddy block for intersection |
726 | * |
727 | * @man: TTM memory type manager |
728 | * @res: The resource to test |
729 | * @place: The place to test against |
730 | * @size: Size of the new allocation |
731 | * |
732 | * Test each drm buddy block for intersection for eviction decision. |
733 | */ |
734 | static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man, |
735 | struct ttm_resource *res, |
736 | const struct ttm_place *place, |
737 | size_t size) |
738 | { |
739 | struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); |
740 | struct drm_buddy_block *block; |
741 | |
742 | /* Check each drm buddy block individually */ |
743 | list_for_each_entry(block, &mgr->blocks, link) { |
744 | unsigned long fpfn = |
745 | amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT; |
746 | unsigned long lpfn = fpfn + |
747 | (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT); |
748 | |
749 | if (place->fpfn < lpfn && |
750 | (!place->lpfn || place->lpfn > fpfn)) |
751 | return true; |
752 | } |
753 | |
754 | return false; |
755 | } |
756 | |
757 | /** |
758 | * amdgpu_vram_mgr_compatible - test each drm buddy block for compatibility |
759 | * |
760 | * @man: TTM memory type manager |
761 | * @res: The resource to test |
762 | * @place: The place to test against |
763 | * @size: Size of the new allocation |
764 | * |
765 | * Test each drm buddy block for placement compatibility. |
766 | */ |
767 | static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man, |
768 | struct ttm_resource *res, |
769 | const struct ttm_place *place, |
770 | size_t size) |
771 | { |
772 | struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); |
773 | struct drm_buddy_block *block; |
774 | |
775 | /* Check each drm buddy block individually */ |
776 | list_for_each_entry(block, &mgr->blocks, link) { |
777 | unsigned long fpfn = |
778 | amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT; |
779 | unsigned long lpfn = fpfn + |
780 | (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT); |
781 | |
782 | if (fpfn < place->fpfn || |
783 | (place->lpfn && lpfn > place->lpfn)) |
784 | return false; |
785 | } |
786 | |
787 | return true; |
788 | } |
789 | |
790 | /** |
791 | * amdgpu_vram_mgr_debug - dump VRAM table |
792 | * |
793 | * @man: TTM memory type manager |
794 | * @printer: DRM printer to use |
795 | * |
796 | * Dump the table content using printk. |
797 | */ |
798 | static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man, |
799 | struct drm_printer *printer) |
800 | { |
801 | struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); |
802 | struct drm_buddy *mm = &mgr->mm; |
803 | struct amdgpu_vram_reservation *rsv; |
804 | |
805 | drm_printf(p: printer, f: " vis usage:%llu\n" , |
806 | amdgpu_vram_mgr_vis_usage(mgr)); |
807 | |
808 | mutex_lock(&mgr->lock); |
809 | drm_printf(p: printer, f: "default_page_size: %lluKiB\n" , |
810 | mgr->default_page_size >> 10); |
811 | |
812 | drm_buddy_print(mm, p: printer); |
813 | |
814 | drm_printf(p: printer, f: "reserved:\n" ); |
815 | list_for_each_entry(rsv, &mgr->reserved_pages, blocks) |
816 | drm_printf(p: printer, f: "%#018llx-%#018llx: %llu\n" , |
817 | rsv->start, rsv->start + rsv->size, rsv->size); |
818 | mutex_unlock(lock: &mgr->lock); |
819 | } |
820 | |
821 | static const struct ttm_resource_manager_func amdgpu_dummy_vram_mgr_func = { |
822 | .alloc = amdgpu_dummy_vram_mgr_new, |
823 | .free = amdgpu_dummy_vram_mgr_del, |
824 | .intersects = amdgpu_dummy_vram_mgr_intersects, |
825 | .compatible = amdgpu_dummy_vram_mgr_compatible, |
826 | .debug = amdgpu_dummy_vram_mgr_debug |
827 | }; |
828 | |
829 | static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = { |
830 | .alloc = amdgpu_vram_mgr_new, |
831 | .free = amdgpu_vram_mgr_del, |
832 | .intersects = amdgpu_vram_mgr_intersects, |
833 | .compatible = amdgpu_vram_mgr_compatible, |
834 | .debug = amdgpu_vram_mgr_debug |
835 | }; |
836 | |
837 | /** |
838 | * amdgpu_vram_mgr_init - init VRAM manager and DRM MM |
839 | * |
840 | * @adev: amdgpu_device pointer |
841 | * |
842 | * Allocate and initialize the VRAM manager. |
843 | */ |
844 | int amdgpu_vram_mgr_init(struct amdgpu_device *adev) |
845 | { |
846 | struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; |
847 | struct ttm_resource_manager *man = &mgr->manager; |
848 | int err; |
849 | |
850 | ttm_resource_manager_init(man, bdev: &adev->mman.bdev, |
851 | size: adev->gmc.real_vram_size); |
852 | |
853 | mutex_init(&mgr->lock); |
854 | INIT_LIST_HEAD(list: &mgr->reservations_pending); |
855 | INIT_LIST_HEAD(list: &mgr->reserved_pages); |
856 | mgr->default_page_size = PAGE_SIZE; |
857 | |
858 | if (!adev->gmc.is_app_apu) { |
859 | man->func = &amdgpu_vram_mgr_func; |
860 | |
861 | err = drm_buddy_init(mm: &mgr->mm, size: man->size, PAGE_SIZE); |
862 | if (err) |
863 | return err; |
864 | } else { |
865 | man->func = &amdgpu_dummy_vram_mgr_func; |
866 | DRM_INFO("Setup dummy vram mgr\n" ); |
867 | } |
868 | |
869 | ttm_set_driver_manager(bdev: &adev->mman.bdev, TTM_PL_VRAM, manager: &mgr->manager); |
870 | ttm_resource_manager_set_used(man, used: true); |
871 | return 0; |
872 | } |
873 | |
874 | /** |
875 | * amdgpu_vram_mgr_fini - free and destroy VRAM manager |
876 | * |
877 | * @adev: amdgpu_device pointer |
878 | * |
879 | * Destroy and free the VRAM manager, returns -EBUSY if ranges are still |
880 | * allocated inside it. |
881 | */ |
882 | void amdgpu_vram_mgr_fini(struct amdgpu_device *adev) |
883 | { |
884 | struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; |
885 | struct ttm_resource_manager *man = &mgr->manager; |
886 | int ret; |
887 | struct amdgpu_vram_reservation *rsv, *temp; |
888 | |
889 | ttm_resource_manager_set_used(man, used: false); |
890 | |
891 | ret = ttm_resource_manager_evict_all(bdev: &adev->mman.bdev, man); |
892 | if (ret) |
893 | return; |
894 | |
895 | mutex_lock(&mgr->lock); |
896 | list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) |
897 | kfree(objp: rsv); |
898 | |
899 | list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) { |
900 | drm_buddy_free_list(mm: &mgr->mm, objects: &rsv->allocated); |
901 | kfree(objp: rsv); |
902 | } |
903 | if (!adev->gmc.is_app_apu) |
904 | drm_buddy_fini(mm: &mgr->mm); |
905 | mutex_unlock(lock: &mgr->lock); |
906 | |
907 | ttm_resource_manager_cleanup(man); |
908 | ttm_set_driver_manager(bdev: &adev->mman.bdev, TTM_PL_VRAM, NULL); |
909 | } |
910 | |