1/*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25#include <linux/dma-mapping.h>
26#include <drm/ttm/ttm_range_manager.h>
27
28#include "amdgpu.h"
29#include "amdgpu_vm.h"
30#include "amdgpu_res_cursor.h"
31#include "amdgpu_atomfirmware.h"
32#include "atom.h"
33
34struct amdgpu_vram_reservation {
35 u64 start;
36 u64 size;
37 struct list_head allocated;
38 struct list_head blocks;
39};
40
41static inline struct amdgpu_vram_mgr *
42to_vram_mgr(struct ttm_resource_manager *man)
43{
44 return container_of(man, struct amdgpu_vram_mgr, manager);
45}
46
47static inline struct amdgpu_device *
48to_amdgpu_device(struct amdgpu_vram_mgr *mgr)
49{
50 return container_of(mgr, struct amdgpu_device, mman.vram_mgr);
51}
52
53static inline struct drm_buddy_block *
54amdgpu_vram_mgr_first_block(struct list_head *list)
55{
56 return list_first_entry_or_null(list, struct drm_buddy_block, link);
57}
58
59static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head)
60{
61 struct drm_buddy_block *block;
62 u64 start, size;
63
64 block = amdgpu_vram_mgr_first_block(list: head);
65 if (!block)
66 return false;
67
68 while (head != block->link.next) {
69 start = amdgpu_vram_mgr_block_start(block);
70 size = amdgpu_vram_mgr_block_size(block);
71
72 block = list_entry(block->link.next, struct drm_buddy_block, link);
73 if (start + size != amdgpu_vram_mgr_block_start(block))
74 return false;
75 }
76
77 return true;
78}
79
80static inline u64 amdgpu_vram_mgr_blocks_size(struct list_head *head)
81{
82 struct drm_buddy_block *block;
83 u64 size = 0;
84
85 list_for_each_entry(block, head, link)
86 size += amdgpu_vram_mgr_block_size(block);
87
88 return size;
89}
90
91/**
92 * DOC: mem_info_vram_total
93 *
94 * The amdgpu driver provides a sysfs API for reporting current total VRAM
95 * available on the device
96 * The file mem_info_vram_total is used for this and returns the total
97 * amount of VRAM in bytes
98 */
99static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev,
100 struct device_attribute *attr, char *buf)
101{
102 struct drm_device *ddev = dev_get_drvdata(dev);
103 struct amdgpu_device *adev = drm_to_adev(ddev);
104
105 return sysfs_emit(buf, fmt: "%llu\n", adev->gmc.real_vram_size);
106}
107
108/**
109 * DOC: mem_info_vis_vram_total
110 *
111 * The amdgpu driver provides a sysfs API for reporting current total
112 * visible VRAM available on the device
113 * The file mem_info_vis_vram_total is used for this and returns the total
114 * amount of visible VRAM in bytes
115 */
116static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev,
117 struct device_attribute *attr, char *buf)
118{
119 struct drm_device *ddev = dev_get_drvdata(dev);
120 struct amdgpu_device *adev = drm_to_adev(ddev);
121
122 return sysfs_emit(buf, fmt: "%llu\n", adev->gmc.visible_vram_size);
123}
124
125/**
126 * DOC: mem_info_vram_used
127 *
128 * The amdgpu driver provides a sysfs API for reporting current total VRAM
129 * available on the device
130 * The file mem_info_vram_used is used for this and returns the total
131 * amount of currently used VRAM in bytes
132 */
133static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev,
134 struct device_attribute *attr,
135 char *buf)
136{
137 struct drm_device *ddev = dev_get_drvdata(dev);
138 struct amdgpu_device *adev = drm_to_adev(ddev);
139 struct ttm_resource_manager *man = &adev->mman.vram_mgr.manager;
140
141 return sysfs_emit(buf, fmt: "%llu\n", ttm_resource_manager_usage(man));
142}
143
144/**
145 * DOC: mem_info_vis_vram_used
146 *
147 * The amdgpu driver provides a sysfs API for reporting current total of
148 * used visible VRAM
149 * The file mem_info_vis_vram_used is used for this and returns the total
150 * amount of currently used visible VRAM in bytes
151 */
152static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev,
153 struct device_attribute *attr,
154 char *buf)
155{
156 struct drm_device *ddev = dev_get_drvdata(dev);
157 struct amdgpu_device *adev = drm_to_adev(ddev);
158
159 return sysfs_emit(buf, fmt: "%llu\n",
160 amdgpu_vram_mgr_vis_usage(mgr: &adev->mman.vram_mgr));
161}
162
163/**
164 * DOC: mem_info_vram_vendor
165 *
166 * The amdgpu driver provides a sysfs API for reporting the vendor of the
167 * installed VRAM
168 * The file mem_info_vram_vendor is used for this and returns the name of the
169 * vendor.
170 */
171static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev,
172 struct device_attribute *attr,
173 char *buf)
174{
175 struct drm_device *ddev = dev_get_drvdata(dev);
176 struct amdgpu_device *adev = drm_to_adev(ddev);
177
178 switch (adev->gmc.vram_vendor) {
179 case SAMSUNG:
180 return sysfs_emit(buf, fmt: "samsung\n");
181 case INFINEON:
182 return sysfs_emit(buf, fmt: "infineon\n");
183 case ELPIDA:
184 return sysfs_emit(buf, fmt: "elpida\n");
185 case ETRON:
186 return sysfs_emit(buf, fmt: "etron\n");
187 case NANYA:
188 return sysfs_emit(buf, fmt: "nanya\n");
189 case HYNIX:
190 return sysfs_emit(buf, fmt: "hynix\n");
191 case MOSEL:
192 return sysfs_emit(buf, fmt: "mosel\n");
193 case WINBOND:
194 return sysfs_emit(buf, fmt: "winbond\n");
195 case ESMT:
196 return sysfs_emit(buf, fmt: "esmt\n");
197 case MICRON:
198 return sysfs_emit(buf, fmt: "micron\n");
199 default:
200 return sysfs_emit(buf, fmt: "unknown\n");
201 }
202}
203
204static DEVICE_ATTR(mem_info_vram_total, S_IRUGO,
205 amdgpu_mem_info_vram_total_show, NULL);
206static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO,
207 amdgpu_mem_info_vis_vram_total_show,NULL);
208static DEVICE_ATTR(mem_info_vram_used, S_IRUGO,
209 amdgpu_mem_info_vram_used_show, NULL);
210static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO,
211 amdgpu_mem_info_vis_vram_used_show, NULL);
212static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO,
213 amdgpu_mem_info_vram_vendor, NULL);
214
215static struct attribute *amdgpu_vram_mgr_attributes[] = {
216 &dev_attr_mem_info_vram_total.attr,
217 &dev_attr_mem_info_vis_vram_total.attr,
218 &dev_attr_mem_info_vram_used.attr,
219 &dev_attr_mem_info_vis_vram_used.attr,
220 &dev_attr_mem_info_vram_vendor.attr,
221 NULL
222};
223
224const struct attribute_group amdgpu_vram_mgr_attr_group = {
225 .attrs = amdgpu_vram_mgr_attributes
226};
227
228/**
229 * amdgpu_vram_mgr_vis_size - Calculate visible block size
230 *
231 * @adev: amdgpu_device pointer
232 * @block: DRM BUDDY block structure
233 *
234 * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM
235 */
236static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
237 struct drm_buddy_block *block)
238{
239 u64 start = amdgpu_vram_mgr_block_start(block);
240 u64 end = start + amdgpu_vram_mgr_block_size(block);
241
242 if (start >= adev->gmc.visible_vram_size)
243 return 0;
244
245 return (end > adev->gmc.visible_vram_size ?
246 adev->gmc.visible_vram_size : end) - start;
247}
248
249/**
250 * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size
251 *
252 * @bo: &amdgpu_bo buffer object (must be in VRAM)
253 *
254 * Returns:
255 * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM.
256 */
257u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo)
258{
259 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
260 struct ttm_resource *res = bo->tbo.resource;
261 struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
262 struct drm_buddy_block *block;
263 u64 usage = 0;
264
265 if (amdgpu_gmc_vram_full_visible(gmc: &adev->gmc))
266 return amdgpu_bo_size(bo);
267
268 if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT)
269 return 0;
270
271 list_for_each_entry(block, &vres->blocks, link)
272 usage += amdgpu_vram_mgr_vis_size(adev, block);
273
274 return usage;
275}
276
277/* Commit the reservation of VRAM pages */
278static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)
279{
280 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
281 struct amdgpu_device *adev = to_amdgpu_device(mgr);
282 struct drm_buddy *mm = &mgr->mm;
283 struct amdgpu_vram_reservation *rsv, *temp;
284 struct drm_buddy_block *block;
285 uint64_t vis_usage;
286
287 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) {
288 if (drm_buddy_alloc_blocks(mm, start: rsv->start, end: rsv->start + rsv->size,
289 size: rsv->size, min_page_size: mm->chunk_size, blocks: &rsv->allocated,
290 DRM_BUDDY_RANGE_ALLOCATION))
291 continue;
292
293 block = amdgpu_vram_mgr_first_block(list: &rsv->allocated);
294 if (!block)
295 continue;
296
297 dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n",
298 rsv->start, rsv->size);
299
300 vis_usage = amdgpu_vram_mgr_vis_size(adev, block);
301 atomic64_add(i: vis_usage, v: &mgr->vis_usage);
302 spin_lock(lock: &man->bdev->lru_lock);
303 man->usage += rsv->size;
304 spin_unlock(lock: &man->bdev->lru_lock);
305 list_move(list: &rsv->blocks, head: &mgr->reserved_pages);
306 }
307}
308
309/**
310 * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM
311 *
312 * @mgr: amdgpu_vram_mgr pointer
313 * @start: start address of the range in VRAM
314 * @size: size of the range
315 *
316 * Reserve memory from start address with the specified size in VRAM
317 */
318int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr,
319 uint64_t start, uint64_t size)
320{
321 struct amdgpu_vram_reservation *rsv;
322
323 rsv = kzalloc(size: sizeof(*rsv), GFP_KERNEL);
324 if (!rsv)
325 return -ENOMEM;
326
327 INIT_LIST_HEAD(list: &rsv->allocated);
328 INIT_LIST_HEAD(list: &rsv->blocks);
329
330 rsv->start = start;
331 rsv->size = size;
332
333 mutex_lock(&mgr->lock);
334 list_add_tail(new: &rsv->blocks, head: &mgr->reservations_pending);
335 amdgpu_vram_mgr_do_reserve(man: &mgr->manager);
336 mutex_unlock(lock: &mgr->lock);
337
338 return 0;
339}
340
341/**
342 * amdgpu_vram_mgr_query_page_status - query the reservation status
343 *
344 * @mgr: amdgpu_vram_mgr pointer
345 * @start: start address of a page in VRAM
346 *
347 * Returns:
348 * -EBUSY: the page is still hold and in pending list
349 * 0: the page has been reserved
350 * -ENOENT: the input page is not a reservation
351 */
352int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr,
353 uint64_t start)
354{
355 struct amdgpu_vram_reservation *rsv;
356 int ret;
357
358 mutex_lock(&mgr->lock);
359
360 list_for_each_entry(rsv, &mgr->reservations_pending, blocks) {
361 if (rsv->start <= start &&
362 (start < (rsv->start + rsv->size))) {
363 ret = -EBUSY;
364 goto out;
365 }
366 }
367
368 list_for_each_entry(rsv, &mgr->reserved_pages, blocks) {
369 if (rsv->start <= start &&
370 (start < (rsv->start + rsv->size))) {
371 ret = 0;
372 goto out;
373 }
374 }
375
376 ret = -ENOENT;
377out:
378 mutex_unlock(lock: &mgr->lock);
379 return ret;
380}
381
382static void amdgpu_dummy_vram_mgr_debug(struct ttm_resource_manager *man,
383 struct drm_printer *printer)
384{
385 DRM_DEBUG_DRIVER("Dummy vram mgr debug\n");
386}
387
388static bool amdgpu_dummy_vram_mgr_compatible(struct ttm_resource_manager *man,
389 struct ttm_resource *res,
390 const struct ttm_place *place,
391 size_t size)
392{
393 DRM_DEBUG_DRIVER("Dummy vram mgr compatible\n");
394 return false;
395}
396
397static bool amdgpu_dummy_vram_mgr_intersects(struct ttm_resource_manager *man,
398 struct ttm_resource *res,
399 const struct ttm_place *place,
400 size_t size)
401{
402 DRM_DEBUG_DRIVER("Dummy vram mgr intersects\n");
403 return true;
404}
405
406static void amdgpu_dummy_vram_mgr_del(struct ttm_resource_manager *man,
407 struct ttm_resource *res)
408{
409 DRM_DEBUG_DRIVER("Dummy vram mgr deleted\n");
410}
411
412static int amdgpu_dummy_vram_mgr_new(struct ttm_resource_manager *man,
413 struct ttm_buffer_object *tbo,
414 const struct ttm_place *place,
415 struct ttm_resource **res)
416{
417 DRM_DEBUG_DRIVER("Dummy vram mgr new\n");
418 return -ENOSPC;
419}
420
421/**
422 * amdgpu_vram_mgr_new - allocate new ranges
423 *
424 * @man: TTM memory type manager
425 * @tbo: TTM BO we need this range for
426 * @place: placement flags and restrictions
427 * @res: the resulting mem object
428 *
429 * Allocate VRAM for the given BO.
430 */
431static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
432 struct ttm_buffer_object *tbo,
433 const struct ttm_place *place,
434 struct ttm_resource **res)
435{
436 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
437 struct amdgpu_device *adev = to_amdgpu_device(mgr);
438 u64 vis_usage = 0, max_bytes, min_block_size;
439 struct amdgpu_vram_mgr_resource *vres;
440 u64 size, remaining_size, lpfn, fpfn;
441 struct drm_buddy *mm = &mgr->mm;
442 struct drm_buddy_block *block;
443 unsigned long pages_per_block;
444 int r;
445
446 lpfn = (u64)place->lpfn << PAGE_SHIFT;
447 if (!lpfn)
448 lpfn = man->size;
449
450 fpfn = (u64)place->fpfn << PAGE_SHIFT;
451
452 max_bytes = adev->gmc.mc_vram_size;
453 if (tbo->type != ttm_bo_type_kernel)
454 max_bytes -= AMDGPU_VM_RESERVED_VRAM;
455
456 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
457 pages_per_block = ~0ul;
458 } else {
459#ifdef CONFIG_TRANSPARENT_HUGEPAGE
460 pages_per_block = HPAGE_PMD_NR;
461#else
462 /* default to 2MB */
463 pages_per_block = 2UL << (20UL - PAGE_SHIFT);
464#endif
465 pages_per_block = max_t(uint32_t, pages_per_block,
466 tbo->page_alignment);
467 }
468
469 vres = kzalloc(size: sizeof(*vres), GFP_KERNEL);
470 if (!vres)
471 return -ENOMEM;
472
473 ttm_resource_init(bo: tbo, place, res: &vres->base);
474
475 /* bail out quickly if there's likely not enough VRAM for this BO */
476 if (ttm_resource_manager_usage(man) > max_bytes) {
477 r = -ENOSPC;
478 goto error_fini;
479 }
480
481 INIT_LIST_HEAD(list: &vres->blocks);
482
483 if (place->flags & TTM_PL_FLAG_TOPDOWN)
484 vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
485
486 if (place->flags & TTM_PL_FLAG_CONTIGUOUS)
487 vres->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION;
488
489 if (fpfn || lpfn != mgr->mm.size)
490 /* Allocate blocks in desired range */
491 vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;
492
493 remaining_size = (u64)vres->base.size;
494
495 mutex_lock(&mgr->lock);
496 while (remaining_size) {
497 if (tbo->page_alignment)
498 min_block_size = (u64)tbo->page_alignment << PAGE_SHIFT;
499 else
500 min_block_size = mgr->default_page_size;
501
502 BUG_ON(min_block_size < mm->chunk_size);
503
504 /* Limit maximum size to 2GiB due to SG table limitations */
505 size = min(remaining_size, 2ULL << 30);
506
507 if ((size >= (u64)pages_per_block << PAGE_SHIFT) &&
508 !(size & (((u64)pages_per_block << PAGE_SHIFT) - 1)))
509 min_block_size = (u64)pages_per_block << PAGE_SHIFT;
510
511 r = drm_buddy_alloc_blocks(mm, start: fpfn,
512 end: lpfn,
513 size,
514 min_page_size: min_block_size,
515 blocks: &vres->blocks,
516 flags: vres->flags);
517 if (unlikely(r))
518 goto error_free_blocks;
519
520 if (size > remaining_size)
521 remaining_size = 0;
522 else
523 remaining_size -= size;
524 }
525 mutex_unlock(lock: &mgr->lock);
526
527 vres->base.start = 0;
528 size = max_t(u64, amdgpu_vram_mgr_blocks_size(&vres->blocks),
529 vres->base.size);
530 list_for_each_entry(block, &vres->blocks, link) {
531 unsigned long start;
532
533 start = amdgpu_vram_mgr_block_start(block) +
534 amdgpu_vram_mgr_block_size(block);
535 start >>= PAGE_SHIFT;
536
537 if (start > PFN_UP(size))
538 start -= PFN_UP(size);
539 else
540 start = 0;
541 vres->base.start = max(vres->base.start, start);
542
543 vis_usage += amdgpu_vram_mgr_vis_size(adev, block);
544 }
545
546 if (amdgpu_is_vram_mgr_blocks_contiguous(head: &vres->blocks))
547 vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS;
548
549 if (adev->gmc.xgmi.connected_to_cpu)
550 vres->base.bus.caching = ttm_cached;
551 else
552 vres->base.bus.caching = ttm_write_combined;
553
554 atomic64_add(i: vis_usage, v: &mgr->vis_usage);
555 *res = &vres->base;
556 return 0;
557
558error_free_blocks:
559 drm_buddy_free_list(mm, objects: &vres->blocks);
560 mutex_unlock(lock: &mgr->lock);
561error_fini:
562 ttm_resource_fini(man, res: &vres->base);
563 kfree(objp: vres);
564
565 return r;
566}
567
568/**
569 * amdgpu_vram_mgr_del - free ranges
570 *
571 * @man: TTM memory type manager
572 * @res: TTM memory object
573 *
574 * Free the allocated VRAM again.
575 */
576static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,
577 struct ttm_resource *res)
578{
579 struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
580 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
581 struct amdgpu_device *adev = to_amdgpu_device(mgr);
582 struct drm_buddy *mm = &mgr->mm;
583 struct drm_buddy_block *block;
584 uint64_t vis_usage = 0;
585
586 mutex_lock(&mgr->lock);
587 list_for_each_entry(block, &vres->blocks, link)
588 vis_usage += amdgpu_vram_mgr_vis_size(adev, block);
589
590 amdgpu_vram_mgr_do_reserve(man);
591
592 drm_buddy_free_list(mm, objects: &vres->blocks);
593 mutex_unlock(lock: &mgr->lock);
594
595 atomic64_sub(i: vis_usage, v: &mgr->vis_usage);
596
597 ttm_resource_fini(man, res);
598 kfree(objp: vres);
599}
600
601/**
602 * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table
603 *
604 * @adev: amdgpu device pointer
605 * @res: TTM memory object
606 * @offset: byte offset from the base of VRAM BO
607 * @length: number of bytes to export in sg_table
608 * @dev: the other device
609 * @dir: dma direction
610 * @sgt: resulting sg table
611 *
612 * Allocate and fill a sg table from a VRAM allocation.
613 */
614int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
615 struct ttm_resource *res,
616 u64 offset, u64 length,
617 struct device *dev,
618 enum dma_data_direction dir,
619 struct sg_table **sgt)
620{
621 struct amdgpu_res_cursor cursor;
622 struct scatterlist *sg;
623 int num_entries = 0;
624 int i, r;
625
626 *sgt = kmalloc(size: sizeof(**sgt), GFP_KERNEL);
627 if (!*sgt)
628 return -ENOMEM;
629
630 /* Determine the number of DRM_BUDDY blocks to export */
631 amdgpu_res_first(res, start: offset, size: length, cur: &cursor);
632 while (cursor.remaining) {
633 num_entries++;
634 amdgpu_res_next(cur: &cursor, size: cursor.size);
635 }
636
637 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
638 if (r)
639 goto error_free;
640
641 /* Initialize scatterlist nodes of sg_table */
642 for_each_sgtable_sg((*sgt), sg, i)
643 sg->length = 0;
644
645 /*
646 * Walk down DRM_BUDDY blocks to populate scatterlist nodes
647 * @note: Use iterator api to get first the DRM_BUDDY block
648 * and the number of bytes from it. Access the following
649 * DRM_BUDDY block(s) if more buffer needs to exported
650 */
651 amdgpu_res_first(res, start: offset, size: length, cur: &cursor);
652 for_each_sgtable_sg((*sgt), sg, i) {
653 phys_addr_t phys = cursor.start + adev->gmc.aper_base;
654 size_t size = cursor.size;
655 dma_addr_t addr;
656
657 addr = dma_map_resource(dev, phys_addr: phys, size, dir,
658 DMA_ATTR_SKIP_CPU_SYNC);
659 r = dma_mapping_error(dev, dma_addr: addr);
660 if (r)
661 goto error_unmap;
662
663 sg_set_page(sg, NULL, len: size, offset: 0);
664 sg_dma_address(sg) = addr;
665 sg_dma_len(sg) = size;
666
667 amdgpu_res_next(cur: &cursor, size: cursor.size);
668 }
669
670 return 0;
671
672error_unmap:
673 for_each_sgtable_sg((*sgt), sg, i) {
674 if (!sg->length)
675 continue;
676
677 dma_unmap_resource(dev, addr: sg->dma_address,
678 size: sg->length, dir,
679 DMA_ATTR_SKIP_CPU_SYNC);
680 }
681 sg_free_table(*sgt);
682
683error_free:
684 kfree(objp: *sgt);
685 return r;
686}
687
688/**
689 * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table
690 *
691 * @dev: device pointer
692 * @dir: data direction of resource to unmap
693 * @sgt: sg table to free
694 *
695 * Free a previously allocate sg table.
696 */
697void amdgpu_vram_mgr_free_sgt(struct device *dev,
698 enum dma_data_direction dir,
699 struct sg_table *sgt)
700{
701 struct scatterlist *sg;
702 int i;
703
704 for_each_sgtable_sg(sgt, sg, i)
705 dma_unmap_resource(dev, addr: sg->dma_address,
706 size: sg->length, dir,
707 DMA_ATTR_SKIP_CPU_SYNC);
708 sg_free_table(sgt);
709 kfree(objp: sgt);
710}
711
712/**
713 * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part
714 *
715 * @mgr: amdgpu_vram_mgr pointer
716 *
717 * Returns how many bytes are used in the visible part of VRAM
718 */
719uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr)
720{
721 return atomic64_read(v: &mgr->vis_usage);
722}
723
724/**
725 * amdgpu_vram_mgr_intersects - test each drm buddy block for intersection
726 *
727 * @man: TTM memory type manager
728 * @res: The resource to test
729 * @place: The place to test against
730 * @size: Size of the new allocation
731 *
732 * Test each drm buddy block for intersection for eviction decision.
733 */
734static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man,
735 struct ttm_resource *res,
736 const struct ttm_place *place,
737 size_t size)
738{
739 struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
740 struct drm_buddy_block *block;
741
742 /* Check each drm buddy block individually */
743 list_for_each_entry(block, &mgr->blocks, link) {
744 unsigned long fpfn =
745 amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;
746 unsigned long lpfn = fpfn +
747 (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT);
748
749 if (place->fpfn < lpfn &&
750 (!place->lpfn || place->lpfn > fpfn))
751 return true;
752 }
753
754 return false;
755}
756
757/**
758 * amdgpu_vram_mgr_compatible - test each drm buddy block for compatibility
759 *
760 * @man: TTM memory type manager
761 * @res: The resource to test
762 * @place: The place to test against
763 * @size: Size of the new allocation
764 *
765 * Test each drm buddy block for placement compatibility.
766 */
767static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man,
768 struct ttm_resource *res,
769 const struct ttm_place *place,
770 size_t size)
771{
772 struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
773 struct drm_buddy_block *block;
774
775 /* Check each drm buddy block individually */
776 list_for_each_entry(block, &mgr->blocks, link) {
777 unsigned long fpfn =
778 amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;
779 unsigned long lpfn = fpfn +
780 (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT);
781
782 if (fpfn < place->fpfn ||
783 (place->lpfn && lpfn > place->lpfn))
784 return false;
785 }
786
787 return true;
788}
789
790/**
791 * amdgpu_vram_mgr_debug - dump VRAM table
792 *
793 * @man: TTM memory type manager
794 * @printer: DRM printer to use
795 *
796 * Dump the table content using printk.
797 */
798static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man,
799 struct drm_printer *printer)
800{
801 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
802 struct drm_buddy *mm = &mgr->mm;
803 struct amdgpu_vram_reservation *rsv;
804
805 drm_printf(p: printer, f: " vis usage:%llu\n",
806 amdgpu_vram_mgr_vis_usage(mgr));
807
808 mutex_lock(&mgr->lock);
809 drm_printf(p: printer, f: "default_page_size: %lluKiB\n",
810 mgr->default_page_size >> 10);
811
812 drm_buddy_print(mm, p: printer);
813
814 drm_printf(p: printer, f: "reserved:\n");
815 list_for_each_entry(rsv, &mgr->reserved_pages, blocks)
816 drm_printf(p: printer, f: "%#018llx-%#018llx: %llu\n",
817 rsv->start, rsv->start + rsv->size, rsv->size);
818 mutex_unlock(lock: &mgr->lock);
819}
820
821static const struct ttm_resource_manager_func amdgpu_dummy_vram_mgr_func = {
822 .alloc = amdgpu_dummy_vram_mgr_new,
823 .free = amdgpu_dummy_vram_mgr_del,
824 .intersects = amdgpu_dummy_vram_mgr_intersects,
825 .compatible = amdgpu_dummy_vram_mgr_compatible,
826 .debug = amdgpu_dummy_vram_mgr_debug
827};
828
829static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = {
830 .alloc = amdgpu_vram_mgr_new,
831 .free = amdgpu_vram_mgr_del,
832 .intersects = amdgpu_vram_mgr_intersects,
833 .compatible = amdgpu_vram_mgr_compatible,
834 .debug = amdgpu_vram_mgr_debug
835};
836
837/**
838 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
839 *
840 * @adev: amdgpu_device pointer
841 *
842 * Allocate and initialize the VRAM manager.
843 */
844int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
845{
846 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
847 struct ttm_resource_manager *man = &mgr->manager;
848 int err;
849
850 ttm_resource_manager_init(man, bdev: &adev->mman.bdev,
851 size: adev->gmc.real_vram_size);
852
853 mutex_init(&mgr->lock);
854 INIT_LIST_HEAD(list: &mgr->reservations_pending);
855 INIT_LIST_HEAD(list: &mgr->reserved_pages);
856 mgr->default_page_size = PAGE_SIZE;
857
858 if (!adev->gmc.is_app_apu) {
859 man->func = &amdgpu_vram_mgr_func;
860
861 err = drm_buddy_init(mm: &mgr->mm, size: man->size, PAGE_SIZE);
862 if (err)
863 return err;
864 } else {
865 man->func = &amdgpu_dummy_vram_mgr_func;
866 DRM_INFO("Setup dummy vram mgr\n");
867 }
868
869 ttm_set_driver_manager(bdev: &adev->mman.bdev, TTM_PL_VRAM, manager: &mgr->manager);
870 ttm_resource_manager_set_used(man, used: true);
871 return 0;
872}
873
874/**
875 * amdgpu_vram_mgr_fini - free and destroy VRAM manager
876 *
877 * @adev: amdgpu_device pointer
878 *
879 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
880 * allocated inside it.
881 */
882void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
883{
884 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
885 struct ttm_resource_manager *man = &mgr->manager;
886 int ret;
887 struct amdgpu_vram_reservation *rsv, *temp;
888
889 ttm_resource_manager_set_used(man, used: false);
890
891 ret = ttm_resource_manager_evict_all(bdev: &adev->mman.bdev, man);
892 if (ret)
893 return;
894
895 mutex_lock(&mgr->lock);
896 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks)
897 kfree(objp: rsv);
898
899 list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) {
900 drm_buddy_free_list(mm: &mgr->mm, objects: &rsv->allocated);
901 kfree(objp: rsv);
902 }
903 if (!adev->gmc.is_app_apu)
904 drm_buddy_fini(mm: &mgr->mm);
905 mutex_unlock(lock: &mgr->lock);
906
907 ttm_resource_manager_cleanup(man);
908 ttm_set_driver_manager(bdev: &adev->mman.bdev, TTM_PL_VRAM, NULL);
909}
910

source code of linux/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c