1	// SPDX-License-Identifier: MIT
2	/*
3	* Copyright 2014-2018 Advanced Micro Devices, Inc.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*/
23	#include <linux/dma-buf.h>
24	#include <linux/list.h>
25	#include <linux/pagemap.h>
26	#include <linux/sched/mm.h>
27	#include <linux/sched/task.h>
28	#include <drm/ttm/ttm_tt.h>
29
30	#include <drm/drm_exec.h>
31
32	#include "amdgpu_object.h"
33	#include "amdgpu_gem.h"
34	#include "amdgpu_vm.h"
35	#include "amdgpu_hmm.h"
36	#include "amdgpu_amdkfd.h"
37	#include "amdgpu_dma_buf.h"
38	#include <uapi/linux/kfd_ioctl.h>
39	#include "amdgpu_xgmi.h"
40	#include "kfd_priv.h"
41	#include "kfd_smi_events.h"
42
43	/* Userptr restore delay, just long enough to allow consecutive VM
44	* changes to accumulate
45	*/
46	#define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
47	#define AMDGPU_RESERVE_MEM_LIMIT (3UL << 29)
48
49	/*
50	* Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB
51	* BO chunk
52	*/
53	#define VRAM_AVAILABLITY_ALIGN (1 << 21)
54
55	/* Impose limit on how much memory KFD can use */
56	static struct {
57	uint64_t max_system_mem_limit;
58	uint64_t max_ttm_mem_limit;
59	int64_t system_mem_used;
60	int64_t ttm_mem_used;
61	spinlock_t mem_limit_lock;
62	} kfd_mem_limit;
63
64	static const char * const domain_bit_to_string[] = {
65	"CPU",
66	"GTT",
67	"VRAM",
68	"GDS",
69	"GWS",
70	"OA"
71	};
72
73	#define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
74
75	static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
76
77	static bool kfd_mem_is_attached(struct amdgpu_vm *avm,
78	struct kgd_mem *mem)
79	{
80	struct kfd_mem_attachment *entry;
81
82	list_for_each_entry(entry, &mem->attachments, list)
83	if (entry->bo_va->base.vm == avm)
84	return true;
85
86	return false;
87	}
88
89	/**
90	* reuse_dmamap() - Check whether adev can share the original
91	* userptr BO
92	*
93	* If both adev and bo_adev are in direct mapping or
94	* in the same iommu group, they can share the original BO.
95	*
96	* @adev: Device to which can or cannot share the original BO
97	* @bo_adev: Device to which allocated BO belongs to
98	*
99	* Return: returns true if adev can share original userptr BO,
100	* false otherwise.
101	*/
102	static bool reuse_dmamap(struct amdgpu_device *adev, struct amdgpu_device *bo_adev)
103	{
104	return (adev->ram_is_direct_mapped && bo_adev->ram_is_direct_mapped) ||
105	(adev->dev->iommu_group == bo_adev->dev->iommu_group);
106	}
107
108	/* Set memory usage limits. Current, limits are
109	* System (TTM + userptr) memory - 15/16th System RAM
110	* TTM memory - 3/8th System RAM
111	*/
112	void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
113	{
114	struct sysinfo si;
115	uint64_t mem;
116
117	if (kfd_mem_limit.max_system_mem_limit)
118	return;
119
120	si_meminfo(val: &si);
121	mem = si.totalram - si.totalhigh;
122	mem *= si.mem_unit;
123
124	spin_lock_init(&kfd_mem_limit.mem_limit_lock);
125	kfd_mem_limit.max_system_mem_limit = mem - (mem >> 6);
126	if (kfd_mem_limit.max_system_mem_limit < 2 * AMDGPU_RESERVE_MEM_LIMIT)
127	kfd_mem_limit.max_system_mem_limit >>= 1;
128	else
129	kfd_mem_limit.max_system_mem_limit -= AMDGPU_RESERVE_MEM_LIMIT;
130
131	kfd_mem_limit.max_ttm_mem_limit = ttm_tt_pages_limit() << PAGE_SHIFT;
132	pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n",
133	(kfd_mem_limit.max_system_mem_limit >> 20),
134	(kfd_mem_limit.max_ttm_mem_limit >> 20));
135	}
136
137	void amdgpu_amdkfd_reserve_system_mem(uint64_t size)
138	{
139	kfd_mem_limit.system_mem_used += size;
140	}
141
142	/* Estimate page table size needed to represent a given memory size
143	*
144	* With 4KB pages, we need one 8 byte PTE for each 4KB of memory
145	* (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB
146	* of memory (factor 256K, >> 18). ROCm user mode tries to optimize
147	* for 2MB pages for TLB efficiency. However, small allocations and
148	* fragmented system memory still need some 4KB pages. We choose a
149	* compromise that should work in most cases without reserving too
150	* much memory for page tables unnecessarily (factor 16K, >> 14).
151	*/
152
153	#define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM)
154
155	/**
156	* amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size
157	* of buffer.
158	*
159	* @adev: Device to which allocated BO belongs to
160	* @size: Size of buffer, in bytes, encapsulated by B0. This should be
161	* equivalent to amdgpu_bo_size(BO)
162	* @alloc_flag: Flag used in allocating a BO as noted above
163	* @xcp_id: xcp_id is used to get xcp from xcp manager, one xcp is
164	* managed as one compute node in driver for app
165	*
166	* Return:
167	* returns -ENOMEM in case of error, ZERO otherwise
168	*/
169	int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
170	uint64_t size, u32 alloc_flag, int8_t xcp_id)
171	{
172	uint64_t reserved_for_pt =
173	ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
174	size_t system_mem_needed, ttm_mem_needed, vram_needed;
175	int ret = 0;
176	uint64_t vram_size = 0;
177
178	system_mem_needed = 0;
179	ttm_mem_needed = 0;
180	vram_needed = 0;
181	if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
182	system_mem_needed = size;
183	ttm_mem_needed = size;
184	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
185	/*
186	* Conservatively round up the allocation requirement to 2 MB
187	* to avoid fragmentation caused by 4K allocations in the tail
188	* 2M BO chunk.
189	*/
190	vram_needed = size;
191	/*
192	* For GFX 9.4.3, get the VRAM size from XCP structs
193	*/
194	if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
195	return -EINVAL;
196
197	vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id);
198	if (adev->gmc.is_app_apu) {
199	system_mem_needed = size;
200	ttm_mem_needed = size;
201	}
202	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
203	system_mem_needed = size;
204	} else if (!(alloc_flag &
205	(KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
206	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
207	pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
208	return -ENOMEM;
209	}
210
211	spin_lock(lock: &kfd_mem_limit.mem_limit_lock);
212
213	if (kfd_mem_limit.system_mem_used + system_mem_needed >
214	kfd_mem_limit.max_system_mem_limit)
215	pr_debug("Set no_system_mem_limit=1 if using shared memory\n");
216
217	if ((kfd_mem_limit.system_mem_used + system_mem_needed >
218	kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) ||
219	(kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
220	kfd_mem_limit.max_ttm_mem_limit) ||
221	(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] + vram_needed >
222	vram_size - reserved_for_pt)) {
223	ret = -ENOMEM;
224	goto release;
225	}
226
227	/* Update memory accounting by decreasing available system
228	* memory, TTM memory and GPU memory as computed above
229	*/
230	WARN_ONCE(vram_needed && !adev,
231	"adev reference can't be null when vram is used");
232	if (adev && xcp_id >= 0) {
233	adev->kfd.vram_used[xcp_id] += vram_needed;
234	adev->kfd.vram_used_aligned[xcp_id] += adev->gmc.is_app_apu ?
235	vram_needed :
236	ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN);
237	}
238	kfd_mem_limit.system_mem_used += system_mem_needed;
239	kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
240
241	release:
242	spin_unlock(lock: &kfd_mem_limit.mem_limit_lock);
243	return ret;
244	}
245
246	void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
247	uint64_t size, u32 alloc_flag, int8_t xcp_id)
248	{
249	spin_lock(lock: &kfd_mem_limit.mem_limit_lock);
250
251	if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
252	kfd_mem_limit.system_mem_used -= size;
253	kfd_mem_limit.ttm_mem_used -= size;
254	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
255	WARN_ONCE(!adev,
256	"adev reference can't be null when alloc mem flags vram is set");
257	if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
258	goto release;
259
260	if (adev) {
261	adev->kfd.vram_used[xcp_id] -= size;
262	if (adev->gmc.is_app_apu) {
263	adev->kfd.vram_used_aligned[xcp_id] -= size;
264	kfd_mem_limit.system_mem_used -= size;
265	kfd_mem_limit.ttm_mem_used -= size;
266	} else {
267	adev->kfd.vram_used_aligned[xcp_id] -=
268	ALIGN(size, VRAM_AVAILABLITY_ALIGN);
269	}
270	}
271	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
272	kfd_mem_limit.system_mem_used -= size;
273	} else if (!(alloc_flag &
274	(KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
275	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
276	pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
277	goto release;
278	}
279	WARN_ONCE(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] < 0,
280	"KFD VRAM memory accounting unbalanced for xcp: %d", xcp_id);
281	WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
282	"KFD TTM memory accounting unbalanced");
283	WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
284	"KFD system memory accounting unbalanced");
285
286	release:
287	spin_unlock(lock: &kfd_mem_limit.mem_limit_lock);
288	}
289
290	void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
291	{
292	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
293	u32 alloc_flags = bo->kfd_bo->alloc_flags;
294	u64 size = amdgpu_bo_size(bo);
295
296	amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flag: alloc_flags,
297	xcp_id: bo->xcp_id);
298
299	kfree(objp: bo->kfd_bo);
300	}
301
302	/**
303	* create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information
304	* about USERPTR or DOOREBELL or MMIO BO.
305	*
306	* @adev: Device for which dmamap BO is being created
307	* @mem: BO of peer device that is being DMA mapped. Provides parameters
308	* in building the dmamap BO
309	* @bo_out: Output parameter updated with handle of dmamap BO
310	*/
311	static int
312	create_dmamap_sg_bo(struct amdgpu_device *adev,
313	struct kgd_mem *mem, struct amdgpu_bo **bo_out)
314	{
315	struct drm_gem_object *gem_obj;
316	int ret;
317	uint64_t flags = 0;
318
319	ret = amdgpu_bo_reserve(bo: mem->bo, no_intr: false);
320	if (ret)
321	return ret;
322
323	if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)
324	flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
325	AMDGPU_GEM_CREATE_UNCACHED);
326
327	ret = amdgpu_gem_object_create(adev, size: mem->bo->tbo.base.size, alignment: 1,
328	AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags,
329	type: ttm_bo_type_sg, resv: mem->bo->tbo.base.resv, obj: &gem_obj, xcp_id_plus1: 0);
330
331	amdgpu_bo_unreserve(bo: mem->bo);
332
333	if (ret) {
334	pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
335	return -EINVAL;
336	}
337
338	*bo_out = gem_to_amdgpu_bo(gem_obj);
339	(*bo_out)->parent = amdgpu_bo_ref(bo: mem->bo);
340	return ret;
341	}
342
343	/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
344	* reservation object.
345	*
346	* @bo: [IN] Remove eviction fence(s) from this BO
347	* @ef: [IN] This eviction fence is removed if it
348	* is present in the shared list.
349	*
350	* NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held.
351	*/
352	static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
353	struct amdgpu_amdkfd_fence *ef)
354	{
355	struct dma_fence *replacement;
356
357	if (!ef)
358	return -EINVAL;
359
360	/* TODO: Instead of block before we should use the fence of the page
361	* table update and TLB flush here directly.
362	*/
363	replacement = dma_fence_get_stub();
364	dma_resv_replace_fences(obj: bo->tbo.base.resv, context: ef->base.context,
365	fence: replacement, usage: DMA_RESV_USAGE_BOOKKEEP);
366	dma_fence_put(fence: replacement);
367	return 0;
368	}
369
370	int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo)
371	{
372	struct amdgpu_bo *root = bo;
373	struct amdgpu_vm_bo_base *vm_bo;
374	struct amdgpu_vm *vm;
375	struct amdkfd_process_info *info;
376	struct amdgpu_amdkfd_fence *ef;
377	int ret;
378
379	/* we can always get vm_bo from root PD bo.*/
380	while (root->parent)
381	root = root->parent;
382
383	vm_bo = root->vm_bo;
384	if (!vm_bo)
385	return 0;
386
387	vm = vm_bo->vm;
388	if (!vm)
389	return 0;
390
391	info = vm->process_info;
392	if (!info || !info->eviction_fence)
393	return 0;
394
395	ef = container_of(dma_fence_get(&info->eviction_fence->base),
396	struct amdgpu_amdkfd_fence, base);
397
398	BUG_ON(!dma_resv_trylock(bo->tbo.base.resv));
399	ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef);
400	dma_resv_unlock(obj: bo->tbo.base.resv);
401
402	dma_fence_put(fence: &ef->base);
403	return ret;
404	}
405
406	static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
407	bool wait)
408	{
409	struct ttm_operation_ctx ctx = { false, false };
410	int ret;
411
412	if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),
413	"Called with userptr BO"))
414	return -EINVAL;
415
416	amdgpu_bo_placement_from_domain(abo: bo, domain);
417
418	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
419	if (ret)
420	goto validate_fail;
421	if (wait)
422	amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, intr: false);
423
424	validate_fail:
425	return ret;
426	}
427
428	static int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo,
429	uint32_t domain,
430	struct dma_fence *fence)
431	{
432	int ret = amdgpu_bo_reserve(bo, no_intr: false);
433
434	if (ret)
435	return ret;
436
437	ret = amdgpu_amdkfd_bo_validate(bo, domain, wait: true);
438	if (ret)
439	goto unreserve_out;
440
441	ret = dma_resv_reserve_fences(obj: bo->tbo.base.resv, num_fences: 1);
442	if (ret)
443	goto unreserve_out;
444
445	dma_resv_add_fence(obj: bo->tbo.base.resv, fence,
446	usage: DMA_RESV_USAGE_BOOKKEEP);
447
448	unreserve_out:
449	amdgpu_bo_unreserve(bo);
450
451	return ret;
452	}
453
454	static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo)
455	{
456	return amdgpu_amdkfd_bo_validate(bo, domain: bo->allowed_domains, wait: false);
457	}
458
459	/* vm_validate_pt_pd_bos - Validate page table and directory BOs
460	*
461	* Page directories are not updated here because huge page handling
462	* during page table updates can invalidate page directory entries
463	* again. Page directories are only updated after updating page
464	* tables.
465	*/
466	static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm)
467	{
468	struct amdgpu_bo *pd = vm->root.bo;
469	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: pd->tbo.bdev);
470	int ret;
471
472	ret = amdgpu_vm_validate_pt_bos(adev, vm, callback: amdgpu_amdkfd_validate_vm_bo, NULL);
473	if (ret) {
474	pr_err("failed to validate PT BOs\n");
475	return ret;
476	}
477
478	vm->pd_phys_addr = amdgpu_gmc_pd_addr(bo: vm->root.bo);
479
480	return 0;
481	}
482
483	static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
484	{
485	struct amdgpu_bo *pd = vm->root.bo;
486	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: pd->tbo.bdev);
487	int ret;
488
489	ret = amdgpu_vm_update_pdes(adev, vm, immediate: false);
490	if (ret)
491	return ret;
492
493	return amdgpu_sync_fence(sync, f: vm->last_update);
494	}
495
496	static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
497	{
498	uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE |
499	AMDGPU_VM_MTYPE_DEFAULT;
500
501	if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE)
502	mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE;
503	if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE)
504	mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
505
506	return amdgpu_gem_va_map_flags(adev, flags: mapping_flags);
507	}
508
509	/**
510	* create_sg_table() - Create an sg_table for a contiguous DMA addr range
511	* @addr: The starting address to point to
512	* @size: Size of memory area in bytes being pointed to
513	*
514	* Allocates an instance of sg_table and initializes it to point to memory
515	* area specified by input parameters. The address used to build is assumed
516	* to be DMA mapped, if needed.
517	*
518	* DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
519	* because they are physically contiguous.
520	*
521	* Return: Initialized instance of SG Table or NULL
522	*/
523	static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
524	{
525	struct sg_table *sg = kmalloc(size: sizeof(*sg), GFP_KERNEL);
526
527	if (!sg)
528	return NULL;
529	if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
530	kfree(objp: sg);
531	return NULL;
532	}
533	sg_dma_address(sg->sgl) = addr;
534	sg->sgl->length = size;
535	#ifdef CONFIG_NEED_SG_DMA_LENGTH
536	sg->sgl->dma_length = size;
537	#endif
538	return sg;
539	}
540
541	static int
542	kfd_mem_dmamap_userptr(struct kgd_mem *mem,
543	struct kfd_mem_attachment *attachment)
544	{
545	enum dma_data_direction direction =
546	mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
547	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
548	struct ttm_operation_ctx ctx = {.interruptible = true};
549	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
550	struct amdgpu_device *adev = attachment->adev;
551	struct ttm_tt *src_ttm = mem->bo->tbo.ttm;
552	struct ttm_tt *ttm = bo->tbo.ttm;
553	int ret;
554
555	if (WARN_ON(ttm->num_pages != src_ttm->num_pages))
556	return -EINVAL;
557
558	ttm->sg = kmalloc(size: sizeof(*ttm->sg), GFP_KERNEL);
559	if (unlikely(!ttm->sg))
560	return -ENOMEM;
561
562	/* Same sequence as in amdgpu_ttm_tt_pin_userptr */
563	ret = sg_alloc_table_from_pages(sgt: ttm->sg, pages: src_ttm->pages,
564	n_pages: ttm->num_pages, offset: 0,
565	size: (u64)ttm->num_pages << PAGE_SHIFT,
566	GFP_KERNEL);
567	if (unlikely(ret))
568	goto free_sg;
569
570	ret = dma_map_sgtable(dev: adev->dev, sgt: ttm->sg, dir: direction, attrs: 0);
571	if (unlikely(ret))
572	goto release_sg;
573
574	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_GTT);
575	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
576	if (ret)
577	goto unmap_sg;
578
579	return 0;
580
581	unmap_sg:
582	dma_unmap_sgtable(dev: adev->dev, sgt: ttm->sg, dir: direction, attrs: 0);
583	release_sg:
584	pr_err("DMA map userptr failed: %d\n", ret);
585	sg_free_table(ttm->sg);
586	free_sg:
587	kfree(objp: ttm->sg);
588	ttm->sg = NULL;
589	return ret;
590	}
591
592	static int
593	kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
594	{
595	struct ttm_operation_ctx ctx = {.interruptible = true};
596	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
597	int ret;
598
599	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_CPU);
600	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
601	if (ret)
602	return ret;
603
604	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_GTT);
605	return ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
606	}
607
608	/**
609	* kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
610	* @mem: SG BO of the DOORBELL or MMIO resource on the owning device
611	* @attachment: Virtual address attachment of the BO on accessing device
612	*
613	* An access request from the device that owns DOORBELL does not require DMA mapping.
614	* This is because the request doesn't go through PCIe root complex i.e. it instead
615	* loops back. The need to DMA map arises only when accessing peer device's DOORBELL
616	*
617	* In contrast, all access requests for MMIO need to be DMA mapped without regard to
618	* device ownership. This is because access requests for MMIO go through PCIe root
619	* complex.
620	*
621	* This is accomplished in two steps:
622	* - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
623	* in updating requesting device's page table
624	* - Signal TTM to mark memory pointed to by requesting device's BO as GPU
625	* accessible. This allows an update of requesting device's page table
626	* with entries associated with DOOREBELL or MMIO memory
627	*
628	* This method is invoked in the following contexts:
629	* - Mapping of DOORBELL or MMIO BO of same or peer device
630	* - Validating an evicted DOOREBELL or MMIO BO on device seeking access
631	*
632	* Return: ZERO if successful, NON-ZERO otherwise
633	*/
634	static int
635	kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
636	struct kfd_mem_attachment *attachment)
637	{
638	struct ttm_operation_ctx ctx = {.interruptible = true};
639	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
640	struct amdgpu_device *adev = attachment->adev;
641	struct ttm_tt *ttm = bo->tbo.ttm;
642	enum dma_data_direction dir;
643	dma_addr_t dma_addr;
644	bool mmio;
645	int ret;
646
647	/* Expect SG Table of dmapmap BO to be NULL */
648	mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
649	if (unlikely(ttm->sg)) {
650	pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
651	return -EINVAL;
652	}
653
654	dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
655	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
656	dma_addr = mem->bo->tbo.sg->sgl->dma_address;
657	pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
658	pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
659	dma_addr = dma_map_resource(dev: adev->dev, phys_addr: dma_addr,
660	size: mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
661	ret = dma_mapping_error(dev: adev->dev, dma_addr);
662	if (unlikely(ret))
663	return ret;
664	pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
665
666	ttm->sg = create_sg_table(addr: dma_addr, size: mem->bo->tbo.sg->sgl->length);
667	if (unlikely(!ttm->sg)) {
668	ret = -ENOMEM;
669	goto unmap_sg;
670	}
671
672	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_GTT);
673	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
674	if (unlikely(ret))
675	goto free_sg;
676
677	return ret;
678
679	free_sg:
680	sg_free_table(ttm->sg);
681	kfree(objp: ttm->sg);
682	ttm->sg = NULL;
683	unmap_sg:
684	dma_unmap_resource(dev: adev->dev, addr: dma_addr, size: mem->bo->tbo.sg->sgl->length,
685	dir, DMA_ATTR_SKIP_CPU_SYNC);
686	return ret;
687	}
688
689	static int
690	kfd_mem_dmamap_attachment(struct kgd_mem *mem,
691	struct kfd_mem_attachment *attachment)
692	{
693	switch (attachment->type) {
694	case KFD_MEM_ATT_SHARED:
695	return 0;
696	case KFD_MEM_ATT_USERPTR:
697	return kfd_mem_dmamap_userptr(mem, attachment);
698	case KFD_MEM_ATT_DMABUF:
699	return kfd_mem_dmamap_dmabuf(attachment);
700	case KFD_MEM_ATT_SG:
701	return kfd_mem_dmamap_sg_bo(mem, attachment);
702	default:
703	WARN_ON_ONCE(1);
704	}
705	return -EINVAL;
706	}
707
708	static void
709	kfd_mem_dmaunmap_userptr(struct kgd_mem *mem,
710	struct kfd_mem_attachment *attachment)
711	{
712	enum dma_data_direction direction =
713	mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
714	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
715	struct ttm_operation_ctx ctx = {.interruptible = false};
716	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
717	struct amdgpu_device *adev = attachment->adev;
718	struct ttm_tt *ttm = bo->tbo.ttm;
719
720	if (unlikely(!ttm->sg))
721	return;
722
723	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_CPU);
724	ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
725
726	dma_unmap_sgtable(dev: adev->dev, sgt: ttm->sg, dir: direction, attrs: 0);
727	sg_free_table(ttm->sg);
728	kfree(objp: ttm->sg);
729	ttm->sg = NULL;
730	}
731
732	static void
733	kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
734	{
735	/* This is a no-op. We don't want to trigger eviction fences when
736	* unmapping DMABufs. Therefore the invalidation (moving to system
737	* domain) is done in kfd_mem_dmamap_dmabuf.
738	*/
739	}
740
741	/**
742	* kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
743	* @mem: SG BO of the DOORBELL or MMIO resource on the owning device
744	* @attachment: Virtual address attachment of the BO on accessing device
745	*
746	* The method performs following steps:
747	* - Signal TTM to mark memory pointed to by BO as GPU inaccessible
748	* - Free SG Table that is used to encapsulate DMA mapped memory of
749	* peer device's DOORBELL or MMIO memory
750	*
751	* This method is invoked in the following contexts:
752	* UNMapping of DOORBELL or MMIO BO on a device having access to its memory
753	* Eviction of DOOREBELL or MMIO BO on device having access to its memory
754	*
755	* Return: void
756	*/
757	static void
758	kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
759	struct kfd_mem_attachment *attachment)
760	{
761	struct ttm_operation_ctx ctx = {.interruptible = true};
762	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
763	struct amdgpu_device *adev = attachment->adev;
764	struct ttm_tt *ttm = bo->tbo.ttm;
765	enum dma_data_direction dir;
766
767	if (unlikely(!ttm->sg)) {
768	pr_debug("SG Table of BO is NULL");
769	return;
770	}
771
772	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_CPU);
773	ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
774
775	dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
776	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
777	dma_unmap_resource(dev: adev->dev, addr: ttm->sg->sgl->dma_address,
778	size: ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
779	sg_free_table(ttm->sg);
780	kfree(objp: ttm->sg);
781	ttm->sg = NULL;
782	bo->tbo.sg = NULL;
783	}
784
785	static void
786	kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
787	struct kfd_mem_attachment *attachment)
788	{
789	switch (attachment->type) {
790	case KFD_MEM_ATT_SHARED:
791	break;
792	case KFD_MEM_ATT_USERPTR:
793	kfd_mem_dmaunmap_userptr(mem, attachment);
794	break;
795	case KFD_MEM_ATT_DMABUF:
796	kfd_mem_dmaunmap_dmabuf(attachment);
797	break;
798	case KFD_MEM_ATT_SG:
799	kfd_mem_dmaunmap_sg_bo(mem, attachment);
800	break;
801	default:
802	WARN_ON_ONCE(1);
803	}
804	}
805
806	static int kfd_mem_export_dmabuf(struct kgd_mem *mem)
807	{
808	if (!mem->dmabuf) {
809	struct dma_buf *ret = amdgpu_gem_prime_export(
810	gobj: &mem->bo->tbo.base,
811	flags: mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
812	DRM_RDWR : 0);
813	if (IS_ERR(ptr: ret))
814	return PTR_ERR(ptr: ret);
815	mem->dmabuf = ret;
816	}
817
818	return 0;
819	}
820
821	static int
822	kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
823	struct amdgpu_bo **bo)
824	{
825	struct drm_gem_object *gobj;
826	int ret;
827
828	ret = kfd_mem_export_dmabuf(mem);
829	if (ret)
830	return ret;
831
832	gobj = amdgpu_gem_prime_import(dev: adev_to_drm(adev), dma_buf: mem->dmabuf);
833	if (IS_ERR(ptr: gobj))
834	return PTR_ERR(ptr: gobj);
835
836	*bo = gem_to_amdgpu_bo(gobj);
837	(*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE;
838
839	return 0;
840	}
841
842	/* kfd_mem_attach - Add a BO to a VM
843	*
844	* Everything that needs to bo done only once when a BO is first added
845	* to a VM. It can later be mapped and unmapped many times without
846	* repeating these steps.
847	*
848	* 0. Create BO for DMA mapping, if needed
849	* 1. Allocate and initialize BO VA entry data structure
850	* 2. Add BO to the VM
851	* 3. Determine ASIC-specific PTE flags
852	* 4. Alloc page tables and directories if needed
853	* 4a. Validate new page tables and directories
854	*/
855	static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
856	struct amdgpu_vm *vm, bool is_aql)
857	{
858	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bdev: mem->bo->tbo.bdev);
859	unsigned long bo_size = mem->bo->tbo.base.size;
860	uint64_t va = mem->va;
861	struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
862	struct amdgpu_bo *bo[2] = {NULL, NULL};
863	struct amdgpu_bo_va *bo_va;
864	bool same_hive = false;
865	int i, ret;
866
867	if (!va) {
868	pr_err("Invalid VA when adding BO to VM\n");
869	return -EINVAL;
870	}
871
872	/* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
873	*
874	* The access path of MMIO and DOORBELL BOs of is always over PCIe.
875	* In contrast the access path of VRAM BOs depens upon the type of
876	* link that connects the peer device. Access over PCIe is allowed
877	* if peer device has large BAR. In contrast, access over xGMI is
878	* allowed for both small and large BAR configurations of peer device
879	*/
880	if ((adev != bo_adev && !adev->gmc.is_app_apu) &&
881	((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
882	(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
883	(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
884	if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
885	same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
886	if (!same_hive && !amdgpu_device_is_peer_accessible(adev: bo_adev, peer_adev: adev))
887	return -EINVAL;
888	}
889
890	for (i = 0; i <= is_aql; i++) {
891	attachment[i] = kzalloc(size: sizeof(*attachment[i]), GFP_KERNEL);
892	if (unlikely(!attachment[i])) {
893	ret = -ENOMEM;
894	goto unwind;
895	}
896
897	pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
898	va + bo_size, vm);
899
900	if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
901	(amdgpu_ttm_tt_get_usermm(ttm: mem->bo->tbo.ttm) && reuse_dmamap(adev, bo_adev)) ||
902	(mem->domain == AMDGPU_GEM_DOMAIN_GTT && reuse_dmamap(adev, bo_adev)) ||
903	same_hive) {
904	/* Mappings on the local GPU, or VRAM mappings in the
905	* local hive, or userptr, or GTT mapping can reuse dma map
906	* address space share the original BO
907	*/
908	attachment[i]->type = KFD_MEM_ATT_SHARED;
909	bo[i] = mem->bo;
910	drm_gem_object_get(obj: &bo[i]->tbo.base);
911	} else if (i > 0) {
912	/* Multiple mappings on the same GPU share the BO */
913	attachment[i]->type = KFD_MEM_ATT_SHARED;
914	bo[i] = bo[0];
915	drm_gem_object_get(obj: &bo[i]->tbo.base);
916	} else if (amdgpu_ttm_tt_get_usermm(ttm: mem->bo->tbo.ttm)) {
917	/* Create an SG BO to DMA-map userptrs on other GPUs */
918	attachment[i]->type = KFD_MEM_ATT_USERPTR;
919	ret = create_dmamap_sg_bo(adev, mem, bo_out: &bo[i]);
920	if (ret)
921	goto unwind;
922	/* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
923	} else if (mem->bo->tbo.type == ttm_bo_type_sg) {
924	WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||
925	mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),
926	"Handing invalid SG BO in ATTACH request");
927	attachment[i]->type = KFD_MEM_ATT_SG;
928	ret = create_dmamap_sg_bo(adev, mem, bo_out: &bo[i]);
929	if (ret)
930	goto unwind;
931	/* Enable acces to GTT and VRAM BOs of peer devices */
932	} else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||
933	mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
934	attachment[i]->type = KFD_MEM_ATT_DMABUF;
935	ret = kfd_mem_attach_dmabuf(adev, mem, bo: &bo[i]);
936	if (ret)
937	goto unwind;
938	pr_debug("Employ DMABUF mechanism to enable peer GPU access\n");
939	} else {
940	WARN_ONCE(true, "Handling invalid ATTACH request");
941	ret = -EINVAL;
942	goto unwind;
943	}
944
945	/* Add BO to VM internal data structures */
946	ret = amdgpu_bo_reserve(bo: bo[i], no_intr: false);
947	if (ret) {
948	pr_debug("Unable to reserve BO during memory attach");
949	goto unwind;
950	}
951	bo_va = amdgpu_vm_bo_find(vm, bo: bo[i]);
952	if (!bo_va)
953	bo_va = amdgpu_vm_bo_add(adev, vm, bo: bo[i]);
954	else
955	++bo_va->ref_count;
956	attachment[i]->bo_va = bo_va;
957	amdgpu_bo_unreserve(bo: bo[i]);
958	if (unlikely(!attachment[i]->bo_va)) {
959	ret = -ENOMEM;
960	pr_err("Failed to add BO object to VM. ret == %d\n",
961	ret);
962	goto unwind;
963	}
964	attachment[i]->va = va;
965	attachment[i]->pte_flags = get_pte_flags(adev, mem);
966	attachment[i]->adev = adev;
967	list_add(new: &attachment[i]->list, head: &mem->attachments);
968
969	va += bo_size;
970	}
971
972	return 0;
973
974	unwind:
975	for (; i >= 0; i--) {
976	if (!attachment[i])
977	continue;
978	if (attachment[i]->bo_va) {
979	amdgpu_bo_reserve(bo: bo[i], no_intr: true);
980	if (--attachment[i]->bo_va->ref_count == 0)
981	amdgpu_vm_bo_del(adev, bo_va: attachment[i]->bo_va);
982	amdgpu_bo_unreserve(bo: bo[i]);
983	list_del(entry: &attachment[i]->list);
984	}
985	if (bo[i])
986	drm_gem_object_put(obj: &bo[i]->tbo.base);
987	kfree(objp: attachment[i]);
988	}
989	return ret;
990	}
991
992	static void kfd_mem_detach(struct kfd_mem_attachment *attachment)
993	{
994	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
995
996	pr_debug("\t remove VA 0x%llx in entry %p\n",
997	attachment->va, attachment);
998	if (--attachment->bo_va->ref_count == 0)
999	amdgpu_vm_bo_del(adev: attachment->adev, bo_va: attachment->bo_va);
1000	drm_gem_object_put(obj: &bo->tbo.base);
1001	list_del(entry: &attachment->list);
1002	kfree(objp: attachment);
1003	}
1004
1005	static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
1006	struct amdkfd_process_info *process_info,
1007	bool userptr)
1008	{
1009	mutex_lock(&process_info->lock);
1010	if (userptr)
1011	list_add_tail(new: &mem->validate_list,
1012	head: &process_info->userptr_valid_list);
1013	else
1014	list_add_tail(new: &mem->validate_list, head: &process_info->kfd_bo_list);
1015	mutex_unlock(lock: &process_info->lock);
1016	}
1017
1018	static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,
1019	struct amdkfd_process_info *process_info)
1020	{
1021	mutex_lock(&process_info->lock);
1022	list_del(entry: &mem->validate_list);
1023	mutex_unlock(lock: &process_info->lock);
1024	}
1025
1026	/* Initializes user pages. It registers the MMU notifier and validates
1027	* the userptr BO in the GTT domain.
1028	*
1029	* The BO must already be on the userptr_valid_list. Otherwise an
1030	* eviction and restore may happen that leaves the new BO unmapped
1031	* with the user mode queues running.
1032	*
1033	* Takes the process_info->lock to protect against concurrent restore
1034	* workers.
1035	*
1036	* Returns 0 for success, negative errno for errors.
1037	*/
1038	static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr,
1039	bool criu_resume)
1040	{
1041	struct amdkfd_process_info *process_info = mem->process_info;
1042	struct amdgpu_bo *bo = mem->bo;
1043	struct ttm_operation_ctx ctx = { true, false };
1044	struct hmm_range *range;
1045	int ret = 0;
1046
1047	mutex_lock(&process_info->lock);
1048
1049	ret = amdgpu_ttm_tt_set_userptr(bo: &bo->tbo, addr: user_addr, flags: 0);
1050	if (ret) {
1051	pr_err("%s: Failed to set userptr: %d\n", __func__, ret);
1052	goto out;
1053	}
1054
1055	ret = amdgpu_hmm_register(bo, addr: user_addr);
1056	if (ret) {
1057	pr_err("%s: Failed to register MMU notifier: %d\n",
1058	__func__, ret);
1059	goto out;
1060	}
1061
1062	if (criu_resume) {
1063	/*
1064	* During a CRIU restore operation, the userptr buffer objects
1065	* will be validated in the restore_userptr_work worker at a
1066	* later stage when it is scheduled by another ioctl called by
1067	* CRIU master process for the target pid for restore.
1068	*/
1069	mutex_lock(&process_info->notifier_lock);
1070	mem->invalid++;
1071	mutex_unlock(lock: &process_info->notifier_lock);
1072	mutex_unlock(lock: &process_info->lock);
1073	return 0;
1074	}
1075
1076	ret = amdgpu_ttm_tt_get_user_pages(bo, pages: bo->tbo.ttm->pages, range: &range);
1077	if (ret) {
1078	pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
1079	goto unregister_out;
1080	}
1081
1082	ret = amdgpu_bo_reserve(bo, no_intr: true);
1083	if (ret) {
1084	pr_err("%s: Failed to reserve BO\n", __func__);
1085	goto release_out;
1086	}
1087	amdgpu_bo_placement_from_domain(abo: bo, domain: mem->domain);
1088	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
1089	if (ret)
1090	pr_err("%s: failed to validate BO\n", __func__);
1091	amdgpu_bo_unreserve(bo);
1092
1093	release_out:
1094	amdgpu_ttm_tt_get_user_pages_done(ttm: bo->tbo.ttm, range);
1095	unregister_out:
1096	if (ret)
1097	amdgpu_hmm_unregister(bo);
1098	out:
1099	mutex_unlock(lock: &process_info->lock);
1100	return ret;
1101	}
1102
1103	/* Reserving a BO and its page table BOs must happen atomically to
1104	* avoid deadlocks. Some operations update multiple VMs at once. Track
1105	* all the reservation info in a context structure. Optionally a sync
1106	* object can track VM updates.
1107	*/
1108	struct bo_vm_reservation_context {
1109	/* DRM execution context for the reservation */
1110	struct drm_exec exec;
1111	/* Number of VMs reserved */
1112	unsigned int n_vms;
1113	/* Pointer to sync object */
1114	struct amdgpu_sync *sync;
1115	};
1116
1117	enum bo_vm_match {
1118	BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */
1119	BO_VM_MAPPED, /* Match VMs where a BO is mapped */
1120	BO_VM_ALL, /* Match all VMs a BO was added to */
1121	};
1122
1123	/**
1124	* reserve_bo_and_vm - reserve a BO and a VM unconditionally.
1125	* @mem: KFD BO structure.
1126	* @vm: the VM to reserve.
1127	* @ctx: the struct that will be used in unreserve_bo_and_vms().
1128	*/
1129	static int reserve_bo_and_vm(struct kgd_mem *mem,
1130	struct amdgpu_vm *vm,
1131	struct bo_vm_reservation_context *ctx)
1132	{
1133	struct amdgpu_bo *bo = mem->bo;
1134	int ret;
1135
1136	WARN_ON(!vm);
1137
1138	ctx->n_vms = 1;
1139	ctx->sync = &mem->sync;
1140	drm_exec_init(exec: &ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
1141	drm_exec_until_all_locked(&ctx->exec) {
1142	ret = amdgpu_vm_lock_pd(vm, exec: &ctx->exec, num_fences: 2);
1143	drm_exec_retry_on_contention(&ctx->exec);
1144	if (unlikely(ret))
1145	goto error;
1146
1147	ret = drm_exec_prepare_obj(exec: &ctx->exec, obj: &bo->tbo.base, num_fences: 1);
1148	drm_exec_retry_on_contention(&ctx->exec);
1149	if (unlikely(ret))
1150	goto error;
1151	}
1152	return 0;
1153
1154	error:
1155	pr_err("Failed to reserve buffers in ttm.\n");
1156	drm_exec_fini(exec: &ctx->exec);
1157	return ret;
1158	}
1159
1160	/**
1161	* reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally
1162	* @mem: KFD BO structure.
1163	* @vm: the VM to reserve. If NULL, then all VMs associated with the BO
1164	* is used. Otherwise, a single VM associated with the BO.
1165	* @map_type: the mapping status that will be used to filter the VMs.
1166	* @ctx: the struct that will be used in unreserve_bo_and_vms().
1167	*
1168	* Returns 0 for success, negative for failure.
1169	*/
1170	static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
1171	struct amdgpu_vm *vm, enum bo_vm_match map_type,
1172	struct bo_vm_reservation_context *ctx)
1173	{
1174	struct kfd_mem_attachment *entry;
1175	struct amdgpu_bo *bo = mem->bo;
1176	int ret;
1177
1178	ctx->sync = &mem->sync;
1179	drm_exec_init(exec: &ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
1180	drm_exec_until_all_locked(&ctx->exec) {
1181	ctx->n_vms = 0;
1182	list_for_each_entry(entry, &mem->attachments, list) {
1183	if ((vm && vm != entry->bo_va->base.vm) ||
1184	(entry->is_mapped != map_type
1185	&& map_type != BO_VM_ALL))
1186	continue;
1187
1188	ret = amdgpu_vm_lock_pd(vm: entry->bo_va->base.vm,
1189	exec: &ctx->exec, num_fences: 2);
1190	drm_exec_retry_on_contention(&ctx->exec);
1191	if (unlikely(ret))
1192	goto error;
1193	++ctx->n_vms;
1194	}
1195
1196	ret = drm_exec_prepare_obj(exec: &ctx->exec, obj: &bo->tbo.base, num_fences: 1);
1197	drm_exec_retry_on_contention(&ctx->exec);
1198	if (unlikely(ret))
1199	goto error;
1200	}
1201	return 0;
1202
1203	error:
1204	pr_err("Failed to reserve buffers in ttm.\n");
1205	drm_exec_fini(exec: &ctx->exec);
1206	return ret;
1207	}
1208
1209	/**
1210	* unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context
1211	* @ctx: Reservation context to unreserve
1212	* @wait: Optionally wait for a sync object representing pending VM updates
1213	* @intr: Whether the wait is interruptible
1214	*
1215	* Also frees any resources allocated in
1216	* reserve_bo_and_(cond_)vm(s). Returns the status from
1217	* amdgpu_sync_wait.
1218	*/
1219	static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx,
1220	bool wait, bool intr)
1221	{
1222	int ret = 0;
1223
1224	if (wait)
1225	ret = amdgpu_sync_wait(sync: ctx->sync, intr);
1226
1227	drm_exec_fini(exec: &ctx->exec);
1228	ctx->sync = NULL;
1229	return ret;
1230	}
1231
1232	static void unmap_bo_from_gpuvm(struct kgd_mem *mem,
1233	struct kfd_mem_attachment *entry,
1234	struct amdgpu_sync *sync)
1235	{
1236	struct amdgpu_bo_va *bo_va = entry->bo_va;
1237	struct amdgpu_device *adev = entry->adev;
1238	struct amdgpu_vm *vm = bo_va->base.vm;
1239
1240	amdgpu_vm_bo_unmap(adev, bo_va, addr: entry->va);
1241
1242	amdgpu_vm_clear_freed(adev, vm, fence: &bo_va->last_pt_update);
1243
1244	amdgpu_sync_fence(sync, f: bo_va->last_pt_update);
1245	}
1246
1247	static int update_gpuvm_pte(struct kgd_mem *mem,
1248	struct kfd_mem_attachment *entry,
1249	struct amdgpu_sync *sync)
1250	{
1251	struct amdgpu_bo_va *bo_va = entry->bo_va;
1252	struct amdgpu_device *adev = entry->adev;
1253	int ret;
1254
1255	ret = kfd_mem_dmamap_attachment(mem, attachment: entry);
1256	if (ret)
1257	return ret;
1258
1259	/* Update the page tables */
1260	ret = amdgpu_vm_bo_update(adev, bo_va, clear: false);
1261	if (ret) {
1262	pr_err("amdgpu_vm_bo_update failed\n");
1263	return ret;
1264	}
1265
1266	return amdgpu_sync_fence(sync, f: bo_va->last_pt_update);
1267	}
1268
1269	static int map_bo_to_gpuvm(struct kgd_mem *mem,
1270	struct kfd_mem_attachment *entry,
1271	struct amdgpu_sync *sync,
1272	bool no_update_pte)
1273	{
1274	int ret;
1275
1276	/* Set virtual address for the allocation */
1277	ret = amdgpu_vm_bo_map(adev: entry->adev, bo_va: entry->bo_va, addr: entry->va, offset: 0,
1278	size: amdgpu_bo_size(bo: entry->bo_va->base.bo),
1279	flags: entry->pte_flags);
1280	if (ret) {
1281	pr_err("Failed to map VA 0x%llx in vm. ret %d\n",
1282	entry->va, ret);
1283	return ret;
1284	}
1285
1286	if (no_update_pte)
1287	return 0;
1288
1289	ret = update_gpuvm_pte(mem, entry, sync);
1290	if (ret) {
1291	pr_err("update_gpuvm_pte() failed\n");
1292	goto update_gpuvm_pte_failed;
1293	}
1294
1295	return 0;
1296
1297	update_gpuvm_pte_failed:
1298	unmap_bo_from_gpuvm(mem, entry, sync);
1299	kfd_mem_dmaunmap_attachment(mem, attachment: entry);
1300	return ret;
1301	}
1302
1303	static int process_validate_vms(struct amdkfd_process_info *process_info)
1304	{
1305	struct amdgpu_vm *peer_vm;
1306	int ret;
1307
1308	list_for_each_entry(peer_vm, &process_info->vm_list_head,
1309	vm_list_node) {
1310	ret = vm_validate_pt_pd_bos(vm: peer_vm);
1311	if (ret)
1312	return ret;
1313	}
1314
1315	return 0;
1316	}
1317
1318	static int process_sync_pds_resv(struct amdkfd_process_info *process_info,
1319	struct amdgpu_sync *sync)
1320	{
1321	struct amdgpu_vm *peer_vm;
1322	int ret;
1323
1324	list_for_each_entry(peer_vm, &process_info->vm_list_head,
1325	vm_list_node) {
1326	struct amdgpu_bo *pd = peer_vm->root.bo;
1327
1328	ret = amdgpu_sync_resv(NULL, sync, resv: pd->tbo.base.resv,
1329	mode: AMDGPU_SYNC_NE_OWNER,
1330	AMDGPU_FENCE_OWNER_KFD);
1331	if (ret)
1332	return ret;
1333	}
1334
1335	return 0;
1336	}
1337
1338	static int process_update_pds(struct amdkfd_process_info *process_info,
1339	struct amdgpu_sync *sync)
1340	{
1341	struct amdgpu_vm *peer_vm;
1342	int ret;
1343
1344	list_for_each_entry(peer_vm, &process_info->vm_list_head,
1345	vm_list_node) {
1346	ret = vm_update_pds(vm: peer_vm, sync);
1347	if (ret)
1348	return ret;
1349	}
1350
1351	return 0;
1352	}
1353
1354	static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
1355	struct dma_fence **ef)
1356	{
1357	struct amdkfd_process_info *info = NULL;
1358	int ret;
1359
1360	if (!*process_info) {
1361	info = kzalloc(size: sizeof(*info), GFP_KERNEL);
1362	if (!info)
1363	return -ENOMEM;
1364
1365	mutex_init(&info->lock);
1366	mutex_init(&info->notifier_lock);
1367	INIT_LIST_HEAD(list: &info->vm_list_head);
1368	INIT_LIST_HEAD(list: &info->kfd_bo_list);
1369	INIT_LIST_HEAD(list: &info->userptr_valid_list);
1370	INIT_LIST_HEAD(list: &info->userptr_inval_list);
1371
1372	info->eviction_fence =
1373	amdgpu_amdkfd_fence_create(context: dma_fence_context_alloc(num: 1),
1374	current->mm,
1375	NULL);
1376	if (!info->eviction_fence) {
1377	pr_err("Failed to create eviction fence\n");
1378	ret = -ENOMEM;
1379	goto create_evict_fence_fail;
1380	}
1381
1382	info->pid = get_task_pid(current->group_leader, type: PIDTYPE_PID);
1383	INIT_DELAYED_WORK(&info->restore_userptr_work,
1384	amdgpu_amdkfd_restore_userptr_worker);
1385
1386	*process_info = info;
1387	*ef = dma_fence_get(fence: &info->eviction_fence->base);
1388	}
1389
1390	vm->process_info = *process_info;
1391
1392	/* Validate page directory and attach eviction fence */
1393	ret = amdgpu_bo_reserve(bo: vm->root.bo, no_intr: true);
1394	if (ret)
1395	goto reserve_pd_fail;
1396	ret = vm_validate_pt_pd_bos(vm);
1397	if (ret) {
1398	pr_err("validate_pt_pd_bos() failed\n");
1399	goto validate_pd_fail;
1400	}
1401	ret = amdgpu_bo_sync_wait(bo: vm->root.bo,
1402	AMDGPU_FENCE_OWNER_KFD, intr: false);
1403	if (ret)
1404	goto wait_pd_fail;
1405	ret = dma_resv_reserve_fences(obj: vm->root.bo->tbo.base.resv, num_fences: 1);
1406	if (ret)
1407	goto reserve_shared_fail;
1408	dma_resv_add_fence(obj: vm->root.bo->tbo.base.resv,
1409	fence: &vm->process_info->eviction_fence->base,
1410	usage: DMA_RESV_USAGE_BOOKKEEP);
1411	amdgpu_bo_unreserve(bo: vm->root.bo);
1412
1413	/* Update process info */
1414	mutex_lock(&vm->process_info->lock);
1415	list_add_tail(new: &vm->vm_list_node,
1416	head: &(vm->process_info->vm_list_head));
1417	vm->process_info->n_vms++;
1418	mutex_unlock(lock: &vm->process_info->lock);
1419
1420	return 0;
1421
1422	reserve_shared_fail:
1423	wait_pd_fail:
1424	validate_pd_fail:
1425	amdgpu_bo_unreserve(bo: vm->root.bo);
1426	reserve_pd_fail:
1427	vm->process_info = NULL;
1428	if (info) {
1429	/* Two fence references: one in info and one in *ef */
1430	dma_fence_put(fence: &info->eviction_fence->base);
1431	dma_fence_put(fence: *ef);
1432	*ef = NULL;
1433	*process_info = NULL;
1434	put_pid(pid: info->pid);
1435	create_evict_fence_fail:
1436	mutex_destroy(lock: &info->lock);
1437	mutex_destroy(lock: &info->notifier_lock);
1438	kfree(objp: info);
1439	}
1440	return ret;
1441	}
1442
1443	/**
1444	* amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria
1445	* @bo: Handle of buffer object being pinned
1446	* @domain: Domain into which BO should be pinned
1447	*
1448	* - USERPTR BOs are UNPINNABLE and will return error
1449	* - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
1450	* PIN count incremented. It is valid to PIN a BO multiple times
1451	*
1452	* Return: ZERO if successful in pinning, Non-Zero in case of error.
1453	*/
1454	static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
1455	{
1456	int ret = 0;
1457
1458	ret = amdgpu_bo_reserve(bo, no_intr: false);
1459	if (unlikely(ret))
1460	return ret;
1461
1462	ret = amdgpu_bo_pin_restricted(bo, domain, min_offset: 0, max_offset: 0);
1463	if (ret)
1464	pr_err("Error in Pinning BO to domain: %d\n", domain);
1465
1466	amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, intr: false);
1467	amdgpu_bo_unreserve(bo);
1468
1469	return ret;
1470	}
1471
1472	/**
1473	* amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria
1474	* @bo: Handle of buffer object being unpinned
1475	*
1476	* - Is a illegal request for USERPTR BOs and is ignored
1477	* - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
1478	* PIN count decremented. Calls to UNPIN must balance calls to PIN
1479	*/
1480	static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
1481	{
1482	int ret = 0;
1483
1484	ret = amdgpu_bo_reserve(bo, no_intr: false);
1485	if (unlikely(ret))
1486	return;
1487
1488	amdgpu_bo_unpin(bo);
1489	amdgpu_bo_unreserve(bo);
1490	}
1491
1492	int amdgpu_amdkfd_gpuvm_set_vm_pasid(struct amdgpu_device *adev,
1493	struct amdgpu_vm *avm, u32 pasid)
1494
1495	{
1496	int ret;
1497
1498	/* Free the original amdgpu allocated pasid,
1499	* will be replaced with kfd allocated pasid.
1500	*/
1501	if (avm->pasid) {
1502	amdgpu_pasid_free(pasid: avm->pasid);
1503	amdgpu_vm_set_pasid(adev, vm: avm, pasid: 0);
1504	}
1505
1506	ret = amdgpu_vm_set_pasid(adev, vm: avm, pasid);
1507	if (ret)
1508	return ret;
1509
1510	return 0;
1511	}
1512
1513	int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
1514	struct amdgpu_vm *avm,
1515	void **process_info,
1516	struct dma_fence **ef)
1517	{
1518	int ret;
1519
1520	/* Already a compute VM? */
1521	if (avm->process_info)
1522	return -EINVAL;
1523
1524	/* Convert VM into a compute VM */
1525	ret = amdgpu_vm_make_compute(adev, vm: avm);
1526	if (ret)
1527	return ret;
1528
1529	/* Initialize KFD part of the VM and process info */
1530	ret = init_kfd_vm(vm: avm, process_info, ef);
1531	if (ret)
1532	return ret;
1533
1534	amdgpu_vm_set_task_info(vm: avm);
1535
1536	return 0;
1537	}
1538
1539	void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
1540	struct amdgpu_vm *vm)
1541	{
1542	struct amdkfd_process_info *process_info = vm->process_info;
1543
1544	if (!process_info)
1545	return;
1546
1547	/* Update process info */
1548	mutex_lock(&process_info->lock);
1549	process_info->n_vms--;
1550	list_del(entry: &vm->vm_list_node);
1551	mutex_unlock(lock: &process_info->lock);
1552
1553	vm->process_info = NULL;
1554
1555	/* Release per-process resources when last compute VM is destroyed */
1556	if (!process_info->n_vms) {
1557	WARN_ON(!list_empty(&process_info->kfd_bo_list));
1558	WARN_ON(!list_empty(&process_info->userptr_valid_list));
1559	WARN_ON(!list_empty(&process_info->userptr_inval_list));
1560
1561	dma_fence_put(fence: &process_info->eviction_fence->base);
1562	cancel_delayed_work_sync(dwork: &process_info->restore_userptr_work);
1563	put_pid(pid: process_info->pid);
1564	mutex_destroy(lock: &process_info->lock);
1565	mutex_destroy(lock: &process_info->notifier_lock);
1566	kfree(objp: process_info);
1567	}
1568	}
1569
1570	void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
1571	void *drm_priv)
1572	{
1573	struct amdgpu_vm *avm;
1574
1575	if (WARN_ON(!adev || !drm_priv))
1576	return;
1577
1578	avm = drm_priv_to_vm(drm_priv);
1579
1580	pr_debug("Releasing process vm %p\n", avm);
1581
1582	/* The original pasid of amdgpu vm has already been
1583	* released during making a amdgpu vm to a compute vm
1584	* The current pasid is managed by kfd and will be
1585	* released on kfd process destroy. Set amdgpu pasid
1586	* to 0 to avoid duplicate release.
1587	*/
1588	amdgpu_vm_release_compute(adev, vm: avm);
1589	}
1590
1591	uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv)
1592	{
1593	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1594	struct amdgpu_bo *pd = avm->root.bo;
1595	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: pd->tbo.bdev);
1596
1597	if (adev->asic_type < CHIP_VEGA10)
1598	return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT;
1599	return avm->pd_phys_addr;
1600	}
1601
1602	void amdgpu_amdkfd_block_mmu_notifications(void *p)
1603	{
1604	struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;
1605
1606	mutex_lock(&pinfo->lock);
1607	WRITE_ONCE(pinfo->block_mmu_notifications, true);
1608	mutex_unlock(lock: &pinfo->lock);
1609	}
1610
1611	int amdgpu_amdkfd_criu_resume(void *p)
1612	{
1613	int ret = 0;
1614	struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;
1615
1616	mutex_lock(&pinfo->lock);
1617	pr_debug("scheduling work\n");
1618	mutex_lock(&pinfo->notifier_lock);
1619	pinfo->evicted_bos++;
1620	mutex_unlock(lock: &pinfo->notifier_lock);
1621	if (!READ_ONCE(pinfo->block_mmu_notifications)) {
1622	ret = -EINVAL;
1623	goto out_unlock;
1624	}
1625	WRITE_ONCE(pinfo->block_mmu_notifications, false);
1626	schedule_delayed_work(dwork: &pinfo->restore_userptr_work, delay: 0);
1627
1628	out_unlock:
1629	mutex_unlock(lock: &pinfo->lock);
1630	return ret;
1631	}
1632
1633	size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
1634	uint8_t xcp_id)
1635	{
1636	uint64_t reserved_for_pt =
1637	ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
1638	ssize_t available;
1639	uint64_t vram_available, system_mem_available, ttm_mem_available;
1640
1641	spin_lock(lock: &kfd_mem_limit.mem_limit_lock);
1642	vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id)
1643	- adev->kfd.vram_used_aligned[xcp_id]
1644	- atomic64_read(v: &adev->vram_pin_size)
1645	- reserved_for_pt;
1646
1647	if (adev->gmc.is_app_apu) {
1648	system_mem_available = no_system_mem_limit ?
1649	kfd_mem_limit.max_system_mem_limit :
1650	kfd_mem_limit.max_system_mem_limit -
1651	kfd_mem_limit.system_mem_used;
1652
1653	ttm_mem_available = kfd_mem_limit.max_ttm_mem_limit -
1654	kfd_mem_limit.ttm_mem_used;
1655
1656	available = min3(system_mem_available, ttm_mem_available,
1657	vram_available);
1658	available = ALIGN_DOWN(available, PAGE_SIZE);
1659	} else {
1660	available = ALIGN_DOWN(vram_available, VRAM_AVAILABLITY_ALIGN);
1661	}
1662
1663	spin_unlock(lock: &kfd_mem_limit.mem_limit_lock);
1664
1665	if (available < 0)
1666	available = 0;
1667
1668	return available;
1669	}
1670
1671	int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1672	struct amdgpu_device *adev, uint64_t va, uint64_t size,
1673	void *drm_priv, struct kgd_mem **mem,
1674	uint64_t *offset, uint32_t flags, bool criu_resume)
1675	{
1676	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1677	struct amdgpu_fpriv *fpriv = container_of(avm, struct amdgpu_fpriv, vm);
1678	enum ttm_bo_type bo_type = ttm_bo_type_device;
1679	struct sg_table *sg = NULL;
1680	uint64_t user_addr = 0;
1681	struct amdgpu_bo *bo;
1682	struct drm_gem_object *gobj = NULL;
1683	u32 domain, alloc_domain;
1684	uint64_t aligned_size;
1685	int8_t xcp_id = -1;
1686	u64 alloc_flags;
1687	int ret;
1688
1689	/*
1690	* Check on which domain to allocate BO
1691	*/
1692	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
1693	domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;
1694
1695	if (adev->gmc.is_app_apu) {
1696	domain = AMDGPU_GEM_DOMAIN_GTT;
1697	alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1698	alloc_flags = 0;
1699	} else {
1700	alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
1701	alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ?
1702	AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0;
1703	}
1704	xcp_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ?
1705	0 : fpriv->xcp_id;
1706	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
1707	domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1708	alloc_flags = 0;
1709	} else {
1710	domain = AMDGPU_GEM_DOMAIN_GTT;
1711	alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
1712	alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE;
1713
1714	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1715	if (!offset || !*offset)
1716	return -EINVAL;
1717	user_addr = untagged_addr(*offset);
1718	} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1719	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1720	bo_type = ttm_bo_type_sg;
1721	if (size > UINT_MAX)
1722	return -EINVAL;
1723	sg = create_sg_table(addr: *offset, size);
1724	if (!sg)
1725	return -ENOMEM;
1726	} else {
1727	return -EINVAL;
1728	}
1729	}
1730
1731	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT)
1732	alloc_flags |= AMDGPU_GEM_CREATE_COHERENT;
1733	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT)
1734	alloc_flags |= AMDGPU_GEM_CREATE_EXT_COHERENT;
1735	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED)
1736	alloc_flags |= AMDGPU_GEM_CREATE_UNCACHED;
1737
1738	*mem = kzalloc(size: sizeof(struct kgd_mem), GFP_KERNEL);
1739	if (!*mem) {
1740	ret = -ENOMEM;
1741	goto err;
1742	}
1743	INIT_LIST_HEAD(list: &(*mem)->attachments);
1744	mutex_init(&(*mem)->lock);
1745	(*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM);
1746
1747	/* Workaround for AQL queue wraparound bug. Map the same
1748	* memory twice. That means we only actually allocate half
1749	* the memory.
1750	*/
1751	if ((*mem)->aql_queue)
1752	size >>= 1;
1753	aligned_size = PAGE_ALIGN(size);
1754
1755	(*mem)->alloc_flags = flags;
1756
1757	amdgpu_sync_create(sync: &(*mem)->sync);
1758
1759	ret = amdgpu_amdkfd_reserve_mem_limit(adev, size: aligned_size, alloc_flag: flags,
1760	xcp_id);
1761	if (ret) {
1762	pr_debug("Insufficient memory\n");
1763	goto err_reserve_limit;
1764	}
1765
1766	pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s xcp_id %d\n",
1767	va, (*mem)->aql_queue ? size << 1 : size,
1768	domain_string(alloc_domain), xcp_id);
1769
1770	ret = amdgpu_gem_object_create(adev, size: aligned_size, alignment: 1, initial_domain: alloc_domain, flags: alloc_flags,
1771	type: bo_type, NULL, obj: &gobj, xcp_id_plus1: xcp_id + 1);
1772	if (ret) {
1773	pr_debug("Failed to create BO on domain %s. ret %d\n",
1774	domain_string(alloc_domain), ret);
1775	goto err_bo_create;
1776	}
1777	ret = drm_vma_node_allow(node: &gobj->vma_node, tag: drm_priv);
1778	if (ret) {
1779	pr_debug("Failed to allow vma node access. ret %d\n", ret);
1780	goto err_node_allow;
1781	}
1782	bo = gem_to_amdgpu_bo(gobj);
1783	if (bo_type == ttm_bo_type_sg) {
1784	bo->tbo.sg = sg;
1785	bo->tbo.ttm->sg = sg;
1786	}
1787	bo->kfd_bo = *mem;
1788	(*mem)->bo = bo;
1789	if (user_addr)
1790	bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO;
1791
1792	(*mem)->va = va;
1793	(*mem)->domain = domain;
1794	(*mem)->mapped_to_gpu_memory = 0;
1795	(*mem)->process_info = avm->process_info;
1796
1797	add_kgd_mem_to_kfd_bo_list(mem: *mem, process_info: avm->process_info, userptr: user_addr);
1798
1799	if (user_addr) {
1800	pr_debug("creating userptr BO for user_addr = %llx\n", user_addr);
1801	ret = init_user_pages(mem: *mem, user_addr, criu_resume);
1802	if (ret)
1803	goto allocate_init_user_pages_failed;
1804	} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1805	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1806	ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);
1807	if (ret) {
1808	pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n");
1809	goto err_pin_bo;
1810	}
1811	bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
1812	bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
1813	} else {
1814	mutex_lock(&avm->process_info->lock);
1815	if (avm->process_info->eviction_fence &&
1816	!dma_fence_is_signaled(fence: &avm->process_info->eviction_fence->base))
1817	ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain,
1818	fence: &avm->process_info->eviction_fence->base);
1819	mutex_unlock(lock: &avm->process_info->lock);
1820	if (ret)
1821	goto err_validate_bo;
1822	}
1823
1824	if (offset)
1825	*offset = amdgpu_bo_mmap_offset(bo);
1826
1827	return 0;
1828
1829	allocate_init_user_pages_failed:
1830	err_pin_bo:
1831	err_validate_bo:
1832	remove_kgd_mem_from_kfd_bo_list(mem: *mem, process_info: avm->process_info);
1833	drm_vma_node_revoke(node: &gobj->vma_node, tag: drm_priv);
1834	err_node_allow:
1835	/* Don't unreserve system mem limit twice */
1836	goto err_reserve_limit;
1837	err_bo_create:
1838	amdgpu_amdkfd_unreserve_mem_limit(adev, size: aligned_size, alloc_flag: flags, xcp_id);
1839	err_reserve_limit:
1840	mutex_destroy(lock: &(*mem)->lock);
1841	if (gobj)
1842	drm_gem_object_put(obj: gobj);
1843	else
1844	kfree(objp: *mem);
1845	err:
1846	if (sg) {
1847	sg_free_table(sg);
1848	kfree(objp: sg);
1849	}
1850	return ret;
1851	}
1852
1853	int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
1854	struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
1855	uint64_t *size)
1856	{
1857	struct amdkfd_process_info *process_info = mem->process_info;
1858	unsigned long bo_size = mem->bo->tbo.base.size;
1859	bool use_release_notifier = (mem->bo->kfd_bo == mem);
1860	struct kfd_mem_attachment *entry, *tmp;
1861	struct bo_vm_reservation_context ctx;
1862	unsigned int mapped_to_gpu_memory;
1863	int ret;
1864	bool is_imported = false;
1865
1866	mutex_lock(&mem->lock);
1867
1868	/* Unpin MMIO/DOORBELL BO's that were pinned during allocation */
1869	if (mem->alloc_flags &
1870	(KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1871	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1872	amdgpu_amdkfd_gpuvm_unpin_bo(bo: mem->bo);
1873	}
1874
1875	mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
1876	is_imported = mem->is_imported;
1877	mutex_unlock(lock: &mem->lock);
1878	/* lock is not needed after this, since mem is unused and will
1879	* be freed anyway
1880	*/
1881
1882	if (mapped_to_gpu_memory > 0) {
1883	pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n",
1884	mem->va, bo_size);
1885	return -EBUSY;
1886	}
1887
1888	/* Make sure restore workers don't access the BO any more */
1889	mutex_lock(&process_info->lock);
1890	list_del(entry: &mem->validate_list);
1891	mutex_unlock(lock: &process_info->lock);
1892
1893	/* Cleanup user pages and MMU notifiers */
1894	if (amdgpu_ttm_tt_get_usermm(ttm: mem->bo->tbo.ttm)) {
1895	amdgpu_hmm_unregister(bo: mem->bo);
1896	mutex_lock(&process_info->notifier_lock);
1897	amdgpu_ttm_tt_discard_user_pages(ttm: mem->bo->tbo.ttm, range: mem->range);
1898	mutex_unlock(lock: &process_info->notifier_lock);
1899	}
1900
1901	ret = reserve_bo_and_cond_vms(mem, NULL, map_type: BO_VM_ALL, ctx: &ctx);
1902	if (unlikely(ret))
1903	return ret;
1904
1905	amdgpu_amdkfd_remove_eviction_fence(bo: mem->bo,
1906	ef: process_info->eviction_fence);
1907	pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,
1908	mem->va + bo_size * (1 + mem->aql_queue));
1909
1910	/* Remove from VM internal data structures */
1911	list_for_each_entry_safe(entry, tmp, &mem->attachments, list) {
1912	kfd_mem_dmaunmap_attachment(mem, attachment: entry);
1913	kfd_mem_detach(attachment: entry);
1914	}
1915
1916	ret = unreserve_bo_and_vms(ctx: &ctx, wait: false, intr: false);
1917
1918	/* Free the sync object */
1919	amdgpu_sync_free(sync: &mem->sync);
1920
1921	/* If the SG is not NULL, it's one we created for a doorbell or mmio
1922	* remap BO. We need to free it.
1923	*/
1924	if (mem->bo->tbo.sg) {
1925	sg_free_table(mem->bo->tbo.sg);
1926	kfree(objp: mem->bo->tbo.sg);
1927	}
1928
1929	/* Update the size of the BO being freed if it was allocated from
1930	* VRAM and is not imported. For APP APU VRAM allocations are done
1931	* in GTT domain
1932	*/
1933	if (size) {
1934	if (!is_imported &&
1935	(mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM ||
1936	(adev->gmc.is_app_apu &&
1937	mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT)))
1938	*size = bo_size;
1939	else
1940	*size = 0;
1941	}
1942
1943	/* Free the BO*/
1944	drm_vma_node_revoke(node: &mem->bo->tbo.base.vma_node, tag: drm_priv);
1945	if (mem->dmabuf)
1946	dma_buf_put(dmabuf: mem->dmabuf);
1947	mutex_destroy(lock: &mem->lock);
1948
1949	/* If this releases the last reference, it will end up calling
1950	* amdgpu_amdkfd_release_notify and kfree the mem struct. That's why
1951	* this needs to be the last call here.
1952	*/
1953	drm_gem_object_put(obj: &mem->bo->tbo.base);
1954
1955	/*
1956	* For kgd_mem allocated in amdgpu_amdkfd_gpuvm_import_dmabuf(),
1957	* explicitly free it here.
1958	*/
1959	if (!use_release_notifier)
1960	kfree(objp: mem);
1961
1962	return ret;
1963	}
1964
1965	int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1966	struct amdgpu_device *adev, struct kgd_mem *mem,
1967	void *drm_priv)
1968	{
1969	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1970	int ret;
1971	struct amdgpu_bo *bo;
1972	uint32_t domain;
1973	struct kfd_mem_attachment *entry;
1974	struct bo_vm_reservation_context ctx;
1975	unsigned long bo_size;
1976	bool is_invalid_userptr = false;
1977
1978	bo = mem->bo;
1979	if (!bo) {
1980	pr_err("Invalid BO when mapping memory to GPU\n");
1981	return -EINVAL;
1982	}
1983
1984	/* Make sure restore is not running concurrently. Since we
1985	* don't map invalid userptr BOs, we rely on the next restore
1986	* worker to do the mapping
1987	*/
1988	mutex_lock(&mem->process_info->lock);
1989
1990	/* Lock notifier lock. If we find an invalid userptr BO, we can be
1991	* sure that the MMU notifier is no longer running
1992	* concurrently and the queues are actually stopped
1993	*/
1994	if (amdgpu_ttm_tt_get_usermm(ttm: bo->tbo.ttm)) {
1995	mutex_lock(&mem->process_info->notifier_lock);
1996	is_invalid_userptr = !!mem->invalid;
1997	mutex_unlock(lock: &mem->process_info->notifier_lock);
1998	}
1999
2000	mutex_lock(&mem->lock);
2001
2002	domain = mem->domain;
2003	bo_size = bo->tbo.base.size;
2004
2005	pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
2006	mem->va,
2007	mem->va + bo_size * (1 + mem->aql_queue),
2008	avm, domain_string(domain));
2009
2010	if (!kfd_mem_is_attached(avm, mem)) {
2011	ret = kfd_mem_attach(adev, mem, vm: avm, is_aql: mem->aql_queue);
2012	if (ret)
2013	goto out;
2014	}
2015
2016	ret = reserve_bo_and_vm(mem, vm: avm, ctx: &ctx);
2017	if (unlikely(ret))
2018	goto out;
2019
2020	/* Userptr can be marked as "not invalid", but not actually be
2021	* validated yet (still in the system domain). In that case
2022	* the queues are still stopped and we can leave mapping for
2023	* the next restore worker
2024	*/
2025	if (amdgpu_ttm_tt_get_usermm(ttm: bo->tbo.ttm) &&
2026	bo->tbo.resource->mem_type == TTM_PL_SYSTEM)
2027	is_invalid_userptr = true;
2028
2029	ret = vm_validate_pt_pd_bos(vm: avm);
2030	if (unlikely(ret))
2031	goto out_unreserve;
2032
2033	list_for_each_entry(entry, &mem->attachments, list) {
2034	if (entry->bo_va->base.vm != avm || entry->is_mapped)
2035	continue;
2036
2037	pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
2038	entry->va, entry->va + bo_size, entry);
2039
2040	ret = map_bo_to_gpuvm(mem, entry, sync: ctx.sync,
2041	no_update_pte: is_invalid_userptr);
2042	if (ret) {
2043	pr_err("Failed to map bo to gpuvm\n");
2044	goto out_unreserve;
2045	}
2046
2047	ret = vm_update_pds(vm: avm, sync: ctx.sync);
2048	if (ret) {
2049	pr_err("Failed to update page directories\n");
2050	goto out_unreserve;
2051	}
2052
2053	entry->is_mapped = true;
2054	mem->mapped_to_gpu_memory++;
2055	pr_debug("\t INC mapping count %d\n",
2056	mem->mapped_to_gpu_memory);
2057	}
2058
2059	ret = unreserve_bo_and_vms(ctx: &ctx, wait: false, intr: false);
2060
2061	goto out;
2062
2063	out_unreserve:
2064	unreserve_bo_and_vms(ctx: &ctx, wait: false, intr: false);
2065	out:
2066	mutex_unlock(lock: &mem->process_info->lock);
2067	mutex_unlock(lock: &mem->lock);
2068	return ret;
2069	}
2070
2071	void amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv)
2072	{
2073	struct kfd_mem_attachment *entry;
2074	struct amdgpu_vm *vm;
2075
2076	vm = drm_priv_to_vm(drm_priv);
2077
2078	mutex_lock(&mem->lock);
2079
2080	list_for_each_entry(entry, &mem->attachments, list) {
2081	if (entry->bo_va->base.vm == vm)
2082	kfd_mem_dmaunmap_attachment(mem, attachment: entry);
2083	}
2084
2085	mutex_unlock(lock: &mem->lock);
2086	}
2087
2088	int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
2089	struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv)
2090	{
2091	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2092	unsigned long bo_size = mem->bo->tbo.base.size;
2093	struct kfd_mem_attachment *entry;
2094	struct bo_vm_reservation_context ctx;
2095	int ret;
2096
2097	mutex_lock(&mem->lock);
2098
2099	ret = reserve_bo_and_cond_vms(mem, vm: avm, map_type: BO_VM_MAPPED, ctx: &ctx);
2100	if (unlikely(ret))
2101	goto out;
2102	/* If no VMs were reserved, it means the BO wasn't actually mapped */
2103	if (ctx.n_vms == 0) {
2104	ret = -EINVAL;
2105	goto unreserve_out;
2106	}
2107
2108	ret = vm_validate_pt_pd_bos(vm: avm);
2109	if (unlikely(ret))
2110	goto unreserve_out;
2111
2112	pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
2113	mem->va,
2114	mem->va + bo_size * (1 + mem->aql_queue),
2115	avm);
2116
2117	list_for_each_entry(entry, &mem->attachments, list) {
2118	if (entry->bo_va->base.vm != avm || !entry->is_mapped)
2119	continue;
2120
2121	pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
2122	entry->va, entry->va + bo_size, entry);
2123
2124	unmap_bo_from_gpuvm(mem, entry, sync: ctx.sync);
2125	entry->is_mapped = false;
2126
2127	mem->mapped_to_gpu_memory--;
2128	pr_debug("\t DEC mapping count %d\n",
2129	mem->mapped_to_gpu_memory);
2130	}
2131
2132	unreserve_out:
2133	unreserve_bo_and_vms(ctx: &ctx, wait: false, intr: false);
2134	out:
2135	mutex_unlock(lock: &mem->lock);
2136	return ret;
2137	}
2138
2139	int amdgpu_amdkfd_gpuvm_sync_memory(
2140	struct amdgpu_device *adev, struct kgd_mem *mem, bool intr)
2141	{
2142	struct amdgpu_sync sync;
2143	int ret;
2144
2145	amdgpu_sync_create(sync: &sync);
2146
2147	mutex_lock(&mem->lock);
2148	amdgpu_sync_clone(source: &mem->sync, clone: &sync);
2149	mutex_unlock(lock: &mem->lock);
2150
2151	ret = amdgpu_sync_wait(sync: &sync, intr);
2152	amdgpu_sync_free(sync: &sync);
2153	return ret;
2154	}
2155
2156	/**
2157	* amdgpu_amdkfd_map_gtt_bo_to_gart - Map BO to GART and increment reference count
2158	* @adev: Device to which allocated BO belongs
2159	* @bo: Buffer object to be mapped
2160	*
2161	* Before return, bo reference count is incremented. To release the reference and unpin/
2162	* unmap the BO, call amdgpu_amdkfd_free_gtt_mem.
2163	*/
2164	int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_device *adev, struct amdgpu_bo *bo)
2165	{
2166	int ret;
2167
2168	ret = amdgpu_bo_reserve(bo, no_intr: true);
2169	if (ret) {
2170	pr_err("Failed to reserve bo. ret %d\n", ret);
2171	goto err_reserve_bo_failed;
2172	}
2173
2174	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
2175	if (ret) {
2176	pr_err("Failed to pin bo. ret %d\n", ret);
2177	goto err_pin_bo_failed;
2178	}
2179
2180	ret = amdgpu_ttm_alloc_gart(bo: &bo->tbo);
2181	if (ret) {
2182	pr_err("Failed to bind bo to GART. ret %d\n", ret);
2183	goto err_map_bo_gart_failed;
2184	}
2185
2186	amdgpu_amdkfd_remove_eviction_fence(
2187	bo, ef: bo->vm_bo->vm->process_info->eviction_fence);
2188
2189	amdgpu_bo_unreserve(bo);
2190
2191	bo = amdgpu_bo_ref(bo);
2192
2193	return 0;
2194
2195	err_map_bo_gart_failed:
2196	amdgpu_bo_unpin(bo);
2197	err_pin_bo_failed:
2198	amdgpu_bo_unreserve(bo);
2199	err_reserve_bo_failed:
2200
2201	return ret;
2202	}
2203
2204	/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Map a GTT BO for kernel CPU access
2205	*
2206	* @mem: Buffer object to be mapped for CPU access
2207	* @kptr[out]: pointer in kernel CPU address space
2208	* @size[out]: size of the buffer
2209	*
2210	* Pins the BO and maps it for kernel CPU access. The eviction fence is removed
2211	* from the BO, since pinned BOs cannot be evicted. The bo must remain on the
2212	* validate_list, so the GPU mapping can be restored after a page table was
2213	* evicted.
2214	*
2215	* Return: 0 on success, error code on failure
2216	*/
2217	int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
2218	void **kptr, uint64_t *size)
2219	{
2220	int ret;
2221	struct amdgpu_bo *bo = mem->bo;
2222
2223	if (amdgpu_ttm_tt_get_usermm(ttm: bo->tbo.ttm)) {
2224	pr_err("userptr can't be mapped to kernel\n");
2225	return -EINVAL;
2226	}
2227
2228	mutex_lock(&mem->process_info->lock);
2229
2230	ret = amdgpu_bo_reserve(bo, no_intr: true);
2231	if (ret) {
2232	pr_err("Failed to reserve bo. ret %d\n", ret);
2233	goto bo_reserve_failed;
2234	}
2235
2236	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
2237	if (ret) {
2238	pr_err("Failed to pin bo. ret %d\n", ret);
2239	goto pin_failed;
2240	}
2241
2242	ret = amdgpu_bo_kmap(bo, ptr: kptr);
2243	if (ret) {
2244	pr_err("Failed to map bo to kernel. ret %d\n", ret);
2245	goto kmap_failed;
2246	}
2247
2248	amdgpu_amdkfd_remove_eviction_fence(
2249	bo, ef: mem->process_info->eviction_fence);
2250
2251	if (size)
2252	*size = amdgpu_bo_size(bo);
2253
2254	amdgpu_bo_unreserve(bo);
2255
2256	mutex_unlock(lock: &mem->process_info->lock);
2257	return 0;
2258
2259	kmap_failed:
2260	amdgpu_bo_unpin(bo);
2261	pin_failed:
2262	amdgpu_bo_unreserve(bo);
2263	bo_reserve_failed:
2264	mutex_unlock(lock: &mem->process_info->lock);
2265
2266	return ret;
2267	}
2268
2269	/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Unmap a GTT BO for kernel CPU access
2270	*
2271	* @mem: Buffer object to be unmapped for CPU access
2272	*
2273	* Removes the kernel CPU mapping and unpins the BO. It does not restore the
2274	* eviction fence, so this function should only be used for cleanup before the
2275	* BO is destroyed.
2276	*/
2277	void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem)
2278	{
2279	struct amdgpu_bo *bo = mem->bo;
2280
2281	amdgpu_bo_reserve(bo, no_intr: true);
2282	amdgpu_bo_kunmap(bo);
2283	amdgpu_bo_unpin(bo);
2284	amdgpu_bo_unreserve(bo);
2285	}
2286
2287	int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev,
2288	struct kfd_vm_fault_info *mem)
2289	{
2290	if (atomic_read(v: &adev->gmc.vm_fault_info_updated) == 1) {
2291	*mem = *adev->gmc.vm_fault_info;
2292	mb(); /* make sure read happened */
2293	atomic_set(v: &adev->gmc.vm_fault_info_updated, i: 0);
2294	}
2295	return 0;
2296	}
2297
2298	int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev,
2299	struct dma_buf *dma_buf,
2300	uint64_t va, void *drm_priv,
2301	struct kgd_mem **mem, uint64_t *size,
2302	uint64_t *mmap_offset)
2303	{
2304	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2305	struct drm_gem_object *obj;
2306	struct amdgpu_bo *bo;
2307	int ret;
2308
2309	obj = amdgpu_gem_prime_import(dev: adev_to_drm(adev), dma_buf);
2310	if (IS_ERR(ptr: obj))
2311	return PTR_ERR(ptr: obj);
2312
2313	bo = gem_to_amdgpu_bo(obj);
2314	if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
2315	AMDGPU_GEM_DOMAIN_GTT))) {
2316	/* Only VRAM and GTT BOs are supported */
2317	ret = -EINVAL;
2318	goto err_put_obj;
2319	}
2320
2321	*mem = kzalloc(size: sizeof(struct kgd_mem), GFP_KERNEL);
2322	if (!*mem) {
2323	ret = -ENOMEM;
2324	goto err_put_obj;
2325	}
2326
2327	ret = drm_vma_node_allow(node: &obj->vma_node, tag: drm_priv);
2328	if (ret)
2329	goto err_free_mem;
2330
2331	if (size)
2332	*size = amdgpu_bo_size(bo);
2333
2334	if (mmap_offset)
2335	*mmap_offset = amdgpu_bo_mmap_offset(bo);
2336
2337	INIT_LIST_HEAD(list: &(*mem)->attachments);
2338	mutex_init(&(*mem)->lock);
2339
2340	(*mem)->alloc_flags =
2341	((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
2342	KFD_IOC_ALLOC_MEM_FLAGS_VRAM : KFD_IOC_ALLOC_MEM_FLAGS_GTT)
2343	| KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE
2344	| KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
2345
2346	get_dma_buf(dmabuf: dma_buf);
2347	(*mem)->dmabuf = dma_buf;
2348	(*mem)->bo = bo;
2349	(*mem)->va = va;
2350	(*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) && !adev->gmc.is_app_apu ?
2351	AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;
2352
2353	(*mem)->mapped_to_gpu_memory = 0;
2354	(*mem)->process_info = avm->process_info;
2355	add_kgd_mem_to_kfd_bo_list(mem: *mem, process_info: avm->process_info, userptr: false);
2356	amdgpu_sync_create(sync: &(*mem)->sync);
2357	(*mem)->is_imported = true;
2358
2359	mutex_lock(&avm->process_info->lock);
2360	if (avm->process_info->eviction_fence &&
2361	!dma_fence_is_signaled(fence: &avm->process_info->eviction_fence->base))
2362	ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain: (*mem)->domain,
2363	fence: &avm->process_info->eviction_fence->base);
2364	mutex_unlock(lock: &avm->process_info->lock);
2365	if (ret)
2366	goto err_remove_mem;
2367
2368	return 0;
2369
2370	err_remove_mem:
2371	remove_kgd_mem_from_kfd_bo_list(mem: *mem, process_info: avm->process_info);
2372	drm_vma_node_revoke(node: &obj->vma_node, tag: drm_priv);
2373	err_free_mem:
2374	kfree(objp: *mem);
2375	err_put_obj:
2376	drm_gem_object_put(obj);
2377	return ret;
2378	}
2379
2380	int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem,
2381	struct dma_buf **dma_buf)
2382	{
2383	int ret;
2384
2385	mutex_lock(&mem->lock);
2386	ret = kfd_mem_export_dmabuf(mem);
2387	if (ret)
2388	goto out;
2389
2390	get_dma_buf(dmabuf: mem->dmabuf);
2391	*dma_buf = mem->dmabuf;
2392	out:
2393	mutex_unlock(lock: &mem->lock);
2394	return ret;
2395	}
2396
2397	/* Evict a userptr BO by stopping the queues if necessary
2398	*
2399	* Runs in MMU notifier, may be in RECLAIM_FS context. This means it
2400	* cannot do any memory allocations, and cannot take any locks that
2401	* are held elsewhere while allocating memory.
2402	*
2403	* It doesn't do anything to the BO itself. The real work happens in
2404	* restore, where we get updated page addresses. This function only
2405	* ensures that GPU access to the BO is stopped.
2406	*/
2407	int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni,
2408	unsigned long cur_seq, struct kgd_mem *mem)
2409	{
2410	struct amdkfd_process_info *process_info = mem->process_info;
2411	int r = 0;
2412
2413	/* Do not process MMU notifications during CRIU restore until
2414	* KFD_CRIU_OP_RESUME IOCTL is received
2415	*/
2416	if (READ_ONCE(process_info->block_mmu_notifications))
2417	return 0;
2418
2419	mutex_lock(&process_info->notifier_lock);
2420	mmu_interval_set_seq(interval_sub: mni, cur_seq);
2421
2422	mem->invalid++;
2423	if (++process_info->evicted_bos == 1) {
2424	/* First eviction, stop the queues */
2425	r = kgd2kfd_quiesce_mm(mm: mni->mm,
2426	trigger: KFD_QUEUE_EVICTION_TRIGGER_USERPTR);
2427	if (r)
2428	pr_err("Failed to quiesce KFD\n");
2429	schedule_delayed_work(dwork: &process_info->restore_userptr_work,
2430	delay: msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
2431	}
2432	mutex_unlock(lock: &process_info->notifier_lock);
2433
2434	return r;
2435	}
2436
2437	/* Update invalid userptr BOs
2438	*
2439	* Moves invalidated (evicted) userptr BOs from userptr_valid_list to
2440	* userptr_inval_list and updates user pages for all BOs that have
2441	* been invalidated since their last update.
2442	*/
2443	static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
2444	struct mm_struct *mm)
2445	{
2446	struct kgd_mem *mem, *tmp_mem;
2447	struct amdgpu_bo *bo;
2448	struct ttm_operation_ctx ctx = { false, false };
2449	uint32_t invalid;
2450	int ret = 0;
2451
2452	mutex_lock(&process_info->notifier_lock);
2453
2454	/* Move all invalidated BOs to the userptr_inval_list */
2455	list_for_each_entry_safe(mem, tmp_mem,
2456	&process_info->userptr_valid_list,
2457	validate_list)
2458	if (mem->invalid)
2459	list_move_tail(list: &mem->validate_list,
2460	head: &process_info->userptr_inval_list);
2461
2462	/* Go through userptr_inval_list and update any invalid user_pages */
2463	list_for_each_entry(mem, &process_info->userptr_inval_list,
2464	validate_list) {
2465	invalid = mem->invalid;
2466	if (!invalid)
2467	/* BO hasn't been invalidated since the last
2468	* revalidation attempt. Keep its page list.
2469	*/
2470	continue;
2471
2472	bo = mem->bo;
2473
2474	amdgpu_ttm_tt_discard_user_pages(ttm: bo->tbo.ttm, range: mem->range);
2475	mem->range = NULL;
2476
2477	/* BO reservations and getting user pages (hmm_range_fault)
2478	* must happen outside the notifier lock
2479	*/
2480	mutex_unlock(lock: &process_info->notifier_lock);
2481
2482	/* Move the BO to system (CPU) domain if necessary to unmap
2483	* and free the SG table
2484	*/
2485	if (bo->tbo.resource->mem_type != TTM_PL_SYSTEM) {
2486	if (amdgpu_bo_reserve(bo, no_intr: true))
2487	return -EAGAIN;
2488	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_CPU);
2489	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
2490	amdgpu_bo_unreserve(bo);
2491	if (ret) {
2492	pr_err("%s: Failed to invalidate userptr BO\n",
2493	__func__);
2494	return -EAGAIN;
2495	}
2496	}
2497
2498	/* Get updated user pages */
2499	ret = amdgpu_ttm_tt_get_user_pages(bo, pages: bo->tbo.ttm->pages,
2500	range: &mem->range);
2501	if (ret) {
2502	pr_debug("Failed %d to get user pages\n", ret);
2503
2504	/* Return -EFAULT bad address error as success. It will
2505	* fail later with a VM fault if the GPU tries to access
2506	* it. Better than hanging indefinitely with stalled
2507	* user mode queues.
2508	*
2509	* Return other error -EBUSY or -ENOMEM to retry restore
2510	*/
2511	if (ret != -EFAULT)
2512	return ret;
2513
2514	ret = 0;
2515	}
2516
2517	mutex_lock(&process_info->notifier_lock);
2518
2519	/* Mark the BO as valid unless it was invalidated
2520	* again concurrently.
2521	*/
2522	if (mem->invalid != invalid) {
2523	ret = -EAGAIN;
2524	goto unlock_out;
2525	}
2526	/* set mem valid if mem has hmm range associated */
2527	if (mem->range)
2528	mem->invalid = 0;
2529	}
2530
2531	unlock_out:
2532	mutex_unlock(lock: &process_info->notifier_lock);
2533
2534	return ret;
2535	}
2536
2537	/* Validate invalid userptr BOs
2538	*
2539	* Validates BOs on the userptr_inval_list. Also updates GPUVM page tables
2540	* with new page addresses and waits for the page table updates to complete.
2541	*/
2542	static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
2543	{
2544	struct ttm_operation_ctx ctx = { false, false };
2545	struct amdgpu_sync sync;
2546	struct drm_exec exec;
2547
2548	struct amdgpu_vm *peer_vm;
2549	struct kgd_mem *mem, *tmp_mem;
2550	struct amdgpu_bo *bo;
2551	int ret;
2552
2553	amdgpu_sync_create(sync: &sync);
2554
2555	drm_exec_init(exec: &exec, flags: 0);
2556	/* Reserve all BOs and page tables for validation */
2557	drm_exec_until_all_locked(&exec) {
2558	/* Reserve all the page directories */
2559	list_for_each_entry(peer_vm, &process_info->vm_list_head,
2560	vm_list_node) {
2561	ret = amdgpu_vm_lock_pd(vm: peer_vm, exec: &exec, num_fences: 2);
2562	drm_exec_retry_on_contention(&exec);
2563	if (unlikely(ret))
2564	goto unreserve_out;
2565	}
2566
2567	/* Reserve the userptr_inval_list entries to resv_list */
2568	list_for_each_entry(mem, &process_info->userptr_inval_list,
2569	validate_list) {
2570	struct drm_gem_object *gobj;
2571
2572	gobj = &mem->bo->tbo.base;
2573	ret = drm_exec_prepare_obj(exec: &exec, obj: gobj, num_fences: 1);
2574	drm_exec_retry_on_contention(&exec);
2575	if (unlikely(ret))
2576	goto unreserve_out;
2577	}
2578	}
2579
2580	ret = process_validate_vms(process_info);
2581	if (ret)
2582	goto unreserve_out;
2583
2584	/* Validate BOs and update GPUVM page tables */
2585	list_for_each_entry_safe(mem, tmp_mem,
2586	&process_info->userptr_inval_list,
2587	validate_list) {
2588	struct kfd_mem_attachment *attachment;
2589
2590	bo = mem->bo;
2591
2592	/* Validate the BO if we got user pages */
2593	if (bo->tbo.ttm->pages[0]) {
2594	amdgpu_bo_placement_from_domain(abo: bo, domain: mem->domain);
2595	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
2596	if (ret) {
2597	pr_err("%s: failed to validate BO\n", __func__);
2598	goto unreserve_out;
2599	}
2600	}
2601
2602	/* Update mapping. If the BO was not validated
2603	* (because we couldn't get user pages), this will
2604	* clear the page table entries, which will result in
2605	* VM faults if the GPU tries to access the invalid
2606	* memory.
2607	*/
2608	list_for_each_entry(attachment, &mem->attachments, list) {
2609	if (!attachment->is_mapped)
2610	continue;
2611
2612	kfd_mem_dmaunmap_attachment(mem, attachment);
2613	ret = update_gpuvm_pte(mem, entry: attachment, sync: &sync);
2614	if (ret) {
2615	pr_err("%s: update PTE failed\n", __func__);
2616	/* make sure this gets validated again */
2617	mutex_lock(&process_info->notifier_lock);
2618	mem->invalid++;
2619	mutex_unlock(lock: &process_info->notifier_lock);
2620	goto unreserve_out;
2621	}
2622	}
2623	}
2624
2625	/* Update page directories */
2626	ret = process_update_pds(process_info, sync: &sync);
2627
2628	unreserve_out:
2629	drm_exec_fini(exec: &exec);
2630	amdgpu_sync_wait(sync: &sync, intr: false);
2631	amdgpu_sync_free(sync: &sync);
2632
2633	return ret;
2634	}
2635
2636	/* Confirm that all user pages are valid while holding the notifier lock
2637	*
2638	* Moves valid BOs from the userptr_inval_list back to userptr_val_list.
2639	*/
2640	static int confirm_valid_user_pages_locked(struct amdkfd_process_info *process_info)
2641	{
2642	struct kgd_mem *mem, *tmp_mem;
2643	int ret = 0;
2644
2645	list_for_each_entry_safe(mem, tmp_mem,
2646	&process_info->userptr_inval_list,
2647	validate_list) {
2648	bool valid;
2649
2650	/* keep mem without hmm range at userptr_inval_list */
2651	if (!mem->range)
2652	continue;
2653
2654	/* Only check mem with hmm range associated */
2655	valid = amdgpu_ttm_tt_get_user_pages_done(
2656	ttm: mem->bo->tbo.ttm, range: mem->range);
2657
2658	mem->range = NULL;
2659	if (!valid) {
2660	WARN(!mem->invalid, "Invalid BO not marked invalid");
2661	ret = -EAGAIN;
2662	continue;
2663	}
2664
2665	if (mem->invalid) {
2666	WARN(1, "Valid BO is marked invalid");
2667	ret = -EAGAIN;
2668	continue;
2669	}
2670
2671	list_move_tail(list: &mem->validate_list,
2672	head: &process_info->userptr_valid_list);
2673	}
2674
2675	return ret;
2676	}
2677
2678	/* Worker callback to restore evicted userptr BOs
2679	*
2680	* Tries to update and validate all userptr BOs. If successful and no
2681	* concurrent evictions happened, the queues are restarted. Otherwise,
2682	* reschedule for another attempt later.
2683	*/
2684	static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)
2685	{
2686	struct delayed_work *dwork = to_delayed_work(work);
2687	struct amdkfd_process_info *process_info =
2688	container_of(dwork, struct amdkfd_process_info,
2689	restore_userptr_work);
2690	struct task_struct *usertask;
2691	struct mm_struct *mm;
2692	uint32_t evicted_bos;
2693
2694	mutex_lock(&process_info->notifier_lock);
2695	evicted_bos = process_info->evicted_bos;
2696	mutex_unlock(lock: &process_info->notifier_lock);
2697	if (!evicted_bos)
2698	return;
2699
2700	/* Reference task and mm in case of concurrent process termination */
2701	usertask = get_pid_task(pid: process_info->pid, PIDTYPE_PID);
2702	if (!usertask)
2703	return;
2704	mm = get_task_mm(task: usertask);
2705	if (!mm) {
2706	put_task_struct(t: usertask);
2707	return;
2708	}
2709
2710	mutex_lock(&process_info->lock);
2711
2712	if (update_invalid_user_pages(process_info, mm))
2713	goto unlock_out;
2714	/* userptr_inval_list can be empty if all evicted userptr BOs
2715	* have been freed. In that case there is nothing to validate
2716	* and we can just restart the queues.
2717	*/
2718	if (!list_empty(head: &process_info->userptr_inval_list)) {
2719	if (validate_invalid_user_pages(process_info))
2720	goto unlock_out;
2721	}
2722	/* Final check for concurrent evicton and atomic update. If
2723	* another eviction happens after successful update, it will
2724	* be a first eviction that calls quiesce_mm. The eviction
2725	* reference counting inside KFD will handle this case.
2726	*/
2727	mutex_lock(&process_info->notifier_lock);
2728	if (process_info->evicted_bos != evicted_bos)
2729	goto unlock_notifier_out;
2730
2731	if (confirm_valid_user_pages_locked(process_info)) {
2732	WARN(1, "User pages unexpectedly invalid");
2733	goto unlock_notifier_out;
2734	}
2735
2736	process_info->evicted_bos = evicted_bos = 0;
2737
2738	if (kgd2kfd_resume_mm(mm)) {
2739	pr_err("%s: Failed to resume KFD\n", __func__);
2740	/* No recovery from this failure. Probably the CP is
2741	* hanging. No point trying again.
2742	*/
2743	}
2744
2745	unlock_notifier_out:
2746	mutex_unlock(lock: &process_info->notifier_lock);
2747	unlock_out:
2748	mutex_unlock(lock: &process_info->lock);
2749
2750	/* If validation failed, reschedule another attempt */
2751	if (evicted_bos) {
2752	schedule_delayed_work(dwork: &process_info->restore_userptr_work,
2753	delay: msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
2754
2755	kfd_smi_event_queue_restore_rescheduled(mm);
2756	}
2757	mmput(mm);
2758	put_task_struct(t: usertask);
2759	}
2760
2761	/** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given
2762	* KFD process identified by process_info
2763	*
2764	* @process_info: amdkfd_process_info of the KFD process
2765	*
2766	* After memory eviction, restore thread calls this function. The function
2767	* should be called when the Process is still valid. BO restore involves -
2768	*
2769	* 1. Release old eviction fence and create new one
2770	* 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list.
2771	* 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of
2772	* BOs that need to be reserved.
2773	* 4. Reserve all the BOs
2774	* 5. Validate of PD and PT BOs.
2775	* 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence
2776	* 7. Add fence to all PD and PT BOs.
2777	* 8. Unreserve all BOs
2778	*/
2779	int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
2780	{
2781	struct amdkfd_process_info *process_info = info;
2782	struct amdgpu_vm *peer_vm;
2783	struct kgd_mem *mem;
2784	struct amdgpu_amdkfd_fence *new_fence;
2785	struct list_head duplicate_save;
2786	struct amdgpu_sync sync_obj;
2787	unsigned long failed_size = 0;
2788	unsigned long total_size = 0;
2789	struct drm_exec exec;
2790	int ret;
2791
2792	INIT_LIST_HEAD(list: &duplicate_save);
2793
2794	mutex_lock(&process_info->lock);
2795
2796	drm_exec_init(exec: &exec, flags: 0);
2797	drm_exec_until_all_locked(&exec) {
2798	list_for_each_entry(peer_vm, &process_info->vm_list_head,
2799	vm_list_node) {
2800	ret = amdgpu_vm_lock_pd(vm: peer_vm, exec: &exec, num_fences: 2);
2801	drm_exec_retry_on_contention(&exec);
2802	if (unlikely(ret))
2803	goto ttm_reserve_fail;
2804	}
2805
2806	/* Reserve all BOs and page tables/directory. Add all BOs from
2807	* kfd_bo_list to ctx.list
2808	*/
2809	list_for_each_entry(mem, &process_info->kfd_bo_list,
2810	validate_list) {
2811	struct drm_gem_object *gobj;
2812
2813	gobj = &mem->bo->tbo.base;
2814	ret = drm_exec_prepare_obj(exec: &exec, obj: gobj, num_fences: 1);
2815	drm_exec_retry_on_contention(&exec);
2816	if (unlikely(ret))
2817	goto ttm_reserve_fail;
2818	}
2819	}
2820
2821	amdgpu_sync_create(sync: &sync_obj);
2822
2823	/* Validate PDs and PTs */
2824	ret = process_validate_vms(process_info);
2825	if (ret)
2826	goto validate_map_fail;
2827
2828	ret = process_sync_pds_resv(process_info, sync: &sync_obj);
2829	if (ret) {
2830	pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n");
2831	goto validate_map_fail;
2832	}
2833
2834	/* Validate BOs and map them to GPUVM (update VM page tables). */
2835	list_for_each_entry(mem, &process_info->kfd_bo_list,
2836	validate_list) {
2837
2838	struct amdgpu_bo *bo = mem->bo;
2839	uint32_t domain = mem->domain;
2840	struct kfd_mem_attachment *attachment;
2841	struct dma_resv_iter cursor;
2842	struct dma_fence *fence;
2843
2844	total_size += amdgpu_bo_size(bo);
2845
2846	ret = amdgpu_amdkfd_bo_validate(bo, domain, wait: false);
2847	if (ret) {
2848	pr_debug("Memory eviction: Validate BOs failed\n");
2849	failed_size += amdgpu_bo_size(bo);
2850	ret = amdgpu_amdkfd_bo_validate(bo,
2851	AMDGPU_GEM_DOMAIN_GTT, wait: false);
2852	if (ret) {
2853	pr_debug("Memory eviction: Try again\n");
2854	goto validate_map_fail;
2855	}
2856	}
2857	dma_resv_for_each_fence(&cursor, bo->tbo.base.resv,
2858	DMA_RESV_USAGE_KERNEL, fence) {
2859	ret = amdgpu_sync_fence(sync: &sync_obj, f: fence);
2860	if (ret) {
2861	pr_debug("Memory eviction: Sync BO fence failed. Try again\n");
2862	goto validate_map_fail;
2863	}
2864	}
2865	list_for_each_entry(attachment, &mem->attachments, list) {
2866	if (!attachment->is_mapped)
2867	continue;
2868
2869	if (attachment->bo_va->base.bo->tbo.pin_count)
2870	continue;
2871
2872	kfd_mem_dmaunmap_attachment(mem, attachment);
2873	ret = update_gpuvm_pte(mem, entry: attachment, sync: &sync_obj);
2874	if (ret) {
2875	pr_debug("Memory eviction: update PTE failed. Try again\n");
2876	goto validate_map_fail;
2877	}
2878	}
2879	}
2880
2881	if (failed_size)
2882	pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size);
2883
2884	/* Update page directories */
2885	ret = process_update_pds(process_info, sync: &sync_obj);
2886	if (ret) {
2887	pr_debug("Memory eviction: update PDs failed. Try again\n");
2888	goto validate_map_fail;
2889	}
2890
2891	/* Wait for validate and PT updates to finish */
2892	amdgpu_sync_wait(sync: &sync_obj, intr: false);
2893
2894	/* Release old eviction fence and create new one, because fence only
2895	* goes from unsignaled to signaled, fence cannot be reused.
2896	* Use context and mm from the old fence.
2897	*/
2898	new_fence = amdgpu_amdkfd_fence_create(
2899	context: process_info->eviction_fence->base.context,
2900	mm: process_info->eviction_fence->mm,
2901	NULL);
2902	if (!new_fence) {
2903	pr_err("Failed to create eviction fence\n");
2904	ret = -ENOMEM;
2905	goto validate_map_fail;
2906	}
2907	dma_fence_put(fence: &process_info->eviction_fence->base);
2908	process_info->eviction_fence = new_fence;
2909	*ef = dma_fence_get(fence: &new_fence->base);
2910
2911	/* Attach new eviction fence to all BOs except pinned ones */
2912	list_for_each_entry(mem, &process_info->kfd_bo_list, validate_list) {
2913	if (mem->bo->tbo.pin_count)
2914	continue;
2915
2916	dma_resv_add_fence(obj: mem->bo->tbo.base.resv,
2917	fence: &process_info->eviction_fence->base,
2918	usage: DMA_RESV_USAGE_BOOKKEEP);
2919	}
2920	/* Attach eviction fence to PD / PT BOs */
2921	list_for_each_entry(peer_vm, &process_info->vm_list_head,
2922	vm_list_node) {
2923	struct amdgpu_bo *bo = peer_vm->root.bo;
2924
2925	dma_resv_add_fence(obj: bo->tbo.base.resv,
2926	fence: &process_info->eviction_fence->base,
2927	usage: DMA_RESV_USAGE_BOOKKEEP);
2928	}
2929
2930	validate_map_fail:
2931	amdgpu_sync_free(sync: &sync_obj);
2932	ttm_reserve_fail:
2933	drm_exec_fini(exec: &exec);
2934	mutex_unlock(lock: &process_info->lock);
2935	return ret;
2936	}
2937
2938	int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem)
2939	{
2940	struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
2941	struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws;
2942	int ret;
2943
2944	if (!info || !gws)
2945	return -EINVAL;
2946
2947	*mem = kzalloc(size: sizeof(struct kgd_mem), GFP_KERNEL);
2948	if (!*mem)
2949	return -ENOMEM;
2950
2951	mutex_init(&(*mem)->lock);
2952	INIT_LIST_HEAD(list: &(*mem)->attachments);
2953	(*mem)->bo = amdgpu_bo_ref(bo: gws_bo);
2954	(*mem)->domain = AMDGPU_GEM_DOMAIN_GWS;
2955	(*mem)->process_info = process_info;
2956	add_kgd_mem_to_kfd_bo_list(mem: *mem, process_info, userptr: false);
2957	amdgpu_sync_create(sync: &(*mem)->sync);
2958
2959
2960	/* Validate gws bo the first time it is added to process */
2961	mutex_lock(&(*mem)->process_info->lock);
2962	ret = amdgpu_bo_reserve(bo: gws_bo, no_intr: false);
2963	if (unlikely(ret)) {
2964	pr_err("Reserve gws bo failed %d\n", ret);
2965	goto bo_reservation_failure;
2966	}
2967
2968	ret = amdgpu_amdkfd_bo_validate(bo: gws_bo, AMDGPU_GEM_DOMAIN_GWS, wait: true);
2969	if (ret) {
2970	pr_err("GWS BO validate failed %d\n", ret);
2971	goto bo_validation_failure;
2972	}
2973	/* GWS resource is shared b/t amdgpu and amdkfd
2974	* Add process eviction fence to bo so they can
2975	* evict each other.
2976	*/
2977	ret = dma_resv_reserve_fences(obj: gws_bo->tbo.base.resv, num_fences: 1);
2978	if (ret)
2979	goto reserve_shared_fail;
2980	dma_resv_add_fence(obj: gws_bo->tbo.base.resv,
2981	fence: &process_info->eviction_fence->base,
2982	usage: DMA_RESV_USAGE_BOOKKEEP);
2983	amdgpu_bo_unreserve(bo: gws_bo);
2984	mutex_unlock(lock: &(*mem)->process_info->lock);
2985
2986	return ret;
2987
2988	reserve_shared_fail:
2989	bo_validation_failure:
2990	amdgpu_bo_unreserve(bo: gws_bo);
2991	bo_reservation_failure:
2992	mutex_unlock(lock: &(*mem)->process_info->lock);
2993	amdgpu_sync_free(sync: &(*mem)->sync);
2994	remove_kgd_mem_from_kfd_bo_list(mem: *mem, process_info);
2995	amdgpu_bo_unref(bo: &gws_bo);
2996	mutex_destroy(lock: &(*mem)->lock);
2997	kfree(objp: *mem);
2998	*mem = NULL;
2999	return ret;
3000	}
3001
3002	int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem)
3003	{
3004	int ret;
3005	struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
3006	struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
3007	struct amdgpu_bo *gws_bo = kgd_mem->bo;
3008
3009	/* Remove BO from process's validate list so restore worker won't touch
3010	* it anymore
3011	*/
3012	remove_kgd_mem_from_kfd_bo_list(mem: kgd_mem, process_info);
3013
3014	ret = amdgpu_bo_reserve(bo: gws_bo, no_intr: false);
3015	if (unlikely(ret)) {
3016	pr_err("Reserve gws bo failed %d\n", ret);
3017	//TODO add BO back to validate_list?
3018	return ret;
3019	}
3020	amdgpu_amdkfd_remove_eviction_fence(bo: gws_bo,
3021	ef: process_info->eviction_fence);
3022	amdgpu_bo_unreserve(bo: gws_bo);
3023	amdgpu_sync_free(sync: &kgd_mem->sync);
3024	amdgpu_bo_unref(bo: &gws_bo);
3025	mutex_destroy(lock: &kgd_mem->lock);
3026	kfree(objp: mem);
3027	return 0;
3028	}
3029
3030	/* Returns GPU-specific tiling mode information */
3031	int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,
3032	struct tile_config *config)
3033	{
3034	config->gb_addr_config = adev->gfx.config.gb_addr_config;
3035	config->tile_config_ptr = adev->gfx.config.tile_mode_array;
3036	config->num_tile_configs =
3037	ARRAY_SIZE(adev->gfx.config.tile_mode_array);
3038	config->macro_tile_config_ptr =
3039	adev->gfx.config.macrotile_mode_array;
3040	config->num_macro_tile_configs =
3041	ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
3042
3043	/* Those values are not set from GFX9 onwards */
3044	config->num_banks = adev->gfx.config.num_banks;
3045	config->num_ranks = adev->gfx.config.num_ranks;
3046
3047	return 0;
3048	}
3049
3050	bool amdgpu_amdkfd_bo_mapped_to_dev(struct amdgpu_device *adev, struct kgd_mem *mem)
3051	{
3052	struct kfd_mem_attachment *entry;
3053
3054	list_for_each_entry(entry, &mem->attachments, list) {
3055	if (entry->is_mapped && entry->adev == adev)
3056	return true;
3057	}
3058	return false;
3059	}
3060
3061	#if defined(CONFIG_DEBUG_FS)
3062
3063	int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data)
3064	{
3065
3066	spin_lock(lock: &kfd_mem_limit.mem_limit_lock);
3067	seq_printf(m, fmt: "System mem used %lldM out of %lluM\n",
3068	(kfd_mem_limit.system_mem_used >> 20),
3069	(kfd_mem_limit.max_system_mem_limit >> 20));
3070	seq_printf(m, fmt: "TTM mem used %lldM out of %lluM\n",
3071	(kfd_mem_limit.ttm_mem_used >> 20),
3072	(kfd_mem_limit.max_ttm_mem_limit >> 20));
3073	spin_unlock(lock: &kfd_mem_limit.mem_limit_lock);
3074
3075	return 0;
3076	}
3077
3078	#endif
3079