1	// SPDX-License-Identifier: GPL-2.0 OR MIT
2	/*
3	* Copyright 2014-2022 Advanced Micro Devices, Inc.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*/
23
24	#include <linux/device.h>
25	#include <linux/export.h>
26	#include <linux/err.h>
27	#include <linux/fs.h>
28	#include <linux/file.h>
29	#include <linux/sched.h>
30	#include <linux/slab.h>
31	#include <linux/uaccess.h>
32	#include <linux/compat.h>
33	#include <uapi/linux/kfd_ioctl.h>
34	#include <linux/time.h>
35	#include <linux/mm.h>
36	#include <linux/mman.h>
37	#include <linux/ptrace.h>
38	#include <linux/dma-buf.h>
39	#include <linux/fdtable.h>
40	#include <linux/processor.h>
41	#include "kfd_priv.h"
42	#include "kfd_device_queue_manager.h"
43	#include "kfd_svm.h"
44	#include "amdgpu_amdkfd.h"
45	#include "kfd_smi_events.h"
46	#include "amdgpu_dma_buf.h"
47	#include "kfd_debug.h"
48
49	static long kfd_ioctl(struct file *, unsigned int, unsigned long);
50	static int kfd_open(struct inode *, struct file *);
51	static int kfd_release(struct inode *, struct file *);
52	static int kfd_mmap(struct file *, struct vm_area_struct *);
53
54	static const char kfd_dev_name[] = "kfd";
55
56	static const struct file_operations kfd_fops = {
57	.owner = THIS_MODULE,
58	.unlocked_ioctl = kfd_ioctl,
59	.compat_ioctl = compat_ptr_ioctl,
60	.open = kfd_open,
61	.release = kfd_release,
62	.mmap = kfd_mmap,
63	};
64
65	static int kfd_char_dev_major = -1;
66	static struct class *kfd_class;
67	struct device *kfd_device;
68
69	static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id)
70	{
71	struct kfd_process_device *pdd;
72
73	mutex_lock(&p->mutex);
74	pdd = kfd_process_device_data_by_id(process: p, gpu_id);
75
76	if (pdd)
77	return pdd;
78
79	mutex_unlock(lock: &p->mutex);
80	return NULL;
81	}
82
83	static inline void kfd_unlock_pdd(struct kfd_process_device *pdd)
84	{
85	mutex_unlock(lock: &pdd->process->mutex);
86	}
87
88	int kfd_chardev_init(void)
89	{
90	int err = 0;
91
92	kfd_char_dev_major = register_chrdev(major: 0, name: kfd_dev_name, fops: &kfd_fops);
93	err = kfd_char_dev_major;
94	if (err < 0)
95	goto err_register_chrdev;
96
97	kfd_class = class_create(name: kfd_dev_name);
98	err = PTR_ERR(ptr: kfd_class);
99	if (IS_ERR(ptr: kfd_class))
100	goto err_class_create;
101
102	kfd_device = device_create(cls: kfd_class, NULL,
103	MKDEV(kfd_char_dev_major, 0),
104	NULL, fmt: kfd_dev_name);
105	err = PTR_ERR(ptr: kfd_device);
106	if (IS_ERR(ptr: kfd_device))
107	goto err_device_create;
108
109	return 0;
110
111	err_device_create:
112	class_destroy(cls: kfd_class);
113	err_class_create:
114	unregister_chrdev(major: kfd_char_dev_major, name: kfd_dev_name);
115	err_register_chrdev:
116	return err;
117	}
118
119	void kfd_chardev_exit(void)
120	{
121	device_destroy(cls: kfd_class, MKDEV(kfd_char_dev_major, 0));
122	class_destroy(cls: kfd_class);
123	unregister_chrdev(major: kfd_char_dev_major, name: kfd_dev_name);
124	kfd_device = NULL;
125	}
126
127
128	static int kfd_open(struct inode *inode, struct file *filep)
129	{
130	struct kfd_process *process;
131	bool is_32bit_user_mode;
132
133	if (iminor(inode) != 0)
134	return -ENODEV;
135
136	is_32bit_user_mode = in_compat_syscall();
137
138	if (is_32bit_user_mode) {
139	dev_warn(kfd_device,
140	"Process %d (32-bit) failed to open /dev/kfd\n"
141	"32-bit processes are not supported by amdkfd\n",
142	current->pid);
143	return -EPERM;
144	}
145
146	process = kfd_create_process(current);
147	if (IS_ERR(ptr: process))
148	return PTR_ERR(ptr: process);
149
150	if (kfd_process_init_cwsr_apu(process, filep)) {
151	kfd_unref_process(p: process);
152	return -EFAULT;
153	}
154
155	/* filep now owns the reference returned by kfd_create_process */
156	filep->private_data = process;
157
158	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
159	process->pasid, process->is_32bit_user_mode);
160
161	return 0;
162	}
163
164	static int kfd_release(struct inode *inode, struct file *filep)
165	{
166	struct kfd_process *process = filep->private_data;
167
168	if (process)
169	kfd_unref_process(p: process);
170
171	return 0;
172	}
173
174	static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
175	void *data)
176	{
177	struct kfd_ioctl_get_version_args *args = data;
178
179	args->major_version = KFD_IOCTL_MAJOR_VERSION;
180	args->minor_version = KFD_IOCTL_MINOR_VERSION;
181
182	return 0;
183	}
184
185	static int set_queue_properties_from_user(struct queue_properties *q_properties,
186	struct kfd_ioctl_create_queue_args *args)
187	{
188	/*
189	* Repurpose queue percentage to accommodate new features:
190	* bit 0-7: queue percentage
191	* bit 8-15: pm4_target_xcc
192	*/
193	if ((args->queue_percentage & 0xFF) > KFD_MAX_QUEUE_PERCENTAGE) {
194	pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
195	return -EINVAL;
196	}
197
198	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
199	pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
200	return -EINVAL;
201	}
202
203	if ((args->ring_base_address) &&
204	(!access_ok((const void __user *) args->ring_base_address,
205	sizeof(uint64_t)))) {
206	pr_err("Can't access ring base address\n");
207	return -EFAULT;
208	}
209
210	if (!is_power_of_2(n: args->ring_size) && (args->ring_size != 0)) {
211	pr_err("Ring size must be a power of 2 or 0\n");
212	return -EINVAL;
213	}
214
215	if (!access_ok((const void __user *) args->read_pointer_address,
216	sizeof(uint32_t))) {
217	pr_err("Can't access read pointer\n");
218	return -EFAULT;
219	}
220
221	if (!access_ok((const void __user *) args->write_pointer_address,
222	sizeof(uint32_t))) {
223	pr_err("Can't access write pointer\n");
224	return -EFAULT;
225	}
226
227	if (args->eop_buffer_address &&
228	!access_ok((const void __user *) args->eop_buffer_address,
229	sizeof(uint32_t))) {
230	pr_debug("Can't access eop buffer");
231	return -EFAULT;
232	}
233
234	if (args->ctx_save_restore_address &&
235	!access_ok((const void __user *) args->ctx_save_restore_address,
236	sizeof(uint32_t))) {
237	pr_debug("Can't access ctx save restore buffer");
238	return -EFAULT;
239	}
240
241	q_properties->is_interop = false;
242	q_properties->is_gws = false;
243	q_properties->queue_percent = args->queue_percentage & 0xFF;
244	/* bit 8-15 are repurposed to be PM4 target XCC */
245	q_properties->pm4_target_xcc = (args->queue_percentage >> 8) & 0xFF;
246	q_properties->priority = args->queue_priority;
247	q_properties->queue_address = args->ring_base_address;
248	q_properties->queue_size = args->ring_size;
249	q_properties->read_ptr = (uint32_t *) args->read_pointer_address;
250	q_properties->write_ptr = (uint32_t *) args->write_pointer_address;
251	q_properties->eop_ring_buffer_address = args->eop_buffer_address;
252	q_properties->eop_ring_buffer_size = args->eop_buffer_size;
253	q_properties->ctx_save_restore_area_address =
254	args->ctx_save_restore_address;
255	q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
256	q_properties->ctl_stack_size = args->ctl_stack_size;
257	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
258	args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
259	q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
260	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
261	q_properties->type = KFD_QUEUE_TYPE_SDMA;
262	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
263	q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
264	else
265	return -ENOTSUPP;
266
267	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
268	q_properties->format = KFD_QUEUE_FORMAT_AQL;
269	else
270	q_properties->format = KFD_QUEUE_FORMAT_PM4;
271
272	pr_debug("Queue Percentage: %d, %d\n",
273	q_properties->queue_percent, args->queue_percentage);
274
275	pr_debug("Queue Priority: %d, %d\n",
276	q_properties->priority, args->queue_priority);
277
278	pr_debug("Queue Address: 0x%llX, 0x%llX\n",
279	q_properties->queue_address, args->ring_base_address);
280
281	pr_debug("Queue Size: 0x%llX, %u\n",
282	q_properties->queue_size, args->ring_size);
283
284	pr_debug("Queue r/w Pointers: %px, %px\n",
285	q_properties->read_ptr,
286	q_properties->write_ptr);
287
288	pr_debug("Queue Format: %d\n", q_properties->format);
289
290	pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
291
292	pr_debug("Queue CTX save area: 0x%llX\n",
293	q_properties->ctx_save_restore_area_address);
294
295	return 0;
296	}
297
298	static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
299	void *data)
300	{
301	struct kfd_ioctl_create_queue_args *args = data;
302	struct kfd_node *dev;
303	int err = 0;
304	unsigned int queue_id;
305	struct kfd_process_device *pdd;
306	struct queue_properties q_properties;
307	uint32_t doorbell_offset_in_process = 0;
308	struct amdgpu_bo *wptr_bo = NULL;
309
310	memset(&q_properties, 0, sizeof(struct queue_properties));
311
312	pr_debug("Creating queue ioctl\n");
313
314	err = set_queue_properties_from_user(q_properties: &q_properties, args);
315	if (err)
316	return err;
317
318	pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
319
320	mutex_lock(&p->mutex);
321
322	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
323	if (!pdd) {
324	pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
325	err = -EINVAL;
326	goto err_pdd;
327	}
328	dev = pdd->dev;
329
330	pdd = kfd_bind_process_to_device(dev, p);
331	if (IS_ERR(ptr: pdd)) {
332	err = -ESRCH;
333	goto err_bind_process;
334	}
335
336	if (!pdd->qpd.proc_doorbells) {
337	err = kfd_alloc_process_doorbells(kfd: dev->kfd, pdd);
338	if (err) {
339	pr_debug("failed to allocate process doorbells\n");
340	goto err_bind_process;
341	}
342	}
343
344	/* Starting with GFX11, wptr BOs must be mapped to GART for MES to determine work
345	* on unmapped queues for usermode queue oversubscription (no aggregated doorbell)
346	*/
347	if (dev->kfd->shared_resources.enable_mes &&
348	((dev->adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK)
349	>> AMDGPU_MES_API_VERSION_SHIFT) >= 2) {
350	struct amdgpu_bo_va_mapping *wptr_mapping;
351	struct amdgpu_vm *wptr_vm;
352
353	wptr_vm = drm_priv_to_vm(pdd->drm_priv);
354	err = amdgpu_bo_reserve(bo: wptr_vm->root.bo, no_intr: false);
355	if (err)
356	goto err_wptr_map_gart;
357
358	wptr_mapping = amdgpu_vm_bo_lookup_mapping(
359	vm: wptr_vm, addr: args->write_pointer_address >> PAGE_SHIFT);
360	amdgpu_bo_unreserve(bo: wptr_vm->root.bo);
361	if (!wptr_mapping) {
362	pr_err("Failed to lookup wptr bo\n");
363	err = -EINVAL;
364	goto err_wptr_map_gart;
365	}
366
367	wptr_bo = wptr_mapping->bo_va->base.bo;
368	if (wptr_bo->tbo.base.size > PAGE_SIZE) {
369	pr_err("Requested GART mapping for wptr bo larger than one page\n");
370	err = -EINVAL;
371	goto err_wptr_map_gart;
372	}
373
374	err = amdgpu_amdkfd_map_gtt_bo_to_gart(adev: dev->adev, bo: wptr_bo);
375	if (err) {
376	pr_err("Failed to map wptr bo to GART\n");
377	goto err_wptr_map_gart;
378	}
379	}
380
381	pr_debug("Creating queue for PASID 0x%x on gpu 0x%x\n",
382	p->pasid,
383	dev->id);
384
385	err = pqm_create_queue(pqm: &p->pqm, dev, f: filep, properties: &q_properties, qid: &queue_id, wptr_bo,
386	NULL, NULL, NULL, p_doorbell_offset_in_process: &doorbell_offset_in_process);
387	if (err != 0)
388	goto err_create_queue;
389
390	args->queue_id = queue_id;
391
392
393	/* Return gpu_id as doorbell offset for mmap usage */
394	args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
395	args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
396	if (KFD_IS_SOC15(dev))
397	/* On SOC15 ASICs, include the doorbell offset within the
398	* process doorbell frame, which is 2 pages.
399	*/
400	args->doorbell_offset |= doorbell_offset_in_process;
401
402	mutex_unlock(lock: &p->mutex);
403
404	pr_debug("Queue id %d was created successfully\n", args->queue_id);
405
406	pr_debug("Ring buffer address == 0x%016llX\n",
407	args->ring_base_address);
408
409	pr_debug("Read ptr address == 0x%016llX\n",
410	args->read_pointer_address);
411
412	pr_debug("Write ptr address == 0x%016llX\n",
413	args->write_pointer_address);
414
415	kfd_dbg_ev_raise(KFD_EC_MASK(EC_QUEUE_NEW), process: p, dev, source_id: queue_id, use_worker: false, NULL, exception_data_size: 0);
416	return 0;
417
418	err_create_queue:
419	if (wptr_bo)
420	amdgpu_amdkfd_free_gtt_mem(adev: dev->adev, mem_obj: wptr_bo);
421	err_wptr_map_gart:
422	err_bind_process:
423	err_pdd:
424	mutex_unlock(lock: &p->mutex);
425	return err;
426	}
427
428	static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
429	void *data)
430	{
431	int retval;
432	struct kfd_ioctl_destroy_queue_args *args = data;
433
434	pr_debug("Destroying queue id %d for pasid 0x%x\n",
435	args->queue_id,
436	p->pasid);
437
438	mutex_lock(&p->mutex);
439
440	retval = pqm_destroy_queue(pqm: &p->pqm, qid: args->queue_id);
441
442	mutex_unlock(lock: &p->mutex);
443	return retval;
444	}
445
446	static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
447	void *data)
448	{
449	int retval;
450	struct kfd_ioctl_update_queue_args *args = data;
451	struct queue_properties properties;
452
453	/*
454	* Repurpose queue percentage to accommodate new features:
455	* bit 0-7: queue percentage
456	* bit 8-15: pm4_target_xcc
457	*/
458	if ((args->queue_percentage & 0xFF) > KFD_MAX_QUEUE_PERCENTAGE) {
459	pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
460	return -EINVAL;
461	}
462
463	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
464	pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
465	return -EINVAL;
466	}
467
468	if ((args->ring_base_address) &&
469	(!access_ok((const void __user *) args->ring_base_address,
470	sizeof(uint64_t)))) {
471	pr_err("Can't access ring base address\n");
472	return -EFAULT;
473	}
474
475	if (!is_power_of_2(n: args->ring_size) && (args->ring_size != 0)) {
476	pr_err("Ring size must be a power of 2 or 0\n");
477	return -EINVAL;
478	}
479
480	properties.queue_address = args->ring_base_address;
481	properties.queue_size = args->ring_size;
482	properties.queue_percent = args->queue_percentage & 0xFF;
483	/* bit 8-15 are repurposed to be PM4 target XCC */
484	properties.pm4_target_xcc = (args->queue_percentage >> 8) & 0xFF;
485	properties.priority = args->queue_priority;
486
487	pr_debug("Updating queue id %d for pasid 0x%x\n",
488	args->queue_id, p->pasid);
489
490	mutex_lock(&p->mutex);
491
492	retval = pqm_update_queue_properties(pqm: &p->pqm, qid: args->queue_id, p: &properties);
493
494	mutex_unlock(lock: &p->mutex);
495
496	return retval;
497	}
498
499	static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
500	void *data)
501	{
502	int retval;
503	const int max_num_cus = 1024;
504	struct kfd_ioctl_set_cu_mask_args *args = data;
505	struct mqd_update_info minfo = {0};
506	uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
507	size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
508
509	if ((args->num_cu_mask % 32) != 0) {
510	pr_debug("num_cu_mask 0x%x must be a multiple of 32",
511	args->num_cu_mask);
512	return -EINVAL;
513	}
514
515	minfo.cu_mask.count = args->num_cu_mask;
516	if (minfo.cu_mask.count == 0) {
517	pr_debug("CU mask cannot be 0");
518	return -EINVAL;
519	}
520
521	/* To prevent an unreasonably large CU mask size, set an arbitrary
522	* limit of max_num_cus bits. We can then just drop any CU mask bits
523	* past max_num_cus bits and just use the first max_num_cus bits.
524	*/
525	if (minfo.cu_mask.count > max_num_cus) {
526	pr_debug("CU mask cannot be greater than 1024 bits");
527	minfo.cu_mask.count = max_num_cus;
528	cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
529	}
530
531	minfo.cu_mask.ptr = kzalloc(size: cu_mask_size, GFP_KERNEL);
532	if (!minfo.cu_mask.ptr)
533	return -ENOMEM;
534
535	retval = copy_from_user(to: minfo.cu_mask.ptr, from: cu_mask_ptr, n: cu_mask_size);
536	if (retval) {
537	pr_debug("Could not copy CU mask from userspace");
538	retval = -EFAULT;
539	goto out;
540	}
541
542	mutex_lock(&p->mutex);
543
544	retval = pqm_update_mqd(pqm: &p->pqm, qid: args->queue_id, minfo: &minfo);
545
546	mutex_unlock(lock: &p->mutex);
547
548	out:
549	kfree(objp: minfo.cu_mask.ptr);
550	return retval;
551	}
552
553	static int kfd_ioctl_get_queue_wave_state(struct file *filep,
554	struct kfd_process *p, void *data)
555	{
556	struct kfd_ioctl_get_queue_wave_state_args *args = data;
557	int r;
558
559	mutex_lock(&p->mutex);
560
561	r = pqm_get_wave_state(pqm: &p->pqm, qid: args->queue_id,
562	ctl_stack: (void __user *)args->ctl_stack_address,
563	ctl_stack_used_size: &args->ctl_stack_used_size,
564	save_area_used_size: &args->save_area_used_size);
565
566	mutex_unlock(lock: &p->mutex);
567
568	return r;
569	}
570
571	static int kfd_ioctl_set_memory_policy(struct file *filep,
572	struct kfd_process *p, void *data)
573	{
574	struct kfd_ioctl_set_memory_policy_args *args = data;
575	int err = 0;
576	struct kfd_process_device *pdd;
577	enum cache_policy default_policy, alternate_policy;
578
579	if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
580	&& args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
581	return -EINVAL;
582	}
583
584	if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
585	&& args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
586	return -EINVAL;
587	}
588
589	mutex_lock(&p->mutex);
590	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
591	if (!pdd) {
592	pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
593	err = -EINVAL;
594	goto err_pdd;
595	}
596
597	pdd = kfd_bind_process_to_device(dev: pdd->dev, p);
598	if (IS_ERR(ptr: pdd)) {
599	err = -ESRCH;
600	goto out;
601	}
602
603	default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
604	? cache_policy_coherent : cache_policy_noncoherent;
605
606	alternate_policy =
607	(args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
608	? cache_policy_coherent : cache_policy_noncoherent;
609
610	if (!pdd->dev->dqm->ops.set_cache_memory_policy(pdd->dev->dqm,
611	&pdd->qpd,
612	default_policy,
613	alternate_policy,
614	(void __user *)args->alternate_aperture_base,
615	args->alternate_aperture_size))
616	err = -EINVAL;
617
618	out:
619	err_pdd:
620	mutex_unlock(lock: &p->mutex);
621
622	return err;
623	}
624
625	static int kfd_ioctl_set_trap_handler(struct file *filep,
626	struct kfd_process *p, void *data)
627	{
628	struct kfd_ioctl_set_trap_handler_args *args = data;
629	int err = 0;
630	struct kfd_process_device *pdd;
631
632	mutex_lock(&p->mutex);
633
634	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
635	if (!pdd) {
636	err = -EINVAL;
637	goto err_pdd;
638	}
639
640	pdd = kfd_bind_process_to_device(dev: pdd->dev, p);
641	if (IS_ERR(ptr: pdd)) {
642	err = -ESRCH;
643	goto out;
644	}
645
646	kfd_process_set_trap_handler(qpd: &pdd->qpd, tba_addr: args->tba_addr, tma_addr: args->tma_addr);
647
648	out:
649	err_pdd:
650	mutex_unlock(lock: &p->mutex);
651
652	return err;
653	}
654
655	static int kfd_ioctl_dbg_register(struct file *filep,
656	struct kfd_process *p, void *data)
657	{
658	return -EPERM;
659	}
660
661	static int kfd_ioctl_dbg_unregister(struct file *filep,
662	struct kfd_process *p, void *data)
663	{
664	return -EPERM;
665	}
666
667	static int kfd_ioctl_dbg_address_watch(struct file *filep,
668	struct kfd_process *p, void *data)
669	{
670	return -EPERM;
671	}
672
673	/* Parse and generate fixed size data structure for wave control */
674	static int kfd_ioctl_dbg_wave_control(struct file *filep,
675	struct kfd_process *p, void *data)
676	{
677	return -EPERM;
678	}
679
680	static int kfd_ioctl_get_clock_counters(struct file *filep,
681	struct kfd_process *p, void *data)
682	{
683	struct kfd_ioctl_get_clock_counters_args *args = data;
684	struct kfd_process_device *pdd;
685
686	mutex_lock(&p->mutex);
687	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
688	mutex_unlock(lock: &p->mutex);
689	if (pdd)
690	/* Reading GPU clock counter from KGD */
691	args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(adev: pdd->dev->adev);
692	else
693	/* Node without GPU resource */
694	args->gpu_clock_counter = 0;
695
696	/* No access to rdtsc. Using raw monotonic time */
697	args->cpu_clock_counter = ktime_get_raw_ns();
698	args->system_clock_counter = ktime_get_boottime_ns();
699
700	/* Since the counter is in nano-seconds we use 1GHz frequency */
701	args->system_clock_freq = 1000000000;
702
703	return 0;
704	}
705
706
707	static int kfd_ioctl_get_process_apertures(struct file *filp,
708	struct kfd_process *p, void *data)
709	{
710	struct kfd_ioctl_get_process_apertures_args *args = data;
711	struct kfd_process_device_apertures *pAperture;
712	int i;
713
714	dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
715
716	args->num_of_nodes = 0;
717
718	mutex_lock(&p->mutex);
719	/* Run over all pdd of the process */
720	for (i = 0; i < p->n_pdds; i++) {
721	struct kfd_process_device *pdd = p->pdds[i];
722
723	pAperture =
724	&args->process_apertures[args->num_of_nodes];
725	pAperture->gpu_id = pdd->dev->id;
726	pAperture->lds_base = pdd->lds_base;
727	pAperture->lds_limit = pdd->lds_limit;
728	pAperture->gpuvm_base = pdd->gpuvm_base;
729	pAperture->gpuvm_limit = pdd->gpuvm_limit;
730	pAperture->scratch_base = pdd->scratch_base;
731	pAperture->scratch_limit = pdd->scratch_limit;
732
733	dev_dbg(kfd_device,
734	"node id %u\n", args->num_of_nodes);
735	dev_dbg(kfd_device,
736	"gpu id %u\n", pdd->dev->id);
737	dev_dbg(kfd_device,
738	"lds_base %llX\n", pdd->lds_base);
739	dev_dbg(kfd_device,
740	"lds_limit %llX\n", pdd->lds_limit);
741	dev_dbg(kfd_device,
742	"gpuvm_base %llX\n", pdd->gpuvm_base);
743	dev_dbg(kfd_device,
744	"gpuvm_limit %llX\n", pdd->gpuvm_limit);
745	dev_dbg(kfd_device,
746	"scratch_base %llX\n", pdd->scratch_base);
747	dev_dbg(kfd_device,
748	"scratch_limit %llX\n", pdd->scratch_limit);
749
750	if (++args->num_of_nodes >= NUM_OF_SUPPORTED_GPUS)
751	break;
752	}
753	mutex_unlock(lock: &p->mutex);
754
755	return 0;
756	}
757
758	static int kfd_ioctl_get_process_apertures_new(struct file *filp,
759	struct kfd_process *p, void *data)
760	{
761	struct kfd_ioctl_get_process_apertures_new_args *args = data;
762	struct kfd_process_device_apertures *pa;
763	int ret;
764	int i;
765
766	dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
767
768	if (args->num_of_nodes == 0) {
769	/* Return number of nodes, so that user space can alloacate
770	* sufficient memory
771	*/
772	mutex_lock(&p->mutex);
773	args->num_of_nodes = p->n_pdds;
774	goto out_unlock;
775	}
776
777	/* Fill in process-aperture information for all available
778	* nodes, but not more than args->num_of_nodes as that is
779	* the amount of memory allocated by user
780	*/
781	pa = kzalloc(size: (sizeof(struct kfd_process_device_apertures) *
782	args->num_of_nodes), GFP_KERNEL);
783	if (!pa)
784	return -ENOMEM;
785
786	mutex_lock(&p->mutex);
787
788	if (!p->n_pdds) {
789	args->num_of_nodes = 0;
790	kfree(objp: pa);
791	goto out_unlock;
792	}
793
794	/* Run over all pdd of the process */
795	for (i = 0; i < min(p->n_pdds, args->num_of_nodes); i++) {
796	struct kfd_process_device *pdd = p->pdds[i];
797
798	pa[i].gpu_id = pdd->dev->id;
799	pa[i].lds_base = pdd->lds_base;
800	pa[i].lds_limit = pdd->lds_limit;
801	pa[i].gpuvm_base = pdd->gpuvm_base;
802	pa[i].gpuvm_limit = pdd->gpuvm_limit;
803	pa[i].scratch_base = pdd->scratch_base;
804	pa[i].scratch_limit = pdd->scratch_limit;
805
806	dev_dbg(kfd_device,
807	"gpu id %u\n", pdd->dev->id);
808	dev_dbg(kfd_device,
809	"lds_base %llX\n", pdd->lds_base);
810	dev_dbg(kfd_device,
811	"lds_limit %llX\n", pdd->lds_limit);
812	dev_dbg(kfd_device,
813	"gpuvm_base %llX\n", pdd->gpuvm_base);
814	dev_dbg(kfd_device,
815	"gpuvm_limit %llX\n", pdd->gpuvm_limit);
816	dev_dbg(kfd_device,
817	"scratch_base %llX\n", pdd->scratch_base);
818	dev_dbg(kfd_device,
819	"scratch_limit %llX\n", pdd->scratch_limit);
820	}
821	mutex_unlock(lock: &p->mutex);
822
823	args->num_of_nodes = i;
824	ret = copy_to_user(
825	to: (void __user *)args->kfd_process_device_apertures_ptr,
826	from: pa,
827	n: (i * sizeof(struct kfd_process_device_apertures)));
828	kfree(objp: pa);
829	return ret ? -EFAULT : 0;
830
831	out_unlock:
832	mutex_unlock(lock: &p->mutex);
833	return 0;
834	}
835
836	static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
837	void *data)
838	{
839	struct kfd_ioctl_create_event_args *args = data;
840	int err;
841
842	/* For dGPUs the event page is allocated in user mode. The
843	* handle is passed to KFD with the first call to this IOCTL
844	* through the event_page_offset field.
845	*/
846	if (args->event_page_offset) {
847	mutex_lock(&p->mutex);
848	err = kfd_kmap_event_page(p, event_page_offset: args->event_page_offset);
849	mutex_unlock(lock: &p->mutex);
850	if (err)
851	return err;
852	}
853
854	err = kfd_event_create(devkfd: filp, p, event_type: args->event_type,
855	auto_reset: args->auto_reset != 0, node_id: args->node_id,
856	event_id: &args->event_id, event_trigger_data: &args->event_trigger_data,
857	event_page_offset: &args->event_page_offset,
858	event_slot_index: &args->event_slot_index);
859
860	pr_debug("Created event (id:0x%08x) (%s)\n", args->event_id, __func__);
861	return err;
862	}
863
864	static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
865	void *data)
866	{
867	struct kfd_ioctl_destroy_event_args *args = data;
868
869	return kfd_event_destroy(p, event_id: args->event_id);
870	}
871
872	static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
873	void *data)
874	{
875	struct kfd_ioctl_set_event_args *args = data;
876
877	return kfd_set_event(p, event_id: args->event_id);
878	}
879
880	static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
881	void *data)
882	{
883	struct kfd_ioctl_reset_event_args *args = data;
884
885	return kfd_reset_event(p, event_id: args->event_id);
886	}
887
888	static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
889	void *data)
890	{
891	struct kfd_ioctl_wait_events_args *args = data;
892
893	return kfd_wait_on_events(p, num_events: args->num_events,
894	data: (void __user *)args->events_ptr,
895	all: (args->wait_for_all != 0),
896	user_timeout_ms: &args->timeout, wait_result: &args->wait_result);
897	}
898	static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
899	struct kfd_process *p, void *data)
900	{
901	struct kfd_ioctl_set_scratch_backing_va_args *args = data;
902	struct kfd_process_device *pdd;
903	struct kfd_node *dev;
904	long err;
905
906	mutex_lock(&p->mutex);
907	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
908	if (!pdd) {
909	err = -EINVAL;
910	goto err_pdd;
911	}
912	dev = pdd->dev;
913
914	pdd = kfd_bind_process_to_device(dev, p);
915	if (IS_ERR(ptr: pdd)) {
916	err = PTR_ERR(ptr: pdd);
917	goto bind_process_to_device_fail;
918	}
919
920	pdd->qpd.sh_hidden_private_base = args->va_addr;
921
922	mutex_unlock(lock: &p->mutex);
923
924	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
925	pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
926	dev->kfd2kgd->set_scratch_backing_va(
927	dev->adev, args->va_addr, pdd->qpd.vmid);
928
929	return 0;
930
931	bind_process_to_device_fail:
932	err_pdd:
933	mutex_unlock(lock: &p->mutex);
934	return err;
935	}
936
937	static int kfd_ioctl_get_tile_config(struct file *filep,
938	struct kfd_process *p, void *data)
939	{
940	struct kfd_ioctl_get_tile_config_args *args = data;
941	struct kfd_process_device *pdd;
942	struct tile_config config;
943	int err = 0;
944
945	mutex_lock(&p->mutex);
946	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
947	mutex_unlock(lock: &p->mutex);
948	if (!pdd)
949	return -EINVAL;
950
951	amdgpu_amdkfd_get_tile_config(adev: pdd->dev->adev, config: &config);
952
953	args->gb_addr_config = config.gb_addr_config;
954	args->num_banks = config.num_banks;
955	args->num_ranks = config.num_ranks;
956
957	if (args->num_tile_configs > config.num_tile_configs)
958	args->num_tile_configs = config.num_tile_configs;
959	err = copy_to_user(to: (void __user *)args->tile_config_ptr,
960	from: config.tile_config_ptr,
961	n: args->num_tile_configs * sizeof(uint32_t));
962	if (err) {
963	args->num_tile_configs = 0;
964	return -EFAULT;
965	}
966
967	if (args->num_macro_tile_configs > config.num_macro_tile_configs)
968	args->num_macro_tile_configs =
969	config.num_macro_tile_configs;
970	err = copy_to_user(to: (void __user *)args->macro_tile_config_ptr,
971	from: config.macro_tile_config_ptr,
972	n: args->num_macro_tile_configs * sizeof(uint32_t));
973	if (err) {
974	args->num_macro_tile_configs = 0;
975	return -EFAULT;
976	}
977
978	return 0;
979	}
980
981	static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
982	void *data)
983	{
984	struct kfd_ioctl_acquire_vm_args *args = data;
985	struct kfd_process_device *pdd;
986	struct file *drm_file;
987	int ret;
988
989	drm_file = fget(fd: args->drm_fd);
990	if (!drm_file)
991	return -EINVAL;
992
993	mutex_lock(&p->mutex);
994	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
995	if (!pdd) {
996	ret = -EINVAL;
997	goto err_pdd;
998	}
999
1000	if (pdd->drm_file) {
1001	ret = pdd->drm_file == drm_file ? 0 : -EBUSY;
1002	goto err_drm_file;
1003	}
1004
1005	ret = kfd_process_device_init_vm(pdd, drm_file);
1006	if (ret)
1007	goto err_unlock;
1008
1009	/* On success, the PDD keeps the drm_file reference */
1010	mutex_unlock(lock: &p->mutex);
1011
1012	return 0;
1013
1014	err_unlock:
1015	err_pdd:
1016	err_drm_file:
1017	mutex_unlock(lock: &p->mutex);
1018	fput(drm_file);
1019	return ret;
1020	}
1021
1022	bool kfd_dev_is_large_bar(struct kfd_node *dev)
1023	{
1024	if (dev->kfd->adev->debug_largebar) {
1025	pr_debug("Simulate large-bar allocation on non large-bar machine\n");
1026	return true;
1027	}
1028
1029	if (dev->local_mem_info.local_mem_size_private == 0 &&
1030	dev->local_mem_info.local_mem_size_public > 0)
1031	return true;
1032
1033	if (dev->local_mem_info.local_mem_size_public == 0 &&
1034	dev->kfd->adev->gmc.is_app_apu) {
1035	pr_debug("APP APU, Consider like a large bar system\n");
1036	return true;
1037	}
1038
1039	return false;
1040	}
1041
1042	static int kfd_ioctl_get_available_memory(struct file *filep,
1043	struct kfd_process *p, void *data)
1044	{
1045	struct kfd_ioctl_get_available_memory_args *args = data;
1046	struct kfd_process_device *pdd = kfd_lock_pdd_by_id(p, gpu_id: args->gpu_id);
1047
1048	if (!pdd)
1049	return -EINVAL;
1050	args->available = amdgpu_amdkfd_get_available_memory(adev: pdd->dev->adev,
1051	xcp_id: pdd->dev->node_id);
1052	kfd_unlock_pdd(pdd);
1053	return 0;
1054	}
1055
1056	static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
1057	struct kfd_process *p, void *data)
1058	{
1059	struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
1060	struct kfd_process_device *pdd;
1061	void *mem;
1062	struct kfd_node *dev;
1063	int idr_handle;
1064	long err;
1065	uint64_t offset = args->mmap_offset;
1066	uint32_t flags = args->flags;
1067
1068	if (args->size == 0)
1069	return -EINVAL;
1070
1071	#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1072	/* Flush pending deferred work to avoid racing with deferred actions
1073	* from previous memory map changes (e.g. munmap).
1074	*/
1075	svm_range_list_lock_and_flush_work(svms: &p->svms, current->mm);
1076	mutex_lock(&p->svms.lock);
1077	mmap_write_unlock(current->mm);
1078	if (interval_tree_iter_first(root: &p->svms.objects,
1079	start: args->va_addr >> PAGE_SHIFT,
1080	last: (args->va_addr + args->size - 1) >> PAGE_SHIFT)) {
1081	pr_err("Address: 0x%llx already allocated by SVM\n",
1082	args->va_addr);
1083	mutex_unlock(lock: &p->svms.lock);
1084	return -EADDRINUSE;
1085	}
1086
1087	/* When register user buffer check if it has been registered by svm by
1088	* buffer cpu virtual address.
1089	*/
1090	if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) &&
1091	interval_tree_iter_first(root: &p->svms.objects,
1092	start: args->mmap_offset >> PAGE_SHIFT,
1093	last: (args->mmap_offset + args->size - 1) >> PAGE_SHIFT)) {
1094	pr_err("User Buffer Address: 0x%llx already allocated by SVM\n",
1095	args->mmap_offset);
1096	mutex_unlock(lock: &p->svms.lock);
1097	return -EADDRINUSE;
1098	}
1099
1100	mutex_unlock(lock: &p->svms.lock);
1101	#endif
1102	mutex_lock(&p->mutex);
1103	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
1104	if (!pdd) {
1105	err = -EINVAL;
1106	goto err_pdd;
1107	}
1108
1109	dev = pdd->dev;
1110
1111	if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) &&
1112	(flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) &&
1113	!kfd_dev_is_large_bar(dev)) {
1114	pr_err("Alloc host visible vram on small bar is not allowed\n");
1115	err = -EINVAL;
1116	goto err_large_bar;
1117	}
1118
1119	pdd = kfd_bind_process_to_device(dev, p);
1120	if (IS_ERR(ptr: pdd)) {
1121	err = PTR_ERR(ptr: pdd);
1122	goto err_unlock;
1123	}
1124
1125	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
1126	if (args->size != kfd_doorbell_process_slice(kfd: dev->kfd)) {
1127	err = -EINVAL;
1128	goto err_unlock;
1129	}
1130	offset = kfd_get_process_doorbells(pdd);
1131	if (!offset) {
1132	err = -ENOMEM;
1133	goto err_unlock;
1134	}
1135	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
1136	if (args->size != PAGE_SIZE) {
1137	err = -EINVAL;
1138	goto err_unlock;
1139	}
1140	offset = dev->adev->rmmio_remap.bus_addr;
1141	if (!offset) {
1142	err = -ENOMEM;
1143	goto err_unlock;
1144	}
1145	}
1146
1147	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1148	adev: dev->adev, va: args->va_addr, size: args->size,
1149	drm_priv: pdd->drm_priv, mem: (struct kgd_mem **) &mem, offset: &offset,
1150	flags, criu_resume: false);
1151
1152	if (err)
1153	goto err_unlock;
1154
1155	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1156	if (idr_handle < 0) {
1157	err = -EFAULT;
1158	goto err_free;
1159	}
1160
1161	/* Update the VRAM usage count */
1162	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
1163	uint64_t size = args->size;
1164
1165	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM)
1166	size >>= 1;
1167	WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + PAGE_ALIGN(size));
1168	}
1169
1170	mutex_unlock(lock: &p->mutex);
1171
1172	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1173	args->mmap_offset = offset;
1174
1175	/* MMIO is mapped through kfd device
1176	* Generate a kfd mmap offset
1177	*/
1178	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1179	args->mmap_offset = KFD_MMAP_TYPE_MMIO
1180	| KFD_MMAP_GPU_ID(args->gpu_id);
1181
1182	return 0;
1183
1184	err_free:
1185	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: dev->adev, mem: (struct kgd_mem *)mem,
1186	drm_priv: pdd->drm_priv, NULL);
1187	err_unlock:
1188	err_pdd:
1189	err_large_bar:
1190	mutex_unlock(lock: &p->mutex);
1191	return err;
1192	}
1193
1194	static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
1195	struct kfd_process *p, void *data)
1196	{
1197	struct kfd_ioctl_free_memory_of_gpu_args *args = data;
1198	struct kfd_process_device *pdd;
1199	void *mem;
1200	int ret;
1201	uint64_t size = 0;
1202
1203	mutex_lock(&p->mutex);
1204	/*
1205	* Safeguard to prevent user space from freeing signal BO.
1206	* It will be freed at process termination.
1207	*/
1208	if (p->signal_handle && (p->signal_handle == args->handle)) {
1209	pr_err("Free signal BO is not allowed\n");
1210	ret = -EPERM;
1211	goto err_unlock;
1212	}
1213
1214	pdd = kfd_process_device_data_by_id(process: p, GET_GPU_ID(args->handle));
1215	if (!pdd) {
1216	pr_err("Process device data doesn't exist\n");
1217	ret = -EINVAL;
1218	goto err_pdd;
1219	}
1220
1221	mem = kfd_process_device_translate_handle(
1222	p: pdd, GET_IDR_HANDLE(args->handle));
1223	if (!mem) {
1224	ret = -EINVAL;
1225	goto err_unlock;
1226	}
1227
1228	ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: pdd->dev->adev,
1229	mem: (struct kgd_mem *)mem, drm_priv: pdd->drm_priv, size: &size);
1230
1231	/* If freeing the buffer failed, leave the handle in place for
1232	* clean-up during process tear-down.
1233	*/
1234	if (!ret)
1235	kfd_process_device_remove_obj_handle(
1236	pdd, GET_IDR_HANDLE(args->handle));
1237
1238	WRITE_ONCE(pdd->vram_usage, pdd->vram_usage - size);
1239
1240	err_unlock:
1241	err_pdd:
1242	mutex_unlock(lock: &p->mutex);
1243	return ret;
1244	}
1245
1246	static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
1247	struct kfd_process *p, void *data)
1248	{
1249	struct kfd_ioctl_map_memory_to_gpu_args *args = data;
1250	struct kfd_process_device *pdd, *peer_pdd;
1251	void *mem;
1252	struct kfd_node *dev;
1253	long err = 0;
1254	int i;
1255	uint32_t *devices_arr = NULL;
1256
1257	if (!args->n_devices) {
1258	pr_debug("Device IDs array empty\n");
1259	return -EINVAL;
1260	}
1261	if (args->n_success > args->n_devices) {
1262	pr_debug("n_success exceeds n_devices\n");
1263	return -EINVAL;
1264	}
1265
1266	devices_arr = kmalloc_array(n: args->n_devices, size: sizeof(*devices_arr),
1267	GFP_KERNEL);
1268	if (!devices_arr)
1269	return -ENOMEM;
1270
1271	err = copy_from_user(to: devices_arr,
1272	from: (void __user *)args->device_ids_array_ptr,
1273	n: args->n_devices * sizeof(*devices_arr));
1274	if (err != 0) {
1275	err = -EFAULT;
1276	goto copy_from_user_failed;
1277	}
1278
1279	mutex_lock(&p->mutex);
1280	pdd = kfd_process_device_data_by_id(process: p, GET_GPU_ID(args->handle));
1281	if (!pdd) {
1282	err = -EINVAL;
1283	goto get_process_device_data_failed;
1284	}
1285	dev = pdd->dev;
1286
1287	pdd = kfd_bind_process_to_device(dev, p);
1288	if (IS_ERR(ptr: pdd)) {
1289	err = PTR_ERR(ptr: pdd);
1290	goto bind_process_to_device_failed;
1291	}
1292
1293	mem = kfd_process_device_translate_handle(p: pdd,
1294	GET_IDR_HANDLE(args->handle));
1295	if (!mem) {
1296	err = -ENOMEM;
1297	goto get_mem_obj_from_handle_failed;
1298	}
1299
1300	for (i = args->n_success; i < args->n_devices; i++) {
1301	peer_pdd = kfd_process_device_data_by_id(process: p, gpu_id: devices_arr[i]);
1302	if (!peer_pdd) {
1303	pr_debug("Getting device by id failed for 0x%x\n",
1304	devices_arr[i]);
1305	err = -EINVAL;
1306	goto get_mem_obj_from_handle_failed;
1307	}
1308
1309	peer_pdd = kfd_bind_process_to_device(dev: peer_pdd->dev, p);
1310	if (IS_ERR(ptr: peer_pdd)) {
1311	err = PTR_ERR(ptr: peer_pdd);
1312	goto get_mem_obj_from_handle_failed;
1313	}
1314
1315	err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1316	adev: peer_pdd->dev->adev, mem: (struct kgd_mem *)mem,
1317	drm_priv: peer_pdd->drm_priv);
1318	if (err) {
1319	struct pci_dev *pdev = peer_pdd->dev->adev->pdev;
1320
1321	dev_err(dev->adev->dev,
1322	"Failed to map peer:%04x:%02x:%02x.%d mem_domain:%d\n",
1323	pci_domain_nr(pdev->bus),
1324	pdev->bus->number,
1325	PCI_SLOT(pdev->devfn),
1326	PCI_FUNC(pdev->devfn),
1327	((struct kgd_mem *)mem)->domain);
1328	goto map_memory_to_gpu_failed;
1329	}
1330	args->n_success = i+1;
1331	}
1332
1333	err = amdgpu_amdkfd_gpuvm_sync_memory(adev: dev->adev, mem: (struct kgd_mem *) mem, intr: true);
1334	if (err) {
1335	pr_debug("Sync memory failed, wait interrupted by user signal\n");
1336	goto sync_memory_failed;
1337	}
1338
1339	mutex_unlock(lock: &p->mutex);
1340
1341	/* Flush TLBs after waiting for the page table updates to complete */
1342	for (i = 0; i < args->n_devices; i++) {
1343	peer_pdd = kfd_process_device_data_by_id(process: p, gpu_id: devices_arr[i]);
1344	if (WARN_ON_ONCE(!peer_pdd))
1345	continue;
1346	kfd_flush_tlb(pdd: peer_pdd, type: TLB_FLUSH_LEGACY);
1347	}
1348	kfree(objp: devices_arr);
1349
1350	return err;
1351
1352	get_process_device_data_failed:
1353	bind_process_to_device_failed:
1354	get_mem_obj_from_handle_failed:
1355	map_memory_to_gpu_failed:
1356	sync_memory_failed:
1357	mutex_unlock(lock: &p->mutex);
1358	copy_from_user_failed:
1359	kfree(objp: devices_arr);
1360
1361	return err;
1362	}
1363
1364	static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
1365	struct kfd_process *p, void *data)
1366	{
1367	struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
1368	struct kfd_process_device *pdd, *peer_pdd;
1369	void *mem;
1370	long err = 0;
1371	uint32_t *devices_arr = NULL, i;
1372	bool flush_tlb;
1373
1374	if (!args->n_devices) {
1375	pr_debug("Device IDs array empty\n");
1376	return -EINVAL;
1377	}
1378	if (args->n_success > args->n_devices) {
1379	pr_debug("n_success exceeds n_devices\n");
1380	return -EINVAL;
1381	}
1382
1383	devices_arr = kmalloc_array(n: args->n_devices, size: sizeof(*devices_arr),
1384	GFP_KERNEL);
1385	if (!devices_arr)
1386	return -ENOMEM;
1387
1388	err = copy_from_user(to: devices_arr,
1389	from: (void __user *)args->device_ids_array_ptr,
1390	n: args->n_devices * sizeof(*devices_arr));
1391	if (err != 0) {
1392	err = -EFAULT;
1393	goto copy_from_user_failed;
1394	}
1395
1396	mutex_lock(&p->mutex);
1397	pdd = kfd_process_device_data_by_id(process: p, GET_GPU_ID(args->handle));
1398	if (!pdd) {
1399	err = -EINVAL;
1400	goto bind_process_to_device_failed;
1401	}
1402
1403	mem = kfd_process_device_translate_handle(p: pdd,
1404	GET_IDR_HANDLE(args->handle));
1405	if (!mem) {
1406	err = -ENOMEM;
1407	goto get_mem_obj_from_handle_failed;
1408	}
1409
1410	for (i = args->n_success; i < args->n_devices; i++) {
1411	peer_pdd = kfd_process_device_data_by_id(process: p, gpu_id: devices_arr[i]);
1412	if (!peer_pdd) {
1413	err = -EINVAL;
1414	goto get_mem_obj_from_handle_failed;
1415	}
1416	err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1417	adev: peer_pdd->dev->adev, mem: (struct kgd_mem *)mem, drm_priv: peer_pdd->drm_priv);
1418	if (err) {
1419	pr_err("Failed to unmap from gpu %d/%d\n",
1420	i, args->n_devices);
1421	goto unmap_memory_from_gpu_failed;
1422	}
1423	args->n_success = i+1;
1424	}
1425
1426	flush_tlb = kfd_flush_tlb_after_unmap(dev: pdd->dev->kfd);
1427	if (flush_tlb) {
1428	err = amdgpu_amdkfd_gpuvm_sync_memory(adev: pdd->dev->adev,
1429	mem: (struct kgd_mem *) mem, intr: true);
1430	if (err) {
1431	pr_debug("Sync memory failed, wait interrupted by user signal\n");
1432	goto sync_memory_failed;
1433	}
1434	}
1435
1436	/* Flush TLBs after waiting for the page table updates to complete */
1437	for (i = 0; i < args->n_devices; i++) {
1438	peer_pdd = kfd_process_device_data_by_id(process: p, gpu_id: devices_arr[i]);
1439	if (WARN_ON_ONCE(!peer_pdd))
1440	continue;
1441	if (flush_tlb)
1442	kfd_flush_tlb(pdd: peer_pdd, type: TLB_FLUSH_HEAVYWEIGHT);
1443
1444	/* Remove dma mapping after tlb flush to avoid IO_PAGE_FAULT */
1445	amdgpu_amdkfd_gpuvm_dmaunmap_mem(mem, drm_priv: peer_pdd->drm_priv);
1446	}
1447
1448	mutex_unlock(lock: &p->mutex);
1449
1450	kfree(objp: devices_arr);
1451
1452	return 0;
1453
1454	bind_process_to_device_failed:
1455	get_mem_obj_from_handle_failed:
1456	unmap_memory_from_gpu_failed:
1457	sync_memory_failed:
1458	mutex_unlock(lock: &p->mutex);
1459	copy_from_user_failed:
1460	kfree(objp: devices_arr);
1461	return err;
1462	}
1463
1464	static int kfd_ioctl_alloc_queue_gws(struct file *filep,
1465	struct kfd_process *p, void *data)
1466	{
1467	int retval;
1468	struct kfd_ioctl_alloc_queue_gws_args *args = data;
1469	struct queue *q;
1470	struct kfd_node *dev;
1471
1472	mutex_lock(&p->mutex);
1473	q = pqm_get_user_queue(pqm: &p->pqm, qid: args->queue_id);
1474
1475	if (q) {
1476	dev = q->device;
1477	} else {
1478	retval = -EINVAL;
1479	goto out_unlock;
1480	}
1481
1482	if (!dev->gws) {
1483	retval = -ENODEV;
1484	goto out_unlock;
1485	}
1486
1487	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1488	retval = -ENODEV;
1489	goto out_unlock;
1490	}
1491
1492	if (p->debug_trap_enabled && (!kfd_dbg_has_gws_support(dev) ||
1493	kfd_dbg_has_cwsr_workaround(dev))) {
1494	retval = -EBUSY;
1495	goto out_unlock;
1496	}
1497
1498	retval = pqm_set_gws(pqm: &p->pqm, qid: args->queue_id, gws: args->num_gws ? dev->gws : NULL);
1499	mutex_unlock(lock: &p->mutex);
1500
1501	args->first_gws = 0;
1502	return retval;
1503
1504	out_unlock:
1505	mutex_unlock(lock: &p->mutex);
1506	return retval;
1507	}
1508
1509	static int kfd_ioctl_get_dmabuf_info(struct file *filep,
1510	struct kfd_process *p, void *data)
1511	{
1512	struct kfd_ioctl_get_dmabuf_info_args *args = data;
1513	struct kfd_node *dev = NULL;
1514	struct amdgpu_device *dmabuf_adev;
1515	void *metadata_buffer = NULL;
1516	uint32_t flags;
1517	int8_t xcp_id;
1518	unsigned int i;
1519	int r;
1520
1521	/* Find a KFD GPU device that supports the get_dmabuf_info query */
1522	for (i = 0; kfd_topology_enum_kfd_devices(idx: i, kdev: &dev) == 0; i++)
1523	if (dev)
1524	break;
1525	if (!dev)
1526	return -EINVAL;
1527
1528	if (args->metadata_ptr) {
1529	metadata_buffer = kzalloc(size: args->metadata_size, GFP_KERNEL);
1530	if (!metadata_buffer)
1531	return -ENOMEM;
1532	}
1533
1534	/* Get dmabuf info from KGD */
1535	r = amdgpu_amdkfd_get_dmabuf_info(adev: dev->adev, dma_buf_fd: args->dmabuf_fd,
1536	dmabuf_adev: &dmabuf_adev, bo_size: &args->size,
1537	metadata_buffer, buffer_size: args->metadata_size,
1538	metadata_size: &args->metadata_size, flags: &flags, xcp_id: &xcp_id);
1539	if (r)
1540	goto exit;
1541
1542	if (xcp_id >= 0)
1543	args->gpu_id = dmabuf_adev->kfd.dev->nodes[xcp_id]->id;
1544	else
1545	args->gpu_id = dmabuf_adev->kfd.dev->nodes[0]->id;
1546	args->flags = flags;
1547
1548	/* Copy metadata buffer to user mode */
1549	if (metadata_buffer) {
1550	r = copy_to_user(to: (void __user *)args->metadata_ptr,
1551	from: metadata_buffer, n: args->metadata_size);
1552	if (r != 0)
1553	r = -EFAULT;
1554	}
1555
1556	exit:
1557	kfree(objp: metadata_buffer);
1558
1559	return r;
1560	}
1561
1562	static int kfd_ioctl_import_dmabuf(struct file *filep,
1563	struct kfd_process *p, void *data)
1564	{
1565	struct kfd_ioctl_import_dmabuf_args *args = data;
1566	struct kfd_process_device *pdd;
1567	struct dma_buf *dmabuf;
1568	int idr_handle;
1569	uint64_t size;
1570	void *mem;
1571	int r;
1572
1573	dmabuf = dma_buf_get(fd: args->dmabuf_fd);
1574	if (IS_ERR(ptr: dmabuf))
1575	return PTR_ERR(ptr: dmabuf);
1576
1577	mutex_lock(&p->mutex);
1578	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
1579	if (!pdd) {
1580	r = -EINVAL;
1581	goto err_unlock;
1582	}
1583
1584	pdd = kfd_bind_process_to_device(dev: pdd->dev, p);
1585	if (IS_ERR(ptr: pdd)) {
1586	r = PTR_ERR(ptr: pdd);
1587	goto err_unlock;
1588	}
1589
1590	r = amdgpu_amdkfd_gpuvm_import_dmabuf(adev: pdd->dev->adev, dmabuf,
1591	va: args->va_addr, drm_priv: pdd->drm_priv,
1592	mem: (struct kgd_mem **)&mem, size: &size,
1593	NULL);
1594	if (r)
1595	goto err_unlock;
1596
1597	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1598	if (idr_handle < 0) {
1599	r = -EFAULT;
1600	goto err_free;
1601	}
1602
1603	mutex_unlock(lock: &p->mutex);
1604	dma_buf_put(dmabuf);
1605
1606	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1607
1608	return 0;
1609
1610	err_free:
1611	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: pdd->dev->adev, mem: (struct kgd_mem *)mem,
1612	drm_priv: pdd->drm_priv, NULL);
1613	err_unlock:
1614	mutex_unlock(lock: &p->mutex);
1615	dma_buf_put(dmabuf);
1616	return r;
1617	}
1618
1619	static int kfd_ioctl_export_dmabuf(struct file *filep,
1620	struct kfd_process *p, void *data)
1621	{
1622	struct kfd_ioctl_export_dmabuf_args *args = data;
1623	struct kfd_process_device *pdd;
1624	struct dma_buf *dmabuf;
1625	struct kfd_node *dev;
1626	void *mem;
1627	int ret = 0;
1628
1629	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1630	if (!dev)
1631	return -EINVAL;
1632
1633	mutex_lock(&p->mutex);
1634
1635	pdd = kfd_get_process_device_data(dev, p);
1636	if (!pdd) {
1637	ret = -EINVAL;
1638	goto err_unlock;
1639	}
1640
1641	mem = kfd_process_device_translate_handle(p: pdd,
1642	GET_IDR_HANDLE(args->handle));
1643	if (!mem) {
1644	ret = -EINVAL;
1645	goto err_unlock;
1646	}
1647
1648	ret = amdgpu_amdkfd_gpuvm_export_dmabuf(mem, dmabuf: &dmabuf);
1649	mutex_unlock(lock: &p->mutex);
1650	if (ret)
1651	goto err_out;
1652
1653	ret = dma_buf_fd(dmabuf, flags: args->flags);
1654	if (ret < 0) {
1655	dma_buf_put(dmabuf);
1656	goto err_out;
1657	}
1658	/* dma_buf_fd assigns the reference count to the fd, no need to
1659	* put the reference here.
1660	*/
1661	args->dmabuf_fd = ret;
1662
1663	return 0;
1664
1665	err_unlock:
1666	mutex_unlock(lock: &p->mutex);
1667	err_out:
1668	return ret;
1669	}
1670
1671	/* Handle requests for watching SMI events */
1672	static int kfd_ioctl_smi_events(struct file *filep,
1673	struct kfd_process *p, void *data)
1674	{
1675	struct kfd_ioctl_smi_events_args *args = data;
1676	struct kfd_process_device *pdd;
1677
1678	mutex_lock(&p->mutex);
1679
1680	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpuid);
1681	mutex_unlock(lock: &p->mutex);
1682	if (!pdd)
1683	return -EINVAL;
1684
1685	return kfd_smi_event_open(dev: pdd->dev, fd: &args->anon_fd);
1686	}
1687
1688	#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1689
1690	static int kfd_ioctl_set_xnack_mode(struct file *filep,
1691	struct kfd_process *p, void *data)
1692	{
1693	struct kfd_ioctl_set_xnack_mode_args *args = data;
1694	int r = 0;
1695
1696	mutex_lock(&p->mutex);
1697	if (args->xnack_enabled >= 0) {
1698	if (!list_empty(head: &p->pqm.queues)) {
1699	pr_debug("Process has user queues running\n");
1700	r = -EBUSY;
1701	goto out_unlock;
1702	}
1703
1704	if (p->xnack_enabled == args->xnack_enabled)
1705	goto out_unlock;
1706
1707	if (args->xnack_enabled && !kfd_process_xnack_mode(p, supported: true)) {
1708	r = -EPERM;
1709	goto out_unlock;
1710	}
1711
1712	r = svm_range_switch_xnack_reserve_mem(p, xnack_enabled: args->xnack_enabled);
1713	} else {
1714	args->xnack_enabled = p->xnack_enabled;
1715	}
1716
1717	out_unlock:
1718	mutex_unlock(lock: &p->mutex);
1719
1720	return r;
1721	}
1722
1723	static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1724	{
1725	struct kfd_ioctl_svm_args *args = data;
1726	int r = 0;
1727
1728	pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n",
1729	args->start_addr, args->size, args->op, args->nattr);
1730
1731	if ((args->start_addr & ~PAGE_MASK) || (args->size & ~PAGE_MASK))
1732	return -EINVAL;
1733	if (!args->start_addr || !args->size)
1734	return -EINVAL;
1735
1736	r = svm_ioctl(p, op: args->op, start: args->start_addr, size: args->size, nattrs: args->nattr,
1737	attrs: args->attrs);
1738
1739	return r;
1740	}
1741	#else
1742	static int kfd_ioctl_set_xnack_mode(struct file *filep,
1743	struct kfd_process *p, void *data)
1744	{
1745	return -EPERM;
1746	}
1747	static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1748	{
1749	return -EPERM;
1750	}
1751	#endif
1752
1753	static int criu_checkpoint_process(struct kfd_process *p,
1754	uint8_t __user *user_priv_data,
1755	uint64_t *priv_offset)
1756	{
1757	struct kfd_criu_process_priv_data process_priv;
1758	int ret;
1759
1760	memset(&process_priv, 0, sizeof(process_priv));
1761
1762	process_priv.version = KFD_CRIU_PRIV_VERSION;
1763	/* For CR, we don't consider negative xnack mode which is used for
1764	* querying without changing it, here 0 simply means disabled and 1
1765	* means enabled so retry for finding a valid PTE.
1766	*/
1767	process_priv.xnack_mode = p->xnack_enabled ? 1 : 0;
1768
1769	ret = copy_to_user(to: user_priv_data + *priv_offset,
1770	from: &process_priv, n: sizeof(process_priv));
1771
1772	if (ret) {
1773	pr_err("Failed to copy process information to user\n");
1774	ret = -EFAULT;
1775	}
1776
1777	*priv_offset += sizeof(process_priv);
1778	return ret;
1779	}
1780
1781	static int criu_checkpoint_devices(struct kfd_process *p,
1782	uint32_t num_devices,
1783	uint8_t __user *user_addr,
1784	uint8_t __user *user_priv_data,
1785	uint64_t *priv_offset)
1786	{
1787	struct kfd_criu_device_priv_data *device_priv = NULL;
1788	struct kfd_criu_device_bucket *device_buckets = NULL;
1789	int ret = 0, i;
1790
1791	device_buckets = kvzalloc(size: num_devices * sizeof(*device_buckets), GFP_KERNEL);
1792	if (!device_buckets) {
1793	ret = -ENOMEM;
1794	goto exit;
1795	}
1796
1797	device_priv = kvzalloc(size: num_devices * sizeof(*device_priv), GFP_KERNEL);
1798	if (!device_priv) {
1799	ret = -ENOMEM;
1800	goto exit;
1801	}
1802
1803	for (i = 0; i < num_devices; i++) {
1804	struct kfd_process_device *pdd = p->pdds[i];
1805
1806	device_buckets[i].user_gpu_id = pdd->user_gpu_id;
1807	device_buckets[i].actual_gpu_id = pdd->dev->id;
1808
1809	/*
1810	* priv_data does not contain useful information for now and is reserved for
1811	* future use, so we do not set its contents.
1812	*/
1813	}
1814
1815	ret = copy_to_user(to: user_addr, from: device_buckets, n: num_devices * sizeof(*device_buckets));
1816	if (ret) {
1817	pr_err("Failed to copy device information to user\n");
1818	ret = -EFAULT;
1819	goto exit;
1820	}
1821
1822	ret = copy_to_user(to: user_priv_data + *priv_offset,
1823	from: device_priv,
1824	n: num_devices * sizeof(*device_priv));
1825	if (ret) {
1826	pr_err("Failed to copy device information to user\n");
1827	ret = -EFAULT;
1828	}
1829	*priv_offset += num_devices * sizeof(*device_priv);
1830
1831	exit:
1832	kvfree(addr: device_buckets);
1833	kvfree(addr: device_priv);
1834	return ret;
1835	}
1836
1837	static uint32_t get_process_num_bos(struct kfd_process *p)
1838	{
1839	uint32_t num_of_bos = 0;
1840	int i;
1841
1842	/* Run over all PDDs of the process */
1843	for (i = 0; i < p->n_pdds; i++) {
1844	struct kfd_process_device *pdd = p->pdds[i];
1845	void *mem;
1846	int id;
1847
1848	idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1849	struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
1850
1851	if (!kgd_mem->va || kgd_mem->va > pdd->gpuvm_base)
1852	num_of_bos++;
1853	}
1854	}
1855	return num_of_bos;
1856	}
1857
1858	static int criu_get_prime_handle(struct kgd_mem *mem, int flags,
1859	u32 *shared_fd)
1860	{
1861	struct dma_buf *dmabuf;
1862	int ret;
1863
1864	ret = amdgpu_amdkfd_gpuvm_export_dmabuf(mem, dmabuf: &dmabuf);
1865	if (ret) {
1866	pr_err("dmabuf export failed for the BO\n");
1867	return ret;
1868	}
1869
1870	ret = dma_buf_fd(dmabuf, flags);
1871	if (ret < 0) {
1872	pr_err("dmabuf create fd failed, ret:%d\n", ret);
1873	goto out_free_dmabuf;
1874	}
1875
1876	*shared_fd = ret;
1877	return 0;
1878
1879	out_free_dmabuf:
1880	dma_buf_put(dmabuf);
1881	return ret;
1882	}
1883
1884	static int criu_checkpoint_bos(struct kfd_process *p,
1885	uint32_t num_bos,
1886	uint8_t __user *user_bos,
1887	uint8_t __user *user_priv_data,
1888	uint64_t *priv_offset)
1889	{
1890	struct kfd_criu_bo_bucket *bo_buckets;
1891	struct kfd_criu_bo_priv_data *bo_privs;
1892	int ret = 0, pdd_index, bo_index = 0, id;
1893	void *mem;
1894
1895	bo_buckets = kvzalloc(size: num_bos * sizeof(*bo_buckets), GFP_KERNEL);
1896	if (!bo_buckets)
1897	return -ENOMEM;
1898
1899	bo_privs = kvzalloc(size: num_bos * sizeof(*bo_privs), GFP_KERNEL);
1900	if (!bo_privs) {
1901	ret = -ENOMEM;
1902	goto exit;
1903	}
1904
1905	for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {
1906	struct kfd_process_device *pdd = p->pdds[pdd_index];
1907	struct amdgpu_bo *dumper_bo;
1908	struct kgd_mem *kgd_mem;
1909
1910	idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1911	struct kfd_criu_bo_bucket *bo_bucket;
1912	struct kfd_criu_bo_priv_data *bo_priv;
1913	int i, dev_idx = 0;
1914
1915	if (!mem) {
1916	ret = -ENOMEM;
1917	goto exit;
1918	}
1919
1920	kgd_mem = (struct kgd_mem *)mem;
1921	dumper_bo = kgd_mem->bo;
1922
1923	/* Skip checkpointing BOs that are used for Trap handler
1924	* code and state. Currently, these BOs have a VA that
1925	* is less GPUVM Base
1926	*/
1927	if (kgd_mem->va && kgd_mem->va <= pdd->gpuvm_base)
1928	continue;
1929
1930	bo_bucket = &bo_buckets[bo_index];
1931	bo_priv = &bo_privs[bo_index];
1932
1933	bo_bucket->gpu_id = pdd->user_gpu_id;
1934	bo_bucket->addr = (uint64_t)kgd_mem->va;
1935	bo_bucket->size = amdgpu_bo_size(bo: dumper_bo);
1936	bo_bucket->alloc_flags = (uint32_t)kgd_mem->alloc_flags;
1937	bo_priv->idr_handle = id;
1938
1939	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1940	ret = amdgpu_ttm_tt_get_userptr(tbo: &dumper_bo->tbo,
1941	user_addr: &bo_priv->user_addr);
1942	if (ret) {
1943	pr_err("Failed to obtain user address for user-pointer bo\n");
1944	goto exit;
1945	}
1946	}
1947	if (bo_bucket->alloc_flags
1948	& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
1949	ret = criu_get_prime_handle(mem: kgd_mem,
1950	flags: bo_bucket->alloc_flags &
1951	KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0,
1952	shared_fd: &bo_bucket->dmabuf_fd);
1953	if (ret)
1954	goto exit;
1955	} else {
1956	bo_bucket->dmabuf_fd = KFD_INVALID_FD;
1957	}
1958
1959	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
1960	bo_bucket->offset = KFD_MMAP_TYPE_DOORBELL |
1961	KFD_MMAP_GPU_ID(pdd->dev->id);
1962	else if (bo_bucket->alloc_flags &
1963	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1964	bo_bucket->offset = KFD_MMAP_TYPE_MMIO |
1965	KFD_MMAP_GPU_ID(pdd->dev->id);
1966	else
1967	bo_bucket->offset = amdgpu_bo_mmap_offset(bo: dumper_bo);
1968
1969	for (i = 0; i < p->n_pdds; i++) {
1970	if (amdgpu_amdkfd_bo_mapped_to_dev(adev: p->pdds[i]->dev->adev, mem: kgd_mem))
1971	bo_priv->mapped_gpuids[dev_idx++] = p->pdds[i]->user_gpu_id;
1972	}
1973
1974	pr_debug("bo_size = 0x%llx, bo_addr = 0x%llx bo_offset = 0x%llx\n"
1975	"gpu_id = 0x%x alloc_flags = 0x%x idr_handle = 0x%x",
1976	bo_bucket->size,
1977	bo_bucket->addr,
1978	bo_bucket->offset,
1979	bo_bucket->gpu_id,
1980	bo_bucket->alloc_flags,
1981	bo_priv->idr_handle);
1982	bo_index++;
1983	}
1984	}
1985
1986	ret = copy_to_user(to: user_bos, from: bo_buckets, n: num_bos * sizeof(*bo_buckets));
1987	if (ret) {
1988	pr_err("Failed to copy BO information to user\n");
1989	ret = -EFAULT;
1990	goto exit;
1991	}
1992
1993	ret = copy_to_user(to: user_priv_data + *priv_offset, from: bo_privs, n: num_bos * sizeof(*bo_privs));
1994	if (ret) {
1995	pr_err("Failed to copy BO priv information to user\n");
1996	ret = -EFAULT;
1997	goto exit;
1998	}
1999
2000	*priv_offset += num_bos * sizeof(*bo_privs);
2001
2002	exit:
2003	while (ret && bo_index--) {
2004	if (bo_buckets[bo_index].alloc_flags
2005	& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT))
2006	close_fd(fd: bo_buckets[bo_index].dmabuf_fd);
2007	}
2008
2009	kvfree(addr: bo_buckets);
2010	kvfree(addr: bo_privs);
2011	return ret;
2012	}
2013
2014	static int criu_get_process_object_info(struct kfd_process *p,
2015	uint32_t *num_devices,
2016	uint32_t *num_bos,
2017	uint32_t *num_objects,
2018	uint64_t *objs_priv_size)
2019	{
2020	uint64_t queues_priv_data_size, svm_priv_data_size, priv_size;
2021	uint32_t num_queues, num_events, num_svm_ranges;
2022	int ret;
2023
2024	*num_devices = p->n_pdds;
2025	*num_bos = get_process_num_bos(p);
2026
2027	ret = kfd_process_get_queue_info(p, num_queues: &num_queues, priv_data_sizes: &queues_priv_data_size);
2028	if (ret)
2029	return ret;
2030
2031	num_events = kfd_get_num_events(p);
2032
2033	ret = svm_range_get_info(p, num_svm_ranges: &num_svm_ranges, svm_priv_data_size: &svm_priv_data_size);
2034	if (ret)
2035	return ret;
2036
2037	*num_objects = num_queues + num_events + num_svm_ranges;
2038
2039	if (objs_priv_size) {
2040	priv_size = sizeof(struct kfd_criu_process_priv_data);
2041	priv_size += *num_devices * sizeof(struct kfd_criu_device_priv_data);
2042	priv_size += *num_bos * sizeof(struct kfd_criu_bo_priv_data);
2043	priv_size += queues_priv_data_size;
2044	priv_size += num_events * sizeof(struct kfd_criu_event_priv_data);
2045	priv_size += svm_priv_data_size;
2046	*objs_priv_size = priv_size;
2047	}
2048	return 0;
2049	}
2050
2051	static int criu_checkpoint(struct file *filep,
2052	struct kfd_process *p,
2053	struct kfd_ioctl_criu_args *args)
2054	{
2055	int ret;
2056	uint32_t num_devices, num_bos, num_objects;
2057	uint64_t priv_size, priv_offset = 0, bo_priv_offset;
2058
2059	if (!args->devices || !args->bos || !args->priv_data)
2060	return -EINVAL;
2061
2062	mutex_lock(&p->mutex);
2063
2064	if (!p->n_pdds) {
2065	pr_err("No pdd for given process\n");
2066	ret = -ENODEV;
2067	goto exit_unlock;
2068	}
2069
2070	/* Confirm all process queues are evicted */
2071	if (!p->queues_paused) {
2072	pr_err("Cannot dump process when queues are not in evicted state\n");
2073	/* CRIU plugin did not call op PROCESS_INFO before checkpointing */
2074	ret = -EINVAL;
2075	goto exit_unlock;
2076	}
2077
2078	ret = criu_get_process_object_info(p, num_devices: &num_devices, num_bos: &num_bos, num_objects: &num_objects, objs_priv_size: &priv_size);
2079	if (ret)
2080	goto exit_unlock;
2081
2082	if (num_devices != args->num_devices ||
2083	num_bos != args->num_bos ||
2084	num_objects != args->num_objects ||
2085	priv_size != args->priv_data_size) {
2086
2087	ret = -EINVAL;
2088	goto exit_unlock;
2089	}
2090
2091	/* each function will store private data inside priv_data and adjust priv_offset */
2092	ret = criu_checkpoint_process(p, user_priv_data: (uint8_t __user *)args->priv_data, priv_offset: &priv_offset);
2093	if (ret)
2094	goto exit_unlock;
2095
2096	ret = criu_checkpoint_devices(p, num_devices, user_addr: (uint8_t __user *)args->devices,
2097	user_priv_data: (uint8_t __user *)args->priv_data, priv_offset: &priv_offset);
2098	if (ret)
2099	goto exit_unlock;
2100
2101	/* Leave room for BOs in the private data. They need to be restored
2102	* before events, but we checkpoint them last to simplify the error
2103	* handling.
2104	*/
2105	bo_priv_offset = priv_offset;
2106	priv_offset += num_bos * sizeof(struct kfd_criu_bo_priv_data);
2107
2108	if (num_objects) {
2109	ret = kfd_criu_checkpoint_queues(p, user_priv_data: (uint8_t __user *)args->priv_data,
2110	priv_data_offset: &priv_offset);
2111	if (ret)
2112	goto exit_unlock;
2113
2114	ret = kfd_criu_checkpoint_events(p, user_priv_data: (uint8_t __user *)args->priv_data,
2115	priv_data_offset: &priv_offset);
2116	if (ret)
2117	goto exit_unlock;
2118
2119	ret = kfd_criu_checkpoint_svm(p, user_priv_data: (uint8_t __user *)args->priv_data, priv_offset: &priv_offset);
2120	if (ret)
2121	goto exit_unlock;
2122	}
2123
2124	/* This must be the last thing in this function that can fail.
2125	* Otherwise we leak dmabuf file descriptors.
2126	*/
2127	ret = criu_checkpoint_bos(p, num_bos, user_bos: (uint8_t __user *)args->bos,
2128	user_priv_data: (uint8_t __user *)args->priv_data, priv_offset: &bo_priv_offset);
2129
2130	exit_unlock:
2131	mutex_unlock(lock: &p->mutex);
2132	if (ret)
2133	pr_err("Failed to dump CRIU ret:%d\n", ret);
2134	else
2135	pr_debug("CRIU dump ret:%d\n", ret);
2136
2137	return ret;
2138	}
2139
2140	static int criu_restore_process(struct kfd_process *p,
2141	struct kfd_ioctl_criu_args *args,
2142	uint64_t *priv_offset,
2143	uint64_t max_priv_data_size)
2144	{
2145	int ret = 0;
2146	struct kfd_criu_process_priv_data process_priv;
2147
2148	if (*priv_offset + sizeof(process_priv) > max_priv_data_size)
2149	return -EINVAL;
2150
2151	ret = copy_from_user(to: &process_priv,
2152	from: (void __user *)(args->priv_data + *priv_offset),
2153	n: sizeof(process_priv));
2154	if (ret) {
2155	pr_err("Failed to copy process private information from user\n");
2156	ret = -EFAULT;
2157	goto exit;
2158	}
2159	*priv_offset += sizeof(process_priv);
2160
2161	if (process_priv.version != KFD_CRIU_PRIV_VERSION) {
2162	pr_err("Invalid CRIU API version (checkpointed:%d current:%d)\n",
2163	process_priv.version, KFD_CRIU_PRIV_VERSION);
2164	return -EINVAL;
2165	}
2166
2167	pr_debug("Setting XNACK mode\n");
2168	if (process_priv.xnack_mode && !kfd_process_xnack_mode(p, supported: true)) {
2169	pr_err("xnack mode cannot be set\n");
2170	ret = -EPERM;
2171	goto exit;
2172	} else {
2173	pr_debug("set xnack mode: %d\n", process_priv.xnack_mode);
2174	p->xnack_enabled = process_priv.xnack_mode;
2175	}
2176
2177	exit:
2178	return ret;
2179	}
2180
2181	static int criu_restore_devices(struct kfd_process *p,
2182	struct kfd_ioctl_criu_args *args,
2183	uint64_t *priv_offset,
2184	uint64_t max_priv_data_size)
2185	{
2186	struct kfd_criu_device_bucket *device_buckets;
2187	struct kfd_criu_device_priv_data *device_privs;
2188	int ret = 0;
2189	uint32_t i;
2190
2191	if (args->num_devices != p->n_pdds)
2192	return -EINVAL;
2193
2194	if (*priv_offset + (args->num_devices * sizeof(*device_privs)) > max_priv_data_size)
2195	return -EINVAL;
2196
2197	device_buckets = kmalloc_array(n: args->num_devices, size: sizeof(*device_buckets), GFP_KERNEL);
2198	if (!device_buckets)
2199	return -ENOMEM;
2200
2201	ret = copy_from_user(to: device_buckets, from: (void __user *)args->devices,
2202	n: args->num_devices * sizeof(*device_buckets));
2203	if (ret) {
2204	pr_err("Failed to copy devices buckets from user\n");
2205	ret = -EFAULT;
2206	goto exit;
2207	}
2208
2209	for (i = 0; i < args->num_devices; i++) {
2210	struct kfd_node *dev;
2211	struct kfd_process_device *pdd;
2212	struct file *drm_file;
2213
2214	/* device private data is not currently used */
2215
2216	if (!device_buckets[i].user_gpu_id) {
2217	pr_err("Invalid user gpu_id\n");
2218	ret = -EINVAL;
2219	goto exit;
2220	}
2221
2222	dev = kfd_device_by_id(gpu_id: device_buckets[i].actual_gpu_id);
2223	if (!dev) {
2224	pr_err("Failed to find device with gpu_id = %x\n",
2225	device_buckets[i].actual_gpu_id);
2226	ret = -EINVAL;
2227	goto exit;
2228	}
2229
2230	pdd = kfd_get_process_device_data(dev, p);
2231	if (!pdd) {
2232	pr_err("Failed to get pdd for gpu_id = %x\n",
2233	device_buckets[i].actual_gpu_id);
2234	ret = -EINVAL;
2235	goto exit;
2236	}
2237	pdd->user_gpu_id = device_buckets[i].user_gpu_id;
2238
2239	drm_file = fget(fd: device_buckets[i].drm_fd);
2240	if (!drm_file) {
2241	pr_err("Invalid render node file descriptor sent from plugin (%d)\n",
2242	device_buckets[i].drm_fd);
2243	ret = -EINVAL;
2244	goto exit;
2245	}
2246
2247	if (pdd->drm_file) {
2248	ret = -EINVAL;
2249	goto exit;
2250	}
2251
2252	/* create the vm using render nodes for kfd pdd */
2253	if (kfd_process_device_init_vm(pdd, drm_file)) {
2254	pr_err("could not init vm for given pdd\n");
2255	/* On success, the PDD keeps the drm_file reference */
2256	fput(drm_file);
2257	ret = -EINVAL;
2258	goto exit;
2259	}
2260	/*
2261	* pdd now already has the vm bound to render node so below api won't create a new
2262	* exclusive kfd mapping but use existing one with renderDXXX but is still needed
2263	* for iommu v2 binding and runtime pm.
2264	*/
2265	pdd = kfd_bind_process_to_device(dev, p);
2266	if (IS_ERR(ptr: pdd)) {
2267	ret = PTR_ERR(ptr: pdd);
2268	goto exit;
2269	}
2270
2271	if (!pdd->qpd.proc_doorbells) {
2272	ret = kfd_alloc_process_doorbells(kfd: dev->kfd, pdd);
2273	if (ret)
2274	goto exit;
2275	}
2276	}
2277
2278	/*
2279	* We are not copying device private data from user as we are not using the data for now,
2280	* but we still adjust for its private data.
2281	*/
2282	*priv_offset += args->num_devices * sizeof(*device_privs);
2283
2284	exit:
2285	kfree(objp: device_buckets);
2286	return ret;
2287	}
2288
2289	static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
2290	struct kfd_criu_bo_bucket *bo_bucket,
2291	struct kfd_criu_bo_priv_data *bo_priv,
2292	struct kgd_mem **kgd_mem)
2293	{
2294	int idr_handle;
2295	int ret;
2296	const bool criu_resume = true;
2297	u64 offset;
2298
2299	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
2300	if (bo_bucket->size !=
2301	kfd_doorbell_process_slice(kfd: pdd->dev->kfd))
2302	return -EINVAL;
2303
2304	offset = kfd_get_process_doorbells(pdd);
2305	if (!offset)
2306	return -ENOMEM;
2307	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2308	/* MMIO BOs need remapped bus address */
2309	if (bo_bucket->size != PAGE_SIZE) {
2310	pr_err("Invalid page size\n");
2311	return -EINVAL;
2312	}
2313	offset = pdd->dev->adev->rmmio_remap.bus_addr;
2314	if (!offset) {
2315	pr_err("amdgpu_amdkfd_get_mmio_remap_phys_addr failed\n");
2316	return -ENOMEM;
2317	}
2318	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
2319	offset = bo_priv->user_addr;
2320	}
2321	/* Create the BO */
2322	ret = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(adev: pdd->dev->adev, va: bo_bucket->addr,
2323	size: bo_bucket->size, drm_priv: pdd->drm_priv, mem: kgd_mem,
2324	offset: &offset, flags: bo_bucket->alloc_flags, criu_resume);
2325	if (ret) {
2326	pr_err("Could not create the BO\n");
2327	return ret;
2328	}
2329	pr_debug("New BO created: size:0x%llx addr:0x%llx offset:0x%llx\n",
2330	bo_bucket->size, bo_bucket->addr, offset);
2331
2332	/* Restore previous IDR handle */
2333	pr_debug("Restoring old IDR handle for the BO");
2334	idr_handle = idr_alloc(&pdd->alloc_idr, ptr: *kgd_mem, start: bo_priv->idr_handle,
2335	end: bo_priv->idr_handle + 1, GFP_KERNEL);
2336
2337	if (idr_handle < 0) {
2338	pr_err("Could not allocate idr\n");
2339	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: pdd->dev->adev, mem: *kgd_mem, drm_priv: pdd->drm_priv,
2340	NULL);
2341	return -ENOMEM;
2342	}
2343
2344	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
2345	bo_bucket->restored_offset = KFD_MMAP_TYPE_DOORBELL | KFD_MMAP_GPU_ID(pdd->dev->id);
2346	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2347	bo_bucket->restored_offset = KFD_MMAP_TYPE_MMIO | KFD_MMAP_GPU_ID(pdd->dev->id);
2348	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
2349	bo_bucket->restored_offset = offset;
2350	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
2351	bo_bucket->restored_offset = offset;
2352	/* Update the VRAM usage count */
2353	WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + bo_bucket->size);
2354	}
2355	return 0;
2356	}
2357
2358	static int criu_restore_bo(struct kfd_process *p,
2359	struct kfd_criu_bo_bucket *bo_bucket,
2360	struct kfd_criu_bo_priv_data *bo_priv)
2361	{
2362	struct kfd_process_device *pdd;
2363	struct kgd_mem *kgd_mem;
2364	int ret;
2365	int j;
2366
2367	pr_debug("Restoring BO size:0x%llx addr:0x%llx gpu_id:0x%x flags:0x%x idr_handle:0x%x\n",
2368	bo_bucket->size, bo_bucket->addr, bo_bucket->gpu_id, bo_bucket->alloc_flags,
2369	bo_priv->idr_handle);
2370
2371	pdd = kfd_process_device_data_by_id(process: p, gpu_id: bo_bucket->gpu_id);
2372	if (!pdd) {
2373	pr_err("Failed to get pdd\n");
2374	return -ENODEV;
2375	}
2376
2377	ret = criu_restore_memory_of_gpu(pdd, bo_bucket, bo_priv, kgd_mem: &kgd_mem);
2378	if (ret)
2379	return ret;
2380
2381	/* now map these BOs to GPU/s */
2382	for (j = 0; j < p->n_pdds; j++) {
2383	struct kfd_node *peer;
2384	struct kfd_process_device *peer_pdd;
2385
2386	if (!bo_priv->mapped_gpuids[j])
2387	break;
2388
2389	peer_pdd = kfd_process_device_data_by_id(process: p, gpu_id: bo_priv->mapped_gpuids[j]);
2390	if (!peer_pdd)
2391	return -EINVAL;
2392
2393	peer = peer_pdd->dev;
2394
2395	peer_pdd = kfd_bind_process_to_device(dev: peer, p);
2396	if (IS_ERR(ptr: peer_pdd))
2397	return PTR_ERR(ptr: peer_pdd);
2398
2399	ret = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(adev: peer->adev, mem: kgd_mem,
2400	drm_priv: peer_pdd->drm_priv);
2401	if (ret) {
2402	pr_err("Failed to map to gpu %d/%d\n", j, p->n_pdds);
2403	return ret;
2404	}
2405	}
2406
2407	pr_debug("map memory was successful for the BO\n");
2408	/* create the dmabuf object and export the bo */
2409	if (bo_bucket->alloc_flags
2410	& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
2411	ret = criu_get_prime_handle(mem: kgd_mem, DRM_RDWR,
2412	shared_fd: &bo_bucket->dmabuf_fd);
2413	if (ret)
2414	return ret;
2415	} else {
2416	bo_bucket->dmabuf_fd = KFD_INVALID_FD;
2417	}
2418
2419	return 0;
2420	}
2421
2422	static int criu_restore_bos(struct kfd_process *p,
2423	struct kfd_ioctl_criu_args *args,
2424	uint64_t *priv_offset,
2425	uint64_t max_priv_data_size)
2426	{
2427	struct kfd_criu_bo_bucket *bo_buckets = NULL;
2428	struct kfd_criu_bo_priv_data *bo_privs = NULL;
2429	int ret = 0;
2430	uint32_t i = 0;
2431
2432	if (*priv_offset + (args->num_bos * sizeof(*bo_privs)) > max_priv_data_size)
2433	return -EINVAL;
2434
2435	/* Prevent MMU notifications until stage-4 IOCTL (CRIU_RESUME) is received */
2436	amdgpu_amdkfd_block_mmu_notifications(p: p->kgd_process_info);
2437
2438	bo_buckets = kvmalloc_array(n: args->num_bos, size: sizeof(*bo_buckets), GFP_KERNEL);
2439	if (!bo_buckets)
2440	return -ENOMEM;
2441
2442	ret = copy_from_user(to: bo_buckets, from: (void __user *)args->bos,
2443	n: args->num_bos * sizeof(*bo_buckets));
2444	if (ret) {
2445	pr_err("Failed to copy BOs information from user\n");
2446	ret = -EFAULT;
2447	goto exit;
2448	}
2449
2450	bo_privs = kvmalloc_array(n: args->num_bos, size: sizeof(*bo_privs), GFP_KERNEL);
2451	if (!bo_privs) {
2452	ret = -ENOMEM;
2453	goto exit;
2454	}
2455
2456	ret = copy_from_user(to: bo_privs, from: (void __user *)args->priv_data + *priv_offset,
2457	n: args->num_bos * sizeof(*bo_privs));
2458	if (ret) {
2459	pr_err("Failed to copy BOs information from user\n");
2460	ret = -EFAULT;
2461	goto exit;
2462	}
2463	*priv_offset += args->num_bos * sizeof(*bo_privs);
2464
2465	/* Create and map new BOs */
2466	for (; i < args->num_bos; i++) {
2467	ret = criu_restore_bo(p, bo_bucket: &bo_buckets[i], bo_priv: &bo_privs[i]);
2468	if (ret) {
2469	pr_debug("Failed to restore BO[%d] ret%d\n", i, ret);
2470	goto exit;
2471	}
2472	} /* done */
2473
2474	/* Copy only the buckets back so user can read bo_buckets[N].restored_offset */
2475	ret = copy_to_user(to: (void __user *)args->bos,
2476	from: bo_buckets,
2477	n: (args->num_bos * sizeof(*bo_buckets)));
2478	if (ret)
2479	ret = -EFAULT;
2480
2481	exit:
2482	while (ret && i--) {
2483	if (bo_buckets[i].alloc_flags
2484	& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT))
2485	close_fd(fd: bo_buckets[i].dmabuf_fd);
2486	}
2487	kvfree(addr: bo_buckets);
2488	kvfree(addr: bo_privs);
2489	return ret;
2490	}
2491
2492	static int criu_restore_objects(struct file *filep,
2493	struct kfd_process *p,
2494	struct kfd_ioctl_criu_args *args,
2495	uint64_t *priv_offset,
2496	uint64_t max_priv_data_size)
2497	{
2498	int ret = 0;
2499	uint32_t i;
2500
2501	BUILD_BUG_ON(offsetof(struct kfd_criu_queue_priv_data, object_type));
2502	BUILD_BUG_ON(offsetof(struct kfd_criu_event_priv_data, object_type));
2503	BUILD_BUG_ON(offsetof(struct kfd_criu_svm_range_priv_data, object_type));
2504
2505	for (i = 0; i < args->num_objects; i++) {
2506	uint32_t object_type;
2507
2508	if (*priv_offset + sizeof(object_type) > max_priv_data_size) {
2509	pr_err("Invalid private data size\n");
2510	return -EINVAL;
2511	}
2512
2513	ret = get_user(object_type, (uint32_t __user *)(args->priv_data + *priv_offset));
2514	if (ret) {
2515	pr_err("Failed to copy private information from user\n");
2516	goto exit;
2517	}
2518
2519	switch (object_type) {
2520	case KFD_CRIU_OBJECT_TYPE_QUEUE:
2521	ret = kfd_criu_restore_queue(p, user_priv_data: (uint8_t __user *)args->priv_data,
2522	priv_data_offset: priv_offset, max_priv_data_size);
2523	if (ret)
2524	goto exit;
2525	break;
2526	case KFD_CRIU_OBJECT_TYPE_EVENT:
2527	ret = kfd_criu_restore_event(devkfd: filep, p, user_priv_data: (uint8_t __user *)args->priv_data,
2528	priv_data_offset: priv_offset, max_priv_data_size);
2529	if (ret)
2530	goto exit;
2531	break;
2532	case KFD_CRIU_OBJECT_TYPE_SVM_RANGE:
2533	ret = kfd_criu_restore_svm(p, user_priv_ptr: (uint8_t __user *)args->priv_data,
2534	priv_data_offset: priv_offset, max_priv_data_size);
2535	if (ret)
2536	goto exit;
2537	break;
2538	default:
2539	pr_err("Invalid object type:%u at index:%d\n", object_type, i);
2540	ret = -EINVAL;
2541	goto exit;
2542	}
2543	}
2544	exit:
2545	return ret;
2546	}
2547
2548	static int criu_restore(struct file *filep,
2549	struct kfd_process *p,
2550	struct kfd_ioctl_criu_args *args)
2551	{
2552	uint64_t priv_offset = 0;
2553	int ret = 0;
2554
2555	pr_debug("CRIU restore (num_devices:%u num_bos:%u num_objects:%u priv_data_size:%llu)\n",
2556	args->num_devices, args->num_bos, args->num_objects, args->priv_data_size);
2557
2558	if (!args->bos || !args->devices || !args->priv_data || !args->priv_data_size ||
2559	!args->num_devices || !args->num_bos)
2560	return -EINVAL;
2561
2562	mutex_lock(&p->mutex);
2563
2564	/*
2565	* Set the process to evicted state to avoid running any new queues before all the memory
2566	* mappings are ready.
2567	*/
2568	ret = kfd_process_evict_queues(p, trigger: KFD_QUEUE_EVICTION_CRIU_RESTORE);
2569	if (ret)
2570	goto exit_unlock;
2571
2572	/* Each function will adjust priv_offset based on how many bytes they consumed */
2573	ret = criu_restore_process(p, args, priv_offset: &priv_offset, max_priv_data_size: args->priv_data_size);
2574	if (ret)
2575	goto exit_unlock;
2576
2577	ret = criu_restore_devices(p, args, priv_offset: &priv_offset, max_priv_data_size: args->priv_data_size);
2578	if (ret)
2579	goto exit_unlock;
2580
2581	ret = criu_restore_bos(p, args, priv_offset: &priv_offset, max_priv_data_size: args->priv_data_size);
2582	if (ret)
2583	goto exit_unlock;
2584
2585	ret = criu_restore_objects(filep, p, args, priv_offset: &priv_offset, max_priv_data_size: args->priv_data_size);
2586	if (ret)
2587	goto exit_unlock;
2588
2589	if (priv_offset != args->priv_data_size) {
2590	pr_err("Invalid private data size\n");
2591	ret = -EINVAL;
2592	}
2593
2594	exit_unlock:
2595	mutex_unlock(lock: &p->mutex);
2596	if (ret)
2597	pr_err("Failed to restore CRIU ret:%d\n", ret);
2598	else
2599	pr_debug("CRIU restore successful\n");
2600
2601	return ret;
2602	}
2603
2604	static int criu_unpause(struct file *filep,
2605	struct kfd_process *p,
2606	struct kfd_ioctl_criu_args *args)
2607	{
2608	int ret;
2609
2610	mutex_lock(&p->mutex);
2611
2612	if (!p->queues_paused) {
2613	mutex_unlock(lock: &p->mutex);
2614	return -EINVAL;
2615	}
2616
2617	ret = kfd_process_restore_queues(p);
2618	if (ret)
2619	pr_err("Failed to unpause queues ret:%d\n", ret);
2620	else
2621	p->queues_paused = false;
2622
2623	mutex_unlock(lock: &p->mutex);
2624
2625	return ret;
2626	}
2627
2628	static int criu_resume(struct file *filep,
2629	struct kfd_process *p,
2630	struct kfd_ioctl_criu_args *args)
2631	{
2632	struct kfd_process *target = NULL;
2633	struct pid *pid = NULL;
2634	int ret = 0;
2635
2636	pr_debug("Inside %s, target pid for criu restore: %d\n", __func__,
2637	args->pid);
2638
2639	pid = find_get_pid(nr: args->pid);
2640	if (!pid) {
2641	pr_err("Cannot find pid info for %i\n", args->pid);
2642	return -ESRCH;
2643	}
2644
2645	pr_debug("calling kfd_lookup_process_by_pid\n");
2646	target = kfd_lookup_process_by_pid(pid);
2647
2648	put_pid(pid);
2649
2650	if (!target) {
2651	pr_debug("Cannot find process info for %i\n", args->pid);
2652	return -ESRCH;
2653	}
2654
2655	mutex_lock(&target->mutex);
2656	ret = kfd_criu_resume_svm(p: target);
2657	if (ret) {
2658	pr_err("kfd_criu_resume_svm failed for %i\n", args->pid);
2659	goto exit;
2660	}
2661
2662	ret = amdgpu_amdkfd_criu_resume(p: target->kgd_process_info);
2663	if (ret)
2664	pr_err("amdgpu_amdkfd_criu_resume failed for %i\n", args->pid);
2665
2666	exit:
2667	mutex_unlock(lock: &target->mutex);
2668
2669	kfd_unref_process(p: target);
2670	return ret;
2671	}
2672
2673	static int criu_process_info(struct file *filep,
2674	struct kfd_process *p,
2675	struct kfd_ioctl_criu_args *args)
2676	{
2677	int ret = 0;
2678
2679	mutex_lock(&p->mutex);
2680
2681	if (!p->n_pdds) {
2682	pr_err("No pdd for given process\n");
2683	ret = -ENODEV;
2684	goto err_unlock;
2685	}
2686
2687	ret = kfd_process_evict_queues(p, trigger: KFD_QUEUE_EVICTION_CRIU_CHECKPOINT);
2688	if (ret)
2689	goto err_unlock;
2690
2691	p->queues_paused = true;
2692
2693	args->pid = task_pid_nr_ns(tsk: p->lead_thread,
2694	ns: task_active_pid_ns(tsk: p->lead_thread));
2695
2696	ret = criu_get_process_object_info(p, num_devices: &args->num_devices, num_bos: &args->num_bos,
2697	num_objects: &args->num_objects, objs_priv_size: &args->priv_data_size);
2698	if (ret)
2699	goto err_unlock;
2700
2701	dev_dbg(kfd_device, "Num of devices:%u bos:%u objects:%u priv_data_size:%lld\n",
2702	args->num_devices, args->num_bos, args->num_objects,
2703	args->priv_data_size);
2704
2705	err_unlock:
2706	if (ret) {
2707	kfd_process_restore_queues(p);
2708	p->queues_paused = false;
2709	}
2710	mutex_unlock(lock: &p->mutex);
2711	return ret;
2712	}
2713
2714	static int kfd_ioctl_criu(struct file *filep, struct kfd_process *p, void *data)
2715	{
2716	struct kfd_ioctl_criu_args *args = data;
2717	int ret;
2718
2719	dev_dbg(kfd_device, "CRIU operation: %d\n", args->op);
2720	switch (args->op) {
2721	case KFD_CRIU_OP_PROCESS_INFO:
2722	ret = criu_process_info(filep, p, args);
2723	break;
2724	case KFD_CRIU_OP_CHECKPOINT:
2725	ret = criu_checkpoint(filep, p, args);
2726	break;
2727	case KFD_CRIU_OP_UNPAUSE:
2728	ret = criu_unpause(filep, p, args);
2729	break;
2730	case KFD_CRIU_OP_RESTORE:
2731	ret = criu_restore(filep, p, args);
2732	break;
2733	case KFD_CRIU_OP_RESUME:
2734	ret = criu_resume(filep, p, args);
2735	break;
2736	default:
2737	dev_dbg(kfd_device, "Unsupported CRIU operation:%d\n", args->op);
2738	ret = -EINVAL;
2739	break;
2740	}
2741
2742	if (ret)
2743	dev_dbg(kfd_device, "CRIU operation:%d err:%d\n", args->op, ret);
2744
2745	return ret;
2746	}
2747
2748	static int runtime_enable(struct kfd_process *p, uint64_t r_debug,
2749	bool enable_ttmp_setup)
2750	{
2751	int i = 0, ret = 0;
2752
2753	if (p->is_runtime_retry)
2754	goto retry;
2755
2756	if (p->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_DISABLED)
2757	return -EBUSY;
2758
2759	for (i = 0; i < p->n_pdds; i++) {
2760	struct kfd_process_device *pdd = p->pdds[i];
2761
2762	if (pdd->qpd.queue_count)
2763	return -EEXIST;
2764
2765	/*
2766	* Setup TTMPs by default.
2767	* Note that this call must remain here for MES ADD QUEUE to
2768	* skip_process_ctx_clear unconditionally as the first call to
2769	* SET_SHADER_DEBUGGER clears any stale process context data
2770	* saved in MES.
2771	*/
2772	if (pdd->dev->kfd->shared_resources.enable_mes)
2773	kfd_dbg_set_mes_debug_mode(pdd, sq_trap_en: !kfd_dbg_has_cwsr_workaround(dev: pdd->dev));
2774	}
2775
2776	p->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_ENABLED;
2777	p->runtime_info.r_debug = r_debug;
2778	p->runtime_info.ttmp_setup = enable_ttmp_setup;
2779
2780	if (p->runtime_info.ttmp_setup) {
2781	for (i = 0; i < p->n_pdds; i++) {
2782	struct kfd_process_device *pdd = p->pdds[i];
2783
2784	if (!kfd_dbg_is_rlc_restore_supported(dev: pdd->dev)) {
2785	amdgpu_gfx_off_ctrl(adev: pdd->dev->adev, enable: false);
2786	pdd->dev->kfd2kgd->enable_debug_trap(
2787	pdd->dev->adev,
2788	true,
2789	pdd->dev->vm_info.last_vmid_kfd);
2790	} else if (kfd_dbg_is_per_vmid_supported(dev: pdd->dev)) {
2791	pdd->spi_dbg_override = pdd->dev->kfd2kgd->enable_debug_trap(
2792	pdd->dev->adev,
2793	false,
2794	0);
2795	}
2796	}
2797	}
2798
2799	retry:
2800	if (p->debug_trap_enabled) {
2801	if (!p->is_runtime_retry) {
2802	kfd_dbg_trap_activate(target: p);
2803	kfd_dbg_ev_raise(KFD_EC_MASK(EC_PROCESS_RUNTIME),
2804	process: p, NULL, source_id: 0, use_worker: false, NULL, exception_data_size: 0);
2805	}
2806
2807	mutex_unlock(lock: &p->mutex);
2808	ret = down_interruptible(sem: &p->runtime_enable_sema);
2809	mutex_lock(&p->mutex);
2810
2811	p->is_runtime_retry = !!ret;
2812	}
2813
2814	return ret;
2815	}
2816
2817	static int runtime_disable(struct kfd_process *p)
2818	{
2819	int i = 0, ret;
2820	bool was_enabled = p->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED;
2821
2822	p->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_DISABLED;
2823	p->runtime_info.r_debug = 0;
2824
2825	if (p->debug_trap_enabled) {
2826	if (was_enabled)
2827	kfd_dbg_trap_deactivate(target: p, unwind: false, unwind_count: 0);
2828
2829	if (!p->is_runtime_retry)
2830	kfd_dbg_ev_raise(KFD_EC_MASK(EC_PROCESS_RUNTIME),
2831	process: p, NULL, source_id: 0, use_worker: false, NULL, exception_data_size: 0);
2832
2833	mutex_unlock(lock: &p->mutex);
2834	ret = down_interruptible(sem: &p->runtime_enable_sema);
2835	mutex_lock(&p->mutex);
2836
2837	p->is_runtime_retry = !!ret;
2838	if (ret)
2839	return ret;
2840	}
2841
2842	if (was_enabled && p->runtime_info.ttmp_setup) {
2843	for (i = 0; i < p->n_pdds; i++) {
2844	struct kfd_process_device *pdd = p->pdds[i];
2845
2846	if (!kfd_dbg_is_rlc_restore_supported(dev: pdd->dev))
2847	amdgpu_gfx_off_ctrl(adev: pdd->dev->adev, enable: true);
2848	}
2849	}
2850
2851	p->runtime_info.ttmp_setup = false;
2852
2853	/* disable ttmp setup */
2854	for (i = 0; i < p->n_pdds; i++) {
2855	struct kfd_process_device *pdd = p->pdds[i];
2856
2857	if (kfd_dbg_is_per_vmid_supported(dev: pdd->dev)) {
2858	pdd->spi_dbg_override =
2859	pdd->dev->kfd2kgd->disable_debug_trap(
2860	pdd->dev->adev,
2861	false,
2862	pdd->dev->vm_info.last_vmid_kfd);
2863
2864	if (!pdd->dev->kfd->shared_resources.enable_mes)
2865	debug_refresh_runlist(dqm: pdd->dev->dqm);
2866	else
2867	kfd_dbg_set_mes_debug_mode(pdd,
2868	sq_trap_en: !kfd_dbg_has_cwsr_workaround(dev: pdd->dev));
2869	}
2870	}
2871
2872	return 0;
2873	}
2874
2875	static int kfd_ioctl_runtime_enable(struct file *filep, struct kfd_process *p, void *data)
2876	{
2877	struct kfd_ioctl_runtime_enable_args *args = data;
2878	int r;
2879
2880	mutex_lock(&p->mutex);
2881
2882	if (args->mode_mask & KFD_RUNTIME_ENABLE_MODE_ENABLE_MASK)
2883	r = runtime_enable(p, r_debug: args->r_debug,
2884	enable_ttmp_setup: !!(args->mode_mask & KFD_RUNTIME_ENABLE_MODE_TTMP_SAVE_MASK));
2885	else
2886	r = runtime_disable(p);
2887
2888	mutex_unlock(lock: &p->mutex);
2889
2890	return r;
2891	}
2892
2893	static int kfd_ioctl_set_debug_trap(struct file *filep, struct kfd_process *p, void *data)
2894	{
2895	struct kfd_ioctl_dbg_trap_args *args = data;
2896	struct task_struct *thread = NULL;
2897	struct mm_struct *mm = NULL;
2898	struct pid *pid = NULL;
2899	struct kfd_process *target = NULL;
2900	struct kfd_process_device *pdd = NULL;
2901	int r = 0;
2902
2903	if (sched_policy == KFD_SCHED_POLICY_NO_HWS) {
2904	pr_err("Debugging does not support sched_policy %i", sched_policy);
2905	return -EINVAL;
2906	}
2907
2908	pid = find_get_pid(nr: args->pid);
2909	if (!pid) {
2910	pr_debug("Cannot find pid info for %i\n", args->pid);
2911	r = -ESRCH;
2912	goto out;
2913	}
2914
2915	thread = get_pid_task(pid, PIDTYPE_PID);
2916	if (!thread) {
2917	r = -ESRCH;
2918	goto out;
2919	}
2920
2921	mm = get_task_mm(task: thread);
2922	if (!mm) {
2923	r = -ESRCH;
2924	goto out;
2925	}
2926
2927	if (args->op == KFD_IOC_DBG_TRAP_ENABLE) {
2928	bool create_process;
2929
2930	rcu_read_lock();
2931	create_process = thread && thread != current && ptrace_parent(task: thread) == current;
2932	rcu_read_unlock();
2933
2934	target = create_process ? kfd_create_process(thread) :
2935	kfd_lookup_process_by_pid(pid);
2936	} else {
2937	target = kfd_lookup_process_by_pid(pid);
2938	}
2939
2940	if (IS_ERR_OR_NULL(ptr: target)) {
2941	pr_debug("Cannot find process PID %i to debug\n", args->pid);
2942	r = target ? PTR_ERR(ptr: target) : -ESRCH;
2943	goto out;
2944	}
2945
2946	/* Check if target is still PTRACED. */
2947	rcu_read_lock();
2948	if (target != p && args->op != KFD_IOC_DBG_TRAP_DISABLE
2949	&& ptrace_parent(task: target->lead_thread) != current) {
2950	pr_err("PID %i is not PTRACED and cannot be debugged\n", args->pid);
2951	r = -EPERM;
2952	}
2953	rcu_read_unlock();
2954
2955	if (r)
2956	goto out;
2957
2958	mutex_lock(&target->mutex);
2959
2960	if (args->op != KFD_IOC_DBG_TRAP_ENABLE && !target->debug_trap_enabled) {
2961	pr_err("PID %i not debug enabled for op %i\n", args->pid, args->op);
2962	r = -EINVAL;
2963	goto unlock_out;
2964	}
2965
2966	if (target->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_ENABLED &&
2967	(args->op == KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE ||
2968	args->op == KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE ||
2969	args->op == KFD_IOC_DBG_TRAP_SUSPEND_QUEUES ||
2970	args->op == KFD_IOC_DBG_TRAP_RESUME_QUEUES ||
2971	args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ||
2972	args->op == KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH ||
2973	args->op == KFD_IOC_DBG_TRAP_SET_FLAGS)) {
2974	r = -EPERM;
2975	goto unlock_out;
2976	}
2977
2978	if (args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ||
2979	args->op == KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH) {
2980	int user_gpu_id = kfd_process_get_user_gpu_id(p: target,
2981	actual_gpu_id: args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ?
2982	args->set_node_address_watch.gpu_id :
2983	args->clear_node_address_watch.gpu_id);
2984
2985	pdd = kfd_process_device_data_by_id(process: target, gpu_id: user_gpu_id);
2986	if (user_gpu_id == -EINVAL || !pdd) {
2987	r = -ENODEV;
2988	goto unlock_out;
2989	}
2990	}
2991
2992	switch (args->op) {
2993	case KFD_IOC_DBG_TRAP_ENABLE:
2994	if (target != p)
2995	target->debugger_process = p;
2996
2997	r = kfd_dbg_trap_enable(target,
2998	fd: args->enable.dbg_fd,
2999	runtime_info: (void __user *)args->enable.rinfo_ptr,
3000	runtime_info_size: &args->enable.rinfo_size);
3001	if (!r)
3002	target->exception_enable_mask = args->enable.exception_mask;
3003
3004	break;
3005	case KFD_IOC_DBG_TRAP_DISABLE:
3006	r = kfd_dbg_trap_disable(target);
3007	break;
3008	case KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT:
3009	r = kfd_dbg_send_exception_to_runtime(p: target,
3010	dev_id: args->send_runtime_event.gpu_id,
3011	queue_id: args->send_runtime_event.queue_id,
3012	error_reason: args->send_runtime_event.exception_mask);
3013	break;
3014	case KFD_IOC_DBG_TRAP_SET_EXCEPTIONS_ENABLED:
3015	kfd_dbg_set_enabled_debug_exception_mask(target,
3016	exception_set_mask: args->set_exceptions_enabled.exception_mask);
3017	break;
3018	case KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE:
3019	r = kfd_dbg_trap_set_wave_launch_override(target,
3020	trap_override: args->launch_override.override_mode,
3021	trap_mask_bits: args->launch_override.enable_mask,
3022	trap_mask_request: args->launch_override.support_request_mask,
3023	trap_mask_prev: &args->launch_override.enable_mask,
3024	trap_mask_supported: &args->launch_override.support_request_mask);
3025	break;
3026	case KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE:
3027	r = kfd_dbg_trap_set_wave_launch_mode(target,
3028	wave_launch_mode: args->launch_mode.launch_mode);
3029	break;
3030	case KFD_IOC_DBG_TRAP_SUSPEND_QUEUES:
3031	r = suspend_queues(p: target,
3032	num_queues: args->suspend_queues.num_queues,
3033	grace_period: args->suspend_queues.grace_period,
3034	exception_clear_mask: args->suspend_queues.exception_mask,
3035	usr_queue_id_array: (uint32_t *)args->suspend_queues.queue_array_ptr);
3036
3037	break;
3038	case KFD_IOC_DBG_TRAP_RESUME_QUEUES:
3039	r = resume_queues(p: target, num_queues: args->resume_queues.num_queues,
3040	usr_queue_id_array: (uint32_t *)args->resume_queues.queue_array_ptr);
3041	break;
3042	case KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH:
3043	r = kfd_dbg_trap_set_dev_address_watch(pdd,
3044	watch_address: args->set_node_address_watch.address,
3045	watch_address_mask: args->set_node_address_watch.mask,
3046	watch_id: &args->set_node_address_watch.id,
3047	watch_mode: args->set_node_address_watch.mode);
3048	break;
3049	case KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH:
3050	r = kfd_dbg_trap_clear_dev_address_watch(pdd,
3051	watch_id: args->clear_node_address_watch.id);
3052	break;
3053	case KFD_IOC_DBG_TRAP_SET_FLAGS:
3054	r = kfd_dbg_trap_set_flags(target, flags: &args->set_flags.flags);
3055	break;
3056	case KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT:
3057	r = kfd_dbg_ev_query_debug_event(process: target,
3058	queue_id: &args->query_debug_event.queue_id,
3059	gpu_id: &args->query_debug_event.gpu_id,
3060	exception_clear_mask: args->query_debug_event.exception_mask,
3061	event_status: &args->query_debug_event.exception_mask);
3062	break;
3063	case KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO:
3064	r = kfd_dbg_trap_query_exception_info(target,
3065	source_id: args->query_exception_info.source_id,
3066	exception_code: args->query_exception_info.exception_code,
3067	clear_exception: args->query_exception_info.clear_exception,
3068	info: (void __user *)args->query_exception_info.info_ptr,
3069	info_size: &args->query_exception_info.info_size);
3070	break;
3071	case KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT:
3072	r = pqm_get_queue_snapshot(pqm: &target->pqm,
3073	exception_clear_mask: args->queue_snapshot.exception_mask,
3074	buf: (void __user *)args->queue_snapshot.snapshot_buf_ptr,
3075	num_qss_entries: &args->queue_snapshot.num_queues,
3076	entry_size: &args->queue_snapshot.entry_size);
3077	break;
3078	case KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT:
3079	r = kfd_dbg_trap_device_snapshot(target,
3080	exception_clear_mask: args->device_snapshot.exception_mask,
3081	user_info: (void __user *)args->device_snapshot.snapshot_buf_ptr,
3082	number_of_device_infos: &args->device_snapshot.num_devices,
3083	entry_size: &args->device_snapshot.entry_size);
3084	break;
3085	default:
3086	pr_err("Invalid option: %i\n", args->op);
3087	r = -EINVAL;
3088	}
3089
3090	unlock_out:
3091	mutex_unlock(lock: &target->mutex);
3092
3093	out:
3094	if (thread)
3095	put_task_struct(t: thread);
3096
3097	if (mm)
3098	mmput(mm);
3099
3100	if (pid)
3101	put_pid(pid);
3102
3103	if (target)
3104	kfd_unref_process(p: target);
3105
3106	return r;
3107	}
3108
3109	#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
3110	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
3111	.cmd_drv = 0, .name = #ioctl}
3112
3113	/** Ioctl table */
3114	static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
3115	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
3116	kfd_ioctl_get_version, 0),
3117
3118	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
3119	kfd_ioctl_create_queue, 0),
3120
3121	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
3122	kfd_ioctl_destroy_queue, 0),
3123
3124	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
3125	kfd_ioctl_set_memory_policy, 0),
3126
3127	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
3128	kfd_ioctl_get_clock_counters, 0),
3129
3130	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
3131	kfd_ioctl_get_process_apertures, 0),
3132
3133	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
3134	kfd_ioctl_update_queue, 0),
3135
3136	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT,
3137	kfd_ioctl_create_event, 0),
3138
3139	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT,
3140	kfd_ioctl_destroy_event, 0),
3141
3142	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT,
3143	kfd_ioctl_set_event, 0),
3144
3145	AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT,
3146	kfd_ioctl_reset_event, 0),
3147
3148	AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS,
3149	kfd_ioctl_wait_events, 0),
3150
3151	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER_DEPRECATED,
3152	kfd_ioctl_dbg_register, 0),
3153
3154	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED,
3155	kfd_ioctl_dbg_unregister, 0),
3156
3157	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED,
3158	kfd_ioctl_dbg_address_watch, 0),
3159
3160	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED,
3161	kfd_ioctl_dbg_wave_control, 0),
3162
3163	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA,
3164	kfd_ioctl_set_scratch_backing_va, 0),
3165
3166	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
3167	kfd_ioctl_get_tile_config, 0),
3168
3169	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
3170	kfd_ioctl_set_trap_handler, 0),
3171
3172	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
3173	kfd_ioctl_get_process_apertures_new, 0),
3174
3175	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ACQUIRE_VM,
3176	kfd_ioctl_acquire_vm, 0),
3177
3178	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
3179	kfd_ioctl_alloc_memory_of_gpu, 0),
3180
3181	AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
3182	kfd_ioctl_free_memory_of_gpu, 0),
3183
3184	AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
3185	kfd_ioctl_map_memory_to_gpu, 0),
3186
3187	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
3188	kfd_ioctl_unmap_memory_from_gpu, 0),
3189
3190	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
3191	kfd_ioctl_set_cu_mask, 0),
3192
3193	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
3194	kfd_ioctl_get_queue_wave_state, 0),
3195
3196	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
3197	kfd_ioctl_get_dmabuf_info, 0),
3198
3199	AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
3200	kfd_ioctl_import_dmabuf, 0),
3201
3202	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
3203	kfd_ioctl_alloc_queue_gws, 0),
3204
3205	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
3206	kfd_ioctl_smi_events, 0),
3207
3208	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
3209
3210	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
3211	kfd_ioctl_set_xnack_mode, 0),
3212
3213	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CRIU_OP,
3214	kfd_ioctl_criu, KFD_IOC_FLAG_CHECKPOINT_RESTORE),
3215
3216	AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
3217	kfd_ioctl_get_available_memory, 0),
3218
3219	AMDKFD_IOCTL_DEF(AMDKFD_IOC_EXPORT_DMABUF,
3220	kfd_ioctl_export_dmabuf, 0),
3221
3222	AMDKFD_IOCTL_DEF(AMDKFD_IOC_RUNTIME_ENABLE,
3223	kfd_ioctl_runtime_enable, 0),
3224
3225	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_TRAP,
3226	kfd_ioctl_set_debug_trap, 0),
3227	};
3228
3229	#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
3230
3231	static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
3232	{
3233	struct kfd_process *process;
3234	amdkfd_ioctl_t *func;
3235	const struct amdkfd_ioctl_desc *ioctl = NULL;
3236	unsigned int nr = _IOC_NR(cmd);
3237	char stack_kdata[128];
3238	char *kdata = NULL;
3239	unsigned int usize, asize;
3240	int retcode = -EINVAL;
3241	bool ptrace_attached = false;
3242
3243	if (nr >= AMDKFD_CORE_IOCTL_COUNT)
3244	goto err_i1;
3245
3246	if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
3247	u32 amdkfd_size;
3248
3249	ioctl = &amdkfd_ioctls[nr];
3250
3251	amdkfd_size = _IOC_SIZE(ioctl->cmd);
3252	usize = asize = _IOC_SIZE(cmd);
3253	if (amdkfd_size > asize)
3254	asize = amdkfd_size;
3255
3256	cmd = ioctl->cmd;
3257	} else
3258	goto err_i1;
3259
3260	dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg);
3261
3262	/* Get the process struct from the filep. Only the process
3263	* that opened /dev/kfd can use the file descriptor. Child
3264	* processes need to create their own KFD device context.
3265	*/
3266	process = filep->private_data;
3267
3268	rcu_read_lock();
3269	if ((ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE) &&
3270	ptrace_parent(task: process->lead_thread) == current)
3271	ptrace_attached = true;
3272	rcu_read_unlock();
3273
3274	if (process->lead_thread != current->group_leader
3275	&& !ptrace_attached) {
3276	dev_dbg(kfd_device, "Using KFD FD in wrong process\n");
3277	retcode = -EBADF;
3278	goto err_i1;
3279	}
3280
3281	/* Do not trust userspace, use our own definition */
3282	func = ioctl->func;
3283
3284	if (unlikely(!func)) {
3285	dev_dbg(kfd_device, "no function\n");
3286	retcode = -EINVAL;
3287	goto err_i1;
3288	}
3289
3290	/*
3291	* Versions of docker shipped in Ubuntu 18.xx and 20.xx do not support
3292	* CAP_CHECKPOINT_RESTORE, so we also allow access if CAP_SYS_ADMIN as CAP_SYS_ADMIN is a
3293	* more priviledged access.
3294	*/
3295	if (unlikely(ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE)) {
3296	if (!capable(CAP_CHECKPOINT_RESTORE) &&
3297	!capable(CAP_SYS_ADMIN)) {
3298	retcode = -EACCES;
3299	goto err_i1;
3300	}
3301	}
3302
3303	if (cmd & (IOC_IN | IOC_OUT)) {
3304	if (asize <= sizeof(stack_kdata)) {
3305	kdata = stack_kdata;
3306	} else {
3307	kdata = kmalloc(size: asize, GFP_KERNEL);
3308	if (!kdata) {
3309	retcode = -ENOMEM;
3310	goto err_i1;
3311	}
3312	}
3313	if (asize > usize)
3314	memset(kdata + usize, 0, asize - usize);
3315	}
3316
3317	if (cmd & IOC_IN) {
3318	if (copy_from_user(to: kdata, from: (void __user *)arg, n: usize) != 0) {
3319	retcode = -EFAULT;
3320	goto err_i1;
3321	}
3322	} else if (cmd & IOC_OUT) {
3323	memset(kdata, 0, usize);
3324	}
3325
3326	retcode = func(filep, process, kdata);
3327
3328	if (cmd & IOC_OUT)
3329	if (copy_to_user(to: (void __user *)arg, from: kdata, n: usize) != 0)
3330	retcode = -EFAULT;
3331
3332	err_i1:
3333	if (!ioctl)
3334	dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
3335	task_pid_nr(current), cmd, nr);
3336
3337	if (kdata != stack_kdata)
3338	kfree(objp: kdata);
3339
3340	if (retcode)
3341	dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n",
3342	nr, arg, retcode);
3343
3344	return retcode;
3345	}
3346
3347	static int kfd_mmio_mmap(struct kfd_node *dev, struct kfd_process *process,
3348	struct vm_area_struct *vma)
3349	{
3350	phys_addr_t address;
3351
3352	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
3353	return -EINVAL;
3354
3355	address = dev->adev->rmmio_remap.bus_addr;
3356
3357	vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
3358	VM_DONTDUMP | VM_PFNMAP);
3359
3360	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
3361
3362	pr_debug("pasid 0x%x mapping mmio page\n"
3363	" target user address == 0x%08llX\n"
3364	" physical address == 0x%08llX\n"
3365	" vm_flags == 0x%04lX\n"
3366	" size == 0x%04lX\n",
3367	process->pasid, (unsigned long long) vma->vm_start,
3368	address, vma->vm_flags, PAGE_SIZE);
3369
3370	return io_remap_pfn_range(vma,
3371	addr: vma->vm_start,
3372	pfn: address >> PAGE_SHIFT,
3373	PAGE_SIZE,
3374	prot: vma->vm_page_prot);
3375	}
3376
3377
3378	static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
3379	{
3380	struct kfd_process *process;
3381	struct kfd_node *dev = NULL;
3382	unsigned long mmap_offset;
3383	unsigned int gpu_id;
3384
3385	process = kfd_get_process(current);
3386	if (IS_ERR(ptr: process))
3387	return PTR_ERR(ptr: process);
3388
3389	mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
3390	gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
3391	if (gpu_id)
3392	dev = kfd_device_by_id(gpu_id);
3393
3394	switch (mmap_offset & KFD_MMAP_TYPE_MASK) {
3395	case KFD_MMAP_TYPE_DOORBELL:
3396	if (!dev)
3397	return -ENODEV;
3398	return kfd_doorbell_mmap(dev, process, vma);
3399
3400	case KFD_MMAP_TYPE_EVENTS:
3401	return kfd_event_mmap(process, vma);
3402
3403	case KFD_MMAP_TYPE_RESERVED_MEM:
3404	if (!dev)
3405	return -ENODEV;
3406	return kfd_reserved_mem_mmap(dev, process, vma);
3407	case KFD_MMAP_TYPE_MMIO:
3408	if (!dev)
3409	return -ENODEV;
3410	return kfd_mmio_mmap(dev, process, vma);
3411	}
3412
3413	return -EFAULT;
3414	}
3415