1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */
24
25#ifndef AMDGPU_AMDKFD_H_INCLUDED
26#define AMDGPU_AMDKFD_H_INCLUDED
27
28#include <linux/types.h>
29#include <linux/mm.h>
30#include <linux/workqueue.h>
31#include <kgd_kfd_interface.h>
32#include <drm/ttm/ttm_execbuf_util.h>
33#include "amdgpu_sync.h"
34#include "amdgpu_vm.h"
35
36extern uint64_t amdgpu_amdkfd_total_mem_size;
37
38struct amdgpu_device;
39
40struct kfd_bo_va_list {
41 struct list_head bo_list;
42 struct amdgpu_bo_va *bo_va;
43 void *kgd_dev;
44 bool is_mapped;
45 uint64_t va;
46 uint64_t pte_flags;
47};
48
49struct kgd_mem {
50 struct mutex lock;
51 struct amdgpu_bo *bo;
52 struct list_head bo_va_list;
53 /* protected by amdkfd_process_info.lock */
54 struct ttm_validate_buffer validate_list;
55 struct ttm_validate_buffer resv_list;
56 uint32_t domain;
57 unsigned int mapped_to_gpu_memory;
58 uint64_t va;
59
60 uint32_t mapping_flags;
61
62 atomic_t invalid;
63 struct amdkfd_process_info *process_info;
64 struct page **user_pages;
65
66 struct amdgpu_sync sync;
67
68 bool aql_queue;
69};
70
71/* KFD Memory Eviction */
72struct amdgpu_amdkfd_fence {
73 struct dma_fence base;
74 struct mm_struct *mm;
75 spinlock_t lock;
76 char timeline_name[TASK_COMM_LEN];
77};
78
79struct amdgpu_kfd_dev {
80 struct kfd_dev *dev;
81 uint64_t vram_used;
82};
83
84struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
85 struct mm_struct *mm);
86bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm);
87struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f);
88
89struct amdkfd_process_info {
90 /* List head of all VMs that belong to a KFD process */
91 struct list_head vm_list_head;
92 /* List head for all KFD BOs that belong to a KFD process. */
93 struct list_head kfd_bo_list;
94 /* List of userptr BOs that are valid or invalid */
95 struct list_head userptr_valid_list;
96 struct list_head userptr_inval_list;
97 /* Lock to protect kfd_bo_list */
98 struct mutex lock;
99
100 /* Number of VMs */
101 unsigned int n_vms;
102 /* Eviction Fence */
103 struct amdgpu_amdkfd_fence *eviction_fence;
104
105 /* MMU-notifier related fields */
106 atomic_t evicted_bos;
107 struct delayed_work restore_userptr_work;
108 struct pid *pid;
109};
110
111int amdgpu_amdkfd_init(void);
112void amdgpu_amdkfd_fini(void);
113
114void amdgpu_amdkfd_suspend(struct amdgpu_device *adev);
115int amdgpu_amdkfd_resume(struct amdgpu_device *adev);
116void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
117 const void *ih_ring_entry);
118void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
119void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
120void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);
121
122int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm);
123int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
124 uint32_t vmid, uint64_t gpu_addr,
125 uint32_t *ib_cmd, uint32_t ib_len);
126void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);
127
128struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
129struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
130struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void);
131
132bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
133
134int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev);
135
136int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);
137
138void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd);
139
140/* Shared API */
141int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
142 void **mem_obj, uint64_t *gpu_addr,
143 void **cpu_ptr, bool mqd_gfx9);
144void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
145void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
146 struct kfd_local_mem_info *mem_info);
147uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct kgd_dev *kgd);
148
149uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
150void amdgpu_amdkfd_get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info);
151int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
152 struct kgd_dev **dmabuf_kgd,
153 uint64_t *bo_size, void *metadata_buffer,
154 size_t buffer_size, uint32_t *metadata_size,
155 uint32_t *flags);
156uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
157uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd);
158
159#define read_user_wptr(mmptr, wptr, dst) \
160 ({ \
161 bool valid = false; \
162 if ((mmptr) && (wptr)) { \
163 if ((mmptr) == current->mm) { \
164 valid = !get_user((dst), (wptr)); \
165 } else if (current->mm == NULL) { \
166 use_mm(mmptr); \
167 valid = !get_user((dst), (wptr)); \
168 unuse_mm(mmptr); \
169 } \
170 } \
171 valid; \
172 })
173
174/* GPUVM API */
175int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, unsigned int pasid,
176 void **vm, void **process_info,
177 struct dma_fence **ef);
178int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
179 struct file *filp, unsigned int pasid,
180 void **vm, void **process_info,
181 struct dma_fence **ef);
182void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
183 struct amdgpu_vm *vm);
184void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm);
185void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *vm);
186uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
187int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
188 struct kgd_dev *kgd, uint64_t va, uint64_t size,
189 void *vm, struct kgd_mem **mem,
190 uint64_t *offset, uint32_t flags);
191int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
192 struct kgd_dev *kgd, struct kgd_mem *mem);
193int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
194 struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
195int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
196 struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
197int amdgpu_amdkfd_gpuvm_sync_memory(
198 struct kgd_dev *kgd, struct kgd_mem *mem, bool intr);
199int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
200 struct kgd_mem *mem, void **kptr, uint64_t *size);
201int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
202 struct dma_fence **ef);
203
204int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
205 struct kfd_vm_fault_info *info);
206
207int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
208 struct dma_buf *dmabuf,
209 uint64_t va, void *vm,
210 struct kgd_mem **mem, uint64_t *size,
211 uint64_t *mmap_offset);
212
213void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
214void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo);
215
216/* KGD2KFD callbacks */
217int kgd2kfd_init(void);
218void kgd2kfd_exit(void);
219struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev,
220 const struct kfd2kgd_calls *f2g);
221bool kgd2kfd_device_init(struct kfd_dev *kfd,
222 const struct kgd2kfd_shared_resources *gpu_resources);
223void kgd2kfd_device_exit(struct kfd_dev *kfd);
224void kgd2kfd_suspend(struct kfd_dev *kfd);
225int kgd2kfd_resume(struct kfd_dev *kfd);
226int kgd2kfd_pre_reset(struct kfd_dev *kfd);
227int kgd2kfd_post_reset(struct kfd_dev *kfd);
228void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
229int kgd2kfd_quiesce_mm(struct mm_struct *mm);
230int kgd2kfd_resume_mm(struct mm_struct *mm);
231int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
232 struct dma_fence *fence);
233
234#endif /* AMDGPU_AMDKFD_H_INCLUDED */
235