1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* |
3 | * Copyright 2014-2022 Advanced Micro Devices, Inc. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * copy of this software and associated documentation files (the "Software"), |
7 | * to deal in the Software without restriction, including without limitation |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
9 | * and/or sell copies of the Software, and to permit persons to whom the |
10 | * Software is furnished to do so, subject to the following conditions: |
11 | * |
12 | * The above copyright notice and this permission notice shall be included in |
13 | * all copies or substantial portions of the Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
21 | * OTHER DEALINGS IN THE SOFTWARE. |
22 | */ |
23 | #include "kfd_priv.h" |
24 | #include <linux/mm.h> |
25 | #include <linux/mman.h> |
26 | #include <linux/slab.h> |
27 | #include <linux/io.h> |
28 | #include <linux/idr.h> |
29 | |
30 | /* |
31 | * This extension supports a kernel level doorbells management for the |
32 | * kernel queues using the first doorbell page reserved for the kernel. |
33 | */ |
34 | |
35 | /* |
36 | * Each device exposes a doorbell aperture, a PCI MMIO aperture that |
37 | * receives 32-bit writes that are passed to queues as wptr values. |
38 | * The doorbells are intended to be written by applications as part |
39 | * of queueing work on user-mode queues. |
40 | * We assign doorbells to applications in PAGE_SIZE-sized and aligned chunks. |
41 | * We map the doorbell address space into user-mode when a process creates |
42 | * its first queue on each device. |
43 | * Although the mapping is done by KFD, it is equivalent to an mmap of |
44 | * the /dev/kfd with the particular device encoded in the mmap offset. |
45 | * There will be other uses for mmap of /dev/kfd, so only a range of |
46 | * offsets (KFD_MMAP_DOORBELL_START-END) is used for doorbells. |
47 | */ |
48 | |
49 | /* # of doorbell bytes allocated for each process. */ |
50 | size_t kfd_doorbell_process_slice(struct kfd_dev *kfd) |
51 | { |
52 | if (!kfd->shared_resources.enable_mes) |
53 | return roundup(kfd->device_info.doorbell_size * |
54 | KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, |
55 | PAGE_SIZE); |
56 | else |
57 | return amdgpu_mes_doorbell_process_slice( |
58 | adev: (struct amdgpu_device *)kfd->adev); |
59 | } |
60 | |
61 | /* Doorbell calculations for device init. */ |
62 | int kfd_doorbell_init(struct kfd_dev *kfd) |
63 | { |
64 | int size = PAGE_SIZE; |
65 | int r; |
66 | |
67 | /* |
68 | * Todo: KFD kernel level operations need only one doorbell for |
69 | * ring test/HWS. So instead of reserving a whole page here for |
70 | * kernel, reserve and consume a doorbell from existing KGD kernel |
71 | * doorbell page. |
72 | */ |
73 | |
74 | /* Bitmap to dynamically allocate doorbells from kernel page */ |
75 | kfd->doorbell_bitmap = bitmap_zalloc(nbits: size / sizeof(u32), GFP_KERNEL); |
76 | if (!kfd->doorbell_bitmap) { |
77 | DRM_ERROR("Failed to allocate kernel doorbell bitmap\n" ); |
78 | return -ENOMEM; |
79 | } |
80 | |
81 | /* Alloc a doorbell page for KFD kernel usages */ |
82 | r = amdgpu_bo_create_kernel(adev: kfd->adev, |
83 | size, |
84 | PAGE_SIZE, |
85 | AMDGPU_GEM_DOMAIN_DOORBELL, |
86 | bo_ptr: &kfd->doorbells, |
87 | NULL, |
88 | cpu_addr: (void **)&kfd->doorbell_kernel_ptr); |
89 | if (r) { |
90 | pr_err("failed to allocate kernel doorbells\n" ); |
91 | bitmap_free(bitmap: kfd->doorbell_bitmap); |
92 | return r; |
93 | } |
94 | |
95 | pr_debug("Doorbell kernel address == %p\n" , kfd->doorbell_kernel_ptr); |
96 | return 0; |
97 | } |
98 | |
99 | void kfd_doorbell_fini(struct kfd_dev *kfd) |
100 | { |
101 | bitmap_free(bitmap: kfd->doorbell_bitmap); |
102 | amdgpu_bo_free_kernel(bo: &kfd->doorbells, NULL, |
103 | cpu_addr: (void **)&kfd->doorbell_kernel_ptr); |
104 | } |
105 | |
106 | int kfd_doorbell_mmap(struct kfd_node *dev, struct kfd_process *process, |
107 | struct vm_area_struct *vma) |
108 | { |
109 | phys_addr_t address; |
110 | struct kfd_process_device *pdd; |
111 | |
112 | /* |
113 | * For simplicitly we only allow mapping of the entire doorbell |
114 | * allocation of a single device & process. |
115 | */ |
116 | if (vma->vm_end - vma->vm_start != kfd_doorbell_process_slice(kfd: dev->kfd)) |
117 | return -EINVAL; |
118 | |
119 | pdd = kfd_get_process_device_data(dev, p: process); |
120 | if (!pdd) |
121 | return -EINVAL; |
122 | |
123 | /* Calculate physical address of doorbell */ |
124 | address = kfd_get_process_doorbells(pdd); |
125 | if (!address) |
126 | return -ENOMEM; |
127 | vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE | |
128 | VM_DONTDUMP | VM_PFNMAP); |
129 | |
130 | vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); |
131 | |
132 | pr_debug("Mapping doorbell page\n" |
133 | " target user address == 0x%08llX\n" |
134 | " physical address == 0x%08llX\n" |
135 | " vm_flags == 0x%04lX\n" |
136 | " size == 0x%04lX\n" , |
137 | (unsigned long long) vma->vm_start, address, vma->vm_flags, |
138 | kfd_doorbell_process_slice(dev->kfd)); |
139 | |
140 | |
141 | return io_remap_pfn_range(vma, |
142 | addr: vma->vm_start, |
143 | pfn: address >> PAGE_SHIFT, |
144 | size: kfd_doorbell_process_slice(kfd: dev->kfd), |
145 | prot: vma->vm_page_prot); |
146 | } |
147 | |
148 | |
149 | /* get kernel iomem pointer for a doorbell */ |
150 | void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, |
151 | unsigned int *doorbell_off) |
152 | { |
153 | u32 inx; |
154 | |
155 | mutex_lock(&kfd->doorbell_mutex); |
156 | inx = find_first_zero_bit(addr: kfd->doorbell_bitmap, PAGE_SIZE / sizeof(u32)); |
157 | |
158 | __set_bit(inx, kfd->doorbell_bitmap); |
159 | mutex_unlock(lock: &kfd->doorbell_mutex); |
160 | |
161 | if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) |
162 | return NULL; |
163 | |
164 | *doorbell_off = amdgpu_doorbell_index_on_bar(adev: kfd->adev, |
165 | db_bo: kfd->doorbells, |
166 | doorbell_index: inx, |
167 | db_size: kfd->device_info.doorbell_size); |
168 | inx *= 2; |
169 | |
170 | pr_debug("Get kernel queue doorbell\n" |
171 | " doorbell offset == 0x%08X\n" |
172 | " doorbell index == 0x%x\n" , |
173 | *doorbell_off, inx); |
174 | |
175 | return kfd->doorbell_kernel_ptr + inx; |
176 | } |
177 | |
178 | void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr) |
179 | { |
180 | unsigned int inx; |
181 | |
182 | inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr); |
183 | inx /= 2; |
184 | |
185 | mutex_lock(&kfd->doorbell_mutex); |
186 | __clear_bit(inx, kfd->doorbell_bitmap); |
187 | mutex_unlock(lock: &kfd->doorbell_mutex); |
188 | } |
189 | |
190 | void write_kernel_doorbell(void __iomem *db, u32 value) |
191 | { |
192 | if (db) { |
193 | writel(val: value, addr: db); |
194 | pr_debug("Writing %d to doorbell address %p\n" , value, db); |
195 | } |
196 | } |
197 | |
198 | void write_kernel_doorbell64(void __iomem *db, u64 value) |
199 | { |
200 | if (db) { |
201 | WARN(((unsigned long)db & 7) != 0, |
202 | "Unaligned 64-bit doorbell" ); |
203 | writeq(val: value, addr: (u64 __iomem *)db); |
204 | pr_debug("writing %llu to doorbell address %p\n" , value, db); |
205 | } |
206 | } |
207 | |
208 | static int init_doorbell_bitmap(struct qcm_process_device *qpd, |
209 | struct kfd_dev *dev) |
210 | { |
211 | unsigned int i; |
212 | int range_start = dev->shared_resources.non_cp_doorbells_start; |
213 | int range_end = dev->shared_resources.non_cp_doorbells_end; |
214 | |
215 | if (!KFD_IS_SOC15(dev)) |
216 | return 0; |
217 | |
218 | /* Mask out doorbells reserved for SDMA, IH, and VCN on SOC15. */ |
219 | pr_debug("reserved doorbell 0x%03x - 0x%03x\n" , range_start, range_end); |
220 | pr_debug("reserved doorbell 0x%03x - 0x%03x\n" , |
221 | range_start + KFD_QUEUE_DOORBELL_MIRROR_OFFSET, |
222 | range_end + KFD_QUEUE_DOORBELL_MIRROR_OFFSET); |
223 | |
224 | for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) { |
225 | if (i >= range_start && i <= range_end) { |
226 | __set_bit(i, qpd->doorbell_bitmap); |
227 | __set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET, |
228 | qpd->doorbell_bitmap); |
229 | } |
230 | } |
231 | |
232 | return 0; |
233 | } |
234 | |
235 | phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd) |
236 | { |
237 | struct amdgpu_device *adev = pdd->dev->adev; |
238 | uint32_t first_db_index; |
239 | |
240 | if (!pdd->qpd.proc_doorbells) { |
241 | if (kfd_alloc_process_doorbells(kfd: pdd->dev->kfd, pdd)) |
242 | /* phys_addr_t 0 is error */ |
243 | return 0; |
244 | } |
245 | |
246 | first_db_index = amdgpu_doorbell_index_on_bar(adev, |
247 | db_bo: pdd->qpd.proc_doorbells, |
248 | doorbell_index: 0, |
249 | db_size: pdd->dev->kfd->device_info.doorbell_size); |
250 | return adev->doorbell.base + first_db_index * sizeof(uint32_t); |
251 | } |
252 | |
253 | int kfd_alloc_process_doorbells(struct kfd_dev *kfd, struct kfd_process_device *pdd) |
254 | { |
255 | int r; |
256 | struct qcm_process_device *qpd = &pdd->qpd; |
257 | |
258 | /* Allocate bitmap for dynamic doorbell allocation */ |
259 | qpd->doorbell_bitmap = bitmap_zalloc(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, |
260 | GFP_KERNEL); |
261 | if (!qpd->doorbell_bitmap) { |
262 | DRM_ERROR("Failed to allocate process doorbell bitmap\n" ); |
263 | return -ENOMEM; |
264 | } |
265 | |
266 | r = init_doorbell_bitmap(qpd: &pdd->qpd, dev: kfd); |
267 | if (r) { |
268 | DRM_ERROR("Failed to initialize process doorbells\n" ); |
269 | r = -ENOMEM; |
270 | goto err; |
271 | } |
272 | |
273 | /* Allocate doorbells for this process */ |
274 | r = amdgpu_bo_create_kernel(adev: kfd->adev, |
275 | size: kfd_doorbell_process_slice(kfd), |
276 | PAGE_SIZE, |
277 | AMDGPU_GEM_DOMAIN_DOORBELL, |
278 | bo_ptr: &qpd->proc_doorbells, |
279 | NULL, |
280 | NULL); |
281 | if (r) { |
282 | DRM_ERROR("Failed to allocate process doorbells\n" ); |
283 | goto err; |
284 | } |
285 | return 0; |
286 | |
287 | err: |
288 | bitmap_free(bitmap: qpd->doorbell_bitmap); |
289 | qpd->doorbell_bitmap = NULL; |
290 | return r; |
291 | } |
292 | |
293 | void kfd_free_process_doorbells(struct kfd_dev *kfd, struct kfd_process_device *pdd) |
294 | { |
295 | struct qcm_process_device *qpd = &pdd->qpd; |
296 | |
297 | if (qpd->doorbell_bitmap) { |
298 | bitmap_free(bitmap: qpd->doorbell_bitmap); |
299 | qpd->doorbell_bitmap = NULL; |
300 | } |
301 | |
302 | amdgpu_bo_free_kernel(bo: &qpd->proc_doorbells, NULL, NULL); |
303 | } |
304 | |