1 | /* |
2 | * Copyright 2008 Advanced Micro Devices, Inc. |
3 | * Copyright 2008 Red Hat Inc. |
4 | * Copyright 2009 Jerome Glisse. |
5 | * |
6 | * Permission is hereby granted, free of charge, to any person obtaining a |
7 | * copy of this software and associated documentation files (the "Software"), |
8 | * to deal in the Software without restriction, including without limitation |
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
10 | * and/or sell copies of the Software, and to permit persons to whom the |
11 | * Software is furnished to do so, subject to the following conditions: |
12 | * |
13 | * The above copyright notice and this permission notice shall be included in |
14 | * all copies or substantial portions of the Software. |
15 | * |
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
19 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
20 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
21 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
22 | * OTHER DEALINGS IN THE SOFTWARE. |
23 | * |
24 | */ |
25 | |
26 | #include <linux/kthread.h> |
27 | #include <linux/pci.h> |
28 | #include <linux/uaccess.h> |
29 | #include <linux/pm_runtime.h> |
30 | |
31 | #include "amdgpu.h" |
32 | #include "amdgpu_pm.h" |
33 | #include "amdgpu_dm_debugfs.h" |
34 | #include "amdgpu_ras.h" |
35 | #include "amdgpu_rap.h" |
36 | #include "amdgpu_securedisplay.h" |
37 | #include "amdgpu_fw_attestation.h" |
38 | #include "amdgpu_umr.h" |
39 | |
40 | #include "amdgpu_reset.h" |
41 | #include "amdgpu_psp_ta.h" |
42 | |
43 | #if defined(CONFIG_DEBUG_FS) |
44 | |
45 | /** |
46 | * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes |
47 | * |
48 | * @read: True if reading |
49 | * @f: open file handle |
50 | * @buf: User buffer to write/read to |
51 | * @size: Number of bytes to write/read |
52 | * @pos: Offset to seek to |
53 | * |
54 | * This debugfs entry has special meaning on the offset being sought. |
55 | * Various bits have different meanings: |
56 | * |
57 | * Bit 62: Indicates a GRBM bank switch is needed |
58 | * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is |
59 | * zero) |
60 | * Bits 24..33: The SE or ME selector if needed |
61 | * Bits 34..43: The SH (or SA) or PIPE selector if needed |
62 | * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed |
63 | * |
64 | * Bit 23: Indicates that the PM power gating lock should be held |
65 | * This is necessary to read registers that might be |
66 | * unreliable during a power gating transistion. |
67 | * |
68 | * The lower bits are the BYTE offset of the register to read. This |
69 | * allows reading multiple registers in a single call and having |
70 | * the returned size reflect that. |
71 | */ |
72 | static int amdgpu_debugfs_process_reg_op(bool read, struct file *f, |
73 | char __user *buf, size_t size, loff_t *pos) |
74 | { |
75 | struct amdgpu_device *adev = file_inode(f)->i_private; |
76 | ssize_t result = 0; |
77 | int r; |
78 | bool pm_pg_lock, use_bank, use_ring; |
79 | unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid; |
80 | |
81 | pm_pg_lock = use_bank = use_ring = false; |
82 | instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0; |
83 | |
84 | if (size & 0x3 || *pos & 0x3 || |
85 | ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61)))) |
86 | return -EINVAL; |
87 | |
88 | /* are we reading registers for which a PG lock is necessary? */ |
89 | pm_pg_lock = (*pos >> 23) & 1; |
90 | |
91 | if (*pos & (1ULL << 62)) { |
92 | se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24; |
93 | sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34; |
94 | instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44; |
95 | |
96 | if (se_bank == 0x3FF) |
97 | se_bank = 0xFFFFFFFF; |
98 | if (sh_bank == 0x3FF) |
99 | sh_bank = 0xFFFFFFFF; |
100 | if (instance_bank == 0x3FF) |
101 | instance_bank = 0xFFFFFFFF; |
102 | use_bank = true; |
103 | } else if (*pos & (1ULL << 61)) { |
104 | |
105 | me = (*pos & GENMASK_ULL(33, 24)) >> 24; |
106 | pipe = (*pos & GENMASK_ULL(43, 34)) >> 34; |
107 | queue = (*pos & GENMASK_ULL(53, 44)) >> 44; |
108 | vmid = (*pos & GENMASK_ULL(58, 54)) >> 54; |
109 | |
110 | use_ring = true; |
111 | } else { |
112 | use_bank = use_ring = false; |
113 | } |
114 | |
115 | *pos &= (1UL << 22) - 1; |
116 | |
117 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
118 | if (r < 0) { |
119 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
120 | return r; |
121 | } |
122 | |
123 | r = amdgpu_virt_enable_access_debugfs(adev); |
124 | if (r < 0) { |
125 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
126 | return r; |
127 | } |
128 | |
129 | if (use_bank) { |
130 | if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) || |
131 | (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) { |
132 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
133 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
134 | amdgpu_virt_disable_access_debugfs(adev); |
135 | return -EINVAL; |
136 | } |
137 | mutex_lock(&adev->grbm_idx_mutex); |
138 | amdgpu_gfx_select_se_sh(adev, se_bank, |
139 | sh_bank, instance_bank, 0); |
140 | } else if (use_ring) { |
141 | mutex_lock(&adev->srbm_mutex); |
142 | amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0); |
143 | } |
144 | |
145 | if (pm_pg_lock) |
146 | mutex_lock(&adev->pm.mutex); |
147 | |
148 | while (size) { |
149 | uint32_t value; |
150 | |
151 | if (read) { |
152 | value = RREG32(*pos >> 2); |
153 | r = put_user(value, (uint32_t *)buf); |
154 | } else { |
155 | r = get_user(value, (uint32_t *)buf); |
156 | if (!r) |
157 | amdgpu_mm_wreg_mmio_rlc(adev, reg: *pos >> 2, v: value, xcc_id: 0); |
158 | } |
159 | if (r) { |
160 | result = r; |
161 | goto end; |
162 | } |
163 | |
164 | result += 4; |
165 | buf += 4; |
166 | *pos += 4; |
167 | size -= 4; |
168 | } |
169 | |
170 | end: |
171 | if (use_bank) { |
172 | amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); |
173 | mutex_unlock(lock: &adev->grbm_idx_mutex); |
174 | } else if (use_ring) { |
175 | amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); |
176 | mutex_unlock(lock: &adev->srbm_mutex); |
177 | } |
178 | |
179 | if (pm_pg_lock) |
180 | mutex_unlock(lock: &adev->pm.mutex); |
181 | |
182 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
183 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
184 | |
185 | amdgpu_virt_disable_access_debugfs(adev); |
186 | return result; |
187 | } |
188 | |
189 | /* |
190 | * amdgpu_debugfs_regs_read - Callback for reading MMIO registers |
191 | */ |
192 | static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf, |
193 | size_t size, loff_t *pos) |
194 | { |
195 | return amdgpu_debugfs_process_reg_op(read: true, f, buf, size, pos); |
196 | } |
197 | |
198 | /* |
199 | * amdgpu_debugfs_regs_write - Callback for writing MMIO registers |
200 | */ |
201 | static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf, |
202 | size_t size, loff_t *pos) |
203 | { |
204 | return amdgpu_debugfs_process_reg_op(read: false, f, buf: (char __user *)buf, size, pos); |
205 | } |
206 | |
207 | static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file) |
208 | { |
209 | struct amdgpu_debugfs_regs2_data *rd; |
210 | |
211 | rd = kzalloc(size: sizeof(*rd), GFP_KERNEL); |
212 | if (!rd) |
213 | return -ENOMEM; |
214 | rd->adev = file_inode(f: file)->i_private; |
215 | file->private_data = rd; |
216 | mutex_init(&rd->lock); |
217 | |
218 | return 0; |
219 | } |
220 | |
221 | static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file) |
222 | { |
223 | struct amdgpu_debugfs_regs2_data *rd = file->private_data; |
224 | |
225 | mutex_destroy(lock: &rd->lock); |
226 | kfree(objp: file->private_data); |
227 | return 0; |
228 | } |
229 | |
230 | static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en) |
231 | { |
232 | struct amdgpu_debugfs_regs2_data *rd = f->private_data; |
233 | struct amdgpu_device *adev = rd->adev; |
234 | ssize_t result = 0; |
235 | int r; |
236 | uint32_t value; |
237 | |
238 | if (size & 0x3 || offset & 0x3) |
239 | return -EINVAL; |
240 | |
241 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
242 | if (r < 0) { |
243 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
244 | return r; |
245 | } |
246 | |
247 | r = amdgpu_virt_enable_access_debugfs(adev); |
248 | if (r < 0) { |
249 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
250 | return r; |
251 | } |
252 | |
253 | mutex_lock(&rd->lock); |
254 | |
255 | if (rd->id.use_grbm) { |
256 | if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) || |
257 | (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) { |
258 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
259 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
260 | amdgpu_virt_disable_access_debugfs(adev); |
261 | mutex_unlock(lock: &rd->lock); |
262 | return -EINVAL; |
263 | } |
264 | mutex_lock(&adev->grbm_idx_mutex); |
265 | amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se, |
266 | rd->id.grbm.sh, |
267 | rd->id.grbm.instance, rd->id.xcc_id); |
268 | } |
269 | |
270 | if (rd->id.use_srbm) { |
271 | mutex_lock(&adev->srbm_mutex); |
272 | amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe, |
273 | rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id); |
274 | } |
275 | |
276 | if (rd->id.pg_lock) |
277 | mutex_lock(&adev->pm.mutex); |
278 | |
279 | while (size) { |
280 | if (!write_en) { |
281 | value = RREG32(offset >> 2); |
282 | r = put_user(value, (uint32_t *)buf); |
283 | } else { |
284 | r = get_user(value, (uint32_t *)buf); |
285 | if (!r) |
286 | amdgpu_mm_wreg_mmio_rlc(adev, reg: offset >> 2, v: value, xcc_id: rd->id.xcc_id); |
287 | } |
288 | if (r) { |
289 | result = r; |
290 | goto end; |
291 | } |
292 | offset += 4; |
293 | size -= 4; |
294 | result += 4; |
295 | buf += 4; |
296 | } |
297 | end: |
298 | if (rd->id.use_grbm) { |
299 | amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id); |
300 | mutex_unlock(lock: &adev->grbm_idx_mutex); |
301 | } |
302 | |
303 | if (rd->id.use_srbm) { |
304 | amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id); |
305 | mutex_unlock(lock: &adev->srbm_mutex); |
306 | } |
307 | |
308 | if (rd->id.pg_lock) |
309 | mutex_unlock(lock: &adev->pm.mutex); |
310 | |
311 | mutex_unlock(lock: &rd->lock); |
312 | |
313 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
314 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
315 | |
316 | amdgpu_virt_disable_access_debugfs(adev); |
317 | return result; |
318 | } |
319 | |
320 | static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data) |
321 | { |
322 | struct amdgpu_debugfs_regs2_data *rd = f->private_data; |
323 | struct amdgpu_debugfs_regs2_iocdata v1_data; |
324 | int r; |
325 | |
326 | mutex_lock(&rd->lock); |
327 | |
328 | switch (cmd) { |
329 | case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2: |
330 | r = copy_from_user(to: &rd->id, from: (struct amdgpu_debugfs_regs2_iocdata_v2 *)data, |
331 | n: sizeof(rd->id)); |
332 | if (r) |
333 | r = -EINVAL; |
334 | goto done; |
335 | case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE: |
336 | r = copy_from_user(to: &v1_data, from: (struct amdgpu_debugfs_regs2_iocdata *)data, |
337 | n: sizeof(v1_data)); |
338 | if (r) { |
339 | r = -EINVAL; |
340 | goto done; |
341 | } |
342 | goto v1_copy; |
343 | default: |
344 | r = -EINVAL; |
345 | goto done; |
346 | } |
347 | |
348 | v1_copy: |
349 | rd->id.use_srbm = v1_data.use_srbm; |
350 | rd->id.use_grbm = v1_data.use_grbm; |
351 | rd->id.pg_lock = v1_data.pg_lock; |
352 | rd->id.grbm.se = v1_data.grbm.se; |
353 | rd->id.grbm.sh = v1_data.grbm.sh; |
354 | rd->id.grbm.instance = v1_data.grbm.instance; |
355 | rd->id.srbm.me = v1_data.srbm.me; |
356 | rd->id.srbm.pipe = v1_data.srbm.pipe; |
357 | rd->id.srbm.queue = v1_data.srbm.queue; |
358 | rd->id.xcc_id = 0; |
359 | done: |
360 | mutex_unlock(lock: &rd->lock); |
361 | return r; |
362 | } |
363 | |
364 | static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos) |
365 | { |
366 | return amdgpu_debugfs_regs2_op(f, buf, offset: *pos, size, write_en: 0); |
367 | } |
368 | |
369 | static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos) |
370 | { |
371 | return amdgpu_debugfs_regs2_op(f, buf: (char __user *)buf, offset: *pos, size, write_en: 1); |
372 | } |
373 | |
374 | static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file) |
375 | { |
376 | struct amdgpu_debugfs_gprwave_data *rd; |
377 | |
378 | rd = kzalloc(size: sizeof(*rd), GFP_KERNEL); |
379 | if (!rd) |
380 | return -ENOMEM; |
381 | rd->adev = file_inode(f: file)->i_private; |
382 | file->private_data = rd; |
383 | mutex_init(&rd->lock); |
384 | |
385 | return 0; |
386 | } |
387 | |
388 | static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file) |
389 | { |
390 | struct amdgpu_debugfs_gprwave_data *rd = file->private_data; |
391 | |
392 | mutex_destroy(lock: &rd->lock); |
393 | kfree(objp: file->private_data); |
394 | return 0; |
395 | } |
396 | |
397 | static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos) |
398 | { |
399 | struct amdgpu_debugfs_gprwave_data *rd = f->private_data; |
400 | struct amdgpu_device *adev = rd->adev; |
401 | ssize_t result = 0; |
402 | int r; |
403 | uint32_t *data, x; |
404 | |
405 | if (size & 0x3 || *pos & 0x3) |
406 | return -EINVAL; |
407 | |
408 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
409 | if (r < 0) { |
410 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
411 | return r; |
412 | } |
413 | |
414 | r = amdgpu_virt_enable_access_debugfs(adev); |
415 | if (r < 0) { |
416 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
417 | return r; |
418 | } |
419 | |
420 | data = kcalloc(n: 1024, size: sizeof(*data), GFP_KERNEL); |
421 | if (!data) { |
422 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
423 | amdgpu_virt_disable_access_debugfs(adev); |
424 | return -ENOMEM; |
425 | } |
426 | |
427 | /* switch to the specific se/sh/cu */ |
428 | mutex_lock(&adev->grbm_idx_mutex); |
429 | amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id); |
430 | |
431 | if (!rd->id.gpr_or_wave) { |
432 | x = 0; |
433 | if (adev->gfx.funcs->read_wave_data) |
434 | adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x); |
435 | } else { |
436 | x = size >> 2; |
437 | if (rd->id.gpr.vpgr_or_sgpr) { |
438 | if (adev->gfx.funcs->read_wave_vgprs) |
439 | adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data); |
440 | } else { |
441 | if (adev->gfx.funcs->read_wave_sgprs) |
442 | adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data); |
443 | } |
444 | } |
445 | |
446 | amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id); |
447 | mutex_unlock(lock: &adev->grbm_idx_mutex); |
448 | |
449 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
450 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
451 | |
452 | if (!x) { |
453 | result = -EINVAL; |
454 | goto done; |
455 | } |
456 | |
457 | while (size && (*pos < x * 4)) { |
458 | uint32_t value; |
459 | |
460 | value = data[*pos >> 2]; |
461 | r = put_user(value, (uint32_t *)buf); |
462 | if (r) { |
463 | result = r; |
464 | goto done; |
465 | } |
466 | |
467 | result += 4; |
468 | buf += 4; |
469 | *pos += 4; |
470 | size -= 4; |
471 | } |
472 | |
473 | done: |
474 | amdgpu_virt_disable_access_debugfs(adev); |
475 | kfree(objp: data); |
476 | return result; |
477 | } |
478 | |
479 | static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data) |
480 | { |
481 | struct amdgpu_debugfs_gprwave_data *rd = f->private_data; |
482 | int r = 0; |
483 | |
484 | mutex_lock(&rd->lock); |
485 | |
486 | switch (cmd) { |
487 | case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE: |
488 | if (copy_from_user(to: &rd->id, |
489 | from: (struct amdgpu_debugfs_gprwave_iocdata *)data, |
490 | n: sizeof(rd->id))) |
491 | r = -EFAULT; |
492 | goto done; |
493 | default: |
494 | r = -EINVAL; |
495 | goto done; |
496 | } |
497 | |
498 | done: |
499 | mutex_unlock(lock: &rd->lock); |
500 | return r; |
501 | } |
502 | |
503 | |
504 | |
505 | |
506 | /** |
507 | * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register |
508 | * |
509 | * @f: open file handle |
510 | * @buf: User buffer to store read data in |
511 | * @size: Number of bytes to read |
512 | * @pos: Offset to seek to |
513 | * |
514 | * The lower bits are the BYTE offset of the register to read. This |
515 | * allows reading multiple registers in a single call and having |
516 | * the returned size reflect that. |
517 | */ |
518 | static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf, |
519 | size_t size, loff_t *pos) |
520 | { |
521 | struct amdgpu_device *adev = file_inode(f)->i_private; |
522 | ssize_t result = 0; |
523 | int r; |
524 | |
525 | if (size & 0x3 || *pos & 0x3) |
526 | return -EINVAL; |
527 | |
528 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
529 | if (r < 0) { |
530 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
531 | return r; |
532 | } |
533 | |
534 | r = amdgpu_virt_enable_access_debugfs(adev); |
535 | if (r < 0) { |
536 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
537 | return r; |
538 | } |
539 | |
540 | while (size) { |
541 | uint32_t value; |
542 | |
543 | value = RREG32_PCIE(*pos); |
544 | r = put_user(value, (uint32_t *)buf); |
545 | if (r) |
546 | goto out; |
547 | |
548 | result += 4; |
549 | buf += 4; |
550 | *pos += 4; |
551 | size -= 4; |
552 | } |
553 | |
554 | r = result; |
555 | out: |
556 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
557 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
558 | amdgpu_virt_disable_access_debugfs(adev); |
559 | return r; |
560 | } |
561 | |
562 | /** |
563 | * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register |
564 | * |
565 | * @f: open file handle |
566 | * @buf: User buffer to write data from |
567 | * @size: Number of bytes to write |
568 | * @pos: Offset to seek to |
569 | * |
570 | * The lower bits are the BYTE offset of the register to write. This |
571 | * allows writing multiple registers in a single call and having |
572 | * the returned size reflect that. |
573 | */ |
574 | static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf, |
575 | size_t size, loff_t *pos) |
576 | { |
577 | struct amdgpu_device *adev = file_inode(f)->i_private; |
578 | ssize_t result = 0; |
579 | int r; |
580 | |
581 | if (size & 0x3 || *pos & 0x3) |
582 | return -EINVAL; |
583 | |
584 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
585 | if (r < 0) { |
586 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
587 | return r; |
588 | } |
589 | |
590 | r = amdgpu_virt_enable_access_debugfs(adev); |
591 | if (r < 0) { |
592 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
593 | return r; |
594 | } |
595 | |
596 | while (size) { |
597 | uint32_t value; |
598 | |
599 | r = get_user(value, (uint32_t *)buf); |
600 | if (r) |
601 | goto out; |
602 | |
603 | WREG32_PCIE(*pos, value); |
604 | |
605 | result += 4; |
606 | buf += 4; |
607 | *pos += 4; |
608 | size -= 4; |
609 | } |
610 | |
611 | r = result; |
612 | out: |
613 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
614 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
615 | amdgpu_virt_disable_access_debugfs(adev); |
616 | return r; |
617 | } |
618 | |
619 | /** |
620 | * amdgpu_debugfs_regs_didt_read - Read from a DIDT register |
621 | * |
622 | * @f: open file handle |
623 | * @buf: User buffer to store read data in |
624 | * @size: Number of bytes to read |
625 | * @pos: Offset to seek to |
626 | * |
627 | * The lower bits are the BYTE offset of the register to read. This |
628 | * allows reading multiple registers in a single call and having |
629 | * the returned size reflect that. |
630 | */ |
631 | static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf, |
632 | size_t size, loff_t *pos) |
633 | { |
634 | struct amdgpu_device *adev = file_inode(f)->i_private; |
635 | ssize_t result = 0; |
636 | int r; |
637 | |
638 | if (size & 0x3 || *pos & 0x3) |
639 | return -EINVAL; |
640 | |
641 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
642 | if (r < 0) { |
643 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
644 | return r; |
645 | } |
646 | |
647 | r = amdgpu_virt_enable_access_debugfs(adev); |
648 | if (r < 0) { |
649 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
650 | return r; |
651 | } |
652 | |
653 | while (size) { |
654 | uint32_t value; |
655 | |
656 | value = RREG32_DIDT(*pos >> 2); |
657 | r = put_user(value, (uint32_t *)buf); |
658 | if (r) |
659 | goto out; |
660 | |
661 | result += 4; |
662 | buf += 4; |
663 | *pos += 4; |
664 | size -= 4; |
665 | } |
666 | |
667 | r = result; |
668 | out: |
669 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
670 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
671 | amdgpu_virt_disable_access_debugfs(adev); |
672 | return r; |
673 | } |
674 | |
675 | /** |
676 | * amdgpu_debugfs_regs_didt_write - Write to a DIDT register |
677 | * |
678 | * @f: open file handle |
679 | * @buf: User buffer to write data from |
680 | * @size: Number of bytes to write |
681 | * @pos: Offset to seek to |
682 | * |
683 | * The lower bits are the BYTE offset of the register to write. This |
684 | * allows writing multiple registers in a single call and having |
685 | * the returned size reflect that. |
686 | */ |
687 | static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf, |
688 | size_t size, loff_t *pos) |
689 | { |
690 | struct amdgpu_device *adev = file_inode(f)->i_private; |
691 | ssize_t result = 0; |
692 | int r; |
693 | |
694 | if (size & 0x3 || *pos & 0x3) |
695 | return -EINVAL; |
696 | |
697 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
698 | if (r < 0) { |
699 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
700 | return r; |
701 | } |
702 | |
703 | r = amdgpu_virt_enable_access_debugfs(adev); |
704 | if (r < 0) { |
705 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
706 | return r; |
707 | } |
708 | |
709 | while (size) { |
710 | uint32_t value; |
711 | |
712 | r = get_user(value, (uint32_t *)buf); |
713 | if (r) |
714 | goto out; |
715 | |
716 | WREG32_DIDT(*pos >> 2, value); |
717 | |
718 | result += 4; |
719 | buf += 4; |
720 | *pos += 4; |
721 | size -= 4; |
722 | } |
723 | |
724 | r = result; |
725 | out: |
726 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
727 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
728 | amdgpu_virt_disable_access_debugfs(adev); |
729 | return r; |
730 | } |
731 | |
732 | /** |
733 | * amdgpu_debugfs_regs_smc_read - Read from a SMC register |
734 | * |
735 | * @f: open file handle |
736 | * @buf: User buffer to store read data in |
737 | * @size: Number of bytes to read |
738 | * @pos: Offset to seek to |
739 | * |
740 | * The lower bits are the BYTE offset of the register to read. This |
741 | * allows reading multiple registers in a single call and having |
742 | * the returned size reflect that. |
743 | */ |
744 | static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf, |
745 | size_t size, loff_t *pos) |
746 | { |
747 | struct amdgpu_device *adev = file_inode(f)->i_private; |
748 | ssize_t result = 0; |
749 | int r; |
750 | |
751 | if (!adev->smc_rreg) |
752 | return -EPERM; |
753 | |
754 | if (size & 0x3 || *pos & 0x3) |
755 | return -EINVAL; |
756 | |
757 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
758 | if (r < 0) { |
759 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
760 | return r; |
761 | } |
762 | |
763 | r = amdgpu_virt_enable_access_debugfs(adev); |
764 | if (r < 0) { |
765 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
766 | return r; |
767 | } |
768 | |
769 | while (size) { |
770 | uint32_t value; |
771 | |
772 | value = RREG32_SMC(*pos); |
773 | r = put_user(value, (uint32_t *)buf); |
774 | if (r) |
775 | goto out; |
776 | |
777 | result += 4; |
778 | buf += 4; |
779 | *pos += 4; |
780 | size -= 4; |
781 | } |
782 | |
783 | r = result; |
784 | out: |
785 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
786 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
787 | amdgpu_virt_disable_access_debugfs(adev); |
788 | return r; |
789 | } |
790 | |
791 | /** |
792 | * amdgpu_debugfs_regs_smc_write - Write to a SMC register |
793 | * |
794 | * @f: open file handle |
795 | * @buf: User buffer to write data from |
796 | * @size: Number of bytes to write |
797 | * @pos: Offset to seek to |
798 | * |
799 | * The lower bits are the BYTE offset of the register to write. This |
800 | * allows writing multiple registers in a single call and having |
801 | * the returned size reflect that. |
802 | */ |
803 | static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf, |
804 | size_t size, loff_t *pos) |
805 | { |
806 | struct amdgpu_device *adev = file_inode(f)->i_private; |
807 | ssize_t result = 0; |
808 | int r; |
809 | |
810 | if (!adev->smc_wreg) |
811 | return -EPERM; |
812 | |
813 | if (size & 0x3 || *pos & 0x3) |
814 | return -EINVAL; |
815 | |
816 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
817 | if (r < 0) { |
818 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
819 | return r; |
820 | } |
821 | |
822 | r = amdgpu_virt_enable_access_debugfs(adev); |
823 | if (r < 0) { |
824 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
825 | return r; |
826 | } |
827 | |
828 | while (size) { |
829 | uint32_t value; |
830 | |
831 | r = get_user(value, (uint32_t *)buf); |
832 | if (r) |
833 | goto out; |
834 | |
835 | WREG32_SMC(*pos, value); |
836 | |
837 | result += 4; |
838 | buf += 4; |
839 | *pos += 4; |
840 | size -= 4; |
841 | } |
842 | |
843 | r = result; |
844 | out: |
845 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
846 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
847 | amdgpu_virt_disable_access_debugfs(adev); |
848 | return r; |
849 | } |
850 | |
851 | /** |
852 | * amdgpu_debugfs_gca_config_read - Read from gfx config data |
853 | * |
854 | * @f: open file handle |
855 | * @buf: User buffer to store read data in |
856 | * @size: Number of bytes to read |
857 | * @pos: Offset to seek to |
858 | * |
859 | * This file is used to access configuration data in a somewhat |
860 | * stable fashion. The format is a series of DWORDs with the first |
861 | * indicating which revision it is. New content is appended to the |
862 | * end so that older software can still read the data. |
863 | */ |
864 | |
865 | static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf, |
866 | size_t size, loff_t *pos) |
867 | { |
868 | struct amdgpu_device *adev = file_inode(f)->i_private; |
869 | ssize_t result = 0; |
870 | int r; |
871 | uint32_t *config, no_regs = 0; |
872 | |
873 | if (size & 0x3 || *pos & 0x3) |
874 | return -EINVAL; |
875 | |
876 | config = kmalloc_array(n: 256, size: sizeof(*config), GFP_KERNEL); |
877 | if (!config) |
878 | return -ENOMEM; |
879 | |
880 | /* version, increment each time something is added */ |
881 | config[no_regs++] = 5; |
882 | config[no_regs++] = adev->gfx.config.max_shader_engines; |
883 | config[no_regs++] = adev->gfx.config.max_tile_pipes; |
884 | config[no_regs++] = adev->gfx.config.max_cu_per_sh; |
885 | config[no_regs++] = adev->gfx.config.max_sh_per_se; |
886 | config[no_regs++] = adev->gfx.config.max_backends_per_se; |
887 | config[no_regs++] = adev->gfx.config.max_texture_channel_caches; |
888 | config[no_regs++] = adev->gfx.config.max_gprs; |
889 | config[no_regs++] = adev->gfx.config.max_gs_threads; |
890 | config[no_regs++] = adev->gfx.config.max_hw_contexts; |
891 | config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend; |
892 | config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend; |
893 | config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size; |
894 | config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size; |
895 | config[no_regs++] = adev->gfx.config.num_tile_pipes; |
896 | config[no_regs++] = adev->gfx.config.backend_enable_mask; |
897 | config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes; |
898 | config[no_regs++] = adev->gfx.config.mem_row_size_in_kb; |
899 | config[no_regs++] = adev->gfx.config.shader_engine_tile_size; |
900 | config[no_regs++] = adev->gfx.config.num_gpus; |
901 | config[no_regs++] = adev->gfx.config.multi_gpu_tile_size; |
902 | config[no_regs++] = adev->gfx.config.mc_arb_ramcfg; |
903 | config[no_regs++] = adev->gfx.config.gb_addr_config; |
904 | config[no_regs++] = adev->gfx.config.num_rbs; |
905 | |
906 | /* rev==1 */ |
907 | config[no_regs++] = adev->rev_id; |
908 | config[no_regs++] = lower_32_bits(adev->pg_flags); |
909 | config[no_regs++] = lower_32_bits(adev->cg_flags); |
910 | |
911 | /* rev==2 */ |
912 | config[no_regs++] = adev->family; |
913 | config[no_regs++] = adev->external_rev_id; |
914 | |
915 | /* rev==3 */ |
916 | config[no_regs++] = adev->pdev->device; |
917 | config[no_regs++] = adev->pdev->revision; |
918 | config[no_regs++] = adev->pdev->subsystem_device; |
919 | config[no_regs++] = adev->pdev->subsystem_vendor; |
920 | |
921 | /* rev==4 APU flag */ |
922 | config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0; |
923 | |
924 | /* rev==5 PG/CG flag upper 32bit */ |
925 | config[no_regs++] = upper_32_bits(adev->pg_flags); |
926 | config[no_regs++] = upper_32_bits(adev->cg_flags); |
927 | |
928 | while (size && (*pos < no_regs * 4)) { |
929 | uint32_t value; |
930 | |
931 | value = config[*pos >> 2]; |
932 | r = put_user(value, (uint32_t *)buf); |
933 | if (r) { |
934 | kfree(objp: config); |
935 | return r; |
936 | } |
937 | |
938 | result += 4; |
939 | buf += 4; |
940 | *pos += 4; |
941 | size -= 4; |
942 | } |
943 | |
944 | kfree(objp: config); |
945 | return result; |
946 | } |
947 | |
948 | /** |
949 | * amdgpu_debugfs_sensor_read - Read from the powerplay sensors |
950 | * |
951 | * @f: open file handle |
952 | * @buf: User buffer to store read data in |
953 | * @size: Number of bytes to read |
954 | * @pos: Offset to seek to |
955 | * |
956 | * The offset is treated as the BYTE address of one of the sensors |
957 | * enumerated in amd/include/kgd_pp_interface.h under the |
958 | * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK |
959 | * you would use the offset 3 * 4 = 12. |
960 | */ |
961 | static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf, |
962 | size_t size, loff_t *pos) |
963 | { |
964 | struct amdgpu_device *adev = file_inode(f)->i_private; |
965 | int idx, x, outsize, r, valuesize; |
966 | uint32_t values[16]; |
967 | |
968 | if (size & 3 || *pos & 0x3) |
969 | return -EINVAL; |
970 | |
971 | if (!adev->pm.dpm_enabled) |
972 | return -EINVAL; |
973 | |
974 | /* convert offset to sensor number */ |
975 | idx = *pos >> 2; |
976 | |
977 | valuesize = sizeof(values); |
978 | |
979 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
980 | if (r < 0) { |
981 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
982 | return r; |
983 | } |
984 | |
985 | r = amdgpu_virt_enable_access_debugfs(adev); |
986 | if (r < 0) { |
987 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
988 | return r; |
989 | } |
990 | |
991 | r = amdgpu_dpm_read_sensor(adev, sensor: idx, data: &values[0], size: &valuesize); |
992 | |
993 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
994 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
995 | |
996 | if (r) { |
997 | amdgpu_virt_disable_access_debugfs(adev); |
998 | return r; |
999 | } |
1000 | |
1001 | if (size > valuesize) { |
1002 | amdgpu_virt_disable_access_debugfs(adev); |
1003 | return -EINVAL; |
1004 | } |
1005 | |
1006 | outsize = 0; |
1007 | x = 0; |
1008 | if (!r) { |
1009 | while (size) { |
1010 | r = put_user(values[x++], (int32_t *)buf); |
1011 | buf += 4; |
1012 | size -= 4; |
1013 | outsize += 4; |
1014 | } |
1015 | } |
1016 | |
1017 | amdgpu_virt_disable_access_debugfs(adev); |
1018 | return !r ? outsize : r; |
1019 | } |
1020 | |
1021 | /** amdgpu_debugfs_wave_read - Read WAVE STATUS data |
1022 | * |
1023 | * @f: open file handle |
1024 | * @buf: User buffer to store read data in |
1025 | * @size: Number of bytes to read |
1026 | * @pos: Offset to seek to |
1027 | * |
1028 | * The offset being sought changes which wave that the status data |
1029 | * will be returned for. The bits are used as follows: |
1030 | * |
1031 | * Bits 0..6: Byte offset into data |
1032 | * Bits 7..14: SE selector |
1033 | * Bits 15..22: SH/SA selector |
1034 | * Bits 23..30: CU/{WGP+SIMD} selector |
1035 | * Bits 31..36: WAVE ID selector |
1036 | * Bits 37..44: SIMD ID selector |
1037 | * |
1038 | * The returned data begins with one DWORD of version information |
1039 | * Followed by WAVE STATUS registers relevant to the GFX IP version |
1040 | * being used. See gfx_v8_0_read_wave_data() for an example output. |
1041 | */ |
1042 | static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf, |
1043 | size_t size, loff_t *pos) |
1044 | { |
1045 | struct amdgpu_device *adev = f->f_inode->i_private; |
1046 | int r, x; |
1047 | ssize_t result = 0; |
1048 | uint32_t offset, se, sh, cu, wave, simd, data[32]; |
1049 | |
1050 | if (size & 3 || *pos & 3) |
1051 | return -EINVAL; |
1052 | |
1053 | /* decode offset */ |
1054 | offset = (*pos & GENMASK_ULL(6, 0)); |
1055 | se = (*pos & GENMASK_ULL(14, 7)) >> 7; |
1056 | sh = (*pos & GENMASK_ULL(22, 15)) >> 15; |
1057 | cu = (*pos & GENMASK_ULL(30, 23)) >> 23; |
1058 | wave = (*pos & GENMASK_ULL(36, 31)) >> 31; |
1059 | simd = (*pos & GENMASK_ULL(44, 37)) >> 37; |
1060 | |
1061 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
1062 | if (r < 0) { |
1063 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1064 | return r; |
1065 | } |
1066 | |
1067 | r = amdgpu_virt_enable_access_debugfs(adev); |
1068 | if (r < 0) { |
1069 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1070 | return r; |
1071 | } |
1072 | |
1073 | /* switch to the specific se/sh/cu */ |
1074 | mutex_lock(&adev->grbm_idx_mutex); |
1075 | amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0); |
1076 | |
1077 | x = 0; |
1078 | if (adev->gfx.funcs->read_wave_data) |
1079 | adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x); |
1080 | |
1081 | amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0); |
1082 | mutex_unlock(lock: &adev->grbm_idx_mutex); |
1083 | |
1084 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
1085 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1086 | |
1087 | if (!x) { |
1088 | amdgpu_virt_disable_access_debugfs(adev); |
1089 | return -EINVAL; |
1090 | } |
1091 | |
1092 | while (size && (offset < x * 4)) { |
1093 | uint32_t value; |
1094 | |
1095 | value = data[offset >> 2]; |
1096 | r = put_user(value, (uint32_t *)buf); |
1097 | if (r) { |
1098 | amdgpu_virt_disable_access_debugfs(adev); |
1099 | return r; |
1100 | } |
1101 | |
1102 | result += 4; |
1103 | buf += 4; |
1104 | offset += 4; |
1105 | size -= 4; |
1106 | } |
1107 | |
1108 | amdgpu_virt_disable_access_debugfs(adev); |
1109 | return result; |
1110 | } |
1111 | |
1112 | /** amdgpu_debugfs_gpr_read - Read wave gprs |
1113 | * |
1114 | * @f: open file handle |
1115 | * @buf: User buffer to store read data in |
1116 | * @size: Number of bytes to read |
1117 | * @pos: Offset to seek to |
1118 | * |
1119 | * The offset being sought changes which wave that the status data |
1120 | * will be returned for. The bits are used as follows: |
1121 | * |
1122 | * Bits 0..11: Byte offset into data |
1123 | * Bits 12..19: SE selector |
1124 | * Bits 20..27: SH/SA selector |
1125 | * Bits 28..35: CU/{WGP+SIMD} selector |
1126 | * Bits 36..43: WAVE ID selector |
1127 | * Bits 37..44: SIMD ID selector |
1128 | * Bits 52..59: Thread selector |
1129 | * Bits 60..61: Bank selector (VGPR=0,SGPR=1) |
1130 | * |
1131 | * The return data comes from the SGPR or VGPR register bank for |
1132 | * the selected operational unit. |
1133 | */ |
1134 | static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf, |
1135 | size_t size, loff_t *pos) |
1136 | { |
1137 | struct amdgpu_device *adev = f->f_inode->i_private; |
1138 | int r; |
1139 | ssize_t result = 0; |
1140 | uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data; |
1141 | |
1142 | if (size > 4096 || size & 3 || *pos & 3) |
1143 | return -EINVAL; |
1144 | |
1145 | /* decode offset */ |
1146 | offset = (*pos & GENMASK_ULL(11, 0)) >> 2; |
1147 | se = (*pos & GENMASK_ULL(19, 12)) >> 12; |
1148 | sh = (*pos & GENMASK_ULL(27, 20)) >> 20; |
1149 | cu = (*pos & GENMASK_ULL(35, 28)) >> 28; |
1150 | wave = (*pos & GENMASK_ULL(43, 36)) >> 36; |
1151 | simd = (*pos & GENMASK_ULL(51, 44)) >> 44; |
1152 | thread = (*pos & GENMASK_ULL(59, 52)) >> 52; |
1153 | bank = (*pos & GENMASK_ULL(61, 60)) >> 60; |
1154 | |
1155 | data = kcalloc(n: 1024, size: sizeof(*data), GFP_KERNEL); |
1156 | if (!data) |
1157 | return -ENOMEM; |
1158 | |
1159 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
1160 | if (r < 0) |
1161 | goto err; |
1162 | |
1163 | r = amdgpu_virt_enable_access_debugfs(adev); |
1164 | if (r < 0) |
1165 | goto err; |
1166 | |
1167 | /* switch to the specific se/sh/cu */ |
1168 | mutex_lock(&adev->grbm_idx_mutex); |
1169 | amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0); |
1170 | |
1171 | if (bank == 0) { |
1172 | if (adev->gfx.funcs->read_wave_vgprs) |
1173 | adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data); |
1174 | } else { |
1175 | if (adev->gfx.funcs->read_wave_sgprs) |
1176 | adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data); |
1177 | } |
1178 | |
1179 | amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0); |
1180 | mutex_unlock(lock: &adev->grbm_idx_mutex); |
1181 | |
1182 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
1183 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1184 | |
1185 | while (size) { |
1186 | uint32_t value; |
1187 | |
1188 | value = data[result >> 2]; |
1189 | r = put_user(value, (uint32_t *)buf); |
1190 | if (r) { |
1191 | amdgpu_virt_disable_access_debugfs(adev); |
1192 | goto err; |
1193 | } |
1194 | |
1195 | result += 4; |
1196 | buf += 4; |
1197 | size -= 4; |
1198 | } |
1199 | |
1200 | kfree(objp: data); |
1201 | amdgpu_virt_disable_access_debugfs(adev); |
1202 | return result; |
1203 | |
1204 | err: |
1205 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1206 | kfree(objp: data); |
1207 | return r; |
1208 | } |
1209 | |
1210 | /** |
1211 | * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency |
1212 | * |
1213 | * @f: open file handle |
1214 | * @buf: User buffer to store read data in |
1215 | * @size: Number of bytes to read |
1216 | * @pos: Offset to seek to |
1217 | * |
1218 | * Read the last residency value logged. It doesn't auto update, one needs to |
1219 | * stop logging before getting the current value. |
1220 | */ |
1221 | static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf, |
1222 | size_t size, loff_t *pos) |
1223 | { |
1224 | struct amdgpu_device *adev = file_inode(f)->i_private; |
1225 | ssize_t result = 0; |
1226 | int r; |
1227 | |
1228 | if (size & 0x3 || *pos & 0x3) |
1229 | return -EINVAL; |
1230 | |
1231 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
1232 | if (r < 0) { |
1233 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1234 | return r; |
1235 | } |
1236 | |
1237 | while (size) { |
1238 | uint32_t value; |
1239 | |
1240 | r = amdgpu_get_gfx_off_residency(adev, residency: &value); |
1241 | if (r) |
1242 | goto out; |
1243 | |
1244 | r = put_user(value, (uint32_t *)buf); |
1245 | if (r) |
1246 | goto out; |
1247 | |
1248 | result += 4; |
1249 | buf += 4; |
1250 | *pos += 4; |
1251 | size -= 4; |
1252 | } |
1253 | |
1254 | r = result; |
1255 | out: |
1256 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
1257 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1258 | |
1259 | return r; |
1260 | } |
1261 | |
1262 | /** |
1263 | * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency |
1264 | * |
1265 | * @f: open file handle |
1266 | * @buf: User buffer to write data from |
1267 | * @size: Number of bytes to write |
1268 | * @pos: Offset to seek to |
1269 | * |
1270 | * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop |
1271 | */ |
1272 | static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf, |
1273 | size_t size, loff_t *pos) |
1274 | { |
1275 | struct amdgpu_device *adev = file_inode(f)->i_private; |
1276 | ssize_t result = 0; |
1277 | int r; |
1278 | |
1279 | if (size & 0x3 || *pos & 0x3) |
1280 | return -EINVAL; |
1281 | |
1282 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
1283 | if (r < 0) { |
1284 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1285 | return r; |
1286 | } |
1287 | |
1288 | while (size) { |
1289 | u32 value; |
1290 | |
1291 | r = get_user(value, (uint32_t *)buf); |
1292 | if (r) |
1293 | goto out; |
1294 | |
1295 | amdgpu_set_gfx_off_residency(adev, value: value ? true : false); |
1296 | |
1297 | result += 4; |
1298 | buf += 4; |
1299 | *pos += 4; |
1300 | size -= 4; |
1301 | } |
1302 | |
1303 | r = result; |
1304 | out: |
1305 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
1306 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1307 | |
1308 | return r; |
1309 | } |
1310 | |
1311 | |
1312 | /** |
1313 | * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count |
1314 | * |
1315 | * @f: open file handle |
1316 | * @buf: User buffer to store read data in |
1317 | * @size: Number of bytes to read |
1318 | * @pos: Offset to seek to |
1319 | */ |
1320 | static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf, |
1321 | size_t size, loff_t *pos) |
1322 | { |
1323 | struct amdgpu_device *adev = file_inode(f)->i_private; |
1324 | ssize_t result = 0; |
1325 | int r; |
1326 | |
1327 | if (size & 0x3 || *pos & 0x3) |
1328 | return -EINVAL; |
1329 | |
1330 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
1331 | if (r < 0) { |
1332 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1333 | return r; |
1334 | } |
1335 | |
1336 | while (size) { |
1337 | u64 value = 0; |
1338 | |
1339 | r = amdgpu_get_gfx_off_entrycount(adev, value: &value); |
1340 | if (r) |
1341 | goto out; |
1342 | |
1343 | r = put_user(value, (u64 *)buf); |
1344 | if (r) |
1345 | goto out; |
1346 | |
1347 | result += 4; |
1348 | buf += 4; |
1349 | *pos += 4; |
1350 | size -= 4; |
1351 | } |
1352 | |
1353 | r = result; |
1354 | out: |
1355 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
1356 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1357 | |
1358 | return r; |
1359 | } |
1360 | |
1361 | /** |
1362 | * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF |
1363 | * |
1364 | * @f: open file handle |
1365 | * @buf: User buffer to write data from |
1366 | * @size: Number of bytes to write |
1367 | * @pos: Offset to seek to |
1368 | * |
1369 | * Write a 32-bit zero to disable or a 32-bit non-zero to enable |
1370 | */ |
1371 | static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf, |
1372 | size_t size, loff_t *pos) |
1373 | { |
1374 | struct amdgpu_device *adev = file_inode(f)->i_private; |
1375 | ssize_t result = 0; |
1376 | int r; |
1377 | |
1378 | if (size & 0x3 || *pos & 0x3) |
1379 | return -EINVAL; |
1380 | |
1381 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
1382 | if (r < 0) { |
1383 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1384 | return r; |
1385 | } |
1386 | |
1387 | while (size) { |
1388 | uint32_t value; |
1389 | |
1390 | r = get_user(value, (uint32_t *)buf); |
1391 | if (r) |
1392 | goto out; |
1393 | |
1394 | amdgpu_gfx_off_ctrl(adev, enable: value ? true : false); |
1395 | |
1396 | result += 4; |
1397 | buf += 4; |
1398 | *pos += 4; |
1399 | size -= 4; |
1400 | } |
1401 | |
1402 | r = result; |
1403 | out: |
1404 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
1405 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1406 | |
1407 | return r; |
1408 | } |
1409 | |
1410 | |
1411 | /** |
1412 | * amdgpu_debugfs_gfxoff_read - read gfxoff status |
1413 | * |
1414 | * @f: open file handle |
1415 | * @buf: User buffer to store read data in |
1416 | * @size: Number of bytes to read |
1417 | * @pos: Offset to seek to |
1418 | */ |
1419 | static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf, |
1420 | size_t size, loff_t *pos) |
1421 | { |
1422 | struct amdgpu_device *adev = file_inode(f)->i_private; |
1423 | ssize_t result = 0; |
1424 | int r; |
1425 | |
1426 | if (size & 0x3 || *pos & 0x3) |
1427 | return -EINVAL; |
1428 | |
1429 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
1430 | if (r < 0) { |
1431 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1432 | return r; |
1433 | } |
1434 | |
1435 | while (size) { |
1436 | u32 value = adev->gfx.gfx_off_state; |
1437 | |
1438 | r = put_user(value, (u32 *)buf); |
1439 | if (r) |
1440 | goto out; |
1441 | |
1442 | result += 4; |
1443 | buf += 4; |
1444 | *pos += 4; |
1445 | size -= 4; |
1446 | } |
1447 | |
1448 | r = result; |
1449 | out: |
1450 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
1451 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1452 | |
1453 | return r; |
1454 | } |
1455 | |
1456 | static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf, |
1457 | size_t size, loff_t *pos) |
1458 | { |
1459 | struct amdgpu_device *adev = file_inode(f)->i_private; |
1460 | ssize_t result = 0; |
1461 | int r; |
1462 | |
1463 | if (size & 0x3 || *pos & 0x3) |
1464 | return -EINVAL; |
1465 | |
1466 | r = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
1467 | if (r < 0) { |
1468 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1469 | return r; |
1470 | } |
1471 | |
1472 | while (size) { |
1473 | u32 value; |
1474 | |
1475 | r = amdgpu_get_gfx_off_status(adev, value: &value); |
1476 | if (r) |
1477 | goto out; |
1478 | |
1479 | r = put_user(value, (u32 *)buf); |
1480 | if (r) |
1481 | goto out; |
1482 | |
1483 | result += 4; |
1484 | buf += 4; |
1485 | *pos += 4; |
1486 | size -= 4; |
1487 | } |
1488 | |
1489 | r = result; |
1490 | out: |
1491 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
1492 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1493 | |
1494 | return r; |
1495 | } |
1496 | |
1497 | static const struct file_operations amdgpu_debugfs_regs2_fops = { |
1498 | .owner = THIS_MODULE, |
1499 | .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl, |
1500 | .read = amdgpu_debugfs_regs2_read, |
1501 | .write = amdgpu_debugfs_regs2_write, |
1502 | .open = amdgpu_debugfs_regs2_open, |
1503 | .release = amdgpu_debugfs_regs2_release, |
1504 | .llseek = default_llseek |
1505 | }; |
1506 | |
1507 | static const struct file_operations amdgpu_debugfs_gprwave_fops = { |
1508 | .owner = THIS_MODULE, |
1509 | .unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl, |
1510 | .read = amdgpu_debugfs_gprwave_read, |
1511 | .open = amdgpu_debugfs_gprwave_open, |
1512 | .release = amdgpu_debugfs_gprwave_release, |
1513 | .llseek = default_llseek |
1514 | }; |
1515 | |
1516 | static const struct file_operations amdgpu_debugfs_regs_fops = { |
1517 | .owner = THIS_MODULE, |
1518 | .read = amdgpu_debugfs_regs_read, |
1519 | .write = amdgpu_debugfs_regs_write, |
1520 | .llseek = default_llseek |
1521 | }; |
1522 | static const struct file_operations amdgpu_debugfs_regs_didt_fops = { |
1523 | .owner = THIS_MODULE, |
1524 | .read = amdgpu_debugfs_regs_didt_read, |
1525 | .write = amdgpu_debugfs_regs_didt_write, |
1526 | .llseek = default_llseek |
1527 | }; |
1528 | static const struct file_operations amdgpu_debugfs_regs_pcie_fops = { |
1529 | .owner = THIS_MODULE, |
1530 | .read = amdgpu_debugfs_regs_pcie_read, |
1531 | .write = amdgpu_debugfs_regs_pcie_write, |
1532 | .llseek = default_llseek |
1533 | }; |
1534 | static const struct file_operations amdgpu_debugfs_regs_smc_fops = { |
1535 | .owner = THIS_MODULE, |
1536 | .read = amdgpu_debugfs_regs_smc_read, |
1537 | .write = amdgpu_debugfs_regs_smc_write, |
1538 | .llseek = default_llseek |
1539 | }; |
1540 | |
1541 | static const struct file_operations amdgpu_debugfs_gca_config_fops = { |
1542 | .owner = THIS_MODULE, |
1543 | .read = amdgpu_debugfs_gca_config_read, |
1544 | .llseek = default_llseek |
1545 | }; |
1546 | |
1547 | static const struct file_operations amdgpu_debugfs_sensors_fops = { |
1548 | .owner = THIS_MODULE, |
1549 | .read = amdgpu_debugfs_sensor_read, |
1550 | .llseek = default_llseek |
1551 | }; |
1552 | |
1553 | static const struct file_operations amdgpu_debugfs_wave_fops = { |
1554 | .owner = THIS_MODULE, |
1555 | .read = amdgpu_debugfs_wave_read, |
1556 | .llseek = default_llseek |
1557 | }; |
1558 | static const struct file_operations amdgpu_debugfs_gpr_fops = { |
1559 | .owner = THIS_MODULE, |
1560 | .read = amdgpu_debugfs_gpr_read, |
1561 | .llseek = default_llseek |
1562 | }; |
1563 | |
1564 | static const struct file_operations amdgpu_debugfs_gfxoff_fops = { |
1565 | .owner = THIS_MODULE, |
1566 | .read = amdgpu_debugfs_gfxoff_read, |
1567 | .write = amdgpu_debugfs_gfxoff_write, |
1568 | .llseek = default_llseek |
1569 | }; |
1570 | |
1571 | static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = { |
1572 | .owner = THIS_MODULE, |
1573 | .read = amdgpu_debugfs_gfxoff_status_read, |
1574 | .llseek = default_llseek |
1575 | }; |
1576 | |
1577 | static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = { |
1578 | .owner = THIS_MODULE, |
1579 | .read = amdgpu_debugfs_gfxoff_count_read, |
1580 | .llseek = default_llseek |
1581 | }; |
1582 | |
1583 | static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = { |
1584 | .owner = THIS_MODULE, |
1585 | .read = amdgpu_debugfs_gfxoff_residency_read, |
1586 | .write = amdgpu_debugfs_gfxoff_residency_write, |
1587 | .llseek = default_llseek |
1588 | }; |
1589 | |
1590 | static const struct file_operations *debugfs_regs[] = { |
1591 | &amdgpu_debugfs_regs_fops, |
1592 | &amdgpu_debugfs_regs2_fops, |
1593 | &amdgpu_debugfs_gprwave_fops, |
1594 | &amdgpu_debugfs_regs_didt_fops, |
1595 | &amdgpu_debugfs_regs_pcie_fops, |
1596 | &amdgpu_debugfs_regs_smc_fops, |
1597 | &amdgpu_debugfs_gca_config_fops, |
1598 | &amdgpu_debugfs_sensors_fops, |
1599 | &amdgpu_debugfs_wave_fops, |
1600 | &amdgpu_debugfs_gpr_fops, |
1601 | &amdgpu_debugfs_gfxoff_fops, |
1602 | &amdgpu_debugfs_gfxoff_status_fops, |
1603 | &amdgpu_debugfs_gfxoff_count_fops, |
1604 | &amdgpu_debugfs_gfxoff_residency_fops, |
1605 | }; |
1606 | |
1607 | static const char * const debugfs_regs_names[] = { |
1608 | "amdgpu_regs" , |
1609 | "amdgpu_regs2" , |
1610 | "amdgpu_gprwave" , |
1611 | "amdgpu_regs_didt" , |
1612 | "amdgpu_regs_pcie" , |
1613 | "amdgpu_regs_smc" , |
1614 | "amdgpu_gca_config" , |
1615 | "amdgpu_sensors" , |
1616 | "amdgpu_wave" , |
1617 | "amdgpu_gpr" , |
1618 | "amdgpu_gfxoff" , |
1619 | "amdgpu_gfxoff_status" , |
1620 | "amdgpu_gfxoff_count" , |
1621 | "amdgpu_gfxoff_residency" , |
1622 | }; |
1623 | |
1624 | /** |
1625 | * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide |
1626 | * register access. |
1627 | * |
1628 | * @adev: The device to attach the debugfs entries to |
1629 | */ |
1630 | int amdgpu_debugfs_regs_init(struct amdgpu_device *adev) |
1631 | { |
1632 | struct drm_minor *minor = adev_to_drm(adev)->primary; |
1633 | struct dentry *ent, *root = minor->debugfs_root; |
1634 | unsigned int i; |
1635 | |
1636 | for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) { |
1637 | ent = debugfs_create_file(name: debugfs_regs_names[i], |
1638 | S_IFREG | 0444, parent: root, |
1639 | data: adev, fops: debugfs_regs[i]); |
1640 | if (!i && !IS_ERR_OR_NULL(ptr: ent)) |
1641 | i_size_write(inode: ent->d_inode, i_size: adev->rmmio_size); |
1642 | } |
1643 | |
1644 | return 0; |
1645 | } |
1646 | |
1647 | static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused) |
1648 | { |
1649 | struct amdgpu_device *adev = m->private; |
1650 | struct drm_device *dev = adev_to_drm(adev); |
1651 | int r = 0, i; |
1652 | |
1653 | r = pm_runtime_get_sync(dev: dev->dev); |
1654 | if (r < 0) { |
1655 | pm_runtime_put_autosuspend(dev: dev->dev); |
1656 | return r; |
1657 | } |
1658 | |
1659 | /* Avoid accidently unparking the sched thread during GPU reset */ |
1660 | r = down_write_killable(sem: &adev->reset_domain->sem); |
1661 | if (r) |
1662 | return r; |
1663 | |
1664 | /* hold on the scheduler */ |
1665 | for (i = 0; i < AMDGPU_MAX_RINGS; i++) { |
1666 | struct amdgpu_ring *ring = adev->rings[i]; |
1667 | |
1668 | if (!ring || !ring->sched.thread) |
1669 | continue; |
1670 | kthread_park(k: ring->sched.thread); |
1671 | } |
1672 | |
1673 | seq_puts(m, s: "run ib test:\n" ); |
1674 | r = amdgpu_ib_ring_tests(adev); |
1675 | if (r) |
1676 | seq_printf(m, fmt: "ib ring tests failed (%d).\n" , r); |
1677 | else |
1678 | seq_puts(m, s: "ib ring tests passed.\n" ); |
1679 | |
1680 | /* go on the scheduler */ |
1681 | for (i = 0; i < AMDGPU_MAX_RINGS; i++) { |
1682 | struct amdgpu_ring *ring = adev->rings[i]; |
1683 | |
1684 | if (!ring || !ring->sched.thread) |
1685 | continue; |
1686 | kthread_unpark(k: ring->sched.thread); |
1687 | } |
1688 | |
1689 | up_write(sem: &adev->reset_domain->sem); |
1690 | |
1691 | pm_runtime_mark_last_busy(dev: dev->dev); |
1692 | pm_runtime_put_autosuspend(dev: dev->dev); |
1693 | |
1694 | return 0; |
1695 | } |
1696 | |
1697 | static int amdgpu_debugfs_evict_vram(void *data, u64 *val) |
1698 | { |
1699 | struct amdgpu_device *adev = (struct amdgpu_device *)data; |
1700 | struct drm_device *dev = adev_to_drm(adev); |
1701 | int r; |
1702 | |
1703 | r = pm_runtime_get_sync(dev: dev->dev); |
1704 | if (r < 0) { |
1705 | pm_runtime_put_autosuspend(dev: dev->dev); |
1706 | return r; |
1707 | } |
1708 | |
1709 | *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM); |
1710 | |
1711 | pm_runtime_mark_last_busy(dev: dev->dev); |
1712 | pm_runtime_put_autosuspend(dev: dev->dev); |
1713 | |
1714 | return 0; |
1715 | } |
1716 | |
1717 | |
1718 | static int amdgpu_debugfs_evict_gtt(void *data, u64 *val) |
1719 | { |
1720 | struct amdgpu_device *adev = (struct amdgpu_device *)data; |
1721 | struct drm_device *dev = adev_to_drm(adev); |
1722 | int r; |
1723 | |
1724 | r = pm_runtime_get_sync(dev: dev->dev); |
1725 | if (r < 0) { |
1726 | pm_runtime_put_autosuspend(dev: dev->dev); |
1727 | return r; |
1728 | } |
1729 | |
1730 | *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT); |
1731 | |
1732 | pm_runtime_mark_last_busy(dev: dev->dev); |
1733 | pm_runtime_put_autosuspend(dev: dev->dev); |
1734 | |
1735 | return 0; |
1736 | } |
1737 | |
1738 | static int amdgpu_debugfs_benchmark(void *data, u64 val) |
1739 | { |
1740 | struct amdgpu_device *adev = (struct amdgpu_device *)data; |
1741 | struct drm_device *dev = adev_to_drm(adev); |
1742 | int r; |
1743 | |
1744 | r = pm_runtime_get_sync(dev: dev->dev); |
1745 | if (r < 0) { |
1746 | pm_runtime_put_autosuspend(dev: dev->dev); |
1747 | return r; |
1748 | } |
1749 | |
1750 | r = amdgpu_benchmark(adev, test_number: val); |
1751 | |
1752 | pm_runtime_mark_last_busy(dev: dev->dev); |
1753 | pm_runtime_put_autosuspend(dev: dev->dev); |
1754 | |
1755 | return r; |
1756 | } |
1757 | |
1758 | static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused) |
1759 | { |
1760 | struct amdgpu_device *adev = m->private; |
1761 | struct drm_device *dev = adev_to_drm(adev); |
1762 | struct drm_file *file; |
1763 | int r; |
1764 | |
1765 | r = mutex_lock_interruptible(&dev->filelist_mutex); |
1766 | if (r) |
1767 | return r; |
1768 | |
1769 | list_for_each_entry(file, &dev->filelist, lhead) { |
1770 | struct amdgpu_fpriv *fpriv = file->driver_priv; |
1771 | struct amdgpu_vm *vm = &fpriv->vm; |
1772 | |
1773 | seq_printf(m, fmt: "pid:%d\tProcess:%s ----------\n" , |
1774 | vm->task_info.pid, vm->task_info.process_name); |
1775 | r = amdgpu_bo_reserve(bo: vm->root.bo, no_intr: true); |
1776 | if (r) |
1777 | break; |
1778 | amdgpu_debugfs_vm_bo_info(vm, m); |
1779 | amdgpu_bo_unreserve(bo: vm->root.bo); |
1780 | } |
1781 | |
1782 | mutex_unlock(lock: &dev->filelist_mutex); |
1783 | |
1784 | return r; |
1785 | } |
1786 | |
1787 | DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib); |
1788 | DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info); |
1789 | DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram, |
1790 | NULL, "%lld\n" ); |
1791 | DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt, |
1792 | NULL, "%lld\n" ); |
1793 | DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark, |
1794 | "%lld\n" ); |
1795 | |
1796 | static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring, |
1797 | struct dma_fence **fences) |
1798 | { |
1799 | struct amdgpu_fence_driver *drv = &ring->fence_drv; |
1800 | uint32_t sync_seq, last_seq; |
1801 | |
1802 | last_seq = atomic_read(v: &ring->fence_drv.last_seq); |
1803 | sync_seq = ring->fence_drv.sync_seq; |
1804 | |
1805 | last_seq &= drv->num_fences_mask; |
1806 | sync_seq &= drv->num_fences_mask; |
1807 | |
1808 | do { |
1809 | struct dma_fence *fence, **ptr; |
1810 | |
1811 | ++last_seq; |
1812 | last_seq &= drv->num_fences_mask; |
1813 | ptr = &drv->fences[last_seq]; |
1814 | |
1815 | fence = rcu_dereference_protected(*ptr, 1); |
1816 | RCU_INIT_POINTER(*ptr, NULL); |
1817 | |
1818 | if (!fence) |
1819 | continue; |
1820 | |
1821 | fences[last_seq] = fence; |
1822 | |
1823 | } while (last_seq != sync_seq); |
1824 | } |
1825 | |
1826 | static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences, |
1827 | int length) |
1828 | { |
1829 | int i; |
1830 | struct dma_fence *fence; |
1831 | |
1832 | for (i = 0; i < length; i++) { |
1833 | fence = fences[i]; |
1834 | if (!fence) |
1835 | continue; |
1836 | dma_fence_signal(fence); |
1837 | dma_fence_put(fence); |
1838 | } |
1839 | } |
1840 | |
1841 | static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched) |
1842 | { |
1843 | struct drm_sched_job *s_job; |
1844 | struct dma_fence *fence; |
1845 | |
1846 | spin_lock(lock: &sched->job_list_lock); |
1847 | list_for_each_entry(s_job, &sched->pending_list, list) { |
1848 | fence = sched->ops->run_job(s_job); |
1849 | dma_fence_put(fence); |
1850 | } |
1851 | spin_unlock(lock: &sched->job_list_lock); |
1852 | } |
1853 | |
1854 | static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring) |
1855 | { |
1856 | struct amdgpu_job *job; |
1857 | struct drm_sched_job *s_job, *tmp; |
1858 | uint32_t preempt_seq; |
1859 | struct dma_fence *fence, **ptr; |
1860 | struct amdgpu_fence_driver *drv = &ring->fence_drv; |
1861 | struct drm_gpu_scheduler *sched = &ring->sched; |
1862 | bool preempted = true; |
1863 | |
1864 | if (ring->funcs->type != AMDGPU_RING_TYPE_GFX) |
1865 | return; |
1866 | |
1867 | preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2)); |
1868 | if (preempt_seq <= atomic_read(v: &drv->last_seq)) { |
1869 | preempted = false; |
1870 | goto no_preempt; |
1871 | } |
1872 | |
1873 | preempt_seq &= drv->num_fences_mask; |
1874 | ptr = &drv->fences[preempt_seq]; |
1875 | fence = rcu_dereference_protected(*ptr, 1); |
1876 | |
1877 | no_preempt: |
1878 | spin_lock(lock: &sched->job_list_lock); |
1879 | list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) { |
1880 | if (dma_fence_is_signaled(fence: &s_job->s_fence->finished)) { |
1881 | /* remove job from ring_mirror_list */ |
1882 | list_del_init(entry: &s_job->list); |
1883 | sched->ops->free_job(s_job); |
1884 | continue; |
1885 | } |
1886 | job = to_amdgpu_job(s_job); |
1887 | if (preempted && (&job->hw_fence) == fence) |
1888 | /* mark the job as preempted */ |
1889 | job->preemption_status |= AMDGPU_IB_PREEMPTED; |
1890 | } |
1891 | spin_unlock(lock: &sched->job_list_lock); |
1892 | } |
1893 | |
1894 | static int amdgpu_debugfs_ib_preempt(void *data, u64 val) |
1895 | { |
1896 | int r, length; |
1897 | struct amdgpu_ring *ring; |
1898 | struct dma_fence **fences = NULL; |
1899 | struct amdgpu_device *adev = (struct amdgpu_device *)data; |
1900 | |
1901 | if (val >= AMDGPU_MAX_RINGS) |
1902 | return -EINVAL; |
1903 | |
1904 | ring = adev->rings[val]; |
1905 | |
1906 | if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread) |
1907 | return -EINVAL; |
1908 | |
1909 | /* the last preemption failed */ |
1910 | if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr)) |
1911 | return -EBUSY; |
1912 | |
1913 | length = ring->fence_drv.num_fences_mask + 1; |
1914 | fences = kcalloc(n: length, size: sizeof(void *), GFP_KERNEL); |
1915 | if (!fences) |
1916 | return -ENOMEM; |
1917 | |
1918 | /* Avoid accidently unparking the sched thread during GPU reset */ |
1919 | r = down_read_killable(sem: &adev->reset_domain->sem); |
1920 | if (r) |
1921 | goto pro_end; |
1922 | |
1923 | /* stop the scheduler */ |
1924 | kthread_park(k: ring->sched.thread); |
1925 | |
1926 | /* preempt the IB */ |
1927 | r = amdgpu_ring_preempt_ib(ring); |
1928 | if (r) { |
1929 | DRM_WARN("failed to preempt ring %d\n" , ring->idx); |
1930 | goto failure; |
1931 | } |
1932 | |
1933 | amdgpu_fence_process(ring); |
1934 | |
1935 | if (atomic_read(v: &ring->fence_drv.last_seq) != |
1936 | ring->fence_drv.sync_seq) { |
1937 | DRM_INFO("ring %d was preempted\n" , ring->idx); |
1938 | |
1939 | amdgpu_ib_preempt_mark_partial_job(ring); |
1940 | |
1941 | /* swap out the old fences */ |
1942 | amdgpu_ib_preempt_fences_swap(ring, fences); |
1943 | |
1944 | amdgpu_fence_driver_force_completion(ring); |
1945 | |
1946 | /* resubmit unfinished jobs */ |
1947 | amdgpu_ib_preempt_job_recovery(sched: &ring->sched); |
1948 | |
1949 | /* wait for jobs finished */ |
1950 | amdgpu_fence_wait_empty(ring); |
1951 | |
1952 | /* signal the old fences */ |
1953 | amdgpu_ib_preempt_signal_fences(fences, length); |
1954 | } |
1955 | |
1956 | failure: |
1957 | /* restart the scheduler */ |
1958 | kthread_unpark(k: ring->sched.thread); |
1959 | |
1960 | up_read(sem: &adev->reset_domain->sem); |
1961 | |
1962 | pro_end: |
1963 | kfree(objp: fences); |
1964 | |
1965 | return r; |
1966 | } |
1967 | |
1968 | static int amdgpu_debugfs_sclk_set(void *data, u64 val) |
1969 | { |
1970 | int ret = 0; |
1971 | uint32_t max_freq, min_freq; |
1972 | struct amdgpu_device *adev = (struct amdgpu_device *)data; |
1973 | |
1974 | if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev)) |
1975 | return -EINVAL; |
1976 | |
1977 | ret = pm_runtime_get_sync(dev: adev_to_drm(adev)->dev); |
1978 | if (ret < 0) { |
1979 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
1980 | return ret; |
1981 | } |
1982 | |
1983 | ret = amdgpu_dpm_get_dpm_freq_range(adev, type: PP_SCLK, min: &min_freq, max: &max_freq); |
1984 | if (ret == -EOPNOTSUPP) { |
1985 | ret = 0; |
1986 | goto out; |
1987 | } |
1988 | if (ret || val > max_freq || val < min_freq) { |
1989 | ret = -EINVAL; |
1990 | goto out; |
1991 | } |
1992 | |
1993 | ret = amdgpu_dpm_set_soft_freq_range(adev, type: PP_SCLK, min: (uint32_t)val, max: (uint32_t)val); |
1994 | if (ret) |
1995 | ret = -EINVAL; |
1996 | |
1997 | out: |
1998 | pm_runtime_mark_last_busy(dev: adev_to_drm(adev)->dev); |
1999 | pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev); |
2000 | |
2001 | return ret; |
2002 | } |
2003 | |
2004 | DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL, |
2005 | amdgpu_debugfs_ib_preempt, "%llu\n" ); |
2006 | |
2007 | DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL, |
2008 | amdgpu_debugfs_sclk_set, "%llu\n" ); |
2009 | |
2010 | static ssize_t amdgpu_reset_dump_register_list_read(struct file *f, |
2011 | char __user *buf, size_t size, loff_t *pos) |
2012 | { |
2013 | struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; |
2014 | char reg_offset[12]; |
2015 | int i, ret, len = 0; |
2016 | |
2017 | if (*pos) |
2018 | return 0; |
2019 | |
2020 | memset(reg_offset, 0, 12); |
2021 | ret = down_read_killable(sem: &adev->reset_domain->sem); |
2022 | if (ret) |
2023 | return ret; |
2024 | |
2025 | for (i = 0; i < adev->reset_info.num_regs; i++) { |
2026 | sprintf(buf: reg_offset, fmt: "0x%x\n" , adev->reset_info.reset_dump_reg_list[i]); |
2027 | up_read(sem: &adev->reset_domain->sem); |
2028 | if (copy_to_user(to: buf + len, from: reg_offset, strlen(reg_offset))) |
2029 | return -EFAULT; |
2030 | |
2031 | len += strlen(reg_offset); |
2032 | ret = down_read_killable(sem: &adev->reset_domain->sem); |
2033 | if (ret) |
2034 | return ret; |
2035 | } |
2036 | |
2037 | up_read(sem: &adev->reset_domain->sem); |
2038 | *pos += len; |
2039 | |
2040 | return len; |
2041 | } |
2042 | |
2043 | static ssize_t amdgpu_reset_dump_register_list_write(struct file *f, |
2044 | const char __user *buf, size_t size, loff_t *pos) |
2045 | { |
2046 | struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; |
2047 | char reg_offset[11]; |
2048 | uint32_t *new = NULL, *tmp = NULL; |
2049 | int ret, i = 0, len = 0; |
2050 | |
2051 | do { |
2052 | memset(reg_offset, 0, 11); |
2053 | if (copy_from_user(to: reg_offset, from: buf + len, |
2054 | min(10, ((int)size-len)))) { |
2055 | ret = -EFAULT; |
2056 | goto error_free; |
2057 | } |
2058 | |
2059 | new = krealloc_array(p: tmp, new_n: i + 1, new_size: sizeof(uint32_t), GFP_KERNEL); |
2060 | if (!new) { |
2061 | ret = -ENOMEM; |
2062 | goto error_free; |
2063 | } |
2064 | tmp = new; |
2065 | if (sscanf(reg_offset, "%X %n" , &tmp[i], &ret) != 1) { |
2066 | ret = -EINVAL; |
2067 | goto error_free; |
2068 | } |
2069 | |
2070 | len += ret; |
2071 | i++; |
2072 | } while (len < size); |
2073 | |
2074 | new = kmalloc_array(n: i, size: sizeof(uint32_t), GFP_KERNEL); |
2075 | if (!new) { |
2076 | ret = -ENOMEM; |
2077 | goto error_free; |
2078 | } |
2079 | ret = down_write_killable(sem: &adev->reset_domain->sem); |
2080 | if (ret) |
2081 | goto error_free; |
2082 | |
2083 | swap(adev->reset_info.reset_dump_reg_list, tmp); |
2084 | swap(adev->reset_info.reset_dump_reg_value, new); |
2085 | adev->reset_info.num_regs = i; |
2086 | up_write(sem: &adev->reset_domain->sem); |
2087 | ret = size; |
2088 | |
2089 | error_free: |
2090 | if (tmp != new) |
2091 | kfree(objp: tmp); |
2092 | kfree(objp: new); |
2093 | return ret; |
2094 | } |
2095 | |
2096 | static const struct file_operations amdgpu_reset_dump_register_list = { |
2097 | .owner = THIS_MODULE, |
2098 | .read = amdgpu_reset_dump_register_list_read, |
2099 | .write = amdgpu_reset_dump_register_list_write, |
2100 | .llseek = default_llseek |
2101 | }; |
2102 | |
2103 | int amdgpu_debugfs_init(struct amdgpu_device *adev) |
2104 | { |
2105 | struct dentry *root = adev_to_drm(adev)->primary->debugfs_root; |
2106 | struct dentry *ent; |
2107 | int r, i; |
2108 | |
2109 | if (!debugfs_initialized()) |
2110 | return 0; |
2111 | |
2112 | debugfs_create_x32(name: "amdgpu_smu_debug" , mode: 0600, parent: root, |
2113 | value: &adev->pm.smu_debug_mask); |
2114 | |
2115 | ent = debugfs_create_file(name: "amdgpu_preempt_ib" , mode: 0600, parent: root, data: adev, |
2116 | fops: &fops_ib_preempt); |
2117 | if (IS_ERR(ptr: ent)) { |
2118 | DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n" ); |
2119 | return PTR_ERR(ptr: ent); |
2120 | } |
2121 | |
2122 | ent = debugfs_create_file(name: "amdgpu_force_sclk" , mode: 0200, parent: root, data: adev, |
2123 | fops: &fops_sclk_set); |
2124 | if (IS_ERR(ptr: ent)) { |
2125 | DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n" ); |
2126 | return PTR_ERR(ptr: ent); |
2127 | } |
2128 | |
2129 | /* Register debugfs entries for amdgpu_ttm */ |
2130 | amdgpu_ttm_debugfs_init(adev); |
2131 | amdgpu_debugfs_pm_init(adev); |
2132 | amdgpu_debugfs_sa_init(adev); |
2133 | amdgpu_debugfs_fence_init(adev); |
2134 | amdgpu_debugfs_gem_init(adev); |
2135 | |
2136 | r = amdgpu_debugfs_regs_init(adev); |
2137 | if (r) |
2138 | DRM_ERROR("registering register debugfs failed (%d).\n" , r); |
2139 | |
2140 | amdgpu_debugfs_firmware_init(adev); |
2141 | amdgpu_ta_if_debugfs_init(adev); |
2142 | |
2143 | #if defined(CONFIG_DRM_AMD_DC) |
2144 | if (adev->dc_enabled) |
2145 | dtn_debugfs_init(adev); |
2146 | #endif |
2147 | |
2148 | for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { |
2149 | struct amdgpu_ring *ring = adev->rings[i]; |
2150 | |
2151 | if (!ring) |
2152 | continue; |
2153 | |
2154 | amdgpu_debugfs_ring_init(adev, ring); |
2155 | } |
2156 | |
2157 | for (i = 0; i < adev->vcn.num_vcn_inst; i++) { |
2158 | if (!amdgpu_vcnfw_log) |
2159 | break; |
2160 | |
2161 | if (adev->vcn.harvest_config & (1 << i)) |
2162 | continue; |
2163 | |
2164 | amdgpu_debugfs_vcn_fwlog_init(adev, i, vcn: &adev->vcn.inst[i]); |
2165 | } |
2166 | |
2167 | amdgpu_ras_debugfs_create_all(adev); |
2168 | amdgpu_rap_debugfs_init(adev); |
2169 | amdgpu_securedisplay_debugfs_init(adev); |
2170 | amdgpu_fw_attestation_debugfs_init(adev); |
2171 | |
2172 | debugfs_create_file(name: "amdgpu_evict_vram" , mode: 0444, parent: root, data: adev, |
2173 | fops: &amdgpu_evict_vram_fops); |
2174 | debugfs_create_file(name: "amdgpu_evict_gtt" , mode: 0444, parent: root, data: adev, |
2175 | fops: &amdgpu_evict_gtt_fops); |
2176 | debugfs_create_file(name: "amdgpu_test_ib" , mode: 0444, parent: root, data: adev, |
2177 | fops: &amdgpu_debugfs_test_ib_fops); |
2178 | debugfs_create_file(name: "amdgpu_vm_info" , mode: 0444, parent: root, data: adev, |
2179 | fops: &amdgpu_debugfs_vm_info_fops); |
2180 | debugfs_create_file(name: "amdgpu_benchmark" , mode: 0200, parent: root, data: adev, |
2181 | fops: &amdgpu_benchmark_fops); |
2182 | debugfs_create_file(name: "amdgpu_reset_dump_register_list" , mode: 0644, parent: root, data: adev, |
2183 | fops: &amdgpu_reset_dump_register_list); |
2184 | |
2185 | adev->debugfs_vbios_blob.data = adev->bios; |
2186 | adev->debugfs_vbios_blob.size = adev->bios_size; |
2187 | debugfs_create_blob(name: "amdgpu_vbios" , mode: 0444, parent: root, |
2188 | blob: &adev->debugfs_vbios_blob); |
2189 | |
2190 | adev->debugfs_discovery_blob.data = adev->mman.discovery_bin; |
2191 | adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size; |
2192 | debugfs_create_blob(name: "amdgpu_discovery" , mode: 0444, parent: root, |
2193 | blob: &adev->debugfs_discovery_blob); |
2194 | |
2195 | return 0; |
2196 | } |
2197 | |
2198 | #else |
2199 | int amdgpu_debugfs_init(struct amdgpu_device *adev) |
2200 | { |
2201 | return 0; |
2202 | } |
2203 | int amdgpu_debugfs_regs_init(struct amdgpu_device *adev) |
2204 | { |
2205 | return 0; |
2206 | } |
2207 | #endif |
2208 | |