1 | /* |
2 | * Copyright 2016 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | */ |
23 | |
24 | #include <linux/module.h> |
25 | |
26 | #ifdef CONFIG_X86 |
27 | #include <asm/hypervisor.h> |
28 | #endif |
29 | |
30 | #include <drm/drm_drv.h> |
31 | #include <xen/xen.h> |
32 | |
33 | #include "amdgpu.h" |
34 | #include "amdgpu_ras.h" |
35 | #include "vi.h" |
36 | #include "soc15.h" |
37 | #include "nv.h" |
38 | |
39 | #define POPULATE_UCODE_INFO(vf2pf_info, ucode, ver) \ |
40 | do { \ |
41 | vf2pf_info->ucode_info[ucode].id = ucode; \ |
42 | vf2pf_info->ucode_info[ucode].version = ver; \ |
43 | } while (0) |
44 | |
45 | bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev) |
46 | { |
47 | /* By now all MMIO pages except mailbox are blocked */ |
48 | /* if blocking is enabled in hypervisor. Choose the */ |
49 | /* SCRATCH_REG0 to test. */ |
50 | return RREG32_NO_KIQ(0xc040) == 0xffffffff; |
51 | } |
52 | |
53 | void amdgpu_virt_init_setting(struct amdgpu_device *adev) |
54 | { |
55 | struct drm_device *ddev = adev_to_drm(adev); |
56 | |
57 | /* enable virtual display */ |
58 | if (adev->asic_type != CHIP_ALDEBARAN && |
59 | adev->asic_type != CHIP_ARCTURUS && |
60 | ((adev->pdev->class >> 8) != PCI_CLASS_ACCELERATOR_PROCESSING)) { |
61 | if (adev->mode_info.num_crtc == 0) |
62 | adev->mode_info.num_crtc = 1; |
63 | adev->enable_virtual_display = true; |
64 | } |
65 | ddev->driver_features &= ~DRIVER_ATOMIC; |
66 | adev->cg_flags = 0; |
67 | adev->pg_flags = 0; |
68 | |
69 | /* Reduce kcq number to 2 to reduce latency */ |
70 | if (amdgpu_num_kcq == -1) |
71 | amdgpu_num_kcq = 2; |
72 | } |
73 | |
74 | void amdgpu_virt_kiq_reg_write_reg_wait(struct amdgpu_device *adev, |
75 | uint32_t reg0, uint32_t reg1, |
76 | uint32_t ref, uint32_t mask) |
77 | { |
78 | struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; |
79 | struct amdgpu_ring *ring = &kiq->ring; |
80 | signed long r, cnt = 0; |
81 | unsigned long flags; |
82 | uint32_t seq; |
83 | |
84 | if (adev->mes.ring.sched.ready) { |
85 | amdgpu_mes_reg_write_reg_wait(adev, reg0, reg1, |
86 | ref, mask); |
87 | return; |
88 | } |
89 | |
90 | spin_lock_irqsave(&kiq->ring_lock, flags); |
91 | amdgpu_ring_alloc(ring, ndw: 32); |
92 | amdgpu_ring_emit_reg_write_reg_wait(ring, reg0, reg1, |
93 | ref, mask); |
94 | r = amdgpu_fence_emit_polling(ring, s: &seq, MAX_KIQ_REG_WAIT); |
95 | if (r) |
96 | goto failed_undo; |
97 | |
98 | amdgpu_ring_commit(ring); |
99 | spin_unlock_irqrestore(lock: &kiq->ring_lock, flags); |
100 | |
101 | r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT); |
102 | |
103 | /* don't wait anymore for IRQ context */ |
104 | if (r < 1 && in_interrupt()) |
105 | goto failed_kiq; |
106 | |
107 | might_sleep(); |
108 | while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { |
109 | |
110 | msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); |
111 | r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT); |
112 | } |
113 | |
114 | if (cnt > MAX_KIQ_REG_TRY) |
115 | goto failed_kiq; |
116 | |
117 | return; |
118 | |
119 | failed_undo: |
120 | amdgpu_ring_undo(ring); |
121 | spin_unlock_irqrestore(lock: &kiq->ring_lock, flags); |
122 | failed_kiq: |
123 | dev_err(adev->dev, "failed to write reg %x wait reg %x\n" , reg0, reg1); |
124 | } |
125 | |
126 | /** |
127 | * amdgpu_virt_request_full_gpu() - request full gpu access |
128 | * @adev: amdgpu device. |
129 | * @init: is driver init time. |
130 | * When start to init/fini driver, first need to request full gpu access. |
131 | * Return: Zero if request success, otherwise will return error. |
132 | */ |
133 | int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init) |
134 | { |
135 | struct amdgpu_virt *virt = &adev->virt; |
136 | int r; |
137 | |
138 | if (virt->ops && virt->ops->req_full_gpu) { |
139 | r = virt->ops->req_full_gpu(adev, init); |
140 | if (r) |
141 | return r; |
142 | |
143 | adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; |
144 | } |
145 | |
146 | return 0; |
147 | } |
148 | |
149 | /** |
150 | * amdgpu_virt_release_full_gpu() - release full gpu access |
151 | * @adev: amdgpu device. |
152 | * @init: is driver init time. |
153 | * When finishing driver init/fini, need to release full gpu access. |
154 | * Return: Zero if release success, otherwise will returen error. |
155 | */ |
156 | int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init) |
157 | { |
158 | struct amdgpu_virt *virt = &adev->virt; |
159 | int r; |
160 | |
161 | if (virt->ops && virt->ops->rel_full_gpu) { |
162 | r = virt->ops->rel_full_gpu(adev, init); |
163 | if (r) |
164 | return r; |
165 | |
166 | adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME; |
167 | } |
168 | return 0; |
169 | } |
170 | |
171 | /** |
172 | * amdgpu_virt_reset_gpu() - reset gpu |
173 | * @adev: amdgpu device. |
174 | * Send reset command to GPU hypervisor to reset GPU that VM is using |
175 | * Return: Zero if reset success, otherwise will return error. |
176 | */ |
177 | int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) |
178 | { |
179 | struct amdgpu_virt *virt = &adev->virt; |
180 | int r; |
181 | |
182 | if (virt->ops && virt->ops->reset_gpu) { |
183 | r = virt->ops->reset_gpu(adev); |
184 | if (r) |
185 | return r; |
186 | |
187 | adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; |
188 | } |
189 | |
190 | return 0; |
191 | } |
192 | |
193 | void amdgpu_virt_request_init_data(struct amdgpu_device *adev) |
194 | { |
195 | struct amdgpu_virt *virt = &adev->virt; |
196 | |
197 | if (virt->ops && virt->ops->req_init_data) |
198 | virt->ops->req_init_data(adev); |
199 | |
200 | if (adev->virt.req_init_data_ver > 0) |
201 | DRM_INFO("host supports REQ_INIT_DATA handshake\n" ); |
202 | else |
203 | DRM_WARN("host doesn't support REQ_INIT_DATA handshake\n" ); |
204 | } |
205 | |
206 | /** |
207 | * amdgpu_virt_wait_reset() - wait for reset gpu completed |
208 | * @adev: amdgpu device. |
209 | * Wait for GPU reset completed. |
210 | * Return: Zero if reset success, otherwise will return error. |
211 | */ |
212 | int amdgpu_virt_wait_reset(struct amdgpu_device *adev) |
213 | { |
214 | struct amdgpu_virt *virt = &adev->virt; |
215 | |
216 | if (!virt->ops || !virt->ops->wait_reset) |
217 | return -EINVAL; |
218 | |
219 | return virt->ops->wait_reset(adev); |
220 | } |
221 | |
222 | /** |
223 | * amdgpu_virt_alloc_mm_table() - alloc memory for mm table |
224 | * @adev: amdgpu device. |
225 | * MM table is used by UVD and VCE for its initialization |
226 | * Return: Zero if allocate success. |
227 | */ |
228 | int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev) |
229 | { |
230 | int r; |
231 | |
232 | if (!amdgpu_sriov_vf(adev) || adev->virt.mm_table.gpu_addr) |
233 | return 0; |
234 | |
235 | r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE, |
236 | AMDGPU_GEM_DOMAIN_VRAM | |
237 | AMDGPU_GEM_DOMAIN_GTT, |
238 | bo_ptr: &adev->virt.mm_table.bo, |
239 | gpu_addr: &adev->virt.mm_table.gpu_addr, |
240 | cpu_addr: (void *)&adev->virt.mm_table.cpu_addr); |
241 | if (r) { |
242 | DRM_ERROR("failed to alloc mm table and error = %d.\n" , r); |
243 | return r; |
244 | } |
245 | |
246 | memset((void *)adev->virt.mm_table.cpu_addr, 0, PAGE_SIZE); |
247 | DRM_INFO("MM table gpu addr = 0x%llx, cpu addr = %p.\n" , |
248 | adev->virt.mm_table.gpu_addr, |
249 | adev->virt.mm_table.cpu_addr); |
250 | return 0; |
251 | } |
252 | |
253 | /** |
254 | * amdgpu_virt_free_mm_table() - free mm table memory |
255 | * @adev: amdgpu device. |
256 | * Free MM table memory |
257 | */ |
258 | void amdgpu_virt_free_mm_table(struct amdgpu_device *adev) |
259 | { |
260 | if (!amdgpu_sriov_vf(adev) || !adev->virt.mm_table.gpu_addr) |
261 | return; |
262 | |
263 | amdgpu_bo_free_kernel(bo: &adev->virt.mm_table.bo, |
264 | gpu_addr: &adev->virt.mm_table.gpu_addr, |
265 | cpu_addr: (void *)&adev->virt.mm_table.cpu_addr); |
266 | adev->virt.mm_table.gpu_addr = 0; |
267 | } |
268 | |
269 | |
270 | unsigned int amd_sriov_msg_checksum(void *obj, |
271 | unsigned long obj_size, |
272 | unsigned int key, |
273 | unsigned int checksum) |
274 | { |
275 | unsigned int ret = key; |
276 | unsigned long i = 0; |
277 | unsigned char *pos; |
278 | |
279 | pos = (char *)obj; |
280 | /* calculate checksum */ |
281 | for (i = 0; i < obj_size; ++i) |
282 | ret += *(pos + i); |
283 | /* minus the checksum itself */ |
284 | pos = (char *)&checksum; |
285 | for (i = 0; i < sizeof(checksum); ++i) |
286 | ret -= *(pos + i); |
287 | return ret; |
288 | } |
289 | |
290 | static int amdgpu_virt_init_ras_err_handler_data(struct amdgpu_device *adev) |
291 | { |
292 | struct amdgpu_virt *virt = &adev->virt; |
293 | struct amdgpu_virt_ras_err_handler_data **data = &virt->virt_eh_data; |
294 | /* GPU will be marked bad on host if bp count more then 10, |
295 | * so alloc 512 is enough. |
296 | */ |
297 | unsigned int align_space = 512; |
298 | void *bps = NULL; |
299 | struct amdgpu_bo **bps_bo = NULL; |
300 | |
301 | *data = kmalloc(size: sizeof(struct amdgpu_virt_ras_err_handler_data), GFP_KERNEL); |
302 | if (!*data) |
303 | goto data_failure; |
304 | |
305 | bps = kmalloc_array(n: align_space, size: sizeof((*data)->bps), GFP_KERNEL); |
306 | if (!bps) |
307 | goto bps_failure; |
308 | |
309 | bps_bo = kmalloc_array(n: align_space, size: sizeof((*data)->bps_bo), GFP_KERNEL); |
310 | if (!bps_bo) |
311 | goto bps_bo_failure; |
312 | |
313 | (*data)->bps = bps; |
314 | (*data)->bps_bo = bps_bo; |
315 | (*data)->count = 0; |
316 | (*data)->last_reserved = 0; |
317 | |
318 | virt->ras_init_done = true; |
319 | |
320 | return 0; |
321 | |
322 | bps_bo_failure: |
323 | kfree(objp: bps); |
324 | bps_failure: |
325 | kfree(objp: *data); |
326 | data_failure: |
327 | return -ENOMEM; |
328 | } |
329 | |
330 | static void amdgpu_virt_ras_release_bp(struct amdgpu_device *adev) |
331 | { |
332 | struct amdgpu_virt *virt = &adev->virt; |
333 | struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; |
334 | struct amdgpu_bo *bo; |
335 | int i; |
336 | |
337 | if (!data) |
338 | return; |
339 | |
340 | for (i = data->last_reserved - 1; i >= 0; i--) { |
341 | bo = data->bps_bo[i]; |
342 | amdgpu_bo_free_kernel(bo: &bo, NULL, NULL); |
343 | data->bps_bo[i] = bo; |
344 | data->last_reserved = i; |
345 | } |
346 | } |
347 | |
348 | void amdgpu_virt_release_ras_err_handler_data(struct amdgpu_device *adev) |
349 | { |
350 | struct amdgpu_virt *virt = &adev->virt; |
351 | struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; |
352 | |
353 | virt->ras_init_done = false; |
354 | |
355 | if (!data) |
356 | return; |
357 | |
358 | amdgpu_virt_ras_release_bp(adev); |
359 | |
360 | kfree(objp: data->bps); |
361 | kfree(objp: data->bps_bo); |
362 | kfree(objp: data); |
363 | virt->virt_eh_data = NULL; |
364 | } |
365 | |
366 | static void amdgpu_virt_ras_add_bps(struct amdgpu_device *adev, |
367 | struct eeprom_table_record *bps, int pages) |
368 | { |
369 | struct amdgpu_virt *virt = &adev->virt; |
370 | struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; |
371 | |
372 | if (!data) |
373 | return; |
374 | |
375 | memcpy(&data->bps[data->count], bps, pages * sizeof(*data->bps)); |
376 | data->count += pages; |
377 | } |
378 | |
379 | static void amdgpu_virt_ras_reserve_bps(struct amdgpu_device *adev) |
380 | { |
381 | struct amdgpu_virt *virt = &adev->virt; |
382 | struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; |
383 | struct amdgpu_bo *bo = NULL; |
384 | uint64_t bp; |
385 | int i; |
386 | |
387 | if (!data) |
388 | return; |
389 | |
390 | for (i = data->last_reserved; i < data->count; i++) { |
391 | bp = data->bps[i].retired_page; |
392 | |
393 | /* There are two cases of reserve error should be ignored: |
394 | * 1) a ras bad page has been allocated (used by someone); |
395 | * 2) a ras bad page has been reserved (duplicate error injection |
396 | * for one page); |
397 | */ |
398 | if (amdgpu_bo_create_kernel_at(adev, offset: bp << AMDGPU_GPU_PAGE_SHIFT, |
399 | AMDGPU_GPU_PAGE_SIZE, |
400 | bo_ptr: &bo, NULL)) |
401 | DRM_DEBUG("RAS WARN: reserve vram for retired page %llx fail\n" , bp); |
402 | |
403 | data->bps_bo[i] = bo; |
404 | data->last_reserved = i + 1; |
405 | bo = NULL; |
406 | } |
407 | } |
408 | |
409 | static bool amdgpu_virt_ras_check_bad_page(struct amdgpu_device *adev, |
410 | uint64_t retired_page) |
411 | { |
412 | struct amdgpu_virt *virt = &adev->virt; |
413 | struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; |
414 | int i; |
415 | |
416 | if (!data) |
417 | return true; |
418 | |
419 | for (i = 0; i < data->count; i++) |
420 | if (retired_page == data->bps[i].retired_page) |
421 | return true; |
422 | |
423 | return false; |
424 | } |
425 | |
426 | static void amdgpu_virt_add_bad_page(struct amdgpu_device *adev, |
427 | uint64_t bp_block_offset, uint32_t bp_block_size) |
428 | { |
429 | struct eeprom_table_record bp; |
430 | uint64_t retired_page; |
431 | uint32_t bp_idx, bp_cnt; |
432 | void *vram_usage_va = NULL; |
433 | |
434 | if (adev->mman.fw_vram_usage_va) |
435 | vram_usage_va = adev->mman.fw_vram_usage_va; |
436 | else |
437 | vram_usage_va = adev->mman.drv_vram_usage_va; |
438 | |
439 | if (bp_block_size) { |
440 | bp_cnt = bp_block_size / sizeof(uint64_t); |
441 | for (bp_idx = 0; bp_idx < bp_cnt; bp_idx++) { |
442 | retired_page = *(uint64_t *)(vram_usage_va + |
443 | bp_block_offset + bp_idx * sizeof(uint64_t)); |
444 | bp.retired_page = retired_page; |
445 | |
446 | if (amdgpu_virt_ras_check_bad_page(adev, retired_page)) |
447 | continue; |
448 | |
449 | amdgpu_virt_ras_add_bps(adev, bps: &bp, pages: 1); |
450 | |
451 | amdgpu_virt_ras_reserve_bps(adev); |
452 | } |
453 | } |
454 | } |
455 | |
456 | static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) |
457 | { |
458 | struct amd_sriov_msg_pf2vf_info_header *pf2vf_info = adev->virt.fw_reserve.p_pf2vf; |
459 | uint32_t checksum; |
460 | uint32_t checkval; |
461 | |
462 | uint32_t i; |
463 | uint32_t tmp; |
464 | |
465 | if (adev->virt.fw_reserve.p_pf2vf == NULL) |
466 | return -EINVAL; |
467 | |
468 | if (pf2vf_info->size > 1024) { |
469 | DRM_ERROR("invalid pf2vf message size\n" ); |
470 | return -EINVAL; |
471 | } |
472 | |
473 | switch (pf2vf_info->version) { |
474 | case 1: |
475 | checksum = ((struct amdgim_pf2vf_info_v1 *)pf2vf_info)->checksum; |
476 | checkval = amd_sriov_msg_checksum( |
477 | obj: adev->virt.fw_reserve.p_pf2vf, obj_size: pf2vf_info->size, |
478 | key: adev->virt.fw_reserve.checksum_key, checksum); |
479 | if (checksum != checkval) { |
480 | DRM_ERROR("invalid pf2vf message\n" ); |
481 | return -EINVAL; |
482 | } |
483 | |
484 | adev->virt.gim_feature = |
485 | ((struct amdgim_pf2vf_info_v1 *)pf2vf_info)->feature_flags; |
486 | break; |
487 | case 2: |
488 | /* TODO: missing key, need to add it later */ |
489 | checksum = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->checksum; |
490 | checkval = amd_sriov_msg_checksum( |
491 | obj: adev->virt.fw_reserve.p_pf2vf, obj_size: pf2vf_info->size, |
492 | key: 0, checksum); |
493 | if (checksum != checkval) { |
494 | DRM_ERROR("invalid pf2vf message\n" ); |
495 | return -EINVAL; |
496 | } |
497 | |
498 | adev->virt.vf2pf_update_interval_ms = |
499 | ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->vf2pf_update_interval_ms; |
500 | adev->virt.gim_feature = |
501 | ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->feature_flags.all; |
502 | adev->virt.reg_access = |
503 | ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->reg_access_flags.all; |
504 | |
505 | adev->virt.decode_max_dimension_pixels = 0; |
506 | adev->virt.decode_max_frame_pixels = 0; |
507 | adev->virt.encode_max_dimension_pixels = 0; |
508 | adev->virt.encode_max_frame_pixels = 0; |
509 | adev->virt.is_mm_bw_enabled = false; |
510 | for (i = 0; i < AMD_SRIOV_MSG_RESERVE_VCN_INST; i++) { |
511 | tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].decode_max_dimension_pixels; |
512 | adev->virt.decode_max_dimension_pixels = max(tmp, adev->virt.decode_max_dimension_pixels); |
513 | |
514 | tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].decode_max_frame_pixels; |
515 | adev->virt.decode_max_frame_pixels = max(tmp, adev->virt.decode_max_frame_pixels); |
516 | |
517 | tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].encode_max_dimension_pixels; |
518 | adev->virt.encode_max_dimension_pixels = max(tmp, adev->virt.encode_max_dimension_pixels); |
519 | |
520 | tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].encode_max_frame_pixels; |
521 | adev->virt.encode_max_frame_pixels = max(tmp, adev->virt.encode_max_frame_pixels); |
522 | } |
523 | if ((adev->virt.decode_max_dimension_pixels > 0) || (adev->virt.encode_max_dimension_pixels > 0)) |
524 | adev->virt.is_mm_bw_enabled = true; |
525 | |
526 | adev->unique_id = |
527 | ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->uuid; |
528 | break; |
529 | default: |
530 | DRM_ERROR("invalid pf2vf version\n" ); |
531 | return -EINVAL; |
532 | } |
533 | |
534 | /* correct too large or too little interval value */ |
535 | if (adev->virt.vf2pf_update_interval_ms < 200 || adev->virt.vf2pf_update_interval_ms > 10000) |
536 | adev->virt.vf2pf_update_interval_ms = 2000; |
537 | |
538 | return 0; |
539 | } |
540 | |
541 | static void amdgpu_virt_populate_vf2pf_ucode_info(struct amdgpu_device *adev) |
542 | { |
543 | struct amd_sriov_msg_vf2pf_info *vf2pf_info; |
544 | vf2pf_info = (struct amd_sriov_msg_vf2pf_info *) adev->virt.fw_reserve.p_vf2pf; |
545 | |
546 | if (adev->virt.fw_reserve.p_vf2pf == NULL) |
547 | return; |
548 | |
549 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_VCE, adev->vce.fw_version); |
550 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_UVD, adev->uvd.fw_version); |
551 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MC, adev->gmc.fw_version); |
552 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ME, adev->gfx.me_fw_version); |
553 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_PFP, adev->gfx.pfp_fw_version); |
554 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_CE, adev->gfx.ce_fw_version); |
555 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC, adev->gfx.rlc_fw_version); |
556 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLC, adev->gfx.rlc_srlc_fw_version); |
557 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLG, adev->gfx.rlc_srlg_fw_version); |
558 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLS, adev->gfx.rlc_srls_fw_version); |
559 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC, adev->gfx.mec_fw_version); |
560 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC2, adev->gfx.mec2_fw_version); |
561 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SOS, adev->psp.sos.fw_version); |
562 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ASD, |
563 | adev->psp.asd_context.bin_desc.fw_version); |
564 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_RAS, |
565 | adev->psp.ras_context.context.bin_desc.fw_version); |
566 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_XGMI, |
567 | adev->psp.xgmi_context.context.bin_desc.fw_version); |
568 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SMC, adev->pm.fw_version); |
569 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SDMA, adev->sdma.instance[0].fw_version); |
570 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SDMA2, adev->sdma.instance[1].fw_version); |
571 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_VCN, adev->vcn.fw_version); |
572 | POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_DMCU, adev->dm.dmcu_fw_version); |
573 | } |
574 | |
575 | static int amdgpu_virt_write_vf2pf_data(struct amdgpu_device *adev) |
576 | { |
577 | struct amd_sriov_msg_vf2pf_info *vf2pf_info; |
578 | |
579 | vf2pf_info = (struct amd_sriov_msg_vf2pf_info *) adev->virt.fw_reserve.p_vf2pf; |
580 | |
581 | if (adev->virt.fw_reserve.p_vf2pf == NULL) |
582 | return -EINVAL; |
583 | |
584 | memset(vf2pf_info, 0, sizeof(struct amd_sriov_msg_vf2pf_info)); |
585 | |
586 | vf2pf_info->header.size = sizeof(struct amd_sriov_msg_vf2pf_info); |
587 | vf2pf_info->header.version = AMD_SRIOV_MSG_FW_VRAM_VF2PF_VER; |
588 | |
589 | #ifdef MODULE |
590 | if (THIS_MODULE->version != NULL) |
591 | strcpy(vf2pf_info->driver_version, THIS_MODULE->version); |
592 | else |
593 | #endif |
594 | strcpy(p: vf2pf_info->driver_version, q: "N/A" ); |
595 | |
596 | vf2pf_info->pf2vf_version_required = 0; // no requirement, guest understands all |
597 | vf2pf_info->driver_cert = 0; |
598 | vf2pf_info->os_info.all = 0; |
599 | |
600 | vf2pf_info->fb_usage = |
601 | ttm_resource_manager_usage(man: &adev->mman.vram_mgr.manager) >> 20; |
602 | vf2pf_info->fb_vis_usage = |
603 | amdgpu_vram_mgr_vis_usage(mgr: &adev->mman.vram_mgr) >> 20; |
604 | vf2pf_info->fb_size = adev->gmc.real_vram_size >> 20; |
605 | vf2pf_info->fb_vis_size = adev->gmc.visible_vram_size >> 20; |
606 | |
607 | amdgpu_virt_populate_vf2pf_ucode_info(adev); |
608 | |
609 | /* TODO: read dynamic info */ |
610 | vf2pf_info->gfx_usage = 0; |
611 | vf2pf_info->compute_usage = 0; |
612 | vf2pf_info->encode_usage = 0; |
613 | vf2pf_info->decode_usage = 0; |
614 | |
615 | vf2pf_info->dummy_page_addr = (uint64_t)adev->dummy_page_addr; |
616 | vf2pf_info->checksum = |
617 | amd_sriov_msg_checksum( |
618 | obj: vf2pf_info, obj_size: vf2pf_info->header.size, key: 0, checksum: 0); |
619 | |
620 | return 0; |
621 | } |
622 | |
623 | static void amdgpu_virt_update_vf2pf_work_item(struct work_struct *work) |
624 | { |
625 | struct amdgpu_device *adev = container_of(work, struct amdgpu_device, virt.vf2pf_work.work); |
626 | int ret; |
627 | |
628 | ret = amdgpu_virt_read_pf2vf_data(adev); |
629 | if (ret) |
630 | goto out; |
631 | amdgpu_virt_write_vf2pf_data(adev); |
632 | |
633 | out: |
634 | schedule_delayed_work(dwork: &(adev->virt.vf2pf_work), delay: adev->virt.vf2pf_update_interval_ms); |
635 | } |
636 | |
637 | void amdgpu_virt_fini_data_exchange(struct amdgpu_device *adev) |
638 | { |
639 | if (adev->virt.vf2pf_update_interval_ms != 0) { |
640 | DRM_INFO("clean up the vf2pf work item\n" ); |
641 | cancel_delayed_work_sync(dwork: &adev->virt.vf2pf_work); |
642 | adev->virt.vf2pf_update_interval_ms = 0; |
643 | } |
644 | } |
645 | |
646 | void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev) |
647 | { |
648 | adev->virt.fw_reserve.p_pf2vf = NULL; |
649 | adev->virt.fw_reserve.p_vf2pf = NULL; |
650 | adev->virt.vf2pf_update_interval_ms = 0; |
651 | |
652 | if (adev->mman.fw_vram_usage_va && adev->mman.drv_vram_usage_va) { |
653 | DRM_WARN("Currently fw_vram and drv_vram should not have values at the same time!" ); |
654 | } else if (adev->mman.fw_vram_usage_va || adev->mman.drv_vram_usage_va) { |
655 | /* go through this logic in ip_init and reset to init workqueue*/ |
656 | amdgpu_virt_exchange_data(adev); |
657 | |
658 | INIT_DELAYED_WORK(&adev->virt.vf2pf_work, amdgpu_virt_update_vf2pf_work_item); |
659 | schedule_delayed_work(dwork: &(adev->virt.vf2pf_work), delay: msecs_to_jiffies(m: adev->virt.vf2pf_update_interval_ms)); |
660 | } else if (adev->bios != NULL) { |
661 | /* got through this logic in early init stage to get necessary flags, e.g. rlcg_acc related*/ |
662 | adev->virt.fw_reserve.p_pf2vf = |
663 | (struct amd_sriov_msg_pf2vf_info_header *) |
664 | (adev->bios + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB << 10)); |
665 | |
666 | amdgpu_virt_read_pf2vf_data(adev); |
667 | } |
668 | } |
669 | |
670 | |
671 | void amdgpu_virt_exchange_data(struct amdgpu_device *adev) |
672 | { |
673 | uint64_t bp_block_offset = 0; |
674 | uint32_t bp_block_size = 0; |
675 | struct amd_sriov_msg_pf2vf_info *pf2vf_v2 = NULL; |
676 | |
677 | if (adev->mman.fw_vram_usage_va || adev->mman.drv_vram_usage_va) { |
678 | if (adev->mman.fw_vram_usage_va) { |
679 | adev->virt.fw_reserve.p_pf2vf = |
680 | (struct amd_sriov_msg_pf2vf_info_header *) |
681 | (adev->mman.fw_vram_usage_va + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB << 10)); |
682 | adev->virt.fw_reserve.p_vf2pf = |
683 | (struct amd_sriov_msg_vf2pf_info_header *) |
684 | (adev->mman.fw_vram_usage_va + (AMD_SRIOV_MSG_VF2PF_OFFSET_KB << 10)); |
685 | } else if (adev->mman.drv_vram_usage_va) { |
686 | adev->virt.fw_reserve.p_pf2vf = |
687 | (struct amd_sriov_msg_pf2vf_info_header *) |
688 | (adev->mman.drv_vram_usage_va + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB << 10)); |
689 | adev->virt.fw_reserve.p_vf2pf = |
690 | (struct amd_sriov_msg_vf2pf_info_header *) |
691 | (adev->mman.drv_vram_usage_va + (AMD_SRIOV_MSG_VF2PF_OFFSET_KB << 10)); |
692 | } |
693 | |
694 | amdgpu_virt_read_pf2vf_data(adev); |
695 | amdgpu_virt_write_vf2pf_data(adev); |
696 | |
697 | /* bad page handling for version 2 */ |
698 | if (adev->virt.fw_reserve.p_pf2vf->version == 2) { |
699 | pf2vf_v2 = (struct amd_sriov_msg_pf2vf_info *)adev->virt.fw_reserve.p_pf2vf; |
700 | |
701 | bp_block_offset = ((uint64_t)pf2vf_v2->bp_block_offset_low & 0xFFFFFFFF) | |
702 | ((((uint64_t)pf2vf_v2->bp_block_offset_high) << 32) & 0xFFFFFFFF00000000); |
703 | bp_block_size = pf2vf_v2->bp_block_size; |
704 | |
705 | if (bp_block_size && !adev->virt.ras_init_done) |
706 | amdgpu_virt_init_ras_err_handler_data(adev); |
707 | |
708 | if (adev->virt.ras_init_done) |
709 | amdgpu_virt_add_bad_page(adev, bp_block_offset, bp_block_size); |
710 | } |
711 | } |
712 | } |
713 | |
714 | void amdgpu_detect_virtualization(struct amdgpu_device *adev) |
715 | { |
716 | uint32_t reg; |
717 | |
718 | switch (adev->asic_type) { |
719 | case CHIP_TONGA: |
720 | case CHIP_FIJI: |
721 | reg = RREG32(mmBIF_IOV_FUNC_IDENTIFIER); |
722 | break; |
723 | case CHIP_VEGA10: |
724 | case CHIP_VEGA20: |
725 | case CHIP_NAVI10: |
726 | case CHIP_NAVI12: |
727 | case CHIP_SIENNA_CICHLID: |
728 | case CHIP_ARCTURUS: |
729 | case CHIP_ALDEBARAN: |
730 | case CHIP_IP_DISCOVERY: |
731 | reg = RREG32(mmRCC_IOV_FUNC_IDENTIFIER); |
732 | break; |
733 | default: /* other chip doesn't support SRIOV */ |
734 | reg = 0; |
735 | break; |
736 | } |
737 | |
738 | if (reg & 1) |
739 | adev->virt.caps |= AMDGPU_SRIOV_CAPS_IS_VF; |
740 | |
741 | if (reg & 0x80000000) |
742 | adev->virt.caps |= AMDGPU_SRIOV_CAPS_ENABLE_IOV; |
743 | |
744 | if (!reg) { |
745 | /* passthrough mode exclus sriov mod */ |
746 | if (is_virtual_machine() && !xen_initial_domain()) |
747 | adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE; |
748 | } |
749 | |
750 | if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID) |
751 | /* VF MMIO access (except mailbox range) from CPU |
752 | * will be blocked during sriov runtime |
753 | */ |
754 | adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT; |
755 | |
756 | /* we have the ability to check now */ |
757 | if (amdgpu_sriov_vf(adev)) { |
758 | switch (adev->asic_type) { |
759 | case CHIP_TONGA: |
760 | case CHIP_FIJI: |
761 | vi_set_virt_ops(adev); |
762 | break; |
763 | case CHIP_VEGA10: |
764 | soc15_set_virt_ops(adev); |
765 | #ifdef CONFIG_X86 |
766 | /* not send GPU_INIT_DATA with MS_HYPERV*/ |
767 | if (!hypervisor_is_type(type: X86_HYPER_MS_HYPERV)) |
768 | #endif |
769 | /* send a dummy GPU_INIT_DATA request to host on vega10 */ |
770 | amdgpu_virt_request_init_data(adev); |
771 | break; |
772 | case CHIP_VEGA20: |
773 | case CHIP_ARCTURUS: |
774 | case CHIP_ALDEBARAN: |
775 | soc15_set_virt_ops(adev); |
776 | break; |
777 | case CHIP_NAVI10: |
778 | case CHIP_NAVI12: |
779 | case CHIP_SIENNA_CICHLID: |
780 | case CHIP_IP_DISCOVERY: |
781 | nv_set_virt_ops(adev); |
782 | /* try send GPU_INIT_DATA request to host */ |
783 | amdgpu_virt_request_init_data(adev); |
784 | break; |
785 | default: /* other chip doesn't support SRIOV */ |
786 | DRM_ERROR("Unknown asic type: %d!\n" , adev->asic_type); |
787 | break; |
788 | } |
789 | } |
790 | } |
791 | |
792 | static bool amdgpu_virt_access_debugfs_is_mmio(struct amdgpu_device *adev) |
793 | { |
794 | return amdgpu_sriov_is_debug(adev) ? true : false; |
795 | } |
796 | |
797 | static bool amdgpu_virt_access_debugfs_is_kiq(struct amdgpu_device *adev) |
798 | { |
799 | return amdgpu_sriov_is_normal(adev) ? true : false; |
800 | } |
801 | |
802 | int amdgpu_virt_enable_access_debugfs(struct amdgpu_device *adev) |
803 | { |
804 | if (!amdgpu_sriov_vf(adev) || |
805 | amdgpu_virt_access_debugfs_is_kiq(adev)) |
806 | return 0; |
807 | |
808 | if (amdgpu_virt_access_debugfs_is_mmio(adev)) |
809 | adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; |
810 | else |
811 | return -EPERM; |
812 | |
813 | return 0; |
814 | } |
815 | |
816 | void amdgpu_virt_disable_access_debugfs(struct amdgpu_device *adev) |
817 | { |
818 | if (amdgpu_sriov_vf(adev)) |
819 | adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME; |
820 | } |
821 | |
822 | enum amdgpu_sriov_vf_mode amdgpu_virt_get_sriov_vf_mode(struct amdgpu_device *adev) |
823 | { |
824 | enum amdgpu_sriov_vf_mode mode; |
825 | |
826 | if (amdgpu_sriov_vf(adev)) { |
827 | if (amdgpu_sriov_is_pp_one_vf(adev)) |
828 | mode = SRIOV_VF_MODE_ONE_VF; |
829 | else |
830 | mode = SRIOV_VF_MODE_MULTI_VF; |
831 | } else { |
832 | mode = SRIOV_VF_MODE_BARE_METAL; |
833 | } |
834 | |
835 | return mode; |
836 | } |
837 | |
838 | void amdgpu_virt_post_reset(struct amdgpu_device *adev) |
839 | { |
840 | if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(11, 0, 3)) { |
841 | /* force set to GFXOFF state after reset, |
842 | * to avoid some invalid operation before GC enable |
843 | */ |
844 | adev->gfx.is_poweron = false; |
845 | } |
846 | } |
847 | |
848 | bool amdgpu_virt_fw_load_skip_check(struct amdgpu_device *adev, uint32_t ucode_id) |
849 | { |
850 | switch (amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0)) { |
851 | case IP_VERSION(13, 0, 0): |
852 | /* no vf autoload, white list */ |
853 | if (ucode_id == AMDGPU_UCODE_ID_VCN1 || |
854 | ucode_id == AMDGPU_UCODE_ID_VCN) |
855 | return false; |
856 | else |
857 | return true; |
858 | case IP_VERSION(11, 0, 9): |
859 | case IP_VERSION(11, 0, 7): |
860 | /* black list for CHIP_NAVI12 and CHIP_SIENNA_CICHLID */ |
861 | if (ucode_id == AMDGPU_UCODE_ID_RLC_G |
862 | || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL |
863 | || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM |
864 | || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM |
865 | || ucode_id == AMDGPU_UCODE_ID_SMC) |
866 | return true; |
867 | else |
868 | return false; |
869 | case IP_VERSION(13, 0, 10): |
870 | /* white list */ |
871 | if (ucode_id == AMDGPU_UCODE_ID_CAP |
872 | || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP |
873 | || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME |
874 | || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC |
875 | || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK |
876 | || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK |
877 | || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK |
878 | || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK |
879 | || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK |
880 | || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK |
881 | || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK |
882 | || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK |
883 | || ucode_id == AMDGPU_UCODE_ID_CP_MES |
884 | || ucode_id == AMDGPU_UCODE_ID_CP_MES_DATA |
885 | || ucode_id == AMDGPU_UCODE_ID_CP_MES1 |
886 | || ucode_id == AMDGPU_UCODE_ID_CP_MES1_DATA |
887 | || ucode_id == AMDGPU_UCODE_ID_VCN1 |
888 | || ucode_id == AMDGPU_UCODE_ID_VCN) |
889 | return false; |
890 | else |
891 | return true; |
892 | default: |
893 | /* lagacy black list */ |
894 | if (ucode_id == AMDGPU_UCODE_ID_SDMA0 |
895 | || ucode_id == AMDGPU_UCODE_ID_SDMA1 |
896 | || ucode_id == AMDGPU_UCODE_ID_SDMA2 |
897 | || ucode_id == AMDGPU_UCODE_ID_SDMA3 |
898 | || ucode_id == AMDGPU_UCODE_ID_SDMA4 |
899 | || ucode_id == AMDGPU_UCODE_ID_SDMA5 |
900 | || ucode_id == AMDGPU_UCODE_ID_SDMA6 |
901 | || ucode_id == AMDGPU_UCODE_ID_SDMA7 |
902 | || ucode_id == AMDGPU_UCODE_ID_RLC_G |
903 | || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL |
904 | || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM |
905 | || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM |
906 | || ucode_id == AMDGPU_UCODE_ID_SMC) |
907 | return true; |
908 | else |
909 | return false; |
910 | } |
911 | } |
912 | |
913 | void amdgpu_virt_update_sriov_video_codec(struct amdgpu_device *adev, |
914 | struct amdgpu_video_codec_info *encode, uint32_t encode_array_size, |
915 | struct amdgpu_video_codec_info *decode, uint32_t decode_array_size) |
916 | { |
917 | uint32_t i; |
918 | |
919 | if (!adev->virt.is_mm_bw_enabled) |
920 | return; |
921 | |
922 | if (encode) { |
923 | for (i = 0; i < encode_array_size; i++) { |
924 | encode[i].max_width = adev->virt.encode_max_dimension_pixels; |
925 | encode[i].max_pixels_per_frame = adev->virt.encode_max_frame_pixels; |
926 | if (encode[i].max_width > 0) |
927 | encode[i].max_height = encode[i].max_pixels_per_frame / encode[i].max_width; |
928 | else |
929 | encode[i].max_height = 0; |
930 | } |
931 | } |
932 | |
933 | if (decode) { |
934 | for (i = 0; i < decode_array_size; i++) { |
935 | decode[i].max_width = adev->virt.decode_max_dimension_pixels; |
936 | decode[i].max_pixels_per_frame = adev->virt.decode_max_frame_pixels; |
937 | if (decode[i].max_width > 0) |
938 | decode[i].max_height = decode[i].max_pixels_per_frame / decode[i].max_width; |
939 | else |
940 | decode[i].max_height = 0; |
941 | } |
942 | } |
943 | } |
944 | |
945 | static bool amdgpu_virt_get_rlcg_reg_access_flag(struct amdgpu_device *adev, |
946 | u32 acc_flags, u32 hwip, |
947 | bool write, u32 *rlcg_flag) |
948 | { |
949 | bool ret = false; |
950 | |
951 | switch (hwip) { |
952 | case GC_HWIP: |
953 | if (amdgpu_sriov_reg_indirect_gc(adev)) { |
954 | *rlcg_flag = |
955 | write ? AMDGPU_RLCG_GC_WRITE : AMDGPU_RLCG_GC_READ; |
956 | ret = true; |
957 | /* only in new version, AMDGPU_REGS_NO_KIQ and |
958 | * AMDGPU_REGS_RLC are enabled simultaneously */ |
959 | } else if ((acc_flags & AMDGPU_REGS_RLC) && |
960 | !(acc_flags & AMDGPU_REGS_NO_KIQ) && write) { |
961 | *rlcg_flag = AMDGPU_RLCG_GC_WRITE_LEGACY; |
962 | ret = true; |
963 | } |
964 | break; |
965 | case MMHUB_HWIP: |
966 | if (amdgpu_sriov_reg_indirect_mmhub(adev) && |
967 | (acc_flags & AMDGPU_REGS_RLC) && write) { |
968 | *rlcg_flag = AMDGPU_RLCG_MMHUB_WRITE; |
969 | ret = true; |
970 | } |
971 | break; |
972 | default: |
973 | break; |
974 | } |
975 | return ret; |
976 | } |
977 | |
978 | static u32 amdgpu_virt_rlcg_reg_rw(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag, u32 xcc_id) |
979 | { |
980 | struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; |
981 | uint32_t timeout = 50000; |
982 | uint32_t i, tmp; |
983 | uint32_t ret = 0; |
984 | void *scratch_reg0; |
985 | void *scratch_reg1; |
986 | void *scratch_reg2; |
987 | void *scratch_reg3; |
988 | void *spare_int; |
989 | |
990 | if (!adev->gfx.rlc.rlcg_reg_access_supported) { |
991 | dev_err(adev->dev, |
992 | "indirect registers access through rlcg is not available\n" ); |
993 | return 0; |
994 | } |
995 | |
996 | if (adev->gfx.xcc_mask && (((1 << xcc_id) & adev->gfx.xcc_mask) == 0)) { |
997 | dev_err(adev->dev, "invalid xcc\n" ); |
998 | return 0; |
999 | } |
1000 | |
1001 | reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[xcc_id]; |
1002 | scratch_reg0 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg0; |
1003 | scratch_reg1 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg1; |
1004 | scratch_reg2 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg2; |
1005 | scratch_reg3 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg3; |
1006 | if (reg_access_ctrl->spare_int) |
1007 | spare_int = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->spare_int; |
1008 | |
1009 | if (offset == reg_access_ctrl->grbm_cntl) { |
1010 | /* if the target reg offset is grbm_cntl, write to scratch_reg2 */ |
1011 | writel(val: v, addr: scratch_reg2); |
1012 | if (flag == AMDGPU_RLCG_GC_WRITE_LEGACY) |
1013 | writel(val: v, addr: ((void __iomem *)adev->rmmio) + (offset * 4)); |
1014 | } else if (offset == reg_access_ctrl->grbm_idx) { |
1015 | /* if the target reg offset is grbm_idx, write to scratch_reg3 */ |
1016 | writel(val: v, addr: scratch_reg3); |
1017 | if (flag == AMDGPU_RLCG_GC_WRITE_LEGACY) |
1018 | writel(val: v, addr: ((void __iomem *)adev->rmmio) + (offset * 4)); |
1019 | } else { |
1020 | /* |
1021 | * SCRATCH_REG0 = read/write value |
1022 | * SCRATCH_REG1[30:28] = command |
1023 | * SCRATCH_REG1[19:0] = address in dword |
1024 | * SCRATCH_REG1[26:24] = Error reporting |
1025 | */ |
1026 | writel(val: v, addr: scratch_reg0); |
1027 | writel(val: (offset | flag), addr: scratch_reg1); |
1028 | if (reg_access_ctrl->spare_int) |
1029 | writel(val: 1, addr: spare_int); |
1030 | |
1031 | for (i = 0; i < timeout; i++) { |
1032 | tmp = readl(addr: scratch_reg1); |
1033 | if (!(tmp & AMDGPU_RLCG_SCRATCH1_ADDRESS_MASK)) |
1034 | break; |
1035 | udelay(10); |
1036 | } |
1037 | |
1038 | if (i >= timeout) { |
1039 | if (amdgpu_sriov_rlcg_error_report_enabled(adev)) { |
1040 | if (tmp & AMDGPU_RLCG_VFGATE_DISABLED) { |
1041 | dev_err(adev->dev, |
1042 | "vfgate is disabled, rlcg failed to program reg: 0x%05x\n" , offset); |
1043 | } else if (tmp & AMDGPU_RLCG_WRONG_OPERATION_TYPE) { |
1044 | dev_err(adev->dev, |
1045 | "wrong operation type, rlcg failed to program reg: 0x%05x\n" , offset); |
1046 | } else if (tmp & AMDGPU_RLCG_REG_NOT_IN_RANGE) { |
1047 | dev_err(adev->dev, |
1048 | "register is not in range, rlcg failed to program reg: 0x%05x\n" , offset); |
1049 | } else { |
1050 | dev_err(adev->dev, |
1051 | "unknown error type, rlcg failed to program reg: 0x%05x\n" , offset); |
1052 | } |
1053 | } else { |
1054 | dev_err(adev->dev, |
1055 | "timeout: rlcg faled to program reg: 0x%05x\n" , offset); |
1056 | } |
1057 | } |
1058 | } |
1059 | |
1060 | ret = readl(addr: scratch_reg0); |
1061 | return ret; |
1062 | } |
1063 | |
1064 | void amdgpu_sriov_wreg(struct amdgpu_device *adev, |
1065 | u32 offset, u32 value, |
1066 | u32 acc_flags, u32 hwip, u32 xcc_id) |
1067 | { |
1068 | u32 rlcg_flag; |
1069 | |
1070 | if (!amdgpu_sriov_runtime(adev) && |
1071 | amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, hwip, write: true, rlcg_flag: &rlcg_flag)) { |
1072 | amdgpu_virt_rlcg_reg_rw(adev, offset, v: value, flag: rlcg_flag, xcc_id); |
1073 | return; |
1074 | } |
1075 | |
1076 | if (acc_flags & AMDGPU_REGS_NO_KIQ) |
1077 | WREG32_NO_KIQ(offset, value); |
1078 | else |
1079 | WREG32(offset, value); |
1080 | } |
1081 | |
1082 | u32 amdgpu_sriov_rreg(struct amdgpu_device *adev, |
1083 | u32 offset, u32 acc_flags, u32 hwip, u32 xcc_id) |
1084 | { |
1085 | u32 rlcg_flag; |
1086 | |
1087 | if (!amdgpu_sriov_runtime(adev) && |
1088 | amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, hwip, write: false, rlcg_flag: &rlcg_flag)) |
1089 | return amdgpu_virt_rlcg_reg_rw(adev, offset, v: 0, flag: rlcg_flag, xcc_id); |
1090 | |
1091 | if (acc_flags & AMDGPU_REGS_NO_KIQ) |
1092 | return RREG32_NO_KIQ(offset); |
1093 | else |
1094 | return RREG32(offset); |
1095 | } |
1096 | |