1 | /* |
2 | * Copyright 2021 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | */ |
23 | |
24 | #include <linux/printk.h> |
25 | #include <linux/slab.h> |
26 | #include <linux/uaccess.h> |
27 | #include "kfd_priv.h" |
28 | #include "kfd_mqd_manager.h" |
29 | #include "v11_structs.h" |
30 | #include "gc/gc_11_0_0_offset.h" |
31 | #include "gc/gc_11_0_0_sh_mask.h" |
32 | #include "amdgpu_amdkfd.h" |
33 | |
34 | static inline struct v11_compute_mqd *get_mqd(void *mqd) |
35 | { |
36 | return (struct v11_compute_mqd *)mqd; |
37 | } |
38 | |
39 | static inline struct v11_sdma_mqd *get_sdma_mqd(void *mqd) |
40 | { |
41 | return (struct v11_sdma_mqd *)mqd; |
42 | } |
43 | |
44 | static void update_cu_mask(struct mqd_manager *mm, void *mqd, |
45 | struct mqd_update_info *minfo) |
46 | { |
47 | struct v11_compute_mqd *m; |
48 | uint32_t se_mask[KFD_MAX_NUM_SE] = {0}; |
49 | bool has_wa_flag = minfo && (minfo->update_flag & (UPDATE_FLAG_DBG_WA_ENABLE | |
50 | UPDATE_FLAG_DBG_WA_DISABLE)); |
51 | |
52 | if (!minfo || !(has_wa_flag || minfo->cu_mask.ptr)) |
53 | return; |
54 | |
55 | m = get_mqd(mqd); |
56 | |
57 | if (has_wa_flag) { |
58 | uint32_t wa_mask = |
59 | (minfo->update_flag & UPDATE_FLAG_DBG_WA_ENABLE) ? 0xffff : 0xffffffff; |
60 | |
61 | m->compute_static_thread_mgmt_se0 = wa_mask; |
62 | m->compute_static_thread_mgmt_se1 = wa_mask; |
63 | m->compute_static_thread_mgmt_se2 = wa_mask; |
64 | m->compute_static_thread_mgmt_se3 = wa_mask; |
65 | m->compute_static_thread_mgmt_se4 = wa_mask; |
66 | m->compute_static_thread_mgmt_se5 = wa_mask; |
67 | m->compute_static_thread_mgmt_se6 = wa_mask; |
68 | m->compute_static_thread_mgmt_se7 = wa_mask; |
69 | |
70 | return; |
71 | } |
72 | |
73 | mqd_symmetrically_map_cu_mask(mm, |
74 | cu_mask: minfo->cu_mask.ptr, cu_mask_count: minfo->cu_mask.count, se_mask, inst: 0); |
75 | |
76 | m->compute_static_thread_mgmt_se0 = se_mask[0]; |
77 | m->compute_static_thread_mgmt_se1 = se_mask[1]; |
78 | m->compute_static_thread_mgmt_se2 = se_mask[2]; |
79 | m->compute_static_thread_mgmt_se3 = se_mask[3]; |
80 | m->compute_static_thread_mgmt_se4 = se_mask[4]; |
81 | m->compute_static_thread_mgmt_se5 = se_mask[5]; |
82 | m->compute_static_thread_mgmt_se6 = se_mask[6]; |
83 | m->compute_static_thread_mgmt_se7 = se_mask[7]; |
84 | |
85 | pr_debug("update cu mask to %#x %#x %#x %#x %#x %#x %#x %#x\n" , |
86 | m->compute_static_thread_mgmt_se0, |
87 | m->compute_static_thread_mgmt_se1, |
88 | m->compute_static_thread_mgmt_se2, |
89 | m->compute_static_thread_mgmt_se3, |
90 | m->compute_static_thread_mgmt_se4, |
91 | m->compute_static_thread_mgmt_se5, |
92 | m->compute_static_thread_mgmt_se6, |
93 | m->compute_static_thread_mgmt_se7); |
94 | } |
95 | |
96 | static void set_priority(struct v11_compute_mqd *m, struct queue_properties *q) |
97 | { |
98 | m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; |
99 | m->cp_hqd_queue_priority = q->priority; |
100 | } |
101 | |
102 | static struct kfd_mem_obj *allocate_mqd(struct kfd_node *node, |
103 | struct queue_properties *q) |
104 | { |
105 | struct kfd_mem_obj *mqd_mem_obj; |
106 | int size; |
107 | |
108 | /* |
109 | * MES write to areas beyond MQD size. So allocate |
110 | * 1 PAGE_SIZE memory for MQD is MES is enabled. |
111 | */ |
112 | if (node->kfd->shared_resources.enable_mes) |
113 | size = PAGE_SIZE; |
114 | else |
115 | size = sizeof(struct v11_compute_mqd); |
116 | |
117 | if (kfd_gtt_sa_allocate(node, size, mem_obj: &mqd_mem_obj)) |
118 | return NULL; |
119 | |
120 | return mqd_mem_obj; |
121 | } |
122 | |
123 | static void init_mqd(struct mqd_manager *mm, void **mqd, |
124 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, |
125 | struct queue_properties *q) |
126 | { |
127 | uint64_t addr; |
128 | struct v11_compute_mqd *m; |
129 | int size; |
130 | uint32_t wa_mask = q->is_dbg_wa ? 0xffff : 0xffffffff; |
131 | |
132 | m = (struct v11_compute_mqd *) mqd_mem_obj->cpu_ptr; |
133 | addr = mqd_mem_obj->gpu_addr; |
134 | |
135 | if (mm->dev->kfd->shared_resources.enable_mes) |
136 | size = PAGE_SIZE; |
137 | else |
138 | size = sizeof(struct v11_compute_mqd); |
139 | |
140 | memset(m, 0, size); |
141 | |
142 | m->header = 0xC0310800; |
143 | m->compute_pipelinestat_enable = 1; |
144 | |
145 | m->compute_static_thread_mgmt_se0 = wa_mask; |
146 | m->compute_static_thread_mgmt_se1 = wa_mask; |
147 | m->compute_static_thread_mgmt_se2 = wa_mask; |
148 | m->compute_static_thread_mgmt_se3 = wa_mask; |
149 | m->compute_static_thread_mgmt_se4 = wa_mask; |
150 | m->compute_static_thread_mgmt_se5 = wa_mask; |
151 | m->compute_static_thread_mgmt_se6 = wa_mask; |
152 | m->compute_static_thread_mgmt_se7 = wa_mask; |
153 | |
154 | m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK | |
155 | 0x55 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT; |
156 | |
157 | m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT; |
158 | |
159 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); |
160 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); |
161 | |
162 | m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT | |
163 | 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT | |
164 | 1 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT; |
165 | |
166 | /* Set cp_hqd_hq_scheduler0 bit 14 to 1 to have the CP set up the |
167 | * DISPATCH_PTR. This is required for the kfd debugger |
168 | */ |
169 | m->cp_hqd_hq_status0 = 1 << 14; |
170 | |
171 | /* |
172 | * GFX11 RS64 CPFW version >= 509 supports PCIe atomics support |
173 | * acknowledgment. |
174 | */ |
175 | if (amdgpu_amdkfd_have_atomics_support(adev: mm->dev->adev)) |
176 | m->cp_hqd_hq_status0 |= 1 << 29; |
177 | |
178 | if (q->format == KFD_QUEUE_FORMAT_AQL) { |
179 | m->cp_hqd_aql_control = |
180 | 1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT; |
181 | } |
182 | |
183 | if (mm->dev->kfd->cwsr_enabled) { |
184 | m->cp_hqd_persistent_state |= |
185 | (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT); |
186 | m->cp_hqd_ctx_save_base_addr_lo = |
187 | lower_32_bits(q->ctx_save_restore_area_address); |
188 | m->cp_hqd_ctx_save_base_addr_hi = |
189 | upper_32_bits(q->ctx_save_restore_area_address); |
190 | m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size; |
191 | m->cp_hqd_cntl_stack_size = q->ctl_stack_size; |
192 | m->cp_hqd_cntl_stack_offset = q->ctl_stack_size; |
193 | m->cp_hqd_wg_state_offset = q->ctl_stack_size; |
194 | } |
195 | |
196 | *mqd = m; |
197 | if (gart_addr) |
198 | *gart_addr = addr; |
199 | mm->update_mqd(mm, m, q, NULL); |
200 | } |
201 | |
202 | static int load_mqd(struct mqd_manager *mm, void *mqd, |
203 | uint32_t pipe_id, uint32_t queue_id, |
204 | struct queue_properties *p, struct mm_struct *mms) |
205 | { |
206 | int r = 0; |
207 | /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */ |
208 | uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0); |
209 | |
210 | r = mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id, |
211 | (uint32_t __user *)p->write_ptr, |
212 | wptr_shift, 0, mms, 0); |
213 | return r; |
214 | } |
215 | |
216 | static void update_mqd(struct mqd_manager *mm, void *mqd, |
217 | struct queue_properties *q, |
218 | struct mqd_update_info *minfo) |
219 | { |
220 | struct v11_compute_mqd *m; |
221 | |
222 | m = get_mqd(mqd); |
223 | |
224 | m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT; |
225 | m->cp_hqd_pq_control |= |
226 | ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1; |
227 | m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK; |
228 | pr_debug("cp_hqd_pq_control 0x%x\n" , m->cp_hqd_pq_control); |
229 | |
230 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); |
231 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); |
232 | |
233 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); |
234 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); |
235 | m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr); |
236 | m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr); |
237 | |
238 | m->cp_hqd_pq_doorbell_control = |
239 | q->doorbell_off << |
240 | CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; |
241 | pr_debug("cp_hqd_pq_doorbell_control 0x%x\n" , |
242 | m->cp_hqd_pq_doorbell_control); |
243 | |
244 | m->cp_hqd_ib_control = 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT; |
245 | |
246 | /* |
247 | * HW does not clamp this field correctly. Maximum EOP queue size |
248 | * is constrained by per-SE EOP done signal count, which is 8-bit. |
249 | * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit |
250 | * more than (EOP entry count - 1) so a queue size of 0x800 dwords |
251 | * is safe, giving a maximum field value of 0xA. |
252 | */ |
253 | m->cp_hqd_eop_control = min(0xA, |
254 | ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1); |
255 | m->cp_hqd_eop_base_addr_lo = |
256 | lower_32_bits(q->eop_ring_buffer_address >> 8); |
257 | m->cp_hqd_eop_base_addr_hi = |
258 | upper_32_bits(q->eop_ring_buffer_address >> 8); |
259 | |
260 | m->cp_hqd_iq_timer = 0; |
261 | |
262 | m->cp_hqd_vmid = q->vmid; |
263 | |
264 | if (q->format == KFD_QUEUE_FORMAT_AQL) { |
265 | /* GC 10 removed WPP_CLAMP from PQ Control */ |
266 | m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK | |
267 | 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT | |
268 | 1 << CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT ; |
269 | m->cp_hqd_pq_doorbell_control |= |
270 | 1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT; |
271 | } |
272 | if (mm->dev->kfd->cwsr_enabled) |
273 | m->cp_hqd_ctx_save_control = 0; |
274 | |
275 | update_cu_mask(mm, mqd, minfo); |
276 | set_priority(m, q); |
277 | |
278 | q->is_active = QUEUE_IS_ACTIVE(*q); |
279 | } |
280 | |
281 | static uint32_t read_doorbell_id(void *mqd) |
282 | { |
283 | struct v11_compute_mqd *m = (struct v11_compute_mqd *)mqd; |
284 | |
285 | return m->queue_doorbell_id0; |
286 | } |
287 | |
288 | static int get_wave_state(struct mqd_manager *mm, void *mqd, |
289 | struct queue_properties *q, |
290 | void __user *ctl_stack, |
291 | u32 *ctl_stack_used_size, |
292 | u32 *save_area_used_size) |
293 | { |
294 | struct v11_compute_mqd *m; |
295 | struct kfd_context_save_area_header ; |
296 | |
297 | m = get_mqd(mqd); |
298 | |
299 | /* Control stack is written backwards, while workgroup context data |
300 | * is written forwards. Both starts from m->cp_hqd_cntl_stack_size. |
301 | * Current position is at m->cp_hqd_cntl_stack_offset and |
302 | * m->cp_hqd_wg_state_offset, respectively. |
303 | */ |
304 | *ctl_stack_used_size = m->cp_hqd_cntl_stack_size - |
305 | m->cp_hqd_cntl_stack_offset; |
306 | *save_area_used_size = m->cp_hqd_wg_state_offset - |
307 | m->cp_hqd_cntl_stack_size; |
308 | |
309 | /* Control stack is not copied to user mode for GFXv11 because |
310 | * it's part of the context save area that is already |
311 | * accessible to user mode |
312 | */ |
313 | header.wave_state.control_stack_size = *ctl_stack_used_size; |
314 | header.wave_state.wave_state_size = *save_area_used_size; |
315 | |
316 | header.wave_state.wave_state_offset = m->cp_hqd_wg_state_offset; |
317 | header.wave_state.control_stack_offset = m->cp_hqd_cntl_stack_offset; |
318 | |
319 | if (copy_to_user(to: ctl_stack, from: &header, n: sizeof(header.wave_state))) |
320 | return -EFAULT; |
321 | |
322 | return 0; |
323 | } |
324 | |
325 | static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst) |
326 | { |
327 | struct v11_compute_mqd *m; |
328 | |
329 | m = get_mqd(mqd); |
330 | |
331 | memcpy(mqd_dst, m, sizeof(struct v11_compute_mqd)); |
332 | } |
333 | |
334 | static void restore_mqd(struct mqd_manager *mm, void **mqd, |
335 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, |
336 | struct queue_properties *qp, |
337 | const void *mqd_src, |
338 | const void *ctl_stack_src, const u32 ctl_stack_size) |
339 | { |
340 | uint64_t addr; |
341 | struct v11_compute_mqd *m; |
342 | |
343 | m = (struct v11_compute_mqd *) mqd_mem_obj->cpu_ptr; |
344 | addr = mqd_mem_obj->gpu_addr; |
345 | |
346 | memcpy(m, mqd_src, sizeof(*m)); |
347 | |
348 | *mqd = m; |
349 | if (gart_addr) |
350 | *gart_addr = addr; |
351 | |
352 | m->cp_hqd_pq_doorbell_control = |
353 | qp->doorbell_off << |
354 | CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; |
355 | pr_debug("cp_hqd_pq_doorbell_control 0x%x\n" , |
356 | m->cp_hqd_pq_doorbell_control); |
357 | |
358 | qp->is_active = 0; |
359 | } |
360 | |
361 | |
362 | static void init_mqd_hiq(struct mqd_manager *mm, void **mqd, |
363 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, |
364 | struct queue_properties *q) |
365 | { |
366 | struct v11_compute_mqd *m; |
367 | |
368 | init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q); |
369 | |
370 | m = get_mqd(mqd: *mqd); |
371 | |
372 | m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT | |
373 | 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT; |
374 | } |
375 | |
376 | static int destroy_hiq_mqd(struct mqd_manager *mm, void *mqd, |
377 | enum kfd_preempt_type type, unsigned int timeout, |
378 | uint32_t pipe_id, uint32_t queue_id) |
379 | { |
380 | int err; |
381 | struct v11_compute_mqd *m; |
382 | u32 doorbell_off; |
383 | |
384 | m = get_mqd(mqd); |
385 | |
386 | doorbell_off = m->cp_hqd_pq_doorbell_control >> |
387 | CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; |
388 | |
389 | err = amdgpu_amdkfd_unmap_hiq(adev: mm->dev->adev, doorbell_off, inst: 0); |
390 | if (err) |
391 | pr_debug("Destroy HIQ MQD failed: %d\n" , err); |
392 | |
393 | return err; |
394 | } |
395 | |
396 | static void init_mqd_sdma(struct mqd_manager *mm, void **mqd, |
397 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, |
398 | struct queue_properties *q) |
399 | { |
400 | struct v11_sdma_mqd *m; |
401 | int size; |
402 | |
403 | m = (struct v11_sdma_mqd *) mqd_mem_obj->cpu_ptr; |
404 | |
405 | if (mm->dev->kfd->shared_resources.enable_mes) |
406 | size = PAGE_SIZE; |
407 | else |
408 | size = sizeof(struct v11_sdma_mqd); |
409 | |
410 | memset(m, 0, size); |
411 | *mqd = m; |
412 | if (gart_addr) |
413 | *gart_addr = mqd_mem_obj->gpu_addr; |
414 | |
415 | mm->update_mqd(mm, m, q, NULL); |
416 | } |
417 | |
418 | #define SDMA_RLC_DUMMY_DEFAULT 0xf |
419 | |
420 | static void update_mqd_sdma(struct mqd_manager *mm, void *mqd, |
421 | struct queue_properties *q, |
422 | struct mqd_update_info *minfo) |
423 | { |
424 | struct v11_sdma_mqd *m; |
425 | |
426 | m = get_sdma_mqd(mqd); |
427 | m->sdmax_rlcx_rb_cntl = (ffs(q->queue_size / sizeof(unsigned int)) - 1) |
428 | << SDMA0_QUEUE0_RB_CNTL__RB_SIZE__SHIFT | |
429 | q->vmid << SDMA0_QUEUE0_RB_CNTL__RB_VMID__SHIFT | |
430 | 1 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT | |
431 | 6 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT | |
432 | 1 << SDMA0_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT; |
433 | |
434 | m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8); |
435 | m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8); |
436 | m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); |
437 | m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); |
438 | m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr); |
439 | m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr); |
440 | m->sdmax_rlcx_doorbell_offset = |
441 | q->doorbell_off << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT; |
442 | |
443 | m->sdmax_rlcx_sched_cntl = (amdgpu_sdma_phase_quantum |
444 | << SDMA0_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT) |
445 | & SDMA0_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK; |
446 | |
447 | m->sdma_engine_id = q->sdma_engine_id; |
448 | m->sdma_queue_id = q->sdma_queue_id; |
449 | m->sdmax_rlcx_dummy_reg = SDMA_RLC_DUMMY_DEFAULT; |
450 | |
451 | q->is_active = QUEUE_IS_ACTIVE(*q); |
452 | } |
453 | |
454 | #if defined(CONFIG_DEBUG_FS) |
455 | |
456 | static int debugfs_show_mqd(struct seq_file *m, void *data) |
457 | { |
458 | seq_hex_dump(m, prefix_str: " " , prefix_type: DUMP_PREFIX_OFFSET, rowsize: 32, groupsize: 4, |
459 | buf: data, len: sizeof(struct v11_compute_mqd), ascii: false); |
460 | return 0; |
461 | } |
462 | |
463 | static int debugfs_show_mqd_sdma(struct seq_file *m, void *data) |
464 | { |
465 | seq_hex_dump(m, prefix_str: " " , prefix_type: DUMP_PREFIX_OFFSET, rowsize: 32, groupsize: 4, |
466 | buf: data, len: sizeof(struct v11_sdma_mqd), ascii: false); |
467 | return 0; |
468 | } |
469 | |
470 | #endif |
471 | |
472 | struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type, |
473 | struct kfd_node *dev) |
474 | { |
475 | struct mqd_manager *mqd; |
476 | |
477 | if (WARN_ON(type >= KFD_MQD_TYPE_MAX)) |
478 | return NULL; |
479 | |
480 | mqd = kzalloc(size: sizeof(*mqd), GFP_KERNEL); |
481 | if (!mqd) |
482 | return NULL; |
483 | |
484 | mqd->dev = dev; |
485 | |
486 | switch (type) { |
487 | case KFD_MQD_TYPE_CP: |
488 | pr_debug("%s@%i\n" , __func__, __LINE__); |
489 | mqd->allocate_mqd = allocate_mqd; |
490 | mqd->init_mqd = init_mqd; |
491 | mqd->free_mqd = kfd_free_mqd_cp; |
492 | mqd->load_mqd = load_mqd; |
493 | mqd->update_mqd = update_mqd; |
494 | mqd->destroy_mqd = kfd_destroy_mqd_cp; |
495 | mqd->is_occupied = kfd_is_occupied_cp; |
496 | mqd->mqd_size = sizeof(struct v11_compute_mqd); |
497 | mqd->get_wave_state = get_wave_state; |
498 | mqd->mqd_stride = kfd_mqd_stride; |
499 | mqd->checkpoint_mqd = checkpoint_mqd; |
500 | mqd->restore_mqd = restore_mqd; |
501 | #if defined(CONFIG_DEBUG_FS) |
502 | mqd->debugfs_show_mqd = debugfs_show_mqd; |
503 | #endif |
504 | pr_debug("%s@%i\n" , __func__, __LINE__); |
505 | break; |
506 | case KFD_MQD_TYPE_HIQ: |
507 | pr_debug("%s@%i\n" , __func__, __LINE__); |
508 | mqd->allocate_mqd = allocate_hiq_mqd; |
509 | mqd->init_mqd = init_mqd_hiq; |
510 | mqd->free_mqd = free_mqd_hiq_sdma; |
511 | mqd->load_mqd = kfd_hiq_load_mqd_kiq; |
512 | mqd->update_mqd = update_mqd; |
513 | mqd->destroy_mqd = destroy_hiq_mqd; |
514 | mqd->is_occupied = kfd_is_occupied_cp; |
515 | mqd->mqd_size = sizeof(struct v11_compute_mqd); |
516 | mqd->mqd_stride = kfd_mqd_stride; |
517 | #if defined(CONFIG_DEBUG_FS) |
518 | mqd->debugfs_show_mqd = debugfs_show_mqd; |
519 | #endif |
520 | mqd->read_doorbell_id = read_doorbell_id; |
521 | pr_debug("%s@%i\n" , __func__, __LINE__); |
522 | break; |
523 | case KFD_MQD_TYPE_DIQ: |
524 | mqd->allocate_mqd = allocate_mqd; |
525 | mqd->init_mqd = init_mqd_hiq; |
526 | mqd->free_mqd = kfd_free_mqd_cp; |
527 | mqd->load_mqd = load_mqd; |
528 | mqd->update_mqd = update_mqd; |
529 | mqd->destroy_mqd = kfd_destroy_mqd_cp; |
530 | mqd->is_occupied = kfd_is_occupied_cp; |
531 | mqd->mqd_size = sizeof(struct v11_compute_mqd); |
532 | #if defined(CONFIG_DEBUG_FS) |
533 | mqd->debugfs_show_mqd = debugfs_show_mqd; |
534 | #endif |
535 | break; |
536 | case KFD_MQD_TYPE_SDMA: |
537 | pr_debug("%s@%i\n" , __func__, __LINE__); |
538 | mqd->allocate_mqd = allocate_sdma_mqd; |
539 | mqd->init_mqd = init_mqd_sdma; |
540 | mqd->free_mqd = free_mqd_hiq_sdma; |
541 | mqd->load_mqd = kfd_load_mqd_sdma; |
542 | mqd->update_mqd = update_mqd_sdma; |
543 | mqd->destroy_mqd = kfd_destroy_mqd_sdma; |
544 | mqd->is_occupied = kfd_is_occupied_sdma; |
545 | mqd->checkpoint_mqd = checkpoint_mqd; |
546 | mqd->restore_mqd = restore_mqd; |
547 | mqd->mqd_size = sizeof(struct v11_sdma_mqd); |
548 | mqd->mqd_stride = kfd_mqd_stride; |
549 | #if defined(CONFIG_DEBUG_FS) |
550 | mqd->debugfs_show_mqd = debugfs_show_mqd_sdma; |
551 | #endif |
552 | /* |
553 | * To allocate SDMA MQDs by generic functions |
554 | * when MES is enabled. |
555 | */ |
556 | if (dev->kfd->shared_resources.enable_mes) { |
557 | mqd->allocate_mqd = allocate_mqd; |
558 | mqd->free_mqd = kfd_free_mqd_cp; |
559 | } |
560 | pr_debug("%s@%i\n" , __func__, __LINE__); |
561 | break; |
562 | default: |
563 | kfree(objp: mqd); |
564 | return NULL; |
565 | } |
566 | |
567 | return mqd; |
568 | } |
569 | |