1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* |
3 | * Copyright 2014-2022 Advanced Micro Devices, Inc. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * copy of this software and associated documentation files (the "Software"), |
7 | * to deal in the Software without restriction, including without limitation |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
9 | * and/or sell copies of the Software, and to permit persons to whom the |
10 | * Software is furnished to do so, subject to the following conditions: |
11 | * |
12 | * The above copyright notice and this permission notice shall be included in |
13 | * all copies or substantial portions of the Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
21 | * OTHER DEALINGS IN THE SOFTWARE. |
22 | * |
23 | */ |
24 | |
25 | #include "kfd_kernel_queue.h" |
26 | #include "kfd_device_queue_manager.h" |
27 | #include "kfd_pm4_headers_vi.h" |
28 | #include "kfd_pm4_opcodes.h" |
29 | |
30 | unsigned int (unsigned int opcode, size_t packet_size) |
31 | { |
32 | union PM4_MES_TYPE_3_HEADER ; |
33 | |
34 | header.u32All = 0; |
35 | header.opcode = opcode; |
36 | header.count = packet_size / 4 - 2; |
37 | header.type = PM4_TYPE_3; |
38 | |
39 | return header.u32All; |
40 | } |
41 | |
42 | static int pm_map_process_vi(struct packet_manager *pm, uint32_t *buffer, |
43 | struct qcm_process_device *qpd) |
44 | { |
45 | struct pm4_mes_map_process *packet; |
46 | |
47 | packet = (struct pm4_mes_map_process *)buffer; |
48 | |
49 | memset(buffer, 0, sizeof(struct pm4_mes_map_process)); |
50 | |
51 | packet->header.u32All = pm_build_pm4_header(opcode: IT_MAP_PROCESS, |
52 | packet_size: sizeof(struct pm4_mes_map_process)); |
53 | packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0; |
54 | packet->bitfields2.process_quantum = 10; |
55 | packet->bitfields2.pasid = qpd->pqm->process->pasid; |
56 | packet->bitfields3.page_table_base = qpd->page_table_base; |
57 | packet->bitfields10.gds_size = qpd->gds_size; |
58 | packet->bitfields10.num_gws = qpd->num_gws; |
59 | packet->bitfields10.num_oac = qpd->num_oac; |
60 | packet->bitfields10.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count; |
61 | |
62 | packet->sh_mem_config = qpd->sh_mem_config; |
63 | packet->sh_mem_bases = qpd->sh_mem_bases; |
64 | packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base; |
65 | packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit; |
66 | |
67 | packet->sh_hidden_private_base_vmid = qpd->sh_hidden_private_base; |
68 | |
69 | packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area); |
70 | packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area); |
71 | |
72 | return 0; |
73 | } |
74 | |
75 | static int pm_runlist_vi(struct packet_manager *pm, uint32_t *buffer, |
76 | uint64_t ib, size_t ib_size_in_dwords, bool chain) |
77 | { |
78 | struct pm4_mes_runlist *packet; |
79 | int concurrent_proc_cnt = 0; |
80 | struct kfd_node *kfd = pm->dqm->dev; |
81 | |
82 | if (WARN_ON(!ib)) |
83 | return -EFAULT; |
84 | |
85 | /* Determine the number of processes to map together to HW: |
86 | * it can not exceed the number of VMIDs available to the |
87 | * scheduler, and it is determined by the smaller of the number |
88 | * of processes in the runlist and kfd module parameter |
89 | * hws_max_conc_proc. |
90 | * Note: the arbitration between the number of VMIDs and |
91 | * hws_max_conc_proc has been done in |
92 | * kgd2kfd_device_init(). |
93 | */ |
94 | concurrent_proc_cnt = min(pm->dqm->processes_count, |
95 | kfd->max_proc_per_quantum); |
96 | |
97 | packet = (struct pm4_mes_runlist *)buffer; |
98 | |
99 | memset(buffer, 0, sizeof(struct pm4_mes_runlist)); |
100 | packet->header.u32All = pm_build_pm4_header(opcode: IT_RUN_LIST, |
101 | packet_size: sizeof(struct pm4_mes_runlist)); |
102 | |
103 | packet->bitfields4.ib_size = ib_size_in_dwords; |
104 | packet->bitfields4.chain = chain ? 1 : 0; |
105 | packet->bitfields4.offload_polling = 0; |
106 | packet->bitfields4.valid = 1; |
107 | packet->bitfields4.process_cnt = concurrent_proc_cnt; |
108 | packet->ordinal2 = lower_32_bits(ib); |
109 | packet->bitfields3.ib_base_hi = upper_32_bits(ib); |
110 | |
111 | return 0; |
112 | } |
113 | |
114 | static int pm_set_resources_vi(struct packet_manager *pm, uint32_t *buffer, |
115 | struct scheduling_resources *res) |
116 | { |
117 | struct pm4_mes_set_resources *packet; |
118 | |
119 | packet = (struct pm4_mes_set_resources *)buffer; |
120 | memset(buffer, 0, sizeof(struct pm4_mes_set_resources)); |
121 | |
122 | packet->header.u32All = pm_build_pm4_header(opcode: IT_SET_RESOURCES, |
123 | packet_size: sizeof(struct pm4_mes_set_resources)); |
124 | |
125 | packet->bitfields2.queue_type = |
126 | queue_type__mes_set_resources__hsa_interface_queue_hiq; |
127 | packet->bitfields2.vmid_mask = res->vmid_mask; |
128 | packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY_MS / 100; |
129 | packet->bitfields7.oac_mask = res->oac_mask; |
130 | packet->bitfields8.gds_heap_base = res->gds_heap_base; |
131 | packet->bitfields8.gds_heap_size = res->gds_heap_size; |
132 | |
133 | packet->gws_mask_lo = lower_32_bits(res->gws_mask); |
134 | packet->gws_mask_hi = upper_32_bits(res->gws_mask); |
135 | |
136 | packet->queue_mask_lo = lower_32_bits(res->queue_mask); |
137 | packet->queue_mask_hi = upper_32_bits(res->queue_mask); |
138 | |
139 | return 0; |
140 | } |
141 | |
142 | static int pm_map_queues_vi(struct packet_manager *pm, uint32_t *buffer, |
143 | struct queue *q, bool is_static) |
144 | { |
145 | struct pm4_mes_map_queues *packet; |
146 | bool use_static = is_static; |
147 | |
148 | packet = (struct pm4_mes_map_queues *)buffer; |
149 | memset(buffer, 0, sizeof(struct pm4_mes_map_queues)); |
150 | |
151 | packet->header.u32All = pm_build_pm4_header(opcode: IT_MAP_QUEUES, |
152 | packet_size: sizeof(struct pm4_mes_map_queues)); |
153 | packet->bitfields2.num_queues = 1; |
154 | packet->bitfields2.queue_sel = |
155 | queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi; |
156 | |
157 | packet->bitfields2.engine_sel = |
158 | engine_sel__mes_map_queues__compute_vi; |
159 | packet->bitfields2.queue_type = |
160 | queue_type__mes_map_queues__normal_compute_vi; |
161 | |
162 | switch (q->properties.type) { |
163 | case KFD_QUEUE_TYPE_COMPUTE: |
164 | if (use_static) |
165 | packet->bitfields2.queue_type = |
166 | queue_type__mes_map_queues__normal_latency_static_queue_vi; |
167 | break; |
168 | case KFD_QUEUE_TYPE_DIQ: |
169 | packet->bitfields2.queue_type = |
170 | queue_type__mes_map_queues__debug_interface_queue_vi; |
171 | break; |
172 | case KFD_QUEUE_TYPE_SDMA: |
173 | case KFD_QUEUE_TYPE_SDMA_XGMI: |
174 | packet->bitfields2.engine_sel = q->properties.sdma_engine_id + |
175 | engine_sel__mes_map_queues__sdma0_vi; |
176 | use_static = false; /* no static queues under SDMA */ |
177 | break; |
178 | default: |
179 | WARN(1, "queue type %d" , q->properties.type); |
180 | return -EINVAL; |
181 | } |
182 | packet->bitfields3.doorbell_offset = |
183 | q->properties.doorbell_off; |
184 | |
185 | packet->mqd_addr_lo = |
186 | lower_32_bits(q->gart_mqd_addr); |
187 | |
188 | packet->mqd_addr_hi = |
189 | upper_32_bits(q->gart_mqd_addr); |
190 | |
191 | packet->wptr_addr_lo = |
192 | lower_32_bits((uint64_t)q->properties.write_ptr); |
193 | |
194 | packet->wptr_addr_hi = |
195 | upper_32_bits((uint64_t)q->properties.write_ptr); |
196 | |
197 | return 0; |
198 | } |
199 | |
200 | static int pm_unmap_queues_vi(struct packet_manager *pm, uint32_t *buffer, |
201 | enum kfd_unmap_queues_filter filter, |
202 | uint32_t filter_param, bool reset) |
203 | { |
204 | struct pm4_mes_unmap_queues *packet; |
205 | |
206 | packet = (struct pm4_mes_unmap_queues *)buffer; |
207 | memset(buffer, 0, sizeof(struct pm4_mes_unmap_queues)); |
208 | |
209 | packet->header.u32All = pm_build_pm4_header(opcode: IT_UNMAP_QUEUES, |
210 | packet_size: sizeof(struct pm4_mes_unmap_queues)); |
211 | |
212 | packet->bitfields2.engine_sel = |
213 | engine_sel__mes_unmap_queues__compute; |
214 | |
215 | if (reset) |
216 | packet->bitfields2.action = |
217 | action__mes_unmap_queues__reset_queues; |
218 | else |
219 | packet->bitfields2.action = |
220 | action__mes_unmap_queues__preempt_queues; |
221 | |
222 | switch (filter) { |
223 | case KFD_UNMAP_QUEUES_FILTER_BY_PASID: |
224 | packet->bitfields2.queue_sel = |
225 | queue_sel__mes_unmap_queues__perform_request_on_pasid_queues; |
226 | packet->bitfields3a.pasid = filter_param; |
227 | break; |
228 | case KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: |
229 | packet->bitfields2.queue_sel = |
230 | queue_sel__mes_unmap_queues__unmap_all_queues; |
231 | break; |
232 | case KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES: |
233 | /* in this case, we do not preempt static queues */ |
234 | packet->bitfields2.queue_sel = |
235 | queue_sel__mes_unmap_queues__unmap_all_non_static_queues; |
236 | break; |
237 | default: |
238 | WARN(1, "filter %d" , filter); |
239 | return -EINVAL; |
240 | } |
241 | |
242 | return 0; |
243 | |
244 | } |
245 | |
246 | static int pm_query_status_vi(struct packet_manager *pm, uint32_t *buffer, |
247 | uint64_t fence_address, uint64_t fence_value) |
248 | { |
249 | struct pm4_mes_query_status *packet; |
250 | |
251 | packet = (struct pm4_mes_query_status *)buffer; |
252 | memset(buffer, 0, sizeof(struct pm4_mes_query_status)); |
253 | |
254 | packet->header.u32All = pm_build_pm4_header(opcode: IT_QUERY_STATUS, |
255 | packet_size: sizeof(struct pm4_mes_query_status)); |
256 | |
257 | packet->bitfields2.context_id = 0; |
258 | packet->bitfields2.interrupt_sel = |
259 | interrupt_sel__mes_query_status__completion_status; |
260 | packet->bitfields2.command = |
261 | command__mes_query_status__fence_only_after_write_ack; |
262 | |
263 | packet->addr_hi = upper_32_bits((uint64_t)fence_address); |
264 | packet->addr_lo = lower_32_bits((uint64_t)fence_address); |
265 | packet->data_hi = upper_32_bits((uint64_t)fence_value); |
266 | packet->data_lo = lower_32_bits((uint64_t)fence_value); |
267 | |
268 | return 0; |
269 | } |
270 | |
271 | static int pm_release_mem_vi(uint64_t gpu_addr, uint32_t *buffer) |
272 | { |
273 | struct pm4_mec_release_mem *packet; |
274 | |
275 | packet = (struct pm4_mec_release_mem *)buffer; |
276 | memset(buffer, 0, sizeof(*packet)); |
277 | |
278 | packet->header.u32All = pm_build_pm4_header(opcode: IT_RELEASE_MEM, |
279 | packet_size: sizeof(*packet)); |
280 | |
281 | packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT; |
282 | packet->bitfields2.event_index = event_index___release_mem__end_of_pipe; |
283 | packet->bitfields2.tcl1_action_ena = 1; |
284 | packet->bitfields2.tc_action_ena = 1; |
285 | packet->bitfields2.cache_policy = cache_policy___release_mem__lru; |
286 | packet->bitfields2.atc = 0; |
287 | |
288 | packet->bitfields3.data_sel = data_sel___release_mem__send_32_bit_low; |
289 | packet->bitfields3.int_sel = |
290 | int_sel___release_mem__send_interrupt_after_write_confirm; |
291 | |
292 | packet->bitfields4.address_lo_32b = (gpu_addr & 0xffffffff) >> 2; |
293 | packet->address_hi = upper_32_bits(gpu_addr); |
294 | |
295 | packet->data_lo = 0; |
296 | |
297 | return 0; |
298 | } |
299 | |
300 | const struct packet_manager_funcs kfd_vi_pm_funcs = { |
301 | .map_process = pm_map_process_vi, |
302 | .runlist = pm_runlist_vi, |
303 | .set_resources = pm_set_resources_vi, |
304 | .map_queues = pm_map_queues_vi, |
305 | .unmap_queues = pm_unmap_queues_vi, |
306 | .set_grace_period = NULL, |
307 | .query_status = pm_query_status_vi, |
308 | .release_mem = pm_release_mem_vi, |
309 | .map_process_size = sizeof(struct pm4_mes_map_process), |
310 | .runlist_size = sizeof(struct pm4_mes_runlist), |
311 | .set_resources_size = sizeof(struct pm4_mes_set_resources), |
312 | .map_queues_size = sizeof(struct pm4_mes_map_queues), |
313 | .unmap_queues_size = sizeof(struct pm4_mes_unmap_queues), |
314 | .set_grace_period_size = 0, |
315 | .query_status_size = sizeof(struct pm4_mes_query_status), |
316 | .release_mem_size = sizeof(struct pm4_mec_release_mem) |
317 | }; |
318 | |