1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* |
3 | * Copyright 2020-2022 Advanced Micro Devices, Inc. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * copy of this software and associated documentation files (the "Software"), |
7 | * to deal in the Software without restriction, including without limitation |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
9 | * and/or sell copies of the Software, and to permit persons to whom the |
10 | * Software is furnished to do so, subject to the following conditions: |
11 | * |
12 | * The above copyright notice and this permission notice shall be included in |
13 | * all copies or substantial portions of the Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
21 | * OTHER DEALINGS IN THE SOFTWARE. |
22 | */ |
23 | |
24 | #include <linux/poll.h> |
25 | #include <linux/wait.h> |
26 | #include <linux/anon_inodes.h> |
27 | #include <uapi/linux/kfd_ioctl.h> |
28 | #include "amdgpu.h" |
29 | #include "amdgpu_vm.h" |
30 | #include "kfd_priv.h" |
31 | #include "kfd_smi_events.h" |
32 | |
33 | struct kfd_smi_client { |
34 | struct list_head list; |
35 | struct kfifo fifo; |
36 | wait_queue_head_t wait_queue; |
37 | /* events enabled */ |
38 | uint64_t events; |
39 | struct kfd_node *dev; |
40 | spinlock_t lock; |
41 | struct rcu_head rcu; |
42 | pid_t pid; |
43 | bool suser; |
44 | }; |
45 | |
46 | #define MAX_KFIFO_SIZE 1024 |
47 | |
48 | static __poll_t kfd_smi_ev_poll(struct file *, struct poll_table_struct *); |
49 | static ssize_t kfd_smi_ev_read(struct file *, char __user *, size_t, loff_t *); |
50 | static ssize_t kfd_smi_ev_write(struct file *, const char __user *, size_t, |
51 | loff_t *); |
52 | static int kfd_smi_ev_release(struct inode *, struct file *); |
53 | |
54 | static const char kfd_smi_name[] = "kfd_smi_ev" ; |
55 | |
56 | static const struct file_operations kfd_smi_ev_fops = { |
57 | .owner = THIS_MODULE, |
58 | .poll = kfd_smi_ev_poll, |
59 | .read = kfd_smi_ev_read, |
60 | .write = kfd_smi_ev_write, |
61 | .release = kfd_smi_ev_release |
62 | }; |
63 | |
64 | static __poll_t kfd_smi_ev_poll(struct file *filep, |
65 | struct poll_table_struct *wait) |
66 | { |
67 | struct kfd_smi_client *client = filep->private_data; |
68 | __poll_t mask = 0; |
69 | |
70 | poll_wait(filp: filep, wait_address: &client->wait_queue, p: wait); |
71 | |
72 | spin_lock(lock: &client->lock); |
73 | if (!kfifo_is_empty(&client->fifo)) |
74 | mask = EPOLLIN | EPOLLRDNORM; |
75 | spin_unlock(lock: &client->lock); |
76 | |
77 | return mask; |
78 | } |
79 | |
80 | static ssize_t kfd_smi_ev_read(struct file *filep, char __user *user, |
81 | size_t size, loff_t *offset) |
82 | { |
83 | int ret; |
84 | size_t to_copy; |
85 | struct kfd_smi_client *client = filep->private_data; |
86 | unsigned char *buf; |
87 | |
88 | size = min_t(size_t, size, MAX_KFIFO_SIZE); |
89 | buf = kmalloc(size, GFP_KERNEL); |
90 | if (!buf) |
91 | return -ENOMEM; |
92 | |
93 | /* kfifo_to_user can sleep so we can't use spinlock protection around |
94 | * it. Instead, we kfifo out as spinlocked then copy them to the user. |
95 | */ |
96 | spin_lock(lock: &client->lock); |
97 | to_copy = kfifo_len(&client->fifo); |
98 | if (!to_copy) { |
99 | spin_unlock(lock: &client->lock); |
100 | ret = -EAGAIN; |
101 | goto ret_err; |
102 | } |
103 | to_copy = min(size, to_copy); |
104 | ret = kfifo_out(&client->fifo, buf, to_copy); |
105 | spin_unlock(lock: &client->lock); |
106 | if (ret <= 0) { |
107 | ret = -EAGAIN; |
108 | goto ret_err; |
109 | } |
110 | |
111 | ret = copy_to_user(to: user, from: buf, n: to_copy); |
112 | if (ret) { |
113 | ret = -EFAULT; |
114 | goto ret_err; |
115 | } |
116 | |
117 | kfree(objp: buf); |
118 | return to_copy; |
119 | |
120 | ret_err: |
121 | kfree(objp: buf); |
122 | return ret; |
123 | } |
124 | |
125 | static ssize_t kfd_smi_ev_write(struct file *filep, const char __user *user, |
126 | size_t size, loff_t *offset) |
127 | { |
128 | struct kfd_smi_client *client = filep->private_data; |
129 | uint64_t events; |
130 | |
131 | if (!access_ok(user, size) || size < sizeof(events)) |
132 | return -EFAULT; |
133 | if (copy_from_user(to: &events, from: user, n: sizeof(events))) |
134 | return -EFAULT; |
135 | |
136 | WRITE_ONCE(client->events, events); |
137 | |
138 | return sizeof(events); |
139 | } |
140 | |
141 | static void kfd_smi_ev_client_free(struct rcu_head *p) |
142 | { |
143 | struct kfd_smi_client *ev = container_of(p, struct kfd_smi_client, rcu); |
144 | |
145 | kfifo_free(&ev->fifo); |
146 | kfree(objp: ev); |
147 | } |
148 | |
149 | static int kfd_smi_ev_release(struct inode *inode, struct file *filep) |
150 | { |
151 | struct kfd_smi_client *client = filep->private_data; |
152 | struct kfd_node *dev = client->dev; |
153 | |
154 | spin_lock(lock: &dev->smi_lock); |
155 | list_del_rcu(entry: &client->list); |
156 | spin_unlock(lock: &dev->smi_lock); |
157 | |
158 | call_rcu(head: &client->rcu, func: kfd_smi_ev_client_free); |
159 | return 0; |
160 | } |
161 | |
162 | static bool kfd_smi_ev_enabled(pid_t pid, struct kfd_smi_client *client, |
163 | unsigned int event) |
164 | { |
165 | uint64_t all = KFD_SMI_EVENT_MASK_FROM_INDEX(KFD_SMI_EVENT_ALL_PROCESS); |
166 | uint64_t events = READ_ONCE(client->events); |
167 | |
168 | if (pid && client->pid != pid && !(client->suser && (events & all))) |
169 | return false; |
170 | |
171 | return events & KFD_SMI_EVENT_MASK_FROM_INDEX(event); |
172 | } |
173 | |
174 | static void add_event_to_kfifo(pid_t pid, struct kfd_node *dev, |
175 | unsigned int smi_event, char *event_msg, int len) |
176 | { |
177 | struct kfd_smi_client *client; |
178 | |
179 | rcu_read_lock(); |
180 | |
181 | list_for_each_entry_rcu(client, &dev->smi_clients, list) { |
182 | if (!kfd_smi_ev_enabled(pid, client, event: smi_event)) |
183 | continue; |
184 | spin_lock(lock: &client->lock); |
185 | if (kfifo_avail(&client->fifo) >= len) { |
186 | kfifo_in(&client->fifo, event_msg, len); |
187 | wake_up_all(&client->wait_queue); |
188 | } else { |
189 | pr_debug("smi_event(EventID: %u): no space left\n" , |
190 | smi_event); |
191 | } |
192 | spin_unlock(lock: &client->lock); |
193 | } |
194 | |
195 | rcu_read_unlock(); |
196 | } |
197 | |
198 | __printf(4, 5) |
199 | static void kfd_smi_event_add(pid_t pid, struct kfd_node *dev, |
200 | unsigned int event, char *fmt, ...) |
201 | { |
202 | char fifo_in[KFD_SMI_EVENT_MSG_SIZE]; |
203 | int len; |
204 | va_list args; |
205 | |
206 | if (list_empty(head: &dev->smi_clients)) |
207 | return; |
208 | |
209 | len = snprintf(buf: fifo_in, size: sizeof(fifo_in), fmt: "%x " , event); |
210 | |
211 | va_start(args, fmt); |
212 | len += vsnprintf(buf: fifo_in + len, size: sizeof(fifo_in) - len, fmt, args); |
213 | va_end(args); |
214 | |
215 | add_event_to_kfifo(pid, dev, smi_event: event, event_msg: fifo_in, len); |
216 | } |
217 | |
218 | void kfd_smi_event_update_gpu_reset(struct kfd_node *dev, bool post_reset) |
219 | { |
220 | unsigned int event; |
221 | |
222 | if (post_reset) { |
223 | event = KFD_SMI_EVENT_GPU_POST_RESET; |
224 | } else { |
225 | event = KFD_SMI_EVENT_GPU_PRE_RESET; |
226 | ++(dev->reset_seq_num); |
227 | } |
228 | kfd_smi_event_add(pid: 0, dev, event, fmt: "%x\n" , dev->reset_seq_num); |
229 | } |
230 | |
231 | void kfd_smi_event_update_thermal_throttling(struct kfd_node *dev, |
232 | uint64_t throttle_bitmask) |
233 | { |
234 | kfd_smi_event_add(pid: 0, dev, event: KFD_SMI_EVENT_THERMAL_THROTTLE, fmt: "%llx:%llx\n" , |
235 | throttle_bitmask, |
236 | amdgpu_dpm_get_thermal_throttling_counter(adev: dev->adev)); |
237 | } |
238 | |
239 | void kfd_smi_event_update_vmfault(struct kfd_node *dev, uint16_t pasid) |
240 | { |
241 | struct amdgpu_task_info *task_info; |
242 | |
243 | task_info = amdgpu_vm_get_task_info_pasid(adev: dev->adev, pasid); |
244 | if (task_info) { |
245 | /* Report VM faults from user applications, not retry from kernel */ |
246 | if (task_info->pid) |
247 | kfd_smi_event_add(pid: 0, dev, event: KFD_SMI_EVENT_VMFAULT, fmt: "%x:%s\n" , |
248 | task_info->pid, task_info->task_name); |
249 | amdgpu_vm_put_task_info(task_info); |
250 | } |
251 | } |
252 | |
253 | void kfd_smi_event_page_fault_start(struct kfd_node *node, pid_t pid, |
254 | unsigned long address, bool write_fault, |
255 | ktime_t ts) |
256 | { |
257 | kfd_smi_event_add(pid, dev: node, event: KFD_SMI_EVENT_PAGE_FAULT_START, |
258 | fmt: "%lld -%d @%lx(%x) %c\n" , ktime_to_ns(kt: ts), pid, |
259 | address, node->id, write_fault ? 'W' : 'R'); |
260 | } |
261 | |
262 | void kfd_smi_event_page_fault_end(struct kfd_node *node, pid_t pid, |
263 | unsigned long address, bool migration) |
264 | { |
265 | kfd_smi_event_add(pid, dev: node, event: KFD_SMI_EVENT_PAGE_FAULT_END, |
266 | fmt: "%lld -%d @%lx(%x) %c\n" , ktime_get_boottime_ns(), |
267 | pid, address, node->id, migration ? 'M' : 'U'); |
268 | } |
269 | |
270 | void kfd_smi_event_migration_start(struct kfd_node *node, pid_t pid, |
271 | unsigned long start, unsigned long end, |
272 | uint32_t from, uint32_t to, |
273 | uint32_t prefetch_loc, uint32_t preferred_loc, |
274 | uint32_t trigger) |
275 | { |
276 | kfd_smi_event_add(pid, dev: node, event: KFD_SMI_EVENT_MIGRATE_START, |
277 | fmt: "%lld -%d @%lx(%lx) %x->%x %x:%x %d\n" , |
278 | ktime_get_boottime_ns(), pid, start, end - start, |
279 | from, to, prefetch_loc, preferred_loc, trigger); |
280 | } |
281 | |
282 | void kfd_smi_event_migration_end(struct kfd_node *node, pid_t pid, |
283 | unsigned long start, unsigned long end, |
284 | uint32_t from, uint32_t to, uint32_t trigger) |
285 | { |
286 | kfd_smi_event_add(pid, dev: node, event: KFD_SMI_EVENT_MIGRATE_END, |
287 | fmt: "%lld -%d @%lx(%lx) %x->%x %d\n" , |
288 | ktime_get_boottime_ns(), pid, start, end - start, |
289 | from, to, trigger); |
290 | } |
291 | |
292 | void kfd_smi_event_queue_eviction(struct kfd_node *node, pid_t pid, |
293 | uint32_t trigger) |
294 | { |
295 | kfd_smi_event_add(pid, dev: node, event: KFD_SMI_EVENT_QUEUE_EVICTION, |
296 | fmt: "%lld -%d %x %d\n" , ktime_get_boottime_ns(), pid, |
297 | node->id, trigger); |
298 | } |
299 | |
300 | void kfd_smi_event_queue_restore(struct kfd_node *node, pid_t pid) |
301 | { |
302 | kfd_smi_event_add(pid, dev: node, event: KFD_SMI_EVENT_QUEUE_RESTORE, |
303 | fmt: "%lld -%d %x\n" , ktime_get_boottime_ns(), pid, |
304 | node->id); |
305 | } |
306 | |
307 | void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm) |
308 | { |
309 | struct kfd_process *p; |
310 | int i; |
311 | |
312 | p = kfd_lookup_process_by_mm(mm); |
313 | if (!p) |
314 | return; |
315 | |
316 | for (i = 0; i < p->n_pdds; i++) { |
317 | struct kfd_process_device *pdd = p->pdds[i]; |
318 | |
319 | kfd_smi_event_add(pid: p->lead_thread->pid, dev: pdd->dev, |
320 | event: KFD_SMI_EVENT_QUEUE_RESTORE, |
321 | fmt: "%lld -%d %x %c\n" , ktime_get_boottime_ns(), |
322 | p->lead_thread->pid, pdd->dev->id, 'R'); |
323 | } |
324 | kfd_unref_process(p); |
325 | } |
326 | |
327 | void kfd_smi_event_unmap_from_gpu(struct kfd_node *node, pid_t pid, |
328 | unsigned long address, unsigned long last, |
329 | uint32_t trigger) |
330 | { |
331 | kfd_smi_event_add(pid, dev: node, event: KFD_SMI_EVENT_UNMAP_FROM_GPU, |
332 | fmt: "%lld -%d @%lx(%lx) %x %d\n" , ktime_get_boottime_ns(), |
333 | pid, address, last - address + 1, node->id, trigger); |
334 | } |
335 | |
336 | int kfd_smi_event_open(struct kfd_node *dev, uint32_t *fd) |
337 | { |
338 | struct kfd_smi_client *client; |
339 | int ret; |
340 | |
341 | client = kzalloc(size: sizeof(struct kfd_smi_client), GFP_KERNEL); |
342 | if (!client) |
343 | return -ENOMEM; |
344 | INIT_LIST_HEAD(list: &client->list); |
345 | |
346 | ret = kfifo_alloc(&client->fifo, MAX_KFIFO_SIZE, GFP_KERNEL); |
347 | if (ret) { |
348 | kfree(objp: client); |
349 | return ret; |
350 | } |
351 | |
352 | init_waitqueue_head(&client->wait_queue); |
353 | spin_lock_init(&client->lock); |
354 | client->events = 0; |
355 | client->dev = dev; |
356 | client->pid = current->tgid; |
357 | client->suser = capable(CAP_SYS_ADMIN); |
358 | |
359 | spin_lock(lock: &dev->smi_lock); |
360 | list_add_rcu(new: &client->list, head: &dev->smi_clients); |
361 | spin_unlock(lock: &dev->smi_lock); |
362 | |
363 | ret = anon_inode_getfd(name: kfd_smi_name, fops: &kfd_smi_ev_fops, priv: (void *)client, |
364 | O_RDWR); |
365 | if (ret < 0) { |
366 | spin_lock(lock: &dev->smi_lock); |
367 | list_del_rcu(entry: &client->list); |
368 | spin_unlock(lock: &dev->smi_lock); |
369 | |
370 | synchronize_rcu(); |
371 | |
372 | kfifo_free(&client->fifo); |
373 | kfree(objp: client); |
374 | return ret; |
375 | } |
376 | *fd = ret; |
377 | |
378 | return 0; |
379 | } |
380 | |