1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Sync File validation framework |
4 | * |
5 | * Copyright (C) 2012 Google, Inc. |
6 | */ |
7 | |
8 | #include <linux/file.h> |
9 | #include <linux/fs.h> |
10 | #include <linux/uaccess.h> |
11 | #include <linux/slab.h> |
12 | #include <linux/sync_file.h> |
13 | |
14 | #include "sync_debug.h" |
15 | |
16 | #define CREATE_TRACE_POINTS |
17 | #include "sync_trace.h" |
18 | |
19 | /* |
20 | * SW SYNC validation framework |
21 | * |
22 | * A sync object driver that uses a 32bit counter to coordinate |
23 | * synchronization. Useful when there is no hardware primitive backing |
24 | * the synchronization. |
25 | * |
26 | * To start the framework just open: |
27 | * |
28 | * <debugfs>/sync/sw_sync |
29 | * |
30 | * That will create a sync timeline, all fences created under this timeline |
31 | * file descriptor will belong to the this timeline. |
32 | * |
33 | * The 'sw_sync' file can be opened many times as to create different |
34 | * timelines. |
35 | * |
36 | * Fences can be created with SW_SYNC_IOC_CREATE_FENCE ioctl with struct |
37 | * sw_sync_create_fence_data as parameter. |
38 | * |
39 | * To increment the timeline counter, SW_SYNC_IOC_INC ioctl should be used |
40 | * with the increment as u32. This will update the last signaled value |
41 | * from the timeline and signal any fence that has a seqno smaller or equal |
42 | * to it. |
43 | * |
44 | * struct sw_sync_create_fence_data |
45 | * @value: the seqno to initialise the fence with |
46 | * @name: the name of the new sync point |
47 | * @fence: return the fd of the new sync_file with the created fence |
48 | */ |
49 | struct sw_sync_create_fence_data { |
50 | __u32 value; |
51 | char name[32]; |
52 | __s32 fence; /* fd of new fence */ |
53 | }; |
54 | |
55 | /** |
56 | * struct sw_sync_get_deadline - get the deadline hint of a sw_sync fence |
57 | * @deadline_ns: absolute time of the deadline |
58 | * @pad: must be zero |
59 | * @fence_fd: the sw_sync fence fd (in) |
60 | * |
61 | * Return the earliest deadline set on the fence. The timebase for the |
62 | * deadline is CLOCK_MONOTONIC (same as vblank). If there is no deadline |
63 | * set on the fence, this ioctl will return -ENOENT. |
64 | */ |
65 | struct sw_sync_get_deadline { |
66 | __u64 deadline_ns; |
67 | __u32 pad; |
68 | __s32 fence_fd; |
69 | }; |
70 | |
71 | #define SW_SYNC_IOC_MAGIC 'W' |
72 | |
73 | #define SW_SYNC_IOC_CREATE_FENCE _IOWR(SW_SYNC_IOC_MAGIC, 0,\ |
74 | struct sw_sync_create_fence_data) |
75 | |
76 | #define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32) |
77 | #define SW_SYNC_GET_DEADLINE _IOWR(SW_SYNC_IOC_MAGIC, 2, \ |
78 | struct sw_sync_get_deadline) |
79 | |
80 | |
81 | #define SW_SYNC_HAS_DEADLINE_BIT DMA_FENCE_FLAG_USER_BITS |
82 | |
83 | static const struct dma_fence_ops timeline_fence_ops; |
84 | |
85 | static inline struct sync_pt *dma_fence_to_sync_pt(struct dma_fence *fence) |
86 | { |
87 | if (fence->ops != &timeline_fence_ops) |
88 | return NULL; |
89 | return container_of(fence, struct sync_pt, base); |
90 | } |
91 | |
92 | /** |
93 | * sync_timeline_create() - creates a sync object |
94 | * @name: sync_timeline name |
95 | * |
96 | * Creates a new sync_timeline. Returns the sync_timeline object or NULL in |
97 | * case of error. |
98 | */ |
99 | static struct sync_timeline *sync_timeline_create(const char *name) |
100 | { |
101 | struct sync_timeline *obj; |
102 | |
103 | obj = kzalloc(size: sizeof(*obj), GFP_KERNEL); |
104 | if (!obj) |
105 | return NULL; |
106 | |
107 | kref_init(kref: &obj->kref); |
108 | obj->context = dma_fence_context_alloc(num: 1); |
109 | strscpy(obj->name, name, sizeof(obj->name)); |
110 | |
111 | obj->pt_tree = RB_ROOT; |
112 | INIT_LIST_HEAD(list: &obj->pt_list); |
113 | spin_lock_init(&obj->lock); |
114 | |
115 | sync_timeline_debug_add(obj); |
116 | |
117 | return obj; |
118 | } |
119 | |
120 | static void sync_timeline_free(struct kref *kref) |
121 | { |
122 | struct sync_timeline *obj = |
123 | container_of(kref, struct sync_timeline, kref); |
124 | |
125 | sync_timeline_debug_remove(obj); |
126 | |
127 | kfree(objp: obj); |
128 | } |
129 | |
130 | static void sync_timeline_get(struct sync_timeline *obj) |
131 | { |
132 | kref_get(kref: &obj->kref); |
133 | } |
134 | |
135 | static void sync_timeline_put(struct sync_timeline *obj) |
136 | { |
137 | kref_put(kref: &obj->kref, release: sync_timeline_free); |
138 | } |
139 | |
140 | static const char *timeline_fence_get_driver_name(struct dma_fence *fence) |
141 | { |
142 | return "sw_sync" ; |
143 | } |
144 | |
145 | static const char *timeline_fence_get_timeline_name(struct dma_fence *fence) |
146 | { |
147 | struct sync_timeline *parent = dma_fence_parent(fence); |
148 | |
149 | return parent->name; |
150 | } |
151 | |
152 | static void timeline_fence_release(struct dma_fence *fence) |
153 | { |
154 | struct sync_pt *pt = dma_fence_to_sync_pt(fence); |
155 | struct sync_timeline *parent = dma_fence_parent(fence); |
156 | unsigned long flags; |
157 | |
158 | spin_lock_irqsave(fence->lock, flags); |
159 | if (!list_empty(head: &pt->link)) { |
160 | list_del(entry: &pt->link); |
161 | rb_erase(&pt->node, &parent->pt_tree); |
162 | } |
163 | spin_unlock_irqrestore(lock: fence->lock, flags); |
164 | |
165 | sync_timeline_put(obj: parent); |
166 | dma_fence_free(fence); |
167 | } |
168 | |
169 | static bool timeline_fence_signaled(struct dma_fence *fence) |
170 | { |
171 | struct sync_timeline *parent = dma_fence_parent(fence); |
172 | |
173 | return !__dma_fence_is_later(f1: fence->seqno, f2: parent->value, ops: fence->ops); |
174 | } |
175 | |
176 | static bool timeline_fence_enable_signaling(struct dma_fence *fence) |
177 | { |
178 | return true; |
179 | } |
180 | |
181 | static void timeline_fence_value_str(struct dma_fence *fence, |
182 | char *str, int size) |
183 | { |
184 | snprintf(buf: str, size, fmt: "%lld" , fence->seqno); |
185 | } |
186 | |
187 | static void timeline_fence_timeline_value_str(struct dma_fence *fence, |
188 | char *str, int size) |
189 | { |
190 | struct sync_timeline *parent = dma_fence_parent(fence); |
191 | |
192 | snprintf(buf: str, size, fmt: "%d" , parent->value); |
193 | } |
194 | |
195 | static void timeline_fence_set_deadline(struct dma_fence *fence, ktime_t deadline) |
196 | { |
197 | struct sync_pt *pt = dma_fence_to_sync_pt(fence); |
198 | unsigned long flags; |
199 | |
200 | spin_lock_irqsave(fence->lock, flags); |
201 | if (test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) { |
202 | if (ktime_before(cmp1: deadline, cmp2: pt->deadline)) |
203 | pt->deadline = deadline; |
204 | } else { |
205 | pt->deadline = deadline; |
206 | __set_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags); |
207 | } |
208 | spin_unlock_irqrestore(lock: fence->lock, flags); |
209 | } |
210 | |
211 | static const struct dma_fence_ops timeline_fence_ops = { |
212 | .get_driver_name = timeline_fence_get_driver_name, |
213 | .get_timeline_name = timeline_fence_get_timeline_name, |
214 | .enable_signaling = timeline_fence_enable_signaling, |
215 | .signaled = timeline_fence_signaled, |
216 | .release = timeline_fence_release, |
217 | .fence_value_str = timeline_fence_value_str, |
218 | .timeline_value_str = timeline_fence_timeline_value_str, |
219 | .set_deadline = timeline_fence_set_deadline, |
220 | }; |
221 | |
222 | /** |
223 | * sync_timeline_signal() - signal a status change on a sync_timeline |
224 | * @obj: sync_timeline to signal |
225 | * @inc: num to increment on timeline->value |
226 | * |
227 | * A sync implementation should call this any time one of it's fences |
228 | * has signaled or has an error condition. |
229 | */ |
230 | static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc) |
231 | { |
232 | LIST_HEAD(signalled); |
233 | struct sync_pt *pt, *next; |
234 | |
235 | trace_sync_timeline(timeline: obj); |
236 | |
237 | spin_lock_irq(lock: &obj->lock); |
238 | |
239 | obj->value += inc; |
240 | |
241 | list_for_each_entry_safe(pt, next, &obj->pt_list, link) { |
242 | if (!timeline_fence_signaled(fence: &pt->base)) |
243 | break; |
244 | |
245 | dma_fence_get(fence: &pt->base); |
246 | |
247 | list_move_tail(list: &pt->link, head: &signalled); |
248 | rb_erase(&pt->node, &obj->pt_tree); |
249 | |
250 | dma_fence_signal_locked(fence: &pt->base); |
251 | } |
252 | |
253 | spin_unlock_irq(lock: &obj->lock); |
254 | |
255 | list_for_each_entry_safe(pt, next, &signalled, link) { |
256 | list_del_init(entry: &pt->link); |
257 | dma_fence_put(fence: &pt->base); |
258 | } |
259 | } |
260 | |
261 | /** |
262 | * sync_pt_create() - creates a sync pt |
263 | * @obj: parent sync_timeline |
264 | * @value: value of the fence |
265 | * |
266 | * Creates a new sync_pt (fence) as a child of @parent. @size bytes will be |
267 | * allocated allowing for implementation specific data to be kept after |
268 | * the generic sync_timeline struct. Returns the sync_pt object or |
269 | * NULL in case of error. |
270 | */ |
271 | static struct sync_pt *sync_pt_create(struct sync_timeline *obj, |
272 | unsigned int value) |
273 | { |
274 | struct sync_pt *pt; |
275 | |
276 | pt = kzalloc(size: sizeof(*pt), GFP_KERNEL); |
277 | if (!pt) |
278 | return NULL; |
279 | |
280 | sync_timeline_get(obj); |
281 | dma_fence_init(fence: &pt->base, ops: &timeline_fence_ops, lock: &obj->lock, |
282 | context: obj->context, seqno: value); |
283 | INIT_LIST_HEAD(list: &pt->link); |
284 | |
285 | spin_lock_irq(lock: &obj->lock); |
286 | if (!dma_fence_is_signaled_locked(fence: &pt->base)) { |
287 | struct rb_node **p = &obj->pt_tree.rb_node; |
288 | struct rb_node *parent = NULL; |
289 | |
290 | while (*p) { |
291 | struct sync_pt *other; |
292 | int cmp; |
293 | |
294 | parent = *p; |
295 | other = rb_entry(parent, typeof(*pt), node); |
296 | cmp = value - other->base.seqno; |
297 | if (cmp > 0) { |
298 | p = &parent->rb_right; |
299 | } else if (cmp < 0) { |
300 | p = &parent->rb_left; |
301 | } else { |
302 | if (dma_fence_get_rcu(fence: &other->base)) { |
303 | sync_timeline_put(obj); |
304 | kfree(objp: pt); |
305 | pt = other; |
306 | goto unlock; |
307 | } |
308 | p = &parent->rb_left; |
309 | } |
310 | } |
311 | rb_link_node(node: &pt->node, parent, rb_link: p); |
312 | rb_insert_color(&pt->node, &obj->pt_tree); |
313 | |
314 | parent = rb_next(&pt->node); |
315 | list_add_tail(new: &pt->link, |
316 | head: parent ? &rb_entry(parent, typeof(*pt), node)->link : &obj->pt_list); |
317 | } |
318 | unlock: |
319 | spin_unlock_irq(lock: &obj->lock); |
320 | |
321 | return pt; |
322 | } |
323 | |
324 | /* |
325 | * *WARNING* |
326 | * |
327 | * improper use of this can result in deadlocking kernel drivers from userspace. |
328 | */ |
329 | |
330 | /* opening sw_sync create a new sync obj */ |
331 | static int sw_sync_debugfs_open(struct inode *inode, struct file *file) |
332 | { |
333 | struct sync_timeline *obj; |
334 | char task_comm[TASK_COMM_LEN]; |
335 | |
336 | get_task_comm(task_comm, current); |
337 | |
338 | obj = sync_timeline_create(name: task_comm); |
339 | if (!obj) |
340 | return -ENOMEM; |
341 | |
342 | file->private_data = obj; |
343 | |
344 | return 0; |
345 | } |
346 | |
347 | static int sw_sync_debugfs_release(struct inode *inode, struct file *file) |
348 | { |
349 | struct sync_timeline *obj = file->private_data; |
350 | struct sync_pt *pt, *next; |
351 | |
352 | spin_lock_irq(lock: &obj->lock); |
353 | |
354 | list_for_each_entry_safe(pt, next, &obj->pt_list, link) { |
355 | dma_fence_set_error(fence: &pt->base, error: -ENOENT); |
356 | dma_fence_signal_locked(fence: &pt->base); |
357 | } |
358 | |
359 | spin_unlock_irq(lock: &obj->lock); |
360 | |
361 | sync_timeline_put(obj); |
362 | return 0; |
363 | } |
364 | |
365 | static long sw_sync_ioctl_create_fence(struct sync_timeline *obj, |
366 | unsigned long arg) |
367 | { |
368 | int fd = get_unused_fd_flags(O_CLOEXEC); |
369 | int err; |
370 | struct sync_pt *pt; |
371 | struct sync_file *sync_file; |
372 | struct sw_sync_create_fence_data data; |
373 | |
374 | if (fd < 0) |
375 | return fd; |
376 | |
377 | if (copy_from_user(to: &data, from: (void __user *)arg, n: sizeof(data))) { |
378 | err = -EFAULT; |
379 | goto err; |
380 | } |
381 | |
382 | pt = sync_pt_create(obj, value: data.value); |
383 | if (!pt) { |
384 | err = -ENOMEM; |
385 | goto err; |
386 | } |
387 | |
388 | sync_file = sync_file_create(fence: &pt->base); |
389 | dma_fence_put(fence: &pt->base); |
390 | if (!sync_file) { |
391 | err = -ENOMEM; |
392 | goto err; |
393 | } |
394 | |
395 | data.fence = fd; |
396 | if (copy_to_user(to: (void __user *)arg, from: &data, n: sizeof(data))) { |
397 | fput(sync_file->file); |
398 | err = -EFAULT; |
399 | goto err; |
400 | } |
401 | |
402 | fd_install(fd, file: sync_file->file); |
403 | |
404 | return 0; |
405 | |
406 | err: |
407 | put_unused_fd(fd); |
408 | return err; |
409 | } |
410 | |
411 | static long sw_sync_ioctl_inc(struct sync_timeline *obj, unsigned long arg) |
412 | { |
413 | u32 value; |
414 | |
415 | if (copy_from_user(to: &value, from: (void __user *)arg, n: sizeof(value))) |
416 | return -EFAULT; |
417 | |
418 | while (value > INT_MAX) { |
419 | sync_timeline_signal(obj, INT_MAX); |
420 | value -= INT_MAX; |
421 | } |
422 | |
423 | sync_timeline_signal(obj, inc: value); |
424 | |
425 | return 0; |
426 | } |
427 | |
428 | static int sw_sync_ioctl_get_deadline(struct sync_timeline *obj, unsigned long arg) |
429 | { |
430 | struct sw_sync_get_deadline data; |
431 | struct dma_fence *fence; |
432 | unsigned long flags; |
433 | struct sync_pt *pt; |
434 | int ret = 0; |
435 | |
436 | if (copy_from_user(to: &data, from: (void __user *)arg, n: sizeof(data))) |
437 | return -EFAULT; |
438 | |
439 | if (data.deadline_ns || data.pad) |
440 | return -EINVAL; |
441 | |
442 | fence = sync_file_get_fence(fd: data.fence_fd); |
443 | if (!fence) |
444 | return -EINVAL; |
445 | |
446 | pt = dma_fence_to_sync_pt(fence); |
447 | if (!pt) |
448 | return -EINVAL; |
449 | |
450 | spin_lock_irqsave(fence->lock, flags); |
451 | if (test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) { |
452 | data.deadline_ns = ktime_to_ns(kt: pt->deadline); |
453 | } else { |
454 | ret = -ENOENT; |
455 | } |
456 | spin_unlock_irqrestore(lock: fence->lock, flags); |
457 | |
458 | dma_fence_put(fence); |
459 | |
460 | if (ret) |
461 | return ret; |
462 | |
463 | if (copy_to_user(to: (void __user *)arg, from: &data, n: sizeof(data))) |
464 | return -EFAULT; |
465 | |
466 | return 0; |
467 | } |
468 | |
469 | static long sw_sync_ioctl(struct file *file, unsigned int cmd, |
470 | unsigned long arg) |
471 | { |
472 | struct sync_timeline *obj = file->private_data; |
473 | |
474 | switch (cmd) { |
475 | case SW_SYNC_IOC_CREATE_FENCE: |
476 | return sw_sync_ioctl_create_fence(obj, arg); |
477 | |
478 | case SW_SYNC_IOC_INC: |
479 | return sw_sync_ioctl_inc(obj, arg); |
480 | |
481 | case SW_SYNC_GET_DEADLINE: |
482 | return sw_sync_ioctl_get_deadline(obj, arg); |
483 | |
484 | default: |
485 | return -ENOTTY; |
486 | } |
487 | } |
488 | |
489 | const struct file_operations sw_sync_debugfs_fops = { |
490 | .open = sw_sync_debugfs_open, |
491 | .release = sw_sync_debugfs_release, |
492 | .unlocked_ioctl = sw_sync_ioctl, |
493 | .compat_ioctl = compat_ptr_ioctl, |
494 | }; |
495 | |