1 | /* SPDX-License-Identifier: MIT */ |
2 | |
3 | /* |
4 | * Copyright © 2019 Intel Corporation |
5 | */ |
6 | |
7 | #include <linux/delay.h> |
8 | #include <linux/dma-fence.h> |
9 | #include <linux/kernel.h> |
10 | #include <linux/kthread.h> |
11 | #include <linux/sched/signal.h> |
12 | #include <linux/slab.h> |
13 | #include <linux/spinlock.h> |
14 | |
15 | #include "selftest.h" |
16 | |
17 | static struct kmem_cache *slab_fences; |
18 | |
19 | static struct mock_fence { |
20 | struct dma_fence base; |
21 | struct spinlock lock; |
22 | } *to_mock_fence(struct dma_fence *f) { |
23 | return container_of(f, struct mock_fence, base); |
24 | } |
25 | |
26 | static const char *mock_name(struct dma_fence *f) |
27 | { |
28 | return "mock" ; |
29 | } |
30 | |
31 | static void mock_fence_release(struct dma_fence *f) |
32 | { |
33 | kmem_cache_free(s: slab_fences, objp: to_mock_fence(f)); |
34 | } |
35 | |
36 | struct wait_cb { |
37 | struct dma_fence_cb cb; |
38 | struct task_struct *task; |
39 | }; |
40 | |
41 | static void mock_wakeup(struct dma_fence *f, struct dma_fence_cb *cb) |
42 | { |
43 | wake_up_process(container_of(cb, struct wait_cb, cb)->task); |
44 | } |
45 | |
46 | static long mock_wait(struct dma_fence *f, bool intr, long timeout) |
47 | { |
48 | const int state = intr ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; |
49 | struct wait_cb cb = { .task = current }; |
50 | |
51 | if (dma_fence_add_callback(fence: f, cb: &cb.cb, func: mock_wakeup)) |
52 | return timeout; |
53 | |
54 | while (timeout) { |
55 | set_current_state(state); |
56 | |
57 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags)) |
58 | break; |
59 | |
60 | if (signal_pending_state(state, current)) |
61 | break; |
62 | |
63 | timeout = schedule_timeout(timeout); |
64 | } |
65 | __set_current_state(TASK_RUNNING); |
66 | |
67 | if (!dma_fence_remove_callback(fence: f, cb: &cb.cb)) |
68 | return timeout; |
69 | |
70 | if (signal_pending_state(state, current)) |
71 | return -ERESTARTSYS; |
72 | |
73 | return -ETIME; |
74 | } |
75 | |
76 | static const struct dma_fence_ops mock_ops = { |
77 | .get_driver_name = mock_name, |
78 | .get_timeline_name = mock_name, |
79 | .wait = mock_wait, |
80 | .release = mock_fence_release, |
81 | }; |
82 | |
83 | static struct dma_fence *mock_fence(void) |
84 | { |
85 | struct mock_fence *f; |
86 | |
87 | f = kmem_cache_alloc(cachep: slab_fences, GFP_KERNEL); |
88 | if (!f) |
89 | return NULL; |
90 | |
91 | spin_lock_init(&f->lock); |
92 | dma_fence_init(fence: &f->base, ops: &mock_ops, lock: &f->lock, context: 0, seqno: 0); |
93 | |
94 | return &f->base; |
95 | } |
96 | |
97 | static int sanitycheck(void *arg) |
98 | { |
99 | struct dma_fence *f; |
100 | |
101 | f = mock_fence(); |
102 | if (!f) |
103 | return -ENOMEM; |
104 | |
105 | dma_fence_enable_sw_signaling(fence: f); |
106 | |
107 | dma_fence_signal(fence: f); |
108 | dma_fence_put(fence: f); |
109 | |
110 | return 0; |
111 | } |
112 | |
113 | static int test_signaling(void *arg) |
114 | { |
115 | struct dma_fence *f; |
116 | int err = -EINVAL; |
117 | |
118 | f = mock_fence(); |
119 | if (!f) |
120 | return -ENOMEM; |
121 | |
122 | dma_fence_enable_sw_signaling(fence: f); |
123 | |
124 | if (dma_fence_is_signaled(fence: f)) { |
125 | pr_err("Fence unexpectedly signaled on creation\n" ); |
126 | goto err_free; |
127 | } |
128 | |
129 | if (dma_fence_signal(fence: f)) { |
130 | pr_err("Fence reported being already signaled\n" ); |
131 | goto err_free; |
132 | } |
133 | |
134 | if (!dma_fence_is_signaled(fence: f)) { |
135 | pr_err("Fence not reporting signaled\n" ); |
136 | goto err_free; |
137 | } |
138 | |
139 | if (!dma_fence_signal(fence: f)) { |
140 | pr_err("Fence reported not being already signaled\n" ); |
141 | goto err_free; |
142 | } |
143 | |
144 | err = 0; |
145 | err_free: |
146 | dma_fence_put(fence: f); |
147 | return err; |
148 | } |
149 | |
150 | struct simple_cb { |
151 | struct dma_fence_cb cb; |
152 | bool seen; |
153 | }; |
154 | |
155 | static void simple_callback(struct dma_fence *f, struct dma_fence_cb *cb) |
156 | { |
157 | smp_store_mb(container_of(cb, struct simple_cb, cb)->seen, true); |
158 | } |
159 | |
160 | static int test_add_callback(void *arg) |
161 | { |
162 | struct simple_cb cb = {}; |
163 | struct dma_fence *f; |
164 | int err = -EINVAL; |
165 | |
166 | f = mock_fence(); |
167 | if (!f) |
168 | return -ENOMEM; |
169 | |
170 | if (dma_fence_add_callback(fence: f, cb: &cb.cb, func: simple_callback)) { |
171 | pr_err("Failed to add callback, fence already signaled!\n" ); |
172 | goto err_free; |
173 | } |
174 | |
175 | dma_fence_signal(fence: f); |
176 | if (!cb.seen) { |
177 | pr_err("Callback failed!\n" ); |
178 | goto err_free; |
179 | } |
180 | |
181 | err = 0; |
182 | err_free: |
183 | dma_fence_put(fence: f); |
184 | return err; |
185 | } |
186 | |
187 | static int test_late_add_callback(void *arg) |
188 | { |
189 | struct simple_cb cb = {}; |
190 | struct dma_fence *f; |
191 | int err = -EINVAL; |
192 | |
193 | f = mock_fence(); |
194 | if (!f) |
195 | return -ENOMEM; |
196 | |
197 | dma_fence_enable_sw_signaling(fence: f); |
198 | |
199 | dma_fence_signal(fence: f); |
200 | |
201 | if (!dma_fence_add_callback(fence: f, cb: &cb.cb, func: simple_callback)) { |
202 | pr_err("Added callback, but fence was already signaled!\n" ); |
203 | goto err_free; |
204 | } |
205 | |
206 | dma_fence_signal(fence: f); |
207 | if (cb.seen) { |
208 | pr_err("Callback called after failed attachment !\n" ); |
209 | goto err_free; |
210 | } |
211 | |
212 | err = 0; |
213 | err_free: |
214 | dma_fence_put(fence: f); |
215 | return err; |
216 | } |
217 | |
218 | static int test_rm_callback(void *arg) |
219 | { |
220 | struct simple_cb cb = {}; |
221 | struct dma_fence *f; |
222 | int err = -EINVAL; |
223 | |
224 | f = mock_fence(); |
225 | if (!f) |
226 | return -ENOMEM; |
227 | |
228 | if (dma_fence_add_callback(fence: f, cb: &cb.cb, func: simple_callback)) { |
229 | pr_err("Failed to add callback, fence already signaled!\n" ); |
230 | goto err_free; |
231 | } |
232 | |
233 | if (!dma_fence_remove_callback(fence: f, cb: &cb.cb)) { |
234 | pr_err("Failed to remove callback!\n" ); |
235 | goto err_free; |
236 | } |
237 | |
238 | dma_fence_signal(fence: f); |
239 | if (cb.seen) { |
240 | pr_err("Callback still signaled after removal!\n" ); |
241 | goto err_free; |
242 | } |
243 | |
244 | err = 0; |
245 | err_free: |
246 | dma_fence_put(fence: f); |
247 | return err; |
248 | } |
249 | |
250 | static int test_late_rm_callback(void *arg) |
251 | { |
252 | struct simple_cb cb = {}; |
253 | struct dma_fence *f; |
254 | int err = -EINVAL; |
255 | |
256 | f = mock_fence(); |
257 | if (!f) |
258 | return -ENOMEM; |
259 | |
260 | if (dma_fence_add_callback(fence: f, cb: &cb.cb, func: simple_callback)) { |
261 | pr_err("Failed to add callback, fence already signaled!\n" ); |
262 | goto err_free; |
263 | } |
264 | |
265 | dma_fence_signal(fence: f); |
266 | if (!cb.seen) { |
267 | pr_err("Callback failed!\n" ); |
268 | goto err_free; |
269 | } |
270 | |
271 | if (dma_fence_remove_callback(fence: f, cb: &cb.cb)) { |
272 | pr_err("Callback removal succeed after being executed!\n" ); |
273 | goto err_free; |
274 | } |
275 | |
276 | err = 0; |
277 | err_free: |
278 | dma_fence_put(fence: f); |
279 | return err; |
280 | } |
281 | |
282 | static int test_status(void *arg) |
283 | { |
284 | struct dma_fence *f; |
285 | int err = -EINVAL; |
286 | |
287 | f = mock_fence(); |
288 | if (!f) |
289 | return -ENOMEM; |
290 | |
291 | dma_fence_enable_sw_signaling(fence: f); |
292 | |
293 | if (dma_fence_get_status(fence: f)) { |
294 | pr_err("Fence unexpectedly has signaled status on creation\n" ); |
295 | goto err_free; |
296 | } |
297 | |
298 | dma_fence_signal(fence: f); |
299 | if (!dma_fence_get_status(fence: f)) { |
300 | pr_err("Fence not reporting signaled status\n" ); |
301 | goto err_free; |
302 | } |
303 | |
304 | err = 0; |
305 | err_free: |
306 | dma_fence_put(fence: f); |
307 | return err; |
308 | } |
309 | |
310 | static int test_error(void *arg) |
311 | { |
312 | struct dma_fence *f; |
313 | int err = -EINVAL; |
314 | |
315 | f = mock_fence(); |
316 | if (!f) |
317 | return -ENOMEM; |
318 | |
319 | dma_fence_enable_sw_signaling(fence: f); |
320 | |
321 | dma_fence_set_error(fence: f, error: -EIO); |
322 | |
323 | if (dma_fence_get_status(fence: f)) { |
324 | pr_err("Fence unexpectedly has error status before signal\n" ); |
325 | goto err_free; |
326 | } |
327 | |
328 | dma_fence_signal(fence: f); |
329 | if (dma_fence_get_status(fence: f) != -EIO) { |
330 | pr_err("Fence not reporting error status, got %d\n" , |
331 | dma_fence_get_status(f)); |
332 | goto err_free; |
333 | } |
334 | |
335 | err = 0; |
336 | err_free: |
337 | dma_fence_put(fence: f); |
338 | return err; |
339 | } |
340 | |
341 | static int test_wait(void *arg) |
342 | { |
343 | struct dma_fence *f; |
344 | int err = -EINVAL; |
345 | |
346 | f = mock_fence(); |
347 | if (!f) |
348 | return -ENOMEM; |
349 | |
350 | dma_fence_enable_sw_signaling(fence: f); |
351 | |
352 | if (dma_fence_wait_timeout(f, intr: false, timeout: 0) != -ETIME) { |
353 | pr_err("Wait reported complete before being signaled\n" ); |
354 | goto err_free; |
355 | } |
356 | |
357 | dma_fence_signal(fence: f); |
358 | |
359 | if (dma_fence_wait_timeout(f, intr: false, timeout: 0) != 0) { |
360 | pr_err("Wait reported incomplete after being signaled\n" ); |
361 | goto err_free; |
362 | } |
363 | |
364 | err = 0; |
365 | err_free: |
366 | dma_fence_signal(fence: f); |
367 | dma_fence_put(fence: f); |
368 | return err; |
369 | } |
370 | |
371 | struct wait_timer { |
372 | struct timer_list timer; |
373 | struct dma_fence *f; |
374 | }; |
375 | |
376 | static void wait_timer(struct timer_list *timer) |
377 | { |
378 | struct wait_timer *wt = from_timer(wt, timer, timer); |
379 | |
380 | dma_fence_signal(fence: wt->f); |
381 | } |
382 | |
383 | static int test_wait_timeout(void *arg) |
384 | { |
385 | struct wait_timer wt; |
386 | int err = -EINVAL; |
387 | |
388 | timer_setup_on_stack(&wt.timer, wait_timer, 0); |
389 | |
390 | wt.f = mock_fence(); |
391 | if (!wt.f) |
392 | return -ENOMEM; |
393 | |
394 | dma_fence_enable_sw_signaling(fence: wt.f); |
395 | |
396 | if (dma_fence_wait_timeout(wt.f, intr: false, timeout: 1) != -ETIME) { |
397 | pr_err("Wait reported complete before being signaled\n" ); |
398 | goto err_free; |
399 | } |
400 | |
401 | mod_timer(timer: &wt.timer, expires: jiffies + 1); |
402 | |
403 | if (dma_fence_wait_timeout(wt.f, intr: false, timeout: 2) == -ETIME) { |
404 | if (timer_pending(timer: &wt.timer)) { |
405 | pr_notice("Timer did not fire within the jiffie!\n" ); |
406 | err = 0; /* not our fault! */ |
407 | } else { |
408 | pr_err("Wait reported incomplete after timeout\n" ); |
409 | } |
410 | goto err_free; |
411 | } |
412 | |
413 | err = 0; |
414 | err_free: |
415 | del_timer_sync(timer: &wt.timer); |
416 | destroy_timer_on_stack(timer: &wt.timer); |
417 | dma_fence_signal(fence: wt.f); |
418 | dma_fence_put(fence: wt.f); |
419 | return err; |
420 | } |
421 | |
422 | static int test_stub(void *arg) |
423 | { |
424 | struct dma_fence *f[64]; |
425 | int err = -EINVAL; |
426 | int i; |
427 | |
428 | for (i = 0; i < ARRAY_SIZE(f); i++) { |
429 | f[i] = dma_fence_get_stub(); |
430 | if (!dma_fence_is_signaled(fence: f[i])) { |
431 | pr_err("Obtained unsignaled stub fence!\n" ); |
432 | goto err; |
433 | } |
434 | } |
435 | |
436 | err = 0; |
437 | err: |
438 | while (i--) |
439 | dma_fence_put(fence: f[i]); |
440 | return err; |
441 | } |
442 | |
443 | /* Now off to the races! */ |
444 | |
445 | struct race_thread { |
446 | struct dma_fence __rcu **fences; |
447 | struct task_struct *task; |
448 | bool before; |
449 | int id; |
450 | }; |
451 | |
452 | static void __wait_for_callbacks(struct dma_fence *f) |
453 | { |
454 | spin_lock_irq(lock: f->lock); |
455 | spin_unlock_irq(lock: f->lock); |
456 | } |
457 | |
458 | static int thread_signal_callback(void *arg) |
459 | { |
460 | const struct race_thread *t = arg; |
461 | unsigned long pass = 0; |
462 | unsigned long miss = 0; |
463 | int err = 0; |
464 | |
465 | while (!err && !kthread_should_stop()) { |
466 | struct dma_fence *f1, *f2; |
467 | struct simple_cb cb; |
468 | |
469 | f1 = mock_fence(); |
470 | if (!f1) { |
471 | err = -ENOMEM; |
472 | break; |
473 | } |
474 | |
475 | dma_fence_enable_sw_signaling(fence: f1); |
476 | |
477 | rcu_assign_pointer(t->fences[t->id], f1); |
478 | smp_wmb(); |
479 | |
480 | rcu_read_lock(); |
481 | do { |
482 | f2 = dma_fence_get_rcu_safe(fencep: &t->fences[!t->id]); |
483 | } while (!f2 && !kthread_should_stop()); |
484 | rcu_read_unlock(); |
485 | |
486 | if (t->before) |
487 | dma_fence_signal(fence: f1); |
488 | |
489 | smp_store_mb(cb.seen, false); |
490 | if (!f2 || |
491 | dma_fence_add_callback(fence: f2, cb: &cb.cb, func: simple_callback)) { |
492 | miss++; |
493 | cb.seen = true; |
494 | } |
495 | |
496 | if (!t->before) |
497 | dma_fence_signal(fence: f1); |
498 | |
499 | if (!cb.seen) { |
500 | dma_fence_wait(fence: f2, intr: false); |
501 | __wait_for_callbacks(f: f2); |
502 | } |
503 | |
504 | if (!READ_ONCE(cb.seen)) { |
505 | pr_err("Callback not seen on thread %d, pass %lu (%lu misses), signaling %s add_callback; fence signaled? %s\n" , |
506 | t->id, pass, miss, |
507 | t->before ? "before" : "after" , |
508 | dma_fence_is_signaled(f2) ? "yes" : "no" ); |
509 | err = -EINVAL; |
510 | } |
511 | |
512 | dma_fence_put(fence: f2); |
513 | |
514 | rcu_assign_pointer(t->fences[t->id], NULL); |
515 | smp_wmb(); |
516 | |
517 | dma_fence_put(fence: f1); |
518 | |
519 | pass++; |
520 | } |
521 | |
522 | pr_info("%s[%d] completed %lu passes, %lu misses\n" , |
523 | __func__, t->id, pass, miss); |
524 | return err; |
525 | } |
526 | |
527 | static int race_signal_callback(void *arg) |
528 | { |
529 | struct dma_fence __rcu *f[2] = {}; |
530 | int ret = 0; |
531 | int pass; |
532 | |
533 | for (pass = 0; !ret && pass <= 1; pass++) { |
534 | struct race_thread t[2]; |
535 | int i; |
536 | |
537 | for (i = 0; i < ARRAY_SIZE(t); i++) { |
538 | t[i].fences = f; |
539 | t[i].id = i; |
540 | t[i].before = pass; |
541 | t[i].task = kthread_run(thread_signal_callback, &t[i], |
542 | "dma-fence:%d" , i); |
543 | get_task_struct(t: t[i].task); |
544 | } |
545 | |
546 | msleep(msecs: 50); |
547 | |
548 | for (i = 0; i < ARRAY_SIZE(t); i++) { |
549 | int err; |
550 | |
551 | err = kthread_stop_put(k: t[i].task); |
552 | if (err && !ret) |
553 | ret = err; |
554 | } |
555 | } |
556 | |
557 | return ret; |
558 | } |
559 | |
560 | int dma_fence(void) |
561 | { |
562 | static const struct subtest tests[] = { |
563 | SUBTEST(sanitycheck), |
564 | SUBTEST(test_signaling), |
565 | SUBTEST(test_add_callback), |
566 | SUBTEST(test_late_add_callback), |
567 | SUBTEST(test_rm_callback), |
568 | SUBTEST(test_late_rm_callback), |
569 | SUBTEST(test_status), |
570 | SUBTEST(test_error), |
571 | SUBTEST(test_wait), |
572 | SUBTEST(test_wait_timeout), |
573 | SUBTEST(test_stub), |
574 | SUBTEST(race_signal_callback), |
575 | }; |
576 | int ret; |
577 | |
578 | pr_info("sizeof(dma_fence)=%zu\n" , sizeof(struct dma_fence)); |
579 | |
580 | slab_fences = KMEM_CACHE(mock_fence, |
581 | SLAB_TYPESAFE_BY_RCU | |
582 | SLAB_HWCACHE_ALIGN); |
583 | if (!slab_fences) |
584 | return -ENOMEM; |
585 | |
586 | ret = subtests(tests, NULL); |
587 | |
588 | kmem_cache_destroy(s: slab_fences); |
589 | |
590 | return ret; |
591 | } |
592 | |