1 | /* |
2 | * Copyright (C) 2007 Ben Skeggs. |
3 | * All Rights Reserved. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining |
6 | * a copy of this software and associated documentation files (the |
7 | * "Software"), to deal in the Software without restriction, including |
8 | * without limitation the rights to use, copy, modify, merge, publish, |
9 | * distribute, sublicense, and/or sell copies of the Software, and to |
10 | * permit persons to whom the Software is furnished to do so, subject to |
11 | * the following conditions: |
12 | * |
13 | * The above copyright notice and this permission notice (including the |
14 | * next paragraph) shall be included in all copies or substantial |
15 | * portions of the Software. |
16 | * |
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. |
20 | * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE |
21 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
22 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
24 | * |
25 | */ |
26 | |
27 | #include <linux/ktime.h> |
28 | #include <linux/hrtimer.h> |
29 | #include <linux/sched/signal.h> |
30 | #include <trace/events/dma_fence.h> |
31 | |
32 | #include <nvif/if0020.h> |
33 | |
34 | #include "nouveau_drv.h" |
35 | #include "nouveau_dma.h" |
36 | #include "nouveau_fence.h" |
37 | |
38 | static const struct dma_fence_ops nouveau_fence_ops_uevent; |
39 | static const struct dma_fence_ops nouveau_fence_ops_legacy; |
40 | |
41 | static inline struct nouveau_fence * |
42 | from_fence(struct dma_fence *fence) |
43 | { |
44 | return container_of(fence, struct nouveau_fence, base); |
45 | } |
46 | |
47 | static inline struct nouveau_fence_chan * |
48 | nouveau_fctx(struct nouveau_fence *fence) |
49 | { |
50 | return container_of(fence->base.lock, struct nouveau_fence_chan, lock); |
51 | } |
52 | |
53 | static int |
54 | nouveau_fence_signal(struct nouveau_fence *fence) |
55 | { |
56 | int drop = 0; |
57 | |
58 | dma_fence_signal_locked(fence: &fence->base); |
59 | list_del(entry: &fence->head); |
60 | rcu_assign_pointer(fence->channel, NULL); |
61 | |
62 | if (test_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags)) { |
63 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); |
64 | |
65 | if (!--fctx->notify_ref) |
66 | drop = 1; |
67 | } |
68 | |
69 | dma_fence_put(fence: &fence->base); |
70 | return drop; |
71 | } |
72 | |
73 | static struct nouveau_fence * |
74 | nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm) |
75 | { |
76 | if (fence->ops != &nouveau_fence_ops_legacy && |
77 | fence->ops != &nouveau_fence_ops_uevent) |
78 | return NULL; |
79 | |
80 | return from_fence(fence); |
81 | } |
82 | |
83 | void |
84 | nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error) |
85 | { |
86 | struct nouveau_fence *fence; |
87 | unsigned long flags; |
88 | |
89 | spin_lock_irqsave(&fctx->lock, flags); |
90 | while (!list_empty(head: &fctx->pending)) { |
91 | fence = list_entry(fctx->pending.next, typeof(*fence), head); |
92 | |
93 | if (error) |
94 | dma_fence_set_error(fence: &fence->base, error); |
95 | |
96 | if (nouveau_fence_signal(fence)) |
97 | nvif_event_block(&fctx->event); |
98 | } |
99 | fctx->killed = 1; |
100 | spin_unlock_irqrestore(lock: &fctx->lock, flags); |
101 | } |
102 | |
103 | void |
104 | nouveau_fence_context_del(struct nouveau_fence_chan *fctx) |
105 | { |
106 | cancel_work_sync(work: &fctx->uevent_work); |
107 | nouveau_fence_context_kill(fctx, error: 0); |
108 | nvif_event_dtor(&fctx->event); |
109 | fctx->dead = 1; |
110 | |
111 | /* |
112 | * Ensure that all accesses to fence->channel complete before freeing |
113 | * the channel. |
114 | */ |
115 | synchronize_rcu(); |
116 | } |
117 | |
118 | static void |
119 | nouveau_fence_context_put(struct kref *fence_ref) |
120 | { |
121 | kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref)); |
122 | } |
123 | |
124 | void |
125 | nouveau_fence_context_free(struct nouveau_fence_chan *fctx) |
126 | { |
127 | kref_put(kref: &fctx->fence_ref, release: nouveau_fence_context_put); |
128 | } |
129 | |
130 | static int |
131 | nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx) |
132 | { |
133 | struct nouveau_fence *fence; |
134 | int drop = 0; |
135 | u32 seq = fctx->read(chan); |
136 | |
137 | while (!list_empty(head: &fctx->pending)) { |
138 | fence = list_entry(fctx->pending.next, typeof(*fence), head); |
139 | |
140 | if ((int)(seq - fence->base.seqno) < 0) |
141 | break; |
142 | |
143 | drop |= nouveau_fence_signal(fence); |
144 | } |
145 | |
146 | return drop; |
147 | } |
148 | |
149 | static void |
150 | nouveau_fence_uevent_work(struct work_struct *work) |
151 | { |
152 | struct nouveau_fence_chan *fctx = container_of(work, struct nouveau_fence_chan, |
153 | uevent_work); |
154 | unsigned long flags; |
155 | int drop = 0; |
156 | |
157 | spin_lock_irqsave(&fctx->lock, flags); |
158 | if (!list_empty(head: &fctx->pending)) { |
159 | struct nouveau_fence *fence; |
160 | struct nouveau_channel *chan; |
161 | |
162 | fence = list_entry(fctx->pending.next, typeof(*fence), head); |
163 | chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock)); |
164 | if (nouveau_fence_update(chan, fctx)) |
165 | drop = 1; |
166 | } |
167 | if (drop) |
168 | nvif_event_block(&fctx->event); |
169 | |
170 | spin_unlock_irqrestore(lock: &fctx->lock, flags); |
171 | } |
172 | |
173 | static int |
174 | nouveau_fence_wait_uevent_handler(struct nvif_event *event, void *repv, u32 repc) |
175 | { |
176 | struct nouveau_fence_chan *fctx = container_of(event, typeof(*fctx), event); |
177 | schedule_work(work: &fctx->uevent_work); |
178 | return NVIF_EVENT_KEEP; |
179 | } |
180 | |
181 | void |
182 | nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx) |
183 | { |
184 | struct nouveau_fence_priv *priv = (void*)chan->drm->fence; |
185 | struct nouveau_cli *cli = (void *)chan->user.client; |
186 | struct { |
187 | struct nvif_event_v0 base; |
188 | struct nvif_chan_event_v0 host; |
189 | } args; |
190 | int ret; |
191 | |
192 | INIT_WORK(&fctx->uevent_work, nouveau_fence_uevent_work); |
193 | INIT_LIST_HEAD(list: &fctx->flip); |
194 | INIT_LIST_HEAD(list: &fctx->pending); |
195 | spin_lock_init(&fctx->lock); |
196 | fctx->context = chan->drm->runl[chan->runlist].context_base + chan->chid; |
197 | |
198 | if (chan == chan->drm->cechan) |
199 | strcpy(p: fctx->name, q: "copy engine channel" ); |
200 | else if (chan == chan->drm->channel) |
201 | strcpy(p: fctx->name, q: "generic kernel channel" ); |
202 | else |
203 | strcpy(fctx->name, nvxx_client(&cli->base)->name); |
204 | |
205 | kref_init(kref: &fctx->fence_ref); |
206 | if (!priv->uevent) |
207 | return; |
208 | |
209 | args.host.version = 0; |
210 | args.host.type = NVIF_CHAN_EVENT_V0_NON_STALL_INTR; |
211 | |
212 | ret = nvif_event_ctor(&chan->user, "fenceNonStallIntr" , (chan->runlist << 16) | chan->chid, |
213 | nouveau_fence_wait_uevent_handler, false, |
214 | &args.base, sizeof(args), &fctx->event); |
215 | |
216 | WARN_ON(ret); |
217 | } |
218 | |
219 | int |
220 | nouveau_fence_emit(struct nouveau_fence *fence) |
221 | { |
222 | struct nouveau_channel *chan = unrcu_pointer(fence->channel); |
223 | struct nouveau_fence_chan *fctx = chan->fence; |
224 | struct nouveau_fence_priv *priv = (void*)chan->drm->fence; |
225 | int ret; |
226 | |
227 | fence->timeout = jiffies + (15 * HZ); |
228 | |
229 | if (priv->uevent) |
230 | dma_fence_init(fence: &fence->base, ops: &nouveau_fence_ops_uevent, |
231 | lock: &fctx->lock, context: fctx->context, seqno: ++fctx->sequence); |
232 | else |
233 | dma_fence_init(fence: &fence->base, ops: &nouveau_fence_ops_legacy, |
234 | lock: &fctx->lock, context: fctx->context, seqno: ++fctx->sequence); |
235 | kref_get(kref: &fctx->fence_ref); |
236 | |
237 | ret = fctx->emit(fence); |
238 | if (!ret) { |
239 | dma_fence_get(fence: &fence->base); |
240 | spin_lock_irq(lock: &fctx->lock); |
241 | |
242 | if (unlikely(fctx->killed)) { |
243 | spin_unlock_irq(lock: &fctx->lock); |
244 | dma_fence_put(fence: &fence->base); |
245 | return -ENODEV; |
246 | } |
247 | |
248 | if (nouveau_fence_update(chan, fctx)) |
249 | nvif_event_block(&fctx->event); |
250 | |
251 | list_add_tail(new: &fence->head, head: &fctx->pending); |
252 | spin_unlock_irq(lock: &fctx->lock); |
253 | } |
254 | |
255 | return ret; |
256 | } |
257 | |
258 | bool |
259 | nouveau_fence_done(struct nouveau_fence *fence) |
260 | { |
261 | if (fence->base.ops == &nouveau_fence_ops_legacy || |
262 | fence->base.ops == &nouveau_fence_ops_uevent) { |
263 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); |
264 | struct nouveau_channel *chan; |
265 | unsigned long flags; |
266 | |
267 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags)) |
268 | return true; |
269 | |
270 | spin_lock_irqsave(&fctx->lock, flags); |
271 | chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock)); |
272 | if (chan && nouveau_fence_update(chan, fctx)) |
273 | nvif_event_block(&fctx->event); |
274 | spin_unlock_irqrestore(lock: &fctx->lock, flags); |
275 | } |
276 | return dma_fence_is_signaled(fence: &fence->base); |
277 | } |
278 | |
279 | static long |
280 | nouveau_fence_wait_legacy(struct dma_fence *f, bool intr, long wait) |
281 | { |
282 | struct nouveau_fence *fence = from_fence(fence: f); |
283 | unsigned long sleep_time = NSEC_PER_MSEC / 1000; |
284 | unsigned long t = jiffies, timeout = t + wait; |
285 | |
286 | while (!nouveau_fence_done(fence)) { |
287 | ktime_t kt; |
288 | |
289 | t = jiffies; |
290 | |
291 | if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) { |
292 | __set_current_state(TASK_RUNNING); |
293 | return 0; |
294 | } |
295 | |
296 | __set_current_state(intr ? TASK_INTERRUPTIBLE : |
297 | TASK_UNINTERRUPTIBLE); |
298 | |
299 | kt = sleep_time; |
300 | schedule_hrtimeout(expires: &kt, mode: HRTIMER_MODE_REL); |
301 | sleep_time *= 2; |
302 | if (sleep_time > NSEC_PER_MSEC) |
303 | sleep_time = NSEC_PER_MSEC; |
304 | |
305 | if (intr && signal_pending(current)) |
306 | return -ERESTARTSYS; |
307 | } |
308 | |
309 | __set_current_state(TASK_RUNNING); |
310 | |
311 | return timeout - t; |
312 | } |
313 | |
314 | static int |
315 | nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr) |
316 | { |
317 | int ret = 0; |
318 | |
319 | while (!nouveau_fence_done(fence)) { |
320 | if (time_after_eq(jiffies, fence->timeout)) { |
321 | ret = -EBUSY; |
322 | break; |
323 | } |
324 | |
325 | __set_current_state(intr ? |
326 | TASK_INTERRUPTIBLE : |
327 | TASK_UNINTERRUPTIBLE); |
328 | |
329 | if (intr && signal_pending(current)) { |
330 | ret = -ERESTARTSYS; |
331 | break; |
332 | } |
333 | } |
334 | |
335 | __set_current_state(TASK_RUNNING); |
336 | return ret; |
337 | } |
338 | |
339 | int |
340 | nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr) |
341 | { |
342 | long ret; |
343 | |
344 | if (!lazy) |
345 | return nouveau_fence_wait_busy(fence, intr); |
346 | |
347 | ret = dma_fence_wait_timeout(&fence->base, intr, timeout: 15 * HZ); |
348 | if (ret < 0) |
349 | return ret; |
350 | else if (!ret) |
351 | return -EBUSY; |
352 | else |
353 | return 0; |
354 | } |
355 | |
356 | int |
357 | nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, |
358 | bool exclusive, bool intr) |
359 | { |
360 | struct nouveau_fence_chan *fctx = chan->fence; |
361 | struct dma_resv *resv = nvbo->bo.base.resv; |
362 | int i, ret; |
363 | |
364 | ret = dma_resv_reserve_fences(obj: resv, num_fences: 1); |
365 | if (ret) |
366 | return ret; |
367 | |
368 | /* Waiting for the writes first causes performance regressions |
369 | * under some circumstances. So manually wait for the reads first. |
370 | */ |
371 | for (i = 0; i < 2; ++i) { |
372 | struct dma_resv_iter cursor; |
373 | struct dma_fence *fence; |
374 | |
375 | dma_resv_for_each_fence(&cursor, resv, |
376 | dma_resv_usage_rw(exclusive), |
377 | fence) { |
378 | enum dma_resv_usage usage; |
379 | struct nouveau_fence *f; |
380 | |
381 | usage = dma_resv_iter_usage(cursor: &cursor); |
382 | if (i == 0 && usage == DMA_RESV_USAGE_WRITE) |
383 | continue; |
384 | |
385 | f = nouveau_local_fence(fence, drm: chan->drm); |
386 | if (f) { |
387 | struct nouveau_channel *prev; |
388 | bool must_wait = true; |
389 | |
390 | rcu_read_lock(); |
391 | prev = rcu_dereference(f->channel); |
392 | if (prev && (prev == chan || |
393 | fctx->sync(f, prev, chan) == 0)) |
394 | must_wait = false; |
395 | rcu_read_unlock(); |
396 | if (!must_wait) |
397 | continue; |
398 | } |
399 | |
400 | ret = dma_fence_wait(fence, intr); |
401 | if (ret) |
402 | return ret; |
403 | } |
404 | } |
405 | |
406 | return 0; |
407 | } |
408 | |
409 | void |
410 | nouveau_fence_unref(struct nouveau_fence **pfence) |
411 | { |
412 | if (*pfence) |
413 | dma_fence_put(fence: &(*pfence)->base); |
414 | *pfence = NULL; |
415 | } |
416 | |
417 | int |
418 | nouveau_fence_create(struct nouveau_fence **pfence, |
419 | struct nouveau_channel *chan) |
420 | { |
421 | struct nouveau_fence *fence; |
422 | |
423 | if (unlikely(!chan->fence)) |
424 | return -ENODEV; |
425 | |
426 | fence = kzalloc(size: sizeof(*fence), GFP_KERNEL); |
427 | if (!fence) |
428 | return -ENOMEM; |
429 | |
430 | fence->channel = chan; |
431 | |
432 | *pfence = fence; |
433 | return 0; |
434 | } |
435 | |
436 | int |
437 | nouveau_fence_new(struct nouveau_fence **pfence, |
438 | struct nouveau_channel *chan) |
439 | { |
440 | int ret = 0; |
441 | |
442 | ret = nouveau_fence_create(pfence, chan); |
443 | if (ret) |
444 | return ret; |
445 | |
446 | ret = nouveau_fence_emit(fence: *pfence); |
447 | if (ret) |
448 | nouveau_fence_unref(pfence); |
449 | |
450 | return ret; |
451 | } |
452 | |
453 | static const char *nouveau_fence_get_get_driver_name(struct dma_fence *fence) |
454 | { |
455 | return "nouveau" ; |
456 | } |
457 | |
458 | static const char *nouveau_fence_get_timeline_name(struct dma_fence *f) |
459 | { |
460 | struct nouveau_fence *fence = from_fence(fence: f); |
461 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); |
462 | |
463 | return !fctx->dead ? fctx->name : "dead channel" ; |
464 | } |
465 | |
466 | /* |
467 | * In an ideal world, read would not assume the channel context is still alive. |
468 | * This function may be called from another device, running into free memory as a |
469 | * result. The drm node should still be there, so we can derive the index from |
470 | * the fence context. |
471 | */ |
472 | static bool nouveau_fence_is_signaled(struct dma_fence *f) |
473 | { |
474 | struct nouveau_fence *fence = from_fence(fence: f); |
475 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); |
476 | struct nouveau_channel *chan; |
477 | bool ret = false; |
478 | |
479 | rcu_read_lock(); |
480 | chan = rcu_dereference(fence->channel); |
481 | if (chan) |
482 | ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0; |
483 | rcu_read_unlock(); |
484 | |
485 | return ret; |
486 | } |
487 | |
488 | static bool nouveau_fence_no_signaling(struct dma_fence *f) |
489 | { |
490 | struct nouveau_fence *fence = from_fence(fence: f); |
491 | |
492 | /* |
493 | * caller should have a reference on the fence, |
494 | * else fence could get freed here |
495 | */ |
496 | WARN_ON(kref_read(&fence->base.refcount) <= 1); |
497 | |
498 | /* |
499 | * This needs uevents to work correctly, but dma_fence_add_callback relies on |
500 | * being able to enable signaling. It will still get signaled eventually, |
501 | * just not right away. |
502 | */ |
503 | if (nouveau_fence_is_signaled(f)) { |
504 | list_del(entry: &fence->head); |
505 | |
506 | dma_fence_put(fence: &fence->base); |
507 | return false; |
508 | } |
509 | |
510 | return true; |
511 | } |
512 | |
513 | static void nouveau_fence_release(struct dma_fence *f) |
514 | { |
515 | struct nouveau_fence *fence = from_fence(fence: f); |
516 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); |
517 | |
518 | kref_put(kref: &fctx->fence_ref, release: nouveau_fence_context_put); |
519 | dma_fence_free(fence: &fence->base); |
520 | } |
521 | |
522 | static const struct dma_fence_ops nouveau_fence_ops_legacy = { |
523 | .get_driver_name = nouveau_fence_get_get_driver_name, |
524 | .get_timeline_name = nouveau_fence_get_timeline_name, |
525 | .enable_signaling = nouveau_fence_no_signaling, |
526 | .signaled = nouveau_fence_is_signaled, |
527 | .wait = nouveau_fence_wait_legacy, |
528 | .release = nouveau_fence_release |
529 | }; |
530 | |
531 | static bool nouveau_fence_enable_signaling(struct dma_fence *f) |
532 | { |
533 | struct nouveau_fence *fence = from_fence(fence: f); |
534 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); |
535 | bool ret; |
536 | |
537 | if (!fctx->notify_ref++) |
538 | nvif_event_allow(&fctx->event); |
539 | |
540 | ret = nouveau_fence_no_signaling(f); |
541 | if (ret) |
542 | set_bit(nr: DMA_FENCE_FLAG_USER_BITS, addr: &fence->base.flags); |
543 | else if (!--fctx->notify_ref) |
544 | nvif_event_block(&fctx->event); |
545 | |
546 | return ret; |
547 | } |
548 | |
549 | static const struct dma_fence_ops nouveau_fence_ops_uevent = { |
550 | .get_driver_name = nouveau_fence_get_get_driver_name, |
551 | .get_timeline_name = nouveau_fence_get_timeline_name, |
552 | .enable_signaling = nouveau_fence_enable_signaling, |
553 | .signaled = nouveau_fence_is_signaled, |
554 | .release = nouveau_fence_release |
555 | }; |
556 | |