1	/*
2	* Copyright 2017 Red Hat
3	* Parts ported from amdgpu (fence wait code).
4	* Copyright 2016 Advanced Micro Devices, Inc.
5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a
7	* copy of this software and associated documentation files (the "Software"),
8	* to deal in the Software without restriction, including without limitation
9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
10	* and/or sell copies of the Software, and to permit persons to whom the
11	* Software is furnished to do so, subject to the following conditions:
12	*
13	* The above copyright notice and this permission notice (including the next
14	* paragraph) shall be included in all copies or substantial portions of the
15	* Software.
16	*
17	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23	* IN THE SOFTWARE.
24	*
25	* Authors:
26	*
27	*/
28
29	/**
30	* DOC: Overview
31	*
32	* DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a
33	* container for a synchronization primitive which can be used by userspace
34	* to explicitly synchronize GPU commands, can be shared between userspace
35	* processes, and can be shared between different DRM drivers.
36	* Their primary use-case is to implement Vulkan fences and semaphores.
37	* The syncobj userspace API provides ioctls for several operations:
38	*
39	* - Creation and destruction of syncobjs
40	* - Import and export of syncobjs to/from a syncobj file descriptor
41	* - Import and export a syncobj's underlying fence to/from a sync file
42	* - Reset a syncobj (set its fence to NULL)
43	* - Signal a syncobj (set a trivially signaled fence)
44	* - Wait for a syncobj's fence to appear and be signaled
45	*
46	* The syncobj userspace API also provides operations to manipulate a syncobj
47	* in terms of a timeline of struct &dma_fence_chain rather than a single
48	* struct &dma_fence, through the following operations:
49	*
50	* - Signal a given point on the timeline
51	* - Wait for a given point to appear and/or be signaled
52	* - Import and export from/to a given point of a timeline
53	*
54	* At it's core, a syncobj is simply a wrapper around a pointer to a struct
55	* &dma_fence which may be NULL.
56	* When a syncobj is first created, its pointer is either NULL or a pointer
57	* to an already signaled fence depending on whether the
58	* &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to
59	* &DRM_IOCTL_SYNCOBJ_CREATE.
60	*
61	* If the syncobj is considered as a binary (its state is either signaled or
62	* unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal
63	* the syncobj, the syncobj's fence is replaced with a fence which will be
64	* signaled by the completion of that work.
65	* If the syncobj is considered as a timeline primitive, when GPU work is
66	* enqueued in a DRM driver to signal the a given point of the syncobj, a new
67	* struct &dma_fence_chain pointing to the DRM driver's fence and also
68	* pointing to the previous fence that was in the syncobj. The new struct
69	* &dma_fence_chain fence replace the syncobj's fence and will be signaled by
70	* completion of the DRM driver's work and also any work associated with the
71	* fence previously in the syncobj.
72	*
73	* When GPU work which waits on a syncobj is enqueued in a DRM driver, at the
74	* time the work is enqueued, it waits on the syncobj's fence before
75	* submitting the work to hardware. That fence is either :
76	*
77	* - The syncobj's current fence if the syncobj is considered as a binary
78	* primitive.
79	* - The struct &dma_fence associated with a given point if the syncobj is
80	* considered as a timeline primitive.
81	*
82	* If the syncobj's fence is NULL or not present in the syncobj's timeline,
83	* the enqueue operation is expected to fail.
84	*
85	* With binary syncobj, all manipulation of the syncobjs's fence happens in
86	* terms of the current fence at the time the ioctl is called by userspace
87	* regardless of whether that operation is an immediate host-side operation
88	* (signal or reset) or or an operation which is enqueued in some driver
89	* queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used
90	* to manipulate a syncobj from the host by resetting its pointer to NULL or
91	* setting its pointer to a fence which is already signaled.
92	*
93	* With a timeline syncobj, all manipulation of the synobj's fence happens in
94	* terms of a u64 value referring to point in the timeline. See
95	* dma_fence_chain_find_seqno() to see how a given point is found in the
96	* timeline.
97	*
98	* Note that applications should be careful to always use timeline set of
99	* ioctl() when dealing with syncobj considered as timeline. Using a binary
100	* set of ioctl() with a syncobj considered as timeline could result incorrect
101	* synchronization. The use of binary syncobj is supported through the
102	* timeline set of ioctl() by using a point value of 0, this will reproduce
103	* the behavior of the binary set of ioctl() (for example replace the
104	* syncobj's fence when signaling).
105	*
106	*
107	* Host-side wait on syncobjs
108	* --------------------------
109	*
110	* &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a
111	* host-side wait on all of the syncobj fences simultaneously.
112	* If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on
113	* all of the syncobj fences to be signaled before it returns.
114	* Otherwise, it returns once at least one syncobj fence has been signaled
115	* and the index of a signaled fence is written back to the client.
116	*
117	* Unlike the enqueued GPU work dependencies which fail if they see a NULL
118	* fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set,
119	* the host-side wait will first wait for the syncobj to receive a non-NULL
120	* fence and then wait on that fence.
121	* If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the
122	* syncobjs in the array has a NULL fence, -EINVAL will be returned.
123	* Assuming the syncobj starts off with a NULL fence, this allows a client
124	* to do a host wait in one thread (or process) which waits on GPU work
125	* submitted in another thread (or process) without having to manually
126	* synchronize between the two.
127	* This requirement is inherited from the Vulkan fence API.
128	*
129	* Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj
130	* handles as well as an array of u64 points and does a host-side wait on all
131	* of syncobj fences at the given points simultaneously.
132	*
133	* &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given
134	* fence to materialize on the timeline without waiting for the fence to be
135	* signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This
136	* requirement is inherited from the wait-before-signal behavior required by
137	* the Vulkan timeline semaphore API.
138	*
139	* Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without
140	* blocking: an eventfd will be signaled when the syncobj is. This is useful to
141	* integrate the wait in an event loop.
142	*
143	*
144	* Import/export of syncobjs
145	* -------------------------
146	*
147	* &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD
148	* provide two mechanisms for import/export of syncobjs.
149	*
150	* The first lets the client import or export an entire syncobj to a file
151	* descriptor.
152	* These fd's are opaque and have no other use case, except passing the
153	* syncobj between processes.
154	* All exported file descriptors and any syncobj handles created as a
155	* result of importing those file descriptors own a reference to the
156	* same underlying struct &drm_syncobj and the syncobj can be used
157	* persistently across all the processes with which it is shared.
158	* The syncobj is freed only once the last reference is dropped.
159	* Unlike dma-buf, importing a syncobj creates a new handle (with its own
160	* reference) for every import instead of de-duplicating.
161	* The primary use-case of this persistent import/export is for shared
162	* Vulkan fences and semaphores.
163	*
164	* The second import/export mechanism, which is indicated by
165	* &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or
166	* &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client
167	* import/export the syncobj's current fence from/to a &sync_file.
168	* When a syncobj is exported to a sync file, that sync file wraps the
169	* sycnobj's fence at the time of export and any later signal or reset
170	* operations on the syncobj will not affect the exported sync file.
171	* When a sync file is imported into a syncobj, the syncobj's fence is set
172	* to the fence wrapped by that sync file.
173	* Because sync files are immutable, resetting or signaling the syncobj
174	* will not affect any sync files whose fences have been imported into the
175	* syncobj.
176	*
177	*
178	* Import/export of timeline points in timeline syncobjs
179	* -----------------------------------------------------
180	*
181	* &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct
182	* &dma_fence_chain of a syncobj at a given u64 point to another u64 point
183	* into another syncobj.
184	*
185	* Note that if you want to transfer a struct &dma_fence_chain from a given
186	* point on a timeline syncobj from/into a binary syncobj, you can use the
187	* point 0 to mean take/replace the fence in the syncobj.
188	*/
189
190	#include <linux/anon_inodes.h>
191	#include <linux/dma-fence-unwrap.h>
192	#include <linux/eventfd.h>
193	#include <linux/file.h>
194	#include <linux/fs.h>
195	#include <linux/sched/signal.h>
196	#include <linux/sync_file.h>
197	#include <linux/uaccess.h>
198
199	#include <drm/drm.h>
200	#include <drm/drm_drv.h>
201	#include <drm/drm_file.h>
202	#include <drm/drm_gem.h>
203	#include <drm/drm_print.h>
204	#include <drm/drm_syncobj.h>
205	#include <drm/drm_utils.h>
206
207	#include "drm_internal.h"
208
209	struct syncobj_wait_entry {
210	struct list_head node;
211	struct task_struct *task;
212	struct dma_fence *fence;
213	struct dma_fence_cb fence_cb;
214	u64 point;
215	};
216
217	static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
218	struct syncobj_wait_entry *wait);
219
220	struct syncobj_eventfd_entry {
221	struct list_head node;
222	struct dma_fence *fence;
223	struct dma_fence_cb fence_cb;
224	struct drm_syncobj *syncobj;
225	struct eventfd_ctx *ev_fd_ctx;
226	u64 point;
227	u32 flags;
228	};
229
230	static void
231	syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
232	struct syncobj_eventfd_entry *entry);
233
234	/**
235	* drm_syncobj_find - lookup and reference a sync object.
236	* @file_private: drm file private pointer
237	* @handle: sync object handle to lookup.
238	*
239	* Returns a reference to the syncobj pointed to by handle or NULL. The
240	* reference must be released by calling drm_syncobj_put().
241	*/
242	struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
243	u32 handle)
244	{
245	struct drm_syncobj *syncobj;
246
247	spin_lock(lock: &file_private->syncobj_table_lock);
248
249	/* Check if we currently have a reference on the object */
250	syncobj = idr_find(&file_private->syncobj_idr, id: handle);
251	if (syncobj)
252	drm_syncobj_get(obj: syncobj);
253
254	spin_unlock(lock: &file_private->syncobj_table_lock);
255
256	return syncobj;
257	}
258	EXPORT_SYMBOL(drm_syncobj_find);
259
260	static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
261	struct syncobj_wait_entry *wait)
262	{
263	struct dma_fence *fence;
264
265	if (wait->fence)
266	return;
267
268	spin_lock(lock: &syncobj->lock);
269	/* We've already tried once to get a fence and failed. Now that we
270	* have the lock, try one more time just to be sure we don't add a
271	* callback when a fence has already been set.
272	*/
273	fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
274	if (!fence || dma_fence_chain_find_seqno(pfence: &fence, seqno: wait->point)) {
275	dma_fence_put(fence);
276	list_add_tail(new: &wait->node, head: &syncobj->cb_list);
277	} else if (!fence) {
278	wait->fence = dma_fence_get_stub();
279	} else {
280	wait->fence = fence;
281	}
282	spin_unlock(lock: &syncobj->lock);
283	}
284
285	static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
286	struct syncobj_wait_entry *wait)
287	{
288	if (!wait->node.next)
289	return;
290
291	spin_lock(lock: &syncobj->lock);
292	list_del_init(entry: &wait->node);
293	spin_unlock(lock: &syncobj->lock);
294	}
295
296	static void
297	syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry)
298	{
299	eventfd_ctx_put(ctx: entry->ev_fd_ctx);
300	dma_fence_put(fence: entry->fence);
301	/* This happens either inside the syncobj lock, or after the node has
302	* already been removed from the list.
303	*/
304	list_del(entry: &entry->node);
305	kfree(objp: entry);
306	}
307
308	static void
309	drm_syncobj_add_eventfd(struct drm_syncobj *syncobj,
310	struct syncobj_eventfd_entry *entry)
311	{
312	spin_lock(lock: &syncobj->lock);
313	list_add_tail(new: &entry->node, head: &syncobj->ev_fd_list);
314	syncobj_eventfd_entry_func(syncobj, entry);
315	spin_unlock(lock: &syncobj->lock);
316	}
317
318	/**
319	* drm_syncobj_add_point - add new timeline point to the syncobj
320	* @syncobj: sync object to add timeline point do
321	* @chain: chain node to use to add the point
322	* @fence: fence to encapsulate in the chain node
323	* @point: sequence number to use for the point
324	*
325	* Add the chain node as new timeline point to the syncobj.
326	*/
327	void drm_syncobj_add_point(struct drm_syncobj *syncobj,
328	struct dma_fence_chain *chain,
329	struct dma_fence *fence,
330	uint64_t point)
331	{
332	struct syncobj_wait_entry *wait_cur, *wait_tmp;
333	struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
334	struct dma_fence *prev;
335
336	dma_fence_get(fence);
337
338	spin_lock(lock: &syncobj->lock);
339
340	prev = drm_syncobj_fence_get(syncobj);
341	/* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
342	if (prev && prev->seqno >= point)
343	DRM_DEBUG("You are adding an unorder point to timeline!\n");
344	dma_fence_chain_init(chain, prev, fence, seqno: point);
345	rcu_assign_pointer(syncobj->fence, &chain->base);
346
347	list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
348	syncobj_wait_syncobj_func(syncobj, wait: wait_cur);
349	list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
350	syncobj_eventfd_entry_func(syncobj, entry: ev_fd_cur);
351	spin_unlock(lock: &syncobj->lock);
352
353	/* Walk the chain once to trigger garbage collection */
354	dma_fence_chain_for_each(fence, prev);
355	dma_fence_put(fence: prev);
356	}
357	EXPORT_SYMBOL(drm_syncobj_add_point);
358
359	/**
360	* drm_syncobj_replace_fence - replace fence in a sync object.
361	* @syncobj: Sync object to replace fence in
362	* @fence: fence to install in sync file.
363	*
364	* This replaces the fence on a sync object.
365	*/
366	void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
367	struct dma_fence *fence)
368	{
369	struct dma_fence *old_fence;
370	struct syncobj_wait_entry *wait_cur, *wait_tmp;
371	struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
372
373	if (fence)
374	dma_fence_get(fence);
375
376	spin_lock(lock: &syncobj->lock);
377
378	old_fence = rcu_dereference_protected(syncobj->fence,
379	lockdep_is_held(&syncobj->lock));
380	rcu_assign_pointer(syncobj->fence, fence);
381
382	if (fence != old_fence) {
383	list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
384	syncobj_wait_syncobj_func(syncobj, wait: wait_cur);
385	list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
386	syncobj_eventfd_entry_func(syncobj, entry: ev_fd_cur);
387	}
388
389	spin_unlock(lock: &syncobj->lock);
390
391	dma_fence_put(fence: old_fence);
392	}
393	EXPORT_SYMBOL(drm_syncobj_replace_fence);
394
395	/**
396	* drm_syncobj_assign_null_handle - assign a stub fence to the sync object
397	* @syncobj: sync object to assign the fence on
398	*
399	* Assign a already signaled stub fence to the sync object.
400	*/
401	static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
402	{
403	struct dma_fence *fence = dma_fence_allocate_private_stub(timestamp: ktime_get());
404
405	if (!fence)
406	return -ENOMEM;
407
408	drm_syncobj_replace_fence(syncobj, fence);
409	dma_fence_put(fence);
410	return 0;
411	}
412
413	/* 5s default for wait submission */
414	#define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
415	/**
416	* drm_syncobj_find_fence - lookup and reference the fence in a sync object
417	* @file_private: drm file private pointer
418	* @handle: sync object handle to lookup.
419	* @point: timeline point
420	* @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
421	* @fence: out parameter for the fence
422	*
423	* This is just a convenience function that combines drm_syncobj_find() and
424	* drm_syncobj_fence_get().
425	*
426	* Returns 0 on success or a negative error value on failure. On success @fence
427	* contains a reference to the fence, which must be released by calling
428	* dma_fence_put().
429	*/
430	int drm_syncobj_find_fence(struct drm_file *file_private,
431	u32 handle, u64 point, u64 flags,
432	struct dma_fence **fence)
433	{
434	struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
435	struct syncobj_wait_entry wait;
436	u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
437	int ret;
438
439	if (!syncobj)
440	return -ENOENT;
441
442	/* Waiting for userspace with locks help is illegal cause that can
443	* trivial deadlock with page faults for example. Make lockdep complain
444	* about it early on.
445	*/
446	if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
447	might_sleep();
448	lockdep_assert_none_held_once();
449	}
450
451	*fence = drm_syncobj_fence_get(syncobj);
452
453	if (*fence) {
454	ret = dma_fence_chain_find_seqno(pfence: fence, seqno: point);
455	if (!ret) {
456	/* If the requested seqno is already signaled
457	* drm_syncobj_find_fence may return a NULL
458	* fence. To make sure the recipient gets
459	* signalled, use a new fence instead.
460	*/
461	if (!*fence)
462	*fence = dma_fence_get_stub();
463
464	goto out;
465	}
466	dma_fence_put(fence: *fence);
467	} else {
468	ret = -EINVAL;
469	}
470
471	if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
472	goto out;
473
474	memset(&wait, 0, sizeof(wait));
475	wait.task = current;
476	wait.point = point;
477	drm_syncobj_fence_add_wait(syncobj, wait: &wait);
478
479	do {
480	set_current_state(TASK_INTERRUPTIBLE);
481	if (wait.fence) {
482	ret = 0;
483	break;
484	}
485	if (timeout == 0) {
486	ret = -ETIME;
487	break;
488	}
489
490	if (signal_pending(current)) {
491	ret = -ERESTARTSYS;
492	break;
493	}
494
495	timeout = schedule_timeout(timeout);
496	} while (1);
497
498	__set_current_state(TASK_RUNNING);
499	*fence = wait.fence;
500
501	if (wait.node.next)
502	drm_syncobj_remove_wait(syncobj, wait: &wait);
503
504	out:
505	drm_syncobj_put(obj: syncobj);
506
507	return ret;
508	}
509	EXPORT_SYMBOL(drm_syncobj_find_fence);
510
511	/**
512	* drm_syncobj_free - free a sync object.
513	* @kref: kref to free.
514	*
515	* Only to be called from kref_put in drm_syncobj_put.
516	*/
517	void drm_syncobj_free(struct kref *kref)
518	{
519	struct drm_syncobj *syncobj = container_of(kref,
520	struct drm_syncobj,
521	refcount);
522	struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
523
524	drm_syncobj_replace_fence(syncobj, NULL);
525
526	list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
527	syncobj_eventfd_entry_free(entry: ev_fd_cur);
528
529	kfree(objp: syncobj);
530	}
531	EXPORT_SYMBOL(drm_syncobj_free);
532
533	/**
534	* drm_syncobj_create - create a new syncobj
535	* @out_syncobj: returned syncobj
536	* @flags: DRM_SYNCOBJ_* flags
537	* @fence: if non-NULL, the syncobj will represent this fence
538	*
539	* This is the first function to create a sync object. After creating, drivers
540	* probably want to make it available to userspace, either through
541	* drm_syncobj_get_handle() or drm_syncobj_get_fd().
542	*
543	* Returns 0 on success or a negative error value on failure.
544	*/
545	int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
546	struct dma_fence *fence)
547	{
548	int ret;
549	struct drm_syncobj *syncobj;
550
551	syncobj = kzalloc(size: sizeof(struct drm_syncobj), GFP_KERNEL);
552	if (!syncobj)
553	return -ENOMEM;
554
555	kref_init(kref: &syncobj->refcount);
556	INIT_LIST_HEAD(list: &syncobj->cb_list);
557	INIT_LIST_HEAD(list: &syncobj->ev_fd_list);
558	spin_lock_init(&syncobj->lock);
559
560	if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
561	ret = drm_syncobj_assign_null_handle(syncobj);
562	if (ret < 0) {
563	drm_syncobj_put(obj: syncobj);
564	return ret;
565	}
566	}
567
568	if (fence)
569	drm_syncobj_replace_fence(syncobj, fence);
570
571	*out_syncobj = syncobj;
572	return 0;
573	}
574	EXPORT_SYMBOL(drm_syncobj_create);
575
576	/**
577	* drm_syncobj_get_handle - get a handle from a syncobj
578	* @file_private: drm file private pointer
579	* @syncobj: Sync object to export
580	* @handle: out parameter with the new handle
581	*
582	* Exports a sync object created with drm_syncobj_create() as a handle on
583	* @file_private to userspace.
584	*
585	* Returns 0 on success or a negative error value on failure.
586	*/
587	int drm_syncobj_get_handle(struct drm_file *file_private,
588	struct drm_syncobj *syncobj, u32 *handle)
589	{
590	int ret;
591
592	/* take a reference to put in the idr */
593	drm_syncobj_get(obj: syncobj);
594
595	idr_preload(GFP_KERNEL);
596	spin_lock(lock: &file_private->syncobj_table_lock);
597	ret = idr_alloc(&file_private->syncobj_idr, ptr: syncobj, start: 1, end: 0, GFP_NOWAIT);
598	spin_unlock(lock: &file_private->syncobj_table_lock);
599
600	idr_preload_end();
601
602	if (ret < 0) {
603	drm_syncobj_put(obj: syncobj);
604	return ret;
605	}
606
607	*handle = ret;
608	return 0;
609	}
610	EXPORT_SYMBOL(drm_syncobj_get_handle);
611
612	static int drm_syncobj_create_as_handle(struct drm_file *file_private,
613	u32 *handle, uint32_t flags)
614	{
615	int ret;
616	struct drm_syncobj *syncobj;
617
618	ret = drm_syncobj_create(&syncobj, flags, NULL);
619	if (ret)
620	return ret;
621
622	ret = drm_syncobj_get_handle(file_private, syncobj, handle);
623	drm_syncobj_put(obj: syncobj);
624	return ret;
625	}
626
627	static int drm_syncobj_destroy(struct drm_file *file_private,
628	u32 handle)
629	{
630	struct drm_syncobj *syncobj;
631
632	spin_lock(lock: &file_private->syncobj_table_lock);
633	syncobj = idr_remove(&file_private->syncobj_idr, id: handle);
634	spin_unlock(lock: &file_private->syncobj_table_lock);
635
636	if (!syncobj)
637	return -EINVAL;
638
639	drm_syncobj_put(obj: syncobj);
640	return 0;
641	}
642
643	static int drm_syncobj_file_release(struct inode *inode, struct file *file)
644	{
645	struct drm_syncobj *syncobj = file->private_data;
646
647	drm_syncobj_put(obj: syncobj);
648	return 0;
649	}
650
651	static const struct file_operations drm_syncobj_file_fops = {
652	.release = drm_syncobj_file_release,
653	};
654
655	/**
656	* drm_syncobj_get_fd - get a file descriptor from a syncobj
657	* @syncobj: Sync object to export
658	* @p_fd: out parameter with the new file descriptor
659	*
660	* Exports a sync object created with drm_syncobj_create() as a file descriptor.
661	*
662	* Returns 0 on success or a negative error value on failure.
663	*/
664	int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
665	{
666	struct file *file;
667	int fd;
668
669	fd = get_unused_fd_flags(O_CLOEXEC);
670	if (fd < 0)
671	return fd;
672
673	file = anon_inode_getfile(name: "syncobj_file",
674	fops: &drm_syncobj_file_fops,
675	priv: syncobj, flags: 0);
676	if (IS_ERR(ptr: file)) {
677	put_unused_fd(fd);
678	return PTR_ERR(ptr: file);
679	}
680
681	drm_syncobj_get(obj: syncobj);
682	fd_install(fd, file);
683
684	*p_fd = fd;
685	return 0;
686	}
687	EXPORT_SYMBOL(drm_syncobj_get_fd);
688
689	static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
690	u32 handle, int *p_fd)
691	{
692	struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
693	int ret;
694
695	if (!syncobj)
696	return -EINVAL;
697
698	ret = drm_syncobj_get_fd(syncobj, p_fd);
699	drm_syncobj_put(obj: syncobj);
700	return ret;
701	}
702
703	static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
704	int fd, u32 *handle)
705	{
706	struct drm_syncobj *syncobj;
707	struct fd f = fdget(fd);
708	int ret;
709
710	if (!f.file)
711	return -EINVAL;
712
713	if (f.file->f_op != &drm_syncobj_file_fops) {
714	fdput(fd: f);
715	return -EINVAL;
716	}
717
718	/* take a reference to put in the idr */
719	syncobj = f.file->private_data;
720	drm_syncobj_get(obj: syncobj);
721
722	idr_preload(GFP_KERNEL);
723	spin_lock(lock: &file_private->syncobj_table_lock);
724	ret = idr_alloc(&file_private->syncobj_idr, ptr: syncobj, start: 1, end: 0, GFP_NOWAIT);
725	spin_unlock(lock: &file_private->syncobj_table_lock);
726	idr_preload_end();
727
728	if (ret > 0) {
729	*handle = ret;
730	ret = 0;
731	} else
732	drm_syncobj_put(obj: syncobj);
733
734	fdput(fd: f);
735	return ret;
736	}
737
738	static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
739	int fd, int handle)
740	{
741	struct dma_fence *fence = sync_file_get_fence(fd);
742	struct drm_syncobj *syncobj;
743
744	if (!fence)
745	return -EINVAL;
746
747	syncobj = drm_syncobj_find(file_private, handle);
748	if (!syncobj) {
749	dma_fence_put(fence);
750	return -ENOENT;
751	}
752
753	drm_syncobj_replace_fence(syncobj, fence);
754	dma_fence_put(fence);
755	drm_syncobj_put(obj: syncobj);
756	return 0;
757	}
758
759	static int drm_syncobj_export_sync_file(struct drm_file *file_private,
760	int handle, int *p_fd)
761	{
762	int ret;
763	struct dma_fence *fence;
764	struct sync_file *sync_file;
765	int fd = get_unused_fd_flags(O_CLOEXEC);
766
767	if (fd < 0)
768	return fd;
769
770	ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
771	if (ret)
772	goto err_put_fd;
773
774	sync_file = sync_file_create(fence);
775
776	dma_fence_put(fence);
777
778	if (!sync_file) {
779	ret = -EINVAL;
780	goto err_put_fd;
781	}
782
783	fd_install(fd, file: sync_file->file);
784
785	*p_fd = fd;
786	return 0;
787	err_put_fd:
788	put_unused_fd(fd);
789	return ret;
790	}
791	/**
792	* drm_syncobj_open - initializes syncobj file-private structures at devnode open time
793	* @file_private: drm file-private structure to set up
794	*
795	* Called at device open time, sets up the structure for handling refcounting
796	* of sync objects.
797	*/
798	void
799	drm_syncobj_open(struct drm_file *file_private)
800	{
801	idr_init_base(idr: &file_private->syncobj_idr, base: 1);
802	spin_lock_init(&file_private->syncobj_table_lock);
803	}
804
805	static int
806	drm_syncobj_release_handle(int id, void *ptr, void *data)
807	{
808	struct drm_syncobj *syncobj = ptr;
809
810	drm_syncobj_put(obj: syncobj);
811	return 0;
812	}
813
814	/**
815	* drm_syncobj_release - release file-private sync object resources
816	* @file_private: drm file-private structure to clean up
817	*
818	* Called at close time when the filp is going away.
819	*
820	* Releases any remaining references on objects by this filp.
821	*/
822	void
823	drm_syncobj_release(struct drm_file *file_private)
824	{
825	idr_for_each(&file_private->syncobj_idr,
826	fn: &drm_syncobj_release_handle, data: file_private);
827	idr_destroy(&file_private->syncobj_idr);
828	}
829
830	int
831	drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
832	struct drm_file *file_private)
833	{
834	struct drm_syncobj_create *args = data;
835
836	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
837	return -EOPNOTSUPP;
838
839	/* no valid flags yet */
840	if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
841	return -EINVAL;
842
843	return drm_syncobj_create_as_handle(file_private,
844	handle: &args->handle, flags: args->flags);
845	}
846
847	int
848	drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
849	struct drm_file *file_private)
850	{
851	struct drm_syncobj_destroy *args = data;
852
853	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
854	return -EOPNOTSUPP;
855
856	/* make sure padding is empty */
857	if (args->pad)
858	return -EINVAL;
859	return drm_syncobj_destroy(file_private, handle: args->handle);
860	}
861
862	int
863	drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
864	struct drm_file *file_private)
865	{
866	struct drm_syncobj_handle *args = data;
867
868	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
869	return -EOPNOTSUPP;
870
871	if (args->pad)
872	return -EINVAL;
873
874	if (args->flags != 0 &&
875	args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
876	return -EINVAL;
877
878	if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
879	return drm_syncobj_export_sync_file(file_private, handle: args->handle,
880	p_fd: &args->fd);
881
882	return drm_syncobj_handle_to_fd(file_private, handle: args->handle,
883	p_fd: &args->fd);
884	}
885
886	int
887	drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
888	struct drm_file *file_private)
889	{
890	struct drm_syncobj_handle *args = data;
891
892	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
893	return -EOPNOTSUPP;
894
895	if (args->pad)
896	return -EINVAL;
897
898	if (args->flags != 0 &&
899	args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
900	return -EINVAL;
901
902	if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
903	return drm_syncobj_import_sync_file_fence(file_private,
904	fd: args->fd,
905	handle: args->handle);
906
907	return drm_syncobj_fd_to_handle(file_private, fd: args->fd,
908	handle: &args->handle);
909	}
910
911	static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
912	struct drm_syncobj_transfer *args)
913	{
914	struct drm_syncobj *timeline_syncobj = NULL;
915	struct dma_fence *fence, *tmp;
916	struct dma_fence_chain *chain;
917	int ret;
918
919	timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
920	if (!timeline_syncobj) {
921	return -ENOENT;
922	}
923	ret = drm_syncobj_find_fence(file_private, args->src_handle,
924	args->src_point, args->flags,
925	&tmp);
926	if (ret)
927	goto err_put_timeline;
928
929	fence = dma_fence_unwrap_merge(tmp);
930	dma_fence_put(fence: tmp);
931	if (!fence) {
932	ret = -ENOMEM;
933	goto err_put_timeline;
934	}
935
936	chain = dma_fence_chain_alloc();
937	if (!chain) {
938	ret = -ENOMEM;
939	goto err_free_fence;
940	}
941
942	drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
943	err_free_fence:
944	dma_fence_put(fence);
945	err_put_timeline:
946	drm_syncobj_put(obj: timeline_syncobj);
947
948	return ret;
949	}
950
951	static int
952	drm_syncobj_transfer_to_binary(struct drm_file *file_private,
953	struct drm_syncobj_transfer *args)
954	{
955	struct drm_syncobj *binary_syncobj = NULL;
956	struct dma_fence *fence;
957	int ret;
958
959	binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
960	if (!binary_syncobj)
961	return -ENOENT;
962	ret = drm_syncobj_find_fence(file_private, args->src_handle,
963	args->src_point, args->flags, &fence);
964	if (ret)
965	goto err;
966	drm_syncobj_replace_fence(binary_syncobj, fence);
967	dma_fence_put(fence);
968	err:
969	drm_syncobj_put(obj: binary_syncobj);
970
971	return ret;
972	}
973	int
974	drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
975	struct drm_file *file_private)
976	{
977	struct drm_syncobj_transfer *args = data;
978	int ret;
979
980	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ_TIMELINE))
981	return -EOPNOTSUPP;
982
983	if (args->pad)
984	return -EINVAL;
985
986	if (args->dst_point)
987	ret = drm_syncobj_transfer_to_timeline(file_private, args);
988	else
989	ret = drm_syncobj_transfer_to_binary(file_private, args);
990
991	return ret;
992	}
993
994	static void syncobj_wait_fence_func(struct dma_fence *fence,
995	struct dma_fence_cb *cb)
996	{
997	struct syncobj_wait_entry *wait =
998	container_of(cb, struct syncobj_wait_entry, fence_cb);
999
1000	wake_up_process(tsk: wait->task);
1001	}
1002
1003	static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
1004	struct syncobj_wait_entry *wait)
1005	{
1006	struct dma_fence *fence;
1007
1008	/* This happens inside the syncobj lock */
1009	fence = rcu_dereference_protected(syncobj->fence,
1010	lockdep_is_held(&syncobj->lock));
1011	dma_fence_get(fence);
1012	if (!fence || dma_fence_chain_find_seqno(pfence: &fence, seqno: wait->point)) {
1013	dma_fence_put(fence);
1014	return;
1015	} else if (!fence) {
1016	wait->fence = dma_fence_get_stub();
1017	} else {
1018	wait->fence = fence;
1019	}
1020
1021	wake_up_process(tsk: wait->task);
1022	list_del_init(entry: &wait->node);
1023	}
1024
1025	static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
1026	void __user *user_points,
1027	uint32_t count,
1028	uint32_t flags,
1029	signed long timeout,
1030	uint32_t *idx)
1031	{
1032	struct syncobj_wait_entry *entries;
1033	struct dma_fence *fence;
1034	uint64_t *points;
1035	uint32_t signaled_count, i;
1036
1037	if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)
1038	lockdep_assert_none_held_once();
1039
1040	points = kmalloc_array(n: count, size: sizeof(*points), GFP_KERNEL);
1041	if (points == NULL)
1042	return -ENOMEM;
1043
1044	if (!user_points) {
1045	memset(points, 0, count * sizeof(uint64_t));
1046
1047	} else if (copy_from_user(to: points, from: user_points,
1048	n: sizeof(uint64_t) * count)) {
1049	timeout = -EFAULT;
1050	goto err_free_points;
1051	}
1052
1053	entries = kcalloc(n: count, size: sizeof(*entries), GFP_KERNEL);
1054	if (!entries) {
1055	timeout = -ENOMEM;
1056	goto err_free_points;
1057	}
1058	/* Walk the list of sync objects and initialize entries. We do
1059	* this up-front so that we can properly return -EINVAL if there is
1060	* a syncobj with a missing fence and then never have the chance of
1061	* returning -EINVAL again.
1062	*/
1063	signaled_count = 0;
1064	for (i = 0; i < count; ++i) {
1065	struct dma_fence *fence;
1066
1067	entries[i].task = current;
1068	entries[i].point = points[i];
1069	fence = drm_syncobj_fence_get(syncobj: syncobjs[i]);
1070	if (!fence || dma_fence_chain_find_seqno(pfence: &fence, seqno: points[i])) {
1071	dma_fence_put(fence);
1072	if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1073	continue;
1074	} else {
1075	timeout = -EINVAL;
1076	goto cleanup_entries;
1077	}
1078	}
1079
1080	if (fence)
1081	entries[i].fence = fence;
1082	else
1083	entries[i].fence = dma_fence_get_stub();
1084
1085	if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1086	dma_fence_is_signaled(fence: entries[i].fence)) {
1087	if (signaled_count == 0 && idx)
1088	*idx = i;
1089	signaled_count++;
1090	}
1091	}
1092
1093	if (signaled_count == count ||
1094	(signaled_count > 0 &&
1095	!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
1096	goto cleanup_entries;
1097
1098	/* There's a very annoying laxness in the dma_fence API here, in
1099	* that backends are not required to automatically report when a
1100	* fence is signaled prior to fence->ops->enable_signaling() being
1101	* called. So here if we fail to match signaled_count, we need to
1102	* fallthough and try a 0 timeout wait!
1103	*/
1104
1105	if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1106	for (i = 0; i < count; ++i)
1107	drm_syncobj_fence_add_wait(syncobj: syncobjs[i], wait: &entries[i]);
1108	}
1109
1110	do {
1111	set_current_state(TASK_INTERRUPTIBLE);
1112
1113	signaled_count = 0;
1114	for (i = 0; i < count; ++i) {
1115	fence = entries[i].fence;
1116	if (!fence)
1117	continue;
1118
1119	if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1120	dma_fence_is_signaled(fence) ||
1121	(!entries[i].fence_cb.func &&
1122	dma_fence_add_callback(fence,
1123	cb: &entries[i].fence_cb,
1124	func: syncobj_wait_fence_func))) {
1125	/* The fence has been signaled */
1126	if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
1127	signaled_count++;
1128	} else {
1129	if (idx)
1130	*idx = i;
1131	goto done_waiting;
1132	}
1133	}
1134	}
1135
1136	if (signaled_count == count)
1137	goto done_waiting;
1138
1139	if (timeout == 0) {
1140	timeout = -ETIME;
1141	goto done_waiting;
1142	}
1143
1144	if (signal_pending(current)) {
1145	timeout = -ERESTARTSYS;
1146	goto done_waiting;
1147	}
1148
1149	timeout = schedule_timeout(timeout);
1150	} while (1);
1151
1152	done_waiting:
1153	__set_current_state(TASK_RUNNING);
1154
1155	cleanup_entries:
1156	for (i = 0; i < count; ++i) {
1157	drm_syncobj_remove_wait(syncobj: syncobjs[i], wait: &entries[i]);
1158	if (entries[i].fence_cb.func)
1159	dma_fence_remove_callback(fence: entries[i].fence,
1160	cb: &entries[i].fence_cb);
1161	dma_fence_put(fence: entries[i].fence);
1162	}
1163	kfree(objp: entries);
1164
1165	err_free_points:
1166	kfree(objp: points);
1167
1168	return timeout;
1169	}
1170
1171	/**
1172	* drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
1173	*
1174	* @timeout_nsec: timeout nsec component in ns, 0 for poll
1175	*
1176	* Calculate the timeout in jiffies from an absolute time in sec/nsec.
1177	*/
1178	signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
1179	{
1180	ktime_t abs_timeout, now;
1181	u64 timeout_ns, timeout_jiffies64;
1182
1183	/* make 0 timeout means poll - absolute 0 doesn't seem valid */
1184	if (timeout_nsec == 0)
1185	return 0;
1186
1187	abs_timeout = ns_to_ktime(ns: timeout_nsec);
1188	now = ktime_get();
1189
1190	if (!ktime_after(cmp1: abs_timeout, cmp2: now))
1191	return 0;
1192
1193	timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
1194
1195	timeout_jiffies64 = nsecs_to_jiffies64(n: timeout_ns);
1196	/* clamp timeout to avoid infinite timeout */
1197	if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
1198	return MAX_SCHEDULE_TIMEOUT - 1;
1199
1200	return timeout_jiffies64 + 1;
1201	}
1202	EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
1203
1204	static int drm_syncobj_array_wait(struct drm_device *dev,
1205	struct drm_file *file_private,
1206	struct drm_syncobj_wait *wait,
1207	struct drm_syncobj_timeline_wait *timeline_wait,
1208	struct drm_syncobj **syncobjs, bool timeline)
1209	{
1210	signed long timeout = 0;
1211	uint32_t first = ~0;
1212
1213	if (!timeline) {
1214	timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
1215	timeout = drm_syncobj_array_wait_timeout(syncobjs,
1216	NULL,
1217	count: wait->count_handles,
1218	flags: wait->flags,
1219	timeout, idx: &first);
1220	if (timeout < 0)
1221	return timeout;
1222	wait->first_signaled = first;
1223	} else {
1224	timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
1225	timeout = drm_syncobj_array_wait_timeout(syncobjs,
1226	u64_to_user_ptr(timeline_wait->points),
1227	count: timeline_wait->count_handles,
1228	flags: timeline_wait->flags,
1229	timeout, idx: &first);
1230	if (timeout < 0)
1231	return timeout;
1232	timeline_wait->first_signaled = first;
1233	}
1234	return 0;
1235	}
1236
1237	static int drm_syncobj_array_find(struct drm_file *file_private,
1238	void __user *user_handles,
1239	uint32_t count_handles,
1240	struct drm_syncobj ***syncobjs_out)
1241	{
1242	uint32_t i, *handles;
1243	struct drm_syncobj **syncobjs;
1244	int ret;
1245
1246	handles = kmalloc_array(n: count_handles, size: sizeof(*handles), GFP_KERNEL);
1247	if (handles == NULL)
1248	return -ENOMEM;
1249
1250	if (copy_from_user(to: handles, from: user_handles,
1251	n: sizeof(uint32_t) * count_handles)) {
1252	ret = -EFAULT;
1253	goto err_free_handles;
1254	}
1255
1256	syncobjs = kmalloc_array(n: count_handles, size: sizeof(*syncobjs), GFP_KERNEL);
1257	if (syncobjs == NULL) {
1258	ret = -ENOMEM;
1259	goto err_free_handles;
1260	}
1261
1262	for (i = 0; i < count_handles; i++) {
1263	syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1264	if (!syncobjs[i]) {
1265	ret = -ENOENT;
1266	goto err_put_syncobjs;
1267	}
1268	}
1269
1270	kfree(objp: handles);
1271	*syncobjs_out = syncobjs;
1272	return 0;
1273
1274	err_put_syncobjs:
1275	while (i-- > 0)
1276	drm_syncobj_put(obj: syncobjs[i]);
1277	kfree(objp: syncobjs);
1278	err_free_handles:
1279	kfree(objp: handles);
1280
1281	return ret;
1282	}
1283
1284	static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1285	uint32_t count)
1286	{
1287	uint32_t i;
1288
1289	for (i = 0; i < count; i++)
1290	drm_syncobj_put(obj: syncobjs[i]);
1291	kfree(objp: syncobjs);
1292	}
1293
1294	int
1295	drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1296	struct drm_file *file_private)
1297	{
1298	struct drm_syncobj_wait *args = data;
1299	struct drm_syncobj **syncobjs;
1300	int ret = 0;
1301
1302	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
1303	return -EOPNOTSUPP;
1304
1305	if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1306	DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
1307	return -EINVAL;
1308
1309	if (args->count_handles == 0)
1310	return -EINVAL;
1311
1312	ret = drm_syncobj_array_find(file_private,
1313	u64_to_user_ptr(args->handles),
1314	count_handles: args->count_handles,
1315	syncobjs_out: &syncobjs);
1316	if (ret < 0)
1317	return ret;
1318
1319	ret = drm_syncobj_array_wait(dev, file_private,
1320	wait: args, NULL, syncobjs, timeline: false);
1321
1322	drm_syncobj_array_free(syncobjs, count: args->count_handles);
1323
1324	return ret;
1325	}
1326
1327	int
1328	drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1329	struct drm_file *file_private)
1330	{
1331	struct drm_syncobj_timeline_wait *args = data;
1332	struct drm_syncobj **syncobjs;
1333	int ret = 0;
1334
1335	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ_TIMELINE))
1336	return -EOPNOTSUPP;
1337
1338	if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1339	DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1340	DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
1341	return -EINVAL;
1342
1343	if (args->count_handles == 0)
1344	return -EINVAL;
1345
1346	ret = drm_syncobj_array_find(file_private,
1347	u64_to_user_ptr(args->handles),
1348	count_handles: args->count_handles,
1349	syncobjs_out: &syncobjs);
1350	if (ret < 0)
1351	return ret;
1352
1353	ret = drm_syncobj_array_wait(dev, file_private,
1354	NULL, timeline_wait: args, syncobjs, timeline: true);
1355
1356	drm_syncobj_array_free(syncobjs, count: args->count_handles);
1357
1358	return ret;
1359	}
1360
1361	static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence,
1362	struct dma_fence_cb *cb)
1363	{
1364	struct syncobj_eventfd_entry *entry =
1365	container_of(cb, struct syncobj_eventfd_entry, fence_cb);
1366
1367	eventfd_signal(ctx: entry->ev_fd_ctx, n: 1);
1368	syncobj_eventfd_entry_free(entry);
1369	}
1370
1371	static void
1372	syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
1373	struct syncobj_eventfd_entry *entry)
1374	{
1375	int ret;
1376	struct dma_fence *fence;
1377
1378	/* This happens inside the syncobj lock */
1379	fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
1380	ret = dma_fence_chain_find_seqno(pfence: &fence, seqno: entry->point);
1381	if (ret != 0 || !fence) {
1382	dma_fence_put(fence);
1383	return;
1384	}
1385
1386	list_del_init(entry: &entry->node);
1387	entry->fence = fence;
1388
1389	if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) {
1390	eventfd_signal(ctx: entry->ev_fd_ctx, n: 1);
1391	syncobj_eventfd_entry_free(entry);
1392	} else {
1393	ret = dma_fence_add_callback(fence, cb: &entry->fence_cb,
1394	func: syncobj_eventfd_entry_fence_func);
1395	if (ret == -ENOENT) {
1396	eventfd_signal(ctx: entry->ev_fd_ctx, n: 1);
1397	syncobj_eventfd_entry_free(entry);
1398	}
1399	}
1400	}
1401
1402	int
1403	drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data,
1404	struct drm_file *file_private)
1405	{
1406	struct drm_syncobj_eventfd *args = data;
1407	struct drm_syncobj *syncobj;
1408	struct eventfd_ctx *ev_fd_ctx;
1409	struct syncobj_eventfd_entry *entry;
1410
1411	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ_TIMELINE))
1412	return -EOPNOTSUPP;
1413
1414	if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)
1415	return -EINVAL;
1416
1417	if (args->pad)
1418	return -EINVAL;
1419
1420	syncobj = drm_syncobj_find(file_private, args->handle);
1421	if (!syncobj)
1422	return -ENOENT;
1423
1424	ev_fd_ctx = eventfd_ctx_fdget(fd: args->fd);
1425	if (IS_ERR(ptr: ev_fd_ctx))
1426	return PTR_ERR(ptr: ev_fd_ctx);
1427
1428	entry = kzalloc(size: sizeof(*entry), GFP_KERNEL);
1429	if (!entry) {
1430	eventfd_ctx_put(ctx: ev_fd_ctx);
1431	return -ENOMEM;
1432	}
1433	entry->syncobj = syncobj;
1434	entry->ev_fd_ctx = ev_fd_ctx;
1435	entry->point = args->point;
1436	entry->flags = args->flags;
1437
1438	drm_syncobj_add_eventfd(syncobj, entry);
1439	drm_syncobj_put(obj: syncobj);
1440
1441	return 0;
1442	}
1443
1444	int
1445	drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1446	struct drm_file *file_private)
1447	{
1448	struct drm_syncobj_array *args = data;
1449	struct drm_syncobj **syncobjs;
1450	uint32_t i;
1451	int ret;
1452
1453	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
1454	return -EOPNOTSUPP;
1455
1456	if (args->pad != 0)
1457	return -EINVAL;
1458
1459	if (args->count_handles == 0)
1460	return -EINVAL;
1461
1462	ret = drm_syncobj_array_find(file_private,
1463	u64_to_user_ptr(args->handles),
1464	count_handles: args->count_handles,
1465	syncobjs_out: &syncobjs);
1466	if (ret < 0)
1467	return ret;
1468
1469	for (i = 0; i < args->count_handles; i++)
1470	drm_syncobj_replace_fence(syncobjs[i], NULL);
1471
1472	drm_syncobj_array_free(syncobjs, count: args->count_handles);
1473
1474	return 0;
1475	}
1476
1477	int
1478	drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1479	struct drm_file *file_private)
1480	{
1481	struct drm_syncobj_array *args = data;
1482	struct drm_syncobj **syncobjs;
1483	uint32_t i;
1484	int ret;
1485
1486	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
1487	return -EOPNOTSUPP;
1488
1489	if (args->pad != 0)
1490	return -EINVAL;
1491
1492	if (args->count_handles == 0)
1493	return -EINVAL;
1494
1495	ret = drm_syncobj_array_find(file_private,
1496	u64_to_user_ptr(args->handles),
1497	count_handles: args->count_handles,
1498	syncobjs_out: &syncobjs);
1499	if (ret < 0)
1500	return ret;
1501
1502	for (i = 0; i < args->count_handles; i++) {
1503	ret = drm_syncobj_assign_null_handle(syncobj: syncobjs[i]);
1504	if (ret < 0)
1505	break;
1506	}
1507
1508	drm_syncobj_array_free(syncobjs, count: args->count_handles);
1509
1510	return ret;
1511	}
1512
1513	int
1514	drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1515	struct drm_file *file_private)
1516	{
1517	struct drm_syncobj_timeline_array *args = data;
1518	struct drm_syncobj **syncobjs;
1519	struct dma_fence_chain **chains;
1520	uint64_t *points;
1521	uint32_t i, j;
1522	int ret;
1523
1524	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ_TIMELINE))
1525	return -EOPNOTSUPP;
1526
1527	if (args->flags != 0)
1528	return -EINVAL;
1529
1530	if (args->count_handles == 0)
1531	return -EINVAL;
1532
1533	ret = drm_syncobj_array_find(file_private,
1534	u64_to_user_ptr(args->handles),
1535	count_handles: args->count_handles,
1536	syncobjs_out: &syncobjs);
1537	if (ret < 0)
1538	return ret;
1539
1540	points = kmalloc_array(n: args->count_handles, size: sizeof(*points),
1541	GFP_KERNEL);
1542	if (!points) {
1543	ret = -ENOMEM;
1544	goto out;
1545	}
1546	if (!u64_to_user_ptr(args->points)) {
1547	memset(points, 0, args->count_handles * sizeof(uint64_t));
1548	} else if (copy_from_user(to: points, u64_to_user_ptr(args->points),
1549	n: sizeof(uint64_t) * args->count_handles)) {
1550	ret = -EFAULT;
1551	goto err_points;
1552	}
1553
1554	chains = kmalloc_array(n: args->count_handles, size: sizeof(void *), GFP_KERNEL);
1555	if (!chains) {
1556	ret = -ENOMEM;
1557	goto err_points;
1558	}
1559	for (i = 0; i < args->count_handles; i++) {
1560	chains[i] = dma_fence_chain_alloc();
1561	if (!chains[i]) {
1562	for (j = 0; j < i; j++)
1563	dma_fence_chain_free(chain: chains[j]);
1564	ret = -ENOMEM;
1565	goto err_chains;
1566	}
1567	}
1568
1569	for (i = 0; i < args->count_handles; i++) {
1570	struct dma_fence *fence = dma_fence_get_stub();
1571
1572	drm_syncobj_add_point(syncobjs[i], chains[i],
1573	fence, points[i]);
1574	dma_fence_put(fence);
1575	}
1576	err_chains:
1577	kfree(objp: chains);
1578	err_points:
1579	kfree(objp: points);
1580	out:
1581	drm_syncobj_array_free(syncobjs, count: args->count_handles);
1582
1583	return ret;
1584	}
1585
1586	int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1587	struct drm_file *file_private)
1588	{
1589	struct drm_syncobj_timeline_array *args = data;
1590	struct drm_syncobj **syncobjs;
1591	uint64_t __user *points = u64_to_user_ptr(args->points);
1592	uint32_t i;
1593	int ret;
1594
1595	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ_TIMELINE))
1596	return -EOPNOTSUPP;
1597
1598	if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)
1599	return -EINVAL;
1600
1601	if (args->count_handles == 0)
1602	return -EINVAL;
1603
1604	ret = drm_syncobj_array_find(file_private,
1605	u64_to_user_ptr(args->handles),
1606	count_handles: args->count_handles,
1607	syncobjs_out: &syncobjs);
1608	if (ret < 0)
1609	return ret;
1610
1611	for (i = 0; i < args->count_handles; i++) {
1612	struct dma_fence_chain *chain;
1613	struct dma_fence *fence;
1614	uint64_t point;
1615
1616	fence = drm_syncobj_fence_get(syncobj: syncobjs[i]);
1617	chain = to_dma_fence_chain(fence);
1618	if (chain) {
1619	struct dma_fence *iter, *last_signaled =
1620	dma_fence_get(fence);
1621
1622	if (args->flags &
1623	DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {
1624	point = fence->seqno;
1625	} else {
1626	dma_fence_chain_for_each(iter, fence) {
1627	if (iter->context != fence->context) {
1628	dma_fence_put(fence: iter);
1629	/* It is most likely that timeline has
1630	* unorder points. */
1631	break;
1632	}
1633	dma_fence_put(fence: last_signaled);
1634	last_signaled = dma_fence_get(fence: iter);
1635	}
1636	point = dma_fence_is_signaled(fence: last_signaled) ?
1637	last_signaled->seqno :
1638	to_dma_fence_chain(fence: last_signaled)->prev_seqno;
1639	}
1640	dma_fence_put(fence: last_signaled);
1641	} else {
1642	point = 0;
1643	}
1644	dma_fence_put(fence);
1645	ret = copy_to_user(to: &points[i], from: &point, n: sizeof(uint64_t));
1646	ret = ret ? -EFAULT : 0;
1647	if (ret)
1648	break;
1649	}
1650	drm_syncobj_array_free(syncobjs, count: args->count_handles);
1651
1652	return ret;
1653	}
1654