1	// SPDX-License-Identifier: GPL-2.0-only OR MIT
2	/*
3	* Copyright (c) 2022 Red Hat.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*
23	* Authors:
24	* Danilo Krummrich <dakr@redhat.com>
25	*
26	*/
27
28	#include <drm/drm_gpuvm.h>
29
30	#include <linux/interval_tree_generic.h>
31	#include <linux/mm.h>
32
33	/**
34	* DOC: Overview
35	*
36	* The DRM GPU VA Manager, represented by struct drm_gpuvm keeps track of a
37	* GPU's virtual address (VA) space and manages the corresponding virtual
38	* mappings represented by &drm_gpuva objects. It also keeps track of the
39	* mapping's backing &drm_gem_object buffers.
40	*
41	* &drm_gem_object buffers maintain a list of &drm_gpuva objects representing
42	* all existent GPU VA mappings using this &drm_gem_object as backing buffer.
43	*
44	* GPU VAs can be flagged as sparse, such that drivers may use GPU VAs to also
45	* keep track of sparse PTEs in order to support Vulkan 'Sparse Resources'.
46	*
47	* The GPU VA manager internally uses a rb-tree to manage the
48	* &drm_gpuva mappings within a GPU's virtual address space.
49	*
50	* The &drm_gpuvm structure contains a special &drm_gpuva representing the
51	* portion of VA space reserved by the kernel. This node is initialized together
52	* with the GPU VA manager instance and removed when the GPU VA manager is
53	* destroyed.
54	*
55	* In a typical application drivers would embed struct drm_gpuvm and
56	* struct drm_gpuva within their own driver specific structures, there won't be
57	* any memory allocations of its own nor memory allocations of &drm_gpuva
58	* entries.
59	*
60	* The data structures needed to store &drm_gpuvas within the &drm_gpuvm are
61	* contained within struct drm_gpuva already. Hence, for inserting &drm_gpuva
62	* entries from within dma-fence signalling critical sections it is enough to
63	* pre-allocate the &drm_gpuva structures.
64	*
65	* &drm_gem_objects which are private to a single VM can share a common
66	* &dma_resv in order to improve locking efficiency (e.g. with &drm_exec).
67	* For this purpose drivers must pass a &drm_gem_object to drm_gpuvm_init(), in
68	* the following called 'resv object', which serves as the container of the
69	* GPUVM's shared &dma_resv. This resv object can be a driver specific
70	* &drm_gem_object, such as the &drm_gem_object containing the root page table,
71	* but it can also be a 'dummy' object, which can be allocated with
72	* drm_gpuvm_resv_object_alloc().
73	*
74	* In order to connect a struct drm_gpuva its backing &drm_gem_object each
75	* &drm_gem_object maintains a list of &drm_gpuvm_bo structures, and each
76	* &drm_gpuvm_bo contains a list of &drm_gpuva structures.
77	*
78	* A &drm_gpuvm_bo is an abstraction that represents a combination of a
79	* &drm_gpuvm and a &drm_gem_object. Every such combination should be unique.
80	* This is ensured by the API through drm_gpuvm_bo_obtain() and
81	* drm_gpuvm_bo_obtain_prealloc() which first look into the corresponding
82	* &drm_gem_object list of &drm_gpuvm_bos for an existing instance of this
83	* particular combination. If not existent a new instance is created and linked
84	* to the &drm_gem_object.
85	*
86	* &drm_gpuvm_bo structures, since unique for a given &drm_gpuvm, are also used
87	* as entry for the &drm_gpuvm's lists of external and evicted objects. Those
88	* lists are maintained in order to accelerate locking of dma-resv locks and
89	* validation of evicted objects bound in a &drm_gpuvm. For instance, all
90	* &drm_gem_object's &dma_resv of a given &drm_gpuvm can be locked by calling
91	* drm_gpuvm_exec_lock(). Once locked drivers can call drm_gpuvm_validate() in
92	* order to validate all evicted &drm_gem_objects. It is also possible to lock
93	* additional &drm_gem_objects by providing the corresponding parameters to
94	* drm_gpuvm_exec_lock() as well as open code the &drm_exec loop while making
95	* use of helper functions such as drm_gpuvm_prepare_range() or
96	* drm_gpuvm_prepare_objects().
97	*
98	* Every bound &drm_gem_object is treated as external object when its &dma_resv
99	* structure is different than the &drm_gpuvm's common &dma_resv structure.
100	*/
101
102	/**
103	* DOC: Split and Merge
104	*
105	* Besides its capability to manage and represent a GPU VA space, the
106	* GPU VA manager also provides functions to let the &drm_gpuvm calculate a
107	* sequence of operations to satisfy a given map or unmap request.
108	*
109	* Therefore the DRM GPU VA manager provides an algorithm implementing splitting
110	* and merging of existent GPU VA mappings with the ones that are requested to
111	* be mapped or unmapped. This feature is required by the Vulkan API to
112	* implement Vulkan 'Sparse Memory Bindings' - drivers UAPIs often refer to this
113	* as VM BIND.
114	*
115	* Drivers can call drm_gpuvm_sm_map() to receive a sequence of callbacks
116	* containing map, unmap and remap operations for a given newly requested
117	* mapping. The sequence of callbacks represents the set of operations to
118	* execute in order to integrate the new mapping cleanly into the current state
119	* of the GPU VA space.
120	*
121	* Depending on how the new GPU VA mapping intersects with the existent mappings
122	* of the GPU VA space the &drm_gpuvm_ops callbacks contain an arbitrary amount
123	* of unmap operations, a maximum of two remap operations and a single map
124	* operation. The caller might receive no callback at all if no operation is
125	* required, e.g. if the requested mapping already exists in the exact same way.
126	*
127	* The single map operation represents the original map operation requested by
128	* the caller.
129	*
130	* &drm_gpuva_op_unmap contains a 'keep' field, which indicates whether the
131	* &drm_gpuva to unmap is physically contiguous with the original mapping
132	* request. Optionally, if 'keep' is set, drivers may keep the actual page table
133	* entries for this &drm_gpuva, adding the missing page table entries only and
134	* update the &drm_gpuvm's view of things accordingly.
135	*
136	* Drivers may do the same optimization, namely delta page table updates, also
137	* for remap operations. This is possible since &drm_gpuva_op_remap consists of
138	* one unmap operation and one or two map operations, such that drivers can
139	* derive the page table update delta accordingly.
140	*
141	* Note that there can't be more than two existent mappings to split up, one at
142	* the beginning and one at the end of the new mapping, hence there is a
143	* maximum of two remap operations.
144	*
145	* Analogous to drm_gpuvm_sm_map() drm_gpuvm_sm_unmap() uses &drm_gpuvm_ops to
146	* call back into the driver in order to unmap a range of GPU VA space. The
147	* logic behind this function is way simpler though: For all existent mappings
148	* enclosed by the given range unmap operations are created. For mappings which
149	* are only partically located within the given range, remap operations are
150	* created such that those mappings are split up and re-mapped partically.
151	*
152	* As an alternative to drm_gpuvm_sm_map() and drm_gpuvm_sm_unmap(),
153	* drm_gpuvm_sm_map_ops_create() and drm_gpuvm_sm_unmap_ops_create() can be used
154	* to directly obtain an instance of struct drm_gpuva_ops containing a list of
155	* &drm_gpuva_op, which can be iterated with drm_gpuva_for_each_op(). This list
156	* contains the &drm_gpuva_ops analogous to the callbacks one would receive when
157	* calling drm_gpuvm_sm_map() or drm_gpuvm_sm_unmap(). While this way requires
158	* more memory (to allocate the &drm_gpuva_ops), it provides drivers a way to
159	* iterate the &drm_gpuva_op multiple times, e.g. once in a context where memory
160	* allocations are possible (e.g. to allocate GPU page tables) and once in the
161	* dma-fence signalling critical path.
162	*
163	* To update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert() and
164	* drm_gpuva_remove() may be used. These functions can safely be used from
165	* &drm_gpuvm_ops callbacks originating from drm_gpuvm_sm_map() or
166	* drm_gpuvm_sm_unmap(). However, it might be more convenient to use the
167	* provided helper functions drm_gpuva_map(), drm_gpuva_remap() and
168	* drm_gpuva_unmap() instead.
169	*
170	* The following diagram depicts the basic relationships of existent GPU VA
171	* mappings, a newly requested mapping and the resulting mappings as implemented
172	* by drm_gpuvm_sm_map() - it doesn't cover any arbitrary combinations of these.
173	*
174	* 1) Requested mapping is identical. Replace it, but indicate the backing PTEs
175	* could be kept.
176	*
177	* ::
178	*
179	* 0 a 1
180	* old: |-----------| (bo_offset=n)
181	*
182	* 0 a 1
183	* req: |-----------| (bo_offset=n)
184	*
185	* 0 a 1
186	* new: |-----------| (bo_offset=n)
187	*
188	*
189	* 2) Requested mapping is identical, except for the BO offset, hence replace
190	* the mapping.
191	*
192	* ::
193	*
194	* 0 a 1
195	* old: |-----------| (bo_offset=n)
196	*
197	* 0 a 1
198	* req: |-----------| (bo_offset=m)
199	*
200	* 0 a 1
201	* new: |-----------| (bo_offset=m)
202	*
203	*
204	* 3) Requested mapping is identical, except for the backing BO, hence replace
205	* the mapping.
206	*
207	* ::
208	*
209	* 0 a 1
210	* old: |-----------| (bo_offset=n)
211	*
212	* 0 b 1
213	* req: |-----------| (bo_offset=n)
214	*
215	* 0 b 1
216	* new: |-----------| (bo_offset=n)
217	*
218	*
219	* 4) Existent mapping is a left aligned subset of the requested one, hence
220	* replace the existent one.
221	*
222	* ::
223	*
224	* 0 a 1
225	* old: |-----| (bo_offset=n)
226	*
227	* 0 a 2
228	* req: |-----------| (bo_offset=n)
229	*
230	* 0 a 2
231	* new: |-----------| (bo_offset=n)
232	*
233	* .. note::
234	* We expect to see the same result for a request with a different BO
235	* and/or non-contiguous BO offset.
236	*
237	*
238	* 5) Requested mapping's range is a left aligned subset of the existent one,
239	* but backed by a different BO. Hence, map the requested mapping and split
240	* the existent one adjusting its BO offset.
241	*
242	* ::
243	*
244	* 0 a 2
245	* old: |-----------| (bo_offset=n)
246	*
247	* 0 b 1
248	* req: |-----| (bo_offset=n)
249	*
250	* 0 b 1 a' 2
251	* new: |-----|-----| (b.bo_offset=n, a.bo_offset=n+1)
252	*
253	* .. note::
254	* We expect to see the same result for a request with a different BO
255	* and/or non-contiguous BO offset.
256	*
257	*
258	* 6) Existent mapping is a superset of the requested mapping. Split it up, but
259	* indicate that the backing PTEs could be kept.
260	*
261	* ::
262	*
263	* 0 a 2
264	* old: |-----------| (bo_offset=n)
265	*
266	* 0 a 1
267	* req: |-----| (bo_offset=n)
268	*
269	* 0 a 1 a' 2
270	* new: |-----|-----| (a.bo_offset=n, a'.bo_offset=n+1)
271	*
272	*
273	* 7) Requested mapping's range is a right aligned subset of the existent one,
274	* but backed by a different BO. Hence, map the requested mapping and split
275	* the existent one, without adjusting the BO offset.
276	*
277	* ::
278	*
279	* 0 a 2
280	* old: |-----------| (bo_offset=n)
281	*
282	* 1 b 2
283	* req: |-----| (bo_offset=m)
284	*
285	* 0 a 1 b 2
286	* new: |-----|-----| (a.bo_offset=n,b.bo_offset=m)
287	*
288	*
289	* 8) Existent mapping is a superset of the requested mapping. Split it up, but
290	* indicate that the backing PTEs could be kept.
291	*
292	* ::
293	*
294	* 0 a 2
295	* old: |-----------| (bo_offset=n)
296	*
297	* 1 a 2
298	* req: |-----| (bo_offset=n+1)
299	*
300	* 0 a' 1 a 2
301	* new: |-----|-----| (a'.bo_offset=n, a.bo_offset=n+1)
302	*
303	*
304	* 9) Existent mapping is overlapped at the end by the requested mapping backed
305	* by a different BO. Hence, map the requested mapping and split up the
306	* existent one, without adjusting the BO offset.
307	*
308	* ::
309	*
310	* 0 a 2
311	* old: |-----------| (bo_offset=n)
312	*
313	* 1 b 3
314	* req: |-----------| (bo_offset=m)
315	*
316	* 0 a 1 b 3
317	* new: |-----|-----------| (a.bo_offset=n,b.bo_offset=m)
318	*
319	*
320	* 10) Existent mapping is overlapped by the requested mapping, both having the
321	* same backing BO with a contiguous offset. Indicate the backing PTEs of
322	* the old mapping could be kept.
323	*
324	* ::
325	*
326	* 0 a 2
327	* old: |-----------| (bo_offset=n)
328	*
329	* 1 a 3
330	* req: |-----------| (bo_offset=n+1)
331	*
332	* 0 a' 1 a 3
333	* new: |-----|-----------| (a'.bo_offset=n, a.bo_offset=n+1)
334	*
335	*
336	* 11) Requested mapping's range is a centered subset of the existent one
337	* having a different backing BO. Hence, map the requested mapping and split
338	* up the existent one in two mappings, adjusting the BO offset of the right
339	* one accordingly.
340	*
341	* ::
342	*
343	* 0 a 3
344	* old: |-----------------| (bo_offset=n)
345	*
346	* 1 b 2
347	* req: |-----| (bo_offset=m)
348	*
349	* 0 a 1 b 2 a' 3
350	* new: |-----|-----|-----| (a.bo_offset=n,b.bo_offset=m,a'.bo_offset=n+2)
351	*
352	*
353	* 12) Requested mapping is a contiguous subset of the existent one. Split it
354	* up, but indicate that the backing PTEs could be kept.
355	*
356	* ::
357	*
358	* 0 a 3
359	* old: |-----------------| (bo_offset=n)
360	*
361	* 1 a 2
362	* req: |-----| (bo_offset=n+1)
363	*
364	* 0 a' 1 a 2 a'' 3
365	* old: |-----|-----|-----| (a'.bo_offset=n, a.bo_offset=n+1, a''.bo_offset=n+2)
366	*
367	*
368	* 13) Existent mapping is a right aligned subset of the requested one, hence
369	* replace the existent one.
370	*
371	* ::
372	*
373	* 1 a 2
374	* old: |-----| (bo_offset=n+1)
375	*
376	* 0 a 2
377	* req: |-----------| (bo_offset=n)
378	*
379	* 0 a 2
380	* new: |-----------| (bo_offset=n)
381	*
382	* .. note::
383	* We expect to see the same result for a request with a different bo
384	* and/or non-contiguous bo_offset.
385	*
386	*
387	* 14) Existent mapping is a centered subset of the requested one, hence
388	* replace the existent one.
389	*
390	* ::
391	*
392	* 1 a 2
393	* old: |-----| (bo_offset=n+1)
394	*
395	* 0 a 3
396	* req: |----------------| (bo_offset=n)
397	*
398	* 0 a 3
399	* new: |----------------| (bo_offset=n)
400	*
401	* .. note::
402	* We expect to see the same result for a request with a different bo
403	* and/or non-contiguous bo_offset.
404	*
405	*
406	* 15) Existent mappings is overlapped at the beginning by the requested mapping
407	* backed by a different BO. Hence, map the requested mapping and split up
408	* the existent one, adjusting its BO offset accordingly.
409	*
410	* ::
411	*
412	* 1 a 3
413	* old: |-----------| (bo_offset=n)
414	*
415	* 0 b 2
416	* req: |-----------| (bo_offset=m)
417	*
418	* 0 b 2 a' 3
419	* new: |-----------|-----| (b.bo_offset=m,a.bo_offset=n+2)
420	*/
421
422	/**
423	* DOC: Locking
424	*
425	* In terms of managing &drm_gpuva entries DRM GPUVM does not take care of
426	* locking itself, it is the drivers responsibility to take care about locking.
427	* Drivers might want to protect the following operations: inserting, removing
428	* and iterating &drm_gpuva objects as well as generating all kinds of
429	* operations, such as split / merge or prefetch.
430	*
431	* DRM GPUVM also does not take care of the locking of the backing
432	* &drm_gem_object buffers GPU VA lists and &drm_gpuvm_bo abstractions by
433	* itself; drivers are responsible to enforce mutual exclusion using either the
434	* GEMs dma_resv lock or alternatively a driver specific external lock. For the
435	* latter see also drm_gem_gpuva_set_lock().
436	*
437	* However, DRM GPUVM contains lockdep checks to ensure callers of its API hold
438	* the corresponding lock whenever the &drm_gem_objects GPU VA list is accessed
439	* by functions such as drm_gpuva_link() or drm_gpuva_unlink(), but also
440	* drm_gpuvm_bo_obtain() and drm_gpuvm_bo_put().
441	*
442	* The latter is required since on creation and destruction of a &drm_gpuvm_bo
443	* the &drm_gpuvm_bo is attached / removed from the &drm_gem_objects gpuva list.
444	* Subsequent calls to drm_gpuvm_bo_obtain() for the same &drm_gpuvm and
445	* &drm_gem_object must be able to observe previous creations and destructions
446	* of &drm_gpuvm_bos in order to keep instances unique.
447	*
448	* The &drm_gpuvm's lists for keeping track of external and evicted objects are
449	* protected against concurrent insertion / removal and iteration internally.
450	*
451	* However, drivers still need ensure to protect concurrent calls to functions
452	* iterating those lists, namely drm_gpuvm_prepare_objects() and
453	* drm_gpuvm_validate().
454	*
455	* Alternatively, drivers can set the &DRM_GPUVM_RESV_PROTECTED flag to indicate
456	* that the corresponding &dma_resv locks are held in order to protect the
457	* lists. If &DRM_GPUVM_RESV_PROTECTED is set, internal locking is disabled and
458	* the corresponding lockdep checks are enabled. This is an optimization for
459	* drivers which are capable of taking the corresponding &dma_resv locks and
460	* hence do not require internal locking.
461	*/
462
463	/**
464	* DOC: Examples
465	*
466	* This section gives two examples on how to let the DRM GPUVA Manager generate
467	* &drm_gpuva_op in order to satisfy a given map or unmap request and how to
468	* make use of them.
469	*
470	* The below code is strictly limited to illustrate the generic usage pattern.
471	* To maintain simplicitly, it doesn't make use of any abstractions for common
472	* code, different (asyncronous) stages with fence signalling critical paths,
473	* any other helpers or error handling in terms of freeing memory and dropping
474	* previously taken locks.
475	*
476	* 1) Obtain a list of &drm_gpuva_op to create a new mapping::
477	*
478	* // Allocates a new &drm_gpuva.
479	* struct drm_gpuva * driver_gpuva_alloc(void);
480	*
481	* // Typically drivers would embedd the &drm_gpuvm and &drm_gpuva
482	* // structure in individual driver structures and lock the dma-resv with
483	* // drm_exec or similar helpers.
484	* int driver_mapping_create(struct drm_gpuvm *gpuvm,
485	* u64 addr, u64 range,
486	* struct drm_gem_object *obj, u64 offset)
487	* {
488	* struct drm_gpuva_ops *ops;
489	* struct drm_gpuva_op *op
490	* struct drm_gpuvm_bo *vm_bo;
491	*
492	* driver_lock_va_space();
493	* ops = drm_gpuvm_sm_map_ops_create(gpuvm, addr, range,
494	* obj, offset);
495	* if (IS_ERR(ops))
496	* return PTR_ERR(ops);
497	*
498	* vm_bo = drm_gpuvm_bo_obtain(gpuvm, obj);
499	* if (IS_ERR(vm_bo))
500	* return PTR_ERR(vm_bo);
501	*
502	* drm_gpuva_for_each_op(op, ops) {
503	* struct drm_gpuva *va;
504	*
505	* switch (op->op) {
506	* case DRM_GPUVA_OP_MAP:
507	* va = driver_gpuva_alloc();
508	* if (!va)
509	* ; // unwind previous VA space updates,
510	* // free memory and unlock
511	*
512	* driver_vm_map();
513	* drm_gpuva_map(gpuvm, va, &op->map);
514	* drm_gpuva_link(va, vm_bo);
515	*
516	* break;
517	* case DRM_GPUVA_OP_REMAP: {
518	* struct drm_gpuva *prev = NULL, *next = NULL;
519	*
520	* va = op->remap.unmap->va;
521	*
522	* if (op->remap.prev) {
523	* prev = driver_gpuva_alloc();
524	* if (!prev)
525	* ; // unwind previous VA space
526	* // updates, free memory and
527	* // unlock
528	* }
529	*
530	* if (op->remap.next) {
531	* next = driver_gpuva_alloc();
532	* if (!next)
533	* ; // unwind previous VA space
534	* // updates, free memory and
535	* // unlock
536	* }
537	*
538	* driver_vm_remap();
539	* drm_gpuva_remap(prev, next, &op->remap);
540	*
541	* if (prev)
542	* drm_gpuva_link(prev, va->vm_bo);
543	* if (next)
544	* drm_gpuva_link(next, va->vm_bo);
545	* drm_gpuva_unlink(va);
546	*
547	* break;
548	* }
549	* case DRM_GPUVA_OP_UNMAP:
550	* va = op->unmap->va;
551	*
552	* driver_vm_unmap();
553	* drm_gpuva_unlink(va);
554	* drm_gpuva_unmap(&op->unmap);
555	*
556	* break;
557	* default:
558	* break;
559	* }
560	* }
561	* drm_gpuvm_bo_put(vm_bo);
562	* driver_unlock_va_space();
563	*
564	* return 0;
565	* }
566	*
567	* 2) Receive a callback for each &drm_gpuva_op to create a new mapping::
568	*
569	* struct driver_context {
570	* struct drm_gpuvm *gpuvm;
571	* struct drm_gpuvm_bo *vm_bo;
572	* struct drm_gpuva *new_va;
573	* struct drm_gpuva *prev_va;
574	* struct drm_gpuva *next_va;
575	* };
576	*
577	* // ops to pass to drm_gpuvm_init()
578	* static const struct drm_gpuvm_ops driver_gpuvm_ops = {
579	* .sm_step_map = driver_gpuva_map,
580	* .sm_step_remap = driver_gpuva_remap,
581	* .sm_step_unmap = driver_gpuva_unmap,
582	* };
583	*
584	* // Typically drivers would embedd the &drm_gpuvm and &drm_gpuva
585	* // structure in individual driver structures and lock the dma-resv with
586	* // drm_exec or similar helpers.
587	* int driver_mapping_create(struct drm_gpuvm *gpuvm,
588	* u64 addr, u64 range,
589	* struct drm_gem_object *obj, u64 offset)
590	* {
591	* struct driver_context ctx;
592	* struct drm_gpuvm_bo *vm_bo;
593	* struct drm_gpuva_ops *ops;
594	* struct drm_gpuva_op *op;
595	* int ret = 0;
596	*
597	* ctx.gpuvm = gpuvm;
598	*
599	* ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL);
600	* ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL);
601	* ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL);
602	* ctx.vm_bo = drm_gpuvm_bo_create(gpuvm, obj);
603	* if (!ctx.new_va || !ctx.prev_va || !ctx.next_va || !vm_bo) {
604	* ret = -ENOMEM;
605	* goto out;
606	* }
607	*
608	* // Typically protected with a driver specific GEM gpuva lock
609	* // used in the fence signaling path for drm_gpuva_link() and
610	* // drm_gpuva_unlink(), hence pre-allocate.
611	* ctx.vm_bo = drm_gpuvm_bo_obtain_prealloc(ctx.vm_bo);
612	*
613	* driver_lock_va_space();
614	* ret = drm_gpuvm_sm_map(gpuvm, &ctx, addr, range, obj, offset);
615	* driver_unlock_va_space();
616	*
617	* out:
618	* drm_gpuvm_bo_put(ctx.vm_bo);
619	* kfree(ctx.new_va);
620	* kfree(ctx.prev_va);
621	* kfree(ctx.next_va);
622	* return ret;
623	* }
624	*
625	* int driver_gpuva_map(struct drm_gpuva_op *op, void *__ctx)
626	* {
627	* struct driver_context *ctx = __ctx;
628	*
629	* drm_gpuva_map(ctx->vm, ctx->new_va, &op->map);
630	*
631	* drm_gpuva_link(ctx->new_va, ctx->vm_bo);
632	*
633	* // prevent the new GPUVA from being freed in
634	* // driver_mapping_create()
635	* ctx->new_va = NULL;
636	*
637	* return 0;
638	* }
639	*
640	* int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx)
641	* {
642	* struct driver_context *ctx = __ctx;
643	* struct drm_gpuva *va = op->remap.unmap->va;
644	*
645	* drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap);
646	*
647	* if (op->remap.prev) {
648	* drm_gpuva_link(ctx->prev_va, va->vm_bo);
649	* ctx->prev_va = NULL;
650	* }
651	*
652	* if (op->remap.next) {
653	* drm_gpuva_link(ctx->next_va, va->vm_bo);
654	* ctx->next_va = NULL;
655	* }
656	*
657	* drm_gpuva_unlink(va);
658	* kfree(va);
659	*
660	* return 0;
661	* }
662	*
663	* int driver_gpuva_unmap(struct drm_gpuva_op *op, void *__ctx)
664	* {
665	* drm_gpuva_unlink(op->unmap.va);
666	* drm_gpuva_unmap(&op->unmap);
667	* kfree(op->unmap.va);
668	*
669	* return 0;
670	* }
671	*/
672
673	/**
674	* get_next_vm_bo_from_list() - get the next vm_bo element
675	* @__gpuvm: the &drm_gpuvm
676	* @__list_name: the name of the list we're iterating on
677	* @__local_list: a pointer to the local list used to store already iterated items
678	* @__prev_vm_bo: the previous element we got from get_next_vm_bo_from_list()
679	*
680	* This helper is here to provide lockless list iteration. Lockless as in, the
681	* iterator releases the lock immediately after picking the first element from
682	* the list, so list insertion deletion can happen concurrently.
683	*
684	* Elements popped from the original list are kept in a local list, so removal
685	* and is_empty checks can still happen while we're iterating the list.
686	*/
687	#define get_next_vm_bo_from_list(__gpuvm, __list_name, __local_list, __prev_vm_bo) \
688	({ \
689	struct drm_gpuvm_bo *__vm_bo = NULL; \
690	\
691	drm_gpuvm_bo_put(__prev_vm_bo); \
692	\
693	spin_lock(&(__gpuvm)->__list_name.lock); \
694	if (!(__gpuvm)->__list_name.local_list) \
695	(__gpuvm)->__list_name.local_list = __local_list; \
696	else \
697	drm_WARN_ON((__gpuvm)->drm, \
698	(__gpuvm)->__list_name.local_list != __local_list); \
699	\
700	while (!list_empty(&(__gpuvm)->__list_name.list)) { \
701	__vm_bo = list_first_entry(&(__gpuvm)->__list_name.list, \
702	struct drm_gpuvm_bo, \
703	list.entry.__list_name); \
704	if (kref_get_unless_zero(&__vm_bo->kref)) { \
705	list_move_tail(&(__vm_bo)->list.entry.__list_name, \
706	__local_list); \
707	break; \
708	} else { \
709	list_del_init(&(__vm_bo)->list.entry.__list_name); \
710	__vm_bo = NULL; \
711	} \
712	} \
713	spin_unlock(&(__gpuvm)->__list_name.lock); \
714	\
715	__vm_bo; \
716	})
717
718	/**
719	* for_each_vm_bo_in_list() - internal vm_bo list iterator
720	* @__gpuvm: the &drm_gpuvm
721	* @__list_name: the name of the list we're iterating on
722	* @__local_list: a pointer to the local list used to store already iterated items
723	* @__vm_bo: the struct drm_gpuvm_bo to assign in each iteration step
724	*
725	* This helper is here to provide lockless list iteration. Lockless as in, the
726	* iterator releases the lock immediately after picking the first element from the
727	* list, hence list insertion and deletion can happen concurrently.
728	*
729	* It is not allowed to re-assign the vm_bo pointer from inside this loop.
730	*
731	* Typical use:
732	*
733	* struct drm_gpuvm_bo *vm_bo;
734	* LIST_HEAD(my_local_list);
735	*
736	* ret = 0;
737	* for_each_vm_bo_in_list(gpuvm, <list_name>, &my_local_list, vm_bo) {
738	* ret = do_something_with_vm_bo(..., vm_bo);
739	* if (ret)
740	* break;
741	* }
742	* // Drop ref in case we break out of the loop.
743	* drm_gpuvm_bo_put(vm_bo);
744	* restore_vm_bo_list(gpuvm, <list_name>, &my_local_list);
745	*
746	*
747	* Only used for internal list iterations, not meant to be exposed to the outside
748	* world.
749	*/
750	#define for_each_vm_bo_in_list(__gpuvm, __list_name, __local_list, __vm_bo) \
751	for (__vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \
752	__local_list, NULL); \
753	__vm_bo; \
754	__vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \
755	__local_list, __vm_bo))
756
757	static void
758	__restore_vm_bo_list(struct drm_gpuvm *gpuvm, spinlock_t *lock,
759	struct list_head *list, struct list_head **local_list)
760	{
761	/* Merge back the two lists, moving local list elements to the
762	* head to preserve previous ordering, in case it matters.
763	*/
764	spin_lock(lock);
765	if (*local_list) {
766	list_splice(list: *local_list, head: list);
767	*local_list = NULL;
768	}
769	spin_unlock(lock);
770	}
771
772	/**
773	* restore_vm_bo_list() - move vm_bo elements back to their original list
774	* @__gpuvm: the &drm_gpuvm
775	* @__list_name: the name of the list we're iterating on
776	*
777	* When we're done iterating a vm_bo list, we should call restore_vm_bo_list()
778	* to restore the original state and let new iterations take place.
779	*/
780	#define restore_vm_bo_list(__gpuvm, __list_name) \
781	__restore_vm_bo_list((__gpuvm), &(__gpuvm)->__list_name.lock, \
782	&(__gpuvm)->__list_name.list, \
783	&(__gpuvm)->__list_name.local_list)
784
785	static void
786	cond_spin_lock(spinlock_t *lock, bool cond)
787	{
788	if (cond)
789	spin_lock(lock);
790	}
791
792	static void
793	cond_spin_unlock(spinlock_t *lock, bool cond)
794	{
795	if (cond)
796	spin_unlock(lock);
797	}
798
799	static void
800	__drm_gpuvm_bo_list_add(struct drm_gpuvm *gpuvm, spinlock_t *lock,
801	struct list_head *entry, struct list_head *list)
802	{
803	cond_spin_lock(lock, cond: !!lock);
804	if (list_empty(head: entry))
805	list_add_tail(new: entry, head: list);
806	cond_spin_unlock(lock, cond: !!lock);
807	}
808
809	/**
810	* drm_gpuvm_bo_list_add() - insert a vm_bo into the given list
811	* @__vm_bo: the &drm_gpuvm_bo
812	* @__list_name: the name of the list to insert into
813	* @__lock: whether to lock with the internal spinlock
814	*
815	* Inserts the given @__vm_bo into the list specified by @__list_name.
816	*/
817	#define drm_gpuvm_bo_list_add(__vm_bo, __list_name, __lock) \
818	__drm_gpuvm_bo_list_add((__vm_bo)->vm, \
819	__lock ? &(__vm_bo)->vm->__list_name.lock : \
820	NULL, \
821	&(__vm_bo)->list.entry.__list_name, \
822	&(__vm_bo)->vm->__list_name.list)
823
824	static void
825	__drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock,
826	struct list_head *entry, bool init)
827	{
828	cond_spin_lock(lock, cond: !!lock);
829	if (init) {
830	if (!list_empty(head: entry))
831	list_del_init(entry);
832	} else {
833	list_del(entry);
834	}
835	cond_spin_unlock(lock, cond: !!lock);
836	}
837
838	/**
839	* drm_gpuvm_bo_list_del_init() - remove a vm_bo from the given list
840	* @__vm_bo: the &drm_gpuvm_bo
841	* @__list_name: the name of the list to insert into
842	* @__lock: whether to lock with the internal spinlock
843	*
844	* Removes the given @__vm_bo from the list specified by @__list_name.
845	*/
846	#define drm_gpuvm_bo_list_del_init(__vm_bo, __list_name, __lock) \
847	__drm_gpuvm_bo_list_del((__vm_bo)->vm, \
848	__lock ? &(__vm_bo)->vm->__list_name.lock : \
849	NULL, \
850	&(__vm_bo)->list.entry.__list_name, \
851	true)
852
853	/**
854	* drm_gpuvm_bo_list_del() - remove a vm_bo from the given list
855	* @__vm_bo: the &drm_gpuvm_bo
856	* @__list_name: the name of the list to insert into
857	* @__lock: whether to lock with the internal spinlock
858	*
859	* Removes the given @__vm_bo from the list specified by @__list_name.
860	*/
861	#define drm_gpuvm_bo_list_del(__vm_bo, __list_name, __lock) \
862	__drm_gpuvm_bo_list_del((__vm_bo)->vm, \
863	__lock ? &(__vm_bo)->vm->__list_name.lock : \
864	NULL, \
865	&(__vm_bo)->list.entry.__list_name, \
866	false)
867
868	#define to_drm_gpuva(__node) container_of((__node), struct drm_gpuva, rb.node)
869
870	#define GPUVA_START(node) ((node)->va.addr)
871	#define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1)
872
873	/* We do not actually use drm_gpuva_it_next(), tell the compiler to not complain
874	* about this.
875	*/
876	INTERVAL_TREE_DEFINE(struct drm_gpuva, rb.node, u64, rb.__subtree_last,
877	GPUVA_START, GPUVA_LAST, static __maybe_unused,
878	drm_gpuva_it)
879
880	static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm,
881	struct drm_gpuva *va);
882	static void __drm_gpuva_remove(struct drm_gpuva *va);
883
884	static bool
885	drm_gpuvm_check_overflow(u64 addr, u64 range)
886	{
887	u64 end;
888
889	return check_add_overflow(addr, range, &end);
890	}
891
892	static bool
893	drm_gpuvm_warn_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
894	{
895	return drm_WARN(gpuvm->drm, drm_gpuvm_check_overflow(addr, range),
896	"GPUVA address limited to %zu bytes.\n", sizeof(addr));
897	}
898
899	static bool
900	drm_gpuvm_in_mm_range(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
901	{
902	u64 end = addr + range;
903	u64 mm_start = gpuvm->mm_start;
904	u64 mm_end = mm_start + gpuvm->mm_range;
905
906	return addr >= mm_start && end <= mm_end;
907	}
908
909	static bool
910	drm_gpuvm_in_kernel_node(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
911	{
912	u64 end = addr + range;
913	u64 kstart = gpuvm->kernel_alloc_node.va.addr;
914	u64 krange = gpuvm->kernel_alloc_node.va.range;
915	u64 kend = kstart + krange;
916
917	return krange && addr < kend && kstart < end;
918	}
919
920	/**
921	* drm_gpuvm_range_valid() - checks whether the given range is valid for the
922	* given &drm_gpuvm
923	* @gpuvm: the GPUVM to check the range for
924	* @addr: the base address
925	* @range: the range starting from the base address
926	*
927	* Checks whether the range is within the GPUVM's managed boundaries.
928	*
929	* Returns: true for a valid range, false otherwise
930	*/
931	bool
932	drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm,
933	u64 addr, u64 range)
934	{
935	return !drm_gpuvm_check_overflow(addr, range) &&
936	drm_gpuvm_in_mm_range(gpuvm, addr, range) &&
937	!drm_gpuvm_in_kernel_node(gpuvm, addr, range);
938	}
939	EXPORT_SYMBOL_GPL(drm_gpuvm_range_valid);
940
941	static void
942	drm_gpuvm_gem_object_free(struct drm_gem_object *obj)
943	{
944	drm_gem_object_release(obj);
945	kfree(objp: obj);
946	}
947
948	static const struct drm_gem_object_funcs drm_gpuvm_object_funcs = {
949	.free = drm_gpuvm_gem_object_free,
950	};
951
952	/**
953	* drm_gpuvm_resv_object_alloc() - allocate a dummy &drm_gem_object
954	* @drm: the drivers &drm_device
955	*
956	* Allocates a dummy &drm_gem_object which can be passed to drm_gpuvm_init() in
957	* order to serve as root GEM object providing the &drm_resv shared across
958	* &drm_gem_objects local to a single GPUVM.
959	*
960	* Returns: the &drm_gem_object on success, NULL on failure
961	*/
962	struct drm_gem_object *
963	drm_gpuvm_resv_object_alloc(struct drm_device *drm)
964	{
965	struct drm_gem_object *obj;
966
967	obj = kzalloc(size: sizeof(*obj), GFP_KERNEL);
968	if (!obj)
969	return NULL;
970
971	obj->funcs = &drm_gpuvm_object_funcs;
972	drm_gem_private_object_init(dev: drm, obj, size: 0);
973
974	return obj;
975	}
976	EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc);
977
978	/**
979	* drm_gpuvm_init() - initialize a &drm_gpuvm
980	* @gpuvm: pointer to the &drm_gpuvm to initialize
981	* @name: the name of the GPU VA space
982	* @flags: the &drm_gpuvm_flags for this GPUVM
983	* @drm: the &drm_device this VM resides in
984	* @r_obj: the resv &drm_gem_object providing the GPUVM's common &dma_resv
985	* @start_offset: the start offset of the GPU VA space
986	* @range: the size of the GPU VA space
987	* @reserve_offset: the start of the kernel reserved GPU VA area
988	* @reserve_range: the size of the kernel reserved GPU VA area
989	* @ops: &drm_gpuvm_ops called on &drm_gpuvm_sm_map / &drm_gpuvm_sm_unmap
990	*
991	* The &drm_gpuvm must be initialized with this function before use.
992	*
993	* Note that @gpuvm must be cleared to 0 before calling this function. The given
994	* &name is expected to be managed by the surrounding driver structures.
995	*/
996	void
997	drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
998	enum drm_gpuvm_flags flags,
999	struct drm_device *drm,
1000	struct drm_gem_object *r_obj,
1001	u64 start_offset, u64 range,
1002	u64 reserve_offset, u64 reserve_range,
1003	const struct drm_gpuvm_ops *ops)
1004	{
1005	gpuvm->rb.tree = RB_ROOT_CACHED;
1006	INIT_LIST_HEAD(list: &gpuvm->rb.list);
1007
1008	INIT_LIST_HEAD(list: &gpuvm->extobj.list);
1009	spin_lock_init(&gpuvm->extobj.lock);
1010
1011	INIT_LIST_HEAD(list: &gpuvm->evict.list);
1012	spin_lock_init(&gpuvm->evict.lock);
1013
1014	kref_init(kref: &gpuvm->kref);
1015
1016	gpuvm->name = name ? name : "unknown";
1017	gpuvm->flags = flags;
1018	gpuvm->ops = ops;
1019	gpuvm->drm = drm;
1020	gpuvm->r_obj = r_obj;
1021
1022	drm_gem_object_get(obj: r_obj);
1023
1024	drm_gpuvm_warn_check_overflow(gpuvm, addr: start_offset, range);
1025	gpuvm->mm_start = start_offset;
1026	gpuvm->mm_range = range;
1027
1028	memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva));
1029	if (reserve_range) {
1030	gpuvm->kernel_alloc_node.va.addr = reserve_offset;
1031	gpuvm->kernel_alloc_node.va.range = reserve_range;
1032
1033	if (likely(!drm_gpuvm_warn_check_overflow(gpuvm, reserve_offset,
1034	reserve_range)))
1035	__drm_gpuva_insert(gpuvm, va: &gpuvm->kernel_alloc_node);
1036	}
1037	}
1038	EXPORT_SYMBOL_GPL(drm_gpuvm_init);
1039
1040	static void
1041	drm_gpuvm_fini(struct drm_gpuvm *gpuvm)
1042	{
1043	gpuvm->name = NULL;
1044
1045	if (gpuvm->kernel_alloc_node.va.range)
1046	__drm_gpuva_remove(va: &gpuvm->kernel_alloc_node);
1047
1048	drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root),
1049	"GPUVA tree is not empty, potentially leaking memory.\n");
1050
1051	drm_WARN(gpuvm->drm, !list_empty(&gpuvm->extobj.list),
1052	"Extobj list should be empty.\n");
1053	drm_WARN(gpuvm->drm, !list_empty(&gpuvm->evict.list),
1054	"Evict list should be empty.\n");
1055
1056	drm_gem_object_put(obj: gpuvm->r_obj);
1057	}
1058
1059	static void
1060	drm_gpuvm_free(struct kref *kref)
1061	{
1062	struct drm_gpuvm *gpuvm = container_of(kref, struct drm_gpuvm, kref);
1063
1064	drm_gpuvm_fini(gpuvm);
1065
1066	if (drm_WARN_ON(gpuvm->drm, !gpuvm->ops->vm_free))
1067	return;
1068
1069	gpuvm->ops->vm_free(gpuvm);
1070	}
1071
1072	/**
1073	* drm_gpuvm_put() - drop a struct drm_gpuvm reference
1074	* @gpuvm: the &drm_gpuvm to release the reference of
1075	*
1076	* This releases a reference to @gpuvm.
1077	*
1078	* This function may be called from atomic context.
1079	*/
1080	void
1081	drm_gpuvm_put(struct drm_gpuvm *gpuvm)
1082	{
1083	if (gpuvm)
1084	kref_put(kref: &gpuvm->kref, release: drm_gpuvm_free);
1085	}
1086	EXPORT_SYMBOL_GPL(drm_gpuvm_put);
1087
1088	static int
1089	exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj,
1090	unsigned int num_fences)
1091	{
1092	return num_fences ? drm_exec_prepare_obj(exec, obj, num_fences) :
1093	drm_exec_lock_obj(exec, obj);
1094	}
1095
1096	/**
1097	* drm_gpuvm_prepare_vm() - prepare the GPUVMs common dma-resv
1098	* @gpuvm: the &drm_gpuvm
1099	* @exec: the &drm_exec context
1100	* @num_fences: the amount of &dma_fences to reserve
1101	*
1102	* Calls drm_exec_prepare_obj() for the GPUVMs dummy &drm_gem_object; if
1103	* @num_fences is zero drm_exec_lock_obj() is called instead.
1104	*
1105	* Using this function directly, it is the drivers responsibility to call
1106	* drm_exec_init() and drm_exec_fini() accordingly.
1107	*
1108	* Returns: 0 on success, negative error code on failure.
1109	*/
1110	int
1111	drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm,
1112	struct drm_exec *exec,
1113	unsigned int num_fences)
1114	{
1115	return exec_prepare_obj(exec, obj: gpuvm->r_obj, num_fences);
1116	}
1117	EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_vm);
1118
1119	static int
1120	__drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
1121	struct drm_exec *exec,
1122	unsigned int num_fences)
1123	{
1124	struct drm_gpuvm_bo *vm_bo;
1125	LIST_HEAD(extobjs);
1126	int ret = 0;
1127
1128	for_each_vm_bo_in_list(gpuvm, extobj, &extobjs, vm_bo) {
1129	ret = exec_prepare_obj(exec, obj: vm_bo->obj, num_fences);
1130	if (ret)
1131	break;
1132	}
1133	/* Drop ref in case we break out of the loop. */
1134	drm_gpuvm_bo_put(vm_bo);
1135	restore_vm_bo_list(gpuvm, extobj);
1136
1137	return ret;
1138	}
1139
1140	static int
1141	drm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm,
1142	struct drm_exec *exec,
1143	unsigned int num_fences)
1144	{
1145	struct drm_gpuvm_bo *vm_bo;
1146	int ret = 0;
1147
1148	drm_gpuvm_resv_assert_held(gpuvm);
1149	list_for_each_entry(vm_bo, &gpuvm->extobj.list, list.entry.extobj) {
1150	ret = exec_prepare_obj(exec, obj: vm_bo->obj, num_fences);
1151	if (ret)
1152	break;
1153
1154	if (vm_bo->evicted)
1155	drm_gpuvm_bo_list_add(vm_bo, evict, false);
1156	}
1157
1158	return ret;
1159	}
1160
1161	/**
1162	* drm_gpuvm_prepare_objects() - prepare all assoiciated BOs
1163	* @gpuvm: the &drm_gpuvm
1164	* @exec: the &drm_exec locking context
1165	* @num_fences: the amount of &dma_fences to reserve
1166	*
1167	* Calls drm_exec_prepare_obj() for all &drm_gem_objects the given
1168	* &drm_gpuvm contains mappings of; if @num_fences is zero drm_exec_lock_obj()
1169	* is called instead.
1170	*
1171	* Using this function directly, it is the drivers responsibility to call
1172	* drm_exec_init() and drm_exec_fini() accordingly.
1173	*
1174	* Note: This function is safe against concurrent insertion and removal of
1175	* external objects, however it is not safe against concurrent usage itself.
1176	*
1177	* Drivers need to make sure to protect this case with either an outer VM lock
1178	* or by calling drm_gpuvm_prepare_vm() before this function within the
1179	* drm_exec_until_all_locked() loop, such that the GPUVM's dma-resv lock ensures
1180	* mutual exclusion.
1181	*
1182	* Returns: 0 on success, negative error code on failure.
1183	*/
1184	int
1185	drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
1186	struct drm_exec *exec,
1187	unsigned int num_fences)
1188	{
1189	if (drm_gpuvm_resv_protected(gpuvm))
1190	return drm_gpuvm_prepare_objects_locked(gpuvm, exec,
1191	num_fences);
1192	else
1193	return __drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);
1194	}
1195	EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_objects);
1196
1197	/**
1198	* drm_gpuvm_prepare_range() - prepare all BOs mapped within a given range
1199	* @gpuvm: the &drm_gpuvm
1200	* @exec: the &drm_exec locking context
1201	* @addr: the start address within the VA space
1202	* @range: the range to iterate within the VA space
1203	* @num_fences: the amount of &dma_fences to reserve
1204	*
1205	* Calls drm_exec_prepare_obj() for all &drm_gem_objects mapped between @addr
1206	* and @addr + @range; if @num_fences is zero drm_exec_lock_obj() is called
1207	* instead.
1208	*
1209	* Returns: 0 on success, negative error code on failure.
1210	*/
1211	int
1212	drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec,
1213	u64 addr, u64 range, unsigned int num_fences)
1214	{
1215	struct drm_gpuva *va;
1216	u64 end = addr + range;
1217	int ret;
1218
1219	drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {
1220	struct drm_gem_object *obj = va->gem.obj;
1221
1222	ret = exec_prepare_obj(exec, obj, num_fences);
1223	if (ret)
1224	return ret;
1225	}
1226
1227	return 0;
1228	}
1229	EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_range);
1230
1231	/**
1232	* drm_gpuvm_exec_lock() - lock all dma-resv of all assoiciated BOs
1233	* @vm_exec: the &drm_gpuvm_exec wrapper
1234	*
1235	* Acquires all dma-resv locks of all &drm_gem_objects the given
1236	* &drm_gpuvm contains mappings of.
1237	*
1238	* Addionally, when calling this function with struct drm_gpuvm_exec::extra
1239	* being set the driver receives the given @fn callback to lock additional
1240	* dma-resv in the context of the &drm_gpuvm_exec instance. Typically, drivers
1241	* would call drm_exec_prepare_obj() from within this callback.
1242	*
1243	* Returns: 0 on success, negative error code on failure.
1244	*/
1245	int
1246	drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec)
1247	{
1248	struct drm_gpuvm *gpuvm = vm_exec->vm;
1249	struct drm_exec *exec = &vm_exec->exec;
1250	unsigned int num_fences = vm_exec->num_fences;
1251	int ret;
1252
1253	drm_exec_init(exec, flags: vm_exec->flags, nr: 0);
1254
1255	drm_exec_until_all_locked(exec) {
1256	ret = drm_gpuvm_prepare_vm(gpuvm, exec, num_fences);
1257	drm_exec_retry_on_contention(exec);
1258	if (ret)
1259	goto err;
1260
1261	ret = drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);
1262	drm_exec_retry_on_contention(exec);
1263	if (ret)
1264	goto err;
1265
1266	if (vm_exec->extra.fn) {
1267	ret = vm_exec->extra.fn(vm_exec);
1268	drm_exec_retry_on_contention(exec);
1269	if (ret)
1270	goto err;
1271	}
1272	}
1273
1274	return 0;
1275
1276	err:
1277	drm_exec_fini(exec);
1278	return ret;
1279	}
1280	EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock);
1281
1282	static int
1283	fn_lock_array(struct drm_gpuvm_exec *vm_exec)
1284	{
1285	struct {
1286	struct drm_gem_object **objs;
1287	unsigned int num_objs;
1288	} *args = vm_exec->extra.priv;
1289
1290	return drm_exec_prepare_array(exec: &vm_exec->exec, objects: args->objs,
1291	num_objects: args->num_objs, num_fences: vm_exec->num_fences);
1292	}
1293
1294	/**
1295	* drm_gpuvm_exec_lock_array() - lock all dma-resv of all assoiciated BOs
1296	* @vm_exec: the &drm_gpuvm_exec wrapper
1297	* @objs: additional &drm_gem_objects to lock
1298	* @num_objs: the number of additional &drm_gem_objects to lock
1299	*
1300	* Acquires all dma-resv locks of all &drm_gem_objects the given &drm_gpuvm
1301	* contains mappings of, plus the ones given through @objs.
1302	*
1303	* Returns: 0 on success, negative error code on failure.
1304	*/
1305	int
1306	drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec,
1307	struct drm_gem_object **objs,
1308	unsigned int num_objs)
1309	{
1310	struct {
1311	struct drm_gem_object **objs;
1312	unsigned int num_objs;
1313	} args;
1314
1315	args.objs = objs;
1316	args.num_objs = num_objs;
1317
1318	vm_exec->extra.fn = fn_lock_array;
1319	vm_exec->extra.priv = &args;
1320
1321	return drm_gpuvm_exec_lock(vm_exec);
1322	}
1323	EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_array);
1324
1325	/**
1326	* drm_gpuvm_exec_lock_range() - prepare all BOs mapped within a given range
1327	* @vm_exec: the &drm_gpuvm_exec wrapper
1328	* @addr: the start address within the VA space
1329	* @range: the range to iterate within the VA space
1330	*
1331	* Acquires all dma-resv locks of all &drm_gem_objects mapped between @addr and
1332	* @addr + @range.
1333	*
1334	* Returns: 0 on success, negative error code on failure.
1335	*/
1336	int
1337	drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec,
1338	u64 addr, u64 range)
1339	{
1340	struct drm_gpuvm *gpuvm = vm_exec->vm;
1341	struct drm_exec *exec = &vm_exec->exec;
1342	int ret;
1343
1344	drm_exec_init(exec, flags: vm_exec->flags, nr: 0);
1345
1346	drm_exec_until_all_locked(exec) {
1347	ret = drm_gpuvm_prepare_range(gpuvm, exec, addr, range,
1348	vm_exec->num_fences);
1349	drm_exec_retry_on_contention(exec);
1350	if (ret)
1351	goto err;
1352	}
1353
1354	return ret;
1355
1356	err:
1357	drm_exec_fini(exec);
1358	return ret;
1359	}
1360	EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_range);
1361
1362	static int
1363	__drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
1364	{
1365	const struct drm_gpuvm_ops *ops = gpuvm->ops;
1366	struct drm_gpuvm_bo *vm_bo;
1367	LIST_HEAD(evict);
1368	int ret = 0;
1369
1370	for_each_vm_bo_in_list(gpuvm, evict, &evict, vm_bo) {
1371	ret = ops->vm_bo_validate(vm_bo, exec);
1372	if (ret)
1373	break;
1374	}
1375	/* Drop ref in case we break out of the loop. */
1376	drm_gpuvm_bo_put(vm_bo);
1377	restore_vm_bo_list(gpuvm, evict);
1378
1379	return ret;
1380	}
1381
1382	static int
1383	drm_gpuvm_validate_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
1384	{
1385	const struct drm_gpuvm_ops *ops = gpuvm->ops;
1386	struct drm_gpuvm_bo *vm_bo, *next;
1387	int ret = 0;
1388
1389	drm_gpuvm_resv_assert_held(gpuvm);
1390
1391	list_for_each_entry_safe(vm_bo, next, &gpuvm->evict.list,
1392	list.entry.evict) {
1393	ret = ops->vm_bo_validate(vm_bo, exec);
1394	if (ret)
1395	break;
1396
1397	dma_resv_assert_held(vm_bo->obj->resv);
1398	if (!vm_bo->evicted)
1399	drm_gpuvm_bo_list_del_init(vm_bo, evict, false);
1400	}
1401
1402	return ret;
1403	}
1404
1405	/**
1406	* drm_gpuvm_validate() - validate all BOs marked as evicted
1407	* @gpuvm: the &drm_gpuvm to validate evicted BOs
1408	* @exec: the &drm_exec instance used for locking the GPUVM
1409	*
1410	* Calls the &drm_gpuvm_ops::vm_bo_validate callback for all evicted buffer
1411	* objects being mapped in the given &drm_gpuvm.
1412	*
1413	* Returns: 0 on success, negative error code on failure.
1414	*/
1415	int
1416	drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
1417	{
1418	const struct drm_gpuvm_ops *ops = gpuvm->ops;
1419
1420	if (unlikely(!ops || !ops->vm_bo_validate))
1421	return -EOPNOTSUPP;
1422
1423	if (drm_gpuvm_resv_protected(gpuvm))
1424	return drm_gpuvm_validate_locked(gpuvm, exec);
1425	else
1426	return __drm_gpuvm_validate(gpuvm, exec);
1427	}
1428	EXPORT_SYMBOL_GPL(drm_gpuvm_validate);
1429
1430	/**
1431	* drm_gpuvm_resv_add_fence - add fence to private and all extobj
1432	* dma-resv
1433	* @gpuvm: the &drm_gpuvm to add a fence to
1434	* @exec: the &drm_exec locking context
1435	* @fence: fence to add
1436	* @private_usage: private dma-resv usage
1437	* @extobj_usage: extobj dma-resv usage
1438	*/
1439	void
1440	drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm,
1441	struct drm_exec *exec,
1442	struct dma_fence *fence,
1443	enum dma_resv_usage private_usage,
1444	enum dma_resv_usage extobj_usage)
1445	{
1446	struct drm_gem_object *obj;
1447	unsigned long index;
1448
1449	drm_exec_for_each_locked_object(exec, index, obj) {
1450	dma_resv_assert_held(obj->resv);
1451	dma_resv_add_fence(obj: obj->resv, fence,
1452	usage: drm_gpuvm_is_extobj(gpuvm, obj) ?
1453	extobj_usage : private_usage);
1454	}
1455	}
1456	EXPORT_SYMBOL_GPL(drm_gpuvm_resv_add_fence);
1457
1458	/**
1459	* drm_gpuvm_bo_create() - create a new instance of struct drm_gpuvm_bo
1460	* @gpuvm: The &drm_gpuvm the @obj is mapped in.
1461	* @obj: The &drm_gem_object being mapped in the @gpuvm.
1462	*
1463	* If provided by the driver, this function uses the &drm_gpuvm_ops
1464	* vm_bo_alloc() callback to allocate.
1465	*
1466	* Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure
1467	*/
1468	struct drm_gpuvm_bo *
1469	drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm,
1470	struct drm_gem_object *obj)
1471	{
1472	const struct drm_gpuvm_ops *ops = gpuvm->ops;
1473	struct drm_gpuvm_bo *vm_bo;
1474
1475	if (ops && ops->vm_bo_alloc)
1476	vm_bo = ops->vm_bo_alloc();
1477	else
1478	vm_bo = kzalloc(size: sizeof(*vm_bo), GFP_KERNEL);
1479
1480	if (unlikely(!vm_bo))
1481	return NULL;
1482
1483	vm_bo->vm = drm_gpuvm_get(gpuvm);
1484	vm_bo->obj = obj;
1485	drm_gem_object_get(obj);
1486
1487	kref_init(kref: &vm_bo->kref);
1488	INIT_LIST_HEAD(list: &vm_bo->list.gpuva);
1489	INIT_LIST_HEAD(list: &vm_bo->list.entry.gem);
1490
1491	INIT_LIST_HEAD(list: &vm_bo->list.entry.extobj);
1492	INIT_LIST_HEAD(list: &vm_bo->list.entry.evict);
1493
1494	return vm_bo;
1495	}
1496	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_create);
1497
1498	static void
1499	drm_gpuvm_bo_destroy(struct kref *kref)
1500	{
1501	struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo,
1502	kref);
1503	struct drm_gpuvm *gpuvm = vm_bo->vm;
1504	const struct drm_gpuvm_ops *ops = gpuvm->ops;
1505	struct drm_gem_object *obj = vm_bo->obj;
1506	bool lock = !drm_gpuvm_resv_protected(gpuvm);
1507
1508	if (!lock)
1509	drm_gpuvm_resv_assert_held(gpuvm);
1510
1511	drm_gpuvm_bo_list_del(vm_bo, extobj, lock);
1512	drm_gpuvm_bo_list_del(vm_bo, evict, lock);
1513
1514	drm_gem_gpuva_assert_lock_held(obj);
1515	list_del(entry: &vm_bo->list.entry.gem);
1516
1517	if (ops && ops->vm_bo_free)
1518	ops->vm_bo_free(vm_bo);
1519	else
1520	kfree(objp: vm_bo);
1521
1522	drm_gpuvm_put(gpuvm);
1523	drm_gem_object_put(obj);
1524	}
1525
1526	/**
1527	* drm_gpuvm_bo_put() - drop a struct drm_gpuvm_bo reference
1528	* @vm_bo: the &drm_gpuvm_bo to release the reference of
1529	*
1530	* This releases a reference to @vm_bo.
1531	*
1532	* If the reference count drops to zero, the &gpuvm_bo is destroyed, which
1533	* includes removing it from the GEMs gpuva list. Hence, if a call to this
1534	* function can potentially let the reference count drop to zero the caller must
1535	* hold the dma-resv or driver specific GEM gpuva lock.
1536	*
1537	* This function may only be called from non-atomic context.
1538	*
1539	* Returns: true if vm_bo was destroyed, false otherwise.
1540	*/
1541	bool
1542	drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo)
1543	{
1544	might_sleep();
1545
1546	if (vm_bo)
1547	return !!kref_put(kref: &vm_bo->kref, release: drm_gpuvm_bo_destroy);
1548
1549	return false;
1550	}
1551	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put);
1552
1553	static struct drm_gpuvm_bo *
1554	__drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
1555	struct drm_gem_object *obj)
1556	{
1557	struct drm_gpuvm_bo *vm_bo;
1558
1559	drm_gem_gpuva_assert_lock_held(obj);
1560	drm_gem_for_each_gpuvm_bo(vm_bo, obj)
1561	if (vm_bo->vm == gpuvm)
1562	return vm_bo;
1563
1564	return NULL;
1565	}
1566
1567	/**
1568	* drm_gpuvm_bo_find() - find the &drm_gpuvm_bo for the given
1569	* &drm_gpuvm and &drm_gem_object
1570	* @gpuvm: The &drm_gpuvm the @obj is mapped in.
1571	* @obj: The &drm_gem_object being mapped in the @gpuvm.
1572	*
1573	* Find the &drm_gpuvm_bo representing the combination of the given
1574	* &drm_gpuvm and &drm_gem_object. If found, increases the reference
1575	* count of the &drm_gpuvm_bo accordingly.
1576	*
1577	* Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure
1578	*/
1579	struct drm_gpuvm_bo *
1580	drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
1581	struct drm_gem_object *obj)
1582	{
1583	struct drm_gpuvm_bo *vm_bo = __drm_gpuvm_bo_find(gpuvm, obj);
1584
1585	return vm_bo ? drm_gpuvm_bo_get(vm_bo) : NULL;
1586	}
1587	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_find);
1588
1589	/**
1590	* drm_gpuvm_bo_obtain() - obtains and instance of the &drm_gpuvm_bo for the
1591	* given &drm_gpuvm and &drm_gem_object
1592	* @gpuvm: The &drm_gpuvm the @obj is mapped in.
1593	* @obj: The &drm_gem_object being mapped in the @gpuvm.
1594	*
1595	* Find the &drm_gpuvm_bo representing the combination of the given
1596	* &drm_gpuvm and &drm_gem_object. If found, increases the reference
1597	* count of the &drm_gpuvm_bo accordingly. If not found, allocates a new
1598	* &drm_gpuvm_bo.
1599	*
1600	* A new &drm_gpuvm_bo is added to the GEMs gpuva list.
1601	*
1602	* Returns: a pointer to the &drm_gpuvm_bo on success, an ERR_PTR on failure
1603	*/
1604	struct drm_gpuvm_bo *
1605	drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm,
1606	struct drm_gem_object *obj)
1607	{
1608	struct drm_gpuvm_bo *vm_bo;
1609
1610	vm_bo = drm_gpuvm_bo_find(gpuvm, obj);
1611	if (vm_bo)
1612	return vm_bo;
1613
1614	vm_bo = drm_gpuvm_bo_create(gpuvm, obj);
1615	if (!vm_bo)
1616	return ERR_PTR(error: -ENOMEM);
1617
1618	drm_gem_gpuva_assert_lock_held(obj);
1619	list_add_tail(new: &vm_bo->list.entry.gem, head: &obj->gpuva.list);
1620
1621	return vm_bo;
1622	}
1623	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain);
1624
1625	/**
1626	* drm_gpuvm_bo_obtain_prealloc() - obtains and instance of the &drm_gpuvm_bo
1627	* for the given &drm_gpuvm and &drm_gem_object
1628	* @__vm_bo: A pre-allocated struct drm_gpuvm_bo.
1629	*
1630	* Find the &drm_gpuvm_bo representing the combination of the given
1631	* &drm_gpuvm and &drm_gem_object. If found, increases the reference
1632	* count of the found &drm_gpuvm_bo accordingly, while the @__vm_bo reference
1633	* count is decreased. If not found @__vm_bo is returned without further
1634	* increase of the reference count.
1635	*
1636	* A new &drm_gpuvm_bo is added to the GEMs gpuva list.
1637	*
1638	* Returns: a pointer to the found &drm_gpuvm_bo or @__vm_bo if no existing
1639	* &drm_gpuvm_bo was found
1640	*/
1641	struct drm_gpuvm_bo *
1642	drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *__vm_bo)
1643	{
1644	struct drm_gpuvm *gpuvm = __vm_bo->vm;
1645	struct drm_gem_object *obj = __vm_bo->obj;
1646	struct drm_gpuvm_bo *vm_bo;
1647
1648	vm_bo = drm_gpuvm_bo_find(gpuvm, obj);
1649	if (vm_bo) {
1650	drm_gpuvm_bo_put(__vm_bo);
1651	return vm_bo;
1652	}
1653
1654	drm_gem_gpuva_assert_lock_held(obj);
1655	list_add_tail(new: &__vm_bo->list.entry.gem, head: &obj->gpuva.list);
1656
1657	return __vm_bo;
1658	}
1659	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_prealloc);
1660
1661	/**
1662	* drm_gpuvm_bo_extobj_add() - adds the &drm_gpuvm_bo to its &drm_gpuvm's
1663	* extobj list
1664	* @vm_bo: The &drm_gpuvm_bo to add to its &drm_gpuvm's the extobj list.
1665	*
1666	* Adds the given @vm_bo to its &drm_gpuvm's extobj list if not on the list
1667	* already and if the corresponding &drm_gem_object is an external object,
1668	* actually.
1669	*/
1670	void
1671	drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo)
1672	{
1673	struct drm_gpuvm *gpuvm = vm_bo->vm;
1674	bool lock = !drm_gpuvm_resv_protected(gpuvm);
1675
1676	if (!lock)
1677	drm_gpuvm_resv_assert_held(gpuvm);
1678
1679	if (drm_gpuvm_is_extobj(gpuvm, obj: vm_bo->obj))
1680	drm_gpuvm_bo_list_add(vm_bo, extobj, lock);
1681	}
1682	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_extobj_add);
1683
1684	/**
1685	* drm_gpuvm_bo_evict() - add / remove a &drm_gpuvm_bo to / from the &drm_gpuvms
1686	* evicted list
1687	* @vm_bo: the &drm_gpuvm_bo to add or remove
1688	* @evict: indicates whether the object is evicted
1689	*
1690	* Adds a &drm_gpuvm_bo to or removes it from the &drm_gpuvms evicted list.
1691	*/
1692	void
1693	drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict)
1694	{
1695	struct drm_gpuvm *gpuvm = vm_bo->vm;
1696	struct drm_gem_object *obj = vm_bo->obj;
1697	bool lock = !drm_gpuvm_resv_protected(gpuvm);
1698
1699	dma_resv_assert_held(obj->resv);
1700	vm_bo->evicted = evict;
1701
1702	/* Can't add external objects to the evicted list directly if not using
1703	* internal spinlocks, since in this case the evicted list is protected
1704	* with the VM's common dma-resv lock.
1705	*/
1706	if (drm_gpuvm_is_extobj(gpuvm, obj) && !lock)
1707	return;
1708
1709	if (evict)
1710	drm_gpuvm_bo_list_add(vm_bo, evict, lock);
1711	else
1712	drm_gpuvm_bo_list_del_init(vm_bo, evict, lock);
1713	}
1714	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_evict);
1715
1716	static int
1717	__drm_gpuva_insert(struct drm_gpuvm *gpuvm,
1718	struct drm_gpuva *va)
1719	{
1720	struct rb_node *node;
1721	struct list_head *head;
1722
1723	if (drm_gpuva_it_iter_first(root: &gpuvm->rb.tree,
1724	GPUVA_START(va),
1725	GPUVA_LAST(va)))
1726	return -EEXIST;
1727
1728	va->vm = gpuvm;
1729
1730	drm_gpuva_it_insert(node: va, root: &gpuvm->rb.tree);
1731
1732	node = rb_prev(&va->rb.node);
1733	if (node)
1734	head = &(to_drm_gpuva(node))->rb.entry;
1735	else
1736	head = &gpuvm->rb.list;
1737
1738	list_add(new: &va->rb.entry, head);
1739
1740	return 0;
1741	}
1742
1743	/**
1744	* drm_gpuva_insert() - insert a &drm_gpuva
1745	* @gpuvm: the &drm_gpuvm to insert the &drm_gpuva in
1746	* @va: the &drm_gpuva to insert
1747	*
1748	* Insert a &drm_gpuva with a given address and range into a
1749	* &drm_gpuvm.
1750	*
1751	* It is safe to use this function using the safe versions of iterating the GPU
1752	* VA space, such as drm_gpuvm_for_each_va_safe() and
1753	* drm_gpuvm_for_each_va_range_safe().
1754	*
1755	* Returns: 0 on success, negative error code on failure.
1756	*/
1757	int
1758	drm_gpuva_insert(struct drm_gpuvm *gpuvm,
1759	struct drm_gpuva *va)
1760	{
1761	u64 addr = va->va.addr;
1762	u64 range = va->va.range;
1763	int ret;
1764
1765	if (unlikely(!drm_gpuvm_range_valid(gpuvm, addr, range)))
1766	return -EINVAL;
1767
1768	ret = __drm_gpuva_insert(gpuvm, va);
1769	if (likely(!ret))
1770	/* Take a reference of the GPUVM for the successfully inserted
1771	* drm_gpuva. We can't take the reference in
1772	* __drm_gpuva_insert() itself, since we don't want to increse
1773	* the reference count for the GPUVM's kernel_alloc_node.
1774	*/
1775	drm_gpuvm_get(gpuvm);
1776
1777	return ret;
1778	}
1779	EXPORT_SYMBOL_GPL(drm_gpuva_insert);
1780
1781	static void
1782	__drm_gpuva_remove(struct drm_gpuva *va)
1783	{
1784	drm_gpuva_it_remove(node: va, root: &va->vm->rb.tree);
1785	list_del_init(entry: &va->rb.entry);
1786	}
1787
1788	/**
1789	* drm_gpuva_remove() - remove a &drm_gpuva
1790	* @va: the &drm_gpuva to remove
1791	*
1792	* This removes the given &va from the underlaying tree.
1793	*
1794	* It is safe to use this function using the safe versions of iterating the GPU
1795	* VA space, such as drm_gpuvm_for_each_va_safe() and
1796	* drm_gpuvm_for_each_va_range_safe().
1797	*/
1798	void
1799	drm_gpuva_remove(struct drm_gpuva *va)
1800	{
1801	struct drm_gpuvm *gpuvm = va->vm;
1802
1803	if (unlikely(va == &gpuvm->kernel_alloc_node)) {
1804	drm_WARN(gpuvm->drm, 1,
1805	"Can't destroy kernel reserved node.\n");
1806	return;
1807	}
1808
1809	__drm_gpuva_remove(va);
1810	drm_gpuvm_put(va->vm);
1811	}
1812	EXPORT_SYMBOL_GPL(drm_gpuva_remove);
1813
1814	/**
1815	* drm_gpuva_link() - link a &drm_gpuva
1816	* @va: the &drm_gpuva to link
1817	* @vm_bo: the &drm_gpuvm_bo to add the &drm_gpuva to
1818	*
1819	* This adds the given &va to the GPU VA list of the &drm_gpuvm_bo and the
1820	* &drm_gpuvm_bo to the &drm_gem_object it is associated with.
1821	*
1822	* For every &drm_gpuva entry added to the &drm_gpuvm_bo an additional
1823	* reference of the latter is taken.
1824	*
1825	* This function expects the caller to protect the GEM's GPUVA list against
1826	* concurrent access using either the GEMs dma_resv lock or a driver specific
1827	* lock set through drm_gem_gpuva_set_lock().
1828	*/
1829	void
1830	drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo)
1831	{
1832	struct drm_gem_object *obj = va->gem.obj;
1833	struct drm_gpuvm *gpuvm = va->vm;
1834
1835	if (unlikely(!obj))
1836	return;
1837
1838	drm_WARN_ON(gpuvm->drm, obj != vm_bo->obj);
1839
1840	va->vm_bo = drm_gpuvm_bo_get(vm_bo);
1841
1842	drm_gem_gpuva_assert_lock_held(obj);
1843	list_add_tail(new: &va->gem.entry, head: &vm_bo->list.gpuva);
1844	}
1845	EXPORT_SYMBOL_GPL(drm_gpuva_link);
1846
1847	/**
1848	* drm_gpuva_unlink() - unlink a &drm_gpuva
1849	* @va: the &drm_gpuva to unlink
1850	*
1851	* This removes the given &va from the GPU VA list of the &drm_gem_object it is
1852	* associated with.
1853	*
1854	* This removes the given &va from the GPU VA list of the &drm_gpuvm_bo and
1855	* the &drm_gpuvm_bo from the &drm_gem_object it is associated with in case
1856	* this call unlinks the last &drm_gpuva from the &drm_gpuvm_bo.
1857	*
1858	* For every &drm_gpuva entry removed from the &drm_gpuvm_bo a reference of
1859	* the latter is dropped.
1860	*
1861	* This function expects the caller to protect the GEM's GPUVA list against
1862	* concurrent access using either the GEMs dma_resv lock or a driver specific
1863	* lock set through drm_gem_gpuva_set_lock().
1864	*/
1865	void
1866	drm_gpuva_unlink(struct drm_gpuva *va)
1867	{
1868	struct drm_gem_object *obj = va->gem.obj;
1869	struct drm_gpuvm_bo *vm_bo = va->vm_bo;
1870
1871	if (unlikely(!obj))
1872	return;
1873
1874	drm_gem_gpuva_assert_lock_held(obj);
1875	list_del_init(entry: &va->gem.entry);
1876
1877	va->vm_bo = NULL;
1878	drm_gpuvm_bo_put(vm_bo);
1879	}
1880	EXPORT_SYMBOL_GPL(drm_gpuva_unlink);
1881
1882	/**
1883	* drm_gpuva_find_first() - find the first &drm_gpuva in the given range
1884	* @gpuvm: the &drm_gpuvm to search in
1885	* @addr: the &drm_gpuvas address
1886	* @range: the &drm_gpuvas range
1887	*
1888	* Returns: the first &drm_gpuva within the given range
1889	*/
1890	struct drm_gpuva *
1891	drm_gpuva_find_first(struct drm_gpuvm *gpuvm,
1892	u64 addr, u64 range)
1893	{
1894	u64 last = addr + range - 1;
1895
1896	return drm_gpuva_it_iter_first(root: &gpuvm->rb.tree, start: addr, last);
1897	}
1898	EXPORT_SYMBOL_GPL(drm_gpuva_find_first);
1899
1900	/**
1901	* drm_gpuva_find() - find a &drm_gpuva
1902	* @gpuvm: the &drm_gpuvm to search in
1903	* @addr: the &drm_gpuvas address
1904	* @range: the &drm_gpuvas range
1905	*
1906	* Returns: the &drm_gpuva at a given &addr and with a given &range
1907	*/
1908	struct drm_gpuva *
1909	drm_gpuva_find(struct drm_gpuvm *gpuvm,
1910	u64 addr, u64 range)
1911	{
1912	struct drm_gpuva *va;
1913
1914	va = drm_gpuva_find_first(gpuvm, addr, range);
1915	if (!va)
1916	goto out;
1917
1918	if (va->va.addr != addr ||
1919	va->va.range != range)
1920	goto out;
1921
1922	return va;
1923
1924	out:
1925	return NULL;
1926	}
1927	EXPORT_SYMBOL_GPL(drm_gpuva_find);
1928
1929	/**
1930	* drm_gpuva_find_prev() - find the &drm_gpuva before the given address
1931	* @gpuvm: the &drm_gpuvm to search in
1932	* @start: the given GPU VA's start address
1933	*
1934	* Find the adjacent &drm_gpuva before the GPU VA with given &start address.
1935	*
1936	* Note that if there is any free space between the GPU VA mappings no mapping
1937	* is returned.
1938	*
1939	* Returns: a pointer to the found &drm_gpuva or NULL if none was found
1940	*/
1941	struct drm_gpuva *
1942	drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start)
1943	{
1944	if (!drm_gpuvm_range_valid(gpuvm, start - 1, 1))
1945	return NULL;
1946
1947	return drm_gpuva_it_iter_first(root: &gpuvm->rb.tree, start: start - 1, last: start);
1948	}
1949	EXPORT_SYMBOL_GPL(drm_gpuva_find_prev);
1950
1951	/**
1952	* drm_gpuva_find_next() - find the &drm_gpuva after the given address
1953	* @gpuvm: the &drm_gpuvm to search in
1954	* @end: the given GPU VA's end address
1955	*
1956	* Find the adjacent &drm_gpuva after the GPU VA with given &end address.
1957	*
1958	* Note that if there is any free space between the GPU VA mappings no mapping
1959	* is returned.
1960	*
1961	* Returns: a pointer to the found &drm_gpuva or NULL if none was found
1962	*/
1963	struct drm_gpuva *
1964	drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end)
1965	{
1966	if (!drm_gpuvm_range_valid(gpuvm, end, 1))
1967	return NULL;
1968
1969	return drm_gpuva_it_iter_first(root: &gpuvm->rb.tree, start: end, last: end + 1);
1970	}
1971	EXPORT_SYMBOL_GPL(drm_gpuva_find_next);
1972
1973	/**
1974	* drm_gpuvm_interval_empty() - indicate whether a given interval of the VA space
1975	* is empty
1976	* @gpuvm: the &drm_gpuvm to check the range for
1977	* @addr: the start address of the range
1978	* @range: the range of the interval
1979	*
1980	* Returns: true if the interval is empty, false otherwise
1981	*/
1982	bool
1983	drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
1984	{
1985	return !drm_gpuva_find_first(gpuvm, addr, range);
1986	}
1987	EXPORT_SYMBOL_GPL(drm_gpuvm_interval_empty);
1988
1989	/**
1990	* drm_gpuva_map() - helper to insert a &drm_gpuva according to a
1991	* &drm_gpuva_op_map
1992	* @gpuvm: the &drm_gpuvm
1993	* @va: the &drm_gpuva to insert
1994	* @op: the &drm_gpuva_op_map to initialize @va with
1995	*
1996	* Initializes the @va from the @op and inserts it into the given @gpuvm.
1997	*/
1998	void
1999	drm_gpuva_map(struct drm_gpuvm *gpuvm,
2000	struct drm_gpuva *va,
2001	struct drm_gpuva_op_map *op)
2002	{
2003	drm_gpuva_init_from_op(va, op);
2004	drm_gpuva_insert(gpuvm, va);
2005	}
2006	EXPORT_SYMBOL_GPL(drm_gpuva_map);
2007
2008	/**
2009	* drm_gpuva_remap() - helper to remap a &drm_gpuva according to a
2010	* &drm_gpuva_op_remap
2011	* @prev: the &drm_gpuva to remap when keeping the start of a mapping
2012	* @next: the &drm_gpuva to remap when keeping the end of a mapping
2013	* @op: the &drm_gpuva_op_remap to initialize @prev and @next with
2014	*
2015	* Removes the currently mapped &drm_gpuva and remaps it using @prev and/or
2016	* @next.
2017	*/
2018	void
2019	drm_gpuva_remap(struct drm_gpuva *prev,
2020	struct drm_gpuva *next,
2021	struct drm_gpuva_op_remap *op)
2022	{
2023	struct drm_gpuva *va = op->unmap->va;
2024	struct drm_gpuvm *gpuvm = va->vm;
2025
2026	drm_gpuva_remove(va);
2027
2028	if (op->prev) {
2029	drm_gpuva_init_from_op(va: prev, op: op->prev);
2030	drm_gpuva_insert(gpuvm, prev);
2031	}
2032
2033	if (op->next) {
2034	drm_gpuva_init_from_op(va: next, op: op->next);
2035	drm_gpuva_insert(gpuvm, next);
2036	}
2037	}
2038	EXPORT_SYMBOL_GPL(drm_gpuva_remap);
2039
2040	/**
2041	* drm_gpuva_unmap() - helper to remove a &drm_gpuva according to a
2042	* &drm_gpuva_op_unmap
2043	* @op: the &drm_gpuva_op_unmap specifying the &drm_gpuva to remove
2044	*
2045	* Removes the &drm_gpuva associated with the &drm_gpuva_op_unmap.
2046	*/
2047	void
2048	drm_gpuva_unmap(struct drm_gpuva_op_unmap *op)
2049	{
2050	drm_gpuva_remove(op->va);
2051	}
2052	EXPORT_SYMBOL_GPL(drm_gpuva_unmap);
2053
2054	static int
2055	op_map_cb(const struct drm_gpuvm_ops *fn, void *priv,
2056	u64 addr, u64 range,
2057	struct drm_gem_object *obj, u64 offset)
2058	{
2059	struct drm_gpuva_op op = {};
2060
2061	op.op = DRM_GPUVA_OP_MAP;
2062	op.map.va.addr = addr;
2063	op.map.va.range = range;
2064	op.map.gem.obj = obj;
2065	op.map.gem.offset = offset;
2066
2067	return fn->sm_step_map(&op, priv);
2068	}
2069
2070	static int
2071	op_remap_cb(const struct drm_gpuvm_ops *fn, void *priv,
2072	struct drm_gpuva_op_map *prev,
2073	struct drm_gpuva_op_map *next,
2074	struct drm_gpuva_op_unmap *unmap)
2075	{
2076	struct drm_gpuva_op op = {};
2077	struct drm_gpuva_op_remap *r;
2078
2079	op.op = DRM_GPUVA_OP_REMAP;
2080	r = &op.remap;
2081	r->prev = prev;
2082	r->next = next;
2083	r->unmap = unmap;
2084
2085	return fn->sm_step_remap(&op, priv);
2086	}
2087
2088	static int
2089	op_unmap_cb(const struct drm_gpuvm_ops *fn, void *priv,
2090	struct drm_gpuva *va, bool merge)
2091	{
2092	struct drm_gpuva_op op = {};
2093
2094	op.op = DRM_GPUVA_OP_UNMAP;
2095	op.unmap.va = va;
2096	op.unmap.keep = merge;
2097
2098	return fn->sm_step_unmap(&op, priv);
2099	}
2100
2101	static int
2102	__drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm,
2103	const struct drm_gpuvm_ops *ops, void *priv,
2104	u64 req_addr, u64 req_range,
2105	struct drm_gem_object *req_obj, u64 req_offset)
2106	{
2107	struct drm_gpuva *va, *next;
2108	u64 req_end = req_addr + req_range;
2109	int ret;
2110
2111	if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range)))
2112	return -EINVAL;
2113
2114	drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) {
2115	struct drm_gem_object *obj = va->gem.obj;
2116	u64 offset = va->gem.offset;
2117	u64 addr = va->va.addr;
2118	u64 range = va->va.range;
2119	u64 end = addr + range;
2120	bool merge = !!va->gem.obj;
2121
2122	if (addr == req_addr) {
2123	merge &= obj == req_obj &&
2124	offset == req_offset;
2125
2126	if (end == req_end) {
2127	ret = op_unmap_cb(fn: ops, priv, va, merge);
2128	if (ret)
2129	return ret;
2130	break;
2131	}
2132
2133	if (end < req_end) {
2134	ret = op_unmap_cb(fn: ops, priv, va, merge);
2135	if (ret)
2136	return ret;
2137	continue;
2138	}
2139
2140	if (end > req_end) {
2141	struct drm_gpuva_op_map n = {
2142	.va.addr = req_end,
2143	.va.range = range - req_range,
2144	.gem.obj = obj,
2145	.gem.offset = offset + req_range,
2146	};
2147	struct drm_gpuva_op_unmap u = {
2148	.va = va,
2149	.keep = merge,
2150	};
2151
2152	ret = op_remap_cb(fn: ops, priv, NULL, next: &n, unmap: &u);
2153	if (ret)
2154	return ret;
2155	break;
2156	}
2157	} else if (addr < req_addr) {
2158	u64 ls_range = req_addr - addr;
2159	struct drm_gpuva_op_map p = {
2160	.va.addr = addr,
2161	.va.range = ls_range,
2162	.gem.obj = obj,
2163	.gem.offset = offset,
2164	};
2165	struct drm_gpuva_op_unmap u = { .va = va };
2166
2167	merge &= obj == req_obj &&
2168	offset + ls_range == req_offset;
2169	u.keep = merge;
2170
2171	if (end == req_end) {
2172	ret = op_remap_cb(fn: ops, priv, prev: &p, NULL, unmap: &u);
2173	if (ret)
2174	return ret;
2175	break;
2176	}
2177
2178	if (end < req_end) {
2179	ret = op_remap_cb(fn: ops, priv, prev: &p, NULL, unmap: &u);
2180	if (ret)
2181	return ret;
2182	continue;
2183	}
2184
2185	if (end > req_end) {
2186	struct drm_gpuva_op_map n = {
2187	.va.addr = req_end,
2188	.va.range = end - req_end,
2189	.gem.obj = obj,
2190	.gem.offset = offset + ls_range +
2191	req_range,
2192	};
2193
2194	ret = op_remap_cb(fn: ops, priv, prev: &p, next: &n, unmap: &u);
2195	if (ret)
2196	return ret;
2197	break;
2198	}
2199	} else if (addr > req_addr) {
2200	merge &= obj == req_obj &&
2201	offset == req_offset +
2202	(addr - req_addr);
2203
2204	if (end == req_end) {
2205	ret = op_unmap_cb(fn: ops, priv, va, merge);
2206	if (ret)
2207	return ret;
2208	break;
2209	}
2210
2211	if (end < req_end) {
2212	ret = op_unmap_cb(fn: ops, priv, va, merge);
2213	if (ret)
2214	return ret;
2215	continue;
2216	}
2217
2218	if (end > req_end) {
2219	struct drm_gpuva_op_map n = {
2220	.va.addr = req_end,
2221	.va.range = end - req_end,
2222	.gem.obj = obj,
2223	.gem.offset = offset + req_end - addr,
2224	};
2225	struct drm_gpuva_op_unmap u = {
2226	.va = va,
2227	.keep = merge,
2228	};
2229
2230	ret = op_remap_cb(fn: ops, priv, NULL, next: &n, unmap: &u);
2231	if (ret)
2232	return ret;
2233	break;
2234	}
2235	}
2236	}
2237
2238	return op_map_cb(fn: ops, priv,
2239	addr: req_addr, range: req_range,
2240	obj: req_obj, offset: req_offset);
2241	}
2242
2243	static int
2244	__drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm,
2245	const struct drm_gpuvm_ops *ops, void *priv,
2246	u64 req_addr, u64 req_range)
2247	{
2248	struct drm_gpuva *va, *next;
2249	u64 req_end = req_addr + req_range;
2250	int ret;
2251
2252	if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range)))
2253	return -EINVAL;
2254
2255	drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) {
2256	struct drm_gpuva_op_map prev = {}, next = {};
2257	bool prev_split = false, next_split = false;
2258	struct drm_gem_object *obj = va->gem.obj;
2259	u64 offset = va->gem.offset;
2260	u64 addr = va->va.addr;
2261	u64 range = va->va.range;
2262	u64 end = addr + range;
2263
2264	if (addr < req_addr) {
2265	prev.va.addr = addr;
2266	prev.va.range = req_addr - addr;
2267	prev.gem.obj = obj;
2268	prev.gem.offset = offset;
2269
2270	prev_split = true;
2271	}
2272
2273	if (end > req_end) {
2274	next.va.addr = req_end;
2275	next.va.range = end - req_end;
2276	next.gem.obj = obj;
2277	next.gem.offset = offset + (req_end - addr);
2278
2279	next_split = true;
2280	}
2281
2282	if (prev_split || next_split) {
2283	struct drm_gpuva_op_unmap unmap = { .va = va };
2284
2285	ret = op_remap_cb(fn: ops, priv,
2286	prev: prev_split ? &prev : NULL,
2287	next: next_split ? &next : NULL,
2288	unmap: &unmap);
2289	if (ret)
2290	return ret;
2291	} else {
2292	ret = op_unmap_cb(fn: ops, priv, va, merge: false);
2293	if (ret)
2294	return ret;
2295	}
2296	}
2297
2298	return 0;
2299	}
2300
2301	/**
2302	* drm_gpuvm_sm_map() - creates the &drm_gpuva_op split/merge steps
2303	* @gpuvm: the &drm_gpuvm representing the GPU VA space
2304	* @req_addr: the start address of the new mapping
2305	* @req_range: the range of the new mapping
2306	* @req_obj: the &drm_gem_object to map
2307	* @req_offset: the offset within the &drm_gem_object
2308	* @priv: pointer to a driver private data structure
2309	*
2310	* This function iterates the given range of the GPU VA space. It utilizes the
2311	* &drm_gpuvm_ops to call back into the driver providing the split and merge
2312	* steps.
2313	*
2314	* Drivers may use these callbacks to update the GPU VA space right away within
2315	* the callback. In case the driver decides to copy and store the operations for
2316	* later processing neither this function nor &drm_gpuvm_sm_unmap is allowed to
2317	* be called before the &drm_gpuvm's view of the GPU VA space was
2318	* updated with the previous set of operations. To update the
2319	* &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),
2320	* drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
2321	* used.
2322	*
2323	* A sequence of callbacks can contain map, unmap and remap operations, but
2324	* the sequence of callbacks might also be empty if no operation is required,
2325	* e.g. if the requested mapping already exists in the exact same way.
2326	*
2327	* There can be an arbitrary amount of unmap operations, a maximum of two remap
2328	* operations and a single map operation. The latter one represents the original
2329	* map operation requested by the caller.
2330	*
2331	* Returns: 0 on success or a negative error code
2332	*/
2333	int
2334	drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv,
2335	u64 req_addr, u64 req_range,
2336	struct drm_gem_object *req_obj, u64 req_offset)
2337	{
2338	const struct drm_gpuvm_ops *ops = gpuvm->ops;
2339
2340	if (unlikely(!(ops && ops->sm_step_map &&
2341	ops->sm_step_remap &&
2342	ops->sm_step_unmap)))
2343	return -EINVAL;
2344
2345	return __drm_gpuvm_sm_map(gpuvm, ops, priv,
2346	req_addr, req_range,
2347	req_obj, req_offset);
2348	}
2349	EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map);
2350
2351	/**
2352	* drm_gpuvm_sm_unmap() - creates the &drm_gpuva_ops to split on unmap
2353	* @gpuvm: the &drm_gpuvm representing the GPU VA space
2354	* @priv: pointer to a driver private data structure
2355	* @req_addr: the start address of the range to unmap
2356	* @req_range: the range of the mappings to unmap
2357	*
2358	* This function iterates the given range of the GPU VA space. It utilizes the
2359	* &drm_gpuvm_ops to call back into the driver providing the operations to
2360	* unmap and, if required, split existent mappings.
2361	*
2362	* Drivers may use these callbacks to update the GPU VA space right away within
2363	* the callback. In case the driver decides to copy and store the operations for
2364	* later processing neither this function nor &drm_gpuvm_sm_map is allowed to be
2365	* called before the &drm_gpuvm's view of the GPU VA space was updated
2366	* with the previous set of operations. To update the &drm_gpuvm's view
2367	* of the GPU VA space drm_gpuva_insert(), drm_gpuva_destroy_locked() and/or
2368	* drm_gpuva_destroy_unlocked() should be used.
2369	*
2370	* A sequence of callbacks can contain unmap and remap operations, depending on
2371	* whether there are actual overlapping mappings to split.
2372	*
2373	* There can be an arbitrary amount of unmap operations and a maximum of two
2374	* remap operations.
2375	*
2376	* Returns: 0 on success or a negative error code
2377	*/
2378	int
2379	drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv,
2380	u64 req_addr, u64 req_range)
2381	{
2382	const struct drm_gpuvm_ops *ops = gpuvm->ops;
2383
2384	if (unlikely(!(ops && ops->sm_step_remap &&
2385	ops->sm_step_unmap)))
2386	return -EINVAL;
2387
2388	return __drm_gpuvm_sm_unmap(gpuvm, ops, priv,
2389	req_addr, req_range);
2390	}
2391	EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap);
2392
2393	static struct drm_gpuva_op *
2394	gpuva_op_alloc(struct drm_gpuvm *gpuvm)
2395	{
2396	const struct drm_gpuvm_ops *fn = gpuvm->ops;
2397	struct drm_gpuva_op *op;
2398
2399	if (fn && fn->op_alloc)
2400	op = fn->op_alloc();
2401	else
2402	op = kzalloc(size: sizeof(*op), GFP_KERNEL);
2403
2404	if (unlikely(!op))
2405	return NULL;
2406
2407	return op;
2408	}
2409
2410	static void
2411	gpuva_op_free(struct drm_gpuvm *gpuvm,
2412	struct drm_gpuva_op *op)
2413	{
2414	const struct drm_gpuvm_ops *fn = gpuvm->ops;
2415
2416	if (fn && fn->op_free)
2417	fn->op_free(op);
2418	else
2419	kfree(objp: op);
2420	}
2421
2422	static int
2423	drm_gpuva_sm_step(struct drm_gpuva_op *__op,
2424	void *priv)
2425	{
2426	struct {
2427	struct drm_gpuvm *vm;
2428	struct drm_gpuva_ops *ops;
2429	} *args = priv;
2430	struct drm_gpuvm *gpuvm = args->vm;
2431	struct drm_gpuva_ops *ops = args->ops;
2432	struct drm_gpuva_op *op;
2433
2434	op = gpuva_op_alloc(gpuvm);
2435	if (unlikely(!op))
2436	goto err;
2437
2438	memcpy(op, __op, sizeof(*op));
2439
2440	if (op->op == DRM_GPUVA_OP_REMAP) {
2441	struct drm_gpuva_op_remap *__r = &__op->remap;
2442	struct drm_gpuva_op_remap *r = &op->remap;
2443
2444	r->unmap = kmemdup(p: __r->unmap, size: sizeof(*r->unmap),
2445	GFP_KERNEL);
2446	if (unlikely(!r->unmap))
2447	goto err_free_op;
2448
2449	if (__r->prev) {
2450	r->prev = kmemdup(p: __r->prev, size: sizeof(*r->prev),
2451	GFP_KERNEL);
2452	if (unlikely(!r->prev))
2453	goto err_free_unmap;
2454	}
2455
2456	if (__r->next) {
2457	r->next = kmemdup(p: __r->next, size: sizeof(*r->next),
2458	GFP_KERNEL);
2459	if (unlikely(!r->next))
2460	goto err_free_prev;
2461	}
2462	}
2463
2464	list_add_tail(new: &op->entry, head: &ops->list);
2465
2466	return 0;
2467
2468	err_free_unmap:
2469	kfree(objp: op->remap.unmap);
2470	err_free_prev:
2471	kfree(objp: op->remap.prev);
2472	err_free_op:
2473	gpuva_op_free(gpuvm, op);
2474	err:
2475	return -ENOMEM;
2476	}
2477
2478	static const struct drm_gpuvm_ops gpuvm_list_ops = {
2479	.sm_step_map = drm_gpuva_sm_step,
2480	.sm_step_remap = drm_gpuva_sm_step,
2481	.sm_step_unmap = drm_gpuva_sm_step,
2482	};
2483
2484	/**
2485	* drm_gpuvm_sm_map_ops_create() - creates the &drm_gpuva_ops to split and merge
2486	* @gpuvm: the &drm_gpuvm representing the GPU VA space
2487	* @req_addr: the start address of the new mapping
2488	* @req_range: the range of the new mapping
2489	* @req_obj: the &drm_gem_object to map
2490	* @req_offset: the offset within the &drm_gem_object
2491	*
2492	* This function creates a list of operations to perform splitting and merging
2493	* of existent mapping(s) with the newly requested one.
2494	*
2495	* The list can be iterated with &drm_gpuva_for_each_op and must be processed
2496	* in the given order. It can contain map, unmap and remap operations, but it
2497	* also can be empty if no operation is required, e.g. if the requested mapping
2498	* already exists is the exact same way.
2499	*
2500	* There can be an arbitrary amount of unmap operations, a maximum of two remap
2501	* operations and a single map operation. The latter one represents the original
2502	* map operation requested by the caller.
2503	*
2504	* Note that before calling this function again with another mapping request it
2505	* is necessary to update the &drm_gpuvm's view of the GPU VA space. The
2506	* previously obtained operations must be either processed or abandoned. To
2507	* update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),
2508	* drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
2509	* used.
2510	*
2511	* After the caller finished processing the returned &drm_gpuva_ops, they must
2512	* be freed with &drm_gpuva_ops_free.
2513	*
2514	* Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
2515	*/
2516	struct drm_gpuva_ops *
2517	drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm,
2518	u64 req_addr, u64 req_range,
2519	struct drm_gem_object *req_obj, u64 req_offset)
2520	{
2521	struct drm_gpuva_ops *ops;
2522	struct {
2523	struct drm_gpuvm *vm;
2524	struct drm_gpuva_ops *ops;
2525	} args;
2526	int ret;
2527
2528	ops = kzalloc(size: sizeof(*ops), GFP_KERNEL);
2529	if (unlikely(!ops))
2530	return ERR_PTR(error: -ENOMEM);
2531
2532	INIT_LIST_HEAD(list: &ops->list);
2533
2534	args.vm = gpuvm;
2535	args.ops = ops;
2536
2537	ret = __drm_gpuvm_sm_map(gpuvm, ops: &gpuvm_list_ops, priv: &args,
2538	req_addr, req_range,
2539	req_obj, req_offset);
2540	if (ret)
2541	goto err_free_ops;
2542
2543	return ops;
2544
2545	err_free_ops:
2546	drm_gpuva_ops_free(gpuvm, ops);
2547	return ERR_PTR(error: ret);
2548	}
2549	EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map_ops_create);
2550
2551	/**
2552	* drm_gpuvm_sm_unmap_ops_create() - creates the &drm_gpuva_ops to split on
2553	* unmap
2554	* @gpuvm: the &drm_gpuvm representing the GPU VA space
2555	* @req_addr: the start address of the range to unmap
2556	* @req_range: the range of the mappings to unmap
2557	*
2558	* This function creates a list of operations to perform unmapping and, if
2559	* required, splitting of the mappings overlapping the unmap range.
2560	*
2561	* The list can be iterated with &drm_gpuva_for_each_op and must be processed
2562	* in the given order. It can contain unmap and remap operations, depending on
2563	* whether there are actual overlapping mappings to split.
2564	*
2565	* There can be an arbitrary amount of unmap operations and a maximum of two
2566	* remap operations.
2567	*
2568	* Note that before calling this function again with another range to unmap it
2569	* is necessary to update the &drm_gpuvm's view of the GPU VA space. The
2570	* previously obtained operations must be processed or abandoned. To update the
2571	* &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),
2572	* drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
2573	* used.
2574	*
2575	* After the caller finished processing the returned &drm_gpuva_ops, they must
2576	* be freed with &drm_gpuva_ops_free.
2577	*
2578	* Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
2579	*/
2580	struct drm_gpuva_ops *
2581	drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm,
2582	u64 req_addr, u64 req_range)
2583	{
2584	struct drm_gpuva_ops *ops;
2585	struct {
2586	struct drm_gpuvm *vm;
2587	struct drm_gpuva_ops *ops;
2588	} args;
2589	int ret;
2590
2591	ops = kzalloc(size: sizeof(*ops), GFP_KERNEL);
2592	if (unlikely(!ops))
2593	return ERR_PTR(error: -ENOMEM);
2594
2595	INIT_LIST_HEAD(list: &ops->list);
2596
2597	args.vm = gpuvm;
2598	args.ops = ops;
2599
2600	ret = __drm_gpuvm_sm_unmap(gpuvm, ops: &gpuvm_list_ops, priv: &args,
2601	req_addr, req_range);
2602	if (ret)
2603	goto err_free_ops;
2604
2605	return ops;
2606
2607	err_free_ops:
2608	drm_gpuva_ops_free(gpuvm, ops);
2609	return ERR_PTR(error: ret);
2610	}
2611	EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap_ops_create);
2612
2613	/**
2614	* drm_gpuvm_prefetch_ops_create() - creates the &drm_gpuva_ops to prefetch
2615	* @gpuvm: the &drm_gpuvm representing the GPU VA space
2616	* @addr: the start address of the range to prefetch
2617	* @range: the range of the mappings to prefetch
2618	*
2619	* This function creates a list of operations to perform prefetching.
2620	*
2621	* The list can be iterated with &drm_gpuva_for_each_op and must be processed
2622	* in the given order. It can contain prefetch operations.
2623	*
2624	* There can be an arbitrary amount of prefetch operations.
2625	*
2626	* After the caller finished processing the returned &drm_gpuva_ops, they must
2627	* be freed with &drm_gpuva_ops_free.
2628	*
2629	* Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
2630	*/
2631	struct drm_gpuva_ops *
2632	drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm,
2633	u64 addr, u64 range)
2634	{
2635	struct drm_gpuva_ops *ops;
2636	struct drm_gpuva_op *op;
2637	struct drm_gpuva *va;
2638	u64 end = addr + range;
2639	int ret;
2640
2641	ops = kzalloc(size: sizeof(*ops), GFP_KERNEL);
2642	if (!ops)
2643	return ERR_PTR(error: -ENOMEM);
2644
2645	INIT_LIST_HEAD(list: &ops->list);
2646
2647	drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {
2648	op = gpuva_op_alloc(gpuvm);
2649	if (!op) {
2650	ret = -ENOMEM;
2651	goto err_free_ops;
2652	}
2653
2654	op->op = DRM_GPUVA_OP_PREFETCH;
2655	op->prefetch.va = va;
2656	list_add_tail(new: &op->entry, head: &ops->list);
2657	}
2658
2659	return ops;
2660
2661	err_free_ops:
2662	drm_gpuva_ops_free(gpuvm, ops);
2663	return ERR_PTR(error: ret);
2664	}
2665	EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create);
2666
2667	/**
2668	* drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM
2669	* @vm_bo: the &drm_gpuvm_bo abstraction
2670	*
2671	* This function creates a list of operations to perform unmapping for every
2672	* GPUVA attached to a GEM.
2673	*
2674	* The list can be iterated with &drm_gpuva_for_each_op and consists out of an
2675	* arbitrary amount of unmap operations.
2676	*
2677	* After the caller finished processing the returned &drm_gpuva_ops, they must
2678	* be freed with &drm_gpuva_ops_free.
2679	*
2680	* It is the callers responsibility to protect the GEMs GPUVA list against
2681	* concurrent access using the GEMs dma_resv lock.
2682	*
2683	* Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
2684	*/
2685	struct drm_gpuva_ops *
2686	drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo)
2687	{
2688	struct drm_gpuva_ops *ops;
2689	struct drm_gpuva_op *op;
2690	struct drm_gpuva *va;
2691	int ret;
2692
2693	drm_gem_gpuva_assert_lock_held(vm_bo->obj);
2694
2695	ops = kzalloc(size: sizeof(*ops), GFP_KERNEL);
2696	if (!ops)
2697	return ERR_PTR(error: -ENOMEM);
2698
2699	INIT_LIST_HEAD(list: &ops->list);
2700
2701	drm_gpuvm_bo_for_each_va(va, vm_bo) {
2702	op = gpuva_op_alloc(gpuvm: vm_bo->vm);
2703	if (!op) {
2704	ret = -ENOMEM;
2705	goto err_free_ops;
2706	}
2707
2708	op->op = DRM_GPUVA_OP_UNMAP;
2709	op->unmap.va = va;
2710	list_add_tail(new: &op->entry, head: &ops->list);
2711	}
2712
2713	return ops;
2714
2715	err_free_ops:
2716	drm_gpuva_ops_free(gpuvm: vm_bo->vm, ops);
2717	return ERR_PTR(error: ret);
2718	}
2719	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_unmap_ops_create);
2720
2721	/**
2722	* drm_gpuva_ops_free() - free the given &drm_gpuva_ops
2723	* @gpuvm: the &drm_gpuvm the ops were created for
2724	* @ops: the &drm_gpuva_ops to free
2725	*
2726	* Frees the given &drm_gpuva_ops structure including all the ops associated
2727	* with it.
2728	*/
2729	void
2730	drm_gpuva_ops_free(struct drm_gpuvm *gpuvm,
2731	struct drm_gpuva_ops *ops)
2732	{
2733	struct drm_gpuva_op *op, *next;
2734
2735	drm_gpuva_for_each_op_safe(op, next, ops) {
2736	list_del(entry: &op->entry);
2737
2738	if (op->op == DRM_GPUVA_OP_REMAP) {
2739	kfree(objp: op->remap.prev);
2740	kfree(objp: op->remap.next);
2741	kfree(objp: op->remap.unmap);
2742	}
2743
2744	gpuva_op_free(gpuvm, op);
2745	}
2746
2747	kfree(objp: ops);
2748	}
2749	EXPORT_SYMBOL_GPL(drm_gpuva_ops_free);
2750
2751	MODULE_DESCRIPTION("DRM GPUVM");
2752	MODULE_LICENSE("GPL");
2753