1 | /* |
2 | * Copyright © 2016 Intel Corporation |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
22 | * |
23 | */ |
24 | |
25 | #ifndef __I915_VMA_H__ |
26 | #define __I915_VMA_H__ |
27 | |
28 | #include <linux/io-mapping.h> |
29 | #include <linux/rbtree.h> |
30 | |
31 | #include <drm/drm_mm.h> |
32 | |
33 | #include "gt/intel_ggtt_fencing.h" |
34 | #include "gem/i915_gem_object.h" |
35 | |
36 | #include "i915_gem_gtt.h" |
37 | |
38 | #include "i915_active.h" |
39 | #include "i915_request.h" |
40 | #include "i915_vma_resource.h" |
41 | #include "i915_vma_types.h" |
42 | |
43 | struct i915_vma * |
44 | i915_vma_instance(struct drm_i915_gem_object *obj, |
45 | struct i915_address_space *vm, |
46 | const struct i915_gtt_view *view); |
47 | |
48 | void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags); |
49 | #define I915_VMA_RELEASE_MAP BIT(0) |
50 | |
51 | static inline bool i915_vma_is_active(const struct i915_vma *vma) |
52 | { |
53 | return !i915_active_is_idle(ref: &vma->active); |
54 | } |
55 | |
56 | /* do not reserve memory to prevent deadlocks */ |
57 | #define __EXEC_OBJECT_NO_RESERVE BIT(31) |
58 | #define __EXEC_OBJECT_NO_REQUEST_AWAIT BIT(30) |
59 | |
60 | int __must_check _i915_vma_move_to_active(struct i915_vma *vma, |
61 | struct i915_request *rq, |
62 | struct dma_fence *fence, |
63 | unsigned int flags); |
64 | static inline int __must_check |
65 | i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq, |
66 | unsigned int flags) |
67 | { |
68 | return _i915_vma_move_to_active(vma, rq, fence: &rq->fence, flags); |
69 | } |
70 | |
71 | #define __i915_vma_flags(v) ((unsigned long *)&(v)->flags.counter) |
72 | |
73 | static inline bool i915_vma_is_ggtt(const struct i915_vma *vma) |
74 | { |
75 | return test_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma)); |
76 | } |
77 | |
78 | static inline bool i915_vma_is_dpt(const struct i915_vma *vma) |
79 | { |
80 | return i915_is_dpt(vma->vm); |
81 | } |
82 | |
83 | static inline bool i915_vma_has_ggtt_write(const struct i915_vma *vma) |
84 | { |
85 | return test_bit(I915_VMA_GGTT_WRITE_BIT, __i915_vma_flags(vma)); |
86 | } |
87 | |
88 | static inline void i915_vma_set_ggtt_write(struct i915_vma *vma) |
89 | { |
90 | GEM_BUG_ON(!i915_vma_is_ggtt(vma)); |
91 | set_bit(I915_VMA_GGTT_WRITE_BIT, __i915_vma_flags(vma)); |
92 | } |
93 | |
94 | static inline bool i915_vma_unset_ggtt_write(struct i915_vma *vma) |
95 | { |
96 | return test_and_clear_bit(I915_VMA_GGTT_WRITE_BIT, |
97 | __i915_vma_flags(vma)); |
98 | } |
99 | |
100 | void i915_vma_flush_writes(struct i915_vma *vma); |
101 | |
102 | static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma) |
103 | { |
104 | return test_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma)); |
105 | } |
106 | |
107 | static inline bool i915_vma_set_userfault(struct i915_vma *vma) |
108 | { |
109 | GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); |
110 | return test_and_set_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma)); |
111 | } |
112 | |
113 | static inline void i915_vma_unset_userfault(struct i915_vma *vma) |
114 | { |
115 | return clear_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma)); |
116 | } |
117 | |
118 | static inline bool i915_vma_has_userfault(const struct i915_vma *vma) |
119 | { |
120 | return test_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma)); |
121 | } |
122 | |
123 | static inline bool i915_vma_is_closed(const struct i915_vma *vma) |
124 | { |
125 | return !list_empty(head: &vma->closed_link); |
126 | } |
127 | |
128 | /* Internal use only. */ |
129 | static inline u64 __i915_vma_size(const struct i915_vma *vma) |
130 | { |
131 | return vma->node.size - 2 * vma->guard; |
132 | } |
133 | |
134 | /** |
135 | * i915_vma_size - Obtain the va range size of the vma |
136 | * @vma: The vma |
137 | * |
138 | * GPU virtual address space may be allocated with padding. This |
139 | * function returns the effective virtual address range size |
140 | * with padding subtracted. |
141 | * |
142 | * Return: The effective virtual address range size. |
143 | */ |
144 | static inline u64 i915_vma_size(const struct i915_vma *vma) |
145 | { |
146 | GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); |
147 | return __i915_vma_size(vma); |
148 | } |
149 | |
150 | /* Internal use only. */ |
151 | static inline u64 __i915_vma_offset(const struct i915_vma *vma) |
152 | { |
153 | /* The actual start of the vma->pages is after the guard pages. */ |
154 | return vma->node.start + vma->guard; |
155 | } |
156 | |
157 | /** |
158 | * i915_vma_offset - Obtain the va offset of the vma |
159 | * @vma: The vma |
160 | * |
161 | * GPU virtual address space may be allocated with padding. This |
162 | * function returns the effective virtual address offset the gpu |
163 | * should use to access the bound data. |
164 | * |
165 | * Return: The effective virtual address offset. |
166 | */ |
167 | static inline u64 i915_vma_offset(const struct i915_vma *vma) |
168 | { |
169 | GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); |
170 | return __i915_vma_offset(vma); |
171 | } |
172 | |
173 | static inline u32 i915_ggtt_offset(const struct i915_vma *vma) |
174 | { |
175 | GEM_BUG_ON(!i915_vma_is_ggtt(vma)); |
176 | GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); |
177 | GEM_BUG_ON(upper_32_bits(i915_vma_offset(vma))); |
178 | GEM_BUG_ON(upper_32_bits(i915_vma_offset(vma) + |
179 | i915_vma_size(vma) - 1)); |
180 | return lower_32_bits(i915_vma_offset(vma)); |
181 | } |
182 | |
183 | static inline u32 i915_ggtt_pin_bias(struct i915_vma *vma) |
184 | { |
185 | return i915_vm_to_ggtt(vm: vma->vm)->pin_bias; |
186 | } |
187 | |
188 | static inline struct i915_vma *i915_vma_get(struct i915_vma *vma) |
189 | { |
190 | i915_gem_object_get(obj: vma->obj); |
191 | return vma; |
192 | } |
193 | |
194 | static inline struct i915_vma *i915_vma_tryget(struct i915_vma *vma) |
195 | { |
196 | if (likely(kref_get_unless_zero(&vma->obj->base.refcount))) |
197 | return vma; |
198 | |
199 | return NULL; |
200 | } |
201 | |
202 | static inline void i915_vma_put(struct i915_vma *vma) |
203 | { |
204 | i915_gem_object_put(obj: vma->obj); |
205 | } |
206 | |
207 | static inline long |
208 | i915_vma_compare(struct i915_vma *vma, |
209 | struct i915_address_space *vm, |
210 | const struct i915_gtt_view *view) |
211 | { |
212 | ptrdiff_t cmp; |
213 | |
214 | GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm)); |
215 | |
216 | cmp = ptrdiff(a: vma->vm, b: vm); |
217 | if (cmp) |
218 | return cmp; |
219 | |
220 | BUILD_BUG_ON(I915_GTT_VIEW_NORMAL != 0); |
221 | cmp = vma->gtt_view.type; |
222 | if (!view) |
223 | return cmp; |
224 | |
225 | cmp -= view->type; |
226 | if (cmp) |
227 | return cmp; |
228 | |
229 | assert_i915_gem_gtt_types(); |
230 | |
231 | /* gtt_view.type also encodes its size so that we both distinguish |
232 | * different views using it as a "type" and also use a compact (no |
233 | * accessing of uninitialised padding bytes) memcmp without storing |
234 | * an extra parameter or adding more code. |
235 | * |
236 | * To ensure that the memcmp is valid for all branches of the union, |
237 | * even though the code looks like it is just comparing one branch, |
238 | * we assert above that all branches have the same address, and that |
239 | * each branch has a unique type/size. |
240 | */ |
241 | BUILD_BUG_ON(I915_GTT_VIEW_NORMAL >= I915_GTT_VIEW_PARTIAL); |
242 | BUILD_BUG_ON(I915_GTT_VIEW_PARTIAL >= I915_GTT_VIEW_ROTATED); |
243 | BUILD_BUG_ON(I915_GTT_VIEW_ROTATED >= I915_GTT_VIEW_REMAPPED); |
244 | BUILD_BUG_ON(offsetof(typeof(*view), rotated) != |
245 | offsetof(typeof(*view), partial)); |
246 | BUILD_BUG_ON(offsetof(typeof(*view), rotated) != |
247 | offsetof(typeof(*view), remapped)); |
248 | return memcmp(p: &vma->gtt_view.partial, q: &view->partial, size: view->type); |
249 | } |
250 | |
251 | struct i915_vma_work *i915_vma_work(void); |
252 | int i915_vma_bind(struct i915_vma *vma, |
253 | unsigned int pat_index, |
254 | u32 flags, |
255 | struct i915_vma_work *work, |
256 | struct i915_vma_resource *vma_res); |
257 | |
258 | bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color); |
259 | bool i915_vma_misplaced(const struct i915_vma *vma, |
260 | u64 size, u64 alignment, u64 flags); |
261 | void __i915_vma_set_map_and_fenceable(struct i915_vma *vma); |
262 | void i915_vma_revoke_mmap(struct i915_vma *vma); |
263 | void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb); |
264 | struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async); |
265 | int __i915_vma_unbind(struct i915_vma *vma); |
266 | int __must_check i915_vma_unbind(struct i915_vma *vma); |
267 | int __must_check i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm); |
268 | int __must_check i915_vma_unbind_unlocked(struct i915_vma *vma); |
269 | void i915_vma_unlink_ctx(struct i915_vma *vma); |
270 | void i915_vma_close(struct i915_vma *vma); |
271 | void i915_vma_reopen(struct i915_vma *vma); |
272 | |
273 | void i915_vma_destroy_locked(struct i915_vma *vma); |
274 | void i915_vma_destroy(struct i915_vma *vma); |
275 | |
276 | #define assert_vma_held(vma) dma_resv_assert_held((vma)->obj->base.resv) |
277 | |
278 | static inline void i915_vma_lock(struct i915_vma *vma) |
279 | { |
280 | dma_resv_lock(obj: vma->obj->base.resv, NULL); |
281 | } |
282 | |
283 | static inline void i915_vma_unlock(struct i915_vma *vma) |
284 | { |
285 | dma_resv_unlock(obj: vma->obj->base.resv); |
286 | } |
287 | |
288 | int __must_check |
289 | i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, |
290 | u64 size, u64 alignment, u64 flags); |
291 | |
292 | static inline int __must_check |
293 | i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) |
294 | { |
295 | struct i915_gem_ww_ctx ww; |
296 | int err; |
297 | |
298 | i915_gem_ww_ctx_init(ctx: &ww, intr: true); |
299 | retry: |
300 | err = i915_gem_object_lock(obj: vma->obj, ww: &ww); |
301 | if (!err) |
302 | err = i915_vma_pin_ww(vma, ww: &ww, size, alignment, flags); |
303 | if (err == -EDEADLK) { |
304 | err = i915_gem_ww_ctx_backoff(ctx: &ww); |
305 | if (!err) |
306 | goto retry; |
307 | } |
308 | i915_gem_ww_ctx_fini(ctx: &ww); |
309 | |
310 | return err; |
311 | } |
312 | |
313 | int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, |
314 | u32 align, unsigned int flags); |
315 | |
316 | static inline int i915_vma_pin_count(const struct i915_vma *vma) |
317 | { |
318 | return atomic_read(v: &vma->flags) & I915_VMA_PIN_MASK; |
319 | } |
320 | |
321 | static inline bool i915_vma_is_pinned(const struct i915_vma *vma) |
322 | { |
323 | return i915_vma_pin_count(vma); |
324 | } |
325 | |
326 | static inline void __i915_vma_pin(struct i915_vma *vma) |
327 | { |
328 | atomic_inc(v: &vma->flags); |
329 | GEM_BUG_ON(!i915_vma_is_pinned(vma)); |
330 | } |
331 | |
332 | static inline void __i915_vma_unpin(struct i915_vma *vma) |
333 | { |
334 | GEM_BUG_ON(!i915_vma_is_pinned(vma)); |
335 | atomic_dec(v: &vma->flags); |
336 | } |
337 | |
338 | static inline void i915_vma_unpin(struct i915_vma *vma) |
339 | { |
340 | GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); |
341 | __i915_vma_unpin(vma); |
342 | } |
343 | |
344 | static inline bool i915_vma_is_bound(const struct i915_vma *vma, |
345 | unsigned int where) |
346 | { |
347 | return atomic_read(v: &vma->flags) & where; |
348 | } |
349 | |
350 | static inline bool i915_node_color_differs(const struct drm_mm_node *node, |
351 | unsigned long color) |
352 | { |
353 | return drm_mm_node_allocated(node) && node->color != color; |
354 | } |
355 | |
356 | /** |
357 | * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture |
358 | * @vma: VMA to iomap |
359 | * |
360 | * The passed in VMA has to be pinned in the global GTT mappable region. |
361 | * An extra pinning of the VMA is acquired for the return iomapping, |
362 | * the caller must call i915_vma_unpin_iomap to relinquish the pinning |
363 | * after the iomapping is no longer required. |
364 | * |
365 | * Returns a valid iomapped pointer or ERR_PTR. |
366 | */ |
367 | void __iomem *i915_vma_pin_iomap(struct i915_vma *vma); |
368 | |
369 | /** |
370 | * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap |
371 | * @vma: VMA to unpin |
372 | * |
373 | * Unpins the previously iomapped VMA from i915_vma_pin_iomap(). |
374 | * |
375 | * This function is only valid to be called on a VMA previously |
376 | * iomapped by the caller with i915_vma_pin_iomap(). |
377 | */ |
378 | void i915_vma_unpin_iomap(struct i915_vma *vma); |
379 | |
380 | /** |
381 | * i915_vma_pin_fence - pin fencing state |
382 | * @vma: vma to pin fencing for |
383 | * |
384 | * This pins the fencing state (whether tiled or untiled) to make sure the |
385 | * vma (and its object) is ready to be used as a scanout target. Fencing |
386 | * status must be synchronize first by calling i915_vma_get_fence(): |
387 | * |
388 | * The resulting fence pin reference must be released again with |
389 | * i915_vma_unpin_fence(). |
390 | * |
391 | * Returns: |
392 | * |
393 | * True if the vma has a fence, false otherwise. |
394 | */ |
395 | int __must_check i915_vma_pin_fence(struct i915_vma *vma); |
396 | void i915_vma_revoke_fence(struct i915_vma *vma); |
397 | |
398 | int __i915_vma_pin_fence(struct i915_vma *vma); |
399 | |
400 | static inline void __i915_vma_unpin_fence(struct i915_vma *vma) |
401 | { |
402 | GEM_BUG_ON(atomic_read(&vma->fence->pin_count) <= 0); |
403 | atomic_dec(v: &vma->fence->pin_count); |
404 | } |
405 | |
406 | /** |
407 | * i915_vma_unpin_fence - unpin fencing state |
408 | * @vma: vma to unpin fencing for |
409 | * |
410 | * This releases the fence pin reference acquired through |
411 | * i915_vma_pin_fence. It will handle both objects with and without an |
412 | * attached fence correctly, callers do not need to distinguish this. |
413 | */ |
414 | static inline void |
415 | i915_vma_unpin_fence(struct i915_vma *vma) |
416 | { |
417 | if (vma->fence) |
418 | __i915_vma_unpin_fence(vma); |
419 | } |
420 | |
421 | static inline int i915_vma_fence_id(const struct i915_vma *vma) |
422 | { |
423 | return vma->fence ? vma->fence->id : -1; |
424 | } |
425 | |
426 | void i915_vma_parked(struct intel_gt *gt); |
427 | |
428 | static inline bool i915_vma_is_scanout(const struct i915_vma *vma) |
429 | { |
430 | return test_bit(I915_VMA_SCANOUT_BIT, __i915_vma_flags(vma)); |
431 | } |
432 | |
433 | static inline void i915_vma_mark_scanout(struct i915_vma *vma) |
434 | { |
435 | set_bit(I915_VMA_SCANOUT_BIT, __i915_vma_flags(vma)); |
436 | } |
437 | |
438 | static inline void i915_vma_clear_scanout(struct i915_vma *vma) |
439 | { |
440 | clear_bit(I915_VMA_SCANOUT_BIT, __i915_vma_flags(vma)); |
441 | } |
442 | |
443 | void i915_ggtt_clear_scanout(struct drm_i915_gem_object *obj); |
444 | |
445 | #define for_each_until(cond) if (cond) break; else |
446 | |
447 | /** |
448 | * for_each_ggtt_vma - Iterate over the GGTT VMA belonging to an object. |
449 | * @V: the #i915_vma iterator |
450 | * @OBJ: the #drm_i915_gem_object |
451 | * |
452 | * GGTT VMA are placed at the being of the object's vma_list, see |
453 | * vma_create(), so we can stop our walk as soon as we see a ppgtt VMA, |
454 | * or the list is empty ofc. |
455 | */ |
456 | #define for_each_ggtt_vma(V, OBJ) \ |
457 | list_for_each_entry(V, &(OBJ)->vma.list, obj_link) \ |
458 | for_each_until(!i915_vma_is_ggtt(V)) |
459 | |
460 | struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma); |
461 | void i915_vma_make_shrinkable(struct i915_vma *vma); |
462 | void i915_vma_make_purgeable(struct i915_vma *vma); |
463 | |
464 | int i915_vma_wait_for_bind(struct i915_vma *vma); |
465 | |
466 | static inline int i915_vma_sync(struct i915_vma *vma) |
467 | { |
468 | /* Wait for the asynchronous bindings and pending GPU reads */ |
469 | return i915_active_wait(ref: &vma->active); |
470 | } |
471 | |
472 | /** |
473 | * i915_vma_get_current_resource - Get the current resource of the vma |
474 | * @vma: The vma to get the current resource from. |
475 | * |
476 | * It's illegal to call this function if the vma is not bound. |
477 | * |
478 | * Return: A refcounted pointer to the current vma resource |
479 | * of the vma, assuming the vma is bound. |
480 | */ |
481 | static inline struct i915_vma_resource * |
482 | i915_vma_get_current_resource(struct i915_vma *vma) |
483 | { |
484 | return i915_vma_resource_get(vma_res: vma->resource); |
485 | } |
486 | |
487 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) |
488 | void i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res, |
489 | struct i915_vma *vma); |
490 | #endif |
491 | |
492 | void i915_vma_module_exit(void); |
493 | int i915_vma_module_init(void); |
494 | |
495 | I915_SELFTEST_DECLARE(int i915_vma_get_pages(struct i915_vma *vma)); |
496 | I915_SELFTEST_DECLARE(void i915_vma_put_pages(struct i915_vma *vma)); |
497 | |
498 | #endif |
499 | |