1 | /* |
2 | * Copyright © 2008-2015 Intel Corporation |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
22 | * |
23 | * Authors: |
24 | * Eric Anholt <eric@anholt.net> |
25 | * |
26 | */ |
27 | |
28 | #include <linux/dma-fence-array.h> |
29 | #include <linux/kthread.h> |
30 | #include <linux/dma-resv.h> |
31 | #include <linux/shmem_fs.h> |
32 | #include <linux/slab.h> |
33 | #include <linux/stop_machine.h> |
34 | #include <linux/swap.h> |
35 | #include <linux/pci.h> |
36 | #include <linux/dma-buf.h> |
37 | #include <linux/mman.h> |
38 | |
39 | #include <drm/drm_cache.h> |
40 | #include <drm/drm_vma_manager.h> |
41 | |
42 | #include "display/intel_display.h" |
43 | |
44 | #include "gem/i915_gem_clflush.h" |
45 | #include "gem/i915_gem_context.h" |
46 | #include "gem/i915_gem_ioctls.h" |
47 | #include "gem/i915_gem_mman.h" |
48 | #include "gem/i915_gem_object_frontbuffer.h" |
49 | #include "gem/i915_gem_pm.h" |
50 | #include "gem/i915_gem_region.h" |
51 | #include "gem/i915_gem_userptr.h" |
52 | #include "gt/intel_engine_user.h" |
53 | #include "gt/intel_gt.h" |
54 | #include "gt/intel_gt_pm.h" |
55 | #include "gt/intel_workarounds.h" |
56 | |
57 | #include "i915_drv.h" |
58 | #include "i915_file_private.h" |
59 | #include "i915_trace.h" |
60 | #include "i915_vgpu.h" |
61 | #include "intel_clock_gating.h" |
62 | |
63 | static int |
64 | insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size) |
65 | { |
66 | int err; |
67 | |
68 | err = mutex_lock_interruptible(&ggtt->vm.mutex); |
69 | if (err) |
70 | return err; |
71 | |
72 | memset(node, 0, sizeof(*node)); |
73 | err = drm_mm_insert_node_in_range(mm: &ggtt->vm.mm, node, |
74 | size, alignment: 0, I915_COLOR_UNEVICTABLE, |
75 | start: 0, end: ggtt->mappable_end, |
76 | mode: DRM_MM_INSERT_LOW); |
77 | |
78 | mutex_unlock(lock: &ggtt->vm.mutex); |
79 | |
80 | return err; |
81 | } |
82 | |
83 | static void |
84 | remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node) |
85 | { |
86 | mutex_lock(&ggtt->vm.mutex); |
87 | drm_mm_remove_node(node); |
88 | mutex_unlock(lock: &ggtt->vm.mutex); |
89 | } |
90 | |
91 | int |
92 | i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, |
93 | struct drm_file *file) |
94 | { |
95 | struct drm_i915_private *i915 = to_i915(dev); |
96 | struct i915_ggtt *ggtt = to_gt(i915)->ggtt; |
97 | struct drm_i915_gem_get_aperture *args = data; |
98 | struct i915_vma *vma; |
99 | u64 pinned; |
100 | |
101 | if (mutex_lock_interruptible(&ggtt->vm.mutex)) |
102 | return -EINTR; |
103 | |
104 | pinned = ggtt->vm.reserved; |
105 | list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link) |
106 | if (i915_vma_is_pinned(vma)) |
107 | pinned += vma->node.size; |
108 | |
109 | mutex_unlock(lock: &ggtt->vm.mutex); |
110 | |
111 | args->aper_size = ggtt->vm.total; |
112 | args->aper_available_size = args->aper_size - pinned; |
113 | |
114 | return 0; |
115 | } |
116 | |
117 | int i915_gem_object_unbind(struct drm_i915_gem_object *obj, |
118 | unsigned long flags) |
119 | { |
120 | struct intel_runtime_pm *rpm = &to_i915(dev: obj->base.dev)->runtime_pm; |
121 | bool vm_trylock = !!(flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK); |
122 | LIST_HEAD(still_in_list); |
123 | intel_wakeref_t wakeref; |
124 | struct i915_vma *vma; |
125 | int ret; |
126 | |
127 | assert_object_held(obj); |
128 | |
129 | if (list_empty(head: &obj->vma.list)) |
130 | return 0; |
131 | |
132 | /* |
133 | * As some machines use ACPI to handle runtime-resume callbacks, and |
134 | * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex |
135 | * as they are required by the shrinker. Ergo, we wake the device up |
136 | * first just in case. |
137 | */ |
138 | wakeref = intel_runtime_pm_get(rpm); |
139 | |
140 | try_again: |
141 | ret = 0; |
142 | spin_lock(lock: &obj->vma.lock); |
143 | while (!ret && (vma = list_first_entry_or_null(&obj->vma.list, |
144 | struct i915_vma, |
145 | obj_link))) { |
146 | list_move_tail(list: &vma->obj_link, head: &still_in_list); |
147 | if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK)) |
148 | continue; |
149 | |
150 | if (flags & I915_GEM_OBJECT_UNBIND_TEST) { |
151 | ret = -EBUSY; |
152 | break; |
153 | } |
154 | |
155 | /* |
156 | * Requiring the vm destructor to take the object lock |
157 | * before destroying a vma would help us eliminate the |
158 | * i915_vm_tryget() here, AND thus also the barrier stuff |
159 | * at the end. That's an easy fix, but sleeping locks in |
160 | * a kthread should generally be avoided. |
161 | */ |
162 | ret = -EAGAIN; |
163 | if (!i915_vm_tryget(vm: vma->vm)) |
164 | break; |
165 | |
166 | spin_unlock(lock: &obj->vma.lock); |
167 | |
168 | /* |
169 | * Since i915_vma_parked() takes the object lock |
170 | * before vma destruction, it won't race us here, |
171 | * and destroy the vma from under us. |
172 | */ |
173 | |
174 | ret = -EBUSY; |
175 | if (flags & I915_GEM_OBJECT_UNBIND_ASYNC) { |
176 | assert_object_held(vma->obj); |
177 | ret = i915_vma_unbind_async(vma, trylock_vm: vm_trylock); |
178 | } |
179 | |
180 | if (ret == -EBUSY && (flags & I915_GEM_OBJECT_UNBIND_ACTIVE || |
181 | !i915_vma_is_active(vma))) { |
182 | if (vm_trylock) { |
183 | if (mutex_trylock(lock: &vma->vm->mutex)) { |
184 | ret = __i915_vma_unbind(vma); |
185 | mutex_unlock(lock: &vma->vm->mutex); |
186 | } |
187 | } else { |
188 | ret = i915_vma_unbind(vma); |
189 | } |
190 | } |
191 | |
192 | i915_vm_put(vm: vma->vm); |
193 | spin_lock(lock: &obj->vma.lock); |
194 | } |
195 | list_splice_init(list: &still_in_list, head: &obj->vma.list); |
196 | spin_unlock(lock: &obj->vma.lock); |
197 | |
198 | if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) { |
199 | rcu_barrier(); /* flush the i915_vm_release() */ |
200 | goto try_again; |
201 | } |
202 | |
203 | intel_runtime_pm_put(rpm, wref: wakeref); |
204 | |
205 | return ret; |
206 | } |
207 | |
208 | static int |
209 | shmem_pread(struct page *page, int offset, int len, char __user *user_data, |
210 | bool needs_clflush) |
211 | { |
212 | char *vaddr; |
213 | int ret; |
214 | |
215 | vaddr = kmap(page); |
216 | |
217 | if (needs_clflush) |
218 | drm_clflush_virt_range(addr: vaddr + offset, length: len); |
219 | |
220 | ret = __copy_to_user(to: user_data, from: vaddr + offset, n: len); |
221 | |
222 | kunmap(page); |
223 | |
224 | return ret ? -EFAULT : 0; |
225 | } |
226 | |
227 | static int |
228 | i915_gem_shmem_pread(struct drm_i915_gem_object *obj, |
229 | struct drm_i915_gem_pread *args) |
230 | { |
231 | unsigned int needs_clflush; |
232 | char __user *user_data; |
233 | unsigned long offset; |
234 | pgoff_t idx; |
235 | u64 remain; |
236 | int ret; |
237 | |
238 | ret = i915_gem_object_lock_interruptible(obj, NULL); |
239 | if (ret) |
240 | return ret; |
241 | |
242 | ret = i915_gem_object_pin_pages(obj); |
243 | if (ret) |
244 | goto err_unlock; |
245 | |
246 | ret = i915_gem_object_prepare_read(obj, needs_clflush: &needs_clflush); |
247 | if (ret) |
248 | goto err_unpin; |
249 | |
250 | i915_gem_object_finish_access(obj); |
251 | i915_gem_object_unlock(obj); |
252 | |
253 | remain = args->size; |
254 | user_data = u64_to_user_ptr(args->data_ptr); |
255 | offset = offset_in_page(args->offset); |
256 | for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { |
257 | struct page *page = i915_gem_object_get_page(obj, idx); |
258 | unsigned int length = min_t(u64, remain, PAGE_SIZE - offset); |
259 | |
260 | ret = shmem_pread(page, offset, len: length, user_data, |
261 | needs_clflush); |
262 | if (ret) |
263 | break; |
264 | |
265 | remain -= length; |
266 | user_data += length; |
267 | offset = 0; |
268 | } |
269 | |
270 | i915_gem_object_unpin_pages(obj); |
271 | return ret; |
272 | |
273 | err_unpin: |
274 | i915_gem_object_unpin_pages(obj); |
275 | err_unlock: |
276 | i915_gem_object_unlock(obj); |
277 | return ret; |
278 | } |
279 | |
280 | static inline bool |
281 | gtt_user_read(struct io_mapping *mapping, |
282 | loff_t base, int offset, |
283 | char __user *user_data, int length) |
284 | { |
285 | void __iomem *vaddr; |
286 | unsigned long unwritten; |
287 | |
288 | /* We can use the cpu mem copy function because this is X86. */ |
289 | vaddr = io_mapping_map_atomic_wc(mapping, offset: base); |
290 | unwritten = __copy_to_user_inatomic(to: user_data, |
291 | from: (void __force *)vaddr + offset, |
292 | n: length); |
293 | io_mapping_unmap_atomic(vaddr); |
294 | if (unwritten) { |
295 | vaddr = io_mapping_map_wc(mapping, offset: base, PAGE_SIZE); |
296 | unwritten = copy_to_user(to: user_data, |
297 | from: (void __force *)vaddr + offset, |
298 | n: length); |
299 | io_mapping_unmap(vaddr); |
300 | } |
301 | return unwritten; |
302 | } |
303 | |
304 | static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj, |
305 | struct drm_mm_node *node, |
306 | bool write) |
307 | { |
308 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
309 | struct i915_ggtt *ggtt = to_gt(i915)->ggtt; |
310 | struct i915_vma *vma; |
311 | struct i915_gem_ww_ctx ww; |
312 | int ret; |
313 | |
314 | i915_gem_ww_ctx_init(ctx: &ww, intr: true); |
315 | retry: |
316 | vma = ERR_PTR(error: -ENODEV); |
317 | ret = i915_gem_object_lock(obj, ww: &ww); |
318 | if (ret) |
319 | goto err_ww; |
320 | |
321 | ret = i915_gem_object_set_to_gtt_domain(obj, write); |
322 | if (ret) |
323 | goto err_ww; |
324 | |
325 | if (!i915_gem_object_is_tiled(obj)) |
326 | vma = i915_gem_object_ggtt_pin_ww(obj, ww: &ww, NULL, size: 0, alignment: 0, |
327 | PIN_MAPPABLE | |
328 | PIN_NONBLOCK /* NOWARN */ | |
329 | PIN_NOEVICT); |
330 | if (vma == ERR_PTR(error: -EDEADLK)) { |
331 | ret = -EDEADLK; |
332 | goto err_ww; |
333 | } else if (!IS_ERR(ptr: vma)) { |
334 | node->start = i915_ggtt_offset(vma); |
335 | node->flags = 0; |
336 | } else { |
337 | ret = insert_mappable_node(ggtt, node, PAGE_SIZE); |
338 | if (ret) |
339 | goto err_ww; |
340 | GEM_BUG_ON(!drm_mm_node_allocated(node)); |
341 | vma = NULL; |
342 | } |
343 | |
344 | ret = i915_gem_object_pin_pages(obj); |
345 | if (ret) { |
346 | if (drm_mm_node_allocated(node)) { |
347 | ggtt->vm.clear_range(&ggtt->vm, node->start, node->size); |
348 | remove_mappable_node(ggtt, node); |
349 | } else { |
350 | i915_vma_unpin(vma); |
351 | } |
352 | } |
353 | |
354 | err_ww: |
355 | if (ret == -EDEADLK) { |
356 | ret = i915_gem_ww_ctx_backoff(ctx: &ww); |
357 | if (!ret) |
358 | goto retry; |
359 | } |
360 | i915_gem_ww_ctx_fini(ctx: &ww); |
361 | |
362 | return ret ? ERR_PTR(error: ret) : vma; |
363 | } |
364 | |
365 | static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj, |
366 | struct drm_mm_node *node, |
367 | struct i915_vma *vma) |
368 | { |
369 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
370 | struct i915_ggtt *ggtt = to_gt(i915)->ggtt; |
371 | |
372 | i915_gem_object_unpin_pages(obj); |
373 | if (drm_mm_node_allocated(node)) { |
374 | ggtt->vm.clear_range(&ggtt->vm, node->start, node->size); |
375 | remove_mappable_node(ggtt, node); |
376 | } else { |
377 | i915_vma_unpin(vma); |
378 | } |
379 | } |
380 | |
381 | static int |
382 | i915_gem_gtt_pread(struct drm_i915_gem_object *obj, |
383 | const struct drm_i915_gem_pread *args) |
384 | { |
385 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
386 | struct i915_ggtt *ggtt = to_gt(i915)->ggtt; |
387 | unsigned long remain, offset; |
388 | intel_wakeref_t wakeref; |
389 | struct drm_mm_node node; |
390 | void __user *user_data; |
391 | struct i915_vma *vma; |
392 | int ret = 0; |
393 | |
394 | if (overflows_type(args->size, remain) || |
395 | overflows_type(args->offset, offset)) |
396 | return -EINVAL; |
397 | |
398 | wakeref = intel_runtime_pm_get(rpm: &i915->runtime_pm); |
399 | |
400 | vma = i915_gem_gtt_prepare(obj, node: &node, write: false); |
401 | if (IS_ERR(ptr: vma)) { |
402 | ret = PTR_ERR(ptr: vma); |
403 | goto out_rpm; |
404 | } |
405 | |
406 | user_data = u64_to_user_ptr(args->data_ptr); |
407 | remain = args->size; |
408 | offset = args->offset; |
409 | |
410 | while (remain > 0) { |
411 | /* Operation in this page |
412 | * |
413 | * page_base = page offset within aperture |
414 | * page_offset = offset within page |
415 | * page_length = bytes to copy for this page |
416 | */ |
417 | u32 page_base = node.start; |
418 | unsigned page_offset = offset_in_page(offset); |
419 | unsigned page_length = PAGE_SIZE - page_offset; |
420 | page_length = remain < page_length ? remain : page_length; |
421 | if (drm_mm_node_allocated(node: &node)) { |
422 | ggtt->vm.insert_page(&ggtt->vm, |
423 | i915_gem_object_get_dma_address(obj, |
424 | offset >> PAGE_SHIFT), |
425 | node.start, |
426 | i915_gem_get_pat_index(i915, |
427 | level: I915_CACHE_NONE), 0); |
428 | } else { |
429 | page_base += offset & PAGE_MASK; |
430 | } |
431 | |
432 | if (gtt_user_read(mapping: &ggtt->iomap, base: page_base, offset: page_offset, |
433 | user_data, length: page_length)) { |
434 | ret = -EFAULT; |
435 | break; |
436 | } |
437 | |
438 | remain -= page_length; |
439 | user_data += page_length; |
440 | offset += page_length; |
441 | } |
442 | |
443 | i915_gem_gtt_cleanup(obj, node: &node, vma); |
444 | out_rpm: |
445 | intel_runtime_pm_put(rpm: &i915->runtime_pm, wref: wakeref); |
446 | return ret; |
447 | } |
448 | |
449 | /** |
450 | * i915_gem_pread_ioctl - Reads data from the object referenced by handle. |
451 | * @dev: drm device pointer |
452 | * @data: ioctl data blob |
453 | * @file: drm file pointer |
454 | * |
455 | * On error, the contents of *data are undefined. |
456 | */ |
457 | int |
458 | i915_gem_pread_ioctl(struct drm_device *dev, void *data, |
459 | struct drm_file *file) |
460 | { |
461 | struct drm_i915_private *i915 = to_i915(dev); |
462 | struct drm_i915_gem_pread *args = data; |
463 | struct drm_i915_gem_object *obj; |
464 | int ret; |
465 | |
466 | /* PREAD is disallowed for all platforms after TGL-LP. This also |
467 | * covers all platforms with local memory. |
468 | */ |
469 | if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915)) |
470 | return -EOPNOTSUPP; |
471 | |
472 | if (args->size == 0) |
473 | return 0; |
474 | |
475 | if (!access_ok(u64_to_user_ptr(args->data_ptr), |
476 | args->size)) |
477 | return -EFAULT; |
478 | |
479 | obj = i915_gem_object_lookup(file, handle: args->handle); |
480 | if (!obj) |
481 | return -ENOENT; |
482 | |
483 | /* Bounds check source. */ |
484 | if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) { |
485 | ret = -EINVAL; |
486 | goto out; |
487 | } |
488 | |
489 | trace_i915_gem_object_pread(obj, offset: args->offset, len: args->size); |
490 | ret = -ENODEV; |
491 | if (obj->ops->pread) |
492 | ret = obj->ops->pread(obj, args); |
493 | if (ret != -ENODEV) |
494 | goto out; |
495 | |
496 | ret = i915_gem_object_wait(obj, |
497 | I915_WAIT_INTERRUPTIBLE, |
498 | MAX_SCHEDULE_TIMEOUT); |
499 | if (ret) |
500 | goto out; |
501 | |
502 | ret = i915_gem_shmem_pread(obj, args); |
503 | if (ret == -EFAULT || ret == -ENODEV) |
504 | ret = i915_gem_gtt_pread(obj, args); |
505 | |
506 | out: |
507 | i915_gem_object_put(obj); |
508 | return ret; |
509 | } |
510 | |
511 | /* This is the fast write path which cannot handle |
512 | * page faults in the source data |
513 | */ |
514 | |
515 | static inline bool |
516 | ggtt_write(struct io_mapping *mapping, |
517 | loff_t base, int offset, |
518 | char __user *user_data, int length) |
519 | { |
520 | void __iomem *vaddr; |
521 | unsigned long unwritten; |
522 | |
523 | /* We can use the cpu mem copy function because this is X86. */ |
524 | vaddr = io_mapping_map_atomic_wc(mapping, offset: base); |
525 | unwritten = __copy_from_user_inatomic_nocache(dst: (void __force *)vaddr + offset, |
526 | src: user_data, size: length); |
527 | io_mapping_unmap_atomic(vaddr); |
528 | if (unwritten) { |
529 | vaddr = io_mapping_map_wc(mapping, offset: base, PAGE_SIZE); |
530 | unwritten = copy_from_user(to: (void __force *)vaddr + offset, |
531 | from: user_data, n: length); |
532 | io_mapping_unmap(vaddr); |
533 | } |
534 | |
535 | return unwritten; |
536 | } |
537 | |
538 | /** |
539 | * i915_gem_gtt_pwrite_fast - This is the fast pwrite path, where we copy the data directly from the |
540 | * user into the GTT, uncached. |
541 | * @obj: i915 GEM object |
542 | * @args: pwrite arguments structure |
543 | */ |
544 | static int |
545 | i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, |
546 | const struct drm_i915_gem_pwrite *args) |
547 | { |
548 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
549 | struct i915_ggtt *ggtt = to_gt(i915)->ggtt; |
550 | struct intel_runtime_pm *rpm = &i915->runtime_pm; |
551 | unsigned long remain, offset; |
552 | intel_wakeref_t wakeref; |
553 | struct drm_mm_node node; |
554 | struct i915_vma *vma; |
555 | void __user *user_data; |
556 | int ret = 0; |
557 | |
558 | if (overflows_type(args->size, remain) || |
559 | overflows_type(args->offset, offset)) |
560 | return -EINVAL; |
561 | |
562 | if (i915_gem_object_has_struct_page(obj)) { |
563 | /* |
564 | * Avoid waking the device up if we can fallback, as |
565 | * waking/resuming is very slow (worst-case 10-100 ms |
566 | * depending on PCI sleeps and our own resume time). |
567 | * This easily dwarfs any performance advantage from |
568 | * using the cache bypass of indirect GGTT access. |
569 | */ |
570 | wakeref = intel_runtime_pm_get_if_in_use(rpm); |
571 | if (!wakeref) |
572 | return -EFAULT; |
573 | } else { |
574 | /* No backing pages, no fallback, we must force GGTT access */ |
575 | wakeref = intel_runtime_pm_get(rpm); |
576 | } |
577 | |
578 | vma = i915_gem_gtt_prepare(obj, node: &node, write: true); |
579 | if (IS_ERR(ptr: vma)) { |
580 | ret = PTR_ERR(ptr: vma); |
581 | goto out_rpm; |
582 | } |
583 | |
584 | i915_gem_object_invalidate_frontbuffer(obj, origin: ORIGIN_CPU); |
585 | |
586 | user_data = u64_to_user_ptr(args->data_ptr); |
587 | offset = args->offset; |
588 | remain = args->size; |
589 | while (remain) { |
590 | /* Operation in this page |
591 | * |
592 | * page_base = page offset within aperture |
593 | * page_offset = offset within page |
594 | * page_length = bytes to copy for this page |
595 | */ |
596 | u32 page_base = node.start; |
597 | unsigned int page_offset = offset_in_page(offset); |
598 | unsigned int page_length = PAGE_SIZE - page_offset; |
599 | page_length = remain < page_length ? remain : page_length; |
600 | if (drm_mm_node_allocated(node: &node)) { |
601 | /* flush the write before we modify the GGTT */ |
602 | intel_gt_flush_ggtt_writes(gt: ggtt->vm.gt); |
603 | ggtt->vm.insert_page(&ggtt->vm, |
604 | i915_gem_object_get_dma_address(obj, |
605 | offset >> PAGE_SHIFT), |
606 | node.start, |
607 | i915_gem_get_pat_index(i915, |
608 | level: I915_CACHE_NONE), 0); |
609 | wmb(); /* flush modifications to the GGTT (insert_page) */ |
610 | } else { |
611 | page_base += offset & PAGE_MASK; |
612 | } |
613 | /* If we get a fault while copying data, then (presumably) our |
614 | * source page isn't available. Return the error and we'll |
615 | * retry in the slow path. |
616 | * If the object is non-shmem backed, we retry again with the |
617 | * path that handles page fault. |
618 | */ |
619 | if (ggtt_write(mapping: &ggtt->iomap, base: page_base, offset: page_offset, |
620 | user_data, length: page_length)) { |
621 | ret = -EFAULT; |
622 | break; |
623 | } |
624 | |
625 | remain -= page_length; |
626 | user_data += page_length; |
627 | offset += page_length; |
628 | } |
629 | |
630 | intel_gt_flush_ggtt_writes(gt: ggtt->vm.gt); |
631 | i915_gem_object_flush_frontbuffer(obj, origin: ORIGIN_CPU); |
632 | |
633 | i915_gem_gtt_cleanup(obj, node: &node, vma); |
634 | out_rpm: |
635 | intel_runtime_pm_put(rpm, wref: wakeref); |
636 | return ret; |
637 | } |
638 | |
639 | /* Per-page copy function for the shmem pwrite fastpath. |
640 | * Flushes invalid cachelines before writing to the target if |
641 | * needs_clflush_before is set and flushes out any written cachelines after |
642 | * writing if needs_clflush is set. |
643 | */ |
644 | static int |
645 | shmem_pwrite(struct page *page, int offset, int len, char __user *user_data, |
646 | bool needs_clflush_before, |
647 | bool needs_clflush_after) |
648 | { |
649 | char *vaddr; |
650 | int ret; |
651 | |
652 | vaddr = kmap(page); |
653 | |
654 | if (needs_clflush_before) |
655 | drm_clflush_virt_range(addr: vaddr + offset, length: len); |
656 | |
657 | ret = __copy_from_user(to: vaddr + offset, from: user_data, n: len); |
658 | if (!ret && needs_clflush_after) |
659 | drm_clflush_virt_range(addr: vaddr + offset, length: len); |
660 | |
661 | kunmap(page); |
662 | |
663 | return ret ? -EFAULT : 0; |
664 | } |
665 | |
666 | static int |
667 | i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, |
668 | const struct drm_i915_gem_pwrite *args) |
669 | { |
670 | unsigned int partial_cacheline_write; |
671 | unsigned int needs_clflush; |
672 | void __user *user_data; |
673 | unsigned long offset; |
674 | pgoff_t idx; |
675 | u64 remain; |
676 | int ret; |
677 | |
678 | ret = i915_gem_object_lock_interruptible(obj, NULL); |
679 | if (ret) |
680 | return ret; |
681 | |
682 | ret = i915_gem_object_pin_pages(obj); |
683 | if (ret) |
684 | goto err_unlock; |
685 | |
686 | ret = i915_gem_object_prepare_write(obj, needs_clflush: &needs_clflush); |
687 | if (ret) |
688 | goto err_unpin; |
689 | |
690 | i915_gem_object_finish_access(obj); |
691 | i915_gem_object_unlock(obj); |
692 | |
693 | /* If we don't overwrite a cacheline completely we need to be |
694 | * careful to have up-to-date data by first clflushing. Don't |
695 | * overcomplicate things and flush the entire patch. |
696 | */ |
697 | partial_cacheline_write = 0; |
698 | if (needs_clflush & CLFLUSH_BEFORE) |
699 | partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1; |
700 | |
701 | user_data = u64_to_user_ptr(args->data_ptr); |
702 | remain = args->size; |
703 | offset = offset_in_page(args->offset); |
704 | for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { |
705 | struct page *page = i915_gem_object_get_page(obj, idx); |
706 | unsigned int length = min_t(u64, remain, PAGE_SIZE - offset); |
707 | |
708 | ret = shmem_pwrite(page, offset, len: length, user_data, |
709 | needs_clflush_before: (offset | length) & partial_cacheline_write, |
710 | needs_clflush_after: needs_clflush & CLFLUSH_AFTER); |
711 | if (ret) |
712 | break; |
713 | |
714 | remain -= length; |
715 | user_data += length; |
716 | offset = 0; |
717 | } |
718 | |
719 | i915_gem_object_flush_frontbuffer(obj, origin: ORIGIN_CPU); |
720 | |
721 | i915_gem_object_unpin_pages(obj); |
722 | return ret; |
723 | |
724 | err_unpin: |
725 | i915_gem_object_unpin_pages(obj); |
726 | err_unlock: |
727 | i915_gem_object_unlock(obj); |
728 | return ret; |
729 | } |
730 | |
731 | /** |
732 | * i915_gem_pwrite_ioctl - Writes data to the object referenced by handle. |
733 | * @dev: drm device |
734 | * @data: ioctl data blob |
735 | * @file: drm file |
736 | * |
737 | * On error, the contents of the buffer that were to be modified are undefined. |
738 | */ |
739 | int |
740 | i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, |
741 | struct drm_file *file) |
742 | { |
743 | struct drm_i915_private *i915 = to_i915(dev); |
744 | struct drm_i915_gem_pwrite *args = data; |
745 | struct drm_i915_gem_object *obj; |
746 | int ret; |
747 | |
748 | /* PWRITE is disallowed for all platforms after TGL-LP. This also |
749 | * covers all platforms with local memory. |
750 | */ |
751 | if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915)) |
752 | return -EOPNOTSUPP; |
753 | |
754 | if (args->size == 0) |
755 | return 0; |
756 | |
757 | if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size)) |
758 | return -EFAULT; |
759 | |
760 | obj = i915_gem_object_lookup(file, handle: args->handle); |
761 | if (!obj) |
762 | return -ENOENT; |
763 | |
764 | /* Bounds check destination. */ |
765 | if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) { |
766 | ret = -EINVAL; |
767 | goto err; |
768 | } |
769 | |
770 | /* Writes not allowed into this read-only object */ |
771 | if (i915_gem_object_is_readonly(obj)) { |
772 | ret = -EINVAL; |
773 | goto err; |
774 | } |
775 | |
776 | trace_i915_gem_object_pwrite(obj, offset: args->offset, len: args->size); |
777 | |
778 | ret = -ENODEV; |
779 | if (obj->ops->pwrite) |
780 | ret = obj->ops->pwrite(obj, args); |
781 | if (ret != -ENODEV) |
782 | goto err; |
783 | |
784 | ret = i915_gem_object_wait(obj, |
785 | I915_WAIT_INTERRUPTIBLE | |
786 | I915_WAIT_ALL, |
787 | MAX_SCHEDULE_TIMEOUT); |
788 | if (ret) |
789 | goto err; |
790 | |
791 | ret = -EFAULT; |
792 | /* We can only do the GTT pwrite on untiled buffers, as otherwise |
793 | * it would end up going through the fenced access, and we'll get |
794 | * different detiling behavior between reading and writing. |
795 | * pread/pwrite currently are reading and writing from the CPU |
796 | * perspective, requiring manual detiling by the client. |
797 | */ |
798 | if (!i915_gem_object_has_struct_page(obj) || |
799 | i915_gem_cpu_write_needs_clflush(obj)) |
800 | /* Note that the gtt paths might fail with non-page-backed user |
801 | * pointers (e.g. gtt mappings when moving data between |
802 | * textures). Fallback to the shmem path in that case. |
803 | */ |
804 | ret = i915_gem_gtt_pwrite_fast(obj, args); |
805 | |
806 | if (ret == -EFAULT || ret == -ENOSPC) { |
807 | if (i915_gem_object_has_struct_page(obj)) |
808 | ret = i915_gem_shmem_pwrite(obj, args); |
809 | } |
810 | |
811 | err: |
812 | i915_gem_object_put(obj); |
813 | return ret; |
814 | } |
815 | |
816 | /** |
817 | * i915_gem_sw_finish_ioctl - Called when user space has done writes to this buffer |
818 | * @dev: drm device |
819 | * @data: ioctl data blob |
820 | * @file: drm file |
821 | */ |
822 | int |
823 | i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, |
824 | struct drm_file *file) |
825 | { |
826 | struct drm_i915_gem_sw_finish *args = data; |
827 | struct drm_i915_gem_object *obj; |
828 | |
829 | obj = i915_gem_object_lookup(file, handle: args->handle); |
830 | if (!obj) |
831 | return -ENOENT; |
832 | |
833 | /* |
834 | * Proxy objects are barred from CPU access, so there is no |
835 | * need to ban sw_finish as it is a nop. |
836 | */ |
837 | |
838 | /* Pinned buffers may be scanout, so flush the cache */ |
839 | i915_gem_object_flush_if_display(obj); |
840 | i915_gem_object_put(obj); |
841 | |
842 | return 0; |
843 | } |
844 | |
845 | void i915_gem_runtime_suspend(struct drm_i915_private *i915) |
846 | { |
847 | struct drm_i915_gem_object *obj, *on; |
848 | int i; |
849 | |
850 | /* |
851 | * Only called during RPM suspend. All users of the userfault_list |
852 | * must be holding an RPM wakeref to ensure that this can not |
853 | * run concurrently with themselves (and use the struct_mutex for |
854 | * protection between themselves). |
855 | */ |
856 | |
857 | list_for_each_entry_safe(obj, on, |
858 | &to_gt(i915)->ggtt->userfault_list, userfault_link) |
859 | __i915_gem_object_release_mmap_gtt(obj); |
860 | |
861 | list_for_each_entry_safe(obj, on, |
862 | &i915->runtime_pm.lmem_userfault_list, userfault_link) |
863 | i915_gem_object_runtime_pm_release_mmap_offset(obj); |
864 | |
865 | /* |
866 | * The fence will be lost when the device powers down. If any were |
867 | * in use by hardware (i.e. they are pinned), we should not be powering |
868 | * down! All other fences will be reacquired by the user upon waking. |
869 | */ |
870 | for (i = 0; i < to_gt(i915)->ggtt->num_fences; i++) { |
871 | struct i915_fence_reg *reg = &to_gt(i915)->ggtt->fence_regs[i]; |
872 | |
873 | /* |
874 | * Ideally we want to assert that the fence register is not |
875 | * live at this point (i.e. that no piece of code will be |
876 | * trying to write through fence + GTT, as that both violates |
877 | * our tracking of activity and associated locking/barriers, |
878 | * but also is illegal given that the hw is powered down). |
879 | * |
880 | * Previously we used reg->pin_count as a "liveness" indicator. |
881 | * That is not sufficient, and we need a more fine-grained |
882 | * tool if we want to have a sanity check here. |
883 | */ |
884 | |
885 | if (!reg->vma) |
886 | continue; |
887 | |
888 | GEM_BUG_ON(i915_vma_has_userfault(reg->vma)); |
889 | reg->dirty = true; |
890 | } |
891 | } |
892 | |
893 | static void discard_ggtt_vma(struct i915_vma *vma) |
894 | { |
895 | struct drm_i915_gem_object *obj = vma->obj; |
896 | |
897 | spin_lock(lock: &obj->vma.lock); |
898 | if (!RB_EMPTY_NODE(&vma->obj_node)) { |
899 | rb_erase(&vma->obj_node, &obj->vma.tree); |
900 | RB_CLEAR_NODE(&vma->obj_node); |
901 | } |
902 | spin_unlock(lock: &obj->vma.lock); |
903 | } |
904 | |
905 | struct i915_vma * |
906 | i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj, |
907 | struct i915_gem_ww_ctx *ww, |
908 | const struct i915_gtt_view *view, |
909 | u64 size, u64 alignment, u64 flags) |
910 | { |
911 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
912 | struct i915_ggtt *ggtt = to_gt(i915)->ggtt; |
913 | struct i915_vma *vma; |
914 | int ret; |
915 | |
916 | GEM_WARN_ON(!ww); |
917 | |
918 | if (flags & PIN_MAPPABLE && |
919 | (!view || view->type == I915_GTT_VIEW_NORMAL)) { |
920 | /* |
921 | * If the required space is larger than the available |
922 | * aperture, we will not able to find a slot for the |
923 | * object and unbinding the object now will be in |
924 | * vain. Worse, doing so may cause us to ping-pong |
925 | * the object in and out of the Global GTT and |
926 | * waste a lot of cycles under the mutex. |
927 | */ |
928 | if (obj->base.size > ggtt->mappable_end) |
929 | return ERR_PTR(error: -E2BIG); |
930 | |
931 | /* |
932 | * If NONBLOCK is set the caller is optimistically |
933 | * trying to cache the full object within the mappable |
934 | * aperture, and *must* have a fallback in place for |
935 | * situations where we cannot bind the object. We |
936 | * can be a little more lax here and use the fallback |
937 | * more often to avoid costly migrations of ourselves |
938 | * and other objects within the aperture. |
939 | * |
940 | * Half-the-aperture is used as a simple heuristic. |
941 | * More interesting would to do search for a free |
942 | * block prior to making the commitment to unbind. |
943 | * That caters for the self-harm case, and with a |
944 | * little more heuristics (e.g. NOFAULT, NOEVICT) |
945 | * we could try to minimise harm to others. |
946 | */ |
947 | if (flags & PIN_NONBLOCK && |
948 | obj->base.size > ggtt->mappable_end / 2) |
949 | return ERR_PTR(error: -ENOSPC); |
950 | } |
951 | |
952 | new_vma: |
953 | vma = i915_vma_instance(obj, vm: &ggtt->vm, view); |
954 | if (IS_ERR(ptr: vma)) |
955 | return vma; |
956 | |
957 | if (i915_vma_misplaced(vma, size, alignment, flags)) { |
958 | if (flags & PIN_NONBLOCK) { |
959 | if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) |
960 | return ERR_PTR(error: -ENOSPC); |
961 | |
962 | /* |
963 | * If this misplaced vma is too big (i.e, at-least |
964 | * half the size of aperture) or hasn't been pinned |
965 | * mappable before, we ignore the misplacement when |
966 | * PIN_NONBLOCK is set in order to avoid the ping-pong |
967 | * issue described above. In other words, we try to |
968 | * avoid the costly operation of unbinding this vma |
969 | * from the GGTT and rebinding it back because there |
970 | * may not be enough space for this vma in the aperture. |
971 | */ |
972 | if (flags & PIN_MAPPABLE && |
973 | (vma->fence_size > ggtt->mappable_end / 2 || |
974 | !i915_vma_is_map_and_fenceable(vma))) |
975 | return ERR_PTR(error: -ENOSPC); |
976 | } |
977 | |
978 | if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) { |
979 | discard_ggtt_vma(vma); |
980 | goto new_vma; |
981 | } |
982 | |
983 | ret = i915_vma_unbind(vma); |
984 | if (ret) |
985 | return ERR_PTR(error: ret); |
986 | } |
987 | |
988 | ret = i915_vma_pin_ww(vma, ww, size, alignment, flags: flags | PIN_GLOBAL); |
989 | |
990 | if (ret) |
991 | return ERR_PTR(error: ret); |
992 | |
993 | if (vma->fence && !i915_gem_object_is_tiled(obj)) { |
994 | mutex_lock(&ggtt->vm.mutex); |
995 | i915_vma_revoke_fence(vma); |
996 | mutex_unlock(lock: &ggtt->vm.mutex); |
997 | } |
998 | |
999 | ret = i915_vma_wait_for_bind(vma); |
1000 | if (ret) { |
1001 | i915_vma_unpin(vma); |
1002 | return ERR_PTR(error: ret); |
1003 | } |
1004 | |
1005 | return vma; |
1006 | } |
1007 | |
1008 | struct i915_vma * __must_check |
1009 | i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, |
1010 | const struct i915_gtt_view *view, |
1011 | u64 size, u64 alignment, u64 flags) |
1012 | { |
1013 | struct i915_gem_ww_ctx ww; |
1014 | struct i915_vma *ret; |
1015 | int err; |
1016 | |
1017 | for_i915_gem_ww(&ww, err, true) { |
1018 | err = i915_gem_object_lock(obj, ww: &ww); |
1019 | if (err) |
1020 | continue; |
1021 | |
1022 | ret = i915_gem_object_ggtt_pin_ww(obj, ww: &ww, view, size, |
1023 | alignment, flags); |
1024 | if (IS_ERR(ptr: ret)) |
1025 | err = PTR_ERR(ptr: ret); |
1026 | } |
1027 | |
1028 | return err ? ERR_PTR(error: err) : ret; |
1029 | } |
1030 | |
1031 | int |
1032 | i915_gem_madvise_ioctl(struct drm_device *dev, void *data, |
1033 | struct drm_file *file_priv) |
1034 | { |
1035 | struct drm_i915_private *i915 = to_i915(dev); |
1036 | struct drm_i915_gem_madvise *args = data; |
1037 | struct drm_i915_gem_object *obj; |
1038 | int err; |
1039 | |
1040 | switch (args->madv) { |
1041 | case I915_MADV_DONTNEED: |
1042 | case I915_MADV_WILLNEED: |
1043 | break; |
1044 | default: |
1045 | return -EINVAL; |
1046 | } |
1047 | |
1048 | obj = i915_gem_object_lookup(file: file_priv, handle: args->handle); |
1049 | if (!obj) |
1050 | return -ENOENT; |
1051 | |
1052 | err = i915_gem_object_lock_interruptible(obj, NULL); |
1053 | if (err) |
1054 | goto out; |
1055 | |
1056 | if (i915_gem_object_has_pages(obj) && |
1057 | i915_gem_object_is_tiled(obj) && |
1058 | i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) { |
1059 | if (obj->mm.madv == I915_MADV_WILLNEED) { |
1060 | GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj)); |
1061 | i915_gem_object_clear_tiling_quirk(obj); |
1062 | i915_gem_object_make_shrinkable(obj); |
1063 | } |
1064 | if (args->madv == I915_MADV_WILLNEED) { |
1065 | GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj)); |
1066 | i915_gem_object_make_unshrinkable(obj); |
1067 | i915_gem_object_set_tiling_quirk(obj); |
1068 | } |
1069 | } |
1070 | |
1071 | if (obj->mm.madv != __I915_MADV_PURGED) { |
1072 | obj->mm.madv = args->madv; |
1073 | if (obj->ops->adjust_lru) |
1074 | obj->ops->adjust_lru(obj); |
1075 | } |
1076 | |
1077 | if (i915_gem_object_has_pages(obj) || |
1078 | i915_gem_object_has_self_managed_shrink_list(obj)) { |
1079 | unsigned long flags; |
1080 | |
1081 | spin_lock_irqsave(&i915->mm.obj_lock, flags); |
1082 | if (!list_empty(head: &obj->mm.link)) { |
1083 | struct list_head *list; |
1084 | |
1085 | if (obj->mm.madv != I915_MADV_WILLNEED) |
1086 | list = &i915->mm.purge_list; |
1087 | else |
1088 | list = &i915->mm.shrink_list; |
1089 | list_move_tail(list: &obj->mm.link, head: list); |
1090 | |
1091 | } |
1092 | spin_unlock_irqrestore(lock: &i915->mm.obj_lock, flags); |
1093 | } |
1094 | |
1095 | /* if the object is no longer attached, discard its backing storage */ |
1096 | if (obj->mm.madv == I915_MADV_DONTNEED && |
1097 | !i915_gem_object_has_pages(obj)) |
1098 | i915_gem_object_truncate(obj); |
1099 | |
1100 | args->retained = obj->mm.madv != __I915_MADV_PURGED; |
1101 | |
1102 | i915_gem_object_unlock(obj); |
1103 | out: |
1104 | i915_gem_object_put(obj); |
1105 | return err; |
1106 | } |
1107 | |
1108 | /* |
1109 | * A single pass should suffice to release all the freed objects (along most |
1110 | * call paths), but be a little more paranoid in that freeing the objects does |
1111 | * take a little amount of time, during which the rcu callbacks could have added |
1112 | * new objects into the freed list, and armed the work again. |
1113 | */ |
1114 | void i915_gem_drain_freed_objects(struct drm_i915_private *i915) |
1115 | { |
1116 | while (atomic_read(v: &i915->mm.free_count)) { |
1117 | flush_work(work: &i915->mm.free_work); |
1118 | drain_workqueue(wq: i915->bdev.wq); |
1119 | rcu_barrier(); |
1120 | } |
1121 | } |
1122 | |
1123 | /* |
1124 | * Similar to objects above (see i915_gem_drain_freed-objects), in general we |
1125 | * have workers that are armed by RCU and then rearm themselves in their |
1126 | * callbacks. To be paranoid, we need to drain the workqueue a second time after |
1127 | * waiting for the RCU grace period so that we catch work queued via RCU from |
1128 | * the first pass. As neither drain_workqueue() nor flush_workqueue() report a |
1129 | * result, we make an assumption that we only don't require more than 3 passes |
1130 | * to catch all _recursive_ RCU delayed work. |
1131 | */ |
1132 | void i915_gem_drain_workqueue(struct drm_i915_private *i915) |
1133 | { |
1134 | int i; |
1135 | |
1136 | for (i = 0; i < 3; i++) { |
1137 | flush_workqueue(i915->wq); |
1138 | rcu_barrier(); |
1139 | i915_gem_drain_freed_objects(i915); |
1140 | } |
1141 | |
1142 | drain_workqueue(wq: i915->wq); |
1143 | } |
1144 | |
1145 | int i915_gem_init(struct drm_i915_private *dev_priv) |
1146 | { |
1147 | struct intel_gt *gt; |
1148 | unsigned int i; |
1149 | int ret; |
1150 | |
1151 | /* |
1152 | * In the proccess of replacing cache_level with pat_index a tricky |
1153 | * dependency is created on the definition of the enum i915_cache_level. |
1154 | * in case this enum is changed, PTE encode would be broken. |
1155 | * Add a WARNING here. And remove when we completely quit using this |
1156 | * enum |
1157 | */ |
1158 | BUILD_BUG_ON(I915_CACHE_NONE != 0 || |
1159 | I915_CACHE_LLC != 1 || |
1160 | I915_CACHE_L3_LLC != 2 || |
1161 | I915_CACHE_WT != 3 || |
1162 | I915_MAX_CACHE_LEVEL != 4); |
1163 | |
1164 | /* We need to fallback to 4K pages if host doesn't support huge gtt. */ |
1165 | if (intel_vgpu_active(i915: dev_priv) && !intel_vgpu_has_huge_gtt(i915: dev_priv)) |
1166 | RUNTIME_INFO(dev_priv)->page_sizes = I915_GTT_PAGE_SIZE_4K; |
1167 | |
1168 | ret = i915_gem_init_userptr(dev_priv); |
1169 | if (ret) |
1170 | return ret; |
1171 | |
1172 | for_each_gt(gt, dev_priv, i) { |
1173 | intel_uc_fetch_firmwares(uc: >->uc); |
1174 | intel_wopcm_init(wopcm: >->wopcm); |
1175 | if (GRAPHICS_VER(dev_priv) >= 8) |
1176 | setup_private_pat(gt); |
1177 | } |
1178 | |
1179 | ret = i915_init_ggtt(i915: dev_priv); |
1180 | if (ret) { |
1181 | GEM_BUG_ON(ret == -EIO); |
1182 | goto err_unlock; |
1183 | } |
1184 | |
1185 | /* |
1186 | * Despite its name intel_clock_gating_init applies both display |
1187 | * clock gating workarounds; GT mmio workarounds and the occasional |
1188 | * GT power context workaround. Worse, sometimes it includes a context |
1189 | * register workaround which we need to apply before we record the |
1190 | * default HW state for all contexts. |
1191 | * |
1192 | * FIXME: break up the workarounds and apply them at the right time! |
1193 | */ |
1194 | intel_clock_gating_init(i915: dev_priv); |
1195 | |
1196 | for_each_gt(gt, dev_priv, i) { |
1197 | ret = intel_gt_init(gt); |
1198 | if (ret) |
1199 | goto err_unlock; |
1200 | } |
1201 | |
1202 | /* |
1203 | * Register engines early to ensure the engine list is in its final |
1204 | * rb-tree form, lowering the amount of code that has to deal with |
1205 | * the intermediate llist state. |
1206 | */ |
1207 | intel_engines_driver_register(i915: dev_priv); |
1208 | |
1209 | return 0; |
1210 | |
1211 | /* |
1212 | * Unwinding is complicated by that we want to handle -EIO to mean |
1213 | * disable GPU submission but keep KMS alive. We want to mark the |
1214 | * HW as irrevisibly wedged, but keep enough state around that the |
1215 | * driver doesn't explode during runtime. |
1216 | */ |
1217 | err_unlock: |
1218 | i915_gem_drain_workqueue(i915: dev_priv); |
1219 | |
1220 | if (ret != -EIO) { |
1221 | for_each_gt(gt, dev_priv, i) { |
1222 | intel_gt_driver_remove(gt); |
1223 | intel_gt_driver_release(gt); |
1224 | intel_uc_cleanup_firmwares(uc: >->uc); |
1225 | } |
1226 | } |
1227 | |
1228 | if (ret == -EIO) { |
1229 | /* |
1230 | * Allow engines or uC initialisation to fail by marking the GPU |
1231 | * as wedged. But we only want to do this when the GPU is angry, |
1232 | * for all other failure, such as an allocation failure, bail. |
1233 | */ |
1234 | for_each_gt(gt, dev_priv, i) { |
1235 | if (!intel_gt_is_wedged(gt)) { |
1236 | i915_probe_error(dev_priv, |
1237 | "Failed to initialize GPU, declaring it wedged!\n" ); |
1238 | intel_gt_set_wedged(gt); |
1239 | } |
1240 | } |
1241 | |
1242 | /* Minimal basic recovery for KMS */ |
1243 | ret = i915_ggtt_enable_hw(i915: dev_priv); |
1244 | i915_ggtt_resume(ggtt: to_gt(i915: dev_priv)->ggtt); |
1245 | intel_clock_gating_init(i915: dev_priv); |
1246 | } |
1247 | |
1248 | i915_gem_drain_freed_objects(i915: dev_priv); |
1249 | |
1250 | return ret; |
1251 | } |
1252 | |
1253 | void i915_gem_driver_register(struct drm_i915_private *i915) |
1254 | { |
1255 | i915_gem_driver_register__shrinker(i915); |
1256 | } |
1257 | |
1258 | void i915_gem_driver_unregister(struct drm_i915_private *i915) |
1259 | { |
1260 | i915_gem_driver_unregister__shrinker(i915); |
1261 | } |
1262 | |
1263 | void i915_gem_driver_remove(struct drm_i915_private *dev_priv) |
1264 | { |
1265 | struct intel_gt *gt; |
1266 | unsigned int i; |
1267 | |
1268 | i915_gem_suspend_late(i915: dev_priv); |
1269 | for_each_gt(gt, dev_priv, i) |
1270 | intel_gt_driver_remove(gt); |
1271 | dev_priv->uabi_engines = RB_ROOT; |
1272 | |
1273 | /* Flush any outstanding unpin_work. */ |
1274 | i915_gem_drain_workqueue(i915: dev_priv); |
1275 | } |
1276 | |
1277 | void i915_gem_driver_release(struct drm_i915_private *dev_priv) |
1278 | { |
1279 | struct intel_gt *gt; |
1280 | unsigned int i; |
1281 | |
1282 | for_each_gt(gt, dev_priv, i) { |
1283 | intel_gt_driver_release(gt); |
1284 | intel_uc_cleanup_firmwares(uc: >->uc); |
1285 | } |
1286 | |
1287 | /* Flush any outstanding work, including i915_gem_context.release_work. */ |
1288 | i915_gem_drain_workqueue(i915: dev_priv); |
1289 | |
1290 | drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list)); |
1291 | } |
1292 | |
1293 | static void i915_gem_init__mm(struct drm_i915_private *i915) |
1294 | { |
1295 | spin_lock_init(&i915->mm.obj_lock); |
1296 | |
1297 | init_llist_head(list: &i915->mm.free_list); |
1298 | |
1299 | INIT_LIST_HEAD(list: &i915->mm.purge_list); |
1300 | INIT_LIST_HEAD(list: &i915->mm.shrink_list); |
1301 | |
1302 | i915_gem_init__objects(i915); |
1303 | } |
1304 | |
1305 | void i915_gem_init_early(struct drm_i915_private *dev_priv) |
1306 | { |
1307 | i915_gem_init__mm(i915: dev_priv); |
1308 | i915_gem_init__contexts(i915: dev_priv); |
1309 | |
1310 | spin_lock_init(&dev_priv->display.fb_tracking.lock); |
1311 | } |
1312 | |
1313 | void i915_gem_cleanup_early(struct drm_i915_private *dev_priv) |
1314 | { |
1315 | i915_gem_drain_workqueue(i915: dev_priv); |
1316 | GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list)); |
1317 | GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count)); |
1318 | drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count); |
1319 | } |
1320 | |
1321 | int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file) |
1322 | { |
1323 | struct drm_i915_file_private *file_priv; |
1324 | struct i915_drm_client *client; |
1325 | int ret = -ENOMEM; |
1326 | |
1327 | drm_dbg(&i915->drm, "\n" ); |
1328 | |
1329 | file_priv = kzalloc(size: sizeof(*file_priv), GFP_KERNEL); |
1330 | if (!file_priv) |
1331 | goto err_alloc; |
1332 | |
1333 | client = i915_drm_client_alloc(); |
1334 | if (!client) |
1335 | goto err_client; |
1336 | |
1337 | file->driver_priv = file_priv; |
1338 | file_priv->i915 = i915; |
1339 | file_priv->file = file; |
1340 | file_priv->client = client; |
1341 | |
1342 | file_priv->bsd_engine = -1; |
1343 | file_priv->hang_timestamp = jiffies; |
1344 | |
1345 | ret = i915_gem_context_open(i915, file); |
1346 | if (ret) |
1347 | goto err_context; |
1348 | |
1349 | return 0; |
1350 | |
1351 | err_context: |
1352 | i915_drm_client_put(client); |
1353 | err_client: |
1354 | kfree(objp: file_priv); |
1355 | err_alloc: |
1356 | return ret; |
1357 | } |
1358 | |
1359 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) |
1360 | #include "selftests/mock_gem_device.c" |
1361 | #include "selftests/i915_gem.c" |
1362 | #endif |
1363 | |