1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * Copyright (C) 2013 Red Hat |
4 | * Author: Rob Clark <robdclark@gmail.com> |
5 | */ |
6 | |
7 | #ifndef __MSM_GEM_H__ |
8 | #define __MSM_GEM_H__ |
9 | |
10 | #include <linux/kref.h> |
11 | #include <linux/dma-resv.h> |
12 | #include "drm/drm_exec.h" |
13 | #include "drm/gpu_scheduler.h" |
14 | #include "msm_drv.h" |
15 | |
16 | /* Make all GEM related WARN_ON()s ratelimited.. when things go wrong they |
17 | * tend to go wrong 1000s of times in a short timespan. |
18 | */ |
19 | #define GEM_WARN_ON(x) WARN_RATELIMIT(x, "%s", __stringify(x)) |
20 | |
21 | /* Additional internal-use only BO flags: */ |
22 | #define MSM_BO_STOLEN 0x10000000 /* try to use stolen/splash memory */ |
23 | #define MSM_BO_MAP_PRIV 0x20000000 /* use IOMMU_PRIV when mapping */ |
24 | |
25 | struct msm_gem_address_space { |
26 | const char *name; |
27 | /* NOTE: mm managed at the page level, size is in # of pages |
28 | * and position mm_node->start is in # of pages: |
29 | */ |
30 | struct drm_mm mm; |
31 | spinlock_t lock; /* Protects drm_mm node allocation/removal */ |
32 | struct msm_mmu *mmu; |
33 | struct kref kref; |
34 | |
35 | /* For address spaces associated with a specific process, this |
36 | * will be non-NULL: |
37 | */ |
38 | struct pid *pid; |
39 | |
40 | /* @faults: the number of GPU hangs associated with this address space */ |
41 | int faults; |
42 | |
43 | /** @va_start: lowest possible address to allocate */ |
44 | uint64_t va_start; |
45 | |
46 | /** @va_size: the size of the address space (in bytes) */ |
47 | uint64_t va_size; |
48 | }; |
49 | |
50 | struct msm_gem_address_space * |
51 | msm_gem_address_space_get(struct msm_gem_address_space *aspace); |
52 | |
53 | void msm_gem_address_space_put(struct msm_gem_address_space *aspace); |
54 | |
55 | struct msm_gem_address_space * |
56 | msm_gem_address_space_create(struct msm_mmu *mmu, const char *name, |
57 | u64 va_start, u64 size); |
58 | |
59 | struct msm_fence_context; |
60 | |
61 | struct msm_gem_vma { |
62 | struct drm_mm_node node; |
63 | uint64_t iova; |
64 | struct msm_gem_address_space *aspace; |
65 | struct list_head list; /* node in msm_gem_object::vmas */ |
66 | bool mapped; |
67 | }; |
68 | |
69 | struct msm_gem_vma *msm_gem_vma_new(struct msm_gem_address_space *aspace); |
70 | int msm_gem_vma_init(struct msm_gem_vma *vma, int size, |
71 | u64 range_start, u64 range_end); |
72 | void msm_gem_vma_purge(struct msm_gem_vma *vma); |
73 | int msm_gem_vma_map(struct msm_gem_vma *vma, int prot, struct sg_table *sgt, int size); |
74 | void msm_gem_vma_close(struct msm_gem_vma *vma); |
75 | |
76 | struct msm_gem_object { |
77 | struct drm_gem_object base; |
78 | |
79 | uint32_t flags; |
80 | |
81 | /** |
82 | * madv: are the backing pages purgeable? |
83 | * |
84 | * Protected by obj lock and LRU lock |
85 | */ |
86 | uint8_t madv; |
87 | |
88 | /** |
89 | * count of active vmap'ing |
90 | */ |
91 | uint8_t vmap_count; |
92 | |
93 | /** |
94 | * Node in list of all objects (mainly for debugfs, protected by |
95 | * priv->obj_lock |
96 | */ |
97 | struct list_head node; |
98 | |
99 | struct page **pages; |
100 | struct sg_table *sgt; |
101 | void *vaddr; |
102 | |
103 | struct list_head vmas; /* list of msm_gem_vma */ |
104 | |
105 | /* For physically contiguous buffers. Used when we don't have |
106 | * an IOMMU. Also used for stolen/splashscreen buffer. |
107 | */ |
108 | struct drm_mm_node *vram_node; |
109 | |
110 | char name[32]; /* Identifier to print for the debugfs files */ |
111 | |
112 | /* userspace metadata backchannel */ |
113 | void *metadata; |
114 | u32 metadata_size; |
115 | |
116 | /** |
117 | * pin_count: Number of times the pages are pinned |
118 | * |
119 | * Protected by LRU lock. |
120 | */ |
121 | int pin_count; |
122 | }; |
123 | #define to_msm_bo(x) container_of(x, struct msm_gem_object, base) |
124 | |
125 | uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj); |
126 | int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma); |
127 | void msm_gem_unpin_locked(struct drm_gem_object *obj); |
128 | void msm_gem_unpin_active(struct drm_gem_object *obj); |
129 | struct msm_gem_vma *msm_gem_get_vma_locked(struct drm_gem_object *obj, |
130 | struct msm_gem_address_space *aspace); |
131 | int msm_gem_get_iova(struct drm_gem_object *obj, |
132 | struct msm_gem_address_space *aspace, uint64_t *iova); |
133 | int msm_gem_set_iova(struct drm_gem_object *obj, |
134 | struct msm_gem_address_space *aspace, uint64_t iova); |
135 | int msm_gem_get_and_pin_iova_range(struct drm_gem_object *obj, |
136 | struct msm_gem_address_space *aspace, uint64_t *iova, |
137 | u64 range_start, u64 range_end); |
138 | int msm_gem_get_and_pin_iova(struct drm_gem_object *obj, |
139 | struct msm_gem_address_space *aspace, uint64_t *iova); |
140 | void msm_gem_unpin_iova(struct drm_gem_object *obj, |
141 | struct msm_gem_address_space *aspace); |
142 | void msm_gem_pin_obj_locked(struct drm_gem_object *obj); |
143 | struct page **msm_gem_pin_pages(struct drm_gem_object *obj); |
144 | void msm_gem_unpin_pages(struct drm_gem_object *obj); |
145 | int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev, |
146 | struct drm_mode_create_dumb *args); |
147 | int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, |
148 | uint32_t handle, uint64_t *offset); |
149 | void *msm_gem_get_vaddr_locked(struct drm_gem_object *obj); |
150 | void *msm_gem_get_vaddr(struct drm_gem_object *obj); |
151 | void *msm_gem_get_vaddr_active(struct drm_gem_object *obj); |
152 | void msm_gem_put_vaddr_locked(struct drm_gem_object *obj); |
153 | void msm_gem_put_vaddr(struct drm_gem_object *obj); |
154 | int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv); |
155 | bool msm_gem_active(struct drm_gem_object *obj); |
156 | int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout); |
157 | int msm_gem_cpu_fini(struct drm_gem_object *obj); |
158 | int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file, |
159 | uint32_t size, uint32_t flags, uint32_t *handle, char *name); |
160 | struct drm_gem_object *msm_gem_new(struct drm_device *dev, |
161 | uint32_t size, uint32_t flags); |
162 | void *msm_gem_kernel_new(struct drm_device *dev, uint32_t size, |
163 | uint32_t flags, struct msm_gem_address_space *aspace, |
164 | struct drm_gem_object **bo, uint64_t *iova); |
165 | void msm_gem_kernel_put(struct drm_gem_object *bo, |
166 | struct msm_gem_address_space *aspace); |
167 | struct drm_gem_object *msm_gem_import(struct drm_device *dev, |
168 | struct dma_buf *dmabuf, struct sg_table *sgt); |
169 | __printf(2, 3) |
170 | void msm_gem_object_set_name(struct drm_gem_object *bo, const char *fmt, ...); |
171 | |
172 | #ifdef CONFIG_DEBUG_FS |
173 | struct msm_gem_stats { |
174 | struct { |
175 | unsigned count; |
176 | size_t size; |
177 | } all, active, resident, purgeable, purged; |
178 | }; |
179 | |
180 | void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m, |
181 | struct msm_gem_stats *stats); |
182 | void msm_gem_describe_objects(struct list_head *list, struct seq_file *m); |
183 | #endif |
184 | |
185 | static inline void |
186 | msm_gem_lock(struct drm_gem_object *obj) |
187 | { |
188 | dma_resv_lock(obj: obj->resv, NULL); |
189 | } |
190 | |
191 | static inline int |
192 | msm_gem_lock_interruptible(struct drm_gem_object *obj) |
193 | { |
194 | return dma_resv_lock_interruptible(obj: obj->resv, NULL); |
195 | } |
196 | |
197 | static inline void |
198 | msm_gem_unlock(struct drm_gem_object *obj) |
199 | { |
200 | dma_resv_unlock(obj: obj->resv); |
201 | } |
202 | |
203 | static inline void |
204 | msm_gem_assert_locked(struct drm_gem_object *obj) |
205 | { |
206 | /* |
207 | * Destroying the object is a special case.. msm_gem_free_object() |
208 | * calls many things that WARN_ON if the obj lock is not held. But |
209 | * acquiring the obj lock in msm_gem_free_object() can cause a |
210 | * locking order inversion between reservation_ww_class_mutex and |
211 | * fs_reclaim. |
212 | * |
213 | * This deadlock is not actually possible, because no one should |
214 | * be already holding the lock when msm_gem_free_object() is called. |
215 | * Unfortunately lockdep is not aware of this detail. So when the |
216 | * refcount drops to zero, we pretend it is already locked. |
217 | */ |
218 | lockdep_assert_once( |
219 | (kref_read(&obj->refcount) == 0) || |
220 | (lockdep_is_held(&obj->resv->lock.base) != LOCK_STATE_NOT_HELD) |
221 | ); |
222 | } |
223 | |
224 | /* imported/exported objects are not purgeable: */ |
225 | static inline bool is_unpurgeable(struct msm_gem_object *msm_obj) |
226 | { |
227 | return msm_obj->base.import_attach || msm_obj->pin_count; |
228 | } |
229 | |
230 | static inline bool is_purgeable(struct msm_gem_object *msm_obj) |
231 | { |
232 | return (msm_obj->madv == MSM_MADV_DONTNEED) && msm_obj->sgt && |
233 | !is_unpurgeable(msm_obj); |
234 | } |
235 | |
236 | static inline bool is_vunmapable(struct msm_gem_object *msm_obj) |
237 | { |
238 | msm_gem_assert_locked(obj: &msm_obj->base); |
239 | return (msm_obj->vmap_count == 0) && msm_obj->vaddr; |
240 | } |
241 | |
242 | static inline bool is_unevictable(struct msm_gem_object *msm_obj) |
243 | { |
244 | return is_unpurgeable(msm_obj) || msm_obj->vaddr; |
245 | } |
246 | |
247 | void msm_gem_purge(struct drm_gem_object *obj); |
248 | void msm_gem_evict(struct drm_gem_object *obj); |
249 | void msm_gem_vunmap(struct drm_gem_object *obj); |
250 | |
251 | /* Created per submit-ioctl, to track bo's and cmdstream bufs, etc, |
252 | * associated with the cmdstream submission for synchronization (and |
253 | * make it easier to unwind when things go wrong, etc). |
254 | */ |
255 | struct msm_gem_submit { |
256 | struct drm_sched_job base; |
257 | struct kref ref; |
258 | struct drm_device *dev; |
259 | struct msm_gpu *gpu; |
260 | struct msm_gem_address_space *aspace; |
261 | struct list_head node; /* node in ring submit list */ |
262 | struct drm_exec exec; |
263 | uint32_t seqno; /* Sequence number of the submit on the ring */ |
264 | |
265 | /* Hw fence, which is created when the scheduler executes the job, and |
266 | * is signaled when the hw finishes (via seqno write from cmdstream) |
267 | */ |
268 | struct dma_fence *hw_fence; |
269 | |
270 | /* Userspace visible fence, which is signaled by the scheduler after |
271 | * the hw_fence is signaled. |
272 | */ |
273 | struct dma_fence *user_fence; |
274 | |
275 | int fence_id; /* key into queue->fence_idr */ |
276 | struct msm_gpu_submitqueue *queue; |
277 | struct pid *pid; /* submitting process */ |
278 | bool bos_pinned : 1; |
279 | bool fault_dumped:1;/* Limit devcoredump dumping to one per submit */ |
280 | bool in_rb : 1; /* "sudo" mode, copy cmds into RB */ |
281 | struct msm_ringbuffer *ring; |
282 | unsigned int nr_cmds; |
283 | unsigned int nr_bos; |
284 | u32 ident; /* A "identifier" for the submit for logging */ |
285 | struct { |
286 | uint32_t type; |
287 | uint32_t size; /* in dwords */ |
288 | uint64_t iova; |
289 | uint32_t offset;/* in dwords */ |
290 | uint32_t idx; /* cmdstream buffer idx in bos[] */ |
291 | uint32_t nr_relocs; |
292 | struct drm_msm_gem_submit_reloc *relocs; |
293 | } *cmd; /* array of size nr_cmds */ |
294 | struct { |
295 | uint32_t flags; |
296 | union { |
297 | struct drm_gem_object *obj; |
298 | uint32_t handle; |
299 | }; |
300 | uint64_t iova; |
301 | } bos[]; |
302 | }; |
303 | |
304 | static inline struct msm_gem_submit *to_msm_submit(struct drm_sched_job *job) |
305 | { |
306 | return container_of(job, struct msm_gem_submit, base); |
307 | } |
308 | |
309 | void __msm_gem_submit_destroy(struct kref *kref); |
310 | |
311 | static inline void msm_gem_submit_get(struct msm_gem_submit *submit) |
312 | { |
313 | kref_get(kref: &submit->ref); |
314 | } |
315 | |
316 | static inline void msm_gem_submit_put(struct msm_gem_submit *submit) |
317 | { |
318 | kref_put(kref: &submit->ref, release: __msm_gem_submit_destroy); |
319 | } |
320 | |
321 | void msm_submit_retire(struct msm_gem_submit *submit); |
322 | |
323 | /* helper to determine of a buffer in submit should be dumped, used for both |
324 | * devcoredump and debugfs cmdstream dumping: |
325 | */ |
326 | static inline bool |
327 | should_dump(struct msm_gem_submit *submit, int idx) |
328 | { |
329 | extern bool rd_full; |
330 | return rd_full || (submit->bos[idx].flags & MSM_SUBMIT_BO_DUMP); |
331 | } |
332 | |
333 | #endif /* __MSM_GEM_H__ */ |
334 | |