1 | /* SPDX-License-Identifier: MIT */ |
2 | #ifndef _INTEL_RINGBUFFER_H_ |
3 | #define _INTEL_RINGBUFFER_H_ |
4 | |
5 | #include <asm/cacheflush.h> |
6 | #include <drm/drm_util.h> |
7 | #include <drm/drm_cache.h> |
8 | |
9 | #include <linux/hashtable.h> |
10 | #include <linux/irq_work.h> |
11 | #include <linux/random.h> |
12 | #include <linux/seqlock.h> |
13 | |
14 | #include "i915_pmu.h" |
15 | #include "i915_request.h" |
16 | #include "i915_selftest.h" |
17 | #include "intel_engine_types.h" |
18 | #include "intel_gt_types.h" |
19 | #include "intel_timeline.h" |
20 | #include "intel_workarounds.h" |
21 | |
22 | struct drm_printer; |
23 | struct intel_context; |
24 | struct intel_gt; |
25 | struct lock_class_key; |
26 | |
27 | /* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill, |
28 | * but keeps the logic simple. Indeed, the whole purpose of this macro is just |
29 | * to give some inclination as to some of the magic values used in the various |
30 | * workarounds! |
31 | */ |
32 | #define CACHELINE_BYTES 64 |
33 | #define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(u32)) |
34 | |
35 | #define ENGINE_TRACE(e, fmt, ...) do { \ |
36 | const struct intel_engine_cs *e__ __maybe_unused = (e); \ |
37 | GEM_TRACE("%s %s: " fmt, \ |
38 | dev_name(e__->i915->drm.dev), e__->name, \ |
39 | ##__VA_ARGS__); \ |
40 | } while (0) |
41 | |
42 | /* |
43 | * The register defines to be used with the following macros need to accept a |
44 | * base param, e.g: |
45 | * |
46 | * REG_FOO(base) _MMIO((base) + <relative offset>) |
47 | * ENGINE_READ(engine, REG_FOO); |
48 | * |
49 | * register arrays are to be defined and accessed as follows: |
50 | * |
51 | * REG_BAR(base, i) _MMIO((base) + <relative offset> + (i) * <shift>) |
52 | * ENGINE_READ_IDX(engine, REG_BAR, i) |
53 | */ |
54 | |
55 | #define __ENGINE_REG_OP(op__, engine__, ...) \ |
56 | intel_uncore_##op__((engine__)->uncore, __VA_ARGS__) |
57 | |
58 | #define __ENGINE_READ_OP(op__, engine__, reg__) \ |
59 | __ENGINE_REG_OP(op__, (engine__), reg__((engine__)->mmio_base)) |
60 | |
61 | #define ENGINE_READ16(...) __ENGINE_READ_OP(read16, __VA_ARGS__) |
62 | #define ENGINE_READ(...) __ENGINE_READ_OP(read, __VA_ARGS__) |
63 | #define ENGINE_READ_FW(...) __ENGINE_READ_OP(read_fw, __VA_ARGS__) |
64 | #define ENGINE_POSTING_READ(...) __ENGINE_READ_OP(posting_read_fw, __VA_ARGS__) |
65 | #define ENGINE_POSTING_READ16(...) __ENGINE_READ_OP(posting_read16, __VA_ARGS__) |
66 | |
67 | #define ENGINE_READ64(engine__, lower_reg__, upper_reg__) \ |
68 | __ENGINE_REG_OP(read64_2x32, (engine__), \ |
69 | lower_reg__((engine__)->mmio_base), \ |
70 | upper_reg__((engine__)->mmio_base)) |
71 | |
72 | #define ENGINE_READ_IDX(engine__, reg__, idx__) \ |
73 | __ENGINE_REG_OP(read, (engine__), reg__((engine__)->mmio_base, (idx__))) |
74 | |
75 | #define __ENGINE_WRITE_OP(op__, engine__, reg__, val__) \ |
76 | __ENGINE_REG_OP(op__, (engine__), reg__((engine__)->mmio_base), (val__)) |
77 | |
78 | #define ENGINE_WRITE16(...) __ENGINE_WRITE_OP(write16, __VA_ARGS__) |
79 | #define ENGINE_WRITE(...) __ENGINE_WRITE_OP(write, __VA_ARGS__) |
80 | #define ENGINE_WRITE_FW(...) __ENGINE_WRITE_OP(write_fw, __VA_ARGS__) |
81 | |
82 | #define GEN6_RING_FAULT_REG_READ(engine__) \ |
83 | intel_uncore_read((engine__)->uncore, RING_FAULT_REG(engine__)) |
84 | |
85 | #define GEN6_RING_FAULT_REG_POSTING_READ(engine__) \ |
86 | intel_uncore_posting_read((engine__)->uncore, RING_FAULT_REG(engine__)) |
87 | |
88 | #define GEN6_RING_FAULT_REG_RMW(engine__, clear__, set__) \ |
89 | ({ \ |
90 | u32 __val; \ |
91 | \ |
92 | __val = intel_uncore_read((engine__)->uncore, \ |
93 | RING_FAULT_REG(engine__)); \ |
94 | __val &= ~(clear__); \ |
95 | __val |= (set__); \ |
96 | intel_uncore_write((engine__)->uncore, RING_FAULT_REG(engine__), \ |
97 | __val); \ |
98 | }) |
99 | |
100 | /* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to |
101 | * do the writes, and that must have qw aligned offsets, simply pretend it's 8b. |
102 | */ |
103 | |
104 | static inline unsigned int |
105 | execlists_num_ports(const struct intel_engine_execlists * const execlists) |
106 | { |
107 | return execlists->port_mask + 1; |
108 | } |
109 | |
110 | static inline struct i915_request * |
111 | execlists_active(const struct intel_engine_execlists *execlists) |
112 | { |
113 | struct i915_request * const *cur, * const *old, *active; |
114 | |
115 | cur = READ_ONCE(execlists->active); |
116 | smp_rmb(); /* pairs with overwrite protection in process_csb() */ |
117 | do { |
118 | old = cur; |
119 | |
120 | active = READ_ONCE(*cur); |
121 | cur = READ_ONCE(execlists->active); |
122 | |
123 | smp_rmb(); /* and complete the seqlock retry */ |
124 | } while (unlikely(cur != old)); |
125 | |
126 | return active; |
127 | } |
128 | |
129 | struct i915_request * |
130 | execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists); |
131 | |
132 | static inline u32 |
133 | intel_read_status_page(const struct intel_engine_cs *engine, int reg) |
134 | { |
135 | /* Ensure that the compiler doesn't optimize away the load. */ |
136 | return READ_ONCE(engine->status_page.addr[reg]); |
137 | } |
138 | |
139 | static inline void |
140 | intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value) |
141 | { |
142 | /* Writing into the status page should be done sparingly. Since |
143 | * we do when we are uncertain of the device state, we take a bit |
144 | * of extra paranoia to try and ensure that the HWS takes the value |
145 | * we give and that it doesn't end up trapped inside the CPU! |
146 | */ |
147 | drm_clflush_virt_range(addr: &engine->status_page.addr[reg], length: sizeof(value)); |
148 | WRITE_ONCE(engine->status_page.addr[reg], value); |
149 | drm_clflush_virt_range(addr: &engine->status_page.addr[reg], length: sizeof(value)); |
150 | } |
151 | |
152 | /* |
153 | * Reads a dword out of the status page, which is written to from the command |
154 | * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or |
155 | * MI_STORE_DATA_IMM. |
156 | * |
157 | * The following dwords have a reserved meaning: |
158 | * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes. |
159 | * 0x04: ring 0 head pointer |
160 | * 0x05: ring 1 head pointer (915-class) |
161 | * 0x06: ring 2 head pointer (915-class) |
162 | * 0x10-0x1b: Context status DWords (GM45) |
163 | * 0x1f: Last written status offset. (GM45) |
164 | * 0x20-0x2f: Reserved (Gen6+) |
165 | * |
166 | * The area from dword 0x30 to 0x3ff is available for driver usage. |
167 | */ |
168 | #define I915_GEM_HWS_PREEMPT 0x32 |
169 | #define I915_GEM_HWS_PREEMPT_ADDR (I915_GEM_HWS_PREEMPT * sizeof(u32)) |
170 | #define I915_GEM_HWS_SEQNO 0x40 |
171 | #define I915_GEM_HWS_SEQNO_ADDR (I915_GEM_HWS_SEQNO * sizeof(u32)) |
172 | #define I915_GEM_HWS_MIGRATE (0x42 * sizeof(u32)) |
173 | #define I915_GEM_HWS_GGTT_BIND 0x46 |
174 | #define I915_GEM_HWS_GGTT_BIND_ADDR (I915_GEM_HWS_GGTT_BIND * sizeof(u32)) |
175 | #define I915_GEM_HWS_PXP 0x60 |
176 | #define I915_GEM_HWS_PXP_ADDR (I915_GEM_HWS_PXP * sizeof(u32)) |
177 | #define I915_GEM_HWS_GSC 0x62 |
178 | #define I915_GEM_HWS_GSC_ADDR (I915_GEM_HWS_GSC * sizeof(u32)) |
179 | #define I915_GEM_HWS_SCRATCH 0x80 |
180 | |
181 | #define I915_HWS_CSB_BUF0_INDEX 0x10 |
182 | #define I915_HWS_CSB_WRITE_INDEX 0x1f |
183 | #define ICL_HWS_CSB_WRITE_INDEX 0x2f |
184 | #define INTEL_HWS_CSB_WRITE_INDEX(__i915) \ |
185 | (GRAPHICS_VER(__i915) >= 11 ? ICL_HWS_CSB_WRITE_INDEX : I915_HWS_CSB_WRITE_INDEX) |
186 | |
187 | void intel_engine_stop(struct intel_engine_cs *engine); |
188 | void intel_engine_cleanup(struct intel_engine_cs *engine); |
189 | |
190 | int intel_engines_init_mmio(struct intel_gt *gt); |
191 | int intel_engines_init(struct intel_gt *gt); |
192 | |
193 | void intel_engine_free_request_pool(struct intel_engine_cs *engine); |
194 | |
195 | void intel_engines_release(struct intel_gt *gt); |
196 | void intel_engines_free(struct intel_gt *gt); |
197 | |
198 | int intel_engine_init_common(struct intel_engine_cs *engine); |
199 | void intel_engine_cleanup_common(struct intel_engine_cs *engine); |
200 | |
201 | int intel_engine_resume(struct intel_engine_cs *engine); |
202 | |
203 | int intel_ring_submission_setup(struct intel_engine_cs *engine); |
204 | |
205 | int intel_engine_stop_cs(struct intel_engine_cs *engine); |
206 | void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine); |
207 | |
208 | void intel_engine_wait_for_pending_mi_fw(struct intel_engine_cs *engine); |
209 | |
210 | void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask); |
211 | |
212 | u64 intel_engine_get_active_head(const struct intel_engine_cs *engine); |
213 | u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine); |
214 | |
215 | void intel_engine_get_instdone(const struct intel_engine_cs *engine, |
216 | struct intel_instdone *instdone); |
217 | |
218 | void intel_engine_init_execlists(struct intel_engine_cs *engine); |
219 | |
220 | bool intel_engine_irq_enable(struct intel_engine_cs *engine); |
221 | void intel_engine_irq_disable(struct intel_engine_cs *engine); |
222 | |
223 | static inline void __intel_engine_reset(struct intel_engine_cs *engine, |
224 | bool stalled) |
225 | { |
226 | if (engine->reset.rewind) |
227 | engine->reset.rewind(engine, stalled); |
228 | engine->serial++; /* contexts lost */ |
229 | } |
230 | |
231 | bool intel_engines_are_idle(struct intel_gt *gt); |
232 | bool intel_engine_is_idle(struct intel_engine_cs *engine); |
233 | |
234 | void __intel_engine_flush_submission(struct intel_engine_cs *engine, bool sync); |
235 | static inline void intel_engine_flush_submission(struct intel_engine_cs *engine) |
236 | { |
237 | __intel_engine_flush_submission(engine, sync: true); |
238 | } |
239 | |
240 | void intel_engines_reset_default_submission(struct intel_gt *gt); |
241 | |
242 | bool intel_engine_can_store_dword(struct intel_engine_cs *engine); |
243 | |
244 | __printf(3, 4) |
245 | void intel_engine_dump(struct intel_engine_cs *engine, |
246 | struct drm_printer *m, |
247 | const char *, ...); |
248 | void intel_engine_dump_active_requests(struct list_head *requests, |
249 | struct i915_request *hung_rq, |
250 | struct drm_printer *m); |
251 | |
252 | ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine, |
253 | ktime_t *now); |
254 | |
255 | void intel_engine_get_hung_entity(struct intel_engine_cs *engine, |
256 | struct intel_context **ce, struct i915_request **rq); |
257 | |
258 | u32 intel_engine_context_size(struct intel_gt *gt, u8 class); |
259 | struct intel_context * |
260 | intel_engine_create_pinned_context(struct intel_engine_cs *engine, |
261 | struct i915_address_space *vm, |
262 | unsigned int ring_size, |
263 | unsigned int hwsp, |
264 | struct lock_class_key *key, |
265 | const char *name); |
266 | |
267 | void intel_engine_destroy_pinned_context(struct intel_context *ce); |
268 | |
269 | void xehp_enable_ccs_engines(struct intel_engine_cs *engine); |
270 | |
271 | #define ENGINE_PHYSICAL 0 |
272 | #define ENGINE_MOCK 1 |
273 | #define ENGINE_VIRTUAL 2 |
274 | |
275 | static inline bool intel_engine_uses_guc(const struct intel_engine_cs *engine) |
276 | { |
277 | return engine->gt->submission_method >= INTEL_SUBMISSION_GUC; |
278 | } |
279 | |
280 | static inline bool |
281 | intel_engine_has_preempt_reset(const struct intel_engine_cs *engine) |
282 | { |
283 | if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT) |
284 | return false; |
285 | |
286 | return intel_engine_has_preemption(engine); |
287 | } |
288 | |
289 | #define FORCE_VIRTUAL BIT(0) |
290 | struct intel_context * |
291 | intel_engine_create_virtual(struct intel_engine_cs **siblings, |
292 | unsigned int count, unsigned long flags); |
293 | |
294 | static inline struct intel_context * |
295 | intel_engine_create_parallel(struct intel_engine_cs **engines, |
296 | unsigned int num_engines, |
297 | unsigned int width) |
298 | { |
299 | GEM_BUG_ON(!engines[0]->cops->create_parallel); |
300 | return engines[0]->cops->create_parallel(engines, num_engines, width); |
301 | } |
302 | |
303 | static inline bool |
304 | intel_virtual_engine_has_heartbeat(const struct intel_engine_cs *engine) |
305 | { |
306 | /* |
307 | * For non-GuC submission we expect the back-end to look at the |
308 | * heartbeat status of the actual physical engine that the work |
309 | * has been (or is being) scheduled on, so we should only reach |
310 | * here with GuC submission enabled. |
311 | */ |
312 | GEM_BUG_ON(!intel_engine_uses_guc(engine)); |
313 | |
314 | return intel_guc_virtual_engine_has_heartbeat(ve: engine); |
315 | } |
316 | |
317 | static inline bool |
318 | intel_engine_has_heartbeat(const struct intel_engine_cs *engine) |
319 | { |
320 | if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL) |
321 | return false; |
322 | |
323 | if (intel_engine_is_virtual(engine)) |
324 | return intel_virtual_engine_has_heartbeat(engine); |
325 | else |
326 | return READ_ONCE(engine->props.heartbeat_interval_ms); |
327 | } |
328 | |
329 | static inline struct intel_engine_cs * |
330 | intel_engine_get_sibling(struct intel_engine_cs *engine, unsigned int sibling) |
331 | { |
332 | GEM_BUG_ON(!intel_engine_is_virtual(engine)); |
333 | return engine->cops->get_sibling(engine, sibling); |
334 | } |
335 | |
336 | static inline void |
337 | intel_engine_set_hung_context(struct intel_engine_cs *engine, |
338 | struct intel_context *ce) |
339 | { |
340 | engine->hung_ce = ce; |
341 | } |
342 | |
343 | static inline void |
344 | intel_engine_clear_hung_context(struct intel_engine_cs *engine) |
345 | { |
346 | intel_engine_set_hung_context(engine, NULL); |
347 | } |
348 | |
349 | static inline struct intel_context * |
350 | intel_engine_get_hung_context(struct intel_engine_cs *engine) |
351 | { |
352 | return engine->hung_ce; |
353 | } |
354 | |
355 | u64 intel_clamp_heartbeat_interval_ms(struct intel_engine_cs *engine, u64 value); |
356 | u64 intel_clamp_max_busywait_duration_ns(struct intel_engine_cs *engine, u64 value); |
357 | u64 intel_clamp_preempt_timeout_ms(struct intel_engine_cs *engine, u64 value); |
358 | u64 intel_clamp_stop_timeout_ms(struct intel_engine_cs *engine, u64 value); |
359 | u64 intel_clamp_timeslice_duration_ms(struct intel_engine_cs *engine, u64 value); |
360 | |
361 | #endif /* _INTEL_RINGBUFFER_H_ */ |
362 | |