1	/*
2	* Copyright © 2013 Intel Corporation
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice (including the next
12	* paragraph) shall be included in all copies or substantial portions of the
13	* Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21	* IN THE SOFTWARE.
22	*/
23
24	#include <drm/drm_managed.h>
25	#include <linux/pm_runtime.h>
26
27	#include "gt/intel_engine_regs.h"
28	#include "gt/intel_gt_regs.h"
29
30	#include "i915_drv.h"
31	#include "i915_iosf_mbi.h"
32	#include "i915_reg.h"
33	#include "i915_trace.h"
34	#include "i915_vgpu.h"
35
36	#define FORCEWAKE_ACK_TIMEOUT_MS 50
37	#define GT_FIFO_TIMEOUT_MS 10
38
39	#define __raw_posting_read(...) ((void)__raw_uncore_read32(__VA_ARGS__))
40
41	static void
42	fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
43	{
44	uncore->fw_get_funcs->force_wake_get(uncore, fw_domains);
45	}
46
47	void
48	intel_uncore_mmio_debug_init_early(struct drm_i915_private *i915)
49	{
50	spin_lock_init(&i915->mmio_debug.lock);
51	i915->mmio_debug.unclaimed_mmio_check = 1;
52
53	i915->uncore.debug = &i915->mmio_debug;
54	}
55
56	static void mmio_debug_suspend(struct intel_uncore *uncore)
57	{
58	if (!uncore->debug)
59	return;
60
61	spin_lock(lock: &uncore->debug->lock);
62
63	/* Save and disable mmio debugging for the user bypass */
64	if (!uncore->debug->suspend_count++) {
65	uncore->debug->saved_mmio_check = uncore->debug->unclaimed_mmio_check;
66	uncore->debug->unclaimed_mmio_check = 0;
67	}
68
69	spin_unlock(lock: &uncore->debug->lock);
70	}
71
72	static bool check_for_unclaimed_mmio(struct intel_uncore *uncore);
73
74	static void mmio_debug_resume(struct intel_uncore *uncore)
75	{
76	if (!uncore->debug)
77	return;
78
79	spin_lock(lock: &uncore->debug->lock);
80
81	if (!--uncore->debug->suspend_count)
82	uncore->debug->unclaimed_mmio_check = uncore->debug->saved_mmio_check;
83
84	if (check_for_unclaimed_mmio(uncore))
85	drm_info(&uncore->i915->drm,
86	"Invalid mmio detected during user access\n");
87
88	spin_unlock(lock: &uncore->debug->lock);
89	}
90
91	static const char * const forcewake_domain_names[] = {
92	"render",
93	"gt",
94	"media",
95	"vdbox0",
96	"vdbox1",
97	"vdbox2",
98	"vdbox3",
99	"vdbox4",
100	"vdbox5",
101	"vdbox6",
102	"vdbox7",
103	"vebox0",
104	"vebox1",
105	"vebox2",
106	"vebox3",
107	"gsc",
108	};
109
110	const char *
111	intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
112	{
113	BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
114
115	if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
116	return forcewake_domain_names[id];
117
118	WARN_ON(id);
119
120	return "unknown";
121	}
122
123	#define fw_ack(d) readl((d)->reg_ack)
124	#define fw_set(d, val) writel(_MASKED_BIT_ENABLE((val)), (d)->reg_set)
125	#define fw_clear(d, val) writel(_MASKED_BIT_DISABLE((val)), (d)->reg_set)
126
127	static inline void
128	fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
129	{
130	/*
131	* We don't really know if the powerwell for the forcewake domain we are
132	* trying to reset here does exist at this point (engines could be fused
133	* off in ICL+), so no waiting for acks
134	*/
135	/* WaRsClearFWBitsAtReset */
136	if (GRAPHICS_VER(d->uncore->i915) >= 12)
137	fw_clear(d, 0xefff);
138	else
139	fw_clear(d, 0xffff);
140	}
141
142	static inline void
143	fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
144	{
145	GEM_BUG_ON(d->uncore->fw_domains_timer & d->mask);
146	d->uncore->fw_domains_timer |= d->mask;
147	d->wake_count++;
148	hrtimer_start_range_ns(timer: &d->timer,
149	NSEC_PER_MSEC,
150	NSEC_PER_MSEC,
151	mode: HRTIMER_MODE_REL);
152	}
153
154	static inline int
155	__wait_for_ack(const struct intel_uncore_forcewake_domain *d,
156	const u32 ack,
157	const u32 value)
158	{
159	return wait_for_atomic((fw_ack(d) & ack) == value,
160	FORCEWAKE_ACK_TIMEOUT_MS);
161	}
162
163	static inline int
164	wait_ack_clear(const struct intel_uncore_forcewake_domain *d,
165	const u32 ack)
166	{
167	return __wait_for_ack(d, ack, value: 0);
168	}
169
170	static inline int
171	wait_ack_set(const struct intel_uncore_forcewake_domain *d,
172	const u32 ack)
173	{
174	return __wait_for_ack(d, ack, value: ack);
175	}
176
177	static inline void
178	fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d)
179	{
180	if (!wait_ack_clear(d, FORCEWAKE_KERNEL))
181	return;
182
183	if (fw_ack(d) == ~0)
184	drm_err(&d->uncore->i915->drm,
185	"%s: MMIO unreliable (forcewake register returns 0xFFFFFFFF)!\n",
186	intel_uncore_forcewake_domain_to_str(d->id));
187	else
188	drm_err(&d->uncore->i915->drm,
189	"%s: timed out waiting for forcewake ack to clear.\n",
190	intel_uncore_forcewake_domain_to_str(d->id));
191
192	add_taint_for_CI(i915: d->uncore->i915, TAINT_WARN); /* CI now unreliable */
193	}
194
195	enum ack_type {
196	ACK_CLEAR = 0,
197	ACK_SET
198	};
199
200	static int
201	fw_domain_wait_ack_with_fallback(const struct intel_uncore_forcewake_domain *d,
202	const enum ack_type type)
203	{
204	const u32 ack_bit = FORCEWAKE_KERNEL;
205	const u32 value = type == ACK_SET ? ack_bit : 0;
206	unsigned int pass;
207	bool ack_detected;
208
209	/*
210	* There is a possibility of driver's wake request colliding
211	* with hardware's own wake requests and that can cause
212	* hardware to not deliver the driver's ack message.
213	*
214	* Use a fallback bit toggle to kick the gpu state machine
215	* in the hope that the original ack will be delivered along with
216	* the fallback ack.
217	*
218	* This workaround is described in HSDES #1604254524 and it's known as:
219	* WaRsForcewakeAddDelayForAck:skl,bxt,kbl,glk,cfl,cnl,icl
220	* although the name is a bit misleading.
221	*/
222
223	pass = 1;
224	do {
225	wait_ack_clear(d, FORCEWAKE_KERNEL_FALLBACK);
226
227	fw_set(d, FORCEWAKE_KERNEL_FALLBACK);
228	/* Give gt some time to relax before the polling frenzy */
229	udelay(10 * pass);
230	wait_ack_set(d, FORCEWAKE_KERNEL_FALLBACK);
231
232	ack_detected = (fw_ack(d) & ack_bit) == value;
233
234	fw_clear(d, FORCEWAKE_KERNEL_FALLBACK);
235	} while (!ack_detected && pass++ < 10);
236
237	drm_dbg(&d->uncore->i915->drm,
238	"%s had to use fallback to %s ack, 0x%x (passes %u)\n",
239	intel_uncore_forcewake_domain_to_str(d->id),
240	type == ACK_SET ? "set" : "clear",
241	fw_ack(d),
242	pass);
243
244	return ack_detected ? 0 : -ETIMEDOUT;
245	}
246
247	static inline void
248	fw_domain_wait_ack_clear_fallback(const struct intel_uncore_forcewake_domain *d)
249	{
250	if (likely(!wait_ack_clear(d, FORCEWAKE_KERNEL)))
251	return;
252
253	if (fw_domain_wait_ack_with_fallback(d, type: ACK_CLEAR))
254	fw_domain_wait_ack_clear(d);
255	}
256
257	static inline void
258	fw_domain_get(const struct intel_uncore_forcewake_domain *d)
259	{
260	fw_set(d, FORCEWAKE_KERNEL);
261	}
262
263	static inline void
264	fw_domain_wait_ack_set(const struct intel_uncore_forcewake_domain *d)
265	{
266	if (wait_ack_set(d, FORCEWAKE_KERNEL)) {
267	drm_err(&d->uncore->i915->drm,
268	"%s: timed out waiting for forcewake ack request.\n",
269	intel_uncore_forcewake_domain_to_str(d->id));
270	add_taint_for_CI(i915: d->uncore->i915, TAINT_WARN); /* CI now unreliable */
271	}
272	}
273
274	static inline void
275	fw_domain_wait_ack_set_fallback(const struct intel_uncore_forcewake_domain *d)
276	{
277	if (likely(!wait_ack_set(d, FORCEWAKE_KERNEL)))
278	return;
279
280	if (fw_domain_wait_ack_with_fallback(d, type: ACK_SET))
281	fw_domain_wait_ack_set(d);
282	}
283
284	static inline void
285	fw_domain_put(const struct intel_uncore_forcewake_domain *d)
286	{
287	fw_clear(d, FORCEWAKE_KERNEL);
288	}
289
290	static void
291	fw_domains_get_normal(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
292	{
293	struct intel_uncore_forcewake_domain *d;
294	unsigned int tmp;
295
296	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
297
298	for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
299	fw_domain_wait_ack_clear(d);
300	fw_domain_get(d);
301	}
302
303	for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
304	fw_domain_wait_ack_set(d);
305
306	uncore->fw_domains_active |= fw_domains;
307	}
308
309	static void
310	fw_domains_get_with_fallback(struct intel_uncore *uncore,
311	enum forcewake_domains fw_domains)
312	{
313	struct intel_uncore_forcewake_domain *d;
314	unsigned int tmp;
315
316	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
317
318	for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
319	fw_domain_wait_ack_clear_fallback(d);
320	fw_domain_get(d);
321	}
322
323	for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
324	fw_domain_wait_ack_set_fallback(d);
325
326	uncore->fw_domains_active |= fw_domains;
327	}
328
329	static void
330	fw_domains_put(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
331	{
332	struct intel_uncore_forcewake_domain *d;
333	unsigned int tmp;
334
335	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
336
337	for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
338	fw_domain_put(d);
339
340	uncore->fw_domains_active &= ~fw_domains;
341	}
342
343	static void
344	fw_domains_reset(struct intel_uncore *uncore,
345	enum forcewake_domains fw_domains)
346	{
347	struct intel_uncore_forcewake_domain *d;
348	unsigned int tmp;
349
350	if (!fw_domains)
351	return;
352
353	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
354
355	for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
356	fw_domain_reset(d);
357	}
358
359	static inline u32 gt_thread_status(struct intel_uncore *uncore)
360	{
361	u32 val;
362
363	val = __raw_uncore_read32(uncore, GEN6_GT_THREAD_STATUS_REG);
364	val &= GEN6_GT_THREAD_STATUS_CORE_MASK;
365
366	return val;
367	}
368
369	static void __gen6_gt_wait_for_thread_c0(struct intel_uncore *uncore)
370	{
371	/*
372	* w/a for a sporadic read returning 0 by waiting for the GT
373	* thread to wake up.
374	*/
375	drm_WARN_ONCE(&uncore->i915->drm,
376	wait_for_atomic_us(gt_thread_status(uncore) == 0, 5000),
377	"GT thread status wait timed out\n");
378	}
379
380	static void fw_domains_get_with_thread_status(struct intel_uncore *uncore,
381	enum forcewake_domains fw_domains)
382	{
383	fw_domains_get_normal(uncore, fw_domains);
384
385	/* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
386	__gen6_gt_wait_for_thread_c0(uncore);
387	}
388
389	static inline u32 fifo_free_entries(struct intel_uncore *uncore)
390	{
391	u32 count = __raw_uncore_read32(uncore, GTFIFOCTL);
392
393	return count & GT_FIFO_FREE_ENTRIES_MASK;
394	}
395
396	static void __gen6_gt_wait_for_fifo(struct intel_uncore *uncore)
397	{
398	u32 n;
399
400	/* On VLV, FIFO will be shared by both SW and HW.
401	* So, we need to read the FREE_ENTRIES everytime */
402	if (IS_VALLEYVIEW(uncore->i915))
403	n = fifo_free_entries(uncore);
404	else
405	n = uncore->fifo_count;
406
407	if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) {
408	if (wait_for_atomic((n = fifo_free_entries(uncore)) >
409	GT_FIFO_NUM_RESERVED_ENTRIES,
410	GT_FIFO_TIMEOUT_MS)) {
411	drm_dbg(&uncore->i915->drm,
412	"GT_FIFO timeout, entries: %u\n", n);
413	return;
414	}
415	}
416
417	uncore->fifo_count = n - 1;
418	}
419
420	static enum hrtimer_restart
421	intel_uncore_fw_release_timer(struct hrtimer *timer)
422	{
423	struct intel_uncore_forcewake_domain *domain =
424	container_of(timer, struct intel_uncore_forcewake_domain, timer);
425	struct intel_uncore *uncore = domain->uncore;
426	unsigned long irqflags;
427
428	assert_rpm_device_not_suspended(rpm: uncore->rpm);
429
430	if (xchg(&domain->active, false))
431	return HRTIMER_RESTART;
432
433	spin_lock_irqsave(&uncore->lock, irqflags);
434
435	uncore->fw_domains_timer &= ~domain->mask;
436
437	GEM_BUG_ON(!domain->wake_count);
438	if (--domain->wake_count == 0)
439	fw_domains_put(uncore, fw_domains: domain->mask);
440
441	spin_unlock_irqrestore(lock: &uncore->lock, flags: irqflags);
442
443	return HRTIMER_NORESTART;
444	}
445
446	/* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */
447	static unsigned int
448	intel_uncore_forcewake_reset(struct intel_uncore *uncore)
449	{
450	unsigned long irqflags;
451	struct intel_uncore_forcewake_domain *domain;
452	int retry_count = 100;
453	enum forcewake_domains fw, active_domains;
454
455	iosf_mbi_assert_punit_acquired();
456
457	/* Hold uncore.lock across reset to prevent any register access
458	* with forcewake not set correctly. Wait until all pending
459	* timers are run before holding.
460	*/
461	while (1) {
462	unsigned int tmp;
463
464	active_domains = 0;
465
466	for_each_fw_domain(domain, uncore, tmp) {
467	smp_store_mb(domain->active, false);
468	if (hrtimer_cancel(timer: &domain->timer) == 0)
469	continue;
470
471	intel_uncore_fw_release_timer(timer: &domain->timer);
472	}
473
474	spin_lock_irqsave(&uncore->lock, irqflags);
475
476	for_each_fw_domain(domain, uncore, tmp) {
477	if (hrtimer_active(timer: &domain->timer))
478	active_domains |= domain->mask;
479	}
480
481	if (active_domains == 0)
482	break;
483
484	if (--retry_count == 0) {
485	drm_err(&uncore->i915->drm, "Timed out waiting for forcewake timers to finish\n");
486	break;
487	}
488
489	spin_unlock_irqrestore(lock: &uncore->lock, flags: irqflags);
490	cond_resched();
491	}
492
493	drm_WARN_ON(&uncore->i915->drm, active_domains);
494
495	fw = uncore->fw_domains_active;
496	if (fw)
497	fw_domains_put(uncore, fw_domains: fw);
498
499	fw_domains_reset(uncore, fw_domains: uncore->fw_domains);
500	assert_forcewakes_inactive(uncore);
501
502	spin_unlock_irqrestore(lock: &uncore->lock, flags: irqflags);
503
504	return fw; /* track the lost user forcewake domains */
505	}
506
507	static bool
508	fpga_check_for_unclaimed_mmio(struct intel_uncore *uncore)
509	{
510	u32 dbg;
511
512	dbg = __raw_uncore_read32(uncore, FPGA_DBG);
513	if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
514	return false;
515
516	/*
517	* Bugs in PCI programming (or failing hardware) can occasionally cause
518	* us to lose access to the MMIO BAR. When this happens, register
519	* reads will come back with 0xFFFFFFFF for every register and things
520	* go bad very quickly. Let's try to detect that special case and at
521	* least try to print a more informative message about what has
522	* happened.
523	*
524	* During normal operation the FPGA_DBG register has several unused
525	* bits that will always read back as 0's so we can use them as canaries
526	* to recognize when MMIO accesses are just busted.
527	*/
528	if (unlikely(dbg == ~0))
529	drm_err(&uncore->i915->drm,
530	"Lost access to MMIO BAR; all registers now read back as 0xFFFFFFFF!\n");
531
532	__raw_uncore_write32(uncore, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
533
534	return true;
535	}
536
537	static bool
538	vlv_check_for_unclaimed_mmio(struct intel_uncore *uncore)
539	{
540	u32 cer;
541
542	cer = __raw_uncore_read32(uncore, CLAIM_ER);
543	if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
544	return false;
545
546	__raw_uncore_write32(uncore, CLAIM_ER, CLAIM_ER_CLR);
547
548	return true;
549	}
550
551	static bool
552	gen6_check_for_fifo_debug(struct intel_uncore *uncore)
553	{
554	u32 fifodbg;
555
556	fifodbg = __raw_uncore_read32(uncore, GTFIFODBG);
557
558	if (unlikely(fifodbg)) {
559	drm_dbg(&uncore->i915->drm, "GTFIFODBG = 0x08%x\n", fifodbg);
560	__raw_uncore_write32(uncore, GTFIFODBG, val: fifodbg);
561	}
562
563	return fifodbg;
564	}
565
566	static bool
567	check_for_unclaimed_mmio(struct intel_uncore *uncore)
568	{
569	bool ret = false;
570
571	lockdep_assert_held(&uncore->debug->lock);
572
573	if (uncore->debug->suspend_count)
574	return false;
575
576	if (intel_uncore_has_fpga_dbg_unclaimed(uncore))
577	ret |= fpga_check_for_unclaimed_mmio(uncore);
578
579	if (intel_uncore_has_dbg_unclaimed(uncore))
580	ret |= vlv_check_for_unclaimed_mmio(uncore);
581
582	if (intel_uncore_has_fifo(uncore))
583	ret |= gen6_check_for_fifo_debug(uncore);
584
585	return ret;
586	}
587
588	static void forcewake_early_sanitize(struct intel_uncore *uncore,
589	unsigned int restore_forcewake)
590	{
591	GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
592
593	/* WaDisableShadowRegForCpd:chv */
594	if (IS_CHERRYVIEW(uncore->i915)) {
595	__raw_uncore_write32(uncore, GTFIFOCTL,
596	val: __raw_uncore_read32(uncore, GTFIFOCTL) |
597	GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
598	GT_FIFO_CTL_RC6_POLICY_STALL);
599	}
600
601	iosf_mbi_punit_acquire();
602	intel_uncore_forcewake_reset(uncore);
603	if (restore_forcewake) {
604	spin_lock_irq(lock: &uncore->lock);
605	fw_domains_get(uncore, fw_domains: restore_forcewake);
606
607	if (intel_uncore_has_fifo(uncore))
608	uncore->fifo_count = fifo_free_entries(uncore);
609	spin_unlock_irq(lock: &uncore->lock);
610	}
611	iosf_mbi_punit_release();
612	}
613
614	void intel_uncore_suspend(struct intel_uncore *uncore)
615	{
616	if (!intel_uncore_has_forcewake(uncore))
617	return;
618
619	iosf_mbi_punit_acquire();
620	iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
621	nb: &uncore->pmic_bus_access_nb);
622	uncore->fw_domains_saved = intel_uncore_forcewake_reset(uncore);
623	iosf_mbi_punit_release();
624	}
625
626	void intel_uncore_resume_early(struct intel_uncore *uncore)
627	{
628	unsigned int restore_forcewake;
629
630	if (intel_uncore_unclaimed_mmio(uncore))
631	drm_dbg(&uncore->i915->drm, "unclaimed mmio detected on resume, clearing\n");
632
633	if (!intel_uncore_has_forcewake(uncore))
634	return;
635
636	restore_forcewake = fetch_and_zero(&uncore->fw_domains_saved);
637	forcewake_early_sanitize(uncore, restore_forcewake);
638
639	iosf_mbi_register_pmic_bus_access_notifier(nb: &uncore->pmic_bus_access_nb);
640	}
641
642	void intel_uncore_runtime_resume(struct intel_uncore *uncore)
643	{
644	if (!intel_uncore_has_forcewake(uncore))
645	return;
646
647	iosf_mbi_register_pmic_bus_access_notifier(nb: &uncore->pmic_bus_access_nb);
648	}
649
650	static void __intel_uncore_forcewake_get(struct intel_uncore *uncore,
651	enum forcewake_domains fw_domains)
652	{
653	struct intel_uncore_forcewake_domain *domain;
654	unsigned int tmp;
655
656	fw_domains &= uncore->fw_domains;
657
658	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
659	if (domain->wake_count++) {
660	fw_domains &= ~domain->mask;
661	domain->active = true;
662	}
663	}
664
665	if (fw_domains)
666	fw_domains_get(uncore, fw_domains);
667	}
668
669	/**
670	* intel_uncore_forcewake_get - grab forcewake domain references
671	* @uncore: the intel_uncore structure
672	* @fw_domains: forcewake domains to get reference on
673	*
674	* This function can be used get GT's forcewake domain references.
675	* Normal register access will handle the forcewake domains automatically.
676	* However if some sequence requires the GT to not power down a particular
677	* forcewake domains this function should be called at the beginning of the
678	* sequence. And subsequently the reference should be dropped by symmetric
679	* call to intel_unforce_forcewake_put(). Usually caller wants all the domains
680	* to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
681	*/
682	void intel_uncore_forcewake_get(struct intel_uncore *uncore,
683	enum forcewake_domains fw_domains)
684	{
685	unsigned long irqflags;
686
687	if (!uncore->fw_get_funcs)
688	return;
689
690	assert_rpm_wakelock_held(rpm: uncore->rpm);
691
692	spin_lock_irqsave(&uncore->lock, irqflags);
693	__intel_uncore_forcewake_get(uncore, fw_domains);
694	spin_unlock_irqrestore(lock: &uncore->lock, flags: irqflags);
695	}
696
697	/**
698	* intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace
699	* @uncore: the intel_uncore structure
700	*
701	* This function is a wrapper around intel_uncore_forcewake_get() to acquire
702	* the GT powerwell and in the process disable our debugging for the
703	* duration of userspace's bypass.
704	*/
705	void intel_uncore_forcewake_user_get(struct intel_uncore *uncore)
706	{
707	spin_lock_irq(lock: &uncore->lock);
708	if (!uncore->user_forcewake_count++) {
709	intel_uncore_forcewake_get__locked(uncore, domains: FORCEWAKE_ALL);
710	mmio_debug_suspend(uncore);
711	}
712	spin_unlock_irq(lock: &uncore->lock);
713	}
714
715	/**
716	* intel_uncore_forcewake_user_put - release forcewake on behalf of userspace
717	* @uncore: the intel_uncore structure
718	*
719	* This function complements intel_uncore_forcewake_user_get() and releases
720	* the GT powerwell taken on behalf of the userspace bypass.
721	*/
722	void intel_uncore_forcewake_user_put(struct intel_uncore *uncore)
723	{
724	spin_lock_irq(lock: &uncore->lock);
725	if (!--uncore->user_forcewake_count) {
726	mmio_debug_resume(uncore);
727	intel_uncore_forcewake_put__locked(uncore, domains: FORCEWAKE_ALL);
728	}
729	spin_unlock_irq(lock: &uncore->lock);
730	}
731
732	/**
733	* intel_uncore_forcewake_get__locked - grab forcewake domain references
734	* @uncore: the intel_uncore structure
735	* @fw_domains: forcewake domains to get reference on
736	*
737	* See intel_uncore_forcewake_get(). This variant places the onus
738	* on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
739	*/
740	void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore,
741	enum forcewake_domains fw_domains)
742	{
743	lockdep_assert_held(&uncore->lock);
744
745	if (!uncore->fw_get_funcs)
746	return;
747
748	__intel_uncore_forcewake_get(uncore, fw_domains);
749	}
750
751	static void __intel_uncore_forcewake_put(struct intel_uncore *uncore,
752	enum forcewake_domains fw_domains,
753	bool delayed)
754	{
755	struct intel_uncore_forcewake_domain *domain;
756	unsigned int tmp;
757
758	fw_domains &= uncore->fw_domains;
759
760	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
761	GEM_BUG_ON(!domain->wake_count);
762
763	if (--domain->wake_count) {
764	domain->active = true;
765	continue;
766	}
767
768	if (delayed &&
769	!(domain->uncore->fw_domains_timer & domain->mask))
770	fw_domain_arm_timer(d: domain);
771	else
772	fw_domains_put(uncore, fw_domains: domain->mask);
773	}
774	}
775
776	/**
777	* intel_uncore_forcewake_put - release a forcewake domain reference
778	* @uncore: the intel_uncore structure
779	* @fw_domains: forcewake domains to put references
780	*
781	* This function drops the device-level forcewakes for specified
782	* domains obtained by intel_uncore_forcewake_get().
783	*/
784	void intel_uncore_forcewake_put(struct intel_uncore *uncore,
785	enum forcewake_domains fw_domains)
786	{
787	unsigned long irqflags;
788
789	if (!uncore->fw_get_funcs)
790	return;
791
792	spin_lock_irqsave(&uncore->lock, irqflags);
793	__intel_uncore_forcewake_put(uncore, fw_domains, delayed: false);
794	spin_unlock_irqrestore(lock: &uncore->lock, flags: irqflags);
795	}
796
797	void intel_uncore_forcewake_put_delayed(struct intel_uncore *uncore,
798	enum forcewake_domains fw_domains)
799	{
800	unsigned long irqflags;
801
802	if (!uncore->fw_get_funcs)
803	return;
804
805	spin_lock_irqsave(&uncore->lock, irqflags);
806	__intel_uncore_forcewake_put(uncore, fw_domains, delayed: true);
807	spin_unlock_irqrestore(lock: &uncore->lock, flags: irqflags);
808	}
809
810	/**
811	* intel_uncore_forcewake_flush - flush the delayed release
812	* @uncore: the intel_uncore structure
813	* @fw_domains: forcewake domains to flush
814	*/
815	void intel_uncore_forcewake_flush(struct intel_uncore *uncore,
816	enum forcewake_domains fw_domains)
817	{
818	struct intel_uncore_forcewake_domain *domain;
819	unsigned int tmp;
820
821	if (!uncore->fw_get_funcs)
822	return;
823
824	fw_domains &= uncore->fw_domains;
825	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
826	WRITE_ONCE(domain->active, false);
827	if (hrtimer_cancel(timer: &domain->timer))
828	intel_uncore_fw_release_timer(timer: &domain->timer);
829	}
830	}
831
832	/**
833	* intel_uncore_forcewake_put__locked - release forcewake domain references
834	* @uncore: the intel_uncore structure
835	* @fw_domains: forcewake domains to put references
836	*
837	* See intel_uncore_forcewake_put(). This variant places the onus
838	* on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
839	*/
840	void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore,
841	enum forcewake_domains fw_domains)
842	{
843	lockdep_assert_held(&uncore->lock);
844
845	if (!uncore->fw_get_funcs)
846	return;
847
848	__intel_uncore_forcewake_put(uncore, fw_domains, delayed: false);
849	}
850
851	void assert_forcewakes_inactive(struct intel_uncore *uncore)
852	{
853	if (!uncore->fw_get_funcs)
854	return;
855
856	drm_WARN(&uncore->i915->drm, uncore->fw_domains_active,
857	"Expected all fw_domains to be inactive, but %08x are still on\n",
858	uncore->fw_domains_active);
859	}
860
861	void assert_forcewakes_active(struct intel_uncore *uncore,
862	enum forcewake_domains fw_domains)
863	{
864	struct intel_uncore_forcewake_domain *domain;
865	unsigned int tmp;
866
867	if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM))
868	return;
869
870	if (!uncore->fw_get_funcs)
871	return;
872
873	spin_lock_irq(lock: &uncore->lock);
874
875	assert_rpm_wakelock_held(rpm: uncore->rpm);
876
877	fw_domains &= uncore->fw_domains;
878	drm_WARN(&uncore->i915->drm, fw_domains & ~uncore->fw_domains_active,
879	"Expected %08x fw_domains to be active, but %08x are off\n",
880	fw_domains, fw_domains & ~uncore->fw_domains_active);
881
882	/*
883	* Check that the caller has an explicit wakeref and we don't mistake
884	* it for the auto wakeref.
885	*/
886	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
887	unsigned int actual = READ_ONCE(domain->wake_count);
888	unsigned int expect = 1;
889
890	if (uncore->fw_domains_timer & domain->mask)
891	expect++; /* pending automatic release */
892
893	if (drm_WARN(&uncore->i915->drm, actual < expect,
894	"Expected domain %d to be held awake by caller, count=%d\n",
895	domain->id, actual))
896	break;
897	}
898
899	spin_unlock_irq(lock: &uncore->lock);
900	}
901
902	/*
903	* We give fast paths for the really cool registers. The second range includes
904	* media domains (and the GSC starting from Xe_LPM+)
905	*/
906	#define NEEDS_FORCE_WAKE(reg) ({ \
907	u32 __reg = (reg); \
908	__reg < 0x40000 || __reg >= 0x116000; \
909	})
910
911	static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
912	{
913	if (offset < entry->start)
914	return -1;
915	else if (offset > entry->end)
916	return 1;
917	else
918	return 0;
919	}
920
921	/* Copied and "macroized" from lib/bsearch.c */
922	#define BSEARCH(key, base, num, cmp) ({ \
923	unsigned int start__ = 0, end__ = (num); \
924	typeof(base) result__ = NULL; \
925	while (start__ < end__) { \
926	unsigned int mid__ = start__ + (end__ - start__) / 2; \
927	int ret__ = (cmp)((key), (base) + mid__); \
928	if (ret__ < 0) { \
929	end__ = mid__; \
930	} else if (ret__ > 0) { \
931	start__ = mid__ + 1; \
932	} else { \
933	result__ = (base) + mid__; \
934	break; \
935	} \
936	} \
937	result__; \
938	})
939
940	static enum forcewake_domains
941	find_fw_domain(struct intel_uncore *uncore, u32 offset)
942	{
943	const struct intel_forcewake_range *entry;
944
945	if (IS_GSI_REG(offset))
946	offset += uncore->gsi_offset;
947
948	entry = BSEARCH(offset,
949	uncore->fw_domains_table,
950	uncore->fw_domains_table_entries,
951	fw_range_cmp);
952
953	if (!entry)
954	return 0;
955
956	/*
957	* The list of FW domains depends on the SKU in gen11+ so we
958	* can't determine it statically. We use FORCEWAKE_ALL and
959	* translate it here to the list of available domains.
960	*/
961	if (entry->domains == FORCEWAKE_ALL)
962	return uncore->fw_domains;
963
964	drm_WARN(&uncore->i915->drm, entry->domains & ~uncore->fw_domains,
965	"Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
966	entry->domains & ~uncore->fw_domains, offset);
967
968	return entry->domains;
969	}
970
971	/*
972	* Shadowed register tables describe special register ranges that i915 is
973	* allowed to write to without acquiring forcewake. If these registers' power
974	* wells are down, the hardware will save values written by i915 to a shadow
975	* copy and automatically transfer them into the real register the next time
976	* the power well is woken up. Shadowing only applies to writes; forcewake
977	* must still be acquired when reading from registers in these ranges.
978	*
979	* The documentation for shadowed registers is somewhat spotty on older
980	* platforms. However missing registers from these lists is non-fatal; it just
981	* means we'll wake up the hardware for some register accesses where we didn't
982	* really need to.
983	*
984	* The ranges listed in these tables must be sorted by offset.
985	*
986	* When adding new tables here, please also add them to
987	* intel_shadow_table_check() in selftests/intel_uncore.c so that they will be
988	* scanned for obvious mistakes or typos by the selftests.
989	*/
990
991	static const struct i915_range gen8_shadowed_regs[] = {
992	{ .start = 0x2030, .end = 0x2030 },
993	{ .start = 0xA008, .end = 0xA00C },
994	{ .start = 0x12030, .end = 0x12030 },
995	{ .start = 0x1a030, .end = 0x1a030 },
996	{ .start = 0x22030, .end = 0x22030 },
997	};
998
999	static const struct i915_range gen11_shadowed_regs[] = {
1000	{ .start = 0x2030, .end = 0x2030 },
1001	{ .start = 0x2550, .end = 0x2550 },
1002	{ .start = 0xA008, .end = 0xA00C },
1003	{ .start = 0x22030, .end = 0x22030 },
1004	{ .start = 0x22230, .end = 0x22230 },
1005	{ .start = 0x22510, .end = 0x22550 },
1006	{ .start = 0x1C0030, .end = 0x1C0030 },
1007	{ .start = 0x1C0230, .end = 0x1C0230 },
1008	{ .start = 0x1C0510, .end = 0x1C0550 },
1009	{ .start = 0x1C4030, .end = 0x1C4030 },
1010	{ .start = 0x1C4230, .end = 0x1C4230 },
1011	{ .start = 0x1C4510, .end = 0x1C4550 },
1012	{ .start = 0x1C8030, .end = 0x1C8030 },
1013	{ .start = 0x1C8230, .end = 0x1C8230 },
1014	{ .start = 0x1C8510, .end = 0x1C8550 },
1015	{ .start = 0x1D0030, .end = 0x1D0030 },
1016	{ .start = 0x1D0230, .end = 0x1D0230 },
1017	{ .start = 0x1D0510, .end = 0x1D0550 },
1018	{ .start = 0x1D4030, .end = 0x1D4030 },
1019	{ .start = 0x1D4230, .end = 0x1D4230 },
1020	{ .start = 0x1D4510, .end = 0x1D4550 },
1021	{ .start = 0x1D8030, .end = 0x1D8030 },
1022	{ .start = 0x1D8230, .end = 0x1D8230 },
1023	{ .start = 0x1D8510, .end = 0x1D8550 },
1024	};
1025
1026	static const struct i915_range gen12_shadowed_regs[] = {
1027	{ .start = 0x2030, .end = 0x2030 },
1028	{ .start = 0x2510, .end = 0x2550 },
1029	{ .start = 0xA008, .end = 0xA00C },
1030	{ .start = 0xA188, .end = 0xA188 },
1031	{ .start = 0xA278, .end = 0xA278 },
1032	{ .start = 0xA540, .end = 0xA56C },
1033	{ .start = 0xC4C8, .end = 0xC4C8 },
1034	{ .start = 0xC4D4, .end = 0xC4D4 },
1035	{ .start = 0xC600, .end = 0xC600 },
1036	{ .start = 0x22030, .end = 0x22030 },
1037	{ .start = 0x22510, .end = 0x22550 },
1038	{ .start = 0x1C0030, .end = 0x1C0030 },
1039	{ .start = 0x1C0510, .end = 0x1C0550 },
1040	{ .start = 0x1C4030, .end = 0x1C4030 },
1041	{ .start = 0x1C4510, .end = 0x1C4550 },
1042	{ .start = 0x1C8030, .end = 0x1C8030 },
1043	{ .start = 0x1C8510, .end = 0x1C8550 },
1044	{ .start = 0x1D0030, .end = 0x1D0030 },
1045	{ .start = 0x1D0510, .end = 0x1D0550 },
1046	{ .start = 0x1D4030, .end = 0x1D4030 },
1047	{ .start = 0x1D4510, .end = 0x1D4550 },
1048	{ .start = 0x1D8030, .end = 0x1D8030 },
1049	{ .start = 0x1D8510, .end = 0x1D8550 },
1050
1051	/*
1052	* The rest of these ranges are specific to Xe_HP and beyond, but
1053	* are reserved/unused ranges on earlier gen12 platforms, so they can
1054	* be safely added to the gen12 table.
1055	*/
1056	{ .start = 0x1E0030, .end = 0x1E0030 },
1057	{ .start = 0x1E0510, .end = 0x1E0550 },
1058	{ .start = 0x1E4030, .end = 0x1E4030 },
1059	{ .start = 0x1E4510, .end = 0x1E4550 },
1060	{ .start = 0x1E8030, .end = 0x1E8030 },
1061	{ .start = 0x1E8510, .end = 0x1E8550 },
1062	{ .start = 0x1F0030, .end = 0x1F0030 },
1063	{ .start = 0x1F0510, .end = 0x1F0550 },
1064	{ .start = 0x1F4030, .end = 0x1F4030 },
1065	{ .start = 0x1F4510, .end = 0x1F4550 },
1066	{ .start = 0x1F8030, .end = 0x1F8030 },
1067	{ .start = 0x1F8510, .end = 0x1F8550 },
1068	};
1069
1070	static const struct i915_range dg2_shadowed_regs[] = {
1071	{ .start = 0x2030, .end = 0x2030 },
1072	{ .start = 0x2510, .end = 0x2550 },
1073	{ .start = 0xA008, .end = 0xA00C },
1074	{ .start = 0xA188, .end = 0xA188 },
1075	{ .start = 0xA278, .end = 0xA278 },
1076	{ .start = 0xA540, .end = 0xA56C },
1077	{ .start = 0xC4C8, .end = 0xC4C8 },
1078	{ .start = 0xC4E0, .end = 0xC4E0 },
1079	{ .start = 0xC600, .end = 0xC600 },
1080	{ .start = 0xC658, .end = 0xC658 },
1081	{ .start = 0x22030, .end = 0x22030 },
1082	{ .start = 0x22510, .end = 0x22550 },
1083	{ .start = 0x1C0030, .end = 0x1C0030 },
1084	{ .start = 0x1C0510, .end = 0x1C0550 },
1085	{ .start = 0x1C4030, .end = 0x1C4030 },
1086	{ .start = 0x1C4510, .end = 0x1C4550 },
1087	{ .start = 0x1C8030, .end = 0x1C8030 },
1088	{ .start = 0x1C8510, .end = 0x1C8550 },
1089	{ .start = 0x1D0030, .end = 0x1D0030 },
1090	{ .start = 0x1D0510, .end = 0x1D0550 },
1091	{ .start = 0x1D4030, .end = 0x1D4030 },
1092	{ .start = 0x1D4510, .end = 0x1D4550 },
1093	{ .start = 0x1D8030, .end = 0x1D8030 },
1094	{ .start = 0x1D8510, .end = 0x1D8550 },
1095	{ .start = 0x1E0030, .end = 0x1E0030 },
1096	{ .start = 0x1E0510, .end = 0x1E0550 },
1097	{ .start = 0x1E4030, .end = 0x1E4030 },
1098	{ .start = 0x1E4510, .end = 0x1E4550 },
1099	{ .start = 0x1E8030, .end = 0x1E8030 },
1100	{ .start = 0x1E8510, .end = 0x1E8550 },
1101	{ .start = 0x1F0030, .end = 0x1F0030 },
1102	{ .start = 0x1F0510, .end = 0x1F0550 },
1103	{ .start = 0x1F4030, .end = 0x1F4030 },
1104	{ .start = 0x1F4510, .end = 0x1F4550 },
1105	{ .start = 0x1F8030, .end = 0x1F8030 },
1106	{ .start = 0x1F8510, .end = 0x1F8550 },
1107	};
1108
1109	static const struct i915_range pvc_shadowed_regs[] = {
1110	{ .start = 0x2030, .end = 0x2030 },
1111	{ .start = 0x2510, .end = 0x2550 },
1112	{ .start = 0xA008, .end = 0xA00C },
1113	{ .start = 0xA188, .end = 0xA188 },
1114	{ .start = 0xA278, .end = 0xA278 },
1115	{ .start = 0xA540, .end = 0xA56C },
1116	{ .start = 0xC4C8, .end = 0xC4C8 },
1117	{ .start = 0xC4E0, .end = 0xC4E0 },
1118	{ .start = 0xC600, .end = 0xC600 },
1119	{ .start = 0xC658, .end = 0xC658 },
1120	{ .start = 0x22030, .end = 0x22030 },
1121	{ .start = 0x22510, .end = 0x22550 },
1122	{ .start = 0x1C0030, .end = 0x1C0030 },
1123	{ .start = 0x1C0510, .end = 0x1C0550 },
1124	{ .start = 0x1C4030, .end = 0x1C4030 },
1125	{ .start = 0x1C4510, .end = 0x1C4550 },
1126	{ .start = 0x1C8030, .end = 0x1C8030 },
1127	{ .start = 0x1C8510, .end = 0x1C8550 },
1128	{ .start = 0x1D0030, .end = 0x1D0030 },
1129	{ .start = 0x1D0510, .end = 0x1D0550 },
1130	{ .start = 0x1D4030, .end = 0x1D4030 },
1131	{ .start = 0x1D4510, .end = 0x1D4550 },
1132	{ .start = 0x1D8030, .end = 0x1D8030 },
1133	{ .start = 0x1D8510, .end = 0x1D8550 },
1134	{ .start = 0x1E0030, .end = 0x1E0030 },
1135	{ .start = 0x1E0510, .end = 0x1E0550 },
1136	{ .start = 0x1E4030, .end = 0x1E4030 },
1137	{ .start = 0x1E4510, .end = 0x1E4550 },
1138	{ .start = 0x1E8030, .end = 0x1E8030 },
1139	{ .start = 0x1E8510, .end = 0x1E8550 },
1140	{ .start = 0x1F0030, .end = 0x1F0030 },
1141	{ .start = 0x1F0510, .end = 0x1F0550 },
1142	{ .start = 0x1F4030, .end = 0x1F4030 },
1143	{ .start = 0x1F4510, .end = 0x1F4550 },
1144	{ .start = 0x1F8030, .end = 0x1F8030 },
1145	{ .start = 0x1F8510, .end = 0x1F8550 },
1146	};
1147
1148	static const struct i915_range mtl_shadowed_regs[] = {
1149	{ .start = 0x2030, .end = 0x2030 },
1150	{ .start = 0x2510, .end = 0x2550 },
1151	{ .start = 0xA008, .end = 0xA00C },
1152	{ .start = 0xA188, .end = 0xA188 },
1153	{ .start = 0xA278, .end = 0xA278 },
1154	{ .start = 0xA540, .end = 0xA56C },
1155	{ .start = 0xC050, .end = 0xC050 },
1156	{ .start = 0xC340, .end = 0xC340 },
1157	{ .start = 0xC4C8, .end = 0xC4C8 },
1158	{ .start = 0xC4E0, .end = 0xC4E0 },
1159	{ .start = 0xC600, .end = 0xC600 },
1160	{ .start = 0xC658, .end = 0xC658 },
1161	{ .start = 0xCFD4, .end = 0xCFDC },
1162	{ .start = 0x22030, .end = 0x22030 },
1163	{ .start = 0x22510, .end = 0x22550 },
1164	};
1165
1166	static const struct i915_range xelpmp_shadowed_regs[] = {
1167	{ .start = 0x1C0030, .end = 0x1C0030 },
1168	{ .start = 0x1C0510, .end = 0x1C0550 },
1169	{ .start = 0x1C8030, .end = 0x1C8030 },
1170	{ .start = 0x1C8510, .end = 0x1C8550 },
1171	{ .start = 0x1D0030, .end = 0x1D0030 },
1172	{ .start = 0x1D0510, .end = 0x1D0550 },
1173	{ .start = 0x38A008, .end = 0x38A00C },
1174	{ .start = 0x38A188, .end = 0x38A188 },
1175	{ .start = 0x38A278, .end = 0x38A278 },
1176	{ .start = 0x38A540, .end = 0x38A56C },
1177	{ .start = 0x38A618, .end = 0x38A618 },
1178	{ .start = 0x38C050, .end = 0x38C050 },
1179	{ .start = 0x38C340, .end = 0x38C340 },
1180	{ .start = 0x38C4C8, .end = 0x38C4C8 },
1181	{ .start = 0x38C4E0, .end = 0x38C4E4 },
1182	{ .start = 0x38C600, .end = 0x38C600 },
1183	{ .start = 0x38C658, .end = 0x38C658 },
1184	{ .start = 0x38CFD4, .end = 0x38CFDC },
1185	};
1186
1187	static int mmio_range_cmp(u32 key, const struct i915_range *range)
1188	{
1189	if (key < range->start)
1190	return -1;
1191	else if (key > range->end)
1192	return 1;
1193	else
1194	return 0;
1195	}
1196
1197	static bool is_shadowed(struct intel_uncore *uncore, u32 offset)
1198	{
1199	if (drm_WARN_ON(&uncore->i915->drm, !uncore->shadowed_reg_table))
1200	return false;
1201
1202	if (IS_GSI_REG(offset))
1203	offset += uncore->gsi_offset;
1204
1205	return BSEARCH(offset,
1206	uncore->shadowed_reg_table,
1207	uncore->shadowed_reg_table_entries,
1208	mmio_range_cmp);
1209	}
1210
1211	static enum forcewake_domains
1212	gen6_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg)
1213	{
1214	return FORCEWAKE_RENDER;
1215	}
1216
1217	#define __fwtable_reg_read_fw_domains(uncore, offset) \
1218	({ \
1219	enum forcewake_domains __fwd = 0; \
1220	if (NEEDS_FORCE_WAKE((offset))) \
1221	__fwd = find_fw_domain(uncore, offset); \
1222	__fwd; \
1223	})
1224
1225	#define __fwtable_reg_write_fw_domains(uncore, offset) \
1226	({ \
1227	enum forcewake_domains __fwd = 0; \
1228	const u32 __offset = (offset); \
1229	if (NEEDS_FORCE_WAKE((__offset)) && !is_shadowed(uncore, __offset)) \
1230	__fwd = find_fw_domain(uncore, __offset); \
1231	__fwd; \
1232	})
1233
1234	#define GEN_FW_RANGE(s, e, d) \
1235	{ .start = (s), .end = (e), .domains = (d) }
1236
1237	/*
1238	* All platforms' forcewake tables below must be sorted by offset ranges.
1239	* Furthermore, new forcewake tables added should be "watertight" and have
1240	* no gaps between ranges.
1241	*
1242	* When there are multiple consecutive ranges listed in the bspec with
1243	* the same forcewake domain, it is customary to combine them into a single
1244	* row in the tables below to keep the tables small and lookups fast.
1245	* Likewise, reserved/unused ranges may be combined with the preceding and/or
1246	* following ranges since the driver will never be making MMIO accesses in
1247	* those ranges.
1248	*
1249	* For example, if the bspec were to list:
1250	*
1251	* ...
1252	* 0x1000 - 0x1fff: GT
1253	* 0x2000 - 0x2cff: GT
1254	* 0x2d00 - 0x2fff: unused/reserved
1255	* 0x3000 - 0xffff: GT
1256	* ...
1257	*
1258	* these could all be represented by a single line in the code:
1259	*
1260	* GEN_FW_RANGE(0x1000, 0xffff, FORCEWAKE_GT)
1261	*
1262	* When adding new forcewake tables here, please also add them to
1263	* intel_uncore_mock_selftests in selftests/intel_uncore.c so that they will be
1264	* scanned for obvious mistakes or typos by the selftests.
1265	*/
1266
1267	static const struct intel_forcewake_range __gen6_fw_ranges[] = {
1268	GEN_FW_RANGE(0x0, 0x3ffff, FORCEWAKE_RENDER),
1269	};
1270
1271	static const struct intel_forcewake_range __vlv_fw_ranges[] = {
1272	GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
1273	GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
1274	GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
1275	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1276	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
1277	GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
1278	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
1279	};
1280
1281	static const struct intel_forcewake_range __chv_fw_ranges[] = {
1282	GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
1283	GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1284	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1285	GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1286	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1287	GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1288	GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
1289	GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1290	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1291	GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
1292	GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
1293	GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1294	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1295	GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
1296	GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
1297	GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
1298	};
1299
1300	static const struct intel_forcewake_range __gen9_fw_ranges[] = {
1301	GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_GT),
1302	GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
1303	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1304	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1305	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1306	GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT),
1307	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1308	GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_GT),
1309	GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
1310	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1311	GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_GT),
1312	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1313	GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_GT),
1314	GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
1315	GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_GT),
1316	GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1317	GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_GT),
1318	GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1319	GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_GT),
1320	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1321	GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_GT),
1322	GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
1323	GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_GT),
1324	GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
1325	GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_GT),
1326	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1327	GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_GT),
1328	GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
1329	GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_GT),
1330	GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
1331	GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_GT),
1332	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
1333	};
1334
1335	static const struct intel_forcewake_range __gen11_fw_ranges[] = {
1336	GEN_FW_RANGE(0x0, 0x1fff, 0), /* uncore range */
1337	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1338	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1339	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1340	GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT),
1341	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1342	GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1343	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1344	GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_GT),
1345	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1346	GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_GT),
1347	GEN_FW_RANGE(0x8800, 0x8bff, 0),
1348	GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1349	GEN_FW_RANGE(0x8d00, 0x94cf, FORCEWAKE_GT),
1350	GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1351	GEN_FW_RANGE(0x9560, 0x95ff, 0),
1352	GEN_FW_RANGE(0x9600, 0xafff, FORCEWAKE_GT),
1353	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1354	GEN_FW_RANGE(0xb480, 0xdeff, FORCEWAKE_GT),
1355	GEN_FW_RANGE(0xdf00, 0xe8ff, FORCEWAKE_RENDER),
1356	GEN_FW_RANGE(0xe900, 0x16dff, FORCEWAKE_GT),
1357	GEN_FW_RANGE(0x16e00, 0x19fff, FORCEWAKE_RENDER),
1358	GEN_FW_RANGE(0x1a000, 0x23fff, FORCEWAKE_GT),
1359	GEN_FW_RANGE(0x24000, 0x2407f, 0),
1360	GEN_FW_RANGE(0x24080, 0x2417f, FORCEWAKE_GT),
1361	GEN_FW_RANGE(0x24180, 0x242ff, FORCEWAKE_RENDER),
1362	GEN_FW_RANGE(0x24300, 0x243ff, FORCEWAKE_GT),
1363	GEN_FW_RANGE(0x24400, 0x24fff, FORCEWAKE_RENDER),
1364	GEN_FW_RANGE(0x25000, 0x3ffff, FORCEWAKE_GT),
1365	GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1366	GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0),
1367	GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0),
1368	GEN_FW_RANGE(0x1c8000, 0x1cffff, FORCEWAKE_MEDIA_VEBOX0),
1369	GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2),
1370	GEN_FW_RANGE(0x1d4000, 0x1dbfff, 0)
1371	};
1372
1373	static const struct intel_forcewake_range __gen12_fw_ranges[] = {
1374	GEN_FW_RANGE(0x0, 0x1fff, 0), /*
1375	0x0 - 0xaff: reserved
1376	0xb00 - 0x1fff: always on */
1377	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1378	GEN_FW_RANGE(0x2700, 0x27ff, FORCEWAKE_GT),
1379	GEN_FW_RANGE(0x2800, 0x2aff, FORCEWAKE_RENDER),
1380	GEN_FW_RANGE(0x2b00, 0x2fff, FORCEWAKE_GT),
1381	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1382	GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), /*
1383	0x4000 - 0x48ff: gt
1384	0x4900 - 0x51ff: reserved */
1385	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), /*
1386	0x5200 - 0x53ff: render
1387	0x5400 - 0x54ff: reserved
1388	0x5500 - 0x7fff: render */
1389	GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1390	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1391	GEN_FW_RANGE(0x8160, 0x81ff, 0), /*
1392	0x8160 - 0x817f: reserved
1393	0x8180 - 0x81ff: always on */
1394	GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT),
1395	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1396	GEN_FW_RANGE(0x8500, 0x94cf, FORCEWAKE_GT), /*
1397	0x8500 - 0x87ff: gt
1398	0x8800 - 0x8fff: reserved
1399	0x9000 - 0x947f: gt
1400	0x9480 - 0x94cf: reserved */
1401	GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1402	GEN_FW_RANGE(0x9560, 0x97ff, 0), /*
1403	0x9560 - 0x95ff: always on
1404	0x9600 - 0x97ff: reserved */
1405	GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_GT),
1406	GEN_FW_RANGE(0xb000, 0xb3ff, FORCEWAKE_RENDER),
1407	GEN_FW_RANGE(0xb400, 0xcfff, FORCEWAKE_GT), /*
1408	0xb400 - 0xbf7f: gt
1409	0xb480 - 0xbfff: reserved
1410	0xc000 - 0xcfff: gt */
1411	GEN_FW_RANGE(0xd000, 0xd7ff, 0),
1412	GEN_FW_RANGE(0xd800, 0xd8ff, FORCEWAKE_RENDER),
1413	GEN_FW_RANGE(0xd900, 0xdbff, FORCEWAKE_GT),
1414	GEN_FW_RANGE(0xdc00, 0xefff, FORCEWAKE_RENDER), /*
1415	0xdc00 - 0xddff: render
1416	0xde00 - 0xde7f: reserved
1417	0xde80 - 0xe8ff: render
1418	0xe900 - 0xefff: reserved */
1419	GEN_FW_RANGE(0xf000, 0x147ff, FORCEWAKE_GT), /*
1420	0xf000 - 0xffff: gt
1421	0x10000 - 0x147ff: reserved */
1422	GEN_FW_RANGE(0x14800, 0x1ffff, FORCEWAKE_RENDER), /*
1423	0x14800 - 0x14fff: render
1424	0x15000 - 0x16dff: reserved
1425	0x16e00 - 0x1bfff: render
1426	0x1c000 - 0x1ffff: reserved */
1427	GEN_FW_RANGE(0x20000, 0x20fff, FORCEWAKE_MEDIA_VDBOX0),
1428	GEN_FW_RANGE(0x21000, 0x21fff, FORCEWAKE_MEDIA_VDBOX2),
1429	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1430	GEN_FW_RANGE(0x24000, 0x2417f, 0), /*
1431	0x24000 - 0x2407f: always on
1432	0x24080 - 0x2417f: reserved */
1433	GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /*
1434	0x24180 - 0x241ff: gt
1435	0x24200 - 0x249ff: reserved */
1436	GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /*
1437	0x24a00 - 0x24a7f: render
1438	0x24a80 - 0x251ff: reserved */
1439	GEN_FW_RANGE(0x25200, 0x255ff, FORCEWAKE_GT), /*
1440	0x25200 - 0x252ff: gt
1441	0x25300 - 0x255ff: reserved */
1442	GEN_FW_RANGE(0x25600, 0x2567f, FORCEWAKE_MEDIA_VDBOX0),
1443	GEN_FW_RANGE(0x25680, 0x259ff, FORCEWAKE_MEDIA_VDBOX2), /*
1444	0x25680 - 0x256ff: VD2
1445	0x25700 - 0x259ff: reserved */
1446	GEN_FW_RANGE(0x25a00, 0x25a7f, FORCEWAKE_MEDIA_VDBOX0),
1447	GEN_FW_RANGE(0x25a80, 0x2ffff, FORCEWAKE_MEDIA_VDBOX2), /*
1448	0x25a80 - 0x25aff: VD2
1449	0x25b00 - 0x2ffff: reserved */
1450	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1451	GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1452	GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*
1453	0x1c0000 - 0x1c2bff: VD0
1454	0x1c2c00 - 0x1c2cff: reserved
1455	0x1c2d00 - 0x1c2dff: VD0
1456	0x1c2e00 - 0x1c3eff: reserved
1457	0x1c3f00 - 0x1c3fff: VD0 */
1458	GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0),
1459	GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
1460	0x1c8000 - 0x1ca0ff: VE0
1461	0x1ca100 - 0x1cbeff: reserved
1462	0x1cbf00 - 0x1cbfff: VE0 */
1463	GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX0), /*
1464	0x1cc000 - 0x1ccfff: VD0
1465	0x1cd000 - 0x1cffff: reserved */
1466	GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /*
1467	0x1d0000 - 0x1d2bff: VD2
1468	0x1d2c00 - 0x1d2cff: reserved
1469	0x1d2d00 - 0x1d2dff: VD2
1470	0x1d2e00 - 0x1d3eff: reserved
1471	0x1d3f00 - 0x1d3fff: VD2 */
1472	};
1473
1474	/*
1475	* Graphics IP version 12.55 brings a slight change to the 0xd800 range,
1476	* switching it from the GT domain to the render domain.
1477	*/
1478	#define XEHP_FWRANGES(FW_RANGE_D800) \
1479	GEN_FW_RANGE(0x0, 0x1fff, 0), /* \
1480	0x0 - 0xaff: reserved \
1481	0xb00 - 0x1fff: always on */ \
1482	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), \
1483	GEN_FW_RANGE(0x2700, 0x4aff, FORCEWAKE_GT), \
1484	GEN_FW_RANGE(0x4b00, 0x51ff, 0), /* \
1485	0x4b00 - 0x4fff: reserved \
1486	0x5000 - 0x51ff: always on */ \
1487	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), \
1488	GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT), \
1489	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), \
1490	GEN_FW_RANGE(0x8160, 0x81ff, 0), /* \
1491	0x8160 - 0x817f: reserved \
1492	0x8180 - 0x81ff: always on */ \
1493	GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT), \
1494	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), \
1495	GEN_FW_RANGE(0x8500, 0x8cff, FORCEWAKE_GT), /* \
1496	0x8500 - 0x87ff: gt \
1497	0x8800 - 0x8c7f: reserved \
1498	0x8c80 - 0x8cff: gt (DG2 only) */ \
1499	GEN_FW_RANGE(0x8d00, 0x8fff, FORCEWAKE_RENDER), /* \
1500	0x8d00 - 0x8dff: render (DG2 only) \
1501	0x8e00 - 0x8fff: reserved */ \
1502	GEN_FW_RANGE(0x9000, 0x94cf, FORCEWAKE_GT), /* \
1503	0x9000 - 0x947f: gt \
1504	0x9480 - 0x94cf: reserved */ \
1505	GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER), \
1506	GEN_FW_RANGE(0x9560, 0x967f, 0), /* \
1507	0x9560 - 0x95ff: always on \
1508	0x9600 - 0x967f: reserved */ \
1509	GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /* \
1510	0x9680 - 0x96ff: render (DG2 only) \
1511	0x9700 - 0x97ff: reserved */ \
1512	GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /* \
1513	0x9800 - 0xb4ff: gt \
1514	0xb500 - 0xbfff: reserved \
1515	0xc000 - 0xcfff: gt */ \
1516	GEN_FW_RANGE(0xd000, 0xd7ff, 0), \
1517	GEN_FW_RANGE(0xd800, 0xd87f, FW_RANGE_D800), \
1518	GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT), \
1519	GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER), \
1520	GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /* \
1521	0xdd00 - 0xddff: gt \
1522	0xde00 - 0xde7f: reserved */ \
1523	GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /* \
1524	0xde80 - 0xdfff: render \
1525	0xe000 - 0xe0ff: reserved \
1526	0xe100 - 0xe8ff: render */ \
1527	GEN_FW_RANGE(0xe900, 0xffff, FORCEWAKE_GT), /* \
1528	0xe900 - 0xe9ff: gt \
1529	0xea00 - 0xefff: reserved \
1530	0xf000 - 0xffff: gt */ \
1531	GEN_FW_RANGE(0x10000, 0x12fff, 0), /* \
1532	0x10000 - 0x11fff: reserved \
1533	0x12000 - 0x127ff: always on \
1534	0x12800 - 0x12fff: reserved */ \
1535	GEN_FW_RANGE(0x13000, 0x131ff, FORCEWAKE_MEDIA_VDBOX0), /* DG2 only */ \
1536	GEN_FW_RANGE(0x13200, 0x13fff, FORCEWAKE_MEDIA_VDBOX2), /* \
1537	0x13200 - 0x133ff: VD2 (DG2 only) \
1538	0x13400 - 0x13fff: reserved */ \
1539	GEN_FW_RANGE(0x14000, 0x141ff, FORCEWAKE_MEDIA_VDBOX0), /* XEHPSDV only */ \
1540	GEN_FW_RANGE(0x14200, 0x143ff, FORCEWAKE_MEDIA_VDBOX2), /* XEHPSDV only */ \
1541	GEN_FW_RANGE(0x14400, 0x145ff, FORCEWAKE_MEDIA_VDBOX4), /* XEHPSDV only */ \
1542	GEN_FW_RANGE(0x14600, 0x147ff, FORCEWAKE_MEDIA_VDBOX6), /* XEHPSDV only */ \
1543	GEN_FW_RANGE(0x14800, 0x14fff, FORCEWAKE_RENDER), \
1544	GEN_FW_RANGE(0x15000, 0x16dff, FORCEWAKE_GT), /* \
1545	0x15000 - 0x15fff: gt (DG2 only) \
1546	0x16000 - 0x16dff: reserved */ \
1547	GEN_FW_RANGE(0x16e00, 0x1ffff, FORCEWAKE_RENDER), \
1548	GEN_FW_RANGE(0x20000, 0x21fff, FORCEWAKE_MEDIA_VDBOX0), /* \
1549	0x20000 - 0x20fff: VD0 (XEHPSDV only) \
1550	0x21000 - 0x21fff: reserved */ \
1551	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT), \
1552	GEN_FW_RANGE(0x24000, 0x2417f, 0), /* \
1553	0x24000 - 0x2407f: always on \
1554	0x24080 - 0x2417f: reserved */ \
1555	GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /* \
1556	0x24180 - 0x241ff: gt \
1557	0x24200 - 0x249ff: reserved */ \
1558	GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /* \
1559	0x24a00 - 0x24a7f: render \
1560	0x24a80 - 0x251ff: reserved */ \
1561	GEN_FW_RANGE(0x25200, 0x25fff, FORCEWAKE_GT), /* \
1562	0x25200 - 0x252ff: gt \
1563	0x25300 - 0x25fff: reserved */ \
1564	GEN_FW_RANGE(0x26000, 0x2ffff, FORCEWAKE_RENDER), /* \
1565	0x26000 - 0x27fff: render \
1566	0x28000 - 0x29fff: reserved \
1567	0x2a000 - 0x2ffff: undocumented */ \
1568	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT), \
1569	GEN_FW_RANGE(0x40000, 0x1bffff, 0), \
1570	GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /* \
1571	0x1c0000 - 0x1c2bff: VD0 \
1572	0x1c2c00 - 0x1c2cff: reserved \
1573	0x1c2d00 - 0x1c2dff: VD0 \
1574	0x1c2e00 - 0x1c3eff: VD0 (DG2 only) \
1575	0x1c3f00 - 0x1c3fff: VD0 */ \
1576	GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), /* \
1577	0x1c4000 - 0x1c6bff: VD1 \
1578	0x1c6c00 - 0x1c6cff: reserved \
1579	0x1c6d00 - 0x1c6dff: VD1 \
1580	0x1c6e00 - 0x1c7fff: reserved */ \
1581	GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /* \
1582	0x1c8000 - 0x1ca0ff: VE0 \
1583	0x1ca100 - 0x1cbfff: reserved */ \
1584	GEN_FW_RANGE(0x1cc000, 0x1ccfff, FORCEWAKE_MEDIA_VDBOX0), \
1585	GEN_FW_RANGE(0x1cd000, 0x1cdfff, FORCEWAKE_MEDIA_VDBOX2), \
1586	GEN_FW_RANGE(0x1ce000, 0x1cefff, FORCEWAKE_MEDIA_VDBOX4), \
1587	GEN_FW_RANGE(0x1cf000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX6), \
1588	GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /* \
1589	0x1d0000 - 0x1d2bff: VD2 \
1590	0x1d2c00 - 0x1d2cff: reserved \
1591	0x1d2d00 - 0x1d2dff: VD2 \
1592	0x1d2e00 - 0x1d3dff: VD2 (DG2 only) \
1593	0x1d3e00 - 0x1d3eff: reserved \
1594	0x1d3f00 - 0x1d3fff: VD2 */ \
1595	GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), /* \
1596	0x1d4000 - 0x1d6bff: VD3 \
1597	0x1d6c00 - 0x1d6cff: reserved \
1598	0x1d6d00 - 0x1d6dff: VD3 \
1599	0x1d6e00 - 0x1d7fff: reserved */ \
1600	GEN_FW_RANGE(0x1d8000, 0x1dffff, FORCEWAKE_MEDIA_VEBOX1), /* \
1601	0x1d8000 - 0x1da0ff: VE1 \
1602	0x1da100 - 0x1dffff: reserved */ \
1603	GEN_FW_RANGE(0x1e0000, 0x1e3fff, FORCEWAKE_MEDIA_VDBOX4), /* \
1604	0x1e0000 - 0x1e2bff: VD4 \
1605	0x1e2c00 - 0x1e2cff: reserved \
1606	0x1e2d00 - 0x1e2dff: VD4 \
1607	0x1e2e00 - 0x1e3eff: reserved \
1608	0x1e3f00 - 0x1e3fff: VD4 */ \
1609	GEN_FW_RANGE(0x1e4000, 0x1e7fff, FORCEWAKE_MEDIA_VDBOX5), /* \
1610	0x1e4000 - 0x1e6bff: VD5 \
1611	0x1e6c00 - 0x1e6cff: reserved \
1612	0x1e6d00 - 0x1e6dff: VD5 \
1613	0x1e6e00 - 0x1e7fff: reserved */ \
1614	GEN_FW_RANGE(0x1e8000, 0x1effff, FORCEWAKE_MEDIA_VEBOX2), /* \
1615	0x1e8000 - 0x1ea0ff: VE2 \
1616	0x1ea100 - 0x1effff: reserved */ \
1617	GEN_FW_RANGE(0x1f0000, 0x1f3fff, FORCEWAKE_MEDIA_VDBOX6), /* \
1618	0x1f0000 - 0x1f2bff: VD6 \
1619	0x1f2c00 - 0x1f2cff: reserved \
1620	0x1f2d00 - 0x1f2dff: VD6 \
1621	0x1f2e00 - 0x1f3eff: reserved \
1622	0x1f3f00 - 0x1f3fff: VD6 */ \
1623	GEN_FW_RANGE(0x1f4000, 0x1f7fff, FORCEWAKE_MEDIA_VDBOX7), /* \
1624	0x1f4000 - 0x1f6bff: VD7 \
1625	0x1f6c00 - 0x1f6cff: reserved \
1626	0x1f6d00 - 0x1f6dff: VD7 \
1627	0x1f6e00 - 0x1f7fff: reserved */ \
1628	GEN_FW_RANGE(0x1f8000, 0x1fa0ff, FORCEWAKE_MEDIA_VEBOX3),
1629
1630	static const struct intel_forcewake_range __xehp_fw_ranges[] = {
1631	XEHP_FWRANGES(FORCEWAKE_GT)
1632	};
1633
1634	static const struct intel_forcewake_range __dg2_fw_ranges[] = {
1635	XEHP_FWRANGES(FORCEWAKE_RENDER)
1636	};
1637
1638	static const struct intel_forcewake_range __pvc_fw_ranges[] = {
1639	GEN_FW_RANGE(0x0, 0xaff, 0),
1640	GEN_FW_RANGE(0xb00, 0xbff, FORCEWAKE_GT),
1641	GEN_FW_RANGE(0xc00, 0xfff, 0),
1642	GEN_FW_RANGE(0x1000, 0x1fff, FORCEWAKE_GT),
1643	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1644	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1645	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1646	GEN_FW_RANGE(0x4000, 0x813f, FORCEWAKE_GT), /*
1647	0x4000 - 0x4aff: gt
1648	0x4b00 - 0x4fff: reserved
1649	0x5000 - 0x51ff: gt
1650	0x5200 - 0x52ff: reserved
1651	0x5300 - 0x53ff: gt
1652	0x5400 - 0x7fff: reserved
1653	0x8000 - 0x813f: gt */
1654	GEN_FW_RANGE(0x8140, 0x817f, FORCEWAKE_RENDER),
1655	GEN_FW_RANGE(0x8180, 0x81ff, 0),
1656	GEN_FW_RANGE(0x8200, 0x94cf, FORCEWAKE_GT), /*
1657	0x8200 - 0x82ff: gt
1658	0x8300 - 0x84ff: reserved
1659	0x8500 - 0x887f: gt
1660	0x8880 - 0x8a7f: reserved
1661	0x8a80 - 0x8aff: gt
1662	0x8b00 - 0x8fff: reserved
1663	0x9000 - 0x947f: gt
1664	0x9480 - 0x94cf: reserved */
1665	GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1666	GEN_FW_RANGE(0x9560, 0x967f, 0), /*
1667	0x9560 - 0x95ff: always on
1668	0x9600 - 0x967f: reserved */
1669	GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /*
1670	0x9680 - 0x96ff: render
1671	0x9700 - 0x97ff: reserved */
1672	GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /*
1673	0x9800 - 0xb4ff: gt
1674	0xb500 - 0xbfff: reserved
1675	0xc000 - 0xcfff: gt */
1676	GEN_FW_RANGE(0xd000, 0xd3ff, 0),
1677	GEN_FW_RANGE(0xd400, 0xdbff, FORCEWAKE_GT),
1678	GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),
1679	GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*
1680	0xdd00 - 0xddff: gt
1681	0xde00 - 0xde7f: reserved */
1682	GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*
1683	0xde80 - 0xdeff: render
1684	0xdf00 - 0xe1ff: reserved
1685	0xe200 - 0xe7ff: render
1686	0xe800 - 0xe8ff: reserved */
1687	GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_GT), /*
1688	0xe900 - 0xe9ff: gt
1689	0xea00 - 0xebff: reserved
1690	0xec00 - 0xffff: gt
1691	0x10000 - 0x11fff: reserved */
1692	GEN_FW_RANGE(0x12000, 0x12fff, 0), /*
1693	0x12000 - 0x127ff: always on
1694	0x12800 - 0x12fff: reserved */
1695	GEN_FW_RANGE(0x13000, 0x19fff, FORCEWAKE_GT), /*
1696	0x13000 - 0x135ff: gt
1697	0x13600 - 0x147ff: reserved
1698	0x14800 - 0x153ff: gt
1699	0x15400 - 0x19fff: reserved */
1700	GEN_FW_RANGE(0x1a000, 0x21fff, FORCEWAKE_RENDER), /*
1701	0x1a000 - 0x1ffff: render
1702	0x20000 - 0x21fff: reserved */
1703	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1704	GEN_FW_RANGE(0x24000, 0x2417f, 0), /*
1705	24000 - 0x2407f: always on
1706	24080 - 0x2417f: reserved */
1707	GEN_FW_RANGE(0x24180, 0x25fff, FORCEWAKE_GT), /*
1708	0x24180 - 0x241ff: gt
1709	0x24200 - 0x251ff: reserved
1710	0x25200 - 0x252ff: gt
1711	0x25300 - 0x25fff: reserved */
1712	GEN_FW_RANGE(0x26000, 0x2ffff, FORCEWAKE_RENDER), /*
1713	0x26000 - 0x27fff: render
1714	0x28000 - 0x2ffff: reserved */
1715	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1716	GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1717	GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*
1718	0x1c0000 - 0x1c2bff: VD0
1719	0x1c2c00 - 0x1c2cff: reserved
1720	0x1c2d00 - 0x1c2dff: VD0
1721	0x1c2e00 - 0x1c3eff: reserved
1722	0x1c3f00 - 0x1c3fff: VD0 */
1723	GEN_FW_RANGE(0x1c4000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX1), /*
1724	0x1c4000 - 0x1c6aff: VD1
1725	0x1c6b00 - 0x1c7eff: reserved
1726	0x1c7f00 - 0x1c7fff: VD1
1727	0x1c8000 - 0x1cffff: reserved */
1728	GEN_FW_RANGE(0x1d0000, 0x23ffff, FORCEWAKE_MEDIA_VDBOX2), /*
1729	0x1d0000 - 0x1d2aff: VD2
1730	0x1d2b00 - 0x1d3eff: reserved
1731	0x1d3f00 - 0x1d3fff: VD2
1732	0x1d4000 - 0x23ffff: reserved */
1733	GEN_FW_RANGE(0x240000, 0x3dffff, 0),
1734	GEN_FW_RANGE(0x3e0000, 0x3effff, FORCEWAKE_GT),
1735	};
1736
1737	static const struct intel_forcewake_range __mtl_fw_ranges[] = {
1738	GEN_FW_RANGE(0x0, 0xaff, 0),
1739	GEN_FW_RANGE(0xb00, 0xbff, FORCEWAKE_GT),
1740	GEN_FW_RANGE(0xc00, 0xfff, 0),
1741	GEN_FW_RANGE(0x1000, 0x1fff, FORCEWAKE_GT),
1742	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1743	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1744	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1745	GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), /*
1746	0x4000 - 0x48ff: render
1747	0x4900 - 0x51ff: reserved */
1748	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), /*
1749	0x5200 - 0x53ff: render
1750	0x5400 - 0x54ff: reserved
1751	0x5500 - 0x7fff: render */
1752	GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1753	GEN_FW_RANGE(0x8140, 0x817f, FORCEWAKE_RENDER), /*
1754	0x8140 - 0x815f: render
1755	0x8160 - 0x817f: reserved */
1756	GEN_FW_RANGE(0x8180, 0x81ff, 0),
1757	GEN_FW_RANGE(0x8200, 0x94cf, FORCEWAKE_GT), /*
1758	0x8200 - 0x87ff: gt
1759	0x8800 - 0x8dff: reserved
1760	0x8e00 - 0x8f7f: gt
1761	0x8f80 - 0x8fff: reserved
1762	0x9000 - 0x947f: gt
1763	0x9480 - 0x94cf: reserved */
1764	GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1765	GEN_FW_RANGE(0x9560, 0x967f, 0), /*
1766	0x9560 - 0x95ff: always on
1767	0x9600 - 0x967f: reserved */
1768	GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /*
1769	0x9680 - 0x96ff: render
1770	0x9700 - 0x97ff: reserved */
1771	GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /*
1772	0x9800 - 0xb4ff: gt
1773	0xb500 - 0xbfff: reserved
1774	0xc000 - 0xcfff: gt */
1775	GEN_FW_RANGE(0xd000, 0xd7ff, 0), /*
1776	0xd000 - 0xd3ff: always on
1777	0xd400 - 0xd7ff: reserved */
1778	GEN_FW_RANGE(0xd800, 0xd87f, FORCEWAKE_RENDER),
1779	GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT),
1780	GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),
1781	GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*
1782	0xdd00 - 0xddff: gt
1783	0xde00 - 0xde7f: reserved */
1784	GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*
1785	0xde80 - 0xdfff: render
1786	0xe000 - 0xe0ff: reserved
1787	0xe100 - 0xe8ff: render */
1788	GEN_FW_RANGE(0xe900, 0xe9ff, FORCEWAKE_GT),
1789	GEN_FW_RANGE(0xea00, 0x147ff, 0), /*
1790	0xea00 - 0x11fff: reserved
1791	0x12000 - 0x127ff: always on
1792	0x12800 - 0x147ff: reserved */
1793	GEN_FW_RANGE(0x14800, 0x19fff, FORCEWAKE_GT), /*
1794	0x14800 - 0x153ff: gt
1795	0x15400 - 0x19fff: reserved */
1796	GEN_FW_RANGE(0x1a000, 0x21fff, FORCEWAKE_RENDER), /*
1797	0x1a000 - 0x1bfff: render
1798	0x1c000 - 0x21fff: reserved */
1799	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1800	GEN_FW_RANGE(0x24000, 0x2ffff, 0), /*
1801	0x24000 - 0x2407f: always on
1802	0x24080 - 0x2ffff: reserved */
1803	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1804	GEN_FW_RANGE(0x40000, 0x1901ef, 0),
1805	GEN_FW_RANGE(0x1901f0, 0x1901f3, FORCEWAKE_GT)
1806	/* FIXME: WA to wake GT while triggering H2G */
1807	};
1808
1809	/*
1810	* Note that the register ranges here are the final offsets after
1811	* translation of the GSI block to the 0x380000 offset.
1812	*
1813	* NOTE: There are a couple MCR ranges near the bottom of this table
1814	* that need to power up either VD0 or VD2 depending on which replicated
1815	* instance of the register we're trying to access. Our forcewake logic
1816	* at the moment doesn't have a good way to take steering into consideration,
1817	* and the driver doesn't even access any registers in those ranges today,
1818	* so for now we just mark those ranges as FORCEWAKE_ALL. That will ensure
1819	* proper operation if we do start using the ranges in the future, and we
1820	* can determine at that time whether it's worth adding extra complexity to
1821	* the forcewake handling to take steering into consideration.
1822	*/
1823	static const struct intel_forcewake_range __xelpmp_fw_ranges[] = {
1824	GEN_FW_RANGE(0x0, 0x115fff, 0), /* render GT range */
1825	GEN_FW_RANGE(0x116000, 0x11ffff, FORCEWAKE_GSC), /*
1826	0x116000 - 0x117fff: gsc
1827	0x118000 - 0x119fff: reserved
1828	0x11a000 - 0x11efff: gsc
1829	0x11f000 - 0x11ffff: reserved */
1830	GEN_FW_RANGE(0x120000, 0x1bffff, 0), /* non-GT range */
1831	GEN_FW_RANGE(0x1c0000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX0), /*
1832	0x1c0000 - 0x1c3dff: VD0
1833	0x1c3e00 - 0x1c3eff: reserved
1834	0x1c3f00 - 0x1c3fff: VD0
1835	0x1c4000 - 0x1c7fff: reserved */
1836	GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
1837	0x1c8000 - 0x1ca0ff: VE0
1838	0x1ca100 - 0x1cbfff: reserved */
1839	GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX0), /*
1840	0x1cc000 - 0x1cdfff: VD0
1841	0x1ce000 - 0x1cffff: reserved */
1842	GEN_FW_RANGE(0x1d0000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX2), /*
1843	0x1d0000 - 0x1d3dff: VD2
1844	0x1d3e00 - 0x1d3eff: reserved
1845	0x1d4000 - 0x1d7fff: VD2 */
1846	GEN_FW_RANGE(0x1d8000, 0x1da0ff, FORCEWAKE_MEDIA_VEBOX1),
1847	GEN_FW_RANGE(0x1da100, 0x380aff, 0), /*
1848	0x1da100 - 0x23ffff: reserved
1849	0x240000 - 0x37ffff: non-GT range
1850	0x380000 - 0x380aff: reserved */
1851	GEN_FW_RANGE(0x380b00, 0x380bff, FORCEWAKE_GT),
1852	GEN_FW_RANGE(0x380c00, 0x380fff, 0),
1853	GEN_FW_RANGE(0x381000, 0x38817f, FORCEWAKE_GT), /*
1854	0x381000 - 0x381fff: gt
1855	0x382000 - 0x383fff: reserved
1856	0x384000 - 0x384aff: gt
1857	0x384b00 - 0x3851ff: reserved
1858	0x385200 - 0x3871ff: gt
1859	0x387200 - 0x387fff: reserved
1860	0x388000 - 0x38813f: gt
1861	0x388140 - 0x38817f: reserved */
1862	GEN_FW_RANGE(0x388180, 0x3882ff, 0), /*
1863	0x388180 - 0x3881ff: always on
1864	0x388200 - 0x3882ff: reserved */
1865	GEN_FW_RANGE(0x388300, 0x38955f, FORCEWAKE_GT), /*
1866	0x388300 - 0x38887f: gt
1867	0x388880 - 0x388fff: reserved
1868	0x389000 - 0x38947f: gt
1869	0x389480 - 0x38955f: reserved */
1870	GEN_FW_RANGE(0x389560, 0x389fff, 0), /*
1871	0x389560 - 0x3895ff: always on
1872	0x389600 - 0x389fff: reserved */
1873	GEN_FW_RANGE(0x38a000, 0x38cfff, FORCEWAKE_GT), /*
1874	0x38a000 - 0x38afff: gt
1875	0x38b000 - 0x38bfff: reserved
1876	0x38c000 - 0x38cfff: gt */
1877	GEN_FW_RANGE(0x38d000, 0x38d11f, 0),
1878	GEN_FW_RANGE(0x38d120, 0x391fff, FORCEWAKE_GT), /*
1879	0x38d120 - 0x38dfff: gt
1880	0x38e000 - 0x38efff: reserved
1881	0x38f000 - 0x38ffff: gt
1882	0x389000 - 0x391fff: reserved */
1883	GEN_FW_RANGE(0x392000, 0x392fff, 0), /*
1884	0x392000 - 0x3927ff: always on
1885	0x392800 - 0x292fff: reserved */
1886	GEN_FW_RANGE(0x393000, 0x3931ff, FORCEWAKE_GT),
1887	GEN_FW_RANGE(0x393200, 0x39323f, FORCEWAKE_ALL), /* instance-based, see note above */
1888	GEN_FW_RANGE(0x393240, 0x3933ff, FORCEWAKE_GT),
1889	GEN_FW_RANGE(0x393400, 0x3934ff, FORCEWAKE_ALL), /* instance-based, see note above */
1890	GEN_FW_RANGE(0x393500, 0x393c7f, 0), /*
1891	0x393500 - 0x393bff: reserved
1892	0x393c00 - 0x393c7f: always on */
1893	GEN_FW_RANGE(0x393c80, 0x393dff, FORCEWAKE_GT),
1894	};
1895
1896	static void
1897	ilk_dummy_write(struct intel_uncore *uncore)
1898	{
1899	/* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
1900	* the chip from rc6 before touching it for real. MI_MODE is masked,
1901	* hence harmless to write 0 into. */
1902	__raw_uncore_write32(uncore, RING_MI_MODE(RENDER_RING_BASE), val: 0);
1903	}
1904
1905	static void
1906	__unclaimed_reg_debug(struct intel_uncore *uncore,
1907	const i915_reg_t reg,
1908	const bool read)
1909	{
1910	if (drm_WARN(&uncore->i915->drm,
1911	check_for_unclaimed_mmio(uncore),
1912	"Unclaimed %s register 0x%x\n",
1913	read ? "read from" : "write to",
1914	i915_mmio_reg_offset(reg)))
1915	/* Only report the first N failures */
1916	uncore->i915->params.mmio_debug--;
1917	}
1918
1919	static void
1920	__unclaimed_previous_reg_debug(struct intel_uncore *uncore,
1921	const i915_reg_t reg,
1922	const bool read)
1923	{
1924	if (check_for_unclaimed_mmio(uncore))
1925	drm_dbg(&uncore->i915->drm,
1926	"Unclaimed access detected before %s register 0x%x\n",
1927	read ? "read from" : "write to",
1928	i915_mmio_reg_offset(reg));
1929	}
1930
1931	static inline bool __must_check
1932	unclaimed_reg_debug_header(struct intel_uncore *uncore,
1933	const i915_reg_t reg, const bool read)
1934	{
1935	if (likely(!uncore->i915->params.mmio_debug) || !uncore->debug)
1936	return false;
1937
1938	/* interrupts are disabled and re-enabled around uncore->lock usage */
1939	lockdep_assert_held(&uncore->lock);
1940
1941	spin_lock(lock: &uncore->debug->lock);
1942	__unclaimed_previous_reg_debug(uncore, reg, read);
1943
1944	return true;
1945	}
1946
1947	static inline void
1948	unclaimed_reg_debug_footer(struct intel_uncore *uncore,
1949	const i915_reg_t reg, const bool read)
1950	{
1951	/* interrupts are disabled and re-enabled around uncore->lock usage */
1952	lockdep_assert_held(&uncore->lock);
1953
1954	__unclaimed_reg_debug(uncore, reg, read);
1955	spin_unlock(lock: &uncore->debug->lock);
1956	}
1957
1958	#define __vgpu_read(x) \
1959	static u##x \
1960	vgpu_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1961	u##x val = __raw_uncore_read##x(uncore, reg); \
1962	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1963	return val; \
1964	}
1965	__vgpu_read(8)
1966	__vgpu_read(16)
1967	__vgpu_read(32)
1968	__vgpu_read(64)
1969
1970	#define GEN2_READ_HEADER(x) \
1971	u##x val = 0; \
1972	assert_rpm_wakelock_held(uncore->rpm);
1973
1974	#define GEN2_READ_FOOTER \
1975	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1976	return val
1977
1978	#define __gen2_read(x) \
1979	static u##x \
1980	gen2_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1981	GEN2_READ_HEADER(x); \
1982	val = __raw_uncore_read##x(uncore, reg); \
1983	GEN2_READ_FOOTER; \
1984	}
1985
1986	#define __gen5_read(x) \
1987	static u##x \
1988	gen5_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1989	GEN2_READ_HEADER(x); \
1990	ilk_dummy_write(uncore); \
1991	val = __raw_uncore_read##x(uncore, reg); \
1992	GEN2_READ_FOOTER; \
1993	}
1994
1995	__gen5_read(8)
1996	__gen5_read(16)
1997	__gen5_read(32)
1998	__gen5_read(64)
1999	__gen2_read(8)
2000	__gen2_read(16)
2001	__gen2_read(32)
2002	__gen2_read(64)
2003
2004	#undef __gen5_read
2005	#undef __gen2_read
2006
2007	#undef GEN2_READ_FOOTER
2008	#undef GEN2_READ_HEADER
2009
2010	#define GEN6_READ_HEADER(x) \
2011	u32 offset = i915_mmio_reg_offset(reg); \
2012	unsigned long irqflags; \
2013	bool unclaimed_reg_debug; \
2014	u##x val = 0; \
2015	assert_rpm_wakelock_held(uncore->rpm); \
2016	spin_lock_irqsave(&uncore->lock, irqflags); \
2017	unclaimed_reg_debug = unclaimed_reg_debug_header(uncore, reg, true)
2018
2019	#define GEN6_READ_FOOTER \
2020	if (unclaimed_reg_debug) \
2021	unclaimed_reg_debug_footer(uncore, reg, true); \
2022	spin_unlock_irqrestore(&uncore->lock, irqflags); \
2023	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
2024	return val
2025
2026	static noinline void ___force_wake_auto(struct intel_uncore *uncore,
2027	enum forcewake_domains fw_domains)
2028	{
2029	struct intel_uncore_forcewake_domain *domain;
2030	unsigned int tmp;
2031
2032	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
2033
2034	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp)
2035	fw_domain_arm_timer(d: domain);
2036
2037	fw_domains_get(uncore, fw_domains);
2038	}
2039
2040	static inline void __force_wake_auto(struct intel_uncore *uncore,
2041	enum forcewake_domains fw_domains)
2042	{
2043	GEM_BUG_ON(!fw_domains);
2044
2045	/* Turn on all requested but inactive supported forcewake domains. */
2046	fw_domains &= uncore->fw_domains;
2047	fw_domains &= ~uncore->fw_domains_active;
2048
2049	if (fw_domains)
2050	___force_wake_auto(uncore, fw_domains);
2051	}
2052
2053	#define __gen_fwtable_read(x) \
2054	static u##x \
2055	fwtable_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) \
2056	{ \
2057	enum forcewake_domains fw_engine; \
2058	GEN6_READ_HEADER(x); \
2059	fw_engine = __fwtable_reg_read_fw_domains(uncore, offset); \
2060	if (fw_engine) \
2061	__force_wake_auto(uncore, fw_engine); \
2062	val = __raw_uncore_read##x(uncore, reg); \
2063	GEN6_READ_FOOTER; \
2064	}
2065
2066	static enum forcewake_domains
2067	fwtable_reg_read_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) {
2068	return __fwtable_reg_read_fw_domains(uncore, i915_mmio_reg_offset(reg));
2069	}
2070
2071	__gen_fwtable_read(8)
2072	__gen_fwtable_read(16)
2073	__gen_fwtable_read(32)
2074	__gen_fwtable_read(64)
2075
2076	#undef __gen_fwtable_read
2077	#undef GEN6_READ_FOOTER
2078	#undef GEN6_READ_HEADER
2079
2080	#define GEN2_WRITE_HEADER \
2081	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
2082	assert_rpm_wakelock_held(uncore->rpm); \
2083
2084	#define GEN2_WRITE_FOOTER
2085
2086	#define __gen2_write(x) \
2087	static void \
2088	gen2_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2089	GEN2_WRITE_HEADER; \
2090	__raw_uncore_write##x(uncore, reg, val); \
2091	GEN2_WRITE_FOOTER; \
2092	}
2093
2094	#define __gen5_write(x) \
2095	static void \
2096	gen5_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2097	GEN2_WRITE_HEADER; \
2098	ilk_dummy_write(uncore); \
2099	__raw_uncore_write##x(uncore, reg, val); \
2100	GEN2_WRITE_FOOTER; \
2101	}
2102
2103	__gen5_write(8)
2104	__gen5_write(16)
2105	__gen5_write(32)
2106	__gen2_write(8)
2107	__gen2_write(16)
2108	__gen2_write(32)
2109
2110	#undef __gen5_write
2111	#undef __gen2_write
2112
2113	#undef GEN2_WRITE_FOOTER
2114	#undef GEN2_WRITE_HEADER
2115
2116	#define GEN6_WRITE_HEADER \
2117	u32 offset = i915_mmio_reg_offset(reg); \
2118	unsigned long irqflags; \
2119	bool unclaimed_reg_debug; \
2120	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
2121	assert_rpm_wakelock_held(uncore->rpm); \
2122	spin_lock_irqsave(&uncore->lock, irqflags); \
2123	unclaimed_reg_debug = unclaimed_reg_debug_header(uncore, reg, false)
2124
2125	#define GEN6_WRITE_FOOTER \
2126	if (unclaimed_reg_debug) \
2127	unclaimed_reg_debug_footer(uncore, reg, false); \
2128	spin_unlock_irqrestore(&uncore->lock, irqflags)
2129
2130	#define __gen6_write(x) \
2131	static void \
2132	gen6_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2133	GEN6_WRITE_HEADER; \
2134	if (NEEDS_FORCE_WAKE(offset)) \
2135	__gen6_gt_wait_for_fifo(uncore); \
2136	__raw_uncore_write##x(uncore, reg, val); \
2137	GEN6_WRITE_FOOTER; \
2138	}
2139	__gen6_write(8)
2140	__gen6_write(16)
2141	__gen6_write(32)
2142
2143	#define __gen_fwtable_write(x) \
2144	static void \
2145	fwtable_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2146	enum forcewake_domains fw_engine; \
2147	GEN6_WRITE_HEADER; \
2148	fw_engine = __fwtable_reg_write_fw_domains(uncore, offset); \
2149	if (fw_engine) \
2150	__force_wake_auto(uncore, fw_engine); \
2151	__raw_uncore_write##x(uncore, reg, val); \
2152	GEN6_WRITE_FOOTER; \
2153	}
2154
2155	static enum forcewake_domains
2156	fwtable_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg)
2157	{
2158	return __fwtable_reg_write_fw_domains(uncore, i915_mmio_reg_offset(reg));
2159	}
2160
2161	__gen_fwtable_write(8)
2162	__gen_fwtable_write(16)
2163	__gen_fwtable_write(32)
2164
2165	#undef __gen_fwtable_write
2166	#undef GEN6_WRITE_FOOTER
2167	#undef GEN6_WRITE_HEADER
2168
2169	#define __vgpu_write(x) \
2170	static void \
2171	vgpu_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2172	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
2173	__raw_uncore_write##x(uncore, reg, val); \
2174	}
2175	__vgpu_write(8)
2176	__vgpu_write(16)
2177	__vgpu_write(32)
2178
2179	#define ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, x) \
2180	do { \
2181	(uncore)->funcs.mmio_writeb = x##_write8; \
2182	(uncore)->funcs.mmio_writew = x##_write16; \
2183	(uncore)->funcs.mmio_writel = x##_write32; \
2184	} while (0)
2185
2186	#define ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x) \
2187	do { \
2188	(uncore)->funcs.mmio_readb = x##_read8; \
2189	(uncore)->funcs.mmio_readw = x##_read16; \
2190	(uncore)->funcs.mmio_readl = x##_read32; \
2191	(uncore)->funcs.mmio_readq = x##_read64; \
2192	} while (0)
2193
2194	#define ASSIGN_WRITE_MMIO_VFUNCS(uncore, x) \
2195	do { \
2196	ASSIGN_RAW_WRITE_MMIO_VFUNCS((uncore), x); \
2197	(uncore)->funcs.write_fw_domains = x##_reg_write_fw_domains; \
2198	} while (0)
2199
2200	#define ASSIGN_READ_MMIO_VFUNCS(uncore, x) \
2201	do { \
2202	ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x); \
2203	(uncore)->funcs.read_fw_domains = x##_reg_read_fw_domains; \
2204	} while (0)
2205
2206	static int __fw_domain_init(struct intel_uncore *uncore,
2207	enum forcewake_domain_id domain_id,
2208	i915_reg_t reg_set,
2209	i915_reg_t reg_ack)
2210	{
2211	struct intel_uncore_forcewake_domain *d;
2212
2213	GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT);
2214	GEM_BUG_ON(uncore->fw_domain[domain_id]);
2215
2216	if (i915_inject_probe_failure(uncore->i915))
2217	return -ENOMEM;
2218
2219	d = kzalloc(size: sizeof(*d), GFP_KERNEL);
2220	if (!d)
2221	return -ENOMEM;
2222
2223	drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_set));
2224	drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_ack));
2225
2226	d->uncore = uncore;
2227	d->wake_count = 0;
2228	d->reg_set = uncore->regs + i915_mmio_reg_offset(reg_set) + uncore->gsi_offset;
2229	d->reg_ack = uncore->regs + i915_mmio_reg_offset(reg_ack) + uncore->gsi_offset;
2230
2231	d->id = domain_id;
2232
2233	BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
2234	BUILD_BUG_ON(FORCEWAKE_GT != (1 << FW_DOMAIN_ID_GT));
2235	BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));
2236	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0));
2237	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1));
2238	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2));
2239	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3));
2240	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX4 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX4));
2241	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX5 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX5));
2242	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX6 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX6));
2243	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX7 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX7));
2244	BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0));
2245	BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1));
2246	BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX2));
2247	BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX3));
2248	BUILD_BUG_ON(FORCEWAKE_GSC != (1 << FW_DOMAIN_ID_GSC));
2249
2250	d->mask = BIT(domain_id);
2251
2252	hrtimer_init(timer: &d->timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
2253	d->timer.function = intel_uncore_fw_release_timer;
2254
2255	uncore->fw_domains |= BIT(domain_id);
2256
2257	fw_domain_reset(d);
2258
2259	uncore->fw_domain[domain_id] = d;
2260
2261	return 0;
2262	}
2263
2264	static void fw_domain_fini(struct intel_uncore *uncore,
2265	enum forcewake_domain_id domain_id)
2266	{
2267	struct intel_uncore_forcewake_domain *d;
2268
2269	GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT);
2270
2271	d = fetch_and_zero(&uncore->fw_domain[domain_id]);
2272	if (!d)
2273	return;
2274
2275	uncore->fw_domains &= ~BIT(domain_id);
2276	drm_WARN_ON(&uncore->i915->drm, d->wake_count);
2277	drm_WARN_ON(&uncore->i915->drm, hrtimer_cancel(&d->timer));
2278	kfree(objp: d);
2279	}
2280
2281	static void intel_uncore_fw_domains_fini(struct intel_uncore *uncore)
2282	{
2283	struct intel_uncore_forcewake_domain *d;
2284	int tmp;
2285
2286	for_each_fw_domain(d, uncore, tmp)
2287	fw_domain_fini(uncore, domain_id: d->id);
2288	}
2289
2290	static const struct intel_uncore_fw_get uncore_get_fallback = {
2291	.force_wake_get = fw_domains_get_with_fallback
2292	};
2293
2294	static const struct intel_uncore_fw_get uncore_get_normal = {
2295	.force_wake_get = fw_domains_get_normal,
2296	};
2297
2298	static const struct intel_uncore_fw_get uncore_get_thread_status = {
2299	.force_wake_get = fw_domains_get_with_thread_status
2300	};
2301
2302	static int intel_uncore_fw_domains_init(struct intel_uncore *uncore)
2303	{
2304	struct drm_i915_private *i915 = uncore->i915;
2305	int ret = 0;
2306
2307	GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
2308
2309	#define fw_domain_init(uncore__, id__, set__, ack__) \
2310	(ret ?: (ret = __fw_domain_init((uncore__), (id__), (set__), (ack__))))
2311
2312	if (GRAPHICS_VER(i915) >= 11) {
2313	intel_engine_mask_t emask;
2314	int i;
2315
2316	/* we'll prune the domains of missing engines later */
2317	emask = uncore->gt->info.engine_mask;
2318
2319	uncore->fw_get_funcs = &uncore_get_fallback;
2320	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2321	fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2322	FORCEWAKE_GT_GEN9,
2323	FORCEWAKE_ACK_GT_MTL);
2324	else
2325	fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2326	FORCEWAKE_GT_GEN9,
2327	FORCEWAKE_ACK_GT_GEN9);
2328
2329	if (RCS_MASK(uncore->gt) || CCS_MASK(uncore->gt))
2330	fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2331	FORCEWAKE_RENDER_GEN9,
2332	FORCEWAKE_ACK_RENDER_GEN9);
2333
2334	for (i = 0; i < I915_MAX_VCS; i++) {
2335	if (!__HAS_ENGINE(emask, _VCS(i)))
2336	continue;
2337
2338	fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VDBOX0 + i,
2339	FORCEWAKE_MEDIA_VDBOX_GEN11(i),
2340	FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i));
2341	}
2342	for (i = 0; i < I915_MAX_VECS; i++) {
2343	if (!__HAS_ENGINE(emask, _VECS(i)))
2344	continue;
2345
2346	fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VEBOX0 + i,
2347	FORCEWAKE_MEDIA_VEBOX_GEN11(i),
2348	FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i));
2349	}
2350
2351	if (uncore->gt->type == GT_MEDIA)
2352	fw_domain_init(uncore, FW_DOMAIN_ID_GSC,
2353	FORCEWAKE_REQ_GSC, FORCEWAKE_ACK_GSC);
2354	} else if (IS_GRAPHICS_VER(i915, 9, 10)) {
2355	uncore->fw_get_funcs = &uncore_get_fallback;
2356	fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2357	FORCEWAKE_RENDER_GEN9,
2358	FORCEWAKE_ACK_RENDER_GEN9);
2359	fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2360	FORCEWAKE_GT_GEN9,
2361	FORCEWAKE_ACK_GT_GEN9);
2362	fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA,
2363	FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
2364	} else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2365	uncore->fw_get_funcs = &uncore_get_normal;
2366	fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2367	FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
2368	fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA,
2369	FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
2370	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2371	uncore->fw_get_funcs = &uncore_get_thread_status;
2372	fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2373	FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
2374	} else if (IS_IVYBRIDGE(i915)) {
2375	u32 ecobus;
2376
2377	/* IVB configs may use multi-threaded forcewake */
2378
2379	/* A small trick here - if the bios hasn't configured
2380	* MT forcewake, and if the device is in RC6, then
2381	* force_wake_mt_get will not wake the device and the
2382	* ECOBUS read will return zero. Which will be
2383	* (correctly) interpreted by the test below as MT
2384	* forcewake being disabled.
2385	*/
2386	uncore->fw_get_funcs = &uncore_get_thread_status;
2387
2388	/* We need to init first for ECOBUS access and then
2389	* determine later if we want to reinit, in case of MT access is
2390	* not working. In this stage we don't know which flavour this
2391	* ivb is, so it is better to reset also the gen6 fw registers
2392	* before the ecobus check.
2393	*/
2394
2395	__raw_uncore_write32(uncore, FORCEWAKE, val: 0);
2396	__raw_posting_read(uncore, ECOBUS);
2397
2398	ret = __fw_domain_init(uncore, domain_id: FW_DOMAIN_ID_RENDER,
2399	FORCEWAKE_MT, FORCEWAKE_MT_ACK);
2400	if (ret)
2401	goto out;
2402
2403	spin_lock_irq(lock: &uncore->lock);
2404	fw_domains_get_with_thread_status(uncore, fw_domains: FORCEWAKE_RENDER);
2405	ecobus = __raw_uncore_read32(uncore, ECOBUS);
2406	fw_domains_put(uncore, fw_domains: FORCEWAKE_RENDER);
2407	spin_unlock_irq(lock: &uncore->lock);
2408
2409	if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
2410	drm_info(&i915->drm, "No MT forcewake available on Ivybridge, this can result in issues\n");
2411	drm_info(&i915->drm, "when using vblank-synced partial screen updates.\n");
2412	fw_domain_fini(uncore, domain_id: FW_DOMAIN_ID_RENDER);
2413	fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2414	FORCEWAKE, FORCEWAKE_ACK);
2415	}
2416	} else if (GRAPHICS_VER(i915) == 6) {
2417	uncore->fw_get_funcs = &uncore_get_thread_status;
2418	fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2419	FORCEWAKE, FORCEWAKE_ACK);
2420	}
2421
2422	#undef fw_domain_init
2423
2424	/* All future platforms are expected to require complex power gating */
2425	drm_WARN_ON(&i915->drm, !ret && uncore->fw_domains == 0);
2426
2427	out:
2428	if (ret)
2429	intel_uncore_fw_domains_fini(uncore);
2430
2431	return ret;
2432	}
2433
2434	#define ASSIGN_FW_DOMAINS_TABLE(uncore, d) \
2435	{ \
2436	(uncore)->fw_domains_table = \
2437	(struct intel_forcewake_range *)(d); \
2438	(uncore)->fw_domains_table_entries = ARRAY_SIZE((d)); \
2439	}
2440
2441	#define ASSIGN_SHADOW_TABLE(uncore, d) \
2442	{ \
2443	(uncore)->shadowed_reg_table = d; \
2444	(uncore)->shadowed_reg_table_entries = ARRAY_SIZE((d)); \
2445	}
2446
2447	static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
2448	unsigned long action, void *data)
2449	{
2450	struct intel_uncore *uncore = container_of(nb,
2451	struct intel_uncore, pmic_bus_access_nb);
2452
2453	switch (action) {
2454	case MBI_PMIC_BUS_ACCESS_BEGIN:
2455	/*
2456	* forcewake all now to make sure that we don't need to do a
2457	* forcewake later which on systems where this notifier gets
2458	* called requires the punit to access to the shared pmic i2c
2459	* bus, which will be busy after this notification, leading to:
2460	* "render: timed out waiting for forcewake ack request."
2461	* errors.
2462	*
2463	* The notifier is unregistered during intel_runtime_suspend(),
2464	* so it's ok to access the HW here without holding a RPM
2465	* wake reference -> disable wakeref asserts for the time of
2466	* the access.
2467	*/
2468	disable_rpm_wakeref_asserts(rpm: uncore->rpm);
2469	intel_uncore_forcewake_get(uncore, fw_domains: FORCEWAKE_ALL);
2470	enable_rpm_wakeref_asserts(rpm: uncore->rpm);
2471	break;
2472	case MBI_PMIC_BUS_ACCESS_END:
2473	intel_uncore_forcewake_put(uncore, fw_domains: FORCEWAKE_ALL);
2474	break;
2475	}
2476
2477	return NOTIFY_OK;
2478	}
2479
2480	static void uncore_unmap_mmio(struct drm_device *drm, void *regs)
2481	{
2482	iounmap(addr: (void __iomem *)regs);
2483	}
2484
2485	int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr)
2486	{
2487	struct drm_i915_private *i915 = uncore->i915;
2488	int mmio_size;
2489
2490	/*
2491	* Before gen4, the registers and the GTT are behind different BARs.
2492	* However, from gen4 onwards, the registers and the GTT are shared
2493	* in the same BAR, so we want to restrict this ioremap from
2494	* clobbering the GTT which we want ioremap_wc instead. Fortunately,
2495	* the register BAR remains the same size for all the earlier
2496	* generations up to Ironlake.
2497	* For dgfx chips register range is expanded to 4MB, and this larger
2498	* range is also used for integrated gpus beginning with Meteor Lake.
2499	*/
2500	if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2501	mmio_size = 4 * 1024 * 1024;
2502	else if (GRAPHICS_VER(i915) >= 5)
2503	mmio_size = 2 * 1024 * 1024;
2504	else
2505	mmio_size = 512 * 1024;
2506
2507	uncore->regs = ioremap(offset: phys_addr, size: mmio_size);
2508	if (uncore->regs == NULL) {
2509	drm_err(&i915->drm, "failed to map registers\n");
2510	return -EIO;
2511	}
2512
2513	return drmm_add_action_or_reset(&i915->drm, uncore_unmap_mmio,
2514	(void __force *)uncore->regs);
2515	}
2516
2517	void intel_uncore_init_early(struct intel_uncore *uncore,
2518	struct intel_gt *gt)
2519	{
2520	spin_lock_init(&uncore->lock);
2521	uncore->i915 = gt->i915;
2522	uncore->gt = gt;
2523	uncore->rpm = &gt->i915->runtime_pm;
2524	}
2525
2526	static void uncore_raw_init(struct intel_uncore *uncore)
2527	{
2528	GEM_BUG_ON(intel_uncore_has_forcewake(uncore));
2529
2530	if (intel_vgpu_active(i915: uncore->i915)) {
2531	ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, vgpu);
2532	ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, vgpu);
2533	} else if (GRAPHICS_VER(uncore->i915) == 5) {
2534	ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen5);
2535	ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen5);
2536	} else {
2537	ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen2);
2538	ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen2);
2539	}
2540	}
2541
2542	static int uncore_media_forcewake_init(struct intel_uncore *uncore)
2543	{
2544	struct drm_i915_private *i915 = uncore->i915;
2545
2546	if (MEDIA_VER(i915) >= 13) {
2547	ASSIGN_FW_DOMAINS_TABLE(uncore, __xelpmp_fw_ranges);
2548	ASSIGN_SHADOW_TABLE(uncore, xelpmp_shadowed_regs);
2549	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2550	} else {
2551	MISSING_CASE(MEDIA_VER(i915));
2552	return -ENODEV;
2553	}
2554
2555	return 0;
2556	}
2557
2558	static int uncore_forcewake_init(struct intel_uncore *uncore)
2559	{
2560	struct drm_i915_private *i915 = uncore->i915;
2561	int ret;
2562
2563	GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
2564
2565	ret = intel_uncore_fw_domains_init(uncore);
2566	if (ret)
2567	return ret;
2568	forcewake_early_sanitize(uncore, restore_forcewake: 0);
2569
2570	ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable);
2571
2572	if (uncore->gt->type == GT_MEDIA)
2573	return uncore_media_forcewake_init(uncore);
2574
2575	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
2576	ASSIGN_FW_DOMAINS_TABLE(uncore, __mtl_fw_ranges);
2577	ASSIGN_SHADOW_TABLE(uncore, mtl_shadowed_regs);
2578	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2579	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 60)) {
2580	ASSIGN_FW_DOMAINS_TABLE(uncore, __pvc_fw_ranges);
2581	ASSIGN_SHADOW_TABLE(uncore, pvc_shadowed_regs);
2582	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2583	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
2584	ASSIGN_FW_DOMAINS_TABLE(uncore, __dg2_fw_ranges);
2585	ASSIGN_SHADOW_TABLE(uncore, dg2_shadowed_regs);
2586	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2587	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
2588	ASSIGN_FW_DOMAINS_TABLE(uncore, __xehp_fw_ranges);
2589	ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs);
2590	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2591	} else if (GRAPHICS_VER(i915) >= 12) {
2592	ASSIGN_FW_DOMAINS_TABLE(uncore, __gen12_fw_ranges);
2593	ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs);
2594	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2595	} else if (GRAPHICS_VER(i915) == 11) {
2596	ASSIGN_FW_DOMAINS_TABLE(uncore, __gen11_fw_ranges);
2597	ASSIGN_SHADOW_TABLE(uncore, gen11_shadowed_regs);
2598	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2599	} else if (IS_GRAPHICS_VER(i915, 9, 10)) {
2600	ASSIGN_FW_DOMAINS_TABLE(uncore, __gen9_fw_ranges);
2601	ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2602	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2603	} else if (IS_CHERRYVIEW(i915)) {
2604	ASSIGN_FW_DOMAINS_TABLE(uncore, __chv_fw_ranges);
2605	ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2606	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2607	} else if (GRAPHICS_VER(i915) == 8) {
2608	ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges);
2609	ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2610	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2611	} else if (IS_VALLEYVIEW(i915)) {
2612	ASSIGN_FW_DOMAINS_TABLE(uncore, __vlv_fw_ranges);
2613	ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6);
2614	} else if (IS_GRAPHICS_VER(i915, 6, 7)) {
2615	ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges);
2616	ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6);
2617	}
2618
2619	uncore->pmic_bus_access_nb.notifier_call = i915_pmic_bus_access_notifier;
2620	iosf_mbi_register_pmic_bus_access_notifier(nb: &uncore->pmic_bus_access_nb);
2621
2622	return 0;
2623	}
2624
2625	static int sanity_check_mmio_access(struct intel_uncore *uncore)
2626	{
2627	struct drm_i915_private *i915 = uncore->i915;
2628
2629	if (GRAPHICS_VER(i915) < 8)
2630	return 0;
2631
2632	/*
2633	* Sanitycheck that MMIO access to the device is working properly. If
2634	* the CPU is unable to communcate with a PCI device, BAR reads will
2635	* return 0xFFFFFFFF. Let's make sure the device isn't in this state
2636	* before we start trying to access registers.
2637	*
2638	* We use the primary GT's forcewake register as our guinea pig since
2639	* it's been around since HSW and it's a masked register so the upper
2640	* 16 bits can never read back as 1's if device access is operating
2641	* properly.
2642	*
2643	* If MMIO isn't working, we'll wait up to 2 seconds to see if it
2644	* recovers, then give up.
2645	*/
2646	#define COND (__raw_uncore_read32(uncore, FORCEWAKE_MT) != ~0)
2647	if (wait_for(COND, 2000) == -ETIMEDOUT) {
2648	drm_err(&i915->drm, "Device is non-operational; MMIO access returns 0xFFFFFFFF!\n");
2649	return -EIO;
2650	}
2651
2652	return 0;
2653	}
2654
2655	int intel_uncore_init_mmio(struct intel_uncore *uncore)
2656	{
2657	struct drm_i915_private *i915 = uncore->i915;
2658	int ret;
2659
2660	ret = sanity_check_mmio_access(uncore);
2661	if (ret)
2662	return ret;
2663
2664	/*
2665	* The boot firmware initializes local memory and assesses its health.
2666	* If memory training fails, the punit will have been instructed to
2667	* keep the GT powered down; we won't be able to communicate with it
2668	* and we should not continue with driver initialization.
2669	*/
2670	if (IS_DGFX(i915) &&
2671	!(__raw_uncore_read32(uncore, GU_CNTL) & LMEM_INIT)) {
2672	drm_err(&i915->drm, "LMEM not initialized by firmware\n");
2673	return -ENODEV;
2674	}
2675
2676	if (GRAPHICS_VER(i915) > 5 && !intel_vgpu_active(i915))
2677	uncore->flags |= UNCORE_HAS_FORCEWAKE;
2678
2679	if (!intel_uncore_has_forcewake(uncore)) {
2680	uncore_raw_init(uncore);
2681	} else {
2682	ret = uncore_forcewake_init(uncore);
2683	if (ret)
2684	return ret;
2685	}
2686
2687	/* make sure fw funcs are set if and only if we have fw*/
2688	GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->fw_get_funcs);
2689	GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.read_fw_domains);
2690	GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.write_fw_domains);
2691
2692	if (HAS_FPGA_DBG_UNCLAIMED(i915))
2693	uncore->flags |= UNCORE_HAS_FPGA_DBG_UNCLAIMED;
2694
2695	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2696	uncore->flags |= UNCORE_HAS_DBG_UNCLAIMED;
2697
2698	if (IS_GRAPHICS_VER(i915, 6, 7))
2699	uncore->flags |= UNCORE_HAS_FIFO;
2700
2701	/* clear out unclaimed reg detection bit */
2702	if (intel_uncore_unclaimed_mmio(uncore))
2703	drm_dbg(&i915->drm, "unclaimed mmio detected on uncore init, clearing\n");
2704
2705	return 0;
2706	}
2707
2708	/*
2709	* We might have detected that some engines are fused off after we initialized
2710	* the forcewake domains. Prune them, to make sure they only reference existing
2711	* engines.
2712	*/
2713	void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore,
2714	struct intel_gt *gt)
2715	{
2716	enum forcewake_domains fw_domains = uncore->fw_domains;
2717	enum forcewake_domain_id domain_id;
2718	int i;
2719
2720	if (!intel_uncore_has_forcewake(uncore) || GRAPHICS_VER(uncore->i915) < 11)
2721	return;
2722
2723	for (i = 0; i < I915_MAX_VCS; i++) {
2724	domain_id = FW_DOMAIN_ID_MEDIA_VDBOX0 + i;
2725
2726	if (HAS_ENGINE(gt, _VCS(i)))
2727	continue;
2728
2729	/*
2730	* Starting with XeHP, the power well for an even-numbered
2731	* VDBOX is also used for shared units within the
2732	* media slice such as SFC. So even if the engine
2733	* itself is fused off, we still need to initialize
2734	* the forcewake domain if any of the other engines
2735	* in the same media slice are present.
2736	*/
2737	if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 50) && i % 2 == 0) {
2738	if ((i + 1 < I915_MAX_VCS) && HAS_ENGINE(gt, _VCS(i + 1)))
2739	continue;
2740
2741	if (HAS_ENGINE(gt, _VECS(i / 2)))
2742	continue;
2743	}
2744
2745	if (fw_domains & BIT(domain_id))
2746	fw_domain_fini(uncore, domain_id);
2747	}
2748
2749	for (i = 0; i < I915_MAX_VECS; i++) {
2750	domain_id = FW_DOMAIN_ID_MEDIA_VEBOX0 + i;
2751
2752	if (HAS_ENGINE(gt, _VECS(i)))
2753	continue;
2754
2755	if (fw_domains & BIT(domain_id))
2756	fw_domain_fini(uncore, domain_id);
2757	}
2758
2759	if ((fw_domains & BIT(FW_DOMAIN_ID_GSC)) && !HAS_ENGINE(gt, GSC0))
2760	fw_domain_fini(uncore, domain_id: FW_DOMAIN_ID_GSC);
2761	}
2762
2763	/*
2764	* The driver-initiated FLR is the highest level of reset that we can trigger
2765	* from within the driver. It is different from the PCI FLR in that it doesn't
2766	* fully reset the SGUnit and doesn't modify the PCI config space and therefore
2767	* it doesn't require a re-enumeration of the PCI BARs. However, the
2768	* driver-initiated FLR does still cause a reset of both GT and display and a
2769	* memory wipe of local and stolen memory, so recovery would require a full HW
2770	* re-init and saving/restoring (or re-populating) the wiped memory. Since we
2771	* perform the FLR as the very last action before releasing access to the HW
2772	* during the driver release flow, we don't attempt recovery at all, because
2773	* if/when a new instance of i915 is bound to the device it will do a full
2774	* re-init anyway.
2775	*/
2776	static void driver_initiated_flr(struct intel_uncore *uncore)
2777	{
2778	struct drm_i915_private *i915 = uncore->i915;
2779	const unsigned int flr_timeout_ms = 3000; /* specs recommend a 3s wait */
2780	int ret;
2781
2782	drm_dbg(&i915->drm, "Triggering Driver-FLR\n");
2783
2784	/*
2785	* Make sure any pending FLR requests have cleared by waiting for the
2786	* FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
2787	* to make sure it's not still set from a prior attempt (it's a write to
2788	* clear bit).
2789	* Note that we should never be in a situation where a previous attempt
2790	* is still pending (unless the HW is totally dead), but better to be
2791	* safe in case something unexpected happens
2792	*/
2793	ret = intel_wait_for_register_fw(uncore, GU_CNTL, DRIVERFLR, value: 0, timeout_ms: flr_timeout_ms);
2794	if (ret) {
2795	drm_err(&i915->drm,
2796	"Failed to wait for Driver-FLR bit to clear! %d\n",
2797	ret);
2798	return;
2799	}
2800	intel_uncore_write_fw(uncore, GU_DEBUG, DRIVERFLR_STATUS);
2801
2802	/* Trigger the actual Driver-FLR */
2803	intel_uncore_rmw_fw(uncore, GU_CNTL, clear: 0, DRIVERFLR);
2804
2805	/* Wait for hardware teardown to complete */
2806	ret = intel_wait_for_register_fw(uncore, GU_CNTL,
2807	DRIVERFLR, value: 0,
2808	timeout_ms: flr_timeout_ms);
2809	if (ret) {
2810	drm_err(&i915->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
2811	return;
2812	}
2813
2814	/* Wait for hardware/firmware re-init to complete */
2815	ret = intel_wait_for_register_fw(uncore, GU_DEBUG,
2816	DRIVERFLR_STATUS, DRIVERFLR_STATUS,
2817	timeout_ms: flr_timeout_ms);
2818	if (ret) {
2819	drm_err(&i915->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
2820	return;
2821	}
2822
2823	/* Clear sticky completion status */
2824	intel_uncore_write_fw(uncore, GU_DEBUG, DRIVERFLR_STATUS);
2825	}
2826
2827	/* Called via drm-managed action */
2828	void intel_uncore_fini_mmio(struct drm_device *dev, void *data)
2829	{
2830	struct intel_uncore *uncore = data;
2831
2832	if (intel_uncore_has_forcewake(uncore)) {
2833	iosf_mbi_punit_acquire();
2834	iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
2835	nb: &uncore->pmic_bus_access_nb);
2836	intel_uncore_forcewake_reset(uncore);
2837	intel_uncore_fw_domains_fini(uncore);
2838	iosf_mbi_punit_release();
2839	}
2840
2841	if (intel_uncore_needs_flr_on_fini(uncore))
2842	driver_initiated_flr(uncore);
2843	}
2844
2845	/**
2846	* __intel_wait_for_register_fw - wait until register matches expected state
2847	* @uncore: the struct intel_uncore
2848	* @reg: the register to read
2849	* @mask: mask to apply to register value
2850	* @value: expected value
2851	* @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
2852	* @slow_timeout_ms: slow timeout in millisecond
2853	* @out_value: optional placeholder to hold registry value
2854	*
2855	* This routine waits until the target register @reg contains the expected
2856	* @value after applying the @mask, i.e. it waits until ::
2857	*
2858	* (intel_uncore_read_fw(uncore, reg) & mask) == value
2859	*
2860	* Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
2861	* For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
2862	* must be not larger than 20,0000 microseconds.
2863	*
2864	* Note that this routine assumes the caller holds forcewake asserted, it is
2865	* not suitable for very long waits. See intel_wait_for_register() if you
2866	* wish to wait without holding forcewake for the duration (i.e. you expect
2867	* the wait to be slow).
2868	*
2869	* Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
2870	*/
2871	int __intel_wait_for_register_fw(struct intel_uncore *uncore,
2872	i915_reg_t reg,
2873	u32 mask,
2874	u32 value,
2875	unsigned int fast_timeout_us,
2876	unsigned int slow_timeout_ms,
2877	u32 *out_value)
2878	{
2879	u32 reg_value = 0;
2880	#define done (((reg_value = intel_uncore_read_fw(uncore, reg)) & mask) == value)
2881	int ret;
2882
2883	/* Catch any overuse of this function */
2884	might_sleep_if(slow_timeout_ms);
2885	GEM_BUG_ON(fast_timeout_us > 20000);
2886	GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms);
2887
2888	ret = -ETIMEDOUT;
2889	if (fast_timeout_us && fast_timeout_us <= 20000)
2890	ret = _wait_for_atomic(done, fast_timeout_us, 0);
2891	if (ret && slow_timeout_ms)
2892	ret = wait_for(done, slow_timeout_ms);
2893
2894	if (out_value)
2895	*out_value = reg_value;
2896
2897	return ret;
2898	#undef done
2899	}
2900
2901	/**
2902	* __intel_wait_for_register - wait until register matches expected state
2903	* @uncore: the struct intel_uncore
2904	* @reg: the register to read
2905	* @mask: mask to apply to register value
2906	* @value: expected value
2907	* @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
2908	* @slow_timeout_ms: slow timeout in millisecond
2909	* @out_value: optional placeholder to hold registry value
2910	*
2911	* This routine waits until the target register @reg contains the expected
2912	* @value after applying the @mask, i.e. it waits until ::
2913	*
2914	* (intel_uncore_read(uncore, reg) & mask) == value
2915	*
2916	* Otherwise, the wait will timeout after @timeout_ms milliseconds.
2917	*
2918	* Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
2919	*/
2920	int __intel_wait_for_register(struct intel_uncore *uncore,
2921	i915_reg_t reg,
2922	u32 mask,
2923	u32 value,
2924	unsigned int fast_timeout_us,
2925	unsigned int slow_timeout_ms,
2926	u32 *out_value)
2927	{
2928	unsigned fw =
2929	intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ);
2930	u32 reg_value;
2931	int ret;
2932
2933	might_sleep_if(slow_timeout_ms);
2934
2935	spin_lock_irq(lock: &uncore->lock);
2936	intel_uncore_forcewake_get__locked(uncore, fw_domains: fw);
2937
2938	ret = __intel_wait_for_register_fw(uncore,
2939	reg, mask, value,
2940	fast_timeout_us, slow_timeout_ms: 0, out_value: &reg_value);
2941
2942	intel_uncore_forcewake_put__locked(uncore, fw_domains: fw);
2943	spin_unlock_irq(lock: &uncore->lock);
2944
2945	if (ret && slow_timeout_ms)
2946	ret = __wait_for(reg_value = intel_uncore_read_notrace(uncore,
2947	reg),
2948	(reg_value & mask) == value,
2949	slow_timeout_ms * 1000, 10, 1000);
2950
2951	/* just trace the final value */
2952	trace_i915_reg_rw(write: false, reg, val: reg_value, len: sizeof(reg_value), trace: true);
2953
2954	if (out_value)
2955	*out_value = reg_value;
2956
2957	return ret;
2958	}
2959
2960	bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore)
2961	{
2962	bool ret;
2963
2964	if (!uncore->debug)
2965	return false;
2966
2967	spin_lock_irq(lock: &uncore->debug->lock);
2968	ret = check_for_unclaimed_mmio(uncore);
2969	spin_unlock_irq(lock: &uncore->debug->lock);
2970
2971	return ret;
2972	}
2973
2974	bool
2975	intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore)
2976	{
2977	bool ret = false;
2978
2979	if (drm_WARN_ON(&uncore->i915->drm, !uncore->debug))
2980	return false;
2981
2982	spin_lock_irq(lock: &uncore->debug->lock);
2983
2984	if (unlikely(uncore->debug->unclaimed_mmio_check <= 0))
2985	goto out;
2986
2987	if (unlikely(check_for_unclaimed_mmio(uncore))) {
2988	if (!uncore->i915->params.mmio_debug) {
2989	drm_dbg(&uncore->i915->drm,
2990	"Unclaimed register detected, "
2991	"enabling oneshot unclaimed register reporting. "
2992	"Please use i915.mmio_debug=N for more information.\n");
2993	uncore->i915->params.mmio_debug++;
2994	}
2995	uncore->debug->unclaimed_mmio_check--;
2996	ret = true;
2997	}
2998
2999	out:
3000	spin_unlock_irq(lock: &uncore->debug->lock);
3001
3002	return ret;
3003	}
3004
3005	/**
3006	* intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
3007	* a register
3008	* @uncore: pointer to struct intel_uncore
3009	* @reg: register in question
3010	* @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
3011	*
3012	* Returns a set of forcewake domains required to be taken with for example
3013	* intel_uncore_forcewake_get for the specified register to be accessible in the
3014	* specified mode (read, write or read/write) with raw mmio accessors.
3015	*
3016	* NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
3017	* callers to do FIFO management on their own or risk losing writes.
3018	*/
3019	enum forcewake_domains
3020	intel_uncore_forcewake_for_reg(struct intel_uncore *uncore,
3021	i915_reg_t reg, unsigned int op)
3022	{
3023	enum forcewake_domains fw_domains = 0;
3024
3025	drm_WARN_ON(&uncore->i915->drm, !op);
3026
3027	if (!intel_uncore_has_forcewake(uncore))
3028	return 0;
3029
3030	if (op & FW_REG_READ)
3031	fw_domains = uncore->funcs.read_fw_domains(uncore, reg);
3032
3033	if (op & FW_REG_WRITE)
3034	fw_domains |= uncore->funcs.write_fw_domains(uncore, reg);
3035
3036	drm_WARN_ON(&uncore->i915->drm, fw_domains & ~uncore->fw_domains);
3037
3038	return fw_domains;
3039	}
3040
3041	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
3042	#include "selftests/mock_uncore.c"
3043	#include "selftests/intel_uncore.c"
3044	#endif
3045