1	/*
2	* Copyright 2008 Advanced Micro Devices, Inc.
3	* Copyright 2008 Red Hat Inc.
4	* Copyright 2009 Jerome Glisse.
5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a
7	* copy of this software and associated documentation files (the "Software"),
8	* to deal in the Software without restriction, including without limitation
9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
10	* and/or sell copies of the Software, and to permit persons to whom the
11	* Software is furnished to do so, subject to the following conditions:
12	*
13	* The above copyright notice and this permission notice shall be included in
14	* all copies or substantial portions of the Software.
15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22	* OTHER DEALINGS IN THE SOFTWARE.
23	*
24	* Authors: Dave Airlie
25	* Alex Deucher
26	* Jerome Glisse
27	*/
28	#include <linux/power_supply.h>
29	#include <linux/kthread.h>
30	#include <linux/module.h>
31	#include <linux/console.h>
32	#include <linux/slab.h>
33	#include <linux/iommu.h>
34	#include <linux/pci.h>
35	#include <linux/pci-p2pdma.h>
36	#include <linux/apple-gmux.h>
37
38	#include <drm/drm_aperture.h>
39	#include <drm/drm_atomic_helper.h>
40	#include <drm/drm_crtc_helper.h>
41	#include <drm/drm_fb_helper.h>
42	#include <drm/drm_probe_helper.h>
43	#include <drm/amdgpu_drm.h>
44	#include <linux/device.h>
45	#include <linux/vgaarb.h>
46	#include <linux/vga_switcheroo.h>
47	#include <linux/efi.h>
48	#include "amdgpu.h"
49	#include "amdgpu_trace.h"
50	#include "amdgpu_i2c.h"
51	#include "atom.h"
52	#include "amdgpu_atombios.h"
53	#include "amdgpu_atomfirmware.h"
54	#include "amd_pcie.h"
55	#ifdef CONFIG_DRM_AMDGPU_SI
56	#include "si.h"
57	#endif
58	#ifdef CONFIG_DRM_AMDGPU_CIK
59	#include "cik.h"
60	#endif
61	#include "vi.h"
62	#include "soc15.h"
63	#include "nv.h"
64	#include "bif/bif_4_1_d.h"
65	#include <linux/firmware.h>
66	#include "amdgpu_vf_error.h"
67
68	#include "amdgpu_amdkfd.h"
69	#include "amdgpu_pm.h"
70
71	#include "amdgpu_xgmi.h"
72	#include "amdgpu_ras.h"
73	#include "amdgpu_pmu.h"
74	#include "amdgpu_fru_eeprom.h"
75	#include "amdgpu_reset.h"
76
77	#include <linux/suspend.h>
78	#include <drm/task_barrier.h>
79	#include <linux/pm_runtime.h>
80
81	#include <drm/drm_drv.h>
82
83	#if IS_ENABLED(CONFIG_X86)
84	#include <asm/intel-family.h>
85	#endif
86
87	MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
88	MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
89	MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
90	MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
91	MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
92	MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
93	MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
94
95	#define AMDGPU_RESUME_MS 2000
96	#define AMDGPU_MAX_RETRY_LIMIT 2
97	#define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
98
99	static const struct drm_driver amdgpu_kms_driver;
100
101	const char *amdgpu_asic_name[] = {
102	"TAHITI",
103	"PITCAIRN",
104	"VERDE",
105	"OLAND",
106	"HAINAN",
107	"BONAIRE",
108	"KAVERI",
109	"KABINI",
110	"HAWAII",
111	"MULLINS",
112	"TOPAZ",
113	"TONGA",
114	"FIJI",
115	"CARRIZO",
116	"STONEY",
117	"POLARIS10",
118	"POLARIS11",
119	"POLARIS12",
120	"VEGAM",
121	"VEGA10",
122	"VEGA12",
123	"VEGA20",
124	"RAVEN",
125	"ARCTURUS",
126	"RENOIR",
127	"ALDEBARAN",
128	"NAVI10",
129	"CYAN_SKILLFISH",
130	"NAVI14",
131	"NAVI12",
132	"SIENNA_CICHLID",
133	"NAVY_FLOUNDER",
134	"VANGOGH",
135	"DIMGREY_CAVEFISH",
136	"BEIGE_GOBY",
137	"YELLOW_CARP",
138	"IP DISCOVERY",
139	"LAST",
140	};
141
142	/**
143	* DOC: pcie_replay_count
144	*
145	* The amdgpu driver provides a sysfs API for reporting the total number
146	* of PCIe replays (NAKs)
147	* The file pcie_replay_count is used for this and returns the total
148	* number of replays as a sum of the NAKs generated and NAKs received
149	*/
150
151	static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
152	struct device_attribute *attr, char *buf)
153	{
154	struct drm_device *ddev = dev_get_drvdata(dev);
155	struct amdgpu_device *adev = drm_to_adev(ddev);
156	uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
157
158	return sysfs_emit(buf, fmt: "%llu\n", cnt);
159	}
160
161	static DEVICE_ATTR(pcie_replay_count, 0444,
162	amdgpu_device_get_pcie_replay_count, NULL);
163
164	/**
165	* DOC: board_info
166	*
167	* The amdgpu driver provides a sysfs API for giving board related information.
168	* It provides the form factor information in the format
169	*
170	* type : form factor
171	*
172	* Possible form factor values
173	*
174	* - "cem" - PCIE CEM card
175	* - "oam" - Open Compute Accelerator Module
176	* - "unknown" - Not known
177	*
178	*/
179
180	static ssize_t amdgpu_device_get_board_info(struct device *dev,
181	struct device_attribute *attr,
182	char *buf)
183	{
184	struct drm_device *ddev = dev_get_drvdata(dev);
185	struct amdgpu_device *adev = drm_to_adev(ddev);
186	enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM;
187	const char *pkg;
188
189	if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)
190	pkg_type = adev->smuio.funcs->get_pkg_type(adev);
191
192	switch (pkg_type) {
193	case AMDGPU_PKG_TYPE_CEM:
194	pkg = "cem";
195	break;
196	case AMDGPU_PKG_TYPE_OAM:
197	pkg = "oam";
198	break;
199	default:
200	pkg = "unknown";
201	break;
202	}
203
204	return sysfs_emit(buf, fmt: "%s : %s\n", "type", pkg);
205	}
206
207	static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL);
208
209	static struct attribute *amdgpu_board_attrs[] = {
210	&dev_attr_board_info.attr,
211	NULL,
212	};
213
214	static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj,
215	struct attribute *attr, int n)
216	{
217	struct device *dev = kobj_to_dev(kobj);
218	struct drm_device *ddev = dev_get_drvdata(dev);
219	struct amdgpu_device *adev = drm_to_adev(ddev);
220
221	if (adev->flags & AMD_IS_APU)
222	return 0;
223
224	return attr->mode;
225	}
226
227	static const struct attribute_group amdgpu_board_attrs_group = {
228	.attrs = amdgpu_board_attrs,
229	.is_visible = amdgpu_board_attrs_is_visible
230	};
231
232	static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
233
234
235	/**
236	* amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
237	*
238	* @dev: drm_device pointer
239	*
240	* Returns true if the device is a dGPU with ATPX power control,
241	* otherwise return false.
242	*/
243	bool amdgpu_device_supports_px(struct drm_device *dev)
244	{
245	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
246
247	if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
248	return true;
249	return false;
250	}
251
252	/**
253	* amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
254	*
255	* @dev: drm_device pointer
256	*
257	* Returns true if the device is a dGPU with ACPI power control,
258	* otherwise return false.
259	*/
260	bool amdgpu_device_supports_boco(struct drm_device *dev)
261	{
262	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
263
264	if (adev->has_pr3 ||
265	((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
266	return true;
267	return false;
268	}
269
270	/**
271	* amdgpu_device_supports_baco - Does the device support BACO
272	*
273	* @dev: drm_device pointer
274	*
275	* Returns true if the device supporte BACO,
276	* otherwise return false.
277	*/
278	bool amdgpu_device_supports_baco(struct drm_device *dev)
279	{
280	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
281
282	return amdgpu_asic_supports_baco(adev);
283	}
284
285	/**
286	* amdgpu_device_supports_smart_shift - Is the device dGPU with
287	* smart shift support
288	*
289	* @dev: drm_device pointer
290	*
291	* Returns true if the device is a dGPU with Smart Shift support,
292	* otherwise returns false.
293	*/
294	bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
295	{
296	return (amdgpu_device_supports_boco(dev) &&
297	amdgpu_acpi_is_power_shift_control_supported());
298	}
299
300	/*
301	* VRAM access helper functions
302	*/
303
304	/**
305	* amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
306	*
307	* @adev: amdgpu_device pointer
308	* @pos: offset of the buffer in vram
309	* @buf: virtual address of the buffer in system memory
310	* @size: read/write size, sizeof(@buf) must > @size
311	* @write: true - write to vram, otherwise - read from vram
312	*/
313	void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
314	void *buf, size_t size, bool write)
315	{
316	unsigned long flags;
317	uint32_t hi = ~0, tmp = 0;
318	uint32_t *data = buf;
319	uint64_t last;
320	int idx;
321
322	if (!drm_dev_enter(dev: adev_to_drm(adev), idx: &idx))
323	return;
324
325	BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
326
327	spin_lock_irqsave(&adev->mmio_idx_lock, flags);
328	for (last = pos + size; pos < last; pos += 4) {
329	tmp = pos >> 31;
330
331	WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
332	if (tmp != hi) {
333	WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
334	hi = tmp;
335	}
336	if (write)
337	WREG32_NO_KIQ(mmMM_DATA, *data++);
338	else
339	*data++ = RREG32_NO_KIQ(mmMM_DATA);
340	}
341
342	spin_unlock_irqrestore(lock: &adev->mmio_idx_lock, flags);
343	drm_dev_exit(idx);
344	}
345
346	/**
347	* amdgpu_device_aper_access - access vram by vram aperature
348	*
349	* @adev: amdgpu_device pointer
350	* @pos: offset of the buffer in vram
351	* @buf: virtual address of the buffer in system memory
352	* @size: read/write size, sizeof(@buf) must > @size
353	* @write: true - write to vram, otherwise - read from vram
354	*
355	* The return value means how many bytes have been transferred.
356	*/
357	size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
358	void *buf, size_t size, bool write)
359	{
360	#ifdef CONFIG_64BIT
361	void __iomem *addr;
362	size_t count = 0;
363	uint64_t last;
364
365	if (!adev->mman.aper_base_kaddr)
366	return 0;
367
368	last = min(pos + size, adev->gmc.visible_vram_size);
369	if (last > pos) {
370	addr = adev->mman.aper_base_kaddr + pos;
371	count = last - pos;
372
373	if (write) {
374	memcpy_toio(addr, buf, count);
375	/* Make sure HDP write cache flush happens without any reordering
376	* after the system memory contents are sent over PCIe device
377	*/
378	mb();
379	amdgpu_device_flush_hdp(adev, NULL);
380	} else {
381	amdgpu_device_invalidate_hdp(adev, NULL);
382	/* Make sure HDP read cache is invalidated before issuing a read
383	* to the PCIe device
384	*/
385	mb();
386	memcpy_fromio(buf, addr, count);
387	}
388
389	}
390
391	return count;
392	#else
393	return 0;
394	#endif
395	}
396
397	/**
398	* amdgpu_device_vram_access - read/write a buffer in vram
399	*
400	* @adev: amdgpu_device pointer
401	* @pos: offset of the buffer in vram
402	* @buf: virtual address of the buffer in system memory
403	* @size: read/write size, sizeof(@buf) must > @size
404	* @write: true - write to vram, otherwise - read from vram
405	*/
406	void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
407	void *buf, size_t size, bool write)
408	{
409	size_t count;
410
411	/* try to using vram apreature to access vram first */
412	count = amdgpu_device_aper_access(adev, pos, buf, size, write);
413	size -= count;
414	if (size) {
415	/* using MM to access rest vram */
416	pos += count;
417	buf += count;
418	amdgpu_device_mm_access(adev, pos, buf, size, write);
419	}
420	}
421
422	/*
423	* register access helper functions.
424	*/
425
426	/* Check if hw access should be skipped because of hotplug or device error */
427	bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
428	{
429	if (adev->no_hw_access)
430	return true;
431
432	#ifdef CONFIG_LOCKDEP
433	/*
434	* This is a bit complicated to understand, so worth a comment. What we assert
435	* here is that the GPU reset is not running on another thread in parallel.
436	*
437	* For this we trylock the read side of the reset semaphore, if that succeeds
438	* we know that the reset is not running in paralell.
439	*
440	* If the trylock fails we assert that we are either already holding the read
441	* side of the lock or are the reset thread itself and hold the write side of
442	* the lock.
443	*/
444	if (in_task()) {
445	if (down_read_trylock(sem: &adev->reset_domain->sem))
446	up_read(sem: &adev->reset_domain->sem);
447	else
448	lockdep_assert_held(&adev->reset_domain->sem);
449	}
450	#endif
451	return false;
452	}
453
454	/**
455	* amdgpu_device_rreg - read a memory mapped IO or indirect register
456	*
457	* @adev: amdgpu_device pointer
458	* @reg: dword aligned register offset
459	* @acc_flags: access flags which require special behavior
460	*
461	* Returns the 32 bit value from the offset specified.
462	*/
463	uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
464	uint32_t reg, uint32_t acc_flags)
465	{
466	uint32_t ret;
467
468	if (amdgpu_device_skip_hw_access(adev))
469	return 0;
470
471	if ((reg * 4) < adev->rmmio_size) {
472	if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
473	amdgpu_sriov_runtime(adev) &&
474	down_read_trylock(sem: &adev->reset_domain->sem)) {
475	ret = amdgpu_kiq_rreg(adev, reg);
476	up_read(sem: &adev->reset_domain->sem);
477	} else {
478	ret = readl(addr: ((void __iomem *)adev->rmmio) + (reg * 4));
479	}
480	} else {
481	ret = adev->pcie_rreg(adev, reg * 4);
482	}
483
484	trace_amdgpu_device_rreg(did: adev->pdev->device, reg, value: ret);
485
486	return ret;
487	}
488
489	/*
490	* MMIO register read with bytes helper functions
491	* @offset:bytes offset from MMIO start
492	*/
493
494	/**
495	* amdgpu_mm_rreg8 - read a memory mapped IO register
496	*
497	* @adev: amdgpu_device pointer
498	* @offset: byte aligned register offset
499	*
500	* Returns the 8 bit value from the offset specified.
501	*/
502	uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
503	{
504	if (amdgpu_device_skip_hw_access(adev))
505	return 0;
506
507	if (offset < adev->rmmio_size)
508	return (readb(addr: adev->rmmio + offset));
509	BUG();
510	}
511
512	/*
513	* MMIO register write with bytes helper functions
514	* @offset:bytes offset from MMIO start
515	* @value: the value want to be written to the register
516	*/
517
518	/**
519	* amdgpu_mm_wreg8 - read a memory mapped IO register
520	*
521	* @adev: amdgpu_device pointer
522	* @offset: byte aligned register offset
523	* @value: 8 bit value to write
524	*
525	* Writes the value specified to the offset specified.
526	*/
527	void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
528	{
529	if (amdgpu_device_skip_hw_access(adev))
530	return;
531
532	if (offset < adev->rmmio_size)
533	writeb(val: value, addr: adev->rmmio + offset);
534	else
535	BUG();
536	}
537
538	/**
539	* amdgpu_device_wreg - write to a memory mapped IO or indirect register
540	*
541	* @adev: amdgpu_device pointer
542	* @reg: dword aligned register offset
543	* @v: 32 bit value to write to the register
544	* @acc_flags: access flags which require special behavior
545	*
546	* Writes the value specified to the offset specified.
547	*/
548	void amdgpu_device_wreg(struct amdgpu_device *adev,
549	uint32_t reg, uint32_t v,
550	uint32_t acc_flags)
551	{
552	if (amdgpu_device_skip_hw_access(adev))
553	return;
554
555	if ((reg * 4) < adev->rmmio_size) {
556	if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
557	amdgpu_sriov_runtime(adev) &&
558	down_read_trylock(sem: &adev->reset_domain->sem)) {
559	amdgpu_kiq_wreg(adev, reg, v);
560	up_read(sem: &adev->reset_domain->sem);
561	} else {
562	writel(val: v, addr: ((void __iomem *)adev->rmmio) + (reg * 4));
563	}
564	} else {
565	adev->pcie_wreg(adev, reg * 4, v);
566	}
567
568	trace_amdgpu_device_wreg(did: adev->pdev->device, reg, value: v);
569	}
570
571	/**
572	* amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range
573	*
574	* @adev: amdgpu_device pointer
575	* @reg: mmio/rlc register
576	* @v: value to write
577	* @xcc_id: xcc accelerated compute core id
578	*
579	* this function is invoked only for the debugfs register access
580	*/
581	void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
582	uint32_t reg, uint32_t v,
583	uint32_t xcc_id)
584	{
585	if (amdgpu_device_skip_hw_access(adev))
586	return;
587
588	if (amdgpu_sriov_fullaccess(adev) &&
589	adev->gfx.rlc.funcs &&
590	adev->gfx.rlc.funcs->is_rlcg_access_range) {
591	if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
592	return amdgpu_sriov_wreg(adev, offset: reg, value: v, acc_flags: 0, hwip: 0, xcc_id);
593	} else if ((reg * 4) >= adev->rmmio_size) {
594	adev->pcie_wreg(adev, reg * 4, v);
595	} else {
596	writel(val: v, addr: ((void __iomem *)adev->rmmio) + (reg * 4));
597	}
598	}
599
600	/**
601	* amdgpu_device_indirect_rreg - read an indirect register
602	*
603	* @adev: amdgpu_device pointer
604	* @reg_addr: indirect register address to read from
605	*
606	* Returns the value of indirect register @reg_addr
607	*/
608	u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
609	u32 reg_addr)
610	{
611	unsigned long flags, pcie_index, pcie_data;
612	void __iomem *pcie_index_offset;
613	void __iomem *pcie_data_offset;
614	u32 r;
615
616	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
617	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
618
619	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
620	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
621	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
622
623	writel(val: reg_addr, addr: pcie_index_offset);
624	readl(addr: pcie_index_offset);
625	r = readl(addr: pcie_data_offset);
626	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
627
628	return r;
629	}
630
631	u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
632	u64 reg_addr)
633	{
634	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
635	u32 r;
636	void __iomem *pcie_index_offset;
637	void __iomem *pcie_index_hi_offset;
638	void __iomem *pcie_data_offset;
639
640	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
641	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
642	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
643	pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
644	else
645	pcie_index_hi = 0;
646
647	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
648	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
649	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
650	if (pcie_index_hi != 0)
651	pcie_index_hi_offset = (void __iomem *)adev->rmmio +
652	pcie_index_hi * 4;
653
654	writel(val: reg_addr, addr: pcie_index_offset);
655	readl(addr: pcie_index_offset);
656	if (pcie_index_hi != 0) {
657	writel(val: (reg_addr >> 32) & 0xff, addr: pcie_index_hi_offset);
658	readl(addr: pcie_index_hi_offset);
659	}
660	r = readl(addr: pcie_data_offset);
661
662	/* clear the high bits */
663	if (pcie_index_hi != 0) {
664	writel(val: 0, addr: pcie_index_hi_offset);
665	readl(addr: pcie_index_hi_offset);
666	}
667
668	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
669
670	return r;
671	}
672
673	/**
674	* amdgpu_device_indirect_rreg64 - read a 64bits indirect register
675	*
676	* @adev: amdgpu_device pointer
677	* @reg_addr: indirect register address to read from
678	*
679	* Returns the value of indirect register @reg_addr
680	*/
681	u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
682	u32 reg_addr)
683	{
684	unsigned long flags, pcie_index, pcie_data;
685	void __iomem *pcie_index_offset;
686	void __iomem *pcie_data_offset;
687	u64 r;
688
689	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
690	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
691
692	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
693	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
694	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
695
696	/* read low 32 bits */
697	writel(val: reg_addr, addr: pcie_index_offset);
698	readl(addr: pcie_index_offset);
699	r = readl(addr: pcie_data_offset);
700	/* read high 32 bits */
701	writel(val: reg_addr + 4, addr: pcie_index_offset);
702	readl(addr: pcie_index_offset);
703	r |= ((u64)readl(addr: pcie_data_offset) << 32);
704	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
705
706	return r;
707	}
708
709	u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,
710	u64 reg_addr)
711	{
712	unsigned long flags, pcie_index, pcie_data;
713	unsigned long pcie_index_hi = 0;
714	void __iomem *pcie_index_offset;
715	void __iomem *pcie_index_hi_offset;
716	void __iomem *pcie_data_offset;
717	u64 r;
718
719	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
720	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
721	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
722	pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
723
724	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
725	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
726	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
727	if (pcie_index_hi != 0)
728	pcie_index_hi_offset = (void __iomem *)adev->rmmio +
729	pcie_index_hi * 4;
730
731	/* read low 32 bits */
732	writel(val: reg_addr, addr: pcie_index_offset);
733	readl(addr: pcie_index_offset);
734	if (pcie_index_hi != 0) {
735	writel(val: (reg_addr >> 32) & 0xff, addr: pcie_index_hi_offset);
736	readl(addr: pcie_index_hi_offset);
737	}
738	r = readl(addr: pcie_data_offset);
739	/* read high 32 bits */
740	writel(val: reg_addr + 4, addr: pcie_index_offset);
741	readl(addr: pcie_index_offset);
742	if (pcie_index_hi != 0) {
743	writel(val: (reg_addr >> 32) & 0xff, addr: pcie_index_hi_offset);
744	readl(addr: pcie_index_hi_offset);
745	}
746	r |= ((u64)readl(addr: pcie_data_offset) << 32);
747
748	/* clear the high bits */
749	if (pcie_index_hi != 0) {
750	writel(val: 0, addr: pcie_index_hi_offset);
751	readl(addr: pcie_index_hi_offset);
752	}
753
754	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
755
756	return r;
757	}
758
759	/**
760	* amdgpu_device_indirect_wreg - write an indirect register address
761	*
762	* @adev: amdgpu_device pointer
763	* @reg_addr: indirect register offset
764	* @reg_data: indirect register data
765	*
766	*/
767	void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
768	u32 reg_addr, u32 reg_data)
769	{
770	unsigned long flags, pcie_index, pcie_data;
771	void __iomem *pcie_index_offset;
772	void __iomem *pcie_data_offset;
773
774	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
775	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
776
777	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
778	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
779	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
780
781	writel(val: reg_addr, addr: pcie_index_offset);
782	readl(addr: pcie_index_offset);
783	writel(val: reg_data, addr: pcie_data_offset);
784	readl(addr: pcie_data_offset);
785	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
786	}
787
788	void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
789	u64 reg_addr, u32 reg_data)
790	{
791	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
792	void __iomem *pcie_index_offset;
793	void __iomem *pcie_index_hi_offset;
794	void __iomem *pcie_data_offset;
795
796	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
797	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
798	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
799	pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
800	else
801	pcie_index_hi = 0;
802
803	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
804	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
805	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
806	if (pcie_index_hi != 0)
807	pcie_index_hi_offset = (void __iomem *)adev->rmmio +
808	pcie_index_hi * 4;
809
810	writel(val: reg_addr, addr: pcie_index_offset);
811	readl(addr: pcie_index_offset);
812	if (pcie_index_hi != 0) {
813	writel(val: (reg_addr >> 32) & 0xff, addr: pcie_index_hi_offset);
814	readl(addr: pcie_index_hi_offset);
815	}
816	writel(val: reg_data, addr: pcie_data_offset);
817	readl(addr: pcie_data_offset);
818
819	/* clear the high bits */
820	if (pcie_index_hi != 0) {
821	writel(val: 0, addr: pcie_index_hi_offset);
822	readl(addr: pcie_index_hi_offset);
823	}
824
825	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
826	}
827
828	/**
829	* amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
830	*
831	* @adev: amdgpu_device pointer
832	* @reg_addr: indirect register offset
833	* @reg_data: indirect register data
834	*
835	*/
836	void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
837	u32 reg_addr, u64 reg_data)
838	{
839	unsigned long flags, pcie_index, pcie_data;
840	void __iomem *pcie_index_offset;
841	void __iomem *pcie_data_offset;
842
843	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
844	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
845
846	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
847	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
848	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
849
850	/* write low 32 bits */
851	writel(val: reg_addr, addr: pcie_index_offset);
852	readl(addr: pcie_index_offset);
853	writel(val: (u32)(reg_data & 0xffffffffULL), addr: pcie_data_offset);
854	readl(addr: pcie_data_offset);
855	/* write high 32 bits */
856	writel(val: reg_addr + 4, addr: pcie_index_offset);
857	readl(addr: pcie_index_offset);
858	writel(val: (u32)(reg_data >> 32), addr: pcie_data_offset);
859	readl(addr: pcie_data_offset);
860	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
861	}
862
863	void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,
864	u64 reg_addr, u64 reg_data)
865	{
866	unsigned long flags, pcie_index, pcie_data;
867	unsigned long pcie_index_hi = 0;
868	void __iomem *pcie_index_offset;
869	void __iomem *pcie_index_hi_offset;
870	void __iomem *pcie_data_offset;
871
872	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
873	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
874	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
875	pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
876
877	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
878	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
879	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
880	if (pcie_index_hi != 0)
881	pcie_index_hi_offset = (void __iomem *)adev->rmmio +
882	pcie_index_hi * 4;
883
884	/* write low 32 bits */
885	writel(val: reg_addr, addr: pcie_index_offset);
886	readl(addr: pcie_index_offset);
887	if (pcie_index_hi != 0) {
888	writel(val: (reg_addr >> 32) & 0xff, addr: pcie_index_hi_offset);
889	readl(addr: pcie_index_hi_offset);
890	}
891	writel(val: (u32)(reg_data & 0xffffffffULL), addr: pcie_data_offset);
892	readl(addr: pcie_data_offset);
893	/* write high 32 bits */
894	writel(val: reg_addr + 4, addr: pcie_index_offset);
895	readl(addr: pcie_index_offset);
896	if (pcie_index_hi != 0) {
897	writel(val: (reg_addr >> 32) & 0xff, addr: pcie_index_hi_offset);
898	readl(addr: pcie_index_hi_offset);
899	}
900	writel(val: (u32)(reg_data >> 32), addr: pcie_data_offset);
901	readl(addr: pcie_data_offset);
902
903	/* clear the high bits */
904	if (pcie_index_hi != 0) {
905	writel(val: 0, addr: pcie_index_hi_offset);
906	readl(addr: pcie_index_hi_offset);
907	}
908
909	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
910	}
911
912	/**
913	* amdgpu_device_get_rev_id - query device rev_id
914	*
915	* @adev: amdgpu_device pointer
916	*
917	* Return device rev_id
918	*/
919	u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
920	{
921	return adev->nbio.funcs->get_rev_id(adev);
922	}
923
924	/**
925	* amdgpu_invalid_rreg - dummy reg read function
926	*
927	* @adev: amdgpu_device pointer
928	* @reg: offset of register
929	*
930	* Dummy register read function. Used for register blocks
931	* that certain asics don't have (all asics).
932	* Returns the value in the register.
933	*/
934	static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
935	{
936	DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
937	BUG();
938	return 0;
939	}
940
941	static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)
942	{
943	DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
944	BUG();
945	return 0;
946	}
947
948	/**
949	* amdgpu_invalid_wreg - dummy reg write function
950	*
951	* @adev: amdgpu_device pointer
952	* @reg: offset of register
953	* @v: value to write to the register
954	*
955	* Dummy register read function. Used for register blocks
956	* that certain asics don't have (all asics).
957	*/
958	static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
959	{
960	DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
961	reg, v);
962	BUG();
963	}
964
965	static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)
966	{
967	DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n",
968	reg, v);
969	BUG();
970	}
971
972	/**
973	* amdgpu_invalid_rreg64 - dummy 64 bit reg read function
974	*
975	* @adev: amdgpu_device pointer
976	* @reg: offset of register
977	*
978	* Dummy register read function. Used for register blocks
979	* that certain asics don't have (all asics).
980	* Returns the value in the register.
981	*/
982	static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
983	{
984	DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
985	BUG();
986	return 0;
987	}
988
989	static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)
990	{
991	DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
992	BUG();
993	return 0;
994	}
995
996	/**
997	* amdgpu_invalid_wreg64 - dummy reg write function
998	*
999	* @adev: amdgpu_device pointer
1000	* @reg: offset of register
1001	* @v: value to write to the register
1002	*
1003	* Dummy register read function. Used for register blocks
1004	* that certain asics don't have (all asics).
1005	*/
1006	static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
1007	{
1008	DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
1009	reg, v);
1010	BUG();
1011	}
1012
1013	static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)
1014	{
1015	DRM_ERROR("Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n",
1016	reg, v);
1017	BUG();
1018	}
1019
1020	/**
1021	* amdgpu_block_invalid_rreg - dummy reg read function
1022	*
1023	* @adev: amdgpu_device pointer
1024	* @block: offset of instance
1025	* @reg: offset of register
1026	*
1027	* Dummy register read function. Used for register blocks
1028	* that certain asics don't have (all asics).
1029	* Returns the value in the register.
1030	*/
1031	static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
1032	uint32_t block, uint32_t reg)
1033	{
1034	DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
1035	reg, block);
1036	BUG();
1037	return 0;
1038	}
1039
1040	/**
1041	* amdgpu_block_invalid_wreg - dummy reg write function
1042	*
1043	* @adev: amdgpu_device pointer
1044	* @block: offset of instance
1045	* @reg: offset of register
1046	* @v: value to write to the register
1047	*
1048	* Dummy register read function. Used for register blocks
1049	* that certain asics don't have (all asics).
1050	*/
1051	static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
1052	uint32_t block,
1053	uint32_t reg, uint32_t v)
1054	{
1055	DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
1056	reg, block, v);
1057	BUG();
1058	}
1059
1060	/**
1061	* amdgpu_device_asic_init - Wrapper for atom asic_init
1062	*
1063	* @adev: amdgpu_device pointer
1064	*
1065	* Does any asic specific work and then calls atom asic init.
1066	*/
1067	static int amdgpu_device_asic_init(struct amdgpu_device *adev)
1068	{
1069	int ret;
1070
1071	amdgpu_asic_pre_asic_init(adev);
1072
1073	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3) ||
1074	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >= IP_VERSION(11, 0, 0)) {
1075	amdgpu_psp_wait_for_bootloader(adev);
1076	ret = amdgpu_atomfirmware_asic_init(adev, fb_reset: true);
1077	/* TODO: check the return val and stop device initialization if boot fails */
1078	amdgpu_psp_query_boot_status(adev);
1079	return ret;
1080	} else {
1081	return amdgpu_atom_asic_init(ctx: adev->mode_info.atom_context);
1082	}
1083
1084	return 0;
1085	}
1086
1087	/**
1088	* amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1089	*
1090	* @adev: amdgpu_device pointer
1091	*
1092	* Allocates a scratch page of VRAM for use by various things in the
1093	* driver.
1094	*/
1095	static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
1096	{
1097	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1098	AMDGPU_GEM_DOMAIN_VRAM |
1099	AMDGPU_GEM_DOMAIN_GTT,
1100	bo_ptr: &adev->mem_scratch.robj,
1101	gpu_addr: &adev->mem_scratch.gpu_addr,
1102	cpu_addr: (void **)&adev->mem_scratch.ptr);
1103	}
1104
1105	/**
1106	* amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1107	*
1108	* @adev: amdgpu_device pointer
1109	*
1110	* Frees the VRAM scratch page.
1111	*/
1112	static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1113	{
1114	amdgpu_bo_free_kernel(bo: &adev->mem_scratch.robj, NULL, NULL);
1115	}
1116
1117	/**
1118	* amdgpu_device_program_register_sequence - program an array of registers.
1119	*
1120	* @adev: amdgpu_device pointer
1121	* @registers: pointer to the register array
1122	* @array_size: size of the register array
1123	*
1124	* Programs an array or registers with and or masks.
1125	* This is a helper for setting golden registers.
1126	*/
1127	void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1128	const u32 *registers,
1129	const u32 array_size)
1130	{
1131	u32 tmp, reg, and_mask, or_mask;
1132	int i;
1133
1134	if (array_size % 3)
1135	return;
1136
1137	for (i = 0; i < array_size; i += 3) {
1138	reg = registers[i + 0];
1139	and_mask = registers[i + 1];
1140	or_mask = registers[i + 2];
1141
1142	if (and_mask == 0xffffffff) {
1143	tmp = or_mask;
1144	} else {
1145	tmp = RREG32(reg);
1146	tmp &= ~and_mask;
1147	if (adev->family >= AMDGPU_FAMILY_AI)
1148	tmp |= (or_mask & and_mask);
1149	else
1150	tmp |= or_mask;
1151	}
1152	WREG32(reg, tmp);
1153	}
1154	}
1155
1156	/**
1157	* amdgpu_device_pci_config_reset - reset the GPU
1158	*
1159	* @adev: amdgpu_device pointer
1160	*
1161	* Resets the GPU using the pci config reset sequence.
1162	* Only applicable to asics prior to vega10.
1163	*/
1164	void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1165	{
1166	pci_write_config_dword(dev: adev->pdev, where: 0x7c, AMDGPU_ASIC_RESET_DATA);
1167	}
1168
1169	/**
1170	* amdgpu_device_pci_reset - reset the GPU using generic PCI means
1171	*
1172	* @adev: amdgpu_device pointer
1173	*
1174	* Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1175	*/
1176	int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1177	{
1178	return pci_reset_function(dev: adev->pdev);
1179	}
1180
1181	/*
1182	* amdgpu_device_wb_*()
1183	* Writeback is the method by which the GPU updates special pages in memory
1184	* with the status of certain GPU events (fences, ring pointers,etc.).
1185	*/
1186
1187	/**
1188	* amdgpu_device_wb_fini - Disable Writeback and free memory
1189	*
1190	* @adev: amdgpu_device pointer
1191	*
1192	* Disables Writeback and frees the Writeback memory (all asics).
1193	* Used at driver shutdown.
1194	*/
1195	static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1196	{
1197	if (adev->wb.wb_obj) {
1198	amdgpu_bo_free_kernel(bo: &adev->wb.wb_obj,
1199	gpu_addr: &adev->wb.gpu_addr,
1200	cpu_addr: (void **)&adev->wb.wb);
1201	adev->wb.wb_obj = NULL;
1202	}
1203	}
1204
1205	/**
1206	* amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1207	*
1208	* @adev: amdgpu_device pointer
1209	*
1210	* Initializes writeback and allocates writeback memory (all asics).
1211	* Used at driver startup.
1212	* Returns 0 on success or an -error on failure.
1213	*/
1214	static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1215	{
1216	int r;
1217
1218	if (adev->wb.wb_obj == NULL) {
1219	/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1220	r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1221	PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1222	bo_ptr: &adev->wb.wb_obj, gpu_addr: &adev->wb.gpu_addr,
1223	cpu_addr: (void **)&adev->wb.wb);
1224	if (r) {
1225	dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1226	return r;
1227	}
1228
1229	adev->wb.num_wb = AMDGPU_MAX_WB;
1230	memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1231
1232	/* clear wb memory */
1233	memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1234	}
1235
1236	return 0;
1237	}
1238
1239	/**
1240	* amdgpu_device_wb_get - Allocate a wb entry
1241	*
1242	* @adev: amdgpu_device pointer
1243	* @wb: wb index
1244	*
1245	* Allocate a wb slot for use by the driver (all asics).
1246	* Returns 0 on success or -EINVAL on failure.
1247	*/
1248	int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1249	{
1250	unsigned long offset = find_first_zero_bit(addr: adev->wb.used, size: adev->wb.num_wb);
1251
1252	if (offset < adev->wb.num_wb) {
1253	__set_bit(offset, adev->wb.used);
1254	*wb = offset << 3; /* convert to dw offset */
1255	return 0;
1256	} else {
1257	return -EINVAL;
1258	}
1259	}
1260
1261	/**
1262	* amdgpu_device_wb_free - Free a wb entry
1263	*
1264	* @adev: amdgpu_device pointer
1265	* @wb: wb index
1266	*
1267	* Free a wb slot allocated for use by the driver (all asics)
1268	*/
1269	void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1270	{
1271	wb >>= 3;
1272	if (wb < adev->wb.num_wb)
1273	__clear_bit(wb, adev->wb.used);
1274	}
1275
1276	/**
1277	* amdgpu_device_resize_fb_bar - try to resize FB BAR
1278	*
1279	* @adev: amdgpu_device pointer
1280	*
1281	* Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1282	* to fail, but if any of the BARs is not accessible after the size we abort
1283	* driver loading by returning -ENODEV.
1284	*/
1285	int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1286	{
1287	int rbar_size = pci_rebar_bytes_to_size(bytes: adev->gmc.real_vram_size);
1288	struct pci_bus *root;
1289	struct resource *res;
1290	unsigned int i;
1291	u16 cmd;
1292	int r;
1293
1294	if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
1295	return 0;
1296
1297	/* Bypass for VF */
1298	if (amdgpu_sriov_vf(adev))
1299	return 0;
1300
1301	/* skip if the bios has already enabled large BAR */
1302	if (adev->gmc.real_vram_size &&
1303	(pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1304	return 0;
1305
1306	/* Check if the root BUS has 64bit memory resources */
1307	root = adev->pdev->bus;
1308	while (root->parent)
1309	root = root->parent;
1310
1311	pci_bus_for_each_resource(root, res, i) {
1312	if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1313	res->start > 0x100000000ull)
1314	break;
1315	}
1316
1317	/* Trying to resize is pointless without a root hub window above 4GB */
1318	if (!res)
1319	return 0;
1320
1321	/* Limit the BAR size to what is available */
1322	rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1323	rbar_size);
1324
1325	/* Disable memory decoding while we change the BAR addresses and size */
1326	pci_read_config_word(dev: adev->pdev, PCI_COMMAND, val: &cmd);
1327	pci_write_config_word(dev: adev->pdev, PCI_COMMAND,
1328	val: cmd & ~PCI_COMMAND_MEMORY);
1329
1330	/* Free the VRAM and doorbell BAR, we most likely need to move both. */
1331	amdgpu_doorbell_fini(adev);
1332	if (adev->asic_type >= CHIP_BONAIRE)
1333	pci_release_resource(dev: adev->pdev, resno: 2);
1334
1335	pci_release_resource(dev: adev->pdev, resno: 0);
1336
1337	r = pci_resize_resource(dev: adev->pdev, i: 0, size: rbar_size);
1338	if (r == -ENOSPC)
1339	DRM_INFO("Not enough PCI address space for a large BAR.");
1340	else if (r && r != -ENOTSUPP)
1341	DRM_ERROR("Problem resizing BAR0 (%d).", r);
1342
1343	pci_assign_unassigned_bus_resources(bus: adev->pdev->bus);
1344
1345	/* When the doorbell or fb BAR isn't available we have no chance of
1346	* using the device.
1347	*/
1348	r = amdgpu_doorbell_init(adev);
1349	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1350	return -ENODEV;
1351
1352	pci_write_config_word(dev: adev->pdev, PCI_COMMAND, val: cmd);
1353
1354	return 0;
1355	}
1356
1357	static bool amdgpu_device_read_bios(struct amdgpu_device *adev)
1358	{
1359	if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
1360	return false;
1361
1362	return true;
1363	}
1364
1365	/*
1366	* GPU helpers function.
1367	*/
1368	/**
1369	* amdgpu_device_need_post - check if the hw need post or not
1370	*
1371	* @adev: amdgpu_device pointer
1372	*
1373	* Check if the asic has been initialized (all asics) at driver startup
1374	* or post is needed if hw reset is performed.
1375	* Returns true if need or false if not.
1376	*/
1377	bool amdgpu_device_need_post(struct amdgpu_device *adev)
1378	{
1379	uint32_t reg;
1380
1381	if (amdgpu_sriov_vf(adev))
1382	return false;
1383
1384	if (!amdgpu_device_read_bios(adev))
1385	return false;
1386
1387	if (amdgpu_passthrough(adev)) {
1388	/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1389	* some old smc fw still need driver do vPost otherwise gpu hang, while
1390	* those smc fw version above 22.15 doesn't have this flaw, so we force
1391	* vpost executed for smc version below 22.15
1392	*/
1393	if (adev->asic_type == CHIP_FIJI) {
1394	int err;
1395	uint32_t fw_ver;
1396
1397	err = request_firmware(fw: &adev->pm.fw, name: "amdgpu/fiji_smc.bin", device: adev->dev);
1398	/* force vPost if error occured */
1399	if (err)
1400	return true;
1401
1402	fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1403	if (fw_ver < 0x00160e00)
1404	return true;
1405	}
1406	}
1407
1408	/* Don't post if we need to reset whole hive on init */
1409	if (adev->gmc.xgmi.pending_reset)
1410	return false;
1411
1412	if (adev->has_hw_reset) {
1413	adev->has_hw_reset = false;
1414	return true;
1415	}
1416
1417	/* bios scratch used on CIK+ */
1418	if (adev->asic_type >= CHIP_BONAIRE)
1419	return amdgpu_atombios_scratch_need_asic_init(adev);
1420
1421	/* check MEM_SIZE for older asics */
1422	reg = amdgpu_asic_get_config_memsize(adev);
1423
1424	if ((reg != 0) && (reg != 0xffffffff))
1425	return false;
1426
1427	return true;
1428	}
1429
1430	/*
1431	* Check whether seamless boot is supported.
1432	*
1433	* So far we only support seamless boot on DCE 3.0 or later.
1434	* If users report that it works on older ASICS as well, we may
1435	* loosen this.
1436	*/
1437	bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
1438	{
1439	switch (amdgpu_seamless) {
1440	case -1:
1441	break;
1442	case 1:
1443	return true;
1444	case 0:
1445	return false;
1446	default:
1447	DRM_ERROR("Invalid value for amdgpu.seamless: %d\n",
1448	amdgpu_seamless);
1449	return false;
1450	}
1451
1452	if (!(adev->flags & AMD_IS_APU))
1453	return false;
1454
1455	if (adev->mman.keep_stolen_vga_memory)
1456	return false;
1457
1458	return adev->ip_versions[DCE_HWIP][0] >= IP_VERSION(3, 0, 0);
1459	}
1460
1461	/*
1462	* Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
1463	* don't support dynamic speed switching. Until we have confirmation from Intel
1464	* that a specific host supports it, it's safer that we keep it disabled for all.
1465	*
1466	* https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1467	* https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1468	*/
1469	static bool amdgpu_device_pcie_dynamic_switching_supported(void)
1470	{
1471	#if IS_ENABLED(CONFIG_X86)
1472	struct cpuinfo_x86 *c = &cpu_data(0);
1473
1474	if (c->x86_vendor == X86_VENDOR_INTEL)
1475	return false;
1476	#endif
1477	return true;
1478	}
1479
1480	/**
1481	* amdgpu_device_should_use_aspm - check if the device should program ASPM
1482	*
1483	* @adev: amdgpu_device pointer
1484	*
1485	* Confirm whether the module parameter and pcie bridge agree that ASPM should
1486	* be set for this device.
1487	*
1488	* Returns true if it should be used or false if not.
1489	*/
1490	bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1491	{
1492	switch (amdgpu_aspm) {
1493	case -1:
1494	break;
1495	case 0:
1496	return false;
1497	case 1:
1498	return true;
1499	default:
1500	return false;
1501	}
1502	if (adev->flags & AMD_IS_APU)
1503	return false;
1504	if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK))
1505	return false;
1506	return pcie_aspm_enabled(pdev: adev->pdev);
1507	}
1508
1509	/* if we get transitioned to only one device, take VGA back */
1510	/**
1511	* amdgpu_device_vga_set_decode - enable/disable vga decode
1512	*
1513	* @pdev: PCI device pointer
1514	* @state: enable/disable vga decode
1515	*
1516	* Enable/disable vga decode (all asics).
1517	* Returns VGA resource flags.
1518	*/
1519	static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1520	bool state)
1521	{
1522	struct amdgpu_device *adev = drm_to_adev(ddev: pci_get_drvdata(pdev));
1523
1524	amdgpu_asic_set_vga_state(adev, state);
1525	if (state)
1526	return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1527	VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1528	else
1529	return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1530	}
1531
1532	/**
1533	* amdgpu_device_check_block_size - validate the vm block size
1534	*
1535	* @adev: amdgpu_device pointer
1536	*
1537	* Validates the vm block size specified via module parameter.
1538	* The vm block size defines number of bits in page table versus page directory,
1539	* a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1540	* page table and the remaining bits are in the page directory.
1541	*/
1542	static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1543	{
1544	/* defines number of bits in page table versus page directory,
1545	* a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1546	* page table and the remaining bits are in the page directory
1547	*/
1548	if (amdgpu_vm_block_size == -1)
1549	return;
1550
1551	if (amdgpu_vm_block_size < 9) {
1552	dev_warn(adev->dev, "VM page table size (%d) too small\n",
1553	amdgpu_vm_block_size);
1554	amdgpu_vm_block_size = -1;
1555	}
1556	}
1557
1558	/**
1559	* amdgpu_device_check_vm_size - validate the vm size
1560	*
1561	* @adev: amdgpu_device pointer
1562	*
1563	* Validates the vm size in GB specified via module parameter.
1564	* The VM size is the size of the GPU virtual memory space in GB.
1565	*/
1566	static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1567	{
1568	/* no need to check the default value */
1569	if (amdgpu_vm_size == -1)
1570	return;
1571
1572	if (amdgpu_vm_size < 1) {
1573	dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1574	amdgpu_vm_size);
1575	amdgpu_vm_size = -1;
1576	}
1577	}
1578
1579	static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1580	{
1581	struct sysinfo si;
1582	bool is_os_64 = (sizeof(void *) == 8);
1583	uint64_t total_memory;
1584	uint64_t dram_size_seven_GB = 0x1B8000000;
1585	uint64_t dram_size_three_GB = 0xB8000000;
1586
1587	if (amdgpu_smu_memory_pool_size == 0)
1588	return;
1589
1590	if (!is_os_64) {
1591	DRM_WARN("Not 64-bit OS, feature not supported\n");
1592	goto def_value;
1593	}
1594	si_meminfo(val: &si);
1595	total_memory = (uint64_t)si.totalram * si.mem_unit;
1596
1597	if ((amdgpu_smu_memory_pool_size == 1) ||
1598	(amdgpu_smu_memory_pool_size == 2)) {
1599	if (total_memory < dram_size_three_GB)
1600	goto def_value1;
1601	} else if ((amdgpu_smu_memory_pool_size == 4) ||
1602	(amdgpu_smu_memory_pool_size == 8)) {
1603	if (total_memory < dram_size_seven_GB)
1604	goto def_value1;
1605	} else {
1606	DRM_WARN("Smu memory pool size not supported\n");
1607	goto def_value;
1608	}
1609	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1610
1611	return;
1612
1613	def_value1:
1614	DRM_WARN("No enough system memory\n");
1615	def_value:
1616	adev->pm.smu_prv_buffer_size = 0;
1617	}
1618
1619	static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1620	{
1621	if (!(adev->flags & AMD_IS_APU) ||
1622	adev->asic_type < CHIP_RAVEN)
1623	return 0;
1624
1625	switch (adev->asic_type) {
1626	case CHIP_RAVEN:
1627	if (adev->pdev->device == 0x15dd)
1628	adev->apu_flags |= AMD_APU_IS_RAVEN;
1629	if (adev->pdev->device == 0x15d8)
1630	adev->apu_flags |= AMD_APU_IS_PICASSO;
1631	break;
1632	case CHIP_RENOIR:
1633	if ((adev->pdev->device == 0x1636) ||
1634	(adev->pdev->device == 0x164c))
1635	adev->apu_flags |= AMD_APU_IS_RENOIR;
1636	else
1637	adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1638	break;
1639	case CHIP_VANGOGH:
1640	adev->apu_flags |= AMD_APU_IS_VANGOGH;
1641	break;
1642	case CHIP_YELLOW_CARP:
1643	break;
1644	case CHIP_CYAN_SKILLFISH:
1645	if ((adev->pdev->device == 0x13FE) ||
1646	(adev->pdev->device == 0x143F))
1647	adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1648	break;
1649	default:
1650	break;
1651	}
1652
1653	return 0;
1654	}
1655
1656	/**
1657	* amdgpu_device_check_arguments - validate module params
1658	*
1659	* @adev: amdgpu_device pointer
1660	*
1661	* Validates certain module parameters and updates
1662	* the associated values used by the driver (all asics).
1663	*/
1664	static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1665	{
1666	if (amdgpu_sched_jobs < 4) {
1667	dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1668	amdgpu_sched_jobs);
1669	amdgpu_sched_jobs = 4;
1670	} else if (!is_power_of_2(n: amdgpu_sched_jobs)) {
1671	dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1672	amdgpu_sched_jobs);
1673	amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1674	}
1675
1676	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1677	/* gart size must be greater or equal to 32M */
1678	dev_warn(adev->dev, "gart size (%d) too small\n",
1679	amdgpu_gart_size);
1680	amdgpu_gart_size = -1;
1681	}
1682
1683	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1684	/* gtt size must be greater or equal to 32M */
1685	dev_warn(adev->dev, "gtt size (%d) too small\n",
1686	amdgpu_gtt_size);
1687	amdgpu_gtt_size = -1;
1688	}
1689
1690	/* valid range is between 4 and 9 inclusive */
1691	if (amdgpu_vm_fragment_size != -1 &&
1692	(amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1693	dev_warn(adev->dev, "valid range is between 4 and 9\n");
1694	amdgpu_vm_fragment_size = -1;
1695	}
1696
1697	if (amdgpu_sched_hw_submission < 2) {
1698	dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1699	amdgpu_sched_hw_submission);
1700	amdgpu_sched_hw_submission = 2;
1701	} else if (!is_power_of_2(n: amdgpu_sched_hw_submission)) {
1702	dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1703	amdgpu_sched_hw_submission);
1704	amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1705	}
1706
1707	if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1708	dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1709	amdgpu_reset_method = -1;
1710	}
1711
1712	amdgpu_device_check_smu_prv_buffer_size(adev);
1713
1714	amdgpu_device_check_vm_size(adev);
1715
1716	amdgpu_device_check_block_size(adev);
1717
1718	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, load_type: amdgpu_fw_load_type);
1719
1720	return 0;
1721	}
1722
1723	/**
1724	* amdgpu_switcheroo_set_state - set switcheroo state
1725	*
1726	* @pdev: pci dev pointer
1727	* @state: vga_switcheroo state
1728	*
1729	* Callback for the switcheroo driver. Suspends or resumes
1730	* the asics before or after it is powered up using ACPI methods.
1731	*/
1732	static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1733	enum vga_switcheroo_state state)
1734	{
1735	struct drm_device *dev = pci_get_drvdata(pdev);
1736	int r;
1737
1738	if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1739	return;
1740
1741	if (state == VGA_SWITCHEROO_ON) {
1742	pr_info("switched on\n");
1743	/* don't suspend or resume card normally */
1744	dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1745
1746	pci_set_power_state(dev: pdev, PCI_D0);
1747	amdgpu_device_load_pci_state(pdev);
1748	r = pci_enable_device(dev: pdev);
1749	if (r)
1750	DRM_WARN("pci_enable_device failed (%d)\n", r);
1751	amdgpu_device_resume(dev, fbcon: true);
1752
1753	dev->switch_power_state = DRM_SWITCH_POWER_ON;
1754	} else {
1755	pr_info("switched off\n");
1756	dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1757	amdgpu_device_prepare(dev);
1758	amdgpu_device_suspend(dev, fbcon: true);
1759	amdgpu_device_cache_pci_state(pdev);
1760	/* Shut down the device */
1761	pci_disable_device(dev: pdev);
1762	pci_set_power_state(dev: pdev, PCI_D3cold);
1763	dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1764	}
1765	}
1766
1767	/**
1768	* amdgpu_switcheroo_can_switch - see if switcheroo state can change
1769	*
1770	* @pdev: pci dev pointer
1771	*
1772	* Callback for the switcheroo driver. Check of the switcheroo
1773	* state can be changed.
1774	* Returns true if the state can be changed, false if not.
1775	*/
1776	static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1777	{
1778	struct drm_device *dev = pci_get_drvdata(pdev);
1779
1780	/*
1781	* FIXME: open_count is protected by drm_global_mutex but that would lead to
1782	* locking inversion with the driver load path. And the access here is
1783	* completely racy anyway. So don't bother with locking for now.
1784	*/
1785	return atomic_read(v: &dev->open_count) == 0;
1786	}
1787
1788	static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1789	.set_gpu_state = amdgpu_switcheroo_set_state,
1790	.reprobe = NULL,
1791	.can_switch = amdgpu_switcheroo_can_switch,
1792	};
1793
1794	/**
1795	* amdgpu_device_ip_set_clockgating_state - set the CG state
1796	*
1797	* @dev: amdgpu_device pointer
1798	* @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1799	* @state: clockgating state (gate or ungate)
1800	*
1801	* Sets the requested clockgating state for all instances of
1802	* the hardware IP specified.
1803	* Returns the error code from the last instance.
1804	*/
1805	int amdgpu_device_ip_set_clockgating_state(void *dev,
1806	enum amd_ip_block_type block_type,
1807	enum amd_clockgating_state state)
1808	{
1809	struct amdgpu_device *adev = dev;
1810	int i, r = 0;
1811
1812	for (i = 0; i < adev->num_ip_blocks; i++) {
1813	if (!adev->ip_blocks[i].status.valid)
1814	continue;
1815	if (adev->ip_blocks[i].version->type != block_type)
1816	continue;
1817	if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1818	continue;
1819	r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1820	(void *)adev, state);
1821	if (r)
1822	DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1823	adev->ip_blocks[i].version->funcs->name, r);
1824	}
1825	return r;
1826	}
1827
1828	/**
1829	* amdgpu_device_ip_set_powergating_state - set the PG state
1830	*
1831	* @dev: amdgpu_device pointer
1832	* @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1833	* @state: powergating state (gate or ungate)
1834	*
1835	* Sets the requested powergating state for all instances of
1836	* the hardware IP specified.
1837	* Returns the error code from the last instance.
1838	*/
1839	int amdgpu_device_ip_set_powergating_state(void *dev,
1840	enum amd_ip_block_type block_type,
1841	enum amd_powergating_state state)
1842	{
1843	struct amdgpu_device *adev = dev;
1844	int i, r = 0;
1845
1846	for (i = 0; i < adev->num_ip_blocks; i++) {
1847	if (!adev->ip_blocks[i].status.valid)
1848	continue;
1849	if (adev->ip_blocks[i].version->type != block_type)
1850	continue;
1851	if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1852	continue;
1853	r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1854	(void *)adev, state);
1855	if (r)
1856	DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1857	adev->ip_blocks[i].version->funcs->name, r);
1858	}
1859	return r;
1860	}
1861
1862	/**
1863	* amdgpu_device_ip_get_clockgating_state - get the CG state
1864	*
1865	* @adev: amdgpu_device pointer
1866	* @flags: clockgating feature flags
1867	*
1868	* Walks the list of IPs on the device and updates the clockgating
1869	* flags for each IP.
1870	* Updates @flags with the feature flags for each hardware IP where
1871	* clockgating is enabled.
1872	*/
1873	void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1874	u64 *flags)
1875	{
1876	int i;
1877
1878	for (i = 0; i < adev->num_ip_blocks; i++) {
1879	if (!adev->ip_blocks[i].status.valid)
1880	continue;
1881	if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1882	adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1883	}
1884	}
1885
1886	/**
1887	* amdgpu_device_ip_wait_for_idle - wait for idle
1888	*
1889	* @adev: amdgpu_device pointer
1890	* @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1891	*
1892	* Waits for the request hardware IP to be idle.
1893	* Returns 0 for success or a negative error code on failure.
1894	*/
1895	int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1896	enum amd_ip_block_type block_type)
1897	{
1898	int i, r;
1899
1900	for (i = 0; i < adev->num_ip_blocks; i++) {
1901	if (!adev->ip_blocks[i].status.valid)
1902	continue;
1903	if (adev->ip_blocks[i].version->type == block_type) {
1904	r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1905	if (r)
1906	return r;
1907	break;
1908	}
1909	}
1910	return 0;
1911
1912	}
1913
1914	/**
1915	* amdgpu_device_ip_is_idle - is the hardware IP idle
1916	*
1917	* @adev: amdgpu_device pointer
1918	* @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1919	*
1920	* Check if the hardware IP is idle or not.
1921	* Returns true if it the IP is idle, false if not.
1922	*/
1923	bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1924	enum amd_ip_block_type block_type)
1925	{
1926	int i;
1927
1928	for (i = 0; i < adev->num_ip_blocks; i++) {
1929	if (!adev->ip_blocks[i].status.valid)
1930	continue;
1931	if (adev->ip_blocks[i].version->type == block_type)
1932	return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1933	}
1934	return true;
1935
1936	}
1937
1938	/**
1939	* amdgpu_device_ip_get_ip_block - get a hw IP pointer
1940	*
1941	* @adev: amdgpu_device pointer
1942	* @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1943	*
1944	* Returns a pointer to the hardware IP block structure
1945	* if it exists for the asic, otherwise NULL.
1946	*/
1947	struct amdgpu_ip_block *
1948	amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1949	enum amd_ip_block_type type)
1950	{
1951	int i;
1952
1953	for (i = 0; i < adev->num_ip_blocks; i++)
1954	if (adev->ip_blocks[i].version->type == type)
1955	return &adev->ip_blocks[i];
1956
1957	return NULL;
1958	}
1959
1960	/**
1961	* amdgpu_device_ip_block_version_cmp
1962	*
1963	* @adev: amdgpu_device pointer
1964	* @type: enum amd_ip_block_type
1965	* @major: major version
1966	* @minor: minor version
1967	*
1968	* return 0 if equal or greater
1969	* return 1 if smaller or the ip_block doesn't exist
1970	*/
1971	int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1972	enum amd_ip_block_type type,
1973	u32 major, u32 minor)
1974	{
1975	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1976
1977	if (ip_block && ((ip_block->version->major > major) ||
1978	((ip_block->version->major == major) &&
1979	(ip_block->version->minor >= minor))))
1980	return 0;
1981
1982	return 1;
1983	}
1984
1985	/**
1986	* amdgpu_device_ip_block_add
1987	*
1988	* @adev: amdgpu_device pointer
1989	* @ip_block_version: pointer to the IP to add
1990	*
1991	* Adds the IP block driver information to the collection of IPs
1992	* on the asic.
1993	*/
1994	int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1995	const struct amdgpu_ip_block_version *ip_block_version)
1996	{
1997	if (!ip_block_version)
1998	return -EINVAL;
1999
2000	switch (ip_block_version->type) {
2001	case AMD_IP_BLOCK_TYPE_VCN:
2002	if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
2003	return 0;
2004	break;
2005	case AMD_IP_BLOCK_TYPE_JPEG:
2006	if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
2007	return 0;
2008	break;
2009	default:
2010	break;
2011	}
2012
2013	DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
2014	ip_block_version->funcs->name);
2015
2016	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
2017
2018	return 0;
2019	}
2020
2021	/**
2022	* amdgpu_device_enable_virtual_display - enable virtual display feature
2023	*
2024	* @adev: amdgpu_device pointer
2025	*
2026	* Enabled the virtual display feature if the user has enabled it via
2027	* the module parameter virtual_display. This feature provides a virtual
2028	* display hardware on headless boards or in virtualized environments.
2029	* This function parses and validates the configuration string specified by
2030	* the user and configues the virtual display configuration (number of
2031	* virtual connectors, crtcs, etc.) specified.
2032	*/
2033	static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
2034	{
2035	adev->enable_virtual_display = false;
2036
2037	if (amdgpu_virtual_display) {
2038	const char *pci_address_name = pci_name(pdev: adev->pdev);
2039	char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
2040
2041	pciaddstr = kstrdup(s: amdgpu_virtual_display, GFP_KERNEL);
2042	pciaddstr_tmp = pciaddstr;
2043	while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
2044	pciaddname = strsep(&pciaddname_tmp, ",");
2045	if (!strcmp("all", pciaddname)
2046	|| !strcmp(pci_address_name, pciaddname)) {
2047	long num_crtc;
2048	int res = -1;
2049
2050	adev->enable_virtual_display = true;
2051
2052	if (pciaddname_tmp)
2053	res = kstrtol(s: pciaddname_tmp, base: 10,
2054	res: &num_crtc);
2055
2056	if (!res) {
2057	if (num_crtc < 1)
2058	num_crtc = 1;
2059	if (num_crtc > 6)
2060	num_crtc = 6;
2061	adev->mode_info.num_crtc = num_crtc;
2062	} else {
2063	adev->mode_info.num_crtc = 1;
2064	}
2065	break;
2066	}
2067	}
2068
2069	DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
2070	amdgpu_virtual_display, pci_address_name,
2071	adev->enable_virtual_display, adev->mode_info.num_crtc);
2072
2073	kfree(objp: pciaddstr);
2074	}
2075	}
2076
2077	void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
2078	{
2079	if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
2080	adev->mode_info.num_crtc = 1;
2081	adev->enable_virtual_display = true;
2082	DRM_INFO("virtual_display:%d, num_crtc:%d\n",
2083	adev->enable_virtual_display, adev->mode_info.num_crtc);
2084	}
2085	}
2086
2087	/**
2088	* amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2089	*
2090	* @adev: amdgpu_device pointer
2091	*
2092	* Parses the asic configuration parameters specified in the gpu info
2093	* firmware and makes them availale to the driver for use in configuring
2094	* the asic.
2095	* Returns 0 on success, -EINVAL on failure.
2096	*/
2097	static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
2098	{
2099	const char *chip_name;
2100	char fw_name[40];
2101	int err;
2102	const struct gpu_info_firmware_header_v1_0 *hdr;
2103
2104	adev->firmware.gpu_info_fw = NULL;
2105
2106	if (adev->mman.discovery_bin) {
2107	/*
2108	* FIXME: The bounding box is still needed by Navi12, so
2109	* temporarily read it from gpu_info firmware. Should be dropped
2110	* when DAL no longer needs it.
2111	*/
2112	if (adev->asic_type != CHIP_NAVI12)
2113	return 0;
2114	}
2115
2116	switch (adev->asic_type) {
2117	default:
2118	return 0;
2119	case CHIP_VEGA10:
2120	chip_name = "vega10";
2121	break;
2122	case CHIP_VEGA12:
2123	chip_name = "vega12";
2124	break;
2125	case CHIP_RAVEN:
2126	if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2127	chip_name = "raven2";
2128	else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2129	chip_name = "picasso";
2130	else
2131	chip_name = "raven";
2132	break;
2133	case CHIP_ARCTURUS:
2134	chip_name = "arcturus";
2135	break;
2136	case CHIP_NAVI12:
2137	chip_name = "navi12";
2138	break;
2139	}
2140
2141	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_gpu_info.bin", chip_name);
2142	err = amdgpu_ucode_request(adev, fw: &adev->firmware.gpu_info_fw, fw_name);
2143	if (err) {
2144	dev_err(adev->dev,
2145	"Failed to get gpu_info firmware \"%s\"\n",
2146	fw_name);
2147	goto out;
2148	}
2149
2150	hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2151	amdgpu_ucode_print_gpu_info_hdr(hdr: &hdr->header);
2152
2153	switch (hdr->version_major) {
2154	case 1:
2155	{
2156	const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2157	(const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2158	le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2159
2160	/*
2161	* Should be droped when DAL no longer needs it.
2162	*/
2163	if (adev->asic_type == CHIP_NAVI12)
2164	goto parse_soc_bounding_box;
2165
2166	adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2167	adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2168	adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2169	adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2170	adev->gfx.config.max_texture_channel_caches =
2171	le32_to_cpu(gpu_info_fw->gc_num_tccs);
2172	adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2173	adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2174	adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2175	adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2176	adev->gfx.config.double_offchip_lds_buf =
2177	le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2178	adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2179	adev->gfx.cu_info.max_waves_per_simd =
2180	le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2181	adev->gfx.cu_info.max_scratch_slots_per_cu =
2182	le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2183	adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2184	if (hdr->version_minor >= 1) {
2185	const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2186	(const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2187	le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2188	adev->gfx.config.num_sc_per_sh =
2189	le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2190	adev->gfx.config.num_packer_per_sc =
2191	le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2192	}
2193
2194	parse_soc_bounding_box:
2195	/*
2196	* soc bounding box info is not integrated in disocovery table,
2197	* we always need to parse it from gpu info firmware if needed.
2198	*/
2199	if (hdr->version_minor == 2) {
2200	const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2201	(const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2202	le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2203	adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2204	}
2205	break;
2206	}
2207	default:
2208	dev_err(adev->dev,
2209	"Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2210	err = -EINVAL;
2211	goto out;
2212	}
2213	out:
2214	return err;
2215	}
2216
2217	/**
2218	* amdgpu_device_ip_early_init - run early init for hardware IPs
2219	*
2220	* @adev: amdgpu_device pointer
2221	*
2222	* Early initialization pass for hardware IPs. The hardware IPs that make
2223	* up each asic are discovered each IP's early_init callback is run. This
2224	* is the first stage in initializing the asic.
2225	* Returns 0 on success, negative error code on failure.
2226	*/
2227	static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2228	{
2229	struct pci_dev *parent;
2230	int i, r;
2231	bool total;
2232
2233	amdgpu_device_enable_virtual_display(adev);
2234
2235	if (amdgpu_sriov_vf(adev)) {
2236	r = amdgpu_virt_request_full_gpu(adev, init: true);
2237	if (r)
2238	return r;
2239	}
2240
2241	switch (adev->asic_type) {
2242	#ifdef CONFIG_DRM_AMDGPU_SI
2243	case CHIP_VERDE:
2244	case CHIP_TAHITI:
2245	case CHIP_PITCAIRN:
2246	case CHIP_OLAND:
2247	case CHIP_HAINAN:
2248	adev->family = AMDGPU_FAMILY_SI;
2249	r = si_set_ip_blocks(adev);
2250	if (r)
2251	return r;
2252	break;
2253	#endif
2254	#ifdef CONFIG_DRM_AMDGPU_CIK
2255	case CHIP_BONAIRE:
2256	case CHIP_HAWAII:
2257	case CHIP_KAVERI:
2258	case CHIP_KABINI:
2259	case CHIP_MULLINS:
2260	if (adev->flags & AMD_IS_APU)
2261	adev->family = AMDGPU_FAMILY_KV;
2262	else
2263	adev->family = AMDGPU_FAMILY_CI;
2264
2265	r = cik_set_ip_blocks(adev);
2266	if (r)
2267	return r;
2268	break;
2269	#endif
2270	case CHIP_TOPAZ:
2271	case CHIP_TONGA:
2272	case CHIP_FIJI:
2273	case CHIP_POLARIS10:
2274	case CHIP_POLARIS11:
2275	case CHIP_POLARIS12:
2276	case CHIP_VEGAM:
2277	case CHIP_CARRIZO:
2278	case CHIP_STONEY:
2279	if (adev->flags & AMD_IS_APU)
2280	adev->family = AMDGPU_FAMILY_CZ;
2281	else
2282	adev->family = AMDGPU_FAMILY_VI;
2283
2284	r = vi_set_ip_blocks(adev);
2285	if (r)
2286	return r;
2287	break;
2288	default:
2289	r = amdgpu_discovery_set_ip_blocks(adev);
2290	if (r)
2291	return r;
2292	break;
2293	}
2294
2295	if (amdgpu_has_atpx() &&
2296	(amdgpu_is_atpx_hybrid() ||
2297	amdgpu_has_atpx_dgpu_power_cntl()) &&
2298	((adev->flags & AMD_IS_APU) == 0) &&
2299	!dev_is_removable(dev: &adev->pdev->dev))
2300	adev->flags |= AMD_IS_PX;
2301
2302	if (!(adev->flags & AMD_IS_APU)) {
2303	parent = pcie_find_root_port(dev: adev->pdev);
2304	adev->has_pr3 = parent ? pci_pr3_present(pdev: parent) : false;
2305	}
2306
2307
2308	adev->pm.pp_feature = amdgpu_pp_feature_mask;
2309	if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2310	adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2311	if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2312	adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2313	if (!amdgpu_device_pcie_dynamic_switching_supported())
2314	adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
2315
2316	total = true;
2317	for (i = 0; i < adev->num_ip_blocks; i++) {
2318	if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2319	DRM_WARN("disabled ip block: %d <%s>\n",
2320	i, adev->ip_blocks[i].version->funcs->name);
2321	adev->ip_blocks[i].status.valid = false;
2322	} else {
2323	if (adev->ip_blocks[i].version->funcs->early_init) {
2324	r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2325	if (r == -ENOENT) {
2326	adev->ip_blocks[i].status.valid = false;
2327	} else if (r) {
2328	DRM_ERROR("early_init of IP block <%s> failed %d\n",
2329	adev->ip_blocks[i].version->funcs->name, r);
2330	total = false;
2331	} else {
2332	adev->ip_blocks[i].status.valid = true;
2333	}
2334	} else {
2335	adev->ip_blocks[i].status.valid = true;
2336	}
2337	}
2338	/* get the vbios after the asic_funcs are set up */
2339	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2340	r = amdgpu_device_parse_gpu_info_fw(adev);
2341	if (r)
2342	return r;
2343
2344	/* Read BIOS */
2345	if (amdgpu_device_read_bios(adev)) {
2346	if (!amdgpu_get_bios(adev))
2347	return -EINVAL;
2348
2349	r = amdgpu_atombios_init(adev);
2350	if (r) {
2351	dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2352	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, error_flags: 0, error_data: 0);
2353	return r;
2354	}
2355	}
2356
2357	/*get pf2vf msg info at it's earliest time*/
2358	if (amdgpu_sriov_vf(adev))
2359	amdgpu_virt_init_data_exchange(adev);
2360
2361	}
2362	}
2363	if (!total)
2364	return -ENODEV;
2365
2366	amdgpu_amdkfd_device_probe(adev);
2367	adev->cg_flags &= amdgpu_cg_mask;
2368	adev->pg_flags &= amdgpu_pg_mask;
2369
2370	return 0;
2371	}
2372
2373	static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2374	{
2375	int i, r;
2376
2377	for (i = 0; i < adev->num_ip_blocks; i++) {
2378	if (!adev->ip_blocks[i].status.sw)
2379	continue;
2380	if (adev->ip_blocks[i].status.hw)
2381	continue;
2382	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2383	(amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2384	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2385	r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2386	if (r) {
2387	DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2388	adev->ip_blocks[i].version->funcs->name, r);
2389	return r;
2390	}
2391	adev->ip_blocks[i].status.hw = true;
2392	}
2393	}
2394
2395	return 0;
2396	}
2397
2398	static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2399	{
2400	int i, r;
2401
2402	for (i = 0; i < adev->num_ip_blocks; i++) {
2403	if (!adev->ip_blocks[i].status.sw)
2404	continue;
2405	if (adev->ip_blocks[i].status.hw)
2406	continue;
2407	r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2408	if (r) {
2409	DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2410	adev->ip_blocks[i].version->funcs->name, r);
2411	return r;
2412	}
2413	adev->ip_blocks[i].status.hw = true;
2414	}
2415
2416	return 0;
2417	}
2418
2419	static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2420	{
2421	int r = 0;
2422	int i;
2423	uint32_t smu_version;
2424
2425	if (adev->asic_type >= CHIP_VEGA10) {
2426	for (i = 0; i < adev->num_ip_blocks; i++) {
2427	if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2428	continue;
2429
2430	if (!adev->ip_blocks[i].status.sw)
2431	continue;
2432
2433	/* no need to do the fw loading again if already done*/
2434	if (adev->ip_blocks[i].status.hw == true)
2435	break;
2436
2437	if (amdgpu_in_reset(adev) || adev->in_suspend) {
2438	r = adev->ip_blocks[i].version->funcs->resume(adev);
2439	if (r) {
2440	DRM_ERROR("resume of IP block <%s> failed %d\n",
2441	adev->ip_blocks[i].version->funcs->name, r);
2442	return r;
2443	}
2444	} else {
2445	r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2446	if (r) {
2447	DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2448	adev->ip_blocks[i].version->funcs->name, r);
2449	return r;
2450	}
2451	}
2452
2453	adev->ip_blocks[i].status.hw = true;
2454	break;
2455	}
2456	}
2457
2458	if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2459	r = amdgpu_pm_load_smu_firmware(adev, smu_version: &smu_version);
2460
2461	return r;
2462	}
2463
2464	static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2465	{
2466	long timeout;
2467	int r, i;
2468
2469	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2470	struct amdgpu_ring *ring = adev->rings[i];
2471
2472	/* No need to setup the GPU scheduler for rings that don't need it */
2473	if (!ring || ring->no_scheduler)
2474	continue;
2475
2476	switch (ring->funcs->type) {
2477	case AMDGPU_RING_TYPE_GFX:
2478	timeout = adev->gfx_timeout;
2479	break;
2480	case AMDGPU_RING_TYPE_COMPUTE:
2481	timeout = adev->compute_timeout;
2482	break;
2483	case AMDGPU_RING_TYPE_SDMA:
2484	timeout = adev->sdma_timeout;
2485	break;
2486	default:
2487	timeout = adev->video_timeout;
2488	break;
2489	}
2490
2491	r = drm_sched_init(sched: &ring->sched, ops: &amdgpu_sched_ops,
2492	num_rqs: DRM_SCHED_PRIORITY_COUNT,
2493	hw_submission: ring->num_hw_submission, hang_limit: 0,
2494	timeout, timeout_wq: adev->reset_domain->wq,
2495	score: ring->sched_score, name: ring->name,
2496	dev: adev->dev);
2497	if (r) {
2498	DRM_ERROR("Failed to create scheduler on ring %s.\n",
2499	ring->name);
2500	return r;
2501	}
2502	}
2503
2504	amdgpu_xcp_update_partition_sched_list(adev);
2505
2506	return 0;
2507	}
2508
2509
2510	/**
2511	* amdgpu_device_ip_init - run init for hardware IPs
2512	*
2513	* @adev: amdgpu_device pointer
2514	*
2515	* Main initialization pass for hardware IPs. The list of all the hardware
2516	* IPs that make up the asic is walked and the sw_init and hw_init callbacks
2517	* are run. sw_init initializes the software state associated with each IP
2518	* and hw_init initializes the hardware associated with each IP.
2519	* Returns 0 on success, negative error code on failure.
2520	*/
2521	static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2522	{
2523	int i, r;
2524
2525	r = amdgpu_ras_init(adev);
2526	if (r)
2527	return r;
2528
2529	for (i = 0; i < adev->num_ip_blocks; i++) {
2530	if (!adev->ip_blocks[i].status.valid)
2531	continue;
2532	r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2533	if (r) {
2534	DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2535	adev->ip_blocks[i].version->funcs->name, r);
2536	goto init_failed;
2537	}
2538	adev->ip_blocks[i].status.sw = true;
2539
2540	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2541	/* need to do common hw init early so everything is set up for gmc */
2542	r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2543	if (r) {
2544	DRM_ERROR("hw_init %d failed %d\n", i, r);
2545	goto init_failed;
2546	}
2547	adev->ip_blocks[i].status.hw = true;
2548	} else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2549	/* need to do gmc hw init early so we can allocate gpu mem */
2550	/* Try to reserve bad pages early */
2551	if (amdgpu_sriov_vf(adev))
2552	amdgpu_virt_exchange_data(adev);
2553
2554	r = amdgpu_device_mem_scratch_init(adev);
2555	if (r) {
2556	DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2557	goto init_failed;
2558	}
2559	r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2560	if (r) {
2561	DRM_ERROR("hw_init %d failed %d\n", i, r);
2562	goto init_failed;
2563	}
2564	r = amdgpu_device_wb_init(adev);
2565	if (r) {
2566	DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2567	goto init_failed;
2568	}
2569	adev->ip_blocks[i].status.hw = true;
2570
2571	/* right after GMC hw init, we create CSA */
2572	if (adev->gfx.mcbp) {
2573	r = amdgpu_allocate_static_csa(adev, bo: &adev->virt.csa_obj,
2574	AMDGPU_GEM_DOMAIN_VRAM |
2575	AMDGPU_GEM_DOMAIN_GTT,
2576	AMDGPU_CSA_SIZE);
2577	if (r) {
2578	DRM_ERROR("allocate CSA failed %d\n", r);
2579	goto init_failed;
2580	}
2581	}
2582	}
2583	}
2584
2585	if (amdgpu_sriov_vf(adev))
2586	amdgpu_virt_init_data_exchange(adev);
2587
2588	r = amdgpu_ib_pool_init(adev);
2589	if (r) {
2590	dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2591	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_IB_INIT_FAIL, error_flags: 0, error_data: r);
2592	goto init_failed;
2593	}
2594
2595	r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2596	if (r)
2597	goto init_failed;
2598
2599	r = amdgpu_device_ip_hw_init_phase1(adev);
2600	if (r)
2601	goto init_failed;
2602
2603	r = amdgpu_device_fw_loading(adev);
2604	if (r)
2605	goto init_failed;
2606
2607	r = amdgpu_device_ip_hw_init_phase2(adev);
2608	if (r)
2609	goto init_failed;
2610
2611	/*
2612	* retired pages will be loaded from eeprom and reserved here,
2613	* it should be called after amdgpu_device_ip_hw_init_phase2 since
2614	* for some ASICs the RAS EEPROM code relies on SMU fully functioning
2615	* for I2C communication which only true at this point.
2616	*
2617	* amdgpu_ras_recovery_init may fail, but the upper only cares the
2618	* failure from bad gpu situation and stop amdgpu init process
2619	* accordingly. For other failed cases, it will still release all
2620	* the resource and print error message, rather than returning one
2621	* negative value to upper level.
2622	*
2623	* Note: theoretically, this should be called before all vram allocations
2624	* to protect retired page from abusing
2625	*/
2626	r = amdgpu_ras_recovery_init(adev);
2627	if (r)
2628	goto init_failed;
2629
2630	/**
2631	* In case of XGMI grab extra reference for reset domain for this device
2632	*/
2633	if (adev->gmc.xgmi.num_physical_nodes > 1) {
2634	if (amdgpu_xgmi_add_device(adev) == 0) {
2635	if (!amdgpu_sriov_vf(adev)) {
2636	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2637
2638	if (WARN_ON(!hive)) {
2639	r = -ENOENT;
2640	goto init_failed;
2641	}
2642
2643	if (!hive->reset_domain ||
2644	!amdgpu_reset_get_reset_domain(domain: hive->reset_domain)) {
2645	r = -ENOENT;
2646	amdgpu_put_xgmi_hive(hive);
2647	goto init_failed;
2648	}
2649
2650	/* Drop the early temporary reset domain we created for device */
2651	amdgpu_reset_put_reset_domain(domain: adev->reset_domain);
2652	adev->reset_domain = hive->reset_domain;
2653	amdgpu_put_xgmi_hive(hive);
2654	}
2655	}
2656	}
2657
2658	r = amdgpu_device_init_schedulers(adev);
2659	if (r)
2660	goto init_failed;
2661
2662	if (adev->mman.buffer_funcs_ring->sched.ready)
2663	amdgpu_ttm_set_buffer_funcs_status(adev, enable: true);
2664
2665	/* Don't init kfd if whole hive need to be reset during init */
2666	if (!adev->gmc.xgmi.pending_reset) {
2667	kgd2kfd_init_zone_device(adev);
2668	amdgpu_amdkfd_device_init(adev);
2669	}
2670
2671	amdgpu_fru_get_product_info(adev);
2672
2673	init_failed:
2674
2675	return r;
2676	}
2677
2678	/**
2679	* amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2680	*
2681	* @adev: amdgpu_device pointer
2682	*
2683	* Writes a reset magic value to the gart pointer in VRAM. The driver calls
2684	* this function before a GPU reset. If the value is retained after a
2685	* GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents.
2686	*/
2687	static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2688	{
2689	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2690	}
2691
2692	/**
2693	* amdgpu_device_check_vram_lost - check if vram is valid
2694	*
2695	* @adev: amdgpu_device pointer
2696	*
2697	* Checks the reset magic value written to the gart pointer in VRAM.
2698	* The driver calls this after a GPU reset to see if the contents of
2699	* VRAM is lost or now.
2700	* returns true if vram is lost, false if not.
2701	*/
2702	static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2703	{
2704	if (memcmp(p: adev->gart.ptr, q: adev->reset_magic,
2705	AMDGPU_RESET_MAGIC_NUM))
2706	return true;
2707
2708	if (!amdgpu_in_reset(adev))
2709	return false;
2710
2711	/*
2712	* For all ASICs with baco/mode1 reset, the VRAM is
2713	* always assumed to be lost.
2714	*/
2715	switch (amdgpu_asic_reset_method(adev)) {
2716	case AMD_RESET_METHOD_BACO:
2717	case AMD_RESET_METHOD_MODE1:
2718	return true;
2719	default:
2720	return false;
2721	}
2722	}
2723
2724	/**
2725	* amdgpu_device_set_cg_state - set clockgating for amdgpu device
2726	*
2727	* @adev: amdgpu_device pointer
2728	* @state: clockgating state (gate or ungate)
2729	*
2730	* The list of all the hardware IPs that make up the asic is walked and the
2731	* set_clockgating_state callbacks are run.
2732	* Late initialization pass enabling clockgating for hardware IPs.
2733	* Fini or suspend, pass disabling clockgating for hardware IPs.
2734	* Returns 0 on success, negative error code on failure.
2735	*/
2736
2737	int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2738	enum amd_clockgating_state state)
2739	{
2740	int i, j, r;
2741
2742	if (amdgpu_emu_mode == 1)
2743	return 0;
2744
2745	for (j = 0; j < adev->num_ip_blocks; j++) {
2746	i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2747	if (!adev->ip_blocks[i].status.late_initialized)
2748	continue;
2749	/* skip CG for GFX, SDMA on S0ix */
2750	if (adev->in_s0ix &&
2751	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2752	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2753	continue;
2754	/* skip CG for VCE/UVD, it's handled specially */
2755	if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2756	adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2757	adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2758	adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2759	adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2760	/* enable clockgating to save power */
2761	r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2762	state);
2763	if (r) {
2764	DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2765	adev->ip_blocks[i].version->funcs->name, r);
2766	return r;
2767	}
2768	}
2769	}
2770
2771	return 0;
2772	}
2773
2774	int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2775	enum amd_powergating_state state)
2776	{
2777	int i, j, r;
2778
2779	if (amdgpu_emu_mode == 1)
2780	return 0;
2781
2782	for (j = 0; j < adev->num_ip_blocks; j++) {
2783	i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2784	if (!adev->ip_blocks[i].status.late_initialized)
2785	continue;
2786	/* skip PG for GFX, SDMA on S0ix */
2787	if (adev->in_s0ix &&
2788	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2789	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2790	continue;
2791	/* skip CG for VCE/UVD, it's handled specially */
2792	if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2793	adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2794	adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2795	adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2796	adev->ip_blocks[i].version->funcs->set_powergating_state) {
2797	/* enable powergating to save power */
2798	r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2799	state);
2800	if (r) {
2801	DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2802	adev->ip_blocks[i].version->funcs->name, r);
2803	return r;
2804	}
2805	}
2806	}
2807	return 0;
2808	}
2809
2810	static int amdgpu_device_enable_mgpu_fan_boost(void)
2811	{
2812	struct amdgpu_gpu_instance *gpu_ins;
2813	struct amdgpu_device *adev;
2814	int i, ret = 0;
2815
2816	mutex_lock(&mgpu_info.mutex);
2817
2818	/*
2819	* MGPU fan boost feature should be enabled
2820	* only when there are two or more dGPUs in
2821	* the system
2822	*/
2823	if (mgpu_info.num_dgpu < 2)
2824	goto out;
2825
2826	for (i = 0; i < mgpu_info.num_dgpu; i++) {
2827	gpu_ins = &(mgpu_info.gpu_ins[i]);
2828	adev = gpu_ins->adev;
2829	if (!(adev->flags & AMD_IS_APU) &&
2830	!gpu_ins->mgpu_fan_enabled) {
2831	ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2832	if (ret)
2833	break;
2834
2835	gpu_ins->mgpu_fan_enabled = 1;
2836	}
2837	}
2838
2839	out:
2840	mutex_unlock(lock: &mgpu_info.mutex);
2841
2842	return ret;
2843	}
2844
2845	/**
2846	* amdgpu_device_ip_late_init - run late init for hardware IPs
2847	*
2848	* @adev: amdgpu_device pointer
2849	*
2850	* Late initialization pass for hardware IPs. The list of all the hardware
2851	* IPs that make up the asic is walked and the late_init callbacks are run.
2852	* late_init covers any special initialization that an IP requires
2853	* after all of the have been initialized or something that needs to happen
2854	* late in the init process.
2855	* Returns 0 on success, negative error code on failure.
2856	*/
2857	static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2858	{
2859	struct amdgpu_gpu_instance *gpu_instance;
2860	int i = 0, r;
2861
2862	for (i = 0; i < adev->num_ip_blocks; i++) {
2863	if (!adev->ip_blocks[i].status.hw)
2864	continue;
2865	if (adev->ip_blocks[i].version->funcs->late_init) {
2866	r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2867	if (r) {
2868	DRM_ERROR("late_init of IP block <%s> failed %d\n",
2869	adev->ip_blocks[i].version->funcs->name, r);
2870	return r;
2871	}
2872	}
2873	adev->ip_blocks[i].status.late_initialized = true;
2874	}
2875
2876	r = amdgpu_ras_late_init(adev);
2877	if (r) {
2878	DRM_ERROR("amdgpu_ras_late_init failed %d", r);
2879	return r;
2880	}
2881
2882	amdgpu_ras_set_error_query_ready(adev, ready: true);
2883
2884	amdgpu_device_set_cg_state(adev, state: AMD_CG_STATE_GATE);
2885	amdgpu_device_set_pg_state(adev, state: AMD_PG_STATE_GATE);
2886
2887	amdgpu_device_fill_reset_magic(adev);
2888
2889	r = amdgpu_device_enable_mgpu_fan_boost();
2890	if (r)
2891	DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2892
2893	/* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2894	if (amdgpu_passthrough(adev) &&
2895	((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
2896	adev->asic_type == CHIP_ALDEBARAN))
2897	amdgpu_dpm_handle_passthrough_sbr(adev, enable: true);
2898
2899	if (adev->gmc.xgmi.num_physical_nodes > 1) {
2900	mutex_lock(&mgpu_info.mutex);
2901
2902	/*
2903	* Reset device p-state to low as this was booted with high.
2904	*
2905	* This should be performed only after all devices from the same
2906	* hive get initialized.
2907	*
2908	* However, it's unknown how many device in the hive in advance.
2909	* As this is counted one by one during devices initializations.
2910	*
2911	* So, we wait for all XGMI interlinked devices initialized.
2912	* This may bring some delays as those devices may come from
2913	* different hives. But that should be OK.
2914	*/
2915	if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2916	for (i = 0; i < mgpu_info.num_gpu; i++) {
2917	gpu_instance = &(mgpu_info.gpu_ins[i]);
2918	if (gpu_instance->adev->flags & AMD_IS_APU)
2919	continue;
2920
2921	r = amdgpu_xgmi_set_pstate(adev: gpu_instance->adev,
2922	pstate: AMDGPU_XGMI_PSTATE_MIN);
2923	if (r) {
2924	DRM_ERROR("pstate setting failed (%d).\n", r);
2925	break;
2926	}
2927	}
2928	}
2929
2930	mutex_unlock(lock: &mgpu_info.mutex);
2931	}
2932
2933	return 0;
2934	}
2935
2936	/**
2937	* amdgpu_device_smu_fini_early - smu hw_fini wrapper
2938	*
2939	* @adev: amdgpu_device pointer
2940	*
2941	* For ASICs need to disable SMC first
2942	*/
2943	static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2944	{
2945	int i, r;
2946
2947	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) > IP_VERSION(9, 0, 0))
2948	return;
2949
2950	for (i = 0; i < adev->num_ip_blocks; i++) {
2951	if (!adev->ip_blocks[i].status.hw)
2952	continue;
2953	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2954	r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2955	/* XXX handle errors */
2956	if (r) {
2957	DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2958	adev->ip_blocks[i].version->funcs->name, r);
2959	}
2960	adev->ip_blocks[i].status.hw = false;
2961	break;
2962	}
2963	}
2964	}
2965
2966	static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2967	{
2968	int i, r;
2969
2970	for (i = 0; i < adev->num_ip_blocks; i++) {
2971	if (!adev->ip_blocks[i].version->funcs->early_fini)
2972	continue;
2973
2974	r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2975	if (r) {
2976	DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2977	adev->ip_blocks[i].version->funcs->name, r);
2978	}
2979	}
2980
2981	amdgpu_device_set_pg_state(adev, state: AMD_PG_STATE_UNGATE);
2982	amdgpu_device_set_cg_state(adev, state: AMD_CG_STATE_UNGATE);
2983
2984	amdgpu_amdkfd_suspend(adev, run_pm: false);
2985
2986	/* Workaroud for ASICs need to disable SMC first */
2987	amdgpu_device_smu_fini_early(adev);
2988
2989	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2990	if (!adev->ip_blocks[i].status.hw)
2991	continue;
2992
2993	r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2994	/* XXX handle errors */
2995	if (r) {
2996	DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2997	adev->ip_blocks[i].version->funcs->name, r);
2998	}
2999
3000	adev->ip_blocks[i].status.hw = false;
3001	}
3002
3003	if (amdgpu_sriov_vf(adev)) {
3004	if (amdgpu_virt_release_full_gpu(adev, init: false))
3005	DRM_ERROR("failed to release exclusive mode on fini\n");
3006	}
3007
3008	return 0;
3009	}
3010
3011	/**
3012	* amdgpu_device_ip_fini - run fini for hardware IPs
3013	*
3014	* @adev: amdgpu_device pointer
3015	*
3016	* Main teardown pass for hardware IPs. The list of all the hardware
3017	* IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
3018	* are run. hw_fini tears down the hardware associated with each IP
3019	* and sw_fini tears down any software state associated with each IP.
3020	* Returns 0 on success, negative error code on failure.
3021	*/
3022	static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
3023	{
3024	int i, r;
3025
3026	if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
3027	amdgpu_virt_release_ras_err_handler_data(adev);
3028
3029	if (adev->gmc.xgmi.num_physical_nodes > 1)
3030	amdgpu_xgmi_remove_device(adev);
3031
3032	amdgpu_amdkfd_device_fini_sw(adev);
3033
3034	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3035	if (!adev->ip_blocks[i].status.sw)
3036	continue;
3037
3038	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
3039	amdgpu_ucode_free_bo(adev);
3040	amdgpu_free_static_csa(bo: &adev->virt.csa_obj);
3041	amdgpu_device_wb_fini(adev);
3042	amdgpu_device_mem_scratch_fini(adev);
3043	amdgpu_ib_pool_fini(adev);
3044	}
3045
3046	r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
3047	/* XXX handle errors */
3048	if (r) {
3049	DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
3050	adev->ip_blocks[i].version->funcs->name, r);
3051	}
3052	adev->ip_blocks[i].status.sw = false;
3053	adev->ip_blocks[i].status.valid = false;
3054	}
3055
3056	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3057	if (!adev->ip_blocks[i].status.late_initialized)
3058	continue;
3059	if (adev->ip_blocks[i].version->funcs->late_fini)
3060	adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
3061	adev->ip_blocks[i].status.late_initialized = false;
3062	}
3063
3064	amdgpu_ras_fini(adev);
3065
3066	return 0;
3067	}
3068
3069	/**
3070	* amdgpu_device_delayed_init_work_handler - work handler for IB tests
3071	*
3072	* @work: work_struct.
3073	*/
3074	static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3075	{
3076	struct amdgpu_device *adev =
3077	container_of(work, struct amdgpu_device, delayed_init_work.work);
3078	int r;
3079
3080	r = amdgpu_ib_ring_tests(adev);
3081	if (r)
3082	DRM_ERROR("ib ring test failed (%d).\n", r);
3083	}
3084
3085	static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3086	{
3087	struct amdgpu_device *adev =
3088	container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3089
3090	WARN_ON_ONCE(adev->gfx.gfx_off_state);
3091	WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3092
3093	if (!amdgpu_dpm_set_powergating_by_smu(adev, block_type: AMD_IP_BLOCK_TYPE_GFX, gate: true))
3094	adev->gfx.gfx_off_state = true;
3095	}
3096
3097	/**
3098	* amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3099	*
3100	* @adev: amdgpu_device pointer
3101	*
3102	* Main suspend function for hardware IPs. The list of all the hardware
3103	* IPs that make up the asic is walked, clockgating is disabled and the
3104	* suspend callbacks are run. suspend puts the hardware and software state
3105	* in each IP into a state suitable for suspend.
3106	* Returns 0 on success, negative error code on failure.
3107	*/
3108	static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3109	{
3110	int i, r;
3111
3112	amdgpu_device_set_pg_state(adev, state: AMD_PG_STATE_UNGATE);
3113	amdgpu_device_set_cg_state(adev, state: AMD_CG_STATE_UNGATE);
3114
3115	/*
3116	* Per PMFW team's suggestion, driver needs to handle gfxoff
3117	* and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3118	* scenario. Add the missing df cstate disablement here.
3119	*/
3120	if (amdgpu_dpm_set_df_cstate(adev, cstate: DF_CSTATE_DISALLOW))
3121	dev_warn(adev->dev, "Failed to disallow df cstate");
3122
3123	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3124	if (!adev->ip_blocks[i].status.valid)
3125	continue;
3126
3127	/* displays are handled separately */
3128	if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3129	continue;
3130
3131	/* XXX handle errors */
3132	r = adev->ip_blocks[i].version->funcs->suspend(adev);
3133	/* XXX handle errors */
3134	if (r) {
3135	DRM_ERROR("suspend of IP block <%s> failed %d\n",
3136	adev->ip_blocks[i].version->funcs->name, r);
3137	return r;
3138	}
3139
3140	adev->ip_blocks[i].status.hw = false;
3141	}
3142
3143	return 0;
3144	}
3145
3146	/**
3147	* amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3148	*
3149	* @adev: amdgpu_device pointer
3150	*
3151	* Main suspend function for hardware IPs. The list of all the hardware
3152	* IPs that make up the asic is walked, clockgating is disabled and the
3153	* suspend callbacks are run. suspend puts the hardware and software state
3154	* in each IP into a state suitable for suspend.
3155	* Returns 0 on success, negative error code on failure.
3156	*/
3157	static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3158	{
3159	int i, r;
3160
3161	if (adev->in_s0ix)
3162	amdgpu_dpm_gfx_state_change(adev, state: sGpuChangeState_D3Entry);
3163
3164	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3165	if (!adev->ip_blocks[i].status.valid)
3166	continue;
3167	/* displays are handled in phase1 */
3168	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3169	continue;
3170	/* PSP lost connection when err_event_athub occurs */
3171	if (amdgpu_ras_intr_triggered() &&
3172	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3173	adev->ip_blocks[i].status.hw = false;
3174	continue;
3175	}
3176
3177	/* skip unnecessary suspend if we do not initialize them yet */
3178	if (adev->gmc.xgmi.pending_reset &&
3179	!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3180	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3181	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3182	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3183	adev->ip_blocks[i].status.hw = false;
3184	continue;
3185	}
3186
3187	/* skip suspend of gfx/mes and psp for S0ix
3188	* gfx is in gfxoff state, so on resume it will exit gfxoff just
3189	* like at runtime. PSP is also part of the always on hardware
3190	* so no need to suspend it.
3191	*/
3192	if (adev->in_s0ix &&
3193	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3194	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3195	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3196	continue;
3197
3198	/* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3199	if (adev->in_s0ix &&
3200	(amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) >=
3201	IP_VERSION(5, 0, 0)) &&
3202	(adev->ip_blocks[i].version->type ==
3203	AMD_IP_BLOCK_TYPE_SDMA))
3204	continue;
3205
3206	/* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3207	* These are in TMR, hence are expected to be reused by PSP-TOS to reload
3208	* from this location and RLC Autoload automatically also gets loaded
3209	* from here based on PMFW -> PSP message during re-init sequence.
3210	* Therefore, the psp suspend & resume should be skipped to avoid destroy
3211	* the TMR and reload FWs again for IMU enabled APU ASICs.
3212	*/
3213	if (amdgpu_in_reset(adev) &&
3214	(adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3215	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3216	continue;
3217
3218	/* XXX handle errors */
3219	r = adev->ip_blocks[i].version->funcs->suspend(adev);
3220	/* XXX handle errors */
3221	if (r) {
3222	DRM_ERROR("suspend of IP block <%s> failed %d\n",
3223	adev->ip_blocks[i].version->funcs->name, r);
3224	}
3225	adev->ip_blocks[i].status.hw = false;
3226	/* handle putting the SMC in the appropriate state */
3227	if (!amdgpu_sriov_vf(adev)) {
3228	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3229	r = amdgpu_dpm_set_mp1_state(adev, mp1_state: adev->mp1_state);
3230	if (r) {
3231	DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3232	adev->mp1_state, r);
3233	return r;
3234	}
3235	}
3236	}
3237	}
3238
3239	return 0;
3240	}
3241
3242	/**
3243	* amdgpu_device_ip_suspend - run suspend for hardware IPs
3244	*
3245	* @adev: amdgpu_device pointer
3246	*
3247	* Main suspend function for hardware IPs. The list of all the hardware
3248	* IPs that make up the asic is walked, clockgating is disabled and the
3249	* suspend callbacks are run. suspend puts the hardware and software state
3250	* in each IP into a state suitable for suspend.
3251	* Returns 0 on success, negative error code on failure.
3252	*/
3253	int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3254	{
3255	int r;
3256
3257	if (amdgpu_sriov_vf(adev)) {
3258	amdgpu_virt_fini_data_exchange(adev);
3259	amdgpu_virt_request_full_gpu(adev, init: false);
3260	}
3261
3262	amdgpu_ttm_set_buffer_funcs_status(adev, enable: false);
3263
3264	r = amdgpu_device_ip_suspend_phase1(adev);
3265	if (r)
3266	return r;
3267	r = amdgpu_device_ip_suspend_phase2(adev);
3268
3269	if (amdgpu_sriov_vf(adev))
3270	amdgpu_virt_release_full_gpu(adev, init: false);
3271
3272	return r;
3273	}
3274
3275	static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3276	{
3277	int i, r;
3278
3279	static enum amd_ip_block_type ip_order[] = {
3280	AMD_IP_BLOCK_TYPE_COMMON,
3281	AMD_IP_BLOCK_TYPE_GMC,
3282	AMD_IP_BLOCK_TYPE_PSP,
3283	AMD_IP_BLOCK_TYPE_IH,
3284	};
3285
3286	for (i = 0; i < adev->num_ip_blocks; i++) {
3287	int j;
3288	struct amdgpu_ip_block *block;
3289
3290	block = &adev->ip_blocks[i];
3291	block->status.hw = false;
3292
3293	for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3294
3295	if (block->version->type != ip_order[j] ||
3296	!block->status.valid)
3297	continue;
3298
3299	r = block->version->funcs->hw_init(adev);
3300	DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3301	if (r)
3302	return r;
3303	block->status.hw = true;
3304	}
3305	}
3306
3307	return 0;
3308	}
3309
3310	static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3311	{
3312	int i, r;
3313
3314	static enum amd_ip_block_type ip_order[] = {
3315	AMD_IP_BLOCK_TYPE_SMC,
3316	AMD_IP_BLOCK_TYPE_DCE,
3317	AMD_IP_BLOCK_TYPE_GFX,
3318	AMD_IP_BLOCK_TYPE_SDMA,
3319	AMD_IP_BLOCK_TYPE_MES,
3320	AMD_IP_BLOCK_TYPE_UVD,
3321	AMD_IP_BLOCK_TYPE_VCE,
3322	AMD_IP_BLOCK_TYPE_VCN,
3323	AMD_IP_BLOCK_TYPE_JPEG
3324	};
3325
3326	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3327	int j;
3328	struct amdgpu_ip_block *block;
3329
3330	for (j = 0; j < adev->num_ip_blocks; j++) {
3331	block = &adev->ip_blocks[j];
3332
3333	if (block->version->type != ip_order[i] ||
3334	!block->status.valid ||
3335	block->status.hw)
3336	continue;
3337
3338	if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3339	r = block->version->funcs->resume(adev);
3340	else
3341	r = block->version->funcs->hw_init(adev);
3342
3343	DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3344	if (r)
3345	return r;
3346	block->status.hw = true;
3347	}
3348	}
3349
3350	return 0;
3351	}
3352
3353	/**
3354	* amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3355	*
3356	* @adev: amdgpu_device pointer
3357	*
3358	* First resume function for hardware IPs. The list of all the hardware
3359	* IPs that make up the asic is walked and the resume callbacks are run for
3360	* COMMON, GMC, and IH. resume puts the hardware into a functional state
3361	* after a suspend and updates the software state as necessary. This
3362	* function is also used for restoring the GPU after a GPU reset.
3363	* Returns 0 on success, negative error code on failure.
3364	*/
3365	static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3366	{
3367	int i, r;
3368
3369	for (i = 0; i < adev->num_ip_blocks; i++) {
3370	if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3371	continue;
3372	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3373	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3374	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3375	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3376
3377	r = adev->ip_blocks[i].version->funcs->resume(adev);
3378	if (r) {
3379	DRM_ERROR("resume of IP block <%s> failed %d\n",
3380	adev->ip_blocks[i].version->funcs->name, r);
3381	return r;
3382	}
3383	adev->ip_blocks[i].status.hw = true;
3384	}
3385	}
3386
3387	return 0;
3388	}
3389
3390	/**
3391	* amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3392	*
3393	* @adev: amdgpu_device pointer
3394	*
3395	* First resume function for hardware IPs. The list of all the hardware
3396	* IPs that make up the asic is walked and the resume callbacks are run for
3397	* all blocks except COMMON, GMC, and IH. resume puts the hardware into a
3398	* functional state after a suspend and updates the software state as
3399	* necessary. This function is also used for restoring the GPU after a GPU
3400	* reset.
3401	* Returns 0 on success, negative error code on failure.
3402	*/
3403	static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3404	{
3405	int i, r;
3406
3407	for (i = 0; i < adev->num_ip_blocks; i++) {
3408	if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3409	continue;
3410	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3411	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3412	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3413	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3414	continue;
3415	r = adev->ip_blocks[i].version->funcs->resume(adev);
3416	if (r) {
3417	DRM_ERROR("resume of IP block <%s> failed %d\n",
3418	adev->ip_blocks[i].version->funcs->name, r);
3419	return r;
3420	}
3421	adev->ip_blocks[i].status.hw = true;
3422	}
3423
3424	return 0;
3425	}
3426
3427	/**
3428	* amdgpu_device_ip_resume - run resume for hardware IPs
3429	*
3430	* @adev: amdgpu_device pointer
3431	*
3432	* Main resume function for hardware IPs. The hardware IPs
3433	* are split into two resume functions because they are
3434	* also used in recovering from a GPU reset and some additional
3435	* steps need to be take between them. In this case (S3/S4) they are
3436	* run sequentially.
3437	* Returns 0 on success, negative error code on failure.
3438	*/
3439	static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3440	{
3441	int r;
3442
3443	r = amdgpu_device_ip_resume_phase1(adev);
3444	if (r)
3445	return r;
3446
3447	r = amdgpu_device_fw_loading(adev);
3448	if (r)
3449	return r;
3450
3451	r = amdgpu_device_ip_resume_phase2(adev);
3452
3453	if (adev->mman.buffer_funcs_ring->sched.ready)
3454	amdgpu_ttm_set_buffer_funcs_status(adev, enable: true);
3455
3456	return r;
3457	}
3458
3459	/**
3460	* amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3461	*
3462	* @adev: amdgpu_device pointer
3463	*
3464	* Query the VBIOS data tables to determine if the board supports SR-IOV.
3465	*/
3466	static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3467	{
3468	if (amdgpu_sriov_vf(adev)) {
3469	if (adev->is_atom_fw) {
3470	if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3471	adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3472	} else {
3473	if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3474	adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3475	}
3476
3477	if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3478	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_NO_VBIOS, error_flags: 0, error_data: 0);
3479	}
3480	}
3481
3482	/**
3483	* amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3484	*
3485	* @asic_type: AMD asic type
3486	*
3487	* Check if there is DC (new modesetting infrastructre) support for an asic.
3488	* returns true if DC has support, false if not.
3489	*/
3490	bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3491	{
3492	switch (asic_type) {
3493	#ifdef CONFIG_DRM_AMDGPU_SI
3494	case CHIP_HAINAN:
3495	#endif
3496	case CHIP_TOPAZ:
3497	/* chips with no display hardware */
3498	return false;
3499	#if defined(CONFIG_DRM_AMD_DC)
3500	case CHIP_TAHITI:
3501	case CHIP_PITCAIRN:
3502	case CHIP_VERDE:
3503	case CHIP_OLAND:
3504	/*
3505	* We have systems in the wild with these ASICs that require
3506	* LVDS and VGA support which is not supported with DC.
3507	*
3508	* Fallback to the non-DC driver here by default so as not to
3509	* cause regressions.
3510	*/
3511	#if defined(CONFIG_DRM_AMD_DC_SI)
3512	return amdgpu_dc > 0;
3513	#else
3514	return false;
3515	#endif
3516	case CHIP_BONAIRE:
3517	case CHIP_KAVERI:
3518	case CHIP_KABINI:
3519	case CHIP_MULLINS:
3520	/*
3521	* We have systems in the wild with these ASICs that require
3522	* VGA support which is not supported with DC.
3523	*
3524	* Fallback to the non-DC driver here by default so as not to
3525	* cause regressions.
3526	*/
3527	return amdgpu_dc > 0;
3528	default:
3529	return amdgpu_dc != 0;
3530	#else
3531	default:
3532	if (amdgpu_dc > 0)
3533	DRM_INFO_ONCE("Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n");
3534	return false;
3535	#endif
3536	}
3537	}
3538
3539	/**
3540	* amdgpu_device_has_dc_support - check if dc is supported
3541	*
3542	* @adev: amdgpu_device pointer
3543	*
3544	* Returns true for supported, false for not supported
3545	*/
3546	bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3547	{
3548	if (adev->enable_virtual_display ||
3549	(adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3550	return false;
3551
3552	return amdgpu_device_asic_has_dc_support(asic_type: adev->asic_type);
3553	}
3554
3555	static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3556	{
3557	struct amdgpu_device *adev =
3558	container_of(__work, struct amdgpu_device, xgmi_reset_work);
3559	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3560
3561	/* It's a bug to not have a hive within this function */
3562	if (WARN_ON(!hive))
3563	return;
3564
3565	/*
3566	* Use task barrier to synchronize all xgmi reset works across the
3567	* hive. task_barrier_enter and task_barrier_exit will block
3568	* until all the threads running the xgmi reset works reach
3569	* those points. task_barrier_full will do both blocks.
3570	*/
3571	if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3572
3573	task_barrier_enter(tb: &hive->tb);
3574	adev->asic_reset_res = amdgpu_device_baco_enter(dev: adev_to_drm(adev));
3575
3576	if (adev->asic_reset_res)
3577	goto fail;
3578
3579	task_barrier_exit(tb: &hive->tb);
3580	adev->asic_reset_res = amdgpu_device_baco_exit(dev: adev_to_drm(adev));
3581
3582	if (adev->asic_reset_res)
3583	goto fail;
3584
3585	amdgpu_ras_reset_error_count(adev, block: AMDGPU_RAS_BLOCK__MMHUB);
3586	} else {
3587
3588	task_barrier_full(tb: &hive->tb);
3589	adev->asic_reset_res = amdgpu_asic_reset(adev);
3590	}
3591
3592	fail:
3593	if (adev->asic_reset_res)
3594	DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3595	adev->asic_reset_res, adev_to_drm(adev)->unique);
3596	amdgpu_put_xgmi_hive(hive);
3597	}
3598
3599	static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3600	{
3601	char *input = amdgpu_lockup_timeout;
3602	char *timeout_setting = NULL;
3603	int index = 0;
3604	long timeout;
3605	int ret = 0;
3606
3607	/*
3608	* By default timeout for non compute jobs is 10000
3609	* and 60000 for compute jobs.
3610	* In SR-IOV or passthrough mode, timeout for compute
3611	* jobs are 60000 by default.
3612	*/
3613	adev->gfx_timeout = msecs_to_jiffies(m: 10000);
3614	adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3615	if (amdgpu_sriov_vf(adev))
3616	adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3617	msecs_to_jiffies(m: 60000) : msecs_to_jiffies(m: 10000);
3618	else
3619	adev->compute_timeout = msecs_to_jiffies(m: 60000);
3620
3621	if (strnlen(p: input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3622	while ((timeout_setting = strsep(&input, ",")) &&
3623	strnlen(p: timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3624	ret = kstrtol(s: timeout_setting, base: 0, res: &timeout);
3625	if (ret)
3626	return ret;
3627
3628	if (timeout == 0) {
3629	index++;
3630	continue;
3631	} else if (timeout < 0) {
3632	timeout = MAX_SCHEDULE_TIMEOUT;
3633	dev_warn(adev->dev, "lockup timeout disabled");
3634	add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3635	} else {
3636	timeout = msecs_to_jiffies(m: timeout);
3637	}
3638
3639	switch (index++) {
3640	case 0:
3641	adev->gfx_timeout = timeout;
3642	break;
3643	case 1:
3644	adev->compute_timeout = timeout;
3645	break;
3646	case 2:
3647	adev->sdma_timeout = timeout;
3648	break;
3649	case 3:
3650	adev->video_timeout = timeout;
3651	break;
3652	default:
3653	break;
3654	}
3655	}
3656	/*
3657	* There is only one value specified and
3658	* it should apply to all non-compute jobs.
3659	*/
3660	if (index == 1) {
3661	adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3662	if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3663	adev->compute_timeout = adev->gfx_timeout;
3664	}
3665	}
3666
3667	return ret;
3668	}
3669
3670	/**
3671	* amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3672	*
3673	* @adev: amdgpu_device pointer
3674	*
3675	* RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3676	*/
3677	static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3678	{
3679	struct iommu_domain *domain;
3680
3681	domain = iommu_get_domain_for_dev(dev: adev->dev);
3682	if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3683	adev->ram_is_direct_mapped = true;
3684	}
3685
3686	static const struct attribute *amdgpu_dev_attributes[] = {
3687	&dev_attr_pcie_replay_count.attr,
3688	NULL
3689	};
3690
3691	static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
3692	{
3693	if (amdgpu_mcbp == 1)
3694	adev->gfx.mcbp = true;
3695	else if (amdgpu_mcbp == 0)
3696	adev->gfx.mcbp = false;
3697	else if ((amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >= IP_VERSION(9, 0, 0)) &&
3698	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) < IP_VERSION(10, 0, 0)) &&
3699	adev->gfx.num_gfx_rings)
3700	adev->gfx.mcbp = true;
3701
3702	if (amdgpu_sriov_vf(adev))
3703	adev->gfx.mcbp = true;
3704
3705	if (adev->gfx.mcbp)
3706	DRM_INFO("MCBP is enabled\n");
3707	}
3708
3709	/**
3710	* amdgpu_device_init - initialize the driver
3711	*
3712	* @adev: amdgpu_device pointer
3713	* @flags: driver flags
3714	*
3715	* Initializes the driver info and hw (all asics).
3716	* Returns 0 for success or an error on failure.
3717	* Called at driver startup.
3718	*/
3719	int amdgpu_device_init(struct amdgpu_device *adev,
3720	uint32_t flags)
3721	{
3722	struct drm_device *ddev = adev_to_drm(adev);
3723	struct pci_dev *pdev = adev->pdev;
3724	int r, i;
3725	bool px = false;
3726	u32 max_MBps;
3727	int tmp;
3728
3729	adev->shutdown = false;
3730	adev->flags = flags;
3731
3732	if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3733	adev->asic_type = amdgpu_force_asic_type;
3734	else
3735	adev->asic_type = flags & AMD_ASIC_MASK;
3736
3737	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3738	if (amdgpu_emu_mode == 1)
3739	adev->usec_timeout *= 10;
3740	adev->gmc.gart_size = 512 * 1024 * 1024;
3741	adev->accel_working = false;
3742	adev->num_rings = 0;
3743	RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3744	adev->mman.buffer_funcs = NULL;
3745	adev->mman.buffer_funcs_ring = NULL;
3746	adev->vm_manager.vm_pte_funcs = NULL;
3747	adev->vm_manager.vm_pte_num_scheds = 0;
3748	adev->gmc.gmc_funcs = NULL;
3749	adev->harvest_ip_mask = 0x0;
3750	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3751	bitmap_zero(dst: adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3752
3753	adev->smc_rreg = &amdgpu_invalid_rreg;
3754	adev->smc_wreg = &amdgpu_invalid_wreg;
3755	adev->pcie_rreg = &amdgpu_invalid_rreg;
3756	adev->pcie_wreg = &amdgpu_invalid_wreg;
3757	adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;
3758	adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;
3759	adev->pciep_rreg = &amdgpu_invalid_rreg;
3760	adev->pciep_wreg = &amdgpu_invalid_wreg;
3761	adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3762	adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3763	adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext;
3764	adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext;
3765	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3766	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3767	adev->didt_rreg = &amdgpu_invalid_rreg;
3768	adev->didt_wreg = &amdgpu_invalid_wreg;
3769	adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3770	adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3771	adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3772	adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3773
3774	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3775	amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3776	pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3777
3778	/* mutex initialization are all done here so we
3779	* can recall function without having locking issues
3780	*/
3781	mutex_init(&adev->firmware.mutex);
3782	mutex_init(&adev->pm.mutex);
3783	mutex_init(&adev->gfx.gpu_clock_mutex);
3784	mutex_init(&adev->srbm_mutex);
3785	mutex_init(&adev->gfx.pipe_reserve_mutex);
3786	mutex_init(&adev->gfx.gfx_off_mutex);
3787	mutex_init(&adev->gfx.partition_mutex);
3788	mutex_init(&adev->grbm_idx_mutex);
3789	mutex_init(&adev->mn_lock);
3790	mutex_init(&adev->virt.vf_errors.lock);
3791	hash_init(adev->mn_hash);
3792	mutex_init(&adev->psp.mutex);
3793	mutex_init(&adev->notifier_lock);
3794	mutex_init(&adev->pm.stable_pstate_ctx_lock);
3795	mutex_init(&adev->benchmark_mutex);
3796
3797	amdgpu_device_init_apu_flags(adev);
3798
3799	r = amdgpu_device_check_arguments(adev);
3800	if (r)
3801	return r;
3802
3803	spin_lock_init(&adev->mmio_idx_lock);
3804	spin_lock_init(&adev->smc_idx_lock);
3805	spin_lock_init(&adev->pcie_idx_lock);
3806	spin_lock_init(&adev->uvd_ctx_idx_lock);
3807	spin_lock_init(&adev->didt_idx_lock);
3808	spin_lock_init(&adev->gc_cac_idx_lock);
3809	spin_lock_init(&adev->se_cac_idx_lock);
3810	spin_lock_init(&adev->audio_endpt_idx_lock);
3811	spin_lock_init(&adev->mm_stats.lock);
3812
3813	INIT_LIST_HEAD(list: &adev->shadow_list);
3814	mutex_init(&adev->shadow_list_lock);
3815
3816	INIT_LIST_HEAD(list: &adev->reset_list);
3817
3818	INIT_LIST_HEAD(list: &adev->ras_list);
3819
3820	INIT_LIST_HEAD(list: &adev->pm.od_kobj_list);
3821
3822	INIT_DELAYED_WORK(&adev->delayed_init_work,
3823	amdgpu_device_delayed_init_work_handler);
3824	INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3825	amdgpu_device_delay_enable_gfx_off);
3826
3827	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3828
3829	adev->gfx.gfx_off_req_count = 1;
3830	adev->gfx.gfx_off_residency = 0;
3831	adev->gfx.gfx_off_entrycount = 0;
3832	adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3833
3834	atomic_set(v: &adev->throttling_logging_enabled, i: 1);
3835	/*
3836	* If throttling continues, logging will be performed every minute
3837	* to avoid log flooding. "-1" is subtracted since the thermal
3838	* throttling interrupt comes every second. Thus, the total logging
3839	* interval is 59 seconds(retelimited printk interval) + 1(waiting
3840	* for throttling interrupt) = 60 seconds.
3841	*/
3842	ratelimit_state_init(rs: &adev->throttling_logging_rs, interval: (60 - 1) * HZ, burst: 1);
3843	ratelimit_set_flags(rs: &adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3844
3845	/* Registers mapping */
3846	/* TODO: block userspace mapping of io register */
3847	if (adev->asic_type >= CHIP_BONAIRE) {
3848	adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3849	adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3850	} else {
3851	adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3852	adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3853	}
3854
3855	for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3856	atomic_set(v: &adev->pm.pwr_state[i], i: POWER_STATE_UNKNOWN);
3857
3858	adev->rmmio = ioremap(offset: adev->rmmio_base, size: adev->rmmio_size);
3859	if (!adev->rmmio)
3860	return -ENOMEM;
3861
3862	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3863	DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size);
3864
3865	/*
3866	* Reset domain needs to be present early, before XGMI hive discovered
3867	* (if any) and intitialized to use reset sem and in_gpu reset flag
3868	* early on during init and before calling to RREG32.
3869	*/
3870	adev->reset_domain = amdgpu_reset_create_reset_domain(type: SINGLE_DEVICE, wq_name: "amdgpu-reset-dev");
3871	if (!adev->reset_domain)
3872	return -ENOMEM;
3873
3874	/* detect hw virtualization here */
3875	amdgpu_detect_virtualization(adev);
3876
3877	amdgpu_device_get_pcie_info(adev);
3878
3879	r = amdgpu_device_get_job_timeout_settings(adev);
3880	if (r) {
3881	dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3882	return r;
3883	}
3884
3885	/* early init functions */
3886	r = amdgpu_device_ip_early_init(adev);
3887	if (r)
3888	return r;
3889
3890	amdgpu_device_set_mcbp(adev);
3891
3892	/* Get rid of things like offb */
3893	r = drm_aperture_remove_conflicting_pci_framebuffers(pdev: adev->pdev, req_driver: &amdgpu_kms_driver);
3894	if (r)
3895	return r;
3896
3897	/* Enable TMZ based on IP_VERSION */
3898	amdgpu_gmc_tmz_set(adev);
3899
3900	amdgpu_gmc_noretry_set(adev);
3901	/* Need to get xgmi info early to decide the reset behavior*/
3902	if (adev->gmc.xgmi.supported) {
3903	r = adev->gfxhub.funcs->get_xgmi_info(adev);
3904	if (r)
3905	return r;
3906	}
3907
3908	/* enable PCIE atomic ops */
3909	if (amdgpu_sriov_vf(adev)) {
3910	if (adev->virt.fw_reserve.p_pf2vf)
3911	adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
3912	adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
3913	(PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3914	/* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
3915	* internal path natively support atomics, set have_atomics_support to true.
3916	*/
3917	} else if ((adev->flags & AMD_IS_APU) &&
3918	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >
3919	IP_VERSION(9, 0, 0))) {
3920	adev->have_atomics_support = true;
3921	} else {
3922	adev->have_atomics_support =
3923	!pci_enable_atomic_ops_to_root(dev: adev->pdev,
3924	PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3925	PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3926	}
3927
3928	if (!adev->have_atomics_support)
3929	dev_info(adev->dev, "PCIE atomic ops is not supported\n");
3930
3931	/* doorbell bar mapping and doorbell index init*/
3932	amdgpu_doorbell_init(adev);
3933
3934	if (amdgpu_emu_mode == 1) {
3935	/* post the asic on emulation mode */
3936	emu_soc_asic_init(adev);
3937	goto fence_driver_init;
3938	}
3939
3940	amdgpu_reset_init(adev);
3941
3942	/* detect if we are with an SRIOV vbios */
3943	if (adev->bios)
3944	amdgpu_device_detect_sriov_bios(adev);
3945
3946	/* check if we need to reset the asic
3947	* E.g., driver was not cleanly unloaded previously, etc.
3948	*/
3949	if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3950	if (adev->gmc.xgmi.num_physical_nodes) {
3951	dev_info(adev->dev, "Pending hive reset.\n");
3952	adev->gmc.xgmi.pending_reset = true;
3953	/* Only need to init necessary block for SMU to handle the reset */
3954	for (i = 0; i < adev->num_ip_blocks; i++) {
3955	if (!adev->ip_blocks[i].status.valid)
3956	continue;
3957	if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3958	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3959	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3960	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3961	DRM_DEBUG("IP %s disabled for hw_init.\n",
3962	adev->ip_blocks[i].version->funcs->name);
3963	adev->ip_blocks[i].status.hw = true;
3964	}
3965	}
3966	} else {
3967	switch (amdgpu_ip_version(adev, ip: MP1_HWIP, inst: 0)) {
3968	case IP_VERSION(13, 0, 0):
3969	case IP_VERSION(13, 0, 7):
3970	case IP_VERSION(13, 0, 10):
3971	r = psp_gpu_reset(adev);
3972	break;
3973	default:
3974	tmp = amdgpu_reset_method;
3975	/* It should do a default reset when loading or reloading the driver,
3976	* regardless of the module parameter reset_method.
3977	*/
3978	amdgpu_reset_method = AMD_RESET_METHOD_NONE;
3979	r = amdgpu_asic_reset(adev);
3980	amdgpu_reset_method = tmp;
3981	break;
3982	}
3983
3984	if (r) {
3985	dev_err(adev->dev, "asic reset on init failed\n");
3986	goto failed;
3987	}
3988	}
3989	}
3990
3991	/* Post card if necessary */
3992	if (amdgpu_device_need_post(adev)) {
3993	if (!adev->bios) {
3994	dev_err(adev->dev, "no vBIOS found\n");
3995	r = -EINVAL;
3996	goto failed;
3997	}
3998	DRM_INFO("GPU posting now...\n");
3999	r = amdgpu_device_asic_init(adev);
4000	if (r) {
4001	dev_err(adev->dev, "gpu post error!\n");
4002	goto failed;
4003	}
4004	}
4005
4006	if (adev->bios) {
4007	if (adev->is_atom_fw) {
4008	/* Initialize clocks */
4009	r = amdgpu_atomfirmware_get_clock_info(adev);
4010	if (r) {
4011	dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
4012	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, error_flags: 0, error_data: 0);
4013	goto failed;
4014	}
4015	} else {
4016	/* Initialize clocks */
4017	r = amdgpu_atombios_get_clock_info(adev);
4018	if (r) {
4019	dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4020	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, error_flags: 0, error_data: 0);
4021	goto failed;
4022	}
4023	/* init i2c buses */
4024	if (!amdgpu_device_has_dc_support(adev))
4025	amdgpu_atombios_i2c_init(adev);
4026	}
4027	}
4028
4029	fence_driver_init:
4030	/* Fence driver */
4031	r = amdgpu_fence_driver_sw_init(adev);
4032	if (r) {
4033	dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4034	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_FENCE_INIT_FAIL, error_flags: 0, error_data: 0);
4035	goto failed;
4036	}
4037
4038	/* init the mode config */
4039	drm_mode_config_init(dev: adev_to_drm(adev));
4040
4041	r = amdgpu_device_ip_init(adev);
4042	if (r) {
4043	dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4044	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, error_flags: 0, error_data: 0);
4045	goto release_ras_con;
4046	}
4047
4048	amdgpu_fence_driver_hw_init(adev);
4049
4050	dev_info(adev->dev,
4051	"SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4052	adev->gfx.config.max_shader_engines,
4053	adev->gfx.config.max_sh_per_se,
4054	adev->gfx.config.max_cu_per_sh,
4055	adev->gfx.cu_info.number);
4056
4057	adev->accel_working = true;
4058
4059	amdgpu_vm_check_compute_bug(adev);
4060
4061	/* Initialize the buffer migration limit. */
4062	if (amdgpu_moverate >= 0)
4063	max_MBps = amdgpu_moverate;
4064	else
4065	max_MBps = 8; /* Allow 8 MB/s. */
4066	/* Get a log2 for easy divisions. */
4067	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4068
4069	/*
4070	* Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4071	* Otherwise the mgpu fan boost feature will be skipped due to the
4072	* gpu instance is counted less.
4073	*/
4074	amdgpu_register_gpu_instance(adev);
4075
4076	/* enable clockgating, etc. after ib tests, etc. since some blocks require
4077	* explicit gating rather than handling it automatically.
4078	*/
4079	if (!adev->gmc.xgmi.pending_reset) {
4080	r = amdgpu_device_ip_late_init(adev);
4081	if (r) {
4082	dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4083	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, error_flags: 0, error_data: r);
4084	goto release_ras_con;
4085	}
4086	/* must succeed. */
4087	amdgpu_ras_resume(adev);
4088	queue_delayed_work(wq: system_wq, dwork: &adev->delayed_init_work,
4089	delay: msecs_to_jiffies(AMDGPU_RESUME_MS));
4090	}
4091
4092	if (amdgpu_sriov_vf(adev)) {
4093	amdgpu_virt_release_full_gpu(adev, init: true);
4094	flush_delayed_work(dwork: &adev->delayed_init_work);
4095	}
4096
4097	/*
4098	* Place those sysfs registering after `late_init`. As some of those
4099	* operations performed in `late_init` might affect the sysfs
4100	* interfaces creating.
4101	*/
4102	r = amdgpu_atombios_sysfs_init(adev);
4103	if (r)
4104	drm_err(&adev->ddev,
4105	"registering atombios sysfs failed (%d).\n", r);
4106
4107	r = amdgpu_pm_sysfs_init(adev);
4108	if (r)
4109	DRM_ERROR("registering pm sysfs failed (%d).\n", r);
4110
4111	r = amdgpu_ucode_sysfs_init(adev);
4112	if (r) {
4113	adev->ucode_sysfs_en = false;
4114	DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
4115	} else
4116	adev->ucode_sysfs_en = true;
4117
4118	r = sysfs_create_files(kobj: &adev->dev->kobj, attr: amdgpu_dev_attributes);
4119	if (r)
4120	dev_err(adev->dev, "Could not create amdgpu device attr\n");
4121
4122	r = devm_device_add_group(dev: adev->dev, grp: &amdgpu_board_attrs_group);
4123	if (r)
4124	dev_err(adev->dev,
4125	"Could not create amdgpu board attributes\n");
4126
4127	amdgpu_fru_sysfs_init(adev);
4128
4129	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4130	r = amdgpu_pmu_init(adev);
4131	if (r)
4132	dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4133
4134	/* Have stored pci confspace at hand for restore in sudden PCI error */
4135	if (amdgpu_device_cache_pci_state(pdev: adev->pdev))
4136	pci_restore_state(dev: pdev);
4137
4138	/* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4139	/* this will fail for cards that aren't VGA class devices, just
4140	* ignore it
4141	*/
4142	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4143	vga_client_register(pdev: adev->pdev, set_decode: amdgpu_device_vga_set_decode);
4144
4145	px = amdgpu_device_supports_px(dev: ddev);
4146
4147	if (px || (!dev_is_removable(dev: &adev->pdev->dev) &&
4148	apple_gmux_detect(NULL, NULL)))
4149	vga_switcheroo_register_client(dev: adev->pdev,
4150	ops: &amdgpu_switcheroo_ops, driver_power_control: px);
4151
4152	if (px)
4153	vga_switcheroo_init_domain_pm_ops(dev: adev->dev, domain: &adev->vga_pm_domain);
4154
4155	if (adev->gmc.xgmi.pending_reset)
4156	queue_delayed_work(wq: system_wq, dwork: &mgpu_info.delayed_reset_work,
4157	delay: msecs_to_jiffies(AMDGPU_RESUME_MS));
4158
4159	amdgpu_device_check_iommu_direct_map(adev);
4160
4161	return 0;
4162
4163	release_ras_con:
4164	if (amdgpu_sriov_vf(adev))
4165	amdgpu_virt_release_full_gpu(adev, init: true);
4166
4167	/* failed in exclusive mode due to timeout */
4168	if (amdgpu_sriov_vf(adev) &&
4169	!amdgpu_sriov_runtime(adev) &&
4170	amdgpu_virt_mmio_blocked(adev) &&
4171	!amdgpu_virt_wait_reset(adev)) {
4172	dev_err(adev->dev, "VF exclusive mode timeout\n");
4173	/* Don't send request since VF is inactive. */
4174	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4175	adev->virt.ops = NULL;
4176	r = -EAGAIN;
4177	}
4178	amdgpu_release_ras_context(adev);
4179
4180	failed:
4181	amdgpu_vf_error_trans_all(adev);
4182
4183	return r;
4184	}
4185
4186	static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4187	{
4188
4189	/* Clear all CPU mappings pointing to this device */
4190	unmap_mapping_range(mapping: adev->ddev.anon_inode->i_mapping, holebegin: 0, holelen: 0, even_cows: 1);
4191
4192	/* Unmap all mapped bars - Doorbell, registers and VRAM */
4193	amdgpu_doorbell_fini(adev);
4194
4195	iounmap(addr: adev->rmmio);
4196	adev->rmmio = NULL;
4197	if (adev->mman.aper_base_kaddr)
4198	iounmap(addr: adev->mman.aper_base_kaddr);
4199	adev->mman.aper_base_kaddr = NULL;
4200
4201	/* Memory manager related */
4202	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4203	arch_phys_wc_del(handle: adev->gmc.vram_mtrr);
4204	arch_io_free_memtype_wc(start: adev->gmc.aper_base, size: adev->gmc.aper_size);
4205	}
4206	}
4207
4208	/**
4209	* amdgpu_device_fini_hw - tear down the driver
4210	*
4211	* @adev: amdgpu_device pointer
4212	*
4213	* Tear down the driver info (all asics).
4214	* Called at driver shutdown.
4215	*/
4216	void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4217	{
4218	dev_info(adev->dev, "amdgpu: finishing device.\n");
4219	flush_delayed_work(dwork: &adev->delayed_init_work);
4220	adev->shutdown = true;
4221
4222	/* make sure IB test finished before entering exclusive mode
4223	* to avoid preemption on IB test
4224	*/
4225	if (amdgpu_sriov_vf(adev)) {
4226	amdgpu_virt_request_full_gpu(adev, init: false);
4227	amdgpu_virt_fini_data_exchange(adev);
4228	}
4229
4230	/* disable all interrupts */
4231	amdgpu_irq_disable_all(adev);
4232	if (adev->mode_info.mode_config_initialized) {
4233	if (!drm_drv_uses_atomic_modeset(dev: adev_to_drm(adev)))
4234	drm_helper_force_disable_all(dev: adev_to_drm(adev));
4235	else
4236	drm_atomic_helper_shutdown(dev: adev_to_drm(adev));
4237	}
4238	amdgpu_fence_driver_hw_fini(adev);
4239
4240	if (adev->mman.initialized)
4241	drain_workqueue(wq: adev->mman.bdev.wq);
4242
4243	if (adev->pm.sysfs_initialized)
4244	amdgpu_pm_sysfs_fini(adev);
4245	if (adev->ucode_sysfs_en)
4246	amdgpu_ucode_sysfs_fini(adev);
4247	sysfs_remove_files(kobj: &adev->dev->kobj, attr: amdgpu_dev_attributes);
4248	amdgpu_fru_sysfs_fini(adev);
4249
4250	/* disable ras feature must before hw fini */
4251	amdgpu_ras_pre_fini(adev);
4252
4253	amdgpu_ttm_set_buffer_funcs_status(adev, enable: false);
4254
4255	amdgpu_device_ip_fini_early(adev);
4256
4257	amdgpu_irq_fini_hw(adev);
4258
4259	if (adev->mman.initialized)
4260	ttm_device_clear_dma_mappings(bdev: &adev->mman.bdev);
4261
4262	amdgpu_gart_dummy_page_fini(adev);
4263
4264	if (drm_dev_is_unplugged(dev: adev_to_drm(adev)))
4265	amdgpu_device_unmap_mmio(adev);
4266
4267	}
4268
4269	void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4270	{
4271	int idx;
4272	bool px;
4273
4274	amdgpu_fence_driver_sw_fini(adev);
4275	amdgpu_device_ip_fini(adev);
4276	amdgpu_ucode_release(fw: &adev->firmware.gpu_info_fw);
4277	adev->accel_working = false;
4278	dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4279
4280	amdgpu_reset_fini(adev);
4281
4282	/* free i2c buses */
4283	if (!amdgpu_device_has_dc_support(adev))
4284	amdgpu_i2c_fini(adev);
4285
4286	if (amdgpu_emu_mode != 1)
4287	amdgpu_atombios_fini(adev);
4288
4289	kfree(objp: adev->bios);
4290	adev->bios = NULL;
4291
4292	kfree(objp: adev->fru_info);
4293	adev->fru_info = NULL;
4294
4295	px = amdgpu_device_supports_px(dev: adev_to_drm(adev));
4296
4297	if (px || (!dev_is_removable(dev: &adev->pdev->dev) &&
4298	apple_gmux_detect(NULL, NULL)))
4299	vga_switcheroo_unregister_client(dev: adev->pdev);
4300
4301	if (px)
4302	vga_switcheroo_fini_domain_pm_ops(dev: adev->dev);
4303
4304	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4305	vga_client_unregister(pdev: adev->pdev);
4306
4307	if (drm_dev_enter(dev: adev_to_drm(adev), idx: &idx)) {
4308
4309	iounmap(addr: adev->rmmio);
4310	adev->rmmio = NULL;
4311	amdgpu_doorbell_fini(adev);
4312	drm_dev_exit(idx);
4313	}
4314
4315	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4316	amdgpu_pmu_fini(adev);
4317	if (adev->mman.discovery_bin)
4318	amdgpu_discovery_fini(adev);
4319
4320	amdgpu_reset_put_reset_domain(domain: adev->reset_domain);
4321	adev->reset_domain = NULL;
4322
4323	kfree(objp: adev->pci_state);
4324
4325	}
4326
4327	/**
4328	* amdgpu_device_evict_resources - evict device resources
4329	* @adev: amdgpu device object
4330	*
4331	* Evicts all ttm device resources(vram BOs, gart table) from the lru list
4332	* of the vram memory type. Mainly used for evicting device resources
4333	* at suspend time.
4334	*
4335	*/
4336	static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4337	{
4338	int ret;
4339
4340	/* No need to evict vram on APUs for suspend to ram or s2idle */
4341	if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4342	return 0;
4343
4344	ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4345	if (ret)
4346	DRM_WARN("evicting device resources failed\n");
4347	return ret;
4348	}
4349
4350	/*
4351	* Suspend & resume.
4352	*/
4353	/**
4354	* amdgpu_device_prepare - prepare for device suspend
4355	*
4356	* @dev: drm dev pointer
4357	*
4358	* Prepare to put the hw in the suspend state (all asics).
4359	* Returns 0 for success or an error on failure.
4360	* Called at driver suspend.
4361	*/
4362	int amdgpu_device_prepare(struct drm_device *dev)
4363	{
4364	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
4365	int i, r;
4366
4367	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4368	return 0;
4369
4370	/* Evict the majority of BOs before starting suspend sequence */
4371	r = amdgpu_device_evict_resources(adev);
4372	if (r)
4373	return r;
4374
4375	for (i = 0; i < adev->num_ip_blocks; i++) {
4376	if (!adev->ip_blocks[i].status.valid)
4377	continue;
4378	if (!adev->ip_blocks[i].version->funcs->prepare_suspend)
4379	continue;
4380	r = adev->ip_blocks[i].version->funcs->prepare_suspend((void *)adev);
4381	if (r)
4382	return r;
4383	}
4384
4385	return 0;
4386	}
4387
4388	/**
4389	* amdgpu_device_suspend - initiate device suspend
4390	*
4391	* @dev: drm dev pointer
4392	* @fbcon : notify the fbdev of suspend
4393	*
4394	* Puts the hw in the suspend state (all asics).
4395	* Returns 0 for success or an error on failure.
4396	* Called at driver suspend.
4397	*/
4398	int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4399	{
4400	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
4401	int r = 0;
4402
4403	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4404	return 0;
4405
4406	adev->in_suspend = true;
4407
4408	if (amdgpu_sriov_vf(adev)) {
4409	amdgpu_virt_fini_data_exchange(adev);
4410	r = amdgpu_virt_request_full_gpu(adev, init: false);
4411	if (r)
4412	return r;
4413	}
4414
4415	if (amdgpu_acpi_smart_shift_update(dev, ss_state: AMDGPU_SS_DEV_D3))
4416	DRM_WARN("smart shift update failed\n");
4417
4418	if (fbcon)
4419	drm_fb_helper_set_suspend_unlocked(fb_helper: adev_to_drm(adev)->fb_helper, suspend: true);
4420
4421	cancel_delayed_work_sync(dwork: &adev->delayed_init_work);
4422	flush_delayed_work(dwork: &adev->gfx.gfx_off_delay_work);
4423
4424	amdgpu_ras_suspend(adev);
4425
4426	amdgpu_ttm_set_buffer_funcs_status(adev, enable: false);
4427
4428	amdgpu_device_ip_suspend_phase1(adev);
4429
4430	if (!adev->in_s0ix)
4431	amdgpu_amdkfd_suspend(adev, run_pm: adev->in_runpm);
4432
4433	r = amdgpu_device_evict_resources(adev);
4434	if (r)
4435	return r;
4436
4437	amdgpu_fence_driver_hw_fini(adev);
4438
4439	amdgpu_device_ip_suspend_phase2(adev);
4440
4441	if (amdgpu_sriov_vf(adev))
4442	amdgpu_virt_release_full_gpu(adev, init: false);
4443
4444	return 0;
4445	}
4446
4447	/**
4448	* amdgpu_device_resume - initiate device resume
4449	*
4450	* @dev: drm dev pointer
4451	* @fbcon : notify the fbdev of resume
4452	*
4453	* Bring the hw back to operating state (all asics).
4454	* Returns 0 for success or an error on failure.
4455	* Called at driver resume.
4456	*/
4457	int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4458	{
4459	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
4460	int r = 0;
4461
4462	if (amdgpu_sriov_vf(adev)) {
4463	r = amdgpu_virt_request_full_gpu(adev, init: true);
4464	if (r)
4465	return r;
4466	}
4467
4468	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4469	return 0;
4470
4471	if (adev->in_s0ix)
4472	amdgpu_dpm_gfx_state_change(adev, state: sGpuChangeState_D0Entry);
4473
4474	/* post card */
4475	if (amdgpu_device_need_post(adev)) {
4476	r = amdgpu_device_asic_init(adev);
4477	if (r)
4478	dev_err(adev->dev, "amdgpu asic init failed\n");
4479	}
4480
4481	r = amdgpu_device_ip_resume(adev);
4482
4483	if (r) {
4484	dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4485	goto exit;
4486	}
4487	amdgpu_fence_driver_hw_init(adev);
4488
4489	r = amdgpu_device_ip_late_init(adev);
4490	if (r)
4491	goto exit;
4492
4493	queue_delayed_work(wq: system_wq, dwork: &adev->delayed_init_work,
4494	delay: msecs_to_jiffies(AMDGPU_RESUME_MS));
4495
4496	if (!adev->in_s0ix) {
4497	r = amdgpu_amdkfd_resume(adev, run_pm: adev->in_runpm);
4498	if (r)
4499	goto exit;
4500	}
4501
4502	exit:
4503	if (amdgpu_sriov_vf(adev)) {
4504	amdgpu_virt_init_data_exchange(adev);
4505	amdgpu_virt_release_full_gpu(adev, init: true);
4506	}
4507
4508	if (r)
4509	return r;
4510
4511	/* Make sure IB tests flushed */
4512	flush_delayed_work(dwork: &adev->delayed_init_work);
4513
4514	if (fbcon)
4515	drm_fb_helper_set_suspend_unlocked(fb_helper: adev_to_drm(adev)->fb_helper, suspend: false);
4516
4517	amdgpu_ras_resume(adev);
4518
4519	if (adev->mode_info.num_crtc) {
4520	/*
4521	* Most of the connector probing functions try to acquire runtime pm
4522	* refs to ensure that the GPU is powered on when connector polling is
4523	* performed. Since we're calling this from a runtime PM callback,
4524	* trying to acquire rpm refs will cause us to deadlock.
4525	*
4526	* Since we're guaranteed to be holding the rpm lock, it's safe to
4527	* temporarily disable the rpm helpers so this doesn't deadlock us.
4528	*/
4529	#ifdef CONFIG_PM
4530	dev->dev->power.disable_depth++;
4531	#endif
4532	if (!adev->dc_enabled)
4533	drm_helper_hpd_irq_event(dev);
4534	else
4535	drm_kms_helper_hotplug_event(dev);
4536	#ifdef CONFIG_PM
4537	dev->dev->power.disable_depth--;
4538	#endif
4539	}
4540	adev->in_suspend = false;
4541
4542	if (adev->enable_mes)
4543	amdgpu_mes_self_test(adev);
4544
4545	if (amdgpu_acpi_smart_shift_update(dev, ss_state: AMDGPU_SS_DEV_D0))
4546	DRM_WARN("smart shift update failed\n");
4547
4548	return 0;
4549	}
4550
4551	/**
4552	* amdgpu_device_ip_check_soft_reset - did soft reset succeed
4553	*
4554	* @adev: amdgpu_device pointer
4555	*
4556	* The list of all the hardware IPs that make up the asic is walked and
4557	* the check_soft_reset callbacks are run. check_soft_reset determines
4558	* if the asic is still hung or not.
4559	* Returns true if any of the IPs are still in a hung state, false if not.
4560	*/
4561	static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4562	{
4563	int i;
4564	bool asic_hang = false;
4565
4566	if (amdgpu_sriov_vf(adev))
4567	return true;
4568
4569	if (amdgpu_asic_need_full_reset(adev))
4570	return true;
4571
4572	for (i = 0; i < adev->num_ip_blocks; i++) {
4573	if (!adev->ip_blocks[i].status.valid)
4574	continue;
4575	if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4576	adev->ip_blocks[i].status.hang =
4577	adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4578	if (adev->ip_blocks[i].status.hang) {
4579	dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4580	asic_hang = true;
4581	}
4582	}
4583	return asic_hang;
4584	}
4585
4586	/**
4587	* amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4588	*
4589	* @adev: amdgpu_device pointer
4590	*
4591	* The list of all the hardware IPs that make up the asic is walked and the
4592	* pre_soft_reset callbacks are run if the block is hung. pre_soft_reset
4593	* handles any IP specific hardware or software state changes that are
4594	* necessary for a soft reset to succeed.
4595	* Returns 0 on success, negative error code on failure.
4596	*/
4597	static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4598	{
4599	int i, r = 0;
4600
4601	for (i = 0; i < adev->num_ip_blocks; i++) {
4602	if (!adev->ip_blocks[i].status.valid)
4603	continue;
4604	if (adev->ip_blocks[i].status.hang &&
4605	adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4606	r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4607	if (r)
4608	return r;
4609	}
4610	}
4611
4612	return 0;
4613	}
4614
4615	/**
4616	* amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4617	*
4618	* @adev: amdgpu_device pointer
4619	*
4620	* Some hardware IPs cannot be soft reset. If they are hung, a full gpu
4621	* reset is necessary to recover.
4622	* Returns true if a full asic reset is required, false if not.
4623	*/
4624	static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4625	{
4626	int i;
4627
4628	if (amdgpu_asic_need_full_reset(adev))
4629	return true;
4630
4631	for (i = 0; i < adev->num_ip_blocks; i++) {
4632	if (!adev->ip_blocks[i].status.valid)
4633	continue;
4634	if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4635	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4636	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4637	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4638	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4639	if (adev->ip_blocks[i].status.hang) {
4640	dev_info(adev->dev, "Some block need full reset!\n");
4641	return true;
4642	}
4643	}
4644	}
4645	return false;
4646	}
4647
4648	/**
4649	* amdgpu_device_ip_soft_reset - do a soft reset
4650	*
4651	* @adev: amdgpu_device pointer
4652	*
4653	* The list of all the hardware IPs that make up the asic is walked and the
4654	* soft_reset callbacks are run if the block is hung. soft_reset handles any
4655	* IP specific hardware or software state changes that are necessary to soft
4656	* reset the IP.
4657	* Returns 0 on success, negative error code on failure.
4658	*/
4659	static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4660	{
4661	int i, r = 0;
4662
4663	for (i = 0; i < adev->num_ip_blocks; i++) {
4664	if (!adev->ip_blocks[i].status.valid)
4665	continue;
4666	if (adev->ip_blocks[i].status.hang &&
4667	adev->ip_blocks[i].version->funcs->soft_reset) {
4668	r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4669	if (r)
4670	return r;
4671	}
4672	}
4673
4674	return 0;
4675	}
4676
4677	/**
4678	* amdgpu_device_ip_post_soft_reset - clean up from soft reset
4679	*
4680	* @adev: amdgpu_device pointer
4681	*
4682	* The list of all the hardware IPs that make up the asic is walked and the
4683	* post_soft_reset callbacks are run if the asic was hung. post_soft_reset
4684	* handles any IP specific hardware or software state changes that are
4685	* necessary after the IP has been soft reset.
4686	* Returns 0 on success, negative error code on failure.
4687	*/
4688	static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4689	{
4690	int i, r = 0;
4691
4692	for (i = 0; i < adev->num_ip_blocks; i++) {
4693	if (!adev->ip_blocks[i].status.valid)
4694	continue;
4695	if (adev->ip_blocks[i].status.hang &&
4696	adev->ip_blocks[i].version->funcs->post_soft_reset)
4697	r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4698	if (r)
4699	return r;
4700	}
4701
4702	return 0;
4703	}
4704
4705	/**
4706	* amdgpu_device_recover_vram - Recover some VRAM contents
4707	*
4708	* @adev: amdgpu_device pointer
4709	*
4710	* Restores the contents of VRAM buffers from the shadows in GTT. Used to
4711	* restore things like GPUVM page tables after a GPU reset where
4712	* the contents of VRAM might be lost.
4713	*
4714	* Returns:
4715	* 0 on success, negative error code on failure.
4716	*/
4717	static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4718	{
4719	struct dma_fence *fence = NULL, *next = NULL;
4720	struct amdgpu_bo *shadow;
4721	struct amdgpu_bo_vm *vmbo;
4722	long r = 1, tmo;
4723
4724	if (amdgpu_sriov_runtime(adev))
4725	tmo = msecs_to_jiffies(m: 8000);
4726	else
4727	tmo = msecs_to_jiffies(m: 100);
4728
4729	dev_info(adev->dev, "recover vram bo from shadow start\n");
4730	mutex_lock(&adev->shadow_list_lock);
4731	list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4732	/* If vm is compute context or adev is APU, shadow will be NULL */
4733	if (!vmbo->shadow)
4734	continue;
4735	shadow = vmbo->shadow;
4736
4737	/* No need to recover an evicted BO */
4738	if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4739	shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4740	shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4741	continue;
4742
4743	r = amdgpu_bo_restore_shadow(shadow, fence: &next);
4744	if (r)
4745	break;
4746
4747	if (fence) {
4748	tmo = dma_fence_wait_timeout(fence, intr: false, timeout: tmo);
4749	dma_fence_put(fence);
4750	fence = next;
4751	if (tmo == 0) {
4752	r = -ETIMEDOUT;
4753	break;
4754	} else if (tmo < 0) {
4755	r = tmo;
4756	break;
4757	}
4758	} else {
4759	fence = next;
4760	}
4761	}
4762	mutex_unlock(lock: &adev->shadow_list_lock);
4763
4764	if (fence)
4765	tmo = dma_fence_wait_timeout(fence, intr: false, timeout: tmo);
4766	dma_fence_put(fence);
4767
4768	if (r < 0 || tmo <= 0) {
4769	dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4770	return -EIO;
4771	}
4772
4773	dev_info(adev->dev, "recover vram bo from shadow done\n");
4774	return 0;
4775	}
4776
4777
4778	/**
4779	* amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4780	*
4781	* @adev: amdgpu_device pointer
4782	* @from_hypervisor: request from hypervisor
4783	*
4784	* do VF FLR and reinitialize Asic
4785	* return 0 means succeeded otherwise failed
4786	*/
4787	static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4788	bool from_hypervisor)
4789	{
4790	int r;
4791	struct amdgpu_hive_info *hive = NULL;
4792	int retry_limit = 0;
4793
4794	retry:
4795	amdgpu_amdkfd_pre_reset(adev);
4796
4797	if (from_hypervisor)
4798	r = amdgpu_virt_request_full_gpu(adev, init: true);
4799	else
4800	r = amdgpu_virt_reset_gpu(adev);
4801	if (r)
4802	return r;
4803	amdgpu_irq_gpu_reset_resume_helper(adev);
4804
4805	/* some sw clean up VF needs to do before recover */
4806	amdgpu_virt_post_reset(adev);
4807
4808	/* Resume IP prior to SMC */
4809	r = amdgpu_device_ip_reinit_early_sriov(adev);
4810	if (r)
4811	goto error;
4812
4813	amdgpu_virt_init_data_exchange(adev);
4814
4815	r = amdgpu_device_fw_loading(adev);
4816	if (r)
4817	return r;
4818
4819	/* now we are okay to resume SMC/CP/SDMA */
4820	r = amdgpu_device_ip_reinit_late_sriov(adev);
4821	if (r)
4822	goto error;
4823
4824	hive = amdgpu_get_xgmi_hive(adev);
4825	/* Update PSP FW topology after reset */
4826	if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4827	r = amdgpu_xgmi_update_topology(hive, adev);
4828
4829	if (hive)
4830	amdgpu_put_xgmi_hive(hive);
4831
4832	if (!r) {
4833	r = amdgpu_ib_ring_tests(adev);
4834
4835	amdgpu_amdkfd_post_reset(adev);
4836	}
4837
4838	error:
4839	if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4840	amdgpu_inc_vram_lost(adev);
4841	r = amdgpu_device_recover_vram(adev);
4842	}
4843	amdgpu_virt_release_full_gpu(adev, init: true);
4844
4845	if (AMDGPU_RETRY_SRIOV_RESET(r)) {
4846	if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
4847	retry_limit++;
4848	goto retry;
4849	} else
4850	DRM_ERROR("GPU reset retry is beyond the retry limit\n");
4851	}
4852
4853	return r;
4854	}
4855
4856	/**
4857	* amdgpu_device_has_job_running - check if there is any job in mirror list
4858	*
4859	* @adev: amdgpu_device pointer
4860	*
4861	* check if there is any job in mirror list
4862	*/
4863	bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4864	{
4865	int i;
4866	struct drm_sched_job *job;
4867
4868	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4869	struct amdgpu_ring *ring = adev->rings[i];
4870
4871	if (!ring || !ring->sched.thread)
4872	continue;
4873
4874	spin_lock(lock: &ring->sched.job_list_lock);
4875	job = list_first_entry_or_null(&ring->sched.pending_list,
4876	struct drm_sched_job, list);
4877	spin_unlock(lock: &ring->sched.job_list_lock);
4878	if (job)
4879	return true;
4880	}
4881	return false;
4882	}
4883
4884	/**
4885	* amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4886	*
4887	* @adev: amdgpu_device pointer
4888	*
4889	* Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4890	* a hung GPU.
4891	*/
4892	bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4893	{
4894
4895	if (amdgpu_gpu_recovery == 0)
4896	goto disabled;
4897
4898	/* Skip soft reset check in fatal error mode */
4899	if (!amdgpu_ras_is_poison_mode_supported(adev))
4900	return true;
4901
4902	if (amdgpu_sriov_vf(adev))
4903	return true;
4904
4905	if (amdgpu_gpu_recovery == -1) {
4906	switch (adev->asic_type) {
4907	#ifdef CONFIG_DRM_AMDGPU_SI
4908	case CHIP_VERDE:
4909	case CHIP_TAHITI:
4910	case CHIP_PITCAIRN:
4911	case CHIP_OLAND:
4912	case CHIP_HAINAN:
4913	#endif
4914	#ifdef CONFIG_DRM_AMDGPU_CIK
4915	case CHIP_KAVERI:
4916	case CHIP_KABINI:
4917	case CHIP_MULLINS:
4918	#endif
4919	case CHIP_CARRIZO:
4920	case CHIP_STONEY:
4921	case CHIP_CYAN_SKILLFISH:
4922	goto disabled;
4923	default:
4924	break;
4925	}
4926	}
4927
4928	return true;
4929
4930	disabled:
4931	dev_info(adev->dev, "GPU recovery disabled.\n");
4932	return false;
4933	}
4934
4935	int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4936	{
4937	u32 i;
4938	int ret = 0;
4939
4940	amdgpu_atombios_scratch_regs_engine_hung(adev, hung: true);
4941
4942	dev_info(adev->dev, "GPU mode1 reset\n");
4943
4944	/* disable BM */
4945	pci_clear_master(dev: adev->pdev);
4946
4947	amdgpu_device_cache_pci_state(pdev: adev->pdev);
4948
4949	if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4950	dev_info(adev->dev, "GPU smu mode1 reset\n");
4951	ret = amdgpu_dpm_mode1_reset(adev);
4952	} else {
4953	dev_info(adev->dev, "GPU psp mode1 reset\n");
4954	ret = psp_gpu_reset(adev);
4955	}
4956
4957	if (ret)
4958	goto mode1_reset_failed;
4959
4960	amdgpu_device_load_pci_state(pdev: adev->pdev);
4961	ret = amdgpu_psp_wait_for_bootloader(adev);
4962	if (ret)
4963	goto mode1_reset_failed;
4964
4965	/* wait for asic to come out of reset */
4966	for (i = 0; i < adev->usec_timeout; i++) {
4967	u32 memsize = adev->nbio.funcs->get_memsize(adev);
4968
4969	if (memsize != 0xffffffff)
4970	break;
4971	udelay(1);
4972	}
4973
4974	if (i >= adev->usec_timeout) {
4975	ret = -ETIMEDOUT;
4976	goto mode1_reset_failed;
4977	}
4978
4979	amdgpu_atombios_scratch_regs_engine_hung(adev, hung: false);
4980
4981	return 0;
4982
4983	mode1_reset_failed:
4984	dev_err(adev->dev, "GPU mode1 reset failed\n");
4985	return ret;
4986	}
4987
4988	int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4989	struct amdgpu_reset_context *reset_context)
4990	{
4991	int i, r = 0;
4992	struct amdgpu_job *job = NULL;
4993	bool need_full_reset =
4994	test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4995
4996	if (reset_context->reset_req_dev == adev)
4997	job = reset_context->job;
4998
4999	if (amdgpu_sriov_vf(adev)) {
5000	/* stop the data exchange thread */
5001	amdgpu_virt_fini_data_exchange(adev);
5002	}
5003
5004	amdgpu_fence_driver_isr_toggle(adev, stop: true);
5005
5006	/* block all schedulers and reset given job's ring */
5007	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5008	struct amdgpu_ring *ring = adev->rings[i];
5009
5010	if (!ring || !ring->sched.thread)
5011	continue;
5012
5013	/* Clear job fence from fence drv to avoid force_completion
5014	* leave NULL and vm flush fence in fence drv
5015	*/
5016	amdgpu_fence_driver_clear_job_fences(ring);
5017
5018	/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
5019	amdgpu_fence_driver_force_completion(ring);
5020	}
5021
5022	amdgpu_fence_driver_isr_toggle(adev, stop: false);
5023
5024	if (job && job->vm)
5025	drm_sched_increase_karma(bad: &job->base);
5026
5027	r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
5028	/* If reset handler not implemented, continue; otherwise return */
5029	if (r == -EOPNOTSUPP)
5030	r = 0;
5031	else
5032	return r;
5033
5034	/* Don't suspend on bare metal if we are not going to HW reset the ASIC */
5035	if (!amdgpu_sriov_vf(adev)) {
5036
5037	if (!need_full_reset)
5038	need_full_reset = amdgpu_device_ip_need_full_reset(adev);
5039
5040	if (!need_full_reset && amdgpu_gpu_recovery &&
5041	amdgpu_device_ip_check_soft_reset(adev)) {
5042	amdgpu_device_ip_pre_soft_reset(adev);
5043	r = amdgpu_device_ip_soft_reset(adev);
5044	amdgpu_device_ip_post_soft_reset(adev);
5045	if (r || amdgpu_device_ip_check_soft_reset(adev)) {
5046	dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
5047	need_full_reset = true;
5048	}
5049	}
5050
5051	if (need_full_reset)
5052	r = amdgpu_device_ip_suspend(adev);
5053	if (need_full_reset)
5054	set_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context->flags);
5055	else
5056	clear_bit(nr: AMDGPU_NEED_FULL_RESET,
5057	addr: &reset_context->flags);
5058	}
5059
5060	return r;
5061	}
5062
5063	static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
5064	{
5065	int i;
5066
5067	lockdep_assert_held(&adev->reset_domain->sem);
5068
5069	for (i = 0; i < adev->reset_info.num_regs; i++) {
5070	adev->reset_info.reset_dump_reg_value[i] =
5071	RREG32(adev->reset_info.reset_dump_reg_list[i]);
5072
5073	trace_amdgpu_reset_reg_dumps(address: adev->reset_info.reset_dump_reg_list[i],
5074	value: adev->reset_info.reset_dump_reg_value[i]);
5075	}
5076
5077	return 0;
5078	}
5079
5080	int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5081	struct amdgpu_reset_context *reset_context)
5082	{
5083	struct amdgpu_device *tmp_adev = NULL;
5084	bool need_full_reset, skip_hw_reset, vram_lost = false;
5085	int r = 0;
5086	bool gpu_reset_for_dev_remove = 0;
5087
5088	/* Try reset handler method first */
5089	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5090	reset_list);
5091	amdgpu_reset_reg_dumps(adev: tmp_adev);
5092
5093	reset_context->reset_device_list = device_list_handle;
5094	r = amdgpu_reset_perform_reset(adev: tmp_adev, reset_context);
5095	/* If reset handler not implemented, continue; otherwise return */
5096	if (r == -EOPNOTSUPP)
5097	r = 0;
5098	else
5099	return r;
5100
5101	/* Reset handler not implemented, use the default method */
5102	need_full_reset =
5103	test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5104	skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5105
5106	gpu_reset_for_dev_remove =
5107	test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5108	test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5109
5110	/*
5111	* ASIC reset has to be done on all XGMI hive nodes ASAP
5112	* to allow proper links negotiation in FW (within 1 sec)
5113	*/
5114	if (!skip_hw_reset && need_full_reset) {
5115	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5116	/* For XGMI run all resets in parallel to speed up the process */
5117	if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5118	tmp_adev->gmc.xgmi.pending_reset = false;
5119	if (!queue_work(wq: system_unbound_wq, work: &tmp_adev->xgmi_reset_work))
5120	r = -EALREADY;
5121	} else
5122	r = amdgpu_asic_reset(tmp_adev);
5123
5124	if (r) {
5125	dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
5126	r, adev_to_drm(tmp_adev)->unique);
5127	goto out;
5128	}
5129	}
5130
5131	/* For XGMI wait for all resets to complete before proceed */
5132	if (!r) {
5133	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5134	if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5135	flush_work(work: &tmp_adev->xgmi_reset_work);
5136	r = tmp_adev->asic_reset_res;
5137	if (r)
5138	break;
5139	}
5140	}
5141	}
5142	}
5143
5144	if (!r && amdgpu_ras_intr_triggered()) {
5145	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5146	amdgpu_ras_reset_error_count(adev: tmp_adev, block: AMDGPU_RAS_BLOCK__MMHUB);
5147	}
5148
5149	amdgpu_ras_intr_cleared();
5150	}
5151
5152	/* Since the mode1 reset affects base ip blocks, the
5153	* phase1 ip blocks need to be resumed. Otherwise there
5154	* will be a BIOS signature error and the psp bootloader
5155	* can't load kdb on the next amdgpu install.
5156	*/
5157	if (gpu_reset_for_dev_remove) {
5158	list_for_each_entry(tmp_adev, device_list_handle, reset_list)
5159	amdgpu_device_ip_resume_phase1(adev: tmp_adev);
5160
5161	goto end;
5162	}
5163
5164	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5165	if (need_full_reset) {
5166	/* post card */
5167	r = amdgpu_device_asic_init(adev: tmp_adev);
5168	if (r) {
5169	dev_warn(tmp_adev->dev, "asic atom init failed!");
5170	} else {
5171	dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5172
5173	r = amdgpu_device_ip_resume_phase1(adev: tmp_adev);
5174	if (r)
5175	goto out;
5176
5177	vram_lost = amdgpu_device_check_vram_lost(adev: tmp_adev);
5178
5179	amdgpu_coredump(adev: tmp_adev, vram_lost, reset_context);
5180
5181	if (vram_lost) {
5182	DRM_INFO("VRAM is lost due to GPU reset!\n");
5183	amdgpu_inc_vram_lost(tmp_adev);
5184	}
5185
5186	r = amdgpu_device_fw_loading(adev: tmp_adev);
5187	if (r)
5188	return r;
5189
5190	r = amdgpu_xcp_restore_partition_mode(
5191	xcp_mgr: tmp_adev->xcp_mgr);
5192	if (r)
5193	goto out;
5194
5195	r = amdgpu_device_ip_resume_phase2(adev: tmp_adev);
5196	if (r)
5197	goto out;
5198
5199	if (tmp_adev->mman.buffer_funcs_ring->sched.ready)
5200	amdgpu_ttm_set_buffer_funcs_status(adev: tmp_adev, enable: true);
5201
5202	if (vram_lost)
5203	amdgpu_device_fill_reset_magic(adev: tmp_adev);
5204
5205	/*
5206	* Add this ASIC as tracked as reset was already
5207	* complete successfully.
5208	*/
5209	amdgpu_register_gpu_instance(adev: tmp_adev);
5210
5211	if (!reset_context->hive &&
5212	tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5213	amdgpu_xgmi_add_device(adev: tmp_adev);
5214
5215	r = amdgpu_device_ip_late_init(adev: tmp_adev);
5216	if (r)
5217	goto out;
5218
5219	drm_fb_helper_set_suspend_unlocked(fb_helper: adev_to_drm(adev: tmp_adev)->fb_helper, suspend: false);
5220
5221	/*
5222	* The GPU enters bad state once faulty pages
5223	* by ECC has reached the threshold, and ras
5224	* recovery is scheduled next. So add one check
5225	* here to break recovery if it indeed exceeds
5226	* bad page threshold, and remind user to
5227	* retire this GPU or setting one bigger
5228	* bad_page_threshold value to fix this once
5229	* probing driver again.
5230	*/
5231	if (!amdgpu_ras_eeprom_check_err_threshold(adev: tmp_adev)) {
5232	/* must succeed. */
5233	amdgpu_ras_resume(adev: tmp_adev);
5234	} else {
5235	r = -EINVAL;
5236	goto out;
5237	}
5238
5239	/* Update PSP FW topology after reset */
5240	if (reset_context->hive &&
5241	tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5242	r = amdgpu_xgmi_update_topology(
5243	hive: reset_context->hive, adev: tmp_adev);
5244	}
5245	}
5246
5247	out:
5248	if (!r) {
5249	amdgpu_irq_gpu_reset_resume_helper(adev: tmp_adev);
5250	r = amdgpu_ib_ring_tests(adev: tmp_adev);
5251	if (r) {
5252	dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5253	need_full_reset = true;
5254	r = -EAGAIN;
5255	goto end;
5256	}
5257	}
5258
5259	if (!r)
5260	r = amdgpu_device_recover_vram(adev: tmp_adev);
5261	else
5262	tmp_adev->asic_reset_res = r;
5263	}
5264
5265	end:
5266	if (need_full_reset)
5267	set_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context->flags);
5268	else
5269	clear_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context->flags);
5270	return r;
5271	}
5272
5273	static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5274	{
5275
5276	switch (amdgpu_asic_reset_method(adev)) {
5277	case AMD_RESET_METHOD_MODE1:
5278	adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5279	break;
5280	case AMD_RESET_METHOD_MODE2:
5281	adev->mp1_state = PP_MP1_STATE_RESET;
5282	break;
5283	default:
5284	adev->mp1_state = PP_MP1_STATE_NONE;
5285	break;
5286	}
5287	}
5288
5289	static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5290	{
5291	amdgpu_vf_error_trans_all(adev);
5292	adev->mp1_state = PP_MP1_STATE_NONE;
5293	}
5294
5295	static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5296	{
5297	struct pci_dev *p = NULL;
5298
5299	p = pci_get_domain_bus_and_slot(domain: pci_domain_nr(bus: adev->pdev->bus),
5300	bus: adev->pdev->bus->number, devfn: 1);
5301	if (p) {
5302	pm_runtime_enable(dev: &(p->dev));
5303	pm_runtime_resume(dev: &(p->dev));
5304	}
5305
5306	pci_dev_put(dev: p);
5307	}
5308
5309	static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5310	{
5311	enum amd_reset_method reset_method;
5312	struct pci_dev *p = NULL;
5313	u64 expires;
5314
5315	/*
5316	* For now, only BACO and mode1 reset are confirmed
5317	* to suffer the audio issue without proper suspended.
5318	*/
5319	reset_method = amdgpu_asic_reset_method(adev);
5320	if ((reset_method != AMD_RESET_METHOD_BACO) &&
5321	(reset_method != AMD_RESET_METHOD_MODE1))
5322	return -EINVAL;
5323
5324	p = pci_get_domain_bus_and_slot(domain: pci_domain_nr(bus: adev->pdev->bus),
5325	bus: adev->pdev->bus->number, devfn: 1);
5326	if (!p)
5327	return -ENODEV;
5328
5329	expires = pm_runtime_autosuspend_expiration(dev: &(p->dev));
5330	if (!expires)
5331	/*
5332	* If we cannot get the audio device autosuspend delay,
5333	* a fixed 4S interval will be used. Considering 3S is
5334	* the audio controller default autosuspend delay setting.
5335	* 4S used here is guaranteed to cover that.
5336	*/
5337	expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5338
5339	while (!pm_runtime_status_suspended(dev: &(p->dev))) {
5340	if (!pm_runtime_suspend(dev: &(p->dev)))
5341	break;
5342
5343	if (expires < ktime_get_mono_fast_ns()) {
5344	dev_warn(adev->dev, "failed to suspend display audio\n");
5345	pci_dev_put(dev: p);
5346	/* TODO: abort the succeeding gpu reset? */
5347	return -ETIMEDOUT;
5348	}
5349	}
5350
5351	pm_runtime_disable(dev: &(p->dev));
5352
5353	pci_dev_put(dev: p);
5354	return 0;
5355	}
5356
5357	static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5358	{
5359	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5360
5361	#if defined(CONFIG_DEBUG_FS)
5362	if (!amdgpu_sriov_vf(adev))
5363	cancel_work(work: &adev->reset_work);
5364	#endif
5365
5366	if (adev->kfd.dev)
5367	cancel_work(work: &adev->kfd.reset_work);
5368
5369	if (amdgpu_sriov_vf(adev))
5370	cancel_work(work: &adev->virt.flr_work);
5371
5372	if (con && adev->ras_enabled)
5373	cancel_work(work: &con->recovery_work);
5374
5375	}
5376
5377	/**
5378	* amdgpu_device_gpu_recover - reset the asic and recover scheduler
5379	*
5380	* @adev: amdgpu_device pointer
5381	* @job: which job trigger hang
5382	* @reset_context: amdgpu reset context pointer
5383	*
5384	* Attempt to reset the GPU if it has hung (all asics).
5385	* Attempt to do soft-reset or full-reset and reinitialize Asic
5386	* Returns 0 for success or an error on failure.
5387	*/
5388
5389	int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5390	struct amdgpu_job *job,
5391	struct amdgpu_reset_context *reset_context)
5392	{
5393	struct list_head device_list, *device_list_handle = NULL;
5394	bool job_signaled = false;
5395	struct amdgpu_hive_info *hive = NULL;
5396	struct amdgpu_device *tmp_adev = NULL;
5397	int i, r = 0;
5398	bool need_emergency_restart = false;
5399	bool audio_suspended = false;
5400	bool gpu_reset_for_dev_remove = false;
5401
5402	gpu_reset_for_dev_remove =
5403	test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5404	test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5405
5406	/*
5407	* Special case: RAS triggered and full reset isn't supported
5408	*/
5409	need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5410
5411	/*
5412	* Flush RAM to disk so that after reboot
5413	* the user can read log and see why the system rebooted.
5414	*/
5415	if (need_emergency_restart && amdgpu_ras_get_context(adev) &&
5416	amdgpu_ras_get_context(adev)->reboot) {
5417	DRM_WARN("Emergency reboot.");
5418
5419	ksys_sync_helper();
5420	emergency_restart();
5421	}
5422
5423	dev_info(adev->dev, "GPU %s begin!\n",
5424	need_emergency_restart ? "jobs stop":"reset");
5425
5426	if (!amdgpu_sriov_vf(adev))
5427	hive = amdgpu_get_xgmi_hive(adev);
5428	if (hive)
5429	mutex_lock(&hive->hive_lock);
5430
5431	reset_context->job = job;
5432	reset_context->hive = hive;
5433	/*
5434	* Build list of devices to reset.
5435	* In case we are in XGMI hive mode, resort the device list
5436	* to put adev in the 1st position.
5437	*/
5438	INIT_LIST_HEAD(list: &device_list);
5439	if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5440	list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5441	list_add_tail(new: &tmp_adev->reset_list, head: &device_list);
5442	if (gpu_reset_for_dev_remove && adev->shutdown)
5443	tmp_adev->shutdown = true;
5444	}
5445	if (!list_is_first(list: &adev->reset_list, head: &device_list))
5446	list_rotate_to_front(list: &adev->reset_list, head: &device_list);
5447	device_list_handle = &device_list;
5448	} else {
5449	list_add_tail(new: &adev->reset_list, head: &device_list);
5450	device_list_handle = &device_list;
5451	}
5452
5453	/* We need to lock reset domain only once both for XGMI and single device */
5454	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5455	reset_list);
5456	amdgpu_device_lock_reset_domain(reset_domain: tmp_adev->reset_domain);
5457
5458	/* block all schedulers and reset given job's ring */
5459	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5460
5461	amdgpu_device_set_mp1_state(adev: tmp_adev);
5462
5463	/*
5464	* Try to put the audio codec into suspend state
5465	* before gpu reset started.
5466	*
5467	* Due to the power domain of the graphics device
5468	* is shared with AZ power domain. Without this,
5469	* we may change the audio hardware from behind
5470	* the audio driver's back. That will trigger
5471	* some audio codec errors.
5472	*/
5473	if (!amdgpu_device_suspend_display_audio(adev: tmp_adev))
5474	audio_suspended = true;
5475
5476	amdgpu_ras_set_error_query_ready(adev: tmp_adev, ready: false);
5477
5478	cancel_delayed_work_sync(dwork: &tmp_adev->delayed_init_work);
5479
5480	if (!amdgpu_sriov_vf(tmp_adev))
5481	amdgpu_amdkfd_pre_reset(adev: tmp_adev);
5482
5483	/*
5484	* Mark these ASICs to be reseted as untracked first
5485	* And add them back after reset completed
5486	*/
5487	amdgpu_unregister_gpu_instance(adev: tmp_adev);
5488
5489	drm_fb_helper_set_suspend_unlocked(fb_helper: adev_to_drm(adev: tmp_adev)->fb_helper, suspend: true);
5490
5491	/* disable ras on ALL IPs */
5492	if (!need_emergency_restart &&
5493	amdgpu_device_ip_need_full_reset(adev: tmp_adev))
5494	amdgpu_ras_suspend(adev: tmp_adev);
5495
5496	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5497	struct amdgpu_ring *ring = tmp_adev->rings[i];
5498
5499	if (!ring || !ring->sched.thread)
5500	continue;
5501
5502	drm_sched_stop(sched: &ring->sched, bad: job ? &job->base : NULL);
5503
5504	if (need_emergency_restart)
5505	amdgpu_job_stop_all_jobs_on_sched(sched: &ring->sched);
5506	}
5507	atomic_inc(v: &tmp_adev->gpu_reset_counter);
5508	}
5509
5510	if (need_emergency_restart)
5511	goto skip_sched_resume;
5512
5513	/*
5514	* Must check guilty signal here since after this point all old
5515	* HW fences are force signaled.
5516	*
5517	* job->base holds a reference to parent fence
5518	*/
5519	if (job && dma_fence_is_signaled(fence: &job->hw_fence)) {
5520	job_signaled = true;
5521	dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5522	goto skip_hw_reset;
5523	}
5524
5525	retry: /* Rest of adevs pre asic reset from XGMI hive. */
5526	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5527	if (gpu_reset_for_dev_remove) {
5528	/* Workaroud for ASICs need to disable SMC first */
5529	amdgpu_device_smu_fini_early(adev: tmp_adev);
5530	}
5531	r = amdgpu_device_pre_asic_reset(adev: tmp_adev, reset_context);
5532	/*TODO Should we stop ?*/
5533	if (r) {
5534	dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5535	r, adev_to_drm(tmp_adev)->unique);
5536	tmp_adev->asic_reset_res = r;
5537	}
5538
5539	/*
5540	* Drop all pending non scheduler resets. Scheduler resets
5541	* were already dropped during drm_sched_stop
5542	*/
5543	amdgpu_device_stop_pending_resets(adev: tmp_adev);
5544	}
5545
5546	/* Actual ASIC resets if needed.*/
5547	/* Host driver will handle XGMI hive reset for SRIOV */
5548	if (amdgpu_sriov_vf(adev)) {
5549	r = amdgpu_device_reset_sriov(adev, from_hypervisor: job ? false : true);
5550	if (r)
5551	adev->asic_reset_res = r;
5552
5553	/* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5554	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) ==
5555	IP_VERSION(9, 4, 2) ||
5556	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(11, 0, 3))
5557	amdgpu_ras_resume(adev);
5558	} else {
5559	r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5560	if (r && r == -EAGAIN)
5561	goto retry;
5562
5563	if (!r && gpu_reset_for_dev_remove)
5564	goto recover_end;
5565	}
5566
5567	skip_hw_reset:
5568
5569	/* Post ASIC reset for all devs .*/
5570	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5571
5572	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5573	struct amdgpu_ring *ring = tmp_adev->rings[i];
5574
5575	if (!ring || !ring->sched.thread)
5576	continue;
5577
5578	drm_sched_start(sched: &ring->sched, full_recovery: true);
5579	}
5580
5581	if (!drm_drv_uses_atomic_modeset(dev: adev_to_drm(adev: tmp_adev)) && !job_signaled)
5582	drm_helper_resume_force_mode(dev: adev_to_drm(adev: tmp_adev));
5583
5584	if (tmp_adev->asic_reset_res)
5585	r = tmp_adev->asic_reset_res;
5586
5587	tmp_adev->asic_reset_res = 0;
5588
5589	if (r) {
5590	/* bad news, how to tell it to userspace ? */
5591	dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5592	amdgpu_vf_error_put(adev: tmp_adev, sub_error_code: AMDGIM_ERROR_VF_GPU_RESET_FAIL, error_flags: 0, error_data: r);
5593	} else {
5594	dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5595	if (amdgpu_acpi_smart_shift_update(dev: adev_to_drm(adev: tmp_adev), ss_state: AMDGPU_SS_DEV_D0))
5596	DRM_WARN("smart shift update failed\n");
5597	}
5598	}
5599
5600	skip_sched_resume:
5601	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5602	/* unlock kfd: SRIOV would do it separately */
5603	if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5604	amdgpu_amdkfd_post_reset(adev: tmp_adev);
5605
5606	/* kfd_post_reset will do nothing if kfd device is not initialized,
5607	* need to bring up kfd here if it's not be initialized before
5608	*/
5609	if (!adev->kfd.init_complete)
5610	amdgpu_amdkfd_device_init(adev);
5611
5612	if (audio_suspended)
5613	amdgpu_device_resume_display_audio(adev: tmp_adev);
5614
5615	amdgpu_device_unset_mp1_state(adev: tmp_adev);
5616
5617	amdgpu_ras_set_error_query_ready(adev: tmp_adev, ready: true);
5618	}
5619
5620	recover_end:
5621	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5622	reset_list);
5623	amdgpu_device_unlock_reset_domain(reset_domain: tmp_adev->reset_domain);
5624
5625	if (hive) {
5626	mutex_unlock(lock: &hive->hive_lock);
5627	amdgpu_put_xgmi_hive(hive);
5628	}
5629
5630	if (r)
5631	dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5632
5633	atomic_set(v: &adev->reset_domain->reset_res, i: r);
5634	return r;
5635	}
5636
5637	/**
5638	* amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5639	*
5640	* @adev: amdgpu_device pointer
5641	*
5642	* Fetchs and stores in the driver the PCIE capabilities (gen speed
5643	* and lanes) of the slot the device is in. Handles APUs and
5644	* virtualized environments where PCIE config space may not be available.
5645	*/
5646	static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5647	{
5648	struct pci_dev *pdev;
5649	enum pci_bus_speed speed_cap, platform_speed_cap;
5650	enum pcie_link_width platform_link_width;
5651
5652	if (amdgpu_pcie_gen_cap)
5653	adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5654
5655	if (amdgpu_pcie_lane_cap)
5656	adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5657
5658	/* covers APUs as well */
5659	if (pci_is_root_bus(pbus: adev->pdev->bus) && !amdgpu_passthrough(adev)) {
5660	if (adev->pm.pcie_gen_mask == 0)
5661	adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5662	if (adev->pm.pcie_mlw_mask == 0)
5663	adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5664	return;
5665	}
5666
5667	if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5668	return;
5669
5670	pcie_bandwidth_available(dev: adev->pdev, NULL,
5671	speed: &platform_speed_cap, width: &platform_link_width);
5672
5673	if (adev->pm.pcie_gen_mask == 0) {
5674	/* asic caps */
5675	pdev = adev->pdev;
5676	speed_cap = pcie_get_speed_cap(dev: pdev);
5677	if (speed_cap == PCI_SPEED_UNKNOWN) {
5678	adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5679	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5680	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5681	} else {
5682	if (speed_cap == PCIE_SPEED_32_0GT)
5683	adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5684	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5685	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5686	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5687	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5688	else if (speed_cap == PCIE_SPEED_16_0GT)
5689	adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5690	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5691	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5692	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5693	else if (speed_cap == PCIE_SPEED_8_0GT)
5694	adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5695	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5696	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5697	else if (speed_cap == PCIE_SPEED_5_0GT)
5698	adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5699	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5700	else
5701	adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5702	}
5703	/* platform caps */
5704	if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5705	adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5706	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5707	} else {
5708	if (platform_speed_cap == PCIE_SPEED_32_0GT)
5709	adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5710	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5711	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5712	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5713	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5714	else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5715	adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5716	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5717	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5718	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5719	else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5720	adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5721	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5722	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5723	else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5724	adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5725	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5726	else
5727	adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5728
5729	}
5730	}
5731	if (adev->pm.pcie_mlw_mask == 0) {
5732	if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5733	adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5734	} else {
5735	switch (platform_link_width) {
5736	case PCIE_LNK_X32:
5737	adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5738	CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5739	CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5740	CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5741	CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5742	CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5743	CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5744	break;
5745	case PCIE_LNK_X16:
5746	adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5747	CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5748	CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5749	CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5750	CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5751	CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5752	break;
5753	case PCIE_LNK_X12:
5754	adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5755	CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5756	CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5757	CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5758	CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5759	break;
5760	case PCIE_LNK_X8:
5761	adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5762	CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5763	CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5764	CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5765	break;
5766	case PCIE_LNK_X4:
5767	adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5768	CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5769	CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5770	break;
5771	case PCIE_LNK_X2:
5772	adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5773	CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5774	break;
5775	case PCIE_LNK_X1:
5776	adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5777	break;
5778	default:
5779	break;
5780	}
5781	}
5782	}
5783	}
5784
5785	/**
5786	* amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5787	*
5788	* @adev: amdgpu_device pointer
5789	* @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5790	*
5791	* Return true if @peer_adev can access (DMA) @adev through the PCIe
5792	* BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5793	* @peer_adev.
5794	*/
5795	bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5796	struct amdgpu_device *peer_adev)
5797	{
5798	#ifdef CONFIG_HSA_AMD_P2P
5799	uint64_t address_mask = peer_adev->dev->dma_mask ?
5800	~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5801	resource_size_t aper_limit =
5802	adev->gmc.aper_base + adev->gmc.aper_size - 1;
5803	bool p2p_access =
5804	!adev->gmc.xgmi.connected_to_cpu &&
5805	!(pci_p2pdma_distance(provider: adev->pdev, client: peer_adev->dev, verbose: false) < 0);
5806
5807	return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5808	adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5809	!(adev->gmc.aper_base & address_mask ||
5810	aper_limit & address_mask));
5811	#else
5812	return false;
5813	#endif
5814	}
5815
5816	int amdgpu_device_baco_enter(struct drm_device *dev)
5817	{
5818	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
5819	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5820
5821	if (!amdgpu_device_supports_baco(dev))
5822	return -ENOTSUPP;
5823
5824	if (ras && adev->ras_enabled &&
5825	adev->nbio.funcs->enable_doorbell_interrupt)
5826	adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5827
5828	return amdgpu_dpm_baco_enter(adev);
5829	}
5830
5831	int amdgpu_device_baco_exit(struct drm_device *dev)
5832	{
5833	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
5834	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5835	int ret = 0;
5836
5837	if (!amdgpu_device_supports_baco(dev))
5838	return -ENOTSUPP;
5839
5840	ret = amdgpu_dpm_baco_exit(adev);
5841	if (ret)
5842	return ret;
5843
5844	if (ras && adev->ras_enabled &&
5845	adev->nbio.funcs->enable_doorbell_interrupt)
5846	adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5847
5848	if (amdgpu_passthrough(adev) &&
5849	adev->nbio.funcs->clear_doorbell_interrupt)
5850	adev->nbio.funcs->clear_doorbell_interrupt(adev);
5851
5852	return 0;
5853	}
5854
5855	/**
5856	* amdgpu_pci_error_detected - Called when a PCI error is detected.
5857	* @pdev: PCI device struct
5858	* @state: PCI channel state
5859	*
5860	* Description: Called when a PCI error is detected.
5861	*
5862	* Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5863	*/
5864	pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5865	{
5866	struct drm_device *dev = pci_get_drvdata(pdev);
5867	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
5868	int i;
5869
5870	DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5871
5872	if (adev->gmc.xgmi.num_physical_nodes > 1) {
5873	DRM_WARN("No support for XGMI hive yet...");
5874	return PCI_ERS_RESULT_DISCONNECT;
5875	}
5876
5877	adev->pci_channel_state = state;
5878
5879	switch (state) {
5880	case pci_channel_io_normal:
5881	return PCI_ERS_RESULT_CAN_RECOVER;
5882	/* Fatal error, prepare for slot reset */
5883	case pci_channel_io_frozen:
5884	/*
5885	* Locking adev->reset_domain->sem will prevent any external access
5886	* to GPU during PCI error recovery
5887	*/
5888	amdgpu_device_lock_reset_domain(reset_domain: adev->reset_domain);
5889	amdgpu_device_set_mp1_state(adev);
5890
5891	/*
5892	* Block any work scheduling as we do for regular GPU reset
5893	* for the duration of the recovery
5894	*/
5895	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5896	struct amdgpu_ring *ring = adev->rings[i];
5897
5898	if (!ring || !ring->sched.thread)
5899	continue;
5900
5901	drm_sched_stop(sched: &ring->sched, NULL);
5902	}
5903	atomic_inc(v: &adev->gpu_reset_counter);
5904	return PCI_ERS_RESULT_NEED_RESET;
5905	case pci_channel_io_perm_failure:
5906	/* Permanent error, prepare for device removal */
5907	return PCI_ERS_RESULT_DISCONNECT;
5908	}
5909
5910	return PCI_ERS_RESULT_NEED_RESET;
5911	}
5912
5913	/**
5914	* amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5915	* @pdev: pointer to PCI device
5916	*/
5917	pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5918	{
5919
5920	DRM_INFO("PCI error: mmio enabled callback!!\n");
5921
5922	/* TODO - dump whatever for debugging purposes */
5923
5924	/* This called only if amdgpu_pci_error_detected returns
5925	* PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5926	* works, no need to reset slot.
5927	*/
5928
5929	return PCI_ERS_RESULT_RECOVERED;
5930	}
5931
5932	/**
5933	* amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5934	* @pdev: PCI device struct
5935	*
5936	* Description: This routine is called by the pci error recovery
5937	* code after the PCI slot has been reset, just before we
5938	* should resume normal operations.
5939	*/
5940	pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5941	{
5942	struct drm_device *dev = pci_get_drvdata(pdev);
5943	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
5944	int r, i;
5945	struct amdgpu_reset_context reset_context;
5946	u32 memsize;
5947	struct list_head device_list;
5948
5949	DRM_INFO("PCI error: slot reset callback!!\n");
5950
5951	memset(&reset_context, 0, sizeof(reset_context));
5952
5953	INIT_LIST_HEAD(list: &device_list);
5954	list_add_tail(new: &adev->reset_list, head: &device_list);
5955
5956	/* wait for asic to come out of reset */
5957	msleep(msecs: 500);
5958
5959	/* Restore PCI confspace */
5960	amdgpu_device_load_pci_state(pdev);
5961
5962	/* confirm ASIC came out of reset */
5963	for (i = 0; i < adev->usec_timeout; i++) {
5964	memsize = amdgpu_asic_get_config_memsize(adev);
5965
5966	if (memsize != 0xffffffff)
5967	break;
5968	udelay(1);
5969	}
5970	if (memsize == 0xffffffff) {
5971	r = -ETIME;
5972	goto out;
5973	}
5974
5975	reset_context.method = AMD_RESET_METHOD_NONE;
5976	reset_context.reset_req_dev = adev;
5977	set_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context.flags);
5978	set_bit(nr: AMDGPU_SKIP_HW_RESET, addr: &reset_context.flags);
5979
5980	adev->no_hw_access = true;
5981	r = amdgpu_device_pre_asic_reset(adev, reset_context: &reset_context);
5982	adev->no_hw_access = false;
5983	if (r)
5984	goto out;
5985
5986	r = amdgpu_do_asic_reset(device_list_handle: &device_list, reset_context: &reset_context);
5987
5988	out:
5989	if (!r) {
5990	if (amdgpu_device_cache_pci_state(pdev: adev->pdev))
5991	pci_restore_state(dev: adev->pdev);
5992
5993	DRM_INFO("PCIe error recovery succeeded\n");
5994	} else {
5995	DRM_ERROR("PCIe error recovery failed, err:%d", r);
5996	amdgpu_device_unset_mp1_state(adev);
5997	amdgpu_device_unlock_reset_domain(reset_domain: adev->reset_domain);
5998	}
5999
6000	return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
6001	}
6002
6003	/**
6004	* amdgpu_pci_resume() - resume normal ops after PCI reset
6005	* @pdev: pointer to PCI device
6006	*
6007	* Called when the error recovery driver tells us that its
6008	* OK to resume normal operation.
6009	*/
6010	void amdgpu_pci_resume(struct pci_dev *pdev)
6011	{
6012	struct drm_device *dev = pci_get_drvdata(pdev);
6013	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
6014	int i;
6015
6016
6017	DRM_INFO("PCI error: resume callback!!\n");
6018
6019	/* Only continue execution for the case of pci_channel_io_frozen */
6020	if (adev->pci_channel_state != pci_channel_io_frozen)
6021	return;
6022
6023	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6024	struct amdgpu_ring *ring = adev->rings[i];
6025
6026	if (!ring || !ring->sched.thread)
6027	continue;
6028
6029	drm_sched_start(sched: &ring->sched, full_recovery: true);
6030	}
6031
6032	amdgpu_device_unset_mp1_state(adev);
6033	amdgpu_device_unlock_reset_domain(reset_domain: adev->reset_domain);
6034	}
6035
6036	bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6037	{
6038	struct drm_device *dev = pci_get_drvdata(pdev);
6039	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
6040	int r;
6041
6042	r = pci_save_state(dev: pdev);
6043	if (!r) {
6044	kfree(objp: adev->pci_state);
6045
6046	adev->pci_state = pci_store_saved_state(dev: pdev);
6047
6048	if (!adev->pci_state) {
6049	DRM_ERROR("Failed to store PCI saved state");
6050	return false;
6051	}
6052	} else {
6053	DRM_WARN("Failed to save PCI state, err:%d\n", r);
6054	return false;
6055	}
6056
6057	return true;
6058	}
6059
6060	bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6061	{
6062	struct drm_device *dev = pci_get_drvdata(pdev);
6063	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
6064	int r;
6065
6066	if (!adev->pci_state)
6067	return false;
6068
6069	r = pci_load_saved_state(dev: pdev, state: adev->pci_state);
6070
6071	if (!r) {
6072	pci_restore_state(dev: pdev);
6073	} else {
6074	DRM_WARN("Failed to load PCI state, err:%d\n", r);
6075	return false;
6076	}
6077
6078	return true;
6079	}
6080
6081	void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6082	struct amdgpu_ring *ring)
6083	{
6084	#ifdef CONFIG_X86_64
6085	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6086	return;
6087	#endif
6088	if (adev->gmc.xgmi.connected_to_cpu)
6089	return;
6090
6091	if (ring && ring->funcs->emit_hdp_flush)
6092	amdgpu_ring_emit_hdp_flush(ring);
6093	else
6094	amdgpu_asic_flush_hdp(adev, ring);
6095	}
6096
6097	void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6098	struct amdgpu_ring *ring)
6099	{
6100	#ifdef CONFIG_X86_64
6101	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6102	return;
6103	#endif
6104	if (adev->gmc.xgmi.connected_to_cpu)
6105	return;
6106
6107	amdgpu_asic_invalidate_hdp(adev, ring);
6108	}
6109
6110	int amdgpu_in_reset(struct amdgpu_device *adev)
6111	{
6112	return atomic_read(v: &adev->reset_domain->in_gpu_reset);
6113	}
6114
6115	/**
6116	* amdgpu_device_halt() - bring hardware to some kind of halt state
6117	*
6118	* @adev: amdgpu_device pointer
6119	*
6120	* Bring hardware to some kind of halt state so that no one can touch it
6121	* any more. It will help to maintain error context when error occurred.
6122	* Compare to a simple hang, the system will keep stable at least for SSH
6123	* access. Then it should be trivial to inspect the hardware state and
6124	* see what's going on. Implemented as following:
6125	*
6126	* 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6127	* clears all CPU mappings to device, disallows remappings through page faults
6128	* 2. amdgpu_irq_disable_all() disables all interrupts
6129	* 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6130	* 4. set adev->no_hw_access to avoid potential crashes after setp 5
6131	* 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6132	* 6. pci_disable_device() and pci_wait_for_pending_transaction()
6133	* flush any in flight DMA operations
6134	*/
6135	void amdgpu_device_halt(struct amdgpu_device *adev)
6136	{
6137	struct pci_dev *pdev = adev->pdev;
6138	struct drm_device *ddev = adev_to_drm(adev);
6139
6140	amdgpu_xcp_dev_unplug(adev);
6141	drm_dev_unplug(dev: ddev);
6142
6143	amdgpu_irq_disable_all(adev);
6144
6145	amdgpu_fence_driver_hw_fini(adev);
6146
6147	adev->no_hw_access = true;
6148
6149	amdgpu_device_unmap_mmio(adev);
6150
6151	pci_disable_device(dev: pdev);
6152	pci_wait_for_pending_transaction(dev: pdev);
6153	}
6154
6155	u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
6156	u32 reg)
6157	{
6158	unsigned long flags, address, data;
6159	u32 r;
6160
6161	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6162	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6163
6164	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6165	WREG32(address, reg * 4);
6166	(void)RREG32(address);
6167	r = RREG32(data);
6168	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
6169	return r;
6170	}
6171
6172	void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
6173	u32 reg, u32 v)
6174	{
6175	unsigned long flags, address, data;
6176
6177	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6178	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6179
6180	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6181	WREG32(address, reg * 4);
6182	(void)RREG32(address);
6183	WREG32(data, v);
6184	(void)RREG32(data);
6185	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
6186	}
6187
6188	/**
6189	* amdgpu_device_switch_gang - switch to a new gang
6190	* @adev: amdgpu_device pointer
6191	* @gang: the gang to switch to
6192	*
6193	* Try to switch to a new gang.
6194	* Returns: NULL if we switched to the new gang or a reference to the current
6195	* gang leader.
6196	*/
6197	struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6198	struct dma_fence *gang)
6199	{
6200	struct dma_fence *old = NULL;
6201
6202	do {
6203	dma_fence_put(fence: old);
6204	rcu_read_lock();
6205	old = dma_fence_get_rcu_safe(fencep: &adev->gang_submit);
6206	rcu_read_unlock();
6207
6208	if (old == gang)
6209	break;
6210
6211	if (!dma_fence_is_signaled(fence: old))
6212	return old;
6213
6214	} while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6215	old, gang) != old);
6216
6217	dma_fence_put(fence: old);
6218	return NULL;
6219	}
6220
6221	bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6222	{
6223	switch (adev->asic_type) {
6224	#ifdef CONFIG_DRM_AMDGPU_SI
6225	case CHIP_HAINAN:
6226	#endif
6227	case CHIP_TOPAZ:
6228	/* chips with no display hardware */
6229	return false;
6230	#ifdef CONFIG_DRM_AMDGPU_SI
6231	case CHIP_TAHITI:
6232	case CHIP_PITCAIRN:
6233	case CHIP_VERDE:
6234	case CHIP_OLAND:
6235	#endif
6236	#ifdef CONFIG_DRM_AMDGPU_CIK
6237	case CHIP_BONAIRE:
6238	case CHIP_HAWAII:
6239	case CHIP_KAVERI:
6240	case CHIP_KABINI:
6241	case CHIP_MULLINS:
6242	#endif
6243	case CHIP_TONGA:
6244	case CHIP_FIJI:
6245	case CHIP_POLARIS10:
6246	case CHIP_POLARIS11:
6247	case CHIP_POLARIS12:
6248	case CHIP_VEGAM:
6249	case CHIP_CARRIZO:
6250	case CHIP_STONEY:
6251	/* chips with display hardware */
6252	return true;
6253	default:
6254	/* IP discovery */
6255	if (!amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0) ||
6256	(adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6257	return false;
6258	return true;
6259	}
6260	}
6261
6262	uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
6263	uint32_t inst, uint32_t reg_addr, char reg_name[],
6264	uint32_t expected_value, uint32_t mask)
6265	{
6266	uint32_t ret = 0;
6267	uint32_t old_ = 0;
6268	uint32_t tmp_ = RREG32(reg_addr);
6269	uint32_t loop = adev->usec_timeout;
6270
6271	while ((tmp_ & (mask)) != (expected_value)) {
6272	if (old_ != tmp_) {
6273	loop = adev->usec_timeout;
6274	old_ = tmp_;
6275	} else
6276	udelay(1);
6277	tmp_ = RREG32(reg_addr);
6278	loop--;
6279	if (!loop) {
6280	DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
6281	inst, reg_name, (uint32_t)expected_value,
6282	(uint32_t)(tmp_ & (mask)));
6283	ret = -ETIMEDOUT;
6284	break;
6285	}
6286	}
6287	return ret;
6288	}
6289