1	/*
2	* Copyright 2016 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	*/
23
24	#include <linux/firmware.h>
25	#include <linux/pci.h>
26
27	#include <drm/drm_cache.h>
28
29	#include "amdgpu.h"
30	#include "gmc_v9_0.h"
31	#include "amdgpu_atomfirmware.h"
32	#include "amdgpu_gem.h"
33
34	#include "gc/gc_9_0_sh_mask.h"
35	#include "dce/dce_12_0_offset.h"
36	#include "dce/dce_12_0_sh_mask.h"
37	#include "vega10_enum.h"
38	#include "mmhub/mmhub_1_0_offset.h"
39	#include "athub/athub_1_0_sh_mask.h"
40	#include "athub/athub_1_0_offset.h"
41	#include "oss/osssys_4_0_offset.h"
42
43	#include "soc15.h"
44	#include "soc15d.h"
45	#include "soc15_common.h"
46	#include "umc/umc_6_0_sh_mask.h"
47
48	#include "gfxhub_v1_0.h"
49	#include "mmhub_v1_0.h"
50	#include "athub_v1_0.h"
51	#include "gfxhub_v1_1.h"
52	#include "gfxhub_v1_2.h"
53	#include "mmhub_v9_4.h"
54	#include "mmhub_v1_7.h"
55	#include "mmhub_v1_8.h"
56	#include "umc_v6_1.h"
57	#include "umc_v6_0.h"
58	#include "umc_v6_7.h"
59	#include "umc_v12_0.h"
60	#include "hdp_v4_0.h"
61	#include "mca_v3_0.h"
62
63	#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
64
65	#include "amdgpu_ras.h"
66	#include "amdgpu_xgmi.h"
67
68	/* add these here since we already include dce12 headers and these are for DCN */
69	#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d
70	#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2
71	#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0
72	#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10
73	#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL
74	#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L
75	#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d
76	#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2
77
78	#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2 0x05ea
79	#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2_BASE_IDX 2
80
81	#define MAX_MEM_RANGES 8
82
83	static const char * const gfxhub_client_ids[] = {
84	"CB",
85	"DB",
86	"IA",
87	"WD",
88	"CPF",
89	"CPC",
90	"CPG",
91	"RLC",
92	"TCP",
93	"SQC (inst)",
94	"SQC (data)",
95	"SQG",
96	"PA",
97	};
98
99	static const char *mmhub_client_ids_raven[][2] = {
100	[0][0] = "MP1",
101	[1][0] = "MP0",
102	[2][0] = "VCN",
103	[3][0] = "VCNU",
104	[4][0] = "HDP",
105	[5][0] = "DCE",
106	[13][0] = "UTCL2",
107	[19][0] = "TLS",
108	[26][0] = "OSS",
109	[27][0] = "SDMA0",
110	[0][1] = "MP1",
111	[1][1] = "MP0",
112	[2][1] = "VCN",
113	[3][1] = "VCNU",
114	[4][1] = "HDP",
115	[5][1] = "XDP",
116	[6][1] = "DBGU0",
117	[7][1] = "DCE",
118	[8][1] = "DCEDWB0",
119	[9][1] = "DCEDWB1",
120	[26][1] = "OSS",
121	[27][1] = "SDMA0",
122	};
123
124	static const char *mmhub_client_ids_renoir[][2] = {
125	[0][0] = "MP1",
126	[1][0] = "MP0",
127	[2][0] = "HDP",
128	[4][0] = "DCEDMC",
129	[5][0] = "DCEVGA",
130	[13][0] = "UTCL2",
131	[19][0] = "TLS",
132	[26][0] = "OSS",
133	[27][0] = "SDMA0",
134	[28][0] = "VCN",
135	[29][0] = "VCNU",
136	[30][0] = "JPEG",
137	[0][1] = "MP1",
138	[1][1] = "MP0",
139	[2][1] = "HDP",
140	[3][1] = "XDP",
141	[6][1] = "DBGU0",
142	[7][1] = "DCEDMC",
143	[8][1] = "DCEVGA",
144	[9][1] = "DCEDWB",
145	[26][1] = "OSS",
146	[27][1] = "SDMA0",
147	[28][1] = "VCN",
148	[29][1] = "VCNU",
149	[30][1] = "JPEG",
150	};
151
152	static const char *mmhub_client_ids_vega10[][2] = {
153	[0][0] = "MP0",
154	[1][0] = "UVD",
155	[2][0] = "UVDU",
156	[3][0] = "HDP",
157	[13][0] = "UTCL2",
158	[14][0] = "OSS",
159	[15][0] = "SDMA1",
160	[32+0][0] = "VCE0",
161	[32+1][0] = "VCE0U",
162	[32+2][0] = "XDMA",
163	[32+3][0] = "DCE",
164	[32+4][0] = "MP1",
165	[32+14][0] = "SDMA0",
166	[0][1] = "MP0",
167	[1][1] = "UVD",
168	[2][1] = "UVDU",
169	[3][1] = "DBGU0",
170	[4][1] = "HDP",
171	[5][1] = "XDP",
172	[14][1] = "OSS",
173	[15][1] = "SDMA0",
174	[32+0][1] = "VCE0",
175	[32+1][1] = "VCE0U",
176	[32+2][1] = "XDMA",
177	[32+3][1] = "DCE",
178	[32+4][1] = "DCEDWB",
179	[32+5][1] = "MP1",
180	[32+6][1] = "DBGU1",
181	[32+14][1] = "SDMA1",
182	};
183
184	static const char *mmhub_client_ids_vega12[][2] = {
185	[0][0] = "MP0",
186	[1][0] = "VCE0",
187	[2][0] = "VCE0U",
188	[3][0] = "HDP",
189	[13][0] = "UTCL2",
190	[14][0] = "OSS",
191	[15][0] = "SDMA1",
192	[32+0][0] = "DCE",
193	[32+1][0] = "XDMA",
194	[32+2][0] = "UVD",
195	[32+3][0] = "UVDU",
196	[32+4][0] = "MP1",
197	[32+15][0] = "SDMA0",
198	[0][1] = "MP0",
199	[1][1] = "VCE0",
200	[2][1] = "VCE0U",
201	[3][1] = "DBGU0",
202	[4][1] = "HDP",
203	[5][1] = "XDP",
204	[14][1] = "OSS",
205	[15][1] = "SDMA0",
206	[32+0][1] = "DCE",
207	[32+1][1] = "DCEDWB",
208	[32+2][1] = "XDMA",
209	[32+3][1] = "UVD",
210	[32+4][1] = "UVDU",
211	[32+5][1] = "MP1",
212	[32+6][1] = "DBGU1",
213	[32+15][1] = "SDMA1",
214	};
215
216	static const char *mmhub_client_ids_vega20[][2] = {
217	[0][0] = "XDMA",
218	[1][0] = "DCE",
219	[2][0] = "VCE0",
220	[3][0] = "VCE0U",
221	[4][0] = "UVD",
222	[5][0] = "UVD1U",
223	[13][0] = "OSS",
224	[14][0] = "HDP",
225	[15][0] = "SDMA0",
226	[32+0][0] = "UVD",
227	[32+1][0] = "UVDU",
228	[32+2][0] = "MP1",
229	[32+3][0] = "MP0",
230	[32+12][0] = "UTCL2",
231	[32+14][0] = "SDMA1",
232	[0][1] = "XDMA",
233	[1][1] = "DCE",
234	[2][1] = "DCEDWB",
235	[3][1] = "VCE0",
236	[4][1] = "VCE0U",
237	[5][1] = "UVD1",
238	[6][1] = "UVD1U",
239	[7][1] = "DBGU0",
240	[8][1] = "XDP",
241	[13][1] = "OSS",
242	[14][1] = "HDP",
243	[15][1] = "SDMA0",
244	[32+0][1] = "UVD",
245	[32+1][1] = "UVDU",
246	[32+2][1] = "DBGU1",
247	[32+3][1] = "MP1",
248	[32+4][1] = "MP0",
249	[32+14][1] = "SDMA1",
250	};
251
252	static const char *mmhub_client_ids_arcturus[][2] = {
253	[0][0] = "DBGU1",
254	[1][0] = "XDP",
255	[2][0] = "MP1",
256	[14][0] = "HDP",
257	[171][0] = "JPEG",
258	[172][0] = "VCN",
259	[173][0] = "VCNU",
260	[203][0] = "JPEG1",
261	[204][0] = "VCN1",
262	[205][0] = "VCN1U",
263	[256][0] = "SDMA0",
264	[257][0] = "SDMA1",
265	[258][0] = "SDMA2",
266	[259][0] = "SDMA3",
267	[260][0] = "SDMA4",
268	[261][0] = "SDMA5",
269	[262][0] = "SDMA6",
270	[263][0] = "SDMA7",
271	[384][0] = "OSS",
272	[0][1] = "DBGU1",
273	[1][1] = "XDP",
274	[2][1] = "MP1",
275	[14][1] = "HDP",
276	[171][1] = "JPEG",
277	[172][1] = "VCN",
278	[173][1] = "VCNU",
279	[203][1] = "JPEG1",
280	[204][1] = "VCN1",
281	[205][1] = "VCN1U",
282	[256][1] = "SDMA0",
283	[257][1] = "SDMA1",
284	[258][1] = "SDMA2",
285	[259][1] = "SDMA3",
286	[260][1] = "SDMA4",
287	[261][1] = "SDMA5",
288	[262][1] = "SDMA6",
289	[263][1] = "SDMA7",
290	[384][1] = "OSS",
291	};
292
293	static const char *mmhub_client_ids_aldebaran[][2] = {
294	[2][0] = "MP1",
295	[3][0] = "MP0",
296	[32+1][0] = "DBGU_IO0",
297	[32+2][0] = "DBGU_IO2",
298	[32+4][0] = "MPIO",
299	[96+11][0] = "JPEG0",
300	[96+12][0] = "VCN0",
301	[96+13][0] = "VCNU0",
302	[128+11][0] = "JPEG1",
303	[128+12][0] = "VCN1",
304	[128+13][0] = "VCNU1",
305	[160+1][0] = "XDP",
306	[160+14][0] = "HDP",
307	[256+0][0] = "SDMA0",
308	[256+1][0] = "SDMA1",
309	[256+2][0] = "SDMA2",
310	[256+3][0] = "SDMA3",
311	[256+4][0] = "SDMA4",
312	[384+0][0] = "OSS",
313	[2][1] = "MP1",
314	[3][1] = "MP0",
315	[32+1][1] = "DBGU_IO0",
316	[32+2][1] = "DBGU_IO2",
317	[32+4][1] = "MPIO",
318	[96+11][1] = "JPEG0",
319	[96+12][1] = "VCN0",
320	[96+13][1] = "VCNU0",
321	[128+11][1] = "JPEG1",
322	[128+12][1] = "VCN1",
323	[128+13][1] = "VCNU1",
324	[160+1][1] = "XDP",
325	[160+14][1] = "HDP",
326	[256+0][1] = "SDMA0",
327	[256+1][1] = "SDMA1",
328	[256+2][1] = "SDMA2",
329	[256+3][1] = "SDMA3",
330	[256+4][1] = "SDMA4",
331	[384+0][1] = "OSS",
332	};
333
334	static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = {
335	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
336	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
337	};
338
339	static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = {
340	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
341	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
342	};
343
344	static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
345	(0x000143c0 + 0x00000000),
346	(0x000143c0 + 0x00000800),
347	(0x000143c0 + 0x00001000),
348	(0x000143c0 + 0x00001800),
349	(0x000543c0 + 0x00000000),
350	(0x000543c0 + 0x00000800),
351	(0x000543c0 + 0x00001000),
352	(0x000543c0 + 0x00001800),
353	(0x000943c0 + 0x00000000),
354	(0x000943c0 + 0x00000800),
355	(0x000943c0 + 0x00001000),
356	(0x000943c0 + 0x00001800),
357	(0x000d43c0 + 0x00000000),
358	(0x000d43c0 + 0x00000800),
359	(0x000d43c0 + 0x00001000),
360	(0x000d43c0 + 0x00001800),
361	(0x001143c0 + 0x00000000),
362	(0x001143c0 + 0x00000800),
363	(0x001143c0 + 0x00001000),
364	(0x001143c0 + 0x00001800),
365	(0x001543c0 + 0x00000000),
366	(0x001543c0 + 0x00000800),
367	(0x001543c0 + 0x00001000),
368	(0x001543c0 + 0x00001800),
369	(0x001943c0 + 0x00000000),
370	(0x001943c0 + 0x00000800),
371	(0x001943c0 + 0x00001000),
372	(0x001943c0 + 0x00001800),
373	(0x001d43c0 + 0x00000000),
374	(0x001d43c0 + 0x00000800),
375	(0x001d43c0 + 0x00001000),
376	(0x001d43c0 + 0x00001800),
377	};
378
379	static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
380	(0x000143e0 + 0x00000000),
381	(0x000143e0 + 0x00000800),
382	(0x000143e0 + 0x00001000),
383	(0x000143e0 + 0x00001800),
384	(0x000543e0 + 0x00000000),
385	(0x000543e0 + 0x00000800),
386	(0x000543e0 + 0x00001000),
387	(0x000543e0 + 0x00001800),
388	(0x000943e0 + 0x00000000),
389	(0x000943e0 + 0x00000800),
390	(0x000943e0 + 0x00001000),
391	(0x000943e0 + 0x00001800),
392	(0x000d43e0 + 0x00000000),
393	(0x000d43e0 + 0x00000800),
394	(0x000d43e0 + 0x00001000),
395	(0x000d43e0 + 0x00001800),
396	(0x001143e0 + 0x00000000),
397	(0x001143e0 + 0x00000800),
398	(0x001143e0 + 0x00001000),
399	(0x001143e0 + 0x00001800),
400	(0x001543e0 + 0x00000000),
401	(0x001543e0 + 0x00000800),
402	(0x001543e0 + 0x00001000),
403	(0x001543e0 + 0x00001800),
404	(0x001943e0 + 0x00000000),
405	(0x001943e0 + 0x00000800),
406	(0x001943e0 + 0x00001000),
407	(0x001943e0 + 0x00001800),
408	(0x001d43e0 + 0x00000000),
409	(0x001d43e0 + 0x00000800),
410	(0x001d43e0 + 0x00001000),
411	(0x001d43e0 + 0x00001800),
412	};
413
414	static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
415	struct amdgpu_irq_src *src,
416	unsigned int type,
417	enum amdgpu_interrupt_state state)
418	{
419	u32 bits, i, tmp, reg;
420
421	/* Devices newer then VEGA10/12 shall have these programming
422	* sequences performed by PSP BL
423	*/
424	if (adev->asic_type >= CHIP_VEGA20)
425	return 0;
426
427	bits = 0x7f;
428
429	switch (state) {
430	case AMDGPU_IRQ_STATE_DISABLE:
431	for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
432	reg = ecc_umc_mcumc_ctrl_addrs[i];
433	tmp = RREG32(reg);
434	tmp &= ~bits;
435	WREG32(reg, tmp);
436	}
437	for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
438	reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
439	tmp = RREG32(reg);
440	tmp &= ~bits;
441	WREG32(reg, tmp);
442	}
443	break;
444	case AMDGPU_IRQ_STATE_ENABLE:
445	for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
446	reg = ecc_umc_mcumc_ctrl_addrs[i];
447	tmp = RREG32(reg);
448	tmp |= bits;
449	WREG32(reg, tmp);
450	}
451	for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
452	reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
453	tmp = RREG32(reg);
454	tmp |= bits;
455	WREG32(reg, tmp);
456	}
457	break;
458	default:
459	break;
460	}
461
462	return 0;
463	}
464
465	static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
466	struct amdgpu_irq_src *src,
467	unsigned int type,
468	enum amdgpu_interrupt_state state)
469	{
470	struct amdgpu_vmhub *hub;
471	u32 tmp, reg, bits, i, j;
472
473	bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
474	VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
475	VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
476	VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
477	VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
478	VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
479	VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
480
481	switch (state) {
482	case AMDGPU_IRQ_STATE_DISABLE:
483	for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
484	hub = &adev->vmhub[j];
485	for (i = 0; i < 16; i++) {
486	reg = hub->vm_context0_cntl + i;
487
488	/* This works because this interrupt is only
489	* enabled at init/resume and disabled in
490	* fini/suspend, so the overall state doesn't
491	* change over the course of suspend/resume.
492	*/
493	if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
494	continue;
495
496	if (j >= AMDGPU_MMHUB0(0))
497	tmp = RREG32_SOC15_IP(MMHUB, reg);
498	else
499	tmp = RREG32_XCC(reg, j);
500
501	tmp &= ~bits;
502
503	if (j >= AMDGPU_MMHUB0(0))
504	WREG32_SOC15_IP(MMHUB, reg, tmp);
505	else
506	WREG32_XCC(reg, tmp, j);
507	}
508	}
509	break;
510	case AMDGPU_IRQ_STATE_ENABLE:
511	for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
512	hub = &adev->vmhub[j];
513	for (i = 0; i < 16; i++) {
514	reg = hub->vm_context0_cntl + i;
515
516	/* This works because this interrupt is only
517	* enabled at init/resume and disabled in
518	* fini/suspend, so the overall state doesn't
519	* change over the course of suspend/resume.
520	*/
521	if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
522	continue;
523
524	if (j >= AMDGPU_MMHUB0(0))
525	tmp = RREG32_SOC15_IP(MMHUB, reg);
526	else
527	tmp = RREG32_XCC(reg, j);
528
529	tmp |= bits;
530
531	if (j >= AMDGPU_MMHUB0(0))
532	WREG32_SOC15_IP(MMHUB, reg, tmp);
533	else
534	WREG32_XCC(reg, tmp, j);
535	}
536	}
537	break;
538	default:
539	break;
540	}
541
542	return 0;
543	}
544
545	static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
546	struct amdgpu_irq_src *source,
547	struct amdgpu_iv_entry *entry)
548	{
549	bool retry_fault = !!(entry->src_data[1] & 0x80);
550	bool write_fault = !!(entry->src_data[1] & 0x20);
551	uint32_t status = 0, cid = 0, rw = 0;
552	struct amdgpu_task_info *task_info;
553	struct amdgpu_vmhub *hub;
554	const char *mmhub_cid;
555	const char *hub_name;
556	unsigned int vmhub;
557	u64 addr;
558	uint32_t cam_index = 0;
559	int ret, xcc_id = 0;
560	uint32_t node_id;
561
562	node_id = entry->node_id;
563
564	addr = (u64)entry->src_data[0] << 12;
565	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
566
567	if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
568	hub_name = "mmhub0";
569	vmhub = AMDGPU_MMHUB0(node_id / 4);
570	} else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
571	hub_name = "mmhub1";
572	vmhub = AMDGPU_MMHUB1(0);
573	} else {
574	hub_name = "gfxhub0";
575	if (adev->gfx.funcs->ih_node_to_logical_xcc) {
576	xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev,
577	node_id);
578	if (xcc_id < 0)
579	xcc_id = 0;
580	}
581	vmhub = xcc_id;
582	}
583	hub = &adev->vmhub[vmhub];
584
585	if (retry_fault) {
586	if (adev->irq.retry_cam_enabled) {
587	/* Delegate it to a different ring if the hardware hasn't
588	* already done it.
589	*/
590	if (entry->ih == &adev->irq.ih) {
591	amdgpu_irq_delegate(adev, entry, num_dw: 8);
592	return 1;
593	}
594
595	cam_index = entry->src_data[2] & 0x3ff;
596
597	ret = amdgpu_vm_handle_fault(adev, pasid: entry->pasid, vmid: entry->vmid, node_id,
598	addr, write_fault);
599	WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
600	if (ret)
601	return 1;
602	} else {
603	/* Process it onyl if it's the first fault for this address */
604	if (entry->ih != &adev->irq.ih_soft &&
605	amdgpu_gmc_filter_faults(adev, ih: entry->ih, addr, pasid: entry->pasid,
606	timestamp: entry->timestamp))
607	return 1;
608
609	/* Delegate it to a different ring if the hardware hasn't
610	* already done it.
611	*/
612	if (entry->ih == &adev->irq.ih) {
613	amdgpu_irq_delegate(adev, entry, num_dw: 8);
614	return 1;
615	}
616
617	/* Try to handle the recoverable page faults by filling page
618	* tables
619	*/
620	if (amdgpu_vm_handle_fault(adev, pasid: entry->pasid, vmid: entry->vmid, node_id,
621	addr, write_fault))
622	return 1;
623	}
624	}
625
626	if (!printk_ratelimit())
627	return 0;
628
629	dev_err(adev->dev,
630	"[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", hub_name,
631	retry_fault ? "retry" : "no-retry",
632	entry->src_id, entry->ring_id, entry->vmid, entry->pasid);
633
634	task_info = amdgpu_vm_get_task_info_pasid(adev, pasid: entry->pasid);
635	if (task_info) {
636	dev_err(adev->dev,
637	" for process %s pid %d thread %s pid %d)\n",
638	task_info->process_name, task_info->tgid,
639	task_info->task_name, task_info->pid);
640	amdgpu_vm_put_task_info(task_info);
641	}
642
643	dev_err(adev->dev, " in page starting at address 0x%016llx from IH client 0x%x (%s)\n",
644	addr, entry->client_id,
645	soc15_ih_clientid_name[entry->client_id]);
646
647	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3))
648	dev_err(adev->dev, " cookie node_id %d fault from die %s%d%s\n",
649	node_id, node_id % 4 == 3 ? "RSV" : "AID", node_id / 4,
650	node_id % 4 == 1 ? ".XCD0" : node_id % 4 == 2 ? ".XCD1" : "");
651
652	if (amdgpu_sriov_vf(adev))
653	return 0;
654
655	/*
656	* Issue a dummy read to wait for the status register to
657	* be updated to avoid reading an incorrect value due to
658	* the new fast GRBM interface.
659	*/
660	if ((entry->vmid_src == AMDGPU_GFXHUB(0)) &&
661	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) < IP_VERSION(9, 4, 2)))
662	RREG32(hub->vm_l2_pro_fault_status);
663
664	status = RREG32(hub->vm_l2_pro_fault_status);
665	cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID);
666	rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW);
667	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
668
669	amdgpu_vm_update_fault_cache(adev, pasid: entry->pasid, addr, status, vmhub);
670
671	dev_err(adev->dev,
672	"VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
673	status);
674	if (entry->vmid_src == AMDGPU_GFXHUB(0)) {
675	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
676	cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" :
677	gfxhub_client_ids[cid],
678	cid);
679	} else {
680	switch (amdgpu_ip_version(adev, ip: MMHUB_HWIP, inst: 0)) {
681	case IP_VERSION(9, 0, 0):
682	mmhub_cid = mmhub_client_ids_vega10[cid][rw];
683	break;
684	case IP_VERSION(9, 3, 0):
685	mmhub_cid = mmhub_client_ids_vega12[cid][rw];
686	break;
687	case IP_VERSION(9, 4, 0):
688	mmhub_cid = mmhub_client_ids_vega20[cid][rw];
689	break;
690	case IP_VERSION(9, 4, 1):
691	mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
692	break;
693	case IP_VERSION(9, 1, 0):
694	case IP_VERSION(9, 2, 0):
695	mmhub_cid = mmhub_client_ids_raven[cid][rw];
696	break;
697	case IP_VERSION(1, 5, 0):
698	case IP_VERSION(2, 4, 0):
699	mmhub_cid = mmhub_client_ids_renoir[cid][rw];
700	break;
701	case IP_VERSION(1, 8, 0):
702	case IP_VERSION(9, 4, 2):
703	mmhub_cid = mmhub_client_ids_aldebaran[cid][rw];
704	break;
705	default:
706	mmhub_cid = NULL;
707	break;
708	}
709	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
710	mmhub_cid ? mmhub_cid : "unknown", cid);
711	}
712	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
713	REG_GET_FIELD(status,
714	VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
715	dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
716	REG_GET_FIELD(status,
717	VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
718	dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
719	REG_GET_FIELD(status,
720	VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
721	dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
722	REG_GET_FIELD(status,
723	VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
724	dev_err(adev->dev, "\t RW: 0x%x\n", rw);
725	return 0;
726	}
727
728	static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
729	.set = gmc_v9_0_vm_fault_interrupt_state,
730	.process = gmc_v9_0_process_interrupt,
731	};
732
733
734	static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
735	.set = gmc_v9_0_ecc_interrupt_state,
736	.process = amdgpu_umc_process_ecc_irq,
737	};
738
739	static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
740	{
741	adev->gmc.vm_fault.num_types = 1;
742	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
743
744	if (!amdgpu_sriov_vf(adev) &&
745	!adev->gmc.xgmi.connected_to_cpu &&
746	!adev->gmc.is_app_apu) {
747	adev->gmc.ecc_irq.num_types = 1;
748	adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
749	}
750	}
751
752	static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
753	uint32_t flush_type)
754	{
755	u32 req = 0;
756
757	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
758	PER_VMID_INVALIDATE_REQ, 1 << vmid);
759	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
760	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
761	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
762	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
763	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
764	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
765	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
766	CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
767
768	return req;
769	}
770
771	/**
772	* gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore
773	*
774	* @adev: amdgpu_device pointer
775	* @vmhub: vmhub type
776	*
777	*/
778	static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
779	uint32_t vmhub)
780	{
781	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2) ||
782	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3))
783	return false;
784
785	return ((vmhub == AMDGPU_MMHUB0(0) ||
786	vmhub == AMDGPU_MMHUB1(0)) &&
787	(!amdgpu_sriov_vf(adev)) &&
788	(!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
789	(adev->apu_flags & AMD_APU_IS_PICASSO))));
790	}
791
792	static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
793	uint8_t vmid, uint16_t *p_pasid)
794	{
795	uint32_t value;
796
797	value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
798	+ vmid);
799	*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
800
801	return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
802	}
803
804	/*
805	* GART
806	* VMID 0 is the physical GPU addresses as used by the kernel.
807	* VMIDs 1-15 are used for userspace clients and are handled
808	* by the amdgpu vm/hsa code.
809	*/
810
811	/**
812	* gmc_v9_0_flush_gpu_tlb - tlb flush with certain type
813	*
814	* @adev: amdgpu_device pointer
815	* @vmid: vm instance to flush
816	* @vmhub: which hub to flush
817	* @flush_type: the flush type
818	*
819	* Flush the TLB for the requested page table using certain type.
820	*/
821	static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
822	uint32_t vmhub, uint32_t flush_type)
823	{
824	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
825	u32 j, inv_req, tmp, sem, req, ack, inst;
826	const unsigned int eng = 17;
827	struct amdgpu_vmhub *hub;
828
829	BUG_ON(vmhub >= AMDGPU_MAX_VMHUBS);
830
831	hub = &adev->vmhub[vmhub];
832	inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
833	sem = hub->vm_inv_eng0_sem + hub->eng_distance * eng;
834	req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
835	ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
836
837	if (vmhub >= AMDGPU_MMHUB0(0))
838	inst = GET_INST(GC, 0);
839	else
840	inst = vmhub;
841
842	/* This is necessary for SRIOV as well as for GFXOFF to function
843	* properly under bare metal
844	*/
845	if (adev->gfx.kiq[inst].ring.sched.ready &&
846	(amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) {
847	uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
848	uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
849
850	amdgpu_gmc_fw_reg_write_reg_wait(adev, reg0: req, reg1: ack, ref: inv_req,
851	mask: 1 << vmid, xcc_inst: inst);
852	return;
853	}
854
855	/* This path is needed before KIQ/MES/GFXOFF are set up */
856	spin_lock(lock: &adev->gmc.invalidate_lock);
857
858	/*
859	* It may lose gpuvm invalidate acknowldege state across power-gating
860	* off cycle, add semaphore acquire before invalidation and semaphore
861	* release after invalidation to avoid entering power gated state
862	* to WA the Issue
863	*/
864
865	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
866	if (use_semaphore) {
867	for (j = 0; j < adev->usec_timeout; j++) {
868	/* a read return value of 1 means semaphore acquire */
869	if (vmhub >= AMDGPU_MMHUB0(0))
870	tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, sem, inst);
871	else
872	tmp = RREG32_SOC15_IP_NO_KIQ(GC, sem, inst);
873	if (tmp & 0x1)
874	break;
875	udelay(1);
876	}
877
878	if (j >= adev->usec_timeout)
879	DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
880	}
881
882	if (vmhub >= AMDGPU_MMHUB0(0))
883	WREG32_SOC15_IP_NO_KIQ(MMHUB, req, inv_req, inst);
884	else
885	WREG32_SOC15_IP_NO_KIQ(GC, req, inv_req, inst);
886
887	/*
888	* Issue a dummy read to wait for the ACK register to
889	* be cleared to avoid a false ACK due to the new fast
890	* GRBM interface.
891	*/
892	if ((vmhub == AMDGPU_GFXHUB(0)) &&
893	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) < IP_VERSION(9, 4, 2)))
894	RREG32_NO_KIQ(req);
895
896	for (j = 0; j < adev->usec_timeout; j++) {
897	if (vmhub >= AMDGPU_MMHUB0(0))
898	tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, ack, inst);
899	else
900	tmp = RREG32_SOC15_IP_NO_KIQ(GC, ack, inst);
901	if (tmp & (1 << vmid))
902	break;
903	udelay(1);
904	}
905
906	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
907	if (use_semaphore) {
908	/*
909	* add semaphore release after invalidation,
910	* write with 0 means semaphore release
911	*/
912	if (vmhub >= AMDGPU_MMHUB0(0))
913	WREG32_SOC15_IP_NO_KIQ(MMHUB, sem, 0, inst);
914	else
915	WREG32_SOC15_IP_NO_KIQ(GC, sem, 0, inst);
916	}
917
918	spin_unlock(lock: &adev->gmc.invalidate_lock);
919
920	if (j < adev->usec_timeout)
921	return;
922
923	DRM_ERROR("Timeout waiting for VM flush ACK!\n");
924	}
925
926	/**
927	* gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid
928	*
929	* @adev: amdgpu_device pointer
930	* @pasid: pasid to be flush
931	* @flush_type: the flush type
932	* @all_hub: flush all hubs
933	* @inst: is used to select which instance of KIQ to use for the invalidation
934	*
935	* Flush the TLB for the requested pasid.
936	*/
937	static void gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
938	uint16_t pasid, uint32_t flush_type,
939	bool all_hub, uint32_t inst)
940	{
941	uint16_t queried;
942	int i, vmid;
943
944	for (vmid = 1; vmid < 16; vmid++) {
945	bool valid;
946
947	valid = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
948	p_pasid: &queried);
949	if (!valid || queried != pasid)
950	continue;
951
952	if (all_hub) {
953	for_each_set_bit(i, adev->vmhubs_mask,
954	AMDGPU_MAX_VMHUBS)
955	gmc_v9_0_flush_gpu_tlb(adev, vmid, vmhub: i,
956	flush_type);
957	} else {
958	gmc_v9_0_flush_gpu_tlb(adev, vmid,
959	AMDGPU_GFXHUB(0),
960	flush_type);
961	}
962	}
963	}
964
965	static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
966	unsigned int vmid, uint64_t pd_addr)
967	{
968	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev: ring->adev, vmhub: ring->vm_hub);
969	struct amdgpu_device *adev = ring->adev;
970	struct amdgpu_vmhub *hub = &adev->vmhub[ring->vm_hub];
971	uint32_t req = gmc_v9_0_get_invalidate_req(vmid, flush_type: 0);
972	unsigned int eng = ring->vm_inv_eng;
973
974	/*
975	* It may lose gpuvm invalidate acknowldege state across power-gating
976	* off cycle, add semaphore acquire before invalidation and semaphore
977	* release after invalidation to avoid entering power gated state
978	* to WA the Issue
979	*/
980
981	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
982	if (use_semaphore)
983	/* a read return value of 1 means semaphore acuqire */
984	amdgpu_ring_emit_reg_wait(ring,
985	hub->vm_inv_eng0_sem +
986	hub->eng_distance * eng, 0x1, 0x1);
987
988	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
989	(hub->ctx_addr_distance * vmid),
990	lower_32_bits(pd_addr));
991
992	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
993	(hub->ctx_addr_distance * vmid),
994	upper_32_bits(pd_addr));
995
996	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
997	hub->eng_distance * eng,
998	hub->vm_inv_eng0_ack +
999	hub->eng_distance * eng,
1000	req, 1 << vmid);
1001
1002	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
1003	if (use_semaphore)
1004	/*
1005	* add semaphore release after invalidation,
1006	* write with 0 means semaphore release
1007	*/
1008	amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
1009	hub->eng_distance * eng, 0);
1010
1011	return pd_addr;
1012	}
1013
1014	static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid,
1015	unsigned int pasid)
1016	{
1017	struct amdgpu_device *adev = ring->adev;
1018	uint32_t reg;
1019
1020	/* Do nothing because there's no lut register for mmhub1. */
1021	if (ring->vm_hub == AMDGPU_MMHUB1(0))
1022	return;
1023
1024	if (ring->vm_hub == AMDGPU_GFXHUB(0))
1025	reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
1026	else
1027	reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
1028
1029	amdgpu_ring_emit_wreg(ring, reg, pasid);
1030	}
1031
1032	/*
1033	* PTE format on VEGA 10:
1034	* 63:59 reserved
1035	* 58:57 mtype
1036	* 56 F
1037	* 55 L
1038	* 54 P
1039	* 53 SW
1040	* 52 T
1041	* 50:48 reserved
1042	* 47:12 4k physical page base address
1043	* 11:7 fragment
1044	* 6 write
1045	* 5 read
1046	* 4 exe
1047	* 3 Z
1048	* 2 snooped
1049	* 1 system
1050	* 0 valid
1051	*
1052	* PDE format on VEGA 10:
1053	* 63:59 block fragment size
1054	* 58:55 reserved
1055	* 54 P
1056	* 53:48 reserved
1057	* 47:6 physical base address of PD or PTE
1058	* 5:3 reserved
1059	* 2 C
1060	* 1 system
1061	* 0 valid
1062	*/
1063
1064	static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
1065
1066	{
1067	switch (flags) {
1068	case AMDGPU_VM_MTYPE_DEFAULT:
1069	return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1070	case AMDGPU_VM_MTYPE_NC:
1071	return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1072	case AMDGPU_VM_MTYPE_WC:
1073	return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC);
1074	case AMDGPU_VM_MTYPE_RW:
1075	return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW);
1076	case AMDGPU_VM_MTYPE_CC:
1077	return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
1078	case AMDGPU_VM_MTYPE_UC:
1079	return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC);
1080	default:
1081	return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1082	}
1083	}
1084
1085	static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1086	uint64_t *addr, uint64_t *flags)
1087	{
1088	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
1089	*addr = amdgpu_gmc_vram_mc2pa(adev, mc_addr: *addr);
1090	BUG_ON(*addr & 0xFFFF00000000003FULL);
1091
1092	if (!adev->gmc.translate_further)
1093	return;
1094
1095	if (level == AMDGPU_VM_PDB1) {
1096	/* Set the block fragment size */
1097	if (!(*flags & AMDGPU_PDE_PTE))
1098	*flags |= AMDGPU_PDE_BFS(0x9);
1099
1100	} else if (level == AMDGPU_VM_PDB0) {
1101	if (*flags & AMDGPU_PDE_PTE) {
1102	*flags &= ~AMDGPU_PDE_PTE;
1103	if (!(*flags & AMDGPU_PTE_VALID))
1104	*addr |= 1 << PAGE_SHIFT;
1105	} else {
1106	*flags |= AMDGPU_PTE_TF;
1107	}
1108	}
1109	}
1110
1111	static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
1112	struct amdgpu_bo *bo,
1113	struct amdgpu_bo_va_mapping *mapping,
1114	uint64_t *flags)
1115	{
1116	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
1117	bool is_vram = bo->tbo.resource->mem_type == TTM_PL_VRAM;
1118	bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT | AMDGPU_GEM_CREATE_EXT_COHERENT);
1119	bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT;
1120	bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
1121	struct amdgpu_vm *vm = mapping->bo_va->base.vm;
1122	unsigned int mtype_local, mtype;
1123	bool snoop = false;
1124	bool is_local;
1125
1126	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1127	case IP_VERSION(9, 4, 1):
1128	case IP_VERSION(9, 4, 2):
1129	if (is_vram) {
1130	if (bo_adev == adev) {
1131	if (uncached)
1132	mtype = MTYPE_UC;
1133	else if (coherent)
1134	mtype = MTYPE_CC;
1135	else
1136	mtype = MTYPE_RW;
1137	/* FIXME: is this still needed? Or does
1138	* amdgpu_ttm_tt_pde_flags already handle this?
1139	*/
1140	if ((amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) ==
1141	IP_VERSION(9, 4, 2) ||
1142	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) ==
1143	IP_VERSION(9, 4, 3)) &&
1144	adev->gmc.xgmi.connected_to_cpu)
1145	snoop = true;
1146	} else {
1147	if (uncached || coherent)
1148	mtype = MTYPE_UC;
1149	else
1150	mtype = MTYPE_NC;
1151	if (mapping->bo_va->is_xgmi)
1152	snoop = true;
1153	}
1154	} else {
1155	if (uncached || coherent)
1156	mtype = MTYPE_UC;
1157	else
1158	mtype = MTYPE_NC;
1159	/* FIXME: is this still needed? Or does
1160	* amdgpu_ttm_tt_pde_flags already handle this?
1161	*/
1162	snoop = true;
1163	}
1164	break;
1165	case IP_VERSION(9, 4, 3):
1166	/* Only local VRAM BOs or system memory on non-NUMA APUs
1167	* can be assumed to be local in their entirety. Choose
1168	* MTYPE_NC as safe fallback for all system memory BOs on
1169	* NUMA systems. Their MTYPE can be overridden per-page in
1170	* gmc_v9_0_override_vm_pte_flags.
1171	*/
1172	mtype_local = MTYPE_RW;
1173	if (amdgpu_mtype_local == 1) {
1174	DRM_INFO_ONCE("Using MTYPE_NC for local memory\n");
1175	mtype_local = MTYPE_NC;
1176	} else if (amdgpu_mtype_local == 2) {
1177	DRM_INFO_ONCE("Using MTYPE_CC for local memory\n");
1178	mtype_local = MTYPE_CC;
1179	} else {
1180	DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
1181	}
1182	is_local = (!is_vram && (adev->flags & AMD_IS_APU) &&
1183	num_possible_nodes() <= 1) ||
1184	(is_vram && adev == bo_adev &&
1185	KFD_XCP_MEM_ID(adev, bo->xcp_id) == vm->mem_id);
1186	snoop = true;
1187	if (uncached) {
1188	mtype = MTYPE_UC;
1189	} else if (ext_coherent) {
1190	if (adev->rev_id)
1191	mtype = is_local ? MTYPE_CC : MTYPE_UC;
1192	else
1193	mtype = MTYPE_UC;
1194	} else if (adev->flags & AMD_IS_APU) {
1195	mtype = is_local ? mtype_local : MTYPE_NC;
1196	} else {
1197	/* dGPU */
1198	if (is_local)
1199	mtype = mtype_local;
1200	else if (is_vram)
1201	mtype = MTYPE_NC;
1202	else
1203	mtype = MTYPE_UC;
1204	}
1205
1206	break;
1207	default:
1208	if (uncached || coherent)
1209	mtype = MTYPE_UC;
1210	else
1211	mtype = MTYPE_NC;
1212
1213	/* FIXME: is this still needed? Or does
1214	* amdgpu_ttm_tt_pde_flags already handle this?
1215	*/
1216	if (!is_vram)
1217	snoop = true;
1218	}
1219
1220	if (mtype != MTYPE_NC)
1221	*flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) |
1222	AMDGPU_PTE_MTYPE_VG10(mtype);
1223	*flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
1224	}
1225
1226	static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1227	struct amdgpu_bo_va_mapping *mapping,
1228	uint64_t *flags)
1229	{
1230	struct amdgpu_bo *bo = mapping->bo_va->base.bo;
1231
1232	*flags &= ~AMDGPU_PTE_EXECUTABLE;
1233	*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1234
1235	*flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1236	*flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK;
1237
1238	if (mapping->flags & AMDGPU_PTE_PRT) {
1239	*flags |= AMDGPU_PTE_PRT;
1240	*flags &= ~AMDGPU_PTE_VALID;
1241	}
1242
1243	if (bo && bo->tbo.resource)
1244	gmc_v9_0_get_coherence_flags(adev, bo: mapping->bo_va->base.bo,
1245	mapping, flags);
1246	}
1247
1248	static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
1249	struct amdgpu_vm *vm,
1250	uint64_t addr, uint64_t *flags)
1251	{
1252	int local_node, nid;
1253
1254	/* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes. Local system
1255	* memory can use more efficient MTYPEs.
1256	*/
1257	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 3))
1258	return;
1259
1260	/* Only direct-mapped memory allows us to determine the NUMA node from
1261	* the DMA address.
1262	*/
1263	if (!adev->ram_is_direct_mapped) {
1264	dev_dbg_ratelimited(adev->dev, "RAM is not direct mapped\n");
1265	return;
1266	}
1267
1268	/* MTYPE_NC is the same default and can be overridden.
1269	* MTYPE_UC will be present if the memory is extended-coherent
1270	* and can also be overridden.
1271	*/
1272	if ((*flags & AMDGPU_PTE_MTYPE_VG10_MASK) !=
1273	AMDGPU_PTE_MTYPE_VG10(MTYPE_NC) &&
1274	(*flags & AMDGPU_PTE_MTYPE_VG10_MASK) !=
1275	AMDGPU_PTE_MTYPE_VG10(MTYPE_UC)) {
1276	dev_dbg_ratelimited(adev->dev, "MTYPE is not NC or UC\n");
1277	return;
1278	}
1279
1280	/* FIXME: Only supported on native mode for now. For carve-out, the
1281	* NUMA affinity of the GPU/VM needs to come from the PCI info because
1282	* memory partitions are not associated with different NUMA nodes.
1283	*/
1284	if (adev->gmc.is_app_apu && vm->mem_id >= 0) {
1285	local_node = adev->gmc.mem_partitions[vm->mem_id].numa.node;
1286	} else {
1287	dev_dbg_ratelimited(adev->dev, "Only native mode APU is supported.\n");
1288	return;
1289	}
1290
1291	/* Only handle real RAM. Mappings of PCIe resources don't have struct
1292	* page or NUMA nodes.
1293	*/
1294	if (!page_is_ram(pfn: addr >> PAGE_SHIFT)) {
1295	dev_dbg_ratelimited(adev->dev, "Page is not RAM.\n");
1296	return;
1297	}
1298	nid = pfn_to_nid(addr >> PAGE_SHIFT);
1299	dev_dbg_ratelimited(adev->dev, "vm->mem_id=%d, local_node=%d, nid=%d\n",
1300	vm->mem_id, local_node, nid);
1301	if (nid == local_node) {
1302	uint64_t old_flags = *flags;
1303	if ((*flags & AMDGPU_PTE_MTYPE_VG10_MASK) ==
1304	AMDGPU_PTE_MTYPE_VG10(MTYPE_NC)) {
1305	unsigned int mtype_local = MTYPE_RW;
1306
1307	if (amdgpu_mtype_local == 1)
1308	mtype_local = MTYPE_NC;
1309	else if (amdgpu_mtype_local == 2)
1310	mtype_local = MTYPE_CC;
1311
1312	*flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) |
1313	AMDGPU_PTE_MTYPE_VG10(mtype_local);
1314	} else if (adev->rev_id) {
1315	/* MTYPE_UC case */
1316	*flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) |
1317	AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
1318	}
1319
1320	dev_dbg_ratelimited(adev->dev, "flags updated from %llx to %llx\n",
1321	old_flags, *flags);
1322	}
1323	}
1324
1325	static unsigned int gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
1326	{
1327	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1328	unsigned int size;
1329
1330	/* TODO move to DC so GMC doesn't need to hard-code DCN registers */
1331
1332	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1333	size = AMDGPU_VBIOS_VGA_ALLOCATION;
1334	} else {
1335	u32 viewport;
1336
1337	switch (amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0)) {
1338	case IP_VERSION(1, 0, 0):
1339	case IP_VERSION(1, 0, 1):
1340	viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1341	size = (REG_GET_FIELD(viewport,
1342	HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1343	REG_GET_FIELD(viewport,
1344	HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1345	4);
1346	break;
1347	case IP_VERSION(2, 1, 0):
1348	viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2);
1349	size = (REG_GET_FIELD(viewport,
1350	HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1351	REG_GET_FIELD(viewport,
1352	HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1353	4);
1354	break;
1355	default:
1356	viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1357	size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1358	REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1359	4);
1360	break;
1361	}
1362	}
1363
1364	return size;
1365	}
1366
1367	static enum amdgpu_memory_partition
1368	gmc_v9_0_get_memory_partition(struct amdgpu_device *adev, u32 *supp_modes)
1369	{
1370	enum amdgpu_memory_partition mode = UNKNOWN_MEMORY_PARTITION_MODE;
1371
1372	if (adev->nbio.funcs->get_memory_partition_mode)
1373	mode = adev->nbio.funcs->get_memory_partition_mode(adev,
1374	supp_modes);
1375
1376	return mode;
1377	}
1378
1379	static enum amdgpu_memory_partition
1380	gmc_v9_0_query_memory_partition(struct amdgpu_device *adev)
1381	{
1382	if (amdgpu_sriov_vf(adev))
1383	return AMDGPU_NPS1_PARTITION_MODE;
1384
1385	return gmc_v9_0_get_memory_partition(adev, NULL);
1386	}
1387
1388	static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1389	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
1390	.flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
1391	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
1392	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
1393	.map_mtype = gmc_v9_0_map_mtype,
1394	.get_vm_pde = gmc_v9_0_get_vm_pde,
1395	.get_vm_pte = gmc_v9_0_get_vm_pte,
1396	.override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags,
1397	.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
1398	.query_mem_partition_mode = &gmc_v9_0_query_memory_partition,
1399	};
1400
1401	static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
1402	{
1403	adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
1404	}
1405
1406	static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1407	{
1408	switch (amdgpu_ip_version(adev, ip: UMC_HWIP, inst: 0)) {
1409	case IP_VERSION(6, 0, 0):
1410	adev->umc.funcs = &umc_v6_0_funcs;
1411	break;
1412	case IP_VERSION(6, 1, 1):
1413	adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1414	adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1415	adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1416	adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
1417	adev->umc.retire_unit = 1;
1418	adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1419	adev->umc.ras = &umc_v6_1_ras;
1420	break;
1421	case IP_VERSION(6, 1, 2):
1422	adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1423	adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1424	adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1425	adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
1426	adev->umc.retire_unit = 1;
1427	adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1428	adev->umc.ras = &umc_v6_1_ras;
1429	break;
1430	case IP_VERSION(6, 7, 0):
1431	adev->umc.max_ras_err_cnt_per_query =
1432	UMC_V6_7_TOTAL_CHANNEL_NUM * UMC_V6_7_BAD_PAGE_NUM_PER_CHANNEL;
1433	adev->umc.channel_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM;
1434	adev->umc.umc_inst_num = UMC_V6_7_UMC_INSTANCE_NUM;
1435	adev->umc.channel_offs = UMC_V6_7_PER_CHANNEL_OFFSET;
1436	adev->umc.retire_unit = (UMC_V6_7_NA_MAP_PA_NUM * 2);
1437	if (!adev->gmc.xgmi.connected_to_cpu)
1438	adev->umc.ras = &umc_v6_7_ras;
1439	if (1 & adev->smuio.funcs->get_die_id(adev))
1440	adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_first[0][0];
1441	else
1442	adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_second[0][0];
1443	break;
1444	case IP_VERSION(12, 0, 0):
1445	adev->umc.max_ras_err_cnt_per_query =
1446	UMC_V12_0_TOTAL_CHANNEL_NUM(adev) * UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL;
1447	adev->umc.channel_inst_num = UMC_V12_0_CHANNEL_INSTANCE_NUM;
1448	adev->umc.umc_inst_num = UMC_V12_0_UMC_INSTANCE_NUM;
1449	adev->umc.node_inst_num /= UMC_V12_0_UMC_INSTANCE_NUM;
1450	adev->umc.channel_offs = UMC_V12_0_PER_CHANNEL_OFFSET;
1451	adev->umc.active_mask = adev->aid_mask;
1452	adev->umc.retire_unit = UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL;
1453	adev->umc.channel_idx_tbl = &umc_v12_0_channel_idx_tbl[0][0][0];
1454	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu)
1455	adev->umc.ras = &umc_v12_0_ras;
1456	break;
1457	default:
1458	break;
1459	}
1460	}
1461
1462	static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1463	{
1464	switch (amdgpu_ip_version(adev, ip: MMHUB_HWIP, inst: 0)) {
1465	case IP_VERSION(9, 4, 1):
1466	adev->mmhub.funcs = &mmhub_v9_4_funcs;
1467	break;
1468	case IP_VERSION(9, 4, 2):
1469	adev->mmhub.funcs = &mmhub_v1_7_funcs;
1470	break;
1471	case IP_VERSION(1, 8, 0):
1472	adev->mmhub.funcs = &mmhub_v1_8_funcs;
1473	break;
1474	default:
1475	adev->mmhub.funcs = &mmhub_v1_0_funcs;
1476	break;
1477	}
1478	}
1479
1480	static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev)
1481	{
1482	switch (amdgpu_ip_version(adev, ip: MMHUB_HWIP, inst: 0)) {
1483	case IP_VERSION(9, 4, 0):
1484	adev->mmhub.ras = &mmhub_v1_0_ras;
1485	break;
1486	case IP_VERSION(9, 4, 1):
1487	adev->mmhub.ras = &mmhub_v9_4_ras;
1488	break;
1489	case IP_VERSION(9, 4, 2):
1490	adev->mmhub.ras = &mmhub_v1_7_ras;
1491	break;
1492	case IP_VERSION(1, 8, 0):
1493	adev->mmhub.ras = &mmhub_v1_8_ras;
1494	break;
1495	default:
1496	/* mmhub ras is not available */
1497	break;
1498	}
1499	}
1500
1501	static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev)
1502	{
1503	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3))
1504	adev->gfxhub.funcs = &gfxhub_v1_2_funcs;
1505	else
1506	adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
1507	}
1508
1509	static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev)
1510	{
1511	adev->hdp.ras = &hdp_v4_0_ras;
1512	}
1513
1514	static void gmc_v9_0_set_mca_ras_funcs(struct amdgpu_device *adev)
1515	{
1516	struct amdgpu_mca *mca = &adev->mca;
1517
1518	/* is UMC the right IP to check for MCA? Maybe DF? */
1519	switch (amdgpu_ip_version(adev, ip: UMC_HWIP, inst: 0)) {
1520	case IP_VERSION(6, 7, 0):
1521	if (!adev->gmc.xgmi.connected_to_cpu) {
1522	mca->mp0.ras = &mca_v3_0_mp0_ras;
1523	mca->mp1.ras = &mca_v3_0_mp1_ras;
1524	mca->mpio.ras = &mca_v3_0_mpio_ras;
1525	}
1526	break;
1527	default:
1528	break;
1529	}
1530	}
1531
1532	static void gmc_v9_0_set_xgmi_ras_funcs(struct amdgpu_device *adev)
1533	{
1534	if (!adev->gmc.xgmi.connected_to_cpu)
1535	adev->gmc.xgmi.ras = &xgmi_ras;
1536	}
1537
1538	static int gmc_v9_0_early_init(void *handle)
1539	{
1540	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1541
1542	/*
1543	* 9.4.0, 9.4.1 and 9.4.3 don't have XGMI defined
1544	* in their IP discovery tables
1545	*/
1546	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 0) ||
1547	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1) ||
1548	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3))
1549	adev->gmc.xgmi.supported = true;
1550
1551	if (amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0) == IP_VERSION(6, 1, 0)) {
1552	adev->gmc.xgmi.supported = true;
1553	adev->gmc.xgmi.connected_to_cpu =
1554	adev->smuio.funcs->is_host_gpu_xgmi_supported(adev);
1555	}
1556
1557	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3)) {
1558	enum amdgpu_pkg_type pkg_type =
1559	adev->smuio.funcs->get_pkg_type(adev);
1560	/* On GFXIP 9.4.3. APU, there is no physical VRAM domain present
1561	* and the APU, can be in used two possible modes:
1562	* - carveout mode
1563	* - native APU mode
1564	* "is_app_apu" can be used to identify the APU in the native
1565	* mode.
1566	*/
1567	adev->gmc.is_app_apu = (pkg_type == AMDGPU_PKG_TYPE_APU &&
1568	!pci_resource_len(adev->pdev, 0));
1569	}
1570
1571	gmc_v9_0_set_gmc_funcs(adev);
1572	gmc_v9_0_set_irq_funcs(adev);
1573	gmc_v9_0_set_umc_funcs(adev);
1574	gmc_v9_0_set_mmhub_funcs(adev);
1575	gmc_v9_0_set_mmhub_ras_funcs(adev);
1576	gmc_v9_0_set_gfxhub_funcs(adev);
1577	gmc_v9_0_set_hdp_ras_funcs(adev);
1578	gmc_v9_0_set_mca_ras_funcs(adev);
1579	gmc_v9_0_set_xgmi_ras_funcs(adev);
1580
1581	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1582	adev->gmc.shared_aperture_end =
1583	adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
1584	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
1585	adev->gmc.private_aperture_end =
1586	adev->gmc.private_aperture_start + (4ULL << 30) - 1;
1587	adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
1588
1589	return 0;
1590	}
1591
1592	static int gmc_v9_0_late_init(void *handle)
1593	{
1594	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1595	int r;
1596
1597	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
1598	if (r)
1599	return r;
1600
1601	/*
1602	* Workaround performance drop issue with VBIOS enables partial
1603	* writes, while disables HBM ECC for vega10.
1604	*/
1605	if (!amdgpu_sriov_vf(adev) &&
1606	(amdgpu_ip_version(adev, ip: UMC_HWIP, inst: 0) == IP_VERSION(6, 0, 0))) {
1607	if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) {
1608	if (adev->df.funcs &&
1609	adev->df.funcs->enable_ecc_force_par_wr_rmw)
1610	adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1611	}
1612	}
1613
1614	if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
1615	amdgpu_ras_reset_error_count(adev, block: AMDGPU_RAS_BLOCK__MMHUB);
1616	amdgpu_ras_reset_error_count(adev, block: AMDGPU_RAS_BLOCK__HDP);
1617	}
1618
1619	r = amdgpu_gmc_ras_late_init(adev);
1620	if (r)
1621	return r;
1622
1623	return amdgpu_irq_get(adev, src: &adev->gmc.vm_fault, type: 0);
1624	}
1625
1626	static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
1627	struct amdgpu_gmc *mc)
1628	{
1629	u64 base = adev->mmhub.funcs->get_fb_location(adev);
1630
1631	amdgpu_gmc_set_agp_default(adev, mc);
1632
1633	/* add the xgmi offset of the physical node */
1634	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1635	if (adev->gmc.xgmi.connected_to_cpu) {
1636	amdgpu_gmc_sysvm_location(adev, mc);
1637	} else {
1638	amdgpu_gmc_vram_location(adev, mc, base);
1639	amdgpu_gmc_gart_location(adev, mc, gart_placement: AMDGPU_GART_PLACEMENT_BEST_FIT);
1640	if (!amdgpu_sriov_vf(adev) && (amdgpu_agp == 1))
1641	amdgpu_gmc_agp_location(adev, mc);
1642	}
1643	/* base offset of vram pages */
1644	adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
1645
1646	/* XXX: add the xgmi offset of the physical node? */
1647	adev->vm_manager.vram_base_offset +=
1648	adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1649	}
1650
1651	/**
1652	* gmc_v9_0_mc_init - initialize the memory controller driver params
1653	*
1654	* @adev: amdgpu_device pointer
1655	*
1656	* Look up the amount of vram, vram width, and decide how to place
1657	* vram and gart within the GPU's physical address space.
1658	* Returns 0 for success.
1659	*/
1660	static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1661	{
1662	int r;
1663
1664	/* size in MB on si */
1665	if (!adev->gmc.is_app_apu) {
1666	adev->gmc.mc_vram_size =
1667	adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
1668	} else {
1669	DRM_DEBUG("Set mc_vram_size = 0 for APP APU\n");
1670	adev->gmc.mc_vram_size = 0;
1671	}
1672	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
1673
1674	if (!(adev->flags & AMD_IS_APU) &&
1675	!adev->gmc.xgmi.connected_to_cpu) {
1676	r = amdgpu_device_resize_fb_bar(adev);
1677	if (r)
1678	return r;
1679	}
1680	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
1681	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
1682
1683	#ifdef CONFIG_X86_64
1684	/*
1685	* AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi
1686	* interface can use VRAM through here as it appears system reserved
1687	* memory in host address space.
1688	*
1689	* For APUs, VRAM is just the stolen system memory and can be accessed
1690	* directly.
1691	*
1692	* Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR.
1693	*/
1694
1695	/* check whether both host-gpu and gpu-gpu xgmi links exist */
1696	if ((!amdgpu_sriov_vf(adev) &&
1697	(adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) ||
1698	(adev->gmc.xgmi.supported &&
1699	adev->gmc.xgmi.connected_to_cpu)) {
1700	adev->gmc.aper_base =
1701	adev->gfxhub.funcs->get_mc_fb_offset(adev) +
1702	adev->gmc.xgmi.physical_node_id *
1703	adev->gmc.xgmi.node_segment_size;
1704	adev->gmc.aper_size = adev->gmc.real_vram_size;
1705	}
1706
1707	#endif
1708	adev->gmc.visible_vram_size = adev->gmc.aper_size;
1709
1710	/* set the gart size */
1711	if (amdgpu_gart_size == -1) {
1712	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1713	case IP_VERSION(9, 0, 1): /* all engines support GPUVM */
1714	case IP_VERSION(9, 2, 1): /* all engines support GPUVM */
1715	case IP_VERSION(9, 4, 0):
1716	case IP_VERSION(9, 4, 1):
1717	case IP_VERSION(9, 4, 2):
1718	case IP_VERSION(9, 4, 3):
1719	default:
1720	adev->gmc.gart_size = 512ULL << 20;
1721	break;
1722	case IP_VERSION(9, 1, 0): /* DCE SG support */
1723	case IP_VERSION(9, 2, 2): /* DCE SG support */
1724	case IP_VERSION(9, 3, 0):
1725	adev->gmc.gart_size = 1024ULL << 20;
1726	break;
1727	}
1728	} else {
1729	adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
1730	}
1731
1732	adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
1733
1734	gmc_v9_0_vram_gtt_location(adev, mc: &adev->gmc);
1735
1736	return 0;
1737	}
1738
1739	static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1740	{
1741	int r;
1742
1743	if (adev->gart.bo) {
1744	WARN(1, "VEGA10 PCIE GART already initialized\n");
1745	return 0;
1746	}
1747
1748	if (adev->gmc.xgmi.connected_to_cpu) {
1749	adev->gmc.vmid0_page_table_depth = 1;
1750	adev->gmc.vmid0_page_table_block_size = 12;
1751	} else {
1752	adev->gmc.vmid0_page_table_depth = 0;
1753	adev->gmc.vmid0_page_table_block_size = 0;
1754	}
1755
1756	/* Initialize common gart structure */
1757	r = amdgpu_gart_init(adev);
1758	if (r)
1759	return r;
1760	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
1761	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) |
1762	AMDGPU_PTE_EXECUTABLE;
1763
1764	if (!adev->gmc.real_vram_size) {
1765	dev_info(adev->dev, "Put GART in system memory for APU\n");
1766	r = amdgpu_gart_table_ram_alloc(adev);
1767	if (r)
1768	dev_err(adev->dev, "Failed to allocate GART in system memory\n");
1769	} else {
1770	r = amdgpu_gart_table_vram_alloc(adev);
1771	if (r)
1772	return r;
1773
1774	if (adev->gmc.xgmi.connected_to_cpu)
1775	r = amdgpu_gmc_pdb0_alloc(adev);
1776	}
1777
1778	return r;
1779	}
1780
1781	/**
1782	* gmc_v9_0_save_registers - saves regs
1783	*
1784	* @adev: amdgpu_device pointer
1785	*
1786	* This saves potential register values that should be
1787	* restored upon resume
1788	*/
1789	static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
1790	{
1791	if ((amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0) == IP_VERSION(1, 0, 0)) ||
1792	(amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0) == IP_VERSION(1, 0, 1)))
1793	adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
1794	}
1795
1796	static bool gmc_v9_0_validate_partition_info(struct amdgpu_device *adev)
1797	{
1798	enum amdgpu_memory_partition mode;
1799	u32 supp_modes;
1800	bool valid;
1801
1802	mode = gmc_v9_0_get_memory_partition(adev, supp_modes: &supp_modes);
1803
1804	/* Mode detected by hardware not present in supported modes */
1805	if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) &&
1806	!(BIT(mode - 1) & supp_modes))
1807	return false;
1808
1809	switch (mode) {
1810	case UNKNOWN_MEMORY_PARTITION_MODE:
1811	case AMDGPU_NPS1_PARTITION_MODE:
1812	valid = (adev->gmc.num_mem_partitions == 1);
1813	break;
1814	case AMDGPU_NPS2_PARTITION_MODE:
1815	valid = (adev->gmc.num_mem_partitions == 2);
1816	break;
1817	case AMDGPU_NPS4_PARTITION_MODE:
1818	valid = (adev->gmc.num_mem_partitions == 3 ||
1819	adev->gmc.num_mem_partitions == 4);
1820	break;
1821	default:
1822	valid = false;
1823	}
1824
1825	return valid;
1826	}
1827
1828	static bool gmc_v9_0_is_node_present(int *node_ids, int num_ids, int nid)
1829	{
1830	int i;
1831
1832	/* Check if node with id 'nid' is present in 'node_ids' array */
1833	for (i = 0; i < num_ids; ++i)
1834	if (node_ids[i] == nid)
1835	return true;
1836
1837	return false;
1838	}
1839
1840	static void
1841	gmc_v9_0_init_acpi_mem_ranges(struct amdgpu_device *adev,
1842	struct amdgpu_mem_partition_info *mem_ranges)
1843	{
1844	struct amdgpu_numa_info numa_info;
1845	int node_ids[MAX_MEM_RANGES];
1846	int num_ranges = 0, ret;
1847	int num_xcc, xcc_id;
1848	uint32_t xcc_mask;
1849
1850	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1851	xcc_mask = (1U << num_xcc) - 1;
1852
1853	for_each_inst(xcc_id, xcc_mask) {
1854	ret = amdgpu_acpi_get_mem_info(adev, xcc_id, numa_info: &numa_info);
1855	if (ret)
1856	continue;
1857
1858	if (numa_info.nid == NUMA_NO_NODE) {
1859	mem_ranges[0].size = numa_info.size;
1860	mem_ranges[0].numa.node = numa_info.nid;
1861	num_ranges = 1;
1862	break;
1863	}
1864
1865	if (gmc_v9_0_is_node_present(node_ids, num_ids: num_ranges,
1866	nid: numa_info.nid))
1867	continue;
1868
1869	node_ids[num_ranges] = numa_info.nid;
1870	mem_ranges[num_ranges].numa.node = numa_info.nid;
1871	mem_ranges[num_ranges].size = numa_info.size;
1872	++num_ranges;
1873	}
1874
1875	adev->gmc.num_mem_partitions = num_ranges;
1876	}
1877
1878	static void
1879	gmc_v9_0_init_sw_mem_ranges(struct amdgpu_device *adev,
1880	struct amdgpu_mem_partition_info *mem_ranges)
1881	{
1882	enum amdgpu_memory_partition mode;
1883	u32 start_addr = 0, size;
1884	int i;
1885
1886	mode = gmc_v9_0_query_memory_partition(adev);
1887
1888	switch (mode) {
1889	case UNKNOWN_MEMORY_PARTITION_MODE:
1890	case AMDGPU_NPS1_PARTITION_MODE:
1891	adev->gmc.num_mem_partitions = 1;
1892	break;
1893	case AMDGPU_NPS2_PARTITION_MODE:
1894	adev->gmc.num_mem_partitions = 2;
1895	break;
1896	case AMDGPU_NPS4_PARTITION_MODE:
1897	if (adev->flags & AMD_IS_APU)
1898	adev->gmc.num_mem_partitions = 3;
1899	else
1900	adev->gmc.num_mem_partitions = 4;
1901	break;
1902	default:
1903	adev->gmc.num_mem_partitions = 1;
1904	break;
1905	}
1906
1907	size = adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT;
1908	size /= adev->gmc.num_mem_partitions;
1909
1910	for (i = 0; i < adev->gmc.num_mem_partitions; ++i) {
1911	mem_ranges[i].range.fpfn = start_addr;
1912	mem_ranges[i].size = ((u64)size << AMDGPU_GPU_PAGE_SHIFT);
1913	mem_ranges[i].range.lpfn = start_addr + size - 1;
1914	start_addr += size;
1915	}
1916
1917	/* Adjust the last one */
1918	mem_ranges[adev->gmc.num_mem_partitions - 1].range.lpfn =
1919	(adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT) - 1;
1920	mem_ranges[adev->gmc.num_mem_partitions - 1].size =
1921	adev->gmc.real_vram_size -
1922	((u64)mem_ranges[adev->gmc.num_mem_partitions - 1].range.fpfn
1923	<< AMDGPU_GPU_PAGE_SHIFT);
1924	}
1925
1926	static int gmc_v9_0_init_mem_ranges(struct amdgpu_device *adev)
1927	{
1928	bool valid;
1929
1930	adev->gmc.mem_partitions = kcalloc(MAX_MEM_RANGES,
1931	size: sizeof(struct amdgpu_mem_partition_info),
1932	GFP_KERNEL);
1933	if (!adev->gmc.mem_partitions)
1934	return -ENOMEM;
1935
1936	/* TODO : Get the range from PSP/Discovery for dGPU */
1937	if (adev->gmc.is_app_apu)
1938	gmc_v9_0_init_acpi_mem_ranges(adev, mem_ranges: adev->gmc.mem_partitions);
1939	else
1940	gmc_v9_0_init_sw_mem_ranges(adev, mem_ranges: adev->gmc.mem_partitions);
1941
1942	if (amdgpu_sriov_vf(adev))
1943	valid = true;
1944	else
1945	valid = gmc_v9_0_validate_partition_info(adev);
1946	if (!valid) {
1947	/* TODO: handle invalid case */
1948	dev_WARN(adev->dev,
1949	"Mem ranges not matching with hardware config");
1950	}
1951
1952	return 0;
1953	}
1954
1955	static void gmc_v9_4_3_init_vram_info(struct amdgpu_device *adev)
1956	{
1957	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM;
1958	adev->gmc.vram_width = 128 * 64;
1959	}
1960
1961	static int gmc_v9_0_sw_init(void *handle)
1962	{
1963	int r, vram_width = 0, vram_type = 0, vram_vendor = 0, dma_addr_bits;
1964	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1965	unsigned long inst_mask = adev->aid_mask;
1966
1967	adev->gfxhub.funcs->init(adev);
1968
1969	adev->mmhub.funcs->init(adev);
1970
1971	spin_lock_init(&adev->gmc.invalidate_lock);
1972
1973	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3)) {
1974	gmc_v9_4_3_init_vram_info(adev);
1975	} else if (!adev->bios) {
1976	if (adev->flags & AMD_IS_APU) {
1977	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_DDR4;
1978	adev->gmc.vram_width = 64 * 64;
1979	} else {
1980	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM;
1981	adev->gmc.vram_width = 128 * 64;
1982	}
1983	} else {
1984	r = amdgpu_atomfirmware_get_vram_info(adev,
1985	vram_width: &vram_width, vram_type: &vram_type, vram_vendor: &vram_vendor);
1986	if (amdgpu_sriov_vf(adev))
1987	/* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN,
1988	* and DF related registers is not readable, seems hardcord is the
1989	* only way to set the correct vram_width
1990	*/
1991	adev->gmc.vram_width = 2048;
1992	else if (amdgpu_emu_mode != 1)
1993	adev->gmc.vram_width = vram_width;
1994
1995	if (!adev->gmc.vram_width) {
1996	int chansize, numchan;
1997
1998	/* hbm memory channel size */
1999	if (adev->flags & AMD_IS_APU)
2000	chansize = 64;
2001	else
2002	chansize = 128;
2003	if (adev->df.funcs &&
2004	adev->df.funcs->get_hbm_channel_number) {
2005	numchan = adev->df.funcs->get_hbm_channel_number(adev);
2006	adev->gmc.vram_width = numchan * chansize;
2007	}
2008	}
2009
2010	adev->gmc.vram_type = vram_type;
2011	adev->gmc.vram_vendor = vram_vendor;
2012	}
2013	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
2014	case IP_VERSION(9, 1, 0):
2015	case IP_VERSION(9, 2, 2):
2016	set_bit(AMDGPU_GFXHUB(0), addr: adev->vmhubs_mask);
2017	set_bit(AMDGPU_MMHUB0(0), addr: adev->vmhubs_mask);
2018
2019	if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
2020	amdgpu_vm_adjust_size(adev, min_vm_size: 256 * 1024, fragment_size_default: 9, max_level: 3, max_bits: 48);
2021	} else {
2022	/* vm_size is 128TB + 512GB for legacy 3-level page support */
2023	amdgpu_vm_adjust_size(adev, min_vm_size: 128 * 1024 + 512, fragment_size_default: 9, max_level: 2, max_bits: 48);
2024	adev->gmc.translate_further =
2025	adev->vm_manager.num_level > 1;
2026	}
2027	break;
2028	case IP_VERSION(9, 0, 1):
2029	case IP_VERSION(9, 2, 1):
2030	case IP_VERSION(9, 4, 0):
2031	case IP_VERSION(9, 3, 0):
2032	case IP_VERSION(9, 4, 2):
2033	set_bit(AMDGPU_GFXHUB(0), addr: adev->vmhubs_mask);
2034	set_bit(AMDGPU_MMHUB0(0), addr: adev->vmhubs_mask);
2035
2036	/*
2037	* To fulfill 4-level page support,
2038	* vm size is 256TB (48bit), maximum size of Vega10,
2039	* block size 512 (9bit)
2040	*/
2041
2042	amdgpu_vm_adjust_size(adev, min_vm_size: 256 * 1024, fragment_size_default: 9, max_level: 3, max_bits: 48);
2043	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2))
2044	adev->gmc.translate_further = adev->vm_manager.num_level > 1;
2045	break;
2046	case IP_VERSION(9, 4, 1):
2047	set_bit(AMDGPU_GFXHUB(0), addr: adev->vmhubs_mask);
2048	set_bit(AMDGPU_MMHUB0(0), addr: adev->vmhubs_mask);
2049	set_bit(AMDGPU_MMHUB1(0), addr: adev->vmhubs_mask);
2050
2051	/* Keep the vm size same with Vega20 */
2052	amdgpu_vm_adjust_size(adev, min_vm_size: 256 * 1024, fragment_size_default: 9, max_level: 3, max_bits: 48);
2053	adev->gmc.translate_further = adev->vm_manager.num_level > 1;
2054	break;
2055	case IP_VERSION(9, 4, 3):
2056	bitmap_set(map: adev->vmhubs_mask, AMDGPU_GFXHUB(0),
2057	NUM_XCC(adev->gfx.xcc_mask));
2058
2059	inst_mask <<= AMDGPU_MMHUB0(0);
2060	bitmap_or(dst: adev->vmhubs_mask, src1: adev->vmhubs_mask, src2: &inst_mask, nbits: 32);
2061
2062	amdgpu_vm_adjust_size(adev, min_vm_size: 256 * 1024, fragment_size_default: 9, max_level: 3, max_bits: 48);
2063	adev->gmc.translate_further = adev->vm_manager.num_level > 1;
2064	break;
2065	default:
2066	break;
2067	}
2068
2069	/* This interrupt is VMC page fault.*/
2070	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
2071	source: &adev->gmc.vm_fault);
2072	if (r)
2073	return r;
2074
2075	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1)) {
2076	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
2077	source: &adev->gmc.vm_fault);
2078	if (r)
2079	return r;
2080	}
2081
2082	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
2083	source: &adev->gmc.vm_fault);
2084
2085	if (r)
2086	return r;
2087
2088	if (!amdgpu_sriov_vf(adev) &&
2089	!adev->gmc.xgmi.connected_to_cpu &&
2090	!adev->gmc.is_app_apu) {
2091	/* interrupt sent to DF. */
2092	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_DF, src_id: 0,
2093	source: &adev->gmc.ecc_irq);
2094	if (r)
2095	return r;
2096	}
2097
2098	/* Set the internal MC address mask
2099	* This is the max address of the GPU's
2100	* internal address space.
2101	*/
2102	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
2103
2104	dma_addr_bits = amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >=
2105	IP_VERSION(9, 4, 2) ?
2106	48 :
2107	44;
2108	r = dma_set_mask_and_coherent(dev: adev->dev, DMA_BIT_MASK(dma_addr_bits));
2109	if (r) {
2110	dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");
2111	return r;
2112	}
2113	adev->need_swiotlb = drm_need_swiotlb(dma_bits: dma_addr_bits);
2114
2115	r = gmc_v9_0_mc_init(adev);
2116	if (r)
2117	return r;
2118
2119	amdgpu_gmc_get_vbios_allocations(adev);
2120
2121	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3)) {
2122	r = gmc_v9_0_init_mem_ranges(adev);
2123	if (r)
2124	return r;
2125	}
2126
2127	/* Memory manager */
2128	r = amdgpu_bo_init(adev);
2129	if (r)
2130	return r;
2131
2132	r = gmc_v9_0_gart_init(adev);
2133	if (r)
2134	return r;
2135
2136	/*
2137	* number of VMs
2138	* VMID 0 is reserved for System
2139	* amdgpu graphics/compute will use VMIDs 1..n-1
2140	* amdkfd will use VMIDs n..15
2141	*
2142	* The first KFD VMID is 8 for GPUs with graphics, 3 for
2143	* compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
2144	* for video processing.
2145	*/
2146	adev->vm_manager.first_kfd_vmid =
2147	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1) ||
2148	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2) ||
2149	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3)) ?
2150	3 :
2151	8;
2152
2153	amdgpu_vm_manager_init(adev);
2154
2155	gmc_v9_0_save_registers(adev);
2156
2157	r = amdgpu_gmc_ras_sw_init(adev);
2158	if (r)
2159	return r;
2160
2161	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3))
2162	amdgpu_gmc_sysfs_init(adev);
2163
2164	return 0;
2165	}
2166
2167	static int gmc_v9_0_sw_fini(void *handle)
2168	{
2169	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2170
2171	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3))
2172	amdgpu_gmc_sysfs_fini(adev);
2173
2174	amdgpu_gmc_ras_fini(adev);
2175	amdgpu_gem_force_release(adev);
2176	amdgpu_vm_manager_fini(adev);
2177	if (!adev->gmc.real_vram_size) {
2178	dev_info(adev->dev, "Put GART in system memory for APU free\n");
2179	amdgpu_gart_table_ram_free(adev);
2180	} else {
2181	amdgpu_gart_table_vram_free(adev);
2182	}
2183	amdgpu_bo_free_kernel(bo: &adev->gmc.pdb0_bo, NULL, cpu_addr: &adev->gmc.ptr_pdb0);
2184	amdgpu_bo_fini(adev);
2185
2186	adev->gmc.num_mem_partitions = 0;
2187	kfree(objp: adev->gmc.mem_partitions);
2188
2189	return 0;
2190	}
2191
2192	static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
2193	{
2194	switch (amdgpu_ip_version(adev, ip: MMHUB_HWIP, inst: 0)) {
2195	case IP_VERSION(9, 0, 0):
2196	if (amdgpu_sriov_vf(adev))
2197	break;
2198	fallthrough;
2199	case IP_VERSION(9, 4, 0):
2200	soc15_program_register_sequence(adev,
2201	registers: golden_settings_mmhub_1_0_0,
2202	ARRAY_SIZE(golden_settings_mmhub_1_0_0));
2203	soc15_program_register_sequence(adev,
2204	registers: golden_settings_athub_1_0_0,
2205	ARRAY_SIZE(golden_settings_athub_1_0_0));
2206	break;
2207	case IP_VERSION(9, 1, 0):
2208	case IP_VERSION(9, 2, 0):
2209	/* TODO for renoir */
2210	soc15_program_register_sequence(adev,
2211	registers: golden_settings_athub_1_0_0,
2212	ARRAY_SIZE(golden_settings_athub_1_0_0));
2213	break;
2214	default:
2215	break;
2216	}
2217	}
2218
2219	/**
2220	* gmc_v9_0_restore_registers - restores regs
2221	*
2222	* @adev: amdgpu_device pointer
2223	*
2224	* This restores register values, saved at suspend.
2225	*/
2226	void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
2227	{
2228	if ((amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0) == IP_VERSION(1, 0, 0)) ||
2229	(amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0) == IP_VERSION(1, 0, 1))) {
2230	WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
2231	WARN_ON(adev->gmc.sdpif_register !=
2232	RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0));
2233	}
2234	}
2235
2236	/**
2237	* gmc_v9_0_gart_enable - gart enable
2238	*
2239	* @adev: amdgpu_device pointer
2240	*/
2241	static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
2242	{
2243	int r;
2244
2245	if (adev->gmc.xgmi.connected_to_cpu)
2246	amdgpu_gmc_init_pdb0(adev);
2247
2248	if (adev->gart.bo == NULL) {
2249	dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
2250	return -EINVAL;
2251	}
2252
2253	amdgpu_gtt_mgr_recover(mgr: &adev->mman.gtt_mgr);
2254
2255	if (!adev->in_s0ix) {
2256	r = adev->gfxhub.funcs->gart_enable(adev);
2257	if (r)
2258	return r;
2259	}
2260
2261	r = adev->mmhub.funcs->gart_enable(adev);
2262	if (r)
2263	return r;
2264
2265	DRM_INFO("PCIE GART of %uM enabled.\n",
2266	(unsigned int)(adev->gmc.gart_size >> 20));
2267	if (adev->gmc.pdb0_bo)
2268	DRM_INFO("PDB0 located at 0x%016llX\n",
2269	(unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo));
2270	DRM_INFO("PTB located at 0x%016llX\n",
2271	(unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
2272
2273	return 0;
2274	}
2275
2276	static int gmc_v9_0_hw_init(void *handle)
2277	{
2278	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2279	bool value;
2280	int i, r;
2281
2282	adev->gmc.flush_pasid_uses_kiq = true;
2283
2284	/* Vega20+XGMI caches PTEs in TC and TLB. Add a heavy-weight TLB flush
2285	* (type 2), which flushes both. Due to a race condition with
2286	* concurrent memory accesses using the same TLB cache line, we still
2287	* need a second TLB flush after this.
2288	*/
2289	adev->gmc.flush_tlb_needs_extra_type_2 =
2290	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 0) &&
2291	adev->gmc.xgmi.num_physical_nodes;
2292	/*
2293	* TODO: This workaround is badly documented and had a buggy
2294	* implementation. We should probably verify what we do here.
2295	*/
2296	adev->gmc.flush_tlb_needs_extra_type_0 =
2297	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3) &&
2298	adev->rev_id == 0;
2299
2300	/* The sequence of these two function calls matters.*/
2301	gmc_v9_0_init_golden_registers(adev);
2302
2303	if (adev->mode_info.num_crtc) {
2304	/* Lockout access through VGA aperture*/
2305	WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
2306	/* disable VGA render */
2307	WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
2308	}
2309
2310	if (adev->mmhub.funcs->update_power_gating)
2311	adev->mmhub.funcs->update_power_gating(adev, true);
2312
2313	adev->hdp.funcs->init_registers(adev);
2314
2315	/* After HDP is initialized, flush HDP.*/
2316	adev->hdp.funcs->flush_hdp(adev, NULL);
2317
2318	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
2319	value = false;
2320	else
2321	value = true;
2322
2323	if (!amdgpu_sriov_vf(adev)) {
2324	if (!adev->in_s0ix)
2325	adev->gfxhub.funcs->set_fault_enable_default(adev, value);
2326	adev->mmhub.funcs->set_fault_enable_default(adev, value);
2327	}
2328	for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
2329	if (adev->in_s0ix && (i == AMDGPU_GFXHUB(0)))
2330	continue;
2331	gmc_v9_0_flush_gpu_tlb(adev, vmid: 0, vmhub: i, flush_type: 0);
2332	}
2333
2334	if (adev->umc.funcs && adev->umc.funcs->init_registers)
2335	adev->umc.funcs->init_registers(adev);
2336
2337	r = gmc_v9_0_gart_enable(adev);
2338	if (r)
2339	return r;
2340
2341	if (amdgpu_emu_mode == 1)
2342	return amdgpu_gmc_vram_checking(adev);
2343
2344	return 0;
2345	}
2346
2347	/**
2348	* gmc_v9_0_gart_disable - gart disable
2349	*
2350	* @adev: amdgpu_device pointer
2351	*
2352	* This disables all VM page table.
2353	*/
2354	static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
2355	{
2356	if (!adev->in_s0ix)
2357	adev->gfxhub.funcs->gart_disable(adev);
2358	adev->mmhub.funcs->gart_disable(adev);
2359	}
2360
2361	static int gmc_v9_0_hw_fini(void *handle)
2362	{
2363	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2364
2365	gmc_v9_0_gart_disable(adev);
2366
2367	if (amdgpu_sriov_vf(adev)) {
2368	/* full access mode, so don't touch any GMC register */
2369	DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
2370	return 0;
2371	}
2372
2373	/*
2374	* Pair the operations did in gmc_v9_0_hw_init and thus maintain
2375	* a correct cached state for GMC. Otherwise, the "gate" again
2376	* operation on S3 resuming will fail due to wrong cached state.
2377	*/
2378	if (adev->mmhub.funcs->update_power_gating)
2379	adev->mmhub.funcs->update_power_gating(adev, false);
2380
2381	amdgpu_irq_put(adev, src: &adev->gmc.vm_fault, type: 0);
2382
2383	if (adev->gmc.ecc_irq.funcs &&
2384	amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__UMC))
2385	amdgpu_irq_put(adev, src: &adev->gmc.ecc_irq, type: 0);
2386
2387	return 0;
2388	}
2389
2390	static int gmc_v9_0_suspend(void *handle)
2391	{
2392	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2393
2394	return gmc_v9_0_hw_fini(handle: adev);
2395	}
2396
2397	static int gmc_v9_0_resume(void *handle)
2398	{
2399	int r;
2400	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2401
2402	r = gmc_v9_0_hw_init(handle: adev);
2403	if (r)
2404	return r;
2405
2406	amdgpu_vmid_reset_all(adev);
2407
2408	return 0;
2409	}
2410
2411	static bool gmc_v9_0_is_idle(void *handle)
2412	{
2413	/* MC is always ready in GMC v9.*/
2414	return true;
2415	}
2416
2417	static int gmc_v9_0_wait_for_idle(void *handle)
2418	{
2419	/* There is no need to wait for MC idle in GMC v9.*/
2420	return 0;
2421	}
2422
2423	static int gmc_v9_0_soft_reset(void *handle)
2424	{
2425	/* XXX for emulation.*/
2426	return 0;
2427	}
2428
2429	static int gmc_v9_0_set_clockgating_state(void *handle,
2430	enum amd_clockgating_state state)
2431	{
2432	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2433
2434	adev->mmhub.funcs->set_clockgating(adev, state);
2435
2436	athub_v1_0_set_clockgating(adev, state);
2437
2438	return 0;
2439	}
2440
2441	static void gmc_v9_0_get_clockgating_state(void *handle, u64 *flags)
2442	{
2443	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2444
2445	adev->mmhub.funcs->get_clockgating(adev, flags);
2446
2447	athub_v1_0_get_clockgating(adev, flags);
2448	}
2449
2450	static int gmc_v9_0_set_powergating_state(void *handle,
2451	enum amd_powergating_state state)
2452	{
2453	return 0;
2454	}
2455
2456	const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
2457	.name = "gmc_v9_0",
2458	.early_init = gmc_v9_0_early_init,
2459	.late_init = gmc_v9_0_late_init,
2460	.sw_init = gmc_v9_0_sw_init,
2461	.sw_fini = gmc_v9_0_sw_fini,
2462	.hw_init = gmc_v9_0_hw_init,
2463	.hw_fini = gmc_v9_0_hw_fini,
2464	.suspend = gmc_v9_0_suspend,
2465	.resume = gmc_v9_0_resume,
2466	.is_idle = gmc_v9_0_is_idle,
2467	.wait_for_idle = gmc_v9_0_wait_for_idle,
2468	.soft_reset = gmc_v9_0_soft_reset,
2469	.set_clockgating_state = gmc_v9_0_set_clockgating_state,
2470	.set_powergating_state = gmc_v9_0_set_powergating_state,
2471	.get_clockgating_state = gmc_v9_0_get_clockgating_state,
2472	};
2473
2474	const struct amdgpu_ip_block_version gmc_v9_0_ip_block = {
2475	.type = AMD_IP_BLOCK_TYPE_GMC,
2476	.major = 9,
2477	.minor = 0,
2478	.rev = 0,
2479	.funcs = &gmc_v9_0_ip_funcs,
2480	};
2481