1	/*
2	* Copyright 2016 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	*/
23
24	#include <linux/delay.h>
25	#include <linux/kernel.h>
26	#include <linux/firmware.h>
27	#include <linux/module.h>
28	#include <linux/pci.h>
29
30	#include "amdgpu.h"
31	#include "amdgpu_gfx.h"
32	#include "soc15.h"
33	#include "soc15d.h"
34	#include "amdgpu_atomfirmware.h"
35	#include "amdgpu_pm.h"
36
37	#include "gc/gc_9_0_offset.h"
38	#include "gc/gc_9_0_sh_mask.h"
39
40	#include "vega10_enum.h"
41
42	#include "soc15_common.h"
43	#include "clearstate_gfx9.h"
44	#include "v9_structs.h"
45
46	#include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
47
48	#include "amdgpu_ras.h"
49
50	#include "amdgpu_ring_mux.h"
51	#include "gfx_v9_4.h"
52	#include "gfx_v9_0.h"
53	#include "gfx_v9_4_2.h"
54
55	#include "asic_reg/pwr/pwr_10_0_offset.h"
56	#include "asic_reg/pwr/pwr_10_0_sh_mask.h"
57	#include "asic_reg/gc/gc_9_0_default.h"
58
59	#define GFX9_NUM_GFX_RINGS 1
60	#define GFX9_NUM_SW_GFX_RINGS 2
61	#define GFX9_MEC_HPD_SIZE 4096
62	#define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
63	#define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
64
65	#define mmGCEA_PROBE_MAP 0x070c
66	#define mmGCEA_PROBE_MAP_BASE_IDX 0
67
68	MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
69	MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
70	MODULE_FIRMWARE("amdgpu/vega10_me.bin");
71	MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
72	MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
73	MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
74
75	MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
76	MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
77	MODULE_FIRMWARE("amdgpu/vega12_me.bin");
78	MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
79	MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
80	MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
81
82	MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
83	MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
84	MODULE_FIRMWARE("amdgpu/vega20_me.bin");
85	MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
86	MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
87	MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
88
89	MODULE_FIRMWARE("amdgpu/raven_ce.bin");
90	MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
91	MODULE_FIRMWARE("amdgpu/raven_me.bin");
92	MODULE_FIRMWARE("amdgpu/raven_mec.bin");
93	MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
94	MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
95
96	MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
97	MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
98	MODULE_FIRMWARE("amdgpu/picasso_me.bin");
99	MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
100	MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
101	MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
102	MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
103
104	MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
105	MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
106	MODULE_FIRMWARE("amdgpu/raven2_me.bin");
107	MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
108	MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
109	MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
110	MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
111
112	MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
113	MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
114
115	MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
116	MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
117	MODULE_FIRMWARE("amdgpu/renoir_me.bin");
118	MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
119	MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
120
121	MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
122	MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
123	MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
124	MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
125	MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
126	MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
127
128	MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
129	MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
130	MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
131	MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin");
132	MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin");
133
134	#define mmTCP_CHAN_STEER_0_ARCT 0x0b03
135	#define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0
136	#define mmTCP_CHAN_STEER_1_ARCT 0x0b04
137	#define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0
138	#define mmTCP_CHAN_STEER_2_ARCT 0x0b09
139	#define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0
140	#define mmTCP_CHAN_STEER_3_ARCT 0x0b0a
141	#define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0
142	#define mmTCP_CHAN_STEER_4_ARCT 0x0b0b
143	#define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0
144	#define mmTCP_CHAN_STEER_5_ARCT 0x0b0c
145	#define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0
146
147	#define mmGOLDEN_TSC_COUNT_UPPER_Renoir 0x0025
148	#define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX 1
149	#define mmGOLDEN_TSC_COUNT_LOWER_Renoir 0x0026
150	#define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX 1
151
152	enum ta_ras_gfx_subblock {
153	/*CPC*/
154	TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
155	TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
156	TA_RAS_BLOCK__GFX_CPC_UCODE,
157	TA_RAS_BLOCK__GFX_DC_STATE_ME1,
158	TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
159	TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
160	TA_RAS_BLOCK__GFX_DC_STATE_ME2,
161	TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
162	TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
163	TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
164	/* CPF*/
165	TA_RAS_BLOCK__GFX_CPF_INDEX_START,
166	TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
167	TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
168	TA_RAS_BLOCK__GFX_CPF_TAG,
169	TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
170	/* CPG*/
171	TA_RAS_BLOCK__GFX_CPG_INDEX_START,
172	TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
173	TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
174	TA_RAS_BLOCK__GFX_CPG_TAG,
175	TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
176	/* GDS*/
177	TA_RAS_BLOCK__GFX_GDS_INDEX_START,
178	TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
179	TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
180	TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
181	TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
182	TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
183	TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
184	/* SPI*/
185	TA_RAS_BLOCK__GFX_SPI_SR_MEM,
186	/* SQ*/
187	TA_RAS_BLOCK__GFX_SQ_INDEX_START,
188	TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
189	TA_RAS_BLOCK__GFX_SQ_LDS_D,
190	TA_RAS_BLOCK__GFX_SQ_LDS_I,
191	TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
192	TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
193	/* SQC (3 ranges)*/
194	TA_RAS_BLOCK__GFX_SQC_INDEX_START,
195	/* SQC range 0*/
196	TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
197	TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
198	TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
199	TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
200	TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
201	TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
202	TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
203	TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
204	TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
205	TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
206	TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
207	/* SQC range 1*/
208	TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
209	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
210	TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
211	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
212	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
213	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
214	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
215	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
216	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
217	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
218	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
219	TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
220	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
221	/* SQC range 2*/
222	TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
223	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
224	TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
225	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
226	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
227	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
228	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
229	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
230	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
231	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
232	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
233	TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
234	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
235	TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
236	/* TA*/
237	TA_RAS_BLOCK__GFX_TA_INDEX_START,
238	TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
239	TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
240	TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
241	TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
242	TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
243	TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
244	/* TCA*/
245	TA_RAS_BLOCK__GFX_TCA_INDEX_START,
246	TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
247	TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
248	TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
249	/* TCC (5 sub-ranges)*/
250	TA_RAS_BLOCK__GFX_TCC_INDEX_START,
251	/* TCC range 0*/
252	TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
253	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
254	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
255	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
256	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
257	TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
258	TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
259	TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
260	TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
261	TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
262	/* TCC range 1*/
263	TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
264	TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
265	TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
266	TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
267	TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
268	/* TCC range 2*/
269	TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
270	TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
271	TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
272	TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
273	TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
274	TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
275	TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
276	TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
277	TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
278	TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
279	TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
280	/* TCC range 3*/
281	TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
282	TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
283	TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
284	TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
285	TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
286	/* TCC range 4*/
287	TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
288	TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
289	TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
290	TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
291	TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
292	TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
293	TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
294	/* TCI*/
295	TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
296	/* TCP*/
297	TA_RAS_BLOCK__GFX_TCP_INDEX_START,
298	TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
299	TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
300	TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
301	TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
302	TA_RAS_BLOCK__GFX_TCP_DB_RAM,
303	TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
304	TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
305	TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
306	/* TD*/
307	TA_RAS_BLOCK__GFX_TD_INDEX_START,
308	TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
309	TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
310	TA_RAS_BLOCK__GFX_TD_CS_FIFO,
311	TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
312	/* EA (3 sub-ranges)*/
313	TA_RAS_BLOCK__GFX_EA_INDEX_START,
314	/* EA range 0*/
315	TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
316	TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
317	TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
318	TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
319	TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
320	TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
321	TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
322	TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
323	TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
324	TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
325	/* EA range 1*/
326	TA_RAS_BLOCK__GFX_EA_INDEX1_START,
327	TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
328	TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
329	TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
330	TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
331	TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
332	TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
333	TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
334	TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
335	/* EA range 2*/
336	TA_RAS_BLOCK__GFX_EA_INDEX2_START,
337	TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
338	TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
339	TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
340	TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
341	TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
342	TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
343	/* UTC VM L2 bank*/
344	TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
345	/* UTC VM walker*/
346	TA_RAS_BLOCK__UTC_VML2_WALKER,
347	/* UTC ATC L2 2MB cache*/
348	TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
349	/* UTC ATC L2 4KB cache*/
350	TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
351	TA_RAS_BLOCK__GFX_MAX
352	};
353
354	struct ras_gfx_subblock {
355	unsigned char *name;
356	int ta_subblock;
357	int hw_supported_error_type;
358	int sw_supported_error_type;
359	};
360
361	#define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \
362	[AMDGPU_RAS_BLOCK__##subblock] = { \
363	#subblock, \
364	TA_RAS_BLOCK__##subblock, \
365	((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \
366	(((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \
367	}
368
369	static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
370	AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
371	AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
372	AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
373	AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
374	AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
375	AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
376	AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
377	AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
378	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
379	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
380	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
381	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
382	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
383	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
384	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
385	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
386	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
387	0),
388	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
389	0),
390	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
391	AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
392	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
393	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
394	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
395	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
396	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
397	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
398	0, 0),
399	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
400	0),
401	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
402	0, 0),
403	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
404	0),
405	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
406	0, 0),
407	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
408	0),
409	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
410	1),
411	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
412	0, 0, 0),
413	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
414	0),
415	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
416	0),
417	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
418	0),
419	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
420	0),
421	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
422	0),
423	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
424	0, 0),
425	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
426	0),
427	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
428	0),
429	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
430	0, 0, 0),
431	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
432	0),
433	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
434	0),
435	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
436	0),
437	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
438	0),
439	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
440	0),
441	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
442	0, 0),
443	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
444	0),
445	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
446	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
447	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
448	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
449	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
450	AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
451	AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
452	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
453	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
454	1),
455	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
456	1),
457	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
458	1),
459	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
460	0),
461	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
462	0),
463	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
464	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
465	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
466	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
467	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
468	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
469	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
470	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
471	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
472	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
473	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
474	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
475	0),
476	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
477	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
478	0),
479	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
480	0, 0),
481	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
482	0),
483	AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
484	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
485	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
486	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
487	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
488	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
489	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
490	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
491	AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
492	AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
493	AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
494	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
495	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
496	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
497	AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
498	AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
499	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
500	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
501	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
502	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
503	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
504	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
505	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
506	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
507	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
508	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
509	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
510	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
511	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
512	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
513	AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
514	AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
515	AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
516	AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
517	};
518
519	static const struct soc15_reg_golden golden_settings_gc_9_0[] =
520	{
521	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
522	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
523	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
524	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
525	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
526	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
527	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
528	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
529	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
530	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
531	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
532	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
533	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
534	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
535	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
536	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
537	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
538	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
539	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
540	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
541	};
542
543	static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
544	{
545	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
546	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
547	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
548	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
549	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
550	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
551	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
552	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
553	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
554	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
555	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
556	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
557	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
558	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
559	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
560	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
561	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
562	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
563	};
564
565	static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
566	{
567	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
568	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
569	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
570	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
571	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
572	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
573	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
574	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
575	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
576	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
577	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
578	};
579
580	static const struct soc15_reg_golden golden_settings_gc_9_1[] =
581	{
582	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
583	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
584	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
585	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
586	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
587	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
588	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
589	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
590	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
591	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
592	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
593	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
594	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
595	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
596	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
597	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
598	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
599	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
600	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
601	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
602	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
603	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
604	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
605	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
606	};
607
608	static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
609	{
610	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
611	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
612	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
613	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
614	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
615	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
616	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
617	};
618
619	static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
620	{
621	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
622	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
623	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
624	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
625	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
626	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
627	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
628	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
629	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
630	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
631	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
632	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
633	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
634	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
635	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
636	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
637	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
638	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
639	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
640	};
641
642	static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
643	{
644	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
645	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
646	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
647	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
648	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
649	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
650	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
651	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
652	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
653	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
654	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
655	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
656	};
657
658	static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
659	{
660	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
661	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
662	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
663	};
664
665	static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
666	{
667	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
668	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
669	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
670	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
671	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
672	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
673	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
674	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
675	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
676	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
677	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
678	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
679	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
680	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
681	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
682	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
683	};
684
685	static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
686	{
687	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
688	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
689	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
690	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
691	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
692	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
693	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
694	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
695	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
696	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
697	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
698	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
699	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
700	};
701
702	static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
703	{
704	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
705	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
706	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
707	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
708	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
709	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
710	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
711	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
712	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
713	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
714	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
715	};
716
717	static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
718	{SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
719	{SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
720	};
721
722	static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
723	{
724	mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
725	mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
726	mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
727	mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
728	mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
729	mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
730	mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
731	mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
732	};
733
734	static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
735	{
736	mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
737	mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
738	mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
739	mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
740	mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
741	mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
742	mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
743	mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
744	};
745
746	#define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
747	#define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
748	#define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
749	#define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
750
751	static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
752	static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
753	static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
754	static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
755	static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
756	struct amdgpu_cu_info *cu_info);
757	static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
758	static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds);
759	static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
760	static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
761	void *ras_error_status);
762	static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
763	void *inject_if, uint32_t instance_mask);
764	static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
765	static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
766	unsigned int vmid);
767
768	static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
769	uint64_t queue_mask)
770	{
771	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
772	amdgpu_ring_write(ring: kiq_ring,
773	PACKET3_SET_RESOURCES_VMID_MASK(0) |
774	/* vmid_mask:0* queue_type:0 (KIQ) */
775	PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
776	amdgpu_ring_write(ring: kiq_ring,
777	lower_32_bits(queue_mask)); /* queue mask lo */
778	amdgpu_ring_write(ring: kiq_ring,
779	upper_32_bits(queue_mask)); /* queue mask hi */
780	amdgpu_ring_write(ring: kiq_ring, v: 0); /* gws mask lo */
781	amdgpu_ring_write(ring: kiq_ring, v: 0); /* gws mask hi */
782	amdgpu_ring_write(ring: kiq_ring, v: 0); /* oac mask */
783	amdgpu_ring_write(ring: kiq_ring, v: 0); /* gds heap base:0, gds heap size:0 */
784	}
785
786	static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
787	struct amdgpu_ring *ring)
788	{
789	uint64_t mqd_addr = amdgpu_bo_gpu_offset(bo: ring->mqd_obj);
790	uint64_t wptr_addr = ring->wptr_gpu_addr;
791	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
792
793	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
794	/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
795	amdgpu_ring_write(ring: kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
796	PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
797	PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
798	PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
799	PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
800	PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
801	/*queue_type: normal compute queue */
802	PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
803	/* alloc format: all_on_one_pipe */
804	PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
805	PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
806	/* num_queues: must be 1 */
807	PACKET3_MAP_QUEUES_NUM_QUEUES(1));
808	amdgpu_ring_write(ring: kiq_ring,
809	PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
810	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(mqd_addr));
811	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(mqd_addr));
812	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(wptr_addr));
813	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(wptr_addr));
814	}
815
816	static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
817	struct amdgpu_ring *ring,
818	enum amdgpu_unmap_queues_action action,
819	u64 gpu_addr, u64 seq)
820	{
821	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
822
823	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
824	amdgpu_ring_write(ring: kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
825	PACKET3_UNMAP_QUEUES_ACTION(action) |
826	PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
827	PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
828	PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
829	amdgpu_ring_write(ring: kiq_ring,
830	PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
831
832	if (action == PREEMPT_QUEUES_NO_UNMAP) {
833	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask));
834	amdgpu_ring_write(ring: kiq_ring, v: 0);
835	amdgpu_ring_write(ring: kiq_ring, v: 0);
836
837	} else {
838	amdgpu_ring_write(ring: kiq_ring, v: 0);
839	amdgpu_ring_write(ring: kiq_ring, v: 0);
840	amdgpu_ring_write(ring: kiq_ring, v: 0);
841	}
842	}
843
844	static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
845	struct amdgpu_ring *ring,
846	u64 addr,
847	u64 seq)
848	{
849	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
850
851	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
852	amdgpu_ring_write(ring: kiq_ring,
853	PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
854	PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
855	PACKET3_QUERY_STATUS_COMMAND(2));
856	/* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
857	amdgpu_ring_write(ring: kiq_ring,
858	PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
859	PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
860	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(addr));
861	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(addr));
862	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(seq));
863	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(seq));
864	}
865
866	static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
867	uint16_t pasid, uint32_t flush_type,
868	bool all_hub)
869	{
870	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
871	amdgpu_ring_write(ring: kiq_ring,
872	PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
873	PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
874	PACKET3_INVALIDATE_TLBS_PASID(pasid) |
875	PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
876	}
877
878	static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
879	.kiq_set_resources = gfx_v9_0_kiq_set_resources,
880	.kiq_map_queues = gfx_v9_0_kiq_map_queues,
881	.kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
882	.kiq_query_status = gfx_v9_0_kiq_query_status,
883	.kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
884	.set_resources_size = 8,
885	.map_queues_size = 7,
886	.unmap_queues_size = 6,
887	.query_status_size = 7,
888	.invalidate_tlbs_size = 2,
889	};
890
891	static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
892	{
893	adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs;
894	}
895
896	static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
897	{
898	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
899	case IP_VERSION(9, 0, 1):
900	soc15_program_register_sequence(adev,
901	registers: golden_settings_gc_9_0,
902	ARRAY_SIZE(golden_settings_gc_9_0));
903	soc15_program_register_sequence(adev,
904	registers: golden_settings_gc_9_0_vg10,
905	ARRAY_SIZE(golden_settings_gc_9_0_vg10));
906	break;
907	case IP_VERSION(9, 2, 1):
908	soc15_program_register_sequence(adev,
909	registers: golden_settings_gc_9_2_1,
910	ARRAY_SIZE(golden_settings_gc_9_2_1));
911	soc15_program_register_sequence(adev,
912	registers: golden_settings_gc_9_2_1_vg12,
913	ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
914	break;
915	case IP_VERSION(9, 4, 0):
916	soc15_program_register_sequence(adev,
917	registers: golden_settings_gc_9_0,
918	ARRAY_SIZE(golden_settings_gc_9_0));
919	soc15_program_register_sequence(adev,
920	registers: golden_settings_gc_9_0_vg20,
921	ARRAY_SIZE(golden_settings_gc_9_0_vg20));
922	break;
923	case IP_VERSION(9, 4, 1):
924	soc15_program_register_sequence(adev,
925	registers: golden_settings_gc_9_4_1_arct,
926	ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
927	break;
928	case IP_VERSION(9, 2, 2):
929	case IP_VERSION(9, 1, 0):
930	soc15_program_register_sequence(adev, registers: golden_settings_gc_9_1,
931	ARRAY_SIZE(golden_settings_gc_9_1));
932	if (adev->apu_flags & AMD_APU_IS_RAVEN2)
933	soc15_program_register_sequence(adev,
934	registers: golden_settings_gc_9_1_rv2,
935	ARRAY_SIZE(golden_settings_gc_9_1_rv2));
936	else
937	soc15_program_register_sequence(adev,
938	registers: golden_settings_gc_9_1_rv1,
939	ARRAY_SIZE(golden_settings_gc_9_1_rv1));
940	break;
941	case IP_VERSION(9, 3, 0):
942	soc15_program_register_sequence(adev,
943	registers: golden_settings_gc_9_1_rn,
944	ARRAY_SIZE(golden_settings_gc_9_1_rn));
945	return; /* for renoir, don't need common goldensetting */
946	case IP_VERSION(9, 4, 2):
947	gfx_v9_4_2_init_golden_registers(adev,
948	die_id: adev->smuio.funcs->get_die_id(adev));
949	break;
950	default:
951	break;
952	}
953
954	if ((amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 1)) &&
955	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 2)))
956	soc15_program_register_sequence(adev, registers: golden_settings_gc_9_x_common,
957	array_size: (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
958	}
959
960	static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
961	bool wc, uint32_t reg, uint32_t val)
962	{
963	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
964	amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
965	WRITE_DATA_DST_SEL(0) |
966	(wc ? WR_CONFIRM : 0));
967	amdgpu_ring_write(ring, v: reg);
968	amdgpu_ring_write(ring, v: 0);
969	amdgpu_ring_write(ring, v: val);
970	}
971
972	static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
973	int mem_space, int opt, uint32_t addr0,
974	uint32_t addr1, uint32_t ref, uint32_t mask,
975	uint32_t inv)
976	{
977	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
978	amdgpu_ring_write(ring,
979	/* memory (1) or register (0) */
980	v: (WAIT_REG_MEM_MEM_SPACE(mem_space) |
981	WAIT_REG_MEM_OPERATION(opt) | /* wait */
982	WAIT_REG_MEM_FUNCTION(3) | /* equal */
983	WAIT_REG_MEM_ENGINE(eng_sel)));
984
985	if (mem_space)
986	BUG_ON(addr0 & 0x3); /* Dword align */
987	amdgpu_ring_write(ring, v: addr0);
988	amdgpu_ring_write(ring, v: addr1);
989	amdgpu_ring_write(ring, v: ref);
990	amdgpu_ring_write(ring, v: mask);
991	amdgpu_ring_write(ring, v: inv); /* poll interval */
992	}
993
994	static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
995	{
996	struct amdgpu_device *adev = ring->adev;
997	uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
998	uint32_t tmp = 0;
999	unsigned i;
1000	int r;
1001
1002	WREG32(scratch, 0xCAFEDEAD);
1003	r = amdgpu_ring_alloc(ring, ndw: 3);
1004	if (r)
1005	return r;
1006
1007	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1008	amdgpu_ring_write(ring, v: scratch - PACKET3_SET_UCONFIG_REG_START);
1009	amdgpu_ring_write(ring, v: 0xDEADBEEF);
1010	amdgpu_ring_commit(ring);
1011
1012	for (i = 0; i < adev->usec_timeout; i++) {
1013	tmp = RREG32(scratch);
1014	if (tmp == 0xDEADBEEF)
1015	break;
1016	udelay(1);
1017	}
1018
1019	if (i >= adev->usec_timeout)
1020	r = -ETIMEDOUT;
1021	return r;
1022	}
1023
1024	static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1025	{
1026	struct amdgpu_device *adev = ring->adev;
1027	struct amdgpu_ib ib;
1028	struct dma_fence *f = NULL;
1029
1030	unsigned index;
1031	uint64_t gpu_addr;
1032	uint32_t tmp;
1033	long r;
1034
1035	r = amdgpu_device_wb_get(adev, wb: &index);
1036	if (r)
1037	return r;
1038
1039	gpu_addr = adev->wb.gpu_addr + (index * 4);
1040	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1041	memset(&ib, 0, sizeof(ib));
1042	r = amdgpu_ib_get(adev, NULL, size: 16,
1043	pool: AMDGPU_IB_POOL_DIRECT, ib: &ib);
1044	if (r)
1045	goto err1;
1046
1047	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1048	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1049	ib.ptr[2] = lower_32_bits(gpu_addr);
1050	ib.ptr[3] = upper_32_bits(gpu_addr);
1051	ib.ptr[4] = 0xDEADBEEF;
1052	ib.length_dw = 5;
1053
1054	r = amdgpu_ib_schedule(ring, num_ibs: 1, ibs: &ib, NULL, f: &f);
1055	if (r)
1056	goto err2;
1057
1058	r = dma_fence_wait_timeout(f, intr: false, timeout);
1059	if (r == 0) {
1060	r = -ETIMEDOUT;
1061	goto err2;
1062	} else if (r < 0) {
1063	goto err2;
1064	}
1065
1066	tmp = adev->wb.wb[index];
1067	if (tmp == 0xDEADBEEF)
1068	r = 0;
1069	else
1070	r = -EINVAL;
1071
1072	err2:
1073	amdgpu_ib_free(adev, ib: &ib, NULL);
1074	dma_fence_put(fence: f);
1075	err1:
1076	amdgpu_device_wb_free(adev, wb: index);
1077	return r;
1078	}
1079
1080
1081	static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1082	{
1083	amdgpu_ucode_release(fw: &adev->gfx.pfp_fw);
1084	amdgpu_ucode_release(fw: &adev->gfx.me_fw);
1085	amdgpu_ucode_release(fw: &adev->gfx.ce_fw);
1086	amdgpu_ucode_release(fw: &adev->gfx.rlc_fw);
1087	amdgpu_ucode_release(fw: &adev->gfx.mec_fw);
1088	amdgpu_ucode_release(fw: &adev->gfx.mec2_fw);
1089
1090	kfree(objp: adev->gfx.rlc.register_list_format);
1091	}
1092
1093	static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1094	{
1095	adev->gfx.me_fw_write_wait = false;
1096	adev->gfx.mec_fw_write_wait = false;
1097
1098	if ((amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 1)) &&
1099	((adev->gfx.mec_fw_version < 0x000001a5) ||
1100	(adev->gfx.mec_feature_version < 46) ||
1101	(adev->gfx.pfp_fw_version < 0x000000b7) ||
1102	(adev->gfx.pfp_feature_version < 46)))
1103	DRM_WARN_ONCE("CP firmware version too old, please update!");
1104
1105	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1106	case IP_VERSION(9, 0, 1):
1107	if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1108	(adev->gfx.me_feature_version >= 42) &&
1109	(adev->gfx.pfp_fw_version >= 0x000000b1) &&
1110	(adev->gfx.pfp_feature_version >= 42))
1111	adev->gfx.me_fw_write_wait = true;
1112
1113	if ((adev->gfx.mec_fw_version >= 0x00000193) &&
1114	(adev->gfx.mec_feature_version >= 42))
1115	adev->gfx.mec_fw_write_wait = true;
1116	break;
1117	case IP_VERSION(9, 2, 1):
1118	if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1119	(adev->gfx.me_feature_version >= 44) &&
1120	(adev->gfx.pfp_fw_version >= 0x000000b2) &&
1121	(adev->gfx.pfp_feature_version >= 44))
1122	adev->gfx.me_fw_write_wait = true;
1123
1124	if ((adev->gfx.mec_fw_version >= 0x00000196) &&
1125	(adev->gfx.mec_feature_version >= 44))
1126	adev->gfx.mec_fw_write_wait = true;
1127	break;
1128	case IP_VERSION(9, 4, 0):
1129	if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1130	(adev->gfx.me_feature_version >= 44) &&
1131	(adev->gfx.pfp_fw_version >= 0x000000b2) &&
1132	(adev->gfx.pfp_feature_version >= 44))
1133	adev->gfx.me_fw_write_wait = true;
1134
1135	if ((adev->gfx.mec_fw_version >= 0x00000197) &&
1136	(adev->gfx.mec_feature_version >= 44))
1137	adev->gfx.mec_fw_write_wait = true;
1138	break;
1139	case IP_VERSION(9, 1, 0):
1140	case IP_VERSION(9, 2, 2):
1141	if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1142	(adev->gfx.me_feature_version >= 42) &&
1143	(adev->gfx.pfp_fw_version >= 0x000000b1) &&
1144	(adev->gfx.pfp_feature_version >= 42))
1145	adev->gfx.me_fw_write_wait = true;
1146
1147	if ((adev->gfx.mec_fw_version >= 0x00000192) &&
1148	(adev->gfx.mec_feature_version >= 42))
1149	adev->gfx.mec_fw_write_wait = true;
1150	break;
1151	default:
1152	adev->gfx.me_fw_write_wait = true;
1153	adev->gfx.mec_fw_write_wait = true;
1154	break;
1155	}
1156	}
1157
1158	struct amdgpu_gfxoff_quirk {
1159	u16 chip_vendor;
1160	u16 chip_device;
1161	u16 subsys_vendor;
1162	u16 subsys_device;
1163	u8 revision;
1164	};
1165
1166	static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1167	/* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1168	{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1169	/* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1170	{ 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1171	/* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1172	{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1173	/* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
1174	{ 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 },
1175	{ 0, 0, 0, 0, 0 },
1176	};
1177
1178	static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1179	{
1180	const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1181
1182	while (p && p->chip_device != 0) {
1183	if (pdev->vendor == p->chip_vendor &&
1184	pdev->device == p->chip_device &&
1185	pdev->subsystem_vendor == p->subsys_vendor &&
1186	pdev->subsystem_device == p->subsys_device &&
1187	pdev->revision == p->revision) {
1188	return true;
1189	}
1190	++p;
1191	}
1192	return false;
1193	}
1194
1195	static bool is_raven_kicker(struct amdgpu_device *adev)
1196	{
1197	if (adev->pm.fw_version >= 0x41e2b)
1198	return true;
1199	else
1200	return false;
1201	}
1202
1203	static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)
1204	{
1205	if ((amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 3, 0)) &&
1206	(adev->gfx.me_fw_version >= 0x000000a5) &&
1207	(adev->gfx.me_feature_version >= 52))
1208	return true;
1209	else
1210	return false;
1211	}
1212
1213	static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1214	{
1215	if (gfx_v9_0_should_disable_gfxoff(pdev: adev->pdev))
1216	adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1217
1218	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1219	case IP_VERSION(9, 0, 1):
1220	case IP_VERSION(9, 2, 1):
1221	case IP_VERSION(9, 4, 0):
1222	break;
1223	case IP_VERSION(9, 2, 2):
1224	case IP_VERSION(9, 1, 0):
1225	if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1226	(adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1227	((!is_raven_kicker(adev) &&
1228	adev->gfx.rlc_fw_version < 531) ||
1229	(adev->gfx.rlc_feature_version < 1) ||
1230	!adev->gfx.rlc.is_rlc_v2_1))
1231	adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1232
1233	if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1234	adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1235	AMD_PG_SUPPORT_CP |
1236	AMD_PG_SUPPORT_RLC_SMU_HS;
1237	break;
1238	case IP_VERSION(9, 3, 0):
1239	if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1240	adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1241	AMD_PG_SUPPORT_CP |
1242	AMD_PG_SUPPORT_RLC_SMU_HS;
1243	break;
1244	default:
1245	break;
1246	}
1247	}
1248
1249	static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1250	char *chip_name)
1251	{
1252	char fw_name[30];
1253	int err;
1254
1255	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_pfp.bin", chip_name);
1256	err = amdgpu_ucode_request(adev, fw: &adev->gfx.pfp_fw, fw_name);
1257	if (err)
1258	goto out;
1259	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_PFP);
1260
1261	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_me.bin", chip_name);
1262	err = amdgpu_ucode_request(adev, fw: &adev->gfx.me_fw, fw_name);
1263	if (err)
1264	goto out;
1265	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_ME);
1266
1267	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_ce.bin", chip_name);
1268	err = amdgpu_ucode_request(adev, fw: &adev->gfx.ce_fw, fw_name);
1269	if (err)
1270	goto out;
1271	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_CE);
1272
1273	out:
1274	if (err) {
1275	amdgpu_ucode_release(fw: &adev->gfx.pfp_fw);
1276	amdgpu_ucode_release(fw: &adev->gfx.me_fw);
1277	amdgpu_ucode_release(fw: &adev->gfx.ce_fw);
1278	}
1279	return err;
1280	}
1281
1282	static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1283	char *chip_name)
1284	{
1285	char fw_name[30];
1286	int err;
1287	const struct rlc_firmware_header_v2_0 *rlc_hdr;
1288	uint16_t version_major;
1289	uint16_t version_minor;
1290	uint32_t smu_version;
1291
1292	/*
1293	* For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1294	* instead of picasso_rlc.bin.
1295	* Judgment method:
1296	* PCO AM4: revision >= 0xC8 && revision <= 0xCF
1297	* or revision >= 0xD8 && revision <= 0xDF
1298	* otherwise is PCO FP5
1299	*/
1300	if (!strcmp(chip_name, "picasso") &&
1301	(((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1302	((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1303	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_rlc_am4.bin", chip_name);
1304	else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, smu_version: &smu_version) == 0) &&
1305	(smu_version >= 0x41e2b))
1306	/**
1307	*SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1308	*/
1309	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_kicker_rlc.bin", chip_name);
1310	else
1311	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_rlc.bin", chip_name);
1312	err = amdgpu_ucode_request(adev, fw: &adev->gfx.rlc_fw, fw_name);
1313	if (err)
1314	goto out;
1315	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1316
1317	version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1318	version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1319	err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
1320	out:
1321	if (err)
1322	amdgpu_ucode_release(fw: &adev->gfx.rlc_fw);
1323
1324	return err;
1325	}
1326
1327	static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1328	{
1329	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2) ||
1330	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1) ||
1331	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 3, 0))
1332	return false;
1333
1334	return true;
1335	}
1336
1337	static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1338	char *chip_name)
1339	{
1340	char fw_name[30];
1341	int err;
1342
1343	if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1344	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_sjt_mec.bin", chip_name);
1345	else
1346	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_mec.bin", chip_name);
1347
1348	err = amdgpu_ucode_request(adev, fw: &adev->gfx.mec_fw, fw_name);
1349	if (err)
1350	goto out;
1351	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_MEC1);
1352	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_MEC1_JT);
1353
1354	if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1355	if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1356	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_sjt_mec2.bin", chip_name);
1357	else
1358	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_mec2.bin", chip_name);
1359
1360	/* ignore failures to load */
1361	err = amdgpu_ucode_request(adev, fw: &adev->gfx.mec2_fw, fw_name);
1362	if (!err) {
1363	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_MEC2);
1364	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_MEC2_JT);
1365	} else {
1366	err = 0;
1367	amdgpu_ucode_release(fw: &adev->gfx.mec2_fw);
1368	}
1369	} else {
1370	adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1371	adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1372	}
1373
1374	gfx_v9_0_check_if_need_gfxoff(adev);
1375	gfx_v9_0_check_fw_write_wait(adev);
1376
1377	out:
1378	if (err)
1379	amdgpu_ucode_release(fw: &adev->gfx.mec_fw);
1380	return err;
1381	}
1382
1383	static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1384	{
1385	char ucode_prefix[30];
1386	int r;
1387
1388	DRM_DEBUG("\n");
1389	amdgpu_ucode_ip_version_decode(adev, block_type: GC_HWIP, ucode_prefix, len: sizeof(ucode_prefix));
1390
1391	/* No CPG in Arcturus */
1392	if (adev->gfx.num_gfx_rings) {
1393	r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name: ucode_prefix);
1394	if (r)
1395	return r;
1396	}
1397
1398	r = gfx_v9_0_init_rlc_microcode(adev, chip_name: ucode_prefix);
1399	if (r)
1400	return r;
1401
1402	r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name: ucode_prefix);
1403	if (r)
1404	return r;
1405
1406	return r;
1407	}
1408
1409	static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1410	{
1411	u32 count = 0;
1412	const struct cs_section_def *sect = NULL;
1413	const struct cs_extent_def *ext = NULL;
1414
1415	/* begin clear state */
1416	count += 2;
1417	/* context control state */
1418	count += 3;
1419
1420	for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1421	for (ext = sect->section; ext->extent != NULL; ++ext) {
1422	if (sect->id == SECT_CONTEXT)
1423	count += 2 + ext->reg_count;
1424	else
1425	return 0;
1426	}
1427	}
1428
1429	/* end clear state */
1430	count += 2;
1431	/* clear state */
1432	count += 2;
1433
1434	return count;
1435	}
1436
1437	static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1438	volatile u32 *buffer)
1439	{
1440	u32 count = 0, i;
1441	const struct cs_section_def *sect = NULL;
1442	const struct cs_extent_def *ext = NULL;
1443
1444	if (adev->gfx.rlc.cs_data == NULL)
1445	return;
1446	if (buffer == NULL)
1447	return;
1448
1449	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1450	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1451
1452	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1453	buffer[count++] = cpu_to_le32(0x80000000);
1454	buffer[count++] = cpu_to_le32(0x80000000);
1455
1456	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1457	for (ext = sect->section; ext->extent != NULL; ++ext) {
1458	if (sect->id == SECT_CONTEXT) {
1459	buffer[count++] =
1460	cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1461	buffer[count++] = cpu_to_le32(ext->reg_index -
1462	PACKET3_SET_CONTEXT_REG_START);
1463	for (i = 0; i < ext->reg_count; i++)
1464	buffer[count++] = cpu_to_le32(ext->extent[i]);
1465	} else {
1466	return;
1467	}
1468	}
1469	}
1470
1471	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1472	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1473
1474	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1475	buffer[count++] = cpu_to_le32(0);
1476	}
1477
1478	static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1479	{
1480	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1481	uint32_t pg_always_on_cu_num = 2;
1482	uint32_t always_on_cu_num;
1483	uint32_t i, j, k;
1484	uint32_t mask, cu_bitmap, counter;
1485
1486	if (adev->flags & AMD_IS_APU)
1487	always_on_cu_num = 4;
1488	else if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 2, 1))
1489	always_on_cu_num = 8;
1490	else
1491	always_on_cu_num = 12;
1492
1493	mutex_lock(&adev->grbm_idx_mutex);
1494	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1495	for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1496	mask = 1;
1497	cu_bitmap = 0;
1498	counter = 0;
1499	amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
1500
1501	for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1502	if (cu_info->bitmap[0][i][j] & mask) {
1503	if (counter == pg_always_on_cu_num)
1504	WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1505	if (counter < always_on_cu_num)
1506	cu_bitmap |= mask;
1507	else
1508	break;
1509	counter++;
1510	}
1511	mask <<= 1;
1512	}
1513
1514	WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1515	cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1516	}
1517	}
1518	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1519	mutex_unlock(lock: &adev->grbm_idx_mutex);
1520	}
1521
1522	static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1523	{
1524	uint32_t data;
1525
1526	/* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1527	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1528	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1529	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1530	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1531
1532	/* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1533	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1534
1535	/* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1536	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1537
1538	mutex_lock(&adev->grbm_idx_mutex);
1539	/* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1540	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1541	WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1542
1543	/* set mmRLC_LB_PARAMS = 0x003F_1006 */
1544	data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1545	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1546	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1547	WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1548
1549	/* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1550	data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1551	data &= 0x0000FFFF;
1552	data |= 0x00C00000;
1553	WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1554
1555	/*
1556	* RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1557	* programmed in gfx_v9_0_init_always_on_cu_mask()
1558	*/
1559
1560	/* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1561	* but used for RLC_LB_CNTL configuration */
1562	data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1563	data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1564	data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1565	WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1566	mutex_unlock(lock: &adev->grbm_idx_mutex);
1567
1568	gfx_v9_0_init_always_on_cu_mask(adev);
1569	}
1570
1571	static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1572	{
1573	uint32_t data;
1574
1575	/* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1576	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1577	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1578	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1579	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1580
1581	/* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1582	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1583
1584	/* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1585	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1586
1587	mutex_lock(&adev->grbm_idx_mutex);
1588	/* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1589	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1590	WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1591
1592	/* set mmRLC_LB_PARAMS = 0x003F_1006 */
1593	data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1594	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1595	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1596	WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1597
1598	/* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1599	data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1600	data &= 0x0000FFFF;
1601	data |= 0x00C00000;
1602	WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1603
1604	/*
1605	* RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1606	* programmed in gfx_v9_0_init_always_on_cu_mask()
1607	*/
1608
1609	/* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1610	* but used for RLC_LB_CNTL configuration */
1611	data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1612	data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1613	data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1614	WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1615	mutex_unlock(lock: &adev->grbm_idx_mutex);
1616
1617	gfx_v9_0_init_always_on_cu_mask(adev);
1618	}
1619
1620	static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1621	{
1622	WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1623	}
1624
1625	static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1626	{
1627	if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1628	return 5;
1629	else
1630	return 4;
1631	}
1632
1633	static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
1634	{
1635	struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
1636
1637	reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
1638	reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1639	reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1);
1640	reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2);
1641	reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3);
1642	reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL);
1643	reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX);
1644	reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT);
1645	adev->gfx.rlc.rlcg_reg_access_supported = true;
1646	}
1647
1648	static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1649	{
1650	const struct cs_section_def *cs_data;
1651	int r;
1652
1653	adev->gfx.rlc.cs_data = gfx9_cs_data;
1654
1655	cs_data = adev->gfx.rlc.cs_data;
1656
1657	if (cs_data) {
1658	/* init clear state block */
1659	r = amdgpu_gfx_rlc_init_csb(adev);
1660	if (r)
1661	return r;
1662	}
1663
1664	if (adev->flags & AMD_IS_APU) {
1665	/* TODO: double check the cp_table_size for RV */
1666	adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1667	r = amdgpu_gfx_rlc_init_cpt(adev);
1668	if (r)
1669	return r;
1670	}
1671
1672	return 0;
1673	}
1674
1675	static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1676	{
1677	amdgpu_bo_free_kernel(bo: &adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1678	amdgpu_bo_free_kernel(bo: &adev->gfx.mec.mec_fw_obj, NULL, NULL);
1679	}
1680
1681	static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1682	{
1683	int r;
1684	u32 *hpd;
1685	const __le32 *fw_data;
1686	unsigned fw_size;
1687	u32 *fw;
1688	size_t mec_hpd_size;
1689
1690	const struct gfx_firmware_header_v1_0 *mec_hdr;
1691
1692	bitmap_zero(dst: adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1693
1694	/* take ownership of the relevant compute queues */
1695	amdgpu_gfx_compute_queue_acquire(adev);
1696	mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1697	if (mec_hpd_size) {
1698	r = amdgpu_bo_create_reserved(adev, size: mec_hpd_size, PAGE_SIZE,
1699	AMDGPU_GEM_DOMAIN_VRAM |
1700	AMDGPU_GEM_DOMAIN_GTT,
1701	bo_ptr: &adev->gfx.mec.hpd_eop_obj,
1702	gpu_addr: &adev->gfx.mec.hpd_eop_gpu_addr,
1703	cpu_addr: (void **)&hpd);
1704	if (r) {
1705	dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1706	gfx_v9_0_mec_fini(adev);
1707	return r;
1708	}
1709
1710	memset(hpd, 0, mec_hpd_size);
1711
1712	amdgpu_bo_kunmap(bo: adev->gfx.mec.hpd_eop_obj);
1713	amdgpu_bo_unreserve(bo: adev->gfx.mec.hpd_eop_obj);
1714	}
1715
1716	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1717
1718	fw_data = (const __le32 *)
1719	(adev->gfx.mec_fw->data +
1720	le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1721	fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1722
1723	r = amdgpu_bo_create_reserved(adev, size: mec_hdr->header.ucode_size_bytes,
1724	PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1725	bo_ptr: &adev->gfx.mec.mec_fw_obj,
1726	gpu_addr: &adev->gfx.mec.mec_fw_gpu_addr,
1727	cpu_addr: (void **)&fw);
1728	if (r) {
1729	dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1730	gfx_v9_0_mec_fini(adev);
1731	return r;
1732	}
1733
1734	memcpy(fw, fw_data, fw_size);
1735
1736	amdgpu_bo_kunmap(bo: adev->gfx.mec.mec_fw_obj);
1737	amdgpu_bo_unreserve(bo: adev->gfx.mec.mec_fw_obj);
1738
1739	return 0;
1740	}
1741
1742	static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1743	{
1744	WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1745	(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1746	(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1747	(address << SQ_IND_INDEX__INDEX__SHIFT) |
1748	(SQ_IND_INDEX__FORCE_READ_MASK));
1749	return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1750	}
1751
1752	static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1753	uint32_t wave, uint32_t thread,
1754	uint32_t regno, uint32_t num, uint32_t *out)
1755	{
1756	WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1757	(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1758	(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1759	(regno << SQ_IND_INDEX__INDEX__SHIFT) |
1760	(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
1761	(SQ_IND_INDEX__FORCE_READ_MASK) |
1762	(SQ_IND_INDEX__AUTO_INCR_MASK));
1763	while (num--)
1764	*(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1765	}
1766
1767	static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1768	{
1769	/* type 1 wave data */
1770	dst[(*no_fields)++] = 1;
1771	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
1772	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
1773	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
1774	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
1775	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
1776	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
1777	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
1778	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
1779	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
1780	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
1781	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
1782	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
1783	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
1784	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
1785	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
1786	}
1787
1788	static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1789	uint32_t wave, uint32_t start,
1790	uint32_t size, uint32_t *dst)
1791	{
1792	wave_read_regs(
1793	adev, simd, wave, thread: 0,
1794	regno: start + SQIND_WAVE_SGPRS_OFFSET, num: size, out: dst);
1795	}
1796
1797	static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1798	uint32_t wave, uint32_t thread,
1799	uint32_t start, uint32_t size,
1800	uint32_t *dst)
1801	{
1802	wave_read_regs(
1803	adev, simd, wave, thread,
1804	regno: start + SQIND_WAVE_VGPRS_OFFSET, num: size, out: dst);
1805	}
1806
1807	static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
1808	u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1809	{
1810	soc15_grbm_select(adev, me, pipe, queue: q, vmid: vm, xcc_id: 0);
1811	}
1812
1813	static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
1814	.get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
1815	.select_se_sh = &gfx_v9_0_select_se_sh,
1816	.read_wave_data = &gfx_v9_0_read_wave_data,
1817	.read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
1818	.read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
1819	.select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
1820	};
1821
1822	const struct amdgpu_ras_block_hw_ops gfx_v9_0_ras_ops = {
1823	.ras_error_inject = &gfx_v9_0_ras_error_inject,
1824	.query_ras_error_count = &gfx_v9_0_query_ras_error_count,
1825	.reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
1826	};
1827
1828	static struct amdgpu_gfx_ras gfx_v9_0_ras = {
1829	.ras_block = {
1830	.hw_ops = &gfx_v9_0_ras_ops,
1831	},
1832	};
1833
1834	static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
1835	{
1836	u32 gb_addr_config;
1837	int err;
1838
1839	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1840	case IP_VERSION(9, 0, 1):
1841	adev->gfx.config.max_hw_contexts = 8;
1842	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1843	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1844	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1845	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1846	gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
1847	break;
1848	case IP_VERSION(9, 2, 1):
1849	adev->gfx.config.max_hw_contexts = 8;
1850	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1851	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1852	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1853	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1854	gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
1855	DRM_INFO("fix gfx.config for vega12\n");
1856	break;
1857	case IP_VERSION(9, 4, 0):
1858	adev->gfx.ras = &gfx_v9_0_ras;
1859	adev->gfx.config.max_hw_contexts = 8;
1860	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1861	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1862	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1863	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1864	gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1865	gb_addr_config &= ~0xf3e777ff;
1866	gb_addr_config |= 0x22014042;
1867	/* check vbios table if gpu info is not available */
1868	err = amdgpu_atomfirmware_get_gfx_info(adev);
1869	if (err)
1870	return err;
1871	break;
1872	case IP_VERSION(9, 2, 2):
1873	case IP_VERSION(9, 1, 0):
1874	adev->gfx.config.max_hw_contexts = 8;
1875	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1876	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1877	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1878	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1879	if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1880	gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
1881	else
1882	gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
1883	break;
1884	case IP_VERSION(9, 4, 1):
1885	adev->gfx.ras = &gfx_v9_4_ras;
1886	adev->gfx.config.max_hw_contexts = 8;
1887	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1888	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1889	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1890	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1891	gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1892	gb_addr_config &= ~0xf3e777ff;
1893	gb_addr_config |= 0x22014042;
1894	break;
1895	case IP_VERSION(9, 3, 0):
1896	adev->gfx.config.max_hw_contexts = 8;
1897	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1898	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1899	adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
1900	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1901	gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1902	gb_addr_config &= ~0xf3e777ff;
1903	gb_addr_config |= 0x22010042;
1904	break;
1905	case IP_VERSION(9, 4, 2):
1906	adev->gfx.ras = &gfx_v9_4_2_ras;
1907	adev->gfx.config.max_hw_contexts = 8;
1908	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1909	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1910	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1911	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1912	gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1913	gb_addr_config &= ~0xf3e777ff;
1914	gb_addr_config |= 0x22014042;
1915	/* check vbios table if gpu info is not available */
1916	err = amdgpu_atomfirmware_get_gfx_info(adev);
1917	if (err)
1918	return err;
1919	break;
1920	default:
1921	BUG();
1922	break;
1923	}
1924
1925	adev->gfx.config.gb_addr_config = gb_addr_config;
1926
1927	adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
1928	REG_GET_FIELD(
1929	adev->gfx.config.gb_addr_config,
1930	GB_ADDR_CONFIG,
1931	NUM_PIPES);
1932
1933	adev->gfx.config.max_tile_pipes =
1934	adev->gfx.config.gb_addr_config_fields.num_pipes;
1935
1936	adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
1937	REG_GET_FIELD(
1938	adev->gfx.config.gb_addr_config,
1939	GB_ADDR_CONFIG,
1940	NUM_BANKS);
1941	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
1942	REG_GET_FIELD(
1943	adev->gfx.config.gb_addr_config,
1944	GB_ADDR_CONFIG,
1945	MAX_COMPRESSED_FRAGS);
1946	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
1947	REG_GET_FIELD(
1948	adev->gfx.config.gb_addr_config,
1949	GB_ADDR_CONFIG,
1950	NUM_RB_PER_SE);
1951	adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
1952	REG_GET_FIELD(
1953	adev->gfx.config.gb_addr_config,
1954	GB_ADDR_CONFIG,
1955	NUM_SHADER_ENGINES);
1956	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
1957	REG_GET_FIELD(
1958	adev->gfx.config.gb_addr_config,
1959	GB_ADDR_CONFIG,
1960	PIPE_INTERLEAVE_SIZE));
1961
1962	return 0;
1963	}
1964
1965	static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1966	int mec, int pipe, int queue)
1967	{
1968	unsigned irq_type;
1969	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
1970	unsigned int hw_prio;
1971
1972	ring = &adev->gfx.compute_ring[ring_id];
1973
1974	/* mec0 is me1 */
1975	ring->me = mec + 1;
1976	ring->pipe = pipe;
1977	ring->queue = queue;
1978
1979	ring->ring_obj = NULL;
1980	ring->use_doorbell = true;
1981	ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
1982	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1983	+ (ring_id * GFX9_MEC_HPD_SIZE);
1984	ring->vm_hub = AMDGPU_GFXHUB(0);
1985	sprintf(buf: ring->name, fmt: "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1986
1987	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1988	+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1989	+ ring->pipe;
1990	hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1991	AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
1992	/* type-2 packets are deprecated on MEC, use type-3 instead */
1993	return amdgpu_ring_init(adev, ring, max_dw: 1024, irq_src: &adev->gfx.eop_irq, irq_type,
1994	hw_prio, NULL);
1995	}
1996
1997	static int gfx_v9_0_sw_init(void *handle)
1998	{
1999	int i, j, k, r, ring_id;
2000	struct amdgpu_ring *ring;
2001	struct amdgpu_kiq *kiq;
2002	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2003	unsigned int hw_prio;
2004
2005	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
2006	case IP_VERSION(9, 0, 1):
2007	case IP_VERSION(9, 2, 1):
2008	case IP_VERSION(9, 4, 0):
2009	case IP_VERSION(9, 2, 2):
2010	case IP_VERSION(9, 1, 0):
2011	case IP_VERSION(9, 4, 1):
2012	case IP_VERSION(9, 3, 0):
2013	case IP_VERSION(9, 4, 2):
2014	adev->gfx.mec.num_mec = 2;
2015	break;
2016	default:
2017	adev->gfx.mec.num_mec = 1;
2018	break;
2019	}
2020
2021	adev->gfx.mec.num_pipe_per_mec = 4;
2022	adev->gfx.mec.num_queue_per_pipe = 8;
2023
2024	/* EOP Event */
2025	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, source: &adev->gfx.eop_irq);
2026	if (r)
2027	return r;
2028
2029	/* Privileged reg */
2030	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2031	source: &adev->gfx.priv_reg_irq);
2032	if (r)
2033	return r;
2034
2035	/* Privileged inst */
2036	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2037	source: &adev->gfx.priv_inst_irq);
2038	if (r)
2039	return r;
2040
2041	/* ECC error */
2042	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2043	source: &adev->gfx.cp_ecc_error_irq);
2044	if (r)
2045	return r;
2046
2047	/* FUE error */
2048	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2049	source: &adev->gfx.cp_ecc_error_irq);
2050	if (r)
2051	return r;
2052
2053	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2054
2055	if (adev->gfx.rlc.funcs) {
2056	if (adev->gfx.rlc.funcs->init) {
2057	r = adev->gfx.rlc.funcs->init(adev);
2058	if (r) {
2059	dev_err(adev->dev, "Failed to init rlc BOs!\n");
2060	return r;
2061	}
2062	}
2063	}
2064
2065	r = gfx_v9_0_mec_init(adev);
2066	if (r) {
2067	DRM_ERROR("Failed to init MEC BOs!\n");
2068	return r;
2069	}
2070
2071	/* set up the gfx ring */
2072	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2073	ring = &adev->gfx.gfx_ring[i];
2074	ring->ring_obj = NULL;
2075	if (!i)
2076	sprintf(buf: ring->name, fmt: "gfx");
2077	else
2078	sprintf(buf: ring->name, fmt: "gfx_%d", i);
2079	ring->use_doorbell = true;
2080	ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2081
2082	/* disable scheduler on the real ring */
2083	ring->no_scheduler = true;
2084	ring->vm_hub = AMDGPU_GFXHUB(0);
2085	r = amdgpu_ring_init(adev, ring, max_dw: 1024, irq_src: &adev->gfx.eop_irq,
2086	irq_type: AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2087	hw_prio: AMDGPU_RING_PRIO_DEFAULT, NULL);
2088	if (r)
2089	return r;
2090	}
2091
2092	/* set up the software rings */
2093	if (adev->gfx.num_gfx_rings) {
2094	for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2095	ring = &adev->gfx.sw_gfx_ring[i];
2096	ring->ring_obj = NULL;
2097	sprintf(buf: ring->name, fmt: amdgpu_sw_ring_name(idx: i));
2098	ring->use_doorbell = true;
2099	ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2100	ring->is_sw_ring = true;
2101	hw_prio = amdgpu_sw_ring_priority(idx: i);
2102	ring->vm_hub = AMDGPU_GFXHUB(0);
2103	r = amdgpu_ring_init(adev, ring, max_dw: 1024, irq_src: &adev->gfx.eop_irq,
2104	irq_type: AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio,
2105	NULL);
2106	if (r)
2107	return r;
2108	ring->wptr = 0;
2109	}
2110
2111	/* init the muxer and add software rings */
2112	r = amdgpu_ring_mux_init(mux: &adev->gfx.muxer, ring: &adev->gfx.gfx_ring[0],
2113	GFX9_NUM_SW_GFX_RINGS);
2114	if (r) {
2115	DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r);
2116	return r;
2117	}
2118	for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2119	r = amdgpu_ring_mux_add_sw_ring(mux: &adev->gfx.muxer,
2120	ring: &adev->gfx.sw_gfx_ring[i]);
2121	if (r) {
2122	DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r);
2123	return r;
2124	}
2125	}
2126	}
2127
2128	/* set up the compute queues - allocate horizontally across pipes */
2129	ring_id = 0;
2130	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2131	for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2132	for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2133	if (!amdgpu_gfx_is_mec_queue_enabled(adev, xcc_id: 0, mec: i,
2134	pipe: k, queue: j))
2135	continue;
2136
2137	r = gfx_v9_0_compute_ring_init(adev,
2138	ring_id,
2139	mec: i, pipe: k, queue: j);
2140	if (r)
2141	return r;
2142
2143	ring_id++;
2144	}
2145	}
2146	}
2147
2148	r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, xcc_id: 0);
2149	if (r) {
2150	DRM_ERROR("Failed to init KIQ BOs!\n");
2151	return r;
2152	}
2153
2154	kiq = &adev->gfx.kiq[0];
2155	r = amdgpu_gfx_kiq_init_ring(adev, ring: &kiq->ring, irq: &kiq->irq, xcc_id: 0);
2156	if (r)
2157	return r;
2158
2159	/* create MQD for all compute queues as wel as KIQ for SRIOV case */
2160	r = amdgpu_gfx_mqd_sw_init(adev, mqd_size: sizeof(struct v9_mqd_allocation), xcc_id: 0);
2161	if (r)
2162	return r;
2163
2164	adev->gfx.ce_ram_size = 0x8000;
2165
2166	r = gfx_v9_0_gpu_early_init(adev);
2167	if (r)
2168	return r;
2169
2170	if (amdgpu_gfx_ras_sw_init(adev)) {
2171	dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
2172	return -EINVAL;
2173	}
2174
2175	return 0;
2176	}
2177
2178
2179	static int gfx_v9_0_sw_fini(void *handle)
2180	{
2181	int i;
2182	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2183
2184	if (adev->gfx.num_gfx_rings) {
2185	for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
2186	amdgpu_ring_fini(ring: &adev->gfx.sw_gfx_ring[i]);
2187	amdgpu_ring_mux_fini(mux: &adev->gfx.muxer);
2188	}
2189
2190	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2191	amdgpu_ring_fini(ring: &adev->gfx.gfx_ring[i]);
2192	for (i = 0; i < adev->gfx.num_compute_rings; i++)
2193	amdgpu_ring_fini(ring: &adev->gfx.compute_ring[i]);
2194
2195	amdgpu_gfx_mqd_sw_fini(adev, xcc_id: 0);
2196	amdgpu_gfx_kiq_free_ring(ring: &adev->gfx.kiq[0].ring);
2197	amdgpu_gfx_kiq_fini(adev, xcc_id: 0);
2198
2199	gfx_v9_0_mec_fini(adev);
2200	amdgpu_bo_free_kernel(bo: &adev->gfx.rlc.clear_state_obj,
2201	gpu_addr: &adev->gfx.rlc.clear_state_gpu_addr,
2202	cpu_addr: (void **)&adev->gfx.rlc.cs_ptr);
2203	if (adev->flags & AMD_IS_APU) {
2204	amdgpu_bo_free_kernel(bo: &adev->gfx.rlc.cp_table_obj,
2205	gpu_addr: &adev->gfx.rlc.cp_table_gpu_addr,
2206	cpu_addr: (void **)&adev->gfx.rlc.cp_table_ptr);
2207	}
2208	gfx_v9_0_free_microcode(adev);
2209
2210	return 0;
2211	}
2212
2213
2214	static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2215	{
2216	/* TODO */
2217	}
2218
2219	void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2220	u32 instance, int xcc_id)
2221	{
2222	u32 data;
2223
2224	if (instance == 0xffffffff)
2225	data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2226	else
2227	data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2228
2229	if (se_num == 0xffffffff)
2230	data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2231	else
2232	data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2233
2234	if (sh_num == 0xffffffff)
2235	data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2236	else
2237	data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2238
2239	WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2240	}
2241
2242	static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2243	{
2244	u32 data, mask;
2245
2246	data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2247	data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2248
2249	data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2250	data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2251
2252	mask = amdgpu_gfx_create_bitmask(bit_width: adev->gfx.config.max_backends_per_se /
2253	adev->gfx.config.max_sh_per_se);
2254
2255	return (~data) & mask;
2256	}
2257
2258	static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2259	{
2260	int i, j;
2261	u32 data;
2262	u32 active_rbs = 0;
2263	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2264	adev->gfx.config.max_sh_per_se;
2265
2266	mutex_lock(&adev->grbm_idx_mutex);
2267	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2268	for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2269	amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2270	data = gfx_v9_0_get_rb_active_bitmap(adev);
2271	active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2272	rb_bitmap_width_per_sh);
2273	}
2274	}
2275	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2276	mutex_unlock(lock: &adev->grbm_idx_mutex);
2277
2278	adev->gfx.config.backend_enable_mask = active_rbs;
2279	adev->gfx.config.num_rbs = hweight32(active_rbs);
2280	}
2281
2282	static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,
2283	uint32_t first_vmid,
2284	uint32_t last_vmid)
2285	{
2286	uint32_t data;
2287	uint32_t trap_config_vmid_mask = 0;
2288	int i;
2289
2290	/* Calculate trap config vmid mask */
2291	for (i = first_vmid; i < last_vmid; i++)
2292	trap_config_vmid_mask |= (1 << i);
2293
2294	data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,
2295	VMID_SEL, trap_config_vmid_mask);
2296	data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
2297	TRAP_EN, 1);
2298	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
2299	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
2300
2301	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0);
2302	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0);
2303	}
2304
2305	#define DEFAULT_SH_MEM_BASES (0x6000)
2306	static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2307	{
2308	int i;
2309	uint32_t sh_mem_config;
2310	uint32_t sh_mem_bases;
2311
2312	/*
2313	* Configure apertures:
2314	* LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
2315	* Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
2316	* GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
2317	*/
2318	sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2319
2320	sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2321	SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2322	SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2323
2324	mutex_lock(&adev->srbm_mutex);
2325	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2326	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: i, xcc_id: 0);
2327	/* CP and shaders */
2328	WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2329	WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2330	}
2331	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
2332	mutex_unlock(lock: &adev->srbm_mutex);
2333
2334	/* Initialize all compute VMIDs to have no GDS, GWS, or OA
2335	access. These should be enabled by FW for target VMIDs. */
2336	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2337	WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2338	WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2339	WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2340	WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2341	}
2342	}
2343
2344	static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2345	{
2346	int vmid;
2347
2348	/*
2349	* Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2350	* access. Compute VMIDs should be enabled by FW for target VMIDs,
2351	* the driver can enable them for graphics. VMID0 should maintain
2352	* access so that HWS firmware can save/restore entries.
2353	*/
2354	for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2355	WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2356	WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2357	WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2358	WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2359	}
2360	}
2361
2362	static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2363	{
2364	uint32_t tmp;
2365
2366	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
2367	case IP_VERSION(9, 4, 1):
2368	tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2369	tmp = REG_SET_FIELD(tmp, SQ_CONFIG, DISABLE_BARRIER_WAITCNT,
2370	!READ_ONCE(adev->barrier_has_auto_waitcnt));
2371	WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2372	break;
2373	default:
2374	break;
2375	}
2376	}
2377
2378	static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2379	{
2380	u32 tmp;
2381	int i;
2382
2383	WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2384
2385	gfx_v9_0_tiling_mode_table_init(adev);
2386
2387	if (adev->gfx.num_gfx_rings)
2388	gfx_v9_0_setup_rb(adev);
2389	gfx_v9_0_get_cu_info(adev, cu_info: &adev->gfx.cu_info);
2390	adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2391
2392	/* XXX SH_MEM regs */
2393	/* where to put LDS, scratch, GPUVM in FSA64 space */
2394	mutex_lock(&adev->srbm_mutex);
2395	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
2396	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: i, xcc_id: 0);
2397	/* CP and shaders */
2398	if (i == 0) {
2399	tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2400	SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2401	tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2402	!!adev->gmc.noretry);
2403	WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2404	WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2405	} else {
2406	tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2407	SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2408	tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2409	!!adev->gmc.noretry);
2410	WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2411	tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2412	(adev->gmc.private_aperture_start >> 48));
2413	tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2414	(adev->gmc.shared_aperture_start >> 48));
2415	WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2416	}
2417	}
2418	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
2419
2420	mutex_unlock(lock: &adev->srbm_mutex);
2421
2422	gfx_v9_0_init_compute_vmid(adev);
2423	gfx_v9_0_init_gds_vmid(adev);
2424	gfx_v9_0_init_sq_config(adev);
2425	}
2426
2427	static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2428	{
2429	u32 i, j, k;
2430	u32 mask;
2431
2432	mutex_lock(&adev->grbm_idx_mutex);
2433	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2434	for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2435	amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2436	for (k = 0; k < adev->usec_timeout; k++) {
2437	if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2438	break;
2439	udelay(1);
2440	}
2441	if (k == adev->usec_timeout) {
2442	amdgpu_gfx_select_se_sh(adev, 0xffffffff,
2443	0xffffffff, 0xffffffff, 0);
2444	mutex_unlock(lock: &adev->grbm_idx_mutex);
2445	DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2446	i, j);
2447	return;
2448	}
2449	}
2450	}
2451	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2452	mutex_unlock(lock: &adev->grbm_idx_mutex);
2453
2454	mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2455	RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2456	RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2457	RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2458	for (k = 0; k < adev->usec_timeout; k++) {
2459	if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2460	break;
2461	udelay(1);
2462	}
2463	}
2464
2465	static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2466	bool enable)
2467	{
2468	u32 tmp;
2469
2470	/* These interrupts should be enabled to drive DS clock */
2471
2472	tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2473
2474	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2475	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2476	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2477	if(adev->gfx.num_gfx_rings)
2478	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2479
2480	WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2481	}
2482
2483	static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2484	{
2485	adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2486	/* csib */
2487	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2488	adev->gfx.rlc.clear_state_gpu_addr >> 32);
2489	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2490	adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2491	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2492	adev->gfx.rlc.clear_state_size);
2493	}
2494
2495	static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2496	int indirect_offset,
2497	int list_size,
2498	int *unique_indirect_regs,
2499	int unique_indirect_reg_count,
2500	int *indirect_start_offsets,
2501	int *indirect_start_offsets_count,
2502	int max_start_offsets_count)
2503	{
2504	int idx;
2505
2506	for (; indirect_offset < list_size; indirect_offset++) {
2507	WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2508	indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2509	*indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2510
2511	while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2512	indirect_offset += 2;
2513
2514	/* look for the matching indice */
2515	for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2516	if (unique_indirect_regs[idx] ==
2517	register_list_format[indirect_offset] ||
2518	!unique_indirect_regs[idx])
2519	break;
2520	}
2521
2522	BUG_ON(idx >= unique_indirect_reg_count);
2523
2524	if (!unique_indirect_regs[idx])
2525	unique_indirect_regs[idx] = register_list_format[indirect_offset];
2526
2527	indirect_offset++;
2528	}
2529	}
2530	}
2531
2532	static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2533	{
2534	int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2535	int unique_indirect_reg_count = 0;
2536
2537	int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2538	int indirect_start_offsets_count = 0;
2539
2540	int list_size = 0;
2541	int i = 0, j = 0;
2542	u32 tmp = 0;
2543
2544	u32 *register_list_format =
2545	kmemdup(p: adev->gfx.rlc.register_list_format,
2546	size: adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2547	if (!register_list_format)
2548	return -ENOMEM;
2549
2550	/* setup unique_indirect_regs array and indirect_start_offsets array */
2551	unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2552	gfx_v9_1_parse_ind_reg_list(register_list_format,
2553	indirect_offset: adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2554	list_size: adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2555	unique_indirect_regs,
2556	unique_indirect_reg_count,
2557	indirect_start_offsets,
2558	indirect_start_offsets_count: &indirect_start_offsets_count,
2559	ARRAY_SIZE(indirect_start_offsets));
2560
2561	/* enable auto inc in case it is disabled */
2562	tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2563	tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2564	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2565
2566	/* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2567	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2568	RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2569	for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2570	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2571	adev->gfx.rlc.register_restore[i]);
2572
2573	/* load indirect register */
2574	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2575	adev->gfx.rlc.reg_list_format_start);
2576
2577	/* direct register portion */
2578	for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2579	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2580	register_list_format[i]);
2581
2582	/* indirect register portion */
2583	while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2584	if (register_list_format[i] == 0xFFFFFFFF) {
2585	WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2586	continue;
2587	}
2588
2589	WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2590	WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2591
2592	for (j = 0; j < unique_indirect_reg_count; j++) {
2593	if (register_list_format[i] == unique_indirect_regs[j]) {
2594	WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2595	break;
2596	}
2597	}
2598
2599	BUG_ON(j >= unique_indirect_reg_count);
2600
2601	i++;
2602	}
2603
2604	/* set save/restore list size */
2605	list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2606	list_size = list_size >> 1;
2607	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2608	adev->gfx.rlc.reg_restore_list_size);
2609	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2610
2611	/* write the starting offsets to RLC scratch ram */
2612	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2613	adev->gfx.rlc.starting_offsets_start);
2614	for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2615	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2616	indirect_start_offsets[i]);
2617
2618	/* load unique indirect regs*/
2619	for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2620	if (unique_indirect_regs[i] != 0) {
2621	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2622	+ GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2623	unique_indirect_regs[i] & 0x3FFFF);
2624
2625	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2626	+ GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2627	unique_indirect_regs[i] >> 20);
2628	}
2629	}
2630
2631	kfree(objp: register_list_format);
2632	return 0;
2633	}
2634
2635	static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2636	{
2637	WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2638	}
2639
2640	static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2641	bool enable)
2642	{
2643	uint32_t data = 0;
2644	uint32_t default_data = 0;
2645
2646	default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2647	if (enable) {
2648	/* enable GFXIP control over CGPG */
2649	data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2650	if(default_data != data)
2651	WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2652
2653	/* update status */
2654	data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2655	data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2656	if(default_data != data)
2657	WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2658	} else {
2659	/* restore GFXIP control over GCPG */
2660	data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2661	if(default_data != data)
2662	WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2663	}
2664	}
2665
2666	static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2667	{
2668	uint32_t data = 0;
2669
2670	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2671	AMD_PG_SUPPORT_GFX_SMG |
2672	AMD_PG_SUPPORT_GFX_DMG)) {
2673	/* init IDLE_POLL_COUNT = 60 */
2674	data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2675	data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2676	data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2677	WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2678
2679	/* init RLC PG Delay */
2680	data = 0;
2681	data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2682	data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2683	data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2684	data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2685	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2686
2687	data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2688	data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2689	data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2690	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2691
2692	data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2693	data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2694	data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2695	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2696
2697	data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2698	data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2699
2700	/* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2701	data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2702	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2703	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 3, 0))
2704	pwr_10_0_gfxip_control_over_cgpg(adev, enable: true);
2705	}
2706	}
2707
2708	static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2709	bool enable)
2710	{
2711	uint32_t data = 0;
2712	uint32_t default_data = 0;
2713
2714	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2715	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2716	SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2717	enable ? 1 : 0);
2718	if (default_data != data)
2719	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2720	}
2721
2722	static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2723	bool enable)
2724	{
2725	uint32_t data = 0;
2726	uint32_t default_data = 0;
2727
2728	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2729	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2730	SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2731	enable ? 1 : 0);
2732	if(default_data != data)
2733	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2734	}
2735
2736	static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2737	bool enable)
2738	{
2739	uint32_t data = 0;
2740	uint32_t default_data = 0;
2741
2742	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2743	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2744	CP_PG_DISABLE,
2745	enable ? 0 : 1);
2746	if(default_data != data)
2747	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2748	}
2749
2750	static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2751	bool enable)
2752	{
2753	uint32_t data, default_data;
2754
2755	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2756	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2757	GFX_POWER_GATING_ENABLE,
2758	enable ? 1 : 0);
2759	if(default_data != data)
2760	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2761	}
2762
2763	static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2764	bool enable)
2765	{
2766	uint32_t data, default_data;
2767
2768	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2769	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2770	GFX_PIPELINE_PG_ENABLE,
2771	enable ? 1 : 0);
2772	if(default_data != data)
2773	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2774
2775	if (!enable)
2776	/* read any GFX register to wake up GFX */
2777	data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
2778	}
2779
2780	static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
2781	bool enable)
2782	{
2783	uint32_t data, default_data;
2784
2785	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2786	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2787	STATIC_PER_CU_PG_ENABLE,
2788	enable ? 1 : 0);
2789	if(default_data != data)
2790	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2791	}
2792
2793	static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
2794	bool enable)
2795	{
2796	uint32_t data, default_data;
2797
2798	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2799	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2800	DYN_PER_CU_PG_ENABLE,
2801	enable ? 1 : 0);
2802	if(default_data != data)
2803	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2804	}
2805
2806	static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
2807	{
2808	gfx_v9_0_init_csb(adev);
2809
2810	/*
2811	* Rlc save restore list is workable since v2_1.
2812	* And it's needed by gfxoff feature.
2813	*/
2814	if (adev->gfx.rlc.is_rlc_v2_1) {
2815	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) ==
2816	IP_VERSION(9, 2, 1) ||
2817	(adev->apu_flags & AMD_APU_IS_RAVEN2))
2818	gfx_v9_1_init_rlc_save_restore_list(adev);
2819	gfx_v9_0_enable_save_restore_machine(adev);
2820	}
2821
2822	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2823	AMD_PG_SUPPORT_GFX_SMG |
2824	AMD_PG_SUPPORT_GFX_DMG |
2825	AMD_PG_SUPPORT_CP |
2826	AMD_PG_SUPPORT_GDS |
2827	AMD_PG_SUPPORT_RLC_SMU_HS)) {
2828	WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE,
2829	adev->gfx.rlc.cp_table_gpu_addr >> 8);
2830	gfx_v9_0_init_gfx_power_gating(adev);
2831	}
2832	}
2833
2834	static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
2835	{
2836	WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
2837	gfx_v9_0_enable_gui_idle_interrupt(adev, enable: false);
2838	gfx_v9_0_wait_for_rlc_serdes(adev);
2839	}
2840
2841	static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
2842	{
2843	WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2844	udelay(50);
2845	WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2846	udelay(50);
2847	}
2848
2849	static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
2850	{
2851	#ifdef AMDGPU_RLC_DEBUG_RETRY
2852	u32 rlc_ucode_ver;
2853	#endif
2854
2855	WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
2856	udelay(50);
2857
2858	/* carrizo do enable cp interrupt after cp inited */
2859	if (!(adev->flags & AMD_IS_APU)) {
2860	gfx_v9_0_enable_gui_idle_interrupt(adev, enable: true);
2861	udelay(50);
2862	}
2863
2864	#ifdef AMDGPU_RLC_DEBUG_RETRY
2865	/* RLC_GPM_GENERAL_6 : RLC Ucode version */
2866	rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
2867	if(rlc_ucode_ver == 0x108) {
2868	DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
2869	rlc_ucode_ver, adev->gfx.rlc_fw_version);
2870	/* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
2871	* default is 0x9C4 to create a 100us interval */
2872	WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
2873	/* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
2874	* to disable the page fault retry interrupts, default is
2875	* 0x100 (256) */
2876	WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
2877	}
2878	#endif
2879	}
2880
2881	static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
2882	{
2883	const struct rlc_firmware_header_v2_0 *hdr;
2884	const __le32 *fw_data;
2885	unsigned i, fw_size;
2886
2887	if (!adev->gfx.rlc_fw)
2888	return -EINVAL;
2889
2890	hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2891	amdgpu_ucode_print_rlc_hdr(hdr: &hdr->header);
2892
2893	fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2894	le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2895	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2896
2897	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
2898	RLCG_UCODE_LOADING_START_ADDRESS);
2899	for (i = 0; i < fw_size; i++)
2900	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
2901	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
2902
2903	return 0;
2904	}
2905
2906	static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
2907	{
2908	int r;
2909
2910	if (amdgpu_sriov_vf(adev)) {
2911	gfx_v9_0_init_csb(adev);
2912	return 0;
2913	}
2914
2915	adev->gfx.rlc.funcs->stop(adev);
2916
2917	/* disable CG */
2918	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
2919
2920	gfx_v9_0_init_pg(adev);
2921
2922	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
2923	/* legacy rlc firmware loading */
2924	r = gfx_v9_0_rlc_load_microcode(adev);
2925	if (r)
2926	return r;
2927	}
2928
2929	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
2930	case IP_VERSION(9, 2, 2):
2931	case IP_VERSION(9, 1, 0):
2932	gfx_v9_0_init_lbpw(adev);
2933	if (amdgpu_lbpw == 0)
2934	gfx_v9_0_enable_lbpw(adev, enable: false);
2935	else
2936	gfx_v9_0_enable_lbpw(adev, enable: true);
2937	break;
2938	case IP_VERSION(9, 4, 0):
2939	gfx_v9_4_init_lbpw(adev);
2940	if (amdgpu_lbpw > 0)
2941	gfx_v9_0_enable_lbpw(adev, enable: true);
2942	else
2943	gfx_v9_0_enable_lbpw(adev, enable: false);
2944	break;
2945	default:
2946	break;
2947	}
2948
2949	gfx_v9_0_update_spm_vmid_internal(adev, vmid: 0xf);
2950
2951	adev->gfx.rlc.funcs->start(adev);
2952
2953	return 0;
2954	}
2955
2956	static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2957	{
2958	u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
2959
2960	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2961	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2962	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
2963	WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
2964	udelay(50);
2965	}
2966
2967	static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
2968	{
2969	const struct gfx_firmware_header_v1_0 *pfp_hdr;
2970	const struct gfx_firmware_header_v1_0 *ce_hdr;
2971	const struct gfx_firmware_header_v1_0 *me_hdr;
2972	const __le32 *fw_data;
2973	unsigned i, fw_size;
2974
2975	if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
2976	return -EINVAL;
2977
2978	pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2979	adev->gfx.pfp_fw->data;
2980	ce_hdr = (const struct gfx_firmware_header_v1_0 *)
2981	adev->gfx.ce_fw->data;
2982	me_hdr = (const struct gfx_firmware_header_v1_0 *)
2983	adev->gfx.me_fw->data;
2984
2985	amdgpu_ucode_print_gfx_hdr(hdr: &pfp_hdr->header);
2986	amdgpu_ucode_print_gfx_hdr(hdr: &ce_hdr->header);
2987	amdgpu_ucode_print_gfx_hdr(hdr: &me_hdr->header);
2988
2989	gfx_v9_0_cp_gfx_enable(adev, enable: false);
2990
2991	/* PFP */
2992	fw_data = (const __le32 *)
2993	(adev->gfx.pfp_fw->data +
2994	le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2995	fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
2996	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
2997	for (i = 0; i < fw_size; i++)
2998	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
2999	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3000
3001	/* CE */
3002	fw_data = (const __le32 *)
3003	(adev->gfx.ce_fw->data +
3004	le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3005	fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3006	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3007	for (i = 0; i < fw_size; i++)
3008	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3009	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3010
3011	/* ME */
3012	fw_data = (const __le32 *)
3013	(adev->gfx.me_fw->data +
3014	le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3015	fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3016	WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3017	for (i = 0; i < fw_size; i++)
3018	WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3019	WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3020
3021	return 0;
3022	}
3023
3024	static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3025	{
3026	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3027	const struct cs_section_def *sect = NULL;
3028	const struct cs_extent_def *ext = NULL;
3029	int r, i, tmp;
3030
3031	/* init the CP */
3032	WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3033	WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3034
3035	gfx_v9_0_cp_gfx_enable(adev, enable: true);
3036
3037	r = amdgpu_ring_alloc(ring, ndw: gfx_v9_0_get_csb_size(adev) + 4 + 3);
3038	if (r) {
3039	DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3040	return r;
3041	}
3042
3043	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3044	amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3045
3046	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3047	amdgpu_ring_write(ring, v: 0x80000000);
3048	amdgpu_ring_write(ring, v: 0x80000000);
3049
3050	for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3051	for (ext = sect->section; ext->extent != NULL; ++ext) {
3052	if (sect->id == SECT_CONTEXT) {
3053	amdgpu_ring_write(ring,
3054	PACKET3(PACKET3_SET_CONTEXT_REG,
3055	ext->reg_count));
3056	amdgpu_ring_write(ring,
3057	v: ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3058	for (i = 0; i < ext->reg_count; i++)
3059	amdgpu_ring_write(ring, v: ext->extent[i]);
3060	}
3061	}
3062	}
3063
3064	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3065	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3066
3067	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3068	amdgpu_ring_write(ring, v: 0);
3069
3070	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3071	amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3072	amdgpu_ring_write(ring, v: 0x8000);
3073	amdgpu_ring_write(ring, v: 0x8000);
3074
3075	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3076	tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3077	(SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3078	amdgpu_ring_write(ring, v: tmp);
3079	amdgpu_ring_write(ring, v: 0);
3080
3081	amdgpu_ring_commit(ring);
3082
3083	return 0;
3084	}
3085
3086	static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3087	{
3088	struct amdgpu_ring *ring;
3089	u32 tmp;
3090	u32 rb_bufsz;
3091	u64 rb_addr, rptr_addr, wptr_gpu_addr;
3092
3093	/* Set the write pointer delay */
3094	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3095
3096	/* set the RB to use vmid 0 */
3097	WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3098
3099	/* Set ring buffer size */
3100	ring = &adev->gfx.gfx_ring[0];
3101	rb_bufsz = order_base_2(ring->ring_size / 8);
3102	tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3103	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3104	#ifdef __BIG_ENDIAN
3105	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3106	#endif
3107	WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3108
3109	/* Initialize the ring buffer's write pointers */
3110	ring->wptr = 0;
3111	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3112	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3113
3114	/* set the wb address wether it's enabled or not */
3115	rptr_addr = ring->rptr_gpu_addr;
3116	WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3117	WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3118
3119	wptr_gpu_addr = ring->wptr_gpu_addr;
3120	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3121	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3122
3123	mdelay(1);
3124	WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3125
3126	rb_addr = ring->gpu_addr >> 8;
3127	WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3128	WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3129
3130	tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3131	if (ring->use_doorbell) {
3132	tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3133	DOORBELL_OFFSET, ring->doorbell_index);
3134	tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3135	DOORBELL_EN, 1);
3136	} else {
3137	tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3138	}
3139	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3140
3141	tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3142	DOORBELL_RANGE_LOWER, ring->doorbell_index);
3143	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3144
3145	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3146	CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3147
3148
3149	/* start the ring */
3150	gfx_v9_0_cp_gfx_start(adev);
3151
3152	return 0;
3153	}
3154
3155	static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3156	{
3157	if (enable) {
3158	WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3159	} else {
3160	WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3161	(CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3162	adev->gfx.kiq[0].ring.sched.ready = false;
3163	}
3164	udelay(50);
3165	}
3166
3167	static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3168	{
3169	const struct gfx_firmware_header_v1_0 *mec_hdr;
3170	const __le32 *fw_data;
3171	unsigned i;
3172	u32 tmp;
3173
3174	if (!adev->gfx.mec_fw)
3175	return -EINVAL;
3176
3177	gfx_v9_0_cp_compute_enable(adev, enable: false);
3178
3179	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3180	amdgpu_ucode_print_gfx_hdr(hdr: &mec_hdr->header);
3181
3182	fw_data = (const __le32 *)
3183	(adev->gfx.mec_fw->data +
3184	le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3185	tmp = 0;
3186	tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3187	tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3188	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3189
3190	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3191	adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3192	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3193	upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3194
3195	/* MEC1 */
3196	WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3197	mec_hdr->jt_offset);
3198	for (i = 0; i < mec_hdr->jt_size; i++)
3199	WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3200	le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3201
3202	WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3203	adev->gfx.mec_fw_version);
3204	/* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3205
3206	return 0;
3207	}
3208
3209	/* KIQ functions */
3210	static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3211	{
3212	uint32_t tmp;
3213	struct amdgpu_device *adev = ring->adev;
3214
3215	/* tell RLC which is KIQ queue */
3216	tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3217	tmp &= 0xffffff00;
3218	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3219	WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3220	tmp |= 0x80;
3221	WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3222	}
3223
3224	static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3225	{
3226	struct amdgpu_device *adev = ring->adev;
3227
3228	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3229	if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3230	mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3231	mqd->cp_hqd_queue_priority =
3232	AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3233	}
3234	}
3235	}
3236
3237	static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3238	{
3239	struct amdgpu_device *adev = ring->adev;
3240	struct v9_mqd *mqd = ring->mqd_ptr;
3241	uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3242	uint32_t tmp;
3243
3244	mqd->header = 0xC0310800;
3245	mqd->compute_pipelinestat_enable = 0x00000001;
3246	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3247	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3248	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3249	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3250	mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3251	mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3252	mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3253	mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3254	mqd->compute_misc_reserved = 0x00000003;
3255
3256	mqd->dynamic_cu_mask_addr_lo =
3257	lower_32_bits(ring->mqd_gpu_addr
3258	+ offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3259	mqd->dynamic_cu_mask_addr_hi =
3260	upper_32_bits(ring->mqd_gpu_addr
3261	+ offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3262
3263	eop_base_addr = ring->eop_gpu_addr >> 8;
3264	mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3265	mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3266
3267	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3268	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3269	tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3270	(order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3271
3272	mqd->cp_hqd_eop_control = tmp;
3273
3274	/* enable doorbell? */
3275	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3276
3277	if (ring->use_doorbell) {
3278	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3279	DOORBELL_OFFSET, ring->doorbell_index);
3280	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3281	DOORBELL_EN, 1);
3282	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3283	DOORBELL_SOURCE, 0);
3284	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3285	DOORBELL_HIT, 0);
3286	} else {
3287	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3288	DOORBELL_EN, 0);
3289	}
3290
3291	mqd->cp_hqd_pq_doorbell_control = tmp;
3292
3293	/* disable the queue if it's active */
3294	ring->wptr = 0;
3295	mqd->cp_hqd_dequeue_request = 0;
3296	mqd->cp_hqd_pq_rptr = 0;
3297	mqd->cp_hqd_pq_wptr_lo = 0;
3298	mqd->cp_hqd_pq_wptr_hi = 0;
3299
3300	/* set the pointer to the MQD */
3301	mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3302	mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3303
3304	/* set MQD vmid to 0 */
3305	tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3306	tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3307	mqd->cp_mqd_control = tmp;
3308
3309	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3310	hqd_gpu_addr = ring->gpu_addr >> 8;
3311	mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3312	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3313
3314	/* set up the HQD, this is similar to CP_RB0_CNTL */
3315	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3316	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3317	(order_base_2(ring->ring_size / 4) - 1));
3318	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3319	(order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
3320	#ifdef __BIG_ENDIAN
3321	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3322	#endif
3323	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3324	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3325	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3326	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3327	mqd->cp_hqd_pq_control = tmp;
3328
3329	/* set the wb address whether it's enabled or not */
3330	wb_gpu_addr = ring->rptr_gpu_addr;
3331	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3332	mqd->cp_hqd_pq_rptr_report_addr_hi =
3333	upper_32_bits(wb_gpu_addr) & 0xffff;
3334
3335	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3336	wb_gpu_addr = ring->wptr_gpu_addr;
3337	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3338	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3339
3340	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3341	ring->wptr = 0;
3342	mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3343
3344	/* set the vmid for the queue */
3345	mqd->cp_hqd_vmid = 0;
3346
3347	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3348	tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3349	mqd->cp_hqd_persistent_state = tmp;
3350
3351	/* set MIN_IB_AVAIL_SIZE */
3352	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3353	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3354	mqd->cp_hqd_ib_control = tmp;
3355
3356	/* set static priority for a queue/ring */
3357	gfx_v9_0_mqd_set_priority(ring, mqd);
3358	mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
3359
3360	/* map_queues packet doesn't need activate the queue,
3361	* so only kiq need set this field.
3362	*/
3363	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3364	mqd->cp_hqd_active = 1;
3365
3366	return 0;
3367	}
3368
3369	static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3370	{
3371	struct amdgpu_device *adev = ring->adev;
3372	struct v9_mqd *mqd = ring->mqd_ptr;
3373	int j;
3374
3375	/* disable wptr polling */
3376	WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3377
3378	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3379	mqd->cp_hqd_eop_base_addr_lo);
3380	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3381	mqd->cp_hqd_eop_base_addr_hi);
3382
3383	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3384	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3385	mqd->cp_hqd_eop_control);
3386
3387	/* enable doorbell? */
3388	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3389	mqd->cp_hqd_pq_doorbell_control);
3390
3391	/* disable the queue if it's active */
3392	if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3393	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3394	for (j = 0; j < adev->usec_timeout; j++) {
3395	if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3396	break;
3397	udelay(1);
3398	}
3399	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3400	mqd->cp_hqd_dequeue_request);
3401	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3402	mqd->cp_hqd_pq_rptr);
3403	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3404	mqd->cp_hqd_pq_wptr_lo);
3405	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3406	mqd->cp_hqd_pq_wptr_hi);
3407	}
3408
3409	/* set the pointer to the MQD */
3410	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3411	mqd->cp_mqd_base_addr_lo);
3412	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3413	mqd->cp_mqd_base_addr_hi);
3414
3415	/* set MQD vmid to 0 */
3416	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3417	mqd->cp_mqd_control);
3418
3419	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3420	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3421	mqd->cp_hqd_pq_base_lo);
3422	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3423	mqd->cp_hqd_pq_base_hi);
3424
3425	/* set up the HQD, this is similar to CP_RB0_CNTL */
3426	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3427	mqd->cp_hqd_pq_control);
3428
3429	/* set the wb address whether it's enabled or not */
3430	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3431	mqd->cp_hqd_pq_rptr_report_addr_lo);
3432	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3433	mqd->cp_hqd_pq_rptr_report_addr_hi);
3434
3435	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3436	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3437	mqd->cp_hqd_pq_wptr_poll_addr_lo);
3438	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3439	mqd->cp_hqd_pq_wptr_poll_addr_hi);
3440
3441	/* enable the doorbell if requested */
3442	if (ring->use_doorbell) {
3443	WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3444	(adev->doorbell_index.kiq * 2) << 2);
3445	/* If GC has entered CGPG, ringing doorbell > first page
3446	* doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to
3447	* workaround this issue. And this change has to align with firmware
3448	* update.
3449	*/
3450	if (check_if_enlarge_doorbell_range(adev))
3451	WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3452	(adev->doorbell.size - 4));
3453	else
3454	WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3455	(adev->doorbell_index.userqueue_end * 2) << 2);
3456	}
3457
3458	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3459	mqd->cp_hqd_pq_doorbell_control);
3460
3461	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3462	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3463	mqd->cp_hqd_pq_wptr_lo);
3464	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3465	mqd->cp_hqd_pq_wptr_hi);
3466
3467	/* set the vmid for the queue */
3468	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3469
3470	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3471	mqd->cp_hqd_persistent_state);
3472
3473	/* activate the queue */
3474	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3475	mqd->cp_hqd_active);
3476
3477	if (ring->use_doorbell)
3478	WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3479
3480	return 0;
3481	}
3482
3483	static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3484	{
3485	struct amdgpu_device *adev = ring->adev;
3486	int j;
3487
3488	/* disable the queue if it's active */
3489	if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3490
3491	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3492
3493	for (j = 0; j < adev->usec_timeout; j++) {
3494	if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3495	break;
3496	udelay(1);
3497	}
3498
3499	if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3500	DRM_DEBUG("KIQ dequeue request failed.\n");
3501
3502	/* Manual disable if dequeue request times out */
3503	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3504	}
3505
3506	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3507	0);
3508	}
3509
3510	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3511	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3512	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3513	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3514	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3515	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3516	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3517	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3518
3519	return 0;
3520	}
3521
3522	static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3523	{
3524	struct amdgpu_device *adev = ring->adev;
3525	struct v9_mqd *mqd = ring->mqd_ptr;
3526	struct v9_mqd *tmp_mqd;
3527
3528	gfx_v9_0_kiq_setting(ring);
3529
3530	/* GPU could be in bad state during probe, driver trigger the reset
3531	* after load the SMU, in this case , the mqd is not be initialized.
3532	* driver need to re-init the mqd.
3533	* check mqd->cp_hqd_pq_control since this value should not be 0
3534	*/
3535	tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup;
3536	if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3537	/* for GPU_RESET case , reset MQD to a clean status */
3538	if (adev->gfx.kiq[0].mqd_backup)
3539	memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation));
3540
3541	/* reset ring buffer */
3542	ring->wptr = 0;
3543	amdgpu_ring_clear_ring(ring);
3544
3545	mutex_lock(&adev->srbm_mutex);
3546	soc15_grbm_select(adev, me: ring->me, pipe: ring->pipe, queue: ring->queue, vmid: 0, xcc_id: 0);
3547	gfx_v9_0_kiq_init_register(ring);
3548	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
3549	mutex_unlock(lock: &adev->srbm_mutex);
3550	} else {
3551	memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3552	((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3553	((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3554	if (amdgpu_sriov_vf(adev) && adev->in_suspend)
3555	amdgpu_ring_clear_ring(ring);
3556	mutex_lock(&adev->srbm_mutex);
3557	soc15_grbm_select(adev, me: ring->me, pipe: ring->pipe, queue: ring->queue, vmid: 0, xcc_id: 0);
3558	gfx_v9_0_mqd_init(ring);
3559	gfx_v9_0_kiq_init_register(ring);
3560	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
3561	mutex_unlock(lock: &adev->srbm_mutex);
3562
3563	if (adev->gfx.kiq[0].mqd_backup)
3564	memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation));
3565	}
3566
3567	return 0;
3568	}
3569
3570	static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3571	{
3572	struct amdgpu_device *adev = ring->adev;
3573	struct v9_mqd *mqd = ring->mqd_ptr;
3574	int mqd_idx = ring - &adev->gfx.compute_ring[0];
3575	struct v9_mqd *tmp_mqd;
3576
3577	/* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3578	* is not be initialized before
3579	*/
3580	tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3581
3582	if (!tmp_mqd->cp_hqd_pq_control ||
3583	(!amdgpu_in_reset(adev) && !adev->in_suspend)) {
3584	memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3585	((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3586	((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3587	mutex_lock(&adev->srbm_mutex);
3588	soc15_grbm_select(adev, me: ring->me, pipe: ring->pipe, queue: ring->queue, vmid: 0, xcc_id: 0);
3589	gfx_v9_0_mqd_init(ring);
3590	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
3591	mutex_unlock(lock: &adev->srbm_mutex);
3592
3593	if (adev->gfx.mec.mqd_backup[mqd_idx])
3594	memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3595	} else {
3596	/* restore MQD to a clean status */
3597	if (adev->gfx.mec.mqd_backup[mqd_idx])
3598	memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3599	/* reset ring buffer */
3600	ring->wptr = 0;
3601	atomic64_set(v: (atomic64_t *)ring->wptr_cpu_addr, i: 0);
3602	amdgpu_ring_clear_ring(ring);
3603	}
3604
3605	return 0;
3606	}
3607
3608	static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3609	{
3610	struct amdgpu_ring *ring;
3611	int r;
3612
3613	ring = &adev->gfx.kiq[0].ring;
3614
3615	r = amdgpu_bo_reserve(bo: ring->mqd_obj, no_intr: false);
3616	if (unlikely(r != 0))
3617	return r;
3618
3619	r = amdgpu_bo_kmap(bo: ring->mqd_obj, ptr: (void **)&ring->mqd_ptr);
3620	if (unlikely(r != 0)) {
3621	amdgpu_bo_unreserve(bo: ring->mqd_obj);
3622	return r;
3623	}
3624
3625	gfx_v9_0_kiq_init_queue(ring);
3626	amdgpu_bo_kunmap(bo: ring->mqd_obj);
3627	ring->mqd_ptr = NULL;
3628	amdgpu_bo_unreserve(bo: ring->mqd_obj);
3629	return 0;
3630	}
3631
3632	static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3633	{
3634	struct amdgpu_ring *ring = NULL;
3635	int r = 0, i;
3636
3637	gfx_v9_0_cp_compute_enable(adev, enable: true);
3638
3639	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3640	ring = &adev->gfx.compute_ring[i];
3641
3642	r = amdgpu_bo_reserve(bo: ring->mqd_obj, no_intr: false);
3643	if (unlikely(r != 0))
3644	goto done;
3645	r = amdgpu_bo_kmap(bo: ring->mqd_obj, ptr: (void **)&ring->mqd_ptr);
3646	if (!r) {
3647	r = gfx_v9_0_kcq_init_queue(ring);
3648	amdgpu_bo_kunmap(bo: ring->mqd_obj);
3649	ring->mqd_ptr = NULL;
3650	}
3651	amdgpu_bo_unreserve(bo: ring->mqd_obj);
3652	if (r)
3653	goto done;
3654	}
3655
3656	r = amdgpu_gfx_enable_kcq(adev, xcc_id: 0);
3657	done:
3658	return r;
3659	}
3660
3661	static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3662	{
3663	int r, i;
3664	struct amdgpu_ring *ring;
3665
3666	if (!(adev->flags & AMD_IS_APU))
3667	gfx_v9_0_enable_gui_idle_interrupt(adev, enable: false);
3668
3669	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3670	if (adev->gfx.num_gfx_rings) {
3671	/* legacy firmware loading */
3672	r = gfx_v9_0_cp_gfx_load_microcode(adev);
3673	if (r)
3674	return r;
3675	}
3676
3677	r = gfx_v9_0_cp_compute_load_microcode(adev);
3678	if (r)
3679	return r;
3680	}
3681
3682	r = gfx_v9_0_kiq_resume(adev);
3683	if (r)
3684	return r;
3685
3686	if (adev->gfx.num_gfx_rings) {
3687	r = gfx_v9_0_cp_gfx_resume(adev);
3688	if (r)
3689	return r;
3690	}
3691
3692	r = gfx_v9_0_kcq_resume(adev);
3693	if (r)
3694	return r;
3695
3696	if (adev->gfx.num_gfx_rings) {
3697	ring = &adev->gfx.gfx_ring[0];
3698	r = amdgpu_ring_test_helper(ring);
3699	if (r)
3700	return r;
3701	}
3702
3703	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3704	ring = &adev->gfx.compute_ring[i];
3705	amdgpu_ring_test_helper(ring);
3706	}
3707
3708	gfx_v9_0_enable_gui_idle_interrupt(adev, enable: true);
3709
3710	return 0;
3711	}
3712
3713	static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3714	{
3715	u32 tmp;
3716
3717	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 1) &&
3718	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 2))
3719	return;
3720
3721	tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3722	tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3723	adev->df.hash_status.hash_64k);
3724	tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3725	adev->df.hash_status.hash_2m);
3726	tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3727	adev->df.hash_status.hash_1g);
3728	WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3729	}
3730
3731	static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3732	{
3733	if (adev->gfx.num_gfx_rings)
3734	gfx_v9_0_cp_gfx_enable(adev, enable);
3735	gfx_v9_0_cp_compute_enable(adev, enable);
3736	}
3737
3738	static int gfx_v9_0_hw_init(void *handle)
3739	{
3740	int r;
3741	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3742
3743	if (!amdgpu_sriov_vf(adev))
3744	gfx_v9_0_init_golden_registers(adev);
3745
3746	gfx_v9_0_constants_init(adev);
3747
3748	gfx_v9_0_init_tcp_config(adev);
3749
3750	r = adev->gfx.rlc.funcs->resume(adev);
3751	if (r)
3752	return r;
3753
3754	r = gfx_v9_0_cp_resume(adev);
3755	if (r)
3756	return r;
3757
3758	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2))
3759	gfx_v9_4_2_set_power_brake_sequence(adev);
3760
3761	return r;
3762	}
3763
3764	static int gfx_v9_0_hw_fini(void *handle)
3765	{
3766	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3767
3768	if (amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
3769	amdgpu_irq_put(adev, src: &adev->gfx.cp_ecc_error_irq, type: 0);
3770	amdgpu_irq_put(adev, src: &adev->gfx.priv_reg_irq, type: 0);
3771	amdgpu_irq_put(adev, src: &adev->gfx.priv_inst_irq, type: 0);
3772
3773	/* DF freeze and kcq disable will fail */
3774	if (!amdgpu_ras_intr_triggered())
3775	/* disable KCQ to avoid CPC touch memory not valid anymore */
3776	amdgpu_gfx_disable_kcq(adev, xcc_id: 0);
3777
3778	if (amdgpu_sriov_vf(adev)) {
3779	gfx_v9_0_cp_gfx_enable(adev, enable: false);
3780	/* must disable polling for SRIOV when hw finished, otherwise
3781	* CPC engine may still keep fetching WB address which is already
3782	* invalid after sw finished and trigger DMAR reading error in
3783	* hypervisor side.
3784	*/
3785	WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3786	return 0;
3787	}
3788
3789	/* Use deinitialize sequence from CAIL when unbinding device from driver,
3790	* otherwise KIQ is hanging when binding back
3791	*/
3792	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3793	mutex_lock(&adev->srbm_mutex);
3794	soc15_grbm_select(adev, me: adev->gfx.kiq[0].ring.me,
3795	pipe: adev->gfx.kiq[0].ring.pipe,
3796	queue: adev->gfx.kiq[0].ring.queue, vmid: 0, xcc_id: 0);
3797	gfx_v9_0_kiq_fini_register(ring: &adev->gfx.kiq[0].ring);
3798	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
3799	mutex_unlock(lock: &adev->srbm_mutex);
3800	}
3801
3802	gfx_v9_0_cp_enable(adev, enable: false);
3803
3804	/* Skip stopping RLC with A+A reset or when RLC controls GFX clock */
3805	if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) ||
3806	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >= IP_VERSION(9, 4, 2))) {
3807	dev_dbg(adev->dev, "Skipping RLC halt\n");
3808	return 0;
3809	}
3810
3811	adev->gfx.rlc.funcs->stop(adev);
3812	return 0;
3813	}
3814
3815	static int gfx_v9_0_suspend(void *handle)
3816	{
3817	return gfx_v9_0_hw_fini(handle);
3818	}
3819
3820	static int gfx_v9_0_resume(void *handle)
3821	{
3822	return gfx_v9_0_hw_init(handle);
3823	}
3824
3825	static bool gfx_v9_0_is_idle(void *handle)
3826	{
3827	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3828
3829	if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
3830	GRBM_STATUS, GUI_ACTIVE))
3831	return false;
3832	else
3833	return true;
3834	}
3835
3836	static int gfx_v9_0_wait_for_idle(void *handle)
3837	{
3838	unsigned i;
3839	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3840
3841	for (i = 0; i < adev->usec_timeout; i++) {
3842	if (gfx_v9_0_is_idle(handle))
3843	return 0;
3844	udelay(1);
3845	}
3846	return -ETIMEDOUT;
3847	}
3848
3849	static int gfx_v9_0_soft_reset(void *handle)
3850	{
3851	u32 grbm_soft_reset = 0;
3852	u32 tmp;
3853	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3854
3855	/* GRBM_STATUS */
3856	tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
3857	if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
3858	GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
3859	GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
3860	GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
3861	GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
3862	GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
3863	grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3864	GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
3865	grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3866	GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
3867	}
3868
3869	if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
3870	grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3871	GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
3872	}
3873
3874	/* GRBM_STATUS2 */
3875	tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
3876	if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
3877	grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3878	GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3879
3880
3881	if (grbm_soft_reset) {
3882	/* stop the rlc */
3883	adev->gfx.rlc.funcs->stop(adev);
3884
3885	if (adev->gfx.num_gfx_rings)
3886	/* Disable GFX parsing/prefetching */
3887	gfx_v9_0_cp_gfx_enable(adev, enable: false);
3888
3889	/* Disable MEC parsing/prefetching */
3890	gfx_v9_0_cp_compute_enable(adev, enable: false);
3891
3892	if (grbm_soft_reset) {
3893	tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3894	tmp |= grbm_soft_reset;
3895	dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3896	WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
3897	tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3898
3899	udelay(50);
3900
3901	tmp &= ~grbm_soft_reset;
3902	WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
3903	tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3904	}
3905
3906	/* Wait a little for things to settle down */
3907	udelay(50);
3908	}
3909	return 0;
3910	}
3911
3912	static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
3913	{
3914	signed long r, cnt = 0;
3915	unsigned long flags;
3916	uint32_t seq, reg_val_offs = 0;
3917	uint64_t value = 0;
3918	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
3919	struct amdgpu_ring *ring = &kiq->ring;
3920
3921	BUG_ON(!ring->funcs->emit_rreg);
3922
3923	spin_lock_irqsave(&kiq->ring_lock, flags);
3924	if (amdgpu_device_wb_get(adev, wb: &reg_val_offs)) {
3925	pr_err("critical bug! too many kiq readers\n");
3926	goto failed_unlock;
3927	}
3928	amdgpu_ring_alloc(ring, ndw: 32);
3929	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
3930	amdgpu_ring_write(ring, v: 9 | /* src: register*/
3931	(5 << 8) | /* dst: memory */
3932	(1 << 16) | /* count sel */
3933	(1 << 20)); /* write confirm */
3934	amdgpu_ring_write(ring, v: 0);
3935	amdgpu_ring_write(ring, v: 0);
3936	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
3937	reg_val_offs * 4));
3938	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
3939	reg_val_offs * 4));
3940	r = amdgpu_fence_emit_polling(ring, s: &seq, MAX_KIQ_REG_WAIT);
3941	if (r)
3942	goto failed_undo;
3943
3944	amdgpu_ring_commit(ring);
3945	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
3946
3947	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
3948
3949	/* don't wait anymore for gpu reset case because this way may
3950	* block gpu_recover() routine forever, e.g. this virt_kiq_rreg
3951	* is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
3952	* never return if we keep waiting in virt_kiq_rreg, which cause
3953	* gpu_recover() hang there.
3954	*
3955	* also don't wait anymore for IRQ context
3956	* */
3957	if (r < 1 && (amdgpu_in_reset(adev)))
3958	goto failed_kiq_read;
3959
3960	might_sleep();
3961	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
3962	msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
3963	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
3964	}
3965
3966	if (cnt > MAX_KIQ_REG_TRY)
3967	goto failed_kiq_read;
3968
3969	mb();
3970	value = (uint64_t)adev->wb.wb[reg_val_offs] |
3971	(uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
3972	amdgpu_device_wb_free(adev, wb: reg_val_offs);
3973	return value;
3974
3975	failed_undo:
3976	amdgpu_ring_undo(ring);
3977	failed_unlock:
3978	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
3979	failed_kiq_read:
3980	if (reg_val_offs)
3981	amdgpu_device_wb_free(adev, wb: reg_val_offs);
3982	pr_err("failed to read gpu clock\n");
3983	return ~0;
3984	}
3985
3986	static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
3987	{
3988	uint64_t clock, clock_lo, clock_hi, hi_check;
3989
3990	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
3991	case IP_VERSION(9, 3, 0):
3992	preempt_disable();
3993	clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
3994	clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
3995	hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
3996	/* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
3997	* roughly every 42 seconds.
3998	*/
3999	if (hi_check != clock_hi) {
4000	clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4001	clock_hi = hi_check;
4002	}
4003	preempt_enable();
4004	clock = clock_lo | (clock_hi << 32ULL);
4005	break;
4006	default:
4007	amdgpu_gfx_off_ctrl(adev, enable: false);
4008	mutex_lock(&adev->gfx.gpu_clock_mutex);
4009	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) ==
4010	IP_VERSION(9, 0, 1) &&
4011	amdgpu_sriov_runtime(adev)) {
4012	clock = gfx_v9_0_kiq_read_clock(adev);
4013	} else {
4014	WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4015	clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4016	((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4017	}
4018	mutex_unlock(lock: &adev->gfx.gpu_clock_mutex);
4019	amdgpu_gfx_off_ctrl(adev, enable: true);
4020	break;
4021	}
4022	return clock;
4023	}
4024
4025	static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4026	uint32_t vmid,
4027	uint32_t gds_base, uint32_t gds_size,
4028	uint32_t gws_base, uint32_t gws_size,
4029	uint32_t oa_base, uint32_t oa_size)
4030	{
4031	struct amdgpu_device *adev = ring->adev;
4032
4033	/* GDS Base */
4034	gfx_v9_0_write_data_to_reg(ring, eng_sel: 0, wc: false,
4035	SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4036	val: gds_base);
4037
4038	/* GDS Size */
4039	gfx_v9_0_write_data_to_reg(ring, eng_sel: 0, wc: false,
4040	SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4041	val: gds_size);
4042
4043	/* GWS */
4044	gfx_v9_0_write_data_to_reg(ring, eng_sel: 0, wc: false,
4045	SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4046	val: gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4047
4048	/* OA */
4049	gfx_v9_0_write_data_to_reg(ring, eng_sel: 0, wc: false,
4050	SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4051	val: (1 << (oa_size + oa_base)) - (1 << oa_base));
4052	}
4053
4054	static const u32 vgpr_init_compute_shader[] =
4055	{
4056	0xb07c0000, 0xbe8000ff,
4057	0x000000f8, 0xbf110800,
4058	0x7e000280, 0x7e020280,
4059	0x7e040280, 0x7e060280,
4060	0x7e080280, 0x7e0a0280,
4061	0x7e0c0280, 0x7e0e0280,
4062	0x80808800, 0xbe803200,
4063	0xbf84fff5, 0xbf9c0000,
4064	0xd28c0001, 0x0001007f,
4065	0xd28d0001, 0x0002027e,
4066	0x10020288, 0xb8810904,
4067	0xb7814000, 0xd1196a01,
4068	0x00000301, 0xbe800087,
4069	0xbefc00c1, 0xd89c4000,
4070	0x00020201, 0xd89cc080,
4071	0x00040401, 0x320202ff,
4072	0x00000800, 0x80808100,
4073	0xbf84fff8, 0x7e020280,
4074	0xbf810000, 0x00000000,
4075	};
4076
4077	static const u32 sgpr_init_compute_shader[] =
4078	{
4079	0xb07c0000, 0xbe8000ff,
4080	0x0000005f, 0xbee50080,
4081	0xbe812c65, 0xbe822c65,
4082	0xbe832c65, 0xbe842c65,
4083	0xbe852c65, 0xb77c0005,
4084	0x80808500, 0xbf84fff8,
4085	0xbe800080, 0xbf810000,
4086	};
4087
4088	static const u32 vgpr_init_compute_shader_arcturus[] = {
4089	0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4090	0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4091	0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4092	0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4093	0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4094	0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4095	0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4096	0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4097	0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4098	0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4099	0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4100	0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4101	0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4102	0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4103	0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4104	0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4105	0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4106	0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4107	0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4108	0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4109	0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4110	0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4111	0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4112	0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4113	0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4114	0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4115	0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4116	0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4117	0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4118	0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4119	0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4120	0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4121	0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4122	0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4123	0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4124	0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4125	0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4126	0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4127	0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4128	0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4129	0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4130	0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4131	0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4132	0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4133	0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4134	0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4135	0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4136	0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4137	0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4138	0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4139	0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4140	0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4141	0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4142	0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4143	0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4144	0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4145	0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4146	0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4147	0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4148	0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4149	0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4150	0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4151	0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4152	0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4153	0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4154	0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4155	0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4156	0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4157	0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4158	0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4159	0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4160	0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4161	0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4162	0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4163	0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4164	0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4165	0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4166	0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4167	0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4168	0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4169	0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4170	0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4171	0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4172	0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4173	0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4174	0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4175	0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4176	0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4177	0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4178	0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4179	0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4180	0xbf84fff8, 0xbf810000,
4181	};
4182
4183	/* When below register arrays changed, please update gpr_reg_size,
4184	and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4185	to cover all gfx9 ASICs */
4186	static const struct soc15_reg_entry vgpr_init_regs[] = {
4187	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4188	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4189	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4190	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4191	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4192	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4193	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4194	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4195	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4196	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4197	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4198	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4199	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4200	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4201	};
4202
4203	static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4204	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4205	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4206	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4207	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4208	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4209	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4210	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4211	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4212	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4213	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4214	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4215	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4216	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4217	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4218	};
4219
4220	static const struct soc15_reg_entry sgpr1_init_regs[] = {
4221	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4222	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4223	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4224	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4225	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4226	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4227	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4228	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4229	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4230	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4231	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4232	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4233	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4234	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4235	};
4236
4237	static const struct soc15_reg_entry sgpr2_init_regs[] = {
4238	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4239	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4240	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4241	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4242	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4243	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4244	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4245	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4246	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4247	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4248	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4249	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4250	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4251	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4252	};
4253
4254	static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4255	{ SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4256	{ SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4257	{ SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4258	{ SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4259	{ SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4260	{ SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4261	{ SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4262	{ SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4263	{ SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4264	{ SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4265	{ SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4266	{ SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4267	{ SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4268	{ SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4269	{ SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4270	{ SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4271	{ SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4272	{ SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4273	{ SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4274	{ SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4275	{ SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4276	{ SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4277	{ SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4278	{ SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4279	{ SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4280	{ SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4281	{ SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4282	{ SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4283	{ SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4284	{ SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4285	{ SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4286	{ SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4287	{ SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4288	};
4289
4290	static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4291	{
4292	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4293	int i, r;
4294
4295	/* only support when RAS is enabled */
4296	if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
4297	return 0;
4298
4299	r = amdgpu_ring_alloc(ring, ndw: 7);
4300	if (r) {
4301	DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4302	ring->name, r);
4303	return r;
4304	}
4305
4306	WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4307	WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4308
4309	amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4310	amdgpu_ring_write(ring, v: (PACKET3_DMA_DATA_CP_SYNC |
4311	PACKET3_DMA_DATA_DST_SEL(1) |
4312	PACKET3_DMA_DATA_SRC_SEL(2) |
4313	PACKET3_DMA_DATA_ENGINE(0)));
4314	amdgpu_ring_write(ring, v: 0);
4315	amdgpu_ring_write(ring, v: 0);
4316	amdgpu_ring_write(ring, v: 0);
4317	amdgpu_ring_write(ring, v: 0);
4318	amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4319	adev->gds.gds_size);
4320
4321	amdgpu_ring_commit(ring);
4322
4323	for (i = 0; i < adev->usec_timeout; i++) {
4324	if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4325	break;
4326	udelay(1);
4327	}
4328
4329	if (i >= adev->usec_timeout)
4330	r = -ETIMEDOUT;
4331
4332	WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4333
4334	return r;
4335	}
4336
4337	static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4338	{
4339	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4340	struct amdgpu_ib ib;
4341	struct dma_fence *f = NULL;
4342	int r, i;
4343	unsigned total_size, vgpr_offset, sgpr_offset;
4344	u64 gpu_addr;
4345
4346	int compute_dim_x = adev->gfx.config.max_shader_engines *
4347	adev->gfx.config.max_cu_per_sh *
4348	adev->gfx.config.max_sh_per_se;
4349	int sgpr_work_group_size = 5;
4350	int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4351	int vgpr_init_shader_size;
4352	const u32 *vgpr_init_shader_ptr;
4353	const struct soc15_reg_entry *vgpr_init_regs_ptr;
4354
4355	/* only support when RAS is enabled */
4356	if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
4357	return 0;
4358
4359	/* bail if the compute ring is not ready */
4360	if (!ring->sched.ready)
4361	return 0;
4362
4363	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1)) {
4364	vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4365	vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4366	vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4367	} else {
4368	vgpr_init_shader_ptr = vgpr_init_compute_shader;
4369	vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4370	vgpr_init_regs_ptr = vgpr_init_regs;
4371	}
4372
4373	total_size =
4374	(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4375	total_size +=
4376	(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4377	total_size +=
4378	(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4379	total_size = ALIGN(total_size, 256);
4380	vgpr_offset = total_size;
4381	total_size += ALIGN(vgpr_init_shader_size, 256);
4382	sgpr_offset = total_size;
4383	total_size += sizeof(sgpr_init_compute_shader);
4384
4385	/* allocate an indirect buffer to put the commands in */
4386	memset(&ib, 0, sizeof(ib));
4387	r = amdgpu_ib_get(adev, NULL, size: total_size,
4388	pool: AMDGPU_IB_POOL_DIRECT, ib: &ib);
4389	if (r) {
4390	DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4391	return r;
4392	}
4393
4394	/* load the compute shaders */
4395	for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4396	ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4397
4398	for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4399	ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4400
4401	/* init the ib length to 0 */
4402	ib.length_dw = 0;
4403
4404	/* VGPR */
4405	/* write the register state for the compute dispatch */
4406	for (i = 0; i < gpr_reg_size; i++) {
4407	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4408	ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4409	- PACKET3_SET_SH_REG_START;
4410	ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4411	}
4412	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4413	gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4414	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4415	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4416	- PACKET3_SET_SH_REG_START;
4417	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4418	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4419
4420	/* write dispatch packet */
4421	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4422	ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4423	ib.ptr[ib.length_dw++] = 1; /* y */
4424	ib.ptr[ib.length_dw++] = 1; /* z */
4425	ib.ptr[ib.length_dw++] =
4426	REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4427
4428	/* write CS partial flush packet */
4429	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4430	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4431
4432	/* SGPR1 */
4433	/* write the register state for the compute dispatch */
4434	for (i = 0; i < gpr_reg_size; i++) {
4435	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4436	ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4437	- PACKET3_SET_SH_REG_START;
4438	ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4439	}
4440	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4441	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4442	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4443	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4444	- PACKET3_SET_SH_REG_START;
4445	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4446	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4447
4448	/* write dispatch packet */
4449	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4450	ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4451	ib.ptr[ib.length_dw++] = 1; /* y */
4452	ib.ptr[ib.length_dw++] = 1; /* z */
4453	ib.ptr[ib.length_dw++] =
4454	REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4455
4456	/* write CS partial flush packet */
4457	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4458	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4459
4460	/* SGPR2 */
4461	/* write the register state for the compute dispatch */
4462	for (i = 0; i < gpr_reg_size; i++) {
4463	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4464	ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4465	- PACKET3_SET_SH_REG_START;
4466	ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4467	}
4468	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4469	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4470	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4471	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4472	- PACKET3_SET_SH_REG_START;
4473	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4474	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4475
4476	/* write dispatch packet */
4477	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4478	ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4479	ib.ptr[ib.length_dw++] = 1; /* y */
4480	ib.ptr[ib.length_dw++] = 1; /* z */
4481	ib.ptr[ib.length_dw++] =
4482	REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4483
4484	/* write CS partial flush packet */
4485	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4486	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4487
4488	/* shedule the ib on the ring */
4489	r = amdgpu_ib_schedule(ring, num_ibs: 1, ibs: &ib, NULL, f: &f);
4490	if (r) {
4491	DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4492	goto fail;
4493	}
4494
4495	/* wait for the GPU to finish processing the IB */
4496	r = dma_fence_wait(fence: f, intr: false);
4497	if (r) {
4498	DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4499	goto fail;
4500	}
4501
4502	fail:
4503	amdgpu_ib_free(adev, ib: &ib, NULL);
4504	dma_fence_put(fence: f);
4505
4506	return r;
4507	}
4508
4509	static int gfx_v9_0_early_init(void *handle)
4510	{
4511	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4512
4513	adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
4514
4515	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1) ||
4516	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2))
4517	adev->gfx.num_gfx_rings = 0;
4518	else
4519	adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4520	adev->gfx.xcc_mask = 1;
4521	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4522	AMDGPU_MAX_COMPUTE_RINGS);
4523	gfx_v9_0_set_kiq_pm4_funcs(adev);
4524	gfx_v9_0_set_ring_funcs(adev);
4525	gfx_v9_0_set_irq_funcs(adev);
4526	gfx_v9_0_set_gds_init(adev);
4527	gfx_v9_0_set_rlc_funcs(adev);
4528
4529	/* init rlcg reg access ctrl */
4530	gfx_v9_0_init_rlcg_reg_access_ctrl(adev);
4531
4532	return gfx_v9_0_init_microcode(adev);
4533	}
4534
4535	static int gfx_v9_0_ecc_late_init(void *handle)
4536	{
4537	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4538	int r;
4539
4540	/*
4541	* Temp workaround to fix the issue that CP firmware fails to
4542	* update read pointer when CPDMA is writing clearing operation
4543	* to GDS in suspend/resume sequence on several cards. So just
4544	* limit this operation in cold boot sequence.
4545	*/
4546	if ((!adev->in_suspend) &&
4547	(adev->gds.gds_size)) {
4548	r = gfx_v9_0_do_edc_gds_workarounds(adev);
4549	if (r)
4550	return r;
4551	}
4552
4553	/* requires IBs so do in late init after IB pool is initialized */
4554	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2))
4555	r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
4556	else
4557	r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4558
4559	if (r)
4560	return r;
4561
4562	if (adev->gfx.ras &&
4563	adev->gfx.ras->enable_watchdog_timer)
4564	adev->gfx.ras->enable_watchdog_timer(adev);
4565
4566	return 0;
4567	}
4568
4569	static int gfx_v9_0_late_init(void *handle)
4570	{
4571	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4572	int r;
4573
4574	r = amdgpu_irq_get(adev, src: &adev->gfx.priv_reg_irq, type: 0);
4575	if (r)
4576	return r;
4577
4578	r = amdgpu_irq_get(adev, src: &adev->gfx.priv_inst_irq, type: 0);
4579	if (r)
4580	return r;
4581
4582	r = gfx_v9_0_ecc_late_init(handle);
4583	if (r)
4584	return r;
4585
4586	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2))
4587	gfx_v9_4_2_debug_trap_config_init(adev,
4588	first_vmid: adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4589	else
4590	gfx_v9_0_debug_trap_config_init(adev,
4591	first_vmid: adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4592
4593	return 0;
4594	}
4595
4596	static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4597	{
4598	uint32_t rlc_setting;
4599
4600	/* if RLC is not enabled, do nothing */
4601	rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4602	if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4603	return false;
4604
4605	return true;
4606	}
4607
4608	static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
4609	{
4610	uint32_t data;
4611	unsigned i;
4612
4613	data = RLC_SAFE_MODE__CMD_MASK;
4614	data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4615	WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4616
4617	/* wait for RLC_SAFE_MODE */
4618	for (i = 0; i < adev->usec_timeout; i++) {
4619	if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4620	break;
4621	udelay(1);
4622	}
4623	}
4624
4625	static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
4626	{
4627	uint32_t data;
4628
4629	data = RLC_SAFE_MODE__CMD_MASK;
4630	WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4631	}
4632
4633	static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4634	bool enable)
4635	{
4636	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id: 0);
4637
4638	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4639	gfx_v9_0_enable_gfx_cg_power_gating(adev, enable: true);
4640	if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4641	gfx_v9_0_enable_gfx_pipeline_powergating(adev, enable: true);
4642	} else {
4643	gfx_v9_0_enable_gfx_cg_power_gating(adev, enable: false);
4644	if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4645	gfx_v9_0_enable_gfx_pipeline_powergating(adev, enable: false);
4646	}
4647
4648	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id: 0);
4649	}
4650
4651	static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4652	bool enable)
4653	{
4654	/* TODO: double check if we need to perform under safe mode */
4655	/* gfx_v9_0_enter_rlc_safe_mode(adev); */
4656
4657	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4658	gfx_v9_0_enable_gfx_static_mg_power_gating(adev, enable: true);
4659	else
4660	gfx_v9_0_enable_gfx_static_mg_power_gating(adev, enable: false);
4661
4662	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4663	gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, enable: true);
4664	else
4665	gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, enable: false);
4666
4667	/* gfx_v9_0_exit_rlc_safe_mode(adev); */
4668	}
4669
4670	static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4671	bool enable)
4672	{
4673	uint32_t data, def;
4674
4675	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id: 0);
4676
4677	/* It is disabled by HW by default */
4678	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4679	/* 1 - RLC_CGTT_MGCG_OVERRIDE */
4680	def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4681
4682	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 2, 1))
4683	data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4684
4685	data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4686	RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4687	RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4688
4689	/* only for Vega10 & Raven1 */
4690	data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4691
4692	if (def != data)
4693	WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4694
4695	/* MGLS is a global flag to control all MGLS in GFX */
4696	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4697	/* 2 - RLC memory Light sleep */
4698	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4699	def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4700	data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4701	if (def != data)
4702	WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4703	}
4704	/* 3 - CP memory Light sleep */
4705	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4706	def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4707	data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4708	if (def != data)
4709	WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4710	}
4711	}
4712	} else {
4713	/* 1 - MGCG_OVERRIDE */
4714	def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4715
4716	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 2, 1))
4717	data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4718
4719	data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4720	RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4721	RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4722	RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4723
4724	if (def != data)
4725	WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4726
4727	/* 2 - disable MGLS in RLC */
4728	data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4729	if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4730	data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4731	WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4732	}
4733
4734	/* 3 - disable MGLS in CP */
4735	data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4736	if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4737	data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4738	WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4739	}
4740	}
4741
4742	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id: 0);
4743	}
4744
4745	static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4746	bool enable)
4747	{
4748	uint32_t data, def;
4749
4750	if (!adev->gfx.num_gfx_rings)
4751	return;
4752
4753	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id: 0);
4754
4755	/* Enable 3D CGCG/CGLS */
4756	if (enable) {
4757	/* write cmd to clear cgcg/cgls ov */
4758	def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4759	/* unset CGCG override */
4760	data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4761	/* update CGCG and CGLS override bits */
4762	if (def != data)
4763	WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4764
4765	/* enable 3Dcgcg FSM(0x0000363f) */
4766	def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4767
4768	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
4769	data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4770	RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4771	else
4772	data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;
4773
4774	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4775	data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4776	RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4777	if (def != data)
4778	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4779
4780	/* set IDLE_POLL_COUNT(0x00900100) */
4781	def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4782	data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4783	(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4784	if (def != data)
4785	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4786	} else {
4787	/* Disable CGCG/CGLS */
4788	def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4789	/* disable cgcg, cgls should be disabled */
4790	data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
4791	RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
4792	/* disable cgcg and cgls in FSM */
4793	if (def != data)
4794	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4795	}
4796
4797	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id: 0);
4798	}
4799
4800	static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4801	bool enable)
4802	{
4803	uint32_t def, data;
4804
4805	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id: 0);
4806
4807	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
4808	def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4809	/* unset CGCG override */
4810	data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4811	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4812	data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4813	else
4814	data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4815	/* update CGCG and CGLS override bits */
4816	if (def != data)
4817	WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4818
4819	/* enable cgcg FSM(0x0000363F) */
4820	def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4821
4822	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1))
4823	data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4824	RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4825	else
4826	data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4827	RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4828	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4829	data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4830	RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
4831	if (def != data)
4832	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4833
4834	/* set IDLE_POLL_COUNT(0x00900100) */
4835	def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4836	data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4837	(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4838	if (def != data)
4839	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4840	} else {
4841	def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4842	/* reset CGCG/CGLS bits */
4843	data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
4844	/* disable cgcg and cgls in FSM */
4845	if (def != data)
4846	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4847	}
4848
4849	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id: 0);
4850	}
4851
4852	static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
4853	bool enable)
4854	{
4855	if (enable) {
4856	/* CGCG/CGLS should be enabled after MGCG/MGLS
4857	* === MGCG + MGLS ===
4858	*/
4859	gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4860	/* === CGCG /CGLS for GFX 3D Only === */
4861	gfx_v9_0_update_3d_clock_gating(adev, enable);
4862	/* === CGCG + CGLS === */
4863	gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4864	} else {
4865	/* CGCG/CGLS should be disabled before MGCG/MGLS
4866	* === CGCG + CGLS ===
4867	*/
4868	gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4869	/* === CGCG /CGLS for GFX 3D Only === */
4870	gfx_v9_0_update_3d_clock_gating(adev, enable);
4871	/* === MGCG + MGLS === */
4872	gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4873	}
4874	return 0;
4875	}
4876
4877	static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
4878	unsigned int vmid)
4879	{
4880	u32 reg, data;
4881
4882	reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
4883	if (amdgpu_sriov_is_pp_one_vf(adev))
4884	data = RREG32_NO_KIQ(reg);
4885	else
4886	data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL);
4887
4888	data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
4889	data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
4890
4891	if (amdgpu_sriov_is_pp_one_vf(adev))
4892	WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
4893	else
4894	WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
4895	}
4896
4897	static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned int vmid)
4898	{
4899	amdgpu_gfx_off_ctrl(adev, enable: false);
4900
4901	gfx_v9_0_update_spm_vmid_internal(adev, vmid);
4902
4903	amdgpu_gfx_off_ctrl(adev, enable: true);
4904	}
4905
4906	static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
4907	uint32_t offset,
4908	struct soc15_reg_rlcg *entries, int arr_size)
4909	{
4910	int i;
4911	uint32_t reg;
4912
4913	if (!entries)
4914	return false;
4915
4916	for (i = 0; i < arr_size; i++) {
4917	const struct soc15_reg_rlcg *entry;
4918
4919	entry = &entries[i];
4920	reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
4921	if (offset == reg)
4922	return true;
4923	}
4924
4925	return false;
4926	}
4927
4928	static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
4929	{
4930	return gfx_v9_0_check_rlcg_range(adev, offset,
4931	entries: (void *)rlcg_access_gc_9_0,
4932	ARRAY_SIZE(rlcg_access_gc_9_0));
4933	}
4934
4935	static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
4936	.is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
4937	.set_safe_mode = gfx_v9_0_set_safe_mode,
4938	.unset_safe_mode = gfx_v9_0_unset_safe_mode,
4939	.init = gfx_v9_0_rlc_init,
4940	.get_csb_size = gfx_v9_0_get_csb_size,
4941	.get_csb_buffer = gfx_v9_0_get_csb_buffer,
4942	.get_cp_table_num = gfx_v9_0_cp_jump_table_num,
4943	.resume = gfx_v9_0_rlc_resume,
4944	.stop = gfx_v9_0_rlc_stop,
4945	.reset = gfx_v9_0_rlc_reset,
4946	.start = gfx_v9_0_rlc_start,
4947	.update_spm_vmid = gfx_v9_0_update_spm_vmid,
4948	.is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
4949	};
4950
4951	static int gfx_v9_0_set_powergating_state(void *handle,
4952	enum amd_powergating_state state)
4953	{
4954	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4955	bool enable = (state == AMD_PG_STATE_GATE);
4956
4957	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
4958	case IP_VERSION(9, 2, 2):
4959	case IP_VERSION(9, 1, 0):
4960	case IP_VERSION(9, 3, 0):
4961	if (!enable)
4962	amdgpu_gfx_off_ctrl(adev, enable: false);
4963
4964	if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
4965	gfx_v9_0_enable_sck_slow_down_on_power_up(adev, enable: true);
4966	gfx_v9_0_enable_sck_slow_down_on_power_down(adev, enable: true);
4967	} else {
4968	gfx_v9_0_enable_sck_slow_down_on_power_up(adev, enable: false);
4969	gfx_v9_0_enable_sck_slow_down_on_power_down(adev, enable: false);
4970	}
4971
4972	if (adev->pg_flags & AMD_PG_SUPPORT_CP)
4973	gfx_v9_0_enable_cp_power_gating(adev, enable: true);
4974	else
4975	gfx_v9_0_enable_cp_power_gating(adev, enable: false);
4976
4977	/* update gfx cgpg state */
4978	gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
4979
4980	/* update mgcg state */
4981	gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
4982
4983	if (enable)
4984	amdgpu_gfx_off_ctrl(adev, enable: true);
4985	break;
4986	case IP_VERSION(9, 2, 1):
4987	amdgpu_gfx_off_ctrl(adev, enable);
4988	break;
4989	default:
4990	break;
4991	}
4992
4993	return 0;
4994	}
4995
4996	static int gfx_v9_0_set_clockgating_state(void *handle,
4997	enum amd_clockgating_state state)
4998	{
4999	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5000
5001	if (amdgpu_sriov_vf(adev))
5002	return 0;
5003
5004	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
5005	case IP_VERSION(9, 0, 1):
5006	case IP_VERSION(9, 2, 1):
5007	case IP_VERSION(9, 4, 0):
5008	case IP_VERSION(9, 2, 2):
5009	case IP_VERSION(9, 1, 0):
5010	case IP_VERSION(9, 4, 1):
5011	case IP_VERSION(9, 3, 0):
5012	case IP_VERSION(9, 4, 2):
5013	gfx_v9_0_update_gfx_clock_gating(adev,
5014	enable: state == AMD_CG_STATE_GATE);
5015	break;
5016	default:
5017	break;
5018	}
5019	return 0;
5020	}
5021
5022	static void gfx_v9_0_get_clockgating_state(void *handle, u64 *flags)
5023	{
5024	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5025	int data;
5026
5027	if (amdgpu_sriov_vf(adev))
5028	*flags = 0;
5029
5030	/* AMD_CG_SUPPORT_GFX_MGCG */
5031	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5032	if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5033	*flags |= AMD_CG_SUPPORT_GFX_MGCG;
5034
5035	/* AMD_CG_SUPPORT_GFX_CGCG */
5036	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5037	if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5038	*flags |= AMD_CG_SUPPORT_GFX_CGCG;
5039
5040	/* AMD_CG_SUPPORT_GFX_CGLS */
5041	if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5042	*flags |= AMD_CG_SUPPORT_GFX_CGLS;
5043
5044	/* AMD_CG_SUPPORT_GFX_RLC_LS */
5045	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5046	if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5047	*flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5048
5049	/* AMD_CG_SUPPORT_GFX_CP_LS */
5050	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5051	if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5052	*flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5053
5054	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 1)) {
5055	/* AMD_CG_SUPPORT_GFX_3D_CGCG */
5056	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5057	if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5058	*flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5059
5060	/* AMD_CG_SUPPORT_GFX_3D_CGLS */
5061	if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5062	*flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5063	}
5064	}
5065
5066	static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5067	{
5068	return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/
5069	}
5070
5071	static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5072	{
5073	struct amdgpu_device *adev = ring->adev;
5074	u64 wptr;
5075
5076	/* XXX check if swapping is necessary on BE */
5077	if (ring->use_doorbell) {
5078	wptr = atomic64_read(v: (atomic64_t *)ring->wptr_cpu_addr);
5079	} else {
5080	wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5081	wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5082	}
5083
5084	return wptr;
5085	}
5086
5087	static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5088	{
5089	struct amdgpu_device *adev = ring->adev;
5090
5091	if (ring->use_doorbell) {
5092	/* XXX check if swapping is necessary on BE */
5093	atomic64_set(v: (atomic64_t *)ring->wptr_cpu_addr, i: ring->wptr);
5094	WDOORBELL64(ring->doorbell_index, ring->wptr);
5095	} else {
5096	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5097	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5098	}
5099	}
5100
5101	static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5102	{
5103	struct amdgpu_device *adev = ring->adev;
5104	u32 ref_and_mask, reg_mem_engine;
5105	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5106
5107	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5108	switch (ring->me) {
5109	case 1:
5110	ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5111	break;
5112	case 2:
5113	ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5114	break;
5115	default:
5116	return;
5117	}
5118	reg_mem_engine = 0;
5119	} else {
5120	ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5121	reg_mem_engine = 1; /* pfp */
5122	}
5123
5124	gfx_v9_0_wait_reg_mem(ring, eng_sel: reg_mem_engine, mem_space: 0, opt: 1,
5125	addr0: adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5126	addr1: adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5127	ref: ref_and_mask, mask: ref_and_mask, inv: 0x20);
5128	}
5129
5130	static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5131	struct amdgpu_job *job,
5132	struct amdgpu_ib *ib,
5133	uint32_t flags)
5134	{
5135	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5136	u32 header, control = 0;
5137
5138	if (ib->flags & AMDGPU_IB_FLAG_CE)
5139	header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5140	else
5141	header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5142
5143	control |= ib->length_dw | (vmid << 24);
5144
5145	if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
5146	control |= INDIRECT_BUFFER_PRE_ENB(1);
5147
5148	if (flags & AMDGPU_IB_PREEMPTED)
5149	control |= INDIRECT_BUFFER_PRE_RESUME(1);
5150
5151	if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5152	gfx_v9_0_ring_emit_de_meta(ring,
5153	resume: (!amdgpu_sriov_vf(ring->adev) &&
5154	flags & AMDGPU_IB_PREEMPTED) ?
5155	true : false,
5156	usegds: job->gds_size > 0 && job->gds_base != 0);
5157	}
5158
5159	amdgpu_ring_write(ring, v: header);
5160	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5161	amdgpu_ring_write(ring,
5162	#ifdef __BIG_ENDIAN
5163	(2 << 0) |
5164	#endif
5165	lower_32_bits(ib->gpu_addr));
5166	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5167	amdgpu_ring_ib_on_emit_cntl(ring);
5168	amdgpu_ring_write(ring, v: control);
5169	}
5170
5171	static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring,
5172	unsigned offset)
5173	{
5174	u32 control = ring->ring[offset];
5175
5176	control |= INDIRECT_BUFFER_PRE_RESUME(1);
5177	ring->ring[offset] = control;
5178	}
5179
5180	static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring,
5181	unsigned offset)
5182	{
5183	struct amdgpu_device *adev = ring->adev;
5184	void *ce_payload_cpu_addr;
5185	uint64_t payload_offset, payload_size;
5186
5187	payload_size = sizeof(struct v9_ce_ib_state);
5188
5189	if (ring->is_mes_queue) {
5190	payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5191	gfx[0].gfx_meta_data) +
5192	offsetof(struct v9_gfx_meta_data, ce_payload);
5193	ce_payload_cpu_addr =
5194	amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);
5195	} else {
5196	payload_offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5197	ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5198	}
5199
5200	if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5201	memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size);
5202	} else {
5203	memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr,
5204	(ring->buf_mask + 1 - offset) << 2);
5205	payload_size -= (ring->buf_mask + 1 - offset) << 2;
5206	memcpy((void *)&ring->ring[0],
5207	ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5208	payload_size);
5209	}
5210	}
5211
5212	static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring,
5213	unsigned offset)
5214	{
5215	struct amdgpu_device *adev = ring->adev;
5216	void *de_payload_cpu_addr;
5217	uint64_t payload_offset, payload_size;
5218
5219	payload_size = sizeof(struct v9_de_ib_state);
5220
5221	if (ring->is_mes_queue) {
5222	payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5223	gfx[0].gfx_meta_data) +
5224	offsetof(struct v9_gfx_meta_data, de_payload);
5225	de_payload_cpu_addr =
5226	amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);
5227	} else {
5228	payload_offset = offsetof(struct v9_gfx_meta_data, de_payload);
5229	de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5230	}
5231
5232	((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status =
5233	IB_COMPLETION_STATUS_PREEMPTED;
5234
5235	if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5236	memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size);
5237	} else {
5238	memcpy((void *)&ring->ring[offset], de_payload_cpu_addr,
5239	(ring->buf_mask + 1 - offset) << 2);
5240	payload_size -= (ring->buf_mask + 1 - offset) << 2;
5241	memcpy((void *)&ring->ring[0],
5242	de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5243	payload_size);
5244	}
5245	}
5246
5247	static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5248	struct amdgpu_job *job,
5249	struct amdgpu_ib *ib,
5250	uint32_t flags)
5251	{
5252	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5253	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5254
5255	/* Currently, there is a high possibility to get wave ID mismatch
5256	* between ME and GDS, leading to a hw deadlock, because ME generates
5257	* different wave IDs than the GDS expects. This situation happens
5258	* randomly when at least 5 compute pipes use GDS ordered append.
5259	* The wave IDs generated by ME are also wrong after suspend/resume.
5260	* Those are probably bugs somewhere else in the kernel driver.
5261	*
5262	* Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5263	* GDS to 0 for this ring (me/pipe).
5264	*/
5265	if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5266	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5267	amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5268	amdgpu_ring_write(ring, v: ring->adev->gds.gds_compute_max_wave_id);
5269	}
5270
5271	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5272	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5273	amdgpu_ring_write(ring,
5274	#ifdef __BIG_ENDIAN
5275	(2 << 0) |
5276	#endif
5277	lower_32_bits(ib->gpu_addr));
5278	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5279	amdgpu_ring_write(ring, v: control);
5280	}
5281
5282	static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5283	u64 seq, unsigned flags)
5284	{
5285	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5286	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5287	bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5288	bool exec = flags & AMDGPU_FENCE_FLAG_EXEC;
5289	uint32_t dw2 = 0;
5290
5291	/* RELEASE_MEM - flush caches, send int */
5292	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5293
5294	if (writeback) {
5295	dw2 = EOP_TC_NC_ACTION_EN;
5296	} else {
5297	dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN |
5298	EOP_TC_MD_ACTION_EN;
5299	}
5300	dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5301	EVENT_INDEX(5);
5302	if (exec)
5303	dw2 |= EOP_EXEC;
5304
5305	amdgpu_ring_write(ring, v: dw2);
5306	amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5307
5308	/*
5309	* the address should be Qword aligned if 64bit write, Dword
5310	* aligned if only send 32bit data low (discard data high)
5311	*/
5312	if (write64bit)
5313	BUG_ON(addr & 0x7);
5314	else
5315	BUG_ON(addr & 0x3);
5316	amdgpu_ring_write(ring, lower_32_bits(addr));
5317	amdgpu_ring_write(ring, upper_32_bits(addr));
5318	amdgpu_ring_write(ring, lower_32_bits(seq));
5319	amdgpu_ring_write(ring, upper_32_bits(seq));
5320	amdgpu_ring_write(ring, v: 0);
5321	}
5322
5323	static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5324	{
5325	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5326	uint32_t seq = ring->fence_drv.sync_seq;
5327	uint64_t addr = ring->fence_drv.gpu_addr;
5328
5329	gfx_v9_0_wait_reg_mem(ring, eng_sel: usepfp, mem_space: 1, opt: 0,
5330	lower_32_bits(addr), upper_32_bits(addr),
5331	ref: seq, mask: 0xffffffff, inv: 4);
5332	}
5333
5334	static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5335	unsigned vmid, uint64_t pd_addr)
5336	{
5337	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5338
5339	/* compute doesn't have PFP */
5340	if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5341	/* sync PFP to ME, otherwise we might get invalid PFP reads */
5342	amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5343	amdgpu_ring_write(ring, v: 0x0);
5344	}
5345	}
5346
5347	static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5348	{
5349	return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */
5350	}
5351
5352	static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5353	{
5354	u64 wptr;
5355
5356	/* XXX check if swapping is necessary on BE */
5357	if (ring->use_doorbell)
5358	wptr = atomic64_read(v: (atomic64_t *)ring->wptr_cpu_addr);
5359	else
5360	BUG();
5361	return wptr;
5362	}
5363
5364	static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5365	{
5366	struct amdgpu_device *adev = ring->adev;
5367
5368	/* XXX check if swapping is necessary on BE */
5369	if (ring->use_doorbell) {
5370	atomic64_set(v: (atomic64_t *)ring->wptr_cpu_addr, i: ring->wptr);
5371	WDOORBELL64(ring->doorbell_index, ring->wptr);
5372	} else{
5373	BUG(); /* only DOORBELL method supported on gfx9 now */
5374	}
5375	}
5376
5377	static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5378	u64 seq, unsigned int flags)
5379	{
5380	struct amdgpu_device *adev = ring->adev;
5381
5382	/* we only allocate 32bit for each seq wb address */
5383	BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5384
5385	/* write fence seq to the "addr" */
5386	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5387	amdgpu_ring_write(ring, v: (WRITE_DATA_ENGINE_SEL(0) |
5388	WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5389	amdgpu_ring_write(ring, lower_32_bits(addr));
5390	amdgpu_ring_write(ring, upper_32_bits(addr));
5391	amdgpu_ring_write(ring, lower_32_bits(seq));
5392
5393	if (flags & AMDGPU_FENCE_FLAG_INT) {
5394	/* set register to trigger INT */
5395	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5396	amdgpu_ring_write(ring, v: (WRITE_DATA_ENGINE_SEL(0) |
5397	WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5398	amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5399	amdgpu_ring_write(ring, v: 0);
5400	amdgpu_ring_write(ring, v: 0x20000000); /* src_id is 178 */
5401	}
5402	}
5403
5404	static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5405	{
5406	amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5407	amdgpu_ring_write(ring, v: 0);
5408	}
5409
5410	static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)
5411	{
5412	struct amdgpu_device *adev = ring->adev;
5413	struct v9_ce_ib_state ce_payload = {0};
5414	uint64_t offset, ce_payload_gpu_addr;
5415	void *ce_payload_cpu_addr;
5416	int cnt;
5417
5418	cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5419
5420	if (ring->is_mes_queue) {
5421	offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5422	gfx[0].gfx_meta_data) +
5423	offsetof(struct v9_gfx_meta_data, ce_payload);
5424	ce_payload_gpu_addr =
5425	amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5426	ce_payload_cpu_addr =
5427	amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5428	} else {
5429	offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5430	ce_payload_gpu_addr = amdgpu_csa_vaddr(adev: ring->adev) + offset;
5431	ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5432	}
5433
5434	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5435	amdgpu_ring_write(ring, v: (WRITE_DATA_ENGINE_SEL(2) |
5436	WRITE_DATA_DST_SEL(8) |
5437	WR_CONFIRM) |
5438	WRITE_DATA_CACHE_POLICY(0));
5439	amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr));
5440	amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr));
5441
5442	amdgpu_ring_ib_on_emit_ce(ring);
5443
5444	if (resume)
5445	amdgpu_ring_write_multiple(ring, src: ce_payload_cpu_addr,
5446	count_dw: sizeof(ce_payload) >> 2);
5447	else
5448	amdgpu_ring_write_multiple(ring, src: (void *)&ce_payload,
5449	count_dw: sizeof(ce_payload) >> 2);
5450	}
5451
5452	static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring)
5453	{
5454	int i, r = 0;
5455	struct amdgpu_device *adev = ring->adev;
5456	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5457	struct amdgpu_ring *kiq_ring = &kiq->ring;
5458	unsigned long flags;
5459
5460	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5461	return -EINVAL;
5462
5463	spin_lock_irqsave(&kiq->ring_lock, flags);
5464
5465	if (amdgpu_ring_alloc(ring: kiq_ring, ndw: kiq->pmf->unmap_queues_size)) {
5466	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
5467	return -ENOMEM;
5468	}
5469
5470	/* assert preemption condition */
5471	amdgpu_ring_set_preempt_cond_exec(ring, cond_exec: false);
5472
5473	ring->trail_seq += 1;
5474	amdgpu_ring_alloc(ring, ndw: 13);
5475	gfx_v9_0_ring_emit_fence(ring, addr: ring->trail_fence_gpu_addr,
5476	seq: ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT);
5477
5478	/* assert IB preemption, emit the trailing fence */
5479	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5480	ring->trail_fence_gpu_addr,
5481	ring->trail_seq);
5482
5483	amdgpu_ring_commit(ring: kiq_ring);
5484	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
5485
5486	/* poll the trailing fence */
5487	for (i = 0; i < adev->usec_timeout; i++) {
5488	if (ring->trail_seq ==
5489	le32_to_cpu(*ring->trail_fence_cpu_addr))
5490	break;
5491	udelay(1);
5492	}
5493
5494	if (i >= adev->usec_timeout) {
5495	r = -EINVAL;
5496	DRM_WARN("ring %d timeout to preempt ib\n", ring->idx);
5497	}
5498
5499	/*reset the CP_VMID_PREEMPT after trailing fence*/
5500	amdgpu_ring_emit_wreg(ring,
5501	SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT),
5502	0x0);
5503	amdgpu_ring_commit(ring);
5504
5505	/* deassert preemption condition */
5506	amdgpu_ring_set_preempt_cond_exec(ring, cond_exec: true);
5507	return r;
5508	}
5509
5510	static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds)
5511	{
5512	struct amdgpu_device *adev = ring->adev;
5513	struct v9_de_ib_state de_payload = {0};
5514	uint64_t offset, gds_addr, de_payload_gpu_addr;
5515	void *de_payload_cpu_addr;
5516	int cnt;
5517
5518	if (ring->is_mes_queue) {
5519	offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5520	gfx[0].gfx_meta_data) +
5521	offsetof(struct v9_gfx_meta_data, de_payload);
5522	de_payload_gpu_addr =
5523	amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5524	de_payload_cpu_addr =
5525	amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5526
5527	offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5528	gfx[0].gds_backup) +
5529	offsetof(struct v9_gfx_meta_data, de_payload);
5530	gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5531	} else {
5532	offset = offsetof(struct v9_gfx_meta_data, de_payload);
5533	de_payload_gpu_addr = amdgpu_csa_vaddr(adev: ring->adev) + offset;
5534	de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5535
5536	gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5537	AMDGPU_CSA_SIZE - adev->gds.gds_size,
5538	PAGE_SIZE);
5539	}
5540
5541	if (usegds) {
5542	de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5543	de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5544	}
5545
5546	cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5547	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5548	amdgpu_ring_write(ring, v: (WRITE_DATA_ENGINE_SEL(1) |
5549	WRITE_DATA_DST_SEL(8) |
5550	WR_CONFIRM) |
5551	WRITE_DATA_CACHE_POLICY(0));
5552	amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5553	amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5554
5555	amdgpu_ring_ib_on_emit_de(ring);
5556	if (resume)
5557	amdgpu_ring_write_multiple(ring, src: de_payload_cpu_addr,
5558	count_dw: sizeof(de_payload) >> 2);
5559	else
5560	amdgpu_ring_write_multiple(ring, src: (void *)&de_payload,
5561	count_dw: sizeof(de_payload) >> 2);
5562	}
5563
5564	static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5565	bool secure)
5566	{
5567	uint32_t v = secure ? FRAME_TMZ : 0;
5568
5569	amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5570	amdgpu_ring_write(ring, v: v | FRAME_CMD(start ? 0 : 1));
5571	}
5572
5573	static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5574	{
5575	uint32_t dw2 = 0;
5576
5577	gfx_v9_0_ring_emit_ce_meta(ring,
5578	resume: (!amdgpu_sriov_vf(ring->adev) &&
5579	flags & AMDGPU_IB_PREEMPTED) ? true : false);
5580
5581	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5582	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5583	/* set load_global_config & load_global_uconfig */
5584	dw2 |= 0x8001;
5585	/* set load_cs_sh_regs */
5586	dw2 |= 0x01000000;
5587	/* set load_per_context_state & load_gfx_sh_regs for GFX */
5588	dw2 |= 0x10002;
5589
5590	/* set load_ce_ram if preamble presented */
5591	if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5592	dw2 |= 0x10000000;
5593	} else {
5594	/* still load_ce_ram if this is the first time preamble presented
5595	* although there is no context switch happens.
5596	*/
5597	if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5598	dw2 |= 0x10000000;
5599	}
5600
5601	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5602	amdgpu_ring_write(ring, v: dw2);
5603	amdgpu_ring_write(ring, v: 0);
5604	}
5605
5606	static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5607	{
5608	unsigned ret;
5609	amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5610	amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5611	amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5612	amdgpu_ring_write(ring, v: 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5613	ret = ring->wptr & ring->buf_mask;
5614	amdgpu_ring_write(ring, v: 0x55aa55aa); /* patch dummy value later */
5615	return ret;
5616	}
5617
5618	static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5619	{
5620	unsigned cur;
5621	BUG_ON(offset > ring->buf_mask);
5622	BUG_ON(ring->ring[offset] != 0x55aa55aa);
5623
5624	cur = (ring->wptr - 1) & ring->buf_mask;
5625	if (likely(cur > offset))
5626	ring->ring[offset] = cur - offset;
5627	else
5628	ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
5629	}
5630
5631	static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5632	uint32_t reg_val_offs)
5633	{
5634	struct amdgpu_device *adev = ring->adev;
5635
5636	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5637	amdgpu_ring_write(ring, v: 0 | /* src: register*/
5638	(5 << 8) | /* dst: memory */
5639	(1 << 20)); /* write confirm */
5640	amdgpu_ring_write(ring, v: reg);
5641	amdgpu_ring_write(ring, v: 0);
5642	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5643	reg_val_offs * 4));
5644	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5645	reg_val_offs * 4));
5646	}
5647
5648	static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5649	uint32_t val)
5650	{
5651	uint32_t cmd = 0;
5652
5653	switch (ring->funcs->type) {
5654	case AMDGPU_RING_TYPE_GFX:
5655	cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5656	break;
5657	case AMDGPU_RING_TYPE_KIQ:
5658	cmd = (1 << 16); /* no inc addr */
5659	break;
5660	default:
5661	cmd = WR_CONFIRM;
5662	break;
5663	}
5664	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5665	amdgpu_ring_write(ring, v: cmd);
5666	amdgpu_ring_write(ring, v: reg);
5667	amdgpu_ring_write(ring, v: 0);
5668	amdgpu_ring_write(ring, v: val);
5669	}
5670
5671	static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5672	uint32_t val, uint32_t mask)
5673	{
5674	gfx_v9_0_wait_reg_mem(ring, eng_sel: 0, mem_space: 0, opt: 0, addr0: reg, addr1: 0, ref: val, mask, inv: 0x20);
5675	}
5676
5677	static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5678	uint32_t reg0, uint32_t reg1,
5679	uint32_t ref, uint32_t mask)
5680	{
5681	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5682	struct amdgpu_device *adev = ring->adev;
5683	bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5684	adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5685
5686	if (fw_version_ok)
5687	gfx_v9_0_wait_reg_mem(ring, eng_sel: usepfp, mem_space: 0, opt: 1, addr0: reg0, addr1: reg1,
5688	ref, mask, inv: 0x20);
5689	else
5690	amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, val0: reg1,
5691	reg1: ref, val1: mask);
5692	}
5693
5694	static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5695	{
5696	struct amdgpu_device *adev = ring->adev;
5697	uint32_t value = 0;
5698
5699	value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5700	value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5701	value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5702	value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5703	WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5704	}
5705
5706	static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5707	enum amdgpu_interrupt_state state)
5708	{
5709	switch (state) {
5710	case AMDGPU_IRQ_STATE_DISABLE:
5711	case AMDGPU_IRQ_STATE_ENABLE:
5712	WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5713	TIME_STAMP_INT_ENABLE,
5714	state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5715	break;
5716	default:
5717	break;
5718	}
5719	}
5720
5721	static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5722	int me, int pipe,
5723	enum amdgpu_interrupt_state state)
5724	{
5725	u32 mec_int_cntl, mec_int_cntl_reg;
5726
5727	/*
5728	* amdgpu controls only the first MEC. That's why this function only
5729	* handles the setting of interrupts for this specific MEC. All other
5730	* pipes' interrupts are set by amdkfd.
5731	*/
5732
5733	if (me == 1) {
5734	switch (pipe) {
5735	case 0:
5736	mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5737	break;
5738	case 1:
5739	mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5740	break;
5741	case 2:
5742	mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5743	break;
5744	case 3:
5745	mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5746	break;
5747	default:
5748	DRM_DEBUG("invalid pipe %d\n", pipe);
5749	return;
5750	}
5751	} else {
5752	DRM_DEBUG("invalid me %d\n", me);
5753	return;
5754	}
5755
5756	switch (state) {
5757	case AMDGPU_IRQ_STATE_DISABLE:
5758	mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg);
5759	mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5760	TIME_STAMP_INT_ENABLE, 0);
5761	WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5762	break;
5763	case AMDGPU_IRQ_STATE_ENABLE:
5764	mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
5765	mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5766	TIME_STAMP_INT_ENABLE, 1);
5767	WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5768	break;
5769	default:
5770	break;
5771	}
5772	}
5773
5774	static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5775	struct amdgpu_irq_src *source,
5776	unsigned type,
5777	enum amdgpu_interrupt_state state)
5778	{
5779	switch (state) {
5780	case AMDGPU_IRQ_STATE_DISABLE:
5781	case AMDGPU_IRQ_STATE_ENABLE:
5782	WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5783	PRIV_REG_INT_ENABLE,
5784	state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5785	break;
5786	default:
5787	break;
5788	}
5789
5790	return 0;
5791	}
5792
5793	static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5794	struct amdgpu_irq_src *source,
5795	unsigned type,
5796	enum amdgpu_interrupt_state state)
5797	{
5798	switch (state) {
5799	case AMDGPU_IRQ_STATE_DISABLE:
5800	case AMDGPU_IRQ_STATE_ENABLE:
5801	WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5802	PRIV_INSTR_INT_ENABLE,
5803	state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5804	break;
5805	default:
5806	break;
5807	}
5808
5809	return 0;
5810	}
5811
5812	#define ENABLE_ECC_ON_ME_PIPE(me, pipe) \
5813	WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5814	CP_ECC_ERROR_INT_ENABLE, 1)
5815
5816	#define DISABLE_ECC_ON_ME_PIPE(me, pipe) \
5817	WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5818	CP_ECC_ERROR_INT_ENABLE, 0)
5819
5820	static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5821	struct amdgpu_irq_src *source,
5822	unsigned type,
5823	enum amdgpu_interrupt_state state)
5824	{
5825	switch (state) {
5826	case AMDGPU_IRQ_STATE_DISABLE:
5827	WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5828	CP_ECC_ERROR_INT_ENABLE, 0);
5829	DISABLE_ECC_ON_ME_PIPE(1, 0);
5830	DISABLE_ECC_ON_ME_PIPE(1, 1);
5831	DISABLE_ECC_ON_ME_PIPE(1, 2);
5832	DISABLE_ECC_ON_ME_PIPE(1, 3);
5833	break;
5834
5835	case AMDGPU_IRQ_STATE_ENABLE:
5836	WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5837	CP_ECC_ERROR_INT_ENABLE, 1);
5838	ENABLE_ECC_ON_ME_PIPE(1, 0);
5839	ENABLE_ECC_ON_ME_PIPE(1, 1);
5840	ENABLE_ECC_ON_ME_PIPE(1, 2);
5841	ENABLE_ECC_ON_ME_PIPE(1, 3);
5842	break;
5843	default:
5844	break;
5845	}
5846
5847	return 0;
5848	}
5849
5850
5851	static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5852	struct amdgpu_irq_src *src,
5853	unsigned type,
5854	enum amdgpu_interrupt_state state)
5855	{
5856	switch (type) {
5857	case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5858	gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5859	break;
5860	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5861	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 1, pipe: 0, state);
5862	break;
5863	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5864	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 1, pipe: 1, state);
5865	break;
5866	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5867	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 1, pipe: 2, state);
5868	break;
5869	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5870	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 1, pipe: 3, state);
5871	break;
5872	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5873	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 2, pipe: 0, state);
5874	break;
5875	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5876	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 2, pipe: 1, state);
5877	break;
5878	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5879	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 2, pipe: 2, state);
5880	break;
5881	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5882	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 2, pipe: 3, state);
5883	break;
5884	default:
5885	break;
5886	}
5887	return 0;
5888	}
5889
5890	static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5891	struct amdgpu_irq_src *source,
5892	struct amdgpu_iv_entry *entry)
5893	{
5894	int i;
5895	u8 me_id, pipe_id, queue_id;
5896	struct amdgpu_ring *ring;
5897
5898	DRM_DEBUG("IH: CP EOP\n");
5899	me_id = (entry->ring_id & 0x0c) >> 2;
5900	pipe_id = (entry->ring_id & 0x03) >> 0;
5901	queue_id = (entry->ring_id & 0x70) >> 4;
5902
5903	switch (me_id) {
5904	case 0:
5905	if (adev->gfx.num_gfx_rings &&
5906	!amdgpu_mcbp_handle_trailing_fence_irq(mux: &adev->gfx.muxer)) {
5907	/* Fence signals are handled on the software rings*/
5908	for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
5909	amdgpu_fence_process(ring: &adev->gfx.sw_gfx_ring[i]);
5910	}
5911	break;
5912	case 1:
5913	case 2:
5914	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5915	ring = &adev->gfx.compute_ring[i];
5916	/* Per-queue interrupt is supported for MEC starting from VI.
5917	* The interrupt can only be enabled/disabled per pipe instead of per queue.
5918	*/
5919	if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5920	amdgpu_fence_process(ring);
5921	}
5922	break;
5923	}
5924	return 0;
5925	}
5926
5927	static void gfx_v9_0_fault(struct amdgpu_device *adev,
5928	struct amdgpu_iv_entry *entry)
5929	{
5930	u8 me_id, pipe_id, queue_id;
5931	struct amdgpu_ring *ring;
5932	int i;
5933
5934	me_id = (entry->ring_id & 0x0c) >> 2;
5935	pipe_id = (entry->ring_id & 0x03) >> 0;
5936	queue_id = (entry->ring_id & 0x70) >> 4;
5937
5938	switch (me_id) {
5939	case 0:
5940	drm_sched_fault(sched: &adev->gfx.gfx_ring[0].sched);
5941	break;
5942	case 1:
5943	case 2:
5944	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5945	ring = &adev->gfx.compute_ring[i];
5946	if (ring->me == me_id && ring->pipe == pipe_id &&
5947	ring->queue == queue_id)
5948	drm_sched_fault(sched: &ring->sched);
5949	}
5950	break;
5951	}
5952	}
5953
5954	static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5955	struct amdgpu_irq_src *source,
5956	struct amdgpu_iv_entry *entry)
5957	{
5958	DRM_ERROR("Illegal register access in command stream\n");
5959	gfx_v9_0_fault(adev, entry);
5960	return 0;
5961	}
5962
5963	static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5964	struct amdgpu_irq_src *source,
5965	struct amdgpu_iv_entry *entry)
5966	{
5967	DRM_ERROR("Illegal instruction in command stream\n");
5968	gfx_v9_0_fault(adev, entry);
5969	return 0;
5970	}
5971
5972
5973	static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
5974	{ "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5975	SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5976	SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5977	},
5978	{ "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5979	SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5980	SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5981	},
5982	{ "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5983	SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5984	0, 0
5985	},
5986	{ "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5987	SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5988	0, 0
5989	},
5990	{ "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5991	SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
5992	SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
5993	},
5994	{ "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5995	SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
5996	0, 0
5997	},
5998	{ "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5999	SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
6000	SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
6001	},
6002	{ "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
6003	SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
6004	SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
6005	},
6006	{ "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
6007	SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
6008	0, 0
6009	},
6010	{ "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
6011	SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
6012	0, 0
6013	},
6014	{ "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
6015	SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
6016	0, 0
6017	},
6018	{ "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6019	SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
6020	SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
6021	},
6022	{ "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6023	SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
6024	0, 0
6025	},
6026	{ "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6027	SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
6028	SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
6029	},
6030	{ "GDS_OA_PHY_PHY_CMD_RAM_MEM",
6031	SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6032	SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
6033	SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
6034	},
6035	{ "GDS_OA_PHY_PHY_DATA_RAM_MEM",
6036	SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6037	SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
6038	0, 0
6039	},
6040	{ "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6041	SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6042	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6043	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6044	},
6045	{ "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6046	SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6047	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6048	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6049	},
6050	{ "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6051	SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6052	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6053	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6054	},
6055	{ "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6056	SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6057	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6058	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6059	},
6060	{ "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6061	SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6062	0, 0
6063	},
6064	{ "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6065	SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6066	SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6067	},
6068	{ "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6069	SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6070	0, 0
6071	},
6072	{ "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6073	SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6074	0, 0
6075	},
6076	{ "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6077	SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6078	0, 0
6079	},
6080	{ "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6081	SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6082	0, 0
6083	},
6084	{ "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6085	SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6086	0, 0
6087	},
6088	{ "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6089	SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6090	0, 0
6091	},
6092	{ "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6093	SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6094	SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6095	},
6096	{ "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6097	SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6098	SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6099	},
6100	{ "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6101	SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6102	SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6103	},
6104	{ "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6105	SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6106	SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6107	},
6108	{ "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6109	SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6110	SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6111	},
6112	{ "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6113	SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6114	0, 0
6115	},
6116	{ "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6117	SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6118	0, 0
6119	},
6120	{ "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6121	SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6122	0, 0
6123	},
6124	{ "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6125	SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6126	0, 0
6127	},
6128	{ "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6129	SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6130	0, 0
6131	},
6132	{ "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6133	SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6134	0, 0
6135	},
6136	{ "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6137	SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6138	0, 0
6139	},
6140	{ "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6141	SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6142	0, 0
6143	},
6144	{ "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6145	SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6146	0, 0
6147	},
6148	{ "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6149	SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6150	0, 0
6151	},
6152	{ "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6153	SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6154	0, 0
6155	},
6156	{ "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6157	SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6158	0, 0
6159	},
6160	{ "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6161	SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6162	0, 0
6163	},
6164	{ "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6165	SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6166	0, 0
6167	},
6168	{ "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6169	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6170	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6171	},
6172	{ "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6173	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6174	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6175	},
6176	{ "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6177	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6178	0, 0
6179	},
6180	{ "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6181	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6182	0, 0
6183	},
6184	{ "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6185	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6186	0, 0
6187	},
6188	{ "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6189	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6190	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6191	},
6192	{ "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6193	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6194	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6195	},
6196	{ "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6197	SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6198	SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6199	},
6200	{ "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6201	SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6202	SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6203	},
6204	{ "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6205	SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6206	0, 0
6207	},
6208	{ "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6209	SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6210	SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6211	},
6212	{ "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6213	SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6214	SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6215	},
6216	{ "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6217	SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6218	SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6219	},
6220	{ "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6221	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6222	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6223	},
6224	{ "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6225	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6226	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6227	},
6228	{ "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6229	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6230	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6231	},
6232	{ "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6233	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6234	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6235	},
6236	{ "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6237	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6238	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6239	},
6240	{ "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6241	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6242	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6243	},
6244	{ "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6245	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6246	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6247	},
6248	{ "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6249	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6250	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6251	},
6252	{ "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6253	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6254	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6255	},
6256	{ "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6257	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6258	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6259	},
6260	{ "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6261	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6262	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6263	},
6264	{ "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6265	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6266	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6267	},
6268	{ "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6269	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6270	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6271	},
6272	{ "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6273	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6274	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6275	},
6276	{ "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6277	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6278	0, 0
6279	},
6280	{ "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6281	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6282	0, 0
6283	},
6284	{ "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6285	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6286	0, 0
6287	},
6288	{ "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6289	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6290	0, 0
6291	},
6292	{ "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6293	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6294	0, 0
6295	},
6296	{ "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6297	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6298	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6299	},
6300	{ "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6301	SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6302	SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6303	},
6304	{ "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6305	SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6306	SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6307	},
6308	{ "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6309	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6310	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6311	},
6312	{ "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6313	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6314	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6315	},
6316	{ "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6317	SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6318	0, 0
6319	},
6320	{ "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6321	SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6322	0, 0
6323	},
6324	{ "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6325	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6326	0, 0
6327	},
6328	{ "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6329	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6330	0, 0
6331	},
6332	{ "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6333	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6334	0, 0
6335	},
6336	{ "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6337	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6338	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6339	},
6340	{ "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6341	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6342	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6343	},
6344	{ "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6345	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6346	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6347	},
6348	{ "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6349	SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6350	SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6351	},
6352	{ "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6353	SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6354	SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6355	},
6356	{ "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6357	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6358	0, 0
6359	},
6360	{ "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6361	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6362	0, 0
6363	},
6364	{ "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6365	SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6366	0, 0
6367	},
6368	{ "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6369	SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6370	0, 0
6371	},
6372	{ "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6373	SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6374	0, 0
6375	},
6376	{ "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6377	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6378	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6379	},
6380	{ "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6381	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6382	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6383	},
6384	{ "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6385	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6386	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6387	},
6388	{ "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6389	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6390	0, 0
6391	},
6392	{ "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6393	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6394	0, 0
6395	},
6396	{ "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6397	SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6398	0, 0
6399	},
6400	{ "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6401	SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6402	0, 0
6403	},
6404	{ "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6405	SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6406	0, 0
6407	},
6408	{ "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6409	SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6410	0, 0
6411	}
6412	};
6413
6414	static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6415	void *inject_if, uint32_t instance_mask)
6416	{
6417	struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6418	int ret;
6419	struct ta_ras_trigger_error_input block_info = { 0 };
6420
6421	if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
6422	return -EINVAL;
6423
6424	if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6425	return -EINVAL;
6426
6427	if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6428	return -EPERM;
6429
6430	if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6431	info->head.type)) {
6432	DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6433	ras_gfx_subblocks[info->head.sub_block_index].name,
6434	info->head.type);
6435	return -EPERM;
6436	}
6437
6438	if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6439	info->head.type)) {
6440	DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6441	ras_gfx_subblocks[info->head.sub_block_index].name,
6442	info->head.type);
6443	return -EPERM;
6444	}
6445
6446	block_info.block_id = amdgpu_ras_block_to_ta(block: info->head.block);
6447	block_info.sub_block_index =
6448	ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6449	block_info.inject_error_type = amdgpu_ras_error_to_ta(error: info->head.type);
6450	block_info.address = info->address;
6451	block_info.value = info->value;
6452
6453	mutex_lock(&adev->grbm_idx_mutex);
6454	ret = psp_ras_trigger_error(psp: &adev->psp, info: &block_info, instance_mask);
6455	mutex_unlock(lock: &adev->grbm_idx_mutex);
6456
6457	return ret;
6458	}
6459
6460	static const char * const vml2_mems[] = {
6461	"UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6462	"UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6463	"UTC_VML2_BANK_CACHE_0_4K_MEM0",
6464	"UTC_VML2_BANK_CACHE_0_4K_MEM1",
6465	"UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6466	"UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6467	"UTC_VML2_BANK_CACHE_1_4K_MEM0",
6468	"UTC_VML2_BANK_CACHE_1_4K_MEM1",
6469	"UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6470	"UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6471	"UTC_VML2_BANK_CACHE_2_4K_MEM0",
6472	"UTC_VML2_BANK_CACHE_2_4K_MEM1",
6473	"UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6474	"UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6475	"UTC_VML2_BANK_CACHE_3_4K_MEM0",
6476	"UTC_VML2_BANK_CACHE_3_4K_MEM1",
6477	};
6478
6479	static const char * const vml2_walker_mems[] = {
6480	"UTC_VML2_CACHE_PDE0_MEM0",
6481	"UTC_VML2_CACHE_PDE0_MEM1",
6482	"UTC_VML2_CACHE_PDE1_MEM0",
6483	"UTC_VML2_CACHE_PDE1_MEM1",
6484	"UTC_VML2_CACHE_PDE2_MEM0",
6485	"UTC_VML2_CACHE_PDE2_MEM1",
6486	"UTC_VML2_RDIF_LOG_FIFO",
6487	};
6488
6489	static const char * const atc_l2_cache_2m_mems[] = {
6490	"UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6491	"UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6492	"UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6493	"UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6494	};
6495
6496	static const char *atc_l2_cache_4k_mems[] = {
6497	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6498	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6499	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6500	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6501	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6502	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6503	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6504	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6505	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6506	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6507	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6508	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6509	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6510	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6511	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6512	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6513	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6514	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6515	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6516	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6517	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6518	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6519	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6520	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6521	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6522	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6523	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6524	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6525	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6526	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6527	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6528	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6529	};
6530
6531	static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6532	struct ras_err_data *err_data)
6533	{
6534	uint32_t i, data;
6535	uint32_t sec_count, ded_count;
6536
6537	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6538	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6539	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6540	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6541	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6542	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6543	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6544	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6545
6546	for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6547	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6548	data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6549
6550	sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6551	if (sec_count) {
6552	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6553	"SEC %d\n", i, vml2_mems[i], sec_count);
6554	err_data->ce_count += sec_count;
6555	}
6556
6557	ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6558	if (ded_count) {
6559	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6560	"DED %d\n", i, vml2_mems[i], ded_count);
6561	err_data->ue_count += ded_count;
6562	}
6563	}
6564
6565	for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6566	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6567	data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6568
6569	sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6570	SEC_COUNT);
6571	if (sec_count) {
6572	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6573	"SEC %d\n", i, vml2_walker_mems[i], sec_count);
6574	err_data->ce_count += sec_count;
6575	}
6576
6577	ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6578	DED_COUNT);
6579	if (ded_count) {
6580	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6581	"DED %d\n", i, vml2_walker_mems[i], ded_count);
6582	err_data->ue_count += ded_count;
6583	}
6584	}
6585
6586	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6587	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6588	data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6589
6590	sec_count = (data & 0x00006000L) >> 0xd;
6591	if (sec_count) {
6592	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6593	"SEC %d\n", i, atc_l2_cache_2m_mems[i],
6594	sec_count);
6595	err_data->ce_count += sec_count;
6596	}
6597	}
6598
6599	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6600	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6601	data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6602
6603	sec_count = (data & 0x00006000L) >> 0xd;
6604	if (sec_count) {
6605	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6606	"SEC %d\n", i, atc_l2_cache_4k_mems[i],
6607	sec_count);
6608	err_data->ce_count += sec_count;
6609	}
6610
6611	ded_count = (data & 0x00018000L) >> 0xf;
6612	if (ded_count) {
6613	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6614	"DED %d\n", i, atc_l2_cache_4k_mems[i],
6615	ded_count);
6616	err_data->ue_count += ded_count;
6617	}
6618	}
6619
6620	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6621	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6622	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6623	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6624
6625	return 0;
6626	}
6627
6628	static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6629	const struct soc15_reg_entry *reg,
6630	uint32_t se_id, uint32_t inst_id, uint32_t value,
6631	uint32_t *sec_count, uint32_t *ded_count)
6632	{
6633	uint32_t i;
6634	uint32_t sec_cnt, ded_cnt;
6635
6636	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6637	if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6638	gfx_v9_0_ras_fields[i].seg != reg->seg ||
6639	gfx_v9_0_ras_fields[i].inst != reg->inst)
6640	continue;
6641
6642	sec_cnt = (value &
6643	gfx_v9_0_ras_fields[i].sec_count_mask) >>
6644	gfx_v9_0_ras_fields[i].sec_count_shift;
6645	if (sec_cnt) {
6646	dev_info(adev->dev, "GFX SubBlock %s, "
6647	"Instance[%d][%d], SEC %d\n",
6648	gfx_v9_0_ras_fields[i].name,
6649	se_id, inst_id,
6650	sec_cnt);
6651	*sec_count += sec_cnt;
6652	}
6653
6654	ded_cnt = (value &
6655	gfx_v9_0_ras_fields[i].ded_count_mask) >>
6656	gfx_v9_0_ras_fields[i].ded_count_shift;
6657	if (ded_cnt) {
6658	dev_info(adev->dev, "GFX SubBlock %s, "
6659	"Instance[%d][%d], DED %d\n",
6660	gfx_v9_0_ras_fields[i].name,
6661	se_id, inst_id,
6662	ded_cnt);
6663	*ded_count += ded_cnt;
6664	}
6665	}
6666
6667	return 0;
6668	}
6669
6670	static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6671	{
6672	int i, j, k;
6673
6674	if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
6675	return;
6676
6677	/* read back registers to clear the counters */
6678	mutex_lock(&adev->grbm_idx_mutex);
6679	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6680	for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6681	for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6682	amdgpu_gfx_select_se_sh(adev, j, 0x0, k, 0);
6683	RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6684	}
6685	}
6686	}
6687	WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
6688	mutex_unlock(lock: &adev->grbm_idx_mutex);
6689
6690	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6691	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6692	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6693	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6694	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6695	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6696	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6697	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6698
6699	for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6700	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6701	RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6702	}
6703
6704	for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6705	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6706	RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6707	}
6708
6709	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6710	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6711	RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6712	}
6713
6714	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6715	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6716	RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6717	}
6718
6719	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6720	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6721	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6722	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6723	}
6724
6725	static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6726	void *ras_error_status)
6727	{
6728	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6729	uint32_t sec_count = 0, ded_count = 0;
6730	uint32_t i, j, k;
6731	uint32_t reg_value;
6732
6733	if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
6734	return;
6735
6736	err_data->ue_count = 0;
6737	err_data->ce_count = 0;
6738
6739	mutex_lock(&adev->grbm_idx_mutex);
6740
6741	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6742	for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6743	for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6744	amdgpu_gfx_select_se_sh(adev, j, 0, k, 0);
6745	reg_value =
6746	RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6747	if (reg_value)
6748	gfx_v9_0_ras_error_count(adev,
6749	reg: &gfx_v9_0_edc_counter_regs[i],
6750	se_id: j, inst_id: k, value: reg_value,
6751	sec_count: &sec_count, ded_count: &ded_count);
6752	}
6753	}
6754	}
6755
6756	err_data->ce_count += sec_count;
6757	err_data->ue_count += ded_count;
6758
6759	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
6760	mutex_unlock(lock: &adev->grbm_idx_mutex);
6761
6762	gfx_v9_0_query_utc_edc_status(adev, err_data);
6763	}
6764
6765	static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
6766	{
6767	const unsigned int cp_coher_cntl =
6768	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
6769	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
6770	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
6771	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
6772	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
6773
6774	/* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
6775	amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6776	amdgpu_ring_write(ring, v: cp_coher_cntl); /* CP_COHER_CNTL */
6777	amdgpu_ring_write(ring, v: 0xffffffff); /* CP_COHER_SIZE */
6778	amdgpu_ring_write(ring, v: 0xffffff); /* CP_COHER_SIZE_HI */
6779	amdgpu_ring_write(ring, v: 0); /* CP_COHER_BASE */
6780	amdgpu_ring_write(ring, v: 0); /* CP_COHER_BASE_HI */
6781	amdgpu_ring_write(ring, v: 0x0000000A); /* POLL_INTERVAL */
6782	}
6783
6784	static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6785	uint32_t pipe, bool enable)
6786	{
6787	struct amdgpu_device *adev = ring->adev;
6788	uint32_t val;
6789	uint32_t wcl_cs_reg;
6790
6791	/* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6792	val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
6793
6794	switch (pipe) {
6795	case 0:
6796	wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
6797	break;
6798	case 1:
6799	wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
6800	break;
6801	case 2:
6802	wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
6803	break;
6804	case 3:
6805	wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
6806	break;
6807	default:
6808	DRM_DEBUG("invalid pipe %d\n", pipe);
6809	return;
6810	}
6811
6812	amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6813
6814	}
6815	static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6816	{
6817	struct amdgpu_device *adev = ring->adev;
6818	uint32_t val;
6819	int i;
6820
6821
6822	/* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
6823	* number of gfx waves. Setting 5 bit will make sure gfx only gets
6824	* around 25% of gpu resources.
6825	*/
6826	val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6827	amdgpu_ring_emit_wreg(ring,
6828	SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
6829	val);
6830
6831	/* Restrict waves for normal/low priority compute queues as well
6832	* to get best QoS for high priority compute jobs.
6833	*
6834	* amdgpu controls only 1st ME(0-3 CS pipes).
6835	*/
6836	for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6837	if (i != ring->pipe)
6838	gfx_v9_0_emit_wave_limit_cs(ring, pipe: i, enable);
6839
6840	}
6841	}
6842
6843	static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6844	.name = "gfx_v9_0",
6845	.early_init = gfx_v9_0_early_init,
6846	.late_init = gfx_v9_0_late_init,
6847	.sw_init = gfx_v9_0_sw_init,
6848	.sw_fini = gfx_v9_0_sw_fini,
6849	.hw_init = gfx_v9_0_hw_init,
6850	.hw_fini = gfx_v9_0_hw_fini,
6851	.suspend = gfx_v9_0_suspend,
6852	.resume = gfx_v9_0_resume,
6853	.is_idle = gfx_v9_0_is_idle,
6854	.wait_for_idle = gfx_v9_0_wait_for_idle,
6855	.soft_reset = gfx_v9_0_soft_reset,
6856	.set_clockgating_state = gfx_v9_0_set_clockgating_state,
6857	.set_powergating_state = gfx_v9_0_set_powergating_state,
6858	.get_clockgating_state = gfx_v9_0_get_clockgating_state,
6859	};
6860
6861	static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6862	.type = AMDGPU_RING_TYPE_GFX,
6863	.align_mask = 0xff,
6864	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6865	.support_64bit_ptrs = true,
6866	.secure_submission_supported = true,
6867	.get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6868	.get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6869	.set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6870	.emit_frame_size = /* totally 242 maximum if 16 IBs */
6871	5 + /* COND_EXEC */
6872	7 + /* PIPELINE_SYNC */
6873	SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6874	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6875	2 + /* VM_FLUSH */
6876	8 + /* FENCE for VM_FLUSH */
6877	20 + /* GDS switch */
6878	4 + /* double SWITCH_BUFFER,
6879	the first COND_EXEC jump to the place just
6880	prior to this double SWITCH_BUFFER */
6881	5 + /* COND_EXEC */
6882	7 + /* HDP_flush */
6883	4 + /* VGT_flush */
6884	14 + /* CE_META */
6885	31 + /* DE_META */
6886	3 + /* CNTX_CTRL */
6887	5 + /* HDP_INVL */
6888	8 + 8 + /* FENCE x2 */
6889	2 + /* SWITCH_BUFFER */
6890	7, /* gfx_v9_0_emit_mem_sync */
6891	.emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6892	.emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6893	.emit_fence = gfx_v9_0_ring_emit_fence,
6894	.emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6895	.emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6896	.emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6897	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6898	.test_ring = gfx_v9_0_ring_test_ring,
6899	.insert_nop = amdgpu_ring_insert_nop,
6900	.pad_ib = amdgpu_ring_generic_pad_ib,
6901	.emit_switch_buffer = gfx_v9_ring_emit_sb,
6902	.emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6903	.init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6904	.patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6905	.preempt_ib = gfx_v9_0_ring_preempt_ib,
6906	.emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6907	.emit_wreg = gfx_v9_0_ring_emit_wreg,
6908	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6909	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6910	.soft_recovery = gfx_v9_0_ring_soft_recovery,
6911	.emit_mem_sync = gfx_v9_0_emit_mem_sync,
6912	};
6913
6914	static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = {
6915	.type = AMDGPU_RING_TYPE_GFX,
6916	.align_mask = 0xff,
6917	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6918	.support_64bit_ptrs = true,
6919	.secure_submission_supported = true,
6920	.get_rptr = amdgpu_sw_ring_get_rptr_gfx,
6921	.get_wptr = amdgpu_sw_ring_get_wptr_gfx,
6922	.set_wptr = amdgpu_sw_ring_set_wptr_gfx,
6923	.emit_frame_size = /* totally 242 maximum if 16 IBs */
6924	5 + /* COND_EXEC */
6925	7 + /* PIPELINE_SYNC */
6926	SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6927	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6928	2 + /* VM_FLUSH */
6929	8 + /* FENCE for VM_FLUSH */
6930	20 + /* GDS switch */
6931	4 + /* double SWITCH_BUFFER,
6932	* the first COND_EXEC jump to the place just
6933	* prior to this double SWITCH_BUFFER
6934	*/
6935	5 + /* COND_EXEC */
6936	7 + /* HDP_flush */
6937	4 + /* VGT_flush */
6938	14 + /* CE_META */
6939	31 + /* DE_META */
6940	3 + /* CNTX_CTRL */
6941	5 + /* HDP_INVL */
6942	8 + 8 + /* FENCE x2 */
6943	2 + /* SWITCH_BUFFER */
6944	7, /* gfx_v9_0_emit_mem_sync */
6945	.emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6946	.emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6947	.emit_fence = gfx_v9_0_ring_emit_fence,
6948	.emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6949	.emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6950	.emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6951	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6952	.test_ring = gfx_v9_0_ring_test_ring,
6953	.test_ib = gfx_v9_0_ring_test_ib,
6954	.insert_nop = amdgpu_sw_ring_insert_nop,
6955	.pad_ib = amdgpu_ring_generic_pad_ib,
6956	.emit_switch_buffer = gfx_v9_ring_emit_sb,
6957	.emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6958	.init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6959	.patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6960	.emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6961	.emit_wreg = gfx_v9_0_ring_emit_wreg,
6962	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6963	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6964	.soft_recovery = gfx_v9_0_ring_soft_recovery,
6965	.emit_mem_sync = gfx_v9_0_emit_mem_sync,
6966	.patch_cntl = gfx_v9_0_ring_patch_cntl,
6967	.patch_de = gfx_v9_0_ring_patch_de_meta,
6968	.patch_ce = gfx_v9_0_ring_patch_ce_meta,
6969	};
6970
6971	static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6972	.type = AMDGPU_RING_TYPE_COMPUTE,
6973	.align_mask = 0xff,
6974	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6975	.support_64bit_ptrs = true,
6976	.get_rptr = gfx_v9_0_ring_get_rptr_compute,
6977	.get_wptr = gfx_v9_0_ring_get_wptr_compute,
6978	.set_wptr = gfx_v9_0_ring_set_wptr_compute,
6979	.emit_frame_size =
6980	20 + /* gfx_v9_0_ring_emit_gds_switch */
6981	7 + /* gfx_v9_0_ring_emit_hdp_flush */
6982	5 + /* hdp invalidate */
6983	7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6984	SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6985	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6986	2 + /* gfx_v9_0_ring_emit_vm_flush */
6987	8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
6988	7 + /* gfx_v9_0_emit_mem_sync */
6989	5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
6990	15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
6991	.emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6992	.emit_ib = gfx_v9_0_ring_emit_ib_compute,
6993	.emit_fence = gfx_v9_0_ring_emit_fence,
6994	.emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6995	.emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6996	.emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6997	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6998	.test_ring = gfx_v9_0_ring_test_ring,
6999	.test_ib = gfx_v9_0_ring_test_ib,
7000	.insert_nop = amdgpu_ring_insert_nop,
7001	.pad_ib = amdgpu_ring_generic_pad_ib,
7002	.emit_wreg = gfx_v9_0_ring_emit_wreg,
7003	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7004	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7005	.emit_mem_sync = gfx_v9_0_emit_mem_sync,
7006	.emit_wave_limit = gfx_v9_0_emit_wave_limit,
7007	};
7008
7009	static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
7010	.type = AMDGPU_RING_TYPE_KIQ,
7011	.align_mask = 0xff,
7012	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
7013	.support_64bit_ptrs = true,
7014	.get_rptr = gfx_v9_0_ring_get_rptr_compute,
7015	.get_wptr = gfx_v9_0_ring_get_wptr_compute,
7016	.set_wptr = gfx_v9_0_ring_set_wptr_compute,
7017	.emit_frame_size =
7018	20 + /* gfx_v9_0_ring_emit_gds_switch */
7019	7 + /* gfx_v9_0_ring_emit_hdp_flush */
7020	5 + /* hdp invalidate */
7021	7 + /* gfx_v9_0_ring_emit_pipeline_sync */
7022	SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7023	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7024	2 + /* gfx_v9_0_ring_emit_vm_flush */
7025	8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
7026	.emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
7027	.emit_fence = gfx_v9_0_ring_emit_fence_kiq,
7028	.test_ring = gfx_v9_0_ring_test_ring,
7029	.insert_nop = amdgpu_ring_insert_nop,
7030	.pad_ib = amdgpu_ring_generic_pad_ib,
7031	.emit_rreg = gfx_v9_0_ring_emit_rreg,
7032	.emit_wreg = gfx_v9_0_ring_emit_wreg,
7033	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7034	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7035	};
7036
7037	static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
7038	{
7039	int i;
7040
7041	adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq;
7042
7043	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7044	adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
7045
7046	if (adev->gfx.num_gfx_rings) {
7047	for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
7048	adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx;
7049	}
7050
7051	for (i = 0; i < adev->gfx.num_compute_rings; i++)
7052	adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
7053	}
7054
7055	static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
7056	.set = gfx_v9_0_set_eop_interrupt_state,
7057	.process = gfx_v9_0_eop_irq,
7058	};
7059
7060	static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
7061	.set = gfx_v9_0_set_priv_reg_fault_state,
7062	.process = gfx_v9_0_priv_reg_irq,
7063	};
7064
7065	static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
7066	.set = gfx_v9_0_set_priv_inst_fault_state,
7067	.process = gfx_v9_0_priv_inst_irq,
7068	};
7069
7070	static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
7071	.set = gfx_v9_0_set_cp_ecc_error_state,
7072	.process = amdgpu_gfx_cp_ecc_error_irq,
7073	};
7074
7075
7076	static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
7077	{
7078	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7079	adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
7080
7081	adev->gfx.priv_reg_irq.num_types = 1;
7082	adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
7083
7084	adev->gfx.priv_inst_irq.num_types = 1;
7085	adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
7086
7087	adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
7088	adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
7089	}
7090
7091	static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
7092	{
7093	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
7094	case IP_VERSION(9, 0, 1):
7095	case IP_VERSION(9, 2, 1):
7096	case IP_VERSION(9, 4, 0):
7097	case IP_VERSION(9, 2, 2):
7098	case IP_VERSION(9, 1, 0):
7099	case IP_VERSION(9, 4, 1):
7100	case IP_VERSION(9, 3, 0):
7101	case IP_VERSION(9, 4, 2):
7102	adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7103	break;
7104	default:
7105	break;
7106	}
7107	}
7108
7109	static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7110	{
7111	/* init asci gds info */
7112	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
7113	case IP_VERSION(9, 0, 1):
7114	case IP_VERSION(9, 2, 1):
7115	case IP_VERSION(9, 4, 0):
7116	adev->gds.gds_size = 0x10000;
7117	break;
7118	case IP_VERSION(9, 2, 2):
7119	case IP_VERSION(9, 1, 0):
7120	case IP_VERSION(9, 4, 1):
7121	adev->gds.gds_size = 0x1000;
7122	break;
7123	case IP_VERSION(9, 4, 2):
7124	/* aldebaran removed all the GDS internal memory,
7125	* only support GWS opcode in kernel, like barrier
7126	* semaphore.etc */
7127	adev->gds.gds_size = 0;
7128	break;
7129	default:
7130	adev->gds.gds_size = 0x10000;
7131	break;
7132	}
7133
7134	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
7135	case IP_VERSION(9, 0, 1):
7136	case IP_VERSION(9, 4, 0):
7137	adev->gds.gds_compute_max_wave_id = 0x7ff;
7138	break;
7139	case IP_VERSION(9, 2, 1):
7140	adev->gds.gds_compute_max_wave_id = 0x27f;
7141	break;
7142	case IP_VERSION(9, 2, 2):
7143	case IP_VERSION(9, 1, 0):
7144	if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7145	adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7146	else
7147	adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7148	break;
7149	case IP_VERSION(9, 4, 1):
7150	adev->gds.gds_compute_max_wave_id = 0xfff;
7151	break;
7152	case IP_VERSION(9, 4, 2):
7153	/* deprecated for Aldebaran, no usage at all */
7154	adev->gds.gds_compute_max_wave_id = 0;
7155	break;
7156	default:
7157	/* this really depends on the chip */
7158	adev->gds.gds_compute_max_wave_id = 0x7ff;
7159	break;
7160	}
7161
7162	adev->gds.gws_size = 64;
7163	adev->gds.oa_size = 16;
7164	}
7165
7166	static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7167	u32 bitmap)
7168	{
7169	u32 data;
7170
7171	if (!bitmap)
7172	return;
7173
7174	data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7175	data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7176
7177	WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7178	}
7179
7180	static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7181	{
7182	u32 data, mask;
7183
7184	data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7185	data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7186
7187	data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7188	data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7189
7190	mask = amdgpu_gfx_create_bitmask(bit_width: adev->gfx.config.max_cu_per_sh);
7191
7192	return (~data) & mask;
7193	}
7194
7195	static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7196	struct amdgpu_cu_info *cu_info)
7197	{
7198	int i, j, k, counter, active_cu_number = 0;
7199	u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7200	unsigned disable_masks[4 * 4];
7201
7202	if (!adev || !cu_info)
7203	return -EINVAL;
7204
7205	/*
7206	* 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7207	*/
7208	if (adev->gfx.config.max_shader_engines *
7209	adev->gfx.config.max_sh_per_se > 16)
7210	return -EINVAL;
7211
7212	amdgpu_gfx_parse_disable_cu(mask: disable_masks,
7213	max_se: adev->gfx.config.max_shader_engines,
7214	max_sh: adev->gfx.config.max_sh_per_se);
7215
7216	mutex_lock(&adev->grbm_idx_mutex);
7217	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7218	for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7219	mask = 1;
7220	ao_bitmap = 0;
7221	counter = 0;
7222	amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
7223	gfx_v9_0_set_user_cu_inactive_bitmap(
7224	adev, bitmap: disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7225	bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7226
7227	/*
7228	* The bitmap(and ao_cu_bitmap) in cu_info structure is
7229	* 4x4 size array, and it's usually suitable for Vega
7230	* ASICs which has 4*2 SE/SH layout.
7231	* But for Arcturus, SE/SH layout is changed to 8*1.
7232	* To mostly reduce the impact, we make it compatible
7233	* with current bitmap array as below:
7234	* SE4,SH0 --> bitmap[0][1]
7235	* SE5,SH0 --> bitmap[1][1]
7236	* SE6,SH0 --> bitmap[2][1]
7237	* SE7,SH0 --> bitmap[3][1]
7238	*/
7239	cu_info->bitmap[0][i % 4][j + i / 4] = bitmap;
7240
7241	for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7242	if (bitmap & mask) {
7243	if (counter < adev->gfx.config.max_cu_per_sh)
7244	ao_bitmap |= mask;
7245	counter ++;
7246	}
7247	mask <<= 1;
7248	}
7249	active_cu_number += counter;
7250	if (i < 2 && j < 2)
7251	ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7252	cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7253	}
7254	}
7255	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7256	mutex_unlock(lock: &adev->grbm_idx_mutex);
7257
7258	cu_info->number = active_cu_number;
7259	cu_info->ao_cu_mask = ao_cu_mask;
7260	cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7261
7262	return 0;
7263	}
7264
7265	const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7266	{
7267	.type = AMD_IP_BLOCK_TYPE_GFX,
7268	.major = 9,
7269	.minor = 0,
7270	.rev = 0,
7271	.funcs = &gfx_v9_0_ip_funcs,
7272	};
7273