1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (c) 2017 Hisilicon Limited.
4 */
5
6#include <linux/sched/clock.h>
7#include "hisi_sas.h"
8#define DRV_NAME "hisi_sas_v3_hw"
9
10/* global registers need init */
11#define DLVRY_QUEUE_ENABLE 0x0
12#define IOST_BASE_ADDR_LO 0x8
13#define IOST_BASE_ADDR_HI 0xc
14#define ITCT_BASE_ADDR_LO 0x10
15#define ITCT_BASE_ADDR_HI 0x14
16#define IO_BROKEN_MSG_ADDR_LO 0x18
17#define IO_BROKEN_MSG_ADDR_HI 0x1c
18#define PHY_CONTEXT 0x20
19#define PHY_STATE 0x24
20#define PHY_PORT_NUM_MA 0x28
21#define PHY_CONN_RATE 0x30
22#define ITCT_CLR 0x44
23#define ITCT_CLR_EN_OFF 16
24#define ITCT_CLR_EN_MSK (0x1 << ITCT_CLR_EN_OFF)
25#define ITCT_DEV_OFF 0
26#define ITCT_DEV_MSK (0x7ff << ITCT_DEV_OFF)
27#define SAS_AXI_USER3 0x50
28#define IO_SATA_BROKEN_MSG_ADDR_LO 0x58
29#define IO_SATA_BROKEN_MSG_ADDR_HI 0x5c
30#define SATA_INITI_D2H_STORE_ADDR_LO 0x60
31#define SATA_INITI_D2H_STORE_ADDR_HI 0x64
32#define CFG_MAX_TAG 0x68
33#define TRANS_LOCK_ICT_TIME 0X70
34#define HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL 0x84
35#define HGC_SAS_TXFAIL_RETRY_CTRL 0x88
36#define HGC_GET_ITV_TIME 0x90
37#define DEVICE_MSG_WORK_MODE 0x94
38#define OPENA_WT_CONTI_TIME 0x9c
39#define I_T_NEXUS_LOSS_TIME 0xa0
40#define MAX_CON_TIME_LIMIT_TIME 0xa4
41#define BUS_INACTIVE_LIMIT_TIME 0xa8
42#define REJECT_TO_OPEN_LIMIT_TIME 0xac
43#define CQ_INT_CONVERGE_EN 0xb0
44#define CFG_AGING_TIME 0xbc
45#define HGC_DFX_CFG2 0xc0
46#define CFG_ABT_SET_QUERY_IPTT 0xd4
47#define CFG_SET_ABORTED_IPTT_OFF 0
48#define CFG_SET_ABORTED_IPTT_MSK (0xfff << CFG_SET_ABORTED_IPTT_OFF)
49#define CFG_SET_ABORTED_EN_OFF 12
50#define CFG_ABT_SET_IPTT_DONE 0xd8
51#define CFG_ABT_SET_IPTT_DONE_OFF 0
52#define HGC_IOMB_PROC1_STATUS 0x104
53#define HGC_LM_DFX_STATUS2 0x128
54#define HGC_LM_DFX_STATUS2_IOSTLIST_OFF 0
55#define HGC_LM_DFX_STATUS2_IOSTLIST_MSK (0xfff << \
56 HGC_LM_DFX_STATUS2_IOSTLIST_OFF)
57#define HGC_LM_DFX_STATUS2_ITCTLIST_OFF 12
58#define HGC_LM_DFX_STATUS2_ITCTLIST_MSK (0x7ff << \
59 HGC_LM_DFX_STATUS2_ITCTLIST_OFF)
60#define HGC_CQE_ECC_ADDR 0x13c
61#define HGC_CQE_ECC_1B_ADDR_OFF 0
62#define HGC_CQE_ECC_1B_ADDR_MSK (0x3f << HGC_CQE_ECC_1B_ADDR_OFF)
63#define HGC_CQE_ECC_MB_ADDR_OFF 8
64#define HGC_CQE_ECC_MB_ADDR_MSK (0x3f << HGC_CQE_ECC_MB_ADDR_OFF)
65#define HGC_IOST_ECC_ADDR 0x140
66#define HGC_IOST_ECC_1B_ADDR_OFF 0
67#define HGC_IOST_ECC_1B_ADDR_MSK (0x3ff << HGC_IOST_ECC_1B_ADDR_OFF)
68#define HGC_IOST_ECC_MB_ADDR_OFF 16
69#define HGC_IOST_ECC_MB_ADDR_MSK (0x3ff << HGC_IOST_ECC_MB_ADDR_OFF)
70#define HGC_DQE_ECC_ADDR 0x144
71#define HGC_DQE_ECC_1B_ADDR_OFF 0
72#define HGC_DQE_ECC_1B_ADDR_MSK (0xfff << HGC_DQE_ECC_1B_ADDR_OFF)
73#define HGC_DQE_ECC_MB_ADDR_OFF 16
74#define HGC_DQE_ECC_MB_ADDR_MSK (0xfff << HGC_DQE_ECC_MB_ADDR_OFF)
75#define CHNL_INT_STATUS 0x148
76#define TAB_DFX 0x14c
77#define HGC_ITCT_ECC_ADDR 0x150
78#define HGC_ITCT_ECC_1B_ADDR_OFF 0
79#define HGC_ITCT_ECC_1B_ADDR_MSK (0x3ff << \
80 HGC_ITCT_ECC_1B_ADDR_OFF)
81#define HGC_ITCT_ECC_MB_ADDR_OFF 16
82#define HGC_ITCT_ECC_MB_ADDR_MSK (0x3ff << \
83 HGC_ITCT_ECC_MB_ADDR_OFF)
84#define HGC_AXI_FIFO_ERR_INFO 0x154
85#define AXI_ERR_INFO_OFF 0
86#define AXI_ERR_INFO_MSK (0xff << AXI_ERR_INFO_OFF)
87#define FIFO_ERR_INFO_OFF 8
88#define FIFO_ERR_INFO_MSK (0xff << FIFO_ERR_INFO_OFF)
89#define TAB_RD_TYPE 0x15c
90#define INT_COAL_EN 0x19c
91#define OQ_INT_COAL_TIME 0x1a0
92#define OQ_INT_COAL_CNT 0x1a4
93#define ENT_INT_COAL_TIME 0x1a8
94#define ENT_INT_COAL_CNT 0x1ac
95#define OQ_INT_SRC 0x1b0
96#define OQ_INT_SRC_MSK 0x1b4
97#define ENT_INT_SRC1 0x1b8
98#define ENT_INT_SRC1_D2H_FIS_CH0_OFF 0
99#define ENT_INT_SRC1_D2H_FIS_CH0_MSK (0x1 << ENT_INT_SRC1_D2H_FIS_CH0_OFF)
100#define ENT_INT_SRC1_D2H_FIS_CH1_OFF 8
101#define ENT_INT_SRC1_D2H_FIS_CH1_MSK (0x1 << ENT_INT_SRC1_D2H_FIS_CH1_OFF)
102#define ENT_INT_SRC2 0x1bc
103#define ENT_INT_SRC3 0x1c0
104#define ENT_INT_SRC3_WP_DEPTH_OFF 8
105#define ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF 9
106#define ENT_INT_SRC3_RP_DEPTH_OFF 10
107#define ENT_INT_SRC3_AXI_OFF 11
108#define ENT_INT_SRC3_FIFO_OFF 12
109#define ENT_INT_SRC3_LM_OFF 14
110#define ENT_INT_SRC3_ITC_INT_OFF 15
111#define ENT_INT_SRC3_ITC_INT_MSK (0x1 << ENT_INT_SRC3_ITC_INT_OFF)
112#define ENT_INT_SRC3_ABT_OFF 16
113#define ENT_INT_SRC3_DQE_POISON_OFF 18
114#define ENT_INT_SRC3_IOST_POISON_OFF 19
115#define ENT_INT_SRC3_ITCT_POISON_OFF 20
116#define ENT_INT_SRC3_ITCT_NCQ_POISON_OFF 21
117#define ENT_INT_SRC_MSK1 0x1c4
118#define ENT_INT_SRC_MSK2 0x1c8
119#define ENT_INT_SRC_MSK3 0x1cc
120#define ENT_INT_SRC_MSK3_ENT95_MSK_OFF 31
121#define CHNL_PHYUPDOWN_INT_MSK 0x1d0
122#define CHNL_ENT_INT_MSK 0x1d4
123#define HGC_COM_INT_MSK 0x1d8
124#define ENT_INT_SRC_MSK3_ENT95_MSK_MSK (0x1 << ENT_INT_SRC_MSK3_ENT95_MSK_OFF)
125#define SAS_ECC_INTR 0x1e8
126#define SAS_ECC_INTR_DQE_ECC_1B_OFF 0
127#define SAS_ECC_INTR_DQE_ECC_MB_OFF 1
128#define SAS_ECC_INTR_IOST_ECC_1B_OFF 2
129#define SAS_ECC_INTR_IOST_ECC_MB_OFF 3
130#define SAS_ECC_INTR_ITCT_ECC_1B_OFF 4
131#define SAS_ECC_INTR_ITCT_ECC_MB_OFF 5
132#define SAS_ECC_INTR_ITCTLIST_ECC_1B_OFF 6
133#define SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF 7
134#define SAS_ECC_INTR_IOSTLIST_ECC_1B_OFF 8
135#define SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF 9
136#define SAS_ECC_INTR_CQE_ECC_1B_OFF 10
137#define SAS_ECC_INTR_CQE_ECC_MB_OFF 11
138#define SAS_ECC_INTR_NCQ_MEM0_ECC_1B_OFF 12
139#define SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF 13
140#define SAS_ECC_INTR_NCQ_MEM1_ECC_1B_OFF 14
141#define SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF 15
142#define SAS_ECC_INTR_NCQ_MEM2_ECC_1B_OFF 16
143#define SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF 17
144#define SAS_ECC_INTR_NCQ_MEM3_ECC_1B_OFF 18
145#define SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF 19
146#define SAS_ECC_INTR_OOO_RAM_ECC_1B_OFF 20
147#define SAS_ECC_INTR_OOO_RAM_ECC_MB_OFF 21
148#define SAS_ECC_INTR_MSK 0x1ec
149#define HGC_ERR_STAT_EN 0x238
150#define CQE_SEND_CNT 0x248
151#define DLVRY_Q_0_BASE_ADDR_LO 0x260
152#define DLVRY_Q_0_BASE_ADDR_HI 0x264
153#define DLVRY_Q_0_DEPTH 0x268
154#define DLVRY_Q_0_WR_PTR 0x26c
155#define DLVRY_Q_0_RD_PTR 0x270
156#define HYPER_STREAM_ID_EN_CFG 0xc80
157#define OQ0_INT_SRC_MSK 0xc90
158#define COMPL_Q_0_BASE_ADDR_LO 0x4e0
159#define COMPL_Q_0_BASE_ADDR_HI 0x4e4
160#define COMPL_Q_0_DEPTH 0x4e8
161#define COMPL_Q_0_WR_PTR 0x4ec
162#define COMPL_Q_0_RD_PTR 0x4f0
163#define HGC_RXM_DFX_STATUS14 0xae8
164#define HGC_RXM_DFX_STATUS14_MEM0_OFF 0
165#define HGC_RXM_DFX_STATUS14_MEM0_MSK (0x1ff << \
166 HGC_RXM_DFX_STATUS14_MEM0_OFF)
167#define HGC_RXM_DFX_STATUS14_MEM1_OFF 9
168#define HGC_RXM_DFX_STATUS14_MEM1_MSK (0x1ff << \
169 HGC_RXM_DFX_STATUS14_MEM1_OFF)
170#define HGC_RXM_DFX_STATUS14_MEM2_OFF 18
171#define HGC_RXM_DFX_STATUS14_MEM2_MSK (0x1ff << \
172 HGC_RXM_DFX_STATUS14_MEM2_OFF)
173#define HGC_RXM_DFX_STATUS15 0xaec
174#define HGC_RXM_DFX_STATUS15_MEM3_OFF 0
175#define HGC_RXM_DFX_STATUS15_MEM3_MSK (0x1ff << \
176 HGC_RXM_DFX_STATUS15_MEM3_OFF)
177#define AWQOS_AWCACHE_CFG 0xc84
178#define ARQOS_ARCACHE_CFG 0xc88
179#define HILINK_ERR_DFX 0xe04
180#define SAS_GPIO_CFG_0 0x1000
181#define SAS_GPIO_CFG_1 0x1004
182#define SAS_GPIO_TX_0_1 0x1040
183#define SAS_CFG_DRIVE_VLD 0x1070
184
185/* phy registers requiring init */
186#define PORT_BASE (0x2000)
187#define PHY_CFG (PORT_BASE + 0x0)
188#define HARD_PHY_LINKRATE (PORT_BASE + 0x4)
189#define PHY_CFG_ENA_OFF 0
190#define PHY_CFG_ENA_MSK (0x1 << PHY_CFG_ENA_OFF)
191#define PHY_CFG_DC_OPT_OFF 2
192#define PHY_CFG_DC_OPT_MSK (0x1 << PHY_CFG_DC_OPT_OFF)
193#define PHY_CFG_PHY_RST_OFF 3
194#define PHY_CFG_PHY_RST_MSK (0x1 << PHY_CFG_PHY_RST_OFF)
195#define PROG_PHY_LINK_RATE (PORT_BASE + 0x8)
196#define CFG_PROG_PHY_LINK_RATE_OFF 0
197#define CFG_PROG_PHY_LINK_RATE_MSK (0xff << CFG_PROG_PHY_LINK_RATE_OFF)
198#define CFG_PROG_OOB_PHY_LINK_RATE_OFF 8
199#define CFG_PROG_OOB_PHY_LINK_RATE_MSK (0xf << CFG_PROG_OOB_PHY_LINK_RATE_OFF)
200#define PHY_CTRL (PORT_BASE + 0x14)
201#define PHY_CTRL_RESET_OFF 0
202#define PHY_CTRL_RESET_MSK (0x1 << PHY_CTRL_RESET_OFF)
203#define CMD_HDR_PIR_OFF 8
204#define CMD_HDR_PIR_MSK (0x1 << CMD_HDR_PIR_OFF)
205#define SERDES_CFG (PORT_BASE + 0x1c)
206#define CFG_ALOS_CHK_DISABLE_OFF 9
207#define CFG_ALOS_CHK_DISABLE_MSK (0x1 << CFG_ALOS_CHK_DISABLE_OFF)
208#define SAS_PHY_BIST_CTRL (PORT_BASE + 0x2c)
209#define CFG_BIST_MODE_SEL_OFF 0
210#define CFG_BIST_MODE_SEL_MSK (0xf << CFG_BIST_MODE_SEL_OFF)
211#define CFG_LOOP_TEST_MODE_OFF 14
212#define CFG_LOOP_TEST_MODE_MSK (0x3 << CFG_LOOP_TEST_MODE_OFF)
213#define CFG_RX_BIST_EN_OFF 16
214#define CFG_RX_BIST_EN_MSK (0x1 << CFG_RX_BIST_EN_OFF)
215#define CFG_TX_BIST_EN_OFF 17
216#define CFG_TX_BIST_EN_MSK (0x1 << CFG_TX_BIST_EN_OFF)
217#define CFG_BIST_TEST_OFF 18
218#define CFG_BIST_TEST_MSK (0x1 << CFG_BIST_TEST_OFF)
219#define SAS_PHY_BIST_CODE (PORT_BASE + 0x30)
220#define SAS_PHY_BIST_CODE1 (PORT_BASE + 0x34)
221#define SAS_BIST_ERR_CNT (PORT_BASE + 0x38)
222#define SL_CFG (PORT_BASE + 0x84)
223#define AIP_LIMIT (PORT_BASE + 0x90)
224#define SL_CONTROL (PORT_BASE + 0x94)
225#define SL_CONTROL_NOTIFY_EN_OFF 0
226#define SL_CONTROL_NOTIFY_EN_MSK (0x1 << SL_CONTROL_NOTIFY_EN_OFF)
227#define SL_CTA_OFF 17
228#define SL_CTA_MSK (0x1 << SL_CTA_OFF)
229#define RX_PRIMS_STATUS (PORT_BASE + 0x98)
230#define RX_BCAST_CHG_OFF 1
231#define RX_BCAST_CHG_MSK (0x1 << RX_BCAST_CHG_OFF)
232#define TX_ID_DWORD0 (PORT_BASE + 0x9c)
233#define TX_ID_DWORD1 (PORT_BASE + 0xa0)
234#define TX_ID_DWORD2 (PORT_BASE + 0xa4)
235#define TX_ID_DWORD3 (PORT_BASE + 0xa8)
236#define TX_ID_DWORD4 (PORT_BASE + 0xaC)
237#define TX_ID_DWORD5 (PORT_BASE + 0xb0)
238#define TX_ID_DWORD6 (PORT_BASE + 0xb4)
239#define TXID_AUTO (PORT_BASE + 0xb8)
240#define CT3_OFF 1
241#define CT3_MSK (0x1 << CT3_OFF)
242#define TX_HARDRST_OFF 2
243#define TX_HARDRST_MSK (0x1 << TX_HARDRST_OFF)
244#define RX_IDAF_DWORD0 (PORT_BASE + 0xc4)
245#define RXOP_CHECK_CFG_H (PORT_BASE + 0xfc)
246#define STP_LINK_TIMER (PORT_BASE + 0x120)
247#define STP_LINK_TIMEOUT_STATE (PORT_BASE + 0x124)
248#define CON_CFG_DRIVER (PORT_BASE + 0x130)
249#define SAS_SSP_CON_TIMER_CFG (PORT_BASE + 0x134)
250#define SAS_SMP_CON_TIMER_CFG (PORT_BASE + 0x138)
251#define SAS_STP_CON_TIMER_CFG (PORT_BASE + 0x13c)
252#define CHL_INT0 (PORT_BASE + 0x1b4)
253#define CHL_INT0_HOTPLUG_TOUT_OFF 0
254#define CHL_INT0_HOTPLUG_TOUT_MSK (0x1 << CHL_INT0_HOTPLUG_TOUT_OFF)
255#define CHL_INT0_SL_RX_BCST_ACK_OFF 1
256#define CHL_INT0_SL_RX_BCST_ACK_MSK (0x1 << CHL_INT0_SL_RX_BCST_ACK_OFF)
257#define CHL_INT0_SL_PHY_ENABLE_OFF 2
258#define CHL_INT0_SL_PHY_ENABLE_MSK (0x1 << CHL_INT0_SL_PHY_ENABLE_OFF)
259#define CHL_INT0_NOT_RDY_OFF 4
260#define CHL_INT0_NOT_RDY_MSK (0x1 << CHL_INT0_NOT_RDY_OFF)
261#define CHL_INT0_PHY_RDY_OFF 5
262#define CHL_INT0_PHY_RDY_MSK (0x1 << CHL_INT0_PHY_RDY_OFF)
263#define CHL_INT1 (PORT_BASE + 0x1b8)
264#define CHL_INT1_DMAC_TX_ECC_MB_ERR_OFF 15
265#define CHL_INT1_DMAC_TX_ECC_1B_ERR_OFF 16
266#define CHL_INT1_DMAC_RX_ECC_MB_ERR_OFF 17
267#define CHL_INT1_DMAC_RX_ECC_1B_ERR_OFF 18
268#define CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF 19
269#define CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF 20
270#define CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF 21
271#define CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF 22
272#define CHL_INT1_DMAC_TX_FIFO_ERR_OFF 23
273#define CHL_INT1_DMAC_RX_FIFO_ERR_OFF 24
274#define CHL_INT1_DMAC_TX_AXI_RUSER_ERR_OFF 26
275#define CHL_INT1_DMAC_RX_AXI_RUSER_ERR_OFF 27
276#define CHL_INT2 (PORT_BASE + 0x1bc)
277#define CHL_INT2_SL_IDAF_TOUT_CONF_OFF 0
278#define CHL_INT2_RX_DISP_ERR_OFF 28
279#define CHL_INT2_RX_CODE_ERR_OFF 29
280#define CHL_INT2_RX_INVLD_DW_OFF 30
281#define CHL_INT2_STP_LINK_TIMEOUT_OFF 31
282#define CHL_INT0_MSK (PORT_BASE + 0x1c0)
283#define CHL_INT1_MSK (PORT_BASE + 0x1c4)
284#define CHL_INT2_MSK (PORT_BASE + 0x1c8)
285#define SAS_EC_INT_COAL_TIME (PORT_BASE + 0x1cc)
286#define CHL_INT_COAL_EN (PORT_BASE + 0x1d0)
287#define SAS_RX_TRAIN_TIMER (PORT_BASE + 0x2a4)
288#define PHY_CTRL_RDY_MSK (PORT_BASE + 0x2b0)
289#define PHYCTRL_NOT_RDY_MSK (PORT_BASE + 0x2b4)
290#define PHYCTRL_DWS_RESET_MSK (PORT_BASE + 0x2b8)
291#define PHYCTRL_PHY_ENA_MSK (PORT_BASE + 0x2bc)
292#define SL_RX_BCAST_CHK_MSK (PORT_BASE + 0x2c0)
293#define PHYCTRL_OOB_RESTART_MSK (PORT_BASE + 0x2c4)
294#define DMA_TX_STATUS (PORT_BASE + 0x2d0)
295#define DMA_TX_STATUS_BUSY_OFF 0
296#define DMA_TX_STATUS_BUSY_MSK (0x1 << DMA_TX_STATUS_BUSY_OFF)
297#define DMA_RX_STATUS (PORT_BASE + 0x2e8)
298#define DMA_RX_STATUS_BUSY_OFF 0
299#define DMA_RX_STATUS_BUSY_MSK (0x1 << DMA_RX_STATUS_BUSY_OFF)
300
301#define COARSETUNE_TIME (PORT_BASE + 0x304)
302#define TXDEEMPH_G1 (PORT_BASE + 0x350)
303#define ERR_CNT_DWS_LOST (PORT_BASE + 0x380)
304#define ERR_CNT_RESET_PROB (PORT_BASE + 0x384)
305#define ERR_CNT_INVLD_DW (PORT_BASE + 0x390)
306#define ERR_CNT_CODE_ERR (PORT_BASE + 0x394)
307#define ERR_CNT_DISP_ERR (PORT_BASE + 0x398)
308#define DFX_FIFO_CTRL (PORT_BASE + 0x3a0)
309#define DFX_FIFO_CTRL_TRIGGER_MODE_OFF 0
310#define DFX_FIFO_CTRL_TRIGGER_MODE_MSK (0x7 << DFX_FIFO_CTRL_TRIGGER_MODE_OFF)
311#define DFX_FIFO_CTRL_DUMP_MODE_OFF 3
312#define DFX_FIFO_CTRL_DUMP_MODE_MSK (0x7 << DFX_FIFO_CTRL_DUMP_MODE_OFF)
313#define DFX_FIFO_CTRL_SIGNAL_SEL_OFF 6
314#define DFX_FIFO_CTRL_SIGNAL_SEL_MSK (0xF << DFX_FIFO_CTRL_SIGNAL_SEL_OFF)
315#define DFX_FIFO_CTRL_DUMP_DISABLE_OFF 10
316#define DFX_FIFO_CTRL_DUMP_DISABLE_MSK (0x1 << DFX_FIFO_CTRL_DUMP_DISABLE_OFF)
317#define DFX_FIFO_TRIGGER (PORT_BASE + 0x3a4)
318#define DFX_FIFO_TRIGGER_MSK (PORT_BASE + 0x3a8)
319#define DFX_FIFO_DUMP_MSK (PORT_BASE + 0x3aC)
320#define DFX_FIFO_RD_DATA (PORT_BASE + 0x3b0)
321
322#define DEFAULT_ITCT_HW 2048 /* reset value, not reprogrammed */
323#if (HISI_SAS_MAX_DEVICES > DEFAULT_ITCT_HW)
324#error Max ITCT exceeded
325#endif
326
327#define AXI_MASTER_CFG_BASE (0x5000)
328#define AM_CTRL_GLOBAL (0x0)
329#define AM_CTRL_SHUTDOWN_REQ_OFF 0
330#define AM_CTRL_SHUTDOWN_REQ_MSK (0x1 << AM_CTRL_SHUTDOWN_REQ_OFF)
331#define AM_CURR_TRANS_RETURN (0x150)
332
333#define AM_CFG_MAX_TRANS (0x5010)
334#define AM_CFG_SINGLE_PORT_MAX_TRANS (0x5014)
335#define AXI_CFG (0x5100)
336#define AM_ROB_ECC_ERR_ADDR (0x510c)
337#define AM_ROB_ECC_ERR_ADDR_OFF 0
338#define AM_ROB_ECC_ERR_ADDR_MSK 0xffffffff
339
340/* RAS registers need init */
341#define RAS_BASE (0x6000)
342#define SAS_RAS_INTR0 (RAS_BASE)
343#define SAS_RAS_INTR1 (RAS_BASE + 0x04)
344#define SAS_RAS_INTR0_MASK (RAS_BASE + 0x08)
345#define SAS_RAS_INTR1_MASK (RAS_BASE + 0x0c)
346#define CFG_SAS_RAS_INTR_MASK (RAS_BASE + 0x1c)
347#define SAS_RAS_INTR2 (RAS_BASE + 0x20)
348#define SAS_RAS_INTR2_MASK (RAS_BASE + 0x24)
349
350/* HW dma structures */
351/* Delivery queue header */
352/* dw0 */
353#define CMD_HDR_ABORT_FLAG_OFF 0
354#define CMD_HDR_ABORT_FLAG_MSK (0x3 << CMD_HDR_ABORT_FLAG_OFF)
355#define CMD_HDR_ABORT_DEVICE_TYPE_OFF 2
356#define CMD_HDR_ABORT_DEVICE_TYPE_MSK (0x1 << CMD_HDR_ABORT_DEVICE_TYPE_OFF)
357#define CMD_HDR_RESP_REPORT_OFF 5
358#define CMD_HDR_RESP_REPORT_MSK (0x1 << CMD_HDR_RESP_REPORT_OFF)
359#define CMD_HDR_TLR_CTRL_OFF 6
360#define CMD_HDR_TLR_CTRL_MSK (0x3 << CMD_HDR_TLR_CTRL_OFF)
361#define CMD_HDR_PORT_OFF 18
362#define CMD_HDR_PORT_MSK (0xf << CMD_HDR_PORT_OFF)
363#define CMD_HDR_PRIORITY_OFF 27
364#define CMD_HDR_PRIORITY_MSK (0x1 << CMD_HDR_PRIORITY_OFF)
365#define CMD_HDR_CMD_OFF 29
366#define CMD_HDR_CMD_MSK (0x7 << CMD_HDR_CMD_OFF)
367/* dw1 */
368#define CMD_HDR_UNCON_CMD_OFF 3
369#define CMD_HDR_DIR_OFF 5
370#define CMD_HDR_DIR_MSK (0x3 << CMD_HDR_DIR_OFF)
371#define CMD_HDR_RESET_OFF 7
372#define CMD_HDR_RESET_MSK (0x1 << CMD_HDR_RESET_OFF)
373#define CMD_HDR_VDTL_OFF 10
374#define CMD_HDR_VDTL_MSK (0x1 << CMD_HDR_VDTL_OFF)
375#define CMD_HDR_FRAME_TYPE_OFF 11
376#define CMD_HDR_FRAME_TYPE_MSK (0x1f << CMD_HDR_FRAME_TYPE_OFF)
377#define CMD_HDR_DEV_ID_OFF 16
378#define CMD_HDR_DEV_ID_MSK (0xffff << CMD_HDR_DEV_ID_OFF)
379/* dw2 */
380#define CMD_HDR_CFL_OFF 0
381#define CMD_HDR_CFL_MSK (0x1ff << CMD_HDR_CFL_OFF)
382#define CMD_HDR_NCQ_TAG_OFF 10
383#define CMD_HDR_NCQ_TAG_MSK (0x1f << CMD_HDR_NCQ_TAG_OFF)
384#define CMD_HDR_MRFL_OFF 15
385#define CMD_HDR_MRFL_MSK (0x1ff << CMD_HDR_MRFL_OFF)
386#define CMD_HDR_SG_MOD_OFF 24
387#define CMD_HDR_SG_MOD_MSK (0x3 << CMD_HDR_SG_MOD_OFF)
388/* dw3 */
389#define CMD_HDR_IPTT_OFF 0
390#define CMD_HDR_IPTT_MSK (0xffff << CMD_HDR_IPTT_OFF)
391/* dw6 */
392#define CMD_HDR_DIF_SGL_LEN_OFF 0
393#define CMD_HDR_DIF_SGL_LEN_MSK (0xffff << CMD_HDR_DIF_SGL_LEN_OFF)
394#define CMD_HDR_DATA_SGL_LEN_OFF 16
395#define CMD_HDR_DATA_SGL_LEN_MSK (0xffff << CMD_HDR_DATA_SGL_LEN_OFF)
396/* dw7 */
397#define CMD_HDR_ADDR_MODE_SEL_OFF 15
398#define CMD_HDR_ADDR_MODE_SEL_MSK (1 << CMD_HDR_ADDR_MODE_SEL_OFF)
399#define CMD_HDR_ABORT_IPTT_OFF 16
400#define CMD_HDR_ABORT_IPTT_MSK (0xffff << CMD_HDR_ABORT_IPTT_OFF)
401
402/* Completion header */
403/* dw0 */
404#define CMPLT_HDR_CMPLT_OFF 0
405#define CMPLT_HDR_CMPLT_MSK (0x3 << CMPLT_HDR_CMPLT_OFF)
406#define CMPLT_HDR_ERROR_PHASE_OFF 2
407#define CMPLT_HDR_ERROR_PHASE_MSK (0xff << CMPLT_HDR_ERROR_PHASE_OFF)
408/* bit[9:2] Error Phase */
409#define ERR_PHASE_RESPONSE_FRAME_REV_STAGE_OFF \
410 8
411#define ERR_PHASE_RESPONSE_FRAME_REV_STAGE_MSK \
412 (0x1 << ERR_PHASE_RESPONSE_FRAME_REV_STAGE_OFF)
413#define CMPLT_HDR_RSPNS_XFRD_OFF 10
414#define CMPLT_HDR_RSPNS_XFRD_MSK (0x1 << CMPLT_HDR_RSPNS_XFRD_OFF)
415#define CMPLT_HDR_RSPNS_GOOD_OFF 11
416#define CMPLT_HDR_RSPNS_GOOD_MSK (0x1 << CMPLT_HDR_RSPNS_GOOD_OFF)
417#define CMPLT_HDR_ERX_OFF 12
418#define CMPLT_HDR_ERX_MSK (0x1 << CMPLT_HDR_ERX_OFF)
419#define CMPLT_HDR_ABORT_STAT_OFF 13
420#define CMPLT_HDR_ABORT_STAT_MSK (0x7 << CMPLT_HDR_ABORT_STAT_OFF)
421/* abort_stat */
422#define STAT_IO_NOT_VALID 0x1
423#define STAT_IO_NO_DEVICE 0x2
424#define STAT_IO_COMPLETE 0x3
425#define STAT_IO_ABORTED 0x4
426/* dw1 */
427#define CMPLT_HDR_IPTT_OFF 0
428#define CMPLT_HDR_IPTT_MSK (0xffff << CMPLT_HDR_IPTT_OFF)
429#define CMPLT_HDR_DEV_ID_OFF 16
430#define CMPLT_HDR_DEV_ID_MSK (0xffff << CMPLT_HDR_DEV_ID_OFF)
431/* dw3 */
432#define SATA_DISK_IN_ERROR_STATUS_OFF 8
433#define SATA_DISK_IN_ERROR_STATUS_MSK (0x1 << SATA_DISK_IN_ERROR_STATUS_OFF)
434#define CMPLT_HDR_SATA_DISK_ERR_OFF 16
435#define CMPLT_HDR_SATA_DISK_ERR_MSK (0x1 << CMPLT_HDR_SATA_DISK_ERR_OFF)
436#define CMPLT_HDR_IO_IN_TARGET_OFF 17
437#define CMPLT_HDR_IO_IN_TARGET_MSK (0x1 << CMPLT_HDR_IO_IN_TARGET_OFF)
438/* bit[23:18] ERR_FIS_ATA_STATUS */
439#define FIS_ATA_STATUS_ERR_OFF 18
440#define FIS_ATA_STATUS_ERR_MSK (0x1 << FIS_ATA_STATUS_ERR_OFF)
441#define FIS_TYPE_SDB_OFF 31
442#define FIS_TYPE_SDB_MSK (0x1 << FIS_TYPE_SDB_OFF)
443
444/* ITCT header */
445/* qw0 */
446#define ITCT_HDR_DEV_TYPE_OFF 0
447#define ITCT_HDR_DEV_TYPE_MSK (0x3 << ITCT_HDR_DEV_TYPE_OFF)
448#define ITCT_HDR_VALID_OFF 2
449#define ITCT_HDR_VALID_MSK (0x1 << ITCT_HDR_VALID_OFF)
450#define ITCT_HDR_MCR_OFF 5
451#define ITCT_HDR_MCR_MSK (0xf << ITCT_HDR_MCR_OFF)
452#define ITCT_HDR_VLN_OFF 9
453#define ITCT_HDR_VLN_MSK (0xf << ITCT_HDR_VLN_OFF)
454#define ITCT_HDR_SMP_TIMEOUT_OFF 16
455#define ITCT_HDR_AWT_CONTINUE_OFF 25
456#define ITCT_HDR_PORT_ID_OFF 28
457#define ITCT_HDR_PORT_ID_MSK (0xf << ITCT_HDR_PORT_ID_OFF)
458/* qw2 */
459#define ITCT_HDR_INLT_OFF 0
460#define ITCT_HDR_INLT_MSK (0xffffULL << ITCT_HDR_INLT_OFF)
461#define ITCT_HDR_RTOLT_OFF 48
462#define ITCT_HDR_RTOLT_MSK (0xffffULL << ITCT_HDR_RTOLT_OFF)
463
464struct hisi_sas_protect_iu_v3_hw {
465 u32 dw0;
466 u32 lbrtcv;
467 u32 lbrtgv;
468 u32 dw3;
469 u32 dw4;
470 u32 dw5;
471 u32 rsv;
472};
473
474struct hisi_sas_complete_v3_hdr {
475 __le32 dw0;
476 __le32 dw1;
477 __le32 act;
478 __le32 dw3;
479};
480
481struct hisi_sas_err_record_v3 {
482 /* dw0 */
483 __le32 trans_tx_fail_type;
484
485 /* dw1 */
486 __le32 trans_rx_fail_type;
487
488 /* dw2 */
489 __le16 dma_tx_err_type;
490 __le16 sipc_rx_err_type;
491
492 /* dw3 */
493 __le32 dma_rx_err_type;
494};
495
496#define RX_DATA_LEN_UNDERFLOW_OFF 6
497#define RX_DATA_LEN_UNDERFLOW_MSK (1 << RX_DATA_LEN_UNDERFLOW_OFF)
498
499#define RX_FIS_STATUS_ERR_OFF 0
500#define RX_FIS_STATUS_ERR_MSK (1 << RX_FIS_STATUS_ERR_OFF)
501
502#define HISI_SAS_COMMAND_ENTRIES_V3_HW 4096
503#define HISI_SAS_MSI_COUNT_V3_HW 32
504
505#define DIR_NO_DATA 0
506#define DIR_TO_INI 1
507#define DIR_TO_DEVICE 2
508#define DIR_RESERVED 3
509
510#define FIS_CMD_IS_UNCONSTRAINED(fis) \
511 ((fis.command == ATA_CMD_READ_LOG_EXT) || \
512 (fis.command == ATA_CMD_READ_LOG_DMA_EXT) || \
513 ((fis.command == ATA_CMD_DEV_RESET) && \
514 ((fis.control & ATA_SRST) != 0)))
515
516#define T10_INSRT_EN_OFF 0
517#define T10_INSRT_EN_MSK (1 << T10_INSRT_EN_OFF)
518#define T10_RMV_EN_OFF 1
519#define T10_RMV_EN_MSK (1 << T10_RMV_EN_OFF)
520#define T10_RPLC_EN_OFF 2
521#define T10_RPLC_EN_MSK (1 << T10_RPLC_EN_OFF)
522#define T10_CHK_EN_OFF 3
523#define T10_CHK_EN_MSK (1 << T10_CHK_EN_OFF)
524#define INCR_LBRT_OFF 5
525#define INCR_LBRT_MSK (1 << INCR_LBRT_OFF)
526#define USR_DATA_BLOCK_SZ_OFF 20
527#define USR_DATA_BLOCK_SZ_MSK (0x3 << USR_DATA_BLOCK_SZ_OFF)
528#define T10_CHK_MSK_OFF 16
529#define T10_CHK_REF_TAG_MSK (0xf0 << T10_CHK_MSK_OFF)
530#define T10_CHK_APP_TAG_MSK (0xc << T10_CHK_MSK_OFF)
531
532#define BASE_VECTORS_V3_HW 16
533#define MIN_AFFINE_VECTORS_V3_HW (BASE_VECTORS_V3_HW + 1)
534
535#define CHNL_INT_STS_MSK 0xeeeeeeee
536#define CHNL_INT_STS_PHY_MSK 0xe
537#define CHNL_INT_STS_INT0_MSK BIT(1)
538#define CHNL_INT_STS_INT1_MSK BIT(2)
539#define CHNL_INT_STS_INT2_MSK BIT(3)
540#define CHNL_WIDTH 4
541
542#define BAR_NO_V3_HW 5
543
544enum {
545 DSM_FUNC_ERR_HANDLE_MSI = 0,
546};
547
548static bool hisi_sas_intr_conv;
549MODULE_PARM_DESC(intr_conv, "interrupt converge enable (0-1)");
550
551/* permit overriding the host protection capabilities mask (EEDP/T10 PI) */
552static int prot_mask;
553module_param(prot_mask, int, 0444);
554MODULE_PARM_DESC(prot_mask, " host protection capabilities mask, def=0x0 ");
555
556/* the index of iopoll queues are bigger than interrupt queues' */
557static int experimental_iopoll_q_cnt;
558module_param(experimental_iopoll_q_cnt, int, 0444);
559MODULE_PARM_DESC(experimental_iopoll_q_cnt, "number of queues to be used as poll mode, def=0");
560
561static int debugfs_snapshot_regs_v3_hw(struct hisi_hba *hisi_hba);
562
563static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off)
564{
565 void __iomem *regs = hisi_hba->regs + off;
566
567 return readl(addr: regs);
568}
569
570static void hisi_sas_write32(struct hisi_hba *hisi_hba, u32 off, u32 val)
571{
572 void __iomem *regs = hisi_hba->regs + off;
573
574 writel(val, addr: regs);
575}
576
577static void hisi_sas_phy_write32(struct hisi_hba *hisi_hba, int phy_no,
578 u32 off, u32 val)
579{
580 void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
581
582 writel(val, addr: regs);
583}
584
585static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba,
586 int phy_no, u32 off)
587{
588 void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
589
590 return readl(addr: regs);
591}
592
593#define hisi_sas_read32_poll_timeout(off, val, cond, delay_us, \
594 timeout_us) \
595({ \
596 void __iomem *regs = hisi_hba->regs + off; \
597 readl_poll_timeout(regs, val, cond, delay_us, timeout_us); \
598})
599
600#define hisi_sas_read32_poll_timeout_atomic(off, val, cond, delay_us, \
601 timeout_us) \
602({ \
603 void __iomem *regs = hisi_hba->regs + off; \
604 readl_poll_timeout_atomic(regs, val, cond, delay_us, timeout_us);\
605})
606
607static void interrupt_enable_v3_hw(struct hisi_hba *hisi_hba)
608{
609 int i;
610
611 for (i = 0; i < hisi_hba->queue_count; i++)
612 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, val: 0);
613
614 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, val: 0xfefefefe);
615 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, val: 0xfefefefe);
616 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, val: 0xffc220ff);
617 hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, val: 0x155555);
618
619 for (i = 0; i < hisi_hba->n_phy; i++) {
620 hisi_sas_phy_write32(hisi_hba, phy_no: i, CHL_INT1_MSK, val: 0xf2057fff);
621 hisi_sas_phy_write32(hisi_hba, phy_no: i, CHL_INT2_MSK, val: 0xffffbfe);
622 hisi_sas_phy_write32(hisi_hba, phy_no: i, PHYCTRL_NOT_RDY_MSK, val: 0x0);
623 hisi_sas_phy_write32(hisi_hba, phy_no: i, PHYCTRL_PHY_ENA_MSK, val: 0x0);
624 hisi_sas_phy_write32(hisi_hba, phy_no: i, SL_RX_BCAST_CHK_MSK, val: 0x0);
625 }
626}
627
628static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
629{
630 struct pci_dev *pdev = hisi_hba->pci_dev;
631 int i, j;
632
633 /* Global registers init */
634 hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
635 val: (u32)((1ULL << hisi_hba->queue_count) - 1));
636 hisi_sas_write32(hisi_hba, CFG_MAX_TAG, val: 0xfff0400);
637 /* time / CLK_AHB = 2.5s / 2ns = 0x4A817C80 */
638 hisi_sas_write32(hisi_hba, TRANS_LOCK_ICT_TIME, val: 0x4A817C80);
639 hisi_sas_write32(hisi_hba, HGC_SAS_TXFAIL_RETRY_CTRL, val: 0x108);
640 hisi_sas_write32(hisi_hba, CFG_AGING_TIME, val: 0x1);
641 hisi_sas_write32(hisi_hba, INT_COAL_EN, val: 0x1);
642 hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME, val: 0x1);
643 hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT, val: 0x1);
644 hisi_sas_write32(hisi_hba, CQ_INT_CONVERGE_EN,
645 val: hisi_sas_intr_conv);
646 hisi_sas_write32(hisi_hba, OQ_INT_SRC, val: 0xffff);
647 hisi_sas_write32(hisi_hba, ENT_INT_SRC1, val: 0xffffffff);
648 hisi_sas_write32(hisi_hba, ENT_INT_SRC2, val: 0xffffffff);
649 hisi_sas_write32(hisi_hba, ENT_INT_SRC3, val: 0xffffffff);
650 hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, val: 0x0);
651 hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, val: 0x0);
652 hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, val: 0x0);
653 hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, val: 0xf0f0);
654 hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, val: 0xf0f0);
655 hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, val: 1);
656
657 if (pdev->revision < 0x30)
658 hisi_sas_write32(hisi_hba, SAS_AXI_USER3, val: 0);
659
660 interrupt_enable_v3_hw(hisi_hba);
661 for (i = 0; i < hisi_hba->n_phy; i++) {
662 enum sas_linkrate max;
663 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
664 struct asd_sas_phy *sas_phy = &phy->sas_phy;
665 u32 prog_phy_link_rate = hisi_sas_phy_read32(hisi_hba, phy_no: i,
666 PROG_PHY_LINK_RATE);
667
668 prog_phy_link_rate &= ~CFG_PROG_PHY_LINK_RATE_MSK;
669 if (!sas_phy->phy || (sas_phy->phy->maximum_linkrate <
670 SAS_LINK_RATE_1_5_GBPS))
671 max = SAS_LINK_RATE_12_0_GBPS;
672 else
673 max = sas_phy->phy->maximum_linkrate;
674 prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
675 hisi_sas_phy_write32(hisi_hba, phy_no: i, PROG_PHY_LINK_RATE,
676 val: prog_phy_link_rate);
677 hisi_sas_phy_write32(hisi_hba, phy_no: i, SAS_RX_TRAIN_TIMER, val: 0x13e80);
678 hisi_sas_phy_write32(hisi_hba, phy_no: i, CHL_INT0, val: 0xffffffff);
679 hisi_sas_phy_write32(hisi_hba, phy_no: i, CHL_INT1, val: 0xffffffff);
680 hisi_sas_phy_write32(hisi_hba, phy_no: i, CHL_INT2, val: 0xffffffff);
681 hisi_sas_phy_write32(hisi_hba, phy_no: i, RXOP_CHECK_CFG_H, val: 0x1000);
682 hisi_sas_phy_write32(hisi_hba, phy_no: i, PHY_CTRL_RDY_MSK, val: 0x0);
683 hisi_sas_phy_write32(hisi_hba, phy_no: i, PHYCTRL_DWS_RESET_MSK, val: 0x0);
684 hisi_sas_phy_write32(hisi_hba, phy_no: i, PHYCTRL_OOB_RESTART_MSK, val: 0x1);
685 hisi_sas_phy_write32(hisi_hba, phy_no: i, STP_LINK_TIMER, val: 0x7f7a120);
686 hisi_sas_phy_write32(hisi_hba, phy_no: i, CON_CFG_DRIVER, val: 0x2a0a01);
687 hisi_sas_phy_write32(hisi_hba, phy_no: i, SAS_EC_INT_COAL_TIME,
688 val: 0x30f4240);
689 hisi_sas_phy_write32(hisi_hba, phy_no: i, AIP_LIMIT, val: 0x2ffff);
690
691 /* set value through firmware for 920B and later version */
692 if (pdev->revision < 0x30) {
693 hisi_sas_phy_write32(hisi_hba, phy_no: i, SAS_SSP_CON_TIMER_CFG, val: 0x32);
694 hisi_sas_phy_write32(hisi_hba, phy_no: i, SERDES_CFG, val: 0xffc00);
695 /* used for 12G negotiate */
696 hisi_sas_phy_write32(hisi_hba, phy_no: i, COARSETUNE_TIME, val: 0x1e);
697 }
698
699 /* get default FFE configuration for BIST */
700 for (j = 0; j < FFE_CFG_MAX; j++) {
701 u32 val = hisi_sas_phy_read32(hisi_hba, phy_no: i,
702 TXDEEMPH_G1 + (j * 0x4));
703 hisi_hba->debugfs_bist_ffe[i][j] = val;
704 }
705 }
706
707 for (i = 0; i < hisi_hba->queue_count; i++) {
708 /* Delivery queue */
709 hisi_sas_write32(hisi_hba,
710 DLVRY_Q_0_BASE_ADDR_HI + (i * 0x14),
711 upper_32_bits(hisi_hba->cmd_hdr_dma[i]));
712
713 hisi_sas_write32(hisi_hba, DLVRY_Q_0_BASE_ADDR_LO + (i * 0x14),
714 lower_32_bits(hisi_hba->cmd_hdr_dma[i]));
715
716 hisi_sas_write32(hisi_hba, DLVRY_Q_0_DEPTH + (i * 0x14),
717 HISI_SAS_QUEUE_SLOTS);
718
719 /* Completion queue */
720 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_HI + (i * 0x14),
721 upper_32_bits(hisi_hba->complete_hdr_dma[i]));
722
723 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_LO + (i * 0x14),
724 lower_32_bits(hisi_hba->complete_hdr_dma[i]));
725
726 hisi_sas_write32(hisi_hba, COMPL_Q_0_DEPTH + (i * 0x14),
727 HISI_SAS_QUEUE_SLOTS);
728 }
729
730 /* itct */
731 hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_LO,
732 lower_32_bits(hisi_hba->itct_dma));
733
734 hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_HI,
735 upper_32_bits(hisi_hba->itct_dma));
736
737 /* iost */
738 hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_LO,
739 lower_32_bits(hisi_hba->iost_dma));
740
741 hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_HI,
742 upper_32_bits(hisi_hba->iost_dma));
743
744 /* breakpoint */
745 hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_LO,
746 lower_32_bits(hisi_hba->breakpoint_dma));
747
748 hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_HI,
749 upper_32_bits(hisi_hba->breakpoint_dma));
750
751 /* SATA broken msg */
752 hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_LO,
753 lower_32_bits(hisi_hba->sata_breakpoint_dma));
754
755 hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_HI,
756 upper_32_bits(hisi_hba->sata_breakpoint_dma));
757
758 /* SATA initial fis */
759 hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_LO,
760 lower_32_bits(hisi_hba->initial_fis_dma));
761
762 hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_HI,
763 upper_32_bits(hisi_hba->initial_fis_dma));
764
765 /* RAS registers init */
766 hisi_sas_write32(hisi_hba, SAS_RAS_INTR0_MASK, val: 0x0);
767 hisi_sas_write32(hisi_hba, SAS_RAS_INTR1_MASK, val: 0x0);
768 hisi_sas_write32(hisi_hba, SAS_RAS_INTR2_MASK, val: 0x0);
769 hisi_sas_write32(hisi_hba, CFG_SAS_RAS_INTR_MASK, val: 0x0);
770
771 /* LED registers init */
772 hisi_sas_write32(hisi_hba, SAS_CFG_DRIVE_VLD, val: 0x80000ff);
773 hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1, val: 0x80808080);
774 hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1 + 0x4, val: 0x80808080);
775 /* Configure blink generator rate A to 1Hz and B to 4Hz */
776 hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_1, val: 0x121700);
777 hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_0, val: 0x800000);
778}
779
780static void config_phy_opt_mode_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
781{
782 u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
783
784 cfg &= ~PHY_CFG_DC_OPT_MSK;
785 cfg |= 1 << PHY_CFG_DC_OPT_OFF;
786 hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, val: cfg);
787}
788
789static void config_id_frame_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
790{
791 struct sas_identify_frame identify_frame;
792 u32 *identify_buffer;
793
794 memset(&identify_frame, 0, sizeof(identify_frame));
795 identify_frame.dev_type = SAS_END_DEVICE;
796 identify_frame.frame_type = 0;
797 identify_frame._un1 = 1;
798 identify_frame.initiator_bits = SAS_PROTOCOL_ALL;
799 identify_frame.target_bits = SAS_PROTOCOL_NONE;
800 memcpy(&identify_frame._un4_11[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
801 memcpy(&identify_frame.sas_addr[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
802 identify_frame.phy_id = phy_no;
803 identify_buffer = (u32 *)(&identify_frame);
804
805 hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD0,
806 __swab32(identify_buffer[0]));
807 hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD1,
808 __swab32(identify_buffer[1]));
809 hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD2,
810 __swab32(identify_buffer[2]));
811 hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD3,
812 __swab32(identify_buffer[3]));
813 hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD4,
814 __swab32(identify_buffer[4]));
815 hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD5,
816 __swab32(identify_buffer[5]));
817}
818
819static void setup_itct_v3_hw(struct hisi_hba *hisi_hba,
820 struct hisi_sas_device *sas_dev)
821{
822 struct domain_device *device = sas_dev->sas_device;
823 struct device *dev = hisi_hba->dev;
824 u64 qw0, device_id = sas_dev->device_id;
825 struct hisi_sas_itct *itct = &hisi_hba->itct[device_id];
826 struct domain_device *parent_dev = device->parent;
827 struct asd_sas_port *sas_port = device->port;
828 struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
829 u64 sas_addr;
830
831 memset(itct, 0, sizeof(*itct));
832
833 /* qw0 */
834 qw0 = 0;
835 switch (sas_dev->dev_type) {
836 case SAS_END_DEVICE:
837 case SAS_EDGE_EXPANDER_DEVICE:
838 case SAS_FANOUT_EXPANDER_DEVICE:
839 qw0 = HISI_SAS_DEV_TYPE_SSP << ITCT_HDR_DEV_TYPE_OFF;
840 break;
841 case SAS_SATA_DEV:
842 case SAS_SATA_PENDING:
843 if (parent_dev && dev_is_expander(type: parent_dev->dev_type))
844 qw0 = HISI_SAS_DEV_TYPE_STP << ITCT_HDR_DEV_TYPE_OFF;
845 else
846 qw0 = HISI_SAS_DEV_TYPE_SATA << ITCT_HDR_DEV_TYPE_OFF;
847 break;
848 default:
849 dev_warn(dev, "setup itct: unsupported dev type (%d)\n",
850 sas_dev->dev_type);
851 }
852
853 qw0 |= ((1 << ITCT_HDR_VALID_OFF) |
854 (device->linkrate << ITCT_HDR_MCR_OFF) |
855 (1 << ITCT_HDR_VLN_OFF) |
856 (0xfa << ITCT_HDR_SMP_TIMEOUT_OFF) |
857 (1 << ITCT_HDR_AWT_CONTINUE_OFF) |
858 (port->id << ITCT_HDR_PORT_ID_OFF));
859 itct->qw0 = cpu_to_le64(qw0);
860
861 /* qw1 */
862 memcpy(&sas_addr, device->sas_addr, SAS_ADDR_SIZE);
863 itct->sas_addr = cpu_to_le64(__swab64(sas_addr));
864
865 /* qw2 */
866 if (!dev_is_sata(dev: device))
867 itct->qw2 = cpu_to_le64((5000ULL << ITCT_HDR_INLT_OFF) |
868 (0x1ULL << ITCT_HDR_RTOLT_OFF));
869}
870
871static int clear_itct_v3_hw(struct hisi_hba *hisi_hba,
872 struct hisi_sas_device *sas_dev)
873{
874 DECLARE_COMPLETION_ONSTACK(completion);
875 u64 dev_id = sas_dev->device_id;
876 struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id];
877 u32 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
878 struct device *dev = hisi_hba->dev;
879
880 sas_dev->completion = &completion;
881
882 /* clear the itct interrupt state */
883 if (ENT_INT_SRC3_ITC_INT_MSK & reg_val)
884 hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
885 ENT_INT_SRC3_ITC_INT_MSK);
886
887 /* clear the itct table */
888 reg_val = ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK);
889 hisi_sas_write32(hisi_hba, ITCT_CLR, val: reg_val);
890
891 if (!wait_for_completion_timeout(x: sas_dev->completion,
892 HISI_SAS_CLEAR_ITCT_TIMEOUT)) {
893 dev_warn(dev, "failed to clear ITCT\n");
894 return -ETIMEDOUT;
895 }
896
897 memset(itct, 0, sizeof(struct hisi_sas_itct));
898 return 0;
899}
900
901static void dereg_device_v3_hw(struct hisi_hba *hisi_hba,
902 struct domain_device *device)
903{
904 struct hisi_sas_slot *slot, *slot2;
905 struct hisi_sas_device *sas_dev = device->lldd_dev;
906 u32 cfg_abt_set_query_iptt;
907
908 cfg_abt_set_query_iptt = hisi_sas_read32(hisi_hba,
909 CFG_ABT_SET_QUERY_IPTT);
910 spin_lock(lock: &sas_dev->lock);
911 list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry) {
912 cfg_abt_set_query_iptt &= ~CFG_SET_ABORTED_IPTT_MSK;
913 cfg_abt_set_query_iptt |= (1 << CFG_SET_ABORTED_EN_OFF) |
914 (slot->idx << CFG_SET_ABORTED_IPTT_OFF);
915 hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
916 val: cfg_abt_set_query_iptt);
917 }
918 spin_unlock(lock: &sas_dev->lock);
919 cfg_abt_set_query_iptt &= ~(1 << CFG_SET_ABORTED_EN_OFF);
920 hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
921 val: cfg_abt_set_query_iptt);
922 hisi_sas_write32(hisi_hba, CFG_ABT_SET_IPTT_DONE,
923 val: 1 << CFG_ABT_SET_IPTT_DONE_OFF);
924}
925
926static int reset_hw_v3_hw(struct hisi_hba *hisi_hba)
927{
928 struct device *dev = hisi_hba->dev;
929 int ret;
930 u32 val;
931
932 hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, val: 0);
933
934 /* Disable all of the PHYs */
935 hisi_sas_stop_phys(hisi_hba);
936 udelay(50);
937
938 /* Ensure axi bus idle */
939 ret = hisi_sas_read32_poll_timeout(AXI_CFG, val, !val,
940 20000, 1000000);
941 if (ret) {
942 dev_err(dev, "axi bus is not idle, ret = %d!\n", ret);
943 return -EIO;
944 }
945
946 if (ACPI_HANDLE(dev)) {
947 acpi_status s;
948
949 s = acpi_evaluate_object(ACPI_HANDLE(dev), pathname: "_RST", NULL, NULL);
950 if (ACPI_FAILURE(s)) {
951 dev_err(dev, "Reset failed\n");
952 return -EIO;
953 }
954 } else {
955 dev_err(dev, "no reset method!\n");
956 return -EINVAL;
957 }
958
959 return 0;
960}
961
962static int hw_init_v3_hw(struct hisi_hba *hisi_hba)
963{
964 struct device *dev = hisi_hba->dev;
965 struct acpi_device *acpi_dev;
966 union acpi_object *obj;
967 guid_t guid;
968 int rc;
969
970 rc = reset_hw_v3_hw(hisi_hba);
971 if (rc) {
972 dev_err(dev, "hisi_sas_reset_hw failed, rc=%d\n", rc);
973 return rc;
974 }
975
976 msleep(msecs: 100);
977 init_reg_v3_hw(hisi_hba);
978
979 if (guid_parse(uuid: "D5918B4B-37AE-4E10-A99F-E5E8A6EF4C1F", u: &guid)) {
980 dev_err(dev, "Parse GUID failed\n");
981 return -EINVAL;
982 }
983
984 /*
985 * This DSM handles some hardware-related configurations:
986 * 1. Switch over to MSI error handling in kernel
987 * 2. BIOS *may* reset some register values through this method
988 */
989 obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), guid: &guid, rev: 0,
990 func: DSM_FUNC_ERR_HANDLE_MSI, NULL);
991 if (!obj)
992 dev_warn(dev, "can not find DSM method, ignore\n");
993 else
994 ACPI_FREE(obj);
995
996 acpi_dev = ACPI_COMPANION(dev);
997 if (!acpi_device_power_manageable(adev: acpi_dev))
998 dev_notice(dev, "neither _PS0 nor _PR0 is defined\n");
999 return 0;
1000}
1001
1002static void enable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1003{
1004 u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
1005
1006 cfg |= PHY_CFG_ENA_MSK;
1007 cfg &= ~PHY_CFG_PHY_RST_MSK;
1008 hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, val: cfg);
1009}
1010
1011static void disable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1012{
1013 u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
1014 u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2_MSK);
1015 static const u32 msk = BIT(CHL_INT2_RX_DISP_ERR_OFF) |
1016 BIT(CHL_INT2_RX_CODE_ERR_OFF) |
1017 BIT(CHL_INT2_RX_INVLD_DW_OFF);
1018 u32 state;
1019
1020 hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2_MSK, val: msk | irq_msk);
1021
1022 cfg &= ~PHY_CFG_ENA_MSK;
1023 hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, val: cfg);
1024
1025 mdelay(50);
1026
1027 state = hisi_sas_read32(hisi_hba, PHY_STATE);
1028 if (state & BIT(phy_no)) {
1029 cfg |= PHY_CFG_PHY_RST_MSK;
1030 hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, val: cfg);
1031 }
1032
1033 udelay(1);
1034
1035 hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW);
1036 hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR);
1037 hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_CODE_ERR);
1038
1039 hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2, val: msk);
1040 hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2_MSK, val: irq_msk);
1041}
1042
1043static void start_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1044{
1045 config_id_frame_v3_hw(hisi_hba, phy_no);
1046 config_phy_opt_mode_v3_hw(hisi_hba, phy_no);
1047 enable_phy_v3_hw(hisi_hba, phy_no);
1048}
1049
1050static void phy_hard_reset_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1051{
1052 struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1053 u32 txid_auto;
1054
1055 hisi_sas_phy_enable(hisi_hba, phy_no, enable: 0);
1056 if (phy->identify.device_type == SAS_END_DEVICE) {
1057 txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
1058 hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
1059 val: txid_auto | TX_HARDRST_MSK);
1060 }
1061 msleep(msecs: 100);
1062 hisi_sas_phy_enable(hisi_hba, phy_no, enable: 1);
1063}
1064
1065static enum sas_linkrate phy_get_max_linkrate_v3_hw(void)
1066{
1067 return SAS_LINK_RATE_12_0_GBPS;
1068}
1069
1070static void phys_init_v3_hw(struct hisi_hba *hisi_hba)
1071{
1072 int i;
1073
1074 for (i = 0; i < hisi_hba->n_phy; i++) {
1075 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
1076 struct asd_sas_phy *sas_phy = &phy->sas_phy;
1077
1078 if (!sas_phy->phy->enabled)
1079 continue;
1080
1081 hisi_sas_phy_enable(hisi_hba, phy_no: i, enable: 1);
1082 }
1083}
1084
1085static void sl_notify_ssp_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1086{
1087 u32 sl_control;
1088
1089 sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1090 sl_control |= SL_CONTROL_NOTIFY_EN_MSK;
1091 hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, val: sl_control);
1092 msleep(msecs: 1);
1093 sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1094 sl_control &= ~SL_CONTROL_NOTIFY_EN_MSK;
1095 hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, val: sl_control);
1096}
1097
1098static int get_wideport_bitmap_v3_hw(struct hisi_hba *hisi_hba, int port_id)
1099{
1100 int i, bitmap = 0;
1101 u32 phy_port_num_ma = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
1102 u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1103
1104 for (i = 0; i < hisi_hba->n_phy; i++)
1105 if (phy_state & BIT(i))
1106 if (((phy_port_num_ma >> (i * 4)) & 0xf) == port_id)
1107 bitmap |= BIT(i);
1108
1109 return bitmap;
1110}
1111
1112static void start_delivery_v3_hw(struct hisi_sas_dq *dq)
1113{
1114 struct hisi_hba *hisi_hba = dq->hisi_hba;
1115 struct hisi_sas_slot *s, *s1, *s2 = NULL;
1116 int dlvry_queue = dq->id;
1117 int wp;
1118
1119 list_for_each_entry_safe(s, s1, &dq->list, delivery) {
1120 if (!s->ready)
1121 break;
1122 s2 = s;
1123 list_del(entry: &s->delivery);
1124 }
1125
1126 if (!s2)
1127 return;
1128
1129 /*
1130 * Ensure that memories for slots built on other CPUs is observed.
1131 */
1132 smp_rmb();
1133 wp = (s2->dlvry_queue_slot + 1) % HISI_SAS_QUEUE_SLOTS;
1134
1135 hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), val: wp);
1136}
1137
1138static void prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
1139 struct hisi_sas_slot *slot,
1140 struct hisi_sas_cmd_hdr *hdr,
1141 struct scatterlist *scatter,
1142 int n_elem)
1143{
1144 struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
1145 struct scatterlist *sg;
1146 int i;
1147
1148 for_each_sg(scatter, sg, n_elem, i) {
1149 struct hisi_sas_sge *entry = &sge_page->sge[i];
1150
1151 entry->addr = cpu_to_le64(sg_dma_address(sg));
1152 entry->page_ctrl_0 = entry->page_ctrl_1 = 0;
1153 entry->data_len = cpu_to_le32(sg_dma_len(sg));
1154 entry->data_off = 0;
1155 }
1156
1157 hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
1158
1159 hdr->sg_len |= cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
1160}
1161
1162static void prep_prd_sge_dif_v3_hw(struct hisi_hba *hisi_hba,
1163 struct hisi_sas_slot *slot,
1164 struct hisi_sas_cmd_hdr *hdr,
1165 struct scatterlist *scatter,
1166 int n_elem)
1167{
1168 struct hisi_sas_sge_dif_page *sge_dif_page;
1169 struct scatterlist *sg;
1170 int i;
1171
1172 sge_dif_page = hisi_sas_sge_dif_addr_mem(slot);
1173
1174 for_each_sg(scatter, sg, n_elem, i) {
1175 struct hisi_sas_sge *entry = &sge_dif_page->sge[i];
1176
1177 entry->addr = cpu_to_le64(sg_dma_address(sg));
1178 entry->page_ctrl_0 = 0;
1179 entry->page_ctrl_1 = 0;
1180 entry->data_len = cpu_to_le32(sg_dma_len(sg));
1181 entry->data_off = 0;
1182 }
1183
1184 hdr->dif_prd_table_addr =
1185 cpu_to_le64(hisi_sas_sge_dif_addr_dma(slot));
1186
1187 hdr->sg_len |= cpu_to_le32(n_elem << CMD_HDR_DIF_SGL_LEN_OFF);
1188}
1189
1190static u32 get_prot_chk_msk_v3_hw(struct scsi_cmnd *scsi_cmnd)
1191{
1192 unsigned char prot_flags = scsi_cmnd->prot_flags;
1193
1194 if (prot_flags & SCSI_PROT_REF_CHECK)
1195 return T10_CHK_APP_TAG_MSK;
1196 return T10_CHK_REF_TAG_MSK | T10_CHK_APP_TAG_MSK;
1197}
1198
1199static void fill_prot_v3_hw(struct scsi_cmnd *scsi_cmnd,
1200 struct hisi_sas_protect_iu_v3_hw *prot)
1201{
1202 unsigned char prot_op = scsi_get_prot_op(scmd: scsi_cmnd);
1203 unsigned int interval = scsi_prot_interval(scmd: scsi_cmnd);
1204 u32 lbrt_chk_val = t10_pi_ref_tag(rq: scsi_cmd_to_rq(scmd: scsi_cmnd));
1205
1206 switch (prot_op) {
1207 case SCSI_PROT_READ_INSERT:
1208 prot->dw0 |= T10_INSRT_EN_MSK;
1209 prot->lbrtgv = lbrt_chk_val;
1210 break;
1211 case SCSI_PROT_READ_STRIP:
1212 prot->dw0 |= (T10_RMV_EN_MSK | T10_CHK_EN_MSK);
1213 prot->lbrtcv = lbrt_chk_val;
1214 prot->dw4 |= get_prot_chk_msk_v3_hw(scsi_cmnd);
1215 break;
1216 case SCSI_PROT_READ_PASS:
1217 prot->dw0 |= T10_CHK_EN_MSK;
1218 prot->lbrtcv = lbrt_chk_val;
1219 prot->dw4 |= get_prot_chk_msk_v3_hw(scsi_cmnd);
1220 break;
1221 case SCSI_PROT_WRITE_INSERT:
1222 prot->dw0 |= T10_INSRT_EN_MSK;
1223 prot->lbrtgv = lbrt_chk_val;
1224 break;
1225 case SCSI_PROT_WRITE_STRIP:
1226 prot->dw0 |= (T10_RMV_EN_MSK | T10_CHK_EN_MSK);
1227 prot->lbrtcv = lbrt_chk_val;
1228 break;
1229 case SCSI_PROT_WRITE_PASS:
1230 prot->dw0 |= T10_CHK_EN_MSK;
1231 prot->lbrtcv = lbrt_chk_val;
1232 prot->dw4 |= get_prot_chk_msk_v3_hw(scsi_cmnd);
1233 break;
1234 default:
1235 WARN(1, "prot_op(0x%x) is not valid\n", prot_op);
1236 break;
1237 }
1238
1239 switch (interval) {
1240 case 512:
1241 break;
1242 case 4096:
1243 prot->dw0 |= (0x1 << USR_DATA_BLOCK_SZ_OFF);
1244 break;
1245 case 520:
1246 prot->dw0 |= (0x2 << USR_DATA_BLOCK_SZ_OFF);
1247 break;
1248 default:
1249 WARN(1, "protection interval (0x%x) invalid\n",
1250 interval);
1251 break;
1252 }
1253
1254 prot->dw0 |= INCR_LBRT_MSK;
1255}
1256
1257static void prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
1258 struct hisi_sas_slot *slot)
1259{
1260 struct sas_task *task = slot->task;
1261 struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1262 struct domain_device *device = task->dev;
1263 struct hisi_sas_device *sas_dev = device->lldd_dev;
1264 struct hisi_sas_port *port = slot->port;
1265 struct sas_ssp_task *ssp_task = &task->ssp_task;
1266 struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
1267 struct sas_tmf_task *tmf = slot->tmf;
1268 int has_data = 0, priority = !!tmf;
1269 unsigned char prot_op;
1270 u8 *buf_cmd;
1271 u32 dw1 = 0, dw2 = 0, len = 0;
1272
1273 hdr->dw0 = cpu_to_le32((1 << CMD_HDR_RESP_REPORT_OFF) |
1274 (2 << CMD_HDR_TLR_CTRL_OFF) |
1275 (port->id << CMD_HDR_PORT_OFF) |
1276 (priority << CMD_HDR_PRIORITY_OFF) |
1277 (1 << CMD_HDR_CMD_OFF)); /* ssp */
1278
1279 dw1 = 1 << CMD_HDR_VDTL_OFF;
1280 if (tmf) {
1281 dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF;
1282 dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF;
1283 } else {
1284 prot_op = scsi_get_prot_op(scmd: scsi_cmnd);
1285 dw1 |= 1 << CMD_HDR_FRAME_TYPE_OFF;
1286 switch (scsi_cmnd->sc_data_direction) {
1287 case DMA_TO_DEVICE:
1288 has_data = 1;
1289 dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
1290 break;
1291 case DMA_FROM_DEVICE:
1292 has_data = 1;
1293 dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
1294 break;
1295 default:
1296 dw1 &= ~CMD_HDR_DIR_MSK;
1297 }
1298 }
1299
1300 /* map itct entry */
1301 dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
1302
1303 dw2 = (((sizeof(struct ssp_command_iu) + sizeof(struct ssp_frame_hdr)
1304 + 3) / 4) << CMD_HDR_CFL_OFF) |
1305 ((HISI_SAS_MAX_SSP_RESP_SZ / 4) << CMD_HDR_MRFL_OFF) |
1306 (2 << CMD_HDR_SG_MOD_OFF);
1307 hdr->dw2 = cpu_to_le32(dw2);
1308 hdr->transfer_tags = cpu_to_le32(slot->idx);
1309
1310 if (has_data) {
1311 prep_prd_sge_v3_hw(hisi_hba, slot, hdr, scatter: task->scatter,
1312 n_elem: slot->n_elem);
1313
1314 if (scsi_prot_sg_count(cmd: scsi_cmnd))
1315 prep_prd_sge_dif_v3_hw(hisi_hba, slot, hdr,
1316 scatter: scsi_prot_sglist(cmd: scsi_cmnd),
1317 n_elem: slot->n_elem_dif);
1318 }
1319
1320 hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
1321 hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1322
1323 buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot) +
1324 sizeof(struct ssp_frame_hdr);
1325
1326 memcpy(buf_cmd, &task->ssp_task.LUN, 8);
1327 if (!tmf) {
1328 buf_cmd[9] = ssp_task->task_attr;
1329 memcpy(buf_cmd + 12, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
1330 } else {
1331 buf_cmd[10] = tmf->tmf;
1332 switch (tmf->tmf) {
1333 case TMF_ABORT_TASK:
1334 case TMF_QUERY_TASK:
1335 buf_cmd[12] =
1336 (tmf->tag_of_task_to_be_managed >> 8) & 0xff;
1337 buf_cmd[13] =
1338 tmf->tag_of_task_to_be_managed & 0xff;
1339 break;
1340 default:
1341 break;
1342 }
1343 }
1344
1345 if (has_data && (prot_op != SCSI_PROT_NORMAL)) {
1346 struct hisi_sas_protect_iu_v3_hw prot;
1347 u8 *buf_cmd_prot;
1348
1349 hdr->dw7 |= cpu_to_le32(1 << CMD_HDR_ADDR_MODE_SEL_OFF);
1350 dw1 |= CMD_HDR_PIR_MSK;
1351 buf_cmd_prot = hisi_sas_cmd_hdr_addr_mem(slot) +
1352 sizeof(struct ssp_frame_hdr) +
1353 sizeof(struct ssp_command_iu);
1354
1355 memset(&prot, 0, sizeof(struct hisi_sas_protect_iu_v3_hw));
1356 fill_prot_v3_hw(scsi_cmnd, prot: &prot);
1357 memcpy(buf_cmd_prot, &prot,
1358 sizeof(struct hisi_sas_protect_iu_v3_hw));
1359 /*
1360 * For READ, we need length of info read to memory, while for
1361 * WRITE we need length of data written to the disk.
1362 */
1363 if (prot_op == SCSI_PROT_WRITE_INSERT ||
1364 prot_op == SCSI_PROT_READ_INSERT ||
1365 prot_op == SCSI_PROT_WRITE_PASS ||
1366 prot_op == SCSI_PROT_READ_PASS) {
1367 unsigned int interval = scsi_prot_interval(scmd: scsi_cmnd);
1368 unsigned int ilog2_interval = ilog2(interval);
1369
1370 len = (task->total_xfer_len >> ilog2_interval) * 8;
1371 }
1372 }
1373
1374 hdr->dw1 = cpu_to_le32(dw1);
1375
1376 hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len + len);
1377}
1378
1379static void prep_smp_v3_hw(struct hisi_hba *hisi_hba,
1380 struct hisi_sas_slot *slot)
1381{
1382 struct sas_task *task = slot->task;
1383 struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1384 struct domain_device *device = task->dev;
1385 struct hisi_sas_port *port = slot->port;
1386 struct scatterlist *sg_req;
1387 struct hisi_sas_device *sas_dev = device->lldd_dev;
1388 dma_addr_t req_dma_addr;
1389 unsigned int req_len;
1390
1391 /* req */
1392 sg_req = &task->smp_task.smp_req;
1393 req_len = sg_dma_len(sg_req);
1394 req_dma_addr = sg_dma_address(sg_req);
1395
1396 /* create header */
1397 /* dw0 */
1398 hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) |
1399 (1 << CMD_HDR_PRIORITY_OFF) | /* high pri */
1400 (2 << CMD_HDR_CMD_OFF)); /* smp */
1401
1402 /* map itct entry */
1403 hdr->dw1 = cpu_to_le32((sas_dev->device_id << CMD_HDR_DEV_ID_OFF) |
1404 (1 << CMD_HDR_FRAME_TYPE_OFF) |
1405 (DIR_NO_DATA << CMD_HDR_DIR_OFF));
1406
1407 /* dw2 */
1408 hdr->dw2 = cpu_to_le32((((req_len - 4) / 4) << CMD_HDR_CFL_OFF) |
1409 (HISI_SAS_MAX_SMP_RESP_SZ / 4 <<
1410 CMD_HDR_MRFL_OFF));
1411
1412 hdr->transfer_tags = cpu_to_le32(slot->idx << CMD_HDR_IPTT_OFF);
1413
1414 hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
1415 hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1416}
1417
1418static void prep_ata_v3_hw(struct hisi_hba *hisi_hba,
1419 struct hisi_sas_slot *slot)
1420{
1421 struct sas_task *task = slot->task;
1422 struct domain_device *device = task->dev;
1423 struct domain_device *parent_dev = device->parent;
1424 struct hisi_sas_device *sas_dev = device->lldd_dev;
1425 struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1426 struct asd_sas_port *sas_port = device->port;
1427 struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
1428 u8 *buf_cmd;
1429 int has_data = 0, hdr_tag = 0;
1430 u32 dw1 = 0, dw2 = 0;
1431
1432 hdr->dw0 = cpu_to_le32(port->id << CMD_HDR_PORT_OFF);
1433 if (parent_dev && dev_is_expander(type: parent_dev->dev_type))
1434 hdr->dw0 |= cpu_to_le32(3 << CMD_HDR_CMD_OFF);
1435 else
1436 hdr->dw0 |= cpu_to_le32(4U << CMD_HDR_CMD_OFF);
1437
1438 switch (task->data_dir) {
1439 case DMA_TO_DEVICE:
1440 has_data = 1;
1441 dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
1442 break;
1443 case DMA_FROM_DEVICE:
1444 has_data = 1;
1445 dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
1446 break;
1447 default:
1448 dw1 &= ~CMD_HDR_DIR_MSK;
1449 }
1450
1451 if ((task->ata_task.fis.command == ATA_CMD_DEV_RESET) &&
1452 (task->ata_task.fis.control & ATA_SRST))
1453 dw1 |= 1 << CMD_HDR_RESET_OFF;
1454
1455 dw1 |= (hisi_sas_get_ata_protocol(
1456 fis: &task->ata_task.fis, direction: task->data_dir))
1457 << CMD_HDR_FRAME_TYPE_OFF;
1458 dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
1459
1460 if (FIS_CMD_IS_UNCONSTRAINED(task->ata_task.fis))
1461 dw1 |= 1 << CMD_HDR_UNCON_CMD_OFF;
1462
1463 hdr->dw1 = cpu_to_le32(dw1);
1464
1465 /* dw2 */
1466 if (task->ata_task.use_ncq) {
1467 struct ata_queued_cmd *qc = task->uldd_task;
1468
1469 hdr_tag = qc->tag;
1470 task->ata_task.fis.sector_count |= (u8) (hdr_tag << 3);
1471 dw2 |= hdr_tag << CMD_HDR_NCQ_TAG_OFF;
1472 }
1473
1474 dw2 |= (HISI_SAS_MAX_STP_RESP_SZ / 4) << CMD_HDR_CFL_OFF |
1475 2 << CMD_HDR_SG_MOD_OFF;
1476 hdr->dw2 = cpu_to_le32(dw2);
1477
1478 /* dw3 */
1479 hdr->transfer_tags = cpu_to_le32(slot->idx);
1480
1481 if (has_data)
1482 prep_prd_sge_v3_hw(hisi_hba, slot, hdr, scatter: task->scatter,
1483 n_elem: slot->n_elem);
1484
1485 hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
1486 hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
1487 hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1488
1489 buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot);
1490
1491 if (likely(!task->ata_task.device_control_reg_update))
1492 task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
1493 /* fill in command FIS */
1494 memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
1495}
1496
1497static void prep_abort_v3_hw(struct hisi_hba *hisi_hba,
1498 struct hisi_sas_slot *slot)
1499{
1500 struct sas_task *task = slot->task;
1501 struct sas_internal_abort_task *abort = &task->abort_task;
1502 struct domain_device *dev = task->dev;
1503 struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1504 struct hisi_sas_port *port = slot->port;
1505 struct hisi_sas_device *sas_dev = dev->lldd_dev;
1506 bool sata = dev_is_sata(dev);
1507
1508 /* dw0 */
1509 hdr->dw0 = cpu_to_le32((5U << CMD_HDR_CMD_OFF) | /* abort */
1510 (port->id << CMD_HDR_PORT_OFF) |
1511 (sata << CMD_HDR_ABORT_DEVICE_TYPE_OFF) |
1512 (abort->type << CMD_HDR_ABORT_FLAG_OFF));
1513
1514 /* dw1 */
1515 hdr->dw1 = cpu_to_le32(sas_dev->device_id
1516 << CMD_HDR_DEV_ID_OFF);
1517
1518 /* dw7 */
1519 hdr->dw7 = cpu_to_le32(abort->tag << CMD_HDR_ABORT_IPTT_OFF);
1520 hdr->transfer_tags = cpu_to_le32(slot->idx);
1521}
1522
1523static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1524{
1525 int i;
1526 irqreturn_t res;
1527 u32 context, port_id, link_rate;
1528 struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1529 struct asd_sas_phy *sas_phy = &phy->sas_phy;
1530 struct device *dev = hisi_hba->dev;
1531
1532 hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, val: 1);
1533
1534 port_id = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
1535 port_id = (port_id >> (4 * phy_no)) & 0xf;
1536 link_rate = hisi_sas_read32(hisi_hba, PHY_CONN_RATE);
1537 link_rate = (link_rate >> (phy_no * 4)) & 0xf;
1538
1539 if (port_id == 0xf) {
1540 dev_err(dev, "phyup: phy%d invalid portid\n", phy_no);
1541 res = IRQ_NONE;
1542 goto end;
1543 }
1544 sas_phy->linkrate = link_rate;
1545 phy->phy_type &= ~(PORT_TYPE_SAS | PORT_TYPE_SATA);
1546
1547 /* Check for SATA dev */
1548 context = hisi_sas_read32(hisi_hba, PHY_CONTEXT);
1549 if (context & (1 << phy_no)) {
1550 struct hisi_sas_initial_fis *initial_fis;
1551 struct dev_to_host_fis *fis;
1552 u8 attached_sas_addr[SAS_ADDR_SIZE] = {0};
1553 struct Scsi_Host *shost = hisi_hba->shost;
1554
1555 dev_info(dev, "phyup: phy%d link_rate=%d(sata)\n", phy_no, link_rate);
1556 initial_fis = &hisi_hba->initial_fis[phy_no];
1557 fis = &initial_fis->fis;
1558
1559 /* check ERR bit of Status Register */
1560 if (fis->status & ATA_ERR) {
1561 dev_warn(dev, "sata int: phy%d FIS status: 0x%x\n",
1562 phy_no, fis->status);
1563 hisi_sas_notify_phy_event(phy, event: HISI_PHYE_LINK_RESET);
1564 res = IRQ_NONE;
1565 goto end;
1566 }
1567
1568 sas_phy->oob_mode = SATA_OOB_MODE;
1569 attached_sas_addr[0] = 0x50;
1570 attached_sas_addr[6] = shost->host_no;
1571 attached_sas_addr[7] = phy_no;
1572 memcpy(sas_phy->attached_sas_addr,
1573 attached_sas_addr,
1574 SAS_ADDR_SIZE);
1575 memcpy(sas_phy->frame_rcvd, fis,
1576 sizeof(struct dev_to_host_fis));
1577 phy->phy_type |= PORT_TYPE_SATA;
1578 phy->identify.device_type = SAS_SATA_DEV;
1579 phy->frame_rcvd_size = sizeof(struct dev_to_host_fis);
1580 phy->identify.target_port_protocols = SAS_PROTOCOL_SATA;
1581 } else {
1582 u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd;
1583 struct sas_identify_frame *id =
1584 (struct sas_identify_frame *)frame_rcvd;
1585
1586 dev_info(dev, "phyup: phy%d link_rate=%d\n", phy_no, link_rate);
1587 for (i = 0; i < 6; i++) {
1588 u32 idaf = hisi_sas_phy_read32(hisi_hba, phy_no,
1589 RX_IDAF_DWORD0 + (i * 4));
1590 frame_rcvd[i] = __swab32(idaf);
1591 }
1592 sas_phy->oob_mode = SAS_OOB_MODE;
1593 memcpy(sas_phy->attached_sas_addr,
1594 &id->sas_addr,
1595 SAS_ADDR_SIZE);
1596 phy->phy_type |= PORT_TYPE_SAS;
1597 phy->identify.device_type = id->dev_type;
1598 phy->frame_rcvd_size = sizeof(struct sas_identify_frame);
1599 if (phy->identify.device_type == SAS_END_DEVICE)
1600 phy->identify.target_port_protocols =
1601 SAS_PROTOCOL_SSP;
1602 else if (phy->identify.device_type != SAS_PHY_UNUSED)
1603 phy->identify.target_port_protocols =
1604 SAS_PROTOCOL_SMP;
1605 }
1606
1607 phy->port_id = port_id;
1608
1609 /*
1610 * Call pm_runtime_get_noresume() which pairs with
1611 * hisi_sas_phyup_pm_work() -> pm_runtime_put_sync().
1612 * For failure call pm_runtime_put() as we are in a hardirq context.
1613 */
1614 pm_runtime_get_noresume(dev);
1615 res = hisi_sas_notify_phy_event(phy, event: HISI_PHYE_PHY_UP_PM);
1616 if (!res)
1617 pm_runtime_put(dev);
1618
1619 res = IRQ_HANDLED;
1620
1621 spin_lock(lock: &phy->lock);
1622 /* Delete timer and set phy_attached atomically */
1623 del_timer(timer: &phy->timer);
1624 phy->phy_attached = 1;
1625 spin_unlock(lock: &phy->lock);
1626end:
1627 if (phy->reset_completion)
1628 complete(phy->reset_completion);
1629 hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1630 CHL_INT0_SL_PHY_ENABLE_MSK);
1631 hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, val: 0);
1632
1633 return res;
1634}
1635
1636static irqreturn_t phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1637{
1638 struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1639 u32 phy_state, sl_ctrl, txid_auto;
1640 struct device *dev = hisi_hba->dev;
1641
1642 atomic_inc(v: &phy->down_cnt);
1643
1644 del_timer(timer: &phy->timer);
1645 hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, val: 1);
1646
1647 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1648 dev_info(dev, "phydown: phy%d phy_state=0x%x\n", phy_no, phy_state);
1649 hisi_sas_phy_down(hisi_hba, phy_no, rdy: (phy_state & 1 << phy_no) ? 1 : 0,
1650 GFP_ATOMIC);
1651
1652 sl_ctrl = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1653 hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL,
1654 val: sl_ctrl&(~SL_CTA_MSK));
1655
1656 txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
1657 hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
1658 val: txid_auto | CT3_MSK);
1659
1660 hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_NOT_RDY_MSK);
1661 hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, val: 0);
1662
1663 return IRQ_HANDLED;
1664}
1665
1666static irqreturn_t phy_bcast_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1667{
1668 struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1669 u32 bcast_status;
1670
1671 hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, val: 1);
1672 bcast_status = hisi_sas_phy_read32(hisi_hba, phy_no, RX_PRIMS_STATUS);
1673 if (bcast_status & RX_BCAST_CHG_MSK)
1674 hisi_sas_phy_bcast(phy);
1675 hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1676 CHL_INT0_SL_RX_BCST_ACK_MSK);
1677 hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, val: 0);
1678
1679 return IRQ_HANDLED;
1680}
1681
1682static irqreturn_t int_phy_up_down_bcast_v3_hw(int irq_no, void *p)
1683{
1684 struct hisi_hba *hisi_hba = p;
1685 u32 irq_msk;
1686 int phy_no = 0;
1687 irqreturn_t res = IRQ_NONE;
1688
1689 irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1690 & 0x11111111;
1691 while (irq_msk) {
1692 if (irq_msk & 1) {
1693 u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1694 CHL_INT0);
1695 u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1696 int rdy = phy_state & (1 << phy_no);
1697
1698 if (rdy) {
1699 if (irq_value & CHL_INT0_SL_PHY_ENABLE_MSK)
1700 /* phy up */
1701 if (phy_up_v3_hw(phy_no, hisi_hba)
1702 == IRQ_HANDLED)
1703 res = IRQ_HANDLED;
1704 if (irq_value & CHL_INT0_SL_RX_BCST_ACK_MSK)
1705 /* phy bcast */
1706 if (phy_bcast_v3_hw(phy_no, hisi_hba)
1707 == IRQ_HANDLED)
1708 res = IRQ_HANDLED;
1709 } else {
1710 if (irq_value & CHL_INT0_NOT_RDY_MSK)
1711 /* phy down */
1712 if (phy_down_v3_hw(phy_no, hisi_hba)
1713 == IRQ_HANDLED)
1714 res = IRQ_HANDLED;
1715 }
1716 }
1717 irq_msk >>= 4;
1718 phy_no++;
1719 }
1720
1721 return res;
1722}
1723
1724static const struct hisi_sas_hw_error port_axi_error[] = {
1725 {
1726 .irq_msk = BIT(CHL_INT1_DMAC_TX_ECC_MB_ERR_OFF),
1727 .msg = "dmac_tx_ecc_bad_err",
1728 },
1729 {
1730 .irq_msk = BIT(CHL_INT1_DMAC_RX_ECC_MB_ERR_OFF),
1731 .msg = "dmac_rx_ecc_bad_err",
1732 },
1733 {
1734 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF),
1735 .msg = "dma_tx_axi_wr_err",
1736 },
1737 {
1738 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF),
1739 .msg = "dma_tx_axi_rd_err",
1740 },
1741 {
1742 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF),
1743 .msg = "dma_rx_axi_wr_err",
1744 },
1745 {
1746 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF),
1747 .msg = "dma_rx_axi_rd_err",
1748 },
1749 {
1750 .irq_msk = BIT(CHL_INT1_DMAC_TX_FIFO_ERR_OFF),
1751 .msg = "dma_tx_fifo_err",
1752 },
1753 {
1754 .irq_msk = BIT(CHL_INT1_DMAC_RX_FIFO_ERR_OFF),
1755 .msg = "dma_rx_fifo_err",
1756 },
1757 {
1758 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_RUSER_ERR_OFF),
1759 .msg = "dma_tx_axi_ruser_err",
1760 },
1761 {
1762 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RUSER_ERR_OFF),
1763 .msg = "dma_rx_axi_ruser_err",
1764 },
1765};
1766
1767static void handle_chl_int1_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1768{
1769 u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT1);
1770 u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT1_MSK);
1771 struct device *dev = hisi_hba->dev;
1772 int i;
1773
1774 irq_value &= ~irq_msk;
1775 if (!irq_value) {
1776 dev_warn(dev, "phy%d channel int 1 received with status bits cleared\n",
1777 phy_no);
1778 return;
1779 }
1780
1781 for (i = 0; i < ARRAY_SIZE(port_axi_error); i++) {
1782 const struct hisi_sas_hw_error *error = &port_axi_error[i];
1783
1784 if (!(irq_value & error->irq_msk))
1785 continue;
1786
1787 dev_err(dev, "%s error (phy%d 0x%x) found!\n",
1788 error->msg, phy_no, irq_value);
1789 queue_work(wq: hisi_hba->wq, work: &hisi_hba->rst_work);
1790 }
1791
1792 hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT1, val: irq_value);
1793}
1794
1795static void phy_get_events_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1796{
1797 struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1798 struct asd_sas_phy *sas_phy = &phy->sas_phy;
1799 struct sas_phy *sphy = sas_phy->phy;
1800 unsigned long flags;
1801 u32 reg_value;
1802
1803 spin_lock_irqsave(&phy->lock, flags);
1804
1805 /* loss dword sync */
1806 reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DWS_LOST);
1807 sphy->loss_of_dword_sync_count += reg_value;
1808
1809 /* phy reset problem */
1810 reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_RESET_PROB);
1811 sphy->phy_reset_problem_count += reg_value;
1812
1813 /* invalid dword */
1814 reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW);
1815 sphy->invalid_dword_count += reg_value;
1816
1817 /* disparity err */
1818 reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR);
1819 sphy->running_disparity_error_count += reg_value;
1820
1821 /* code violation error */
1822 reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_CODE_ERR);
1823 phy->code_violation_err_count += reg_value;
1824
1825 spin_unlock_irqrestore(lock: &phy->lock, flags);
1826}
1827
1828static void handle_chl_int2_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1829{
1830 u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2_MSK);
1831 u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2);
1832 struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1833 struct pci_dev *pci_dev = hisi_hba->pci_dev;
1834 struct device *dev = hisi_hba->dev;
1835 static const u32 msk = BIT(CHL_INT2_RX_DISP_ERR_OFF) |
1836 BIT(CHL_INT2_RX_CODE_ERR_OFF) |
1837 BIT(CHL_INT2_RX_INVLD_DW_OFF);
1838
1839 irq_value &= ~irq_msk;
1840 if (!irq_value) {
1841 dev_warn(dev, "phy%d channel int 2 received with status bits cleared\n",
1842 phy_no);
1843 return;
1844 }
1845
1846 if (irq_value & BIT(CHL_INT2_SL_IDAF_TOUT_CONF_OFF)) {
1847 dev_warn(dev, "phy%d identify timeout\n", phy_no);
1848 hisi_sas_notify_phy_event(phy, event: HISI_PHYE_LINK_RESET);
1849 }
1850
1851 if (irq_value & BIT(CHL_INT2_STP_LINK_TIMEOUT_OFF)) {
1852 u32 reg_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1853 STP_LINK_TIMEOUT_STATE);
1854
1855 dev_warn(dev, "phy%d stp link timeout (0x%x)\n",
1856 phy_no, reg_value);
1857 if (reg_value & BIT(4))
1858 hisi_sas_notify_phy_event(phy, event: HISI_PHYE_LINK_RESET);
1859 }
1860
1861 if (pci_dev->revision > 0x20 && (irq_value & msk)) {
1862 struct asd_sas_phy *sas_phy = &phy->sas_phy;
1863 struct sas_phy *sphy = sas_phy->phy;
1864
1865 phy_get_events_v3_hw(hisi_hba, phy_no);
1866
1867 if (irq_value & BIT(CHL_INT2_RX_INVLD_DW_OFF))
1868 dev_info(dev, "phy%d invalid dword cnt: %u\n", phy_no,
1869 sphy->invalid_dword_count);
1870
1871 if (irq_value & BIT(CHL_INT2_RX_CODE_ERR_OFF))
1872 dev_info(dev, "phy%d code violation cnt: %u\n", phy_no,
1873 phy->code_violation_err_count);
1874
1875 if (irq_value & BIT(CHL_INT2_RX_DISP_ERR_OFF))
1876 dev_info(dev, "phy%d disparity error cnt: %u\n", phy_no,
1877 sphy->running_disparity_error_count);
1878 }
1879
1880 if ((irq_value & BIT(CHL_INT2_RX_INVLD_DW_OFF)) &&
1881 (pci_dev->revision == 0x20)) {
1882 u32 reg_value;
1883 int rc;
1884
1885 rc = hisi_sas_read32_poll_timeout_atomic(
1886 HILINK_ERR_DFX, reg_value,
1887 !((reg_value >> 8) & BIT(phy_no)),
1888 1000, 10000);
1889 if (rc)
1890 hisi_sas_notify_phy_event(phy, event: HISI_PHYE_LINK_RESET);
1891 }
1892
1893 hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2, val: irq_value);
1894}
1895
1896static void handle_chl_int0_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1897{
1898 u32 irq_value0 = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT0);
1899
1900 if (irq_value0 & CHL_INT0_PHY_RDY_MSK)
1901 hisi_sas_phy_oob_ready(hisi_hba, phy_no);
1902
1903 hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1904 val: irq_value0 & (~CHL_INT0_SL_RX_BCST_ACK_MSK)
1905 & (~CHL_INT0_SL_PHY_ENABLE_MSK)
1906 & (~CHL_INT0_NOT_RDY_MSK));
1907}
1908
1909static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
1910{
1911 struct hisi_hba *hisi_hba = p;
1912 u32 irq_msk;
1913 int phy_no = 0;
1914
1915 irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1916 & CHNL_INT_STS_MSK;
1917
1918 while (irq_msk) {
1919 if (irq_msk & (CHNL_INT_STS_INT0_MSK << (phy_no * CHNL_WIDTH)))
1920 handle_chl_int0_v3_hw(hisi_hba, phy_no);
1921
1922 if (irq_msk & (CHNL_INT_STS_INT1_MSK << (phy_no * CHNL_WIDTH)))
1923 handle_chl_int1_v3_hw(hisi_hba, phy_no);
1924
1925 if (irq_msk & (CHNL_INT_STS_INT2_MSK << (phy_no * CHNL_WIDTH)))
1926 handle_chl_int2_v3_hw(hisi_hba, phy_no);
1927
1928 irq_msk &= ~(CHNL_INT_STS_PHY_MSK << (phy_no * CHNL_WIDTH));
1929 phy_no++;
1930 }
1931
1932 return IRQ_HANDLED;
1933}
1934
1935static const struct hisi_sas_hw_error multi_bit_ecc_errors[] = {
1936 {
1937 .irq_msk = BIT(SAS_ECC_INTR_DQE_ECC_MB_OFF),
1938 .msk = HGC_DQE_ECC_MB_ADDR_MSK,
1939 .shift = HGC_DQE_ECC_MB_ADDR_OFF,
1940 .msg = "hgc_dqe_eccbad_intr",
1941 .reg = HGC_DQE_ECC_ADDR,
1942 },
1943 {
1944 .irq_msk = BIT(SAS_ECC_INTR_IOST_ECC_MB_OFF),
1945 .msk = HGC_IOST_ECC_MB_ADDR_MSK,
1946 .shift = HGC_IOST_ECC_MB_ADDR_OFF,
1947 .msg = "hgc_iost_eccbad_intr",
1948 .reg = HGC_IOST_ECC_ADDR,
1949 },
1950 {
1951 .irq_msk = BIT(SAS_ECC_INTR_ITCT_ECC_MB_OFF),
1952 .msk = HGC_ITCT_ECC_MB_ADDR_MSK,
1953 .shift = HGC_ITCT_ECC_MB_ADDR_OFF,
1954 .msg = "hgc_itct_eccbad_intr",
1955 .reg = HGC_ITCT_ECC_ADDR,
1956 },
1957 {
1958 .irq_msk = BIT(SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF),
1959 .msk = HGC_LM_DFX_STATUS2_IOSTLIST_MSK,
1960 .shift = HGC_LM_DFX_STATUS2_IOSTLIST_OFF,
1961 .msg = "hgc_iostl_eccbad_intr",
1962 .reg = HGC_LM_DFX_STATUS2,
1963 },
1964 {
1965 .irq_msk = BIT(SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF),
1966 .msk = HGC_LM_DFX_STATUS2_ITCTLIST_MSK,
1967 .shift = HGC_LM_DFX_STATUS2_ITCTLIST_OFF,
1968 .msg = "hgc_itctl_eccbad_intr",
1969 .reg = HGC_LM_DFX_STATUS2,
1970 },
1971 {
1972 .irq_msk = BIT(SAS_ECC_INTR_CQE_ECC_MB_OFF),
1973 .msk = HGC_CQE_ECC_MB_ADDR_MSK,
1974 .shift = HGC_CQE_ECC_MB_ADDR_OFF,
1975 .msg = "hgc_cqe_eccbad_intr",
1976 .reg = HGC_CQE_ECC_ADDR,
1977 },
1978 {
1979 .irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF),
1980 .msk = HGC_RXM_DFX_STATUS14_MEM0_MSK,
1981 .shift = HGC_RXM_DFX_STATUS14_MEM0_OFF,
1982 .msg = "rxm_mem0_eccbad_intr",
1983 .reg = HGC_RXM_DFX_STATUS14,
1984 },
1985 {
1986 .irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF),
1987 .msk = HGC_RXM_DFX_STATUS14_MEM1_MSK,
1988 .shift = HGC_RXM_DFX_STATUS14_MEM1_OFF,
1989 .msg = "rxm_mem1_eccbad_intr",
1990 .reg = HGC_RXM_DFX_STATUS14,
1991 },
1992 {
1993 .irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF),
1994 .msk = HGC_RXM_DFX_STATUS14_MEM2_MSK,
1995 .shift = HGC_RXM_DFX_STATUS14_MEM2_OFF,
1996 .msg = "rxm_mem2_eccbad_intr",
1997 .reg = HGC_RXM_DFX_STATUS14,
1998 },
1999 {
2000 .irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF),
2001 .msk = HGC_RXM_DFX_STATUS15_MEM3_MSK,
2002 .shift = HGC_RXM_DFX_STATUS15_MEM3_OFF,
2003 .msg = "rxm_mem3_eccbad_intr",
2004 .reg = HGC_RXM_DFX_STATUS15,
2005 },
2006 {
2007 .irq_msk = BIT(SAS_ECC_INTR_OOO_RAM_ECC_MB_OFF),
2008 .msk = AM_ROB_ECC_ERR_ADDR_MSK,
2009 .shift = AM_ROB_ECC_ERR_ADDR_OFF,
2010 .msg = "ooo_ram_eccbad_intr",
2011 .reg = AM_ROB_ECC_ERR_ADDR,
2012 },
2013};
2014
2015static void multi_bit_ecc_error_process_v3_hw(struct hisi_hba *hisi_hba,
2016 u32 irq_value)
2017{
2018 struct device *dev = hisi_hba->dev;
2019 const struct hisi_sas_hw_error *ecc_error;
2020 u32 val;
2021 int i;
2022
2023 for (i = 0; i < ARRAY_SIZE(multi_bit_ecc_errors); i++) {
2024 ecc_error = &multi_bit_ecc_errors[i];
2025 if (irq_value & ecc_error->irq_msk) {
2026 val = hisi_sas_read32(hisi_hba, off: ecc_error->reg);
2027 val &= ecc_error->msk;
2028 val >>= ecc_error->shift;
2029 dev_err(dev, "%s (0x%x) found: mem addr is 0x%08X\n",
2030 ecc_error->msg, irq_value, val);
2031 queue_work(wq: hisi_hba->wq, work: &hisi_hba->rst_work);
2032 }
2033 }
2034}
2035
2036static void fatal_ecc_int_v3_hw(struct hisi_hba *hisi_hba)
2037{
2038 u32 irq_value, irq_msk;
2039
2040 irq_msk = hisi_sas_read32(hisi_hba, SAS_ECC_INTR_MSK);
2041 hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, val: 0xffffffff);
2042
2043 irq_value = hisi_sas_read32(hisi_hba, SAS_ECC_INTR);
2044 if (irq_value)
2045 multi_bit_ecc_error_process_v3_hw(hisi_hba, irq_value);
2046
2047 hisi_sas_write32(hisi_hba, SAS_ECC_INTR, val: irq_value);
2048 hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, val: irq_msk);
2049}
2050
2051static const struct hisi_sas_hw_error axi_error[] = {
2052 { .msk = BIT(0), .msg = "IOST_AXI_W_ERR" },
2053 { .msk = BIT(1), .msg = "IOST_AXI_R_ERR" },
2054 { .msk = BIT(2), .msg = "ITCT_AXI_W_ERR" },
2055 { .msk = BIT(3), .msg = "ITCT_AXI_R_ERR" },
2056 { .msk = BIT(4), .msg = "SATA_AXI_W_ERR" },
2057 { .msk = BIT(5), .msg = "SATA_AXI_R_ERR" },
2058 { .msk = BIT(6), .msg = "DQE_AXI_R_ERR" },
2059 { .msk = BIT(7), .msg = "CQE_AXI_W_ERR" },
2060 {}
2061};
2062
2063static const struct hisi_sas_hw_error fifo_error[] = {
2064 { .msk = BIT(8), .msg = "CQE_WINFO_FIFO" },
2065 { .msk = BIT(9), .msg = "CQE_MSG_FIFIO" },
2066 { .msk = BIT(10), .msg = "GETDQE_FIFO" },
2067 { .msk = BIT(11), .msg = "CMDP_FIFO" },
2068 { .msk = BIT(12), .msg = "AWTCTRL_FIFO" },
2069 {}
2070};
2071
2072static const struct hisi_sas_hw_error fatal_axi_error[] = {
2073 {
2074 .irq_msk = BIT(ENT_INT_SRC3_WP_DEPTH_OFF),
2075 .msg = "write pointer and depth",
2076 },
2077 {
2078 .irq_msk = BIT(ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF),
2079 .msg = "iptt no match slot",
2080 },
2081 {
2082 .irq_msk = BIT(ENT_INT_SRC3_RP_DEPTH_OFF),
2083 .msg = "read pointer and depth",
2084 },
2085 {
2086 .irq_msk = BIT(ENT_INT_SRC3_AXI_OFF),
2087 .reg = HGC_AXI_FIFO_ERR_INFO,
2088 .sub = axi_error,
2089 },
2090 {
2091 .irq_msk = BIT(ENT_INT_SRC3_FIFO_OFF),
2092 .reg = HGC_AXI_FIFO_ERR_INFO,
2093 .sub = fifo_error,
2094 },
2095 {
2096 .irq_msk = BIT(ENT_INT_SRC3_LM_OFF),
2097 .msg = "LM add/fetch list",
2098 },
2099 {
2100 .irq_msk = BIT(ENT_INT_SRC3_ABT_OFF),
2101 .msg = "SAS_HGC_ABT fetch LM list",
2102 },
2103 {
2104 .irq_msk = BIT(ENT_INT_SRC3_DQE_POISON_OFF),
2105 .msg = "read dqe poison",
2106 },
2107 {
2108 .irq_msk = BIT(ENT_INT_SRC3_IOST_POISON_OFF),
2109 .msg = "read iost poison",
2110 },
2111 {
2112 .irq_msk = BIT(ENT_INT_SRC3_ITCT_POISON_OFF),
2113 .msg = "read itct poison",
2114 },
2115 {
2116 .irq_msk = BIT(ENT_INT_SRC3_ITCT_NCQ_POISON_OFF),
2117 .msg = "read itct ncq poison",
2118 },
2119
2120};
2121
2122static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p)
2123{
2124 u32 irq_value, irq_msk;
2125 struct hisi_hba *hisi_hba = p;
2126 struct device *dev = hisi_hba->dev;
2127 struct pci_dev *pdev = hisi_hba->pci_dev;
2128 int i;
2129
2130 irq_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
2131 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, val: irq_msk | 0x1df00);
2132
2133 irq_value = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
2134 irq_value &= ~irq_msk;
2135
2136 for (i = 0; i < ARRAY_SIZE(fatal_axi_error); i++) {
2137 const struct hisi_sas_hw_error *error = &fatal_axi_error[i];
2138
2139 if (!(irq_value & error->irq_msk))
2140 continue;
2141
2142 if (error->sub) {
2143 const struct hisi_sas_hw_error *sub = error->sub;
2144 u32 err_value = hisi_sas_read32(hisi_hba, off: error->reg);
2145
2146 for (; sub->msk || sub->msg; sub++) {
2147 if (!(err_value & sub->msk))
2148 continue;
2149
2150 dev_err(dev, "%s error (0x%x) found!\n",
2151 sub->msg, irq_value);
2152 queue_work(wq: hisi_hba->wq, work: &hisi_hba->rst_work);
2153 }
2154 } else {
2155 dev_err(dev, "%s error (0x%x) found!\n",
2156 error->msg, irq_value);
2157 queue_work(wq: hisi_hba->wq, work: &hisi_hba->rst_work);
2158 }
2159
2160 if (pdev->revision < 0x21) {
2161 u32 reg_val;
2162
2163 reg_val = hisi_sas_read32(hisi_hba,
2164 AXI_MASTER_CFG_BASE +
2165 AM_CTRL_GLOBAL);
2166 reg_val |= AM_CTRL_SHUTDOWN_REQ_MSK;
2167 hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
2168 AM_CTRL_GLOBAL, val: reg_val);
2169 }
2170 }
2171
2172 fatal_ecc_int_v3_hw(hisi_hba);
2173
2174 if (irq_value & BIT(ENT_INT_SRC3_ITC_INT_OFF)) {
2175 u32 reg_val = hisi_sas_read32(hisi_hba, ITCT_CLR);
2176 u32 dev_id = reg_val & ITCT_DEV_MSK;
2177 struct hisi_sas_device *sas_dev =
2178 &hisi_hba->devices[dev_id];
2179
2180 hisi_sas_write32(hisi_hba, ITCT_CLR, val: 0);
2181 dev_dbg(dev, "clear ITCT ok\n");
2182 complete(sas_dev->completion);
2183 }
2184
2185 hisi_sas_write32(hisi_hba, ENT_INT_SRC3, val: irq_value & 0x1df00);
2186 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, val: irq_msk);
2187
2188 return IRQ_HANDLED;
2189}
2190
2191static bool is_ncq_err_v3_hw(struct hisi_sas_complete_v3_hdr *complete_hdr)
2192{
2193 u32 dw0, dw3;
2194
2195 dw0 = le32_to_cpu(complete_hdr->dw0);
2196 dw3 = le32_to_cpu(complete_hdr->dw3);
2197
2198 return (dw0 & ERR_PHASE_RESPONSE_FRAME_REV_STAGE_MSK) &&
2199 (dw3 & FIS_TYPE_SDB_MSK) &&
2200 (dw3 & FIS_ATA_STATUS_ERR_MSK);
2201}
2202
2203static bool
2204slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task,
2205 struct hisi_sas_slot *slot)
2206{
2207 struct task_status_struct *ts = &task->task_status;
2208 struct hisi_sas_complete_v3_hdr *complete_queue =
2209 hisi_hba->complete_hdr[slot->cmplt_queue];
2210 struct hisi_sas_complete_v3_hdr *complete_hdr =
2211 &complete_queue[slot->cmplt_queue_slot];
2212 struct hisi_sas_err_record_v3 *record =
2213 hisi_sas_status_buf_addr_mem(slot);
2214 u32 dma_rx_err_type = le32_to_cpu(record->dma_rx_err_type);
2215 u32 trans_tx_fail_type = le32_to_cpu(record->trans_tx_fail_type);
2216 u16 sipc_rx_err_type = le16_to_cpu(record->sipc_rx_err_type);
2217 u32 dw3 = le32_to_cpu(complete_hdr->dw3);
2218 u32 dw0 = le32_to_cpu(complete_hdr->dw0);
2219
2220 switch (task->task_proto) {
2221 case SAS_PROTOCOL_SSP:
2222 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
2223 /*
2224 * If returned response frame is incorrect because of data underflow,
2225 * but I/O information has been written to the host memory, we examine
2226 * response IU.
2227 */
2228 if (!(dw0 & CMPLT_HDR_RSPNS_GOOD_MSK) &&
2229 (dw0 & CMPLT_HDR_RSPNS_XFRD_MSK))
2230 return false;
2231
2232 ts->residual = trans_tx_fail_type;
2233 ts->stat = SAS_DATA_UNDERRUN;
2234 } else if (dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
2235 ts->stat = SAS_QUEUE_FULL;
2236 slot->abort = 1;
2237 } else {
2238 ts->stat = SAS_OPEN_REJECT;
2239 ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
2240 }
2241 break;
2242 case SAS_PROTOCOL_SATA:
2243 case SAS_PROTOCOL_STP:
2244 case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
2245 if ((dw0 & CMPLT_HDR_RSPNS_XFRD_MSK) &&
2246 (sipc_rx_err_type & RX_FIS_STATUS_ERR_MSK)) {
2247 ts->stat = SAS_PROTO_RESPONSE;
2248 } else if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
2249 ts->residual = trans_tx_fail_type;
2250 ts->stat = SAS_DATA_UNDERRUN;
2251 } else if ((dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) ||
2252 (dw3 & SATA_DISK_IN_ERROR_STATUS_MSK)) {
2253 ts->stat = SAS_PHY_DOWN;
2254 slot->abort = 1;
2255 } else {
2256 ts->stat = SAS_OPEN_REJECT;
2257 ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
2258 }
2259 if (dw0 & CMPLT_HDR_RSPNS_XFRD_MSK)
2260 hisi_sas_sata_done(task, slot);
2261 break;
2262 case SAS_PROTOCOL_SMP:
2263 ts->stat = SAS_SAM_STAT_CHECK_CONDITION;
2264 break;
2265 default:
2266 break;
2267 }
2268 return true;
2269}
2270
2271static void slot_complete_v3_hw(struct hisi_hba *hisi_hba,
2272 struct hisi_sas_slot *slot)
2273{
2274 struct sas_task *task = slot->task;
2275 struct hisi_sas_device *sas_dev;
2276 struct device *dev = hisi_hba->dev;
2277 struct task_status_struct *ts;
2278 struct domain_device *device;
2279 struct sas_ha_struct *ha;
2280 struct hisi_sas_complete_v3_hdr *complete_queue =
2281 hisi_hba->complete_hdr[slot->cmplt_queue];
2282 struct hisi_sas_complete_v3_hdr *complete_hdr =
2283 &complete_queue[slot->cmplt_queue_slot];
2284 unsigned long flags;
2285 bool is_internal = slot->is_internal;
2286 u32 dw0, dw1, dw3;
2287
2288 if (unlikely(!task || !task->lldd_task || !task->dev))
2289 return;
2290
2291 ts = &task->task_status;
2292 device = task->dev;
2293 ha = device->port->ha;
2294 sas_dev = device->lldd_dev;
2295
2296 spin_lock_irqsave(&task->task_state_lock, flags);
2297 task->task_state_flags &= ~SAS_TASK_STATE_PENDING;
2298 spin_unlock_irqrestore(lock: &task->task_state_lock, flags);
2299
2300 memset(ts, 0, sizeof(*ts));
2301 ts->resp = SAS_TASK_COMPLETE;
2302
2303 if (unlikely(!sas_dev)) {
2304 dev_dbg(dev, "slot complete: port has not device\n");
2305 ts->stat = SAS_PHY_DOWN;
2306 goto out;
2307 }
2308
2309 dw0 = le32_to_cpu(complete_hdr->dw0);
2310 dw1 = le32_to_cpu(complete_hdr->dw1);
2311 dw3 = le32_to_cpu(complete_hdr->dw3);
2312
2313 /*
2314 * Use SAS+TMF status codes
2315 */
2316 switch ((dw0 & CMPLT_HDR_ABORT_STAT_MSK) >> CMPLT_HDR_ABORT_STAT_OFF) {
2317 case STAT_IO_ABORTED:
2318 /* this IO has been aborted by abort command */
2319 ts->stat = SAS_ABORTED_TASK;
2320 goto out;
2321 case STAT_IO_COMPLETE:
2322 /* internal abort command complete */
2323 ts->stat = TMF_RESP_FUNC_SUCC;
2324 goto out;
2325 case STAT_IO_NO_DEVICE:
2326 ts->stat = TMF_RESP_FUNC_COMPLETE;
2327 goto out;
2328 case STAT_IO_NOT_VALID:
2329 /*
2330 * abort single IO, the controller can't find the IO
2331 */
2332 ts->stat = TMF_RESP_FUNC_FAILED;
2333 goto out;
2334 default:
2335 break;
2336 }
2337
2338 /* check for erroneous completion */
2339 if ((dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) {
2340 u32 *error_info = hisi_sas_status_buf_addr_mem(slot);
2341
2342 if (slot_err_v3_hw(hisi_hba, task, slot)) {
2343 if (ts->stat != SAS_DATA_UNDERRUN)
2344 dev_info(dev, "erroneous completion iptt=%d task=%pK dev id=%d addr=%016llx CQ hdr: 0x%x 0x%x 0x%x 0x%x Error info: 0x%x 0x%x 0x%x 0x%x\n",
2345 slot->idx, task, sas_dev->device_id,
2346 SAS_ADDR(device->sas_addr),
2347 dw0, dw1, complete_hdr->act, dw3,
2348 error_info[0], error_info[1],
2349 error_info[2], error_info[3]);
2350 if (unlikely(slot->abort)) {
2351 if (dev_is_sata(dev: device) && task->ata_task.use_ncq)
2352 sas_ata_device_link_abort(dev: device, force_reset: true);
2353 else
2354 sas_task_abort(task);
2355
2356 return;
2357 }
2358 goto out;
2359 }
2360 }
2361
2362 switch (task->task_proto) {
2363 case SAS_PROTOCOL_SSP: {
2364 struct ssp_response_iu *iu =
2365 hisi_sas_status_buf_addr_mem(slot) +
2366 sizeof(struct hisi_sas_err_record);
2367
2368 sas_ssp_task_response(dev, task, iu);
2369 break;
2370 }
2371 case SAS_PROTOCOL_SMP: {
2372 struct scatterlist *sg_resp = &task->smp_task.smp_resp;
2373 void *to = page_address(sg_page(sg_resp));
2374
2375 ts->stat = SAS_SAM_STAT_GOOD;
2376
2377 memcpy(to + sg_resp->offset,
2378 hisi_sas_status_buf_addr_mem(slot) +
2379 sizeof(struct hisi_sas_err_record),
2380 sg_resp->length);
2381 break;
2382 }
2383 case SAS_PROTOCOL_SATA:
2384 case SAS_PROTOCOL_STP:
2385 case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
2386 ts->stat = SAS_SAM_STAT_GOOD;
2387 if (dw0 & CMPLT_HDR_RSPNS_XFRD_MSK)
2388 hisi_sas_sata_done(task, slot);
2389 break;
2390 default:
2391 ts->stat = SAS_SAM_STAT_CHECK_CONDITION;
2392 break;
2393 }
2394
2395 if (!slot->port->port_attached) {
2396 dev_warn(dev, "slot complete: port %d has removed\n",
2397 slot->port->sas_port.id);
2398 ts->stat = SAS_PHY_DOWN;
2399 }
2400
2401out:
2402 spin_lock_irqsave(&task->task_state_lock, flags);
2403 if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
2404 spin_unlock_irqrestore(lock: &task->task_state_lock, flags);
2405 dev_info(dev, "slot complete: task(%pK) aborted\n", task);
2406 return;
2407 }
2408 task->task_state_flags |= SAS_TASK_STATE_DONE;
2409 spin_unlock_irqrestore(lock: &task->task_state_lock, flags);
2410 hisi_sas_slot_task_free(hisi_hba, task, slot, need_lock: true);
2411
2412 if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
2413 spin_lock_irqsave(&device->done_lock, flags);
2414 if (test_bit(SAS_HA_FROZEN, &ha->state)) {
2415 spin_unlock_irqrestore(lock: &device->done_lock, flags);
2416 dev_info(dev, "slot complete: task(%pK) ignored\n ",
2417 task);
2418 return;
2419 }
2420 spin_unlock_irqrestore(lock: &device->done_lock, flags);
2421 }
2422
2423 if (task->task_done)
2424 task->task_done(task);
2425}
2426
2427static int complete_v3_hw(struct hisi_sas_cq *cq)
2428{
2429 struct hisi_sas_complete_v3_hdr *complete_queue;
2430 struct hisi_hba *hisi_hba = cq->hisi_hba;
2431 u32 rd_point, wr_point;
2432 int queue = cq->id;
2433 int completed;
2434
2435 rd_point = cq->rd_point;
2436 complete_queue = hisi_hba->complete_hdr[queue];
2437
2438 wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
2439 (0x14 * queue));
2440 completed = (wr_point + HISI_SAS_QUEUE_SLOTS - rd_point) % HISI_SAS_QUEUE_SLOTS;
2441
2442 while (rd_point != wr_point) {
2443 struct hisi_sas_complete_v3_hdr *complete_hdr;
2444 struct device *dev = hisi_hba->dev;
2445 struct hisi_sas_slot *slot;
2446 u32 dw0, dw1, dw3;
2447 int iptt;
2448
2449 complete_hdr = &complete_queue[rd_point];
2450 dw0 = le32_to_cpu(complete_hdr->dw0);
2451 dw1 = le32_to_cpu(complete_hdr->dw1);
2452 dw3 = le32_to_cpu(complete_hdr->dw3);
2453
2454 iptt = dw1 & CMPLT_HDR_IPTT_MSK;
2455 if (unlikely((dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) &&
2456 (dw3 & CMPLT_HDR_SATA_DISK_ERR_MSK)) {
2457 int device_id = (dw1 & CMPLT_HDR_DEV_ID_MSK) >>
2458 CMPLT_HDR_DEV_ID_OFF;
2459 struct hisi_sas_itct *itct =
2460 &hisi_hba->itct[device_id];
2461 struct hisi_sas_device *sas_dev =
2462 &hisi_hba->devices[device_id];
2463 struct domain_device *device = sas_dev->sas_device;
2464
2465 dev_err(dev, "erroneous completion disk err dev id=%d sas_addr=0x%llx CQ hdr: 0x%x 0x%x 0x%x 0x%x\n",
2466 device_id, itct->sas_addr, dw0, dw1,
2467 complete_hdr->act, dw3);
2468
2469 if (is_ncq_err_v3_hw(complete_hdr))
2470 sas_dev->dev_status = HISI_SAS_DEV_NCQ_ERR;
2471
2472 sas_ata_device_link_abort(dev: device, force_reset: true);
2473 } else if (likely(iptt < HISI_SAS_COMMAND_ENTRIES_V3_HW)) {
2474 slot = &hisi_hba->slot_info[iptt];
2475 slot->cmplt_queue_slot = rd_point;
2476 slot->cmplt_queue = queue;
2477 slot_complete_v3_hw(hisi_hba, slot);
2478 } else
2479 dev_err(dev, "IPTT %d is invalid, discard it.\n", iptt);
2480
2481 if (++rd_point >= HISI_SAS_QUEUE_SLOTS)
2482 rd_point = 0;
2483 }
2484
2485 /* update rd_point */
2486 cq->rd_point = rd_point;
2487 hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), val: rd_point);
2488
2489 return completed;
2490}
2491
2492static int queue_complete_v3_hw(struct Scsi_Host *shost, unsigned int queue)
2493{
2494 struct hisi_hba *hisi_hba = shost_priv(shost);
2495 struct hisi_sas_cq *cq = &hisi_hba->cq[queue];
2496 int completed;
2497
2498 spin_lock(lock: &cq->poll_lock);
2499 completed = complete_v3_hw(cq);
2500 spin_unlock(lock: &cq->poll_lock);
2501
2502 return completed;
2503}
2504
2505static irqreturn_t cq_thread_v3_hw(int irq_no, void *p)
2506{
2507 struct hisi_sas_cq *cq = p;
2508
2509 complete_v3_hw(cq);
2510
2511 return IRQ_HANDLED;
2512}
2513
2514static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p)
2515{
2516 struct hisi_sas_cq *cq = p;
2517 struct hisi_hba *hisi_hba = cq->hisi_hba;
2518 int queue = cq->id;
2519
2520 hisi_sas_write32(hisi_hba, OQ_INT_SRC, val: 1 << queue);
2521
2522 return IRQ_WAKE_THREAD;
2523}
2524
2525static void hisi_sas_v3_free_vectors(void *data)
2526{
2527 struct pci_dev *pdev = data;
2528
2529 pci_free_irq_vectors(dev: pdev);
2530}
2531
2532static int interrupt_preinit_v3_hw(struct hisi_hba *hisi_hba)
2533{
2534 /* Allocate all MSI vectors to avoid re-insertion issue */
2535 int max_msi = HISI_SAS_MSI_COUNT_V3_HW;
2536 int vectors, min_msi;
2537 struct Scsi_Host *shost = hisi_hba->shost;
2538 struct pci_dev *pdev = hisi_hba->pci_dev;
2539 struct irq_affinity desc = {
2540 .pre_vectors = BASE_VECTORS_V3_HW,
2541 };
2542
2543 min_msi = MIN_AFFINE_VECTORS_V3_HW;
2544 vectors = pci_alloc_irq_vectors_affinity(dev: pdev,
2545 min_vecs: min_msi, max_vecs: max_msi,
2546 PCI_IRQ_MSI |
2547 PCI_IRQ_AFFINITY,
2548 affd: &desc);
2549 if (vectors < 0)
2550 return -ENOENT;
2551
2552
2553 hisi_hba->cq_nvecs = vectors - BASE_VECTORS_V3_HW - hisi_hba->iopoll_q_cnt;
2554 shost->nr_hw_queues = hisi_hba->cq_nvecs + hisi_hba->iopoll_q_cnt;
2555
2556 return devm_add_action(&pdev->dev, hisi_sas_v3_free_vectors, pdev);
2557}
2558
2559static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba)
2560{
2561 struct device *dev = hisi_hba->dev;
2562 struct pci_dev *pdev = hisi_hba->pci_dev;
2563 int rc, i;
2564
2565 rc = devm_request_irq(dev, irq: pci_irq_vector(dev: pdev, nr: 1),
2566 handler: int_phy_up_down_bcast_v3_hw, irqflags: 0,
2567 DRV_NAME " phy", dev_id: hisi_hba);
2568 if (rc) {
2569 dev_err(dev, "could not request phy interrupt, rc=%d\n", rc);
2570 return -ENOENT;
2571 }
2572
2573 rc = devm_request_irq(dev, irq: pci_irq_vector(dev: pdev, nr: 2),
2574 handler: int_chnl_int_v3_hw, irqflags: 0,
2575 DRV_NAME " channel", dev_id: hisi_hba);
2576 if (rc) {
2577 dev_err(dev, "could not request chnl interrupt, rc=%d\n", rc);
2578 return -ENOENT;
2579 }
2580
2581 rc = devm_request_irq(dev, irq: pci_irq_vector(dev: pdev, nr: 11),
2582 handler: fatal_axi_int_v3_hw, irqflags: 0,
2583 DRV_NAME " fatal", dev_id: hisi_hba);
2584 if (rc) {
2585 dev_err(dev, "could not request fatal interrupt, rc=%d\n", rc);
2586 return -ENOENT;
2587 }
2588
2589 if (hisi_sas_intr_conv)
2590 dev_info(dev, "Enable interrupt converge\n");
2591
2592 for (i = 0; i < hisi_hba->cq_nvecs; i++) {
2593 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
2594 int nr = hisi_sas_intr_conv ? 16 : 16 + i;
2595 unsigned long irqflags = hisi_sas_intr_conv ? IRQF_SHARED :
2596 IRQF_ONESHOT;
2597
2598 cq->irq_no = pci_irq_vector(dev: pdev, nr);
2599 rc = devm_request_threaded_irq(dev, irq: cq->irq_no,
2600 handler: cq_interrupt_v3_hw,
2601 thread_fn: cq_thread_v3_hw,
2602 irqflags,
2603 DRV_NAME " cq", dev_id: cq);
2604 if (rc) {
2605 dev_err(dev, "could not request cq%d interrupt, rc=%d\n",
2606 i, rc);
2607 return -ENOENT;
2608 }
2609 cq->irq_mask = pci_irq_get_affinity(pdev, vec: i + BASE_VECTORS_V3_HW);
2610 if (!cq->irq_mask) {
2611 dev_err(dev, "could not get cq%d irq affinity!\n", i);
2612 return -ENOENT;
2613 }
2614 }
2615
2616 return 0;
2617}
2618
2619static int hisi_sas_v3_init(struct hisi_hba *hisi_hba)
2620{
2621 int rc;
2622
2623 rc = hw_init_v3_hw(hisi_hba);
2624 if (rc)
2625 return rc;
2626
2627 rc = interrupt_init_v3_hw(hisi_hba);
2628 if (rc)
2629 return rc;
2630
2631 return 0;
2632}
2633
2634static void phy_set_linkrate_v3_hw(struct hisi_hba *hisi_hba, int phy_no,
2635 struct sas_phy_linkrates *r)
2636{
2637 enum sas_linkrate max = r->maximum_linkrate;
2638 u32 prog_phy_link_rate = hisi_sas_phy_read32(hisi_hba, phy_no,
2639 PROG_PHY_LINK_RATE);
2640
2641 prog_phy_link_rate &= ~CFG_PROG_PHY_LINK_RATE_MSK;
2642 prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
2643 hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
2644 val: prog_phy_link_rate);
2645}
2646
2647static void interrupt_disable_v3_hw(struct hisi_hba *hisi_hba)
2648{
2649 struct pci_dev *pdev = hisi_hba->pci_dev;
2650 int i;
2651
2652 synchronize_irq(irq: pci_irq_vector(dev: pdev, nr: 1));
2653 synchronize_irq(irq: pci_irq_vector(dev: pdev, nr: 2));
2654 synchronize_irq(irq: pci_irq_vector(dev: pdev, nr: 11));
2655 for (i = 0; i < hisi_hba->queue_count; i++)
2656 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, val: 0x1);
2657
2658 for (i = 0; i < hisi_hba->cq_nvecs; i++)
2659 synchronize_irq(irq: pci_irq_vector(dev: pdev, nr: i + 16));
2660
2661 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, val: 0xffffffff);
2662 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, val: 0xffffffff);
2663 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, val: 0xffffffff);
2664 hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, val: 0xffffffff);
2665
2666 for (i = 0; i < hisi_hba->n_phy; i++) {
2667 hisi_sas_phy_write32(hisi_hba, phy_no: i, CHL_INT1_MSK, val: 0xffffffff);
2668 hisi_sas_phy_write32(hisi_hba, phy_no: i, CHL_INT2_MSK, val: 0xffffffff);
2669 hisi_sas_phy_write32(hisi_hba, phy_no: i, PHYCTRL_NOT_RDY_MSK, val: 0x1);
2670 hisi_sas_phy_write32(hisi_hba, phy_no: i, PHYCTRL_PHY_ENA_MSK, val: 0x1);
2671 hisi_sas_phy_write32(hisi_hba, phy_no: i, SL_RX_BCAST_CHK_MSK, val: 0x1);
2672 }
2673}
2674
2675static u32 get_phys_state_v3_hw(struct hisi_hba *hisi_hba)
2676{
2677 return hisi_sas_read32(hisi_hba, PHY_STATE);
2678}
2679
2680static int disable_host_v3_hw(struct hisi_hba *hisi_hba)
2681{
2682 struct device *dev = hisi_hba->dev;
2683 u32 status, reg_val;
2684 int rc;
2685
2686 hisi_sas_sync_poll_cqs(hisi_hba);
2687 hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, val: 0x0);
2688
2689 hisi_sas_stop_phys(hisi_hba);
2690
2691 mdelay(10);
2692
2693 reg_val = hisi_sas_read32(hisi_hba, AXI_MASTER_CFG_BASE +
2694 AM_CTRL_GLOBAL);
2695 reg_val |= AM_CTRL_SHUTDOWN_REQ_MSK;
2696 hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
2697 AM_CTRL_GLOBAL, val: reg_val);
2698
2699 /* wait until bus idle */
2700 rc = hisi_sas_read32_poll_timeout(AXI_MASTER_CFG_BASE +
2701 AM_CURR_TRANS_RETURN, status,
2702 status == 0x3, 10, 100);
2703 if (rc) {
2704 dev_err(dev, "axi bus is not idle, rc=%d\n", rc);
2705 return rc;
2706 }
2707
2708 return 0;
2709}
2710
2711static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
2712{
2713 struct device *dev = hisi_hba->dev;
2714 int rc;
2715
2716 interrupt_disable_v3_hw(hisi_hba);
2717 rc = disable_host_v3_hw(hisi_hba);
2718 if (rc) {
2719 dev_err(dev, "soft reset: disable host failed rc=%d\n", rc);
2720 return rc;
2721 }
2722
2723 hisi_sas_init_mem(hisi_hba);
2724
2725 return hw_init_v3_hw(hisi_hba);
2726}
2727
2728static int write_gpio_v3_hw(struct hisi_hba *hisi_hba, u8 reg_type,
2729 u8 reg_index, u8 reg_count, u8 *write_data)
2730{
2731 struct device *dev = hisi_hba->dev;
2732 u32 *data = (u32 *)write_data;
2733 int i;
2734
2735 switch (reg_type) {
2736 case SAS_GPIO_REG_TX:
2737 if ((reg_index + reg_count) > ((hisi_hba->n_phy + 3) / 4)) {
2738 dev_err(dev, "write gpio: invalid reg range[%d, %d]\n",
2739 reg_index, reg_index + reg_count - 1);
2740 return -EINVAL;
2741 }
2742
2743 for (i = 0; i < reg_count; i++)
2744 hisi_sas_write32(hisi_hba,
2745 SAS_GPIO_TX_0_1 + (reg_index + i) * 4,
2746 val: data[i]);
2747 break;
2748 default:
2749 dev_err(dev, "write gpio: unsupported or bad reg type %d\n",
2750 reg_type);
2751 return -EINVAL;
2752 }
2753
2754 return 0;
2755}
2756
2757static void wait_cmds_complete_timeout_v3_hw(struct hisi_hba *hisi_hba,
2758 int delay_ms, int timeout_ms)
2759{
2760 struct device *dev = hisi_hba->dev;
2761 int entries, entries_old = 0, time;
2762
2763 for (time = 0; time < timeout_ms; time += delay_ms) {
2764 entries = hisi_sas_read32(hisi_hba, CQE_SEND_CNT);
2765 if (entries == entries_old)
2766 break;
2767
2768 entries_old = entries;
2769 msleep(msecs: delay_ms);
2770 }
2771
2772 if (time >= timeout_ms) {
2773 dev_dbg(dev, "Wait commands complete timeout!\n");
2774 return;
2775 }
2776
2777 dev_dbg(dev, "wait commands complete %dms\n", time);
2778}
2779
2780static ssize_t intr_conv_v3_hw_show(struct device *dev,
2781 struct device_attribute *attr, char *buf)
2782{
2783 return scnprintf(buf, PAGE_SIZE, fmt: "%u\n", hisi_sas_intr_conv);
2784}
2785static DEVICE_ATTR_RO(intr_conv_v3_hw);
2786
2787static void config_intr_coal_v3_hw(struct hisi_hba *hisi_hba)
2788{
2789 /* config those registers between enable and disable PHYs */
2790 hisi_sas_stop_phys(hisi_hba);
2791
2792 if (hisi_hba->intr_coal_ticks == 0 ||
2793 hisi_hba->intr_coal_count == 0) {
2794 hisi_sas_write32(hisi_hba, INT_COAL_EN, val: 0x1);
2795 hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME, val: 0x1);
2796 hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT, val: 0x1);
2797 } else {
2798 hisi_sas_write32(hisi_hba, INT_COAL_EN, val: 0x3);
2799 hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME,
2800 val: hisi_hba->intr_coal_ticks);
2801 hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT,
2802 val: hisi_hba->intr_coal_count);
2803 }
2804 phys_init_v3_hw(hisi_hba);
2805}
2806
2807static ssize_t intr_coal_ticks_v3_hw_show(struct device *dev,
2808 struct device_attribute *attr,
2809 char *buf)
2810{
2811 struct Scsi_Host *shost = class_to_shost(dev);
2812 struct hisi_hba *hisi_hba = shost_priv(shost);
2813
2814 return scnprintf(buf, PAGE_SIZE, fmt: "%u\n",
2815 hisi_hba->intr_coal_ticks);
2816}
2817
2818static ssize_t intr_coal_ticks_v3_hw_store(struct device *dev,
2819 struct device_attribute *attr,
2820 const char *buf, size_t count)
2821{
2822 struct Scsi_Host *shost = class_to_shost(dev);
2823 struct hisi_hba *hisi_hba = shost_priv(shost);
2824 u32 intr_coal_ticks;
2825 int ret;
2826
2827 ret = kstrtou32(s: buf, base: 10, res: &intr_coal_ticks);
2828 if (ret) {
2829 dev_err(dev, "Input data of interrupt coalesce unmatch\n");
2830 return -EINVAL;
2831 }
2832
2833 if (intr_coal_ticks >= BIT(24)) {
2834 dev_err(dev, "intr_coal_ticks must be less than 2^24!\n");
2835 return -EINVAL;
2836 }
2837
2838 hisi_hba->intr_coal_ticks = intr_coal_ticks;
2839
2840 config_intr_coal_v3_hw(hisi_hba);
2841
2842 return count;
2843}
2844static DEVICE_ATTR_RW(intr_coal_ticks_v3_hw);
2845
2846static ssize_t intr_coal_count_v3_hw_show(struct device *dev,
2847 struct device_attribute
2848 *attr, char *buf)
2849{
2850 struct Scsi_Host *shost = class_to_shost(dev);
2851 struct hisi_hba *hisi_hba = shost_priv(shost);
2852
2853 return scnprintf(buf, PAGE_SIZE, fmt: "%u\n",
2854 hisi_hba->intr_coal_count);
2855}
2856
2857static ssize_t intr_coal_count_v3_hw_store(struct device *dev,
2858 struct device_attribute
2859 *attr, const char *buf, size_t count)
2860{
2861 struct Scsi_Host *shost = class_to_shost(dev);
2862 struct hisi_hba *hisi_hba = shost_priv(shost);
2863 u32 intr_coal_count;
2864 int ret;
2865
2866 ret = kstrtou32(s: buf, base: 10, res: &intr_coal_count);
2867 if (ret) {
2868 dev_err(dev, "Input data of interrupt coalesce unmatch\n");
2869 return -EINVAL;
2870 }
2871
2872 if (intr_coal_count >= BIT(8)) {
2873 dev_err(dev, "intr_coal_count must be less than 2^8!\n");
2874 return -EINVAL;
2875 }
2876
2877 hisi_hba->intr_coal_count = intr_coal_count;
2878
2879 config_intr_coal_v3_hw(hisi_hba);
2880
2881 return count;
2882}
2883static DEVICE_ATTR_RW(intr_coal_count_v3_hw);
2884
2885static ssize_t iopoll_q_cnt_v3_hw_show(struct device *dev,
2886 struct device_attribute
2887 *attr, char *buf)
2888{
2889 struct Scsi_Host *shost = class_to_shost(dev);
2890 struct hisi_hba *hisi_hba = shost_priv(shost);
2891
2892 return scnprintf(buf, PAGE_SIZE, fmt: "%u\n",
2893 hisi_hba->iopoll_q_cnt);
2894}
2895static DEVICE_ATTR_RO(iopoll_q_cnt_v3_hw);
2896
2897static int slave_configure_v3_hw(struct scsi_device *sdev)
2898{
2899 struct Scsi_Host *shost = dev_to_shost(dev: &sdev->sdev_gendev);
2900 struct hisi_hba *hisi_hba = shost_priv(shost);
2901 int ret = hisi_sas_slave_configure(sdev);
2902 struct device *dev = hisi_hba->dev;
2903
2904 if (ret)
2905 return ret;
2906
2907 if (sdev->type == TYPE_ENCLOSURE)
2908 return 0;
2909
2910 if (!device_link_add(consumer: &sdev->sdev_gendev, supplier: dev,
2911 DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE)) {
2912 if (pm_runtime_enabled(dev)) {
2913 dev_info(dev, "add device link failed, disable runtime PM for the host\n");
2914 pm_runtime_disable(dev);
2915 }
2916 }
2917
2918 return 0;
2919}
2920
2921static struct attribute *host_v3_hw_attrs[] = {
2922 &dev_attr_phy_event_threshold.attr,
2923 &dev_attr_intr_conv_v3_hw.attr,
2924 &dev_attr_intr_coal_ticks_v3_hw.attr,
2925 &dev_attr_intr_coal_count_v3_hw.attr,
2926 &dev_attr_iopoll_q_cnt_v3_hw.attr,
2927 NULL
2928};
2929
2930ATTRIBUTE_GROUPS(host_v3_hw);
2931
2932#define HISI_SAS_DEBUGFS_REG(x) {#x, x}
2933
2934struct hisi_sas_debugfs_reg_lu {
2935 char *name;
2936 int off;
2937};
2938
2939struct hisi_sas_debugfs_reg {
2940 const struct hisi_sas_debugfs_reg_lu *lu;
2941 int count;
2942 int base_off;
2943};
2944
2945static const struct hisi_sas_debugfs_reg_lu debugfs_port_reg_lu[] = {
2946 HISI_SAS_DEBUGFS_REG(PHY_CFG),
2947 HISI_SAS_DEBUGFS_REG(HARD_PHY_LINKRATE),
2948 HISI_SAS_DEBUGFS_REG(PROG_PHY_LINK_RATE),
2949 HISI_SAS_DEBUGFS_REG(PHY_CTRL),
2950 HISI_SAS_DEBUGFS_REG(SL_CFG),
2951 HISI_SAS_DEBUGFS_REG(AIP_LIMIT),
2952 HISI_SAS_DEBUGFS_REG(SL_CONTROL),
2953 HISI_SAS_DEBUGFS_REG(RX_PRIMS_STATUS),
2954 HISI_SAS_DEBUGFS_REG(TX_ID_DWORD0),
2955 HISI_SAS_DEBUGFS_REG(TX_ID_DWORD1),
2956 HISI_SAS_DEBUGFS_REG(TX_ID_DWORD2),
2957 HISI_SAS_DEBUGFS_REG(TX_ID_DWORD3),
2958 HISI_SAS_DEBUGFS_REG(TX_ID_DWORD4),
2959 HISI_SAS_DEBUGFS_REG(TX_ID_DWORD5),
2960 HISI_SAS_DEBUGFS_REG(TX_ID_DWORD6),
2961 HISI_SAS_DEBUGFS_REG(TXID_AUTO),
2962 HISI_SAS_DEBUGFS_REG(RX_IDAF_DWORD0),
2963 HISI_SAS_DEBUGFS_REG(RXOP_CHECK_CFG_H),
2964 HISI_SAS_DEBUGFS_REG(STP_LINK_TIMER),
2965 HISI_SAS_DEBUGFS_REG(STP_LINK_TIMEOUT_STATE),
2966 HISI_SAS_DEBUGFS_REG(CON_CFG_DRIVER),
2967 HISI_SAS_DEBUGFS_REG(SAS_SSP_CON_TIMER_CFG),
2968 HISI_SAS_DEBUGFS_REG(SAS_SMP_CON_TIMER_CFG),
2969 HISI_SAS_DEBUGFS_REG(SAS_STP_CON_TIMER_CFG),
2970 HISI_SAS_DEBUGFS_REG(CHL_INT0),
2971 HISI_SAS_DEBUGFS_REG(CHL_INT1),
2972 HISI_SAS_DEBUGFS_REG(CHL_INT2),
2973 HISI_SAS_DEBUGFS_REG(CHL_INT0_MSK),
2974 HISI_SAS_DEBUGFS_REG(CHL_INT1_MSK),
2975 HISI_SAS_DEBUGFS_REG(CHL_INT2_MSK),
2976 HISI_SAS_DEBUGFS_REG(SAS_EC_INT_COAL_TIME),
2977 HISI_SAS_DEBUGFS_REG(CHL_INT_COAL_EN),
2978 HISI_SAS_DEBUGFS_REG(SAS_RX_TRAIN_TIMER),
2979 HISI_SAS_DEBUGFS_REG(PHY_CTRL_RDY_MSK),
2980 HISI_SAS_DEBUGFS_REG(PHYCTRL_NOT_RDY_MSK),
2981 HISI_SAS_DEBUGFS_REG(PHYCTRL_DWS_RESET_MSK),
2982 HISI_SAS_DEBUGFS_REG(PHYCTRL_PHY_ENA_MSK),
2983 HISI_SAS_DEBUGFS_REG(SL_RX_BCAST_CHK_MSK),
2984 HISI_SAS_DEBUGFS_REG(PHYCTRL_OOB_RESTART_MSK),
2985 HISI_SAS_DEBUGFS_REG(DMA_TX_STATUS),
2986 HISI_SAS_DEBUGFS_REG(DMA_RX_STATUS),
2987 HISI_SAS_DEBUGFS_REG(COARSETUNE_TIME),
2988 HISI_SAS_DEBUGFS_REG(ERR_CNT_DWS_LOST),
2989 HISI_SAS_DEBUGFS_REG(ERR_CNT_RESET_PROB),
2990 HISI_SAS_DEBUGFS_REG(ERR_CNT_INVLD_DW),
2991 HISI_SAS_DEBUGFS_REG(ERR_CNT_CODE_ERR),
2992 HISI_SAS_DEBUGFS_REG(ERR_CNT_DISP_ERR),
2993 {}
2994};
2995
2996static const struct hisi_sas_debugfs_reg debugfs_port_reg = {
2997 .lu = debugfs_port_reg_lu,
2998 .count = 0x100,
2999 .base_off = PORT_BASE,
3000};
3001
3002static const struct hisi_sas_debugfs_reg_lu debugfs_global_reg_lu[] = {
3003 HISI_SAS_DEBUGFS_REG(DLVRY_QUEUE_ENABLE),
3004 HISI_SAS_DEBUGFS_REG(PHY_CONTEXT),
3005 HISI_SAS_DEBUGFS_REG(PHY_STATE),
3006 HISI_SAS_DEBUGFS_REG(PHY_PORT_NUM_MA),
3007 HISI_SAS_DEBUGFS_REG(PHY_CONN_RATE),
3008 HISI_SAS_DEBUGFS_REG(ITCT_CLR),
3009 HISI_SAS_DEBUGFS_REG(IO_SATA_BROKEN_MSG_ADDR_LO),
3010 HISI_SAS_DEBUGFS_REG(IO_SATA_BROKEN_MSG_ADDR_HI),
3011 HISI_SAS_DEBUGFS_REG(SATA_INITI_D2H_STORE_ADDR_LO),
3012 HISI_SAS_DEBUGFS_REG(SATA_INITI_D2H_STORE_ADDR_HI),
3013 HISI_SAS_DEBUGFS_REG(CFG_MAX_TAG),
3014 HISI_SAS_DEBUGFS_REG(TRANS_LOCK_ICT_TIME),
3015 HISI_SAS_DEBUGFS_REG(HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL),
3016 HISI_SAS_DEBUGFS_REG(HGC_SAS_TXFAIL_RETRY_CTRL),
3017 HISI_SAS_DEBUGFS_REG(HGC_GET_ITV_TIME),
3018 HISI_SAS_DEBUGFS_REG(DEVICE_MSG_WORK_MODE),
3019 HISI_SAS_DEBUGFS_REG(OPENA_WT_CONTI_TIME),
3020 HISI_SAS_DEBUGFS_REG(I_T_NEXUS_LOSS_TIME),
3021 HISI_SAS_DEBUGFS_REG(MAX_CON_TIME_LIMIT_TIME),
3022 HISI_SAS_DEBUGFS_REG(BUS_INACTIVE_LIMIT_TIME),
3023 HISI_SAS_DEBUGFS_REG(REJECT_TO_OPEN_LIMIT_TIME),
3024 HISI_SAS_DEBUGFS_REG(CQ_INT_CONVERGE_EN),
3025 HISI_SAS_DEBUGFS_REG(CFG_AGING_TIME),
3026 HISI_SAS_DEBUGFS_REG(HGC_DFX_CFG2),
3027 HISI_SAS_DEBUGFS_REG(CFG_ABT_SET_QUERY_IPTT),
3028 HISI_SAS_DEBUGFS_REG(CFG_ABT_SET_IPTT_DONE),
3029 HISI_SAS_DEBUGFS_REG(HGC_IOMB_PROC1_STATUS),
3030 HISI_SAS_DEBUGFS_REG(CHNL_INT_STATUS),
3031 HISI_SAS_DEBUGFS_REG(HGC_AXI_FIFO_ERR_INFO),
3032 HISI_SAS_DEBUGFS_REG(INT_COAL_EN),
3033 HISI_SAS_DEBUGFS_REG(OQ_INT_COAL_TIME),
3034 HISI_SAS_DEBUGFS_REG(OQ_INT_COAL_CNT),
3035 HISI_SAS_DEBUGFS_REG(ENT_INT_COAL_TIME),
3036 HISI_SAS_DEBUGFS_REG(ENT_INT_COAL_CNT),
3037 HISI_SAS_DEBUGFS_REG(OQ_INT_SRC),
3038 HISI_SAS_DEBUGFS_REG(OQ_INT_SRC_MSK),
3039 HISI_SAS_DEBUGFS_REG(ENT_INT_SRC1),
3040 HISI_SAS_DEBUGFS_REG(ENT_INT_SRC2),
3041 HISI_SAS_DEBUGFS_REG(ENT_INT_SRC3),
3042 HISI_SAS_DEBUGFS_REG(ENT_INT_SRC_MSK1),
3043 HISI_SAS_DEBUGFS_REG(ENT_INT_SRC_MSK2),
3044 HISI_SAS_DEBUGFS_REG(ENT_INT_SRC_MSK3),
3045 HISI_SAS_DEBUGFS_REG(CHNL_PHYUPDOWN_INT_MSK),
3046 HISI_SAS_DEBUGFS_REG(CHNL_ENT_INT_MSK),
3047 HISI_SAS_DEBUGFS_REG(HGC_COM_INT_MSK),
3048 HISI_SAS_DEBUGFS_REG(SAS_ECC_INTR),
3049 HISI_SAS_DEBUGFS_REG(SAS_ECC_INTR_MSK),
3050 HISI_SAS_DEBUGFS_REG(HGC_ERR_STAT_EN),
3051 HISI_SAS_DEBUGFS_REG(CQE_SEND_CNT),
3052 HISI_SAS_DEBUGFS_REG(DLVRY_Q_0_DEPTH),
3053 HISI_SAS_DEBUGFS_REG(DLVRY_Q_0_WR_PTR),
3054 HISI_SAS_DEBUGFS_REG(DLVRY_Q_0_RD_PTR),
3055 HISI_SAS_DEBUGFS_REG(HYPER_STREAM_ID_EN_CFG),
3056 HISI_SAS_DEBUGFS_REG(OQ0_INT_SRC_MSK),
3057 HISI_SAS_DEBUGFS_REG(COMPL_Q_0_DEPTH),
3058 HISI_SAS_DEBUGFS_REG(COMPL_Q_0_WR_PTR),
3059 HISI_SAS_DEBUGFS_REG(COMPL_Q_0_RD_PTR),
3060 HISI_SAS_DEBUGFS_REG(AWQOS_AWCACHE_CFG),
3061 HISI_SAS_DEBUGFS_REG(ARQOS_ARCACHE_CFG),
3062 HISI_SAS_DEBUGFS_REG(HILINK_ERR_DFX),
3063 HISI_SAS_DEBUGFS_REG(SAS_GPIO_CFG_0),
3064 HISI_SAS_DEBUGFS_REG(SAS_GPIO_CFG_1),
3065 HISI_SAS_DEBUGFS_REG(SAS_GPIO_TX_0_1),
3066 HISI_SAS_DEBUGFS_REG(SAS_CFG_DRIVE_VLD),
3067 {}
3068};
3069
3070static const struct hisi_sas_debugfs_reg debugfs_global_reg = {
3071 .lu = debugfs_global_reg_lu,
3072 .count = 0x800,
3073};
3074
3075static const struct hisi_sas_debugfs_reg_lu debugfs_axi_reg_lu[] = {
3076 HISI_SAS_DEBUGFS_REG(AM_CFG_MAX_TRANS),
3077 HISI_SAS_DEBUGFS_REG(AM_CFG_SINGLE_PORT_MAX_TRANS),
3078 HISI_SAS_DEBUGFS_REG(AXI_CFG),
3079 HISI_SAS_DEBUGFS_REG(AM_ROB_ECC_ERR_ADDR),
3080 {}
3081};
3082
3083static const struct hisi_sas_debugfs_reg debugfs_axi_reg = {
3084 .lu = debugfs_axi_reg_lu,
3085 .count = 0x61,
3086 .base_off = AXI_MASTER_CFG_BASE,
3087};
3088
3089static const struct hisi_sas_debugfs_reg_lu debugfs_ras_reg_lu[] = {
3090 HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR0),
3091 HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR1),
3092 HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR0_MASK),
3093 HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR1_MASK),
3094 HISI_SAS_DEBUGFS_REG(CFG_SAS_RAS_INTR_MASK),
3095 HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR2),
3096 HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR2_MASK),
3097 {}
3098};
3099
3100static const struct hisi_sas_debugfs_reg debugfs_ras_reg = {
3101 .lu = debugfs_ras_reg_lu,
3102 .count = 0x10,
3103 .base_off = RAS_BASE,
3104};
3105
3106static void debugfs_snapshot_prepare_v3_hw(struct hisi_hba *hisi_hba)
3107{
3108 struct Scsi_Host *shost = hisi_hba->shost;
3109
3110 scsi_block_requests(shost);
3111 wait_cmds_complete_timeout_v3_hw(hisi_hba, delay_ms: 100, timeout_ms: 5000);
3112
3113 set_bit(HISI_SAS_REJECT_CMD_BIT, addr: &hisi_hba->flags);
3114 hisi_sas_sync_cqs(hisi_hba);
3115 hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, val: 0);
3116}
3117
3118static void debugfs_snapshot_restore_v3_hw(struct hisi_hba *hisi_hba)
3119{
3120 struct Scsi_Host *shost = hisi_hba->shost;
3121
3122 hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
3123 val: (u32)((1ULL << hisi_hba->queue_count) - 1));
3124
3125 clear_bit(HISI_SAS_REJECT_CMD_BIT, addr: &hisi_hba->flags);
3126 scsi_unblock_requests(shost);
3127}
3128
3129static void read_iost_itct_cache_v3_hw(struct hisi_hba *hisi_hba,
3130 enum hisi_sas_debugfs_cache_type type,
3131 u32 *cache)
3132{
3133 u32 cache_dw_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ *
3134 HISI_SAS_IOST_ITCT_CACHE_NUM;
3135 struct device *dev = hisi_hba->dev;
3136 u32 *buf = cache;
3137 u32 i, val;
3138
3139 hisi_sas_write32(hisi_hba, TAB_RD_TYPE, val: type);
3140
3141 for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_DW_SZ; i++) {
3142 val = hisi_sas_read32(hisi_hba, TAB_DFX);
3143 if (val == 0xffffffff)
3144 break;
3145 }
3146
3147 if (val != 0xffffffff) {
3148 dev_err(dev, "Issue occurred in reading IOST/ITCT cache!\n");
3149 return;
3150 }
3151
3152 memset(buf, 0, cache_dw_size * 4);
3153 buf[0] = val;
3154
3155 for (i = 1; i < cache_dw_size; i++)
3156 buf[i] = hisi_sas_read32(hisi_hba, TAB_DFX);
3157}
3158
3159static void hisi_sas_bist_test_prep_v3_hw(struct hisi_hba *hisi_hba)
3160{
3161 u32 reg_val;
3162 int phy_no = hisi_hba->debugfs_bist_phy_no;
3163 int i;
3164
3165 /* disable PHY */
3166 hisi_sas_phy_enable(hisi_hba, phy_no, enable: 0);
3167
3168 /* update FFE */
3169 for (i = 0; i < FFE_CFG_MAX; i++)
3170 hisi_sas_phy_write32(hisi_hba, phy_no, TXDEEMPH_G1 + (i * 0x4),
3171 val: hisi_hba->debugfs_bist_ffe[phy_no][i]);
3172
3173 /* disable ALOS */
3174 reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, SERDES_CFG);
3175 reg_val |= CFG_ALOS_CHK_DISABLE_MSK;
3176 hisi_sas_phy_write32(hisi_hba, phy_no, SERDES_CFG, val: reg_val);
3177}
3178
3179static void hisi_sas_bist_test_restore_v3_hw(struct hisi_hba *hisi_hba)
3180{
3181 u32 reg_val;
3182 int phy_no = hisi_hba->debugfs_bist_phy_no;
3183
3184 /* disable loopback */
3185 reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL);
3186 reg_val &= ~(CFG_RX_BIST_EN_MSK | CFG_TX_BIST_EN_MSK |
3187 CFG_BIST_TEST_MSK);
3188 hisi_sas_phy_write32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL, val: reg_val);
3189
3190 /* enable ALOS */
3191 reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, SERDES_CFG);
3192 reg_val &= ~CFG_ALOS_CHK_DISABLE_MSK;
3193 hisi_sas_phy_write32(hisi_hba, phy_no, SERDES_CFG, val: reg_val);
3194
3195 /* restore the linkrate */
3196 reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE);
3197 /* init OOB link rate as 1.5 Gbits */
3198 reg_val &= ~CFG_PROG_OOB_PHY_LINK_RATE_MSK;
3199 reg_val |= (0x8 << CFG_PROG_OOB_PHY_LINK_RATE_OFF);
3200 hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE, val: reg_val);
3201
3202 /* enable PHY */
3203 hisi_sas_phy_enable(hisi_hba, phy_no, enable: 1);
3204}
3205
3206#define SAS_PHY_BIST_CODE_INIT 0x1
3207#define SAS_PHY_BIST_CODE1_INIT 0X80
3208static int debugfs_set_bist_v3_hw(struct hisi_hba *hisi_hba, bool enable)
3209{
3210 u32 reg_val, mode_tmp;
3211 u32 linkrate = hisi_hba->debugfs_bist_linkrate;
3212 u32 phy_no = hisi_hba->debugfs_bist_phy_no;
3213 u32 *ffe = hisi_hba->debugfs_bist_ffe[phy_no];
3214 u32 code_mode = hisi_hba->debugfs_bist_code_mode;
3215 u32 path_mode = hisi_hba->debugfs_bist_mode;
3216 u32 *fix_code = &hisi_hba->debugfs_bist_fixed_code[0];
3217 struct device *dev = hisi_hba->dev;
3218
3219 dev_info(dev, "BIST info:phy%d link_rate=%d code_mode=%d path_mode=%d ffe={0x%x, 0x%x, 0x%x, 0x%x, 0x%x, 0x%x, 0x%x} fixed_code={0x%x, 0x%x}\n",
3220 phy_no, linkrate, code_mode, path_mode,
3221 ffe[FFE_SAS_1_5_GBPS], ffe[FFE_SAS_3_0_GBPS],
3222 ffe[FFE_SAS_6_0_GBPS], ffe[FFE_SAS_12_0_GBPS],
3223 ffe[FFE_SATA_1_5_GBPS], ffe[FFE_SATA_3_0_GBPS],
3224 ffe[FFE_SATA_6_0_GBPS], fix_code[FIXED_CODE],
3225 fix_code[FIXED_CODE_1]);
3226 mode_tmp = path_mode ? 2 : 1;
3227 if (enable) {
3228 /* some preparations before bist test */
3229 hisi_sas_bist_test_prep_v3_hw(hisi_hba);
3230
3231 /* set linkrate of bit test*/
3232 reg_val = hisi_sas_phy_read32(hisi_hba, phy_no,
3233 PROG_PHY_LINK_RATE);
3234 reg_val &= ~CFG_PROG_OOB_PHY_LINK_RATE_MSK;
3235 reg_val |= (linkrate << CFG_PROG_OOB_PHY_LINK_RATE_OFF);
3236 hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
3237 val: reg_val);
3238
3239 /* set code mode of bit test */
3240 reg_val = hisi_sas_phy_read32(hisi_hba, phy_no,
3241 SAS_PHY_BIST_CTRL);
3242 reg_val &= ~(CFG_BIST_MODE_SEL_MSK | CFG_LOOP_TEST_MODE_MSK |
3243 CFG_RX_BIST_EN_MSK | CFG_TX_BIST_EN_MSK |
3244 CFG_BIST_TEST_MSK);
3245 reg_val |= ((code_mode << CFG_BIST_MODE_SEL_OFF) |
3246 (mode_tmp << CFG_LOOP_TEST_MODE_OFF) |
3247 CFG_BIST_TEST_MSK);
3248 hisi_sas_phy_write32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL,
3249 val: reg_val);
3250
3251 /* set the bist init value */
3252 if (code_mode == HISI_SAS_BIST_CODE_MODE_FIXED_DATA) {
3253 reg_val = hisi_hba->debugfs_bist_fixed_code[0];
3254 hisi_sas_phy_write32(hisi_hba, phy_no,
3255 SAS_PHY_BIST_CODE, val: reg_val);
3256
3257 reg_val = hisi_hba->debugfs_bist_fixed_code[1];
3258 hisi_sas_phy_write32(hisi_hba, phy_no,
3259 SAS_PHY_BIST_CODE1, val: reg_val);
3260 } else {
3261 hisi_sas_phy_write32(hisi_hba, phy_no,
3262 SAS_PHY_BIST_CODE,
3263 SAS_PHY_BIST_CODE_INIT);
3264 hisi_sas_phy_write32(hisi_hba, phy_no,
3265 SAS_PHY_BIST_CODE1,
3266 SAS_PHY_BIST_CODE1_INIT);
3267 }
3268
3269 mdelay(100);
3270 reg_val |= (CFG_RX_BIST_EN_MSK | CFG_TX_BIST_EN_MSK);
3271 hisi_sas_phy_write32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL,
3272 val: reg_val);
3273
3274 /* clear error bit */
3275 mdelay(100);
3276 hisi_sas_phy_read32(hisi_hba, phy_no, SAS_BIST_ERR_CNT);
3277 } else {
3278 /* disable bist test and recover it */
3279 hisi_hba->debugfs_bist_cnt += hisi_sas_phy_read32(hisi_hba,
3280 phy_no, SAS_BIST_ERR_CNT);
3281 hisi_sas_bist_test_restore_v3_hw(hisi_hba);
3282 }
3283
3284 return 0;
3285}
3286
3287static void hisi_sas_map_queues(struct Scsi_Host *shost)
3288{
3289 struct hisi_hba *hisi_hba = shost_priv(shost);
3290 struct blk_mq_queue_map *qmap;
3291 int i, qoff;
3292
3293 for (i = 0, qoff = 0; i < shost->nr_maps; i++) {
3294 qmap = &shost->tag_set.map[i];
3295 if (i == HCTX_TYPE_DEFAULT) {
3296 qmap->nr_queues = hisi_hba->cq_nvecs;
3297 } else if (i == HCTX_TYPE_POLL) {
3298 qmap->nr_queues = hisi_hba->iopoll_q_cnt;
3299 } else {
3300 qmap->nr_queues = 0;
3301 continue;
3302 }
3303
3304 /* At least one interrupt hardware queue */
3305 if (!qmap->nr_queues)
3306 WARN_ON(i == HCTX_TYPE_DEFAULT);
3307 qmap->queue_offset = qoff;
3308 if (i == HCTX_TYPE_POLL)
3309 blk_mq_map_queues(qmap);
3310 else
3311 blk_mq_pci_map_queues(qmap, pdev: hisi_hba->pci_dev,
3312 BASE_VECTORS_V3_HW);
3313 qoff += qmap->nr_queues;
3314 }
3315}
3316
3317static const struct scsi_host_template sht_v3_hw = {
3318 .name = DRV_NAME,
3319 .proc_name = DRV_NAME,
3320 .module = THIS_MODULE,
3321 .queuecommand = sas_queuecommand,
3322 .dma_need_drain = ata_scsi_dma_need_drain,
3323 .target_alloc = sas_target_alloc,
3324 .slave_configure = slave_configure_v3_hw,
3325 .scan_finished = hisi_sas_scan_finished,
3326 .scan_start = hisi_sas_scan_start,
3327 .map_queues = hisi_sas_map_queues,
3328 .change_queue_depth = sas_change_queue_depth,
3329 .bios_param = sas_bios_param,
3330 .this_id = -1,
3331 .sg_tablesize = HISI_SAS_SGE_PAGE_CNT,
3332 .sg_prot_tablesize = HISI_SAS_SGE_PAGE_CNT,
3333 .max_sectors = SCSI_DEFAULT_MAX_SECTORS,
3334 .eh_device_reset_handler = sas_eh_device_reset_handler,
3335 .eh_target_reset_handler = sas_eh_target_reset_handler,
3336 .slave_alloc = hisi_sas_slave_alloc,
3337 .target_destroy = sas_target_destroy,
3338 .ioctl = sas_ioctl,
3339#ifdef CONFIG_COMPAT
3340 .compat_ioctl = sas_ioctl,
3341#endif
3342 .shost_groups = host_v3_hw_groups,
3343 .tag_alloc_policy = BLK_TAG_ALLOC_RR,
3344 .host_reset = hisi_sas_host_reset,
3345 .host_tagset = 1,
3346 .mq_poll = queue_complete_v3_hw,
3347};
3348
3349static const struct hisi_sas_hw hisi_sas_v3_hw = {
3350 .setup_itct = setup_itct_v3_hw,
3351 .get_wideport_bitmap = get_wideport_bitmap_v3_hw,
3352 .complete_hdr_size = sizeof(struct hisi_sas_complete_v3_hdr),
3353 .clear_itct = clear_itct_v3_hw,
3354 .sl_notify_ssp = sl_notify_ssp_v3_hw,
3355 .prep_ssp = prep_ssp_v3_hw,
3356 .prep_smp = prep_smp_v3_hw,
3357 .prep_stp = prep_ata_v3_hw,
3358 .prep_abort = prep_abort_v3_hw,
3359 .start_delivery = start_delivery_v3_hw,
3360 .phys_init = phys_init_v3_hw,
3361 .phy_start = start_phy_v3_hw,
3362 .phy_disable = disable_phy_v3_hw,
3363 .phy_hard_reset = phy_hard_reset_v3_hw,
3364 .phy_get_max_linkrate = phy_get_max_linkrate_v3_hw,
3365 .phy_set_linkrate = phy_set_linkrate_v3_hw,
3366 .dereg_device = dereg_device_v3_hw,
3367 .soft_reset = soft_reset_v3_hw,
3368 .get_phys_state = get_phys_state_v3_hw,
3369 .get_events = phy_get_events_v3_hw,
3370 .write_gpio = write_gpio_v3_hw,
3371 .wait_cmds_complete_timeout = wait_cmds_complete_timeout_v3_hw,
3372 .debugfs_snapshot_regs = debugfs_snapshot_regs_v3_hw,
3373};
3374
3375static struct Scsi_Host *
3376hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
3377{
3378 struct Scsi_Host *shost;
3379 struct hisi_hba *hisi_hba;
3380 struct device *dev = &pdev->dev;
3381
3382 shost = scsi_host_alloc(&sht_v3_hw, sizeof(*hisi_hba));
3383 if (!shost) {
3384 dev_err(dev, "shost alloc failed\n");
3385 return NULL;
3386 }
3387 hisi_hba = shost_priv(shost);
3388
3389 INIT_WORK(&hisi_hba->rst_work, hisi_sas_rst_work_handler);
3390 hisi_hba->hw = &hisi_sas_v3_hw;
3391 hisi_hba->pci_dev = pdev;
3392 hisi_hba->dev = dev;
3393 hisi_hba->shost = shost;
3394 SHOST_TO_SAS_HA(shost) = &hisi_hba->sha;
3395
3396 if (prot_mask & ~HISI_SAS_PROT_MASK)
3397 dev_err(dev, "unsupported protection mask 0x%x, using default (0x0)\n",
3398 prot_mask);
3399 else
3400 hisi_hba->prot_mask = prot_mask;
3401
3402 if (hisi_sas_get_fw_info(hisi_hba) < 0)
3403 goto err_out;
3404
3405 if (experimental_iopoll_q_cnt < 0 ||
3406 experimental_iopoll_q_cnt >= hisi_hba->queue_count)
3407 dev_err(dev, "iopoll queue count %d cannot exceed or equal 16, using default 0\n",
3408 experimental_iopoll_q_cnt);
3409 else
3410 hisi_hba->iopoll_q_cnt = experimental_iopoll_q_cnt;
3411
3412 if (hisi_sas_alloc(hisi_hba)) {
3413 hisi_sas_free(hisi_hba);
3414 goto err_out;
3415 }
3416
3417 return shost;
3418err_out:
3419 scsi_host_put(t: shost);
3420 dev_err(dev, "shost alloc failed\n");
3421 return NULL;
3422}
3423
3424static void debugfs_snapshot_cq_reg_v3_hw(struct hisi_hba *hisi_hba)
3425{
3426 int queue_entry_size = hisi_hba->hw->complete_hdr_size;
3427 int dump_index = hisi_hba->debugfs_dump_index;
3428 int i;
3429
3430 for (i = 0; i < hisi_hba->queue_count; i++)
3431 memcpy(hisi_hba->debugfs_cq[dump_index][i].complete_hdr,
3432 hisi_hba->complete_hdr[i],
3433 HISI_SAS_QUEUE_SLOTS * queue_entry_size);
3434}
3435
3436static void debugfs_snapshot_dq_reg_v3_hw(struct hisi_hba *hisi_hba)
3437{
3438 int queue_entry_size = sizeof(struct hisi_sas_cmd_hdr);
3439 int dump_index = hisi_hba->debugfs_dump_index;
3440 int i;
3441
3442 for (i = 0; i < hisi_hba->queue_count; i++) {
3443 struct hisi_sas_cmd_hdr *debugfs_cmd_hdr, *cmd_hdr;
3444 int j;
3445
3446 debugfs_cmd_hdr = hisi_hba->debugfs_dq[dump_index][i].hdr;
3447 cmd_hdr = hisi_hba->cmd_hdr[i];
3448
3449 for (j = 0; j < HISI_SAS_QUEUE_SLOTS; j++)
3450 memcpy(&debugfs_cmd_hdr[j], &cmd_hdr[j],
3451 queue_entry_size);
3452 }
3453}
3454
3455static void debugfs_snapshot_port_reg_v3_hw(struct hisi_hba *hisi_hba)
3456{
3457 int dump_index = hisi_hba->debugfs_dump_index;
3458 const struct hisi_sas_debugfs_reg *port = &debugfs_port_reg;
3459 int i, phy_cnt;
3460 u32 offset;
3461 u32 *databuf;
3462
3463 for (phy_cnt = 0; phy_cnt < hisi_hba->n_phy; phy_cnt++) {
3464 databuf = hisi_hba->debugfs_port_reg[dump_index][phy_cnt].data;
3465 for (i = 0; i < port->count; i++, databuf++) {
3466 offset = port->base_off + 4 * i;
3467 *databuf = hisi_sas_phy_read32(hisi_hba, phy_no: phy_cnt,
3468 off: offset);
3469 }
3470 }
3471}
3472
3473static void debugfs_snapshot_global_reg_v3_hw(struct hisi_hba *hisi_hba)
3474{
3475 int dump_index = hisi_hba->debugfs_dump_index;
3476 u32 *databuf = hisi_hba->debugfs_regs[dump_index][DEBUGFS_GLOBAL].data;
3477 int i;
3478
3479 for (i = 0; i < debugfs_axi_reg.count; i++, databuf++)
3480 *databuf = hisi_sas_read32(hisi_hba, off: 4 * i);
3481}
3482
3483static void debugfs_snapshot_axi_reg_v3_hw(struct hisi_hba *hisi_hba)
3484{
3485 int dump_index = hisi_hba->debugfs_dump_index;
3486 u32 *databuf = hisi_hba->debugfs_regs[dump_index][DEBUGFS_AXI].data;
3487 const struct hisi_sas_debugfs_reg *axi = &debugfs_axi_reg;
3488 int i;
3489
3490 for (i = 0; i < axi->count; i++, databuf++)
3491 *databuf = hisi_sas_read32(hisi_hba, off: 4 * i + axi->base_off);
3492}
3493
3494static void debugfs_snapshot_ras_reg_v3_hw(struct hisi_hba *hisi_hba)
3495{
3496 int dump_index = hisi_hba->debugfs_dump_index;
3497 u32 *databuf = hisi_hba->debugfs_regs[dump_index][DEBUGFS_RAS].data;
3498 const struct hisi_sas_debugfs_reg *ras = &debugfs_ras_reg;
3499 int i;
3500
3501 for (i = 0; i < ras->count; i++, databuf++)
3502 *databuf = hisi_sas_read32(hisi_hba, off: 4 * i + ras->base_off);
3503}
3504
3505static void debugfs_snapshot_itct_reg_v3_hw(struct hisi_hba *hisi_hba)
3506{
3507 int dump_index = hisi_hba->debugfs_dump_index;
3508 void *cachebuf = hisi_hba->debugfs_itct_cache[dump_index].cache;
3509 void *databuf = hisi_hba->debugfs_itct[dump_index].itct;
3510 struct hisi_sas_itct *itct;
3511 int i;
3512
3513 read_iost_itct_cache_v3_hw(hisi_hba, type: HISI_SAS_ITCT_CACHE, cache: cachebuf);
3514
3515 itct = hisi_hba->itct;
3516
3517 for (i = 0; i < HISI_SAS_MAX_ITCT_ENTRIES; i++, itct++) {
3518 memcpy(databuf, itct, sizeof(struct hisi_sas_itct));
3519 databuf += sizeof(struct hisi_sas_itct);
3520 }
3521}
3522
3523static void debugfs_snapshot_iost_reg_v3_hw(struct hisi_hba *hisi_hba)
3524{
3525 int dump_index = hisi_hba->debugfs_dump_index;
3526 int max_command_entries = HISI_SAS_MAX_COMMANDS;
3527 void *cachebuf = hisi_hba->debugfs_iost_cache[dump_index].cache;
3528 void *databuf = hisi_hba->debugfs_iost[dump_index].iost;
3529 struct hisi_sas_iost *iost;
3530 int i;
3531
3532 read_iost_itct_cache_v3_hw(hisi_hba, type: HISI_SAS_IOST_CACHE, cache: cachebuf);
3533
3534 iost = hisi_hba->iost;
3535
3536 for (i = 0; i < max_command_entries; i++, iost++) {
3537 memcpy(databuf, iost, sizeof(struct hisi_sas_iost));
3538 databuf += sizeof(struct hisi_sas_iost);
3539 }
3540}
3541
3542static const char *
3543debugfs_to_reg_name_v3_hw(int off, int base_off,
3544 const struct hisi_sas_debugfs_reg_lu *lu)
3545{
3546 for (; lu->name; lu++) {
3547 if (off == lu->off - base_off)
3548 return lu->name;
3549 }
3550
3551 return NULL;
3552}
3553
3554static void debugfs_print_reg_v3_hw(u32 *regs_val, struct seq_file *s,
3555 const struct hisi_sas_debugfs_reg *reg)
3556{
3557 int i;
3558
3559 for (i = 0; i < reg->count; i++) {
3560 int off = i * 4;
3561 const char *name;
3562
3563 name = debugfs_to_reg_name_v3_hw(off, base_off: reg->base_off,
3564 lu: reg->lu);
3565
3566 if (name)
3567 seq_printf(m: s, fmt: "0x%08x 0x%08x %s\n", off,
3568 regs_val[i], name);
3569 else
3570 seq_printf(m: s, fmt: "0x%08x 0x%08x\n", off,
3571 regs_val[i]);
3572 }
3573}
3574
3575static int debugfs_global_v3_hw_show(struct seq_file *s, void *p)
3576{
3577 struct hisi_sas_debugfs_regs *global = s->private;
3578
3579 debugfs_print_reg_v3_hw(regs_val: global->data, s,
3580 reg: &debugfs_global_reg);
3581
3582 return 0;
3583}
3584DEFINE_SHOW_ATTRIBUTE(debugfs_global_v3_hw);
3585
3586static int debugfs_axi_v3_hw_show(struct seq_file *s, void *p)
3587{
3588 struct hisi_sas_debugfs_regs *axi = s->private;
3589
3590 debugfs_print_reg_v3_hw(regs_val: axi->data, s,
3591 reg: &debugfs_axi_reg);
3592
3593 return 0;
3594}
3595DEFINE_SHOW_ATTRIBUTE(debugfs_axi_v3_hw);
3596
3597static int debugfs_ras_v3_hw_show(struct seq_file *s, void *p)
3598{
3599 struct hisi_sas_debugfs_regs *ras = s->private;
3600
3601 debugfs_print_reg_v3_hw(regs_val: ras->data, s,
3602 reg: &debugfs_ras_reg);
3603
3604 return 0;
3605}
3606DEFINE_SHOW_ATTRIBUTE(debugfs_ras_v3_hw);
3607
3608static int debugfs_port_v3_hw_show(struct seq_file *s, void *p)
3609{
3610 struct hisi_sas_debugfs_port *port = s->private;
3611 const struct hisi_sas_debugfs_reg *reg_port = &debugfs_port_reg;
3612
3613 debugfs_print_reg_v3_hw(regs_val: port->data, s, reg: reg_port);
3614
3615 return 0;
3616}
3617DEFINE_SHOW_ATTRIBUTE(debugfs_port_v3_hw);
3618
3619static void debugfs_show_row_64_v3_hw(struct seq_file *s, int index,
3620 int sz, __le64 *ptr)
3621{
3622 int i;
3623
3624 /* completion header size not fixed per HW version */
3625 seq_printf(m: s, fmt: "index %04d:\n\t", index);
3626 for (i = 1; i <= sz / 8; i++, ptr++) {
3627 seq_printf(m: s, fmt: " 0x%016llx", le64_to_cpu(*ptr));
3628 if (!(i % 2))
3629 seq_puts(m: s, s: "\n\t");
3630 }
3631
3632 seq_puts(m: s, s: "\n");
3633}
3634
3635static void debugfs_show_row_32_v3_hw(struct seq_file *s, int index,
3636 int sz, __le32 *ptr)
3637{
3638 int i;
3639
3640 /* completion header size not fixed per HW version */
3641 seq_printf(m: s, fmt: "index %04d:\n\t", index);
3642 for (i = 1; i <= sz / 4; i++, ptr++) {
3643 seq_printf(m: s, fmt: " 0x%08x", le32_to_cpu(*ptr));
3644 if (!(i % 4))
3645 seq_puts(m: s, s: "\n\t");
3646 }
3647 seq_puts(m: s, s: "\n");
3648}
3649
3650static void debugfs_cq_show_slot_v3_hw(struct seq_file *s, int slot,
3651 struct hisi_sas_debugfs_cq *debugfs_cq)
3652{
3653 struct hisi_sas_cq *cq = debugfs_cq->cq;
3654 struct hisi_hba *hisi_hba = cq->hisi_hba;
3655 __le32 *complete_hdr = debugfs_cq->complete_hdr +
3656 (hisi_hba->hw->complete_hdr_size * slot);
3657
3658 debugfs_show_row_32_v3_hw(s, index: slot,
3659 sz: hisi_hba->hw->complete_hdr_size,
3660 ptr: complete_hdr);
3661}
3662
3663static int debugfs_cq_v3_hw_show(struct seq_file *s, void *p)
3664{
3665 struct hisi_sas_debugfs_cq *debugfs_cq = s->private;
3666 int slot;
3667
3668 for (slot = 0; slot < HISI_SAS_QUEUE_SLOTS; slot++)
3669 debugfs_cq_show_slot_v3_hw(s, slot, debugfs_cq);
3670
3671 return 0;
3672}
3673DEFINE_SHOW_ATTRIBUTE(debugfs_cq_v3_hw);
3674
3675static void debugfs_dq_show_slot_v3_hw(struct seq_file *s, int slot,
3676 void *dq_ptr)
3677{
3678 struct hisi_sas_debugfs_dq *debugfs_dq = dq_ptr;
3679 void *cmd_queue = debugfs_dq->hdr;
3680 __le32 *cmd_hdr = cmd_queue +
3681 sizeof(struct hisi_sas_cmd_hdr) * slot;
3682
3683 debugfs_show_row_32_v3_hw(s, index: slot, sz: sizeof(struct hisi_sas_cmd_hdr),
3684 ptr: cmd_hdr);
3685}
3686
3687static int debugfs_dq_v3_hw_show(struct seq_file *s, void *p)
3688{
3689 int slot;
3690
3691 for (slot = 0; slot < HISI_SAS_QUEUE_SLOTS; slot++)
3692 debugfs_dq_show_slot_v3_hw(s, slot, dq_ptr: s->private);
3693
3694 return 0;
3695}
3696DEFINE_SHOW_ATTRIBUTE(debugfs_dq_v3_hw);
3697
3698static int debugfs_iost_v3_hw_show(struct seq_file *s, void *p)
3699{
3700 struct hisi_sas_debugfs_iost *debugfs_iost = s->private;
3701 struct hisi_sas_iost *iost = debugfs_iost->iost;
3702 int i, max_command_entries = HISI_SAS_MAX_COMMANDS;
3703
3704 for (i = 0; i < max_command_entries; i++, iost++) {
3705 __le64 *data = &iost->qw0;
3706
3707 debugfs_show_row_64_v3_hw(s, index: i, sz: sizeof(*iost), ptr: data);
3708 }
3709
3710 return 0;
3711}
3712DEFINE_SHOW_ATTRIBUTE(debugfs_iost_v3_hw);
3713
3714static int debugfs_iost_cache_v3_hw_show(struct seq_file *s, void *p)
3715{
3716 struct hisi_sas_debugfs_iost_cache *debugfs_iost_cache = s->private;
3717 struct hisi_sas_iost_itct_cache *iost_cache =
3718 debugfs_iost_cache->cache;
3719 u32 cache_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ * 4;
3720 int i, tab_idx;
3721 __le64 *iost;
3722
3723 for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_NUM; i++, iost_cache++) {
3724 /*
3725 * Data struct of IOST cache:
3726 * Data[1]: BIT0~15: Table index
3727 * Bit16: Valid mask
3728 * Data[2]~[9]: IOST table
3729 */
3730 tab_idx = (iost_cache->data[1] & 0xffff);
3731 iost = (__le64 *)iost_cache;
3732
3733 debugfs_show_row_64_v3_hw(s, index: tab_idx, sz: cache_size, ptr: iost);
3734 }
3735
3736 return 0;
3737}
3738DEFINE_SHOW_ATTRIBUTE(debugfs_iost_cache_v3_hw);
3739
3740static int debugfs_itct_v3_hw_show(struct seq_file *s, void *p)
3741{
3742 int i;
3743 struct hisi_sas_debugfs_itct *debugfs_itct = s->private;
3744 struct hisi_sas_itct *itct = debugfs_itct->itct;
3745
3746 for (i = 0; i < HISI_SAS_MAX_ITCT_ENTRIES; i++, itct++) {
3747 __le64 *data = &itct->qw0;
3748
3749 debugfs_show_row_64_v3_hw(s, index: i, sz: sizeof(*itct), ptr: data);
3750 }
3751
3752 return 0;
3753}
3754DEFINE_SHOW_ATTRIBUTE(debugfs_itct_v3_hw);
3755
3756static int debugfs_itct_cache_v3_hw_show(struct seq_file *s, void *p)
3757{
3758 struct hisi_sas_debugfs_itct_cache *debugfs_itct_cache = s->private;
3759 struct hisi_sas_iost_itct_cache *itct_cache =
3760 debugfs_itct_cache->cache;
3761 u32 cache_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ * 4;
3762 int i, tab_idx;
3763 __le64 *itct;
3764
3765 for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_NUM; i++, itct_cache++) {
3766 /*
3767 * Data struct of ITCT cache:
3768 * Data[1]: BIT0~15: Table index
3769 * Bit16: Valid mask
3770 * Data[2]~[9]: ITCT table
3771 */
3772 tab_idx = itct_cache->data[1] & 0xffff;
3773 itct = (__le64 *)itct_cache;
3774
3775 debugfs_show_row_64_v3_hw(s, index: tab_idx, sz: cache_size, ptr: itct);
3776 }
3777
3778 return 0;
3779}
3780DEFINE_SHOW_ATTRIBUTE(debugfs_itct_cache_v3_hw);
3781
3782static void debugfs_create_files_v3_hw(struct hisi_hba *hisi_hba)
3783{
3784 u64 *debugfs_timestamp;
3785 int dump_index = hisi_hba->debugfs_dump_index;
3786 struct dentry *dump_dentry;
3787 struct dentry *dentry;
3788 char name[256];
3789 int p;
3790 int c;
3791 int d;
3792
3793 snprintf(buf: name, size: 256, fmt: "%d", dump_index);
3794
3795 dump_dentry = debugfs_create_dir(name, parent: hisi_hba->debugfs_dump_dentry);
3796
3797 debugfs_timestamp = &hisi_hba->debugfs_timestamp[dump_index];
3798
3799 debugfs_create_u64(name: "timestamp", mode: 0400, parent: dump_dentry,
3800 value: debugfs_timestamp);
3801
3802 debugfs_create_file(name: "global", mode: 0400, parent: dump_dentry,
3803 data: &hisi_hba->debugfs_regs[dump_index][DEBUGFS_GLOBAL],
3804 fops: &debugfs_global_v3_hw_fops);
3805
3806 /* Create port dir and files */
3807 dentry = debugfs_create_dir(name: "port", parent: dump_dentry);
3808 for (p = 0; p < hisi_hba->n_phy; p++) {
3809 snprintf(buf: name, size: 256, fmt: "%d", p);
3810
3811 debugfs_create_file(name, mode: 0400, parent: dentry,
3812 data: &hisi_hba->debugfs_port_reg[dump_index][p],
3813 fops: &debugfs_port_v3_hw_fops);
3814 }
3815
3816 /* Create CQ dir and files */
3817 dentry = debugfs_create_dir(name: "cq", parent: dump_dentry);
3818 for (c = 0; c < hisi_hba->queue_count; c++) {
3819 snprintf(buf: name, size: 256, fmt: "%d", c);
3820
3821 debugfs_create_file(name, mode: 0400, parent: dentry,
3822 data: &hisi_hba->debugfs_cq[dump_index][c],
3823 fops: &debugfs_cq_v3_hw_fops);
3824 }
3825
3826 /* Create DQ dir and files */
3827 dentry = debugfs_create_dir(name: "dq", parent: dump_dentry);
3828 for (d = 0; d < hisi_hba->queue_count; d++) {
3829 snprintf(buf: name, size: 256, fmt: "%d", d);
3830
3831 debugfs_create_file(name, mode: 0400, parent: dentry,
3832 data: &hisi_hba->debugfs_dq[dump_index][d],
3833 fops: &debugfs_dq_v3_hw_fops);
3834 }
3835
3836 debugfs_create_file(name: "iost", mode: 0400, parent: dump_dentry,
3837 data: &hisi_hba->debugfs_iost[dump_index],
3838 fops: &debugfs_iost_v3_hw_fops);
3839
3840 debugfs_create_file(name: "iost_cache", mode: 0400, parent: dump_dentry,
3841 data: &hisi_hba->debugfs_iost_cache[dump_index],
3842 fops: &debugfs_iost_cache_v3_hw_fops);
3843
3844 debugfs_create_file(name: "itct", mode: 0400, parent: dump_dentry,
3845 data: &hisi_hba->debugfs_itct[dump_index],
3846 fops: &debugfs_itct_v3_hw_fops);
3847
3848 debugfs_create_file(name: "itct_cache", mode: 0400, parent: dump_dentry,
3849 data: &hisi_hba->debugfs_itct_cache[dump_index],
3850 fops: &debugfs_itct_cache_v3_hw_fops);
3851
3852 debugfs_create_file(name: "axi", mode: 0400, parent: dump_dentry,
3853 data: &hisi_hba->debugfs_regs[dump_index][DEBUGFS_AXI],
3854 fops: &debugfs_axi_v3_hw_fops);
3855
3856 debugfs_create_file(name: "ras", mode: 0400, parent: dump_dentry,
3857 data: &hisi_hba->debugfs_regs[dump_index][DEBUGFS_RAS],
3858 fops: &debugfs_ras_v3_hw_fops);
3859}
3860
3861static ssize_t debugfs_trigger_dump_v3_hw_write(struct file *file,
3862 const char __user *user_buf,
3863 size_t count, loff_t *ppos)
3864{
3865 struct hisi_hba *hisi_hba = file->f_inode->i_private;
3866 char buf[8];
3867
3868 if (count > 8)
3869 return -EFAULT;
3870
3871 if (copy_from_user(to: buf, from: user_buf, n: count))
3872 return -EFAULT;
3873
3874 if (buf[0] != '1')
3875 return -EFAULT;
3876
3877 down(sem: &hisi_hba->sem);
3878 if (debugfs_snapshot_regs_v3_hw(hisi_hba)) {
3879 up(sem: &hisi_hba->sem);
3880 return -EFAULT;
3881 }
3882 up(sem: &hisi_hba->sem);
3883
3884 return count;
3885}
3886
3887static const struct file_operations debugfs_trigger_dump_v3_hw_fops = {
3888 .write = &debugfs_trigger_dump_v3_hw_write,
3889 .owner = THIS_MODULE,
3890};
3891
3892enum {
3893 HISI_SAS_BIST_LOOPBACK_MODE_DIGITAL = 0,
3894 HISI_SAS_BIST_LOOPBACK_MODE_SERDES,
3895 HISI_SAS_BIST_LOOPBACK_MODE_REMOTE,
3896};
3897
3898static const struct {
3899 int value;
3900 char *name;
3901} debugfs_loop_linkrate_v3_hw[] = {
3902 { SAS_LINK_RATE_1_5_GBPS, "1.5 Gbit" },
3903 { SAS_LINK_RATE_3_0_GBPS, "3.0 Gbit" },
3904 { SAS_LINK_RATE_6_0_GBPS, "6.0 Gbit" },
3905 { SAS_LINK_RATE_12_0_GBPS, "12.0 Gbit" },
3906};
3907
3908static int debugfs_bist_linkrate_v3_hw_show(struct seq_file *s, void *p)
3909{
3910 struct hisi_hba *hisi_hba = s->private;
3911 int i;
3912
3913 for (i = 0; i < ARRAY_SIZE(debugfs_loop_linkrate_v3_hw); i++) {
3914 int match = (hisi_hba->debugfs_bist_linkrate ==
3915 debugfs_loop_linkrate_v3_hw[i].value);
3916
3917 seq_printf(m: s, fmt: "%s%s%s ", match ? "[" : "",
3918 debugfs_loop_linkrate_v3_hw[i].name,
3919 match ? "]" : "");
3920 }
3921 seq_puts(m: s, s: "\n");
3922
3923 return 0;
3924}
3925
3926static ssize_t debugfs_bist_linkrate_v3_hw_write(struct file *filp,
3927 const char __user *buf,
3928 size_t count, loff_t *ppos)
3929{
3930 struct seq_file *m = filp->private_data;
3931 struct hisi_hba *hisi_hba = m->private;
3932 char kbuf[16] = {}, *pkbuf;
3933 bool found = false;
3934 int i;
3935
3936 if (hisi_hba->debugfs_bist_enable)
3937 return -EPERM;
3938
3939 if (count >= sizeof(kbuf))
3940 return -EOVERFLOW;
3941
3942 if (copy_from_user(to: kbuf, from: buf, n: count))
3943 return -EINVAL;
3944
3945 pkbuf = strstrip(str: kbuf);
3946
3947 for (i = 0; i < ARRAY_SIZE(debugfs_loop_linkrate_v3_hw); i++) {
3948 if (!strncmp(debugfs_loop_linkrate_v3_hw[i].name,
3949 pkbuf, 16)) {
3950 hisi_hba->debugfs_bist_linkrate =
3951 debugfs_loop_linkrate_v3_hw[i].value;
3952 found = true;
3953 break;
3954 }
3955 }
3956
3957 if (!found)
3958 return -EINVAL;
3959
3960 return count;
3961}
3962DEFINE_SHOW_STORE_ATTRIBUTE(debugfs_bist_linkrate_v3_hw);
3963
3964static const struct {
3965 int value;
3966 char *name;
3967} debugfs_loop_code_mode_v3_hw[] = {
3968 { HISI_SAS_BIST_CODE_MODE_PRBS7, "PRBS7" },
3969 { HISI_SAS_BIST_CODE_MODE_PRBS23, "PRBS23" },
3970 { HISI_SAS_BIST_CODE_MODE_PRBS31, "PRBS31" },
3971 { HISI_SAS_BIST_CODE_MODE_JTPAT, "JTPAT" },
3972 { HISI_SAS_BIST_CODE_MODE_CJTPAT, "CJTPAT" },
3973 { HISI_SAS_BIST_CODE_MODE_SCRAMBED_0, "SCRAMBED_0" },
3974 { HISI_SAS_BIST_CODE_MODE_TRAIN, "TRAIN" },
3975 { HISI_SAS_BIST_CODE_MODE_TRAIN_DONE, "TRAIN_DONE" },
3976 { HISI_SAS_BIST_CODE_MODE_HFTP, "HFTP" },
3977 { HISI_SAS_BIST_CODE_MODE_MFTP, "MFTP" },
3978 { HISI_SAS_BIST_CODE_MODE_LFTP, "LFTP" },
3979 { HISI_SAS_BIST_CODE_MODE_FIXED_DATA, "FIXED_DATA" },
3980};
3981
3982static int debugfs_bist_code_mode_v3_hw_show(struct seq_file *s, void *p)
3983{
3984 struct hisi_hba *hisi_hba = s->private;
3985 int i;
3986
3987 for (i = 0; i < ARRAY_SIZE(debugfs_loop_code_mode_v3_hw); i++) {
3988 int match = (hisi_hba->debugfs_bist_code_mode ==
3989 debugfs_loop_code_mode_v3_hw[i].value);
3990
3991 seq_printf(m: s, fmt: "%s%s%s ", match ? "[" : "",
3992 debugfs_loop_code_mode_v3_hw[i].name,
3993 match ? "]" : "");
3994 }
3995 seq_puts(m: s, s: "\n");
3996
3997 return 0;
3998}
3999
4000static ssize_t debugfs_bist_code_mode_v3_hw_write(struct file *filp,
4001 const char __user *buf,
4002 size_t count,
4003 loff_t *ppos)
4004{
4005 struct seq_file *m = filp->private_data;
4006 struct hisi_hba *hisi_hba = m->private;
4007 char kbuf[16] = {}, *pkbuf;
4008 bool found = false;
4009 int i;
4010
4011 if (hisi_hba->debugfs_bist_enable)
4012 return -EPERM;
4013
4014 if (count >= sizeof(kbuf))
4015 return -EINVAL;
4016
4017 if (copy_from_user(to: kbuf, from: buf, n: count))
4018 return -EOVERFLOW;
4019
4020 pkbuf = strstrip(str: kbuf);
4021
4022 for (i = 0; i < ARRAY_SIZE(debugfs_loop_code_mode_v3_hw); i++) {
4023 if (!strncmp(debugfs_loop_code_mode_v3_hw[i].name,
4024 pkbuf, 16)) {
4025 hisi_hba->debugfs_bist_code_mode =
4026 debugfs_loop_code_mode_v3_hw[i].value;
4027 found = true;
4028 break;
4029 }
4030 }
4031
4032 if (!found)
4033 return -EINVAL;
4034
4035 return count;
4036}
4037DEFINE_SHOW_STORE_ATTRIBUTE(debugfs_bist_code_mode_v3_hw);
4038
4039static ssize_t debugfs_bist_phy_v3_hw_write(struct file *filp,
4040 const char __user *buf,
4041 size_t count, loff_t *ppos)
4042{
4043 struct seq_file *m = filp->private_data;
4044 struct hisi_hba *hisi_hba = m->private;
4045 unsigned int phy_no;
4046 int val;
4047
4048 if (hisi_hba->debugfs_bist_enable)
4049 return -EPERM;
4050
4051 val = kstrtouint_from_user(s: buf, count, base: 0, res: &phy_no);
4052 if (val)
4053 return val;
4054
4055 if (phy_no >= hisi_hba->n_phy)
4056 return -EINVAL;
4057
4058 hisi_hba->debugfs_bist_phy_no = phy_no;
4059
4060 return count;
4061}
4062
4063static int debugfs_bist_phy_v3_hw_show(struct seq_file *s, void *p)
4064{
4065 struct hisi_hba *hisi_hba = s->private;
4066
4067 seq_printf(m: s, fmt: "%d\n", hisi_hba->debugfs_bist_phy_no);
4068
4069 return 0;
4070}
4071DEFINE_SHOW_STORE_ATTRIBUTE(debugfs_bist_phy_v3_hw);
4072
4073static ssize_t debugfs_bist_cnt_v3_hw_write(struct file *filp,
4074 const char __user *buf,
4075 size_t count, loff_t *ppos)
4076{
4077 struct seq_file *m = filp->private_data;
4078 struct hisi_hba *hisi_hba = m->private;
4079 unsigned int cnt;
4080 int val;
4081
4082 if (hisi_hba->debugfs_bist_enable)
4083 return -EPERM;
4084
4085 val = kstrtouint_from_user(s: buf, count, base: 0, res: &cnt);
4086 if (val)
4087 return val;
4088
4089 if (cnt)
4090 return -EINVAL;
4091
4092 hisi_hba->debugfs_bist_cnt = 0;
4093 return count;
4094}
4095
4096static int debugfs_bist_cnt_v3_hw_show(struct seq_file *s, void *p)
4097{
4098 struct hisi_hba *hisi_hba = s->private;
4099
4100 seq_printf(m: s, fmt: "%u\n", hisi_hba->debugfs_bist_cnt);
4101
4102 return 0;
4103}
4104DEFINE_SHOW_STORE_ATTRIBUTE(debugfs_bist_cnt_v3_hw);
4105
4106static const struct {
4107 int value;
4108 char *name;
4109} debugfs_loop_modes_v3_hw[] = {
4110 { HISI_SAS_BIST_LOOPBACK_MODE_DIGITAL, "digital" },
4111 { HISI_SAS_BIST_LOOPBACK_MODE_SERDES, "serdes" },
4112 { HISI_SAS_BIST_LOOPBACK_MODE_REMOTE, "remote" },
4113};
4114
4115static int debugfs_bist_mode_v3_hw_show(struct seq_file *s, void *p)
4116{
4117 struct hisi_hba *hisi_hba = s->private;
4118 int i;
4119
4120 for (i = 0; i < ARRAY_SIZE(debugfs_loop_modes_v3_hw); i++) {
4121 int match = (hisi_hba->debugfs_bist_mode ==
4122 debugfs_loop_modes_v3_hw[i].value);
4123
4124 seq_printf(m: s, fmt: "%s%s%s ", match ? "[" : "",
4125 debugfs_loop_modes_v3_hw[i].name,
4126 match ? "]" : "");
4127 }
4128 seq_puts(m: s, s: "\n");
4129
4130 return 0;
4131}
4132
4133static ssize_t debugfs_bist_mode_v3_hw_write(struct file *filp,
4134 const char __user *buf,
4135 size_t count, loff_t *ppos)
4136{
4137 struct seq_file *m = filp->private_data;
4138 struct hisi_hba *hisi_hba = m->private;
4139 char kbuf[16] = {}, *pkbuf;
4140 bool found = false;
4141 int i;
4142
4143 if (hisi_hba->debugfs_bist_enable)
4144 return -EPERM;
4145
4146 if (count >= sizeof(kbuf))
4147 return -EINVAL;
4148
4149 if (copy_from_user(to: kbuf, from: buf, n: count))
4150 return -EOVERFLOW;
4151
4152 pkbuf = strstrip(str: kbuf);
4153
4154 for (i = 0; i < ARRAY_SIZE(debugfs_loop_modes_v3_hw); i++) {
4155 if (!strncmp(debugfs_loop_modes_v3_hw[i].name, pkbuf, 16)) {
4156 hisi_hba->debugfs_bist_mode =
4157 debugfs_loop_modes_v3_hw[i].value;
4158 found = true;
4159 break;
4160 }
4161 }
4162
4163 if (!found)
4164 return -EINVAL;
4165
4166 return count;
4167}
4168DEFINE_SHOW_STORE_ATTRIBUTE(debugfs_bist_mode_v3_hw);
4169
4170static ssize_t debugfs_bist_enable_v3_hw_write(struct file *filp,
4171 const char __user *buf,
4172 size_t count, loff_t *ppos)
4173{
4174 struct seq_file *m = filp->private_data;
4175 struct hisi_hba *hisi_hba = m->private;
4176 unsigned int enable;
4177 int val;
4178
4179 val = kstrtouint_from_user(s: buf, count, base: 0, res: &enable);
4180 if (val)
4181 return val;
4182
4183 if (enable > 1)
4184 return -EINVAL;
4185
4186 if (enable == hisi_hba->debugfs_bist_enable)
4187 return count;
4188
4189 val = debugfs_set_bist_v3_hw(hisi_hba, enable);
4190 if (val < 0)
4191 return val;
4192
4193 hisi_hba->debugfs_bist_enable = enable;
4194
4195 return count;
4196}
4197
4198static int debugfs_bist_enable_v3_hw_show(struct seq_file *s, void *p)
4199{
4200 struct hisi_hba *hisi_hba = s->private;
4201
4202 seq_printf(m: s, fmt: "%d\n", hisi_hba->debugfs_bist_enable);
4203
4204 return 0;
4205}
4206DEFINE_SHOW_STORE_ATTRIBUTE(debugfs_bist_enable_v3_hw);
4207
4208static const struct {
4209 char *name;
4210} debugfs_ffe_name_v3_hw[FFE_CFG_MAX] = {
4211 { "SAS_1_5_GBPS" },
4212 { "SAS_3_0_GBPS" },
4213 { "SAS_6_0_GBPS" },
4214 { "SAS_12_0_GBPS" },
4215 { "FFE_RESV" },
4216 { "SATA_1_5_GBPS" },
4217 { "SATA_3_0_GBPS" },
4218 { "SATA_6_0_GBPS" },
4219};
4220
4221static ssize_t debugfs_v3_hw_write(struct file *filp,
4222 const char __user *buf,
4223 size_t count, loff_t *ppos)
4224{
4225 struct seq_file *m = filp->private_data;
4226 u32 *val = m->private;
4227 int res;
4228
4229 res = kstrtouint_from_user(s: buf, count, base: 0, res: val);
4230 if (res)
4231 return res;
4232
4233 return count;
4234}
4235
4236static int debugfs_v3_hw_show(struct seq_file *s, void *p)
4237{
4238 u32 *val = s->private;
4239
4240 seq_printf(m: s, fmt: "0x%x\n", *val);
4241
4242 return 0;
4243}
4244DEFINE_SHOW_STORE_ATTRIBUTE(debugfs_v3_hw);
4245
4246static ssize_t debugfs_phy_down_cnt_v3_hw_write(struct file *filp,
4247 const char __user *buf,
4248 size_t count, loff_t *ppos)
4249{
4250 struct seq_file *s = filp->private_data;
4251 struct hisi_sas_phy *phy = s->private;
4252 unsigned int set_val;
4253 int res;
4254
4255 res = kstrtouint_from_user(s: buf, count, base: 0, res: &set_val);
4256 if (res)
4257 return res;
4258
4259 if (set_val > 0)
4260 return -EINVAL;
4261
4262 atomic_set(v: &phy->down_cnt, i: 0);
4263
4264 return count;
4265}
4266
4267static int debugfs_phy_down_cnt_v3_hw_show(struct seq_file *s, void *p)
4268{
4269 struct hisi_sas_phy *phy = s->private;
4270
4271 seq_printf(m: s, fmt: "%d\n", atomic_read(v: &phy->down_cnt));
4272
4273 return 0;
4274}
4275DEFINE_SHOW_STORE_ATTRIBUTE(debugfs_phy_down_cnt_v3_hw);
4276
4277enum fifo_dump_mode_v3_hw {
4278 FIFO_DUMP_FORVER = (1U << 0),
4279 FIFO_DUMP_AFTER_TRIGGER = (1U << 1),
4280 FIFO_DUMP_UNTILL_TRIGGER = (1U << 2),
4281};
4282
4283enum fifo_trigger_mode_v3_hw {
4284 FIFO_TRIGGER_EDGE = (1U << 0),
4285 FIFO_TRIGGER_SAME_LEVEL = (1U << 1),
4286 FIFO_TRIGGER_DIFF_LEVEL = (1U << 2),
4287};
4288
4289static int debugfs_is_fifo_config_valid_v3_hw(struct hisi_sas_phy *phy)
4290{
4291 struct hisi_hba *hisi_hba = phy->hisi_hba;
4292
4293 if (phy->fifo.signal_sel > 0xf) {
4294 dev_info(hisi_hba->dev, "Invalid signal select: %u\n",
4295 phy->fifo.signal_sel);
4296 return -EINVAL;
4297 }
4298
4299 switch (phy->fifo.dump_mode) {
4300 case FIFO_DUMP_FORVER:
4301 case FIFO_DUMP_AFTER_TRIGGER:
4302 case FIFO_DUMP_UNTILL_TRIGGER:
4303 break;
4304 default:
4305 dev_info(hisi_hba->dev, "Invalid dump mode: %u\n",
4306 phy->fifo.dump_mode);
4307 return -EINVAL;
4308 }
4309
4310 /* when FIFO_DUMP_FORVER, no need to check trigger_mode */
4311 if (phy->fifo.dump_mode == FIFO_DUMP_FORVER)
4312 return 0;
4313
4314 switch (phy->fifo.trigger_mode) {
4315 case FIFO_TRIGGER_EDGE:
4316 case FIFO_TRIGGER_SAME_LEVEL:
4317 case FIFO_TRIGGER_DIFF_LEVEL:
4318 break;
4319 default:
4320 dev_info(hisi_hba->dev, "Invalid trigger mode: %u\n",
4321 phy->fifo.trigger_mode);
4322 return -EINVAL;
4323 }
4324 return 0;
4325}
4326
4327static int debugfs_update_fifo_config_v3_hw(struct hisi_sas_phy *phy)
4328{
4329 u32 trigger_mode = phy->fifo.trigger_mode;
4330 u32 signal_sel = phy->fifo.signal_sel;
4331 u32 dump_mode = phy->fifo.dump_mode;
4332 struct hisi_hba *hisi_hba = phy->hisi_hba;
4333 int phy_no = phy->sas_phy.id;
4334 u32 reg_val;
4335 int res;
4336
4337 /* Check the validity of trace FIFO configuration */
4338 res = debugfs_is_fifo_config_valid_v3_hw(phy);
4339 if (res)
4340 return res;
4341
4342 reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4343 /* Disable trace FIFO before update configuration */
4344 reg_val |= DFX_FIFO_CTRL_DUMP_DISABLE_MSK;
4345
4346 /* Update trace FIFO configuration */
4347 reg_val &= ~(DFX_FIFO_CTRL_DUMP_MODE_MSK |
4348 DFX_FIFO_CTRL_SIGNAL_SEL_MSK |
4349 DFX_FIFO_CTRL_TRIGGER_MODE_MSK);
4350
4351 reg_val |= ((trigger_mode << DFX_FIFO_CTRL_TRIGGER_MODE_OFF) |
4352 (dump_mode << DFX_FIFO_CTRL_DUMP_MODE_OFF) |
4353 (signal_sel << DFX_FIFO_CTRL_SIGNAL_SEL_OFF));
4354 hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_CTRL, val: reg_val);
4355
4356 hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_DUMP_MSK,
4357 val: phy->fifo.dump_msk);
4358
4359 hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_TRIGGER,
4360 val: phy->fifo.trigger);
4361
4362 hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_TRIGGER_MSK,
4363 val: phy->fifo.trigger_msk);
4364
4365 /* Enable trace FIFO after updated configuration */
4366 reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4367 reg_val &= ~DFX_FIFO_CTRL_DUMP_DISABLE_MSK;
4368 hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_CTRL, val: reg_val);
4369
4370 return 0;
4371}
4372
4373static ssize_t debugfs_fifo_update_cfg_v3_hw_write(struct file *filp,
4374 const char __user *buf,
4375 size_t count, loff_t *ppos)
4376{
4377 struct hisi_sas_phy *phy = filp->private_data;
4378 bool update;
4379 int val;
4380
4381 val = kstrtobool_from_user(s: buf, count, res: &update);
4382 if (val)
4383 return val;
4384
4385 if (update != 1)
4386 return -EINVAL;
4387
4388 val = debugfs_update_fifo_config_v3_hw(phy);
4389 if (val)
4390 return val;
4391
4392 return count;
4393}
4394
4395static const struct file_operations debugfs_fifo_update_cfg_v3_hw_fops = {
4396 .open = simple_open,
4397 .write = debugfs_fifo_update_cfg_v3_hw_write,
4398 .owner = THIS_MODULE,
4399};
4400
4401static void debugfs_read_fifo_data_v3_hw(struct hisi_sas_phy *phy)
4402{
4403 struct hisi_hba *hisi_hba = phy->hisi_hba;
4404 u32 *buf = phy->fifo.rd_data;
4405 int phy_no = phy->sas_phy.id;
4406 u32 val;
4407 int i;
4408
4409 memset(buf, 0, sizeof(phy->fifo.rd_data));
4410
4411 /* Disable trace FIFO before read data */
4412 val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4413 val |= DFX_FIFO_CTRL_DUMP_DISABLE_MSK;
4414 hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_CTRL, val);
4415
4416 for (i = 0; i < HISI_SAS_FIFO_DATA_DW_SIZE; i++) {
4417 val = hisi_sas_phy_read32(hisi_hba, phy_no,
4418 DFX_FIFO_RD_DATA);
4419 buf[i] = val;
4420 }
4421
4422 /* Enable trace FIFO after read data */
4423 val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4424 val &= ~DFX_FIFO_CTRL_DUMP_DISABLE_MSK;
4425 hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_CTRL, val);
4426}
4427
4428static int debugfs_fifo_data_v3_hw_show(struct seq_file *s, void *p)
4429{
4430 struct hisi_sas_phy *phy = s->private;
4431
4432 debugfs_read_fifo_data_v3_hw(phy);
4433
4434 debugfs_show_row_32_v3_hw(s, index: 0, HISI_SAS_FIFO_DATA_DW_SIZE * 4,
4435 ptr: (__le32 *)phy->fifo.rd_data);
4436
4437 return 0;
4438}
4439DEFINE_SHOW_ATTRIBUTE(debugfs_fifo_data_v3_hw);
4440
4441static void debugfs_fifo_init_v3_hw(struct hisi_hba *hisi_hba)
4442{
4443 int phy_no;
4444
4445 hisi_hba->debugfs_fifo_dentry =
4446 debugfs_create_dir(name: "fifo", parent: hisi_hba->debugfs_dir);
4447
4448 for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) {
4449 struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
4450 struct dentry *port_dentry;
4451 char name[256];
4452 u32 val;
4453
4454 /* get default configuration for trace FIFO */
4455 val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4456 val &= DFX_FIFO_CTRL_DUMP_MODE_MSK;
4457 val >>= DFX_FIFO_CTRL_DUMP_MODE_OFF;
4458 phy->fifo.dump_mode = val;
4459
4460 val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4461 val &= DFX_FIFO_CTRL_TRIGGER_MODE_MSK;
4462 val >>= DFX_FIFO_CTRL_TRIGGER_MODE_OFF;
4463 phy->fifo.trigger_mode = val;
4464
4465 val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4466 val &= DFX_FIFO_CTRL_SIGNAL_SEL_MSK;
4467 val >>= DFX_FIFO_CTRL_SIGNAL_SEL_OFF;
4468 phy->fifo.signal_sel = val;
4469
4470 val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_DUMP_MSK);
4471 phy->fifo.dump_msk = val;
4472
4473 val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_TRIGGER);
4474 phy->fifo.trigger = val;
4475 val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_TRIGGER_MSK);
4476 phy->fifo.trigger_msk = val;
4477
4478 snprintf(buf: name, size: 256, fmt: "%d", phy_no);
4479 port_dentry = debugfs_create_dir(name,
4480 parent: hisi_hba->debugfs_fifo_dentry);
4481
4482 debugfs_create_file(name: "update_config", mode: 0200, parent: port_dentry, data: phy,
4483 fops: &debugfs_fifo_update_cfg_v3_hw_fops);
4484
4485 debugfs_create_file(name: "signal_sel", mode: 0600, parent: port_dentry,
4486 data: &phy->fifo.signal_sel,
4487 fops: &debugfs_v3_hw_fops);
4488
4489 debugfs_create_file(name: "dump_msk", mode: 0600, parent: port_dentry,
4490 data: &phy->fifo.dump_msk,
4491 fops: &debugfs_v3_hw_fops);
4492
4493 debugfs_create_file(name: "dump_mode", mode: 0600, parent: port_dentry,
4494 data: &phy->fifo.dump_mode,
4495 fops: &debugfs_v3_hw_fops);
4496
4497 debugfs_create_file(name: "trigger_mode", mode: 0600, parent: port_dentry,
4498 data: &phy->fifo.trigger_mode,
4499 fops: &debugfs_v3_hw_fops);
4500
4501 debugfs_create_file(name: "trigger", mode: 0600, parent: port_dentry,
4502 data: &phy->fifo.trigger,
4503 fops: &debugfs_v3_hw_fops);
4504
4505 debugfs_create_file(name: "trigger_msk", mode: 0600, parent: port_dentry,
4506 data: &phy->fifo.trigger_msk,
4507 fops: &debugfs_v3_hw_fops);
4508
4509 debugfs_create_file(name: "fifo_data", mode: 0400, parent: port_dentry, data: phy,
4510 fops: &debugfs_fifo_data_v3_hw_fops);
4511 }
4512}
4513
4514static void debugfs_release_v3_hw(struct hisi_hba *hisi_hba, int dump_index)
4515{
4516 struct device *dev = hisi_hba->dev;
4517 int i;
4518
4519 devm_kfree(dev, p: hisi_hba->debugfs_iost_cache[dump_index].cache);
4520 devm_kfree(dev, p: hisi_hba->debugfs_itct_cache[dump_index].cache);
4521 devm_kfree(dev, p: hisi_hba->debugfs_iost[dump_index].iost);
4522 devm_kfree(dev, p: hisi_hba->debugfs_itct[dump_index].itct);
4523
4524 for (i = 0; i < hisi_hba->queue_count; i++)
4525 devm_kfree(dev, p: hisi_hba->debugfs_dq[dump_index][i].hdr);
4526
4527 for (i = 0; i < hisi_hba->queue_count; i++)
4528 devm_kfree(dev,
4529 p: hisi_hba->debugfs_cq[dump_index][i].complete_hdr);
4530
4531 for (i = 0; i < DEBUGFS_REGS_NUM; i++)
4532 devm_kfree(dev, p: hisi_hba->debugfs_regs[dump_index][i].data);
4533
4534 for (i = 0; i < hisi_hba->n_phy; i++)
4535 devm_kfree(dev, p: hisi_hba->debugfs_port_reg[dump_index][i].data);
4536}
4537
4538static const struct hisi_sas_debugfs_reg *debugfs_reg_array_v3_hw[DEBUGFS_REGS_NUM] = {
4539 [DEBUGFS_GLOBAL] = &debugfs_global_reg,
4540 [DEBUGFS_AXI] = &debugfs_axi_reg,
4541 [DEBUGFS_RAS] = &debugfs_ras_reg,
4542};
4543
4544static int debugfs_alloc_v3_hw(struct hisi_hba *hisi_hba, int dump_index)
4545{
4546 const struct hisi_sas_hw *hw = hisi_hba->hw;
4547 struct device *dev = hisi_hba->dev;
4548 int p, c, d, r;
4549 size_t sz;
4550
4551 for (r = 0; r < DEBUGFS_REGS_NUM; r++) {
4552 struct hisi_sas_debugfs_regs *regs =
4553 &hisi_hba->debugfs_regs[dump_index][r];
4554
4555 sz = debugfs_reg_array_v3_hw[r]->count * 4;
4556 regs->data = devm_kmalloc(dev, size: sz, GFP_KERNEL);
4557 if (!regs->data)
4558 goto fail;
4559 regs->hisi_hba = hisi_hba;
4560 }
4561
4562 sz = debugfs_port_reg.count * 4;
4563 for (p = 0; p < hisi_hba->n_phy; p++) {
4564 struct hisi_sas_debugfs_port *port =
4565 &hisi_hba->debugfs_port_reg[dump_index][p];
4566
4567 port->data = devm_kmalloc(dev, size: sz, GFP_KERNEL);
4568 if (!port->data)
4569 goto fail;
4570 port->phy = &hisi_hba->phy[p];
4571 }
4572
4573 sz = hw->complete_hdr_size * HISI_SAS_QUEUE_SLOTS;
4574 for (c = 0; c < hisi_hba->queue_count; c++) {
4575 struct hisi_sas_debugfs_cq *cq =
4576 &hisi_hba->debugfs_cq[dump_index][c];
4577
4578 cq->complete_hdr = devm_kmalloc(dev, size: sz, GFP_KERNEL);
4579 if (!cq->complete_hdr)
4580 goto fail;
4581 cq->cq = &hisi_hba->cq[c];
4582 }
4583
4584 sz = sizeof(struct hisi_sas_cmd_hdr) * HISI_SAS_QUEUE_SLOTS;
4585 for (d = 0; d < hisi_hba->queue_count; d++) {
4586 struct hisi_sas_debugfs_dq *dq =
4587 &hisi_hba->debugfs_dq[dump_index][d];
4588
4589 dq->hdr = devm_kmalloc(dev, size: sz, GFP_KERNEL);
4590 if (!dq->hdr)
4591 goto fail;
4592 dq->dq = &hisi_hba->dq[d];
4593 }
4594
4595 sz = HISI_SAS_MAX_COMMANDS * sizeof(struct hisi_sas_iost);
4596
4597 hisi_hba->debugfs_iost[dump_index].iost =
4598 devm_kmalloc(dev, size: sz, GFP_KERNEL);
4599 if (!hisi_hba->debugfs_iost[dump_index].iost)
4600 goto fail;
4601
4602 sz = HISI_SAS_IOST_ITCT_CACHE_NUM *
4603 sizeof(struct hisi_sas_iost_itct_cache);
4604
4605 hisi_hba->debugfs_iost_cache[dump_index].cache =
4606 devm_kmalloc(dev, size: sz, GFP_KERNEL);
4607 if (!hisi_hba->debugfs_iost_cache[dump_index].cache)
4608 goto fail;
4609
4610 sz = HISI_SAS_IOST_ITCT_CACHE_NUM *
4611 sizeof(struct hisi_sas_iost_itct_cache);
4612
4613 hisi_hba->debugfs_itct_cache[dump_index].cache =
4614 devm_kmalloc(dev, size: sz, GFP_KERNEL);
4615 if (!hisi_hba->debugfs_itct_cache[dump_index].cache)
4616 goto fail;
4617
4618 /* New memory allocation must be locate before itct */
4619 sz = HISI_SAS_MAX_ITCT_ENTRIES * sizeof(struct hisi_sas_itct);
4620
4621 hisi_hba->debugfs_itct[dump_index].itct =
4622 devm_kmalloc(dev, size: sz, GFP_KERNEL);
4623 if (!hisi_hba->debugfs_itct[dump_index].itct)
4624 goto fail;
4625
4626 return 0;
4627fail:
4628 debugfs_release_v3_hw(hisi_hba, dump_index);
4629 return -ENOMEM;
4630}
4631
4632static int debugfs_snapshot_regs_v3_hw(struct hisi_hba *hisi_hba)
4633{
4634 int debugfs_dump_index = hisi_hba->debugfs_dump_index;
4635 struct device *dev = hisi_hba->dev;
4636 u64 timestamp = local_clock();
4637
4638 if (debugfs_dump_index >= hisi_sas_debugfs_dump_count) {
4639 dev_warn(dev, "dump count exceeded!\n");
4640 return -EINVAL;
4641 }
4642
4643 if (debugfs_alloc_v3_hw(hisi_hba, dump_index: debugfs_dump_index)) {
4644 dev_warn(dev, "failed to alloc memory\n");
4645 return -ENOMEM;
4646 }
4647
4648 do_div(timestamp, NSEC_PER_MSEC);
4649 hisi_hba->debugfs_timestamp[debugfs_dump_index] = timestamp;
4650
4651 debugfs_snapshot_prepare_v3_hw(hisi_hba);
4652
4653 debugfs_snapshot_global_reg_v3_hw(hisi_hba);
4654 debugfs_snapshot_port_reg_v3_hw(hisi_hba);
4655 debugfs_snapshot_axi_reg_v3_hw(hisi_hba);
4656 debugfs_snapshot_ras_reg_v3_hw(hisi_hba);
4657 debugfs_snapshot_cq_reg_v3_hw(hisi_hba);
4658 debugfs_snapshot_dq_reg_v3_hw(hisi_hba);
4659 debugfs_snapshot_itct_reg_v3_hw(hisi_hba);
4660 debugfs_snapshot_iost_reg_v3_hw(hisi_hba);
4661
4662 debugfs_create_files_v3_hw(hisi_hba);
4663
4664 debugfs_snapshot_restore_v3_hw(hisi_hba);
4665 hisi_hba->debugfs_dump_index++;
4666
4667 return 0;
4668}
4669
4670static void debugfs_phy_down_cnt_init_v3_hw(struct hisi_hba *hisi_hba)
4671{
4672 struct dentry *dir = debugfs_create_dir(name: "phy_down_cnt",
4673 parent: hisi_hba->debugfs_dir);
4674 char name[16];
4675 int phy_no;
4676
4677 for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) {
4678 snprintf(buf: name, size: 16, fmt: "%d", phy_no);
4679 debugfs_create_file(name, mode: 0600, parent: dir,
4680 data: &hisi_hba->phy[phy_no],
4681 fops: &debugfs_phy_down_cnt_v3_hw_fops);
4682 }
4683}
4684
4685static void debugfs_bist_init_v3_hw(struct hisi_hba *hisi_hba)
4686{
4687 struct dentry *ports_dentry;
4688 int phy_no;
4689
4690 hisi_hba->debugfs_bist_dentry =
4691 debugfs_create_dir(name: "bist", parent: hisi_hba->debugfs_dir);
4692 debugfs_create_file(name: "link_rate", mode: 0600,
4693 parent: hisi_hba->debugfs_bist_dentry, data: hisi_hba,
4694 fops: &debugfs_bist_linkrate_v3_hw_fops);
4695
4696 debugfs_create_file(name: "code_mode", mode: 0600,
4697 parent: hisi_hba->debugfs_bist_dentry, data: hisi_hba,
4698 fops: &debugfs_bist_code_mode_v3_hw_fops);
4699
4700 debugfs_create_file(name: "fixed_code", mode: 0600,
4701 parent: hisi_hba->debugfs_bist_dentry,
4702 data: &hisi_hba->debugfs_bist_fixed_code[0],
4703 fops: &debugfs_v3_hw_fops);
4704
4705 debugfs_create_file(name: "fixed_code_1", mode: 0600,
4706 parent: hisi_hba->debugfs_bist_dentry,
4707 data: &hisi_hba->debugfs_bist_fixed_code[1],
4708 fops: &debugfs_v3_hw_fops);
4709
4710 debugfs_create_file(name: "phy_id", mode: 0600, parent: hisi_hba->debugfs_bist_dentry,
4711 data: hisi_hba, fops: &debugfs_bist_phy_v3_hw_fops);
4712
4713 debugfs_create_file(name: "cnt", mode: 0600, parent: hisi_hba->debugfs_bist_dentry,
4714 data: hisi_hba, fops: &debugfs_bist_cnt_v3_hw_fops);
4715
4716 debugfs_create_file(name: "loopback_mode", mode: 0600,
4717 parent: hisi_hba->debugfs_bist_dentry,
4718 data: hisi_hba, fops: &debugfs_bist_mode_v3_hw_fops);
4719
4720 debugfs_create_file(name: "enable", mode: 0600, parent: hisi_hba->debugfs_bist_dentry,
4721 data: hisi_hba, fops: &debugfs_bist_enable_v3_hw_fops);
4722
4723 ports_dentry = debugfs_create_dir(name: "port", parent: hisi_hba->debugfs_bist_dentry);
4724
4725 for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) {
4726 struct dentry *port_dentry;
4727 struct dentry *ffe_dentry;
4728 char name[256];
4729 int i;
4730
4731 snprintf(buf: name, size: 256, fmt: "%d", phy_no);
4732 port_dentry = debugfs_create_dir(name, parent: ports_dentry);
4733 ffe_dentry = debugfs_create_dir(name: "ffe", parent: port_dentry);
4734 for (i = 0; i < FFE_CFG_MAX; i++) {
4735 if (i == FFE_RESV)
4736 continue;
4737 debugfs_create_file(name: debugfs_ffe_name_v3_hw[i].name,
4738 mode: 0600, parent: ffe_dentry,
4739 data: &hisi_hba->debugfs_bist_ffe[phy_no][i],
4740 fops: &debugfs_v3_hw_fops);
4741 }
4742 }
4743
4744 hisi_hba->debugfs_bist_linkrate = SAS_LINK_RATE_1_5_GBPS;
4745}
4746
4747static void debugfs_exit_v3_hw(struct hisi_hba *hisi_hba)
4748{
4749 debugfs_remove_recursive(dentry: hisi_hba->debugfs_dir);
4750 hisi_hba->debugfs_dir = NULL;
4751}
4752
4753static void debugfs_init_v3_hw(struct hisi_hba *hisi_hba)
4754{
4755 struct device *dev = hisi_hba->dev;
4756
4757 hisi_hba->debugfs_dir = debugfs_create_dir(name: dev_name(dev),
4758 parent: hisi_sas_debugfs_dir);
4759 debugfs_create_file(name: "trigger_dump", mode: 0200,
4760 parent: hisi_hba->debugfs_dir,
4761 data: hisi_hba,
4762 fops: &debugfs_trigger_dump_v3_hw_fops);
4763
4764 /* create bist structures */
4765 debugfs_bist_init_v3_hw(hisi_hba);
4766
4767 hisi_hba->debugfs_dump_dentry =
4768 debugfs_create_dir(name: "dump", parent: hisi_hba->debugfs_dir);
4769
4770 debugfs_phy_down_cnt_init_v3_hw(hisi_hba);
4771 debugfs_fifo_init_v3_hw(hisi_hba);
4772}
4773
4774static int
4775hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
4776{
4777 struct Scsi_Host *shost;
4778 struct hisi_hba *hisi_hba;
4779 struct device *dev = &pdev->dev;
4780 struct asd_sas_phy **arr_phy;
4781 struct asd_sas_port **arr_port;
4782 struct sas_ha_struct *sha;
4783 int rc, phy_nr, port_nr, i;
4784
4785 rc = pcim_enable_device(pdev);
4786 if (rc)
4787 goto err_out;
4788
4789 pci_set_master(dev: pdev);
4790
4791 rc = pcim_iomap_regions(pdev, mask: 1 << BAR_NO_V3_HW, DRV_NAME);
4792 if (rc)
4793 goto err_out;
4794
4795 rc = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
4796 if (rc) {
4797 dev_err(dev, "No usable DMA addressing method\n");
4798 rc = -ENODEV;
4799 goto err_out;
4800 }
4801
4802 shost = hisi_sas_shost_alloc_pci(pdev);
4803 if (!shost) {
4804 rc = -ENOMEM;
4805 goto err_out;
4806 }
4807
4808 sha = SHOST_TO_SAS_HA(shost);
4809 hisi_hba = shost_priv(shost);
4810 dev_set_drvdata(dev, data: sha);
4811
4812 hisi_hba->regs = pcim_iomap_table(pdev)[BAR_NO_V3_HW];
4813 if (!hisi_hba->regs) {
4814 dev_err(dev, "cannot map register\n");
4815 rc = -ENOMEM;
4816 goto err_out_free_host;
4817 }
4818
4819 phy_nr = port_nr = hisi_hba->n_phy;
4820
4821 arr_phy = devm_kcalloc(dev, n: phy_nr, size: sizeof(void *), GFP_KERNEL);
4822 arr_port = devm_kcalloc(dev, n: port_nr, size: sizeof(void *), GFP_KERNEL);
4823 if (!arr_phy || !arr_port) {
4824 rc = -ENOMEM;
4825 goto err_out_free_host;
4826 }
4827
4828 sha->sas_phy = arr_phy;
4829 sha->sas_port = arr_port;
4830 sha->shost = shost;
4831 sha->lldd_ha = hisi_hba;
4832
4833 shost->transportt = hisi_sas_stt;
4834 shost->max_id = HISI_SAS_MAX_DEVICES;
4835 shost->max_lun = ~0;
4836 shost->max_channel = 1;
4837 shost->max_cmd_len = 16;
4838 shost->can_queue = HISI_SAS_UNRESERVED_IPTT;
4839 shost->cmd_per_lun = HISI_SAS_UNRESERVED_IPTT;
4840 if (hisi_hba->iopoll_q_cnt)
4841 shost->nr_maps = 3;
4842 else
4843 shost->nr_maps = 1;
4844
4845 sha->sas_ha_name = DRV_NAME;
4846 sha->dev = dev;
4847 sha->sas_addr = &hisi_hba->sas_addr[0];
4848 sha->num_phys = hisi_hba->n_phy;
4849
4850 for (i = 0; i < hisi_hba->n_phy; i++) {
4851 sha->sas_phy[i] = &hisi_hba->phy[i].sas_phy;
4852 sha->sas_port[i] = &hisi_hba->port[i].sas_port;
4853 }
4854
4855 if (hisi_hba->prot_mask) {
4856 dev_info(dev, "Registering for DIF/DIX prot_mask=0x%x\n",
4857 prot_mask);
4858 scsi_host_set_prot(shost: hisi_hba->shost, mask: prot_mask);
4859 if (hisi_hba->prot_mask & HISI_SAS_DIX_PROT_MASK)
4860 scsi_host_set_guard(shost: hisi_hba->shost,
4861 type: SHOST_DIX_GUARD_CRC);
4862 }
4863
4864 if (hisi_sas_debugfs_enable)
4865 debugfs_init_v3_hw(hisi_hba);
4866
4867 rc = interrupt_preinit_v3_hw(hisi_hba);
4868 if (rc)
4869 goto err_out_undo_debugfs;
4870
4871 rc = scsi_add_host(host: shost, dev);
4872 if (rc)
4873 goto err_out_undo_debugfs;
4874
4875 rc = sas_register_ha(sha);
4876 if (rc)
4877 goto err_out_remove_host;
4878
4879 rc = hisi_sas_v3_init(hisi_hba);
4880 if (rc)
4881 goto err_out_unregister_ha;
4882
4883 scsi_scan_host(shost);
4884
4885 pm_runtime_set_autosuspend_delay(dev, delay: 5000);
4886 pm_runtime_use_autosuspend(dev);
4887 /*
4888 * For the situation that there are ATA disks connected with SAS
4889 * controller, it additionally creates ata_port which will affect the
4890 * child_count of hisi_hba->dev. Even if suspended all the disks,
4891 * ata_port is still and the child_count of hisi_hba->dev is not 0.
4892 * So use pm_suspend_ignore_children() to ignore the effect to
4893 * hisi_hba->dev.
4894 */
4895 pm_suspend_ignore_children(dev, enable: true);
4896 pm_runtime_put_noidle(dev: &pdev->dev);
4897
4898 return 0;
4899
4900err_out_unregister_ha:
4901 sas_unregister_ha(sha);
4902err_out_remove_host:
4903 scsi_remove_host(shost);
4904err_out_undo_debugfs:
4905 debugfs_exit_v3_hw(hisi_hba);
4906err_out_free_host:
4907 hisi_sas_free(hisi_hba);
4908 scsi_host_put(t: shost);
4909err_out:
4910 return rc;
4911}
4912
4913static void
4914hisi_sas_v3_destroy_irqs(struct pci_dev *pdev, struct hisi_hba *hisi_hba)
4915{
4916 int i;
4917
4918 devm_free_irq(dev: &pdev->dev, irq: pci_irq_vector(dev: pdev, nr: 1), dev_id: hisi_hba);
4919 devm_free_irq(dev: &pdev->dev, irq: pci_irq_vector(dev: pdev, nr: 2), dev_id: hisi_hba);
4920 devm_free_irq(dev: &pdev->dev, irq: pci_irq_vector(dev: pdev, nr: 11), dev_id: hisi_hba);
4921 for (i = 0; i < hisi_hba->cq_nvecs; i++) {
4922 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
4923 int nr = hisi_sas_intr_conv ? 16 : 16 + i;
4924
4925 devm_free_irq(dev: &pdev->dev, irq: pci_irq_vector(dev: pdev, nr), dev_id: cq);
4926 }
4927}
4928
4929static void hisi_sas_v3_remove(struct pci_dev *pdev)
4930{
4931 struct device *dev = &pdev->dev;
4932 struct sas_ha_struct *sha = dev_get_drvdata(dev);
4933 struct hisi_hba *hisi_hba = sha->lldd_ha;
4934 struct Scsi_Host *shost = sha->shost;
4935
4936 pm_runtime_get_noresume(dev);
4937 del_timer_sync(timer: &hisi_hba->timer);
4938
4939 sas_unregister_ha(sha);
4940 flush_workqueue(hisi_hba->wq);
4941 sas_remove_host(shost);
4942
4943 hisi_sas_v3_destroy_irqs(pdev, hisi_hba);
4944 hisi_sas_free(hisi_hba);
4945 debugfs_exit_v3_hw(hisi_hba);
4946 scsi_host_put(t: shost);
4947}
4948
4949static void hisi_sas_reset_prepare_v3_hw(struct pci_dev *pdev)
4950{
4951 struct sas_ha_struct *sha = pci_get_drvdata(pdev);
4952 struct hisi_hba *hisi_hba = sha->lldd_ha;
4953 struct device *dev = hisi_hba->dev;
4954 int rc;
4955
4956 dev_info(dev, "FLR prepare\n");
4957 down(sem: &hisi_hba->sem);
4958 set_bit(HISI_SAS_RESETTING_BIT, addr: &hisi_hba->flags);
4959 hisi_sas_controller_reset_prepare(hisi_hba);
4960
4961 interrupt_disable_v3_hw(hisi_hba);
4962 rc = disable_host_v3_hw(hisi_hba);
4963 if (rc)
4964 dev_err(dev, "FLR: disable host failed rc=%d\n", rc);
4965}
4966
4967static void hisi_sas_reset_done_v3_hw(struct pci_dev *pdev)
4968{
4969 struct sas_ha_struct *sha = pci_get_drvdata(pdev);
4970 struct hisi_hba *hisi_hba = sha->lldd_ha;
4971 struct device *dev = hisi_hba->dev;
4972 int rc;
4973
4974 hisi_sas_init_mem(hisi_hba);
4975
4976 rc = hw_init_v3_hw(hisi_hba);
4977 if (rc) {
4978 dev_err(dev, "FLR: hw init failed rc=%d\n", rc);
4979 return;
4980 }
4981
4982 hisi_sas_controller_reset_done(hisi_hba);
4983 dev_info(dev, "FLR done\n");
4984}
4985
4986enum {
4987 /* instances of the controller */
4988 hip08,
4989};
4990
4991static void enable_host_v3_hw(struct hisi_hba *hisi_hba)
4992{
4993 u32 reg_val;
4994
4995 hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
4996 val: (u32)((1ULL << hisi_hba->queue_count) - 1));
4997
4998 phys_init_v3_hw(hisi_hba);
4999 reg_val = hisi_sas_read32(hisi_hba, AXI_MASTER_CFG_BASE +
5000 AM_CTRL_GLOBAL);
5001 reg_val &= ~AM_CTRL_SHUTDOWN_REQ_MSK;
5002 hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
5003 AM_CTRL_GLOBAL, val: reg_val);
5004}
5005
5006static int _suspend_v3_hw(struct device *device)
5007{
5008 struct pci_dev *pdev = to_pci_dev(device);
5009 struct sas_ha_struct *sha = pci_get_drvdata(pdev);
5010 struct hisi_hba *hisi_hba = sha->lldd_ha;
5011 struct device *dev = hisi_hba->dev;
5012 struct Scsi_Host *shost = hisi_hba->shost;
5013 int rc;
5014
5015 if (!pdev->pm_cap) {
5016 dev_err(dev, "PCI PM not supported\n");
5017 return -ENODEV;
5018 }
5019
5020 if (test_and_set_bit(HISI_SAS_RESETTING_BIT, addr: &hisi_hba->flags))
5021 return -1;
5022
5023 dev_warn(dev, "entering suspend state\n");
5024
5025 scsi_block_requests(shost);
5026 set_bit(HISI_SAS_REJECT_CMD_BIT, addr: &hisi_hba->flags);
5027 flush_workqueue(hisi_hba->wq);
5028 interrupt_disable_v3_hw(hisi_hba);
5029
5030#ifdef CONFIG_PM
5031 if (atomic_read(v: &device->power.usage_count)) {
5032 dev_err(dev, "PM suspend: host status cannot be suspended\n");
5033 rc = -EBUSY;
5034 goto err_out;
5035 }
5036#endif
5037
5038 rc = disable_host_v3_hw(hisi_hba);
5039 if (rc) {
5040 dev_err(dev, "PM suspend: disable host failed rc=%d\n", rc);
5041 goto err_out_recover_host;
5042 }
5043
5044 hisi_sas_init_mem(hisi_hba);
5045
5046 hisi_sas_release_tasks(hisi_hba);
5047
5048 sas_suspend_ha(sas_ha: sha);
5049
5050 dev_warn(dev, "end of suspending controller\n");
5051 return 0;
5052
5053err_out_recover_host:
5054 enable_host_v3_hw(hisi_hba);
5055#ifdef CONFIG_PM
5056err_out:
5057#endif
5058 interrupt_enable_v3_hw(hisi_hba);
5059 clear_bit(HISI_SAS_REJECT_CMD_BIT, addr: &hisi_hba->flags);
5060 clear_bit(HISI_SAS_RESETTING_BIT, addr: &hisi_hba->flags);
5061 scsi_unblock_requests(shost);
5062 return rc;
5063}
5064
5065static int _resume_v3_hw(struct device *device)
5066{
5067 struct pci_dev *pdev = to_pci_dev(device);
5068 struct sas_ha_struct *sha = pci_get_drvdata(pdev);
5069 struct hisi_hba *hisi_hba = sha->lldd_ha;
5070 struct Scsi_Host *shost = hisi_hba->shost;
5071 struct device *dev = hisi_hba->dev;
5072 unsigned int rc;
5073 pci_power_t device_state = pdev->current_state;
5074
5075 dev_warn(dev, "resuming from operating state [D%d]\n",
5076 device_state);
5077
5078 scsi_unblock_requests(shost);
5079 clear_bit(HISI_SAS_REJECT_CMD_BIT, addr: &hisi_hba->flags);
5080
5081 sas_prep_resume_ha(sas_ha: sha);
5082 rc = hw_init_v3_hw(hisi_hba);
5083 if (rc) {
5084 scsi_remove_host(shost);
5085 return rc;
5086 }
5087 phys_init_v3_hw(hisi_hba);
5088
5089 /*
5090 * If a directly-attached disk is removed during suspend, a deadlock
5091 * may occur, as the PHYE_RESUME_TIMEOUT processing will require the
5092 * hisi_hba->device to be active, which can only happen when resume
5093 * completes. So don't wait for the HA event workqueue to drain upon
5094 * resume.
5095 */
5096 sas_resume_ha_no_sync(sas_ha: sha);
5097 clear_bit(HISI_SAS_RESETTING_BIT, addr: &hisi_hba->flags);
5098
5099 dev_warn(dev, "end of resuming controller\n");
5100
5101 return 0;
5102}
5103
5104static int __maybe_unused suspend_v3_hw(struct device *device)
5105{
5106 struct pci_dev *pdev = to_pci_dev(device);
5107 struct sas_ha_struct *sha = pci_get_drvdata(pdev);
5108 struct hisi_hba *hisi_hba = sha->lldd_ha;
5109 int rc;
5110
5111 set_bit(HISI_SAS_PM_BIT, addr: &hisi_hba->flags);
5112
5113 rc = _suspend_v3_hw(device);
5114 if (rc)
5115 clear_bit(HISI_SAS_PM_BIT, addr: &hisi_hba->flags);
5116
5117 return rc;
5118}
5119
5120static int __maybe_unused resume_v3_hw(struct device *device)
5121{
5122 struct pci_dev *pdev = to_pci_dev(device);
5123 struct sas_ha_struct *sha = pci_get_drvdata(pdev);
5124 struct hisi_hba *hisi_hba = sha->lldd_ha;
5125 int rc = _resume_v3_hw(device);
5126
5127 clear_bit(HISI_SAS_PM_BIT, addr: &hisi_hba->flags);
5128
5129 return rc;
5130}
5131
5132static const struct pci_device_id sas_v3_pci_table[] = {
5133 { PCI_VDEVICE(HUAWEI, 0xa230), hip08 },
5134 {}
5135};
5136MODULE_DEVICE_TABLE(pci, sas_v3_pci_table);
5137
5138static const struct pci_error_handlers hisi_sas_err_handler = {
5139 .reset_prepare = hisi_sas_reset_prepare_v3_hw,
5140 .reset_done = hisi_sas_reset_done_v3_hw,
5141};
5142
5143static UNIVERSAL_DEV_PM_OPS(hisi_sas_v3_pm_ops,
5144 suspend_v3_hw,
5145 resume_v3_hw,
5146 NULL);
5147
5148static struct pci_driver sas_v3_pci_driver = {
5149 .name = DRV_NAME,
5150 .id_table = sas_v3_pci_table,
5151 .probe = hisi_sas_v3_probe,
5152 .remove = hisi_sas_v3_remove,
5153 .err_handler = &hisi_sas_err_handler,
5154 .driver.pm = &hisi_sas_v3_pm_ops,
5155};
5156
5157module_pci_driver(sas_v3_pci_driver);
5158module_param_named(intr_conv, hisi_sas_intr_conv, bool, 0444);
5159
5160MODULE_LICENSE("GPL");
5161MODULE_AUTHOR("John Garry <john.garry@huawei.com>");
5162MODULE_DESCRIPTION("HISILICON SAS controller v3 hw driver based on pci device");
5163MODULE_ALIAS("pci:" DRV_NAME);
5164

source code of linux/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c