1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2021, HiSilicon Ltd. |
4 | */ |
5 | |
6 | #include <linux/device.h> |
7 | #include <linux/eventfd.h> |
8 | #include <linux/file.h> |
9 | #include <linux/hisi_acc_qm.h> |
10 | #include <linux/interrupt.h> |
11 | #include <linux/module.h> |
12 | #include <linux/pci.h> |
13 | #include <linux/vfio.h> |
14 | #include <linux/vfio_pci_core.h> |
15 | #include <linux/anon_inodes.h> |
16 | |
17 | #include "hisi_acc_vfio_pci.h" |
18 | |
19 | /* Return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */ |
20 | static int qm_wait_dev_not_ready(struct hisi_qm *qm) |
21 | { |
22 | u32 val; |
23 | |
24 | return readl_relaxed_poll_timeout(qm->io_base + QM_VF_STATE, |
25 | val, !(val & 0x1), MB_POLL_PERIOD_US, |
26 | MB_POLL_TIMEOUT_US); |
27 | } |
28 | |
29 | /* |
30 | * Each state Reg is checked 100 times, |
31 | * with a delay of 100 microseconds after each check |
32 | */ |
33 | static u32 qm_check_reg_state(struct hisi_qm *qm, u32 regs) |
34 | { |
35 | int check_times = 0; |
36 | u32 state; |
37 | |
38 | state = readl(addr: qm->io_base + regs); |
39 | while (state && check_times < ERROR_CHECK_TIMEOUT) { |
40 | udelay(CHECK_DELAY_TIME); |
41 | state = readl(addr: qm->io_base + regs); |
42 | check_times++; |
43 | } |
44 | |
45 | return state; |
46 | } |
47 | |
48 | static int qm_read_regs(struct hisi_qm *qm, u32 reg_addr, |
49 | u32 *data, u8 nums) |
50 | { |
51 | int i; |
52 | |
53 | if (nums < 1 || nums > QM_REGS_MAX_LEN) |
54 | return -EINVAL; |
55 | |
56 | for (i = 0; i < nums; i++) { |
57 | data[i] = readl(addr: qm->io_base + reg_addr); |
58 | reg_addr += QM_REG_ADDR_OFFSET; |
59 | } |
60 | |
61 | return 0; |
62 | } |
63 | |
64 | static int qm_write_regs(struct hisi_qm *qm, u32 reg, |
65 | u32 *data, u8 nums) |
66 | { |
67 | int i; |
68 | |
69 | if (nums < 1 || nums > QM_REGS_MAX_LEN) |
70 | return -EINVAL; |
71 | |
72 | for (i = 0; i < nums; i++) |
73 | writel(val: data[i], addr: qm->io_base + reg + i * QM_REG_ADDR_OFFSET); |
74 | |
75 | return 0; |
76 | } |
77 | |
78 | static int qm_get_vft(struct hisi_qm *qm, u32 *base) |
79 | { |
80 | u64 sqc_vft; |
81 | u32 qp_num; |
82 | int ret; |
83 | |
84 | ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, dma_addr: 0, queue: 0, op: 1); |
85 | if (ret) |
86 | return ret; |
87 | |
88 | sqc_vft = readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_L) | |
89 | ((u64)readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_H) << |
90 | QM_XQC_ADDR_OFFSET); |
91 | *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2); |
92 | qp_num = (QM_SQC_VFT_NUM_MASK_V2 & |
93 | (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1; |
94 | |
95 | return qp_num; |
96 | } |
97 | |
98 | static int qm_get_sqc(struct hisi_qm *qm, u64 *addr) |
99 | { |
100 | int ret; |
101 | |
102 | ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, dma_addr: 0, queue: 0, op: 1); |
103 | if (ret) |
104 | return ret; |
105 | |
106 | *addr = readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_L) | |
107 | ((u64)readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_H) << |
108 | QM_XQC_ADDR_OFFSET); |
109 | |
110 | return 0; |
111 | } |
112 | |
113 | static int qm_get_cqc(struct hisi_qm *qm, u64 *addr) |
114 | { |
115 | int ret; |
116 | |
117 | ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, dma_addr: 0, queue: 0, op: 1); |
118 | if (ret) |
119 | return ret; |
120 | |
121 | *addr = readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_L) | |
122 | ((u64)readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_H) << |
123 | QM_XQC_ADDR_OFFSET); |
124 | |
125 | return 0; |
126 | } |
127 | |
128 | static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data) |
129 | { |
130 | struct device *dev = &qm->pdev->dev; |
131 | int ret; |
132 | |
133 | ret = qm_read_regs(qm, QM_VF_AEQ_INT_MASK, data: &vf_data->aeq_int_mask, nums: 1); |
134 | if (ret) { |
135 | dev_err(dev, "failed to read QM_VF_AEQ_INT_MASK\n" ); |
136 | return ret; |
137 | } |
138 | |
139 | ret = qm_read_regs(qm, QM_VF_EQ_INT_MASK, data: &vf_data->eq_int_mask, nums: 1); |
140 | if (ret) { |
141 | dev_err(dev, "failed to read QM_VF_EQ_INT_MASK\n" ); |
142 | return ret; |
143 | } |
144 | |
145 | ret = qm_read_regs(qm, QM_IFC_INT_SOURCE_V, |
146 | data: &vf_data->ifc_int_source, nums: 1); |
147 | if (ret) { |
148 | dev_err(dev, "failed to read QM_IFC_INT_SOURCE_V\n" ); |
149 | return ret; |
150 | } |
151 | |
152 | ret = qm_read_regs(qm, QM_IFC_INT_MASK, data: &vf_data->ifc_int_mask, nums: 1); |
153 | if (ret) { |
154 | dev_err(dev, "failed to read QM_IFC_INT_MASK\n" ); |
155 | return ret; |
156 | } |
157 | |
158 | ret = qm_read_regs(qm, QM_IFC_INT_SET_V, data: &vf_data->ifc_int_set, nums: 1); |
159 | if (ret) { |
160 | dev_err(dev, "failed to read QM_IFC_INT_SET_V\n" ); |
161 | return ret; |
162 | } |
163 | |
164 | ret = qm_read_regs(qm, QM_PAGE_SIZE, data: &vf_data->page_size, nums: 1); |
165 | if (ret) { |
166 | dev_err(dev, "failed to read QM_PAGE_SIZE\n" ); |
167 | return ret; |
168 | } |
169 | |
170 | /* QM_EQC_DW has 7 regs */ |
171 | ret = qm_read_regs(qm, QM_EQC_DW0, data: vf_data->qm_eqc_dw, nums: 7); |
172 | if (ret) { |
173 | dev_err(dev, "failed to read QM_EQC_DW\n" ); |
174 | return ret; |
175 | } |
176 | |
177 | /* QM_AEQC_DW has 7 regs */ |
178 | ret = qm_read_regs(qm, QM_AEQC_DW0, data: vf_data->qm_aeqc_dw, nums: 7); |
179 | if (ret) { |
180 | dev_err(dev, "failed to read QM_AEQC_DW\n" ); |
181 | return ret; |
182 | } |
183 | |
184 | return 0; |
185 | } |
186 | |
187 | static int qm_set_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data) |
188 | { |
189 | struct device *dev = &qm->pdev->dev; |
190 | int ret; |
191 | |
192 | /* Check VF state */ |
193 | if (unlikely(hisi_qm_wait_mb_ready(qm))) { |
194 | dev_err(&qm->pdev->dev, "QM device is not ready to write\n" ); |
195 | return -EBUSY; |
196 | } |
197 | |
198 | ret = qm_write_regs(qm, QM_VF_AEQ_INT_MASK, data: &vf_data->aeq_int_mask, nums: 1); |
199 | if (ret) { |
200 | dev_err(dev, "failed to write QM_VF_AEQ_INT_MASK\n" ); |
201 | return ret; |
202 | } |
203 | |
204 | ret = qm_write_regs(qm, QM_VF_EQ_INT_MASK, data: &vf_data->eq_int_mask, nums: 1); |
205 | if (ret) { |
206 | dev_err(dev, "failed to write QM_VF_EQ_INT_MASK\n" ); |
207 | return ret; |
208 | } |
209 | |
210 | ret = qm_write_regs(qm, QM_IFC_INT_SOURCE_V, |
211 | data: &vf_data->ifc_int_source, nums: 1); |
212 | if (ret) { |
213 | dev_err(dev, "failed to write QM_IFC_INT_SOURCE_V\n" ); |
214 | return ret; |
215 | } |
216 | |
217 | ret = qm_write_regs(qm, QM_IFC_INT_MASK, data: &vf_data->ifc_int_mask, nums: 1); |
218 | if (ret) { |
219 | dev_err(dev, "failed to write QM_IFC_INT_MASK\n" ); |
220 | return ret; |
221 | } |
222 | |
223 | ret = qm_write_regs(qm, QM_IFC_INT_SET_V, data: &vf_data->ifc_int_set, nums: 1); |
224 | if (ret) { |
225 | dev_err(dev, "failed to write QM_IFC_INT_SET_V\n" ); |
226 | return ret; |
227 | } |
228 | |
229 | ret = qm_write_regs(qm, QM_QUE_ISO_CFG_V, data: &vf_data->que_iso_cfg, nums: 1); |
230 | if (ret) { |
231 | dev_err(dev, "failed to write QM_QUE_ISO_CFG_V\n" ); |
232 | return ret; |
233 | } |
234 | |
235 | ret = qm_write_regs(qm, QM_PAGE_SIZE, data: &vf_data->page_size, nums: 1); |
236 | if (ret) { |
237 | dev_err(dev, "failed to write QM_PAGE_SIZE\n" ); |
238 | return ret; |
239 | } |
240 | |
241 | /* QM_EQC_DW has 7 regs */ |
242 | ret = qm_write_regs(qm, QM_EQC_DW0, data: vf_data->qm_eqc_dw, nums: 7); |
243 | if (ret) { |
244 | dev_err(dev, "failed to write QM_EQC_DW\n" ); |
245 | return ret; |
246 | } |
247 | |
248 | /* QM_AEQC_DW has 7 regs */ |
249 | ret = qm_write_regs(qm, QM_AEQC_DW0, data: vf_data->qm_aeqc_dw, nums: 7); |
250 | if (ret) { |
251 | dev_err(dev, "failed to write QM_AEQC_DW\n" ); |
252 | return ret; |
253 | } |
254 | |
255 | return 0; |
256 | } |
257 | |
258 | static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, |
259 | u16 index, u8 priority) |
260 | { |
261 | u64 doorbell; |
262 | u64 dbase; |
263 | u16 randata = 0; |
264 | |
265 | if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ) |
266 | dbase = QM_DOORBELL_SQ_CQ_BASE_V2; |
267 | else |
268 | dbase = QM_DOORBELL_EQ_AEQ_BASE_V2; |
269 | |
270 | doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) | |
271 | ((u64)randata << QM_DB_RAND_SHIFT_V2) | |
272 | ((u64)index << QM_DB_INDEX_SHIFT_V2) | |
273 | ((u64)priority << QM_DB_PRIORITY_SHIFT_V2); |
274 | |
275 | writeq(val: doorbell, addr: qm->io_base + dbase); |
276 | } |
277 | |
278 | static int pf_qm_get_qp_num(struct hisi_qm *qm, int vf_id, u32 *rbase) |
279 | { |
280 | unsigned int val; |
281 | u64 sqc_vft; |
282 | u32 qp_num; |
283 | int ret; |
284 | |
285 | ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, |
286 | val & BIT(0), MB_POLL_PERIOD_US, |
287 | MB_POLL_TIMEOUT_US); |
288 | if (ret) |
289 | return ret; |
290 | |
291 | writel(val: 0x1, addr: qm->io_base + QM_VFT_CFG_OP_WR); |
292 | /* 0 mean SQC VFT */ |
293 | writel(val: 0x0, addr: qm->io_base + QM_VFT_CFG_TYPE); |
294 | writel(val: vf_id, addr: qm->io_base + QM_VFT_CFG); |
295 | |
296 | writel(val: 0x0, addr: qm->io_base + QM_VFT_CFG_RDY); |
297 | writel(val: 0x1, addr: qm->io_base + QM_VFT_CFG_OP_ENABLE); |
298 | |
299 | ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, |
300 | val & BIT(0), MB_POLL_PERIOD_US, |
301 | MB_POLL_TIMEOUT_US); |
302 | if (ret) |
303 | return ret; |
304 | |
305 | sqc_vft = readl(addr: qm->io_base + QM_VFT_CFG_DATA_L) | |
306 | ((u64)readl(addr: qm->io_base + QM_VFT_CFG_DATA_H) << |
307 | QM_XQC_ADDR_OFFSET); |
308 | *rbase = QM_SQC_VFT_BASE_MASK_V2 & |
309 | (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2); |
310 | qp_num = (QM_SQC_VFT_NUM_MASK_V2 & |
311 | (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1; |
312 | |
313 | return qp_num; |
314 | } |
315 | |
316 | static void qm_dev_cmd_init(struct hisi_qm *qm) |
317 | { |
318 | /* Clear VF communication status registers. */ |
319 | writel(val: 0x1, addr: qm->io_base + QM_IFC_INT_SOURCE_V); |
320 | |
321 | /* Enable pf and vf communication. */ |
322 | writel(val: 0x0, addr: qm->io_base + QM_IFC_INT_MASK); |
323 | } |
324 | |
325 | static int vf_qm_cache_wb(struct hisi_qm *qm) |
326 | { |
327 | unsigned int val; |
328 | |
329 | writel(val: 0x1, addr: qm->io_base + QM_CACHE_WB_START); |
330 | if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, |
331 | val, val & BIT(0), MB_POLL_PERIOD_US, |
332 | MB_POLL_TIMEOUT_US)) { |
333 | dev_err(&qm->pdev->dev, "vf QM writeback sqc cache fail\n" ); |
334 | return -EINVAL; |
335 | } |
336 | |
337 | return 0; |
338 | } |
339 | |
340 | static void vf_qm_fun_reset(struct hisi_qm *qm) |
341 | { |
342 | int i; |
343 | |
344 | for (i = 0; i < qm->qp_num; i++) |
345 | qm_db(qm, qn: i, QM_DOORBELL_CMD_SQ, index: 0, priority: 1); |
346 | } |
347 | |
348 | static int vf_qm_func_stop(struct hisi_qm *qm) |
349 | { |
350 | return hisi_qm_mb(qm, QM_MB_CMD_PAUSE_QM, dma_addr: 0, queue: 0, op: 0); |
351 | } |
352 | |
353 | static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, |
354 | struct hisi_acc_vf_migration_file *migf) |
355 | { |
356 | struct acc_vf_data *vf_data = &migf->vf_data; |
357 | struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; |
358 | struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm; |
359 | struct device *dev = &vf_qm->pdev->dev; |
360 | u32 que_iso_state; |
361 | int ret; |
362 | |
363 | if (migf->total_length < QM_MATCH_SIZE || hisi_acc_vdev->match_done) |
364 | return 0; |
365 | |
366 | if (vf_data->acc_magic != ACC_DEV_MAGIC) { |
367 | dev_err(dev, "failed to match ACC_DEV_MAGIC\n" ); |
368 | return -EINVAL; |
369 | } |
370 | |
371 | if (vf_data->dev_id != hisi_acc_vdev->vf_dev->device) { |
372 | dev_err(dev, "failed to match VF devices\n" ); |
373 | return -EINVAL; |
374 | } |
375 | |
376 | /* VF qp num check */ |
377 | ret = qm_get_vft(qm: vf_qm, base: &vf_qm->qp_base); |
378 | if (ret <= 0) { |
379 | dev_err(dev, "failed to get vft qp nums\n" ); |
380 | return -EINVAL; |
381 | } |
382 | |
383 | if (ret != vf_data->qp_num) { |
384 | dev_err(dev, "failed to match VF qp num\n" ); |
385 | return -EINVAL; |
386 | } |
387 | |
388 | vf_qm->qp_num = ret; |
389 | |
390 | /* VF isolation state check */ |
391 | ret = qm_read_regs(qm: pf_qm, QM_QUE_ISO_CFG_V, data: &que_iso_state, nums: 1); |
392 | if (ret) { |
393 | dev_err(dev, "failed to read QM_QUE_ISO_CFG_V\n" ); |
394 | return ret; |
395 | } |
396 | |
397 | if (vf_data->que_iso_cfg != que_iso_state) { |
398 | dev_err(dev, "failed to match isolation state\n" ); |
399 | return -EINVAL; |
400 | } |
401 | |
402 | ret = qm_write_regs(qm: vf_qm, QM_VF_STATE, data: &vf_data->vf_qm_state, nums: 1); |
403 | if (ret) { |
404 | dev_err(dev, "failed to write QM_VF_STATE\n" ); |
405 | return ret; |
406 | } |
407 | |
408 | hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; |
409 | hisi_acc_vdev->match_done = true; |
410 | return 0; |
411 | } |
412 | |
413 | static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, |
414 | struct acc_vf_data *vf_data) |
415 | { |
416 | struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm; |
417 | struct device *dev = &pf_qm->pdev->dev; |
418 | int vf_id = hisi_acc_vdev->vf_id; |
419 | int ret; |
420 | |
421 | vf_data->acc_magic = ACC_DEV_MAGIC; |
422 | /* Save device id */ |
423 | vf_data->dev_id = hisi_acc_vdev->vf_dev->device; |
424 | |
425 | /* VF qp num save from PF */ |
426 | ret = pf_qm_get_qp_num(qm: pf_qm, vf_id, rbase: &vf_data->qp_base); |
427 | if (ret <= 0) { |
428 | dev_err(dev, "failed to get vft qp nums!\n" ); |
429 | return -EINVAL; |
430 | } |
431 | |
432 | vf_data->qp_num = ret; |
433 | |
434 | /* VF isolation state save from PF */ |
435 | ret = qm_read_regs(qm: pf_qm, QM_QUE_ISO_CFG_V, data: &vf_data->que_iso_cfg, nums: 1); |
436 | if (ret) { |
437 | dev_err(dev, "failed to read QM_QUE_ISO_CFG_V!\n" ); |
438 | return ret; |
439 | } |
440 | |
441 | return 0; |
442 | } |
443 | |
444 | static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, |
445 | struct hisi_acc_vf_migration_file *migf) |
446 | { |
447 | struct hisi_qm *qm = &hisi_acc_vdev->vf_qm; |
448 | struct device *dev = &qm->pdev->dev; |
449 | struct acc_vf_data *vf_data = &migf->vf_data; |
450 | int ret; |
451 | |
452 | /* Return if only match data was transferred */ |
453 | if (migf->total_length == QM_MATCH_SIZE) |
454 | return 0; |
455 | |
456 | if (migf->total_length < sizeof(struct acc_vf_data)) |
457 | return -EINVAL; |
458 | |
459 | qm->eqe_dma = vf_data->eqe_dma; |
460 | qm->aeqe_dma = vf_data->aeqe_dma; |
461 | qm->sqc_dma = vf_data->sqc_dma; |
462 | qm->cqc_dma = vf_data->cqc_dma; |
463 | |
464 | qm->qp_base = vf_data->qp_base; |
465 | qm->qp_num = vf_data->qp_num; |
466 | |
467 | ret = qm_set_regs(qm, vf_data); |
468 | if (ret) { |
469 | dev_err(dev, "set VF regs failed\n" ); |
470 | return ret; |
471 | } |
472 | |
473 | ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, dma_addr: qm->sqc_dma, queue: 0, op: 0); |
474 | if (ret) { |
475 | dev_err(dev, "set sqc failed\n" ); |
476 | return ret; |
477 | } |
478 | |
479 | ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, dma_addr: qm->cqc_dma, queue: 0, op: 0); |
480 | if (ret) { |
481 | dev_err(dev, "set cqc failed\n" ); |
482 | return ret; |
483 | } |
484 | |
485 | qm_dev_cmd_init(qm); |
486 | return 0; |
487 | } |
488 | |
489 | static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev, |
490 | struct hisi_acc_vf_migration_file *migf) |
491 | { |
492 | struct acc_vf_data *vf_data = &migf->vf_data; |
493 | struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; |
494 | struct device *dev = &vf_qm->pdev->dev; |
495 | int ret; |
496 | |
497 | if (unlikely(qm_wait_dev_not_ready(vf_qm))) { |
498 | /* Update state and return with match data */ |
499 | vf_data->vf_qm_state = QM_NOT_READY; |
500 | hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; |
501 | migf->total_length = QM_MATCH_SIZE; |
502 | return 0; |
503 | } |
504 | |
505 | vf_data->vf_qm_state = QM_READY; |
506 | hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; |
507 | |
508 | ret = vf_qm_cache_wb(qm: vf_qm); |
509 | if (ret) { |
510 | dev_err(dev, "failed to writeback QM Cache!\n" ); |
511 | return ret; |
512 | } |
513 | |
514 | ret = qm_get_regs(qm: vf_qm, vf_data); |
515 | if (ret) |
516 | return -EINVAL; |
517 | |
518 | /* Every reg is 32 bit, the dma address is 64 bit. */ |
519 | vf_data->eqe_dma = vf_data->qm_eqc_dw[1]; |
520 | vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET; |
521 | vf_data->eqe_dma |= vf_data->qm_eqc_dw[0]; |
522 | vf_data->aeqe_dma = vf_data->qm_aeqc_dw[1]; |
523 | vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET; |
524 | vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[0]; |
525 | |
526 | /* Through SQC_BT/CQC_BT to get sqc and cqc address */ |
527 | ret = qm_get_sqc(qm: vf_qm, addr: &vf_data->sqc_dma); |
528 | if (ret) { |
529 | dev_err(dev, "failed to read SQC addr!\n" ); |
530 | return -EINVAL; |
531 | } |
532 | |
533 | ret = qm_get_cqc(qm: vf_qm, addr: &vf_data->cqc_dma); |
534 | if (ret) { |
535 | dev_err(dev, "failed to read CQC addr!\n" ); |
536 | return -EINVAL; |
537 | } |
538 | |
539 | migf->total_length = sizeof(struct acc_vf_data); |
540 | return 0; |
541 | } |
542 | |
543 | static struct hisi_acc_vf_core_device *hisi_acc_drvdata(struct pci_dev *pdev) |
544 | { |
545 | struct vfio_pci_core_device *core_device = dev_get_drvdata(dev: &pdev->dev); |
546 | |
547 | return container_of(core_device, struct hisi_acc_vf_core_device, |
548 | core_device); |
549 | } |
550 | |
551 | /* Check the PF's RAS state and Function INT state */ |
552 | static int |
553 | hisi_acc_check_int_state(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
554 | { |
555 | struct hisi_qm *vfqm = &hisi_acc_vdev->vf_qm; |
556 | struct hisi_qm *qm = hisi_acc_vdev->pf_qm; |
557 | struct pci_dev *vf_pdev = hisi_acc_vdev->vf_dev; |
558 | struct device *dev = &qm->pdev->dev; |
559 | u32 state; |
560 | |
561 | /* Check RAS state */ |
562 | state = qm_check_reg_state(qm, QM_ABNORMAL_INT_STATUS); |
563 | if (state) { |
564 | dev_err(dev, "failed to check QM RAS state!\n" ); |
565 | return -EBUSY; |
566 | } |
567 | |
568 | /* Check Function Communication state between PF and VF */ |
569 | state = qm_check_reg_state(qm: vfqm, QM_IFC_INT_STATUS); |
570 | if (state) { |
571 | dev_err(dev, "failed to check QM IFC INT state!\n" ); |
572 | return -EBUSY; |
573 | } |
574 | state = qm_check_reg_state(qm: vfqm, QM_IFC_INT_SET_V); |
575 | if (state) { |
576 | dev_err(dev, "failed to check QM IFC INT SET state!\n" ); |
577 | return -EBUSY; |
578 | } |
579 | |
580 | /* Check submodule task state */ |
581 | switch (vf_pdev->device) { |
582 | case PCI_DEVICE_ID_HUAWEI_SEC_VF: |
583 | state = qm_check_reg_state(qm, SEC_CORE_INT_STATUS); |
584 | if (state) { |
585 | dev_err(dev, "failed to check QM SEC Core INT state!\n" ); |
586 | return -EBUSY; |
587 | } |
588 | return 0; |
589 | case PCI_DEVICE_ID_HUAWEI_HPRE_VF: |
590 | state = qm_check_reg_state(qm, HPRE_HAC_INT_STATUS); |
591 | if (state) { |
592 | dev_err(dev, "failed to check QM HPRE HAC INT state!\n" ); |
593 | return -EBUSY; |
594 | } |
595 | return 0; |
596 | case PCI_DEVICE_ID_HUAWEI_ZIP_VF: |
597 | state = qm_check_reg_state(qm, HZIP_CORE_INT_STATUS); |
598 | if (state) { |
599 | dev_err(dev, "failed to check QM ZIP Core INT state!\n" ); |
600 | return -EBUSY; |
601 | } |
602 | return 0; |
603 | default: |
604 | dev_err(dev, "failed to detect acc module type!\n" ); |
605 | return -EINVAL; |
606 | } |
607 | } |
608 | |
609 | static void hisi_acc_vf_disable_fd(struct hisi_acc_vf_migration_file *migf) |
610 | { |
611 | mutex_lock(&migf->lock); |
612 | migf->disabled = true; |
613 | migf->total_length = 0; |
614 | migf->filp->f_pos = 0; |
615 | mutex_unlock(lock: &migf->lock); |
616 | } |
617 | |
618 | static void hisi_acc_vf_disable_fds(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
619 | { |
620 | if (hisi_acc_vdev->resuming_migf) { |
621 | hisi_acc_vf_disable_fd(migf: hisi_acc_vdev->resuming_migf); |
622 | fput(hisi_acc_vdev->resuming_migf->filp); |
623 | hisi_acc_vdev->resuming_migf = NULL; |
624 | } |
625 | |
626 | if (hisi_acc_vdev->saving_migf) { |
627 | hisi_acc_vf_disable_fd(migf: hisi_acc_vdev->saving_migf); |
628 | fput(hisi_acc_vdev->saving_migf->filp); |
629 | hisi_acc_vdev->saving_migf = NULL; |
630 | } |
631 | } |
632 | |
633 | static void hisi_acc_vf_reset(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
634 | { |
635 | hisi_acc_vdev->vf_qm_state = QM_NOT_READY; |
636 | hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; |
637 | hisi_acc_vf_disable_fds(hisi_acc_vdev); |
638 | } |
639 | |
640 | static void hisi_acc_vf_start_device(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
641 | { |
642 | struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; |
643 | |
644 | if (hisi_acc_vdev->vf_qm_state != QM_READY) |
645 | return; |
646 | |
647 | /* Make sure the device is enabled */ |
648 | qm_dev_cmd_init(qm: vf_qm); |
649 | |
650 | vf_qm_fun_reset(qm: vf_qm); |
651 | } |
652 | |
653 | static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
654 | { |
655 | struct device *dev = &hisi_acc_vdev->vf_dev->dev; |
656 | struct hisi_acc_vf_migration_file *migf = hisi_acc_vdev->resuming_migf; |
657 | int ret; |
658 | |
659 | /* Recover data to VF */ |
660 | ret = vf_qm_load_data(hisi_acc_vdev, migf); |
661 | if (ret) { |
662 | dev_err(dev, "failed to recover the VF!\n" ); |
663 | return ret; |
664 | } |
665 | |
666 | return 0; |
667 | } |
668 | |
669 | static int hisi_acc_vf_release_file(struct inode *inode, struct file *filp) |
670 | { |
671 | struct hisi_acc_vf_migration_file *migf = filp->private_data; |
672 | |
673 | hisi_acc_vf_disable_fd(migf); |
674 | mutex_destroy(lock: &migf->lock); |
675 | kfree(objp: migf); |
676 | return 0; |
677 | } |
678 | |
679 | static ssize_t hisi_acc_vf_resume_write(struct file *filp, const char __user *buf, |
680 | size_t len, loff_t *pos) |
681 | { |
682 | struct hisi_acc_vf_migration_file *migf = filp->private_data; |
683 | u8 *vf_data = (u8 *)&migf->vf_data; |
684 | loff_t requested_length; |
685 | ssize_t done = 0; |
686 | int ret; |
687 | |
688 | if (pos) |
689 | return -ESPIPE; |
690 | pos = &filp->f_pos; |
691 | |
692 | if (*pos < 0 || |
693 | check_add_overflow((loff_t)len, *pos, &requested_length)) |
694 | return -EINVAL; |
695 | |
696 | if (requested_length > sizeof(struct acc_vf_data)) |
697 | return -ENOMEM; |
698 | |
699 | mutex_lock(&migf->lock); |
700 | if (migf->disabled) { |
701 | done = -ENODEV; |
702 | goto out_unlock; |
703 | } |
704 | |
705 | ret = copy_from_user(to: vf_data + *pos, from: buf, n: len); |
706 | if (ret) { |
707 | done = -EFAULT; |
708 | goto out_unlock; |
709 | } |
710 | *pos += len; |
711 | done = len; |
712 | migf->total_length += len; |
713 | |
714 | ret = vf_qm_check_match(hisi_acc_vdev: migf->hisi_acc_vdev, migf); |
715 | if (ret) |
716 | done = -EFAULT; |
717 | out_unlock: |
718 | mutex_unlock(lock: &migf->lock); |
719 | return done; |
720 | } |
721 | |
722 | static const struct file_operations hisi_acc_vf_resume_fops = { |
723 | .owner = THIS_MODULE, |
724 | .write = hisi_acc_vf_resume_write, |
725 | .release = hisi_acc_vf_release_file, |
726 | .llseek = no_llseek, |
727 | }; |
728 | |
729 | static struct hisi_acc_vf_migration_file * |
730 | hisi_acc_vf_pci_resume(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
731 | { |
732 | struct hisi_acc_vf_migration_file *migf; |
733 | |
734 | migf = kzalloc(size: sizeof(*migf), GFP_KERNEL_ACCOUNT); |
735 | if (!migf) |
736 | return ERR_PTR(error: -ENOMEM); |
737 | |
738 | migf->filp = anon_inode_getfile(name: "hisi_acc_vf_mig" , fops: &hisi_acc_vf_resume_fops, priv: migf, |
739 | O_WRONLY); |
740 | if (IS_ERR(ptr: migf->filp)) { |
741 | int err = PTR_ERR(ptr: migf->filp); |
742 | |
743 | kfree(objp: migf); |
744 | return ERR_PTR(error: err); |
745 | } |
746 | |
747 | stream_open(inode: migf->filp->f_inode, filp: migf->filp); |
748 | mutex_init(&migf->lock); |
749 | migf->hisi_acc_vdev = hisi_acc_vdev; |
750 | return migf; |
751 | } |
752 | |
753 | static long hisi_acc_vf_precopy_ioctl(struct file *filp, |
754 | unsigned int cmd, unsigned long arg) |
755 | { |
756 | struct hisi_acc_vf_migration_file *migf = filp->private_data; |
757 | struct hisi_acc_vf_core_device *hisi_acc_vdev = migf->hisi_acc_vdev; |
758 | loff_t *pos = &filp->f_pos; |
759 | struct vfio_precopy_info info; |
760 | unsigned long minsz; |
761 | int ret; |
762 | |
763 | if (cmd != VFIO_MIG_GET_PRECOPY_INFO) |
764 | return -ENOTTY; |
765 | |
766 | minsz = offsetofend(struct vfio_precopy_info, dirty_bytes); |
767 | |
768 | if (copy_from_user(to: &info, from: (void __user *)arg, n: minsz)) |
769 | return -EFAULT; |
770 | if (info.argsz < minsz) |
771 | return -EINVAL; |
772 | |
773 | mutex_lock(&hisi_acc_vdev->state_mutex); |
774 | if (hisi_acc_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY) { |
775 | mutex_unlock(lock: &hisi_acc_vdev->state_mutex); |
776 | return -EINVAL; |
777 | } |
778 | |
779 | mutex_lock(&migf->lock); |
780 | |
781 | if (migf->disabled) { |
782 | ret = -ENODEV; |
783 | goto out; |
784 | } |
785 | |
786 | if (*pos > migf->total_length) { |
787 | ret = -EINVAL; |
788 | goto out; |
789 | } |
790 | |
791 | info.dirty_bytes = 0; |
792 | info.initial_bytes = migf->total_length - *pos; |
793 | mutex_unlock(lock: &migf->lock); |
794 | mutex_unlock(lock: &hisi_acc_vdev->state_mutex); |
795 | |
796 | return copy_to_user(to: (void __user *)arg, from: &info, n: minsz) ? -EFAULT : 0; |
797 | out: |
798 | mutex_unlock(lock: &migf->lock); |
799 | mutex_unlock(lock: &hisi_acc_vdev->state_mutex); |
800 | return ret; |
801 | } |
802 | |
803 | static ssize_t hisi_acc_vf_save_read(struct file *filp, char __user *buf, size_t len, |
804 | loff_t *pos) |
805 | { |
806 | struct hisi_acc_vf_migration_file *migf = filp->private_data; |
807 | ssize_t done = 0; |
808 | int ret; |
809 | |
810 | if (pos) |
811 | return -ESPIPE; |
812 | pos = &filp->f_pos; |
813 | |
814 | mutex_lock(&migf->lock); |
815 | if (*pos > migf->total_length) { |
816 | done = -EINVAL; |
817 | goto out_unlock; |
818 | } |
819 | |
820 | if (migf->disabled) { |
821 | done = -ENODEV; |
822 | goto out_unlock; |
823 | } |
824 | |
825 | len = min_t(size_t, migf->total_length - *pos, len); |
826 | if (len) { |
827 | u8 *vf_data = (u8 *)&migf->vf_data; |
828 | |
829 | ret = copy_to_user(to: buf, from: vf_data + *pos, n: len); |
830 | if (ret) { |
831 | done = -EFAULT; |
832 | goto out_unlock; |
833 | } |
834 | *pos += len; |
835 | done = len; |
836 | } |
837 | out_unlock: |
838 | mutex_unlock(lock: &migf->lock); |
839 | return done; |
840 | } |
841 | |
842 | static const struct file_operations hisi_acc_vf_save_fops = { |
843 | .owner = THIS_MODULE, |
844 | .read = hisi_acc_vf_save_read, |
845 | .unlocked_ioctl = hisi_acc_vf_precopy_ioctl, |
846 | .compat_ioctl = compat_ptr_ioctl, |
847 | .release = hisi_acc_vf_release_file, |
848 | .llseek = no_llseek, |
849 | }; |
850 | |
851 | static struct hisi_acc_vf_migration_file * |
852 | hisi_acc_open_saving_migf(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
853 | { |
854 | struct hisi_acc_vf_migration_file *migf; |
855 | int ret; |
856 | |
857 | migf = kzalloc(size: sizeof(*migf), GFP_KERNEL_ACCOUNT); |
858 | if (!migf) |
859 | return ERR_PTR(error: -ENOMEM); |
860 | |
861 | migf->filp = anon_inode_getfile(name: "hisi_acc_vf_mig" , fops: &hisi_acc_vf_save_fops, priv: migf, |
862 | O_RDONLY); |
863 | if (IS_ERR(ptr: migf->filp)) { |
864 | int err = PTR_ERR(ptr: migf->filp); |
865 | |
866 | kfree(objp: migf); |
867 | return ERR_PTR(error: err); |
868 | } |
869 | |
870 | stream_open(inode: migf->filp->f_inode, filp: migf->filp); |
871 | mutex_init(&migf->lock); |
872 | migf->hisi_acc_vdev = hisi_acc_vdev; |
873 | |
874 | ret = vf_qm_get_match_data(hisi_acc_vdev, vf_data: &migf->vf_data); |
875 | if (ret) { |
876 | fput(migf->filp); |
877 | return ERR_PTR(error: ret); |
878 | } |
879 | |
880 | return migf; |
881 | } |
882 | |
883 | static struct hisi_acc_vf_migration_file * |
884 | hisi_acc_vf_pre_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
885 | { |
886 | struct hisi_acc_vf_migration_file *migf; |
887 | |
888 | migf = hisi_acc_open_saving_migf(hisi_acc_vdev); |
889 | if (IS_ERR(ptr: migf)) |
890 | return migf; |
891 | |
892 | migf->total_length = QM_MATCH_SIZE; |
893 | return migf; |
894 | } |
895 | |
896 | static struct hisi_acc_vf_migration_file * |
897 | hisi_acc_vf_stop_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev, bool open) |
898 | { |
899 | int ret; |
900 | struct hisi_acc_vf_migration_file *migf = NULL; |
901 | |
902 | if (open) { |
903 | /* |
904 | * Userspace didn't use PRECOPY support. Hence saving_migf |
905 | * is not opened yet. |
906 | */ |
907 | migf = hisi_acc_open_saving_migf(hisi_acc_vdev); |
908 | if (IS_ERR(ptr: migf)) |
909 | return migf; |
910 | } else { |
911 | migf = hisi_acc_vdev->saving_migf; |
912 | } |
913 | |
914 | ret = vf_qm_state_save(hisi_acc_vdev, migf); |
915 | if (ret) |
916 | return ERR_PTR(error: ret); |
917 | |
918 | return open ? migf : NULL; |
919 | } |
920 | |
921 | static int hisi_acc_vf_stop_device(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
922 | { |
923 | struct device *dev = &hisi_acc_vdev->vf_dev->dev; |
924 | struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; |
925 | int ret; |
926 | |
927 | ret = vf_qm_func_stop(qm: vf_qm); |
928 | if (ret) { |
929 | dev_err(dev, "failed to stop QM VF function!\n" ); |
930 | return ret; |
931 | } |
932 | |
933 | ret = hisi_acc_check_int_state(hisi_acc_vdev); |
934 | if (ret) { |
935 | dev_err(dev, "failed to check QM INT state!\n" ); |
936 | return ret; |
937 | } |
938 | return 0; |
939 | } |
940 | |
941 | static struct file * |
942 | hisi_acc_vf_set_device_state(struct hisi_acc_vf_core_device *hisi_acc_vdev, |
943 | u32 new) |
944 | { |
945 | u32 cur = hisi_acc_vdev->mig_state; |
946 | int ret; |
947 | |
948 | if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_PRE_COPY) { |
949 | struct hisi_acc_vf_migration_file *migf; |
950 | |
951 | migf = hisi_acc_vf_pre_copy(hisi_acc_vdev); |
952 | if (IS_ERR(ptr: migf)) |
953 | return ERR_CAST(ptr: migf); |
954 | get_file(f: migf->filp); |
955 | hisi_acc_vdev->saving_migf = migf; |
956 | return migf->filp; |
957 | } |
958 | |
959 | if (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_STOP_COPY) { |
960 | struct hisi_acc_vf_migration_file *migf; |
961 | |
962 | ret = hisi_acc_vf_stop_device(hisi_acc_vdev); |
963 | if (ret) |
964 | return ERR_PTR(error: ret); |
965 | |
966 | migf = hisi_acc_vf_stop_copy(hisi_acc_vdev, open: false); |
967 | if (IS_ERR(ptr: migf)) |
968 | return ERR_CAST(ptr: migf); |
969 | |
970 | return NULL; |
971 | } |
972 | |
973 | if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_STOP) { |
974 | ret = hisi_acc_vf_stop_device(hisi_acc_vdev); |
975 | if (ret) |
976 | return ERR_PTR(error: ret); |
977 | return NULL; |
978 | } |
979 | |
980 | if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) { |
981 | struct hisi_acc_vf_migration_file *migf; |
982 | |
983 | migf = hisi_acc_vf_stop_copy(hisi_acc_vdev, open: true); |
984 | if (IS_ERR(ptr: migf)) |
985 | return ERR_CAST(ptr: migf); |
986 | get_file(f: migf->filp); |
987 | hisi_acc_vdev->saving_migf = migf; |
988 | return migf->filp; |
989 | } |
990 | |
991 | if ((cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP)) { |
992 | hisi_acc_vf_disable_fds(hisi_acc_vdev); |
993 | return NULL; |
994 | } |
995 | |
996 | if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) { |
997 | struct hisi_acc_vf_migration_file *migf; |
998 | |
999 | migf = hisi_acc_vf_pci_resume(hisi_acc_vdev); |
1000 | if (IS_ERR(ptr: migf)) |
1001 | return ERR_CAST(ptr: migf); |
1002 | get_file(f: migf->filp); |
1003 | hisi_acc_vdev->resuming_migf = migf; |
1004 | return migf->filp; |
1005 | } |
1006 | |
1007 | if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) { |
1008 | ret = hisi_acc_vf_load_state(hisi_acc_vdev); |
1009 | if (ret) |
1010 | return ERR_PTR(error: ret); |
1011 | hisi_acc_vf_disable_fds(hisi_acc_vdev); |
1012 | return NULL; |
1013 | } |
1014 | |
1015 | if (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_RUNNING) { |
1016 | hisi_acc_vf_disable_fds(hisi_acc_vdev); |
1017 | return NULL; |
1018 | } |
1019 | |
1020 | if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING) { |
1021 | hisi_acc_vf_start_device(hisi_acc_vdev); |
1022 | return NULL; |
1023 | } |
1024 | |
1025 | /* |
1026 | * vfio_mig_get_next_state() does not use arcs other than the above |
1027 | */ |
1028 | WARN_ON(true); |
1029 | return ERR_PTR(error: -EINVAL); |
1030 | } |
1031 | |
1032 | static struct file * |
1033 | hisi_acc_vfio_pci_set_device_state(struct vfio_device *vdev, |
1034 | enum vfio_device_mig_state new_state) |
1035 | { |
1036 | struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev, |
1037 | struct hisi_acc_vf_core_device, core_device.vdev); |
1038 | enum vfio_device_mig_state next_state; |
1039 | struct file *res = NULL; |
1040 | int ret; |
1041 | |
1042 | mutex_lock(&hisi_acc_vdev->state_mutex); |
1043 | while (new_state != hisi_acc_vdev->mig_state) { |
1044 | ret = vfio_mig_get_next_state(device: vdev, |
1045 | cur_fsm: hisi_acc_vdev->mig_state, |
1046 | new_fsm: new_state, next_fsm: &next_state); |
1047 | if (ret) { |
1048 | res = ERR_PTR(error: -EINVAL); |
1049 | break; |
1050 | } |
1051 | |
1052 | res = hisi_acc_vf_set_device_state(hisi_acc_vdev, new: next_state); |
1053 | if (IS_ERR(ptr: res)) |
1054 | break; |
1055 | hisi_acc_vdev->mig_state = next_state; |
1056 | if (WARN_ON(res && new_state != hisi_acc_vdev->mig_state)) { |
1057 | fput(res); |
1058 | res = ERR_PTR(error: -EINVAL); |
1059 | break; |
1060 | } |
1061 | } |
1062 | mutex_unlock(lock: &hisi_acc_vdev->state_mutex); |
1063 | return res; |
1064 | } |
1065 | |
1066 | static int |
1067 | hisi_acc_vfio_pci_get_data_size(struct vfio_device *vdev, |
1068 | unsigned long *stop_copy_length) |
1069 | { |
1070 | *stop_copy_length = sizeof(struct acc_vf_data); |
1071 | return 0; |
1072 | } |
1073 | |
1074 | static int |
1075 | hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev, |
1076 | enum vfio_device_mig_state *curr_state) |
1077 | { |
1078 | struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev, |
1079 | struct hisi_acc_vf_core_device, core_device.vdev); |
1080 | |
1081 | mutex_lock(&hisi_acc_vdev->state_mutex); |
1082 | *curr_state = hisi_acc_vdev->mig_state; |
1083 | mutex_unlock(lock: &hisi_acc_vdev->state_mutex); |
1084 | return 0; |
1085 | } |
1086 | |
1087 | static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev) |
1088 | { |
1089 | struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); |
1090 | |
1091 | if (hisi_acc_vdev->core_device.vdev.migration_flags != |
1092 | VFIO_MIGRATION_STOP_COPY) |
1093 | return; |
1094 | |
1095 | mutex_lock(&hisi_acc_vdev->state_mutex); |
1096 | hisi_acc_vf_reset(hisi_acc_vdev); |
1097 | mutex_unlock(lock: &hisi_acc_vdev->state_mutex); |
1098 | } |
1099 | |
1100 | static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
1101 | { |
1102 | struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device; |
1103 | struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; |
1104 | struct pci_dev *vf_dev = vdev->pdev; |
1105 | |
1106 | /* |
1107 | * ACC VF dev BAR2 region consists of both functional register space |
1108 | * and migration control register space. For migration to work, we |
1109 | * need access to both. Hence, we map the entire BAR2 region here. |
1110 | * But unnecessarily exposing the migration BAR region to the Guest |
1111 | * has the potential to prevent/corrupt the Guest migration. Hence, |
1112 | * we restrict access to the migration control space from |
1113 | * Guest(Please see mmap/ioctl/read/write override functions). |
1114 | * |
1115 | * Please note that it is OK to expose the entire VF BAR if migration |
1116 | * is not supported or required as this cannot affect the ACC PF |
1117 | * configurations. |
1118 | * |
1119 | * Also the HiSilicon ACC VF devices supported by this driver on |
1120 | * HiSilicon hardware platforms are integrated end point devices |
1121 | * and the platform lacks the capability to perform any PCIe P2P |
1122 | * between these devices. |
1123 | */ |
1124 | |
1125 | vf_qm->io_base = |
1126 | ioremap(pci_resource_start(vf_dev, VFIO_PCI_BAR2_REGION_INDEX), |
1127 | pci_resource_len(vf_dev, VFIO_PCI_BAR2_REGION_INDEX)); |
1128 | if (!vf_qm->io_base) |
1129 | return -EIO; |
1130 | |
1131 | vf_qm->fun_type = QM_HW_VF; |
1132 | vf_qm->pdev = vf_dev; |
1133 | mutex_init(&vf_qm->mailbox_lock); |
1134 | |
1135 | return 0; |
1136 | } |
1137 | |
1138 | static struct hisi_qm *hisi_acc_get_pf_qm(struct pci_dev *pdev) |
1139 | { |
1140 | struct hisi_qm *pf_qm; |
1141 | struct pci_driver *pf_driver; |
1142 | |
1143 | if (!pdev->is_virtfn) |
1144 | return NULL; |
1145 | |
1146 | switch (pdev->device) { |
1147 | case PCI_DEVICE_ID_HUAWEI_SEC_VF: |
1148 | pf_driver = hisi_sec_get_pf_driver(); |
1149 | break; |
1150 | case PCI_DEVICE_ID_HUAWEI_HPRE_VF: |
1151 | pf_driver = hisi_hpre_get_pf_driver(); |
1152 | break; |
1153 | case PCI_DEVICE_ID_HUAWEI_ZIP_VF: |
1154 | pf_driver = hisi_zip_get_pf_driver(); |
1155 | break; |
1156 | default: |
1157 | return NULL; |
1158 | } |
1159 | |
1160 | if (!pf_driver) |
1161 | return NULL; |
1162 | |
1163 | pf_qm = pci_iov_get_pf_drvdata(dev: pdev, pf_driver); |
1164 | |
1165 | return !IS_ERR(ptr: pf_qm) ? pf_qm : NULL; |
1166 | } |
1167 | |
1168 | static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev, |
1169 | size_t count, loff_t *ppos, |
1170 | size_t *new_count) |
1171 | { |
1172 | unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); |
1173 | struct vfio_pci_core_device *vdev = |
1174 | container_of(core_vdev, struct vfio_pci_core_device, vdev); |
1175 | |
1176 | if (index == VFIO_PCI_BAR2_REGION_INDEX) { |
1177 | loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; |
1178 | resource_size_t end = pci_resource_len(vdev->pdev, index) / 2; |
1179 | |
1180 | /* Check if access is for migration control region */ |
1181 | if (pos >= end) |
1182 | return -EINVAL; |
1183 | |
1184 | *new_count = min(count, (size_t)(end - pos)); |
1185 | } |
1186 | |
1187 | return 0; |
1188 | } |
1189 | |
1190 | static int hisi_acc_vfio_pci_mmap(struct vfio_device *core_vdev, |
1191 | struct vm_area_struct *vma) |
1192 | { |
1193 | struct vfio_pci_core_device *vdev = |
1194 | container_of(core_vdev, struct vfio_pci_core_device, vdev); |
1195 | unsigned int index; |
1196 | |
1197 | index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); |
1198 | if (index == VFIO_PCI_BAR2_REGION_INDEX) { |
1199 | u64 req_len, pgoff, req_start; |
1200 | resource_size_t end = pci_resource_len(vdev->pdev, index) / 2; |
1201 | |
1202 | req_len = vma->vm_end - vma->vm_start; |
1203 | pgoff = vma->vm_pgoff & |
1204 | ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); |
1205 | req_start = pgoff << PAGE_SHIFT; |
1206 | |
1207 | if (req_start + req_len > end) |
1208 | return -EINVAL; |
1209 | } |
1210 | |
1211 | return vfio_pci_core_mmap(core_vdev, vma); |
1212 | } |
1213 | |
1214 | static ssize_t hisi_acc_vfio_pci_write(struct vfio_device *core_vdev, |
1215 | const char __user *buf, size_t count, |
1216 | loff_t *ppos) |
1217 | { |
1218 | size_t new_count = count; |
1219 | int ret; |
1220 | |
1221 | ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, new_count: &new_count); |
1222 | if (ret) |
1223 | return ret; |
1224 | |
1225 | return vfio_pci_core_write(core_vdev, buf, count: new_count, ppos); |
1226 | } |
1227 | |
1228 | static ssize_t hisi_acc_vfio_pci_read(struct vfio_device *core_vdev, |
1229 | char __user *buf, size_t count, |
1230 | loff_t *ppos) |
1231 | { |
1232 | size_t new_count = count; |
1233 | int ret; |
1234 | |
1235 | ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, new_count: &new_count); |
1236 | if (ret) |
1237 | return ret; |
1238 | |
1239 | return vfio_pci_core_read(core_vdev, buf, count: new_count, ppos); |
1240 | } |
1241 | |
1242 | static long hisi_acc_vfio_pci_ioctl(struct vfio_device *core_vdev, unsigned int cmd, |
1243 | unsigned long arg) |
1244 | { |
1245 | if (cmd == VFIO_DEVICE_GET_REGION_INFO) { |
1246 | struct vfio_pci_core_device *vdev = |
1247 | container_of(core_vdev, struct vfio_pci_core_device, vdev); |
1248 | struct pci_dev *pdev = vdev->pdev; |
1249 | struct vfio_region_info info; |
1250 | unsigned long minsz; |
1251 | |
1252 | minsz = offsetofend(struct vfio_region_info, offset); |
1253 | |
1254 | if (copy_from_user(to: &info, from: (void __user *)arg, n: minsz)) |
1255 | return -EFAULT; |
1256 | |
1257 | if (info.argsz < minsz) |
1258 | return -EINVAL; |
1259 | |
1260 | if (info.index == VFIO_PCI_BAR2_REGION_INDEX) { |
1261 | info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); |
1262 | |
1263 | /* |
1264 | * ACC VF dev BAR2 region consists of both functional |
1265 | * register space and migration control register space. |
1266 | * Report only the functional region to Guest. |
1267 | */ |
1268 | info.size = pci_resource_len(pdev, info.index) / 2; |
1269 | |
1270 | info.flags = VFIO_REGION_INFO_FLAG_READ | |
1271 | VFIO_REGION_INFO_FLAG_WRITE | |
1272 | VFIO_REGION_INFO_FLAG_MMAP; |
1273 | |
1274 | return copy_to_user(to: (void __user *)arg, from: &info, n: minsz) ? |
1275 | -EFAULT : 0; |
1276 | } |
1277 | } |
1278 | return vfio_pci_core_ioctl(core_vdev, cmd, arg); |
1279 | } |
1280 | |
1281 | static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev) |
1282 | { |
1283 | struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev, |
1284 | struct hisi_acc_vf_core_device, core_device.vdev); |
1285 | struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device; |
1286 | int ret; |
1287 | |
1288 | ret = vfio_pci_core_enable(vdev); |
1289 | if (ret) |
1290 | return ret; |
1291 | |
1292 | if (core_vdev->mig_ops) { |
1293 | ret = hisi_acc_vf_qm_init(hisi_acc_vdev); |
1294 | if (ret) { |
1295 | vfio_pci_core_disable(vdev); |
1296 | return ret; |
1297 | } |
1298 | hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; |
1299 | } |
1300 | |
1301 | vfio_pci_core_finish_enable(vdev); |
1302 | return 0; |
1303 | } |
1304 | |
1305 | static void hisi_acc_vfio_pci_close_device(struct vfio_device *core_vdev) |
1306 | { |
1307 | struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev, |
1308 | struct hisi_acc_vf_core_device, core_device.vdev); |
1309 | struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; |
1310 | |
1311 | iounmap(addr: vf_qm->io_base); |
1312 | vfio_pci_core_close_device(core_vdev); |
1313 | } |
1314 | |
1315 | static const struct vfio_migration_ops hisi_acc_vfio_pci_migrn_state_ops = { |
1316 | .migration_set_state = hisi_acc_vfio_pci_set_device_state, |
1317 | .migration_get_state = hisi_acc_vfio_pci_get_device_state, |
1318 | .migration_get_data_size = hisi_acc_vfio_pci_get_data_size, |
1319 | }; |
1320 | |
1321 | static int hisi_acc_vfio_pci_migrn_init_dev(struct vfio_device *core_vdev) |
1322 | { |
1323 | struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev, |
1324 | struct hisi_acc_vf_core_device, core_device.vdev); |
1325 | struct pci_dev *pdev = to_pci_dev(core_vdev->dev); |
1326 | struct hisi_qm *pf_qm = hisi_acc_get_pf_qm(pdev); |
1327 | |
1328 | hisi_acc_vdev->vf_id = pci_iov_vf_id(dev: pdev) + 1; |
1329 | hisi_acc_vdev->pf_qm = pf_qm; |
1330 | hisi_acc_vdev->vf_dev = pdev; |
1331 | mutex_init(&hisi_acc_vdev->state_mutex); |
1332 | |
1333 | core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY; |
1334 | core_vdev->mig_ops = &hisi_acc_vfio_pci_migrn_state_ops; |
1335 | |
1336 | return vfio_pci_core_init_dev(core_vdev); |
1337 | } |
1338 | |
1339 | static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = { |
1340 | .name = "hisi-acc-vfio-pci-migration" , |
1341 | .init = hisi_acc_vfio_pci_migrn_init_dev, |
1342 | .release = vfio_pci_core_release_dev, |
1343 | .open_device = hisi_acc_vfio_pci_open_device, |
1344 | .close_device = hisi_acc_vfio_pci_close_device, |
1345 | .ioctl = hisi_acc_vfio_pci_ioctl, |
1346 | .device_feature = vfio_pci_core_ioctl_feature, |
1347 | .read = hisi_acc_vfio_pci_read, |
1348 | .write = hisi_acc_vfio_pci_write, |
1349 | .mmap = hisi_acc_vfio_pci_mmap, |
1350 | .request = vfio_pci_core_request, |
1351 | .match = vfio_pci_core_match, |
1352 | .bind_iommufd = vfio_iommufd_physical_bind, |
1353 | .unbind_iommufd = vfio_iommufd_physical_unbind, |
1354 | .attach_ioas = vfio_iommufd_physical_attach_ioas, |
1355 | .detach_ioas = vfio_iommufd_physical_detach_ioas, |
1356 | }; |
1357 | |
1358 | static const struct vfio_device_ops hisi_acc_vfio_pci_ops = { |
1359 | .name = "hisi-acc-vfio-pci" , |
1360 | .init = vfio_pci_core_init_dev, |
1361 | .release = vfio_pci_core_release_dev, |
1362 | .open_device = hisi_acc_vfio_pci_open_device, |
1363 | .close_device = vfio_pci_core_close_device, |
1364 | .ioctl = vfio_pci_core_ioctl, |
1365 | .device_feature = vfio_pci_core_ioctl_feature, |
1366 | .read = vfio_pci_core_read, |
1367 | .write = vfio_pci_core_write, |
1368 | .mmap = vfio_pci_core_mmap, |
1369 | .request = vfio_pci_core_request, |
1370 | .match = vfio_pci_core_match, |
1371 | .bind_iommufd = vfio_iommufd_physical_bind, |
1372 | .unbind_iommufd = vfio_iommufd_physical_unbind, |
1373 | .attach_ioas = vfio_iommufd_physical_attach_ioas, |
1374 | .detach_ioas = vfio_iommufd_physical_detach_ioas, |
1375 | }; |
1376 | |
1377 | static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
1378 | { |
1379 | struct hisi_acc_vf_core_device *hisi_acc_vdev; |
1380 | const struct vfio_device_ops *ops = &hisi_acc_vfio_pci_ops; |
1381 | struct hisi_qm *pf_qm; |
1382 | int vf_id; |
1383 | int ret; |
1384 | |
1385 | pf_qm = hisi_acc_get_pf_qm(pdev); |
1386 | if (pf_qm && pf_qm->ver >= QM_HW_V3) { |
1387 | vf_id = pci_iov_vf_id(dev: pdev); |
1388 | if (vf_id >= 0) |
1389 | ops = &hisi_acc_vfio_pci_migrn_ops; |
1390 | else |
1391 | pci_warn(pdev, "migration support failed, continue with generic interface\n" ); |
1392 | } |
1393 | |
1394 | hisi_acc_vdev = vfio_alloc_device(hisi_acc_vf_core_device, |
1395 | core_device.vdev, &pdev->dev, ops); |
1396 | if (IS_ERR(ptr: hisi_acc_vdev)) |
1397 | return PTR_ERR(ptr: hisi_acc_vdev); |
1398 | |
1399 | dev_set_drvdata(dev: &pdev->dev, data: &hisi_acc_vdev->core_device); |
1400 | ret = vfio_pci_core_register_device(vdev: &hisi_acc_vdev->core_device); |
1401 | if (ret) |
1402 | goto out_put_vdev; |
1403 | return 0; |
1404 | |
1405 | out_put_vdev: |
1406 | vfio_put_device(device: &hisi_acc_vdev->core_device.vdev); |
1407 | return ret; |
1408 | } |
1409 | |
1410 | static void hisi_acc_vfio_pci_remove(struct pci_dev *pdev) |
1411 | { |
1412 | struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); |
1413 | |
1414 | vfio_pci_core_unregister_device(vdev: &hisi_acc_vdev->core_device); |
1415 | vfio_put_device(device: &hisi_acc_vdev->core_device.vdev); |
1416 | } |
1417 | |
1418 | static const struct pci_device_id hisi_acc_vfio_pci_table[] = { |
1419 | { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) }, |
1420 | { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_HPRE_VF) }, |
1421 | { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) }, |
1422 | { } |
1423 | }; |
1424 | |
1425 | MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table); |
1426 | |
1427 | static const struct pci_error_handlers hisi_acc_vf_err_handlers = { |
1428 | .reset_done = hisi_acc_vf_pci_aer_reset_done, |
1429 | .error_detected = vfio_pci_core_aer_err_detected, |
1430 | }; |
1431 | |
1432 | static struct pci_driver hisi_acc_vfio_pci_driver = { |
1433 | .name = KBUILD_MODNAME, |
1434 | .id_table = hisi_acc_vfio_pci_table, |
1435 | .probe = hisi_acc_vfio_pci_probe, |
1436 | .remove = hisi_acc_vfio_pci_remove, |
1437 | .err_handler = &hisi_acc_vf_err_handlers, |
1438 | .driver_managed_dma = true, |
1439 | }; |
1440 | |
1441 | module_pci_driver(hisi_acc_vfio_pci_driver); |
1442 | |
1443 | MODULE_LICENSE("GPL v2" ); |
1444 | MODULE_AUTHOR("Liu Longfang <liulongfang@huawei.com>" ); |
1445 | MODULE_AUTHOR("Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>" ); |
1446 | MODULE_DESCRIPTION("HiSilicon VFIO PCI - VFIO PCI driver with live migration support for HiSilicon ACC device family" ); |
1447 | |