1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2023 Advanced Micro Devices, Inc. */ |
3 | |
4 | #include <linux/io.h> |
5 | #include <linux/types.h> |
6 | #include <linux/delay.h> |
7 | |
8 | #include <linux/pds/pds_common.h> |
9 | #include <linux/pds/pds_core_if.h> |
10 | #include <linux/pds/pds_adminq.h> |
11 | |
12 | #include "vfio_dev.h" |
13 | #include "cmds.h" |
14 | |
15 | #define SUSPEND_TIMEOUT_S 5 |
16 | #define SUSPEND_CHECK_INTERVAL_MS 1 |
17 | |
18 | static int pds_vfio_client_adminq_cmd(struct pds_vfio_pci_device *pds_vfio, |
19 | union pds_core_adminq_cmd *req, |
20 | union pds_core_adminq_comp *resp, |
21 | bool fast_poll) |
22 | { |
23 | struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio); |
24 | union pds_core_adminq_cmd cmd = {}; |
25 | struct pdsc *pdsc; |
26 | int err; |
27 | |
28 | /* Wrap the client request */ |
29 | cmd.client_request.opcode = PDS_AQ_CMD_CLIENT_CMD; |
30 | cmd.client_request.client_id = cpu_to_le16(pds_vfio->client_id); |
31 | memcpy(cmd.client_request.client_cmd, req, |
32 | sizeof(cmd.client_request.client_cmd)); |
33 | |
34 | pdsc = pdsc_get_pf_struct(vf_pdev: pdev); |
35 | if (IS_ERR(ptr: pdsc)) |
36 | return PTR_ERR(ptr: pdsc); |
37 | |
38 | err = pdsc_adminq_post(pdsc, cmd: &cmd, comp: resp, fast_poll); |
39 | if (err && err != -EAGAIN) |
40 | dev_err(pds_vfio_to_dev(pds_vfio), |
41 | "client admin cmd failed: %pe\n" , ERR_PTR(err)); |
42 | |
43 | return err; |
44 | } |
45 | |
46 | int pds_vfio_register_client_cmd(struct pds_vfio_pci_device *pds_vfio) |
47 | { |
48 | struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio); |
49 | char devname[PDS_DEVNAME_LEN]; |
50 | struct pdsc *pdsc; |
51 | int ci; |
52 | |
53 | snprintf(buf: devname, size: sizeof(devname), fmt: "%s.%d-%u" , PDS_VFIO_LM_DEV_NAME, |
54 | pci_domain_nr(bus: pdev->bus), |
55 | PCI_DEVID(pdev->bus->number, pdev->devfn)); |
56 | |
57 | pdsc = pdsc_get_pf_struct(vf_pdev: pdev); |
58 | if (IS_ERR(ptr: pdsc)) |
59 | return PTR_ERR(ptr: pdsc); |
60 | |
61 | ci = pds_client_register(pf: pdsc, devname); |
62 | if (ci < 0) |
63 | return ci; |
64 | |
65 | pds_vfio->client_id = ci; |
66 | |
67 | return 0; |
68 | } |
69 | |
70 | void pds_vfio_unregister_client_cmd(struct pds_vfio_pci_device *pds_vfio) |
71 | { |
72 | struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio); |
73 | struct pdsc *pdsc; |
74 | int err; |
75 | |
76 | pdsc = pdsc_get_pf_struct(vf_pdev: pdev); |
77 | if (IS_ERR(ptr: pdsc)) |
78 | return; |
79 | |
80 | err = pds_client_unregister(pf: pdsc, client_id: pds_vfio->client_id); |
81 | if (err) |
82 | dev_err(&pdev->dev, "unregister from DSC failed: %pe\n" , |
83 | ERR_PTR(err)); |
84 | |
85 | pds_vfio->client_id = 0; |
86 | } |
87 | |
88 | static int |
89 | pds_vfio_suspend_wait_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type) |
90 | { |
91 | union pds_core_adminq_cmd cmd = { |
92 | .lm_suspend_status = { |
93 | .opcode = PDS_LM_CMD_SUSPEND_STATUS, |
94 | .vf_id = cpu_to_le16(pds_vfio->vf_id), |
95 | .type = type, |
96 | }, |
97 | }; |
98 | struct device *dev = pds_vfio_to_dev(pds_vfio); |
99 | union pds_core_adminq_comp comp = {}; |
100 | unsigned long time_limit; |
101 | unsigned long time_start; |
102 | unsigned long time_done; |
103 | int err; |
104 | |
105 | time_start = jiffies; |
106 | time_limit = time_start + HZ * SUSPEND_TIMEOUT_S; |
107 | do { |
108 | err = pds_vfio_client_adminq_cmd(pds_vfio, req: &cmd, resp: &comp, fast_poll: true); |
109 | if (err != -EAGAIN) |
110 | break; |
111 | |
112 | msleep(SUSPEND_CHECK_INTERVAL_MS); |
113 | } while (time_before(jiffies, time_limit)); |
114 | |
115 | time_done = jiffies; |
116 | dev_dbg(dev, "%s: vf%u: Suspend comp received in %d msecs\n" , __func__, |
117 | pds_vfio->vf_id, jiffies_to_msecs(time_done - time_start)); |
118 | |
119 | /* Check the results */ |
120 | if (time_after_eq(time_done, time_limit)) { |
121 | dev_err(dev, "%s: vf%u: Suspend comp timeout\n" , __func__, |
122 | pds_vfio->vf_id); |
123 | err = -ETIMEDOUT; |
124 | } |
125 | |
126 | return err; |
127 | } |
128 | |
129 | int pds_vfio_suspend_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type) |
130 | { |
131 | union pds_core_adminq_cmd cmd = { |
132 | .lm_suspend = { |
133 | .opcode = PDS_LM_CMD_SUSPEND, |
134 | .vf_id = cpu_to_le16(pds_vfio->vf_id), |
135 | .type = type, |
136 | }, |
137 | }; |
138 | struct device *dev = pds_vfio_to_dev(pds_vfio); |
139 | union pds_core_adminq_comp comp = {}; |
140 | int err; |
141 | |
142 | dev_dbg(dev, "vf%u: Suspend device\n" , pds_vfio->vf_id); |
143 | |
144 | /* |
145 | * The initial suspend request to the firmware starts the device suspend |
146 | * operation and the firmware returns success if it's started |
147 | * successfully. |
148 | */ |
149 | err = pds_vfio_client_adminq_cmd(pds_vfio, req: &cmd, resp: &comp, fast_poll: true); |
150 | if (err) { |
151 | dev_err(dev, "vf%u: Suspend failed: %pe\n" , pds_vfio->vf_id, |
152 | ERR_PTR(err)); |
153 | return err; |
154 | } |
155 | |
156 | /* |
157 | * The subsequent suspend status request(s) check if the firmware has |
158 | * completed the device suspend process. |
159 | */ |
160 | return pds_vfio_suspend_wait_device_cmd(pds_vfio, type); |
161 | } |
162 | |
163 | int pds_vfio_resume_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type) |
164 | { |
165 | union pds_core_adminq_cmd cmd = { |
166 | .lm_resume = { |
167 | .opcode = PDS_LM_CMD_RESUME, |
168 | .vf_id = cpu_to_le16(pds_vfio->vf_id), |
169 | .type = type, |
170 | }, |
171 | }; |
172 | struct device *dev = pds_vfio_to_dev(pds_vfio); |
173 | union pds_core_adminq_comp comp = {}; |
174 | |
175 | dev_dbg(dev, "vf%u: Resume device\n" , pds_vfio->vf_id); |
176 | |
177 | return pds_vfio_client_adminq_cmd(pds_vfio, req: &cmd, resp: &comp, fast_poll: true); |
178 | } |
179 | |
180 | int pds_vfio_get_lm_state_size_cmd(struct pds_vfio_pci_device *pds_vfio, u64 *size) |
181 | { |
182 | union pds_core_adminq_cmd cmd = { |
183 | .lm_state_size = { |
184 | .opcode = PDS_LM_CMD_STATE_SIZE, |
185 | .vf_id = cpu_to_le16(pds_vfio->vf_id), |
186 | }, |
187 | }; |
188 | struct device *dev = pds_vfio_to_dev(pds_vfio); |
189 | union pds_core_adminq_comp comp = {}; |
190 | int err; |
191 | |
192 | dev_dbg(dev, "vf%u: Get migration status\n" , pds_vfio->vf_id); |
193 | |
194 | err = pds_vfio_client_adminq_cmd(pds_vfio, req: &cmd, resp: &comp, fast_poll: false); |
195 | if (err) |
196 | return err; |
197 | |
198 | *size = le64_to_cpu(comp.lm_state_size.size); |
199 | return 0; |
200 | } |
201 | |
202 | static int pds_vfio_dma_map_lm_file(struct device *dev, |
203 | enum dma_data_direction dir, |
204 | struct pds_vfio_lm_file *lm_file) |
205 | { |
206 | struct pds_lm_sg_elem *sgl, *sge; |
207 | struct scatterlist *sg; |
208 | dma_addr_t sgl_addr; |
209 | size_t sgl_size; |
210 | int err; |
211 | int i; |
212 | |
213 | if (!lm_file) |
214 | return -EINVAL; |
215 | |
216 | /* dma map file pages */ |
217 | err = dma_map_sgtable(dev, sgt: &lm_file->sg_table, dir, attrs: 0); |
218 | if (err) |
219 | return err; |
220 | |
221 | lm_file->num_sge = lm_file->sg_table.nents; |
222 | |
223 | /* alloc sgl */ |
224 | sgl_size = lm_file->num_sge * sizeof(struct pds_lm_sg_elem); |
225 | sgl = kzalloc(size: sgl_size, GFP_KERNEL); |
226 | if (!sgl) { |
227 | err = -ENOMEM; |
228 | goto out_unmap_sgtable; |
229 | } |
230 | |
231 | /* fill sgl */ |
232 | sge = sgl; |
233 | for_each_sgtable_dma_sg(&lm_file->sg_table, sg, i) { |
234 | sge->addr = cpu_to_le64(sg_dma_address(sg)); |
235 | sge->len = cpu_to_le32(sg_dma_len(sg)); |
236 | dev_dbg(dev, "addr = %llx, len = %u\n" , sge->addr, sge->len); |
237 | sge++; |
238 | } |
239 | |
240 | sgl_addr = dma_map_single(dev, sgl, sgl_size, DMA_TO_DEVICE); |
241 | if (dma_mapping_error(dev, dma_addr: sgl_addr)) { |
242 | err = -EIO; |
243 | goto out_free_sgl; |
244 | } |
245 | |
246 | lm_file->sgl = sgl; |
247 | lm_file->sgl_addr = sgl_addr; |
248 | |
249 | return 0; |
250 | |
251 | out_free_sgl: |
252 | kfree(objp: sgl); |
253 | out_unmap_sgtable: |
254 | lm_file->num_sge = 0; |
255 | dma_unmap_sgtable(dev, sgt: &lm_file->sg_table, dir, attrs: 0); |
256 | return err; |
257 | } |
258 | |
259 | static void pds_vfio_dma_unmap_lm_file(struct device *dev, |
260 | enum dma_data_direction dir, |
261 | struct pds_vfio_lm_file *lm_file) |
262 | { |
263 | if (!lm_file) |
264 | return; |
265 | |
266 | /* free sgl */ |
267 | if (lm_file->sgl) { |
268 | dma_unmap_single(dev, lm_file->sgl_addr, |
269 | lm_file->num_sge * sizeof(*lm_file->sgl), |
270 | DMA_TO_DEVICE); |
271 | kfree(objp: lm_file->sgl); |
272 | lm_file->sgl = NULL; |
273 | lm_file->sgl_addr = DMA_MAPPING_ERROR; |
274 | lm_file->num_sge = 0; |
275 | } |
276 | |
277 | /* dma unmap file pages */ |
278 | dma_unmap_sgtable(dev, sgt: &lm_file->sg_table, dir, attrs: 0); |
279 | } |
280 | |
281 | int pds_vfio_get_lm_state_cmd(struct pds_vfio_pci_device *pds_vfio) |
282 | { |
283 | union pds_core_adminq_cmd cmd = { |
284 | .lm_save = { |
285 | .opcode = PDS_LM_CMD_SAVE, |
286 | .vf_id = cpu_to_le16(pds_vfio->vf_id), |
287 | }, |
288 | }; |
289 | struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio); |
290 | struct device *pdsc_dev = &pci_physfn(dev: pdev)->dev; |
291 | union pds_core_adminq_comp comp = {}; |
292 | struct pds_vfio_lm_file *lm_file; |
293 | int err; |
294 | |
295 | dev_dbg(&pdev->dev, "vf%u: Get migration state\n" , pds_vfio->vf_id); |
296 | |
297 | lm_file = pds_vfio->save_file; |
298 | |
299 | err = pds_vfio_dma_map_lm_file(dev: pdsc_dev, dir: DMA_FROM_DEVICE, lm_file); |
300 | if (err) { |
301 | dev_err(&pdev->dev, "failed to map save migration file: %pe\n" , |
302 | ERR_PTR(err)); |
303 | return err; |
304 | } |
305 | |
306 | cmd.lm_save.sgl_addr = cpu_to_le64(lm_file->sgl_addr); |
307 | cmd.lm_save.num_sge = cpu_to_le32(lm_file->num_sge); |
308 | |
309 | err = pds_vfio_client_adminq_cmd(pds_vfio, req: &cmd, resp: &comp, fast_poll: false); |
310 | if (err) |
311 | dev_err(&pdev->dev, "failed to get migration state: %pe\n" , |
312 | ERR_PTR(err)); |
313 | |
314 | pds_vfio_dma_unmap_lm_file(dev: pdsc_dev, dir: DMA_FROM_DEVICE, lm_file); |
315 | |
316 | return err; |
317 | } |
318 | |
319 | int pds_vfio_set_lm_state_cmd(struct pds_vfio_pci_device *pds_vfio) |
320 | { |
321 | union pds_core_adminq_cmd cmd = { |
322 | .lm_restore = { |
323 | .opcode = PDS_LM_CMD_RESTORE, |
324 | .vf_id = cpu_to_le16(pds_vfio->vf_id), |
325 | }, |
326 | }; |
327 | struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio); |
328 | struct device *pdsc_dev = &pci_physfn(dev: pdev)->dev; |
329 | union pds_core_adminq_comp comp = {}; |
330 | struct pds_vfio_lm_file *lm_file; |
331 | int err; |
332 | |
333 | dev_dbg(&pdev->dev, "vf%u: Set migration state\n" , pds_vfio->vf_id); |
334 | |
335 | lm_file = pds_vfio->restore_file; |
336 | |
337 | err = pds_vfio_dma_map_lm_file(dev: pdsc_dev, dir: DMA_TO_DEVICE, lm_file); |
338 | if (err) { |
339 | dev_err(&pdev->dev, |
340 | "failed to map restore migration file: %pe\n" , |
341 | ERR_PTR(err)); |
342 | return err; |
343 | } |
344 | |
345 | cmd.lm_restore.sgl_addr = cpu_to_le64(lm_file->sgl_addr); |
346 | cmd.lm_restore.num_sge = cpu_to_le32(lm_file->num_sge); |
347 | |
348 | err = pds_vfio_client_adminq_cmd(pds_vfio, req: &cmd, resp: &comp, fast_poll: false); |
349 | if (err) |
350 | dev_err(&pdev->dev, "failed to set migration state: %pe\n" , |
351 | ERR_PTR(err)); |
352 | |
353 | pds_vfio_dma_unmap_lm_file(dev: pdsc_dev, dir: DMA_TO_DEVICE, lm_file); |
354 | |
355 | return err; |
356 | } |
357 | |
358 | void pds_vfio_send_host_vf_lm_status_cmd(struct pds_vfio_pci_device *pds_vfio, |
359 | enum pds_lm_host_vf_status vf_status) |
360 | { |
361 | union pds_core_adminq_cmd cmd = { |
362 | .lm_host_vf_status = { |
363 | .opcode = PDS_LM_CMD_HOST_VF_STATUS, |
364 | .vf_id = cpu_to_le16(pds_vfio->vf_id), |
365 | .status = vf_status, |
366 | }, |
367 | }; |
368 | struct device *dev = pds_vfio_to_dev(pds_vfio); |
369 | union pds_core_adminq_comp comp = {}; |
370 | int err; |
371 | |
372 | dev_dbg(dev, "vf%u: Set host VF LM status: %u" , pds_vfio->vf_id, |
373 | vf_status); |
374 | if (vf_status != PDS_LM_STA_IN_PROGRESS && |
375 | vf_status != PDS_LM_STA_NONE) { |
376 | dev_warn(dev, "Invalid host VF migration status, %d\n" , |
377 | vf_status); |
378 | return; |
379 | } |
380 | |
381 | err = pds_vfio_client_adminq_cmd(pds_vfio, req: &cmd, resp: &comp, fast_poll: false); |
382 | if (err) |
383 | dev_warn(dev, "failed to send host VF migration status: %pe\n" , |
384 | ERR_PTR(err)); |
385 | } |
386 | |
387 | int pds_vfio_dirty_status_cmd(struct pds_vfio_pci_device *pds_vfio, |
388 | u64 regions_dma, u8 *max_regions, u8 *num_regions) |
389 | { |
390 | union pds_core_adminq_cmd cmd = { |
391 | .lm_dirty_status = { |
392 | .opcode = PDS_LM_CMD_DIRTY_STATUS, |
393 | .vf_id = cpu_to_le16(pds_vfio->vf_id), |
394 | }, |
395 | }; |
396 | struct device *dev = pds_vfio_to_dev(pds_vfio); |
397 | union pds_core_adminq_comp comp = {}; |
398 | int err; |
399 | |
400 | dev_dbg(dev, "vf%u: Dirty status\n" , pds_vfio->vf_id); |
401 | |
402 | cmd.lm_dirty_status.regions_dma = cpu_to_le64(regions_dma); |
403 | cmd.lm_dirty_status.max_regions = *max_regions; |
404 | |
405 | err = pds_vfio_client_adminq_cmd(pds_vfio, req: &cmd, resp: &comp, fast_poll: false); |
406 | if (err) { |
407 | dev_err(dev, "failed to get dirty status: %pe\n" , ERR_PTR(err)); |
408 | return err; |
409 | } |
410 | |
411 | /* only support seq_ack approach for now */ |
412 | if (!(le32_to_cpu(comp.lm_dirty_status.bmp_type_mask) & |
413 | BIT(PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK))) { |
414 | dev_err(dev, "Dirty bitmap tracking SEQ_ACK not supported\n" ); |
415 | return -EOPNOTSUPP; |
416 | } |
417 | |
418 | *num_regions = comp.lm_dirty_status.num_regions; |
419 | *max_regions = comp.lm_dirty_status.max_regions; |
420 | |
421 | dev_dbg(dev, |
422 | "Page Tracking Status command successful, max_regions: %d, num_regions: %d, bmp_type: %s\n" , |
423 | *max_regions, *num_regions, "PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK" ); |
424 | |
425 | return 0; |
426 | } |
427 | |
428 | int pds_vfio_dirty_enable_cmd(struct pds_vfio_pci_device *pds_vfio, |
429 | u64 regions_dma, u8 num_regions) |
430 | { |
431 | union pds_core_adminq_cmd cmd = { |
432 | .lm_dirty_enable = { |
433 | .opcode = PDS_LM_CMD_DIRTY_ENABLE, |
434 | .vf_id = cpu_to_le16(pds_vfio->vf_id), |
435 | .regions_dma = cpu_to_le64(regions_dma), |
436 | .bmp_type = PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK, |
437 | .num_regions = num_regions, |
438 | }, |
439 | }; |
440 | struct device *dev = pds_vfio_to_dev(pds_vfio); |
441 | union pds_core_adminq_comp comp = {}; |
442 | int err; |
443 | |
444 | err = pds_vfio_client_adminq_cmd(pds_vfio, req: &cmd, resp: &comp, fast_poll: false); |
445 | if (err) { |
446 | dev_err(dev, "failed dirty tracking enable: %pe\n" , |
447 | ERR_PTR(err)); |
448 | return err; |
449 | } |
450 | |
451 | return 0; |
452 | } |
453 | |
454 | int pds_vfio_dirty_disable_cmd(struct pds_vfio_pci_device *pds_vfio) |
455 | { |
456 | union pds_core_adminq_cmd cmd = { |
457 | .lm_dirty_disable = { |
458 | .opcode = PDS_LM_CMD_DIRTY_DISABLE, |
459 | .vf_id = cpu_to_le16(pds_vfio->vf_id), |
460 | }, |
461 | }; |
462 | struct device *dev = pds_vfio_to_dev(pds_vfio); |
463 | union pds_core_adminq_comp comp = {}; |
464 | int err; |
465 | |
466 | err = pds_vfio_client_adminq_cmd(pds_vfio, req: &cmd, resp: &comp, fast_poll: false); |
467 | if (err || comp.lm_dirty_status.num_regions != 0) { |
468 | /* in case num_regions is still non-zero after disable */ |
469 | err = err ? err : -EIO; |
470 | dev_err(dev, |
471 | "failed dirty tracking disable: %pe, num_regions %d\n" , |
472 | ERR_PTR(err), comp.lm_dirty_status.num_regions); |
473 | return err; |
474 | } |
475 | |
476 | return 0; |
477 | } |
478 | |
479 | int pds_vfio_dirty_seq_ack_cmd(struct pds_vfio_pci_device *pds_vfio, |
480 | u64 sgl_dma, u16 num_sge, u32 offset, |
481 | u32 total_len, bool read_seq) |
482 | { |
483 | const char *cmd_type_str = read_seq ? "read_seq" : "write_ack" ; |
484 | union pds_core_adminq_cmd cmd = { |
485 | .lm_dirty_seq_ack = { |
486 | .vf_id = cpu_to_le16(pds_vfio->vf_id), |
487 | .len_bytes = cpu_to_le32(total_len), |
488 | .off_bytes = cpu_to_le32(offset), |
489 | .sgl_addr = cpu_to_le64(sgl_dma), |
490 | .num_sge = cpu_to_le16(num_sge), |
491 | }, |
492 | }; |
493 | struct device *dev = pds_vfio_to_dev(pds_vfio); |
494 | union pds_core_adminq_comp comp = {}; |
495 | int err; |
496 | |
497 | if (read_seq) |
498 | cmd.lm_dirty_seq_ack.opcode = PDS_LM_CMD_DIRTY_READ_SEQ; |
499 | else |
500 | cmd.lm_dirty_seq_ack.opcode = PDS_LM_CMD_DIRTY_WRITE_ACK; |
501 | |
502 | err = pds_vfio_client_adminq_cmd(pds_vfio, req: &cmd, resp: &comp, fast_poll: false); |
503 | if (err) { |
504 | dev_err(dev, "failed cmd Page Tracking %s: %pe\n" , cmd_type_str, |
505 | ERR_PTR(err)); |
506 | return err; |
507 | } |
508 | |
509 | return 0; |
510 | } |
511 | |