1 | /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ |
2 | /* |
3 | * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
4 | */ |
5 | |
6 | #ifndef MLX5_VFIO_CMD_H |
7 | #define MLX5_VFIO_CMD_H |
8 | |
9 | #include <linux/kernel.h> |
10 | #include <linux/vfio_pci_core.h> |
11 | #include <linux/mlx5/driver.h> |
12 | #include <linux/mlx5/vport.h> |
13 | #include <linux/mlx5/cq.h> |
14 | #include <linux/mlx5/qp.h> |
15 | |
16 | enum mlx5_vf_migf_state { |
17 | MLX5_MIGF_STATE_ERROR = 1, |
18 | MLX5_MIGF_STATE_PRE_COPY_ERROR, |
19 | MLX5_MIGF_STATE_PRE_COPY, |
20 | MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK, |
21 | MLX5_MIGF_STATE_COMPLETE, |
22 | }; |
23 | |
24 | enum mlx5_vf_load_state { |
25 | , |
26 | , |
27 | , |
28 | MLX5_VF_LOAD_STATE_PREP_IMAGE, |
29 | MLX5_VF_LOAD_STATE_READ_IMAGE, |
30 | MLX5_VF_LOAD_STATE_LOAD_IMAGE, |
31 | }; |
32 | |
33 | struct mlx5_vf_migration_tag_stop_copy_data { |
34 | __le64 stop_copy_size; |
35 | }; |
36 | |
37 | enum { |
38 | MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0, |
39 | = 1 << 0, |
40 | }; |
41 | |
42 | enum { |
43 | = 0, |
44 | = 1 << 0, |
45 | }; |
46 | |
47 | struct { |
48 | __le64 ; |
49 | /* For future use in case we may need to change the kernel protocol */ |
50 | __le32 ; /* Use mlx5_vf_migf_header_flags */ |
51 | __le32 ; /* Use mlx5_vf_migf_header_tag */ |
52 | __u8 []; /* Its size is given in the record_size */ |
53 | }; |
54 | |
55 | struct mlx5_vhca_data_buffer { |
56 | struct sg_append_table table; |
57 | loff_t start_pos; |
58 | u64 length; |
59 | u64 allocated_length; |
60 | u32 mkey; |
61 | enum dma_data_direction dma_dir; |
62 | u8 dmaed:1; |
63 | u8 stop_copy_chunk_num; |
64 | struct list_head buf_elm; |
65 | struct mlx5_vf_migration_file *migf; |
66 | /* Optimize mlx5vf_get_migration_page() for sequential access */ |
67 | struct scatterlist *last_offset_sg; |
68 | unsigned int sg_last_entry; |
69 | unsigned long last_offset; |
70 | }; |
71 | |
72 | struct mlx5vf_async_data { |
73 | struct mlx5_async_work cb_work; |
74 | struct work_struct work; |
75 | struct mlx5_vhca_data_buffer *buf; |
76 | struct mlx5_vhca_data_buffer *; |
77 | int status; |
78 | u8 stop_copy_chunk:1; |
79 | void *out; |
80 | }; |
81 | |
82 | struct mlx5vf_save_work_data { |
83 | struct mlx5_vf_migration_file *migf; |
84 | size_t next_required_umem_size; |
85 | struct work_struct work; |
86 | u8 chunk_num; |
87 | }; |
88 | |
89 | #define MAX_NUM_CHUNKS 2 |
90 | |
91 | struct mlx5_vf_migration_file { |
92 | struct file *filp; |
93 | struct mutex lock; |
94 | enum mlx5_vf_migf_state state; |
95 | |
96 | enum mlx5_vf_load_state load_state; |
97 | u32 pdn; |
98 | loff_t max_pos; |
99 | u64 record_size; |
100 | u32 record_tag; |
101 | u64 stop_copy_prep_size; |
102 | u64 pre_copy_initial_bytes; |
103 | size_t next_required_umem_size; |
104 | u8 num_ready_chunks; |
105 | /* Upon chunk mode preserve another set of buffers for stop_copy phase */ |
106 | struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS]; |
107 | struct mlx5_vhca_data_buffer *[MAX_NUM_CHUNKS]; |
108 | struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS]; |
109 | spinlock_t list_lock; |
110 | struct list_head buf_list; |
111 | struct list_head avail_list; |
112 | struct mlx5vf_pci_core_device *mvdev; |
113 | wait_queue_head_t poll_wait; |
114 | struct completion save_comp; |
115 | struct mlx5_async_ctx async_ctx; |
116 | struct mlx5vf_async_data async_data; |
117 | }; |
118 | |
119 | struct mlx5_vhca_cq_buf { |
120 | struct mlx5_frag_buf_ctrl fbc; |
121 | struct mlx5_frag_buf frag_buf; |
122 | int cqe_size; |
123 | int nent; |
124 | }; |
125 | |
126 | struct mlx5_vhca_cq { |
127 | struct mlx5_vhca_cq_buf buf; |
128 | struct mlx5_db db; |
129 | struct mlx5_core_cq mcq; |
130 | size_t ncqe; |
131 | }; |
132 | |
133 | struct mlx5_vhca_recv_buf { |
134 | u32 npages; |
135 | struct page **page_list; |
136 | dma_addr_t *dma_addrs; |
137 | u32 next_rq_offset; |
138 | u32 mkey; |
139 | }; |
140 | |
141 | struct mlx5_vhca_qp { |
142 | struct mlx5_frag_buf buf; |
143 | struct mlx5_db db; |
144 | struct mlx5_vhca_recv_buf recv_buf; |
145 | u32 tracked_page_size; |
146 | u32 max_msg_size; |
147 | u32 qpn; |
148 | struct { |
149 | unsigned int pc; |
150 | unsigned int cc; |
151 | unsigned int wqe_cnt; |
152 | __be32 *db; |
153 | struct mlx5_frag_buf_ctrl fbc; |
154 | } rq; |
155 | }; |
156 | |
157 | struct mlx5_vhca_page_tracker { |
158 | u32 id; |
159 | u32 pdn; |
160 | u8 is_err:1; |
161 | u8 object_changed:1; |
162 | struct mlx5_uars_page *uar; |
163 | struct mlx5_vhca_cq cq; |
164 | struct mlx5_vhca_qp *host_qp; |
165 | struct mlx5_vhca_qp *fw_qp; |
166 | struct mlx5_nb nb; |
167 | int status; |
168 | }; |
169 | |
170 | struct mlx5vf_pci_core_device { |
171 | struct vfio_pci_core_device core_device; |
172 | int vf_id; |
173 | u16 vhca_id; |
174 | u8 migrate_cap:1; |
175 | u8 deferred_reset:1; |
176 | u8 mdev_detach:1; |
177 | u8 log_active:1; |
178 | u8 chunk_mode:1; |
179 | struct completion tracker_comp; |
180 | /* protect migration state */ |
181 | struct mutex state_mutex; |
182 | enum vfio_device_mig_state mig_state; |
183 | /* protect the reset_done flow */ |
184 | spinlock_t reset_lock; |
185 | struct mlx5_vf_migration_file *resuming_migf; |
186 | struct mlx5_vf_migration_file *saving_migf; |
187 | struct mlx5_vhca_page_tracker tracker; |
188 | struct workqueue_struct *cb_wq; |
189 | struct notifier_block nb; |
190 | struct mlx5_core_dev *mdev; |
191 | }; |
192 | |
193 | enum { |
194 | MLX5VF_QUERY_INC = (1UL << 0), |
195 | MLX5VF_QUERY_FINAL = (1UL << 1), |
196 | MLX5VF_QUERY_CLEANUP = (1UL << 2), |
197 | }; |
198 | |
199 | int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); |
200 | int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); |
201 | int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev, |
202 | size_t *state_size, u64 *total_size, |
203 | u8 query_flags); |
204 | void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev, |
205 | const struct vfio_migration_ops *mig_ops, |
206 | const struct vfio_log_ops *log_ops); |
207 | void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev); |
208 | void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev); |
209 | int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev, |
210 | struct mlx5_vf_migration_file *migf, |
211 | struct mlx5_vhca_data_buffer *buf, bool inc, |
212 | bool track); |
213 | int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev, |
214 | struct mlx5_vf_migration_file *migf, |
215 | struct mlx5_vhca_data_buffer *buf); |
216 | int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf); |
217 | void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf); |
218 | void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf); |
219 | struct mlx5_vhca_data_buffer * |
220 | mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf, |
221 | size_t length, enum dma_data_direction dma_dir); |
222 | void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf); |
223 | struct mlx5_vhca_data_buffer * |
224 | mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf, |
225 | size_t length, enum dma_data_direction dma_dir); |
226 | void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf); |
227 | struct page *mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf, |
228 | unsigned long offset); |
229 | void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev); |
230 | void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev, |
231 | enum mlx5_vf_migf_state *last_save_state); |
232 | void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work); |
233 | void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf, |
234 | u8 chunk_num, size_t next_required_umem_size); |
235 | int mlx5vf_start_page_tracker(struct vfio_device *vdev, |
236 | struct rb_root_cached *ranges, u32 nnodes, u64 *page_size); |
237 | int mlx5vf_stop_page_tracker(struct vfio_device *vdev); |
238 | int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova, |
239 | unsigned long length, struct iova_bitmap *dirty); |
240 | #endif /* MLX5_VFIO_CMD_H */ |
241 | |