1 | /* |
2 | * Copyright (c) 2014, Mellanox Technologies inc. All rights reserved. |
3 | * |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU |
6 | * General Public License (GPL) Version 2, available from the file |
7 | * COPYING in the main directory of this source tree, or the |
8 | * OpenIB.org BSD license below: |
9 | * |
10 | * Redistribution and use in source and binary forms, with or |
11 | * without modification, are permitted provided that the following |
12 | * conditions are met: |
13 | * |
14 | * - Redistributions of source code must retain the above |
15 | * copyright notice, this list of conditions and the following |
16 | * disclaimer. |
17 | * |
18 | * - Redistributions in binary form must reproduce the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer in the documentation and/or other materials |
21 | * provided with the distribution. |
22 | * |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
30 | * SOFTWARE. |
31 | */ |
32 | |
33 | #include <linux/pci.h> |
34 | #include <linux/mlx5/driver.h> |
35 | #include <linux/mlx5/vport.h> |
36 | #include "mlx5_core.h" |
37 | #include "mlx5_irq.h" |
38 | #include "eswitch.h" |
39 | |
40 | static int sriov_restore_guids(struct mlx5_core_dev *dev, int vf, u16 func_id) |
41 | { |
42 | struct mlx5_core_sriov *sriov = &dev->priv.sriov; |
43 | struct mlx5_hca_vport_context *in; |
44 | int err = 0; |
45 | |
46 | /* Restore sriov guid and policy settings */ |
47 | if (sriov->vfs_ctx[vf].node_guid || |
48 | sriov->vfs_ctx[vf].port_guid || |
49 | sriov->vfs_ctx[vf].policy != MLX5_POLICY_INVALID) { |
50 | in = kzalloc(size: sizeof(*in), GFP_KERNEL); |
51 | if (!in) |
52 | return -ENOMEM; |
53 | |
54 | in->node_guid = sriov->vfs_ctx[vf].node_guid; |
55 | in->port_guid = sriov->vfs_ctx[vf].port_guid; |
56 | in->policy = sriov->vfs_ctx[vf].policy; |
57 | in->field_select = |
58 | !!(in->port_guid) * MLX5_HCA_VPORT_SEL_PORT_GUID | |
59 | !!(in->node_guid) * MLX5_HCA_VPORT_SEL_NODE_GUID | |
60 | !!(in->policy) * MLX5_HCA_VPORT_SEL_STATE_POLICY; |
61 | |
62 | err = mlx5_core_modify_hca_vport_context(dev, other_vport: 1, port_num: 1, vf: func_id, req: in); |
63 | if (err) |
64 | mlx5_core_warn(dev, "modify vport context failed, unable to restore VF %d settings\n" , vf); |
65 | |
66 | kfree(objp: in); |
67 | } |
68 | |
69 | return err; |
70 | } |
71 | |
72 | static int mlx5_device_enable_sriov(struct mlx5_core_dev *dev, int num_vfs) |
73 | { |
74 | struct mlx5_core_sriov *sriov = &dev->priv.sriov; |
75 | int err, vf, num_msix_count; |
76 | int vport_num; |
77 | |
78 | err = mlx5_eswitch_enable(esw: dev->priv.eswitch, num_vfs); |
79 | if (err) { |
80 | mlx5_core_warn(dev, |
81 | "failed to enable eswitch SRIOV (%d)\n" , err); |
82 | return err; |
83 | } |
84 | |
85 | num_msix_count = mlx5_get_default_msix_vec_count(dev, num_vfs); |
86 | for (vf = 0; vf < num_vfs; vf++) { |
87 | /* Notify the VF before its enablement to let it set |
88 | * some stuff. |
89 | */ |
90 | blocking_notifier_call_chain(nh: &sriov->vfs_ctx[vf].notifier, |
91 | val: MLX5_PF_NOTIFY_ENABLE_VF, v: dev); |
92 | err = mlx5_core_enable_hca(dev, func_id: vf + 1); |
93 | if (err) { |
94 | mlx5_core_warn(dev, "failed to enable VF %d (%d)\n" , vf, err); |
95 | continue; |
96 | } |
97 | |
98 | err = mlx5_set_msix_vec_count(dev, devfn: vf + 1, msix_vec_count: num_msix_count); |
99 | if (err) { |
100 | mlx5_core_warn(dev, |
101 | "failed to set MSI-X vector counts VF %d, err %d\n" , |
102 | vf, err); |
103 | continue; |
104 | } |
105 | |
106 | sriov->vfs_ctx[vf].enabled = 1; |
107 | if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) { |
108 | vport_num = mlx5_core_ec_sriov_enabled(dev) ? |
109 | mlx5_core_ec_vf_vport_base(dev) + vf |
110 | : vf + 1; |
111 | err = sriov_restore_guids(dev, vf, func_id: vport_num); |
112 | if (err) { |
113 | mlx5_core_warn(dev, |
114 | "failed to restore VF %d settings, err %d\n" , |
115 | vf, err); |
116 | continue; |
117 | } |
118 | } |
119 | mlx5_core_dbg(dev, "successfully enabled VF* %d\n" , vf); |
120 | } |
121 | |
122 | return 0; |
123 | } |
124 | |
125 | static void |
126 | mlx5_device_disable_sriov(struct mlx5_core_dev *dev, int num_vfs, bool clear_vf, bool num_vf_change) |
127 | { |
128 | struct mlx5_core_sriov *sriov = &dev->priv.sriov; |
129 | bool wait_for_ec_vf_pages = true; |
130 | bool wait_for_vf_pages = true; |
131 | int err; |
132 | int vf; |
133 | |
134 | for (vf = num_vfs - 1; vf >= 0; vf--) { |
135 | if (!sriov->vfs_ctx[vf].enabled) |
136 | continue; |
137 | /* Notify the VF before its disablement to let it clean |
138 | * some resources. |
139 | */ |
140 | blocking_notifier_call_chain(nh: &sriov->vfs_ctx[vf].notifier, |
141 | val: MLX5_PF_NOTIFY_DISABLE_VF, v: dev); |
142 | err = mlx5_core_disable_hca(dev, func_id: vf + 1); |
143 | if (err) { |
144 | mlx5_core_warn(dev, "failed to disable VF %d\n" , vf); |
145 | continue; |
146 | } |
147 | sriov->vfs_ctx[vf].enabled = 0; |
148 | } |
149 | |
150 | mlx5_eswitch_disable_sriov(esw: dev->priv.eswitch, clear_vf); |
151 | |
152 | /* There are a number of scenarios when SRIOV is being disabled: |
153 | * 1. VFs or ECVFs had been created, and now set back to 0 (num_vf_change == true). |
154 | * - If EC SRIOV is enabled then this flow is happening on the |
155 | * embedded platform, wait for only EC VF pages. |
156 | * - If EC SRIOV is not enabled this flow is happening on non-embedded |
157 | * platform, wait for the VF pages. |
158 | * |
159 | * 2. The driver is being unloaded. In this case wait for all pages. |
160 | */ |
161 | if (num_vf_change) { |
162 | if (mlx5_core_ec_sriov_enabled(dev)) |
163 | wait_for_vf_pages = false; |
164 | else |
165 | wait_for_ec_vf_pages = false; |
166 | } |
167 | |
168 | if (wait_for_ec_vf_pages && mlx5_wait_for_pages(dev, pages: &dev->priv.page_counters[MLX5_EC_VF])) |
169 | mlx5_core_warn(dev, "timeout reclaiming EC VFs pages\n" ); |
170 | |
171 | /* For ECPFs, skip waiting for host VF pages until ECPF is destroyed */ |
172 | if (mlx5_core_is_ecpf(dev)) |
173 | return; |
174 | |
175 | if (wait_for_vf_pages && mlx5_wait_for_pages(dev, pages: &dev->priv.page_counters[MLX5_VF])) |
176 | mlx5_core_warn(dev, "timeout reclaiming VFs pages\n" ); |
177 | } |
178 | |
179 | static int mlx5_sriov_enable(struct pci_dev *pdev, int num_vfs) |
180 | { |
181 | struct mlx5_core_dev *dev = pci_get_drvdata(pdev); |
182 | struct devlink *devlink = priv_to_devlink(priv: dev); |
183 | int err; |
184 | |
185 | devl_lock(devlink); |
186 | err = mlx5_device_enable_sriov(dev, num_vfs); |
187 | devl_unlock(devlink); |
188 | if (err) { |
189 | mlx5_core_warn(dev, "mlx5_device_enable_sriov failed : %d\n" , err); |
190 | return err; |
191 | } |
192 | |
193 | err = pci_enable_sriov(dev: pdev, nr_virtfn: num_vfs); |
194 | if (err) { |
195 | mlx5_core_warn(dev, "pci_enable_sriov failed : %d\n" , err); |
196 | mlx5_device_disable_sriov(dev, num_vfs, clear_vf: true, num_vf_change: true); |
197 | } |
198 | return err; |
199 | } |
200 | |
201 | void mlx5_sriov_disable(struct pci_dev *pdev, bool num_vf_change) |
202 | { |
203 | struct mlx5_core_dev *dev = pci_get_drvdata(pdev); |
204 | struct devlink *devlink = priv_to_devlink(priv: dev); |
205 | int num_vfs = pci_num_vf(dev: dev->pdev); |
206 | |
207 | pci_disable_sriov(dev: pdev); |
208 | devl_lock(devlink); |
209 | mlx5_device_disable_sriov(dev, num_vfs, clear_vf: true, num_vf_change); |
210 | devl_unlock(devlink); |
211 | } |
212 | |
213 | int mlx5_core_sriov_configure(struct pci_dev *pdev, int num_vfs) |
214 | { |
215 | struct mlx5_core_dev *dev = pci_get_drvdata(pdev); |
216 | struct mlx5_core_sriov *sriov = &dev->priv.sriov; |
217 | int err = 0; |
218 | |
219 | mlx5_core_dbg(dev, "requested num_vfs %d\n" , num_vfs); |
220 | |
221 | if (num_vfs) |
222 | err = mlx5_sriov_enable(pdev, num_vfs); |
223 | else |
224 | mlx5_sriov_disable(pdev, num_vf_change: true); |
225 | |
226 | if (!err) |
227 | sriov->num_vfs = num_vfs; |
228 | return err ? err : num_vfs; |
229 | } |
230 | |
231 | int mlx5_core_sriov_set_msix_vec_count(struct pci_dev *vf, int msix_vec_count) |
232 | { |
233 | struct pci_dev *pf = pci_physfn(dev: vf); |
234 | struct mlx5_core_sriov *sriov; |
235 | struct mlx5_core_dev *dev; |
236 | int num_vf_msix, id; |
237 | |
238 | dev = pci_get_drvdata(pdev: pf); |
239 | num_vf_msix = MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix); |
240 | if (!num_vf_msix) |
241 | return -EOPNOTSUPP; |
242 | |
243 | if (!msix_vec_count) |
244 | msix_vec_count = |
245 | mlx5_get_default_msix_vec_count(dev, num_vfs: pci_num_vf(dev: pf)); |
246 | |
247 | sriov = &dev->priv.sriov; |
248 | id = pci_iov_vf_id(dev: vf); |
249 | if (id < 0 || !sriov->vfs_ctx[id].enabled) |
250 | return -EINVAL; |
251 | |
252 | return mlx5_set_msix_vec_count(dev, devfn: id + 1, msix_vec_count); |
253 | } |
254 | |
255 | int mlx5_sriov_attach(struct mlx5_core_dev *dev) |
256 | { |
257 | if (!mlx5_core_is_pf(dev) || !pci_num_vf(dev: dev->pdev)) |
258 | return 0; |
259 | |
260 | /* If sriov VFs exist in PCI level, enable them in device level */ |
261 | return mlx5_device_enable_sriov(dev, num_vfs: pci_num_vf(dev: dev->pdev)); |
262 | } |
263 | |
264 | void mlx5_sriov_detach(struct mlx5_core_dev *dev) |
265 | { |
266 | if (!mlx5_core_is_pf(dev)) |
267 | return; |
268 | |
269 | mlx5_device_disable_sriov(dev, num_vfs: pci_num_vf(dev: dev->pdev), clear_vf: false, num_vf_change: false); |
270 | } |
271 | |
272 | static u16 mlx5_get_max_vfs(struct mlx5_core_dev *dev) |
273 | { |
274 | u16 host_total_vfs; |
275 | const u32 *out; |
276 | |
277 | if (mlx5_core_is_ecpf_esw_manager(dev)) { |
278 | out = mlx5_esw_query_functions(dev); |
279 | |
280 | /* Old FW doesn't support getting total_vfs from esw func |
281 | * but supports getting it from pci_sriov. |
282 | */ |
283 | if (IS_ERR(ptr: out)) |
284 | goto done; |
285 | host_total_vfs = MLX5_GET(query_esw_functions_out, out, |
286 | host_params_context.host_total_vfs); |
287 | kvfree(addr: out); |
288 | return host_total_vfs; |
289 | } |
290 | |
291 | done: |
292 | return pci_sriov_get_totalvfs(dev: dev->pdev); |
293 | } |
294 | |
295 | int mlx5_sriov_init(struct mlx5_core_dev *dev) |
296 | { |
297 | struct mlx5_core_sriov *sriov = &dev->priv.sriov; |
298 | struct pci_dev *pdev = dev->pdev; |
299 | int total_vfs, i; |
300 | |
301 | if (!mlx5_core_is_pf(dev)) |
302 | return 0; |
303 | |
304 | total_vfs = pci_sriov_get_totalvfs(dev: pdev); |
305 | sriov->max_vfs = mlx5_get_max_vfs(dev); |
306 | sriov->num_vfs = pci_num_vf(dev: pdev); |
307 | sriov->max_ec_vfs = mlx5_core_ec_sriov_enabled(dev) ? pci_sriov_get_totalvfs(dev: dev->pdev) : 0; |
308 | sriov->vfs_ctx = kcalloc(n: total_vfs, size: sizeof(*sriov->vfs_ctx), GFP_KERNEL); |
309 | if (!sriov->vfs_ctx) |
310 | return -ENOMEM; |
311 | |
312 | for (i = 0; i < total_vfs; i++) |
313 | BLOCKING_INIT_NOTIFIER_HEAD(&sriov->vfs_ctx[i].notifier); |
314 | |
315 | return 0; |
316 | } |
317 | |
318 | void mlx5_sriov_cleanup(struct mlx5_core_dev *dev) |
319 | { |
320 | struct mlx5_core_sriov *sriov = &dev->priv.sriov; |
321 | |
322 | if (!mlx5_core_is_pf(dev)) |
323 | return; |
324 | |
325 | kfree(objp: sriov->vfs_ctx); |
326 | } |
327 | |
328 | /** |
329 | * mlx5_sriov_blocking_notifier_unregister - Unregister a VF from |
330 | * a notification block chain. |
331 | * |
332 | * @mdev: The mlx5 core device. |
333 | * @vf_id: The VF id. |
334 | * @nb: The notifier block to be unregistered. |
335 | */ |
336 | void mlx5_sriov_blocking_notifier_unregister(struct mlx5_core_dev *mdev, |
337 | int vf_id, |
338 | struct notifier_block *nb) |
339 | { |
340 | struct mlx5_vf_context *vfs_ctx; |
341 | struct mlx5_core_sriov *sriov; |
342 | |
343 | sriov = &mdev->priv.sriov; |
344 | if (WARN_ON(vf_id < 0 || vf_id >= sriov->num_vfs)) |
345 | return; |
346 | |
347 | vfs_ctx = &sriov->vfs_ctx[vf_id]; |
348 | blocking_notifier_chain_unregister(nh: &vfs_ctx->notifier, nb); |
349 | } |
350 | EXPORT_SYMBOL(mlx5_sriov_blocking_notifier_unregister); |
351 | |
352 | /** |
353 | * mlx5_sriov_blocking_notifier_register - Register a VF notification |
354 | * block chain. |
355 | * |
356 | * @mdev: The mlx5 core device. |
357 | * @vf_id: The VF id. |
358 | * @nb: The notifier block to be called upon the VF events. |
359 | * |
360 | * Returns 0 on success or an error code. |
361 | */ |
362 | int mlx5_sriov_blocking_notifier_register(struct mlx5_core_dev *mdev, |
363 | int vf_id, |
364 | struct notifier_block *nb) |
365 | { |
366 | struct mlx5_vf_context *vfs_ctx; |
367 | struct mlx5_core_sriov *sriov; |
368 | |
369 | sriov = &mdev->priv.sriov; |
370 | if (vf_id < 0 || vf_id >= sriov->num_vfs) |
371 | return -EINVAL; |
372 | |
373 | vfs_ctx = &sriov->vfs_ctx[vf_id]; |
374 | return blocking_notifier_chain_register(nh: &vfs_ctx->notifier, nb); |
375 | } |
376 | EXPORT_SYMBOL(mlx5_sriov_blocking_notifier_register); |
377 | |