1 | /* |
2 | * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. |
3 | * |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU |
6 | * General Public License (GPL) Version 2, available from the file |
7 | * COPYING in the main directory of this source tree, or the |
8 | * OpenIB.org BSD license below: |
9 | * |
10 | * Redistribution and use in source and binary forms, with or |
11 | * without modification, are permitted provided that the following |
12 | * conditions are met: |
13 | * |
14 | * - Redistributions of source code must retain the above |
15 | * copyright notice, this list of conditions and the following |
16 | * disclaimer. |
17 | * |
18 | * - Redistributions in binary form must reproduce the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer in the documentation and/or other materials |
21 | * provided with the distribution. |
22 | * |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
30 | * SOFTWARE. |
31 | */ |
32 | |
33 | #include <linux/highmem.h> |
34 | #include <linux/kernel.h> |
35 | #include <linux/delay.h> |
36 | #include <linux/mlx5/driver.h> |
37 | #include <linux/xarray.h> |
38 | #include "mlx5_core.h" |
39 | #include "lib/eq.h" |
40 | #include "lib/tout.h" |
41 | |
42 | enum { |
43 | MLX5_PAGES_CANT_GIVE = 0, |
44 | MLX5_PAGES_GIVE = 1, |
45 | MLX5_PAGES_TAKE = 2 |
46 | }; |
47 | |
48 | struct mlx5_pages_req { |
49 | struct mlx5_core_dev *dev; |
50 | u16 func_id; |
51 | u8 ec_function; |
52 | s32 npages; |
53 | struct work_struct work; |
54 | u8 release_all; |
55 | }; |
56 | |
57 | struct fw_page { |
58 | struct rb_node rb_node; |
59 | u64 addr; |
60 | struct page *page; |
61 | u32 function; |
62 | unsigned long bitmask; |
63 | struct list_head list; |
64 | unsigned int free_count; |
65 | }; |
66 | |
67 | enum { |
68 | MLX5_MAX_RECLAIM_TIME_MILI = 5000, |
69 | MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE, |
70 | }; |
71 | |
72 | static u32 get_function(u16 func_id, bool ec_function) |
73 | { |
74 | return (u32)func_id | (ec_function << 16); |
75 | } |
76 | |
77 | static u16 func_id_to_type(struct mlx5_core_dev *dev, u16 func_id, bool ec_function) |
78 | { |
79 | if (!func_id) |
80 | return mlx5_core_is_ecpf(dev) && !ec_function ? MLX5_HOST_PF : MLX5_PF; |
81 | |
82 | if (func_id <= max(mlx5_core_max_vfs(dev), mlx5_core_max_ec_vfs(dev))) { |
83 | if (ec_function) |
84 | return MLX5_EC_VF; |
85 | else |
86 | return MLX5_VF; |
87 | } |
88 | return MLX5_SF; |
89 | } |
90 | |
91 | static u32 mlx5_get_ec_function(u32 function) |
92 | { |
93 | return function >> 16; |
94 | } |
95 | |
96 | static u32 mlx5_get_func_id(u32 function) |
97 | { |
98 | return function & 0xffff; |
99 | } |
100 | |
101 | static struct rb_root *page_root_per_function(struct mlx5_core_dev *dev, u32 function) |
102 | { |
103 | struct rb_root *root; |
104 | int err; |
105 | |
106 | root = xa_load(&dev->priv.page_root_xa, index: function); |
107 | if (root) |
108 | return root; |
109 | |
110 | root = kzalloc(size: sizeof(*root), GFP_KERNEL); |
111 | if (!root) |
112 | return ERR_PTR(error: -ENOMEM); |
113 | |
114 | err = xa_insert(xa: &dev->priv.page_root_xa, index: function, entry: root, GFP_KERNEL); |
115 | if (err) { |
116 | kfree(objp: root); |
117 | return ERR_PTR(error: err); |
118 | } |
119 | |
120 | *root = RB_ROOT; |
121 | |
122 | return root; |
123 | } |
124 | |
125 | static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u32 function) |
126 | { |
127 | struct rb_node *parent = NULL; |
128 | struct rb_root *root; |
129 | struct rb_node **new; |
130 | struct fw_page *nfp; |
131 | struct fw_page *tfp; |
132 | int i; |
133 | |
134 | root = page_root_per_function(dev, function); |
135 | if (IS_ERR(ptr: root)) |
136 | return PTR_ERR(ptr: root); |
137 | |
138 | new = &root->rb_node; |
139 | |
140 | while (*new) { |
141 | parent = *new; |
142 | tfp = rb_entry(parent, struct fw_page, rb_node); |
143 | if (tfp->addr < addr) |
144 | new = &parent->rb_left; |
145 | else if (tfp->addr > addr) |
146 | new = &parent->rb_right; |
147 | else |
148 | return -EEXIST; |
149 | } |
150 | |
151 | nfp = kzalloc(size: sizeof(*nfp), GFP_KERNEL); |
152 | if (!nfp) |
153 | return -ENOMEM; |
154 | |
155 | nfp->addr = addr; |
156 | nfp->page = page; |
157 | nfp->function = function; |
158 | nfp->free_count = MLX5_NUM_4K_IN_PAGE; |
159 | for (i = 0; i < MLX5_NUM_4K_IN_PAGE; i++) |
160 | set_bit(nr: i, addr: &nfp->bitmask); |
161 | |
162 | rb_link_node(node: &nfp->rb_node, parent, rb_link: new); |
163 | rb_insert_color(&nfp->rb_node, root); |
164 | list_add(new: &nfp->list, head: &dev->priv.free_list); |
165 | |
166 | return 0; |
167 | } |
168 | |
169 | static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr, |
170 | u32 function) |
171 | { |
172 | struct fw_page *result = NULL; |
173 | struct rb_root *root; |
174 | struct rb_node *tmp; |
175 | struct fw_page *tfp; |
176 | |
177 | root = xa_load(&dev->priv.page_root_xa, index: function); |
178 | if (WARN_ON_ONCE(!root)) |
179 | return NULL; |
180 | |
181 | tmp = root->rb_node; |
182 | |
183 | while (tmp) { |
184 | tfp = rb_entry(tmp, struct fw_page, rb_node); |
185 | if (tfp->addr < addr) { |
186 | tmp = tmp->rb_left; |
187 | } else if (tfp->addr > addr) { |
188 | tmp = tmp->rb_right; |
189 | } else { |
190 | result = tfp; |
191 | break; |
192 | } |
193 | } |
194 | |
195 | return result; |
196 | } |
197 | |
198 | static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, |
199 | s32 *npages, int boot) |
200 | { |
201 | u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {}; |
202 | u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {}; |
203 | int err; |
204 | |
205 | MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES); |
206 | MLX5_SET(query_pages_in, in, op_mod, boot ? |
207 | MLX5_QUERY_PAGES_IN_OP_MOD_BOOT_PAGES : |
208 | MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES); |
209 | MLX5_SET(query_pages_in, in, embedded_cpu_function, mlx5_core_is_ecpf(dev)); |
210 | |
211 | err = mlx5_cmd_exec_inout(dev, query_pages, in, out); |
212 | if (err) |
213 | return err; |
214 | |
215 | *npages = MLX5_GET(query_pages_out, out, num_pages); |
216 | *func_id = MLX5_GET(query_pages_out, out, function_id); |
217 | |
218 | return err; |
219 | } |
220 | |
221 | static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr, u32 function) |
222 | { |
223 | struct fw_page *fp = NULL; |
224 | struct fw_page *iter; |
225 | unsigned n; |
226 | |
227 | list_for_each_entry(iter, &dev->priv.free_list, list) { |
228 | if (iter->function != function) |
229 | continue; |
230 | fp = iter; |
231 | } |
232 | |
233 | if (list_empty(head: &dev->priv.free_list) || !fp) |
234 | return -ENOMEM; |
235 | |
236 | n = find_first_bit(addr: &fp->bitmask, size: 8 * sizeof(fp->bitmask)); |
237 | if (n >= MLX5_NUM_4K_IN_PAGE) { |
238 | mlx5_core_warn(dev, "alloc 4k bug: fw page = 0x%llx, n = %u, bitmask: %lu, max num of 4K pages: %d\n" , |
239 | fp->addr, n, fp->bitmask, MLX5_NUM_4K_IN_PAGE); |
240 | return -ENOENT; |
241 | } |
242 | clear_bit(nr: n, addr: &fp->bitmask); |
243 | fp->free_count--; |
244 | if (!fp->free_count) |
245 | list_del(entry: &fp->list); |
246 | |
247 | *addr = fp->addr + n * MLX5_ADAPTER_PAGE_SIZE; |
248 | |
249 | return 0; |
250 | } |
251 | |
252 | #define MLX5_U64_4K_PAGE_MASK ((~(u64)0U) << PAGE_SHIFT) |
253 | |
254 | static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp, |
255 | bool in_free_list) |
256 | { |
257 | struct rb_root *root; |
258 | |
259 | root = xa_load(&dev->priv.page_root_xa, index: fwp->function); |
260 | if (WARN_ON_ONCE(!root)) |
261 | return; |
262 | |
263 | rb_erase(&fwp->rb_node, root); |
264 | if (in_free_list) |
265 | list_del(entry: &fwp->list); |
266 | dma_unmap_page(mlx5_core_dma_dev(dev), fwp->addr & MLX5_U64_4K_PAGE_MASK, |
267 | PAGE_SIZE, DMA_BIDIRECTIONAL); |
268 | __free_page(fwp->page); |
269 | kfree(objp: fwp); |
270 | } |
271 | |
272 | static void free_4k(struct mlx5_core_dev *dev, u64 addr, u32 function) |
273 | { |
274 | struct fw_page *fwp; |
275 | int n; |
276 | |
277 | fwp = find_fw_page(dev, addr: addr & MLX5_U64_4K_PAGE_MASK, function); |
278 | if (!fwp) { |
279 | mlx5_core_warn_rl(dev, "page not found\n" ); |
280 | return; |
281 | } |
282 | n = (addr & ~MLX5_U64_4K_PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT; |
283 | fwp->free_count++; |
284 | set_bit(nr: n, addr: &fwp->bitmask); |
285 | if (fwp->free_count == MLX5_NUM_4K_IN_PAGE) |
286 | free_fwp(dev, fwp, in_free_list: fwp->free_count != 1); |
287 | else if (fwp->free_count == 1) |
288 | list_add(new: &fwp->list, head: &dev->priv.free_list); |
289 | } |
290 | |
291 | static int alloc_system_page(struct mlx5_core_dev *dev, u32 function) |
292 | { |
293 | struct device *device = mlx5_core_dma_dev(dev); |
294 | int nid = dev_to_node(dev: device); |
295 | struct page *page; |
296 | u64 zero_addr = 1; |
297 | u64 addr; |
298 | int err; |
299 | |
300 | page = alloc_pages_node(nid, GFP_HIGHUSER, order: 0); |
301 | if (!page) { |
302 | mlx5_core_warn(dev, "failed to allocate page\n" ); |
303 | return -ENOMEM; |
304 | } |
305 | map: |
306 | addr = dma_map_page(device, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); |
307 | if (dma_mapping_error(dev: device, dma_addr: addr)) { |
308 | mlx5_core_warn(dev, "failed dma mapping page\n" ); |
309 | err = -ENOMEM; |
310 | goto err_mapping; |
311 | } |
312 | |
313 | /* Firmware doesn't support page with physical address 0 */ |
314 | if (addr == 0) { |
315 | zero_addr = addr; |
316 | goto map; |
317 | } |
318 | |
319 | err = insert_page(dev, addr, page, function); |
320 | if (err) { |
321 | mlx5_core_err(dev, "failed to track allocated page\n" ); |
322 | dma_unmap_page(device, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); |
323 | } |
324 | |
325 | err_mapping: |
326 | if (err) |
327 | __free_page(page); |
328 | |
329 | if (zero_addr == 0) |
330 | dma_unmap_page(device, zero_addr, PAGE_SIZE, |
331 | DMA_BIDIRECTIONAL); |
332 | |
333 | return err; |
334 | } |
335 | |
336 | static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id, |
337 | bool ec_function) |
338 | { |
339 | u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {}; |
340 | int err; |
341 | |
342 | MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); |
343 | MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_CANT_GIVE); |
344 | MLX5_SET(manage_pages_in, in, function_id, func_id); |
345 | MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function); |
346 | |
347 | err = mlx5_cmd_exec_in(dev, manage_pages, in); |
348 | if (err) |
349 | mlx5_core_warn(dev, "page notify failed func_id(%d) err(%d)\n" , |
350 | func_id, err); |
351 | } |
352 | |
353 | static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, |
354 | int event, bool ec_function) |
355 | { |
356 | u32 function = get_function(func_id, ec_function); |
357 | u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0}; |
358 | int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); |
359 | int notify_fail = event; |
360 | u16 func_type; |
361 | u64 addr; |
362 | int err; |
363 | u32 *in; |
364 | int i; |
365 | |
366 | inlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_in, pas[0]); |
367 | in = kvzalloc(size: inlen, GFP_KERNEL); |
368 | if (!in) { |
369 | err = -ENOMEM; |
370 | mlx5_core_warn(dev, "vzalloc failed %d\n" , inlen); |
371 | goto out_free; |
372 | } |
373 | |
374 | for (i = 0; i < npages; i++) { |
375 | retry: |
376 | err = alloc_4k(dev, addr: &addr, function); |
377 | if (err) { |
378 | if (err == -ENOMEM) |
379 | err = alloc_system_page(dev, function); |
380 | if (err) { |
381 | dev->priv.fw_pages_alloc_failed += (npages - i); |
382 | goto out_4k; |
383 | } |
384 | |
385 | goto retry; |
386 | } |
387 | MLX5_ARRAY_SET64(manage_pages_in, in, pas, i, addr); |
388 | } |
389 | |
390 | MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); |
391 | MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_GIVE); |
392 | MLX5_SET(manage_pages_in, in, function_id, func_id); |
393 | MLX5_SET(manage_pages_in, in, input_num_entries, npages); |
394 | MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function); |
395 | |
396 | err = mlx5_cmd_do(dev, in, in_size: inlen, out, out_size: sizeof(out)); |
397 | if (err == -EREMOTEIO) { |
398 | notify_fail = 0; |
399 | /* if triggered by FW and failed by FW ignore */ |
400 | if (event) { |
401 | err = 0; |
402 | goto out_dropped; |
403 | } |
404 | } |
405 | err = mlx5_cmd_check(dev, err, in, out); |
406 | if (err) { |
407 | mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n" , |
408 | func_id, npages, err); |
409 | goto out_dropped; |
410 | } |
411 | |
412 | func_type = func_id_to_type(dev, func_id, ec_function); |
413 | dev->priv.page_counters[func_type] += npages; |
414 | dev->priv.fw_pages += npages; |
415 | |
416 | mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x, err %d\n" , |
417 | npages, ec_function, func_id, err); |
418 | |
419 | kvfree(addr: in); |
420 | return 0; |
421 | |
422 | out_dropped: |
423 | dev->priv.give_pages_dropped += npages; |
424 | out_4k: |
425 | for (i--; i >= 0; i--) |
426 | free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i]), function); |
427 | out_free: |
428 | kvfree(addr: in); |
429 | if (notify_fail) |
430 | page_notify_fail(dev, func_id, ec_function); |
431 | return err; |
432 | } |
433 | |
434 | static void release_all_pages(struct mlx5_core_dev *dev, u16 func_id, |
435 | bool ec_function) |
436 | { |
437 | u32 function = get_function(func_id, ec_function); |
438 | struct rb_root *root; |
439 | struct rb_node *p; |
440 | int npages = 0; |
441 | u16 func_type; |
442 | |
443 | root = xa_load(&dev->priv.page_root_xa, index: function); |
444 | if (WARN_ON_ONCE(!root)) |
445 | return; |
446 | |
447 | p = rb_first(root); |
448 | while (p) { |
449 | struct fw_page *fwp = rb_entry(p, struct fw_page, rb_node); |
450 | |
451 | p = rb_next(p); |
452 | npages += (MLX5_NUM_4K_IN_PAGE - fwp->free_count); |
453 | free_fwp(dev, fwp, in_free_list: fwp->free_count); |
454 | } |
455 | |
456 | func_type = func_id_to_type(dev, func_id, ec_function); |
457 | dev->priv.page_counters[func_type] -= npages; |
458 | dev->priv.fw_pages -= npages; |
459 | |
460 | mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x\n" , |
461 | npages, ec_function, func_id); |
462 | } |
463 | |
464 | static u32 fwp_fill_manage_pages_out(struct fw_page *fwp, u32 *out, u32 index, |
465 | u32 npages) |
466 | { |
467 | u32 pages_set = 0; |
468 | unsigned int n; |
469 | |
470 | for_each_clear_bit(n, &fwp->bitmask, MLX5_NUM_4K_IN_PAGE) { |
471 | MLX5_ARRAY_SET64(manage_pages_out, out, pas, index + pages_set, |
472 | fwp->addr + (n * MLX5_ADAPTER_PAGE_SIZE)); |
473 | pages_set++; |
474 | |
475 | if (!--npages) |
476 | break; |
477 | } |
478 | |
479 | return pages_set; |
480 | } |
481 | |
482 | static int reclaim_pages_cmd(struct mlx5_core_dev *dev, |
483 | u32 *in, int in_size, u32 *out, int out_size) |
484 | { |
485 | struct rb_root *root; |
486 | struct fw_page *fwp; |
487 | struct rb_node *p; |
488 | bool ec_function; |
489 | u32 func_id; |
490 | u32 npages; |
491 | u32 i = 0; |
492 | |
493 | if (!mlx5_cmd_is_down(dev)) |
494 | return mlx5_cmd_do(dev, in, in_size, out, out_size); |
495 | |
496 | /* No hard feelings, we want our pages back! */ |
497 | npages = MLX5_GET(manage_pages_in, in, input_num_entries); |
498 | func_id = MLX5_GET(manage_pages_in, in, function_id); |
499 | ec_function = MLX5_GET(manage_pages_in, in, embedded_cpu_function); |
500 | |
501 | root = xa_load(&dev->priv.page_root_xa, index: get_function(func_id, ec_function)); |
502 | if (WARN_ON_ONCE(!root)) |
503 | return -EEXIST; |
504 | |
505 | p = rb_first(root); |
506 | while (p && i < npages) { |
507 | fwp = rb_entry(p, struct fw_page, rb_node); |
508 | p = rb_next(p); |
509 | |
510 | i += fwp_fill_manage_pages_out(fwp, out, index: i, npages: npages - i); |
511 | } |
512 | |
513 | MLX5_SET(manage_pages_out, out, output_num_entries, i); |
514 | return 0; |
515 | } |
516 | |
517 | static int reclaim_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, |
518 | int *nclaimed, bool event, bool ec_function) |
519 | { |
520 | u32 function = get_function(func_id, ec_function); |
521 | int outlen = MLX5_ST_SZ_BYTES(manage_pages_out); |
522 | u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {}; |
523 | int num_claimed; |
524 | u16 func_type; |
525 | u32 *out; |
526 | int err; |
527 | int i; |
528 | |
529 | if (nclaimed) |
530 | *nclaimed = 0; |
531 | |
532 | outlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); |
533 | out = kvzalloc(size: outlen, GFP_KERNEL); |
534 | if (!out) |
535 | return -ENOMEM; |
536 | |
537 | MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); |
538 | MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_TAKE); |
539 | MLX5_SET(manage_pages_in, in, function_id, func_id); |
540 | MLX5_SET(manage_pages_in, in, input_num_entries, npages); |
541 | MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function); |
542 | |
543 | mlx5_core_dbg(dev, "func 0x%x, npages %d, outlen %d\n" , |
544 | func_id, npages, outlen); |
545 | err = reclaim_pages_cmd(dev, in, in_size: sizeof(in), out, out_size: outlen); |
546 | if (err) { |
547 | npages = MLX5_GET(manage_pages_in, in, input_num_entries); |
548 | dev->priv.reclaim_pages_discard += npages; |
549 | } |
550 | /* if triggered by FW event and failed by FW then ignore */ |
551 | if (event && err == -EREMOTEIO) { |
552 | err = 0; |
553 | goto out_free; |
554 | } |
555 | |
556 | err = mlx5_cmd_check(dev, err, in, out); |
557 | if (err) { |
558 | mlx5_core_err(dev, "failed reclaiming pages: err %d\n" , err); |
559 | goto out_free; |
560 | } |
561 | |
562 | num_claimed = MLX5_GET(manage_pages_out, out, output_num_entries); |
563 | if (num_claimed > npages) { |
564 | mlx5_core_warn(dev, "fw returned %d, driver asked %d => corruption\n" , |
565 | num_claimed, npages); |
566 | err = -EINVAL; |
567 | goto out_free; |
568 | } |
569 | |
570 | for (i = 0; i < num_claimed; i++) |
571 | free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i]), function); |
572 | |
573 | if (nclaimed) |
574 | *nclaimed = num_claimed; |
575 | |
576 | func_type = func_id_to_type(dev, func_id, ec_function); |
577 | dev->priv.page_counters[func_type] -= num_claimed; |
578 | dev->priv.fw_pages -= num_claimed; |
579 | |
580 | out_free: |
581 | kvfree(addr: out); |
582 | return err; |
583 | } |
584 | |
585 | static void pages_work_handler(struct work_struct *work) |
586 | { |
587 | struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); |
588 | struct mlx5_core_dev *dev = req->dev; |
589 | int err = 0; |
590 | |
591 | if (req->release_all) |
592 | release_all_pages(dev, func_id: req->func_id, ec_function: req->ec_function); |
593 | else if (req->npages < 0) |
594 | err = reclaim_pages(dev, func_id: req->func_id, npages: -1 * req->npages, NULL, |
595 | event: true, ec_function: req->ec_function); |
596 | else if (req->npages > 0) |
597 | err = give_pages(dev, func_id: req->func_id, npages: req->npages, event: 1, ec_function: req->ec_function); |
598 | |
599 | if (err) |
600 | mlx5_core_warn(dev, "%s fail %d\n" , |
601 | req->npages < 0 ? "reclaim" : "give" , err); |
602 | |
603 | kfree(objp: req); |
604 | } |
605 | |
606 | enum { |
607 | EC_FUNCTION_MASK = 0x8000, |
608 | RELEASE_ALL_PAGES_MASK = 0x4000, |
609 | }; |
610 | |
611 | static int req_pages_handler(struct notifier_block *nb, |
612 | unsigned long type, void *data) |
613 | { |
614 | struct mlx5_pages_req *req; |
615 | struct mlx5_core_dev *dev; |
616 | struct mlx5_priv *priv; |
617 | struct mlx5_eqe *eqe; |
618 | bool ec_function; |
619 | bool release_all; |
620 | u16 func_id; |
621 | s32 npages; |
622 | |
623 | priv = mlx5_nb_cof(nb, struct mlx5_priv, pg_nb); |
624 | dev = container_of(priv, struct mlx5_core_dev, priv); |
625 | eqe = data; |
626 | |
627 | func_id = be16_to_cpu(eqe->data.req_pages.func_id); |
628 | npages = be32_to_cpu(eqe->data.req_pages.num_pages); |
629 | ec_function = be16_to_cpu(eqe->data.req_pages.ec_function) & EC_FUNCTION_MASK; |
630 | release_all = be16_to_cpu(eqe->data.req_pages.ec_function) & |
631 | RELEASE_ALL_PAGES_MASK; |
632 | mlx5_core_dbg(dev, "page request for func 0x%x, npages %d, release_all %d\n" , |
633 | func_id, npages, release_all); |
634 | req = kzalloc(size: sizeof(*req), GFP_ATOMIC); |
635 | if (!req) { |
636 | mlx5_core_warn(dev, "failed to allocate pages request\n" ); |
637 | return NOTIFY_DONE; |
638 | } |
639 | |
640 | req->dev = dev; |
641 | req->func_id = func_id; |
642 | req->npages = npages; |
643 | req->ec_function = ec_function; |
644 | req->release_all = release_all; |
645 | INIT_WORK(&req->work, pages_work_handler); |
646 | queue_work(wq: dev->priv.pg_wq, work: &req->work); |
647 | return NOTIFY_OK; |
648 | } |
649 | |
650 | int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) |
651 | { |
652 | u16 func_id; |
653 | s32 npages; |
654 | int err; |
655 | |
656 | err = mlx5_cmd_query_pages(dev, func_id: &func_id, npages: &npages, boot); |
657 | if (err) |
658 | return err; |
659 | |
660 | mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n" , |
661 | npages, boot ? "boot" : "init" , func_id); |
662 | |
663 | return give_pages(dev, func_id, npages, event: 0, ec_function: mlx5_core_is_ecpf(dev)); |
664 | } |
665 | |
666 | enum { |
667 | MLX5_BLKS_FOR_RECLAIM_PAGES = 12 |
668 | }; |
669 | |
670 | static int optimal_reclaimed_pages(void) |
671 | { |
672 | struct mlx5_cmd_prot_block *block; |
673 | struct mlx5_cmd_layout *lay; |
674 | int ret; |
675 | |
676 | ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - |
677 | MLX5_ST_SZ_BYTES(manage_pages_out)) / |
678 | MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); |
679 | |
680 | return ret; |
681 | } |
682 | |
683 | static int mlx5_reclaim_root_pages(struct mlx5_core_dev *dev, |
684 | struct rb_root *root, u32 function) |
685 | { |
686 | u64 recl_pages_to_jiffies = msecs_to_jiffies(mlx5_tout_ms(dev, RECLAIM_PAGES)); |
687 | unsigned long end = jiffies + recl_pages_to_jiffies; |
688 | |
689 | while (!RB_EMPTY_ROOT(root)) { |
690 | u32 ec_function = mlx5_get_ec_function(function); |
691 | u32 function_id = mlx5_get_func_id(function); |
692 | int nclaimed; |
693 | int err; |
694 | |
695 | err = reclaim_pages(dev, func_id: function_id, npages: optimal_reclaimed_pages(), |
696 | nclaimed: &nclaimed, event: false, ec_function); |
697 | if (err) { |
698 | mlx5_core_warn(dev, "reclaim_pages err (%d) func_id=0x%x ec_func=0x%x\n" , |
699 | err, function_id, ec_function); |
700 | return err; |
701 | } |
702 | |
703 | if (nclaimed) |
704 | end = jiffies + recl_pages_to_jiffies; |
705 | |
706 | if (time_after(jiffies, end)) { |
707 | mlx5_core_warn(dev, "FW did not return all pages. giving up...\n" ); |
708 | break; |
709 | } |
710 | } |
711 | |
712 | return 0; |
713 | } |
714 | |
715 | int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) |
716 | { |
717 | struct rb_root *root; |
718 | unsigned long id; |
719 | void *entry; |
720 | |
721 | xa_for_each(&dev->priv.page_root_xa, id, entry) { |
722 | root = entry; |
723 | mlx5_reclaim_root_pages(dev, root, function: id); |
724 | xa_erase(&dev->priv.page_root_xa, index: id); |
725 | kfree(objp: root); |
726 | } |
727 | |
728 | WARN_ON(!xa_empty(&dev->priv.page_root_xa)); |
729 | |
730 | WARN(dev->priv.fw_pages, |
731 | "FW pages counter is %d after reclaiming all pages\n" , |
732 | dev->priv.fw_pages); |
733 | WARN(dev->priv.page_counters[MLX5_VF], |
734 | "VFs FW pages counter is %d after reclaiming all pages\n" , |
735 | dev->priv.page_counters[MLX5_VF]); |
736 | WARN(dev->priv.page_counters[MLX5_HOST_PF], |
737 | "External host PF FW pages counter is %d after reclaiming all pages\n" , |
738 | dev->priv.page_counters[MLX5_HOST_PF]); |
739 | WARN(dev->priv.page_counters[MLX5_EC_VF], |
740 | "EC VFs FW pages counter is %d after reclaiming all pages\n" , |
741 | dev->priv.page_counters[MLX5_EC_VF]); |
742 | |
743 | return 0; |
744 | } |
745 | |
746 | int mlx5_pagealloc_init(struct mlx5_core_dev *dev) |
747 | { |
748 | INIT_LIST_HEAD(list: &dev->priv.free_list); |
749 | dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator" ); |
750 | if (!dev->priv.pg_wq) |
751 | return -ENOMEM; |
752 | |
753 | xa_init(xa: &dev->priv.page_root_xa); |
754 | mlx5_pages_debugfs_init(dev); |
755 | |
756 | return 0; |
757 | } |
758 | |
759 | void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) |
760 | { |
761 | mlx5_pages_debugfs_cleanup(dev); |
762 | xa_destroy(&dev->priv.page_root_xa); |
763 | destroy_workqueue(wq: dev->priv.pg_wq); |
764 | } |
765 | |
766 | void mlx5_pagealloc_start(struct mlx5_core_dev *dev) |
767 | { |
768 | MLX5_NB_INIT(&dev->priv.pg_nb, req_pages_handler, PAGE_REQUEST); |
769 | mlx5_eq_notifier_register(dev, nb: &dev->priv.pg_nb); |
770 | } |
771 | |
772 | void mlx5_pagealloc_stop(struct mlx5_core_dev *dev) |
773 | { |
774 | mlx5_eq_notifier_unregister(dev, nb: &dev->priv.pg_nb); |
775 | flush_workqueue(dev->priv.pg_wq); |
776 | } |
777 | |
778 | int mlx5_wait_for_pages(struct mlx5_core_dev *dev, int *pages) |
779 | { |
780 | u64 recl_vf_pages_to_jiffies = msecs_to_jiffies(mlx5_tout_ms(dev, RECLAIM_VFS_PAGES)); |
781 | unsigned long end = jiffies + recl_vf_pages_to_jiffies; |
782 | int prev_pages = *pages; |
783 | |
784 | /* In case of internal error we will free the pages manually later */ |
785 | if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { |
786 | mlx5_core_warn(dev, "Skipping wait for vf pages stage" ); |
787 | return 0; |
788 | } |
789 | |
790 | mlx5_core_dbg(dev, "Waiting for %d pages\n" , prev_pages); |
791 | while (*pages) { |
792 | if (time_after(jiffies, end)) { |
793 | mlx5_core_warn(dev, "aborting while there are %d pending pages\n" , *pages); |
794 | return -ETIMEDOUT; |
795 | } |
796 | if (*pages < prev_pages) { |
797 | end = jiffies + recl_vf_pages_to_jiffies; |
798 | prev_pages = *pages; |
799 | } |
800 | msleep(msecs: 50); |
801 | } |
802 | |
803 | mlx5_core_dbg(dev, "All pages received\n" ); |
804 | return 0; |
805 | } |
806 | |