1 | /* |
2 | * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. |
3 | * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. |
4 | * |
5 | * This software is available to you under a choice of one of two |
6 | * licenses. You may choose to be licensed under the terms of the GNU |
7 | * General Public License (GPL) Version 2, available from the file |
8 | * COPYING in the main directory of this source tree, or the |
9 | * OpenIB.org BSD license below: |
10 | * |
11 | * Redistribution and use in source and binary forms, with or |
12 | * without modification, are permitted provided that the following |
13 | * conditions are met: |
14 | * |
15 | * - Redistributions of source code must retain the above |
16 | * copyright notice, this list of conditions and the following |
17 | * disclaimer. |
18 | * |
19 | * - Redistributions in binary form must reproduce the above |
20 | * copyright notice, this list of conditions and the following |
21 | * disclaimer in the documentation and/or other materials |
22 | * provided with the distribution. |
23 | * |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
31 | * SOFTWARE. |
32 | */ |
33 | |
34 | #include <linux/errno.h> |
35 | #include <linux/mm.h> |
36 | #include <linux/scatterlist.h> |
37 | #include <linux/slab.h> |
38 | |
39 | #include <linux/mlx4/cmd.h> |
40 | |
41 | #include "mlx4.h" |
42 | #include "icm.h" |
43 | #include "fw.h" |
44 | |
45 | /* |
46 | * We allocate in as big chunks as we can, up to a maximum of 256 KB |
47 | * per chunk. Note that the chunks are not necessarily in contiguous |
48 | * physical memory. |
49 | */ |
50 | enum { |
51 | MLX4_ICM_ALLOC_SIZE = 1 << 18, |
52 | MLX4_TABLE_CHUNK_SIZE = 1 << 18, |
53 | }; |
54 | |
55 | static void mlx4_free_icm_pages(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk) |
56 | { |
57 | int i; |
58 | |
59 | if (chunk->nsg > 0) |
60 | dma_unmap_sg(&dev->persist->pdev->dev, chunk->sg, chunk->npages, |
61 | DMA_BIDIRECTIONAL); |
62 | |
63 | for (i = 0; i < chunk->npages; ++i) |
64 | __free_pages(page: sg_page(sg: &chunk->sg[i]), |
65 | order: get_order(size: chunk->sg[i].length)); |
66 | } |
67 | |
68 | static void mlx4_free_icm_coherent(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk) |
69 | { |
70 | int i; |
71 | |
72 | for (i = 0; i < chunk->npages; ++i) |
73 | dma_free_coherent(dev: &dev->persist->pdev->dev, |
74 | size: chunk->buf[i].size, |
75 | cpu_addr: chunk->buf[i].addr, |
76 | dma_handle: chunk->buf[i].dma_addr); |
77 | } |
78 | |
79 | void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent) |
80 | { |
81 | struct mlx4_icm_chunk *chunk, *tmp; |
82 | |
83 | if (!icm) |
84 | return; |
85 | |
86 | list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) { |
87 | if (coherent) |
88 | mlx4_free_icm_coherent(dev, chunk); |
89 | else |
90 | mlx4_free_icm_pages(dev, chunk); |
91 | |
92 | kfree(objp: chunk); |
93 | } |
94 | |
95 | kfree(objp: icm); |
96 | } |
97 | |
98 | static int mlx4_alloc_icm_pages(struct scatterlist *mem, int order, |
99 | gfp_t gfp_mask, int node) |
100 | { |
101 | struct page *page; |
102 | |
103 | page = alloc_pages_node(nid: node, gfp_mask, order); |
104 | if (!page) { |
105 | page = alloc_pages(gfp: gfp_mask, order); |
106 | if (!page) |
107 | return -ENOMEM; |
108 | } |
109 | |
110 | sg_set_page(sg: mem, page, PAGE_SIZE << order, offset: 0); |
111 | return 0; |
112 | } |
113 | |
114 | static int mlx4_alloc_icm_coherent(struct device *dev, struct mlx4_icm_buf *buf, |
115 | int order, gfp_t gfp_mask) |
116 | { |
117 | buf->addr = dma_alloc_coherent(dev, PAGE_SIZE << order, |
118 | dma_handle: &buf->dma_addr, gfp: gfp_mask); |
119 | if (!buf->addr) |
120 | return -ENOMEM; |
121 | |
122 | if (offset_in_page(buf->addr)) { |
123 | dma_free_coherent(dev, PAGE_SIZE << order, cpu_addr: buf->addr, |
124 | dma_handle: buf->dma_addr); |
125 | return -ENOMEM; |
126 | } |
127 | |
128 | buf->size = PAGE_SIZE << order; |
129 | return 0; |
130 | } |
131 | |
132 | struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages, |
133 | gfp_t gfp_mask, int coherent) |
134 | { |
135 | struct mlx4_icm *icm; |
136 | struct mlx4_icm_chunk *chunk = NULL; |
137 | int cur_order; |
138 | gfp_t mask; |
139 | int ret; |
140 | |
141 | /* We use sg_set_buf for coherent allocs, which assumes low memory */ |
142 | BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM)); |
143 | |
144 | icm = kmalloc_node(size: sizeof(*icm), |
145 | flags: gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN), |
146 | node: dev->numa_node); |
147 | if (!icm) { |
148 | icm = kmalloc(size: sizeof(*icm), |
149 | flags: gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); |
150 | if (!icm) |
151 | return NULL; |
152 | } |
153 | |
154 | icm->refcount = 0; |
155 | INIT_LIST_HEAD(list: &icm->chunk_list); |
156 | |
157 | cur_order = get_order(size: MLX4_ICM_ALLOC_SIZE); |
158 | |
159 | while (npages > 0) { |
160 | if (!chunk) { |
161 | chunk = kzalloc_node(size: sizeof(*chunk), |
162 | flags: gfp_mask & ~(__GFP_HIGHMEM | |
163 | __GFP_NOWARN), |
164 | node: dev->numa_node); |
165 | if (!chunk) { |
166 | chunk = kzalloc(size: sizeof(*chunk), |
167 | flags: gfp_mask & ~(__GFP_HIGHMEM | |
168 | __GFP_NOWARN)); |
169 | if (!chunk) |
170 | goto fail; |
171 | } |
172 | chunk->coherent = coherent; |
173 | |
174 | if (!coherent) |
175 | sg_init_table(chunk->sg, MLX4_ICM_CHUNK_LEN); |
176 | list_add_tail(new: &chunk->list, head: &icm->chunk_list); |
177 | } |
178 | |
179 | while (1 << cur_order > npages) |
180 | --cur_order; |
181 | |
182 | mask = gfp_mask; |
183 | if (cur_order) |
184 | mask &= ~__GFP_DIRECT_RECLAIM; |
185 | |
186 | if (coherent) |
187 | ret = mlx4_alloc_icm_coherent(dev: &dev->persist->pdev->dev, |
188 | buf: &chunk->buf[chunk->npages], |
189 | order: cur_order, gfp_mask: mask); |
190 | else |
191 | ret = mlx4_alloc_icm_pages(mem: &chunk->sg[chunk->npages], |
192 | order: cur_order, gfp_mask: mask, |
193 | node: dev->numa_node); |
194 | |
195 | if (ret) { |
196 | if (--cur_order < 0) |
197 | goto fail; |
198 | else |
199 | continue; |
200 | } |
201 | |
202 | ++chunk->npages; |
203 | |
204 | if (coherent) |
205 | ++chunk->nsg; |
206 | else if (chunk->npages == MLX4_ICM_CHUNK_LEN) { |
207 | chunk->nsg = dma_map_sg(&dev->persist->pdev->dev, |
208 | chunk->sg, chunk->npages, |
209 | DMA_BIDIRECTIONAL); |
210 | |
211 | if (!chunk->nsg) |
212 | goto fail; |
213 | } |
214 | |
215 | if (chunk->npages == MLX4_ICM_CHUNK_LEN) |
216 | chunk = NULL; |
217 | |
218 | npages -= 1 << cur_order; |
219 | } |
220 | |
221 | if (!coherent && chunk) { |
222 | chunk->nsg = dma_map_sg(&dev->persist->pdev->dev, chunk->sg, |
223 | chunk->npages, DMA_BIDIRECTIONAL); |
224 | |
225 | if (!chunk->nsg) |
226 | goto fail; |
227 | } |
228 | |
229 | return icm; |
230 | |
231 | fail: |
232 | mlx4_free_icm(dev, icm, coherent); |
233 | return NULL; |
234 | } |
235 | |
236 | static int mlx4_MAP_ICM(struct mlx4_dev *dev, struct mlx4_icm *icm, u64 virt) |
237 | { |
238 | return mlx4_map_cmd(dev, op: MLX4_CMD_MAP_ICM, icm, virt); |
239 | } |
240 | |
241 | static int mlx4_UNMAP_ICM(struct mlx4_dev *dev, u64 virt, u32 page_count) |
242 | { |
243 | return mlx4_cmd(dev, in_param: virt, in_modifier: page_count, op_modifier: 0, op: MLX4_CMD_UNMAP_ICM, |
244 | timeout: MLX4_CMD_TIME_CLASS_B, native: MLX4_CMD_NATIVE); |
245 | } |
246 | |
247 | int mlx4_MAP_ICM_AUX(struct mlx4_dev *dev, struct mlx4_icm *icm) |
248 | { |
249 | return mlx4_map_cmd(dev, op: MLX4_CMD_MAP_ICM_AUX, icm, virt: -1); |
250 | } |
251 | |
252 | int mlx4_UNMAP_ICM_AUX(struct mlx4_dev *dev) |
253 | { |
254 | return mlx4_cmd(dev, in_param: 0, in_modifier: 0, op_modifier: 0, op: MLX4_CMD_UNMAP_ICM_AUX, |
255 | timeout: MLX4_CMD_TIME_CLASS_B, native: MLX4_CMD_NATIVE); |
256 | } |
257 | |
258 | int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj) |
259 | { |
260 | u32 i = (obj & (table->num_obj - 1)) / |
261 | (MLX4_TABLE_CHUNK_SIZE / table->obj_size); |
262 | int ret = 0; |
263 | |
264 | mutex_lock(&table->mutex); |
265 | |
266 | if (table->icm[i]) { |
267 | ++table->icm[i]->refcount; |
268 | goto out; |
269 | } |
270 | |
271 | table->icm[i] = mlx4_alloc_icm(dev, npages: MLX4_TABLE_CHUNK_SIZE >> PAGE_SHIFT, |
272 | gfp_mask: (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | |
273 | __GFP_NOWARN, coherent: table->coherent); |
274 | if (!table->icm[i]) { |
275 | ret = -ENOMEM; |
276 | goto out; |
277 | } |
278 | |
279 | if (mlx4_MAP_ICM(dev, icm: table->icm[i], virt: table->virt + |
280 | (u64) i * MLX4_TABLE_CHUNK_SIZE)) { |
281 | mlx4_free_icm(dev, icm: table->icm[i], coherent: table->coherent); |
282 | table->icm[i] = NULL; |
283 | ret = -ENOMEM; |
284 | goto out; |
285 | } |
286 | |
287 | ++table->icm[i]->refcount; |
288 | |
289 | out: |
290 | mutex_unlock(lock: &table->mutex); |
291 | return ret; |
292 | } |
293 | |
294 | void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj) |
295 | { |
296 | u32 i; |
297 | u64 offset; |
298 | |
299 | i = (obj & (table->num_obj - 1)) / (MLX4_TABLE_CHUNK_SIZE / table->obj_size); |
300 | |
301 | mutex_lock(&table->mutex); |
302 | |
303 | if (--table->icm[i]->refcount == 0) { |
304 | offset = (u64) i * MLX4_TABLE_CHUNK_SIZE; |
305 | mlx4_UNMAP_ICM(dev, virt: table->virt + offset, |
306 | page_count: MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE); |
307 | mlx4_free_icm(dev, icm: table->icm[i], coherent: table->coherent); |
308 | table->icm[i] = NULL; |
309 | } |
310 | |
311 | mutex_unlock(lock: &table->mutex); |
312 | } |
313 | |
314 | void *mlx4_table_find(struct mlx4_icm_table *table, u32 obj, |
315 | dma_addr_t *dma_handle) |
316 | { |
317 | int offset, dma_offset, i; |
318 | u64 idx; |
319 | struct mlx4_icm_chunk *chunk; |
320 | struct mlx4_icm *icm; |
321 | void *addr = NULL; |
322 | |
323 | if (!table->lowmem) |
324 | return NULL; |
325 | |
326 | mutex_lock(&table->mutex); |
327 | |
328 | idx = (u64) (obj & (table->num_obj - 1)) * table->obj_size; |
329 | icm = table->icm[idx / MLX4_TABLE_CHUNK_SIZE]; |
330 | dma_offset = offset = idx % MLX4_TABLE_CHUNK_SIZE; |
331 | |
332 | if (!icm) |
333 | goto out; |
334 | |
335 | list_for_each_entry(chunk, &icm->chunk_list, list) { |
336 | for (i = 0; i < chunk->npages; ++i) { |
337 | dma_addr_t dma_addr; |
338 | size_t len; |
339 | |
340 | if (table->coherent) { |
341 | len = chunk->buf[i].size; |
342 | dma_addr = chunk->buf[i].dma_addr; |
343 | addr = chunk->buf[i].addr; |
344 | } else { |
345 | struct page *page; |
346 | |
347 | len = sg_dma_len(&chunk->sg[i]); |
348 | dma_addr = sg_dma_address(&chunk->sg[i]); |
349 | |
350 | /* XXX: we should never do this for highmem |
351 | * allocation. This function either needs |
352 | * to be split, or the kernel virtual address |
353 | * return needs to be made optional. |
354 | */ |
355 | page = sg_page(sg: &chunk->sg[i]); |
356 | addr = lowmem_page_address(page); |
357 | } |
358 | |
359 | if (dma_handle && dma_offset >= 0) { |
360 | if (len > dma_offset) |
361 | *dma_handle = dma_addr + dma_offset; |
362 | dma_offset -= len; |
363 | } |
364 | |
365 | /* |
366 | * DMA mapping can merge pages but not split them, |
367 | * so if we found the page, dma_handle has already |
368 | * been assigned to. |
369 | */ |
370 | if (len > offset) |
371 | goto out; |
372 | offset -= len; |
373 | } |
374 | } |
375 | |
376 | addr = NULL; |
377 | out: |
378 | mutex_unlock(lock: &table->mutex); |
379 | return addr ? addr + offset : NULL; |
380 | } |
381 | |
382 | int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, |
383 | u32 start, u32 end) |
384 | { |
385 | int inc = MLX4_TABLE_CHUNK_SIZE / table->obj_size; |
386 | int err; |
387 | u32 i; |
388 | |
389 | for (i = start; i <= end; i += inc) { |
390 | err = mlx4_table_get(dev, table, obj: i); |
391 | if (err) |
392 | goto fail; |
393 | } |
394 | |
395 | return 0; |
396 | |
397 | fail: |
398 | while (i > start) { |
399 | i -= inc; |
400 | mlx4_table_put(dev, table, obj: i); |
401 | } |
402 | |
403 | return err; |
404 | } |
405 | |
406 | void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, |
407 | u32 start, u32 end) |
408 | { |
409 | u32 i; |
410 | |
411 | for (i = start; i <= end; i += MLX4_TABLE_CHUNK_SIZE / table->obj_size) |
412 | mlx4_table_put(dev, table, obj: i); |
413 | } |
414 | |
415 | int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table, |
416 | u64 virt, int obj_size, u32 nobj, int reserved, |
417 | int use_lowmem, int use_coherent) |
418 | { |
419 | int obj_per_chunk; |
420 | int num_icm; |
421 | unsigned chunk_size; |
422 | int i; |
423 | u64 size; |
424 | |
425 | obj_per_chunk = MLX4_TABLE_CHUNK_SIZE / obj_size; |
426 | if (WARN_ON(!obj_per_chunk)) |
427 | return -EINVAL; |
428 | num_icm = DIV_ROUND_UP(nobj, obj_per_chunk); |
429 | |
430 | table->icm = kvcalloc(n: num_icm, size: sizeof(*table->icm), GFP_KERNEL); |
431 | if (!table->icm) |
432 | return -ENOMEM; |
433 | table->virt = virt; |
434 | table->num_icm = num_icm; |
435 | table->num_obj = nobj; |
436 | table->obj_size = obj_size; |
437 | table->lowmem = use_lowmem; |
438 | table->coherent = use_coherent; |
439 | mutex_init(&table->mutex); |
440 | |
441 | size = (u64) nobj * obj_size; |
442 | for (i = 0; i * MLX4_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) { |
443 | chunk_size = MLX4_TABLE_CHUNK_SIZE; |
444 | if ((i + 1) * MLX4_TABLE_CHUNK_SIZE > size) |
445 | chunk_size = PAGE_ALIGN(size - |
446 | i * MLX4_TABLE_CHUNK_SIZE); |
447 | |
448 | table->icm[i] = mlx4_alloc_icm(dev, npages: chunk_size >> PAGE_SHIFT, |
449 | gfp_mask: (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) | |
450 | __GFP_NOWARN, coherent: use_coherent); |
451 | if (!table->icm[i]) |
452 | goto err; |
453 | if (mlx4_MAP_ICM(dev, icm: table->icm[i], virt: virt + i * MLX4_TABLE_CHUNK_SIZE)) { |
454 | mlx4_free_icm(dev, icm: table->icm[i], coherent: use_coherent); |
455 | table->icm[i] = NULL; |
456 | goto err; |
457 | } |
458 | |
459 | /* |
460 | * Add a reference to this ICM chunk so that it never |
461 | * gets freed (since it contains reserved firmware objects). |
462 | */ |
463 | ++table->icm[i]->refcount; |
464 | } |
465 | |
466 | return 0; |
467 | |
468 | err: |
469 | for (i = 0; i < num_icm; ++i) |
470 | if (table->icm[i]) { |
471 | mlx4_UNMAP_ICM(dev, virt: virt + i * MLX4_TABLE_CHUNK_SIZE, |
472 | page_count: MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE); |
473 | mlx4_free_icm(dev, icm: table->icm[i], coherent: use_coherent); |
474 | } |
475 | |
476 | kvfree(addr: table->icm); |
477 | |
478 | return -ENOMEM; |
479 | } |
480 | |
481 | void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table) |
482 | { |
483 | int i; |
484 | |
485 | for (i = 0; i < table->num_icm; ++i) |
486 | if (table->icm[i]) { |
487 | mlx4_UNMAP_ICM(dev, virt: table->virt + i * MLX4_TABLE_CHUNK_SIZE, |
488 | page_count: MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE); |
489 | mlx4_free_icm(dev, icm: table->icm[i], coherent: table->coherent); |
490 | } |
491 | |
492 | kvfree(addr: table->icm); |
493 | } |
494 | |