1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * DMABUF System heap exporter |
4 | * |
5 | * Copyright (C) 2011 Google, Inc. |
6 | * Copyright (C) 2019, 2020 Linaro Ltd. |
7 | * |
8 | * Portions based off of Andrew Davis' SRAM heap: |
9 | * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ |
10 | * Andrew F. Davis <afd@ti.com> |
11 | */ |
12 | |
13 | #include <linux/dma-buf.h> |
14 | #include <linux/dma-mapping.h> |
15 | #include <linux/dma-heap.h> |
16 | #include <linux/err.h> |
17 | #include <linux/highmem.h> |
18 | #include <linux/mm.h> |
19 | #include <linux/module.h> |
20 | #include <linux/scatterlist.h> |
21 | #include <linux/slab.h> |
22 | #include <linux/vmalloc.h> |
23 | |
24 | static struct dma_heap *sys_heap; |
25 | |
26 | struct system_heap_buffer { |
27 | struct dma_heap *heap; |
28 | struct list_head attachments; |
29 | struct mutex lock; |
30 | unsigned long len; |
31 | struct sg_table sg_table; |
32 | int vmap_cnt; |
33 | void *vaddr; |
34 | }; |
35 | |
36 | struct dma_heap_attachment { |
37 | struct device *dev; |
38 | struct sg_table *table; |
39 | struct list_head list; |
40 | bool mapped; |
41 | }; |
42 | |
43 | #define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO) |
44 | #define HIGH_ORDER_GFP (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \ |
45 | | __GFP_NORETRY) & ~__GFP_RECLAIM) \ |
46 | | __GFP_COMP) |
47 | static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP, LOW_ORDER_GFP}; |
48 | /* |
49 | * The selection of the orders used for allocation (1MB, 64K, 4K) is designed |
50 | * to match with the sizes often found in IOMMUs. Using order 4 pages instead |
51 | * of order 0 pages can significantly improve the performance of many IOMMUs |
52 | * by reducing TLB pressure and time spent updating page tables. |
53 | */ |
54 | static const unsigned int orders[] = {8, 4, 0}; |
55 | #define NUM_ORDERS ARRAY_SIZE(orders) |
56 | |
57 | static struct sg_table *dup_sg_table(struct sg_table *table) |
58 | { |
59 | struct sg_table *new_table; |
60 | int ret, i; |
61 | struct scatterlist *sg, *new_sg; |
62 | |
63 | new_table = kzalloc(size: sizeof(*new_table), GFP_KERNEL); |
64 | if (!new_table) |
65 | return ERR_PTR(error: -ENOMEM); |
66 | |
67 | ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL); |
68 | if (ret) { |
69 | kfree(objp: new_table); |
70 | return ERR_PTR(error: -ENOMEM); |
71 | } |
72 | |
73 | new_sg = new_table->sgl; |
74 | for_each_sgtable_sg(table, sg, i) { |
75 | sg_set_page(sg: new_sg, page: sg_page(sg), len: sg->length, offset: sg->offset); |
76 | new_sg = sg_next(new_sg); |
77 | } |
78 | |
79 | return new_table; |
80 | } |
81 | |
82 | static int system_heap_attach(struct dma_buf *dmabuf, |
83 | struct dma_buf_attachment *attachment) |
84 | { |
85 | struct system_heap_buffer *buffer = dmabuf->priv; |
86 | struct dma_heap_attachment *a; |
87 | struct sg_table *table; |
88 | |
89 | a = kzalloc(size: sizeof(*a), GFP_KERNEL); |
90 | if (!a) |
91 | return -ENOMEM; |
92 | |
93 | table = dup_sg_table(table: &buffer->sg_table); |
94 | if (IS_ERR(ptr: table)) { |
95 | kfree(objp: a); |
96 | return -ENOMEM; |
97 | } |
98 | |
99 | a->table = table; |
100 | a->dev = attachment->dev; |
101 | INIT_LIST_HEAD(list: &a->list); |
102 | a->mapped = false; |
103 | |
104 | attachment->priv = a; |
105 | |
106 | mutex_lock(&buffer->lock); |
107 | list_add(new: &a->list, head: &buffer->attachments); |
108 | mutex_unlock(lock: &buffer->lock); |
109 | |
110 | return 0; |
111 | } |
112 | |
113 | static void system_heap_detach(struct dma_buf *dmabuf, |
114 | struct dma_buf_attachment *attachment) |
115 | { |
116 | struct system_heap_buffer *buffer = dmabuf->priv; |
117 | struct dma_heap_attachment *a = attachment->priv; |
118 | |
119 | mutex_lock(&buffer->lock); |
120 | list_del(entry: &a->list); |
121 | mutex_unlock(lock: &buffer->lock); |
122 | |
123 | sg_free_table(a->table); |
124 | kfree(objp: a->table); |
125 | kfree(objp: a); |
126 | } |
127 | |
128 | static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment, |
129 | enum dma_data_direction direction) |
130 | { |
131 | struct dma_heap_attachment *a = attachment->priv; |
132 | struct sg_table *table = a->table; |
133 | int ret; |
134 | |
135 | ret = dma_map_sgtable(dev: attachment->dev, sgt: table, dir: direction, attrs: 0); |
136 | if (ret) |
137 | return ERR_PTR(error: ret); |
138 | |
139 | a->mapped = true; |
140 | return table; |
141 | } |
142 | |
143 | static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment, |
144 | struct sg_table *table, |
145 | enum dma_data_direction direction) |
146 | { |
147 | struct dma_heap_attachment *a = attachment->priv; |
148 | |
149 | a->mapped = false; |
150 | dma_unmap_sgtable(dev: attachment->dev, sgt: table, dir: direction, attrs: 0); |
151 | } |
152 | |
153 | static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, |
154 | enum dma_data_direction direction) |
155 | { |
156 | struct system_heap_buffer *buffer = dmabuf->priv; |
157 | struct dma_heap_attachment *a; |
158 | |
159 | mutex_lock(&buffer->lock); |
160 | |
161 | if (buffer->vmap_cnt) |
162 | invalidate_kernel_vmap_range(vaddr: buffer->vaddr, size: buffer->len); |
163 | |
164 | list_for_each_entry(a, &buffer->attachments, list) { |
165 | if (!a->mapped) |
166 | continue; |
167 | dma_sync_sgtable_for_cpu(dev: a->dev, sgt: a->table, dir: direction); |
168 | } |
169 | mutex_unlock(lock: &buffer->lock); |
170 | |
171 | return 0; |
172 | } |
173 | |
174 | static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf, |
175 | enum dma_data_direction direction) |
176 | { |
177 | struct system_heap_buffer *buffer = dmabuf->priv; |
178 | struct dma_heap_attachment *a; |
179 | |
180 | mutex_lock(&buffer->lock); |
181 | |
182 | if (buffer->vmap_cnt) |
183 | flush_kernel_vmap_range(vaddr: buffer->vaddr, size: buffer->len); |
184 | |
185 | list_for_each_entry(a, &buffer->attachments, list) { |
186 | if (!a->mapped) |
187 | continue; |
188 | dma_sync_sgtable_for_device(dev: a->dev, sgt: a->table, dir: direction); |
189 | } |
190 | mutex_unlock(lock: &buffer->lock); |
191 | |
192 | return 0; |
193 | } |
194 | |
195 | static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) |
196 | { |
197 | struct system_heap_buffer *buffer = dmabuf->priv; |
198 | struct sg_table *table = &buffer->sg_table; |
199 | unsigned long addr = vma->vm_start; |
200 | struct sg_page_iter piter; |
201 | int ret; |
202 | |
203 | for_each_sgtable_page(table, &piter, vma->vm_pgoff) { |
204 | struct page *page = sg_page_iter_page(piter: &piter); |
205 | |
206 | ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE, |
207 | vma->vm_page_prot); |
208 | if (ret) |
209 | return ret; |
210 | addr += PAGE_SIZE; |
211 | if (addr >= vma->vm_end) |
212 | return 0; |
213 | } |
214 | return 0; |
215 | } |
216 | |
217 | static void *system_heap_do_vmap(struct system_heap_buffer *buffer) |
218 | { |
219 | struct sg_table *table = &buffer->sg_table; |
220 | int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE; |
221 | struct page **pages = vmalloc(size: sizeof(struct page *) * npages); |
222 | struct page **tmp = pages; |
223 | struct sg_page_iter piter; |
224 | void *vaddr; |
225 | |
226 | if (!pages) |
227 | return ERR_PTR(error: -ENOMEM); |
228 | |
229 | for_each_sgtable_page(table, &piter, 0) { |
230 | WARN_ON(tmp - pages >= npages); |
231 | *tmp++ = sg_page_iter_page(piter: &piter); |
232 | } |
233 | |
234 | vaddr = vmap(pages, count: npages, VM_MAP, PAGE_KERNEL); |
235 | vfree(addr: pages); |
236 | |
237 | if (!vaddr) |
238 | return ERR_PTR(error: -ENOMEM); |
239 | |
240 | return vaddr; |
241 | } |
242 | |
243 | static int system_heap_vmap(struct dma_buf *dmabuf, struct iosys_map *map) |
244 | { |
245 | struct system_heap_buffer *buffer = dmabuf->priv; |
246 | void *vaddr; |
247 | int ret = 0; |
248 | |
249 | mutex_lock(&buffer->lock); |
250 | if (buffer->vmap_cnt) { |
251 | buffer->vmap_cnt++; |
252 | iosys_map_set_vaddr(map, vaddr: buffer->vaddr); |
253 | goto out; |
254 | } |
255 | |
256 | vaddr = system_heap_do_vmap(buffer); |
257 | if (IS_ERR(ptr: vaddr)) { |
258 | ret = PTR_ERR(ptr: vaddr); |
259 | goto out; |
260 | } |
261 | |
262 | buffer->vaddr = vaddr; |
263 | buffer->vmap_cnt++; |
264 | iosys_map_set_vaddr(map, vaddr: buffer->vaddr); |
265 | out: |
266 | mutex_unlock(lock: &buffer->lock); |
267 | |
268 | return ret; |
269 | } |
270 | |
271 | static void system_heap_vunmap(struct dma_buf *dmabuf, struct iosys_map *map) |
272 | { |
273 | struct system_heap_buffer *buffer = dmabuf->priv; |
274 | |
275 | mutex_lock(&buffer->lock); |
276 | if (!--buffer->vmap_cnt) { |
277 | vunmap(addr: buffer->vaddr); |
278 | buffer->vaddr = NULL; |
279 | } |
280 | mutex_unlock(lock: &buffer->lock); |
281 | iosys_map_clear(map); |
282 | } |
283 | |
284 | static void system_heap_dma_buf_release(struct dma_buf *dmabuf) |
285 | { |
286 | struct system_heap_buffer *buffer = dmabuf->priv; |
287 | struct sg_table *table; |
288 | struct scatterlist *sg; |
289 | int i; |
290 | |
291 | table = &buffer->sg_table; |
292 | for_each_sgtable_sg(table, sg, i) { |
293 | struct page *page = sg_page(sg); |
294 | |
295 | __free_pages(page, order: compound_order(page)); |
296 | } |
297 | sg_free_table(table); |
298 | kfree(objp: buffer); |
299 | } |
300 | |
301 | static const struct dma_buf_ops system_heap_buf_ops = { |
302 | .attach = system_heap_attach, |
303 | .detach = system_heap_detach, |
304 | .map_dma_buf = system_heap_map_dma_buf, |
305 | .unmap_dma_buf = system_heap_unmap_dma_buf, |
306 | .begin_cpu_access = system_heap_dma_buf_begin_cpu_access, |
307 | .end_cpu_access = system_heap_dma_buf_end_cpu_access, |
308 | .mmap = system_heap_mmap, |
309 | .vmap = system_heap_vmap, |
310 | .vunmap = system_heap_vunmap, |
311 | .release = system_heap_dma_buf_release, |
312 | }; |
313 | |
314 | static struct page *alloc_largest_available(unsigned long size, |
315 | unsigned int max_order) |
316 | { |
317 | struct page *page; |
318 | int i; |
319 | |
320 | for (i = 0; i < NUM_ORDERS; i++) { |
321 | if (size < (PAGE_SIZE << orders[i])) |
322 | continue; |
323 | if (max_order < orders[i]) |
324 | continue; |
325 | |
326 | page = alloc_pages(gfp: order_flags[i], order: orders[i]); |
327 | if (!page) |
328 | continue; |
329 | return page; |
330 | } |
331 | return NULL; |
332 | } |
333 | |
334 | static struct dma_buf *system_heap_allocate(struct dma_heap *heap, |
335 | unsigned long len, |
336 | unsigned long fd_flags, |
337 | unsigned long heap_flags) |
338 | { |
339 | struct system_heap_buffer *buffer; |
340 | DEFINE_DMA_BUF_EXPORT_INFO(exp_info); |
341 | unsigned long size_remaining = len; |
342 | unsigned int max_order = orders[0]; |
343 | struct dma_buf *dmabuf; |
344 | struct sg_table *table; |
345 | struct scatterlist *sg; |
346 | struct list_head pages; |
347 | struct page *page, *tmp_page; |
348 | int i, ret = -ENOMEM; |
349 | |
350 | buffer = kzalloc(size: sizeof(*buffer), GFP_KERNEL); |
351 | if (!buffer) |
352 | return ERR_PTR(error: -ENOMEM); |
353 | |
354 | INIT_LIST_HEAD(list: &buffer->attachments); |
355 | mutex_init(&buffer->lock); |
356 | buffer->heap = heap; |
357 | buffer->len = len; |
358 | |
359 | INIT_LIST_HEAD(list: &pages); |
360 | i = 0; |
361 | while (size_remaining > 0) { |
362 | /* |
363 | * Avoid trying to allocate memory if the process |
364 | * has been killed by SIGKILL |
365 | */ |
366 | if (fatal_signal_pending(current)) { |
367 | ret = -EINTR; |
368 | goto free_buffer; |
369 | } |
370 | |
371 | page = alloc_largest_available(size: size_remaining, max_order); |
372 | if (!page) |
373 | goto free_buffer; |
374 | |
375 | list_add_tail(new: &page->lru, head: &pages); |
376 | size_remaining -= page_size(page); |
377 | max_order = compound_order(page); |
378 | i++; |
379 | } |
380 | |
381 | table = &buffer->sg_table; |
382 | if (sg_alloc_table(table, i, GFP_KERNEL)) |
383 | goto free_buffer; |
384 | |
385 | sg = table->sgl; |
386 | list_for_each_entry_safe(page, tmp_page, &pages, lru) { |
387 | sg_set_page(sg, page, len: page_size(page), offset: 0); |
388 | sg = sg_next(sg); |
389 | list_del(entry: &page->lru); |
390 | } |
391 | |
392 | /* create the dmabuf */ |
393 | exp_info.exp_name = dma_heap_get_name(heap); |
394 | exp_info.ops = &system_heap_buf_ops; |
395 | exp_info.size = buffer->len; |
396 | exp_info.flags = fd_flags; |
397 | exp_info.priv = buffer; |
398 | dmabuf = dma_buf_export(exp_info: &exp_info); |
399 | if (IS_ERR(ptr: dmabuf)) { |
400 | ret = PTR_ERR(ptr: dmabuf); |
401 | goto free_pages; |
402 | } |
403 | return dmabuf; |
404 | |
405 | free_pages: |
406 | for_each_sgtable_sg(table, sg, i) { |
407 | struct page *p = sg_page(sg); |
408 | |
409 | __free_pages(page: p, order: compound_order(page: p)); |
410 | } |
411 | sg_free_table(table); |
412 | free_buffer: |
413 | list_for_each_entry_safe(page, tmp_page, &pages, lru) |
414 | __free_pages(page, order: compound_order(page)); |
415 | kfree(objp: buffer); |
416 | |
417 | return ERR_PTR(error: ret); |
418 | } |
419 | |
420 | static const struct dma_heap_ops system_heap_ops = { |
421 | .allocate = system_heap_allocate, |
422 | }; |
423 | |
424 | static int system_heap_create(void) |
425 | { |
426 | struct dma_heap_export_info exp_info; |
427 | |
428 | exp_info.name = "system" ; |
429 | exp_info.ops = &system_heap_ops; |
430 | exp_info.priv = NULL; |
431 | |
432 | sys_heap = dma_heap_add(exp_info: &exp_info); |
433 | if (IS_ERR(ptr: sys_heap)) |
434 | return PTR_ERR(ptr: sys_heap); |
435 | |
436 | return 0; |
437 | } |
438 | module_init(system_heap_create); |
439 | |