1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2015-2017, 2019-2021 Linaro Limited
4 */
5#include <linux/anon_inodes.h>
6#include <linux/device.h>
7#include <linux/idr.h>
8#include <linux/mm.h>
9#include <linux/sched.h>
10#include <linux/slab.h>
11#include <linux/tee_drv.h>
12#include <linux/uaccess.h>
13#include <linux/uio.h>
14#include <linux/highmem.h>
15#include "tee_private.h"
16
17static void shm_put_kernel_pages(struct page **pages, size_t page_count)
18{
19 size_t n;
20
21 for (n = 0; n < page_count; n++)
22 put_page(page: pages[n]);
23}
24
25static int shm_get_kernel_pages(unsigned long start, size_t page_count,
26 struct page **pages)
27{
28 struct page *page;
29 size_t n;
30
31 if (WARN_ON_ONCE(is_vmalloc_addr((void *)start) ||
32 is_kmap_addr((void *)start)))
33 return -EINVAL;
34
35 page = virt_to_page((void *)start);
36 for (n = 0; n < page_count; n++) {
37 pages[n] = page + n;
38 get_page(page: pages[n]);
39 }
40
41 return page_count;
42}
43
44static void release_registered_pages(struct tee_shm *shm)
45{
46 if (shm->pages) {
47 if (shm->flags & TEE_SHM_USER_MAPPED)
48 unpin_user_pages(pages: shm->pages, npages: shm->num_pages);
49 else
50 shm_put_kernel_pages(pages: shm->pages, page_count: shm->num_pages);
51
52 kfree(objp: shm->pages);
53 }
54}
55
56static void tee_shm_release(struct tee_device *teedev, struct tee_shm *shm)
57{
58 if (shm->flags & TEE_SHM_POOL) {
59 teedev->pool->ops->free(teedev->pool, shm);
60 } else if (shm->flags & TEE_SHM_DYNAMIC) {
61 int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);
62
63 if (rc)
64 dev_err(teedev->dev.parent,
65 "unregister shm %p failed: %d", shm, rc);
66
67 release_registered_pages(shm);
68 }
69
70 teedev_ctx_put(ctx: shm->ctx);
71
72 kfree(objp: shm);
73
74 tee_device_put(teedev);
75}
76
77static struct tee_shm *shm_alloc_helper(struct tee_context *ctx, size_t size,
78 size_t align, u32 flags, int id)
79{
80 struct tee_device *teedev = ctx->teedev;
81 struct tee_shm *shm;
82 void *ret;
83 int rc;
84
85 if (!tee_device_get(teedev))
86 return ERR_PTR(error: -EINVAL);
87
88 if (!teedev->pool) {
89 /* teedev has been detached from driver */
90 ret = ERR_PTR(error: -EINVAL);
91 goto err_dev_put;
92 }
93
94 shm = kzalloc(size: sizeof(*shm), GFP_KERNEL);
95 if (!shm) {
96 ret = ERR_PTR(error: -ENOMEM);
97 goto err_dev_put;
98 }
99
100 refcount_set(r: &shm->refcount, n: 1);
101 shm->flags = flags;
102 shm->id = id;
103
104 /*
105 * We're assigning this as it is needed if the shm is to be
106 * registered. If this function returns OK then the caller expected
107 * to call teedev_ctx_get() or clear shm->ctx in case it's not
108 * needed any longer.
109 */
110 shm->ctx = ctx;
111
112 rc = teedev->pool->ops->alloc(teedev->pool, shm, size, align);
113 if (rc) {
114 ret = ERR_PTR(error: rc);
115 goto err_kfree;
116 }
117
118 teedev_ctx_get(ctx);
119 return shm;
120err_kfree:
121 kfree(objp: shm);
122err_dev_put:
123 tee_device_put(teedev);
124 return ret;
125}
126
127/**
128 * tee_shm_alloc_user_buf() - Allocate shared memory for user space
129 * @ctx: Context that allocates the shared memory
130 * @size: Requested size of shared memory
131 *
132 * Memory allocated as user space shared memory is automatically freed when
133 * the TEE file pointer is closed. The primary usage of this function is
134 * when the TEE driver doesn't support registering ordinary user space
135 * memory.
136 *
137 * @returns a pointer to 'struct tee_shm'
138 */
139struct tee_shm *tee_shm_alloc_user_buf(struct tee_context *ctx, size_t size)
140{
141 u32 flags = TEE_SHM_DYNAMIC | TEE_SHM_POOL;
142 struct tee_device *teedev = ctx->teedev;
143 struct tee_shm *shm;
144 void *ret;
145 int id;
146
147 mutex_lock(&teedev->mutex);
148 id = idr_alloc(&teedev->idr, NULL, start: 1, end: 0, GFP_KERNEL);
149 mutex_unlock(lock: &teedev->mutex);
150 if (id < 0)
151 return ERR_PTR(error: id);
152
153 shm = shm_alloc_helper(ctx, size, PAGE_SIZE, flags, id);
154 if (IS_ERR(ptr: shm)) {
155 mutex_lock(&teedev->mutex);
156 idr_remove(&teedev->idr, id);
157 mutex_unlock(lock: &teedev->mutex);
158 return shm;
159 }
160
161 mutex_lock(&teedev->mutex);
162 ret = idr_replace(&teedev->idr, shm, id);
163 mutex_unlock(lock: &teedev->mutex);
164 if (IS_ERR(ptr: ret)) {
165 tee_shm_free(shm);
166 return ret;
167 }
168
169 return shm;
170}
171
172/**
173 * tee_shm_alloc_kernel_buf() - Allocate shared memory for kernel buffer
174 * @ctx: Context that allocates the shared memory
175 * @size: Requested size of shared memory
176 *
177 * The returned memory registered in secure world and is suitable to be
178 * passed as a memory buffer in parameter argument to
179 * tee_client_invoke_func(). The memory allocated is later freed with a
180 * call to tee_shm_free().
181 *
182 * @returns a pointer to 'struct tee_shm'
183 */
184struct tee_shm *tee_shm_alloc_kernel_buf(struct tee_context *ctx, size_t size)
185{
186 u32 flags = TEE_SHM_DYNAMIC | TEE_SHM_POOL;
187
188 return shm_alloc_helper(ctx, size, PAGE_SIZE, flags, id: -1);
189}
190EXPORT_SYMBOL_GPL(tee_shm_alloc_kernel_buf);
191
192/**
193 * tee_shm_alloc_priv_buf() - Allocate shared memory for a privately shared
194 * kernel buffer
195 * @ctx: Context that allocates the shared memory
196 * @size: Requested size of shared memory
197 *
198 * This function returns similar shared memory as
199 * tee_shm_alloc_kernel_buf(), but with the difference that the memory
200 * might not be registered in secure world in case the driver supports
201 * passing memory not registered in advance.
202 *
203 * This function should normally only be used internally in the TEE
204 * drivers.
205 *
206 * @returns a pointer to 'struct tee_shm'
207 */
208struct tee_shm *tee_shm_alloc_priv_buf(struct tee_context *ctx, size_t size)
209{
210 u32 flags = TEE_SHM_PRIV | TEE_SHM_POOL;
211
212 return shm_alloc_helper(ctx, size, align: sizeof(long) * 2, flags, id: -1);
213}
214EXPORT_SYMBOL_GPL(tee_shm_alloc_priv_buf);
215
216static struct tee_shm *
217register_shm_helper(struct tee_context *ctx, unsigned long addr,
218 size_t length, u32 flags, int id)
219{
220 struct tee_device *teedev = ctx->teedev;
221 struct tee_shm *shm;
222 unsigned long start;
223 size_t num_pages;
224 void *ret;
225 int rc;
226
227 if (!tee_device_get(teedev))
228 return ERR_PTR(error: -EINVAL);
229
230 if (!teedev->desc->ops->shm_register ||
231 !teedev->desc->ops->shm_unregister) {
232 ret = ERR_PTR(error: -ENOTSUPP);
233 goto err_dev_put;
234 }
235
236 teedev_ctx_get(ctx);
237
238 shm = kzalloc(size: sizeof(*shm), GFP_KERNEL);
239 if (!shm) {
240 ret = ERR_PTR(error: -ENOMEM);
241 goto err_ctx_put;
242 }
243
244 refcount_set(r: &shm->refcount, n: 1);
245 shm->flags = flags;
246 shm->ctx = ctx;
247 shm->id = id;
248 addr = untagged_addr(addr);
249 start = rounddown(addr, PAGE_SIZE);
250 shm->offset = addr - start;
251 shm->size = length;
252 num_pages = (roundup(addr + length, PAGE_SIZE) - start) / PAGE_SIZE;
253 shm->pages = kcalloc(n: num_pages, size: sizeof(*shm->pages), GFP_KERNEL);
254 if (!shm->pages) {
255 ret = ERR_PTR(error: -ENOMEM);
256 goto err_free_shm;
257 }
258
259 if (flags & TEE_SHM_USER_MAPPED)
260 rc = pin_user_pages_fast(start, nr_pages: num_pages, gup_flags: FOLL_WRITE,
261 pages: shm->pages);
262 else
263 rc = shm_get_kernel_pages(start, page_count: num_pages, pages: shm->pages);
264 if (rc > 0)
265 shm->num_pages = rc;
266 if (rc != num_pages) {
267 if (rc >= 0)
268 rc = -ENOMEM;
269 ret = ERR_PTR(error: rc);
270 goto err_put_shm_pages;
271 }
272
273 rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,
274 shm->num_pages, start);
275 if (rc) {
276 ret = ERR_PTR(error: rc);
277 goto err_put_shm_pages;
278 }
279
280 return shm;
281err_put_shm_pages:
282 if (flags & TEE_SHM_USER_MAPPED)
283 unpin_user_pages(pages: shm->pages, npages: shm->num_pages);
284 else
285 shm_put_kernel_pages(pages: shm->pages, page_count: shm->num_pages);
286 kfree(objp: shm->pages);
287err_free_shm:
288 kfree(objp: shm);
289err_ctx_put:
290 teedev_ctx_put(ctx);
291err_dev_put:
292 tee_device_put(teedev);
293 return ret;
294}
295
296/**
297 * tee_shm_register_user_buf() - Register a userspace shared memory buffer
298 * @ctx: Context that registers the shared memory
299 * @addr: The userspace address of the shared buffer
300 * @length: Length of the shared buffer
301 *
302 * @returns a pointer to 'struct tee_shm'
303 */
304struct tee_shm *tee_shm_register_user_buf(struct tee_context *ctx,
305 unsigned long addr, size_t length)
306{
307 u32 flags = TEE_SHM_USER_MAPPED | TEE_SHM_DYNAMIC;
308 struct tee_device *teedev = ctx->teedev;
309 struct tee_shm *shm;
310 void *ret;
311 int id;
312
313 if (!access_ok((void __user *)addr, length))
314 return ERR_PTR(error: -EFAULT);
315
316 mutex_lock(&teedev->mutex);
317 id = idr_alloc(&teedev->idr, NULL, start: 1, end: 0, GFP_KERNEL);
318 mutex_unlock(lock: &teedev->mutex);
319 if (id < 0)
320 return ERR_PTR(error: id);
321
322 shm = register_shm_helper(ctx, addr, length, flags, id);
323 if (IS_ERR(ptr: shm)) {
324 mutex_lock(&teedev->mutex);
325 idr_remove(&teedev->idr, id);
326 mutex_unlock(lock: &teedev->mutex);
327 return shm;
328 }
329
330 mutex_lock(&teedev->mutex);
331 ret = idr_replace(&teedev->idr, shm, id);
332 mutex_unlock(lock: &teedev->mutex);
333 if (IS_ERR(ptr: ret)) {
334 tee_shm_free(shm);
335 return ret;
336 }
337
338 return shm;
339}
340
341/**
342 * tee_shm_register_kernel_buf() - Register kernel memory to be shared with
343 * secure world
344 * @ctx: Context that registers the shared memory
345 * @addr: The buffer
346 * @length: Length of the buffer
347 *
348 * @returns a pointer to 'struct tee_shm'
349 */
350
351struct tee_shm *tee_shm_register_kernel_buf(struct tee_context *ctx,
352 void *addr, size_t length)
353{
354 u32 flags = TEE_SHM_DYNAMIC;
355
356 return register_shm_helper(ctx, addr: (unsigned long)addr, length, flags, id: -1);
357}
358EXPORT_SYMBOL_GPL(tee_shm_register_kernel_buf);
359
360static int tee_shm_fop_release(struct inode *inode, struct file *filp)
361{
362 tee_shm_put(shm: filp->private_data);
363 return 0;
364}
365
366static int tee_shm_fop_mmap(struct file *filp, struct vm_area_struct *vma)
367{
368 struct tee_shm *shm = filp->private_data;
369 size_t size = vma->vm_end - vma->vm_start;
370
371 /* Refuse sharing shared memory provided by application */
372 if (shm->flags & TEE_SHM_USER_MAPPED)
373 return -EINVAL;
374
375 /* check for overflowing the buffer's size */
376 if (vma->vm_pgoff + vma_pages(vma) > shm->size >> PAGE_SHIFT)
377 return -EINVAL;
378
379 return remap_pfn_range(vma, addr: vma->vm_start, pfn: shm->paddr >> PAGE_SHIFT,
380 size, vma->vm_page_prot);
381}
382
383static const struct file_operations tee_shm_fops = {
384 .owner = THIS_MODULE,
385 .release = tee_shm_fop_release,
386 .mmap = tee_shm_fop_mmap,
387};
388
389/**
390 * tee_shm_get_fd() - Increase reference count and return file descriptor
391 * @shm: Shared memory handle
392 * @returns user space file descriptor to shared memory
393 */
394int tee_shm_get_fd(struct tee_shm *shm)
395{
396 int fd;
397
398 if (shm->id < 0)
399 return -EINVAL;
400
401 /* matched by tee_shm_put() in tee_shm_op_release() */
402 refcount_inc(r: &shm->refcount);
403 fd = anon_inode_getfd(name: "tee_shm", fops: &tee_shm_fops, priv: shm, O_RDWR);
404 if (fd < 0)
405 tee_shm_put(shm);
406 return fd;
407}
408
409/**
410 * tee_shm_free() - Free shared memory
411 * @shm: Handle to shared memory to free
412 */
413void tee_shm_free(struct tee_shm *shm)
414{
415 tee_shm_put(shm);
416}
417EXPORT_SYMBOL_GPL(tee_shm_free);
418
419/**
420 * tee_shm_get_va() - Get virtual address of a shared memory plus an offset
421 * @shm: Shared memory handle
422 * @offs: Offset from start of this shared memory
423 * @returns virtual address of the shared memory + offs if offs is within
424 * the bounds of this shared memory, else an ERR_PTR
425 */
426void *tee_shm_get_va(struct tee_shm *shm, size_t offs)
427{
428 if (!shm->kaddr)
429 return ERR_PTR(error: -EINVAL);
430 if (offs >= shm->size)
431 return ERR_PTR(error: -EINVAL);
432 return (char *)shm->kaddr + offs;
433}
434EXPORT_SYMBOL_GPL(tee_shm_get_va);
435
436/**
437 * tee_shm_get_pa() - Get physical address of a shared memory plus an offset
438 * @shm: Shared memory handle
439 * @offs: Offset from start of this shared memory
440 * @pa: Physical address to return
441 * @returns 0 if offs is within the bounds of this shared memory, else an
442 * error code.
443 */
444int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa)
445{
446 if (offs >= shm->size)
447 return -EINVAL;
448 if (pa)
449 *pa = shm->paddr + offs;
450 return 0;
451}
452EXPORT_SYMBOL_GPL(tee_shm_get_pa);
453
454/**
455 * tee_shm_get_from_id() - Find shared memory object and increase reference
456 * count
457 * @ctx: Context owning the shared memory
458 * @id: Id of shared memory object
459 * @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
460 */
461struct tee_shm *tee_shm_get_from_id(struct tee_context *ctx, int id)
462{
463 struct tee_device *teedev;
464 struct tee_shm *shm;
465
466 if (!ctx)
467 return ERR_PTR(error: -EINVAL);
468
469 teedev = ctx->teedev;
470 mutex_lock(&teedev->mutex);
471 shm = idr_find(&teedev->idr, id);
472 /*
473 * If the tee_shm was found in the IDR it must have a refcount
474 * larger than 0 due to the guarantee in tee_shm_put() below. So
475 * it's safe to use refcount_inc().
476 */
477 if (!shm || shm->ctx != ctx)
478 shm = ERR_PTR(error: -EINVAL);
479 else
480 refcount_inc(r: &shm->refcount);
481 mutex_unlock(lock: &teedev->mutex);
482 return shm;
483}
484EXPORT_SYMBOL_GPL(tee_shm_get_from_id);
485
486/**
487 * tee_shm_put() - Decrease reference count on a shared memory handle
488 * @shm: Shared memory handle
489 */
490void tee_shm_put(struct tee_shm *shm)
491{
492 struct tee_device *teedev = shm->ctx->teedev;
493 bool do_release = false;
494
495 mutex_lock(&teedev->mutex);
496 if (refcount_dec_and_test(r: &shm->refcount)) {
497 /*
498 * refcount has reached 0, we must now remove it from the
499 * IDR before releasing the mutex. This will guarantee that
500 * the refcount_inc() in tee_shm_get_from_id() never starts
501 * from 0.
502 */
503 if (shm->id >= 0)
504 idr_remove(&teedev->idr, id: shm->id);
505 do_release = true;
506 }
507 mutex_unlock(lock: &teedev->mutex);
508
509 if (do_release)
510 tee_shm_release(teedev, shm);
511}
512EXPORT_SYMBOL_GPL(tee_shm_put);
513

source code of linux/drivers/tee/tee_shm.c