1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (c) 2015-2021, Linaro Limited
4	* Copyright (c) 2016, EPAM Systems
5	*/
6
7	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9	#include <linux/arm-smccc.h>
10	#include <linux/cpuhotplug.h>
11	#include <linux/errno.h>
12	#include <linux/firmware.h>
13	#include <linux/interrupt.h>
14	#include <linux/io.h>
15	#include <linux/irqdomain.h>
16	#include <linux/kernel.h>
17	#include <linux/mm.h>
18	#include <linux/module.h>
19	#include <linux/of.h>
20	#include <linux/of_irq.h>
21	#include <linux/of_platform.h>
22	#include <linux/platform_device.h>
23	#include <linux/sched.h>
24	#include <linux/slab.h>
25	#include <linux/string.h>
26	#include <linux/tee_drv.h>
27	#include <linux/types.h>
28	#include <linux/workqueue.h>
29	#include "optee_private.h"
30	#include "optee_smc.h"
31	#include "optee_rpc_cmd.h"
32	#include <linux/kmemleak.h>
33	#define CREATE_TRACE_POINTS
34	#include "optee_trace.h"
35
36	/*
37	* This file implement the SMC ABI used when communicating with secure world
38	* OP-TEE OS via raw SMCs.
39	* This file is divided into the following sections:
40	* 1. Convert between struct tee_param and struct optee_msg_param
41	* 2. Low level support functions to register shared memory in secure world
42	* 3. Dynamic shared memory pool based on alloc_pages()
43	* 4. Do a normal scheduled call into secure world
44	* 5. Asynchronous notification
45	* 6. Driver initialization.
46	*/
47
48	/*
49	* A typical OP-TEE private shm allocation is 224 bytes (argument struct
50	* with 6 parameters, needed for open session). So with an alignment of 512
51	* we'll waste a bit more than 50%. However, it's only expected that we'll
52	* have a handful of these structs allocated at a time. Most memory will
53	* be allocated aligned to the page size, So all in all this should scale
54	* up and down quite well.
55	*/
56	#define OPTEE_MIN_STATIC_POOL_ALIGN 9 /* 512 bytes aligned */
57
58	/* SMC ABI considers at most a single TEE firmware */
59	static unsigned int pcpu_irq_num;
60
61	static int optee_cpuhp_enable_pcpu_irq(unsigned int cpu)
62	{
63	enable_percpu_irq(irq: pcpu_irq_num, type: IRQ_TYPE_NONE);
64
65	return 0;
66	}
67
68	static int optee_cpuhp_disable_pcpu_irq(unsigned int cpu)
69	{
70	disable_percpu_irq(irq: pcpu_irq_num);
71
72	return 0;
73	}
74
75	/*
76	* 1. Convert between struct tee_param and struct optee_msg_param
77	*
78	* optee_from_msg_param() and optee_to_msg_param() are the main
79	* functions.
80	*/
81
82	static int from_msg_param_tmp_mem(struct tee_param *p, u32 attr,
83	const struct optee_msg_param *mp)
84	{
85	struct tee_shm *shm;
86	phys_addr_t pa;
87	int rc;
88
89	p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
90	attr - OPTEE_MSG_ATTR_TYPE_TMEM_INPUT;
91	p->u.memref.size = mp->u.tmem.size;
92	shm = (struct tee_shm *)(unsigned long)mp->u.tmem.shm_ref;
93	if (!shm) {
94	p->u.memref.shm_offs = 0;
95	p->u.memref.shm = NULL;
96	return 0;
97	}
98
99	rc = tee_shm_get_pa(shm, offs: 0, pa: &pa);
100	if (rc)
101	return rc;
102
103	p->u.memref.shm_offs = mp->u.tmem.buf_ptr - pa;
104	p->u.memref.shm = shm;
105
106	return 0;
107	}
108
109	static void from_msg_param_reg_mem(struct tee_param *p, u32 attr,
110	const struct optee_msg_param *mp)
111	{
112	struct tee_shm *shm;
113
114	p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
115	attr - OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
116	p->u.memref.size = mp->u.rmem.size;
117	shm = (struct tee_shm *)(unsigned long)mp->u.rmem.shm_ref;
118
119	if (shm) {
120	p->u.memref.shm_offs = mp->u.rmem.offs;
121	p->u.memref.shm = shm;
122	} else {
123	p->u.memref.shm_offs = 0;
124	p->u.memref.shm = NULL;
125	}
126	}
127
128	/**
129	* optee_from_msg_param() - convert from OPTEE_MSG parameters to
130	* struct tee_param
131	* @optee: main service struct
132	* @params: subsystem internal parameter representation
133	* @num_params: number of elements in the parameter arrays
134	* @msg_params: OPTEE_MSG parameters
135	* Returns 0 on success or <0 on failure
136	*/
137	static int optee_from_msg_param(struct optee *optee, struct tee_param *params,
138	size_t num_params,
139	const struct optee_msg_param *msg_params)
140	{
141	int rc;
142	size_t n;
143
144	for (n = 0; n < num_params; n++) {
145	struct tee_param *p = params + n;
146	const struct optee_msg_param *mp = msg_params + n;
147	u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK;
148
149	switch (attr) {
150	case OPTEE_MSG_ATTR_TYPE_NONE:
151	p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
152	memset(&p->u, 0, sizeof(p->u));
153	break;
154	case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT:
155	case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT:
156	case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT:
157	optee_from_msg_param_value(p, attr, mp);
158	break;
159	case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT:
160	case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT:
161	case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT:
162	rc = from_msg_param_tmp_mem(p, attr, mp);
163	if (rc)
164	return rc;
165	break;
166	case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT:
167	case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT:
168	case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT:
169	from_msg_param_reg_mem(p, attr, mp);
170	break;
171
172	default:
173	return -EINVAL;
174	}
175	}
176	return 0;
177	}
178
179	static int to_msg_param_tmp_mem(struct optee_msg_param *mp,
180	const struct tee_param *p)
181	{
182	int rc;
183	phys_addr_t pa;
184
185	mp->attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + p->attr -
186	TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
187
188	mp->u.tmem.shm_ref = (unsigned long)p->u.memref.shm;
189	mp->u.tmem.size = p->u.memref.size;
190
191	if (!p->u.memref.shm) {
192	mp->u.tmem.buf_ptr = 0;
193	return 0;
194	}
195
196	rc = tee_shm_get_pa(shm: p->u.memref.shm, offs: p->u.memref.shm_offs, pa: &pa);
197	if (rc)
198	return rc;
199
200	mp->u.tmem.buf_ptr = pa;
201	mp->attr |= OPTEE_MSG_ATTR_CACHE_PREDEFINED <<
202	OPTEE_MSG_ATTR_CACHE_SHIFT;
203
204	return 0;
205	}
206
207	static int to_msg_param_reg_mem(struct optee_msg_param *mp,
208	const struct tee_param *p)
209	{
210	mp->attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT + p->attr -
211	TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
212
213	mp->u.rmem.shm_ref = (unsigned long)p->u.memref.shm;
214	mp->u.rmem.size = p->u.memref.size;
215	mp->u.rmem.offs = p->u.memref.shm_offs;
216	return 0;
217	}
218
219	/**
220	* optee_to_msg_param() - convert from struct tee_params to OPTEE_MSG parameters
221	* @optee: main service struct
222	* @msg_params: OPTEE_MSG parameters
223	* @num_params: number of elements in the parameter arrays
224	* @params: subsystem itnernal parameter representation
225	* Returns 0 on success or <0 on failure
226	*/
227	static int optee_to_msg_param(struct optee *optee,
228	struct optee_msg_param *msg_params,
229	size_t num_params, const struct tee_param *params)
230	{
231	int rc;
232	size_t n;
233
234	for (n = 0; n < num_params; n++) {
235	const struct tee_param *p = params + n;
236	struct optee_msg_param *mp = msg_params + n;
237
238	switch (p->attr) {
239	case TEE_IOCTL_PARAM_ATTR_TYPE_NONE:
240	mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
241	memset(&mp->u, 0, sizeof(mp->u));
242	break;
243	case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT:
244	case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT:
245	case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT:
246	optee_to_msg_param_value(mp, p);
247	break;
248	case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT:
249	case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT:
250	case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT:
251	if (tee_shm_is_dynamic(shm: p->u.memref.shm))
252	rc = to_msg_param_reg_mem(mp, p);
253	else
254	rc = to_msg_param_tmp_mem(mp, p);
255	if (rc)
256	return rc;
257	break;
258	default:
259	return -EINVAL;
260	}
261	}
262	return 0;
263	}
264
265	/*
266	* 2. Low level support functions to register shared memory in secure world
267	*
268	* Functions to enable/disable shared memory caching in secure world, that
269	* is, lazy freeing of previously allocated shared memory. Freeing is
270	* performed when a request has been compled.
271	*
272	* Functions to register and unregister shared memory both for normal
273	* clients and for tee-supplicant.
274	*/
275
276	/**
277	* optee_enable_shm_cache() - Enables caching of some shared memory allocation
278	* in OP-TEE
279	* @optee: main service struct
280	*/
281	static void optee_enable_shm_cache(struct optee *optee)
282	{
283	struct optee_call_waiter w;
284
285	/* We need to retry until secure world isn't busy. */
286	optee_cq_wait_init(cq: &optee->call_queue, w: &w);
287	while (true) {
288	struct arm_smccc_res res;
289
290	optee->smc.invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE,
291	0, 0, 0, 0, 0, 0, 0, &res);
292	if (res.a0 == OPTEE_SMC_RETURN_OK)
293	break;
294	optee_cq_wait_for_completion(cq: &optee->call_queue, w: &w);
295	}
296	optee_cq_wait_final(cq: &optee->call_queue, w: &w);
297	}
298
299	/**
300	* __optee_disable_shm_cache() - Disables caching of some shared memory
301	* allocation in OP-TEE
302	* @optee: main service struct
303	* @is_mapped: true if the cached shared memory addresses were mapped by this
304	* kernel, are safe to dereference, and should be freed
305	*/
306	static void __optee_disable_shm_cache(struct optee *optee, bool is_mapped)
307	{
308	struct optee_call_waiter w;
309
310	/* We need to retry until secure world isn't busy. */
311	optee_cq_wait_init(cq: &optee->call_queue, w: &w);
312	while (true) {
313	union {
314	struct arm_smccc_res smccc;
315	struct optee_smc_disable_shm_cache_result result;
316	} res;
317
318	optee->smc.invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE,
319	0, 0, 0, 0, 0, 0, 0, &res.smccc);
320	if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL)
321	break; /* All shm's freed */
322	if (res.result.status == OPTEE_SMC_RETURN_OK) {
323	struct tee_shm *shm;
324
325	/*
326	* Shared memory references that were not mapped by
327	* this kernel must be ignored to prevent a crash.
328	*/
329	if (!is_mapped)
330	continue;
331
332	shm = reg_pair_to_ptr(reg0: res.result.shm_upper32,
333	reg1: res.result.shm_lower32);
334	tee_shm_free(shm);
335	} else {
336	optee_cq_wait_for_completion(cq: &optee->call_queue, w: &w);
337	}
338	}
339	optee_cq_wait_final(cq: &optee->call_queue, w: &w);
340	}
341
342	/**
343	* optee_disable_shm_cache() - Disables caching of mapped shared memory
344	* allocations in OP-TEE
345	* @optee: main service struct
346	*/
347	static void optee_disable_shm_cache(struct optee *optee)
348	{
349	return __optee_disable_shm_cache(optee, is_mapped: true);
350	}
351
352	/**
353	* optee_disable_unmapped_shm_cache() - Disables caching of shared memory
354	* allocations in OP-TEE which are not
355	* currently mapped
356	* @optee: main service struct
357	*/
358	static void optee_disable_unmapped_shm_cache(struct optee *optee)
359	{
360	return __optee_disable_shm_cache(optee, is_mapped: false);
361	}
362
363	#define PAGELIST_ENTRIES_PER_PAGE \
364	((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
365
366	/*
367	* The final entry in each pagelist page is a pointer to the next
368	* pagelist page.
369	*/
370	static size_t get_pages_list_size(size_t num_entries)
371	{
372	int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
373
374	return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;
375	}
376
377	static u64 *optee_allocate_pages_list(size_t num_entries)
378	{
379	return alloc_pages_exact(size: get_pages_list_size(num_entries), GFP_KERNEL);
380	}
381
382	static void optee_free_pages_list(void *list, size_t num_entries)
383	{
384	free_pages_exact(virt: list, size: get_pages_list_size(num_entries));
385	}
386
387	/**
388	* optee_fill_pages_list() - write list of user pages to given shared
389	* buffer.
390	*
391	* @dst: page-aligned buffer where list of pages will be stored
392	* @pages: array of pages that represents shared buffer
393	* @num_pages: number of entries in @pages
394	* @page_offset: offset of user buffer from page start
395	*
396	* @dst should be big enough to hold list of user page addresses and
397	* links to the next pages of buffer
398	*/
399	static void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages,
400	size_t page_offset)
401	{
402	int n = 0;
403	phys_addr_t optee_page;
404	/*
405	* Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h
406	* for details.
407	*/
408	struct {
409	u64 pages_list[PAGELIST_ENTRIES_PER_PAGE];
410	u64 next_page_data;
411	} *pages_data;
412
413	/*
414	* Currently OP-TEE uses 4k page size and it does not looks
415	* like this will change in the future. On other hand, there are
416	* no know ARM architectures with page size < 4k.
417	* Thus the next built assert looks redundant. But the following
418	* code heavily relies on this assumption, so it is better be
419	* safe than sorry.
420	*/
421	BUILD_BUG_ON(PAGE_SIZE < OPTEE_MSG_NONCONTIG_PAGE_SIZE);
422
423	pages_data = (void *)dst;
424	/*
425	* If linux page is bigger than 4k, and user buffer offset is
426	* larger than 4k/8k/12k/etc this will skip first 4k pages,
427	* because they bear no value data for OP-TEE.
428	*/
429	optee_page = page_to_phys(*pages) +
430	round_down(page_offset, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
431
432	while (true) {
433	pages_data->pages_list[n++] = optee_page;
434
435	if (n == PAGELIST_ENTRIES_PER_PAGE) {
436	pages_data->next_page_data =
437	virt_to_phys(address: pages_data + 1);
438	pages_data++;
439	n = 0;
440	}
441
442	optee_page += OPTEE_MSG_NONCONTIG_PAGE_SIZE;
443	if (!(optee_page & ~PAGE_MASK)) {
444	if (!--num_pages)
445	break;
446	pages++;
447	optee_page = page_to_phys(*pages);
448	}
449	}
450	}
451
452	static int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm,
453	struct page **pages, size_t num_pages,
454	unsigned long start)
455	{
456	struct optee *optee = tee_get_drvdata(teedev: ctx->teedev);
457	struct optee_msg_arg *msg_arg;
458	struct tee_shm *shm_arg;
459	u64 *pages_list;
460	size_t sz;
461	int rc;
462
463	if (!num_pages)
464	return -EINVAL;
465
466	rc = optee_check_mem_type(start, num_pages);
467	if (rc)
468	return rc;
469
470	pages_list = optee_allocate_pages_list(num_entries: num_pages);
471	if (!pages_list)
472	return -ENOMEM;
473
474	/*
475	* We're about to register shared memory we can't register shared
476	* memory for this request or there's a catch-22.
477	*
478	* So in this we'll have to do the good old temporary private
479	* allocation instead of using optee_get_msg_arg().
480	*/
481	sz = optee_msg_arg_size(rpc_param_count: optee->rpc_param_count);
482	shm_arg = tee_shm_alloc_priv_buf(ctx, size: sz);
483	if (IS_ERR(ptr: shm_arg)) {
484	rc = PTR_ERR(ptr: shm_arg);
485	goto out;
486	}
487	msg_arg = tee_shm_get_va(shm: shm_arg, offs: 0);
488	if (IS_ERR(ptr: msg_arg)) {
489	rc = PTR_ERR(ptr: msg_arg);
490	goto out;
491	}
492
493	optee_fill_pages_list(dst: pages_list, pages, num_pages,
494	page_offset: tee_shm_get_page_offset(shm));
495
496	memset(msg_arg, 0, OPTEE_MSG_GET_ARG_SIZE(1));
497	msg_arg->num_params = 1;
498	msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM;
499	msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
500	OPTEE_MSG_ATTR_NONCONTIG;
501	msg_arg->params->u.tmem.shm_ref = (unsigned long)shm;
502	msg_arg->params->u.tmem.size = tee_shm_get_size(shm);
503	/*
504	* In the least bits of msg_arg->params->u.tmem.buf_ptr we
505	* store buffer offset from 4k page, as described in OP-TEE ABI.
506	*/
507	msg_arg->params->u.tmem.buf_ptr = virt_to_phys(address: pages_list) |
508	(tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
509
510	if (optee->ops->do_call_with_arg(ctx, shm_arg, 0) ||
511	msg_arg->ret != TEEC_SUCCESS)
512	rc = -EINVAL;
513
514	tee_shm_free(shm: shm_arg);
515	out:
516	optee_free_pages_list(list: pages_list, num_entries: num_pages);
517	return rc;
518	}
519
520	static int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm)
521	{
522	struct optee *optee = tee_get_drvdata(teedev: ctx->teedev);
523	struct optee_msg_arg *msg_arg;
524	struct tee_shm *shm_arg;
525	int rc = 0;
526	size_t sz;
527
528	/*
529	* We're about to unregister shared memory and we may not be able
530	* register shared memory for this request in case we're called
531	* from optee_shm_arg_cache_uninit().
532	*
533	* So in order to keep things simple in this function just as in
534	* optee_shm_register() we'll use temporary private allocation
535	* instead of using optee_get_msg_arg().
536	*/
537	sz = optee_msg_arg_size(rpc_param_count: optee->rpc_param_count);
538	shm_arg = tee_shm_alloc_priv_buf(ctx, size: sz);
539	if (IS_ERR(ptr: shm_arg))
540	return PTR_ERR(ptr: shm_arg);
541	msg_arg = tee_shm_get_va(shm: shm_arg, offs: 0);
542	if (IS_ERR(ptr: msg_arg)) {
543	rc = PTR_ERR(ptr: msg_arg);
544	goto out;
545	}
546
547	memset(msg_arg, 0, sz);
548	msg_arg->num_params = 1;
549	msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM;
550	msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
551	msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm;
552
553	if (optee->ops->do_call_with_arg(ctx, shm_arg, 0) ||
554	msg_arg->ret != TEEC_SUCCESS)
555	rc = -EINVAL;
556	out:
557	tee_shm_free(shm: shm_arg);
558	return rc;
559	}
560
561	static int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm,
562	struct page **pages, size_t num_pages,
563	unsigned long start)
564	{
565	/*
566	* We don't want to register supplicant memory in OP-TEE.
567	* Instead information about it will be passed in RPC code.
568	*/
569	return optee_check_mem_type(start, num_pages);
570	}
571
572	static int optee_shm_unregister_supp(struct tee_context *ctx,
573	struct tee_shm *shm)
574	{
575	return 0;
576	}
577
578	/*
579	* 3. Dynamic shared memory pool based on alloc_pages()
580	*
581	* Implements an OP-TEE specific shared memory pool which is used
582	* when dynamic shared memory is supported by secure world.
583	*
584	* The main function is optee_shm_pool_alloc_pages().
585	*/
586
587	static int pool_op_alloc(struct tee_shm_pool *pool,
588	struct tee_shm *shm, size_t size, size_t align)
589	{
590	/*
591	* Shared memory private to the OP-TEE driver doesn't need
592	* to be registered with OP-TEE.
593	*/
594	if (shm->flags & TEE_SHM_PRIV)
595	return optee_pool_op_alloc_helper(pool, shm, size, align, NULL);
596
597	return optee_pool_op_alloc_helper(pool, shm, size, align,
598	shm_register: optee_shm_register);
599	}
600
601	static void pool_op_free(struct tee_shm_pool *pool,
602	struct tee_shm *shm)
603	{
604	if (!(shm->flags & TEE_SHM_PRIV))
605	optee_pool_op_free_helper(pool, shm, shm_unregister: optee_shm_unregister);
606	else
607	optee_pool_op_free_helper(pool, shm, NULL);
608	}
609
610	static void pool_op_destroy_pool(struct tee_shm_pool *pool)
611	{
612	kfree(objp: pool);
613	}
614
615	static const struct tee_shm_pool_ops pool_ops = {
616	.alloc = pool_op_alloc,
617	.free = pool_op_free,
618	.destroy_pool = pool_op_destroy_pool,
619	};
620
621	/**
622	* optee_shm_pool_alloc_pages() - create page-based allocator pool
623	*
624	* This pool is used when OP-TEE supports dymanic SHM. In this case
625	* command buffers and such are allocated from kernel's own memory.
626	*/
627	static struct tee_shm_pool *optee_shm_pool_alloc_pages(void)
628	{
629	struct tee_shm_pool *pool = kzalloc(size: sizeof(*pool), GFP_KERNEL);
630
631	if (!pool)
632	return ERR_PTR(error: -ENOMEM);
633
634	pool->ops = &pool_ops;
635
636	return pool;
637	}
638
639	/*
640	* 4. Do a normal scheduled call into secure world
641	*
642	* The function optee_smc_do_call_with_arg() performs a normal scheduled
643	* call into secure world. During this call may normal world request help
644	* from normal world using RPCs, Remote Procedure Calls. This includes
645	* delivery of non-secure interrupts to for instance allow rescheduling of
646	* the current task.
647	*/
648
649	static void handle_rpc_func_cmd_shm_free(struct tee_context *ctx,
650	struct optee_msg_arg *arg)
651	{
652	struct tee_shm *shm;
653
654	arg->ret_origin = TEEC_ORIGIN_COMMS;
655
656	if (arg->num_params != 1 ||
657	arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
658	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
659	return;
660	}
661
662	shm = (struct tee_shm *)(unsigned long)arg->params[0].u.value.b;
663	switch (arg->params[0].u.value.a) {
664	case OPTEE_RPC_SHM_TYPE_APPL:
665	optee_rpc_cmd_free_suppl(ctx, shm);
666	break;
667	case OPTEE_RPC_SHM_TYPE_KERNEL:
668	tee_shm_free(shm);
669	break;
670	default:
671	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
672	}
673	arg->ret = TEEC_SUCCESS;
674	}
675
676	static void handle_rpc_func_cmd_shm_alloc(struct tee_context *ctx,
677	struct optee *optee,
678	struct optee_msg_arg *arg,
679	struct optee_call_ctx *call_ctx)
680	{
681	phys_addr_t pa;
682	struct tee_shm *shm;
683	size_t sz;
684	size_t n;
685
686	arg->ret_origin = TEEC_ORIGIN_COMMS;
687
688	if (!arg->num_params ||
689	arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
690	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
691	return;
692	}
693
694	for (n = 1; n < arg->num_params; n++) {
695	if (arg->params[n].attr != OPTEE_MSG_ATTR_TYPE_NONE) {
696	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
697	return;
698	}
699	}
700
701	sz = arg->params[0].u.value.b;
702	switch (arg->params[0].u.value.a) {
703	case OPTEE_RPC_SHM_TYPE_APPL:
704	shm = optee_rpc_cmd_alloc_suppl(ctx, sz);
705	break;
706	case OPTEE_RPC_SHM_TYPE_KERNEL:
707	shm = tee_shm_alloc_priv_buf(ctx: optee->ctx, size: sz);
708	break;
709	default:
710	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
711	return;
712	}
713
714	if (IS_ERR(ptr: shm)) {
715	arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
716	return;
717	}
718
719	if (tee_shm_get_pa(shm, offs: 0, pa: &pa)) {
720	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
721	goto bad;
722	}
723
724	sz = tee_shm_get_size(shm);
725
726	if (tee_shm_is_dynamic(shm)) {
727	struct page **pages;
728	u64 *pages_list;
729	size_t page_num;
730
731	pages = tee_shm_get_pages(shm, num_pages: &page_num);
732	if (!pages || !page_num) {
733	arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
734	goto bad;
735	}
736
737	pages_list = optee_allocate_pages_list(num_entries: page_num);
738	if (!pages_list) {
739	arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
740	goto bad;
741	}
742
743	call_ctx->pages_list = pages_list;
744	call_ctx->num_entries = page_num;
745
746	arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
747	OPTEE_MSG_ATTR_NONCONTIG;
748	/*
749	* In the least bits of u.tmem.buf_ptr we store buffer offset
750	* from 4k page, as described in OP-TEE ABI.
751	*/
752	arg->params[0].u.tmem.buf_ptr = virt_to_phys(address: pages_list) |
753	(tee_shm_get_page_offset(shm) &
754	(OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
755	arg->params[0].u.tmem.size = tee_shm_get_size(shm);
756	arg->params[0].u.tmem.shm_ref = (unsigned long)shm;
757
758	optee_fill_pages_list(dst: pages_list, pages, num_pages: page_num,
759	page_offset: tee_shm_get_page_offset(shm));
760	} else {
761	arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT;
762	arg->params[0].u.tmem.buf_ptr = pa;
763	arg->params[0].u.tmem.size = sz;
764	arg->params[0].u.tmem.shm_ref = (unsigned long)shm;
765	}
766
767	arg->ret = TEEC_SUCCESS;
768	return;
769	bad:
770	tee_shm_free(shm);
771	}
772
773	static void free_pages_list(struct optee_call_ctx *call_ctx)
774	{
775	if (call_ctx->pages_list) {
776	optee_free_pages_list(list: call_ctx->pages_list,
777	num_entries: call_ctx->num_entries);
778	call_ctx->pages_list = NULL;
779	call_ctx->num_entries = 0;
780	}
781	}
782
783	static void optee_rpc_finalize_call(struct optee_call_ctx *call_ctx)
784	{
785	free_pages_list(call_ctx);
786	}
787
788	static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee,
789	struct optee_msg_arg *arg,
790	struct optee_call_ctx *call_ctx)
791	{
792
793	switch (arg->cmd) {
794	case OPTEE_RPC_CMD_SHM_ALLOC:
795	free_pages_list(call_ctx);
796	handle_rpc_func_cmd_shm_alloc(ctx, optee, arg, call_ctx);
797	break;
798	case OPTEE_RPC_CMD_SHM_FREE:
799	handle_rpc_func_cmd_shm_free(ctx, arg);
800	break;
801	default:
802	optee_rpc_cmd(ctx, optee, arg);
803	}
804	}
805
806	/**
807	* optee_handle_rpc() - handle RPC from secure world
808	* @ctx: context doing the RPC
809	* @param: value of registers for the RPC
810	* @call_ctx: call context. Preserved during one OP-TEE invocation
811	*
812	* Result of RPC is written back into @param.
813	*/
814	static void optee_handle_rpc(struct tee_context *ctx,
815	struct optee_msg_arg *rpc_arg,
816	struct optee_rpc_param *param,
817	struct optee_call_ctx *call_ctx)
818	{
819	struct tee_device *teedev = ctx->teedev;
820	struct optee *optee = tee_get_drvdata(teedev);
821	struct optee_msg_arg *arg;
822	struct tee_shm *shm;
823	phys_addr_t pa;
824
825	switch (OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)) {
826	case OPTEE_SMC_RPC_FUNC_ALLOC:
827	shm = tee_shm_alloc_priv_buf(ctx: optee->ctx, size: param->a1);
828	if (!IS_ERR(ptr: shm) && !tee_shm_get_pa(shm, offs: 0, pa: &pa)) {
829	reg_pair_from_64(reg0: &param->a1, reg1: &param->a2, val: pa);
830	reg_pair_from_64(reg0: &param->a4, reg1: &param->a5,
831	val: (unsigned long)shm);
832	} else {
833	param->a1 = 0;
834	param->a2 = 0;
835	param->a4 = 0;
836	param->a5 = 0;
837	}
838	kmemleak_not_leak(ptr: shm);
839	break;
840	case OPTEE_SMC_RPC_FUNC_FREE:
841	shm = reg_pair_to_ptr(reg0: param->a1, reg1: param->a2);
842	tee_shm_free(shm);
843	break;
844	case OPTEE_SMC_RPC_FUNC_FOREIGN_INTR:
845	/*
846	* A foreign interrupt was raised while secure world was
847	* executing, since they are handled in Linux a dummy RPC is
848	* performed to let Linux take the interrupt through the normal
849	* vector.
850	*/
851	break;
852	case OPTEE_SMC_RPC_FUNC_CMD:
853	if (rpc_arg) {
854	arg = rpc_arg;
855	} else {
856	shm = reg_pair_to_ptr(reg0: param->a1, reg1: param->a2);
857	arg = tee_shm_get_va(shm, offs: 0);
858	if (IS_ERR(ptr: arg)) {
859	pr_err("%s: tee_shm_get_va %p failed\n",
860	__func__, shm);
861	break;
862	}
863	}
864
865	handle_rpc_func_cmd(ctx, optee, arg, call_ctx);
866	break;
867	default:
868	pr_warn("Unknown RPC func 0x%x\n",
869	(u32)OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0));
870	break;
871	}
872
873	param->a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC;
874	}
875
876	/**
877	* optee_smc_do_call_with_arg() - Do an SMC to OP-TEE in secure world
878	* @ctx: calling context
879	* @shm: shared memory holding the message to pass to secure world
880	* @offs: offset of the message in @shm
881	*
882	* Does and SMC to OP-TEE in secure world and handles eventual resulting
883	* Remote Procedure Calls (RPC) from OP-TEE.
884	*
885	* Returns return code from secure world, 0 is OK
886	*/
887	static int optee_smc_do_call_with_arg(struct tee_context *ctx,
888	struct tee_shm *shm, u_int offs)
889	{
890	struct optee *optee = tee_get_drvdata(teedev: ctx->teedev);
891	struct optee_call_waiter w;
892	struct optee_rpc_param param = { };
893	struct optee_call_ctx call_ctx = { };
894	struct optee_msg_arg *rpc_arg = NULL;
895	int rc;
896
897	if (optee->rpc_param_count) {
898	struct optee_msg_arg *arg;
899	unsigned int rpc_arg_offs;
900
901	arg = tee_shm_get_va(shm, offs);
902	if (IS_ERR(ptr: arg))
903	return PTR_ERR(ptr: arg);
904
905	rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params);
906	rpc_arg = tee_shm_get_va(shm, offs: offs + rpc_arg_offs);
907	if (IS_ERR(ptr: rpc_arg))
908	return PTR_ERR(ptr: rpc_arg);
909	}
910
911	if (rpc_arg && tee_shm_is_dynamic(shm)) {
912	param.a0 = OPTEE_SMC_CALL_WITH_REGD_ARG;
913	reg_pair_from_64(reg0: &param.a1, reg1: &param.a2, val: (u_long)shm);
914	param.a3 = offs;
915	} else {
916	phys_addr_t parg;
917
918	rc = tee_shm_get_pa(shm, offs, pa: &parg);
919	if (rc)
920	return rc;
921
922	if (rpc_arg)
923	param.a0 = OPTEE_SMC_CALL_WITH_RPC_ARG;
924	else
925	param.a0 = OPTEE_SMC_CALL_WITH_ARG;
926	reg_pair_from_64(reg0: &param.a1, reg1: &param.a2, val: parg);
927	}
928	/* Initialize waiter */
929	optee_cq_wait_init(cq: &optee->call_queue, w: &w);
930	while (true) {
931	struct arm_smccc_res res;
932
933	trace_optee_invoke_fn_begin(param: &param);
934	optee->smc.invoke_fn(param.a0, param.a1, param.a2, param.a3,
935	param.a4, param.a5, param.a6, param.a7,
936	&res);
937	trace_optee_invoke_fn_end(param: &param, res: &res);
938
939	if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) {
940	/*
941	* Out of threads in secure world, wait for a thread
942	* become available.
943	*/
944	optee_cq_wait_for_completion(cq: &optee->call_queue, w: &w);
945	} else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) {
946	cond_resched();
947	param.a0 = res.a0;
948	param.a1 = res.a1;
949	param.a2 = res.a2;
950	param.a3 = res.a3;
951	optee_handle_rpc(ctx, rpc_arg, param: &param, call_ctx: &call_ctx);
952	} else {
953	rc = res.a0;
954	break;
955	}
956	}
957
958	optee_rpc_finalize_call(call_ctx: &call_ctx);
959	/*
960	* We're done with our thread in secure world, if there's any
961	* thread waiters wake up one.
962	*/
963	optee_cq_wait_final(cq: &optee->call_queue, w: &w);
964
965	return rc;
966	}
967
968	static int simple_call_with_arg(struct tee_context *ctx, u32 cmd)
969	{
970	struct optee_shm_arg_entry *entry;
971	struct optee_msg_arg *msg_arg;
972	struct tee_shm *shm;
973	u_int offs;
974
975	msg_arg = optee_get_msg_arg(ctx, num_params: 0, entry: &entry, shm_ret: &shm, offs: &offs);
976	if (IS_ERR(ptr: msg_arg))
977	return PTR_ERR(ptr: msg_arg);
978
979	msg_arg->cmd = cmd;
980	optee_smc_do_call_with_arg(ctx, shm, offs);
981
982	optee_free_msg_arg(ctx, entry, offs);
983	return 0;
984	}
985
986	static int optee_smc_do_bottom_half(struct tee_context *ctx)
987	{
988	return simple_call_with_arg(ctx, OPTEE_MSG_CMD_DO_BOTTOM_HALF);
989	}
990
991	static int optee_smc_stop_async_notif(struct tee_context *ctx)
992	{
993	return simple_call_with_arg(ctx, OPTEE_MSG_CMD_STOP_ASYNC_NOTIF);
994	}
995
996	/*
997	* 5. Asynchronous notification
998	*/
999
1000	static u32 get_async_notif_value(optee_invoke_fn *invoke_fn, bool *value_valid,
1001	bool *value_pending)
1002	{
1003	struct arm_smccc_res res;
1004
1005	invoke_fn(OPTEE_SMC_GET_ASYNC_NOTIF_VALUE, 0, 0, 0, 0, 0, 0, 0, &res);
1006
1007	if (res.a0) {
1008	*value_valid = false;
1009	return 0;
1010	}
1011	*value_valid = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_VALID);
1012	*value_pending = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_PENDING);
1013	return res.a1;
1014	}
1015
1016	static irqreturn_t irq_handler(struct optee *optee)
1017	{
1018	bool do_bottom_half = false;
1019	bool value_valid;
1020	bool value_pending;
1021	u32 value;
1022
1023	do {
1024	value = get_async_notif_value(invoke_fn: optee->smc.invoke_fn,
1025	value_valid: &value_valid, value_pending: &value_pending);
1026	if (!value_valid)
1027	break;
1028
1029	if (value == OPTEE_SMC_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF)
1030	do_bottom_half = true;
1031	else
1032	optee_notif_send(optee, key: value);
1033	} while (value_pending);
1034
1035	if (do_bottom_half)
1036	return IRQ_WAKE_THREAD;
1037	return IRQ_HANDLED;
1038	}
1039
1040	static irqreturn_t notif_irq_handler(int irq, void *dev_id)
1041	{
1042	struct optee *optee = dev_id;
1043
1044	return irq_handler(optee);
1045	}
1046
1047	static irqreturn_t notif_irq_thread_fn(int irq, void *dev_id)
1048	{
1049	struct optee *optee = dev_id;
1050
1051	optee_smc_do_bottom_half(ctx: optee->ctx);
1052
1053	return IRQ_HANDLED;
1054	}
1055
1056	static int init_irq(struct optee *optee, u_int irq)
1057	{
1058	int rc;
1059
1060	rc = request_threaded_irq(irq, handler: notif_irq_handler,
1061	thread_fn: notif_irq_thread_fn,
1062	flags: 0, name: "optee_notification", dev: optee);
1063	if (rc)
1064	return rc;
1065
1066	optee->smc.notif_irq = irq;
1067
1068	return 0;
1069	}
1070
1071	static irqreturn_t notif_pcpu_irq_handler(int irq, void *dev_id)
1072	{
1073	struct optee_pcpu *pcpu = dev_id;
1074	struct optee *optee = pcpu->optee;
1075
1076	if (irq_handler(optee) == IRQ_WAKE_THREAD)
1077	queue_work(wq: optee->smc.notif_pcpu_wq,
1078	work: &optee->smc.notif_pcpu_work);
1079
1080	return IRQ_HANDLED;
1081	}
1082
1083	static void notif_pcpu_irq_work_fn(struct work_struct *work)
1084	{
1085	struct optee_smc *optee_smc = container_of(work, struct optee_smc,
1086	notif_pcpu_work);
1087	struct optee *optee = container_of(optee_smc, struct optee, smc);
1088
1089	optee_smc_do_bottom_half(ctx: optee->ctx);
1090	}
1091
1092	static int init_pcpu_irq(struct optee *optee, u_int irq)
1093	{
1094	struct optee_pcpu __percpu *optee_pcpu;
1095	int cpu, rc;
1096
1097	optee_pcpu = alloc_percpu(struct optee_pcpu);
1098	if (!optee_pcpu)
1099	return -ENOMEM;
1100
1101	for_each_present_cpu(cpu)
1102	per_cpu_ptr(optee_pcpu, cpu)->optee = optee;
1103
1104	rc = request_percpu_irq(irq, handler: notif_pcpu_irq_handler,
1105	devname: "optee_pcpu_notification", percpu_dev_id: optee_pcpu);
1106	if (rc)
1107	goto err_free_pcpu;
1108
1109	INIT_WORK(&optee->smc.notif_pcpu_work, notif_pcpu_irq_work_fn);
1110	optee->smc.notif_pcpu_wq = create_workqueue("optee_pcpu_notification");
1111	if (!optee->smc.notif_pcpu_wq) {
1112	rc = -EINVAL;
1113	goto err_free_pcpu_irq;
1114	}
1115
1116	optee->smc.optee_pcpu = optee_pcpu;
1117	optee->smc.notif_irq = irq;
1118
1119	pcpu_irq_num = irq;
1120	rc = cpuhp_setup_state(state: CPUHP_AP_ONLINE_DYN, name: "optee/pcpu-notif:starting",
1121	startup: optee_cpuhp_enable_pcpu_irq,
1122	teardown: optee_cpuhp_disable_pcpu_irq);
1123	if (!rc)
1124	rc = -EINVAL;
1125	if (rc < 0)
1126	goto err_free_pcpu_irq;
1127
1128	optee->smc.notif_cpuhp_state = rc;
1129
1130	return 0;
1131
1132	err_free_pcpu_irq:
1133	free_percpu_irq(irq, optee_pcpu);
1134	err_free_pcpu:
1135	free_percpu(pdata: optee_pcpu);
1136
1137	return rc;
1138	}
1139
1140	static int optee_smc_notif_init_irq(struct optee *optee, u_int irq)
1141	{
1142	if (irq_is_percpu_devid(irq))
1143	return init_pcpu_irq(optee, irq);
1144	else
1145	return init_irq(optee, irq);
1146	}
1147
1148	static void uninit_pcpu_irq(struct optee *optee)
1149	{
1150	cpuhp_remove_state(state: optee->smc.notif_cpuhp_state);
1151
1152	destroy_workqueue(wq: optee->smc.notif_pcpu_wq);
1153
1154	free_percpu_irq(optee->smc.notif_irq, optee->smc.optee_pcpu);
1155	free_percpu(pdata: optee->smc.optee_pcpu);
1156	}
1157
1158	static void optee_smc_notif_uninit_irq(struct optee *optee)
1159	{
1160	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
1161	optee_smc_stop_async_notif(ctx: optee->ctx);
1162	if (optee->smc.notif_irq) {
1163	if (irq_is_percpu_devid(irq: optee->smc.notif_irq))
1164	uninit_pcpu_irq(optee);
1165	else
1166	free_irq(optee->smc.notif_irq, optee);
1167
1168	irq_dispose_mapping(virq: optee->smc.notif_irq);
1169	}
1170	}
1171	}
1172
1173	/*
1174	* 6. Driver initialization
1175	*
1176	* During driver initialization is secure world probed to find out which
1177	* features it supports so the driver can be initialized with a matching
1178	* configuration. This involves for instance support for dynamic shared
1179	* memory instead of a static memory carvout.
1180	*/
1181
1182	static void optee_get_version(struct tee_device *teedev,
1183	struct tee_ioctl_version_data *vers)
1184	{
1185	struct tee_ioctl_version_data v = {
1186	.impl_id = TEE_IMPL_ID_OPTEE,
1187	.impl_caps = TEE_OPTEE_CAP_TZ,
1188	.gen_caps = TEE_GEN_CAP_GP,
1189	};
1190	struct optee *optee = tee_get_drvdata(teedev);
1191
1192	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
1193	v.gen_caps |= TEE_GEN_CAP_REG_MEM;
1194	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL)
1195	v.gen_caps |= TEE_GEN_CAP_MEMREF_NULL;
1196	*vers = v;
1197	}
1198
1199	static int optee_smc_open(struct tee_context *ctx)
1200	{
1201	struct optee *optee = tee_get_drvdata(teedev: ctx->teedev);
1202	u32 sec_caps = optee->smc.sec_caps;
1203
1204	return optee_open(ctx, cap_memref_null: sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL);
1205	}
1206
1207	static const struct tee_driver_ops optee_clnt_ops = {
1208	.get_version = optee_get_version,
1209	.open = optee_smc_open,
1210	.release = optee_release,
1211	.open_session = optee_open_session,
1212	.close_session = optee_close_session,
1213	.invoke_func = optee_invoke_func,
1214	.cancel_req = optee_cancel_req,
1215	.shm_register = optee_shm_register,
1216	.shm_unregister = optee_shm_unregister,
1217	};
1218
1219	static const struct tee_desc optee_clnt_desc = {
1220	.name = DRIVER_NAME "-clnt",
1221	.ops = &optee_clnt_ops,
1222	.owner = THIS_MODULE,
1223	};
1224
1225	static const struct tee_driver_ops optee_supp_ops = {
1226	.get_version = optee_get_version,
1227	.open = optee_smc_open,
1228	.release = optee_release_supp,
1229	.supp_recv = optee_supp_recv,
1230	.supp_send = optee_supp_send,
1231	.shm_register = optee_shm_register_supp,
1232	.shm_unregister = optee_shm_unregister_supp,
1233	};
1234
1235	static const struct tee_desc optee_supp_desc = {
1236	.name = DRIVER_NAME "-supp",
1237	.ops = &optee_supp_ops,
1238	.owner = THIS_MODULE,
1239	.flags = TEE_DESC_PRIVILEGED,
1240	};
1241
1242	static const struct optee_ops optee_ops = {
1243	.do_call_with_arg = optee_smc_do_call_with_arg,
1244	.to_msg_param = optee_to_msg_param,
1245	.from_msg_param = optee_from_msg_param,
1246	};
1247
1248	static int enable_async_notif(optee_invoke_fn *invoke_fn)
1249	{
1250	struct arm_smccc_res res;
1251
1252	invoke_fn(OPTEE_SMC_ENABLE_ASYNC_NOTIF, 0, 0, 0, 0, 0, 0, 0, &res);
1253
1254	if (res.a0)
1255	return -EINVAL;
1256	return 0;
1257	}
1258
1259	static bool optee_msg_api_uid_is_optee_api(optee_invoke_fn *invoke_fn)
1260	{
1261	struct arm_smccc_res res;
1262
1263	invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1264
1265	if (res.a0 == OPTEE_MSG_UID_0 && res.a1 == OPTEE_MSG_UID_1 &&
1266	res.a2 == OPTEE_MSG_UID_2 && res.a3 == OPTEE_MSG_UID_3)
1267	return true;
1268	return false;
1269	}
1270
1271	#ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
1272	static bool optee_msg_api_uid_is_optee_image_load(optee_invoke_fn *invoke_fn)
1273	{
1274	struct arm_smccc_res res;
1275
1276	invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1277
1278	if (res.a0 == OPTEE_MSG_IMAGE_LOAD_UID_0 &&
1279	res.a1 == OPTEE_MSG_IMAGE_LOAD_UID_1 &&
1280	res.a2 == OPTEE_MSG_IMAGE_LOAD_UID_2 &&
1281	res.a3 == OPTEE_MSG_IMAGE_LOAD_UID_3)
1282	return true;
1283	return false;
1284	}
1285	#endif
1286
1287	static void optee_msg_get_os_revision(optee_invoke_fn *invoke_fn)
1288	{
1289	union {
1290	struct arm_smccc_res smccc;
1291	struct optee_smc_call_get_os_revision_result result;
1292	} res = {
1293	.result = {
1294	.build_id = 0
1295	}
1296	};
1297
1298	invoke_fn(OPTEE_SMC_CALL_GET_OS_REVISION, 0, 0, 0, 0, 0, 0, 0,
1299	&res.smccc);
1300
1301	if (res.result.build_id)
1302	pr_info("revision %lu.%lu (%08lx)", res.result.major,
1303	res.result.minor, res.result.build_id);
1304	else
1305	pr_info("revision %lu.%lu", res.result.major, res.result.minor);
1306	}
1307
1308	static bool optee_msg_api_revision_is_compatible(optee_invoke_fn *invoke_fn)
1309	{
1310	union {
1311	struct arm_smccc_res smccc;
1312	struct optee_smc_calls_revision_result result;
1313	} res;
1314
1315	invoke_fn(OPTEE_SMC_CALLS_REVISION, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
1316
1317	if (res.result.major == OPTEE_MSG_REVISION_MAJOR &&
1318	(int)res.result.minor >= OPTEE_MSG_REVISION_MINOR)
1319	return true;
1320	return false;
1321	}
1322
1323	static bool optee_msg_exchange_capabilities(optee_invoke_fn *invoke_fn,
1324	u32 *sec_caps, u32 *max_notif_value,
1325	unsigned int *rpc_param_count)
1326	{
1327	union {
1328	struct arm_smccc_res smccc;
1329	struct optee_smc_exchange_capabilities_result result;
1330	} res;
1331	u32 a1 = 0;
1332
1333	/*
1334	* TODO This isn't enough to tell if it's UP system (from kernel
1335	* point of view) or not, is_smp() returns the information
1336	* needed, but can't be called directly from here.
1337	*/
1338	if (!IS_ENABLED(CONFIG_SMP) || nr_cpu_ids == 1)
1339	a1 |= OPTEE_SMC_NSEC_CAP_UNIPROCESSOR;
1340
1341	invoke_fn(OPTEE_SMC_EXCHANGE_CAPABILITIES, a1, 0, 0, 0, 0, 0, 0,
1342	&res.smccc);
1343
1344	if (res.result.status != OPTEE_SMC_RETURN_OK)
1345	return false;
1346
1347	*sec_caps = res.result.capabilities;
1348	if (*sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF)
1349	*max_notif_value = res.result.max_notif_value;
1350	else
1351	*max_notif_value = OPTEE_DEFAULT_MAX_NOTIF_VALUE;
1352	if (*sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
1353	*rpc_param_count = (u8)res.result.data;
1354	else
1355	*rpc_param_count = 0;
1356
1357	return true;
1358	}
1359
1360	static struct tee_shm_pool *
1361	optee_config_shm_memremap(optee_invoke_fn *invoke_fn, void **memremaped_shm)
1362	{
1363	union {
1364	struct arm_smccc_res smccc;
1365	struct optee_smc_get_shm_config_result result;
1366	} res;
1367	unsigned long vaddr;
1368	phys_addr_t paddr;
1369	size_t size;
1370	phys_addr_t begin;
1371	phys_addr_t end;
1372	void *va;
1373	void *rc;
1374
1375	invoke_fn(OPTEE_SMC_GET_SHM_CONFIG, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
1376	if (res.result.status != OPTEE_SMC_RETURN_OK) {
1377	pr_err("static shm service not available\n");
1378	return ERR_PTR(error: -ENOENT);
1379	}
1380
1381	if (res.result.settings != OPTEE_SMC_SHM_CACHED) {
1382	pr_err("only normal cached shared memory supported\n");
1383	return ERR_PTR(error: -EINVAL);
1384	}
1385
1386	begin = roundup(res.result.start, PAGE_SIZE);
1387	end = rounddown(res.result.start + res.result.size, PAGE_SIZE);
1388	paddr = begin;
1389	size = end - begin;
1390
1391	va = memremap(offset: paddr, size, flags: MEMREMAP_WB);
1392	if (!va) {
1393	pr_err("shared memory ioremap failed\n");
1394	return ERR_PTR(error: -EINVAL);
1395	}
1396	vaddr = (unsigned long)va;
1397
1398	rc = tee_shm_pool_alloc_res_mem(vaddr, paddr, size,
1399	OPTEE_MIN_STATIC_POOL_ALIGN);
1400	if (IS_ERR(ptr: rc))
1401	memunmap(addr: va);
1402	else
1403	*memremaped_shm = va;
1404
1405	return rc;
1406	}
1407
1408	/* Simple wrapper functions to be able to use a function pointer */
1409	static void optee_smccc_smc(unsigned long a0, unsigned long a1,
1410	unsigned long a2, unsigned long a3,
1411	unsigned long a4, unsigned long a5,
1412	unsigned long a6, unsigned long a7,
1413	struct arm_smccc_res *res)
1414	{
1415	arm_smccc_smc(a0, a1, a2, a3, a4, a5, a6, a7, res);
1416	}
1417
1418	static void optee_smccc_hvc(unsigned long a0, unsigned long a1,
1419	unsigned long a2, unsigned long a3,
1420	unsigned long a4, unsigned long a5,
1421	unsigned long a6, unsigned long a7,
1422	struct arm_smccc_res *res)
1423	{
1424	arm_smccc_hvc(a0, a1, a2, a3, a4, a5, a6, a7, res);
1425	}
1426
1427	static optee_invoke_fn *get_invoke_func(struct device *dev)
1428	{
1429	const char *method;
1430
1431	pr_info("probing for conduit method.\n");
1432
1433	if (device_property_read_string(dev, propname: "method", val: &method)) {
1434	pr_warn("missing \"method\" property\n");
1435	return ERR_PTR(error: -ENXIO);
1436	}
1437
1438	if (!strcmp("hvc", method))
1439	return optee_smccc_hvc;
1440	else if (!strcmp("smc", method))
1441	return optee_smccc_smc;
1442
1443	pr_warn("invalid \"method\" property: %s\n", method);
1444	return ERR_PTR(error: -EINVAL);
1445	}
1446
1447	/* optee_remove - Device Removal Routine
1448	* @pdev: platform device information struct
1449	*
1450	* optee_remove is called by platform subsystem to alert the driver
1451	* that it should release the device
1452	*/
1453	static int optee_smc_remove(struct platform_device *pdev)
1454	{
1455	struct optee *optee = platform_get_drvdata(pdev);
1456
1457	/*
1458	* Ask OP-TEE to free all cached shared memory objects to decrease
1459	* reference counters and also avoid wild pointers in secure world
1460	* into the old shared memory range.
1461	*/
1462	if (!optee->rpc_param_count)
1463	optee_disable_shm_cache(optee);
1464
1465	optee_smc_notif_uninit_irq(optee);
1466
1467	optee_remove_common(optee);
1468
1469	if (optee->smc.memremaped_shm)
1470	memunmap(addr: optee->smc.memremaped_shm);
1471
1472	kfree(objp: optee);
1473
1474	return 0;
1475	}
1476
1477	/* optee_shutdown - Device Removal Routine
1478	* @pdev: platform device information struct
1479	*
1480	* platform_shutdown is called by the platform subsystem to alert
1481	* the driver that a shutdown, reboot, or kexec is happening and
1482	* device must be disabled.
1483	*/
1484	static void optee_shutdown(struct platform_device *pdev)
1485	{
1486	struct optee *optee = platform_get_drvdata(pdev);
1487
1488	if (!optee->rpc_param_count)
1489	optee_disable_shm_cache(optee);
1490	}
1491
1492	#ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
1493
1494	#define OPTEE_FW_IMAGE "optee/tee.bin"
1495
1496	static optee_invoke_fn *cpuhp_invoke_fn;
1497
1498	static int optee_cpuhp_probe(unsigned int cpu)
1499	{
1500	/*
1501	* Invoking a call on a CPU will cause OP-TEE to perform the required
1502	* setup for that CPU. Just invoke the call to get the UID since that
1503	* has no side effects.
1504	*/
1505	if (optee_msg_api_uid_is_optee_api(cpuhp_invoke_fn))
1506	return 0;
1507	else
1508	return -EINVAL;
1509	}
1510
1511	static int optee_load_fw(struct platform_device *pdev,
1512	optee_invoke_fn *invoke_fn)
1513	{
1514	const struct firmware *fw = NULL;
1515	struct arm_smccc_res res;
1516	phys_addr_t data_pa;
1517	u8 *data_buf = NULL;
1518	u64 data_size;
1519	u32 data_pa_high, data_pa_low;
1520	u32 data_size_high, data_size_low;
1521	int rc;
1522	int hp_state;
1523
1524	if (!optee_msg_api_uid_is_optee_image_load(invoke_fn))
1525	return 0;
1526
1527	rc = request_firmware(&fw, OPTEE_FW_IMAGE, &pdev->dev);
1528	if (rc) {
1529	/*
1530	* The firmware in the rootfs will not be accessible until we
1531	* are in the SYSTEM_RUNNING state, so return EPROBE_DEFER until
1532	* that point.
1533	*/
1534	if (system_state < SYSTEM_RUNNING)
1535	return -EPROBE_DEFER;
1536	goto fw_err;
1537	}
1538
1539	data_size = fw->size;
1540	/*
1541	* This uses the GFP_DMA flag to ensure we are allocated memory in the
1542	* 32-bit space since TF-A cannot map memory beyond the 32-bit boundary.
1543	*/
1544	data_buf = kmemdup(fw->data, fw->size, GFP_KERNEL | GFP_DMA);
1545	if (!data_buf) {
1546	rc = -ENOMEM;
1547	goto fw_err;
1548	}
1549	data_pa = virt_to_phys(data_buf);
1550	reg_pair_from_64(&data_pa_high, &data_pa_low, data_pa);
1551	reg_pair_from_64(&data_size_high, &data_size_low, data_size);
1552	goto fw_load;
1553
1554	fw_err:
1555	pr_warn("image loading failed\n");
1556	data_pa_high = 0;
1557	data_pa_low = 0;
1558	data_size_high = 0;
1559	data_size_low = 0;
1560
1561	fw_load:
1562	/*
1563	* Always invoke the SMC, even if loading the image fails, to indicate
1564	* to EL3 that we have passed the point where it should allow invoking
1565	* this SMC.
1566	*/
1567	pr_warn("OP-TEE image loaded from kernel, this can be insecure");
1568	invoke_fn(OPTEE_SMC_CALL_LOAD_IMAGE, data_size_high, data_size_low,
1569	data_pa_high, data_pa_low, 0, 0, 0, &res);
1570	if (!rc)
1571	rc = res.a0;
1572	if (fw)
1573	release_firmware(fw);
1574	kfree(data_buf);
1575
1576	if (!rc) {
1577	/*
1578	* We need to initialize OP-TEE on all other running cores as
1579	* well. Any cores that aren't running yet will get initialized
1580	* when they are brought up by the power management functions in
1581	* TF-A which are registered by the OP-TEE SPD. Due to that we
1582	* can un-register the callback right after registering it.
1583	*/
1584	cpuhp_invoke_fn = invoke_fn;
1585	hp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee:probe",
1586	optee_cpuhp_probe, NULL);
1587	if (hp_state < 0) {
1588	pr_warn("Failed with CPU hotplug setup for OP-TEE");
1589	return -EINVAL;
1590	}
1591	cpuhp_remove_state(hp_state);
1592	cpuhp_invoke_fn = NULL;
1593	}
1594
1595	return rc;
1596	}
1597	#else
1598	static inline int optee_load_fw(struct platform_device *pdev,
1599	optee_invoke_fn *invoke_fn)
1600	{
1601	return 0;
1602	}
1603	#endif
1604
1605	static int optee_probe(struct platform_device *pdev)
1606	{
1607	optee_invoke_fn *invoke_fn;
1608	struct tee_shm_pool *pool = ERR_PTR(error: -EINVAL);
1609	struct optee *optee = NULL;
1610	void *memremaped_shm = NULL;
1611	unsigned int rpc_param_count;
1612	struct tee_device *teedev;
1613	struct tee_context *ctx;
1614	u32 max_notif_value;
1615	u32 arg_cache_flags;
1616	u32 sec_caps;
1617	int rc;
1618
1619	invoke_fn = get_invoke_func(dev: &pdev->dev);
1620	if (IS_ERR(ptr: invoke_fn))
1621	return PTR_ERR(ptr: invoke_fn);
1622
1623	rc = optee_load_fw(pdev, invoke_fn);
1624	if (rc)
1625	return rc;
1626
1627	if (!optee_msg_api_uid_is_optee_api(invoke_fn)) {
1628	pr_warn("api uid mismatch\n");
1629	return -EINVAL;
1630	}
1631
1632	optee_msg_get_os_revision(invoke_fn);
1633
1634	if (!optee_msg_api_revision_is_compatible(invoke_fn)) {
1635	pr_warn("api revision mismatch\n");
1636	return -EINVAL;
1637	}
1638
1639	if (!optee_msg_exchange_capabilities(invoke_fn, sec_caps: &sec_caps,
1640	max_notif_value: &max_notif_value,
1641	rpc_param_count: &rpc_param_count)) {
1642	pr_warn("capabilities mismatch\n");
1643	return -EINVAL;
1644	}
1645
1646	/*
1647	* Try to use dynamic shared memory if possible
1648	*/
1649	if (sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) {
1650	/*
1651	* If we have OPTEE_SMC_SEC_CAP_RPC_ARG we can ask
1652	* optee_get_msg_arg() to pre-register (by having
1653	* OPTEE_SHM_ARG_ALLOC_PRIV cleared) the page used to pass
1654	* an argument struct.
1655	*
1656	* With the page is pre-registered we can use a non-zero
1657	* offset for argument struct, this is indicated with
1658	* OPTEE_SHM_ARG_SHARED.
1659	*
1660	* This means that optee_smc_do_call_with_arg() will use
1661	* OPTEE_SMC_CALL_WITH_REGD_ARG for pre-registered pages.
1662	*/
1663	if (sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
1664	arg_cache_flags = OPTEE_SHM_ARG_SHARED;
1665	else
1666	arg_cache_flags = OPTEE_SHM_ARG_ALLOC_PRIV;
1667
1668	pool = optee_shm_pool_alloc_pages();
1669	}
1670
1671	/*
1672	* If dynamic shared memory is not available or failed - try static one
1673	*/
1674	if (IS_ERR(ptr: pool) && (sec_caps & OPTEE_SMC_SEC_CAP_HAVE_RESERVED_SHM)) {
1675	/*
1676	* The static memory pool can use non-zero page offsets so
1677	* let optee_get_msg_arg() know that with OPTEE_SHM_ARG_SHARED.
1678	*
1679	* optee_get_msg_arg() should not pre-register the
1680	* allocated page used to pass an argument struct, this is
1681	* indicated with OPTEE_SHM_ARG_ALLOC_PRIV.
1682	*
1683	* This means that optee_smc_do_call_with_arg() will use
1684	* OPTEE_SMC_CALL_WITH_ARG if rpc_param_count is 0, else
1685	* OPTEE_SMC_CALL_WITH_RPC_ARG.
1686	*/
1687	arg_cache_flags = OPTEE_SHM_ARG_SHARED |
1688	OPTEE_SHM_ARG_ALLOC_PRIV;
1689	pool = optee_config_shm_memremap(invoke_fn, memremaped_shm: &memremaped_shm);
1690	}
1691
1692	if (IS_ERR(ptr: pool))
1693	return PTR_ERR(ptr: pool);
1694
1695	optee = kzalloc(size: sizeof(*optee), GFP_KERNEL);
1696	if (!optee) {
1697	rc = -ENOMEM;
1698	goto err_free_pool;
1699	}
1700
1701	optee->ops = &optee_ops;
1702	optee->smc.invoke_fn = invoke_fn;
1703	optee->smc.sec_caps = sec_caps;
1704	optee->rpc_param_count = rpc_param_count;
1705
1706	teedev = tee_device_alloc(teedesc: &optee_clnt_desc, NULL, pool, driver_data: optee);
1707	if (IS_ERR(ptr: teedev)) {
1708	rc = PTR_ERR(ptr: teedev);
1709	goto err_free_optee;
1710	}
1711	optee->teedev = teedev;
1712
1713	teedev = tee_device_alloc(teedesc: &optee_supp_desc, NULL, pool, driver_data: optee);
1714	if (IS_ERR(ptr: teedev)) {
1715	rc = PTR_ERR(ptr: teedev);
1716	goto err_unreg_teedev;
1717	}
1718	optee->supp_teedev = teedev;
1719
1720	rc = tee_device_register(teedev: optee->teedev);
1721	if (rc)
1722	goto err_unreg_supp_teedev;
1723
1724	rc = tee_device_register(teedev: optee->supp_teedev);
1725	if (rc)
1726	goto err_unreg_supp_teedev;
1727
1728	mutex_init(&optee->call_queue.mutex);
1729	INIT_LIST_HEAD(list: &optee->call_queue.waiters);
1730	optee_supp_init(supp: &optee->supp);
1731	optee->smc.memremaped_shm = memremaped_shm;
1732	optee->pool = pool;
1733	optee_shm_arg_cache_init(optee, flags: arg_cache_flags);
1734
1735	platform_set_drvdata(pdev, data: optee);
1736	ctx = teedev_open(teedev: optee->teedev);
1737	if (IS_ERR(ptr: ctx)) {
1738	rc = PTR_ERR(ptr: ctx);
1739	goto err_supp_uninit;
1740	}
1741	optee->ctx = ctx;
1742	rc = optee_notif_init(optee, max_key: max_notif_value);
1743	if (rc)
1744	goto err_close_ctx;
1745
1746	if (sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
1747	unsigned int irq;
1748
1749	rc = platform_get_irq(pdev, 0);
1750	if (rc < 0) {
1751	pr_err("platform_get_irq: ret %d\n", rc);
1752	goto err_notif_uninit;
1753	}
1754	irq = rc;
1755
1756	rc = optee_smc_notif_init_irq(optee, irq);
1757	if (rc) {
1758	irq_dispose_mapping(virq: irq);
1759	goto err_notif_uninit;
1760	}
1761	enable_async_notif(invoke_fn: optee->smc.invoke_fn);
1762	pr_info("Asynchronous notifications enabled\n");
1763	}
1764
1765	/*
1766	* Ensure that there are no pre-existing shm objects before enabling
1767	* the shm cache so that there's no chance of receiving an invalid
1768	* address during shutdown. This could occur, for example, if we're
1769	* kexec booting from an older kernel that did not properly cleanup the
1770	* shm cache.
1771	*/
1772	optee_disable_unmapped_shm_cache(optee);
1773
1774	/*
1775	* Only enable the shm cache in case we're not able to pass the RPC
1776	* arg struct right after the normal arg struct.
1777	*/
1778	if (!optee->rpc_param_count)
1779	optee_enable_shm_cache(optee);
1780
1781	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
1782	pr_info("dynamic shared memory is enabled\n");
1783
1784	rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES);
1785	if (rc)
1786	goto err_disable_shm_cache;
1787
1788	pr_info("initialized driver\n");
1789	return 0;
1790
1791	err_disable_shm_cache:
1792	if (!optee->rpc_param_count)
1793	optee_disable_shm_cache(optee);
1794	optee_smc_notif_uninit_irq(optee);
1795	optee_unregister_devices();
1796	err_notif_uninit:
1797	optee_notif_uninit(optee);
1798	err_close_ctx:
1799	teedev_close_context(ctx);
1800	err_supp_uninit:
1801	optee_shm_arg_cache_uninit(optee);
1802	optee_supp_uninit(supp: &optee->supp);
1803	mutex_destroy(lock: &optee->call_queue.mutex);
1804	err_unreg_supp_teedev:
1805	tee_device_unregister(teedev: optee->supp_teedev);
1806	err_unreg_teedev:
1807	tee_device_unregister(teedev: optee->teedev);
1808	err_free_optee:
1809	kfree(objp: optee);
1810	err_free_pool:
1811	tee_shm_pool_free(pool);
1812	if (memremaped_shm)
1813	memunmap(addr: memremaped_shm);
1814	return rc;
1815	}
1816
1817	static const struct of_device_id optee_dt_match[] = {
1818	{ .compatible = "linaro,optee-tz" },
1819	{},
1820	};
1821	MODULE_DEVICE_TABLE(of, optee_dt_match);
1822
1823	static struct platform_driver optee_driver = {
1824	.probe = optee_probe,
1825	.remove = optee_smc_remove,
1826	.shutdown = optee_shutdown,
1827	.driver = {
1828	.name = "optee",
1829	.of_match_table = optee_dt_match,
1830	},
1831	};
1832
1833	int optee_smc_abi_register(void)
1834	{
1835	return platform_driver_register(&optee_driver);
1836	}
1837
1838	void optee_smc_abi_unregister(void)
1839	{
1840	platform_driver_unregister(&optee_driver);
1841	}
1842