1	// SPDX-License-Identifier: GPL-2.0
2	// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
3	// Copyright (c) 2018, Linaro Limited
4
5	#include <linux/completion.h>
6	#include <linux/device.h>
7	#include <linux/dma-buf.h>
8	#include <linux/dma-mapping.h>
9	#include <linux/dma-resv.h>
10	#include <linux/idr.h>
11	#include <linux/list.h>
12	#include <linux/miscdevice.h>
13	#include <linux/module.h>
14	#include <linux/of_address.h>
15	#include <linux/of.h>
16	#include <linux/platform_device.h>
17	#include <linux/sort.h>
18	#include <linux/of_platform.h>
19	#include <linux/rpmsg.h>
20	#include <linux/scatterlist.h>
21	#include <linux/slab.h>
22	#include <linux/firmware/qcom/qcom_scm.h>
23	#include <uapi/misc/fastrpc.h>
24	#include <linux/of_reserved_mem.h>
25
26	#define ADSP_DOMAIN_ID (0)
27	#define MDSP_DOMAIN_ID (1)
28	#define SDSP_DOMAIN_ID (2)
29	#define CDSP_DOMAIN_ID (3)
30	#define FASTRPC_DEV_MAX 4 /* adsp, mdsp, slpi, cdsp*/
31	#define FASTRPC_MAX_SESSIONS 14
32	#define FASTRPC_MAX_VMIDS 16
33	#define FASTRPC_ALIGN 128
34	#define FASTRPC_MAX_FDLIST 16
35	#define FASTRPC_MAX_CRCLIST 64
36	#define FASTRPC_PHYS(p) ((p) & 0xffffffff)
37	#define FASTRPC_CTX_MAX (256)
38	#define FASTRPC_INIT_HANDLE 1
39	#define FASTRPC_DSP_UTILITIES_HANDLE 2
40	#define FASTRPC_CTXID_MASK (0xFF0)
41	#define INIT_FILELEN_MAX (2 * 1024 * 1024)
42	#define INIT_FILE_NAMELEN_MAX (128)
43	#define FASTRPC_DEVICE_NAME "fastrpc"
44
45	/* Add memory to static PD pool, protection thru XPU */
46	#define ADSP_MMAP_HEAP_ADDR 4
47	/* MAP static DMA buffer on DSP User PD */
48	#define ADSP_MMAP_DMA_BUFFER 6
49	/* Add memory to static PD pool protection thru hypervisor */
50	#define ADSP_MMAP_REMOTE_HEAP_ADDR 8
51	/* Add memory to userPD pool, for user heap */
52	#define ADSP_MMAP_ADD_PAGES 0x1000
53	/* Add memory to userPD pool, for LLC heap */
54	#define ADSP_MMAP_ADD_PAGES_LLC 0x3000,
55
56	#define DSP_UNSUPPORTED_API (0x80000414)
57	/* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */
58	#define FASTRPC_MAX_DSP_ATTRIBUTES (256)
59	#define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES)
60
61	/* Retrives number of input buffers from the scalars parameter */
62	#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
63
64	/* Retrives number of output buffers from the scalars parameter */
65	#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
66
67	/* Retrives number of input handles from the scalars parameter */
68	#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
69
70	/* Retrives number of output handles from the scalars parameter */
71	#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
72
73	#define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \
74	REMOTE_SCALARS_OUTBUFS(sc) + \
75	REMOTE_SCALARS_INHANDLES(sc)+ \
76	REMOTE_SCALARS_OUTHANDLES(sc))
77	#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \
78	(((attr & 0x07) << 29) | \
79	((method & 0x1f) << 24) | \
80	((in & 0xff) << 16) | \
81	((out & 0xff) << 8) | \
82	((oin & 0x0f) << 4) | \
83	(oout & 0x0f))
84
85	#define FASTRPC_SCALARS(method, in, out) \
86	FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
87
88	#define FASTRPC_CREATE_PROCESS_NARGS 6
89	#define FASTRPC_CREATE_STATIC_PROCESS_NARGS 3
90	/* Remote Method id table */
91	#define FASTRPC_RMID_INIT_ATTACH 0
92	#define FASTRPC_RMID_INIT_RELEASE 1
93	#define FASTRPC_RMID_INIT_MMAP 4
94	#define FASTRPC_RMID_INIT_MUNMAP 5
95	#define FASTRPC_RMID_INIT_CREATE 6
96	#define FASTRPC_RMID_INIT_CREATE_ATTR 7
97	#define FASTRPC_RMID_INIT_CREATE_STATIC 8
98	#define FASTRPC_RMID_INIT_MEM_MAP 10
99	#define FASTRPC_RMID_INIT_MEM_UNMAP 11
100
101	/* Protection Domain(PD) ids */
102	#define ROOT_PD (0)
103	#define USER_PD (1)
104	#define SENSORS_PD (2)
105
106	#define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)
107
108	static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp",
109	"sdsp", "cdsp"};
110	struct fastrpc_phy_page {
111	u64 addr; /* physical address */
112	u64 size; /* size of contiguous region */
113	};
114
115	struct fastrpc_invoke_buf {
116	u32 num; /* number of contiguous regions */
117	u32 pgidx; /* index to start of contiguous region */
118	};
119
120	struct fastrpc_remote_dmahandle {
121	s32 fd; /* dma handle fd */
122	u32 offset; /* dma handle offset */
123	u32 len; /* dma handle length */
124	};
125
126	struct fastrpc_remote_buf {
127	u64 pv; /* buffer pointer */
128	u64 len; /* length of buffer */
129	};
130
131	union fastrpc_remote_arg {
132	struct fastrpc_remote_buf buf;
133	struct fastrpc_remote_dmahandle dma;
134	};
135
136	struct fastrpc_mmap_rsp_msg {
137	u64 vaddr;
138	};
139
140	struct fastrpc_mmap_req_msg {
141	s32 pgid;
142	u32 flags;
143	u64 vaddr;
144	s32 num;
145	};
146
147	struct fastrpc_mem_map_req_msg {
148	s32 pgid;
149	s32 fd;
150	s32 offset;
151	u32 flags;
152	u64 vaddrin;
153	s32 num;
154	s32 data_len;
155	};
156
157	struct fastrpc_munmap_req_msg {
158	s32 pgid;
159	u64 vaddr;
160	u64 size;
161	};
162
163	struct fastrpc_mem_unmap_req_msg {
164	s32 pgid;
165	s32 fd;
166	u64 vaddrin;
167	u64 len;
168	};
169
170	struct fastrpc_msg {
171	int pid; /* process group id */
172	int tid; /* thread id */
173	u64 ctx; /* invoke caller context */
174	u32 handle; /* handle to invoke */
175	u32 sc; /* scalars structure describing the data */
176	u64 addr; /* physical address */
177	u64 size; /* size of contiguous region */
178	};
179
180	struct fastrpc_invoke_rsp {
181	u64 ctx; /* invoke caller context */
182	int retval; /* invoke return value */
183	};
184
185	struct fastrpc_buf_overlap {
186	u64 start;
187	u64 end;
188	int raix;
189	u64 mstart;
190	u64 mend;
191	u64 offset;
192	};
193
194	struct fastrpc_buf {
195	struct fastrpc_user *fl;
196	struct dma_buf *dmabuf;
197	struct device *dev;
198	void *virt;
199	u64 phys;
200	u64 size;
201	/* Lock for dma buf attachments */
202	struct mutex lock;
203	struct list_head attachments;
204	/* mmap support */
205	struct list_head node; /* list of user requested mmaps */
206	uintptr_t raddr;
207	};
208
209	struct fastrpc_dma_buf_attachment {
210	struct device *dev;
211	struct sg_table sgt;
212	struct list_head node;
213	};
214
215	struct fastrpc_map {
216	struct list_head node;
217	struct fastrpc_user *fl;
218	int fd;
219	struct dma_buf *buf;
220	struct sg_table *table;
221	struct dma_buf_attachment *attach;
222	u64 phys;
223	u64 size;
224	void *va;
225	u64 len;
226	u64 raddr;
227	u32 attr;
228	struct kref refcount;
229	};
230
231	struct fastrpc_invoke_ctx {
232	int nscalars;
233	int nbufs;
234	int retval;
235	int pid;
236	int tgid;
237	u32 sc;
238	u32 *crc;
239	u64 ctxid;
240	u64 msg_sz;
241	struct kref refcount;
242	struct list_head node; /* list of ctxs */
243	struct completion work;
244	struct work_struct put_work;
245	struct fastrpc_msg msg;
246	struct fastrpc_user *fl;
247	union fastrpc_remote_arg *rpra;
248	struct fastrpc_map **maps;
249	struct fastrpc_buf *buf;
250	struct fastrpc_invoke_args *args;
251	struct fastrpc_buf_overlap *olaps;
252	struct fastrpc_channel_ctx *cctx;
253	};
254
255	struct fastrpc_session_ctx {
256	struct device *dev;
257	int sid;
258	bool used;
259	bool valid;
260	};
261
262	struct fastrpc_channel_ctx {
263	int domain_id;
264	int sesscount;
265	int vmcount;
266	u64 perms;
267	struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS];
268	struct rpmsg_device *rpdev;
269	struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
270	spinlock_t lock;
271	struct idr ctx_idr;
272	struct list_head users;
273	struct kref refcount;
274	/* Flag if dsp attributes are cached */
275	bool valid_attributes;
276	u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES];
277	struct fastrpc_device *secure_fdevice;
278	struct fastrpc_device *fdevice;
279	struct fastrpc_buf *remote_heap;
280	struct list_head invoke_interrupted_mmaps;
281	bool secure;
282	bool unsigned_support;
283	u64 dma_mask;
284	};
285
286	struct fastrpc_device {
287	struct fastrpc_channel_ctx *cctx;
288	struct miscdevice miscdev;
289	bool secure;
290	};
291
292	struct fastrpc_user {
293	struct list_head user;
294	struct list_head maps;
295	struct list_head pending;
296	struct list_head mmaps;
297
298	struct fastrpc_channel_ctx *cctx;
299	struct fastrpc_session_ctx *sctx;
300	struct fastrpc_buf *init_mem;
301
302	int tgid;
303	int pd;
304	bool is_secure_dev;
305	/* Lock for lists */
306	spinlock_t lock;
307	/* lock for allocations */
308	struct mutex mutex;
309	};
310
311	static void fastrpc_free_map(struct kref *ref)
312	{
313	struct fastrpc_map *map;
314
315	map = container_of(ref, struct fastrpc_map, refcount);
316
317	if (map->table) {
318	if (map->attr & FASTRPC_ATTR_SECUREMAP) {
319	struct qcom_scm_vmperm perm;
320	int vmid = map->fl->cctx->vmperms[0].vmid;
321	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid);
322	int err = 0;
323
324	perm.vmid = QCOM_SCM_VMID_HLOS;
325	perm.perm = QCOM_SCM_PERM_RWX;
326	err = qcom_scm_assign_mem(mem_addr: map->phys, mem_sz: map->size,
327	src: &src_perms, newvm: &perm, dest_cnt: 1);
328	if (err) {
329	dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
330	map->phys, map->size, err);
331	return;
332	}
333	}
334	dma_buf_unmap_attachment_unlocked(attach: map->attach, sg_table: map->table,
335	direction: DMA_BIDIRECTIONAL);
336	dma_buf_detach(dmabuf: map->buf, attach: map->attach);
337	dma_buf_put(dmabuf: map->buf);
338	}
339
340	if (map->fl) {
341	spin_lock(lock: &map->fl->lock);
342	list_del(entry: &map->node);
343	spin_unlock(lock: &map->fl->lock);
344	map->fl = NULL;
345	}
346
347	kfree(objp: map);
348	}
349
350	static void fastrpc_map_put(struct fastrpc_map *map)
351	{
352	if (map)
353	kref_put(kref: &map->refcount, release: fastrpc_free_map);
354	}
355
356	static int fastrpc_map_get(struct fastrpc_map *map)
357	{
358	if (!map)
359	return -ENOENT;
360
361	return kref_get_unless_zero(kref: &map->refcount) ? 0 : -ENOENT;
362	}
363
364
365	static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd,
366	struct fastrpc_map **ppmap, bool take_ref)
367	{
368	struct fastrpc_session_ctx *sess = fl->sctx;
369	struct fastrpc_map *map = NULL;
370	int ret = -ENOENT;
371
372	spin_lock(lock: &fl->lock);
373	list_for_each_entry(map, &fl->maps, node) {
374	if (map->fd != fd)
375	continue;
376
377	if (take_ref) {
378	ret = fastrpc_map_get(map);
379	if (ret) {
380	dev_dbg(sess->dev, "%s: Failed to get map fd=%d ret=%d\n",
381	__func__, fd, ret);
382	break;
383	}
384	}
385
386	*ppmap = map;
387	ret = 0;
388	break;
389	}
390	spin_unlock(lock: &fl->lock);
391
392	return ret;
393	}
394
395	static void fastrpc_buf_free(struct fastrpc_buf *buf)
396	{
397	dma_free_coherent(dev: buf->dev, size: buf->size, cpu_addr: buf->virt,
398	FASTRPC_PHYS(buf->phys));
399	kfree(objp: buf);
400	}
401
402	static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
403	u64 size, struct fastrpc_buf **obuf)
404	{
405	struct fastrpc_buf *buf;
406
407	buf = kzalloc(size: sizeof(*buf), GFP_KERNEL);
408	if (!buf)
409	return -ENOMEM;
410
411	INIT_LIST_HEAD(list: &buf->attachments);
412	INIT_LIST_HEAD(list: &buf->node);
413	mutex_init(&buf->lock);
414
415	buf->fl = fl;
416	buf->virt = NULL;
417	buf->phys = 0;
418	buf->size = size;
419	buf->dev = dev;
420	buf->raddr = 0;
421
422	buf->virt = dma_alloc_coherent(dev, size: buf->size, dma_handle: (dma_addr_t *)&buf->phys,
423	GFP_KERNEL);
424	if (!buf->virt) {
425	mutex_destroy(lock: &buf->lock);
426	kfree(objp: buf);
427	return -ENOMEM;
428	}
429
430	*obuf = buf;
431
432	return 0;
433	}
434
435	static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
436	u64 size, struct fastrpc_buf **obuf)
437	{
438	int ret;
439	struct fastrpc_buf *buf;
440
441	ret = __fastrpc_buf_alloc(fl, dev, size, obuf);
442	if (ret)
443	return ret;
444
445	buf = *obuf;
446
447	if (fl->sctx && fl->sctx->sid)
448	buf->phys += ((u64)fl->sctx->sid << 32);
449
450	return 0;
451	}
452
453	static int fastrpc_remote_heap_alloc(struct fastrpc_user *fl, struct device *dev,
454	u64 size, struct fastrpc_buf **obuf)
455	{
456	struct device *rdev = &fl->cctx->rpdev->dev;
457
458	return __fastrpc_buf_alloc(fl, dev: rdev, size, obuf);
459	}
460
461	static void fastrpc_channel_ctx_free(struct kref *ref)
462	{
463	struct fastrpc_channel_ctx *cctx;
464
465	cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
466
467	kfree(objp: cctx);
468	}
469
470	static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)
471	{
472	kref_get(kref: &cctx->refcount);
473	}
474
475	static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
476	{
477	kref_put(kref: &cctx->refcount, release: fastrpc_channel_ctx_free);
478	}
479
480	static void fastrpc_context_free(struct kref *ref)
481	{
482	struct fastrpc_invoke_ctx *ctx;
483	struct fastrpc_channel_ctx *cctx;
484	unsigned long flags;
485	int i;
486
487	ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
488	cctx = ctx->cctx;
489
490	for (i = 0; i < ctx->nbufs; i++)
491	fastrpc_map_put(map: ctx->maps[i]);
492
493	if (ctx->buf)
494	fastrpc_buf_free(buf: ctx->buf);
495
496	spin_lock_irqsave(&cctx->lock, flags);
497	idr_remove(&cctx->ctx_idr, id: ctx->ctxid >> 4);
498	spin_unlock_irqrestore(lock: &cctx->lock, flags);
499
500	kfree(objp: ctx->maps);
501	kfree(objp: ctx->olaps);
502	kfree(objp: ctx);
503
504	fastrpc_channel_ctx_put(cctx);
505	}
506
507	static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)
508	{
509	kref_get(kref: &ctx->refcount);
510	}
511
512	static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
513	{
514	kref_put(kref: &ctx->refcount, release: fastrpc_context_free);
515	}
516
517	static void fastrpc_context_put_wq(struct work_struct *work)
518	{
519	struct fastrpc_invoke_ctx *ctx =
520	container_of(work, struct fastrpc_invoke_ctx, put_work);
521
522	fastrpc_context_put(ctx);
523	}
524
525	#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
526	static int olaps_cmp(const void *a, const void *b)
527	{
528	struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;
529	struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;
530	/* sort with lowest starting buffer first */
531	int st = CMP(pa->start, pb->start);
532	/* sort with highest ending buffer first */
533	int ed = CMP(pb->end, pa->end);
534
535	return st == 0 ? ed : st;
536	}
537
538	static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
539	{
540	u64 max_end = 0;
541	int i;
542
543	for (i = 0; i < ctx->nbufs; ++i) {
544	ctx->olaps[i].start = ctx->args[i].ptr;
545	ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
546	ctx->olaps[i].raix = i;
547	}
548
549	sort(base: ctx->olaps, num: ctx->nbufs, size: sizeof(*ctx->olaps), cmp_func: olaps_cmp, NULL);
550
551	for (i = 0; i < ctx->nbufs; ++i) {
552	/* Falling inside previous range */
553	if (ctx->olaps[i].start < max_end) {
554	ctx->olaps[i].mstart = max_end;
555	ctx->olaps[i].mend = ctx->olaps[i].end;
556	ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
557
558	if (ctx->olaps[i].end > max_end) {
559	max_end = ctx->olaps[i].end;
560	} else {
561	ctx->olaps[i].mend = 0;
562	ctx->olaps[i].mstart = 0;
563	}
564
565	} else {
566	ctx->olaps[i].mend = ctx->olaps[i].end;
567	ctx->olaps[i].mstart = ctx->olaps[i].start;
568	ctx->olaps[i].offset = 0;
569	max_end = ctx->olaps[i].end;
570	}
571	}
572	}
573
574	static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
575	struct fastrpc_user *user, u32 kernel, u32 sc,
576	struct fastrpc_invoke_args *args)
577	{
578	struct fastrpc_channel_ctx *cctx = user->cctx;
579	struct fastrpc_invoke_ctx *ctx = NULL;
580	unsigned long flags;
581	int ret;
582
583	ctx = kzalloc(size: sizeof(*ctx), GFP_KERNEL);
584	if (!ctx)
585	return ERR_PTR(error: -ENOMEM);
586
587	INIT_LIST_HEAD(list: &ctx->node);
588	ctx->fl = user;
589	ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);
590	ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +
591	REMOTE_SCALARS_OUTBUFS(sc);
592
593	if (ctx->nscalars) {
594	ctx->maps = kcalloc(n: ctx->nscalars,
595	size: sizeof(*ctx->maps), GFP_KERNEL);
596	if (!ctx->maps) {
597	kfree(objp: ctx);
598	return ERR_PTR(error: -ENOMEM);
599	}
600	ctx->olaps = kcalloc(n: ctx->nscalars,
601	size: sizeof(*ctx->olaps), GFP_KERNEL);
602	if (!ctx->olaps) {
603	kfree(objp: ctx->maps);
604	kfree(objp: ctx);
605	return ERR_PTR(error: -ENOMEM);
606	}
607	ctx->args = args;
608	fastrpc_get_buff_overlaps(ctx);
609	}
610
611	/* Released in fastrpc_context_put() */
612	fastrpc_channel_ctx_get(cctx);
613
614	ctx->sc = sc;
615	ctx->retval = -1;
616	ctx->pid = current->pid;
617	ctx->tgid = user->tgid;
618	ctx->cctx = cctx;
619	init_completion(x: &ctx->work);
620	INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
621
622	spin_lock(lock: &user->lock);
623	list_add_tail(new: &ctx->node, head: &user->pending);
624	spin_unlock(lock: &user->lock);
625
626	spin_lock_irqsave(&cctx->lock, flags);
627	ret = idr_alloc_cyclic(&cctx->ctx_idr, ptr: ctx, start: 1,
628	FASTRPC_CTX_MAX, GFP_ATOMIC);
629	if (ret < 0) {
630	spin_unlock_irqrestore(lock: &cctx->lock, flags);
631	goto err_idr;
632	}
633	ctx->ctxid = ret << 4;
634	spin_unlock_irqrestore(lock: &cctx->lock, flags);
635
636	kref_init(kref: &ctx->refcount);
637
638	return ctx;
639	err_idr:
640	spin_lock(lock: &user->lock);
641	list_del(entry: &ctx->node);
642	spin_unlock(lock: &user->lock);
643	fastrpc_channel_ctx_put(cctx);
644	kfree(objp: ctx->maps);
645	kfree(objp: ctx->olaps);
646	kfree(objp: ctx);
647
648	return ERR_PTR(error: ret);
649	}
650
651	static struct sg_table *
652	fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,
653	enum dma_data_direction dir)
654	{
655	struct fastrpc_dma_buf_attachment *a = attachment->priv;
656	struct sg_table *table;
657	int ret;
658
659	table = &a->sgt;
660
661	ret = dma_map_sgtable(dev: attachment->dev, sgt: table, dir, attrs: 0);
662	if (ret)
663	table = ERR_PTR(error: ret);
664	return table;
665	}
666
667	static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,
668	struct sg_table *table,
669	enum dma_data_direction dir)
670	{
671	dma_unmap_sgtable(dev: attach->dev, sgt: table, dir, attrs: 0);
672	}
673
674	static void fastrpc_release(struct dma_buf *dmabuf)
675	{
676	struct fastrpc_buf *buffer = dmabuf->priv;
677
678	fastrpc_buf_free(buf: buffer);
679	}
680
681	static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,
682	struct dma_buf_attachment *attachment)
683	{
684	struct fastrpc_dma_buf_attachment *a;
685	struct fastrpc_buf *buffer = dmabuf->priv;
686	int ret;
687
688	a = kzalloc(size: sizeof(*a), GFP_KERNEL);
689	if (!a)
690	return -ENOMEM;
691
692	ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,
693	FASTRPC_PHYS(buffer->phys), buffer->size);
694	if (ret < 0) {
695	dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");
696	kfree(objp: a);
697	return -EINVAL;
698	}
699
700	a->dev = attachment->dev;
701	INIT_LIST_HEAD(list: &a->node);
702	attachment->priv = a;
703
704	mutex_lock(&buffer->lock);
705	list_add(new: &a->node, head: &buffer->attachments);
706	mutex_unlock(lock: &buffer->lock);
707
708	return 0;
709	}
710
711	static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,
712	struct dma_buf_attachment *attachment)
713	{
714	struct fastrpc_dma_buf_attachment *a = attachment->priv;
715	struct fastrpc_buf *buffer = dmabuf->priv;
716
717	mutex_lock(&buffer->lock);
718	list_del(entry: &a->node);
719	mutex_unlock(lock: &buffer->lock);
720	sg_free_table(&a->sgt);
721	kfree(objp: a);
722	}
723
724	static int fastrpc_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
725	{
726	struct fastrpc_buf *buf = dmabuf->priv;
727
728	iosys_map_set_vaddr(map, vaddr: buf->virt);
729
730	return 0;
731	}
732
733	static int fastrpc_mmap(struct dma_buf *dmabuf,
734	struct vm_area_struct *vma)
735	{
736	struct fastrpc_buf *buf = dmabuf->priv;
737	size_t size = vma->vm_end - vma->vm_start;
738
739	dma_resv_assert_held(dmabuf->resv);
740
741	return dma_mmap_coherent(buf->dev, vma, buf->virt,
742	FASTRPC_PHYS(buf->phys), size);
743	}
744
745	static const struct dma_buf_ops fastrpc_dma_buf_ops = {
746	.attach = fastrpc_dma_buf_attach,
747	.detach = fastrpc_dma_buf_detatch,
748	.map_dma_buf = fastrpc_map_dma_buf,
749	.unmap_dma_buf = fastrpc_unmap_dma_buf,
750	.mmap = fastrpc_mmap,
751	.vmap = fastrpc_vmap,
752	.release = fastrpc_release,
753	};
754
755	static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
756	u64 len, u32 attr, struct fastrpc_map **ppmap)
757	{
758	struct fastrpc_session_ctx *sess = fl->sctx;
759	struct fastrpc_map *map = NULL;
760	struct sg_table *table;
761	int err = 0;
762
763	if (!fastrpc_map_lookup(fl, fd, ppmap, take_ref: true))
764	return 0;
765
766	map = kzalloc(size: sizeof(*map), GFP_KERNEL);
767	if (!map)
768	return -ENOMEM;
769
770	INIT_LIST_HEAD(list: &map->node);
771	kref_init(kref: &map->refcount);
772
773	map->fl = fl;
774	map->fd = fd;
775	map->buf = dma_buf_get(fd);
776	if (IS_ERR(ptr: map->buf)) {
777	err = PTR_ERR(ptr: map->buf);
778	goto get_err;
779	}
780
781	map->attach = dma_buf_attach(dmabuf: map->buf, dev: sess->dev);
782	if (IS_ERR(ptr: map->attach)) {
783	dev_err(sess->dev, "Failed to attach dmabuf\n");
784	err = PTR_ERR(ptr: map->attach);
785	goto attach_err;
786	}
787
788	table = dma_buf_map_attachment_unlocked(attach: map->attach, direction: DMA_BIDIRECTIONAL);
789	if (IS_ERR(ptr: table)) {
790	err = PTR_ERR(ptr: table);
791	goto map_err;
792	}
793	map->table = table;
794
795	if (attr & FASTRPC_ATTR_SECUREMAP) {
796	map->phys = sg_phys(sg: map->table->sgl);
797	} else {
798	map->phys = sg_dma_address(map->table->sgl);
799	map->phys += ((u64)fl->sctx->sid << 32);
800	}
801	map->size = len;
802	map->va = sg_virt(sg: map->table->sgl);
803	map->len = len;
804
805	if (attr & FASTRPC_ATTR_SECUREMAP) {
806	/*
807	* If subsystem VMIDs are defined in DTSI, then do
808	* hyp_assign from HLOS to those VM(s)
809	*/
810	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
811	struct qcom_scm_vmperm dst_perms[2] = {0};
812
813	dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
814	dst_perms[0].perm = QCOM_SCM_PERM_RW;
815	dst_perms[1].vmid = fl->cctx->vmperms[0].vmid;
816	dst_perms[1].perm = QCOM_SCM_PERM_RWX;
817	map->attr = attr;
818	err = qcom_scm_assign_mem(mem_addr: map->phys, mem_sz: (u64)map->size, src: &src_perms, newvm: dst_perms, dest_cnt: 2);
819	if (err) {
820	dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d",
821	map->phys, map->size, err);
822	goto map_err;
823	}
824	}
825	spin_lock(lock: &fl->lock);
826	list_add_tail(new: &map->node, head: &fl->maps);
827	spin_unlock(lock: &fl->lock);
828	*ppmap = map;
829
830	return 0;
831
832	map_err:
833	dma_buf_detach(dmabuf: map->buf, attach: map->attach);
834	attach_err:
835	dma_buf_put(dmabuf: map->buf);
836	get_err:
837	fastrpc_map_put(map);
838
839	return err;
840	}
841
842	/*
843	* Fastrpc payload buffer with metadata looks like:
844	*
845	* >>>>>> START of METADATA <<<<<<<<<
846	* +---------------------------------+
847	* | Arguments |
848	* | type:(union fastrpc_remote_arg)|
849	* | (0 - N) |
850	* +---------------------------------+
851	* | Invoke Buffer list |
852	* | type:(struct fastrpc_invoke_buf)|
853	* | (0 - N) |
854	* +---------------------------------+
855	* | Page info list |
856	* | type:(struct fastrpc_phy_page) |
857	* | (0 - N) |
858	* +---------------------------------+
859	* | Optional info |
860	* |(can be specific to SoC/Firmware)|
861	* +---------------------------------+
862	* >>>>>>>> END of METADATA <<<<<<<<<
863	* +---------------------------------+
864	* | Inline ARGS |
865	* | (0-N) |
866	* +---------------------------------+
867	*/
868
869	static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
870	{
871	int size = 0;
872
873	size = (sizeof(struct fastrpc_remote_buf) +
874	sizeof(struct fastrpc_invoke_buf) +
875	sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
876	sizeof(u64) * FASTRPC_MAX_FDLIST +
877	sizeof(u32) * FASTRPC_MAX_CRCLIST;
878
879	return size;
880	}
881
882	static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
883	{
884	u64 size = 0;
885	int oix;
886
887	size = ALIGN(metalen, FASTRPC_ALIGN);
888	for (oix = 0; oix < ctx->nbufs; oix++) {
889	int i = ctx->olaps[oix].raix;
890
891	if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
892
893	if (ctx->olaps[oix].offset == 0)
894	size = ALIGN(size, FASTRPC_ALIGN);
895
896	size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart);
897	}
898	}
899
900	return size;
901	}
902
903	static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
904	{
905	struct device *dev = ctx->fl->sctx->dev;
906	int i, err;
907
908	for (i = 0; i < ctx->nscalars; ++i) {
909
910	if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
911	ctx->args[i].length == 0)
912	continue;
913
914	err = fastrpc_map_create(fl: ctx->fl, fd: ctx->args[i].fd,
915	len: ctx->args[i].length, attr: ctx->args[i].attr, ppmap: &ctx->maps[i]);
916	if (err) {
917	dev_err(dev, "Error Creating map %d\n", err);
918	return -EINVAL;
919	}
920
921	}
922	return 0;
923	}
924
925	static struct fastrpc_invoke_buf *fastrpc_invoke_buf_start(union fastrpc_remote_arg *pra, int len)
926	{
927	return (struct fastrpc_invoke_buf *)(&pra[len]);
928	}
929
930	static struct fastrpc_phy_page *fastrpc_phy_page_start(struct fastrpc_invoke_buf *buf, int len)
931	{
932	return (struct fastrpc_phy_page *)(&buf[len]);
933	}
934
935	static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
936	{
937	struct device *dev = ctx->fl->sctx->dev;
938	union fastrpc_remote_arg *rpra;
939	struct fastrpc_invoke_buf *list;
940	struct fastrpc_phy_page *pages;
941	int inbufs, i, oix, err = 0;
942	u64 len, rlen, pkt_size;
943	u64 pg_start, pg_end;
944	uintptr_t args;
945	int metalen;
946
947	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
948	metalen = fastrpc_get_meta_size(ctx);
949	pkt_size = fastrpc_get_payload_size(ctx, metalen);
950
951	err = fastrpc_create_maps(ctx);
952	if (err)
953	return err;
954
955	ctx->msg_sz = pkt_size;
956
957	err = fastrpc_buf_alloc(fl: ctx->fl, dev, size: pkt_size, obuf: &ctx->buf);
958	if (err)
959	return err;
960
961	memset(ctx->buf->virt, 0, pkt_size);
962	rpra = ctx->buf->virt;
963	list = fastrpc_invoke_buf_start(pra: rpra, len: ctx->nscalars);
964	pages = fastrpc_phy_page_start(buf: list, len: ctx->nscalars);
965	args = (uintptr_t)ctx->buf->virt + metalen;
966	rlen = pkt_size - metalen;
967	ctx->rpra = rpra;
968
969	for (oix = 0; oix < ctx->nbufs; ++oix) {
970	int mlen;
971
972	i = ctx->olaps[oix].raix;
973	len = ctx->args[i].length;
974
975	rpra[i].buf.pv = 0;
976	rpra[i].buf.len = len;
977	list[i].num = len ? 1 : 0;
978	list[i].pgidx = i;
979
980	if (!len)
981	continue;
982
983	if (ctx->maps[i]) {
984	struct vm_area_struct *vma = NULL;
985
986	rpra[i].buf.pv = (u64) ctx->args[i].ptr;
987	pages[i].addr = ctx->maps[i]->phys;
988
989	mmap_read_lock(current->mm);
990	vma = find_vma(current->mm, addr: ctx->args[i].ptr);
991	if (vma)
992	pages[i].addr += ctx->args[i].ptr -
993	vma->vm_start;
994	mmap_read_unlock(current->mm);
995
996	pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
997	pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
998	PAGE_SHIFT;
999	pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1000
1001	} else {
1002
1003	if (ctx->olaps[oix].offset == 0) {
1004	rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
1005	args = ALIGN(args, FASTRPC_ALIGN);
1006	}
1007
1008	mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
1009
1010	if (rlen < mlen)
1011	goto bail;
1012
1013	rpra[i].buf.pv = args - ctx->olaps[oix].offset;
1014	pages[i].addr = ctx->buf->phys -
1015	ctx->olaps[oix].offset +
1016	(pkt_size - rlen);
1017	pages[i].addr = pages[i].addr & PAGE_MASK;
1018
1019	pg_start = (args & PAGE_MASK) >> PAGE_SHIFT;
1020	pg_end = ((args + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
1021	pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1022	args = args + mlen;
1023	rlen -= mlen;
1024	}
1025
1026	if (i < inbufs && !ctx->maps[i]) {
1027	void *dst = (void *)(uintptr_t)rpra[i].buf.pv;
1028	void *src = (void *)(uintptr_t)ctx->args[i].ptr;
1029
1030	if (!kernel) {
1031	if (copy_from_user(to: dst, from: (void __user *)src,
1032	n: len)) {
1033	err = -EFAULT;
1034	goto bail;
1035	}
1036	} else {
1037	memcpy(dst, src, len);
1038	}
1039	}
1040	}
1041
1042	for (i = ctx->nbufs; i < ctx->nscalars; ++i) {
1043	list[i].num = ctx->args[i].length ? 1 : 0;
1044	list[i].pgidx = i;
1045	if (ctx->maps[i]) {
1046	pages[i].addr = ctx->maps[i]->phys;
1047	pages[i].size = ctx->maps[i]->size;
1048	}
1049	rpra[i].dma.fd = ctx->args[i].fd;
1050	rpra[i].dma.len = ctx->args[i].length;
1051	rpra[i].dma.offset = (u64) ctx->args[i].ptr;
1052	}
1053
1054	bail:
1055	if (err)
1056	dev_err(dev, "Error: get invoke args failed:%d\n", err);
1057
1058	return err;
1059	}
1060
1061	static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,
1062	u32 kernel)
1063	{
1064	union fastrpc_remote_arg *rpra = ctx->rpra;
1065	struct fastrpc_user *fl = ctx->fl;
1066	struct fastrpc_map *mmap = NULL;
1067	struct fastrpc_invoke_buf *list;
1068	struct fastrpc_phy_page *pages;
1069	u64 *fdlist;
1070	int i, inbufs, outbufs, handles;
1071
1072	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
1073	outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
1074	handles = REMOTE_SCALARS_INHANDLES(ctx->sc) + REMOTE_SCALARS_OUTHANDLES(ctx->sc);
1075	list = fastrpc_invoke_buf_start(pra: rpra, len: ctx->nscalars);
1076	pages = fastrpc_phy_page_start(buf: list, len: ctx->nscalars);
1077	fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
1078
1079	for (i = inbufs; i < ctx->nbufs; ++i) {
1080	if (!ctx->maps[i]) {
1081	void *src = (void *)(uintptr_t)rpra[i].buf.pv;
1082	void *dst = (void *)(uintptr_t)ctx->args[i].ptr;
1083	u64 len = rpra[i].buf.len;
1084
1085	if (!kernel) {
1086	if (copy_to_user(to: (void __user *)dst, from: src, n: len))
1087	return -EFAULT;
1088	} else {
1089	memcpy(dst, src, len);
1090	}
1091	}
1092	}
1093
1094	/* Clean up fdlist which is updated by DSP */
1095	for (i = 0; i < FASTRPC_MAX_FDLIST; i++) {
1096	if (!fdlist[i])
1097	break;
1098	if (!fastrpc_map_lookup(fl, fd: (int)fdlist[i], ppmap: &mmap, take_ref: false))
1099	fastrpc_map_put(map: mmap);
1100	}
1101
1102	return 0;
1103	}
1104
1105	static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
1106	struct fastrpc_invoke_ctx *ctx,
1107	u32 kernel, uint32_t handle)
1108	{
1109	struct fastrpc_channel_ctx *cctx;
1110	struct fastrpc_user *fl = ctx->fl;
1111	struct fastrpc_msg *msg = &ctx->msg;
1112	int ret;
1113
1114	cctx = fl->cctx;
1115	msg->pid = fl->tgid;
1116	msg->tid = current->pid;
1117
1118	if (kernel)
1119	msg->pid = 0;
1120
1121	msg->ctx = ctx->ctxid | fl->pd;
1122	msg->handle = handle;
1123	msg->sc = ctx->sc;
1124	msg->addr = ctx->buf ? ctx->buf->phys : 0;
1125	msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
1126	fastrpc_context_get(ctx);
1127
1128	ret = rpmsg_send(ept: cctx->rpdev->ept, data: (void *)msg, len: sizeof(*msg));
1129
1130	if (ret)
1131	fastrpc_context_put(ctx);
1132
1133	return ret;
1134
1135	}
1136
1137	static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
1138	u32 handle, u32 sc,
1139	struct fastrpc_invoke_args *args)
1140	{
1141	struct fastrpc_invoke_ctx *ctx = NULL;
1142	struct fastrpc_buf *buf, *b;
1143
1144	int err = 0;
1145
1146	if (!fl->sctx)
1147	return -EINVAL;
1148
1149	if (!fl->cctx->rpdev)
1150	return -EPIPE;
1151
1152	if (handle == FASTRPC_INIT_HANDLE && !kernel) {
1153	dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n", handle);
1154	return -EPERM;
1155	}
1156
1157	ctx = fastrpc_context_alloc(user: fl, kernel, sc, args);
1158	if (IS_ERR(ptr: ctx))
1159	return PTR_ERR(ptr: ctx);
1160
1161	err = fastrpc_get_args(kernel, ctx);
1162	if (err)
1163	goto bail;
1164
1165	/* make sure that all CPU memory writes are seen by DSP */
1166	dma_wmb();
1167	/* Send invoke buffer to remote dsp */
1168	err = fastrpc_invoke_send(sctx: fl->sctx, ctx, kernel, handle);
1169	if (err)
1170	goto bail;
1171
1172	if (kernel) {
1173	if (!wait_for_completion_timeout(x: &ctx->work, timeout: 10 * HZ))
1174	err = -ETIMEDOUT;
1175	} else {
1176	err = wait_for_completion_interruptible(x: &ctx->work);
1177	}
1178
1179	if (err)
1180	goto bail;
1181
1182	/* make sure that all memory writes by DSP are seen by CPU */
1183	dma_rmb();
1184	/* populate all the output buffers with results */
1185	err = fastrpc_put_args(ctx, kernel);
1186	if (err)
1187	goto bail;
1188
1189	/* Check the response from remote dsp */
1190	err = ctx->retval;
1191	if (err)
1192	goto bail;
1193
1194	bail:
1195	if (err != -ERESTARTSYS && err != -ETIMEDOUT) {
1196	/* We are done with this compute context */
1197	spin_lock(lock: &fl->lock);
1198	list_del(entry: &ctx->node);
1199	spin_unlock(lock: &fl->lock);
1200	fastrpc_context_put(ctx);
1201	}
1202
1203	if (err == -ERESTARTSYS) {
1204	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1205	list_del(entry: &buf->node);
1206	list_add_tail(new: &buf->node, head: &fl->cctx->invoke_interrupted_mmaps);
1207	}
1208	}
1209
1210	if (err)
1211	dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
1212
1213	return err;
1214	}
1215
1216	static bool is_session_rejected(struct fastrpc_user *fl, bool unsigned_pd_request)
1217	{
1218	/* Check if the device node is non-secure and channel is secure*/
1219	if (!fl->is_secure_dev && fl->cctx->secure) {
1220	/*
1221	* Allow untrusted applications to offload only to Unsigned PD when
1222	* channel is configured as secure and block untrusted apps on channel
1223	* that does not support unsigned PD offload
1224	*/
1225	if (!fl->cctx->unsigned_support || !unsigned_pd_request) {
1226	dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD");
1227	return true;
1228	}
1229	}
1230
1231	return false;
1232	}
1233
1234	static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
1235	char __user *argp)
1236	{
1237	struct fastrpc_init_create_static init;
1238	struct fastrpc_invoke_args *args;
1239	struct fastrpc_phy_page pages[1];
1240	char *name;
1241	int err;
1242	struct {
1243	int pgid;
1244	u32 namelen;
1245	u32 pageslen;
1246	} inbuf;
1247	u32 sc;
1248
1249	args = kcalloc(FASTRPC_CREATE_STATIC_PROCESS_NARGS, size: sizeof(*args), GFP_KERNEL);
1250	if (!args)
1251	return -ENOMEM;
1252
1253	if (copy_from_user(to: &init, from: argp, n: sizeof(init))) {
1254	err = -EFAULT;
1255	goto err;
1256	}
1257
1258	if (init.namelen > INIT_FILE_NAMELEN_MAX) {
1259	err = -EINVAL;
1260	goto err;
1261	}
1262
1263	name = kzalloc(size: init.namelen, GFP_KERNEL);
1264	if (!name) {
1265	err = -ENOMEM;
1266	goto err;
1267	}
1268
1269	if (copy_from_user(to: name, from: (void __user *)(uintptr_t)init.name, n: init.namelen)) {
1270	err = -EFAULT;
1271	goto err_name;
1272	}
1273
1274	if (!fl->cctx->remote_heap) {
1275	err = fastrpc_remote_heap_alloc(fl, dev: fl->sctx->dev, size: init.memlen,
1276	obuf: &fl->cctx->remote_heap);
1277	if (err)
1278	goto err_name;
1279
1280	/* Map if we have any heap VMIDs associated with this ADSP Static Process. */
1281	if (fl->cctx->vmcount) {
1282	err = qcom_scm_assign_mem(mem_addr: fl->cctx->remote_heap->phys,
1283	mem_sz: (u64)fl->cctx->remote_heap->size,
1284	src: &fl->cctx->perms,
1285	newvm: fl->cctx->vmperms, dest_cnt: fl->cctx->vmcount);
1286	if (err) {
1287	dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d",
1288	fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1289	goto err_map;
1290	}
1291	}
1292	}
1293
1294	inbuf.pgid = fl->tgid;
1295	inbuf.namelen = init.namelen;
1296	inbuf.pageslen = 0;
1297	fl->pd = USER_PD;
1298
1299	args[0].ptr = (u64)(uintptr_t)&inbuf;
1300	args[0].length = sizeof(inbuf);
1301	args[0].fd = -1;
1302
1303	args[1].ptr = (u64)(uintptr_t)name;
1304	args[1].length = inbuf.namelen;
1305	args[1].fd = -1;
1306
1307	pages[0].addr = fl->cctx->remote_heap->phys;
1308	pages[0].size = fl->cctx->remote_heap->size;
1309
1310	args[2].ptr = (u64)(uintptr_t) pages;
1311	args[2].length = sizeof(*pages);
1312	args[2].fd = -1;
1313
1314	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_STATIC, 3, 0);
1315
1316	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE,
1317	sc, args);
1318	if (err)
1319	goto err_invoke;
1320
1321	kfree(objp: args);
1322
1323	return 0;
1324	err_invoke:
1325	if (fl->cctx->vmcount) {
1326	u64 src_perms = 0;
1327	struct qcom_scm_vmperm dst_perms;
1328	u32 i;
1329
1330	for (i = 0; i < fl->cctx->vmcount; i++)
1331	src_perms |= BIT(fl->cctx->vmperms[i].vmid);
1332
1333	dst_perms.vmid = QCOM_SCM_VMID_HLOS;
1334	dst_perms.perm = QCOM_SCM_PERM_RWX;
1335	err = qcom_scm_assign_mem(mem_addr: fl->cctx->remote_heap->phys,
1336	mem_sz: (u64)fl->cctx->remote_heap->size,
1337	src: &src_perms, newvm: &dst_perms, dest_cnt: 1);
1338	if (err)
1339	dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
1340	fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1341	}
1342	err_map:
1343	fastrpc_buf_free(buf: fl->cctx->remote_heap);
1344	err_name:
1345	kfree(objp: name);
1346	err:
1347	kfree(objp: args);
1348
1349	return err;
1350	}
1351
1352	static int fastrpc_init_create_process(struct fastrpc_user *fl,
1353	char __user *argp)
1354	{
1355	struct fastrpc_init_create init;
1356	struct fastrpc_invoke_args *args;
1357	struct fastrpc_phy_page pages[1];
1358	struct fastrpc_map *map = NULL;
1359	struct fastrpc_buf *imem = NULL;
1360	int memlen;
1361	int err;
1362	struct {
1363	int pgid;
1364	u32 namelen;
1365	u32 filelen;
1366	u32 pageslen;
1367	u32 attrs;
1368	u32 siglen;
1369	} inbuf;
1370	u32 sc;
1371	bool unsigned_module = false;
1372
1373	args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, size: sizeof(*args), GFP_KERNEL);
1374	if (!args)
1375	return -ENOMEM;
1376
1377	if (copy_from_user(to: &init, from: argp, n: sizeof(init))) {
1378	err = -EFAULT;
1379	goto err;
1380	}
1381
1382	if (init.attrs & FASTRPC_MODE_UNSIGNED_MODULE)
1383	unsigned_module = true;
1384
1385	if (is_session_rejected(fl, unsigned_pd_request: unsigned_module)) {
1386	err = -ECONNREFUSED;
1387	goto err;
1388	}
1389
1390	if (init.filelen > INIT_FILELEN_MAX) {
1391	err = -EINVAL;
1392	goto err;
1393	}
1394
1395	inbuf.pgid = fl->tgid;
1396	inbuf.namelen = strlen(current->comm) + 1;
1397	inbuf.filelen = init.filelen;
1398	inbuf.pageslen = 1;
1399	inbuf.attrs = init.attrs;
1400	inbuf.siglen = init.siglen;
1401	fl->pd = USER_PD;
1402
1403	if (init.filelen && init.filefd) {
1404	err = fastrpc_map_create(fl, fd: init.filefd, len: init.filelen, attr: 0, ppmap: &map);
1405	if (err)
1406	goto err;
1407	}
1408
1409	memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1410	1024 * 1024);
1411	err = fastrpc_buf_alloc(fl, dev: fl->sctx->dev, size: memlen,
1412	obuf: &imem);
1413	if (err)
1414	goto err_alloc;
1415
1416	fl->init_mem = imem;
1417	args[0].ptr = (u64)(uintptr_t)&inbuf;
1418	args[0].length = sizeof(inbuf);
1419	args[0].fd = -1;
1420
1421	args[1].ptr = (u64)(uintptr_t)current->comm;
1422	args[1].length = inbuf.namelen;
1423	args[1].fd = -1;
1424
1425	args[2].ptr = (u64) init.file;
1426	args[2].length = inbuf.filelen;
1427	args[2].fd = init.filefd;
1428
1429	pages[0].addr = imem->phys;
1430	pages[0].size = imem->size;
1431
1432	args[3].ptr = (u64)(uintptr_t) pages;
1433	args[3].length = 1 * sizeof(*pages);
1434	args[3].fd = -1;
1435
1436	args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;
1437	args[4].length = sizeof(inbuf.attrs);
1438	args[4].fd = -1;
1439
1440	args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;
1441	args[5].length = sizeof(inbuf.siglen);
1442	args[5].fd = -1;
1443
1444	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);
1445	if (init.attrs)
1446	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 4, 0);
1447
1448	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE,
1449	sc, args);
1450	if (err)
1451	goto err_invoke;
1452
1453	kfree(objp: args);
1454
1455	return 0;
1456
1457	err_invoke:
1458	fl->init_mem = NULL;
1459	fastrpc_buf_free(buf: imem);
1460	err_alloc:
1461	fastrpc_map_put(map);
1462	err:
1463	kfree(objp: args);
1464
1465	return err;
1466	}
1467
1468	static struct fastrpc_session_ctx *fastrpc_session_alloc(
1469	struct fastrpc_channel_ctx *cctx)
1470	{
1471	struct fastrpc_session_ctx *session = NULL;
1472	unsigned long flags;
1473	int i;
1474
1475	spin_lock_irqsave(&cctx->lock, flags);
1476	for (i = 0; i < cctx->sesscount; i++) {
1477	if (!cctx->session[i].used && cctx->session[i].valid) {
1478	cctx->session[i].used = true;
1479	session = &cctx->session[i];
1480	break;
1481	}
1482	}
1483	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1484
1485	return session;
1486	}
1487
1488	static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
1489	struct fastrpc_session_ctx *session)
1490	{
1491	unsigned long flags;
1492
1493	spin_lock_irqsave(&cctx->lock, flags);
1494	session->used = false;
1495	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1496	}
1497
1498	static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
1499	{
1500	struct fastrpc_invoke_args args[1];
1501	int tgid = 0;
1502	u32 sc;
1503
1504	tgid = fl->tgid;
1505	args[0].ptr = (u64)(uintptr_t) &tgid;
1506	args[0].length = sizeof(tgid);
1507	args[0].fd = -1;
1508	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1509
1510	return fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE,
1511	sc, args: &args[0]);
1512	}
1513
1514	static int fastrpc_device_release(struct inode *inode, struct file *file)
1515	{
1516	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1517	struct fastrpc_channel_ctx *cctx = fl->cctx;
1518	struct fastrpc_invoke_ctx *ctx, *n;
1519	struct fastrpc_map *map, *m;
1520	struct fastrpc_buf *buf, *b;
1521	unsigned long flags;
1522
1523	fastrpc_release_current_dsp_process(fl);
1524
1525	spin_lock_irqsave(&cctx->lock, flags);
1526	list_del(entry: &fl->user);
1527	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1528
1529	if (fl->init_mem)
1530	fastrpc_buf_free(buf: fl->init_mem);
1531
1532	list_for_each_entry_safe(ctx, n, &fl->pending, node) {
1533	list_del(entry: &ctx->node);
1534	fastrpc_context_put(ctx);
1535	}
1536
1537	list_for_each_entry_safe(map, m, &fl->maps, node)
1538	fastrpc_map_put(map);
1539
1540	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1541	list_del(entry: &buf->node);
1542	fastrpc_buf_free(buf);
1543	}
1544
1545	fastrpc_session_free(cctx, session: fl->sctx);
1546	fastrpc_channel_ctx_put(cctx);
1547
1548	mutex_destroy(lock: &fl->mutex);
1549	kfree(objp: fl);
1550	file->private_data = NULL;
1551
1552	return 0;
1553	}
1554
1555	static int fastrpc_device_open(struct inode *inode, struct file *filp)
1556	{
1557	struct fastrpc_channel_ctx *cctx;
1558	struct fastrpc_device *fdevice;
1559	struct fastrpc_user *fl = NULL;
1560	unsigned long flags;
1561
1562	fdevice = miscdev_to_fdevice(filp->private_data);
1563	cctx = fdevice->cctx;
1564
1565	fl = kzalloc(size: sizeof(*fl), GFP_KERNEL);
1566	if (!fl)
1567	return -ENOMEM;
1568
1569	/* Released in fastrpc_device_release() */
1570	fastrpc_channel_ctx_get(cctx);
1571
1572	filp->private_data = fl;
1573	spin_lock_init(&fl->lock);
1574	mutex_init(&fl->mutex);
1575	INIT_LIST_HEAD(list: &fl->pending);
1576	INIT_LIST_HEAD(list: &fl->maps);
1577	INIT_LIST_HEAD(list: &fl->mmaps);
1578	INIT_LIST_HEAD(list: &fl->user);
1579	fl->tgid = current->tgid;
1580	fl->cctx = cctx;
1581	fl->is_secure_dev = fdevice->secure;
1582
1583	fl->sctx = fastrpc_session_alloc(cctx);
1584	if (!fl->sctx) {
1585	dev_err(&cctx->rpdev->dev, "No session available\n");
1586	mutex_destroy(lock: &fl->mutex);
1587	kfree(objp: fl);
1588
1589	return -EBUSY;
1590	}
1591
1592	spin_lock_irqsave(&cctx->lock, flags);
1593	list_add_tail(new: &fl->user, head: &cctx->users);
1594	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1595
1596	return 0;
1597	}
1598
1599	static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)
1600	{
1601	struct fastrpc_alloc_dma_buf bp;
1602	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1603	struct fastrpc_buf *buf = NULL;
1604	int err;
1605
1606	if (copy_from_user(to: &bp, from: argp, n: sizeof(bp)))
1607	return -EFAULT;
1608
1609	err = fastrpc_buf_alloc(fl, dev: fl->sctx->dev, size: bp.size, obuf: &buf);
1610	if (err)
1611	return err;
1612	exp_info.ops = &fastrpc_dma_buf_ops;
1613	exp_info.size = bp.size;
1614	exp_info.flags = O_RDWR;
1615	exp_info.priv = buf;
1616	buf->dmabuf = dma_buf_export(exp_info: &exp_info);
1617	if (IS_ERR(ptr: buf->dmabuf)) {
1618	err = PTR_ERR(ptr: buf->dmabuf);
1619	fastrpc_buf_free(buf);
1620	return err;
1621	}
1622
1623	bp.fd = dma_buf_fd(dmabuf: buf->dmabuf, O_ACCMODE);
1624	if (bp.fd < 0) {
1625	dma_buf_put(dmabuf: buf->dmabuf);
1626	return -EINVAL;
1627	}
1628
1629	if (copy_to_user(to: argp, from: &bp, n: sizeof(bp))) {
1630	/*
1631	* The usercopy failed, but we can't do much about it, as
1632	* dma_buf_fd() already called fd_install() and made the
1633	* file descriptor accessible for the current process. It
1634	* might already be closed and dmabuf no longer valid when
1635	* we reach this point. Therefore "leak" the fd and rely on
1636	* the process exit path to do any required cleanup.
1637	*/
1638	return -EFAULT;
1639	}
1640
1641	return 0;
1642	}
1643
1644	static int fastrpc_init_attach(struct fastrpc_user *fl, int pd)
1645	{
1646	struct fastrpc_invoke_args args[1];
1647	int tgid = fl->tgid;
1648	u32 sc;
1649
1650	args[0].ptr = (u64)(uintptr_t) &tgid;
1651	args[0].length = sizeof(tgid);
1652	args[0].fd = -1;
1653	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1654	fl->pd = pd;
1655
1656	return fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE,
1657	sc, args: &args[0]);
1658	}
1659
1660	static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)
1661	{
1662	struct fastrpc_invoke_args *args = NULL;
1663	struct fastrpc_invoke inv;
1664	u32 nscalars;
1665	int err;
1666
1667	if (copy_from_user(to: &inv, from: argp, n: sizeof(inv)))
1668	return -EFAULT;
1669
1670	/* nscalars is truncated here to max supported value */
1671	nscalars = REMOTE_SCALARS_LENGTH(inv.sc);
1672	if (nscalars) {
1673	args = kcalloc(n: nscalars, size: sizeof(*args), GFP_KERNEL);
1674	if (!args)
1675	return -ENOMEM;
1676
1677	if (copy_from_user(to: args, from: (void __user *)(uintptr_t)inv.args,
1678	n: nscalars * sizeof(*args))) {
1679	kfree(objp: args);
1680	return -EFAULT;
1681	}
1682	}
1683
1684	err = fastrpc_internal_invoke(fl, kernel: false, handle: inv.handle, sc: inv.sc, args);
1685	kfree(objp: args);
1686
1687	return err;
1688	}
1689
1690	static int fastrpc_get_info_from_dsp(struct fastrpc_user *fl, uint32_t *dsp_attr_buf,
1691	uint32_t dsp_attr_buf_len)
1692	{
1693	struct fastrpc_invoke_args args[2] = { 0 };
1694
1695	/* Capability filled in userspace */
1696	dsp_attr_buf[0] = 0;
1697
1698	args[0].ptr = (u64)(uintptr_t)&dsp_attr_buf_len;
1699	args[0].length = sizeof(dsp_attr_buf_len);
1700	args[0].fd = -1;
1701	args[1].ptr = (u64)(uintptr_t)&dsp_attr_buf[1];
1702	args[1].length = dsp_attr_buf_len;
1703	args[1].fd = -1;
1704	fl->pd = USER_PD;
1705
1706	return fastrpc_internal_invoke(fl, kernel: true, FASTRPC_DSP_UTILITIES_HANDLE,
1707	FASTRPC_SCALARS(0, 1, 1), args);
1708	}
1709
1710	static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap,
1711	struct fastrpc_user *fl)
1712	{
1713	struct fastrpc_channel_ctx *cctx = fl->cctx;
1714	uint32_t attribute_id = cap->attribute_id;
1715	uint32_t *dsp_attributes;
1716	unsigned long flags;
1717	uint32_t domain = cap->domain;
1718	int err;
1719
1720	spin_lock_irqsave(&cctx->lock, flags);
1721	/* check if we already have queried dsp for attributes */
1722	if (cctx->valid_attributes) {
1723	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1724	goto done;
1725	}
1726	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1727
1728	dsp_attributes = kzalloc(FASTRPC_MAX_DSP_ATTRIBUTES_LEN, GFP_KERNEL);
1729	if (!dsp_attributes)
1730	return -ENOMEM;
1731
1732	err = fastrpc_get_info_from_dsp(fl, dsp_attr_buf: dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN);
1733	if (err == DSP_UNSUPPORTED_API) {
1734	dev_info(&cctx->rpdev->dev,
1735	"Warning: DSP capabilities not supported on domain: %d\n", domain);
1736	kfree(objp: dsp_attributes);
1737	return -EOPNOTSUPP;
1738	} else if (err) {
1739	dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err);
1740	kfree(objp: dsp_attributes);
1741	return err;
1742	}
1743
1744	spin_lock_irqsave(&cctx->lock, flags);
1745	memcpy(cctx->dsp_attributes, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN);
1746	cctx->valid_attributes = true;
1747	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1748	kfree(objp: dsp_attributes);
1749	done:
1750	cap->capability = cctx->dsp_attributes[attribute_id];
1751	return 0;
1752	}
1753
1754	static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp)
1755	{
1756	struct fastrpc_ioctl_capability cap = {0};
1757	int err = 0;
1758
1759	if (copy_from_user(to: &cap, from: argp, n: sizeof(cap)))
1760	return -EFAULT;
1761
1762	cap.capability = 0;
1763	if (cap.domain >= FASTRPC_DEV_MAX) {
1764	dev_err(&fl->cctx->rpdev->dev, "Error: Invalid domain id:%d, err:%d\n",
1765	cap.domain, err);
1766	return -ECHRNG;
1767	}
1768
1769	/* Fastrpc Capablities does not support modem domain */
1770	if (cap.domain == MDSP_DOMAIN_ID) {
1771	dev_err(&fl->cctx->rpdev->dev, "Error: modem not supported %d\n", err);
1772	return -ECHRNG;
1773	}
1774
1775	if (cap.attribute_id >= FASTRPC_MAX_DSP_ATTRIBUTES) {
1776	dev_err(&fl->cctx->rpdev->dev, "Error: invalid attribute: %d, err: %d\n",
1777	cap.attribute_id, err);
1778	return -EOVERFLOW;
1779	}
1780
1781	err = fastrpc_get_info_from_kernel(cap: &cap, fl);
1782	if (err)
1783	return err;
1784
1785	if (copy_to_user(to: argp, from: &cap.capability, n: sizeof(cap.capability)))
1786	return -EFAULT;
1787
1788	return 0;
1789	}
1790
1791	static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf *buf)
1792	{
1793	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1794	struct fastrpc_munmap_req_msg req_msg;
1795	struct device *dev = fl->sctx->dev;
1796	int err;
1797	u32 sc;
1798
1799	req_msg.pgid = fl->tgid;
1800	req_msg.size = buf->size;
1801	req_msg.vaddr = buf->raddr;
1802
1803	args[0].ptr = (u64) (uintptr_t) &req_msg;
1804	args[0].length = sizeof(req_msg);
1805
1806	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0);
1807	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE, sc,
1808	args: &args[0]);
1809	if (!err) {
1810	dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);
1811	spin_lock(lock: &fl->lock);
1812	list_del(entry: &buf->node);
1813	spin_unlock(lock: &fl->lock);
1814	fastrpc_buf_free(buf);
1815	} else {
1816	dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);
1817	}
1818
1819	return err;
1820	}
1821
1822	static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
1823	{
1824	struct fastrpc_buf *buf = NULL, *iter, *b;
1825	struct fastrpc_req_munmap req;
1826	struct device *dev = fl->sctx->dev;
1827
1828	if (copy_from_user(to: &req, from: argp, n: sizeof(req)))
1829	return -EFAULT;
1830
1831	spin_lock(lock: &fl->lock);
1832	list_for_each_entry_safe(iter, b, &fl->mmaps, node) {
1833	if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) {
1834	buf = iter;
1835	break;
1836	}
1837	}
1838	spin_unlock(lock: &fl->lock);
1839
1840	if (!buf) {
1841	dev_err(dev, "mmap\t\tpt 0x%09llx [len 0x%08llx] not in list\n",
1842	req.vaddrout, req.size);
1843	return -EINVAL;
1844	}
1845
1846	return fastrpc_req_munmap_impl(fl, buf);
1847	}
1848
1849	static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
1850	{
1851	struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } };
1852	struct fastrpc_buf *buf = NULL;
1853	struct fastrpc_mmap_req_msg req_msg;
1854	struct fastrpc_mmap_rsp_msg rsp_msg;
1855	struct fastrpc_phy_page pages;
1856	struct fastrpc_req_mmap req;
1857	struct device *dev = fl->sctx->dev;
1858	int err;
1859	u32 sc;
1860
1861	if (copy_from_user(to: &req, from: argp, n: sizeof(req)))
1862	return -EFAULT;
1863
1864	if (req.flags != ADSP_MMAP_ADD_PAGES && req.flags != ADSP_MMAP_REMOTE_HEAP_ADDR) {
1865	dev_err(dev, "flag not supported 0x%x\n", req.flags);
1866
1867	return -EINVAL;
1868	}
1869
1870	if (req.vaddrin) {
1871	dev_err(dev, "adding user allocated pages is not supported\n");
1872	return -EINVAL;
1873	}
1874
1875	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR)
1876	err = fastrpc_remote_heap_alloc(fl, dev, size: req.size, obuf: &buf);
1877	else
1878	err = fastrpc_buf_alloc(fl, dev, size: req.size, obuf: &buf);
1879
1880	if (err) {
1881	dev_err(dev, "failed to allocate buffer\n");
1882	return err;
1883	}
1884
1885	req_msg.pgid = fl->tgid;
1886	req_msg.flags = req.flags;
1887	req_msg.vaddr = req.vaddrin;
1888	req_msg.num = sizeof(pages);
1889
1890	args[0].ptr = (u64) (uintptr_t) &req_msg;
1891	args[0].length = sizeof(req_msg);
1892
1893	pages.addr = buf->phys;
1894	pages.size = buf->size;
1895
1896	args[1].ptr = (u64) (uintptr_t) &pages;
1897	args[1].length = sizeof(pages);
1898
1899	args[2].ptr = (u64) (uintptr_t) &rsp_msg;
1900	args[2].length = sizeof(rsp_msg);
1901
1902	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1);
1903	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE, sc,
1904	args: &args[0]);
1905	if (err) {
1906	dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size);
1907	goto err_invoke;
1908	}
1909
1910	/* update the buffer to be able to deallocate the memory on the DSP */
1911	buf->raddr = (uintptr_t) rsp_msg.vaddr;
1912
1913	/* let the client know the address to use */
1914	req.vaddrout = rsp_msg.vaddr;
1915
1916	/* Add memory to static PD pool, protection thru hypervisor */
1917	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR && fl->cctx->vmcount) {
1918	err = qcom_scm_assign_mem(mem_addr: buf->phys, mem_sz: (u64)buf->size,
1919	src: &fl->cctx->perms, newvm: fl->cctx->vmperms, dest_cnt: fl->cctx->vmcount);
1920	if (err) {
1921	dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
1922	buf->phys, buf->size, err);
1923	goto err_assign;
1924	}
1925	}
1926
1927	spin_lock(lock: &fl->lock);
1928	list_add_tail(new: &buf->node, head: &fl->mmaps);
1929	spin_unlock(lock: &fl->lock);
1930
1931	if (copy_to_user(to: (void __user *)argp, from: &req, n: sizeof(req))) {
1932	err = -EFAULT;
1933	goto err_assign;
1934	}
1935
1936	dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",
1937	buf->raddr, buf->size);
1938
1939	return 0;
1940
1941	err_assign:
1942	fastrpc_req_munmap_impl(fl, buf);
1943	err_invoke:
1944	fastrpc_buf_free(buf);
1945
1946	return err;
1947	}
1948
1949	static int fastrpc_req_mem_unmap_impl(struct fastrpc_user *fl, struct fastrpc_mem_unmap *req)
1950	{
1951	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1952	struct fastrpc_map *map = NULL, *iter, *m;
1953	struct fastrpc_mem_unmap_req_msg req_msg = { 0 };
1954	int err = 0;
1955	u32 sc;
1956	struct device *dev = fl->sctx->dev;
1957
1958	spin_lock(lock: &fl->lock);
1959	list_for_each_entry_safe(iter, m, &fl->maps, node) {
1960	if ((req->fd < 0 || iter->fd == req->fd) && (iter->raddr == req->vaddr)) {
1961	map = iter;
1962	break;
1963	}
1964	}
1965
1966	spin_unlock(lock: &fl->lock);
1967
1968	if (!map) {
1969	dev_err(dev, "map not in list\n");
1970	return -EINVAL;
1971	}
1972
1973	req_msg.pgid = fl->tgid;
1974	req_msg.len = map->len;
1975	req_msg.vaddrin = map->raddr;
1976	req_msg.fd = map->fd;
1977
1978	args[0].ptr = (u64) (uintptr_t) &req_msg;
1979	args[0].length = sizeof(req_msg);
1980
1981	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_UNMAP, 1, 0);
1982	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE, sc,
1983	args: &args[0]);
1984	if (err) {
1985	dev_err(dev, "unmmap\tpt fd = %d, 0x%09llx error\n", map->fd, map->raddr);
1986	return err;
1987	}
1988	fastrpc_map_put(map);
1989
1990	return 0;
1991	}
1992
1993	static int fastrpc_req_mem_unmap(struct fastrpc_user *fl, char __user *argp)
1994	{
1995	struct fastrpc_mem_unmap req;
1996
1997	if (copy_from_user(to: &req, from: argp, n: sizeof(req)))
1998	return -EFAULT;
1999
2000	return fastrpc_req_mem_unmap_impl(fl, req: &req);
2001	}
2002
2003	static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp)
2004	{
2005	struct fastrpc_invoke_args args[4] = { [0 ... 3] = { 0 } };
2006	struct fastrpc_mem_map_req_msg req_msg = { 0 };
2007	struct fastrpc_mmap_rsp_msg rsp_msg = { 0 };
2008	struct fastrpc_mem_unmap req_unmap = { 0 };
2009	struct fastrpc_phy_page pages = { 0 };
2010	struct fastrpc_mem_map req;
2011	struct device *dev = fl->sctx->dev;
2012	struct fastrpc_map *map = NULL;
2013	int err;
2014	u32 sc;
2015
2016	if (copy_from_user(to: &req, from: argp, n: sizeof(req)))
2017	return -EFAULT;
2018
2019	/* create SMMU mapping */
2020	err = fastrpc_map_create(fl, fd: req.fd, len: req.length, attr: 0, ppmap: &map);
2021	if (err) {
2022	dev_err(dev, "failed to map buffer, fd = %d\n", req.fd);
2023	return err;
2024	}
2025
2026	req_msg.pgid = fl->tgid;
2027	req_msg.fd = req.fd;
2028	req_msg.offset = req.offset;
2029	req_msg.vaddrin = req.vaddrin;
2030	map->va = (void *) (uintptr_t) req.vaddrin;
2031	req_msg.flags = req.flags;
2032	req_msg.num = sizeof(pages);
2033	req_msg.data_len = 0;
2034
2035	args[0].ptr = (u64) (uintptr_t) &req_msg;
2036	args[0].length = sizeof(req_msg);
2037
2038	pages.addr = map->phys;
2039	pages.size = map->size;
2040
2041	args[1].ptr = (u64) (uintptr_t) &pages;
2042	args[1].length = sizeof(pages);
2043
2044	args[2].ptr = (u64) (uintptr_t) &pages;
2045	args[2].length = 0;
2046
2047	args[3].ptr = (u64) (uintptr_t) &rsp_msg;
2048	args[3].length = sizeof(rsp_msg);
2049
2050	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_MAP, 3, 1);
2051	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE, sc, args: &args[0]);
2052	if (err) {
2053	dev_err(dev, "mem mmap error, fd %d, vaddr %llx, size %lld\n",
2054	req.fd, req.vaddrin, map->size);
2055	goto err_invoke;
2056	}
2057
2058	/* update the buffer to be able to deallocate the memory on the DSP */
2059	map->raddr = rsp_msg.vaddr;
2060
2061	/* let the client know the address to use */
2062	req.vaddrout = rsp_msg.vaddr;
2063
2064	if (copy_to_user(to: (void __user *)argp, from: &req, n: sizeof(req))) {
2065	/* unmap the memory and release the buffer */
2066	req_unmap.vaddr = (uintptr_t) rsp_msg.vaddr;
2067	req_unmap.length = map->size;
2068	fastrpc_req_mem_unmap_impl(fl, req: &req_unmap);
2069	return -EFAULT;
2070	}
2071
2072	return 0;
2073
2074	err_invoke:
2075	fastrpc_map_put(map);
2076
2077	return err;
2078	}
2079
2080	static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,
2081	unsigned long arg)
2082	{
2083	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
2084	char __user *argp = (char __user *)arg;
2085	int err;
2086
2087	switch (cmd) {
2088	case FASTRPC_IOCTL_INVOKE:
2089	err = fastrpc_invoke(fl, argp);
2090	break;
2091	case FASTRPC_IOCTL_INIT_ATTACH:
2092	err = fastrpc_init_attach(fl, ROOT_PD);
2093	break;
2094	case FASTRPC_IOCTL_INIT_ATTACH_SNS:
2095	err = fastrpc_init_attach(fl, SENSORS_PD);
2096	break;
2097	case FASTRPC_IOCTL_INIT_CREATE_STATIC:
2098	err = fastrpc_init_create_static_process(fl, argp);
2099	break;
2100	case FASTRPC_IOCTL_INIT_CREATE:
2101	err = fastrpc_init_create_process(fl, argp);
2102	break;
2103	case FASTRPC_IOCTL_ALLOC_DMA_BUFF:
2104	err = fastrpc_dmabuf_alloc(fl, argp);
2105	break;
2106	case FASTRPC_IOCTL_MMAP:
2107	err = fastrpc_req_mmap(fl, argp);
2108	break;
2109	case FASTRPC_IOCTL_MUNMAP:
2110	err = fastrpc_req_munmap(fl, argp);
2111	break;
2112	case FASTRPC_IOCTL_MEM_MAP:
2113	err = fastrpc_req_mem_map(fl, argp);
2114	break;
2115	case FASTRPC_IOCTL_MEM_UNMAP:
2116	err = fastrpc_req_mem_unmap(fl, argp);
2117	break;
2118	case FASTRPC_IOCTL_GET_DSP_INFO:
2119	err = fastrpc_get_dsp_info(fl, argp);
2120	break;
2121	default:
2122	err = -ENOTTY;
2123	break;
2124	}
2125
2126	return err;
2127	}
2128
2129	static const struct file_operations fastrpc_fops = {
2130	.open = fastrpc_device_open,
2131	.release = fastrpc_device_release,
2132	.unlocked_ioctl = fastrpc_device_ioctl,
2133	.compat_ioctl = fastrpc_device_ioctl,
2134	};
2135
2136	static int fastrpc_cb_probe(struct platform_device *pdev)
2137	{
2138	struct fastrpc_channel_ctx *cctx;
2139	struct fastrpc_session_ctx *sess;
2140	struct device *dev = &pdev->dev;
2141	int i, sessions = 0;
2142	unsigned long flags;
2143	int rc;
2144
2145	cctx = dev_get_drvdata(dev: dev->parent);
2146	if (!cctx)
2147	return -EINVAL;
2148
2149	of_property_read_u32(np: dev->of_node, propname: "qcom,nsessions", out_value: &sessions);
2150
2151	spin_lock_irqsave(&cctx->lock, flags);
2152	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) {
2153	dev_err(&pdev->dev, "too many sessions\n");
2154	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2155	return -ENOSPC;
2156	}
2157	sess = &cctx->session[cctx->sesscount++];
2158	sess->used = false;
2159	sess->valid = true;
2160	sess->dev = dev;
2161	dev_set_drvdata(dev, data: sess);
2162
2163	if (of_property_read_u32(np: dev->of_node, propname: "reg", out_value: &sess->sid))
2164	dev_info(dev, "FastRPC Session ID not specified in DT\n");
2165
2166	if (sessions > 0) {
2167	struct fastrpc_session_ctx *dup_sess;
2168
2169	for (i = 1; i < sessions; i++) {
2170	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS)
2171	break;
2172	dup_sess = &cctx->session[cctx->sesscount++];
2173	memcpy(dup_sess, sess, sizeof(*dup_sess));
2174	}
2175	}
2176	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2177	rc = dma_set_mask(dev, DMA_BIT_MASK(32));
2178	if (rc) {
2179	dev_err(dev, "32-bit DMA enable failed\n");
2180	return rc;
2181	}
2182
2183	return 0;
2184	}
2185
2186	static int fastrpc_cb_remove(struct platform_device *pdev)
2187	{
2188	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(dev: pdev->dev.parent);
2189	struct fastrpc_session_ctx *sess = dev_get_drvdata(dev: &pdev->dev);
2190	unsigned long flags;
2191	int i;
2192
2193	spin_lock_irqsave(&cctx->lock, flags);
2194	for (i = 1; i < FASTRPC_MAX_SESSIONS; i++) {
2195	if (cctx->session[i].sid == sess->sid) {
2196	cctx->session[i].valid = false;
2197	cctx->sesscount--;
2198	}
2199	}
2200	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2201
2202	return 0;
2203	}
2204
2205	static const struct of_device_id fastrpc_match_table[] = {
2206	{ .compatible = "qcom,fastrpc-compute-cb", },
2207	{}
2208	};
2209
2210	static struct platform_driver fastrpc_cb_driver = {
2211	.probe = fastrpc_cb_probe,
2212	.remove = fastrpc_cb_remove,
2213	.driver = {
2214	.name = "qcom,fastrpc-cb",
2215	.of_match_table = fastrpc_match_table,
2216	.suppress_bind_attrs = true,
2217	},
2218	};
2219
2220	static int fastrpc_device_register(struct device *dev, struct fastrpc_channel_ctx *cctx,
2221	bool is_secured, const char *domain)
2222	{
2223	struct fastrpc_device *fdev;
2224	int err;
2225
2226	fdev = devm_kzalloc(dev, size: sizeof(*fdev), GFP_KERNEL);
2227	if (!fdev)
2228	return -ENOMEM;
2229
2230	fdev->secure = is_secured;
2231	fdev->cctx = cctx;
2232	fdev->miscdev.minor = MISC_DYNAMIC_MINOR;
2233	fdev->miscdev.fops = &fastrpc_fops;
2234	fdev->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, fmt: "fastrpc-%s%s",
2235	domain, is_secured ? "-secure" : "");
2236	if (!fdev->miscdev.name)
2237	return -ENOMEM;
2238
2239	err = misc_register(misc: &fdev->miscdev);
2240	if (!err) {
2241	if (is_secured)
2242	cctx->secure_fdevice = fdev;
2243	else
2244	cctx->fdevice = fdev;
2245	}
2246
2247	return err;
2248	}
2249
2250	static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
2251	{
2252	struct device *rdev = &rpdev->dev;
2253	struct fastrpc_channel_ctx *data;
2254	int i, err, domain_id = -1, vmcount;
2255	const char *domain;
2256	bool secure_dsp;
2257	unsigned int vmids[FASTRPC_MAX_VMIDS];
2258
2259	err = of_property_read_string(np: rdev->of_node, propname: "label", out_string: &domain);
2260	if (err) {
2261	dev_info(rdev, "FastRPC Domain not specified in DT\n");
2262	return err;
2263	}
2264
2265	for (i = 0; i <= CDSP_DOMAIN_ID; i++) {
2266	if (!strcmp(domains[i], domain)) {
2267	domain_id = i;
2268	break;
2269	}
2270	}
2271
2272	if (domain_id < 0) {
2273	dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);
2274	return -EINVAL;
2275	}
2276
2277	if (of_reserved_mem_device_init_by_idx(dev: rdev, np: rdev->of_node, idx: 0))
2278	dev_info(rdev, "no reserved DMA memory for FASTRPC\n");
2279
2280	vmcount = of_property_read_variable_u32_array(np: rdev->of_node,
2281	propname: "qcom,vmids", out_values: &vmids[0], sz_min: 0, FASTRPC_MAX_VMIDS);
2282	if (vmcount < 0)
2283	vmcount = 0;
2284	else if (!qcom_scm_is_available())
2285	return -EPROBE_DEFER;
2286
2287	data = kzalloc(size: sizeof(*data), GFP_KERNEL);
2288	if (!data)
2289	return -ENOMEM;
2290
2291	if (vmcount) {
2292	data->vmcount = vmcount;
2293	data->perms = BIT(QCOM_SCM_VMID_HLOS);
2294	for (i = 0; i < data->vmcount; i++) {
2295	data->vmperms[i].vmid = vmids[i];
2296	data->vmperms[i].perm = QCOM_SCM_PERM_RWX;
2297	}
2298	}
2299
2300	secure_dsp = !(of_property_read_bool(np: rdev->of_node, propname: "qcom,non-secure-domain"));
2301	data->secure = secure_dsp;
2302
2303	switch (domain_id) {
2304	case ADSP_DOMAIN_ID:
2305	case MDSP_DOMAIN_ID:
2306	case SDSP_DOMAIN_ID:
2307	/* Unsigned PD offloading is only supported on CDSP*/
2308	data->unsigned_support = false;
2309	err = fastrpc_device_register(dev: rdev, cctx: data, is_secured: secure_dsp, domain: domains[domain_id]);
2310	if (err)
2311	goto fdev_error;
2312	break;
2313	case CDSP_DOMAIN_ID:
2314	data->unsigned_support = true;
2315	/* Create both device nodes so that we can allow both Signed and Unsigned PD */
2316	err = fastrpc_device_register(dev: rdev, cctx: data, is_secured: true, domain: domains[domain_id]);
2317	if (err)
2318	goto fdev_error;
2319
2320	err = fastrpc_device_register(dev: rdev, cctx: data, is_secured: false, domain: domains[domain_id]);
2321	if (err)
2322	goto fdev_error;
2323	break;
2324	default:
2325	err = -EINVAL;
2326	goto fdev_error;
2327	}
2328
2329	kref_init(kref: &data->refcount);
2330
2331	dev_set_drvdata(dev: &rpdev->dev, data);
2332	rdev->dma_mask = &data->dma_mask;
2333	dma_set_mask_and_coherent(dev: rdev, DMA_BIT_MASK(32));
2334	INIT_LIST_HEAD(list: &data->users);
2335	INIT_LIST_HEAD(list: &data->invoke_interrupted_mmaps);
2336	spin_lock_init(&data->lock);
2337	idr_init(idr: &data->ctx_idr);
2338	data->domain_id = domain_id;
2339	data->rpdev = rpdev;
2340
2341	err = of_platform_populate(root: rdev->of_node, NULL, NULL, parent: rdev);
2342	if (err)
2343	goto populate_error;
2344
2345	return 0;
2346
2347	populate_error:
2348	if (data->fdevice)
2349	misc_deregister(misc: &data->fdevice->miscdev);
2350	if (data->secure_fdevice)
2351	misc_deregister(misc: &data->secure_fdevice->miscdev);
2352
2353	fdev_error:
2354	kfree(objp: data);
2355	return err;
2356	}
2357
2358	static void fastrpc_notify_users(struct fastrpc_user *user)
2359	{
2360	struct fastrpc_invoke_ctx *ctx;
2361
2362	spin_lock(lock: &user->lock);
2363	list_for_each_entry(ctx, &user->pending, node) {
2364	ctx->retval = -EPIPE;
2365	complete(&ctx->work);
2366	}
2367	spin_unlock(lock: &user->lock);
2368	}
2369
2370	static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
2371	{
2372	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(dev: &rpdev->dev);
2373	struct fastrpc_buf *buf, *b;
2374	struct fastrpc_user *user;
2375	unsigned long flags;
2376
2377	/* No invocations past this point */
2378	spin_lock_irqsave(&cctx->lock, flags);
2379	cctx->rpdev = NULL;
2380	list_for_each_entry(user, &cctx->users, user)
2381	fastrpc_notify_users(user);
2382	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2383
2384	if (cctx->fdevice)
2385	misc_deregister(misc: &cctx->fdevice->miscdev);
2386
2387	if (cctx->secure_fdevice)
2388	misc_deregister(misc: &cctx->secure_fdevice->miscdev);
2389
2390	list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node)
2391	list_del(entry: &buf->node);
2392
2393	if (cctx->remote_heap)
2394	fastrpc_buf_free(buf: cctx->remote_heap);
2395
2396	of_platform_depopulate(parent: &rpdev->dev);
2397
2398	fastrpc_channel_ctx_put(cctx);
2399	}
2400
2401	static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
2402	int len, void *priv, u32 addr)
2403	{
2404	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(dev: &rpdev->dev);
2405	struct fastrpc_invoke_rsp *rsp = data;
2406	struct fastrpc_invoke_ctx *ctx;
2407	unsigned long flags;
2408	unsigned long ctxid;
2409
2410	if (len < sizeof(*rsp))
2411	return -EINVAL;
2412
2413	ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);
2414
2415	spin_lock_irqsave(&cctx->lock, flags);
2416	ctx = idr_find(&cctx->ctx_idr, id: ctxid);
2417	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2418
2419	if (!ctx) {
2420	dev_err(&rpdev->dev, "No context ID matches response\n");
2421	return -ENOENT;
2422	}
2423
2424	ctx->retval = rsp->retval;
2425	complete(&ctx->work);
2426
2427	/*
2428	* The DMA buffer associated with the context cannot be freed in
2429	* interrupt context so schedule it through a worker thread to
2430	* avoid a kernel BUG.
2431	*/
2432	schedule_work(work: &ctx->put_work);
2433
2434	return 0;
2435	}
2436
2437	static const struct of_device_id fastrpc_rpmsg_of_match[] = {
2438	{ .compatible = "qcom,fastrpc" },
2439	{ },
2440	};
2441	MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
2442
2443	static struct rpmsg_driver fastrpc_driver = {
2444	.probe = fastrpc_rpmsg_probe,
2445	.remove = fastrpc_rpmsg_remove,
2446	.callback = fastrpc_rpmsg_callback,
2447	.drv = {
2448	.name = "qcom,fastrpc",
2449	.of_match_table = fastrpc_rpmsg_of_match,
2450	},
2451	};
2452
2453	static int fastrpc_init(void)
2454	{
2455	int ret;
2456
2457	ret = platform_driver_register(&fastrpc_cb_driver);
2458	if (ret < 0) {
2459	pr_err("fastrpc: failed to register cb driver\n");
2460	return ret;
2461	}
2462
2463	ret = register_rpmsg_driver(&fastrpc_driver);
2464	if (ret < 0) {
2465	pr_err("fastrpc: failed to register rpmsg driver\n");
2466	platform_driver_unregister(&fastrpc_cb_driver);
2467	return ret;
2468	}
2469
2470	return 0;
2471	}
2472	module_init(fastrpc_init);
2473
2474	static void fastrpc_exit(void)
2475	{
2476	platform_driver_unregister(&fastrpc_cb_driver);
2477	unregister_rpmsg_driver(drv: &fastrpc_driver);
2478	}
2479	module_exit(fastrpc_exit);
2480
2481	MODULE_LICENSE("GPL v2");
2482	MODULE_IMPORT_NS(DMA_BUF);
2483