1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* f_fs.c -- user mode file system API for USB composite function controllers
4	*
5	* Copyright (C) 2010 Samsung Electronics
6	* Author: Michal Nazarewicz <mina86@mina86.com>
7	*
8	* Based on inode.c (GadgetFS) which was:
9	* Copyright (C) 2003-2004 David Brownell
10	* Copyright (C) 2003 Agilent Technologies
11	*/
12
13
14	/* #define DEBUG */
15	/* #define VERBOSE_DEBUG */
16
17	#include <linux/blkdev.h>
18	#include <linux/dma-buf.h>
19	#include <linux/dma-fence.h>
20	#include <linux/dma-resv.h>
21	#include <linux/pagemap.h>
22	#include <linux/export.h>
23	#include <linux/fs_parser.h>
24	#include <linux/hid.h>
25	#include <linux/mm.h>
26	#include <linux/module.h>
27	#include <linux/scatterlist.h>
28	#include <linux/sched/signal.h>
29	#include <linux/uio.h>
30	#include <linux/vmalloc.h>
31	#include <asm/unaligned.h>
32
33	#include <linux/usb/ccid.h>
34	#include <linux/usb/composite.h>
35	#include <linux/usb/functionfs.h>
36
37	#include <linux/aio.h>
38	#include <linux/kthread.h>
39	#include <linux/poll.h>
40	#include <linux/eventfd.h>
41
42	#include "u_fs.h"
43	#include "u_f.h"
44	#include "u_os_desc.h"
45	#include "configfs.h"
46
47	#define FUNCTIONFS_MAGIC 0xa647361 /* Chosen by a honest dice roll ;) */
48
49	#define DMABUF_ENQUEUE_TIMEOUT_MS 5000
50
51	MODULE_IMPORT_NS(DMA_BUF);
52
53	/* Reference counter handling */
54	static void ffs_data_get(struct ffs_data *ffs);
55	static void ffs_data_put(struct ffs_data *ffs);
56	/* Creates new ffs_data object. */
57	static struct ffs_data *__must_check ffs_data_new(const char *dev_name)
58	__attribute__((malloc));
59
60	/* Opened counter handling. */
61	static void ffs_data_opened(struct ffs_data *ffs);
62	static void ffs_data_closed(struct ffs_data *ffs);
63
64	/* Called with ffs->mutex held; take over ownership of data. */
65	static int __must_check
66	__ffs_data_got_descs(struct ffs_data *ffs, char *data, size_t len);
67	static int __must_check
68	__ffs_data_got_strings(struct ffs_data *ffs, char *data, size_t len);
69
70
71	/* The function structure ***************************************************/
72
73	struct ffs_ep;
74
75	struct ffs_function {
76	struct usb_configuration *conf;
77	struct usb_gadget *gadget;
78	struct ffs_data *ffs;
79
80	struct ffs_ep *eps;
81	u8 eps_revmap[16];
82	short *interfaces_nums;
83
84	struct usb_function function;
85	};
86
87
88	static struct ffs_function *ffs_func_from_usb(struct usb_function *f)
89	{
90	return container_of(f, struct ffs_function, function);
91	}
92
93
94	static inline enum ffs_setup_state
95	ffs_setup_state_clear_cancelled(struct ffs_data *ffs)
96	{
97	return (enum ffs_setup_state)
98	cmpxchg(&ffs->setup_state, FFS_SETUP_CANCELLED, FFS_NO_SETUP);
99	}
100
101
102	static void ffs_func_eps_disable(struct ffs_function *func);
103	static int __must_check ffs_func_eps_enable(struct ffs_function *func);
104
105	static int ffs_func_bind(struct usb_configuration *,
106	struct usb_function *);
107	static int ffs_func_set_alt(struct usb_function *, unsigned, unsigned);
108	static void ffs_func_disable(struct usb_function *);
109	static int ffs_func_setup(struct usb_function *,
110	const struct usb_ctrlrequest *);
111	static bool ffs_func_req_match(struct usb_function *,
112	const struct usb_ctrlrequest *,
113	bool config0);
114	static void ffs_func_suspend(struct usb_function *);
115	static void ffs_func_resume(struct usb_function *);
116
117
118	static int ffs_func_revmap_ep(struct ffs_function *func, u8 num);
119	static int ffs_func_revmap_intf(struct ffs_function *func, u8 intf);
120
121
122	/* The endpoints structures *************************************************/
123
124	struct ffs_ep {
125	struct usb_ep *ep; /* P: ffs->eps_lock */
126	struct usb_request *req; /* P: epfile->mutex */
127
128	/* [0]: full speed, [1]: high speed, [2]: super speed */
129	struct usb_endpoint_descriptor *descs[3];
130
131	u8 num;
132	};
133
134	struct ffs_dmabuf_priv {
135	struct list_head entry;
136	struct kref ref;
137	struct ffs_data *ffs;
138	struct dma_buf_attachment *attach;
139	struct sg_table *sgt;
140	enum dma_data_direction dir;
141	spinlock_t lock;
142	u64 context;
143	struct usb_request *req; /* P: ffs->eps_lock */
144	struct usb_ep *ep; /* P: ffs->eps_lock */
145	};
146
147	struct ffs_dma_fence {
148	struct dma_fence base;
149	struct ffs_dmabuf_priv *priv;
150	struct work_struct work;
151	};
152
153	struct ffs_epfile {
154	/* Protects ep->ep and ep->req. */
155	struct mutex mutex;
156
157	struct ffs_data *ffs;
158	struct ffs_ep *ep; /* P: ffs->eps_lock */
159
160	struct dentry *dentry;
161
162	/*
163	* Buffer for holding data from partial reads which may happen since
164	* we’re rounding user read requests to a multiple of a max packet size.
165	*
166	* The pointer is initialised with NULL value and may be set by
167	* __ffs_epfile_read_data function to point to a temporary buffer.
168	*
169	* In normal operation, calls to __ffs_epfile_read_buffered will consume
170	* data from said buffer and eventually free it. Importantly, while the
171	* function is using the buffer, it sets the pointer to NULL. This is
172	* all right since __ffs_epfile_read_data and __ffs_epfile_read_buffered
173	* can never run concurrently (they are synchronised by epfile->mutex)
174	* so the latter will not assign a new value to the pointer.
175	*
176	* Meanwhile ffs_func_eps_disable frees the buffer (if the pointer is
177	* valid) and sets the pointer to READ_BUFFER_DROP value. This special
178	* value is crux of the synchronisation between ffs_func_eps_disable and
179	* __ffs_epfile_read_data.
180	*
181	* Once __ffs_epfile_read_data is about to finish it will try to set the
182	* pointer back to its old value (as described above), but seeing as the
183	* pointer is not-NULL (namely READ_BUFFER_DROP) it will instead free
184	* the buffer.
185	*
186	* == State transitions ==
187	*
188	* • ptr == NULL: (initial state)
189	* ◦ __ffs_epfile_read_buffer_free: go to ptr == DROP
190	* ◦ __ffs_epfile_read_buffered: nop
191	* ◦ __ffs_epfile_read_data allocates temp buffer: go to ptr == buf
192	* ◦ reading finishes: n/a, not in ‘and reading’ state
193	* • ptr == DROP:
194	* ◦ __ffs_epfile_read_buffer_free: nop
195	* ◦ __ffs_epfile_read_buffered: go to ptr == NULL
196	* ◦ __ffs_epfile_read_data allocates temp buffer: free buf, nop
197	* ◦ reading finishes: n/a, not in ‘and reading’ state
198	* • ptr == buf:
199	* ◦ __ffs_epfile_read_buffer_free: free buf, go to ptr == DROP
200	* ◦ __ffs_epfile_read_buffered: go to ptr == NULL and reading
201	* ◦ __ffs_epfile_read_data: n/a, __ffs_epfile_read_buffered
202	* is always called first
203	* ◦ reading finishes: n/a, not in ‘and reading’ state
204	* • ptr == NULL and reading:
205	* ◦ __ffs_epfile_read_buffer_free: go to ptr == DROP and reading
206	* ◦ __ffs_epfile_read_buffered: n/a, mutex is held
207	* ◦ __ffs_epfile_read_data: n/a, mutex is held
208	* ◦ reading finishes and …
209	* … all data read: free buf, go to ptr == NULL
210	* … otherwise: go to ptr == buf and reading
211	* • ptr == DROP and reading:
212	* ◦ __ffs_epfile_read_buffer_free: nop
213	* ◦ __ffs_epfile_read_buffered: n/a, mutex is held
214	* ◦ __ffs_epfile_read_data: n/a, mutex is held
215	* ◦ reading finishes: free buf, go to ptr == DROP
216	*/
217	struct ffs_buffer *read_buffer;
218	#define READ_BUFFER_DROP ((struct ffs_buffer *)ERR_PTR(-ESHUTDOWN))
219
220	char name[5];
221
222	unsigned char in; /* P: ffs->eps_lock */
223	unsigned char isoc; /* P: ffs->eps_lock */
224
225	unsigned char _pad;
226
227	/* Protects dmabufs */
228	struct mutex dmabufs_mutex;
229	struct list_head dmabufs; /* P: dmabufs_mutex */
230	atomic_t seqno;
231	};
232
233	struct ffs_buffer {
234	size_t length;
235	char *data;
236	char storage[] __counted_by(length);
237	};
238
239	/* ffs_io_data structure ***************************************************/
240
241	struct ffs_io_data {
242	bool aio;
243	bool read;
244
245	struct kiocb *kiocb;
246	struct iov_iter data;
247	const void *to_free;
248	char *buf;
249
250	struct mm_struct *mm;
251	struct work_struct work;
252
253	struct usb_ep *ep;
254	struct usb_request *req;
255	struct sg_table sgt;
256	bool use_sg;
257
258	struct ffs_data *ffs;
259
260	int status;
261	struct completion done;
262	};
263
264	struct ffs_desc_helper {
265	struct ffs_data *ffs;
266	unsigned interfaces_count;
267	unsigned eps_count;
268	};
269
270	static int __must_check ffs_epfiles_create(struct ffs_data *ffs);
271	static void ffs_epfiles_destroy(struct ffs_epfile *epfiles, unsigned count);
272
273	static struct dentry *
274	ffs_sb_create_file(struct super_block *sb, const char *name, void *data,
275	const struct file_operations *fops);
276
277	/* Devices management *******************************************************/
278
279	DEFINE_MUTEX(ffs_lock);
280	EXPORT_SYMBOL_GPL(ffs_lock);
281
282	static struct ffs_dev *_ffs_find_dev(const char *name);
283	static struct ffs_dev *_ffs_alloc_dev(void);
284	static void _ffs_free_dev(struct ffs_dev *dev);
285	static int ffs_acquire_dev(const char *dev_name, struct ffs_data *ffs_data);
286	static void ffs_release_dev(struct ffs_dev *ffs_dev);
287	static int ffs_ready(struct ffs_data *ffs);
288	static void ffs_closed(struct ffs_data *ffs);
289
290	/* Misc helper functions ****************************************************/
291
292	static int ffs_mutex_lock(struct mutex *mutex, unsigned nonblock)
293	__attribute__((warn_unused_result, nonnull));
294	static char *ffs_prepare_buffer(const char __user *buf, size_t len)
295	__attribute__((warn_unused_result, nonnull));
296
297
298	/* Control file aka ep0 *****************************************************/
299
300	static void ffs_ep0_complete(struct usb_ep *ep, struct usb_request *req)
301	{
302	struct ffs_data *ffs = req->context;
303
304	complete(&ffs->ep0req_completion);
305	}
306
307	static int __ffs_ep0_queue_wait(struct ffs_data *ffs, char *data, size_t len)
308	__releases(&ffs->ev.waitq.lock)
309	{
310	struct usb_request *req = ffs->ep0req;
311	int ret;
312
313	if (!req) {
314	spin_unlock_irq(lock: &ffs->ev.waitq.lock);
315	return -EINVAL;
316	}
317
318	req->zero = len < le16_to_cpu(ffs->ev.setup.wLength);
319
320	spin_unlock_irq(lock: &ffs->ev.waitq.lock);
321
322	req->buf = data;
323	req->length = len;
324
325	/*
326	* UDC layer requires to provide a buffer even for ZLP, but should
327	* not use it at all. Let's provide some poisoned pointer to catch
328	* possible bug in the driver.
329	*/
330	if (req->buf == NULL)
331	req->buf = (void *)0xDEADBABE;
332
333	reinit_completion(x: &ffs->ep0req_completion);
334
335	ret = usb_ep_queue(ep: ffs->gadget->ep0, req, GFP_ATOMIC);
336	if (ret < 0)
337	return ret;
338
339	ret = wait_for_completion_interruptible(x: &ffs->ep0req_completion);
340	if (ret) {
341	usb_ep_dequeue(ep: ffs->gadget->ep0, req);
342	return -EINTR;
343	}
344
345	ffs->setup_state = FFS_NO_SETUP;
346	return req->status ? req->status : req->actual;
347	}
348
349	static int __ffs_ep0_stall(struct ffs_data *ffs)
350	{
351	if (ffs->ev.can_stall) {
352	pr_vdebug("ep0 stall\n");
353	usb_ep_set_halt(ep: ffs->gadget->ep0);
354	ffs->setup_state = FFS_NO_SETUP;
355	return -EL2HLT;
356	} else {
357	pr_debug("bogus ep0 stall!\n");
358	return -ESRCH;
359	}
360	}
361
362	static ssize_t ffs_ep0_write(struct file *file, const char __user *buf,
363	size_t len, loff_t *ptr)
364	{
365	struct ffs_data *ffs = file->private_data;
366	ssize_t ret;
367	char *data;
368
369	/* Fast check if setup was canceled */
370	if (ffs_setup_state_clear_cancelled(ffs) == FFS_SETUP_CANCELLED)
371	return -EIDRM;
372
373	/* Acquire mutex */
374	ret = ffs_mutex_lock(mutex: &ffs->mutex, nonblock: file->f_flags & O_NONBLOCK);
375	if (ret < 0)
376	return ret;
377
378	/* Check state */
379	switch (ffs->state) {
380	case FFS_READ_DESCRIPTORS:
381	case FFS_READ_STRINGS:
382	/* Copy data */
383	if (len < 16) {
384	ret = -EINVAL;
385	break;
386	}
387
388	data = ffs_prepare_buffer(buf, len);
389	if (IS_ERR(ptr: data)) {
390	ret = PTR_ERR(ptr: data);
391	break;
392	}
393
394	/* Handle data */
395	if (ffs->state == FFS_READ_DESCRIPTORS) {
396	pr_info("read descriptors\n");
397	ret = __ffs_data_got_descs(ffs, data, len);
398	if (ret < 0)
399	break;
400
401	ffs->state = FFS_READ_STRINGS;
402	ret = len;
403	} else {
404	pr_info("read strings\n");
405	ret = __ffs_data_got_strings(ffs, data, len);
406	if (ret < 0)
407	break;
408
409	ret = ffs_epfiles_create(ffs);
410	if (ret) {
411	ffs->state = FFS_CLOSING;
412	break;
413	}
414
415	ffs->state = FFS_ACTIVE;
416	mutex_unlock(lock: &ffs->mutex);
417
418	ret = ffs_ready(ffs);
419	if (ret < 0) {
420	ffs->state = FFS_CLOSING;
421	return ret;
422	}
423
424	return len;
425	}
426	break;
427
428	case FFS_ACTIVE:
429	data = NULL;
430	/*
431	* We're called from user space, we can use _irq
432	* rather then _irqsave
433	*/
434	spin_lock_irq(lock: &ffs->ev.waitq.lock);
435	switch (ffs_setup_state_clear_cancelled(ffs)) {
436	case FFS_SETUP_CANCELLED:
437	ret = -EIDRM;
438	goto done_spin;
439
440	case FFS_NO_SETUP:
441	ret = -ESRCH;
442	goto done_spin;
443
444	case FFS_SETUP_PENDING:
445	break;
446	}
447
448	/* FFS_SETUP_PENDING */
449	if (!(ffs->ev.setup.bRequestType & USB_DIR_IN)) {
450	spin_unlock_irq(lock: &ffs->ev.waitq.lock);
451	ret = __ffs_ep0_stall(ffs);
452	break;
453	}
454
455	/* FFS_SETUP_PENDING and not stall */
456	len = min(len, (size_t)le16_to_cpu(ffs->ev.setup.wLength));
457
458	spin_unlock_irq(lock: &ffs->ev.waitq.lock);
459
460	data = ffs_prepare_buffer(buf, len);
461	if (IS_ERR(ptr: data)) {
462	ret = PTR_ERR(ptr: data);
463	break;
464	}
465
466	spin_lock_irq(lock: &ffs->ev.waitq.lock);
467
468	/*
469	* We are guaranteed to be still in FFS_ACTIVE state
470	* but the state of setup could have changed from
471	* FFS_SETUP_PENDING to FFS_SETUP_CANCELLED so we need
472	* to check for that. If that happened we copied data
473	* from user space in vain but it's unlikely.
474	*
475	* For sure we are not in FFS_NO_SETUP since this is
476	* the only place FFS_SETUP_PENDING -> FFS_NO_SETUP
477	* transition can be performed and it's protected by
478	* mutex.
479	*/
480	if (ffs_setup_state_clear_cancelled(ffs) ==
481	FFS_SETUP_CANCELLED) {
482	ret = -EIDRM;
483	done_spin:
484	spin_unlock_irq(lock: &ffs->ev.waitq.lock);
485	} else {
486	/* unlocks spinlock */
487	ret = __ffs_ep0_queue_wait(ffs, data, len);
488	}
489	kfree(objp: data);
490	break;
491
492	default:
493	ret = -EBADFD;
494	break;
495	}
496
497	mutex_unlock(lock: &ffs->mutex);
498	return ret;
499	}
500
501	/* Called with ffs->ev.waitq.lock and ffs->mutex held, both released on exit. */
502	static ssize_t __ffs_ep0_read_events(struct ffs_data *ffs, char __user *buf,
503	size_t n)
504	__releases(&ffs->ev.waitq.lock)
505	{
506	/*
507	* n cannot be bigger than ffs->ev.count, which cannot be bigger than
508	* size of ffs->ev.types array (which is four) so that's how much space
509	* we reserve.
510	*/
511	struct usb_functionfs_event events[ARRAY_SIZE(ffs->ev.types)];
512	const size_t size = n * sizeof *events;
513	unsigned i = 0;
514
515	memset(events, 0, size);
516
517	do {
518	events[i].type = ffs->ev.types[i];
519	if (events[i].type == FUNCTIONFS_SETUP) {
520	events[i].u.setup = ffs->ev.setup;
521	ffs->setup_state = FFS_SETUP_PENDING;
522	}
523	} while (++i < n);
524
525	ffs->ev.count -= n;
526	if (ffs->ev.count)
527	memmove(ffs->ev.types, ffs->ev.types + n,
528	ffs->ev.count * sizeof *ffs->ev.types);
529
530	spin_unlock_irq(lock: &ffs->ev.waitq.lock);
531	mutex_unlock(lock: &ffs->mutex);
532
533	return copy_to_user(to: buf, from: events, n: size) ? -EFAULT : size;
534	}
535
536	static ssize_t ffs_ep0_read(struct file *file, char __user *buf,
537	size_t len, loff_t *ptr)
538	{
539	struct ffs_data *ffs = file->private_data;
540	char *data = NULL;
541	size_t n;
542	int ret;
543
544	/* Fast check if setup was canceled */
545	if (ffs_setup_state_clear_cancelled(ffs) == FFS_SETUP_CANCELLED)
546	return -EIDRM;
547
548	/* Acquire mutex */
549	ret = ffs_mutex_lock(mutex: &ffs->mutex, nonblock: file->f_flags & O_NONBLOCK);
550	if (ret < 0)
551	return ret;
552
553	/* Check state */
554	if (ffs->state != FFS_ACTIVE) {
555	ret = -EBADFD;
556	goto done_mutex;
557	}
558
559	/*
560	* We're called from user space, we can use _irq rather then
561	* _irqsave
562	*/
563	spin_lock_irq(lock: &ffs->ev.waitq.lock);
564
565	switch (ffs_setup_state_clear_cancelled(ffs)) {
566	case FFS_SETUP_CANCELLED:
567	ret = -EIDRM;
568	break;
569
570	case FFS_NO_SETUP:
571	n = len / sizeof(struct usb_functionfs_event);
572	if (!n) {
573	ret = -EINVAL;
574	break;
575	}
576
577	if ((file->f_flags & O_NONBLOCK) && !ffs->ev.count) {
578	ret = -EAGAIN;
579	break;
580	}
581
582	if (wait_event_interruptible_exclusive_locked_irq(ffs->ev.waitq,
583	ffs->ev.count)) {
584	ret = -EINTR;
585	break;
586	}
587
588	/* unlocks spinlock */
589	return __ffs_ep0_read_events(ffs, buf,
590	min(n, (size_t)ffs->ev.count));
591
592	case FFS_SETUP_PENDING:
593	if (ffs->ev.setup.bRequestType & USB_DIR_IN) {
594	spin_unlock_irq(lock: &ffs->ev.waitq.lock);
595	ret = __ffs_ep0_stall(ffs);
596	goto done_mutex;
597	}
598
599	len = min(len, (size_t)le16_to_cpu(ffs->ev.setup.wLength));
600
601	spin_unlock_irq(lock: &ffs->ev.waitq.lock);
602
603	if (len) {
604	data = kmalloc(size: len, GFP_KERNEL);
605	if (!data) {
606	ret = -ENOMEM;
607	goto done_mutex;
608	}
609	}
610
611	spin_lock_irq(lock: &ffs->ev.waitq.lock);
612
613	/* See ffs_ep0_write() */
614	if (ffs_setup_state_clear_cancelled(ffs) ==
615	FFS_SETUP_CANCELLED) {
616	ret = -EIDRM;
617	break;
618	}
619
620	/* unlocks spinlock */
621	ret = __ffs_ep0_queue_wait(ffs, data, len);
622	if ((ret > 0) && (copy_to_user(to: buf, from: data, n: len)))
623	ret = -EFAULT;
624	goto done_mutex;
625
626	default:
627	ret = -EBADFD;
628	break;
629	}
630
631	spin_unlock_irq(lock: &ffs->ev.waitq.lock);
632	done_mutex:
633	mutex_unlock(lock: &ffs->mutex);
634	kfree(objp: data);
635	return ret;
636	}
637
638	static int ffs_ep0_open(struct inode *inode, struct file *file)
639	{
640	struct ffs_data *ffs = inode->i_private;
641
642	if (ffs->state == FFS_CLOSING)
643	return -EBUSY;
644
645	file->private_data = ffs;
646	ffs_data_opened(ffs);
647
648	return stream_open(inode, filp: file);
649	}
650
651	static int ffs_ep0_release(struct inode *inode, struct file *file)
652	{
653	struct ffs_data *ffs = file->private_data;
654
655	ffs_data_closed(ffs);
656
657	return 0;
658	}
659
660	static long ffs_ep0_ioctl(struct file *file, unsigned code, unsigned long value)
661	{
662	struct ffs_data *ffs = file->private_data;
663	struct usb_gadget *gadget = ffs->gadget;
664	long ret;
665
666	if (code == FUNCTIONFS_INTERFACE_REVMAP) {
667	struct ffs_function *func = ffs->func;
668	ret = func ? ffs_func_revmap_intf(func, intf: value) : -ENODEV;
669	} else if (gadget && gadget->ops->ioctl) {
670	ret = gadget->ops->ioctl(gadget, code, value);
671	} else {
672	ret = -ENOTTY;
673	}
674
675	return ret;
676	}
677
678	static __poll_t ffs_ep0_poll(struct file *file, poll_table *wait)
679	{
680	struct ffs_data *ffs = file->private_data;
681	__poll_t mask = EPOLLWRNORM;
682	int ret;
683
684	poll_wait(filp: file, wait_address: &ffs->ev.waitq, p: wait);
685
686	ret = ffs_mutex_lock(mutex: &ffs->mutex, nonblock: file->f_flags & O_NONBLOCK);
687	if (ret < 0)
688	return mask;
689
690	switch (ffs->state) {
691	case FFS_READ_DESCRIPTORS:
692	case FFS_READ_STRINGS:
693	mask |= EPOLLOUT;
694	break;
695
696	case FFS_ACTIVE:
697	switch (ffs->setup_state) {
698	case FFS_NO_SETUP:
699	if (ffs->ev.count)
700	mask |= EPOLLIN;
701	break;
702
703	case FFS_SETUP_PENDING:
704	case FFS_SETUP_CANCELLED:
705	mask |= (EPOLLIN | EPOLLOUT);
706	break;
707	}
708	break;
709
710	case FFS_CLOSING:
711	break;
712	case FFS_DEACTIVATED:
713	break;
714	}
715
716	mutex_unlock(lock: &ffs->mutex);
717
718	return mask;
719	}
720
721	static const struct file_operations ffs_ep0_operations = {
722	.llseek = no_llseek,
723
724	.open = ffs_ep0_open,
725	.write = ffs_ep0_write,
726	.read = ffs_ep0_read,
727	.release = ffs_ep0_release,
728	.unlocked_ioctl = ffs_ep0_ioctl,
729	.poll = ffs_ep0_poll,
730	};
731
732
733	/* "Normal" endpoints operations ********************************************/
734
735	static void ffs_epfile_io_complete(struct usb_ep *_ep, struct usb_request *req)
736	{
737	struct ffs_io_data *io_data = req->context;
738
739	if (req->status)
740	io_data->status = req->status;
741	else
742	io_data->status = req->actual;
743
744	complete(&io_data->done);
745	}
746
747	static ssize_t ffs_copy_to_iter(void *data, int data_len, struct iov_iter *iter)
748	{
749	ssize_t ret = copy_to_iter(addr: data, bytes: data_len, i: iter);
750	if (ret == data_len)
751	return ret;
752
753	if (iov_iter_count(i: iter))
754	return -EFAULT;
755
756	/*
757	* Dear user space developer!
758	*
759	* TL;DR: To stop getting below error message in your kernel log, change
760	* user space code using functionfs to align read buffers to a max
761	* packet size.
762	*
763	* Some UDCs (e.g. dwc3) require request sizes to be a multiple of a max
764	* packet size. When unaligned buffer is passed to functionfs, it
765	* internally uses a larger, aligned buffer so that such UDCs are happy.
766	*
767	* Unfortunately, this means that host may send more data than was
768	* requested in read(2) system call. f_fs doesn’t know what to do with
769	* that excess data so it simply drops it.
770	*
771	* Was the buffer aligned in the first place, no such problem would
772	* happen.
773	*
774	* Data may be dropped only in AIO reads. Synchronous reads are handled
775	* by splitting a request into multiple parts. This splitting may still
776	* be a problem though so it’s likely best to align the buffer
777	* regardless of it being AIO or not..
778	*
779	* This only affects OUT endpoints, i.e. reading data with a read(2),
780	* aio_read(2) etc. system calls. Writing data to an IN endpoint is not
781	* affected.
782	*/
783	pr_err("functionfs read size %d > requested size %zd, dropping excess data. "
784	"Align read buffer size to max packet size to avoid the problem.\n",
785	data_len, ret);
786
787	return ret;
788	}
789
790	/*
791	* allocate a virtually contiguous buffer and create a scatterlist describing it
792	* @sg_table - pointer to a place to be filled with sg_table contents
793	* @size - required buffer size
794	*/
795	static void *ffs_build_sg_list(struct sg_table *sgt, size_t sz)
796	{
797	struct page **pages;
798	void *vaddr, *ptr;
799	unsigned int n_pages;
800	int i;
801
802	vaddr = vmalloc(size: sz);
803	if (!vaddr)
804	return NULL;
805
806	n_pages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
807	pages = kvmalloc_array(n: n_pages, size: sizeof(struct page *), GFP_KERNEL);
808	if (!pages) {
809	vfree(addr: vaddr);
810
811	return NULL;
812	}
813	for (i = 0, ptr = vaddr; i < n_pages; ++i, ptr += PAGE_SIZE)
814	pages[i] = vmalloc_to_page(addr: ptr);
815
816	if (sg_alloc_table_from_pages(sgt, pages, n_pages, offset: 0, size: sz, GFP_KERNEL)) {
817	kvfree(addr: pages);
818	vfree(addr: vaddr);
819
820	return NULL;
821	}
822	kvfree(addr: pages);
823
824	return vaddr;
825	}
826
827	static inline void *ffs_alloc_buffer(struct ffs_io_data *io_data,
828	size_t data_len)
829	{
830	if (io_data->use_sg)
831	return ffs_build_sg_list(sgt: &io_data->sgt, sz: data_len);
832
833	return kmalloc(size: data_len, GFP_KERNEL);
834	}
835
836	static inline void ffs_free_buffer(struct ffs_io_data *io_data)
837	{
838	if (!io_data->buf)
839	return;
840
841	if (io_data->use_sg) {
842	sg_free_table(&io_data->sgt);
843	vfree(addr: io_data->buf);
844	} else {
845	kfree(objp: io_data->buf);
846	}
847	}
848
849	static void ffs_user_copy_worker(struct work_struct *work)
850	{
851	struct ffs_io_data *io_data = container_of(work, struct ffs_io_data,
852	work);
853	int ret = io_data->status;
854	bool kiocb_has_eventfd = io_data->kiocb->ki_flags & IOCB_EVENTFD;
855
856	if (io_data->read && ret > 0) {
857	kthread_use_mm(mm: io_data->mm);
858	ret = ffs_copy_to_iter(data: io_data->buf, data_len: ret, iter: &io_data->data);
859	kthread_unuse_mm(mm: io_data->mm);
860	}
861
862	io_data->kiocb->ki_complete(io_data->kiocb, ret);
863
864	if (io_data->ffs->ffs_eventfd && !kiocb_has_eventfd)
865	eventfd_signal(ctx: io_data->ffs->ffs_eventfd);
866
867	if (io_data->read)
868	kfree(objp: io_data->to_free);
869	ffs_free_buffer(io_data);
870	kfree(objp: io_data);
871	}
872
873	static void ffs_epfile_async_io_complete(struct usb_ep *_ep,
874	struct usb_request *req)
875	{
876	struct ffs_io_data *io_data = req->context;
877	struct ffs_data *ffs = io_data->ffs;
878
879	io_data->status = req->status ? req->status : req->actual;
880	usb_ep_free_request(ep: _ep, req);
881
882	INIT_WORK(&io_data->work, ffs_user_copy_worker);
883	queue_work(wq: ffs->io_completion_wq, work: &io_data->work);
884	}
885
886	static void __ffs_epfile_read_buffer_free(struct ffs_epfile *epfile)
887	{
888	/*
889	* See comment in struct ffs_epfile for full read_buffer pointer
890	* synchronisation story.
891	*/
892	struct ffs_buffer *buf = xchg(&epfile->read_buffer, READ_BUFFER_DROP);
893	if (buf && buf != READ_BUFFER_DROP)
894	kfree(objp: buf);
895	}
896
897	/* Assumes epfile->mutex is held. */
898	static ssize_t __ffs_epfile_read_buffered(struct ffs_epfile *epfile,
899	struct iov_iter *iter)
900	{
901	/*
902	* Null out epfile->read_buffer so ffs_func_eps_disable does not free
903	* the buffer while we are using it. See comment in struct ffs_epfile
904	* for full read_buffer pointer synchronisation story.
905	*/
906	struct ffs_buffer *buf = xchg(&epfile->read_buffer, NULL);
907	ssize_t ret;
908	if (!buf || buf == READ_BUFFER_DROP)
909	return 0;
910
911	ret = copy_to_iter(addr: buf->data, bytes: buf->length, i: iter);
912	if (buf->length == ret) {
913	kfree(objp: buf);
914	return ret;
915	}
916
917	if (iov_iter_count(i: iter)) {
918	ret = -EFAULT;
919	} else {
920	buf->length -= ret;
921	buf->data += ret;
922	}
923
924	if (cmpxchg(&epfile->read_buffer, NULL, buf))
925	kfree(objp: buf);
926
927	return ret;
928	}
929
930	/* Assumes epfile->mutex is held. */
931	static ssize_t __ffs_epfile_read_data(struct ffs_epfile *epfile,
932	void *data, int data_len,
933	struct iov_iter *iter)
934	{
935	struct ffs_buffer *buf;
936
937	ssize_t ret = copy_to_iter(addr: data, bytes: data_len, i: iter);
938	if (data_len == ret)
939	return ret;
940
941	if (iov_iter_count(i: iter))
942	return -EFAULT;
943
944	/* See ffs_copy_to_iter for more context. */
945	pr_warn("functionfs read size %d > requested size %zd, splitting request into multiple reads.",
946	data_len, ret);
947
948	data_len -= ret;
949	buf = kmalloc(struct_size(buf, storage, data_len), GFP_KERNEL);
950	if (!buf)
951	return -ENOMEM;
952	buf->length = data_len;
953	buf->data = buf->storage;
954	memcpy(buf->storage, data + ret, flex_array_size(buf, storage, data_len));
955
956	/*
957	* At this point read_buffer is NULL or READ_BUFFER_DROP (if
958	* ffs_func_eps_disable has been called in the meanwhile). See comment
959	* in struct ffs_epfile for full read_buffer pointer synchronisation
960	* story.
961	*/
962	if (cmpxchg(&epfile->read_buffer, NULL, buf))
963	kfree(objp: buf);
964
965	return ret;
966	}
967
968	static struct ffs_ep *ffs_epfile_wait_ep(struct file *file)
969	{
970	struct ffs_epfile *epfile = file->private_data;
971	struct ffs_ep *ep;
972	int ret;
973
974	/* Wait for endpoint to be enabled */
975	ep = epfile->ep;
976	if (!ep) {
977	if (file->f_flags & O_NONBLOCK)
978	return ERR_PTR(error: -EAGAIN);
979
980	ret = wait_event_interruptible(
981	epfile->ffs->wait, (ep = epfile->ep));
982	if (ret)
983	return ERR_PTR(error: -EINTR);
984	}
985
986	return ep;
987	}
988
989	static ssize_t ffs_epfile_io(struct file *file, struct ffs_io_data *io_data)
990	{
991	struct ffs_epfile *epfile = file->private_data;
992	struct usb_request *req;
993	struct ffs_ep *ep;
994	char *data = NULL;
995	ssize_t ret, data_len = -EINVAL;
996	int halt;
997
998	/* Are we still active? */
999	if (WARN_ON(epfile->ffs->state != FFS_ACTIVE))
1000	return -ENODEV;
1001
1002	ep = ffs_epfile_wait_ep(file);
1003	if (IS_ERR(ptr: ep))
1004	return PTR_ERR(ptr: ep);
1005
1006	/* Do we halt? */
1007	halt = (!io_data->read == !epfile->in);
1008	if (halt && epfile->isoc)
1009	return -EINVAL;
1010
1011	/* We will be using request and read_buffer */
1012	ret = ffs_mutex_lock(mutex: &epfile->mutex, nonblock: file->f_flags & O_NONBLOCK);
1013	if (ret)
1014	goto error;
1015
1016	/* Allocate & copy */
1017	if (!halt) {
1018	struct usb_gadget *gadget;
1019
1020	/*
1021	* Do we have buffered data from previous partial read? Check
1022	* that for synchronous case only because we do not have
1023	* facility to ‘wake up’ a pending asynchronous read and push
1024	* buffered data to it which we would need to make things behave
1025	* consistently.
1026	*/
1027	if (!io_data->aio && io_data->read) {
1028	ret = __ffs_epfile_read_buffered(epfile, iter: &io_data->data);
1029	if (ret)
1030	goto error_mutex;
1031	}
1032
1033	/*
1034	* if we _do_ wait above, the epfile->ffs->gadget might be NULL
1035	* before the waiting completes, so do not assign to 'gadget'
1036	* earlier
1037	*/
1038	gadget = epfile->ffs->gadget;
1039
1040	spin_lock_irq(lock: &epfile->ffs->eps_lock);
1041	/* In the meantime, endpoint got disabled or changed. */
1042	if (epfile->ep != ep) {
1043	ret = -ESHUTDOWN;
1044	goto error_lock;
1045	}
1046	data_len = iov_iter_count(i: &io_data->data);
1047	/*
1048	* Controller may require buffer size to be aligned to
1049	* maxpacketsize of an out endpoint.
1050	*/
1051	if (io_data->read)
1052	data_len = usb_ep_align_maybe(g: gadget, ep: ep->ep, len: data_len);
1053
1054	io_data->use_sg = gadget->sg_supported && data_len > PAGE_SIZE;
1055	spin_unlock_irq(lock: &epfile->ffs->eps_lock);
1056
1057	data = ffs_alloc_buffer(io_data, data_len);
1058	if (!data) {
1059	ret = -ENOMEM;
1060	goto error_mutex;
1061	}
1062	if (!io_data->read &&
1063	!copy_from_iter_full(addr: data, bytes: data_len, i: &io_data->data)) {
1064	ret = -EFAULT;
1065	goto error_mutex;
1066	}
1067	}
1068
1069	spin_lock_irq(lock: &epfile->ffs->eps_lock);
1070
1071	if (epfile->ep != ep) {
1072	/* In the meantime, endpoint got disabled or changed. */
1073	ret = -ESHUTDOWN;
1074	} else if (halt) {
1075	ret = usb_ep_set_halt(ep: ep->ep);
1076	if (!ret)
1077	ret = -EBADMSG;
1078	} else if (data_len == -EINVAL) {
1079	/*
1080	* Sanity Check: even though data_len can't be used
1081	* uninitialized at the time I write this comment, some
1082	* compilers complain about this situation.
1083	* In order to keep the code clean from warnings, data_len is
1084	* being initialized to -EINVAL during its declaration, which
1085	* means we can't rely on compiler anymore to warn no future
1086	* changes won't result in data_len being used uninitialized.
1087	* For such reason, we're adding this redundant sanity check
1088	* here.
1089	*/
1090	WARN(1, "%s: data_len == -EINVAL\n", __func__);
1091	ret = -EINVAL;
1092	} else if (!io_data->aio) {
1093	bool interrupted = false;
1094
1095	req = ep->req;
1096	if (io_data->use_sg) {
1097	req->buf = NULL;
1098	req->sg = io_data->sgt.sgl;
1099	req->num_sgs = io_data->sgt.nents;
1100	} else {
1101	req->buf = data;
1102	req->num_sgs = 0;
1103	}
1104	req->length = data_len;
1105
1106	io_data->buf = data;
1107
1108	init_completion(x: &io_data->done);
1109	req->context = io_data;
1110	req->complete = ffs_epfile_io_complete;
1111
1112	ret = usb_ep_queue(ep: ep->ep, req, GFP_ATOMIC);
1113	if (ret < 0)
1114	goto error_lock;
1115
1116	spin_unlock_irq(lock: &epfile->ffs->eps_lock);
1117
1118	if (wait_for_completion_interruptible(x: &io_data->done)) {
1119	spin_lock_irq(lock: &epfile->ffs->eps_lock);
1120	if (epfile->ep != ep) {
1121	ret = -ESHUTDOWN;
1122	goto error_lock;
1123	}
1124	/*
1125	* To avoid race condition with ffs_epfile_io_complete,
1126	* dequeue the request first then check
1127	* status. usb_ep_dequeue API should guarantee no race
1128	* condition with req->complete callback.
1129	*/
1130	usb_ep_dequeue(ep: ep->ep, req);
1131	spin_unlock_irq(lock: &epfile->ffs->eps_lock);
1132	wait_for_completion(&io_data->done);
1133	interrupted = io_data->status < 0;
1134	}
1135
1136	if (interrupted)
1137	ret = -EINTR;
1138	else if (io_data->read && io_data->status > 0)
1139	ret = __ffs_epfile_read_data(epfile, data, data_len: io_data->status,
1140	iter: &io_data->data);
1141	else
1142	ret = io_data->status;
1143	goto error_mutex;
1144	} else if (!(req = usb_ep_alloc_request(ep: ep->ep, GFP_ATOMIC))) {
1145	ret = -ENOMEM;
1146	} else {
1147	if (io_data->use_sg) {
1148	req->buf = NULL;
1149	req->sg = io_data->sgt.sgl;
1150	req->num_sgs = io_data->sgt.nents;
1151	} else {
1152	req->buf = data;
1153	req->num_sgs = 0;
1154	}
1155	req->length = data_len;
1156
1157	io_data->buf = data;
1158	io_data->ep = ep->ep;
1159	io_data->req = req;
1160	io_data->ffs = epfile->ffs;
1161
1162	req->context = io_data;
1163	req->complete = ffs_epfile_async_io_complete;
1164
1165	ret = usb_ep_queue(ep: ep->ep, req, GFP_ATOMIC);
1166	if (ret) {
1167	io_data->req = NULL;
1168	usb_ep_free_request(ep: ep->ep, req);
1169	goto error_lock;
1170	}
1171
1172	ret = -EIOCBQUEUED;
1173	/*
1174	* Do not kfree the buffer in this function. It will be freed
1175	* by ffs_user_copy_worker.
1176	*/
1177	data = NULL;
1178	}
1179
1180	error_lock:
1181	spin_unlock_irq(lock: &epfile->ffs->eps_lock);
1182	error_mutex:
1183	mutex_unlock(lock: &epfile->mutex);
1184	error:
1185	if (ret != -EIOCBQUEUED) /* don't free if there is iocb queued */
1186	ffs_free_buffer(io_data);
1187	return ret;
1188	}
1189
1190	static int
1191	ffs_epfile_open(struct inode *inode, struct file *file)
1192	{
1193	struct ffs_epfile *epfile = inode->i_private;
1194
1195	if (WARN_ON(epfile->ffs->state != FFS_ACTIVE))
1196	return -ENODEV;
1197
1198	file->private_data = epfile;
1199	ffs_data_opened(ffs: epfile->ffs);
1200
1201	return stream_open(inode, filp: file);
1202	}
1203
1204	static int ffs_aio_cancel(struct kiocb *kiocb)
1205	{
1206	struct ffs_io_data *io_data = kiocb->private;
1207	struct ffs_epfile *epfile = kiocb->ki_filp->private_data;
1208	unsigned long flags;
1209	int value;
1210
1211	spin_lock_irqsave(&epfile->ffs->eps_lock, flags);
1212
1213	if (io_data && io_data->ep && io_data->req)
1214	value = usb_ep_dequeue(ep: io_data->ep, req: io_data->req);
1215	else
1216	value = -EINVAL;
1217
1218	spin_unlock_irqrestore(lock: &epfile->ffs->eps_lock, flags);
1219
1220	return value;
1221	}
1222
1223	static ssize_t ffs_epfile_write_iter(struct kiocb *kiocb, struct iov_iter *from)
1224	{
1225	struct ffs_io_data io_data, *p = &io_data;
1226	ssize_t res;
1227
1228	if (!is_sync_kiocb(kiocb)) {
1229	p = kzalloc(size: sizeof(io_data), GFP_KERNEL);
1230	if (!p)
1231	return -ENOMEM;
1232	p->aio = true;
1233	} else {
1234	memset(p, 0, sizeof(*p));
1235	p->aio = false;
1236	}
1237
1238	p->read = false;
1239	p->kiocb = kiocb;
1240	p->data = *from;
1241	p->mm = current->mm;
1242
1243	kiocb->private = p;
1244
1245	if (p->aio)
1246	kiocb_set_cancel_fn(req: kiocb, cancel: ffs_aio_cancel);
1247
1248	res = ffs_epfile_io(file: kiocb->ki_filp, io_data: p);
1249	if (res == -EIOCBQUEUED)
1250	return res;
1251	if (p->aio)
1252	kfree(objp: p);
1253	else
1254	*from = p->data;
1255	return res;
1256	}
1257
1258	static ssize_t ffs_epfile_read_iter(struct kiocb *kiocb, struct iov_iter *to)
1259	{
1260	struct ffs_io_data io_data, *p = &io_data;
1261	ssize_t res;
1262
1263	if (!is_sync_kiocb(kiocb)) {
1264	p = kzalloc(size: sizeof(io_data), GFP_KERNEL);
1265	if (!p)
1266	return -ENOMEM;
1267	p->aio = true;
1268	} else {
1269	memset(p, 0, sizeof(*p));
1270	p->aio = false;
1271	}
1272
1273	p->read = true;
1274	p->kiocb = kiocb;
1275	if (p->aio) {
1276	p->to_free = dup_iter(new: &p->data, old: to, GFP_KERNEL);
1277	if (!iter_is_ubuf(i: &p->data) && !p->to_free) {
1278	kfree(objp: p);
1279	return -ENOMEM;
1280	}
1281	} else {
1282	p->data = *to;
1283	p->to_free = NULL;
1284	}
1285	p->mm = current->mm;
1286
1287	kiocb->private = p;
1288
1289	if (p->aio)
1290	kiocb_set_cancel_fn(req: kiocb, cancel: ffs_aio_cancel);
1291
1292	res = ffs_epfile_io(file: kiocb->ki_filp, io_data: p);
1293	if (res == -EIOCBQUEUED)
1294	return res;
1295
1296	if (p->aio) {
1297	kfree(objp: p->to_free);
1298	kfree(objp: p);
1299	} else {
1300	*to = p->data;
1301	}
1302	return res;
1303	}
1304
1305	static void ffs_dmabuf_release(struct kref *ref)
1306	{
1307	struct ffs_dmabuf_priv *priv = container_of(ref, struct ffs_dmabuf_priv, ref);
1308	struct dma_buf_attachment *attach = priv->attach;
1309	struct dma_buf *dmabuf = attach->dmabuf;
1310
1311	pr_vdebug("FFS DMABUF release\n");
1312	dma_resv_lock(obj: dmabuf->resv, NULL);
1313	dma_buf_unmap_attachment(attach, priv->sgt, priv->dir);
1314	dma_resv_unlock(obj: dmabuf->resv);
1315
1316	dma_buf_detach(dmabuf: attach->dmabuf, attach);
1317	dma_buf_put(dmabuf);
1318	kfree(objp: priv);
1319	}
1320
1321	static void ffs_dmabuf_get(struct dma_buf_attachment *attach)
1322	{
1323	struct ffs_dmabuf_priv *priv = attach->importer_priv;
1324
1325	kref_get(kref: &priv->ref);
1326	}
1327
1328	static void ffs_dmabuf_put(struct dma_buf_attachment *attach)
1329	{
1330	struct ffs_dmabuf_priv *priv = attach->importer_priv;
1331
1332	kref_put(kref: &priv->ref, release: ffs_dmabuf_release);
1333	}
1334
1335	static int
1336	ffs_epfile_release(struct inode *inode, struct file *file)
1337	{
1338	struct ffs_epfile *epfile = inode->i_private;
1339	struct ffs_dmabuf_priv *priv, *tmp;
1340	struct ffs_data *ffs = epfile->ffs;
1341
1342	mutex_lock(&epfile->dmabufs_mutex);
1343
1344	/* Close all attached DMABUFs */
1345	list_for_each_entry_safe(priv, tmp, &epfile->dmabufs, entry) {
1346	/* Cancel any pending transfer */
1347	spin_lock_irq(lock: &ffs->eps_lock);
1348	if (priv->ep && priv->req)
1349	usb_ep_dequeue(ep: priv->ep, req: priv->req);
1350	spin_unlock_irq(lock: &ffs->eps_lock);
1351
1352	list_del(entry: &priv->entry);
1353	ffs_dmabuf_put(attach: priv->attach);
1354	}
1355
1356	mutex_unlock(lock: &epfile->dmabufs_mutex);
1357
1358	__ffs_epfile_read_buffer_free(epfile);
1359	ffs_data_closed(ffs: epfile->ffs);
1360
1361	return 0;
1362	}
1363
1364	static void ffs_dmabuf_cleanup(struct work_struct *work)
1365	{
1366	struct ffs_dma_fence *dma_fence =
1367	container_of(work, struct ffs_dma_fence, work);
1368	struct ffs_dmabuf_priv *priv = dma_fence->priv;
1369	struct dma_buf_attachment *attach = priv->attach;
1370	struct dma_fence *fence = &dma_fence->base;
1371
1372	ffs_dmabuf_put(attach);
1373	dma_fence_put(fence);
1374	}
1375
1376	static void ffs_dmabuf_signal_done(struct ffs_dma_fence *dma_fence, int ret)
1377	{
1378	struct ffs_dmabuf_priv *priv = dma_fence->priv;
1379	struct dma_fence *fence = &dma_fence->base;
1380	bool cookie = dma_fence_begin_signalling();
1381
1382	dma_fence_get(fence);
1383	fence->error = ret;
1384	dma_fence_signal(fence);
1385	dma_fence_end_signalling(cookie);
1386
1387	/*
1388	* The fence will be unref'd in ffs_dmabuf_cleanup.
1389	* It can't be done here, as the unref functions might try to lock
1390	* the resv object, which would deadlock.
1391	*/
1392	INIT_WORK(&dma_fence->work, ffs_dmabuf_cleanup);
1393	queue_work(wq: priv->ffs->io_completion_wq, work: &dma_fence->work);
1394	}
1395
1396	static void ffs_epfile_dmabuf_io_complete(struct usb_ep *ep,
1397	struct usb_request *req)
1398	{
1399	pr_vdebug("FFS: DMABUF transfer complete, status=%d\n", req->status);
1400	ffs_dmabuf_signal_done(dma_fence: req->context, ret: req->status);
1401	usb_ep_free_request(ep, req);
1402	}
1403
1404	static const char *ffs_dmabuf_get_driver_name(struct dma_fence *fence)
1405	{
1406	return "functionfs";
1407	}
1408
1409	static const char *ffs_dmabuf_get_timeline_name(struct dma_fence *fence)
1410	{
1411	return "";
1412	}
1413
1414	static void ffs_dmabuf_fence_release(struct dma_fence *fence)
1415	{
1416	struct ffs_dma_fence *dma_fence =
1417	container_of(fence, struct ffs_dma_fence, base);
1418
1419	kfree(objp: dma_fence);
1420	}
1421
1422	static const struct dma_fence_ops ffs_dmabuf_fence_ops = {
1423	.get_driver_name = ffs_dmabuf_get_driver_name,
1424	.get_timeline_name = ffs_dmabuf_get_timeline_name,
1425	.release = ffs_dmabuf_fence_release,
1426	};
1427
1428	static int ffs_dma_resv_lock(struct dma_buf *dmabuf, bool nonblock)
1429	{
1430	if (!nonblock)
1431	return dma_resv_lock_interruptible(obj: dmabuf->resv, NULL);
1432
1433	if (!dma_resv_trylock(obj: dmabuf->resv))
1434	return -EBUSY;
1435
1436	return 0;
1437	}
1438
1439	static struct dma_buf_attachment *
1440	ffs_dmabuf_find_attachment(struct ffs_epfile *epfile, struct dma_buf *dmabuf)
1441	{
1442	struct device *dev = epfile->ffs->gadget->dev.parent;
1443	struct dma_buf_attachment *attach = NULL;
1444	struct ffs_dmabuf_priv *priv;
1445
1446	mutex_lock(&epfile->dmabufs_mutex);
1447
1448	list_for_each_entry(priv, &epfile->dmabufs, entry) {
1449	if (priv->attach->dev == dev
1450	&& priv->attach->dmabuf == dmabuf) {
1451	attach = priv->attach;
1452	break;
1453	}
1454	}
1455
1456	if (attach)
1457	ffs_dmabuf_get(attach);
1458
1459	mutex_unlock(lock: &epfile->dmabufs_mutex);
1460
1461	return attach ?: ERR_PTR(error: -EPERM);
1462	}
1463
1464	static int ffs_dmabuf_attach(struct file *file, int fd)
1465	{
1466	bool nonblock = file->f_flags & O_NONBLOCK;
1467	struct ffs_epfile *epfile = file->private_data;
1468	struct usb_gadget *gadget = epfile->ffs->gadget;
1469	struct dma_buf_attachment *attach;
1470	struct ffs_dmabuf_priv *priv;
1471	enum dma_data_direction dir;
1472	struct sg_table *sg_table;
1473	struct dma_buf *dmabuf;
1474	int err;
1475
1476	if (!gadget || !gadget->sg_supported)
1477	return -EPERM;
1478
1479	dmabuf = dma_buf_get(fd);
1480	if (IS_ERR(ptr: dmabuf))
1481	return PTR_ERR(ptr: dmabuf);
1482
1483	attach = dma_buf_attach(dmabuf, dev: gadget->dev.parent);
1484	if (IS_ERR(ptr: attach)) {
1485	err = PTR_ERR(ptr: attach);
1486	goto err_dmabuf_put;
1487	}
1488
1489	priv = kzalloc(size: sizeof(*priv), GFP_KERNEL);
1490	if (!priv) {
1491	err = -ENOMEM;
1492	goto err_dmabuf_detach;
1493	}
1494
1495	dir = epfile->in ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1496
1497	err = ffs_dma_resv_lock(dmabuf, nonblock);
1498	if (err)
1499	goto err_free_priv;
1500
1501	sg_table = dma_buf_map_attachment(attach, dir);
1502	dma_resv_unlock(obj: dmabuf->resv);
1503
1504	if (IS_ERR(ptr: sg_table)) {
1505	err = PTR_ERR(ptr: sg_table);
1506	goto err_free_priv;
1507	}
1508
1509	attach->importer_priv = priv;
1510
1511	priv->sgt = sg_table;
1512	priv->dir = dir;
1513	priv->ffs = epfile->ffs;
1514	priv->attach = attach;
1515	spin_lock_init(&priv->lock);
1516	kref_init(kref: &priv->ref);
1517	priv->context = dma_fence_context_alloc(num: 1);
1518
1519	mutex_lock(&epfile->dmabufs_mutex);
1520	list_add(new: &priv->entry, head: &epfile->dmabufs);
1521	mutex_unlock(lock: &epfile->dmabufs_mutex);
1522
1523	return 0;
1524
1525	err_free_priv:
1526	kfree(objp: priv);
1527	err_dmabuf_detach:
1528	dma_buf_detach(dmabuf, attach);
1529	err_dmabuf_put:
1530	dma_buf_put(dmabuf);
1531
1532	return err;
1533	}
1534
1535	static int ffs_dmabuf_detach(struct file *file, int fd)
1536	{
1537	struct ffs_epfile *epfile = file->private_data;
1538	struct ffs_data *ffs = epfile->ffs;
1539	struct device *dev = ffs->gadget->dev.parent;
1540	struct ffs_dmabuf_priv *priv, *tmp;
1541	struct dma_buf *dmabuf;
1542	int ret = -EPERM;
1543
1544	dmabuf = dma_buf_get(fd);
1545	if (IS_ERR(ptr: dmabuf))
1546	return PTR_ERR(ptr: dmabuf);
1547
1548	mutex_lock(&epfile->dmabufs_mutex);
1549
1550	list_for_each_entry_safe(priv, tmp, &epfile->dmabufs, entry) {
1551	if (priv->attach->dev == dev
1552	&& priv->attach->dmabuf == dmabuf) {
1553	/* Cancel any pending transfer */
1554	spin_lock_irq(lock: &ffs->eps_lock);
1555	if (priv->ep && priv->req)
1556	usb_ep_dequeue(ep: priv->ep, req: priv->req);
1557	spin_unlock_irq(lock: &ffs->eps_lock);
1558
1559	list_del(entry: &priv->entry);
1560
1561	/* Unref the reference from ffs_dmabuf_attach() */
1562	ffs_dmabuf_put(attach: priv->attach);
1563	ret = 0;
1564	break;
1565	}
1566	}
1567
1568	mutex_unlock(lock: &epfile->dmabufs_mutex);
1569	dma_buf_put(dmabuf);
1570
1571	return ret;
1572	}
1573
1574	static int ffs_dmabuf_transfer(struct file *file,
1575	const struct usb_ffs_dmabuf_transfer_req *req)
1576	{
1577	bool nonblock = file->f_flags & O_NONBLOCK;
1578	struct ffs_epfile *epfile = file->private_data;
1579	struct dma_buf_attachment *attach;
1580	struct ffs_dmabuf_priv *priv;
1581	struct ffs_dma_fence *fence;
1582	struct usb_request *usb_req;
1583	enum dma_resv_usage resv_dir;
1584	struct dma_buf *dmabuf;
1585	unsigned long timeout;
1586	struct ffs_ep *ep;
1587	bool cookie;
1588	u32 seqno;
1589	long retl;
1590	int ret;
1591
1592	if (req->flags & ~USB_FFS_DMABUF_TRANSFER_MASK)
1593	return -EINVAL;
1594
1595	dmabuf = dma_buf_get(fd: req->fd);
1596	if (IS_ERR(ptr: dmabuf))
1597	return PTR_ERR(ptr: dmabuf);
1598
1599	if (req->length > dmabuf->size || req->length == 0) {
1600	ret = -EINVAL;
1601	goto err_dmabuf_put;
1602	}
1603
1604	attach = ffs_dmabuf_find_attachment(epfile, dmabuf);
1605	if (IS_ERR(ptr: attach)) {
1606	ret = PTR_ERR(ptr: attach);
1607	goto err_dmabuf_put;
1608	}
1609
1610	priv = attach->importer_priv;
1611
1612	ep = ffs_epfile_wait_ep(file);
1613	if (IS_ERR(ptr: ep)) {
1614	ret = PTR_ERR(ptr: ep);
1615	goto err_attachment_put;
1616	}
1617
1618	ret = ffs_dma_resv_lock(dmabuf, nonblock);
1619	if (ret)
1620	goto err_attachment_put;
1621
1622	/* Make sure we don't have writers */
1623	timeout = nonblock ? 0 : msecs_to_jiffies(DMABUF_ENQUEUE_TIMEOUT_MS);
1624	retl = dma_resv_wait_timeout(obj: dmabuf->resv,
1625	usage: dma_resv_usage_rw(write: epfile->in),
1626	intr: true, timeout);
1627	if (retl == 0)
1628	retl = -EBUSY;
1629	if (retl < 0) {
1630	ret = (int)retl;
1631	goto err_resv_unlock;
1632	}
1633
1634	ret = dma_resv_reserve_fences(obj: dmabuf->resv, num_fences: 1);
1635	if (ret)
1636	goto err_resv_unlock;
1637
1638	fence = kmalloc(size: sizeof(*fence), GFP_KERNEL);
1639	if (!fence) {
1640	ret = -ENOMEM;
1641	goto err_resv_unlock;
1642	}
1643
1644	fence->priv = priv;
1645
1646	spin_lock_irq(lock: &epfile->ffs->eps_lock);
1647
1648	/* In the meantime, endpoint got disabled or changed. */
1649	if (epfile->ep != ep) {
1650	ret = -ESHUTDOWN;
1651	goto err_fence_put;
1652	}
1653
1654	usb_req = usb_ep_alloc_request(ep: ep->ep, GFP_ATOMIC);
1655	if (!usb_req) {
1656	ret = -ENOMEM;
1657	goto err_fence_put;
1658	}
1659
1660	/*
1661	* usb_ep_queue() guarantees that all transfers are processed in the
1662	* order they are enqueued, so we can use a simple incrementing
1663	* sequence number for the dma_fence.
1664	*/
1665	seqno = atomic_add_return(i: 1, v: &epfile->seqno);
1666
1667	dma_fence_init(fence: &fence->base, ops: &ffs_dmabuf_fence_ops,
1668	lock: &priv->lock, context: priv->context, seqno);
1669
1670	resv_dir = epfile->in ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ;
1671
1672	dma_resv_add_fence(obj: dmabuf->resv, fence: &fence->base, usage: resv_dir);
1673	dma_resv_unlock(obj: dmabuf->resv);
1674
1675	/* Now that the dma_fence is in place, queue the transfer. */
1676
1677	usb_req->length = req->length;
1678	usb_req->buf = NULL;
1679	usb_req->sg = priv->sgt->sgl;
1680	usb_req->num_sgs = sg_nents_for_len(sg: priv->sgt->sgl, len: req->length);
1681	usb_req->sg_was_mapped = true;
1682	usb_req->context = fence;
1683	usb_req->complete = ffs_epfile_dmabuf_io_complete;
1684
1685	cookie = dma_fence_begin_signalling();
1686	ret = usb_ep_queue(ep: ep->ep, req: usb_req, GFP_ATOMIC);
1687	dma_fence_end_signalling(cookie);
1688	if (!ret) {
1689	priv->req = usb_req;
1690	priv->ep = ep->ep;
1691	} else {
1692	pr_warn("FFS: Failed to queue DMABUF: %d\n", ret);
1693	ffs_dmabuf_signal_done(dma_fence: fence, ret);
1694	usb_ep_free_request(ep: ep->ep, req: usb_req);
1695	}
1696
1697	spin_unlock_irq(lock: &epfile->ffs->eps_lock);
1698	dma_buf_put(dmabuf);
1699
1700	return ret;
1701
1702	err_fence_put:
1703	spin_unlock_irq(lock: &epfile->ffs->eps_lock);
1704	dma_fence_put(fence: &fence->base);
1705	err_resv_unlock:
1706	dma_resv_unlock(obj: dmabuf->resv);
1707	err_attachment_put:
1708	ffs_dmabuf_put(attach);
1709	err_dmabuf_put:
1710	dma_buf_put(dmabuf);
1711
1712	return ret;
1713	}
1714
1715	static long ffs_epfile_ioctl(struct file *file, unsigned code,
1716	unsigned long value)
1717	{
1718	struct ffs_epfile *epfile = file->private_data;
1719	struct ffs_ep *ep;
1720	int ret;
1721
1722	if (WARN_ON(epfile->ffs->state != FFS_ACTIVE))
1723	return -ENODEV;
1724
1725	switch (code) {
1726	case FUNCTIONFS_DMABUF_ATTACH:
1727	{
1728	int fd;
1729
1730	if (copy_from_user(to: &fd, from: (void __user *)value, n: sizeof(fd))) {
1731	ret = -EFAULT;
1732	break;
1733	}
1734
1735	return ffs_dmabuf_attach(file, fd);
1736	}
1737	case FUNCTIONFS_DMABUF_DETACH:
1738	{
1739	int fd;
1740
1741	if (copy_from_user(to: &fd, from: (void __user *)value, n: sizeof(fd))) {
1742	ret = -EFAULT;
1743	break;
1744	}
1745
1746	return ffs_dmabuf_detach(file, fd);
1747	}
1748	case FUNCTIONFS_DMABUF_TRANSFER:
1749	{
1750	struct usb_ffs_dmabuf_transfer_req req;
1751
1752	if (copy_from_user(to: &req, from: (void __user *)value, n: sizeof(req))) {
1753	ret = -EFAULT;
1754	break;
1755	}
1756
1757	return ffs_dmabuf_transfer(file, req: &req);
1758	}
1759	default:
1760	break;
1761	}
1762
1763	/* Wait for endpoint to be enabled */
1764	ep = ffs_epfile_wait_ep(file);
1765	if (IS_ERR(ptr: ep))
1766	return PTR_ERR(ptr: ep);
1767
1768	spin_lock_irq(lock: &epfile->ffs->eps_lock);
1769
1770	/* In the meantime, endpoint got disabled or changed. */
1771	if (epfile->ep != ep) {
1772	spin_unlock_irq(lock: &epfile->ffs->eps_lock);
1773	return -ESHUTDOWN;
1774	}
1775
1776	switch (code) {
1777	case FUNCTIONFS_FIFO_STATUS:
1778	ret = usb_ep_fifo_status(ep: epfile->ep->ep);
1779	break;
1780	case FUNCTIONFS_FIFO_FLUSH:
1781	usb_ep_fifo_flush(ep: epfile->ep->ep);
1782	ret = 0;
1783	break;
1784	case FUNCTIONFS_CLEAR_HALT:
1785	ret = usb_ep_clear_halt(ep: epfile->ep->ep);
1786	break;
1787	case FUNCTIONFS_ENDPOINT_REVMAP:
1788	ret = epfile->ep->num;
1789	break;
1790	case FUNCTIONFS_ENDPOINT_DESC:
1791	{
1792	int desc_idx;
1793	struct usb_endpoint_descriptor desc1, *desc;
1794
1795	switch (epfile->ffs->gadget->speed) {
1796	case USB_SPEED_SUPER:
1797	case USB_SPEED_SUPER_PLUS:
1798	desc_idx = 2;
1799	break;
1800	case USB_SPEED_HIGH:
1801	desc_idx = 1;
1802	break;
1803	default:
1804	desc_idx = 0;
1805	}
1806
1807	desc = epfile->ep->descs[desc_idx];
1808	memcpy(&desc1, desc, desc->bLength);
1809
1810	spin_unlock_irq(lock: &epfile->ffs->eps_lock);
1811	ret = copy_to_user(to: (void __user *)value, from: &desc1, n: desc1.bLength);
1812	if (ret)
1813	ret = -EFAULT;
1814	return ret;
1815	}
1816	default:
1817	ret = -ENOTTY;
1818	}
1819	spin_unlock_irq(lock: &epfile->ffs->eps_lock);
1820
1821	return ret;
1822	}
1823
1824	static const struct file_operations ffs_epfile_operations = {
1825	.llseek = no_llseek,
1826
1827	.open = ffs_epfile_open,
1828	.write_iter = ffs_epfile_write_iter,
1829	.read_iter = ffs_epfile_read_iter,
1830	.release = ffs_epfile_release,
1831	.unlocked_ioctl = ffs_epfile_ioctl,
1832	.compat_ioctl = compat_ptr_ioctl,
1833	};
1834
1835
1836	/* File system and super block operations ***********************************/
1837
1838	/*
1839	* Mounting the file system creates a controller file, used first for
1840	* function configuration then later for event monitoring.
1841	*/
1842
1843	static struct inode *__must_check
1844	ffs_sb_make_inode(struct super_block *sb, void *data,
1845	const struct file_operations *fops,
1846	const struct inode_operations *iops,
1847	struct ffs_file_perms *perms)
1848	{
1849	struct inode *inode;
1850
1851	inode = new_inode(sb);
1852
1853	if (inode) {
1854	struct timespec64 ts = inode_set_ctime_current(inode);
1855
1856	inode->i_ino = get_next_ino();
1857	inode->i_mode = perms->mode;
1858	inode->i_uid = perms->uid;
1859	inode->i_gid = perms->gid;
1860	inode_set_atime_to_ts(inode, ts);
1861	inode_set_mtime_to_ts(inode, ts);
1862	inode->i_private = data;
1863	if (fops)
1864	inode->i_fop = fops;
1865	if (iops)
1866	inode->i_op = iops;
1867	}
1868
1869	return inode;
1870	}
1871
1872	/* Create "regular" file */
1873	static struct dentry *ffs_sb_create_file(struct super_block *sb,
1874	const char *name, void *data,
1875	const struct file_operations *fops)
1876	{
1877	struct ffs_data *ffs = sb->s_fs_info;
1878	struct dentry *dentry;
1879	struct inode *inode;
1880
1881	dentry = d_alloc_name(sb->s_root, name);
1882	if (!dentry)
1883	return NULL;
1884
1885	inode = ffs_sb_make_inode(sb, data, fops, NULL, perms: &ffs->file_perms);
1886	if (!inode) {
1887	dput(dentry);
1888	return NULL;
1889	}
1890
1891	d_add(dentry, inode);
1892	return dentry;
1893	}
1894
1895	/* Super block */
1896	static const struct super_operations ffs_sb_operations = {
1897	.statfs = simple_statfs,
1898	.drop_inode = generic_delete_inode,
1899	};
1900
1901	struct ffs_sb_fill_data {
1902	struct ffs_file_perms perms;
1903	umode_t root_mode;
1904	const char *dev_name;
1905	bool no_disconnect;
1906	struct ffs_data *ffs_data;
1907	};
1908
1909	static int ffs_sb_fill(struct super_block *sb, struct fs_context *fc)
1910	{
1911	struct ffs_sb_fill_data *data = fc->fs_private;
1912	struct inode *inode;
1913	struct ffs_data *ffs = data->ffs_data;
1914
1915	ffs->sb = sb;
1916	data->ffs_data = NULL;
1917	sb->s_fs_info = ffs;
1918	sb->s_blocksize = PAGE_SIZE;
1919	sb->s_blocksize_bits = PAGE_SHIFT;
1920	sb->s_magic = FUNCTIONFS_MAGIC;
1921	sb->s_op = &ffs_sb_operations;
1922	sb->s_time_gran = 1;
1923
1924	/* Root inode */
1925	data->perms.mode = data->root_mode;
1926	inode = ffs_sb_make_inode(sb, NULL,
1927	fops: &simple_dir_operations,
1928	iops: &simple_dir_inode_operations,
1929	perms: &data->perms);
1930	sb->s_root = d_make_root(inode);
1931	if (!sb->s_root)
1932	return -ENOMEM;
1933
1934	/* EP0 file */
1935	if (!ffs_sb_create_file(sb, name: "ep0", data: ffs, fops: &ffs_ep0_operations))
1936	return -ENOMEM;
1937
1938	return 0;
1939	}
1940
1941	enum {
1942	Opt_no_disconnect,
1943	Opt_rmode,
1944	Opt_fmode,
1945	Opt_mode,
1946	Opt_uid,
1947	Opt_gid,
1948	};
1949
1950	static const struct fs_parameter_spec ffs_fs_fs_parameters[] = {
1951	fsparam_bool ("no_disconnect", Opt_no_disconnect),
1952	fsparam_u32 ("rmode", Opt_rmode),
1953	fsparam_u32 ("fmode", Opt_fmode),
1954	fsparam_u32 ("mode", Opt_mode),
1955	fsparam_u32 ("uid", Opt_uid),
1956	fsparam_u32 ("gid", Opt_gid),
1957	{}
1958	};
1959
1960	static int ffs_fs_parse_param(struct fs_context *fc, struct fs_parameter *param)
1961	{
1962	struct ffs_sb_fill_data *data = fc->fs_private;
1963	struct fs_parse_result result;
1964	int opt;
1965
1966	opt = fs_parse(fc, desc: ffs_fs_fs_parameters, param, result: &result);
1967	if (opt < 0)
1968	return opt;
1969
1970	switch (opt) {
1971	case Opt_no_disconnect:
1972	data->no_disconnect = result.boolean;
1973	break;
1974	case Opt_rmode:
1975	data->root_mode = (result.uint_32 & 0555) | S_IFDIR;
1976	break;
1977	case Opt_fmode:
1978	data->perms.mode = (result.uint_32 & 0666) | S_IFREG;
1979	break;
1980	case Opt_mode:
1981	data->root_mode = (result.uint_32 & 0555) | S_IFDIR;
1982	data->perms.mode = (result.uint_32 & 0666) | S_IFREG;
1983	break;
1984
1985	case Opt_uid:
1986	data->perms.uid = make_kuid(current_user_ns(), uid: result.uint_32);
1987	if (!uid_valid(uid: data->perms.uid))
1988	goto unmapped_value;
1989	break;
1990	case Opt_gid:
1991	data->perms.gid = make_kgid(current_user_ns(), gid: result.uint_32);
1992	if (!gid_valid(gid: data->perms.gid))
1993	goto unmapped_value;
1994	break;
1995
1996	default:
1997	return -ENOPARAM;
1998	}
1999
2000	return 0;
2001
2002	unmapped_value:
2003	return invalf(fc, "%s: unmapped value: %u", param->key, result.uint_32);
2004	}
2005
2006	/*
2007	* Set up the superblock for a mount.
2008	*/
2009	static int ffs_fs_get_tree(struct fs_context *fc)
2010	{
2011	struct ffs_sb_fill_data *ctx = fc->fs_private;
2012	struct ffs_data *ffs;
2013	int ret;
2014
2015	if (!fc->source)
2016	return invalf(fc, "No source specified");
2017
2018	ffs = ffs_data_new(dev_name: fc->source);
2019	if (!ffs)
2020	return -ENOMEM;
2021	ffs->file_perms = ctx->perms;
2022	ffs->no_disconnect = ctx->no_disconnect;
2023
2024	ffs->dev_name = kstrdup(s: fc->source, GFP_KERNEL);
2025	if (!ffs->dev_name) {
2026	ffs_data_put(ffs);
2027	return -ENOMEM;
2028	}
2029
2030	ret = ffs_acquire_dev(dev_name: ffs->dev_name, ffs_data: ffs);
2031	if (ret) {
2032	ffs_data_put(ffs);
2033	return ret;
2034	}
2035
2036	ctx->ffs_data = ffs;
2037	return get_tree_nodev(fc, fill_super: ffs_sb_fill);
2038	}
2039
2040	static void ffs_fs_free_fc(struct fs_context *fc)
2041	{
2042	struct ffs_sb_fill_data *ctx = fc->fs_private;
2043
2044	if (ctx) {
2045	if (ctx->ffs_data) {
2046	ffs_data_put(ffs: ctx->ffs_data);
2047	}
2048
2049	kfree(objp: ctx);
2050	}
2051	}
2052
2053	static const struct fs_context_operations ffs_fs_context_ops = {
2054	.free = ffs_fs_free_fc,
2055	.parse_param = ffs_fs_parse_param,
2056	.get_tree = ffs_fs_get_tree,
2057	};
2058
2059	static int ffs_fs_init_fs_context(struct fs_context *fc)
2060	{
2061	struct ffs_sb_fill_data *ctx;
2062
2063	ctx = kzalloc(size: sizeof(struct ffs_sb_fill_data), GFP_KERNEL);
2064	if (!ctx)
2065	return -ENOMEM;
2066
2067	ctx->perms.mode = S_IFREG | 0600;
2068	ctx->perms.uid = GLOBAL_ROOT_UID;
2069	ctx->perms.gid = GLOBAL_ROOT_GID;
2070	ctx->root_mode = S_IFDIR | 0500;
2071	ctx->no_disconnect = false;
2072
2073	fc->fs_private = ctx;
2074	fc->ops = &ffs_fs_context_ops;
2075	return 0;
2076	}
2077
2078	static void
2079	ffs_fs_kill_sb(struct super_block *sb)
2080	{
2081	kill_litter_super(sb);
2082	if (sb->s_fs_info)
2083	ffs_data_closed(ffs: sb->s_fs_info);
2084	}
2085
2086	static struct file_system_type ffs_fs_type = {
2087	.owner = THIS_MODULE,
2088	.name = "functionfs",
2089	.init_fs_context = ffs_fs_init_fs_context,
2090	.parameters = ffs_fs_fs_parameters,
2091	.kill_sb = ffs_fs_kill_sb,
2092	};
2093	MODULE_ALIAS_FS("functionfs");
2094
2095
2096	/* Driver's main init/cleanup functions *************************************/
2097
2098	static int functionfs_init(void)
2099	{
2100	int ret;
2101
2102	ret = register_filesystem(&ffs_fs_type);
2103	if (!ret)
2104	pr_info("file system registered\n");
2105	else
2106	pr_err("failed registering file system (%d)\n", ret);
2107
2108	return ret;
2109	}
2110
2111	static void functionfs_cleanup(void)
2112	{
2113	pr_info("unloading\n");
2114	unregister_filesystem(&ffs_fs_type);
2115	}
2116
2117
2118	/* ffs_data and ffs_function construction and destruction code **************/
2119
2120	static void ffs_data_clear(struct ffs_data *ffs);
2121	static void ffs_data_reset(struct ffs_data *ffs);
2122
2123	static void ffs_data_get(struct ffs_data *ffs)
2124	{
2125	refcount_inc(r: &ffs->ref);
2126	}
2127
2128	static void ffs_data_opened(struct ffs_data *ffs)
2129	{
2130	refcount_inc(r: &ffs->ref);
2131	if (atomic_add_return(i: 1, v: &ffs->opened) == 1 &&
2132	ffs->state == FFS_DEACTIVATED) {
2133	ffs->state = FFS_CLOSING;
2134	ffs_data_reset(ffs);
2135	}
2136	}
2137
2138	static void ffs_data_put(struct ffs_data *ffs)
2139	{
2140	if (refcount_dec_and_test(r: &ffs->ref)) {
2141	pr_info("%s(): freeing\n", __func__);
2142	ffs_data_clear(ffs);
2143	ffs_release_dev(ffs_dev: ffs->private_data);
2144	BUG_ON(waitqueue_active(&ffs->ev.waitq) ||
2145	swait_active(&ffs->ep0req_completion.wait) ||
2146	waitqueue_active(&ffs->wait));
2147	destroy_workqueue(wq: ffs->io_completion_wq);
2148	kfree(objp: ffs->dev_name);
2149	kfree(objp: ffs);
2150	}
2151	}
2152
2153	static void ffs_data_closed(struct ffs_data *ffs)
2154	{
2155	struct ffs_epfile *epfiles;
2156	unsigned long flags;
2157
2158	if (atomic_dec_and_test(v: &ffs->opened)) {
2159	if (ffs->no_disconnect) {
2160	ffs->state = FFS_DEACTIVATED;
2161	spin_lock_irqsave(&ffs->eps_lock, flags);
2162	epfiles = ffs->epfiles;
2163	ffs->epfiles = NULL;
2164	spin_unlock_irqrestore(lock: &ffs->eps_lock,
2165	flags);
2166
2167	if (epfiles)
2168	ffs_epfiles_destroy(epfiles,
2169	count: ffs->eps_count);
2170
2171	if (ffs->setup_state == FFS_SETUP_PENDING)
2172	__ffs_ep0_stall(ffs);
2173	} else {
2174	ffs->state = FFS_CLOSING;
2175	ffs_data_reset(ffs);
2176	}
2177	}
2178	if (atomic_read(v: &ffs->opened) < 0) {
2179	ffs->state = FFS_CLOSING;
2180	ffs_data_reset(ffs);
2181	}
2182
2183	ffs_data_put(ffs);
2184	}
2185
2186	static struct ffs_data *ffs_data_new(const char *dev_name)
2187	{
2188	struct ffs_data *ffs = kzalloc(size: sizeof *ffs, GFP_KERNEL);
2189	if (!ffs)
2190	return NULL;
2191
2192	ffs->io_completion_wq = alloc_ordered_workqueue("%s", 0, dev_name);
2193	if (!ffs->io_completion_wq) {
2194	kfree(objp: ffs);
2195	return NULL;
2196	}
2197
2198	refcount_set(r: &ffs->ref, n: 1);
2199	atomic_set(v: &ffs->opened, i: 0);
2200	ffs->state = FFS_READ_DESCRIPTORS;
2201	mutex_init(&ffs->mutex);
2202	spin_lock_init(&ffs->eps_lock);
2203	init_waitqueue_head(&ffs->ev.waitq);
2204	init_waitqueue_head(&ffs->wait);
2205	init_completion(x: &ffs->ep0req_completion);
2206
2207	/* XXX REVISIT need to update it in some places, or do we? */
2208	ffs->ev.can_stall = 1;
2209
2210	return ffs;
2211	}
2212
2213	static void ffs_data_clear(struct ffs_data *ffs)
2214	{
2215	struct ffs_epfile *epfiles;
2216	unsigned long flags;
2217
2218	ffs_closed(ffs);
2219
2220	BUG_ON(ffs->gadget);
2221
2222	spin_lock_irqsave(&ffs->eps_lock, flags);
2223	epfiles = ffs->epfiles;
2224	ffs->epfiles = NULL;
2225	spin_unlock_irqrestore(lock: &ffs->eps_lock, flags);
2226
2227	/*
2228	* potential race possible between ffs_func_eps_disable
2229	* & ffs_epfile_release therefore maintaining a local
2230	* copy of epfile will save us from use-after-free.
2231	*/
2232	if (epfiles) {
2233	ffs_epfiles_destroy(epfiles, count: ffs->eps_count);
2234	ffs->epfiles = NULL;
2235	}
2236
2237	if (ffs->ffs_eventfd) {
2238	eventfd_ctx_put(ctx: ffs->ffs_eventfd);
2239	ffs->ffs_eventfd = NULL;
2240	}
2241
2242	kfree(objp: ffs->raw_descs_data);
2243	kfree(objp: ffs->raw_strings);
2244	kfree(objp: ffs->stringtabs);
2245	}
2246
2247	static void ffs_data_reset(struct ffs_data *ffs)
2248	{
2249	ffs_data_clear(ffs);
2250
2251	ffs->raw_descs_data = NULL;
2252	ffs->raw_descs = NULL;
2253	ffs->raw_strings = NULL;
2254	ffs->stringtabs = NULL;
2255
2256	ffs->raw_descs_length = 0;
2257	ffs->fs_descs_count = 0;
2258	ffs->hs_descs_count = 0;
2259	ffs->ss_descs_count = 0;
2260
2261	ffs->strings_count = 0;
2262	ffs->interfaces_count = 0;
2263	ffs->eps_count = 0;
2264
2265	ffs->ev.count = 0;
2266
2267	ffs->state = FFS_READ_DESCRIPTORS;
2268	ffs->setup_state = FFS_NO_SETUP;
2269	ffs->flags = 0;
2270
2271	ffs->ms_os_descs_ext_prop_count = 0;
2272	ffs->ms_os_descs_ext_prop_name_len = 0;
2273	ffs->ms_os_descs_ext_prop_data_len = 0;
2274	}
2275
2276
2277	static int functionfs_bind(struct ffs_data *ffs, struct usb_composite_dev *cdev)
2278	{
2279	struct usb_gadget_strings **lang;
2280	int first_id;
2281
2282	if (WARN_ON(ffs->state != FFS_ACTIVE
2283	|| test_and_set_bit(FFS_FL_BOUND, &ffs->flags)))
2284	return -EBADFD;
2285
2286	first_id = usb_string_ids_n(c: cdev, n: ffs->strings_count);
2287	if (first_id < 0)
2288	return first_id;
2289
2290	ffs->ep0req = usb_ep_alloc_request(ep: cdev->gadget->ep0, GFP_KERNEL);
2291	if (!ffs->ep0req)
2292	return -ENOMEM;
2293	ffs->ep0req->complete = ffs_ep0_complete;
2294	ffs->ep0req->context = ffs;
2295
2296	lang = ffs->stringtabs;
2297	if (lang) {
2298	for (; *lang; ++lang) {
2299	struct usb_string *str = (*lang)->strings;
2300	int id = first_id;
2301	for (; str->s; ++id, ++str)
2302	str->id = id;
2303	}
2304	}
2305
2306	ffs->gadget = cdev->gadget;
2307	ffs_data_get(ffs);
2308	return 0;
2309	}
2310
2311	static void functionfs_unbind(struct ffs_data *ffs)
2312	{
2313	if (!WARN_ON(!ffs->gadget)) {
2314	/* dequeue before freeing ep0req */
2315	usb_ep_dequeue(ep: ffs->gadget->ep0, req: ffs->ep0req);
2316	mutex_lock(&ffs->mutex);
2317	usb_ep_free_request(ep: ffs->gadget->ep0, req: ffs->ep0req);
2318	ffs->ep0req = NULL;
2319	ffs->gadget = NULL;
2320	clear_bit(FFS_FL_BOUND, addr: &ffs->flags);
2321	mutex_unlock(lock: &ffs->mutex);
2322	ffs_data_put(ffs);
2323	}
2324	}
2325
2326	static int ffs_epfiles_create(struct ffs_data *ffs)
2327	{
2328	struct ffs_epfile *epfile, *epfiles;
2329	unsigned i, count;
2330
2331	count = ffs->eps_count;
2332	epfiles = kcalloc(n: count, size: sizeof(*epfiles), GFP_KERNEL);
2333	if (!epfiles)
2334	return -ENOMEM;
2335
2336	epfile = epfiles;
2337	for (i = 1; i <= count; ++i, ++epfile) {
2338	epfile->ffs = ffs;
2339	mutex_init(&epfile->mutex);
2340	mutex_init(&epfile->dmabufs_mutex);
2341	INIT_LIST_HEAD(list: &epfile->dmabufs);
2342	if (ffs->user_flags & FUNCTIONFS_VIRTUAL_ADDR)
2343	sprintf(buf: epfile->name, fmt: "ep%02x", ffs->eps_addrmap[i]);
2344	else
2345	sprintf(buf: epfile->name, fmt: "ep%u", i);
2346	epfile->dentry = ffs_sb_create_file(sb: ffs->sb, name: epfile->name,
2347	data: epfile,
2348	fops: &ffs_epfile_operations);
2349	if (!epfile->dentry) {
2350	ffs_epfiles_destroy(epfiles, count: i - 1);
2351	return -ENOMEM;
2352	}
2353	}
2354
2355	ffs->epfiles = epfiles;
2356	return 0;
2357	}
2358
2359	static void ffs_epfiles_destroy(struct ffs_epfile *epfiles, unsigned count)
2360	{
2361	struct ffs_epfile *epfile = epfiles;
2362
2363	for (; count; --count, ++epfile) {
2364	BUG_ON(mutex_is_locked(&epfile->mutex));
2365	if (epfile->dentry) {
2366	d_delete(epfile->dentry);
2367	dput(epfile->dentry);
2368	epfile->dentry = NULL;
2369	}
2370	}
2371
2372	kfree(objp: epfiles);
2373	}
2374
2375	static void ffs_func_eps_disable(struct ffs_function *func)
2376	{
2377	struct ffs_ep *ep;
2378	struct ffs_epfile *epfile;
2379	unsigned short count;
2380	unsigned long flags;
2381
2382	spin_lock_irqsave(&func->ffs->eps_lock, flags);
2383	count = func->ffs->eps_count;
2384	epfile = func->ffs->epfiles;
2385	ep = func->eps;
2386	while (count--) {
2387	/* pending requests get nuked */
2388	if (ep->ep)
2389	usb_ep_disable(ep: ep->ep);
2390	++ep;
2391
2392	if (epfile) {
2393	epfile->ep = NULL;
2394	__ffs_epfile_read_buffer_free(epfile);
2395	++epfile;
2396	}
2397	}
2398	spin_unlock_irqrestore(lock: &func->ffs->eps_lock, flags);
2399	}
2400
2401	static int ffs_func_eps_enable(struct ffs_function *func)
2402	{
2403	struct ffs_data *ffs;
2404	struct ffs_ep *ep;
2405	struct ffs_epfile *epfile;
2406	unsigned short count;
2407	unsigned long flags;
2408	int ret = 0;
2409
2410	spin_lock_irqsave(&func->ffs->eps_lock, flags);
2411	ffs = func->ffs;
2412	ep = func->eps;
2413	epfile = ffs->epfiles;
2414	count = ffs->eps_count;
2415	while(count--) {
2416	ep->ep->driver_data = ep;
2417
2418	ret = config_ep_by_speed(g: func->gadget, f: &func->function, ep: ep->ep);
2419	if (ret) {
2420	pr_err("%s: config_ep_by_speed(%s) returned %d\n",
2421	__func__, ep->ep->name, ret);
2422	break;
2423	}
2424
2425	ret = usb_ep_enable(ep: ep->ep);
2426	if (!ret) {
2427	epfile->ep = ep;
2428	epfile->in = usb_endpoint_dir_in(epd: ep->ep->desc);
2429	epfile->isoc = usb_endpoint_xfer_isoc(epd: ep->ep->desc);
2430	} else {
2431	break;
2432	}
2433
2434	++ep;
2435	++epfile;
2436	}
2437
2438	wake_up_interruptible(&ffs->wait);
2439	spin_unlock_irqrestore(lock: &func->ffs->eps_lock, flags);
2440
2441	return ret;
2442	}
2443
2444
2445	/* Parsing and building descriptors and strings *****************************/
2446
2447	/*
2448	* This validates if data pointed by data is a valid USB descriptor as
2449	* well as record how many interfaces, endpoints and strings are
2450	* required by given configuration. Returns address after the
2451	* descriptor or NULL if data is invalid.
2452	*/
2453
2454	enum ffs_entity_type {
2455	FFS_DESCRIPTOR, FFS_INTERFACE, FFS_STRING, FFS_ENDPOINT
2456	};
2457
2458	enum ffs_os_desc_type {
2459	FFS_OS_DESC, FFS_OS_DESC_EXT_COMPAT, FFS_OS_DESC_EXT_PROP
2460	};
2461
2462	typedef int (*ffs_entity_callback)(enum ffs_entity_type entity,
2463	u8 *valuep,
2464	struct usb_descriptor_header *desc,
2465	void *priv);
2466
2467	typedef int (*ffs_os_desc_callback)(enum ffs_os_desc_type entity,
2468	struct usb_os_desc_header *h, void *data,
2469	unsigned len, void *priv);
2470
2471	static int __must_check ffs_do_single_desc(char *data, unsigned len,
2472	ffs_entity_callback entity,
2473	void *priv, int *current_class)
2474	{
2475	struct usb_descriptor_header *_ds = (void *)data;
2476	u8 length;
2477	int ret;
2478
2479	/* At least two bytes are required: length and type */
2480	if (len < 2) {
2481	pr_vdebug("descriptor too short\n");
2482	return -EINVAL;
2483	}
2484
2485	/* If we have at least as many bytes as the descriptor takes? */
2486	length = _ds->bLength;
2487	if (len < length) {
2488	pr_vdebug("descriptor longer then available data\n");
2489	return -EINVAL;
2490	}
2491
2492	#define __entity_check_INTERFACE(val) 1
2493	#define __entity_check_STRING(val) (val)
2494	#define __entity_check_ENDPOINT(val) ((val) & USB_ENDPOINT_NUMBER_MASK)
2495	#define __entity(type, val) do { \
2496	pr_vdebug("entity " #type "(%02x)\n", (val)); \
2497	if (!__entity_check_ ##type(val)) { \
2498	pr_vdebug("invalid entity's value\n"); \
2499	return -EINVAL; \
2500	} \
2501	ret = entity(FFS_ ##type, &val, _ds, priv); \
2502	if (ret < 0) { \
2503	pr_debug("entity " #type "(%02x); ret = %d\n", \
2504	(val), ret); \
2505	return ret; \
2506	} \
2507	} while (0)
2508
2509	/* Parse descriptor depending on type. */
2510	switch (_ds->bDescriptorType) {
2511	case USB_DT_DEVICE:
2512	case USB_DT_CONFIG:
2513	case USB_DT_STRING:
2514	case USB_DT_DEVICE_QUALIFIER:
2515	/* function can't have any of those */
2516	pr_vdebug("descriptor reserved for gadget: %d\n",
2517	_ds->bDescriptorType);
2518	return -EINVAL;
2519
2520	case USB_DT_INTERFACE: {
2521	struct usb_interface_descriptor *ds = (void *)_ds;
2522	pr_vdebug("interface descriptor\n");
2523	if (length != sizeof *ds)
2524	goto inv_length;
2525
2526	__entity(INTERFACE, ds->bInterfaceNumber);
2527	if (ds->iInterface)
2528	__entity(STRING, ds->iInterface);
2529	*current_class = ds->bInterfaceClass;
2530	}
2531	break;
2532
2533	case USB_DT_ENDPOINT: {
2534	struct usb_endpoint_descriptor *ds = (void *)_ds;
2535	pr_vdebug("endpoint descriptor\n");
2536	if (length != USB_DT_ENDPOINT_SIZE &&
2537	length != USB_DT_ENDPOINT_AUDIO_SIZE)
2538	goto inv_length;
2539	__entity(ENDPOINT, ds->bEndpointAddress);
2540	}
2541	break;
2542
2543	case USB_TYPE_CLASS | 0x01:
2544	if (*current_class == USB_INTERFACE_CLASS_HID) {
2545	pr_vdebug("hid descriptor\n");
2546	if (length != sizeof(struct hid_descriptor))
2547	goto inv_length;
2548	break;
2549	} else if (*current_class == USB_INTERFACE_CLASS_CCID) {
2550	pr_vdebug("ccid descriptor\n");
2551	if (length != sizeof(struct ccid_descriptor))
2552	goto inv_length;
2553	break;
2554	} else {
2555	pr_vdebug("unknown descriptor: %d for class %d\n",
2556	_ds->bDescriptorType, *current_class);
2557	return -EINVAL;
2558	}
2559
2560	case USB_DT_OTG:
2561	if (length != sizeof(struct usb_otg_descriptor))
2562	goto inv_length;
2563	break;
2564
2565	case USB_DT_INTERFACE_ASSOCIATION: {
2566	struct usb_interface_assoc_descriptor *ds = (void *)_ds;
2567	pr_vdebug("interface association descriptor\n");
2568	if (length != sizeof *ds)
2569	goto inv_length;
2570	if (ds->iFunction)
2571	__entity(STRING, ds->iFunction);
2572	}
2573	break;
2574
2575	case USB_DT_SS_ENDPOINT_COMP:
2576	pr_vdebug("EP SS companion descriptor\n");
2577	if (length != sizeof(struct usb_ss_ep_comp_descriptor))
2578	goto inv_length;
2579	break;
2580
2581	case USB_DT_OTHER_SPEED_CONFIG:
2582	case USB_DT_INTERFACE_POWER:
2583	case USB_DT_DEBUG:
2584	case USB_DT_SECURITY:
2585	case USB_DT_CS_RADIO_CONTROL:
2586	/* TODO */
2587	pr_vdebug("unimplemented descriptor: %d\n", _ds->bDescriptorType);
2588	return -EINVAL;
2589
2590	default:
2591	/* We should never be here */
2592	pr_vdebug("unknown descriptor: %d\n", _ds->bDescriptorType);
2593	return -EINVAL;
2594
2595	inv_length:
2596	pr_vdebug("invalid length: %d (descriptor %d)\n",
2597	_ds->bLength, _ds->bDescriptorType);
2598	return -EINVAL;
2599	}
2600
2601	#undef __entity
2602	#undef __entity_check_DESCRIPTOR
2603	#undef __entity_check_INTERFACE
2604	#undef __entity_check_STRING
2605	#undef __entity_check_ENDPOINT
2606
2607	return length;
2608	}
2609
2610	static int __must_check ffs_do_descs(unsigned count, char *data, unsigned len,
2611	ffs_entity_callback entity, void *priv)
2612	{
2613	const unsigned _len = len;
2614	unsigned long num = 0;
2615	int current_class = -1;
2616
2617	for (;;) {
2618	int ret;
2619
2620	if (num == count)
2621	data = NULL;
2622
2623	/* Record "descriptor" entity */
2624	ret = entity(FFS_DESCRIPTOR, (u8 *)num, (void *)data, priv);
2625	if (ret < 0) {
2626	pr_debug("entity DESCRIPTOR(%02lx); ret = %d\n",
2627	num, ret);
2628	return ret;
2629	}
2630
2631	if (!data)
2632	return _len - len;
2633
2634	ret = ffs_do_single_desc(data, len, entity, priv,
2635	current_class: &current_class);
2636	if (ret < 0) {
2637	pr_debug("%s returns %d\n", __func__, ret);
2638	return ret;
2639	}
2640
2641	len -= ret;
2642	data += ret;
2643	++num;
2644	}
2645	}
2646
2647	static int __ffs_data_do_entity(enum ffs_entity_type type,
2648	u8 *valuep, struct usb_descriptor_header *desc,
2649	void *priv)
2650	{
2651	struct ffs_desc_helper *helper = priv;
2652	struct usb_endpoint_descriptor *d;
2653
2654	switch (type) {
2655	case FFS_DESCRIPTOR:
2656	break;
2657
2658	case FFS_INTERFACE:
2659	/*
2660	* Interfaces are indexed from zero so if we
2661	* encountered interface "n" then there are at least
2662	* "n+1" interfaces.
2663	*/
2664	if (*valuep >= helper->interfaces_count)
2665	helper->interfaces_count = *valuep + 1;
2666	break;
2667
2668	case FFS_STRING:
2669	/*
2670	* Strings are indexed from 1 (0 is reserved
2671	* for languages list)
2672	*/
2673	if (*valuep > helper->ffs->strings_count)
2674	helper->ffs->strings_count = *valuep;
2675	break;
2676
2677	case FFS_ENDPOINT:
2678	d = (void *)desc;
2679	helper->eps_count++;
2680	if (helper->eps_count >= FFS_MAX_EPS_COUNT)
2681	return -EINVAL;
2682	/* Check if descriptors for any speed were already parsed */
2683	if (!helper->ffs->eps_count && !helper->ffs->interfaces_count)
2684	helper->ffs->eps_addrmap[helper->eps_count] =
2685	d->bEndpointAddress;
2686	else if (helper->ffs->eps_addrmap[helper->eps_count] !=
2687	d->bEndpointAddress)
2688	return -EINVAL;
2689	break;
2690	}
2691
2692	return 0;
2693	}
2694
2695	static int __ffs_do_os_desc_header(enum ffs_os_desc_type *next_type,
2696	struct usb_os_desc_header *desc)
2697	{
2698	u16 bcd_version = le16_to_cpu(desc->bcdVersion);
2699	u16 w_index = le16_to_cpu(desc->wIndex);
2700
2701	if (bcd_version == 0x1) {
2702	pr_warn("bcdVersion must be 0x0100, stored in Little Endian order. "
2703	"Userspace driver should be fixed, accepting 0x0001 for compatibility.\n");
2704	} else if (bcd_version != 0x100) {
2705	pr_vdebug("unsupported os descriptors version: 0x%x\n",
2706	bcd_version);
2707	return -EINVAL;
2708	}
2709	switch (w_index) {
2710	case 0x4:
2711	*next_type = FFS_OS_DESC_EXT_COMPAT;
2712	break;
2713	case 0x5:
2714	*next_type = FFS_OS_DESC_EXT_PROP;
2715	break;
2716	default:
2717	pr_vdebug("unsupported os descriptor type: %d", w_index);
2718	return -EINVAL;
2719	}
2720
2721	return sizeof(*desc);
2722	}
2723
2724	/*
2725	* Process all extended compatibility/extended property descriptors
2726	* of a feature descriptor
2727	*/
2728	static int __must_check ffs_do_single_os_desc(char *data, unsigned len,
2729	enum ffs_os_desc_type type,
2730	u16 feature_count,
2731	ffs_os_desc_callback entity,
2732	void *priv,
2733	struct usb_os_desc_header *h)
2734	{
2735	int ret;
2736	const unsigned _len = len;
2737
2738	/* loop over all ext compat/ext prop descriptors */
2739	while (feature_count--) {
2740	ret = entity(type, h, data, len, priv);
2741	if (ret < 0) {
2742	pr_debug("bad OS descriptor, type: %d\n", type);
2743	return ret;
2744	}
2745	data += ret;
2746	len -= ret;
2747	}
2748	return _len - len;
2749	}
2750
2751	/* Process a number of complete Feature Descriptors (Ext Compat or Ext Prop) */
2752	static int __must_check ffs_do_os_descs(unsigned count,
2753	char *data, unsigned len,
2754	ffs_os_desc_callback entity, void *priv)
2755	{
2756	const unsigned _len = len;
2757	unsigned long num = 0;
2758
2759	for (num = 0; num < count; ++num) {
2760	int ret;
2761	enum ffs_os_desc_type type;
2762	u16 feature_count;
2763	struct usb_os_desc_header *desc = (void *)data;
2764
2765	if (len < sizeof(*desc))
2766	return -EINVAL;
2767
2768	/*
2769	* Record "descriptor" entity.
2770	* Process dwLength, bcdVersion, wIndex, get b/wCount.
2771	* Move the data pointer to the beginning of extended
2772	* compatibilities proper or extended properties proper
2773	* portions of the data
2774	*/
2775	if (le32_to_cpu(desc->dwLength) > len)
2776	return -EINVAL;
2777
2778	ret = __ffs_do_os_desc_header(next_type: &type, desc);
2779	if (ret < 0) {
2780	pr_debug("entity OS_DESCRIPTOR(%02lx); ret = %d\n",
2781	num, ret);
2782	return ret;
2783	}
2784	/*
2785	* 16-bit hex "?? 00" Little Endian looks like 8-bit hex "??"
2786	*/
2787	feature_count = le16_to_cpu(desc->wCount);
2788	if (type == FFS_OS_DESC_EXT_COMPAT &&
2789	(feature_count > 255 || desc->Reserved))
2790	return -EINVAL;
2791	len -= ret;
2792	data += ret;
2793
2794	/*
2795	* Process all function/property descriptors
2796	* of this Feature Descriptor
2797	*/
2798	ret = ffs_do_single_os_desc(data, len, type,
2799	feature_count, entity, priv, h: desc);
2800	if (ret < 0) {
2801	pr_debug("%s returns %d\n", __func__, ret);
2802	return ret;
2803	}
2804
2805	len -= ret;
2806	data += ret;
2807	}
2808	return _len - len;
2809	}
2810
2811	/*
2812	* Validate contents of the buffer from userspace related to OS descriptors.
2813	*/
2814	static int __ffs_data_do_os_desc(enum ffs_os_desc_type type,
2815	struct usb_os_desc_header *h, void *data,
2816	unsigned len, void *priv)
2817	{
2818	struct ffs_data *ffs = priv;
2819	u8 length;
2820
2821	switch (type) {
2822	case FFS_OS_DESC_EXT_COMPAT: {
2823	struct usb_ext_compat_desc *d = data;
2824	int i;
2825
2826	if (len < sizeof(*d) ||
2827	d->bFirstInterfaceNumber >= ffs->interfaces_count)
2828	return -EINVAL;
2829	if (d->Reserved1 != 1) {
2830	/*
2831	* According to the spec, Reserved1 must be set to 1
2832	* but older kernels incorrectly rejected non-zero
2833	* values. We fix it here to avoid returning EINVAL
2834	* in response to values we used to accept.
2835	*/
2836	pr_debug("usb_ext_compat_desc::Reserved1 forced to 1\n");
2837	d->Reserved1 = 1;
2838	}
2839	for (i = 0; i < ARRAY_SIZE(d->Reserved2); ++i)
2840	if (d->Reserved2[i])
2841	return -EINVAL;
2842
2843	length = sizeof(struct usb_ext_compat_desc);
2844	}
2845	break;
2846	case FFS_OS_DESC_EXT_PROP: {
2847	struct usb_ext_prop_desc *d = data;
2848	u32 type, pdl;
2849	u16 pnl;
2850
2851	if (len < sizeof(*d) || h->interface >= ffs->interfaces_count)
2852	return -EINVAL;
2853	length = le32_to_cpu(d->dwSize);
2854	if (len < length)
2855	return -EINVAL;
2856	type = le32_to_cpu(d->dwPropertyDataType);
2857	if (type < USB_EXT_PROP_UNICODE ||
2858	type > USB_EXT_PROP_UNICODE_MULTI) {
2859	pr_vdebug("unsupported os descriptor property type: %d",
2860	type);
2861	return -EINVAL;
2862	}
2863	pnl = le16_to_cpu(d->wPropertyNameLength);
2864	if (length < 14 + pnl) {
2865	pr_vdebug("invalid os descriptor length: %d pnl:%d (descriptor %d)\n",
2866	length, pnl, type);
2867	return -EINVAL;
2868	}
2869	pdl = le32_to_cpu(*(__le32 *)((u8 *)data + 10 + pnl));
2870	if (length != 14 + pnl + pdl) {
2871	pr_vdebug("invalid os descriptor length: %d pnl:%d pdl:%d (descriptor %d)\n",
2872	length, pnl, pdl, type);
2873	return -EINVAL;
2874	}
2875	++ffs->ms_os_descs_ext_prop_count;
2876	/* property name reported to the host as "WCHAR"s */
2877	ffs->ms_os_descs_ext_prop_name_len += pnl * 2;
2878	ffs->ms_os_descs_ext_prop_data_len += pdl;
2879	}
2880	break;
2881	default:
2882	pr_vdebug("unknown descriptor: %d\n", type);
2883	return -EINVAL;
2884	}
2885	return length;
2886	}
2887
2888	static int __ffs_data_got_descs(struct ffs_data *ffs,
2889	char *const _data, size_t len)
2890	{
2891	char *data = _data, *raw_descs;
2892	unsigned os_descs_count = 0, counts[3], flags;
2893	int ret = -EINVAL, i;
2894	struct ffs_desc_helper helper;
2895
2896	if (get_unaligned_le32(p: data + 4) != len)
2897	goto error;
2898
2899	switch (get_unaligned_le32(p: data)) {
2900	case FUNCTIONFS_DESCRIPTORS_MAGIC:
2901	flags = FUNCTIONFS_HAS_FS_DESC | FUNCTIONFS_HAS_HS_DESC;
2902	data += 8;
2903	len -= 8;
2904	break;
2905	case FUNCTIONFS_DESCRIPTORS_MAGIC_V2:
2906	flags = get_unaligned_le32(p: data + 8);
2907	ffs->user_flags = flags;
2908	if (flags & ~(FUNCTIONFS_HAS_FS_DESC |
2909	FUNCTIONFS_HAS_HS_DESC |
2910	FUNCTIONFS_HAS_SS_DESC |
2911	FUNCTIONFS_HAS_MS_OS_DESC |
2912	FUNCTIONFS_VIRTUAL_ADDR |
2913	FUNCTIONFS_EVENTFD |
2914	FUNCTIONFS_ALL_CTRL_RECIP |
2915	FUNCTIONFS_CONFIG0_SETUP)) {
2916	ret = -ENOSYS;
2917	goto error;
2918	}
2919	data += 12;
2920	len -= 12;
2921	break;
2922	default:
2923	goto error;
2924	}
2925
2926	if (flags & FUNCTIONFS_EVENTFD) {
2927	if (len < 4)
2928	goto error;
2929	ffs->ffs_eventfd =
2930	eventfd_ctx_fdget(fd: (int)get_unaligned_le32(p: data));
2931	if (IS_ERR(ptr: ffs->ffs_eventfd)) {
2932	ret = PTR_ERR(ptr: ffs->ffs_eventfd);
2933	ffs->ffs_eventfd = NULL;
2934	goto error;
2935	}
2936	data += 4;
2937	len -= 4;
2938	}
2939
2940	/* Read fs_count, hs_count and ss_count (if present) */
2941	for (i = 0; i < 3; ++i) {
2942	if (!(flags & (1 << i))) {
2943	counts[i] = 0;
2944	} else if (len < 4) {
2945	goto error;
2946	} else {
2947	counts[i] = get_unaligned_le32(p: data);
2948	data += 4;
2949	len -= 4;
2950	}
2951	}
2952	if (flags & (1 << i)) {
2953	if (len < 4) {
2954	goto error;
2955	}
2956	os_descs_count = get_unaligned_le32(p: data);
2957	data += 4;
2958	len -= 4;
2959	}
2960
2961	/* Read descriptors */
2962	raw_descs = data;
2963	helper.ffs = ffs;
2964	for (i = 0; i < 3; ++i) {
2965	if (!counts[i])
2966	continue;
2967	helper.interfaces_count = 0;
2968	helper.eps_count = 0;
2969	ret = ffs_do_descs(count: counts[i], data, len,
2970	entity: __ffs_data_do_entity, priv: &helper);
2971	if (ret < 0)
2972	goto error;
2973	if (!ffs->eps_count && !ffs->interfaces_count) {
2974	ffs->eps_count = helper.eps_count;
2975	ffs->interfaces_count = helper.interfaces_count;
2976	} else {
2977	if (ffs->eps_count != helper.eps_count) {
2978	ret = -EINVAL;
2979	goto error;
2980	}
2981	if (ffs->interfaces_count != helper.interfaces_count) {
2982	ret = -EINVAL;
2983	goto error;
2984	}
2985	}
2986	data += ret;
2987	len -= ret;
2988	}
2989	if (os_descs_count) {
2990	ret = ffs_do_os_descs(count: os_descs_count, data, len,
2991	entity: __ffs_data_do_os_desc, priv: ffs);
2992	if (ret < 0)
2993	goto error;
2994	data += ret;
2995	len -= ret;
2996	}
2997
2998	if (raw_descs == data || len) {
2999	ret = -EINVAL;
3000	goto error;
3001	}
3002
3003	ffs->raw_descs_data = _data;
3004	ffs->raw_descs = raw_descs;
3005	ffs->raw_descs_length = data - raw_descs;
3006	ffs->fs_descs_count = counts[0];
3007	ffs->hs_descs_count = counts[1];
3008	ffs->ss_descs_count = counts[2];
3009	ffs->ms_os_descs_count = os_descs_count;
3010
3011	return 0;
3012
3013	error:
3014	kfree(objp: _data);
3015	return ret;
3016	}
3017
3018	static int __ffs_data_got_strings(struct ffs_data *ffs,
3019	char *const _data, size_t len)
3020	{
3021	u32 str_count, needed_count, lang_count;
3022	struct usb_gadget_strings **stringtabs, *t;
3023	const char *data = _data;
3024	struct usb_string *s;
3025
3026	if (len < 16 ||
3027	get_unaligned_le32(p: data) != FUNCTIONFS_STRINGS_MAGIC ||
3028	get_unaligned_le32(p: data + 4) != len)
3029	goto error;
3030	str_count = get_unaligned_le32(p: data + 8);
3031	lang_count = get_unaligned_le32(p: data + 12);
3032
3033	/* if one is zero the other must be zero */
3034	if (!str_count != !lang_count)
3035	goto error;
3036
3037	/* Do we have at least as many strings as descriptors need? */
3038	needed_count = ffs->strings_count;
3039	if (str_count < needed_count)
3040	goto error;
3041
3042	/*
3043	* If we don't need any strings just return and free all
3044	* memory.
3045	*/
3046	if (!needed_count) {
3047	kfree(objp: _data);
3048	return 0;
3049	}
3050
3051	/* Allocate everything in one chunk so there's less maintenance. */
3052	{
3053	unsigned i = 0;
3054	vla_group(d);
3055	vla_item(d, struct usb_gadget_strings *, stringtabs,
3056	size_add(lang_count, 1));
3057	vla_item(d, struct usb_gadget_strings, stringtab, lang_count);
3058	vla_item(d, struct usb_string, strings,
3059	size_mul(lang_count, (needed_count + 1)));
3060
3061	char *vlabuf = kmalloc(vla_group_size(d), GFP_KERNEL);
3062
3063	if (!vlabuf) {
3064	kfree(objp: _data);
3065	return -ENOMEM;
3066	}
3067
3068	/* Initialize the VLA pointers */
3069	stringtabs = vla_ptr(vlabuf, d, stringtabs);
3070	t = vla_ptr(vlabuf, d, stringtab);
3071	i = lang_count;
3072	do {
3073	*stringtabs++ = t++;
3074	} while (--i);
3075	*stringtabs = NULL;
3076
3077	/* stringtabs = vlabuf = d_stringtabs for later kfree */
3078	stringtabs = vla_ptr(vlabuf, d, stringtabs);
3079	t = vla_ptr(vlabuf, d, stringtab);
3080	s = vla_ptr(vlabuf, d, strings);
3081	}
3082
3083	/* For each language */
3084	data += 16;
3085	len -= 16;
3086
3087	do { /* lang_count > 0 so we can use do-while */
3088	unsigned needed = needed_count;
3089	u32 str_per_lang = str_count;
3090
3091	if (len < 3)
3092	goto error_free;
3093	t->language = get_unaligned_le16(p: data);
3094	t->strings = s;
3095	++t;
3096
3097	data += 2;
3098	len -= 2;
3099
3100	/* For each string */
3101	do { /* str_count > 0 so we can use do-while */
3102	size_t length = strnlen(p: data, maxlen: len);
3103
3104	if (length == len)
3105	goto error_free;
3106
3107	/*
3108	* User may provide more strings then we need,
3109	* if that's the case we simply ignore the
3110	* rest
3111	*/
3112	if (needed) {
3113	/*
3114	* s->id will be set while adding
3115	* function to configuration so for
3116	* now just leave garbage here.
3117	*/
3118	s->s = data;
3119	--needed;
3120	++s;
3121	}
3122
3123	data += length + 1;
3124	len -= length + 1;
3125	} while (--str_per_lang);
3126
3127	s->id = 0; /* terminator */
3128	s->s = NULL;
3129	++s;
3130
3131	} while (--lang_count);
3132
3133	/* Some garbage left? */
3134	if (len)
3135	goto error_free;
3136
3137	/* Done! */
3138	ffs->stringtabs = stringtabs;
3139	ffs->raw_strings = _data;
3140
3141	return 0;
3142
3143	error_free:
3144	kfree(objp: stringtabs);
3145	error:
3146	kfree(objp: _data);
3147	return -EINVAL;
3148	}
3149
3150
3151	/* Events handling and management *******************************************/
3152
3153	static void __ffs_event_add(struct ffs_data *ffs,
3154	enum usb_functionfs_event_type type)
3155	{
3156	enum usb_functionfs_event_type rem_type1, rem_type2 = type;
3157	int neg = 0;
3158
3159	/*
3160	* Abort any unhandled setup
3161	*
3162	* We do not need to worry about some cmpxchg() changing value
3163	* of ffs->setup_state without holding the lock because when
3164	* state is FFS_SETUP_PENDING cmpxchg() in several places in
3165	* the source does nothing.
3166	*/
3167	if (ffs->setup_state == FFS_SETUP_PENDING)
3168	ffs->setup_state = FFS_SETUP_CANCELLED;
3169
3170	/*
3171	* Logic of this function guarantees that there are at most four pending
3172	* evens on ffs->ev.types queue. This is important because the queue
3173	* has space for four elements only and __ffs_ep0_read_events function
3174	* depends on that limit as well. If more event types are added, those
3175	* limits have to be revisited or guaranteed to still hold.
3176	*/
3177	switch (type) {
3178	case FUNCTIONFS_RESUME:
3179	rem_type2 = FUNCTIONFS_SUSPEND;
3180	fallthrough;
3181	case FUNCTIONFS_SUSPEND:
3182	case FUNCTIONFS_SETUP:
3183	rem_type1 = type;
3184	/* Discard all similar events */
3185	break;
3186
3187	case FUNCTIONFS_BIND:
3188	case FUNCTIONFS_UNBIND:
3189	case FUNCTIONFS_DISABLE:
3190	case FUNCTIONFS_ENABLE:
3191	/* Discard everything other then power management. */
3192	rem_type1 = FUNCTIONFS_SUSPEND;
3193	rem_type2 = FUNCTIONFS_RESUME;
3194	neg = 1;
3195	break;
3196
3197	default:
3198	WARN(1, "%d: unknown event, this should not happen\n", type);
3199	return;
3200	}
3201
3202	{
3203	u8 *ev = ffs->ev.types, *out = ev;
3204	unsigned n = ffs->ev.count;
3205	for (; n; --n, ++ev)
3206	if ((*ev == rem_type1 || *ev == rem_type2) == neg)
3207	*out++ = *ev;
3208	else
3209	pr_vdebug("purging event %d\n", *ev);
3210	ffs->ev.count = out - ffs->ev.types;
3211	}
3212
3213	pr_vdebug("adding event %d\n", type);
3214	ffs->ev.types[ffs->ev.count++] = type;
3215	wake_up_locked(&ffs->ev.waitq);
3216	if (ffs->ffs_eventfd)
3217	eventfd_signal(ctx: ffs->ffs_eventfd);
3218	}
3219
3220	static void ffs_event_add(struct ffs_data *ffs,
3221	enum usb_functionfs_event_type type)
3222	{
3223	unsigned long flags;
3224	spin_lock_irqsave(&ffs->ev.waitq.lock, flags);
3225	__ffs_event_add(ffs, type);
3226	spin_unlock_irqrestore(lock: &ffs->ev.waitq.lock, flags);
3227	}
3228
3229	/* Bind/unbind USB function hooks *******************************************/
3230
3231	static int ffs_ep_addr2idx(struct ffs_data *ffs, u8 endpoint_address)
3232	{
3233	int i;
3234
3235	for (i = 1; i < ARRAY_SIZE(ffs->eps_addrmap); ++i)
3236	if (ffs->eps_addrmap[i] == endpoint_address)
3237	return i;
3238	return -ENOENT;
3239	}
3240
3241	static int __ffs_func_bind_do_descs(enum ffs_entity_type type, u8 *valuep,
3242	struct usb_descriptor_header *desc,
3243	void *priv)
3244	{
3245	struct usb_endpoint_descriptor *ds = (void *)desc;
3246	struct ffs_function *func = priv;
3247	struct ffs_ep *ffs_ep;
3248	unsigned ep_desc_id;
3249	int idx;
3250	static const char *speed_names[] = { "full", "high", "super" };
3251
3252	if (type != FFS_DESCRIPTOR)
3253	return 0;
3254
3255	/*
3256	* If ss_descriptors is not NULL, we are reading super speed
3257	* descriptors; if hs_descriptors is not NULL, we are reading high
3258	* speed descriptors; otherwise, we are reading full speed
3259	* descriptors.
3260	*/
3261	if (func->function.ss_descriptors) {
3262	ep_desc_id = 2;
3263	func->function.ss_descriptors[(long)valuep] = desc;
3264	} else if (func->function.hs_descriptors) {
3265	ep_desc_id = 1;
3266	func->function.hs_descriptors[(long)valuep] = desc;
3267	} else {
3268	ep_desc_id = 0;
3269	func->function.fs_descriptors[(long)valuep] = desc;
3270	}
3271
3272	if (!desc || desc->bDescriptorType != USB_DT_ENDPOINT)
3273	return 0;
3274
3275	idx = ffs_ep_addr2idx(ffs: func->ffs, endpoint_address: ds->bEndpointAddress) - 1;
3276	if (idx < 0)
3277	return idx;
3278
3279	ffs_ep = func->eps + idx;
3280
3281	if (ffs_ep->descs[ep_desc_id]) {
3282	pr_err("two %sspeed descriptors for EP %d\n",
3283	speed_names[ep_desc_id],
3284	ds->bEndpointAddress & USB_ENDPOINT_NUMBER_MASK);
3285	return -EINVAL;
3286	}
3287	ffs_ep->descs[ep_desc_id] = ds;
3288
3289	ffs_dump_mem(": Original ep desc", ds, ds->bLength);
3290	if (ffs_ep->ep) {
3291	ds->bEndpointAddress = ffs_ep->descs[0]->bEndpointAddress;
3292	if (!ds->wMaxPacketSize)
3293	ds->wMaxPacketSize = ffs_ep->descs[0]->wMaxPacketSize;
3294	} else {
3295	struct usb_request *req;
3296	struct usb_ep *ep;
3297	u8 bEndpointAddress;
3298	u16 wMaxPacketSize;
3299
3300	/*
3301	* We back up bEndpointAddress because autoconfig overwrites
3302	* it with physical endpoint address.
3303	*/
3304	bEndpointAddress = ds->bEndpointAddress;
3305	/*
3306	* We back up wMaxPacketSize because autoconfig treats
3307	* endpoint descriptors as if they were full speed.
3308	*/
3309	wMaxPacketSize = ds->wMaxPacketSize;
3310	pr_vdebug("autoconfig\n");
3311	ep = usb_ep_autoconfig(func->gadget, ds);
3312	if (!ep)
3313	return -ENOTSUPP;
3314	ep->driver_data = func->eps + idx;
3315
3316	req = usb_ep_alloc_request(ep, GFP_KERNEL);
3317	if (!req)
3318	return -ENOMEM;
3319
3320	ffs_ep->ep = ep;
3321	ffs_ep->req = req;
3322	func->eps_revmap[ds->bEndpointAddress &
3323	USB_ENDPOINT_NUMBER_MASK] = idx + 1;
3324	/*
3325	* If we use virtual address mapping, we restore
3326	* original bEndpointAddress value.
3327	*/
3328	if (func->ffs->user_flags & FUNCTIONFS_VIRTUAL_ADDR)
3329	ds->bEndpointAddress = bEndpointAddress;
3330	/*
3331	* Restore wMaxPacketSize which was potentially
3332	* overwritten by autoconfig.
3333	*/
3334	ds->wMaxPacketSize = wMaxPacketSize;
3335	}
3336	ffs_dump_mem(": Rewritten ep desc", ds, ds->bLength);
3337
3338	return 0;
3339	}
3340
3341	static int __ffs_func_bind_do_nums(enum ffs_entity_type type, u8 *valuep,
3342	struct usb_descriptor_header *desc,
3343	void *priv)
3344	{
3345	struct ffs_function *func = priv;
3346	unsigned idx;
3347	u8 newValue;
3348
3349	switch (type) {
3350	default:
3351	case FFS_DESCRIPTOR:
3352	/* Handled in previous pass by __ffs_func_bind_do_descs() */
3353	return 0;
3354
3355	case FFS_INTERFACE:
3356	idx = *valuep;
3357	if (func->interfaces_nums[idx] < 0) {
3358	int id = usb_interface_id(func->conf, &func->function);
3359	if (id < 0)
3360	return id;
3361	func->interfaces_nums[idx] = id;
3362	}
3363	newValue = func->interfaces_nums[idx];
3364	break;
3365
3366	case FFS_STRING:
3367	/* String' IDs are allocated when fsf_data is bound to cdev */
3368	newValue = func->ffs->stringtabs[0]->strings[*valuep - 1].id;
3369	break;
3370
3371	case FFS_ENDPOINT:
3372	/*
3373	* USB_DT_ENDPOINT are handled in
3374	* __ffs_func_bind_do_descs().
3375	*/
3376	if (desc->bDescriptorType == USB_DT_ENDPOINT)
3377	return 0;
3378
3379	idx = (*valuep & USB_ENDPOINT_NUMBER_MASK) - 1;
3380	if (!func->eps[idx].ep)
3381	return -EINVAL;
3382
3383	{
3384	struct usb_endpoint_descriptor **descs;
3385	descs = func->eps[idx].descs;
3386	newValue = descs[descs[0] ? 0 : 1]->bEndpointAddress;
3387	}
3388	break;
3389	}
3390
3391	pr_vdebug("%02x -> %02x\n", *valuep, newValue);
3392	*valuep = newValue;
3393	return 0;
3394	}
3395
3396	static int __ffs_func_bind_do_os_desc(enum ffs_os_desc_type type,
3397	struct usb_os_desc_header *h, void *data,
3398	unsigned len, void *priv)
3399	{
3400	struct ffs_function *func = priv;
3401	u8 length = 0;
3402
3403	switch (type) {
3404	case FFS_OS_DESC_EXT_COMPAT: {
3405	struct usb_ext_compat_desc *desc = data;
3406	struct usb_os_desc_table *t;
3407
3408	t = &func->function.os_desc_table[desc->bFirstInterfaceNumber];
3409	t->if_id = func->interfaces_nums[desc->bFirstInterfaceNumber];
3410	memcpy(t->os_desc->ext_compat_id, &desc->IDs,
3411	sizeof_field(struct usb_ext_compat_desc, IDs));
3412	length = sizeof(*desc);
3413	}
3414	break;
3415	case FFS_OS_DESC_EXT_PROP: {
3416	struct usb_ext_prop_desc *desc = data;
3417	struct usb_os_desc_table *t;
3418	struct usb_os_desc_ext_prop *ext_prop;
3419	char *ext_prop_name;
3420	char *ext_prop_data;
3421
3422	t = &func->function.os_desc_table[h->interface];
3423	t->if_id = func->interfaces_nums[h->interface];
3424
3425	ext_prop = func->ffs->ms_os_descs_ext_prop_avail;
3426	func->ffs->ms_os_descs_ext_prop_avail += sizeof(*ext_prop);
3427
3428	ext_prop->type = le32_to_cpu(desc->dwPropertyDataType);
3429	ext_prop->name_len = le16_to_cpu(desc->wPropertyNameLength);
3430	ext_prop->data_len = le32_to_cpu(*(__le32 *)
3431	usb_ext_prop_data_len_ptr(data, ext_prop->name_len));
3432	length = ext_prop->name_len + ext_prop->data_len + 14;
3433
3434	ext_prop_name = func->ffs->ms_os_descs_ext_prop_name_avail;
3435	func->ffs->ms_os_descs_ext_prop_name_avail +=
3436	ext_prop->name_len;
3437
3438	ext_prop_data = func->ffs->ms_os_descs_ext_prop_data_avail;
3439	func->ffs->ms_os_descs_ext_prop_data_avail +=
3440	ext_prop->data_len;
3441	memcpy(ext_prop_data,
3442	usb_ext_prop_data_ptr(data, ext_prop->name_len),
3443	ext_prop->data_len);
3444	/* unicode data reported to the host as "WCHAR"s */
3445	switch (ext_prop->type) {
3446	case USB_EXT_PROP_UNICODE:
3447	case USB_EXT_PROP_UNICODE_ENV:
3448	case USB_EXT_PROP_UNICODE_LINK:
3449	case USB_EXT_PROP_UNICODE_MULTI:
3450	ext_prop->data_len *= 2;
3451	break;
3452	}
3453	ext_prop->data = ext_prop_data;
3454
3455	memcpy(ext_prop_name, usb_ext_prop_name_ptr(data),
3456	ext_prop->name_len);
3457	/* property name reported to the host as "WCHAR"s */
3458	ext_prop->name_len *= 2;
3459	ext_prop->name = ext_prop_name;
3460
3461	t->os_desc->ext_prop_len +=
3462	ext_prop->name_len + ext_prop->data_len + 14;
3463	++t->os_desc->ext_prop_count;
3464	list_add_tail(new: &ext_prop->entry, head: &t->os_desc->ext_prop);
3465	}
3466	break;
3467	default:
3468	pr_vdebug("unknown descriptor: %d\n", type);
3469	}
3470
3471	return length;
3472	}
3473
3474	static inline struct f_fs_opts *ffs_do_functionfs_bind(struct usb_function *f,
3475	struct usb_configuration *c)
3476	{
3477	struct ffs_function *func = ffs_func_from_usb(f);
3478	struct f_fs_opts *ffs_opts =
3479	container_of(f->fi, struct f_fs_opts, func_inst);
3480	struct ffs_data *ffs_data;
3481	int ret;
3482
3483	/*
3484	* Legacy gadget triggers binding in functionfs_ready_callback,
3485	* which already uses locking; taking the same lock here would
3486	* cause a deadlock.
3487	*
3488	* Configfs-enabled gadgets however do need ffs_dev_lock.
3489	*/
3490	if (!ffs_opts->no_configfs)
3491	ffs_dev_lock();
3492	ret = ffs_opts->dev->desc_ready ? 0 : -ENODEV;
3493	ffs_data = ffs_opts->dev->ffs_data;
3494	if (!ffs_opts->no_configfs)
3495	ffs_dev_unlock();
3496	if (ret)
3497	return ERR_PTR(error: ret);
3498
3499	func->ffs = ffs_data;
3500	func->conf = c;
3501	func->gadget = c->cdev->gadget;
3502
3503	/*
3504	* in drivers/usb/gadget/configfs.c:configfs_composite_bind()
3505	* configurations are bound in sequence with list_for_each_entry,
3506	* in each configuration its functions are bound in sequence
3507	* with list_for_each_entry, so we assume no race condition
3508	* with regard to ffs_opts->bound access
3509	*/
3510	if (!ffs_opts->refcnt) {
3511	ret = functionfs_bind(ffs: func->ffs, cdev: c->cdev);
3512	if (ret)
3513	return ERR_PTR(error: ret);
3514	}
3515	ffs_opts->refcnt++;
3516	func->function.strings = func->ffs->stringtabs;
3517
3518	return ffs_opts;
3519	}
3520
3521	static int _ffs_func_bind(struct usb_configuration *c,
3522	struct usb_function *f)
3523	{
3524	struct ffs_function *func = ffs_func_from_usb(f);
3525	struct ffs_data *ffs = func->ffs;
3526
3527	const int full = !!func->ffs->fs_descs_count;
3528	const int high = !!func->ffs->hs_descs_count;
3529	const int super = !!func->ffs->ss_descs_count;
3530
3531	int fs_len, hs_len, ss_len, ret, i;
3532	struct ffs_ep *eps_ptr;
3533
3534	/* Make it a single chunk, less management later on */
3535	vla_group(d);
3536	vla_item_with_sz(d, struct ffs_ep, eps, ffs->eps_count);
3537	vla_item_with_sz(d, struct usb_descriptor_header *, fs_descs,
3538	full ? ffs->fs_descs_count + 1 : 0);
3539	vla_item_with_sz(d, struct usb_descriptor_header *, hs_descs,
3540	high ? ffs->hs_descs_count + 1 : 0);
3541	vla_item_with_sz(d, struct usb_descriptor_header *, ss_descs,
3542	super ? ffs->ss_descs_count + 1 : 0);
3543	vla_item_with_sz(d, short, inums, ffs->interfaces_count);
3544	vla_item_with_sz(d, struct usb_os_desc_table, os_desc_table,
3545	c->cdev->use_os_string ? ffs->interfaces_count : 0);
3546	vla_item_with_sz(d, char[16], ext_compat,
3547	c->cdev->use_os_string ? ffs->interfaces_count : 0);
3548	vla_item_with_sz(d, struct usb_os_desc, os_desc,
3549	c->cdev->use_os_string ? ffs->interfaces_count : 0);
3550	vla_item_with_sz(d, struct usb_os_desc_ext_prop, ext_prop,
3551	ffs->ms_os_descs_ext_prop_count);
3552	vla_item_with_sz(d, char, ext_prop_name,
3553	ffs->ms_os_descs_ext_prop_name_len);
3554	vla_item_with_sz(d, char, ext_prop_data,
3555	ffs->ms_os_descs_ext_prop_data_len);
3556	vla_item_with_sz(d, char, raw_descs, ffs->raw_descs_length);
3557	char *vlabuf;
3558
3559	/* Has descriptors only for speeds gadget does not support */
3560	if (!(full | high | super))
3561	return -ENOTSUPP;
3562
3563	/* Allocate a single chunk, less management later on */
3564	vlabuf = kzalloc(vla_group_size(d), GFP_KERNEL);
3565	if (!vlabuf)
3566	return -ENOMEM;
3567
3568	ffs->ms_os_descs_ext_prop_avail = vla_ptr(vlabuf, d, ext_prop);
3569	ffs->ms_os_descs_ext_prop_name_avail =
3570	vla_ptr(vlabuf, d, ext_prop_name);
3571	ffs->ms_os_descs_ext_prop_data_avail =
3572	vla_ptr(vlabuf, d, ext_prop_data);
3573
3574	/* Copy descriptors */
3575	memcpy(vla_ptr(vlabuf, d, raw_descs), ffs->raw_descs,
3576	ffs->raw_descs_length);
3577
3578	memset(vla_ptr(vlabuf, d, inums), 0xff, d_inums__sz);
3579	eps_ptr = vla_ptr(vlabuf, d, eps);
3580	for (i = 0; i < ffs->eps_count; i++)
3581	eps_ptr[i].num = -1;
3582
3583	/* Save pointers
3584	* d_eps == vlabuf, func->eps used to kfree vlabuf later
3585	*/
3586	func->eps = vla_ptr(vlabuf, d, eps);
3587	func->interfaces_nums = vla_ptr(vlabuf, d, inums);
3588
3589	/*
3590	* Go through all the endpoint descriptors and allocate
3591	* endpoints first, so that later we can rewrite the endpoint
3592	* numbers without worrying that it may be described later on.
3593	*/
3594	if (full) {
3595	func->function.fs_descriptors = vla_ptr(vlabuf, d, fs_descs);
3596	fs_len = ffs_do_descs(count: ffs->fs_descs_count,
3597	vla_ptr(vlabuf, d, raw_descs),
3598	len: d_raw_descs__sz,
3599	entity: __ffs_func_bind_do_descs, priv: func);
3600	if (fs_len < 0) {
3601	ret = fs_len;
3602	goto error;
3603	}
3604	} else {
3605	fs_len = 0;
3606	}
3607
3608	if (high) {
3609	func->function.hs_descriptors = vla_ptr(vlabuf, d, hs_descs);
3610	hs_len = ffs_do_descs(count: ffs->hs_descs_count,
3611	vla_ptr(vlabuf, d, raw_descs) + fs_len,
3612	len: d_raw_descs__sz - fs_len,
3613	entity: __ffs_func_bind_do_descs, priv: func);
3614	if (hs_len < 0) {
3615	ret = hs_len;
3616	goto error;
3617	}
3618	} else {
3619	hs_len = 0;
3620	}
3621
3622	if (super) {
3623	func->function.ss_descriptors = func->function.ssp_descriptors =
3624	vla_ptr(vlabuf, d, ss_descs);
3625	ss_len = ffs_do_descs(count: ffs->ss_descs_count,
3626	vla_ptr(vlabuf, d, raw_descs) + fs_len + hs_len,
3627	len: d_raw_descs__sz - fs_len - hs_len,
3628	entity: __ffs_func_bind_do_descs, priv: func);
3629	if (ss_len < 0) {
3630	ret = ss_len;
3631	goto error;
3632	}
3633	} else {
3634	ss_len = 0;
3635	}
3636
3637	/*
3638	* Now handle interface numbers allocation and interface and
3639	* endpoint numbers rewriting. We can do that in one go
3640	* now.
3641	*/
3642	ret = ffs_do_descs(count: ffs->fs_descs_count +
3643	(high ? ffs->hs_descs_count : 0) +
3644	(super ? ffs->ss_descs_count : 0),
3645	vla_ptr(vlabuf, d, raw_descs), len: d_raw_descs__sz,
3646	entity: __ffs_func_bind_do_nums, priv: func);
3647	if (ret < 0)
3648	goto error;
3649
3650	func->function.os_desc_table = vla_ptr(vlabuf, d, os_desc_table);
3651	if (c->cdev->use_os_string) {
3652	for (i = 0; i < ffs->interfaces_count; ++i) {
3653	struct usb_os_desc *desc;
3654
3655	desc = func->function.os_desc_table[i].os_desc =
3656	vla_ptr(vlabuf, d, os_desc) +
3657	i * sizeof(struct usb_os_desc);
3658	desc->ext_compat_id =
3659	vla_ptr(vlabuf, d, ext_compat) + i * 16;
3660	INIT_LIST_HEAD(list: &desc->ext_prop);
3661	}
3662	ret = ffs_do_os_descs(count: ffs->ms_os_descs_count,
3663	vla_ptr(vlabuf, d, raw_descs) +
3664	fs_len + hs_len + ss_len,
3665	len: d_raw_descs__sz - fs_len - hs_len -
3666	ss_len,
3667	entity: __ffs_func_bind_do_os_desc, priv: func);
3668	if (ret < 0)
3669	goto error;
3670	}
3671	func->function.os_desc_n =
3672	c->cdev->use_os_string ? ffs->interfaces_count : 0;
3673
3674	/* And we're done */
3675	ffs_event_add(ffs, type: FUNCTIONFS_BIND);
3676	return 0;
3677
3678	error:
3679	/* XXX Do we need to release all claimed endpoints here? */
3680	return ret;
3681	}
3682
3683	static int ffs_func_bind(struct usb_configuration *c,
3684	struct usb_function *f)
3685	{
3686	struct f_fs_opts *ffs_opts = ffs_do_functionfs_bind(f, c);
3687	struct ffs_function *func = ffs_func_from_usb(f);
3688	int ret;
3689
3690	if (IS_ERR(ptr: ffs_opts))
3691	return PTR_ERR(ptr: ffs_opts);
3692
3693	ret = _ffs_func_bind(c, f);
3694	if (ret && !--ffs_opts->refcnt)
3695	functionfs_unbind(ffs: func->ffs);
3696
3697	return ret;
3698	}
3699
3700
3701	/* Other USB function hooks *************************************************/
3702
3703	static void ffs_reset_work(struct work_struct *work)
3704	{
3705	struct ffs_data *ffs = container_of(work,
3706	struct ffs_data, reset_work);
3707	ffs_data_reset(ffs);
3708	}
3709
3710	static int ffs_func_set_alt(struct usb_function *f,
3711	unsigned interface, unsigned alt)
3712	{
3713	struct ffs_function *func = ffs_func_from_usb(f);
3714	struct ffs_data *ffs = func->ffs;
3715	int ret = 0, intf;
3716
3717	if (alt != (unsigned)-1) {
3718	intf = ffs_func_revmap_intf(func, intf: interface);
3719	if (intf < 0)
3720	return intf;
3721	}
3722
3723	if (ffs->func)
3724	ffs_func_eps_disable(func: ffs->func);
3725
3726	if (ffs->state == FFS_DEACTIVATED) {
3727	ffs->state = FFS_CLOSING;
3728	INIT_WORK(&ffs->reset_work, ffs_reset_work);
3729	schedule_work(work: &ffs->reset_work);
3730	return -ENODEV;
3731	}
3732
3733	if (ffs->state != FFS_ACTIVE)
3734	return -ENODEV;
3735
3736	if (alt == (unsigned)-1) {
3737	ffs->func = NULL;
3738	ffs_event_add(ffs, type: FUNCTIONFS_DISABLE);
3739	return 0;
3740	}
3741
3742	ffs->func = func;
3743	ret = ffs_func_eps_enable(func);
3744	if (ret >= 0)
3745	ffs_event_add(ffs, type: FUNCTIONFS_ENABLE);
3746	return ret;
3747	}
3748
3749	static void ffs_func_disable(struct usb_function *f)
3750	{
3751	ffs_func_set_alt(f, interface: 0, alt: (unsigned)-1);
3752	}
3753
3754	static int ffs_func_setup(struct usb_function *f,
3755	const struct usb_ctrlrequest *creq)
3756	{
3757	struct ffs_function *func = ffs_func_from_usb(f);
3758	struct ffs_data *ffs = func->ffs;
3759	unsigned long flags;
3760	int ret;
3761
3762	pr_vdebug("creq->bRequestType = %02x\n", creq->bRequestType);
3763	pr_vdebug("creq->bRequest = %02x\n", creq->bRequest);
3764	pr_vdebug("creq->wValue = %04x\n", le16_to_cpu(creq->wValue));
3765	pr_vdebug("creq->wIndex = %04x\n", le16_to_cpu(creq->wIndex));
3766	pr_vdebug("creq->wLength = %04x\n", le16_to_cpu(creq->wLength));
3767
3768	/*
3769	* Most requests directed to interface go through here
3770	* (notable exceptions are set/get interface) so we need to
3771	* handle them. All other either handled by composite or
3772	* passed to usb_configuration->setup() (if one is set). No
3773	* matter, we will handle requests directed to endpoint here
3774	* as well (as it's straightforward). Other request recipient
3775	* types are only handled when the user flag FUNCTIONFS_ALL_CTRL_RECIP
3776	* is being used.
3777	*/
3778	if (ffs->state != FFS_ACTIVE)
3779	return -ENODEV;
3780
3781	switch (creq->bRequestType & USB_RECIP_MASK) {
3782	case USB_RECIP_INTERFACE:
3783	ret = ffs_func_revmap_intf(func, le16_to_cpu(creq->wIndex));
3784	if (ret < 0)
3785	return ret;
3786	break;
3787
3788	case USB_RECIP_ENDPOINT:
3789	ret = ffs_func_revmap_ep(func, le16_to_cpu(creq->wIndex));
3790	if (ret < 0)
3791	return ret;
3792	if (func->ffs->user_flags & FUNCTIONFS_VIRTUAL_ADDR)
3793	ret = func->ffs->eps_addrmap[ret];
3794	break;
3795
3796	default:
3797	if (func->ffs->user_flags & FUNCTIONFS_ALL_CTRL_RECIP)
3798	ret = le16_to_cpu(creq->wIndex);
3799	else
3800	return -EOPNOTSUPP;
3801	}
3802
3803	spin_lock_irqsave(&ffs->ev.waitq.lock, flags);
3804	ffs->ev.setup = *creq;
3805	ffs->ev.setup.wIndex = cpu_to_le16(ret);
3806	__ffs_event_add(ffs, type: FUNCTIONFS_SETUP);
3807	spin_unlock_irqrestore(lock: &ffs->ev.waitq.lock, flags);
3808
3809	return creq->wLength == 0 ? USB_GADGET_DELAYED_STATUS : 0;
3810	}
3811
3812	static bool ffs_func_req_match(struct usb_function *f,
3813	const struct usb_ctrlrequest *creq,
3814	bool config0)
3815	{
3816	struct ffs_function *func = ffs_func_from_usb(f);
3817
3818	if (config0 && !(func->ffs->user_flags & FUNCTIONFS_CONFIG0_SETUP))
3819	return false;
3820
3821	switch (creq->bRequestType & USB_RECIP_MASK) {
3822	case USB_RECIP_INTERFACE:
3823	return (ffs_func_revmap_intf(func,
3824	le16_to_cpu(creq->wIndex)) >= 0);
3825	case USB_RECIP_ENDPOINT:
3826	return (ffs_func_revmap_ep(func,
3827	le16_to_cpu(creq->wIndex)) >= 0);
3828	default:
3829	return (bool) (func->ffs->user_flags &
3830	FUNCTIONFS_ALL_CTRL_RECIP);
3831	}
3832	}
3833
3834	static void ffs_func_suspend(struct usb_function *f)
3835	{
3836	ffs_event_add(ffs: ffs_func_from_usb(f)->ffs, type: FUNCTIONFS_SUSPEND);
3837	}
3838
3839	static void ffs_func_resume(struct usb_function *f)
3840	{
3841	ffs_event_add(ffs: ffs_func_from_usb(f)->ffs, type: FUNCTIONFS_RESUME);
3842	}
3843
3844
3845	/* Endpoint and interface numbers reverse mapping ***************************/
3846
3847	static int ffs_func_revmap_ep(struct ffs_function *func, u8 num)
3848	{
3849	num = func->eps_revmap[num & USB_ENDPOINT_NUMBER_MASK];
3850	return num ? num : -EDOM;
3851	}
3852
3853	static int ffs_func_revmap_intf(struct ffs_function *func, u8 intf)
3854	{
3855	short *nums = func->interfaces_nums;
3856	unsigned count = func->ffs->interfaces_count;
3857
3858	for (; count; --count, ++nums) {
3859	if (*nums >= 0 && *nums == intf)
3860	return nums - func->interfaces_nums;
3861	}
3862
3863	return -EDOM;
3864	}
3865
3866
3867	/* Devices management *******************************************************/
3868
3869	static LIST_HEAD(ffs_devices);
3870
3871	static struct ffs_dev *_ffs_do_find_dev(const char *name)
3872	{
3873	struct ffs_dev *dev;
3874
3875	if (!name)
3876	return NULL;
3877
3878	list_for_each_entry(dev, &ffs_devices, entry) {
3879	if (strcmp(dev->name, name) == 0)
3880	return dev;
3881	}
3882
3883	return NULL;
3884	}
3885
3886	/*
3887	* ffs_lock must be taken by the caller of this function
3888	*/
3889	static struct ffs_dev *_ffs_get_single_dev(void)
3890	{
3891	struct ffs_dev *dev;
3892
3893	if (list_is_singular(head: &ffs_devices)) {
3894	dev = list_first_entry(&ffs_devices, struct ffs_dev, entry);
3895	if (dev->single)
3896	return dev;
3897	}
3898
3899	return NULL;
3900	}
3901
3902	/*
3903	* ffs_lock must be taken by the caller of this function
3904	*/
3905	static struct ffs_dev *_ffs_find_dev(const char *name)
3906	{
3907	struct ffs_dev *dev;
3908
3909	dev = _ffs_get_single_dev();
3910	if (dev)
3911	return dev;
3912
3913	return _ffs_do_find_dev(name);
3914	}
3915
3916	/* Configfs support *********************************************************/
3917
3918	static inline struct f_fs_opts *to_ffs_opts(struct config_item *item)
3919	{
3920	return container_of(to_config_group(item), struct f_fs_opts,
3921	func_inst.group);
3922	}
3923
3924	static ssize_t f_fs_opts_ready_show(struct config_item *item, char *page)
3925	{
3926	struct f_fs_opts *opts = to_ffs_opts(item);
3927	int ready;
3928
3929	ffs_dev_lock();
3930	ready = opts->dev->desc_ready;
3931	ffs_dev_unlock();
3932
3933	return sprintf(buf: page, fmt: "%d\n", ready);
3934	}
3935
3936	CONFIGFS_ATTR_RO(f_fs_opts_, ready);
3937
3938	static struct configfs_attribute *ffs_attrs[] = {
3939	&f_fs_opts_attr_ready,
3940	NULL,
3941	};
3942
3943	static void ffs_attr_release(struct config_item *item)
3944	{
3945	struct f_fs_opts *opts = to_ffs_opts(item);
3946
3947	usb_put_function_instance(fi: &opts->func_inst);
3948	}
3949
3950	static struct configfs_item_operations ffs_item_ops = {
3951	.release = ffs_attr_release,
3952	};
3953
3954	static const struct config_item_type ffs_func_type = {
3955	.ct_item_ops = &ffs_item_ops,
3956	.ct_attrs = ffs_attrs,
3957	.ct_owner = THIS_MODULE,
3958	};
3959
3960
3961	/* Function registration interface ******************************************/
3962
3963	static void ffs_free_inst(struct usb_function_instance *f)
3964	{
3965	struct f_fs_opts *opts;
3966
3967	opts = to_f_fs_opts(fi: f);
3968	ffs_release_dev(ffs_dev: opts->dev);
3969	ffs_dev_lock();
3970	_ffs_free_dev(dev: opts->dev);
3971	ffs_dev_unlock();
3972	kfree(objp: opts);
3973	}
3974
3975	static int ffs_set_inst_name(struct usb_function_instance *fi, const char *name)
3976	{
3977	if (strlen(name) >= sizeof_field(struct ffs_dev, name))
3978	return -ENAMETOOLONG;
3979	return ffs_name_dev(dev: to_f_fs_opts(fi)->dev, name);
3980	}
3981
3982	static struct usb_function_instance *ffs_alloc_inst(void)
3983	{
3984	struct f_fs_opts *opts;
3985	struct ffs_dev *dev;
3986
3987	opts = kzalloc(size: sizeof(*opts), GFP_KERNEL);
3988	if (!opts)
3989	return ERR_PTR(error: -ENOMEM);
3990
3991	opts->func_inst.set_inst_name = ffs_set_inst_name;
3992	opts->func_inst.free_func_inst = ffs_free_inst;
3993	ffs_dev_lock();
3994	dev = _ffs_alloc_dev();
3995	ffs_dev_unlock();
3996	if (IS_ERR(ptr: dev)) {
3997	kfree(objp: opts);
3998	return ERR_CAST(ptr: dev);
3999	}
4000	opts->dev = dev;
4001	dev->opts = opts;
4002
4003	config_group_init_type_name(group: &opts->func_inst.group, name: "",
4004	type: &ffs_func_type);
4005	return &opts->func_inst;
4006	}
4007
4008	static void ffs_free(struct usb_function *f)
4009	{
4010	kfree(objp: ffs_func_from_usb(f));
4011	}
4012
4013	static void ffs_func_unbind(struct usb_configuration *c,
4014	struct usb_function *f)
4015	{
4016	struct ffs_function *func = ffs_func_from_usb(f);
4017	struct ffs_data *ffs = func->ffs;
4018	struct f_fs_opts *opts =
4019	container_of(f->fi, struct f_fs_opts, func_inst);
4020	struct ffs_ep *ep = func->eps;
4021	unsigned count = ffs->eps_count;
4022	unsigned long flags;
4023
4024	if (ffs->func == func) {
4025	ffs_func_eps_disable(func);
4026	ffs->func = NULL;
4027	}
4028
4029	/* Drain any pending AIO completions */
4030	drain_workqueue(wq: ffs->io_completion_wq);
4031
4032	ffs_event_add(ffs, type: FUNCTIONFS_UNBIND);
4033	if (!--opts->refcnt)
4034	functionfs_unbind(ffs);
4035
4036	/* cleanup after autoconfig */
4037	spin_lock_irqsave(&func->ffs->eps_lock, flags);
4038	while (count--) {
4039	if (ep->ep && ep->req)
4040	usb_ep_free_request(ep: ep->ep, req: ep->req);
4041	ep->req = NULL;
4042	++ep;
4043	}
4044	spin_unlock_irqrestore(lock: &func->ffs->eps_lock, flags);
4045	kfree(objp: func->eps);
4046	func->eps = NULL;
4047	/*
4048	* eps, descriptors and interfaces_nums are allocated in the
4049	* same chunk so only one free is required.
4050	*/
4051	func->function.fs_descriptors = NULL;
4052	func->function.hs_descriptors = NULL;
4053	func->function.ss_descriptors = NULL;
4054	func->function.ssp_descriptors = NULL;
4055	func->interfaces_nums = NULL;
4056
4057	}
4058
4059	static struct usb_function *ffs_alloc(struct usb_function_instance *fi)
4060	{
4061	struct ffs_function *func;
4062
4063	func = kzalloc(size: sizeof(*func), GFP_KERNEL);
4064	if (!func)
4065	return ERR_PTR(error: -ENOMEM);
4066
4067	func->function.name = "Function FS Gadget";
4068
4069	func->function.bind = ffs_func_bind;
4070	func->function.unbind = ffs_func_unbind;
4071	func->function.set_alt = ffs_func_set_alt;
4072	func->function.disable = ffs_func_disable;
4073	func->function.setup = ffs_func_setup;
4074	func->function.req_match = ffs_func_req_match;
4075	func->function.suspend = ffs_func_suspend;
4076	func->function.resume = ffs_func_resume;
4077	func->function.free_func = ffs_free;
4078
4079	return &func->function;
4080	}
4081
4082	/*
4083	* ffs_lock must be taken by the caller of this function
4084	*/
4085	static struct ffs_dev *_ffs_alloc_dev(void)
4086	{
4087	struct ffs_dev *dev;
4088	int ret;
4089
4090	if (_ffs_get_single_dev())
4091	return ERR_PTR(error: -EBUSY);
4092
4093	dev = kzalloc(size: sizeof(*dev), GFP_KERNEL);
4094	if (!dev)
4095	return ERR_PTR(error: -ENOMEM);
4096
4097	if (list_empty(head: &ffs_devices)) {
4098	ret = functionfs_init();
4099	if (ret) {
4100	kfree(objp: dev);
4101	return ERR_PTR(error: ret);
4102	}
4103	}
4104
4105	list_add(new: &dev->entry, head: &ffs_devices);
4106
4107	return dev;
4108	}
4109
4110	int ffs_name_dev(struct ffs_dev *dev, const char *name)
4111	{
4112	struct ffs_dev *existing;
4113	int ret = 0;
4114
4115	ffs_dev_lock();
4116
4117	existing = _ffs_do_find_dev(name);
4118	if (!existing)
4119	strscpy(dev->name, name, ARRAY_SIZE(dev->name));
4120	else if (existing != dev)
4121	ret = -EBUSY;
4122
4123	ffs_dev_unlock();
4124
4125	return ret;
4126	}
4127	EXPORT_SYMBOL_GPL(ffs_name_dev);
4128
4129	int ffs_single_dev(struct ffs_dev *dev)
4130	{
4131	int ret;
4132
4133	ret = 0;
4134	ffs_dev_lock();
4135
4136	if (!list_is_singular(head: &ffs_devices))
4137	ret = -EBUSY;
4138	else
4139	dev->single = true;
4140
4141	ffs_dev_unlock();
4142	return ret;
4143	}
4144	EXPORT_SYMBOL_GPL(ffs_single_dev);
4145
4146	/*
4147	* ffs_lock must be taken by the caller of this function
4148	*/
4149	static void _ffs_free_dev(struct ffs_dev *dev)
4150	{
4151	list_del(entry: &dev->entry);
4152
4153	kfree(objp: dev);
4154	if (list_empty(head: &ffs_devices))
4155	functionfs_cleanup();
4156	}
4157
4158	static int ffs_acquire_dev(const char *dev_name, struct ffs_data *ffs_data)
4159	{
4160	int ret = 0;
4161	struct ffs_dev *ffs_dev;
4162
4163	ffs_dev_lock();
4164
4165	ffs_dev = _ffs_find_dev(name: dev_name);
4166	if (!ffs_dev) {
4167	ret = -ENOENT;
4168	} else if (ffs_dev->mounted) {
4169	ret = -EBUSY;
4170	} else if (ffs_dev->ffs_acquire_dev_callback &&
4171	ffs_dev->ffs_acquire_dev_callback(ffs_dev)) {
4172	ret = -ENOENT;
4173	} else {
4174	ffs_dev->mounted = true;
4175	ffs_dev->ffs_data = ffs_data;
4176	ffs_data->private_data = ffs_dev;
4177	}
4178
4179	ffs_dev_unlock();
4180	return ret;
4181	}
4182
4183	static void ffs_release_dev(struct ffs_dev *ffs_dev)
4184	{
4185	ffs_dev_lock();
4186
4187	if (ffs_dev && ffs_dev->mounted) {
4188	ffs_dev->mounted = false;
4189	if (ffs_dev->ffs_data) {
4190	ffs_dev->ffs_data->private_data = NULL;
4191	ffs_dev->ffs_data = NULL;
4192	}
4193
4194	if (ffs_dev->ffs_release_dev_callback)
4195	ffs_dev->ffs_release_dev_callback(ffs_dev);
4196	}
4197
4198	ffs_dev_unlock();
4199	}
4200
4201	static int ffs_ready(struct ffs_data *ffs)
4202	{
4203	struct ffs_dev *ffs_obj;
4204	int ret = 0;
4205
4206	ffs_dev_lock();
4207
4208	ffs_obj = ffs->private_data;
4209	if (!ffs_obj) {
4210	ret = -EINVAL;
4211	goto done;
4212	}
4213	if (WARN_ON(ffs_obj->desc_ready)) {
4214	ret = -EBUSY;
4215	goto done;
4216	}
4217
4218	ffs_obj->desc_ready = true;
4219
4220	if (ffs_obj->ffs_ready_callback) {
4221	ret = ffs_obj->ffs_ready_callback(ffs);
4222	if (ret)
4223	goto done;
4224	}
4225
4226	set_bit(FFS_FL_CALL_CLOSED_CALLBACK, addr: &ffs->flags);
4227	done:
4228	ffs_dev_unlock();
4229	return ret;
4230	}
4231
4232	static void ffs_closed(struct ffs_data *ffs)
4233	{
4234	struct ffs_dev *ffs_obj;
4235	struct f_fs_opts *opts;
4236	struct config_item *ci;
4237
4238	ffs_dev_lock();
4239
4240	ffs_obj = ffs->private_data;
4241	if (!ffs_obj)
4242	goto done;
4243
4244	ffs_obj->desc_ready = false;
4245
4246	if (test_and_clear_bit(FFS_FL_CALL_CLOSED_CALLBACK, addr: &ffs->flags) &&
4247	ffs_obj->ffs_closed_callback)
4248	ffs_obj->ffs_closed_callback(ffs);
4249
4250	if (ffs_obj->opts)
4251	opts = ffs_obj->opts;
4252	else
4253	goto done;
4254
4255	if (opts->no_configfs || !opts->func_inst.group.cg_item.ci_parent
4256	|| !kref_read(kref: &opts->func_inst.group.cg_item.ci_kref))
4257	goto done;
4258
4259	ci = opts->func_inst.group.cg_item.ci_parent->ci_parent;
4260	ffs_dev_unlock();
4261
4262	if (test_bit(FFS_FL_BOUND, &ffs->flags))
4263	unregister_gadget_item(item: ci);
4264	return;
4265	done:
4266	ffs_dev_unlock();
4267	}
4268
4269	/* Misc helper functions ****************************************************/
4270
4271	static int ffs_mutex_lock(struct mutex *mutex, unsigned nonblock)
4272	{
4273	return nonblock
4274	? mutex_trylock(lock: mutex) ? 0 : -EAGAIN
4275	: mutex_lock_interruptible(mutex);
4276	}
4277
4278	static char *ffs_prepare_buffer(const char __user *buf, size_t len)
4279	{
4280	char *data;
4281
4282	if (!len)
4283	return NULL;
4284
4285	data = memdup_user(buf, len);
4286	if (IS_ERR(ptr: data))
4287	return data;
4288
4289	pr_vdebug("Buffer from user space:\n");
4290	ffs_dump_mem("", data, len);
4291
4292	return data;
4293	}
4294
4295	DECLARE_USB_FUNCTION_INIT(ffs, ffs_alloc_inst, ffs_alloc);
4296	MODULE_LICENSE("GPL");
4297	MODULE_AUTHOR("Michal Nazarewicz");
4298