1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright (C) 2006, 2007, 2009 Rusty Russell, IBM Corporation
4	* Copyright (C) 2009, 2010, 2011 Red Hat, Inc.
5	* Copyright (C) 2009, 2010, 2011 Amit Shah <amit.shah@redhat.com>
6	*/
7	#include <linux/cdev.h>
8	#include <linux/debugfs.h>
9	#include <linux/completion.h>
10	#include <linux/device.h>
11	#include <linux/err.h>
12	#include <linux/freezer.h>
13	#include <linux/fs.h>
14	#include <linux/splice.h>
15	#include <linux/pagemap.h>
16	#include <linux/idr.h>
17	#include <linux/init.h>
18	#include <linux/list.h>
19	#include <linux/poll.h>
20	#include <linux/sched.h>
21	#include <linux/slab.h>
22	#include <linux/spinlock.h>
23	#include <linux/virtio.h>
24	#include <linux/virtio_console.h>
25	#include <linux/wait.h>
26	#include <linux/workqueue.h>
27	#include <linux/module.h>
28	#include <linux/dma-mapping.h>
29	#include "../tty/hvc/hvc_console.h"
30
31	#define is_rproc_enabled IS_ENABLED(CONFIG_REMOTEPROC)
32	#define VIRTCONS_MAX_PORTS 0x8000
33
34	/*
35	* This is a global struct for storing common data for all the devices
36	* this driver handles.
37	*
38	* Mainly, it has a linked list for all the consoles in one place so
39	* that callbacks from hvc for get_chars(), put_chars() work properly
40	* across multiple devices and multiple ports per device.
41	*/
42	struct ports_driver_data {
43	/* Used for exporting per-port information to debugfs */
44	struct dentry *debugfs_dir;
45
46	/* List of all the devices we're handling */
47	struct list_head portdevs;
48
49	/* All the console devices handled by this driver */
50	struct list_head consoles;
51	};
52
53	static struct ports_driver_data pdrvdata;
54
55	static const struct class port_class = {
56	.name = "virtio-ports",
57	};
58
59	static DEFINE_SPINLOCK(pdrvdata_lock);
60	static DECLARE_COMPLETION(early_console_added);
61
62	/* This struct holds information that's relevant only for console ports */
63	struct console {
64	/* We'll place all consoles in a list in the pdrvdata struct */
65	struct list_head list;
66
67	/* The hvc device associated with this console port */
68	struct hvc_struct *hvc;
69
70	/* The size of the console */
71	struct winsize ws;
72
73	/*
74	* This number identifies the number that we used to register
75	* with hvc in hvc_instantiate() and hvc_alloc(); this is the
76	* number passed on by the hvc callbacks to us to
77	* differentiate between the other console ports handled by
78	* this driver
79	*/
80	u32 vtermno;
81	};
82
83	static DEFINE_IDA(vtermno_ida);
84
85	struct port_buffer {
86	char *buf;
87
88	/* size of the buffer in *buf above */
89	size_t size;
90
91	/* used length of the buffer */
92	size_t len;
93	/* offset in the buf from which to consume data */
94	size_t offset;
95
96	/* DMA address of buffer */
97	dma_addr_t dma;
98
99	/* Device we got DMA memory from */
100	struct device *dev;
101
102	/* List of pending dma buffers to free */
103	struct list_head list;
104
105	/* If sgpages == 0 then buf is used */
106	unsigned int sgpages;
107
108	/* sg is used if spages > 0. sg must be the last in is struct */
109	struct scatterlist sg[] __counted_by(sgpages);
110	};
111
112	/*
113	* This is a per-device struct that stores data common to all the
114	* ports for that device (vdev->priv).
115	*/
116	struct ports_device {
117	/* Next portdev in the list, head is in the pdrvdata struct */
118	struct list_head list;
119
120	/*
121	* Workqueue handlers where we process deferred work after
122	* notification
123	*/
124	struct work_struct control_work;
125	struct work_struct config_work;
126
127	struct list_head ports;
128
129	/* To protect the list of ports */
130	spinlock_t ports_lock;
131
132	/* To protect the vq operations for the control channel */
133	spinlock_t c_ivq_lock;
134	spinlock_t c_ovq_lock;
135
136	/* max. number of ports this device can hold */
137	u32 max_nr_ports;
138
139	/* The virtio device we're associated with */
140	struct virtio_device *vdev;
141
142	/*
143	* A couple of virtqueues for the control channel: one for
144	* guest->host transfers, one for host->guest transfers
145	*/
146	struct virtqueue *c_ivq, *c_ovq;
147
148	/*
149	* A control packet buffer for guest->host requests, protected
150	* by c_ovq_lock.
151	*/
152	struct virtio_console_control cpkt;
153
154	/* Array of per-port IO virtqueues */
155	struct virtqueue **in_vqs, **out_vqs;
156
157	/* Major number for this device. Ports will be created as minors. */
158	int chr_major;
159	};
160
161	struct port_stats {
162	unsigned long bytes_sent, bytes_received, bytes_discarded;
163	};
164
165	/* This struct holds the per-port data */
166	struct port {
167	/* Next port in the list, head is in the ports_device */
168	struct list_head list;
169
170	/* Pointer to the parent virtio_console device */
171	struct ports_device *portdev;
172
173	/* The current buffer from which data has to be fed to readers */
174	struct port_buffer *inbuf;
175
176	/*
177	* To protect the operations on the in_vq associated with this
178	* port. Has to be a spinlock because it can be called from
179	* interrupt context (get_char()).
180	*/
181	spinlock_t inbuf_lock;
182
183	/* Protect the operations on the out_vq. */
184	spinlock_t outvq_lock;
185
186	/* The IO vqs for this port */
187	struct virtqueue *in_vq, *out_vq;
188
189	/* File in the debugfs directory that exposes this port's information */
190	struct dentry *debugfs_file;
191
192	/*
193	* Keep count of the bytes sent, received and discarded for
194	* this port for accounting and debugging purposes. These
195	* counts are not reset across port open / close events.
196	*/
197	struct port_stats stats;
198
199	/*
200	* The entries in this struct will be valid if this port is
201	* hooked up to an hvc console
202	*/
203	struct console cons;
204
205	/* Each port associates with a separate char device */
206	struct cdev *cdev;
207	struct device *dev;
208
209	/* Reference-counting to handle port hot-unplugs and file operations */
210	struct kref kref;
211
212	/* A waitqueue for poll() or blocking read operations */
213	wait_queue_head_t waitqueue;
214
215	/* The 'name' of the port that we expose via sysfs properties */
216	char *name;
217
218	/* We can notify apps of host connect / disconnect events via SIGIO */
219	struct fasync_struct *async_queue;
220
221	/* The 'id' to identify the port with the Host */
222	u32 id;
223
224	bool outvq_full;
225
226	/* Is the host device open */
227	bool host_connected;
228
229	/* We should allow only one process to open a port */
230	bool guest_connected;
231	};
232
233	static struct port *find_port_by_vtermno(u32 vtermno)
234	{
235	struct port *port;
236	struct console *cons;
237	unsigned long flags;
238
239	spin_lock_irqsave(&pdrvdata_lock, flags);
240	list_for_each_entry(cons, &pdrvdata.consoles, list) {
241	if (cons->vtermno == vtermno) {
242	port = container_of(cons, struct port, cons);
243	goto out;
244	}
245	}
246	port = NULL;
247	out:
248	spin_unlock_irqrestore(lock: &pdrvdata_lock, flags);
249	return port;
250	}
251
252	static struct port *find_port_by_devt_in_portdev(struct ports_device *portdev,
253	dev_t dev)
254	{
255	struct port *port;
256	unsigned long flags;
257
258	spin_lock_irqsave(&portdev->ports_lock, flags);
259	list_for_each_entry(port, &portdev->ports, list) {
260	if (port->cdev->dev == dev) {
261	kref_get(kref: &port->kref);
262	goto out;
263	}
264	}
265	port = NULL;
266	out:
267	spin_unlock_irqrestore(lock: &portdev->ports_lock, flags);
268
269	return port;
270	}
271
272	static struct port *find_port_by_devt(dev_t dev)
273	{
274	struct ports_device *portdev;
275	struct port *port;
276	unsigned long flags;
277
278	spin_lock_irqsave(&pdrvdata_lock, flags);
279	list_for_each_entry(portdev, &pdrvdata.portdevs, list) {
280	port = find_port_by_devt_in_portdev(portdev, dev);
281	if (port)
282	goto out;
283	}
284	port = NULL;
285	out:
286	spin_unlock_irqrestore(lock: &pdrvdata_lock, flags);
287	return port;
288	}
289
290	static struct port *find_port_by_id(struct ports_device *portdev, u32 id)
291	{
292	struct port *port;
293	unsigned long flags;
294
295	spin_lock_irqsave(&portdev->ports_lock, flags);
296	list_for_each_entry(port, &portdev->ports, list)
297	if (port->id == id)
298	goto out;
299	port = NULL;
300	out:
301	spin_unlock_irqrestore(lock: &portdev->ports_lock, flags);
302
303	return port;
304	}
305
306	static struct port *find_port_by_vq(struct ports_device *portdev,
307	struct virtqueue *vq)
308	{
309	struct port *port;
310	unsigned long flags;
311
312	spin_lock_irqsave(&portdev->ports_lock, flags);
313	list_for_each_entry(port, &portdev->ports, list)
314	if (port->in_vq == vq || port->out_vq == vq)
315	goto out;
316	port = NULL;
317	out:
318	spin_unlock_irqrestore(lock: &portdev->ports_lock, flags);
319	return port;
320	}
321
322	static bool is_console_port(struct port *port)
323	{
324	if (port->cons.hvc)
325	return true;
326	return false;
327	}
328
329	static bool is_rproc_serial(const struct virtio_device *vdev)
330	{
331	return is_rproc_enabled && vdev->id.device == VIRTIO_ID_RPROC_SERIAL;
332	}
333
334	static inline bool use_multiport(struct ports_device *portdev)
335	{
336	/*
337	* This condition can be true when put_chars is called from
338	* early_init
339	*/
340	if (!portdev->vdev)
341	return false;
342	return __virtio_test_bit(vdev: portdev->vdev, VIRTIO_CONSOLE_F_MULTIPORT);
343	}
344
345	static DEFINE_SPINLOCK(dma_bufs_lock);
346	static LIST_HEAD(pending_free_dma_bufs);
347
348	static void free_buf(struct port_buffer *buf, bool can_sleep)
349	{
350	unsigned int i;
351
352	for (i = 0; i < buf->sgpages; i++) {
353	struct page *page = sg_page(sg: &buf->sg[i]);
354	if (!page)
355	break;
356	put_page(page);
357	}
358
359	if (!buf->dev) {
360	kfree(objp: buf->buf);
361	} else if (is_rproc_enabled) {
362	unsigned long flags;
363
364	/* dma_free_coherent requires interrupts to be enabled. */
365	if (!can_sleep) {
366	/* queue up dma-buffers to be freed later */
367	spin_lock_irqsave(&dma_bufs_lock, flags);
368	list_add_tail(new: &buf->list, head: &pending_free_dma_bufs);
369	spin_unlock_irqrestore(lock: &dma_bufs_lock, flags);
370	return;
371	}
372	dma_free_coherent(dev: buf->dev, size: buf->size, cpu_addr: buf->buf, dma_handle: buf->dma);
373
374	/* Release device refcnt and allow it to be freed */
375	put_device(dev: buf->dev);
376	}
377
378	kfree(objp: buf);
379	}
380
381	static void reclaim_dma_bufs(void)
382	{
383	unsigned long flags;
384	struct port_buffer *buf, *tmp;
385	LIST_HEAD(tmp_list);
386
387	if (list_empty(head: &pending_free_dma_bufs))
388	return;
389
390	/* Create a copy of the pending_free_dma_bufs while holding the lock */
391	spin_lock_irqsave(&dma_bufs_lock, flags);
392	list_cut_position(list: &tmp_list, head: &pending_free_dma_bufs,
393	entry: pending_free_dma_bufs.prev);
394	spin_unlock_irqrestore(lock: &dma_bufs_lock, flags);
395
396	/* Release the dma buffers, without irqs enabled */
397	list_for_each_entry_safe(buf, tmp, &tmp_list, list) {
398	list_del(entry: &buf->list);
399	free_buf(buf, can_sleep: true);
400	}
401	}
402
403	static struct port_buffer *alloc_buf(struct virtio_device *vdev, size_t buf_size,
404	int pages)
405	{
406	struct port_buffer *buf;
407
408	reclaim_dma_bufs();
409
410	/*
411	* Allocate buffer and the sg list. The sg list array is allocated
412	* directly after the port_buffer struct.
413	*/
414	buf = kmalloc(struct_size(buf, sg, pages), GFP_KERNEL);
415	if (!buf)
416	goto fail;
417
418	buf->sgpages = pages;
419	if (pages > 0) {
420	buf->dev = NULL;
421	buf->buf = NULL;
422	return buf;
423	}
424
425	if (is_rproc_serial(vdev)) {
426	/*
427	* Allocate DMA memory from ancestor. When a virtio
428	* device is created by remoteproc, the DMA memory is
429	* associated with the parent device:
430	* virtioY => remoteprocX#vdevYbuffer.
431	*/
432	buf->dev = vdev->dev.parent;
433	if (!buf->dev)
434	goto free_buf;
435
436	/* Increase device refcnt to avoid freeing it */
437	get_device(dev: buf->dev);
438	buf->buf = dma_alloc_coherent(dev: buf->dev, size: buf_size, dma_handle: &buf->dma,
439	GFP_KERNEL);
440	} else {
441	buf->dev = NULL;
442	buf->buf = kmalloc(size: buf_size, GFP_KERNEL);
443	}
444
445	if (!buf->buf)
446	goto free_buf;
447	buf->len = 0;
448	buf->offset = 0;
449	buf->size = buf_size;
450	return buf;
451
452	free_buf:
453	kfree(objp: buf);
454	fail:
455	return NULL;
456	}
457
458	/* Callers should take appropriate locks */
459	static struct port_buffer *get_inbuf(struct port *port)
460	{
461	struct port_buffer *buf;
462	unsigned int len;
463
464	if (port->inbuf)
465	return port->inbuf;
466
467	buf = virtqueue_get_buf(vq: port->in_vq, len: &len);
468	if (buf) {
469	buf->len = min_t(size_t, len, buf->size);
470	buf->offset = 0;
471	port->stats.bytes_received += len;
472	}
473	return buf;
474	}
475
476	/*
477	* Create a scatter-gather list representing our input buffer and put
478	* it in the queue.
479	*
480	* Callers should take appropriate locks.
481	*/
482	static int add_inbuf(struct virtqueue *vq, struct port_buffer *buf)
483	{
484	struct scatterlist sg[1];
485	int ret;
486
487	sg_init_one(sg, buf->buf, buf->size);
488
489	ret = virtqueue_add_inbuf(vq, sg, num: 1, data: buf, GFP_ATOMIC);
490	virtqueue_kick(vq);
491	if (!ret)
492	ret = vq->num_free;
493	return ret;
494	}
495
496	/* Discard any unread data this port has. Callers lockers. */
497	static void discard_port_data(struct port *port)
498	{
499	struct port_buffer *buf;
500	unsigned int err;
501
502	if (!port->portdev) {
503	/* Device has been unplugged. vqs are already gone. */
504	return;
505	}
506	buf = get_inbuf(port);
507
508	err = 0;
509	while (buf) {
510	port->stats.bytes_discarded += buf->len - buf->offset;
511	if (add_inbuf(vq: port->in_vq, buf) < 0) {
512	err++;
513	free_buf(buf, can_sleep: false);
514	}
515	port->inbuf = NULL;
516	buf = get_inbuf(port);
517	}
518	if (err)
519	dev_warn(port->dev, "Errors adding %d buffers back to vq\n",
520	err);
521	}
522
523	static bool port_has_data(struct port *port)
524	{
525	unsigned long flags;
526	bool ret;
527
528	ret = false;
529	spin_lock_irqsave(&port->inbuf_lock, flags);
530	port->inbuf = get_inbuf(port);
531	if (port->inbuf)
532	ret = true;
533
534	spin_unlock_irqrestore(lock: &port->inbuf_lock, flags);
535	return ret;
536	}
537
538	static ssize_t __send_control_msg(struct ports_device *portdev, u32 port_id,
539	unsigned int event, unsigned int value)
540	{
541	struct scatterlist sg[1];
542	struct virtqueue *vq;
543	unsigned int len;
544
545	if (!use_multiport(portdev))
546	return 0;
547
548	vq = portdev->c_ovq;
549
550	spin_lock(lock: &portdev->c_ovq_lock);
551
552	portdev->cpkt.id = cpu_to_virtio32(vdev: portdev->vdev, val: port_id);
553	portdev->cpkt.event = cpu_to_virtio16(vdev: portdev->vdev, val: event);
554	portdev->cpkt.value = cpu_to_virtio16(vdev: portdev->vdev, val: value);
555
556	sg_init_one(sg, &portdev->cpkt, sizeof(struct virtio_console_control));
557
558	if (virtqueue_add_outbuf(vq, sg, num: 1, data: &portdev->cpkt, GFP_ATOMIC) == 0) {
559	virtqueue_kick(vq);
560	while (!virtqueue_get_buf(vq, len: &len)
561	&& !virtqueue_is_broken(vq))
562	cpu_relax();
563	}
564
565	spin_unlock(lock: &portdev->c_ovq_lock);
566	return 0;
567	}
568
569	static ssize_t send_control_msg(struct port *port, unsigned int event,
570	unsigned int value)
571	{
572	/* Did the port get unplugged before userspace closed it? */
573	if (port->portdev)
574	return __send_control_msg(portdev: port->portdev, port_id: port->id, event, value);
575	return 0;
576	}
577
578
579	/* Callers must take the port->outvq_lock */
580	static void reclaim_consumed_buffers(struct port *port)
581	{
582	struct port_buffer *buf;
583	unsigned int len;
584
585	if (!port->portdev) {
586	/* Device has been unplugged. vqs are already gone. */
587	return;
588	}
589	while ((buf = virtqueue_get_buf(vq: port->out_vq, len: &len))) {
590	free_buf(buf, can_sleep: false);
591	port->outvq_full = false;
592	}
593	}
594
595	static ssize_t __send_to_port(struct port *port, struct scatterlist *sg,
596	int nents, size_t in_count,
597	void *data, bool nonblock)
598	{
599	struct virtqueue *out_vq;
600	int err;
601	unsigned long flags;
602	unsigned int len;
603
604	out_vq = port->out_vq;
605
606	spin_lock_irqsave(&port->outvq_lock, flags);
607
608	reclaim_consumed_buffers(port);
609
610	err = virtqueue_add_outbuf(vq: out_vq, sg, num: nents, data, GFP_ATOMIC);
611
612	/* Tell Host to go! */
613	virtqueue_kick(vq: out_vq);
614
615	if (err) {
616	in_count = 0;
617	goto done;
618	}
619
620	if (out_vq->num_free == 0)
621	port->outvq_full = true;
622
623	if (nonblock)
624	goto done;
625
626	/*
627	* Wait till the host acknowledges it pushed out the data we
628	* sent. This is done for data from the hvc_console; the tty
629	* operations are performed with spinlocks held so we can't
630	* sleep here. An alternative would be to copy the data to a
631	* buffer and relax the spinning requirement. The downside is
632	* we need to kmalloc a GFP_ATOMIC buffer each time the
633	* console driver writes something out.
634	*/
635	while (!virtqueue_get_buf(vq: out_vq, len: &len)
636	&& !virtqueue_is_broken(vq: out_vq))
637	cpu_relax();
638	done:
639	spin_unlock_irqrestore(lock: &port->outvq_lock, flags);
640
641	port->stats.bytes_sent += in_count;
642	/*
643	* We're expected to return the amount of data we wrote -- all
644	* of it
645	*/
646	return in_count;
647	}
648
649	/*
650	* Give out the data that's requested from the buffer that we have
651	* queued up.
652	*/
653	static ssize_t fill_readbuf(struct port *port, u8 __user *out_buf,
654	size_t out_count, bool to_user)
655	{
656	struct port_buffer *buf;
657	unsigned long flags;
658
659	if (!out_count || !port_has_data(port))
660	return 0;
661
662	buf = port->inbuf;
663	out_count = min(out_count, buf->len - buf->offset);
664
665	if (to_user) {
666	ssize_t ret;
667
668	ret = copy_to_user(to: out_buf, from: buf->buf + buf->offset, n: out_count);
669	if (ret)
670	return -EFAULT;
671	} else {
672	memcpy((__force u8 *)out_buf, buf->buf + buf->offset,
673	out_count);
674	}
675
676	buf->offset += out_count;
677
678	if (buf->offset == buf->len) {
679	/*
680	* We're done using all the data in this buffer.
681	* Re-queue so that the Host can send us more data.
682	*/
683	spin_lock_irqsave(&port->inbuf_lock, flags);
684	port->inbuf = NULL;
685
686	if (add_inbuf(vq: port->in_vq, buf) < 0)
687	dev_warn(port->dev, "failed add_buf\n");
688
689	spin_unlock_irqrestore(lock: &port->inbuf_lock, flags);
690	}
691	/* Return the number of bytes actually copied */
692	return out_count;
693	}
694
695	/* The condition that must be true for polling to end */
696	static bool will_read_block(struct port *port)
697	{
698	if (!port->guest_connected) {
699	/* Port got hot-unplugged. Let's exit. */
700	return false;
701	}
702	return !port_has_data(port) && port->host_connected;
703	}
704
705	static bool will_write_block(struct port *port)
706	{
707	bool ret;
708
709	if (!port->guest_connected) {
710	/* Port got hot-unplugged. Let's exit. */
711	return false;
712	}
713	if (!port->host_connected)
714	return true;
715
716	spin_lock_irq(lock: &port->outvq_lock);
717	/*
718	* Check if the Host has consumed any buffers since we last
719	* sent data (this is only applicable for nonblocking ports).
720	*/
721	reclaim_consumed_buffers(port);
722	ret = port->outvq_full;
723	spin_unlock_irq(lock: &port->outvq_lock);
724
725	return ret;
726	}
727
728	static ssize_t port_fops_read(struct file *filp, char __user *ubuf,
729	size_t count, loff_t *offp)
730	{
731	struct port *port;
732	ssize_t ret;
733
734	port = filp->private_data;
735
736	/* Port is hot-unplugged. */
737	if (!port->guest_connected)
738	return -ENODEV;
739
740	if (!port_has_data(port)) {
741	/*
742	* If nothing's connected on the host just return 0 in
743	* case of list_empty; this tells the userspace app
744	* that there's no connection
745	*/
746	if (!port->host_connected)
747	return 0;
748	if (filp->f_flags & O_NONBLOCK)
749	return -EAGAIN;
750
751	ret = wait_event_freezable(port->waitqueue,
752	!will_read_block(port));
753	if (ret < 0)
754	return ret;
755	}
756	/* Port got hot-unplugged while we were waiting above. */
757	if (!port->guest_connected)
758	return -ENODEV;
759	/*
760	* We could've received a disconnection message while we were
761	* waiting for more data.
762	*
763	* This check is not clubbed in the if() statement above as we
764	* might receive some data as well as the host could get
765	* disconnected after we got woken up from our wait. So we
766	* really want to give off whatever data we have and only then
767	* check for host_connected.
768	*/
769	if (!port_has_data(port) && !port->host_connected)
770	return 0;
771
772	return fill_readbuf(port, out_buf: ubuf, out_count: count, to_user: true);
773	}
774
775	static int wait_port_writable(struct port *port, bool nonblock)
776	{
777	int ret;
778
779	if (will_write_block(port)) {
780	if (nonblock)
781	return -EAGAIN;
782
783	ret = wait_event_freezable(port->waitqueue,
784	!will_write_block(port));
785	if (ret < 0)
786	return ret;
787	}
788	/* Port got hot-unplugged. */
789	if (!port->guest_connected)
790	return -ENODEV;
791
792	return 0;
793	}
794
795	static ssize_t port_fops_write(struct file *filp, const char __user *ubuf,
796	size_t count, loff_t *offp)
797	{
798	struct port *port;
799	struct port_buffer *buf;
800	ssize_t ret;
801	bool nonblock;
802	struct scatterlist sg[1];
803
804	/* Userspace could be out to fool us */
805	if (!count)
806	return 0;
807
808	port = filp->private_data;
809
810	nonblock = filp->f_flags & O_NONBLOCK;
811
812	ret = wait_port_writable(port, nonblock);
813	if (ret < 0)
814	return ret;
815
816	count = min((size_t)(32 * 1024), count);
817
818	buf = alloc_buf(vdev: port->portdev->vdev, buf_size: count, pages: 0);
819	if (!buf)
820	return -ENOMEM;
821
822	ret = copy_from_user(to: buf->buf, from: ubuf, n: count);
823	if (ret) {
824	ret = -EFAULT;
825	goto free_buf;
826	}
827
828	/*
829	* We now ask send_buf() to not spin for generic ports -- we
830	* can re-use the same code path that non-blocking file
831	* descriptors take for blocking file descriptors since the
832	* wait is already done and we're certain the write will go
833	* through to the host.
834	*/
835	nonblock = true;
836	sg_init_one(sg, buf->buf, count);
837	ret = __send_to_port(port, sg, nents: 1, in_count: count, data: buf, nonblock);
838
839	if (nonblock && ret > 0)
840	goto out;
841
842	free_buf:
843	free_buf(buf, can_sleep: true);
844	out:
845	return ret;
846	}
847
848	struct sg_list {
849	unsigned int n;
850	unsigned int size;
851	size_t len;
852	struct scatterlist *sg;
853	};
854
855	static int pipe_to_sg(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
856	struct splice_desc *sd)
857	{
858	struct sg_list *sgl = sd->u.data;
859	unsigned int offset, len;
860
861	if (sgl->n == sgl->size)
862	return 0;
863
864	/* Try lock this page */
865	if (pipe_buf_try_steal(pipe, buf)) {
866	/* Get reference and unlock page for moving */
867	get_page(page: buf->page);
868	unlock_page(page: buf->page);
869
870	len = min(buf->len, sd->len);
871	sg_set_page(sg: &(sgl->sg[sgl->n]), page: buf->page, len, offset: buf->offset);
872	} else {
873	/* Failback to copying a page */
874	struct page *page = alloc_page(GFP_KERNEL);
875	char *src;
876
877	if (!page)
878	return -ENOMEM;
879
880	offset = sd->pos & ~PAGE_MASK;
881
882	len = sd->len;
883	if (len + offset > PAGE_SIZE)
884	len = PAGE_SIZE - offset;
885
886	src = kmap_atomic(page: buf->page);
887	memcpy(page_address(page) + offset, src + buf->offset, len);
888	kunmap_atomic(src);
889
890	sg_set_page(sg: &(sgl->sg[sgl->n]), page, len, offset);
891	}
892	sgl->n++;
893	sgl->len += len;
894
895	return len;
896	}
897
898	/* Faster zero-copy write by splicing */
899	static ssize_t port_fops_splice_write(struct pipe_inode_info *pipe,
900	struct file *filp, loff_t *ppos,
901	size_t len, unsigned int flags)
902	{
903	struct port *port = filp->private_data;
904	struct sg_list sgl;
905	ssize_t ret;
906	struct port_buffer *buf;
907	struct splice_desc sd = {
908	.total_len = len,
909	.flags = flags,
910	.pos = *ppos,
911	.u.data = &sgl,
912	};
913	unsigned int occupancy;
914
915	/*
916	* Rproc_serial does not yet support splice. To support splice
917	* pipe_to_sg() must allocate dma-buffers and copy content from
918	* regular pages to dma pages. And alloc_buf and free_buf must
919	* support allocating and freeing such a list of dma-buffers.
920	*/
921	if (is_rproc_serial(vdev: port->out_vq->vdev))
922	return -EINVAL;
923
924	pipe_lock(pipe);
925	ret = 0;
926	if (pipe_empty(head: pipe->head, tail: pipe->tail))
927	goto error_out;
928
929	ret = wait_port_writable(port, nonblock: filp->f_flags & O_NONBLOCK);
930	if (ret < 0)
931	goto error_out;
932
933	occupancy = pipe_occupancy(head: pipe->head, tail: pipe->tail);
934	buf = alloc_buf(vdev: port->portdev->vdev, buf_size: 0, pages: occupancy);
935
936	if (!buf) {
937	ret = -ENOMEM;
938	goto error_out;
939	}
940
941	sgl.n = 0;
942	sgl.len = 0;
943	sgl.size = occupancy;
944	sgl.sg = buf->sg;
945	sg_init_table(sgl.sg, sgl.size);
946	ret = __splice_from_pipe(pipe, sd: &sd, actor: pipe_to_sg);
947	pipe_unlock(pipe);
948	if (likely(ret > 0))
949	ret = __send_to_port(port, sg: buf->sg, nents: sgl.n, in_count: sgl.len, data: buf, nonblock: true);
950
951	if (unlikely(ret <= 0))
952	free_buf(buf, can_sleep: true);
953	return ret;
954
955	error_out:
956	pipe_unlock(pipe);
957	return ret;
958	}
959
960	static __poll_t port_fops_poll(struct file *filp, poll_table *wait)
961	{
962	struct port *port;
963	__poll_t ret;
964
965	port = filp->private_data;
966	poll_wait(filp, wait_address: &port->waitqueue, p: wait);
967
968	if (!port->guest_connected) {
969	/* Port got unplugged */
970	return EPOLLHUP;
971	}
972	ret = 0;
973	if (!will_read_block(port))
974	ret |= EPOLLIN | EPOLLRDNORM;
975	if (!will_write_block(port))
976	ret |= EPOLLOUT;
977	if (!port->host_connected)
978	ret |= EPOLLHUP;
979
980	return ret;
981	}
982
983	static void remove_port(struct kref *kref);
984
985	static int port_fops_release(struct inode *inode, struct file *filp)
986	{
987	struct port *port;
988
989	port = filp->private_data;
990
991	/* Notify host of port being closed */
992	send_control_msg(port, VIRTIO_CONSOLE_PORT_OPEN, value: 0);
993
994	spin_lock_irq(lock: &port->inbuf_lock);
995	port->guest_connected = false;
996
997	discard_port_data(port);
998
999	spin_unlock_irq(lock: &port->inbuf_lock);
1000
1001	spin_lock_irq(lock: &port->outvq_lock);
1002	reclaim_consumed_buffers(port);
1003	spin_unlock_irq(lock: &port->outvq_lock);
1004
1005	reclaim_dma_bufs();
1006	/*
1007	* Locks aren't necessary here as a port can't be opened after
1008	* unplug, and if a port isn't unplugged, a kref would already
1009	* exist for the port. Plus, taking ports_lock here would
1010	* create a dependency on other locks taken by functions
1011	* inside remove_port if we're the last holder of the port,
1012	* creating many problems.
1013	*/
1014	kref_put(kref: &port->kref, release: remove_port);
1015
1016	return 0;
1017	}
1018
1019	static int port_fops_open(struct inode *inode, struct file *filp)
1020	{
1021	struct cdev *cdev = inode->i_cdev;
1022	struct port *port;
1023	int ret;
1024
1025	/* We get the port with a kref here */
1026	port = find_port_by_devt(dev: cdev->dev);
1027	if (!port) {
1028	/* Port was unplugged before we could proceed */
1029	return -ENXIO;
1030	}
1031	filp->private_data = port;
1032
1033	/*
1034	* Don't allow opening of console port devices -- that's done
1035	* via /dev/hvc
1036	*/
1037	if (is_console_port(port)) {
1038	ret = -ENXIO;
1039	goto out;
1040	}
1041
1042	/* Allow only one process to open a particular port at a time */
1043	spin_lock_irq(lock: &port->inbuf_lock);
1044	if (port->guest_connected) {
1045	spin_unlock_irq(lock: &port->inbuf_lock);
1046	ret = -EBUSY;
1047	goto out;
1048	}
1049
1050	port->guest_connected = true;
1051	spin_unlock_irq(lock: &port->inbuf_lock);
1052
1053	spin_lock_irq(lock: &port->outvq_lock);
1054	/*
1055	* There might be a chance that we missed reclaiming a few
1056	* buffers in the window of the port getting previously closed
1057	* and opening now.
1058	*/
1059	reclaim_consumed_buffers(port);
1060	spin_unlock_irq(lock: &port->outvq_lock);
1061
1062	nonseekable_open(inode, filp);
1063
1064	/* Notify host of port being opened */
1065	send_control_msg(port: filp->private_data, VIRTIO_CONSOLE_PORT_OPEN, value: 1);
1066
1067	return 0;
1068	out:
1069	kref_put(kref: &port->kref, release: remove_port);
1070	return ret;
1071	}
1072
1073	static int port_fops_fasync(int fd, struct file *filp, int mode)
1074	{
1075	struct port *port;
1076
1077	port = filp->private_data;
1078	return fasync_helper(fd, filp, mode, &port->async_queue);
1079	}
1080
1081	/*
1082	* The file operations that we support: programs in the guest can open
1083	* a console device, read from it, write to it, poll for data and
1084	* close it. The devices are at
1085	* /dev/vport<device number>p<port number>
1086	*/
1087	static const struct file_operations port_fops = {
1088	.owner = THIS_MODULE,
1089	.open = port_fops_open,
1090	.read = port_fops_read,
1091	.write = port_fops_write,
1092	.splice_write = port_fops_splice_write,
1093	.poll = port_fops_poll,
1094	.release = port_fops_release,
1095	.fasync = port_fops_fasync,
1096	.llseek = no_llseek,
1097	};
1098
1099	/*
1100	* The put_chars() callback is pretty straightforward.
1101	*
1102	* We turn the characters into a scatter-gather list, add it to the
1103	* output queue and then kick the Host. Then we sit here waiting for
1104	* it to finish: inefficient in theory, but in practice
1105	* implementations will do it immediately.
1106	*/
1107	static ssize_t put_chars(u32 vtermno, const u8 *buf, size_t count)
1108	{
1109	struct port *port;
1110	struct scatterlist sg[1];
1111	void *data;
1112	int ret;
1113
1114	port = find_port_by_vtermno(vtermno);
1115	if (!port)
1116	return -EPIPE;
1117
1118	data = kmemdup(p: buf, size: count, GFP_ATOMIC);
1119	if (!data)
1120	return -ENOMEM;
1121
1122	sg_init_one(sg, data, count);
1123	ret = __send_to_port(port, sg, nents: 1, in_count: count, data, nonblock: false);
1124	kfree(objp: data);
1125	return ret;
1126	}
1127
1128	/*
1129	* get_chars() is the callback from the hvc_console infrastructure
1130	* when an interrupt is received.
1131	*
1132	* We call out to fill_readbuf that gets us the required data from the
1133	* buffers that are queued up.
1134	*/
1135	static ssize_t get_chars(u32 vtermno, u8 *buf, size_t count)
1136	{
1137	struct port *port;
1138
1139	port = find_port_by_vtermno(vtermno);
1140	if (!port)
1141	return -EPIPE;
1142
1143	/* If we don't have an input queue yet, we can't get input. */
1144	BUG_ON(!port->in_vq);
1145
1146	return fill_readbuf(port, out_buf: (__force u8 __user *)buf, out_count: count, to_user: false);
1147	}
1148
1149	static void resize_console(struct port *port)
1150	{
1151	struct virtio_device *vdev;
1152
1153	/* The port could have been hot-unplugged */
1154	if (!port || !is_console_port(port))
1155	return;
1156
1157	vdev = port->portdev->vdev;
1158
1159	/* Don't test F_SIZE at all if we're rproc: not a valid feature! */
1160	if (!is_rproc_serial(vdev) &&
1161	virtio_has_feature(vdev, VIRTIO_CONSOLE_F_SIZE))
1162	hvc_resize(hp: port->cons.hvc, ws: port->cons.ws);
1163	}
1164
1165	/* We set the configuration at this point, since we now have a tty */
1166	static int notifier_add_vio(struct hvc_struct *hp, int data)
1167	{
1168	struct port *port;
1169
1170	port = find_port_by_vtermno(vtermno: hp->vtermno);
1171	if (!port)
1172	return -EINVAL;
1173
1174	hp->irq_requested = 1;
1175	resize_console(port);
1176
1177	return 0;
1178	}
1179
1180	static void notifier_del_vio(struct hvc_struct *hp, int data)
1181	{
1182	hp->irq_requested = 0;
1183	}
1184
1185	/* The operations for console ports. */
1186	static const struct hv_ops hv_ops = {
1187	.get_chars = get_chars,
1188	.put_chars = put_chars,
1189	.notifier_add = notifier_add_vio,
1190	.notifier_del = notifier_del_vio,
1191	.notifier_hangup = notifier_del_vio,
1192	};
1193
1194	static int init_port_console(struct port *port)
1195	{
1196	int ret;
1197
1198	/*
1199	* The Host's telling us this port is a console port. Hook it
1200	* up with an hvc console.
1201	*
1202	* To set up and manage our virtual console, we call
1203	* hvc_alloc().
1204	*
1205	* The first argument of hvc_alloc() is the virtual console
1206	* number. The second argument is the parameter for the
1207	* notification mechanism (like irq number). We currently
1208	* leave this as zero, virtqueues have implicit notifications.
1209	*
1210	* The third argument is a "struct hv_ops" containing the
1211	* put_chars() get_chars(), notifier_add() and notifier_del()
1212	* pointers. The final argument is the output buffer size: we
1213	* can do any size, so we put PAGE_SIZE here.
1214	*/
1215	ret = ida_alloc_min(ida: &vtermno_ida, min: 1, GFP_KERNEL);
1216	if (ret < 0)
1217	return ret;
1218
1219	port->cons.vtermno = ret;
1220	port->cons.hvc = hvc_alloc(vtermno: port->cons.vtermno, data: 0, ops: &hv_ops, PAGE_SIZE);
1221	if (IS_ERR(ptr: port->cons.hvc)) {
1222	ret = PTR_ERR(ptr: port->cons.hvc);
1223	dev_err(port->dev,
1224	"error %d allocating hvc for port\n", ret);
1225	port->cons.hvc = NULL;
1226	ida_free(&vtermno_ida, id: port->cons.vtermno);
1227	return ret;
1228	}
1229	spin_lock_irq(lock: &pdrvdata_lock);
1230	list_add_tail(new: &port->cons.list, head: &pdrvdata.consoles);
1231	spin_unlock_irq(lock: &pdrvdata_lock);
1232	port->guest_connected = true;
1233
1234	/* Notify host of port being opened */
1235	send_control_msg(port, VIRTIO_CONSOLE_PORT_OPEN, value: 1);
1236
1237	return 0;
1238	}
1239
1240	static ssize_t show_port_name(struct device *dev,
1241	struct device_attribute *attr, char *buffer)
1242	{
1243	struct port *port;
1244
1245	port = dev_get_drvdata(dev);
1246
1247	return sprintf(buf: buffer, fmt: "%s\n", port->name);
1248	}
1249
1250	static DEVICE_ATTR(name, S_IRUGO, show_port_name, NULL);
1251
1252	static struct attribute *port_sysfs_entries[] = {
1253	&dev_attr_name.attr,
1254	NULL
1255	};
1256
1257	static const struct attribute_group port_attribute_group = {
1258	.name = NULL, /* put in device directory */
1259	.attrs = port_sysfs_entries,
1260	};
1261
1262	static int port_debugfs_show(struct seq_file *s, void *data)
1263	{
1264	struct port *port = s->private;
1265
1266	seq_printf(m: s, fmt: "name: %s\n", port->name ? port->name : "");
1267	seq_printf(m: s, fmt: "guest_connected: %d\n", port->guest_connected);
1268	seq_printf(m: s, fmt: "host_connected: %d\n", port->host_connected);
1269	seq_printf(m: s, fmt: "outvq_full: %d\n", port->outvq_full);
1270	seq_printf(m: s, fmt: "bytes_sent: %lu\n", port->stats.bytes_sent);
1271	seq_printf(m: s, fmt: "bytes_received: %lu\n", port->stats.bytes_received);
1272	seq_printf(m: s, fmt: "bytes_discarded: %lu\n", port->stats.bytes_discarded);
1273	seq_printf(m: s, fmt: "is_console: %s\n",
1274	is_console_port(port) ? "yes" : "no");
1275	seq_printf(m: s, fmt: "console_vtermno: %u\n", port->cons.vtermno);
1276
1277	return 0;
1278	}
1279
1280	DEFINE_SHOW_ATTRIBUTE(port_debugfs);
1281
1282	static void set_console_size(struct port *port, u16 rows, u16 cols)
1283	{
1284	if (!port || !is_console_port(port))
1285	return;
1286
1287	port->cons.ws.ws_row = rows;
1288	port->cons.ws.ws_col = cols;
1289	}
1290
1291	static int fill_queue(struct virtqueue *vq, spinlock_t *lock)
1292	{
1293	struct port_buffer *buf;
1294	int nr_added_bufs;
1295	int ret;
1296
1297	nr_added_bufs = 0;
1298	do {
1299	buf = alloc_buf(vdev: vq->vdev, PAGE_SIZE, pages: 0);
1300	if (!buf)
1301	return -ENOMEM;
1302
1303	spin_lock_irq(lock);
1304	ret = add_inbuf(vq, buf);
1305	if (ret < 0) {
1306	spin_unlock_irq(lock);
1307	free_buf(buf, can_sleep: true);
1308	return ret;
1309	}
1310	nr_added_bufs++;
1311	spin_unlock_irq(lock);
1312	} while (ret > 0);
1313
1314	return nr_added_bufs;
1315	}
1316
1317	static void send_sigio_to_port(struct port *port)
1318	{
1319	if (port->async_queue && port->guest_connected)
1320	kill_fasync(&port->async_queue, SIGIO, POLL_OUT);
1321	}
1322
1323	static int add_port(struct ports_device *portdev, u32 id)
1324	{
1325	char debugfs_name[16];
1326	struct port *port;
1327	dev_t devt;
1328	int err;
1329
1330	port = kmalloc(size: sizeof(*port), GFP_KERNEL);
1331	if (!port) {
1332	err = -ENOMEM;
1333	goto fail;
1334	}
1335	kref_init(kref: &port->kref);
1336
1337	port->portdev = portdev;
1338	port->id = id;
1339
1340	port->name = NULL;
1341	port->inbuf = NULL;
1342	port->cons.hvc = NULL;
1343	port->async_queue = NULL;
1344
1345	port->cons.ws.ws_row = port->cons.ws.ws_col = 0;
1346	port->cons.vtermno = 0;
1347
1348	port->host_connected = port->guest_connected = false;
1349	port->stats = (struct port_stats) { 0 };
1350
1351	port->outvq_full = false;
1352
1353	port->in_vq = portdev->in_vqs[port->id];
1354	port->out_vq = portdev->out_vqs[port->id];
1355
1356	port->cdev = cdev_alloc();
1357	if (!port->cdev) {
1358	dev_err(&port->portdev->vdev->dev, "Error allocating cdev\n");
1359	err = -ENOMEM;
1360	goto free_port;
1361	}
1362	port->cdev->ops = &port_fops;
1363
1364	devt = MKDEV(portdev->chr_major, id);
1365	err = cdev_add(port->cdev, devt, 1);
1366	if (err < 0) {
1367	dev_err(&port->portdev->vdev->dev,
1368	"Error %d adding cdev for port %u\n", err, id);
1369	goto free_cdev;
1370	}
1371	port->dev = device_create(&port_class, &port->portdev->vdev->dev,
1372	devt, port, "vport%up%u",
1373	port->portdev->vdev->index, id);
1374	if (IS_ERR(port->dev)) {
1375	err = PTR_ERR(port->dev);
1376	dev_err(&port->portdev->vdev->dev,
1377	"Error %d creating device for port %u\n",
1378	err, id);
1379	goto free_cdev;
1380	}
1381
1382	spin_lock_init(&port->inbuf_lock);
1383	spin_lock_init(&port->outvq_lock);
1384	init_waitqueue_head(&port->waitqueue);
1385
1386	/* We can safely ignore ENOSPC because it means
1387	* the queue already has buffers. Buffers are removed
1388	* only by virtcons_remove(), not by unplug_port()
1389	*/
1390	err = fill_queue(port->in_vq, &port->inbuf_lock);
1391	if (err < 0 && err != -ENOSPC) {
1392	dev_err(port->dev, "Error allocating inbufs\n");
1393	goto free_device;
1394	}
1395
1396	if (is_rproc_serial(port->portdev->vdev))
1397	/*
1398	* For rproc_serial assume remote processor is connected.
1399	* rproc_serial does not want the console port, only
1400	* the generic port implementation.
1401	*/
1402	port->host_connected = true;
1403	else if (!use_multiport(port->portdev)) {
1404	/*
1405	* If we're not using multiport support,
1406	* this has to be a console port.
1407	*/
1408	err = init_port_console(port);
1409	if (err)
1410	goto free_inbufs;
1411	}
1412
1413	spin_lock_irq(&portdev->ports_lock);
1414	list_add_tail(&port->list, &port->portdev->ports);
1415	spin_unlock_irq(&portdev->ports_lock);
1416
1417	/*
1418	* Tell the Host we're set so that it can send us various
1419	* configuration parameters for this port (eg, port name,
1420	* caching, whether this is a console port, etc.)
1421	*/
1422	send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, 1);
1423
1424	/*
1425	* Finally, create the debugfs file that we can use to
1426	* inspect a port's state at any time
1427	*/
1428	snprintf(debugfs_name, sizeof(debugfs_name), "vport%up%u",
1429	port->portdev->vdev->index, id);
1430	port->debugfs_file = debugfs_create_file(debugfs_name, 0444,
1431	pdrvdata.debugfs_dir,
1432	port, &port_debugfs_fops);
1433	return 0;
1434
1435	free_inbufs:
1436	free_device:
1437	device_destroy(&port_class, port->dev->devt);
1438	free_cdev:
1439	cdev_del(port->cdev);
1440	free_port:
1441	kfree(port);
1442	fail:
1443	/* The host might want to notify management sw about port add failure */
1444	__send_control_msg(portdev, id, VIRTIO_CONSOLE_PORT_READY, 0);
1445	return err;
1446	}
1447
1448	/* No users remain, remove all port-specific data. */
1449	static void remove_port(struct kref *kref)
1450	{
1451	struct port *port;
1452
1453	port = container_of(kref, struct port, kref);
1454
1455	kfree(objp: port);
1456	}
1457
1458	static void remove_port_data(struct port *port)
1459	{
1460	spin_lock_irq(lock: &port->inbuf_lock);
1461	/* Remove unused data this port might have received. */
1462	discard_port_data(port);
1463	spin_unlock_irq(lock: &port->inbuf_lock);
1464
1465	spin_lock_irq(lock: &port->outvq_lock);
1466	reclaim_consumed_buffers(port);
1467	spin_unlock_irq(lock: &port->outvq_lock);
1468	}
1469
1470	/*
1471	* Port got unplugged. Remove port from portdev's list and drop the
1472	* kref reference. If no userspace has this port opened, it will
1473	* result in immediate removal the port.
1474	*/
1475	static void unplug_port(struct port *port)
1476	{
1477	spin_lock_irq(lock: &port->portdev->ports_lock);
1478	list_del(entry: &port->list);
1479	spin_unlock_irq(lock: &port->portdev->ports_lock);
1480
1481	spin_lock_irq(lock: &port->inbuf_lock);
1482	if (port->guest_connected) {
1483	/* Let the app know the port is going down. */
1484	send_sigio_to_port(port);
1485
1486	/* Do this after sigio is actually sent */
1487	port->guest_connected = false;
1488	port->host_connected = false;
1489
1490	wake_up_interruptible(&port->waitqueue);
1491	}
1492	spin_unlock_irq(lock: &port->inbuf_lock);
1493
1494	if (is_console_port(port)) {
1495	spin_lock_irq(lock: &pdrvdata_lock);
1496	list_del(entry: &port->cons.list);
1497	spin_unlock_irq(lock: &pdrvdata_lock);
1498	hvc_remove(hp: port->cons.hvc);
1499	ida_free(&vtermno_ida, id: port->cons.vtermno);
1500	}
1501
1502	remove_port_data(port);
1503
1504	/*
1505	* We should just assume the device itself has gone off --
1506	* else a close on an open port later will try to send out a
1507	* control message.
1508	*/
1509	port->portdev = NULL;
1510
1511	sysfs_remove_group(kobj: &port->dev->kobj, grp: &port_attribute_group);
1512	device_destroy(cls: &port_class, devt: port->dev->devt);
1513	cdev_del(port->cdev);
1514
1515	debugfs_remove(dentry: port->debugfs_file);
1516	kfree(objp: port->name);
1517
1518	/*
1519	* Locks around here are not necessary - a port can't be
1520	* opened after we removed the port struct from ports_list
1521	* above.
1522	*/
1523	kref_put(kref: &port->kref, release: remove_port);
1524	}
1525
1526	/* Any private messages that the Host and Guest want to share */
1527	static void handle_control_message(struct virtio_device *vdev,
1528	struct ports_device *portdev,
1529	struct port_buffer *buf)
1530	{
1531	struct virtio_console_control *cpkt;
1532	struct port *port;
1533	size_t name_size;
1534	int err;
1535
1536	cpkt = (struct virtio_console_control *)(buf->buf + buf->offset);
1537
1538	port = find_port_by_id(portdev, id: virtio32_to_cpu(vdev, val: cpkt->id));
1539	if (!port &&
1540	cpkt->event != cpu_to_virtio16(vdev, VIRTIO_CONSOLE_PORT_ADD)) {
1541	/* No valid header at start of buffer. Drop it. */
1542	dev_dbg(&portdev->vdev->dev,
1543	"Invalid index %u in control packet\n", cpkt->id);
1544	return;
1545	}
1546
1547	switch (virtio16_to_cpu(vdev, val: cpkt->event)) {
1548	case VIRTIO_CONSOLE_PORT_ADD:
1549	if (port) {
1550	dev_dbg(&portdev->vdev->dev,
1551	"Port %u already added\n", port->id);
1552	send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, value: 1);
1553	break;
1554	}
1555	if (virtio32_to_cpu(vdev, val: cpkt->id) >=
1556	portdev->max_nr_ports) {
1557	dev_warn(&portdev->vdev->dev,
1558	"Request for adding port with "
1559	"out-of-bound id %u, max. supported id: %u\n",
1560	cpkt->id, portdev->max_nr_ports - 1);
1561	break;
1562	}
1563	add_port(portdev, id: virtio32_to_cpu(vdev, val: cpkt->id));
1564	break;
1565	case VIRTIO_CONSOLE_PORT_REMOVE:
1566	unplug_port(port);
1567	break;
1568	case VIRTIO_CONSOLE_CONSOLE_PORT:
1569	if (!cpkt->value)
1570	break;
1571	if (is_console_port(port))
1572	break;
1573
1574	init_port_console(port);
1575	complete(&early_console_added);
1576	/*
1577	* Could remove the port here in case init fails - but
1578	* have to notify the host first.
1579	*/
1580	break;
1581	case VIRTIO_CONSOLE_RESIZE: {
1582	struct {
1583	__u16 rows;
1584	__u16 cols;
1585	} size;
1586
1587	if (!is_console_port(port))
1588	break;
1589
1590	memcpy(&size, buf->buf + buf->offset + sizeof(*cpkt),
1591	sizeof(size));
1592	set_console_size(port, rows: size.rows, cols: size.cols);
1593
1594	port->cons.hvc->irq_requested = 1;
1595	resize_console(port);
1596	break;
1597	}
1598	case VIRTIO_CONSOLE_PORT_OPEN:
1599	port->host_connected = virtio16_to_cpu(vdev, val: cpkt->value);
1600	wake_up_interruptible(&port->waitqueue);
1601	/*
1602	* If the host port got closed and the host had any
1603	* unconsumed buffers, we'll be able to reclaim them
1604	* now.
1605	*/
1606	spin_lock_irq(lock: &port->outvq_lock);
1607	reclaim_consumed_buffers(port);
1608	spin_unlock_irq(lock: &port->outvq_lock);
1609
1610	/*
1611	* If the guest is connected, it'll be interested in
1612	* knowing the host connection state changed.
1613	*/
1614	spin_lock_irq(lock: &port->inbuf_lock);
1615	send_sigio_to_port(port);
1616	spin_unlock_irq(lock: &port->inbuf_lock);
1617	break;
1618	case VIRTIO_CONSOLE_PORT_NAME:
1619	/*
1620	* If we woke up after hibernation, we can get this
1621	* again. Skip it in that case.
1622	*/
1623	if (port->name)
1624	break;
1625
1626	/*
1627	* Skip the size of the header and the cpkt to get the size
1628	* of the name that was sent
1629	*/
1630	name_size = buf->len - buf->offset - sizeof(*cpkt) + 1;
1631
1632	port->name = kmalloc(size: name_size, GFP_KERNEL);
1633	if (!port->name) {
1634	dev_err(port->dev,
1635	"Not enough space to store port name\n");
1636	break;
1637	}
1638	strscpy(port->name, buf->buf + buf->offset + sizeof(*cpkt),
1639	name_size);
1640
1641	/*
1642	* Since we only have one sysfs attribute, 'name',
1643	* create it only if we have a name for the port.
1644	*/
1645	err = sysfs_create_group(kobj: &port->dev->kobj,
1646	grp: &port_attribute_group);
1647	if (err) {
1648	dev_err(port->dev,
1649	"Error %d creating sysfs device attributes\n",
1650	err);
1651	} else {
1652	/*
1653	* Generate a udev event so that appropriate
1654	* symlinks can be created based on udev
1655	* rules.
1656	*/
1657	kobject_uevent(kobj: &port->dev->kobj, action: KOBJ_CHANGE);
1658	}
1659	break;
1660	}
1661	}
1662
1663	static void control_work_handler(struct work_struct *work)
1664	{
1665	struct ports_device *portdev;
1666	struct virtqueue *vq;
1667	struct port_buffer *buf;
1668	unsigned int len;
1669
1670	portdev = container_of(work, struct ports_device, control_work);
1671	vq = portdev->c_ivq;
1672
1673	spin_lock(lock: &portdev->c_ivq_lock);
1674	while ((buf = virtqueue_get_buf(vq, len: &len))) {
1675	spin_unlock(lock: &portdev->c_ivq_lock);
1676
1677	buf->len = min_t(size_t, len, buf->size);
1678	buf->offset = 0;
1679
1680	handle_control_message(vdev: vq->vdev, portdev, buf);
1681
1682	spin_lock(lock: &portdev->c_ivq_lock);
1683	if (add_inbuf(vq: portdev->c_ivq, buf) < 0) {
1684	dev_warn(&portdev->vdev->dev,
1685	"Error adding buffer to queue\n");
1686	free_buf(buf, can_sleep: false);
1687	}
1688	}
1689	spin_unlock(lock: &portdev->c_ivq_lock);
1690	}
1691
1692	static void flush_bufs(struct virtqueue *vq, bool can_sleep)
1693	{
1694	struct port_buffer *buf;
1695	unsigned int len;
1696
1697	while ((buf = virtqueue_get_buf(vq, len: &len)))
1698	free_buf(buf, can_sleep);
1699	}
1700
1701	static void out_intr(struct virtqueue *vq)
1702	{
1703	struct port *port;
1704
1705	port = find_port_by_vq(portdev: vq->vdev->priv, vq);
1706	if (!port) {
1707	flush_bufs(vq, can_sleep: false);
1708	return;
1709	}
1710
1711	wake_up_interruptible(&port->waitqueue);
1712	}
1713
1714	static void in_intr(struct virtqueue *vq)
1715	{
1716	struct port *port;
1717	unsigned long flags;
1718
1719	port = find_port_by_vq(portdev: vq->vdev->priv, vq);
1720	if (!port) {
1721	flush_bufs(vq, can_sleep: false);
1722	return;
1723	}
1724
1725	spin_lock_irqsave(&port->inbuf_lock, flags);
1726	port->inbuf = get_inbuf(port);
1727
1728	/*
1729	* Normally the port should not accept data when the port is
1730	* closed. For generic serial ports, the host won't (shouldn't)
1731	* send data till the guest is connected. But this condition
1732	* can be reached when a console port is not yet connected (no
1733	* tty is spawned) and the other side sends out data over the
1734	* vring, or when a remote devices start sending data before
1735	* the ports are opened.
1736	*
1737	* A generic serial port will discard data if not connected,
1738	* while console ports and rproc-serial ports accepts data at
1739	* any time. rproc-serial is initiated with guest_connected to
1740	* false because port_fops_open expects this. Console ports are
1741	* hooked up with an HVC console and is initialized with
1742	* guest_connected to true.
1743	*/
1744
1745	if (!port->guest_connected && !is_rproc_serial(vdev: port->portdev->vdev))
1746	discard_port_data(port);
1747
1748	/* Send a SIGIO indicating new data in case the process asked for it */
1749	send_sigio_to_port(port);
1750
1751	spin_unlock_irqrestore(lock: &port->inbuf_lock, flags);
1752
1753	wake_up_interruptible(&port->waitqueue);
1754
1755	if (is_console_port(port) && hvc_poll(hp: port->cons.hvc))
1756	hvc_kick();
1757	}
1758
1759	static void control_intr(struct virtqueue *vq)
1760	{
1761	struct ports_device *portdev;
1762
1763	portdev = vq->vdev->priv;
1764	schedule_work(work: &portdev->control_work);
1765	}
1766
1767	static void config_intr(struct virtio_device *vdev)
1768	{
1769	struct ports_device *portdev;
1770
1771	portdev = vdev->priv;
1772
1773	if (!use_multiport(portdev))
1774	schedule_work(work: &portdev->config_work);
1775	}
1776
1777	static void config_work_handler(struct work_struct *work)
1778	{
1779	struct ports_device *portdev;
1780
1781	portdev = container_of(work, struct ports_device, config_work);
1782	if (!use_multiport(portdev)) {
1783	struct virtio_device *vdev;
1784	struct port *port;
1785	u16 rows, cols;
1786
1787	vdev = portdev->vdev;
1788	virtio_cread(vdev, struct virtio_console_config, cols, &cols);
1789	virtio_cread(vdev, struct virtio_console_config, rows, &rows);
1790
1791	port = find_port_by_id(portdev, id: 0);
1792	set_console_size(port, rows, cols);
1793
1794	/*
1795	* We'll use this way of resizing only for legacy
1796	* support. For newer userspace
1797	* (VIRTIO_CONSOLE_F_MULTPORT+), use control messages
1798	* to indicate console size changes so that it can be
1799	* done per-port.
1800	*/
1801	resize_console(port);
1802	}
1803	}
1804
1805	static int init_vqs(struct ports_device *portdev)
1806	{
1807	vq_callback_t **io_callbacks;
1808	char **io_names;
1809	struct virtqueue **vqs;
1810	u32 i, j, nr_ports, nr_queues;
1811	int err;
1812
1813	nr_ports = portdev->max_nr_ports;
1814	nr_queues = use_multiport(portdev) ? (nr_ports + 1) * 2 : 2;
1815
1816	vqs = kmalloc_array(n: nr_queues, size: sizeof(struct virtqueue *), GFP_KERNEL);
1817	io_callbacks = kmalloc_array(n: nr_queues, size: sizeof(vq_callback_t *),
1818	GFP_KERNEL);
1819	io_names = kmalloc_array(n: nr_queues, size: sizeof(char *), GFP_KERNEL);
1820	portdev->in_vqs = kmalloc_array(n: nr_ports, size: sizeof(struct virtqueue *),
1821	GFP_KERNEL);
1822	portdev->out_vqs = kmalloc_array(n: nr_ports, size: sizeof(struct virtqueue *),
1823	GFP_KERNEL);
1824	if (!vqs || !io_callbacks || !io_names || !portdev->in_vqs ||
1825	!portdev->out_vqs) {
1826	err = -ENOMEM;
1827	goto free;
1828	}
1829
1830	/*
1831	* For backward compat (newer host but older guest), the host
1832	* spawns a console port first and also inits the vqs for port
1833	* 0 before others.
1834	*/
1835	j = 0;
1836	io_callbacks[j] = in_intr;
1837	io_callbacks[j + 1] = out_intr;
1838	io_names[j] = "input";
1839	io_names[j + 1] = "output";
1840	j += 2;
1841
1842	if (use_multiport(portdev)) {
1843	io_callbacks[j] = control_intr;
1844	io_callbacks[j + 1] = NULL;
1845	io_names[j] = "control-i";
1846	io_names[j + 1] = "control-o";
1847
1848	for (i = 1; i < nr_ports; i++) {
1849	j += 2;
1850	io_callbacks[j] = in_intr;
1851	io_callbacks[j + 1] = out_intr;
1852	io_names[j] = "input";
1853	io_names[j + 1] = "output";
1854	}
1855	}
1856	/* Find the queues. */
1857	err = virtio_find_vqs(vdev: portdev->vdev, nvqs: nr_queues, vqs,
1858	callbacks: io_callbacks,
1859	names: (const char **)io_names, NULL);
1860	if (err)
1861	goto free;
1862
1863	j = 0;
1864	portdev->in_vqs[0] = vqs[0];
1865	portdev->out_vqs[0] = vqs[1];
1866	j += 2;
1867	if (use_multiport(portdev)) {
1868	portdev->c_ivq = vqs[j];
1869	portdev->c_ovq = vqs[j + 1];
1870
1871	for (i = 1; i < nr_ports; i++) {
1872	j += 2;
1873	portdev->in_vqs[i] = vqs[j];
1874	portdev->out_vqs[i] = vqs[j + 1];
1875	}
1876	}
1877	kfree(objp: io_names);
1878	kfree(objp: io_callbacks);
1879	kfree(objp: vqs);
1880
1881	return 0;
1882
1883	free:
1884	kfree(objp: portdev->out_vqs);
1885	kfree(objp: portdev->in_vqs);
1886	kfree(objp: io_names);
1887	kfree(objp: io_callbacks);
1888	kfree(objp: vqs);
1889
1890	return err;
1891	}
1892
1893	static const struct file_operations portdev_fops = {
1894	.owner = THIS_MODULE,
1895	};
1896
1897	static void remove_vqs(struct ports_device *portdev)
1898	{
1899	struct virtqueue *vq;
1900
1901	virtio_device_for_each_vq(portdev->vdev, vq) {
1902	struct port_buffer *buf;
1903
1904	flush_bufs(vq, can_sleep: true);
1905	while ((buf = virtqueue_detach_unused_buf(vq)))
1906	free_buf(buf, can_sleep: true);
1907	cond_resched();
1908	}
1909	portdev->vdev->config->del_vqs(portdev->vdev);
1910	kfree(objp: portdev->in_vqs);
1911	kfree(objp: portdev->out_vqs);
1912	}
1913
1914	static void virtcons_remove(struct virtio_device *vdev)
1915	{
1916	struct ports_device *portdev;
1917	struct port *port, *port2;
1918
1919	portdev = vdev->priv;
1920
1921	spin_lock_irq(lock: &pdrvdata_lock);
1922	list_del(entry: &portdev->list);
1923	spin_unlock_irq(lock: &pdrvdata_lock);
1924
1925	/* Device is going away, exit any polling for buffers */
1926	virtio_break_device(dev: vdev);
1927	if (use_multiport(portdev))
1928	flush_work(work: &portdev->control_work);
1929	else
1930	flush_work(work: &portdev->config_work);
1931
1932	/* Disable interrupts for vqs */
1933	virtio_reset_device(dev: vdev);
1934	/* Finish up work that's lined up */
1935	if (use_multiport(portdev))
1936	cancel_work_sync(work: &portdev->control_work);
1937	else
1938	cancel_work_sync(work: &portdev->config_work);
1939
1940	list_for_each_entry_safe(port, port2, &portdev->ports, list)
1941	unplug_port(port);
1942
1943	unregister_chrdev(major: portdev->chr_major, name: "virtio-portsdev");
1944
1945	/*
1946	* When yanking out a device, we immediately lose the
1947	* (device-side) queues. So there's no point in keeping the
1948	* guest side around till we drop our final reference. This
1949	* also means that any ports which are in an open state will
1950	* have to just stop using the port, as the vqs are going
1951	* away.
1952	*/
1953	remove_vqs(portdev);
1954	kfree(objp: portdev);
1955	}
1956
1957	/*
1958	* Once we're further in boot, we get probed like any other virtio
1959	* device.
1960	*
1961	* If the host also supports multiple console ports, we check the
1962	* config space to see how many ports the host has spawned. We
1963	* initialize each port found.
1964	*/
1965	static int virtcons_probe(struct virtio_device *vdev)
1966	{
1967	struct ports_device *portdev;
1968	int err;
1969	bool multiport;
1970
1971	/* We only need a config space if features are offered */
1972	if (!vdev->config->get &&
1973	(virtio_has_feature(vdev, VIRTIO_CONSOLE_F_SIZE)
1974	|| virtio_has_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT))) {
1975	dev_err(&vdev->dev, "%s failure: config access disabled\n",
1976	__func__);
1977	return -EINVAL;
1978	}
1979
1980	portdev = kmalloc(size: sizeof(*portdev), GFP_KERNEL);
1981	if (!portdev) {
1982	err = -ENOMEM;
1983	goto fail;
1984	}
1985
1986	/* Attach this portdev to this virtio_device, and vice-versa. */
1987	portdev->vdev = vdev;
1988	vdev->priv = portdev;
1989
1990	portdev->chr_major = register_chrdev(major: 0, name: "virtio-portsdev",
1991	fops: &portdev_fops);
1992	if (portdev->chr_major < 0) {
1993	dev_err(&vdev->dev,
1994	"Error %d registering chrdev for device %u\n",
1995	portdev->chr_major, vdev->index);
1996	err = portdev->chr_major;
1997	goto free;
1998	}
1999
2000	multiport = false;
2001	portdev->max_nr_ports = 1;
2002
2003	/* Don't test MULTIPORT at all if we're rproc: not a valid feature! */
2004	if (!is_rproc_serial(vdev) &&
2005	virtio_cread_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT,
2006	struct virtio_console_config, max_nr_ports,
2007	&portdev->max_nr_ports) == 0) {
2008	if (portdev->max_nr_ports == 0 ||
2009	portdev->max_nr_ports > VIRTCONS_MAX_PORTS) {
2010	dev_err(&vdev->dev,
2011	"Invalidate max_nr_ports %d",
2012	portdev->max_nr_ports);
2013	err = -EINVAL;
2014	goto free;
2015	}
2016	multiport = true;
2017	}
2018
2019	err = init_vqs(portdev);
2020	if (err < 0) {
2021	dev_err(&vdev->dev, "Error %d initializing vqs\n", err);
2022	goto free_chrdev;
2023	}
2024
2025	spin_lock_init(&portdev->ports_lock);
2026	INIT_LIST_HEAD(list: &portdev->ports);
2027	INIT_LIST_HEAD(list: &portdev->list);
2028
2029	virtio_device_ready(dev: portdev->vdev);
2030
2031	INIT_WORK(&portdev->config_work, &config_work_handler);
2032	INIT_WORK(&portdev->control_work, &control_work_handler);
2033
2034	if (multiport) {
2035	spin_lock_init(&portdev->c_ivq_lock);
2036	spin_lock_init(&portdev->c_ovq_lock);
2037
2038	err = fill_queue(vq: portdev->c_ivq, lock: &portdev->c_ivq_lock);
2039	if (err < 0) {
2040	dev_err(&vdev->dev,
2041	"Error allocating buffers for control queue\n");
2042	/*
2043	* The host might want to notify mgmt sw about device
2044	* add failure.
2045	*/
2046	__send_control_msg(portdev, VIRTIO_CONSOLE_BAD_ID,
2047	VIRTIO_CONSOLE_DEVICE_READY, value: 0);
2048	/* Device was functional: we need full cleanup. */
2049	virtcons_remove(vdev);
2050	return err;
2051	}
2052	} else {
2053	/*
2054	* For backward compatibility: Create a console port
2055	* if we're running on older host.
2056	*/
2057	add_port(portdev, id: 0);
2058	}
2059
2060	spin_lock_irq(lock: &pdrvdata_lock);
2061	list_add_tail(new: &portdev->list, head: &pdrvdata.portdevs);
2062	spin_unlock_irq(lock: &pdrvdata_lock);
2063
2064	__send_control_msg(portdev, VIRTIO_CONSOLE_BAD_ID,
2065	VIRTIO_CONSOLE_DEVICE_READY, value: 1);
2066
2067	return 0;
2068
2069	free_chrdev:
2070	unregister_chrdev(major: portdev->chr_major, name: "virtio-portsdev");
2071	free:
2072	kfree(objp: portdev);
2073	fail:
2074	return err;
2075	}
2076
2077	static const struct virtio_device_id id_table[] = {
2078	{ VIRTIO_ID_CONSOLE, VIRTIO_DEV_ANY_ID },
2079	{ 0 },
2080	};
2081	MODULE_DEVICE_TABLE(virtio, id_table);
2082
2083	static const unsigned int features[] = {
2084	VIRTIO_CONSOLE_F_SIZE,
2085	VIRTIO_CONSOLE_F_MULTIPORT,
2086	};
2087
2088	static const struct virtio_device_id rproc_serial_id_table[] = {
2089	#if IS_ENABLED(CONFIG_REMOTEPROC)
2090	{ VIRTIO_ID_RPROC_SERIAL, VIRTIO_DEV_ANY_ID },
2091	#endif
2092	{ 0 },
2093	};
2094	MODULE_DEVICE_TABLE(virtio, rproc_serial_id_table);
2095
2096	static const unsigned int rproc_serial_features[] = {
2097	};
2098
2099	#ifdef CONFIG_PM_SLEEP
2100	static int virtcons_freeze(struct virtio_device *vdev)
2101	{
2102	struct ports_device *portdev;
2103	struct port *port;
2104
2105	portdev = vdev->priv;
2106
2107	virtio_reset_device(dev: vdev);
2108
2109	if (use_multiport(portdev))
2110	virtqueue_disable_cb(vq: portdev->c_ivq);
2111	cancel_work_sync(work: &portdev->control_work);
2112	cancel_work_sync(work: &portdev->config_work);
2113	/*
2114	* Once more: if control_work_handler() was running, it would
2115	* enable the cb as the last step.
2116	*/
2117	if (use_multiport(portdev))
2118	virtqueue_disable_cb(vq: portdev->c_ivq);
2119
2120	list_for_each_entry(port, &portdev->ports, list) {
2121	virtqueue_disable_cb(vq: port->in_vq);
2122	virtqueue_disable_cb(vq: port->out_vq);
2123	/*
2124	* We'll ask the host later if the new invocation has
2125	* the port opened or closed.
2126	*/
2127	port->host_connected = false;
2128	remove_port_data(port);
2129	}
2130	remove_vqs(portdev);
2131
2132	return 0;
2133	}
2134
2135	static int virtcons_restore(struct virtio_device *vdev)
2136	{
2137	struct ports_device *portdev;
2138	struct port *port;
2139	int ret;
2140
2141	portdev = vdev->priv;
2142
2143	ret = init_vqs(portdev);
2144	if (ret)
2145	return ret;
2146
2147	virtio_device_ready(dev: portdev->vdev);
2148
2149	if (use_multiport(portdev))
2150	fill_queue(vq: portdev->c_ivq, lock: &portdev->c_ivq_lock);
2151
2152	list_for_each_entry(port, &portdev->ports, list) {
2153	port->in_vq = portdev->in_vqs[port->id];
2154	port->out_vq = portdev->out_vqs[port->id];
2155
2156	fill_queue(vq: port->in_vq, lock: &port->inbuf_lock);
2157
2158	/* Get port open/close status on the host */
2159	send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, value: 1);
2160
2161	/*
2162	* If a port was open at the time of suspending, we
2163	* have to let the host know that it's still open.
2164	*/
2165	if (port->guest_connected)
2166	send_control_msg(port, VIRTIO_CONSOLE_PORT_OPEN, value: 1);
2167	}
2168	return 0;
2169	}
2170	#endif
2171
2172	static struct virtio_driver virtio_console = {
2173	.feature_table = features,
2174	.feature_table_size = ARRAY_SIZE(features),
2175	.driver.name = KBUILD_MODNAME,
2176	.driver.owner = THIS_MODULE,
2177	.id_table = id_table,
2178	.probe = virtcons_probe,
2179	.remove = virtcons_remove,
2180	.config_changed = config_intr,
2181	#ifdef CONFIG_PM_SLEEP
2182	.freeze = virtcons_freeze,
2183	.restore = virtcons_restore,
2184	#endif
2185	};
2186
2187	static struct virtio_driver virtio_rproc_serial = {
2188	.feature_table = rproc_serial_features,
2189	.feature_table_size = ARRAY_SIZE(rproc_serial_features),
2190	.driver.name = "virtio_rproc_serial",
2191	.driver.owner = THIS_MODULE,
2192	.id_table = rproc_serial_id_table,
2193	.probe = virtcons_probe,
2194	.remove = virtcons_remove,
2195	};
2196
2197	static int __init virtio_console_init(void)
2198	{
2199	int err;
2200
2201	err = class_register(class: &port_class);
2202	if (err)
2203	return err;
2204
2205	pdrvdata.debugfs_dir = debugfs_create_dir(name: "virtio-ports", NULL);
2206	INIT_LIST_HEAD(list: &pdrvdata.consoles);
2207	INIT_LIST_HEAD(list: &pdrvdata.portdevs);
2208
2209	err = register_virtio_driver(&virtio_console);
2210	if (err < 0) {
2211	pr_err("Error %d registering virtio driver\n", err);
2212	goto free;
2213	}
2214	err = register_virtio_driver(&virtio_rproc_serial);
2215	if (err < 0) {
2216	pr_err("Error %d registering virtio rproc serial driver\n",
2217	err);
2218	goto unregister;
2219	}
2220	return 0;
2221	unregister:
2222	unregister_virtio_driver(drv: &virtio_console);
2223	free:
2224	debugfs_remove_recursive(dentry: pdrvdata.debugfs_dir);
2225	class_unregister(class: &port_class);
2226	return err;
2227	}
2228
2229	static void __exit virtio_console_fini(void)
2230	{
2231	reclaim_dma_bufs();
2232
2233	unregister_virtio_driver(drv: &virtio_console);
2234	unregister_virtio_driver(drv: &virtio_rproc_serial);
2235
2236	class_unregister(class: &port_class);
2237	debugfs_remove_recursive(dentry: pdrvdata.debugfs_dir);
2238	}
2239	module_init(virtio_console_init);
2240	module_exit(virtio_console_fini);
2241
2242	MODULE_DESCRIPTION("Virtio console driver");
2243	MODULE_LICENSE("GPL");
2244