1/*
2 * Copyright (C) 2006, 2007, 2009 Rusty Russell, IBM Corporation
3 * Copyright (C) 2009, 2010, 2011 Red Hat, Inc.
4 * Copyright (C) 2009, 2010, 2011 Amit Shah <amit.shah@redhat.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20#include <linux/cdev.h>
21#include <linux/debugfs.h>
22#include <linux/completion.h>
23#include <linux/device.h>
24#include <linux/err.h>
25#include <linux/freezer.h>
26#include <linux/fs.h>
27#include <linux/splice.h>
28#include <linux/pagemap.h>
29#include <linux/init.h>
30#include <linux/list.h>
31#include <linux/poll.h>
32#include <linux/sched.h>
33#include <linux/slab.h>
34#include <linux/spinlock.h>
35#include <linux/virtio.h>
36#include <linux/virtio_console.h>
37#include <linux/wait.h>
38#include <linux/workqueue.h>
39#include <linux/module.h>
40#include <linux/dma-mapping.h>
41#include "../tty/hvc/hvc_console.h"
42
43#define is_rproc_enabled IS_ENABLED(CONFIG_REMOTEPROC)
44
45/*
46 * This is a global struct for storing common data for all the devices
47 * this driver handles.
48 *
49 * Mainly, it has a linked list for all the consoles in one place so
50 * that callbacks from hvc for get_chars(), put_chars() work properly
51 * across multiple devices and multiple ports per device.
52 */
53struct ports_driver_data {
54 /* Used for registering chardevs */
55 struct class *class;
56
57 /* Used for exporting per-port information to debugfs */
58 struct dentry *debugfs_dir;
59
60 /* List of all the devices we're handling */
61 struct list_head portdevs;
62
63 /*
64 * This is used to keep track of the number of hvc consoles
65 * spawned by this driver. This number is given as the first
66 * argument to hvc_alloc(). To correctly map an initial
67 * console spawned via hvc_instantiate to the console being
68 * hooked up via hvc_alloc, we need to pass the same vtermno.
69 *
70 * We also just assume the first console being initialised was
71 * the first one that got used as the initial console.
72 */
73 unsigned int next_vtermno;
74
75 /* All the console devices handled by this driver */
76 struct list_head consoles;
77};
78static struct ports_driver_data pdrvdata;
79
80static DEFINE_SPINLOCK(pdrvdata_lock);
81static DECLARE_COMPLETION(early_console_added);
82
83/* This struct holds information that's relevant only for console ports */
84struct console {
85 /* We'll place all consoles in a list in the pdrvdata struct */
86 struct list_head list;
87
88 /* The hvc device associated with this console port */
89 struct hvc_struct *hvc;
90
91 /* The size of the console */
92 struct winsize ws;
93
94 /*
95 * This number identifies the number that we used to register
96 * with hvc in hvc_instantiate() and hvc_alloc(); this is the
97 * number passed on by the hvc callbacks to us to
98 * differentiate between the other console ports handled by
99 * this driver
100 */
101 u32 vtermno;
102};
103
104struct port_buffer {
105 char *buf;
106
107 /* size of the buffer in *buf above */
108 size_t size;
109
110 /* used length of the buffer */
111 size_t len;
112 /* offset in the buf from which to consume data */
113 size_t offset;
114
115 /* DMA address of buffer */
116 dma_addr_t dma;
117
118 /* Device we got DMA memory from */
119 struct device *dev;
120
121 /* List of pending dma buffers to free */
122 struct list_head list;
123
124 /* If sgpages == 0 then buf is used */
125 unsigned int sgpages;
126
127 /* sg is used if spages > 0. sg must be the last in is struct */
128 struct scatterlist sg[0];
129};
130
131/*
132 * This is a per-device struct that stores data common to all the
133 * ports for that device (vdev->priv).
134 */
135struct ports_device {
136 /* Next portdev in the list, head is in the pdrvdata struct */
137 struct list_head list;
138
139 /*
140 * Workqueue handlers where we process deferred work after
141 * notification
142 */
143 struct work_struct control_work;
144 struct work_struct config_work;
145
146 struct list_head ports;
147
148 /* To protect the list of ports */
149 spinlock_t ports_lock;
150
151 /* To protect the vq operations for the control channel */
152 spinlock_t c_ivq_lock;
153 spinlock_t c_ovq_lock;
154
155 /* max. number of ports this device can hold */
156 u32 max_nr_ports;
157
158 /* The virtio device we're associated with */
159 struct virtio_device *vdev;
160
161 /*
162 * A couple of virtqueues for the control channel: one for
163 * guest->host transfers, one for host->guest transfers
164 */
165 struct virtqueue *c_ivq, *c_ovq;
166
167 /*
168 * A control packet buffer for guest->host requests, protected
169 * by c_ovq_lock.
170 */
171 struct virtio_console_control cpkt;
172
173 /* Array of per-port IO virtqueues */
174 struct virtqueue **in_vqs, **out_vqs;
175
176 /* Major number for this device. Ports will be created as minors. */
177 int chr_major;
178};
179
180struct port_stats {
181 unsigned long bytes_sent, bytes_received, bytes_discarded;
182};
183
184/* This struct holds the per-port data */
185struct port {
186 /* Next port in the list, head is in the ports_device */
187 struct list_head list;
188
189 /* Pointer to the parent virtio_console device */
190 struct ports_device *portdev;
191
192 /* The current buffer from which data has to be fed to readers */
193 struct port_buffer *inbuf;
194
195 /*
196 * To protect the operations on the in_vq associated with this
197 * port. Has to be a spinlock because it can be called from
198 * interrupt context (get_char()).
199 */
200 spinlock_t inbuf_lock;
201
202 /* Protect the operations on the out_vq. */
203 spinlock_t outvq_lock;
204
205 /* The IO vqs for this port */
206 struct virtqueue *in_vq, *out_vq;
207
208 /* File in the debugfs directory that exposes this port's information */
209 struct dentry *debugfs_file;
210
211 /*
212 * Keep count of the bytes sent, received and discarded for
213 * this port for accounting and debugging purposes. These
214 * counts are not reset across port open / close events.
215 */
216 struct port_stats stats;
217
218 /*
219 * The entries in this struct will be valid if this port is
220 * hooked up to an hvc console
221 */
222 struct console cons;
223
224 /* Each port associates with a separate char device */
225 struct cdev *cdev;
226 struct device *dev;
227
228 /* Reference-counting to handle port hot-unplugs and file operations */
229 struct kref kref;
230
231 /* A waitqueue for poll() or blocking read operations */
232 wait_queue_head_t waitqueue;
233
234 /* The 'name' of the port that we expose via sysfs properties */
235 char *name;
236
237 /* We can notify apps of host connect / disconnect events via SIGIO */
238 struct fasync_struct *async_queue;
239
240 /* The 'id' to identify the port with the Host */
241 u32 id;
242
243 bool outvq_full;
244
245 /* Is the host device open */
246 bool host_connected;
247
248 /* We should allow only one process to open a port */
249 bool guest_connected;
250};
251
252/* This is the very early arch-specified put chars function. */
253static int (*early_put_chars)(u32, const char *, int);
254
255static struct port *find_port_by_vtermno(u32 vtermno)
256{
257 struct port *port;
258 struct console *cons;
259 unsigned long flags;
260
261 spin_lock_irqsave(&pdrvdata_lock, flags);
262 list_for_each_entry(cons, &pdrvdata.consoles, list) {
263 if (cons->vtermno == vtermno) {
264 port = container_of(cons, struct port, cons);
265 goto out;
266 }
267 }
268 port = NULL;
269out:
270 spin_unlock_irqrestore(&pdrvdata_lock, flags);
271 return port;
272}
273
274static struct port *find_port_by_devt_in_portdev(struct ports_device *portdev,
275 dev_t dev)
276{
277 struct port *port;
278 unsigned long flags;
279
280 spin_lock_irqsave(&portdev->ports_lock, flags);
281 list_for_each_entry(port, &portdev->ports, list) {
282 if (port->cdev->dev == dev) {
283 kref_get(&port->kref);
284 goto out;
285 }
286 }
287 port = NULL;
288out:
289 spin_unlock_irqrestore(&portdev->ports_lock, flags);
290
291 return port;
292}
293
294static struct port *find_port_by_devt(dev_t dev)
295{
296 struct ports_device *portdev;
297 struct port *port;
298 unsigned long flags;
299
300 spin_lock_irqsave(&pdrvdata_lock, flags);
301 list_for_each_entry(portdev, &pdrvdata.portdevs, list) {
302 port = find_port_by_devt_in_portdev(portdev, dev);
303 if (port)
304 goto out;
305 }
306 port = NULL;
307out:
308 spin_unlock_irqrestore(&pdrvdata_lock, flags);
309 return port;
310}
311
312static struct port *find_port_by_id(struct ports_device *portdev, u32 id)
313{
314 struct port *port;
315 unsigned long flags;
316
317 spin_lock_irqsave(&portdev->ports_lock, flags);
318 list_for_each_entry(port, &portdev->ports, list)
319 if (port->id == id)
320 goto out;
321 port = NULL;
322out:
323 spin_unlock_irqrestore(&portdev->ports_lock, flags);
324
325 return port;
326}
327
328static struct port *find_port_by_vq(struct ports_device *portdev,
329 struct virtqueue *vq)
330{
331 struct port *port;
332 unsigned long flags;
333
334 spin_lock_irqsave(&portdev->ports_lock, flags);
335 list_for_each_entry(port, &portdev->ports, list)
336 if (port->in_vq == vq || port->out_vq == vq)
337 goto out;
338 port = NULL;
339out:
340 spin_unlock_irqrestore(&portdev->ports_lock, flags);
341 return port;
342}
343
344static bool is_console_port(struct port *port)
345{
346 if (port->cons.hvc)
347 return true;
348 return false;
349}
350
351static bool is_rproc_serial(const struct virtio_device *vdev)
352{
353 return is_rproc_enabled && vdev->id.device == VIRTIO_ID_RPROC_SERIAL;
354}
355
356static inline bool use_multiport(struct ports_device *portdev)
357{
358 /*
359 * This condition can be true when put_chars is called from
360 * early_init
361 */
362 if (!portdev->vdev)
363 return false;
364 return __virtio_test_bit(portdev->vdev, VIRTIO_CONSOLE_F_MULTIPORT);
365}
366
367static DEFINE_SPINLOCK(dma_bufs_lock);
368static LIST_HEAD(pending_free_dma_bufs);
369
370static void free_buf(struct port_buffer *buf, bool can_sleep)
371{
372 unsigned int i;
373
374 for (i = 0; i < buf->sgpages; i++) {
375 struct page *page = sg_page(&buf->sg[i]);
376 if (!page)
377 break;
378 put_page(page);
379 }
380
381 if (!buf->dev) {
382 kfree(buf->buf);
383 } else if (is_rproc_enabled) {
384 unsigned long flags;
385
386 /* dma_free_coherent requires interrupts to be enabled. */
387 if (!can_sleep) {
388 /* queue up dma-buffers to be freed later */
389 spin_lock_irqsave(&dma_bufs_lock, flags);
390 list_add_tail(&buf->list, &pending_free_dma_bufs);
391 spin_unlock_irqrestore(&dma_bufs_lock, flags);
392 return;
393 }
394 dma_free_coherent(buf->dev, buf->size, buf->buf, buf->dma);
395
396 /* Release device refcnt and allow it to be freed */
397 put_device(buf->dev);
398 }
399
400 kfree(buf);
401}
402
403static void reclaim_dma_bufs(void)
404{
405 unsigned long flags;
406 struct port_buffer *buf, *tmp;
407 LIST_HEAD(tmp_list);
408
409 if (list_empty(&pending_free_dma_bufs))
410 return;
411
412 /* Create a copy of the pending_free_dma_bufs while holding the lock */
413 spin_lock_irqsave(&dma_bufs_lock, flags);
414 list_cut_position(&tmp_list, &pending_free_dma_bufs,
415 pending_free_dma_bufs.prev);
416 spin_unlock_irqrestore(&dma_bufs_lock, flags);
417
418 /* Release the dma buffers, without irqs enabled */
419 list_for_each_entry_safe(buf, tmp, &tmp_list, list) {
420 list_del(&buf->list);
421 free_buf(buf, true);
422 }
423}
424
425static struct port_buffer *alloc_buf(struct virtio_device *vdev, size_t buf_size,
426 int pages)
427{
428 struct port_buffer *buf;
429
430 reclaim_dma_bufs();
431
432 /*
433 * Allocate buffer and the sg list. The sg list array is allocated
434 * directly after the port_buffer struct.
435 */
436 buf = kmalloc(struct_size(buf, sg, pages), GFP_KERNEL);
437 if (!buf)
438 goto fail;
439
440 buf->sgpages = pages;
441 if (pages > 0) {
442 buf->dev = NULL;
443 buf->buf = NULL;
444 return buf;
445 }
446
447 if (is_rproc_serial(vdev)) {
448 /*
449 * Allocate DMA memory from ancestor. When a virtio
450 * device is created by remoteproc, the DMA memory is
451 * associated with the grandparent device:
452 * vdev => rproc => platform-dev.
453 */
454 if (!vdev->dev.parent || !vdev->dev.parent->parent)
455 goto free_buf;
456 buf->dev = vdev->dev.parent->parent;
457
458 /* Increase device refcnt to avoid freeing it */
459 get_device(buf->dev);
460 buf->buf = dma_alloc_coherent(buf->dev, buf_size, &buf->dma,
461 GFP_KERNEL);
462 } else {
463 buf->dev = NULL;
464 buf->buf = kmalloc(buf_size, GFP_KERNEL);
465 }
466
467 if (!buf->buf)
468 goto free_buf;
469 buf->len = 0;
470 buf->offset = 0;
471 buf->size = buf_size;
472 return buf;
473
474free_buf:
475 kfree(buf);
476fail:
477 return NULL;
478}
479
480/* Callers should take appropriate locks */
481static struct port_buffer *get_inbuf(struct port *port)
482{
483 struct port_buffer *buf;
484 unsigned int len;
485
486 if (port->inbuf)
487 return port->inbuf;
488
489 buf = virtqueue_get_buf(port->in_vq, &len);
490 if (buf) {
491 buf->len = len;
492 buf->offset = 0;
493 port->stats.bytes_received += len;
494 }
495 return buf;
496}
497
498/*
499 * Create a scatter-gather list representing our input buffer and put
500 * it in the queue.
501 *
502 * Callers should take appropriate locks.
503 */
504static int add_inbuf(struct virtqueue *vq, struct port_buffer *buf)
505{
506 struct scatterlist sg[1];
507 int ret;
508
509 sg_init_one(sg, buf->buf, buf->size);
510
511 ret = virtqueue_add_inbuf(vq, sg, 1, buf, GFP_ATOMIC);
512 virtqueue_kick(vq);
513 if (!ret)
514 ret = vq->num_free;
515 return ret;
516}
517
518/* Discard any unread data this port has. Callers lockers. */
519static void discard_port_data(struct port *port)
520{
521 struct port_buffer *buf;
522 unsigned int err;
523
524 if (!port->portdev) {
525 /* Device has been unplugged. vqs are already gone. */
526 return;
527 }
528 buf = get_inbuf(port);
529
530 err = 0;
531 while (buf) {
532 port->stats.bytes_discarded += buf->len - buf->offset;
533 if (add_inbuf(port->in_vq, buf) < 0) {
534 err++;
535 free_buf(buf, false);
536 }
537 port->inbuf = NULL;
538 buf = get_inbuf(port);
539 }
540 if (err)
541 dev_warn(port->dev, "Errors adding %d buffers back to vq\n",
542 err);
543}
544
545static bool port_has_data(struct port *port)
546{
547 unsigned long flags;
548 bool ret;
549
550 ret = false;
551 spin_lock_irqsave(&port->inbuf_lock, flags);
552 port->inbuf = get_inbuf(port);
553 if (port->inbuf)
554 ret = true;
555
556 spin_unlock_irqrestore(&port->inbuf_lock, flags);
557 return ret;
558}
559
560static ssize_t __send_control_msg(struct ports_device *portdev, u32 port_id,
561 unsigned int event, unsigned int value)
562{
563 struct scatterlist sg[1];
564 struct virtqueue *vq;
565 unsigned int len;
566
567 if (!use_multiport(portdev))
568 return 0;
569
570 vq = portdev->c_ovq;
571
572 spin_lock(&portdev->c_ovq_lock);
573
574 portdev->cpkt.id = cpu_to_virtio32(portdev->vdev, port_id);
575 portdev->cpkt.event = cpu_to_virtio16(portdev->vdev, event);
576 portdev->cpkt.value = cpu_to_virtio16(portdev->vdev, value);
577
578 sg_init_one(sg, &portdev->cpkt, sizeof(struct virtio_console_control));
579
580 if (virtqueue_add_outbuf(vq, sg, 1, &portdev->cpkt, GFP_ATOMIC) == 0) {
581 virtqueue_kick(vq);
582 while (!virtqueue_get_buf(vq, &len)
583 && !virtqueue_is_broken(vq))
584 cpu_relax();
585 }
586
587 spin_unlock(&portdev->c_ovq_lock);
588 return 0;
589}
590
591static ssize_t send_control_msg(struct port *port, unsigned int event,
592 unsigned int value)
593{
594 /* Did the port get unplugged before userspace closed it? */
595 if (port->portdev)
596 return __send_control_msg(port->portdev, port->id, event, value);
597 return 0;
598}
599
600
601/* Callers must take the port->outvq_lock */
602static void reclaim_consumed_buffers(struct port *port)
603{
604 struct port_buffer *buf;
605 unsigned int len;
606
607 if (!port->portdev) {
608 /* Device has been unplugged. vqs are already gone. */
609 return;
610 }
611 while ((buf = virtqueue_get_buf(port->out_vq, &len))) {
612 free_buf(buf, false);
613 port->outvq_full = false;
614 }
615}
616
617static ssize_t __send_to_port(struct port *port, struct scatterlist *sg,
618 int nents, size_t in_count,
619 void *data, bool nonblock)
620{
621 struct virtqueue *out_vq;
622 int err;
623 unsigned long flags;
624 unsigned int len;
625
626 out_vq = port->out_vq;
627
628 spin_lock_irqsave(&port->outvq_lock, flags);
629
630 reclaim_consumed_buffers(port);
631
632 err = virtqueue_add_outbuf(out_vq, sg, nents, data, GFP_ATOMIC);
633
634 /* Tell Host to go! */
635 virtqueue_kick(out_vq);
636
637 if (err) {
638 in_count = 0;
639 goto done;
640 }
641
642 if (out_vq->num_free == 0)
643 port->outvq_full = true;
644
645 if (nonblock)
646 goto done;
647
648 /*
649 * Wait till the host acknowledges it pushed out the data we
650 * sent. This is done for data from the hvc_console; the tty
651 * operations are performed with spinlocks held so we can't
652 * sleep here. An alternative would be to copy the data to a
653 * buffer and relax the spinning requirement. The downside is
654 * we need to kmalloc a GFP_ATOMIC buffer each time the
655 * console driver writes something out.
656 */
657 while (!virtqueue_get_buf(out_vq, &len)
658 && !virtqueue_is_broken(out_vq))
659 cpu_relax();
660done:
661 spin_unlock_irqrestore(&port->outvq_lock, flags);
662
663 port->stats.bytes_sent += in_count;
664 /*
665 * We're expected to return the amount of data we wrote -- all
666 * of it
667 */
668 return in_count;
669}
670
671/*
672 * Give out the data that's requested from the buffer that we have
673 * queued up.
674 */
675static ssize_t fill_readbuf(struct port *port, char __user *out_buf,
676 size_t out_count, bool to_user)
677{
678 struct port_buffer *buf;
679 unsigned long flags;
680
681 if (!out_count || !port_has_data(port))
682 return 0;
683
684 buf = port->inbuf;
685 out_count = min(out_count, buf->len - buf->offset);
686
687 if (to_user) {
688 ssize_t ret;
689
690 ret = copy_to_user(out_buf, buf->buf + buf->offset, out_count);
691 if (ret)
692 return -EFAULT;
693 } else {
694 memcpy((__force char *)out_buf, buf->buf + buf->offset,
695 out_count);
696 }
697
698 buf->offset += out_count;
699
700 if (buf->offset == buf->len) {
701 /*
702 * We're done using all the data in this buffer.
703 * Re-queue so that the Host can send us more data.
704 */
705 spin_lock_irqsave(&port->inbuf_lock, flags);
706 port->inbuf = NULL;
707
708 if (add_inbuf(port->in_vq, buf) < 0)
709 dev_warn(port->dev, "failed add_buf\n");
710
711 spin_unlock_irqrestore(&port->inbuf_lock, flags);
712 }
713 /* Return the number of bytes actually copied */
714 return out_count;
715}
716
717/* The condition that must be true for polling to end */
718static bool will_read_block(struct port *port)
719{
720 if (!port->guest_connected) {
721 /* Port got hot-unplugged. Let's exit. */
722 return false;
723 }
724 return !port_has_data(port) && port->host_connected;
725}
726
727static bool will_write_block(struct port *port)
728{
729 bool ret;
730
731 if (!port->guest_connected) {
732 /* Port got hot-unplugged. Let's exit. */
733 return false;
734 }
735 if (!port->host_connected)
736 return true;
737
738 spin_lock_irq(&port->outvq_lock);
739 /*
740 * Check if the Host has consumed any buffers since we last
741 * sent data (this is only applicable for nonblocking ports).
742 */
743 reclaim_consumed_buffers(port);
744 ret = port->outvq_full;
745 spin_unlock_irq(&port->outvq_lock);
746
747 return ret;
748}
749
750static ssize_t port_fops_read(struct file *filp, char __user *ubuf,
751 size_t count, loff_t *offp)
752{
753 struct port *port;
754 ssize_t ret;
755
756 port = filp->private_data;
757
758 /* Port is hot-unplugged. */
759 if (!port->guest_connected)
760 return -ENODEV;
761
762 if (!port_has_data(port)) {
763 /*
764 * If nothing's connected on the host just return 0 in
765 * case of list_empty; this tells the userspace app
766 * that there's no connection
767 */
768 if (!port->host_connected)
769 return 0;
770 if (filp->f_flags & O_NONBLOCK)
771 return -EAGAIN;
772
773 ret = wait_event_freezable(port->waitqueue,
774 !will_read_block(port));
775 if (ret < 0)
776 return ret;
777 }
778 /* Port got hot-unplugged while we were waiting above. */
779 if (!port->guest_connected)
780 return -ENODEV;
781 /*
782 * We could've received a disconnection message while we were
783 * waiting for more data.
784 *
785 * This check is not clubbed in the if() statement above as we
786 * might receive some data as well as the host could get
787 * disconnected after we got woken up from our wait. So we
788 * really want to give off whatever data we have and only then
789 * check for host_connected.
790 */
791 if (!port_has_data(port) && !port->host_connected)
792 return 0;
793
794 return fill_readbuf(port, ubuf, count, true);
795}
796
797static int wait_port_writable(struct port *port, bool nonblock)
798{
799 int ret;
800
801 if (will_write_block(port)) {
802 if (nonblock)
803 return -EAGAIN;
804
805 ret = wait_event_freezable(port->waitqueue,
806 !will_write_block(port));
807 if (ret < 0)
808 return ret;
809 }
810 /* Port got hot-unplugged. */
811 if (!port->guest_connected)
812 return -ENODEV;
813
814 return 0;
815}
816
817static ssize_t port_fops_write(struct file *filp, const char __user *ubuf,
818 size_t count, loff_t *offp)
819{
820 struct port *port;
821 struct port_buffer *buf;
822 ssize_t ret;
823 bool nonblock;
824 struct scatterlist sg[1];
825
826 /* Userspace could be out to fool us */
827 if (!count)
828 return 0;
829
830 port = filp->private_data;
831
832 nonblock = filp->f_flags & O_NONBLOCK;
833
834 ret = wait_port_writable(port, nonblock);
835 if (ret < 0)
836 return ret;
837
838 count = min((size_t)(32 * 1024), count);
839
840 buf = alloc_buf(port->portdev->vdev, count, 0);
841 if (!buf)
842 return -ENOMEM;
843
844 ret = copy_from_user(buf->buf, ubuf, count);
845 if (ret) {
846 ret = -EFAULT;
847 goto free_buf;
848 }
849
850 /*
851 * We now ask send_buf() to not spin for generic ports -- we
852 * can re-use the same code path that non-blocking file
853 * descriptors take for blocking file descriptors since the
854 * wait is already done and we're certain the write will go
855 * through to the host.
856 */
857 nonblock = true;
858 sg_init_one(sg, buf->buf, count);
859 ret = __send_to_port(port, sg, 1, count, buf, nonblock);
860
861 if (nonblock && ret > 0)
862 goto out;
863
864free_buf:
865 free_buf(buf, true);
866out:
867 return ret;
868}
869
870struct sg_list {
871 unsigned int n;
872 unsigned int size;
873 size_t len;
874 struct scatterlist *sg;
875};
876
877static int pipe_to_sg(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
878 struct splice_desc *sd)
879{
880 struct sg_list *sgl = sd->u.data;
881 unsigned int offset, len;
882
883 if (sgl->n == sgl->size)
884 return 0;
885
886 /* Try lock this page */
887 if (pipe_buf_steal(pipe, buf) == 0) {
888 /* Get reference and unlock page for moving */
889 get_page(buf->page);
890 unlock_page(buf->page);
891
892 len = min(buf->len, sd->len);
893 sg_set_page(&(sgl->sg[sgl->n]), buf->page, len, buf->offset);
894 } else {
895 /* Failback to copying a page */
896 struct page *page = alloc_page(GFP_KERNEL);
897 char *src;
898
899 if (!page)
900 return -ENOMEM;
901
902 offset = sd->pos & ~PAGE_MASK;
903
904 len = sd->len;
905 if (len + offset > PAGE_SIZE)
906 len = PAGE_SIZE - offset;
907
908 src = kmap_atomic(buf->page);
909 memcpy(page_address(page) + offset, src + buf->offset, len);
910 kunmap_atomic(src);
911
912 sg_set_page(&(sgl->sg[sgl->n]), page, len, offset);
913 }
914 sgl->n++;
915 sgl->len += len;
916
917 return len;
918}
919
920/* Faster zero-copy write by splicing */
921static ssize_t port_fops_splice_write(struct pipe_inode_info *pipe,
922 struct file *filp, loff_t *ppos,
923 size_t len, unsigned int flags)
924{
925 struct port *port = filp->private_data;
926 struct sg_list sgl;
927 ssize_t ret;
928 struct port_buffer *buf;
929 struct splice_desc sd = {
930 .total_len = len,
931 .flags = flags,
932 .pos = *ppos,
933 .u.data = &sgl,
934 };
935
936 /*
937 * Rproc_serial does not yet support splice. To support splice
938 * pipe_to_sg() must allocate dma-buffers and copy content from
939 * regular pages to dma pages. And alloc_buf and free_buf must
940 * support allocating and freeing such a list of dma-buffers.
941 */
942 if (is_rproc_serial(port->out_vq->vdev))
943 return -EINVAL;
944
945 /*
946 * pipe->nrbufs == 0 means there are no data to transfer,
947 * so this returns just 0 for no data.
948 */
949 pipe_lock(pipe);
950 if (!pipe->nrbufs) {
951 ret = 0;
952 goto error_out;
953 }
954
955 ret = wait_port_writable(port, filp->f_flags & O_NONBLOCK);
956 if (ret < 0)
957 goto error_out;
958
959 buf = alloc_buf(port->portdev->vdev, 0, pipe->nrbufs);
960 if (!buf) {
961 ret = -ENOMEM;
962 goto error_out;
963 }
964
965 sgl.n = 0;
966 sgl.len = 0;
967 sgl.size = pipe->nrbufs;
968 sgl.sg = buf->sg;
969 sg_init_table(sgl.sg, sgl.size);
970 ret = __splice_from_pipe(pipe, &sd, pipe_to_sg);
971 pipe_unlock(pipe);
972 if (likely(ret > 0))
973 ret = __send_to_port(port, buf->sg, sgl.n, sgl.len, buf, true);
974
975 if (unlikely(ret <= 0))
976 free_buf(buf, true);
977 return ret;
978
979error_out:
980 pipe_unlock(pipe);
981 return ret;
982}
983
984static __poll_t port_fops_poll(struct file *filp, poll_table *wait)
985{
986 struct port *port;
987 __poll_t ret;
988
989 port = filp->private_data;
990 poll_wait(filp, &port->waitqueue, wait);
991
992 if (!port->guest_connected) {
993 /* Port got unplugged */
994 return EPOLLHUP;
995 }
996 ret = 0;
997 if (!will_read_block(port))
998 ret |= EPOLLIN | EPOLLRDNORM;
999 if (!will_write_block(port))
1000 ret |= EPOLLOUT;
1001 if (!port->host_connected)
1002 ret |= EPOLLHUP;
1003
1004 return ret;
1005}
1006
1007static void remove_port(struct kref *kref);
1008
1009static int port_fops_release(struct inode *inode, struct file *filp)
1010{
1011 struct port *port;
1012
1013 port = filp->private_data;
1014
1015 /* Notify host of port being closed */
1016 send_control_msg(port, VIRTIO_CONSOLE_PORT_OPEN, 0);
1017
1018 spin_lock_irq(&port->inbuf_lock);
1019 port->guest_connected = false;
1020
1021 discard_port_data(port);
1022
1023 spin_unlock_irq(&port->inbuf_lock);
1024
1025 spin_lock_irq(&port->outvq_lock);
1026 reclaim_consumed_buffers(port);
1027 spin_unlock_irq(&port->outvq_lock);
1028
1029 reclaim_dma_bufs();
1030 /*
1031 * Locks aren't necessary here as a port can't be opened after
1032 * unplug, and if a port isn't unplugged, a kref would already
1033 * exist for the port. Plus, taking ports_lock here would
1034 * create a dependency on other locks taken by functions
1035 * inside remove_port if we're the last holder of the port,
1036 * creating many problems.
1037 */
1038 kref_put(&port->kref, remove_port);
1039
1040 return 0;
1041}
1042
1043static int port_fops_open(struct inode *inode, struct file *filp)
1044{
1045 struct cdev *cdev = inode->i_cdev;
1046 struct port *port;
1047 int ret;
1048
1049 /* We get the port with a kref here */
1050 port = find_port_by_devt(cdev->dev);
1051 if (!port) {
1052 /* Port was unplugged before we could proceed */
1053 return -ENXIO;
1054 }
1055 filp->private_data = port;
1056
1057 /*
1058 * Don't allow opening of console port devices -- that's done
1059 * via /dev/hvc
1060 */
1061 if (is_console_port(port)) {
1062 ret = -ENXIO;
1063 goto out;
1064 }
1065
1066 /* Allow only one process to open a particular port at a time */
1067 spin_lock_irq(&port->inbuf_lock);
1068 if (port->guest_connected) {
1069 spin_unlock_irq(&port->inbuf_lock);
1070 ret = -EBUSY;
1071 goto out;
1072 }
1073
1074 port->guest_connected = true;
1075 spin_unlock_irq(&port->inbuf_lock);
1076
1077 spin_lock_irq(&port->outvq_lock);
1078 /*
1079 * There might be a chance that we missed reclaiming a few
1080 * buffers in the window of the port getting previously closed
1081 * and opening now.
1082 */
1083 reclaim_consumed_buffers(port);
1084 spin_unlock_irq(&port->outvq_lock);
1085
1086 nonseekable_open(inode, filp);
1087
1088 /* Notify host of port being opened */
1089 send_control_msg(filp->private_data, VIRTIO_CONSOLE_PORT_OPEN, 1);
1090
1091 return 0;
1092out:
1093 kref_put(&port->kref, remove_port);
1094 return ret;
1095}
1096
1097static int port_fops_fasync(int fd, struct file *filp, int mode)
1098{
1099 struct port *port;
1100
1101 port = filp->private_data;
1102 return fasync_helper(fd, filp, mode, &port->async_queue);
1103}
1104
1105/*
1106 * The file operations that we support: programs in the guest can open
1107 * a console device, read from it, write to it, poll for data and
1108 * close it. The devices are at
1109 * /dev/vport<device number>p<port number>
1110 */
1111static const struct file_operations port_fops = {
1112 .owner = THIS_MODULE,
1113 .open = port_fops_open,
1114 .read = port_fops_read,
1115 .write = port_fops_write,
1116 .splice_write = port_fops_splice_write,
1117 .poll = port_fops_poll,
1118 .release = port_fops_release,
1119 .fasync = port_fops_fasync,
1120 .llseek = no_llseek,
1121};
1122
1123/*
1124 * The put_chars() callback is pretty straightforward.
1125 *
1126 * We turn the characters into a scatter-gather list, add it to the
1127 * output queue and then kick the Host. Then we sit here waiting for
1128 * it to finish: inefficient in theory, but in practice
1129 * implementations will do it immediately.
1130 */
1131static int put_chars(u32 vtermno, const char *buf, int count)
1132{
1133 struct port *port;
1134 struct scatterlist sg[1];
1135 void *data;
1136 int ret;
1137
1138 if (unlikely(early_put_chars))
1139 return early_put_chars(vtermno, buf, count);
1140
1141 port = find_port_by_vtermno(vtermno);
1142 if (!port)
1143 return -EPIPE;
1144
1145 data = kmemdup(buf, count, GFP_ATOMIC);
1146 if (!data)
1147 return -ENOMEM;
1148
1149 sg_init_one(sg, data, count);
1150 ret = __send_to_port(port, sg, 1, count, data, false);
1151 kfree(data);
1152 return ret;
1153}
1154
1155/*
1156 * get_chars() is the callback from the hvc_console infrastructure
1157 * when an interrupt is received.
1158 *
1159 * We call out to fill_readbuf that gets us the required data from the
1160 * buffers that are queued up.
1161 */
1162static int get_chars(u32 vtermno, char *buf, int count)
1163{
1164 struct port *port;
1165
1166 /* If we've not set up the port yet, we have no input to give. */
1167 if (unlikely(early_put_chars))
1168 return 0;
1169
1170 port = find_port_by_vtermno(vtermno);
1171 if (!port)
1172 return -EPIPE;
1173
1174 /* If we don't have an input queue yet, we can't get input. */
1175 BUG_ON(!port->in_vq);
1176
1177 return fill_readbuf(port, (__force char __user *)buf, count, false);
1178}
1179
1180static void resize_console(struct port *port)
1181{
1182 struct virtio_device *vdev;
1183
1184 /* The port could have been hot-unplugged */
1185 if (!port || !is_console_port(port))
1186 return;
1187
1188 vdev = port->portdev->vdev;
1189
1190 /* Don't test F_SIZE at all if we're rproc: not a valid feature! */
1191 if (!is_rproc_serial(vdev) &&
1192 virtio_has_feature(vdev, VIRTIO_CONSOLE_F_SIZE))
1193 hvc_resize(port->cons.hvc, port->cons.ws);
1194}
1195
1196/* We set the configuration at this point, since we now have a tty */
1197static int notifier_add_vio(struct hvc_struct *hp, int data)
1198{
1199 struct port *port;
1200
1201 port = find_port_by_vtermno(hp->vtermno);
1202 if (!port)
1203 return -EINVAL;
1204
1205 hp->irq_requested = 1;
1206 resize_console(port);
1207
1208 return 0;
1209}
1210
1211static void notifier_del_vio(struct hvc_struct *hp, int data)
1212{
1213 hp->irq_requested = 0;
1214}
1215
1216/* The operations for console ports. */
1217static const struct hv_ops hv_ops = {
1218 .get_chars = get_chars,
1219 .put_chars = put_chars,
1220 .notifier_add = notifier_add_vio,
1221 .notifier_del = notifier_del_vio,
1222 .notifier_hangup = notifier_del_vio,
1223};
1224
1225/*
1226 * Console drivers are initialized very early so boot messages can go
1227 * out, so we do things slightly differently from the generic virtio
1228 * initialization of the net and block drivers.
1229 *
1230 * At this stage, the console is output-only. It's too early to set
1231 * up a virtqueue, so we let the drivers do some boutique early-output
1232 * thing.
1233 */
1234int __init virtio_cons_early_init(int (*put_chars)(u32, const char *, int))
1235{
1236 early_put_chars = put_chars;
1237 return hvc_instantiate(0, 0, &hv_ops);
1238}
1239
1240static int init_port_console(struct port *port)
1241{
1242 int ret;
1243
1244 /*
1245 * The Host's telling us this port is a console port. Hook it
1246 * up with an hvc console.
1247 *
1248 * To set up and manage our virtual console, we call
1249 * hvc_alloc().
1250 *
1251 * The first argument of hvc_alloc() is the virtual console
1252 * number. The second argument is the parameter for the
1253 * notification mechanism (like irq number). We currently
1254 * leave this as zero, virtqueues have implicit notifications.
1255 *
1256 * The third argument is a "struct hv_ops" containing the
1257 * put_chars() get_chars(), notifier_add() and notifier_del()
1258 * pointers. The final argument is the output buffer size: we
1259 * can do any size, so we put PAGE_SIZE here.
1260 */
1261 port->cons.vtermno = pdrvdata.next_vtermno;
1262
1263 port->cons.hvc = hvc_alloc(port->cons.vtermno, 0, &hv_ops, PAGE_SIZE);
1264 if (IS_ERR(port->cons.hvc)) {
1265 ret = PTR_ERR(port->cons.hvc);
1266 dev_err(port->dev,
1267 "error %d allocating hvc for port\n", ret);
1268 port->cons.hvc = NULL;
1269 return ret;
1270 }
1271 spin_lock_irq(&pdrvdata_lock);
1272 pdrvdata.next_vtermno++;
1273 list_add_tail(&port->cons.list, &pdrvdata.consoles);
1274 spin_unlock_irq(&pdrvdata_lock);
1275 port->guest_connected = true;
1276
1277 /*
1278 * Start using the new console output if this is the first
1279 * console to come up.
1280 */
1281 if (early_put_chars)
1282 early_put_chars = NULL;
1283
1284 /* Notify host of port being opened */
1285 send_control_msg(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
1286
1287 return 0;
1288}
1289
1290static ssize_t show_port_name(struct device *dev,
1291 struct device_attribute *attr, char *buffer)
1292{
1293 struct port *port;
1294
1295 port = dev_get_drvdata(dev);
1296
1297 return sprintf(buffer, "%s\n", port->name);
1298}
1299
1300static DEVICE_ATTR(name, S_IRUGO, show_port_name, NULL);
1301
1302static struct attribute *port_sysfs_entries[] = {
1303 &dev_attr_name.attr,
1304 NULL
1305};
1306
1307static const struct attribute_group port_attribute_group = {
1308 .name = NULL, /* put in device directory */
1309 .attrs = port_sysfs_entries,
1310};
1311
1312static int port_debugfs_show(struct seq_file *s, void *data)
1313{
1314 struct port *port = s->private;
1315
1316 seq_printf(s, "name: %s\n", port->name ? port->name : "");
1317 seq_printf(s, "guest_connected: %d\n", port->guest_connected);
1318 seq_printf(s, "host_connected: %d\n", port->host_connected);
1319 seq_printf(s, "outvq_full: %d\n", port->outvq_full);
1320 seq_printf(s, "bytes_sent: %lu\n", port->stats.bytes_sent);
1321 seq_printf(s, "bytes_received: %lu\n", port->stats.bytes_received);
1322 seq_printf(s, "bytes_discarded: %lu\n", port->stats.bytes_discarded);
1323 seq_printf(s, "is_console: %s\n",
1324 is_console_port(port) ? "yes" : "no");
1325 seq_printf(s, "console_vtermno: %u\n", port->cons.vtermno);
1326
1327 return 0;
1328}
1329
1330DEFINE_SHOW_ATTRIBUTE(port_debugfs);
1331
1332static void set_console_size(struct port *port, u16 rows, u16 cols)
1333{
1334 if (!port || !is_console_port(port))
1335 return;
1336
1337 port->cons.ws.ws_row = rows;
1338 port->cons.ws.ws_col = cols;
1339}
1340
1341static unsigned int fill_queue(struct virtqueue *vq, spinlock_t *lock)
1342{
1343 struct port_buffer *buf;
1344 unsigned int nr_added_bufs;
1345 int ret;
1346
1347 nr_added_bufs = 0;
1348 do {
1349 buf = alloc_buf(vq->vdev, PAGE_SIZE, 0);
1350 if (!buf)
1351 break;
1352
1353 spin_lock_irq(lock);
1354 ret = add_inbuf(vq, buf);
1355 if (ret < 0) {
1356 spin_unlock_irq(lock);
1357 free_buf(buf, true);
1358 break;
1359 }
1360 nr_added_bufs++;
1361 spin_unlock_irq(lock);
1362 } while (ret > 0);
1363
1364 return nr_added_bufs;
1365}
1366
1367static void send_sigio_to_port(struct port *port)
1368{
1369 if (port->async_queue && port->guest_connected)
1370 kill_fasync(&port->async_queue, SIGIO, POLL_OUT);
1371}
1372
1373static int add_port(struct ports_device *portdev, u32 id)
1374{
1375 char debugfs_name[16];
1376 struct port *port;
1377 dev_t devt;
1378 unsigned int nr_added_bufs;
1379 int err;
1380
1381 port = kmalloc(sizeof(*port), GFP_KERNEL);
1382 if (!port) {
1383 err = -ENOMEM;
1384 goto fail;
1385 }
1386 kref_init(&port->kref);
1387
1388 port->portdev = portdev;
1389 port->id = id;
1390
1391 port->name = NULL;
1392 port->inbuf = NULL;
1393 port->cons.hvc = NULL;
1394 port->async_queue = NULL;
1395
1396 port->cons.ws.ws_row = port->cons.ws.ws_col = 0;
1397
1398 port->host_connected = port->guest_connected = false;
1399 port->stats = (struct port_stats) { 0 };
1400
1401 port->outvq_full = false;
1402
1403 port->in_vq = portdev->in_vqs[port->id];
1404 port->out_vq = portdev->out_vqs[port->id];
1405
1406 port->cdev = cdev_alloc();
1407 if (!port->cdev) {
1408 dev_err(&port->portdev->vdev->dev, "Error allocating cdev\n");
1409 err = -ENOMEM;
1410 goto free_port;
1411 }
1412 port->cdev->ops = &port_fops;
1413
1414 devt = MKDEV(portdev->chr_major, id);
1415 err = cdev_add(port->cdev, devt, 1);
1416 if (err < 0) {
1417 dev_err(&port->portdev->vdev->dev,
1418 "Error %d adding cdev for port %u\n", err, id);
1419 goto free_cdev;
1420 }
1421 port->dev = device_create(pdrvdata.class, &port->portdev->vdev->dev,
1422 devt, port, "vport%up%u",
1423 port->portdev->vdev->index, id);
1424 if (IS_ERR(port->dev)) {
1425 err = PTR_ERR(port->dev);
1426 dev_err(&port->portdev->vdev->dev,
1427 "Error %d creating device for port %u\n",
1428 err, id);
1429 goto free_cdev;
1430 }
1431
1432 spin_lock_init(&port->inbuf_lock);
1433 spin_lock_init(&port->outvq_lock);
1434 init_waitqueue_head(&port->waitqueue);
1435
1436 /* Fill the in_vq with buffers so the host can send us data. */
1437 nr_added_bufs = fill_queue(port->in_vq, &port->inbuf_lock);
1438 if (!nr_added_bufs) {
1439 dev_err(port->dev, "Error allocating inbufs\n");
1440 err = -ENOMEM;
1441 goto free_device;
1442 }
1443
1444 if (is_rproc_serial(port->portdev->vdev))
1445 /*
1446 * For rproc_serial assume remote processor is connected.
1447 * rproc_serial does not want the console port, only
1448 * the generic port implementation.
1449 */
1450 port->host_connected = true;
1451 else if (!use_multiport(port->portdev)) {
1452 /*
1453 * If we're not using multiport support,
1454 * this has to be a console port.
1455 */
1456 err = init_port_console(port);
1457 if (err)
1458 goto free_inbufs;
1459 }
1460
1461 spin_lock_irq(&portdev->ports_lock);
1462 list_add_tail(&port->list, &port->portdev->ports);
1463 spin_unlock_irq(&portdev->ports_lock);
1464
1465 /*
1466 * Tell the Host we're set so that it can send us various
1467 * configuration parameters for this port (eg, port name,
1468 * caching, whether this is a console port, etc.)
1469 */
1470 send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, 1);
1471
1472 if (pdrvdata.debugfs_dir) {
1473 /*
1474 * Finally, create the debugfs file that we can use to
1475 * inspect a port's state at any time
1476 */
1477 snprintf(debugfs_name, sizeof(debugfs_name), "vport%up%u",
1478 port->portdev->vdev->index, id);
1479 port->debugfs_file = debugfs_create_file(debugfs_name, 0444,
1480 pdrvdata.debugfs_dir,
1481 port,
1482 &port_debugfs_fops);
1483 }
1484 return 0;
1485
1486free_inbufs:
1487free_device:
1488 device_destroy(pdrvdata.class, port->dev->devt);
1489free_cdev:
1490 cdev_del(port->cdev);
1491free_port:
1492 kfree(port);
1493fail:
1494 /* The host might want to notify management sw about port add failure */
1495 __send_control_msg(portdev, id, VIRTIO_CONSOLE_PORT_READY, 0);
1496 return err;
1497}
1498
1499/* No users remain, remove all port-specific data. */
1500static void remove_port(struct kref *kref)
1501{
1502 struct port *port;
1503
1504 port = container_of(kref, struct port, kref);
1505
1506 kfree(port);
1507}
1508
1509static void remove_port_data(struct port *port)
1510{
1511 spin_lock_irq(&port->inbuf_lock);
1512 /* Remove unused data this port might have received. */
1513 discard_port_data(port);
1514 spin_unlock_irq(&port->inbuf_lock);
1515
1516 spin_lock_irq(&port->outvq_lock);
1517 reclaim_consumed_buffers(port);
1518 spin_unlock_irq(&port->outvq_lock);
1519}
1520
1521/*
1522 * Port got unplugged. Remove port from portdev's list and drop the
1523 * kref reference. If no userspace has this port opened, it will
1524 * result in immediate removal the port.
1525 */
1526static void unplug_port(struct port *port)
1527{
1528 spin_lock_irq(&port->portdev->ports_lock);
1529 list_del(&port->list);
1530 spin_unlock_irq(&port->portdev->ports_lock);
1531
1532 spin_lock_irq(&port->inbuf_lock);
1533 if (port->guest_connected) {
1534 /* Let the app know the port is going down. */
1535 send_sigio_to_port(port);
1536
1537 /* Do this after sigio is actually sent */
1538 port->guest_connected = false;
1539 port->host_connected = false;
1540
1541 wake_up_interruptible(&port->waitqueue);
1542 }
1543 spin_unlock_irq(&port->inbuf_lock);
1544
1545 if (is_console_port(port)) {
1546 spin_lock_irq(&pdrvdata_lock);
1547 list_del(&port->cons.list);
1548 spin_unlock_irq(&pdrvdata_lock);
1549 hvc_remove(port->cons.hvc);
1550 }
1551
1552 remove_port_data(port);
1553
1554 /*
1555 * We should just assume the device itself has gone off --
1556 * else a close on an open port later will try to send out a
1557 * control message.
1558 */
1559 port->portdev = NULL;
1560
1561 sysfs_remove_group(&port->dev->kobj, &port_attribute_group);
1562 device_destroy(pdrvdata.class, port->dev->devt);
1563 cdev_del(port->cdev);
1564
1565 debugfs_remove(port->debugfs_file);
1566 kfree(port->name);
1567
1568 /*
1569 * Locks around here are not necessary - a port can't be
1570 * opened after we removed the port struct from ports_list
1571 * above.
1572 */
1573 kref_put(&port->kref, remove_port);
1574}
1575
1576/* Any private messages that the Host and Guest want to share */
1577static void handle_control_message(struct virtio_device *vdev,
1578 struct ports_device *portdev,
1579 struct port_buffer *buf)
1580{
1581 struct virtio_console_control *cpkt;
1582 struct port *port;
1583 size_t name_size;
1584 int err;
1585
1586 cpkt = (struct virtio_console_control *)(buf->buf + buf->offset);
1587
1588 port = find_port_by_id(portdev, virtio32_to_cpu(vdev, cpkt->id));
1589 if (!port &&
1590 cpkt->event != cpu_to_virtio16(vdev, VIRTIO_CONSOLE_PORT_ADD)) {
1591 /* No valid header at start of buffer. Drop it. */
1592 dev_dbg(&portdev->vdev->dev,
1593 "Invalid index %u in control packet\n", cpkt->id);
1594 return;
1595 }
1596
1597 switch (virtio16_to_cpu(vdev, cpkt->event)) {
1598 case VIRTIO_CONSOLE_PORT_ADD:
1599 if (port) {
1600 dev_dbg(&portdev->vdev->dev,
1601 "Port %u already added\n", port->id);
1602 send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, 1);
1603 break;
1604 }
1605 if (virtio32_to_cpu(vdev, cpkt->id) >=
1606 portdev->max_nr_ports) {
1607 dev_warn(&portdev->vdev->dev,
1608 "Request for adding port with "
1609 "out-of-bound id %u, max. supported id: %u\n",
1610 cpkt->id, portdev->max_nr_ports - 1);
1611 break;
1612 }
1613 add_port(portdev, virtio32_to_cpu(vdev, cpkt->id));
1614 break;
1615 case VIRTIO_CONSOLE_PORT_REMOVE:
1616 unplug_port(port);
1617 break;
1618 case VIRTIO_CONSOLE_CONSOLE_PORT:
1619 if (!cpkt->value)
1620 break;
1621 if (is_console_port(port))
1622 break;
1623
1624 init_port_console(port);
1625 complete(&early_console_added);
1626 /*
1627 * Could remove the port here in case init fails - but
1628 * have to notify the host first.
1629 */
1630 break;
1631 case VIRTIO_CONSOLE_RESIZE: {
1632 struct {
1633 __u16 rows;
1634 __u16 cols;
1635 } size;
1636
1637 if (!is_console_port(port))
1638 break;
1639
1640 memcpy(&size, buf->buf + buf->offset + sizeof(*cpkt),
1641 sizeof(size));
1642 set_console_size(port, size.rows, size.cols);
1643
1644 port->cons.hvc->irq_requested = 1;
1645 resize_console(port);
1646 break;
1647 }
1648 case VIRTIO_CONSOLE_PORT_OPEN:
1649 port->host_connected = virtio16_to_cpu(vdev, cpkt->value);
1650 wake_up_interruptible(&port->waitqueue);
1651 /*
1652 * If the host port got closed and the host had any
1653 * unconsumed buffers, we'll be able to reclaim them
1654 * now.
1655 */
1656 spin_lock_irq(&port->outvq_lock);
1657 reclaim_consumed_buffers(port);
1658 spin_unlock_irq(&port->outvq_lock);
1659
1660 /*
1661 * If the guest is connected, it'll be interested in
1662 * knowing the host connection state changed.
1663 */
1664 spin_lock_irq(&port->inbuf_lock);
1665 send_sigio_to_port(port);
1666 spin_unlock_irq(&port->inbuf_lock);
1667 break;
1668 case VIRTIO_CONSOLE_PORT_NAME:
1669 /*
1670 * If we woke up after hibernation, we can get this
1671 * again. Skip it in that case.
1672 */
1673 if (port->name)
1674 break;
1675
1676 /*
1677 * Skip the size of the header and the cpkt to get the size
1678 * of the name that was sent
1679 */
1680 name_size = buf->len - buf->offset - sizeof(*cpkt) + 1;
1681
1682 port->name = kmalloc(name_size, GFP_KERNEL);
1683 if (!port->name) {
1684 dev_err(port->dev,
1685 "Not enough space to store port name\n");
1686 break;
1687 }
1688 strncpy(port->name, buf->buf + buf->offset + sizeof(*cpkt),
1689 name_size - 1);
1690 port->name[name_size - 1] = 0;
1691
1692 /*
1693 * Since we only have one sysfs attribute, 'name',
1694 * create it only if we have a name for the port.
1695 */
1696 err = sysfs_create_group(&port->dev->kobj,
1697 &port_attribute_group);
1698 if (err) {
1699 dev_err(port->dev,
1700 "Error %d creating sysfs device attributes\n",
1701 err);
1702 } else {
1703 /*
1704 * Generate a udev event so that appropriate
1705 * symlinks can be created based on udev
1706 * rules.
1707 */
1708 kobject_uevent(&port->dev->kobj, KOBJ_CHANGE);
1709 }
1710 break;
1711 }
1712}
1713
1714static void control_work_handler(struct work_struct *work)
1715{
1716 struct ports_device *portdev;
1717 struct virtqueue *vq;
1718 struct port_buffer *buf;
1719 unsigned int len;
1720
1721 portdev = container_of(work, struct ports_device, control_work);
1722 vq = portdev->c_ivq;
1723
1724 spin_lock(&portdev->c_ivq_lock);
1725 while ((buf = virtqueue_get_buf(vq, &len))) {
1726 spin_unlock(&portdev->c_ivq_lock);
1727
1728 buf->len = len;
1729 buf->offset = 0;
1730
1731 handle_control_message(vq->vdev, portdev, buf);
1732
1733 spin_lock(&portdev->c_ivq_lock);
1734 if (add_inbuf(portdev->c_ivq, buf) < 0) {
1735 dev_warn(&portdev->vdev->dev,
1736 "Error adding buffer to queue\n");
1737 free_buf(buf, false);
1738 }
1739 }
1740 spin_unlock(&portdev->c_ivq_lock);
1741}
1742
1743static void flush_bufs(struct virtqueue *vq, bool can_sleep)
1744{
1745 struct port_buffer *buf;
1746 unsigned int len;
1747
1748 while ((buf = virtqueue_get_buf(vq, &len)))
1749 free_buf(buf, can_sleep);
1750}
1751
1752static void out_intr(struct virtqueue *vq)
1753{
1754 struct port *port;
1755
1756 port = find_port_by_vq(vq->vdev->priv, vq);
1757 if (!port) {
1758 flush_bufs(vq, false);
1759 return;
1760 }
1761
1762 wake_up_interruptible(&port->waitqueue);
1763}
1764
1765static void in_intr(struct virtqueue *vq)
1766{
1767 struct port *port;
1768 unsigned long flags;
1769
1770 port = find_port_by_vq(vq->vdev->priv, vq);
1771 if (!port) {
1772 flush_bufs(vq, false);
1773 return;
1774 }
1775
1776 spin_lock_irqsave(&port->inbuf_lock, flags);
1777 port->inbuf = get_inbuf(port);
1778
1779 /*
1780 * Normally the port should not accept data when the port is
1781 * closed. For generic serial ports, the host won't (shouldn't)
1782 * send data till the guest is connected. But this condition
1783 * can be reached when a console port is not yet connected (no
1784 * tty is spawned) and the other side sends out data over the
1785 * vring, or when a remote devices start sending data before
1786 * the ports are opened.
1787 *
1788 * A generic serial port will discard data if not connected,
1789 * while console ports and rproc-serial ports accepts data at
1790 * any time. rproc-serial is initiated with guest_connected to
1791 * false because port_fops_open expects this. Console ports are
1792 * hooked up with an HVC console and is initialized with
1793 * guest_connected to true.
1794 */
1795
1796 if (!port->guest_connected && !is_rproc_serial(port->portdev->vdev))
1797 discard_port_data(port);
1798
1799 /* Send a SIGIO indicating new data in case the process asked for it */
1800 send_sigio_to_port(port);
1801
1802 spin_unlock_irqrestore(&port->inbuf_lock, flags);
1803
1804 wake_up_interruptible(&port->waitqueue);
1805
1806 if (is_console_port(port) && hvc_poll(port->cons.hvc))
1807 hvc_kick();
1808}
1809
1810static void control_intr(struct virtqueue *vq)
1811{
1812 struct ports_device *portdev;
1813
1814 portdev = vq->vdev->priv;
1815 schedule_work(&portdev->control_work);
1816}
1817
1818static void config_intr(struct virtio_device *vdev)
1819{
1820 struct ports_device *portdev;
1821
1822 portdev = vdev->priv;
1823
1824 if (!use_multiport(portdev))
1825 schedule_work(&portdev->config_work);
1826}
1827
1828static void config_work_handler(struct work_struct *work)
1829{
1830 struct ports_device *portdev;
1831
1832 portdev = container_of(work, struct ports_device, config_work);
1833 if (!use_multiport(portdev)) {
1834 struct virtio_device *vdev;
1835 struct port *port;
1836 u16 rows, cols;
1837
1838 vdev = portdev->vdev;
1839 virtio_cread(vdev, struct virtio_console_config, cols, &cols);
1840 virtio_cread(vdev, struct virtio_console_config, rows, &rows);
1841
1842 port = find_port_by_id(portdev, 0);
1843 set_console_size(port, rows, cols);
1844
1845 /*
1846 * We'll use this way of resizing only for legacy
1847 * support. For newer userspace
1848 * (VIRTIO_CONSOLE_F_MULTPORT+), use control messages
1849 * to indicate console size changes so that it can be
1850 * done per-port.
1851 */
1852 resize_console(port);
1853 }
1854}
1855
1856static int init_vqs(struct ports_device *portdev)
1857{
1858 vq_callback_t **io_callbacks;
1859 char **io_names;
1860 struct virtqueue **vqs;
1861 u32 i, j, nr_ports, nr_queues;
1862 int err;
1863
1864 nr_ports = portdev->max_nr_ports;
1865 nr_queues = use_multiport(portdev) ? (nr_ports + 1) * 2 : 2;
1866
1867 vqs = kmalloc_array(nr_queues, sizeof(struct virtqueue *), GFP_KERNEL);
1868 io_callbacks = kmalloc_array(nr_queues, sizeof(vq_callback_t *),
1869 GFP_KERNEL);
1870 io_names = kmalloc_array(nr_queues, sizeof(char *), GFP_KERNEL);
1871 portdev->in_vqs = kmalloc_array(nr_ports, sizeof(struct virtqueue *),
1872 GFP_KERNEL);
1873 portdev->out_vqs = kmalloc_array(nr_ports, sizeof(struct virtqueue *),
1874 GFP_KERNEL);
1875 if (!vqs || !io_callbacks || !io_names || !portdev->in_vqs ||
1876 !portdev->out_vqs) {
1877 err = -ENOMEM;
1878 goto free;
1879 }
1880
1881 /*
1882 * For backward compat (newer host but older guest), the host
1883 * spawns a console port first and also inits the vqs for port
1884 * 0 before others.
1885 */
1886 j = 0;
1887 io_callbacks[j] = in_intr;
1888 io_callbacks[j + 1] = out_intr;
1889 io_names[j] = "input";
1890 io_names[j + 1] = "output";
1891 j += 2;
1892
1893 if (use_multiport(portdev)) {
1894 io_callbacks[j] = control_intr;
1895 io_callbacks[j + 1] = NULL;
1896 io_names[j] = "control-i";
1897 io_names[j + 1] = "control-o";
1898
1899 for (i = 1; i < nr_ports; i++) {
1900 j += 2;
1901 io_callbacks[j] = in_intr;
1902 io_callbacks[j + 1] = out_intr;
1903 io_names[j] = "input";
1904 io_names[j + 1] = "output";
1905 }
1906 }
1907 /* Find the queues. */
1908 err = virtio_find_vqs(portdev->vdev, nr_queues, vqs,
1909 io_callbacks,
1910 (const char **)io_names, NULL);
1911 if (err)
1912 goto free;
1913
1914 j = 0;
1915 portdev->in_vqs[0] = vqs[0];
1916 portdev->out_vqs[0] = vqs[1];
1917 j += 2;
1918 if (use_multiport(portdev)) {
1919 portdev->c_ivq = vqs[j];
1920 portdev->c_ovq = vqs[j + 1];
1921
1922 for (i = 1; i < nr_ports; i++) {
1923 j += 2;
1924 portdev->in_vqs[i] = vqs[j];
1925 portdev->out_vqs[i] = vqs[j + 1];
1926 }
1927 }
1928 kfree(io_names);
1929 kfree(io_callbacks);
1930 kfree(vqs);
1931
1932 return 0;
1933
1934free:
1935 kfree(portdev->out_vqs);
1936 kfree(portdev->in_vqs);
1937 kfree(io_names);
1938 kfree(io_callbacks);
1939 kfree(vqs);
1940
1941 return err;
1942}
1943
1944static const struct file_operations portdev_fops = {
1945 .owner = THIS_MODULE,
1946};
1947
1948static void remove_vqs(struct ports_device *portdev)
1949{
1950 struct virtqueue *vq;
1951
1952 virtio_device_for_each_vq(portdev->vdev, vq) {
1953 struct port_buffer *buf;
1954
1955 flush_bufs(vq, true);
1956 while ((buf = virtqueue_detach_unused_buf(vq)))
1957 free_buf(buf, true);
1958 }
1959 portdev->vdev->config->del_vqs(portdev->vdev);
1960 kfree(portdev->in_vqs);
1961 kfree(portdev->out_vqs);
1962}
1963
1964static void virtcons_remove(struct virtio_device *vdev)
1965{
1966 struct ports_device *portdev;
1967 struct port *port, *port2;
1968
1969 portdev = vdev->priv;
1970
1971 spin_lock_irq(&pdrvdata_lock);
1972 list_del(&portdev->list);
1973 spin_unlock_irq(&pdrvdata_lock);
1974
1975 /* Disable interrupts for vqs */
1976 vdev->config->reset(vdev);
1977 /* Finish up work that's lined up */
1978 if (use_multiport(portdev))
1979 cancel_work_sync(&portdev->control_work);
1980 else
1981 cancel_work_sync(&portdev->config_work);
1982
1983 list_for_each_entry_safe(port, port2, &portdev->ports, list)
1984 unplug_port(port);
1985
1986 unregister_chrdev(portdev->chr_major, "virtio-portsdev");
1987
1988 /*
1989 * When yanking out a device, we immediately lose the
1990 * (device-side) queues. So there's no point in keeping the
1991 * guest side around till we drop our final reference. This
1992 * also means that any ports which are in an open state will
1993 * have to just stop using the port, as the vqs are going
1994 * away.
1995 */
1996 remove_vqs(portdev);
1997 kfree(portdev);
1998}
1999
2000/*
2001 * Once we're further in boot, we get probed like any other virtio
2002 * device.
2003 *
2004 * If the host also supports multiple console ports, we check the
2005 * config space to see how many ports the host has spawned. We
2006 * initialize each port found.
2007 */
2008static int virtcons_probe(struct virtio_device *vdev)
2009{
2010 struct ports_device *portdev;
2011 int err;
2012 bool multiport;
2013 bool early = early_put_chars != NULL;
2014
2015 /* We only need a config space if features are offered */
2016 if (!vdev->config->get &&
2017 (virtio_has_feature(vdev, VIRTIO_CONSOLE_F_SIZE)
2018 || virtio_has_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT))) {
2019 dev_err(&vdev->dev, "%s failure: config access disabled\n",
2020 __func__);
2021 return -EINVAL;
2022 }
2023
2024 /* Ensure to read early_put_chars now */
2025 barrier();
2026
2027 portdev = kmalloc(sizeof(*portdev), GFP_KERNEL);
2028 if (!portdev) {
2029 err = -ENOMEM;
2030 goto fail;
2031 }
2032
2033 /* Attach this portdev to this virtio_device, and vice-versa. */
2034 portdev->vdev = vdev;
2035 vdev->priv = portdev;
2036
2037 portdev->chr_major = register_chrdev(0, "virtio-portsdev",
2038 &portdev_fops);
2039 if (portdev->chr_major < 0) {
2040 dev_err(&vdev->dev,
2041 "Error %d registering chrdev for device %u\n",
2042 portdev->chr_major, vdev->index);
2043 err = portdev->chr_major;
2044 goto free;
2045 }
2046
2047 multiport = false;
2048 portdev->max_nr_ports = 1;
2049
2050 /* Don't test MULTIPORT at all if we're rproc: not a valid feature! */
2051 if (!is_rproc_serial(vdev) &&
2052 virtio_cread_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT,
2053 struct virtio_console_config, max_nr_ports,
2054 &portdev->max_nr_ports) == 0) {
2055 multiport = true;
2056 }
2057
2058 err = init_vqs(portdev);
2059 if (err < 0) {
2060 dev_err(&vdev->dev, "Error %d initializing vqs\n", err);
2061 goto free_chrdev;
2062 }
2063
2064 spin_lock_init(&portdev->ports_lock);
2065 INIT_LIST_HEAD(&portdev->ports);
2066 INIT_LIST_HEAD(&portdev->list);
2067
2068 virtio_device_ready(portdev->vdev);
2069
2070 INIT_WORK(&portdev->config_work, &config_work_handler);
2071 INIT_WORK(&portdev->control_work, &control_work_handler);
2072
2073 if (multiport) {
2074 unsigned int nr_added_bufs;
2075
2076 spin_lock_init(&portdev->c_ivq_lock);
2077 spin_lock_init(&portdev->c_ovq_lock);
2078
2079 nr_added_bufs = fill_queue(portdev->c_ivq,
2080 &portdev->c_ivq_lock);
2081 if (!nr_added_bufs) {
2082 dev_err(&vdev->dev,
2083 "Error allocating buffers for control queue\n");
2084 /*
2085 * The host might want to notify mgmt sw about device
2086 * add failure.
2087 */
2088 __send_control_msg(portdev, VIRTIO_CONSOLE_BAD_ID,
2089 VIRTIO_CONSOLE_DEVICE_READY, 0);
2090 /* Device was functional: we need full cleanup. */
2091 virtcons_remove(vdev);
2092 return -ENOMEM;
2093 }
2094 } else {
2095 /*
2096 * For backward compatibility: Create a console port
2097 * if we're running on older host.
2098 */
2099 add_port(portdev, 0);
2100 }
2101
2102 spin_lock_irq(&pdrvdata_lock);
2103 list_add_tail(&portdev->list, &pdrvdata.portdevs);
2104 spin_unlock_irq(&pdrvdata_lock);
2105
2106 __send_control_msg(portdev, VIRTIO_CONSOLE_BAD_ID,
2107 VIRTIO_CONSOLE_DEVICE_READY, 1);
2108
2109 /*
2110 * If there was an early virtio console, assume that there are no
2111 * other consoles. We need to wait until the hvc_alloc matches the
2112 * hvc_instantiate, otherwise tty_open will complain, resulting in
2113 * a "Warning: unable to open an initial console" boot failure.
2114 * Without multiport this is done in add_port above. With multiport
2115 * this might take some host<->guest communication - thus we have to
2116 * wait.
2117 */
2118 if (multiport && early)
2119 wait_for_completion(&early_console_added);
2120
2121 return 0;
2122
2123free_chrdev:
2124 unregister_chrdev(portdev->chr_major, "virtio-portsdev");
2125free:
2126 kfree(portdev);
2127fail:
2128 return err;
2129}
2130
2131static struct virtio_device_id id_table[] = {
2132 { VIRTIO_ID_CONSOLE, VIRTIO_DEV_ANY_ID },
2133 { 0 },
2134};
2135
2136static unsigned int features[] = {
2137 VIRTIO_CONSOLE_F_SIZE,
2138 VIRTIO_CONSOLE_F_MULTIPORT,
2139};
2140
2141static struct virtio_device_id rproc_serial_id_table[] = {
2142#if IS_ENABLED(CONFIG_REMOTEPROC)
2143 { VIRTIO_ID_RPROC_SERIAL, VIRTIO_DEV_ANY_ID },
2144#endif
2145 { 0 },
2146};
2147
2148static unsigned int rproc_serial_features[] = {
2149};
2150
2151#ifdef CONFIG_PM_SLEEP
2152static int virtcons_freeze(struct virtio_device *vdev)
2153{
2154 struct ports_device *portdev;
2155 struct port *port;
2156
2157 portdev = vdev->priv;
2158
2159 vdev->config->reset(vdev);
2160
2161 if (use_multiport(portdev))
2162 virtqueue_disable_cb(portdev->c_ivq);
2163 cancel_work_sync(&portdev->control_work);
2164 cancel_work_sync(&portdev->config_work);
2165 /*
2166 * Once more: if control_work_handler() was running, it would
2167 * enable the cb as the last step.
2168 */
2169 if (use_multiport(portdev))
2170 virtqueue_disable_cb(portdev->c_ivq);
2171
2172 list_for_each_entry(port, &portdev->ports, list) {
2173 virtqueue_disable_cb(port->in_vq);
2174 virtqueue_disable_cb(port->out_vq);
2175 /*
2176 * We'll ask the host later if the new invocation has
2177 * the port opened or closed.
2178 */
2179 port->host_connected = false;
2180 remove_port_data(port);
2181 }
2182 remove_vqs(portdev);
2183
2184 return 0;
2185}
2186
2187static int virtcons_restore(struct virtio_device *vdev)
2188{
2189 struct ports_device *portdev;
2190 struct port *port;
2191 int ret;
2192
2193 portdev = vdev->priv;
2194
2195 ret = init_vqs(portdev);
2196 if (ret)
2197 return ret;
2198
2199 virtio_device_ready(portdev->vdev);
2200
2201 if (use_multiport(portdev))
2202 fill_queue(portdev->c_ivq, &portdev->c_ivq_lock);
2203
2204 list_for_each_entry(port, &portdev->ports, list) {
2205 port->in_vq = portdev->in_vqs[port->id];
2206 port->out_vq = portdev->out_vqs[port->id];
2207
2208 fill_queue(port->in_vq, &port->inbuf_lock);
2209
2210 /* Get port open/close status on the host */
2211 send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, 1);
2212
2213 /*
2214 * If a port was open at the time of suspending, we
2215 * have to let the host know that it's still open.
2216 */
2217 if (port->guest_connected)
2218 send_control_msg(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
2219 }
2220 return 0;
2221}
2222#endif
2223
2224static struct virtio_driver virtio_console = {
2225 .feature_table = features,
2226 .feature_table_size = ARRAY_SIZE(features),
2227 .driver.name = KBUILD_MODNAME,
2228 .driver.owner = THIS_MODULE,
2229 .id_table = id_table,
2230 .probe = virtcons_probe,
2231 .remove = virtcons_remove,
2232 .config_changed = config_intr,
2233#ifdef CONFIG_PM_SLEEP
2234 .freeze = virtcons_freeze,
2235 .restore = virtcons_restore,
2236#endif
2237};
2238
2239static struct virtio_driver virtio_rproc_serial = {
2240 .feature_table = rproc_serial_features,
2241 .feature_table_size = ARRAY_SIZE(rproc_serial_features),
2242 .driver.name = "virtio_rproc_serial",
2243 .driver.owner = THIS_MODULE,
2244 .id_table = rproc_serial_id_table,
2245 .probe = virtcons_probe,
2246 .remove = virtcons_remove,
2247};
2248
2249static int __init init(void)
2250{
2251 int err;
2252
2253 pdrvdata.class = class_create(THIS_MODULE, "virtio-ports");
2254 if (IS_ERR(pdrvdata.class)) {
2255 err = PTR_ERR(pdrvdata.class);
2256 pr_err("Error %d creating virtio-ports class\n", err);
2257 return err;
2258 }
2259
2260 pdrvdata.debugfs_dir = debugfs_create_dir("virtio-ports", NULL);
2261 if (!pdrvdata.debugfs_dir)
2262 pr_warn("Error creating debugfs dir for virtio-ports\n");
2263 INIT_LIST_HEAD(&pdrvdata.consoles);
2264 INIT_LIST_HEAD(&pdrvdata.portdevs);
2265
2266 err = register_virtio_driver(&virtio_console);
2267 if (err < 0) {
2268 pr_err("Error %d registering virtio driver\n", err);
2269 goto free;
2270 }
2271 err = register_virtio_driver(&virtio_rproc_serial);
2272 if (err < 0) {
2273 pr_err("Error %d registering virtio rproc serial driver\n",
2274 err);
2275 goto unregister;
2276 }
2277 return 0;
2278unregister:
2279 unregister_virtio_driver(&virtio_console);
2280free:
2281 debugfs_remove_recursive(pdrvdata.debugfs_dir);
2282 class_destroy(pdrvdata.class);
2283 return err;
2284}
2285
2286static void __exit fini(void)
2287{
2288 reclaim_dma_bufs();
2289
2290 unregister_virtio_driver(&virtio_console);
2291 unregister_virtio_driver(&virtio_rproc_serial);
2292
2293 class_destroy(pdrvdata.class);
2294 debugfs_remove_recursive(pdrvdata.debugfs_dir);
2295}
2296module_init(init);
2297module_exit(fini);
2298
2299MODULE_DEVICE_TABLE(virtio, id_table);
2300MODULE_DESCRIPTION("Virtio console driver");
2301MODULE_LICENSE("GPL");
2302