1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Physical device callbacks for vfio_ccw
4 *
5 * Copyright IBM Corp. 2017
6 * Copyright Red Hat, Inc. 2019
7 *
8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com>
9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com>
10 * Cornelia Huck <cohuck@redhat.com>
11 */
12
13#include <linux/vfio.h>
14#include <linux/nospec.h>
15#include <linux/slab.h>
16
17#include "vfio_ccw_private.h"
18
19static const struct vfio_device_ops vfio_ccw_dev_ops;
20
21static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private)
22{
23 /*
24 * If the FSM state is seen as Not Operational after closing
25 * and re-opening the mdev, return an error.
26 */
27 vfio_ccw_fsm_event(private, event: VFIO_CCW_EVENT_CLOSE);
28 vfio_ccw_fsm_event(private, event: VFIO_CCW_EVENT_OPEN);
29 if (private->state == VFIO_CCW_STATE_NOT_OPER)
30 return -EINVAL;
31
32 return 0;
33}
34
35static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length)
36{
37 struct vfio_ccw_private *private =
38 container_of(vdev, struct vfio_ccw_private, vdev);
39
40 /* Drivers MUST unpin pages in response to an invalidation. */
41 if (!cp_iova_pinned(cp: &private->cp, iova, length))
42 return;
43
44 vfio_ccw_mdev_reset(private);
45}
46
47static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev)
48{
49 struct vfio_ccw_private *private =
50 container_of(vdev, struct vfio_ccw_private, vdev);
51
52 mutex_init(&private->io_mutex);
53 private->state = VFIO_CCW_STATE_STANDBY;
54 INIT_LIST_HEAD(list: &private->crw);
55 INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo);
56 INIT_WORK(&private->crw_work, vfio_ccw_crw_todo);
57
58 private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1),
59 GFP_KERNEL);
60 if (!private->cp.guest_cp)
61 goto out_free_private;
62
63 private->io_region = kmem_cache_zalloc(k: vfio_ccw_io_region,
64 GFP_KERNEL | GFP_DMA);
65 if (!private->io_region)
66 goto out_free_cp;
67
68 private->cmd_region = kmem_cache_zalloc(k: vfio_ccw_cmd_region,
69 GFP_KERNEL | GFP_DMA);
70 if (!private->cmd_region)
71 goto out_free_io;
72
73 private->schib_region = kmem_cache_zalloc(k: vfio_ccw_schib_region,
74 GFP_KERNEL | GFP_DMA);
75 if (!private->schib_region)
76 goto out_free_cmd;
77
78 private->crw_region = kmem_cache_zalloc(k: vfio_ccw_crw_region,
79 GFP_KERNEL | GFP_DMA);
80 if (!private->crw_region)
81 goto out_free_schib;
82
83 return 0;
84
85out_free_schib:
86 kmem_cache_free(s: vfio_ccw_schib_region, objp: private->schib_region);
87out_free_cmd:
88 kmem_cache_free(s: vfio_ccw_cmd_region, objp: private->cmd_region);
89out_free_io:
90 kmem_cache_free(s: vfio_ccw_io_region, objp: private->io_region);
91out_free_cp:
92 kfree(objp: private->cp.guest_cp);
93out_free_private:
94 mutex_destroy(lock: &private->io_mutex);
95 return -ENOMEM;
96}
97
98static int vfio_ccw_mdev_probe(struct mdev_device *mdev)
99{
100 struct subchannel *sch = to_subchannel(mdev->dev.parent);
101 struct vfio_ccw_parent *parent = dev_get_drvdata(dev: &sch->dev);
102 struct vfio_ccw_private *private;
103 int ret;
104
105 private = vfio_alloc_device(vfio_ccw_private, vdev, &mdev->dev,
106 &vfio_ccw_dev_ops);
107 if (IS_ERR(ptr: private))
108 return PTR_ERR(ptr: private);
109
110 dev_set_drvdata(dev: &parent->dev, data: private);
111
112 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n",
113 sch->schid.cssid,
114 sch->schid.ssid,
115 sch->schid.sch_no);
116
117 ret = vfio_register_emulated_iommu_dev(device: &private->vdev);
118 if (ret)
119 goto err_put_vdev;
120 dev_set_drvdata(dev: &mdev->dev, data: private);
121 return 0;
122
123err_put_vdev:
124 dev_set_drvdata(dev: &parent->dev, NULL);
125 vfio_put_device(device: &private->vdev);
126 return ret;
127}
128
129static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev)
130{
131 struct vfio_ccw_private *private =
132 container_of(vdev, struct vfio_ccw_private, vdev);
133 struct vfio_ccw_crw *crw, *temp;
134
135 list_for_each_entry_safe(crw, temp, &private->crw, next) {
136 list_del(entry: &crw->next);
137 kfree(objp: crw);
138 }
139
140 kmem_cache_free(s: vfio_ccw_crw_region, objp: private->crw_region);
141 kmem_cache_free(s: vfio_ccw_schib_region, objp: private->schib_region);
142 kmem_cache_free(s: vfio_ccw_cmd_region, objp: private->cmd_region);
143 kmem_cache_free(s: vfio_ccw_io_region, objp: private->io_region);
144 kfree(objp: private->cp.guest_cp);
145 mutex_destroy(lock: &private->io_mutex);
146}
147
148static void vfio_ccw_mdev_remove(struct mdev_device *mdev)
149{
150 struct subchannel *sch = to_subchannel(mdev->dev.parent);
151 struct vfio_ccw_parent *parent = dev_get_drvdata(dev: &sch->dev);
152 struct vfio_ccw_private *private = dev_get_drvdata(dev: &parent->dev);
153
154 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: remove\n",
155 sch->schid.cssid,
156 sch->schid.ssid,
157 sch->schid.sch_no);
158
159 vfio_unregister_group_dev(device: &private->vdev);
160
161 dev_set_drvdata(dev: &parent->dev, NULL);
162 vfio_put_device(device: &private->vdev);
163}
164
165static int vfio_ccw_mdev_open_device(struct vfio_device *vdev)
166{
167 struct vfio_ccw_private *private =
168 container_of(vdev, struct vfio_ccw_private, vdev);
169 int ret;
170
171 /* Device cannot simply be opened again from this state */
172 if (private->state == VFIO_CCW_STATE_NOT_OPER)
173 return -EINVAL;
174
175 ret = vfio_ccw_register_async_dev_regions(private);
176 if (ret)
177 return ret;
178
179 ret = vfio_ccw_register_schib_dev_regions(private);
180 if (ret)
181 goto out_unregister;
182
183 ret = vfio_ccw_register_crw_dev_regions(private);
184 if (ret)
185 goto out_unregister;
186
187 vfio_ccw_fsm_event(private, event: VFIO_CCW_EVENT_OPEN);
188 if (private->state == VFIO_CCW_STATE_NOT_OPER) {
189 ret = -EINVAL;
190 goto out_unregister;
191 }
192
193 return ret;
194
195out_unregister:
196 vfio_ccw_unregister_dev_regions(private);
197 return ret;
198}
199
200static void vfio_ccw_mdev_close_device(struct vfio_device *vdev)
201{
202 struct vfio_ccw_private *private =
203 container_of(vdev, struct vfio_ccw_private, vdev);
204
205 vfio_ccw_fsm_event(private, event: VFIO_CCW_EVENT_CLOSE);
206 vfio_ccw_unregister_dev_regions(private);
207}
208
209static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private,
210 char __user *buf, size_t count,
211 loff_t *ppos)
212{
213 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK;
214 struct ccw_io_region *region;
215 int ret;
216
217 if (pos + count > sizeof(*region))
218 return -EINVAL;
219
220 mutex_lock(&private->io_mutex);
221 region = private->io_region;
222 if (copy_to_user(to: buf, from: (void *)region + pos, n: count))
223 ret = -EFAULT;
224 else
225 ret = count;
226 mutex_unlock(lock: &private->io_mutex);
227 return ret;
228}
229
230static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev,
231 char __user *buf,
232 size_t count,
233 loff_t *ppos)
234{
235 struct vfio_ccw_private *private =
236 container_of(vdev, struct vfio_ccw_private, vdev);
237 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos);
238
239 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions)
240 return -EINVAL;
241
242 switch (index) {
243 case VFIO_CCW_CONFIG_REGION_INDEX:
244 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos);
245 default:
246 index -= VFIO_CCW_NUM_REGIONS;
247 return private->region[index].ops->read(private, buf, count,
248 ppos);
249 }
250
251 return -EINVAL;
252}
253
254static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private,
255 const char __user *buf,
256 size_t count, loff_t *ppos)
257{
258 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK;
259 struct ccw_io_region *region;
260 int ret;
261
262 if (pos + count > sizeof(*region))
263 return -EINVAL;
264
265 if (!mutex_trylock(lock: &private->io_mutex))
266 return -EAGAIN;
267
268 region = private->io_region;
269 if (copy_from_user(to: (void *)region + pos, from: buf, n: count)) {
270 ret = -EFAULT;
271 goto out_unlock;
272 }
273
274 vfio_ccw_fsm_event(private, event: VFIO_CCW_EVENT_IO_REQ);
275 ret = (region->ret_code != 0) ? region->ret_code : count;
276
277out_unlock:
278 mutex_unlock(lock: &private->io_mutex);
279 return ret;
280}
281
282static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev,
283 const char __user *buf,
284 size_t count,
285 loff_t *ppos)
286{
287 struct vfio_ccw_private *private =
288 container_of(vdev, struct vfio_ccw_private, vdev);
289 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos);
290
291 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions)
292 return -EINVAL;
293
294 switch (index) {
295 case VFIO_CCW_CONFIG_REGION_INDEX:
296 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos);
297 default:
298 index -= VFIO_CCW_NUM_REGIONS;
299 return private->region[index].ops->write(private, buf, count,
300 ppos);
301 }
302
303 return -EINVAL;
304}
305
306static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private,
307 struct vfio_device_info *info)
308{
309 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET;
310 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions;
311 info->num_irqs = VFIO_CCW_NUM_IRQS;
312
313 return 0;
314}
315
316static int vfio_ccw_mdev_get_region_info(struct vfio_ccw_private *private,
317 struct vfio_region_info *info,
318 unsigned long arg)
319{
320 int i;
321
322 switch (info->index) {
323 case VFIO_CCW_CONFIG_REGION_INDEX:
324 info->offset = 0;
325 info->size = sizeof(struct ccw_io_region);
326 info->flags = VFIO_REGION_INFO_FLAG_READ
327 | VFIO_REGION_INFO_FLAG_WRITE;
328 return 0;
329 default: /* all other regions are handled via capability chain */
330 {
331 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
332 struct vfio_region_info_cap_type cap_type = {
333 .header.id = VFIO_REGION_INFO_CAP_TYPE,
334 .header.version = 1 };
335 int ret;
336
337 if (info->index >=
338 VFIO_CCW_NUM_REGIONS + private->num_regions)
339 return -EINVAL;
340
341 info->index = array_index_nospec(info->index,
342 VFIO_CCW_NUM_REGIONS +
343 private->num_regions);
344
345 i = info->index - VFIO_CCW_NUM_REGIONS;
346
347 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index);
348 info->size = private->region[i].size;
349 info->flags = private->region[i].flags;
350
351 cap_type.type = private->region[i].type;
352 cap_type.subtype = private->region[i].subtype;
353
354 ret = vfio_info_add_capability(caps: &caps, cap: &cap_type.header,
355 size: sizeof(cap_type));
356 if (ret)
357 return ret;
358
359 info->flags |= VFIO_REGION_INFO_FLAG_CAPS;
360 if (info->argsz < sizeof(*info) + caps.size) {
361 info->argsz = sizeof(*info) + caps.size;
362 info->cap_offset = 0;
363 } else {
364 vfio_info_cap_shift(caps: &caps, offset: sizeof(*info));
365 if (copy_to_user(to: (void __user *)arg + sizeof(*info),
366 from: caps.buf, n: caps.size)) {
367 kfree(objp: caps.buf);
368 return -EFAULT;
369 }
370 info->cap_offset = sizeof(*info);
371 }
372
373 kfree(objp: caps.buf);
374
375 }
376 }
377 return 0;
378}
379
380static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info)
381{
382 switch (info->index) {
383 case VFIO_CCW_IO_IRQ_INDEX:
384 case VFIO_CCW_CRW_IRQ_INDEX:
385 case VFIO_CCW_REQ_IRQ_INDEX:
386 info->count = 1;
387 info->flags = VFIO_IRQ_INFO_EVENTFD;
388 break;
389 default:
390 return -EINVAL;
391 }
392
393 return 0;
394}
395
396static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private,
397 uint32_t flags,
398 uint32_t index,
399 void __user *data)
400{
401 struct eventfd_ctx **ctx;
402
403 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER))
404 return -EINVAL;
405
406 switch (index) {
407 case VFIO_CCW_IO_IRQ_INDEX:
408 ctx = &private->io_trigger;
409 break;
410 case VFIO_CCW_CRW_IRQ_INDEX:
411 ctx = &private->crw_trigger;
412 break;
413 case VFIO_CCW_REQ_IRQ_INDEX:
414 ctx = &private->req_trigger;
415 break;
416 default:
417 return -EINVAL;
418 }
419
420 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
421 case VFIO_IRQ_SET_DATA_NONE:
422 {
423 if (*ctx)
424 eventfd_signal(ctx: *ctx, n: 1);
425 return 0;
426 }
427 case VFIO_IRQ_SET_DATA_BOOL:
428 {
429 uint8_t trigger;
430
431 if (get_user(trigger, (uint8_t __user *)data))
432 return -EFAULT;
433
434 if (trigger && *ctx)
435 eventfd_signal(ctx: *ctx, n: 1);
436 return 0;
437 }
438 case VFIO_IRQ_SET_DATA_EVENTFD:
439 {
440 int32_t fd;
441
442 if (get_user(fd, (int32_t __user *)data))
443 return -EFAULT;
444
445 if (fd == -1) {
446 if (*ctx)
447 eventfd_ctx_put(ctx: *ctx);
448 *ctx = NULL;
449 } else if (fd >= 0) {
450 struct eventfd_ctx *efdctx;
451
452 efdctx = eventfd_ctx_fdget(fd);
453 if (IS_ERR(ptr: efdctx))
454 return PTR_ERR(ptr: efdctx);
455
456 if (*ctx)
457 eventfd_ctx_put(ctx: *ctx);
458
459 *ctx = efdctx;
460 } else
461 return -EINVAL;
462
463 return 0;
464 }
465 default:
466 return -EINVAL;
467 }
468}
469
470int vfio_ccw_register_dev_region(struct vfio_ccw_private *private,
471 unsigned int subtype,
472 const struct vfio_ccw_regops *ops,
473 size_t size, u32 flags, void *data)
474{
475 struct vfio_ccw_region *region;
476
477 region = krealloc(objp: private->region,
478 new_size: (private->num_regions + 1) * sizeof(*region),
479 GFP_KERNEL);
480 if (!region)
481 return -ENOMEM;
482
483 private->region = region;
484 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW;
485 private->region[private->num_regions].subtype = subtype;
486 private->region[private->num_regions].ops = ops;
487 private->region[private->num_regions].size = size;
488 private->region[private->num_regions].flags = flags;
489 private->region[private->num_regions].data = data;
490
491 private->num_regions++;
492
493 return 0;
494}
495
496void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private)
497{
498 int i;
499
500 for (i = 0; i < private->num_regions; i++)
501 private->region[i].ops->release(private, &private->region[i]);
502 private->num_regions = 0;
503 kfree(objp: private->region);
504 private->region = NULL;
505}
506
507static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev,
508 unsigned int cmd,
509 unsigned long arg)
510{
511 struct vfio_ccw_private *private =
512 container_of(vdev, struct vfio_ccw_private, vdev);
513 int ret = 0;
514 unsigned long minsz;
515
516 switch (cmd) {
517 case VFIO_DEVICE_GET_INFO:
518 {
519 struct vfio_device_info info;
520
521 minsz = offsetofend(struct vfio_device_info, num_irqs);
522
523 if (copy_from_user(to: &info, from: (void __user *)arg, n: minsz))
524 return -EFAULT;
525
526 if (info.argsz < minsz)
527 return -EINVAL;
528
529 ret = vfio_ccw_mdev_get_device_info(private, info: &info);
530 if (ret)
531 return ret;
532
533 return copy_to_user(to: (void __user *)arg, from: &info, n: minsz) ? -EFAULT : 0;
534 }
535 case VFIO_DEVICE_GET_REGION_INFO:
536 {
537 struct vfio_region_info info;
538
539 minsz = offsetofend(struct vfio_region_info, offset);
540
541 if (copy_from_user(to: &info, from: (void __user *)arg, n: minsz))
542 return -EFAULT;
543
544 if (info.argsz < minsz)
545 return -EINVAL;
546
547 ret = vfio_ccw_mdev_get_region_info(private, info: &info, arg);
548 if (ret)
549 return ret;
550
551 return copy_to_user(to: (void __user *)arg, from: &info, n: minsz) ? -EFAULT : 0;
552 }
553 case VFIO_DEVICE_GET_IRQ_INFO:
554 {
555 struct vfio_irq_info info;
556
557 minsz = offsetofend(struct vfio_irq_info, count);
558
559 if (copy_from_user(to: &info, from: (void __user *)arg, n: minsz))
560 return -EFAULT;
561
562 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS)
563 return -EINVAL;
564
565 ret = vfio_ccw_mdev_get_irq_info(info: &info);
566 if (ret)
567 return ret;
568
569 if (info.count == -1)
570 return -EINVAL;
571
572 return copy_to_user(to: (void __user *)arg, from: &info, n: minsz) ? -EFAULT : 0;
573 }
574 case VFIO_DEVICE_SET_IRQS:
575 {
576 struct vfio_irq_set hdr;
577 size_t data_size;
578 void __user *data;
579
580 minsz = offsetofend(struct vfio_irq_set, count);
581
582 if (copy_from_user(to: &hdr, from: (void __user *)arg, n: minsz))
583 return -EFAULT;
584
585 ret = vfio_set_irqs_validate_and_prepare(hdr: &hdr, num_irqs: 1,
586 max_irq_type: VFIO_CCW_NUM_IRQS,
587 data_size: &data_size);
588 if (ret)
589 return ret;
590
591 data = (void __user *)(arg + minsz);
592 return vfio_ccw_mdev_set_irqs(private, flags: hdr.flags, index: hdr.index,
593 data);
594 }
595 case VFIO_DEVICE_RESET:
596 return vfio_ccw_mdev_reset(private);
597 default:
598 return -ENOTTY;
599 }
600}
601
602/* Request removal of the device*/
603static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count)
604{
605 struct vfio_ccw_private *private =
606 container_of(vdev, struct vfio_ccw_private, vdev);
607 struct device *dev = vdev->dev;
608
609 if (private->req_trigger) {
610 if (!(count % 10))
611 dev_notice_ratelimited(dev,
612 "Relaying device request to user (#%u)\n",
613 count);
614
615 eventfd_signal(ctx: private->req_trigger, n: 1);
616 } else if (count == 0) {
617 dev_notice(dev,
618 "No device request channel registered, blocked until released by user\n");
619 }
620}
621
622static const struct vfio_device_ops vfio_ccw_dev_ops = {
623 .init = vfio_ccw_mdev_init_dev,
624 .release = vfio_ccw_mdev_release_dev,
625 .open_device = vfio_ccw_mdev_open_device,
626 .close_device = vfio_ccw_mdev_close_device,
627 .read = vfio_ccw_mdev_read,
628 .write = vfio_ccw_mdev_write,
629 .ioctl = vfio_ccw_mdev_ioctl,
630 .request = vfio_ccw_mdev_request,
631 .dma_unmap = vfio_ccw_dma_unmap,
632 .bind_iommufd = vfio_iommufd_emulated_bind,
633 .unbind_iommufd = vfio_iommufd_emulated_unbind,
634 .attach_ioas = vfio_iommufd_emulated_attach_ioas,
635 .detach_ioas = vfio_iommufd_emulated_detach_ioas,
636};
637
638struct mdev_driver vfio_ccw_mdev_driver = {
639 .device_api = VFIO_DEVICE_API_CCW_STRING,
640 .max_instances = 1,
641 .driver = {
642 .name = "vfio_ccw_mdev",
643 .owner = THIS_MODULE,
644 .mod_name = KBUILD_MODNAME,
645 },
646 .probe = vfio_ccw_mdev_probe,
647 .remove = vfio_ccw_mdev_remove,
648};
649

source code of linux/drivers/s390/cio/vfio_ccw_ops.c