1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * VFIO PCI interrupt handling |
4 | * |
5 | * Copyright (C) 2012 Red Hat, Inc. All rights reserved. |
6 | * Author: Alex Williamson <alex.williamson@redhat.com> |
7 | * |
8 | * Derived from original vfio: |
9 | * Copyright 2010 Cisco Systems, Inc. All rights reserved. |
10 | * Author: Tom Lyon, pugs@cisco.com |
11 | */ |
12 | |
13 | #include <linux/device.h> |
14 | #include <linux/interrupt.h> |
15 | #include <linux/eventfd.h> |
16 | #include <linux/msi.h> |
17 | #include <linux/pci.h> |
18 | #include <linux/file.h> |
19 | #include <linux/vfio.h> |
20 | #include <linux/wait.h> |
21 | #include <linux/slab.h> |
22 | |
23 | #include "vfio_pci_priv.h" |
24 | |
25 | struct vfio_pci_irq_ctx { |
26 | struct eventfd_ctx *trigger; |
27 | struct virqfd *unmask; |
28 | struct virqfd *mask; |
29 | char *name; |
30 | bool masked; |
31 | struct irq_bypass_producer producer; |
32 | }; |
33 | |
34 | static bool irq_is(struct vfio_pci_core_device *vdev, int type) |
35 | { |
36 | return vdev->irq_type == type; |
37 | } |
38 | |
39 | static bool is_intx(struct vfio_pci_core_device *vdev) |
40 | { |
41 | return vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX; |
42 | } |
43 | |
44 | static bool is_irq_none(struct vfio_pci_core_device *vdev) |
45 | { |
46 | return !(vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX || |
47 | vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX || |
48 | vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX); |
49 | } |
50 | |
51 | static |
52 | struct vfio_pci_irq_ctx *vfio_irq_ctx_get(struct vfio_pci_core_device *vdev, |
53 | unsigned long index) |
54 | { |
55 | return xa_load(&vdev->ctx, index); |
56 | } |
57 | |
58 | static void vfio_irq_ctx_free(struct vfio_pci_core_device *vdev, |
59 | struct vfio_pci_irq_ctx *ctx, unsigned long index) |
60 | { |
61 | xa_erase(&vdev->ctx, index); |
62 | kfree(objp: ctx); |
63 | } |
64 | |
65 | static struct vfio_pci_irq_ctx * |
66 | vfio_irq_ctx_alloc(struct vfio_pci_core_device *vdev, unsigned long index) |
67 | { |
68 | struct vfio_pci_irq_ctx *ctx; |
69 | int ret; |
70 | |
71 | ctx = kzalloc(size: sizeof(*ctx), GFP_KERNEL_ACCOUNT); |
72 | if (!ctx) |
73 | return NULL; |
74 | |
75 | ret = xa_insert(xa: &vdev->ctx, index, entry: ctx, GFP_KERNEL_ACCOUNT); |
76 | if (ret) { |
77 | kfree(objp: ctx); |
78 | return NULL; |
79 | } |
80 | |
81 | return ctx; |
82 | } |
83 | |
84 | /* |
85 | * INTx |
86 | */ |
87 | static void vfio_send_intx_eventfd(void *opaque, void *unused) |
88 | { |
89 | struct vfio_pci_core_device *vdev = opaque; |
90 | |
91 | if (likely(is_intx(vdev) && !vdev->virq_disabled)) { |
92 | struct vfio_pci_irq_ctx *ctx; |
93 | |
94 | ctx = vfio_irq_ctx_get(vdev, index: 0); |
95 | if (WARN_ON_ONCE(!ctx)) |
96 | return; |
97 | eventfd_signal(ctx: ctx->trigger, n: 1); |
98 | } |
99 | } |
100 | |
101 | /* Returns true if the INTx vfio_pci_irq_ctx.masked value is changed. */ |
102 | bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) |
103 | { |
104 | struct pci_dev *pdev = vdev->pdev; |
105 | struct vfio_pci_irq_ctx *ctx; |
106 | unsigned long flags; |
107 | bool masked_changed = false; |
108 | |
109 | spin_lock_irqsave(&vdev->irqlock, flags); |
110 | |
111 | /* |
112 | * Masking can come from interrupt, ioctl, or config space |
113 | * via INTx disable. The latter means this can get called |
114 | * even when not using intx delivery. In this case, just |
115 | * try to have the physical bit follow the virtual bit. |
116 | */ |
117 | if (unlikely(!is_intx(vdev))) { |
118 | if (vdev->pci_2_3) |
119 | pci_intx(dev: pdev, enable: 0); |
120 | goto out_unlock; |
121 | } |
122 | |
123 | ctx = vfio_irq_ctx_get(vdev, index: 0); |
124 | if (WARN_ON_ONCE(!ctx)) |
125 | goto out_unlock; |
126 | |
127 | if (!ctx->masked) { |
128 | /* |
129 | * Can't use check_and_mask here because we always want to |
130 | * mask, not just when something is pending. |
131 | */ |
132 | if (vdev->pci_2_3) |
133 | pci_intx(dev: pdev, enable: 0); |
134 | else |
135 | disable_irq_nosync(irq: pdev->irq); |
136 | |
137 | ctx->masked = true; |
138 | masked_changed = true; |
139 | } |
140 | |
141 | out_unlock: |
142 | spin_unlock_irqrestore(lock: &vdev->irqlock, flags); |
143 | return masked_changed; |
144 | } |
145 | |
146 | /* |
147 | * If this is triggered by an eventfd, we can't call eventfd_signal |
148 | * or else we'll deadlock on the eventfd wait queue. Return >0 when |
149 | * a signal is necessary, which can then be handled via a work queue |
150 | * or directly depending on the caller. |
151 | */ |
152 | static int vfio_pci_intx_unmask_handler(void *opaque, void *unused) |
153 | { |
154 | struct vfio_pci_core_device *vdev = opaque; |
155 | struct pci_dev *pdev = vdev->pdev; |
156 | struct vfio_pci_irq_ctx *ctx; |
157 | unsigned long flags; |
158 | int ret = 0; |
159 | |
160 | spin_lock_irqsave(&vdev->irqlock, flags); |
161 | |
162 | /* |
163 | * Unmasking comes from ioctl or config, so again, have the |
164 | * physical bit follow the virtual even when not using INTx. |
165 | */ |
166 | if (unlikely(!is_intx(vdev))) { |
167 | if (vdev->pci_2_3) |
168 | pci_intx(dev: pdev, enable: 1); |
169 | goto out_unlock; |
170 | } |
171 | |
172 | ctx = vfio_irq_ctx_get(vdev, index: 0); |
173 | if (WARN_ON_ONCE(!ctx)) |
174 | goto out_unlock; |
175 | |
176 | if (ctx->masked && !vdev->virq_disabled) { |
177 | /* |
178 | * A pending interrupt here would immediately trigger, |
179 | * but we can avoid that overhead by just re-sending |
180 | * the interrupt to the user. |
181 | */ |
182 | if (vdev->pci_2_3) { |
183 | if (!pci_check_and_unmask_intx(dev: pdev)) |
184 | ret = 1; |
185 | } else |
186 | enable_irq(irq: pdev->irq); |
187 | |
188 | ctx->masked = (ret > 0); |
189 | } |
190 | |
191 | out_unlock: |
192 | spin_unlock_irqrestore(lock: &vdev->irqlock, flags); |
193 | |
194 | return ret; |
195 | } |
196 | |
197 | void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) |
198 | { |
199 | if (vfio_pci_intx_unmask_handler(opaque: vdev, NULL) > 0) |
200 | vfio_send_intx_eventfd(opaque: vdev, NULL); |
201 | } |
202 | |
203 | static irqreturn_t vfio_intx_handler(int irq, void *dev_id) |
204 | { |
205 | struct vfio_pci_core_device *vdev = dev_id; |
206 | struct vfio_pci_irq_ctx *ctx; |
207 | unsigned long flags; |
208 | int ret = IRQ_NONE; |
209 | |
210 | ctx = vfio_irq_ctx_get(vdev, index: 0); |
211 | if (WARN_ON_ONCE(!ctx)) |
212 | return ret; |
213 | |
214 | spin_lock_irqsave(&vdev->irqlock, flags); |
215 | |
216 | if (!vdev->pci_2_3) { |
217 | disable_irq_nosync(irq: vdev->pdev->irq); |
218 | ctx->masked = true; |
219 | ret = IRQ_HANDLED; |
220 | } else if (!ctx->masked && /* may be shared */ |
221 | pci_check_and_mask_intx(dev: vdev->pdev)) { |
222 | ctx->masked = true; |
223 | ret = IRQ_HANDLED; |
224 | } |
225 | |
226 | spin_unlock_irqrestore(lock: &vdev->irqlock, flags); |
227 | |
228 | if (ret == IRQ_HANDLED) |
229 | vfio_send_intx_eventfd(opaque: vdev, NULL); |
230 | |
231 | return ret; |
232 | } |
233 | |
234 | static int vfio_intx_enable(struct vfio_pci_core_device *vdev) |
235 | { |
236 | struct vfio_pci_irq_ctx *ctx; |
237 | |
238 | if (!is_irq_none(vdev)) |
239 | return -EINVAL; |
240 | |
241 | if (!vdev->pdev->irq) |
242 | return -ENODEV; |
243 | |
244 | ctx = vfio_irq_ctx_alloc(vdev, index: 0); |
245 | if (!ctx) |
246 | return -ENOMEM; |
247 | |
248 | /* |
249 | * If the virtual interrupt is masked, restore it. Devices |
250 | * supporting DisINTx can be masked at the hardware level |
251 | * here, non-PCI-2.3 devices will have to wait until the |
252 | * interrupt is enabled. |
253 | */ |
254 | ctx->masked = vdev->virq_disabled; |
255 | if (vdev->pci_2_3) |
256 | pci_intx(dev: vdev->pdev, enable: !ctx->masked); |
257 | |
258 | vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX; |
259 | |
260 | return 0; |
261 | } |
262 | |
263 | static int vfio_intx_set_signal(struct vfio_pci_core_device *vdev, int fd) |
264 | { |
265 | struct pci_dev *pdev = vdev->pdev; |
266 | unsigned long irqflags = IRQF_SHARED; |
267 | struct vfio_pci_irq_ctx *ctx; |
268 | struct eventfd_ctx *trigger; |
269 | unsigned long flags; |
270 | int ret; |
271 | |
272 | ctx = vfio_irq_ctx_get(vdev, index: 0); |
273 | if (WARN_ON_ONCE(!ctx)) |
274 | return -EINVAL; |
275 | |
276 | if (ctx->trigger) { |
277 | free_irq(pdev->irq, vdev); |
278 | kfree(objp: ctx->name); |
279 | eventfd_ctx_put(ctx: ctx->trigger); |
280 | ctx->trigger = NULL; |
281 | } |
282 | |
283 | if (fd < 0) /* Disable only */ |
284 | return 0; |
285 | |
286 | ctx->name = kasprintf(GFP_KERNEL_ACCOUNT, fmt: "vfio-intx(%s)" , |
287 | pci_name(pdev)); |
288 | if (!ctx->name) |
289 | return -ENOMEM; |
290 | |
291 | trigger = eventfd_ctx_fdget(fd); |
292 | if (IS_ERR(ptr: trigger)) { |
293 | kfree(objp: ctx->name); |
294 | return PTR_ERR(ptr: trigger); |
295 | } |
296 | |
297 | ctx->trigger = trigger; |
298 | |
299 | if (!vdev->pci_2_3) |
300 | irqflags = 0; |
301 | |
302 | ret = request_irq(irq: pdev->irq, handler: vfio_intx_handler, |
303 | flags: irqflags, name: ctx->name, dev: vdev); |
304 | if (ret) { |
305 | ctx->trigger = NULL; |
306 | kfree(objp: ctx->name); |
307 | eventfd_ctx_put(ctx: trigger); |
308 | return ret; |
309 | } |
310 | |
311 | /* |
312 | * INTx disable will stick across the new irq setup, |
313 | * disable_irq won't. |
314 | */ |
315 | spin_lock_irqsave(&vdev->irqlock, flags); |
316 | if (!vdev->pci_2_3 && ctx->masked) |
317 | disable_irq_nosync(irq: pdev->irq); |
318 | spin_unlock_irqrestore(lock: &vdev->irqlock, flags); |
319 | |
320 | return 0; |
321 | } |
322 | |
323 | static void vfio_intx_disable(struct vfio_pci_core_device *vdev) |
324 | { |
325 | struct vfio_pci_irq_ctx *ctx; |
326 | |
327 | ctx = vfio_irq_ctx_get(vdev, index: 0); |
328 | WARN_ON_ONCE(!ctx); |
329 | if (ctx) { |
330 | vfio_virqfd_disable(pvirqfd: &ctx->unmask); |
331 | vfio_virqfd_disable(pvirqfd: &ctx->mask); |
332 | } |
333 | vfio_intx_set_signal(vdev, fd: -1); |
334 | vdev->irq_type = VFIO_PCI_NUM_IRQS; |
335 | vfio_irq_ctx_free(vdev, ctx, index: 0); |
336 | } |
337 | |
338 | /* |
339 | * MSI/MSI-X |
340 | */ |
341 | static irqreturn_t vfio_msihandler(int irq, void *arg) |
342 | { |
343 | struct eventfd_ctx *trigger = arg; |
344 | |
345 | eventfd_signal(ctx: trigger, n: 1); |
346 | return IRQ_HANDLED; |
347 | } |
348 | |
349 | static int vfio_msi_enable(struct vfio_pci_core_device *vdev, int nvec, bool msix) |
350 | { |
351 | struct pci_dev *pdev = vdev->pdev; |
352 | unsigned int flag = msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI; |
353 | int ret; |
354 | u16 cmd; |
355 | |
356 | if (!is_irq_none(vdev)) |
357 | return -EINVAL; |
358 | |
359 | /* return the number of supported vectors if we can't get all: */ |
360 | cmd = vfio_pci_memory_lock_and_enable(vdev); |
361 | ret = pci_alloc_irq_vectors(dev: pdev, min_vecs: 1, max_vecs: nvec, flags: flag); |
362 | if (ret < nvec) { |
363 | if (ret > 0) |
364 | pci_free_irq_vectors(dev: pdev); |
365 | vfio_pci_memory_unlock_and_restore(vdev, cmd); |
366 | return ret; |
367 | } |
368 | vfio_pci_memory_unlock_and_restore(vdev, cmd); |
369 | |
370 | vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX : |
371 | VFIO_PCI_MSI_IRQ_INDEX; |
372 | |
373 | if (!msix) { |
374 | /* |
375 | * Compute the virtual hardware field for max msi vectors - |
376 | * it is the log base 2 of the number of vectors. |
377 | */ |
378 | vdev->msi_qmax = fls(x: nvec * 2 - 1) - 1; |
379 | } |
380 | |
381 | return 0; |
382 | } |
383 | |
384 | /* |
385 | * vfio_msi_alloc_irq() returns the Linux IRQ number of an MSI or MSI-X device |
386 | * interrupt vector. If a Linux IRQ number is not available then a new |
387 | * interrupt is allocated if dynamic MSI-X is supported. |
388 | * |
389 | * Where is vfio_msi_free_irq()? Allocated interrupts are maintained, |
390 | * essentially forming a cache that subsequent allocations can draw from. |
391 | * Interrupts are freed using pci_free_irq_vectors() when MSI/MSI-X is |
392 | * disabled. |
393 | */ |
394 | static int vfio_msi_alloc_irq(struct vfio_pci_core_device *vdev, |
395 | unsigned int vector, bool msix) |
396 | { |
397 | struct pci_dev *pdev = vdev->pdev; |
398 | struct msi_map map; |
399 | int irq; |
400 | u16 cmd; |
401 | |
402 | irq = pci_irq_vector(dev: pdev, nr: vector); |
403 | if (WARN_ON_ONCE(irq == 0)) |
404 | return -EINVAL; |
405 | if (irq > 0 || !msix || !vdev->has_dyn_msix) |
406 | return irq; |
407 | |
408 | cmd = vfio_pci_memory_lock_and_enable(vdev); |
409 | map = pci_msix_alloc_irq_at(dev: pdev, index: vector, NULL); |
410 | vfio_pci_memory_unlock_and_restore(vdev, cmd); |
411 | |
412 | return map.index < 0 ? map.index : map.virq; |
413 | } |
414 | |
415 | static int vfio_msi_set_vector_signal(struct vfio_pci_core_device *vdev, |
416 | unsigned int vector, int fd, bool msix) |
417 | { |
418 | struct pci_dev *pdev = vdev->pdev; |
419 | struct vfio_pci_irq_ctx *ctx; |
420 | struct eventfd_ctx *trigger; |
421 | int irq = -EINVAL, ret; |
422 | u16 cmd; |
423 | |
424 | ctx = vfio_irq_ctx_get(vdev, index: vector); |
425 | |
426 | if (ctx) { |
427 | irq_bypass_unregister_producer(&ctx->producer); |
428 | irq = pci_irq_vector(dev: pdev, nr: vector); |
429 | cmd = vfio_pci_memory_lock_and_enable(vdev); |
430 | free_irq(irq, ctx->trigger); |
431 | vfio_pci_memory_unlock_and_restore(vdev, cmd); |
432 | /* Interrupt stays allocated, will be freed at MSI-X disable. */ |
433 | kfree(objp: ctx->name); |
434 | eventfd_ctx_put(ctx: ctx->trigger); |
435 | vfio_irq_ctx_free(vdev, ctx, index: vector); |
436 | } |
437 | |
438 | if (fd < 0) |
439 | return 0; |
440 | |
441 | if (irq == -EINVAL) { |
442 | /* Interrupt stays allocated, will be freed at MSI-X disable. */ |
443 | irq = vfio_msi_alloc_irq(vdev, vector, msix); |
444 | if (irq < 0) |
445 | return irq; |
446 | } |
447 | |
448 | ctx = vfio_irq_ctx_alloc(vdev, index: vector); |
449 | if (!ctx) |
450 | return -ENOMEM; |
451 | |
452 | ctx->name = kasprintf(GFP_KERNEL_ACCOUNT, fmt: "vfio-msi%s[%d](%s)" , |
453 | msix ? "x" : "" , vector, pci_name(pdev)); |
454 | if (!ctx->name) { |
455 | ret = -ENOMEM; |
456 | goto out_free_ctx; |
457 | } |
458 | |
459 | trigger = eventfd_ctx_fdget(fd); |
460 | if (IS_ERR(ptr: trigger)) { |
461 | ret = PTR_ERR(ptr: trigger); |
462 | goto out_free_name; |
463 | } |
464 | |
465 | /* |
466 | * If the vector was previously allocated, refresh the on-device |
467 | * message data before enabling in case it had been cleared or |
468 | * corrupted (e.g. due to backdoor resets) since writing. |
469 | */ |
470 | cmd = vfio_pci_memory_lock_and_enable(vdev); |
471 | if (msix) { |
472 | struct msi_msg msg; |
473 | |
474 | get_cached_msi_msg(irq, msg: &msg); |
475 | pci_write_msi_msg(irq, msg: &msg); |
476 | } |
477 | |
478 | ret = request_irq(irq, handler: vfio_msihandler, flags: 0, name: ctx->name, dev: trigger); |
479 | vfio_pci_memory_unlock_and_restore(vdev, cmd); |
480 | if (ret) |
481 | goto out_put_eventfd_ctx; |
482 | |
483 | ctx->producer.token = trigger; |
484 | ctx->producer.irq = irq; |
485 | ret = irq_bypass_register_producer(&ctx->producer); |
486 | if (unlikely(ret)) { |
487 | dev_info(&pdev->dev, |
488 | "irq bypass producer (token %p) registration fails: %d\n" , |
489 | ctx->producer.token, ret); |
490 | |
491 | ctx->producer.token = NULL; |
492 | } |
493 | ctx->trigger = trigger; |
494 | |
495 | return 0; |
496 | |
497 | out_put_eventfd_ctx: |
498 | eventfd_ctx_put(ctx: trigger); |
499 | out_free_name: |
500 | kfree(objp: ctx->name); |
501 | out_free_ctx: |
502 | vfio_irq_ctx_free(vdev, ctx, index: vector); |
503 | return ret; |
504 | } |
505 | |
506 | static int vfio_msi_set_block(struct vfio_pci_core_device *vdev, unsigned start, |
507 | unsigned count, int32_t *fds, bool msix) |
508 | { |
509 | unsigned int i, j; |
510 | int ret = 0; |
511 | |
512 | for (i = 0, j = start; i < count && !ret; i++, j++) { |
513 | int fd = fds ? fds[i] : -1; |
514 | ret = vfio_msi_set_vector_signal(vdev, vector: j, fd, msix); |
515 | } |
516 | |
517 | if (ret) { |
518 | for (i = start; i < j; i++) |
519 | vfio_msi_set_vector_signal(vdev, vector: i, fd: -1, msix); |
520 | } |
521 | |
522 | return ret; |
523 | } |
524 | |
525 | static void vfio_msi_disable(struct vfio_pci_core_device *vdev, bool msix) |
526 | { |
527 | struct pci_dev *pdev = vdev->pdev; |
528 | struct vfio_pci_irq_ctx *ctx; |
529 | unsigned long i; |
530 | u16 cmd; |
531 | |
532 | xa_for_each(&vdev->ctx, i, ctx) { |
533 | vfio_virqfd_disable(pvirqfd: &ctx->unmask); |
534 | vfio_virqfd_disable(pvirqfd: &ctx->mask); |
535 | vfio_msi_set_vector_signal(vdev, vector: i, fd: -1, msix); |
536 | } |
537 | |
538 | cmd = vfio_pci_memory_lock_and_enable(vdev); |
539 | pci_free_irq_vectors(dev: pdev); |
540 | vfio_pci_memory_unlock_and_restore(vdev, cmd); |
541 | |
542 | /* |
543 | * Both disable paths above use pci_intx_for_msi() to clear DisINTx |
544 | * via their shutdown paths. Restore for NoINTx devices. |
545 | */ |
546 | if (vdev->nointx) |
547 | pci_intx(dev: pdev, enable: 0); |
548 | |
549 | vdev->irq_type = VFIO_PCI_NUM_IRQS; |
550 | } |
551 | |
552 | /* |
553 | * IOCTL support |
554 | */ |
555 | static int vfio_pci_set_intx_unmask(struct vfio_pci_core_device *vdev, |
556 | unsigned index, unsigned start, |
557 | unsigned count, uint32_t flags, void *data) |
558 | { |
559 | if (!is_intx(vdev) || start != 0 || count != 1) |
560 | return -EINVAL; |
561 | |
562 | if (flags & VFIO_IRQ_SET_DATA_NONE) { |
563 | vfio_pci_intx_unmask(vdev); |
564 | } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
565 | uint8_t unmask = *(uint8_t *)data; |
566 | if (unmask) |
567 | vfio_pci_intx_unmask(vdev); |
568 | } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
569 | struct vfio_pci_irq_ctx *ctx = vfio_irq_ctx_get(vdev, index: 0); |
570 | int32_t fd = *(int32_t *)data; |
571 | |
572 | if (WARN_ON_ONCE(!ctx)) |
573 | return -EINVAL; |
574 | if (fd >= 0) |
575 | return vfio_virqfd_enable(opaque: (void *) vdev, |
576 | handler: vfio_pci_intx_unmask_handler, |
577 | thread: vfio_send_intx_eventfd, NULL, |
578 | pvirqfd: &ctx->unmask, fd); |
579 | |
580 | vfio_virqfd_disable(pvirqfd: &ctx->unmask); |
581 | } |
582 | |
583 | return 0; |
584 | } |
585 | |
586 | static int vfio_pci_set_intx_mask(struct vfio_pci_core_device *vdev, |
587 | unsigned index, unsigned start, |
588 | unsigned count, uint32_t flags, void *data) |
589 | { |
590 | if (!is_intx(vdev) || start != 0 || count != 1) |
591 | return -EINVAL; |
592 | |
593 | if (flags & VFIO_IRQ_SET_DATA_NONE) { |
594 | vfio_pci_intx_mask(vdev); |
595 | } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
596 | uint8_t mask = *(uint8_t *)data; |
597 | if (mask) |
598 | vfio_pci_intx_mask(vdev); |
599 | } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
600 | return -ENOTTY; /* XXX implement me */ |
601 | } |
602 | |
603 | return 0; |
604 | } |
605 | |
606 | static int vfio_pci_set_intx_trigger(struct vfio_pci_core_device *vdev, |
607 | unsigned index, unsigned start, |
608 | unsigned count, uint32_t flags, void *data) |
609 | { |
610 | if (is_intx(vdev) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { |
611 | vfio_intx_disable(vdev); |
612 | return 0; |
613 | } |
614 | |
615 | if (!(is_intx(vdev) || is_irq_none(vdev)) || start != 0 || count != 1) |
616 | return -EINVAL; |
617 | |
618 | if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
619 | int32_t fd = *(int32_t *)data; |
620 | int ret; |
621 | |
622 | if (is_intx(vdev)) |
623 | return vfio_intx_set_signal(vdev, fd); |
624 | |
625 | ret = vfio_intx_enable(vdev); |
626 | if (ret) |
627 | return ret; |
628 | |
629 | ret = vfio_intx_set_signal(vdev, fd); |
630 | if (ret) |
631 | vfio_intx_disable(vdev); |
632 | |
633 | return ret; |
634 | } |
635 | |
636 | if (!is_intx(vdev)) |
637 | return -EINVAL; |
638 | |
639 | if (flags & VFIO_IRQ_SET_DATA_NONE) { |
640 | vfio_send_intx_eventfd(opaque: vdev, NULL); |
641 | } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
642 | uint8_t trigger = *(uint8_t *)data; |
643 | if (trigger) |
644 | vfio_send_intx_eventfd(opaque: vdev, NULL); |
645 | } |
646 | return 0; |
647 | } |
648 | |
649 | static int vfio_pci_set_msi_trigger(struct vfio_pci_core_device *vdev, |
650 | unsigned index, unsigned start, |
651 | unsigned count, uint32_t flags, void *data) |
652 | { |
653 | struct vfio_pci_irq_ctx *ctx; |
654 | unsigned int i; |
655 | bool msix = (index == VFIO_PCI_MSIX_IRQ_INDEX) ? true : false; |
656 | |
657 | if (irq_is(vdev, type: index) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { |
658 | vfio_msi_disable(vdev, msix); |
659 | return 0; |
660 | } |
661 | |
662 | if (!(irq_is(vdev, type: index) || is_irq_none(vdev))) |
663 | return -EINVAL; |
664 | |
665 | if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
666 | int32_t *fds = data; |
667 | int ret; |
668 | |
669 | if (vdev->irq_type == index) |
670 | return vfio_msi_set_block(vdev, start, count, |
671 | fds, msix); |
672 | |
673 | ret = vfio_msi_enable(vdev, nvec: start + count, msix); |
674 | if (ret) |
675 | return ret; |
676 | |
677 | ret = vfio_msi_set_block(vdev, start, count, fds, msix); |
678 | if (ret) |
679 | vfio_msi_disable(vdev, msix); |
680 | |
681 | return ret; |
682 | } |
683 | |
684 | if (!irq_is(vdev, type: index)) |
685 | return -EINVAL; |
686 | |
687 | for (i = start; i < start + count; i++) { |
688 | ctx = vfio_irq_ctx_get(vdev, index: i); |
689 | if (!ctx) |
690 | continue; |
691 | if (flags & VFIO_IRQ_SET_DATA_NONE) { |
692 | eventfd_signal(ctx: ctx->trigger, n: 1); |
693 | } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
694 | uint8_t *bools = data; |
695 | if (bools[i - start]) |
696 | eventfd_signal(ctx: ctx->trigger, n: 1); |
697 | } |
698 | } |
699 | return 0; |
700 | } |
701 | |
702 | static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx, |
703 | unsigned int count, uint32_t flags, |
704 | void *data) |
705 | { |
706 | /* DATA_NONE/DATA_BOOL enables loopback testing */ |
707 | if (flags & VFIO_IRQ_SET_DATA_NONE) { |
708 | if (*ctx) { |
709 | if (count) { |
710 | eventfd_signal(ctx: *ctx, n: 1); |
711 | } else { |
712 | eventfd_ctx_put(ctx: *ctx); |
713 | *ctx = NULL; |
714 | } |
715 | return 0; |
716 | } |
717 | } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
718 | uint8_t trigger; |
719 | |
720 | if (!count) |
721 | return -EINVAL; |
722 | |
723 | trigger = *(uint8_t *)data; |
724 | if (trigger && *ctx) |
725 | eventfd_signal(ctx: *ctx, n: 1); |
726 | |
727 | return 0; |
728 | } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
729 | int32_t fd; |
730 | |
731 | if (!count) |
732 | return -EINVAL; |
733 | |
734 | fd = *(int32_t *)data; |
735 | if (fd == -1) { |
736 | if (*ctx) |
737 | eventfd_ctx_put(ctx: *ctx); |
738 | *ctx = NULL; |
739 | } else if (fd >= 0) { |
740 | struct eventfd_ctx *efdctx; |
741 | |
742 | efdctx = eventfd_ctx_fdget(fd); |
743 | if (IS_ERR(ptr: efdctx)) |
744 | return PTR_ERR(ptr: efdctx); |
745 | |
746 | if (*ctx) |
747 | eventfd_ctx_put(ctx: *ctx); |
748 | |
749 | *ctx = efdctx; |
750 | } |
751 | return 0; |
752 | } |
753 | |
754 | return -EINVAL; |
755 | } |
756 | |
757 | static int vfio_pci_set_err_trigger(struct vfio_pci_core_device *vdev, |
758 | unsigned index, unsigned start, |
759 | unsigned count, uint32_t flags, void *data) |
760 | { |
761 | if (index != VFIO_PCI_ERR_IRQ_INDEX || start != 0 || count > 1) |
762 | return -EINVAL; |
763 | |
764 | return vfio_pci_set_ctx_trigger_single(ctx: &vdev->err_trigger, |
765 | count, flags, data); |
766 | } |
767 | |
768 | static int vfio_pci_set_req_trigger(struct vfio_pci_core_device *vdev, |
769 | unsigned index, unsigned start, |
770 | unsigned count, uint32_t flags, void *data) |
771 | { |
772 | if (index != VFIO_PCI_REQ_IRQ_INDEX || start != 0 || count > 1) |
773 | return -EINVAL; |
774 | |
775 | return vfio_pci_set_ctx_trigger_single(ctx: &vdev->req_trigger, |
776 | count, flags, data); |
777 | } |
778 | |
779 | int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags, |
780 | unsigned index, unsigned start, unsigned count, |
781 | void *data) |
782 | { |
783 | int (*func)(struct vfio_pci_core_device *vdev, unsigned index, |
784 | unsigned start, unsigned count, uint32_t flags, |
785 | void *data) = NULL; |
786 | |
787 | switch (index) { |
788 | case VFIO_PCI_INTX_IRQ_INDEX: |
789 | switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { |
790 | case VFIO_IRQ_SET_ACTION_MASK: |
791 | func = vfio_pci_set_intx_mask; |
792 | break; |
793 | case VFIO_IRQ_SET_ACTION_UNMASK: |
794 | func = vfio_pci_set_intx_unmask; |
795 | break; |
796 | case VFIO_IRQ_SET_ACTION_TRIGGER: |
797 | func = vfio_pci_set_intx_trigger; |
798 | break; |
799 | } |
800 | break; |
801 | case VFIO_PCI_MSI_IRQ_INDEX: |
802 | case VFIO_PCI_MSIX_IRQ_INDEX: |
803 | switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { |
804 | case VFIO_IRQ_SET_ACTION_MASK: |
805 | case VFIO_IRQ_SET_ACTION_UNMASK: |
806 | /* XXX Need masking support exported */ |
807 | break; |
808 | case VFIO_IRQ_SET_ACTION_TRIGGER: |
809 | func = vfio_pci_set_msi_trigger; |
810 | break; |
811 | } |
812 | break; |
813 | case VFIO_PCI_ERR_IRQ_INDEX: |
814 | switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { |
815 | case VFIO_IRQ_SET_ACTION_TRIGGER: |
816 | if (pci_is_pcie(dev: vdev->pdev)) |
817 | func = vfio_pci_set_err_trigger; |
818 | break; |
819 | } |
820 | break; |
821 | case VFIO_PCI_REQ_IRQ_INDEX: |
822 | switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { |
823 | case VFIO_IRQ_SET_ACTION_TRIGGER: |
824 | func = vfio_pci_set_req_trigger; |
825 | break; |
826 | } |
827 | break; |
828 | } |
829 | |
830 | if (!func) |
831 | return -ENOTTY; |
832 | |
833 | return func(vdev, index, start, count, flags, data); |
834 | } |
835 | |