1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/virtio.h> |
3 | #include <linux/spinlock.h> |
4 | #include <linux/virtio_config.h> |
5 | #include <linux/virtio_anchor.h> |
6 | #include <linux/module.h> |
7 | #include <linux/idr.h> |
8 | #include <linux/of.h> |
9 | #include <uapi/linux/virtio_ids.h> |
10 | |
11 | /* Unique numbering for virtio devices. */ |
12 | static DEFINE_IDA(virtio_index_ida); |
13 | |
14 | static ssize_t device_show(struct device *_d, |
15 | struct device_attribute *attr, char *buf) |
16 | { |
17 | struct virtio_device *dev = dev_to_virtio(_d); |
18 | return sysfs_emit(buf, fmt: "0x%04x\n" , dev->id.device); |
19 | } |
20 | static DEVICE_ATTR_RO(device); |
21 | |
22 | static ssize_t vendor_show(struct device *_d, |
23 | struct device_attribute *attr, char *buf) |
24 | { |
25 | struct virtio_device *dev = dev_to_virtio(_d); |
26 | return sysfs_emit(buf, fmt: "0x%04x\n" , dev->id.vendor); |
27 | } |
28 | static DEVICE_ATTR_RO(vendor); |
29 | |
30 | static ssize_t status_show(struct device *_d, |
31 | struct device_attribute *attr, char *buf) |
32 | { |
33 | struct virtio_device *dev = dev_to_virtio(_d); |
34 | return sysfs_emit(buf, fmt: "0x%08x\n" , dev->config->get_status(dev)); |
35 | } |
36 | static DEVICE_ATTR_RO(status); |
37 | |
38 | static ssize_t modalias_show(struct device *_d, |
39 | struct device_attribute *attr, char *buf) |
40 | { |
41 | struct virtio_device *dev = dev_to_virtio(_d); |
42 | return sysfs_emit(buf, fmt: "virtio:d%08Xv%08X\n" , |
43 | dev->id.device, dev->id.vendor); |
44 | } |
45 | static DEVICE_ATTR_RO(modalias); |
46 | |
47 | static ssize_t features_show(struct device *_d, |
48 | struct device_attribute *attr, char *buf) |
49 | { |
50 | struct virtio_device *dev = dev_to_virtio(_d); |
51 | unsigned int i; |
52 | ssize_t len = 0; |
53 | |
54 | /* We actually represent this as a bitstring, as it could be |
55 | * arbitrary length in future. */ |
56 | for (i = 0; i < sizeof(dev->features)*8; i++) |
57 | len += sysfs_emit_at(buf, at: len, fmt: "%c" , |
58 | __virtio_test_bit(vdev: dev, fbit: i) ? '1' : '0'); |
59 | len += sysfs_emit_at(buf, at: len, fmt: "\n" ); |
60 | return len; |
61 | } |
62 | static DEVICE_ATTR_RO(features); |
63 | |
64 | static struct attribute *virtio_dev_attrs[] = { |
65 | &dev_attr_device.attr, |
66 | &dev_attr_vendor.attr, |
67 | &dev_attr_status.attr, |
68 | &dev_attr_modalias.attr, |
69 | &dev_attr_features.attr, |
70 | NULL, |
71 | }; |
72 | ATTRIBUTE_GROUPS(virtio_dev); |
73 | |
74 | static inline int virtio_id_match(const struct virtio_device *dev, |
75 | const struct virtio_device_id *id) |
76 | { |
77 | if (id->device != dev->id.device && id->device != VIRTIO_DEV_ANY_ID) |
78 | return 0; |
79 | |
80 | return id->vendor == VIRTIO_DEV_ANY_ID || id->vendor == dev->id.vendor; |
81 | } |
82 | |
83 | /* This looks through all the IDs a driver claims to support. If any of them |
84 | * match, we return 1 and the kernel will call virtio_dev_probe(). */ |
85 | static int virtio_dev_match(struct device *_dv, struct device_driver *_dr) |
86 | { |
87 | unsigned int i; |
88 | struct virtio_device *dev = dev_to_virtio(_dv); |
89 | const struct virtio_device_id *ids; |
90 | |
91 | ids = drv_to_virtio(drv: _dr)->id_table; |
92 | for (i = 0; ids[i].device; i++) |
93 | if (virtio_id_match(dev, id: &ids[i])) |
94 | return 1; |
95 | return 0; |
96 | } |
97 | |
98 | static int virtio_uevent(const struct device *_dv, struct kobj_uevent_env *env) |
99 | { |
100 | const struct virtio_device *dev = dev_to_virtio(_dv); |
101 | |
102 | return add_uevent_var(env, format: "MODALIAS=virtio:d%08Xv%08X" , |
103 | dev->id.device, dev->id.vendor); |
104 | } |
105 | |
106 | void virtio_check_driver_offered_feature(const struct virtio_device *vdev, |
107 | unsigned int fbit) |
108 | { |
109 | unsigned int i; |
110 | struct virtio_driver *drv = drv_to_virtio(drv: vdev->dev.driver); |
111 | |
112 | for (i = 0; i < drv->feature_table_size; i++) |
113 | if (drv->feature_table[i] == fbit) |
114 | return; |
115 | |
116 | if (drv->feature_table_legacy) { |
117 | for (i = 0; i < drv->feature_table_size_legacy; i++) |
118 | if (drv->feature_table_legacy[i] == fbit) |
119 | return; |
120 | } |
121 | |
122 | BUG(); |
123 | } |
124 | EXPORT_SYMBOL_GPL(virtio_check_driver_offered_feature); |
125 | |
126 | static void __virtio_config_changed(struct virtio_device *dev) |
127 | { |
128 | struct virtio_driver *drv = drv_to_virtio(drv: dev->dev.driver); |
129 | |
130 | if (!dev->config_enabled) |
131 | dev->config_change_pending = true; |
132 | else if (drv && drv->config_changed) |
133 | drv->config_changed(dev); |
134 | } |
135 | |
136 | void virtio_config_changed(struct virtio_device *dev) |
137 | { |
138 | unsigned long flags; |
139 | |
140 | spin_lock_irqsave(&dev->config_lock, flags); |
141 | __virtio_config_changed(dev); |
142 | spin_unlock_irqrestore(lock: &dev->config_lock, flags); |
143 | } |
144 | EXPORT_SYMBOL_GPL(virtio_config_changed); |
145 | |
146 | static void virtio_config_disable(struct virtio_device *dev) |
147 | { |
148 | spin_lock_irq(lock: &dev->config_lock); |
149 | dev->config_enabled = false; |
150 | spin_unlock_irq(lock: &dev->config_lock); |
151 | } |
152 | |
153 | static void virtio_config_enable(struct virtio_device *dev) |
154 | { |
155 | spin_lock_irq(lock: &dev->config_lock); |
156 | dev->config_enabled = true; |
157 | if (dev->config_change_pending) |
158 | __virtio_config_changed(dev); |
159 | dev->config_change_pending = false; |
160 | spin_unlock_irq(lock: &dev->config_lock); |
161 | } |
162 | |
163 | void virtio_add_status(struct virtio_device *dev, unsigned int status) |
164 | { |
165 | might_sleep(); |
166 | dev->config->set_status(dev, dev->config->get_status(dev) | status); |
167 | } |
168 | EXPORT_SYMBOL_GPL(virtio_add_status); |
169 | |
170 | /* Do some validation, then set FEATURES_OK */ |
171 | static int virtio_features_ok(struct virtio_device *dev) |
172 | { |
173 | unsigned int status; |
174 | |
175 | might_sleep(); |
176 | |
177 | if (virtio_check_mem_acc_cb(dev)) { |
178 | if (!virtio_has_feature(vdev: dev, VIRTIO_F_VERSION_1)) { |
179 | dev_warn(&dev->dev, |
180 | "device must provide VIRTIO_F_VERSION_1\n" ); |
181 | return -ENODEV; |
182 | } |
183 | |
184 | if (!virtio_has_feature(vdev: dev, VIRTIO_F_ACCESS_PLATFORM)) { |
185 | dev_warn(&dev->dev, |
186 | "device must provide VIRTIO_F_ACCESS_PLATFORM\n" ); |
187 | return -ENODEV; |
188 | } |
189 | } |
190 | |
191 | if (!virtio_has_feature(vdev: dev, VIRTIO_F_VERSION_1)) |
192 | return 0; |
193 | |
194 | virtio_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK); |
195 | status = dev->config->get_status(dev); |
196 | if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) { |
197 | dev_err(&dev->dev, "virtio: device refuses features: %x\n" , |
198 | status); |
199 | return -ENODEV; |
200 | } |
201 | return 0; |
202 | } |
203 | |
204 | /** |
205 | * virtio_reset_device - quiesce device for removal |
206 | * @dev: the device to reset |
207 | * |
208 | * Prevents device from sending interrupts and accessing memory. |
209 | * |
210 | * Generally used for cleanup during driver / device removal. |
211 | * |
212 | * Once this has been invoked, caller must ensure that |
213 | * virtqueue_notify / virtqueue_kick are not in progress. |
214 | * |
215 | * Note: this guarantees that vq callbacks are not in progress, however caller |
216 | * is responsible for preventing access from other contexts, such as a system |
217 | * call/workqueue/bh. Invoking virtio_break_device then flushing any such |
218 | * contexts is one way to handle that. |
219 | * */ |
220 | void virtio_reset_device(struct virtio_device *dev) |
221 | { |
222 | #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION |
223 | /* |
224 | * The below virtio_synchronize_cbs() guarantees that any |
225 | * interrupt for this line arriving after |
226 | * virtio_synchronize_vqs() has completed is guaranteed to see |
227 | * vq->broken as true. |
228 | */ |
229 | virtio_break_device(dev); |
230 | virtio_synchronize_cbs(dev); |
231 | #endif |
232 | |
233 | dev->config->reset(dev); |
234 | } |
235 | EXPORT_SYMBOL_GPL(virtio_reset_device); |
236 | |
237 | static int virtio_dev_probe(struct device *_d) |
238 | { |
239 | int err, i; |
240 | struct virtio_device *dev = dev_to_virtio(_d); |
241 | struct virtio_driver *drv = drv_to_virtio(drv: dev->dev.driver); |
242 | u64 device_features; |
243 | u64 driver_features; |
244 | u64 driver_features_legacy; |
245 | |
246 | /* We have a driver! */ |
247 | virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER); |
248 | |
249 | /* Figure out what features the device supports. */ |
250 | device_features = dev->config->get_features(dev); |
251 | |
252 | /* Figure out what features the driver supports. */ |
253 | driver_features = 0; |
254 | for (i = 0; i < drv->feature_table_size; i++) { |
255 | unsigned int f = drv->feature_table[i]; |
256 | BUG_ON(f >= 64); |
257 | driver_features |= (1ULL << f); |
258 | } |
259 | |
260 | /* Some drivers have a separate feature table for virtio v1.0 */ |
261 | if (drv->feature_table_legacy) { |
262 | driver_features_legacy = 0; |
263 | for (i = 0; i < drv->feature_table_size_legacy; i++) { |
264 | unsigned int f = drv->feature_table_legacy[i]; |
265 | BUG_ON(f >= 64); |
266 | driver_features_legacy |= (1ULL << f); |
267 | } |
268 | } else { |
269 | driver_features_legacy = driver_features; |
270 | } |
271 | |
272 | if (device_features & (1ULL << VIRTIO_F_VERSION_1)) |
273 | dev->features = driver_features & device_features; |
274 | else |
275 | dev->features = driver_features_legacy & device_features; |
276 | |
277 | /* Transport features always preserved to pass to finalize_features. */ |
278 | for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) |
279 | if (device_features & (1ULL << i)) |
280 | __virtio_set_bit(vdev: dev, fbit: i); |
281 | |
282 | err = dev->config->finalize_features(dev); |
283 | if (err) |
284 | goto err; |
285 | |
286 | if (drv->validate) { |
287 | u64 features = dev->features; |
288 | |
289 | err = drv->validate(dev); |
290 | if (err) |
291 | goto err; |
292 | |
293 | /* Did validation change any features? Then write them again. */ |
294 | if (features != dev->features) { |
295 | err = dev->config->finalize_features(dev); |
296 | if (err) |
297 | goto err; |
298 | } |
299 | } |
300 | |
301 | err = virtio_features_ok(dev); |
302 | if (err) |
303 | goto err; |
304 | |
305 | err = drv->probe(dev); |
306 | if (err) |
307 | goto err; |
308 | |
309 | /* If probe didn't do it, mark device DRIVER_OK ourselves. */ |
310 | if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK)) |
311 | virtio_device_ready(dev); |
312 | |
313 | if (drv->scan) |
314 | drv->scan(dev); |
315 | |
316 | virtio_config_enable(dev); |
317 | |
318 | return 0; |
319 | err: |
320 | virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED); |
321 | return err; |
322 | |
323 | } |
324 | |
325 | static void virtio_dev_remove(struct device *_d) |
326 | { |
327 | struct virtio_device *dev = dev_to_virtio(_d); |
328 | struct virtio_driver *drv = drv_to_virtio(drv: dev->dev.driver); |
329 | |
330 | virtio_config_disable(dev); |
331 | |
332 | drv->remove(dev); |
333 | |
334 | /* Driver should have reset device. */ |
335 | WARN_ON_ONCE(dev->config->get_status(dev)); |
336 | |
337 | /* Acknowledge the device's existence again. */ |
338 | virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE); |
339 | |
340 | of_node_put(node: dev->dev.of_node); |
341 | } |
342 | |
343 | static struct bus_type virtio_bus = { |
344 | .name = "virtio" , |
345 | .match = virtio_dev_match, |
346 | .dev_groups = virtio_dev_groups, |
347 | .uevent = virtio_uevent, |
348 | .probe = virtio_dev_probe, |
349 | .remove = virtio_dev_remove, |
350 | }; |
351 | |
352 | int register_virtio_driver(struct virtio_driver *driver) |
353 | { |
354 | /* Catch this early. */ |
355 | BUG_ON(driver->feature_table_size && !driver->feature_table); |
356 | driver->driver.bus = &virtio_bus; |
357 | return driver_register(drv: &driver->driver); |
358 | } |
359 | EXPORT_SYMBOL_GPL(register_virtio_driver); |
360 | |
361 | void unregister_virtio_driver(struct virtio_driver *driver) |
362 | { |
363 | driver_unregister(drv: &driver->driver); |
364 | } |
365 | EXPORT_SYMBOL_GPL(unregister_virtio_driver); |
366 | |
367 | static int virtio_device_of_init(struct virtio_device *dev) |
368 | { |
369 | struct device_node *np, *pnode = dev_of_node(dev: dev->dev.parent); |
370 | char compat[] = "virtio,deviceXXXXXXXX" ; |
371 | int ret, count; |
372 | |
373 | if (!pnode) |
374 | return 0; |
375 | |
376 | count = of_get_available_child_count(np: pnode); |
377 | if (!count) |
378 | return 0; |
379 | |
380 | /* There can be only 1 child node */ |
381 | if (WARN_ON(count > 1)) |
382 | return -EINVAL; |
383 | |
384 | np = of_get_next_available_child(node: pnode, NULL); |
385 | if (WARN_ON(!np)) |
386 | return -ENODEV; |
387 | |
388 | ret = snprintf(buf: compat, size: sizeof(compat), fmt: "virtio,device%x" , dev->id.device); |
389 | BUG_ON(ret >= sizeof(compat)); |
390 | |
391 | /* |
392 | * On powerpc/pseries virtio devices are PCI devices so PCI |
393 | * vendor/device ids play the role of the "compatible" property. |
394 | * Simply don't init of_node in this case. |
395 | */ |
396 | if (!of_device_is_compatible(device: np, compat)) { |
397 | ret = 0; |
398 | goto out; |
399 | } |
400 | |
401 | dev->dev.of_node = np; |
402 | return 0; |
403 | |
404 | out: |
405 | of_node_put(node: np); |
406 | return ret; |
407 | } |
408 | |
409 | /** |
410 | * register_virtio_device - register virtio device |
411 | * @dev : virtio device to be registered |
412 | * |
413 | * On error, the caller must call put_device on &@dev->dev (and not kfree), |
414 | * as another code path may have obtained a reference to @dev. |
415 | * |
416 | * Returns: 0 on suceess, -error on failure |
417 | */ |
418 | int register_virtio_device(struct virtio_device *dev) |
419 | { |
420 | int err; |
421 | |
422 | dev->dev.bus = &virtio_bus; |
423 | device_initialize(dev: &dev->dev); |
424 | |
425 | /* Assign a unique device index and hence name. */ |
426 | err = ida_alloc(ida: &virtio_index_ida, GFP_KERNEL); |
427 | if (err < 0) |
428 | goto out; |
429 | |
430 | dev->index = err; |
431 | err = dev_set_name(dev: &dev->dev, name: "virtio%u" , dev->index); |
432 | if (err) |
433 | goto out_ida_remove; |
434 | |
435 | err = virtio_device_of_init(dev); |
436 | if (err) |
437 | goto out_ida_remove; |
438 | |
439 | spin_lock_init(&dev->config_lock); |
440 | dev->config_enabled = false; |
441 | dev->config_change_pending = false; |
442 | |
443 | INIT_LIST_HEAD(list: &dev->vqs); |
444 | spin_lock_init(&dev->vqs_list_lock); |
445 | |
446 | /* We always start by resetting the device, in case a previous |
447 | * driver messed it up. This also tests that code path a little. */ |
448 | virtio_reset_device(dev); |
449 | |
450 | /* Acknowledge that we've seen the device. */ |
451 | virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE); |
452 | |
453 | /* |
454 | * device_add() causes the bus infrastructure to look for a matching |
455 | * driver. |
456 | */ |
457 | err = device_add(dev: &dev->dev); |
458 | if (err) |
459 | goto out_of_node_put; |
460 | |
461 | return 0; |
462 | |
463 | out_of_node_put: |
464 | of_node_put(node: dev->dev.of_node); |
465 | out_ida_remove: |
466 | ida_free(&virtio_index_ida, id: dev->index); |
467 | out: |
468 | virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED); |
469 | return err; |
470 | } |
471 | EXPORT_SYMBOL_GPL(register_virtio_device); |
472 | |
473 | bool is_virtio_device(struct device *dev) |
474 | { |
475 | return dev->bus == &virtio_bus; |
476 | } |
477 | EXPORT_SYMBOL_GPL(is_virtio_device); |
478 | |
479 | void unregister_virtio_device(struct virtio_device *dev) |
480 | { |
481 | int index = dev->index; /* save for after device release */ |
482 | |
483 | device_unregister(dev: &dev->dev); |
484 | ida_free(&virtio_index_ida, id: index); |
485 | } |
486 | EXPORT_SYMBOL_GPL(unregister_virtio_device); |
487 | |
488 | #ifdef CONFIG_PM_SLEEP |
489 | int virtio_device_freeze(struct virtio_device *dev) |
490 | { |
491 | struct virtio_driver *drv = drv_to_virtio(drv: dev->dev.driver); |
492 | |
493 | virtio_config_disable(dev); |
494 | |
495 | dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED; |
496 | |
497 | if (drv && drv->freeze) |
498 | return drv->freeze(dev); |
499 | |
500 | return 0; |
501 | } |
502 | EXPORT_SYMBOL_GPL(virtio_device_freeze); |
503 | |
504 | int virtio_device_restore(struct virtio_device *dev) |
505 | { |
506 | struct virtio_driver *drv = drv_to_virtio(drv: dev->dev.driver); |
507 | int ret; |
508 | |
509 | /* We always start by resetting the device, in case a previous |
510 | * driver messed it up. */ |
511 | virtio_reset_device(dev); |
512 | |
513 | /* Acknowledge that we've seen the device. */ |
514 | virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE); |
515 | |
516 | /* Maybe driver failed before freeze. |
517 | * Restore the failed status, for debugging. */ |
518 | if (dev->failed) |
519 | virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED); |
520 | |
521 | if (!drv) |
522 | return 0; |
523 | |
524 | /* We have a driver! */ |
525 | virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER); |
526 | |
527 | ret = dev->config->finalize_features(dev); |
528 | if (ret) |
529 | goto err; |
530 | |
531 | ret = virtio_features_ok(dev); |
532 | if (ret) |
533 | goto err; |
534 | |
535 | if (drv->restore) { |
536 | ret = drv->restore(dev); |
537 | if (ret) |
538 | goto err; |
539 | } |
540 | |
541 | /* If restore didn't do it, mark device DRIVER_OK ourselves. */ |
542 | if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK)) |
543 | virtio_device_ready(dev); |
544 | |
545 | virtio_config_enable(dev); |
546 | |
547 | return 0; |
548 | |
549 | err: |
550 | virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED); |
551 | return ret; |
552 | } |
553 | EXPORT_SYMBOL_GPL(virtio_device_restore); |
554 | #endif |
555 | |
556 | static int virtio_init(void) |
557 | { |
558 | if (bus_register(bus: &virtio_bus) != 0) |
559 | panic(fmt: "virtio bus registration failed" ); |
560 | return 0; |
561 | } |
562 | |
563 | static void __exit virtio_exit(void) |
564 | { |
565 | bus_unregister(bus: &virtio_bus); |
566 | ida_destroy(ida: &virtio_index_ida); |
567 | } |
568 | core_initcall(virtio_init); |
569 | module_exit(virtio_exit); |
570 | |
571 | MODULE_LICENSE("GPL" ); |
572 | |