1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * VIRTIO based driver for vDPA device |
4 | * |
5 | * Copyright (c) 2020, Red Hat. All rights reserved. |
6 | * Author: Jason Wang <jasowang@redhat.com> |
7 | * |
8 | */ |
9 | |
10 | #include <linux/init.h> |
11 | #include <linux/module.h> |
12 | #include <linux/device.h> |
13 | #include <linux/kernel.h> |
14 | #include <linux/slab.h> |
15 | #include <linux/uuid.h> |
16 | #include <linux/group_cpus.h> |
17 | #include <linux/virtio.h> |
18 | #include <linux/vdpa.h> |
19 | #include <linux/virtio_config.h> |
20 | #include <linux/virtio_ring.h> |
21 | |
22 | #define MOD_VERSION "0.1" |
23 | #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>" |
24 | #define MOD_DESC "vDPA bus driver for virtio devices" |
25 | #define MOD_LICENSE "GPL v2" |
26 | |
27 | struct virtio_vdpa_device { |
28 | struct virtio_device vdev; |
29 | struct vdpa_device *vdpa; |
30 | u64 features; |
31 | |
32 | /* The lock to protect virtqueue list */ |
33 | spinlock_t lock; |
34 | /* List of virtio_vdpa_vq_info */ |
35 | struct list_head virtqueues; |
36 | }; |
37 | |
38 | struct virtio_vdpa_vq_info { |
39 | /* the actual virtqueue */ |
40 | struct virtqueue *vq; |
41 | |
42 | /* the list node for the virtqueues list */ |
43 | struct list_head node; |
44 | }; |
45 | |
46 | static inline struct virtio_vdpa_device * |
47 | to_virtio_vdpa_device(struct virtio_device *dev) |
48 | { |
49 | return container_of(dev, struct virtio_vdpa_device, vdev); |
50 | } |
51 | |
52 | static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev) |
53 | { |
54 | return to_virtio_vdpa_device(dev: vdev)->vdpa; |
55 | } |
56 | |
57 | static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset, |
58 | void *buf, unsigned int len) |
59 | { |
60 | struct vdpa_device *vdpa = vd_get_vdpa(vdev); |
61 | |
62 | vdpa_get_config(vdev: vdpa, offset, buf, len); |
63 | } |
64 | |
65 | static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset, |
66 | const void *buf, unsigned int len) |
67 | { |
68 | struct vdpa_device *vdpa = vd_get_vdpa(vdev); |
69 | |
70 | vdpa_set_config(dev: vdpa, offset, buf, length: len); |
71 | } |
72 | |
73 | static u32 virtio_vdpa_generation(struct virtio_device *vdev) |
74 | { |
75 | struct vdpa_device *vdpa = vd_get_vdpa(vdev); |
76 | const struct vdpa_config_ops *ops = vdpa->config; |
77 | |
78 | if (ops->get_generation) |
79 | return ops->get_generation(vdpa); |
80 | |
81 | return 0; |
82 | } |
83 | |
84 | static u8 virtio_vdpa_get_status(struct virtio_device *vdev) |
85 | { |
86 | struct vdpa_device *vdpa = vd_get_vdpa(vdev); |
87 | const struct vdpa_config_ops *ops = vdpa->config; |
88 | |
89 | return ops->get_status(vdpa); |
90 | } |
91 | |
92 | static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status) |
93 | { |
94 | struct vdpa_device *vdpa = vd_get_vdpa(vdev); |
95 | |
96 | return vdpa_set_status(vdev: vdpa, status); |
97 | } |
98 | |
99 | static void virtio_vdpa_reset(struct virtio_device *vdev) |
100 | { |
101 | struct vdpa_device *vdpa = vd_get_vdpa(vdev); |
102 | |
103 | vdpa_reset(vdev: vdpa, flags: 0); |
104 | } |
105 | |
106 | static bool virtio_vdpa_notify(struct virtqueue *vq) |
107 | { |
108 | struct vdpa_device *vdpa = vd_get_vdpa(vdev: vq->vdev); |
109 | const struct vdpa_config_ops *ops = vdpa->config; |
110 | |
111 | ops->kick_vq(vdpa, vq->index); |
112 | |
113 | return true; |
114 | } |
115 | |
116 | static bool virtio_vdpa_notify_with_data(struct virtqueue *vq) |
117 | { |
118 | struct vdpa_device *vdpa = vd_get_vdpa(vdev: vq->vdev); |
119 | const struct vdpa_config_ops *ops = vdpa->config; |
120 | u32 data = vring_notification_data(vq: vq); |
121 | |
122 | ops->kick_vq_with_data(vdpa, data); |
123 | |
124 | return true; |
125 | } |
126 | |
127 | static irqreturn_t virtio_vdpa_config_cb(void *private) |
128 | { |
129 | struct virtio_vdpa_device *vd_dev = private; |
130 | |
131 | virtio_config_changed(dev: &vd_dev->vdev); |
132 | |
133 | return IRQ_HANDLED; |
134 | } |
135 | |
136 | static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) |
137 | { |
138 | struct virtio_vdpa_vq_info *info = private; |
139 | |
140 | return vring_interrupt(irq: 0, vq: info->vq); |
141 | } |
142 | |
143 | static struct virtqueue * |
144 | virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, |
145 | void (*callback)(struct virtqueue *vq), |
146 | const char *name, bool ctx) |
147 | { |
148 | struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vdev); |
149 | struct vdpa_device *vdpa = vd_get_vdpa(vdev); |
150 | struct device *dma_dev; |
151 | const struct vdpa_config_ops *ops = vdpa->config; |
152 | struct virtio_vdpa_vq_info *info; |
153 | bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify; |
154 | struct vdpa_callback cb; |
155 | struct virtqueue *vq; |
156 | u64 desc_addr, driver_addr, device_addr; |
157 | /* Assume split virtqueue, switch to packed if necessary */ |
158 | struct vdpa_vq_state state = {0}; |
159 | unsigned long flags; |
160 | u32 align, max_num, min_num = 1; |
161 | bool may_reduce_num = true; |
162 | int err; |
163 | |
164 | if (!name) |
165 | return NULL; |
166 | |
167 | if (index >= vdpa->nvqs) |
168 | return ERR_PTR(error: -ENOENT); |
169 | |
170 | /* We cannot accept VIRTIO_F_NOTIFICATION_DATA without kick_vq_with_data */ |
171 | if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) { |
172 | if (ops->kick_vq_with_data) |
173 | notify = virtio_vdpa_notify_with_data; |
174 | else |
175 | __virtio_clear_bit(vdev, VIRTIO_F_NOTIFICATION_DATA); |
176 | } |
177 | |
178 | /* Queue shouldn't already be set up. */ |
179 | if (ops->get_vq_ready(vdpa, index)) |
180 | return ERR_PTR(error: -ENOENT); |
181 | |
182 | /* Allocate and fill out our active queue description */ |
183 | info = kmalloc(size: sizeof(*info), GFP_KERNEL); |
184 | if (!info) |
185 | return ERR_PTR(error: -ENOMEM); |
186 | if (ops->get_vq_size) |
187 | max_num = ops->get_vq_size(vdpa, index); |
188 | else |
189 | max_num = ops->get_vq_num_max(vdpa); |
190 | |
191 | if (max_num == 0) { |
192 | err = -ENOENT; |
193 | goto error_new_virtqueue; |
194 | } |
195 | |
196 | if (ops->get_vq_num_min) |
197 | min_num = ops->get_vq_num_min(vdpa); |
198 | |
199 | may_reduce_num = (max_num == min_num) ? false : true; |
200 | |
201 | /* Create the vring */ |
202 | align = ops->get_vq_align(vdpa); |
203 | |
204 | if (ops->get_vq_dma_dev) |
205 | dma_dev = ops->get_vq_dma_dev(vdpa, index); |
206 | else |
207 | dma_dev = vdpa_get_dma_dev(vdev: vdpa); |
208 | vq = vring_create_virtqueue_dma(index, num: max_num, vring_align: align, vdev, |
209 | weak_barriers: true, may_reduce_num, ctx, |
210 | notify, callback, name, dma_dev); |
211 | if (!vq) { |
212 | err = -ENOMEM; |
213 | goto error_new_virtqueue; |
214 | } |
215 | |
216 | vq->num_max = max_num; |
217 | |
218 | /* Setup virtqueue callback */ |
219 | cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL; |
220 | cb.private = info; |
221 | cb.trigger = NULL; |
222 | ops->set_vq_cb(vdpa, index, &cb); |
223 | ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq)); |
224 | |
225 | desc_addr = virtqueue_get_desc_addr(vq); |
226 | driver_addr = virtqueue_get_avail_addr(vq); |
227 | device_addr = virtqueue_get_used_addr(vq); |
228 | |
229 | if (ops->set_vq_address(vdpa, index, |
230 | desc_addr, driver_addr, |
231 | device_addr)) { |
232 | err = -EINVAL; |
233 | goto err_vq; |
234 | } |
235 | |
236 | /* reset virtqueue state index */ |
237 | if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
238 | struct vdpa_vq_state_packed *s = &state.packed; |
239 | |
240 | s->last_avail_counter = 1; |
241 | s->last_avail_idx = 0; |
242 | s->last_used_counter = 1; |
243 | s->last_used_idx = 0; |
244 | } |
245 | err = ops->set_vq_state(vdpa, index, &state); |
246 | if (err) |
247 | goto err_vq; |
248 | |
249 | ops->set_vq_ready(vdpa, index, 1); |
250 | |
251 | vq->priv = info; |
252 | info->vq = vq; |
253 | |
254 | spin_lock_irqsave(&vd_dev->lock, flags); |
255 | list_add(new: &info->node, head: &vd_dev->virtqueues); |
256 | spin_unlock_irqrestore(lock: &vd_dev->lock, flags); |
257 | |
258 | return vq; |
259 | |
260 | err_vq: |
261 | vring_del_virtqueue(vq); |
262 | error_new_virtqueue: |
263 | ops->set_vq_ready(vdpa, index, 0); |
264 | /* VDPA driver should make sure vq is stopeed here */ |
265 | WARN_ON(ops->get_vq_ready(vdpa, index)); |
266 | kfree(objp: info); |
267 | return ERR_PTR(error: err); |
268 | } |
269 | |
270 | static void virtio_vdpa_del_vq(struct virtqueue *vq) |
271 | { |
272 | struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vq->vdev); |
273 | struct vdpa_device *vdpa = vd_dev->vdpa; |
274 | const struct vdpa_config_ops *ops = vdpa->config; |
275 | struct virtio_vdpa_vq_info *info = vq->priv; |
276 | unsigned int index = vq->index; |
277 | unsigned long flags; |
278 | |
279 | spin_lock_irqsave(&vd_dev->lock, flags); |
280 | list_del(entry: &info->node); |
281 | spin_unlock_irqrestore(lock: &vd_dev->lock, flags); |
282 | |
283 | /* Select and deactivate the queue (best effort) */ |
284 | ops->set_vq_ready(vdpa, index, 0); |
285 | |
286 | vring_del_virtqueue(vq); |
287 | |
288 | kfree(objp: info); |
289 | } |
290 | |
291 | static void virtio_vdpa_del_vqs(struct virtio_device *vdev) |
292 | { |
293 | struct virtqueue *vq, *n; |
294 | |
295 | list_for_each_entry_safe(vq, n, &vdev->vqs, list) |
296 | virtio_vdpa_del_vq(vq); |
297 | } |
298 | |
299 | static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs) |
300 | { |
301 | affd->nr_sets = 1; |
302 | affd->set_size[0] = affvecs; |
303 | } |
304 | |
305 | static struct cpumask * |
306 | create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd) |
307 | { |
308 | unsigned int affvecs = 0, curvec, usedvecs, i; |
309 | struct cpumask *masks = NULL; |
310 | |
311 | if (nvecs > affd->pre_vectors + affd->post_vectors) |
312 | affvecs = nvecs - affd->pre_vectors - affd->post_vectors; |
313 | |
314 | if (!affd->calc_sets) |
315 | affd->calc_sets = default_calc_sets; |
316 | |
317 | affd->calc_sets(affd, affvecs); |
318 | |
319 | if (!affvecs) |
320 | return NULL; |
321 | |
322 | masks = kcalloc(n: nvecs, size: sizeof(*masks), GFP_KERNEL); |
323 | if (!masks) |
324 | return NULL; |
325 | |
326 | /* Fill out vectors at the beginning that don't need affinity */ |
327 | for (curvec = 0; curvec < affd->pre_vectors; curvec++) |
328 | cpumask_setall(dstp: &masks[curvec]); |
329 | |
330 | for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) { |
331 | unsigned int this_vecs = affd->set_size[i]; |
332 | int j; |
333 | struct cpumask *result = group_cpus_evenly(numgrps: this_vecs); |
334 | |
335 | if (!result) { |
336 | kfree(objp: masks); |
337 | return NULL; |
338 | } |
339 | |
340 | for (j = 0; j < this_vecs; j++) |
341 | cpumask_copy(dstp: &masks[curvec + j], srcp: &result[j]); |
342 | kfree(objp: result); |
343 | |
344 | curvec += this_vecs; |
345 | usedvecs += this_vecs; |
346 | } |
347 | |
348 | /* Fill out vectors at the end that don't need affinity */ |
349 | if (usedvecs >= affvecs) |
350 | curvec = affd->pre_vectors + affvecs; |
351 | else |
352 | curvec = affd->pre_vectors + usedvecs; |
353 | for (; curvec < nvecs; curvec++) |
354 | cpumask_setall(dstp: &masks[curvec]); |
355 | |
356 | return masks; |
357 | } |
358 | |
359 | static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, |
360 | struct virtqueue *vqs[], |
361 | vq_callback_t *callbacks[], |
362 | const char * const names[], |
363 | const bool *ctx, |
364 | struct irq_affinity *desc) |
365 | { |
366 | struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vdev); |
367 | struct vdpa_device *vdpa = vd_get_vdpa(vdev); |
368 | const struct vdpa_config_ops *ops = vdpa->config; |
369 | struct irq_affinity default_affd = { 0 }; |
370 | struct cpumask *masks; |
371 | struct vdpa_callback cb; |
372 | bool has_affinity = desc && ops->set_vq_affinity; |
373 | int i, err, queue_idx = 0; |
374 | |
375 | if (has_affinity) { |
376 | masks = create_affinity_masks(nvecs: nvqs, affd: desc ? desc : &default_affd); |
377 | if (!masks) |
378 | return -ENOMEM; |
379 | } |
380 | |
381 | for (i = 0; i < nvqs; ++i) { |
382 | if (!names[i]) { |
383 | vqs[i] = NULL; |
384 | continue; |
385 | } |
386 | |
387 | vqs[i] = virtio_vdpa_setup_vq(vdev, index: queue_idx++, |
388 | callback: callbacks[i], name: names[i], ctx: ctx ? |
389 | ctx[i] : false); |
390 | if (IS_ERR(ptr: vqs[i])) { |
391 | err = PTR_ERR(ptr: vqs[i]); |
392 | goto err_setup_vq; |
393 | } |
394 | |
395 | if (has_affinity) |
396 | ops->set_vq_affinity(vdpa, i, &masks[i]); |
397 | } |
398 | |
399 | cb.callback = virtio_vdpa_config_cb; |
400 | cb.private = vd_dev; |
401 | ops->set_config_cb(vdpa, &cb); |
402 | if (has_affinity) |
403 | kfree(objp: masks); |
404 | |
405 | return 0; |
406 | |
407 | err_setup_vq: |
408 | virtio_vdpa_del_vqs(vdev); |
409 | if (has_affinity) |
410 | kfree(objp: masks); |
411 | return err; |
412 | } |
413 | |
414 | static u64 virtio_vdpa_get_features(struct virtio_device *vdev) |
415 | { |
416 | struct vdpa_device *vdpa = vd_get_vdpa(vdev); |
417 | const struct vdpa_config_ops *ops = vdpa->config; |
418 | |
419 | return ops->get_device_features(vdpa); |
420 | } |
421 | |
422 | static int virtio_vdpa_finalize_features(struct virtio_device *vdev) |
423 | { |
424 | struct vdpa_device *vdpa = vd_get_vdpa(vdev); |
425 | |
426 | /* Give virtio_ring a chance to accept features. */ |
427 | vring_transport_features(vdev); |
428 | |
429 | return vdpa_set_features(vdev: vdpa, features: vdev->features); |
430 | } |
431 | |
432 | static const char *virtio_vdpa_bus_name(struct virtio_device *vdev) |
433 | { |
434 | struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vdev); |
435 | struct vdpa_device *vdpa = vd_dev->vdpa; |
436 | |
437 | return dev_name(dev: &vdpa->dev); |
438 | } |
439 | |
440 | static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq, |
441 | const struct cpumask *cpu_mask) |
442 | { |
443 | struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vq->vdev); |
444 | struct vdpa_device *vdpa = vd_dev->vdpa; |
445 | const struct vdpa_config_ops *ops = vdpa->config; |
446 | unsigned int index = vq->index; |
447 | |
448 | if (ops->set_vq_affinity) |
449 | return ops->set_vq_affinity(vdpa, index, cpu_mask); |
450 | |
451 | return 0; |
452 | } |
453 | |
454 | static const struct cpumask * |
455 | virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index) |
456 | { |
457 | struct vdpa_device *vdpa = vd_get_vdpa(vdev); |
458 | const struct vdpa_config_ops *ops = vdpa->config; |
459 | |
460 | if (ops->get_vq_affinity) |
461 | return ops->get_vq_affinity(vdpa, index); |
462 | |
463 | return NULL; |
464 | } |
465 | |
466 | static const struct virtio_config_ops virtio_vdpa_config_ops = { |
467 | .get = virtio_vdpa_get, |
468 | .set = virtio_vdpa_set, |
469 | .generation = virtio_vdpa_generation, |
470 | .get_status = virtio_vdpa_get_status, |
471 | .set_status = virtio_vdpa_set_status, |
472 | .reset = virtio_vdpa_reset, |
473 | .find_vqs = virtio_vdpa_find_vqs, |
474 | .del_vqs = virtio_vdpa_del_vqs, |
475 | .get_features = virtio_vdpa_get_features, |
476 | .finalize_features = virtio_vdpa_finalize_features, |
477 | .bus_name = virtio_vdpa_bus_name, |
478 | .set_vq_affinity = virtio_vdpa_set_vq_affinity, |
479 | .get_vq_affinity = virtio_vdpa_get_vq_affinity, |
480 | }; |
481 | |
482 | static void virtio_vdpa_release_dev(struct device *_d) |
483 | { |
484 | struct virtio_device *vdev = |
485 | container_of(_d, struct virtio_device, dev); |
486 | struct virtio_vdpa_device *vd_dev = |
487 | container_of(vdev, struct virtio_vdpa_device, vdev); |
488 | |
489 | kfree(objp: vd_dev); |
490 | } |
491 | |
492 | static int virtio_vdpa_probe(struct vdpa_device *vdpa) |
493 | { |
494 | const struct vdpa_config_ops *ops = vdpa->config; |
495 | struct virtio_vdpa_device *vd_dev, *reg_dev = NULL; |
496 | int ret = -EINVAL; |
497 | |
498 | vd_dev = kzalloc(size: sizeof(*vd_dev), GFP_KERNEL); |
499 | if (!vd_dev) |
500 | return -ENOMEM; |
501 | |
502 | vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdev: vdpa); |
503 | vd_dev->vdev.dev.release = virtio_vdpa_release_dev; |
504 | vd_dev->vdev.config = &virtio_vdpa_config_ops; |
505 | vd_dev->vdpa = vdpa; |
506 | INIT_LIST_HEAD(list: &vd_dev->virtqueues); |
507 | spin_lock_init(&vd_dev->lock); |
508 | |
509 | vd_dev->vdev.id.device = ops->get_device_id(vdpa); |
510 | if (vd_dev->vdev.id.device == 0) |
511 | goto err; |
512 | |
513 | vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa); |
514 | ret = register_virtio_device(dev: &vd_dev->vdev); |
515 | reg_dev = vd_dev; |
516 | if (ret) |
517 | goto err; |
518 | |
519 | vdpa_set_drvdata(vdev: vdpa, data: vd_dev); |
520 | |
521 | return 0; |
522 | |
523 | err: |
524 | if (reg_dev) |
525 | put_device(dev: &vd_dev->vdev.dev); |
526 | else |
527 | kfree(objp: vd_dev); |
528 | return ret; |
529 | } |
530 | |
531 | static void virtio_vdpa_remove(struct vdpa_device *vdpa) |
532 | { |
533 | struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdev: vdpa); |
534 | |
535 | unregister_virtio_device(dev: &vd_dev->vdev); |
536 | } |
537 | |
538 | static struct vdpa_driver virtio_vdpa_driver = { |
539 | .driver = { |
540 | .name = "virtio_vdpa" , |
541 | }, |
542 | .probe = virtio_vdpa_probe, |
543 | .remove = virtio_vdpa_remove, |
544 | }; |
545 | |
546 | module_vdpa_driver(virtio_vdpa_driver); |
547 | |
548 | MODULE_VERSION(MOD_VERSION); |
549 | MODULE_LICENSE(MOD_LICENSE); |
550 | MODULE_AUTHOR(MOD_AUTHOR); |
551 | MODULE_DESCRIPTION(MOD_DESC); |
552 | |