1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VIRTIO based driver for vDPA device
4 *
5 * Copyright (c) 2020, Red Hat. All rights reserved.
6 * Author: Jason Wang <jasowang@redhat.com>
7 *
8 */
9
10#include <linux/init.h>
11#include <linux/module.h>
12#include <linux/device.h>
13#include <linux/kernel.h>
14#include <linux/slab.h>
15#include <linux/uuid.h>
16#include <linux/group_cpus.h>
17#include <linux/virtio.h>
18#include <linux/vdpa.h>
19#include <linux/virtio_config.h>
20#include <linux/virtio_ring.h>
21
22#define MOD_VERSION "0.1"
23#define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>"
24#define MOD_DESC "vDPA bus driver for virtio devices"
25#define MOD_LICENSE "GPL v2"
26
27struct virtio_vdpa_device {
28 struct virtio_device vdev;
29 struct vdpa_device *vdpa;
30 u64 features;
31
32 /* The lock to protect virtqueue list */
33 spinlock_t lock;
34 /* List of virtio_vdpa_vq_info */
35 struct list_head virtqueues;
36};
37
38struct virtio_vdpa_vq_info {
39 /* the actual virtqueue */
40 struct virtqueue *vq;
41
42 /* the list node for the virtqueues list */
43 struct list_head node;
44};
45
46static inline struct virtio_vdpa_device *
47to_virtio_vdpa_device(struct virtio_device *dev)
48{
49 return container_of(dev, struct virtio_vdpa_device, vdev);
50}
51
52static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev)
53{
54 return to_virtio_vdpa_device(dev: vdev)->vdpa;
55}
56
57static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset,
58 void *buf, unsigned int len)
59{
60 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
61
62 vdpa_get_config(vdev: vdpa, offset, buf, len);
63}
64
65static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset,
66 const void *buf, unsigned int len)
67{
68 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
69
70 vdpa_set_config(dev: vdpa, offset, buf, length: len);
71}
72
73static u32 virtio_vdpa_generation(struct virtio_device *vdev)
74{
75 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
76 const struct vdpa_config_ops *ops = vdpa->config;
77
78 if (ops->get_generation)
79 return ops->get_generation(vdpa);
80
81 return 0;
82}
83
84static u8 virtio_vdpa_get_status(struct virtio_device *vdev)
85{
86 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
87 const struct vdpa_config_ops *ops = vdpa->config;
88
89 return ops->get_status(vdpa);
90}
91
92static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
93{
94 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
95
96 return vdpa_set_status(vdev: vdpa, status);
97}
98
99static void virtio_vdpa_reset(struct virtio_device *vdev)
100{
101 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
102
103 vdpa_reset(vdev: vdpa, flags: 0);
104}
105
106static bool virtio_vdpa_notify(struct virtqueue *vq)
107{
108 struct vdpa_device *vdpa = vd_get_vdpa(vdev: vq->vdev);
109 const struct vdpa_config_ops *ops = vdpa->config;
110
111 ops->kick_vq(vdpa, vq->index);
112
113 return true;
114}
115
116static bool virtio_vdpa_notify_with_data(struct virtqueue *vq)
117{
118 struct vdpa_device *vdpa = vd_get_vdpa(vdev: vq->vdev);
119 const struct vdpa_config_ops *ops = vdpa->config;
120 u32 data = vring_notification_data(vq: vq);
121
122 ops->kick_vq_with_data(vdpa, data);
123
124 return true;
125}
126
127static irqreturn_t virtio_vdpa_config_cb(void *private)
128{
129 struct virtio_vdpa_device *vd_dev = private;
130
131 virtio_config_changed(dev: &vd_dev->vdev);
132
133 return IRQ_HANDLED;
134}
135
136static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)
137{
138 struct virtio_vdpa_vq_info *info = private;
139
140 return vring_interrupt(irq: 0, vq: info->vq);
141}
142
143static struct virtqueue *
144virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
145 void (*callback)(struct virtqueue *vq),
146 const char *name, bool ctx)
147{
148 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vdev);
149 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
150 struct device *dma_dev;
151 const struct vdpa_config_ops *ops = vdpa->config;
152 struct virtio_vdpa_vq_info *info;
153 bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify;
154 struct vdpa_callback cb;
155 struct virtqueue *vq;
156 u64 desc_addr, driver_addr, device_addr;
157 /* Assume split virtqueue, switch to packed if necessary */
158 struct vdpa_vq_state state = {0};
159 unsigned long flags;
160 u32 align, max_num, min_num = 1;
161 bool may_reduce_num = true;
162 int err;
163
164 if (!name)
165 return NULL;
166
167 if (index >= vdpa->nvqs)
168 return ERR_PTR(error: -ENOENT);
169
170 /* We cannot accept VIRTIO_F_NOTIFICATION_DATA without kick_vq_with_data */
171 if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) {
172 if (ops->kick_vq_with_data)
173 notify = virtio_vdpa_notify_with_data;
174 else
175 __virtio_clear_bit(vdev, VIRTIO_F_NOTIFICATION_DATA);
176 }
177
178 /* Queue shouldn't already be set up. */
179 if (ops->get_vq_ready(vdpa, index))
180 return ERR_PTR(error: -ENOENT);
181
182 /* Allocate and fill out our active queue description */
183 info = kmalloc(size: sizeof(*info), GFP_KERNEL);
184 if (!info)
185 return ERR_PTR(error: -ENOMEM);
186 if (ops->get_vq_size)
187 max_num = ops->get_vq_size(vdpa, index);
188 else
189 max_num = ops->get_vq_num_max(vdpa);
190
191 if (max_num == 0) {
192 err = -ENOENT;
193 goto error_new_virtqueue;
194 }
195
196 if (ops->get_vq_num_min)
197 min_num = ops->get_vq_num_min(vdpa);
198
199 may_reduce_num = (max_num == min_num) ? false : true;
200
201 /* Create the vring */
202 align = ops->get_vq_align(vdpa);
203
204 if (ops->get_vq_dma_dev)
205 dma_dev = ops->get_vq_dma_dev(vdpa, index);
206 else
207 dma_dev = vdpa_get_dma_dev(vdev: vdpa);
208 vq = vring_create_virtqueue_dma(index, num: max_num, vring_align: align, vdev,
209 weak_barriers: true, may_reduce_num, ctx,
210 notify, callback, name, dma_dev);
211 if (!vq) {
212 err = -ENOMEM;
213 goto error_new_virtqueue;
214 }
215
216 vq->num_max = max_num;
217
218 /* Setup virtqueue callback */
219 cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;
220 cb.private = info;
221 cb.trigger = NULL;
222 ops->set_vq_cb(vdpa, index, &cb);
223 ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
224
225 desc_addr = virtqueue_get_desc_addr(vq);
226 driver_addr = virtqueue_get_avail_addr(vq);
227 device_addr = virtqueue_get_used_addr(vq);
228
229 if (ops->set_vq_address(vdpa, index,
230 desc_addr, driver_addr,
231 device_addr)) {
232 err = -EINVAL;
233 goto err_vq;
234 }
235
236 /* reset virtqueue state index */
237 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
238 struct vdpa_vq_state_packed *s = &state.packed;
239
240 s->last_avail_counter = 1;
241 s->last_avail_idx = 0;
242 s->last_used_counter = 1;
243 s->last_used_idx = 0;
244 }
245 err = ops->set_vq_state(vdpa, index, &state);
246 if (err)
247 goto err_vq;
248
249 ops->set_vq_ready(vdpa, index, 1);
250
251 vq->priv = info;
252 info->vq = vq;
253
254 spin_lock_irqsave(&vd_dev->lock, flags);
255 list_add(new: &info->node, head: &vd_dev->virtqueues);
256 spin_unlock_irqrestore(lock: &vd_dev->lock, flags);
257
258 return vq;
259
260err_vq:
261 vring_del_virtqueue(vq);
262error_new_virtqueue:
263 ops->set_vq_ready(vdpa, index, 0);
264 /* VDPA driver should make sure vq is stopeed here */
265 WARN_ON(ops->get_vq_ready(vdpa, index));
266 kfree(objp: info);
267 return ERR_PTR(error: err);
268}
269
270static void virtio_vdpa_del_vq(struct virtqueue *vq)
271{
272 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vq->vdev);
273 struct vdpa_device *vdpa = vd_dev->vdpa;
274 const struct vdpa_config_ops *ops = vdpa->config;
275 struct virtio_vdpa_vq_info *info = vq->priv;
276 unsigned int index = vq->index;
277 unsigned long flags;
278
279 spin_lock_irqsave(&vd_dev->lock, flags);
280 list_del(entry: &info->node);
281 spin_unlock_irqrestore(lock: &vd_dev->lock, flags);
282
283 /* Select and deactivate the queue (best effort) */
284 ops->set_vq_ready(vdpa, index, 0);
285
286 vring_del_virtqueue(vq);
287
288 kfree(objp: info);
289}
290
291static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
292{
293 struct virtqueue *vq, *n;
294
295 list_for_each_entry_safe(vq, n, &vdev->vqs, list)
296 virtio_vdpa_del_vq(vq);
297}
298
299static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs)
300{
301 affd->nr_sets = 1;
302 affd->set_size[0] = affvecs;
303}
304
305static struct cpumask *
306create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd)
307{
308 unsigned int affvecs = 0, curvec, usedvecs, i;
309 struct cpumask *masks = NULL;
310
311 if (nvecs > affd->pre_vectors + affd->post_vectors)
312 affvecs = nvecs - affd->pre_vectors - affd->post_vectors;
313
314 if (!affd->calc_sets)
315 affd->calc_sets = default_calc_sets;
316
317 affd->calc_sets(affd, affvecs);
318
319 if (!affvecs)
320 return NULL;
321
322 masks = kcalloc(n: nvecs, size: sizeof(*masks), GFP_KERNEL);
323 if (!masks)
324 return NULL;
325
326 /* Fill out vectors at the beginning that don't need affinity */
327 for (curvec = 0; curvec < affd->pre_vectors; curvec++)
328 cpumask_setall(dstp: &masks[curvec]);
329
330 for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) {
331 unsigned int this_vecs = affd->set_size[i];
332 int j;
333 struct cpumask *result = group_cpus_evenly(numgrps: this_vecs);
334
335 if (!result) {
336 kfree(objp: masks);
337 return NULL;
338 }
339
340 for (j = 0; j < this_vecs; j++)
341 cpumask_copy(dstp: &masks[curvec + j], srcp: &result[j]);
342 kfree(objp: result);
343
344 curvec += this_vecs;
345 usedvecs += this_vecs;
346 }
347
348 /* Fill out vectors at the end that don't need affinity */
349 if (usedvecs >= affvecs)
350 curvec = affd->pre_vectors + affvecs;
351 else
352 curvec = affd->pre_vectors + usedvecs;
353 for (; curvec < nvecs; curvec++)
354 cpumask_setall(dstp: &masks[curvec]);
355
356 return masks;
357}
358
359static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
360 struct virtqueue *vqs[],
361 vq_callback_t *callbacks[],
362 const char * const names[],
363 const bool *ctx,
364 struct irq_affinity *desc)
365{
366 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vdev);
367 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
368 const struct vdpa_config_ops *ops = vdpa->config;
369 struct irq_affinity default_affd = { 0 };
370 struct cpumask *masks;
371 struct vdpa_callback cb;
372 bool has_affinity = desc && ops->set_vq_affinity;
373 int i, err, queue_idx = 0;
374
375 if (has_affinity) {
376 masks = create_affinity_masks(nvecs: nvqs, affd: desc ? desc : &default_affd);
377 if (!masks)
378 return -ENOMEM;
379 }
380
381 for (i = 0; i < nvqs; ++i) {
382 if (!names[i]) {
383 vqs[i] = NULL;
384 continue;
385 }
386
387 vqs[i] = virtio_vdpa_setup_vq(vdev, index: queue_idx++,
388 callback: callbacks[i], name: names[i], ctx: ctx ?
389 ctx[i] : false);
390 if (IS_ERR(ptr: vqs[i])) {
391 err = PTR_ERR(ptr: vqs[i]);
392 goto err_setup_vq;
393 }
394
395 if (has_affinity)
396 ops->set_vq_affinity(vdpa, i, &masks[i]);
397 }
398
399 cb.callback = virtio_vdpa_config_cb;
400 cb.private = vd_dev;
401 ops->set_config_cb(vdpa, &cb);
402 if (has_affinity)
403 kfree(objp: masks);
404
405 return 0;
406
407err_setup_vq:
408 virtio_vdpa_del_vqs(vdev);
409 if (has_affinity)
410 kfree(objp: masks);
411 return err;
412}
413
414static u64 virtio_vdpa_get_features(struct virtio_device *vdev)
415{
416 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
417 const struct vdpa_config_ops *ops = vdpa->config;
418
419 return ops->get_device_features(vdpa);
420}
421
422static int virtio_vdpa_finalize_features(struct virtio_device *vdev)
423{
424 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
425
426 /* Give virtio_ring a chance to accept features. */
427 vring_transport_features(vdev);
428
429 return vdpa_set_features(vdev: vdpa, features: vdev->features);
430}
431
432static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
433{
434 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vdev);
435 struct vdpa_device *vdpa = vd_dev->vdpa;
436
437 return dev_name(dev: &vdpa->dev);
438}
439
440static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq,
441 const struct cpumask *cpu_mask)
442{
443 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vq->vdev);
444 struct vdpa_device *vdpa = vd_dev->vdpa;
445 const struct vdpa_config_ops *ops = vdpa->config;
446 unsigned int index = vq->index;
447
448 if (ops->set_vq_affinity)
449 return ops->set_vq_affinity(vdpa, index, cpu_mask);
450
451 return 0;
452}
453
454static const struct cpumask *
455virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index)
456{
457 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
458 const struct vdpa_config_ops *ops = vdpa->config;
459
460 if (ops->get_vq_affinity)
461 return ops->get_vq_affinity(vdpa, index);
462
463 return NULL;
464}
465
466static const struct virtio_config_ops virtio_vdpa_config_ops = {
467 .get = virtio_vdpa_get,
468 .set = virtio_vdpa_set,
469 .generation = virtio_vdpa_generation,
470 .get_status = virtio_vdpa_get_status,
471 .set_status = virtio_vdpa_set_status,
472 .reset = virtio_vdpa_reset,
473 .find_vqs = virtio_vdpa_find_vqs,
474 .del_vqs = virtio_vdpa_del_vqs,
475 .get_features = virtio_vdpa_get_features,
476 .finalize_features = virtio_vdpa_finalize_features,
477 .bus_name = virtio_vdpa_bus_name,
478 .set_vq_affinity = virtio_vdpa_set_vq_affinity,
479 .get_vq_affinity = virtio_vdpa_get_vq_affinity,
480};
481
482static void virtio_vdpa_release_dev(struct device *_d)
483{
484 struct virtio_device *vdev =
485 container_of(_d, struct virtio_device, dev);
486 struct virtio_vdpa_device *vd_dev =
487 container_of(vdev, struct virtio_vdpa_device, vdev);
488
489 kfree(objp: vd_dev);
490}
491
492static int virtio_vdpa_probe(struct vdpa_device *vdpa)
493{
494 const struct vdpa_config_ops *ops = vdpa->config;
495 struct virtio_vdpa_device *vd_dev, *reg_dev = NULL;
496 int ret = -EINVAL;
497
498 vd_dev = kzalloc(size: sizeof(*vd_dev), GFP_KERNEL);
499 if (!vd_dev)
500 return -ENOMEM;
501
502 vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdev: vdpa);
503 vd_dev->vdev.dev.release = virtio_vdpa_release_dev;
504 vd_dev->vdev.config = &virtio_vdpa_config_ops;
505 vd_dev->vdpa = vdpa;
506 INIT_LIST_HEAD(list: &vd_dev->virtqueues);
507 spin_lock_init(&vd_dev->lock);
508
509 vd_dev->vdev.id.device = ops->get_device_id(vdpa);
510 if (vd_dev->vdev.id.device == 0)
511 goto err;
512
513 vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa);
514 ret = register_virtio_device(dev: &vd_dev->vdev);
515 reg_dev = vd_dev;
516 if (ret)
517 goto err;
518
519 vdpa_set_drvdata(vdev: vdpa, data: vd_dev);
520
521 return 0;
522
523err:
524 if (reg_dev)
525 put_device(dev: &vd_dev->vdev.dev);
526 else
527 kfree(objp: vd_dev);
528 return ret;
529}
530
531static void virtio_vdpa_remove(struct vdpa_device *vdpa)
532{
533 struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdev: vdpa);
534
535 unregister_virtio_device(dev: &vd_dev->vdev);
536}
537
538static struct vdpa_driver virtio_vdpa_driver = {
539 .driver = {
540 .name = "virtio_vdpa",
541 },
542 .probe = virtio_vdpa_probe,
543 .remove = virtio_vdpa_remove,
544};
545
546module_vdpa_driver(virtio_vdpa_driver);
547
548MODULE_VERSION(MOD_VERSION);
549MODULE_LICENSE(MOD_LICENSE);
550MODULE_AUTHOR(MOD_AUTHOR);
551MODULE_DESCRIPTION(MOD_DESC);
552

source code of linux/drivers/virtio/virtio_vdpa.c