1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #define _GNU_SOURCE |
3 | #include <getopt.h> |
4 | #include <limits.h> |
5 | #include <string.h> |
6 | #include <poll.h> |
7 | #include <sys/eventfd.h> |
8 | #include <stdlib.h> |
9 | #include <assert.h> |
10 | #include <unistd.h> |
11 | #include <sys/ioctl.h> |
12 | #include <sys/stat.h> |
13 | #include <sys/types.h> |
14 | #include <fcntl.h> |
15 | #include <stdbool.h> |
16 | #include <linux/virtio_types.h> |
17 | #include <linux/vhost.h> |
18 | #include <linux/virtio.h> |
19 | #include <linux/virtio_ring.h> |
20 | #include "../../drivers/vhost/test.h" |
21 | |
22 | #define RANDOM_BATCH -1 |
23 | |
24 | /* Unused */ |
25 | void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end; |
26 | |
27 | struct vq_info { |
28 | int kick; |
29 | int call; |
30 | int num; |
31 | int idx; |
32 | void *ring; |
33 | /* copy used for control */ |
34 | struct vring vring; |
35 | struct virtqueue *vq; |
36 | }; |
37 | |
38 | struct vdev_info { |
39 | struct virtio_device vdev; |
40 | int control; |
41 | struct pollfd fds[1]; |
42 | struct vq_info vqs[1]; |
43 | int nvqs; |
44 | void *buf; |
45 | size_t buf_size; |
46 | struct vhost_memory *mem; |
47 | }; |
48 | |
49 | static const struct vhost_vring_file no_backend = { .fd = -1 }, |
50 | backend = { .fd = 1 }; |
51 | static const struct vhost_vring_state null_state = {}; |
52 | |
53 | bool vq_notify(struct virtqueue *vq) |
54 | { |
55 | struct vq_info *info = vq->priv; |
56 | unsigned long long v = 1; |
57 | int r; |
58 | r = write(info->kick, &v, sizeof v); |
59 | assert(r == sizeof v); |
60 | return true; |
61 | } |
62 | |
63 | void vq_callback(struct virtqueue *vq) |
64 | { |
65 | } |
66 | |
67 | |
68 | void vhost_vq_setup(struct vdev_info *dev, struct vq_info *info) |
69 | { |
70 | struct vhost_vring_state state = { .index = info->idx }; |
71 | struct vhost_vring_file file = { .index = info->idx }; |
72 | unsigned long long features = dev->vdev.features; |
73 | struct vhost_vring_addr addr = { |
74 | .index = info->idx, |
75 | .desc_user_addr = (uint64_t)(unsigned long)info->vring.desc, |
76 | .avail_user_addr = (uint64_t)(unsigned long)info->vring.avail, |
77 | .used_user_addr = (uint64_t)(unsigned long)info->vring.used, |
78 | }; |
79 | int r; |
80 | r = ioctl(dev->control, VHOST_SET_FEATURES, &features); |
81 | assert(r >= 0); |
82 | state.num = info->vring.num; |
83 | r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state); |
84 | assert(r >= 0); |
85 | state.num = 0; |
86 | r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state); |
87 | assert(r >= 0); |
88 | r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr); |
89 | assert(r >= 0); |
90 | file.fd = info->kick; |
91 | r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file); |
92 | assert(r >= 0); |
93 | file.fd = info->call; |
94 | r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file); |
95 | assert(r >= 0); |
96 | } |
97 | |
98 | static void vq_reset(struct vq_info *info, int num, struct virtio_device *vdev) |
99 | { |
100 | if (info->vq) |
101 | vring_del_virtqueue(vq: info->vq); |
102 | |
103 | memset(info->ring, 0, vring_size(num, 4096)); |
104 | vring_init(vr: &info->vring, num, p: info->ring, align: 4096); |
105 | info->vq = vring_new_virtqueue(index: info->idx, num, vring_align: 4096, vdev, weak_barriers: true, ctx: false, |
106 | pages: info->ring, notify: vq_notify, callback: vq_callback, name: "test" ); |
107 | assert(info->vq); |
108 | info->vq->priv = info; |
109 | } |
110 | |
111 | static void vq_info_add(struct vdev_info *dev, int num) |
112 | { |
113 | struct vq_info *info = &dev->vqs[dev->nvqs]; |
114 | int r; |
115 | info->idx = dev->nvqs; |
116 | info->kick = eventfd(0, EFD_NONBLOCK); |
117 | info->call = eventfd(0, EFD_NONBLOCK); |
118 | r = posix_memalign(&info->ring, 4096, vring_size(num, align: 4096)); |
119 | assert(r >= 0); |
120 | vq_reset(info, num, vdev: &dev->vdev); |
121 | vhost_vq_setup(dev, info); |
122 | dev->fds[info->idx].fd = info->call; |
123 | dev->fds[info->idx].events = POLLIN; |
124 | dev->nvqs++; |
125 | } |
126 | |
127 | static void vdev_info_init(struct vdev_info* dev, unsigned long long features) |
128 | { |
129 | int r; |
130 | memset(dev, 0, sizeof *dev); |
131 | dev->vdev.features = features; |
132 | INIT_LIST_HEAD(list: &dev->vdev.vqs); |
133 | spin_lock_init(&dev->vdev.vqs_list_lock); |
134 | dev->buf_size = 1024; |
135 | dev->buf = malloc(dev->buf_size); |
136 | assert(dev->buf); |
137 | dev->control = open("/dev/vhost-test" , O_RDWR); |
138 | assert(dev->control >= 0); |
139 | r = ioctl(dev->control, VHOST_SET_OWNER, NULL); |
140 | assert(r >= 0); |
141 | dev->mem = malloc(offsetof(struct vhost_memory, regions) + |
142 | sizeof dev->mem->regions[0]); |
143 | assert(dev->mem); |
144 | memset(dev->mem, 0, offsetof(struct vhost_memory, regions) + |
145 | sizeof dev->mem->regions[0]); |
146 | dev->mem->nregions = 1; |
147 | dev->mem->regions[0].guest_phys_addr = (long)dev->buf; |
148 | dev->mem->regions[0].userspace_addr = (long)dev->buf; |
149 | dev->mem->regions[0].memory_size = dev->buf_size; |
150 | r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem); |
151 | assert(r >= 0); |
152 | } |
153 | |
154 | /* TODO: this is pretty bad: we get a cache line bounce |
155 | * for the wait queue on poll and another one on read, |
156 | * plus the read which is there just to clear the |
157 | * current state. */ |
158 | static void wait_for_interrupt(struct vdev_info *dev) |
159 | { |
160 | int i; |
161 | unsigned long long val; |
162 | poll(dev->fds, dev->nvqs, -1); |
163 | for (i = 0; i < dev->nvqs; ++i) |
164 | if (dev->fds[i].revents & POLLIN) { |
165 | read(dev->fds[i].fd, &val, sizeof val); |
166 | } |
167 | } |
168 | |
169 | static void run_test(struct vdev_info *dev, struct vq_info *vq, |
170 | bool delayed, int batch, int reset_n, int bufs) |
171 | { |
172 | struct scatterlist sl; |
173 | long started = 0, completed = 0, next_reset = reset_n; |
174 | long completed_before, started_before; |
175 | int r, test = 1; |
176 | unsigned int len; |
177 | long long spurious = 0; |
178 | const bool random_batch = batch == RANDOM_BATCH; |
179 | |
180 | r = ioctl(dev->control, VHOST_TEST_RUN, &test); |
181 | assert(r >= 0); |
182 | if (!reset_n) { |
183 | next_reset = INT_MAX; |
184 | } |
185 | |
186 | for (;;) { |
187 | virtqueue_disable_cb(vq: vq->vq); |
188 | completed_before = completed; |
189 | started_before = started; |
190 | do { |
191 | const bool reset = completed > next_reset; |
192 | if (random_batch) |
193 | batch = (random() % vq->vring.num) + 1; |
194 | |
195 | while (started < bufs && |
196 | (started - completed) < batch) { |
197 | sg_init_one(&sl, dev->buf, dev->buf_size); |
198 | r = virtqueue_add_outbuf(vq: vq->vq, sg: &sl, num: 1, |
199 | data: dev->buf + started, |
200 | GFP_ATOMIC); |
201 | if (unlikely(r != 0)) { |
202 | if (r == -ENOSPC && |
203 | started > started_before) |
204 | r = 0; |
205 | else |
206 | r = -1; |
207 | break; |
208 | } |
209 | |
210 | ++started; |
211 | |
212 | if (unlikely(!virtqueue_kick(vq->vq))) { |
213 | r = -1; |
214 | break; |
215 | } |
216 | } |
217 | |
218 | if (started >= bufs) |
219 | r = -1; |
220 | |
221 | if (reset) { |
222 | r = ioctl(dev->control, VHOST_TEST_SET_BACKEND, |
223 | &no_backend); |
224 | assert(!r); |
225 | } |
226 | |
227 | /* Flush out completed bufs if any */ |
228 | while (virtqueue_get_buf(vq: vq->vq, len: &len)) { |
229 | ++completed; |
230 | r = 0; |
231 | } |
232 | |
233 | if (reset) { |
234 | struct vhost_vring_state s = { .index = 0 }; |
235 | |
236 | vq_reset(info: vq, num: vq->vring.num, vdev: &dev->vdev); |
237 | |
238 | r = ioctl(dev->control, VHOST_GET_VRING_BASE, |
239 | &s); |
240 | assert(!r); |
241 | |
242 | s.num = 0; |
243 | r = ioctl(dev->control, VHOST_SET_VRING_BASE, |
244 | &null_state); |
245 | assert(!r); |
246 | |
247 | r = ioctl(dev->control, VHOST_TEST_SET_BACKEND, |
248 | &backend); |
249 | assert(!r); |
250 | |
251 | started = completed; |
252 | while (completed > next_reset) |
253 | next_reset += completed; |
254 | } |
255 | } while (r == 0); |
256 | if (completed == completed_before && started == started_before) |
257 | ++spurious; |
258 | assert(completed <= bufs); |
259 | assert(started <= bufs); |
260 | if (completed == bufs) |
261 | break; |
262 | if (delayed) { |
263 | if (virtqueue_enable_cb_delayed(vq: vq->vq)) |
264 | wait_for_interrupt(dev); |
265 | } else { |
266 | if (virtqueue_enable_cb(vq: vq->vq)) |
267 | wait_for_interrupt(dev); |
268 | } |
269 | } |
270 | test = 0; |
271 | r = ioctl(dev->control, VHOST_TEST_RUN, &test); |
272 | assert(r >= 0); |
273 | fprintf(stderr, |
274 | "spurious wakeups: 0x%llx started=0x%lx completed=0x%lx\n" , |
275 | spurious, started, completed); |
276 | } |
277 | |
278 | const char optstring[] = "h" ; |
279 | const struct option longopts[] = { |
280 | { |
281 | .name = "help" , |
282 | .val = 'h', |
283 | }, |
284 | { |
285 | .name = "event-idx" , |
286 | .val = 'E', |
287 | }, |
288 | { |
289 | .name = "no-event-idx" , |
290 | .val = 'e', |
291 | }, |
292 | { |
293 | .name = "indirect" , |
294 | .val = 'I', |
295 | }, |
296 | { |
297 | .name = "no-indirect" , |
298 | .val = 'i', |
299 | }, |
300 | { |
301 | .name = "virtio-1" , |
302 | .val = '1', |
303 | }, |
304 | { |
305 | .name = "no-virtio-1" , |
306 | .val = '0', |
307 | }, |
308 | { |
309 | .name = "delayed-interrupt" , |
310 | .val = 'D', |
311 | }, |
312 | { |
313 | .name = "no-delayed-interrupt" , |
314 | .val = 'd', |
315 | }, |
316 | { |
317 | .name = "batch" , |
318 | .val = 'b', |
319 | .has_arg = required_argument, |
320 | }, |
321 | { |
322 | .name = "reset" , |
323 | .val = 'r', |
324 | .has_arg = optional_argument, |
325 | }, |
326 | { |
327 | } |
328 | }; |
329 | |
330 | static void help(int status) |
331 | { |
332 | fprintf(stderr, "Usage: virtio_test [--help]" |
333 | " [--no-indirect]" |
334 | " [--no-event-idx]" |
335 | " [--no-virtio-1]" |
336 | " [--delayed-interrupt]" |
337 | " [--batch=random/N]" |
338 | " [--reset=N]" |
339 | "\n" ); |
340 | |
341 | exit(status); |
342 | } |
343 | |
344 | int main(int argc, char **argv) |
345 | { |
346 | struct vdev_info dev; |
347 | unsigned long long features = (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | |
348 | (1ULL << VIRTIO_RING_F_EVENT_IDX) | (1ULL << VIRTIO_F_VERSION_1); |
349 | long batch = 1, reset = 0; |
350 | int o; |
351 | bool delayed = false; |
352 | |
353 | for (;;) { |
354 | o = getopt_long(argc, argv, optstring, longopts, NULL); |
355 | switch (o) { |
356 | case -1: |
357 | goto done; |
358 | case '?': |
359 | help(status: 2); |
360 | case 'e': |
361 | features &= ~(1ULL << VIRTIO_RING_F_EVENT_IDX); |
362 | break; |
363 | case 'h': |
364 | help(status: 0); |
365 | case 'i': |
366 | features &= ~(1ULL << VIRTIO_RING_F_INDIRECT_DESC); |
367 | break; |
368 | case '0': |
369 | features &= ~(1ULL << VIRTIO_F_VERSION_1); |
370 | break; |
371 | case 'D': |
372 | delayed = true; |
373 | break; |
374 | case 'b': |
375 | if (0 == strcmp(optarg, "random" )) { |
376 | batch = RANDOM_BATCH; |
377 | } else { |
378 | batch = strtol(optarg, NULL, 10); |
379 | assert(batch > 0); |
380 | assert(batch < (long)INT_MAX + 1); |
381 | } |
382 | break; |
383 | case 'r': |
384 | if (!optarg) { |
385 | reset = 1; |
386 | } else { |
387 | reset = strtol(optarg, NULL, 10); |
388 | assert(reset > 0); |
389 | assert(reset < (long)INT_MAX + 1); |
390 | } |
391 | break; |
392 | default: |
393 | assert(0); |
394 | break; |
395 | } |
396 | } |
397 | |
398 | done: |
399 | vdev_info_init(dev: &dev, features); |
400 | vq_info_add(dev: &dev, num: 256); |
401 | run_test(dev: &dev, vq: &dev.vqs[0], delayed, batch, reset_n: reset, bufs: 0x100000); |
402 | return 0; |
403 | } |
404 | |