1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Simple test of virtio code, entirely in userpsace. */ |
3 | #define _GNU_SOURCE |
4 | #include <sched.h> |
5 | #include <err.h> |
6 | #include <linux/kernel.h> |
7 | #include <linux/err.h> |
8 | #include <linux/virtio.h> |
9 | #include <linux/vringh.h> |
10 | #include <linux/virtio_ring.h> |
11 | #include <linux/virtio_config.h> |
12 | #include <linux/uaccess.h> |
13 | #include <sys/types.h> |
14 | #include <sys/stat.h> |
15 | #include <sys/mman.h> |
16 | #include <sys/wait.h> |
17 | #include <fcntl.h> |
18 | |
19 | #define USER_MEM (1024*1024) |
20 | void *__user_addr_min, *__user_addr_max; |
21 | void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end; |
22 | static u64 user_addr_offset; |
23 | |
24 | #define RINGSIZE 256 |
25 | #define ALIGN 4096 |
26 | |
27 | static bool never_notify_host(struct virtqueue *vq) |
28 | { |
29 | abort(); |
30 | } |
31 | |
32 | static void never_callback_guest(struct virtqueue *vq) |
33 | { |
34 | abort(); |
35 | } |
36 | |
37 | static bool getrange_iov(struct vringh *vrh, u64 addr, struct vringh_range *r) |
38 | { |
39 | if (addr < (u64)(unsigned long)__user_addr_min - user_addr_offset) |
40 | return false; |
41 | if (addr >= (u64)(unsigned long)__user_addr_max - user_addr_offset) |
42 | return false; |
43 | |
44 | r->start = (u64)(unsigned long)__user_addr_min - user_addr_offset; |
45 | r->end_incl = (u64)(unsigned long)__user_addr_max - 1 - user_addr_offset; |
46 | r->offset = user_addr_offset; |
47 | return true; |
48 | } |
49 | |
50 | /* We return single byte ranges. */ |
51 | static bool getrange_slow(struct vringh *vrh, u64 addr, struct vringh_range *r) |
52 | { |
53 | if (addr < (u64)(unsigned long)__user_addr_min - user_addr_offset) |
54 | return false; |
55 | if (addr >= (u64)(unsigned long)__user_addr_max - user_addr_offset) |
56 | return false; |
57 | |
58 | r->start = addr; |
59 | r->end_incl = r->start; |
60 | r->offset = user_addr_offset; |
61 | return true; |
62 | } |
63 | |
64 | struct guest_virtio_device { |
65 | struct virtio_device vdev; |
66 | int to_host_fd; |
67 | unsigned long notifies; |
68 | }; |
69 | |
70 | static bool parallel_notify_host(struct virtqueue *vq) |
71 | { |
72 | int rc; |
73 | struct guest_virtio_device *gvdev; |
74 | |
75 | gvdev = container_of(vq->vdev, struct guest_virtio_device, vdev); |
76 | rc = write(gvdev->to_host_fd, "" , 1); |
77 | if (rc < 0) |
78 | return false; |
79 | gvdev->notifies++; |
80 | return true; |
81 | } |
82 | |
83 | static bool no_notify_host(struct virtqueue *vq) |
84 | { |
85 | return true; |
86 | } |
87 | |
88 | #define NUM_XFERS (10000000) |
89 | |
90 | /* We aim for two "distant" cpus. */ |
91 | static void find_cpus(unsigned int *first, unsigned int *last) |
92 | { |
93 | unsigned int i; |
94 | |
95 | *first = -1U; |
96 | *last = 0; |
97 | for (i = 0; i < 4096; i++) { |
98 | cpu_set_t set; |
99 | CPU_ZERO(&set); |
100 | CPU_SET(i, &set); |
101 | if (sched_setaffinity(pid: getpid(), new_mask: sizeof(set), &set) == 0) { |
102 | if (i < *first) |
103 | *first = i; |
104 | if (i > *last) |
105 | *last = i; |
106 | } |
107 | } |
108 | } |
109 | |
110 | /* Opencoded version for fast mode */ |
111 | static inline int vringh_get_head(struct vringh *vrh, u16 *head) |
112 | { |
113 | u16 avail_idx, i; |
114 | int err; |
115 | |
116 | err = get_user(avail_idx, &vrh->vring.avail->idx); |
117 | if (err) |
118 | return err; |
119 | |
120 | if (vrh->last_avail_idx == avail_idx) |
121 | return 0; |
122 | |
123 | /* Only get avail ring entries after they have been exposed by guest. */ |
124 | virtio_rmb(weak_barriers: vrh->weak_barriers); |
125 | |
126 | i = vrh->last_avail_idx & (vrh->vring.num - 1); |
127 | |
128 | err = get_user(*head, &vrh->vring.avail->ring[i]); |
129 | if (err) |
130 | return err; |
131 | |
132 | vrh->last_avail_idx++; |
133 | return 1; |
134 | } |
135 | |
136 | static int parallel_test(u64 features, |
137 | bool (*getrange)(struct vringh *vrh, |
138 | u64 addr, struct vringh_range *r), |
139 | bool fast_vringh) |
140 | { |
141 | void *host_map, *guest_map; |
142 | int fd, mapsize, to_guest[2], to_host[2]; |
143 | unsigned long xfers = 0, notifies = 0, receives = 0; |
144 | unsigned int first_cpu, last_cpu; |
145 | cpu_set_t cpu_set; |
146 | char buf[128]; |
147 | |
148 | /* Create real file to mmap. */ |
149 | fd = open("/tmp/vringh_test-file" , O_RDWR|O_CREAT|O_TRUNC, 0600); |
150 | if (fd < 0) |
151 | err(1, "Opening /tmp/vringh_test-file" ); |
152 | |
153 | /* Extra room at the end for some data, and indirects */ |
154 | mapsize = vring_size(RINGSIZE, ALIGN) |
155 | + RINGSIZE * 2 * sizeof(int) |
156 | + RINGSIZE * 6 * sizeof(struct vring_desc); |
157 | mapsize = (mapsize + getpagesize() - 1) & ~(getpagesize() - 1); |
158 | ftruncate(fd, mapsize); |
159 | |
160 | /* Parent and child use separate addresses, to check our mapping logic! */ |
161 | host_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); |
162 | guest_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); |
163 | |
164 | pipe(to_guest); |
165 | pipe(to_host); |
166 | |
167 | CPU_ZERO(&cpu_set); |
168 | find_cpus(first: &first_cpu, last: &last_cpu); |
169 | printf("Using CPUS %u and %u\n" , first_cpu, last_cpu); |
170 | fflush(stdout); |
171 | |
172 | if (fork() != 0) { |
173 | struct vringh vrh; |
174 | int status, err, rlen = 0; |
175 | char rbuf[5]; |
176 | |
177 | /* We are the host: never access guest addresses! */ |
178 | munmap(guest_map, mapsize); |
179 | |
180 | __user_addr_min = host_map; |
181 | __user_addr_max = __user_addr_min + mapsize; |
182 | user_addr_offset = host_map - guest_map; |
183 | assert(user_addr_offset); |
184 | |
185 | close(to_guest[0]); |
186 | close(to_host[1]); |
187 | |
188 | vring_init(vr: &vrh.vring, RINGSIZE, p: host_map, ALIGN); |
189 | vringh_init_user(vrh: &vrh, features, RINGSIZE, weak_barriers: true, |
190 | desc: vrh.vring.desc, avail: vrh.vring.avail, used: vrh.vring.used); |
191 | CPU_SET(first_cpu, &cpu_set); |
192 | if (sched_setaffinity(getpid(), sizeof(cpu_set), &cpu_set)) |
193 | errx(1, "Could not set affinity to cpu %u" , first_cpu); |
194 | |
195 | while (xfers < NUM_XFERS) { |
196 | struct iovec host_riov[2], host_wiov[2]; |
197 | struct vringh_iov riov, wiov; |
198 | u16 head, written; |
199 | |
200 | if (fast_vringh) { |
201 | for (;;) { |
202 | err = vringh_get_head(vrh: &vrh, head: &head); |
203 | if (err != 0) |
204 | break; |
205 | err = vringh_need_notify_user(vrh: &vrh); |
206 | if (err < 0) |
207 | errx(1, "vringh_need_notify_user: %i" , |
208 | err); |
209 | if (err) { |
210 | write(to_guest[1], "" , 1); |
211 | notifies++; |
212 | } |
213 | } |
214 | if (err != 1) |
215 | errx(1, "vringh_get_head" ); |
216 | written = 0; |
217 | goto complete; |
218 | } else { |
219 | vringh_iov_init(iov: &riov, |
220 | iovec: host_riov, |
221 | ARRAY_SIZE(host_riov)); |
222 | vringh_iov_init(iov: &wiov, |
223 | iovec: host_wiov, |
224 | ARRAY_SIZE(host_wiov)); |
225 | |
226 | err = vringh_getdesc_user(vrh: &vrh, riov: &riov, wiov: &wiov, |
227 | getrange, head: &head); |
228 | } |
229 | if (err == 0) { |
230 | err = vringh_need_notify_user(vrh: &vrh); |
231 | if (err < 0) |
232 | errx(1, "vringh_need_notify_user: %i" , |
233 | err); |
234 | if (err) { |
235 | write(to_guest[1], "" , 1); |
236 | notifies++; |
237 | } |
238 | |
239 | if (!vringh_notify_enable_user(vrh: &vrh)) |
240 | continue; |
241 | |
242 | /* Swallow all notifies at once. */ |
243 | if (read(to_host[0], buf, sizeof(buf)) < 1) |
244 | break; |
245 | |
246 | vringh_notify_disable_user(vrh: &vrh); |
247 | receives++; |
248 | continue; |
249 | } |
250 | if (err != 1) |
251 | errx(1, "vringh_getdesc_user: %i" , err); |
252 | |
253 | /* We simply copy bytes. */ |
254 | if (riov.used) { |
255 | rlen = vringh_iov_pull_user(riov: &riov, dst: rbuf, |
256 | len: sizeof(rbuf)); |
257 | if (rlen != 4) |
258 | errx(1, "vringh_iov_pull_user: %i" , |
259 | rlen); |
260 | assert(riov.i == riov.used); |
261 | written = 0; |
262 | } else { |
263 | err = vringh_iov_push_user(wiov: &wiov, src: rbuf, len: rlen); |
264 | if (err != rlen) |
265 | errx(1, "vringh_iov_push_user: %i" , |
266 | err); |
267 | assert(wiov.i == wiov.used); |
268 | written = err; |
269 | } |
270 | complete: |
271 | xfers++; |
272 | |
273 | err = vringh_complete_user(vrh: &vrh, head, len: written); |
274 | if (err != 0) |
275 | errx(1, "vringh_complete_user: %i" , err); |
276 | } |
277 | |
278 | err = vringh_need_notify_user(vrh: &vrh); |
279 | if (err < 0) |
280 | errx(1, "vringh_need_notify_user: %i" , err); |
281 | if (err) { |
282 | write(to_guest[1], "" , 1); |
283 | notifies++; |
284 | } |
285 | wait(&status); |
286 | if (!WIFEXITED(status)) |
287 | errx(1, "Child died with signal %i?" , WTERMSIG(status)); |
288 | if (WEXITSTATUS(status) != 0) |
289 | errx(1, "Child exited %i?" , WEXITSTATUS(status)); |
290 | printf("Host: notified %lu, pinged %lu\n" , notifies, receives); |
291 | return 0; |
292 | } else { |
293 | struct guest_virtio_device gvdev; |
294 | struct virtqueue *vq; |
295 | unsigned int *data; |
296 | struct vring_desc *indirects; |
297 | unsigned int finished = 0; |
298 | |
299 | /* We pass sg[]s pointing into here, but we need RINGSIZE+1 */ |
300 | data = guest_map + vring_size(RINGSIZE, ALIGN); |
301 | indirects = (void *)data + (RINGSIZE + 1) * 2 * sizeof(int); |
302 | |
303 | /* We are the guest. */ |
304 | munmap(host_map, mapsize); |
305 | |
306 | close(to_guest[1]); |
307 | close(to_host[0]); |
308 | |
309 | gvdev.vdev.features = features; |
310 | INIT_LIST_HEAD(list: &gvdev.vdev.vqs); |
311 | spin_lock_init(&gvdev.vdev.vqs_list_lock); |
312 | gvdev.to_host_fd = to_host[1]; |
313 | gvdev.notifies = 0; |
314 | |
315 | CPU_SET(first_cpu, &cpu_set); |
316 | if (sched_setaffinity(getpid(), sizeof(cpu_set), &cpu_set)) |
317 | err(1, "Could not set affinity to cpu %u" , first_cpu); |
318 | |
319 | vq = vring_new_virtqueue(index: 0, RINGSIZE, ALIGN, vdev: &gvdev.vdev, weak_barriers: true, |
320 | ctx: false, pages: guest_map, |
321 | notify: fast_vringh ? no_notify_host |
322 | : parallel_notify_host, |
323 | callback: never_callback_guest, name: "guest vq" ); |
324 | |
325 | /* Don't kfree indirects. */ |
326 | __kfree_ignore_start = indirects; |
327 | __kfree_ignore_end = indirects + RINGSIZE * 6; |
328 | |
329 | while (xfers < NUM_XFERS) { |
330 | struct scatterlist sg[4]; |
331 | unsigned int num_sg, len; |
332 | int *dbuf, err; |
333 | bool output = !(xfers % 2); |
334 | |
335 | /* Consume bufs. */ |
336 | while ((dbuf = virtqueue_get_buf(vq, len: &len)) != NULL) { |
337 | if (len == 4) |
338 | assert(*dbuf == finished - 1); |
339 | else if (!fast_vringh) |
340 | assert(*dbuf == finished); |
341 | finished++; |
342 | } |
343 | |
344 | /* Produce a buffer. */ |
345 | dbuf = data + (xfers % (RINGSIZE + 1)); |
346 | |
347 | if (output) |
348 | *dbuf = xfers; |
349 | else |
350 | *dbuf = -1; |
351 | |
352 | switch ((xfers / sizeof(*dbuf)) % 4) { |
353 | case 0: |
354 | /* Nasty three-element sg list. */ |
355 | sg_init_table(sg, num_sg = 3); |
356 | sg_set_buf(sg: &sg[0], buf: (void *)dbuf, buflen: 1); |
357 | sg_set_buf(sg: &sg[1], buf: (void *)dbuf + 1, buflen: 2); |
358 | sg_set_buf(sg: &sg[2], buf: (void *)dbuf + 3, buflen: 1); |
359 | break; |
360 | case 1: |
361 | sg_init_table(sg, num_sg = 2); |
362 | sg_set_buf(sg: &sg[0], buf: (void *)dbuf, buflen: 1); |
363 | sg_set_buf(sg: &sg[1], buf: (void *)dbuf + 1, buflen: 3); |
364 | break; |
365 | case 2: |
366 | sg_init_table(sg, num_sg = 1); |
367 | sg_set_buf(sg: &sg[0], buf: (void *)dbuf, buflen: 4); |
368 | break; |
369 | case 3: |
370 | sg_init_table(sg, num_sg = 4); |
371 | sg_set_buf(sg: &sg[0], buf: (void *)dbuf, buflen: 1); |
372 | sg_set_buf(sg: &sg[1], buf: (void *)dbuf + 1, buflen: 1); |
373 | sg_set_buf(sg: &sg[2], buf: (void *)dbuf + 2, buflen: 1); |
374 | sg_set_buf(sg: &sg[3], buf: (void *)dbuf + 3, buflen: 1); |
375 | break; |
376 | } |
377 | |
378 | /* May allocate an indirect, so force it to allocate |
379 | * user addr */ |
380 | __kmalloc_fake = indirects + (xfers % RINGSIZE) * 4; |
381 | if (output) |
382 | err = virtqueue_add_outbuf(vq, sg, num: num_sg, data: dbuf, |
383 | GFP_KERNEL); |
384 | else |
385 | err = virtqueue_add_inbuf(vq, sg, num: num_sg, |
386 | data: dbuf, GFP_KERNEL); |
387 | |
388 | if (err == -ENOSPC) { |
389 | if (!virtqueue_enable_cb_delayed(vq)) |
390 | continue; |
391 | /* Swallow all notifies at once. */ |
392 | if (read(to_guest[0], buf, sizeof(buf)) < 1) |
393 | break; |
394 | |
395 | receives++; |
396 | virtqueue_disable_cb(vq); |
397 | continue; |
398 | } |
399 | |
400 | if (err) |
401 | errx(1, "virtqueue_add_in/outbuf: %i" , err); |
402 | |
403 | xfers++; |
404 | virtqueue_kick(vq); |
405 | } |
406 | |
407 | /* Any extra? */ |
408 | while (finished != xfers) { |
409 | int *dbuf; |
410 | unsigned int len; |
411 | |
412 | /* Consume bufs. */ |
413 | dbuf = virtqueue_get_buf(vq, len: &len); |
414 | if (dbuf) { |
415 | if (len == 4) |
416 | assert(*dbuf == finished - 1); |
417 | else |
418 | assert(len == 0); |
419 | finished++; |
420 | continue; |
421 | } |
422 | |
423 | if (!virtqueue_enable_cb_delayed(vq)) |
424 | continue; |
425 | if (read(to_guest[0], buf, sizeof(buf)) < 1) |
426 | break; |
427 | |
428 | receives++; |
429 | virtqueue_disable_cb(vq); |
430 | } |
431 | |
432 | printf("Guest: notified %lu, pinged %lu\n" , |
433 | gvdev.notifies, receives); |
434 | vring_del_virtqueue(vq); |
435 | return 0; |
436 | } |
437 | } |
438 | |
439 | int main(int argc, char *argv[]) |
440 | { |
441 | struct virtio_device vdev; |
442 | struct virtqueue *vq; |
443 | struct vringh vrh; |
444 | struct scatterlist guest_sg[RINGSIZE], *sgs[2]; |
445 | struct iovec host_riov[2], host_wiov[2]; |
446 | struct vringh_iov riov, wiov; |
447 | struct vring_used_elem used[RINGSIZE]; |
448 | char buf[28]; |
449 | u16 head; |
450 | int err; |
451 | unsigned i; |
452 | void *ret; |
453 | bool (*getrange)(struct vringh *vrh, u64 addr, struct vringh_range *r); |
454 | bool fast_vringh = false, parallel = false; |
455 | |
456 | getrange = getrange_iov; |
457 | vdev.features = 0; |
458 | INIT_LIST_HEAD(list: &vdev.vqs); |
459 | spin_lock_init(&vdev.vqs_list_lock); |
460 | |
461 | while (argv[1]) { |
462 | if (strcmp(argv[1], "--indirect" ) == 0) |
463 | __virtio_set_bit(vdev: &vdev, VIRTIO_RING_F_INDIRECT_DESC); |
464 | else if (strcmp(argv[1], "--eventidx" ) == 0) |
465 | __virtio_set_bit(vdev: &vdev, VIRTIO_RING_F_EVENT_IDX); |
466 | else if (strcmp(argv[1], "--virtio-1" ) == 0) |
467 | __virtio_set_bit(vdev: &vdev, VIRTIO_F_VERSION_1); |
468 | else if (strcmp(argv[1], "--slow-range" ) == 0) |
469 | getrange = getrange_slow; |
470 | else if (strcmp(argv[1], "--fast-vringh" ) == 0) |
471 | fast_vringh = true; |
472 | else if (strcmp(argv[1], "--parallel" ) == 0) |
473 | parallel = true; |
474 | else |
475 | errx(1, "Unknown arg %s" , argv[1]); |
476 | argv++; |
477 | } |
478 | |
479 | if (parallel) |
480 | return parallel_test(features: vdev.features, getrange, fast_vringh); |
481 | |
482 | if (posix_memalign(&__user_addr_min, PAGE_SIZE, USER_MEM) != 0) |
483 | abort(); |
484 | __user_addr_max = __user_addr_min + USER_MEM; |
485 | memset(__user_addr_min, 0, vring_size(RINGSIZE, ALIGN)); |
486 | |
487 | /* Set up guest side. */ |
488 | vq = vring_new_virtqueue(index: 0, RINGSIZE, ALIGN, vdev: &vdev, weak_barriers: true, ctx: false, |
489 | pages: __user_addr_min, |
490 | notify: never_notify_host, callback: never_callback_guest, |
491 | name: "guest vq" ); |
492 | |
493 | /* Set up host side. */ |
494 | vring_init(vr: &vrh.vring, RINGSIZE, p: __user_addr_min, ALIGN); |
495 | vringh_init_user(vrh: &vrh, features: vdev.features, RINGSIZE, weak_barriers: true, |
496 | desc: vrh.vring.desc, avail: vrh.vring.avail, used: vrh.vring.used); |
497 | |
498 | /* No descriptor to get yet... */ |
499 | err = vringh_getdesc_user(vrh: &vrh, riov: &riov, wiov: &wiov, getrange, head: &head); |
500 | if (err != 0) |
501 | errx(1, "vringh_getdesc_user: %i" , err); |
502 | |
503 | /* Guest puts in a descriptor. */ |
504 | memcpy(__user_addr_max - 1, "a" , 1); |
505 | sg_init_table(guest_sg, 1); |
506 | sg_set_buf(sg: &guest_sg[0], buf: __user_addr_max - 1, buflen: 1); |
507 | sg_init_table(guest_sg+1, 1); |
508 | sg_set_buf(sg: &guest_sg[1], buf: __user_addr_max - 3, buflen: 2); |
509 | sgs[0] = &guest_sg[0]; |
510 | sgs[1] = &guest_sg[1]; |
511 | |
512 | /* May allocate an indirect, so force it to allocate user addr */ |
513 | __kmalloc_fake = __user_addr_min + vring_size(RINGSIZE, ALIGN); |
514 | err = virtqueue_add_sgs(vq, sgs, out_sgs: 1, in_sgs: 1, data: &err, GFP_KERNEL); |
515 | if (err) |
516 | errx(1, "virtqueue_add_sgs: %i" , err); |
517 | __kmalloc_fake = NULL; |
518 | |
519 | /* Host retreives it. */ |
520 | vringh_iov_init(iov: &riov, iovec: host_riov, ARRAY_SIZE(host_riov)); |
521 | vringh_iov_init(iov: &wiov, iovec: host_wiov, ARRAY_SIZE(host_wiov)); |
522 | |
523 | err = vringh_getdesc_user(vrh: &vrh, riov: &riov, wiov: &wiov, getrange, head: &head); |
524 | if (err != 1) |
525 | errx(1, "vringh_getdesc_user: %i" , err); |
526 | |
527 | assert(riov.used == 1); |
528 | assert(riov.iov[0].iov_base == __user_addr_max - 1); |
529 | assert(riov.iov[0].iov_len == 1); |
530 | if (getrange != getrange_slow) { |
531 | assert(wiov.used == 1); |
532 | assert(wiov.iov[0].iov_base == __user_addr_max - 3); |
533 | assert(wiov.iov[0].iov_len == 2); |
534 | } else { |
535 | assert(wiov.used == 2); |
536 | assert(wiov.iov[0].iov_base == __user_addr_max - 3); |
537 | assert(wiov.iov[0].iov_len == 1); |
538 | assert(wiov.iov[1].iov_base == __user_addr_max - 2); |
539 | assert(wiov.iov[1].iov_len == 1); |
540 | } |
541 | |
542 | err = vringh_iov_pull_user(riov: &riov, dst: buf, len: 5); |
543 | if (err != 1) |
544 | errx(1, "vringh_iov_pull_user: %i" , err); |
545 | assert(buf[0] == 'a'); |
546 | assert(riov.i == 1); |
547 | assert(vringh_iov_pull_user(riov: &riov, dst: buf, len: 5) == 0); |
548 | |
549 | memcpy(buf, "bcdef" , 5); |
550 | err = vringh_iov_push_user(wiov: &wiov, src: buf, len: 5); |
551 | if (err != 2) |
552 | errx(1, "vringh_iov_push_user: %i" , err); |
553 | assert(memcmp(p: __user_addr_max - 3, q: "bc" , size: 2) == 0); |
554 | assert(wiov.i == wiov.used); |
555 | assert(vringh_iov_push_user(wiov: &wiov, src: buf, len: 5) == 0); |
556 | |
557 | /* Host is done. */ |
558 | err = vringh_complete_user(vrh: &vrh, head, len: err); |
559 | if (err != 0) |
560 | errx(1, "vringh_complete_user: %i" , err); |
561 | |
562 | /* Guest should see used token now. */ |
563 | __kfree_ignore_start = __user_addr_min + vring_size(RINGSIZE, ALIGN); |
564 | __kfree_ignore_end = __kfree_ignore_start + 1; |
565 | ret = virtqueue_get_buf(vq, len: &i); |
566 | if (ret != &err) |
567 | errx(1, "virtqueue_get_buf: %p" , ret); |
568 | assert(i == 2); |
569 | |
570 | /* Guest puts in a huge descriptor. */ |
571 | sg_init_table(guest_sg, RINGSIZE); |
572 | for (i = 0; i < RINGSIZE; i++) { |
573 | sg_set_buf(sg: &guest_sg[i], |
574 | buf: __user_addr_max - USER_MEM/4, USER_MEM/4); |
575 | } |
576 | |
577 | /* Fill contents with recognisable garbage. */ |
578 | for (i = 0; i < USER_MEM/4; i++) |
579 | ((char *)__user_addr_max - USER_MEM/4)[i] = i; |
580 | |
581 | /* This will allocate an indirect, so force it to allocate user addr */ |
582 | __kmalloc_fake = __user_addr_min + vring_size(RINGSIZE, ALIGN); |
583 | err = virtqueue_add_outbuf(vq, sg: guest_sg, RINGSIZE, data: &err, GFP_KERNEL); |
584 | if (err) |
585 | errx(1, "virtqueue_add_outbuf (large): %i" , err); |
586 | __kmalloc_fake = NULL; |
587 | |
588 | /* Host picks it up (allocates new iov). */ |
589 | vringh_iov_init(iov: &riov, iovec: host_riov, ARRAY_SIZE(host_riov)); |
590 | vringh_iov_init(iov: &wiov, iovec: host_wiov, ARRAY_SIZE(host_wiov)); |
591 | |
592 | err = vringh_getdesc_user(vrh: &vrh, riov: &riov, wiov: &wiov, getrange, head: &head); |
593 | if (err != 1) |
594 | errx(1, "vringh_getdesc_user: %i" , err); |
595 | |
596 | assert(riov.max_num & VRINGH_IOV_ALLOCATED); |
597 | assert(riov.iov != host_riov); |
598 | if (getrange != getrange_slow) |
599 | assert(riov.used == RINGSIZE); |
600 | else |
601 | assert(riov.used == RINGSIZE * USER_MEM/4); |
602 | |
603 | assert(!(wiov.max_num & VRINGH_IOV_ALLOCATED)); |
604 | assert(wiov.used == 0); |
605 | |
606 | /* Pull data back out (in odd chunks), should be as expected. */ |
607 | for (i = 0; i < RINGSIZE * USER_MEM/4; i += 3) { |
608 | err = vringh_iov_pull_user(riov: &riov, dst: buf, len: 3); |
609 | if (err != 3 && i + err != RINGSIZE * USER_MEM/4) |
610 | errx(1, "vringh_iov_pull_user large: %i" , err); |
611 | assert(buf[0] == (char)i); |
612 | assert(err < 2 || buf[1] == (char)(i + 1)); |
613 | assert(err < 3 || buf[2] == (char)(i + 2)); |
614 | } |
615 | assert(riov.i == riov.used); |
616 | vringh_iov_cleanup(iov: &riov); |
617 | vringh_iov_cleanup(iov: &wiov); |
618 | |
619 | /* Complete using multi interface, just because we can. */ |
620 | used[0].id = head; |
621 | used[0].len = 0; |
622 | err = vringh_complete_multi_user(vrh: &vrh, used, num_used: 1); |
623 | if (err) |
624 | errx(1, "vringh_complete_multi_user(1): %i" , err); |
625 | |
626 | /* Free up those descriptors. */ |
627 | ret = virtqueue_get_buf(vq, len: &i); |
628 | if (ret != &err) |
629 | errx(1, "virtqueue_get_buf: %p" , ret); |
630 | |
631 | /* Add lots of descriptors. */ |
632 | sg_init_table(guest_sg, 1); |
633 | sg_set_buf(sg: &guest_sg[0], buf: __user_addr_max - 1, buflen: 1); |
634 | for (i = 0; i < RINGSIZE; i++) { |
635 | err = virtqueue_add_outbuf(vq, sg: guest_sg, num: 1, data: &err, GFP_KERNEL); |
636 | if (err) |
637 | errx(1, "virtqueue_add_outbuf (multiple): %i" , err); |
638 | } |
639 | |
640 | /* Now get many, and consume them all at once. */ |
641 | vringh_iov_init(iov: &riov, iovec: host_riov, ARRAY_SIZE(host_riov)); |
642 | vringh_iov_init(iov: &wiov, iovec: host_wiov, ARRAY_SIZE(host_wiov)); |
643 | |
644 | for (i = 0; i < RINGSIZE; i++) { |
645 | err = vringh_getdesc_user(vrh: &vrh, riov: &riov, wiov: &wiov, getrange, head: &head); |
646 | if (err != 1) |
647 | errx(1, "vringh_getdesc_user: %i" , err); |
648 | used[i].id = head; |
649 | used[i].len = 0; |
650 | } |
651 | /* Make sure it wraps around ring, to test! */ |
652 | assert(vrh.vring.used->idx % RINGSIZE != 0); |
653 | err = vringh_complete_multi_user(vrh: &vrh, used, RINGSIZE); |
654 | if (err) |
655 | errx(1, "vringh_complete_multi_user: %i" , err); |
656 | |
657 | /* Free those buffers. */ |
658 | for (i = 0; i < RINGSIZE; i++) { |
659 | unsigned len; |
660 | assert(virtqueue_get_buf(vq, len: &len) != NULL); |
661 | } |
662 | |
663 | /* Test weird (but legal!) indirect. */ |
664 | if (__virtio_test_bit(vdev: &vdev, VIRTIO_RING_F_INDIRECT_DESC)) { |
665 | char *data = __user_addr_max - USER_MEM/4; |
666 | struct vring_desc *d = __user_addr_max - USER_MEM/2; |
667 | struct vring vring; |
668 | |
669 | /* Force creation of direct, which we modify. */ |
670 | __virtio_clear_bit(vdev: &vdev, VIRTIO_RING_F_INDIRECT_DESC); |
671 | vq = vring_new_virtqueue(index: 0, RINGSIZE, ALIGN, vdev: &vdev, weak_barriers: true, |
672 | ctx: false, pages: __user_addr_min, |
673 | notify: never_notify_host, |
674 | callback: never_callback_guest, |
675 | name: "guest vq" ); |
676 | |
677 | sg_init_table(guest_sg, 4); |
678 | sg_set_buf(sg: &guest_sg[0], buf: d, buflen: sizeof(*d)*2); |
679 | sg_set_buf(sg: &guest_sg[1], buf: d + 2, buflen: sizeof(*d)*1); |
680 | sg_set_buf(sg: &guest_sg[2], buf: data + 6, buflen: 4); |
681 | sg_set_buf(sg: &guest_sg[3], buf: d + 3, buflen: sizeof(*d)*3); |
682 | |
683 | err = virtqueue_add_outbuf(vq, sg: guest_sg, num: 4, data: &err, GFP_KERNEL); |
684 | if (err) |
685 | errx(1, "virtqueue_add_outbuf (indirect): %i" , err); |
686 | |
687 | vring_init(vr: &vring, RINGSIZE, p: __user_addr_min, ALIGN); |
688 | |
689 | /* They're used in order, but double-check... */ |
690 | assert(vring.desc[0].addr == (unsigned long)d); |
691 | assert(vring.desc[1].addr == (unsigned long)(d+2)); |
692 | assert(vring.desc[2].addr == (unsigned long)data + 6); |
693 | assert(vring.desc[3].addr == (unsigned long)(d+3)); |
694 | vring.desc[0].flags |= VRING_DESC_F_INDIRECT; |
695 | vring.desc[1].flags |= VRING_DESC_F_INDIRECT; |
696 | vring.desc[3].flags |= VRING_DESC_F_INDIRECT; |
697 | |
698 | /* First indirect */ |
699 | d[0].addr = (unsigned long)data; |
700 | d[0].len = 1; |
701 | d[0].flags = VRING_DESC_F_NEXT; |
702 | d[0].next = 1; |
703 | d[1].addr = (unsigned long)data + 1; |
704 | d[1].len = 2; |
705 | d[1].flags = 0; |
706 | |
707 | /* Second indirect */ |
708 | d[2].addr = (unsigned long)data + 3; |
709 | d[2].len = 3; |
710 | d[2].flags = 0; |
711 | |
712 | /* Third indirect */ |
713 | d[3].addr = (unsigned long)data + 10; |
714 | d[3].len = 5; |
715 | d[3].flags = VRING_DESC_F_NEXT; |
716 | d[3].next = 1; |
717 | d[4].addr = (unsigned long)data + 15; |
718 | d[4].len = 6; |
719 | d[4].flags = VRING_DESC_F_NEXT; |
720 | d[4].next = 2; |
721 | d[5].addr = (unsigned long)data + 21; |
722 | d[5].len = 7; |
723 | d[5].flags = 0; |
724 | |
725 | /* Host picks it up (allocates new iov). */ |
726 | vringh_iov_init(iov: &riov, iovec: host_riov, ARRAY_SIZE(host_riov)); |
727 | vringh_iov_init(iov: &wiov, iovec: host_wiov, ARRAY_SIZE(host_wiov)); |
728 | |
729 | err = vringh_getdesc_user(vrh: &vrh, riov: &riov, wiov: &wiov, getrange, head: &head); |
730 | if (err != 1) |
731 | errx(1, "vringh_getdesc_user: %i" , err); |
732 | |
733 | if (head != 0) |
734 | errx(1, "vringh_getdesc_user: head %i not 0" , head); |
735 | |
736 | assert(riov.max_num & VRINGH_IOV_ALLOCATED); |
737 | if (getrange != getrange_slow) |
738 | assert(riov.used == 7); |
739 | else |
740 | assert(riov.used == 28); |
741 | err = vringh_iov_pull_user(riov: &riov, dst: buf, len: 29); |
742 | assert(err == 28); |
743 | |
744 | /* Data should be linear. */ |
745 | for (i = 0; i < err; i++) |
746 | assert(buf[i] == i); |
747 | vringh_iov_cleanup(iov: &riov); |
748 | } |
749 | |
750 | /* Don't leak memory... */ |
751 | vring_del_virtqueue(vq); |
752 | free(__user_addr_min); |
753 | |
754 | return 0; |
755 | } |
756 | |