1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2016 Red Hat, Inc. |
4 | * Author: Michael S. Tsirkin <mst@redhat.com> |
5 | * |
6 | * Partial implementation of virtio 0.9. event index is used for signalling, |
7 | * unconditionally. Design roughly follows linux kernel implementation in order |
8 | * to be able to judge its performance. |
9 | */ |
10 | #define _GNU_SOURCE |
11 | #include "main.h" |
12 | #include <stdlib.h> |
13 | #include <stdio.h> |
14 | #include <assert.h> |
15 | #include <string.h> |
16 | #include <linux/virtio_ring.h> |
17 | |
18 | struct data { |
19 | void *data; |
20 | } *data; |
21 | |
22 | struct vring ring; |
23 | |
24 | /* enabling the below activates experimental ring polling code |
25 | * (which skips index reads on consumer in favor of looking at |
26 | * high bits of ring id ^ 0x8000). |
27 | */ |
28 | /* #ifdef RING_POLL */ |
29 | /* enabling the below activates experimental in-order code |
30 | * (which skips ring updates and reads and writes len in descriptor). |
31 | */ |
32 | /* #ifdef INORDER */ |
33 | |
34 | #if defined(RING_POLL) && defined(INORDER) |
35 | #error "RING_POLL and INORDER are mutually exclusive" |
36 | #endif |
37 | |
38 | /* how much padding is needed to avoid false cache sharing */ |
39 | #define HOST_GUEST_PADDING 0x80 |
40 | |
41 | struct guest { |
42 | unsigned short avail_idx; |
43 | unsigned short last_used_idx; |
44 | unsigned short num_free; |
45 | unsigned short kicked_avail_idx; |
46 | #ifndef INORDER |
47 | unsigned short free_head; |
48 | #else |
49 | unsigned short reserved_free_head; |
50 | #endif |
51 | unsigned char reserved[HOST_GUEST_PADDING - 10]; |
52 | } guest; |
53 | |
54 | struct host { |
55 | /* we do not need to track last avail index |
56 | * unless we have more than one in flight. |
57 | */ |
58 | unsigned short used_idx; |
59 | unsigned short called_used_idx; |
60 | unsigned char reserved[HOST_GUEST_PADDING - 4]; |
61 | } host; |
62 | |
63 | /* implemented by ring */ |
64 | void alloc_ring(void) |
65 | { |
66 | int ret; |
67 | int i; |
68 | void *p; |
69 | |
70 | ret = posix_memalign(&p, 0x1000, vring_size(num: ring_size, align: 0x1000)); |
71 | if (ret) { |
72 | perror("Unable to allocate ring buffer.\n" ); |
73 | exit(3); |
74 | } |
75 | memset(p, 0, vring_size(num: ring_size, align: 0x1000)); |
76 | vring_init(vr: &ring, num: ring_size, p, align: 0x1000); |
77 | |
78 | guest.avail_idx = 0; |
79 | guest.kicked_avail_idx = -1; |
80 | guest.last_used_idx = 0; |
81 | #ifndef INORDER |
82 | /* Put everything in free lists. */ |
83 | guest.free_head = 0; |
84 | #endif |
85 | for (i = 0; i < ring_size - 1; i++) |
86 | ring.desc[i].next = i + 1; |
87 | host.used_idx = 0; |
88 | host.called_used_idx = -1; |
89 | guest.num_free = ring_size; |
90 | data = malloc(ring_size * sizeof *data); |
91 | if (!data) { |
92 | perror("Unable to allocate data buffer.\n" ); |
93 | exit(3); |
94 | } |
95 | memset(data, 0, ring_size * sizeof *data); |
96 | } |
97 | |
98 | /* guest side */ |
99 | int add_inbuf(unsigned len, void *buf, void *datap) |
100 | { |
101 | unsigned head; |
102 | #ifndef INORDER |
103 | unsigned avail; |
104 | #endif |
105 | struct vring_desc *desc; |
106 | |
107 | if (!guest.num_free) |
108 | return -1; |
109 | |
110 | #ifdef INORDER |
111 | head = (ring_size - 1) & (guest.avail_idx++); |
112 | #else |
113 | head = guest.free_head; |
114 | #endif |
115 | guest.num_free--; |
116 | |
117 | desc = ring.desc; |
118 | desc[head].flags = VRING_DESC_F_NEXT; |
119 | desc[head].addr = (unsigned long)(void *)buf; |
120 | desc[head].len = len; |
121 | /* We do it like this to simulate the way |
122 | * we'd have to flip it if we had multiple |
123 | * descriptors. |
124 | */ |
125 | desc[head].flags &= ~VRING_DESC_F_NEXT; |
126 | #ifndef INORDER |
127 | guest.free_head = desc[head].next; |
128 | #endif |
129 | |
130 | data[head].data = datap; |
131 | |
132 | #ifdef RING_POLL |
133 | /* Barrier A (for pairing) */ |
134 | smp_release(); |
135 | avail = guest.avail_idx++; |
136 | ring.avail->ring[avail & (ring_size - 1)] = |
137 | (head | (avail & ~(ring_size - 1))) ^ 0x8000; |
138 | #else |
139 | #ifndef INORDER |
140 | /* Barrier A (for pairing) */ |
141 | smp_release(); |
142 | avail = (ring_size - 1) & (guest.avail_idx++); |
143 | ring.avail->ring[avail] = head; |
144 | #endif |
145 | /* Barrier A (for pairing) */ |
146 | smp_release(); |
147 | #endif |
148 | ring.avail->idx = guest.avail_idx; |
149 | return 0; |
150 | } |
151 | |
152 | void *get_buf(unsigned *lenp, void **bufp) |
153 | { |
154 | unsigned head; |
155 | unsigned index; |
156 | void *datap; |
157 | |
158 | #ifdef RING_POLL |
159 | head = (ring_size - 1) & guest.last_used_idx; |
160 | index = ring.used->ring[head].id; |
161 | if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1)) |
162 | return NULL; |
163 | /* Barrier B (for pairing) */ |
164 | smp_acquire(); |
165 | index &= ring_size - 1; |
166 | #else |
167 | if (ring.used->idx == guest.last_used_idx) |
168 | return NULL; |
169 | /* Barrier B (for pairing) */ |
170 | smp_acquire(); |
171 | #ifdef INORDER |
172 | head = (ring_size - 1) & guest.last_used_idx; |
173 | index = head; |
174 | #else |
175 | head = (ring_size - 1) & guest.last_used_idx; |
176 | index = ring.used->ring[head].id; |
177 | #endif |
178 | |
179 | #endif |
180 | #ifdef INORDER |
181 | *lenp = ring.desc[index].len; |
182 | #else |
183 | *lenp = ring.used->ring[head].len; |
184 | #endif |
185 | datap = data[index].data; |
186 | *bufp = (void*)(unsigned long)ring.desc[index].addr; |
187 | data[index].data = NULL; |
188 | #ifndef INORDER |
189 | ring.desc[index].next = guest.free_head; |
190 | guest.free_head = index; |
191 | #endif |
192 | guest.num_free++; |
193 | guest.last_used_idx++; |
194 | return datap; |
195 | } |
196 | |
197 | bool used_empty() |
198 | { |
199 | unsigned short last_used_idx = guest.last_used_idx; |
200 | #ifdef RING_POLL |
201 | unsigned short head = last_used_idx & (ring_size - 1); |
202 | unsigned index = ring.used->ring[head].id; |
203 | |
204 | return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1); |
205 | #else |
206 | return ring.used->idx == last_used_idx; |
207 | #endif |
208 | } |
209 | |
210 | void disable_call() |
211 | { |
212 | /* Doing nothing to disable calls might cause |
213 | * extra interrupts, but reduces the number of cache misses. |
214 | */ |
215 | } |
216 | |
217 | bool enable_call() |
218 | { |
219 | vring_used_event(&ring) = guest.last_used_idx; |
220 | /* Flush call index write */ |
221 | /* Barrier D (for pairing) */ |
222 | smp_mb(); |
223 | return used_empty(); |
224 | } |
225 | |
226 | void kick_available(void) |
227 | { |
228 | bool need; |
229 | |
230 | /* Flush in previous flags write */ |
231 | /* Barrier C (for pairing) */ |
232 | smp_mb(); |
233 | need = vring_need_event(vring_avail_event(&ring), |
234 | new_idx: guest.avail_idx, |
235 | old: guest.kicked_avail_idx); |
236 | |
237 | guest.kicked_avail_idx = guest.avail_idx; |
238 | if (need) |
239 | kick(); |
240 | } |
241 | |
242 | /* host side */ |
243 | void disable_kick() |
244 | { |
245 | /* Doing nothing to disable kicks might cause |
246 | * extra interrupts, but reduces the number of cache misses. |
247 | */ |
248 | } |
249 | |
250 | bool enable_kick() |
251 | { |
252 | vring_avail_event(&ring) = host.used_idx; |
253 | /* Barrier C (for pairing) */ |
254 | smp_mb(); |
255 | return avail_empty(); |
256 | } |
257 | |
258 | bool avail_empty() |
259 | { |
260 | unsigned head = host.used_idx; |
261 | #ifdef RING_POLL |
262 | unsigned index = ring.avail->ring[head & (ring_size - 1)]; |
263 | |
264 | return ((index ^ head ^ 0x8000) & ~(ring_size - 1)); |
265 | #else |
266 | return head == ring.avail->idx; |
267 | #endif |
268 | } |
269 | |
270 | bool use_buf(unsigned *lenp, void **bufp) |
271 | { |
272 | unsigned used_idx = host.used_idx; |
273 | struct vring_desc *desc; |
274 | unsigned head; |
275 | |
276 | #ifdef RING_POLL |
277 | head = ring.avail->ring[used_idx & (ring_size - 1)]; |
278 | if ((used_idx ^ head ^ 0x8000) & ~(ring_size - 1)) |
279 | return false; |
280 | /* Barrier A (for pairing) */ |
281 | smp_acquire(); |
282 | |
283 | used_idx &= ring_size - 1; |
284 | desc = &ring.desc[head & (ring_size - 1)]; |
285 | #else |
286 | if (used_idx == ring.avail->idx) |
287 | return false; |
288 | |
289 | /* Barrier A (for pairing) */ |
290 | smp_acquire(); |
291 | |
292 | used_idx &= ring_size - 1; |
293 | #ifdef INORDER |
294 | head = used_idx; |
295 | #else |
296 | head = ring.avail->ring[used_idx]; |
297 | #endif |
298 | desc = &ring.desc[head]; |
299 | #endif |
300 | |
301 | *lenp = desc->len; |
302 | *bufp = (void *)(unsigned long)desc->addr; |
303 | |
304 | #ifdef INORDER |
305 | desc->len = desc->len - 1; |
306 | #else |
307 | /* now update used ring */ |
308 | ring.used->ring[used_idx].id = head; |
309 | ring.used->ring[used_idx].len = desc->len - 1; |
310 | #endif |
311 | /* Barrier B (for pairing) */ |
312 | smp_release(); |
313 | host.used_idx++; |
314 | ring.used->idx = host.used_idx; |
315 | |
316 | return true; |
317 | } |
318 | |
319 | void call_used(void) |
320 | { |
321 | bool need; |
322 | |
323 | /* Flush in previous flags write */ |
324 | /* Barrier D (for pairing) */ |
325 | smp_mb(); |
326 | need = vring_need_event(vring_used_event(&ring), |
327 | new_idx: host.used_idx, |
328 | old: host.called_used_idx); |
329 | |
330 | host.called_used_idx = host.used_idx; |
331 | if (need) |
332 | call(); |
333 | } |
334 | |