1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* MSG_ZEROCOPY feature tests for vsock |
3 | * |
4 | * Copyright (C) 2023 SberDevices. |
5 | * |
6 | * Author: Arseniy Krasnov <avkrasnov@salutedevices.com> |
7 | */ |
8 | |
9 | #include <stdio.h> |
10 | #include <stdlib.h> |
11 | #include <string.h> |
12 | #include <sys/mman.h> |
13 | #include <unistd.h> |
14 | #include <poll.h> |
15 | #include <linux/errqueue.h> |
16 | #include <linux/kernel.h> |
17 | #include <errno.h> |
18 | |
19 | #include "control.h" |
20 | #include "vsock_test_zerocopy.h" |
21 | #include "msg_zerocopy_common.h" |
22 | |
23 | #ifndef PAGE_SIZE |
24 | #define PAGE_SIZE 4096 |
25 | #endif |
26 | |
27 | #define VSOCK_TEST_DATA_MAX_IOV 3 |
28 | |
29 | struct vsock_test_data { |
30 | /* This test case if for SOCK_STREAM only. */ |
31 | bool stream_only; |
32 | /* Data must be zerocopied. This field is checked against |
33 | * field 'ee_code' of the 'struct sock_extended_err', which |
34 | * contains bit to detect that zerocopy transmission was |
35 | * fallbacked to copy mode. |
36 | */ |
37 | bool zerocopied; |
38 | /* Enable SO_ZEROCOPY option on the socket. Without enabled |
39 | * SO_ZEROCOPY, every MSG_ZEROCOPY transmission will behave |
40 | * like without MSG_ZEROCOPY flag. |
41 | */ |
42 | bool so_zerocopy; |
43 | /* 'errno' after 'sendmsg()' call. */ |
44 | int sendmsg_errno; |
45 | /* Number of valid elements in 'vecs'. */ |
46 | int vecs_cnt; |
47 | struct iovec vecs[VSOCK_TEST_DATA_MAX_IOV]; |
48 | }; |
49 | |
50 | static struct vsock_test_data test_data_array[] = { |
51 | /* Last element has non-page aligned size. */ |
52 | { |
53 | .zerocopied = true, |
54 | .so_zerocopy = true, |
55 | .sendmsg_errno = 0, |
56 | .vecs_cnt = 3, |
57 | { |
58 | { NULL, PAGE_SIZE }, |
59 | { NULL, PAGE_SIZE }, |
60 | { NULL, 200 } |
61 | } |
62 | }, |
63 | /* All elements have page aligned base and size. */ |
64 | { |
65 | .zerocopied = true, |
66 | .so_zerocopy = true, |
67 | .sendmsg_errno = 0, |
68 | .vecs_cnt = 3, |
69 | { |
70 | { NULL, PAGE_SIZE }, |
71 | { NULL, PAGE_SIZE * 2 }, |
72 | { NULL, PAGE_SIZE * 3 } |
73 | } |
74 | }, |
75 | /* All elements have page aligned base and size. But |
76 | * data length is bigger than 64Kb. |
77 | */ |
78 | { |
79 | .zerocopied = true, |
80 | .so_zerocopy = true, |
81 | .sendmsg_errno = 0, |
82 | .vecs_cnt = 3, |
83 | { |
84 | { NULL, PAGE_SIZE * 16 }, |
85 | { NULL, PAGE_SIZE * 16 }, |
86 | { NULL, PAGE_SIZE * 16 } |
87 | } |
88 | }, |
89 | /* Middle element has both non-page aligned base and size. */ |
90 | { |
91 | .zerocopied = true, |
92 | .so_zerocopy = true, |
93 | .sendmsg_errno = 0, |
94 | .vecs_cnt = 3, |
95 | { |
96 | { NULL, PAGE_SIZE }, |
97 | { (void *)1, 100 }, |
98 | { NULL, PAGE_SIZE } |
99 | } |
100 | }, |
101 | /* Middle element is unmapped. */ |
102 | { |
103 | .zerocopied = false, |
104 | .so_zerocopy = true, |
105 | .sendmsg_errno = ENOMEM, |
106 | .vecs_cnt = 3, |
107 | { |
108 | { NULL, PAGE_SIZE }, |
109 | { MAP_FAILED, PAGE_SIZE }, |
110 | { NULL, PAGE_SIZE } |
111 | } |
112 | }, |
113 | /* Valid data, but SO_ZEROCOPY is off. This |
114 | * will trigger fallback to copy. |
115 | */ |
116 | { |
117 | .zerocopied = false, |
118 | .so_zerocopy = false, |
119 | .sendmsg_errno = 0, |
120 | .vecs_cnt = 1, |
121 | { |
122 | { NULL, PAGE_SIZE } |
123 | } |
124 | }, |
125 | /* Valid data, but message is bigger than peer's |
126 | * buffer, so this will trigger fallback to copy. |
127 | * This test is for SOCK_STREAM only, because |
128 | * for SOCK_SEQPACKET, 'sendmsg()' returns EMSGSIZE. |
129 | */ |
130 | { |
131 | .stream_only = true, |
132 | .zerocopied = false, |
133 | .so_zerocopy = true, |
134 | .sendmsg_errno = 0, |
135 | .vecs_cnt = 1, |
136 | { |
137 | { NULL, 100 * PAGE_SIZE } |
138 | } |
139 | }, |
140 | }; |
141 | |
142 | #define POLL_TIMEOUT_MS 100 |
143 | |
144 | static void test_client(const struct test_opts *opts, |
145 | const struct vsock_test_data *test_data, |
146 | bool sock_seqpacket) |
147 | { |
148 | struct pollfd fds = { 0 }; |
149 | struct msghdr msg = { 0 }; |
150 | ssize_t sendmsg_res; |
151 | struct iovec *iovec; |
152 | int fd; |
153 | |
154 | if (sock_seqpacket) |
155 | fd = vsock_seqpacket_connect(cid: opts->peer_cid, port: opts->peer_port); |
156 | else |
157 | fd = vsock_stream_connect(cid: opts->peer_cid, port: opts->peer_port); |
158 | |
159 | if (fd < 0) { |
160 | perror("connect" ); |
161 | exit(EXIT_FAILURE); |
162 | } |
163 | |
164 | if (test_data->so_zerocopy) |
165 | enable_so_zerocopy(fd); |
166 | |
167 | iovec = alloc_test_iovec(test_iovec: test_data->vecs, iovnum: test_data->vecs_cnt); |
168 | |
169 | msg.msg_iov = iovec; |
170 | msg.msg_iovlen = test_data->vecs_cnt; |
171 | |
172 | errno = 0; |
173 | |
174 | sendmsg_res = sendmsg(fd, &msg, MSG_ZEROCOPY); |
175 | if (errno != test_data->sendmsg_errno) { |
176 | fprintf(stderr, "expected 'errno' == %i, got %i\n" , |
177 | test_data->sendmsg_errno, errno); |
178 | exit(EXIT_FAILURE); |
179 | } |
180 | |
181 | if (!errno) { |
182 | if (sendmsg_res != iovec_bytes(iov: iovec, iovnum: test_data->vecs_cnt)) { |
183 | fprintf(stderr, "expected 'sendmsg()' == %li, got %li\n" , |
184 | iovec_bytes(iov: iovec, iovnum: test_data->vecs_cnt), |
185 | sendmsg_res); |
186 | exit(EXIT_FAILURE); |
187 | } |
188 | } |
189 | |
190 | fds.fd = fd; |
191 | fds.events = 0; |
192 | |
193 | if (poll(&fds, 1, POLL_TIMEOUT_MS) < 0) { |
194 | perror("poll" ); |
195 | exit(EXIT_FAILURE); |
196 | } |
197 | |
198 | if (fds.revents & POLLERR) { |
199 | vsock_recv_completion(fd, zerocopied: &test_data->zerocopied); |
200 | } else if (test_data->so_zerocopy && !test_data->sendmsg_errno) { |
201 | /* If we don't have data in the error queue, but |
202 | * SO_ZEROCOPY was enabled and 'sendmsg()' was |
203 | * successful - this is an error. |
204 | */ |
205 | fprintf(stderr, "POLLERR expected\n" ); |
206 | exit(EXIT_FAILURE); |
207 | } |
208 | |
209 | if (!test_data->sendmsg_errno) |
210 | control_writeulong(value: iovec_hash_djb2(iov: iovec, iovnum: test_data->vecs_cnt)); |
211 | else |
212 | control_writeulong(value: 0); |
213 | |
214 | control_writeln(str: "DONE" ); |
215 | free_test_iovec(test_iovec: test_data->vecs, iovec, iovnum: test_data->vecs_cnt); |
216 | close(fd); |
217 | } |
218 | |
219 | void test_stream_msgzcopy_client(const struct test_opts *opts) |
220 | { |
221 | int i; |
222 | |
223 | for (i = 0; i < ARRAY_SIZE(test_data_array); i++) |
224 | test_client(opts, &test_data_array[i], false); |
225 | } |
226 | |
227 | void test_seqpacket_msgzcopy_client(const struct test_opts *opts) |
228 | { |
229 | int i; |
230 | |
231 | for (i = 0; i < ARRAY_SIZE(test_data_array); i++) { |
232 | if (test_data_array[i].stream_only) |
233 | continue; |
234 | |
235 | test_client(opts, &test_data_array[i], true); |
236 | } |
237 | } |
238 | |
239 | static void test_server(const struct test_opts *opts, |
240 | const struct vsock_test_data *test_data, |
241 | bool sock_seqpacket) |
242 | { |
243 | unsigned long remote_hash; |
244 | unsigned long local_hash; |
245 | ssize_t total_bytes_rec; |
246 | unsigned char *data; |
247 | size_t data_len; |
248 | int fd; |
249 | |
250 | if (sock_seqpacket) |
251 | fd = vsock_seqpacket_accept(VMADDR_CID_ANY, port: opts->peer_port, NULL); |
252 | else |
253 | fd = vsock_stream_accept(VMADDR_CID_ANY, port: opts->peer_port, NULL); |
254 | |
255 | if (fd < 0) { |
256 | perror("accept" ); |
257 | exit(EXIT_FAILURE); |
258 | } |
259 | |
260 | data_len = iovec_bytes(iov: test_data->vecs, iovnum: test_data->vecs_cnt); |
261 | |
262 | data = malloc(data_len); |
263 | if (!data) { |
264 | perror("malloc" ); |
265 | exit(EXIT_FAILURE); |
266 | } |
267 | |
268 | total_bytes_rec = 0; |
269 | |
270 | while (total_bytes_rec != data_len) { |
271 | ssize_t bytes_rec; |
272 | |
273 | bytes_rec = read(fd, data + total_bytes_rec, |
274 | data_len - total_bytes_rec); |
275 | if (bytes_rec <= 0) |
276 | break; |
277 | |
278 | total_bytes_rec += bytes_rec; |
279 | } |
280 | |
281 | if (test_data->sendmsg_errno == 0) |
282 | local_hash = hash_djb2(data, len: data_len); |
283 | else |
284 | local_hash = 0; |
285 | |
286 | free(data); |
287 | |
288 | /* Waiting for some result. */ |
289 | remote_hash = control_readulong(); |
290 | if (remote_hash != local_hash) { |
291 | fprintf(stderr, "hash mismatch\n" ); |
292 | exit(EXIT_FAILURE); |
293 | } |
294 | |
295 | control_expectln(str: "DONE" ); |
296 | close(fd); |
297 | } |
298 | |
299 | void test_stream_msgzcopy_server(const struct test_opts *opts) |
300 | { |
301 | int i; |
302 | |
303 | for (i = 0; i < ARRAY_SIZE(test_data_array); i++) |
304 | test_server(opts, &test_data_array[i], false); |
305 | } |
306 | |
307 | void test_seqpacket_msgzcopy_server(const struct test_opts *opts) |
308 | { |
309 | int i; |
310 | |
311 | for (i = 0; i < ARRAY_SIZE(test_data_array); i++) { |
312 | if (test_data_array[i].stream_only) |
313 | continue; |
314 | |
315 | test_server(opts, &test_data_array[i], true); |
316 | } |
317 | } |
318 | |
319 | void test_stream_msgzcopy_empty_errq_client(const struct test_opts *opts) |
320 | { |
321 | struct msghdr msg = { 0 }; |
322 | char cmsg_data[128]; |
323 | ssize_t res; |
324 | int fd; |
325 | |
326 | fd = vsock_stream_connect(cid: opts->peer_cid, port: opts->peer_port); |
327 | if (fd < 0) { |
328 | perror("connect" ); |
329 | exit(EXIT_FAILURE); |
330 | } |
331 | |
332 | msg.msg_control = cmsg_data; |
333 | msg.msg_controllen = sizeof(cmsg_data); |
334 | |
335 | res = recvmsg(fd, &msg, MSG_ERRQUEUE); |
336 | if (res != -1) { |
337 | fprintf(stderr, "expected 'recvmsg(2)' failure, got %zi\n" , |
338 | res); |
339 | exit(EXIT_FAILURE); |
340 | } |
341 | |
342 | control_writeln(str: "DONE" ); |
343 | close(fd); |
344 | } |
345 | |
346 | void test_stream_msgzcopy_empty_errq_server(const struct test_opts *opts) |
347 | { |
348 | int fd; |
349 | |
350 | fd = vsock_stream_accept(VMADDR_CID_ANY, port: opts->peer_port, NULL); |
351 | if (fd < 0) { |
352 | perror("accept" ); |
353 | exit(EXIT_FAILURE); |
354 | } |
355 | |
356 | control_expectln(str: "DONE" ); |
357 | close(fd); |
358 | } |
359 | |