1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * mem-memcpy.c |
4 | * |
5 | * Simple memcpy() and memset() benchmarks |
6 | * |
7 | * Written by Hitoshi Mitake <mitake@dcl.info.waseda.ac.jp> |
8 | */ |
9 | |
10 | #include "debug.h" |
11 | #include "../perf-sys.h" |
12 | #include <subcmd/parse-options.h> |
13 | #include "../util/header.h" |
14 | #include "../util/cloexec.h" |
15 | #include "../util/string2.h" |
16 | #include "bench.h" |
17 | #include "mem-memcpy-arch.h" |
18 | #include "mem-memset-arch.h" |
19 | |
20 | #include <stdio.h> |
21 | #include <stdlib.h> |
22 | #include <string.h> |
23 | #include <unistd.h> |
24 | #include <sys/time.h> |
25 | #include <errno.h> |
26 | #include <linux/time64.h> |
27 | #include <linux/zalloc.h> |
28 | |
29 | #define K 1024 |
30 | |
31 | static const char *size_str = "1MB" ; |
32 | static const char *function_str = "all" ; |
33 | static int nr_loops = 1; |
34 | static bool use_cycles; |
35 | static int cycles_fd; |
36 | |
37 | static const struct option options[] = { |
38 | OPT_STRING('s', "size" , &size_str, "1MB" , |
39 | "Specify the size of the memory buffers. " |
40 | "Available units: B, KB, MB, GB and TB (case insensitive)" ), |
41 | |
42 | OPT_STRING('f', "function" , &function_str, "all" , |
43 | "Specify the function to run, \"all\" runs all available functions, \"help\" lists them" ), |
44 | |
45 | OPT_INTEGER('l', "nr_loops" , &nr_loops, |
46 | "Specify the number of loops to run. (default: 1)" ), |
47 | |
48 | OPT_BOOLEAN('c', "cycles" , &use_cycles, |
49 | "Use a cycles event instead of gettimeofday() to measure performance" ), |
50 | |
51 | OPT_END() |
52 | }; |
53 | |
54 | typedef void *(*memcpy_t)(void *, const void *, size_t); |
55 | typedef void *(*memset_t)(void *, int, size_t); |
56 | |
57 | struct function { |
58 | const char *name; |
59 | const char *desc; |
60 | union { |
61 | memcpy_t memcpy; |
62 | memset_t memset; |
63 | } fn; |
64 | }; |
65 | |
66 | static struct perf_event_attr cycle_attr = { |
67 | .type = PERF_TYPE_HARDWARE, |
68 | .config = PERF_COUNT_HW_CPU_CYCLES |
69 | }; |
70 | |
71 | static int init_cycles(void) |
72 | { |
73 | cycles_fd = sys_perf_event_open(attr: &cycle_attr, pid: getpid(), cpu: -1, group_fd: -1, flags: perf_event_open_cloexec_flag()); |
74 | |
75 | if (cycles_fd < 0 && errno == ENOSYS) { |
76 | pr_debug("No CONFIG_PERF_EVENTS=y kernel support configured?\n" ); |
77 | return -1; |
78 | } |
79 | |
80 | return cycles_fd; |
81 | } |
82 | |
83 | static u64 get_cycles(void) |
84 | { |
85 | int ret; |
86 | u64 clk; |
87 | |
88 | ret = read(cycles_fd, &clk, sizeof(u64)); |
89 | BUG_ON(ret != sizeof(u64)); |
90 | |
91 | return clk; |
92 | } |
93 | |
94 | static double timeval2double(struct timeval *ts) |
95 | { |
96 | return (double)ts->tv_sec + (double)ts->tv_usec / (double)USEC_PER_SEC; |
97 | } |
98 | |
99 | #define print_bps(x) do { \ |
100 | if (x < K) \ |
101 | printf(" %14lf bytes/sec\n", x); \ |
102 | else if (x < K * K) \ |
103 | printf(" %14lfd KB/sec\n", x / K); \ |
104 | else if (x < K * K * K) \ |
105 | printf(" %14lf MB/sec\n", x / K / K); \ |
106 | else \ |
107 | printf(" %14lf GB/sec\n", x / K / K / K); \ |
108 | } while (0) |
109 | |
110 | struct bench_mem_info { |
111 | const struct function *functions; |
112 | u64 (*do_cycles)(const struct function *r, size_t size, void *src, void *dst); |
113 | double (*do_gettimeofday)(const struct function *r, size_t size, void *src, void *dst); |
114 | const char *const *usage; |
115 | bool alloc_src; |
116 | }; |
117 | |
118 | static void __bench_mem_function(struct bench_mem_info *info, int r_idx, size_t size, double size_total) |
119 | { |
120 | const struct function *r = &info->functions[r_idx]; |
121 | double result_bps = 0.0; |
122 | u64 result_cycles = 0; |
123 | void *src = NULL, *dst = zalloc(size); |
124 | |
125 | printf("# function '%s' (%s)\n" , r->name, r->desc); |
126 | |
127 | if (dst == NULL) |
128 | goto out_alloc_failed; |
129 | |
130 | if (info->alloc_src) { |
131 | src = zalloc(size); |
132 | if (src == NULL) |
133 | goto out_alloc_failed; |
134 | } |
135 | |
136 | if (bench_format == BENCH_FORMAT_DEFAULT) |
137 | printf("# Copying %s bytes ...\n\n" , size_str); |
138 | |
139 | if (use_cycles) { |
140 | result_cycles = info->do_cycles(r, size, src, dst); |
141 | } else { |
142 | result_bps = info->do_gettimeofday(r, size, src, dst); |
143 | } |
144 | |
145 | switch (bench_format) { |
146 | case BENCH_FORMAT_DEFAULT: |
147 | if (use_cycles) { |
148 | printf(" %14lf cycles/byte\n" , (double)result_cycles/size_total); |
149 | } else { |
150 | print_bps(result_bps); |
151 | } |
152 | break; |
153 | |
154 | case BENCH_FORMAT_SIMPLE: |
155 | if (use_cycles) { |
156 | printf("%lf\n" , (double)result_cycles/size_total); |
157 | } else { |
158 | printf("%lf\n" , result_bps); |
159 | } |
160 | break; |
161 | |
162 | default: |
163 | BUG_ON(1); |
164 | break; |
165 | } |
166 | |
167 | out_free: |
168 | free(src); |
169 | free(dst); |
170 | return; |
171 | out_alloc_failed: |
172 | printf("# Memory allocation failed - maybe size (%s) is too large?\n" , size_str); |
173 | goto out_free; |
174 | } |
175 | |
176 | static int bench_mem_common(int argc, const char **argv, struct bench_mem_info *info) |
177 | { |
178 | int i; |
179 | size_t size; |
180 | double size_total; |
181 | |
182 | argc = parse_options(argc, argv, options, info->usage, 0); |
183 | |
184 | if (use_cycles) { |
185 | i = init_cycles(); |
186 | if (i < 0) { |
187 | fprintf(stderr, "Failed to open cycles counter\n" ); |
188 | return i; |
189 | } |
190 | } |
191 | |
192 | size = (size_t)perf_atoll(str: (char *)size_str); |
193 | size_total = (double)size * nr_loops; |
194 | |
195 | if ((s64)size <= 0) { |
196 | fprintf(stderr, "Invalid size:%s\n" , size_str); |
197 | return 1; |
198 | } |
199 | |
200 | if (!strncmp(function_str, "all" , 3)) { |
201 | for (i = 0; info->functions[i].name; i++) |
202 | __bench_mem_function(info, r_idx: i, size, size_total); |
203 | return 0; |
204 | } |
205 | |
206 | for (i = 0; info->functions[i].name; i++) { |
207 | if (!strcmp(info->functions[i].name, function_str)) |
208 | break; |
209 | } |
210 | if (!info->functions[i].name) { |
211 | if (strcmp(function_str, "help" ) && strcmp(function_str, "h" )) |
212 | printf("Unknown function: %s\n" , function_str); |
213 | printf("Available functions:\n" ); |
214 | for (i = 0; info->functions[i].name; i++) { |
215 | printf("\t%s ... %s\n" , |
216 | info->functions[i].name, info->functions[i].desc); |
217 | } |
218 | return 1; |
219 | } |
220 | |
221 | __bench_mem_function(info, r_idx: i, size, size_total); |
222 | |
223 | return 0; |
224 | } |
225 | |
226 | static void memcpy_prefault(memcpy_t fn, size_t size, void *src, void *dst) |
227 | { |
228 | /* Make sure to always prefault zero pages even if MMAP_THRESH is crossed: */ |
229 | memset(src, 0, size); |
230 | |
231 | /* |
232 | * We prefault the freshly allocated memory range here, |
233 | * to not measure page fault overhead: |
234 | */ |
235 | fn(dst, src, size); |
236 | } |
237 | |
238 | static u64 do_memcpy_cycles(const struct function *r, size_t size, void *src, void *dst) |
239 | { |
240 | u64 cycle_start = 0ULL, cycle_end = 0ULL; |
241 | memcpy_t fn = r->fn.memcpy; |
242 | int i; |
243 | |
244 | memcpy_prefault(fn, size, src, dst); |
245 | |
246 | cycle_start = get_cycles(); |
247 | for (i = 0; i < nr_loops; ++i) |
248 | fn(dst, src, size); |
249 | cycle_end = get_cycles(); |
250 | |
251 | return cycle_end - cycle_start; |
252 | } |
253 | |
254 | static double do_memcpy_gettimeofday(const struct function *r, size_t size, void *src, void *dst) |
255 | { |
256 | struct timeval tv_start, tv_end, tv_diff; |
257 | memcpy_t fn = r->fn.memcpy; |
258 | int i; |
259 | |
260 | memcpy_prefault(fn, size, src, dst); |
261 | |
262 | BUG_ON(gettimeofday(&tv_start, NULL)); |
263 | for (i = 0; i < nr_loops; ++i) |
264 | fn(dst, src, size); |
265 | BUG_ON(gettimeofday(&tv_end, NULL)); |
266 | |
267 | timersub(&tv_end, &tv_start, &tv_diff); |
268 | |
269 | return (double)(((double)size * nr_loops) / timeval2double(ts: &tv_diff)); |
270 | } |
271 | |
272 | struct function memcpy_functions[] = { |
273 | { .name = "default" , |
274 | .desc = "Default memcpy() provided by glibc" , |
275 | .fn.memcpy = memcpy }, |
276 | |
277 | #ifdef HAVE_ARCH_X86_64_SUPPORT |
278 | # define MEMCPY_FN(_fn, _name, _desc) {.name = _name, .desc = _desc, .fn.memcpy = _fn}, |
279 | # include "mem-memcpy-x86-64-asm-def.h" |
280 | # undef MEMCPY_FN |
281 | #endif |
282 | |
283 | { .name = NULL, } |
284 | }; |
285 | |
286 | static const char * const bench_mem_memcpy_usage[] = { |
287 | "perf bench mem memcpy <options>" , |
288 | NULL |
289 | }; |
290 | |
291 | int bench_mem_memcpy(int argc, const char **argv) |
292 | { |
293 | struct bench_mem_info info = { |
294 | .functions = memcpy_functions, |
295 | .do_cycles = do_memcpy_cycles, |
296 | .do_gettimeofday = do_memcpy_gettimeofday, |
297 | .usage = bench_mem_memcpy_usage, |
298 | .alloc_src = true, |
299 | }; |
300 | |
301 | return bench_mem_common(argc, argv, info: &info); |
302 | } |
303 | |
304 | static u64 do_memset_cycles(const struct function *r, size_t size, void *src __maybe_unused, void *dst) |
305 | { |
306 | u64 cycle_start = 0ULL, cycle_end = 0ULL; |
307 | memset_t fn = r->fn.memset; |
308 | int i; |
309 | |
310 | /* |
311 | * We prefault the freshly allocated memory range here, |
312 | * to not measure page fault overhead: |
313 | */ |
314 | fn(dst, -1, size); |
315 | |
316 | cycle_start = get_cycles(); |
317 | for (i = 0; i < nr_loops; ++i) |
318 | fn(dst, i, size); |
319 | cycle_end = get_cycles(); |
320 | |
321 | return cycle_end - cycle_start; |
322 | } |
323 | |
324 | static double do_memset_gettimeofday(const struct function *r, size_t size, void *src __maybe_unused, void *dst) |
325 | { |
326 | struct timeval tv_start, tv_end, tv_diff; |
327 | memset_t fn = r->fn.memset; |
328 | int i; |
329 | |
330 | /* |
331 | * We prefault the freshly allocated memory range here, |
332 | * to not measure page fault overhead: |
333 | */ |
334 | fn(dst, -1, size); |
335 | |
336 | BUG_ON(gettimeofday(&tv_start, NULL)); |
337 | for (i = 0; i < nr_loops; ++i) |
338 | fn(dst, i, size); |
339 | BUG_ON(gettimeofday(&tv_end, NULL)); |
340 | |
341 | timersub(&tv_end, &tv_start, &tv_diff); |
342 | |
343 | return (double)(((double)size * nr_loops) / timeval2double(ts: &tv_diff)); |
344 | } |
345 | |
346 | static const char * const bench_mem_memset_usage[] = { |
347 | "perf bench mem memset <options>" , |
348 | NULL |
349 | }; |
350 | |
351 | static const struct function memset_functions[] = { |
352 | { .name = "default" , |
353 | .desc = "Default memset() provided by glibc" , |
354 | .fn.memset = memset }, |
355 | |
356 | #ifdef HAVE_ARCH_X86_64_SUPPORT |
357 | # define MEMSET_FN(_fn, _name, _desc) { .name = _name, .desc = _desc, .fn.memset = _fn }, |
358 | # include "mem-memset-x86-64-asm-def.h" |
359 | # undef MEMSET_FN |
360 | #endif |
361 | |
362 | { .name = NULL, } |
363 | }; |
364 | |
365 | int bench_mem_memset(int argc, const char **argv) |
366 | { |
367 | struct bench_mem_info info = { |
368 | .functions = memset_functions, |
369 | .do_cycles = do_memset_cycles, |
370 | .do_gettimeofday = do_memset_gettimeofday, |
371 | .usage = bench_mem_memset_usage, |
372 | }; |
373 | |
374 | return bench_mem_common(argc, argv, info: &info); |
375 | } |
376 | |