1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * builtin-test.c |
4 | * |
5 | * Builtin regression testing command: ever growing number of sanity tests |
6 | */ |
7 | #include <fcntl.h> |
8 | #include <errno.h> |
9 | #include <poll.h> |
10 | #include <unistd.h> |
11 | #include <string.h> |
12 | #include <stdlib.h> |
13 | #include <sys/types.h> |
14 | #include <dirent.h> |
15 | #include <sys/wait.h> |
16 | #include <sys/stat.h> |
17 | #include "builtin.h" |
18 | #include "config.h" |
19 | #include "hist.h" |
20 | #include "intlist.h" |
21 | #include "tests.h" |
22 | #include "debug.h" |
23 | #include "color.h" |
24 | #include <subcmd/parse-options.h> |
25 | #include <subcmd/run-command.h> |
26 | #include "string2.h" |
27 | #include "symbol.h" |
28 | #include "util/rlimit.h" |
29 | #include "util/strbuf.h" |
30 | #include <linux/kernel.h> |
31 | #include <linux/string.h> |
32 | #include <subcmd/exec-cmd.h> |
33 | #include <linux/zalloc.h> |
34 | |
35 | #include "tests-scripts.h" |
36 | |
37 | /* |
38 | * Command line option to not fork the test running in the same process and |
39 | * making them easier to debug. |
40 | */ |
41 | static bool dont_fork; |
42 | /* Fork the tests in parallel and then wait for their completion. */ |
43 | static bool parallel; |
44 | const char *dso_to_test; |
45 | const char *test_objdump_path = "objdump" ; |
46 | |
47 | /* |
48 | * List of architecture specific tests. Not a weak symbol as the array length is |
49 | * dependent on the initialization, as such GCC with LTO complains of |
50 | * conflicting definitions with a weak symbol. |
51 | */ |
52 | #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) || defined(__powerpc64__) |
53 | extern struct test_suite *arch_tests[]; |
54 | #else |
55 | static struct test_suite *arch_tests[] = { |
56 | NULL, |
57 | }; |
58 | #endif |
59 | |
60 | static struct test_suite *generic_tests[] = { |
61 | &suite__vmlinux_matches_kallsyms, |
62 | #ifdef HAVE_LIBTRACEEVENT |
63 | &suite__openat_syscall_event, |
64 | &suite__openat_syscall_event_on_all_cpus, |
65 | &suite__basic_mmap, |
66 | #endif |
67 | &suite__mem, |
68 | &suite__parse_events, |
69 | &suite__expr, |
70 | &suite__PERF_RECORD, |
71 | &suite__pmu, |
72 | &suite__pmu_events, |
73 | &suite__dso_data, |
74 | &suite__perf_evsel__roundtrip_name_test, |
75 | #ifdef HAVE_LIBTRACEEVENT |
76 | &suite__perf_evsel__tp_sched_test, |
77 | &suite__syscall_openat_tp_fields, |
78 | #endif |
79 | &suite__attr, |
80 | &suite__hists_link, |
81 | &suite__python_use, |
82 | &suite__bp_signal, |
83 | &suite__bp_signal_overflow, |
84 | &suite__bp_accounting, |
85 | &suite__wp, |
86 | &suite__task_exit, |
87 | &suite__sw_clock_freq, |
88 | &suite__code_reading, |
89 | &suite__sample_parsing, |
90 | &suite__keep_tracking, |
91 | &suite__parse_no_sample_id_all, |
92 | &suite__hists_filter, |
93 | &suite__mmap_thread_lookup, |
94 | &suite__thread_maps_share, |
95 | &suite__hists_output, |
96 | &suite__hists_cumulate, |
97 | #ifdef HAVE_LIBTRACEEVENT |
98 | &suite__switch_tracking, |
99 | #endif |
100 | &suite__fdarray__filter, |
101 | &suite__fdarray__add, |
102 | &suite__kmod_path__parse, |
103 | &suite__thread_map, |
104 | &suite__session_topology, |
105 | &suite__thread_map_synthesize, |
106 | &suite__thread_map_remove, |
107 | &suite__cpu_map, |
108 | &suite__synthesize_stat_config, |
109 | &suite__synthesize_stat, |
110 | &suite__synthesize_stat_round, |
111 | &suite__event_update, |
112 | &suite__event_times, |
113 | &suite__backward_ring_buffer, |
114 | &suite__sdt_event, |
115 | &suite__is_printable_array, |
116 | &suite__bitmap_print, |
117 | &suite__perf_hooks, |
118 | &suite__unit_number__scnprint, |
119 | &suite__mem2node, |
120 | &suite__time_utils, |
121 | &suite__jit_write_elf, |
122 | &suite__pfm, |
123 | &suite__api_io, |
124 | &suite__maps__merge_in, |
125 | &suite__demangle_java, |
126 | &suite__demangle_ocaml, |
127 | &suite__parse_metric, |
128 | &suite__pe_file_parsing, |
129 | &suite__expand_cgroup_events, |
130 | &suite__perf_time_to_tsc, |
131 | &suite__dlfilter, |
132 | &suite__sigtrap, |
133 | &suite__event_groups, |
134 | &suite__symbols, |
135 | &suite__util, |
136 | NULL, |
137 | }; |
138 | |
139 | static struct test_suite **tests[] = { |
140 | generic_tests, |
141 | arch_tests, |
142 | NULL, /* shell tests created at runtime. */ |
143 | }; |
144 | |
145 | static struct test_workload *workloads[] = { |
146 | &workload__noploop, |
147 | &workload__thloop, |
148 | &workload__leafloop, |
149 | &workload__sqrtloop, |
150 | &workload__brstack, |
151 | &workload__datasym, |
152 | }; |
153 | |
154 | static int num_subtests(const struct test_suite *t) |
155 | { |
156 | int num; |
157 | |
158 | if (!t->test_cases) |
159 | return 0; |
160 | |
161 | num = 0; |
162 | while (t->test_cases[num].name) |
163 | num++; |
164 | |
165 | return num; |
166 | } |
167 | |
168 | static bool has_subtests(const struct test_suite *t) |
169 | { |
170 | return num_subtests(t) > 1; |
171 | } |
172 | |
173 | static const char *skip_reason(const struct test_suite *t, int subtest) |
174 | { |
175 | if (!t->test_cases) |
176 | return NULL; |
177 | |
178 | return t->test_cases[subtest >= 0 ? subtest : 0].skip_reason; |
179 | } |
180 | |
181 | static const char *test_description(const struct test_suite *t, int subtest) |
182 | { |
183 | if (t->test_cases && subtest >= 0) |
184 | return t->test_cases[subtest].desc; |
185 | |
186 | return t->desc; |
187 | } |
188 | |
189 | static test_fnptr test_function(const struct test_suite *t, int subtest) |
190 | { |
191 | if (subtest <= 0) |
192 | return t->test_cases[0].run_case; |
193 | |
194 | return t->test_cases[subtest].run_case; |
195 | } |
196 | |
197 | static bool perf_test__matches(const char *desc, int curr, int argc, const char *argv[]) |
198 | { |
199 | int i; |
200 | |
201 | if (argc == 0) |
202 | return true; |
203 | |
204 | for (i = 0; i < argc; ++i) { |
205 | char *end; |
206 | long nr = strtoul(argv[i], &end, 10); |
207 | |
208 | if (*end == '\0') { |
209 | if (nr == curr + 1) |
210 | return true; |
211 | continue; |
212 | } |
213 | |
214 | if (strcasestr(desc, argv[i])) |
215 | return true; |
216 | } |
217 | |
218 | return false; |
219 | } |
220 | |
221 | struct child_test { |
222 | struct child_process process; |
223 | struct test_suite *test; |
224 | int test_num; |
225 | int subtest; |
226 | }; |
227 | |
228 | static int run_test_child(struct child_process *process) |
229 | { |
230 | struct child_test *child = container_of(process, struct child_test, process); |
231 | int err; |
232 | |
233 | pr_debug("--- start ---\n" ); |
234 | pr_debug("test child forked, pid %d\n" , getpid()); |
235 | err = test_function(t: child->test, subtest: child->subtest)(child->test, child->subtest); |
236 | pr_debug("---- end(%d) ----\n" , err); |
237 | fflush(NULL); |
238 | return -err; |
239 | } |
240 | |
241 | static int print_test_result(struct test_suite *t, int i, int subtest, int result, int width) |
242 | { |
243 | if (has_subtests(t)) { |
244 | int subw = width > 2 ? width - 2 : width; |
245 | |
246 | pr_info("%3d.%1d: %-*s:" , i + 1, subtest + 1, subw, test_description(t, subtest)); |
247 | } else |
248 | pr_info("%3d: %-*s:" , i + 1, width, test_description(t, subtest)); |
249 | |
250 | switch (result) { |
251 | case TEST_OK: |
252 | pr_info(" Ok\n" ); |
253 | break; |
254 | case TEST_SKIP: { |
255 | const char *reason = skip_reason(t, subtest); |
256 | |
257 | if (reason) |
258 | color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (%s)\n" , reason); |
259 | else |
260 | color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n" ); |
261 | } |
262 | break; |
263 | case TEST_FAIL: |
264 | default: |
265 | color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n" ); |
266 | break; |
267 | } |
268 | |
269 | return 0; |
270 | } |
271 | |
272 | static int finish_test(struct child_test *child_test, int width) |
273 | { |
274 | struct test_suite *t = child_test->test; |
275 | int i = child_test->test_num; |
276 | int subi = child_test->subtest; |
277 | int out = child_test->process.out; |
278 | int err = child_test->process.err; |
279 | bool out_done = out <= 0; |
280 | bool err_done = err <= 0; |
281 | struct strbuf out_output = STRBUF_INIT; |
282 | struct strbuf err_output = STRBUF_INIT; |
283 | int ret; |
284 | |
285 | /* |
286 | * For test suites with subtests, display the suite name ahead of the |
287 | * sub test names. |
288 | */ |
289 | if (has_subtests(t) && subi == 0) |
290 | pr_info("%3d: %-*s:\n" , i + 1, width, test_description(t, -1)); |
291 | |
292 | /* |
293 | * Busy loop reading from the child's stdout and stderr that are set to |
294 | * be non-blocking until EOF. |
295 | */ |
296 | if (!out_done) |
297 | fcntl(out, F_SETFL, O_NONBLOCK); |
298 | if (!err_done) |
299 | fcntl(err, F_SETFL, O_NONBLOCK); |
300 | if (verbose > 1) { |
301 | if (has_subtests(t)) |
302 | pr_info("%3d.%1d: %s:\n" , i + 1, subi + 1, test_description(t, subi)); |
303 | else |
304 | pr_info("%3d: %s:\n" , i + 1, test_description(t, -1)); |
305 | } |
306 | while (!out_done || !err_done) { |
307 | struct pollfd pfds[2] = { |
308 | { .fd = out, |
309 | .events = POLLIN | POLLERR | POLLHUP | POLLNVAL, |
310 | }, |
311 | { .fd = err, |
312 | .events = POLLIN | POLLERR | POLLHUP | POLLNVAL, |
313 | }, |
314 | }; |
315 | char buf[512]; |
316 | ssize_t len; |
317 | |
318 | /* Poll to avoid excessive spinning, timeout set for 1000ms. */ |
319 | poll(pfds, ARRAY_SIZE(pfds), /*timeout=*/1000); |
320 | if (!out_done && pfds[0].revents) { |
321 | errno = 0; |
322 | len = read(out, buf, sizeof(buf) - 1); |
323 | |
324 | if (len <= 0) { |
325 | out_done = errno != EAGAIN; |
326 | } else { |
327 | buf[len] = '\0'; |
328 | if (verbose > 1) |
329 | fprintf(stdout, "%s" , buf); |
330 | else |
331 | strbuf_addstr(&out_output, buf); |
332 | } |
333 | } |
334 | if (!err_done && pfds[1].revents) { |
335 | errno = 0; |
336 | len = read(err, buf, sizeof(buf) - 1); |
337 | |
338 | if (len <= 0) { |
339 | err_done = errno != EAGAIN; |
340 | } else { |
341 | buf[len] = '\0'; |
342 | if (verbose > 1) |
343 | fprintf(stdout, "%s" , buf); |
344 | else |
345 | strbuf_addstr(&err_output, buf); |
346 | } |
347 | } |
348 | } |
349 | /* Clean up child process. */ |
350 | ret = finish_command(&child_test->process); |
351 | if (verbose == 1 && ret == TEST_FAIL) { |
352 | /* Add header for test that was skipped above. */ |
353 | if (has_subtests(t)) |
354 | pr_info("%3d.%1d: %s:\n" , i + 1, subi + 1, test_description(t, subi)); |
355 | else |
356 | pr_info("%3d: %s:\n" , i + 1, test_description(t, -1)); |
357 | fprintf(stdout, "%s" , out_output.buf); |
358 | fprintf(stderr, "%s" , err_output.buf); |
359 | } |
360 | strbuf_release(&out_output); |
361 | strbuf_release(&err_output); |
362 | print_test_result(t, i, subtest: subi, result: ret, width); |
363 | if (out > 0) |
364 | close(out); |
365 | if (err > 0) |
366 | close(err); |
367 | return 0; |
368 | } |
369 | |
370 | static int start_test(struct test_suite *test, int i, int subi, struct child_test **child, |
371 | int width) |
372 | { |
373 | int err; |
374 | |
375 | *child = NULL; |
376 | if (dont_fork) { |
377 | pr_debug("--- start ---\n" ); |
378 | err = test_function(t: test, subtest: subi)(test, subi); |
379 | pr_debug("---- end ----\n" ); |
380 | print_test_result(t: test, i, subtest: subi, result: err, width); |
381 | return 0; |
382 | } |
383 | |
384 | *child = zalloc(sizeof(**child)); |
385 | if (!*child) |
386 | return -ENOMEM; |
387 | |
388 | (*child)->test = test; |
389 | (*child)->test_num = i; |
390 | (*child)->subtest = subi; |
391 | (*child)->process.pid = -1; |
392 | (*child)->process.no_stdin = 1; |
393 | if (verbose <= 0) { |
394 | (*child)->process.no_stdout = 1; |
395 | (*child)->process.no_stderr = 1; |
396 | } else { |
397 | (*child)->process.out = -1; |
398 | (*child)->process.err = -1; |
399 | } |
400 | (*child)->process.no_exec_cmd = run_test_child; |
401 | err = start_command(&(*child)->process); |
402 | if (err || parallel) |
403 | return err; |
404 | return finish_test(child_test: *child, width); |
405 | } |
406 | |
407 | #define for_each_test(j, k, t) \ |
408 | for (j = 0, k = 0; j < ARRAY_SIZE(tests); j++, k = 0) \ |
409 | while ((t = tests[j][k++]) != NULL) |
410 | |
411 | static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist) |
412 | { |
413 | struct test_suite *t; |
414 | unsigned int j, k; |
415 | int i = 0; |
416 | int width = 0; |
417 | size_t num_tests = 0; |
418 | struct child_test **child_tests; |
419 | int child_test_num = 0; |
420 | |
421 | for_each_test(j, k, t) { |
422 | int len = strlen(test_description(t, -1)); |
423 | |
424 | if (width < len) |
425 | width = len; |
426 | |
427 | if (has_subtests(t)) { |
428 | for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) { |
429 | len = strlen(test_description(t, subi)); |
430 | if (width < len) |
431 | width = len; |
432 | num_tests++; |
433 | } |
434 | } else { |
435 | num_tests++; |
436 | } |
437 | } |
438 | child_tests = calloc(num_tests, sizeof(*child_tests)); |
439 | if (!child_tests) |
440 | return -ENOMEM; |
441 | |
442 | for_each_test(j, k, t) { |
443 | int curr = i++; |
444 | |
445 | if (!perf_test__matches(desc: test_description(t, subtest: -1), curr, argc, argv)) { |
446 | bool skip = true; |
447 | |
448 | for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) { |
449 | if (perf_test__matches(desc: test_description(t, subtest: subi), |
450 | curr, argc, argv)) |
451 | skip = false; |
452 | } |
453 | |
454 | if (skip) |
455 | continue; |
456 | } |
457 | |
458 | if (intlist__find(skiplist, i)) { |
459 | pr_info("%3d: %-*s:" , curr + 1, width, test_description(t, -1)); |
460 | color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n" ); |
461 | continue; |
462 | } |
463 | |
464 | if (!has_subtests(t)) { |
465 | int err = start_test(test: t, i: curr, subi: -1, child: &child_tests[child_test_num++], width); |
466 | |
467 | if (err) { |
468 | /* TODO: if parallel waitpid the already forked children. */ |
469 | free(child_tests); |
470 | return err; |
471 | } |
472 | } else { |
473 | for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) { |
474 | int err; |
475 | |
476 | if (!perf_test__matches(desc: test_description(t, subtest: subi), |
477 | curr, argc, argv)) |
478 | continue; |
479 | |
480 | err = start_test(test: t, i: curr, subi, child: &child_tests[child_test_num++], |
481 | width); |
482 | if (err) |
483 | return err; |
484 | } |
485 | } |
486 | } |
487 | for (i = 0; i < child_test_num; i++) { |
488 | if (parallel) { |
489 | int ret = finish_test(child_test: child_tests[i], width); |
490 | |
491 | if (ret) |
492 | return ret; |
493 | } |
494 | free(child_tests[i]); |
495 | } |
496 | free(child_tests); |
497 | return 0; |
498 | } |
499 | |
500 | static int perf_test__list(int argc, const char **argv) |
501 | { |
502 | unsigned int j, k; |
503 | struct test_suite *t; |
504 | int i = 0; |
505 | |
506 | for_each_test(j, k, t) { |
507 | int curr = i++; |
508 | |
509 | if (!perf_test__matches(desc: test_description(t, subtest: -1), curr, argc, argv)) |
510 | continue; |
511 | |
512 | pr_info("%3d: %s\n" , i, test_description(t, -1)); |
513 | |
514 | if (has_subtests(t)) { |
515 | int subn = num_subtests(t); |
516 | int subi; |
517 | |
518 | for (subi = 0; subi < subn; subi++) |
519 | pr_info("%3d:%1d: %s\n" , i, subi + 1, |
520 | test_description(t, subi)); |
521 | } |
522 | } |
523 | return 0; |
524 | } |
525 | |
526 | static int run_workload(const char *work, int argc, const char **argv) |
527 | { |
528 | unsigned int i = 0; |
529 | struct test_workload *twl; |
530 | |
531 | for (i = 0; i < ARRAY_SIZE(workloads); i++) { |
532 | twl = workloads[i]; |
533 | if (!strcmp(twl->name, work)) |
534 | return twl->func(argc, argv); |
535 | } |
536 | |
537 | pr_info("No workload found: %s\n" , work); |
538 | return -1; |
539 | } |
540 | |
541 | static int perf_test__config(const char *var, const char *value, |
542 | void *data __maybe_unused) |
543 | { |
544 | if (!strcmp(var, "annotate.objdump" )) |
545 | test_objdump_path = value; |
546 | |
547 | return 0; |
548 | } |
549 | |
550 | int cmd_test(int argc, const char **argv) |
551 | { |
552 | const char *test_usage[] = { |
553 | "perf test [<options>] [{list <test-name-fragment>|[<test-name-fragments>|<test-numbers>]}]" , |
554 | NULL, |
555 | }; |
556 | const char *skip = NULL; |
557 | const char *workload = NULL; |
558 | const struct option test_options[] = { |
559 | OPT_STRING('s', "skip" , &skip, "tests" , "tests to skip" ), |
560 | OPT_INCR('v', "verbose" , &verbose, |
561 | "be more verbose (show symbol address, etc)" ), |
562 | OPT_BOOLEAN('F', "dont-fork" , &dont_fork, |
563 | "Do not fork for testcase" ), |
564 | OPT_BOOLEAN('p', "parallel" , ¶llel, |
565 | "Run the tests altogether in parallel" ), |
566 | OPT_STRING('w', "workload" , &workload, "work" , "workload to run for testing" ), |
567 | OPT_STRING(0, "dso" , &dso_to_test, "dso" , "dso to test" ), |
568 | OPT_STRING(0, "objdump" , &test_objdump_path, "path" , |
569 | "objdump binary to use for disassembly and annotations" ), |
570 | OPT_END() |
571 | }; |
572 | const char * const test_subcommands[] = { "list" , NULL }; |
573 | struct intlist *skiplist = NULL; |
574 | int ret = hists__init(); |
575 | |
576 | if (ret < 0) |
577 | return ret; |
578 | |
579 | perf_config(perf_test__config, NULL); |
580 | |
581 | /* Unbuffered output */ |
582 | setvbuf(stdout, NULL, _IONBF, 0); |
583 | |
584 | tests[2] = create_script_test_suites(); |
585 | argc = parse_options_subcommand(argc, argv, test_options, test_subcommands, test_usage, 0); |
586 | if (argc >= 1 && !strcmp(argv[0], "list" )) |
587 | return perf_test__list(argc: argc - 1, argv: argv + 1); |
588 | |
589 | if (workload) |
590 | return run_workload(work: workload, argc, argv); |
591 | |
592 | symbol_conf.priv_size = sizeof(int); |
593 | symbol_conf.try_vmlinux_path = true; |
594 | |
595 | if (symbol__init(NULL) < 0) |
596 | return -1; |
597 | |
598 | if (skip != NULL) |
599 | skiplist = intlist__new(skip); |
600 | /* |
601 | * Tests that create BPF maps, for instance, need more than the 64K |
602 | * default: |
603 | */ |
604 | rlimit__bump_memlock(); |
605 | |
606 | return __cmd_test(argc, argv, skiplist); |
607 | } |
608 | |