1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Manage affinity to optimize IPIs inside the kernel perf API. */ |
3 | #define _GNU_SOURCE 1 |
4 | #include <sched.h> |
5 | #include <stdlib.h> |
6 | #include <linux/bitmap.h> |
7 | #include <linux/zalloc.h> |
8 | #include "perf.h" |
9 | #include "cpumap.h" |
10 | #include "affinity.h" |
11 | |
12 | static int get_cpu_set_size(void) |
13 | { |
14 | int sz = cpu__max_cpu().cpu + 8 - 1; |
15 | /* |
16 | * sched_getaffinity doesn't like masks smaller than the kernel. |
17 | * Hopefully that's big enough. |
18 | */ |
19 | if (sz < 4096) |
20 | sz = 4096; |
21 | return sz / 8; |
22 | } |
23 | |
24 | int affinity__setup(struct affinity *a) |
25 | { |
26 | int cpu_set_size = get_cpu_set_size(); |
27 | |
28 | a->orig_cpus = bitmap_zalloc(cpu_set_size * 8); |
29 | if (!a->orig_cpus) |
30 | return -1; |
31 | sched_getaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus); |
32 | a->sched_cpus = bitmap_zalloc(cpu_set_size * 8); |
33 | if (!a->sched_cpus) { |
34 | zfree(&a->orig_cpus); |
35 | return -1; |
36 | } |
37 | bitmap_zero(dst: (unsigned long *)a->sched_cpus, nbits: cpu_set_size); |
38 | a->changed = false; |
39 | return 0; |
40 | } |
41 | |
42 | /* |
43 | * perf_event_open does an IPI internally to the target CPU. |
44 | * It is more efficient to change perf's affinity to the target |
45 | * CPU and then set up all events on that CPU, so we amortize |
46 | * CPU communication. |
47 | */ |
48 | void affinity__set(struct affinity *a, int cpu) |
49 | { |
50 | int cpu_set_size = get_cpu_set_size(); |
51 | |
52 | /* |
53 | * Return: |
54 | * - if cpu is -1 |
55 | * - restrict out of bound access to sched_cpus |
56 | */ |
57 | if (cpu == -1 || ((cpu >= (cpu_set_size * 8)))) |
58 | return; |
59 | |
60 | a->changed = true; |
61 | __set_bit(cpu, a->sched_cpus); |
62 | /* |
63 | * We ignore errors because affinity is just an optimization. |
64 | * This could happen for example with isolated CPUs or cpusets. |
65 | * In this case the IPIs inside the kernel's perf API still work. |
66 | */ |
67 | sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->sched_cpus); |
68 | __clear_bit(cpu, a->sched_cpus); |
69 | } |
70 | |
71 | static void __affinity__cleanup(struct affinity *a) |
72 | { |
73 | int cpu_set_size = get_cpu_set_size(); |
74 | |
75 | if (a->changed) |
76 | sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus); |
77 | zfree(&a->sched_cpus); |
78 | zfree(&a->orig_cpus); |
79 | } |
80 | |
81 | void affinity__cleanup(struct affinity *a) |
82 | { |
83 | if (a != NULL) |
84 | __affinity__cleanup(a); |
85 | } |
86 | |