1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2013 Davidlohr Bueso <davidlohr@hp.com> |
4 | * |
5 | * futex-requeue: Block a bunch of threads on futex1 and requeue them |
6 | * on futex2, N at a time. |
7 | * |
8 | * This program is particularly useful to measure the latency of nthread |
9 | * requeues without waking up any tasks (in the non-pi case) -- thus |
10 | * mimicking a regular futex_wait. |
11 | */ |
12 | |
13 | /* For the CLR_() macros */ |
14 | #include <string.h> |
15 | #include <pthread.h> |
16 | |
17 | #include <signal.h> |
18 | #include "../util/mutex.h" |
19 | #include "../util/stat.h" |
20 | #include <subcmd/parse-options.h> |
21 | #include <linux/compiler.h> |
22 | #include <linux/kernel.h> |
23 | #include <linux/time64.h> |
24 | #include <errno.h> |
25 | #include <perf/cpumap.h> |
26 | #include "bench.h" |
27 | #include "futex.h" |
28 | |
29 | #include <err.h> |
30 | #include <stdlib.h> |
31 | #include <sys/time.h> |
32 | #include <sys/mman.h> |
33 | |
34 | static u_int32_t futex1 = 0, futex2 = 0; |
35 | |
36 | static pthread_t *worker; |
37 | static bool done = false; |
38 | static struct mutex thread_lock; |
39 | static struct cond thread_parent, thread_worker; |
40 | static struct stats requeuetime_stats, requeued_stats; |
41 | static unsigned int threads_starting; |
42 | static int futex_flag = 0; |
43 | |
44 | static struct bench_futex_parameters params = { |
45 | /* |
46 | * How many tasks to requeue at a time. |
47 | * Default to 1 in order to make the kernel work more. |
48 | */ |
49 | .nrequeue = 1, |
50 | }; |
51 | |
52 | static const struct option options[] = { |
53 | OPT_UINTEGER('t', "threads" , ¶ms.nthreads, "Specify amount of threads" ), |
54 | OPT_UINTEGER('q', "nrequeue" , ¶ms.nrequeue, "Specify amount of threads to requeue at once" ), |
55 | OPT_BOOLEAN( 's', "silent" , ¶ms.silent, "Silent mode: do not display data/details" ), |
56 | OPT_BOOLEAN( 'S', "shared" , ¶ms.fshared, "Use shared futexes instead of private ones" ), |
57 | OPT_BOOLEAN( 'm', "mlockall" , ¶ms.mlockall, "Lock all current and future memory" ), |
58 | OPT_BOOLEAN( 'B', "broadcast" , ¶ms.broadcast, "Requeue all threads at once" ), |
59 | OPT_BOOLEAN( 'p', "pi" , ¶ms.pi, "Use PI-aware variants of FUTEX_CMP_REQUEUE" ), |
60 | |
61 | OPT_END() |
62 | }; |
63 | |
64 | static const char * const bench_futex_requeue_usage[] = { |
65 | "perf bench futex requeue <options>" , |
66 | NULL |
67 | }; |
68 | |
69 | static void print_summary(void) |
70 | { |
71 | double requeuetime_avg = avg_stats(stats: &requeuetime_stats); |
72 | double requeuetime_stddev = stddev_stats(stats: &requeuetime_stats); |
73 | unsigned int requeued_avg = avg_stats(stats: &requeued_stats); |
74 | |
75 | printf("Requeued %d of %d threads in %.4f ms (+-%.2f%%)\n" , |
76 | requeued_avg, |
77 | params.nthreads, |
78 | requeuetime_avg / USEC_PER_MSEC, |
79 | rel_stddev_stats(stddev: requeuetime_stddev, avg: requeuetime_avg)); |
80 | } |
81 | |
82 | static void *workerfn(void *arg __maybe_unused) |
83 | { |
84 | int ret; |
85 | |
86 | mutex_lock(mtx: &thread_lock); |
87 | threads_starting--; |
88 | if (!threads_starting) |
89 | cond_signal(cnd: &thread_parent); |
90 | cond_wait(cnd: &thread_worker, mtx: &thread_lock); |
91 | mutex_unlock(mtx: &thread_lock); |
92 | |
93 | while (1) { |
94 | if (!params.pi) { |
95 | ret = futex_wait(uaddr: &futex1, val: 0, NULL, opflags: futex_flag); |
96 | if (!ret) |
97 | break; |
98 | |
99 | if (ret && errno != EAGAIN) { |
100 | if (!params.silent) |
101 | warnx("futex_wait" ); |
102 | break; |
103 | } |
104 | } else { |
105 | ret = futex_wait_requeue_pi(uaddr: &futex1, val: 0, uaddr2: &futex2, |
106 | NULL, opflags: futex_flag); |
107 | if (!ret) { |
108 | /* got the lock at futex2 */ |
109 | futex_unlock_pi(uaddr: &futex2, opflags: futex_flag); |
110 | break; |
111 | } |
112 | |
113 | if (ret && errno != EAGAIN) { |
114 | if (!params.silent) |
115 | warnx("futex_wait_requeue_pi" ); |
116 | break; |
117 | } |
118 | } |
119 | } |
120 | |
121 | return NULL; |
122 | } |
123 | |
124 | static void block_threads(pthread_t *w, struct perf_cpu_map *cpu) |
125 | { |
126 | cpu_set_t *cpuset; |
127 | unsigned int i; |
128 | int nrcpus = perf_cpu_map__nr(cpu); |
129 | size_t size; |
130 | |
131 | threads_starting = params.nthreads; |
132 | |
133 | cpuset = CPU_ALLOC(nrcpus); |
134 | BUG_ON(!cpuset); |
135 | size = CPU_ALLOC_SIZE(nrcpus); |
136 | |
137 | /* create and block all threads */ |
138 | for (i = 0; i < params.nthreads; i++) { |
139 | pthread_attr_t thread_attr; |
140 | |
141 | pthread_attr_init(&thread_attr); |
142 | CPU_ZERO_S(size, cpuset); |
143 | CPU_SET_S(perf_cpu_map__cpu(cpu, i % perf_cpu_map__nr(cpu)).cpu, size, cpuset); |
144 | |
145 | if (pthread_attr_setaffinity_np(&thread_attr, size, cpuset)) { |
146 | CPU_FREE(cpuset); |
147 | err(EXIT_FAILURE, "pthread_attr_setaffinity_np" ); |
148 | } |
149 | |
150 | if (pthread_create(&w[i], &thread_attr, workerfn, NULL)) { |
151 | CPU_FREE(cpuset); |
152 | err(EXIT_FAILURE, "pthread_create" ); |
153 | } |
154 | pthread_attr_destroy(&thread_attr); |
155 | } |
156 | CPU_FREE(cpuset); |
157 | } |
158 | |
159 | static void toggle_done(int sig __maybe_unused, |
160 | siginfo_t *info __maybe_unused, |
161 | void *uc __maybe_unused) |
162 | { |
163 | done = true; |
164 | } |
165 | |
166 | int bench_futex_requeue(int argc, const char **argv) |
167 | { |
168 | int ret = 0; |
169 | unsigned int i, j; |
170 | struct sigaction act; |
171 | struct perf_cpu_map *cpu; |
172 | |
173 | argc = parse_options(argc, argv, options, bench_futex_requeue_usage, 0); |
174 | if (argc) |
175 | goto err; |
176 | |
177 | cpu = perf_cpu_map__new_online_cpus(); |
178 | if (!cpu) |
179 | err(EXIT_FAILURE, "cpu_map__new" ); |
180 | |
181 | memset(&act, 0, sizeof(act)); |
182 | sigfillset(&act.sa_mask); |
183 | act.sa_sigaction = toggle_done; |
184 | sigaction(SIGINT, &act, NULL); |
185 | |
186 | if (params.mlockall) { |
187 | if (mlockall(MCL_CURRENT | MCL_FUTURE)) |
188 | err(EXIT_FAILURE, "mlockall" ); |
189 | } |
190 | |
191 | if (!params.nthreads) |
192 | params.nthreads = perf_cpu_map__nr(cpu); |
193 | |
194 | worker = calloc(params.nthreads, sizeof(*worker)); |
195 | if (!worker) |
196 | err(EXIT_FAILURE, "calloc" ); |
197 | |
198 | if (!params.fshared) |
199 | futex_flag = FUTEX_PRIVATE_FLAG; |
200 | |
201 | if (params.nrequeue > params.nthreads) |
202 | params.nrequeue = params.nthreads; |
203 | |
204 | if (params.broadcast) |
205 | params.nrequeue = params.nthreads; |
206 | |
207 | printf("Run summary [PID %d]: Requeuing %d threads (from [%s] %p to %s%p), " |
208 | "%d at a time.\n\n" , getpid(), params.nthreads, |
209 | params.fshared ? "shared" :"private" , &futex1, |
210 | params.pi ? "PI " : "" , &futex2, params.nrequeue); |
211 | |
212 | init_stats(stats: &requeued_stats); |
213 | init_stats(stats: &requeuetime_stats); |
214 | mutex_init(mtx: &thread_lock); |
215 | cond_init(cnd: &thread_parent); |
216 | cond_init(cnd: &thread_worker); |
217 | |
218 | for (j = 0; j < bench_repeat && !done; j++) { |
219 | unsigned int nrequeued = 0, wakeups = 0; |
220 | struct timeval start, end, runtime; |
221 | |
222 | /* create, launch & block all threads */ |
223 | block_threads(worker, cpu); |
224 | |
225 | /* make sure all threads are already blocked */ |
226 | mutex_lock(mtx: &thread_lock); |
227 | while (threads_starting) |
228 | cond_wait(cnd: &thread_parent, mtx: &thread_lock); |
229 | cond_broadcast(cnd: &thread_worker); |
230 | mutex_unlock(mtx: &thread_lock); |
231 | |
232 | usleep(100000); |
233 | |
234 | /* Ok, all threads are patiently blocked, start requeueing */ |
235 | gettimeofday(&start, NULL); |
236 | while (nrequeued < params.nthreads) { |
237 | int r; |
238 | |
239 | /* |
240 | * For the regular non-pi case, do not wakeup any tasks |
241 | * blocked on futex1, allowing us to really measure |
242 | * futex_wait functionality. For the PI case the first |
243 | * waiter is always awoken. |
244 | */ |
245 | if (!params.pi) { |
246 | r = futex_cmp_requeue(uaddr: &futex1, val: 0, uaddr2: &futex2, nr_wake: 0, |
247 | nr_requeue: params.nrequeue, |
248 | opflags: futex_flag); |
249 | } else { |
250 | r = futex_cmp_requeue_pi(uaddr: &futex1, val: 0, uaddr2: &futex2, |
251 | nr_requeue: params.nrequeue, |
252 | opflags: futex_flag); |
253 | wakeups++; /* assume no error */ |
254 | } |
255 | |
256 | if (r < 0) |
257 | err(EXIT_FAILURE, "couldn't requeue from %p to %p" , |
258 | &futex1, &futex2); |
259 | |
260 | nrequeued += r; |
261 | } |
262 | |
263 | gettimeofday(&end, NULL); |
264 | timersub(&end, &start, &runtime); |
265 | |
266 | update_stats(stats: &requeued_stats, val: nrequeued); |
267 | update_stats(stats: &requeuetime_stats, val: runtime.tv_usec); |
268 | |
269 | if (!params.silent) { |
270 | if (!params.pi) |
271 | printf("[Run %d]: Requeued %d of %d threads in " |
272 | "%.4f ms\n" , j + 1, nrequeued, |
273 | params.nthreads, |
274 | runtime.tv_usec / (double)USEC_PER_MSEC); |
275 | else { |
276 | nrequeued -= wakeups; |
277 | printf("[Run %d]: Awoke and Requeued (%d+%d) of " |
278 | "%d threads in %.4f ms\n" , |
279 | j + 1, wakeups, nrequeued, |
280 | params.nthreads, |
281 | runtime.tv_usec / (double)USEC_PER_MSEC); |
282 | } |
283 | |
284 | } |
285 | |
286 | if (!params.pi) { |
287 | /* everybody should be blocked on futex2, wake'em up */ |
288 | nrequeued = futex_wake(uaddr: &futex2, nr_wake: nrequeued, opflags: futex_flag); |
289 | if (params.nthreads != nrequeued) |
290 | warnx("couldn't wakeup all tasks (%d/%d)" , |
291 | nrequeued, params.nthreads); |
292 | } |
293 | |
294 | for (i = 0; i < params.nthreads; i++) { |
295 | ret = pthread_join(worker[i], NULL); |
296 | if (ret) |
297 | err(EXIT_FAILURE, "pthread_join" ); |
298 | } |
299 | } |
300 | |
301 | /* cleanup & report results */ |
302 | cond_destroy(cnd: &thread_parent); |
303 | cond_destroy(cnd: &thread_worker); |
304 | mutex_destroy(mtx: &thread_lock); |
305 | |
306 | print_summary(); |
307 | |
308 | free(worker); |
309 | perf_cpu_map__put(cpu); |
310 | return ret; |
311 | err: |
312 | usage_with_options(bench_futex_requeue_usage, options); |
313 | exit(EXIT_FAILURE); |
314 | } |
315 | |