1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright 2014, Michael Ellerman, IBM Corp. |
4 | */ |
5 | |
6 | #define _GNU_SOURCE /* For CPU_ZERO etc. */ |
7 | |
8 | #include <sched.h> |
9 | #include <sys/wait.h> |
10 | #include <setjmp.h> |
11 | #include <signal.h> |
12 | #include <stdio.h> |
13 | #include <stdlib.h> |
14 | #include <string.h> |
15 | #include <sys/ioctl.h> |
16 | |
17 | #include "trace.h" |
18 | #include "ebb.h" |
19 | |
20 | |
21 | void (*ebb_user_func)(void); |
22 | |
23 | void ebb_hook(void) |
24 | { |
25 | if (ebb_user_func) |
26 | ebb_user_func(); |
27 | } |
28 | |
29 | struct ebb_state ebb_state; |
30 | |
31 | u64 sample_period = 0x40000000ull; |
32 | |
33 | void reset_ebb_with_clear_mask(unsigned long mmcr0_clear_mask) |
34 | { |
35 | u64 val; |
36 | |
37 | /* 2) clear MMCR0[PMAO] - docs say BESCR[PMEO] should do this */ |
38 | /* 3) set MMCR0[PMAE] - docs say BESCR[PME] should do this */ |
39 | val = mfspr(SPRN_MMCR0); |
40 | mtspr(SPRN_MMCR0, (val & ~mmcr0_clear_mask) | MMCR0_PMAE); |
41 | |
42 | /* 4) clear BESCR[PMEO] */ |
43 | mtspr(SPRN_BESCRR, BESCR_PMEO); |
44 | |
45 | /* 5) set BESCR[PME] */ |
46 | mtspr(SPRN_BESCRS, BESCR_PME); |
47 | |
48 | /* 6) rfebb 1 - done in our caller */ |
49 | } |
50 | |
51 | void reset_ebb(void) |
52 | { |
53 | reset_ebb_with_clear_mask(mmcr0_clear_mask: MMCR0_PMAO | MMCR0_FC); |
54 | } |
55 | |
56 | /* Called outside of the EBB handler to check MMCR0 is sane */ |
57 | int ebb_check_mmcr0(void) |
58 | { |
59 | u64 val; |
60 | |
61 | val = mfspr(SPRN_MMCR0); |
62 | if ((val & (MMCR0_FC | MMCR0_PMAO)) == MMCR0_FC) { |
63 | /* It's OK if we see FC & PMAO, but not FC by itself */ |
64 | printf("Outside of loop, only FC set 0x%llx\n" , val); |
65 | return 1; |
66 | } |
67 | |
68 | return 0; |
69 | } |
70 | |
71 | bool ebb_check_count(int pmc, u64 sample_period, int fudge) |
72 | { |
73 | u64 count, upper, lower; |
74 | |
75 | count = ebb_state.stats.pmc_count[PMC_INDEX(pmc)]; |
76 | |
77 | lower = ebb_state.stats.ebb_count * (sample_period - fudge); |
78 | |
79 | if (count < lower) { |
80 | printf("PMC%d count (0x%llx) below lower limit 0x%llx (-0x%llx)\n" , |
81 | pmc, count, lower, lower - count); |
82 | return false; |
83 | } |
84 | |
85 | upper = ebb_state.stats.ebb_count * (sample_period + fudge); |
86 | |
87 | if (count > upper) { |
88 | printf("PMC%d count (0x%llx) above upper limit 0x%llx (+0x%llx)\n" , |
89 | pmc, count, upper, count - upper); |
90 | return false; |
91 | } |
92 | |
93 | printf("PMC%d count (0x%llx) is between 0x%llx and 0x%llx delta +0x%llx/-0x%llx\n" , |
94 | pmc, count, lower, upper, count - lower, upper - count); |
95 | |
96 | return true; |
97 | } |
98 | |
99 | void standard_ebb_callee(void) |
100 | { |
101 | int found, i; |
102 | u64 val; |
103 | |
104 | val = mfspr(SPRN_BESCR); |
105 | if (!(val & BESCR_PMEO)) { |
106 | ebb_state.stats.spurious++; |
107 | goto out; |
108 | } |
109 | |
110 | ebb_state.stats.ebb_count++; |
111 | trace_log_counter(ebb_state.trace, ebb_state.stats.ebb_count); |
112 | |
113 | val = mfspr(SPRN_MMCR0); |
114 | trace_log_reg(ebb_state.trace, SPRN_MMCR0, val); |
115 | |
116 | found = 0; |
117 | for (i = 1; i <= 6; i++) { |
118 | if (ebb_state.pmc_enable[PMC_INDEX(i)]) |
119 | found += count_pmc(pmc: i, sample_period); |
120 | } |
121 | |
122 | if (!found) |
123 | ebb_state.stats.no_overflow++; |
124 | |
125 | out: |
126 | reset_ebb(); |
127 | } |
128 | |
129 | extern void ebb_handler(void); |
130 | |
131 | void setup_ebb_handler(void (*callee)(void)) |
132 | { |
133 | u64 entry; |
134 | |
135 | #if defined(_CALL_ELF) && _CALL_ELF == 2 |
136 | entry = (u64)ebb_handler; |
137 | #else |
138 | struct opd |
139 | { |
140 | u64 entry; |
141 | u64 toc; |
142 | } *opd; |
143 | |
144 | opd = (struct opd *)ebb_handler; |
145 | entry = opd->entry; |
146 | #endif |
147 | printf("EBB Handler is at %#llx\n" , entry); |
148 | |
149 | ebb_user_func = callee; |
150 | |
151 | /* Ensure ebb_user_func is set before we set the handler */ |
152 | mb(); |
153 | mtspr(SPRN_EBBHR, entry); |
154 | |
155 | /* Make sure the handler is set before we return */ |
156 | mb(); |
157 | } |
158 | |
159 | void clear_ebb_stats(void) |
160 | { |
161 | memset(&ebb_state.stats, 0, sizeof(ebb_state.stats)); |
162 | } |
163 | |
164 | void dump_summary_ebb_state(void) |
165 | { |
166 | printf("ebb_state:\n" \ |
167 | " ebb_count = %d\n" \ |
168 | " spurious = %d\n" \ |
169 | " negative = %d\n" \ |
170 | " no_overflow = %d\n" \ |
171 | " pmc[1] count = 0x%llx\n" \ |
172 | " pmc[2] count = 0x%llx\n" \ |
173 | " pmc[3] count = 0x%llx\n" \ |
174 | " pmc[4] count = 0x%llx\n" \ |
175 | " pmc[5] count = 0x%llx\n" \ |
176 | " pmc[6] count = 0x%llx\n" , |
177 | ebb_state.stats.ebb_count, ebb_state.stats.spurious, |
178 | ebb_state.stats.negative, ebb_state.stats.no_overflow, |
179 | ebb_state.stats.pmc_count[0], ebb_state.stats.pmc_count[1], |
180 | ebb_state.stats.pmc_count[2], ebb_state.stats.pmc_count[3], |
181 | ebb_state.stats.pmc_count[4], ebb_state.stats.pmc_count[5]); |
182 | } |
183 | |
184 | static char *decode_mmcr0(u32 value) |
185 | { |
186 | static char buf[16]; |
187 | |
188 | buf[0] = '\0'; |
189 | |
190 | if (value & (1 << 31)) |
191 | strcat(p: buf, q: "FC " ); |
192 | if (value & (1 << 26)) |
193 | strcat(p: buf, q: "PMAE " ); |
194 | if (value & (1 << 7)) |
195 | strcat(p: buf, q: "PMAO " ); |
196 | |
197 | return buf; |
198 | } |
199 | |
200 | static char *decode_bescr(u64 value) |
201 | { |
202 | static char buf[16]; |
203 | |
204 | buf[0] = '\0'; |
205 | |
206 | if (value & (1ull << 63)) |
207 | strcat(p: buf, q: "GE " ); |
208 | if (value & (1ull << 32)) |
209 | strcat(p: buf, q: "PMAE " ); |
210 | if (value & 1) |
211 | strcat(p: buf, q: "PMAO " ); |
212 | |
213 | return buf; |
214 | } |
215 | |
216 | void dump_ebb_hw_state(void) |
217 | { |
218 | u64 bescr; |
219 | u32 mmcr0; |
220 | |
221 | mmcr0 = mfspr(SPRN_MMCR0); |
222 | bescr = mfspr(SPRN_BESCR); |
223 | |
224 | printf("HW state:\n" \ |
225 | "MMCR0 0x%016x %s\n" \ |
226 | "MMCR2 0x%016lx\n" \ |
227 | "EBBHR 0x%016lx\n" \ |
228 | "BESCR 0x%016llx %s\n" \ |
229 | "PMC1 0x%016lx\n" \ |
230 | "PMC2 0x%016lx\n" \ |
231 | "PMC3 0x%016lx\n" \ |
232 | "PMC4 0x%016lx\n" \ |
233 | "PMC5 0x%016lx\n" \ |
234 | "PMC6 0x%016lx\n" \ |
235 | "SIAR 0x%016lx\n" , |
236 | mmcr0, decode_mmcr0(value: mmcr0), mfspr(SPRN_MMCR2), |
237 | mfspr(SPRN_EBBHR), bescr, decode_bescr(value: bescr), |
238 | mfspr(SPRN_PMC1), mfspr(SPRN_PMC2), mfspr(SPRN_PMC3), |
239 | mfspr(SPRN_PMC4), mfspr(SPRN_PMC5), mfspr(SPRN_PMC6), |
240 | mfspr(SPRN_SIAR)); |
241 | } |
242 | |
243 | void dump_ebb_state(void) |
244 | { |
245 | dump_summary_ebb_state(); |
246 | |
247 | dump_ebb_hw_state(); |
248 | |
249 | trace_buffer_print(tb: ebb_state.trace); |
250 | } |
251 | |
252 | int count_pmc(int pmc, uint32_t sample_period) |
253 | { |
254 | uint32_t start_value; |
255 | u64 val; |
256 | |
257 | /* 0) Read PMC */ |
258 | start_value = pmc_sample_period(value: sample_period); |
259 | |
260 | val = read_pmc(pmc); |
261 | if (val < start_value) |
262 | ebb_state.stats.negative++; |
263 | else |
264 | ebb_state.stats.pmc_count[PMC_INDEX(pmc)] += val - start_value; |
265 | |
266 | trace_log_reg(ebb_state.trace, SPRN_PMC1 + pmc - 1, val); |
267 | |
268 | /* 1) Reset PMC */ |
269 | write_pmc(pmc, value: start_value); |
270 | |
271 | /* Report if we overflowed */ |
272 | return val >= COUNTER_OVERFLOW; |
273 | } |
274 | |
275 | int ebb_event_enable(struct event *e) |
276 | { |
277 | int rc; |
278 | |
279 | /* Ensure any SPR writes are ordered vs us */ |
280 | mb(); |
281 | |
282 | rc = ioctl(e->fd, PERF_EVENT_IOC_ENABLE); |
283 | if (rc) |
284 | return rc; |
285 | |
286 | rc = event_read(e); |
287 | |
288 | /* Ditto */ |
289 | mb(); |
290 | |
291 | return rc; |
292 | } |
293 | |
294 | void ebb_freeze_pmcs(void) |
295 | { |
296 | mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC); |
297 | mb(); |
298 | } |
299 | |
300 | void ebb_unfreeze_pmcs(void) |
301 | { |
302 | /* Unfreeze counters */ |
303 | mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC); |
304 | mb(); |
305 | } |
306 | |
307 | void ebb_global_enable(void) |
308 | { |
309 | /* Enable EBBs globally and PMU EBBs */ |
310 | mtspr(SPRN_BESCR, 0x8000000100000000ull); |
311 | mb(); |
312 | } |
313 | |
314 | void ebb_global_disable(void) |
315 | { |
316 | /* Disable EBBs & freeze counters, events are still scheduled */ |
317 | mtspr(SPRN_BESCRR, BESCR_PME); |
318 | mb(); |
319 | } |
320 | |
321 | bool ebb_is_supported(void) |
322 | { |
323 | #ifdef PPC_FEATURE2_EBB |
324 | /* EBB requires at least POWER8 */ |
325 | return have_hwcap2(PPC_FEATURE2_EBB); |
326 | #else |
327 | return false; |
328 | #endif |
329 | } |
330 | |
331 | void event_ebb_init(struct event *e) |
332 | { |
333 | e->attr.config |= (1ull << 63); |
334 | } |
335 | |
336 | void event_bhrb_init(struct event *e, unsigned ifm) |
337 | { |
338 | e->attr.config |= (1ull << 62) | ((u64)ifm << 60); |
339 | } |
340 | |
341 | void event_leader_ebb_init(struct event *e) |
342 | { |
343 | event_ebb_init(e); |
344 | |
345 | e->attr.exclusive = 1; |
346 | e->attr.pinned = 1; |
347 | } |
348 | |
349 | int ebb_child(union pipe read_pipe, union pipe write_pipe) |
350 | { |
351 | struct event event; |
352 | uint64_t val; |
353 | |
354 | FAIL_IF(wait_for_parent(read_pipe)); |
355 | |
356 | event_init_named(e: &event, config: 0x1001e, name: "cycles" ); |
357 | event_leader_ebb_init(e: &event); |
358 | |
359 | event.attr.exclude_kernel = 1; |
360 | event.attr.exclude_hv = 1; |
361 | event.attr.exclude_idle = 1; |
362 | |
363 | FAIL_IF(event_open(e: &event)); |
364 | |
365 | ebb_enable_pmc_counting(pmc: 1); |
366 | setup_ebb_handler(standard_ebb_callee); |
367 | ebb_global_enable(); |
368 | |
369 | FAIL_IF(event_enable(e: &event)); |
370 | |
371 | if (event_read(e: &event)) { |
372 | /* |
373 | * Some tests expect to fail here, so don't report an error on |
374 | * this line, and return a distinguisable error code. Tell the |
375 | * parent an error happened. |
376 | */ |
377 | notify_parent_of_error(write_pipe); |
378 | return 2; |
379 | } |
380 | |
381 | mtspr(SPRN_PMC1, pmc_sample_period(sample_period)); |
382 | |
383 | FAIL_IF(notify_parent(write_pipe)); |
384 | FAIL_IF(wait_for_parent(read_pipe)); |
385 | FAIL_IF(notify_parent(write_pipe)); |
386 | |
387 | while (ebb_state.stats.ebb_count < 20) { |
388 | FAIL_IF(core_busy_loop()); |
389 | |
390 | /* To try and hit SIGILL case */ |
391 | val = mfspr(SPRN_MMCRA); |
392 | val |= mfspr(SPRN_MMCR2); |
393 | val |= mfspr(SPRN_MMCR0); |
394 | } |
395 | |
396 | ebb_global_disable(); |
397 | ebb_freeze_pmcs(); |
398 | |
399 | dump_ebb_state(); |
400 | |
401 | event_close(e: &event); |
402 | |
403 | FAIL_IF(ebb_state.stats.ebb_count == 0); |
404 | |
405 | return 0; |
406 | } |
407 | |
408 | static jmp_buf setjmp_env; |
409 | |
410 | static void sigill_handler(int signal) |
411 | { |
412 | printf("Took sigill\n" ); |
413 | longjmp(setjmp_env, 1); |
414 | } |
415 | |
416 | static struct sigaction sigill_action = { |
417 | .sa_handler = sigill_handler, |
418 | }; |
419 | |
420 | int catch_sigill(void (*func)(void)) |
421 | { |
422 | if (sigaction(SIGILL, &sigill_action, NULL)) { |
423 | perror("sigaction" ); |
424 | return 1; |
425 | } |
426 | |
427 | if (setjmp(setjmp_env) == 0) { |
428 | func(); |
429 | return 1; |
430 | } |
431 | |
432 | return 0; |
433 | } |
434 | |
435 | void write_pmc1(void) |
436 | { |
437 | mtspr(SPRN_PMC1, 0); |
438 | } |
439 | |
440 | void write_pmc(int pmc, u64 value) |
441 | { |
442 | switch (pmc) { |
443 | case 1: mtspr(SPRN_PMC1, value); break; |
444 | case 2: mtspr(SPRN_PMC2, value); break; |
445 | case 3: mtspr(SPRN_PMC3, value); break; |
446 | case 4: mtspr(SPRN_PMC4, value); break; |
447 | case 5: mtspr(SPRN_PMC5, value); break; |
448 | case 6: mtspr(SPRN_PMC6, value); break; |
449 | } |
450 | } |
451 | |
452 | u64 read_pmc(int pmc) |
453 | { |
454 | switch (pmc) { |
455 | case 1: return mfspr(SPRN_PMC1); |
456 | case 2: return mfspr(SPRN_PMC2); |
457 | case 3: return mfspr(SPRN_PMC3); |
458 | case 4: return mfspr(SPRN_PMC4); |
459 | case 5: return mfspr(SPRN_PMC5); |
460 | case 6: return mfspr(SPRN_PMC6); |
461 | } |
462 | |
463 | return 0; |
464 | } |
465 | |
466 | static void term_handler(int signal) |
467 | { |
468 | dump_summary_ebb_state(); |
469 | dump_ebb_hw_state(); |
470 | abort(); |
471 | } |
472 | |
473 | struct sigaction term_action = { |
474 | .sa_handler = term_handler, |
475 | }; |
476 | |
477 | static void __attribute__((constructor)) ebb_init(void) |
478 | { |
479 | clear_ebb_stats(); |
480 | |
481 | if (sigaction(SIGTERM, &term_action, NULL)) |
482 | perror("sigaction" ); |
483 | |
484 | ebb_state.trace = trace_buffer_allocate(1 * 1024 * 1024); |
485 | } |
486 | |