1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright 2022, Athira Rajeev, IBM Corp. |
4 | * Copyright 2022, Madhavan Srinivasan, IBM Corp. |
5 | * Copyright 2022, Kajol Jain, IBM Corp. |
6 | */ |
7 | |
8 | #include <unistd.h> |
9 | #include <sys/syscall.h> |
10 | #include <string.h> |
11 | #include <stdio.h> |
12 | #include <sys/ioctl.h> |
13 | #include <sys/mman.h> |
14 | #include <stdlib.h> |
15 | #include <ctype.h> |
16 | |
17 | #include "misc.h" |
18 | |
19 | #define PAGE_SIZE sysconf(_SC_PAGESIZE) |
20 | |
21 | /* Storage for platform version */ |
22 | int pvr; |
23 | u64 platform_extended_mask; |
24 | |
25 | /* Mask and Shift for Event code fields */ |
26 | int ev_mask_pmcxsel, ev_shift_pmcxsel; //pmcxsel field |
27 | int ev_mask_marked, ev_shift_marked; //marked filed |
28 | int ev_mask_comb, ev_shift_comb; //combine field |
29 | int ev_mask_unit, ev_shift_unit; //unit field |
30 | int ev_mask_pmc, ev_shift_pmc; //pmc field |
31 | int ev_mask_cache, ev_shift_cache; //Cache sel field |
32 | int ev_mask_sample, ev_shift_sample; //Random sampling field |
33 | int ev_mask_thd_sel, ev_shift_thd_sel; //thresh_sel field |
34 | int ev_mask_thd_start, ev_shift_thd_start; //thresh_start field |
35 | int ev_mask_thd_stop, ev_shift_thd_stop; //thresh_stop field |
36 | int ev_mask_thd_cmp, ev_shift_thd_cmp; //thresh cmp field |
37 | int ev_mask_sm, ev_shift_sm; //SDAR mode field |
38 | int ev_mask_rsq, ev_shift_rsq; //radix scope qual field |
39 | int ev_mask_l2l3, ev_shift_l2l3; //l2l3 sel field |
40 | int ev_mask_mmcr3_src, ev_shift_mmcr3_src; //mmcr3 field |
41 | |
42 | static void init_ev_encodes(void) |
43 | { |
44 | ev_mask_pmcxsel = 0xff; |
45 | ev_shift_pmcxsel = 0; |
46 | ev_mask_marked = 1; |
47 | ev_shift_marked = 8; |
48 | ev_mask_unit = 0xf; |
49 | ev_shift_unit = 12; |
50 | ev_mask_pmc = 0xf; |
51 | ev_shift_pmc = 16; |
52 | ev_mask_sample = 0x1f; |
53 | ev_shift_sample = 24; |
54 | ev_mask_thd_sel = 0x7; |
55 | ev_shift_thd_sel = 29; |
56 | ev_mask_thd_start = 0xf; |
57 | ev_shift_thd_start = 36; |
58 | ev_mask_thd_stop = 0xf; |
59 | ev_shift_thd_stop = 32; |
60 | |
61 | switch (pvr) { |
62 | case POWER10: |
63 | ev_mask_thd_cmp = 0x3ffff; |
64 | ev_shift_thd_cmp = 0; |
65 | ev_mask_rsq = 1; |
66 | ev_shift_rsq = 9; |
67 | ev_mask_comb = 3; |
68 | ev_shift_comb = 10; |
69 | ev_mask_cache = 3; |
70 | ev_shift_cache = 20; |
71 | ev_mask_sm = 0x3; |
72 | ev_shift_sm = 22; |
73 | ev_mask_l2l3 = 0x1f; |
74 | ev_shift_l2l3 = 40; |
75 | ev_mask_mmcr3_src = 0x7fff; |
76 | ev_shift_mmcr3_src = 45; |
77 | break; |
78 | case POWER9: |
79 | ev_mask_comb = 3; |
80 | ev_shift_comb = 10; |
81 | ev_mask_cache = 0xf; |
82 | ev_shift_cache = 20; |
83 | ev_mask_thd_cmp = 0x3ff; |
84 | ev_shift_thd_cmp = 40; |
85 | ev_mask_sm = 0x3; |
86 | ev_shift_sm = 50; |
87 | break; |
88 | default: |
89 | FAIL_IF_EXIT(1); |
90 | } |
91 | } |
92 | |
93 | /* Return the extended regs mask value */ |
94 | static u64 perf_get_platform_reg_mask(void) |
95 | { |
96 | if (have_hwcap2(PPC_FEATURE2_ARCH_3_1)) |
97 | return PERF_POWER10_MASK; |
98 | if (have_hwcap2(PPC_FEATURE2_ARCH_3_00)) |
99 | return PERF_POWER9_MASK; |
100 | |
101 | return -1; |
102 | } |
103 | |
104 | int check_extended_regs_support(void) |
105 | { |
106 | int fd; |
107 | struct event event; |
108 | |
109 | event_init(e: &event, config: 0x1001e); |
110 | |
111 | event.attr.type = 4; |
112 | event.attr.sample_period = 1; |
113 | event.attr.disabled = 1; |
114 | event.attr.sample_type = PERF_SAMPLE_REGS_INTR; |
115 | event.attr.sample_regs_intr = platform_extended_mask; |
116 | |
117 | fd = event_open(e: &event); |
118 | if (fd != -1) |
119 | return 0; |
120 | |
121 | return -1; |
122 | } |
123 | |
124 | int platform_check_for_tests(void) |
125 | { |
126 | pvr = PVR_VER(mfspr(SPRN_PVR)); |
127 | |
128 | /* |
129 | * Check for supported platforms |
130 | * for sampling test |
131 | */ |
132 | if ((pvr != POWER10) && (pvr != POWER9)) |
133 | goto out; |
134 | |
135 | /* |
136 | * Check PMU driver registered by looking for |
137 | * PPC_FEATURE2_EBB bit in AT_HWCAP2 |
138 | */ |
139 | if (!have_hwcap2(PPC_FEATURE2_EBB) || !have_hwcap2(PPC_FEATURE2_ARCH_3_00)) |
140 | goto out; |
141 | |
142 | return 0; |
143 | |
144 | out: |
145 | printf("%s: Tests unsupported for this platform\n" , __func__); |
146 | return -1; |
147 | } |
148 | |
149 | int check_pvr_for_sampling_tests(void) |
150 | { |
151 | SKIP_IF(platform_check_for_tests()); |
152 | |
153 | platform_extended_mask = perf_get_platform_reg_mask(); |
154 | /* check if platform supports extended regs */ |
155 | if (check_extended_regs_support()) |
156 | goto out; |
157 | |
158 | init_ev_encodes(); |
159 | return 0; |
160 | |
161 | out: |
162 | printf("%s: Sampling tests un-supported\n" , __func__); |
163 | return -1; |
164 | } |
165 | |
166 | /* |
167 | * Allocate mmap buffer of "mmap_pages" number of |
168 | * pages. |
169 | */ |
170 | void *event_sample_buf_mmap(int fd, int mmap_pages) |
171 | { |
172 | size_t page_size = sysconf(_SC_PAGESIZE); |
173 | size_t mmap_size; |
174 | void *buff; |
175 | |
176 | if (mmap_pages <= 0) |
177 | return NULL; |
178 | |
179 | if (fd <= 0) |
180 | return NULL; |
181 | |
182 | mmap_size = page_size * (1 + mmap_pages); |
183 | buff = mmap(NULL, mmap_size, |
184 | PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); |
185 | |
186 | if (buff == MAP_FAILED) { |
187 | perror("mmap() failed." ); |
188 | return NULL; |
189 | } |
190 | return buff; |
191 | } |
192 | |
193 | /* |
194 | * Post process the mmap buffer. |
195 | * - If sample_count != NULL then return count of total |
196 | * number of samples present in the mmap buffer. |
197 | * - If sample_count == NULL then return the address |
198 | * of first sample from the mmap buffer |
199 | */ |
200 | void *__event_read_samples(void *sample_buff, size_t *size, u64 *sample_count) |
201 | { |
202 | size_t page_size = sysconf(_SC_PAGESIZE); |
203 | struct perf_event_header * = sample_buff + page_size; |
204 | struct perf_event_mmap_page *metadata_page = sample_buff; |
205 | unsigned long data_head, data_tail; |
206 | |
207 | /* |
208 | * PERF_RECORD_SAMPLE: |
209 | * struct { |
210 | * struct perf_event_header hdr; |
211 | * u64 data[]; |
212 | * }; |
213 | */ |
214 | |
215 | data_head = metadata_page->data_head; |
216 | /* sync memory before reading sample */ |
217 | mb(); |
218 | data_tail = metadata_page->data_tail; |
219 | |
220 | /* Check for sample_count */ |
221 | if (sample_count) |
222 | *sample_count = 0; |
223 | |
224 | while (1) { |
225 | /* |
226 | * Reads the mmap data buffer by moving |
227 | * the data_tail to know the last read data. |
228 | * data_head points to head in data buffer. |
229 | * refer "struct perf_event_mmap_page" in |
230 | * "include/uapi/linux/perf_event.h". |
231 | */ |
232 | if (data_head - data_tail < sizeof(header)) |
233 | return NULL; |
234 | |
235 | data_tail += sizeof(header); |
236 | if (header->type == PERF_RECORD_SAMPLE) { |
237 | *size = (header->size - sizeof(header)); |
238 | if (!sample_count) |
239 | return sample_buff + page_size + data_tail; |
240 | data_tail += *size; |
241 | *sample_count += 1; |
242 | } else { |
243 | *size = (header->size - sizeof(header)); |
244 | if ((metadata_page->data_tail + *size) > metadata_page->data_head) |
245 | data_tail = metadata_page->data_head; |
246 | else |
247 | data_tail += *size; |
248 | } |
249 | header = (struct perf_event_header *)((void *)header + header->size); |
250 | } |
251 | return NULL; |
252 | } |
253 | |
254 | int collect_samples(void *sample_buff) |
255 | { |
256 | u64 sample_count; |
257 | size_t size = 0; |
258 | |
259 | __event_read_samples(sample_buff, size: &size, sample_count: &sample_count); |
260 | return sample_count; |
261 | } |
262 | |
263 | static void *perf_read_first_sample(void *sample_buff, size_t *size) |
264 | { |
265 | return __event_read_samples(sample_buff, size, NULL); |
266 | } |
267 | |
268 | u64 *get_intr_regs(struct event *event, void *sample_buff) |
269 | { |
270 | u64 type = event->attr.sample_type; |
271 | u64 *intr_regs; |
272 | size_t size = 0; |
273 | |
274 | if ((type ^ (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_BRANCH_STACK)) && |
275 | (type ^ PERF_SAMPLE_REGS_INTR)) |
276 | return NULL; |
277 | |
278 | intr_regs = (u64 *)perf_read_first_sample(sample_buff, size: &size); |
279 | if (!intr_regs) |
280 | return NULL; |
281 | |
282 | if (type & PERF_SAMPLE_BRANCH_STACK) { |
283 | /* |
284 | * PERF_RECORD_SAMPLE and PERF_SAMPLE_BRANCH_STACK: |
285 | * struct { |
286 | * struct perf_event_header hdr; |
287 | * u64 number_of_branches; |
288 | * struct perf_branch_entry[number_of_branches]; |
289 | * u64 data[]; |
290 | * }; |
291 | * struct perf_branch_entry { |
292 | * u64 from; |
293 | * u64 to; |
294 | * u64 misc; |
295 | * }; |
296 | */ |
297 | intr_regs += ((*intr_regs) * 3) + 1; |
298 | } |
299 | |
300 | /* |
301 | * First entry in the sample buffer used to specify |
302 | * PERF_SAMPLE_REGS_ABI_64, skip perf regs abi to access |
303 | * interrupt registers. |
304 | */ |
305 | ++intr_regs; |
306 | |
307 | return intr_regs; |
308 | } |
309 | |
310 | static const int __perf_reg_mask(const char *register_name) |
311 | { |
312 | if (!strcmp(register_name, "R0" )) |
313 | return 0; |
314 | else if (!strcmp(register_name, "R1" )) |
315 | return 1; |
316 | else if (!strcmp(register_name, "R2" )) |
317 | return 2; |
318 | else if (!strcmp(register_name, "R3" )) |
319 | return 3; |
320 | else if (!strcmp(register_name, "R4" )) |
321 | return 4; |
322 | else if (!strcmp(register_name, "R5" )) |
323 | return 5; |
324 | else if (!strcmp(register_name, "R6" )) |
325 | return 6; |
326 | else if (!strcmp(register_name, "R7" )) |
327 | return 7; |
328 | else if (!strcmp(register_name, "R8" )) |
329 | return 8; |
330 | else if (!strcmp(register_name, "R9" )) |
331 | return 9; |
332 | else if (!strcmp(register_name, "R10" )) |
333 | return 10; |
334 | else if (!strcmp(register_name, "R11" )) |
335 | return 11; |
336 | else if (!strcmp(register_name, "R12" )) |
337 | return 12; |
338 | else if (!strcmp(register_name, "R13" )) |
339 | return 13; |
340 | else if (!strcmp(register_name, "R14" )) |
341 | return 14; |
342 | else if (!strcmp(register_name, "R15" )) |
343 | return 15; |
344 | else if (!strcmp(register_name, "R16" )) |
345 | return 16; |
346 | else if (!strcmp(register_name, "R17" )) |
347 | return 17; |
348 | else if (!strcmp(register_name, "R18" )) |
349 | return 18; |
350 | else if (!strcmp(register_name, "R19" )) |
351 | return 19; |
352 | else if (!strcmp(register_name, "R20" )) |
353 | return 20; |
354 | else if (!strcmp(register_name, "R21" )) |
355 | return 21; |
356 | else if (!strcmp(register_name, "R22" )) |
357 | return 22; |
358 | else if (!strcmp(register_name, "R23" )) |
359 | return 23; |
360 | else if (!strcmp(register_name, "R24" )) |
361 | return 24; |
362 | else if (!strcmp(register_name, "R25" )) |
363 | return 25; |
364 | else if (!strcmp(register_name, "R26" )) |
365 | return 26; |
366 | else if (!strcmp(register_name, "R27" )) |
367 | return 27; |
368 | else if (!strcmp(register_name, "R28" )) |
369 | return 28; |
370 | else if (!strcmp(register_name, "R29" )) |
371 | return 29; |
372 | else if (!strcmp(register_name, "R30" )) |
373 | return 30; |
374 | else if (!strcmp(register_name, "R31" )) |
375 | return 31; |
376 | else if (!strcmp(register_name, "NIP" )) |
377 | return 32; |
378 | else if (!strcmp(register_name, "MSR" )) |
379 | return 33; |
380 | else if (!strcmp(register_name, "ORIG_R3" )) |
381 | return 34; |
382 | else if (!strcmp(register_name, "CTR" )) |
383 | return 35; |
384 | else if (!strcmp(register_name, "LINK" )) |
385 | return 36; |
386 | else if (!strcmp(register_name, "XER" )) |
387 | return 37; |
388 | else if (!strcmp(register_name, "CCR" )) |
389 | return 38; |
390 | else if (!strcmp(register_name, "SOFTE" )) |
391 | return 39; |
392 | else if (!strcmp(register_name, "TRAP" )) |
393 | return 40; |
394 | else if (!strcmp(register_name, "DAR" )) |
395 | return 41; |
396 | else if (!strcmp(register_name, "DSISR" )) |
397 | return 42; |
398 | else if (!strcmp(register_name, "SIER" )) |
399 | return 43; |
400 | else if (!strcmp(register_name, "MMCRA" )) |
401 | return 44; |
402 | else if (!strcmp(register_name, "MMCR0" )) |
403 | return 45; |
404 | else if (!strcmp(register_name, "MMCR1" )) |
405 | return 46; |
406 | else if (!strcmp(register_name, "MMCR2" )) |
407 | return 47; |
408 | else if (!strcmp(register_name, "MMCR3" )) |
409 | return 48; |
410 | else if (!strcmp(register_name, "SIER2" )) |
411 | return 49; |
412 | else if (!strcmp(register_name, "SIER3" )) |
413 | return 50; |
414 | else if (!strcmp(register_name, "PMC1" )) |
415 | return 51; |
416 | else if (!strcmp(register_name, "PMC2" )) |
417 | return 52; |
418 | else if (!strcmp(register_name, "PMC3" )) |
419 | return 53; |
420 | else if (!strcmp(register_name, "PMC4" )) |
421 | return 54; |
422 | else if (!strcmp(register_name, "PMC5" )) |
423 | return 55; |
424 | else if (!strcmp(register_name, "PMC6" )) |
425 | return 56; |
426 | else if (!strcmp(register_name, "SDAR" )) |
427 | return 57; |
428 | else if (!strcmp(register_name, "SIAR" )) |
429 | return 58; |
430 | else |
431 | return -1; |
432 | } |
433 | |
434 | u64 get_reg_value(u64 *intr_regs, char *register_name) |
435 | { |
436 | int register_bit_position; |
437 | |
438 | register_bit_position = __perf_reg_mask(register_name); |
439 | |
440 | if (register_bit_position < 0 || (!((platform_extended_mask >> |
441 | (register_bit_position - 1)) & 1))) |
442 | return -1; |
443 | |
444 | return *(intr_regs + register_bit_position); |
445 | } |
446 | |
447 | int get_thresh_cmp_val(struct event event) |
448 | { |
449 | int exp = 0; |
450 | u64 result = 0; |
451 | u64 value; |
452 | |
453 | if (!have_hwcap2(PPC_FEATURE2_ARCH_3_1)) |
454 | return EV_CODE_EXTRACT(event.attr.config, thd_cmp); |
455 | |
456 | value = EV_CODE_EXTRACT(event.attr.config1, thd_cmp); |
457 | |
458 | if (!value) |
459 | return value; |
460 | |
461 | /* |
462 | * Incase of P10, thresh_cmp value is not part of raw event code |
463 | * and provided via attr.config1 parameter. To program threshold in MMCRA, |
464 | * take a 18 bit number N and shift right 2 places and increment |
465 | * the exponent E by 1 until the upper 10 bits of N are zero. |
466 | * Write E to the threshold exponent and write the lower 8 bits of N |
467 | * to the threshold mantissa. |
468 | * The max threshold that can be written is 261120. |
469 | */ |
470 | if (value > 261120) |
471 | value = 261120; |
472 | while ((64 - __builtin_clzl(value)) > 8) { |
473 | exp++; |
474 | value >>= 2; |
475 | } |
476 | |
477 | /* |
478 | * Note that it is invalid to write a mantissa with the |
479 | * upper 2 bits of mantissa being zero, unless the |
480 | * exponent is also zero. |
481 | */ |
482 | if (!(value & 0xC0) && exp) |
483 | result = -1; |
484 | else |
485 | result = (exp << 8) | value; |
486 | return result; |
487 | } |
488 | |
489 | /* |
490 | * Utility function to check for generic compat PMU |
491 | * by comparing base_platform value from auxv and real |
492 | * PVR value. |
493 | */ |
494 | static bool auxv_generic_compat_pmu(void) |
495 | { |
496 | int base_pvr = 0; |
497 | |
498 | if (!strcmp(auxv_base_platform(), "power9" )) |
499 | base_pvr = POWER9; |
500 | else if (!strcmp(auxv_base_platform(), "power10" )) |
501 | base_pvr = POWER10; |
502 | |
503 | return (!base_pvr); |
504 | } |
505 | |
506 | /* |
507 | * Check for generic compat PMU. |
508 | * First check for presence of pmu_name from |
509 | * "/sys/bus/event_source/devices/cpu/caps". |
510 | * If doesn't exist, fallback to using value |
511 | * auxv. |
512 | */ |
513 | bool check_for_generic_compat_pmu(void) |
514 | { |
515 | char pmu_name[256]; |
516 | |
517 | memset(pmu_name, 0, sizeof(pmu_name)); |
518 | if (read_sysfs_file("bus/event_source/devices/cpu/caps/pmu_name" , |
519 | pmu_name, sizeof(pmu_name)) < 0) |
520 | return auxv_generic_compat_pmu(); |
521 | |
522 | if (!strcmp(pmu_name, "ISAv3" )) |
523 | return true; |
524 | else |
525 | return false; |
526 | } |
527 | |
528 | /* |
529 | * Check if system is booted in compat mode. |
530 | */ |
531 | bool check_for_compat_mode(void) |
532 | { |
533 | char *platform = auxv_platform(); |
534 | char *base_platform = auxv_base_platform(); |
535 | |
536 | return strcmp(platform, base_platform); |
537 | } |
538 | |