1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * intel_pt.c: Intel Processor Trace support |
4 | * Copyright (c) 2013-2015, Intel Corporation. |
5 | */ |
6 | |
7 | #include <errno.h> |
8 | #include <stdbool.h> |
9 | #include <linux/kernel.h> |
10 | #include <linux/types.h> |
11 | #include <linux/bitops.h> |
12 | #include <linux/log2.h> |
13 | #include <linux/zalloc.h> |
14 | #include <linux/err.h> |
15 | #include <cpuid.h> |
16 | |
17 | #include "../../../util/session.h" |
18 | #include "../../../util/event.h" |
19 | #include "../../../util/evlist.h" |
20 | #include "../../../util/evsel.h" |
21 | #include "../../../util/evsel_config.h" |
22 | #include "../../../util/cpumap.h" |
23 | #include "../../../util/mmap.h" |
24 | #include <subcmd/parse-options.h> |
25 | #include "../../../util/parse-events.h" |
26 | #include "../../../util/pmus.h" |
27 | #include "../../../util/debug.h" |
28 | #include "../../../util/auxtrace.h" |
29 | #include "../../../util/perf_api_probe.h" |
30 | #include "../../../util/record.h" |
31 | #include "../../../util/target.h" |
32 | #include "../../../util/tsc.h" |
33 | #include <internal/lib.h> // page_size |
34 | #include "../../../util/intel-pt.h" |
35 | |
36 | #define KiB(x) ((x) * 1024) |
37 | #define MiB(x) ((x) * 1024 * 1024) |
38 | #define KiB_MASK(x) (KiB(x) - 1) |
39 | #define MiB_MASK(x) (MiB(x) - 1) |
40 | |
41 | #define INTEL_PT_PSB_PERIOD_NEAR 256 |
42 | |
43 | struct intel_pt_snapshot_ref { |
44 | void *ref_buf; |
45 | size_t ref_offset; |
46 | bool wrapped; |
47 | }; |
48 | |
49 | struct intel_pt_recording { |
50 | struct auxtrace_record itr; |
51 | struct perf_pmu *intel_pt_pmu; |
52 | int have_sched_switch; |
53 | struct evlist *evlist; |
54 | bool snapshot_mode; |
55 | bool snapshot_init_done; |
56 | size_t snapshot_size; |
57 | size_t snapshot_ref_buf_size; |
58 | int snapshot_ref_cnt; |
59 | struct intel_pt_snapshot_ref *snapshot_refs; |
60 | size_t priv_size; |
61 | }; |
62 | |
63 | static int intel_pt_parse_terms_with_default(const struct perf_pmu *pmu, |
64 | const char *str, |
65 | u64 *config) |
66 | { |
67 | struct parse_events_terms terms; |
68 | struct perf_event_attr attr = { .size = 0, }; |
69 | int err; |
70 | |
71 | parse_events_terms__init(terms: &terms); |
72 | err = parse_events_terms(&terms, str, /*input=*/ NULL); |
73 | if (err) |
74 | goto out_free; |
75 | |
76 | attr.config = *config; |
77 | err = perf_pmu__config_terms(pmu, attr: &attr, terms: &terms, /*zero=*/true, /*err=*/NULL); |
78 | if (err) |
79 | goto out_free; |
80 | |
81 | *config = attr.config; |
82 | out_free: |
83 | parse_events_terms__exit(terms: &terms); |
84 | return err; |
85 | } |
86 | |
87 | static int intel_pt_parse_terms(const struct perf_pmu *pmu, const char *str, u64 *config) |
88 | { |
89 | *config = 0; |
90 | return intel_pt_parse_terms_with_default(pmu, str, config); |
91 | } |
92 | |
93 | static u64 intel_pt_masked_bits(u64 mask, u64 bits) |
94 | { |
95 | const u64 top_bit = 1ULL << 63; |
96 | u64 res = 0; |
97 | int i; |
98 | |
99 | for (i = 0; i < 64; i++) { |
100 | if (mask & top_bit) { |
101 | res <<= 1; |
102 | if (bits & top_bit) |
103 | res |= 1; |
104 | } |
105 | mask <<= 1; |
106 | bits <<= 1; |
107 | } |
108 | |
109 | return res; |
110 | } |
111 | |
112 | static int intel_pt_read_config(struct perf_pmu *intel_pt_pmu, const char *str, |
113 | struct evlist *evlist, u64 *res) |
114 | { |
115 | struct evsel *evsel; |
116 | u64 mask; |
117 | |
118 | *res = 0; |
119 | |
120 | mask = perf_pmu__format_bits(pmu: intel_pt_pmu, name: str); |
121 | if (!mask) |
122 | return -EINVAL; |
123 | |
124 | evlist__for_each_entry(evlist, evsel) { |
125 | if (evsel->core.attr.type == intel_pt_pmu->type) { |
126 | *res = intel_pt_masked_bits(mask, bits: evsel->core.attr.config); |
127 | return 0; |
128 | } |
129 | } |
130 | |
131 | return -EINVAL; |
132 | } |
133 | |
134 | static size_t intel_pt_psb_period(struct perf_pmu *intel_pt_pmu, |
135 | struct evlist *evlist) |
136 | { |
137 | u64 val; |
138 | int err, topa_multiple_entries; |
139 | size_t psb_period; |
140 | |
141 | if (perf_pmu__scan_file(pmu: intel_pt_pmu, name: "caps/topa_multiple_entries" , |
142 | fmt: "%d" , &topa_multiple_entries) != 1) |
143 | topa_multiple_entries = 0; |
144 | |
145 | /* |
146 | * Use caps/topa_multiple_entries to indicate early hardware that had |
147 | * extra frequent PSBs. |
148 | */ |
149 | if (!topa_multiple_entries) { |
150 | psb_period = 256; |
151 | goto out; |
152 | } |
153 | |
154 | err = intel_pt_read_config(intel_pt_pmu, str: "psb_period" , evlist, res: &val); |
155 | if (err) |
156 | val = 0; |
157 | |
158 | psb_period = 1 << (val + 11); |
159 | out: |
160 | pr_debug2("%s psb_period %zu\n" , intel_pt_pmu->name, psb_period); |
161 | return psb_period; |
162 | } |
163 | |
164 | static int intel_pt_pick_bit(int bits, int target) |
165 | { |
166 | int pos, pick = -1; |
167 | |
168 | for (pos = 0; bits; bits >>= 1, pos++) { |
169 | if (bits & 1) { |
170 | if (pos <= target || pick < 0) |
171 | pick = pos; |
172 | if (pos >= target) |
173 | break; |
174 | } |
175 | } |
176 | |
177 | return pick; |
178 | } |
179 | |
180 | static u64 intel_pt_default_config(const struct perf_pmu *intel_pt_pmu) |
181 | { |
182 | char buf[256]; |
183 | int mtc, mtc_periods = 0, mtc_period; |
184 | int psb_cyc, psb_periods, psb_period; |
185 | int pos = 0; |
186 | u64 config; |
187 | char c; |
188 | int dirfd; |
189 | |
190 | dirfd = perf_pmu__event_source_devices_fd(); |
191 | |
192 | pos += scnprintf(buf: buf + pos, size: sizeof(buf) - pos, fmt: "tsc" ); |
193 | |
194 | if (perf_pmu__scan_file_at(pmu: intel_pt_pmu, dirfd, name: "caps/mtc" , fmt: "%d" , |
195 | &mtc) != 1) |
196 | mtc = 1; |
197 | |
198 | if (mtc) { |
199 | if (perf_pmu__scan_file_at(pmu: intel_pt_pmu, dirfd, name: "caps/mtc_periods" , fmt: "%x" , |
200 | &mtc_periods) != 1) |
201 | mtc_periods = 0; |
202 | if (mtc_periods) { |
203 | mtc_period = intel_pt_pick_bit(bits: mtc_periods, target: 3); |
204 | pos += scnprintf(buf: buf + pos, size: sizeof(buf) - pos, |
205 | fmt: ",mtc,mtc_period=%d" , mtc_period); |
206 | } |
207 | } |
208 | |
209 | if (perf_pmu__scan_file_at(pmu: intel_pt_pmu, dirfd, name: "caps/psb_cyc" , fmt: "%d" , |
210 | &psb_cyc) != 1) |
211 | psb_cyc = 1; |
212 | |
213 | if (psb_cyc && mtc_periods) { |
214 | if (perf_pmu__scan_file_at(pmu: intel_pt_pmu, dirfd, name: "caps/psb_periods" , fmt: "%x" , |
215 | &psb_periods) != 1) |
216 | psb_periods = 0; |
217 | if (psb_periods) { |
218 | psb_period = intel_pt_pick_bit(bits: psb_periods, target: 3); |
219 | pos += scnprintf(buf: buf + pos, size: sizeof(buf) - pos, |
220 | fmt: ",psb_period=%d" , psb_period); |
221 | } |
222 | } |
223 | |
224 | if (perf_pmu__scan_file_at(pmu: intel_pt_pmu, dirfd, name: "format/pt" , fmt: "%c" , &c) == 1 && |
225 | perf_pmu__scan_file_at(pmu: intel_pt_pmu, dirfd, name: "format/branch" , fmt: "%c" , &c) == 1) |
226 | pos += scnprintf(buf: buf + pos, size: sizeof(buf) - pos, fmt: ",pt,branch" ); |
227 | |
228 | pr_debug2("%s default config: %s\n" , intel_pt_pmu->name, buf); |
229 | |
230 | intel_pt_parse_terms(pmu: intel_pt_pmu, str: buf, config: &config); |
231 | |
232 | close(dirfd); |
233 | return config; |
234 | } |
235 | |
236 | static int intel_pt_parse_snapshot_options(struct auxtrace_record *itr, |
237 | struct record_opts *opts, |
238 | const char *str) |
239 | { |
240 | struct intel_pt_recording *ptr = |
241 | container_of(itr, struct intel_pt_recording, itr); |
242 | unsigned long long snapshot_size = 0; |
243 | char *endptr; |
244 | |
245 | if (str) { |
246 | snapshot_size = strtoull(str, &endptr, 0); |
247 | if (*endptr || snapshot_size > SIZE_MAX) |
248 | return -1; |
249 | } |
250 | |
251 | opts->auxtrace_snapshot_mode = true; |
252 | opts->auxtrace_snapshot_size = snapshot_size; |
253 | |
254 | ptr->snapshot_size = snapshot_size; |
255 | |
256 | return 0; |
257 | } |
258 | |
259 | void intel_pt_pmu_default_config(const struct perf_pmu *intel_pt_pmu, |
260 | struct perf_event_attr *attr) |
261 | { |
262 | static u64 config; |
263 | static bool initialized; |
264 | |
265 | if (!initialized) { |
266 | config = intel_pt_default_config(intel_pt_pmu); |
267 | initialized = true; |
268 | } |
269 | attr->config = config; |
270 | } |
271 | |
272 | static const char *intel_pt_find_filter(struct evlist *evlist, |
273 | struct perf_pmu *intel_pt_pmu) |
274 | { |
275 | struct evsel *evsel; |
276 | |
277 | evlist__for_each_entry(evlist, evsel) { |
278 | if (evsel->core.attr.type == intel_pt_pmu->type) |
279 | return evsel->filter; |
280 | } |
281 | |
282 | return NULL; |
283 | } |
284 | |
285 | static size_t intel_pt_filter_bytes(const char *filter) |
286 | { |
287 | size_t len = filter ? strlen(filter) : 0; |
288 | |
289 | return len ? roundup(len + 1, 8) : 0; |
290 | } |
291 | |
292 | static size_t |
293 | intel_pt_info_priv_size(struct auxtrace_record *itr, struct evlist *evlist) |
294 | { |
295 | struct intel_pt_recording *ptr = |
296 | container_of(itr, struct intel_pt_recording, itr); |
297 | const char *filter = intel_pt_find_filter(evlist, intel_pt_pmu: ptr->intel_pt_pmu); |
298 | |
299 | ptr->priv_size = (INTEL_PT_AUXTRACE_PRIV_MAX * sizeof(u64)) + |
300 | intel_pt_filter_bytes(filter); |
301 | ptr->priv_size += sizeof(u64); /* Cap Event Trace */ |
302 | |
303 | return ptr->priv_size; |
304 | } |
305 | |
306 | static void intel_pt_tsc_ctc_ratio(u32 *n, u32 *d) |
307 | { |
308 | unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; |
309 | |
310 | __get_cpuid(0x15, &eax, &ebx, &ecx, &edx); |
311 | *n = ebx; |
312 | *d = eax; |
313 | } |
314 | |
315 | static int intel_pt_info_fill(struct auxtrace_record *itr, |
316 | struct perf_session *session, |
317 | struct perf_record_auxtrace_info *auxtrace_info, |
318 | size_t priv_size) |
319 | { |
320 | struct intel_pt_recording *ptr = |
321 | container_of(itr, struct intel_pt_recording, itr); |
322 | struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu; |
323 | struct perf_event_mmap_page *pc; |
324 | struct perf_tsc_conversion tc = { .time_mult = 0, }; |
325 | bool cap_user_time_zero = false, per_cpu_mmaps; |
326 | u64 tsc_bit, mtc_bit, mtc_freq_bits, cyc_bit, noretcomp_bit; |
327 | u32 tsc_ctc_ratio_n, tsc_ctc_ratio_d; |
328 | unsigned long max_non_turbo_ratio; |
329 | size_t filter_str_len; |
330 | const char *filter; |
331 | int event_trace; |
332 | __u64 *info; |
333 | int err; |
334 | |
335 | if (priv_size != ptr->priv_size) |
336 | return -EINVAL; |
337 | |
338 | intel_pt_parse_terms(pmu: intel_pt_pmu, str: "tsc" , config: &tsc_bit); |
339 | intel_pt_parse_terms(pmu: intel_pt_pmu, str: "noretcomp" , config: &noretcomp_bit); |
340 | intel_pt_parse_terms(pmu: intel_pt_pmu, str: "mtc" , config: &mtc_bit); |
341 | mtc_freq_bits = perf_pmu__format_bits(pmu: intel_pt_pmu, name: "mtc_period" ); |
342 | intel_pt_parse_terms(pmu: intel_pt_pmu, str: "cyc" , config: &cyc_bit); |
343 | |
344 | intel_pt_tsc_ctc_ratio(n: &tsc_ctc_ratio_n, d: &tsc_ctc_ratio_d); |
345 | |
346 | if (perf_pmu__scan_file(pmu: intel_pt_pmu, name: "max_nonturbo_ratio" , |
347 | fmt: "%lu" , &max_non_turbo_ratio) != 1) |
348 | max_non_turbo_ratio = 0; |
349 | if (perf_pmu__scan_file(pmu: intel_pt_pmu, name: "caps/event_trace" , |
350 | fmt: "%d" , &event_trace) != 1) |
351 | event_trace = 0; |
352 | |
353 | filter = intel_pt_find_filter(evlist: session->evlist, intel_pt_pmu: ptr->intel_pt_pmu); |
354 | filter_str_len = filter ? strlen(filter) : 0; |
355 | |
356 | if (!session->evlist->core.nr_mmaps) |
357 | return -EINVAL; |
358 | |
359 | pc = session->evlist->mmap[0].core.base; |
360 | if (pc) { |
361 | err = perf_read_tsc_conversion(pc, tc: &tc); |
362 | if (err) { |
363 | if (err != -EOPNOTSUPP) |
364 | return err; |
365 | } else { |
366 | cap_user_time_zero = tc.time_mult != 0; |
367 | } |
368 | if (!cap_user_time_zero) |
369 | ui__warning(format: "Intel Processor Trace: TSC not available\n" ); |
370 | } |
371 | |
372 | per_cpu_mmaps = !perf_cpu_map__has_any_cpu_or_is_empty(session->evlist->core.user_requested_cpus); |
373 | |
374 | auxtrace_info->type = PERF_AUXTRACE_INTEL_PT; |
375 | auxtrace_info->priv[INTEL_PT_PMU_TYPE] = intel_pt_pmu->type; |
376 | auxtrace_info->priv[INTEL_PT_TIME_SHIFT] = tc.time_shift; |
377 | auxtrace_info->priv[INTEL_PT_TIME_MULT] = tc.time_mult; |
378 | auxtrace_info->priv[INTEL_PT_TIME_ZERO] = tc.time_zero; |
379 | auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO] = cap_user_time_zero; |
380 | auxtrace_info->priv[INTEL_PT_TSC_BIT] = tsc_bit; |
381 | auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT] = noretcomp_bit; |
382 | auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH] = ptr->have_sched_switch; |
383 | auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE] = ptr->snapshot_mode; |
384 | auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS] = per_cpu_mmaps; |
385 | auxtrace_info->priv[INTEL_PT_MTC_BIT] = mtc_bit; |
386 | auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS] = mtc_freq_bits; |
387 | auxtrace_info->priv[INTEL_PT_TSC_CTC_N] = tsc_ctc_ratio_n; |
388 | auxtrace_info->priv[INTEL_PT_TSC_CTC_D] = tsc_ctc_ratio_d; |
389 | auxtrace_info->priv[INTEL_PT_CYC_BIT] = cyc_bit; |
390 | auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO] = max_non_turbo_ratio; |
391 | auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] = filter_str_len; |
392 | |
393 | info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1; |
394 | |
395 | if (filter_str_len) { |
396 | size_t len = intel_pt_filter_bytes(filter); |
397 | |
398 | strncpy(p: (char *)info, q: filter, size: len); |
399 | info += len >> 3; |
400 | } |
401 | |
402 | *info++ = event_trace; |
403 | |
404 | return 0; |
405 | } |
406 | |
407 | #ifdef HAVE_LIBTRACEEVENT |
408 | static int intel_pt_track_switches(struct evlist *evlist) |
409 | { |
410 | const char *sched_switch = "sched:sched_switch" ; |
411 | struct evsel *evsel; |
412 | int err; |
413 | |
414 | if (!evlist__can_select_event(evlist, sched_switch)) |
415 | return -EPERM; |
416 | |
417 | evsel = evlist__add_sched_switch(evlist, true); |
418 | if (IS_ERR(evsel)) { |
419 | err = PTR_ERR(evsel); |
420 | pr_debug2("%s: failed to create %s, error = %d\n" , |
421 | __func__, sched_switch, err); |
422 | return err; |
423 | } |
424 | |
425 | evsel->immediate = true; |
426 | |
427 | return 0; |
428 | } |
429 | #endif |
430 | |
431 | static void intel_pt_valid_str(char *str, size_t len, u64 valid) |
432 | { |
433 | unsigned int val, last = 0, state = 1; |
434 | int p = 0; |
435 | |
436 | str[0] = '\0'; |
437 | |
438 | for (val = 0; val <= 64; val++, valid >>= 1) { |
439 | if (valid & 1) { |
440 | last = val; |
441 | switch (state) { |
442 | case 0: |
443 | p += scnprintf(buf: str + p, size: len - p, fmt: "," ); |
444 | /* Fall through */ |
445 | case 1: |
446 | p += scnprintf(buf: str + p, size: len - p, fmt: "%u" , val); |
447 | state = 2; |
448 | break; |
449 | case 2: |
450 | state = 3; |
451 | break; |
452 | case 3: |
453 | state = 4; |
454 | break; |
455 | default: |
456 | break; |
457 | } |
458 | } else { |
459 | switch (state) { |
460 | case 3: |
461 | p += scnprintf(buf: str + p, size: len - p, fmt: ",%u" , last); |
462 | state = 0; |
463 | break; |
464 | case 4: |
465 | p += scnprintf(buf: str + p, size: len - p, fmt: "-%u" , last); |
466 | state = 0; |
467 | break; |
468 | default: |
469 | break; |
470 | } |
471 | if (state != 1) |
472 | state = 0; |
473 | } |
474 | } |
475 | } |
476 | |
477 | static int intel_pt_val_config_term(struct perf_pmu *intel_pt_pmu, int dirfd, |
478 | const char *caps, const char *name, |
479 | const char *supported, u64 config) |
480 | { |
481 | char valid_str[256]; |
482 | unsigned int shift; |
483 | unsigned long long valid; |
484 | u64 bits; |
485 | int ok; |
486 | |
487 | if (perf_pmu__scan_file_at(pmu: intel_pt_pmu, dirfd, name: caps, fmt: "%llx" , &valid) != 1) |
488 | valid = 0; |
489 | |
490 | if (supported && |
491 | perf_pmu__scan_file_at(pmu: intel_pt_pmu, dirfd, name: supported, fmt: "%d" , &ok) == 1 && !ok) |
492 | valid = 0; |
493 | |
494 | valid |= 1; |
495 | |
496 | bits = perf_pmu__format_bits(pmu: intel_pt_pmu, name); |
497 | |
498 | config &= bits; |
499 | |
500 | for (shift = 0; bits && !(bits & 1); shift++) |
501 | bits >>= 1; |
502 | |
503 | config >>= shift; |
504 | |
505 | if (config > 63) |
506 | goto out_err; |
507 | |
508 | if (valid & (1 << config)) |
509 | return 0; |
510 | out_err: |
511 | intel_pt_valid_str(str: valid_str, len: sizeof(valid_str), valid); |
512 | pr_err("Invalid %s for %s. Valid values are: %s\n" , |
513 | name, INTEL_PT_PMU_NAME, valid_str); |
514 | return -EINVAL; |
515 | } |
516 | |
517 | static int intel_pt_validate_config(struct perf_pmu *intel_pt_pmu, |
518 | struct evsel *evsel) |
519 | { |
520 | int err, dirfd; |
521 | char c; |
522 | |
523 | if (!evsel) |
524 | return 0; |
525 | |
526 | dirfd = perf_pmu__event_source_devices_fd(); |
527 | if (dirfd < 0) |
528 | return dirfd; |
529 | |
530 | /* |
531 | * If supported, force pass-through config term (pt=1) even if user |
532 | * sets pt=0, which avoids senseless kernel errors. |
533 | */ |
534 | if (perf_pmu__scan_file_at(pmu: intel_pt_pmu, dirfd, name: "format/pt" , fmt: "%c" , &c) == 1 && |
535 | !(evsel->core.attr.config & 1)) { |
536 | pr_warning("pt=0 doesn't make sense, forcing pt=1\n" ); |
537 | evsel->core.attr.config |= 1; |
538 | } |
539 | |
540 | err = intel_pt_val_config_term(intel_pt_pmu, dirfd, caps: "caps/cycle_thresholds" , |
541 | name: "cyc_thresh" , supported: "caps/psb_cyc" , |
542 | config: evsel->core.attr.config); |
543 | if (err) |
544 | goto out; |
545 | |
546 | err = intel_pt_val_config_term(intel_pt_pmu, dirfd, caps: "caps/mtc_periods" , |
547 | name: "mtc_period" , supported: "caps/mtc" , |
548 | config: evsel->core.attr.config); |
549 | if (err) |
550 | goto out; |
551 | |
552 | err = intel_pt_val_config_term(intel_pt_pmu, dirfd, caps: "caps/psb_periods" , |
553 | name: "psb_period" , supported: "caps/psb_cyc" , |
554 | config: evsel->core.attr.config); |
555 | |
556 | out: |
557 | close(dirfd); |
558 | return err; |
559 | } |
560 | |
561 | static void intel_pt_min_max_sample_sz(struct evlist *evlist, |
562 | size_t *min_sz, size_t *max_sz) |
563 | { |
564 | struct evsel *evsel; |
565 | |
566 | evlist__for_each_entry(evlist, evsel) { |
567 | size_t sz = evsel->core.attr.aux_sample_size; |
568 | |
569 | if (!sz) |
570 | continue; |
571 | if (min_sz && (sz < *min_sz || !*min_sz)) |
572 | *min_sz = sz; |
573 | if (max_sz && sz > *max_sz) |
574 | *max_sz = sz; |
575 | } |
576 | } |
577 | |
578 | /* |
579 | * Currently, there is not enough information to disambiguate different PEBS |
580 | * events, so only allow one. |
581 | */ |
582 | static bool intel_pt_too_many_aux_output(struct evlist *evlist) |
583 | { |
584 | struct evsel *evsel; |
585 | int aux_output_cnt = 0; |
586 | |
587 | evlist__for_each_entry(evlist, evsel) |
588 | aux_output_cnt += !!evsel->core.attr.aux_output; |
589 | |
590 | if (aux_output_cnt > 1) { |
591 | pr_err(INTEL_PT_PMU_NAME " supports at most one event with aux-output\n" ); |
592 | return true; |
593 | } |
594 | |
595 | return false; |
596 | } |
597 | |
598 | static int intel_pt_recording_options(struct auxtrace_record *itr, |
599 | struct evlist *evlist, |
600 | struct record_opts *opts) |
601 | { |
602 | struct intel_pt_recording *ptr = |
603 | container_of(itr, struct intel_pt_recording, itr); |
604 | struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu; |
605 | bool have_timing_info, need_immediate = false; |
606 | struct evsel *evsel, *intel_pt_evsel = NULL; |
607 | const struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; |
608 | bool privileged = perf_event_paranoid_check(max_level: -1); |
609 | u64 tsc_bit; |
610 | int err; |
611 | |
612 | ptr->evlist = evlist; |
613 | ptr->snapshot_mode = opts->auxtrace_snapshot_mode; |
614 | |
615 | evlist__for_each_entry(evlist, evsel) { |
616 | if (evsel->core.attr.type == intel_pt_pmu->type) { |
617 | if (intel_pt_evsel) { |
618 | pr_err("There may be only one " INTEL_PT_PMU_NAME " event\n" ); |
619 | return -EINVAL; |
620 | } |
621 | evsel->core.attr.freq = 0; |
622 | evsel->core.attr.sample_period = 1; |
623 | evsel->no_aux_samples = true; |
624 | evsel->needs_auxtrace_mmap = true; |
625 | intel_pt_evsel = evsel; |
626 | opts->full_auxtrace = true; |
627 | } |
628 | } |
629 | |
630 | if (opts->auxtrace_snapshot_mode && !opts->full_auxtrace) { |
631 | pr_err("Snapshot mode (-S option) requires " INTEL_PT_PMU_NAME " PMU event (-e " INTEL_PT_PMU_NAME ")\n" ); |
632 | return -EINVAL; |
633 | } |
634 | |
635 | if (opts->auxtrace_snapshot_mode && opts->auxtrace_sample_mode) { |
636 | pr_err("Snapshot mode (" INTEL_PT_PMU_NAME " PMU) and sample trace cannot be used together\n" ); |
637 | return -EINVAL; |
638 | } |
639 | |
640 | if (opts->use_clockid) { |
641 | pr_err("Cannot use clockid (-k option) with " INTEL_PT_PMU_NAME "\n" ); |
642 | return -EINVAL; |
643 | } |
644 | |
645 | if (intel_pt_too_many_aux_output(evlist)) |
646 | return -EINVAL; |
647 | |
648 | if (!opts->full_auxtrace) |
649 | return 0; |
650 | |
651 | if (opts->auxtrace_sample_mode) |
652 | evsel__set_config_if_unset(pmu: intel_pt_pmu, evsel: intel_pt_evsel, |
653 | config_name: "psb_period" , val: 0); |
654 | |
655 | err = intel_pt_validate_config(intel_pt_pmu, evsel: intel_pt_evsel); |
656 | if (err) |
657 | return err; |
658 | |
659 | /* Set default sizes for snapshot mode */ |
660 | if (opts->auxtrace_snapshot_mode) { |
661 | size_t psb_period = intel_pt_psb_period(intel_pt_pmu, evlist); |
662 | |
663 | if (!opts->auxtrace_snapshot_size && !opts->auxtrace_mmap_pages) { |
664 | if (privileged) { |
665 | opts->auxtrace_mmap_pages = MiB(4) / page_size; |
666 | } else { |
667 | opts->auxtrace_mmap_pages = KiB(128) / page_size; |
668 | if (opts->mmap_pages == UINT_MAX) |
669 | opts->mmap_pages = KiB(256) / page_size; |
670 | } |
671 | } else if (!opts->auxtrace_mmap_pages && !privileged && |
672 | opts->mmap_pages == UINT_MAX) { |
673 | opts->mmap_pages = KiB(256) / page_size; |
674 | } |
675 | if (!opts->auxtrace_snapshot_size) |
676 | opts->auxtrace_snapshot_size = |
677 | opts->auxtrace_mmap_pages * (size_t)page_size; |
678 | if (!opts->auxtrace_mmap_pages) { |
679 | size_t sz = opts->auxtrace_snapshot_size; |
680 | |
681 | sz = round_up(sz, page_size) / page_size; |
682 | opts->auxtrace_mmap_pages = roundup_pow_of_two(sz); |
683 | } |
684 | if (opts->auxtrace_snapshot_size > |
685 | opts->auxtrace_mmap_pages * (size_t)page_size) { |
686 | pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n" , |
687 | opts->auxtrace_snapshot_size, |
688 | opts->auxtrace_mmap_pages * (size_t)page_size); |
689 | return -EINVAL; |
690 | } |
691 | if (!opts->auxtrace_snapshot_size || !opts->auxtrace_mmap_pages) { |
692 | pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n" ); |
693 | return -EINVAL; |
694 | } |
695 | pr_debug2("Intel PT snapshot size: %zu\n" , |
696 | opts->auxtrace_snapshot_size); |
697 | if (psb_period && |
698 | opts->auxtrace_snapshot_size <= psb_period + |
699 | INTEL_PT_PSB_PERIOD_NEAR) |
700 | ui__warning(format: "Intel PT snapshot size (%zu) may be too small for PSB period (%zu)\n" , |
701 | opts->auxtrace_snapshot_size, psb_period); |
702 | } |
703 | |
704 | /* Set default sizes for sample mode */ |
705 | if (opts->auxtrace_sample_mode) { |
706 | size_t psb_period = intel_pt_psb_period(intel_pt_pmu, evlist); |
707 | size_t min_sz = 0, max_sz = 0; |
708 | |
709 | intel_pt_min_max_sample_sz(evlist, min_sz: &min_sz, max_sz: &max_sz); |
710 | if (!opts->auxtrace_mmap_pages && !privileged && |
711 | opts->mmap_pages == UINT_MAX) |
712 | opts->mmap_pages = KiB(256) / page_size; |
713 | if (!opts->auxtrace_mmap_pages) { |
714 | size_t sz = round_up(max_sz, page_size) / page_size; |
715 | |
716 | opts->auxtrace_mmap_pages = roundup_pow_of_two(sz); |
717 | } |
718 | if (max_sz > opts->auxtrace_mmap_pages * (size_t)page_size) { |
719 | pr_err("Sample size %zu must not be greater than AUX area tracing mmap size %zu\n" , |
720 | max_sz, |
721 | opts->auxtrace_mmap_pages * (size_t)page_size); |
722 | return -EINVAL; |
723 | } |
724 | pr_debug2("Intel PT min. sample size: %zu max. sample size: %zu\n" , |
725 | min_sz, max_sz); |
726 | if (psb_period && |
727 | min_sz <= psb_period + INTEL_PT_PSB_PERIOD_NEAR) |
728 | ui__warning(format: "Intel PT sample size (%zu) may be too small for PSB period (%zu)\n" , |
729 | min_sz, psb_period); |
730 | } |
731 | |
732 | /* Set default sizes for full trace mode */ |
733 | if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) { |
734 | if (privileged) { |
735 | opts->auxtrace_mmap_pages = MiB(4) / page_size; |
736 | } else { |
737 | opts->auxtrace_mmap_pages = KiB(128) / page_size; |
738 | if (opts->mmap_pages == UINT_MAX) |
739 | opts->mmap_pages = KiB(256) / page_size; |
740 | } |
741 | } |
742 | |
743 | /* Validate auxtrace_mmap_pages */ |
744 | if (opts->auxtrace_mmap_pages) { |
745 | size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size; |
746 | size_t min_sz; |
747 | |
748 | if (opts->auxtrace_snapshot_mode || opts->auxtrace_sample_mode) |
749 | min_sz = KiB(4); |
750 | else |
751 | min_sz = KiB(8); |
752 | |
753 | if (sz < min_sz || !is_power_of_2(n: sz)) { |
754 | pr_err("Invalid mmap size for Intel Processor Trace: must be at least %zuKiB and a power of 2\n" , |
755 | min_sz / 1024); |
756 | return -EINVAL; |
757 | } |
758 | } |
759 | |
760 | if (!opts->auxtrace_snapshot_mode && !opts->auxtrace_sample_mode) { |
761 | u32 aux_watermark = opts->auxtrace_mmap_pages * page_size / 4; |
762 | |
763 | intel_pt_evsel->core.attr.aux_watermark = aux_watermark; |
764 | } |
765 | |
766 | intel_pt_parse_terms(pmu: intel_pt_pmu, str: "tsc" , config: &tsc_bit); |
767 | |
768 | if (opts->full_auxtrace && (intel_pt_evsel->core.attr.config & tsc_bit)) |
769 | have_timing_info = true; |
770 | else |
771 | have_timing_info = false; |
772 | |
773 | /* |
774 | * Per-cpu recording needs sched_switch events to distinguish different |
775 | * threads. |
776 | */ |
777 | if (have_timing_info && !perf_cpu_map__has_any_cpu_or_is_empty(cpus) && |
778 | !record_opts__no_switch_events(opts)) { |
779 | if (perf_can_record_switch_events()) { |
780 | bool cpu_wide = !target__none(target: &opts->target) && |
781 | !target__has_task(target: &opts->target); |
782 | |
783 | if (!cpu_wide && perf_can_record_cpu_wide()) { |
784 | struct evsel *switch_evsel; |
785 | |
786 | switch_evsel = evlist__add_dummy_on_all_cpus(evlist); |
787 | if (!switch_evsel) |
788 | return -ENOMEM; |
789 | |
790 | switch_evsel->core.attr.context_switch = 1; |
791 | switch_evsel->immediate = true; |
792 | |
793 | evsel__set_sample_bit(switch_evsel, TID); |
794 | evsel__set_sample_bit(switch_evsel, TIME); |
795 | evsel__set_sample_bit(switch_evsel, CPU); |
796 | evsel__reset_sample_bit(switch_evsel, BRANCH_STACK); |
797 | |
798 | opts->record_switch_events = false; |
799 | ptr->have_sched_switch = 3; |
800 | } else { |
801 | opts->record_switch_events = true; |
802 | need_immediate = true; |
803 | if (cpu_wide) |
804 | ptr->have_sched_switch = 3; |
805 | else |
806 | ptr->have_sched_switch = 2; |
807 | } |
808 | } else { |
809 | #ifdef HAVE_LIBTRACEEVENT |
810 | err = intel_pt_track_switches(evlist); |
811 | if (err == -EPERM) |
812 | pr_debug2("Unable to select sched:sched_switch\n" ); |
813 | else if (err) |
814 | return err; |
815 | else |
816 | ptr->have_sched_switch = 1; |
817 | #endif |
818 | } |
819 | } |
820 | |
821 | if (have_timing_info && !intel_pt_evsel->core.attr.exclude_kernel && |
822 | perf_can_record_text_poke_events() && perf_can_record_cpu_wide()) |
823 | opts->text_poke = true; |
824 | |
825 | if (intel_pt_evsel) { |
826 | /* |
827 | * To obtain the auxtrace buffer file descriptor, the auxtrace |
828 | * event must come first. |
829 | */ |
830 | evlist__to_front(evlist, move_evsel: intel_pt_evsel); |
831 | /* |
832 | * In the case of per-cpu mmaps, we need the CPU on the |
833 | * AUX event. |
834 | */ |
835 | if (!perf_cpu_map__has_any_cpu_or_is_empty(cpus)) |
836 | evsel__set_sample_bit(intel_pt_evsel, CPU); |
837 | } |
838 | |
839 | /* Add dummy event to keep tracking */ |
840 | if (opts->full_auxtrace) { |
841 | bool need_system_wide_tracking; |
842 | struct evsel *tracking_evsel; |
843 | |
844 | /* |
845 | * User space tasks can migrate between CPUs, so when tracing |
846 | * selected CPUs, sideband for all CPUs is still needed. |
847 | */ |
848 | need_system_wide_tracking = opts->target.cpu_list && |
849 | !intel_pt_evsel->core.attr.exclude_user; |
850 | |
851 | tracking_evsel = evlist__add_aux_dummy(evlist, system_wide: need_system_wide_tracking); |
852 | if (!tracking_evsel) |
853 | return -ENOMEM; |
854 | |
855 | evlist__set_tracking_event(evlist, tracking_evsel); |
856 | |
857 | if (need_immediate) |
858 | tracking_evsel->immediate = true; |
859 | |
860 | /* In per-cpu case, always need the time of mmap events etc */ |
861 | if (!perf_cpu_map__has_any_cpu_or_is_empty(cpus)) { |
862 | evsel__set_sample_bit(tracking_evsel, TIME); |
863 | /* And the CPU for switch events */ |
864 | evsel__set_sample_bit(tracking_evsel, CPU); |
865 | } |
866 | evsel__reset_sample_bit(tracking_evsel, BRANCH_STACK); |
867 | } |
868 | |
869 | /* |
870 | * Warn the user when we do not have enough information to decode i.e. |
871 | * per-cpu with no sched_switch (except workload-only). |
872 | */ |
873 | if (!ptr->have_sched_switch && !perf_cpu_map__has_any_cpu_or_is_empty(cpus) && |
874 | !target__none(target: &opts->target) && |
875 | !intel_pt_evsel->core.attr.exclude_user) |
876 | ui__warning(format: "Intel Processor Trace decoding will not be possible except for kernel tracing!\n" ); |
877 | |
878 | return 0; |
879 | } |
880 | |
881 | static int intel_pt_snapshot_start(struct auxtrace_record *itr) |
882 | { |
883 | struct intel_pt_recording *ptr = |
884 | container_of(itr, struct intel_pt_recording, itr); |
885 | struct evsel *evsel; |
886 | |
887 | evlist__for_each_entry(ptr->evlist, evsel) { |
888 | if (evsel->core.attr.type == ptr->intel_pt_pmu->type) |
889 | return evsel__disable(evsel); |
890 | } |
891 | return -EINVAL; |
892 | } |
893 | |
894 | static int intel_pt_snapshot_finish(struct auxtrace_record *itr) |
895 | { |
896 | struct intel_pt_recording *ptr = |
897 | container_of(itr, struct intel_pt_recording, itr); |
898 | struct evsel *evsel; |
899 | |
900 | evlist__for_each_entry(ptr->evlist, evsel) { |
901 | if (evsel->core.attr.type == ptr->intel_pt_pmu->type) |
902 | return evsel__enable(evsel); |
903 | } |
904 | return -EINVAL; |
905 | } |
906 | |
907 | static int intel_pt_alloc_snapshot_refs(struct intel_pt_recording *ptr, int idx) |
908 | { |
909 | const size_t sz = sizeof(struct intel_pt_snapshot_ref); |
910 | int cnt = ptr->snapshot_ref_cnt, new_cnt = cnt * 2; |
911 | struct intel_pt_snapshot_ref *refs; |
912 | |
913 | if (!new_cnt) |
914 | new_cnt = 16; |
915 | |
916 | while (new_cnt <= idx) |
917 | new_cnt *= 2; |
918 | |
919 | refs = calloc(new_cnt, sz); |
920 | if (!refs) |
921 | return -ENOMEM; |
922 | |
923 | memcpy(refs, ptr->snapshot_refs, cnt * sz); |
924 | |
925 | ptr->snapshot_refs = refs; |
926 | ptr->snapshot_ref_cnt = new_cnt; |
927 | |
928 | return 0; |
929 | } |
930 | |
931 | static void intel_pt_free_snapshot_refs(struct intel_pt_recording *ptr) |
932 | { |
933 | int i; |
934 | |
935 | for (i = 0; i < ptr->snapshot_ref_cnt; i++) |
936 | zfree(&ptr->snapshot_refs[i].ref_buf); |
937 | zfree(&ptr->snapshot_refs); |
938 | } |
939 | |
940 | static void intel_pt_recording_free(struct auxtrace_record *itr) |
941 | { |
942 | struct intel_pt_recording *ptr = |
943 | container_of(itr, struct intel_pt_recording, itr); |
944 | |
945 | intel_pt_free_snapshot_refs(ptr); |
946 | free(ptr); |
947 | } |
948 | |
949 | static int intel_pt_alloc_snapshot_ref(struct intel_pt_recording *ptr, int idx, |
950 | size_t snapshot_buf_size) |
951 | { |
952 | size_t ref_buf_size = ptr->snapshot_ref_buf_size; |
953 | void *ref_buf; |
954 | |
955 | ref_buf = zalloc(ref_buf_size); |
956 | if (!ref_buf) |
957 | return -ENOMEM; |
958 | |
959 | ptr->snapshot_refs[idx].ref_buf = ref_buf; |
960 | ptr->snapshot_refs[idx].ref_offset = snapshot_buf_size - ref_buf_size; |
961 | |
962 | return 0; |
963 | } |
964 | |
965 | static size_t intel_pt_snapshot_ref_buf_size(struct intel_pt_recording *ptr, |
966 | size_t snapshot_buf_size) |
967 | { |
968 | const size_t max_size = 256 * 1024; |
969 | size_t buf_size = 0, psb_period; |
970 | |
971 | if (ptr->snapshot_size <= 64 * 1024) |
972 | return 0; |
973 | |
974 | psb_period = intel_pt_psb_period(intel_pt_pmu: ptr->intel_pt_pmu, evlist: ptr->evlist); |
975 | if (psb_period) |
976 | buf_size = psb_period * 2; |
977 | |
978 | if (!buf_size || buf_size > max_size) |
979 | buf_size = max_size; |
980 | |
981 | if (buf_size >= snapshot_buf_size) |
982 | return 0; |
983 | |
984 | if (buf_size >= ptr->snapshot_size / 2) |
985 | return 0; |
986 | |
987 | return buf_size; |
988 | } |
989 | |
990 | static int intel_pt_snapshot_init(struct intel_pt_recording *ptr, |
991 | size_t snapshot_buf_size) |
992 | { |
993 | if (ptr->snapshot_init_done) |
994 | return 0; |
995 | |
996 | ptr->snapshot_init_done = true; |
997 | |
998 | ptr->snapshot_ref_buf_size = intel_pt_snapshot_ref_buf_size(ptr, |
999 | snapshot_buf_size); |
1000 | |
1001 | return 0; |
1002 | } |
1003 | |
1004 | /** |
1005 | * intel_pt_compare_buffers - compare bytes in a buffer to a circular buffer. |
1006 | * @buf1: first buffer |
1007 | * @compare_size: number of bytes to compare |
1008 | * @buf2: second buffer (a circular buffer) |
1009 | * @offs2: offset in second buffer |
1010 | * @buf2_size: size of second buffer |
1011 | * |
1012 | * The comparison allows for the possibility that the bytes to compare in the |
1013 | * circular buffer are not contiguous. It is assumed that @compare_size <= |
1014 | * @buf2_size. This function returns %false if the bytes are identical, %true |
1015 | * otherwise. |
1016 | */ |
1017 | static bool intel_pt_compare_buffers(void *buf1, size_t compare_size, |
1018 | void *buf2, size_t offs2, size_t buf2_size) |
1019 | { |
1020 | size_t end2 = offs2 + compare_size, part_size; |
1021 | |
1022 | if (end2 <= buf2_size) |
1023 | return memcmp(p: buf1, q: buf2 + offs2, size: compare_size); |
1024 | |
1025 | part_size = end2 - buf2_size; |
1026 | if (memcmp(p: buf1, q: buf2 + offs2, size: part_size)) |
1027 | return true; |
1028 | |
1029 | compare_size -= part_size; |
1030 | |
1031 | return memcmp(p: buf1 + part_size, q: buf2, size: compare_size); |
1032 | } |
1033 | |
1034 | static bool intel_pt_compare_ref(void *ref_buf, size_t ref_offset, |
1035 | size_t ref_size, size_t buf_size, |
1036 | void *data, size_t head) |
1037 | { |
1038 | size_t ref_end = ref_offset + ref_size; |
1039 | |
1040 | if (ref_end > buf_size) { |
1041 | if (head > ref_offset || head < ref_end - buf_size) |
1042 | return true; |
1043 | } else if (head > ref_offset && head < ref_end) { |
1044 | return true; |
1045 | } |
1046 | |
1047 | return intel_pt_compare_buffers(buf1: ref_buf, compare_size: ref_size, buf2: data, offs2: ref_offset, |
1048 | buf2_size: buf_size); |
1049 | } |
1050 | |
1051 | static void intel_pt_copy_ref(void *ref_buf, size_t ref_size, size_t buf_size, |
1052 | void *data, size_t head) |
1053 | { |
1054 | if (head >= ref_size) { |
1055 | memcpy(ref_buf, data + head - ref_size, ref_size); |
1056 | } else { |
1057 | memcpy(ref_buf, data, head); |
1058 | ref_size -= head; |
1059 | memcpy(ref_buf + head, data + buf_size - ref_size, ref_size); |
1060 | } |
1061 | } |
1062 | |
1063 | static bool intel_pt_wrapped(struct intel_pt_recording *ptr, int idx, |
1064 | struct auxtrace_mmap *mm, unsigned char *data, |
1065 | u64 head) |
1066 | { |
1067 | struct intel_pt_snapshot_ref *ref = &ptr->snapshot_refs[idx]; |
1068 | bool wrapped; |
1069 | |
1070 | wrapped = intel_pt_compare_ref(ref_buf: ref->ref_buf, ref_offset: ref->ref_offset, |
1071 | ref_size: ptr->snapshot_ref_buf_size, buf_size: mm->len, |
1072 | data, head); |
1073 | |
1074 | intel_pt_copy_ref(ref_buf: ref->ref_buf, ref_size: ptr->snapshot_ref_buf_size, buf_size: mm->len, |
1075 | data, head); |
1076 | |
1077 | return wrapped; |
1078 | } |
1079 | |
1080 | static bool intel_pt_first_wrap(u64 *data, size_t buf_size) |
1081 | { |
1082 | int i, a, b; |
1083 | |
1084 | b = buf_size >> 3; |
1085 | a = b - 512; |
1086 | if (a < 0) |
1087 | a = 0; |
1088 | |
1089 | for (i = a; i < b; i++) { |
1090 | if (data[i]) |
1091 | return true; |
1092 | } |
1093 | |
1094 | return false; |
1095 | } |
1096 | |
1097 | static int intel_pt_find_snapshot(struct auxtrace_record *itr, int idx, |
1098 | struct auxtrace_mmap *mm, unsigned char *data, |
1099 | u64 *head, u64 *old) |
1100 | { |
1101 | struct intel_pt_recording *ptr = |
1102 | container_of(itr, struct intel_pt_recording, itr); |
1103 | bool wrapped; |
1104 | int err; |
1105 | |
1106 | pr_debug3("%s: mmap index %d old head %zu new head %zu\n" , |
1107 | __func__, idx, (size_t)*old, (size_t)*head); |
1108 | |
1109 | err = intel_pt_snapshot_init(ptr, snapshot_buf_size: mm->len); |
1110 | if (err) |
1111 | goto out_err; |
1112 | |
1113 | if (idx >= ptr->snapshot_ref_cnt) { |
1114 | err = intel_pt_alloc_snapshot_refs(ptr, idx); |
1115 | if (err) |
1116 | goto out_err; |
1117 | } |
1118 | |
1119 | if (ptr->snapshot_ref_buf_size) { |
1120 | if (!ptr->snapshot_refs[idx].ref_buf) { |
1121 | err = intel_pt_alloc_snapshot_ref(ptr, idx, snapshot_buf_size: mm->len); |
1122 | if (err) |
1123 | goto out_err; |
1124 | } |
1125 | wrapped = intel_pt_wrapped(ptr, idx, mm, data, head: *head); |
1126 | } else { |
1127 | wrapped = ptr->snapshot_refs[idx].wrapped; |
1128 | if (!wrapped && intel_pt_first_wrap(data: (u64 *)data, buf_size: mm->len)) { |
1129 | ptr->snapshot_refs[idx].wrapped = true; |
1130 | wrapped = true; |
1131 | } |
1132 | } |
1133 | |
1134 | /* |
1135 | * In full trace mode 'head' continually increases. However in snapshot |
1136 | * mode 'head' is an offset within the buffer. Here 'old' and 'head' |
1137 | * are adjusted to match the full trace case which expects that 'old' is |
1138 | * always less than 'head'. |
1139 | */ |
1140 | if (wrapped) { |
1141 | *old = *head; |
1142 | *head += mm->len; |
1143 | } else { |
1144 | if (mm->mask) |
1145 | *old &= mm->mask; |
1146 | else |
1147 | *old %= mm->len; |
1148 | if (*old > *head) |
1149 | *head += mm->len; |
1150 | } |
1151 | |
1152 | pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n" , |
1153 | __func__, wrapped ? "" : "not " , (size_t)*old, (size_t)*head); |
1154 | |
1155 | return 0; |
1156 | |
1157 | out_err: |
1158 | pr_err("%s: failed, error %d\n" , __func__, err); |
1159 | return err; |
1160 | } |
1161 | |
1162 | static u64 intel_pt_reference(struct auxtrace_record *itr __maybe_unused) |
1163 | { |
1164 | return rdtsc(); |
1165 | } |
1166 | |
1167 | struct auxtrace_record *intel_pt_recording_init(int *err) |
1168 | { |
1169 | struct perf_pmu *intel_pt_pmu = perf_pmus__find(INTEL_PT_PMU_NAME); |
1170 | struct intel_pt_recording *ptr; |
1171 | |
1172 | if (!intel_pt_pmu) |
1173 | return NULL; |
1174 | |
1175 | if (setenv("JITDUMP_USE_ARCH_TIMESTAMP" , "1" , 1)) { |
1176 | *err = -errno; |
1177 | return NULL; |
1178 | } |
1179 | |
1180 | ptr = zalloc(sizeof(struct intel_pt_recording)); |
1181 | if (!ptr) { |
1182 | *err = -ENOMEM; |
1183 | return NULL; |
1184 | } |
1185 | |
1186 | ptr->intel_pt_pmu = intel_pt_pmu; |
1187 | ptr->itr.pmu = intel_pt_pmu; |
1188 | ptr->itr.recording_options = intel_pt_recording_options; |
1189 | ptr->itr.info_priv_size = intel_pt_info_priv_size; |
1190 | ptr->itr.info_fill = intel_pt_info_fill; |
1191 | ptr->itr.free = intel_pt_recording_free; |
1192 | ptr->itr.snapshot_start = intel_pt_snapshot_start; |
1193 | ptr->itr.snapshot_finish = intel_pt_snapshot_finish; |
1194 | ptr->itr.find_snapshot = intel_pt_find_snapshot; |
1195 | ptr->itr.parse_snapshot_options = intel_pt_parse_snapshot_options; |
1196 | ptr->itr.reference = intel_pt_reference; |
1197 | ptr->itr.read_finish = auxtrace_record__read_finish; |
1198 | /* |
1199 | * Decoding starts at a PSB packet. Minimum PSB period is 2K so 4K |
1200 | * should give at least 1 PSB per sample. |
1201 | */ |
1202 | ptr->itr.default_aux_sample_size = 4096; |
1203 | return &ptr->itr; |
1204 | } |
1205 | |