1/*
2 * BTS PMU driver for perf
3 * Copyright (c) 2013-2014, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 */
14
15#undef DEBUG
16
17#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18
19#include <linux/bitops.h>
20#include <linux/types.h>
21#include <linux/slab.h>
22#include <linux/debugfs.h>
23#include <linux/device.h>
24#include <linux/coredump.h>
25
26#include <asm-generic/sizes.h>
27#include <asm/perf_event.h>
28
29#include "../perf_event.h"
30
31struct bts_ctx {
32 struct perf_output_handle handle;
33 struct debug_store ds_back;
34 int state;
35};
36
37/* BTS context states: */
38enum {
39 /* no ongoing AUX transactions */
40 BTS_STATE_STOPPED = 0,
41 /* AUX transaction is on, BTS tracing is disabled */
42 BTS_STATE_INACTIVE,
43 /* AUX transaction is on, BTS tracing is running */
44 BTS_STATE_ACTIVE,
45};
46
47static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
48
49#define BTS_RECORD_SIZE 24
50#define BTS_SAFETY_MARGIN 4080
51
52struct bts_phys {
53 struct page *page;
54 unsigned long size;
55 unsigned long offset;
56 unsigned long displacement;
57};
58
59struct bts_buffer {
60 size_t real_size; /* multiple of BTS_RECORD_SIZE */
61 unsigned int nr_pages;
62 unsigned int nr_bufs;
63 unsigned int cur_buf;
64 bool snapshot;
65 local_t data_size;
66 local_t head;
67 unsigned long end;
68 void **data_pages;
69 struct bts_phys buf[0];
70};
71
72static struct pmu bts_pmu;
73
74static size_t buf_size(struct page *page)
75{
76 return 1 << (PAGE_SHIFT + page_private(page));
77}
78
79static void *
80bts_buffer_setup_aux(struct perf_event *event, void **pages,
81 int nr_pages, bool overwrite)
82{
83 struct bts_buffer *buf;
84 struct page *page;
85 int cpu = event->cpu;
86 int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
87 unsigned long offset;
88 size_t size = nr_pages << PAGE_SHIFT;
89 int pg, nbuf, pad;
90
91 /* count all the high order buffers */
92 for (pg = 0, nbuf = 0; pg < nr_pages;) {
93 page = virt_to_page(pages[pg]);
94 if (WARN_ON_ONCE(!PagePrivate(page) && nr_pages > 1))
95 return NULL;
96 pg += 1 << page_private(page);
97 nbuf++;
98 }
99
100 /*
101 * to avoid interrupts in overwrite mode, only allow one physical
102 */
103 if (overwrite && nbuf > 1)
104 return NULL;
105
106 buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
107 if (!buf)
108 return NULL;
109
110 buf->nr_pages = nr_pages;
111 buf->nr_bufs = nbuf;
112 buf->snapshot = overwrite;
113 buf->data_pages = pages;
114 buf->real_size = size - size % BTS_RECORD_SIZE;
115
116 for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
117 unsigned int __nr_pages;
118
119 page = virt_to_page(pages[pg]);
120 __nr_pages = PagePrivate(page) ? 1 << page_private(page) : 1;
121 buf->buf[nbuf].page = page;
122 buf->buf[nbuf].offset = offset;
123 buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
124 buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
125 pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
126 buf->buf[nbuf].size -= pad;
127
128 pg += __nr_pages;
129 offset += __nr_pages << PAGE_SHIFT;
130 }
131
132 return buf;
133}
134
135static void bts_buffer_free_aux(void *data)
136{
137 kfree(data);
138}
139
140static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
141{
142 return buf->buf[idx].offset + buf->buf[idx].displacement;
143}
144
145static void
146bts_config_buffer(struct bts_buffer *buf)
147{
148 int cpu = raw_smp_processor_id();
149 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
150 struct bts_phys *phys = &buf->buf[buf->cur_buf];
151 unsigned long index, thresh = 0, end = phys->size;
152 struct page *page = phys->page;
153
154 index = local_read(&buf->head);
155
156 if (!buf->snapshot) {
157 if (buf->end < phys->offset + buf_size(page))
158 end = buf->end - phys->offset - phys->displacement;
159
160 index -= phys->offset + phys->displacement;
161
162 if (end - index > BTS_SAFETY_MARGIN)
163 thresh = end - BTS_SAFETY_MARGIN;
164 else if (end - index > BTS_RECORD_SIZE)
165 thresh = end - BTS_RECORD_SIZE;
166 else
167 thresh = end;
168 }
169
170 ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
171 ds->bts_index = ds->bts_buffer_base + index;
172 ds->bts_absolute_maximum = ds->bts_buffer_base + end;
173 ds->bts_interrupt_threshold = !buf->snapshot
174 ? ds->bts_buffer_base + thresh
175 : ds->bts_absolute_maximum + BTS_RECORD_SIZE;
176}
177
178static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
179{
180 unsigned long index = head - phys->offset;
181
182 memset(page_address(phys->page) + index, 0, phys->size - index);
183}
184
185static void bts_update(struct bts_ctx *bts)
186{
187 int cpu = raw_smp_processor_id();
188 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
189 struct bts_buffer *buf = perf_get_aux(&bts->handle);
190 unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
191
192 if (!buf)
193 return;
194
195 head = index + bts_buffer_offset(buf, buf->cur_buf);
196 old = local_xchg(&buf->head, head);
197
198 if (!buf->snapshot) {
199 if (old == head)
200 return;
201
202 if (ds->bts_index >= ds->bts_absolute_maximum)
203 perf_aux_output_flag(&bts->handle,
204 PERF_AUX_FLAG_TRUNCATED);
205
206 /*
207 * old and head are always in the same physical buffer, so we
208 * can subtract them to get the data size.
209 */
210 local_add(head - old, &buf->data_size);
211 } else {
212 local_set(&buf->data_size, head);
213 }
214}
215
216static int
217bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle);
218
219/*
220 * Ordering PMU callbacks wrt themselves and the PMI is done by means
221 * of bts::state, which:
222 * - is set when bts::handle::event is valid, that is, between
223 * perf_aux_output_begin() and perf_aux_output_end();
224 * - is zero otherwise;
225 * - is ordered against bts::handle::event with a compiler barrier.
226 */
227
228static void __bts_event_start(struct perf_event *event)
229{
230 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
231 struct bts_buffer *buf = perf_get_aux(&bts->handle);
232 u64 config = 0;
233
234 if (!buf->snapshot)
235 config |= ARCH_PERFMON_EVENTSEL_INT;
236 if (!event->attr.exclude_kernel)
237 config |= ARCH_PERFMON_EVENTSEL_OS;
238 if (!event->attr.exclude_user)
239 config |= ARCH_PERFMON_EVENTSEL_USR;
240
241 bts_config_buffer(buf);
242
243 /*
244 * local barrier to make sure that ds configuration made it
245 * before we enable BTS and bts::state goes ACTIVE
246 */
247 wmb();
248
249 /* INACTIVE/STOPPED -> ACTIVE */
250 WRITE_ONCE(bts->state, BTS_STATE_ACTIVE);
251
252 intel_pmu_enable_bts(config);
253
254}
255
256static void bts_event_start(struct perf_event *event, int flags)
257{
258 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
259 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
260 struct bts_buffer *buf;
261
262 buf = perf_aux_output_begin(&bts->handle, event);
263 if (!buf)
264 goto fail_stop;
265
266 if (bts_buffer_reset(buf, &bts->handle))
267 goto fail_end_stop;
268
269 bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
270 bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
271 bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
272
273 perf_event_itrace_started(event);
274 event->hw.state = 0;
275
276 __bts_event_start(event);
277
278 return;
279
280fail_end_stop:
281 perf_aux_output_end(&bts->handle, 0);
282
283fail_stop:
284 event->hw.state = PERF_HES_STOPPED;
285}
286
287static void __bts_event_stop(struct perf_event *event, int state)
288{
289 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
290
291 /* ACTIVE -> INACTIVE(PMI)/STOPPED(->stop()) */
292 WRITE_ONCE(bts->state, state);
293
294 /*
295 * No extra synchronization is mandated by the documentation to have
296 * BTS data stores globally visible.
297 */
298 intel_pmu_disable_bts();
299}
300
301static void bts_event_stop(struct perf_event *event, int flags)
302{
303 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
304 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
305 struct bts_buffer *buf = NULL;
306 int state = READ_ONCE(bts->state);
307
308 if (state == BTS_STATE_ACTIVE)
309 __bts_event_stop(event, BTS_STATE_STOPPED);
310
311 if (state != BTS_STATE_STOPPED)
312 buf = perf_get_aux(&bts->handle);
313
314 event->hw.state |= PERF_HES_STOPPED;
315
316 if (flags & PERF_EF_UPDATE) {
317 bts_update(bts);
318
319 if (buf) {
320 if (buf->snapshot)
321 bts->handle.head =
322 local_xchg(&buf->data_size,
323 buf->nr_pages << PAGE_SHIFT);
324 perf_aux_output_end(&bts->handle,
325 local_xchg(&buf->data_size, 0));
326 }
327
328 cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
329 cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
330 cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
331 cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
332 }
333}
334
335void intel_bts_enable_local(void)
336{
337 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
338 int state = READ_ONCE(bts->state);
339
340 /*
341 * Here we transition from INACTIVE to ACTIVE;
342 * if we instead are STOPPED from the interrupt handler,
343 * stay that way. Can't be ACTIVE here though.
344 */
345 if (WARN_ON_ONCE(state == BTS_STATE_ACTIVE))
346 return;
347
348 if (state == BTS_STATE_STOPPED)
349 return;
350
351 if (bts->handle.event)
352 __bts_event_start(bts->handle.event);
353}
354
355void intel_bts_disable_local(void)
356{
357 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
358
359 /*
360 * Here we transition from ACTIVE to INACTIVE;
361 * do nothing for STOPPED or INACTIVE.
362 */
363 if (READ_ONCE(bts->state) != BTS_STATE_ACTIVE)
364 return;
365
366 if (bts->handle.event)
367 __bts_event_stop(bts->handle.event, BTS_STATE_INACTIVE);
368}
369
370static int
371bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
372{
373 unsigned long head, space, next_space, pad, gap, skip, wakeup;
374 unsigned int next_buf;
375 struct bts_phys *phys, *next_phys;
376 int ret;
377
378 if (buf->snapshot)
379 return 0;
380
381 head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
382
383 phys = &buf->buf[buf->cur_buf];
384 space = phys->offset + phys->displacement + phys->size - head;
385 pad = space;
386 if (space > handle->size) {
387 space = handle->size;
388 space -= space % BTS_RECORD_SIZE;
389 }
390 if (space <= BTS_SAFETY_MARGIN) {
391 /* See if next phys buffer has more space */
392 next_buf = buf->cur_buf + 1;
393 if (next_buf >= buf->nr_bufs)
394 next_buf = 0;
395 next_phys = &buf->buf[next_buf];
396 gap = buf_size(phys->page) - phys->displacement - phys->size +
397 next_phys->displacement;
398 skip = pad + gap;
399 if (handle->size >= skip) {
400 next_space = next_phys->size;
401 if (next_space + skip > handle->size) {
402 next_space = handle->size - skip;
403 next_space -= next_space % BTS_RECORD_SIZE;
404 }
405 if (next_space > space || !space) {
406 if (pad)
407 bts_buffer_pad_out(phys, head);
408 ret = perf_aux_output_skip(handle, skip);
409 if (ret)
410 return ret;
411 /* Advance to next phys buffer */
412 phys = next_phys;
413 space = next_space;
414 head = phys->offset + phys->displacement;
415 /*
416 * After this, cur_buf and head won't match ds
417 * anymore, so we must not be racing with
418 * bts_update().
419 */
420 buf->cur_buf = next_buf;
421 local_set(&buf->head, head);
422 }
423 }
424 }
425
426 /* Don't go far beyond wakeup watermark */
427 wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
428 handle->head;
429 if (space > wakeup) {
430 space = wakeup;
431 space -= space % BTS_RECORD_SIZE;
432 }
433
434 buf->end = head + space;
435
436 /*
437 * If we have no space, the lost notification would have been sent when
438 * we hit absolute_maximum - see bts_update()
439 */
440 if (!space)
441 return -ENOSPC;
442
443 return 0;
444}
445
446int intel_bts_interrupt(void)
447{
448 struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds;
449 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
450 struct perf_event *event = bts->handle.event;
451 struct bts_buffer *buf;
452 s64 old_head;
453 int err = -ENOSPC, handled = 0;
454
455 /*
456 * The only surefire way of knowing if this NMI is ours is by checking
457 * the write ptr against the PMI threshold.
458 */
459 if (ds && (ds->bts_index >= ds->bts_interrupt_threshold))
460 handled = 1;
461
462 /*
463 * this is wrapped in intel_bts_enable_local/intel_bts_disable_local,
464 * so we can only be INACTIVE or STOPPED
465 */
466 if (READ_ONCE(bts->state) == BTS_STATE_STOPPED)
467 return handled;
468
469 buf = perf_get_aux(&bts->handle);
470 if (!buf)
471 return handled;
472
473 /*
474 * Skip snapshot counters: they don't use the interrupt, but
475 * there's no other way of telling, because the pointer will
476 * keep moving
477 */
478 if (buf->snapshot)
479 return 0;
480
481 old_head = local_read(&buf->head);
482 bts_update(bts);
483
484 /* no new data */
485 if (old_head == local_read(&buf->head))
486 return handled;
487
488 perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0));
489
490 buf = perf_aux_output_begin(&bts->handle, event);
491 if (buf)
492 err = bts_buffer_reset(buf, &bts->handle);
493
494 if (err) {
495 WRITE_ONCE(bts->state, BTS_STATE_STOPPED);
496
497 if (buf) {
498 /*
499 * BTS_STATE_STOPPED should be visible before
500 * cleared handle::event
501 */
502 barrier();
503 perf_aux_output_end(&bts->handle, 0);
504 }
505 }
506
507 return 1;
508}
509
510static void bts_event_del(struct perf_event *event, int mode)
511{
512 bts_event_stop(event, PERF_EF_UPDATE);
513}
514
515static int bts_event_add(struct perf_event *event, int mode)
516{
517 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
518 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
519 struct hw_perf_event *hwc = &event->hw;
520
521 event->hw.state = PERF_HES_STOPPED;
522
523 if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
524 return -EBUSY;
525
526 if (bts->handle.event)
527 return -EBUSY;
528
529 if (mode & PERF_EF_START) {
530 bts_event_start(event, 0);
531 if (hwc->state & PERF_HES_STOPPED)
532 return -EINVAL;
533 }
534
535 return 0;
536}
537
538static void bts_event_destroy(struct perf_event *event)
539{
540 x86_release_hardware();
541 x86_del_exclusive(x86_lbr_exclusive_bts);
542}
543
544static int bts_event_init(struct perf_event *event)
545{
546 int ret;
547
548 if (event->attr.type != bts_pmu.type)
549 return -ENOENT;
550
551 /*
552 * BTS leaks kernel addresses even when CPL0 tracing is
553 * disabled, so disallow intel_bts driver for unprivileged
554 * users on paranoid systems since it provides trace data
555 * to the user in a zero-copy fashion.
556 *
557 * Note that the default paranoia setting permits unprivileged
558 * users to profile the kernel.
559 */
560 if (event->attr.exclude_kernel && perf_paranoid_kernel() &&
561 !capable(CAP_SYS_ADMIN))
562 return -EACCES;
563
564 if (x86_add_exclusive(x86_lbr_exclusive_bts))
565 return -EBUSY;
566
567 ret = x86_reserve_hardware();
568 if (ret) {
569 x86_del_exclusive(x86_lbr_exclusive_bts);
570 return ret;
571 }
572
573 event->destroy = bts_event_destroy;
574
575 return 0;
576}
577
578static void bts_event_read(struct perf_event *event)
579{
580}
581
582static __init int bts_init(void)
583{
584 if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
585 return -ENODEV;
586
587 if (boot_cpu_has(X86_FEATURE_PTI)) {
588 /*
589 * BTS hardware writes through a virtual memory map we must
590 * either use the kernel physical map, or the user mapping of
591 * the AUX buffer.
592 *
593 * However, since this driver supports per-CPU and per-task inherit
594 * we cannot use the user mapping since it will not be available
595 * if we're not running the owning process.
596 *
597 * With PTI we can't use the kernal map either, because its not
598 * there when we run userspace.
599 *
600 * For now, disable this driver when using PTI.
601 */
602 return -ENODEV;
603 }
604
605 bts_pmu.capabilities = PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE |
606 PERF_PMU_CAP_EXCLUSIVE;
607 bts_pmu.task_ctx_nr = perf_sw_context;
608 bts_pmu.event_init = bts_event_init;
609 bts_pmu.add = bts_event_add;
610 bts_pmu.del = bts_event_del;
611 bts_pmu.start = bts_event_start;
612 bts_pmu.stop = bts_event_stop;
613 bts_pmu.read = bts_event_read;
614 bts_pmu.setup_aux = bts_buffer_setup_aux;
615 bts_pmu.free_aux = bts_buffer_free_aux;
616
617 return perf_pmu_register(&bts_pmu, "intel_bts", -1);
618}
619arch_initcall(bts_init);
620