1/*
2 * Performance events:
3 *
4 * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
7 *
8 * Data type definitions, declarations, prototypes.
9 *
10 * Started by: Thomas Gleixner and Ingo Molnar
11 *
12 * For licencing details see kernel-base/COPYING
13 */
14#ifndef _LINUX_PERF_EVENT_H
15#define _LINUX_PERF_EVENT_H
16
17#include <uapi/linux/perf_event.h>
18#include <uapi/linux/bpf_perf_event.h>
19
20/*
21 * Kernel-internal data types and definitions:
22 */
23
24#ifdef CONFIG_PERF_EVENTS
25# include <asm/perf_event.h>
26# include <asm/local64.h>
27#endif
28
29struct perf_guest_info_callbacks {
30 int (*is_in_guest)(void);
31 int (*is_user_mode)(void);
32 unsigned long (*get_guest_ip)(void);
33};
34
35#ifdef CONFIG_HAVE_HW_BREAKPOINT
36#include <asm/hw_breakpoint.h>
37#endif
38
39#include <linux/list.h>
40#include <linux/mutex.h>
41#include <linux/rculist.h>
42#include <linux/rcupdate.h>
43#include <linux/spinlock.h>
44#include <linux/hrtimer.h>
45#include <linux/fs.h>
46#include <linux/pid_namespace.h>
47#include <linux/workqueue.h>
48#include <linux/ftrace.h>
49#include <linux/cpu.h>
50#include <linux/irq_work.h>
51#include <linux/static_key.h>
52#include <linux/jump_label_ratelimit.h>
53#include <linux/atomic.h>
54#include <linux/sysfs.h>
55#include <linux/perf_regs.h>
56#include <linux/cgroup.h>
57#include <linux/refcount.h>
58#include <asm/local.h>
59
60struct perf_callchain_entry {
61 __u64 nr;
62 __u64 ip[0]; /* /proc/sys/kernel/perf_event_max_stack */
63};
64
65struct perf_callchain_entry_ctx {
66 struct perf_callchain_entry *entry;
67 u32 max_stack;
68 u32 nr;
69 short contexts;
70 bool contexts_maxed;
71};
72
73typedef unsigned long (*perf_copy_f)(void *dst, const void *src,
74 unsigned long off, unsigned long len);
75
76struct perf_raw_frag {
77 union {
78 struct perf_raw_frag *next;
79 unsigned long pad;
80 };
81 perf_copy_f copy;
82 void *data;
83 u32 size;
84} __packed;
85
86struct perf_raw_record {
87 struct perf_raw_frag frag;
88 u32 size;
89};
90
91/*
92 * branch stack layout:
93 * nr: number of taken branches stored in entries[]
94 *
95 * Note that nr can vary from sample to sample
96 * branches (to, from) are stored from most recent
97 * to least recent, i.e., entries[0] contains the most
98 * recent branch.
99 */
100struct perf_branch_stack {
101 __u64 nr;
102 struct perf_branch_entry entries[0];
103};
104
105struct task_struct;
106
107/*
108 * extra PMU register associated with an event
109 */
110struct hw_perf_event_extra {
111 u64 config; /* register value */
112 unsigned int reg; /* register address or index */
113 int alloc; /* extra register already allocated */
114 int idx; /* index in shared_regs->regs[] */
115};
116
117/**
118 * struct hw_perf_event - performance event hardware details:
119 */
120struct hw_perf_event {
121#ifdef CONFIG_PERF_EVENTS
122 union {
123 struct { /* hardware */
124 u64 config;
125 u64 last_tag;
126 unsigned long config_base;
127 unsigned long event_base;
128 int event_base_rdpmc;
129 int idx;
130 int last_cpu;
131 int flags;
132
133 struct hw_perf_event_extra extra_reg;
134 struct hw_perf_event_extra branch_reg;
135 };
136 struct { /* software */
137 struct hrtimer hrtimer;
138 };
139 struct { /* tracepoint */
140 /* for tp_event->class */
141 struct list_head tp_list;
142 };
143 struct { /* amd_power */
144 u64 pwr_acc;
145 u64 ptsc;
146 };
147#ifdef CONFIG_HAVE_HW_BREAKPOINT
148 struct { /* breakpoint */
149 /*
150 * Crufty hack to avoid the chicken and egg
151 * problem hw_breakpoint has with context
152 * creation and event initalization.
153 */
154 struct arch_hw_breakpoint info;
155 struct list_head bp_list;
156 };
157#endif
158 struct { /* amd_iommu */
159 u8 iommu_bank;
160 u8 iommu_cntr;
161 u16 padding;
162 u64 conf;
163 u64 conf1;
164 };
165 };
166 /*
167 * If the event is a per task event, this will point to the task in
168 * question. See the comment in perf_event_alloc().
169 */
170 struct task_struct *target;
171
172 /*
173 * PMU would store hardware filter configuration
174 * here.
175 */
176 void *addr_filters;
177
178 /* Last sync'ed generation of filters */
179 unsigned long addr_filters_gen;
180
181/*
182 * hw_perf_event::state flags; used to track the PERF_EF_* state.
183 */
184#define PERF_HES_STOPPED 0x01 /* the counter is stopped */
185#define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */
186#define PERF_HES_ARCH 0x04
187
188 int state;
189
190 /*
191 * The last observed hardware counter value, updated with a
192 * local64_cmpxchg() such that pmu::read() can be called nested.
193 */
194 local64_t prev_count;
195
196 /*
197 * The period to start the next sample with.
198 */
199 u64 sample_period;
200
201 /*
202 * The period we started this sample with.
203 */
204 u64 last_period;
205
206 /*
207 * However much is left of the current period; note that this is
208 * a full 64bit value and allows for generation of periods longer
209 * than hardware might allow.
210 */
211 local64_t period_left;
212
213 /*
214 * State for throttling the event, see __perf_event_overflow() and
215 * perf_adjust_freq_unthr_context().
216 */
217 u64 interrupts_seq;
218 u64 interrupts;
219
220 /*
221 * State for freq target events, see __perf_event_overflow() and
222 * perf_adjust_freq_unthr_context().
223 */
224 u64 freq_time_stamp;
225 u64 freq_count_stamp;
226#endif
227};
228
229struct perf_event;
230
231/*
232 * Common implementation detail of pmu::{start,commit,cancel}_txn
233 */
234#define PERF_PMU_TXN_ADD 0x1 /* txn to add/schedule event on PMU */
235#define PERF_PMU_TXN_READ 0x2 /* txn to read event group from PMU */
236
237/**
238 * pmu::capabilities flags
239 */
240#define PERF_PMU_CAP_NO_INTERRUPT 0x01
241#define PERF_PMU_CAP_NO_NMI 0x02
242#define PERF_PMU_CAP_AUX_NO_SG 0x04
243#define PERF_PMU_CAP_AUX_SW_DOUBLEBUF 0x08
244#define PERF_PMU_CAP_EXCLUSIVE 0x10
245#define PERF_PMU_CAP_ITRACE 0x20
246#define PERF_PMU_CAP_HETEROGENEOUS_CPUS 0x40
247#define PERF_PMU_CAP_NO_EXCLUDE 0x80
248
249/**
250 * struct pmu - generic performance monitoring unit
251 */
252struct pmu {
253 struct list_head entry;
254
255 struct module *module;
256 struct device *dev;
257 const struct attribute_group **attr_groups;
258 const char *name;
259 int type;
260
261 /*
262 * various common per-pmu feature flags
263 */
264 int capabilities;
265
266 int __percpu *pmu_disable_count;
267 struct perf_cpu_context __percpu *pmu_cpu_context;
268 atomic_t exclusive_cnt; /* < 0: cpu; > 0: tsk */
269 int task_ctx_nr;
270 int hrtimer_interval_ms;
271
272 /* number of address filters this PMU can do */
273 unsigned int nr_addr_filters;
274
275 /*
276 * Fully disable/enable this PMU, can be used to protect from the PMI
277 * as well as for lazy/batch writing of the MSRs.
278 */
279 void (*pmu_enable) (struct pmu *pmu); /* optional */
280 void (*pmu_disable) (struct pmu *pmu); /* optional */
281
282 /*
283 * Try and initialize the event for this PMU.
284 *
285 * Returns:
286 * -ENOENT -- @event is not for this PMU
287 *
288 * -ENODEV -- @event is for this PMU but PMU not present
289 * -EBUSY -- @event is for this PMU but PMU temporarily unavailable
290 * -EINVAL -- @event is for this PMU but @event is not valid
291 * -EOPNOTSUPP -- @event is for this PMU, @event is valid, but not supported
292 * -EACCESS -- @event is for this PMU, @event is valid, but no privilidges
293 *
294 * 0 -- @event is for this PMU and valid
295 *
296 * Other error return values are allowed.
297 */
298 int (*event_init) (struct perf_event *event);
299
300 /*
301 * Notification that the event was mapped or unmapped. Called
302 * in the context of the mapping task.
303 */
304 void (*event_mapped) (struct perf_event *event, struct mm_struct *mm); /* optional */
305 void (*event_unmapped) (struct perf_event *event, struct mm_struct *mm); /* optional */
306
307 /*
308 * Flags for ->add()/->del()/ ->start()/->stop(). There are
309 * matching hw_perf_event::state flags.
310 */
311#define PERF_EF_START 0x01 /* start the counter when adding */
312#define PERF_EF_RELOAD 0x02 /* reload the counter when starting */
313#define PERF_EF_UPDATE 0x04 /* update the counter when stopping */
314
315 /*
316 * Adds/Removes a counter to/from the PMU, can be done inside a
317 * transaction, see the ->*_txn() methods.
318 *
319 * The add/del callbacks will reserve all hardware resources required
320 * to service the event, this includes any counter constraint
321 * scheduling etc.
322 *
323 * Called with IRQs disabled and the PMU disabled on the CPU the event
324 * is on.
325 *
326 * ->add() called without PERF_EF_START should result in the same state
327 * as ->add() followed by ->stop().
328 *
329 * ->del() must always PERF_EF_UPDATE stop an event. If it calls
330 * ->stop() that must deal with already being stopped without
331 * PERF_EF_UPDATE.
332 */
333 int (*add) (struct perf_event *event, int flags);
334 void (*del) (struct perf_event *event, int flags);
335
336 /*
337 * Starts/Stops a counter present on the PMU.
338 *
339 * The PMI handler should stop the counter when perf_event_overflow()
340 * returns !0. ->start() will be used to continue.
341 *
342 * Also used to change the sample period.
343 *
344 * Called with IRQs disabled and the PMU disabled on the CPU the event
345 * is on -- will be called from NMI context with the PMU generates
346 * NMIs.
347 *
348 * ->stop() with PERF_EF_UPDATE will read the counter and update
349 * period/count values like ->read() would.
350 *
351 * ->start() with PERF_EF_RELOAD will reprogram the the counter
352 * value, must be preceded by a ->stop() with PERF_EF_UPDATE.
353 */
354 void (*start) (struct perf_event *event, int flags);
355 void (*stop) (struct perf_event *event, int flags);
356
357 /*
358 * Updates the counter value of the event.
359 *
360 * For sampling capable PMUs this will also update the software period
361 * hw_perf_event::period_left field.
362 */
363 void (*read) (struct perf_event *event);
364
365 /*
366 * Group events scheduling is treated as a transaction, add
367 * group events as a whole and perform one schedulability test.
368 * If the test fails, roll back the whole group
369 *
370 * Start the transaction, after this ->add() doesn't need to
371 * do schedulability tests.
372 *
373 * Optional.
374 */
375 void (*start_txn) (struct pmu *pmu, unsigned int txn_flags);
376 /*
377 * If ->start_txn() disabled the ->add() schedulability test
378 * then ->commit_txn() is required to perform one. On success
379 * the transaction is closed. On error the transaction is kept
380 * open until ->cancel_txn() is called.
381 *
382 * Optional.
383 */
384 int (*commit_txn) (struct pmu *pmu);
385 /*
386 * Will cancel the transaction, assumes ->del() is called
387 * for each successful ->add() during the transaction.
388 *
389 * Optional.
390 */
391 void (*cancel_txn) (struct pmu *pmu);
392
393 /*
394 * Will return the value for perf_event_mmap_page::index for this event,
395 * if no implementation is provided it will default to: event->hw.idx + 1.
396 */
397 int (*event_idx) (struct perf_event *event); /*optional */
398
399 /*
400 * context-switches callback
401 */
402 void (*sched_task) (struct perf_event_context *ctx,
403 bool sched_in);
404 /*
405 * PMU specific data size
406 */
407 size_t task_ctx_size;
408
409
410 /*
411 * Set up pmu-private data structures for an AUX area
412 */
413 void *(*setup_aux) (struct perf_event *event, void **pages,
414 int nr_pages, bool overwrite);
415 /* optional */
416
417 /*
418 * Free pmu-private AUX data structures
419 */
420 void (*free_aux) (void *aux); /* optional */
421
422 /*
423 * Validate address range filters: make sure the HW supports the
424 * requested configuration and number of filters; return 0 if the
425 * supplied filters are valid, -errno otherwise.
426 *
427 * Runs in the context of the ioctl()ing process and is not serialized
428 * with the rest of the PMU callbacks.
429 */
430 int (*addr_filters_validate) (struct list_head *filters);
431 /* optional */
432
433 /*
434 * Synchronize address range filter configuration:
435 * translate hw-agnostic filters into hardware configuration in
436 * event::hw::addr_filters.
437 *
438 * Runs as a part of filter sync sequence that is done in ->start()
439 * callback by calling perf_event_addr_filters_sync().
440 *
441 * May (and should) traverse event::addr_filters::list, for which its
442 * caller provides necessary serialization.
443 */
444 void (*addr_filters_sync) (struct perf_event *event);
445 /* optional */
446
447 /*
448 * Filter events for PMU-specific reasons.
449 */
450 int (*filter_match) (struct perf_event *event); /* optional */
451
452 /*
453 * Check period value for PERF_EVENT_IOC_PERIOD ioctl.
454 */
455 int (*check_period) (struct perf_event *event, u64 value); /* optional */
456};
457
458enum perf_addr_filter_action_t {
459 PERF_ADDR_FILTER_ACTION_STOP = 0,
460 PERF_ADDR_FILTER_ACTION_START,
461 PERF_ADDR_FILTER_ACTION_FILTER,
462};
463
464/**
465 * struct perf_addr_filter - address range filter definition
466 * @entry: event's filter list linkage
467 * @inode: object file's inode for file-based filters
468 * @offset: filter range offset
469 * @size: filter range size (size==0 means single address trigger)
470 * @action: filter/start/stop
471 *
472 * This is a hardware-agnostic filter configuration as specified by the user.
473 */
474struct perf_addr_filter {
475 struct list_head entry;
476 struct path path;
477 unsigned long offset;
478 unsigned long size;
479 enum perf_addr_filter_action_t action;
480};
481
482/**
483 * struct perf_addr_filters_head - container for address range filters
484 * @list: list of filters for this event
485 * @lock: spinlock that serializes accesses to the @list and event's
486 * (and its children's) filter generations.
487 * @nr_file_filters: number of file-based filters
488 *
489 * A child event will use parent's @list (and therefore @lock), so they are
490 * bundled together; see perf_event_addr_filters().
491 */
492struct perf_addr_filters_head {
493 struct list_head list;
494 raw_spinlock_t lock;
495 unsigned int nr_file_filters;
496};
497
498struct perf_addr_filter_range {
499 unsigned long start;
500 unsigned long size;
501};
502
503/**
504 * enum perf_event_state - the states of an event:
505 */
506enum perf_event_state {
507 PERF_EVENT_STATE_DEAD = -4,
508 PERF_EVENT_STATE_EXIT = -3,
509 PERF_EVENT_STATE_ERROR = -2,
510 PERF_EVENT_STATE_OFF = -1,
511 PERF_EVENT_STATE_INACTIVE = 0,
512 PERF_EVENT_STATE_ACTIVE = 1,
513};
514
515struct file;
516struct perf_sample_data;
517
518typedef void (*perf_overflow_handler_t)(struct perf_event *,
519 struct perf_sample_data *,
520 struct pt_regs *regs);
521
522/*
523 * Event capabilities. For event_caps and groups caps.
524 *
525 * PERF_EV_CAP_SOFTWARE: Is a software event.
526 * PERF_EV_CAP_READ_ACTIVE_PKG: A CPU event (or cgroup event) that can be read
527 * from any CPU in the package where it is active.
528 */
529#define PERF_EV_CAP_SOFTWARE BIT(0)
530#define PERF_EV_CAP_READ_ACTIVE_PKG BIT(1)
531
532#define SWEVENT_HLIST_BITS 8
533#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
534
535struct swevent_hlist {
536 struct hlist_head heads[SWEVENT_HLIST_SIZE];
537 struct rcu_head rcu_head;
538};
539
540#define PERF_ATTACH_CONTEXT 0x01
541#define PERF_ATTACH_GROUP 0x02
542#define PERF_ATTACH_TASK 0x04
543#define PERF_ATTACH_TASK_DATA 0x08
544#define PERF_ATTACH_ITRACE 0x10
545
546struct perf_cgroup;
547struct ring_buffer;
548
549struct pmu_event_list {
550 raw_spinlock_t lock;
551 struct list_head list;
552};
553
554#define for_each_sibling_event(sibling, event) \
555 if ((event)->group_leader == (event)) \
556 list_for_each_entry((sibling), &(event)->sibling_list, sibling_list)
557
558/**
559 * struct perf_event - performance event kernel representation:
560 */
561struct perf_event {
562#ifdef CONFIG_PERF_EVENTS
563 /*
564 * entry onto perf_event_context::event_list;
565 * modifications require ctx->lock
566 * RCU safe iterations.
567 */
568 struct list_head event_entry;
569
570 /*
571 * Locked for modification by both ctx->mutex and ctx->lock; holding
572 * either sufficies for read.
573 */
574 struct list_head sibling_list;
575 struct list_head active_list;
576 /*
577 * Node on the pinned or flexible tree located at the event context;
578 */
579 struct rb_node group_node;
580 u64 group_index;
581 /*
582 * We need storage to track the entries in perf_pmu_migrate_context; we
583 * cannot use the event_entry because of RCU and we want to keep the
584 * group in tact which avoids us using the other two entries.
585 */
586 struct list_head migrate_entry;
587
588 struct hlist_node hlist_entry;
589 struct list_head active_entry;
590 int nr_siblings;
591
592 /* Not serialized. Only written during event initialization. */
593 int event_caps;
594 /* The cumulative AND of all event_caps for events in this group. */
595 int group_caps;
596
597 struct perf_event *group_leader;
598 struct pmu *pmu;
599 void *pmu_private;
600
601 enum perf_event_state state;
602 unsigned int attach_state;
603 local64_t count;
604 atomic64_t child_count;
605
606 /*
607 * These are the total time in nanoseconds that the event
608 * has been enabled (i.e. eligible to run, and the task has
609 * been scheduled in, if this is a per-task event)
610 * and running (scheduled onto the CPU), respectively.
611 */
612 u64 total_time_enabled;
613 u64 total_time_running;
614 u64 tstamp;
615
616 /*
617 * timestamp shadows the actual context timing but it can
618 * be safely used in NMI interrupt context. It reflects the
619 * context time as it was when the event was last scheduled in.
620 *
621 * ctx_time already accounts for ctx->timestamp. Therefore to
622 * compute ctx_time for a sample, simply add perf_clock().
623 */
624 u64 shadow_ctx_time;
625
626 struct perf_event_attr attr;
627 u16 header_size;
628 u16 id_header_size;
629 u16 read_size;
630 struct hw_perf_event hw;
631
632 struct perf_event_context *ctx;
633 atomic_long_t refcount;
634
635 /*
636 * These accumulate total time (in nanoseconds) that children
637 * events have been enabled and running, respectively.
638 */
639 atomic64_t child_total_time_enabled;
640 atomic64_t child_total_time_running;
641
642 /*
643 * Protect attach/detach and child_list:
644 */
645 struct mutex child_mutex;
646 struct list_head child_list;
647 struct perf_event *parent;
648
649 int oncpu;
650 int cpu;
651
652 struct list_head owner_entry;
653 struct task_struct *owner;
654
655 /* mmap bits */
656 struct mutex mmap_mutex;
657 atomic_t mmap_count;
658
659 struct ring_buffer *rb;
660 struct list_head rb_entry;
661 unsigned long rcu_batches;
662 int rcu_pending;
663
664 /* poll related */
665 wait_queue_head_t waitq;
666 struct fasync_struct *fasync;
667
668 /* delayed work for NMIs and such */
669 int pending_wakeup;
670 int pending_kill;
671 int pending_disable;
672 struct irq_work pending;
673
674 atomic_t event_limit;
675
676 /* address range filters */
677 struct perf_addr_filters_head addr_filters;
678 /* vma address array for file-based filders */
679 struct perf_addr_filter_range *addr_filter_ranges;
680 unsigned long addr_filters_gen;
681
682 void (*destroy)(struct perf_event *);
683 struct rcu_head rcu_head;
684
685 struct pid_namespace *ns;
686 u64 id;
687
688 u64 (*clock)(void);
689 perf_overflow_handler_t overflow_handler;
690 void *overflow_handler_context;
691#ifdef CONFIG_BPF_SYSCALL
692 perf_overflow_handler_t orig_overflow_handler;
693 struct bpf_prog *prog;
694#endif
695
696#ifdef CONFIG_EVENT_TRACING
697 struct trace_event_call *tp_event;
698 struct event_filter *filter;
699#ifdef CONFIG_FUNCTION_TRACER
700 struct ftrace_ops ftrace_ops;
701#endif
702#endif
703
704#ifdef CONFIG_CGROUP_PERF
705 struct perf_cgroup *cgrp; /* cgroup event is attach to */
706#endif
707
708 struct list_head sb_list;
709#endif /* CONFIG_PERF_EVENTS */
710};
711
712
713struct perf_event_groups {
714 struct rb_root tree;
715 u64 index;
716};
717
718/**
719 * struct perf_event_context - event context structure
720 *
721 * Used as a container for task events and CPU events as well:
722 */
723struct perf_event_context {
724 struct pmu *pmu;
725 /*
726 * Protect the states of the events in the list,
727 * nr_active, and the list:
728 */
729 raw_spinlock_t lock;
730 /*
731 * Protect the list of events. Locking either mutex or lock
732 * is sufficient to ensure the list doesn't change; to change
733 * the list you need to lock both the mutex and the spinlock.
734 */
735 struct mutex mutex;
736
737 struct list_head active_ctx_list;
738 struct perf_event_groups pinned_groups;
739 struct perf_event_groups flexible_groups;
740 struct list_head event_list;
741
742 struct list_head pinned_active;
743 struct list_head flexible_active;
744
745 int nr_events;
746 int nr_active;
747 int is_active;
748 int nr_stat;
749 int nr_freq;
750 int rotate_disable;
751 refcount_t refcount;
752 struct task_struct *task;
753
754 /*
755 * Context clock, runs when context enabled.
756 */
757 u64 time;
758 u64 timestamp;
759
760 /*
761 * These fields let us detect when two contexts have both
762 * been cloned (inherited) from a common ancestor.
763 */
764 struct perf_event_context *parent_ctx;
765 u64 parent_gen;
766 u64 generation;
767 int pin_count;
768#ifdef CONFIG_CGROUP_PERF
769 int nr_cgroups; /* cgroup evts */
770#endif
771 void *task_ctx_data; /* pmu specific data */
772 struct rcu_head rcu_head;
773};
774
775/*
776 * Number of contexts where an event can trigger:
777 * task, softirq, hardirq, nmi.
778 */
779#define PERF_NR_CONTEXTS 4
780
781/**
782 * struct perf_event_cpu_context - per cpu event context structure
783 */
784struct perf_cpu_context {
785 struct perf_event_context ctx;
786 struct perf_event_context *task_ctx;
787 int active_oncpu;
788 int exclusive;
789
790 raw_spinlock_t hrtimer_lock;
791 struct hrtimer hrtimer;
792 ktime_t hrtimer_interval;
793 unsigned int hrtimer_active;
794
795#ifdef CONFIG_CGROUP_PERF
796 struct perf_cgroup *cgrp;
797 struct list_head cgrp_cpuctx_entry;
798#endif
799
800 struct list_head sched_cb_entry;
801 int sched_cb_usage;
802
803 int online;
804};
805
806struct perf_output_handle {
807 struct perf_event *event;
808 struct ring_buffer *rb;
809 unsigned long wakeup;
810 unsigned long size;
811 u64 aux_flags;
812 union {
813 void *addr;
814 unsigned long head;
815 };
816 int page;
817};
818
819struct bpf_perf_event_data_kern {
820 bpf_user_pt_regs_t *regs;
821 struct perf_sample_data *data;
822 struct perf_event *event;
823};
824
825#ifdef CONFIG_CGROUP_PERF
826
827/*
828 * perf_cgroup_info keeps track of time_enabled for a cgroup.
829 * This is a per-cpu dynamically allocated data structure.
830 */
831struct perf_cgroup_info {
832 u64 time;
833 u64 timestamp;
834};
835
836struct perf_cgroup {
837 struct cgroup_subsys_state css;
838 struct perf_cgroup_info __percpu *info;
839};
840
841/*
842 * Must ensure cgroup is pinned (css_get) before calling
843 * this function. In other words, we cannot call this function
844 * if there is no cgroup event for the current CPU context.
845 */
846static inline struct perf_cgroup *
847perf_cgroup_from_task(struct task_struct *task, struct perf_event_context *ctx)
848{
849 return container_of(task_css_check(task, perf_event_cgrp_id,
850 ctx ? lockdep_is_held(&ctx->lock)
851 : true),
852 struct perf_cgroup, css);
853}
854#endif /* CONFIG_CGROUP_PERF */
855
856#ifdef CONFIG_PERF_EVENTS
857
858extern void *perf_aux_output_begin(struct perf_output_handle *handle,
859 struct perf_event *event);
860extern void perf_aux_output_end(struct perf_output_handle *handle,
861 unsigned long size);
862extern int perf_aux_output_skip(struct perf_output_handle *handle,
863 unsigned long size);
864extern void *perf_get_aux(struct perf_output_handle *handle);
865extern void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags);
866extern void perf_event_itrace_started(struct perf_event *event);
867
868extern int perf_pmu_register(struct pmu *pmu, const char *name, int type);
869extern void perf_pmu_unregister(struct pmu *pmu);
870
871extern int perf_num_counters(void);
872extern const char *perf_pmu_name(void);
873extern void __perf_event_task_sched_in(struct task_struct *prev,
874 struct task_struct *task);
875extern void __perf_event_task_sched_out(struct task_struct *prev,
876 struct task_struct *next);
877extern int perf_event_init_task(struct task_struct *child);
878extern void perf_event_exit_task(struct task_struct *child);
879extern void perf_event_free_task(struct task_struct *task);
880extern void perf_event_delayed_put(struct task_struct *task);
881extern struct file *perf_event_get(unsigned int fd);
882extern const struct perf_event *perf_get_event(struct file *file);
883extern const struct perf_event_attr *perf_event_attrs(struct perf_event *event);
884extern void perf_event_print_debug(void);
885extern void perf_pmu_disable(struct pmu *pmu);
886extern void perf_pmu_enable(struct pmu *pmu);
887extern void perf_sched_cb_dec(struct pmu *pmu);
888extern void perf_sched_cb_inc(struct pmu *pmu);
889extern int perf_event_task_disable(void);
890extern int perf_event_task_enable(void);
891extern int perf_event_refresh(struct perf_event *event, int refresh);
892extern void perf_event_update_userpage(struct perf_event *event);
893extern int perf_event_release_kernel(struct perf_event *event);
894extern struct perf_event *
895perf_event_create_kernel_counter(struct perf_event_attr *attr,
896 int cpu,
897 struct task_struct *task,
898 perf_overflow_handler_t callback,
899 void *context);
900extern void perf_pmu_migrate_context(struct pmu *pmu,
901 int src_cpu, int dst_cpu);
902int perf_event_read_local(struct perf_event *event, u64 *value,
903 u64 *enabled, u64 *running);
904extern u64 perf_event_read_value(struct perf_event *event,
905 u64 *enabled, u64 *running);
906
907
908struct perf_sample_data {
909 /*
910 * Fields set by perf_sample_data_init(), group so as to
911 * minimize the cachelines touched.
912 */
913 u64 addr;
914 struct perf_raw_record *raw;
915 struct perf_branch_stack *br_stack;
916 u64 period;
917 u64 weight;
918 u64 txn;
919 union perf_mem_data_src data_src;
920
921 /*
922 * The other fields, optionally {set,used} by
923 * perf_{prepare,output}_sample().
924 */
925 u64 type;
926 u64 ip;
927 struct {
928 u32 pid;
929 u32 tid;
930 } tid_entry;
931 u64 time;
932 u64 id;
933 u64 stream_id;
934 struct {
935 u32 cpu;
936 u32 reserved;
937 } cpu_entry;
938 struct perf_callchain_entry *callchain;
939
940 /*
941 * regs_user may point to task_pt_regs or to regs_user_copy, depending
942 * on arch details.
943 */
944 struct perf_regs regs_user;
945 struct pt_regs regs_user_copy;
946
947 struct perf_regs regs_intr;
948 u64 stack_user_size;
949
950 u64 phys_addr;
951} ____cacheline_aligned;
952
953/* default value for data source */
954#define PERF_MEM_NA (PERF_MEM_S(OP, NA) |\
955 PERF_MEM_S(LVL, NA) |\
956 PERF_MEM_S(SNOOP, NA) |\
957 PERF_MEM_S(LOCK, NA) |\
958 PERF_MEM_S(TLB, NA))
959
960static inline void perf_sample_data_init(struct perf_sample_data *data,
961 u64 addr, u64 period)
962{
963 /* remaining struct members initialized in perf_prepare_sample() */
964 data->addr = addr;
965 data->raw = NULL;
966 data->br_stack = NULL;
967 data->period = period;
968 data->weight = 0;
969 data->data_src.val = PERF_MEM_NA;
970 data->txn = 0;
971}
972
973extern void perf_output_sample(struct perf_output_handle *handle,
974 struct perf_event_header *header,
975 struct perf_sample_data *data,
976 struct perf_event *event);
977extern void perf_prepare_sample(struct perf_event_header *header,
978 struct perf_sample_data *data,
979 struct perf_event *event,
980 struct pt_regs *regs);
981
982extern int perf_event_overflow(struct perf_event *event,
983 struct perf_sample_data *data,
984 struct pt_regs *regs);
985
986extern void perf_event_output_forward(struct perf_event *event,
987 struct perf_sample_data *data,
988 struct pt_regs *regs);
989extern void perf_event_output_backward(struct perf_event *event,
990 struct perf_sample_data *data,
991 struct pt_regs *regs);
992extern int perf_event_output(struct perf_event *event,
993 struct perf_sample_data *data,
994 struct pt_regs *regs);
995
996static inline bool
997is_default_overflow_handler(struct perf_event *event)
998{
999 if (likely(event->overflow_handler == perf_event_output_forward))
1000 return true;
1001 if (unlikely(event->overflow_handler == perf_event_output_backward))
1002 return true;
1003 return false;
1004}
1005
1006extern void
1007perf_event_header__init_id(struct perf_event_header *header,
1008 struct perf_sample_data *data,
1009 struct perf_event *event);
1010extern void
1011perf_event__output_id_sample(struct perf_event *event,
1012 struct perf_output_handle *handle,
1013 struct perf_sample_data *sample);
1014
1015extern void
1016perf_log_lost_samples(struct perf_event *event, u64 lost);
1017
1018static inline bool event_has_any_exclude_flag(struct perf_event *event)
1019{
1020 struct perf_event_attr *attr = &event->attr;
1021
1022 return attr->exclude_idle || attr->exclude_user ||
1023 attr->exclude_kernel || attr->exclude_hv ||
1024 attr->exclude_guest || attr->exclude_host;
1025}
1026
1027static inline bool is_sampling_event(struct perf_event *event)
1028{
1029 return event->attr.sample_period != 0;
1030}
1031
1032/*
1033 * Return 1 for a software event, 0 for a hardware event
1034 */
1035static inline int is_software_event(struct perf_event *event)
1036{
1037 return event->event_caps & PERF_EV_CAP_SOFTWARE;
1038}
1039
1040/*
1041 * Return 1 for event in sw context, 0 for event in hw context
1042 */
1043static inline int in_software_context(struct perf_event *event)
1044{
1045 return event->ctx->pmu->task_ctx_nr == perf_sw_context;
1046}
1047
1048extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
1049
1050extern void ___perf_sw_event(u32, u64, struct pt_regs *, u64);
1051extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
1052
1053#ifndef perf_arch_fetch_caller_regs
1054static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
1055#endif
1056
1057/*
1058 * Take a snapshot of the regs. Skip ip and frame pointer to
1059 * the nth caller. We only need a few of the regs:
1060 * - ip for PERF_SAMPLE_IP
1061 * - cs for user_mode() tests
1062 * - bp for callchains
1063 * - eflags, for future purposes, just in case
1064 */
1065static inline void perf_fetch_caller_regs(struct pt_regs *regs)
1066{
1067 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
1068}
1069
1070static __always_inline void
1071perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
1072{
1073 if (static_key_false(&perf_swevent_enabled[event_id]))
1074 __perf_sw_event(event_id, nr, regs, addr);
1075}
1076
1077DECLARE_PER_CPU(struct pt_regs, __perf_regs[4]);
1078
1079/*
1080 * 'Special' version for the scheduler, it hard assumes no recursion,
1081 * which is guaranteed by us not actually scheduling inside other swevents
1082 * because those disable preemption.
1083 */
1084static __always_inline void
1085perf_sw_event_sched(u32 event_id, u64 nr, u64 addr)
1086{
1087 if (static_key_false(&perf_swevent_enabled[event_id])) {
1088 struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]);
1089
1090 perf_fetch_caller_regs(regs);
1091 ___perf_sw_event(event_id, nr, regs, addr);
1092 }
1093}
1094
1095extern struct static_key_false perf_sched_events;
1096
1097static __always_inline bool
1098perf_sw_migrate_enabled(void)
1099{
1100 if (static_key_false(&perf_swevent_enabled[PERF_COUNT_SW_CPU_MIGRATIONS]))
1101 return true;
1102 return false;
1103}
1104
1105static inline void perf_event_task_migrate(struct task_struct *task)
1106{
1107 if (perf_sw_migrate_enabled())
1108 task->sched_migrated = 1;
1109}
1110
1111static inline void perf_event_task_sched_in(struct task_struct *prev,
1112 struct task_struct *task)
1113{
1114 if (static_branch_unlikely(&perf_sched_events))
1115 __perf_event_task_sched_in(prev, task);
1116
1117 if (perf_sw_migrate_enabled() && task->sched_migrated) {
1118 struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]);
1119
1120 perf_fetch_caller_regs(regs);
1121 ___perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, regs, 0);
1122 task->sched_migrated = 0;
1123 }
1124}
1125
1126static inline void perf_event_task_sched_out(struct task_struct *prev,
1127 struct task_struct *next)
1128{
1129 perf_sw_event_sched(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 0);
1130
1131 if (static_branch_unlikely(&perf_sched_events))
1132 __perf_event_task_sched_out(prev, next);
1133}
1134
1135extern void perf_event_mmap(struct vm_area_struct *vma);
1136
1137extern void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len,
1138 bool unregister, const char *sym);
1139extern void perf_event_bpf_event(struct bpf_prog *prog,
1140 enum perf_bpf_event_type type,
1141 u16 flags);
1142
1143extern struct perf_guest_info_callbacks *perf_guest_cbs;
1144extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
1145extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
1146
1147extern void perf_event_exec(void);
1148extern void perf_event_comm(struct task_struct *tsk, bool exec);
1149extern void perf_event_namespaces(struct task_struct *tsk);
1150extern void perf_event_fork(struct task_struct *tsk);
1151
1152/* Callchains */
1153DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
1154
1155extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
1156extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
1157extern struct perf_callchain_entry *
1158get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
1159 u32 max_stack, bool crosstask, bool add_mark);
1160extern struct perf_callchain_entry *perf_callchain(struct perf_event *event, struct pt_regs *regs);
1161extern int get_callchain_buffers(int max_stack);
1162extern void put_callchain_buffers(void);
1163
1164extern int sysctl_perf_event_max_stack;
1165extern int sysctl_perf_event_max_contexts_per_stack;
1166
1167static inline int perf_callchain_store_context(struct perf_callchain_entry_ctx *ctx, u64 ip)
1168{
1169 if (ctx->contexts < sysctl_perf_event_max_contexts_per_stack) {
1170 struct perf_callchain_entry *entry = ctx->entry;
1171 entry->ip[entry->nr++] = ip;
1172 ++ctx->contexts;
1173 return 0;
1174 } else {
1175 ctx->contexts_maxed = true;
1176 return -1; /* no more room, stop walking the stack */
1177 }
1178}
1179
1180static inline int perf_callchain_store(struct perf_callchain_entry_ctx *ctx, u64 ip)
1181{
1182 if (ctx->nr < ctx->max_stack && !ctx->contexts_maxed) {
1183 struct perf_callchain_entry *entry = ctx->entry;
1184 entry->ip[entry->nr++] = ip;
1185 ++ctx->nr;
1186 return 0;
1187 } else {
1188 return -1; /* no more room, stop walking the stack */
1189 }
1190}
1191
1192extern int sysctl_perf_event_paranoid;
1193extern int sysctl_perf_event_mlock;
1194extern int sysctl_perf_event_sample_rate;
1195extern int sysctl_perf_cpu_time_max_percent;
1196
1197extern void perf_sample_event_took(u64 sample_len_ns);
1198
1199extern int perf_proc_update_handler(struct ctl_table *table, int write,
1200 void __user *buffer, size_t *lenp,
1201 loff_t *ppos);
1202extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
1203 void __user *buffer, size_t *lenp,
1204 loff_t *ppos);
1205
1206int perf_event_max_stack_handler(struct ctl_table *table, int write,
1207 void __user *buffer, size_t *lenp, loff_t *ppos);
1208
1209static inline bool perf_paranoid_tracepoint_raw(void)
1210{
1211 return sysctl_perf_event_paranoid > -1;
1212}
1213
1214static inline bool perf_paranoid_cpu(void)
1215{
1216 return sysctl_perf_event_paranoid > 0;
1217}
1218
1219static inline bool perf_paranoid_kernel(void)
1220{
1221 return sysctl_perf_event_paranoid > 1;
1222}
1223
1224extern void perf_event_init(void);
1225extern void perf_tp_event(u16 event_type, u64 count, void *record,
1226 int entry_size, struct pt_regs *regs,
1227 struct hlist_head *head, int rctx,
1228 struct task_struct *task);
1229extern void perf_bp_event(struct perf_event *event, void *data);
1230
1231#ifndef perf_misc_flags
1232# define perf_misc_flags(regs) \
1233 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
1234# define perf_instruction_pointer(regs) instruction_pointer(regs)
1235#endif
1236#ifndef perf_arch_bpf_user_pt_regs
1237# define perf_arch_bpf_user_pt_regs(regs) regs
1238#endif
1239
1240static inline bool has_branch_stack(struct perf_event *event)
1241{
1242 return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
1243}
1244
1245static inline bool needs_branch_stack(struct perf_event *event)
1246{
1247 return event->attr.branch_sample_type != 0;
1248}
1249
1250static inline bool has_aux(struct perf_event *event)
1251{
1252 return event->pmu->setup_aux;
1253}
1254
1255static inline bool is_write_backward(struct perf_event *event)
1256{
1257 return !!event->attr.write_backward;
1258}
1259
1260static inline bool has_addr_filter(struct perf_event *event)
1261{
1262 return event->pmu->nr_addr_filters;
1263}
1264
1265/*
1266 * An inherited event uses parent's filters
1267 */
1268static inline struct perf_addr_filters_head *
1269perf_event_addr_filters(struct perf_event *event)
1270{
1271 struct perf_addr_filters_head *ifh = &event->addr_filters;
1272
1273 if (event->parent)
1274 ifh = &event->parent->addr_filters;
1275
1276 return ifh;
1277}
1278
1279extern void perf_event_addr_filters_sync(struct perf_event *event);
1280
1281extern int perf_output_begin(struct perf_output_handle *handle,
1282 struct perf_event *event, unsigned int size);
1283extern int perf_output_begin_forward(struct perf_output_handle *handle,
1284 struct perf_event *event,
1285 unsigned int size);
1286extern int perf_output_begin_backward(struct perf_output_handle *handle,
1287 struct perf_event *event,
1288 unsigned int size);
1289
1290extern void perf_output_end(struct perf_output_handle *handle);
1291extern unsigned int perf_output_copy(struct perf_output_handle *handle,
1292 const void *buf, unsigned int len);
1293extern unsigned int perf_output_skip(struct perf_output_handle *handle,
1294 unsigned int len);
1295extern int perf_swevent_get_recursion_context(void);
1296extern void perf_swevent_put_recursion_context(int rctx);
1297extern u64 perf_swevent_set_period(struct perf_event *event);
1298extern void perf_event_enable(struct perf_event *event);
1299extern void perf_event_disable(struct perf_event *event);
1300extern void perf_event_disable_local(struct perf_event *event);
1301extern void perf_event_disable_inatomic(struct perf_event *event);
1302extern void perf_event_task_tick(void);
1303extern int perf_event_account_interrupt(struct perf_event *event);
1304#else /* !CONFIG_PERF_EVENTS: */
1305static inline void *
1306perf_aux_output_begin(struct perf_output_handle *handle,
1307 struct perf_event *event) { return NULL; }
1308static inline void
1309perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)
1310 { }
1311static inline int
1312perf_aux_output_skip(struct perf_output_handle *handle,
1313 unsigned long size) { return -EINVAL; }
1314static inline void *
1315perf_get_aux(struct perf_output_handle *handle) { return NULL; }
1316static inline void
1317perf_event_task_migrate(struct task_struct *task) { }
1318static inline void
1319perf_event_task_sched_in(struct task_struct *prev,
1320 struct task_struct *task) { }
1321static inline void
1322perf_event_task_sched_out(struct task_struct *prev,
1323 struct task_struct *next) { }
1324static inline int perf_event_init_task(struct task_struct *child) { return 0; }
1325static inline void perf_event_exit_task(struct task_struct *child) { }
1326static inline void perf_event_free_task(struct task_struct *task) { }
1327static inline void perf_event_delayed_put(struct task_struct *task) { }
1328static inline struct file *perf_event_get(unsigned int fd) { return ERR_PTR(-EINVAL); }
1329static inline const struct perf_event *perf_get_event(struct file *file)
1330{
1331 return ERR_PTR(-EINVAL);
1332}
1333static inline const struct perf_event_attr *perf_event_attrs(struct perf_event *event)
1334{
1335 return ERR_PTR(-EINVAL);
1336}
1337static inline int perf_event_read_local(struct perf_event *event, u64 *value,
1338 u64 *enabled, u64 *running)
1339{
1340 return -EINVAL;
1341}
1342static inline void perf_event_print_debug(void) { }
1343static inline int perf_event_task_disable(void) { return -EINVAL; }
1344static inline int perf_event_task_enable(void) { return -EINVAL; }
1345static inline int perf_event_refresh(struct perf_event *event, int refresh)
1346{
1347 return -EINVAL;
1348}
1349
1350static inline void
1351perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { }
1352static inline void
1353perf_sw_event_sched(u32 event_id, u64 nr, u64 addr) { }
1354static inline void
1355perf_bp_event(struct perf_event *event, void *data) { }
1356
1357static inline int perf_register_guest_info_callbacks
1358(struct perf_guest_info_callbacks *callbacks) { return 0; }
1359static inline int perf_unregister_guest_info_callbacks
1360(struct perf_guest_info_callbacks *callbacks) { return 0; }
1361
1362static inline void perf_event_mmap(struct vm_area_struct *vma) { }
1363
1364typedef int (perf_ksymbol_get_name_f)(char *name, int name_len, void *data);
1365static inline void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len,
1366 bool unregister, const char *sym) { }
1367static inline void perf_event_bpf_event(struct bpf_prog *prog,
1368 enum perf_bpf_event_type type,
1369 u16 flags) { }
1370static inline void perf_event_exec(void) { }
1371static inline void perf_event_comm(struct task_struct *tsk, bool exec) { }
1372static inline void perf_event_namespaces(struct task_struct *tsk) { }
1373static inline void perf_event_fork(struct task_struct *tsk) { }
1374static inline void perf_event_init(void) { }
1375static inline int perf_swevent_get_recursion_context(void) { return -1; }
1376static inline void perf_swevent_put_recursion_context(int rctx) { }
1377static inline u64 perf_swevent_set_period(struct perf_event *event) { return 0; }
1378static inline void perf_event_enable(struct perf_event *event) { }
1379static inline void perf_event_disable(struct perf_event *event) { }
1380static inline int __perf_event_disable(void *info) { return -1; }
1381static inline void perf_event_task_tick(void) { }
1382static inline int perf_event_release_kernel(struct perf_event *event) { return 0; }
1383#endif
1384
1385#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
1386extern void perf_restore_debug_store(void);
1387#else
1388static inline void perf_restore_debug_store(void) { }
1389#endif
1390
1391static __always_inline bool perf_raw_frag_last(const struct perf_raw_frag *frag)
1392{
1393 return frag->pad < sizeof(u64);
1394}
1395
1396#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
1397
1398struct perf_pmu_events_attr {
1399 struct device_attribute attr;
1400 u64 id;
1401 const char *event_str;
1402};
1403
1404struct perf_pmu_events_ht_attr {
1405 struct device_attribute attr;
1406 u64 id;
1407 const char *event_str_ht;
1408 const char *event_str_noht;
1409};
1410
1411ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
1412 char *page);
1413
1414#define PMU_EVENT_ATTR(_name, _var, _id, _show) \
1415static struct perf_pmu_events_attr _var = { \
1416 .attr = __ATTR(_name, 0444, _show, NULL), \
1417 .id = _id, \
1418};
1419
1420#define PMU_EVENT_ATTR_STRING(_name, _var, _str) \
1421static struct perf_pmu_events_attr _var = { \
1422 .attr = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \
1423 .id = 0, \
1424 .event_str = _str, \
1425};
1426
1427#define PMU_FORMAT_ATTR(_name, _format) \
1428static ssize_t \
1429_name##_show(struct device *dev, \
1430 struct device_attribute *attr, \
1431 char *page) \
1432{ \
1433 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
1434 return sprintf(page, _format "\n"); \
1435} \
1436 \
1437static struct device_attribute format_attr_##_name = __ATTR_RO(_name)
1438
1439/* Performance counter hotplug functions */
1440#ifdef CONFIG_PERF_EVENTS
1441int perf_event_init_cpu(unsigned int cpu);
1442int perf_event_exit_cpu(unsigned int cpu);
1443#else
1444#define perf_event_init_cpu NULL
1445#define perf_event_exit_cpu NULL
1446#endif
1447
1448#endif /* _LINUX_PERF_EVENT_H */
1449