1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Performance events core code:
4	*
5	* Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
6	* Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7	* Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
8	* Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9	*/
10
11	#include <linux/fs.h>
12	#include <linux/mm.h>
13	#include <linux/cpu.h>
14	#include <linux/smp.h>
15	#include <linux/idr.h>
16	#include <linux/file.h>
17	#include <linux/poll.h>
18	#include <linux/slab.h>
19	#include <linux/hash.h>
20	#include <linux/tick.h>
21	#include <linux/sysfs.h>
22	#include <linux/dcache.h>
23	#include <linux/percpu.h>
24	#include <linux/ptrace.h>
25	#include <linux/reboot.h>
26	#include <linux/vmstat.h>
27	#include <linux/device.h>
28	#include <linux/export.h>
29	#include <linux/vmalloc.h>
30	#include <linux/hardirq.h>
31	#include <linux/hugetlb.h>
32	#include <linux/rculist.h>
33	#include <linux/uaccess.h>
34	#include <linux/syscalls.h>
35	#include <linux/anon_inodes.h>
36	#include <linux/kernel_stat.h>
37	#include <linux/cgroup.h>
38	#include <linux/perf_event.h>
39	#include <linux/trace_events.h>
40	#include <linux/hw_breakpoint.h>
41	#include <linux/mm_types.h>
42	#include <linux/module.h>
43	#include <linux/mman.h>
44	#include <linux/compat.h>
45	#include <linux/bpf.h>
46	#include <linux/filter.h>
47	#include <linux/namei.h>
48	#include <linux/parser.h>
49	#include <linux/sched/clock.h>
50	#include <linux/sched/mm.h>
51	#include <linux/proc_ns.h>
52	#include <linux/mount.h>
53	#include <linux/min_heap.h>
54	#include <linux/highmem.h>
55	#include <linux/pgtable.h>
56	#include <linux/buildid.h>
57	#include <linux/task_work.h>
58
59	#include "internal.h"
60
61	#include <asm/irq_regs.h>
62
63	typedef int (*remote_function_f)(void *);
64
65	struct remote_function_call {
66	struct task_struct *p;
67	remote_function_f func;
68	void *info;
69	int ret;
70	};
71
72	static void remote_function(void *data)
73	{
74	struct remote_function_call *tfc = data;
75	struct task_struct *p = tfc->p;
76
77	if (p) {
78	/* -EAGAIN */
79	if (task_cpu(p) != smp_processor_id())
80	return;
81
82	/*
83	* Now that we're on right CPU with IRQs disabled, we can test
84	* if we hit the right task without races.
85	*/
86
87	tfc->ret = -ESRCH; /* No such (running) process */
88	if (p != current)
89	return;
90	}
91
92	tfc->ret = tfc->func(tfc->info);
93	}
94
95	/**
96	* task_function_call - call a function on the cpu on which a task runs
97	* @p: the task to evaluate
98	* @func: the function to be called
99	* @info: the function call argument
100	*
101	* Calls the function @func when the task is currently running. This might
102	* be on the current CPU, which just calls the function directly. This will
103	* retry due to any failures in smp_call_function_single(), such as if the
104	* task_cpu() goes offline concurrently.
105	*
106	* returns @func return value or -ESRCH or -ENXIO when the process isn't running
107	*/
108	static int
109	task_function_call(struct task_struct *p, remote_function_f func, void *info)
110	{
111	struct remote_function_call data = {
112	.p = p,
113	.func = func,
114	.info = info,
115	.ret = -EAGAIN,
116	};
117	int ret;
118
119	for (;;) {
120	ret = smp_call_function_single(cpuid: task_cpu(p), func: remote_function,
121	info: &data, wait: 1);
122	if (!ret)
123	ret = data.ret;
124
125	if (ret != -EAGAIN)
126	break;
127
128	cond_resched();
129	}
130
131	return ret;
132	}
133
134	/**
135	* cpu_function_call - call a function on the cpu
136	* @cpu: target cpu to queue this function
137	* @func: the function to be called
138	* @info: the function call argument
139	*
140	* Calls the function @func on the remote cpu.
141	*
142	* returns: @func return value or -ENXIO when the cpu is offline
143	*/
144	static int cpu_function_call(int cpu, remote_function_f func, void *info)
145	{
146	struct remote_function_call data = {
147	.p = NULL,
148	.func = func,
149	.info = info,
150	.ret = -ENXIO, /* No such CPU */
151	};
152
153	smp_call_function_single(cpuid: cpu, func: remote_function, info: &data, wait: 1);
154
155	return data.ret;
156	}
157
158	static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
159	struct perf_event_context *ctx)
160	{
161	raw_spin_lock(&cpuctx->ctx.lock);
162	if (ctx)
163	raw_spin_lock(&ctx->lock);
164	}
165
166	static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
167	struct perf_event_context *ctx)
168	{
169	if (ctx)
170	raw_spin_unlock(&ctx->lock);
171	raw_spin_unlock(&cpuctx->ctx.lock);
172	}
173
174	#define TASK_TOMBSTONE ((void *)-1L)
175
176	static bool is_kernel_event(struct perf_event *event)
177	{
178	return READ_ONCE(event->owner) == TASK_TOMBSTONE;
179	}
180
181	static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
182
183	struct perf_event_context *perf_cpu_task_ctx(void)
184	{
185	lockdep_assert_irqs_disabled();
186	return this_cpu_ptr(&perf_cpu_context)->task_ctx;
187	}
188
189	/*
190	* On task ctx scheduling...
191	*
192	* When !ctx->nr_events a task context will not be scheduled. This means
193	* we can disable the scheduler hooks (for performance) without leaving
194	* pending task ctx state.
195	*
196	* This however results in two special cases:
197	*
198	* - removing the last event from a task ctx; this is relatively straight
199	* forward and is done in __perf_remove_from_context.
200	*
201	* - adding the first event to a task ctx; this is tricky because we cannot
202	* rely on ctx->is_active and therefore cannot use event_function_call().
203	* See perf_install_in_context().
204	*
205	* If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
206	*/
207
208	typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *,
209	struct perf_event_context *, void *);
210
211	struct event_function_struct {
212	struct perf_event *event;
213	event_f func;
214	void *data;
215	};
216
217	static int event_function(void *info)
218	{
219	struct event_function_struct *efs = info;
220	struct perf_event *event = efs->event;
221	struct perf_event_context *ctx = event->ctx;
222	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
223	struct perf_event_context *task_ctx = cpuctx->task_ctx;
224	int ret = 0;
225
226	lockdep_assert_irqs_disabled();
227
228	perf_ctx_lock(cpuctx, ctx: task_ctx);
229	/*
230	* Since we do the IPI call without holding ctx->lock things can have
231	* changed, double check we hit the task we set out to hit.
232	*/
233	if (ctx->task) {
234	if (ctx->task != current) {
235	ret = -ESRCH;
236	goto unlock;
237	}
238
239	/*
240	* We only use event_function_call() on established contexts,
241	* and event_function() is only ever called when active (or
242	* rather, we'll have bailed in task_function_call() or the
243	* above ctx->task != current test), therefore we must have
244	* ctx->is_active here.
245	*/
246	WARN_ON_ONCE(!ctx->is_active);
247	/*
248	* And since we have ctx->is_active, cpuctx->task_ctx must
249	* match.
250	*/
251	WARN_ON_ONCE(task_ctx != ctx);
252	} else {
253	WARN_ON_ONCE(&cpuctx->ctx != ctx);
254	}
255
256	efs->func(event, cpuctx, ctx, efs->data);
257	unlock:
258	perf_ctx_unlock(cpuctx, ctx: task_ctx);
259
260	return ret;
261	}
262
263	static void event_function_call(struct perf_event *event, event_f func, void *data)
264	{
265	struct perf_event_context *ctx = event->ctx;
266	struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */
267	struct event_function_struct efs = {
268	.event = event,
269	.func = func,
270	.data = data,
271	};
272
273	if (!event->parent) {
274	/*
275	* If this is a !child event, we must hold ctx::mutex to
276	* stabilize the event->ctx relation. See
277	* perf_event_ctx_lock().
278	*/
279	lockdep_assert_held(&ctx->mutex);
280	}
281
282	if (!task) {
283	cpu_function_call(cpu: event->cpu, func: event_function, info: &efs);
284	return;
285	}
286
287	if (task == TASK_TOMBSTONE)
288	return;
289
290	again:
291	if (!task_function_call(p: task, func: event_function, info: &efs))
292	return;
293
294	raw_spin_lock_irq(&ctx->lock);
295	/*
296	* Reload the task pointer, it might have been changed by
297	* a concurrent perf_event_context_sched_out().
298	*/
299	task = ctx->task;
300	if (task == TASK_TOMBSTONE) {
301	raw_spin_unlock_irq(&ctx->lock);
302	return;
303	}
304	if (ctx->is_active) {
305	raw_spin_unlock_irq(&ctx->lock);
306	goto again;
307	}
308	func(event, NULL, ctx, data);
309	raw_spin_unlock_irq(&ctx->lock);
310	}
311
312	/*
313	* Similar to event_function_call() + event_function(), but hard assumes IRQs
314	* are already disabled and we're on the right CPU.
315	*/
316	static void event_function_local(struct perf_event *event, event_f func, void *data)
317	{
318	struct perf_event_context *ctx = event->ctx;
319	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
320	struct task_struct *task = READ_ONCE(ctx->task);
321	struct perf_event_context *task_ctx = NULL;
322
323	lockdep_assert_irqs_disabled();
324
325	if (task) {
326	if (task == TASK_TOMBSTONE)
327	return;
328
329	task_ctx = ctx;
330	}
331
332	perf_ctx_lock(cpuctx, ctx: task_ctx);
333
334	task = ctx->task;
335	if (task == TASK_TOMBSTONE)
336	goto unlock;
337
338	if (task) {
339	/*
340	* We must be either inactive or active and the right task,
341	* otherwise we're screwed, since we cannot IPI to somewhere
342	* else.
343	*/
344	if (ctx->is_active) {
345	if (WARN_ON_ONCE(task != current))
346	goto unlock;
347
348	if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))
349	goto unlock;
350	}
351	} else {
352	WARN_ON_ONCE(&cpuctx->ctx != ctx);
353	}
354
355	func(event, cpuctx, ctx, data);
356	unlock:
357	perf_ctx_unlock(cpuctx, ctx: task_ctx);
358	}
359
360	#define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
361	PERF_FLAG_FD_OUTPUT |\
362	PERF_FLAG_PID_CGROUP |\
363	PERF_FLAG_FD_CLOEXEC)
364
365	/*
366	* branch priv levels that need permission checks
367	*/
368	#define PERF_SAMPLE_BRANCH_PERM_PLM \
369	(PERF_SAMPLE_BRANCH_KERNEL |\
370	PERF_SAMPLE_BRANCH_HV)
371
372	enum event_type_t {
373	EVENT_FLEXIBLE = 0x1,
374	EVENT_PINNED = 0x2,
375	EVENT_TIME = 0x4,
376	/* see ctx_resched() for details */
377	EVENT_CPU = 0x8,
378	EVENT_CGROUP = 0x10,
379	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
380	};
381
382	/*
383	* perf_sched_events : >0 events exist
384	*/
385
386	static void perf_sched_delayed(struct work_struct *work);
387	DEFINE_STATIC_KEY_FALSE(perf_sched_events);
388	static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed);
389	static DEFINE_MUTEX(perf_sched_mutex);
390	static atomic_t perf_sched_count;
391
392	static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
393
394	static atomic_t nr_mmap_events __read_mostly;
395	static atomic_t nr_comm_events __read_mostly;
396	static atomic_t nr_namespaces_events __read_mostly;
397	static atomic_t nr_task_events __read_mostly;
398	static atomic_t nr_freq_events __read_mostly;
399	static atomic_t nr_switch_events __read_mostly;
400	static atomic_t nr_ksymbol_events __read_mostly;
401	static atomic_t nr_bpf_events __read_mostly;
402	static atomic_t nr_cgroup_events __read_mostly;
403	static atomic_t nr_text_poke_events __read_mostly;
404	static atomic_t nr_build_id_events __read_mostly;
405
406	static LIST_HEAD(pmus);
407	static DEFINE_MUTEX(pmus_lock);
408	static struct srcu_struct pmus_srcu;
409	static cpumask_var_t perf_online_mask;
410	static struct kmem_cache *perf_event_cache;
411
412	/*
413	* perf event paranoia level:
414	* -1 - not paranoid at all
415	* 0 - disallow raw tracepoint access for unpriv
416	* 1 - disallow cpu events for unpriv
417	* 2 - disallow kernel profiling for unpriv
418	*/
419	int sysctl_perf_event_paranoid __read_mostly = 2;
420
421	/* Minimum for 512 kiB + 1 user control page */
422	int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
423
424	/*
425	* max perf event sample rate
426	*/
427	#define DEFAULT_MAX_SAMPLE_RATE 100000
428	#define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
429	#define DEFAULT_CPU_TIME_MAX_PERCENT 25
430
431	int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
432
433	static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
434	static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS;
435
436	static int perf_sample_allowed_ns __read_mostly =
437	DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
438
439	static void update_perf_cpu_limits(void)
440	{
441	u64 tmp = perf_sample_period_ns;
442
443	tmp *= sysctl_perf_cpu_time_max_percent;
444	tmp = div_u64(dividend: tmp, divisor: 100);
445	if (!tmp)
446	tmp = 1;
447
448	WRITE_ONCE(perf_sample_allowed_ns, tmp);
449	}
450
451	static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc);
452
453	int perf_event_max_sample_rate_handler(struct ctl_table *table, int write,
454	void *buffer, size_t *lenp, loff_t *ppos)
455	{
456	int ret;
457	int perf_cpu = sysctl_perf_cpu_time_max_percent;
458	/*
459	* If throttling is disabled don't allow the write:
460	*/
461	if (write && (perf_cpu == 100 || perf_cpu == 0))
462	return -EINVAL;
463
464	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
465	if (ret || !write)
466	return ret;
467
468	max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
469	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
470	update_perf_cpu_limits();
471
472	return 0;
473	}
474
475	int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
476
477	int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
478	void *buffer, size_t *lenp, loff_t *ppos)
479	{
480	int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
481
482	if (ret || !write)
483	return ret;
484
485	if (sysctl_perf_cpu_time_max_percent == 100 ||
486	sysctl_perf_cpu_time_max_percent == 0) {
487	printk(KERN_WARNING
488	"perf: Dynamic interrupt throttling disabled, can hang your system!\n");
489	WRITE_ONCE(perf_sample_allowed_ns, 0);
490	} else {
491	update_perf_cpu_limits();
492	}
493
494	return 0;
495	}
496
497	/*
498	* perf samples are done in some very critical code paths (NMIs).
499	* If they take too much CPU time, the system can lock up and not
500	* get any real work done. This will drop the sample rate when
501	* we detect that events are taking too long.
502	*/
503	#define NR_ACCUMULATED_SAMPLES 128
504	static DEFINE_PER_CPU(u64, running_sample_length);
505
506	static u64 __report_avg;
507	static u64 __report_allowed;
508
509	static void perf_duration_warn(struct irq_work *w)
510	{
511	printk_ratelimited(KERN_INFO
512	"perf: interrupt took too long (%lld > %lld), lowering "
513	"kernel.perf_event_max_sample_rate to %d\n",
514	__report_avg, __report_allowed,
515	sysctl_perf_event_sample_rate);
516	}
517
518	static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
519
520	void perf_sample_event_took(u64 sample_len_ns)
521	{
522	u64 max_len = READ_ONCE(perf_sample_allowed_ns);
523	u64 running_len;
524	u64 avg_len;
525	u32 max;
526
527	if (max_len == 0)
528	return;
529
530	/* Decay the counter by 1 average sample. */
531	running_len = __this_cpu_read(running_sample_length);
532	running_len -= running_len/NR_ACCUMULATED_SAMPLES;
533	running_len += sample_len_ns;
534	__this_cpu_write(running_sample_length, running_len);
535
536	/*
537	* Note: this will be biased artifically low until we have
538	* seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
539	* from having to maintain a count.
540	*/
541	avg_len = running_len/NR_ACCUMULATED_SAMPLES;
542	if (avg_len <= max_len)
543	return;
544
545	__report_avg = avg_len;
546	__report_allowed = max_len;
547
548	/*
549	* Compute a throttle threshold 25% below the current duration.
550	*/
551	avg_len += avg_len / 4;
552	max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent;
553	if (avg_len < max)
554	max /= (u32)avg_len;
555	else
556	max = 1;
557
558	WRITE_ONCE(perf_sample_allowed_ns, avg_len);
559	WRITE_ONCE(max_samples_per_tick, max);
560
561	sysctl_perf_event_sample_rate = max * HZ;
562	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
563
564	if (!irq_work_queue(work: &perf_duration_work)) {
565	early_printk(fmt: "perf: interrupt took too long (%lld > %lld), lowering "
566	"kernel.perf_event_max_sample_rate to %d\n",
567	__report_avg, __report_allowed,
568	sysctl_perf_event_sample_rate);
569	}
570	}
571
572	static atomic64_t perf_event_id;
573
574	static void update_context_time(struct perf_event_context *ctx);
575	static u64 perf_event_time(struct perf_event *event);
576
577	void __weak perf_event_print_debug(void) { }
578
579	static inline u64 perf_clock(void)
580	{
581	return local_clock();
582	}
583
584	static inline u64 perf_event_clock(struct perf_event *event)
585	{
586	return event->clock();
587	}
588
589	/*
590	* State based event timekeeping...
591	*
592	* The basic idea is to use event->state to determine which (if any) time
593	* fields to increment with the current delta. This means we only need to
594	* update timestamps when we change state or when they are explicitly requested
595	* (read).
596	*
597	* Event groups make things a little more complicated, but not terribly so. The
598	* rules for a group are that if the group leader is OFF the entire group is
599	* OFF, irrespecive of what the group member states are. This results in
600	* __perf_effective_state().
601	*
602	* A futher ramification is that when a group leader flips between OFF and
603	* !OFF, we need to update all group member times.
604	*
605	*
606	* NOTE: perf_event_time() is based on the (cgroup) context time, and thus we
607	* need to make sure the relevant context time is updated before we try and
608	* update our timestamps.
609	*/
610
611	static __always_inline enum perf_event_state
612	__perf_effective_state(struct perf_event *event)
613	{
614	struct perf_event *leader = event->group_leader;
615
616	if (leader->state <= PERF_EVENT_STATE_OFF)
617	return leader->state;
618
619	return event->state;
620	}
621
622	static __always_inline void
623	__perf_update_times(struct perf_event *event, u64 now, u64 *enabled, u64 *running)
624	{
625	enum perf_event_state state = __perf_effective_state(event);
626	u64 delta = now - event->tstamp;
627
628	*enabled = event->total_time_enabled;
629	if (state >= PERF_EVENT_STATE_INACTIVE)
630	*enabled += delta;
631
632	*running = event->total_time_running;
633	if (state >= PERF_EVENT_STATE_ACTIVE)
634	*running += delta;
635	}
636
637	static void perf_event_update_time(struct perf_event *event)
638	{
639	u64 now = perf_event_time(event);
640
641	__perf_update_times(event, now, enabled: &event->total_time_enabled,
642	running: &event->total_time_running);
643	event->tstamp = now;
644	}
645
646	static void perf_event_update_sibling_time(struct perf_event *leader)
647	{
648	struct perf_event *sibling;
649
650	for_each_sibling_event(sibling, leader)
651	perf_event_update_time(event: sibling);
652	}
653
654	static void
655	perf_event_set_state(struct perf_event *event, enum perf_event_state state)
656	{
657	if (event->state == state)
658	return;
659
660	perf_event_update_time(event);
661	/*
662	* If a group leader gets enabled/disabled all its siblings
663	* are affected too.
664	*/
665	if ((event->state < 0) ^ (state < 0))
666	perf_event_update_sibling_time(leader: event);
667
668	WRITE_ONCE(event->state, state);
669	}
670
671	/*
672	* UP store-release, load-acquire
673	*/
674
675	#define __store_release(ptr, val) \
676	do { \
677	barrier(); \
678	WRITE_ONCE(*(ptr), (val)); \
679	} while (0)
680
681	#define __load_acquire(ptr) \
682	({ \
683	__unqual_scalar_typeof(*(ptr)) ___p = READ_ONCE(*(ptr)); \
684	barrier(); \
685	___p; \
686	})
687
688	static void perf_ctx_disable(struct perf_event_context *ctx, bool cgroup)
689	{
690	struct perf_event_pmu_context *pmu_ctx;
691
692	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
693	if (cgroup && !pmu_ctx->nr_cgroups)
694	continue;
695	perf_pmu_disable(pmu: pmu_ctx->pmu);
696	}
697	}
698
699	static void perf_ctx_enable(struct perf_event_context *ctx, bool cgroup)
700	{
701	struct perf_event_pmu_context *pmu_ctx;
702
703	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
704	if (cgroup && !pmu_ctx->nr_cgroups)
705	continue;
706	perf_pmu_enable(pmu: pmu_ctx->pmu);
707	}
708	}
709
710	static void ctx_sched_out(struct perf_event_context *ctx, enum event_type_t event_type);
711	static void ctx_sched_in(struct perf_event_context *ctx, enum event_type_t event_type);
712
713	#ifdef CONFIG_CGROUP_PERF
714
715	static inline bool
716	perf_cgroup_match(struct perf_event *event)
717	{
718	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
719
720	/* @event doesn't care about cgroup */
721	if (!event->cgrp)
722	return true;
723
724	/* wants specific cgroup scope but @cpuctx isn't associated with any */
725	if (!cpuctx->cgrp)
726	return false;
727
728	/*
729	* Cgroup scoping is recursive. An event enabled for a cgroup is
730	* also enabled for all its descendant cgroups. If @cpuctx's
731	* cgroup is a descendant of @event's (the test covers identity
732	* case), it's a match.
733	*/
734	return cgroup_is_descendant(cgrp: cpuctx->cgrp->css.cgroup,
735	ancestor: event->cgrp->css.cgroup);
736	}
737
738	static inline void perf_detach_cgroup(struct perf_event *event)
739	{
740	css_put(css: &event->cgrp->css);
741	event->cgrp = NULL;
742	}
743
744	static inline int is_cgroup_event(struct perf_event *event)
745	{
746	return event->cgrp != NULL;
747	}
748
749	static inline u64 perf_cgroup_event_time(struct perf_event *event)
750	{
751	struct perf_cgroup_info *t;
752
753	t = per_cpu_ptr(event->cgrp->info, event->cpu);
754	return t->time;
755	}
756
757	static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
758	{
759	struct perf_cgroup_info *t;
760
761	t = per_cpu_ptr(event->cgrp->info, event->cpu);
762	if (!__load_acquire(&t->active))
763	return t->time;
764	now += READ_ONCE(t->timeoffset);
765	return now;
766	}
767
768	static inline void __update_cgrp_time(struct perf_cgroup_info *info, u64 now, bool adv)
769	{
770	if (adv)
771	info->time += now - info->timestamp;
772	info->timestamp = now;
773	/*
774	* see update_context_time()
775	*/
776	WRITE_ONCE(info->timeoffset, info->time - info->timestamp);
777	}
778
779	static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final)
780	{
781	struct perf_cgroup *cgrp = cpuctx->cgrp;
782	struct cgroup_subsys_state *css;
783	struct perf_cgroup_info *info;
784
785	if (cgrp) {
786	u64 now = perf_clock();
787
788	for (css = &cgrp->css; css; css = css->parent) {
789	cgrp = container_of(css, struct perf_cgroup, css);
790	info = this_cpu_ptr(cgrp->info);
791
792	__update_cgrp_time(info, now, adv: true);
793	if (final)
794	__store_release(&info->active, 0);
795	}
796	}
797	}
798
799	static inline void update_cgrp_time_from_event(struct perf_event *event)
800	{
801	struct perf_cgroup_info *info;
802
803	/*
804	* ensure we access cgroup data only when needed and
805	* when we know the cgroup is pinned (css_get)
806	*/
807	if (!is_cgroup_event(event))
808	return;
809
810	info = this_cpu_ptr(event->cgrp->info);
811	/*
812	* Do not update time when cgroup is not active
813	*/
814	if (info->active)
815	__update_cgrp_time(info, now: perf_clock(), adv: true);
816	}
817
818	static inline void
819	perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx)
820	{
821	struct perf_event_context *ctx = &cpuctx->ctx;
822	struct perf_cgroup *cgrp = cpuctx->cgrp;
823	struct perf_cgroup_info *info;
824	struct cgroup_subsys_state *css;
825
826	/*
827	* ctx->lock held by caller
828	* ensure we do not access cgroup data
829	* unless we have the cgroup pinned (css_get)
830	*/
831	if (!cgrp)
832	return;
833
834	WARN_ON_ONCE(!ctx->nr_cgroups);
835
836	for (css = &cgrp->css; css; css = css->parent) {
837	cgrp = container_of(css, struct perf_cgroup, css);
838	info = this_cpu_ptr(cgrp->info);
839	__update_cgrp_time(info, now: ctx->timestamp, adv: false);
840	__store_release(&info->active, 1);
841	}
842	}
843
844	/*
845	* reschedule events based on the cgroup constraint of task.
846	*/
847	static void perf_cgroup_switch(struct task_struct *task)
848	{
849	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
850	struct perf_cgroup *cgrp;
851
852	/*
853	* cpuctx->cgrp is set when the first cgroup event enabled,
854	* and is cleared when the last cgroup event disabled.
855	*/
856	if (READ_ONCE(cpuctx->cgrp) == NULL)
857	return;
858
859	WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);
860
861	cgrp = perf_cgroup_from_task(task, NULL);
862	if (READ_ONCE(cpuctx->cgrp) == cgrp)
863	return;
864
865	perf_ctx_lock(cpuctx, ctx: cpuctx->task_ctx);
866	perf_ctx_disable(ctx: &cpuctx->ctx, cgroup: true);
867
868	ctx_sched_out(ctx: &cpuctx->ctx, event_type: EVENT_ALL|EVENT_CGROUP);
869	/*
870	* must not be done before ctxswout due
871	* to update_cgrp_time_from_cpuctx() in
872	* ctx_sched_out()
873	*/
874	cpuctx->cgrp = cgrp;
875	/*
876	* set cgrp before ctxsw in to allow
877	* perf_cgroup_set_timestamp() in ctx_sched_in()
878	* to not have to pass task around
879	*/
880	ctx_sched_in(ctx: &cpuctx->ctx, event_type: EVENT_ALL|EVENT_CGROUP);
881
882	perf_ctx_enable(ctx: &cpuctx->ctx, cgroup: true);
883	perf_ctx_unlock(cpuctx, ctx: cpuctx->task_ctx);
884	}
885
886	static int perf_cgroup_ensure_storage(struct perf_event *event,
887	struct cgroup_subsys_state *css)
888	{
889	struct perf_cpu_context *cpuctx;
890	struct perf_event **storage;
891	int cpu, heap_size, ret = 0;
892
893	/*
894	* Allow storage to have sufficent space for an iterator for each
895	* possibly nested cgroup plus an iterator for events with no cgroup.
896	*/
897	for (heap_size = 1; css; css = css->parent)
898	heap_size++;
899
900	for_each_possible_cpu(cpu) {
901	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
902	if (heap_size <= cpuctx->heap_size)
903	continue;
904
905	storage = kmalloc_node(size: heap_size * sizeof(struct perf_event *),
906	GFP_KERNEL, cpu_to_node(cpu));
907	if (!storage) {
908	ret = -ENOMEM;
909	break;
910	}
911
912	raw_spin_lock_irq(&cpuctx->ctx.lock);
913	if (cpuctx->heap_size < heap_size) {
914	swap(cpuctx->heap, storage);
915	if (storage == cpuctx->heap_default)
916	storage = NULL;
917	cpuctx->heap_size = heap_size;
918	}
919	raw_spin_unlock_irq(&cpuctx->ctx.lock);
920
921	kfree(objp: storage);
922	}
923
924	return ret;
925	}
926
927	static inline int perf_cgroup_connect(int fd, struct perf_event *event,
928	struct perf_event_attr *attr,
929	struct perf_event *group_leader)
930	{
931	struct perf_cgroup *cgrp;
932	struct cgroup_subsys_state *css;
933	struct fd f = fdget(fd);
934	int ret = 0;
935
936	if (!f.file)
937	return -EBADF;
938
939	css = css_tryget_online_from_dir(dentry: f.file->f_path.dentry,
940	ss: &perf_event_cgrp_subsys);
941	if (IS_ERR(ptr: css)) {
942	ret = PTR_ERR(ptr: css);
943	goto out;
944	}
945
946	ret = perf_cgroup_ensure_storage(event, css);
947	if (ret)
948	goto out;
949
950	cgrp = container_of(css, struct perf_cgroup, css);
951	event->cgrp = cgrp;
952
953	/*
954	* all events in a group must monitor
955	* the same cgroup because a task belongs
956	* to only one perf cgroup at a time
957	*/
958	if (group_leader && group_leader->cgrp != cgrp) {
959	perf_detach_cgroup(event);
960	ret = -EINVAL;
961	}
962	out:
963	fdput(fd: f);
964	return ret;
965	}
966
967	static inline void
968	perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
969	{
970	struct perf_cpu_context *cpuctx;
971
972	if (!is_cgroup_event(event))
973	return;
974
975	event->pmu_ctx->nr_cgroups++;
976
977	/*
978	* Because cgroup events are always per-cpu events,
979	* @ctx == &cpuctx->ctx.
980	*/
981	cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
982
983	if (ctx->nr_cgroups++)
984	return;
985
986	cpuctx->cgrp = perf_cgroup_from_task(current, ctx);
987	}
988
989	static inline void
990	perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
991	{
992	struct perf_cpu_context *cpuctx;
993
994	if (!is_cgroup_event(event))
995	return;
996
997	event->pmu_ctx->nr_cgroups--;
998
999	/*
1000	* Because cgroup events are always per-cpu events,
1001	* @ctx == &cpuctx->ctx.
1002	*/
1003	cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
1004
1005	if (--ctx->nr_cgroups)
1006	return;
1007
1008	cpuctx->cgrp = NULL;
1009	}
1010
1011	#else /* !CONFIG_CGROUP_PERF */
1012
1013	static inline bool
1014	perf_cgroup_match(struct perf_event *event)
1015	{
1016	return true;
1017	}
1018
1019	static inline void perf_detach_cgroup(struct perf_event *event)
1020	{}
1021
1022	static inline int is_cgroup_event(struct perf_event *event)
1023	{
1024	return 0;
1025	}
1026
1027	static inline void update_cgrp_time_from_event(struct perf_event *event)
1028	{
1029	}
1030
1031	static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx,
1032	bool final)
1033	{
1034	}
1035
1036	static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
1037	struct perf_event_attr *attr,
1038	struct perf_event *group_leader)
1039	{
1040	return -EINVAL;
1041	}
1042
1043	static inline void
1044	perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx)
1045	{
1046	}
1047
1048	static inline u64 perf_cgroup_event_time(struct perf_event *event)
1049	{
1050	return 0;
1051	}
1052
1053	static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
1054	{
1055	return 0;
1056	}
1057
1058	static inline void
1059	perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
1060	{
1061	}
1062
1063	static inline void
1064	perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
1065	{
1066	}
1067
1068	static void perf_cgroup_switch(struct task_struct *task)
1069	{
1070	}
1071	#endif
1072
1073	/*
1074	* set default to be dependent on timer tick just
1075	* like original code
1076	*/
1077	#define PERF_CPU_HRTIMER (1000 / HZ)
1078	/*
1079	* function must be called with interrupts disabled
1080	*/
1081	static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)
1082	{
1083	struct perf_cpu_pmu_context *cpc;
1084	bool rotations;
1085
1086	lockdep_assert_irqs_disabled();
1087
1088	cpc = container_of(hr, struct perf_cpu_pmu_context, hrtimer);
1089	rotations = perf_rotate_context(cpc);
1090
1091	raw_spin_lock(&cpc->hrtimer_lock);
1092	if (rotations)
1093	hrtimer_forward_now(timer: hr, interval: cpc->hrtimer_interval);
1094	else
1095	cpc->hrtimer_active = 0;
1096	raw_spin_unlock(&cpc->hrtimer_lock);
1097
1098	return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART;
1099	}
1100
1101	static void __perf_mux_hrtimer_init(struct perf_cpu_pmu_context *cpc, int cpu)
1102	{
1103	struct hrtimer *timer = &cpc->hrtimer;
1104	struct pmu *pmu = cpc->epc.pmu;
1105	u64 interval;
1106
1107	/*
1108	* check default is sane, if not set then force to
1109	* default interval (1/tick)
1110	*/
1111	interval = pmu->hrtimer_interval_ms;
1112	if (interval < 1)
1113	interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
1114
1115	cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
1116
1117	raw_spin_lock_init(&cpc->hrtimer_lock);
1118	hrtimer_init(timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_ABS_PINNED_HARD);
1119	timer->function = perf_mux_hrtimer_handler;
1120	}
1121
1122	static int perf_mux_hrtimer_restart(struct perf_cpu_pmu_context *cpc)
1123	{
1124	struct hrtimer *timer = &cpc->hrtimer;
1125	unsigned long flags;
1126
1127	raw_spin_lock_irqsave(&cpc->hrtimer_lock, flags);
1128	if (!cpc->hrtimer_active) {
1129	cpc->hrtimer_active = 1;
1130	hrtimer_forward_now(timer, interval: cpc->hrtimer_interval);
1131	hrtimer_start_expires(timer, mode: HRTIMER_MODE_ABS_PINNED_HARD);
1132	}
1133	raw_spin_unlock_irqrestore(&cpc->hrtimer_lock, flags);
1134
1135	return 0;
1136	}
1137
1138	static int perf_mux_hrtimer_restart_ipi(void *arg)
1139	{
1140	return perf_mux_hrtimer_restart(cpc: arg);
1141	}
1142
1143	void perf_pmu_disable(struct pmu *pmu)
1144	{
1145	int *count = this_cpu_ptr(pmu->pmu_disable_count);
1146	if (!(*count)++)
1147	pmu->pmu_disable(pmu);
1148	}
1149
1150	void perf_pmu_enable(struct pmu *pmu)
1151	{
1152	int *count = this_cpu_ptr(pmu->pmu_disable_count);
1153	if (!--(*count))
1154	pmu->pmu_enable(pmu);
1155	}
1156
1157	static void perf_assert_pmu_disabled(struct pmu *pmu)
1158	{
1159	WARN_ON_ONCE(*this_cpu_ptr(pmu->pmu_disable_count) == 0);
1160	}
1161
1162	static void get_ctx(struct perf_event_context *ctx)
1163	{
1164	refcount_inc(r: &ctx->refcount);
1165	}
1166
1167	static void *alloc_task_ctx_data(struct pmu *pmu)
1168	{
1169	if (pmu->task_ctx_cache)
1170	return kmem_cache_zalloc(k: pmu->task_ctx_cache, GFP_KERNEL);
1171
1172	return NULL;
1173	}
1174
1175	static void free_task_ctx_data(struct pmu *pmu, void *task_ctx_data)
1176	{
1177	if (pmu->task_ctx_cache && task_ctx_data)
1178	kmem_cache_free(s: pmu->task_ctx_cache, objp: task_ctx_data);
1179	}
1180
1181	static void free_ctx(struct rcu_head *head)
1182	{
1183	struct perf_event_context *ctx;
1184
1185	ctx = container_of(head, struct perf_event_context, rcu_head);
1186	kfree(objp: ctx);
1187	}
1188
1189	static void put_ctx(struct perf_event_context *ctx)
1190	{
1191	if (refcount_dec_and_test(r: &ctx->refcount)) {
1192	if (ctx->parent_ctx)
1193	put_ctx(ctx: ctx->parent_ctx);
1194	if (ctx->task && ctx->task != TASK_TOMBSTONE)
1195	put_task_struct(t: ctx->task);
1196	call_rcu(head: &ctx->rcu_head, func: free_ctx);
1197	}
1198	}
1199
1200	/*
1201	* Because of perf_event::ctx migration in sys_perf_event_open::move_group and
1202	* perf_pmu_migrate_context() we need some magic.
1203	*
1204	* Those places that change perf_event::ctx will hold both
1205	* perf_event_ctx::mutex of the 'old' and 'new' ctx value.
1206	*
1207	* Lock ordering is by mutex address. There are two other sites where
1208	* perf_event_context::mutex nests and those are:
1209	*
1210	* - perf_event_exit_task_context() [ child , 0 ]
1211	* perf_event_exit_event()
1212	* put_event() [ parent, 1 ]
1213	*
1214	* - perf_event_init_context() [ parent, 0 ]
1215	* inherit_task_group()
1216	* inherit_group()
1217	* inherit_event()
1218	* perf_event_alloc()
1219	* perf_init_event()
1220	* perf_try_init_event() [ child , 1 ]
1221	*
1222	* While it appears there is an obvious deadlock here -- the parent and child
1223	* nesting levels are inverted between the two. This is in fact safe because
1224	* life-time rules separate them. That is an exiting task cannot fork, and a
1225	* spawning task cannot (yet) exit.
1226	*
1227	* But remember that these are parent<->child context relations, and
1228	* migration does not affect children, therefore these two orderings should not
1229	* interact.
1230	*
1231	* The change in perf_event::ctx does not affect children (as claimed above)
1232	* because the sys_perf_event_open() case will install a new event and break
1233	* the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1234	* concerned with cpuctx and that doesn't have children.
1235	*
1236	* The places that change perf_event::ctx will issue:
1237	*
1238	* perf_remove_from_context();
1239	* synchronize_rcu();
1240	* perf_install_in_context();
1241	*
1242	* to affect the change. The remove_from_context() + synchronize_rcu() should
1243	* quiesce the event, after which we can install it in the new location. This
1244	* means that only external vectors (perf_fops, prctl) can perturb the event
1245	* while in transit. Therefore all such accessors should also acquire
1246	* perf_event_context::mutex to serialize against this.
1247	*
1248	* However; because event->ctx can change while we're waiting to acquire
1249	* ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1250	* function.
1251	*
1252	* Lock order:
1253	* exec_update_lock
1254	* task_struct::perf_event_mutex
1255	* perf_event_context::mutex
1256	* perf_event::child_mutex;
1257	* perf_event_context::lock
1258	* perf_event::mmap_mutex
1259	* mmap_lock
1260	* perf_addr_filters_head::lock
1261	*
1262	* cpu_hotplug_lock
1263	* pmus_lock
1264	* cpuctx->mutex / perf_event_context::mutex
1265	*/
1266	static struct perf_event_context *
1267	perf_event_ctx_lock_nested(struct perf_event *event, int nesting)
1268	{
1269	struct perf_event_context *ctx;
1270
1271	again:
1272	rcu_read_lock();
1273	ctx = READ_ONCE(event->ctx);
1274	if (!refcount_inc_not_zero(r: &ctx->refcount)) {
1275	rcu_read_unlock();
1276	goto again;
1277	}
1278	rcu_read_unlock();
1279
1280	mutex_lock_nested(lock: &ctx->mutex, subclass: nesting);
1281	if (event->ctx != ctx) {
1282	mutex_unlock(lock: &ctx->mutex);
1283	put_ctx(ctx);
1284	goto again;
1285	}
1286
1287	return ctx;
1288	}
1289
1290	static inline struct perf_event_context *
1291	perf_event_ctx_lock(struct perf_event *event)
1292	{
1293	return perf_event_ctx_lock_nested(event, nesting: 0);
1294	}
1295
1296	static void perf_event_ctx_unlock(struct perf_event *event,
1297	struct perf_event_context *ctx)
1298	{
1299	mutex_unlock(lock: &ctx->mutex);
1300	put_ctx(ctx);
1301	}
1302
1303	/*
1304	* This must be done under the ctx->lock, such as to serialize against
1305	* context_equiv(), therefore we cannot call put_ctx() since that might end up
1306	* calling scheduler related locks and ctx->lock nests inside those.
1307	*/
1308	static __must_check struct perf_event_context *
1309	unclone_ctx(struct perf_event_context *ctx)
1310	{
1311	struct perf_event_context *parent_ctx = ctx->parent_ctx;
1312
1313	lockdep_assert_held(&ctx->lock);
1314
1315	if (parent_ctx)
1316	ctx->parent_ctx = NULL;
1317	ctx->generation++;
1318
1319	return parent_ctx;
1320	}
1321
1322	static u32 perf_event_pid_type(struct perf_event *event, struct task_struct *p,
1323	enum pid_type type)
1324	{
1325	u32 nr;
1326	/*
1327	* only top level events have the pid namespace they were created in
1328	*/
1329	if (event->parent)
1330	event = event->parent;
1331
1332	nr = __task_pid_nr_ns(task: p, type, ns: event->ns);
1333	/* avoid -1 if it is idle thread or runs in another ns */
1334	if (!nr && !pid_alive(p))
1335	nr = -1;
1336	return nr;
1337	}
1338
1339	static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
1340	{
1341	return perf_event_pid_type(event, p, type: PIDTYPE_TGID);
1342	}
1343
1344	static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
1345	{
1346	return perf_event_pid_type(event, p, type: PIDTYPE_PID);
1347	}
1348
1349	/*
1350	* If we inherit events we want to return the parent event id
1351	* to userspace.
1352	*/
1353	static u64 primary_event_id(struct perf_event *event)
1354	{
1355	u64 id = event->id;
1356
1357	if (event->parent)
1358	id = event->parent->id;
1359
1360	return id;
1361	}
1362
1363	/*
1364	* Get the perf_event_context for a task and lock it.
1365	*
1366	* This has to cope with the fact that until it is locked,
1367	* the context could get moved to another task.
1368	*/
1369	static struct perf_event_context *
1370	perf_lock_task_context(struct task_struct *task, unsigned long *flags)
1371	{
1372	struct perf_event_context *ctx;
1373
1374	retry:
1375	/*
1376	* One of the few rules of preemptible RCU is that one cannot do
1377	* rcu_read_unlock() while holding a scheduler (or nested) lock when
1378	* part of the read side critical section was irqs-enabled -- see
1379	* rcu_read_unlock_special().
1380	*
1381	* Since ctx->lock nests under rq->lock we must ensure the entire read
1382	* side critical section has interrupts disabled.
1383	*/
1384	local_irq_save(*flags);
1385	rcu_read_lock();
1386	ctx = rcu_dereference(task->perf_event_ctxp);
1387	if (ctx) {
1388	/*
1389	* If this context is a clone of another, it might
1390	* get swapped for another underneath us by
1391	* perf_event_task_sched_out, though the
1392	* rcu_read_lock() protects us from any context
1393	* getting freed. Lock the context and check if it
1394	* got swapped before we could get the lock, and retry
1395	* if so. If we locked the right context, then it
1396	* can't get swapped on us any more.
1397	*/
1398	raw_spin_lock(&ctx->lock);
1399	if (ctx != rcu_dereference(task->perf_event_ctxp)) {
1400	raw_spin_unlock(&ctx->lock);
1401	rcu_read_unlock();
1402	local_irq_restore(*flags);
1403	goto retry;
1404	}
1405
1406	if (ctx->task == TASK_TOMBSTONE ||
1407	!refcount_inc_not_zero(r: &ctx->refcount)) {
1408	raw_spin_unlock(&ctx->lock);
1409	ctx = NULL;
1410	} else {
1411	WARN_ON_ONCE(ctx->task != task);
1412	}
1413	}
1414	rcu_read_unlock();
1415	if (!ctx)
1416	local_irq_restore(*flags);
1417	return ctx;
1418	}
1419
1420	/*
1421	* Get the context for a task and increment its pin_count so it
1422	* can't get swapped to another task. This also increments its
1423	* reference count so that the context can't get freed.
1424	*/
1425	static struct perf_event_context *
1426	perf_pin_task_context(struct task_struct *task)
1427	{
1428	struct perf_event_context *ctx;
1429	unsigned long flags;
1430
1431	ctx = perf_lock_task_context(task, flags: &flags);
1432	if (ctx) {
1433	++ctx->pin_count;
1434	raw_spin_unlock_irqrestore(&ctx->lock, flags);
1435	}
1436	return ctx;
1437	}
1438
1439	static void perf_unpin_context(struct perf_event_context *ctx)
1440	{
1441	unsigned long flags;
1442
1443	raw_spin_lock_irqsave(&ctx->lock, flags);
1444	--ctx->pin_count;
1445	raw_spin_unlock_irqrestore(&ctx->lock, flags);
1446	}
1447
1448	/*
1449	* Update the record of the current time in a context.
1450	*/
1451	static void __update_context_time(struct perf_event_context *ctx, bool adv)
1452	{
1453	u64 now = perf_clock();
1454
1455	lockdep_assert_held(&ctx->lock);
1456
1457	if (adv)
1458	ctx->time += now - ctx->timestamp;
1459	ctx->timestamp = now;
1460
1461	/*
1462	* The above: time' = time + (now - timestamp), can be re-arranged
1463	* into: time` = now + (time - timestamp), which gives a single value
1464	* offset to compute future time without locks on.
1465	*
1466	* See perf_event_time_now(), which can be used from NMI context where
1467	* it's (obviously) not possible to acquire ctx->lock in order to read
1468	* both the above values in a consistent manner.
1469	*/
1470	WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp);
1471	}
1472
1473	static void update_context_time(struct perf_event_context *ctx)
1474	{
1475	__update_context_time(ctx, adv: true);
1476	}
1477
1478	static u64 perf_event_time(struct perf_event *event)
1479	{
1480	struct perf_event_context *ctx = event->ctx;
1481
1482	if (unlikely(!ctx))
1483	return 0;
1484
1485	if (is_cgroup_event(event))
1486	return perf_cgroup_event_time(event);
1487
1488	return ctx->time;
1489	}
1490
1491	static u64 perf_event_time_now(struct perf_event *event, u64 now)
1492	{
1493	struct perf_event_context *ctx = event->ctx;
1494
1495	if (unlikely(!ctx))
1496	return 0;
1497
1498	if (is_cgroup_event(event))
1499	return perf_cgroup_event_time_now(event, now);
1500
1501	if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))
1502	return ctx->time;
1503
1504	now += READ_ONCE(ctx->timeoffset);
1505	return now;
1506	}
1507
1508	static enum event_type_t get_event_type(struct perf_event *event)
1509	{
1510	struct perf_event_context *ctx = event->ctx;
1511	enum event_type_t event_type;
1512
1513	lockdep_assert_held(&ctx->lock);
1514
1515	/*
1516	* It's 'group type', really, because if our group leader is
1517	* pinned, so are we.
1518	*/
1519	if (event->group_leader != event)
1520	event = event->group_leader;
1521
1522	event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;
1523	if (!ctx->task)
1524	event_type |= EVENT_CPU;
1525
1526	return event_type;
1527	}
1528
1529	/*
1530	* Helper function to initialize event group nodes.
1531	*/
1532	static void init_event_group(struct perf_event *event)
1533	{
1534	RB_CLEAR_NODE(&event->group_node);
1535	event->group_index = 0;
1536	}
1537
1538	/*
1539	* Extract pinned or flexible groups from the context
1540	* based on event attrs bits.
1541	*/
1542	static struct perf_event_groups *
1543	get_event_groups(struct perf_event *event, struct perf_event_context *ctx)
1544	{
1545	if (event->attr.pinned)
1546	return &ctx->pinned_groups;
1547	else
1548	return &ctx->flexible_groups;
1549	}
1550
1551	/*
1552	* Helper function to initializes perf_event_group trees.
1553	*/
1554	static void perf_event_groups_init(struct perf_event_groups *groups)
1555	{
1556	groups->tree = RB_ROOT;
1557	groups->index = 0;
1558	}
1559
1560	static inline struct cgroup *event_cgroup(const struct perf_event *event)
1561	{
1562	struct cgroup *cgroup = NULL;
1563
1564	#ifdef CONFIG_CGROUP_PERF
1565	if (event->cgrp)
1566	cgroup = event->cgrp->css.cgroup;
1567	#endif
1568
1569	return cgroup;
1570	}
1571
1572	/*
1573	* Compare function for event groups;
1574	*
1575	* Implements complex key that first sorts by CPU and then by virtual index
1576	* which provides ordering when rotating groups for the same CPU.
1577	*/
1578	static __always_inline int
1579	perf_event_groups_cmp(const int left_cpu, const struct pmu *left_pmu,
1580	const struct cgroup *left_cgroup, const u64 left_group_index,
1581	const struct perf_event *right)
1582	{
1583	if (left_cpu < right->cpu)
1584	return -1;
1585	if (left_cpu > right->cpu)
1586	return 1;
1587
1588	if (left_pmu) {
1589	if (left_pmu < right->pmu_ctx->pmu)
1590	return -1;
1591	if (left_pmu > right->pmu_ctx->pmu)
1592	return 1;
1593	}
1594
1595	#ifdef CONFIG_CGROUP_PERF
1596	{
1597	const struct cgroup *right_cgroup = event_cgroup(event: right);
1598
1599	if (left_cgroup != right_cgroup) {
1600	if (!left_cgroup) {
1601	/*
1602	* Left has no cgroup but right does, no
1603	* cgroups come first.
1604	*/
1605	return -1;
1606	}
1607	if (!right_cgroup) {
1608	/*
1609	* Right has no cgroup but left does, no
1610	* cgroups come first.
1611	*/
1612	return 1;
1613	}
1614	/* Two dissimilar cgroups, order by id. */
1615	if (cgroup_id(cgrp: left_cgroup) < cgroup_id(cgrp: right_cgroup))
1616	return -1;
1617
1618	return 1;
1619	}
1620	}
1621	#endif
1622
1623	if (left_group_index < right->group_index)
1624	return -1;
1625	if (left_group_index > right->group_index)
1626	return 1;
1627
1628	return 0;
1629	}
1630
1631	#define __node_2_pe(node) \
1632	rb_entry((node), struct perf_event, group_node)
1633
1634	static inline bool __group_less(struct rb_node *a, const struct rb_node *b)
1635	{
1636	struct perf_event *e = __node_2_pe(a);
1637	return perf_event_groups_cmp(left_cpu: e->cpu, left_pmu: e->pmu_ctx->pmu, left_cgroup: event_cgroup(event: e),
1638	left_group_index: e->group_index, __node_2_pe(b)) < 0;
1639	}
1640
1641	struct __group_key {
1642	int cpu;
1643	struct pmu *pmu;
1644	struct cgroup *cgroup;
1645	};
1646
1647	static inline int __group_cmp(const void *key, const struct rb_node *node)
1648	{
1649	const struct __group_key *a = key;
1650	const struct perf_event *b = __node_2_pe(node);
1651
1652	/* partial/subtree match: @cpu, @pmu, @cgroup; ignore: @group_index */
1653	return perf_event_groups_cmp(left_cpu: a->cpu, left_pmu: a->pmu, left_cgroup: a->cgroup, left_group_index: b->group_index, right: b);
1654	}
1655
1656	static inline int
1657	__group_cmp_ignore_cgroup(const void *key, const struct rb_node *node)
1658	{
1659	const struct __group_key *a = key;
1660	const struct perf_event *b = __node_2_pe(node);
1661
1662	/* partial/subtree match: @cpu, @pmu, ignore: @cgroup, @group_index */
1663	return perf_event_groups_cmp(left_cpu: a->cpu, left_pmu: a->pmu, left_cgroup: event_cgroup(event: b),
1664	left_group_index: b->group_index, right: b);
1665	}
1666
1667	/*
1668	* Insert @event into @groups' tree; using
1669	* {@event->cpu, @event->pmu_ctx->pmu, event_cgroup(@event), ++@groups->index}
1670	* as key. This places it last inside the {cpu,pmu,cgroup} subtree.
1671	*/
1672	static void
1673	perf_event_groups_insert(struct perf_event_groups *groups,
1674	struct perf_event *event)
1675	{
1676	event->group_index = ++groups->index;
1677
1678	rb_add(node: &event->group_node, tree: &groups->tree, less: __group_less);
1679	}
1680
1681	/*
1682	* Helper function to insert event into the pinned or flexible groups.
1683	*/
1684	static void
1685	add_event_to_groups(struct perf_event *event, struct perf_event_context *ctx)
1686	{
1687	struct perf_event_groups *groups;
1688
1689	groups = get_event_groups(event, ctx);
1690	perf_event_groups_insert(groups, event);
1691	}
1692
1693	/*
1694	* Delete a group from a tree.
1695	*/
1696	static void
1697	perf_event_groups_delete(struct perf_event_groups *groups,
1698	struct perf_event *event)
1699	{
1700	WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) ||
1701	RB_EMPTY_ROOT(&groups->tree));
1702
1703	rb_erase(&event->group_node, &groups->tree);
1704	init_event_group(event);
1705	}
1706
1707	/*
1708	* Helper function to delete event from its groups.
1709	*/
1710	static void
1711	del_event_from_groups(struct perf_event *event, struct perf_event_context *ctx)
1712	{
1713	struct perf_event_groups *groups;
1714
1715	groups = get_event_groups(event, ctx);
1716	perf_event_groups_delete(groups, event);
1717	}
1718
1719	/*
1720	* Get the leftmost event in the {cpu,pmu,cgroup} subtree.
1721	*/
1722	static struct perf_event *
1723	perf_event_groups_first(struct perf_event_groups *groups, int cpu,
1724	struct pmu *pmu, struct cgroup *cgrp)
1725	{
1726	struct __group_key key = {
1727	.cpu = cpu,
1728	.pmu = pmu,
1729	.cgroup = cgrp,
1730	};
1731	struct rb_node *node;
1732
1733	node = rb_find_first(key: &key, tree: &groups->tree, cmp: __group_cmp);
1734	if (node)
1735	return __node_2_pe(node);
1736
1737	return NULL;
1738	}
1739
1740	static struct perf_event *
1741	perf_event_groups_next(struct perf_event *event, struct pmu *pmu)
1742	{
1743	struct __group_key key = {
1744	.cpu = event->cpu,
1745	.pmu = pmu,
1746	.cgroup = event_cgroup(event),
1747	};
1748	struct rb_node *next;
1749
1750	next = rb_next_match(key: &key, node: &event->group_node, cmp: __group_cmp);
1751	if (next)
1752	return __node_2_pe(next);
1753
1754	return NULL;
1755	}
1756
1757	#define perf_event_groups_for_cpu_pmu(event, groups, cpu, pmu) \
1758	for (event = perf_event_groups_first(groups, cpu, pmu, NULL); \
1759	event; event = perf_event_groups_next(event, pmu))
1760
1761	/*
1762	* Iterate through the whole groups tree.
1763	*/
1764	#define perf_event_groups_for_each(event, groups) \
1765	for (event = rb_entry_safe(rb_first(&((groups)->tree)), \
1766	typeof(*event), group_node); event; \
1767	event = rb_entry_safe(rb_next(&event->group_node), \
1768	typeof(*event), group_node))
1769
1770	/*
1771	* Add an event from the lists for its context.
1772	* Must be called with ctx->mutex and ctx->lock held.
1773	*/
1774	static void
1775	list_add_event(struct perf_event *event, struct perf_event_context *ctx)
1776	{
1777	lockdep_assert_held(&ctx->lock);
1778
1779	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1780	event->attach_state |= PERF_ATTACH_CONTEXT;
1781
1782	event->tstamp = perf_event_time(event);
1783
1784	/*
1785	* If we're a stand alone event or group leader, we go to the context
1786	* list, group events are kept attached to the group so that
1787	* perf_group_detach can, at all times, locate all siblings.
1788	*/
1789	if (event->group_leader == event) {
1790	event->group_caps = event->event_caps;
1791	add_event_to_groups(event, ctx);
1792	}
1793
1794	list_add_rcu(new: &event->event_entry, head: &ctx->event_list);
1795	ctx->nr_events++;
1796	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)
1797	ctx->nr_user++;
1798	if (event->attr.inherit_stat)
1799	ctx->nr_stat++;
1800
1801	if (event->state > PERF_EVENT_STATE_OFF)
1802	perf_cgroup_event_enable(event, ctx);
1803
1804	ctx->generation++;
1805	event->pmu_ctx->nr_events++;
1806	}
1807
1808	/*
1809	* Initialize event state based on the perf_event_attr::disabled.
1810	*/
1811	static inline void perf_event__state_init(struct perf_event *event)
1812	{
1813	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1814	PERF_EVENT_STATE_INACTIVE;
1815	}
1816
1817	static void __perf_event_read_size(struct perf_event *event, int nr_siblings)
1818	{
1819	int entry = sizeof(u64); /* value */
1820	int size = 0;
1821	int nr = 1;
1822
1823	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1824	size += sizeof(u64);
1825
1826	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1827	size += sizeof(u64);
1828
1829	if (event->attr.read_format & PERF_FORMAT_ID)
1830	entry += sizeof(u64);
1831
1832	if (event->attr.read_format & PERF_FORMAT_LOST)
1833	entry += sizeof(u64);
1834
1835	if (event->attr.read_format & PERF_FORMAT_GROUP) {
1836	nr += nr_siblings;
1837	size += sizeof(u64);
1838	}
1839
1840	size += entry * nr;
1841	event->read_size = size;
1842	}
1843
1844	static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
1845	{
1846	struct perf_sample_data *data;
1847	u16 size = 0;
1848
1849	if (sample_type & PERF_SAMPLE_IP)
1850	size += sizeof(data->ip);
1851
1852	if (sample_type & PERF_SAMPLE_ADDR)
1853	size += sizeof(data->addr);
1854
1855	if (sample_type & PERF_SAMPLE_PERIOD)
1856	size += sizeof(data->period);
1857
1858	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
1859	size += sizeof(data->weight.full);
1860
1861	if (sample_type & PERF_SAMPLE_READ)
1862	size += event->read_size;
1863
1864	if (sample_type & PERF_SAMPLE_DATA_SRC)
1865	size += sizeof(data->data_src.val);
1866
1867	if (sample_type & PERF_SAMPLE_TRANSACTION)
1868	size += sizeof(data->txn);
1869
1870	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
1871	size += sizeof(data->phys_addr);
1872
1873	if (sample_type & PERF_SAMPLE_CGROUP)
1874	size += sizeof(data->cgroup);
1875
1876	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
1877	size += sizeof(data->data_page_size);
1878
1879	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
1880	size += sizeof(data->code_page_size);
1881
1882	event->header_size = size;
1883	}
1884
1885	/*
1886	* Called at perf_event creation and when events are attached/detached from a
1887	* group.
1888	*/
1889	static void perf_event__header_size(struct perf_event *event)
1890	{
1891	__perf_event_read_size(event,
1892	nr_siblings: event->group_leader->nr_siblings);
1893	__perf_event_header_size(event, sample_type: event->attr.sample_type);
1894	}
1895
1896	static void perf_event__id_header_size(struct perf_event *event)
1897	{
1898	struct perf_sample_data *data;
1899	u64 sample_type = event->attr.sample_type;
1900	u16 size = 0;
1901
1902	if (sample_type & PERF_SAMPLE_TID)
1903	size += sizeof(data->tid_entry);
1904
1905	if (sample_type & PERF_SAMPLE_TIME)
1906	size += sizeof(data->time);
1907
1908	if (sample_type & PERF_SAMPLE_IDENTIFIER)
1909	size += sizeof(data->id);
1910
1911	if (sample_type & PERF_SAMPLE_ID)
1912	size += sizeof(data->id);
1913
1914	if (sample_type & PERF_SAMPLE_STREAM_ID)
1915	size += sizeof(data->stream_id);
1916
1917	if (sample_type & PERF_SAMPLE_CPU)
1918	size += sizeof(data->cpu_entry);
1919
1920	event->id_header_size = size;
1921	}
1922
1923	static bool perf_event_validate_size(struct perf_event *event)
1924	{
1925	/*
1926	* The values computed here will be over-written when we actually
1927	* attach the event.
1928	*/
1929	__perf_event_read_size(event, nr_siblings: event->group_leader->nr_siblings + 1);
1930	__perf_event_header_size(event, sample_type: event->attr.sample_type & ~PERF_SAMPLE_READ);
1931	perf_event__id_header_size(event);
1932
1933	/*
1934	* Sum the lot; should not exceed the 64k limit we have on records.
1935	* Conservative limit to allow for callchains and other variable fields.
1936	*/
1937	if (event->read_size + event->header_size +
1938	event->id_header_size + sizeof(struct perf_event_header) >= 16*1024)
1939	return false;
1940
1941	return true;
1942	}
1943
1944	static void perf_group_attach(struct perf_event *event)
1945	{
1946	struct perf_event *group_leader = event->group_leader, *pos;
1947
1948	lockdep_assert_held(&event->ctx->lock);
1949
1950	/*
1951	* We can have double attach due to group movement (move_group) in
1952	* perf_event_open().
1953	*/
1954	if (event->attach_state & PERF_ATTACH_GROUP)
1955	return;
1956
1957	event->attach_state |= PERF_ATTACH_GROUP;
1958
1959	if (group_leader == event)
1960	return;
1961
1962	WARN_ON_ONCE(group_leader->ctx != event->ctx);
1963
1964	group_leader->group_caps &= event->event_caps;
1965
1966	list_add_tail(new: &event->sibling_list, head: &group_leader->sibling_list);
1967	group_leader->nr_siblings++;
1968	group_leader->group_generation++;
1969
1970	perf_event__header_size(event: group_leader);
1971
1972	for_each_sibling_event(pos, group_leader)
1973	perf_event__header_size(event: pos);
1974	}
1975
1976	/*
1977	* Remove an event from the lists for its context.
1978	* Must be called with ctx->mutex and ctx->lock held.
1979	*/
1980	static void
1981	list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1982	{
1983	WARN_ON_ONCE(event->ctx != ctx);
1984	lockdep_assert_held(&ctx->lock);
1985
1986	/*
1987	* We can have double detach due to exit/hot-unplug + close.
1988	*/
1989	if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1990	return;
1991
1992	event->attach_state &= ~PERF_ATTACH_CONTEXT;
1993
1994	ctx->nr_events--;
1995	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)
1996	ctx->nr_user--;
1997	if (event->attr.inherit_stat)
1998	ctx->nr_stat--;
1999
2000	list_del_rcu(entry: &event->event_entry);
2001
2002	if (event->group_leader == event)
2003	del_event_from_groups(event, ctx);
2004
2005	/*
2006	* If event was in error state, then keep it
2007	* that way, otherwise bogus counts will be
2008	* returned on read(). The only way to get out
2009	* of error state is by explicit re-enabling
2010	* of the event
2011	*/
2012	if (event->state > PERF_EVENT_STATE_OFF) {
2013	perf_cgroup_event_disable(event, ctx);
2014	perf_event_set_state(event, state: PERF_EVENT_STATE_OFF);
2015	}
2016
2017	ctx->generation++;
2018	event->pmu_ctx->nr_events--;
2019	}
2020
2021	static int
2022	perf_aux_output_match(struct perf_event *event, struct perf_event *aux_event)
2023	{
2024	if (!has_aux(event: aux_event))
2025	return 0;
2026
2027	if (!event->pmu->aux_output_match)
2028	return 0;
2029
2030	return event->pmu->aux_output_match(aux_event);
2031	}
2032
2033	static void put_event(struct perf_event *event);
2034	static void event_sched_out(struct perf_event *event,
2035	struct perf_event_context *ctx);
2036
2037	static void perf_put_aux_event(struct perf_event *event)
2038	{
2039	struct perf_event_context *ctx = event->ctx;
2040	struct perf_event *iter;
2041
2042	/*
2043	* If event uses aux_event tear down the link
2044	*/
2045	if (event->aux_event) {
2046	iter = event->aux_event;
2047	event->aux_event = NULL;
2048	put_event(event: iter);
2049	return;
2050	}
2051
2052	/*
2053	* If the event is an aux_event, tear down all links to
2054	* it from other events.
2055	*/
2056	for_each_sibling_event(iter, event->group_leader) {
2057	if (iter->aux_event != event)
2058	continue;
2059
2060	iter->aux_event = NULL;
2061	put_event(event);
2062
2063	/*
2064	* If it's ACTIVE, schedule it out and put it into ERROR
2065	* state so that we don't try to schedule it again. Note
2066	* that perf_event_enable() will clear the ERROR status.
2067	*/
2068	event_sched_out(event: iter, ctx);
2069	perf_event_set_state(event, state: PERF_EVENT_STATE_ERROR);
2070	}
2071	}
2072
2073	static bool perf_need_aux_event(struct perf_event *event)
2074	{
2075	return !!event->attr.aux_output || !!event->attr.aux_sample_size;
2076	}
2077
2078	static int perf_get_aux_event(struct perf_event *event,
2079	struct perf_event *group_leader)
2080	{
2081	/*
2082	* Our group leader must be an aux event if we want to be
2083	* an aux_output. This way, the aux event will precede its
2084	* aux_output events in the group, and therefore will always
2085	* schedule first.
2086	*/
2087	if (!group_leader)
2088	return 0;
2089
2090	/*
2091	* aux_output and aux_sample_size are mutually exclusive.
2092	*/
2093	if (event->attr.aux_output && event->attr.aux_sample_size)
2094	return 0;
2095
2096	if (event->attr.aux_output &&
2097	!perf_aux_output_match(event, aux_event: group_leader))
2098	return 0;
2099
2100	if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux)
2101	return 0;
2102
2103	if (!atomic_long_inc_not_zero(v: &group_leader->refcount))
2104	return 0;
2105
2106	/*
2107	* Link aux_outputs to their aux event; this is undone in
2108	* perf_group_detach() by perf_put_aux_event(). When the
2109	* group in torn down, the aux_output events loose their
2110	* link to the aux_event and can't schedule any more.
2111	*/
2112	event->aux_event = group_leader;
2113
2114	return 1;
2115	}
2116
2117	static inline struct list_head *get_event_list(struct perf_event *event)
2118	{
2119	return event->attr.pinned ? &event->pmu_ctx->pinned_active :
2120	&event->pmu_ctx->flexible_active;
2121	}
2122
2123	/*
2124	* Events that have PERF_EV_CAP_SIBLING require being part of a group and
2125	* cannot exist on their own, schedule them out and move them into the ERROR
2126	* state. Also see _perf_event_enable(), it will not be able to recover
2127	* this ERROR state.
2128	*/
2129	static inline void perf_remove_sibling_event(struct perf_event *event)
2130	{
2131	event_sched_out(event, ctx: event->ctx);
2132	perf_event_set_state(event, state: PERF_EVENT_STATE_ERROR);
2133	}
2134
2135	static void perf_group_detach(struct perf_event *event)
2136	{
2137	struct perf_event *leader = event->group_leader;
2138	struct perf_event *sibling, *tmp;
2139	struct perf_event_context *ctx = event->ctx;
2140
2141	lockdep_assert_held(&ctx->lock);
2142
2143	/*
2144	* We can have double detach due to exit/hot-unplug + close.
2145	*/
2146	if (!(event->attach_state & PERF_ATTACH_GROUP))
2147	return;
2148
2149	event->attach_state &= ~PERF_ATTACH_GROUP;
2150
2151	perf_put_aux_event(event);
2152
2153	/*
2154	* If this is a sibling, remove it from its group.
2155	*/
2156	if (leader != event) {
2157	list_del_init(entry: &event->sibling_list);
2158	event->group_leader->nr_siblings--;
2159	event->group_leader->group_generation++;
2160	goto out;
2161	}
2162
2163	/*
2164	* If this was a group event with sibling events then
2165	* upgrade the siblings to singleton events by adding them
2166	* to whatever list we are on.
2167	*/
2168	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {
2169
2170	if (sibling->event_caps & PERF_EV_CAP_SIBLING)
2171	perf_remove_sibling_event(event: sibling);
2172
2173	sibling->group_leader = sibling;
2174	list_del_init(entry: &sibling->sibling_list);
2175
2176	/* Inherit group flags from the previous leader */
2177	sibling->group_caps = event->group_caps;
2178
2179	if (sibling->attach_state & PERF_ATTACH_CONTEXT) {
2180	add_event_to_groups(event: sibling, ctx: event->ctx);
2181
2182	if (sibling->state == PERF_EVENT_STATE_ACTIVE)
2183	list_add_tail(new: &sibling->active_list, head: get_event_list(event: sibling));
2184	}
2185
2186	WARN_ON_ONCE(sibling->ctx != event->ctx);
2187	}
2188
2189	out:
2190	for_each_sibling_event(tmp, leader)
2191	perf_event__header_size(event: tmp);
2192
2193	perf_event__header_size(event: leader);
2194	}
2195
2196	static void sync_child_event(struct perf_event *child_event);
2197
2198	static void perf_child_detach(struct perf_event *event)
2199	{
2200	struct perf_event *parent_event = event->parent;
2201
2202	if (!(event->attach_state & PERF_ATTACH_CHILD))
2203	return;
2204
2205	event->attach_state &= ~PERF_ATTACH_CHILD;
2206
2207	if (WARN_ON_ONCE(!parent_event))
2208	return;
2209
2210	lockdep_assert_held(&parent_event->child_mutex);
2211
2212	sync_child_event(child_event: event);
2213	list_del_init(entry: &event->child_list);
2214	}
2215
2216	static bool is_orphaned_event(struct perf_event *event)
2217	{
2218	return event->state == PERF_EVENT_STATE_DEAD;
2219	}
2220
2221	static inline int
2222	event_filter_match(struct perf_event *event)
2223	{
2224	return (event->cpu == -1 || event->cpu == smp_processor_id()) &&
2225	perf_cgroup_match(event);
2226	}
2227
2228	static void
2229	event_sched_out(struct perf_event *event, struct perf_event_context *ctx)
2230	{
2231	struct perf_event_pmu_context *epc = event->pmu_ctx;
2232	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);
2233	enum perf_event_state state = PERF_EVENT_STATE_INACTIVE;
2234
2235	// XXX cpc serialization, probably per-cpu IRQ disabled
2236
2237	WARN_ON_ONCE(event->ctx != ctx);
2238	lockdep_assert_held(&ctx->lock);
2239
2240	if (event->state != PERF_EVENT_STATE_ACTIVE)
2241	return;
2242
2243	/*
2244	* Asymmetry; we only schedule events _IN_ through ctx_sched_in(), but
2245	* we can schedule events _OUT_ individually through things like
2246	* __perf_remove_from_context().
2247	*/
2248	list_del_init(entry: &event->active_list);
2249
2250	perf_pmu_disable(pmu: event->pmu);
2251
2252	event->pmu->del(event, 0);
2253	event->oncpu = -1;
2254
2255	if (event->pending_disable) {
2256	event->pending_disable = 0;
2257	perf_cgroup_event_disable(event, ctx);
2258	state = PERF_EVENT_STATE_OFF;
2259	}
2260
2261	if (event->pending_sigtrap) {
2262	bool dec = true;
2263
2264	event->pending_sigtrap = 0;
2265	if (state != PERF_EVENT_STATE_OFF &&
2266	!event->pending_work) {
2267	event->pending_work = 1;
2268	dec = false;
2269	WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));
2270	task_work_add(current, twork: &event->pending_task, mode: TWA_RESUME);
2271	}
2272	if (dec)
2273	local_dec(l: &event->ctx->nr_pending);
2274	}
2275
2276	perf_event_set_state(event, state);
2277
2278	if (!is_software_event(event))
2279	cpc->active_oncpu--;
2280	if (event->attr.freq && event->attr.sample_freq)
2281	ctx->nr_freq--;
2282	if (event->attr.exclusive || !cpc->active_oncpu)
2283	cpc->exclusive = 0;
2284
2285	perf_pmu_enable(pmu: event->pmu);
2286	}
2287
2288	static void
2289	group_sched_out(struct perf_event *group_event, struct perf_event_context *ctx)
2290	{
2291	struct perf_event *event;
2292
2293	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
2294	return;
2295
2296	perf_assert_pmu_disabled(pmu: group_event->pmu_ctx->pmu);
2297
2298	event_sched_out(event: group_event, ctx);
2299
2300	/*
2301	* Schedule out siblings (if any):
2302	*/
2303	for_each_sibling_event(event, group_event)
2304	event_sched_out(event, ctx);
2305	}
2306
2307	#define DETACH_GROUP 0x01UL
2308	#define DETACH_CHILD 0x02UL
2309	#define DETACH_DEAD 0x04UL
2310
2311	/*
2312	* Cross CPU call to remove a performance event
2313	*
2314	* We disable the event on the hardware level first. After that we
2315	* remove it from the context list.
2316	*/
2317	static void
2318	__perf_remove_from_context(struct perf_event *event,
2319	struct perf_cpu_context *cpuctx,
2320	struct perf_event_context *ctx,
2321	void *info)
2322	{
2323	struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;
2324	unsigned long flags = (unsigned long)info;
2325
2326	if (ctx->is_active & EVENT_TIME) {
2327	update_context_time(ctx);
2328	update_cgrp_time_from_cpuctx(cpuctx, final: false);
2329	}
2330
2331	/*
2332	* Ensure event_sched_out() switches to OFF, at the very least
2333	* this avoids raising perf_pending_task() at this time.
2334	*/
2335	if (flags & DETACH_DEAD)
2336	event->pending_disable = 1;
2337	event_sched_out(event, ctx);
2338	if (flags & DETACH_GROUP)
2339	perf_group_detach(event);
2340	if (flags & DETACH_CHILD)
2341	perf_child_detach(event);
2342	list_del_event(event, ctx);
2343	if (flags & DETACH_DEAD)
2344	event->state = PERF_EVENT_STATE_DEAD;
2345
2346	if (!pmu_ctx->nr_events) {
2347	pmu_ctx->rotate_necessary = 0;
2348
2349	if (ctx->task && ctx->is_active) {
2350	struct perf_cpu_pmu_context *cpc;
2351
2352	cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);
2353	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
2354	cpc->task_epc = NULL;
2355	}
2356	}
2357
2358	if (!ctx->nr_events && ctx->is_active) {
2359	if (ctx == &cpuctx->ctx)
2360	update_cgrp_time_from_cpuctx(cpuctx, final: true);
2361
2362	ctx->is_active = 0;
2363	if (ctx->task) {
2364	WARN_ON_ONCE(cpuctx->task_ctx != ctx);
2365	cpuctx->task_ctx = NULL;
2366	}
2367	}
2368	}
2369
2370	/*
2371	* Remove the event from a task's (or a CPU's) list of events.
2372	*
2373	* If event->ctx is a cloned context, callers must make sure that
2374	* every task struct that event->ctx->task could possibly point to
2375	* remains valid. This is OK when called from perf_release since
2376	* that only calls us on the top-level context, which can't be a clone.
2377	* When called from perf_event_exit_task, it's OK because the
2378	* context has been detached from its task.
2379	*/
2380	static void perf_remove_from_context(struct perf_event *event, unsigned long flags)
2381	{
2382	struct perf_event_context *ctx = event->ctx;
2383
2384	lockdep_assert_held(&ctx->mutex);
2385
2386	/*
2387	* Because of perf_event_exit_task(), perf_remove_from_context() ought
2388	* to work in the face of TASK_TOMBSTONE, unlike every other
2389	* event_function_call() user.
2390	*/
2391	raw_spin_lock_irq(&ctx->lock);
2392	if (!ctx->is_active) {
2393	__perf_remove_from_context(event, this_cpu_ptr(&perf_cpu_context),
2394	ctx, info: (void *)flags);
2395	raw_spin_unlock_irq(&ctx->lock);
2396	return;
2397	}
2398	raw_spin_unlock_irq(&ctx->lock);
2399
2400	event_function_call(event, func: __perf_remove_from_context, data: (void *)flags);
2401	}
2402
2403	/*
2404	* Cross CPU call to disable a performance event
2405	*/
2406	static void __perf_event_disable(struct perf_event *event,
2407	struct perf_cpu_context *cpuctx,
2408	struct perf_event_context *ctx,
2409	void *info)
2410	{
2411	if (event->state < PERF_EVENT_STATE_INACTIVE)
2412	return;
2413
2414	if (ctx->is_active & EVENT_TIME) {
2415	update_context_time(ctx);
2416	update_cgrp_time_from_event(event);
2417	}
2418
2419	perf_pmu_disable(pmu: event->pmu_ctx->pmu);
2420
2421	if (event == event->group_leader)
2422	group_sched_out(group_event: event, ctx);
2423	else
2424	event_sched_out(event, ctx);
2425
2426	perf_event_set_state(event, state: PERF_EVENT_STATE_OFF);
2427	perf_cgroup_event_disable(event, ctx);
2428
2429	perf_pmu_enable(pmu: event->pmu_ctx->pmu);
2430	}
2431
2432	/*
2433	* Disable an event.
2434	*
2435	* If event->ctx is a cloned context, callers must make sure that
2436	* every task struct that event->ctx->task could possibly point to
2437	* remains valid. This condition is satisfied when called through
2438	* perf_event_for_each_child or perf_event_for_each because they
2439	* hold the top-level event's child_mutex, so any descendant that
2440	* goes to exit will block in perf_event_exit_event().
2441	*
2442	* When called from perf_pending_irq it's OK because event->ctx
2443	* is the current context on this CPU and preemption is disabled,
2444	* hence we can't get into perf_event_task_sched_out for this context.
2445	*/
2446	static void _perf_event_disable(struct perf_event *event)
2447	{
2448	struct perf_event_context *ctx = event->ctx;
2449
2450	raw_spin_lock_irq(&ctx->lock);
2451	if (event->state <= PERF_EVENT_STATE_OFF) {
2452	raw_spin_unlock_irq(&ctx->lock);
2453	return;
2454	}
2455	raw_spin_unlock_irq(&ctx->lock);
2456
2457	event_function_call(event, func: __perf_event_disable, NULL);
2458	}
2459
2460	void perf_event_disable_local(struct perf_event *event)
2461	{
2462	event_function_local(event, func: __perf_event_disable, NULL);
2463	}
2464
2465	/*
2466	* Strictly speaking kernel users cannot create groups and therefore this
2467	* interface does not need the perf_event_ctx_lock() magic.
2468	*/
2469	void perf_event_disable(struct perf_event *event)
2470	{
2471	struct perf_event_context *ctx;
2472
2473	ctx = perf_event_ctx_lock(event);
2474	_perf_event_disable(event);
2475	perf_event_ctx_unlock(event, ctx);
2476	}
2477	EXPORT_SYMBOL_GPL(perf_event_disable);
2478
2479	void perf_event_disable_inatomic(struct perf_event *event)
2480	{
2481	event->pending_disable = 1;
2482	irq_work_queue(work: &event->pending_irq);
2483	}
2484
2485	#define MAX_INTERRUPTS (~0ULL)
2486
2487	static void perf_log_throttle(struct perf_event *event, int enable);
2488	static void perf_log_itrace_start(struct perf_event *event);
2489
2490	static int
2491	event_sched_in(struct perf_event *event, struct perf_event_context *ctx)
2492	{
2493	struct perf_event_pmu_context *epc = event->pmu_ctx;
2494	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);
2495	int ret = 0;
2496
2497	WARN_ON_ONCE(event->ctx != ctx);
2498
2499	lockdep_assert_held(&ctx->lock);
2500
2501	if (event->state <= PERF_EVENT_STATE_OFF)
2502	return 0;
2503
2504	WRITE_ONCE(event->oncpu, smp_processor_id());
2505	/*
2506	* Order event::oncpu write to happen before the ACTIVE state is
2507	* visible. This allows perf_event_{stop,read}() to observe the correct
2508	* ->oncpu if it sees ACTIVE.
2509	*/
2510	smp_wmb();
2511	perf_event_set_state(event, state: PERF_EVENT_STATE_ACTIVE);
2512
2513	/*
2514	* Unthrottle events, since we scheduled we might have missed several
2515	* ticks already, also for a heavily scheduling task there is little
2516	* guarantee it'll get a tick in a timely manner.
2517	*/
2518	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
2519	perf_log_throttle(event, enable: 1);
2520	event->hw.interrupts = 0;
2521	}
2522
2523	perf_pmu_disable(pmu: event->pmu);
2524
2525	perf_log_itrace_start(event);
2526
2527	if (event->pmu->add(event, PERF_EF_START)) {
2528	perf_event_set_state(event, state: PERF_EVENT_STATE_INACTIVE);
2529	event->oncpu = -1;
2530	ret = -EAGAIN;
2531	goto out;
2532	}
2533
2534	if (!is_software_event(event))
2535	cpc->active_oncpu++;
2536	if (event->attr.freq && event->attr.sample_freq)
2537	ctx->nr_freq++;
2538
2539	if (event->attr.exclusive)
2540	cpc->exclusive = 1;
2541
2542	out:
2543	perf_pmu_enable(pmu: event->pmu);
2544
2545	return ret;
2546	}
2547
2548	static int
2549	group_sched_in(struct perf_event *group_event, struct perf_event_context *ctx)
2550	{
2551	struct perf_event *event, *partial_group = NULL;
2552	struct pmu *pmu = group_event->pmu_ctx->pmu;
2553
2554	if (group_event->state == PERF_EVENT_STATE_OFF)
2555	return 0;
2556
2557	pmu->start_txn(pmu, PERF_PMU_TXN_ADD);
2558
2559	if (event_sched_in(event: group_event, ctx))
2560	goto error;
2561
2562	/*
2563	* Schedule in siblings as one group (if any):
2564	*/
2565	for_each_sibling_event(event, group_event) {
2566	if (event_sched_in(event, ctx)) {
2567	partial_group = event;
2568	goto group_error;
2569	}
2570	}
2571
2572	if (!pmu->commit_txn(pmu))
2573	return 0;
2574
2575	group_error:
2576	/*
2577	* Groups can be scheduled in as one unit only, so undo any
2578	* partial group before returning:
2579	* The events up to the failed event are scheduled out normally.
2580	*/
2581	for_each_sibling_event(event, group_event) {
2582	if (event == partial_group)
2583	break;
2584
2585	event_sched_out(event, ctx);
2586	}
2587	event_sched_out(event: group_event, ctx);
2588
2589	error:
2590	pmu->cancel_txn(pmu);
2591	return -EAGAIN;
2592	}
2593
2594	/*
2595	* Work out whether we can put this event group on the CPU now.
2596	*/
2597	static int group_can_go_on(struct perf_event *event, int can_add_hw)
2598	{
2599	struct perf_event_pmu_context *epc = event->pmu_ctx;
2600	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);
2601
2602	/*
2603	* Groups consisting entirely of software events can always go on.
2604	*/
2605	if (event->group_caps & PERF_EV_CAP_SOFTWARE)
2606	return 1;
2607	/*
2608	* If an exclusive group is already on, no other hardware
2609	* events can go on.
2610	*/
2611	if (cpc->exclusive)
2612	return 0;
2613	/*
2614	* If this group is exclusive and there are already
2615	* events on the CPU, it can't go on.
2616	*/
2617	if (event->attr.exclusive && !list_empty(head: get_event_list(event)))
2618	return 0;
2619	/*
2620	* Otherwise, try to add it if all previous groups were able
2621	* to go on.
2622	*/
2623	return can_add_hw;
2624	}
2625
2626	static void add_event_to_ctx(struct perf_event *event,
2627	struct perf_event_context *ctx)
2628	{
2629	list_add_event(event, ctx);
2630	perf_group_attach(event);
2631	}
2632
2633	static void task_ctx_sched_out(struct perf_event_context *ctx,
2634	enum event_type_t event_type)
2635	{
2636	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
2637
2638	if (!cpuctx->task_ctx)
2639	return;
2640
2641	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2642	return;
2643
2644	ctx_sched_out(ctx, event_type);
2645	}
2646
2647	static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
2648	struct perf_event_context *ctx)
2649	{
2650	ctx_sched_in(ctx: &cpuctx->ctx, event_type: EVENT_PINNED);
2651	if (ctx)
2652	ctx_sched_in(ctx, event_type: EVENT_PINNED);
2653	ctx_sched_in(ctx: &cpuctx->ctx, event_type: EVENT_FLEXIBLE);
2654	if (ctx)
2655	ctx_sched_in(ctx, event_type: EVENT_FLEXIBLE);
2656	}
2657
2658	/*
2659	* We want to maintain the following priority of scheduling:
2660	* - CPU pinned (EVENT_CPU | EVENT_PINNED)
2661	* - task pinned (EVENT_PINNED)
2662	* - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE)
2663	* - task flexible (EVENT_FLEXIBLE).
2664	*
2665	* In order to avoid unscheduling and scheduling back in everything every
2666	* time an event is added, only do it for the groups of equal priority and
2667	* below.
2668	*
2669	* This can be called after a batch operation on task events, in which case
2670	* event_type is a bit mask of the types of events involved. For CPU events,
2671	* event_type is only either EVENT_PINNED or EVENT_FLEXIBLE.
2672	*/
2673	/*
2674	* XXX: ctx_resched() reschedule entire perf_event_context while adding new
2675	* event to the context or enabling existing event in the context. We can
2676	* probably optimize it by rescheduling only affected pmu_ctx.
2677	*/
2678	static void ctx_resched(struct perf_cpu_context *cpuctx,
2679	struct perf_event_context *task_ctx,
2680	enum event_type_t event_type)
2681	{
2682	bool cpu_event = !!(event_type & EVENT_CPU);
2683
2684	/*
2685	* If pinned groups are involved, flexible groups also need to be
2686	* scheduled out.
2687	*/
2688	if (event_type & EVENT_PINNED)
2689	event_type |= EVENT_FLEXIBLE;
2690
2691	event_type &= EVENT_ALL;
2692
2693	perf_ctx_disable(ctx: &cpuctx->ctx, cgroup: false);
2694	if (task_ctx) {
2695	perf_ctx_disable(ctx: task_ctx, cgroup: false);
2696	task_ctx_sched_out(ctx: task_ctx, event_type);
2697	}
2698
2699	/*
2700	* Decide which cpu ctx groups to schedule out based on the types
2701	* of events that caused rescheduling:
2702	* - EVENT_CPU: schedule out corresponding groups;
2703	* - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups;
2704	* - otherwise, do nothing more.
2705	*/
2706	if (cpu_event)
2707	ctx_sched_out(ctx: &cpuctx->ctx, event_type);
2708	else if (event_type & EVENT_PINNED)
2709	ctx_sched_out(ctx: &cpuctx->ctx, event_type: EVENT_FLEXIBLE);
2710
2711	perf_event_sched_in(cpuctx, ctx: task_ctx);
2712
2713	perf_ctx_enable(ctx: &cpuctx->ctx, cgroup: false);
2714	if (task_ctx)
2715	perf_ctx_enable(ctx: task_ctx, cgroup: false);
2716	}
2717
2718	void perf_pmu_resched(struct pmu *pmu)
2719	{
2720	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
2721	struct perf_event_context *task_ctx = cpuctx->task_ctx;
2722
2723	perf_ctx_lock(cpuctx, ctx: task_ctx);
2724	ctx_resched(cpuctx, task_ctx, event_type: EVENT_ALL|EVENT_CPU);
2725	perf_ctx_unlock(cpuctx, ctx: task_ctx);
2726	}
2727
2728	/*
2729	* Cross CPU call to install and enable a performance event
2730	*
2731	* Very similar to remote_function() + event_function() but cannot assume that
2732	* things like ctx->is_active and cpuctx->task_ctx are set.
2733	*/
2734	static int __perf_install_in_context(void *info)
2735	{
2736	struct perf_event *event = info;
2737	struct perf_event_context *ctx = event->ctx;
2738	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
2739	struct perf_event_context *task_ctx = cpuctx->task_ctx;
2740	bool reprogram = true;
2741	int ret = 0;
2742
2743	raw_spin_lock(&cpuctx->ctx.lock);
2744	if (ctx->task) {
2745	raw_spin_lock(&ctx->lock);
2746	task_ctx = ctx;
2747
2748	reprogram = (ctx->task == current);
2749
2750	/*
2751	* If the task is running, it must be running on this CPU,
2752	* otherwise we cannot reprogram things.
2753	*
2754	* If its not running, we don't care, ctx->lock will
2755	* serialize against it becoming runnable.
2756	*/
2757	if (task_curr(p: ctx->task) && !reprogram) {
2758	ret = -ESRCH;
2759	goto unlock;
2760	}
2761
2762	WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);
2763	} else if (task_ctx) {
2764	raw_spin_lock(&task_ctx->lock);
2765	}
2766
2767	#ifdef CONFIG_CGROUP_PERF
2768	if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {
2769	/*
2770	* If the current cgroup doesn't match the event's
2771	* cgroup, we should not try to schedule it.
2772	*/
2773	struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);
2774	reprogram = cgroup_is_descendant(cgrp: cgrp->css.cgroup,
2775	ancestor: event->cgrp->css.cgroup);
2776	}
2777	#endif
2778
2779	if (reprogram) {
2780	ctx_sched_out(ctx, event_type: EVENT_TIME);
2781	add_event_to_ctx(event, ctx);
2782	ctx_resched(cpuctx, task_ctx, event_type: get_event_type(event));
2783	} else {
2784	add_event_to_ctx(event, ctx);
2785	}
2786
2787	unlock:
2788	perf_ctx_unlock(cpuctx, ctx: task_ctx);
2789
2790	return ret;
2791	}
2792
2793	static bool exclusive_event_installable(struct perf_event *event,
2794	struct perf_event_context *ctx);
2795
2796	/*
2797	* Attach a performance event to a context.
2798	*
2799	* Very similar to event_function_call, see comment there.
2800	*/
2801	static void
2802	perf_install_in_context(struct perf_event_context *ctx,
2803	struct perf_event *event,
2804	int cpu)
2805	{
2806	struct task_struct *task = READ_ONCE(ctx->task);
2807
2808	lockdep_assert_held(&ctx->mutex);
2809
2810	WARN_ON_ONCE(!exclusive_event_installable(event, ctx));
2811
2812	if (event->cpu != -1)
2813	WARN_ON_ONCE(event->cpu != cpu);
2814
2815	/*
2816	* Ensures that if we can observe event->ctx, both the event and ctx
2817	* will be 'complete'. See perf_iterate_sb_cpu().
2818	*/
2819	smp_store_release(&event->ctx, ctx);
2820
2821	/*
2822	* perf_event_attr::disabled events will not run and can be initialized
2823	* without IPI. Except when this is the first event for the context, in
2824	* that case we need the magic of the IPI to set ctx->is_active.
2825	*
2826	* The IOC_ENABLE that is sure to follow the creation of a disabled
2827	* event will issue the IPI and reprogram the hardware.
2828	*/
2829	if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF &&
2830	ctx->nr_events && !is_cgroup_event(event)) {
2831	raw_spin_lock_irq(&ctx->lock);
2832	if (ctx->task == TASK_TOMBSTONE) {
2833	raw_spin_unlock_irq(&ctx->lock);
2834	return;
2835	}
2836	add_event_to_ctx(event, ctx);
2837	raw_spin_unlock_irq(&ctx->lock);
2838	return;
2839	}
2840
2841	if (!task) {
2842	cpu_function_call(cpu, func: __perf_install_in_context, info: event);
2843	return;
2844	}
2845
2846	/*
2847	* Should not happen, we validate the ctx is still alive before calling.
2848	*/
2849	if (WARN_ON_ONCE(task == TASK_TOMBSTONE))
2850	return;
2851
2852	/*
2853	* Installing events is tricky because we cannot rely on ctx->is_active
2854	* to be set in case this is the nr_events 0 -> 1 transition.
2855	*
2856	* Instead we use task_curr(), which tells us if the task is running.
2857	* However, since we use task_curr() outside of rq::lock, we can race
2858	* against the actual state. This means the result can be wrong.
2859	*
2860	* If we get a false positive, we retry, this is harmless.
2861	*
2862	* If we get a false negative, things are complicated. If we are after
2863	* perf_event_context_sched_in() ctx::lock will serialize us, and the
2864	* value must be correct. If we're before, it doesn't matter since
2865	* perf_event_context_sched_in() will program the counter.
2866	*
2867	* However, this hinges on the remote context switch having observed
2868	* our task->perf_event_ctxp[] store, such that it will in fact take
2869	* ctx::lock in perf_event_context_sched_in().
2870	*
2871	* We do this by task_function_call(), if the IPI fails to hit the task
2872	* we know any future context switch of task must see the
2873	* perf_event_ctpx[] store.
2874	*/
2875
2876	/*
2877	* This smp_mb() orders the task->perf_event_ctxp[] store with the
2878	* task_cpu() load, such that if the IPI then does not find the task
2879	* running, a future context switch of that task must observe the
2880	* store.
2881	*/
2882	smp_mb();
2883	again:
2884	if (!task_function_call(p: task, func: __perf_install_in_context, info: event))
2885	return;
2886
2887	raw_spin_lock_irq(&ctx->lock);
2888	task = ctx->task;
2889	if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) {
2890	/*
2891	* Cannot happen because we already checked above (which also
2892	* cannot happen), and we hold ctx->mutex, which serializes us
2893	* against perf_event_exit_task_context().
2894	*/
2895	raw_spin_unlock_irq(&ctx->lock);
2896	return;
2897	}
2898	/*
2899	* If the task is not running, ctx->lock will avoid it becoming so,
2900	* thus we can safely install the event.
2901	*/
2902	if (task_curr(p: task)) {
2903	raw_spin_unlock_irq(&ctx->lock);
2904	goto again;
2905	}
2906	add_event_to_ctx(event, ctx);
2907	raw_spin_unlock_irq(&ctx->lock);
2908	}
2909
2910	/*
2911	* Cross CPU call to enable a performance event
2912	*/
2913	static void __perf_event_enable(struct perf_event *event,
2914	struct perf_cpu_context *cpuctx,
2915	struct perf_event_context *ctx,
2916	void *info)
2917	{
2918	struct perf_event *leader = event->group_leader;
2919	struct perf_event_context *task_ctx;
2920
2921	if (event->state >= PERF_EVENT_STATE_INACTIVE ||
2922	event->state <= PERF_EVENT_STATE_ERROR)
2923	return;
2924
2925	if (ctx->is_active)
2926	ctx_sched_out(ctx, event_type: EVENT_TIME);
2927
2928	perf_event_set_state(event, state: PERF_EVENT_STATE_INACTIVE);
2929	perf_cgroup_event_enable(event, ctx);
2930
2931	if (!ctx->is_active)
2932	return;
2933
2934	if (!event_filter_match(event)) {
2935	ctx_sched_in(ctx, event_type: EVENT_TIME);
2936	return;
2937	}
2938
2939	/*
2940	* If the event is in a group and isn't the group leader,
2941	* then don't put it on unless the group is on.
2942	*/
2943	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) {
2944	ctx_sched_in(ctx, event_type: EVENT_TIME);
2945	return;
2946	}
2947
2948	task_ctx = cpuctx->task_ctx;
2949	if (ctx->task)
2950	WARN_ON_ONCE(task_ctx != ctx);
2951
2952	ctx_resched(cpuctx, task_ctx, event_type: get_event_type(event));
2953	}
2954
2955	/*
2956	* Enable an event.
2957	*
2958	* If event->ctx is a cloned context, callers must make sure that
2959	* every task struct that event->ctx->task could possibly point to
2960	* remains valid. This condition is satisfied when called through
2961	* perf_event_for_each_child or perf_event_for_each as described
2962	* for perf_event_disable.
2963	*/
2964	static void _perf_event_enable(struct perf_event *event)
2965	{
2966	struct perf_event_context *ctx = event->ctx;
2967
2968	raw_spin_lock_irq(&ctx->lock);
2969	if (event->state >= PERF_EVENT_STATE_INACTIVE ||
2970	event->state < PERF_EVENT_STATE_ERROR) {
2971	out:
2972	raw_spin_unlock_irq(&ctx->lock);
2973	return;
2974	}
2975
2976	/*
2977	* If the event is in error state, clear that first.
2978	*
2979	* That way, if we see the event in error state below, we know that it
2980	* has gone back into error state, as distinct from the task having
2981	* been scheduled away before the cross-call arrived.
2982	*/
2983	if (event->state == PERF_EVENT_STATE_ERROR) {
2984	/*
2985	* Detached SIBLING events cannot leave ERROR state.
2986	*/
2987	if (event->event_caps & PERF_EV_CAP_SIBLING &&
2988	event->group_leader == event)
2989	goto out;
2990
2991	event->state = PERF_EVENT_STATE_OFF;
2992	}
2993	raw_spin_unlock_irq(&ctx->lock);
2994
2995	event_function_call(event, func: __perf_event_enable, NULL);
2996	}
2997
2998	/*
2999	* See perf_event_disable();
3000	*/
3001	void perf_event_enable(struct perf_event *event)
3002	{
3003	struct perf_event_context *ctx;
3004
3005	ctx = perf_event_ctx_lock(event);
3006	_perf_event_enable(event);
3007	perf_event_ctx_unlock(event, ctx);
3008	}
3009	EXPORT_SYMBOL_GPL(perf_event_enable);
3010
3011	struct stop_event_data {
3012	struct perf_event *event;
3013	unsigned int restart;
3014	};
3015
3016	static int __perf_event_stop(void *info)
3017	{
3018	struct stop_event_data *sd = info;
3019	struct perf_event *event = sd->event;
3020
3021	/* if it's already INACTIVE, do nothing */
3022	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
3023	return 0;
3024
3025	/* matches smp_wmb() in event_sched_in() */
3026	smp_rmb();
3027
3028	/*
3029	* There is a window with interrupts enabled before we get here,
3030	* so we need to check again lest we try to stop another CPU's event.
3031	*/
3032	if (READ_ONCE(event->oncpu) != smp_processor_id())
3033	return -EAGAIN;
3034
3035	event->pmu->stop(event, PERF_EF_UPDATE);
3036
3037	/*
3038	* May race with the actual stop (through perf_pmu_output_stop()),
3039	* but it is only used for events with AUX ring buffer, and such
3040	* events will refuse to restart because of rb::aux_mmap_count==0,
3041	* see comments in perf_aux_output_begin().
3042	*
3043	* Since this is happening on an event-local CPU, no trace is lost
3044	* while restarting.
3045	*/
3046	if (sd->restart)
3047	event->pmu->start(event, 0);
3048
3049	return 0;
3050	}
3051
3052	static int perf_event_stop(struct perf_event *event, int restart)
3053	{
3054	struct stop_event_data sd = {
3055	.event = event,
3056	.restart = restart,
3057	};
3058	int ret = 0;
3059
3060	do {
3061	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
3062	return 0;
3063
3064	/* matches smp_wmb() in event_sched_in() */
3065	smp_rmb();
3066
3067	/*
3068	* We only want to restart ACTIVE events, so if the event goes
3069	* inactive here (event->oncpu==-1), there's nothing more to do;
3070	* fall through with ret==-ENXIO.
3071	*/
3072	ret = cpu_function_call(READ_ONCE(event->oncpu),
3073	func: __perf_event_stop, info: &sd);
3074	} while (ret == -EAGAIN);
3075
3076	return ret;
3077	}
3078
3079	/*
3080	* In order to contain the amount of racy and tricky in the address filter
3081	* configuration management, it is a two part process:
3082	*
3083	* (p1) when userspace mappings change as a result of (1) or (2) or (3) below,
3084	* we update the addresses of corresponding vmas in
3085	* event::addr_filter_ranges array and bump the event::addr_filters_gen;
3086	* (p2) when an event is scheduled in (pmu::add), it calls
3087	* perf_event_addr_filters_sync() which calls pmu::addr_filters_sync()
3088	* if the generation has changed since the previous call.
3089	*
3090	* If (p1) happens while the event is active, we restart it to force (p2).
3091	*
3092	* (1) perf_addr_filters_apply(): adjusting filters' offsets based on
3093	* pre-existing mappings, called once when new filters arrive via SET_FILTER
3094	* ioctl;
3095	* (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly
3096	* registered mapping, called for every new mmap(), with mm::mmap_lock down
3097	* for reading;
3098	* (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process
3099	* of exec.
3100	*/
3101	void perf_event_addr_filters_sync(struct perf_event *event)
3102	{
3103	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
3104
3105	if (!has_addr_filter(event))
3106	return;
3107
3108	raw_spin_lock(&ifh->lock);
3109	if (event->addr_filters_gen != event->hw.addr_filters_gen) {
3110	event->pmu->addr_filters_sync(event);
3111	event->hw.addr_filters_gen = event->addr_filters_gen;
3112	}
3113	raw_spin_unlock(&ifh->lock);
3114	}
3115	EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync);
3116
3117	static int _perf_event_refresh(struct perf_event *event, int refresh)
3118	{
3119	/*
3120	* not supported on inherited events
3121	*/
3122	if (event->attr.inherit || !is_sampling_event(event))
3123	return -EINVAL;
3124
3125	atomic_add(i: refresh, v: &event->event_limit);
3126	_perf_event_enable(event);
3127
3128	return 0;
3129	}
3130
3131	/*
3132	* See perf_event_disable()
3133	*/
3134	int perf_event_refresh(struct perf_event *event, int refresh)
3135	{
3136	struct perf_event_context *ctx;
3137	int ret;
3138
3139	ctx = perf_event_ctx_lock(event);
3140	ret = _perf_event_refresh(event, refresh);
3141	perf_event_ctx_unlock(event, ctx);
3142
3143	return ret;
3144	}
3145	EXPORT_SYMBOL_GPL(perf_event_refresh);
3146
3147	static int perf_event_modify_breakpoint(struct perf_event *bp,
3148	struct perf_event_attr *attr)
3149	{
3150	int err;
3151
3152	_perf_event_disable(event: bp);
3153
3154	err = modify_user_hw_breakpoint_check(bp, attr, check: true);
3155
3156	if (!bp->attr.disabled)
3157	_perf_event_enable(event: bp);
3158
3159	return err;
3160	}
3161
3162	/*
3163	* Copy event-type-independent attributes that may be modified.
3164	*/
3165	static void perf_event_modify_copy_attr(struct perf_event_attr *to,
3166	const struct perf_event_attr *from)
3167	{
3168	to->sig_data = from->sig_data;
3169	}
3170
3171	static int perf_event_modify_attr(struct perf_event *event,
3172	struct perf_event_attr *attr)
3173	{
3174	int (*func)(struct perf_event *, struct perf_event_attr *);
3175	struct perf_event *child;
3176	int err;
3177
3178	if (event->attr.type != attr->type)
3179	return -EINVAL;
3180
3181	switch (event->attr.type) {
3182	case PERF_TYPE_BREAKPOINT:
3183	func = perf_event_modify_breakpoint;
3184	break;
3185	default:
3186	/* Place holder for future additions. */
3187	return -EOPNOTSUPP;
3188	}
3189
3190	WARN_ON_ONCE(event->ctx->parent_ctx);
3191
3192	mutex_lock(&event->child_mutex);
3193	/*
3194	* Event-type-independent attributes must be copied before event-type
3195	* modification, which will validate that final attributes match the
3196	* source attributes after all relevant attributes have been copied.
3197	*/
3198	perf_event_modify_copy_attr(to: &event->attr, from: attr);
3199	err = func(event, attr);
3200	if (err)
3201	goto out;
3202	list_for_each_entry(child, &event->child_list, child_list) {
3203	perf_event_modify_copy_attr(to: &child->attr, from: attr);
3204	err = func(child, attr);
3205	if (err)
3206	goto out;
3207	}
3208	out:
3209	mutex_unlock(lock: &event->child_mutex);
3210	return err;
3211	}
3212
3213	static void __pmu_ctx_sched_out(struct perf_event_pmu_context *pmu_ctx,
3214	enum event_type_t event_type)
3215	{
3216	struct perf_event_context *ctx = pmu_ctx->ctx;
3217	struct perf_event *event, *tmp;
3218	struct pmu *pmu = pmu_ctx->pmu;
3219
3220	if (ctx->task && !ctx->is_active) {
3221	struct perf_cpu_pmu_context *cpc;
3222
3223	cpc = this_cpu_ptr(pmu->cpu_pmu_context);
3224	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
3225	cpc->task_epc = NULL;
3226	}
3227
3228	if (!event_type)
3229	return;
3230
3231	perf_pmu_disable(pmu);
3232	if (event_type & EVENT_PINNED) {
3233	list_for_each_entry_safe(event, tmp,
3234	&pmu_ctx->pinned_active,
3235	active_list)
3236	group_sched_out(group_event: event, ctx);
3237	}
3238
3239	if (event_type & EVENT_FLEXIBLE) {
3240	list_for_each_entry_safe(event, tmp,
3241	&pmu_ctx->flexible_active,
3242	active_list)
3243	group_sched_out(group_event: event, ctx);
3244	/*
3245	* Since we cleared EVENT_FLEXIBLE, also clear
3246	* rotate_necessary, is will be reset by
3247	* ctx_flexible_sched_in() when needed.
3248	*/
3249	pmu_ctx->rotate_necessary = 0;
3250	}
3251	perf_pmu_enable(pmu);
3252	}
3253
3254	static void
3255	ctx_sched_out(struct perf_event_context *ctx, enum event_type_t event_type)
3256	{
3257	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3258	struct perf_event_pmu_context *pmu_ctx;
3259	int is_active = ctx->is_active;
3260	bool cgroup = event_type & EVENT_CGROUP;
3261
3262	event_type &= ~EVENT_CGROUP;
3263
3264	lockdep_assert_held(&ctx->lock);
3265
3266	if (likely(!ctx->nr_events)) {
3267	/*
3268	* See __perf_remove_from_context().
3269	*/
3270	WARN_ON_ONCE(ctx->is_active);
3271	if (ctx->task)
3272	WARN_ON_ONCE(cpuctx->task_ctx);
3273	return;
3274	}
3275
3276	/*
3277	* Always update time if it was set; not only when it changes.
3278	* Otherwise we can 'forget' to update time for any but the last
3279	* context we sched out. For example:
3280	*
3281	* ctx_sched_out(.event_type = EVENT_FLEXIBLE)
3282	* ctx_sched_out(.event_type = EVENT_PINNED)
3283	*
3284	* would only update time for the pinned events.
3285	*/
3286	if (is_active & EVENT_TIME) {
3287	/* update (and stop) ctx time */
3288	update_context_time(ctx);
3289	update_cgrp_time_from_cpuctx(cpuctx, final: ctx == &cpuctx->ctx);
3290	/*
3291	* CPU-release for the below ->is_active store,
3292	* see __load_acquire() in perf_event_time_now()
3293	*/
3294	barrier();
3295	}
3296
3297	ctx->is_active &= ~event_type;
3298	if (!(ctx->is_active & EVENT_ALL))
3299	ctx->is_active = 0;
3300
3301	if (ctx->task) {
3302	WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3303	if (!ctx->is_active)
3304	cpuctx->task_ctx = NULL;
3305	}
3306
3307	is_active ^= ctx->is_active; /* changed bits */
3308
3309	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
3310	if (cgroup && !pmu_ctx->nr_cgroups)
3311	continue;
3312	__pmu_ctx_sched_out(pmu_ctx, event_type: is_active);
3313	}
3314	}
3315
3316	/*
3317	* Test whether two contexts are equivalent, i.e. whether they have both been
3318	* cloned from the same version of the same context.
3319	*
3320	* Equivalence is measured using a generation number in the context that is
3321	* incremented on each modification to it; see unclone_ctx(), list_add_event()
3322	* and list_del_event().
3323	*/
3324	static int context_equiv(struct perf_event_context *ctx1,
3325	struct perf_event_context *ctx2)
3326	{
3327	lockdep_assert_held(&ctx1->lock);
3328	lockdep_assert_held(&ctx2->lock);
3329
3330	/* Pinning disables the swap optimization */
3331	if (ctx1->pin_count || ctx2->pin_count)
3332	return 0;
3333
3334	/* If ctx1 is the parent of ctx2 */
3335	if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
3336	return 1;
3337
3338	/* If ctx2 is the parent of ctx1 */
3339	if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
3340	return 1;
3341
3342	/*
3343	* If ctx1 and ctx2 have the same parent; we flatten the parent
3344	* hierarchy, see perf_event_init_context().
3345	*/
3346	if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
3347	ctx1->parent_gen == ctx2->parent_gen)
3348	return 1;
3349
3350	/* Unmatched */
3351	return 0;
3352	}
3353
3354	static void __perf_event_sync_stat(struct perf_event *event,
3355	struct perf_event *next_event)
3356	{
3357	u64 value;
3358
3359	if (!event->attr.inherit_stat)
3360	return;
3361
3362	/*
3363	* Update the event value, we cannot use perf_event_read()
3364	* because we're in the middle of a context switch and have IRQs
3365	* disabled, which upsets smp_call_function_single(), however
3366	* we know the event must be on the current CPU, therefore we
3367	* don't need to use it.
3368	*/
3369	if (event->state == PERF_EVENT_STATE_ACTIVE)
3370	event->pmu->read(event);
3371
3372	perf_event_update_time(event);
3373
3374	/*
3375	* In order to keep per-task stats reliable we need to flip the event
3376	* values when we flip the contexts.
3377	*/
3378	value = local64_read(&next_event->count);
3379	value = local64_xchg(&event->count, value);
3380	local64_set(&next_event->count, value);
3381
3382	swap(event->total_time_enabled, next_event->total_time_enabled);
3383	swap(event->total_time_running, next_event->total_time_running);
3384
3385	/*
3386	* Since we swizzled the values, update the user visible data too.
3387	*/
3388	perf_event_update_userpage(event);
3389	perf_event_update_userpage(event: next_event);
3390	}
3391
3392	static void perf_event_sync_stat(struct perf_event_context *ctx,
3393	struct perf_event_context *next_ctx)
3394	{
3395	struct perf_event *event, *next_event;
3396
3397	if (!ctx->nr_stat)
3398	return;
3399
3400	update_context_time(ctx);
3401
3402	event = list_first_entry(&ctx->event_list,
3403	struct perf_event, event_entry);
3404
3405	next_event = list_first_entry(&next_ctx->event_list,
3406	struct perf_event, event_entry);
3407
3408	while (&event->event_entry != &ctx->event_list &&
3409	&next_event->event_entry != &next_ctx->event_list) {
3410
3411	__perf_event_sync_stat(event, next_event);
3412
3413	event = list_next_entry(event, event_entry);
3414	next_event = list_next_entry(next_event, event_entry);
3415	}
3416	}
3417
3418	#define double_list_for_each_entry(pos1, pos2, head1, head2, member) \
3419	for (pos1 = list_first_entry(head1, typeof(*pos1), member), \
3420	pos2 = list_first_entry(head2, typeof(*pos2), member); \
3421	!list_entry_is_head(pos1, head1, member) && \
3422	!list_entry_is_head(pos2, head2, member); \
3423	pos1 = list_next_entry(pos1, member), \
3424	pos2 = list_next_entry(pos2, member))
3425
3426	static void perf_event_swap_task_ctx_data(struct perf_event_context *prev_ctx,
3427	struct perf_event_context *next_ctx)
3428	{
3429	struct perf_event_pmu_context *prev_epc, *next_epc;
3430
3431	if (!prev_ctx->nr_task_data)
3432	return;
3433
3434	double_list_for_each_entry(prev_epc, next_epc,
3435	&prev_ctx->pmu_ctx_list, &next_ctx->pmu_ctx_list,
3436	pmu_ctx_entry) {
3437
3438	if (WARN_ON_ONCE(prev_epc->pmu != next_epc->pmu))
3439	continue;
3440
3441	/*
3442	* PMU specific parts of task perf context can require
3443	* additional synchronization. As an example of such
3444	* synchronization see implementation details of Intel
3445	* LBR call stack data profiling;
3446	*/
3447	if (prev_epc->pmu->swap_task_ctx)
3448	prev_epc->pmu->swap_task_ctx(prev_epc, next_epc);
3449	else
3450	swap(prev_epc->task_ctx_data, next_epc->task_ctx_data);
3451	}
3452	}
3453
3454	static void perf_ctx_sched_task_cb(struct perf_event_context *ctx, bool sched_in)
3455	{
3456	struct perf_event_pmu_context *pmu_ctx;
3457	struct perf_cpu_pmu_context *cpc;
3458
3459	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
3460	cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);
3461
3462	if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task)
3463	pmu_ctx->pmu->sched_task(pmu_ctx, sched_in);
3464	}
3465	}
3466
3467	static void
3468	perf_event_context_sched_out(struct task_struct *task, struct task_struct *next)
3469	{
3470	struct perf_event_context *ctx = task->perf_event_ctxp;
3471	struct perf_event_context *next_ctx;
3472	struct perf_event_context *parent, *next_parent;
3473	int do_switch = 1;
3474
3475	if (likely(!ctx))
3476	return;
3477
3478	rcu_read_lock();
3479	next_ctx = rcu_dereference(next->perf_event_ctxp);
3480	if (!next_ctx)
3481	goto unlock;
3482
3483	parent = rcu_dereference(ctx->parent_ctx);
3484	next_parent = rcu_dereference(next_ctx->parent_ctx);
3485
3486	/* If neither context have a parent context; they cannot be clones. */
3487	if (!parent && !next_parent)
3488	goto unlock;
3489
3490	if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
3491	/*
3492	* Looks like the two contexts are clones, so we might be
3493	* able to optimize the context switch. We lock both
3494	* contexts and check that they are clones under the
3495	* lock (including re-checking that neither has been
3496	* uncloned in the meantime). It doesn't matter which
3497	* order we take the locks because no other cpu could
3498	* be trying to lock both of these tasks.
3499	*/
3500	raw_spin_lock(&ctx->lock);
3501	raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
3502	if (context_equiv(ctx1: ctx, ctx2: next_ctx)) {
3503
3504	perf_ctx_disable(ctx, cgroup: false);
3505
3506	/* PMIs are disabled; ctx->nr_pending is stable. */
3507	if (local_read(&ctx->nr_pending) ||
3508	local_read(&next_ctx->nr_pending)) {
3509	/*
3510	* Must not swap out ctx when there's pending
3511	* events that rely on the ctx->task relation.
3512	*/
3513	raw_spin_unlock(&next_ctx->lock);
3514	rcu_read_unlock();
3515	goto inside_switch;
3516	}
3517
3518	WRITE_ONCE(ctx->task, next);
3519	WRITE_ONCE(next_ctx->task, task);
3520
3521	perf_ctx_sched_task_cb(ctx, sched_in: false);
3522	perf_event_swap_task_ctx_data(prev_ctx: ctx, next_ctx);
3523
3524	perf_ctx_enable(ctx, cgroup: false);
3525
3526	/*
3527	* RCU_INIT_POINTER here is safe because we've not
3528	* modified the ctx and the above modification of
3529	* ctx->task and ctx->task_ctx_data are immaterial
3530	* since those values are always verified under
3531	* ctx->lock which we're now holding.
3532	*/
3533	RCU_INIT_POINTER(task->perf_event_ctxp, next_ctx);
3534	RCU_INIT_POINTER(next->perf_event_ctxp, ctx);
3535
3536	do_switch = 0;
3537
3538	perf_event_sync_stat(ctx, next_ctx);
3539	}
3540	raw_spin_unlock(&next_ctx->lock);
3541	raw_spin_unlock(&ctx->lock);
3542	}
3543	unlock:
3544	rcu_read_unlock();
3545
3546	if (do_switch) {
3547	raw_spin_lock(&ctx->lock);
3548	perf_ctx_disable(ctx, cgroup: false);
3549
3550	inside_switch:
3551	perf_ctx_sched_task_cb(ctx, sched_in: false);
3552	task_ctx_sched_out(ctx, event_type: EVENT_ALL);
3553
3554	perf_ctx_enable(ctx, cgroup: false);
3555	raw_spin_unlock(&ctx->lock);
3556	}
3557	}
3558
3559	static DEFINE_PER_CPU(struct list_head, sched_cb_list);
3560	static DEFINE_PER_CPU(int, perf_sched_cb_usages);
3561
3562	void perf_sched_cb_dec(struct pmu *pmu)
3563	{
3564	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(pmu->cpu_pmu_context);
3565
3566	this_cpu_dec(perf_sched_cb_usages);
3567	barrier();
3568
3569	if (!--cpc->sched_cb_usage)
3570	list_del(entry: &cpc->sched_cb_entry);
3571	}
3572
3573
3574	void perf_sched_cb_inc(struct pmu *pmu)
3575	{
3576	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(pmu->cpu_pmu_context);
3577
3578	if (!cpc->sched_cb_usage++)
3579	list_add(new: &cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list));
3580
3581	barrier();
3582	this_cpu_inc(perf_sched_cb_usages);
3583	}
3584
3585	/*
3586	* This function provides the context switch callback to the lower code
3587	* layer. It is invoked ONLY when the context switch callback is enabled.
3588	*
3589	* This callback is relevant even to per-cpu events; for example multi event
3590	* PEBS requires this to provide PID/TID information. This requires we flush
3591	* all queued PEBS records before we context switch to a new task.
3592	*/
3593	static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc, bool sched_in)
3594	{
3595	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3596	struct pmu *pmu;
3597
3598	pmu = cpc->epc.pmu;
3599
3600	/* software PMUs will not have sched_task */
3601	if (WARN_ON_ONCE(!pmu->sched_task))
3602	return;
3603
3604	perf_ctx_lock(cpuctx, ctx: cpuctx->task_ctx);
3605	perf_pmu_disable(pmu);
3606
3607	pmu->sched_task(cpc->task_epc, sched_in);
3608
3609	perf_pmu_enable(pmu);
3610	perf_ctx_unlock(cpuctx, ctx: cpuctx->task_ctx);
3611	}
3612
3613	static void perf_pmu_sched_task(struct task_struct *prev,
3614	struct task_struct *next,
3615	bool sched_in)
3616	{
3617	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3618	struct perf_cpu_pmu_context *cpc;
3619
3620	/* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */
3621	if (prev == next || cpuctx->task_ctx)
3622	return;
3623
3624	list_for_each_entry(cpc, this_cpu_ptr(&sched_cb_list), sched_cb_entry)
3625	__perf_pmu_sched_task(cpc, sched_in);
3626	}
3627
3628	static void perf_event_switch(struct task_struct *task,
3629	struct task_struct *next_prev, bool sched_in);
3630
3631	/*
3632	* Called from scheduler to remove the events of the current task,
3633	* with interrupts disabled.
3634	*
3635	* We stop each event and update the event value in event->count.
3636	*
3637	* This does not protect us against NMI, but disable()
3638	* sets the disabled bit in the control field of event _before_
3639	* accessing the event control register. If a NMI hits, then it will
3640	* not restart the event.
3641	*/
3642	void __perf_event_task_sched_out(struct task_struct *task,
3643	struct task_struct *next)
3644	{
3645	if (__this_cpu_read(perf_sched_cb_usages))
3646	perf_pmu_sched_task(prev: task, next, sched_in: false);
3647
3648	if (atomic_read(v: &nr_switch_events))
3649	perf_event_switch(task, next_prev: next, sched_in: false);
3650
3651	perf_event_context_sched_out(task, next);
3652
3653	/*
3654	* if cgroup events exist on this CPU, then we need
3655	* to check if we have to switch out PMU state.
3656	* cgroup event are system-wide mode only
3657	*/
3658	perf_cgroup_switch(task: next);
3659	}
3660
3661	static bool perf_less_group_idx(const void *l, const void *r)
3662	{
3663	const struct perf_event *le = *(const struct perf_event **)l;
3664	const struct perf_event *re = *(const struct perf_event **)r;
3665
3666	return le->group_index < re->group_index;
3667	}
3668
3669	static void swap_ptr(void *l, void *r)
3670	{
3671	void **lp = l, **rp = r;
3672
3673	swap(*lp, *rp);
3674	}
3675
3676	static const struct min_heap_callbacks perf_min_heap = {
3677	.elem_size = sizeof(struct perf_event *),
3678	.less = perf_less_group_idx,
3679	.swp = swap_ptr,
3680	};
3681
3682	static void __heap_add(struct min_heap *heap, struct perf_event *event)
3683	{
3684	struct perf_event **itrs = heap->data;
3685
3686	if (event) {
3687	itrs[heap->nr] = event;
3688	heap->nr++;
3689	}
3690	}
3691
3692	static void __link_epc(struct perf_event_pmu_context *pmu_ctx)
3693	{
3694	struct perf_cpu_pmu_context *cpc;
3695
3696	if (!pmu_ctx->ctx->task)
3697	return;
3698
3699	cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);
3700	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
3701	cpc->task_epc = pmu_ctx;
3702	}
3703
3704	static noinline int visit_groups_merge(struct perf_event_context *ctx,
3705	struct perf_event_groups *groups, int cpu,
3706	struct pmu *pmu,
3707	int (*func)(struct perf_event *, void *),
3708	void *data)
3709	{
3710	#ifdef CONFIG_CGROUP_PERF
3711	struct cgroup_subsys_state *css = NULL;
3712	#endif
3713	struct perf_cpu_context *cpuctx = NULL;
3714	/* Space for per CPU and/or any CPU event iterators. */
3715	struct perf_event *itrs[2];
3716	struct min_heap event_heap;
3717	struct perf_event **evt;
3718	int ret;
3719
3720	if (pmu->filter && pmu->filter(pmu, cpu))
3721	return 0;
3722
3723	if (!ctx->task) {
3724	cpuctx = this_cpu_ptr(&perf_cpu_context);
3725	event_heap = (struct min_heap){
3726	.data = cpuctx->heap,
3727	.nr = 0,
3728	.size = cpuctx->heap_size,
3729	};
3730
3731	lockdep_assert_held(&cpuctx->ctx.lock);
3732
3733	#ifdef CONFIG_CGROUP_PERF
3734	if (cpuctx->cgrp)
3735	css = &cpuctx->cgrp->css;
3736	#endif
3737	} else {
3738	event_heap = (struct min_heap){
3739	.data = itrs,
3740	.nr = 0,
3741	.size = ARRAY_SIZE(itrs),
3742	};
3743	/* Events not within a CPU context may be on any CPU. */
3744	__heap_add(heap: &event_heap, event: perf_event_groups_first(groups, cpu: -1, pmu, NULL));
3745	}
3746	evt = event_heap.data;
3747
3748	__heap_add(heap: &event_heap, event: perf_event_groups_first(groups, cpu, pmu, NULL));
3749
3750	#ifdef CONFIG_CGROUP_PERF
3751	for (; css; css = css->parent)
3752	__heap_add(heap: &event_heap, event: perf_event_groups_first(groups, cpu, pmu, cgrp: css->cgroup));
3753	#endif
3754
3755	if (event_heap.nr) {
3756	__link_epc(pmu_ctx: (*evt)->pmu_ctx);
3757	perf_assert_pmu_disabled(pmu: (*evt)->pmu_ctx->pmu);
3758	}
3759
3760	min_heapify_all(heap: &event_heap, func: &perf_min_heap);
3761
3762	while (event_heap.nr) {
3763	ret = func(*evt, data);
3764	if (ret)
3765	return ret;
3766
3767	*evt = perf_event_groups_next(event: *evt, pmu);
3768	if (*evt)
3769	min_heapify(heap: &event_heap, pos: 0, func: &perf_min_heap);
3770	else
3771	min_heap_pop(heap: &event_heap, func: &perf_min_heap);
3772	}
3773
3774	return 0;
3775	}
3776
3777	/*
3778	* Because the userpage is strictly per-event (there is no concept of context,
3779	* so there cannot be a context indirection), every userpage must be updated
3780	* when context time starts :-(
3781	*
3782	* IOW, we must not miss EVENT_TIME edges.
3783	*/
3784	static inline bool event_update_userpage(struct perf_event *event)
3785	{
3786	if (likely(!atomic_read(&event->mmap_count)))
3787	return false;
3788
3789	perf_event_update_time(event);
3790	perf_event_update_userpage(event);
3791
3792	return true;
3793	}
3794
3795	static inline void group_update_userpage(struct perf_event *group_event)
3796	{
3797	struct perf_event *event;
3798
3799	if (!event_update_userpage(event: group_event))
3800	return;
3801
3802	for_each_sibling_event(event, group_event)
3803	event_update_userpage(event);
3804	}
3805
3806	static int merge_sched_in(struct perf_event *event, void *data)
3807	{
3808	struct perf_event_context *ctx = event->ctx;
3809	int *can_add_hw = data;
3810
3811	if (event->state <= PERF_EVENT_STATE_OFF)
3812	return 0;
3813
3814	if (!event_filter_match(event))
3815	return 0;
3816
3817	if (group_can_go_on(event, can_add_hw: *can_add_hw)) {
3818	if (!group_sched_in(group_event: event, ctx))
3819	list_add_tail(new: &event->active_list, head: get_event_list(event));
3820	}
3821
3822	if (event->state == PERF_EVENT_STATE_INACTIVE) {
3823	*can_add_hw = 0;
3824	if (event->attr.pinned) {
3825	perf_cgroup_event_disable(event, ctx);
3826	perf_event_set_state(event, state: PERF_EVENT_STATE_ERROR);
3827	} else {
3828	struct perf_cpu_pmu_context *cpc;
3829
3830	event->pmu_ctx->rotate_necessary = 1;
3831	cpc = this_cpu_ptr(event->pmu_ctx->pmu->cpu_pmu_context);
3832	perf_mux_hrtimer_restart(cpc);
3833	group_update_userpage(group_event: event);
3834	}
3835	}
3836
3837	return 0;
3838	}
3839
3840	static void pmu_groups_sched_in(struct perf_event_context *ctx,
3841	struct perf_event_groups *groups,
3842	struct pmu *pmu)
3843	{
3844	int can_add_hw = 1;
3845	visit_groups_merge(ctx, groups, smp_processor_id(), pmu,
3846	func: merge_sched_in, data: &can_add_hw);
3847	}
3848
3849	static void ctx_groups_sched_in(struct perf_event_context *ctx,
3850	struct perf_event_groups *groups,
3851	bool cgroup)
3852	{
3853	struct perf_event_pmu_context *pmu_ctx;
3854
3855	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
3856	if (cgroup && !pmu_ctx->nr_cgroups)
3857	continue;
3858	pmu_groups_sched_in(ctx, groups, pmu: pmu_ctx->pmu);
3859	}
3860	}
3861
3862	static void __pmu_ctx_sched_in(struct perf_event_context *ctx,
3863	struct pmu *pmu)
3864	{
3865	pmu_groups_sched_in(ctx, groups: &ctx->flexible_groups, pmu);
3866	}
3867
3868	static void
3869	ctx_sched_in(struct perf_event_context *ctx, enum event_type_t event_type)
3870	{
3871	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3872	int is_active = ctx->is_active;
3873	bool cgroup = event_type & EVENT_CGROUP;
3874
3875	event_type &= ~EVENT_CGROUP;
3876
3877	lockdep_assert_held(&ctx->lock);
3878
3879	if (likely(!ctx->nr_events))
3880	return;
3881
3882	if (!(is_active & EVENT_TIME)) {
3883	/* start ctx time */
3884	__update_context_time(ctx, adv: false);
3885	perf_cgroup_set_timestamp(cpuctx);
3886	/*
3887	* CPU-release for the below ->is_active store,
3888	* see __load_acquire() in perf_event_time_now()
3889	*/
3890	barrier();
3891	}
3892
3893	ctx->is_active |= (event_type | EVENT_TIME);
3894	if (ctx->task) {
3895	if (!is_active)
3896	cpuctx->task_ctx = ctx;
3897	else
3898	WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3899	}
3900
3901	is_active ^= ctx->is_active; /* changed bits */
3902
3903	/*
3904	* First go through the list and put on any pinned groups
3905	* in order to give them the best chance of going on.
3906	*/
3907	if (is_active & EVENT_PINNED)
3908	ctx_groups_sched_in(ctx, groups: &ctx->pinned_groups, cgroup);
3909
3910	/* Then walk through the lower prio flexible groups */
3911	if (is_active & EVENT_FLEXIBLE)
3912	ctx_groups_sched_in(ctx, groups: &ctx->flexible_groups, cgroup);
3913	}
3914
3915	static void perf_event_context_sched_in(struct task_struct *task)
3916	{
3917	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3918	struct perf_event_context *ctx;
3919
3920	rcu_read_lock();
3921	ctx = rcu_dereference(task->perf_event_ctxp);
3922	if (!ctx)
3923	goto rcu_unlock;
3924
3925	if (cpuctx->task_ctx == ctx) {
3926	perf_ctx_lock(cpuctx, ctx);
3927	perf_ctx_disable(ctx, cgroup: false);
3928
3929	perf_ctx_sched_task_cb(ctx, sched_in: true);
3930
3931	perf_ctx_enable(ctx, cgroup: false);
3932	perf_ctx_unlock(cpuctx, ctx);
3933	goto rcu_unlock;
3934	}
3935
3936	perf_ctx_lock(cpuctx, ctx);
3937	/*
3938	* We must check ctx->nr_events while holding ctx->lock, such
3939	* that we serialize against perf_install_in_context().
3940	*/
3941	if (!ctx->nr_events)
3942	goto unlock;
3943
3944	perf_ctx_disable(ctx, cgroup: false);
3945	/*
3946	* We want to keep the following priority order:
3947	* cpu pinned (that don't need to move), task pinned,
3948	* cpu flexible, task flexible.
3949	*
3950	* However, if task's ctx is not carrying any pinned
3951	* events, no need to flip the cpuctx's events around.
3952	*/
3953	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) {
3954	perf_ctx_disable(ctx: &cpuctx->ctx, cgroup: false);
3955	ctx_sched_out(ctx: &cpuctx->ctx, event_type: EVENT_FLEXIBLE);
3956	}
3957
3958	perf_event_sched_in(cpuctx, ctx);
3959
3960	perf_ctx_sched_task_cb(ctx: cpuctx->task_ctx, sched_in: true);
3961
3962	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
3963	perf_ctx_enable(ctx: &cpuctx->ctx, cgroup: false);
3964
3965	perf_ctx_enable(ctx, cgroup: false);
3966
3967	unlock:
3968	perf_ctx_unlock(cpuctx, ctx);
3969	rcu_unlock:
3970	rcu_read_unlock();
3971	}
3972
3973	/*
3974	* Called from scheduler to add the events of the current task
3975	* with interrupts disabled.
3976	*
3977	* We restore the event value and then enable it.
3978	*
3979	* This does not protect us against NMI, but enable()
3980	* sets the enabled bit in the control field of event _before_
3981	* accessing the event control register. If a NMI hits, then it will
3982	* keep the event running.
3983	*/
3984	void __perf_event_task_sched_in(struct task_struct *prev,
3985	struct task_struct *task)
3986	{
3987	perf_event_context_sched_in(task);
3988
3989	if (atomic_read(v: &nr_switch_events))
3990	perf_event_switch(task, next_prev: prev, sched_in: true);
3991
3992	if (__this_cpu_read(perf_sched_cb_usages))
3993	perf_pmu_sched_task(prev, next: task, sched_in: true);
3994	}
3995
3996	static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
3997	{
3998	u64 frequency = event->attr.sample_freq;
3999	u64 sec = NSEC_PER_SEC;
4000	u64 divisor, dividend;
4001
4002	int count_fls, nsec_fls, frequency_fls, sec_fls;
4003
4004	count_fls = fls64(x: count);
4005	nsec_fls = fls64(x: nsec);
4006	frequency_fls = fls64(x: frequency);
4007	sec_fls = 30;
4008
4009	/*
4010	* We got @count in @nsec, with a target of sample_freq HZ
4011	* the target period becomes:
4012	*
4013	* @count * 10^9
4014	* period = -------------------
4015	* @nsec * sample_freq
4016	*
4017	*/
4018
4019	/*
4020	* Reduce accuracy by one bit such that @a and @b converge
4021	* to a similar magnitude.
4022	*/
4023	#define REDUCE_FLS(a, b) \
4024	do { \
4025	if (a##_fls > b##_fls) { \
4026	a >>= 1; \
4027	a##_fls--; \
4028	} else { \
4029	b >>= 1; \
4030	b##_fls--; \
4031	} \
4032	} while (0)
4033
4034	/*
4035	* Reduce accuracy until either term fits in a u64, then proceed with
4036	* the other, so that finally we can do a u64/u64 division.
4037	*/
4038	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
4039	REDUCE_FLS(nsec, frequency);
4040	REDUCE_FLS(sec, count);
4041	}
4042
4043	if (count_fls + sec_fls > 64) {
4044	divisor = nsec * frequency;
4045
4046	while (count_fls + sec_fls > 64) {
4047	REDUCE_FLS(count, sec);
4048	divisor >>= 1;
4049	}
4050
4051	dividend = count * sec;
4052	} else {
4053	dividend = count * sec;
4054
4055	while (nsec_fls + frequency_fls > 64) {
4056	REDUCE_FLS(nsec, frequency);
4057	dividend >>= 1;
4058	}
4059
4060	divisor = nsec * frequency;
4061	}
4062
4063	if (!divisor)
4064	return dividend;
4065
4066	return div64_u64(dividend, divisor);
4067	}
4068
4069	static DEFINE_PER_CPU(int, perf_throttled_count);
4070	static DEFINE_PER_CPU(u64, perf_throttled_seq);
4071
4072	static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
4073	{
4074	struct hw_perf_event *hwc = &event->hw;
4075	s64 period, sample_period;
4076	s64 delta;
4077
4078	period = perf_calculate_period(event, nsec, count);
4079
4080	delta = (s64)(period - hwc->sample_period);
4081	delta = (delta + 7) / 8; /* low pass filter */
4082
4083	sample_period = hwc->sample_period + delta;
4084
4085	if (!sample_period)
4086	sample_period = 1;
4087
4088	hwc->sample_period = sample_period;
4089
4090	if (local64_read(&hwc->period_left) > 8*sample_period) {
4091	if (disable)
4092	event->pmu->stop(event, PERF_EF_UPDATE);
4093
4094	local64_set(&hwc->period_left, 0);
4095
4096	if (disable)
4097	event->pmu->start(event, PERF_EF_RELOAD);
4098	}
4099	}
4100
4101	/*
4102	* combine freq adjustment with unthrottling to avoid two passes over the
4103	* events. At the same time, make sure, having freq events does not change
4104	* the rate of unthrottling as that would introduce bias.
4105	*/
4106	static void
4107	perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
4108	{
4109	struct perf_event *event;
4110	struct hw_perf_event *hwc;
4111	u64 now, period = TICK_NSEC;
4112	s64 delta;
4113
4114	/*
4115	* only need to iterate over all events iff:
4116	* - context have events in frequency mode (needs freq adjust)
4117	* - there are events to unthrottle on this cpu
4118	*/
4119	if (!(ctx->nr_freq || unthrottle))
4120	return;
4121
4122	raw_spin_lock(&ctx->lock);
4123
4124	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4125	if (event->state != PERF_EVENT_STATE_ACTIVE)
4126	continue;
4127
4128	// XXX use visit thingy to avoid the -1,cpu match
4129	if (!event_filter_match(event))
4130	continue;
4131
4132	perf_pmu_disable(pmu: event->pmu);
4133
4134	hwc = &event->hw;
4135
4136	if (hwc->interrupts == MAX_INTERRUPTS) {
4137	hwc->interrupts = 0;
4138	perf_log_throttle(event, enable: 1);
4139	event->pmu->start(event, 0);
4140	}
4141
4142	if (!event->attr.freq || !event->attr.sample_freq)
4143	goto next;
4144
4145	/*
4146	* stop the event and update event->count
4147	*/
4148	event->pmu->stop(event, PERF_EF_UPDATE);
4149
4150	now = local64_read(&event->count);
4151	delta = now - hwc->freq_count_stamp;
4152	hwc->freq_count_stamp = now;
4153
4154	/*
4155	* restart the event
4156	* reload only if value has changed
4157	* we have stopped the event so tell that
4158	* to perf_adjust_period() to avoid stopping it
4159	* twice.
4160	*/
4161	if (delta > 0)
4162	perf_adjust_period(event, nsec: period, count: delta, disable: false);
4163
4164	event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
4165	next:
4166	perf_pmu_enable(pmu: event->pmu);
4167	}
4168
4169	raw_spin_unlock(&ctx->lock);
4170	}
4171
4172	/*
4173	* Move @event to the tail of the @ctx's elegible events.
4174	*/
4175	static void rotate_ctx(struct perf_event_context *ctx, struct perf_event *event)
4176	{
4177	/*
4178	* Rotate the first entry last of non-pinned groups. Rotation might be
4179	* disabled by the inheritance code.
4180	*/
4181	if (ctx->rotate_disable)
4182	return;
4183
4184	perf_event_groups_delete(groups: &ctx->flexible_groups, event);
4185	perf_event_groups_insert(groups: &ctx->flexible_groups, event);
4186	}
4187
4188	/* pick an event from the flexible_groups to rotate */
4189	static inline struct perf_event *
4190	ctx_event_to_rotate(struct perf_event_pmu_context *pmu_ctx)
4191	{
4192	struct perf_event *event;
4193	struct rb_node *node;
4194	struct rb_root *tree;
4195	struct __group_key key = {
4196	.pmu = pmu_ctx->pmu,
4197	};
4198
4199	/* pick the first active flexible event */
4200	event = list_first_entry_or_null(&pmu_ctx->flexible_active,
4201	struct perf_event, active_list);
4202	if (event)
4203	goto out;
4204
4205	/* if no active flexible event, pick the first event */
4206	tree = &pmu_ctx->ctx->flexible_groups.tree;
4207
4208	if (!pmu_ctx->ctx->task) {
4209	key.cpu = smp_processor_id();
4210
4211	node = rb_find_first(key: &key, tree, cmp: __group_cmp_ignore_cgroup);
4212	if (node)
4213	event = __node_2_pe(node);
4214	goto out;
4215	}
4216
4217	key.cpu = -1;
4218	node = rb_find_first(key: &key, tree, cmp: __group_cmp_ignore_cgroup);
4219	if (node) {
4220	event = __node_2_pe(node);
4221	goto out;
4222	}
4223
4224	key.cpu = smp_processor_id();
4225	node = rb_find_first(key: &key, tree, cmp: __group_cmp_ignore_cgroup);
4226	if (node)
4227	event = __node_2_pe(node);
4228
4229	out:
4230	/*
4231	* Unconditionally clear rotate_necessary; if ctx_flexible_sched_in()
4232	* finds there are unschedulable events, it will set it again.
4233	*/
4234	pmu_ctx->rotate_necessary = 0;
4235
4236	return event;
4237	}
4238
4239	static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc)
4240	{
4241	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4242	struct perf_event_pmu_context *cpu_epc, *task_epc = NULL;
4243	struct perf_event *cpu_event = NULL, *task_event = NULL;
4244	int cpu_rotate, task_rotate;
4245	struct pmu *pmu;
4246
4247	/*
4248	* Since we run this from IRQ context, nobody can install new
4249	* events, thus the event count values are stable.
4250	*/
4251
4252	cpu_epc = &cpc->epc;
4253	pmu = cpu_epc->pmu;
4254	task_epc = cpc->task_epc;
4255
4256	cpu_rotate = cpu_epc->rotate_necessary;
4257	task_rotate = task_epc ? task_epc->rotate_necessary : 0;
4258
4259	if (!(cpu_rotate || task_rotate))
4260	return false;
4261
4262	perf_ctx_lock(cpuctx, ctx: cpuctx->task_ctx);
4263	perf_pmu_disable(pmu);
4264
4265	if (task_rotate)
4266	task_event = ctx_event_to_rotate(pmu_ctx: task_epc);
4267	if (cpu_rotate)
4268	cpu_event = ctx_event_to_rotate(pmu_ctx: cpu_epc);
4269
4270	/*
4271	* As per the order given at ctx_resched() first 'pop' task flexible
4272	* and then, if needed CPU flexible.
4273	*/
4274	if (task_event || (task_epc && cpu_event)) {
4275	update_context_time(ctx: task_epc->ctx);
4276	__pmu_ctx_sched_out(pmu_ctx: task_epc, event_type: EVENT_FLEXIBLE);
4277	}
4278
4279	if (cpu_event) {
4280	update_context_time(ctx: &cpuctx->ctx);
4281	__pmu_ctx_sched_out(pmu_ctx: cpu_epc, event_type: EVENT_FLEXIBLE);
4282	rotate_ctx(ctx: &cpuctx->ctx, event: cpu_event);
4283	__pmu_ctx_sched_in(ctx: &cpuctx->ctx, pmu);
4284	}
4285
4286	if (task_event)
4287	rotate_ctx(ctx: task_epc->ctx, event: task_event);
4288
4289	if (task_event || (task_epc && cpu_event))
4290	__pmu_ctx_sched_in(ctx: task_epc->ctx, pmu);
4291
4292	perf_pmu_enable(pmu);
4293	perf_ctx_unlock(cpuctx, ctx: cpuctx->task_ctx);
4294
4295	return true;
4296	}
4297
4298	void perf_event_task_tick(void)
4299	{
4300	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4301	struct perf_event_context *ctx;
4302	int throttled;
4303
4304	lockdep_assert_irqs_disabled();
4305
4306	__this_cpu_inc(perf_throttled_seq);
4307	throttled = __this_cpu_xchg(perf_throttled_count, 0);
4308	tick_dep_clear_cpu(smp_processor_id(), bit: TICK_DEP_BIT_PERF_EVENTS);
4309
4310	perf_adjust_freq_unthr_context(ctx: &cpuctx->ctx, unthrottle: !!throttled);
4311
4312	rcu_read_lock();
4313	ctx = rcu_dereference(current->perf_event_ctxp);
4314	if (ctx)
4315	perf_adjust_freq_unthr_context(ctx, unthrottle: !!throttled);
4316	rcu_read_unlock();
4317	}
4318
4319	static int event_enable_on_exec(struct perf_event *event,
4320	struct perf_event_context *ctx)
4321	{
4322	if (!event->attr.enable_on_exec)
4323	return 0;
4324
4325	event->attr.enable_on_exec = 0;
4326	if (event->state >= PERF_EVENT_STATE_INACTIVE)
4327	return 0;
4328
4329	perf_event_set_state(event, state: PERF_EVENT_STATE_INACTIVE);
4330
4331	return 1;
4332	}
4333
4334	/*
4335	* Enable all of a task's events that have been marked enable-on-exec.
4336	* This expects task == current.
4337	*/
4338	static void perf_event_enable_on_exec(struct perf_event_context *ctx)
4339	{
4340	struct perf_event_context *clone_ctx = NULL;
4341	enum event_type_t event_type = 0;
4342	struct perf_cpu_context *cpuctx;
4343	struct perf_event *event;
4344	unsigned long flags;
4345	int enabled = 0;
4346
4347	local_irq_save(flags);
4348	if (WARN_ON_ONCE(current->perf_event_ctxp != ctx))
4349	goto out;
4350
4351	if (!ctx->nr_events)
4352	goto out;
4353
4354	cpuctx = this_cpu_ptr(&perf_cpu_context);
4355	perf_ctx_lock(cpuctx, ctx);
4356	ctx_sched_out(ctx, event_type: EVENT_TIME);
4357
4358	list_for_each_entry(event, &ctx->event_list, event_entry) {
4359	enabled |= event_enable_on_exec(event, ctx);
4360	event_type |= get_event_type(event);
4361	}
4362
4363	/*
4364	* Unclone and reschedule this context if we enabled any event.
4365	*/
4366	if (enabled) {
4367	clone_ctx = unclone_ctx(ctx);
4368	ctx_resched(cpuctx, task_ctx: ctx, event_type);
4369	} else {
4370	ctx_sched_in(ctx, event_type: EVENT_TIME);
4371	}
4372	perf_ctx_unlock(cpuctx, ctx);
4373
4374	out:
4375	local_irq_restore(flags);
4376
4377	if (clone_ctx)
4378	put_ctx(ctx: clone_ctx);
4379	}
4380
4381	static void perf_remove_from_owner(struct perf_event *event);
4382	static void perf_event_exit_event(struct perf_event *event,
4383	struct perf_event_context *ctx);
4384
4385	/*
4386	* Removes all events from the current task that have been marked
4387	* remove-on-exec, and feeds their values back to parent events.
4388	*/
4389	static void perf_event_remove_on_exec(struct perf_event_context *ctx)
4390	{
4391	struct perf_event_context *clone_ctx = NULL;
4392	struct perf_event *event, *next;
4393	unsigned long flags;
4394	bool modified = false;
4395
4396	mutex_lock(&ctx->mutex);
4397
4398	if (WARN_ON_ONCE(ctx->task != current))
4399	goto unlock;
4400
4401	list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) {
4402	if (!event->attr.remove_on_exec)
4403	continue;
4404
4405	if (!is_kernel_event(event))
4406	perf_remove_from_owner(event);
4407
4408	modified = true;
4409
4410	perf_event_exit_event(event, ctx);
4411	}
4412
4413	raw_spin_lock_irqsave(&ctx->lock, flags);
4414	if (modified)
4415	clone_ctx = unclone_ctx(ctx);
4416	raw_spin_unlock_irqrestore(&ctx->lock, flags);
4417
4418	unlock:
4419	mutex_unlock(lock: &ctx->mutex);
4420
4421	if (clone_ctx)
4422	put_ctx(ctx: clone_ctx);
4423	}
4424
4425	struct perf_read_data {
4426	struct perf_event *event;
4427	bool group;
4428	int ret;
4429	};
4430
4431	static int __perf_event_read_cpu(struct perf_event *event, int event_cpu)
4432	{
4433	u16 local_pkg, event_pkg;
4434
4435	if ((unsigned)event_cpu >= nr_cpu_ids)
4436	return event_cpu;
4437
4438	if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {
4439	int local_cpu = smp_processor_id();
4440
4441	event_pkg = topology_physical_package_id(event_cpu);
4442	local_pkg = topology_physical_package_id(local_cpu);
4443
4444	if (event_pkg == local_pkg)
4445	return local_cpu;
4446	}
4447
4448	return event_cpu;
4449	}
4450
4451	/*
4452	* Cross CPU call to read the hardware event
4453	*/
4454	static void __perf_event_read(void *info)
4455	{
4456	struct perf_read_data *data = info;
4457	struct perf_event *sub, *event = data->event;
4458	struct perf_event_context *ctx = event->ctx;
4459	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4460	struct pmu *pmu = event->pmu;
4461
4462	/*
4463	* If this is a task context, we need to check whether it is
4464	* the current task context of this cpu. If not it has been
4465	* scheduled out before the smp call arrived. In that case
4466	* event->count would have been updated to a recent sample
4467	* when the event was scheduled out.
4468	*/
4469	if (ctx->task && cpuctx->task_ctx != ctx)
4470	return;
4471
4472	raw_spin_lock(&ctx->lock);
4473	if (ctx->is_active & EVENT_TIME) {
4474	update_context_time(ctx);
4475	update_cgrp_time_from_event(event);
4476	}
4477
4478	perf_event_update_time(event);
4479	if (data->group)
4480	perf_event_update_sibling_time(leader: event);
4481
4482	if (event->state != PERF_EVENT_STATE_ACTIVE)
4483	goto unlock;
4484
4485	if (!data->group) {
4486	pmu->read(event);
4487	data->ret = 0;
4488	goto unlock;
4489	}
4490
4491	pmu->start_txn(pmu, PERF_PMU_TXN_READ);
4492
4493	pmu->read(event);
4494
4495	for_each_sibling_event(sub, event) {
4496	if (sub->state == PERF_EVENT_STATE_ACTIVE) {
4497	/*
4498	* Use sibling's PMU rather than @event's since
4499	* sibling could be on different (eg: software) PMU.
4500	*/
4501	sub->pmu->read(sub);
4502	}
4503	}
4504
4505	data->ret = pmu->commit_txn(pmu);
4506
4507	unlock:
4508	raw_spin_unlock(&ctx->lock);
4509	}
4510
4511	static inline u64 perf_event_count(struct perf_event *event)
4512	{
4513	return local64_read(&event->count) + atomic64_read(v: &event->child_count);
4514	}
4515
4516	static void calc_timer_values(struct perf_event *event,
4517	u64 *now,
4518	u64 *enabled,
4519	u64 *running)
4520	{
4521	u64 ctx_time;
4522
4523	*now = perf_clock();
4524	ctx_time = perf_event_time_now(event, now: *now);
4525	__perf_update_times(event, now: ctx_time, enabled, running);
4526	}
4527
4528	/*
4529	* NMI-safe method to read a local event, that is an event that
4530	* is:
4531	* - either for the current task, or for this CPU
4532	* - does not have inherit set, for inherited task events
4533	* will not be local and we cannot read them atomically
4534	* - must not have a pmu::count method
4535	*/
4536	int perf_event_read_local(struct perf_event *event, u64 *value,
4537	u64 *enabled, u64 *running)
4538	{
4539	unsigned long flags;
4540	int event_oncpu;
4541	int event_cpu;
4542	int ret = 0;
4543
4544	/*
4545	* Disabling interrupts avoids all counter scheduling (context
4546	* switches, timer based rotation and IPIs).
4547	*/
4548	local_irq_save(flags);
4549
4550	/*
4551	* It must not be an event with inherit set, we cannot read
4552	* all child counters from atomic context.
4553	*/
4554	if (event->attr.inherit) {
4555	ret = -EOPNOTSUPP;
4556	goto out;
4557	}
4558
4559	/* If this is a per-task event, it must be for current */
4560	if ((event->attach_state & PERF_ATTACH_TASK) &&
4561	event->hw.target != current) {
4562	ret = -EINVAL;
4563	goto out;
4564	}
4565
4566	/*
4567	* Get the event CPU numbers, and adjust them to local if the event is
4568	* a per-package event that can be read locally
4569	*/
4570	event_oncpu = __perf_event_read_cpu(event, event_cpu: event->oncpu);
4571	event_cpu = __perf_event_read_cpu(event, event_cpu: event->cpu);
4572
4573	/* If this is a per-CPU event, it must be for this CPU */
4574	if (!(event->attach_state & PERF_ATTACH_TASK) &&
4575	event_cpu != smp_processor_id()) {
4576	ret = -EINVAL;
4577	goto out;
4578	}
4579
4580	/* If this is a pinned event it must be running on this CPU */
4581	if (event->attr.pinned && event_oncpu != smp_processor_id()) {
4582	ret = -EBUSY;
4583	goto out;
4584	}
4585
4586	/*
4587	* If the event is currently on this CPU, its either a per-task event,
4588	* or local to this CPU. Furthermore it means its ACTIVE (otherwise
4589	* oncpu == -1).
4590	*/
4591	if (event_oncpu == smp_processor_id())
4592	event->pmu->read(event);
4593
4594	*value = local64_read(&event->count);
4595	if (enabled || running) {
4596	u64 __enabled, __running, __now;
4597
4598	calc_timer_values(event, now: &__now, enabled: &__enabled, running: &__running);
4599	if (enabled)
4600	*enabled = __enabled;
4601	if (running)
4602	*running = __running;
4603	}
4604	out:
4605	local_irq_restore(flags);
4606
4607	return ret;
4608	}
4609
4610	static int perf_event_read(struct perf_event *event, bool group)
4611	{
4612	enum perf_event_state state = READ_ONCE(event->state);
4613	int event_cpu, ret = 0;
4614
4615	/*
4616	* If event is enabled and currently active on a CPU, update the
4617	* value in the event structure:
4618	*/
4619	again:
4620	if (state == PERF_EVENT_STATE_ACTIVE) {
4621	struct perf_read_data data;
4622
4623	/*
4624	* Orders the ->state and ->oncpu loads such that if we see
4625	* ACTIVE we must also see the right ->oncpu.
4626	*
4627	* Matches the smp_wmb() from event_sched_in().
4628	*/
4629	smp_rmb();
4630
4631	event_cpu = READ_ONCE(event->oncpu);
4632	if ((unsigned)event_cpu >= nr_cpu_ids)
4633	return 0;
4634
4635	data = (struct perf_read_data){
4636	.event = event,
4637	.group = group,
4638	.ret = 0,
4639	};
4640
4641	preempt_disable();
4642	event_cpu = __perf_event_read_cpu(event, event_cpu);
4643
4644	/*
4645	* Purposely ignore the smp_call_function_single() return
4646	* value.
4647	*
4648	* If event_cpu isn't a valid CPU it means the event got
4649	* scheduled out and that will have updated the event count.
4650	*
4651	* Therefore, either way, we'll have an up-to-date event count
4652	* after this.
4653	*/
4654	(void)smp_call_function_single(cpuid: event_cpu, func: __perf_event_read, info: &data, wait: 1);
4655	preempt_enable();
4656	ret = data.ret;
4657
4658	} else if (state == PERF_EVENT_STATE_INACTIVE) {
4659	struct perf_event_context *ctx = event->ctx;
4660	unsigned long flags;
4661
4662	raw_spin_lock_irqsave(&ctx->lock, flags);
4663	state = event->state;
4664	if (state != PERF_EVENT_STATE_INACTIVE) {
4665	raw_spin_unlock_irqrestore(&ctx->lock, flags);
4666	goto again;
4667	}
4668
4669	/*
4670	* May read while context is not active (e.g., thread is
4671	* blocked), in that case we cannot update context time
4672	*/
4673	if (ctx->is_active & EVENT_TIME) {
4674	update_context_time(ctx);
4675	update_cgrp_time_from_event(event);
4676	}
4677
4678	perf_event_update_time(event);
4679	if (group)
4680	perf_event_update_sibling_time(leader: event);
4681	raw_spin_unlock_irqrestore(&ctx->lock, flags);
4682	}
4683
4684	return ret;
4685	}
4686
4687	/*
4688	* Initialize the perf_event context in a task_struct:
4689	*/
4690	static void __perf_event_init_context(struct perf_event_context *ctx)
4691	{
4692	raw_spin_lock_init(&ctx->lock);
4693	mutex_init(&ctx->mutex);
4694	INIT_LIST_HEAD(list: &ctx->pmu_ctx_list);
4695	perf_event_groups_init(groups: &ctx->pinned_groups);
4696	perf_event_groups_init(groups: &ctx->flexible_groups);
4697	INIT_LIST_HEAD(list: &ctx->event_list);
4698	refcount_set(r: &ctx->refcount, n: 1);
4699	}
4700
4701	static void
4702	__perf_init_event_pmu_context(struct perf_event_pmu_context *epc, struct pmu *pmu)
4703	{
4704	epc->pmu = pmu;
4705	INIT_LIST_HEAD(list: &epc->pmu_ctx_entry);
4706	INIT_LIST_HEAD(list: &epc->pinned_active);
4707	INIT_LIST_HEAD(list: &epc->flexible_active);
4708	atomic_set(v: &epc->refcount, i: 1);
4709	}
4710
4711	static struct perf_event_context *
4712	alloc_perf_context(struct task_struct *task)
4713	{
4714	struct perf_event_context *ctx;
4715
4716	ctx = kzalloc(size: sizeof(struct perf_event_context), GFP_KERNEL);
4717	if (!ctx)
4718	return NULL;
4719
4720	__perf_event_init_context(ctx);
4721	if (task)
4722	ctx->task = get_task_struct(t: task);
4723
4724	return ctx;
4725	}
4726
4727	static struct task_struct *
4728	find_lively_task_by_vpid(pid_t vpid)
4729	{
4730	struct task_struct *task;
4731
4732	rcu_read_lock();
4733	if (!vpid)
4734	task = current;
4735	else
4736	task = find_task_by_vpid(nr: vpid);
4737	if (task)
4738	get_task_struct(t: task);
4739	rcu_read_unlock();
4740
4741	if (!task)
4742	return ERR_PTR(error: -ESRCH);
4743
4744	return task;
4745	}
4746
4747	/*
4748	* Returns a matching context with refcount and pincount.
4749	*/
4750	static struct perf_event_context *
4751	find_get_context(struct task_struct *task, struct perf_event *event)
4752	{
4753	struct perf_event_context *ctx, *clone_ctx = NULL;
4754	struct perf_cpu_context *cpuctx;
4755	unsigned long flags;
4756	int err;
4757
4758	if (!task) {
4759	/* Must be root to operate on a CPU event: */
4760	err = perf_allow_cpu(attr: &event->attr);
4761	if (err)
4762	return ERR_PTR(error: err);
4763
4764	cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);
4765	ctx = &cpuctx->ctx;
4766	get_ctx(ctx);
4767	raw_spin_lock_irqsave(&ctx->lock, flags);
4768	++ctx->pin_count;
4769	raw_spin_unlock_irqrestore(&ctx->lock, flags);
4770
4771	return ctx;
4772	}
4773
4774	err = -EINVAL;
4775	retry:
4776	ctx = perf_lock_task_context(task, flags: &flags);
4777	if (ctx) {
4778	clone_ctx = unclone_ctx(ctx);
4779	++ctx->pin_count;
4780
4781	raw_spin_unlock_irqrestore(&ctx->lock, flags);
4782
4783	if (clone_ctx)
4784	put_ctx(ctx: clone_ctx);
4785	} else {
4786	ctx = alloc_perf_context(task);
4787	err = -ENOMEM;
4788	if (!ctx)
4789	goto errout;
4790
4791	err = 0;
4792	mutex_lock(&task->perf_event_mutex);
4793	/*
4794	* If it has already passed perf_event_exit_task().
4795	* we must see PF_EXITING, it takes this mutex too.
4796	*/
4797	if (task->flags & PF_EXITING)
4798	err = -ESRCH;
4799	else if (task->perf_event_ctxp)
4800	err = -EAGAIN;
4801	else {
4802	get_ctx(ctx);
4803	++ctx->pin_count;
4804	rcu_assign_pointer(task->perf_event_ctxp, ctx);
4805	}
4806	mutex_unlock(lock: &task->perf_event_mutex);
4807
4808	if (unlikely(err)) {
4809	put_ctx(ctx);
4810
4811	if (err == -EAGAIN)
4812	goto retry;
4813	goto errout;
4814	}
4815	}
4816
4817	return ctx;
4818
4819	errout:
4820	return ERR_PTR(error: err);
4821	}
4822
4823	static struct perf_event_pmu_context *
4824	find_get_pmu_context(struct pmu *pmu, struct perf_event_context *ctx,
4825	struct perf_event *event)
4826	{
4827	struct perf_event_pmu_context *new = NULL, *epc;
4828	void *task_ctx_data = NULL;
4829
4830	if (!ctx->task) {
4831	struct perf_cpu_pmu_context *cpc;
4832
4833	cpc = per_cpu_ptr(pmu->cpu_pmu_context, event->cpu);
4834	epc = &cpc->epc;
4835	raw_spin_lock_irq(&ctx->lock);
4836	if (!epc->ctx) {
4837	atomic_set(v: &epc->refcount, i: 1);
4838	epc->embedded = 1;
4839	list_add(new: &epc->pmu_ctx_entry, head: &ctx->pmu_ctx_list);
4840	epc->ctx = ctx;
4841	} else {
4842	WARN_ON_ONCE(epc->ctx != ctx);
4843	atomic_inc(v: &epc->refcount);
4844	}
4845	raw_spin_unlock_irq(&ctx->lock);
4846	return epc;
4847	}
4848
4849	new = kzalloc(size: sizeof(*epc), GFP_KERNEL);
4850	if (!new)
4851	return ERR_PTR(error: -ENOMEM);
4852
4853	if (event->attach_state & PERF_ATTACH_TASK_DATA) {
4854	task_ctx_data = alloc_task_ctx_data(pmu);
4855	if (!task_ctx_data) {
4856	kfree(objp: new);
4857	return ERR_PTR(error: -ENOMEM);
4858	}
4859	}
4860
4861	__perf_init_event_pmu_context(epc: new, pmu);
4862
4863	/*
4864	* XXX
4865	*
4866	* lockdep_assert_held(&ctx->mutex);
4867	*
4868	* can't because perf_event_init_task() doesn't actually hold the
4869	* child_ctx->mutex.
4870	*/
4871
4872	raw_spin_lock_irq(&ctx->lock);
4873	list_for_each_entry(epc, &ctx->pmu_ctx_list, pmu_ctx_entry) {
4874	if (epc->pmu == pmu) {
4875	WARN_ON_ONCE(epc->ctx != ctx);
4876	atomic_inc(v: &epc->refcount);
4877	goto found_epc;
4878	}
4879	}
4880
4881	epc = new;
4882	new = NULL;
4883
4884	list_add(new: &epc->pmu_ctx_entry, head: &ctx->pmu_ctx_list);
4885	epc->ctx = ctx;
4886
4887	found_epc:
4888	if (task_ctx_data && !epc->task_ctx_data) {
4889	epc->task_ctx_data = task_ctx_data;
4890	task_ctx_data = NULL;
4891	ctx->nr_task_data++;
4892	}
4893	raw_spin_unlock_irq(&ctx->lock);
4894
4895	free_task_ctx_data(pmu, task_ctx_data);
4896	kfree(objp: new);
4897
4898	return epc;
4899	}
4900
4901	static void get_pmu_ctx(struct perf_event_pmu_context *epc)
4902	{
4903	WARN_ON_ONCE(!atomic_inc_not_zero(&epc->refcount));
4904	}
4905
4906	static void free_epc_rcu(struct rcu_head *head)
4907	{
4908	struct perf_event_pmu_context *epc = container_of(head, typeof(*epc), rcu_head);
4909
4910	kfree(objp: epc->task_ctx_data);
4911	kfree(objp: epc);
4912	}
4913
4914	static void put_pmu_ctx(struct perf_event_pmu_context *epc)
4915	{
4916	struct perf_event_context *ctx = epc->ctx;
4917	unsigned long flags;
4918
4919	/*
4920	* XXX
4921	*
4922	* lockdep_assert_held(&ctx->mutex);
4923	*
4924	* can't because of the call-site in _free_event()/put_event()
4925	* which isn't always called under ctx->mutex.
4926	*/
4927	if (!atomic_dec_and_raw_lock_irqsave(&epc->refcount, &ctx->lock, flags))
4928	return;
4929
4930	WARN_ON_ONCE(list_empty(&epc->pmu_ctx_entry));
4931
4932	list_del_init(entry: &epc->pmu_ctx_entry);
4933	epc->ctx = NULL;
4934
4935	WARN_ON_ONCE(!list_empty(&epc->pinned_active));
4936	WARN_ON_ONCE(!list_empty(&epc->flexible_active));
4937
4938	raw_spin_unlock_irqrestore(&ctx->lock, flags);
4939
4940	if (epc->embedded)
4941	return;
4942
4943	call_rcu(head: &epc->rcu_head, func: free_epc_rcu);
4944	}
4945
4946	static void perf_event_free_filter(struct perf_event *event);
4947
4948	static void free_event_rcu(struct rcu_head *head)
4949	{
4950	struct perf_event *event = container_of(head, typeof(*event), rcu_head);
4951
4952	if (event->ns)
4953	put_pid_ns(ns: event->ns);
4954	perf_event_free_filter(event);
4955	kmem_cache_free(s: perf_event_cache, objp: event);
4956	}
4957
4958	static void ring_buffer_attach(struct perf_event *event,
4959	struct perf_buffer *rb);
4960
4961	static void detach_sb_event(struct perf_event *event)
4962	{
4963	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
4964
4965	raw_spin_lock(&pel->lock);
4966	list_del_rcu(entry: &event->sb_list);
4967	raw_spin_unlock(&pel->lock);
4968	}
4969
4970	static bool is_sb_event(struct perf_event *event)
4971	{
4972	struct perf_event_attr *attr = &event->attr;
4973
4974	if (event->parent)
4975	return false;
4976
4977	if (event->attach_state & PERF_ATTACH_TASK)
4978	return false;
4979
4980	if (attr->mmap || attr->mmap_data || attr->mmap2 ||
4981	attr->comm || attr->comm_exec ||
4982	attr->task || attr->ksymbol ||
4983	attr->context_switch || attr->text_poke ||
4984	attr->bpf_event)
4985	return true;
4986	return false;
4987	}
4988
4989	static void unaccount_pmu_sb_event(struct perf_event *event)
4990	{
4991	if (is_sb_event(event))
4992	detach_sb_event(event);
4993	}
4994
4995	#ifdef CONFIG_NO_HZ_FULL
4996	static DEFINE_SPINLOCK(nr_freq_lock);
4997	#endif
4998
4999	static void unaccount_freq_event_nohz(void)
5000	{
5001	#ifdef CONFIG_NO_HZ_FULL
5002	spin_lock(&nr_freq_lock);
5003	if (atomic_dec_and_test(&nr_freq_events))
5004	tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS);
5005	spin_unlock(&nr_freq_lock);
5006	#endif
5007	}
5008
5009	static void unaccount_freq_event(void)
5010	{
5011	if (tick_nohz_full_enabled())
5012	unaccount_freq_event_nohz();
5013	else
5014	atomic_dec(v: &nr_freq_events);
5015	}
5016
5017	static void unaccount_event(struct perf_event *event)
5018	{
5019	bool dec = false;
5020
5021	if (event->parent)
5022	return;
5023
5024	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
5025	dec = true;
5026	if (event->attr.mmap || event->attr.mmap_data)
5027	atomic_dec(v: &nr_mmap_events);
5028	if (event->attr.build_id)
5029	atomic_dec(v: &nr_build_id_events);
5030	if (event->attr.comm)
5031	atomic_dec(v: &nr_comm_events);
5032	if (event->attr.namespaces)
5033	atomic_dec(v: &nr_namespaces_events);
5034	if (event->attr.cgroup)
5035	atomic_dec(v: &nr_cgroup_events);
5036	if (event->attr.task)
5037	atomic_dec(v: &nr_task_events);
5038	if (event->attr.freq)
5039	unaccount_freq_event();
5040	if (event->attr.context_switch) {
5041	dec = true;
5042	atomic_dec(v: &nr_switch_events);
5043	}
5044	if (is_cgroup_event(event))
5045	dec = true;
5046	if (has_branch_stack(event))
5047	dec = true;
5048	if (event->attr.ksymbol)
5049	atomic_dec(v: &nr_ksymbol_events);
5050	if (event->attr.bpf_event)
5051	atomic_dec(v: &nr_bpf_events);
5052	if (event->attr.text_poke)
5053	atomic_dec(v: &nr_text_poke_events);
5054
5055	if (dec) {
5056	if (!atomic_add_unless(v: &perf_sched_count, a: -1, u: 1))
5057	schedule_delayed_work(dwork: &perf_sched_work, HZ);
5058	}
5059
5060	unaccount_pmu_sb_event(event);
5061	}
5062
5063	static void perf_sched_delayed(struct work_struct *work)
5064	{
5065	mutex_lock(&perf_sched_mutex);
5066	if (atomic_dec_and_test(v: &perf_sched_count))
5067	static_branch_disable(&perf_sched_events);
5068	mutex_unlock(lock: &perf_sched_mutex);
5069	}
5070
5071	/*
5072	* The following implement mutual exclusion of events on "exclusive" pmus
5073	* (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled
5074	* at a time, so we disallow creating events that might conflict, namely:
5075	*
5076	* 1) cpu-wide events in the presence of per-task events,
5077	* 2) per-task events in the presence of cpu-wide events,
5078	* 3) two matching events on the same perf_event_context.
5079	*
5080	* The former two cases are handled in the allocation path (perf_event_alloc(),
5081	* _free_event()), the latter -- before the first perf_install_in_context().
5082	*/
5083	static int exclusive_event_init(struct perf_event *event)
5084	{
5085	struct pmu *pmu = event->pmu;
5086
5087	if (!is_exclusive_pmu(pmu))
5088	return 0;
5089
5090	/*
5091	* Prevent co-existence of per-task and cpu-wide events on the
5092	* same exclusive pmu.
5093	*
5094	* Negative pmu::exclusive_cnt means there are cpu-wide
5095	* events on this "exclusive" pmu, positive means there are
5096	* per-task events.
5097	*
5098	* Since this is called in perf_event_alloc() path, event::ctx
5099	* doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK
5100	* to mean "per-task event", because unlike other attach states it
5101	* never gets cleared.
5102	*/
5103	if (event->attach_state & PERF_ATTACH_TASK) {
5104	if (!atomic_inc_unless_negative(v: &pmu->exclusive_cnt))
5105	return -EBUSY;
5106	} else {
5107	if (!atomic_dec_unless_positive(v: &pmu->exclusive_cnt))
5108	return -EBUSY;
5109	}
5110
5111	return 0;
5112	}
5113
5114	static void exclusive_event_destroy(struct perf_event *event)
5115	{
5116	struct pmu *pmu = event->pmu;
5117
5118	if (!is_exclusive_pmu(pmu))
5119	return;
5120
5121	/* see comment in exclusive_event_init() */
5122	if (event->attach_state & PERF_ATTACH_TASK)
5123	atomic_dec(v: &pmu->exclusive_cnt);
5124	else
5125	atomic_inc(v: &pmu->exclusive_cnt);
5126	}
5127
5128	static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2)
5129	{
5130	if ((e1->pmu == e2->pmu) &&
5131	(e1->cpu == e2->cpu ||
5132	e1->cpu == -1 ||
5133	e2->cpu == -1))
5134	return true;
5135	return false;
5136	}
5137
5138	static bool exclusive_event_installable(struct perf_event *event,
5139	struct perf_event_context *ctx)
5140	{
5141	struct perf_event *iter_event;
5142	struct pmu *pmu = event->pmu;
5143
5144	lockdep_assert_held(&ctx->mutex);
5145
5146	if (!is_exclusive_pmu(pmu))
5147	return true;
5148
5149	list_for_each_entry(iter_event, &ctx->event_list, event_entry) {
5150	if (exclusive_event_match(e1: iter_event, e2: event))
5151	return false;
5152	}
5153
5154	return true;
5155	}
5156
5157	static void perf_addr_filters_splice(struct perf_event *event,
5158	struct list_head *head);
5159
5160	static void _free_event(struct perf_event *event)
5161	{
5162	irq_work_sync(work: &event->pending_irq);
5163
5164	unaccount_event(event);
5165
5166	security_perf_event_free(event);
5167
5168	if (event->rb) {
5169	/*
5170	* Can happen when we close an event with re-directed output.
5171	*
5172	* Since we have a 0 refcount, perf_mmap_close() will skip
5173	* over us; possibly making our ring_buffer_put() the last.
5174	*/
5175	mutex_lock(&event->mmap_mutex);
5176	ring_buffer_attach(event, NULL);
5177	mutex_unlock(lock: &event->mmap_mutex);
5178	}
5179
5180	if (is_cgroup_event(event))
5181	perf_detach_cgroup(event);
5182
5183	if (!event->parent) {
5184	if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
5185	put_callchain_buffers();
5186	}
5187
5188	perf_event_free_bpf_prog(event);
5189	perf_addr_filters_splice(event, NULL);
5190	kfree(objp: event->addr_filter_ranges);
5191
5192	if (event->destroy)
5193	event->destroy(event);
5194
5195	/*
5196	* Must be after ->destroy(), due to uprobe_perf_close() using
5197	* hw.target.
5198	*/
5199	if (event->hw.target)
5200	put_task_struct(t: event->hw.target);
5201
5202	if (event->pmu_ctx)
5203	put_pmu_ctx(epc: event->pmu_ctx);
5204
5205	/*
5206	* perf_event_free_task() relies on put_ctx() being 'last', in particular
5207	* all task references must be cleaned up.
5208	*/
5209	if (event->ctx)
5210	put_ctx(ctx: event->ctx);
5211
5212	exclusive_event_destroy(event);
5213	module_put(module: event->pmu->module);
5214
5215	call_rcu(head: &event->rcu_head, func: free_event_rcu);
5216	}
5217
5218	/*
5219	* Used to free events which have a known refcount of 1, such as in error paths
5220	* where the event isn't exposed yet and inherited events.
5221	*/
5222	static void free_event(struct perf_event *event)
5223	{
5224	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
5225	"unexpected event refcount: %ld; ptr=%p\n",
5226	atomic_long_read(&event->refcount), event)) {
5227	/* leak to avoid use-after-free */
5228	return;
5229	}
5230
5231	_free_event(event);
5232	}
5233
5234	/*
5235	* Remove user event from the owner task.
5236	*/
5237	static void perf_remove_from_owner(struct perf_event *event)
5238	{
5239	struct task_struct *owner;
5240
5241	rcu_read_lock();
5242	/*
5243	* Matches the smp_store_release() in perf_event_exit_task(). If we
5244	* observe !owner it means the list deletion is complete and we can
5245	* indeed free this event, otherwise we need to serialize on
5246	* owner->perf_event_mutex.
5247	*/
5248	owner = READ_ONCE(event->owner);
5249	if (owner) {
5250	/*
5251	* Since delayed_put_task_struct() also drops the last
5252	* task reference we can safely take a new reference
5253	* while holding the rcu_read_lock().
5254	*/
5255	get_task_struct(t: owner);
5256	}
5257	rcu_read_unlock();
5258
5259	if (owner) {
5260	/*
5261	* If we're here through perf_event_exit_task() we're already
5262	* holding ctx->mutex which would be an inversion wrt. the
5263	* normal lock order.
5264	*
5265	* However we can safely take this lock because its the child
5266	* ctx->mutex.
5267	*/
5268	mutex_lock_nested(lock: &owner->perf_event_mutex, SINGLE_DEPTH_NESTING);
5269
5270	/*
5271	* We have to re-check the event->owner field, if it is cleared
5272	* we raced with perf_event_exit_task(), acquiring the mutex
5273	* ensured they're done, and we can proceed with freeing the
5274	* event.
5275	*/
5276	if (event->owner) {
5277	list_del_init(entry: &event->owner_entry);
5278	smp_store_release(&event->owner, NULL);
5279	}
5280	mutex_unlock(lock: &owner->perf_event_mutex);
5281	put_task_struct(t: owner);
5282	}
5283	}
5284
5285	static void put_event(struct perf_event *event)
5286	{
5287	if (!atomic_long_dec_and_test(v: &event->refcount))
5288	return;
5289
5290	_free_event(event);
5291	}
5292
5293	/*
5294	* Kill an event dead; while event:refcount will preserve the event
5295	* object, it will not preserve its functionality. Once the last 'user'
5296	* gives up the object, we'll destroy the thing.
5297	*/
5298	int perf_event_release_kernel(struct perf_event *event)
5299	{
5300	struct perf_event_context *ctx = event->ctx;
5301	struct perf_event *child, *tmp;
5302	LIST_HEAD(free_list);
5303
5304	/*
5305	* If we got here through err_alloc: free_event(event); we will not
5306	* have attached to a context yet.
5307	*/
5308	if (!ctx) {
5309	WARN_ON_ONCE(event->attach_state &
5310	(PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP));
5311	goto no_ctx;
5312	}
5313
5314	if (!is_kernel_event(event))
5315	perf_remove_from_owner(event);
5316
5317	ctx = perf_event_ctx_lock(event);
5318	WARN_ON_ONCE(ctx->parent_ctx);
5319
5320	/*
5321	* Mark this event as STATE_DEAD, there is no external reference to it
5322	* anymore.
5323	*
5324	* Anybody acquiring event->child_mutex after the below loop _must_
5325	* also see this, most importantly inherit_event() which will avoid
5326	* placing more children on the list.
5327	*
5328	* Thus this guarantees that we will in fact observe and kill _ALL_
5329	* child events.
5330	*/
5331	perf_remove_from_context(event, DETACH_GROUP|DETACH_DEAD);
5332
5333	perf_event_ctx_unlock(event, ctx);
5334
5335	again:
5336	mutex_lock(&event->child_mutex);
5337	list_for_each_entry(child, &event->child_list, child_list) {
5338
5339	/*
5340	* Cannot change, child events are not migrated, see the
5341	* comment with perf_event_ctx_lock_nested().
5342	*/
5343	ctx = READ_ONCE(child->ctx);
5344	/*
5345	* Since child_mutex nests inside ctx::mutex, we must jump
5346	* through hoops. We start by grabbing a reference on the ctx.
5347	*
5348	* Since the event cannot get freed while we hold the
5349	* child_mutex, the context must also exist and have a !0
5350	* reference count.
5351	*/
5352	get_ctx(ctx);
5353
5354	/*
5355	* Now that we have a ctx ref, we can drop child_mutex, and
5356	* acquire ctx::mutex without fear of it going away. Then we
5357	* can re-acquire child_mutex.
5358	*/
5359	mutex_unlock(lock: &event->child_mutex);
5360	mutex_lock(&ctx->mutex);
5361	mutex_lock(&event->child_mutex);
5362
5363	/*
5364	* Now that we hold ctx::mutex and child_mutex, revalidate our
5365	* state, if child is still the first entry, it didn't get freed
5366	* and we can continue doing so.
5367	*/
5368	tmp = list_first_entry_or_null(&event->child_list,
5369	struct perf_event, child_list);
5370	if (tmp == child) {
5371	perf_remove_from_context(event: child, DETACH_GROUP);
5372	list_move(list: &child->child_list, head: &free_list);
5373	/*
5374	* This matches the refcount bump in inherit_event();
5375	* this can't be the last reference.
5376	*/
5377	put_event(event);
5378	}
5379
5380	mutex_unlock(lock: &event->child_mutex);
5381	mutex_unlock(lock: &ctx->mutex);
5382	put_ctx(ctx);
5383	goto again;
5384	}
5385	mutex_unlock(lock: &event->child_mutex);
5386
5387	list_for_each_entry_safe(child, tmp, &free_list, child_list) {
5388	void *var = &child->ctx->refcount;
5389
5390	list_del(entry: &child->child_list);
5391	free_event(event: child);
5392
5393	/*
5394	* Wake any perf_event_free_task() waiting for this event to be
5395	* freed.
5396	*/
5397	smp_mb(); /* pairs with wait_var_event() */
5398	wake_up_var(var);
5399	}
5400
5401	no_ctx:
5402	put_event(event); /* Must be the 'last' reference */
5403	return 0;
5404	}
5405	EXPORT_SYMBOL_GPL(perf_event_release_kernel);
5406
5407	/*
5408	* Called when the last reference to the file is gone.
5409	*/
5410	static int perf_release(struct inode *inode, struct file *file)
5411	{
5412	perf_event_release_kernel(file->private_data);
5413	return 0;
5414	}
5415
5416	static u64 __perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
5417	{
5418	struct perf_event *child;
5419	u64 total = 0;
5420
5421	*enabled = 0;
5422	*running = 0;
5423
5424	mutex_lock(&event->child_mutex);
5425
5426	(void)perf_event_read(event, group: false);
5427	total += perf_event_count(event);
5428
5429	*enabled += event->total_time_enabled +
5430	atomic64_read(v: &event->child_total_time_enabled);
5431	*running += event->total_time_running +
5432	atomic64_read(v: &event->child_total_time_running);
5433
5434	list_for_each_entry(child, &event->child_list, child_list) {
5435	(void)perf_event_read(event: child, group: false);
5436	total += perf_event_count(event: child);
5437	*enabled += child->total_time_enabled;
5438	*running += child->total_time_running;
5439	}
5440	mutex_unlock(lock: &event->child_mutex);
5441
5442	return total;
5443	}
5444
5445	u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
5446	{
5447	struct perf_event_context *ctx;
5448	u64 count;
5449
5450	ctx = perf_event_ctx_lock(event);
5451	count = __perf_event_read_value(event, enabled, running);
5452	perf_event_ctx_unlock(event, ctx);
5453
5454	return count;
5455	}
5456	EXPORT_SYMBOL_GPL(perf_event_read_value);
5457
5458	static int __perf_read_group_add(struct perf_event *leader,
5459	u64 read_format, u64 *values)
5460	{
5461	struct perf_event_context *ctx = leader->ctx;
5462	struct perf_event *sub, *parent;
5463	unsigned long flags;
5464	int n = 1; /* skip @nr */
5465	int ret;
5466
5467	ret = perf_event_read(event: leader, group: true);
5468	if (ret)
5469	return ret;
5470
5471	raw_spin_lock_irqsave(&ctx->lock, flags);
5472	/*
5473	* Verify the grouping between the parent and child (inherited)
5474	* events is still in tact.
5475	*
5476	* Specifically:
5477	* - leader->ctx->lock pins leader->sibling_list
5478	* - parent->child_mutex pins parent->child_list
5479	* - parent->ctx->mutex pins parent->sibling_list
5480	*
5481	* Because parent->ctx != leader->ctx (and child_list nests inside
5482	* ctx->mutex), group destruction is not atomic between children, also
5483	* see perf_event_release_kernel(). Additionally, parent can grow the
5484	* group.
5485	*
5486	* Therefore it is possible to have parent and child groups in a
5487	* different configuration and summing over such a beast makes no sense
5488	* what so ever.
5489	*
5490	* Reject this.
5491	*/
5492	parent = leader->parent;
5493	if (parent &&
5494	(parent->group_generation != leader->group_generation ||
5495	parent->nr_siblings != leader->nr_siblings)) {
5496	ret = -ECHILD;
5497	goto unlock;
5498	}
5499
5500	/*
5501	* Since we co-schedule groups, {enabled,running} times of siblings
5502	* will be identical to those of the leader, so we only publish one
5503	* set.
5504	*/
5505	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
5506	values[n++] += leader->total_time_enabled +
5507	atomic64_read(v: &leader->child_total_time_enabled);
5508	}
5509
5510	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
5511	values[n++] += leader->total_time_running +
5512	atomic64_read(v: &leader->child_total_time_running);
5513	}
5514
5515	/*
5516	* Write {count,id} tuples for every sibling.
5517	*/
5518	values[n++] += perf_event_count(event: leader);
5519	if (read_format & PERF_FORMAT_ID)
5520	values[n++] = primary_event_id(event: leader);
5521	if (read_format & PERF_FORMAT_LOST)
5522	values[n++] = atomic64_read(v: &leader->lost_samples);
5523
5524	for_each_sibling_event(sub, leader) {
5525	values[n++] += perf_event_count(event: sub);
5526	if (read_format & PERF_FORMAT_ID)
5527	values[n++] = primary_event_id(event: sub);
5528	if (read_format & PERF_FORMAT_LOST)
5529	values[n++] = atomic64_read(v: &sub->lost_samples);
5530	}
5531
5532	unlock:
5533	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5534	return ret;
5535	}
5536
5537	static int perf_read_group(struct perf_event *event,
5538	u64 read_format, char __user *buf)
5539	{
5540	struct perf_event *leader = event->group_leader, *child;
5541	struct perf_event_context *ctx = leader->ctx;
5542	int ret;
5543	u64 *values;
5544
5545	lockdep_assert_held(&ctx->mutex);
5546
5547	values = kzalloc(size: event->read_size, GFP_KERNEL);
5548	if (!values)
5549	return -ENOMEM;
5550
5551	values[0] = 1 + leader->nr_siblings;
5552
5553	mutex_lock(&leader->child_mutex);
5554
5555	ret = __perf_read_group_add(leader, read_format, values);
5556	if (ret)
5557	goto unlock;
5558
5559	list_for_each_entry(child, &leader->child_list, child_list) {
5560	ret = __perf_read_group_add(leader: child, read_format, values);
5561	if (ret)
5562	goto unlock;
5563	}
5564
5565	mutex_unlock(lock: &leader->child_mutex);
5566
5567	ret = event->read_size;
5568	if (copy_to_user(to: buf, from: values, n: event->read_size))
5569	ret = -EFAULT;
5570	goto out;
5571
5572	unlock:
5573	mutex_unlock(lock: &leader->child_mutex);
5574	out:
5575	kfree(objp: values);
5576	return ret;
5577	}
5578
5579	static int perf_read_one(struct perf_event *event,
5580	u64 read_format, char __user *buf)
5581	{
5582	u64 enabled, running;
5583	u64 values[5];
5584	int n = 0;
5585
5586	values[n++] = __perf_event_read_value(event, enabled: &enabled, running: &running);
5587	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
5588	values[n++] = enabled;
5589	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
5590	values[n++] = running;
5591	if (read_format & PERF_FORMAT_ID)
5592	values[n++] = primary_event_id(event);
5593	if (read_format & PERF_FORMAT_LOST)
5594	values[n++] = atomic64_read(v: &event->lost_samples);
5595
5596	if (copy_to_user(to: buf, from: values, n: n * sizeof(u64)))
5597	return -EFAULT;
5598
5599	return n * sizeof(u64);
5600	}
5601
5602	static bool is_event_hup(struct perf_event *event)
5603	{
5604	bool no_children;
5605
5606	if (event->state > PERF_EVENT_STATE_EXIT)
5607	return false;
5608
5609	mutex_lock(&event->child_mutex);
5610	no_children = list_empty(head: &event->child_list);
5611	mutex_unlock(lock: &event->child_mutex);
5612	return no_children;
5613	}
5614
5615	/*
5616	* Read the performance event - simple non blocking version for now
5617	*/
5618	static ssize_t
5619	__perf_read(struct perf_event *event, char __user *buf, size_t count)
5620	{
5621	u64 read_format = event->attr.read_format;
5622	int ret;
5623
5624	/*
5625	* Return end-of-file for a read on an event that is in
5626	* error state (i.e. because it was pinned but it couldn't be
5627	* scheduled on to the CPU at some point).
5628	*/
5629	if (event->state == PERF_EVENT_STATE_ERROR)
5630	return 0;
5631
5632	if (count < event->read_size)
5633	return -ENOSPC;
5634
5635	WARN_ON_ONCE(event->ctx->parent_ctx);
5636	if (read_format & PERF_FORMAT_GROUP)
5637	ret = perf_read_group(event, read_format, buf);
5638	else
5639	ret = perf_read_one(event, read_format, buf);
5640
5641	return ret;
5642	}
5643
5644	static ssize_t
5645	perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
5646	{
5647	struct perf_event *event = file->private_data;
5648	struct perf_event_context *ctx;
5649	int ret;
5650
5651	ret = security_perf_event_read(event);
5652	if (ret)
5653	return ret;
5654
5655	ctx = perf_event_ctx_lock(event);
5656	ret = __perf_read(event, buf, count);
5657	perf_event_ctx_unlock(event, ctx);
5658
5659	return ret;
5660	}
5661
5662	static __poll_t perf_poll(struct file *file, poll_table *wait)
5663	{
5664	struct perf_event *event = file->private_data;
5665	struct perf_buffer *rb;
5666	__poll_t events = EPOLLHUP;
5667
5668	poll_wait(filp: file, wait_address: &event->waitq, p: wait);
5669
5670	if (is_event_hup(event))
5671	return events;
5672
5673	/*
5674	* Pin the event->rb by taking event->mmap_mutex; otherwise
5675	* perf_event_set_output() can swizzle our rb and make us miss wakeups.
5676	*/
5677	mutex_lock(&event->mmap_mutex);
5678	rb = event->rb;
5679	if (rb)
5680	events = atomic_xchg(v: &rb->poll, new: 0);
5681	mutex_unlock(lock: &event->mmap_mutex);
5682	return events;
5683	}
5684
5685	static void _perf_event_reset(struct perf_event *event)
5686	{
5687	(void)perf_event_read(event, group: false);
5688	local64_set(&event->count, 0);
5689	perf_event_update_userpage(event);
5690	}
5691
5692	/* Assume it's not an event with inherit set. */
5693	u64 perf_event_pause(struct perf_event *event, bool reset)
5694	{
5695	struct perf_event_context *ctx;
5696	u64 count;
5697
5698	ctx = perf_event_ctx_lock(event);
5699	WARN_ON_ONCE(event->attr.inherit);
5700	_perf_event_disable(event);
5701	count = local64_read(&event->count);
5702	if (reset)
5703	local64_set(&event->count, 0);
5704	perf_event_ctx_unlock(event, ctx);
5705
5706	return count;
5707	}
5708	EXPORT_SYMBOL_GPL(perf_event_pause);
5709
5710	/*
5711	* Holding the top-level event's child_mutex means that any
5712	* descendant process that has inherited this event will block
5713	* in perf_event_exit_event() if it goes to exit, thus satisfying the
5714	* task existence requirements of perf_event_enable/disable.
5715	*/
5716	static void perf_event_for_each_child(struct perf_event *event,
5717	void (*func)(struct perf_event *))
5718	{
5719	struct perf_event *child;
5720
5721	WARN_ON_ONCE(event->ctx->parent_ctx);
5722
5723	mutex_lock(&event->child_mutex);
5724	func(event);
5725	list_for_each_entry(child, &event->child_list, child_list)
5726	func(child);
5727	mutex_unlock(lock: &event->child_mutex);
5728	}
5729
5730	static void perf_event_for_each(struct perf_event *event,
5731	void (*func)(struct perf_event *))
5732	{
5733	struct perf_event_context *ctx = event->ctx;
5734	struct perf_event *sibling;
5735
5736	lockdep_assert_held(&ctx->mutex);
5737
5738	event = event->group_leader;
5739
5740	perf_event_for_each_child(event, func);
5741	for_each_sibling_event(sibling, event)
5742	perf_event_for_each_child(event: sibling, func);
5743	}
5744
5745	static void __perf_event_period(struct perf_event *event,
5746	struct perf_cpu_context *cpuctx,
5747	struct perf_event_context *ctx,
5748	void *info)
5749	{
5750	u64 value = *((u64 *)info);
5751	bool active;
5752
5753	if (event->attr.freq) {
5754	event->attr.sample_freq = value;
5755	} else {
5756	event->attr.sample_period = value;
5757	event->hw.sample_period = value;
5758	}
5759
5760	active = (event->state == PERF_EVENT_STATE_ACTIVE);
5761	if (active) {
5762	perf_pmu_disable(pmu: event->pmu);
5763	/*
5764	* We could be throttled; unthrottle now to avoid the tick
5765	* trying to unthrottle while we already re-started the event.
5766	*/
5767	if (event->hw.interrupts == MAX_INTERRUPTS) {
5768	event->hw.interrupts = 0;
5769	perf_log_throttle(event, enable: 1);
5770	}
5771	event->pmu->stop(event, PERF_EF_UPDATE);
5772	}
5773
5774	local64_set(&event->hw.period_left, 0);
5775
5776	if (active) {
5777	event->pmu->start(event, PERF_EF_RELOAD);
5778	perf_pmu_enable(pmu: event->pmu);
5779	}
5780	}
5781
5782	static int perf_event_check_period(struct perf_event *event, u64 value)
5783	{
5784	return event->pmu->check_period(event, value);
5785	}
5786
5787	static int _perf_event_period(struct perf_event *event, u64 value)
5788	{
5789	if (!is_sampling_event(event))
5790	return -EINVAL;
5791
5792	if (!value)
5793	return -EINVAL;
5794
5795	if (event->attr.freq && value > sysctl_perf_event_sample_rate)
5796	return -EINVAL;
5797
5798	if (perf_event_check_period(event, value))
5799	return -EINVAL;
5800
5801	if (!event->attr.freq && (value & (1ULL << 63)))
5802	return -EINVAL;
5803
5804	event_function_call(event, func: __perf_event_period, data: &value);
5805
5806	return 0;
5807	}
5808
5809	int perf_event_period(struct perf_event *event, u64 value)
5810	{
5811	struct perf_event_context *ctx;
5812	int ret;
5813
5814	ctx = perf_event_ctx_lock(event);
5815	ret = _perf_event_period(event, value);
5816	perf_event_ctx_unlock(event, ctx);
5817
5818	return ret;
5819	}
5820	EXPORT_SYMBOL_GPL(perf_event_period);
5821
5822	static const struct file_operations perf_fops;
5823
5824	static inline int perf_fget_light(int fd, struct fd *p)
5825	{
5826	struct fd f = fdget(fd);
5827	if (!f.file)
5828	return -EBADF;
5829
5830	if (f.file->f_op != &perf_fops) {
5831	fdput(fd: f);
5832	return -EBADF;
5833	}
5834	*p = f;
5835	return 0;
5836	}
5837
5838	static int perf_event_set_output(struct perf_event *event,
5839	struct perf_event *output_event);
5840	static int perf_event_set_filter(struct perf_event *event, void __user *arg);
5841	static int perf_copy_attr(struct perf_event_attr __user *uattr,
5842	struct perf_event_attr *attr);
5843
5844	static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
5845	{
5846	void (*func)(struct perf_event *);
5847	u32 flags = arg;
5848
5849	switch (cmd) {
5850	case PERF_EVENT_IOC_ENABLE:
5851	func = _perf_event_enable;
5852	break;
5853	case PERF_EVENT_IOC_DISABLE:
5854	func = _perf_event_disable;
5855	break;
5856	case PERF_EVENT_IOC_RESET:
5857	func = _perf_event_reset;
5858	break;
5859
5860	case PERF_EVENT_IOC_REFRESH:
5861	return _perf_event_refresh(event, refresh: arg);
5862
5863	case PERF_EVENT_IOC_PERIOD:
5864	{
5865	u64 value;
5866
5867	if (copy_from_user(to: &value, from: (u64 __user *)arg, n: sizeof(value)))
5868	return -EFAULT;
5869
5870	return _perf_event_period(event, value);
5871	}
5872	case PERF_EVENT_IOC_ID:
5873	{
5874	u64 id = primary_event_id(event);
5875
5876	if (copy_to_user(to: (void __user *)arg, from: &id, n: sizeof(id)))
5877	return -EFAULT;
5878	return 0;
5879	}
5880
5881	case PERF_EVENT_IOC_SET_OUTPUT:
5882	{
5883	int ret;
5884	if (arg != -1) {
5885	struct perf_event *output_event;
5886	struct fd output;
5887	ret = perf_fget_light(fd: arg, p: &output);
5888	if (ret)
5889	return ret;
5890	output_event = output.file->private_data;
5891	ret = perf_event_set_output(event, output_event);
5892	fdput(fd: output);
5893	} else {
5894	ret = perf_event_set_output(event, NULL);
5895	}
5896	return ret;
5897	}
5898
5899	case PERF_EVENT_IOC_SET_FILTER:
5900	return perf_event_set_filter(event, arg: (void __user *)arg);
5901
5902	case PERF_EVENT_IOC_SET_BPF:
5903	{
5904	struct bpf_prog *prog;
5905	int err;
5906
5907	prog = bpf_prog_get(ufd: arg);
5908	if (IS_ERR(ptr: prog))
5909	return PTR_ERR(ptr: prog);
5910
5911	err = perf_event_set_bpf_prog(event, prog, bpf_cookie: 0);
5912	if (err) {
5913	bpf_prog_put(prog);
5914	return err;
5915	}
5916
5917	return 0;
5918	}
5919
5920	case PERF_EVENT_IOC_PAUSE_OUTPUT: {
5921	struct perf_buffer *rb;
5922
5923	rcu_read_lock();
5924	rb = rcu_dereference(event->rb);
5925	if (!rb || !rb->nr_pages) {
5926	rcu_read_unlock();
5927	return -EINVAL;
5928	}
5929	rb_toggle_paused(rb, pause: !!arg);
5930	rcu_read_unlock();
5931	return 0;
5932	}
5933
5934	case PERF_EVENT_IOC_QUERY_BPF:
5935	return perf_event_query_prog_array(event, info: (void __user *)arg);
5936
5937	case PERF_EVENT_IOC_MODIFY_ATTRIBUTES: {
5938	struct perf_event_attr new_attr;
5939	int err = perf_copy_attr(uattr: (struct perf_event_attr __user *)arg,
5940	attr: &new_attr);
5941
5942	if (err)
5943	return err;
5944
5945	return perf_event_modify_attr(event, attr: &new_attr);
5946	}
5947	default:
5948	return -ENOTTY;
5949	}
5950
5951	if (flags & PERF_IOC_FLAG_GROUP)
5952	perf_event_for_each(event, func);
5953	else
5954	perf_event_for_each_child(event, func);
5955
5956	return 0;
5957	}
5958
5959	static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
5960	{
5961	struct perf_event *event = file->private_data;
5962	struct perf_event_context *ctx;
5963	long ret;
5964
5965	/* Treat ioctl like writes as it is likely a mutating operation. */
5966	ret = security_perf_event_write(event);
5967	if (ret)
5968	return ret;
5969
5970	ctx = perf_event_ctx_lock(event);
5971	ret = _perf_ioctl(event, cmd, arg);
5972	perf_event_ctx_unlock(event, ctx);
5973
5974	return ret;
5975	}
5976
5977	#ifdef CONFIG_COMPAT
5978	static long perf_compat_ioctl(struct file *file, unsigned int cmd,
5979	unsigned long arg)
5980	{
5981	switch (_IOC_NR(cmd)) {
5982	case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
5983	case _IOC_NR(PERF_EVENT_IOC_ID):
5984	case _IOC_NR(PERF_EVENT_IOC_QUERY_BPF):
5985	case _IOC_NR(PERF_EVENT_IOC_MODIFY_ATTRIBUTES):
5986	/* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
5987	if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
5988	cmd &= ~IOCSIZE_MASK;
5989	cmd |= sizeof(void *) << IOCSIZE_SHIFT;
5990	}
5991	break;
5992	}
5993	return perf_ioctl(file, cmd, arg);
5994	}
5995	#else
5996	# define perf_compat_ioctl NULL
5997	#endif
5998
5999	int perf_event_task_enable(void)
6000	{
6001	struct perf_event_context *ctx;
6002	struct perf_event *event;
6003
6004	mutex_lock(&current->perf_event_mutex);
6005	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
6006	ctx = perf_event_ctx_lock(event);
6007	perf_event_for_each_child(event, func: _perf_event_enable);
6008	perf_event_ctx_unlock(event, ctx);
6009	}
6010	mutex_unlock(lock: &current->perf_event_mutex);
6011
6012	return 0;
6013	}
6014
6015	int perf_event_task_disable(void)
6016	{
6017	struct perf_event_context *ctx;
6018	struct perf_event *event;
6019
6020	mutex_lock(&current->perf_event_mutex);
6021	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
6022	ctx = perf_event_ctx_lock(event);
6023	perf_event_for_each_child(event, func: _perf_event_disable);
6024	perf_event_ctx_unlock(event, ctx);
6025	}
6026	mutex_unlock(lock: &current->perf_event_mutex);
6027
6028	return 0;
6029	}
6030
6031	static int perf_event_index(struct perf_event *event)
6032	{
6033	if (event->hw.state & PERF_HES_STOPPED)
6034	return 0;
6035
6036	if (event->state != PERF_EVENT_STATE_ACTIVE)
6037	return 0;
6038
6039	return event->pmu->event_idx(event);
6040	}
6041
6042	static void perf_event_init_userpage(struct perf_event *event)
6043	{
6044	struct perf_event_mmap_page *userpg;
6045	struct perf_buffer *rb;
6046
6047	rcu_read_lock();
6048	rb = rcu_dereference(event->rb);
6049	if (!rb)
6050	goto unlock;
6051
6052	userpg = rb->user_page;
6053
6054	/* Allow new userspace to detect that bit 0 is deprecated */
6055	userpg->cap_bit0_is_deprecated = 1;
6056	userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
6057	userpg->data_offset = PAGE_SIZE;
6058	userpg->data_size = perf_data_size(rb);
6059
6060	unlock:
6061	rcu_read_unlock();
6062	}
6063
6064	void __weak arch_perf_update_userpage(
6065	struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now)
6066	{
6067	}
6068
6069	/*
6070	* Callers need to ensure there can be no nesting of this function, otherwise
6071	* the seqlock logic goes bad. We can not serialize this because the arch
6072	* code calls this from NMI context.
6073	*/
6074	void perf_event_update_userpage(struct perf_event *event)
6075	{
6076	struct perf_event_mmap_page *userpg;
6077	struct perf_buffer *rb;
6078	u64 enabled, running, now;
6079
6080	rcu_read_lock();
6081	rb = rcu_dereference(event->rb);
6082	if (!rb)
6083	goto unlock;
6084
6085	/*
6086	* compute total_time_enabled, total_time_running
6087	* based on snapshot values taken when the event
6088	* was last scheduled in.
6089	*
6090	* we cannot simply called update_context_time()
6091	* because of locking issue as we can be called in
6092	* NMI context
6093	*/
6094	calc_timer_values(event, now: &now, enabled: &enabled, running: &running);
6095
6096	userpg = rb->user_page;
6097	/*
6098	* Disable preemption to guarantee consistent time stamps are stored to
6099	* the user page.
6100	*/
6101	preempt_disable();
6102	++userpg->lock;
6103	barrier();
6104	userpg->index = perf_event_index(event);
6105	userpg->offset = perf_event_count(event);
6106	if (userpg->index)
6107	userpg->offset -= local64_read(&event->hw.prev_count);
6108
6109	userpg->time_enabled = enabled +
6110	atomic64_read(v: &event->child_total_time_enabled);
6111
6112	userpg->time_running = running +
6113	atomic64_read(v: &event->child_total_time_running);
6114
6115	arch_perf_update_userpage(event, userpg, now);
6116
6117	barrier();
6118	++userpg->lock;
6119	preempt_enable();
6120	unlock:
6121	rcu_read_unlock();
6122	}
6123	EXPORT_SYMBOL_GPL(perf_event_update_userpage);
6124
6125	static vm_fault_t perf_mmap_fault(struct vm_fault *vmf)
6126	{
6127	struct perf_event *event = vmf->vma->vm_file->private_data;
6128	struct perf_buffer *rb;
6129	vm_fault_t ret = VM_FAULT_SIGBUS;
6130
6131	if (vmf->flags & FAULT_FLAG_MKWRITE) {
6132	if (vmf->pgoff == 0)
6133	ret = 0;
6134	return ret;
6135	}
6136
6137	rcu_read_lock();
6138	rb = rcu_dereference(event->rb);
6139	if (!rb)
6140	goto unlock;
6141
6142	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
6143	goto unlock;
6144
6145	vmf->page = perf_mmap_to_page(rb, pgoff: vmf->pgoff);
6146	if (!vmf->page)
6147	goto unlock;
6148
6149	get_page(page: vmf->page);
6150	vmf->page->mapping = vmf->vma->vm_file->f_mapping;
6151	vmf->page->index = vmf->pgoff;
6152
6153	ret = 0;
6154	unlock:
6155	rcu_read_unlock();
6156
6157	return ret;
6158	}
6159
6160	static void ring_buffer_attach(struct perf_event *event,
6161	struct perf_buffer *rb)
6162	{
6163	struct perf_buffer *old_rb = NULL;
6164	unsigned long flags;
6165
6166	WARN_ON_ONCE(event->parent);
6167
6168	if (event->rb) {
6169	/*
6170	* Should be impossible, we set this when removing
6171	* event->rb_entry and wait/clear when adding event->rb_entry.
6172	*/
6173	WARN_ON_ONCE(event->rcu_pending);
6174
6175	old_rb = event->rb;
6176	spin_lock_irqsave(&old_rb->event_lock, flags);
6177	list_del_rcu(entry: &event->rb_entry);
6178	spin_unlock_irqrestore(lock: &old_rb->event_lock, flags);
6179
6180	event->rcu_batches = get_state_synchronize_rcu();
6181	event->rcu_pending = 1;
6182	}
6183
6184	if (rb) {
6185	if (event->rcu_pending) {
6186	cond_synchronize_rcu(oldstate: event->rcu_batches);
6187	event->rcu_pending = 0;
6188	}
6189
6190	spin_lock_irqsave(&rb->event_lock, flags);
6191	list_add_rcu(new: &event->rb_entry, head: &rb->event_list);
6192	spin_unlock_irqrestore(lock: &rb->event_lock, flags);
6193	}
6194
6195	/*
6196	* Avoid racing with perf_mmap_close(AUX): stop the event
6197	* before swizzling the event::rb pointer; if it's getting
6198	* unmapped, its aux_mmap_count will be 0 and it won't
6199	* restart. See the comment in __perf_pmu_output_stop().
6200	*
6201	* Data will inevitably be lost when set_output is done in
6202	* mid-air, but then again, whoever does it like this is
6203	* not in for the data anyway.
6204	*/
6205	if (has_aux(event))
6206	perf_event_stop(event, restart: 0);
6207
6208	rcu_assign_pointer(event->rb, rb);
6209
6210	if (old_rb) {
6211	ring_buffer_put(rb: old_rb);
6212	/*
6213	* Since we detached before setting the new rb, so that we
6214	* could attach the new rb, we could have missed a wakeup.
6215	* Provide it now.
6216	*/
6217	wake_up_all(&event->waitq);
6218	}
6219	}
6220
6221	static void ring_buffer_wakeup(struct perf_event *event)
6222	{
6223	struct perf_buffer *rb;
6224
6225	if (event->parent)
6226	event = event->parent;
6227
6228	rcu_read_lock();
6229	rb = rcu_dereference(event->rb);
6230	if (rb) {
6231	list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
6232	wake_up_all(&event->waitq);
6233	}
6234	rcu_read_unlock();
6235	}
6236
6237	struct perf_buffer *ring_buffer_get(struct perf_event *event)
6238	{
6239	struct perf_buffer *rb;
6240
6241	if (event->parent)
6242	event = event->parent;
6243
6244	rcu_read_lock();
6245	rb = rcu_dereference(event->rb);
6246	if (rb) {
6247	if (!refcount_inc_not_zero(r: &rb->refcount))
6248	rb = NULL;
6249	}
6250	rcu_read_unlock();
6251
6252	return rb;
6253	}
6254
6255	void ring_buffer_put(struct perf_buffer *rb)
6256	{
6257	if (!refcount_dec_and_test(r: &rb->refcount))
6258	return;
6259
6260	WARN_ON_ONCE(!list_empty(&rb->event_list));
6261
6262	call_rcu(head: &rb->rcu_head, func: rb_free_rcu);
6263	}
6264
6265	static void perf_mmap_open(struct vm_area_struct *vma)
6266	{
6267	struct perf_event *event = vma->vm_file->private_data;
6268
6269	atomic_inc(v: &event->mmap_count);
6270	atomic_inc(v: &event->rb->mmap_count);
6271
6272	if (vma->vm_pgoff)
6273	atomic_inc(v: &event->rb->aux_mmap_count);
6274
6275	if (event->pmu->event_mapped)
6276	event->pmu->event_mapped(event, vma->vm_mm);
6277	}
6278
6279	static void perf_pmu_output_stop(struct perf_event *event);
6280
6281	/*
6282	* A buffer can be mmap()ed multiple times; either directly through the same
6283	* event, or through other events by use of perf_event_set_output().
6284	*
6285	* In order to undo the VM accounting done by perf_mmap() we need to destroy
6286	* the buffer here, where we still have a VM context. This means we need
6287	* to detach all events redirecting to us.
6288	*/
6289	static void perf_mmap_close(struct vm_area_struct *vma)
6290	{
6291	struct perf_event *event = vma->vm_file->private_data;
6292	struct perf_buffer *rb = ring_buffer_get(event);
6293	struct user_struct *mmap_user = rb->mmap_user;
6294	int mmap_locked = rb->mmap_locked;
6295	unsigned long size = perf_data_size(rb);
6296	bool detach_rest = false;
6297
6298	if (event->pmu->event_unmapped)
6299	event->pmu->event_unmapped(event, vma->vm_mm);
6300
6301	/*
6302	* rb->aux_mmap_count will always drop before rb->mmap_count and
6303	* event->mmap_count, so it is ok to use event->mmap_mutex to
6304	* serialize with perf_mmap here.
6305	*/
6306	if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&
6307	atomic_dec_and_mutex_lock(cnt: &rb->aux_mmap_count, lock: &event->mmap_mutex)) {
6308	/*
6309	* Stop all AUX events that are writing to this buffer,
6310	* so that we can free its AUX pages and corresponding PMU
6311	* data. Note that after rb::aux_mmap_count dropped to zero,
6312	* they won't start any more (see perf_aux_output_begin()).
6313	*/
6314	perf_pmu_output_stop(event);
6315
6316	/* now it's safe to free the pages */
6317	atomic_long_sub(i: rb->aux_nr_pages - rb->aux_mmap_locked, v: &mmap_user->locked_vm);
6318	atomic64_sub(i: rb->aux_mmap_locked, v: &vma->vm_mm->pinned_vm);
6319
6320	/* this has to be the last one */
6321	rb_free_aux(rb);
6322	WARN_ON_ONCE(refcount_read(&rb->aux_refcount));
6323
6324	mutex_unlock(lock: &event->mmap_mutex);
6325	}
6326
6327	if (atomic_dec_and_test(v: &rb->mmap_count))
6328	detach_rest = true;
6329
6330	if (!atomic_dec_and_mutex_lock(cnt: &event->mmap_count, lock: &event->mmap_mutex))
6331	goto out_put;
6332
6333	ring_buffer_attach(event, NULL);
6334	mutex_unlock(lock: &event->mmap_mutex);
6335
6336	/* If there's still other mmap()s of this buffer, we're done. */
6337	if (!detach_rest)
6338	goto out_put;
6339
6340	/*
6341	* No other mmap()s, detach from all other events that might redirect
6342	* into the now unreachable buffer. Somewhat complicated by the
6343	* fact that rb::event_lock otherwise nests inside mmap_mutex.
6344	*/
6345	again:
6346	rcu_read_lock();
6347	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
6348	if (!atomic_long_inc_not_zero(v: &event->refcount)) {
6349	/*
6350	* This event is en-route to free_event() which will
6351	* detach it and remove it from the list.
6352	*/
6353	continue;
6354	}
6355	rcu_read_unlock();
6356
6357	mutex_lock(&event->mmap_mutex);
6358	/*
6359	* Check we didn't race with perf_event_set_output() which can
6360	* swizzle the rb from under us while we were waiting to
6361	* acquire mmap_mutex.
6362	*
6363	* If we find a different rb; ignore this event, a next
6364	* iteration will no longer find it on the list. We have to
6365	* still restart the iteration to make sure we're not now
6366	* iterating the wrong list.
6367	*/
6368	if (event->rb == rb)
6369	ring_buffer_attach(event, NULL);
6370
6371	mutex_unlock(lock: &event->mmap_mutex);
6372	put_event(event);
6373
6374	/*
6375	* Restart the iteration; either we're on the wrong list or
6376	* destroyed its integrity by doing a deletion.
6377	*/
6378	goto again;
6379	}
6380	rcu_read_unlock();
6381
6382	/*
6383	* It could be there's still a few 0-ref events on the list; they'll
6384	* get cleaned up by free_event() -- they'll also still have their
6385	* ref on the rb and will free it whenever they are done with it.
6386	*
6387	* Aside from that, this buffer is 'fully' detached and unmapped,
6388	* undo the VM accounting.
6389	*/
6390
6391	atomic_long_sub(i: (size >> PAGE_SHIFT) + 1 - mmap_locked,
6392	v: &mmap_user->locked_vm);
6393	atomic64_sub(i: mmap_locked, v: &vma->vm_mm->pinned_vm);
6394	free_uid(mmap_user);
6395
6396	out_put:
6397	ring_buffer_put(rb); /* could be last */
6398	}
6399
6400	static const struct vm_operations_struct perf_mmap_vmops = {
6401	.open = perf_mmap_open,
6402	.close = perf_mmap_close, /* non mergeable */
6403	.fault = perf_mmap_fault,
6404	.page_mkwrite = perf_mmap_fault,
6405	};
6406
6407	static int perf_mmap(struct file *file, struct vm_area_struct *vma)
6408	{
6409	struct perf_event *event = file->private_data;
6410	unsigned long user_locked, user_lock_limit;
6411	struct user_struct *user = current_user();
6412	struct perf_buffer *rb = NULL;
6413	unsigned long locked, lock_limit;
6414	unsigned long vma_size;
6415	unsigned long nr_pages;
6416	long user_extra = 0, extra = 0;
6417	int ret = 0, flags = 0;
6418
6419	/*
6420	* Don't allow mmap() of inherited per-task counters. This would
6421	* create a performance issue due to all children writing to the
6422	* same rb.
6423	*/
6424	if (event->cpu == -1 && event->attr.inherit)
6425	return -EINVAL;
6426
6427	if (!(vma->vm_flags & VM_SHARED))
6428	return -EINVAL;
6429
6430	ret = security_perf_event_read(event);
6431	if (ret)
6432	return ret;
6433
6434	vma_size = vma->vm_end - vma->vm_start;
6435
6436	if (vma->vm_pgoff == 0) {
6437	nr_pages = (vma_size / PAGE_SIZE) - 1;
6438	} else {
6439	/*
6440	* AUX area mapping: if rb->aux_nr_pages != 0, it's already
6441	* mapped, all subsequent mappings should have the same size
6442	* and offset. Must be above the normal perf buffer.
6443	*/
6444	u64 aux_offset, aux_size;
6445
6446	if (!event->rb)
6447	return -EINVAL;
6448
6449	nr_pages = vma_size / PAGE_SIZE;
6450
6451	mutex_lock(&event->mmap_mutex);
6452	ret = -EINVAL;
6453
6454	rb = event->rb;
6455	if (!rb)
6456	goto aux_unlock;
6457
6458	aux_offset = READ_ONCE(rb->user_page->aux_offset);
6459	aux_size = READ_ONCE(rb->user_page->aux_size);
6460
6461	if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
6462	goto aux_unlock;
6463
6464	if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)
6465	goto aux_unlock;
6466
6467	/* already mapped with a different offset */
6468	if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
6469	goto aux_unlock;
6470
6471	if (aux_size != vma_size || aux_size != nr_pages * PAGE_SIZE)
6472	goto aux_unlock;
6473
6474	/* already mapped with a different size */
6475	if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
6476	goto aux_unlock;
6477
6478	if (!is_power_of_2(n: nr_pages))
6479	goto aux_unlock;
6480
6481	if (!atomic_inc_not_zero(v: &rb->mmap_count))
6482	goto aux_unlock;
6483
6484	if (rb_has_aux(rb)) {
6485	atomic_inc(v: &rb->aux_mmap_count);
6486	ret = 0;
6487	goto unlock;
6488	}
6489
6490	atomic_set(v: &rb->aux_mmap_count, i: 1);
6491	user_extra = nr_pages;
6492
6493	goto accounting;
6494	}
6495
6496	/*
6497	* If we have rb pages ensure they're a power-of-two number, so we
6498	* can do bitmasks instead of modulo.
6499	*/
6500	if (nr_pages != 0 && !is_power_of_2(n: nr_pages))
6501	return -EINVAL;
6502
6503	if (vma_size != PAGE_SIZE * (1 + nr_pages))
6504	return -EINVAL;
6505
6506	WARN_ON_ONCE(event->ctx->parent_ctx);
6507	again:
6508	mutex_lock(&event->mmap_mutex);
6509	if (event->rb) {
6510	if (data_page_nr(rb: event->rb) != nr_pages) {
6511	ret = -EINVAL;
6512	goto unlock;
6513	}
6514
6515	if (!atomic_inc_not_zero(v: &event->rb->mmap_count)) {
6516	/*
6517	* Raced against perf_mmap_close(); remove the
6518	* event and try again.
6519	*/
6520	ring_buffer_attach(event, NULL);
6521	mutex_unlock(lock: &event->mmap_mutex);
6522	goto again;
6523	}
6524
6525	goto unlock;
6526	}
6527
6528	user_extra = nr_pages + 1;
6529
6530	accounting:
6531	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
6532
6533	/*
6534	* Increase the limit linearly with more CPUs:
6535	*/
6536	user_lock_limit *= num_online_cpus();
6537
6538	user_locked = atomic_long_read(v: &user->locked_vm);
6539
6540	/*
6541	* sysctl_perf_event_mlock may have changed, so that
6542	* user->locked_vm > user_lock_limit
6543	*/
6544	if (user_locked > user_lock_limit)
6545	user_locked = user_lock_limit;
6546	user_locked += user_extra;
6547
6548	if (user_locked > user_lock_limit) {
6549	/*
6550	* charge locked_vm until it hits user_lock_limit;
6551	* charge the rest from pinned_vm
6552	*/
6553	extra = user_locked - user_lock_limit;
6554	user_extra -= extra;
6555	}
6556
6557	lock_limit = rlimit(RLIMIT_MEMLOCK);
6558	lock_limit >>= PAGE_SHIFT;
6559	locked = atomic64_read(v: &vma->vm_mm->pinned_vm) + extra;
6560
6561	if ((locked > lock_limit) && perf_is_paranoid() &&
6562	!capable(CAP_IPC_LOCK)) {
6563	ret = -EPERM;
6564	goto unlock;
6565	}
6566
6567	WARN_ON(!rb && event->rb);
6568
6569	if (vma->vm_flags & VM_WRITE)
6570	flags |= RING_BUFFER_WRITABLE;
6571
6572	if (!rb) {
6573	rb = rb_alloc(nr_pages,
6574	watermark: event->attr.watermark ? event->attr.wakeup_watermark : 0,
6575	cpu: event->cpu, flags);
6576
6577	if (!rb) {
6578	ret = -ENOMEM;
6579	goto unlock;
6580	}
6581
6582	atomic_set(v: &rb->mmap_count, i: 1);
6583	rb->mmap_user = get_current_user();
6584	rb->mmap_locked = extra;
6585
6586	ring_buffer_attach(event, rb);
6587
6588	perf_event_update_time(event);
6589	perf_event_init_userpage(event);
6590	perf_event_update_userpage(event);
6591	} else {
6592	ret = rb_alloc_aux(rb, event, pgoff: vma->vm_pgoff, nr_pages,
6593	watermark: event->attr.aux_watermark, flags);
6594	if (!ret)
6595	rb->aux_mmap_locked = extra;
6596	}
6597
6598	unlock:
6599	if (!ret) {
6600	atomic_long_add(i: user_extra, v: &user->locked_vm);
6601	atomic64_add(i: extra, v: &vma->vm_mm->pinned_vm);
6602
6603	atomic_inc(v: &event->mmap_count);
6604	} else if (rb) {
6605	atomic_dec(v: &rb->mmap_count);
6606	}
6607	aux_unlock:
6608	mutex_unlock(lock: &event->mmap_mutex);
6609
6610	/*
6611	* Since pinned accounting is per vm we cannot allow fork() to copy our
6612	* vma.
6613	*/
6614	vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP);
6615	vma->vm_ops = &perf_mmap_vmops;
6616
6617	if (event->pmu->event_mapped)
6618	event->pmu->event_mapped(event, vma->vm_mm);
6619
6620	return ret;
6621	}
6622
6623	static int perf_fasync(int fd, struct file *filp, int on)
6624	{
6625	struct inode *inode = file_inode(f: filp);
6626	struct perf_event *event = filp->private_data;
6627	int retval;
6628
6629	inode_lock(inode);
6630	retval = fasync_helper(fd, filp, on, &event->fasync);
6631	inode_unlock(inode);
6632
6633	if (retval < 0)
6634	return retval;
6635
6636	return 0;
6637	}
6638
6639	static const struct file_operations perf_fops = {
6640	.llseek = no_llseek,
6641	.release = perf_release,
6642	.read = perf_read,
6643	.poll = perf_poll,
6644	.unlocked_ioctl = perf_ioctl,
6645	.compat_ioctl = perf_compat_ioctl,
6646	.mmap = perf_mmap,
6647	.fasync = perf_fasync,
6648	};
6649
6650	/*
6651	* Perf event wakeup
6652	*
6653	* If there's data, ensure we set the poll() state and publish everything
6654	* to user-space before waking everybody up.
6655	*/
6656
6657	static inline struct fasync_struct **perf_event_fasync(struct perf_event *event)
6658	{
6659	/* only the parent has fasync state */
6660	if (event->parent)
6661	event = event->parent;
6662	return &event->fasync;
6663	}
6664
6665	void perf_event_wakeup(struct perf_event *event)
6666	{
6667	ring_buffer_wakeup(event);
6668
6669	if (event->pending_kill) {
6670	kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);
6671	event->pending_kill = 0;
6672	}
6673	}
6674
6675	static void perf_sigtrap(struct perf_event *event)
6676	{
6677	/*
6678	* We'd expect this to only occur if the irq_work is delayed and either
6679	* ctx->task or current has changed in the meantime. This can be the
6680	* case on architectures that do not implement arch_irq_work_raise().
6681	*/
6682	if (WARN_ON_ONCE(event->ctx->task != current))
6683	return;
6684
6685	/*
6686	* Both perf_pending_task() and perf_pending_irq() can race with the
6687	* task exiting.
6688	*/
6689	if (current->flags & PF_EXITING)
6690	return;
6691
6692	send_sig_perf(addr: (void __user *)event->pending_addr,
6693	type: event->orig_type, sig_data: event->attr.sig_data);
6694	}
6695
6696	/*
6697	* Deliver the pending work in-event-context or follow the context.
6698	*/
6699	static void __perf_pending_irq(struct perf_event *event)
6700	{
6701	int cpu = READ_ONCE(event->oncpu);
6702
6703	/*
6704	* If the event isn't running; we done. event_sched_out() will have
6705	* taken care of things.
6706	*/
6707	if (cpu < 0)
6708	return;
6709
6710	/*
6711	* Yay, we hit home and are in the context of the event.
6712	*/
6713	if (cpu == smp_processor_id()) {
6714	if (event->pending_sigtrap) {
6715	event->pending_sigtrap = 0;
6716	perf_sigtrap(event);
6717	local_dec(l: &event->ctx->nr_pending);
6718	}
6719	if (event->pending_disable) {
6720	event->pending_disable = 0;
6721	perf_event_disable_local(event);
6722	}
6723	return;
6724	}
6725
6726	/*
6727	* CPU-A CPU-B
6728	*
6729	* perf_event_disable_inatomic()
6730	* @pending_disable = CPU-A;
6731	* irq_work_queue();
6732	*
6733	* sched-out
6734	* @pending_disable = -1;
6735	*
6736	* sched-in
6737	* perf_event_disable_inatomic()
6738	* @pending_disable = CPU-B;
6739	* irq_work_queue(); // FAILS
6740	*
6741	* irq_work_run()
6742	* perf_pending_irq()
6743	*
6744	* But the event runs on CPU-B and wants disabling there.
6745	*/
6746	irq_work_queue_on(work: &event->pending_irq, cpu);
6747	}
6748
6749	static void perf_pending_irq(struct irq_work *entry)
6750	{
6751	struct perf_event *event = container_of(entry, struct perf_event, pending_irq);
6752	int rctx;
6753
6754	/*
6755	* If we 'fail' here, that's OK, it means recursion is already disabled
6756	* and we won't recurse 'further'.
6757	*/
6758	rctx = perf_swevent_get_recursion_context();
6759
6760	/*
6761	* The wakeup isn't bound to the context of the event -- it can happen
6762	* irrespective of where the event is.
6763	*/
6764	if (event->pending_wakeup) {
6765	event->pending_wakeup = 0;
6766	perf_event_wakeup(event);
6767	}
6768
6769	__perf_pending_irq(event);
6770
6771	if (rctx >= 0)
6772	perf_swevent_put_recursion_context(rctx);
6773	}
6774
6775	static void perf_pending_task(struct callback_head *head)
6776	{
6777	struct perf_event *event = container_of(head, struct perf_event, pending_task);
6778	int rctx;
6779
6780	/*
6781	* If we 'fail' here, that's OK, it means recursion is already disabled
6782	* and we won't recurse 'further'.
6783	*/
6784	preempt_disable_notrace();
6785	rctx = perf_swevent_get_recursion_context();
6786
6787	if (event->pending_work) {
6788	event->pending_work = 0;
6789	perf_sigtrap(event);
6790	local_dec(l: &event->ctx->nr_pending);
6791	}
6792
6793	if (rctx >= 0)
6794	perf_swevent_put_recursion_context(rctx);
6795	preempt_enable_notrace();
6796
6797	put_event(event);
6798	}
6799
6800	#ifdef CONFIG_GUEST_PERF_EVENTS
6801	struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
6802
6803	DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
6804	DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
6805	DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
6806
6807	void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
6808	{
6809	if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs)))
6810	return;
6811
6812	rcu_assign_pointer(perf_guest_cbs, cbs);
6813	static_call_update(__perf_guest_state, cbs->state);
6814	static_call_update(__perf_guest_get_ip, cbs->get_ip);
6815
6816	/* Implementing ->handle_intel_pt_intr is optional. */
6817	if (cbs->handle_intel_pt_intr)
6818	static_call_update(__perf_guest_handle_intel_pt_intr,
6819	cbs->handle_intel_pt_intr);
6820	}
6821	EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
6822
6823	void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
6824	{
6825	if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs) != cbs))
6826	return;
6827
6828	rcu_assign_pointer(perf_guest_cbs, NULL);
6829	static_call_update(__perf_guest_state, (void *)&__static_call_return0);
6830	static_call_update(__perf_guest_get_ip, (void *)&__static_call_return0);
6831	static_call_update(__perf_guest_handle_intel_pt_intr,
6832	(void *)&__static_call_return0);
6833	synchronize_rcu();
6834	}
6835	EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
6836	#endif
6837
6838	static void
6839	perf_output_sample_regs(struct perf_output_handle *handle,
6840	struct pt_regs *regs, u64 mask)
6841	{
6842	int bit;
6843	DECLARE_BITMAP(_mask, 64);
6844
6845	bitmap_from_u64(dst: _mask, mask);
6846	for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) {
6847	u64 val;
6848
6849	val = perf_reg_value(regs, idx: bit);
6850	perf_output_put(handle, val);
6851	}
6852	}
6853
6854	static void perf_sample_regs_user(struct perf_regs *regs_user,
6855	struct pt_regs *regs)
6856	{
6857	if (user_mode(regs)) {
6858	regs_user->abi = perf_reg_abi(current);
6859	regs_user->regs = regs;
6860	} else if (!(current->flags & PF_KTHREAD)) {
6861	perf_get_regs_user(regs_user, regs);
6862	} else {
6863	regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
6864	regs_user->regs = NULL;
6865	}
6866	}
6867
6868	static void perf_sample_regs_intr(struct perf_regs *regs_intr,
6869	struct pt_regs *regs)
6870	{
6871	regs_intr->regs = regs;
6872	regs_intr->abi = perf_reg_abi(current);
6873	}
6874
6875
6876	/*
6877	* Get remaining task size from user stack pointer.
6878	*
6879	* It'd be better to take stack vma map and limit this more
6880	* precisely, but there's no way to get it safely under interrupt,
6881	* so using TASK_SIZE as limit.
6882	*/
6883	static u64 perf_ustack_task_size(struct pt_regs *regs)
6884	{
6885	unsigned long addr = perf_user_stack_pointer(regs);
6886
6887	if (!addr || addr >= TASK_SIZE)
6888	return 0;
6889
6890	return TASK_SIZE - addr;
6891	}
6892
6893	static u16
6894	perf_sample_ustack_size(u16 stack_size, u16 header_size,
6895	struct pt_regs *regs)
6896	{
6897	u64 task_size;
6898
6899	/* No regs, no stack pointer, no dump. */
6900	if (!regs)
6901	return 0;
6902
6903	/*
6904	* Check if we fit in with the requested stack size into the:
6905	* - TASK_SIZE
6906	* If we don't, we limit the size to the TASK_SIZE.
6907	*
6908	* - remaining sample size
6909	* If we don't, we customize the stack size to
6910	* fit in to the remaining sample size.
6911	*/
6912
6913	task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
6914	stack_size = min(stack_size, (u16) task_size);
6915
6916	/* Current header size plus static size and dynamic size. */
6917	header_size += 2 * sizeof(u64);
6918
6919	/* Do we fit in with the current stack dump size? */
6920	if ((u16) (header_size + stack_size) < header_size) {
6921	/*
6922	* If we overflow the maximum size for the sample,
6923	* we customize the stack dump size to fit in.
6924	*/
6925	stack_size = USHRT_MAX - header_size - sizeof(u64);
6926	stack_size = round_up(stack_size, sizeof(u64));
6927	}
6928
6929	return stack_size;
6930	}
6931
6932	static void
6933	perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
6934	struct pt_regs *regs)
6935	{
6936	/* Case of a kernel thread, nothing to dump */
6937	if (!regs) {
6938	u64 size = 0;
6939	perf_output_put(handle, size);
6940	} else {
6941	unsigned long sp;
6942	unsigned int rem;
6943	u64 dyn_size;
6944
6945	/*
6946	* We dump:
6947	* static size
6948	* - the size requested by user or the best one we can fit
6949	* in to the sample max size
6950	* data
6951	* - user stack dump data
6952	* dynamic size
6953	* - the actual dumped size
6954	*/
6955
6956	/* Static size. */
6957	perf_output_put(handle, dump_size);
6958
6959	/* Data. */
6960	sp = perf_user_stack_pointer(regs);
6961	rem = __output_copy_user(handle, buf: (void *) sp, len: dump_size);
6962	dyn_size = dump_size - rem;
6963
6964	perf_output_skip(handle, len: rem);
6965
6966	/* Dynamic size. */
6967	perf_output_put(handle, dyn_size);
6968	}
6969	}
6970
6971	static unsigned long perf_prepare_sample_aux(struct perf_event *event,
6972	struct perf_sample_data *data,
6973	size_t size)
6974	{
6975	struct perf_event *sampler = event->aux_event;
6976	struct perf_buffer *rb;
6977
6978	data->aux_size = 0;
6979
6980	if (!sampler)
6981	goto out;
6982
6983	if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE))
6984	goto out;
6985
6986	if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id()))
6987	goto out;
6988
6989	rb = ring_buffer_get(event: sampler);
6990	if (!rb)
6991	goto out;
6992
6993	/*
6994	* If this is an NMI hit inside sampling code, don't take
6995	* the sample. See also perf_aux_sample_output().
6996	*/
6997	if (READ_ONCE(rb->aux_in_sampling)) {
6998	data->aux_size = 0;
6999	} else {
7000	size = min_t(size_t, size, perf_aux_size(rb));
7001	data->aux_size = ALIGN(size, sizeof(u64));
7002	}
7003	ring_buffer_put(rb);
7004
7005	out:
7006	return data->aux_size;
7007	}
7008
7009	static long perf_pmu_snapshot_aux(struct perf_buffer *rb,
7010	struct perf_event *event,
7011	struct perf_output_handle *handle,
7012	unsigned long size)
7013	{
7014	unsigned long flags;
7015	long ret;
7016
7017	/*
7018	* Normal ->start()/->stop() callbacks run in IRQ mode in scheduler
7019	* paths. If we start calling them in NMI context, they may race with
7020	* the IRQ ones, that is, for example, re-starting an event that's just
7021	* been stopped, which is why we're using a separate callback that
7022	* doesn't change the event state.
7023	*
7024	* IRQs need to be disabled to prevent IPIs from racing with us.
7025	*/
7026	local_irq_save(flags);
7027	/*
7028	* Guard against NMI hits inside the critical section;
7029	* see also perf_prepare_sample_aux().
7030	*/
7031	WRITE_ONCE(rb->aux_in_sampling, 1);
7032	barrier();
7033
7034	ret = event->pmu->snapshot_aux(event, handle, size);
7035
7036	barrier();
7037	WRITE_ONCE(rb->aux_in_sampling, 0);
7038	local_irq_restore(flags);
7039
7040	return ret;
7041	}
7042
7043	static void perf_aux_sample_output(struct perf_event *event,
7044	struct perf_output_handle *handle,
7045	struct perf_sample_data *data)
7046	{
7047	struct perf_event *sampler = event->aux_event;
7048	struct perf_buffer *rb;
7049	unsigned long pad;
7050	long size;
7051
7052	if (WARN_ON_ONCE(!sampler || !data->aux_size))
7053	return;
7054
7055	rb = ring_buffer_get(event: sampler);
7056	if (!rb)
7057	return;
7058
7059	size = perf_pmu_snapshot_aux(rb, event: sampler, handle, size: data->aux_size);
7060
7061	/*
7062	* An error here means that perf_output_copy() failed (returned a
7063	* non-zero surplus that it didn't copy), which in its current
7064	* enlightened implementation is not possible. If that changes, we'd
7065	* like to know.
7066	*/
7067	if (WARN_ON_ONCE(size < 0))
7068	goto out_put;
7069
7070	/*
7071	* The pad comes from ALIGN()ing data->aux_size up to u64 in
7072	* perf_prepare_sample_aux(), so should not be more than that.
7073	*/
7074	pad = data->aux_size - size;
7075	if (WARN_ON_ONCE(pad >= sizeof(u64)))
7076	pad = 8;
7077
7078	if (pad) {
7079	u64 zero = 0;
7080	perf_output_copy(handle, buf: &zero, len: pad);
7081	}
7082
7083	out_put:
7084	ring_buffer_put(rb);
7085	}
7086
7087	/*
7088	* A set of common sample data types saved even for non-sample records
7089	* when event->attr.sample_id_all is set.
7090	*/
7091	#define PERF_SAMPLE_ID_ALL (PERF_SAMPLE_TID | PERF_SAMPLE_TIME | \
7092	PERF_SAMPLE_ID | PERF_SAMPLE_STREAM_ID | \
7093	PERF_SAMPLE_CPU | PERF_SAMPLE_IDENTIFIER)
7094
7095	static void __perf_event_header__init_id(struct perf_sample_data *data,
7096	struct perf_event *event,
7097	u64 sample_type)
7098	{
7099	data->type = event->attr.sample_type;
7100	data->sample_flags |= data->type & PERF_SAMPLE_ID_ALL;
7101
7102	if (sample_type & PERF_SAMPLE_TID) {
7103	/* namespace issues */
7104	data->tid_entry.pid = perf_event_pid(event, current);
7105	data->tid_entry.tid = perf_event_tid(event, current);
7106	}
7107
7108	if (sample_type & PERF_SAMPLE_TIME)
7109	data->time = perf_event_clock(event);
7110
7111	if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
7112	data->id = primary_event_id(event);
7113
7114	if (sample_type & PERF_SAMPLE_STREAM_ID)
7115	data->stream_id = event->id;
7116
7117	if (sample_type & PERF_SAMPLE_CPU) {
7118	data->cpu_entry.cpu = raw_smp_processor_id();
7119	data->cpu_entry.reserved = 0;
7120	}
7121	}
7122
7123	void perf_event_header__init_id(struct perf_event_header *header,
7124	struct perf_sample_data *data,
7125	struct perf_event *event)
7126	{
7127	if (event->attr.sample_id_all) {
7128	header->size += event->id_header_size;
7129	__perf_event_header__init_id(data, event, sample_type: event->attr.sample_type);
7130	}
7131	}
7132
7133	static void __perf_event__output_id_sample(struct perf_output_handle *handle,
7134	struct perf_sample_data *data)
7135	{
7136	u64 sample_type = data->type;
7137
7138	if (sample_type & PERF_SAMPLE_TID)
7139	perf_output_put(handle, data->tid_entry);
7140
7141	if (sample_type & PERF_SAMPLE_TIME)
7142	perf_output_put(handle, data->time);
7143
7144	if (sample_type & PERF_SAMPLE_ID)
7145	perf_output_put(handle, data->id);
7146
7147	if (sample_type & PERF_SAMPLE_STREAM_ID)
7148	perf_output_put(handle, data->stream_id);
7149
7150	if (sample_type & PERF_SAMPLE_CPU)
7151	perf_output_put(handle, data->cpu_entry);
7152
7153	if (sample_type & PERF_SAMPLE_IDENTIFIER)
7154	perf_output_put(handle, data->id);
7155	}
7156
7157	void perf_event__output_id_sample(struct perf_event *event,
7158	struct perf_output_handle *handle,
7159	struct perf_sample_data *sample)
7160	{
7161	if (event->attr.sample_id_all)
7162	__perf_event__output_id_sample(handle, data: sample);
7163	}
7164
7165	static void perf_output_read_one(struct perf_output_handle *handle,
7166	struct perf_event *event,
7167	u64 enabled, u64 running)
7168	{
7169	u64 read_format = event->attr.read_format;
7170	u64 values[5];
7171	int n = 0;
7172
7173	values[n++] = perf_event_count(event);
7174	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
7175	values[n++] = enabled +
7176	atomic64_read(v: &event->child_total_time_enabled);
7177	}
7178	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
7179	values[n++] = running +
7180	atomic64_read(v: &event->child_total_time_running);
7181	}
7182	if (read_format & PERF_FORMAT_ID)
7183	values[n++] = primary_event_id(event);
7184	if (read_format & PERF_FORMAT_LOST)
7185	values[n++] = atomic64_read(v: &event->lost_samples);
7186
7187	__output_copy(handle, buf: values, len: n * sizeof(u64));
7188	}
7189
7190	static void perf_output_read_group(struct perf_output_handle *handle,
7191	struct perf_event *event,
7192	u64 enabled, u64 running)
7193	{
7194	struct perf_event *leader = event->group_leader, *sub;
7195	u64 read_format = event->attr.read_format;
7196	unsigned long flags;
7197	u64 values[6];
7198	int n = 0;
7199
7200	/*
7201	* Disabling interrupts avoids all counter scheduling
7202	* (context switches, timer based rotation and IPIs).
7203	*/
7204	local_irq_save(flags);
7205
7206	values[n++] = 1 + leader->nr_siblings;
7207
7208	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
7209	values[n++] = enabled;
7210
7211	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
7212	values[n++] = running;
7213
7214	if ((leader != event) &&
7215	(leader->state == PERF_EVENT_STATE_ACTIVE))
7216	leader->pmu->read(leader);
7217
7218	values[n++] = perf_event_count(event: leader);
7219	if (read_format & PERF_FORMAT_ID)
7220	values[n++] = primary_event_id(event: leader);
7221	if (read_format & PERF_FORMAT_LOST)
7222	values[n++] = atomic64_read(v: &leader->lost_samples);
7223
7224	__output_copy(handle, buf: values, len: n * sizeof(u64));
7225
7226	for_each_sibling_event(sub, leader) {
7227	n = 0;
7228
7229	if ((sub != event) &&
7230	(sub->state == PERF_EVENT_STATE_ACTIVE))
7231	sub->pmu->read(sub);
7232
7233	values[n++] = perf_event_count(event: sub);
7234	if (read_format & PERF_FORMAT_ID)
7235	values[n++] = primary_event_id(event: sub);
7236	if (read_format & PERF_FORMAT_LOST)
7237	values[n++] = atomic64_read(v: &sub->lost_samples);
7238
7239	__output_copy(handle, buf: values, len: n * sizeof(u64));
7240	}
7241
7242	local_irq_restore(flags);
7243	}
7244
7245	#define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
7246	PERF_FORMAT_TOTAL_TIME_RUNNING)
7247
7248	/*
7249	* XXX PERF_SAMPLE_READ vs inherited events seems difficult.
7250	*
7251	* The problem is that its both hard and excessively expensive to iterate the
7252	* child list, not to mention that its impossible to IPI the children running
7253	* on another CPU, from interrupt/NMI context.
7254	*/
7255	static void perf_output_read(struct perf_output_handle *handle,
7256	struct perf_event *event)
7257	{
7258	u64 enabled = 0, running = 0, now;
7259	u64 read_format = event->attr.read_format;
7260
7261	/*
7262	* compute total_time_enabled, total_time_running
7263	* based on snapshot values taken when the event
7264	* was last scheduled in.
7265	*
7266	* we cannot simply called update_context_time()
7267	* because of locking issue as we are called in
7268	* NMI context
7269	*/
7270	if (read_format & PERF_FORMAT_TOTAL_TIMES)
7271	calc_timer_values(event, now: &now, enabled: &enabled, running: &running);
7272
7273	if (event->attr.read_format & PERF_FORMAT_GROUP)
7274	perf_output_read_group(handle, event, enabled, running);
7275	else
7276	perf_output_read_one(handle, event, enabled, running);
7277	}
7278
7279	void perf_output_sample(struct perf_output_handle *handle,
7280	struct perf_event_header *header,
7281	struct perf_sample_data *data,
7282	struct perf_event *event)
7283	{
7284	u64 sample_type = data->type;
7285
7286	perf_output_put(handle, *header);
7287
7288	if (sample_type & PERF_SAMPLE_IDENTIFIER)
7289	perf_output_put(handle, data->id);
7290
7291	if (sample_type & PERF_SAMPLE_IP)
7292	perf_output_put(handle, data->ip);
7293
7294	if (sample_type & PERF_SAMPLE_TID)
7295	perf_output_put(handle, data->tid_entry);
7296
7297	if (sample_type & PERF_SAMPLE_TIME)
7298	perf_output_put(handle, data->time);
7299
7300	if (sample_type & PERF_SAMPLE_ADDR)
7301	perf_output_put(handle, data->addr);
7302
7303	if (sample_type & PERF_SAMPLE_ID)
7304	perf_output_put(handle, data->id);
7305
7306	if (sample_type & PERF_SAMPLE_STREAM_ID)
7307	perf_output_put(handle, data->stream_id);
7308
7309	if (sample_type & PERF_SAMPLE_CPU)
7310	perf_output_put(handle, data->cpu_entry);
7311
7312	if (sample_type & PERF_SAMPLE_PERIOD)
7313	perf_output_put(handle, data->period);
7314
7315	if (sample_type & PERF_SAMPLE_READ)
7316	perf_output_read(handle, event);
7317
7318	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
7319	int size = 1;
7320
7321	size += data->callchain->nr;
7322	size *= sizeof(u64);
7323	__output_copy(handle, buf: data->callchain, len: size);
7324	}
7325
7326	if (sample_type & PERF_SAMPLE_RAW) {
7327	struct perf_raw_record *raw = data->raw;
7328
7329	if (raw) {
7330	struct perf_raw_frag *frag = &raw->frag;
7331
7332	perf_output_put(handle, raw->size);
7333	do {
7334	if (frag->copy) {
7335	__output_custom(handle, copy_func: frag->copy,
7336	buf: frag->data, len: frag->size);
7337	} else {
7338	__output_copy(handle, buf: frag->data,
7339	len: frag->size);
7340	}
7341	if (perf_raw_frag_last(frag))
7342	break;
7343	frag = frag->next;
7344	} while (1);
7345	if (frag->pad)
7346	__output_skip(handle, NULL, len: frag->pad);
7347	} else {
7348	struct {
7349	u32 size;
7350	u32 data;
7351	} raw = {
7352	.size = sizeof(u32),
7353	.data = 0,
7354	};
7355	perf_output_put(handle, raw);
7356	}
7357	}
7358
7359	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
7360	if (data->br_stack) {
7361	size_t size;
7362
7363	size = data->br_stack->nr
7364	* sizeof(struct perf_branch_entry);
7365
7366	perf_output_put(handle, data->br_stack->nr);
7367	if (branch_sample_hw_index(event))
7368	perf_output_put(handle, data->br_stack->hw_idx);
7369	perf_output_copy(handle, buf: data->br_stack->entries, len: size);
7370	} else {
7371	/*
7372	* we always store at least the value of nr
7373	*/
7374	u64 nr = 0;
7375	perf_output_put(handle, nr);
7376	}
7377	}
7378
7379	if (sample_type & PERF_SAMPLE_REGS_USER) {
7380	u64 abi = data->regs_user.abi;
7381
7382	/*
7383	* If there are no regs to dump, notice it through
7384	* first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
7385	*/
7386	perf_output_put(handle, abi);
7387
7388	if (abi) {
7389	u64 mask = event->attr.sample_regs_user;
7390	perf_output_sample_regs(handle,
7391	regs: data->regs_user.regs,
7392	mask);
7393	}
7394	}
7395
7396	if (sample_type & PERF_SAMPLE_STACK_USER) {
7397	perf_output_sample_ustack(handle,
7398	dump_size: data->stack_user_size,
7399	regs: data->regs_user.regs);
7400	}
7401
7402	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
7403	perf_output_put(handle, data->weight.full);
7404
7405	if (sample_type & PERF_SAMPLE_DATA_SRC)
7406	perf_output_put(handle, data->data_src.val);
7407
7408	if (sample_type & PERF_SAMPLE_TRANSACTION)
7409	perf_output_put(handle, data->txn);
7410
7411	if (sample_type & PERF_SAMPLE_REGS_INTR) {
7412	u64 abi = data->regs_intr.abi;
7413	/*
7414	* If there are no regs to dump, notice it through
7415	* first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
7416	*/
7417	perf_output_put(handle, abi);
7418
7419	if (abi) {
7420	u64 mask = event->attr.sample_regs_intr;
7421
7422	perf_output_sample_regs(handle,
7423	regs: data->regs_intr.regs,
7424	mask);
7425	}
7426	}
7427
7428	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
7429	perf_output_put(handle, data->phys_addr);
7430
7431	if (sample_type & PERF_SAMPLE_CGROUP)
7432	perf_output_put(handle, data->cgroup);
7433
7434	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
7435	perf_output_put(handle, data->data_page_size);
7436
7437	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
7438	perf_output_put(handle, data->code_page_size);
7439
7440	if (sample_type & PERF_SAMPLE_AUX) {
7441	perf_output_put(handle, data->aux_size);
7442
7443	if (data->aux_size)
7444	perf_aux_sample_output(event, handle, data);
7445	}
7446
7447	if (!event->attr.watermark) {
7448	int wakeup_events = event->attr.wakeup_events;
7449
7450	if (wakeup_events) {
7451	struct perf_buffer *rb = handle->rb;
7452	int events = local_inc_return(&rb->events);
7453
7454	if (events >= wakeup_events) {
7455	local_sub(i: wakeup_events, l: &rb->events);
7456	local_inc(l: &rb->wakeup);
7457	}
7458	}
7459	}
7460	}
7461
7462	static u64 perf_virt_to_phys(u64 virt)
7463	{
7464	u64 phys_addr = 0;
7465
7466	if (!virt)
7467	return 0;
7468
7469	if (virt >= TASK_SIZE) {
7470	/* If it's vmalloc()d memory, leave phys_addr as 0 */
7471	if (virt_addr_valid((void *)(uintptr_t)virt) &&
7472	!(virt >= VMALLOC_START && virt < VMALLOC_END))
7473	phys_addr = (u64)virt_to_phys(address: (void *)(uintptr_t)virt);
7474	} else {
7475	/*
7476	* Walking the pages tables for user address.
7477	* Interrupts are disabled, so it prevents any tear down
7478	* of the page tables.
7479	* Try IRQ-safe get_user_page_fast_only first.
7480	* If failed, leave phys_addr as 0.
7481	*/
7482	if (current->mm != NULL) {
7483	struct page *p;
7484
7485	pagefault_disable();
7486	if (get_user_page_fast_only(addr: virt, gup_flags: 0, pagep: &p)) {
7487	phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
7488	put_page(page: p);
7489	}
7490	pagefault_enable();
7491	}
7492	}
7493
7494	return phys_addr;
7495	}
7496
7497	/*
7498	* Return the pagetable size of a given virtual address.
7499	*/
7500	static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr)
7501	{
7502	u64 size = 0;
7503
7504	#ifdef CONFIG_HAVE_FAST_GUP
7505	pgd_t *pgdp, pgd;
7506	p4d_t *p4dp, p4d;
7507	pud_t *pudp, pud;
7508	pmd_t *pmdp, pmd;
7509	pte_t *ptep, pte;
7510
7511	pgdp = pgd_offset(mm, addr);
7512	pgd = READ_ONCE(*pgdp);
7513	if (pgd_none(pgd))
7514	return 0;
7515
7516	if (pgd_leaf(pgd))
7517	return pgd_leaf_size(pgd);
7518
7519	p4dp = p4d_offset_lockless(pgdp, pgd, addr);
7520	p4d = READ_ONCE(*p4dp);
7521	if (!p4d_present(p4d))
7522	return 0;
7523
7524	if (p4d_leaf(p4d))
7525	return p4d_leaf_size(p4d);
7526
7527	pudp = pud_offset_lockless(p4dp, p4d, addr);
7528	pud = READ_ONCE(*pudp);
7529	if (!pud_present(pud))
7530	return 0;
7531
7532	if (pud_leaf(pud))
7533	return pud_leaf_size(pud);
7534
7535	pmdp = pmd_offset_lockless(pudp, pud, addr);
7536	again:
7537	pmd = pmdp_get_lockless(pmdp);
7538	if (!pmd_present(pmd))
7539	return 0;
7540
7541	if (pmd_leaf(pte: pmd))
7542	return pmd_leaf_size(pmd);
7543
7544	ptep = pte_offset_map(pmd: &pmd, addr);
7545	if (!ptep)
7546	goto again;
7547
7548	pte = ptep_get_lockless(ptep);
7549	if (pte_present(a: pte))
7550	size = pte_leaf_size(pte);
7551	pte_unmap(pte: ptep);
7552	#endif /* CONFIG_HAVE_FAST_GUP */
7553
7554	return size;
7555	}
7556
7557	static u64 perf_get_page_size(unsigned long addr)
7558	{
7559	struct mm_struct *mm;
7560	unsigned long flags;
7561	u64 size;
7562
7563	if (!addr)
7564	return 0;
7565
7566	/*
7567	* Software page-table walkers must disable IRQs,
7568	* which prevents any tear down of the page tables.
7569	*/
7570	local_irq_save(flags);
7571
7572	mm = current->mm;
7573	if (!mm) {
7574	/*
7575	* For kernel threads and the like, use init_mm so that
7576	* we can find kernel memory.
7577	*/
7578	mm = &init_mm;
7579	}
7580
7581	size = perf_get_pgtable_size(mm, addr);
7582
7583	local_irq_restore(flags);
7584
7585	return size;
7586	}
7587
7588	static struct perf_callchain_entry __empty_callchain = { .nr = 0, };
7589
7590	struct perf_callchain_entry *
7591	perf_callchain(struct perf_event *event, struct pt_regs *regs)
7592	{
7593	bool kernel = !event->attr.exclude_callchain_kernel;
7594	bool user = !event->attr.exclude_callchain_user;
7595	/* Disallow cross-task user callchains. */
7596	bool crosstask = event->ctx->task && event->ctx->task != current;
7597	const u32 max_stack = event->attr.sample_max_stack;
7598	struct perf_callchain_entry *callchain;
7599
7600	if (!kernel && !user)
7601	return &__empty_callchain;
7602
7603	callchain = get_perf_callchain(regs, init_nr: 0, kernel, user,
7604	max_stack, crosstask, add_mark: true);
7605	return callchain ?: &__empty_callchain;
7606	}
7607
7608	static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d)
7609	{
7610	return d * !!(flags & s);
7611	}
7612
7613	void perf_prepare_sample(struct perf_sample_data *data,
7614	struct perf_event *event,
7615	struct pt_regs *regs)
7616	{
7617	u64 sample_type = event->attr.sample_type;
7618	u64 filtered_sample_type;
7619
7620	/*
7621	* Add the sample flags that are dependent to others. And clear the
7622	* sample flags that have already been done by the PMU driver.
7623	*/
7624	filtered_sample_type = sample_type;
7625	filtered_sample_type |= __cond_set(flags: sample_type, s: PERF_SAMPLE_CODE_PAGE_SIZE,
7626	d: PERF_SAMPLE_IP);
7627	filtered_sample_type |= __cond_set(flags: sample_type, s: PERF_SAMPLE_DATA_PAGE_SIZE |
7628	PERF_SAMPLE_PHYS_ADDR, d: PERF_SAMPLE_ADDR);
7629	filtered_sample_type |= __cond_set(flags: sample_type, s: PERF_SAMPLE_STACK_USER,
7630	d: PERF_SAMPLE_REGS_USER);
7631	filtered_sample_type &= ~data->sample_flags;
7632
7633	if (filtered_sample_type == 0) {
7634	/* Make sure it has the correct data->type for output */
7635	data->type = event->attr.sample_type;
7636	return;
7637	}
7638
7639	__perf_event_header__init_id(data, event, sample_type: filtered_sample_type);
7640
7641	if (filtered_sample_type & PERF_SAMPLE_IP) {
7642	data->ip = perf_instruction_pointer(regs);
7643	data->sample_flags |= PERF_SAMPLE_IP;
7644	}
7645
7646	if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN)
7647	perf_sample_save_callchain(data, event, regs);
7648
7649	if (filtered_sample_type & PERF_SAMPLE_RAW) {
7650	data->raw = NULL;
7651	data->dyn_size += sizeof(u64);
7652	data->sample_flags |= PERF_SAMPLE_RAW;
7653	}
7654
7655	if (filtered_sample_type & PERF_SAMPLE_BRANCH_STACK) {
7656	data->br_stack = NULL;
7657	data->dyn_size += sizeof(u64);
7658	data->sample_flags |= PERF_SAMPLE_BRANCH_STACK;
7659	}
7660
7661	if (filtered_sample_type & PERF_SAMPLE_REGS_USER)
7662	perf_sample_regs_user(regs_user: &data->regs_user, regs);
7663
7664	/*
7665	* It cannot use the filtered_sample_type here as REGS_USER can be set
7666	* by STACK_USER (using __cond_set() above) and we don't want to update
7667	* the dyn_size if it's not requested by users.
7668	*/
7669	if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) {
7670	/* regs dump ABI info */
7671	int size = sizeof(u64);
7672
7673	if (data->regs_user.regs) {
7674	u64 mask = event->attr.sample_regs_user;
7675	size += hweight64(mask) * sizeof(u64);
7676	}
7677
7678	data->dyn_size += size;
7679	data->sample_flags |= PERF_SAMPLE_REGS_USER;
7680	}
7681
7682	if (filtered_sample_type & PERF_SAMPLE_STACK_USER) {
7683	/*
7684	* Either we need PERF_SAMPLE_STACK_USER bit to be always
7685	* processed as the last one or have additional check added
7686	* in case new sample type is added, because we could eat
7687	* up the rest of the sample size.
7688	*/
7689	u16 stack_size = event->attr.sample_stack_user;
7690	u16 header_size = perf_sample_data_size(data, event);
7691	u16 size = sizeof(u64);
7692
7693	stack_size = perf_sample_ustack_size(stack_size, header_size,
7694	regs: data->regs_user.regs);
7695
7696	/*
7697	* If there is something to dump, add space for the dump
7698	* itself and for the field that tells the dynamic size,
7699	* which is how many have been actually dumped.
7700	*/
7701	if (stack_size)
7702	size += sizeof(u64) + stack_size;
7703
7704	data->stack_user_size = stack_size;
7705	data->dyn_size += size;
7706	data->sample_flags |= PERF_SAMPLE_STACK_USER;
7707	}
7708
7709	if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE) {
7710	data->weight.full = 0;
7711	data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;
7712	}
7713
7714	if (filtered_sample_type & PERF_SAMPLE_DATA_SRC) {
7715	data->data_src.val = PERF_MEM_NA;
7716	data->sample_flags |= PERF_SAMPLE_DATA_SRC;
7717	}
7718
7719	if (filtered_sample_type & PERF_SAMPLE_TRANSACTION) {
7720	data->txn = 0;
7721	data->sample_flags |= PERF_SAMPLE_TRANSACTION;
7722	}
7723
7724	if (filtered_sample_type & PERF_SAMPLE_ADDR) {
7725	data->addr = 0;
7726	data->sample_flags |= PERF_SAMPLE_ADDR;
7727	}
7728
7729	if (filtered_sample_type & PERF_SAMPLE_REGS_INTR) {
7730	/* regs dump ABI info */
7731	int size = sizeof(u64);
7732
7733	perf_sample_regs_intr(regs_intr: &data->regs_intr, regs);
7734
7735	if (data->regs_intr.regs) {
7736	u64 mask = event->attr.sample_regs_intr;
7737
7738	size += hweight64(mask) * sizeof(u64);
7739	}
7740
7741	data->dyn_size += size;
7742	data->sample_flags |= PERF_SAMPLE_REGS_INTR;
7743	}
7744
7745	if (filtered_sample_type & PERF_SAMPLE_PHYS_ADDR) {
7746	data->phys_addr = perf_virt_to_phys(virt: data->addr);
7747	data->sample_flags |= PERF_SAMPLE_PHYS_ADDR;
7748	}
7749
7750	#ifdef CONFIG_CGROUP_PERF
7751	if (filtered_sample_type & PERF_SAMPLE_CGROUP) {
7752	struct cgroup *cgrp;
7753
7754	/* protected by RCU */
7755	cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup;
7756	data->cgroup = cgroup_id(cgrp);
7757	data->sample_flags |= PERF_SAMPLE_CGROUP;
7758	}
7759	#endif
7760
7761	/*
7762	* PERF_DATA_PAGE_SIZE requires PERF_SAMPLE_ADDR. If the user doesn't
7763	* require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr,
7764	* but the value will not dump to the userspace.
7765	*/
7766	if (filtered_sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) {
7767	data->data_page_size = perf_get_page_size(addr: data->addr);
7768	data->sample_flags |= PERF_SAMPLE_DATA_PAGE_SIZE;
7769	}
7770
7771	if (filtered_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) {
7772	data->code_page_size = perf_get_page_size(addr: data->ip);
7773	data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE;
7774	}
7775
7776	if (filtered_sample_type & PERF_SAMPLE_AUX) {
7777	u64 size;
7778	u16 header_size = perf_sample_data_size(data, event);
7779
7780	header_size += sizeof(u64); /* size */
7781
7782	/*
7783	* Given the 16bit nature of header::size, an AUX sample can
7784	* easily overflow it, what with all the preceding sample bits.
7785	* Make sure this doesn't happen by using up to U16_MAX bytes
7786	* per sample in total (rounded down to 8 byte boundary).
7787	*/
7788	size = min_t(size_t, U16_MAX - header_size,
7789	event->attr.aux_sample_size);
7790	size = rounddown(size, 8);
7791	size = perf_prepare_sample_aux(event, data, size);
7792
7793	WARN_ON_ONCE(size + header_size > U16_MAX);
7794	data->dyn_size += size + sizeof(u64); /* size above */
7795	data->sample_flags |= PERF_SAMPLE_AUX;
7796	}
7797	}
7798
7799	void perf_prepare_header(struct perf_event_header *header,
7800	struct perf_sample_data *data,
7801	struct perf_event *event,
7802	struct pt_regs *regs)
7803	{
7804	header->type = PERF_RECORD_SAMPLE;
7805	header->size = perf_sample_data_size(data, event);
7806	header->misc = perf_misc_flags(regs);
7807
7808	/*
7809	* If you're adding more sample types here, you likely need to do
7810	* something about the overflowing header::size, like repurpose the
7811	* lowest 3 bits of size, which should be always zero at the moment.
7812	* This raises a more important question, do we really need 512k sized
7813	* samples and why, so good argumentation is in order for whatever you
7814	* do here next.
7815	*/
7816	WARN_ON_ONCE(header->size & 7);
7817	}
7818
7819	static __always_inline int
7820	__perf_event_output(struct perf_event *event,
7821	struct perf_sample_data *data,
7822	struct pt_regs *regs,
7823	int (*output_begin)(struct perf_output_handle *,
7824	struct perf_sample_data *,
7825	struct perf_event *,
7826	unsigned int))
7827	{
7828	struct perf_output_handle handle;
7829	struct perf_event_header header;
7830	int err;
7831
7832	/* protect the callchain buffers */
7833	rcu_read_lock();
7834
7835	perf_prepare_sample(data, event, regs);
7836	perf_prepare_header(header: &header, data, event, regs);
7837
7838	err = output_begin(&handle, data, event, header.size);
7839	if (err)
7840	goto exit;
7841
7842	perf_output_sample(handle: &handle, header: &header, data, event);
7843
7844	perf_output_end(handle: &handle);
7845
7846	exit:
7847	rcu_read_unlock();
7848	return err;
7849	}
7850
7851	void
7852	perf_event_output_forward(struct perf_event *event,
7853	struct perf_sample_data *data,
7854	struct pt_regs *regs)
7855	{
7856	__perf_event_output(event, data, regs, output_begin: perf_output_begin_forward);
7857	}
7858
7859	void
7860	perf_event_output_backward(struct perf_event *event,
7861	struct perf_sample_data *data,
7862	struct pt_regs *regs)
7863	{
7864	__perf_event_output(event, data, regs, output_begin: perf_output_begin_backward);
7865	}
7866
7867	int
7868	perf_event_output(struct perf_event *event,
7869	struct perf_sample_data *data,
7870	struct pt_regs *regs)
7871	{
7872	return __perf_event_output(event, data, regs, output_begin: perf_output_begin);
7873	}
7874
7875	/*
7876	* read event_id
7877	*/
7878
7879	struct perf_read_event {
7880	struct perf_event_header header;
7881
7882	u32 pid;
7883	u32 tid;
7884	};
7885
7886	static void
7887	perf_event_read_event(struct perf_event *event,
7888	struct task_struct *task)
7889	{
7890	struct perf_output_handle handle;
7891	struct perf_sample_data sample;
7892	struct perf_read_event read_event = {
7893	.header = {
7894	.type = PERF_RECORD_READ,
7895	.misc = 0,
7896	.size = sizeof(read_event) + event->read_size,
7897	},
7898	.pid = perf_event_pid(event, p: task),
7899	.tid = perf_event_tid(event, p: task),
7900	};
7901	int ret;
7902
7903	perf_event_header__init_id(header: &read_event.header, data: &sample, event);
7904	ret = perf_output_begin(handle: &handle, data: &sample, event, size: read_event.header.size);
7905	if (ret)
7906	return;
7907
7908	perf_output_put(&handle, read_event);
7909	perf_output_read(handle: &handle, event);
7910	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
7911
7912	perf_output_end(handle: &handle);
7913	}
7914
7915	typedef void (perf_iterate_f)(struct perf_event *event, void *data);
7916
7917	static void
7918	perf_iterate_ctx(struct perf_event_context *ctx,
7919	perf_iterate_f output,
7920	void *data, bool all)
7921	{
7922	struct perf_event *event;
7923
7924	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
7925	if (!all) {
7926	if (event->state < PERF_EVENT_STATE_INACTIVE)
7927	continue;
7928	if (!event_filter_match(event))
7929	continue;
7930	}
7931
7932	output(event, data);
7933	}
7934	}
7935
7936	static void perf_iterate_sb_cpu(perf_iterate_f output, void *data)
7937	{
7938	struct pmu_event_list *pel = this_cpu_ptr(&pmu_sb_events);
7939	struct perf_event *event;
7940
7941	list_for_each_entry_rcu(event, &pel->list, sb_list) {
7942	/*
7943	* Skip events that are not fully formed yet; ensure that
7944	* if we observe event->ctx, both event and ctx will be
7945	* complete enough. See perf_install_in_context().
7946	*/
7947	if (!smp_load_acquire(&event->ctx))
7948	continue;
7949
7950	if (event->state < PERF_EVENT_STATE_INACTIVE)
7951	continue;
7952	if (!event_filter_match(event))
7953	continue;
7954	output(event, data);
7955	}
7956	}
7957
7958	/*
7959	* Iterate all events that need to receive side-band events.
7960	*
7961	* For new callers; ensure that account_pmu_sb_event() includes
7962	* your event, otherwise it might not get delivered.
7963	*/
7964	static void
7965	perf_iterate_sb(perf_iterate_f output, void *data,
7966	struct perf_event_context *task_ctx)
7967	{
7968	struct perf_event_context *ctx;
7969
7970	rcu_read_lock();
7971	preempt_disable();
7972
7973	/*
7974	* If we have task_ctx != NULL we only notify the task context itself.
7975	* The task_ctx is set only for EXIT events before releasing task
7976	* context.
7977	*/
7978	if (task_ctx) {
7979	perf_iterate_ctx(ctx: task_ctx, output, data, all: false);
7980	goto done;
7981	}
7982
7983	perf_iterate_sb_cpu(output, data);
7984
7985	ctx = rcu_dereference(current->perf_event_ctxp);
7986	if (ctx)
7987	perf_iterate_ctx(ctx, output, data, all: false);
7988	done:
7989	preempt_enable();
7990	rcu_read_unlock();
7991	}
7992
7993	/*
7994	* Clear all file-based filters at exec, they'll have to be
7995	* re-instated when/if these objects are mmapped again.
7996	*/
7997	static void perf_event_addr_filters_exec(struct perf_event *event, void *data)
7998	{
7999	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
8000	struct perf_addr_filter *filter;
8001	unsigned int restart = 0, count = 0;
8002	unsigned long flags;
8003
8004	if (!has_addr_filter(event))
8005	return;
8006
8007	raw_spin_lock_irqsave(&ifh->lock, flags);
8008	list_for_each_entry(filter, &ifh->list, entry) {
8009	if (filter->path.dentry) {
8010	event->addr_filter_ranges[count].start = 0;
8011	event->addr_filter_ranges[count].size = 0;
8012	restart++;
8013	}
8014
8015	count++;
8016	}
8017
8018	if (restart)
8019	event->addr_filters_gen++;
8020	raw_spin_unlock_irqrestore(&ifh->lock, flags);
8021
8022	if (restart)
8023	perf_event_stop(event, restart: 1);
8024	}
8025
8026	void perf_event_exec(void)
8027	{
8028	struct perf_event_context *ctx;
8029
8030	ctx = perf_pin_task_context(current);
8031	if (!ctx)
8032	return;
8033
8034	perf_event_enable_on_exec(ctx);
8035	perf_event_remove_on_exec(ctx);
8036	perf_iterate_ctx(ctx, output: perf_event_addr_filters_exec, NULL, all: true);
8037
8038	perf_unpin_context(ctx);
8039	put_ctx(ctx);
8040	}
8041
8042	struct remote_output {
8043	struct perf_buffer *rb;
8044	int err;
8045	};
8046
8047	static void __perf_event_output_stop(struct perf_event *event, void *data)
8048	{
8049	struct perf_event *parent = event->parent;
8050	struct remote_output *ro = data;
8051	struct perf_buffer *rb = ro->rb;
8052	struct stop_event_data sd = {
8053	.event = event,
8054	};
8055
8056	if (!has_aux(event))
8057	return;
8058
8059	if (!parent)
8060	parent = event;
8061
8062	/*
8063	* In case of inheritance, it will be the parent that links to the
8064	* ring-buffer, but it will be the child that's actually using it.
8065	*
8066	* We are using event::rb to determine if the event should be stopped,
8067	* however this may race with ring_buffer_attach() (through set_output),
8068	* which will make us skip the event that actually needs to be stopped.
8069	* So ring_buffer_attach() has to stop an aux event before re-assigning
8070	* its rb pointer.
8071	*/
8072	if (rcu_dereference(parent->rb) == rb)
8073	ro->err = __perf_event_stop(info: &sd);
8074	}
8075
8076	static int __perf_pmu_output_stop(void *info)
8077	{
8078	struct perf_event *event = info;
8079	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
8080	struct remote_output ro = {
8081	.rb = event->rb,
8082	};
8083
8084	rcu_read_lock();
8085	perf_iterate_ctx(ctx: &cpuctx->ctx, output: __perf_event_output_stop, data: &ro, all: false);
8086	if (cpuctx->task_ctx)
8087	perf_iterate_ctx(ctx: cpuctx->task_ctx, output: __perf_event_output_stop,
8088	data: &ro, all: false);
8089	rcu_read_unlock();
8090
8091	return ro.err;
8092	}
8093
8094	static void perf_pmu_output_stop(struct perf_event *event)
8095	{
8096	struct perf_event *iter;
8097	int err, cpu;
8098
8099	restart:
8100	rcu_read_lock();
8101	list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) {
8102	/*
8103	* For per-CPU events, we need to make sure that neither they
8104	* nor their children are running; for cpu==-1 events it's
8105	* sufficient to stop the event itself if it's active, since
8106	* it can't have children.
8107	*/
8108	cpu = iter->cpu;
8109	if (cpu == -1)
8110	cpu = READ_ONCE(iter->oncpu);
8111
8112	if (cpu == -1)
8113	continue;
8114
8115	err = cpu_function_call(cpu, func: __perf_pmu_output_stop, info: event);
8116	if (err == -EAGAIN) {
8117	rcu_read_unlock();
8118	goto restart;
8119	}
8120	}
8121	rcu_read_unlock();
8122	}
8123
8124	/*
8125	* task tracking -- fork/exit
8126	*
8127	* enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
8128	*/
8129
8130	struct perf_task_event {
8131	struct task_struct *task;
8132	struct perf_event_context *task_ctx;
8133
8134	struct {
8135	struct perf_event_header header;
8136
8137	u32 pid;
8138	u32 ppid;
8139	u32 tid;
8140	u32 ptid;
8141	u64 time;
8142	} event_id;
8143	};
8144
8145	static int perf_event_task_match(struct perf_event *event)
8146	{
8147	return event->attr.comm || event->attr.mmap ||
8148	event->attr.mmap2 || event->attr.mmap_data ||
8149	event->attr.task;
8150	}
8151
8152	static void perf_event_task_output(struct perf_event *event,
8153	void *data)
8154	{
8155	struct perf_task_event *task_event = data;
8156	struct perf_output_handle handle;
8157	struct perf_sample_data sample;
8158	struct task_struct *task = task_event->task;
8159	int ret, size = task_event->event_id.header.size;
8160
8161	if (!perf_event_task_match(event))
8162	return;
8163
8164	perf_event_header__init_id(header: &task_event->event_id.header, data: &sample, event);
8165
8166	ret = perf_output_begin(handle: &handle, data: &sample, event,
8167	size: task_event->event_id.header.size);
8168	if (ret)
8169	goto out;
8170
8171	task_event->event_id.pid = perf_event_pid(event, p: task);
8172	task_event->event_id.tid = perf_event_tid(event, p: task);
8173
8174	if (task_event->event_id.header.type == PERF_RECORD_EXIT) {
8175	task_event->event_id.ppid = perf_event_pid(event,
8176	p: task->real_parent);
8177	task_event->event_id.ptid = perf_event_pid(event,
8178	p: task->real_parent);
8179	} else { /* PERF_RECORD_FORK */
8180	task_event->event_id.ppid = perf_event_pid(event, current);
8181	task_event->event_id.ptid = perf_event_tid(event, current);
8182	}
8183
8184	task_event->event_id.time = perf_event_clock(event);
8185
8186	perf_output_put(&handle, task_event->event_id);
8187
8188	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
8189
8190	perf_output_end(handle: &handle);
8191	out:
8192	task_event->event_id.header.size = size;
8193	}
8194
8195	static void perf_event_task(struct task_struct *task,
8196	struct perf_event_context *task_ctx,
8197	int new)
8198	{
8199	struct perf_task_event task_event;
8200
8201	if (!atomic_read(v: &nr_comm_events) &&
8202	!atomic_read(v: &nr_mmap_events) &&
8203	!atomic_read(v: &nr_task_events))
8204	return;
8205
8206	task_event = (struct perf_task_event){
8207	.task = task,
8208	.task_ctx = task_ctx,
8209	.event_id = {
8210	.header = {
8211	.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
8212	.misc = 0,
8213	.size = sizeof(task_event.event_id),
8214	},
8215	/* .pid */
8216	/* .ppid */
8217	/* .tid */
8218	/* .ptid */
8219	/* .time */
8220	},
8221	};
8222
8223	perf_iterate_sb(output: perf_event_task_output,
8224	data: &task_event,
8225	task_ctx);
8226	}
8227
8228	void perf_event_fork(struct task_struct *task)
8229	{
8230	perf_event_task(task, NULL, new: 1);
8231	perf_event_namespaces(tsk: task);
8232	}
8233
8234	/*
8235	* comm tracking
8236	*/
8237
8238	struct perf_comm_event {
8239	struct task_struct *task;
8240	char *comm;
8241	int comm_size;
8242
8243	struct {
8244	struct perf_event_header header;
8245
8246	u32 pid;
8247	u32 tid;
8248	} event_id;
8249	};
8250
8251	static int perf_event_comm_match(struct perf_event *event)
8252	{
8253	return event->attr.comm;
8254	}
8255
8256	static void perf_event_comm_output(struct perf_event *event,
8257	void *data)
8258	{
8259	struct perf_comm_event *comm_event = data;
8260	struct perf_output_handle handle;
8261	struct perf_sample_data sample;
8262	int size = comm_event->event_id.header.size;
8263	int ret;
8264
8265	if (!perf_event_comm_match(event))
8266	return;
8267
8268	perf_event_header__init_id(header: &comm_event->event_id.header, data: &sample, event);
8269	ret = perf_output_begin(handle: &handle, data: &sample, event,
8270	size: comm_event->event_id.header.size);
8271
8272	if (ret)
8273	goto out;
8274
8275	comm_event->event_id.pid = perf_event_pid(event, p: comm_event->task);
8276	comm_event->event_id.tid = perf_event_tid(event, p: comm_event->task);
8277
8278	perf_output_put(&handle, comm_event->event_id);
8279	__output_copy(handle: &handle, buf: comm_event->comm,
8280	len: comm_event->comm_size);
8281
8282	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
8283
8284	perf_output_end(handle: &handle);
8285	out:
8286	comm_event->event_id.header.size = size;
8287	}
8288
8289	static void perf_event_comm_event(struct perf_comm_event *comm_event)
8290	{
8291	char comm[TASK_COMM_LEN];
8292	unsigned int size;
8293
8294	memset(comm, 0, sizeof(comm));
8295	strscpy(p: comm, q: comm_event->task->comm, size: sizeof(comm));
8296	size = ALIGN(strlen(comm)+1, sizeof(u64));
8297
8298	comm_event->comm = comm;
8299	comm_event->comm_size = size;
8300
8301	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
8302
8303	perf_iterate_sb(output: perf_event_comm_output,
8304	data: comm_event,
8305	NULL);
8306	}
8307
8308	void perf_event_comm(struct task_struct *task, bool exec)
8309	{
8310	struct perf_comm_event comm_event;
8311
8312	if (!atomic_read(v: &nr_comm_events))
8313	return;
8314
8315	comm_event = (struct perf_comm_event){
8316	.task = task,
8317	/* .comm */
8318	/* .comm_size */
8319	.event_id = {
8320	.header = {
8321	.type = PERF_RECORD_COMM,
8322	.misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
8323	/* .size */
8324	},
8325	/* .pid */
8326	/* .tid */
8327	},
8328	};
8329
8330	perf_event_comm_event(comm_event: &comm_event);
8331	}
8332
8333	/*
8334	* namespaces tracking
8335	*/
8336
8337	struct perf_namespaces_event {
8338	struct task_struct *task;
8339
8340	struct {
8341	struct perf_event_header header;
8342
8343	u32 pid;
8344	u32 tid;
8345	u64 nr_namespaces;
8346	struct perf_ns_link_info link_info[NR_NAMESPACES];
8347	} event_id;
8348	};
8349
8350	static int perf_event_namespaces_match(struct perf_event *event)
8351	{
8352	return event->attr.namespaces;
8353	}
8354
8355	static void perf_event_namespaces_output(struct perf_event *event,
8356	void *data)
8357	{
8358	struct perf_namespaces_event *namespaces_event = data;
8359	struct perf_output_handle handle;
8360	struct perf_sample_data sample;
8361	u16 header_size = namespaces_event->event_id.header.size;
8362	int ret;
8363
8364	if (!perf_event_namespaces_match(event))
8365	return;
8366
8367	perf_event_header__init_id(header: &namespaces_event->event_id.header,
8368	data: &sample, event);
8369	ret = perf_output_begin(handle: &handle, data: &sample, event,
8370	size: namespaces_event->event_id.header.size);
8371	if (ret)
8372	goto out;
8373
8374	namespaces_event->event_id.pid = perf_event_pid(event,
8375	p: namespaces_event->task);
8376	namespaces_event->event_id.tid = perf_event_tid(event,
8377	p: namespaces_event->task);
8378
8379	perf_output_put(&handle, namespaces_event->event_id);
8380
8381	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
8382
8383	perf_output_end(handle: &handle);
8384	out:
8385	namespaces_event->event_id.header.size = header_size;
8386	}
8387
8388	static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info,
8389	struct task_struct *task,
8390	const struct proc_ns_operations *ns_ops)
8391	{
8392	struct path ns_path;
8393	struct inode *ns_inode;
8394	int error;
8395
8396	error = ns_get_path(path: &ns_path, task, ns_ops);
8397	if (!error) {
8398	ns_inode = ns_path.dentry->d_inode;
8399	ns_link_info->dev = new_encode_dev(dev: ns_inode->i_sb->s_dev);
8400	ns_link_info->ino = ns_inode->i_ino;
8401	path_put(&ns_path);
8402	}
8403	}
8404
8405	void perf_event_namespaces(struct task_struct *task)
8406	{
8407	struct perf_namespaces_event namespaces_event;
8408	struct perf_ns_link_info *ns_link_info;
8409
8410	if (!atomic_read(v: &nr_namespaces_events))
8411	return;
8412
8413	namespaces_event = (struct perf_namespaces_event){
8414	.task = task,
8415	.event_id = {
8416	.header = {
8417	.type = PERF_RECORD_NAMESPACES,
8418	.misc = 0,
8419	.size = sizeof(namespaces_event.event_id),
8420	},
8421	/* .pid */
8422	/* .tid */
8423	.nr_namespaces = NR_NAMESPACES,
8424	/* .link_info[NR_NAMESPACES] */
8425	},
8426	};
8427
8428	ns_link_info = namespaces_event.event_id.link_info;
8429
8430	perf_fill_ns_link_info(ns_link_info: &ns_link_info[MNT_NS_INDEX],
8431	task, ns_ops: &mntns_operations);
8432
8433	#ifdef CONFIG_USER_NS
8434	perf_fill_ns_link_info(ns_link_info: &ns_link_info[USER_NS_INDEX],
8435	task, ns_ops: &userns_operations);
8436	#endif
8437	#ifdef CONFIG_NET_NS
8438	perf_fill_ns_link_info(ns_link_info: &ns_link_info[NET_NS_INDEX],
8439	task, ns_ops: &netns_operations);
8440	#endif
8441	#ifdef CONFIG_UTS_NS
8442	perf_fill_ns_link_info(ns_link_info: &ns_link_info[UTS_NS_INDEX],
8443	task, ns_ops: &utsns_operations);
8444	#endif
8445	#ifdef CONFIG_IPC_NS
8446	perf_fill_ns_link_info(ns_link_info: &ns_link_info[IPC_NS_INDEX],
8447	task, ns_ops: &ipcns_operations);
8448	#endif
8449	#ifdef CONFIG_PID_NS
8450	perf_fill_ns_link_info(ns_link_info: &ns_link_info[PID_NS_INDEX],
8451	task, ns_ops: &pidns_operations);
8452	#endif
8453	#ifdef CONFIG_CGROUPS
8454	perf_fill_ns_link_info(ns_link_info: &ns_link_info[CGROUP_NS_INDEX],
8455	task, ns_ops: &cgroupns_operations);
8456	#endif
8457
8458	perf_iterate_sb(output: perf_event_namespaces_output,
8459	data: &namespaces_event,
8460	NULL);
8461	}
8462
8463	/*
8464	* cgroup tracking
8465	*/
8466	#ifdef CONFIG_CGROUP_PERF
8467
8468	struct perf_cgroup_event {
8469	char *path;
8470	int path_size;
8471	struct {
8472	struct perf_event_header header;
8473	u64 id;
8474	char path[];
8475	} event_id;
8476	};
8477
8478	static int perf_event_cgroup_match(struct perf_event *event)
8479	{
8480	return event->attr.cgroup;
8481	}
8482
8483	static void perf_event_cgroup_output(struct perf_event *event, void *data)
8484	{
8485	struct perf_cgroup_event *cgroup_event = data;
8486	struct perf_output_handle handle;
8487	struct perf_sample_data sample;
8488	u16 header_size = cgroup_event->event_id.header.size;
8489	int ret;
8490
8491	if (!perf_event_cgroup_match(event))
8492	return;
8493
8494	perf_event_header__init_id(header: &cgroup_event->event_id.header,
8495	data: &sample, event);
8496	ret = perf_output_begin(handle: &handle, data: &sample, event,
8497	size: cgroup_event->event_id.header.size);
8498	if (ret)
8499	goto out;
8500
8501	perf_output_put(&handle, cgroup_event->event_id);
8502	__output_copy(handle: &handle, buf: cgroup_event->path, len: cgroup_event->path_size);
8503
8504	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
8505
8506	perf_output_end(handle: &handle);
8507	out:
8508	cgroup_event->event_id.header.size = header_size;
8509	}
8510
8511	static void perf_event_cgroup(struct cgroup *cgrp)
8512	{
8513	struct perf_cgroup_event cgroup_event;
8514	char path_enomem[16] = "//enomem";
8515	char *pathname;
8516	size_t size;
8517
8518	if (!atomic_read(v: &nr_cgroup_events))
8519	return;
8520
8521	cgroup_event = (struct perf_cgroup_event){
8522	.event_id = {
8523	.header = {
8524	.type = PERF_RECORD_CGROUP,
8525	.misc = 0,
8526	.size = sizeof(cgroup_event.event_id),
8527	},
8528	.id = cgroup_id(cgrp),
8529	},
8530	};
8531
8532	pathname = kmalloc(PATH_MAX, GFP_KERNEL);
8533	if (pathname == NULL) {
8534	cgroup_event.path = path_enomem;
8535	} else {
8536	/* just to be sure to have enough space for alignment */
8537	cgroup_path(cgrp, buf: pathname, PATH_MAX - sizeof(u64));
8538	cgroup_event.path = pathname;
8539	}
8540
8541	/*
8542	* Since our buffer works in 8 byte units we need to align our string
8543	* size to a multiple of 8. However, we must guarantee the tail end is
8544	* zero'd out to avoid leaking random bits to userspace.
8545	*/
8546	size = strlen(cgroup_event.path) + 1;
8547	while (!IS_ALIGNED(size, sizeof(u64)))
8548	cgroup_event.path[size++] = '\0';
8549
8550	cgroup_event.event_id.header.size += size;
8551	cgroup_event.path_size = size;
8552
8553	perf_iterate_sb(output: perf_event_cgroup_output,
8554	data: &cgroup_event,
8555	NULL);
8556
8557	kfree(objp: pathname);
8558	}
8559
8560	#endif
8561
8562	/*
8563	* mmap tracking
8564	*/
8565
8566	struct perf_mmap_event {
8567	struct vm_area_struct *vma;
8568
8569	const char *file_name;
8570	int file_size;
8571	int maj, min;
8572	u64 ino;
8573	u64 ino_generation;
8574	u32 prot, flags;
8575	u8 build_id[BUILD_ID_SIZE_MAX];
8576	u32 build_id_size;
8577
8578	struct {
8579	struct perf_event_header header;
8580
8581	u32 pid;
8582	u32 tid;
8583	u64 start;
8584	u64 len;
8585	u64 pgoff;
8586	} event_id;
8587	};
8588
8589	static int perf_event_mmap_match(struct perf_event *event,
8590	void *data)
8591	{
8592	struct perf_mmap_event *mmap_event = data;
8593	struct vm_area_struct *vma = mmap_event->vma;
8594	int executable = vma->vm_flags & VM_EXEC;
8595
8596	return (!executable && event->attr.mmap_data) ||
8597	(executable && (event->attr.mmap || event->attr.mmap2));
8598	}
8599
8600	static void perf_event_mmap_output(struct perf_event *event,
8601	void *data)
8602	{
8603	struct perf_mmap_event *mmap_event = data;
8604	struct perf_output_handle handle;
8605	struct perf_sample_data sample;
8606	int size = mmap_event->event_id.header.size;
8607	u32 type = mmap_event->event_id.header.type;
8608	bool use_build_id;
8609	int ret;
8610
8611	if (!perf_event_mmap_match(event, data))
8612	return;
8613
8614	if (event->attr.mmap2) {
8615	mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
8616	mmap_event->event_id.header.size += sizeof(mmap_event->maj);
8617	mmap_event->event_id.header.size += sizeof(mmap_event->min);
8618	mmap_event->event_id.header.size += sizeof(mmap_event->ino);
8619	mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);
8620	mmap_event->event_id.header.size += sizeof(mmap_event->prot);
8621	mmap_event->event_id.header.size += sizeof(mmap_event->flags);
8622	}
8623
8624	perf_event_header__init_id(header: &mmap_event->event_id.header, data: &sample, event);
8625	ret = perf_output_begin(handle: &handle, data: &sample, event,
8626	size: mmap_event->event_id.header.size);
8627	if (ret)
8628	goto out;
8629
8630	mmap_event->event_id.pid = perf_event_pid(event, current);
8631	mmap_event->event_id.tid = perf_event_tid(event, current);
8632
8633	use_build_id = event->attr.build_id && mmap_event->build_id_size;
8634
8635	if (event->attr.mmap2 && use_build_id)
8636	mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID;
8637
8638	perf_output_put(&handle, mmap_event->event_id);
8639
8640	if (event->attr.mmap2) {
8641	if (use_build_id) {
8642	u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 };
8643
8644	__output_copy(handle: &handle, buf: size, len: 4);
8645	__output_copy(handle: &handle, buf: mmap_event->build_id, BUILD_ID_SIZE_MAX);
8646	} else {
8647	perf_output_put(&handle, mmap_event->maj);
8648	perf_output_put(&handle, mmap_event->min);
8649	perf_output_put(&handle, mmap_event->ino);
8650	perf_output_put(&handle, mmap_event->ino_generation);
8651	}
8652	perf_output_put(&handle, mmap_event->prot);
8653	perf_output_put(&handle, mmap_event->flags);
8654	}
8655
8656	__output_copy(handle: &handle, buf: mmap_event->file_name,
8657	len: mmap_event->file_size);
8658
8659	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
8660
8661	perf_output_end(handle: &handle);
8662	out:
8663	mmap_event->event_id.header.size = size;
8664	mmap_event->event_id.header.type = type;
8665	}
8666
8667	static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
8668	{
8669	struct vm_area_struct *vma = mmap_event->vma;
8670	struct file *file = vma->vm_file;
8671	int maj = 0, min = 0;
8672	u64 ino = 0, gen = 0;
8673	u32 prot = 0, flags = 0;
8674	unsigned int size;
8675	char tmp[16];
8676	char *buf = NULL;
8677	char *name = NULL;
8678
8679	if (vma->vm_flags & VM_READ)
8680	prot |= PROT_READ;
8681	if (vma->vm_flags & VM_WRITE)
8682	prot |= PROT_WRITE;
8683	if (vma->vm_flags & VM_EXEC)
8684	prot |= PROT_EXEC;
8685
8686	if (vma->vm_flags & VM_MAYSHARE)
8687	flags = MAP_SHARED;
8688	else
8689	flags = MAP_PRIVATE;
8690
8691	if (vma->vm_flags & VM_LOCKED)
8692	flags |= MAP_LOCKED;
8693	if (is_vm_hugetlb_page(vma))
8694	flags |= MAP_HUGETLB;
8695
8696	if (file) {
8697	struct inode *inode;
8698	dev_t dev;
8699
8700	buf = kmalloc(PATH_MAX, GFP_KERNEL);
8701	if (!buf) {
8702	name = "//enomem";
8703	goto cpy_name;
8704	}
8705	/*
8706	* d_path() works from the end of the rb backwards, so we
8707	* need to add enough zero bytes after the string to handle
8708	* the 64bit alignment we do later.
8709	*/
8710	name = file_path(file, buf, PATH_MAX - sizeof(u64));
8711	if (IS_ERR(ptr: name)) {
8712	name = "//toolong";
8713	goto cpy_name;
8714	}
8715	inode = file_inode(f: vma->vm_file);
8716	dev = inode->i_sb->s_dev;
8717	ino = inode->i_ino;
8718	gen = inode->i_generation;
8719	maj = MAJOR(dev);
8720	min = MINOR(dev);
8721
8722	goto got_name;
8723	} else {
8724	if (vma->vm_ops && vma->vm_ops->name)
8725	name = (char *) vma->vm_ops->name(vma);
8726	if (!name)
8727	name = (char *)arch_vma_name(vma);
8728	if (!name) {
8729	if (vma_is_initial_heap(vma))
8730	name = "[heap]";
8731	else if (vma_is_initial_stack(vma))
8732	name = "[stack]";
8733	else
8734	name = "//anon";
8735	}
8736	}
8737
8738	cpy_name:
8739	strscpy(p: tmp, q: name, size: sizeof(tmp));
8740	name = tmp;
8741	got_name:
8742	/*
8743	* Since our buffer works in 8 byte units we need to align our string
8744	* size to a multiple of 8. However, we must guarantee the tail end is
8745	* zero'd out to avoid leaking random bits to userspace.
8746	*/
8747	size = strlen(name)+1;
8748	while (!IS_ALIGNED(size, sizeof(u64)))
8749	name[size++] = '\0';
8750
8751	mmap_event->file_name = name;
8752	mmap_event->file_size = size;
8753	mmap_event->maj = maj;
8754	mmap_event->min = min;
8755	mmap_event->ino = ino;
8756	mmap_event->ino_generation = gen;
8757	mmap_event->prot = prot;
8758	mmap_event->flags = flags;
8759
8760	if (!(vma->vm_flags & VM_EXEC))
8761	mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
8762
8763	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
8764
8765	if (atomic_read(v: &nr_build_id_events))
8766	build_id_parse(vma, build_id: mmap_event->build_id, size: &mmap_event->build_id_size);
8767
8768	perf_iterate_sb(output: perf_event_mmap_output,
8769	data: mmap_event,
8770	NULL);
8771
8772	kfree(objp: buf);
8773	}
8774
8775	/*
8776	* Check whether inode and address range match filter criteria.
8777	*/
8778	static bool perf_addr_filter_match(struct perf_addr_filter *filter,
8779	struct file *file, unsigned long offset,
8780	unsigned long size)
8781	{
8782	/* d_inode(NULL) won't be equal to any mapped user-space file */
8783	if (!filter->path.dentry)
8784	return false;
8785
8786	if (d_inode(dentry: filter->path.dentry) != file_inode(f: file))
8787	return false;
8788
8789	if (filter->offset > offset + size)
8790	return false;
8791
8792	if (filter->offset + filter->size < offset)
8793	return false;
8794
8795	return true;
8796	}
8797
8798	static bool perf_addr_filter_vma_adjust(struct perf_addr_filter *filter,
8799	struct vm_area_struct *vma,
8800	struct perf_addr_filter_range *fr)
8801	{
8802	unsigned long vma_size = vma->vm_end - vma->vm_start;
8803	unsigned long off = vma->vm_pgoff << PAGE_SHIFT;
8804	struct file *file = vma->vm_file;
8805
8806	if (!perf_addr_filter_match(filter, file, offset: off, size: vma_size))
8807	return false;
8808
8809	if (filter->offset < off) {
8810	fr->start = vma->vm_start;
8811	fr->size = min(vma_size, filter->size - (off - filter->offset));
8812	} else {
8813	fr->start = vma->vm_start + filter->offset - off;
8814	fr->size = min(vma->vm_end - fr->start, filter->size);
8815	}
8816
8817	return true;
8818	}
8819
8820	static void __perf_addr_filters_adjust(struct perf_event *event, void *data)
8821	{
8822	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
8823	struct vm_area_struct *vma = data;
8824	struct perf_addr_filter *filter;
8825	unsigned int restart = 0, count = 0;
8826	unsigned long flags;
8827
8828	if (!has_addr_filter(event))
8829	return;
8830
8831	if (!vma->vm_file)
8832	return;
8833
8834	raw_spin_lock_irqsave(&ifh->lock, flags);
8835	list_for_each_entry(filter, &ifh->list, entry) {
8836	if (perf_addr_filter_vma_adjust(filter, vma,
8837	fr: &event->addr_filter_ranges[count]))
8838	restart++;
8839
8840	count++;
8841	}
8842
8843	if (restart)
8844	event->addr_filters_gen++;
8845	raw_spin_unlock_irqrestore(&ifh->lock, flags);
8846
8847	if (restart)
8848	perf_event_stop(event, restart: 1);
8849	}
8850
8851	/*
8852	* Adjust all task's events' filters to the new vma
8853	*/
8854	static void perf_addr_filters_adjust(struct vm_area_struct *vma)
8855	{
8856	struct perf_event_context *ctx;
8857
8858	/*
8859	* Data tracing isn't supported yet and as such there is no need
8860	* to keep track of anything that isn't related to executable code:
8861	*/
8862	if (!(vma->vm_flags & VM_EXEC))
8863	return;
8864
8865	rcu_read_lock();
8866	ctx = rcu_dereference(current->perf_event_ctxp);
8867	if (ctx)
8868	perf_iterate_ctx(ctx, output: __perf_addr_filters_adjust, data: vma, all: true);
8869	rcu_read_unlock();
8870	}
8871
8872	void perf_event_mmap(struct vm_area_struct *vma)
8873	{
8874	struct perf_mmap_event mmap_event;
8875
8876	if (!atomic_read(v: &nr_mmap_events))
8877	return;
8878
8879	mmap_event = (struct perf_mmap_event){
8880	.vma = vma,
8881	/* .file_name */
8882	/* .file_size */
8883	.event_id = {
8884	.header = {
8885	.type = PERF_RECORD_MMAP,
8886	.misc = PERF_RECORD_MISC_USER,
8887	/* .size */
8888	},
8889	/* .pid */
8890	/* .tid */
8891	.start = vma->vm_start,
8892	.len = vma->vm_end - vma->vm_start,
8893	.pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT,
8894	},
8895	/* .maj (attr_mmap2 only) */
8896	/* .min (attr_mmap2 only) */
8897	/* .ino (attr_mmap2 only) */
8898	/* .ino_generation (attr_mmap2 only) */
8899	/* .prot (attr_mmap2 only) */
8900	/* .flags (attr_mmap2 only) */
8901	};
8902
8903	perf_addr_filters_adjust(vma);
8904	perf_event_mmap_event(mmap_event: &mmap_event);
8905	}
8906
8907	void perf_event_aux_event(struct perf_event *event, unsigned long head,
8908	unsigned long size, u64 flags)
8909	{
8910	struct perf_output_handle handle;
8911	struct perf_sample_data sample;
8912	struct perf_aux_event {
8913	struct perf_event_header header;
8914	u64 offset;
8915	u64 size;
8916	u64 flags;
8917	} rec = {
8918	.header = {
8919	.type = PERF_RECORD_AUX,
8920	.misc = 0,
8921	.size = sizeof(rec),
8922	},
8923	.offset = head,
8924	.size = size,
8925	.flags = flags,
8926	};
8927	int ret;
8928
8929	perf_event_header__init_id(header: &rec.header, data: &sample, event);
8930	ret = perf_output_begin(handle: &handle, data: &sample, event, size: rec.header.size);
8931
8932	if (ret)
8933	return;
8934
8935	perf_output_put(&handle, rec);
8936	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
8937
8938	perf_output_end(handle: &handle);
8939	}
8940
8941	/*
8942	* Lost/dropped samples logging
8943	*/
8944	void perf_log_lost_samples(struct perf_event *event, u64 lost)
8945	{
8946	struct perf_output_handle handle;
8947	struct perf_sample_data sample;
8948	int ret;
8949
8950	struct {
8951	struct perf_event_header header;
8952	u64 lost;
8953	} lost_samples_event = {
8954	.header = {
8955	.type = PERF_RECORD_LOST_SAMPLES,
8956	.misc = 0,
8957	.size = sizeof(lost_samples_event),
8958	},
8959	.lost = lost,
8960	};
8961
8962	perf_event_header__init_id(header: &lost_samples_event.header, data: &sample, event);
8963
8964	ret = perf_output_begin(handle: &handle, data: &sample, event,
8965	size: lost_samples_event.header.size);
8966	if (ret)
8967	return;
8968
8969	perf_output_put(&handle, lost_samples_event);
8970	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
8971	perf_output_end(handle: &handle);
8972	}
8973
8974	/*
8975	* context_switch tracking
8976	*/
8977
8978	struct perf_switch_event {
8979	struct task_struct *task;
8980	struct task_struct *next_prev;
8981
8982	struct {
8983	struct perf_event_header header;
8984	u32 next_prev_pid;
8985	u32 next_prev_tid;
8986	} event_id;
8987	};
8988
8989	static int perf_event_switch_match(struct perf_event *event)
8990	{
8991	return event->attr.context_switch;
8992	}
8993
8994	static void perf_event_switch_output(struct perf_event *event, void *data)
8995	{
8996	struct perf_switch_event *se = data;
8997	struct perf_output_handle handle;
8998	struct perf_sample_data sample;
8999	int ret;
9000
9001	if (!perf_event_switch_match(event))
9002	return;
9003
9004	/* Only CPU-wide events are allowed to see next/prev pid/tid */
9005	if (event->ctx->task) {
9006	se->event_id.header.type = PERF_RECORD_SWITCH;
9007	se->event_id.header.size = sizeof(se->event_id.header);
9008	} else {
9009	se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE;
9010	se->event_id.header.size = sizeof(se->event_id);
9011	se->event_id.next_prev_pid =
9012	perf_event_pid(event, p: se->next_prev);
9013	se->event_id.next_prev_tid =
9014	perf_event_tid(event, p: se->next_prev);
9015	}
9016
9017	perf_event_header__init_id(header: &se->event_id.header, data: &sample, event);
9018
9019	ret = perf_output_begin(handle: &handle, data: &sample, event, size: se->event_id.header.size);
9020	if (ret)
9021	return;
9022
9023	if (event->ctx->task)
9024	perf_output_put(&handle, se->event_id.header);
9025	else
9026	perf_output_put(&handle, se->event_id);
9027
9028	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
9029
9030	perf_output_end(handle: &handle);
9031	}
9032
9033	static void perf_event_switch(struct task_struct *task,
9034	struct task_struct *next_prev, bool sched_in)
9035	{
9036	struct perf_switch_event switch_event;
9037
9038	/* N.B. caller checks nr_switch_events != 0 */
9039
9040	switch_event = (struct perf_switch_event){
9041	.task = task,
9042	.next_prev = next_prev,
9043	.event_id = {
9044	.header = {
9045	/* .type */
9046	.misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT,
9047	/* .size */
9048	},
9049	/* .next_prev_pid */
9050	/* .next_prev_tid */
9051	},
9052	};
9053
9054	if (!sched_in && task->on_rq) {
9055	switch_event.event_id.header.misc |=
9056	PERF_RECORD_MISC_SWITCH_OUT_PREEMPT;
9057	}
9058
9059	perf_iterate_sb(output: perf_event_switch_output, data: &switch_event, NULL);
9060	}
9061
9062	/*
9063	* IRQ throttle logging
9064	*/
9065
9066	static void perf_log_throttle(struct perf_event *event, int enable)
9067	{
9068	struct perf_output_handle handle;
9069	struct perf_sample_data sample;
9070	int ret;
9071
9072	struct {
9073	struct perf_event_header header;
9074	u64 time;
9075	u64 id;
9076	u64 stream_id;
9077	} throttle_event = {
9078	.header = {
9079	.type = PERF_RECORD_THROTTLE,
9080	.misc = 0,
9081	.size = sizeof(throttle_event),
9082	},
9083	.time = perf_event_clock(event),
9084	.id = primary_event_id(event),
9085	.stream_id = event->id,
9086	};
9087
9088	if (enable)
9089	throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
9090
9091	perf_event_header__init_id(header: &throttle_event.header, data: &sample, event);
9092
9093	ret = perf_output_begin(handle: &handle, data: &sample, event,
9094	size: throttle_event.header.size);
9095	if (ret)
9096	return;
9097
9098	perf_output_put(&handle, throttle_event);
9099	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
9100	perf_output_end(handle: &handle);
9101	}
9102
9103	/*
9104	* ksymbol register/unregister tracking
9105	*/
9106
9107	struct perf_ksymbol_event {
9108	const char *name;
9109	int name_len;
9110	struct {
9111	struct perf_event_header header;
9112	u64 addr;
9113	u32 len;
9114	u16 ksym_type;
9115	u16 flags;
9116	} event_id;
9117	};
9118
9119	static int perf_event_ksymbol_match(struct perf_event *event)
9120	{
9121	return event->attr.ksymbol;
9122	}
9123
9124	static void perf_event_ksymbol_output(struct perf_event *event, void *data)
9125	{
9126	struct perf_ksymbol_event *ksymbol_event = data;
9127	struct perf_output_handle handle;
9128	struct perf_sample_data sample;
9129	int ret;
9130
9131	if (!perf_event_ksymbol_match(event))
9132	return;
9133
9134	perf_event_header__init_id(header: &ksymbol_event->event_id.header,
9135	data: &sample, event);
9136	ret = perf_output_begin(handle: &handle, data: &sample, event,
9137	size: ksymbol_event->event_id.header.size);
9138	if (ret)
9139	return;
9140
9141	perf_output_put(&handle, ksymbol_event->event_id);
9142	__output_copy(handle: &handle, buf: ksymbol_event->name, len: ksymbol_event->name_len);
9143	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
9144
9145	perf_output_end(handle: &handle);
9146	}
9147
9148	void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister,
9149	const char *sym)
9150	{
9151	struct perf_ksymbol_event ksymbol_event;
9152	char name[KSYM_NAME_LEN];
9153	u16 flags = 0;
9154	int name_len;
9155
9156	if (!atomic_read(v: &nr_ksymbol_events))
9157	return;
9158
9159	if (ksym_type >= PERF_RECORD_KSYMBOL_TYPE_MAX ||
9160	ksym_type == PERF_RECORD_KSYMBOL_TYPE_UNKNOWN)
9161	goto err;
9162
9163	strscpy(p: name, q: sym, KSYM_NAME_LEN);
9164	name_len = strlen(name) + 1;
9165	while (!IS_ALIGNED(name_len, sizeof(u64)))
9166	name[name_len++] = '\0';
9167	BUILD_BUG_ON(KSYM_NAME_LEN % sizeof(u64));
9168
9169	if (unregister)
9170	flags |= PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER;
9171
9172	ksymbol_event = (struct perf_ksymbol_event){
9173	.name = name,
9174	.name_len = name_len,
9175	.event_id = {
9176	.header = {
9177	.type = PERF_RECORD_KSYMBOL,
9178	.size = sizeof(ksymbol_event.event_id) +
9179	name_len,
9180	},
9181	.addr = addr,
9182	.len = len,
9183	.ksym_type = ksym_type,
9184	.flags = flags,
9185	},
9186	};
9187
9188	perf_iterate_sb(output: perf_event_ksymbol_output, data: &ksymbol_event, NULL);
9189	return;
9190	err:
9191	WARN_ONCE(1, "%s: Invalid KSYMBOL type 0x%x\n", __func__, ksym_type);
9192	}
9193
9194	/*
9195	* bpf program load/unload tracking
9196	*/
9197
9198	struct perf_bpf_event {
9199	struct bpf_prog *prog;
9200	struct {
9201	struct perf_event_header header;
9202	u16 type;
9203	u16 flags;
9204	u32 id;
9205	u8 tag[BPF_TAG_SIZE];
9206	} event_id;
9207	};
9208
9209	static int perf_event_bpf_match(struct perf_event *event)
9210	{
9211	return event->attr.bpf_event;
9212	}
9213
9214	static void perf_event_bpf_output(struct perf_event *event, void *data)
9215	{
9216	struct perf_bpf_event *bpf_event = data;
9217	struct perf_output_handle handle;
9218	struct perf_sample_data sample;
9219	int ret;
9220
9221	if (!perf_event_bpf_match(event))
9222	return;
9223
9224	perf_event_header__init_id(header: &bpf_event->event_id.header,
9225	data: &sample, event);
9226	ret = perf_output_begin(handle: &handle, data: &sample, event,
9227	size: bpf_event->event_id.header.size);
9228	if (ret)
9229	return;
9230
9231	perf_output_put(&handle, bpf_event->event_id);
9232	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
9233
9234	perf_output_end(handle: &handle);
9235	}
9236
9237	static void perf_event_bpf_emit_ksymbols(struct bpf_prog *prog,
9238	enum perf_bpf_event_type type)
9239	{
9240	bool unregister = type == PERF_BPF_EVENT_PROG_UNLOAD;
9241	int i;
9242
9243	if (prog->aux->func_cnt == 0) {
9244	perf_event_ksymbol(ksym_type: PERF_RECORD_KSYMBOL_TYPE_BPF,
9245	addr: (u64)(unsigned long)prog->bpf_func,
9246	len: prog->jited_len, unregister,
9247	sym: prog->aux->ksym.name);
9248	} else {
9249	for (i = 0; i < prog->aux->func_cnt; i++) {
9250	struct bpf_prog *subprog = prog->aux->func[i];
9251
9252	perf_event_ksymbol(
9253	ksym_type: PERF_RECORD_KSYMBOL_TYPE_BPF,
9254	addr: (u64)(unsigned long)subprog->bpf_func,
9255	len: subprog->jited_len, unregister,
9256	sym: subprog->aux->ksym.name);
9257	}
9258	}
9259	}
9260
9261	void perf_event_bpf_event(struct bpf_prog *prog,
9262	enum perf_bpf_event_type type,
9263	u16 flags)
9264	{
9265	struct perf_bpf_event bpf_event;
9266
9267	if (type <= PERF_BPF_EVENT_UNKNOWN ||
9268	type >= PERF_BPF_EVENT_MAX)
9269	return;
9270
9271	switch (type) {
9272	case PERF_BPF_EVENT_PROG_LOAD:
9273	case PERF_BPF_EVENT_PROG_UNLOAD:
9274	if (atomic_read(v: &nr_ksymbol_events))
9275	perf_event_bpf_emit_ksymbols(prog, type);
9276	break;
9277	default:
9278	break;
9279	}
9280
9281	if (!atomic_read(v: &nr_bpf_events))
9282	return;
9283
9284	bpf_event = (struct perf_bpf_event){
9285	.prog = prog,
9286	.event_id = {
9287	.header = {
9288	.type = PERF_RECORD_BPF_EVENT,
9289	.size = sizeof(bpf_event.event_id),
9290	},
9291	.type = type,
9292	.flags = flags,
9293	.id = prog->aux->id,
9294	},
9295	};
9296
9297	BUILD_BUG_ON(BPF_TAG_SIZE % sizeof(u64));
9298
9299	memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE);
9300	perf_iterate_sb(output: perf_event_bpf_output, data: &bpf_event, NULL);
9301	}
9302
9303	struct perf_text_poke_event {
9304	const void *old_bytes;
9305	const void *new_bytes;
9306	size_t pad;
9307	u16 old_len;
9308	u16 new_len;
9309
9310	struct {
9311	struct perf_event_header header;
9312
9313	u64 addr;
9314	} event_id;
9315	};
9316
9317	static int perf_event_text_poke_match(struct perf_event *event)
9318	{
9319	return event->attr.text_poke;
9320	}
9321
9322	static void perf_event_text_poke_output(struct perf_event *event, void *data)
9323	{
9324	struct perf_text_poke_event *text_poke_event = data;
9325	struct perf_output_handle handle;
9326	struct perf_sample_data sample;
9327	u64 padding = 0;
9328	int ret;
9329
9330	if (!perf_event_text_poke_match(event))
9331	return;
9332
9333	perf_event_header__init_id(header: &text_poke_event->event_id.header, data: &sample, event);
9334
9335	ret = perf_output_begin(handle: &handle, data: &sample, event,
9336	size: text_poke_event->event_id.header.size);
9337	if (ret)
9338	return;
9339
9340	perf_output_put(&handle, text_poke_event->event_id);
9341	perf_output_put(&handle, text_poke_event->old_len);
9342	perf_output_put(&handle, text_poke_event->new_len);
9343
9344	__output_copy(handle: &handle, buf: text_poke_event->old_bytes, len: text_poke_event->old_len);
9345	__output_copy(handle: &handle, buf: text_poke_event->new_bytes, len: text_poke_event->new_len);
9346
9347	if (text_poke_event->pad)
9348	__output_copy(handle: &handle, buf: &padding, len: text_poke_event->pad);
9349
9350	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
9351
9352	perf_output_end(handle: &handle);
9353	}
9354
9355	void perf_event_text_poke(const void *addr, const void *old_bytes,
9356	size_t old_len, const void *new_bytes, size_t new_len)
9357	{
9358	struct perf_text_poke_event text_poke_event;
9359	size_t tot, pad;
9360
9361	if (!atomic_read(v: &nr_text_poke_events))
9362	return;
9363
9364	tot = sizeof(text_poke_event.old_len) + old_len;
9365	tot += sizeof(text_poke_event.new_len) + new_len;
9366	pad = ALIGN(tot, sizeof(u64)) - tot;
9367
9368	text_poke_event = (struct perf_text_poke_event){
9369	.old_bytes = old_bytes,
9370	.new_bytes = new_bytes,
9371	.pad = pad,
9372	.old_len = old_len,
9373	.new_len = new_len,
9374	.event_id = {
9375	.header = {
9376	.type = PERF_RECORD_TEXT_POKE,
9377	.misc = PERF_RECORD_MISC_KERNEL,
9378	.size = sizeof(text_poke_event.event_id) + tot + pad,
9379	},
9380	.addr = (unsigned long)addr,
9381	},
9382	};
9383
9384	perf_iterate_sb(output: perf_event_text_poke_output, data: &text_poke_event, NULL);
9385	}
9386
9387	void perf_event_itrace_started(struct perf_event *event)
9388	{
9389	event->attach_state |= PERF_ATTACH_ITRACE;
9390	}
9391
9392	static void perf_log_itrace_start(struct perf_event *event)
9393	{
9394	struct perf_output_handle handle;
9395	struct perf_sample_data sample;
9396	struct perf_aux_event {
9397	struct perf_event_header header;
9398	u32 pid;
9399	u32 tid;
9400	} rec;
9401	int ret;
9402
9403	if (event->parent)
9404	event = event->parent;
9405
9406	if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) ||
9407	event->attach_state & PERF_ATTACH_ITRACE)
9408	return;
9409
9410	rec.header.type = PERF_RECORD_ITRACE_START;
9411	rec.header.misc = 0;
9412	rec.header.size = sizeof(rec);
9413	rec.pid = perf_event_pid(event, current);
9414	rec.tid = perf_event_tid(event, current);
9415
9416	perf_event_header__init_id(header: &rec.header, data: &sample, event);
9417	ret = perf_output_begin(handle: &handle, data: &sample, event, size: rec.header.size);
9418
9419	if (ret)
9420	return;
9421
9422	perf_output_put(&handle, rec);
9423	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
9424
9425	perf_output_end(handle: &handle);
9426	}
9427
9428	void perf_report_aux_output_id(struct perf_event *event, u64 hw_id)
9429	{
9430	struct perf_output_handle handle;
9431	struct perf_sample_data sample;
9432	struct perf_aux_event {
9433	struct perf_event_header header;
9434	u64 hw_id;
9435	} rec;
9436	int ret;
9437
9438	if (event->parent)
9439	event = event->parent;
9440
9441	rec.header.type = PERF_RECORD_AUX_OUTPUT_HW_ID;
9442	rec.header.misc = 0;
9443	rec.header.size = sizeof(rec);
9444	rec.hw_id = hw_id;
9445
9446	perf_event_header__init_id(header: &rec.header, data: &sample, event);
9447	ret = perf_output_begin(handle: &handle, data: &sample, event, size: rec.header.size);
9448
9449	if (ret)
9450	return;
9451
9452	perf_output_put(&handle, rec);
9453	perf_event__output_id_sample(event, handle: &handle, sample: &sample);
9454
9455	perf_output_end(handle: &handle);
9456	}
9457	EXPORT_SYMBOL_GPL(perf_report_aux_output_id);
9458
9459	static int
9460	__perf_event_account_interrupt(struct perf_event *event, int throttle)
9461	{
9462	struct hw_perf_event *hwc = &event->hw;
9463	int ret = 0;
9464	u64 seq;
9465
9466	seq = __this_cpu_read(perf_throttled_seq);
9467	if (seq != hwc->interrupts_seq) {
9468	hwc->interrupts_seq = seq;
9469	hwc->interrupts = 1;
9470	} else {
9471	hwc->interrupts++;
9472	if (unlikely(throttle &&
9473	hwc->interrupts > max_samples_per_tick)) {
9474	__this_cpu_inc(perf_throttled_count);
9475	tick_dep_set_cpu(smp_processor_id(), bit: TICK_DEP_BIT_PERF_EVENTS);
9476	hwc->interrupts = MAX_INTERRUPTS;
9477	perf_log_throttle(event, enable: 0);
9478	ret = 1;
9479	}
9480	}
9481
9482	if (event->attr.freq) {
9483	u64 now = perf_clock();
9484	s64 delta = now - hwc->freq_time_stamp;
9485
9486	hwc->freq_time_stamp = now;
9487
9488	if (delta > 0 && delta < 2*TICK_NSEC)
9489	perf_adjust_period(event, nsec: delta, count: hwc->last_period, disable: true);
9490	}
9491
9492	return ret;
9493	}
9494
9495	int perf_event_account_interrupt(struct perf_event *event)
9496	{
9497	return __perf_event_account_interrupt(event, throttle: 1);
9498	}
9499
9500	static inline bool sample_is_allowed(struct perf_event *event, struct pt_regs *regs)
9501	{
9502	/*
9503	* Due to interrupt latency (AKA "skid"), we may enter the
9504	* kernel before taking an overflow, even if the PMU is only
9505	* counting user events.
9506	*/
9507	if (event->attr.exclude_kernel && !user_mode(regs))
9508	return false;
9509
9510	return true;
9511	}
9512
9513	/*
9514	* Generic event overflow handling, sampling.
9515	*/
9516
9517	static int __perf_event_overflow(struct perf_event *event,
9518	int throttle, struct perf_sample_data *data,
9519	struct pt_regs *regs)
9520	{
9521	int events = atomic_read(v: &event->event_limit);
9522	int ret = 0;
9523
9524	/*
9525	* Non-sampling counters might still use the PMI to fold short
9526	* hardware counters, ignore those.
9527	*/
9528	if (unlikely(!is_sampling_event(event)))
9529	return 0;
9530
9531	ret = __perf_event_account_interrupt(event, throttle);
9532
9533	/*
9534	* XXX event_limit might not quite work as expected on inherited
9535	* events
9536	*/
9537
9538	event->pending_kill = POLL_IN;
9539	if (events && atomic_dec_and_test(v: &event->event_limit)) {
9540	ret = 1;
9541	event->pending_kill = POLL_HUP;
9542	perf_event_disable_inatomic(event);
9543	}
9544
9545	if (event->attr.sigtrap) {
9546	/*
9547	* The desired behaviour of sigtrap vs invalid samples is a bit
9548	* tricky; on the one hand, one should not loose the SIGTRAP if
9549	* it is the first event, on the other hand, we should also not
9550	* trigger the WARN or override the data address.
9551	*/
9552	bool valid_sample = sample_is_allowed(event, regs);
9553	unsigned int pending_id = 1;
9554
9555	if (regs)
9556	pending_id = hash32_ptr(ptr: (void *)instruction_pointer(regs)) ?: 1;
9557	if (!event->pending_sigtrap) {
9558	event->pending_sigtrap = pending_id;
9559	local_inc(l: &event->ctx->nr_pending);
9560	} else if (event->attr.exclude_kernel && valid_sample) {
9561	/*
9562	* Should not be able to return to user space without
9563	* consuming pending_sigtrap; with exceptions:
9564	*
9565	* 1. Where !exclude_kernel, events can overflow again
9566	* in the kernel without returning to user space.
9567	*
9568	* 2. Events that can overflow again before the IRQ-
9569	* work without user space progress (e.g. hrtimer).
9570	* To approximate progress (with false negatives),
9571	* check 32-bit hash of the current IP.
9572	*/
9573	WARN_ON_ONCE(event->pending_sigtrap != pending_id);
9574	}
9575
9576	event->pending_addr = 0;
9577	if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR))
9578	event->pending_addr = data->addr;
9579	irq_work_queue(work: &event->pending_irq);
9580	}
9581
9582	READ_ONCE(event->overflow_handler)(event, data, regs);
9583
9584	if (*perf_event_fasync(event) && event->pending_kill) {
9585	event->pending_wakeup = 1;
9586	irq_work_queue(work: &event->pending_irq);
9587	}
9588
9589	return ret;
9590	}
9591
9592	int perf_event_overflow(struct perf_event *event,
9593	struct perf_sample_data *data,
9594	struct pt_regs *regs)
9595	{
9596	return __perf_event_overflow(event, throttle: 1, data, regs);
9597	}
9598
9599	/*
9600	* Generic software event infrastructure
9601	*/
9602
9603	struct swevent_htable {
9604	struct swevent_hlist *swevent_hlist;
9605	struct mutex hlist_mutex;
9606	int hlist_refcount;
9607
9608	/* Recursion avoidance in each contexts */
9609	int recursion[PERF_NR_CONTEXTS];
9610	};
9611
9612	static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
9613
9614	/*
9615	* We directly increment event->count and keep a second value in
9616	* event->hw.period_left to count intervals. This period event
9617	* is kept in the range [-sample_period, 0] so that we can use the
9618	* sign as trigger.
9619	*/
9620
9621	u64 perf_swevent_set_period(struct perf_event *event)
9622	{
9623	struct hw_perf_event *hwc = &event->hw;
9624	u64 period = hwc->last_period;
9625	u64 nr, offset;
9626	s64 old, val;
9627
9628	hwc->last_period = hwc->sample_period;
9629
9630	old = local64_read(&hwc->period_left);
9631	do {
9632	val = old;
9633	if (val < 0)
9634	return 0;
9635
9636	nr = div64_u64(dividend: period + val, divisor: period);
9637	offset = nr * period;
9638	val -= offset;
9639	} while (!local64_try_cmpxchg(l: &hwc->period_left, old: &old, new: val));
9640
9641	return nr;
9642	}
9643
9644	static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
9645	struct perf_sample_data *data,
9646	struct pt_regs *regs)
9647	{
9648	struct hw_perf_event *hwc = &event->hw;
9649	int throttle = 0;
9650
9651	if (!overflow)
9652	overflow = perf_swevent_set_period(event);
9653
9654	if (hwc->interrupts == MAX_INTERRUPTS)
9655	return;
9656
9657	for (; overflow; overflow--) {
9658	if (__perf_event_overflow(event, throttle,
9659	data, regs)) {
9660	/*
9661	* We inhibit the overflow from happening when
9662	* hwc->interrupts == MAX_INTERRUPTS.
9663	*/
9664	break;
9665	}
9666	throttle = 1;
9667	}
9668	}
9669
9670	static void perf_swevent_event(struct perf_event *event, u64 nr,
9671	struct perf_sample_data *data,
9672	struct pt_regs *regs)
9673	{
9674	struct hw_perf_event *hwc = &event->hw;
9675
9676	local64_add(nr, &event->count);
9677
9678	if (!regs)
9679	return;
9680
9681	if (!is_sampling_event(event))
9682	return;
9683
9684	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
9685	data->period = nr;
9686	return perf_swevent_overflow(event, overflow: 1, data, regs);
9687	} else
9688	data->period = event->hw.last_period;
9689
9690	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
9691	return perf_swevent_overflow(event, overflow: 1, data, regs);
9692
9693	if (local64_add_negative(nr, &hwc->period_left))
9694	return;
9695
9696	perf_swevent_overflow(event, overflow: 0, data, regs);
9697	}
9698
9699	static int perf_exclude_event(struct perf_event *event,
9700	struct pt_regs *regs)
9701	{
9702	if (event->hw.state & PERF_HES_STOPPED)
9703	return 1;
9704
9705	if (regs) {
9706	if (event->attr.exclude_user && user_mode(regs))
9707	return 1;
9708
9709	if (event->attr.exclude_kernel && !user_mode(regs))
9710	return 1;
9711	}
9712
9713	return 0;
9714	}
9715
9716	static int perf_swevent_match(struct perf_event *event,
9717	enum perf_type_id type,
9718	u32 event_id,
9719	struct perf_sample_data *data,
9720	struct pt_regs *regs)
9721	{
9722	if (event->attr.type != type)
9723	return 0;
9724
9725	if (event->attr.config != event_id)
9726	return 0;
9727
9728	if (perf_exclude_event(event, regs))
9729	return 0;
9730
9731	return 1;
9732	}
9733
9734	static inline u64 swevent_hash(u64 type, u32 event_id)
9735	{
9736	u64 val = event_id | (type << 32);
9737
9738	return hash_64(val, SWEVENT_HLIST_BITS);
9739	}
9740
9741	static inline struct hlist_head *
9742	__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
9743	{
9744	u64 hash = swevent_hash(type, event_id);
9745
9746	return &hlist->heads[hash];
9747	}
9748
9749	/* For the read side: events when they trigger */
9750	static inline struct hlist_head *
9751	find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
9752	{
9753	struct swevent_hlist *hlist;
9754
9755	hlist = rcu_dereference(swhash->swevent_hlist);
9756	if (!hlist)
9757	return NULL;
9758
9759	return __find_swevent_head(hlist, type, event_id);
9760	}
9761
9762	/* For the event head insertion and removal in the hlist */
9763	static inline struct hlist_head *
9764	find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
9765	{
9766	struct swevent_hlist *hlist;
9767	u32 event_id = event->attr.config;
9768	u64 type = event->attr.type;
9769
9770	/*
9771	* Event scheduling is always serialized against hlist allocation
9772	* and release. Which makes the protected version suitable here.
9773	* The context lock guarantees that.
9774	*/
9775	hlist = rcu_dereference_protected(swhash->swevent_hlist,
9776	lockdep_is_held(&event->ctx->lock));
9777	if (!hlist)
9778	return NULL;
9779
9780	return __find_swevent_head(hlist, type, event_id);
9781	}
9782
9783	static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
9784	u64 nr,
9785	struct perf_sample_data *data,
9786	struct pt_regs *regs)
9787	{
9788	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
9789	struct perf_event *event;
9790	struct hlist_head *head;
9791
9792	rcu_read_lock();
9793	head = find_swevent_head_rcu(swhash, type, event_id);
9794	if (!head)
9795	goto end;
9796
9797	hlist_for_each_entry_rcu(event, head, hlist_entry) {
9798	if (perf_swevent_match(event, type, event_id, data, regs))
9799	perf_swevent_event(event, nr, data, regs);
9800	}
9801	end:
9802	rcu_read_unlock();
9803	}
9804
9805	DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]);
9806
9807	int perf_swevent_get_recursion_context(void)
9808	{
9809	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
9810
9811	return get_recursion_context(recursion: swhash->recursion);
9812	}
9813	EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
9814
9815	void perf_swevent_put_recursion_context(int rctx)
9816	{
9817	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
9818
9819	put_recursion_context(recursion: swhash->recursion, rctx);
9820	}
9821
9822	void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
9823	{
9824	struct perf_sample_data data;
9825
9826	if (WARN_ON_ONCE(!regs))
9827	return;
9828
9829	perf_sample_data_init(data: &data, addr, period: 0);
9830	do_perf_sw_event(type: PERF_TYPE_SOFTWARE, event_id, nr, data: &data, regs);
9831	}
9832
9833	void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
9834	{
9835	int rctx;
9836
9837	preempt_disable_notrace();
9838	rctx = perf_swevent_get_recursion_context();
9839	if (unlikely(rctx < 0))
9840	goto fail;
9841
9842	___perf_sw_event(event_id, nr, regs, addr);
9843
9844	perf_swevent_put_recursion_context(rctx);
9845	fail:
9846	preempt_enable_notrace();
9847	}
9848
9849	static void perf_swevent_read(struct perf_event *event)
9850	{
9851	}
9852
9853	static int perf_swevent_add(struct perf_event *event, int flags)
9854	{
9855	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
9856	struct hw_perf_event *hwc = &event->hw;
9857	struct hlist_head *head;
9858
9859	if (is_sampling_event(event)) {
9860	hwc->last_period = hwc->sample_period;
9861	perf_swevent_set_period(event);
9862	}
9863
9864	hwc->state = !(flags & PERF_EF_START);
9865
9866	head = find_swevent_head(swhash, event);
9867	if (WARN_ON_ONCE(!head))
9868	return -EINVAL;
9869
9870	hlist_add_head_rcu(n: &event->hlist_entry, h: head);
9871	perf_event_update_userpage(event);
9872
9873	return 0;
9874	}
9875
9876	static void perf_swevent_del(struct perf_event *event, int flags)
9877	{
9878	hlist_del_rcu(n: &event->hlist_entry);
9879	}
9880
9881	static void perf_swevent_start(struct perf_event *event, int flags)
9882	{
9883	event->hw.state = 0;
9884	}
9885
9886	static void perf_swevent_stop(struct perf_event *event, int flags)
9887	{
9888	event->hw.state = PERF_HES_STOPPED;
9889	}
9890
9891	/* Deref the hlist from the update side */
9892	static inline struct swevent_hlist *
9893	swevent_hlist_deref(struct swevent_htable *swhash)
9894	{
9895	return rcu_dereference_protected(swhash->swevent_hlist,
9896	lockdep_is_held(&swhash->hlist_mutex));
9897	}
9898
9899	static void swevent_hlist_release(struct swevent_htable *swhash)
9900	{
9901	struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
9902
9903	if (!hlist)
9904	return;
9905
9906	RCU_INIT_POINTER(swhash->swevent_hlist, NULL);
9907	kfree_rcu(hlist, rcu_head);
9908	}
9909
9910	static void swevent_hlist_put_cpu(int cpu)
9911	{
9912	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
9913
9914	mutex_lock(&swhash->hlist_mutex);
9915
9916	if (!--swhash->hlist_refcount)
9917	swevent_hlist_release(swhash);
9918
9919	mutex_unlock(lock: &swhash->hlist_mutex);
9920	}
9921
9922	static void swevent_hlist_put(void)
9923	{
9924	int cpu;
9925
9926	for_each_possible_cpu(cpu)
9927	swevent_hlist_put_cpu(cpu);
9928	}
9929
9930	static int swevent_hlist_get_cpu(int cpu)
9931	{
9932	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
9933	int err = 0;
9934
9935	mutex_lock(&swhash->hlist_mutex);
9936	if (!swevent_hlist_deref(swhash) &&
9937	cpumask_test_cpu(cpu, cpumask: perf_online_mask)) {
9938	struct swevent_hlist *hlist;
9939
9940	hlist = kzalloc(size: sizeof(*hlist), GFP_KERNEL);
9941	if (!hlist) {
9942	err = -ENOMEM;
9943	goto exit;
9944	}
9945	rcu_assign_pointer(swhash->swevent_hlist, hlist);
9946	}
9947	swhash->hlist_refcount++;
9948	exit:
9949	mutex_unlock(lock: &swhash->hlist_mutex);
9950
9951	return err;
9952	}
9953
9954	static int swevent_hlist_get(void)
9955	{
9956	int err, cpu, failed_cpu;
9957
9958	mutex_lock(&pmus_lock);
9959	for_each_possible_cpu(cpu) {
9960	err = swevent_hlist_get_cpu(cpu);
9961	if (err) {
9962	failed_cpu = cpu;
9963	goto fail;
9964	}
9965	}
9966	mutex_unlock(lock: &pmus_lock);
9967	return 0;
9968	fail:
9969	for_each_possible_cpu(cpu) {
9970	if (cpu == failed_cpu)
9971	break;
9972	swevent_hlist_put_cpu(cpu);
9973	}
9974	mutex_unlock(lock: &pmus_lock);
9975	return err;
9976	}
9977
9978	struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
9979
9980	static void sw_perf_event_destroy(struct perf_event *event)
9981	{
9982	u64 event_id = event->attr.config;
9983
9984	WARN_ON(event->parent);
9985
9986	static_key_slow_dec(key: &perf_swevent_enabled[event_id]);
9987	swevent_hlist_put();
9988	}
9989
9990	static struct pmu perf_cpu_clock; /* fwd declaration */
9991	static struct pmu perf_task_clock;
9992
9993	static int perf_swevent_init(struct perf_event *event)
9994	{
9995	u64 event_id = event->attr.config;
9996
9997	if (event->attr.type != PERF_TYPE_SOFTWARE)
9998	return -ENOENT;
9999
10000	/*
10001	* no branch sampling for software events
10002	*/
10003	if (has_branch_stack(event))
10004	return -EOPNOTSUPP;
10005
10006	switch (event_id) {
10007	case PERF_COUNT_SW_CPU_CLOCK:
10008	event->attr.type = perf_cpu_clock.type;
10009	return -ENOENT;
10010	case PERF_COUNT_SW_TASK_CLOCK:
10011	event->attr.type = perf_task_clock.type;
10012	return -ENOENT;
10013
10014	default:
10015	break;
10016	}
10017
10018	if (event_id >= PERF_COUNT_SW_MAX)
10019	return -ENOENT;
10020
10021	if (!event->parent) {
10022	int err;
10023
10024	err = swevent_hlist_get();
10025	if (err)
10026	return err;
10027
10028	static_key_slow_inc(key: &perf_swevent_enabled[event_id]);
10029	event->destroy = sw_perf_event_destroy;
10030	}
10031
10032	return 0;
10033	}
10034
10035	static struct pmu perf_swevent = {
10036	.task_ctx_nr = perf_sw_context,
10037
10038	.capabilities = PERF_PMU_CAP_NO_NMI,
10039
10040	.event_init = perf_swevent_init,
10041	.add = perf_swevent_add,
10042	.del = perf_swevent_del,
10043	.start = perf_swevent_start,
10044	.stop = perf_swevent_stop,
10045	.read = perf_swevent_read,
10046	};
10047
10048	#ifdef CONFIG_EVENT_TRACING
10049
10050	static void tp_perf_event_destroy(struct perf_event *event)
10051	{
10052	perf_trace_destroy(event);
10053	}
10054
10055	static int perf_tp_event_init(struct perf_event *event)
10056	{
10057	int err;
10058
10059	if (event->attr.type != PERF_TYPE_TRACEPOINT)
10060	return -ENOENT;
10061
10062	/*
10063	* no branch sampling for tracepoint events
10064	*/
10065	if (has_branch_stack(event))
10066	return -EOPNOTSUPP;
10067
10068	err = perf_trace_init(event);
10069	if (err)
10070	return err;
10071
10072	event->destroy = tp_perf_event_destroy;
10073
10074	return 0;
10075	}
10076
10077	static struct pmu perf_tracepoint = {
10078	.task_ctx_nr = perf_sw_context,
10079
10080	.event_init = perf_tp_event_init,
10081	.add = perf_trace_add,
10082	.del = perf_trace_del,
10083	.start = perf_swevent_start,
10084	.stop = perf_swevent_stop,
10085	.read = perf_swevent_read,
10086	};
10087
10088	static int perf_tp_filter_match(struct perf_event *event,
10089	struct perf_sample_data *data)
10090	{
10091	void *record = data->raw->frag.data;
10092
10093	/* only top level events have filters set */
10094	if (event->parent)
10095	event = event->parent;
10096
10097	if (likely(!event->filter) || filter_match_preds(filter: event->filter, rec: record))
10098	return 1;
10099	return 0;
10100	}
10101
10102	static int perf_tp_event_match(struct perf_event *event,
10103	struct perf_sample_data *data,
10104	struct pt_regs *regs)
10105	{
10106	if (event->hw.state & PERF_HES_STOPPED)
10107	return 0;
10108	/*
10109	* If exclude_kernel, only trace user-space tracepoints (uprobes)
10110	*/
10111	if (event->attr.exclude_kernel && !user_mode(regs))
10112	return 0;
10113
10114	if (!perf_tp_filter_match(event, data))
10115	return 0;
10116
10117	return 1;
10118	}
10119
10120	void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
10121	struct trace_event_call *call, u64 count,
10122	struct pt_regs *regs, struct hlist_head *head,
10123	struct task_struct *task)
10124	{
10125	if (bpf_prog_array_valid(call)) {
10126	*(struct pt_regs **)raw_data = regs;
10127	if (!trace_call_bpf(call, ctx: raw_data) || hlist_empty(h: head)) {
10128	perf_swevent_put_recursion_context(rctx);
10129	return;
10130	}
10131	}
10132	perf_tp_event(event_type: call->event.type, count, record: raw_data, entry_size: size, regs, head,
10133	rctx, task);
10134	}
10135	EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit);
10136
10137	static void __perf_tp_event_target_task(u64 count, void *record,
10138	struct pt_regs *regs,
10139	struct perf_sample_data *data,
10140	struct perf_event *event)
10141	{
10142	struct trace_entry *entry = record;
10143
10144	if (event->attr.config != entry->type)
10145	return;
10146	/* Cannot deliver synchronous signal to other task. */
10147	if (event->attr.sigtrap)
10148	return;
10149	if (perf_tp_event_match(event, data, regs))
10150	perf_swevent_event(event, nr: count, data, regs);
10151	}
10152
10153	static void perf_tp_event_target_task(u64 count, void *record,
10154	struct pt_regs *regs,
10155	struct perf_sample_data *data,
10156	struct perf_event_context *ctx)
10157	{
10158	unsigned int cpu = smp_processor_id();
10159	struct pmu *pmu = &perf_tracepoint;
10160	struct perf_event *event, *sibling;
10161
10162	perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) {
10163	__perf_tp_event_target_task(count, record, regs, data, event);
10164	for_each_sibling_event(sibling, event)
10165	__perf_tp_event_target_task(count, record, regs, data, event: sibling);
10166	}
10167
10168	perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) {
10169	__perf_tp_event_target_task(count, record, regs, data, event);
10170	for_each_sibling_event(sibling, event)
10171	__perf_tp_event_target_task(count, record, regs, data, event: sibling);
10172	}
10173	}
10174
10175	void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
10176	struct pt_regs *regs, struct hlist_head *head, int rctx,
10177	struct task_struct *task)
10178	{
10179	struct perf_sample_data data;
10180	struct perf_event *event;
10181
10182	struct perf_raw_record raw = {
10183	.frag = {
10184	.size = entry_size,
10185	.data = record,
10186	},
10187	};
10188
10189	perf_sample_data_init(data: &data, addr: 0, period: 0);
10190	perf_sample_save_raw_data(data: &data, raw: &raw);
10191
10192	perf_trace_buf_update(record, type: event_type);
10193
10194	hlist_for_each_entry_rcu(event, head, hlist_entry) {
10195	if (perf_tp_event_match(event, data: &data, regs)) {
10196	perf_swevent_event(event, nr: count, data: &data, regs);
10197
10198	/*
10199	* Here use the same on-stack perf_sample_data,
10200	* some members in data are event-specific and
10201	* need to be re-computed for different sweveents.
10202	* Re-initialize data->sample_flags safely to avoid
10203	* the problem that next event skips preparing data
10204	* because data->sample_flags is set.
10205	*/
10206	perf_sample_data_init(data: &data, addr: 0, period: 0);
10207	perf_sample_save_raw_data(data: &data, raw: &raw);
10208	}
10209	}
10210
10211	/*
10212	* If we got specified a target task, also iterate its context and
10213	* deliver this event there too.
10214	*/
10215	if (task && task != current) {
10216	struct perf_event_context *ctx;
10217
10218	rcu_read_lock();
10219	ctx = rcu_dereference(task->perf_event_ctxp);
10220	if (!ctx)
10221	goto unlock;
10222
10223	raw_spin_lock(&ctx->lock);
10224	perf_tp_event_target_task(count, record, regs, data: &data, ctx);
10225	raw_spin_unlock(&ctx->lock);
10226	unlock:
10227	rcu_read_unlock();
10228	}
10229
10230	perf_swevent_put_recursion_context(rctx);
10231	}
10232	EXPORT_SYMBOL_GPL(perf_tp_event);
10233
10234	#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
10235	/*
10236	* Flags in config, used by dynamic PMU kprobe and uprobe
10237	* The flags should match following PMU_FORMAT_ATTR().
10238	*
10239	* PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe
10240	* if not set, create kprobe/uprobe
10241	*
10242	* The following values specify a reference counter (or semaphore in the
10243	* terminology of tools like dtrace, systemtap, etc.) Userspace Statically
10244	* Defined Tracepoints (USDT). Currently, we use 40 bit for the offset.
10245	*
10246	* PERF_UPROBE_REF_CTR_OFFSET_BITS # of bits in config as th offset
10247	* PERF_UPROBE_REF_CTR_OFFSET_SHIFT # of bits to shift left
10248	*/
10249	enum perf_probe_config {
10250	PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0, /* [k,u]retprobe */
10251	PERF_UPROBE_REF_CTR_OFFSET_BITS = 32,
10252	PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
10253	};
10254
10255	PMU_FORMAT_ATTR(retprobe, "config:0");
10256	#endif
10257
10258	#ifdef CONFIG_KPROBE_EVENTS
10259	static struct attribute *kprobe_attrs[] = {
10260	&format_attr_retprobe.attr,
10261	NULL,
10262	};
10263
10264	static struct attribute_group kprobe_format_group = {
10265	.name = "format",
10266	.attrs = kprobe_attrs,
10267	};
10268
10269	static const struct attribute_group *kprobe_attr_groups[] = {
10270	&kprobe_format_group,
10271	NULL,
10272	};
10273
10274	static int perf_kprobe_event_init(struct perf_event *event);
10275	static struct pmu perf_kprobe = {
10276	.task_ctx_nr = perf_sw_context,
10277	.event_init = perf_kprobe_event_init,
10278	.add = perf_trace_add,
10279	.del = perf_trace_del,
10280	.start = perf_swevent_start,
10281	.stop = perf_swevent_stop,
10282	.read = perf_swevent_read,
10283	.attr_groups = kprobe_attr_groups,
10284	};
10285
10286	static int perf_kprobe_event_init(struct perf_event *event)
10287	{
10288	int err;
10289	bool is_retprobe;
10290
10291	if (event->attr.type != perf_kprobe.type)
10292	return -ENOENT;
10293
10294	if (!perfmon_capable())
10295	return -EACCES;
10296
10297	/*
10298	* no branch sampling for probe events
10299	*/
10300	if (has_branch_stack(event))
10301	return -EOPNOTSUPP;
10302
10303	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
10304	err = perf_kprobe_init(event, is_retprobe);
10305	if (err)
10306	return err;
10307
10308	event->destroy = perf_kprobe_destroy;
10309
10310	return 0;
10311	}
10312	#endif /* CONFIG_KPROBE_EVENTS */
10313
10314	#ifdef CONFIG_UPROBE_EVENTS
10315	PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
10316
10317	static struct attribute *uprobe_attrs[] = {
10318	&format_attr_retprobe.attr,
10319	&format_attr_ref_ctr_offset.attr,
10320	NULL,
10321	};
10322
10323	static struct attribute_group uprobe_format_group = {
10324	.name = "format",
10325	.attrs = uprobe_attrs,
10326	};
10327
10328	static const struct attribute_group *uprobe_attr_groups[] = {
10329	&uprobe_format_group,
10330	NULL,
10331	};
10332
10333	static int perf_uprobe_event_init(struct perf_event *event);
10334	static struct pmu perf_uprobe = {
10335	.task_ctx_nr = perf_sw_context,
10336	.event_init = perf_uprobe_event_init,
10337	.add = perf_trace_add,
10338	.del = perf_trace_del,
10339	.start = perf_swevent_start,
10340	.stop = perf_swevent_stop,
10341	.read = perf_swevent_read,
10342	.attr_groups = uprobe_attr_groups,
10343	};
10344
10345	static int perf_uprobe_event_init(struct perf_event *event)
10346	{
10347	int err;
10348	unsigned long ref_ctr_offset;
10349	bool is_retprobe;
10350
10351	if (event->attr.type != perf_uprobe.type)
10352	return -ENOENT;
10353
10354	if (!perfmon_capable())
10355	return -EACCES;
10356
10357	/*
10358	* no branch sampling for probe events
10359	*/
10360	if (has_branch_stack(event))
10361	return -EOPNOTSUPP;
10362
10363	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
10364	ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
10365	err = perf_uprobe_init(event, ref_ctr_offset, is_retprobe);
10366	if (err)
10367	return err;
10368
10369	event->destroy = perf_uprobe_destroy;
10370
10371	return 0;
10372	}
10373	#endif /* CONFIG_UPROBE_EVENTS */
10374
10375	static inline void perf_tp_register(void)
10376	{
10377	perf_pmu_register(pmu: &perf_tracepoint, name: "tracepoint", type: PERF_TYPE_TRACEPOINT);
10378	#ifdef CONFIG_KPROBE_EVENTS
10379	perf_pmu_register(pmu: &perf_kprobe, name: "kprobe", type: -1);
10380	#endif
10381	#ifdef CONFIG_UPROBE_EVENTS
10382	perf_pmu_register(pmu: &perf_uprobe, name: "uprobe", type: -1);
10383	#endif
10384	}
10385
10386	static void perf_event_free_filter(struct perf_event *event)
10387	{
10388	ftrace_profile_free_filter(event);
10389	}
10390
10391	#ifdef CONFIG_BPF_SYSCALL
10392	static void bpf_overflow_handler(struct perf_event *event,
10393	struct perf_sample_data *data,
10394	struct pt_regs *regs)
10395	{
10396	struct bpf_perf_event_data_kern ctx = {
10397	.data = data,
10398	.event = event,
10399	};
10400	struct bpf_prog *prog;
10401	int ret = 0;
10402
10403	ctx.regs = perf_arch_bpf_user_pt_regs(regs);
10404	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1))
10405	goto out;
10406	rcu_read_lock();
10407	prog = READ_ONCE(event->prog);
10408	if (prog) {
10409	perf_prepare_sample(data, event, regs);
10410	ret = bpf_prog_run(prog, ctx: &ctx);
10411	}
10412	rcu_read_unlock();
10413	out:
10414	__this_cpu_dec(bpf_prog_active);
10415	if (!ret)
10416	return;
10417
10418	event->orig_overflow_handler(event, data, regs);
10419	}
10420
10421	static int perf_event_set_bpf_handler(struct perf_event *event,
10422	struct bpf_prog *prog,
10423	u64 bpf_cookie)
10424	{
10425	if (event->overflow_handler_context)
10426	/* hw breakpoint or kernel counter */
10427	return -EINVAL;
10428
10429	if (event->prog)
10430	return -EEXIST;
10431
10432	if (prog->type != BPF_PROG_TYPE_PERF_EVENT)
10433	return -EINVAL;
10434
10435	if (event->attr.precise_ip &&
10436	prog->call_get_stack &&
10437	(!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) ||
10438	event->attr.exclude_callchain_kernel ||
10439	event->attr.exclude_callchain_user)) {
10440	/*
10441	* On perf_event with precise_ip, calling bpf_get_stack()
10442	* may trigger unwinder warnings and occasional crashes.
10443	* bpf_get_[stack|stackid] works around this issue by using
10444	* callchain attached to perf_sample_data. If the
10445	* perf_event does not full (kernel and user) callchain
10446	* attached to perf_sample_data, do not allow attaching BPF
10447	* program that calls bpf_get_[stack|stackid].
10448	*/
10449	return -EPROTO;
10450	}
10451
10452	event->prog = prog;
10453	event->bpf_cookie = bpf_cookie;
10454	event->orig_overflow_handler = READ_ONCE(event->overflow_handler);
10455	WRITE_ONCE(event->overflow_handler, bpf_overflow_handler);
10456	return 0;
10457	}
10458
10459	static void perf_event_free_bpf_handler(struct perf_event *event)
10460	{
10461	struct bpf_prog *prog = event->prog;
10462
10463	if (!prog)
10464	return;
10465
10466	WRITE_ONCE(event->overflow_handler, event->orig_overflow_handler);
10467	event->prog = NULL;
10468	bpf_prog_put(prog);
10469	}
10470	#else
10471	static int perf_event_set_bpf_handler(struct perf_event *event,
10472	struct bpf_prog *prog,
10473	u64 bpf_cookie)
10474	{
10475	return -EOPNOTSUPP;
10476	}
10477	static void perf_event_free_bpf_handler(struct perf_event *event)
10478	{
10479	}
10480	#endif
10481
10482	/*
10483	* returns true if the event is a tracepoint, or a kprobe/upprobe created
10484	* with perf_event_open()
10485	*/
10486	static inline bool perf_event_is_tracing(struct perf_event *event)
10487	{
10488	if (event->pmu == &perf_tracepoint)
10489	return true;
10490	#ifdef CONFIG_KPROBE_EVENTS
10491	if (event->pmu == &perf_kprobe)
10492	return true;
10493	#endif
10494	#ifdef CONFIG_UPROBE_EVENTS
10495	if (event->pmu == &perf_uprobe)
10496	return true;
10497	#endif
10498	return false;
10499	}
10500
10501	int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,
10502	u64 bpf_cookie)
10503	{
10504	bool is_kprobe, is_uprobe, is_tracepoint, is_syscall_tp;
10505
10506	if (!perf_event_is_tracing(event))
10507	return perf_event_set_bpf_handler(event, prog, bpf_cookie);
10508
10509	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE;
10510	is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE;
10511	is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;
10512	is_syscall_tp = is_syscall_trace_event(tp_event: event->tp_event);
10513	if (!is_kprobe && !is_uprobe && !is_tracepoint && !is_syscall_tp)
10514	/* bpf programs can only be attached to u/kprobe or tracepoint */
10515	return -EINVAL;
10516
10517	if (((is_kprobe || is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) ||
10518	(is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) ||
10519	(is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT))
10520	return -EINVAL;
10521
10522	if (prog->type == BPF_PROG_TYPE_KPROBE && prog->aux->sleepable && !is_uprobe)
10523	/* only uprobe programs are allowed to be sleepable */
10524	return -EINVAL;
10525
10526	/* Kprobe override only works for kprobes, not uprobes. */
10527	if (prog->kprobe_override && !is_kprobe)
10528	return -EINVAL;
10529
10530	if (is_tracepoint || is_syscall_tp) {
10531	int off = trace_event_get_offsets(call: event->tp_event);
10532
10533	if (prog->aux->max_ctx_offset > off)
10534	return -EACCES;
10535	}
10536
10537	return perf_event_attach_bpf_prog(event, prog, bpf_cookie);
10538	}
10539
10540	void perf_event_free_bpf_prog(struct perf_event *event)
10541	{
10542	if (!perf_event_is_tracing(event)) {
10543	perf_event_free_bpf_handler(event);
10544	return;
10545	}
10546	perf_event_detach_bpf_prog(event);
10547	}
10548
10549	#else
10550
10551	static inline void perf_tp_register(void)
10552	{
10553	}
10554
10555	static void perf_event_free_filter(struct perf_event *event)
10556	{
10557	}
10558
10559	int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,
10560	u64 bpf_cookie)
10561	{
10562	return -ENOENT;
10563	}
10564
10565	void perf_event_free_bpf_prog(struct perf_event *event)
10566	{
10567	}
10568	#endif /* CONFIG_EVENT_TRACING */
10569
10570	#ifdef CONFIG_HAVE_HW_BREAKPOINT
10571	void perf_bp_event(struct perf_event *bp, void *data)
10572	{
10573	struct perf_sample_data sample;
10574	struct pt_regs *regs = data;
10575
10576	perf_sample_data_init(data: &sample, addr: bp->attr.bp_addr, period: 0);
10577
10578	if (!bp->hw.state && !perf_exclude_event(event: bp, regs))
10579	perf_swevent_event(event: bp, nr: 1, data: &sample, regs);
10580	}
10581	#endif
10582
10583	/*
10584	* Allocate a new address filter
10585	*/
10586	static struct perf_addr_filter *
10587	perf_addr_filter_new(struct perf_event *event, struct list_head *filters)
10588	{
10589	int node = cpu_to_node(cpu: event->cpu == -1 ? 0 : event->cpu);
10590	struct perf_addr_filter *filter;
10591
10592	filter = kzalloc_node(size: sizeof(*filter), GFP_KERNEL, node);
10593	if (!filter)
10594	return NULL;
10595
10596	INIT_LIST_HEAD(list: &filter->entry);
10597	list_add_tail(new: &filter->entry, head: filters);
10598
10599	return filter;
10600	}
10601
10602	static void free_filters_list(struct list_head *filters)
10603	{
10604	struct perf_addr_filter *filter, *iter;
10605
10606	list_for_each_entry_safe(filter, iter, filters, entry) {
10607	path_put(&filter->path);
10608	list_del(entry: &filter->entry);
10609	kfree(objp: filter);
10610	}
10611	}
10612
10613	/*
10614	* Free existing address filters and optionally install new ones
10615	*/
10616	static void perf_addr_filters_splice(struct perf_event *event,
10617	struct list_head *head)
10618	{
10619	unsigned long flags;
10620	LIST_HEAD(list);
10621
10622	if (!has_addr_filter(event))
10623	return;
10624
10625	/* don't bother with children, they don't have their own filters */
10626	if (event->parent)
10627	return;
10628
10629	raw_spin_lock_irqsave(&event->addr_filters.lock, flags);
10630
10631	list_splice_init(list: &event->addr_filters.list, head: &list);
10632	if (head)
10633	list_splice(list: head, head: &event->addr_filters.list);
10634
10635	raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags);
10636
10637	free_filters_list(filters: &list);
10638	}
10639
10640	/*
10641	* Scan through mm's vmas and see if one of them matches the
10642	* @filter; if so, adjust filter's address range.
10643	* Called with mm::mmap_lock down for reading.
10644	*/
10645	static void perf_addr_filter_apply(struct perf_addr_filter *filter,
10646	struct mm_struct *mm,
10647	struct perf_addr_filter_range *fr)
10648	{
10649	struct vm_area_struct *vma;
10650	VMA_ITERATOR(vmi, mm, 0);
10651
10652	for_each_vma(vmi, vma) {
10653	if (!vma->vm_file)
10654	continue;
10655
10656	if (perf_addr_filter_vma_adjust(filter, vma, fr))
10657	return;
10658	}
10659	}
10660
10661	/*
10662	* Update event's address range filters based on the
10663	* task's existing mappings, if any.
10664	*/
10665	static void perf_event_addr_filters_apply(struct perf_event *event)
10666	{
10667	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
10668	struct task_struct *task = READ_ONCE(event->ctx->task);
10669	struct perf_addr_filter *filter;
10670	struct mm_struct *mm = NULL;
10671	unsigned int count = 0;
10672	unsigned long flags;
10673
10674	/*
10675	* We may observe TASK_TOMBSTONE, which means that the event tear-down
10676	* will stop on the parent's child_mutex that our caller is also holding
10677	*/
10678	if (task == TASK_TOMBSTONE)
10679	return;
10680
10681	if (ifh->nr_file_filters) {
10682	mm = get_task_mm(task);
10683	if (!mm)
10684	goto restart;
10685
10686	mmap_read_lock(mm);
10687	}
10688
10689	raw_spin_lock_irqsave(&ifh->lock, flags);
10690	list_for_each_entry(filter, &ifh->list, entry) {
10691	if (filter->path.dentry) {
10692	/*
10693	* Adjust base offset if the filter is associated to a
10694	* binary that needs to be mapped:
10695	*/
10696	event->addr_filter_ranges[count].start = 0;
10697	event->addr_filter_ranges[count].size = 0;
10698
10699	perf_addr_filter_apply(filter, mm, fr: &event->addr_filter_ranges[count]);
10700	} else {
10701	event->addr_filter_ranges[count].start = filter->offset;
10702	event->addr_filter_ranges[count].size = filter->size;
10703	}
10704
10705	count++;
10706	}
10707
10708	event->addr_filters_gen++;
10709	raw_spin_unlock_irqrestore(&ifh->lock, flags);
10710
10711	if (ifh->nr_file_filters) {
10712	mmap_read_unlock(mm);
10713
10714	mmput(mm);
10715	}
10716
10717	restart:
10718	perf_event_stop(event, restart: 1);
10719	}
10720
10721	/*
10722	* Address range filtering: limiting the data to certain
10723	* instruction address ranges. Filters are ioctl()ed to us from
10724	* userspace as ascii strings.
10725	*
10726	* Filter string format:
10727	*
10728	* ACTION RANGE_SPEC
10729	* where ACTION is one of the
10730	* * "filter": limit the trace to this region
10731	* * "start": start tracing from this address
10732	* * "stop": stop tracing at this address/region;
10733	* RANGE_SPEC is
10734	* * for kernel addresses: <start address>[/<size>]
10735	* * for object files: <start address>[/<size>]@</path/to/object/file>
10736	*
10737	* if <size> is not specified or is zero, the range is treated as a single
10738	* address; not valid for ACTION=="filter".
10739	*/
10740	enum {
10741	IF_ACT_NONE = -1,
10742	IF_ACT_FILTER,
10743	IF_ACT_START,
10744	IF_ACT_STOP,
10745	IF_SRC_FILE,
10746	IF_SRC_KERNEL,
10747	IF_SRC_FILEADDR,
10748	IF_SRC_KERNELADDR,
10749	};
10750
10751	enum {
10752	IF_STATE_ACTION = 0,
10753	IF_STATE_SOURCE,
10754	IF_STATE_END,
10755	};
10756
10757	static const match_table_t if_tokens = {
10758	{ IF_ACT_FILTER, "filter" },
10759	{ IF_ACT_START, "start" },
10760	{ IF_ACT_STOP, "stop" },
10761	{ IF_SRC_FILE, "%u/%u@%s" },
10762	{ IF_SRC_KERNEL, "%u/%u" },
10763	{ IF_SRC_FILEADDR, "%u@%s" },
10764	{ IF_SRC_KERNELADDR, "%u" },
10765	{ IF_ACT_NONE, NULL },
10766	};
10767
10768	/*
10769	* Address filter string parser
10770	*/
10771	static int
10772	perf_event_parse_addr_filter(struct perf_event *event, char *fstr,
10773	struct list_head *filters)
10774	{
10775	struct perf_addr_filter *filter = NULL;
10776	char *start, *orig, *filename = NULL;
10777	substring_t args[MAX_OPT_ARGS];
10778	int state = IF_STATE_ACTION, token;
10779	unsigned int kernel = 0;
10780	int ret = -EINVAL;
10781
10782	orig = fstr = kstrdup(s: fstr, GFP_KERNEL);
10783	if (!fstr)
10784	return -ENOMEM;
10785
10786	while ((start = strsep(&fstr, " ,\n")) != NULL) {
10787	static const enum perf_addr_filter_action_t actions[] = {
10788	[IF_ACT_FILTER] = PERF_ADDR_FILTER_ACTION_FILTER,
10789	[IF_ACT_START] = PERF_ADDR_FILTER_ACTION_START,
10790	[IF_ACT_STOP] = PERF_ADDR_FILTER_ACTION_STOP,
10791	};
10792	ret = -EINVAL;
10793
10794	if (!*start)
10795	continue;
10796
10797	/* filter definition begins */
10798	if (state == IF_STATE_ACTION) {
10799	filter = perf_addr_filter_new(event, filters);
10800	if (!filter)
10801	goto fail;
10802	}
10803
10804	token = match_token(start, table: if_tokens, args);
10805	switch (token) {
10806	case IF_ACT_FILTER:
10807	case IF_ACT_START:
10808	case IF_ACT_STOP:
10809	if (state != IF_STATE_ACTION)
10810	goto fail;
10811
10812	filter->action = actions[token];
10813	state = IF_STATE_SOURCE;
10814	break;
10815
10816	case IF_SRC_KERNELADDR:
10817	case IF_SRC_KERNEL:
10818	kernel = 1;
10819	fallthrough;
10820
10821	case IF_SRC_FILEADDR:
10822	case IF_SRC_FILE:
10823	if (state != IF_STATE_SOURCE)
10824	goto fail;
10825
10826	*args[0].to = 0;
10827	ret = kstrtoul(s: args[0].from, base: 0, res: &filter->offset);
10828	if (ret)
10829	goto fail;
10830
10831	if (token == IF_SRC_KERNEL || token == IF_SRC_FILE) {
10832	*args[1].to = 0;
10833	ret = kstrtoul(s: args[1].from, base: 0, res: &filter->size);
10834	if (ret)
10835	goto fail;
10836	}
10837
10838	if (token == IF_SRC_FILE || token == IF_SRC_FILEADDR) {
10839	int fpos = token == IF_SRC_FILE ? 2 : 1;
10840
10841	kfree(objp: filename);
10842	filename = match_strdup(&args[fpos]);
10843	if (!filename) {
10844	ret = -ENOMEM;
10845	goto fail;
10846	}
10847	}
10848
10849	state = IF_STATE_END;
10850	break;
10851
10852	default:
10853	goto fail;
10854	}
10855
10856	/*
10857	* Filter definition is fully parsed, validate and install it.
10858	* Make sure that it doesn't contradict itself or the event's
10859	* attribute.
10860	*/
10861	if (state == IF_STATE_END) {
10862	ret = -EINVAL;
10863
10864	/*
10865	* ACTION "filter" must have a non-zero length region
10866	* specified.
10867	*/
10868	if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER &&
10869	!filter->size)
10870	goto fail;
10871
10872	if (!kernel) {
10873	if (!filename)
10874	goto fail;
10875
10876	/*
10877	* For now, we only support file-based filters
10878	* in per-task events; doing so for CPU-wide
10879	* events requires additional context switching
10880	* trickery, since same object code will be
10881	* mapped at different virtual addresses in
10882	* different processes.
10883	*/
10884	ret = -EOPNOTSUPP;
10885	if (!event->ctx->task)
10886	goto fail;
10887
10888	/* look up the path and grab its inode */
10889	ret = kern_path(filename, LOOKUP_FOLLOW,
10890	&filter->path);
10891	if (ret)
10892	goto fail;
10893
10894	ret = -EINVAL;
10895	if (!filter->path.dentry ||
10896	!S_ISREG(d_inode(filter->path.dentry)
10897	->i_mode))
10898	goto fail;
10899
10900	event->addr_filters.nr_file_filters++;
10901	}
10902
10903	/* ready to consume more filters */
10904	kfree(objp: filename);
10905	filename = NULL;
10906	state = IF_STATE_ACTION;
10907	filter = NULL;
10908	kernel = 0;
10909	}
10910	}
10911
10912	if (state != IF_STATE_ACTION)
10913	goto fail;
10914
10915	kfree(objp: filename);
10916	kfree(objp: orig);
10917
10918	return 0;
10919
10920	fail:
10921	kfree(objp: filename);
10922	free_filters_list(filters);
10923	kfree(objp: orig);
10924
10925	return ret;
10926	}
10927
10928	static int
10929	perf_event_set_addr_filter(struct perf_event *event, char *filter_str)
10930	{
10931	LIST_HEAD(filters);
10932	int ret;
10933
10934	/*
10935	* Since this is called in perf_ioctl() path, we're already holding
10936	* ctx::mutex.
10937	*/
10938	lockdep_assert_held(&event->ctx->mutex);
10939
10940	if (WARN_ON_ONCE(event->parent))
10941	return -EINVAL;
10942
10943	ret = perf_event_parse_addr_filter(event, fstr: filter_str, filters: &filters);
10944	if (ret)
10945	goto fail_clear_files;
10946
10947	ret = event->pmu->addr_filters_validate(&filters);
10948	if (ret)
10949	goto fail_free_filters;
10950
10951	/* remove existing filters, if any */
10952	perf_addr_filters_splice(event, head: &filters);
10953
10954	/* install new filters */
10955	perf_event_for_each_child(event, func: perf_event_addr_filters_apply);
10956
10957	return ret;
10958
10959	fail_free_filters:
10960	free_filters_list(filters: &filters);
10961
10962	fail_clear_files:
10963	event->addr_filters.nr_file_filters = 0;
10964
10965	return ret;
10966	}
10967
10968	static int perf_event_set_filter(struct perf_event *event, void __user *arg)
10969	{
10970	int ret = -EINVAL;
10971	char *filter_str;
10972
10973	filter_str = strndup_user(arg, PAGE_SIZE);
10974	if (IS_ERR(ptr: filter_str))
10975	return PTR_ERR(ptr: filter_str);
10976
10977	#ifdef CONFIG_EVENT_TRACING
10978	if (perf_event_is_tracing(event)) {
10979	struct perf_event_context *ctx = event->ctx;
10980
10981	/*
10982	* Beware, here be dragons!!
10983	*
10984	* the tracepoint muck will deadlock against ctx->mutex, but
10985	* the tracepoint stuff does not actually need it. So
10986	* temporarily drop ctx->mutex. As per perf_event_ctx_lock() we
10987	* already have a reference on ctx.
10988	*
10989	* This can result in event getting moved to a different ctx,
10990	* but that does not affect the tracepoint state.
10991	*/
10992	mutex_unlock(lock: &ctx->mutex);
10993	ret = ftrace_profile_set_filter(event, event_id: event->attr.config, filter_str);
10994	mutex_lock(&ctx->mutex);
10995	} else
10996	#endif
10997	if (has_addr_filter(event))
10998	ret = perf_event_set_addr_filter(event, filter_str);
10999
11000	kfree(objp: filter_str);
11001	return ret;
11002	}
11003
11004	/*
11005	* hrtimer based swevent callback
11006	*/
11007
11008	static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
11009	{
11010	enum hrtimer_restart ret = HRTIMER_RESTART;
11011	struct perf_sample_data data;
11012	struct pt_regs *regs;
11013	struct perf_event *event;
11014	u64 period;
11015
11016	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
11017
11018	if (event->state != PERF_EVENT_STATE_ACTIVE)
11019	return HRTIMER_NORESTART;
11020
11021	event->pmu->read(event);
11022
11023	perf_sample_data_init(data: &data, addr: 0, period: event->hw.last_period);
11024	regs = get_irq_regs();
11025
11026	if (regs && !perf_exclude_event(event, regs)) {
11027	if (!(event->attr.exclude_idle && is_idle_task(current)))
11028	if (__perf_event_overflow(event, throttle: 1, data: &data, regs))
11029	ret = HRTIMER_NORESTART;
11030	}
11031
11032	period = max_t(u64, 10000, event->hw.sample_period);
11033	hrtimer_forward_now(timer: hrtimer, interval: ns_to_ktime(ns: period));
11034
11035	return ret;
11036	}
11037
11038	static void perf_swevent_start_hrtimer(struct perf_event *event)
11039	{
11040	struct hw_perf_event *hwc = &event->hw;
11041	s64 period;
11042
11043	if (!is_sampling_event(event))
11044	return;
11045
11046	period = local64_read(&hwc->period_left);
11047	if (period) {
11048	if (period < 0)
11049	period = 10000;
11050
11051	local64_set(&hwc->period_left, 0);
11052	} else {
11053	period = max_t(u64, 10000, hwc->sample_period);
11054	}
11055	hrtimer_start(timer: &hwc->hrtimer, tim: ns_to_ktime(ns: period),
11056	mode: HRTIMER_MODE_REL_PINNED_HARD);
11057	}
11058
11059	static void perf_swevent_cancel_hrtimer(struct perf_event *event)
11060	{
11061	struct hw_perf_event *hwc = &event->hw;
11062
11063	if (is_sampling_event(event)) {
11064	ktime_t remaining = hrtimer_get_remaining(timer: &hwc->hrtimer);
11065	local64_set(&hwc->period_left, ktime_to_ns(remaining));
11066
11067	hrtimer_cancel(timer: &hwc->hrtimer);
11068	}
11069	}
11070
11071	static void perf_swevent_init_hrtimer(struct perf_event *event)
11072	{
11073	struct hw_perf_event *hwc = &event->hw;
11074
11075	if (!is_sampling_event(event))
11076	return;
11077
11078	hrtimer_init(timer: &hwc->hrtimer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL_HARD);
11079	hwc->hrtimer.function = perf_swevent_hrtimer;
11080
11081	/*
11082	* Since hrtimers have a fixed rate, we can do a static freq->period
11083	* mapping and avoid the whole period adjust feedback stuff.
11084	*/
11085	if (event->attr.freq) {
11086	long freq = event->attr.sample_freq;
11087
11088	event->attr.sample_period = NSEC_PER_SEC / freq;
11089	hwc->sample_period = event->attr.sample_period;
11090	local64_set(&hwc->period_left, hwc->sample_period);
11091	hwc->last_period = hwc->sample_period;
11092	event->attr.freq = 0;
11093	}
11094	}
11095
11096	/*
11097	* Software event: cpu wall time clock
11098	*/
11099
11100	static void cpu_clock_event_update(struct perf_event *event)
11101	{
11102	s64 prev;
11103	u64 now;
11104
11105	now = local_clock();
11106	prev = local64_xchg(&event->hw.prev_count, now);
11107	local64_add(now - prev, &event->count);
11108	}
11109
11110	static void cpu_clock_event_start(struct perf_event *event, int flags)
11111	{
11112	local64_set(&event->hw.prev_count, local_clock());
11113	perf_swevent_start_hrtimer(event);
11114	}
11115
11116	static void cpu_clock_event_stop(struct perf_event *event, int flags)
11117	{
11118	perf_swevent_cancel_hrtimer(event);
11119	cpu_clock_event_update(event);
11120	}
11121
11122	static int cpu_clock_event_add(struct perf_event *event, int flags)
11123	{
11124	if (flags & PERF_EF_START)
11125	cpu_clock_event_start(event, flags);
11126	perf_event_update_userpage(event);
11127
11128	return 0;
11129	}
11130
11131	static void cpu_clock_event_del(struct perf_event *event, int flags)
11132	{
11133	cpu_clock_event_stop(event, flags);
11134	}
11135
11136	static void cpu_clock_event_read(struct perf_event *event)
11137	{
11138	cpu_clock_event_update(event);
11139	}
11140
11141	static int cpu_clock_event_init(struct perf_event *event)
11142	{
11143	if (event->attr.type != perf_cpu_clock.type)
11144	return -ENOENT;
11145
11146	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
11147	return -ENOENT;
11148
11149	/*
11150	* no branch sampling for software events
11151	*/
11152	if (has_branch_stack(event))
11153	return -EOPNOTSUPP;
11154
11155	perf_swevent_init_hrtimer(event);
11156
11157	return 0;
11158	}
11159
11160	static struct pmu perf_cpu_clock = {
11161	.task_ctx_nr = perf_sw_context,
11162
11163	.capabilities = PERF_PMU_CAP_NO_NMI,
11164	.dev = PMU_NULL_DEV,
11165
11166	.event_init = cpu_clock_event_init,
11167	.add = cpu_clock_event_add,
11168	.del = cpu_clock_event_del,
11169	.start = cpu_clock_event_start,
11170	.stop = cpu_clock_event_stop,
11171	.read = cpu_clock_event_read,
11172	};
11173
11174	/*
11175	* Software event: task time clock
11176	*/
11177
11178	static void task_clock_event_update(struct perf_event *event, u64 now)
11179	{
11180	u64 prev;
11181	s64 delta;
11182
11183	prev = local64_xchg(&event->hw.prev_count, now);
11184	delta = now - prev;
11185	local64_add(delta, &event->count);
11186	}
11187
11188	static void task_clock_event_start(struct perf_event *event, int flags)
11189	{
11190	local64_set(&event->hw.prev_count, event->ctx->time);
11191	perf_swevent_start_hrtimer(event);
11192	}
11193
11194	static void task_clock_event_stop(struct perf_event *event, int flags)
11195	{
11196	perf_swevent_cancel_hrtimer(event);
11197	task_clock_event_update(event, now: event->ctx->time);
11198	}
11199
11200	static int task_clock_event_add(struct perf_event *event, int flags)
11201	{
11202	if (flags & PERF_EF_START)
11203	task_clock_event_start(event, flags);
11204	perf_event_update_userpage(event);
11205
11206	return 0;
11207	}
11208
11209	static void task_clock_event_del(struct perf_event *event, int flags)
11210	{
11211	task_clock_event_stop(event, PERF_EF_UPDATE);
11212	}
11213
11214	static void task_clock_event_read(struct perf_event *event)
11215	{
11216	u64 now = perf_clock();
11217	u64 delta = now - event->ctx->timestamp;
11218	u64 time = event->ctx->time + delta;
11219
11220	task_clock_event_update(event, now: time);
11221	}
11222
11223	static int task_clock_event_init(struct perf_event *event)
11224	{
11225	if (event->attr.type != perf_task_clock.type)
11226	return -ENOENT;
11227
11228	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
11229	return -ENOENT;
11230
11231	/*
11232	* no branch sampling for software events
11233	*/
11234	if (has_branch_stack(event))
11235	return -EOPNOTSUPP;
11236
11237	perf_swevent_init_hrtimer(event);
11238
11239	return 0;
11240	}
11241
11242	static struct pmu perf_task_clock = {
11243	.task_ctx_nr = perf_sw_context,
11244
11245	.capabilities = PERF_PMU_CAP_NO_NMI,
11246	.dev = PMU_NULL_DEV,
11247
11248	.event_init = task_clock_event_init,
11249	.add = task_clock_event_add,
11250	.del = task_clock_event_del,
11251	.start = task_clock_event_start,
11252	.stop = task_clock_event_stop,
11253	.read = task_clock_event_read,
11254	};
11255
11256	static void perf_pmu_nop_void(struct pmu *pmu)
11257	{
11258	}
11259
11260	static void perf_pmu_nop_txn(struct pmu *pmu, unsigned int flags)
11261	{
11262	}
11263
11264	static int perf_pmu_nop_int(struct pmu *pmu)
11265	{
11266	return 0;
11267	}
11268
11269	static int perf_event_nop_int(struct perf_event *event, u64 value)
11270	{
11271	return 0;
11272	}
11273
11274	static DEFINE_PER_CPU(unsigned int, nop_txn_flags);
11275
11276	static void perf_pmu_start_txn(struct pmu *pmu, unsigned int flags)
11277	{
11278	__this_cpu_write(nop_txn_flags, flags);
11279
11280	if (flags & ~PERF_PMU_TXN_ADD)
11281	return;
11282
11283	perf_pmu_disable(pmu);
11284	}
11285
11286	static int perf_pmu_commit_txn(struct pmu *pmu)
11287	{
11288	unsigned int flags = __this_cpu_read(nop_txn_flags);
11289
11290	__this_cpu_write(nop_txn_flags, 0);
11291
11292	if (flags & ~PERF_PMU_TXN_ADD)
11293	return 0;
11294
11295	perf_pmu_enable(pmu);
11296	return 0;
11297	}
11298
11299	static void perf_pmu_cancel_txn(struct pmu *pmu)
11300	{
11301	unsigned int flags = __this_cpu_read(nop_txn_flags);
11302
11303	__this_cpu_write(nop_txn_flags, 0);
11304
11305	if (flags & ~PERF_PMU_TXN_ADD)
11306	return;
11307
11308	perf_pmu_enable(pmu);
11309	}
11310
11311	static int perf_event_idx_default(struct perf_event *event)
11312	{
11313	return 0;
11314	}
11315
11316	static void free_pmu_context(struct pmu *pmu)
11317	{
11318	free_percpu(pdata: pmu->cpu_pmu_context);
11319	}
11320
11321	/*
11322	* Let userspace know that this PMU supports address range filtering:
11323	*/
11324	static ssize_t nr_addr_filters_show(struct device *dev,
11325	struct device_attribute *attr,
11326	char *page)
11327	{
11328	struct pmu *pmu = dev_get_drvdata(dev);
11329
11330	return scnprintf(buf: page, PAGE_SIZE - 1, fmt: "%d\n", pmu->nr_addr_filters);
11331	}
11332	DEVICE_ATTR_RO(nr_addr_filters);
11333
11334	static struct idr pmu_idr;
11335
11336	static ssize_t
11337	type_show(struct device *dev, struct device_attribute *attr, char *page)
11338	{
11339	struct pmu *pmu = dev_get_drvdata(dev);
11340
11341	return scnprintf(buf: page, PAGE_SIZE - 1, fmt: "%d\n", pmu->type);
11342	}
11343	static DEVICE_ATTR_RO(type);
11344
11345	static ssize_t
11346	perf_event_mux_interval_ms_show(struct device *dev,
11347	struct device_attribute *attr,
11348	char *page)
11349	{
11350	struct pmu *pmu = dev_get_drvdata(dev);
11351
11352	return scnprintf(buf: page, PAGE_SIZE - 1, fmt: "%d\n", pmu->hrtimer_interval_ms);
11353	}
11354
11355	static DEFINE_MUTEX(mux_interval_mutex);
11356
11357	static ssize_t
11358	perf_event_mux_interval_ms_store(struct device *dev,
11359	struct device_attribute *attr,
11360	const char *buf, size_t count)
11361	{
11362	struct pmu *pmu = dev_get_drvdata(dev);
11363	int timer, cpu, ret;
11364
11365	ret = kstrtoint(s: buf, base: 0, res: &timer);
11366	if (ret)
11367	return ret;
11368
11369	if (timer < 1)
11370	return -EINVAL;
11371
11372	/* same value, noting to do */
11373	if (timer == pmu->hrtimer_interval_ms)
11374	return count;
11375
11376	mutex_lock(&mux_interval_mutex);
11377	pmu->hrtimer_interval_ms = timer;
11378
11379	/* update all cpuctx for this PMU */
11380	cpus_read_lock();
11381	for_each_online_cpu(cpu) {
11382	struct perf_cpu_pmu_context *cpc;
11383	cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu);
11384	cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
11385
11386	cpu_function_call(cpu, func: perf_mux_hrtimer_restart_ipi, info: cpc);
11387	}
11388	cpus_read_unlock();
11389	mutex_unlock(lock: &mux_interval_mutex);
11390
11391	return count;
11392	}
11393	static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
11394
11395	static struct attribute *pmu_dev_attrs[] = {
11396	&dev_attr_type.attr,
11397	&dev_attr_perf_event_mux_interval_ms.attr,
11398	NULL,
11399	};
11400	ATTRIBUTE_GROUPS(pmu_dev);
11401
11402	static int pmu_bus_running;
11403	static struct bus_type pmu_bus = {
11404	.name = "event_source",
11405	.dev_groups = pmu_dev_groups,
11406	};
11407
11408	static void pmu_dev_release(struct device *dev)
11409	{
11410	kfree(objp: dev);
11411	}
11412
11413	static int pmu_dev_alloc(struct pmu *pmu)
11414	{
11415	int ret = -ENOMEM;
11416
11417	pmu->dev = kzalloc(size: sizeof(struct device), GFP_KERNEL);
11418	if (!pmu->dev)
11419	goto out;
11420
11421	pmu->dev->groups = pmu->attr_groups;
11422	device_initialize(dev: pmu->dev);
11423
11424	dev_set_drvdata(dev: pmu->dev, data: pmu);
11425	pmu->dev->bus = &pmu_bus;
11426	pmu->dev->parent = pmu->parent;
11427	pmu->dev->release = pmu_dev_release;
11428
11429	ret = dev_set_name(dev: pmu->dev, name: "%s", pmu->name);
11430	if (ret)
11431	goto free_dev;
11432
11433	ret = device_add(dev: pmu->dev);
11434	if (ret)
11435	goto free_dev;
11436
11437	/* For PMUs with address filters, throw in an extra attribute: */
11438	if (pmu->nr_addr_filters)
11439	ret = device_create_file(device: pmu->dev, entry: &dev_attr_nr_addr_filters);
11440
11441	if (ret)
11442	goto del_dev;
11443
11444	if (pmu->attr_update)
11445	ret = sysfs_update_groups(kobj: &pmu->dev->kobj, groups: pmu->attr_update);
11446
11447	if (ret)
11448	goto del_dev;
11449
11450	out:
11451	return ret;
11452
11453	del_dev:
11454	device_del(dev: pmu->dev);
11455
11456	free_dev:
11457	put_device(dev: pmu->dev);
11458	goto out;
11459	}
11460
11461	static struct lock_class_key cpuctx_mutex;
11462	static struct lock_class_key cpuctx_lock;
11463
11464	int perf_pmu_register(struct pmu *pmu, const char *name, int type)
11465	{
11466	int cpu, ret, max = PERF_TYPE_MAX;
11467
11468	mutex_lock(&pmus_lock);
11469	ret = -ENOMEM;
11470	pmu->pmu_disable_count = alloc_percpu(int);
11471	if (!pmu->pmu_disable_count)
11472	goto unlock;
11473
11474	pmu->type = -1;
11475	if (WARN_ONCE(!name, "Can not register anonymous pmu.\n")) {
11476	ret = -EINVAL;
11477	goto free_pdc;
11478	}
11479
11480	pmu->name = name;
11481
11482	if (type >= 0)
11483	max = type;
11484
11485	ret = idr_alloc(&pmu_idr, ptr: pmu, start: max, end: 0, GFP_KERNEL);
11486	if (ret < 0)
11487	goto free_pdc;
11488
11489	WARN_ON(type >= 0 && ret != type);
11490
11491	type = ret;
11492	pmu->type = type;
11493
11494	if (pmu_bus_running && !pmu->dev) {
11495	ret = pmu_dev_alloc(pmu);
11496	if (ret)
11497	goto free_idr;
11498	}
11499
11500	ret = -ENOMEM;
11501	pmu->cpu_pmu_context = alloc_percpu(struct perf_cpu_pmu_context);
11502	if (!pmu->cpu_pmu_context)
11503	goto free_dev;
11504
11505	for_each_possible_cpu(cpu) {
11506	struct perf_cpu_pmu_context *cpc;
11507
11508	cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu);
11509	__perf_init_event_pmu_context(epc: &cpc->epc, pmu);
11510	__perf_mux_hrtimer_init(cpc, cpu);
11511	}
11512
11513	if (!pmu->start_txn) {
11514	if (pmu->pmu_enable) {
11515	/*
11516	* If we have pmu_enable/pmu_disable calls, install
11517	* transaction stubs that use that to try and batch
11518	* hardware accesses.
11519	*/
11520	pmu->start_txn = perf_pmu_start_txn;
11521	pmu->commit_txn = perf_pmu_commit_txn;
11522	pmu->cancel_txn = perf_pmu_cancel_txn;
11523	} else {
11524	pmu->start_txn = perf_pmu_nop_txn;
11525	pmu->commit_txn = perf_pmu_nop_int;
11526	pmu->cancel_txn = perf_pmu_nop_void;
11527	}
11528	}
11529
11530	if (!pmu->pmu_enable) {
11531	pmu->pmu_enable = perf_pmu_nop_void;
11532	pmu->pmu_disable = perf_pmu_nop_void;
11533	}
11534
11535	if (!pmu->check_period)
11536	pmu->check_period = perf_event_nop_int;
11537
11538	if (!pmu->event_idx)
11539	pmu->event_idx = perf_event_idx_default;
11540
11541	list_add_rcu(new: &pmu->entry, head: &pmus);
11542	atomic_set(v: &pmu->exclusive_cnt, i: 0);
11543	ret = 0;
11544	unlock:
11545	mutex_unlock(lock: &pmus_lock);
11546
11547	return ret;
11548
11549	free_dev:
11550	if (pmu->dev && pmu->dev != PMU_NULL_DEV) {
11551	device_del(dev: pmu->dev);
11552	put_device(dev: pmu->dev);
11553	}
11554
11555	free_idr:
11556	idr_remove(&pmu_idr, id: pmu->type);
11557
11558	free_pdc:
11559	free_percpu(pdata: pmu->pmu_disable_count);
11560	goto unlock;
11561	}
11562	EXPORT_SYMBOL_GPL(perf_pmu_register);
11563
11564	void perf_pmu_unregister(struct pmu *pmu)
11565	{
11566	mutex_lock(&pmus_lock);
11567	list_del_rcu(entry: &pmu->entry);
11568
11569	/*
11570	* We dereference the pmu list under both SRCU and regular RCU, so
11571	* synchronize against both of those.
11572	*/
11573	synchronize_srcu(ssp: &pmus_srcu);
11574	synchronize_rcu();
11575
11576	free_percpu(pdata: pmu->pmu_disable_count);
11577	idr_remove(&pmu_idr, id: pmu->type);
11578	if (pmu_bus_running && pmu->dev && pmu->dev != PMU_NULL_DEV) {
11579	if (pmu->nr_addr_filters)
11580	device_remove_file(dev: pmu->dev, attr: &dev_attr_nr_addr_filters);
11581	device_del(dev: pmu->dev);
11582	put_device(dev: pmu->dev);
11583	}
11584	free_pmu_context(pmu);
11585	mutex_unlock(lock: &pmus_lock);
11586	}
11587	EXPORT_SYMBOL_GPL(perf_pmu_unregister);
11588
11589	static inline bool has_extended_regs(struct perf_event *event)
11590	{
11591	return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) ||
11592	(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK);
11593	}
11594
11595	static int perf_try_init_event(struct pmu *pmu, struct perf_event *event)
11596	{
11597	struct perf_event_context *ctx = NULL;
11598	int ret;
11599
11600	if (!try_module_get(module: pmu->module))
11601	return -ENODEV;
11602
11603	/*
11604	* A number of pmu->event_init() methods iterate the sibling_list to,
11605	* for example, validate if the group fits on the PMU. Therefore,
11606	* if this is a sibling event, acquire the ctx->mutex to protect
11607	* the sibling_list.
11608	*/
11609	if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) {
11610	/*
11611	* This ctx->mutex can nest when we're called through
11612	* inheritance. See the perf_event_ctx_lock_nested() comment.
11613	*/
11614	ctx = perf_event_ctx_lock_nested(event: event->group_leader,
11615	SINGLE_DEPTH_NESTING);
11616	BUG_ON(!ctx);
11617	}
11618
11619	event->pmu = pmu;
11620	ret = pmu->event_init(event);
11621
11622	if (ctx)
11623	perf_event_ctx_unlock(event: event->group_leader, ctx);
11624
11625	if (!ret) {
11626	if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) &&
11627	has_extended_regs(event))
11628	ret = -EOPNOTSUPP;
11629
11630	if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE &&
11631	event_has_any_exclude_flag(event))
11632	ret = -EINVAL;
11633
11634	if (ret && event->destroy)
11635	event->destroy(event);
11636	}
11637
11638	if (ret)
11639	module_put(module: pmu->module);
11640
11641	return ret;
11642	}
11643
11644	static struct pmu *perf_init_event(struct perf_event *event)
11645	{
11646	bool extended_type = false;
11647	int idx, type, ret;
11648	struct pmu *pmu;
11649
11650	idx = srcu_read_lock(ssp: &pmus_srcu);
11651
11652	/*
11653	* Save original type before calling pmu->event_init() since certain
11654	* pmus overwrites event->attr.type to forward event to another pmu.
11655	*/
11656	event->orig_type = event->attr.type;
11657
11658	/* Try parent's PMU first: */
11659	if (event->parent && event->parent->pmu) {
11660	pmu = event->parent->pmu;
11661	ret = perf_try_init_event(pmu, event);
11662	if (!ret)
11663	goto unlock;
11664	}
11665
11666	/*
11667	* PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
11668	* are often aliases for PERF_TYPE_RAW.
11669	*/
11670	type = event->attr.type;
11671	if (type == PERF_TYPE_HARDWARE || type == PERF_TYPE_HW_CACHE) {
11672	type = event->attr.config >> PERF_PMU_TYPE_SHIFT;
11673	if (!type) {
11674	type = PERF_TYPE_RAW;
11675	} else {
11676	extended_type = true;
11677	event->attr.config &= PERF_HW_EVENT_MASK;
11678	}
11679	}
11680
11681	again:
11682	rcu_read_lock();
11683	pmu = idr_find(&pmu_idr, id: type);
11684	rcu_read_unlock();
11685	if (pmu) {
11686	if (event->attr.type != type && type != PERF_TYPE_RAW &&
11687	!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE))
11688	goto fail;
11689
11690	ret = perf_try_init_event(pmu, event);
11691	if (ret == -ENOENT && event->attr.type != type && !extended_type) {
11692	type = event->attr.type;
11693	goto again;
11694	}
11695
11696	if (ret)
11697	pmu = ERR_PTR(error: ret);
11698
11699	goto unlock;
11700	}
11701
11702	list_for_each_entry_rcu(pmu, &pmus, entry, lockdep_is_held(&pmus_srcu)) {
11703	ret = perf_try_init_event(pmu, event);
11704	if (!ret)
11705	goto unlock;
11706
11707	if (ret != -ENOENT) {
11708	pmu = ERR_PTR(error: ret);
11709	goto unlock;
11710	}
11711	}
11712	fail:
11713	pmu = ERR_PTR(error: -ENOENT);
11714	unlock:
11715	srcu_read_unlock(ssp: &pmus_srcu, idx);
11716
11717	return pmu;
11718	}
11719
11720	static void attach_sb_event(struct perf_event *event)
11721	{
11722	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
11723
11724	raw_spin_lock(&pel->lock);
11725	list_add_rcu(new: &event->sb_list, head: &pel->list);
11726	raw_spin_unlock(&pel->lock);
11727	}
11728
11729	/*
11730	* We keep a list of all !task (and therefore per-cpu) events
11731	* that need to receive side-band records.
11732	*
11733	* This avoids having to scan all the various PMU per-cpu contexts
11734	* looking for them.
11735	*/
11736	static void account_pmu_sb_event(struct perf_event *event)
11737	{
11738	if (is_sb_event(event))
11739	attach_sb_event(event);
11740	}
11741
11742	/* Freq events need the tick to stay alive (see perf_event_task_tick). */
11743	static void account_freq_event_nohz(void)
11744	{
11745	#ifdef CONFIG_NO_HZ_FULL
11746	/* Lock so we don't race with concurrent unaccount */
11747	spin_lock(&nr_freq_lock);
11748	if (atomic_inc_return(&nr_freq_events) == 1)
11749	tick_nohz_dep_set(TICK_DEP_BIT_PERF_EVENTS);
11750	spin_unlock(&nr_freq_lock);
11751	#endif
11752	}
11753
11754	static void account_freq_event(void)
11755	{
11756	if (tick_nohz_full_enabled())
11757	account_freq_event_nohz();
11758	else
11759	atomic_inc(v: &nr_freq_events);
11760	}
11761
11762
11763	static void account_event(struct perf_event *event)
11764	{
11765	bool inc = false;
11766
11767	if (event->parent)
11768	return;
11769
11770	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
11771	inc = true;
11772	if (event->attr.mmap || event->attr.mmap_data)
11773	atomic_inc(v: &nr_mmap_events);
11774	if (event->attr.build_id)
11775	atomic_inc(v: &nr_build_id_events);
11776	if (event->attr.comm)
11777	atomic_inc(v: &nr_comm_events);
11778	if (event->attr.namespaces)
11779	atomic_inc(v: &nr_namespaces_events);
11780	if (event->attr.cgroup)
11781	atomic_inc(v: &nr_cgroup_events);
11782	if (event->attr.task)
11783	atomic_inc(v: &nr_task_events);
11784	if (event->attr.freq)
11785	account_freq_event();
11786	if (event->attr.context_switch) {
11787	atomic_inc(v: &nr_switch_events);
11788	inc = true;
11789	}
11790	if (has_branch_stack(event))
11791	inc = true;
11792	if (is_cgroup_event(event))
11793	inc = true;
11794	if (event->attr.ksymbol)
11795	atomic_inc(v: &nr_ksymbol_events);
11796	if (event->attr.bpf_event)
11797	atomic_inc(v: &nr_bpf_events);
11798	if (event->attr.text_poke)
11799	atomic_inc(v: &nr_text_poke_events);
11800
11801	if (inc) {
11802	/*
11803	* We need the mutex here because static_branch_enable()
11804	* must complete *before* the perf_sched_count increment
11805	* becomes visible.
11806	*/
11807	if (atomic_inc_not_zero(v: &perf_sched_count))
11808	goto enabled;
11809
11810	mutex_lock(&perf_sched_mutex);
11811	if (!atomic_read(v: &perf_sched_count)) {
11812	static_branch_enable(&perf_sched_events);
11813	/*
11814	* Guarantee that all CPUs observe they key change and
11815	* call the perf scheduling hooks before proceeding to
11816	* install events that need them.
11817	*/
11818	synchronize_rcu();
11819	}
11820	/*
11821	* Now that we have waited for the sync_sched(), allow further
11822	* increments to by-pass the mutex.
11823	*/
11824	atomic_inc(v: &perf_sched_count);
11825	mutex_unlock(lock: &perf_sched_mutex);
11826	}
11827	enabled:
11828
11829	account_pmu_sb_event(event);
11830	}
11831
11832	/*
11833	* Allocate and initialize an event structure
11834	*/
11835	static struct perf_event *
11836	perf_event_alloc(struct perf_event_attr *attr, int cpu,
11837	struct task_struct *task,
11838	struct perf_event *group_leader,
11839	struct perf_event *parent_event,
11840	perf_overflow_handler_t overflow_handler,
11841	void *context, int cgroup_fd)
11842	{
11843	struct pmu *pmu;
11844	struct perf_event *event;
11845	struct hw_perf_event *hwc;
11846	long err = -EINVAL;
11847	int node;
11848
11849	if ((unsigned)cpu >= nr_cpu_ids) {
11850	if (!task || cpu != -1)
11851	return ERR_PTR(error: -EINVAL);
11852	}
11853	if (attr->sigtrap && !task) {
11854	/* Requires a task: avoid signalling random tasks. */
11855	return ERR_PTR(error: -EINVAL);
11856	}
11857
11858	node = (cpu >= 0) ? cpu_to_node(cpu) : -1;
11859	event = kmem_cache_alloc_node(s: perf_event_cache, GFP_KERNEL | __GFP_ZERO,
11860	node);
11861	if (!event)
11862	return ERR_PTR(error: -ENOMEM);
11863
11864	/*
11865	* Single events are their own group leaders, with an
11866	* empty sibling list:
11867	*/
11868	if (!group_leader)
11869	group_leader = event;
11870
11871	mutex_init(&event->child_mutex);
11872	INIT_LIST_HEAD(list: &event->child_list);
11873
11874	INIT_LIST_HEAD(list: &event->event_entry);
11875	INIT_LIST_HEAD(list: &event->sibling_list);
11876	INIT_LIST_HEAD(list: &event->active_list);
11877	init_event_group(event);
11878	INIT_LIST_HEAD(list: &event->rb_entry);
11879	INIT_LIST_HEAD(list: &event->active_entry);
11880	INIT_LIST_HEAD(list: &event->addr_filters.list);
11881	INIT_HLIST_NODE(h: &event->hlist_entry);
11882
11883
11884	init_waitqueue_head(&event->waitq);
11885	init_irq_work(work: &event->pending_irq, func: perf_pending_irq);
11886	init_task_work(twork: &event->pending_task, func: perf_pending_task);
11887
11888	mutex_init(&event->mmap_mutex);
11889	raw_spin_lock_init(&event->addr_filters.lock);
11890
11891	atomic_long_set(v: &event->refcount, i: 1);
11892	event->cpu = cpu;
11893	event->attr = *attr;
11894	event->group_leader = group_leader;
11895	event->pmu = NULL;
11896	event->oncpu = -1;
11897
11898	event->parent = parent_event;
11899
11900	event->ns = get_pid_ns(ns: task_active_pid_ns(current));
11901	event->id = atomic64_inc_return(v: &perf_event_id);
11902
11903	event->state = PERF_EVENT_STATE_INACTIVE;
11904
11905	if (parent_event)
11906	event->event_caps = parent_event->event_caps;
11907
11908	if (task) {
11909	event->attach_state = PERF_ATTACH_TASK;
11910	/*
11911	* XXX pmu::event_init needs to know what task to account to
11912	* and we cannot use the ctx information because we need the
11913	* pmu before we get a ctx.
11914	*/
11915	event->hw.target = get_task_struct(t: task);
11916	}
11917
11918	event->clock = &local_clock;
11919	if (parent_event)
11920	event->clock = parent_event->clock;
11921
11922	if (!overflow_handler && parent_event) {
11923	overflow_handler = parent_event->overflow_handler;
11924	context = parent_event->overflow_handler_context;
11925	#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING)
11926	if (overflow_handler == bpf_overflow_handler) {
11927	struct bpf_prog *prog = parent_event->prog;
11928
11929	bpf_prog_inc(prog);
11930	event->prog = prog;
11931	event->orig_overflow_handler =
11932	parent_event->orig_overflow_handler;
11933	}
11934	#endif
11935	}
11936
11937	if (overflow_handler) {
11938	event->overflow_handler = overflow_handler;
11939	event->overflow_handler_context = context;
11940	} else if (is_write_backward(event)){
11941	event->overflow_handler = perf_event_output_backward;
11942	event->overflow_handler_context = NULL;
11943	} else {
11944	event->overflow_handler = perf_event_output_forward;
11945	event->overflow_handler_context = NULL;
11946	}
11947
11948	perf_event__state_init(event);
11949
11950	pmu = NULL;
11951
11952	hwc = &event->hw;
11953	hwc->sample_period = attr->sample_period;
11954	if (attr->freq && attr->sample_freq)
11955	hwc->sample_period = 1;
11956	hwc->last_period = hwc->sample_period;
11957
11958	local64_set(&hwc->period_left, hwc->sample_period);
11959
11960	/*
11961	* We currently do not support PERF_SAMPLE_READ on inherited events.
11962	* See perf_output_read().
11963	*/
11964	if (attr->inherit && (attr->sample_type & PERF_SAMPLE_READ))
11965	goto err_ns;
11966
11967	if (!has_branch_stack(event))
11968	event->attr.branch_sample_type = 0;
11969
11970	pmu = perf_init_event(event);
11971	if (IS_ERR(ptr: pmu)) {
11972	err = PTR_ERR(ptr: pmu);
11973	goto err_ns;
11974	}
11975
11976	/*
11977	* Disallow uncore-task events. Similarly, disallow uncore-cgroup
11978	* events (they don't make sense as the cgroup will be different
11979	* on other CPUs in the uncore mask).
11980	*/
11981	if (pmu->task_ctx_nr == perf_invalid_context && (task || cgroup_fd != -1)) {
11982	err = -EINVAL;
11983	goto err_pmu;
11984	}
11985
11986	if (event->attr.aux_output &&
11987	!(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT)) {
11988	err = -EOPNOTSUPP;
11989	goto err_pmu;
11990	}
11991
11992	if (cgroup_fd != -1) {
11993	err = perf_cgroup_connect(fd: cgroup_fd, event, attr, group_leader);
11994	if (err)
11995	goto err_pmu;
11996	}
11997
11998	err = exclusive_event_init(event);
11999	if (err)
12000	goto err_pmu;
12001
12002	if (has_addr_filter(event)) {
12003	event->addr_filter_ranges = kcalloc(n: pmu->nr_addr_filters,
12004	size: sizeof(struct perf_addr_filter_range),
12005	GFP_KERNEL);
12006	if (!event->addr_filter_ranges) {
12007	err = -ENOMEM;
12008	goto err_per_task;
12009	}
12010
12011	/*
12012	* Clone the parent's vma offsets: they are valid until exec()
12013	* even if the mm is not shared with the parent.
12014	*/
12015	if (event->parent) {
12016	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
12017
12018	raw_spin_lock_irq(&ifh->lock);
12019	memcpy(event->addr_filter_ranges,
12020	event->parent->addr_filter_ranges,
12021	pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range));
12022	raw_spin_unlock_irq(&ifh->lock);
12023	}
12024
12025	/* force hw sync on the address filters */
12026	event->addr_filters_gen = 1;
12027	}
12028
12029	if (!event->parent) {
12030	if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
12031	err = get_callchain_buffers(max_stack: attr->sample_max_stack);
12032	if (err)
12033	goto err_addr_filters;
12034	}
12035	}
12036
12037	err = security_perf_event_alloc(event);
12038	if (err)
12039	goto err_callchain_buffer;
12040
12041	/* symmetric to unaccount_event() in _free_event() */
12042	account_event(event);
12043
12044	return event;
12045
12046	err_callchain_buffer:
12047	if (!event->parent) {
12048	if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
12049	put_callchain_buffers();
12050	}
12051	err_addr_filters:
12052	kfree(objp: event->addr_filter_ranges);
12053
12054	err_per_task:
12055	exclusive_event_destroy(event);
12056
12057	err_pmu:
12058	if (is_cgroup_event(event))
12059	perf_detach_cgroup(event);
12060	if (event->destroy)
12061	event->destroy(event);
12062	module_put(module: pmu->module);
12063	err_ns:
12064	if (event->hw.target)
12065	put_task_struct(t: event->hw.target);
12066	call_rcu(head: &event->rcu_head, func: free_event_rcu);
12067
12068	return ERR_PTR(error: err);
12069	}
12070
12071	static int perf_copy_attr(struct perf_event_attr __user *uattr,
12072	struct perf_event_attr *attr)
12073	{
12074	u32 size;
12075	int ret;
12076
12077	/* Zero the full structure, so that a short copy will be nice. */
12078	memset(attr, 0, sizeof(*attr));
12079
12080	ret = get_user(size, &uattr->size);
12081	if (ret)
12082	return ret;
12083
12084	/* ABI compatibility quirk: */
12085	if (!size)
12086	size = PERF_ATTR_SIZE_VER0;
12087	if (size < PERF_ATTR_SIZE_VER0 || size > PAGE_SIZE)
12088	goto err_size;
12089
12090	ret = copy_struct_from_user(dst: attr, ksize: sizeof(*attr), src: uattr, usize: size);
12091	if (ret) {
12092	if (ret == -E2BIG)
12093	goto err_size;
12094	return ret;
12095	}
12096
12097	attr->size = size;
12098
12099	if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)
12100	return -EINVAL;
12101
12102	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
12103	return -EINVAL;
12104
12105	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
12106	return -EINVAL;
12107
12108	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
12109	u64 mask = attr->branch_sample_type;
12110
12111	/* only using defined bits */
12112	if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
12113	return -EINVAL;
12114
12115	/* at least one branch bit must be set */
12116	if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
12117	return -EINVAL;
12118
12119	/* propagate priv level, when not set for branch */
12120	if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
12121
12122	/* exclude_kernel checked on syscall entry */
12123	if (!attr->exclude_kernel)
12124	mask |= PERF_SAMPLE_BRANCH_KERNEL;
12125
12126	if (!attr->exclude_user)
12127	mask |= PERF_SAMPLE_BRANCH_USER;
12128
12129	if (!attr->exclude_hv)
12130	mask |= PERF_SAMPLE_BRANCH_HV;
12131	/*
12132	* adjust user setting (for HW filter setup)
12133	*/
12134	attr->branch_sample_type = mask;
12135	}
12136	/* privileged levels capture (kernel, hv): check permissions */
12137	if (mask & PERF_SAMPLE_BRANCH_PERM_PLM) {
12138	ret = perf_allow_kernel(attr);
12139	if (ret)
12140	return ret;
12141	}
12142	}
12143
12144	if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
12145	ret = perf_reg_validate(mask: attr->sample_regs_user);
12146	if (ret)
12147	return ret;
12148	}
12149
12150	if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
12151	if (!arch_perf_have_user_stack_dump())
12152	return -ENOSYS;
12153
12154	/*
12155	* We have __u32 type for the size, but so far
12156	* we can only use __u16 as maximum due to the
12157	* __u16 sample size limit.
12158	*/
12159	if (attr->sample_stack_user >= USHRT_MAX)
12160	return -EINVAL;
12161	else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
12162	return -EINVAL;
12163	}
12164
12165	if (!attr->sample_max_stack)
12166	attr->sample_max_stack = sysctl_perf_event_max_stack;
12167
12168	if (attr->sample_type & PERF_SAMPLE_REGS_INTR)
12169	ret = perf_reg_validate(mask: attr->sample_regs_intr);
12170
12171	#ifndef CONFIG_CGROUP_PERF
12172	if (attr->sample_type & PERF_SAMPLE_CGROUP)
12173	return -EINVAL;
12174	#endif
12175	if ((attr->sample_type & PERF_SAMPLE_WEIGHT) &&
12176	(attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT))
12177	return -EINVAL;
12178
12179	if (!attr->inherit && attr->inherit_thread)
12180	return -EINVAL;
12181
12182	if (attr->remove_on_exec && attr->enable_on_exec)
12183	return -EINVAL;
12184
12185	if (attr->sigtrap && !attr->remove_on_exec)
12186	return -EINVAL;
12187
12188	out:
12189	return ret;
12190
12191	err_size:
12192	put_user(sizeof(*attr), &uattr->size);
12193	ret = -E2BIG;
12194	goto out;
12195	}
12196
12197	static void mutex_lock_double(struct mutex *a, struct mutex *b)
12198	{
12199	if (b < a)
12200	swap(a, b);
12201
12202	mutex_lock(a);
12203	mutex_lock_nested(lock: b, SINGLE_DEPTH_NESTING);
12204	}
12205
12206	static int
12207	perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
12208	{
12209	struct perf_buffer *rb = NULL;
12210	int ret = -EINVAL;
12211
12212	if (!output_event) {
12213	mutex_lock(&event->mmap_mutex);
12214	goto set;
12215	}
12216
12217	/* don't allow circular references */
12218	if (event == output_event)
12219	goto out;
12220
12221	/*
12222	* Don't allow cross-cpu buffers
12223	*/
12224	if (output_event->cpu != event->cpu)
12225	goto out;
12226
12227	/*
12228	* If its not a per-cpu rb, it must be the same task.
12229	*/
12230	if (output_event->cpu == -1 && output_event->hw.target != event->hw.target)
12231	goto out;
12232
12233	/*
12234	* Mixing clocks in the same buffer is trouble you don't need.
12235	*/
12236	if (output_event->clock != event->clock)
12237	goto out;
12238
12239	/*
12240	* Either writing ring buffer from beginning or from end.
12241	* Mixing is not allowed.
12242	*/
12243	if (is_write_backward(event: output_event) != is_write_backward(event))
12244	goto out;
12245
12246	/*
12247	* If both events generate aux data, they must be on the same PMU
12248	*/
12249	if (has_aux(event) && has_aux(event: output_event) &&
12250	event->pmu != output_event->pmu)
12251	goto out;
12252
12253	/*
12254	* Hold both mmap_mutex to serialize against perf_mmap_close(). Since
12255	* output_event is already on rb->event_list, and the list iteration
12256	* restarts after every removal, it is guaranteed this new event is
12257	* observed *OR* if output_event is already removed, it's guaranteed we
12258	* observe !rb->mmap_count.
12259	*/
12260	mutex_lock_double(a: &event->mmap_mutex, b: &output_event->mmap_mutex);
12261	set:
12262	/* Can't redirect output if we've got an active mmap() */
12263	if (atomic_read(v: &event->mmap_count))
12264	goto unlock;
12265
12266	if (output_event) {
12267	/* get the rb we want to redirect to */
12268	rb = ring_buffer_get(event: output_event);
12269	if (!rb)
12270	goto unlock;
12271
12272	/* did we race against perf_mmap_close() */
12273	if (!atomic_read(v: &rb->mmap_count)) {
12274	ring_buffer_put(rb);
12275	goto unlock;
12276	}
12277	}
12278
12279	ring_buffer_attach(event, rb);
12280
12281	ret = 0;
12282	unlock:
12283	mutex_unlock(lock: &event->mmap_mutex);
12284	if (output_event)
12285	mutex_unlock(lock: &output_event->mmap_mutex);
12286
12287	out:
12288	return ret;
12289	}
12290
12291	static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id)
12292	{
12293	bool nmi_safe = false;
12294
12295	switch (clk_id) {
12296	case CLOCK_MONOTONIC:
12297	event->clock = &ktime_get_mono_fast_ns;
12298	nmi_safe = true;
12299	break;
12300
12301	case CLOCK_MONOTONIC_RAW:
12302	event->clock = &ktime_get_raw_fast_ns;
12303	nmi_safe = true;
12304	break;
12305
12306	case CLOCK_REALTIME:
12307	event->clock = &ktime_get_real_ns;
12308	break;
12309
12310	case CLOCK_BOOTTIME:
12311	event->clock = &ktime_get_boottime_ns;
12312	break;
12313
12314	case CLOCK_TAI:
12315	event->clock = &ktime_get_clocktai_ns;
12316	break;
12317
12318	default:
12319	return -EINVAL;
12320	}
12321
12322	if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))
12323	return -EINVAL;
12324
12325	return 0;
12326	}
12327
12328	static bool
12329	perf_check_permission(struct perf_event_attr *attr, struct task_struct *task)
12330	{
12331	unsigned int ptrace_mode = PTRACE_MODE_READ_REALCREDS;
12332	bool is_capable = perfmon_capable();
12333
12334	if (attr->sigtrap) {
12335	/*
12336	* perf_event_attr::sigtrap sends signals to the other task.
12337	* Require the current task to also have CAP_KILL.
12338	*/
12339	rcu_read_lock();
12340	is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL);
12341	rcu_read_unlock();
12342
12343	/*
12344	* If the required capabilities aren't available, checks for
12345	* ptrace permissions: upgrade to ATTACH, since sending signals
12346	* can effectively change the target task.
12347	*/
12348	ptrace_mode = PTRACE_MODE_ATTACH_REALCREDS;
12349	}
12350
12351	/*
12352	* Preserve ptrace permission check for backwards compatibility. The
12353	* ptrace check also includes checks that the current task and other
12354	* task have matching uids, and is therefore not done here explicitly.
12355	*/
12356	return is_capable || ptrace_may_access(task, mode: ptrace_mode);
12357	}
12358
12359	/**
12360	* sys_perf_event_open - open a performance event, associate it to a task/cpu
12361	*
12362	* @attr_uptr: event_id type attributes for monitoring/sampling
12363	* @pid: target pid
12364	* @cpu: target cpu
12365	* @group_fd: group leader event fd
12366	* @flags: perf event open flags
12367	*/
12368	SYSCALL_DEFINE5(perf_event_open,
12369	struct perf_event_attr __user *, attr_uptr,
12370	pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
12371	{
12372	struct perf_event *group_leader = NULL, *output_event = NULL;
12373	struct perf_event_pmu_context *pmu_ctx;
12374	struct perf_event *event, *sibling;
12375	struct perf_event_attr attr;
12376	struct perf_event_context *ctx;
12377	struct file *event_file = NULL;
12378	struct fd group = {NULL, 0};
12379	struct task_struct *task = NULL;
12380	struct pmu *pmu;
12381	int event_fd;
12382	int move_group = 0;
12383	int err;
12384	int f_flags = O_RDWR;
12385	int cgroup_fd = -1;
12386
12387	/* for future expandability... */
12388	if (flags & ~PERF_FLAG_ALL)
12389	return -EINVAL;
12390
12391	err = perf_copy_attr(uattr: attr_uptr, attr: &attr);
12392	if (err)
12393	return err;
12394
12395	/* Do we allow access to perf_event_open(2) ? */
12396	err = security_perf_event_open(attr: &attr, PERF_SECURITY_OPEN);
12397	if (err)
12398	return err;
12399
12400	if (!attr.exclude_kernel) {
12401	err = perf_allow_kernel(attr: &attr);
12402	if (err)
12403	return err;
12404	}
12405
12406	if (attr.namespaces) {
12407	if (!perfmon_capable())
12408	return -EACCES;
12409	}
12410
12411	if (attr.freq) {
12412	if (attr.sample_freq > sysctl_perf_event_sample_rate)
12413	return -EINVAL;
12414	} else {
12415	if (attr.sample_period & (1ULL << 63))
12416	return -EINVAL;
12417	}
12418
12419	/* Only privileged users can get physical addresses */
12420	if ((attr.sample_type & PERF_SAMPLE_PHYS_ADDR)) {
12421	err = perf_allow_kernel(attr: &attr);
12422	if (err)
12423	return err;
12424	}
12425
12426	/* REGS_INTR can leak data, lockdown must prevent this */
12427	if (attr.sample_type & PERF_SAMPLE_REGS_INTR) {
12428	err = security_locked_down(what: LOCKDOWN_PERF);
12429	if (err)
12430	return err;
12431	}
12432
12433	/*
12434	* In cgroup mode, the pid argument is used to pass the fd
12435	* opened to the cgroup directory in cgroupfs. The cpu argument
12436	* designates the cpu on which to monitor threads from that
12437	* cgroup.
12438	*/
12439	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
12440	return -EINVAL;
12441
12442	if (flags & PERF_FLAG_FD_CLOEXEC)
12443	f_flags |= O_CLOEXEC;
12444
12445	event_fd = get_unused_fd_flags(flags: f_flags);
12446	if (event_fd < 0)
12447	return event_fd;
12448
12449	if (group_fd != -1) {
12450	err = perf_fget_light(fd: group_fd, p: &group);
12451	if (err)
12452	goto err_fd;
12453	group_leader = group.file->private_data;
12454	if (flags & PERF_FLAG_FD_OUTPUT)
12455	output_event = group_leader;
12456	if (flags & PERF_FLAG_FD_NO_GROUP)
12457	group_leader = NULL;
12458	}
12459
12460	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
12461	task = find_lively_task_by_vpid(vpid: pid);
12462	if (IS_ERR(ptr: task)) {
12463	err = PTR_ERR(ptr: task);
12464	goto err_group_fd;
12465	}
12466	}
12467
12468	if (task && group_leader &&
12469	group_leader->attr.inherit != attr.inherit) {
12470	err = -EINVAL;
12471	goto err_task;
12472	}
12473
12474	if (flags & PERF_FLAG_PID_CGROUP)
12475	cgroup_fd = pid;
12476
12477	event = perf_event_alloc(attr: &attr, cpu, task, group_leader, NULL,
12478	NULL, NULL, cgroup_fd);
12479	if (IS_ERR(ptr: event)) {
12480	err = PTR_ERR(ptr: event);
12481	goto err_task;
12482	}
12483
12484	if (is_sampling_event(event)) {
12485	if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
12486	err = -EOPNOTSUPP;
12487	goto err_alloc;
12488	}
12489	}
12490
12491	/*
12492	* Special case software events and allow them to be part of
12493	* any hardware group.
12494	*/
12495	pmu = event->pmu;
12496
12497	if (attr.use_clockid) {
12498	err = perf_event_set_clock(event, clk_id: attr.clockid);
12499	if (err)
12500	goto err_alloc;
12501	}
12502
12503	if (pmu->task_ctx_nr == perf_sw_context)
12504	event->event_caps |= PERF_EV_CAP_SOFTWARE;
12505
12506	if (task) {
12507	err = down_read_interruptible(sem: &task->signal->exec_update_lock);
12508	if (err)
12509	goto err_alloc;
12510
12511	/*
12512	* We must hold exec_update_lock across this and any potential
12513	* perf_install_in_context() call for this new event to
12514	* serialize against exec() altering our credentials (and the
12515	* perf_event_exit_task() that could imply).
12516	*/
12517	err = -EACCES;
12518	if (!perf_check_permission(attr: &attr, task))
12519	goto err_cred;
12520	}
12521
12522	/*
12523	* Get the target context (task or percpu):
12524	*/
12525	ctx = find_get_context(task, event);
12526	if (IS_ERR(ptr: ctx)) {
12527	err = PTR_ERR(ptr: ctx);
12528	goto err_cred;
12529	}
12530
12531	mutex_lock(&ctx->mutex);
12532
12533	if (ctx->task == TASK_TOMBSTONE) {
12534	err = -ESRCH;
12535	goto err_locked;
12536	}
12537
12538	if (!task) {
12539	/*
12540	* Check if the @cpu we're creating an event for is online.
12541	*
12542	* We use the perf_cpu_context::ctx::mutex to serialize against
12543	* the hotplug notifiers. See perf_event_{init,exit}_cpu().
12544	*/
12545	struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);
12546
12547	if (!cpuctx->online) {
12548	err = -ENODEV;
12549	goto err_locked;
12550	}
12551	}
12552
12553	if (group_leader) {
12554	err = -EINVAL;
12555
12556	/*
12557	* Do not allow a recursive hierarchy (this new sibling
12558	* becoming part of another group-sibling):
12559	*/
12560	if (group_leader->group_leader != group_leader)
12561	goto err_locked;
12562
12563	/* All events in a group should have the same clock */
12564	if (group_leader->clock != event->clock)
12565	goto err_locked;
12566
12567	/*
12568	* Make sure we're both events for the same CPU;
12569	* grouping events for different CPUs is broken; since
12570	* you can never concurrently schedule them anyhow.
12571	*/
12572	if (group_leader->cpu != event->cpu)
12573	goto err_locked;
12574
12575	/*
12576	* Make sure we're both on the same context; either task or cpu.
12577	*/
12578	if (group_leader->ctx != ctx)
12579	goto err_locked;
12580
12581	/*
12582	* Only a group leader can be exclusive or pinned
12583	*/
12584	if (attr.exclusive || attr.pinned)
12585	goto err_locked;
12586
12587	if (is_software_event(event) &&
12588	!in_software_context(event: group_leader)) {
12589	/*
12590	* If the event is a sw event, but the group_leader
12591	* is on hw context.
12592	*
12593	* Allow the addition of software events to hw
12594	* groups, this is safe because software events
12595	* never fail to schedule.
12596	*
12597	* Note the comment that goes with struct
12598	* perf_event_pmu_context.
12599	*/
12600	pmu = group_leader->pmu_ctx->pmu;
12601	} else if (!is_software_event(event)) {
12602	if (is_software_event(event: group_leader) &&
12603	(group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) {
12604	/*
12605	* In case the group is a pure software group, and we
12606	* try to add a hardware event, move the whole group to
12607	* the hardware context.
12608	*/
12609	move_group = 1;
12610	}
12611
12612	/* Don't allow group of multiple hw events from different pmus */
12613	if (!in_software_context(event: group_leader) &&
12614	group_leader->pmu_ctx->pmu != pmu)
12615	goto err_locked;
12616	}
12617	}
12618
12619	/*
12620	* Now that we're certain of the pmu; find the pmu_ctx.
12621	*/
12622	pmu_ctx = find_get_pmu_context(pmu, ctx, event);
12623	if (IS_ERR(ptr: pmu_ctx)) {
12624	err = PTR_ERR(ptr: pmu_ctx);
12625	goto err_locked;
12626	}
12627	event->pmu_ctx = pmu_ctx;
12628
12629	if (output_event) {
12630	err = perf_event_set_output(event, output_event);
12631	if (err)
12632	goto err_context;
12633	}
12634
12635	if (!perf_event_validate_size(event)) {
12636	err = -E2BIG;
12637	goto err_context;
12638	}
12639
12640	if (perf_need_aux_event(event) && !perf_get_aux_event(event, group_leader)) {
12641	err = -EINVAL;
12642	goto err_context;
12643	}
12644
12645	/*
12646	* Must be under the same ctx::mutex as perf_install_in_context(),
12647	* because we need to serialize with concurrent event creation.
12648	*/
12649	if (!exclusive_event_installable(event, ctx)) {
12650	err = -EBUSY;
12651	goto err_context;
12652	}
12653
12654	WARN_ON_ONCE(ctx->parent_ctx);
12655
12656	event_file = anon_inode_getfile(name: "[perf_event]", fops: &perf_fops, priv: event, flags: f_flags);
12657	if (IS_ERR(ptr: event_file)) {
12658	err = PTR_ERR(ptr: event_file);
12659	event_file = NULL;
12660	goto err_context;
12661	}
12662
12663	/*
12664	* This is the point on no return; we cannot fail hereafter. This is
12665	* where we start modifying current state.
12666	*/
12667
12668	if (move_group) {
12669	perf_remove_from_context(event: group_leader, flags: 0);
12670	put_pmu_ctx(epc: group_leader->pmu_ctx);
12671
12672	for_each_sibling_event(sibling, group_leader) {
12673	perf_remove_from_context(event: sibling, flags: 0);
12674	put_pmu_ctx(epc: sibling->pmu_ctx);
12675	}
12676
12677	/*
12678	* Install the group siblings before the group leader.
12679	*
12680	* Because a group leader will try and install the entire group
12681	* (through the sibling list, which is still in-tact), we can
12682	* end up with siblings installed in the wrong context.
12683	*
12684	* By installing siblings first we NO-OP because they're not
12685	* reachable through the group lists.
12686	*/
12687	for_each_sibling_event(sibling, group_leader) {
12688	sibling->pmu_ctx = pmu_ctx;
12689	get_pmu_ctx(epc: pmu_ctx);
12690	perf_event__state_init(event: sibling);
12691	perf_install_in_context(ctx, event: sibling, cpu: sibling->cpu);
12692	}
12693
12694	/*
12695	* Removing from the context ends up with disabled
12696	* event. What we want here is event in the initial
12697	* startup state, ready to be add into new context.
12698	*/
12699	group_leader->pmu_ctx = pmu_ctx;
12700	get_pmu_ctx(epc: pmu_ctx);
12701	perf_event__state_init(event: group_leader);
12702	perf_install_in_context(ctx, event: group_leader, cpu: group_leader->cpu);
12703	}
12704
12705	/*
12706	* Precalculate sample_data sizes; do while holding ctx::mutex such
12707	* that we're serialized against further additions and before
12708	* perf_install_in_context() which is the point the event is active and
12709	* can use these values.
12710	*/
12711	perf_event__header_size(event);
12712	perf_event__id_header_size(event);
12713
12714	event->owner = current;
12715
12716	perf_install_in_context(ctx, event, cpu: event->cpu);
12717	perf_unpin_context(ctx);
12718
12719	mutex_unlock(lock: &ctx->mutex);
12720
12721	if (task) {
12722	up_read(sem: &task->signal->exec_update_lock);
12723	put_task_struct(t: task);
12724	}
12725
12726	mutex_lock(&current->perf_event_mutex);
12727	list_add_tail(new: &event->owner_entry, head: &current->perf_event_list);
12728	mutex_unlock(lock: &current->perf_event_mutex);
12729
12730	/*
12731	* Drop the reference on the group_event after placing the
12732	* new event on the sibling_list. This ensures destruction
12733	* of the group leader will find the pointer to itself in
12734	* perf_group_detach().
12735	*/
12736	fdput(fd: group);
12737	fd_install(fd: event_fd, file: event_file);
12738	return event_fd;
12739
12740	err_context:
12741	put_pmu_ctx(epc: event->pmu_ctx);
12742	event->pmu_ctx = NULL; /* _free_event() */
12743	err_locked:
12744	mutex_unlock(lock: &ctx->mutex);
12745	perf_unpin_context(ctx);
12746	put_ctx(ctx);
12747	err_cred:
12748	if (task)
12749	up_read(sem: &task->signal->exec_update_lock);
12750	err_alloc:
12751	free_event(event);
12752	err_task:
12753	if (task)
12754	put_task_struct(t: task);
12755	err_group_fd:
12756	fdput(fd: group);
12757	err_fd:
12758	put_unused_fd(fd: event_fd);
12759	return err;
12760	}
12761
12762	/**
12763	* perf_event_create_kernel_counter
12764	*
12765	* @attr: attributes of the counter to create
12766	* @cpu: cpu in which the counter is bound
12767	* @task: task to profile (NULL for percpu)
12768	* @overflow_handler: callback to trigger when we hit the event
12769	* @context: context data could be used in overflow_handler callback
12770	*/
12771	struct perf_event *
12772	perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
12773	struct task_struct *task,
12774	perf_overflow_handler_t overflow_handler,
12775	void *context)
12776	{
12777	struct perf_event_pmu_context *pmu_ctx;
12778	struct perf_event_context *ctx;
12779	struct perf_event *event;
12780	struct pmu *pmu;
12781	int err;
12782
12783	/*
12784	* Grouping is not supported for kernel events, neither is 'AUX',
12785	* make sure the caller's intentions are adjusted.
12786	*/
12787	if (attr->aux_output)
12788	return ERR_PTR(error: -EINVAL);
12789
12790	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
12791	overflow_handler, context, cgroup_fd: -1);
12792	if (IS_ERR(ptr: event)) {
12793	err = PTR_ERR(ptr: event);
12794	goto err;
12795	}
12796
12797	/* Mark owner so we could distinguish it from user events. */
12798	event->owner = TASK_TOMBSTONE;
12799	pmu = event->pmu;
12800
12801	if (pmu->task_ctx_nr == perf_sw_context)
12802	event->event_caps |= PERF_EV_CAP_SOFTWARE;
12803
12804	/*
12805	* Get the target context (task or percpu):
12806	*/
12807	ctx = find_get_context(task, event);
12808	if (IS_ERR(ptr: ctx)) {
12809	err = PTR_ERR(ptr: ctx);
12810	goto err_alloc;
12811	}
12812
12813	WARN_ON_ONCE(ctx->parent_ctx);
12814	mutex_lock(&ctx->mutex);
12815	if (ctx->task == TASK_TOMBSTONE) {
12816	err = -ESRCH;
12817	goto err_unlock;
12818	}
12819
12820	pmu_ctx = find_get_pmu_context(pmu, ctx, event);
12821	if (IS_ERR(ptr: pmu_ctx)) {
12822	err = PTR_ERR(ptr: pmu_ctx);
12823	goto err_unlock;
12824	}
12825	event->pmu_ctx = pmu_ctx;
12826
12827	if (!task) {
12828	/*
12829	* Check if the @cpu we're creating an event for is online.
12830	*
12831	* We use the perf_cpu_context::ctx::mutex to serialize against
12832	* the hotplug notifiers. See perf_event_{init,exit}_cpu().
12833	*/
12834	struct perf_cpu_context *cpuctx =
12835	container_of(ctx, struct perf_cpu_context, ctx);
12836	if (!cpuctx->online) {
12837	err = -ENODEV;
12838	goto err_pmu_ctx;
12839	}
12840	}
12841
12842	if (!exclusive_event_installable(event, ctx)) {
12843	err = -EBUSY;
12844	goto err_pmu_ctx;
12845	}
12846
12847	perf_install_in_context(ctx, event, cpu: event->cpu);
12848	perf_unpin_context(ctx);
12849	mutex_unlock(lock: &ctx->mutex);
12850
12851	return event;
12852
12853	err_pmu_ctx:
12854	put_pmu_ctx(epc: pmu_ctx);
12855	event->pmu_ctx = NULL; /* _free_event() */
12856	err_unlock:
12857	mutex_unlock(lock: &ctx->mutex);
12858	perf_unpin_context(ctx);
12859	put_ctx(ctx);
12860	err_alloc:
12861	free_event(event);
12862	err:
12863	return ERR_PTR(error: err);
12864	}
12865	EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
12866
12867	static void __perf_pmu_remove(struct perf_event_context *ctx,
12868	int cpu, struct pmu *pmu,
12869	struct perf_event_groups *groups,
12870	struct list_head *events)
12871	{
12872	struct perf_event *event, *sibling;
12873
12874	perf_event_groups_for_cpu_pmu(event, groups, cpu, pmu) {
12875	perf_remove_from_context(event, flags: 0);
12876	put_pmu_ctx(epc: event->pmu_ctx);
12877	list_add(new: &event->migrate_entry, head: events);
12878
12879	for_each_sibling_event(sibling, event) {
12880	perf_remove_from_context(event: sibling, flags: 0);
12881	put_pmu_ctx(epc: sibling->pmu_ctx);
12882	list_add(new: &sibling->migrate_entry, head: events);
12883	}
12884	}
12885	}
12886
12887	static void __perf_pmu_install_event(struct pmu *pmu,
12888	struct perf_event_context *ctx,
12889	int cpu, struct perf_event *event)
12890	{
12891	struct perf_event_pmu_context *epc;
12892
12893	event->cpu = cpu;
12894	epc = find_get_pmu_context(pmu, ctx, event);
12895	event->pmu_ctx = epc;
12896
12897	if (event->state >= PERF_EVENT_STATE_OFF)
12898	event->state = PERF_EVENT_STATE_INACTIVE;
12899	perf_install_in_context(ctx, event, cpu);
12900	}
12901
12902	static void __perf_pmu_install(struct perf_event_context *ctx,
12903	int cpu, struct pmu *pmu, struct list_head *events)
12904	{
12905	struct perf_event *event, *tmp;
12906
12907	/*
12908	* Re-instate events in 2 passes.
12909	*
12910	* Skip over group leaders and only install siblings on this first
12911	* pass, siblings will not get enabled without a leader, however a
12912	* leader will enable its siblings, even if those are still on the old
12913	* context.
12914	*/
12915	list_for_each_entry_safe(event, tmp, events, migrate_entry) {
12916	if (event->group_leader == event)
12917	continue;
12918
12919	list_del(entry: &event->migrate_entry);
12920	__perf_pmu_install_event(pmu, ctx, cpu, event);
12921	}
12922
12923	/*
12924	* Once all the siblings are setup properly, install the group leaders
12925	* to make it go.
12926	*/
12927	list_for_each_entry_safe(event, tmp, events, migrate_entry) {
12928	list_del(entry: &event->migrate_entry);
12929	__perf_pmu_install_event(pmu, ctx, cpu, event);
12930	}
12931	}
12932
12933	void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
12934	{
12935	struct perf_event_context *src_ctx, *dst_ctx;
12936	LIST_HEAD(events);
12937
12938	src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx;
12939	dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx;
12940
12941	/*
12942	* See perf_event_ctx_lock() for comments on the details
12943	* of swizzling perf_event::ctx.
12944	*/
12945	mutex_lock_double(a: &src_ctx->mutex, b: &dst_ctx->mutex);
12946
12947	__perf_pmu_remove(ctx: src_ctx, cpu: src_cpu, pmu, groups: &src_ctx->pinned_groups, events: &events);
12948	__perf_pmu_remove(ctx: src_ctx, cpu: src_cpu, pmu, groups: &src_ctx->flexible_groups, events: &events);
12949
12950	if (!list_empty(head: &events)) {
12951	/*
12952	* Wait for the events to quiesce before re-instating them.
12953	*/
12954	synchronize_rcu();
12955
12956	__perf_pmu_install(ctx: dst_ctx, cpu: dst_cpu, pmu, events: &events);
12957	}
12958
12959	mutex_unlock(lock: &dst_ctx->mutex);
12960	mutex_unlock(lock: &src_ctx->mutex);
12961	}
12962	EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
12963
12964	static void sync_child_event(struct perf_event *child_event)
12965	{
12966	struct perf_event *parent_event = child_event->parent;
12967	u64 child_val;
12968
12969	if (child_event->attr.inherit_stat) {
12970	struct task_struct *task = child_event->ctx->task;
12971
12972	if (task && task != TASK_TOMBSTONE)
12973	perf_event_read_event(event: child_event, task);
12974	}
12975
12976	child_val = perf_event_count(event: child_event);
12977
12978	/*
12979	* Add back the child's count to the parent's count:
12980	*/
12981	atomic64_add(i: child_val, v: &parent_event->child_count);
12982	atomic64_add(i: child_event->total_time_enabled,
12983	v: &parent_event->child_total_time_enabled);
12984	atomic64_add(i: child_event->total_time_running,
12985	v: &parent_event->child_total_time_running);
12986	}
12987
12988	static void
12989	perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx)
12990	{
12991	struct perf_event *parent_event = event->parent;
12992	unsigned long detach_flags = 0;
12993
12994	if (parent_event) {
12995	/*
12996	* Do not destroy the 'original' grouping; because of the
12997	* context switch optimization the original events could've
12998	* ended up in a random child task.
12999	*
13000	* If we were to destroy the original group, all group related
13001	* operations would cease to function properly after this
13002	* random child dies.
13003	*
13004	* Do destroy all inherited groups, we don't care about those
13005	* and being thorough is better.
13006	*/
13007	detach_flags = DETACH_GROUP | DETACH_CHILD;
13008	mutex_lock(&parent_event->child_mutex);
13009	}
13010
13011	perf_remove_from_context(event, flags: detach_flags);
13012
13013	raw_spin_lock_irq(&ctx->lock);
13014	if (event->state > PERF_EVENT_STATE_EXIT)
13015	perf_event_set_state(event, state: PERF_EVENT_STATE_EXIT);
13016	raw_spin_unlock_irq(&ctx->lock);
13017
13018	/*
13019	* Child events can be freed.
13020	*/
13021	if (parent_event) {
13022	mutex_unlock(lock: &parent_event->child_mutex);
13023	/*
13024	* Kick perf_poll() for is_event_hup();
13025	*/
13026	perf_event_wakeup(event: parent_event);
13027	free_event(event);
13028	put_event(event: parent_event);
13029	return;
13030	}
13031
13032	/*
13033	* Parent events are governed by their filedesc, retain them.
13034	*/
13035	perf_event_wakeup(event);
13036	}
13037
13038	static void perf_event_exit_task_context(struct task_struct *child)
13039	{
13040	struct perf_event_context *child_ctx, *clone_ctx = NULL;
13041	struct perf_event *child_event, *next;
13042
13043	WARN_ON_ONCE(child != current);
13044
13045	child_ctx = perf_pin_task_context(task: child);
13046	if (!child_ctx)
13047	return;
13048
13049	/*
13050	* In order to reduce the amount of tricky in ctx tear-down, we hold
13051	* ctx::mutex over the entire thing. This serializes against almost
13052	* everything that wants to access the ctx.
13053	*
13054	* The exception is sys_perf_event_open() /
13055	* perf_event_create_kernel_count() which does find_get_context()
13056	* without ctx::mutex (it cannot because of the move_group double mutex
13057	* lock thing). See the comments in perf_install_in_context().
13058	*/
13059	mutex_lock(&child_ctx->mutex);
13060
13061	/*
13062	* In a single ctx::lock section, de-schedule the events and detach the
13063	* context from the task such that we cannot ever get it scheduled back
13064	* in.
13065	*/
13066	raw_spin_lock_irq(&child_ctx->lock);
13067	task_ctx_sched_out(ctx: child_ctx, event_type: EVENT_ALL);
13068
13069	/*
13070	* Now that the context is inactive, destroy the task <-> ctx relation
13071	* and mark the context dead.
13072	*/
13073	RCU_INIT_POINTER(child->perf_event_ctxp, NULL);
13074	put_ctx(ctx: child_ctx); /* cannot be last */
13075	WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE);
13076	put_task_struct(current); /* cannot be last */
13077
13078	clone_ctx = unclone_ctx(ctx: child_ctx);
13079	raw_spin_unlock_irq(&child_ctx->lock);
13080
13081	if (clone_ctx)
13082	put_ctx(ctx: clone_ctx);
13083
13084	/*
13085	* Report the task dead after unscheduling the events so that we
13086	* won't get any samples after PERF_RECORD_EXIT. We can however still
13087	* get a few PERF_RECORD_READ events.
13088	*/
13089	perf_event_task(task: child, task_ctx: child_ctx, new: 0);
13090
13091	list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)
13092	perf_event_exit_event(event: child_event, ctx: child_ctx);
13093
13094	mutex_unlock(lock: &child_ctx->mutex);
13095
13096	put_ctx(ctx: child_ctx);
13097	}
13098
13099	/*
13100	* When a child task exits, feed back event values to parent events.
13101	*
13102	* Can be called with exec_update_lock held when called from
13103	* setup_new_exec().
13104	*/
13105	void perf_event_exit_task(struct task_struct *child)
13106	{
13107	struct perf_event *event, *tmp;
13108
13109	mutex_lock(&child->perf_event_mutex);
13110	list_for_each_entry_safe(event, tmp, &child->perf_event_list,
13111	owner_entry) {
13112	list_del_init(entry: &event->owner_entry);
13113
13114	/*
13115	* Ensure the list deletion is visible before we clear
13116	* the owner, closes a race against perf_release() where
13117	* we need to serialize on the owner->perf_event_mutex.
13118	*/
13119	smp_store_release(&event->owner, NULL);
13120	}
13121	mutex_unlock(lock: &child->perf_event_mutex);
13122
13123	perf_event_exit_task_context(child);
13124
13125	/*
13126	* The perf_event_exit_task_context calls perf_event_task
13127	* with child's task_ctx, which generates EXIT events for
13128	* child contexts and sets child->perf_event_ctxp[] to NULL.
13129	* At this point we need to send EXIT events to cpu contexts.
13130	*/
13131	perf_event_task(task: child, NULL, new: 0);
13132	}
13133
13134	static void perf_free_event(struct perf_event *event,
13135	struct perf_event_context *ctx)
13136	{
13137	struct perf_event *parent = event->parent;
13138
13139	if (WARN_ON_ONCE(!parent))
13140	return;
13141
13142	mutex_lock(&parent->child_mutex);
13143	list_del_init(entry: &event->child_list);
13144	mutex_unlock(lock: &parent->child_mutex);
13145
13146	put_event(event: parent);
13147
13148	raw_spin_lock_irq(&ctx->lock);
13149	perf_group_detach(event);
13150	list_del_event(event, ctx);
13151	raw_spin_unlock_irq(&ctx->lock);
13152	free_event(event);
13153	}
13154
13155	/*
13156	* Free a context as created by inheritance by perf_event_init_task() below,
13157	* used by fork() in case of fail.
13158	*
13159	* Even though the task has never lived, the context and events have been
13160	* exposed through the child_list, so we must take care tearing it all down.
13161	*/
13162	void perf_event_free_task(struct task_struct *task)
13163	{
13164	struct perf_event_context *ctx;
13165	struct perf_event *event, *tmp;
13166
13167	ctx = rcu_access_pointer(task->perf_event_ctxp);
13168	if (!ctx)
13169	return;
13170
13171	mutex_lock(&ctx->mutex);
13172	raw_spin_lock_irq(&ctx->lock);
13173	/*
13174	* Destroy the task <-> ctx relation and mark the context dead.
13175	*
13176	* This is important because even though the task hasn't been
13177	* exposed yet the context has been (through child_list).
13178	*/
13179	RCU_INIT_POINTER(task->perf_event_ctxp, NULL);
13180	WRITE_ONCE(ctx->task, TASK_TOMBSTONE);
13181	put_task_struct(t: task); /* cannot be last */
13182	raw_spin_unlock_irq(&ctx->lock);
13183
13184
13185	list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry)
13186	perf_free_event(event, ctx);
13187
13188	mutex_unlock(lock: &ctx->mutex);
13189
13190	/*
13191	* perf_event_release_kernel() could've stolen some of our
13192	* child events and still have them on its free_list. In that
13193	* case we must wait for these events to have been freed (in
13194	* particular all their references to this task must've been
13195	* dropped).
13196	*
13197	* Without this copy_process() will unconditionally free this
13198	* task (irrespective of its reference count) and
13199	* _free_event()'s put_task_struct(event->hw.target) will be a
13200	* use-after-free.
13201	*
13202	* Wait for all events to drop their context reference.
13203	*/
13204	wait_var_event(&ctx->refcount, refcount_read(&ctx->refcount) == 1);
13205	put_ctx(ctx); /* must be last */
13206	}
13207
13208	void perf_event_delayed_put(struct task_struct *task)
13209	{
13210	WARN_ON_ONCE(task->perf_event_ctxp);
13211	}
13212
13213	struct file *perf_event_get(unsigned int fd)
13214	{
13215	struct file *file = fget(fd);
13216	if (!file)
13217	return ERR_PTR(error: -EBADF);
13218
13219	if (file->f_op != &perf_fops) {
13220	fput(file);
13221	return ERR_PTR(error: -EBADF);
13222	}
13223
13224	return file;
13225	}
13226
13227	const struct perf_event *perf_get_event(struct file *file)
13228	{
13229	if (file->f_op != &perf_fops)
13230	return ERR_PTR(error: -EINVAL);
13231
13232	return file->private_data;
13233	}
13234
13235	const struct perf_event_attr *perf_event_attrs(struct perf_event *event)
13236	{
13237	if (!event)
13238	return ERR_PTR(error: -EINVAL);
13239
13240	return &event->attr;
13241	}
13242
13243	/*
13244	* Inherit an event from parent task to child task.
13245	*
13246	* Returns:
13247	* - valid pointer on success
13248	* - NULL for orphaned events
13249	* - IS_ERR() on error
13250	*/
13251	static struct perf_event *
13252	inherit_event(struct perf_event *parent_event,
13253	struct task_struct *parent,
13254	struct perf_event_context *parent_ctx,
13255	struct task_struct *child,
13256	struct perf_event *group_leader,
13257	struct perf_event_context *child_ctx)
13258	{
13259	enum perf_event_state parent_state = parent_event->state;
13260	struct perf_event_pmu_context *pmu_ctx;
13261	struct perf_event *child_event;
13262	unsigned long flags;
13263
13264	/*
13265	* Instead of creating recursive hierarchies of events,
13266	* we link inherited events back to the original parent,
13267	* which has a filp for sure, which we use as the reference
13268	* count:
13269	*/
13270	if (parent_event->parent)
13271	parent_event = parent_event->parent;
13272
13273	child_event = perf_event_alloc(attr: &parent_event->attr,
13274	cpu: parent_event->cpu,
13275	task: child,
13276	group_leader, parent_event,
13277	NULL, NULL, cgroup_fd: -1);
13278	if (IS_ERR(ptr: child_event))
13279	return child_event;
13280
13281	pmu_ctx = find_get_pmu_context(pmu: child_event->pmu, ctx: child_ctx, event: child_event);
13282	if (IS_ERR(ptr: pmu_ctx)) {
13283	free_event(event: child_event);
13284	return ERR_CAST(ptr: pmu_ctx);
13285	}
13286	child_event->pmu_ctx = pmu_ctx;
13287
13288	/*
13289	* is_orphaned_event() and list_add_tail(&parent_event->child_list)
13290	* must be under the same lock in order to serialize against
13291	* perf_event_release_kernel(), such that either we must observe
13292	* is_orphaned_event() or they will observe us on the child_list.
13293	*/
13294	mutex_lock(&parent_event->child_mutex);
13295	if (is_orphaned_event(event: parent_event) ||
13296	!atomic_long_inc_not_zero(v: &parent_event->refcount)) {
13297	mutex_unlock(lock: &parent_event->child_mutex);
13298	/* task_ctx_data is freed with child_ctx */
13299	free_event(event: child_event);
13300	return NULL;
13301	}
13302
13303	get_ctx(ctx: child_ctx);
13304
13305	/*
13306	* Make the child state follow the state of the parent event,
13307	* not its attr.disabled bit. We hold the parent's mutex,
13308	* so we won't race with perf_event_{en, dis}able_family.
13309	*/
13310	if (parent_state >= PERF_EVENT_STATE_INACTIVE)
13311	child_event->state = PERF_EVENT_STATE_INACTIVE;
13312	else
13313	child_event->state = PERF_EVENT_STATE_OFF;
13314
13315	if (parent_event->attr.freq) {
13316	u64 sample_period = parent_event->hw.sample_period;
13317	struct hw_perf_event *hwc = &child_event->hw;
13318
13319	hwc->sample_period = sample_period;
13320	hwc->last_period = sample_period;
13321
13322	local64_set(&hwc->period_left, sample_period);
13323	}
13324
13325	child_event->ctx = child_ctx;
13326	child_event->overflow_handler = parent_event->overflow_handler;
13327	child_event->overflow_handler_context
13328	= parent_event->overflow_handler_context;
13329
13330	/*
13331	* Precalculate sample_data sizes
13332	*/
13333	perf_event__header_size(event: child_event);
13334	perf_event__id_header_size(event: child_event);
13335
13336	/*
13337	* Link it up in the child's context:
13338	*/
13339	raw_spin_lock_irqsave(&child_ctx->lock, flags);
13340	add_event_to_ctx(event: child_event, ctx: child_ctx);
13341	child_event->attach_state |= PERF_ATTACH_CHILD;
13342	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
13343
13344	/*
13345	* Link this into the parent event's child list
13346	*/
13347	list_add_tail(new: &child_event->child_list, head: &parent_event->child_list);
13348	mutex_unlock(lock: &parent_event->child_mutex);
13349
13350	return child_event;
13351	}
13352
13353	/*
13354	* Inherits an event group.
13355	*
13356	* This will quietly suppress orphaned events; !inherit_event() is not an error.
13357	* This matches with perf_event_release_kernel() removing all child events.
13358	*
13359	* Returns:
13360	* - 0 on success
13361	* - <0 on error
13362	*/
13363	static int inherit_group(struct perf_event *parent_event,
13364	struct task_struct *parent,
13365	struct perf_event_context *parent_ctx,
13366	struct task_struct *child,
13367	struct perf_event_context *child_ctx)
13368	{
13369	struct perf_event *leader;
13370	struct perf_event *sub;
13371	struct perf_event *child_ctr;
13372
13373	leader = inherit_event(parent_event, parent, parent_ctx,
13374	child, NULL, child_ctx);
13375	if (IS_ERR(ptr: leader))
13376	return PTR_ERR(ptr: leader);
13377	/*
13378	* @leader can be NULL here because of is_orphaned_event(). In this
13379	* case inherit_event() will create individual events, similar to what
13380	* perf_group_detach() would do anyway.
13381	*/
13382	for_each_sibling_event(sub, parent_event) {
13383	child_ctr = inherit_event(parent_event: sub, parent, parent_ctx,
13384	child, group_leader: leader, child_ctx);
13385	if (IS_ERR(ptr: child_ctr))
13386	return PTR_ERR(ptr: child_ctr);
13387
13388	if (sub->aux_event == parent_event && child_ctr &&
13389	!perf_get_aux_event(event: child_ctr, group_leader: leader))
13390	return -EINVAL;
13391	}
13392	if (leader)
13393	leader->group_generation = parent_event->group_generation;
13394	return 0;
13395	}
13396
13397	/*
13398	* Creates the child task context and tries to inherit the event-group.
13399	*
13400	* Clears @inherited_all on !attr.inherited or error. Note that we'll leave
13401	* inherited_all set when we 'fail' to inherit an orphaned event; this is
13402	* consistent with perf_event_release_kernel() removing all child events.
13403	*
13404	* Returns:
13405	* - 0 on success
13406	* - <0 on error
13407	*/
13408	static int
13409	inherit_task_group(struct perf_event *event, struct task_struct *parent,
13410	struct perf_event_context *parent_ctx,
13411	struct task_struct *child,
13412	u64 clone_flags, int *inherited_all)
13413	{
13414	struct perf_event_context *child_ctx;
13415	int ret;
13416
13417	if (!event->attr.inherit ||
13418	(event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) ||
13419	/* Do not inherit if sigtrap and signal handlers were cleared. */
13420	(event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) {
13421	*inherited_all = 0;
13422	return 0;
13423	}
13424
13425	child_ctx = child->perf_event_ctxp;
13426	if (!child_ctx) {
13427	/*
13428	* This is executed from the parent task context, so
13429	* inherit events that have been marked for cloning.
13430	* First allocate and initialize a context for the
13431	* child.
13432	*/
13433	child_ctx = alloc_perf_context(task: child);
13434	if (!child_ctx)
13435	return -ENOMEM;
13436
13437	child->perf_event_ctxp = child_ctx;
13438	}
13439
13440	ret = inherit_group(parent_event: event, parent, parent_ctx, child, child_ctx);
13441	if (ret)
13442	*inherited_all = 0;
13443
13444	return ret;
13445	}
13446
13447	/*
13448	* Initialize the perf_event context in task_struct
13449	*/
13450	static int perf_event_init_context(struct task_struct *child, u64 clone_flags)
13451	{
13452	struct perf_event_context *child_ctx, *parent_ctx;
13453	struct perf_event_context *cloned_ctx;
13454	struct perf_event *event;
13455	struct task_struct *parent = current;
13456	int inherited_all = 1;
13457	unsigned long flags;
13458	int ret = 0;
13459
13460	if (likely(!parent->perf_event_ctxp))
13461	return 0;
13462
13463	/*
13464	* If the parent's context is a clone, pin it so it won't get
13465	* swapped under us.
13466	*/
13467	parent_ctx = perf_pin_task_context(task: parent);
13468	if (!parent_ctx)
13469	return 0;
13470
13471	/*
13472	* No need to check if parent_ctx != NULL here; since we saw
13473	* it non-NULL earlier, the only reason for it to become NULL
13474	* is if we exit, and since we're currently in the middle of
13475	* a fork we can't be exiting at the same time.
13476	*/
13477
13478	/*
13479	* Lock the parent list. No need to lock the child - not PID
13480	* hashed yet and not running, so nobody can access it.
13481	*/
13482	mutex_lock(&parent_ctx->mutex);
13483
13484	/*
13485	* We dont have to disable NMIs - we are only looking at
13486	* the list, not manipulating it:
13487	*/
13488	perf_event_groups_for_each(event, &parent_ctx->pinned_groups) {
13489	ret = inherit_task_group(event, parent, parent_ctx,
13490	child, clone_flags, inherited_all: &inherited_all);
13491	if (ret)
13492	goto out_unlock;
13493	}
13494
13495	/*
13496	* We can't hold ctx->lock when iterating the ->flexible_group list due
13497	* to allocations, but we need to prevent rotation because
13498	* rotate_ctx() will change the list from interrupt context.
13499	*/
13500	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
13501	parent_ctx->rotate_disable = 1;
13502	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
13503
13504	perf_event_groups_for_each(event, &parent_ctx->flexible_groups) {
13505	ret = inherit_task_group(event, parent, parent_ctx,
13506	child, clone_flags, inherited_all: &inherited_all);
13507	if (ret)
13508	goto out_unlock;
13509	}
13510
13511	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
13512	parent_ctx->rotate_disable = 0;
13513
13514	child_ctx = child->perf_event_ctxp;
13515
13516	if (child_ctx && inherited_all) {
13517	/*
13518	* Mark the child context as a clone of the parent
13519	* context, or of whatever the parent is a clone of.
13520	*
13521	* Note that if the parent is a clone, the holding of
13522	* parent_ctx->lock avoids it from being uncloned.
13523	*/
13524	cloned_ctx = parent_ctx->parent_ctx;
13525	if (cloned_ctx) {
13526	child_ctx->parent_ctx = cloned_ctx;
13527	child_ctx->parent_gen = parent_ctx->parent_gen;
13528	} else {
13529	child_ctx->parent_ctx = parent_ctx;
13530	child_ctx->parent_gen = parent_ctx->generation;
13531	}
13532	get_ctx(ctx: child_ctx->parent_ctx);
13533	}
13534
13535	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
13536	out_unlock:
13537	mutex_unlock(lock: &parent_ctx->mutex);
13538
13539	perf_unpin_context(ctx: parent_ctx);
13540	put_ctx(ctx: parent_ctx);
13541
13542	return ret;
13543	}
13544
13545	/*
13546	* Initialize the perf_event context in task_struct
13547	*/
13548	int perf_event_init_task(struct task_struct *child, u64 clone_flags)
13549	{
13550	int ret;
13551
13552	child->perf_event_ctxp = NULL;
13553	mutex_init(&child->perf_event_mutex);
13554	INIT_LIST_HEAD(list: &child->perf_event_list);
13555
13556	ret = perf_event_init_context(child, clone_flags);
13557	if (ret) {
13558	perf_event_free_task(task: child);
13559	return ret;
13560	}
13561
13562	return 0;
13563	}
13564
13565	static void __init perf_event_init_all_cpus(void)
13566	{
13567	struct swevent_htable *swhash;
13568	struct perf_cpu_context *cpuctx;
13569	int cpu;
13570
13571	zalloc_cpumask_var(mask: &perf_online_mask, GFP_KERNEL);
13572
13573	for_each_possible_cpu(cpu) {
13574	swhash = &per_cpu(swevent_htable, cpu);
13575	mutex_init(&swhash->hlist_mutex);
13576
13577	INIT_LIST_HEAD(list: &per_cpu(pmu_sb_events.list, cpu));
13578	raw_spin_lock_init(&per_cpu(pmu_sb_events.lock, cpu));
13579
13580	INIT_LIST_HEAD(list: &per_cpu(sched_cb_list, cpu));
13581
13582	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
13583	__perf_event_init_context(ctx: &cpuctx->ctx);
13584	lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
13585	lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
13586	cpuctx->online = cpumask_test_cpu(cpu, cpumask: perf_online_mask);
13587	cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default);
13588	cpuctx->heap = cpuctx->heap_default;
13589	}
13590	}
13591
13592	static void perf_swevent_init_cpu(unsigned int cpu)
13593	{
13594	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
13595
13596	mutex_lock(&swhash->hlist_mutex);
13597	if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) {
13598	struct swevent_hlist *hlist;
13599
13600	hlist = kzalloc_node(size: sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
13601	WARN_ON(!hlist);
13602	rcu_assign_pointer(swhash->swevent_hlist, hlist);
13603	}
13604	mutex_unlock(lock: &swhash->hlist_mutex);
13605	}
13606
13607	#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC_CORE
13608	static void __perf_event_exit_context(void *__info)
13609	{
13610	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
13611	struct perf_event_context *ctx = __info;
13612	struct perf_event *event;
13613
13614	raw_spin_lock(&ctx->lock);
13615	ctx_sched_out(ctx, event_type: EVENT_TIME);
13616	list_for_each_entry(event, &ctx->event_list, event_entry)
13617	__perf_remove_from_context(event, cpuctx, ctx, info: (void *)DETACH_GROUP);
13618	raw_spin_unlock(&ctx->lock);
13619	}
13620
13621	static void perf_event_exit_cpu_context(int cpu)
13622	{
13623	struct perf_cpu_context *cpuctx;
13624	struct perf_event_context *ctx;
13625
13626	// XXX simplify cpuctx->online
13627	mutex_lock(&pmus_lock);
13628	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
13629	ctx = &cpuctx->ctx;
13630
13631	mutex_lock(&ctx->mutex);
13632	smp_call_function_single(cpuid: cpu, func: __perf_event_exit_context, info: ctx, wait: 1);
13633	cpuctx->online = 0;
13634	mutex_unlock(lock: &ctx->mutex);
13635	cpumask_clear_cpu(cpu, dstp: perf_online_mask);
13636	mutex_unlock(lock: &pmus_lock);
13637	}
13638	#else
13639
13640	static void perf_event_exit_cpu_context(int cpu) { }
13641
13642	#endif
13643
13644	int perf_event_init_cpu(unsigned int cpu)
13645	{
13646	struct perf_cpu_context *cpuctx;
13647	struct perf_event_context *ctx;
13648
13649	perf_swevent_init_cpu(cpu);
13650
13651	mutex_lock(&pmus_lock);
13652	cpumask_set_cpu(cpu, dstp: perf_online_mask);
13653	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
13654	ctx = &cpuctx->ctx;
13655
13656	mutex_lock(&ctx->mutex);
13657	cpuctx->online = 1;
13658	mutex_unlock(lock: &ctx->mutex);
13659	mutex_unlock(lock: &pmus_lock);
13660
13661	return 0;
13662	}
13663
13664	int perf_event_exit_cpu(unsigned int cpu)
13665	{
13666	perf_event_exit_cpu_context(cpu);
13667	return 0;
13668	}
13669
13670	static int
13671	perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
13672	{
13673	int cpu;
13674
13675	for_each_online_cpu(cpu)
13676	perf_event_exit_cpu(cpu);
13677
13678	return NOTIFY_OK;
13679	}
13680
13681	/*
13682	* Run the perf reboot notifier at the very last possible moment so that
13683	* the generic watchdog code runs as long as possible.
13684	*/
13685	static struct notifier_block perf_reboot_notifier = {
13686	.notifier_call = perf_reboot,
13687	.priority = INT_MIN,
13688	};
13689
13690	void __init perf_event_init(void)
13691	{
13692	int ret;
13693
13694	idr_init(idr: &pmu_idr);
13695
13696	perf_event_init_all_cpus();
13697	init_srcu_struct(&pmus_srcu);
13698	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
13699	perf_pmu_register(&perf_cpu_clock, "cpu_clock", -1);
13700	perf_pmu_register(&perf_task_clock, "task_clock", -1);
13701	perf_tp_register();
13702	perf_event_init_cpu(smp_processor_id());
13703	register_reboot_notifier(&perf_reboot_notifier);
13704
13705	ret = init_hw_breakpoint();
13706	WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
13707
13708	perf_event_cache = KMEM_CACHE(perf_event, SLAB_PANIC);
13709
13710	/*
13711	* Build time assertion that we keep the data_head at the intended
13712	* location. IOW, validation we got the __reserved[] size right.
13713	*/
13714	BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
13715	!= 1024);
13716	}
13717
13718	ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
13719	char *page)
13720	{
13721	struct perf_pmu_events_attr *pmu_attr =
13722	container_of(attr, struct perf_pmu_events_attr, attr);
13723
13724	if (pmu_attr->event_str)
13725	return sprintf(buf: page, fmt: "%s\n", pmu_attr->event_str);
13726
13727	return 0;
13728	}
13729	EXPORT_SYMBOL_GPL(perf_event_sysfs_show);
13730
13731	static int __init perf_event_sysfs_init(void)
13732	{
13733	struct pmu *pmu;
13734	int ret;
13735
13736	mutex_lock(&pmus_lock);
13737
13738	ret = bus_register(bus: &pmu_bus);
13739	if (ret)
13740	goto unlock;
13741
13742	list_for_each_entry(pmu, &pmus, entry) {
13743	if (pmu->dev)
13744	continue;
13745
13746	ret = pmu_dev_alloc(pmu);
13747	WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
13748	}
13749	pmu_bus_running = 1;
13750	ret = 0;
13751
13752	unlock:
13753	mutex_unlock(lock: &pmus_lock);
13754
13755	return ret;
13756	}
13757	device_initcall(perf_event_sysfs_init);
13758
13759	#ifdef CONFIG_CGROUP_PERF
13760	static struct cgroup_subsys_state *
13761	perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
13762	{
13763	struct perf_cgroup *jc;
13764
13765	jc = kzalloc(size: sizeof(*jc), GFP_KERNEL);
13766	if (!jc)
13767	return ERR_PTR(error: -ENOMEM);
13768
13769	jc->info = alloc_percpu(struct perf_cgroup_info);
13770	if (!jc->info) {
13771	kfree(objp: jc);
13772	return ERR_PTR(error: -ENOMEM);
13773	}
13774
13775	return &jc->css;
13776	}
13777
13778	static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
13779	{
13780	struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
13781
13782	free_percpu(pdata: jc->info);
13783	kfree(objp: jc);
13784	}
13785
13786	static int perf_cgroup_css_online(struct cgroup_subsys_state *css)
13787	{
13788	perf_event_cgroup(cgrp: css->cgroup);
13789	return 0;
13790	}
13791
13792	static int __perf_cgroup_move(void *info)
13793	{
13794	struct task_struct *task = info;
13795
13796	preempt_disable();
13797	perf_cgroup_switch(task);
13798	preempt_enable();
13799
13800	return 0;
13801	}
13802
13803	static void perf_cgroup_attach(struct cgroup_taskset *tset)
13804	{
13805	struct task_struct *task;
13806	struct cgroup_subsys_state *css;
13807
13808	cgroup_taskset_for_each(task, css, tset)
13809	task_function_call(p: task, func: __perf_cgroup_move, info: task);
13810	}
13811
13812	struct cgroup_subsys perf_event_cgrp_subsys = {
13813	.css_alloc = perf_cgroup_css_alloc,
13814	.css_free = perf_cgroup_css_free,
13815	.css_online = perf_cgroup_css_online,
13816	.attach = perf_cgroup_attach,
13817	/*
13818	* Implicitly enable on dfl hierarchy so that perf events can
13819	* always be filtered by cgroup2 path as long as perf_event
13820	* controller is not mounted on a legacy hierarchy.
13821	*/
13822	.implicit_on_dfl = true,
13823	.threaded = true,
13824	};
13825	#endif /* CONFIG_CGROUP_PERF */
13826
13827	DEFINE_STATIC_CALL_RET0(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t);
13828