1	// SPDX-License-Identifier: GPL-2.0
2	#include <dirent.h>
3	#include <errno.h>
4	#include <inttypes.h>
5	#include <regex.h>
6	#include <stdlib.h>
7	#include "callchain.h"
8	#include "debug.h"
9	#include "dso.h"
10	#include "env.h"
11	#include "event.h"
12	#include "evsel.h"
13	#include "hist.h"
14	#include "machine.h"
15	#include "map.h"
16	#include "map_symbol.h"
17	#include "branch.h"
18	#include "mem-events.h"
19	#include "path.h"
20	#include "srcline.h"
21	#include "symbol.h"
22	#include "sort.h"
23	#include "strlist.h"
24	#include "target.h"
25	#include "thread.h"
26	#include "util.h"
27	#include "vdso.h"
28	#include <stdbool.h>
29	#include <sys/types.h>
30	#include <sys/stat.h>
31	#include <unistd.h>
32	#include "unwind.h"
33	#include "linux/hash.h"
34	#include "asm/bug.h"
35	#include "bpf-event.h"
36	#include <internal/lib.h> // page_size
37	#include "cgroup.h"
38	#include "arm64-frame-pointer-unwind-support.h"
39
40	#include <linux/ctype.h>
41	#include <symbol/kallsyms.h>
42	#include <linux/mman.h>
43	#include <linux/string.h>
44	#include <linux/zalloc.h>
45
46	static struct dso *machine__kernel_dso(struct machine *machine)
47	{
48	return map__dso(map: machine->vmlinux_map);
49	}
50
51	static void dsos__init(struct dsos *dsos)
52	{
53	INIT_LIST_HEAD(list: &dsos->head);
54	dsos->root = RB_ROOT;
55	init_rwsem(&dsos->lock);
56	}
57
58	static int machine__set_mmap_name(struct machine *machine)
59	{
60	if (machine__is_host(machine))
61	machine->mmap_name = strdup("[kernel.kallsyms]");
62	else if (machine__is_default_guest(machine))
63	machine->mmap_name = strdup("[guest.kernel.kallsyms]");
64	else if (asprintf(&machine->mmap_name, "[guest.kernel.kallsyms.%d]",
65	machine->pid) < 0)
66	machine->mmap_name = NULL;
67
68	return machine->mmap_name ? 0 : -ENOMEM;
69	}
70
71	static void thread__set_guest_comm(struct thread *thread, pid_t pid)
72	{
73	char comm[64];
74
75	snprintf(buf: comm, size: sizeof(comm), fmt: "[guest/%d]", pid);
76	thread__set_comm(thread, comm, timestamp: 0);
77	}
78
79	int machine__init(struct machine *machine, const char *root_dir, pid_t pid)
80	{
81	int err = -ENOMEM;
82
83	memset(machine, 0, sizeof(*machine));
84	machine->kmaps = maps__new(machine);
85	if (machine->kmaps == NULL)
86	return -ENOMEM;
87
88	RB_CLEAR_NODE(&machine->rb_node);
89	dsos__init(dsos: &machine->dsos);
90
91	threads__init(threads: &machine->threads);
92
93	machine->vdso_info = NULL;
94	machine->env = NULL;
95
96	machine->pid = pid;
97
98	machine->id_hdr_size = 0;
99	machine->kptr_restrict_warned = false;
100	machine->comm_exec = false;
101	machine->kernel_start = 0;
102	machine->vmlinux_map = NULL;
103
104	machine->root_dir = strdup(root_dir);
105	if (machine->root_dir == NULL)
106	goto out;
107
108	if (machine__set_mmap_name(machine))
109	goto out;
110
111	if (pid != HOST_KERNEL_ID) {
112	struct thread *thread = machine__findnew_thread(machine, pid: -1,
113	tid: pid);
114
115	if (thread == NULL)
116	goto out;
117
118	thread__set_guest_comm(thread, pid);
119	thread__put(thread);
120	}
121
122	machine->current_tid = NULL;
123	err = 0;
124
125	out:
126	if (err) {
127	zfree(&machine->kmaps);
128	zfree(&machine->root_dir);
129	zfree(&machine->mmap_name);
130	}
131	return 0;
132	}
133
134	struct machine *machine__new_host(void)
135	{
136	struct machine *machine = malloc(sizeof(*machine));
137
138	if (machine != NULL) {
139	machine__init(machine, root_dir: "", HOST_KERNEL_ID);
140
141	if (machine__create_kernel_maps(machine) < 0)
142	goto out_delete;
143	}
144
145	return machine;
146	out_delete:
147	free(machine);
148	return NULL;
149	}
150
151	struct machine *machine__new_kallsyms(void)
152	{
153	struct machine *machine = machine__new_host();
154	/*
155	* FIXME:
156	* 1) We should switch to machine__load_kallsyms(), i.e. not explicitly
157	* ask for not using the kcore parsing code, once this one is fixed
158	* to create a map per module.
159	*/
160	if (machine && machine__load_kallsyms(machine, filename: "/proc/kallsyms") <= 0) {
161	machine__delete(machine);
162	machine = NULL;
163	}
164
165	return machine;
166	}
167
168	static void dsos__purge(struct dsos *dsos)
169	{
170	struct dso *pos, *n;
171
172	down_write(sem: &dsos->lock);
173
174	list_for_each_entry_safe(pos, n, &dsos->head, node) {
175	RB_CLEAR_NODE(&pos->rb_node);
176	pos->root = NULL;
177	list_del_init(entry: &pos->node);
178	dso__put(dso: pos);
179	}
180
181	up_write(sem: &dsos->lock);
182	}
183
184	static void dsos__exit(struct dsos *dsos)
185	{
186	dsos__purge(dsos);
187	exit_rwsem(sem: &dsos->lock);
188	}
189
190	void machine__delete_threads(struct machine *machine)
191	{
192	threads__remove_all_threads(threads: &machine->threads);
193	}
194
195	void machine__exit(struct machine *machine)
196	{
197	if (machine == NULL)
198	return;
199
200	machine__destroy_kernel_maps(machine);
201	maps__zput(machine->kmaps);
202	dsos__exit(dsos: &machine->dsos);
203	machine__exit_vdso(machine);
204	zfree(&machine->root_dir);
205	zfree(&machine->mmap_name);
206	zfree(&machine->current_tid);
207	zfree(&machine->kallsyms_filename);
208
209	threads__exit(threads: &machine->threads);
210	}
211
212	void machine__delete(struct machine *machine)
213	{
214	if (machine) {
215	machine__exit(machine);
216	free(machine);
217	}
218	}
219
220	void machines__init(struct machines *machines)
221	{
222	machine__init(machine: &machines->host, root_dir: "", HOST_KERNEL_ID);
223	machines->guests = RB_ROOT_CACHED;
224	}
225
226	void machines__exit(struct machines *machines)
227	{
228	machine__exit(machine: &machines->host);
229	/* XXX exit guest */
230	}
231
232	struct machine *machines__add(struct machines *machines, pid_t pid,
233	const char *root_dir)
234	{
235	struct rb_node **p = &machines->guests.rb_root.rb_node;
236	struct rb_node *parent = NULL;
237	struct machine *pos, *machine = malloc(sizeof(*machine));
238	bool leftmost = true;
239
240	if (machine == NULL)
241	return NULL;
242
243	if (machine__init(machine, root_dir, pid) != 0) {
244	free(machine);
245	return NULL;
246	}
247
248	while (*p != NULL) {
249	parent = *p;
250	pos = rb_entry(parent, struct machine, rb_node);
251	if (pid < pos->pid)
252	p = &(*p)->rb_left;
253	else {
254	p = &(*p)->rb_right;
255	leftmost = false;
256	}
257	}
258
259	rb_link_node(node: &machine->rb_node, parent, rb_link: p);
260	rb_insert_color_cached(node: &machine->rb_node, root: &machines->guests, leftmost);
261
262	machine->machines = machines;
263
264	return machine;
265	}
266
267	void machines__set_comm_exec(struct machines *machines, bool comm_exec)
268	{
269	struct rb_node *nd;
270
271	machines->host.comm_exec = comm_exec;
272
273	for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
274	struct machine *machine = rb_entry(nd, struct machine, rb_node);
275
276	machine->comm_exec = comm_exec;
277	}
278	}
279
280	struct machine *machines__find(struct machines *machines, pid_t pid)
281	{
282	struct rb_node **p = &machines->guests.rb_root.rb_node;
283	struct rb_node *parent = NULL;
284	struct machine *machine;
285	struct machine *default_machine = NULL;
286
287	if (pid == HOST_KERNEL_ID)
288	return &machines->host;
289
290	while (*p != NULL) {
291	parent = *p;
292	machine = rb_entry(parent, struct machine, rb_node);
293	if (pid < machine->pid)
294	p = &(*p)->rb_left;
295	else if (pid > machine->pid)
296	p = &(*p)->rb_right;
297	else
298	return machine;
299	if (!machine->pid)
300	default_machine = machine;
301	}
302
303	return default_machine;
304	}
305
306	struct machine *machines__findnew(struct machines *machines, pid_t pid)
307	{
308	char path[PATH_MAX];
309	const char *root_dir = "";
310	struct machine *machine = machines__find(machines, pid);
311
312	if (machine && (machine->pid == pid))
313	goto out;
314
315	if ((pid != HOST_KERNEL_ID) &&
316	(pid != DEFAULT_GUEST_KERNEL_ID) &&
317	(symbol_conf.guestmount)) {
318	sprintf(buf: path, fmt: "%s/%d", symbol_conf.guestmount, pid);
319	if (access(path, R_OK)) {
320	static struct strlist *seen;
321
322	if (!seen)
323	seen = strlist__new(NULL, NULL);
324
325	if (!strlist__has_entry(slist: seen, entry: path)) {
326	pr_err("Can't access file %s\n", path);
327	strlist__add(slist: seen, str: path);
328	}
329	machine = NULL;
330	goto out;
331	}
332	root_dir = path;
333	}
334
335	machine = machines__add(machines, pid, root_dir);
336	out:
337	return machine;
338	}
339
340	struct machine *machines__find_guest(struct machines *machines, pid_t pid)
341	{
342	struct machine *machine = machines__find(machines, pid);
343
344	if (!machine)
345	machine = machines__findnew(machines, DEFAULT_GUEST_KERNEL_ID);
346	return machine;
347	}
348
349	/*
350	* A common case for KVM test programs is that the test program acts as the
351	* hypervisor, creating, running and destroying the virtual machine, and
352	* providing the guest object code from its own object code. In this case,
353	* the VM is not running an OS, but only the functions loaded into it by the
354	* hypervisor test program, and conveniently, loaded at the same virtual
355	* addresses.
356	*
357	* Normally to resolve addresses, MMAP events are needed to map addresses
358	* back to the object code and debug symbols for that object code.
359	*
360	* Currently, there is no way to get such mapping information from guests
361	* but, in the scenario described above, the guest has the same mappings
362	* as the hypervisor, so support for that scenario can be achieved.
363	*
364	* To support that, copy the host thread's maps to the guest thread's maps.
365	* Note, we do not discover the guest until we encounter a guest event,
366	* which works well because it is not until then that we know that the host
367	* thread's maps have been set up.
368	*
369	* This function returns the guest thread. Apart from keeping the data
370	* structures sane, using a thread belonging to the guest machine, instead
371	* of the host thread, allows it to have its own comm (refer
372	* thread__set_guest_comm()).
373	*/
374	static struct thread *findnew_guest_code(struct machine *machine,
375	struct machine *host_machine,
376	pid_t pid)
377	{
378	struct thread *host_thread;
379	struct thread *thread;
380	int err;
381
382	if (!machine)
383	return NULL;
384
385	thread = machine__findnew_thread(machine, pid: -1, tid: pid);
386	if (!thread)
387	return NULL;
388
389	/* Assume maps are set up if there are any */
390	if (!maps__empty(maps: thread__maps(thread)))
391	return thread;
392
393	host_thread = machine__find_thread(machine: host_machine, pid: -1, tid: pid);
394	if (!host_thread)
395	goto out_err;
396
397	thread__set_guest_comm(thread, pid);
398
399	/*
400	* Guest code can be found in hypervisor process at the same address
401	* so copy host maps.
402	*/
403	err = maps__copy_from(maps: thread__maps(thread), parent: thread__maps(thread: host_thread));
404	thread__put(thread: host_thread);
405	if (err)
406	goto out_err;
407
408	return thread;
409
410	out_err:
411	thread__zput(thread);
412	return NULL;
413	}
414
415	struct thread *machines__findnew_guest_code(struct machines *machines, pid_t pid)
416	{
417	struct machine *host_machine = machines__find(machines, HOST_KERNEL_ID);
418	struct machine *machine = machines__findnew(machines, pid);
419
420	return findnew_guest_code(machine, host_machine, pid);
421	}
422
423	struct thread *machine__findnew_guest_code(struct machine *machine, pid_t pid)
424	{
425	struct machines *machines = machine->machines;
426	struct machine *host_machine;
427
428	if (!machines)
429	return NULL;
430
431	host_machine = machines__find(machines, HOST_KERNEL_ID);
432
433	return findnew_guest_code(machine, host_machine, pid);
434	}
435
436	void machines__process_guests(struct machines *machines,
437	machine__process_t process, void *data)
438	{
439	struct rb_node *nd;
440
441	for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
442	struct machine *pos = rb_entry(nd, struct machine, rb_node);
443	process(pos, data);
444	}
445	}
446
447	void machines__set_id_hdr_size(struct machines *machines, u16 id_hdr_size)
448	{
449	struct rb_node *node;
450	struct machine *machine;
451
452	machines->host.id_hdr_size = id_hdr_size;
453
454	for (node = rb_first_cached(&machines->guests); node;
455	node = rb_next(node)) {
456	machine = rb_entry(node, struct machine, rb_node);
457	machine->id_hdr_size = id_hdr_size;
458	}
459
460	return;
461	}
462
463	static void machine__update_thread_pid(struct machine *machine,
464	struct thread *th, pid_t pid)
465	{
466	struct thread *leader;
467
468	if (pid == thread__pid(thread: th) || pid == -1 || thread__pid(thread: th) != -1)
469	return;
470
471	thread__set_pid(thread: th, pid_: pid);
472
473	if (thread__pid(thread: th) == thread__tid(thread: th))
474	return;
475
476	leader = machine__findnew_thread(machine, pid: thread__pid(thread: th), tid: thread__pid(thread: th));
477	if (!leader)
478	goto out_err;
479
480	if (!thread__maps(thread: leader))
481	thread__set_maps(thread: leader, maps: maps__new(machine));
482
483	if (!thread__maps(thread: leader))
484	goto out_err;
485
486	if (thread__maps(thread: th) == thread__maps(thread: leader))
487	goto out_put;
488
489	if (thread__maps(thread: th)) {
490	/*
491	* Maps are created from MMAP events which provide the pid and
492	* tid. Consequently there never should be any maps on a thread
493	* with an unknown pid. Just print an error if there are.
494	*/
495	if (!maps__empty(maps: thread__maps(thread: th)))
496	pr_err("Discarding thread maps for %d:%d\n",
497	thread__pid(th), thread__tid(th));
498	maps__put(maps: thread__maps(thread: th));
499	}
500
501	thread__set_maps(thread: th, maps: maps__get(maps: thread__maps(thread: leader)));
502	out_put:
503	thread__put(thread: leader);
504	return;
505	out_err:
506	pr_err("Failed to join map groups for %d:%d\n", thread__pid(th), thread__tid(th));
507	goto out_put;
508	}
509
510	/*
511	* Caller must eventually drop thread->refcnt returned with a successful
512	* lookup/new thread inserted.
513	*/
514	static struct thread *__machine__findnew_thread(struct machine *machine,
515	pid_t pid,
516	pid_t tid,
517	bool create)
518	{
519	struct thread *th = threads__find(threads: &machine->threads, tid);
520	bool created;
521
522	if (th) {
523	machine__update_thread_pid(machine, th, pid);
524	return th;
525	}
526	if (!create)
527	return NULL;
528
529	th = threads__findnew(threads: &machine->threads, pid, tid, created: &created);
530	if (created) {
531	/*
532	* We have to initialize maps separately after rb tree is
533	* updated.
534	*
535	* The reason is that we call machine__findnew_thread within
536	* thread__init_maps to find the thread leader and that would
537	* screwed the rb tree.
538	*/
539	if (thread__init_maps(thread: th, machine)) {
540	pr_err("Thread init failed thread %d\n", pid);
541	threads__remove(threads: &machine->threads, thread: th);
542	thread__put(thread: th);
543	return NULL;
544	}
545	} else
546	machine__update_thread_pid(machine, th, pid);
547
548	return th;
549	}
550
551	struct thread *machine__findnew_thread(struct machine *machine, pid_t pid, pid_t tid)
552	{
553	return __machine__findnew_thread(machine, pid, tid, /*create=*/true);
554	}
555
556	struct thread *machine__find_thread(struct machine *machine, pid_t pid,
557	pid_t tid)
558	{
559	return __machine__findnew_thread(machine, pid, tid, /*create=*/false);
560	}
561
562	/*
563	* Threads are identified by pid and tid, and the idle task has pid == tid == 0.
564	* So here a single thread is created for that, but actually there is a separate
565	* idle task per cpu, so there should be one 'struct thread' per cpu, but there
566	* is only 1. That causes problems for some tools, requiring workarounds. For
567	* example get_idle_thread() in builtin-sched.c, or thread_stack__per_cpu().
568	*/
569	struct thread *machine__idle_thread(struct machine *machine)
570	{
571	struct thread *thread = machine__findnew_thread(machine, pid: 0, tid: 0);
572
573	if (!thread || thread__set_comm(thread, comm: "swapper", timestamp: 0) ||
574	thread__set_namespaces(thread, timestamp: 0, NULL))
575	pr_err("problem inserting idle task for machine pid %d\n", machine->pid);
576
577	return thread;
578	}
579
580	struct comm *machine__thread_exec_comm(struct machine *machine,
581	struct thread *thread)
582	{
583	if (machine->comm_exec)
584	return thread__exec_comm(thread);
585	else
586	return thread__comm(thread);
587	}
588
589	int machine__process_comm_event(struct machine *machine, union perf_event *event,
590	struct perf_sample *sample)
591	{
592	struct thread *thread = machine__findnew_thread(machine,
593	pid: event->comm.pid,
594	tid: event->comm.tid);
595	bool exec = event->header.misc & PERF_RECORD_MISC_COMM_EXEC;
596	int err = 0;
597
598	if (exec)
599	machine->comm_exec = true;
600
601	if (dump_trace)
602	perf_event__fprintf_comm(event, stdout);
603
604	if (thread == NULL ||
605	__thread__set_comm(thread, comm: event->comm.comm, timestamp: sample->time, exec)) {
606	dump_printf(fmt: "problem processing PERF_RECORD_COMM, skipping event.\n");
607	err = -1;
608	}
609
610	thread__put(thread);
611
612	return err;
613	}
614
615	int machine__process_namespaces_event(struct machine *machine __maybe_unused,
616	union perf_event *event,
617	struct perf_sample *sample __maybe_unused)
618	{
619	struct thread *thread = machine__findnew_thread(machine,
620	pid: event->namespaces.pid,
621	tid: event->namespaces.tid);
622	int err = 0;
623
624	WARN_ONCE(event->namespaces.nr_namespaces > NR_NAMESPACES,
625	"\nWARNING: kernel seems to support more namespaces than perf"
626	" tool.\nTry updating the perf tool..\n\n");
627
628	WARN_ONCE(event->namespaces.nr_namespaces < NR_NAMESPACES,
629	"\nWARNING: perf tool seems to support more namespaces than"
630	" the kernel.\nTry updating the kernel..\n\n");
631
632	if (dump_trace)
633	perf_event__fprintf_namespaces(event, stdout);
634
635	if (thread == NULL ||
636	thread__set_namespaces(thread, timestamp: sample->time, event: &event->namespaces)) {
637	dump_printf(fmt: "problem processing PERF_RECORD_NAMESPACES, skipping event.\n");
638	err = -1;
639	}
640
641	thread__put(thread);
642
643	return err;
644	}
645
646	int machine__process_cgroup_event(struct machine *machine,
647	union perf_event *event,
648	struct perf_sample *sample __maybe_unused)
649	{
650	struct cgroup *cgrp;
651
652	if (dump_trace)
653	perf_event__fprintf_cgroup(event, stdout);
654
655	cgrp = cgroup__findnew(env: machine->env, id: event->cgroup.id, path: event->cgroup.path);
656	if (cgrp == NULL)
657	return -ENOMEM;
658
659	return 0;
660	}
661
662	int machine__process_lost_event(struct machine *machine __maybe_unused,
663	union perf_event *event, struct perf_sample *sample __maybe_unused)
664	{
665	dump_printf(": id:%" PRI_lu64 ": lost:%" PRI_lu64 "\n",
666	event->lost.id, event->lost.lost);
667	return 0;
668	}
669
670	int machine__process_lost_samples_event(struct machine *machine __maybe_unused,
671	union perf_event *event, struct perf_sample *sample)
672	{
673	dump_printf(": id:%" PRIu64 ": lost samples :%" PRI_lu64 "\n",
674	sample->id, event->lost_samples.lost);
675	return 0;
676	}
677
678	static struct dso *machine__findnew_module_dso(struct machine *machine,
679	struct kmod_path *m,
680	const char *filename)
681	{
682	struct dso *dso;
683
684	down_write(sem: &machine->dsos.lock);
685
686	dso = __dsos__find(dsos: &machine->dsos, name: m->name, cmp_short: true);
687	if (!dso) {
688	dso = __dsos__addnew(dsos: &machine->dsos, name: m->name);
689	if (dso == NULL)
690	goto out_unlock;
691
692	dso__set_module_info(dso, m, machine);
693	dso__set_long_name(dso, name: strdup(filename), name_allocated: true);
694	dso->kernel = DSO_SPACE__KERNEL;
695	}
696
697	dso__get(dso);
698	out_unlock:
699	up_write(sem: &machine->dsos.lock);
700	return dso;
701	}
702
703	int machine__process_aux_event(struct machine *machine __maybe_unused,
704	union perf_event *event)
705	{
706	if (dump_trace)
707	perf_event__fprintf_aux(event, stdout);
708	return 0;
709	}
710
711	int machine__process_itrace_start_event(struct machine *machine __maybe_unused,
712	union perf_event *event)
713	{
714	if (dump_trace)
715	perf_event__fprintf_itrace_start(event, stdout);
716	return 0;
717	}
718
719	int machine__process_aux_output_hw_id_event(struct machine *machine __maybe_unused,
720	union perf_event *event)
721	{
722	if (dump_trace)
723	perf_event__fprintf_aux_output_hw_id(event, stdout);
724	return 0;
725	}
726
727	int machine__process_switch_event(struct machine *machine __maybe_unused,
728	union perf_event *event)
729	{
730	if (dump_trace)
731	perf_event__fprintf_switch(event, stdout);
732	return 0;
733	}
734
735	static int machine__process_ksymbol_register(struct machine *machine,
736	union perf_event *event,
737	struct perf_sample *sample __maybe_unused)
738	{
739	struct symbol *sym;
740	struct dso *dso;
741	struct map *map = maps__find(maps: machine__kernel_maps(machine), addr: event->ksymbol.addr);
742	int err = 0;
743
744	if (!map) {
745	dso = dso__new(name: event->ksymbol.name);
746
747	if (!dso) {
748	err = -ENOMEM;
749	goto out;
750	}
751	dso->kernel = DSO_SPACE__KERNEL;
752	map = map__new2(start: 0, dso);
753	dso__put(dso);
754	if (!map) {
755	err = -ENOMEM;
756	goto out;
757	}
758	if (event->ksymbol.ksym_type == PERF_RECORD_KSYMBOL_TYPE_OOL) {
759	dso->binary_type = DSO_BINARY_TYPE__OOL;
760	dso->data.file_size = event->ksymbol.len;
761	dso__set_loaded(dso);
762	}
763
764	map__set_start(map, start: event->ksymbol.addr);
765	map__set_end(map, end: map__start(map) + event->ksymbol.len);
766	err = maps__insert(maps: machine__kernel_maps(machine), map);
767	if (err) {
768	err = -ENOMEM;
769	goto out;
770	}
771
772	dso__set_loaded(dso);
773
774	if (is_bpf_image(name: event->ksymbol.name)) {
775	dso->binary_type = DSO_BINARY_TYPE__BPF_IMAGE;
776	dso__set_long_name(dso, name: "", name_allocated: false);
777	}
778	} else {
779	dso = map__dso(map);
780	}
781
782	sym = symbol__new(start: map__map_ip(map, ip_or_rip: map__start(map)),
783	len: event->ksymbol.len,
784	binding: 0, type: 0, name: event->ksymbol.name);
785	if (!sym) {
786	err = -ENOMEM;
787	goto out;
788	}
789	dso__insert_symbol(dso, sym);
790	out:
791	map__put(map);
792	return err;
793	}
794
795	static int machine__process_ksymbol_unregister(struct machine *machine,
796	union perf_event *event,
797	struct perf_sample *sample __maybe_unused)
798	{
799	struct symbol *sym;
800	struct map *map;
801
802	map = maps__find(maps: machine__kernel_maps(machine), addr: event->ksymbol.addr);
803	if (!map)
804	return 0;
805
806	if (!RC_CHK_EQUAL(map, machine->vmlinux_map))
807	maps__remove(maps: machine__kernel_maps(machine), map);
808	else {
809	struct dso *dso = map__dso(map);
810
811	sym = dso__find_symbol(dso, addr: map__map_ip(map, ip_or_rip: map__start(map)));
812	if (sym)
813	dso__delete_symbol(dso, sym);
814	}
815	map__put(map);
816	return 0;
817	}
818
819	int machine__process_ksymbol(struct machine *machine __maybe_unused,
820	union perf_event *event,
821	struct perf_sample *sample)
822	{
823	if (dump_trace)
824	perf_event__fprintf_ksymbol(event, stdout);
825
826	if (event->ksymbol.flags & PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER)
827	return machine__process_ksymbol_unregister(machine, event,
828	sample);
829	return machine__process_ksymbol_register(machine, event, sample);
830	}
831
832	int machine__process_text_poke(struct machine *machine, union perf_event *event,
833	struct perf_sample *sample __maybe_unused)
834	{
835	struct map *map = maps__find(maps: machine__kernel_maps(machine), addr: event->text_poke.addr);
836	u8 cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
837	struct dso *dso = map ? map__dso(map) : NULL;
838
839	if (dump_trace)
840	perf_event__fprintf_text_poke(event, machine, stdout);
841
842	if (!event->text_poke.new_len)
843	goto out;
844
845	if (cpumode != PERF_RECORD_MISC_KERNEL) {
846	pr_debug("%s: unsupported cpumode - ignoring\n", __func__);
847	goto out;
848	}
849
850	if (dso) {
851	u8 *new_bytes = event->text_poke.bytes + event->text_poke.old_len;
852	int ret;
853
854	/*
855	* Kernel maps might be changed when loading symbols so loading
856	* must be done prior to using kernel maps.
857	*/
858	map__load(map);
859	ret = dso__data_write_cache_addr(dso, map, machine,
860	addr: event->text_poke.addr,
861	data: new_bytes,
862	size: event->text_poke.new_len);
863	if (ret != event->text_poke.new_len)
864	pr_debug("Failed to write kernel text poke at %#" PRI_lx64 "\n",
865	event->text_poke.addr);
866	} else {
867	pr_debug("Failed to find kernel text poke address map for %#" PRI_lx64 "\n",
868	event->text_poke.addr);
869	}
870	out:
871	map__put(map);
872	return 0;
873	}
874
875	static struct map *machine__addnew_module_map(struct machine *machine, u64 start,
876	const char *filename)
877	{
878	struct map *map = NULL;
879	struct kmod_path m;
880	struct dso *dso;
881	int err;
882
883	if (kmod_path__parse_name(&m, filename))
884	return NULL;
885
886	dso = machine__findnew_module_dso(machine, m: &m, filename);
887	if (dso == NULL)
888	goto out;
889
890	map = map__new2(start, dso);
891	if (map == NULL)
892	goto out;
893
894	err = maps__insert(maps: machine__kernel_maps(machine), map);
895	/* If maps__insert failed, return NULL. */
896	if (err) {
897	map__put(map);
898	map = NULL;
899	}
900	out:
901	/* put the dso here, corresponding to machine__findnew_module_dso */
902	dso__put(dso);
903	zfree(&m.name);
904	return map;
905	}
906
907	size_t machines__fprintf_dsos(struct machines *machines, FILE *fp)
908	{
909	struct rb_node *nd;
910	size_t ret = __dsos__fprintf(&machines->host.dsos.head, fp);
911
912	for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
913	struct machine *pos = rb_entry(nd, struct machine, rb_node);
914	ret += __dsos__fprintf(&pos->dsos.head, fp);
915	}
916
917	return ret;
918	}
919
920	size_t machine__fprintf_dsos_buildid(struct machine *m, FILE *fp,
921	bool (skip)(struct dso *dso, int parm), int parm)
922	{
923	return __dsos__fprintf_buildid(&m->dsos.head, fp, skip, parm);
924	}
925
926	size_t machines__fprintf_dsos_buildid(struct machines *machines, FILE *fp,
927	bool (skip)(struct dso *dso, int parm), int parm)
928	{
929	struct rb_node *nd;
930	size_t ret = machine__fprintf_dsos_buildid(&machines->host, fp, skip, parm);
931
932	for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
933	struct machine *pos = rb_entry(nd, struct machine, rb_node);
934	ret += machine__fprintf_dsos_buildid(pos, fp, skip, parm);
935	}
936	return ret;
937	}
938
939	size_t machine__fprintf_vmlinux_path(struct machine *machine, FILE *fp)
940	{
941	int i;
942	size_t printed = 0;
943	struct dso *kdso = machine__kernel_dso(machine);
944
945	if (kdso->has_build_id) {
946	char filename[PATH_MAX];
947	if (dso__build_id_filename(dso: kdso, bf: filename, size: sizeof(filename),
948	is_debug: false))
949	printed += fprintf(fp, "[0] %s\n", filename);
950	}
951
952	for (i = 0; i < vmlinux_path__nr_entries; ++i)
953	printed += fprintf(fp, "[%d] %s\n",
954	i + kdso->has_build_id, vmlinux_path[i]);
955
956	return printed;
957	}
958
959	struct machine_fprintf_cb_args {
960	FILE *fp;
961	size_t printed;
962	};
963
964	static int machine_fprintf_cb(struct thread *thread, void *data)
965	{
966	struct machine_fprintf_cb_args *args = data;
967
968	/* TODO: handle fprintf errors. */
969	args->printed += thread__fprintf(thread, args->fp);
970	return 0;
971	}
972
973	size_t machine__fprintf(struct machine *machine, FILE *fp)
974	{
975	struct machine_fprintf_cb_args args = {
976	.fp = fp,
977	.printed = 0,
978	};
979	size_t ret = fprintf(fp, "Threads: %zu\n", threads__nr(threads: &machine->threads));
980
981	machine__for_each_thread(machine, fn: machine_fprintf_cb, priv: &args);
982	return ret + args.printed;
983	}
984
985	static struct dso *machine__get_kernel(struct machine *machine)
986	{
987	const char *vmlinux_name = machine->mmap_name;
988	struct dso *kernel;
989
990	if (machine__is_host(machine)) {
991	if (symbol_conf.vmlinux_name)
992	vmlinux_name = symbol_conf.vmlinux_name;
993
994	kernel = machine__findnew_kernel(machine, name: vmlinux_name,
995	short_name: "[kernel]", dso_type: DSO_SPACE__KERNEL);
996	} else {
997	if (symbol_conf.default_guest_vmlinux_name)
998	vmlinux_name = symbol_conf.default_guest_vmlinux_name;
999
1000	kernel = machine__findnew_kernel(machine, name: vmlinux_name,
1001	short_name: "[guest.kernel]",
1002	dso_type: DSO_SPACE__KERNEL_GUEST);
1003	}
1004
1005	if (kernel != NULL && (!kernel->has_build_id))
1006	dso__read_running_kernel_build_id(dso: kernel, machine);
1007
1008	return kernel;
1009	}
1010
1011	void machine__get_kallsyms_filename(struct machine *machine, char *buf,
1012	size_t bufsz)
1013	{
1014	if (machine__is_default_guest(machine))
1015	scnprintf(buf, size: bufsz, fmt: "%s", symbol_conf.default_guest_kallsyms);
1016	else
1017	scnprintf(buf, size: bufsz, fmt: "%s/proc/kallsyms", machine->root_dir);
1018	}
1019
1020	const char *ref_reloc_sym_names[] = {"_text", "_stext", NULL};
1021
1022	/* Figure out the start address of kernel map from /proc/kallsyms.
1023	* Returns the name of the start symbol in *symbol_name. Pass in NULL as
1024	* symbol_name if it's not that important.
1025	*/
1026	static int machine__get_running_kernel_start(struct machine *machine,
1027	const char **symbol_name,
1028	u64 *start, u64 *end)
1029	{
1030	char filename[PATH_MAX];
1031	int i, err = -1;
1032	const char *name;
1033	u64 addr = 0;
1034
1035	machine__get_kallsyms_filename(machine, buf: filename, PATH_MAX);
1036
1037	if (symbol__restricted_filename(filename, restricted_filename: "/proc/kallsyms"))
1038	return 0;
1039
1040	for (i = 0; (name = ref_reloc_sym_names[i]) != NULL; i++) {
1041	err = kallsyms__get_function_start(kallsyms_filename: filename, symbol_name: name, addr: &addr);
1042	if (!err)
1043	break;
1044	}
1045
1046	if (err)
1047	return -1;
1048
1049	if (symbol_name)
1050	*symbol_name = name;
1051
1052	*start = addr;
1053
1054	err = kallsyms__get_symbol_start(kallsyms_filename: filename, symbol_name: "_edata", addr: &addr);
1055	if (err)
1056	err = kallsyms__get_function_start(kallsyms_filename: filename, symbol_name: "_etext", addr: &addr);
1057	if (!err)
1058	*end = addr;
1059
1060	return 0;
1061	}
1062
1063	int machine__create_extra_kernel_map(struct machine *machine,
1064	struct dso *kernel,
1065	struct extra_kernel_map *xm)
1066	{
1067	struct kmap *kmap;
1068	struct map *map;
1069	int err;
1070
1071	map = map__new2(start: xm->start, dso: kernel);
1072	if (!map)
1073	return -ENOMEM;
1074
1075	map__set_end(map, end: xm->end);
1076	map__set_pgoff(map, pgoff: xm->pgoff);
1077
1078	kmap = map__kmap(map);
1079
1080	strlcpy(kmap->name, xm->name, KMAP_NAME_LEN);
1081
1082	err = maps__insert(maps: machine__kernel_maps(machine), map);
1083
1084	if (!err) {
1085	pr_debug2("Added extra kernel map %s %" PRIx64 "-%" PRIx64 "\n",
1086	kmap->name, map__start(map), map__end(map));
1087	}
1088
1089	map__put(map);
1090
1091	return err;
1092	}
1093
1094	static u64 find_entry_trampoline(struct dso *dso)
1095	{
1096	/* Duplicates are removed so lookup all aliases */
1097	const char *syms[] = {
1098	"_entry_trampoline",
1099	"__entry_trampoline_start",
1100	"entry_SYSCALL_64_trampoline",
1101	};
1102	struct symbol *sym = dso__first_symbol(dso);
1103	unsigned int i;
1104
1105	for (; sym; sym = dso__next_symbol(sym)) {
1106	if (sym->binding != STB_GLOBAL)
1107	continue;
1108	for (i = 0; i < ARRAY_SIZE(syms); i++) {
1109	if (!strcmp(sym->name, syms[i]))
1110	return sym->start;
1111	}
1112	}
1113
1114	return 0;
1115	}
1116
1117	/*
1118	* These values can be used for kernels that do not have symbols for the entry
1119	* trampolines in kallsyms.
1120	*/
1121	#define X86_64_CPU_ENTRY_AREA_PER_CPU 0xfffffe0000000000ULL
1122	#define X86_64_CPU_ENTRY_AREA_SIZE 0x2c000
1123	#define X86_64_ENTRY_TRAMPOLINE 0x6000
1124
1125	struct machine__map_x86_64_entry_trampolines_args {
1126	struct maps *kmaps;
1127	bool found;
1128	};
1129
1130	static int machine__map_x86_64_entry_trampolines_cb(struct map *map, void *data)
1131	{
1132	struct machine__map_x86_64_entry_trampolines_args *args = data;
1133	struct map *dest_map;
1134	struct kmap *kmap = __map__kmap(map);
1135
1136	if (!kmap || !is_entry_trampoline(name: kmap->name))
1137	return 0;
1138
1139	dest_map = maps__find(maps: args->kmaps, addr: map__pgoff(map));
1140	if (RC_CHK_ACCESS(dest_map) != RC_CHK_ACCESS(map))
1141	map__set_pgoff(map, pgoff: map__map_ip(map: dest_map, ip_or_rip: map__pgoff(map)));
1142
1143	map__put(map: dest_map);
1144	args->found = true;
1145	return 0;
1146	}
1147
1148	/* Map x86_64 PTI entry trampolines */
1149	int machine__map_x86_64_entry_trampolines(struct machine *machine,
1150	struct dso *kernel)
1151	{
1152	struct machine__map_x86_64_entry_trampolines_args args = {
1153	.kmaps = machine__kernel_maps(machine),
1154	.found = false,
1155	};
1156	int nr_cpus_avail, cpu;
1157	u64 pgoff;
1158
1159	/*
1160	* In the vmlinux case, pgoff is a virtual address which must now be
1161	* mapped to a vmlinux offset.
1162	*/
1163	maps__for_each_map(maps: args.kmaps, cb: machine__map_x86_64_entry_trampolines_cb, data: &args);
1164
1165	if (args.found || machine->trampolines_mapped)
1166	return 0;
1167
1168	pgoff = find_entry_trampoline(dso: kernel);
1169	if (!pgoff)
1170	return 0;
1171
1172	nr_cpus_avail = machine__nr_cpus_avail(machine);
1173
1174	/* Add a 1 page map for each CPU's entry trampoline */
1175	for (cpu = 0; cpu < nr_cpus_avail; cpu++) {
1176	u64 va = X86_64_CPU_ENTRY_AREA_PER_CPU +
1177	cpu * X86_64_CPU_ENTRY_AREA_SIZE +
1178	X86_64_ENTRY_TRAMPOLINE;
1179	struct extra_kernel_map xm = {
1180	.start = va,
1181	.end = va + page_size,
1182	.pgoff = pgoff,
1183	};
1184
1185	strlcpy(xm.name, ENTRY_TRAMPOLINE_NAME, KMAP_NAME_LEN);
1186
1187	if (machine__create_extra_kernel_map(machine, kernel, xm: &xm) < 0)
1188	return -1;
1189	}
1190
1191	machine->trampolines_mapped = nr_cpus_avail;
1192
1193	return 0;
1194	}
1195
1196	int __weak machine__create_extra_kernel_maps(struct machine *machine __maybe_unused,
1197	struct dso *kernel __maybe_unused)
1198	{
1199	return 0;
1200	}
1201
1202	static int
1203	__machine__create_kernel_maps(struct machine *machine, struct dso *kernel)
1204	{
1205	/* In case of renewal the kernel map, destroy previous one */
1206	machine__destroy_kernel_maps(machine);
1207
1208	map__put(map: machine->vmlinux_map);
1209	machine->vmlinux_map = map__new2(start: 0, dso: kernel);
1210	if (machine->vmlinux_map == NULL)
1211	return -ENOMEM;
1212
1213	map__set_mapping_type(map: machine->vmlinux_map, type: MAPPING_TYPE__IDENTITY);
1214	return maps__insert(maps: machine__kernel_maps(machine), map: machine->vmlinux_map);
1215	}
1216
1217	void machine__destroy_kernel_maps(struct machine *machine)
1218	{
1219	struct kmap *kmap;
1220	struct map *map = machine__kernel_map(machine);
1221
1222	if (map == NULL)
1223	return;
1224
1225	kmap = map__kmap(map);
1226	maps__remove(maps: machine__kernel_maps(machine), map);
1227	if (kmap && kmap->ref_reloc_sym) {
1228	zfree((char **)&kmap->ref_reloc_sym->name);
1229	zfree(&kmap->ref_reloc_sym);
1230	}
1231
1232	map__zput(machine->vmlinux_map);
1233	}
1234
1235	int machines__create_guest_kernel_maps(struct machines *machines)
1236	{
1237	int ret = 0;
1238	struct dirent **namelist = NULL;
1239	int i, items = 0;
1240	char path[PATH_MAX];
1241	pid_t pid;
1242	char *endp;
1243
1244	if (symbol_conf.default_guest_vmlinux_name ||
1245	symbol_conf.default_guest_modules ||
1246	symbol_conf.default_guest_kallsyms) {
1247	machines__create_kernel_maps(machines, DEFAULT_GUEST_KERNEL_ID);
1248	}
1249
1250	if (symbol_conf.guestmount) {
1251	items = scandir(symbol_conf.guestmount, &namelist, NULL, NULL);
1252	if (items <= 0)
1253	return -ENOENT;
1254	for (i = 0; i < items; i++) {
1255	if (!isdigit(c: namelist[i]->d_name[0])) {
1256	/* Filter out . and .. */
1257	continue;
1258	}
1259	pid = (pid_t)strtol(namelist[i]->d_name, &endp, 10);
1260	if ((*endp != '\0') ||
1261	(endp == namelist[i]->d_name) ||
1262	(errno == ERANGE)) {
1263	pr_debug("invalid directory (%s). Skipping.\n",
1264	namelist[i]->d_name);
1265	continue;
1266	}
1267	sprintf(buf: path, fmt: "%s/%s/proc/kallsyms",
1268	symbol_conf.guestmount,
1269	namelist[i]->d_name);
1270	ret = access(path, R_OK);
1271	if (ret) {
1272	pr_debug("Can't access file %s\n", path);
1273	goto failure;
1274	}
1275	machines__create_kernel_maps(machines, pid);
1276	}
1277	failure:
1278	free(namelist);
1279	}
1280
1281	return ret;
1282	}
1283
1284	void machines__destroy_kernel_maps(struct machines *machines)
1285	{
1286	struct rb_node *next = rb_first_cached(&machines->guests);
1287
1288	machine__destroy_kernel_maps(machine: &machines->host);
1289
1290	while (next) {
1291	struct machine *pos = rb_entry(next, struct machine, rb_node);
1292
1293	next = rb_next(&pos->rb_node);
1294	rb_erase_cached(node: &pos->rb_node, root: &machines->guests);
1295	machine__delete(machine: pos);
1296	}
1297	}
1298
1299	int machines__create_kernel_maps(struct machines *machines, pid_t pid)
1300	{
1301	struct machine *machine = machines__findnew(machines, pid);
1302
1303	if (machine == NULL)
1304	return -1;
1305
1306	return machine__create_kernel_maps(machine);
1307	}
1308
1309	int machine__load_kallsyms(struct machine *machine, const char *filename)
1310	{
1311	struct map *map = machine__kernel_map(machine);
1312	struct dso *dso = map__dso(map);
1313	int ret = __dso__load_kallsyms(dso, filename, map, no_kcore: true);
1314
1315	if (ret > 0) {
1316	dso__set_loaded(dso);
1317	/*
1318	* Since /proc/kallsyms will have multiple sessions for the
1319	* kernel, with modules between them, fixup the end of all
1320	* sections.
1321	*/
1322	maps__fixup_end(maps: machine__kernel_maps(machine));
1323	}
1324
1325	return ret;
1326	}
1327
1328	int machine__load_vmlinux_path(struct machine *machine)
1329	{
1330	struct map *map = machine__kernel_map(machine);
1331	struct dso *dso = map__dso(map);
1332	int ret = dso__load_vmlinux_path(dso, map);
1333
1334	if (ret > 0)
1335	dso__set_loaded(dso);
1336
1337	return ret;
1338	}
1339
1340	static char *get_kernel_version(const char *root_dir)
1341	{
1342	char version[PATH_MAX];
1343	FILE *file;
1344	char *name, *tmp;
1345	const char *prefix = "Linux version ";
1346
1347	sprintf(buf: version, fmt: "%s/proc/version", root_dir);
1348	file = fopen(version, "r");
1349	if (!file)
1350	return NULL;
1351
1352	tmp = fgets(version, sizeof(version), file);
1353	fclose(file);
1354	if (!tmp)
1355	return NULL;
1356
1357	name = strstr(version, prefix);
1358	if (!name)
1359	return NULL;
1360	name += strlen(prefix);
1361	tmp = strchr(name, ' ');
1362	if (tmp)
1363	*tmp = '\0';
1364
1365	return strdup(name);
1366	}
1367
1368	static bool is_kmod_dso(struct dso *dso)
1369	{
1370	return dso->symtab_type == DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE ||
1371	dso->symtab_type == DSO_BINARY_TYPE__GUEST_KMODULE;
1372	}
1373
1374	static int maps__set_module_path(struct maps *maps, const char *path, struct kmod_path *m)
1375	{
1376	char *long_name;
1377	struct dso *dso;
1378	struct map *map = maps__find_by_name(maps, name: m->name);
1379
1380	if (map == NULL)
1381	return 0;
1382
1383	long_name = strdup(path);
1384	if (long_name == NULL) {
1385	map__put(map);
1386	return -ENOMEM;
1387	}
1388
1389	dso = map__dso(map);
1390	dso__set_long_name(dso, name: long_name, name_allocated: true);
1391	dso__kernel_module_get_build_id(dso, root_dir: "");
1392
1393	/*
1394	* Full name could reveal us kmod compression, so
1395	* we need to update the symtab_type if needed.
1396	*/
1397	if (m->comp && is_kmod_dso(dso)) {
1398	dso->symtab_type++;
1399	dso->comp = m->comp;
1400	}
1401	map__put(map);
1402	return 0;
1403	}
1404
1405	static int maps__set_modules_path_dir(struct maps *maps, const char *dir_name, int depth)
1406	{
1407	struct dirent *dent;
1408	DIR *dir = opendir(dir_name);
1409	int ret = 0;
1410
1411	if (!dir) {
1412	pr_debug("%s: cannot open %s dir\n", __func__, dir_name);
1413	return -1;
1414	}
1415
1416	while ((dent = readdir(dir)) != NULL) {
1417	char path[PATH_MAX];
1418	struct stat st;
1419
1420	/*sshfs might return bad dent->d_type, so we have to stat*/
1421	path__join(bf: path, size: sizeof(path), path1: dir_name, path2: dent->d_name);
1422	if (stat(path, &st))
1423	continue;
1424
1425	if (S_ISDIR(st.st_mode)) {
1426	if (!strcmp(dent->d_name, ".") ||
1427	!strcmp(dent->d_name, ".."))
1428	continue;
1429
1430	/* Do not follow top-level source and build symlinks */
1431	if (depth == 0) {
1432	if (!strcmp(dent->d_name, "source") ||
1433	!strcmp(dent->d_name, "build"))
1434	continue;
1435	}
1436
1437	ret = maps__set_modules_path_dir(maps, dir_name: path, depth: depth + 1);
1438	if (ret < 0)
1439	goto out;
1440	} else {
1441	struct kmod_path m;
1442
1443	ret = kmod_path__parse_name(&m, dent->d_name);
1444	if (ret)
1445	goto out;
1446
1447	if (m.kmod)
1448	ret = maps__set_module_path(maps, path, m: &m);
1449
1450	zfree(&m.name);
1451
1452	if (ret)
1453	goto out;
1454	}
1455	}
1456
1457	out:
1458	closedir(dir);
1459	return ret;
1460	}
1461
1462	static int machine__set_modules_path(struct machine *machine)
1463	{
1464	char *version;
1465	char modules_path[PATH_MAX];
1466
1467	version = get_kernel_version(root_dir: machine->root_dir);
1468	if (!version)
1469	return -1;
1470
1471	snprintf(buf: modules_path, size: sizeof(modules_path), fmt: "%s/lib/modules/%s",
1472	machine->root_dir, version);
1473	free(version);
1474
1475	return maps__set_modules_path_dir(maps: machine__kernel_maps(machine), dir_name: modules_path, depth: 0);
1476	}
1477	int __weak arch__fix_module_text_start(u64 *start __maybe_unused,
1478	u64 *size __maybe_unused,
1479	const char *name __maybe_unused)
1480	{
1481	return 0;
1482	}
1483
1484	static int machine__create_module(void *arg, const char *name, u64 start,
1485	u64 size)
1486	{
1487	struct machine *machine = arg;
1488	struct map *map;
1489
1490	if (arch__fix_module_text_start(start: &start, size: &size, name) < 0)
1491	return -1;
1492
1493	map = machine__addnew_module_map(machine, start, filename: name);
1494	if (map == NULL)
1495	return -1;
1496	map__set_end(map, end: start + size);
1497
1498	dso__kernel_module_get_build_id(dso: map__dso(map), root_dir: machine->root_dir);
1499	map__put(map);
1500	return 0;
1501	}
1502
1503	static int machine__create_modules(struct machine *machine)
1504	{
1505	const char *modules;
1506	char path[PATH_MAX];
1507
1508	if (machine__is_default_guest(machine)) {
1509	modules = symbol_conf.default_guest_modules;
1510	} else {
1511	snprintf(buf: path, PATH_MAX, fmt: "%s/proc/modules", machine->root_dir);
1512	modules = path;
1513	}
1514
1515	if (symbol__restricted_filename(filename: modules, restricted_filename: "/proc/modules"))
1516	return -1;
1517
1518	if (modules__parse(filename: modules, arg: machine, process_module: machine__create_module))
1519	return -1;
1520
1521	if (!machine__set_modules_path(machine))
1522	return 0;
1523
1524	pr_debug("Problems setting modules path maps, continuing anyway...\n");
1525
1526	return 0;
1527	}
1528
1529	static void machine__set_kernel_mmap(struct machine *machine,
1530	u64 start, u64 end)
1531	{
1532	map__set_start(map: machine->vmlinux_map, start);
1533	map__set_end(map: machine->vmlinux_map, end);
1534	/*
1535	* Be a bit paranoid here, some perf.data file came with
1536	* a zero sized synthesized MMAP event for the kernel.
1537	*/
1538	if (start == 0 && end == 0)
1539	map__set_end(map: machine->vmlinux_map, end: ~0ULL);
1540	}
1541
1542	static int machine__update_kernel_mmap(struct machine *machine,
1543	u64 start, u64 end)
1544	{
1545	struct map *orig, *updated;
1546	int err;
1547
1548	orig = machine->vmlinux_map;
1549	updated = map__get(map: orig);
1550
1551	machine->vmlinux_map = updated;
1552	machine__set_kernel_mmap(machine, start, end);
1553	maps__remove(maps: machine__kernel_maps(machine), map: orig);
1554	err = maps__insert(maps: machine__kernel_maps(machine), map: updated);
1555	map__put(map: orig);
1556
1557	return err;
1558	}
1559
1560	int machine__create_kernel_maps(struct machine *machine)
1561	{
1562	struct dso *kernel = machine__get_kernel(machine);
1563	const char *name = NULL;
1564	u64 start = 0, end = ~0ULL;
1565	int ret;
1566
1567	if (kernel == NULL)
1568	return -1;
1569
1570	ret = __machine__create_kernel_maps(machine, kernel);
1571	if (ret < 0)
1572	goto out_put;
1573
1574	if (symbol_conf.use_modules && machine__create_modules(machine) < 0) {
1575	if (machine__is_host(machine))
1576	pr_debug("Problems creating module maps, "
1577	"continuing anyway...\n");
1578	else
1579	pr_debug("Problems creating module maps for guest %d, "
1580	"continuing anyway...\n", machine->pid);
1581	}
1582
1583	if (!machine__get_running_kernel_start(machine, symbol_name: &name, start: &start, end: &end)) {
1584	if (name &&
1585	map__set_kallsyms_ref_reloc_sym(map: machine->vmlinux_map, symbol_name: name, addr: start)) {
1586	machine__destroy_kernel_maps(machine);
1587	ret = -1;
1588	goto out_put;
1589	}
1590
1591	/*
1592	* we have a real start address now, so re-order the kmaps
1593	* assume it's the last in the kmaps
1594	*/
1595	ret = machine__update_kernel_mmap(machine, start, end);
1596	if (ret < 0)
1597	goto out_put;
1598	}
1599
1600	if (machine__create_extra_kernel_maps(machine, kernel))
1601	pr_debug("Problems creating extra kernel maps, continuing anyway...\n");
1602
1603	if (end == ~0ULL) {
1604	/* update end address of the kernel map using adjacent module address */
1605	struct map *next = maps__find_next_entry(maps: machine__kernel_maps(machine),
1606	map: machine__kernel_map(machine));
1607
1608	if (next) {
1609	machine__set_kernel_mmap(machine, start, end: map__start(map: next));
1610	map__put(map: next);
1611	}
1612	}
1613
1614	out_put:
1615	dso__put(dso: kernel);
1616	return ret;
1617	}
1618
1619	static bool machine__uses_kcore(struct machine *machine)
1620	{
1621	struct dso *dso;
1622
1623	list_for_each_entry(dso, &machine->dsos.head, node) {
1624	if (dso__is_kcore(dso))
1625	return true;
1626	}
1627
1628	return false;
1629	}
1630
1631	static bool perf_event__is_extra_kernel_mmap(struct machine *machine,
1632	struct extra_kernel_map *xm)
1633	{
1634	return machine__is(machine, arch: "x86_64") &&
1635	is_entry_trampoline(name: xm->name);
1636	}
1637
1638	static int machine__process_extra_kernel_map(struct machine *machine,
1639	struct extra_kernel_map *xm)
1640	{
1641	struct dso *kernel = machine__kernel_dso(machine);
1642
1643	if (kernel == NULL)
1644	return -1;
1645
1646	return machine__create_extra_kernel_map(machine, kernel, xm);
1647	}
1648
1649	static int machine__process_kernel_mmap_event(struct machine *machine,
1650	struct extra_kernel_map *xm,
1651	struct build_id *bid)
1652	{
1653	enum dso_space_type dso_space;
1654	bool is_kernel_mmap;
1655	const char *mmap_name = machine->mmap_name;
1656
1657	/* If we have maps from kcore then we do not need or want any others */
1658	if (machine__uses_kcore(machine))
1659	return 0;
1660
1661	if (machine__is_host(machine))
1662	dso_space = DSO_SPACE__KERNEL;
1663	else
1664	dso_space = DSO_SPACE__KERNEL_GUEST;
1665
1666	is_kernel_mmap = memcmp(p: xm->name, q: mmap_name, strlen(mmap_name) - 1) == 0;
1667	if (!is_kernel_mmap && !machine__is_host(machine)) {
1668	/*
1669	* If the event was recorded inside the guest and injected into
1670	* the host perf.data file, then it will match a host mmap_name,
1671	* so try that - see machine__set_mmap_name().
1672	*/
1673	mmap_name = "[kernel.kallsyms]";
1674	is_kernel_mmap = memcmp(p: xm->name, q: mmap_name, strlen(mmap_name) - 1) == 0;
1675	}
1676	if (xm->name[0] == '/' ||
1677	(!is_kernel_mmap && xm->name[0] == '[')) {
1678	struct map *map = machine__addnew_module_map(machine, start: xm->start, filename: xm->name);
1679
1680	if (map == NULL)
1681	goto out_problem;
1682
1683	map__set_end(map, end: map__start(map) + xm->end - xm->start);
1684
1685	if (build_id__is_defined(bid))
1686	dso__set_build_id(dso: map__dso(map), bid);
1687
1688	map__put(map);
1689	} else if (is_kernel_mmap) {
1690	const char *symbol_name = xm->name + strlen(mmap_name);
1691	/*
1692	* Should be there already, from the build-id table in
1693	* the header.
1694	*/
1695	struct dso *kernel = NULL;
1696	struct dso *dso;
1697
1698	down_read(sem: &machine->dsos.lock);
1699
1700	list_for_each_entry(dso, &machine->dsos.head, node) {
1701
1702	/*
1703	* The cpumode passed to is_kernel_module is not the
1704	* cpumode of *this* event. If we insist on passing
1705	* correct cpumode to is_kernel_module, we should
1706	* record the cpumode when we adding this dso to the
1707	* linked list.
1708	*
1709	* However we don't really need passing correct
1710	* cpumode. We know the correct cpumode must be kernel
1711	* mode (if not, we should not link it onto kernel_dsos
1712	* list).
1713	*
1714	* Therefore, we pass PERF_RECORD_MISC_CPUMODE_UNKNOWN.
1715	* is_kernel_module() treats it as a kernel cpumode.
1716	*/
1717
1718	if (!dso->kernel ||
1719	is_kernel_module(pathname: dso->long_name,
1720	PERF_RECORD_MISC_CPUMODE_UNKNOWN))
1721	continue;
1722
1723
1724	kernel = dso__get(dso);
1725	break;
1726	}
1727
1728	up_read(sem: &machine->dsos.lock);
1729
1730	if (kernel == NULL)
1731	kernel = machine__findnew_dso(machine, filename: machine->mmap_name);
1732	if (kernel == NULL)
1733	goto out_problem;
1734
1735	kernel->kernel = dso_space;
1736	if (__machine__create_kernel_maps(machine, kernel) < 0) {
1737	dso__put(dso: kernel);
1738	goto out_problem;
1739	}
1740
1741	if (strstr(kernel->long_name, "vmlinux"))
1742	dso__set_short_name(dso: kernel, name: "[kernel.vmlinux]", name_allocated: false);
1743
1744	if (machine__update_kernel_mmap(machine, start: xm->start, end: xm->end) < 0) {
1745	dso__put(dso: kernel);
1746	goto out_problem;
1747	}
1748
1749	if (build_id__is_defined(bid))
1750	dso__set_build_id(dso: kernel, bid);
1751
1752	/*
1753	* Avoid using a zero address (kptr_restrict) for the ref reloc
1754	* symbol. Effectively having zero here means that at record
1755	* time /proc/sys/kernel/kptr_restrict was non zero.
1756	*/
1757	if (xm->pgoff != 0) {
1758	map__set_kallsyms_ref_reloc_sym(map: machine->vmlinux_map,
1759	symbol_name,
1760	addr: xm->pgoff);
1761	}
1762
1763	if (machine__is_default_guest(machine)) {
1764	/*
1765	* preload dso of guest kernel and modules
1766	*/
1767	dso__load(dso: kernel, map: machine__kernel_map(machine));
1768	}
1769	dso__put(dso: kernel);
1770	} else if (perf_event__is_extra_kernel_mmap(machine, xm)) {
1771	return machine__process_extra_kernel_map(machine, xm);
1772	}
1773	return 0;
1774	out_problem:
1775	return -1;
1776	}
1777
1778	int machine__process_mmap2_event(struct machine *machine,
1779	union perf_event *event,
1780	struct perf_sample *sample)
1781	{
1782	struct thread *thread;
1783	struct map *map;
1784	struct dso_id dso_id = {
1785	.maj = event->mmap2.maj,
1786	.min = event->mmap2.min,
1787	.ino = event->mmap2.ino,
1788	.ino_generation = event->mmap2.ino_generation,
1789	};
1790	struct build_id __bid, *bid = NULL;
1791	int ret = 0;
1792
1793	if (dump_trace)
1794	perf_event__fprintf_mmap2(event, stdout);
1795
1796	if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) {
1797	bid = &__bid;
1798	build_id__init(bid, data: event->mmap2.build_id, size: event->mmap2.build_id_size);
1799	}
1800
1801	if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
1802	sample->cpumode == PERF_RECORD_MISC_KERNEL) {
1803	struct extra_kernel_map xm = {
1804	.start = event->mmap2.start,
1805	.end = event->mmap2.start + event->mmap2.len,
1806	.pgoff = event->mmap2.pgoff,
1807	};
1808
1809	strlcpy(xm.name, event->mmap2.filename, KMAP_NAME_LEN);
1810	ret = machine__process_kernel_mmap_event(machine, xm: &xm, bid);
1811	if (ret < 0)
1812	goto out_problem;
1813	return 0;
1814	}
1815
1816	thread = machine__findnew_thread(machine, pid: event->mmap2.pid,
1817	tid: event->mmap2.tid);
1818	if (thread == NULL)
1819	goto out_problem;
1820
1821	map = map__new(machine, start: event->mmap2.start,
1822	len: event->mmap2.len, pgoff: event->mmap2.pgoff,
1823	id: &dso_id, prot: event->mmap2.prot,
1824	flags: event->mmap2.flags, bid,
1825	filename: event->mmap2.filename, thread);
1826
1827	if (map == NULL)
1828	goto out_problem_map;
1829
1830	ret = thread__insert_map(thread, map);
1831	if (ret)
1832	goto out_problem_insert;
1833
1834	thread__put(thread);
1835	map__put(map);
1836	return 0;
1837
1838	out_problem_insert:
1839	map__put(map);
1840	out_problem_map:
1841	thread__put(thread);
1842	out_problem:
1843	dump_printf(fmt: "problem processing PERF_RECORD_MMAP2, skipping event.\n");
1844	return 0;
1845	}
1846
1847	int machine__process_mmap_event(struct machine *machine, union perf_event *event,
1848	struct perf_sample *sample)
1849	{
1850	struct thread *thread;
1851	struct map *map;
1852	u32 prot = 0;
1853	int ret = 0;
1854
1855	if (dump_trace)
1856	perf_event__fprintf_mmap(event, stdout);
1857
1858	if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
1859	sample->cpumode == PERF_RECORD_MISC_KERNEL) {
1860	struct extra_kernel_map xm = {
1861	.start = event->mmap.start,
1862	.end = event->mmap.start + event->mmap.len,
1863	.pgoff = event->mmap.pgoff,
1864	};
1865
1866	strlcpy(xm.name, event->mmap.filename, KMAP_NAME_LEN);
1867	ret = machine__process_kernel_mmap_event(machine, xm: &xm, NULL);
1868	if (ret < 0)
1869	goto out_problem;
1870	return 0;
1871	}
1872
1873	thread = machine__findnew_thread(machine, pid: event->mmap.pid,
1874	tid: event->mmap.tid);
1875	if (thread == NULL)
1876	goto out_problem;
1877
1878	if (!(event->header.misc & PERF_RECORD_MISC_MMAP_DATA))
1879	prot = PROT_EXEC;
1880
1881	map = map__new(machine, start: event->mmap.start,
1882	len: event->mmap.len, pgoff: event->mmap.pgoff,
1883	NULL, prot, flags: 0, NULL, filename: event->mmap.filename, thread);
1884
1885	if (map == NULL)
1886	goto out_problem_map;
1887
1888	ret = thread__insert_map(thread, map);
1889	if (ret)
1890	goto out_problem_insert;
1891
1892	thread__put(thread);
1893	map__put(map);
1894	return 0;
1895
1896	out_problem_insert:
1897	map__put(map);
1898	out_problem_map:
1899	thread__put(thread);
1900	out_problem:
1901	dump_printf(fmt: "problem processing PERF_RECORD_MMAP, skipping event.\n");
1902	return 0;
1903	}
1904
1905	void machine__remove_thread(struct machine *machine, struct thread *th)
1906	{
1907	return threads__remove(threads: &machine->threads, thread: th);
1908	}
1909
1910	int machine__process_fork_event(struct machine *machine, union perf_event *event,
1911	struct perf_sample *sample)
1912	{
1913	struct thread *thread = machine__find_thread(machine,
1914	pid: event->fork.pid,
1915	tid: event->fork.tid);
1916	struct thread *parent = machine__findnew_thread(machine,
1917	pid: event->fork.ppid,
1918	tid: event->fork.ptid);
1919	bool do_maps_clone = true;
1920	int err = 0;
1921
1922	if (dump_trace)
1923	perf_event__fprintf_task(event, stdout);
1924
1925	/*
1926	* There may be an existing thread that is not actually the parent,
1927	* either because we are processing events out of order, or because the
1928	* (fork) event that would have removed the thread was lost. Assume the
1929	* latter case and continue on as best we can.
1930	*/
1931	if (thread__pid(thread: parent) != (pid_t)event->fork.ppid) {
1932	dump_printf(fmt: "removing erroneous parent thread %d/%d\n",
1933	thread__pid(thread: parent), thread__tid(thread: parent));
1934	machine__remove_thread(machine, th: parent);
1935	thread__put(thread: parent);
1936	parent = machine__findnew_thread(machine, pid: event->fork.ppid,
1937	tid: event->fork.ptid);
1938	}
1939
1940	/* if a thread currently exists for the thread id remove it */
1941	if (thread != NULL) {
1942	machine__remove_thread(machine, th: thread);
1943	thread__put(thread);
1944	}
1945
1946	thread = machine__findnew_thread(machine, pid: event->fork.pid,
1947	tid: event->fork.tid);
1948	/*
1949	* When synthesizing FORK events, we are trying to create thread
1950	* objects for the already running tasks on the machine.
1951	*
1952	* Normally, for a kernel FORK event, we want to clone the parent's
1953	* maps because that is what the kernel just did.
1954	*
1955	* But when synthesizing, this should not be done. If we do, we end up
1956	* with overlapping maps as we process the synthesized MMAP2 events that
1957	* get delivered shortly thereafter.
1958	*
1959	* Use the FORK event misc flags in an internal way to signal this
1960	* situation, so we can elide the map clone when appropriate.
1961	*/
1962	if (event->fork.header.misc & PERF_RECORD_MISC_FORK_EXEC)
1963	do_maps_clone = false;
1964
1965	if (thread == NULL || parent == NULL ||
1966	thread__fork(thread, parent, timestamp: sample->time, do_maps_clone) < 0) {
1967	dump_printf(fmt: "problem processing PERF_RECORD_FORK, skipping event.\n");
1968	err = -1;
1969	}
1970	thread__put(thread);
1971	thread__put(thread: parent);
1972
1973	return err;
1974	}
1975
1976	int machine__process_exit_event(struct machine *machine, union perf_event *event,
1977	struct perf_sample *sample __maybe_unused)
1978	{
1979	struct thread *thread = machine__find_thread(machine,
1980	pid: event->fork.pid,
1981	tid: event->fork.tid);
1982
1983	if (dump_trace)
1984	perf_event__fprintf_task(event, stdout);
1985
1986	if (thread != NULL) {
1987	if (symbol_conf.keep_exited_threads)
1988	thread__set_exited(thread, /*exited=*/true);
1989	else
1990	machine__remove_thread(machine, th: thread);
1991	}
1992	thread__put(thread);
1993	return 0;
1994	}
1995
1996	int machine__process_event(struct machine *machine, union perf_event *event,
1997	struct perf_sample *sample)
1998	{
1999	int ret;
2000
2001	switch (event->header.type) {
2002	case PERF_RECORD_COMM:
2003	ret = machine__process_comm_event(machine, event, sample); break;
2004	case PERF_RECORD_MMAP:
2005	ret = machine__process_mmap_event(machine, event, sample); break;
2006	case PERF_RECORD_NAMESPACES:
2007	ret = machine__process_namespaces_event(machine, event, sample); break;
2008	case PERF_RECORD_CGROUP:
2009	ret = machine__process_cgroup_event(machine, event, sample); break;
2010	case PERF_RECORD_MMAP2:
2011	ret = machine__process_mmap2_event(machine, event, sample); break;
2012	case PERF_RECORD_FORK:
2013	ret = machine__process_fork_event(machine, event, sample); break;
2014	case PERF_RECORD_EXIT:
2015	ret = machine__process_exit_event(machine, event, sample); break;
2016	case PERF_RECORD_LOST:
2017	ret = machine__process_lost_event(machine, event, sample); break;
2018	case PERF_RECORD_AUX:
2019	ret = machine__process_aux_event(machine, event); break;
2020	case PERF_RECORD_ITRACE_START:
2021	ret = machine__process_itrace_start_event(machine, event); break;
2022	case PERF_RECORD_LOST_SAMPLES:
2023	ret = machine__process_lost_samples_event(machine, event, sample); break;
2024	case PERF_RECORD_SWITCH:
2025	case PERF_RECORD_SWITCH_CPU_WIDE:
2026	ret = machine__process_switch_event(machine, event); break;
2027	case PERF_RECORD_KSYMBOL:
2028	ret = machine__process_ksymbol(machine, event, sample); break;
2029	case PERF_RECORD_BPF_EVENT:
2030	ret = machine__process_bpf(machine, event, sample); break;
2031	case PERF_RECORD_TEXT_POKE:
2032	ret = machine__process_text_poke(machine, event, sample); break;
2033	case PERF_RECORD_AUX_OUTPUT_HW_ID:
2034	ret = machine__process_aux_output_hw_id_event(machine, event); break;
2035	default:
2036	ret = -1;
2037	break;
2038	}
2039
2040	return ret;
2041	}
2042
2043	static bool symbol__match_regex(struct symbol *sym, regex_t *regex)
2044	{
2045	return regexec(regex, sym->name, 0, NULL, 0) == 0;
2046	}
2047
2048	static void ip__resolve_ams(struct thread *thread,
2049	struct addr_map_symbol *ams,
2050	u64 ip)
2051	{
2052	struct addr_location al;
2053
2054	addr_location__init(al: &al);
2055	/*
2056	* We cannot use the header.misc hint to determine whether a
2057	* branch stack address is user, kernel, guest, hypervisor.
2058	* Branches may straddle the kernel/user/hypervisor boundaries.
2059	* Thus, we have to try consecutively until we find a match
2060	* or else, the symbol is unknown
2061	*/
2062	thread__find_cpumode_addr_location(thread, addr: ip, al: &al);
2063
2064	ams->addr = ip;
2065	ams->al_addr = al.addr;
2066	ams->al_level = al.level;
2067	ams->ms.maps = maps__get(maps: al.maps);
2068	ams->ms.sym = al.sym;
2069	ams->ms.map = map__get(map: al.map);
2070	ams->phys_addr = 0;
2071	ams->data_page_size = 0;
2072	addr_location__exit(al: &al);
2073	}
2074
2075	static void ip__resolve_data(struct thread *thread,
2076	u8 m, struct addr_map_symbol *ams,
2077	u64 addr, u64 phys_addr, u64 daddr_page_size)
2078	{
2079	struct addr_location al;
2080
2081	addr_location__init(al: &al);
2082
2083	thread__find_symbol(thread, cpumode: m, addr, al: &al);
2084
2085	ams->addr = addr;
2086	ams->al_addr = al.addr;
2087	ams->al_level = al.level;
2088	ams->ms.maps = maps__get(maps: al.maps);
2089	ams->ms.sym = al.sym;
2090	ams->ms.map = map__get(map: al.map);
2091	ams->phys_addr = phys_addr;
2092	ams->data_page_size = daddr_page_size;
2093	addr_location__exit(al: &al);
2094	}
2095
2096	struct mem_info *sample__resolve_mem(struct perf_sample *sample,
2097	struct addr_location *al)
2098	{
2099	struct mem_info *mi = mem_info__new();
2100
2101	if (!mi)
2102	return NULL;
2103
2104	ip__resolve_ams(thread: al->thread, ams: &mi->iaddr, ip: sample->ip);
2105	ip__resolve_data(thread: al->thread, m: al->cpumode, ams: &mi->daddr,
2106	addr: sample->addr, phys_addr: sample->phys_addr,
2107	daddr_page_size: sample->data_page_size);
2108	mi->data_src.val = sample->data_src;
2109
2110	return mi;
2111	}
2112
2113	static char *callchain_srcline(struct map_symbol *ms, u64 ip)
2114	{
2115	struct map *map = ms->map;
2116	char *srcline = NULL;
2117	struct dso *dso;
2118
2119	if (!map || callchain_param.key == CCKEY_FUNCTION)
2120	return srcline;
2121
2122	dso = map__dso(map);
2123	srcline = srcline__tree_find(tree: &dso->srclines, addr: ip);
2124	if (!srcline) {
2125	bool show_sym = false;
2126	bool show_addr = callchain_param.key == CCKEY_ADDRESS;
2127
2128	srcline = get_srcline(dso, addr: map__rip_2objdump(map, rip: ip),
2129	sym: ms->sym, show_sym, show_addr, ip);
2130	srcline__tree_insert(tree: &dso->srclines, addr: ip, srcline);
2131	}
2132
2133	return srcline;
2134	}
2135
2136	struct iterations {
2137	int nr_loop_iter;
2138	u64 cycles;
2139	};
2140
2141	static int add_callchain_ip(struct thread *thread,
2142	struct callchain_cursor *cursor,
2143	struct symbol **parent,
2144	struct addr_location *root_al,
2145	u8 *cpumode,
2146	u64 ip,
2147	bool branch,
2148	struct branch_flags *flags,
2149	struct iterations *iter,
2150	u64 branch_from)
2151	{
2152	struct map_symbol ms = {};
2153	struct addr_location al;
2154	int nr_loop_iter = 0, err = 0;
2155	u64 iter_cycles = 0;
2156	const char *srcline = NULL;
2157
2158	addr_location__init(al: &al);
2159	al.filtered = 0;
2160	al.sym = NULL;
2161	al.srcline = NULL;
2162	if (!cpumode) {
2163	thread__find_cpumode_addr_location(thread, addr: ip, al: &al);
2164	} else {
2165	if (ip >= PERF_CONTEXT_MAX) {
2166	switch (ip) {
2167	case PERF_CONTEXT_HV:
2168	*cpumode = PERF_RECORD_MISC_HYPERVISOR;
2169	break;
2170	case PERF_CONTEXT_KERNEL:
2171	*cpumode = PERF_RECORD_MISC_KERNEL;
2172	break;
2173	case PERF_CONTEXT_USER:
2174	*cpumode = PERF_RECORD_MISC_USER;
2175	break;
2176	default:
2177	pr_debug("invalid callchain context: "
2178	"%"PRId64"\n", (s64) ip);
2179	/*
2180	* It seems the callchain is corrupted.
2181	* Discard all.
2182	*/
2183	callchain_cursor_reset(cursor);
2184	err = 1;
2185	goto out;
2186	}
2187	goto out;
2188	}
2189	thread__find_symbol(thread, cpumode: *cpumode, addr: ip, al: &al);
2190	}
2191
2192	if (al.sym != NULL) {
2193	if (perf_hpp_list.parent && !*parent &&
2194	symbol__match_regex(al.sym, &parent_regex))
2195	*parent = al.sym;
2196	else if (have_ignore_callees && root_al &&
2197	symbol__match_regex(al.sym, &ignore_callees_regex)) {
2198	/* Treat this symbol as the root,
2199	forgetting its callees. */
2200	addr_location__copy(dst: root_al, src: &al);
2201	callchain_cursor_reset(cursor);
2202	}
2203	}
2204
2205	if (symbol_conf.hide_unresolved && al.sym == NULL)
2206	goto out;
2207
2208	if (iter) {
2209	nr_loop_iter = iter->nr_loop_iter;
2210	iter_cycles = iter->cycles;
2211	}
2212
2213	ms.maps = maps__get(maps: al.maps);
2214	ms.map = map__get(map: al.map);
2215	ms.sym = al.sym;
2216	srcline = callchain_srcline(ms: &ms, ip: al.addr);
2217	err = callchain_cursor_append(cursor, ip, ms: &ms,
2218	branch, flags, nr_loop_iter,
2219	iter_cycles, branch_from, srcline);
2220	out:
2221	addr_location__exit(al: &al);
2222	map_symbol__exit(ms: &ms);
2223	return err;
2224	}
2225
2226	struct branch_info *sample__resolve_bstack(struct perf_sample *sample,
2227	struct addr_location *al)
2228	{
2229	unsigned int i;
2230	const struct branch_stack *bs = sample->branch_stack;
2231	struct branch_entry *entries = perf_sample__branch_entries(sample);
2232	struct branch_info *bi = calloc(bs->nr, sizeof(struct branch_info));
2233
2234	if (!bi)
2235	return NULL;
2236
2237	for (i = 0; i < bs->nr; i++) {
2238	ip__resolve_ams(thread: al->thread, ams: &bi[i].to, ip: entries[i].to);
2239	ip__resolve_ams(thread: al->thread, ams: &bi[i].from, ip: entries[i].from);
2240	bi[i].flags = entries[i].flags;
2241	}
2242	return bi;
2243	}
2244
2245	static void save_iterations(struct iterations *iter,
2246	struct branch_entry *be, int nr)
2247	{
2248	int i;
2249
2250	iter->nr_loop_iter++;
2251	iter->cycles = 0;
2252
2253	for (i = 0; i < nr; i++)
2254	iter->cycles += be[i].flags.cycles;
2255	}
2256
2257	#define CHASHSZ 127
2258	#define CHASHBITS 7
2259	#define NO_ENTRY 0xff
2260
2261	#define PERF_MAX_BRANCH_DEPTH 127
2262
2263	/* Remove loops. */
2264	static int remove_loops(struct branch_entry *l, int nr,
2265	struct iterations *iter)
2266	{
2267	int i, j, off;
2268	unsigned char chash[CHASHSZ];
2269
2270	memset(chash, NO_ENTRY, sizeof(chash));
2271
2272	BUG_ON(PERF_MAX_BRANCH_DEPTH > 255);
2273
2274	for (i = 0; i < nr; i++) {
2275	int h = hash_64(val: l[i].from, CHASHBITS) % CHASHSZ;
2276
2277	/* no collision handling for now */
2278	if (chash[h] == NO_ENTRY) {
2279	chash[h] = i;
2280	} else if (l[chash[h]].from == l[i].from) {
2281	bool is_loop = true;
2282	/* check if it is a real loop */
2283	off = 0;
2284	for (j = chash[h]; j < i && i + off < nr; j++, off++)
2285	if (l[j].from != l[i + off].from) {
2286	is_loop = false;
2287	break;
2288	}
2289	if (is_loop) {
2290	j = nr - (i + off);
2291	if (j > 0) {
2292	save_iterations(iter: iter + i + off,
2293	be: l + i, nr: off);
2294
2295	memmove(iter + i, iter + i + off,
2296	j * sizeof(*iter));
2297
2298	memmove(l + i, l + i + off,
2299	j * sizeof(*l));
2300	}
2301
2302	nr -= off;
2303	}
2304	}
2305	}
2306	return nr;
2307	}
2308
2309	static int lbr_callchain_add_kernel_ip(struct thread *thread,
2310	struct callchain_cursor *cursor,
2311	struct perf_sample *sample,
2312	struct symbol **parent,
2313	struct addr_location *root_al,
2314	u64 branch_from,
2315	bool callee, int end)
2316	{
2317	struct ip_callchain *chain = sample->callchain;
2318	u8 cpumode = PERF_RECORD_MISC_USER;
2319	int err, i;
2320
2321	if (callee) {
2322	for (i = 0; i < end + 1; i++) {
2323	err = add_callchain_ip(thread, cursor, parent,
2324	root_al, cpumode: &cpumode, ip: chain->ips[i],
2325	branch: false, NULL, NULL, branch_from);
2326	if (err)
2327	return err;
2328	}
2329	return 0;
2330	}
2331
2332	for (i = end; i >= 0; i--) {
2333	err = add_callchain_ip(thread, cursor, parent,
2334	root_al, cpumode: &cpumode, ip: chain->ips[i],
2335	branch: false, NULL, NULL, branch_from);
2336	if (err)
2337	return err;
2338	}
2339
2340	return 0;
2341	}
2342
2343	static void save_lbr_cursor_node(struct thread *thread,
2344	struct callchain_cursor *cursor,
2345	int idx)
2346	{
2347	struct lbr_stitch *lbr_stitch = thread__lbr_stitch(thread);
2348
2349	if (!lbr_stitch)
2350	return;
2351
2352	if (cursor->pos == cursor->nr) {
2353	lbr_stitch->prev_lbr_cursor[idx].valid = false;
2354	return;
2355	}
2356
2357	if (!cursor->curr)
2358	cursor->curr = cursor->first;
2359	else
2360	cursor->curr = cursor->curr->next;
2361	memcpy(&lbr_stitch->prev_lbr_cursor[idx], cursor->curr,
2362	sizeof(struct callchain_cursor_node));
2363
2364	lbr_stitch->prev_lbr_cursor[idx].valid = true;
2365	cursor->pos++;
2366	}
2367
2368	static int lbr_callchain_add_lbr_ip(struct thread *thread,
2369	struct callchain_cursor *cursor,
2370	struct perf_sample *sample,
2371	struct symbol **parent,
2372	struct addr_location *root_al,
2373	u64 *branch_from,
2374	bool callee)
2375	{
2376	struct branch_stack *lbr_stack = sample->branch_stack;
2377	struct branch_entry *entries = perf_sample__branch_entries(sample);
2378	u8 cpumode = PERF_RECORD_MISC_USER;
2379	int lbr_nr = lbr_stack->nr;
2380	struct branch_flags *flags;
2381	int err, i;
2382	u64 ip;
2383
2384	/*
2385	* The curr and pos are not used in writing session. They are cleared
2386	* in callchain_cursor_commit() when the writing session is closed.
2387	* Using curr and pos to track the current cursor node.
2388	*/
2389	if (thread__lbr_stitch(thread)) {
2390	cursor->curr = NULL;
2391	cursor->pos = cursor->nr;
2392	if (cursor->nr) {
2393	cursor->curr = cursor->first;
2394	for (i = 0; i < (int)(cursor->nr - 1); i++)
2395	cursor->curr = cursor->curr->next;
2396	}
2397	}
2398
2399	if (callee) {
2400	/* Add LBR ip from first entries.to */
2401	ip = entries[0].to;
2402	flags = &entries[0].flags;
2403	*branch_from = entries[0].from;
2404	err = add_callchain_ip(thread, cursor, parent,
2405	root_al, cpumode: &cpumode, ip,
2406	branch: true, flags, NULL,
2407	branch_from: *branch_from);
2408	if (err)
2409	return err;
2410
2411	/*
2412	* The number of cursor node increases.
2413	* Move the current cursor node.
2414	* But does not need to save current cursor node for entry 0.
2415	* It's impossible to stitch the whole LBRs of previous sample.
2416	*/
2417	if (thread__lbr_stitch(thread) && (cursor->pos != cursor->nr)) {
2418	if (!cursor->curr)
2419	cursor->curr = cursor->first;
2420	else
2421	cursor->curr = cursor->curr->next;
2422	cursor->pos++;
2423	}
2424
2425	/* Add LBR ip from entries.from one by one. */
2426	for (i = 0; i < lbr_nr; i++) {
2427	ip = entries[i].from;
2428	flags = &entries[i].flags;
2429	err = add_callchain_ip(thread, cursor, parent,
2430	root_al, cpumode: &cpumode, ip,
2431	branch: true, flags, NULL,
2432	branch_from: *branch_from);
2433	if (err)
2434	return err;
2435	save_lbr_cursor_node(thread, cursor, idx: i);
2436	}
2437	return 0;
2438	}
2439
2440	/* Add LBR ip from entries.from one by one. */
2441	for (i = lbr_nr - 1; i >= 0; i--) {
2442	ip = entries[i].from;
2443	flags = &entries[i].flags;
2444	err = add_callchain_ip(thread, cursor, parent,
2445	root_al, cpumode: &cpumode, ip,
2446	branch: true, flags, NULL,
2447	branch_from: *branch_from);
2448	if (err)
2449	return err;
2450	save_lbr_cursor_node(thread, cursor, idx: i);
2451	}
2452
2453	if (lbr_nr > 0) {
2454	/* Add LBR ip from first entries.to */
2455	ip = entries[0].to;
2456	flags = &entries[0].flags;
2457	*branch_from = entries[0].from;
2458	err = add_callchain_ip(thread, cursor, parent,
2459	root_al, cpumode: &cpumode, ip,
2460	branch: true, flags, NULL,
2461	branch_from: *branch_from);
2462	if (err)
2463	return err;
2464	}
2465
2466	return 0;
2467	}
2468
2469	static int lbr_callchain_add_stitched_lbr_ip(struct thread *thread,
2470	struct callchain_cursor *cursor)
2471	{
2472	struct lbr_stitch *lbr_stitch = thread__lbr_stitch(thread);
2473	struct callchain_cursor_node *cnode;
2474	struct stitch_list *stitch_node;
2475	int err;
2476
2477	list_for_each_entry(stitch_node, &lbr_stitch->lists, node) {
2478	cnode = &stitch_node->cursor;
2479
2480	err = callchain_cursor_append(cursor, ip: cnode->ip,
2481	ms: &cnode->ms,
2482	branch: cnode->branch,
2483	flags: &cnode->branch_flags,
2484	nr_loop_iter: cnode->nr_loop_iter,
2485	iter_cycles: cnode->iter_cycles,
2486	branch_from: cnode->branch_from,
2487	srcline: cnode->srcline);
2488	if (err)
2489	return err;
2490	}
2491	return 0;
2492	}
2493
2494	static struct stitch_list *get_stitch_node(struct thread *thread)
2495	{
2496	struct lbr_stitch *lbr_stitch = thread__lbr_stitch(thread);
2497	struct stitch_list *stitch_node;
2498
2499	if (!list_empty(head: &lbr_stitch->free_lists)) {
2500	stitch_node = list_first_entry(&lbr_stitch->free_lists,
2501	struct stitch_list, node);
2502	list_del(entry: &stitch_node->node);
2503
2504	return stitch_node;
2505	}
2506
2507	return malloc(sizeof(struct stitch_list));
2508	}
2509
2510	static bool has_stitched_lbr(struct thread *thread,
2511	struct perf_sample *cur,
2512	struct perf_sample *prev,
2513	unsigned int max_lbr,
2514	bool callee)
2515	{
2516	struct branch_stack *cur_stack = cur->branch_stack;
2517	struct branch_entry *cur_entries = perf_sample__branch_entries(sample: cur);
2518	struct branch_stack *prev_stack = prev->branch_stack;
2519	struct branch_entry *prev_entries = perf_sample__branch_entries(sample: prev);
2520	struct lbr_stitch *lbr_stitch = thread__lbr_stitch(thread);
2521	int i, j, nr_identical_branches = 0;
2522	struct stitch_list *stitch_node;
2523	u64 cur_base, distance;
2524
2525	if (!cur_stack || !prev_stack)
2526	return false;
2527
2528	/* Find the physical index of the base-of-stack for current sample. */
2529	cur_base = max_lbr - cur_stack->nr + cur_stack->hw_idx + 1;
2530
2531	distance = (prev_stack->hw_idx > cur_base) ? (prev_stack->hw_idx - cur_base) :
2532	(max_lbr + prev_stack->hw_idx - cur_base);
2533	/* Previous sample has shorter stack. Nothing can be stitched. */
2534	if (distance + 1 > prev_stack->nr)
2535	return false;
2536
2537	/*
2538	* Check if there are identical LBRs between two samples.
2539	* Identical LBRs must have same from, to and flags values. Also,
2540	* they have to be saved in the same LBR registers (same physical
2541	* index).
2542	*
2543	* Starts from the base-of-stack of current sample.
2544	*/
2545	for (i = distance, j = cur_stack->nr - 1; (i >= 0) && (j >= 0); i--, j--) {
2546	if ((prev_entries[i].from != cur_entries[j].from) ||
2547	(prev_entries[i].to != cur_entries[j].to) ||
2548	(prev_entries[i].flags.value != cur_entries[j].flags.value))
2549	break;
2550	nr_identical_branches++;
2551	}
2552
2553	if (!nr_identical_branches)
2554	return false;
2555
2556	/*
2557	* Save the LBRs between the base-of-stack of previous sample
2558	* and the base-of-stack of current sample into lbr_stitch->lists.
2559	* These LBRs will be stitched later.
2560	*/
2561	for (i = prev_stack->nr - 1; i > (int)distance; i--) {
2562
2563	if (!lbr_stitch->prev_lbr_cursor[i].valid)
2564	continue;
2565
2566	stitch_node = get_stitch_node(thread);
2567	if (!stitch_node)
2568	return false;
2569
2570	memcpy(&stitch_node->cursor, &lbr_stitch->prev_lbr_cursor[i],
2571	sizeof(struct callchain_cursor_node));
2572
2573	if (callee)
2574	list_add(new: &stitch_node->node, head: &lbr_stitch->lists);
2575	else
2576	list_add_tail(new: &stitch_node->node, head: &lbr_stitch->lists);
2577	}
2578
2579	return true;
2580	}
2581
2582	static bool alloc_lbr_stitch(struct thread *thread, unsigned int max_lbr)
2583	{
2584	if (thread__lbr_stitch(thread))
2585	return true;
2586
2587	thread__set_lbr_stitch(thread, lbrs: zalloc(sizeof(struct lbr_stitch)));
2588	if (!thread__lbr_stitch(thread))
2589	goto err;
2590
2591	thread__lbr_stitch(thread)->prev_lbr_cursor =
2592	calloc(max_lbr + 1, sizeof(struct callchain_cursor_node));
2593	if (!thread__lbr_stitch(thread)->prev_lbr_cursor)
2594	goto free_lbr_stitch;
2595
2596	INIT_LIST_HEAD(list: &thread__lbr_stitch(thread)->lists);
2597	INIT_LIST_HEAD(list: &thread__lbr_stitch(thread)->free_lists);
2598
2599	return true;
2600
2601	free_lbr_stitch:
2602	free(thread__lbr_stitch(thread));
2603	thread__set_lbr_stitch(thread, NULL);
2604	err:
2605	pr_warning("Failed to allocate space for stitched LBRs. Disable LBR stitch\n");
2606	thread__set_lbr_stitch_enable(thread, en: false);
2607	return false;
2608	}
2609
2610	/*
2611	* Resolve LBR callstack chain sample
2612	* Return:
2613	* 1 on success get LBR callchain information
2614	* 0 no available LBR callchain information, should try fp
2615	* negative error code on other errors.
2616	*/
2617	static int resolve_lbr_callchain_sample(struct thread *thread,
2618	struct callchain_cursor *cursor,
2619	struct perf_sample *sample,
2620	struct symbol **parent,
2621	struct addr_location *root_al,
2622	int max_stack,
2623	unsigned int max_lbr)
2624	{
2625	bool callee = (callchain_param.order == ORDER_CALLEE);
2626	struct ip_callchain *chain = sample->callchain;
2627	int chain_nr = min(max_stack, (int)chain->nr), i;
2628	struct lbr_stitch *lbr_stitch;
2629	bool stitched_lbr = false;
2630	u64 branch_from = 0;
2631	int err;
2632
2633	for (i = 0; i < chain_nr; i++) {
2634	if (chain->ips[i] == PERF_CONTEXT_USER)
2635	break;
2636	}
2637
2638	/* LBR only affects the user callchain */
2639	if (i == chain_nr)
2640	return 0;
2641
2642	if (thread__lbr_stitch_enable(thread) && !sample->no_hw_idx &&
2643	(max_lbr > 0) && alloc_lbr_stitch(thread, max_lbr)) {
2644	lbr_stitch = thread__lbr_stitch(thread);
2645
2646	stitched_lbr = has_stitched_lbr(thread, cur: sample,
2647	prev: &lbr_stitch->prev_sample,
2648	max_lbr, callee);
2649
2650	if (!stitched_lbr && !list_empty(head: &lbr_stitch->lists)) {
2651	list_replace_init(old: &lbr_stitch->lists,
2652	new: &lbr_stitch->free_lists);
2653	}
2654	memcpy(&lbr_stitch->prev_sample, sample, sizeof(*sample));
2655	}
2656
2657	if (callee) {
2658	/* Add kernel ip */
2659	err = lbr_callchain_add_kernel_ip(thread, cursor, sample,
2660	parent, root_al, branch_from,
2661	callee: true, end: i);
2662	if (err)
2663	goto error;
2664
2665	err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent,
2666	root_al, branch_from: &branch_from, callee: true);
2667	if (err)
2668	goto error;
2669
2670	if (stitched_lbr) {
2671	err = lbr_callchain_add_stitched_lbr_ip(thread, cursor);
2672	if (err)
2673	goto error;
2674	}
2675
2676	} else {
2677	if (stitched_lbr) {
2678	err = lbr_callchain_add_stitched_lbr_ip(thread, cursor);
2679	if (err)
2680	goto error;
2681	}
2682	err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent,
2683	root_al, branch_from: &branch_from, callee: false);
2684	if (err)
2685	goto error;
2686
2687	/* Add kernel ip */
2688	err = lbr_callchain_add_kernel_ip(thread, cursor, sample,
2689	parent, root_al, branch_from,
2690	callee: false, end: i);
2691	if (err)
2692	goto error;
2693	}
2694	return 1;
2695
2696	error:
2697	return (err < 0) ? err : 0;
2698	}
2699
2700	static int find_prev_cpumode(struct ip_callchain *chain, struct thread *thread,
2701	struct callchain_cursor *cursor,
2702	struct symbol **parent,
2703	struct addr_location *root_al,
2704	u8 *cpumode, int ent)
2705	{
2706	int err = 0;
2707
2708	while (--ent >= 0) {
2709	u64 ip = chain->ips[ent];
2710
2711	if (ip >= PERF_CONTEXT_MAX) {
2712	err = add_callchain_ip(thread, cursor, parent,
2713	root_al, cpumode, ip,
2714	branch: false, NULL, NULL, branch_from: 0);
2715	break;
2716	}
2717	}
2718	return err;
2719	}
2720
2721	static u64 get_leaf_frame_caller(struct perf_sample *sample,
2722	struct thread *thread, int usr_idx)
2723	{
2724	if (machine__normalized_is(machine: maps__machine(maps: thread__maps(thread)), arch: "arm64"))
2725	return get_leaf_frame_caller_aarch64(sample, thread, user_idx: usr_idx);
2726	else
2727	return 0;
2728	}
2729
2730	static int thread__resolve_callchain_sample(struct thread *thread,
2731	struct callchain_cursor *cursor,
2732	struct evsel *evsel,
2733	struct perf_sample *sample,
2734	struct symbol **parent,
2735	struct addr_location *root_al,
2736	int max_stack)
2737	{
2738	struct branch_stack *branch = sample->branch_stack;
2739	struct branch_entry *entries = perf_sample__branch_entries(sample);
2740	struct ip_callchain *chain = sample->callchain;
2741	int chain_nr = 0;
2742	u8 cpumode = PERF_RECORD_MISC_USER;
2743	int i, j, err, nr_entries, usr_idx;
2744	int skip_idx = -1;
2745	int first_call = 0;
2746	u64 leaf_frame_caller;
2747
2748	if (chain)
2749	chain_nr = chain->nr;
2750
2751	if (evsel__has_branch_callstack(evsel)) {
2752	struct perf_env *env = evsel__env(evsel);
2753
2754	err = resolve_lbr_callchain_sample(thread, cursor, sample, parent,
2755	root_al, max_stack,
2756	max_lbr: !env ? 0 : env->max_branches);
2757	if (err)
2758	return (err < 0) ? err : 0;
2759	}
2760
2761	/*
2762	* Based on DWARF debug information, some architectures skip
2763	* a callchain entry saved by the kernel.
2764	*/
2765	skip_idx = arch_skip_callchain_idx(thread, chain);
2766
2767	/*
2768	* Add branches to call stack for easier browsing. This gives
2769	* more context for a sample than just the callers.
2770	*
2771	* This uses individual histograms of paths compared to the
2772	* aggregated histograms the normal LBR mode uses.
2773	*
2774	* Limitations for now:
2775	* - No extra filters
2776	* - No annotations (should annotate somehow)
2777	*/
2778
2779	if (branch && callchain_param.branch_callstack) {
2780	int nr = min(max_stack, (int)branch->nr);
2781	struct branch_entry be[nr];
2782	struct iterations iter[nr];
2783
2784	if (branch->nr > PERF_MAX_BRANCH_DEPTH) {
2785	pr_warning("corrupted branch chain. skipping...\n");
2786	goto check_calls;
2787	}
2788
2789	for (i = 0; i < nr; i++) {
2790	if (callchain_param.order == ORDER_CALLEE) {
2791	be[i] = entries[i];
2792
2793	if (chain == NULL)
2794	continue;
2795
2796	/*
2797	* Check for overlap into the callchain.
2798	* The return address is one off compared to
2799	* the branch entry. To adjust for this
2800	* assume the calling instruction is not longer
2801	* than 8 bytes.
2802	*/
2803	if (i == skip_idx ||
2804	chain->ips[first_call] >= PERF_CONTEXT_MAX)
2805	first_call++;
2806	else if (be[i].from < chain->ips[first_call] &&
2807	be[i].from >= chain->ips[first_call] - 8)
2808	first_call++;
2809	} else
2810	be[i] = entries[branch->nr - i - 1];
2811	}
2812
2813	memset(iter, 0, sizeof(struct iterations) * nr);
2814	nr = remove_loops(l: be, nr, iter);
2815
2816	for (i = 0; i < nr; i++) {
2817	err = add_callchain_ip(thread, cursor, parent,
2818	root_al,
2819	NULL, ip: be[i].to,
2820	branch: true, flags: &be[i].flags,
2821	NULL, branch_from: be[i].from);
2822
2823	if (!err)
2824	err = add_callchain_ip(thread, cursor, parent, root_al,
2825	NULL, ip: be[i].from,
2826	branch: true, flags: &be[i].flags,
2827	iter: &iter[i], branch_from: 0);
2828	if (err == -EINVAL)
2829	break;
2830	if (err)
2831	return err;
2832	}
2833
2834	if (chain_nr == 0)
2835	return 0;
2836
2837	chain_nr -= nr;
2838	}
2839
2840	check_calls:
2841	if (chain && callchain_param.order != ORDER_CALLEE) {
2842	err = find_prev_cpumode(chain, thread, cursor, parent, root_al,
2843	cpumode: &cpumode, ent: chain->nr - first_call);
2844	if (err)
2845	return (err < 0) ? err : 0;
2846	}
2847	for (i = first_call, nr_entries = 0;
2848	i < chain_nr && nr_entries < max_stack; i++) {
2849	u64 ip;
2850
2851	if (callchain_param.order == ORDER_CALLEE)
2852	j = i;
2853	else
2854	j = chain->nr - i - 1;
2855
2856	#ifdef HAVE_SKIP_CALLCHAIN_IDX
2857	if (j == skip_idx)
2858	continue;
2859	#endif
2860	ip = chain->ips[j];
2861	if (ip < PERF_CONTEXT_MAX)
2862	++nr_entries;
2863	else if (callchain_param.order != ORDER_CALLEE) {
2864	err = find_prev_cpumode(chain, thread, cursor, parent,
2865	root_al, cpumode: &cpumode, ent: j);
2866	if (err)
2867	return (err < 0) ? err : 0;
2868	continue;
2869	}
2870
2871	/*
2872	* PERF_CONTEXT_USER allows us to locate where the user stack ends.
2873	* Depending on callchain_param.order and the position of PERF_CONTEXT_USER,
2874	* the index will be different in order to add the missing frame
2875	* at the right place.
2876	*/
2877
2878	usr_idx = callchain_param.order == ORDER_CALLEE ? j-2 : j-1;
2879
2880	if (usr_idx >= 0 && chain->ips[usr_idx] == PERF_CONTEXT_USER) {
2881
2882	leaf_frame_caller = get_leaf_frame_caller(sample, thread, usr_idx);
2883
2884	/*
2885	* check if leaf_frame_Caller != ip to not add the same
2886	* value twice.
2887	*/
2888
2889	if (leaf_frame_caller && leaf_frame_caller != ip) {
2890
2891	err = add_callchain_ip(thread, cursor, parent,
2892	root_al, cpumode: &cpumode, ip: leaf_frame_caller,
2893	branch: false, NULL, NULL, branch_from: 0);
2894	if (err)
2895	return (err < 0) ? err : 0;
2896	}
2897	}
2898
2899	err = add_callchain_ip(thread, cursor, parent,
2900	root_al, cpumode: &cpumode, ip,
2901	branch: false, NULL, NULL, branch_from: 0);
2902
2903	if (err)
2904	return (err < 0) ? err : 0;
2905	}
2906
2907	return 0;
2908	}
2909
2910	static int append_inlines(struct callchain_cursor *cursor, struct map_symbol *ms, u64 ip)
2911	{
2912	struct symbol *sym = ms->sym;
2913	struct map *map = ms->map;
2914	struct inline_node *inline_node;
2915	struct inline_list *ilist;
2916	struct dso *dso;
2917	u64 addr;
2918	int ret = 1;
2919	struct map_symbol ilist_ms;
2920
2921	if (!symbol_conf.inline_name || !map || !sym)
2922	return ret;
2923
2924	addr = map__dso_map_ip(map, ip);
2925	addr = map__rip_2objdump(map, rip: addr);
2926	dso = map__dso(map);
2927
2928	inline_node = inlines__tree_find(tree: &dso->inlined_nodes, addr);
2929	if (!inline_node) {
2930	inline_node = dso__parse_addr_inlines(dso, addr, sym);
2931	if (!inline_node)
2932	return ret;
2933	inlines__tree_insert(tree: &dso->inlined_nodes, inlines: inline_node);
2934	}
2935
2936	ilist_ms = (struct map_symbol) {
2937	.maps = maps__get(maps: ms->maps),
2938	.map = map__get(map),
2939	};
2940	list_for_each_entry(ilist, &inline_node->val, list) {
2941	ilist_ms.sym = ilist->symbol;
2942	ret = callchain_cursor_append(cursor, ip, ms: &ilist_ms, branch: false,
2943	NULL, nr_loop_iter: 0, iter_cycles: 0, branch_from: 0, srcline: ilist->srcline);
2944
2945	if (ret != 0)
2946	return ret;
2947	}
2948	map_symbol__exit(ms: &ilist_ms);
2949
2950	return ret;
2951	}
2952
2953	static int unwind_entry(struct unwind_entry *entry, void *arg)
2954	{
2955	struct callchain_cursor *cursor = arg;
2956	const char *srcline = NULL;
2957	u64 addr = entry->ip;
2958
2959	if (symbol_conf.hide_unresolved && entry->ms.sym == NULL)
2960	return 0;
2961
2962	if (append_inlines(cursor, ms: &entry->ms, ip: entry->ip) == 0)
2963	return 0;
2964
2965	/*
2966	* Convert entry->ip from a virtual address to an offset in
2967	* its corresponding binary.
2968	*/
2969	if (entry->ms.map)
2970	addr = map__dso_map_ip(map: entry->ms.map, ip: entry->ip);
2971
2972	srcline = callchain_srcline(ms: &entry->ms, ip: addr);
2973	return callchain_cursor_append(cursor, ip: entry->ip, ms: &entry->ms,
2974	branch: false, NULL, nr_loop_iter: 0, iter_cycles: 0, branch_from: 0, srcline);
2975	}
2976
2977	static int thread__resolve_callchain_unwind(struct thread *thread,
2978	struct callchain_cursor *cursor,
2979	struct evsel *evsel,
2980	struct perf_sample *sample,
2981	int max_stack)
2982	{
2983	/* Can we do dwarf post unwind? */
2984	if (!((evsel->core.attr.sample_type & PERF_SAMPLE_REGS_USER) &&
2985	(evsel->core.attr.sample_type & PERF_SAMPLE_STACK_USER)))
2986	return 0;
2987
2988	/* Bail out if nothing was captured. */
2989	if ((!sample->user_regs.regs) ||
2990	(!sample->user_stack.size))
2991	return 0;
2992
2993	return unwind__get_entries(cb: unwind_entry, arg: cursor,
2994	thread, data: sample, max_stack, best_effort: false);
2995	}
2996
2997	int thread__resolve_callchain(struct thread *thread,
2998	struct callchain_cursor *cursor,
2999	struct evsel *evsel,
3000	struct perf_sample *sample,
3001	struct symbol **parent,
3002	struct addr_location *root_al,
3003	int max_stack)
3004	{
3005	int ret = 0;
3006
3007	if (cursor == NULL)
3008	return -ENOMEM;
3009
3010	callchain_cursor_reset(cursor);
3011
3012	if (callchain_param.order == ORDER_CALLEE) {
3013	ret = thread__resolve_callchain_sample(thread, cursor,
3014	evsel, sample,
3015	parent, root_al,
3016	max_stack);
3017	if (ret)
3018	return ret;
3019	ret = thread__resolve_callchain_unwind(thread, cursor,
3020	evsel, sample,
3021	max_stack);
3022	} else {
3023	ret = thread__resolve_callchain_unwind(thread, cursor,
3024	evsel, sample,
3025	max_stack);
3026	if (ret)
3027	return ret;
3028	ret = thread__resolve_callchain_sample(thread, cursor,
3029	evsel, sample,
3030	parent, root_al,
3031	max_stack);
3032	}
3033
3034	return ret;
3035	}
3036
3037	int machine__for_each_thread(struct machine *machine,
3038	int (*fn)(struct thread *thread, void *p),
3039	void *priv)
3040	{
3041	return threads__for_each_thread(threads: &machine->threads, fn, data: priv);
3042	}
3043
3044	int machines__for_each_thread(struct machines *machines,
3045	int (*fn)(struct thread *thread, void *p),
3046	void *priv)
3047	{
3048	struct rb_node *nd;
3049	int rc = 0;
3050
3051	rc = machine__for_each_thread(machine: &machines->host, fn, priv);
3052	if (rc != 0)
3053	return rc;
3054
3055	for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
3056	struct machine *machine = rb_entry(nd, struct machine, rb_node);
3057
3058	rc = machine__for_each_thread(machine, fn, priv);
3059	if (rc != 0)
3060	return rc;
3061	}
3062	return rc;
3063	}
3064
3065
3066	static int thread_list_cb(struct thread *thread, void *data)
3067	{
3068	struct list_head *list = data;
3069	struct thread_list *entry = malloc(sizeof(*entry));
3070
3071	if (!entry)
3072	return -ENOMEM;
3073
3074	entry->thread = thread__get(thread);
3075	list_add_tail(new: &entry->list, head: list);
3076	return 0;
3077	}
3078
3079	int machine__thread_list(struct machine *machine, struct list_head *list)
3080	{
3081	return machine__for_each_thread(machine, fn: thread_list_cb, priv: list);
3082	}
3083
3084	void thread_list__delete(struct list_head *list)
3085	{
3086	struct thread_list *pos, *next;
3087
3088	list_for_each_entry_safe(pos, next, list, list) {
3089	thread__zput(pos->thread);
3090	list_del(entry: &pos->list);
3091	free(pos);
3092	}
3093	}
3094
3095	pid_t machine__get_current_tid(struct machine *machine, int cpu)
3096	{
3097	if (cpu < 0 || (size_t)cpu >= machine->current_tid_sz)
3098	return -1;
3099
3100	return machine->current_tid[cpu];
3101	}
3102
3103	int machine__set_current_tid(struct machine *machine, int cpu, pid_t pid,
3104	pid_t tid)
3105	{
3106	struct thread *thread;
3107	const pid_t init_val = -1;
3108
3109	if (cpu < 0)
3110	return -EINVAL;
3111
3112	if (realloc_array_as_needed(machine->current_tid,
3113	machine->current_tid_sz,
3114	(unsigned int)cpu,
3115	&init_val))
3116	return -ENOMEM;
3117
3118	machine->current_tid[cpu] = tid;
3119
3120	thread = machine__findnew_thread(machine, pid, tid);
3121	if (!thread)
3122	return -ENOMEM;
3123
3124	thread__set_cpu(thread, cpu);
3125	thread__put(thread);
3126
3127	return 0;
3128	}
3129
3130	/*
3131	* Compares the raw arch string. N.B. see instead perf_env__arch() or
3132	* machine__normalized_is() if a normalized arch is needed.
3133	*/
3134	bool machine__is(struct machine *machine, const char *arch)
3135	{
3136	return machine && !strcmp(perf_env__raw_arch(env: machine->env), arch);
3137	}
3138
3139	bool machine__normalized_is(struct machine *machine, const char *arch)
3140	{
3141	return machine && !strcmp(perf_env__arch(env: machine->env), arch);
3142	}
3143
3144	int machine__nr_cpus_avail(struct machine *machine)
3145	{
3146	return machine ? perf_env__nr_cpus_avail(env: machine->env) : 0;
3147	}
3148
3149	int machine__get_kernel_start(struct machine *machine)
3150	{
3151	struct map *map = machine__kernel_map(machine);
3152	int err = 0;
3153
3154	/*
3155	* The only addresses above 2^63 are kernel addresses of a 64-bit
3156	* kernel. Note that addresses are unsigned so that on a 32-bit system
3157	* all addresses including kernel addresses are less than 2^32. In
3158	* that case (32-bit system), if the kernel mapping is unknown, all
3159	* addresses will be assumed to be in user space - see
3160	* machine__kernel_ip().
3161	*/
3162	machine->kernel_start = 1ULL << 63;
3163	if (map) {
3164	err = map__load(map);
3165	/*
3166	* On x86_64, PTI entry trampolines are less than the
3167	* start of kernel text, but still above 2^63. So leave
3168	* kernel_start = 1ULL << 63 for x86_64.
3169	*/
3170	if (!err && !machine__is(machine, arch: "x86_64"))
3171	machine->kernel_start = map__start(map);
3172	}
3173	return err;
3174	}
3175
3176	u8 machine__addr_cpumode(struct machine *machine, u8 cpumode, u64 addr)
3177	{
3178	u8 addr_cpumode = cpumode;
3179	bool kernel_ip;
3180
3181	if (!machine->single_address_space)
3182	goto out;
3183
3184	kernel_ip = machine__kernel_ip(machine, ip: addr);
3185	switch (cpumode) {
3186	case PERF_RECORD_MISC_KERNEL:
3187	case PERF_RECORD_MISC_USER:
3188	addr_cpumode = kernel_ip ? PERF_RECORD_MISC_KERNEL :
3189	PERF_RECORD_MISC_USER;
3190	break;
3191	case PERF_RECORD_MISC_GUEST_KERNEL:
3192	case PERF_RECORD_MISC_GUEST_USER:
3193	addr_cpumode = kernel_ip ? PERF_RECORD_MISC_GUEST_KERNEL :
3194	PERF_RECORD_MISC_GUEST_USER;
3195	break;
3196	default:
3197	break;
3198	}
3199	out:
3200	return addr_cpumode;
3201	}
3202
3203	struct dso *machine__findnew_dso_id(struct machine *machine, const char *filename, struct dso_id *id)
3204	{
3205	return dsos__findnew_id(dsos: &machine->dsos, name: filename, id);
3206	}
3207
3208	struct dso *machine__findnew_dso(struct machine *machine, const char *filename)
3209	{
3210	return machine__findnew_dso_id(machine, filename, NULL);
3211	}
3212
3213	char *machine__resolve_kernel_addr(void *vmachine, unsigned long long *addrp, char **modp)
3214	{
3215	struct machine *machine = vmachine;
3216	struct map *map;
3217	struct symbol *sym = machine__find_kernel_symbol(machine, addr: *addrp, mapp: &map);
3218
3219	if (sym == NULL)
3220	return NULL;
3221
3222	*modp = __map__is_kmodule(map) ? (char *)map__dso(map)->short_name : NULL;
3223	*addrp = map__unmap_ip(map, ip_or_rip: sym->start);
3224	return sym->name;
3225	}
3226
3227	int machine__for_each_dso(struct machine *machine, machine__dso_t fn, void *priv)
3228	{
3229	struct dso *pos;
3230	int err = 0;
3231
3232	list_for_each_entry(pos, &machine->dsos.head, node) {
3233	if (fn(pos, machine, priv))
3234	err = -1;
3235	}
3236	return err;
3237	}
3238
3239	int machine__for_each_kernel_map(struct machine *machine, machine__map_t fn, void *priv)
3240	{
3241	struct maps *maps = machine__kernel_maps(machine);
3242
3243	return maps__for_each_map(maps, cb: fn, data: priv);
3244	}
3245
3246	bool machine__is_lock_function(struct machine *machine, u64 addr)
3247	{
3248	if (!machine->sched.text_start) {
3249	struct map *kmap;
3250	struct symbol *sym = machine__find_kernel_symbol_by_name(machine, name: "__sched_text_start", mapp: &kmap);
3251
3252	if (!sym) {
3253	/* to avoid retry */
3254	machine->sched.text_start = 1;
3255	return false;
3256	}
3257
3258	machine->sched.text_start = map__unmap_ip(map: kmap, ip_or_rip: sym->start);
3259
3260	/* should not fail from here */
3261	sym = machine__find_kernel_symbol_by_name(machine, name: "__sched_text_end", mapp: &kmap);
3262	machine->sched.text_end = map__unmap_ip(map: kmap, ip_or_rip: sym->start);
3263
3264	sym = machine__find_kernel_symbol_by_name(machine, name: "__lock_text_start", mapp: &kmap);
3265	machine->lock.text_start = map__unmap_ip(map: kmap, ip_or_rip: sym->start);
3266
3267	sym = machine__find_kernel_symbol_by_name(machine, name: "__lock_text_end", mapp: &kmap);
3268	machine->lock.text_end = map__unmap_ip(map: kmap, ip_or_rip: sym->start);
3269	}
3270
3271	/* failed to get kernel symbols */
3272	if (machine->sched.text_start == 1)
3273	return false;
3274
3275	/* mutex and rwsem functions are in sched text section */
3276	if (machine->sched.text_start <= addr && addr < machine->sched.text_end)
3277	return true;
3278
3279	/* spinlock functions are in lock text section */
3280	if (machine->lock.text_start <= addr && addr < machine->lock.text_end)
3281	return true;
3282
3283	return false;
3284	}
3285