1	// SPDX-License-Identifier: GPL-2.0
2	#include "builtin.h"
3
4	#include "util/dso.h"
5	#include "util/evlist.h"
6	#include "util/evsel.h"
7	#include "util/config.h"
8	#include "util/map.h"
9	#include "util/symbol.h"
10	#include "util/thread.h"
11	#include "util/header.h"
12	#include "util/session.h"
13	#include "util/tool.h"
14	#include "util/callchain.h"
15	#include "util/time-utils.h"
16	#include <linux/err.h>
17
18	#include <subcmd/pager.h>
19	#include <subcmd/parse-options.h>
20	#include "util/trace-event.h"
21	#include "util/data.h"
22	#include "util/cpumap.h"
23
24	#include "util/debug.h"
25	#include "util/string2.h"
26	#include "util/util.h"
27
28	#include <linux/kernel.h>
29	#include <linux/numa.h>
30	#include <linux/rbtree.h>
31	#include <linux/string.h>
32	#include <linux/zalloc.h>
33	#include <errno.h>
34	#include <inttypes.h>
35	#include <locale.h>
36	#include <regex.h>
37
38	#include <linux/ctype.h>
39	#include <traceevent/event-parse.h>
40
41	static int kmem_slab;
42	static int kmem_page;
43
44	static long kmem_page_size;
45	static enum {
46	KMEM_SLAB,
47	KMEM_PAGE,
48	} kmem_default = KMEM_SLAB; /* for backward compatibility */
49
50	struct alloc_stat;
51	typedef int (*sort_fn_t)(void *, void *);
52
53	static int alloc_flag;
54	static int caller_flag;
55
56	static int alloc_lines = -1;
57	static int caller_lines = -1;
58
59	static bool raw_ip;
60
61	struct alloc_stat {
62	u64 call_site;
63	u64 ptr;
64	u64 bytes_req;
65	u64 bytes_alloc;
66	u64 last_alloc;
67	u32 hit;
68	u32 pingpong;
69
70	short alloc_cpu;
71
72	struct rb_node node;
73	};
74
75	static struct rb_root root_alloc_stat;
76	static struct rb_root root_alloc_sorted;
77	static struct rb_root root_caller_stat;
78	static struct rb_root root_caller_sorted;
79
80	static unsigned long total_requested, total_allocated, total_freed;
81	static unsigned long nr_allocs, nr_cross_allocs;
82
83	/* filters for controlling start and stop of time of analysis */
84	static struct perf_time_interval ptime;
85	const char *time_str;
86
87	static int insert_alloc_stat(unsigned long call_site, unsigned long ptr,
88	int bytes_req, int bytes_alloc, int cpu)
89	{
90	struct rb_node **node = &root_alloc_stat.rb_node;
91	struct rb_node *parent = NULL;
92	struct alloc_stat *data = NULL;
93
94	while (*node) {
95	parent = *node;
96	data = rb_entry(*node, struct alloc_stat, node);
97
98	if (ptr > data->ptr)
99	node = &(*node)->rb_right;
100	else if (ptr < data->ptr)
101	node = &(*node)->rb_left;
102	else
103	break;
104	}
105
106	if (data && data->ptr == ptr) {
107	data->hit++;
108	data->bytes_req += bytes_req;
109	data->bytes_alloc += bytes_alloc;
110	} else {
111	data = malloc(sizeof(*data));
112	if (!data) {
113	pr_err("%s: malloc failed\n", __func__);
114	return -1;
115	}
116	data->ptr = ptr;
117	data->pingpong = 0;
118	data->hit = 1;
119	data->bytes_req = bytes_req;
120	data->bytes_alloc = bytes_alloc;
121
122	rb_link_node(node: &data->node, parent, rb_link: node);
123	rb_insert_color(&data->node, &root_alloc_stat);
124	}
125	data->call_site = call_site;
126	data->alloc_cpu = cpu;
127	data->last_alloc = bytes_alloc;
128
129	return 0;
130	}
131
132	static int insert_caller_stat(unsigned long call_site,
133	int bytes_req, int bytes_alloc)
134	{
135	struct rb_node **node = &root_caller_stat.rb_node;
136	struct rb_node *parent = NULL;
137	struct alloc_stat *data = NULL;
138
139	while (*node) {
140	parent = *node;
141	data = rb_entry(*node, struct alloc_stat, node);
142
143	if (call_site > data->call_site)
144	node = &(*node)->rb_right;
145	else if (call_site < data->call_site)
146	node = &(*node)->rb_left;
147	else
148	break;
149	}
150
151	if (data && data->call_site == call_site) {
152	data->hit++;
153	data->bytes_req += bytes_req;
154	data->bytes_alloc += bytes_alloc;
155	} else {
156	data = malloc(sizeof(*data));
157	if (!data) {
158	pr_err("%s: malloc failed\n", __func__);
159	return -1;
160	}
161	data->call_site = call_site;
162	data->pingpong = 0;
163	data->hit = 1;
164	data->bytes_req = bytes_req;
165	data->bytes_alloc = bytes_alloc;
166
167	rb_link_node(node: &data->node, parent, rb_link: node);
168	rb_insert_color(&data->node, &root_caller_stat);
169	}
170
171	return 0;
172	}
173
174	static int evsel__process_alloc_event(struct evsel *evsel, struct perf_sample *sample)
175	{
176	unsigned long ptr = evsel__intval(evsel, sample, "ptr"),
177	call_site = evsel__intval(evsel, sample, "call_site");
178	int bytes_req = evsel__intval(evsel, sample, "bytes_req"),
179	bytes_alloc = evsel__intval(evsel, sample, "bytes_alloc");
180
181	if (insert_alloc_stat(call_site, ptr, bytes_req, bytes_alloc, cpu: sample->cpu) ||
182	insert_caller_stat(call_site, bytes_req, bytes_alloc))
183	return -1;
184
185	total_requested += bytes_req;
186	total_allocated += bytes_alloc;
187
188	nr_allocs++;
189
190	/*
191	* Commit 11e9734bcb6a ("mm/slab_common: unify NUMA and UMA
192	* version of tracepoints") adds the field "node" into the
193	* tracepoints 'kmalloc' and 'kmem_cache_alloc'.
194	*
195	* The legacy tracepoints 'kmalloc_node' and 'kmem_cache_alloc_node'
196	* also contain the field "node".
197	*
198	* If the tracepoint contains the field "node" the tool stats the
199	* cross allocation.
200	*/
201	if (evsel__field(evsel, name: "node")) {
202	int node1, node2;
203
204	node1 = cpu__get_node((struct perf_cpu){.cpu = sample->cpu});
205	node2 = evsel__intval(evsel, sample, "node");
206
207	/*
208	* If the field "node" is NUMA_NO_NODE (-1), we don't take it
209	* as a cross allocation.
210	*/
211	if ((node2 != NUMA_NO_NODE) && (node1 != node2))
212	nr_cross_allocs++;
213	}
214
215	return 0;
216	}
217
218	static int ptr_cmp(void *, void *);
219	static int slab_callsite_cmp(void *, void *);
220
221	static struct alloc_stat *search_alloc_stat(unsigned long ptr,
222	unsigned long call_site,
223	struct rb_root *root,
224	sort_fn_t sort_fn)
225	{
226	struct rb_node *node = root->rb_node;
227	struct alloc_stat key = { .ptr = ptr, .call_site = call_site };
228
229	while (node) {
230	struct alloc_stat *data;
231	int cmp;
232
233	data = rb_entry(node, struct alloc_stat, node);
234
235	cmp = sort_fn(&key, data);
236	if (cmp < 0)
237	node = node->rb_left;
238	else if (cmp > 0)
239	node = node->rb_right;
240	else
241	return data;
242	}
243	return NULL;
244	}
245
246	static int evsel__process_free_event(struct evsel *evsel, struct perf_sample *sample)
247	{
248	unsigned long ptr = evsel__intval(evsel, sample, "ptr");
249	struct alloc_stat *s_alloc, *s_caller;
250
251	s_alloc = search_alloc_stat(ptr, call_site: 0, root: &root_alloc_stat, sort_fn: ptr_cmp);
252	if (!s_alloc)
253	return 0;
254
255	total_freed += s_alloc->last_alloc;
256
257	if ((short)sample->cpu != s_alloc->alloc_cpu) {
258	s_alloc->pingpong++;
259
260	s_caller = search_alloc_stat(ptr: 0, call_site: s_alloc->call_site,
261	root: &root_caller_stat,
262	sort_fn: slab_callsite_cmp);
263	if (!s_caller)
264	return -1;
265	s_caller->pingpong++;
266	}
267	s_alloc->alloc_cpu = -1;
268
269	return 0;
270	}
271
272	static u64 total_page_alloc_bytes;
273	static u64 total_page_free_bytes;
274	static u64 total_page_nomatch_bytes;
275	static u64 total_page_fail_bytes;
276	static unsigned long nr_page_allocs;
277	static unsigned long nr_page_frees;
278	static unsigned long nr_page_fails;
279	static unsigned long nr_page_nomatch;
280
281	static bool use_pfn;
282	static bool live_page;
283	static struct perf_session *kmem_session;
284
285	#define MAX_MIGRATE_TYPES 6
286	#define MAX_PAGE_ORDER 11
287
288	static int order_stats[MAX_PAGE_ORDER][MAX_MIGRATE_TYPES];
289
290	struct page_stat {
291	struct rb_node node;
292	u64 page;
293	u64 callsite;
294	int order;
295	unsigned gfp_flags;
296	unsigned migrate_type;
297	u64 alloc_bytes;
298	u64 free_bytes;
299	int nr_alloc;
300	int nr_free;
301	};
302
303	static struct rb_root page_live_tree;
304	static struct rb_root page_alloc_tree;
305	static struct rb_root page_alloc_sorted;
306	static struct rb_root page_caller_tree;
307	static struct rb_root page_caller_sorted;
308
309	struct alloc_func {
310	u64 start;
311	u64 end;
312	char *name;
313	};
314
315	static int nr_alloc_funcs;
316	static struct alloc_func *alloc_func_list;
317
318	static int funcmp(const void *a, const void *b)
319	{
320	const struct alloc_func *fa = a;
321	const struct alloc_func *fb = b;
322
323	if (fa->start > fb->start)
324	return 1;
325	else
326	return -1;
327	}
328
329	static int callcmp(const void *a, const void *b)
330	{
331	const struct alloc_func *fa = a;
332	const struct alloc_func *fb = b;
333
334	if (fb->start <= fa->start && fa->end < fb->end)
335	return 0;
336
337	if (fa->start > fb->start)
338	return 1;
339	else
340	return -1;
341	}
342
343	static int build_alloc_func_list(void)
344	{
345	int ret;
346	struct map *kernel_map;
347	struct symbol *sym;
348	struct rb_node *node;
349	struct alloc_func *func;
350	struct machine *machine = &kmem_session->machines.host;
351	regex_t alloc_func_regex;
352	static const char pattern[] = "^_?_?(alloc|get_free|get_zeroed)_pages?";
353
354	ret = regcomp(&alloc_func_regex, pattern, REG_EXTENDED);
355	if (ret) {
356	char err[BUFSIZ];
357
358	regerror(ret, &alloc_func_regex, err, sizeof(err));
359	pr_err("Invalid regex: %s\n%s", pattern, err);
360	return -EINVAL;
361	}
362
363	kernel_map = machine__kernel_map(machine);
364	if (map__load(map: kernel_map) < 0) {
365	pr_err("cannot load kernel map\n");
366	return -ENOENT;
367	}
368
369	map__for_each_symbol(kernel_map, sym, node) {
370	if (regexec(&alloc_func_regex, sym->name, 0, NULL, 0))
371	continue;
372
373	func = realloc(alloc_func_list,
374	(nr_alloc_funcs + 1) * sizeof(*func));
375	if (func == NULL)
376	return -ENOMEM;
377
378	pr_debug("alloc func: %s\n", sym->name);
379	func[nr_alloc_funcs].start = sym->start;
380	func[nr_alloc_funcs].end = sym->end;
381	func[nr_alloc_funcs].name = sym->name;
382
383	alloc_func_list = func;
384	nr_alloc_funcs++;
385	}
386
387	qsort(alloc_func_list, nr_alloc_funcs, sizeof(*func), funcmp);
388
389	regfree(&alloc_func_regex);
390	return 0;
391	}
392
393	/*
394	* Find first non-memory allocation function from callchain.
395	* The allocation functions are in the 'alloc_func_list'.
396	*/
397	static u64 find_callsite(struct evsel *evsel, struct perf_sample *sample)
398	{
399	struct addr_location al;
400	struct machine *machine = &kmem_session->machines.host;
401	struct callchain_cursor_node *node;
402	struct callchain_cursor *cursor;
403	u64 result = sample->ip;
404
405	addr_location__init(al: &al);
406	if (alloc_func_list == NULL) {
407	if (build_alloc_func_list() < 0)
408	goto out;
409	}
410
411	al.thread = machine__findnew_thread(machine, pid: sample->pid, tid: sample->tid);
412
413	cursor = get_tls_callchain_cursor();
414	if (cursor == NULL)
415	goto out;
416
417	sample__resolve_callchain(sample, cursor, NULL, evsel, al: &al, max_stack: 16);
418
419	callchain_cursor_commit(cursor);
420	while (true) {
421	struct alloc_func key, *caller;
422	u64 addr;
423
424	node = callchain_cursor_current(cursor);
425	if (node == NULL)
426	break;
427
428	key.start = key.end = node->ip;
429	caller = bsearch(&key, alloc_func_list, nr_alloc_funcs,
430	sizeof(key), callcmp);
431	if (!caller) {
432	/* found */
433	if (node->ms.map)
434	addr = map__dso_unmap_ip(map: node->ms.map, ip: node->ip);
435	else
436	addr = node->ip;
437
438	result = addr;
439	goto out;
440	} else
441	pr_debug3("skipping alloc function: %s\n", caller->name);
442
443	callchain_cursor_advance(cursor);
444	}
445
446	pr_debug2("unknown callsite: %"PRIx64 "\n", sample->ip);
447	out:
448	addr_location__exit(al: &al);
449	return result;
450	}
451
452	struct sort_dimension {
453	const char name[20];
454	sort_fn_t cmp;
455	struct list_head list;
456	};
457
458	static LIST_HEAD(page_alloc_sort_input);
459	static LIST_HEAD(page_caller_sort_input);
460
461	static struct page_stat *
462	__page_stat__findnew_page(struct page_stat *pstat, bool create)
463	{
464	struct rb_node **node = &page_live_tree.rb_node;
465	struct rb_node *parent = NULL;
466	struct page_stat *data;
467
468	while (*node) {
469	s64 cmp;
470
471	parent = *node;
472	data = rb_entry(*node, struct page_stat, node);
473
474	cmp = data->page - pstat->page;
475	if (cmp < 0)
476	node = &parent->rb_left;
477	else if (cmp > 0)
478	node = &parent->rb_right;
479	else
480	return data;
481	}
482
483	if (!create)
484	return NULL;
485
486	data = zalloc(sizeof(*data));
487	if (data != NULL) {
488	data->page = pstat->page;
489	data->order = pstat->order;
490	data->gfp_flags = pstat->gfp_flags;
491	data->migrate_type = pstat->migrate_type;
492
493	rb_link_node(node: &data->node, parent, rb_link: node);
494	rb_insert_color(&data->node, &page_live_tree);
495	}
496
497	return data;
498	}
499
500	static struct page_stat *page_stat__find_page(struct page_stat *pstat)
501	{
502	return __page_stat__findnew_page(pstat, create: false);
503	}
504
505	static struct page_stat *page_stat__findnew_page(struct page_stat *pstat)
506	{
507	return __page_stat__findnew_page(pstat, create: true);
508	}
509
510	static struct page_stat *
511	__page_stat__findnew_alloc(struct page_stat *pstat, bool create)
512	{
513	struct rb_node **node = &page_alloc_tree.rb_node;
514	struct rb_node *parent = NULL;
515	struct page_stat *data;
516	struct sort_dimension *sort;
517
518	while (*node) {
519	int cmp = 0;
520
521	parent = *node;
522	data = rb_entry(*node, struct page_stat, node);
523
524	list_for_each_entry(sort, &page_alloc_sort_input, list) {
525	cmp = sort->cmp(pstat, data);
526	if (cmp)
527	break;
528	}
529
530	if (cmp < 0)
531	node = &parent->rb_left;
532	else if (cmp > 0)
533	node = &parent->rb_right;
534	else
535	return data;
536	}
537
538	if (!create)
539	return NULL;
540
541	data = zalloc(sizeof(*data));
542	if (data != NULL) {
543	data->page = pstat->page;
544	data->order = pstat->order;
545	data->gfp_flags = pstat->gfp_flags;
546	data->migrate_type = pstat->migrate_type;
547
548	rb_link_node(node: &data->node, parent, rb_link: node);
549	rb_insert_color(&data->node, &page_alloc_tree);
550	}
551
552	return data;
553	}
554
555	static struct page_stat *page_stat__find_alloc(struct page_stat *pstat)
556	{
557	return __page_stat__findnew_alloc(pstat, create: false);
558	}
559
560	static struct page_stat *page_stat__findnew_alloc(struct page_stat *pstat)
561	{
562	return __page_stat__findnew_alloc(pstat, create: true);
563	}
564
565	static struct page_stat *
566	__page_stat__findnew_caller(struct page_stat *pstat, bool create)
567	{
568	struct rb_node **node = &page_caller_tree.rb_node;
569	struct rb_node *parent = NULL;
570	struct page_stat *data;
571	struct sort_dimension *sort;
572
573	while (*node) {
574	int cmp = 0;
575
576	parent = *node;
577	data = rb_entry(*node, struct page_stat, node);
578
579	list_for_each_entry(sort, &page_caller_sort_input, list) {
580	cmp = sort->cmp(pstat, data);
581	if (cmp)
582	break;
583	}
584
585	if (cmp < 0)
586	node = &parent->rb_left;
587	else if (cmp > 0)
588	node = &parent->rb_right;
589	else
590	return data;
591	}
592
593	if (!create)
594	return NULL;
595
596	data = zalloc(sizeof(*data));
597	if (data != NULL) {
598	data->callsite = pstat->callsite;
599	data->order = pstat->order;
600	data->gfp_flags = pstat->gfp_flags;
601	data->migrate_type = pstat->migrate_type;
602
603	rb_link_node(node: &data->node, parent, rb_link: node);
604	rb_insert_color(&data->node, &page_caller_tree);
605	}
606
607	return data;
608	}
609
610	static struct page_stat *page_stat__find_caller(struct page_stat *pstat)
611	{
612	return __page_stat__findnew_caller(pstat, create: false);
613	}
614
615	static struct page_stat *page_stat__findnew_caller(struct page_stat *pstat)
616	{
617	return __page_stat__findnew_caller(pstat, create: true);
618	}
619
620	static bool valid_page(u64 pfn_or_page)
621	{
622	if (use_pfn && pfn_or_page == -1UL)
623	return false;
624	if (!use_pfn && pfn_or_page == 0)
625	return false;
626	return true;
627	}
628
629	struct gfp_flag {
630	unsigned int flags;
631	char *compact_str;
632	char *human_readable;
633	};
634
635	static struct gfp_flag *gfps;
636	static int nr_gfps;
637
638	static int gfpcmp(const void *a, const void *b)
639	{
640	const struct gfp_flag *fa = a;
641	const struct gfp_flag *fb = b;
642
643	return fa->flags - fb->flags;
644	}
645
646	/* see include/trace/events/mmflags.h */
647	static const struct {
648	const char *original;
649	const char *compact;
650	} gfp_compact_table[] = {
651	{ "GFP_TRANSHUGE", "THP" },
652	{ "GFP_TRANSHUGE_LIGHT", "THL" },
653	{ "GFP_HIGHUSER_MOVABLE", "HUM" },
654	{ "GFP_HIGHUSER", "HU" },
655	{ "GFP_USER", "U" },
656	{ "GFP_KERNEL_ACCOUNT", "KAC" },
657	{ "GFP_KERNEL", "K" },
658	{ "GFP_NOFS", "NF" },
659	{ "GFP_ATOMIC", "A" },
660	{ "GFP_NOIO", "NI" },
661	{ "GFP_NOWAIT", "NW" },
662	{ "GFP_DMA", "D" },
663	{ "__GFP_HIGHMEM", "HM" },
664	{ "GFP_DMA32", "D32" },
665	{ "__GFP_HIGH", "H" },
666	{ "__GFP_IO", "I" },
667	{ "__GFP_FS", "F" },
668	{ "__GFP_NOWARN", "NWR" },
669	{ "__GFP_RETRY_MAYFAIL", "R" },
670	{ "__GFP_NOFAIL", "NF" },
671	{ "__GFP_NORETRY", "NR" },
672	{ "__GFP_COMP", "C" },
673	{ "__GFP_ZERO", "Z" },
674	{ "__GFP_NOMEMALLOC", "NMA" },
675	{ "__GFP_MEMALLOC", "MA" },
676	{ "__GFP_HARDWALL", "HW" },
677	{ "__GFP_THISNODE", "TN" },
678	{ "__GFP_RECLAIMABLE", "RC" },
679	{ "__GFP_MOVABLE", "M" },
680	{ "__GFP_ACCOUNT", "AC" },
681	{ "__GFP_WRITE", "WR" },
682	{ "__GFP_RECLAIM", "R" },
683	{ "__GFP_DIRECT_RECLAIM", "DR" },
684	{ "__GFP_KSWAPD_RECLAIM", "KR" },
685	};
686
687	static size_t max_gfp_len;
688
689	static char *compact_gfp_flags(char *gfp_flags)
690	{
691	char *orig_flags = strdup(gfp_flags);
692	char *new_flags = NULL;
693	char *str, *pos = NULL;
694	size_t len = 0;
695
696	if (orig_flags == NULL)
697	return NULL;
698
699	str = strtok_r(orig_flags, "|", &pos);
700	while (str) {
701	size_t i;
702	char *new;
703	const char *cpt;
704
705	for (i = 0; i < ARRAY_SIZE(gfp_compact_table); i++) {
706	if (strcmp(gfp_compact_table[i].original, str))
707	continue;
708
709	cpt = gfp_compact_table[i].compact;
710	new = realloc(new_flags, len + strlen(cpt) + 2);
711	if (new == NULL) {
712	free(new_flags);
713	free(orig_flags);
714	return NULL;
715	}
716
717	new_flags = new;
718
719	if (!len) {
720	strcpy(p: new_flags, q: cpt);
721	} else {
722	strcat(p: new_flags, q: "|");
723	strcat(p: new_flags, q: cpt);
724	len++;
725	}
726
727	len += strlen(cpt);
728	}
729
730	str = strtok_r(NULL, "|", &pos);
731	}
732
733	if (max_gfp_len < len)
734	max_gfp_len = len;
735
736	free(orig_flags);
737	return new_flags;
738	}
739
740	static char *compact_gfp_string(unsigned long gfp_flags)
741	{
742	struct gfp_flag key = {
743	.flags = gfp_flags,
744	};
745	struct gfp_flag *gfp;
746
747	gfp = bsearch(&key, gfps, nr_gfps, sizeof(*gfps), gfpcmp);
748	if (gfp)
749	return gfp->compact_str;
750
751	return NULL;
752	}
753
754	static int parse_gfp_flags(struct evsel *evsel, struct perf_sample *sample,
755	unsigned int gfp_flags)
756	{
757	struct tep_record record = {
758	.cpu = sample->cpu,
759	.data = sample->raw_data,
760	.size = sample->raw_size,
761	};
762	struct trace_seq seq;
763	char *str, *pos = NULL;
764
765	if (nr_gfps) {
766	struct gfp_flag key = {
767	.flags = gfp_flags,
768	};
769
770	if (bsearch(&key, gfps, nr_gfps, sizeof(*gfps), gfpcmp))
771	return 0;
772	}
773
774	trace_seq_init(&seq);
775	tep_print_event(evsel->tp_format->tep,
776	&seq, &record, "%s", TEP_PRINT_INFO);
777
778	str = strtok_r(seq.buffer, " ", &pos);
779	while (str) {
780	if (!strncmp(str, "gfp_flags=", 10)) {
781	struct gfp_flag *new;
782
783	new = realloc(gfps, (nr_gfps + 1) * sizeof(*gfps));
784	if (new == NULL)
785	return -ENOMEM;
786
787	gfps = new;
788	new += nr_gfps++;
789
790	new->flags = gfp_flags;
791	new->human_readable = strdup(str + 10);
792	new->compact_str = compact_gfp_flags(gfp_flags: str + 10);
793	if (!new->human_readable || !new->compact_str)
794	return -ENOMEM;
795
796	qsort(gfps, nr_gfps, sizeof(*gfps), gfpcmp);
797	}
798
799	str = strtok_r(NULL, " ", &pos);
800	}
801
802	trace_seq_destroy(&seq);
803	return 0;
804	}
805
806	static int evsel__process_page_alloc_event(struct evsel *evsel, struct perf_sample *sample)
807	{
808	u64 page;
809	unsigned int order = evsel__intval(evsel, sample, "order");
810	unsigned int gfp_flags = evsel__intval(evsel, sample, "gfp_flags");
811	unsigned int migrate_type = evsel__intval(evsel, sample,
812	"migratetype");
813	u64 bytes = kmem_page_size << order;
814	u64 callsite;
815	struct page_stat *pstat;
816	struct page_stat this = {
817	.order = order,
818	.gfp_flags = gfp_flags,
819	.migrate_type = migrate_type,
820	};
821
822	if (use_pfn)
823	page = evsel__intval(evsel, sample, "pfn");
824	else
825	page = evsel__intval(evsel, sample, "page");
826
827	nr_page_allocs++;
828	total_page_alloc_bytes += bytes;
829
830	if (!valid_page(pfn_or_page: page)) {
831	nr_page_fails++;
832	total_page_fail_bytes += bytes;
833
834	return 0;
835	}
836
837	if (parse_gfp_flags(evsel, sample, gfp_flags) < 0)
838	return -1;
839
840	callsite = find_callsite(evsel, sample);
841
842	/*
843	* This is to find the current page (with correct gfp flags and
844	* migrate type) at free event.
845	*/
846	this.page = page;
847	pstat = page_stat__findnew_page(pstat: &this);
848	if (pstat == NULL)
849	return -ENOMEM;
850
851	pstat->nr_alloc++;
852	pstat->alloc_bytes += bytes;
853	pstat->callsite = callsite;
854
855	if (!live_page) {
856	pstat = page_stat__findnew_alloc(pstat: &this);
857	if (pstat == NULL)
858	return -ENOMEM;
859
860	pstat->nr_alloc++;
861	pstat->alloc_bytes += bytes;
862	pstat->callsite = callsite;
863	}
864
865	this.callsite = callsite;
866	pstat = page_stat__findnew_caller(pstat: &this);
867	if (pstat == NULL)
868	return -ENOMEM;
869
870	pstat->nr_alloc++;
871	pstat->alloc_bytes += bytes;
872
873	order_stats[order][migrate_type]++;
874
875	return 0;
876	}
877
878	static int evsel__process_page_free_event(struct evsel *evsel, struct perf_sample *sample)
879	{
880	u64 page;
881	unsigned int order = evsel__intval(evsel, sample, "order");
882	u64 bytes = kmem_page_size << order;
883	struct page_stat *pstat;
884	struct page_stat this = {
885	.order = order,
886	};
887
888	if (use_pfn)
889	page = evsel__intval(evsel, sample, "pfn");
890	else
891	page = evsel__intval(evsel, sample, "page");
892
893	nr_page_frees++;
894	total_page_free_bytes += bytes;
895
896	this.page = page;
897	pstat = page_stat__find_page(pstat: &this);
898	if (pstat == NULL) {
899	pr_debug2("missing free at page %"PRIx64" (order: %d)\n",
900	page, order);
901
902	nr_page_nomatch++;
903	total_page_nomatch_bytes += bytes;
904
905	return 0;
906	}
907
908	this.gfp_flags = pstat->gfp_flags;
909	this.migrate_type = pstat->migrate_type;
910	this.callsite = pstat->callsite;
911
912	rb_erase(&pstat->node, &page_live_tree);
913	free(pstat);
914
915	if (live_page) {
916	order_stats[this.order][this.migrate_type]--;
917	} else {
918	pstat = page_stat__find_alloc(pstat: &this);
919	if (pstat == NULL)
920	return -ENOMEM;
921
922	pstat->nr_free++;
923	pstat->free_bytes += bytes;
924	}
925
926	pstat = page_stat__find_caller(pstat: &this);
927	if (pstat == NULL)
928	return -ENOENT;
929
930	pstat->nr_free++;
931	pstat->free_bytes += bytes;
932
933	if (live_page) {
934	pstat->nr_alloc--;
935	pstat->alloc_bytes -= bytes;
936
937	if (pstat->nr_alloc == 0) {
938	rb_erase(&pstat->node, &page_caller_tree);
939	free(pstat);
940	}
941	}
942
943	return 0;
944	}
945
946	static bool perf_kmem__skip_sample(struct perf_sample *sample)
947	{
948	/* skip sample based on time? */
949	if (perf_time__skip_sample(ptime: &ptime, timestamp: sample->time))
950	return true;
951
952	return false;
953	}
954
955	typedef int (*tracepoint_handler)(struct evsel *evsel,
956	struct perf_sample *sample);
957
958	static int process_sample_event(struct perf_tool *tool __maybe_unused,
959	union perf_event *event,
960	struct perf_sample *sample,
961	struct evsel *evsel,
962	struct machine *machine)
963	{
964	int err = 0;
965	struct thread *thread = machine__findnew_thread(machine, pid: sample->pid,
966	tid: sample->tid);
967
968	if (thread == NULL) {
969	pr_debug("problem processing %d event, skipping it.\n",
970	event->header.type);
971	return -1;
972	}
973
974	if (perf_kmem__skip_sample(sample))
975	return 0;
976
977	dump_printf(fmt: " ... thread: %s:%d\n", thread__comm_str(thread), thread__tid(thread));
978
979	if (evsel->handler != NULL) {
980	tracepoint_handler f = evsel->handler;
981	err = f(evsel, sample);
982	}
983
984	thread__put(thread);
985
986	return err;
987	}
988
989	static struct perf_tool perf_kmem = {
990	.sample = process_sample_event,
991	.comm = perf_event__process_comm,
992	.mmap = perf_event__process_mmap,
993	.mmap2 = perf_event__process_mmap2,
994	.namespaces = perf_event__process_namespaces,
995	.ordered_events = true,
996	};
997
998	static double fragmentation(unsigned long n_req, unsigned long n_alloc)
999	{
1000	if (n_alloc == 0)
1001	return 0.0;
1002	else
1003	return 100.0 - (100.0 * n_req / n_alloc);
1004	}
1005
1006	static void __print_slab_result(struct rb_root *root,
1007	struct perf_session *session,
1008	int n_lines, int is_caller)
1009	{
1010	struct rb_node *next;
1011	struct machine *machine = &session->machines.host;
1012
1013	printf("%.105s\n", graph_dotted_line);
1014	printf(" %-34s |", is_caller ? "Callsite": "Alloc Ptr");
1015	printf(" Total_alloc/Per | Total_req/Per | Hit | Ping-pong | Frag\n");
1016	printf("%.105s\n", graph_dotted_line);
1017
1018	next = rb_first(root);
1019
1020	while (next && n_lines--) {
1021	struct alloc_stat *data = rb_entry(next, struct alloc_stat,
1022	node);
1023	struct symbol *sym = NULL;
1024	struct map *map;
1025	char buf[BUFSIZ];
1026	u64 addr;
1027
1028	if (is_caller) {
1029	addr = data->call_site;
1030	if (!raw_ip)
1031	sym = machine__find_kernel_symbol(machine, addr, mapp: &map);
1032	} else
1033	addr = data->ptr;
1034
1035	if (sym != NULL)
1036	snprintf(buf, sizeof(buf), "%s+%" PRIx64 "", sym->name,
1037	addr - map__unmap_ip(map, sym->start));
1038	else
1039	snprintf(buf, sizeof(buf), "%#" PRIx64 "", addr);
1040	printf(" %-34s |", buf);
1041
1042	printf(" %9llu/%-5lu | %9llu/%-5lu | %8lu | %9lu | %6.3f%%\n",
1043	(unsigned long long)data->bytes_alloc,
1044	(unsigned long)data->bytes_alloc / data->hit,
1045	(unsigned long long)data->bytes_req,
1046	(unsigned long)data->bytes_req / data->hit,
1047	(unsigned long)data->hit,
1048	(unsigned long)data->pingpong,
1049	fragmentation(n_req: data->bytes_req, n_alloc: data->bytes_alloc));
1050
1051	next = rb_next(next);
1052	}
1053
1054	if (n_lines == -1)
1055	printf(" ... | ... | ... | ... | ... | ... \n");
1056
1057	printf("%.105s\n", graph_dotted_line);
1058	}
1059
1060	static const char * const migrate_type_str[] = {
1061	"UNMOVABL",
1062	"RECLAIM",
1063	"MOVABLE",
1064	"RESERVED",
1065	"CMA/ISLT",
1066	"UNKNOWN",
1067	};
1068
1069	static void __print_page_alloc_result(struct perf_session *session, int n_lines)
1070	{
1071	struct rb_node *next = rb_first(&page_alloc_sorted);
1072	struct machine *machine = &session->machines.host;
1073	const char *format;
1074	int gfp_len = max(strlen("GFP flags"), max_gfp_len);
1075
1076	printf("\n%.105s\n", graph_dotted_line);
1077	printf(" %-16s | %5s alloc (KB) | Hits | Order | Mig.type | %-*s | Callsite\n",
1078	use_pfn ? "PFN" : "Page", live_page ? "Live" : "Total",
1079	gfp_len, "GFP flags");
1080	printf("%.105s\n", graph_dotted_line);
1081
1082	if (use_pfn)
1083	format = " %16llu | %'16llu | %'9d | %5d | %8s | %-*s | %s\n";
1084	else
1085	format = " %016llx | %'16llu | %'9d | %5d | %8s | %-*s | %s\n";
1086
1087	while (next && n_lines--) {
1088	struct page_stat *data;
1089	struct symbol *sym;
1090	struct map *map;
1091	char buf[32];
1092	char *caller = buf;
1093
1094	data = rb_entry(next, struct page_stat, node);
1095	sym = machine__find_kernel_symbol(machine, addr: data->callsite, mapp: &map);
1096	if (sym)
1097	caller = sym->name;
1098	else
1099	scnprintf(buf, sizeof(buf), "%"PRIx64, data->callsite);
1100
1101	printf(format, (unsigned long long)data->page,
1102	(unsigned long long)data->alloc_bytes / 1024,
1103	data->nr_alloc, data->order,
1104	migrate_type_str[data->migrate_type],
1105	gfp_len, compact_gfp_string(gfp_flags: data->gfp_flags), caller);
1106
1107	next = rb_next(next);
1108	}
1109
1110	if (n_lines == -1) {
1111	printf(" ... | ... | ... | ... | ... | %-*s | ...\n",
1112	gfp_len, "...");
1113	}
1114
1115	printf("%.105s\n", graph_dotted_line);
1116	}
1117
1118	static void __print_page_caller_result(struct perf_session *session, int n_lines)
1119	{
1120	struct rb_node *next = rb_first(&page_caller_sorted);
1121	struct machine *machine = &session->machines.host;
1122	int gfp_len = max(strlen("GFP flags"), max_gfp_len);
1123
1124	printf("\n%.105s\n", graph_dotted_line);
1125	printf(" %5s alloc (KB) | Hits | Order | Mig.type | %-*s | Callsite\n",
1126	live_page ? "Live" : "Total", gfp_len, "GFP flags");
1127	printf("%.105s\n", graph_dotted_line);
1128
1129	while (next && n_lines--) {
1130	struct page_stat *data;
1131	struct symbol *sym;
1132	struct map *map;
1133	char buf[32];
1134	char *caller = buf;
1135
1136	data = rb_entry(next, struct page_stat, node);
1137	sym = machine__find_kernel_symbol(machine, addr: data->callsite, mapp: &map);
1138	if (sym)
1139	caller = sym->name;
1140	else
1141	scnprintf(buf, sizeof(buf), "%"PRIx64, data->callsite);
1142
1143	printf(" %'16llu | %'9d | %5d | %8s | %-*s | %s\n",
1144	(unsigned long long)data->alloc_bytes / 1024,
1145	data->nr_alloc, data->order,
1146	migrate_type_str[data->migrate_type],
1147	gfp_len, compact_gfp_string(gfp_flags: data->gfp_flags), caller);
1148
1149	next = rb_next(next);
1150	}
1151
1152	if (n_lines == -1) {
1153	printf(" ... | ... | ... | ... | %-*s | ...\n",
1154	gfp_len, "...");
1155	}
1156
1157	printf("%.105s\n", graph_dotted_line);
1158	}
1159
1160	static void print_gfp_flags(void)
1161	{
1162	int i;
1163
1164	printf("#\n");
1165	printf("# GFP flags\n");
1166	printf("# ---------\n");
1167	for (i = 0; i < nr_gfps; i++) {
1168	printf("# %08x: %*s: %s\n", gfps[i].flags,
1169	(int) max_gfp_len, gfps[i].compact_str,
1170	gfps[i].human_readable);
1171	}
1172	}
1173
1174	static void print_slab_summary(void)
1175	{
1176	printf("\nSUMMARY (SLAB allocator)");
1177	printf("\n========================\n");
1178	printf("Total bytes requested: %'lu\n", total_requested);
1179	printf("Total bytes allocated: %'lu\n", total_allocated);
1180	printf("Total bytes freed: %'lu\n", total_freed);
1181	if (total_allocated > total_freed) {
1182	printf("Net total bytes allocated: %'lu\n",
1183	total_allocated - total_freed);
1184	}
1185	printf("Total bytes wasted on internal fragmentation: %'lu\n",
1186	total_allocated - total_requested);
1187	printf("Internal fragmentation: %f%%\n",
1188	fragmentation(n_req: total_requested, n_alloc: total_allocated));
1189	printf("Cross CPU allocations: %'lu/%'lu\n", nr_cross_allocs, nr_allocs);
1190	}
1191
1192	static void print_page_summary(void)
1193	{
1194	int o, m;
1195	u64 nr_alloc_freed = nr_page_frees - nr_page_nomatch;
1196	u64 total_alloc_freed_bytes = total_page_free_bytes - total_page_nomatch_bytes;
1197
1198	printf("\nSUMMARY (page allocator)");
1199	printf("\n========================\n");
1200	printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total allocation requests",
1201	nr_page_allocs, total_page_alloc_bytes / 1024);
1202	printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total free requests",
1203	nr_page_frees, total_page_free_bytes / 1024);
1204	printf("\n");
1205
1206	printf("%-30s: %'16"PRIu64" [ %'16"PRIu64" KB ]\n", "Total alloc+freed requests",
1207	nr_alloc_freed, (total_alloc_freed_bytes) / 1024);
1208	printf("%-30s: %'16"PRIu64" [ %'16"PRIu64" KB ]\n", "Total alloc-only requests",
1209	nr_page_allocs - nr_alloc_freed,
1210	(total_page_alloc_bytes - total_alloc_freed_bytes) / 1024);
1211	printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total free-only requests",
1212	nr_page_nomatch, total_page_nomatch_bytes / 1024);
1213	printf("\n");
1214
1215	printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total allocation failures",
1216	nr_page_fails, total_page_fail_bytes / 1024);
1217	printf("\n");
1218
1219	printf("%5s %12s %12s %12s %12s %12s\n", "Order", "Unmovable",
1220	"Reclaimable", "Movable", "Reserved", "CMA/Isolated");
1221	printf("%.5s %.12s %.12s %.12s %.12s %.12s\n", graph_dotted_line,
1222	graph_dotted_line, graph_dotted_line, graph_dotted_line,
1223	graph_dotted_line, graph_dotted_line);
1224
1225	for (o = 0; o < MAX_PAGE_ORDER; o++) {
1226	printf("%5d", o);
1227	for (m = 0; m < MAX_MIGRATE_TYPES - 1; m++) {
1228	if (order_stats[o][m])
1229	printf(" %'12d", order_stats[o][m]);
1230	else
1231	printf(" %12c", '.');
1232	}
1233	printf("\n");
1234	}
1235	}
1236
1237	static void print_slab_result(struct perf_session *session)
1238	{
1239	if (caller_flag)
1240	__print_slab_result(root: &root_caller_sorted, session, n_lines: caller_lines, is_caller: 1);
1241	if (alloc_flag)
1242	__print_slab_result(root: &root_alloc_sorted, session, n_lines: alloc_lines, is_caller: 0);
1243	print_slab_summary();
1244	}
1245
1246	static void print_page_result(struct perf_session *session)
1247	{
1248	if (caller_flag || alloc_flag)
1249	print_gfp_flags();
1250	if (caller_flag)
1251	__print_page_caller_result(session, n_lines: caller_lines);
1252	if (alloc_flag)
1253	__print_page_alloc_result(session, n_lines: alloc_lines);
1254	print_page_summary();
1255	}
1256
1257	static void print_result(struct perf_session *session)
1258	{
1259	if (kmem_slab)
1260	print_slab_result(session);
1261	if (kmem_page)
1262	print_page_result(session);
1263	}
1264
1265	static LIST_HEAD(slab_caller_sort);
1266	static LIST_HEAD(slab_alloc_sort);
1267	static LIST_HEAD(page_caller_sort);
1268	static LIST_HEAD(page_alloc_sort);
1269
1270	static void sort_slab_insert(struct rb_root *root, struct alloc_stat *data,
1271	struct list_head *sort_list)
1272	{
1273	struct rb_node **new = &(root->rb_node);
1274	struct rb_node *parent = NULL;
1275	struct sort_dimension *sort;
1276
1277	while (*new) {
1278	struct alloc_stat *this;
1279	int cmp = 0;
1280
1281	this = rb_entry(*new, struct alloc_stat, node);
1282	parent = *new;
1283
1284	list_for_each_entry(sort, sort_list, list) {
1285	cmp = sort->cmp(data, this);
1286	if (cmp)
1287	break;
1288	}
1289
1290	if (cmp > 0)
1291	new = &((*new)->rb_left);
1292	else
1293	new = &((*new)->rb_right);
1294	}
1295
1296	rb_link_node(node: &data->node, parent, rb_link: new);
1297	rb_insert_color(&data->node, root);
1298	}
1299
1300	static void __sort_slab_result(struct rb_root *root, struct rb_root *root_sorted,
1301	struct list_head *sort_list)
1302	{
1303	struct rb_node *node;
1304	struct alloc_stat *data;
1305
1306	for (;;) {
1307	node = rb_first(root);
1308	if (!node)
1309	break;
1310
1311	rb_erase(node, root);
1312	data = rb_entry(node, struct alloc_stat, node);
1313	sort_slab_insert(root: root_sorted, data, sort_list);
1314	}
1315	}
1316
1317	static void sort_page_insert(struct rb_root *root, struct page_stat *data,
1318	struct list_head *sort_list)
1319	{
1320	struct rb_node **new = &root->rb_node;
1321	struct rb_node *parent = NULL;
1322	struct sort_dimension *sort;
1323
1324	while (*new) {
1325	struct page_stat *this;
1326	int cmp = 0;
1327
1328	this = rb_entry(*new, struct page_stat, node);
1329	parent = *new;
1330
1331	list_for_each_entry(sort, sort_list, list) {
1332	cmp = sort->cmp(data, this);
1333	if (cmp)
1334	break;
1335	}
1336
1337	if (cmp > 0)
1338	new = &parent->rb_left;
1339	else
1340	new = &parent->rb_right;
1341	}
1342
1343	rb_link_node(node: &data->node, parent, rb_link: new);
1344	rb_insert_color(&data->node, root);
1345	}
1346
1347	static void __sort_page_result(struct rb_root *root, struct rb_root *root_sorted,
1348	struct list_head *sort_list)
1349	{
1350	struct rb_node *node;
1351	struct page_stat *data;
1352
1353	for (;;) {
1354	node = rb_first(root);
1355	if (!node)
1356	break;
1357
1358	rb_erase(node, root);
1359	data = rb_entry(node, struct page_stat, node);
1360	sort_page_insert(root: root_sorted, data, sort_list);
1361	}
1362	}
1363
1364	static void sort_result(void)
1365	{
1366	if (kmem_slab) {
1367	__sort_slab_result(root: &root_alloc_stat, root_sorted: &root_alloc_sorted,
1368	sort_list: &slab_alloc_sort);
1369	__sort_slab_result(root: &root_caller_stat, root_sorted: &root_caller_sorted,
1370	sort_list: &slab_caller_sort);
1371	}
1372	if (kmem_page) {
1373	if (live_page)
1374	__sort_page_result(root: &page_live_tree, root_sorted: &page_alloc_sorted,
1375	sort_list: &page_alloc_sort);
1376	else
1377	__sort_page_result(root: &page_alloc_tree, root_sorted: &page_alloc_sorted,
1378	sort_list: &page_alloc_sort);
1379
1380	__sort_page_result(root: &page_caller_tree, root_sorted: &page_caller_sorted,
1381	sort_list: &page_caller_sort);
1382	}
1383	}
1384
1385	static int __cmd_kmem(struct perf_session *session)
1386	{
1387	int err = -EINVAL;
1388	struct evsel *evsel;
1389	const struct evsel_str_handler kmem_tracepoints[] = {
1390	/* slab allocator */
1391	{ "kmem:kmalloc", evsel__process_alloc_event, },
1392	{ "kmem:kmem_cache_alloc", evsel__process_alloc_event, },
1393	{ "kmem:kmalloc_node", evsel__process_alloc_event, },
1394	{ "kmem:kmem_cache_alloc_node", evsel__process_alloc_event, },
1395	{ "kmem:kfree", evsel__process_free_event, },
1396	{ "kmem:kmem_cache_free", evsel__process_free_event, },
1397	/* page allocator */
1398	{ "kmem:mm_page_alloc", evsel__process_page_alloc_event, },
1399	{ "kmem:mm_page_free", evsel__process_page_free_event, },
1400	};
1401
1402	if (!perf_session__has_traces(session, msg: "kmem record"))
1403	goto out;
1404
1405	if (perf_session__set_tracepoints_handlers(session, kmem_tracepoints)) {
1406	pr_err("Initializing perf session tracepoint handlers failed\n");
1407	goto out;
1408	}
1409
1410	evlist__for_each_entry(session->evlist, evsel) {
1411	if (!strcmp(evsel__name(evsel), "kmem:mm_page_alloc") &&
1412	evsel__field(evsel, name: "pfn")) {
1413	use_pfn = true;
1414	break;
1415	}
1416	}
1417
1418	setup_pager();
1419	err = perf_session__process_events(session);
1420	if (err != 0) {
1421	pr_err("error during process events: %d\n", err);
1422	goto out;
1423	}
1424	sort_result();
1425	print_result(session);
1426	out:
1427	return err;
1428	}
1429
1430	/* slab sort keys */
1431	static int ptr_cmp(void *a, void *b)
1432	{
1433	struct alloc_stat *l = a;
1434	struct alloc_stat *r = b;
1435
1436	if (l->ptr < r->ptr)
1437	return -1;
1438	else if (l->ptr > r->ptr)
1439	return 1;
1440	return 0;
1441	}
1442
1443	static struct sort_dimension ptr_sort_dimension = {
1444	.name = "ptr",
1445	.cmp = ptr_cmp,
1446	};
1447
1448	static int slab_callsite_cmp(void *a, void *b)
1449	{
1450	struct alloc_stat *l = a;
1451	struct alloc_stat *r = b;
1452
1453	if (l->call_site < r->call_site)
1454	return -1;
1455	else if (l->call_site > r->call_site)
1456	return 1;
1457	return 0;
1458	}
1459
1460	static struct sort_dimension callsite_sort_dimension = {
1461	.name = "callsite",
1462	.cmp = slab_callsite_cmp,
1463	};
1464
1465	static int hit_cmp(void *a, void *b)
1466	{
1467	struct alloc_stat *l = a;
1468	struct alloc_stat *r = b;
1469
1470	if (l->hit < r->hit)
1471	return -1;
1472	else if (l->hit > r->hit)
1473	return 1;
1474	return 0;
1475	}
1476
1477	static struct sort_dimension hit_sort_dimension = {
1478	.name = "hit",
1479	.cmp = hit_cmp,
1480	};
1481
1482	static int bytes_cmp(void *a, void *b)
1483	{
1484	struct alloc_stat *l = a;
1485	struct alloc_stat *r = b;
1486
1487	if (l->bytes_alloc < r->bytes_alloc)
1488	return -1;
1489	else if (l->bytes_alloc > r->bytes_alloc)
1490	return 1;
1491	return 0;
1492	}
1493
1494	static struct sort_dimension bytes_sort_dimension = {
1495	.name = "bytes",
1496	.cmp = bytes_cmp,
1497	};
1498
1499	static int frag_cmp(void *a, void *b)
1500	{
1501	double x, y;
1502	struct alloc_stat *l = a;
1503	struct alloc_stat *r = b;
1504
1505	x = fragmentation(n_req: l->bytes_req, n_alloc: l->bytes_alloc);
1506	y = fragmentation(n_req: r->bytes_req, n_alloc: r->bytes_alloc);
1507
1508	if (x < y)
1509	return -1;
1510	else if (x > y)
1511	return 1;
1512	return 0;
1513	}
1514
1515	static struct sort_dimension frag_sort_dimension = {
1516	.name = "frag",
1517	.cmp = frag_cmp,
1518	};
1519
1520	static int pingpong_cmp(void *a, void *b)
1521	{
1522	struct alloc_stat *l = a;
1523	struct alloc_stat *r = b;
1524
1525	if (l->pingpong < r->pingpong)
1526	return -1;
1527	else if (l->pingpong > r->pingpong)
1528	return 1;
1529	return 0;
1530	}
1531
1532	static struct sort_dimension pingpong_sort_dimension = {
1533	.name = "pingpong",
1534	.cmp = pingpong_cmp,
1535	};
1536
1537	/* page sort keys */
1538	static int page_cmp(void *a, void *b)
1539	{
1540	struct page_stat *l = a;
1541	struct page_stat *r = b;
1542
1543	if (l->page < r->page)
1544	return -1;
1545	else if (l->page > r->page)
1546	return 1;
1547	return 0;
1548	}
1549
1550	static struct sort_dimension page_sort_dimension = {
1551	.name = "page",
1552	.cmp = page_cmp,
1553	};
1554
1555	static int page_callsite_cmp(void *a, void *b)
1556	{
1557	struct page_stat *l = a;
1558	struct page_stat *r = b;
1559
1560	if (l->callsite < r->callsite)
1561	return -1;
1562	else if (l->callsite > r->callsite)
1563	return 1;
1564	return 0;
1565	}
1566
1567	static struct sort_dimension page_callsite_sort_dimension = {
1568	.name = "callsite",
1569	.cmp = page_callsite_cmp,
1570	};
1571
1572	static int page_hit_cmp(void *a, void *b)
1573	{
1574	struct page_stat *l = a;
1575	struct page_stat *r = b;
1576
1577	if (l->nr_alloc < r->nr_alloc)
1578	return -1;
1579	else if (l->nr_alloc > r->nr_alloc)
1580	return 1;
1581	return 0;
1582	}
1583
1584	static struct sort_dimension page_hit_sort_dimension = {
1585	.name = "hit",
1586	.cmp = page_hit_cmp,
1587	};
1588
1589	static int page_bytes_cmp(void *a, void *b)
1590	{
1591	struct page_stat *l = a;
1592	struct page_stat *r = b;
1593
1594	if (l->alloc_bytes < r->alloc_bytes)
1595	return -1;
1596	else if (l->alloc_bytes > r->alloc_bytes)
1597	return 1;
1598	return 0;
1599	}
1600
1601	static struct sort_dimension page_bytes_sort_dimension = {
1602	.name = "bytes",
1603	.cmp = page_bytes_cmp,
1604	};
1605
1606	static int page_order_cmp(void *a, void *b)
1607	{
1608	struct page_stat *l = a;
1609	struct page_stat *r = b;
1610
1611	if (l->order < r->order)
1612	return -1;
1613	else if (l->order > r->order)
1614	return 1;
1615	return 0;
1616	}
1617
1618	static struct sort_dimension page_order_sort_dimension = {
1619	.name = "order",
1620	.cmp = page_order_cmp,
1621	};
1622
1623	static int migrate_type_cmp(void *a, void *b)
1624	{
1625	struct page_stat *l = a;
1626	struct page_stat *r = b;
1627
1628	/* for internal use to find free'd page */
1629	if (l->migrate_type == -1U)
1630	return 0;
1631
1632	if (l->migrate_type < r->migrate_type)
1633	return -1;
1634	else if (l->migrate_type > r->migrate_type)
1635	return 1;
1636	return 0;
1637	}
1638
1639	static struct sort_dimension migrate_type_sort_dimension = {
1640	.name = "migtype",
1641	.cmp = migrate_type_cmp,
1642	};
1643
1644	static int gfp_flags_cmp(void *a, void *b)
1645	{
1646	struct page_stat *l = a;
1647	struct page_stat *r = b;
1648
1649	/* for internal use to find free'd page */
1650	if (l->gfp_flags == -1U)
1651	return 0;
1652
1653	if (l->gfp_flags < r->gfp_flags)
1654	return -1;
1655	else if (l->gfp_flags > r->gfp_flags)
1656	return 1;
1657	return 0;
1658	}
1659
1660	static struct sort_dimension gfp_flags_sort_dimension = {
1661	.name = "gfp",
1662	.cmp = gfp_flags_cmp,
1663	};
1664
1665	static struct sort_dimension *slab_sorts[] = {
1666	&ptr_sort_dimension,
1667	&callsite_sort_dimension,
1668	&hit_sort_dimension,
1669	&bytes_sort_dimension,
1670	&frag_sort_dimension,
1671	&pingpong_sort_dimension,
1672	};
1673
1674	static struct sort_dimension *page_sorts[] = {
1675	&page_sort_dimension,
1676	&page_callsite_sort_dimension,
1677	&page_hit_sort_dimension,
1678	&page_bytes_sort_dimension,
1679	&page_order_sort_dimension,
1680	&migrate_type_sort_dimension,
1681	&gfp_flags_sort_dimension,
1682	};
1683
1684	static int slab_sort_dimension__add(const char *tok, struct list_head *list)
1685	{
1686	struct sort_dimension *sort;
1687	int i;
1688
1689	for (i = 0; i < (int)ARRAY_SIZE(slab_sorts); i++) {
1690	if (!strcmp(slab_sorts[i]->name, tok)) {
1691	sort = memdup(slab_sorts[i], sizeof(*slab_sorts[i]));
1692	if (!sort) {
1693	pr_err("%s: memdup failed\n", __func__);
1694	return -1;
1695	}
1696	list_add_tail(new: &sort->list, head: list);
1697	return 0;
1698	}
1699	}
1700
1701	return -1;
1702	}
1703
1704	static int page_sort_dimension__add(const char *tok, struct list_head *list)
1705	{
1706	struct sort_dimension *sort;
1707	int i;
1708
1709	for (i = 0; i < (int)ARRAY_SIZE(page_sorts); i++) {
1710	if (!strcmp(page_sorts[i]->name, tok)) {
1711	sort = memdup(page_sorts[i], sizeof(*page_sorts[i]));
1712	if (!sort) {
1713	pr_err("%s: memdup failed\n", __func__);
1714	return -1;
1715	}
1716	list_add_tail(new: &sort->list, head: list);
1717	return 0;
1718	}
1719	}
1720
1721	return -1;
1722	}
1723
1724	static int setup_slab_sorting(struct list_head *sort_list, const char *arg)
1725	{
1726	char *tok;
1727	char *str = strdup(arg);
1728	char *pos = str;
1729
1730	if (!str) {
1731	pr_err("%s: strdup failed\n", __func__);
1732	return -1;
1733	}
1734
1735	while (true) {
1736	tok = strsep(&pos, ",");
1737	if (!tok)
1738	break;
1739	if (slab_sort_dimension__add(tok, list: sort_list) < 0) {
1740	pr_err("Unknown slab --sort key: '%s'", tok);
1741	free(str);
1742	return -1;
1743	}
1744	}
1745
1746	free(str);
1747	return 0;
1748	}
1749
1750	static int setup_page_sorting(struct list_head *sort_list, const char *arg)
1751	{
1752	char *tok;
1753	char *str = strdup(arg);
1754	char *pos = str;
1755
1756	if (!str) {
1757	pr_err("%s: strdup failed\n", __func__);
1758	return -1;
1759	}
1760
1761	while (true) {
1762	tok = strsep(&pos, ",");
1763	if (!tok)
1764	break;
1765	if (page_sort_dimension__add(tok, list: sort_list) < 0) {
1766	pr_err("Unknown page --sort key: '%s'", tok);
1767	free(str);
1768	return -1;
1769	}
1770	}
1771
1772	free(str);
1773	return 0;
1774	}
1775
1776	static int parse_sort_opt(const struct option *opt __maybe_unused,
1777	const char *arg, int unset __maybe_unused)
1778	{
1779	if (!arg)
1780	return -1;
1781
1782	if (kmem_page > kmem_slab ||
1783	(kmem_page == 0 && kmem_slab == 0 && kmem_default == KMEM_PAGE)) {
1784	if (caller_flag > alloc_flag)
1785	return setup_page_sorting(sort_list: &page_caller_sort, arg);
1786	else
1787	return setup_page_sorting(sort_list: &page_alloc_sort, arg);
1788	} else {
1789	if (caller_flag > alloc_flag)
1790	return setup_slab_sorting(sort_list: &slab_caller_sort, arg);
1791	else
1792	return setup_slab_sorting(sort_list: &slab_alloc_sort, arg);
1793	}
1794
1795	return 0;
1796	}
1797
1798	static int parse_caller_opt(const struct option *opt __maybe_unused,
1799	const char *arg __maybe_unused,
1800	int unset __maybe_unused)
1801	{
1802	caller_flag = (alloc_flag + 1);
1803	return 0;
1804	}
1805
1806	static int parse_alloc_opt(const struct option *opt __maybe_unused,
1807	const char *arg __maybe_unused,
1808	int unset __maybe_unused)
1809	{
1810	alloc_flag = (caller_flag + 1);
1811	return 0;
1812	}
1813
1814	static int parse_slab_opt(const struct option *opt __maybe_unused,
1815	const char *arg __maybe_unused,
1816	int unset __maybe_unused)
1817	{
1818	kmem_slab = (kmem_page + 1);
1819	return 0;
1820	}
1821
1822	static int parse_page_opt(const struct option *opt __maybe_unused,
1823	const char *arg __maybe_unused,
1824	int unset __maybe_unused)
1825	{
1826	kmem_page = (kmem_slab + 1);
1827	return 0;
1828	}
1829
1830	static int parse_line_opt(const struct option *opt __maybe_unused,
1831	const char *arg, int unset __maybe_unused)
1832	{
1833	int lines;
1834
1835	if (!arg)
1836	return -1;
1837
1838	lines = strtoul(arg, NULL, 10);
1839
1840	if (caller_flag > alloc_flag)
1841	caller_lines = lines;
1842	else
1843	alloc_lines = lines;
1844
1845	return 0;
1846	}
1847
1848	static bool slab_legacy_tp_is_exposed(void)
1849	{
1850	/*
1851	* The tracepoints "kmem:kmalloc_node" and
1852	* "kmem:kmem_cache_alloc_node" have been removed on the latest
1853	* kernel, if the tracepoint "kmem:kmalloc_node" is existed it
1854	* means the tool is running on an old kernel, we need to
1855	* rollback to support these legacy tracepoints.
1856	*/
1857	return IS_ERR(ptr: trace_event__tp_format(sys: "kmem", name: "kmalloc_node")) ?
1858	false : true;
1859	}
1860
1861	static int __cmd_record(int argc, const char **argv)
1862	{
1863	const char * const record_args[] = {
1864	"record", "-a", "-R", "-c", "1",
1865	};
1866	const char * const slab_events[] = {
1867	"-e", "kmem:kmalloc",
1868	"-e", "kmem:kfree",
1869	"-e", "kmem:kmem_cache_alloc",
1870	"-e", "kmem:kmem_cache_free",
1871	};
1872	const char * const slab_legacy_events[] = {
1873	"-e", "kmem:kmalloc_node",
1874	"-e", "kmem:kmem_cache_alloc_node",
1875	};
1876	const char * const page_events[] = {
1877	"-e", "kmem:mm_page_alloc",
1878	"-e", "kmem:mm_page_free",
1879	};
1880	unsigned int rec_argc, i, j;
1881	const char **rec_argv;
1882	unsigned int slab_legacy_tp_exposed = slab_legacy_tp_is_exposed();
1883
1884	rec_argc = ARRAY_SIZE(record_args) + argc - 1;
1885	if (kmem_slab) {
1886	rec_argc += ARRAY_SIZE(slab_events);
1887	if (slab_legacy_tp_exposed)
1888	rec_argc += ARRAY_SIZE(slab_legacy_events);
1889	}
1890	if (kmem_page)
1891	rec_argc += ARRAY_SIZE(page_events) + 1; /* for -g */
1892
1893	rec_argv = calloc(rec_argc + 1, sizeof(char *));
1894
1895	if (rec_argv == NULL)
1896	return -ENOMEM;
1897
1898	for (i = 0; i < ARRAY_SIZE(record_args); i++)
1899	rec_argv[i] = strdup(record_args[i]);
1900
1901	if (kmem_slab) {
1902	for (j = 0; j < ARRAY_SIZE(slab_events); j++, i++)
1903	rec_argv[i] = strdup(slab_events[j]);
1904	if (slab_legacy_tp_exposed) {
1905	for (j = 0; j < ARRAY_SIZE(slab_legacy_events); j++, i++)
1906	rec_argv[i] = strdup(slab_legacy_events[j]);
1907	}
1908	}
1909	if (kmem_page) {
1910	rec_argv[i++] = strdup("-g");
1911
1912	for (j = 0; j < ARRAY_SIZE(page_events); j++, i++)
1913	rec_argv[i] = strdup(page_events[j]);
1914	}
1915
1916	for (j = 1; j < (unsigned int)argc; j++, i++)
1917	rec_argv[i] = argv[j];
1918
1919	return cmd_record(argc: i, argv: rec_argv);
1920	}
1921
1922	static int kmem_config(const char *var, const char *value, void *cb __maybe_unused)
1923	{
1924	if (!strcmp(var, "kmem.default")) {
1925	if (!strcmp(value, "slab"))
1926	kmem_default = KMEM_SLAB;
1927	else if (!strcmp(value, "page"))
1928	kmem_default = KMEM_PAGE;
1929	else
1930	pr_err("invalid default value ('slab' or 'page' required): %s\n",
1931	value);
1932	return 0;
1933	}
1934
1935	return 0;
1936	}
1937
1938	int cmd_kmem(int argc, const char **argv)
1939	{
1940	const char * const default_slab_sort = "frag,hit,bytes";
1941	const char * const default_page_sort = "bytes,hit";
1942	struct perf_data data = {
1943	.mode = PERF_DATA_MODE_READ,
1944	};
1945	const struct option kmem_options[] = {
1946	OPT_STRING('i', "input", &input_name, "file", "input file name"),
1947	OPT_INCR('v', "verbose", &verbose,
1948	"be more verbose (show symbol address, etc)"),
1949	OPT_CALLBACK_NOOPT(0, "caller", NULL, NULL,
1950	"show per-callsite statistics", parse_caller_opt),
1951	OPT_CALLBACK_NOOPT(0, "alloc", NULL, NULL,
1952	"show per-allocation statistics", parse_alloc_opt),
1953	OPT_CALLBACK('s', "sort", NULL, "key[,key2...]",
1954	"sort by keys: ptr, callsite, bytes, hit, pingpong, frag, "
1955	"page, order, migtype, gfp", parse_sort_opt),
1956	OPT_CALLBACK('l', "line", NULL, "num", "show n lines", parse_line_opt),
1957	OPT_BOOLEAN(0, "raw-ip", &raw_ip, "show raw ip instead of symbol"),
1958	OPT_BOOLEAN('f', "force", &data.force, "don't complain, do it"),
1959	OPT_CALLBACK_NOOPT(0, "slab", NULL, NULL, "Analyze slab allocator",
1960	parse_slab_opt),
1961	OPT_CALLBACK_NOOPT(0, "page", NULL, NULL, "Analyze page allocator",
1962	parse_page_opt),
1963	OPT_BOOLEAN(0, "live", &live_page, "Show live page stat"),
1964	OPT_STRING(0, "time", &time_str, "str",
1965	"Time span of interest (start,stop)"),
1966	OPT_END()
1967	};
1968	const char *const kmem_subcommands[] = { "record", "stat", NULL };
1969	const char *kmem_usage[] = {
1970	NULL,
1971	NULL
1972	};
1973	struct perf_session *session;
1974	static const char errmsg[] = "No %s allocation events found. Have you run 'perf kmem record --%s'?\n";
1975	int ret = perf_config(fn: kmem_config, NULL);
1976
1977	if (ret)
1978	return ret;
1979
1980	argc = parse_options_subcommand(argc, argv, kmem_options,
1981	kmem_subcommands, kmem_usage,
1982	PARSE_OPT_STOP_AT_NON_OPTION);
1983
1984	if (!argc)
1985	usage_with_options(kmem_usage, kmem_options);
1986
1987	if (kmem_slab == 0 && kmem_page == 0) {
1988	if (kmem_default == KMEM_SLAB)
1989	kmem_slab = 1;
1990	else
1991	kmem_page = 1;
1992	}
1993
1994	if (strlen(argv[0]) > 2 && strstarts(str: "record", prefix: argv[0])) {
1995	symbol__init(NULL);
1996	return __cmd_record(argc, argv);
1997	}
1998
1999	data.path = input_name;
2000
2001	kmem_session = session = perf_session__new(data: &data, tool: &perf_kmem);
2002	if (IS_ERR(ptr: session))
2003	return PTR_ERR(ptr: session);
2004
2005	ret = -1;
2006
2007	if (kmem_slab) {
2008	if (!evlist__find_tracepoint_by_name(evlist: session->evlist, name: "kmem:kmalloc")) {
2009	pr_err(errmsg, "slab", "slab");
2010	goto out_delete;
2011	}
2012	}
2013
2014	if (kmem_page) {
2015	struct evsel *evsel = evlist__find_tracepoint_by_name(evlist: session->evlist, name: "kmem:mm_page_alloc");
2016
2017	if (evsel == NULL) {
2018	pr_err(errmsg, "page", "page");
2019	goto out_delete;
2020	}
2021
2022	kmem_page_size = tep_get_page_size(evsel->tp_format->tep);
2023	symbol_conf.use_callchain = true;
2024	}
2025
2026	symbol__init(env: &session->header.env);
2027
2028	if (perf_time__parse_str(ptime: &ptime, ostr: time_str) != 0) {
2029	pr_err("Invalid time string\n");
2030	ret = -EINVAL;
2031	goto out_delete;
2032	}
2033
2034	if (!strcmp(argv[0], "stat")) {
2035	setlocale(LC_ALL, "");
2036
2037	if (cpu__setup_cpunode_map())
2038	goto out_delete;
2039
2040	if (list_empty(head: &slab_caller_sort))
2041	setup_slab_sorting(sort_list: &slab_caller_sort, arg: default_slab_sort);
2042	if (list_empty(head: &slab_alloc_sort))
2043	setup_slab_sorting(sort_list: &slab_alloc_sort, arg: default_slab_sort);
2044	if (list_empty(head: &page_caller_sort))
2045	setup_page_sorting(sort_list: &page_caller_sort, arg: default_page_sort);
2046	if (list_empty(head: &page_alloc_sort))
2047	setup_page_sorting(sort_list: &page_alloc_sort, arg: default_page_sort);
2048
2049	if (kmem_page) {
2050	setup_page_sorting(sort_list: &page_alloc_sort_input,
2051	arg: "page,order,migtype,gfp");
2052	setup_page_sorting(sort_list: &page_caller_sort_input,
2053	arg: "callsite,order,migtype,gfp");
2054	}
2055	ret = __cmd_kmem(session);
2056	} else
2057	usage_with_options(kmem_usage, kmem_options);
2058
2059	out_delete:
2060	perf_session__delete(session);
2061
2062	return ret;
2063	}
2064
2065