1	// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2	/* Copyright (C) 2017-2018 Netronome Systems, Inc. */
3
4	#ifndef _GNU_SOURCE
5	#define _GNU_SOURCE
6	#endif
7	#include <errno.h>
8	#include <fcntl.h>
9	#include <signal.h>
10	#include <stdarg.h>
11	#include <stdio.h>
12	#include <stdlib.h>
13	#include <string.h>
14	#include <time.h>
15	#include <unistd.h>
16	#include <net/if.h>
17	#include <sys/ioctl.h>
18	#include <sys/types.h>
19	#include <sys/stat.h>
20	#include <sys/syscall.h>
21	#include <dirent.h>
22
23	#include <linux/err.h>
24	#include <linux/perf_event.h>
25	#include <linux/sizes.h>
26
27	#include <bpf/bpf.h>
28	#include <bpf/btf.h>
29	#include <bpf/hashmap.h>
30	#include <bpf/libbpf.h>
31	#include <bpf/libbpf_internal.h>
32	#include <bpf/skel_internal.h>
33
34	#include "cfg.h"
35	#include "main.h"
36	#include "xlated_dumper.h"
37
38	#define BPF_METADATA_PREFIX "bpf_metadata_"
39	#define BPF_METADATA_PREFIX_LEN (sizeof(BPF_METADATA_PREFIX) - 1)
40
41	enum dump_mode {
42	DUMP_JITED,
43	DUMP_XLATED,
44	};
45
46	static const bool attach_types[] = {
47	[BPF_SK_SKB_STREAM_PARSER] = true,
48	[BPF_SK_SKB_STREAM_VERDICT] = true,
49	[BPF_SK_SKB_VERDICT] = true,
50	[BPF_SK_MSG_VERDICT] = true,
51	[BPF_FLOW_DISSECTOR] = true,
52	[__MAX_BPF_ATTACH_TYPE] = false,
53	};
54
55	/* Textual representations traditionally used by the program and kept around
56	* for the sake of backwards compatibility.
57	*/
58	static const char * const attach_type_strings[] = {
59	[BPF_SK_SKB_STREAM_PARSER] = "stream_parser",
60	[BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",
61	[BPF_SK_SKB_VERDICT] = "skb_verdict",
62	[BPF_SK_MSG_VERDICT] = "msg_verdict",
63	[__MAX_BPF_ATTACH_TYPE] = NULL,
64	};
65
66	static struct hashmap *prog_table;
67
68	static enum bpf_attach_type parse_attach_type(const char *str)
69	{
70	enum bpf_attach_type type;
71
72	for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) {
73	if (attach_types[type]) {
74	const char *attach_type_str;
75
76	attach_type_str = libbpf_bpf_attach_type_str(type);
77	if (!strcmp(str, attach_type_str))
78	return type;
79	}
80
81	if (attach_type_strings[type] &&
82	is_prefix(pfx: str, str: attach_type_strings[type]))
83	return type;
84	}
85
86	return __MAX_BPF_ATTACH_TYPE;
87	}
88
89	static int prep_prog_info(struct bpf_prog_info *const info, enum dump_mode mode,
90	void **info_data, size_t *const info_data_sz)
91	{
92	struct bpf_prog_info holder = {};
93	size_t needed = 0;
94	void *ptr;
95
96	if (mode == DUMP_JITED) {
97	holder.jited_prog_len = info->jited_prog_len;
98	needed += info->jited_prog_len;
99	} else {
100	holder.xlated_prog_len = info->xlated_prog_len;
101	needed += info->xlated_prog_len;
102	}
103
104	holder.nr_jited_ksyms = info->nr_jited_ksyms;
105	needed += info->nr_jited_ksyms * sizeof(__u64);
106
107	holder.nr_jited_func_lens = info->nr_jited_func_lens;
108	needed += info->nr_jited_func_lens * sizeof(__u32);
109
110	holder.nr_func_info = info->nr_func_info;
111	holder.func_info_rec_size = info->func_info_rec_size;
112	needed += info->nr_func_info * info->func_info_rec_size;
113
114	holder.nr_line_info = info->nr_line_info;
115	holder.line_info_rec_size = info->line_info_rec_size;
116	needed += info->nr_line_info * info->line_info_rec_size;
117
118	holder.nr_jited_line_info = info->nr_jited_line_info;
119	holder.jited_line_info_rec_size = info->jited_line_info_rec_size;
120	needed += info->nr_jited_line_info * info->jited_line_info_rec_size;
121
122	if (needed > *info_data_sz) {
123	ptr = realloc(*info_data, needed);
124	if (!ptr)
125	return -1;
126
127	*info_data = ptr;
128	*info_data_sz = needed;
129	}
130	ptr = *info_data;
131
132	if (mode == DUMP_JITED) {
133	holder.jited_prog_insns = ptr_to_u64(ptr);
134	ptr += holder.jited_prog_len;
135	} else {
136	holder.xlated_prog_insns = ptr_to_u64(ptr);
137	ptr += holder.xlated_prog_len;
138	}
139
140	holder.jited_ksyms = ptr_to_u64(ptr);
141	ptr += holder.nr_jited_ksyms * sizeof(__u64);
142
143	holder.jited_func_lens = ptr_to_u64(ptr);
144	ptr += holder.nr_jited_func_lens * sizeof(__u32);
145
146	holder.func_info = ptr_to_u64(ptr);
147	ptr += holder.nr_func_info * holder.func_info_rec_size;
148
149	holder.line_info = ptr_to_u64(ptr);
150	ptr += holder.nr_line_info * holder.line_info_rec_size;
151
152	holder.jited_line_info = ptr_to_u64(ptr);
153	ptr += holder.nr_jited_line_info * holder.jited_line_info_rec_size;
154
155	*info = holder;
156	return 0;
157	}
158
159	static void print_boot_time(__u64 nsecs, char *buf, unsigned int size)
160	{
161	struct timespec real_time_ts, boot_time_ts;
162	time_t wallclock_secs;
163	struct tm load_tm;
164
165	buf[--size] = '\0';
166
167	if (clock_gettime(CLOCK_REALTIME, &real_time_ts) ||
168	clock_gettime(CLOCK_BOOTTIME, &boot_time_ts)) {
169	perror("Can't read clocks");
170	snprintf(buf, size, fmt: "%llu", nsecs / 1000000000);
171	return;
172	}
173
174	wallclock_secs = (real_time_ts.tv_sec - boot_time_ts.tv_sec) +
175	(real_time_ts.tv_nsec - boot_time_ts.tv_nsec + nsecs) /
176	1000000000;
177
178
179	if (!localtime_r(&wallclock_secs, &load_tm)) {
180	snprintf(buf, size, fmt: "%llu", nsecs / 1000000000);
181	return;
182	}
183
184	if (json_output)
185	strftime(buf, size, "%s", &load_tm);
186	else
187	strftime(buf, size, "%FT%T%z", &load_tm);
188	}
189
190	static void show_prog_maps(int fd, __u32 num_maps)
191	{
192	struct bpf_prog_info info = {};
193	__u32 len = sizeof(info);
194	__u32 map_ids[num_maps];
195	unsigned int i;
196	int err;
197
198	info.nr_map_ids = num_maps;
199	info.map_ids = ptr_to_u64(ptr: map_ids);
200
201	err = bpf_prog_get_info_by_fd(fd, &info, &len);
202	if (err || !info.nr_map_ids)
203	return;
204
205	if (json_output) {
206	jsonw_name(self: json_wtr, name: "map_ids");
207	jsonw_start_array(self: json_wtr);
208	for (i = 0; i < info.nr_map_ids; i++)
209	jsonw_uint(self: json_wtr, number: map_ids[i]);
210	jsonw_end_array(self: json_wtr);
211	} else {
212	printf(" map_ids ");
213	for (i = 0; i < info.nr_map_ids; i++)
214	printf("%u%s", map_ids[i],
215	i == info.nr_map_ids - 1 ? "" : ",");
216	}
217	}
218
219	static void *find_metadata(int prog_fd, struct bpf_map_info *map_info)
220	{
221	struct bpf_prog_info prog_info;
222	__u32 prog_info_len;
223	__u32 map_info_len;
224	void *value = NULL;
225	__u32 *map_ids;
226	int nr_maps;
227	int key = 0;
228	int map_fd;
229	int ret;
230	__u32 i;
231
232	memset(&prog_info, 0, sizeof(prog_info));
233	prog_info_len = sizeof(prog_info);
234	ret = bpf_prog_get_info_by_fd(prog_fd, &prog_info, &prog_info_len);
235	if (ret)
236	return NULL;
237
238	if (!prog_info.nr_map_ids)
239	return NULL;
240
241	map_ids = calloc(prog_info.nr_map_ids, sizeof(__u32));
242	if (!map_ids)
243	return NULL;
244
245	nr_maps = prog_info.nr_map_ids;
246	memset(&prog_info, 0, sizeof(prog_info));
247	prog_info.nr_map_ids = nr_maps;
248	prog_info.map_ids = ptr_to_u64(ptr: map_ids);
249	prog_info_len = sizeof(prog_info);
250
251	ret = bpf_prog_get_info_by_fd(prog_fd, &prog_info, &prog_info_len);
252	if (ret)
253	goto free_map_ids;
254
255	for (i = 0; i < prog_info.nr_map_ids; i++) {
256	map_fd = bpf_map_get_fd_by_id(map_ids[i]);
257	if (map_fd < 0)
258	goto free_map_ids;
259
260	memset(map_info, 0, sizeof(*map_info));
261	map_info_len = sizeof(*map_info);
262	ret = bpf_map_get_info_by_fd(map_fd, map_info, &map_info_len);
263	if (ret < 0) {
264	close(map_fd);
265	goto free_map_ids;
266	}
267
268	if (map_info->type != BPF_MAP_TYPE_ARRAY ||
269	map_info->key_size != sizeof(int) ||
270	map_info->max_entries != 1 ||
271	!map_info->btf_value_type_id ||
272	!strstr(map_info->name, ".rodata")) {
273	close(map_fd);
274	continue;
275	}
276
277	value = malloc(map_info->value_size);
278	if (!value) {
279	close(map_fd);
280	goto free_map_ids;
281	}
282
283	if (bpf_map_lookup_elem(map_fd, &key, value)) {
284	close(map_fd);
285	free(value);
286	value = NULL;
287	goto free_map_ids;
288	}
289
290	close(map_fd);
291	break;
292	}
293
294	free_map_ids:
295	free(map_ids);
296	return value;
297	}
298
299	static bool has_metadata_prefix(const char *s)
300	{
301	return strncmp(s, BPF_METADATA_PREFIX, BPF_METADATA_PREFIX_LEN) == 0;
302	}
303
304	static void show_prog_metadata(int fd, __u32 num_maps)
305	{
306	const struct btf_type *t_datasec, *t_var;
307	struct bpf_map_info map_info;
308	struct btf_var_secinfo *vsi;
309	bool printed_header = false;
310	unsigned int i, vlen;
311	void *value = NULL;
312	const char *name;
313	struct btf *btf;
314	int err;
315
316	if (!num_maps)
317	return;
318
319	memset(&map_info, 0, sizeof(map_info));
320	value = find_metadata(prog_fd: fd, map_info: &map_info);
321	if (!value)
322	return;
323
324	btf = btf__load_from_kernel_by_id(map_info.btf_id);
325	if (!btf)
326	goto out_free;
327
328	t_datasec = btf__type_by_id(btf, map_info.btf_value_type_id);
329	if (!btf_is_datasec(t_datasec))
330	goto out_free;
331
332	vlen = btf_vlen(t: t_datasec);
333	vsi = btf_var_secinfos(t_datasec);
334
335	/* We don't proceed to check the kinds of the elements of the DATASEC.
336	* The verifier enforces them to be BTF_KIND_VAR.
337	*/
338
339	if (json_output) {
340	struct btf_dumper d = {
341	.btf = btf,
342	.jw = json_wtr,
343	.is_plain_text = false,
344	};
345
346	for (i = 0; i < vlen; i++, vsi++) {
347	t_var = btf__type_by_id(btf, vsi->type);
348	name = btf__name_by_offset(btf, t_var->name_off);
349
350	if (!has_metadata_prefix(s: name))
351	continue;
352
353	if (!printed_header) {
354	jsonw_name(self: json_wtr, name: "metadata");
355	jsonw_start_object(self: json_wtr);
356	printed_header = true;
357	}
358
359	jsonw_name(self: json_wtr, name: name + BPF_METADATA_PREFIX_LEN);
360	err = btf_dumper_type(d: &d, type_id: t_var->type, data: value + vsi->offset);
361	if (err) {
362	p_err(fmt: "btf dump failed: %d", err);
363	break;
364	}
365	}
366	if (printed_header)
367	jsonw_end_object(self: json_wtr);
368	} else {
369	json_writer_t *btf_wtr;
370	struct btf_dumper d = {
371	.btf = btf,
372	.is_plain_text = true,
373	};
374
375	for (i = 0; i < vlen; i++, vsi++) {
376	t_var = btf__type_by_id(btf, vsi->type);
377	name = btf__name_by_offset(btf, t_var->name_off);
378
379	if (!has_metadata_prefix(s: name))
380	continue;
381
382	if (!printed_header) {
383	printf("\tmetadata:");
384
385	btf_wtr = jsonw_new(stdout);
386	if (!btf_wtr) {
387	p_err(fmt: "jsonw alloc failed");
388	goto out_free;
389	}
390	d.jw = btf_wtr,
391
392	printed_header = true;
393	}
394
395	printf("\n\t\t%s = ", name + BPF_METADATA_PREFIX_LEN);
396
397	jsonw_reset(self: btf_wtr);
398	err = btf_dumper_type(d: &d, type_id: t_var->type, data: value + vsi->offset);
399	if (err) {
400	p_err(fmt: "btf dump failed: %d", err);
401	break;
402	}
403	}
404	if (printed_header)
405	jsonw_destroy(self_p: &btf_wtr);
406	}
407
408	out_free:
409	btf__free(btf);
410	free(value);
411	}
412
413	static void print_prog_header_json(struct bpf_prog_info *info, int fd)
414	{
415	const char *prog_type_str;
416	char prog_name[MAX_PROG_FULL_NAME];
417
418	jsonw_uint_field(self: json_wtr, prop: "id", num: info->id);
419	prog_type_str = libbpf_bpf_prog_type_str(info->type);
420
421	if (prog_type_str)
422	jsonw_string_field(self: json_wtr, prop: "type", val: prog_type_str);
423	else
424	jsonw_uint_field(self: json_wtr, prop: "type", num: info->type);
425
426	if (*info->name) {
427	get_prog_full_name(prog_info: info, prog_fd: fd, name_buff: prog_name, buff_len: sizeof(prog_name));
428	jsonw_string_field(self: json_wtr, prop: "name", val: prog_name);
429	}
430
431	jsonw_name(self: json_wtr, name: "tag");
432	jsonw_printf(self: json_wtr, fmt: "\"" BPF_TAG_FMT "\"",
433	info->tag[0], info->tag[1], info->tag[2], info->tag[3],
434	info->tag[4], info->tag[5], info->tag[6], info->tag[7]);
435
436	jsonw_bool_field(self: json_wtr, prop: "gpl_compatible", value: info->gpl_compatible);
437	if (info->run_time_ns) {
438	jsonw_uint_field(self: json_wtr, prop: "run_time_ns", num: info->run_time_ns);
439	jsonw_uint_field(self: json_wtr, prop: "run_cnt", num: info->run_cnt);
440	}
441	if (info->recursion_misses)
442	jsonw_uint_field(self: json_wtr, prop: "recursion_misses", num: info->recursion_misses);
443	}
444
445	static void print_prog_json(struct bpf_prog_info *info, int fd)
446	{
447	char *memlock;
448
449	jsonw_start_object(self: json_wtr);
450	print_prog_header_json(info, fd);
451	print_dev_json(ifindex: info->ifindex, ns_dev: info->netns_dev, ns_inode: info->netns_ino);
452
453	if (info->load_time) {
454	char buf[32];
455
456	print_boot_time(nsecs: info->load_time, buf, size: sizeof(buf));
457
458	/* Piggy back on load_time, since 0 uid is a valid one */
459	jsonw_name(self: json_wtr, name: "loaded_at");
460	jsonw_printf(self: json_wtr, fmt: "%s", buf);
461	jsonw_uint_field(self: json_wtr, prop: "uid", num: info->created_by_uid);
462	}
463
464	jsonw_uint_field(self: json_wtr, prop: "bytes_xlated", num: info->xlated_prog_len);
465
466	if (info->jited_prog_len) {
467	jsonw_bool_field(self: json_wtr, prop: "jited", value: true);
468	jsonw_uint_field(self: json_wtr, prop: "bytes_jited", num: info->jited_prog_len);
469	} else {
470	jsonw_bool_field(self: json_wtr, prop: "jited", value: false);
471	}
472
473	memlock = get_fdinfo(fd, key: "memlock");
474	if (memlock)
475	jsonw_int_field(self: json_wtr, prop: "bytes_memlock", num: atoll(memlock));
476	free(memlock);
477
478	if (info->nr_map_ids)
479	show_prog_maps(fd, num_maps: info->nr_map_ids);
480
481	if (info->btf_id)
482	jsonw_int_field(self: json_wtr, prop: "btf_id", num: info->btf_id);
483
484	if (!hashmap__empty(map: prog_table)) {
485	struct hashmap_entry *entry;
486
487	jsonw_name(self: json_wtr, name: "pinned");
488	jsonw_start_array(self: json_wtr);
489	hashmap__for_each_key_entry(prog_table, entry, info->id)
490	jsonw_string(self: json_wtr, value: entry->pvalue);
491	jsonw_end_array(self: json_wtr);
492	}
493
494	emit_obj_refs_json(table: refs_table, id: info->id, json_wtr);
495
496	show_prog_metadata(fd, num_maps: info->nr_map_ids);
497
498	jsonw_end_object(self: json_wtr);
499	}
500
501	static void print_prog_header_plain(struct bpf_prog_info *info, int fd)
502	{
503	const char *prog_type_str;
504	char prog_name[MAX_PROG_FULL_NAME];
505
506	printf("%u: ", info->id);
507	prog_type_str = libbpf_bpf_prog_type_str(info->type);
508	if (prog_type_str)
509	printf("%s ", prog_type_str);
510	else
511	printf("type %u ", info->type);
512
513	if (*info->name) {
514	get_prog_full_name(prog_info: info, prog_fd: fd, name_buff: prog_name, buff_len: sizeof(prog_name));
515	printf("name %s ", prog_name);
516	}
517
518	printf("tag ");
519	fprint_hex(stdout, info->tag, BPF_TAG_SIZE, "");
520	print_dev_plain(ifindex: info->ifindex, ns_dev: info->netns_dev, ns_inode: info->netns_ino);
521	printf("%s", info->gpl_compatible ? " gpl" : "");
522	if (info->run_time_ns)
523	printf(" run_time_ns %lld run_cnt %lld",
524	info->run_time_ns, info->run_cnt);
525	if (info->recursion_misses)
526	printf(" recursion_misses %lld", info->recursion_misses);
527	printf("\n");
528	}
529
530	static void print_prog_plain(struct bpf_prog_info *info, int fd)
531	{
532	char *memlock;
533
534	print_prog_header_plain(info, fd);
535
536	if (info->load_time) {
537	char buf[32];
538
539	print_boot_time(nsecs: info->load_time, buf, size: sizeof(buf));
540
541	/* Piggy back on load_time, since 0 uid is a valid one */
542	printf("\tloaded_at %s uid %u\n", buf, info->created_by_uid);
543	}
544
545	printf("\txlated %uB", info->xlated_prog_len);
546
547	if (info->jited_prog_len)
548	printf(" jited %uB", info->jited_prog_len);
549	else
550	printf(" not jited");
551
552	memlock = get_fdinfo(fd, key: "memlock");
553	if (memlock)
554	printf(" memlock %sB", memlock);
555	free(memlock);
556
557	if (info->nr_map_ids)
558	show_prog_maps(fd, num_maps: info->nr_map_ids);
559
560	if (!hashmap__empty(map: prog_table)) {
561	struct hashmap_entry *entry;
562
563	hashmap__for_each_key_entry(prog_table, entry, info->id)
564	printf("\n\tpinned %s", (char *)entry->pvalue);
565	}
566
567	if (info->btf_id)
568	printf("\n\tbtf_id %d", info->btf_id);
569
570	emit_obj_refs_plain(table: refs_table, id: info->id, prefix: "\n\tpids ");
571
572	printf("\n");
573
574	show_prog_metadata(fd, num_maps: info->nr_map_ids);
575	}
576
577	static int show_prog(int fd)
578	{
579	struct bpf_prog_info info = {};
580	__u32 len = sizeof(info);
581	int err;
582
583	err = bpf_prog_get_info_by_fd(fd, &info, &len);
584	if (err) {
585	p_err(fmt: "can't get prog info: %s", strerror(errno));
586	return -1;
587	}
588
589	if (json_output)
590	print_prog_json(info: &info, fd);
591	else
592	print_prog_plain(info: &info, fd);
593
594	return 0;
595	}
596
597	static int do_show_subset(int argc, char **argv)
598	{
599	int *fds = NULL;
600	int nb_fds, i;
601	int err = -1;
602
603	fds = malloc(sizeof(int));
604	if (!fds) {
605	p_err(fmt: "mem alloc failed");
606	return -1;
607	}
608	nb_fds = prog_parse_fds(argc: &argc, argv: &argv, fds: &fds);
609	if (nb_fds < 1)
610	goto exit_free;
611
612	if (json_output && nb_fds > 1)
613	jsonw_start_array(self: json_wtr); /* root array */
614	for (i = 0; i < nb_fds; i++) {
615	err = show_prog(fd: fds[i]);
616	if (err) {
617	for (; i < nb_fds; i++)
618	close(fds[i]);
619	break;
620	}
621	close(fds[i]);
622	}
623	if (json_output && nb_fds > 1)
624	jsonw_end_array(self: json_wtr); /* root array */
625
626	exit_free:
627	free(fds);
628	return err;
629	}
630
631	static int do_show(int argc, char **argv)
632	{
633	__u32 id = 0;
634	int err;
635	int fd;
636
637	if (show_pinned) {
638	prog_table = hashmap__new(hash_fn_for_key_as_id,
639	equal_fn_for_key_as_id, NULL);
640	if (IS_ERR(ptr: prog_table)) {
641	p_err(fmt: "failed to create hashmap for pinned paths");
642	return -1;
643	}
644	build_pinned_obj_table(table: prog_table, type: BPF_OBJ_PROG);
645	}
646	build_obj_refs_table(table: &refs_table, type: BPF_OBJ_PROG);
647
648	if (argc == 2)
649	return do_show_subset(argc, argv);
650
651	if (argc)
652	return BAD_ARG();
653
654	if (json_output)
655	jsonw_start_array(self: json_wtr);
656	while (true) {
657	err = bpf_prog_get_next_id(id, &id);
658	if (err) {
659	if (errno == ENOENT) {
660	err = 0;
661	break;
662	}
663	p_err(fmt: "can't get next program: %s%s", strerror(errno),
664	errno == EINVAL ? " -- kernel too old?" : "");
665	err = -1;
666	break;
667	}
668
669	fd = bpf_prog_get_fd_by_id(id);
670	if (fd < 0) {
671	if (errno == ENOENT)
672	continue;
673	p_err(fmt: "can't get prog by id (%u): %s",
674	id, strerror(errno));
675	err = -1;
676	break;
677	}
678
679	err = show_prog(fd);
680	close(fd);
681	if (err)
682	break;
683	}
684
685	if (json_output)
686	jsonw_end_array(self: json_wtr);
687
688	delete_obj_refs_table(table: refs_table);
689
690	if (show_pinned)
691	delete_pinned_obj_table(table: prog_table);
692
693	return err;
694	}
695
696	static int
697	prog_dump(struct bpf_prog_info *info, enum dump_mode mode,
698	char *filepath, bool opcodes, bool visual, bool linum)
699	{
700	struct bpf_prog_linfo *prog_linfo = NULL;
701	const char *disasm_opt = NULL;
702	struct dump_data dd = {};
703	void *func_info = NULL;
704	struct btf *btf = NULL;
705	char func_sig[1024];
706	unsigned char *buf;
707	__u32 member_len;
708	int fd, err = -1;
709	ssize_t n;
710
711	if (mode == DUMP_JITED) {
712	if (info->jited_prog_len == 0 || !info->jited_prog_insns) {
713	p_info(fmt: "no instructions returned");
714	return -1;
715	}
716	buf = u64_to_ptr(ptr: info->jited_prog_insns);
717	member_len = info->jited_prog_len;
718	} else { /* DUMP_XLATED */
719	if (info->xlated_prog_len == 0 || !info->xlated_prog_insns) {
720	p_err(fmt: "error retrieving insn dump: kernel.kptr_restrict set?");
721	return -1;
722	}
723	buf = u64_to_ptr(ptr: info->xlated_prog_insns);
724	member_len = info->xlated_prog_len;
725	}
726
727	if (info->btf_id) {
728	btf = btf__load_from_kernel_by_id(info->btf_id);
729	if (!btf) {
730	p_err(fmt: "failed to get btf");
731	return -1;
732	}
733	}
734
735	func_info = u64_to_ptr(ptr: info->func_info);
736
737	if (info->nr_line_info) {
738	prog_linfo = bpf_prog_linfo__new(info);
739	if (!prog_linfo)
740	p_info(fmt: "error in processing bpf_line_info. continue without it.");
741	}
742
743	if (filepath) {
744	fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0600);
745	if (fd < 0) {
746	p_err(fmt: "can't open file %s: %s", filepath,
747	strerror(errno));
748	goto exit_free;
749	}
750
751	n = write(fd, buf, member_len);
752	close(fd);
753	if (n != (ssize_t)member_len) {
754	p_err(fmt: "error writing output file: %s",
755	n < 0 ? strerror(errno) : "short write");
756	goto exit_free;
757	}
758
759	if (json_output)
760	jsonw_null(self: json_wtr);
761	} else if (mode == DUMP_JITED) {
762	const char *name = NULL;
763
764	if (info->ifindex) {
765	name = ifindex_to_arch(ifindex: info->ifindex, ns_dev: info->netns_dev,
766	ns_ino: info->netns_ino, opt: &disasm_opt);
767	if (!name)
768	goto exit_free;
769	}
770
771	if (info->nr_jited_func_lens && info->jited_func_lens) {
772	struct kernel_sym *sym = NULL;
773	struct bpf_func_info *record;
774	char sym_name[SYM_MAX_NAME];
775	unsigned char *img = buf;
776	__u64 *ksyms = NULL;
777	__u32 *lens;
778	__u32 i;
779	if (info->nr_jited_ksyms) {
780	kernel_syms_load(dd: &dd);
781	ksyms = u64_to_ptr(ptr: info->jited_ksyms);
782	}
783
784	if (json_output)
785	jsonw_start_array(self: json_wtr);
786
787	lens = u64_to_ptr(ptr: info->jited_func_lens);
788	for (i = 0; i < info->nr_jited_func_lens; i++) {
789	if (ksyms) {
790	sym = kernel_syms_search(dd: &dd, key: ksyms[i]);
791	if (sym)
792	sprintf(buf: sym_name, fmt: "%s", sym->name);
793	else
794	sprintf(buf: sym_name, fmt: "0x%016llx", ksyms[i]);
795	} else {
796	strcpy(p: sym_name, q: "unknown");
797	}
798
799	if (func_info) {
800	record = func_info + i * info->func_info_rec_size;
801	btf_dumper_type_only(btf, func_type_id: record->type_id,
802	func_only: func_sig,
803	size: sizeof(func_sig));
804	}
805
806	if (json_output) {
807	jsonw_start_object(self: json_wtr);
808	if (func_info && func_sig[0] != '\0') {
809	jsonw_name(self: json_wtr, name: "proto");
810	jsonw_string(self: json_wtr, value: func_sig);
811	}
812	jsonw_name(self: json_wtr, name: "name");
813	jsonw_string(self: json_wtr, value: sym_name);
814	jsonw_name(self: json_wtr, name: "insns");
815	} else {
816	if (func_info && func_sig[0] != '\0')
817	printf("%s:\n", func_sig);
818	printf("%s:\n", sym_name);
819	}
820
821	if (disasm_print_insn(image: img, len: lens[i], opcodes,
822	arch: name, disassembler_options: disasm_opt, btf,
823	prog_linfo, func_ksym: ksyms[i], func_idx: i,
824	linum))
825	goto exit_free;
826
827	img += lens[i];
828
829	if (json_output)
830	jsonw_end_object(self: json_wtr);
831	else
832	printf("\n");
833	}
834
835	if (json_output)
836	jsonw_end_array(self: json_wtr);
837	} else {
838	if (disasm_print_insn(image: buf, len: member_len, opcodes, arch: name,
839	disassembler_options: disasm_opt, btf, NULL, func_ksym: 0, func_idx: 0,
840	linum: false))
841	goto exit_free;
842	}
843	} else {
844	kernel_syms_load(dd: &dd);
845	dd.nr_jited_ksyms = info->nr_jited_ksyms;
846	dd.jited_ksyms = u64_to_ptr(ptr: info->jited_ksyms);
847	dd.btf = btf;
848	dd.func_info = func_info;
849	dd.finfo_rec_size = info->func_info_rec_size;
850	dd.prog_linfo = prog_linfo;
851
852	if (json_output)
853	dump_xlated_json(dd: &dd, buf, len: member_len, opcodes, linum);
854	else if (visual)
855	dump_xlated_cfg(dd: &dd, buf, len: member_len, opcodes, linum);
856	else
857	dump_xlated_plain(dd: &dd, buf, len: member_len, opcodes, linum);
858	kernel_syms_destroy(dd: &dd);
859	}
860
861	err = 0;
862
863	exit_free:
864	btf__free(btf);
865	bpf_prog_linfo__free(prog_linfo);
866	return err;
867	}
868
869	static int do_dump(int argc, char **argv)
870	{
871	struct bpf_prog_info info;
872	__u32 info_len = sizeof(info);
873	size_t info_data_sz = 0;
874	void *info_data = NULL;
875	char *filepath = NULL;
876	bool opcodes = false;
877	bool visual = false;
878	enum dump_mode mode;
879	bool linum = false;
880	int nb_fds, i = 0;
881	int *fds = NULL;
882	int err = -1;
883
884	if (is_prefix(pfx: *argv, str: "jited")) {
885	if (disasm_init())
886	return -1;
887	mode = DUMP_JITED;
888	} else if (is_prefix(pfx: *argv, str: "xlated")) {
889	mode = DUMP_XLATED;
890	} else {
891	p_err(fmt: "expected 'xlated' or 'jited', got: %s", *argv);
892	return -1;
893	}
894	NEXT_ARG();
895
896	if (argc < 2)
897	usage();
898
899	fds = malloc(sizeof(int));
900	if (!fds) {
901	p_err(fmt: "mem alloc failed");
902	return -1;
903	}
904	nb_fds = prog_parse_fds(argc: &argc, argv: &argv, fds: &fds);
905	if (nb_fds < 1)
906	goto exit_free;
907
908	while (argc) {
909	if (is_prefix(pfx: *argv, str: "file")) {
910	NEXT_ARG();
911	if (!argc) {
912	p_err(fmt: "expected file path");
913	goto exit_close;
914	}
915	if (nb_fds > 1) {
916	p_err(fmt: "several programs matched");
917	goto exit_close;
918	}
919
920	filepath = *argv;
921	NEXT_ARG();
922	} else if (is_prefix(pfx: *argv, str: "opcodes")) {
923	opcodes = true;
924	NEXT_ARG();
925	} else if (is_prefix(pfx: *argv, str: "visual")) {
926	if (nb_fds > 1) {
927	p_err(fmt: "several programs matched");
928	goto exit_close;
929	}
930
931	visual = true;
932	NEXT_ARG();
933	} else if (is_prefix(pfx: *argv, str: "linum")) {
934	linum = true;
935	NEXT_ARG();
936	} else {
937	usage();
938	goto exit_close;
939	}
940	}
941
942	if (filepath && (opcodes || visual || linum)) {
943	p_err(fmt: "'file' is not compatible with 'opcodes', 'visual', or 'linum'");
944	goto exit_close;
945	}
946	if (json_output && visual) {
947	p_err(fmt: "'visual' is not compatible with JSON output");
948	goto exit_close;
949	}
950
951	if (json_output && nb_fds > 1)
952	jsonw_start_array(self: json_wtr); /* root array */
953	for (i = 0; i < nb_fds; i++) {
954	memset(&info, 0, sizeof(info));
955
956	err = bpf_prog_get_info_by_fd(fds[i], &info, &info_len);
957	if (err) {
958	p_err("can't get prog info: %s", strerror(errno));
959	break;
960	}
961
962	err = prep_prog_info(info: &info, mode, info_data: &info_data, info_data_sz: &info_data_sz);
963	if (err) {
964	p_err(fmt: "can't grow prog info_data");
965	break;
966	}
967
968	err = bpf_prog_get_info_by_fd(fds[i], &info, &info_len);
969	if (err) {
970	p_err("can't get prog info: %s", strerror(errno));
971	break;
972	}
973
974	if (json_output && nb_fds > 1) {
975	jsonw_start_object(self: json_wtr); /* prog object */
976	print_prog_header_json(info: &info, fd: fds[i]);
977	jsonw_name(self: json_wtr, name: "insns");
978	} else if (nb_fds > 1) {
979	print_prog_header_plain(info: &info, fd: fds[i]);
980	}
981
982	err = prog_dump(info: &info, mode, filepath, opcodes, visual, linum);
983
984	if (json_output && nb_fds > 1)
985	jsonw_end_object(self: json_wtr); /* prog object */
986	else if (i != nb_fds - 1 && nb_fds > 1)
987	printf("\n");
988
989	if (err)
990	break;
991	close(fds[i]);
992	}
993	if (json_output && nb_fds > 1)
994	jsonw_end_array(self: json_wtr); /* root array */
995
996	exit_close:
997	for (; i < nb_fds; i++)
998	close(fds[i]);
999	exit_free:
1000	free(info_data);
1001	free(fds);
1002	return err;
1003	}
1004
1005	static int do_pin(int argc, char **argv)
1006	{
1007	int err;
1008
1009	err = do_pin_any(argc, argv, get_fd_by_id: prog_parse_fd);
1010	if (!err && json_output)
1011	jsonw_null(self: json_wtr);
1012	return err;
1013	}
1014
1015	struct map_replace {
1016	int idx;
1017	int fd;
1018	char *name;
1019	};
1020
1021	static int map_replace_compar(const void *p1, const void *p2)
1022	{
1023	const struct map_replace *a = p1, *b = p2;
1024
1025	return a->idx - b->idx;
1026	}
1027
1028	static int parse_attach_detach_args(int argc, char **argv, int *progfd,
1029	enum bpf_attach_type *attach_type,
1030	int *mapfd)
1031	{
1032	if (!REQ_ARGS(3))
1033	return -EINVAL;
1034
1035	*progfd = prog_parse_fd(argc: &argc, argv: &argv);
1036	if (*progfd < 0)
1037	return *progfd;
1038
1039	*attach_type = parse_attach_type(str: *argv);
1040	if (*attach_type == __MAX_BPF_ATTACH_TYPE) {
1041	p_err(fmt: "invalid attach/detach type");
1042	return -EINVAL;
1043	}
1044
1045	if (*attach_type == BPF_FLOW_DISSECTOR) {
1046	*mapfd = 0;
1047	return 0;
1048	}
1049
1050	NEXT_ARG();
1051	if (!REQ_ARGS(2))
1052	return -EINVAL;
1053
1054	*mapfd = map_parse_fd(argc: &argc, argv: &argv);
1055	if (*mapfd < 0)
1056	return *mapfd;
1057
1058	return 0;
1059	}
1060
1061	static int do_attach(int argc, char **argv)
1062	{
1063	enum bpf_attach_type attach_type;
1064	int err, progfd;
1065	int mapfd;
1066
1067	err = parse_attach_detach_args(argc, argv,
1068	progfd: &progfd, attach_type: &attach_type, mapfd: &mapfd);
1069	if (err)
1070	return err;
1071
1072	err = bpf_prog_attach(progfd, mapfd, attach_type, 0);
1073	if (err) {
1074	p_err(fmt: "failed prog attach to map");
1075	return -EINVAL;
1076	}
1077
1078	if (json_output)
1079	jsonw_null(self: json_wtr);
1080	return 0;
1081	}
1082
1083	static int do_detach(int argc, char **argv)
1084	{
1085	enum bpf_attach_type attach_type;
1086	int err, progfd;
1087	int mapfd;
1088
1089	err = parse_attach_detach_args(argc, argv,
1090	progfd: &progfd, attach_type: &attach_type, mapfd: &mapfd);
1091	if (err)
1092	return err;
1093
1094	err = bpf_prog_detach2(progfd, mapfd, attach_type);
1095	if (err) {
1096	p_err(fmt: "failed prog detach from map");
1097	return -EINVAL;
1098	}
1099
1100	if (json_output)
1101	jsonw_null(self: json_wtr);
1102	return 0;
1103	}
1104
1105	static int check_single_stdin(char *file_data_in, char *file_ctx_in)
1106	{
1107	if (file_data_in && file_ctx_in &&
1108	!strcmp(file_data_in, "-") && !strcmp(file_ctx_in, "-")) {
1109	p_err(fmt: "cannot use standard input for both data_in and ctx_in");
1110	return -1;
1111	}
1112
1113	return 0;
1114	}
1115
1116	static int get_run_data(const char *fname, void **data_ptr, unsigned int *size)
1117	{
1118	size_t block_size = 256;
1119	size_t buf_size = block_size;
1120	size_t nb_read = 0;
1121	void *tmp;
1122	FILE *f;
1123
1124	if (!fname) {
1125	*data_ptr = NULL;
1126	*size = 0;
1127	return 0;
1128	}
1129
1130	if (!strcmp(fname, "-"))
1131	f = stdin;
1132	else
1133	f = fopen(fname, "r");
1134	if (!f) {
1135	p_err("failed to open %s: %s", fname, strerror(errno));
1136	return -1;
1137	}
1138
1139	*data_ptr = malloc(block_size);
1140	if (!*data_ptr) {
1141	p_err("failed to allocate memory for data_in/ctx_in: %s",
1142	strerror(errno));
1143	goto err_fclose;
1144	}
1145
1146	while ((nb_read += fread(*data_ptr + nb_read, 1, block_size, f))) {
1147	if (feof(f))
1148	break;
1149	if (ferror(f)) {
1150	p_err("failed to read data_in/ctx_in from %s: %s",
1151	fname, strerror(errno));
1152	goto err_free;
1153	}
1154	if (nb_read > buf_size - block_size) {
1155	if (buf_size == UINT32_MAX) {
1156	p_err("data_in/ctx_in is too long (max: %d)",
1157	UINT32_MAX);
1158	goto err_free;
1159	}
1160	/* No space for fread()-ing next chunk; realloc() */
1161	buf_size *= 2;
1162	tmp = realloc(*data_ptr, buf_size);
1163	if (!tmp) {
1164	p_err("failed to reallocate data_in/ctx_in: %s",
1165	strerror(errno));
1166	goto err_free;
1167	}
1168	*data_ptr = tmp;
1169	}
1170	}
1171	if (f != stdin)
1172	fclose(f);
1173
1174	*size = nb_read;
1175	return 0;
1176
1177	err_free:
1178	free(*data_ptr);
1179	*data_ptr = NULL;
1180	err_fclose:
1181	if (f != stdin)
1182	fclose(f);
1183	return -1;
1184	}
1185
1186	static void hex_print(void *data, unsigned int size, FILE *f)
1187	{
1188	size_t i, j;
1189	char c;
1190
1191	for (i = 0; i < size; i += 16) {
1192	/* Row offset */
1193	fprintf(f, "%07zx\t", i);
1194
1195	/* Hexadecimal values */
1196	for (j = i; j < i + 16 && j < size; j++)
1197	fprintf(f, "%02x%s", *(uint8_t *)(data + j),
1198	j % 2 ? " " : "");
1199	for (; j < i + 16; j++)
1200	fprintf(f, " %s", j % 2 ? " " : "");
1201
1202	/* ASCII values (if relevant), '.' otherwise */
1203	fprintf(f, "| ");
1204	for (j = i; j < i + 16 && j < size; j++) {
1205	c = *(char *)(data + j);
1206	if (c < ' ' || c > '~')
1207	c = '.';
1208	fprintf(f, "%c%s", c, j == i + 7 ? " " : "");
1209	}
1210
1211	fprintf(f, "\n");
1212	}
1213	}
1214
1215	static int
1216	print_run_output(void *data, unsigned int size, const char *fname,
1217	const char *json_key)
1218	{
1219	size_t nb_written;
1220	FILE *f;
1221
1222	if (!fname)
1223	return 0;
1224
1225	if (!strcmp(fname, "-")) {
1226	f = stdout;
1227	if (json_output) {
1228	jsonw_name(self: json_wtr, name: json_key);
1229	print_data_json(data, len: size);
1230	} else {
1231	hex_print(data, size, f);
1232	}
1233	return 0;
1234	}
1235
1236	f = fopen(fname, "w");
1237	if (!f) {
1238	p_err("failed to open %s: %s", fname, strerror(errno));
1239	return -1;
1240	}
1241
1242	nb_written = fwrite(data, 1, size, f);
1243	fclose(f);
1244	if (nb_written != size) {
1245	p_err("failed to write output data/ctx: %s", strerror(errno));
1246	return -1;
1247	}
1248
1249	return 0;
1250	}
1251
1252	static int alloc_run_data(void **data_ptr, unsigned int size_out)
1253	{
1254	*data_ptr = calloc(size_out, 1);
1255	if (!*data_ptr) {
1256	p_err("failed to allocate memory for output data/ctx: %s",
1257	strerror(errno));
1258	return -1;
1259	}
1260
1261	return 0;
1262	}
1263
1264	static int do_run(int argc, char **argv)
1265	{
1266	char *data_fname_in = NULL, *data_fname_out = NULL;
1267	char *ctx_fname_in = NULL, *ctx_fname_out = NULL;
1268	const unsigned int default_size = SZ_32K;
1269	void *data_in = NULL, *data_out = NULL;
1270	void *ctx_in = NULL, *ctx_out = NULL;
1271	unsigned int repeat = 1;
1272	int fd, err;
1273	LIBBPF_OPTS(bpf_test_run_opts, test_attr);
1274
1275	if (!REQ_ARGS(4))
1276	return -1;
1277
1278	fd = prog_parse_fd(argc: &argc, argv: &argv);
1279	if (fd < 0)
1280	return -1;
1281
1282	while (argc) {
1283	if (detect_common_prefix(arg: *argv, "data_in", "data_out",
1284	"data_size_out", NULL))
1285	return -1;
1286	if (detect_common_prefix(arg: *argv, "ctx_in", "ctx_out",
1287	"ctx_size_out", NULL))
1288	return -1;
1289
1290	if (is_prefix(pfx: *argv, str: "data_in")) {
1291	NEXT_ARG();
1292	if (!REQ_ARGS(1))
1293	return -1;
1294
1295	data_fname_in = GET_ARG();
1296	if (check_single_stdin(file_data_in: data_fname_in, file_ctx_in: ctx_fname_in))
1297	return -1;
1298	} else if (is_prefix(pfx: *argv, str: "data_out")) {
1299	NEXT_ARG();
1300	if (!REQ_ARGS(1))
1301	return -1;
1302
1303	data_fname_out = GET_ARG();
1304	} else if (is_prefix(pfx: *argv, str: "data_size_out")) {
1305	char *endptr;
1306
1307	NEXT_ARG();
1308	if (!REQ_ARGS(1))
1309	return -1;
1310
1311	test_attr.data_size_out = strtoul(*argv, &endptr, 0);
1312	if (*endptr) {
1313	p_err(fmt: "can't parse %s as output data size",
1314	*argv);
1315	return -1;
1316	}
1317	NEXT_ARG();
1318	} else if (is_prefix(pfx: *argv, str: "ctx_in")) {
1319	NEXT_ARG();
1320	if (!REQ_ARGS(1))
1321	return -1;
1322
1323	ctx_fname_in = GET_ARG();
1324	if (check_single_stdin(file_data_in: data_fname_in, file_ctx_in: ctx_fname_in))
1325	return -1;
1326	} else if (is_prefix(pfx: *argv, str: "ctx_out")) {
1327	NEXT_ARG();
1328	if (!REQ_ARGS(1))
1329	return -1;
1330
1331	ctx_fname_out = GET_ARG();
1332	} else if (is_prefix(pfx: *argv, str: "ctx_size_out")) {
1333	char *endptr;
1334
1335	NEXT_ARG();
1336	if (!REQ_ARGS(1))
1337	return -1;
1338
1339	test_attr.ctx_size_out = strtoul(*argv, &endptr, 0);
1340	if (*endptr) {
1341	p_err(fmt: "can't parse %s as output context size",
1342	*argv);
1343	return -1;
1344	}
1345	NEXT_ARG();
1346	} else if (is_prefix(pfx: *argv, str: "repeat")) {
1347	char *endptr;
1348
1349	NEXT_ARG();
1350	if (!REQ_ARGS(1))
1351	return -1;
1352
1353	repeat = strtoul(*argv, &endptr, 0);
1354	if (*endptr) {
1355	p_err(fmt: "can't parse %s as repeat number",
1356	*argv);
1357	return -1;
1358	}
1359	NEXT_ARG();
1360	} else {
1361	p_err(fmt: "expected no more arguments, 'data_in', 'data_out', 'data_size_out', 'ctx_in', 'ctx_out', 'ctx_size_out' or 'repeat', got: '%s'?",
1362	*argv);
1363	return -1;
1364	}
1365	}
1366
1367	err = get_run_data(data_fname_in, &data_in, &test_attr.data_size_in);
1368	if (err)
1369	return -1;
1370
1371	if (data_in) {
1372	if (!test_attr.data_size_out)
1373	test_attr.data_size_out = default_size;
1374	err = alloc_run_data(&data_out, test_attr.data_size_out);
1375	if (err)
1376	goto free_data_in;
1377	}
1378
1379	err = get_run_data(ctx_fname_in, &ctx_in, &test_attr.ctx_size_in);
1380	if (err)
1381	goto free_data_out;
1382
1383	if (ctx_in) {
1384	if (!test_attr.ctx_size_out)
1385	test_attr.ctx_size_out = default_size;
1386	err = alloc_run_data(&ctx_out, test_attr.ctx_size_out);
1387	if (err)
1388	goto free_ctx_in;
1389	}
1390
1391	test_attr.repeat = repeat;
1392	test_attr.data_in = data_in;
1393	test_attr.data_out = data_out;
1394	test_attr.ctx_in = ctx_in;
1395	test_attr.ctx_out = ctx_out;
1396
1397	err = bpf_prog_test_run_opts(fd, &test_attr);
1398	if (err) {
1399	p_err("failed to run program: %s", strerror(errno));
1400	goto free_ctx_out;
1401	}
1402
1403	err = 0;
1404
1405	if (json_output)
1406	jsonw_start_object(self: json_wtr); /* root */
1407
1408	/* Do not exit on errors occurring when printing output data/context,
1409	* we still want to print return value and duration for program run.
1410	*/
1411	if (test_attr.data_size_out)
1412	err += print_run_output(test_attr.data_out,
1413	test_attr.data_size_out,
1414	data_fname_out, "data_out");
1415	if (test_attr.ctx_size_out)
1416	err += print_run_output(test_attr.ctx_out,
1417	test_attr.ctx_size_out,
1418	ctx_fname_out, "ctx_out");
1419
1420	if (json_output) {
1421	jsonw_uint_field(json_wtr, "retval", test_attr.retval);
1422	jsonw_uint_field(json_wtr, "duration", test_attr.duration);
1423	jsonw_end_object(self: json_wtr); /* root */
1424	} else {
1425	fprintf(stdout, "Return value: %u, duration%s: %uns\n",
1426	test_attr.retval,
1427	repeat > 1 ? " (average)" : "", test_attr.duration);
1428	}
1429
1430	free_ctx_out:
1431	free(ctx_out);
1432	free_ctx_in:
1433	free(ctx_in);
1434	free_data_out:
1435	free(data_out);
1436	free_data_in:
1437	free(data_in);
1438
1439	return err;
1440	}
1441
1442	static int
1443	get_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
1444	enum bpf_attach_type *expected_attach_type)
1445	{
1446	libbpf_print_fn_t print_backup;
1447	int ret;
1448
1449	ret = libbpf_prog_type_by_name(name, prog_type, expected_attach_type);
1450	if (!ret)
1451	return ret;
1452
1453	/* libbpf_prog_type_by_name() failed, let's re-run with debug level */
1454	print_backup = libbpf_set_print(print_all_levels);
1455	ret = libbpf_prog_type_by_name(name, prog_type, expected_attach_type);
1456	libbpf_set_print(print_backup);
1457
1458	return ret;
1459	}
1460
1461	static int
1462	auto_attach_program(struct bpf_program *prog, const char *path)
1463	{
1464	struct bpf_link *link;
1465	int err;
1466
1467	link = bpf_program__attach(prog);
1468	if (!link) {
1469	p_info(fmt: "Program %s does not support autoattach, falling back to pinning",
1470	bpf_program__name(prog));
1471	return bpf_obj_pin(bpf_program__fd(prog), path);
1472	}
1473
1474	err = bpf_link__pin(link, path);
1475	bpf_link__destroy(link);
1476	return err;
1477	}
1478
1479	static int
1480	auto_attach_programs(struct bpf_object *obj, const char *path)
1481	{
1482	struct bpf_program *prog;
1483	char buf[PATH_MAX];
1484	int err;
1485
1486	bpf_object__for_each_program(prog, obj) {
1487	err = pathname_concat(buf, buf_sz: sizeof(buf), path, name: bpf_program__name(prog));
1488	if (err)
1489	goto err_unpin_programs;
1490
1491	err = auto_attach_program(prog, path: buf);
1492	if (err)
1493	goto err_unpin_programs;
1494	}
1495
1496	return 0;
1497
1498	err_unpin_programs:
1499	while ((prog = bpf_object__prev_program(obj, prog))) {
1500	if (pathname_concat(buf, buf_sz: sizeof(buf), path, name: bpf_program__name(prog)))
1501	continue;
1502
1503	bpf_program__unpin(prog, buf);
1504	}
1505
1506	return err;
1507	}
1508
1509	static int load_with_options(int argc, char **argv, bool first_prog_only)
1510	{
1511	enum bpf_prog_type common_prog_type = BPF_PROG_TYPE_UNSPEC;
1512	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts,
1513	.relaxed_maps = relaxed_maps,
1514	);
1515	enum bpf_attach_type expected_attach_type;
1516	struct map_replace *map_replace = NULL;
1517	struct bpf_program *prog = NULL, *pos;
1518	unsigned int old_map_fds = 0;
1519	const char *pinmaps = NULL;
1520	__u32 xdpmeta_ifindex = 0;
1521	__u32 offload_ifindex = 0;
1522	bool auto_attach = false;
1523	struct bpf_object *obj;
1524	struct bpf_map *map;
1525	const char *pinfile;
1526	unsigned int i, j;
1527	const char *file;
1528	int idx, err;
1529
1530
1531	if (!REQ_ARGS(2))
1532	return -1;
1533	file = GET_ARG();
1534	pinfile = GET_ARG();
1535
1536	while (argc) {
1537	if (is_prefix(pfx: *argv, str: "type")) {
1538	NEXT_ARG();
1539
1540	if (common_prog_type != BPF_PROG_TYPE_UNSPEC) {
1541	p_err(fmt: "program type already specified");
1542	goto err_free_reuse_maps;
1543	}
1544	if (!REQ_ARGS(1))
1545	goto err_free_reuse_maps;
1546
1547	err = libbpf_prog_type_by_name(*argv, &common_prog_type,
1548	&expected_attach_type);
1549	if (err < 0) {
1550	/* Put a '/' at the end of type to appease libbpf */
1551	char *type = malloc(strlen(*argv) + 2);
1552
1553	if (!type) {
1554	p_err(fmt: "mem alloc failed");
1555	goto err_free_reuse_maps;
1556	}
1557	*type = 0;
1558	strcat(p: type, q: *argv);
1559	strcat(p: type, q: "/");
1560
1561	err = get_prog_type_by_name(name: type, prog_type: &common_prog_type,
1562	expected_attach_type: &expected_attach_type);
1563	free(type);
1564	if (err < 0)
1565	goto err_free_reuse_maps;
1566	}
1567
1568	NEXT_ARG();
1569	} else if (is_prefix(pfx: *argv, str: "map")) {
1570	void *new_map_replace;
1571	char *endptr, *name;
1572	int fd;
1573
1574	NEXT_ARG();
1575
1576	if (!REQ_ARGS(4))
1577	goto err_free_reuse_maps;
1578
1579	if (is_prefix(pfx: *argv, str: "idx")) {
1580	NEXT_ARG();
1581
1582	idx = strtoul(*argv, &endptr, 0);
1583	if (*endptr) {
1584	p_err(fmt: "can't parse %s as IDX", *argv);
1585	goto err_free_reuse_maps;
1586	}
1587	name = NULL;
1588	} else if (is_prefix(pfx: *argv, str: "name")) {
1589	NEXT_ARG();
1590
1591	name = *argv;
1592	idx = -1;
1593	} else {
1594	p_err(fmt: "expected 'idx' or 'name', got: '%s'?",
1595	*argv);
1596	goto err_free_reuse_maps;
1597	}
1598	NEXT_ARG();
1599
1600	fd = map_parse_fd(argc: &argc, argv: &argv);
1601	if (fd < 0)
1602	goto err_free_reuse_maps;
1603
1604	new_map_replace = libbpf_reallocarray(map_replace,
1605	old_map_fds + 1,
1606	sizeof(*map_replace));
1607	if (!new_map_replace) {
1608	p_err(fmt: "mem alloc failed");
1609	goto err_free_reuse_maps;
1610	}
1611	map_replace = new_map_replace;
1612
1613	map_replace[old_map_fds].idx = idx;
1614	map_replace[old_map_fds].name = name;
1615	map_replace[old_map_fds].fd = fd;
1616	old_map_fds++;
1617	} else if (is_prefix(pfx: *argv, str: "dev")) {
1618	p_info(fmt: "Warning: 'bpftool prog load [...] dev <ifname>' syntax is deprecated.\n"
1619	"Going further, please use 'offload_dev <ifname>' to offload program to device.\n"
1620	"For applications using XDP hints only, use 'xdpmeta_dev <ifname>'.");
1621	goto offload_dev;
1622	} else if (is_prefix(pfx: *argv, str: "offload_dev")) {
1623	offload_dev:
1624	NEXT_ARG();
1625
1626	if (offload_ifindex) {
1627	p_err(fmt: "offload_dev already specified");
1628	goto err_free_reuse_maps;
1629	} else if (xdpmeta_ifindex) {
1630	p_err(fmt: "xdpmeta_dev and offload_dev are mutually exclusive");
1631	goto err_free_reuse_maps;
1632	}
1633	if (!REQ_ARGS(1))
1634	goto err_free_reuse_maps;
1635
1636	offload_ifindex = if_nametoindex(*argv);
1637	if (!offload_ifindex) {
1638	p_err("unrecognized netdevice '%s': %s",
1639	*argv, strerror(errno));
1640	goto err_free_reuse_maps;
1641	}
1642	NEXT_ARG();
1643	} else if (is_prefix(pfx: *argv, str: "xdpmeta_dev")) {
1644	NEXT_ARG();
1645
1646	if (xdpmeta_ifindex) {
1647	p_err(fmt: "xdpmeta_dev already specified");
1648	goto err_free_reuse_maps;
1649	} else if (offload_ifindex) {
1650	p_err(fmt: "xdpmeta_dev and offload_dev are mutually exclusive");
1651	goto err_free_reuse_maps;
1652	}
1653	if (!REQ_ARGS(1))
1654	goto err_free_reuse_maps;
1655
1656	xdpmeta_ifindex = if_nametoindex(*argv);
1657	if (!xdpmeta_ifindex) {
1658	p_err("unrecognized netdevice '%s': %s",
1659	*argv, strerror(errno));
1660	goto err_free_reuse_maps;
1661	}
1662	NEXT_ARG();
1663	} else if (is_prefix(pfx: *argv, str: "pinmaps")) {
1664	NEXT_ARG();
1665
1666	if (!REQ_ARGS(1))
1667	goto err_free_reuse_maps;
1668
1669	pinmaps = GET_ARG();
1670	} else if (is_prefix(pfx: *argv, str: "autoattach")) {
1671	auto_attach = true;
1672	NEXT_ARG();
1673	} else {
1674	p_err(fmt: "expected no more arguments, 'type', 'map' or 'dev', got: '%s'?",
1675	*argv);
1676	goto err_free_reuse_maps;
1677	}
1678	}
1679
1680	set_max_rlimit();
1681
1682	if (verifier_logs)
1683	/* log_level1 + log_level2 + stats, but not stable UAPI */
1684	open_opts.kernel_log_level = 1 + 2 + 4;
1685
1686	obj = bpf_object__open_file(file, &open_opts);
1687	if (!obj) {
1688	p_err(fmt: "failed to open object file");
1689	goto err_free_reuse_maps;
1690	}
1691
1692	bpf_object__for_each_program(pos, obj) {
1693	enum bpf_prog_type prog_type = common_prog_type;
1694
1695	if (prog_type == BPF_PROG_TYPE_UNSPEC) {
1696	const char *sec_name = bpf_program__section_name(pos);
1697
1698	err = get_prog_type_by_name(name: sec_name, prog_type: &prog_type,
1699	expected_attach_type: &expected_attach_type);
1700	if (err < 0)
1701	goto err_close_obj;
1702	}
1703
1704	if (prog_type == BPF_PROG_TYPE_XDP && xdpmeta_ifindex) {
1705	bpf_program__set_flags(pos, BPF_F_XDP_DEV_BOUND_ONLY);
1706	bpf_program__set_ifindex(pos, xdpmeta_ifindex);
1707	} else {
1708	bpf_program__set_ifindex(pos, offload_ifindex);
1709	}
1710	if (bpf_program__type(pos) != prog_type)
1711	bpf_program__set_type(pos, prog_type);
1712	bpf_program__set_expected_attach_type(pos, expected_attach_type);
1713	}
1714
1715	qsort(map_replace, old_map_fds, sizeof(*map_replace),
1716	map_replace_compar);
1717
1718	/* After the sort maps by name will be first on the list, because they
1719	* have idx == -1. Resolve them.
1720	*/
1721	j = 0;
1722	while (j < old_map_fds && map_replace[j].name) {
1723	i = 0;
1724	bpf_object__for_each_map(map, obj) {
1725	if (!strcmp(bpf_map__name(map), map_replace[j].name)) {
1726	map_replace[j].idx = i;
1727	break;
1728	}
1729	i++;
1730	}
1731	if (map_replace[j].idx == -1) {
1732	p_err(fmt: "unable to find map '%s'", map_replace[j].name);
1733	goto err_close_obj;
1734	}
1735	j++;
1736	}
1737	/* Resort if any names were resolved */
1738	if (j)
1739	qsort(map_replace, old_map_fds, sizeof(*map_replace),
1740	map_replace_compar);
1741
1742	/* Set ifindex and name reuse */
1743	j = 0;
1744	idx = 0;
1745	bpf_object__for_each_map(map, obj) {
1746	if (bpf_map__type(map) != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
1747	bpf_map__set_ifindex(map, offload_ifindex);
1748
1749	if (j < old_map_fds && idx == map_replace[j].idx) {
1750	err = bpf_map__reuse_fd(map, map_replace[j++].fd);
1751	if (err) {
1752	p_err(fmt: "unable to set up map reuse: %d", err);
1753	goto err_close_obj;
1754	}
1755
1756	/* Next reuse wants to apply to the same map */
1757	if (j < old_map_fds && map_replace[j].idx == idx) {
1758	p_err(fmt: "replacement for map idx %d specified more than once",
1759	idx);
1760	goto err_close_obj;
1761	}
1762	}
1763
1764	idx++;
1765	}
1766	if (j < old_map_fds) {
1767	p_err(fmt: "map idx '%d' not used", map_replace[j].idx);
1768	goto err_close_obj;
1769	}
1770
1771	err = bpf_object__load(obj);
1772	if (err) {
1773	p_err(fmt: "failed to load object file");
1774	goto err_close_obj;
1775	}
1776
1777	err = mount_bpffs_for_pin(name: pinfile, is_dir: !first_prog_only);
1778	if (err)
1779	goto err_close_obj;
1780
1781	if (first_prog_only) {
1782	prog = bpf_object__next_program(obj, NULL);
1783	if (!prog) {
1784	p_err(fmt: "object file doesn't contain any bpf program");
1785	goto err_close_obj;
1786	}
1787
1788	if (auto_attach)
1789	err = auto_attach_program(prog, path: pinfile);
1790	else
1791	err = bpf_obj_pin(bpf_program__fd(prog), pinfile);
1792	if (err) {
1793	p_err(fmt: "failed to pin program %s",
1794	bpf_program__section_name(prog));
1795	goto err_close_obj;
1796	}
1797	} else {
1798	if (auto_attach)
1799	err = auto_attach_programs(obj, path: pinfile);
1800	else
1801	err = bpf_object__pin_programs(obj, pinfile);
1802	if (err) {
1803	p_err(fmt: "failed to pin all programs");
1804	goto err_close_obj;
1805	}
1806	}
1807
1808	if (pinmaps) {
1809	err = bpf_object__pin_maps(obj, pinmaps);
1810	if (err) {
1811	p_err(fmt: "failed to pin all maps");
1812	goto err_unpin;
1813	}
1814	}
1815
1816	if (json_output)
1817	jsonw_null(self: json_wtr);
1818
1819	bpf_object__close(obj);
1820	for (i = 0; i < old_map_fds; i++)
1821	close(map_replace[i].fd);
1822	free(map_replace);
1823
1824	return 0;
1825
1826	err_unpin:
1827	if (first_prog_only)
1828	unlink(pinfile);
1829	else
1830	bpf_object__unpin_programs(obj, pinfile);
1831	err_close_obj:
1832	bpf_object__close(obj);
1833	err_free_reuse_maps:
1834	for (i = 0; i < old_map_fds; i++)
1835	close(map_replace[i].fd);
1836	free(map_replace);
1837	return -1;
1838	}
1839
1840	static int count_open_fds(void)
1841	{
1842	DIR *dp = opendir("/proc/self/fd");
1843	struct dirent *de;
1844	int cnt = -3;
1845
1846	if (!dp)
1847	return -1;
1848
1849	while ((de = readdir(dp)))
1850	cnt++;
1851
1852	closedir(dp);
1853	return cnt;
1854	}
1855
1856	static int try_loader(struct gen_loader_opts *gen)
1857	{
1858	struct bpf_load_and_run_opts opts = {};
1859	struct bpf_loader_ctx *ctx;
1860	int ctx_sz = sizeof(*ctx) + 64 * max(sizeof(struct bpf_map_desc),
1861	sizeof(struct bpf_prog_desc));
1862	int log_buf_sz = (1u << 24) - 1;
1863	int err, fds_before, fd_delta;
1864	char *log_buf = NULL;
1865
1866	ctx = alloca(ctx_sz);
1867	memset(ctx, 0, ctx_sz);
1868	ctx->sz = ctx_sz;
1869	if (verifier_logs) {
1870	ctx->log_level = 1 + 2 + 4;
1871	ctx->log_size = log_buf_sz;
1872	log_buf = malloc(log_buf_sz);
1873	if (!log_buf)
1874	return -ENOMEM;
1875	ctx->log_buf = (long) log_buf;
1876	}
1877	opts.ctx = ctx;
1878	opts.data = gen->data;
1879	opts.data_sz = gen->data_sz;
1880	opts.insns = gen->insns;
1881	opts.insns_sz = gen->insns_sz;
1882	fds_before = count_open_fds();
1883	err = bpf_load_and_run(&opts);
1884	fd_delta = count_open_fds() - fds_before;
1885	if (err < 0 || verifier_logs) {
1886	fprintf(stderr, "err %d\n%s\n%s", err, opts.errstr, log_buf);
1887	if (fd_delta && err < 0)
1888	fprintf(stderr, "loader prog leaked %d FDs\n",
1889	fd_delta);
1890	}
1891	free(log_buf);
1892	return err;
1893	}
1894
1895	static int do_loader(int argc, char **argv)
1896	{
1897	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts);
1898	DECLARE_LIBBPF_OPTS(gen_loader_opts, gen);
1899	struct bpf_object *obj;
1900	const char *file;
1901	int err = 0;
1902
1903	if (!REQ_ARGS(1))
1904	return -1;
1905	file = GET_ARG();
1906
1907	if (verifier_logs)
1908	/* log_level1 + log_level2 + stats, but not stable UAPI */
1909	open_opts.kernel_log_level = 1 + 2 + 4;
1910
1911	obj = bpf_object__open_file(file, &open_opts);
1912	if (!obj) {
1913	p_err(fmt: "failed to open object file");
1914	goto err_close_obj;
1915	}
1916
1917	err = bpf_object__gen_loader(obj, &gen);
1918	if (err)
1919	goto err_close_obj;
1920
1921	err = bpf_object__load(obj);
1922	if (err) {
1923	p_err(fmt: "failed to load object file");
1924	goto err_close_obj;
1925	}
1926
1927	if (verifier_logs) {
1928	struct dump_data dd = {};
1929
1930	kernel_syms_load(dd: &dd);
1931	dump_xlated_plain(&dd, (void *)gen.insns, gen.insns_sz, false, false);
1932	kernel_syms_destroy(dd: &dd);
1933	}
1934	err = try_loader(&gen);
1935	err_close_obj:
1936	bpf_object__close(obj);
1937	return err;
1938	}
1939
1940	static int do_load(int argc, char **argv)
1941	{
1942	if (use_loader)
1943	return do_loader(argc, argv);
1944	return load_with_options(argc, argv, first_prog_only: true);
1945	}
1946
1947	static int do_loadall(int argc, char **argv)
1948	{
1949	return load_with_options(argc, argv, first_prog_only: false);
1950	}
1951
1952	#ifdef BPFTOOL_WITHOUT_SKELETONS
1953
1954	static int do_profile(int argc, char **argv)
1955	{
1956	p_err("bpftool prog profile command is not supported. Please build bpftool with clang >= 10.0.0");
1957	return 0;
1958	}
1959
1960	#else /* BPFTOOL_WITHOUT_SKELETONS */
1961
1962	#include "profiler.skel.h"
1963
1964	struct profile_metric {
1965	const char *name;
1966	struct bpf_perf_event_value val;
1967	struct perf_event_attr attr;
1968	bool selected;
1969
1970	/* calculate ratios like instructions per cycle */
1971	const int ratio_metric; /* 0 for N/A, 1 for index 0 (cycles) */
1972	const char *ratio_desc;
1973	const float ratio_mul;
1974	} metrics[] = {
1975	{
1976	.name = "cycles",
1977	.attr = {
1978	.type = PERF_TYPE_HARDWARE,
1979	.config = PERF_COUNT_HW_CPU_CYCLES,
1980	.exclude_user = 1,
1981	},
1982	},
1983	{
1984	.name = "instructions",
1985	.attr = {
1986	.type = PERF_TYPE_HARDWARE,
1987	.config = PERF_COUNT_HW_INSTRUCTIONS,
1988	.exclude_user = 1,
1989	},
1990	.ratio_metric = 1,
1991	.ratio_desc = "insns per cycle",
1992	.ratio_mul = 1.0,
1993	},
1994	{
1995	.name = "l1d_loads",
1996	.attr = {
1997	.type = PERF_TYPE_HW_CACHE,
1998	.config =
1999	PERF_COUNT_HW_CACHE_L1D |
2000	(PERF_COUNT_HW_CACHE_OP_READ << 8) |
2001	(PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
2002	.exclude_user = 1,
2003	},
2004	},
2005	{
2006	.name = "llc_misses",
2007	.attr = {
2008	.type = PERF_TYPE_HW_CACHE,
2009	.config =
2010	PERF_COUNT_HW_CACHE_LL |
2011	(PERF_COUNT_HW_CACHE_OP_READ << 8) |
2012	(PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
2013	.exclude_user = 1
2014	},
2015	.ratio_metric = 2,
2016	.ratio_desc = "LLC misses per million insns",
2017	.ratio_mul = 1e6,
2018	},
2019	{
2020	.name = "itlb_misses",
2021	.attr = {
2022	.type = PERF_TYPE_HW_CACHE,
2023	.config =
2024	PERF_COUNT_HW_CACHE_ITLB |
2025	(PERF_COUNT_HW_CACHE_OP_READ << 8) |
2026	(PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
2027	.exclude_user = 1
2028	},
2029	.ratio_metric = 2,
2030	.ratio_desc = "itlb misses per million insns",
2031	.ratio_mul = 1e6,
2032	},
2033	{
2034	.name = "dtlb_misses",
2035	.attr = {
2036	.type = PERF_TYPE_HW_CACHE,
2037	.config =
2038	PERF_COUNT_HW_CACHE_DTLB |
2039	(PERF_COUNT_HW_CACHE_OP_READ << 8) |
2040	(PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
2041	.exclude_user = 1
2042	},
2043	.ratio_metric = 2,
2044	.ratio_desc = "dtlb misses per million insns",
2045	.ratio_mul = 1e6,
2046	},
2047	};
2048
2049	static __u64 profile_total_count;
2050
2051	#define MAX_NUM_PROFILE_METRICS 4
2052
2053	static int profile_parse_metrics(int argc, char **argv)
2054	{
2055	unsigned int metric_cnt;
2056	int selected_cnt = 0;
2057	unsigned int i;
2058
2059	metric_cnt = ARRAY_SIZE(metrics);
2060
2061	while (argc > 0) {
2062	for (i = 0; i < metric_cnt; i++) {
2063	if (is_prefix(pfx: argv[0], str: metrics[i].name)) {
2064	if (!metrics[i].selected)
2065	selected_cnt++;
2066	metrics[i].selected = true;
2067	break;
2068	}
2069	}
2070	if (i == metric_cnt) {
2071	p_err(fmt: "unknown metric %s", argv[0]);
2072	return -1;
2073	}
2074	NEXT_ARG();
2075	}
2076	if (selected_cnt > MAX_NUM_PROFILE_METRICS) {
2077	p_err(fmt: "too many (%d) metrics, please specify no more than %d metrics at at time",
2078	selected_cnt, MAX_NUM_PROFILE_METRICS);
2079	return -1;
2080	}
2081	return selected_cnt;
2082	}
2083
2084	static void profile_read_values(struct profiler_bpf *obj)
2085	{
2086	__u32 m, cpu, num_cpu = obj->rodata->num_cpu;
2087	int reading_map_fd, count_map_fd;
2088	__u64 counts[num_cpu];
2089	__u32 key = 0;
2090	int err;
2091
2092	reading_map_fd = bpf_map__fd(obj->maps.accum_readings);
2093	count_map_fd = bpf_map__fd(obj->maps.counts);
2094	if (reading_map_fd < 0 || count_map_fd < 0) {
2095	p_err(fmt: "failed to get fd for map");
2096	return;
2097	}
2098
2099	err = bpf_map_lookup_elem(count_map_fd, &key, counts);
2100	if (err) {
2101	p_err("failed to read count_map: %s", strerror(errno));
2102	return;
2103	}
2104
2105	profile_total_count = 0;
2106	for (cpu = 0; cpu < num_cpu; cpu++)
2107	profile_total_count += counts[cpu];
2108
2109	for (m = 0; m < ARRAY_SIZE(metrics); m++) {
2110	struct bpf_perf_event_value values[num_cpu];
2111
2112	if (!metrics[m].selected)
2113	continue;
2114
2115	err = bpf_map_lookup_elem(reading_map_fd, &key, values);
2116	if (err) {
2117	p_err("failed to read reading_map: %s",
2118	strerror(errno));
2119	return;
2120	}
2121	for (cpu = 0; cpu < num_cpu; cpu++) {
2122	metrics[m].val.counter += values[cpu].counter;
2123	metrics[m].val.enabled += values[cpu].enabled;
2124	metrics[m].val.running += values[cpu].running;
2125	}
2126	key++;
2127	}
2128	}
2129
2130	static void profile_print_readings_json(void)
2131	{
2132	__u32 m;
2133
2134	jsonw_start_array(self: json_wtr);
2135	for (m = 0; m < ARRAY_SIZE(metrics); m++) {
2136	if (!metrics[m].selected)
2137	continue;
2138	jsonw_start_object(self: json_wtr);
2139	jsonw_string_field(self: json_wtr, prop: "metric", val: metrics[m].name);
2140	jsonw_lluint_field(self: json_wtr, prop: "run_cnt", num: profile_total_count);
2141	jsonw_lluint_field(self: json_wtr, prop: "value", num: metrics[m].val.counter);
2142	jsonw_lluint_field(self: json_wtr, prop: "enabled", num: metrics[m].val.enabled);
2143	jsonw_lluint_field(self: json_wtr, prop: "running", num: metrics[m].val.running);
2144
2145	jsonw_end_object(self: json_wtr);
2146	}
2147	jsonw_end_array(self: json_wtr);
2148	}
2149
2150	static void profile_print_readings_plain(void)
2151	{
2152	__u32 m;
2153
2154	printf("\n%18llu %-20s\n", profile_total_count, "run_cnt");
2155	for (m = 0; m < ARRAY_SIZE(metrics); m++) {
2156	struct bpf_perf_event_value *val = &metrics[m].val;
2157	int r;
2158
2159	if (!metrics[m].selected)
2160	continue;
2161	printf("%18llu %-20s", val->counter, metrics[m].name);
2162
2163	r = metrics[m].ratio_metric - 1;
2164	if (r >= 0 && metrics[r].selected &&
2165	metrics[r].val.counter > 0) {
2166	printf("# %8.2f %-30s",
2167	val->counter * metrics[m].ratio_mul /
2168	metrics[r].val.counter,
2169	metrics[m].ratio_desc);
2170	} else {
2171	printf("%-41s", "");
2172	}
2173
2174	if (val->enabled > val->running)
2175	printf("(%4.2f%%)",
2176	val->running * 100.0 / val->enabled);
2177	printf("\n");
2178	}
2179	}
2180
2181	static void profile_print_readings(void)
2182	{
2183	if (json_output)
2184	profile_print_readings_json();
2185	else
2186	profile_print_readings_plain();
2187	}
2188
2189	static char *profile_target_name(int tgt_fd)
2190	{
2191	struct bpf_func_info func_info;
2192	struct bpf_prog_info info = {};
2193	__u32 info_len = sizeof(info);
2194	const struct btf_type *t;
2195	__u32 func_info_rec_size;
2196	struct btf *btf = NULL;
2197	char *name = NULL;
2198	int err;
2199
2200	err = bpf_prog_get_info_by_fd(tgt_fd, &info, &info_len);
2201	if (err) {
2202	p_err(fmt: "failed to get info for prog FD %d", tgt_fd);
2203	goto out;
2204	}
2205
2206	if (info.btf_id == 0) {
2207	p_err(fmt: "prog FD %d doesn't have valid btf", tgt_fd);
2208	goto out;
2209	}
2210
2211	func_info_rec_size = info.func_info_rec_size;
2212	if (info.nr_func_info == 0) {
2213	p_err(fmt: "found 0 func_info for prog FD %d", tgt_fd);
2214	goto out;
2215	}
2216
2217	memset(&info, 0, sizeof(info));
2218	info.nr_func_info = 1;
2219	info.func_info_rec_size = func_info_rec_size;
2220	info.func_info = ptr_to_u64(ptr: &func_info);
2221
2222	err = bpf_prog_get_info_by_fd(tgt_fd, &info, &info_len);
2223	if (err) {
2224	p_err(fmt: "failed to get func_info for prog FD %d", tgt_fd);
2225	goto out;
2226	}
2227
2228	btf = btf__load_from_kernel_by_id(info.btf_id);
2229	if (!btf) {
2230	p_err(fmt: "failed to load btf for prog FD %d", tgt_fd);
2231	goto out;
2232	}
2233
2234	t = btf__type_by_id(btf, func_info.type_id);
2235	if (!t) {
2236	p_err(fmt: "btf %d doesn't have type %d",
2237	info.btf_id, func_info.type_id);
2238	goto out;
2239	}
2240	name = strdup(btf__name_by_offset(btf, t->name_off));
2241	out:
2242	btf__free(btf);
2243	return name;
2244	}
2245
2246	static struct profiler_bpf *profile_obj;
2247	static int profile_tgt_fd = -1;
2248	static char *profile_tgt_name;
2249	static int *profile_perf_events;
2250	static int profile_perf_event_cnt;
2251
2252	static void profile_close_perf_events(struct profiler_bpf *obj)
2253	{
2254	int i;
2255
2256	for (i = profile_perf_event_cnt - 1; i >= 0; i--)
2257	close(profile_perf_events[i]);
2258
2259	free(profile_perf_events);
2260	profile_perf_event_cnt = 0;
2261	}
2262
2263	static int profile_open_perf_event(int mid, int cpu, int map_fd)
2264	{
2265	int pmu_fd;
2266
2267	pmu_fd = syscall(__NR_perf_event_open, &metrics[mid].attr,
2268	-1 /*pid*/, cpu, -1 /*group_fd*/, 0);
2269	if (pmu_fd < 0) {
2270	if (errno == ENODEV) {
2271	p_info(fmt: "cpu %d may be offline, skip %s profiling.",
2272	cpu, metrics[mid].name);
2273	profile_perf_event_cnt++;
2274	return 0;
2275	}
2276	return -1;
2277	}
2278
2279	if (bpf_map_update_elem(map_fd,
2280	&profile_perf_event_cnt,
2281	&pmu_fd, BPF_ANY) ||
2282	ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0)) {
2283	close(pmu_fd);
2284	return -1;
2285	}
2286
2287	profile_perf_events[profile_perf_event_cnt++] = pmu_fd;
2288	return 0;
2289	}
2290
2291	static int profile_open_perf_events(struct profiler_bpf *obj)
2292	{
2293	unsigned int cpu, m;
2294	int map_fd;
2295
2296	profile_perf_events = calloc(
2297	sizeof(int), obj->rodata->num_cpu * obj->rodata->num_metric);
2298	if (!profile_perf_events) {
2299	p_err("failed to allocate memory for perf_event array: %s",
2300	strerror(errno));
2301	return -1;
2302	}
2303	map_fd = bpf_map__fd(obj->maps.events);
2304	if (map_fd < 0) {
2305	p_err(fmt: "failed to get fd for events map");
2306	return -1;
2307	}
2308
2309	for (m = 0; m < ARRAY_SIZE(metrics); m++) {
2310	if (!metrics[m].selected)
2311	continue;
2312	for (cpu = 0; cpu < obj->rodata->num_cpu; cpu++) {
2313	if (profile_open_perf_event(mid: m, cpu, map_fd)) {
2314	p_err(fmt: "failed to create event %s on cpu %d",
2315	metrics[m].name, cpu);
2316	return -1;
2317	}
2318	}
2319	}
2320	return 0;
2321	}
2322
2323	static void profile_print_and_cleanup(void)
2324	{
2325	profile_close_perf_events(obj: profile_obj);
2326	profile_read_values(obj: profile_obj);
2327	profile_print_readings();
2328	profiler_bpf__destroy(profile_obj);
2329
2330	close(profile_tgt_fd);
2331	free(profile_tgt_name);
2332	}
2333
2334	static void int_exit(int signo)
2335	{
2336	profile_print_and_cleanup();
2337	exit(0);
2338	}
2339
2340	static int do_profile(int argc, char **argv)
2341	{
2342	int num_metric, num_cpu, err = -1;
2343	struct bpf_program *prog;
2344	unsigned long duration;
2345	char *endptr;
2346
2347	/* we at least need two args for the prog and one metric */
2348	if (!REQ_ARGS(3))
2349	return -EINVAL;
2350
2351	/* parse target fd */
2352	profile_tgt_fd = prog_parse_fd(argc: &argc, argv: &argv);
2353	if (profile_tgt_fd < 0) {
2354	p_err(fmt: "failed to parse fd");
2355	return -1;
2356	}
2357
2358	/* parse profiling optional duration */
2359	if (argc > 2 && is_prefix(pfx: argv[0], str: "duration")) {
2360	NEXT_ARG();
2361	duration = strtoul(*argv, &endptr, 0);
2362	if (*endptr)
2363	usage();
2364	NEXT_ARG();
2365	} else {
2366	duration = UINT_MAX;
2367	}
2368
2369	num_metric = profile_parse_metrics(argc, argv);
2370	if (num_metric <= 0)
2371	goto out;
2372
2373	num_cpu = libbpf_num_possible_cpus();
2374	if (num_cpu <= 0) {
2375	p_err(fmt: "failed to identify number of CPUs");
2376	goto out;
2377	}
2378
2379	profile_obj = profiler_bpf__open();
2380	if (!profile_obj) {
2381	p_err(fmt: "failed to open and/or load BPF object");
2382	goto out;
2383	}
2384
2385	profile_obj->rodata->num_cpu = num_cpu;
2386	profile_obj->rodata->num_metric = num_metric;
2387
2388	/* adjust map sizes */
2389	bpf_map__set_max_entries(profile_obj->maps.events, num_metric * num_cpu);
2390	bpf_map__set_max_entries(profile_obj->maps.fentry_readings, num_metric);
2391	bpf_map__set_max_entries(profile_obj->maps.accum_readings, num_metric);
2392	bpf_map__set_max_entries(profile_obj->maps.counts, 1);
2393
2394	/* change target name */
2395	profile_tgt_name = profile_target_name(tgt_fd: profile_tgt_fd);
2396	if (!profile_tgt_name)
2397	goto out;
2398
2399	bpf_object__for_each_program(prog, profile_obj->obj) {
2400	err = bpf_program__set_attach_target(prog, profile_tgt_fd,
2401	profile_tgt_name);
2402	if (err) {
2403	p_err(fmt: "failed to set attach target\n");
2404	goto out;
2405	}
2406	}
2407
2408	set_max_rlimit();
2409	err = profiler_bpf__load(profile_obj);
2410	if (err) {
2411	p_err(fmt: "failed to load profile_obj");
2412	goto out;
2413	}
2414
2415	err = profile_open_perf_events(obj: profile_obj);
2416	if (err)
2417	goto out;
2418
2419	err = profiler_bpf__attach(profile_obj);
2420	if (err) {
2421	p_err(fmt: "failed to attach profile_obj");
2422	goto out;
2423	}
2424	signal(SIGINT, int_exit);
2425
2426	sleep(duration);
2427	profile_print_and_cleanup();
2428	return 0;
2429
2430	out:
2431	profile_close_perf_events(obj: profile_obj);
2432	if (profile_obj)
2433	profiler_bpf__destroy(profile_obj);
2434	close(profile_tgt_fd);
2435	free(profile_tgt_name);
2436	return err;
2437	}
2438
2439	#endif /* BPFTOOL_WITHOUT_SKELETONS */
2440
2441	static int do_help(int argc, char **argv)
2442	{
2443	if (json_output) {
2444	jsonw_null(self: json_wtr);
2445	return 0;
2446	}
2447
2448	fprintf(stderr,
2449	"Usage: %1$s %2$s { show | list } [PROG]\n"
2450	" %1$s %2$s dump xlated PROG [{ file FILE | [opcodes] [linum] [visual] }]\n"
2451	" %1$s %2$s dump jited PROG [{ file FILE | [opcodes] [linum] }]\n"
2452	" %1$s %2$s pin PROG FILE\n"
2453	" %1$s %2$s { load | loadall } OBJ PATH \\\n"
2454	" [type TYPE] [{ offload_dev | xdpmeta_dev } NAME] \\\n"
2455	" [map { idx IDX | name NAME } MAP]\\\n"
2456	" [pinmaps MAP_DIR]\n"
2457	" [autoattach]\n"
2458	" %1$s %2$s attach PROG ATTACH_TYPE [MAP]\n"
2459	" %1$s %2$s detach PROG ATTACH_TYPE [MAP]\n"
2460	" %1$s %2$s run PROG \\\n"
2461	" data_in FILE \\\n"
2462	" [data_out FILE [data_size_out L]] \\\n"
2463	" [ctx_in FILE [ctx_out FILE [ctx_size_out M]]] \\\n"
2464	" [repeat N]\n"
2465	" %1$s %2$s profile PROG [duration DURATION] METRICs\n"
2466	" %1$s %2$s tracelog\n"
2467	" %1$s %2$s help\n"
2468	"\n"
2469	" " HELP_SPEC_MAP "\n"
2470	" " HELP_SPEC_PROGRAM "\n"
2471	" TYPE := { socket | kprobe | kretprobe | classifier | action |\n"
2472	" tracepoint | raw_tracepoint | xdp | perf_event | cgroup/skb |\n"
2473	" cgroup/sock | cgroup/dev | lwt_in | lwt_out | lwt_xmit |\n"
2474	" lwt_seg6local | sockops | sk_skb | sk_msg | lirc_mode2 |\n"
2475	" sk_reuseport | flow_dissector | cgroup/sysctl |\n"
2476	" cgroup/bind4 | cgroup/bind6 | cgroup/post_bind4 |\n"
2477	" cgroup/post_bind6 | cgroup/connect4 | cgroup/connect6 |\n"
2478	" cgroup/connect_unix | cgroup/getpeername4 | cgroup/getpeername6 |\n"
2479	" cgroup/getpeername_unix | cgroup/getsockname4 | cgroup/getsockname6 |\n"
2480	" cgroup/getsockname_unix | cgroup/sendmsg4 | cgroup/sendmsg6 |\n"
2481	" cgroup/sendmsg°unix | cgroup/recvmsg4 | cgroup/recvmsg6 | cgroup/recvmsg_unix |\n"
2482	" cgroup/getsockopt | cgroup/setsockopt | cgroup/sock_release |\n"
2483	" struct_ops | fentry | fexit | freplace | sk_lookup }\n"
2484	" ATTACH_TYPE := { sk_msg_verdict | sk_skb_verdict | sk_skb_stream_verdict |\n"
2485	" sk_skb_stream_parser | flow_dissector }\n"
2486	" METRIC := { cycles | instructions | l1d_loads | llc_misses | itlb_misses | dtlb_misses }\n"
2487	" " HELP_SPEC_OPTIONS " |\n"
2488	" {-f|--bpffs} | {-m|--mapcompat} | {-n|--nomount} |\n"
2489	" {-L|--use-loader} }\n"
2490	"",
2491	bin_name, argv[-2]);
2492
2493	return 0;
2494	}
2495
2496	static const struct cmd cmds[] = {
2497	{ "show", do_show },
2498	{ "list", do_show },
2499	{ "help", do_help },
2500	{ "dump", do_dump },
2501	{ "pin", do_pin },
2502	{ "load", do_load },
2503	{ "loadall", do_loadall },
2504	{ "attach", do_attach },
2505	{ "detach", do_detach },
2506	{ "tracelog", do_tracelog },
2507	{ "run", do_run },
2508	{ "profile", do_profile },
2509	{ 0 }
2510	};
2511
2512	int do_prog(int argc, char **argv)
2513	{
2514	return cmd_select(cmds, argc, argv, help: do_help);
2515	}
2516