1	// SPDX-License-Identifier: GPL-2.0-only
2	#define _GNU_SOURCE
3
4	#include <arpa/inet.h>
5	#include <bpf/bpf.h>
6	#include <bpf/libbpf.h>
7	#include <errno.h>
8	#include <fcntl.h>
9	#include <getopt.h>
10	#include <linux/ethtool.h>
11	#include <linux/hashtable.h>
12	#include <linux/if_link.h>
13	#include <linux/jhash.h>
14	#include <linux/limits.h>
15	#include <linux/list.h>
16	#include <linux/sockios.h>
17	#include <locale.h>
18	#include <math.h>
19	#include <net/if.h>
20	#include <poll.h>
21	#include <signal.h>
22	#include <stdbool.h>
23	#include <stdio.h>
24	#include <stdlib.h>
25	#include <string.h>
26	#include <sys/ioctl.h>
27	#include <sys/mman.h>
28	#include <sys/signalfd.h>
29	#include <sys/sysinfo.h>
30	#include <sys/timerfd.h>
31	#include <sys/utsname.h>
32	#include <time.h>
33	#include <unistd.h>
34
35	#include "bpf_util.h"
36	#include "xdp_sample_user.h"
37
38	#define __sample_print(fmt, cond, ...) \
39	({ \
40	if (cond) \
41	printf(fmt, ##__VA_ARGS__); \
42	})
43
44	#define print_always(fmt, ...) __sample_print(fmt, 1, ##__VA_ARGS__)
45	#define print_default(fmt, ...) \
46	__sample_print(fmt, sample_log_level & LL_DEFAULT, ##__VA_ARGS__)
47	#define __print_err(err, fmt, ...) \
48	({ \
49	__sample_print(fmt, err > 0 || sample_log_level & LL_DEFAULT, \
50	##__VA_ARGS__); \
51	sample_err_exp = sample_err_exp ? true : err > 0; \
52	})
53	#define print_err(err, fmt, ...) __print_err(err, fmt, ##__VA_ARGS__)
54
55	#define __COLUMN(x) "%'10" x " %-13s"
56	#define FMT_COLUMNf __COLUMN(".0f")
57	#define FMT_COLUMNd __COLUMN("d")
58	#define FMT_COLUMNl __COLUMN("llu")
59	#define RX(rx) rx, "rx/s"
60	#define PPS(pps) pps, "pkt/s"
61	#define DROP(drop) drop, "drop/s"
62	#define ERR(err) err, "error/s"
63	#define HITS(hits) hits, "hit/s"
64	#define XMIT(xmit) xmit, "xmit/s"
65	#define PASS(pass) pass, "pass/s"
66	#define REDIR(redir) redir, "redir/s"
67	#define NANOSEC_PER_SEC 1000000000 /* 10^9 */
68
69	#define XDP_UNKNOWN (XDP_REDIRECT + 1)
70	#define XDP_ACTION_MAX (XDP_UNKNOWN + 1)
71	#define XDP_REDIRECT_ERR_MAX 7
72
73	enum map_type {
74	MAP_RX,
75	MAP_REDIRECT_ERR,
76	MAP_CPUMAP_ENQUEUE,
77	MAP_CPUMAP_KTHREAD,
78	MAP_EXCEPTION,
79	MAP_DEVMAP_XMIT,
80	MAP_DEVMAP_XMIT_MULTI,
81	NUM_MAP,
82	};
83
84	enum log_level {
85	LL_DEFAULT = 1U << 0,
86	LL_SIMPLE = 1U << 1,
87	LL_DEBUG = 1U << 2,
88	};
89
90	struct record {
91	__u64 timestamp;
92	struct datarec total;
93	struct datarec *cpu;
94	};
95
96	struct map_entry {
97	struct hlist_node node;
98	__u64 pair;
99	struct record val;
100	};
101
102	struct stats_record {
103	struct record rx_cnt;
104	struct record redir_err[XDP_REDIRECT_ERR_MAX];
105	struct record kthread;
106	struct record exception[XDP_ACTION_MAX];
107	struct record devmap_xmit;
108	DECLARE_HASHTABLE(xmit_map, 5);
109	struct record enq[];
110	};
111
112	struct sample_output {
113	struct {
114	__u64 rx;
115	__u64 redir;
116	__u64 drop;
117	__u64 drop_xmit;
118	__u64 err;
119	__u64 xmit;
120	} totals;
121	struct {
122	union {
123	__u64 pps;
124	__u64 num;
125	};
126	__u64 drop;
127	__u64 err;
128	} rx_cnt;
129	struct {
130	__u64 suc;
131	__u64 err;
132	} redir_cnt;
133	struct {
134	__u64 hits;
135	} except_cnt;
136	struct {
137	__u64 pps;
138	__u64 drop;
139	__u64 err;
140	double bavg;
141	} xmit_cnt;
142	};
143
144	struct xdp_desc {
145	int ifindex;
146	__u32 prog_id;
147	int flags;
148	} sample_xdp_progs[32];
149
150	struct datarec *sample_mmap[NUM_MAP];
151	struct bpf_map *sample_map[NUM_MAP];
152	size_t sample_map_count[NUM_MAP];
153	enum log_level sample_log_level;
154	struct sample_output sample_out;
155	unsigned long sample_interval;
156	bool sample_err_exp;
157	int sample_xdp_cnt;
158	int sample_n_cpus;
159	int sample_sig_fd;
160	int sample_mask;
161
162	static const char *xdp_redirect_err_names[XDP_REDIRECT_ERR_MAX] = {
163	/* Key=1 keeps unknown errors */
164	"Success",
165	"Unknown",
166	"EINVAL",
167	"ENETDOWN",
168	"EMSGSIZE",
169	"EOPNOTSUPP",
170	"ENOSPC",
171	};
172
173	/* Keyed from Unknown */
174	static const char *xdp_redirect_err_help[XDP_REDIRECT_ERR_MAX - 1] = {
175	"Unknown error",
176	"Invalid redirection",
177	"Device being redirected to is down",
178	"Packet length too large for device",
179	"Operation not supported",
180	"No space in ptr_ring of cpumap kthread",
181	};
182
183	static const char *xdp_action_names[XDP_ACTION_MAX] = {
184	[XDP_ABORTED] = "XDP_ABORTED",
185	[XDP_DROP] = "XDP_DROP",
186	[XDP_PASS] = "XDP_PASS",
187	[XDP_TX] = "XDP_TX",
188	[XDP_REDIRECT] = "XDP_REDIRECT",
189	[XDP_UNKNOWN] = "XDP_UNKNOWN",
190	};
191
192	static __u64 gettime(void)
193	{
194	struct timespec t;
195	int res;
196
197	res = clock_gettime(CLOCK_MONOTONIC, tp: &t);
198	if (res < 0) {
199	fprintf(stderr, format: "Error with gettimeofday! (%i)\n", res);
200	return UINT64_MAX;
201	}
202	return (__u64)t.tv_sec * NANOSEC_PER_SEC + t.tv_nsec;
203	}
204
205	static const char *action2str(int action)
206	{
207	if (action < XDP_ACTION_MAX)
208	return xdp_action_names[action];
209	return NULL;
210	}
211
212	static void sample_print_help(int mask)
213	{
214	printf(format: "Output format description\n\n"
215	"By default, redirect success statistics are disabled, use -s to enable.\n"
216	"The terse output mode is default, verbose mode can be activated using -v\n"
217	"Use SIGQUIT (Ctrl + \\) to switch the mode dynamically at runtime\n\n"
218	"Terse mode displays at most the following fields:\n"
219	" rx/s Number of packets received per second\n"
220	" redir/s Number of packets successfully redirected per second\n"
221	" err,drop/s Aggregated count of errors per second (including dropped packets)\n"
222	" xmit/s Number of packets transmitted on the output device per second\n\n"
223	"Output description for verbose mode:\n"
224	" FIELD DESCRIPTION\n");
225
226	if (mask & SAMPLE_RX_CNT) {
227	printf(format: " receive\t\tDisplays the number of packets received & errors encountered\n"
228	" \t\t\tWhenever an error or packet drop occurs, details of per CPU error\n"
229	" \t\t\tand drop statistics will be expanded inline in terse mode.\n"
230	" \t\t\t\tpkt/s - Packets received per second\n"
231	" \t\t\t\tdrop/s - Packets dropped per second\n"
232	" \t\t\t\terror/s - Errors encountered per second\n\n");
233	}
234	if (mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) {
235	printf(format: " redirect\t\tDisplays the number of packets successfully redirected\n"
236	" \t\t\tErrors encountered are expanded under redirect_err field\n"
237	" \t\t\tNote that passing -s to enable it has a per packet overhead\n"
238	" \t\t\t\tredir/s - Packets redirected successfully per second\n\n"
239	" redirect_err\t\tDisplays the number of packets that failed redirection\n"
240	" \t\t\tThe errno is expanded under this field with per CPU count\n"
241	" \t\t\tThe recognized errors are:\n");
242
243	for (int i = 2; i < XDP_REDIRECT_ERR_MAX; i++)
244	printf(format: "\t\t\t %s: %s\n", xdp_redirect_err_names[i],
245	xdp_redirect_err_help[i - 1]);
246
247	printf(format: " \n\t\t\t\terror/s - Packets that failed redirection per second\n\n");
248	}
249
250	if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT) {
251	printf(format: " enqueue to cpu N\tDisplays the number of packets enqueued to bulk queue of CPU N\n"
252	" \t\t\tExpands to cpu:FROM->N to display enqueue stats for each CPU enqueuing to CPU N\n"
253	" \t\t\tReceived packets can be associated with the CPU redirect program is enqueuing \n"
254	" \t\t\tpackets to.\n"
255	" \t\t\t\tpkt/s - Packets enqueued per second from other CPU to CPU N\n"
256	" \t\t\t\tdrop/s - Packets dropped when trying to enqueue to CPU N\n"
257	" \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n");
258	}
259
260	if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
261	printf(format: " kthread\t\tDisplays the number of packets processed in CPUMAP kthread for each CPU\n"
262	" \t\t\tPackets consumed from ptr_ring in kthread, and its xdp_stats (after calling \n"
263	" \t\t\tCPUMAP bpf prog) are expanded below this. xdp_stats are expanded as a total and\n"
264	" \t\t\tthen per-CPU to associate it to each CPU's pinned CPUMAP kthread.\n"
265	" \t\t\t\tpkt/s - Packets consumed per second from ptr_ring\n"
266	" \t\t\t\tdrop/s - Packets dropped per second in kthread\n"
267	" \t\t\t\tsched - Number of times kthread called schedule()\n\n"
268	" \t\t\txdp_stats (also expands to per-CPU counts)\n"
269	" \t\t\t\tpass/s - XDP_PASS count for CPUMAP program execution\n"
270	" \t\t\t\tdrop/s - XDP_DROP count for CPUMAP program execution\n"
271	" \t\t\t\tredir/s - XDP_REDIRECT count for CPUMAP program execution\n\n");
272	}
273
274	if (mask & SAMPLE_EXCEPTION_CNT) {
275	printf(format: " xdp_exception\t\tDisplays xdp_exception tracepoint events\n"
276	" \t\t\tThis can occur due to internal driver errors, unrecognized\n"
277	" \t\t\tXDP actions and due to explicit user trigger by use of XDP_ABORTED\n"
278	" \t\t\tEach action is expanded below this field with its count\n"
279	" \t\t\t\thit/s - Number of times the tracepoint was hit per second\n\n");
280	}
281
282	if (mask & SAMPLE_DEVMAP_XMIT_CNT) {
283	printf(format: " devmap_xmit\t\tDisplays devmap_xmit tracepoint events\n"
284	" \t\t\tThis tracepoint is invoked for successful transmissions on output\n"
285	" \t\t\tdevice but these statistics are not available for generic XDP mode,\n"
286	" \t\t\thence they will be omitted from the output when using SKB mode\n"
287	" \t\t\t\txmit/s - Number of packets that were transmitted per second\n"
288	" \t\t\t\tdrop/s - Number of packets that failed transmissions per second\n"
289	" \t\t\t\tdrv_err/s - Number of internal driver errors per second\n"
290	" \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n");
291	}
292	}
293
294	void sample_usage(char *argv[], const struct option *long_options,
295	const char *doc, int mask, bool error)
296	{
297	int i;
298
299	if (!error)
300	sample_print_help(mask);
301
302	printf(format: "\n%s\nOption for %s:\n", doc, argv[0]);
303	for (i = 0; long_options[i].name != 0; i++) {
304	printf(format: " --%-15s", long_options[i].name);
305	if (long_options[i].flag != NULL)
306	printf(format: " flag (internal value: %d)",
307	*long_options[i].flag);
308	else
309	printf(format: "\t short-option: -%c", long_options[i].val);
310	printf(format: "\n");
311	}
312	printf(format: "\n");
313	}
314
315	static struct datarec *alloc_record_per_cpu(void)
316	{
317	unsigned int nr_cpus = libbpf_num_possible_cpus();
318	struct datarec *array;
319
320	array = calloc(nmemb: nr_cpus, size: sizeof(*array));
321	if (!array) {
322	fprintf(stderr, format: "Failed to allocate memory (nr_cpus: %u)\n",
323	nr_cpus);
324	return NULL;
325	}
326	return array;
327	}
328
329	static int map_entry_init(struct map_entry *e, __u64 pair)
330	{
331	e->pair = pair;
332	INIT_HLIST_NODE(&e->node);
333	e->val.timestamp = gettime();
334	e->val.cpu = alloc_record_per_cpu();
335	if (!e->val.cpu)
336	return -ENOMEM;
337	return 0;
338	}
339
340	static void map_collect_percpu(struct datarec *values, struct record *rec)
341	{
342	/* For percpu maps, userspace gets a value per possible CPU */
343	unsigned int nr_cpus = libbpf_num_possible_cpus();
344	__u64 sum_xdp_redirect = 0;
345	__u64 sum_processed = 0;
346	__u64 sum_xdp_pass = 0;
347	__u64 sum_xdp_drop = 0;
348	__u64 sum_dropped = 0;
349	__u64 sum_issue = 0;
350	int i;
351
352	/* Get time as close as possible to reading map contents */
353	rec->timestamp = gettime();
354
355	/* Record and sum values from each CPU */
356	for (i = 0; i < nr_cpus; i++) {
357	rec->cpu[i].processed = READ_ONCE(values[i].processed);
358	rec->cpu[i].dropped = READ_ONCE(values[i].dropped);
359	rec->cpu[i].issue = READ_ONCE(values[i].issue);
360	rec->cpu[i].xdp_pass = READ_ONCE(values[i].xdp_pass);
361	rec->cpu[i].xdp_drop = READ_ONCE(values[i].xdp_drop);
362	rec->cpu[i].xdp_redirect = READ_ONCE(values[i].xdp_redirect);
363
364	sum_processed += rec->cpu[i].processed;
365	sum_dropped += rec->cpu[i].dropped;
366	sum_issue += rec->cpu[i].issue;
367	sum_xdp_pass += rec->cpu[i].xdp_pass;
368	sum_xdp_drop += rec->cpu[i].xdp_drop;
369	sum_xdp_redirect += rec->cpu[i].xdp_redirect;
370	}
371
372	rec->total.processed = sum_processed;
373	rec->total.dropped = sum_dropped;
374	rec->total.issue = sum_issue;
375	rec->total.xdp_pass = sum_xdp_pass;
376	rec->total.xdp_drop = sum_xdp_drop;
377	rec->total.xdp_redirect = sum_xdp_redirect;
378	}
379
380	static int map_collect_percpu_devmap(int map_fd, struct stats_record *rec)
381	{
382	unsigned int nr_cpus = bpf_num_possible_cpus();
383	__u32 batch, count = 32;
384	struct datarec *values;
385	bool init = false;
386	__u64 *keys;
387	int i, ret;
388
389	keys = calloc(nmemb: count, size: sizeof(__u64));
390	if (!keys)
391	return -ENOMEM;
392	values = calloc(nmemb: count * nr_cpus, size: sizeof(struct datarec));
393	if (!values) {
394	free(ptr: keys);
395	return -ENOMEM;
396	}
397
398	for (;;) {
399	bool exit = false;
400
401	ret = bpf_map_lookup_batch(map_fd, init ? &batch : NULL, &batch,
402	keys, values, &count, NULL);
403	if (ret < 0 && errno != ENOENT)
404	break;
405	if (errno == ENOENT)
406	exit = true;
407
408	init = true;
409	for (i = 0; i < count; i++) {
410	struct map_entry *e, *x = NULL;
411	__u64 pair = keys[i];
412	struct datarec *arr;
413
414	arr = &values[i * nr_cpus];
415	hash_for_each_possible(rec->xmit_map, e, node, pair) {
416	if (e->pair == pair) {
417	x = e;
418	break;
419	}
420	}
421	if (!x) {
422	x = calloc(nmemb: 1, size: sizeof(*x));
423	if (!x)
424	goto cleanup;
425	if (map_entry_init(e: x, pair) < 0) {
426	free(ptr: x);
427	goto cleanup;
428	}
429	hash_add(rec->xmit_map, &x->node, pair);
430	}
431	map_collect_percpu(values: arr, rec: &x->val);
432	}
433
434	if (exit)
435	break;
436	count = 32;
437	}
438
439	free(ptr: values);
440	free(ptr: keys);
441	return 0;
442	cleanup:
443	free(ptr: values);
444	free(ptr: keys);
445	return -ENOMEM;
446	}
447
448	static struct stats_record *alloc_stats_record(void)
449	{
450	struct stats_record *rec;
451	int i;
452
453	rec = calloc(nmemb: 1, size: sizeof(*rec) + sample_n_cpus * sizeof(struct record));
454	if (!rec) {
455	fprintf(stderr, format: "Failed to allocate memory\n");
456	return NULL;
457	}
458
459	if (sample_mask & SAMPLE_RX_CNT) {
460	rec->rx_cnt.cpu = alloc_record_per_cpu();
461	if (!rec->rx_cnt.cpu) {
462	fprintf(stderr,
463	format: "Failed to allocate rx_cnt per-CPU array\n");
464	goto end_rec;
465	}
466	}
467	if (sample_mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) {
468	for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) {
469	rec->redir_err[i].cpu = alloc_record_per_cpu();
470	if (!rec->redir_err[i].cpu) {
471	fprintf(stderr,
472	format: "Failed to allocate redir_err per-CPU array for "
473	"\"%s\" case\n",
474	xdp_redirect_err_names[i]);
475	while (i--)
476	free(ptr: rec->redir_err[i].cpu);
477	goto end_rx_cnt;
478	}
479	}
480	}
481	if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
482	rec->kthread.cpu = alloc_record_per_cpu();
483	if (!rec->kthread.cpu) {
484	fprintf(stderr,
485	format: "Failed to allocate kthread per-CPU array\n");
486	goto end_redir;
487	}
488	}
489	if (sample_mask & SAMPLE_EXCEPTION_CNT) {
490	for (i = 0; i < XDP_ACTION_MAX; i++) {
491	rec->exception[i].cpu = alloc_record_per_cpu();
492	if (!rec->exception[i].cpu) {
493	fprintf(stderr,
494	"Failed to allocate exception per-CPU array for "
495	"\"%s\" case\n",
496	action2str(i));
497	while (i--)
498	free(rec->exception[i].cpu);
499	goto end_kthread;
500	}
501	}
502	}
503	if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT) {
504	rec->devmap_xmit.cpu = alloc_record_per_cpu();
505	if (!rec->devmap_xmit.cpu) {
506	fprintf(stderr,
507	format: "Failed to allocate devmap_xmit per-CPU array\n");
508	goto end_exception;
509	}
510	}
511	if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
512	hash_init(rec->xmit_map);
513	if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT) {
514	for (i = 0; i < sample_n_cpus; i++) {
515	rec->enq[i].cpu = alloc_record_per_cpu();
516	if (!rec->enq[i].cpu) {
517	fprintf(stderr,
518	format: "Failed to allocate enqueue per-CPU array for "
519	"CPU %d\n",
520	i);
521	while (i--)
522	free(ptr: rec->enq[i].cpu);
523	goto end_devmap_xmit;
524	}
525	}
526	}
527
528	return rec;
529
530	end_devmap_xmit:
531	free(ptr: rec->devmap_xmit.cpu);
532	end_exception:
533	for (i = 0; i < XDP_ACTION_MAX; i++)
534	free(rec->exception[i].cpu);
535	end_kthread:
536	free(ptr: rec->kthread.cpu);
537	end_redir:
538	for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++)
539	free(ptr: rec->redir_err[i].cpu);
540	end_rx_cnt:
541	free(ptr: rec->rx_cnt.cpu);
542	end_rec:
543	free(ptr: rec);
544	return NULL;
545	}
546
547	static void free_stats_record(struct stats_record *r)
548	{
549	struct hlist_node *tmp;
550	struct map_entry *e;
551	int i;
552
553	for (i = 0; i < sample_n_cpus; i++)
554	free(ptr: r->enq[i].cpu);
555	hash_for_each_safe(r->xmit_map, i, tmp, e, node) {
556	hash_del(&e->node);
557	free(ptr: e->val.cpu);
558	free(ptr: e);
559	}
560	free(ptr: r->devmap_xmit.cpu);
561	for (i = 0; i < XDP_ACTION_MAX; i++)
562	free(r->exception[i].cpu);
563	free(ptr: r->kthread.cpu);
564	for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++)
565	free(ptr: r->redir_err[i].cpu);
566	free(ptr: r->rx_cnt.cpu);
567	free(ptr: r);
568	}
569
570	static double calc_period(struct record *r, struct record *p)
571	{
572	double period_ = 0;
573	__u64 period = 0;
574
575	period = r->timestamp - p->timestamp;
576	if (period > 0)
577	period_ = ((double)period / NANOSEC_PER_SEC);
578
579	return period_;
580	}
581
582	static double sample_round(double val)
583	{
584	if (val - floor(x: val) < 0.5)
585	return floor(x: val);
586	return ceil(x: val);
587	}
588
589	static __u64 calc_pps(struct datarec *r, struct datarec *p, double period_)
590	{
591	__u64 packets = 0;
592	__u64 pps = 0;
593
594	if (period_ > 0) {
595	packets = r->processed - p->processed;
596	pps = sample_round(val: packets / period_);
597	}
598	return pps;
599	}
600
601	static __u64 calc_drop_pps(struct datarec *r, struct datarec *p, double period_)
602	{
603	__u64 packets = 0;
604	__u64 pps = 0;
605
606	if (period_ > 0) {
607	packets = r->dropped - p->dropped;
608	pps = sample_round(val: packets / period_);
609	}
610	return pps;
611	}
612
613	static __u64 calc_errs_pps(struct datarec *r, struct datarec *p, double period_)
614	{
615	__u64 packets = 0;
616	__u64 pps = 0;
617
618	if (period_ > 0) {
619	packets = r->issue - p->issue;
620	pps = sample_round(val: packets / period_);
621	}
622	return pps;
623	}
624
625	static __u64 calc_info_pps(struct datarec *r, struct datarec *p, double period_)
626	{
627	__u64 packets = 0;
628	__u64 pps = 0;
629
630	if (period_ > 0) {
631	packets = r->info - p->info;
632	pps = sample_round(val: packets / period_);
633	}
634	return pps;
635	}
636
637	static void calc_xdp_pps(struct datarec *r, struct datarec *p, double *xdp_pass,
638	double *xdp_drop, double *xdp_redirect, double period_)
639	{
640	*xdp_pass = 0, *xdp_drop = 0, *xdp_redirect = 0;
641	if (period_ > 0) {
642	*xdp_redirect = (r->xdp_redirect - p->xdp_redirect) / period_;
643	*xdp_pass = (r->xdp_pass - p->xdp_pass) / period_;
644	*xdp_drop = (r->xdp_drop - p->xdp_drop) / period_;
645	}
646	}
647
648	static void stats_get_rx_cnt(struct stats_record *stats_rec,
649	struct stats_record *stats_prev,
650	unsigned int nr_cpus, struct sample_output *out)
651	{
652	struct record *rec, *prev;
653	double t, pps, drop, err;
654	int i;
655
656	rec = &stats_rec->rx_cnt;
657	prev = &stats_prev->rx_cnt;
658	t = calc_period(r: rec, p: prev);
659
660	for (i = 0; i < nr_cpus; i++) {
661	struct datarec *r = &rec->cpu[i];
662	struct datarec *p = &prev->cpu[i];
663	char str[64];
664
665	pps = calc_pps(r, p, period_: t);
666	drop = calc_drop_pps(r, p, period_: t);
667	err = calc_errs_pps(r, p, period_: t);
668	if (!pps && !drop && !err)
669	continue;
670
671	snprintf(s: str, maxlen: sizeof(str), format: "cpu:%d", i);
672	print_default(" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
673	"\n",
674	str, PPS(pps), DROP(drop), ERR(err));
675	}
676
677	if (out) {
678	pps = calc_pps(r: &rec->total, p: &prev->total, period_: t);
679	drop = calc_drop_pps(r: &rec->total, p: &prev->total, period_: t);
680	err = calc_errs_pps(r: &rec->total, p: &prev->total, period_: t);
681
682	out->rx_cnt.pps = pps;
683	out->rx_cnt.drop = drop;
684	out->rx_cnt.err = err;
685	out->totals.rx += pps;
686	out->totals.drop += drop;
687	out->totals.err += err;
688	}
689	}
690
691	static void stats_get_cpumap_enqueue(struct stats_record *stats_rec,
692	struct stats_record *stats_prev,
693	unsigned int nr_cpus)
694	{
695	struct record *rec, *prev;
696	double t, pps, drop, err;
697	int i, to_cpu;
698
699	/* cpumap enqueue stats */
700	for (to_cpu = 0; to_cpu < sample_n_cpus; to_cpu++) {
701	rec = &stats_rec->enq[to_cpu];
702	prev = &stats_prev->enq[to_cpu];
703	t = calc_period(r: rec, p: prev);
704
705	pps = calc_pps(r: &rec->total, p: &prev->total, period_: t);
706	drop = calc_drop_pps(r: &rec->total, p: &prev->total, period_: t);
707	err = calc_errs_pps(r: &rec->total, p: &prev->total, period_: t);
708
709	if (pps > 0 || drop > 0) {
710	char str[64];
711
712	snprintf(s: str, maxlen: sizeof(str), format: "enqueue to cpu %d", to_cpu);
713
714	if (err > 0)
715	err = pps / err; /* calc average bulk size */
716
717	print_err(drop,
718	" %-20s " FMT_COLUMNf FMT_COLUMNf __COLUMN(
719	".2f") "\n",
720	str, PPS(pps), DROP(drop), err, "bulk-avg");
721	}
722
723	for (i = 0; i < nr_cpus; i++) {
724	struct datarec *r = &rec->cpu[i];
725	struct datarec *p = &prev->cpu[i];
726	char str[64];
727
728	pps = calc_pps(r, p, period_: t);
729	drop = calc_drop_pps(r, p, period_: t);
730	err = calc_errs_pps(r, p, period_: t);
731	if (!pps && !drop && !err)
732	continue;
733
734	snprintf(s: str, maxlen: sizeof(str), format: "cpu:%d->%d", i, to_cpu);
735	if (err > 0)
736	err = pps / err; /* calc average bulk size */
737	print_default(
738	" %-18s " FMT_COLUMNf FMT_COLUMNf __COLUMN(
739	".2f") "\n",
740	str, PPS(pps), DROP(drop), err, "bulk-avg");
741	}
742	}
743	}
744
745	static void stats_get_cpumap_remote(struct stats_record *stats_rec,
746	struct stats_record *stats_prev,
747	unsigned int nr_cpus)
748	{
749	double xdp_pass, xdp_drop, xdp_redirect;
750	struct record *rec, *prev;
751	double t;
752	int i;
753
754	rec = &stats_rec->kthread;
755	prev = &stats_prev->kthread;
756	t = calc_period(r: rec, p: prev);
757
758	calc_xdp_pps(r: &rec->total, p: &prev->total, xdp_pass: &xdp_pass, xdp_drop: &xdp_drop,
759	xdp_redirect: &xdp_redirect, period_: t);
760	if (xdp_pass || xdp_drop || xdp_redirect) {
761	print_err(xdp_drop,
762	" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n",
763	"xdp_stats", PASS(xdp_pass), DROP(xdp_drop),
764	REDIR(xdp_redirect));
765	}
766
767	for (i = 0; i < nr_cpus; i++) {
768	struct datarec *r = &rec->cpu[i];
769	struct datarec *p = &prev->cpu[i];
770	char str[64];
771
772	calc_xdp_pps(r, p, xdp_pass: &xdp_pass, xdp_drop: &xdp_drop, xdp_redirect: &xdp_redirect, period_: t);
773	if (!xdp_pass && !xdp_drop && !xdp_redirect)
774	continue;
775
776	snprintf(s: str, maxlen: sizeof(str), format: "cpu:%d", i);
777	print_default(" %-16s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
778	"\n",
779	str, PASS(xdp_pass), DROP(xdp_drop),
780	REDIR(xdp_redirect));
781	}
782	}
783
784	static void stats_get_cpumap_kthread(struct stats_record *stats_rec,
785	struct stats_record *stats_prev,
786	unsigned int nr_cpus)
787	{
788	struct record *rec, *prev;
789	double t, pps, drop, err;
790	int i;
791
792	rec = &stats_rec->kthread;
793	prev = &stats_prev->kthread;
794	t = calc_period(r: rec, p: prev);
795
796	pps = calc_pps(r: &rec->total, p: &prev->total, period_: t);
797	drop = calc_drop_pps(r: &rec->total, p: &prev->total, period_: t);
798	err = calc_errs_pps(r: &rec->total, p: &prev->total, period_: t);
799
800	print_err(drop, " %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n",
801	pps ? "kthread total" : "kthread", PPS(pps), DROP(drop), err,
802	"sched");
803
804	for (i = 0; i < nr_cpus; i++) {
805	struct datarec *r = &rec->cpu[i];
806	struct datarec *p = &prev->cpu[i];
807	char str[64];
808
809	pps = calc_pps(r, p, period_: t);
810	drop = calc_drop_pps(r, p, period_: t);
811	err = calc_errs_pps(r, p, period_: t);
812	if (!pps && !drop && !err)
813	continue;
814
815	snprintf(s: str, maxlen: sizeof(str), format: "cpu:%d", i);
816	print_default(" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
817	"\n",
818	str, PPS(pps), DROP(drop), err, "sched");
819	}
820	}
821
822	static void stats_get_redirect_cnt(struct stats_record *stats_rec,
823	struct stats_record *stats_prev,
824	unsigned int nr_cpus,
825	struct sample_output *out)
826	{
827	struct record *rec, *prev;
828	double t, pps;
829	int i;
830
831	rec = &stats_rec->redir_err[0];
832	prev = &stats_prev->redir_err[0];
833	t = calc_period(r: rec, p: prev);
834	for (i = 0; i < nr_cpus; i++) {
835	struct datarec *r = &rec->cpu[i];
836	struct datarec *p = &prev->cpu[i];
837	char str[64];
838
839	pps = calc_pps(r, p, period_: t);
840	if (!pps)
841	continue;
842
843	snprintf(s: str, maxlen: sizeof(str), format: "cpu:%d", i);
844	print_default(" %-18s " FMT_COLUMNf "\n", str, REDIR(pps));
845	}
846
847	if (out) {
848	pps = calc_pps(r: &rec->total, p: &prev->total, period_: t);
849	out->redir_cnt.suc = pps;
850	out->totals.redir += pps;
851	}
852	}
853
854	static void stats_get_redirect_err_cnt(struct stats_record *stats_rec,
855	struct stats_record *stats_prev,
856	unsigned int nr_cpus,
857	struct sample_output *out)
858	{
859	struct record *rec, *prev;
860	double t, drop, sum = 0;
861	int rec_i, i;
862
863	for (rec_i = 1; rec_i < XDP_REDIRECT_ERR_MAX; rec_i++) {
864	char str[64];
865
866	rec = &stats_rec->redir_err[rec_i];
867	prev = &stats_prev->redir_err[rec_i];
868	t = calc_period(r: rec, p: prev);
869
870	drop = calc_drop_pps(r: &rec->total, p: &prev->total, period_: t);
871	if (drop > 0 && !out) {
872	snprintf(s: str, maxlen: sizeof(str),
873	format: sample_log_level & LL_DEFAULT ? "%s total" :
874	"%s",
875	xdp_redirect_err_names[rec_i]);
876	print_err(drop, " %-18s " FMT_COLUMNf "\n", str,
877	ERR(drop));
878	}
879
880	for (i = 0; i < nr_cpus; i++) {
881	struct datarec *r = &rec->cpu[i];
882	struct datarec *p = &prev->cpu[i];
883	double drop;
884
885	drop = calc_drop_pps(r, p, period_: t);
886	if (!drop)
887	continue;
888
889	snprintf(s: str, maxlen: sizeof(str), format: "cpu:%d", i);
890	print_default(" %-16s" FMT_COLUMNf "\n", str,
891	ERR(drop));
892	}
893
894	sum += drop;
895	}
896
897	if (out) {
898	out->redir_cnt.err = sum;
899	out->totals.err += sum;
900	}
901	}
902
903	static void stats_get_exception_cnt(struct stats_record *stats_rec,
904	struct stats_record *stats_prev,
905	unsigned int nr_cpus,
906	struct sample_output *out)
907	{
908	double t, drop, sum = 0;
909	struct record *rec, *prev;
910	int rec_i, i;
911
912	for (rec_i = 0; rec_i < XDP_ACTION_MAX; rec_i++) {
913	rec = &stats_rec->exception[rec_i];
914	prev = &stats_prev->exception[rec_i];
915	t = calc_period(rec, prev);
916
917	drop = calc_drop_pps(&rec->total, &prev->total, t);
918	/* Fold out errors after heading */
919	sum += drop;
920
921	if (drop > 0 && !out) {
922	print_always(" %-18s " FMT_COLUMNf "\n",
923	action2str(rec_i), ERR(drop));
924
925	for (i = 0; i < nr_cpus; i++) {
926	struct datarec *r = &rec->cpu[i];
927	struct datarec *p = &prev->cpu[i];
928	char str[64];
929	double drop;
930
931	drop = calc_drop_pps(r, p, t);
932	if (!drop)
933	continue;
934
935	snprintf(str, sizeof(str), "cpu:%d", i);
936	print_default(" %-16s" FMT_COLUMNf "\n",
937	str, ERR(drop));
938	}
939	}
940	}
941
942	if (out) {
943	out->except_cnt.hits = sum;
944	out->totals.err += sum;
945	}
946	}
947
948	static void stats_get_devmap_xmit(struct stats_record *stats_rec,
949	struct stats_record *stats_prev,
950	unsigned int nr_cpus,
951	struct sample_output *out)
952	{
953	double pps, drop, info, err;
954	struct record *rec, *prev;
955	double t;
956	int i;
957
958	rec = &stats_rec->devmap_xmit;
959	prev = &stats_prev->devmap_xmit;
960	t = calc_period(r: rec, p: prev);
961	for (i = 0; i < nr_cpus; i++) {
962	struct datarec *r = &rec->cpu[i];
963	struct datarec *p = &prev->cpu[i];
964	char str[64];
965
966	pps = calc_pps(r, p, period_: t);
967	drop = calc_drop_pps(r, p, period_: t);
968	err = calc_errs_pps(r, p, period_: t);
969
970	if (!pps && !drop && !err)
971	continue;
972
973	snprintf(s: str, maxlen: sizeof(str), format: "cpu:%d", i);
974	info = calc_info_pps(r, p, period_: t);
975	if (info > 0)
976	info = (pps + drop) / info; /* calc avg bulk */
977	print_default(" %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
978	__COLUMN(".2f") "\n",
979	str, XMIT(pps), DROP(drop), err, "drv_err/s",
980	info, "bulk-avg");
981	}
982	if (out) {
983	pps = calc_pps(r: &rec->total, p: &prev->total, period_: t);
984	drop = calc_drop_pps(r: &rec->total, p: &prev->total, period_: t);
985	info = calc_info_pps(r: &rec->total, p: &prev->total, period_: t);
986	if (info > 0)
987	info = (pps + drop) / info; /* calc avg bulk */
988	err = calc_errs_pps(r: &rec->total, p: &prev->total, period_: t);
989
990	out->xmit_cnt.pps = pps;
991	out->xmit_cnt.drop = drop;
992	out->xmit_cnt.bavg = info;
993	out->xmit_cnt.err = err;
994	out->totals.xmit += pps;
995	out->totals.drop_xmit += drop;
996	out->totals.err += err;
997	}
998	}
999
1000	static void stats_get_devmap_xmit_multi(struct stats_record *stats_rec,
1001	struct stats_record *stats_prev,
1002	unsigned int nr_cpus,
1003	struct sample_output *out,
1004	bool xmit_total)
1005	{
1006	double pps, drop, info, err;
1007	struct map_entry *entry;
1008	struct record *r, *p;
1009	double t;
1010	int bkt;
1011
1012	hash_for_each(stats_rec->xmit_map, bkt, entry, node) {
1013	struct map_entry *e, *x = NULL;
1014	char ifname_from[IFNAMSIZ];
1015	char ifname_to[IFNAMSIZ];
1016	const char *fstr, *tstr;
1017	unsigned long prev_time;
1018	struct record beg = {};
1019	__u32 from_idx, to_idx;
1020	char str[128];
1021	__u64 pair;
1022	int i;
1023
1024	prev_time = sample_interval * NANOSEC_PER_SEC;
1025
1026	pair = entry->pair;
1027	from_idx = pair >> 32;
1028	to_idx = pair & 0xFFFFFFFF;
1029
1030	r = &entry->val;
1031	beg.timestamp = r->timestamp - prev_time;
1032
1033	/* Find matching entry from stats_prev map */
1034	hash_for_each_possible(stats_prev->xmit_map, e, node, pair) {
1035	if (e->pair == pair) {
1036	x = e;
1037	break;
1038	}
1039	}
1040	if (x)
1041	p = &x->val;
1042	else
1043	p = &beg;
1044	t = calc_period(r, p);
1045	pps = calc_pps(r: &r->total, p: &p->total, period_: t);
1046	drop = calc_drop_pps(r: &r->total, p: &p->total, period_: t);
1047	info = calc_info_pps(r: &r->total, p: &p->total, period_: t);
1048	if (info > 0)
1049	info = (pps + drop) / info; /* calc avg bulk */
1050	err = calc_errs_pps(r: &r->total, p: &p->total, period_: t);
1051
1052	if (out) {
1053	/* We are responsible for filling out totals */
1054	out->totals.xmit += pps;
1055	out->totals.drop_xmit += drop;
1056	out->totals.err += err;
1057	continue;
1058	}
1059
1060	fstr = tstr = NULL;
1061	if (if_indextoname(ifindex: from_idx, ifname: ifname_from))
1062	fstr = ifname_from;
1063	if (if_indextoname(ifindex: to_idx, ifname: ifname_to))
1064	tstr = ifname_to;
1065
1066	snprintf(s: str, maxlen: sizeof(str), format: "xmit %s->%s", fstr ?: "?",
1067	tstr ?: "?");
1068	/* Skip idle streams of redirection */
1069	if (pps || drop || err) {
1070	print_err(drop,
1071	" %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
1072	__COLUMN(".2f") "\n", str, XMIT(pps), DROP(drop),
1073	err, "drv_err/s", info, "bulk-avg");
1074	}
1075
1076	for (i = 0; i < nr_cpus; i++) {
1077	struct datarec *rc = &r->cpu[i];
1078	struct datarec *pc, p_beg = {};
1079	char str[64];
1080
1081	pc = p == &beg ? &p_beg : &p->cpu[i];
1082
1083	pps = calc_pps(r: rc, p: pc, period_: t);
1084	drop = calc_drop_pps(r: rc, p: pc, period_: t);
1085	err = calc_errs_pps(r: rc, p: pc, period_: t);
1086
1087	if (!pps && !drop && !err)
1088	continue;
1089
1090	snprintf(s: str, maxlen: sizeof(str), format: "cpu:%d", i);
1091	info = calc_info_pps(r: rc, p: pc, period_: t);
1092	if (info > 0)
1093	info = (pps + drop) / info; /* calc avg bulk */
1094
1095	print_default(" %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
1096	__COLUMN(".2f") "\n", str, XMIT(pps),
1097	DROP(drop), err, "drv_err/s", info, "bulk-avg");
1098	}
1099	}
1100	}
1101
1102	static void stats_print(const char *prefix, int mask, struct stats_record *r,
1103	struct stats_record *p, struct sample_output *out)
1104	{
1105	int nr_cpus = libbpf_num_possible_cpus();
1106	const char *str;
1107
1108	print_always("%-23s", prefix ?: "Summary");
1109	if (mask & SAMPLE_RX_CNT)
1110	print_always(FMT_COLUMNl, RX(out->totals.rx));
1111	if (mask & SAMPLE_REDIRECT_CNT)
1112	print_always(FMT_COLUMNl, REDIR(out->totals.redir));
1113	printf(FMT_COLUMNl,
1114	out->totals.err + out->totals.drop + out->totals.drop_xmit,
1115	"err,drop/s");
1116	if (mask & SAMPLE_DEVMAP_XMIT_CNT ||
1117	mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1118	printf(FMT_COLUMNl, XMIT(out->totals.xmit));
1119	printf(format: "\n");
1120
1121	if (mask & SAMPLE_RX_CNT) {
1122	str = (sample_log_level & LL_DEFAULT) && out->rx_cnt.pps ?
1123	"receive total" :
1124	"receive";
1125	print_err((out->rx_cnt.err || out->rx_cnt.drop),
1126	" %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl "\n",
1127	str, PPS(out->rx_cnt.pps), DROP(out->rx_cnt.drop),
1128	ERR(out->rx_cnt.err));
1129
1130	stats_get_rx_cnt(stats_rec: r, stats_prev: p, nr_cpus, NULL);
1131	}
1132
1133	if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT)
1134	stats_get_cpumap_enqueue(stats_rec: r, stats_prev: p, nr_cpus);
1135
1136	if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
1137	stats_get_cpumap_kthread(stats_rec: r, stats_prev: p, nr_cpus);
1138	stats_get_cpumap_remote(stats_rec: r, stats_prev: p, nr_cpus);
1139	}
1140
1141	if (mask & SAMPLE_REDIRECT_CNT) {
1142	str = out->redir_cnt.suc ? "redirect total" : "redirect";
1143	print_default(" %-20s " FMT_COLUMNl "\n", str,
1144	REDIR(out->redir_cnt.suc));
1145
1146	stats_get_redirect_cnt(stats_rec: r, stats_prev: p, nr_cpus, NULL);
1147	}
1148
1149	if (mask & SAMPLE_REDIRECT_ERR_CNT) {
1150	str = (sample_log_level & LL_DEFAULT) && out->redir_cnt.err ?
1151	"redirect_err total" :
1152	"redirect_err";
1153	print_err(out->redir_cnt.err, " %-20s " FMT_COLUMNl "\n", str,
1154	ERR(out->redir_cnt.err));
1155
1156	stats_get_redirect_err_cnt(stats_rec: r, stats_prev: p, nr_cpus, NULL);
1157	}
1158
1159	if (mask & SAMPLE_EXCEPTION_CNT) {
1160	str = out->except_cnt.hits ? "xdp_exception total" :
1161	"xdp_exception";
1162
1163	print_err(out->except_cnt.hits, " %-20s " FMT_COLUMNl "\n", str,
1164	HITS(out->except_cnt.hits));
1165
1166	stats_get_exception_cnt(stats_rec: r, stats_prev: p, nr_cpus, NULL);
1167	}
1168
1169	if (mask & SAMPLE_DEVMAP_XMIT_CNT) {
1170	str = (sample_log_level & LL_DEFAULT) && out->xmit_cnt.pps ?
1171	"devmap_xmit total" :
1172	"devmap_xmit";
1173
1174	print_err(out->xmit_cnt.err || out->xmit_cnt.drop,
1175	" %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl
1176	__COLUMN(".2f") "\n",
1177	str, XMIT(out->xmit_cnt.pps),
1178	DROP(out->xmit_cnt.drop), out->xmit_cnt.err,
1179	"drv_err/s", out->xmit_cnt.bavg, "bulk-avg");
1180
1181	stats_get_devmap_xmit(stats_rec: r, stats_prev: p, nr_cpus, NULL);
1182	}
1183
1184	if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1185	stats_get_devmap_xmit_multi(stats_rec: r, stats_prev: p, nr_cpus, NULL,
1186	xmit_total: mask & SAMPLE_DEVMAP_XMIT_CNT);
1187
1188	if (sample_log_level & LL_DEFAULT ||
1189	((sample_log_level & LL_SIMPLE) && sample_err_exp)) {
1190	sample_err_exp = false;
1191	printf(format: "\n");
1192	}
1193	}
1194
1195	int sample_setup_maps(struct bpf_map **maps)
1196	{
1197	sample_n_cpus = libbpf_num_possible_cpus();
1198
1199	for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) {
1200	sample_map[i] = maps[i];
1201
1202	switch (i) {
1203	case MAP_RX:
1204	case MAP_CPUMAP_KTHREAD:
1205	case MAP_DEVMAP_XMIT:
1206	sample_map_count[i] = sample_n_cpus;
1207	break;
1208	case MAP_REDIRECT_ERR:
1209	sample_map_count[i] =
1210	XDP_REDIRECT_ERR_MAX * sample_n_cpus;
1211	break;
1212	case MAP_EXCEPTION:
1213	sample_map_count[i] = XDP_ACTION_MAX * sample_n_cpus;
1214	case MAP_CPUMAP_ENQUEUE:
1215	sample_map_count[i] = sample_n_cpus * sample_n_cpus;
1216	break;
1217	default:
1218	return -EINVAL;
1219	}
1220	if (bpf_map__set_max_entries(sample_map[i], sample_map_count[i]) < 0)
1221	return -errno;
1222	}
1223	sample_map[MAP_DEVMAP_XMIT_MULTI] = maps[MAP_DEVMAP_XMIT_MULTI];
1224	return 0;
1225	}
1226
1227	static int sample_setup_maps_mappings(void)
1228	{
1229	for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) {
1230	size_t size = sample_map_count[i] * sizeof(struct datarec);
1231
1232	sample_mmap[i] = mmap(NULL, len: size, PROT_READ | PROT_WRITE,
1233	MAP_SHARED, fd: bpf_map__fd(sample_map[i]), offset: 0);
1234	if (sample_mmap[i] == MAP_FAILED)
1235	return -errno;
1236	}
1237	return 0;
1238	}
1239
1240	int __sample_init(int mask)
1241	{
1242	sigset_t st;
1243
1244	sigemptyset(set: &st);
1245	sigaddset(set: &st, SIGQUIT);
1246	sigaddset(set: &st, SIGINT);
1247	sigaddset(set: &st, SIGTERM);
1248
1249	if (sigprocmask(SIG_BLOCK, set: &st, NULL) < 0)
1250	return -errno;
1251
1252	sample_sig_fd = signalfd(fd: -1, mask: &st, SFD_CLOEXEC | SFD_NONBLOCK);
1253	if (sample_sig_fd < 0)
1254	return -errno;
1255
1256	sample_mask = mask;
1257
1258	return sample_setup_maps_mappings();
1259	}
1260
1261	static int __sample_remove_xdp(int ifindex, __u32 prog_id, int xdp_flags)
1262	{
1263	__u32 cur_prog_id = 0;
1264	int ret;
1265
1266	if (prog_id) {
1267	ret = bpf_xdp_query_id(ifindex, xdp_flags, &cur_prog_id);
1268	if (ret < 0)
1269	return -errno;
1270
1271	if (prog_id != cur_prog_id) {
1272	print_always(
1273	"Program on ifindex %d does not match installed "
1274	"program, skipping unload\n",
1275	ifindex);
1276	return -ENOENT;
1277	}
1278	}
1279
1280	return bpf_xdp_detach(ifindex, xdp_flags, NULL);
1281	}
1282
1283	int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic,
1284	bool force)
1285	{
1286	int ret, xdp_flags = 0;
1287	__u32 prog_id = 0;
1288
1289	if (sample_xdp_cnt == 32) {
1290	fprintf(stderr,
1291	format: "Total limit for installed XDP programs in a sample reached\n");
1292	return -ENOTSUP;
1293	}
1294
1295	xdp_flags |= !force ? XDP_FLAGS_UPDATE_IF_NOEXIST : 0;
1296	xdp_flags |= generic ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;
1297	ret = bpf_xdp_attach(ifindex, bpf_program__fd(xdp_prog), xdp_flags, NULL);
1298	if (ret < 0) {
1299	ret = -errno;
1300	fprintf(stderr,
1301	format: "Failed to install program \"%s\" on ifindex %d, mode = %s, "
1302	"force = %s: %s\n",
1303	bpf_program__name(xdp_prog), ifindex,
1304	generic ? "skb" : "native", force ? "true" : "false",
1305	strerror(errnum: -ret));
1306	return ret;
1307	}
1308
1309	ret = bpf_xdp_query_id(ifindex, xdp_flags, &prog_id);
1310	if (ret < 0) {
1311	ret = -errno;
1312	fprintf(stderr,
1313	format: "Failed to get XDP program id for ifindex %d, removing program: %s\n",
1314	ifindex, strerror(errno));
1315	__sample_remove_xdp(ifindex, prog_id: 0, xdp_flags);
1316	return ret;
1317	}
1318	sample_xdp_progs[sample_xdp_cnt++] =
1319	(struct xdp_desc){ ifindex, prog_id, xdp_flags };
1320
1321	return 0;
1322	}
1323
1324	static void sample_summary_print(void)
1325	{
1326	double num = sample_out.rx_cnt.num;
1327
1328	if (sample_out.totals.rx) {
1329	double pkts = sample_out.totals.rx;
1330
1331	print_always(" Packets received : %'-10llu\n",
1332	sample_out.totals.rx);
1333	print_always(" Average packets/s : %'-10.0f\n",
1334	sample_round(pkts / num));
1335	}
1336	if (sample_out.totals.redir) {
1337	double pkts = sample_out.totals.redir;
1338
1339	print_always(" Packets redirected : %'-10llu\n",
1340	sample_out.totals.redir);
1341	print_always(" Average redir/s : %'-10.0f\n",
1342	sample_round(pkts / num));
1343	}
1344	if (sample_out.totals.drop)
1345	print_always(" Rx dropped : %'-10llu\n",
1346	sample_out.totals.drop);
1347	if (sample_out.totals.drop_xmit)
1348	print_always(" Tx dropped : %'-10llu\n",
1349	sample_out.totals.drop_xmit);
1350	if (sample_out.totals.err)
1351	print_always(" Errors recorded : %'-10llu\n",
1352	sample_out.totals.err);
1353	if (sample_out.totals.xmit) {
1354	double pkts = sample_out.totals.xmit;
1355
1356	print_always(" Packets transmitted : %'-10llu\n",
1357	sample_out.totals.xmit);
1358	print_always(" Average transmit/s : %'-10.0f\n",
1359	sample_round(pkts / num));
1360	}
1361	}
1362
1363	void sample_exit(int status)
1364	{
1365	size_t size;
1366
1367	for (int i = 0; i < NUM_MAP; i++) {
1368	size = sample_map_count[i] * sizeof(**sample_mmap);
1369	munmap(addr: sample_mmap[i], len: size);
1370	}
1371	while (sample_xdp_cnt--) {
1372	int i = sample_xdp_cnt, ifindex, xdp_flags;
1373	__u32 prog_id;
1374
1375	prog_id = sample_xdp_progs[i].prog_id;
1376	ifindex = sample_xdp_progs[i].ifindex;
1377	xdp_flags = sample_xdp_progs[i].flags;
1378
1379	__sample_remove_xdp(ifindex, prog_id, xdp_flags);
1380	}
1381	sample_summary_print();
1382	close(fd: sample_sig_fd);
1383	exit(status: status);
1384	}
1385
1386	static int sample_stats_collect(struct stats_record *rec)
1387	{
1388	int i;
1389
1390	if (sample_mask & SAMPLE_RX_CNT)
1391	map_collect_percpu(values: sample_mmap[MAP_RX], rec: &rec->rx_cnt);
1392
1393	if (sample_mask & SAMPLE_REDIRECT_CNT)
1394	map_collect_percpu(values: sample_mmap[MAP_REDIRECT_ERR], rec: &rec->redir_err[0]);
1395
1396	if (sample_mask & SAMPLE_REDIRECT_ERR_CNT) {
1397	for (i = 1; i < XDP_REDIRECT_ERR_MAX; i++)
1398	map_collect_percpu(values: &sample_mmap[MAP_REDIRECT_ERR][i * sample_n_cpus],
1399	rec: &rec->redir_err[i]);
1400	}
1401
1402	if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT)
1403	for (i = 0; i < sample_n_cpus; i++)
1404	map_collect_percpu(values: &sample_mmap[MAP_CPUMAP_ENQUEUE][i * sample_n_cpus],
1405	rec: &rec->enq[i]);
1406
1407	if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT)
1408	map_collect_percpu(values: sample_mmap[MAP_CPUMAP_KTHREAD],
1409	rec: &rec->kthread);
1410
1411	if (sample_mask & SAMPLE_EXCEPTION_CNT)
1412	for (i = 0; i < XDP_ACTION_MAX; i++)
1413	map_collect_percpu(&sample_mmap[MAP_EXCEPTION][i * sample_n_cpus],
1414	&rec->exception[i]);
1415
1416	if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT)
1417	map_collect_percpu(values: sample_mmap[MAP_DEVMAP_XMIT], rec: &rec->devmap_xmit);
1418
1419	if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) {
1420	if (map_collect_percpu_devmap(map_fd: bpf_map__fd(sample_map[MAP_DEVMAP_XMIT_MULTI]), rec) < 0)
1421	return -EINVAL;
1422	}
1423	return 0;
1424	}
1425
1426	static void sample_summary_update(struct sample_output *out)
1427	{
1428	sample_out.totals.rx += out->totals.rx;
1429	sample_out.totals.redir += out->totals.redir;
1430	sample_out.totals.drop += out->totals.drop;
1431	sample_out.totals.drop_xmit += out->totals.drop_xmit;
1432	sample_out.totals.err += out->totals.err;
1433	sample_out.totals.xmit += out->totals.xmit;
1434	sample_out.rx_cnt.num++;
1435	}
1436
1437	static void sample_stats_print(int mask, struct stats_record *cur,
1438	struct stats_record *prev, char *prog_name)
1439	{
1440	struct sample_output out = {};
1441
1442	if (mask & SAMPLE_RX_CNT)
1443	stats_get_rx_cnt(stats_rec: cur, stats_prev: prev, nr_cpus: 0, out: &out);
1444	if (mask & SAMPLE_REDIRECT_CNT)
1445	stats_get_redirect_cnt(stats_rec: cur, stats_prev: prev, nr_cpus: 0, out: &out);
1446	if (mask & SAMPLE_REDIRECT_ERR_CNT)
1447	stats_get_redirect_err_cnt(stats_rec: cur, stats_prev: prev, nr_cpus: 0, out: &out);
1448	if (mask & SAMPLE_EXCEPTION_CNT)
1449	stats_get_exception_cnt(stats_rec: cur, stats_prev: prev, nr_cpus: 0, out: &out);
1450	if (mask & SAMPLE_DEVMAP_XMIT_CNT)
1451	stats_get_devmap_xmit(stats_rec: cur, stats_prev: prev, nr_cpus: 0, out: &out);
1452	else if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1453	stats_get_devmap_xmit_multi(stats_rec: cur, stats_prev: prev, nr_cpus: 0, out: &out,
1454	xmit_total: mask & SAMPLE_DEVMAP_XMIT_CNT);
1455	sample_summary_update(out: &out);
1456
1457	stats_print(prefix: prog_name, mask, r: cur, p: prev, out: &out);
1458	}
1459
1460	void sample_switch_mode(void)
1461	{
1462	sample_log_level ^= LL_DEBUG - 1;
1463	}
1464
1465	static int sample_signal_cb(void)
1466	{
1467	struct signalfd_siginfo si;
1468	int r;
1469
1470	r = read(fd: sample_sig_fd, buf: &si, nbytes: sizeof(si));
1471	if (r < 0)
1472	return -errno;
1473
1474	switch (si.ssi_signo) {
1475	case SIGQUIT:
1476	sample_switch_mode();
1477	printf(format: "\n");
1478	break;
1479	default:
1480	printf(format: "\n");
1481	return 1;
1482	}
1483
1484	return 0;
1485	}
1486
1487	/* Pointer swap trick */
1488	static void swap(struct stats_record **a, struct stats_record **b)
1489	{
1490	struct stats_record *tmp;
1491
1492	tmp = *a;
1493	*a = *b;
1494	*b = tmp;
1495	}
1496
1497	static int sample_timer_cb(int timerfd, struct stats_record **rec,
1498	struct stats_record **prev)
1499	{
1500	char line[64] = "Summary";
1501	int ret;
1502	__u64 t;
1503
1504	ret = read(fd: timerfd, buf: &t, nbytes: sizeof(t));
1505	if (ret < 0)
1506	return -errno;
1507
1508	swap(a: prev, b: rec);
1509	ret = sample_stats_collect(rec: *rec);
1510	if (ret < 0)
1511	return ret;
1512
1513	if (sample_xdp_cnt == 2 && !(sample_mask & SAMPLE_SKIP_HEADING)) {
1514	char fi[IFNAMSIZ];
1515	char to[IFNAMSIZ];
1516	const char *f, *t;
1517
1518	f = t = NULL;
1519	if (if_indextoname(ifindex: sample_xdp_progs[0].ifindex, ifname: fi))
1520	f = fi;
1521	if (if_indextoname(ifindex: sample_xdp_progs[1].ifindex, ifname: to))
1522	t = to;
1523
1524	snprintf(s: line, maxlen: sizeof(line), format: "%s->%s", f ?: "?", t ?: "?");
1525	}
1526
1527	sample_stats_print(mask: sample_mask, cur: *rec, prev: *prev, prog_name: line);
1528	return 0;
1529	}
1530
1531	int sample_run(int interval, void (*post_cb)(void *), void *ctx)
1532	{
1533	struct timespec ts = { interval, 0 };
1534	struct itimerspec its = { ts, ts };
1535	struct stats_record *rec, *prev;
1536	struct pollfd pfd[2] = {};
1537	int timerfd, ret;
1538
1539	if (!interval) {
1540	fprintf(stderr, format: "Incorrect interval 0\n");
1541	return -EINVAL;
1542	}
1543	sample_interval = interval;
1544	/* Pretty print numbers */
1545	setlocale(LC_NUMERIC, locale: "en_US.UTF-8");
1546
1547	timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC | TFD_NONBLOCK);
1548	if (timerfd < 0)
1549	return -errno;
1550	timerfd_settime(ufd: timerfd, flags: 0, utmr: &its, NULL);
1551
1552	pfd[0].fd = sample_sig_fd;
1553	pfd[0].events = POLLIN;
1554
1555	pfd[1].fd = timerfd;
1556	pfd[1].events = POLLIN;
1557
1558	ret = -ENOMEM;
1559	rec = alloc_stats_record();
1560	if (!rec)
1561	goto end;
1562	prev = alloc_stats_record();
1563	if (!prev)
1564	goto end_rec;
1565
1566	ret = sample_stats_collect(rec);
1567	if (ret < 0)
1568	goto end_rec_prev;
1569
1570	for (;;) {
1571	ret = poll(fds: pfd, nfds: 2, timeout: -1);
1572	if (ret < 0) {
1573	if (errno == EINTR)
1574	continue;
1575	else
1576	break;
1577	}
1578
1579	if (pfd[0].revents & POLLIN)
1580	ret = sample_signal_cb();
1581	else if (pfd[1].revents & POLLIN)
1582	ret = sample_timer_cb(timerfd, rec: &rec, prev: &prev);
1583
1584	if (ret)
1585	break;
1586
1587	if (post_cb)
1588	post_cb(ctx);
1589	}
1590
1591	end_rec_prev:
1592	free_stats_record(r: prev);
1593	end_rec:
1594	free_stats_record(r: rec);
1595	end:
1596	close(fd: timerfd);
1597
1598	return ret;
1599	}
1600
1601	const char *get_driver_name(int ifindex)
1602	{
1603	struct ethtool_drvinfo drv = {};
1604	char ifname[IF_NAMESIZE];
1605	static char drvname[32];
1606	struct ifreq ifr = {};
1607	int fd, r = 0;
1608
1609	fd = socket(AF_INET, SOCK_DGRAM, protocol: 0);
1610	if (fd < 0)
1611	return "[error]";
1612
1613	if (!if_indextoname(ifindex: ifindex, ifname: ifname))
1614	goto end;
1615
1616	drv.cmd = ETHTOOL_GDRVINFO;
1617	safe_strncpy(dst: ifr.ifr_name, src: ifname, size: sizeof(ifr.ifr_name));
1618	ifr.ifr_data = (void *)&drv;
1619
1620	r = ioctl(fd: fd, SIOCETHTOOL, &ifr);
1621	if (r)
1622	goto end;
1623
1624	safe_strncpy(dst: drvname, src: drv.driver, size: sizeof(drvname));
1625
1626	close(fd: fd);
1627	return drvname;
1628
1629	end:
1630	r = errno;
1631	close(fd: fd);
1632	return r == EOPNOTSUPP ? "loopback" : "[error]";
1633	}
1634
1635	int get_mac_addr(int ifindex, void *mac_addr)
1636	{
1637	char ifname[IF_NAMESIZE];
1638	struct ifreq ifr = {};
1639	int fd, r;
1640
1641	fd = socket(AF_INET, SOCK_DGRAM, protocol: 0);
1642	if (fd < 0)
1643	return -errno;
1644
1645	if (!if_indextoname(ifindex: ifindex, ifname: ifname)) {
1646	r = -errno;
1647	goto end;
1648	}
1649
1650	safe_strncpy(dst: ifr.ifr_name, src: ifname, size: sizeof(ifr.ifr_name));
1651
1652	r = ioctl(fd: fd, SIOCGIFHWADDR, &ifr);
1653	if (r) {
1654	r = -errno;
1655	goto end;
1656	}
1657
1658	memcpy(dest: mac_addr, src: ifr.ifr_hwaddr.sa_data, n: 6 * sizeof(char));
1659
1660	end:
1661	close(fd: fd);
1662	return r;
1663	}
1664
1665	__attribute__((constructor)) static void sample_ctor(void)
1666	{
1667	if (libbpf_set_strict_mode(LIBBPF_STRICT_ALL) < 0) {
1668	fprintf(stderr, format: "Failed to set libbpf strict mode: %s\n",
1669	strerror(errno));
1670	/* Just exit, nothing to cleanup right now */
1671	exit(EXIT_FAIL_BPF);
1672	}
1673	}
1674