1// SPDX-License-Identifier: GPL-2.0
2#include "util/bpf_counter.h"
3#include "util/debug.h"
4#include "util/evsel.h"
5#include "util/evlist.h"
6#include "util/off_cpu.h"
7#include "util/perf-hooks.h"
8#include "util/record.h"
9#include "util/session.h"
10#include "util/target.h"
11#include "util/cpumap.h"
12#include "util/thread_map.h"
13#include "util/cgroup.h"
14#include "util/strlist.h"
15#include <bpf/bpf.h>
16
17#include "bpf_skel/off_cpu.skel.h"
18
19#define MAX_STACKS 32
20#define MAX_PROC 4096
21/* we don't need actual timestamp, just want to put the samples at last */
22#define OFF_CPU_TIMESTAMP (~0ull << 32)
23
24static struct off_cpu_bpf *skel;
25
26struct off_cpu_key {
27 u32 pid;
28 u32 tgid;
29 u32 stack_id;
30 u32 state;
31 u64 cgroup_id;
32};
33
34union off_cpu_data {
35 struct perf_event_header hdr;
36 u64 array[1024 / sizeof(u64)];
37};
38
39static int off_cpu_config(struct evlist *evlist)
40{
41 struct evsel *evsel;
42 struct perf_event_attr attr = {
43 .type = PERF_TYPE_SOFTWARE,
44 .config = PERF_COUNT_SW_BPF_OUTPUT,
45 .size = sizeof(attr), /* to capture ABI version */
46 };
47 char *evname = strdup(OFFCPU_EVENT);
48
49 if (evname == NULL)
50 return -ENOMEM;
51
52 evsel = evsel__new(&attr);
53 if (!evsel) {
54 free(evname);
55 return -ENOMEM;
56 }
57
58 evsel->core.attr.freq = 1;
59 evsel->core.attr.sample_period = 1;
60 /* off-cpu analysis depends on stack trace */
61 evsel->core.attr.sample_type = PERF_SAMPLE_CALLCHAIN;
62
63 evlist__add(evlist, evsel);
64
65 free(evsel->name);
66 evsel->name = evname;
67
68 return 0;
69}
70
71static void off_cpu_start(void *arg)
72{
73 struct evlist *evlist = arg;
74
75 /* update task filter for the given workload */
76 if (!skel->bss->has_cpu && !skel->bss->has_task &&
77 perf_thread_map__pid(evlist->core.threads, 0) != -1) {
78 int fd;
79 u32 pid;
80 u8 val = 1;
81
82 skel->bss->has_task = 1;
83 skel->bss->uses_tgid = 1;
84 fd = bpf_map__fd(skel->maps.task_filter);
85 pid = perf_thread_map__pid(evlist->core.threads, 0);
86 bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
87 }
88
89 skel->bss->enabled = 1;
90}
91
92static void off_cpu_finish(void *arg __maybe_unused)
93{
94 skel->bss->enabled = 0;
95 off_cpu_bpf__destroy(skel);
96}
97
98/* v5.18 kernel added prev_state arg, so it needs to check the signature */
99static void check_sched_switch_args(void)
100{
101 struct btf *btf = btf__load_vmlinux_btf();
102 const struct btf_type *t1, *t2, *t3;
103 u32 type_id;
104
105 type_id = btf__find_by_name_kind(btf, "btf_trace_sched_switch",
106 BTF_KIND_TYPEDEF);
107 if ((s32)type_id < 0)
108 goto cleanup;
109
110 t1 = btf__type_by_id(btf, type_id);
111 if (t1 == NULL)
112 goto cleanup;
113
114 t2 = btf__type_by_id(btf, t1->type);
115 if (t2 == NULL || !btf_is_ptr(t2))
116 goto cleanup;
117
118 t3 = btf__type_by_id(btf, t2->type);
119 /* btf_trace func proto has one more argument for the context */
120 if (t3 && btf_is_func_proto(t3) && btf_vlen(t3) == 5) {
121 /* new format: pass prev_state as 4th arg */
122 skel->rodata->has_prev_state = true;
123 }
124cleanup:
125 btf__free(btf);
126}
127
128int off_cpu_prepare(struct evlist *evlist, struct target *target,
129 struct record_opts *opts)
130{
131 int err, fd, i;
132 int ncpus = 1, ntasks = 1, ncgrps = 1;
133 struct strlist *pid_slist = NULL;
134 struct str_node *pos;
135
136 if (off_cpu_config(evlist) < 0) {
137 pr_err("Failed to config off-cpu BPF event\n");
138 return -1;
139 }
140
141 skel = off_cpu_bpf__open();
142 if (!skel) {
143 pr_err("Failed to open off-cpu BPF skeleton\n");
144 return -1;
145 }
146
147 /* don't need to set cpu filter for system-wide mode */
148 if (target->cpu_list) {
149 ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus);
150 bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus);
151 }
152
153 if (target->pid) {
154 pid_slist = strlist__new(target->pid, NULL);
155 if (!pid_slist) {
156 pr_err("Failed to create a strlist for pid\n");
157 return -1;
158 }
159
160 ntasks = 0;
161 strlist__for_each_entry(pos, pid_slist) {
162 char *end_ptr;
163 int pid = strtol(pos->s, &end_ptr, 10);
164
165 if (pid == INT_MIN || pid == INT_MAX ||
166 (*end_ptr != '\0' && *end_ptr != ','))
167 continue;
168
169 ntasks++;
170 }
171
172 if (ntasks < MAX_PROC)
173 ntasks = MAX_PROC;
174
175 bpf_map__set_max_entries(skel->maps.task_filter, ntasks);
176 } else if (target__has_task(target)) {
177 ntasks = perf_thread_map__nr(evlist->core.threads);
178 bpf_map__set_max_entries(skel->maps.task_filter, ntasks);
179 } else if (target__none(target)) {
180 bpf_map__set_max_entries(skel->maps.task_filter, MAX_PROC);
181 }
182
183 if (evlist__first(evlist)->cgrp) {
184 ncgrps = evlist->core.nr_entries - 1; /* excluding a dummy */
185 bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps);
186
187 if (!cgroup_is_v2("perf_event"))
188 skel->rodata->uses_cgroup_v1 = true;
189 }
190
191 if (opts->record_cgroup) {
192 skel->rodata->needs_cgroup = true;
193
194 if (!cgroup_is_v2("perf_event"))
195 skel->rodata->uses_cgroup_v1 = true;
196 }
197
198 set_max_rlimit();
199 check_sched_switch_args();
200
201 err = off_cpu_bpf__load(skel);
202 if (err) {
203 pr_err("Failed to load off-cpu skeleton\n");
204 goto out;
205 }
206
207 if (target->cpu_list) {
208 u32 cpu;
209 u8 val = 1;
210
211 skel->bss->has_cpu = 1;
212 fd = bpf_map__fd(skel->maps.cpu_filter);
213
214 for (i = 0; i < ncpus; i++) {
215 cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu;
216 bpf_map_update_elem(fd, &cpu, &val, BPF_ANY);
217 }
218 }
219
220 if (target->pid) {
221 u8 val = 1;
222
223 skel->bss->has_task = 1;
224 skel->bss->uses_tgid = 1;
225 fd = bpf_map__fd(skel->maps.task_filter);
226
227 strlist__for_each_entry(pos, pid_slist) {
228 char *end_ptr;
229 u32 tgid;
230 int pid = strtol(pos->s, &end_ptr, 10);
231
232 if (pid == INT_MIN || pid == INT_MAX ||
233 (*end_ptr != '\0' && *end_ptr != ','))
234 continue;
235
236 tgid = pid;
237 bpf_map_update_elem(fd, &tgid, &val, BPF_ANY);
238 }
239 } else if (target__has_task(target)) {
240 u32 pid;
241 u8 val = 1;
242
243 skel->bss->has_task = 1;
244 fd = bpf_map__fd(skel->maps.task_filter);
245
246 for (i = 0; i < ntasks; i++) {
247 pid = perf_thread_map__pid(evlist->core.threads, i);
248 bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
249 }
250 }
251
252 if (evlist__first(evlist)->cgrp) {
253 struct evsel *evsel;
254 u8 val = 1;
255
256 skel->bss->has_cgroup = 1;
257 fd = bpf_map__fd(skel->maps.cgroup_filter);
258
259 evlist__for_each_entry(evlist, evsel) {
260 struct cgroup *cgrp = evsel->cgrp;
261
262 if (cgrp == NULL)
263 continue;
264
265 if (!cgrp->id && read_cgroup_id(cgrp) < 0) {
266 pr_err("Failed to read cgroup id of %s\n",
267 cgrp->name);
268 goto out;
269 }
270
271 bpf_map_update_elem(fd, &cgrp->id, &val, BPF_ANY);
272 }
273 }
274
275 err = off_cpu_bpf__attach(skel);
276 if (err) {
277 pr_err("Failed to attach off-cpu BPF skeleton\n");
278 goto out;
279 }
280
281 if (perf_hooks__set_hook("record_start", off_cpu_start, evlist) ||
282 perf_hooks__set_hook("record_end", off_cpu_finish, evlist)) {
283 pr_err("Failed to attach off-cpu skeleton\n");
284 goto out;
285 }
286
287 return 0;
288
289out:
290 off_cpu_bpf__destroy(skel);
291 return -1;
292}
293
294int off_cpu_write(struct perf_session *session)
295{
296 int bytes = 0, size;
297 int fd, stack;
298 u64 sample_type, val, sid = 0;
299 struct evsel *evsel;
300 struct perf_data_file *file = &session->data->file;
301 struct off_cpu_key prev, key;
302 union off_cpu_data data = {
303 .hdr = {
304 .type = PERF_RECORD_SAMPLE,
305 .misc = PERF_RECORD_MISC_USER,
306 },
307 };
308 u64 tstamp = OFF_CPU_TIMESTAMP;
309
310 skel->bss->enabled = 0;
311
312 evsel = evlist__find_evsel_by_str(session->evlist, OFFCPU_EVENT);
313 if (evsel == NULL) {
314 pr_err("%s evsel not found\n", OFFCPU_EVENT);
315 return 0;
316 }
317
318 sample_type = evsel->core.attr.sample_type;
319
320 if (sample_type & ~OFFCPU_SAMPLE_TYPES) {
321 pr_err("not supported sample type: %llx\n",
322 (unsigned long long)sample_type);
323 return -1;
324 }
325
326 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) {
327 if (evsel->core.id)
328 sid = evsel->core.id[0];
329 }
330
331 fd = bpf_map__fd(skel->maps.off_cpu);
332 stack = bpf_map__fd(skel->maps.stacks);
333 memset(&prev, 0, sizeof(prev));
334
335 while (!bpf_map_get_next_key(fd, &prev, &key)) {
336 int n = 1; /* start from perf_event_header */
337 int ip_pos = -1;
338
339 bpf_map_lookup_elem(fd, &key, &val);
340
341 if (sample_type & PERF_SAMPLE_IDENTIFIER)
342 data.array[n++] = sid;
343 if (sample_type & PERF_SAMPLE_IP) {
344 ip_pos = n;
345 data.array[n++] = 0; /* will be updated */
346 }
347 if (sample_type & PERF_SAMPLE_TID)
348 data.array[n++] = (u64)key.pid << 32 | key.tgid;
349 if (sample_type & PERF_SAMPLE_TIME)
350 data.array[n++] = tstamp;
351 if (sample_type & PERF_SAMPLE_ID)
352 data.array[n++] = sid;
353 if (sample_type & PERF_SAMPLE_CPU)
354 data.array[n++] = 0;
355 if (sample_type & PERF_SAMPLE_PERIOD)
356 data.array[n++] = val;
357 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
358 int len = 0;
359
360 /* data.array[n] is callchain->nr (updated later) */
361 data.array[n + 1] = PERF_CONTEXT_USER;
362 data.array[n + 2] = 0;
363
364 bpf_map_lookup_elem(stack, &key.stack_id, &data.array[n + 2]);
365 while (data.array[n + 2 + len])
366 len++;
367
368 /* update length of callchain */
369 data.array[n] = len + 1;
370
371 /* update sample ip with the first callchain entry */
372 if (ip_pos >= 0)
373 data.array[ip_pos] = data.array[n + 2];
374
375 /* calculate sample callchain data array length */
376 n += len + 2;
377 }
378 if (sample_type & PERF_SAMPLE_CGROUP)
379 data.array[n++] = key.cgroup_id;
380
381 size = n * sizeof(u64);
382 data.hdr.size = size;
383 bytes += size;
384
385 if (perf_data_file__write(file, &data, size) < 0) {
386 pr_err("failed to write perf data, error: %m\n");
387 return bytes;
388 }
389
390 prev = key;
391 /* increase dummy timestamp to sort later samples */
392 tstamp++;
393 }
394 return bytes;
395}
396

source code of linux/tools/perf/util/bpf_off_cpu.c