1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * bpf_kwork.c |
4 | * |
5 | * Copyright (c) 2022 Huawei Inc, Yang Jihong <yangjihong1@huawei.com> |
6 | */ |
7 | |
8 | #include <time.h> |
9 | #include <fcntl.h> |
10 | #include <signal.h> |
11 | #include <stdio.h> |
12 | #include <unistd.h> |
13 | |
14 | #include <linux/time64.h> |
15 | |
16 | #include "util/debug.h" |
17 | #include "util/evsel.h" |
18 | #include "util/kwork.h" |
19 | |
20 | #include <bpf/bpf.h> |
21 | #include <perf/cpumap.h> |
22 | |
23 | #include "util/bpf_skel/kwork_trace.skel.h" |
24 | |
25 | /* |
26 | * This should be in sync with "util/kwork_trace.bpf.c" |
27 | */ |
28 | #define MAX_KWORKNAME 128 |
29 | |
30 | struct work_key { |
31 | u32 type; |
32 | u32 cpu; |
33 | u64 id; |
34 | }; |
35 | |
36 | struct report_data { |
37 | u64 nr; |
38 | u64 total_time; |
39 | u64 max_time; |
40 | u64 max_time_start; |
41 | u64 max_time_end; |
42 | }; |
43 | |
44 | struct kwork_class_bpf { |
45 | struct kwork_class *class; |
46 | |
47 | void (*load_prepare)(struct perf_kwork *kwork); |
48 | int (*get_work_name)(struct work_key *key, char **ret_name); |
49 | }; |
50 | |
51 | static struct kwork_trace_bpf *skel; |
52 | |
53 | static struct timespec ts_start; |
54 | static struct timespec ts_end; |
55 | |
56 | void perf_kwork__trace_start(void) |
57 | { |
58 | clock_gettime(CLOCK_MONOTONIC, &ts_start); |
59 | skel->bss->enabled = 1; |
60 | } |
61 | |
62 | void perf_kwork__trace_finish(void) |
63 | { |
64 | clock_gettime(CLOCK_MONOTONIC, &ts_end); |
65 | skel->bss->enabled = 0; |
66 | } |
67 | |
68 | static int get_work_name_from_map(struct work_key *key, char **ret_name) |
69 | { |
70 | char name[MAX_KWORKNAME] = { 0 }; |
71 | int fd = bpf_map__fd(skel->maps.perf_kwork_names); |
72 | |
73 | *ret_name = NULL; |
74 | |
75 | if (fd < 0) { |
76 | pr_debug("Invalid names map fd\n" ); |
77 | return 0; |
78 | } |
79 | |
80 | if ((bpf_map_lookup_elem(fd, key, name) == 0) && (strlen(name) != 0)) { |
81 | *ret_name = strdup(name); |
82 | if (*ret_name == NULL) { |
83 | pr_err("Failed to copy work name\n" ); |
84 | return -1; |
85 | } |
86 | } |
87 | |
88 | return 0; |
89 | } |
90 | |
91 | static void irq_load_prepare(struct perf_kwork *kwork) |
92 | { |
93 | if (kwork->report == KWORK_REPORT_RUNTIME) { |
94 | bpf_program__set_autoload(skel->progs.report_irq_handler_entry, true); |
95 | bpf_program__set_autoload(skel->progs.report_irq_handler_exit, true); |
96 | } |
97 | } |
98 | |
99 | static struct kwork_class_bpf kwork_irq_bpf = { |
100 | .load_prepare = irq_load_prepare, |
101 | .get_work_name = get_work_name_from_map, |
102 | }; |
103 | |
104 | static void softirq_load_prepare(struct perf_kwork *kwork) |
105 | { |
106 | if (kwork->report == KWORK_REPORT_RUNTIME) { |
107 | bpf_program__set_autoload(skel->progs.report_softirq_entry, true); |
108 | bpf_program__set_autoload(skel->progs.report_softirq_exit, true); |
109 | } else if (kwork->report == KWORK_REPORT_LATENCY) { |
110 | bpf_program__set_autoload(skel->progs.latency_softirq_raise, true); |
111 | bpf_program__set_autoload(skel->progs.latency_softirq_entry, true); |
112 | } |
113 | } |
114 | |
115 | static struct kwork_class_bpf kwork_softirq_bpf = { |
116 | .load_prepare = softirq_load_prepare, |
117 | .get_work_name = get_work_name_from_map, |
118 | }; |
119 | |
120 | static void workqueue_load_prepare(struct perf_kwork *kwork) |
121 | { |
122 | if (kwork->report == KWORK_REPORT_RUNTIME) { |
123 | bpf_program__set_autoload(skel->progs.report_workqueue_execute_start, true); |
124 | bpf_program__set_autoload(skel->progs.report_workqueue_execute_end, true); |
125 | } else if (kwork->report == KWORK_REPORT_LATENCY) { |
126 | bpf_program__set_autoload(skel->progs.latency_workqueue_activate_work, true); |
127 | bpf_program__set_autoload(skel->progs.latency_workqueue_execute_start, true); |
128 | } |
129 | } |
130 | |
131 | static struct kwork_class_bpf kwork_workqueue_bpf = { |
132 | .load_prepare = workqueue_load_prepare, |
133 | .get_work_name = get_work_name_from_map, |
134 | }; |
135 | |
136 | static struct kwork_class_bpf * |
137 | kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { |
138 | [KWORK_CLASS_IRQ] = &kwork_irq_bpf, |
139 | [KWORK_CLASS_SOFTIRQ] = &kwork_softirq_bpf, |
140 | [KWORK_CLASS_WORKQUEUE] = &kwork_workqueue_bpf, |
141 | }; |
142 | |
143 | static bool valid_kwork_class_type(enum kwork_class_type type) |
144 | { |
145 | return type >= 0 && type < KWORK_CLASS_MAX ? true : false; |
146 | } |
147 | |
148 | static int setup_filters(struct perf_kwork *kwork) |
149 | { |
150 | u8 val = 1; |
151 | int i, nr_cpus, key, fd; |
152 | struct perf_cpu_map *map; |
153 | |
154 | if (kwork->cpu_list != NULL) { |
155 | fd = bpf_map__fd(skel->maps.perf_kwork_cpu_filter); |
156 | if (fd < 0) { |
157 | pr_debug("Invalid cpu filter fd\n" ); |
158 | return -1; |
159 | } |
160 | |
161 | map = perf_cpu_map__new(kwork->cpu_list); |
162 | if (map == NULL) { |
163 | pr_debug("Invalid cpu_list\n" ); |
164 | return -1; |
165 | } |
166 | |
167 | nr_cpus = libbpf_num_possible_cpus(); |
168 | for (i = 0; i < perf_cpu_map__nr(map); i++) { |
169 | struct perf_cpu cpu = perf_cpu_map__cpu(map, i); |
170 | |
171 | if (cpu.cpu >= nr_cpus) { |
172 | perf_cpu_map__put(map); |
173 | pr_err("Requested cpu %d too large\n" , cpu.cpu); |
174 | return -1; |
175 | } |
176 | bpf_map_update_elem(fd, &cpu.cpu, &val, BPF_ANY); |
177 | } |
178 | perf_cpu_map__put(map); |
179 | |
180 | skel->bss->has_cpu_filter = 1; |
181 | } |
182 | |
183 | if (kwork->profile_name != NULL) { |
184 | if (strlen(kwork->profile_name) >= MAX_KWORKNAME) { |
185 | pr_err("Requested name filter %s too large, limit to %d\n" , |
186 | kwork->profile_name, MAX_KWORKNAME - 1); |
187 | return -1; |
188 | } |
189 | |
190 | fd = bpf_map__fd(skel->maps.perf_kwork_name_filter); |
191 | if (fd < 0) { |
192 | pr_debug("Invalid name filter fd\n" ); |
193 | return -1; |
194 | } |
195 | |
196 | key = 0; |
197 | bpf_map_update_elem(fd, &key, kwork->profile_name, BPF_ANY); |
198 | |
199 | skel->bss->has_name_filter = 1; |
200 | } |
201 | |
202 | return 0; |
203 | } |
204 | |
205 | int perf_kwork__trace_prepare_bpf(struct perf_kwork *kwork) |
206 | { |
207 | struct bpf_program *prog; |
208 | struct kwork_class *class; |
209 | struct kwork_class_bpf *class_bpf; |
210 | enum kwork_class_type type; |
211 | |
212 | skel = kwork_trace_bpf__open(); |
213 | if (!skel) { |
214 | pr_debug("Failed to open kwork trace skeleton\n" ); |
215 | return -1; |
216 | } |
217 | |
218 | /* |
219 | * set all progs to non-autoload, |
220 | * then set corresponding progs according to config |
221 | */ |
222 | bpf_object__for_each_program(prog, skel->obj) |
223 | bpf_program__set_autoload(prog, false); |
224 | |
225 | list_for_each_entry(class, &kwork->class_list, list) { |
226 | type = class->type; |
227 | if (!valid_kwork_class_type(type: type) || |
228 | (kwork_class_bpf_supported_list[type] == NULL)) { |
229 | pr_err("Unsupported bpf trace class %s\n" , class->name); |
230 | goto out; |
231 | } |
232 | |
233 | class_bpf = kwork_class_bpf_supported_list[type]; |
234 | class_bpf->class = class; |
235 | |
236 | if (class_bpf->load_prepare != NULL) |
237 | class_bpf->load_prepare(kwork); |
238 | } |
239 | |
240 | if (kwork_trace_bpf__load(skel)) { |
241 | pr_debug("Failed to load kwork trace skeleton\n" ); |
242 | goto out; |
243 | } |
244 | |
245 | if (setup_filters(kwork)) |
246 | goto out; |
247 | |
248 | if (kwork_trace_bpf__attach(skel)) { |
249 | pr_debug("Failed to attach kwork trace skeleton\n" ); |
250 | goto out; |
251 | } |
252 | |
253 | return 0; |
254 | |
255 | out: |
256 | kwork_trace_bpf__destroy(skel); |
257 | return -1; |
258 | } |
259 | |
260 | static int add_work(struct perf_kwork *kwork, |
261 | struct work_key *key, |
262 | struct report_data *data) |
263 | { |
264 | struct kwork_work *work; |
265 | struct kwork_class_bpf *bpf_trace; |
266 | struct kwork_work tmp = { |
267 | .id = key->id, |
268 | .name = NULL, |
269 | .cpu = key->cpu, |
270 | }; |
271 | enum kwork_class_type type = key->type; |
272 | |
273 | if (!valid_kwork_class_type(type: type)) { |
274 | pr_debug("Invalid class type %d to add work\n" , type); |
275 | return -1; |
276 | } |
277 | |
278 | bpf_trace = kwork_class_bpf_supported_list[type]; |
279 | tmp.class = bpf_trace->class; |
280 | |
281 | if ((bpf_trace->get_work_name != NULL) && |
282 | (bpf_trace->get_work_name(key, &tmp.name))) |
283 | return -1; |
284 | |
285 | work = perf_kwork_add_work(kwork, tmp.class, &tmp); |
286 | if (work == NULL) |
287 | return -1; |
288 | |
289 | if (kwork->report == KWORK_REPORT_RUNTIME) { |
290 | work->nr_atoms = data->nr; |
291 | work->total_runtime = data->total_time; |
292 | work->max_runtime = data->max_time; |
293 | work->max_runtime_start = data->max_time_start; |
294 | work->max_runtime_end = data->max_time_end; |
295 | } else if (kwork->report == KWORK_REPORT_LATENCY) { |
296 | work->nr_atoms = data->nr; |
297 | work->total_latency = data->total_time; |
298 | work->max_latency = data->max_time; |
299 | work->max_latency_start = data->max_time_start; |
300 | work->max_latency_end = data->max_time_end; |
301 | } else { |
302 | pr_debug("Invalid bpf report type %d\n" , kwork->report); |
303 | return -1; |
304 | } |
305 | |
306 | kwork->timestart = (u64)ts_start.tv_sec * NSEC_PER_SEC + ts_start.tv_nsec; |
307 | kwork->timeend = (u64)ts_end.tv_sec * NSEC_PER_SEC + ts_end.tv_nsec; |
308 | |
309 | return 0; |
310 | } |
311 | |
312 | int perf_kwork__report_read_bpf(struct perf_kwork *kwork) |
313 | { |
314 | struct report_data data; |
315 | struct work_key key = { |
316 | .type = 0, |
317 | .cpu = 0, |
318 | .id = 0, |
319 | }; |
320 | struct work_key prev = { |
321 | .type = 0, |
322 | .cpu = 0, |
323 | .id = 0, |
324 | }; |
325 | int fd = bpf_map__fd(skel->maps.perf_kwork_report); |
326 | |
327 | if (fd < 0) { |
328 | pr_debug("Invalid report fd\n" ); |
329 | return -1; |
330 | } |
331 | |
332 | while (!bpf_map_get_next_key(fd, &prev, &key)) { |
333 | if ((bpf_map_lookup_elem(fd, &key, &data)) != 0) { |
334 | pr_debug("Failed to lookup report elem\n" ); |
335 | return -1; |
336 | } |
337 | |
338 | if ((data.nr != 0) && (add_work(kwork, key: &key, data: &data) != 0)) |
339 | return -1; |
340 | |
341 | prev = key; |
342 | } |
343 | return 0; |
344 | } |
345 | |
346 | void perf_kwork__report_cleanup_bpf(void) |
347 | { |
348 | kwork_trace_bpf__destroy(skel); |
349 | } |
350 | |