1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (c) 2022 Google */ |
3 | #include <linux/bpf.h> |
4 | #include <linux/btf_ids.h> |
5 | #include <linux/cgroup.h> |
6 | #include <linux/kernel.h> |
7 | #include <linux/seq_file.h> |
8 | |
9 | #include "../cgroup/cgroup-internal.h" /* cgroup_mutex and cgroup_is_dead */ |
10 | |
11 | /* cgroup_iter provides four modes of traversal to the cgroup hierarchy. |
12 | * |
13 | * 1. Walk the descendants of a cgroup in pre-order. |
14 | * 2. Walk the descendants of a cgroup in post-order. |
15 | * 3. Walk the ancestors of a cgroup. |
16 | * 4. Show the given cgroup only. |
17 | * |
18 | * For walking descendants, cgroup_iter can walk in either pre-order or |
19 | * post-order. For walking ancestors, the iter walks up from a cgroup to |
20 | * the root. |
21 | * |
22 | * The iter program can terminate the walk early by returning 1. Walk |
23 | * continues if prog returns 0. |
24 | * |
25 | * The prog can check (seq->num == 0) to determine whether this is |
26 | * the first element. The prog may also be passed a NULL cgroup, |
27 | * which means the walk has completed and the prog has a chance to |
28 | * do post-processing, such as outputting an epilogue. |
29 | * |
30 | * Note: the iter_prog is called with cgroup_mutex held. |
31 | * |
32 | * Currently only one session is supported, which means, depending on the |
33 | * volume of data bpf program intends to send to user space, the number |
34 | * of cgroups that can be walked is limited. For example, given the current |
35 | * buffer size is 8 * PAGE_SIZE, if the program sends 64B data for each |
36 | * cgroup, assuming PAGE_SIZE is 4kb, the total number of cgroups that can |
37 | * be walked is 512. This is a limitation of cgroup_iter. If the output data |
38 | * is larger than the kernel buffer size, after all data in the kernel buffer |
39 | * is consumed by user space, the subsequent read() syscall will signal |
40 | * EOPNOTSUPP. In order to work around, the user may have to update their |
41 | * program to reduce the volume of data sent to output. For example, skip |
42 | * some uninteresting cgroups. |
43 | */ |
44 | |
45 | struct bpf_iter__cgroup { |
46 | __bpf_md_ptr(struct bpf_iter_meta *, meta); |
47 | __bpf_md_ptr(struct cgroup *, cgroup); |
48 | }; |
49 | |
50 | struct cgroup_iter_priv { |
51 | struct cgroup_subsys_state *start_css; |
52 | bool visited_all; |
53 | bool terminate; |
54 | int order; |
55 | }; |
56 | |
57 | static void *cgroup_iter_seq_start(struct seq_file *seq, loff_t *pos) |
58 | { |
59 | struct cgroup_iter_priv *p = seq->private; |
60 | |
61 | cgroup_lock(); |
62 | |
63 | /* cgroup_iter doesn't support read across multiple sessions. */ |
64 | if (*pos > 0) { |
65 | if (p->visited_all) |
66 | return NULL; |
67 | |
68 | /* Haven't visited all, but because cgroup_mutex has dropped, |
69 | * return -EOPNOTSUPP to indicate incomplete iteration. |
70 | */ |
71 | return ERR_PTR(error: -EOPNOTSUPP); |
72 | } |
73 | |
74 | ++*pos; |
75 | p->terminate = false; |
76 | p->visited_all = false; |
77 | if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE) |
78 | return css_next_descendant_pre(NULL, css: p->start_css); |
79 | else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST) |
80 | return css_next_descendant_post(NULL, css: p->start_css); |
81 | else /* BPF_CGROUP_ITER_SELF_ONLY and BPF_CGROUP_ITER_ANCESTORS_UP */ |
82 | return p->start_css; |
83 | } |
84 | |
85 | static int __cgroup_iter_seq_show(struct seq_file *seq, |
86 | struct cgroup_subsys_state *css, int in_stop); |
87 | |
88 | static void cgroup_iter_seq_stop(struct seq_file *seq, void *v) |
89 | { |
90 | struct cgroup_iter_priv *p = seq->private; |
91 | |
92 | cgroup_unlock(); |
93 | |
94 | /* pass NULL to the prog for post-processing */ |
95 | if (!v) { |
96 | __cgroup_iter_seq_show(seq, NULL, in_stop: true); |
97 | p->visited_all = true; |
98 | } |
99 | } |
100 | |
101 | static void *cgroup_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos) |
102 | { |
103 | struct cgroup_subsys_state *curr = (struct cgroup_subsys_state *)v; |
104 | struct cgroup_iter_priv *p = seq->private; |
105 | |
106 | ++*pos; |
107 | if (p->terminate) |
108 | return NULL; |
109 | |
110 | if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE) |
111 | return css_next_descendant_pre(pos: curr, css: p->start_css); |
112 | else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST) |
113 | return css_next_descendant_post(pos: curr, css: p->start_css); |
114 | else if (p->order == BPF_CGROUP_ITER_ANCESTORS_UP) |
115 | return curr->parent; |
116 | else /* BPF_CGROUP_ITER_SELF_ONLY */ |
117 | return NULL; |
118 | } |
119 | |
120 | static int __cgroup_iter_seq_show(struct seq_file *seq, |
121 | struct cgroup_subsys_state *css, int in_stop) |
122 | { |
123 | struct cgroup_iter_priv *p = seq->private; |
124 | struct bpf_iter__cgroup ctx; |
125 | struct bpf_iter_meta meta; |
126 | struct bpf_prog *prog; |
127 | int ret = 0; |
128 | |
129 | /* cgroup is dead, skip this element */ |
130 | if (css && cgroup_is_dead(cgrp: css->cgroup)) |
131 | return 0; |
132 | |
133 | ctx.meta = &meta; |
134 | ctx.cgroup = css ? css->cgroup : NULL; |
135 | meta.seq = seq; |
136 | prog = bpf_iter_get_info(meta: &meta, in_stop); |
137 | if (prog) |
138 | ret = bpf_iter_run_prog(prog, ctx: &ctx); |
139 | |
140 | /* if prog returns > 0, terminate after this element. */ |
141 | if (ret != 0) |
142 | p->terminate = true; |
143 | |
144 | return 0; |
145 | } |
146 | |
147 | static int cgroup_iter_seq_show(struct seq_file *seq, void *v) |
148 | { |
149 | return __cgroup_iter_seq_show(seq, css: (struct cgroup_subsys_state *)v, |
150 | in_stop: false); |
151 | } |
152 | |
153 | static const struct seq_operations cgroup_iter_seq_ops = { |
154 | .start = cgroup_iter_seq_start, |
155 | .next = cgroup_iter_seq_next, |
156 | .stop = cgroup_iter_seq_stop, |
157 | .show = cgroup_iter_seq_show, |
158 | }; |
159 | |
160 | BTF_ID_LIST_GLOBAL_SINGLE(bpf_cgroup_btf_id, struct, cgroup) |
161 | |
162 | static int cgroup_iter_seq_init(void *priv, struct bpf_iter_aux_info *aux) |
163 | { |
164 | struct cgroup_iter_priv *p = (struct cgroup_iter_priv *)priv; |
165 | struct cgroup *cgrp = aux->cgroup.start; |
166 | |
167 | /* bpf_iter_attach_cgroup() has already acquired an extra reference |
168 | * for the start cgroup, but the reference may be released after |
169 | * cgroup_iter_seq_init(), so acquire another reference for the |
170 | * start cgroup. |
171 | */ |
172 | p->start_css = &cgrp->self; |
173 | css_get(css: p->start_css); |
174 | p->terminate = false; |
175 | p->visited_all = false; |
176 | p->order = aux->cgroup.order; |
177 | return 0; |
178 | } |
179 | |
180 | static void cgroup_iter_seq_fini(void *priv) |
181 | { |
182 | struct cgroup_iter_priv *p = (struct cgroup_iter_priv *)priv; |
183 | |
184 | css_put(css: p->start_css); |
185 | } |
186 | |
187 | static const struct bpf_iter_seq_info cgroup_iter_seq_info = { |
188 | .seq_ops = &cgroup_iter_seq_ops, |
189 | .init_seq_private = cgroup_iter_seq_init, |
190 | .fini_seq_private = cgroup_iter_seq_fini, |
191 | .seq_priv_size = sizeof(struct cgroup_iter_priv), |
192 | }; |
193 | |
194 | static int bpf_iter_attach_cgroup(struct bpf_prog *prog, |
195 | union bpf_iter_link_info *linfo, |
196 | struct bpf_iter_aux_info *aux) |
197 | { |
198 | int fd = linfo->cgroup.cgroup_fd; |
199 | u64 id = linfo->cgroup.cgroup_id; |
200 | int order = linfo->cgroup.order; |
201 | struct cgroup *cgrp; |
202 | |
203 | if (order != BPF_CGROUP_ITER_DESCENDANTS_PRE && |
204 | order != BPF_CGROUP_ITER_DESCENDANTS_POST && |
205 | order != BPF_CGROUP_ITER_ANCESTORS_UP && |
206 | order != BPF_CGROUP_ITER_SELF_ONLY) |
207 | return -EINVAL; |
208 | |
209 | if (fd && id) |
210 | return -EINVAL; |
211 | |
212 | if (fd) |
213 | cgrp = cgroup_v1v2_get_from_fd(fd); |
214 | else if (id) |
215 | cgrp = cgroup_get_from_id(id); |
216 | else /* walk the entire hierarchy by default. */ |
217 | cgrp = cgroup_get_from_path(path: "/" ); |
218 | |
219 | if (IS_ERR(ptr: cgrp)) |
220 | return PTR_ERR(ptr: cgrp); |
221 | |
222 | aux->cgroup.start = cgrp; |
223 | aux->cgroup.order = order; |
224 | return 0; |
225 | } |
226 | |
227 | static void bpf_iter_detach_cgroup(struct bpf_iter_aux_info *aux) |
228 | { |
229 | cgroup_put(cgrp: aux->cgroup.start); |
230 | } |
231 | |
232 | static void bpf_iter_cgroup_show_fdinfo(const struct bpf_iter_aux_info *aux, |
233 | struct seq_file *seq) |
234 | { |
235 | char *buf; |
236 | |
237 | buf = kzalloc(PATH_MAX, GFP_KERNEL); |
238 | if (!buf) { |
239 | seq_puts(m: seq, s: "cgroup_path:\t<unknown>\n" ); |
240 | goto show_order; |
241 | } |
242 | |
243 | /* If cgroup_path_ns() fails, buf will be an empty string, cgroup_path |
244 | * will print nothing. |
245 | * |
246 | * Path is in the calling process's cgroup namespace. |
247 | */ |
248 | cgroup_path_ns(cgrp: aux->cgroup.start, buf, PATH_MAX, |
249 | current->nsproxy->cgroup_ns); |
250 | seq_printf(m: seq, fmt: "cgroup_path:\t%s\n" , buf); |
251 | kfree(objp: buf); |
252 | |
253 | show_order: |
254 | if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_PRE) |
255 | seq_puts(m: seq, s: "order: descendants_pre\n" ); |
256 | else if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_POST) |
257 | seq_puts(m: seq, s: "order: descendants_post\n" ); |
258 | else if (aux->cgroup.order == BPF_CGROUP_ITER_ANCESTORS_UP) |
259 | seq_puts(m: seq, s: "order: ancestors_up\n" ); |
260 | else /* BPF_CGROUP_ITER_SELF_ONLY */ |
261 | seq_puts(m: seq, s: "order: self_only\n" ); |
262 | } |
263 | |
264 | static int bpf_iter_cgroup_fill_link_info(const struct bpf_iter_aux_info *aux, |
265 | struct bpf_link_info *info) |
266 | { |
267 | info->iter.cgroup.order = aux->cgroup.order; |
268 | info->iter.cgroup.cgroup_id = cgroup_id(cgrp: aux->cgroup.start); |
269 | return 0; |
270 | } |
271 | |
272 | DEFINE_BPF_ITER_FUNC(cgroup, struct bpf_iter_meta *meta, |
273 | struct cgroup *cgroup) |
274 | |
275 | static struct bpf_iter_reg bpf_cgroup_reg_info = { |
276 | .target = "cgroup" , |
277 | .feature = BPF_ITER_RESCHED, |
278 | .attach_target = bpf_iter_attach_cgroup, |
279 | .detach_target = bpf_iter_detach_cgroup, |
280 | .show_fdinfo = bpf_iter_cgroup_show_fdinfo, |
281 | .fill_link_info = bpf_iter_cgroup_fill_link_info, |
282 | .ctx_arg_info_size = 1, |
283 | .ctx_arg_info = { |
284 | { offsetof(struct bpf_iter__cgroup, cgroup), |
285 | PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED }, |
286 | }, |
287 | .seq_info = &cgroup_iter_seq_info, |
288 | }; |
289 | |
290 | static int __init bpf_cgroup_iter_init(void) |
291 | { |
292 | bpf_cgroup_reg_info.ctx_arg_info[0].btf_id = bpf_cgroup_btf_id[0]; |
293 | return bpf_iter_reg_target(reg_info: &bpf_cgroup_reg_info); |
294 | } |
295 | |
296 | late_initcall(bpf_cgroup_iter_init); |
297 | |
298 | struct bpf_iter_css { |
299 | __u64 __opaque[3]; |
300 | } __attribute__((aligned(8))); |
301 | |
302 | struct bpf_iter_css_kern { |
303 | struct cgroup_subsys_state *start; |
304 | struct cgroup_subsys_state *pos; |
305 | unsigned int flags; |
306 | } __attribute__((aligned(8))); |
307 | |
308 | __bpf_kfunc_start_defs(); |
309 | |
310 | __bpf_kfunc int bpf_iter_css_new(struct bpf_iter_css *it, |
311 | struct cgroup_subsys_state *start, unsigned int flags) |
312 | { |
313 | struct bpf_iter_css_kern *kit = (void *)it; |
314 | |
315 | BUILD_BUG_ON(sizeof(struct bpf_iter_css_kern) > sizeof(struct bpf_iter_css)); |
316 | BUILD_BUG_ON(__alignof__(struct bpf_iter_css_kern) != __alignof__(struct bpf_iter_css)); |
317 | |
318 | kit->start = NULL; |
319 | switch (flags) { |
320 | case BPF_CGROUP_ITER_DESCENDANTS_PRE: |
321 | case BPF_CGROUP_ITER_DESCENDANTS_POST: |
322 | case BPF_CGROUP_ITER_ANCESTORS_UP: |
323 | break; |
324 | default: |
325 | return -EINVAL; |
326 | } |
327 | |
328 | kit->start = start; |
329 | kit->pos = NULL; |
330 | kit->flags = flags; |
331 | return 0; |
332 | } |
333 | |
334 | __bpf_kfunc struct cgroup_subsys_state *bpf_iter_css_next(struct bpf_iter_css *it) |
335 | { |
336 | struct bpf_iter_css_kern *kit = (void *)it; |
337 | |
338 | if (!kit->start) |
339 | return NULL; |
340 | |
341 | switch (kit->flags) { |
342 | case BPF_CGROUP_ITER_DESCENDANTS_PRE: |
343 | kit->pos = css_next_descendant_pre(pos: kit->pos, css: kit->start); |
344 | break; |
345 | case BPF_CGROUP_ITER_DESCENDANTS_POST: |
346 | kit->pos = css_next_descendant_post(pos: kit->pos, css: kit->start); |
347 | break; |
348 | case BPF_CGROUP_ITER_ANCESTORS_UP: |
349 | kit->pos = kit->pos ? kit->pos->parent : kit->start; |
350 | } |
351 | |
352 | return kit->pos; |
353 | } |
354 | |
355 | __bpf_kfunc void bpf_iter_css_destroy(struct bpf_iter_css *it) |
356 | { |
357 | } |
358 | |
359 | __bpf_kfunc_end_defs(); |
360 | |