1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * fprobe - Simple ftrace probe wrapper for function entry. |
4 | */ |
5 | #define pr_fmt(fmt) "fprobe: " fmt |
6 | |
7 | #include <linux/err.h> |
8 | #include <linux/fprobe.h> |
9 | #include <linux/kallsyms.h> |
10 | #include <linux/kprobes.h> |
11 | #include <linux/rethook.h> |
12 | #include <linux/slab.h> |
13 | #include <linux/sort.h> |
14 | |
15 | #include "trace.h" |
16 | |
17 | struct fprobe_rethook_node { |
18 | struct rethook_node node; |
19 | unsigned long entry_ip; |
20 | unsigned long entry_parent_ip; |
21 | char data[]; |
22 | }; |
23 | |
24 | static inline void __fprobe_handler(unsigned long ip, unsigned long parent_ip, |
25 | struct ftrace_ops *ops, struct ftrace_regs *fregs) |
26 | { |
27 | struct fprobe_rethook_node *fpr; |
28 | struct rethook_node *rh = NULL; |
29 | struct fprobe *fp; |
30 | void *entry_data = NULL; |
31 | int ret = 0; |
32 | |
33 | fp = container_of(ops, struct fprobe, ops); |
34 | |
35 | if (fp->exit_handler) { |
36 | rh = rethook_try_get(rh: fp->rethook); |
37 | if (!rh) { |
38 | fp->nmissed++; |
39 | return; |
40 | } |
41 | fpr = container_of(rh, struct fprobe_rethook_node, node); |
42 | fpr->entry_ip = ip; |
43 | fpr->entry_parent_ip = parent_ip; |
44 | if (fp->entry_data_size) |
45 | entry_data = fpr->data; |
46 | } |
47 | |
48 | if (fp->entry_handler) |
49 | ret = fp->entry_handler(fp, ip, parent_ip, ftrace_get_regs(fregs), entry_data); |
50 | |
51 | /* If entry_handler returns !0, nmissed is not counted. */ |
52 | if (rh) { |
53 | if (ret) |
54 | rethook_recycle(node: rh); |
55 | else |
56 | rethook_hook(node: rh, regs: ftrace_get_regs(fregs), mcount: true); |
57 | } |
58 | } |
59 | |
60 | static void fprobe_handler(unsigned long ip, unsigned long parent_ip, |
61 | struct ftrace_ops *ops, struct ftrace_regs *fregs) |
62 | { |
63 | struct fprobe *fp; |
64 | int bit; |
65 | |
66 | fp = container_of(ops, struct fprobe, ops); |
67 | if (fprobe_disabled(fp)) |
68 | return; |
69 | |
70 | /* recursion detection has to go before any traceable function and |
71 | * all functions before this point should be marked as notrace |
72 | */ |
73 | bit = ftrace_test_recursion_trylock(ip, parent_ip); |
74 | if (bit < 0) { |
75 | fp->nmissed++; |
76 | return; |
77 | } |
78 | __fprobe_handler(ip, parent_ip, ops, fregs); |
79 | ftrace_test_recursion_unlock(bit); |
80 | |
81 | } |
82 | NOKPROBE_SYMBOL(fprobe_handler); |
83 | |
84 | static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip, |
85 | struct ftrace_ops *ops, struct ftrace_regs *fregs) |
86 | { |
87 | struct fprobe *fp; |
88 | int bit; |
89 | |
90 | fp = container_of(ops, struct fprobe, ops); |
91 | if (fprobe_disabled(fp)) |
92 | return; |
93 | |
94 | /* recursion detection has to go before any traceable function and |
95 | * all functions called before this point should be marked as notrace |
96 | */ |
97 | bit = ftrace_test_recursion_trylock(ip, parent_ip); |
98 | if (bit < 0) { |
99 | fp->nmissed++; |
100 | return; |
101 | } |
102 | |
103 | /* |
104 | * This user handler is shared with other kprobes and is not expected to be |
105 | * called recursively. So if any other kprobe handler is running, this will |
106 | * exit as kprobe does. See the section 'Share the callbacks with kprobes' |
107 | * in Documentation/trace/fprobe.rst for more information. |
108 | */ |
109 | if (unlikely(kprobe_running())) { |
110 | fp->nmissed++; |
111 | goto recursion_unlock; |
112 | } |
113 | |
114 | kprobe_busy_begin(); |
115 | __fprobe_handler(ip, parent_ip, ops, fregs); |
116 | kprobe_busy_end(); |
117 | |
118 | recursion_unlock: |
119 | ftrace_test_recursion_unlock(bit); |
120 | } |
121 | |
122 | static void fprobe_exit_handler(struct rethook_node *rh, void *data, |
123 | unsigned long ret_ip, struct pt_regs *regs) |
124 | { |
125 | struct fprobe *fp = (struct fprobe *)data; |
126 | struct fprobe_rethook_node *fpr; |
127 | int bit; |
128 | |
129 | if (!fp || fprobe_disabled(fp)) |
130 | return; |
131 | |
132 | fpr = container_of(rh, struct fprobe_rethook_node, node); |
133 | |
134 | /* |
135 | * we need to assure no calls to traceable functions in-between the |
136 | * end of fprobe_handler and the beginning of fprobe_exit_handler. |
137 | */ |
138 | bit = ftrace_test_recursion_trylock(ip: fpr->entry_ip, parent_ip: fpr->entry_parent_ip); |
139 | if (bit < 0) { |
140 | fp->nmissed++; |
141 | return; |
142 | } |
143 | |
144 | fp->exit_handler(fp, fpr->entry_ip, ret_ip, regs, |
145 | fp->entry_data_size ? (void *)fpr->data : NULL); |
146 | ftrace_test_recursion_unlock(bit); |
147 | } |
148 | NOKPROBE_SYMBOL(fprobe_exit_handler); |
149 | |
150 | static int symbols_cmp(const void *a, const void *b) |
151 | { |
152 | const char **str_a = (const char **) a; |
153 | const char **str_b = (const char **) b; |
154 | |
155 | return strcmp(*str_a, *str_b); |
156 | } |
157 | |
158 | /* Convert ftrace location address from symbols */ |
159 | static unsigned long *get_ftrace_locations(const char **syms, int num) |
160 | { |
161 | unsigned long *addrs; |
162 | |
163 | /* Convert symbols to symbol address */ |
164 | addrs = kcalloc(n: num, size: sizeof(*addrs), GFP_KERNEL); |
165 | if (!addrs) |
166 | return ERR_PTR(error: -ENOMEM); |
167 | |
168 | /* ftrace_lookup_symbols expects sorted symbols */ |
169 | sort(base: syms, num, size: sizeof(*syms), cmp_func: symbols_cmp, NULL); |
170 | |
171 | if (!ftrace_lookup_symbols(sorted_syms: syms, cnt: num, addrs)) |
172 | return addrs; |
173 | |
174 | kfree(objp: addrs); |
175 | return ERR_PTR(error: -ENOENT); |
176 | } |
177 | |
178 | static void fprobe_init(struct fprobe *fp) |
179 | { |
180 | fp->nmissed = 0; |
181 | if (fprobe_shared_with_kprobes(fp)) |
182 | fp->ops.func = fprobe_kprobe_handler; |
183 | else |
184 | fp->ops.func = fprobe_handler; |
185 | fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS; |
186 | } |
187 | |
188 | static int fprobe_init_rethook(struct fprobe *fp, int num) |
189 | { |
190 | int size; |
191 | |
192 | if (!fp->exit_handler) { |
193 | fp->rethook = NULL; |
194 | return 0; |
195 | } |
196 | |
197 | /* Initialize rethook if needed */ |
198 | if (fp->nr_maxactive) |
199 | num = fp->nr_maxactive; |
200 | else |
201 | num *= num_possible_cpus() * 2; |
202 | if (num <= 0) |
203 | return -EINVAL; |
204 | |
205 | size = sizeof(struct fprobe_rethook_node) + fp->entry_data_size; |
206 | |
207 | /* Initialize rethook */ |
208 | fp->rethook = rethook_alloc(data: (void *)fp, handler: fprobe_exit_handler, size, num); |
209 | if (IS_ERR(ptr: fp->rethook)) |
210 | return PTR_ERR(ptr: fp->rethook); |
211 | |
212 | return 0; |
213 | } |
214 | |
215 | static void fprobe_fail_cleanup(struct fprobe *fp) |
216 | { |
217 | if (!IS_ERR_OR_NULL(ptr: fp->rethook)) { |
218 | /* Don't need to cleanup rethook->handler because this is not used. */ |
219 | rethook_free(rh: fp->rethook); |
220 | fp->rethook = NULL; |
221 | } |
222 | ftrace_free_filter(ops: &fp->ops); |
223 | } |
224 | |
225 | /** |
226 | * register_fprobe() - Register fprobe to ftrace by pattern. |
227 | * @fp: A fprobe data structure to be registered. |
228 | * @filter: A wildcard pattern of probed symbols. |
229 | * @notfilter: A wildcard pattern of NOT probed symbols. |
230 | * |
231 | * Register @fp to ftrace for enabling the probe on the symbols matched to @filter. |
232 | * If @notfilter is not NULL, the symbols matched the @notfilter are not probed. |
233 | * |
234 | * Return 0 if @fp is registered successfully, -errno if not. |
235 | */ |
236 | int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter) |
237 | { |
238 | struct ftrace_hash *hash; |
239 | unsigned char *str; |
240 | int ret, len; |
241 | |
242 | if (!fp || !filter) |
243 | return -EINVAL; |
244 | |
245 | fprobe_init(fp); |
246 | |
247 | len = strlen(filter); |
248 | str = kstrdup(s: filter, GFP_KERNEL); |
249 | ret = ftrace_set_filter(ops: &fp->ops, buf: str, len, reset: 0); |
250 | kfree(objp: str); |
251 | if (ret) |
252 | return ret; |
253 | |
254 | if (notfilter) { |
255 | len = strlen(notfilter); |
256 | str = kstrdup(s: notfilter, GFP_KERNEL); |
257 | ret = ftrace_set_notrace(ops: &fp->ops, buf: str, len, reset: 0); |
258 | kfree(objp: str); |
259 | if (ret) |
260 | goto out; |
261 | } |
262 | |
263 | /* TODO: |
264 | * correctly calculate the total number of filtered symbols |
265 | * from both filter and notfilter. |
266 | */ |
267 | hash = rcu_access_pointer(fp->ops.local_hash.filter_hash); |
268 | if (WARN_ON_ONCE(!hash)) |
269 | goto out; |
270 | |
271 | ret = fprobe_init_rethook(fp, num: (int)hash->count); |
272 | if (!ret) |
273 | ret = register_ftrace_function(ops: &fp->ops); |
274 | |
275 | out: |
276 | if (ret) |
277 | fprobe_fail_cleanup(fp); |
278 | return ret; |
279 | } |
280 | EXPORT_SYMBOL_GPL(register_fprobe); |
281 | |
282 | /** |
283 | * register_fprobe_ips() - Register fprobe to ftrace by address. |
284 | * @fp: A fprobe data structure to be registered. |
285 | * @addrs: An array of target ftrace location addresses. |
286 | * @num: The number of entries of @addrs. |
287 | * |
288 | * Register @fp to ftrace for enabling the probe on the address given by @addrs. |
289 | * The @addrs must be the addresses of ftrace location address, which may be |
290 | * the symbol address + arch-dependent offset. |
291 | * If you unsure what this mean, please use other registration functions. |
292 | * |
293 | * Return 0 if @fp is registered successfully, -errno if not. |
294 | */ |
295 | int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num) |
296 | { |
297 | int ret; |
298 | |
299 | if (!fp || !addrs || num <= 0) |
300 | return -EINVAL; |
301 | |
302 | fprobe_init(fp); |
303 | |
304 | ret = ftrace_set_filter_ips(ops: &fp->ops, ips: addrs, cnt: num, remove: 0, reset: 0); |
305 | if (ret) |
306 | return ret; |
307 | |
308 | ret = fprobe_init_rethook(fp, num); |
309 | if (!ret) |
310 | ret = register_ftrace_function(ops: &fp->ops); |
311 | |
312 | if (ret) |
313 | fprobe_fail_cleanup(fp); |
314 | return ret; |
315 | } |
316 | EXPORT_SYMBOL_GPL(register_fprobe_ips); |
317 | |
318 | /** |
319 | * register_fprobe_syms() - Register fprobe to ftrace by symbols. |
320 | * @fp: A fprobe data structure to be registered. |
321 | * @syms: An array of target symbols. |
322 | * @num: The number of entries of @syms. |
323 | * |
324 | * Register @fp to the symbols given by @syms array. This will be useful if |
325 | * you are sure the symbols exist in the kernel. |
326 | * |
327 | * Return 0 if @fp is registered successfully, -errno if not. |
328 | */ |
329 | int register_fprobe_syms(struct fprobe *fp, const char **syms, int num) |
330 | { |
331 | unsigned long *addrs; |
332 | int ret; |
333 | |
334 | if (!fp || !syms || num <= 0) |
335 | return -EINVAL; |
336 | |
337 | addrs = get_ftrace_locations(syms, num); |
338 | if (IS_ERR(ptr: addrs)) |
339 | return PTR_ERR(ptr: addrs); |
340 | |
341 | ret = register_fprobe_ips(fp, addrs, num); |
342 | |
343 | kfree(objp: addrs); |
344 | |
345 | return ret; |
346 | } |
347 | EXPORT_SYMBOL_GPL(register_fprobe_syms); |
348 | |
349 | bool fprobe_is_registered(struct fprobe *fp) |
350 | { |
351 | if (!fp || (fp->ops.saved_func != fprobe_handler && |
352 | fp->ops.saved_func != fprobe_kprobe_handler)) |
353 | return false; |
354 | return true; |
355 | } |
356 | |
357 | /** |
358 | * unregister_fprobe() - Unregister fprobe from ftrace |
359 | * @fp: A fprobe data structure to be unregistered. |
360 | * |
361 | * Unregister fprobe (and remove ftrace hooks from the function entries). |
362 | * |
363 | * Return 0 if @fp is unregistered successfully, -errno if not. |
364 | */ |
365 | int unregister_fprobe(struct fprobe *fp) |
366 | { |
367 | int ret; |
368 | |
369 | if (!fprobe_is_registered(fp)) |
370 | return -EINVAL; |
371 | |
372 | if (!IS_ERR_OR_NULL(ptr: fp->rethook)) |
373 | rethook_stop(rh: fp->rethook); |
374 | |
375 | ret = unregister_ftrace_function(ops: &fp->ops); |
376 | if (ret < 0) |
377 | return ret; |
378 | |
379 | if (!IS_ERR_OR_NULL(ptr: fp->rethook)) |
380 | rethook_free(rh: fp->rethook); |
381 | |
382 | ftrace_free_filter(ops: &fp->ops); |
383 | |
384 | return ret; |
385 | } |
386 | EXPORT_SYMBOL_GPL(unregister_fprobe); |
387 | |