1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | |
3 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
4 | |
5 | #include <linux/ftrace.h> |
6 | #include <linux/ktime.h> |
7 | #include <linux/module.h> |
8 | |
9 | #include <asm/barrier.h> |
10 | |
11 | /* |
12 | * Arbitrary large value chosen to be sufficiently large to minimize noise but |
13 | * sufficiently small to complete quickly. |
14 | */ |
15 | static unsigned int nr_function_calls = 100000; |
16 | module_param(nr_function_calls, uint, 0); |
17 | MODULE_PARM_DESC(nr_function_calls, "How many times to call the relevant tracee" ); |
18 | |
19 | /* |
20 | * The number of ops associated with a call site affects whether a tracer can |
21 | * be called directly or whether it's necessary to go via the list func, which |
22 | * can be significantly more expensive. |
23 | */ |
24 | static unsigned int nr_ops_relevant = 1; |
25 | module_param(nr_ops_relevant, uint, 0); |
26 | MODULE_PARM_DESC(nr_ops_relevant, "How many ftrace_ops to associate with the relevant tracee" ); |
27 | |
28 | /* |
29 | * On architectures where all call sites share the same trampoline, having |
30 | * tracers enabled for distinct functions can force the use of the list func |
31 | * and incur overhead for all call sites. |
32 | */ |
33 | static unsigned int nr_ops_irrelevant; |
34 | module_param(nr_ops_irrelevant, uint, 0); |
35 | MODULE_PARM_DESC(nr_ops_irrelevant, "How many ftrace_ops to associate with the irrelevant tracee" ); |
36 | |
37 | /* |
38 | * On architectures with DYNAMIC_FTRACE_WITH_REGS, saving the full pt_regs can |
39 | * be more expensive than only saving the minimal necessary regs. |
40 | */ |
41 | static bool save_regs; |
42 | module_param(save_regs, bool, 0); |
43 | MODULE_PARM_DESC(save_regs, "Register ops with FTRACE_OPS_FL_SAVE_REGS (save all registers in the trampoline)" ); |
44 | |
45 | static bool assist_recursion; |
46 | module_param(assist_recursion, bool, 0); |
47 | MODULE_PARM_DESC(assist_reursion, "Register ops with FTRACE_OPS_FL_RECURSION" ); |
48 | |
49 | static bool assist_rcu; |
50 | module_param(assist_rcu, bool, 0); |
51 | MODULE_PARM_DESC(assist_reursion, "Register ops with FTRACE_OPS_FL_RCU" ); |
52 | |
53 | /* |
54 | * By default, a trivial tracer is used which immediately returns to mimimize |
55 | * overhead. Sometimes a consistency check using a more expensive tracer is |
56 | * desireable. |
57 | */ |
58 | static bool check_count; |
59 | module_param(check_count, bool, 0); |
60 | MODULE_PARM_DESC(check_count, "Check that tracers are called the expected number of times\n" ); |
61 | |
62 | /* |
63 | * Usually it's not interesting to leave the ops registered after the test |
64 | * runs, but sometimes it can be useful to leave them registered so that they |
65 | * can be inspected through the tracefs 'enabled_functions' file. |
66 | */ |
67 | static bool persist; |
68 | module_param(persist, bool, 0); |
69 | MODULE_PARM_DESC(persist, "Successfully load module and leave ftrace ops registered after test completes\n" ); |
70 | |
71 | /* |
72 | * Marked as noinline to ensure that an out-of-line traceable copy is |
73 | * generated by the compiler. |
74 | * |
75 | * The barrier() ensures the compiler won't elide calls by determining there |
76 | * are no side-effects. |
77 | */ |
78 | static noinline void tracee_relevant(void) |
79 | { |
80 | barrier(); |
81 | } |
82 | |
83 | /* |
84 | * Marked as noinline to ensure that an out-of-line traceable copy is |
85 | * generated by the compiler. |
86 | * |
87 | * The barrier() ensures the compiler won't elide calls by determining there |
88 | * are no side-effects. |
89 | */ |
90 | static noinline void tracee_irrelevant(void) |
91 | { |
92 | barrier(); |
93 | } |
94 | |
95 | struct sample_ops { |
96 | struct ftrace_ops ops; |
97 | unsigned int count; |
98 | }; |
99 | |
100 | static void ops_func_nop(unsigned long ip, unsigned long parent_ip, |
101 | struct ftrace_ops *op, |
102 | struct ftrace_regs *fregs) |
103 | { |
104 | /* do nothing */ |
105 | } |
106 | |
107 | static void ops_func_count(unsigned long ip, unsigned long parent_ip, |
108 | struct ftrace_ops *op, |
109 | struct ftrace_regs *fregs) |
110 | { |
111 | struct sample_ops *self; |
112 | |
113 | self = container_of(op, struct sample_ops, ops); |
114 | self->count++; |
115 | } |
116 | |
117 | static struct sample_ops *ops_relevant; |
118 | static struct sample_ops *ops_irrelevant; |
119 | |
120 | static struct sample_ops *ops_alloc_init(void *tracee, ftrace_func_t func, |
121 | unsigned long flags, int nr) |
122 | { |
123 | struct sample_ops *ops; |
124 | |
125 | ops = kcalloc(n: nr, size: sizeof(*ops), GFP_KERNEL); |
126 | if (WARN_ON_ONCE(!ops)) |
127 | return NULL; |
128 | |
129 | for (unsigned int i = 0; i < nr; i++) { |
130 | ops[i].ops.func = func; |
131 | ops[i].ops.flags = flags; |
132 | WARN_ON_ONCE(ftrace_set_filter_ip(&ops[i].ops, (unsigned long)tracee, 0, 0)); |
133 | WARN_ON_ONCE(register_ftrace_function(&ops[i].ops)); |
134 | } |
135 | |
136 | return ops; |
137 | } |
138 | |
139 | static void ops_destroy(struct sample_ops *ops, int nr) |
140 | { |
141 | if (!ops) |
142 | return; |
143 | |
144 | for (unsigned int i = 0; i < nr; i++) { |
145 | WARN_ON_ONCE(unregister_ftrace_function(&ops[i].ops)); |
146 | ftrace_free_filter(ops: &ops[i].ops); |
147 | } |
148 | |
149 | kfree(objp: ops); |
150 | } |
151 | |
152 | static void ops_check(struct sample_ops *ops, int nr, |
153 | unsigned int expected_count) |
154 | { |
155 | if (!ops || !check_count) |
156 | return; |
157 | |
158 | for (unsigned int i = 0; i < nr; i++) { |
159 | if (ops->count == expected_count) |
160 | continue; |
161 | pr_warn("Counter called %u times (expected %u)\n" , |
162 | ops->count, expected_count); |
163 | } |
164 | } |
165 | |
166 | static ftrace_func_t tracer_relevant = ops_func_nop; |
167 | static ftrace_func_t tracer_irrelevant = ops_func_nop; |
168 | |
169 | static int __init ftrace_ops_sample_init(void) |
170 | { |
171 | unsigned long flags = 0; |
172 | ktime_t start, end; |
173 | u64 period; |
174 | |
175 | if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && save_regs) { |
176 | pr_info("this kernel does not support saving registers\n" ); |
177 | save_regs = false; |
178 | } else if (save_regs) { |
179 | flags |= FTRACE_OPS_FL_SAVE_REGS; |
180 | } |
181 | |
182 | if (assist_recursion) |
183 | flags |= FTRACE_OPS_FL_RECURSION; |
184 | |
185 | if (assist_rcu) |
186 | flags |= FTRACE_OPS_FL_RCU; |
187 | |
188 | if (check_count) { |
189 | tracer_relevant = ops_func_count; |
190 | tracer_irrelevant = ops_func_count; |
191 | } |
192 | |
193 | pr_info("registering:\n" |
194 | " relevant ops: %u\n" |
195 | " tracee: %ps\n" |
196 | " tracer: %ps\n" |
197 | " irrelevant ops: %u\n" |
198 | " tracee: %ps\n" |
199 | " tracer: %ps\n" |
200 | " saving registers: %s\n" |
201 | " assist recursion: %s\n" |
202 | " assist RCU: %s\n" , |
203 | nr_ops_relevant, tracee_relevant, tracer_relevant, |
204 | nr_ops_irrelevant, tracee_irrelevant, tracer_irrelevant, |
205 | save_regs ? "YES" : "NO" , |
206 | assist_recursion ? "YES" : "NO" , |
207 | assist_rcu ? "YES" : "NO" ); |
208 | |
209 | ops_relevant = ops_alloc_init(tracee: tracee_relevant, func: tracer_relevant, |
210 | flags, nr: nr_ops_relevant); |
211 | ops_irrelevant = ops_alloc_init(tracee: tracee_irrelevant, func: tracer_irrelevant, |
212 | flags, nr: nr_ops_irrelevant); |
213 | |
214 | start = ktime_get(); |
215 | for (unsigned int i = 0; i < nr_function_calls; i++) |
216 | tracee_relevant(); |
217 | end = ktime_get(); |
218 | |
219 | ops_check(ops: ops_relevant, nr: nr_ops_relevant, expected_count: nr_function_calls); |
220 | ops_check(ops: ops_irrelevant, nr: nr_ops_irrelevant, expected_count: 0); |
221 | |
222 | period = ktime_to_ns(ktime_sub(end, start)); |
223 | |
224 | pr_info("Attempted %u calls to %ps in %lluns (%lluns / call)\n" , |
225 | nr_function_calls, tracee_relevant, |
226 | period, div_u64(period, nr_function_calls)); |
227 | |
228 | if (persist) |
229 | return 0; |
230 | |
231 | ops_destroy(ops: ops_relevant, nr: nr_ops_relevant); |
232 | ops_destroy(ops: ops_irrelevant, nr: nr_ops_irrelevant); |
233 | |
234 | /* |
235 | * The benchmark completed sucessfully, but there's no reason to keep |
236 | * the module around. Return an error do the user doesn't have to |
237 | * manually unload the module. |
238 | */ |
239 | return -EINVAL; |
240 | } |
241 | module_init(ftrace_ops_sample_init); |
242 | |
243 | static void __exit ftrace_ops_sample_exit(void) |
244 | { |
245 | ops_destroy(ops: ops_relevant, nr: nr_ops_relevant); |
246 | ops_destroy(ops: ops_irrelevant, nr: nr_ops_irrelevant); |
247 | } |
248 | module_exit(ftrace_ops_sample_exit); |
249 | |
250 | MODULE_AUTHOR("Mark Rutland" ); |
251 | MODULE_DESCRIPTION("Example of using custom ftrace_ops" ); |
252 | MODULE_LICENSE("GPL" ); |
253 | |