1// SPDX-License-Identifier: GPL-2.0-only
2
3#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
4
5#include <linux/ftrace.h>
6#include <linux/ktime.h>
7#include <linux/module.h>
8
9#include <asm/barrier.h>
10
11/*
12 * Arbitrary large value chosen to be sufficiently large to minimize noise but
13 * sufficiently small to complete quickly.
14 */
15static unsigned int nr_function_calls = 100000;
16module_param(nr_function_calls, uint, 0);
17MODULE_PARM_DESC(nr_function_calls, "How many times to call the relevant tracee");
18
19/*
20 * The number of ops associated with a call site affects whether a tracer can
21 * be called directly or whether it's necessary to go via the list func, which
22 * can be significantly more expensive.
23 */
24static unsigned int nr_ops_relevant = 1;
25module_param(nr_ops_relevant, uint, 0);
26MODULE_PARM_DESC(nr_ops_relevant, "How many ftrace_ops to associate with the relevant tracee");
27
28/*
29 * On architectures where all call sites share the same trampoline, having
30 * tracers enabled for distinct functions can force the use of the list func
31 * and incur overhead for all call sites.
32 */
33static unsigned int nr_ops_irrelevant;
34module_param(nr_ops_irrelevant, uint, 0);
35MODULE_PARM_DESC(nr_ops_irrelevant, "How many ftrace_ops to associate with the irrelevant tracee");
36
37/*
38 * On architectures with DYNAMIC_FTRACE_WITH_REGS, saving the full pt_regs can
39 * be more expensive than only saving the minimal necessary regs.
40 */
41static bool save_regs;
42module_param(save_regs, bool, 0);
43MODULE_PARM_DESC(save_regs, "Register ops with FTRACE_OPS_FL_SAVE_REGS (save all registers in the trampoline)");
44
45static bool assist_recursion;
46module_param(assist_recursion, bool, 0);
47MODULE_PARM_DESC(assist_reursion, "Register ops with FTRACE_OPS_FL_RECURSION");
48
49static bool assist_rcu;
50module_param(assist_rcu, bool, 0);
51MODULE_PARM_DESC(assist_reursion, "Register ops with FTRACE_OPS_FL_RCU");
52
53/*
54 * By default, a trivial tracer is used which immediately returns to mimimize
55 * overhead. Sometimes a consistency check using a more expensive tracer is
56 * desireable.
57 */
58static bool check_count;
59module_param(check_count, bool, 0);
60MODULE_PARM_DESC(check_count, "Check that tracers are called the expected number of times\n");
61
62/*
63 * Usually it's not interesting to leave the ops registered after the test
64 * runs, but sometimes it can be useful to leave them registered so that they
65 * can be inspected through the tracefs 'enabled_functions' file.
66 */
67static bool persist;
68module_param(persist, bool, 0);
69MODULE_PARM_DESC(persist, "Successfully load module and leave ftrace ops registered after test completes\n");
70
71/*
72 * Marked as noinline to ensure that an out-of-line traceable copy is
73 * generated by the compiler.
74 *
75 * The barrier() ensures the compiler won't elide calls by determining there
76 * are no side-effects.
77 */
78static noinline void tracee_relevant(void)
79{
80 barrier();
81}
82
83/*
84 * Marked as noinline to ensure that an out-of-line traceable copy is
85 * generated by the compiler.
86 *
87 * The barrier() ensures the compiler won't elide calls by determining there
88 * are no side-effects.
89 */
90static noinline void tracee_irrelevant(void)
91{
92 barrier();
93}
94
95struct sample_ops {
96 struct ftrace_ops ops;
97 unsigned int count;
98};
99
100static void ops_func_nop(unsigned long ip, unsigned long parent_ip,
101 struct ftrace_ops *op,
102 struct ftrace_regs *fregs)
103{
104 /* do nothing */
105}
106
107static void ops_func_count(unsigned long ip, unsigned long parent_ip,
108 struct ftrace_ops *op,
109 struct ftrace_regs *fregs)
110{
111 struct sample_ops *self;
112
113 self = container_of(op, struct sample_ops, ops);
114 self->count++;
115}
116
117static struct sample_ops *ops_relevant;
118static struct sample_ops *ops_irrelevant;
119
120static struct sample_ops *ops_alloc_init(void *tracee, ftrace_func_t func,
121 unsigned long flags, int nr)
122{
123 struct sample_ops *ops;
124
125 ops = kcalloc(n: nr, size: sizeof(*ops), GFP_KERNEL);
126 if (WARN_ON_ONCE(!ops))
127 return NULL;
128
129 for (unsigned int i = 0; i < nr; i++) {
130 ops[i].ops.func = func;
131 ops[i].ops.flags = flags;
132 WARN_ON_ONCE(ftrace_set_filter_ip(&ops[i].ops, (unsigned long)tracee, 0, 0));
133 WARN_ON_ONCE(register_ftrace_function(&ops[i].ops));
134 }
135
136 return ops;
137}
138
139static void ops_destroy(struct sample_ops *ops, int nr)
140{
141 if (!ops)
142 return;
143
144 for (unsigned int i = 0; i < nr; i++) {
145 WARN_ON_ONCE(unregister_ftrace_function(&ops[i].ops));
146 ftrace_free_filter(ops: &ops[i].ops);
147 }
148
149 kfree(objp: ops);
150}
151
152static void ops_check(struct sample_ops *ops, int nr,
153 unsigned int expected_count)
154{
155 if (!ops || !check_count)
156 return;
157
158 for (unsigned int i = 0; i < nr; i++) {
159 if (ops->count == expected_count)
160 continue;
161 pr_warn("Counter called %u times (expected %u)\n",
162 ops->count, expected_count);
163 }
164}
165
166static ftrace_func_t tracer_relevant = ops_func_nop;
167static ftrace_func_t tracer_irrelevant = ops_func_nop;
168
169static int __init ftrace_ops_sample_init(void)
170{
171 unsigned long flags = 0;
172 ktime_t start, end;
173 u64 period;
174
175 if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && save_regs) {
176 pr_info("this kernel does not support saving registers\n");
177 save_regs = false;
178 } else if (save_regs) {
179 flags |= FTRACE_OPS_FL_SAVE_REGS;
180 }
181
182 if (assist_recursion)
183 flags |= FTRACE_OPS_FL_RECURSION;
184
185 if (assist_rcu)
186 flags |= FTRACE_OPS_FL_RCU;
187
188 if (check_count) {
189 tracer_relevant = ops_func_count;
190 tracer_irrelevant = ops_func_count;
191 }
192
193 pr_info("registering:\n"
194 " relevant ops: %u\n"
195 " tracee: %ps\n"
196 " tracer: %ps\n"
197 " irrelevant ops: %u\n"
198 " tracee: %ps\n"
199 " tracer: %ps\n"
200 " saving registers: %s\n"
201 " assist recursion: %s\n"
202 " assist RCU: %s\n",
203 nr_ops_relevant, tracee_relevant, tracer_relevant,
204 nr_ops_irrelevant, tracee_irrelevant, tracer_irrelevant,
205 save_regs ? "YES" : "NO",
206 assist_recursion ? "YES" : "NO",
207 assist_rcu ? "YES" : "NO");
208
209 ops_relevant = ops_alloc_init(tracee: tracee_relevant, func: tracer_relevant,
210 flags, nr: nr_ops_relevant);
211 ops_irrelevant = ops_alloc_init(tracee: tracee_irrelevant, func: tracer_irrelevant,
212 flags, nr: nr_ops_irrelevant);
213
214 start = ktime_get();
215 for (unsigned int i = 0; i < nr_function_calls; i++)
216 tracee_relevant();
217 end = ktime_get();
218
219 ops_check(ops: ops_relevant, nr: nr_ops_relevant, expected_count: nr_function_calls);
220 ops_check(ops: ops_irrelevant, nr: nr_ops_irrelevant, expected_count: 0);
221
222 period = ktime_to_ns(ktime_sub(end, start));
223
224 pr_info("Attempted %u calls to %ps in %lluns (%lluns / call)\n",
225 nr_function_calls, tracee_relevant,
226 period, div_u64(period, nr_function_calls));
227
228 if (persist)
229 return 0;
230
231 ops_destroy(ops: ops_relevant, nr: nr_ops_relevant);
232 ops_destroy(ops: ops_irrelevant, nr: nr_ops_irrelevant);
233
234 /*
235 * The benchmark completed sucessfully, but there's no reason to keep
236 * the module around. Return an error do the user doesn't have to
237 * manually unload the module.
238 */
239 return -EINVAL;
240}
241module_init(ftrace_ops_sample_init);
242
243static void __exit ftrace_ops_sample_exit(void)
244{
245 ops_destroy(ops: ops_relevant, nr: nr_ops_relevant);
246 ops_destroy(ops: ops_irrelevant, nr: nr_ops_irrelevant);
247}
248module_exit(ftrace_ops_sample_exit);
249
250MODULE_AUTHOR("Mark Rutland");
251MODULE_DESCRIPTION("Example of using custom ftrace_ops");
252MODULE_LICENSE("GPL");
253

source code of linux/samples/ftrace/ftrace-ops.c