1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | |
3 | #define pr_fmt(fmt) "callthunks: " fmt |
4 | |
5 | #include <linux/debugfs.h> |
6 | #include <linux/kallsyms.h> |
7 | #include <linux/memory.h> |
8 | #include <linux/moduleloader.h> |
9 | #include <linux/static_call.h> |
10 | |
11 | #include <asm/alternative.h> |
12 | #include <asm/asm-offsets.h> |
13 | #include <asm/cpu.h> |
14 | #include <asm/ftrace.h> |
15 | #include <asm/insn.h> |
16 | #include <asm/kexec.h> |
17 | #include <asm/nospec-branch.h> |
18 | #include <asm/paravirt.h> |
19 | #include <asm/sections.h> |
20 | #include <asm/switch_to.h> |
21 | #include <asm/sync_core.h> |
22 | #include <asm/text-patching.h> |
23 | #include <asm/xen/hypercall.h> |
24 | |
25 | static int __initdata_or_module debug_callthunks; |
26 | |
27 | #define MAX_PATCH_LEN (255-1) |
28 | |
29 | #define prdbg(fmt, args...) \ |
30 | do { \ |
31 | if (debug_callthunks) \ |
32 | printk(KERN_DEBUG pr_fmt(fmt), ##args); \ |
33 | } while(0) |
34 | |
35 | static int __init debug_thunks(char *str) |
36 | { |
37 | debug_callthunks = 1; |
38 | return 1; |
39 | } |
40 | __setup("debug-callthunks" , debug_thunks); |
41 | |
42 | #ifdef CONFIG_CALL_THUNKS_DEBUG |
43 | DEFINE_PER_CPU(u64, __x86_call_count); |
44 | DEFINE_PER_CPU(u64, __x86_ret_count); |
45 | DEFINE_PER_CPU(u64, __x86_stuffs_count); |
46 | DEFINE_PER_CPU(u64, __x86_ctxsw_count); |
47 | EXPORT_PER_CPU_SYMBOL_GPL(__x86_ctxsw_count); |
48 | EXPORT_PER_CPU_SYMBOL_GPL(__x86_call_count); |
49 | #endif |
50 | |
51 | extern s32 __call_sites[], __call_sites_end[]; |
52 | |
53 | struct core_text { |
54 | unsigned long base; |
55 | unsigned long end; |
56 | const char *name; |
57 | }; |
58 | |
59 | static bool thunks_initialized __ro_after_init; |
60 | |
61 | static const struct core_text builtin_coretext = { |
62 | .base = (unsigned long)_text, |
63 | .end = (unsigned long)_etext, |
64 | .name = "builtin" , |
65 | }; |
66 | |
67 | asm ( |
68 | ".pushsection .rodata \n" |
69 | ".global skl_call_thunk_template \n" |
70 | "skl_call_thunk_template: \n" |
71 | __stringify(INCREMENT_CALL_DEPTH)" \n" |
72 | ".global skl_call_thunk_tail \n" |
73 | "skl_call_thunk_tail: \n" |
74 | ".popsection \n" |
75 | ); |
76 | |
77 | extern u8 skl_call_thunk_template[]; |
78 | extern u8 skl_call_thunk_tail[]; |
79 | |
80 | #define SKL_TMPL_SIZE \ |
81 | ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template)) |
82 | |
83 | extern void error_entry(void); |
84 | extern void xen_error_entry(void); |
85 | extern void paranoid_entry(void); |
86 | |
87 | static inline bool within_coretext(const struct core_text *ct, void *addr) |
88 | { |
89 | unsigned long p = (unsigned long)addr; |
90 | |
91 | return ct->base <= p && p < ct->end; |
92 | } |
93 | |
94 | static inline bool within_module_coretext(void *addr) |
95 | { |
96 | bool ret = false; |
97 | |
98 | #ifdef CONFIG_MODULES |
99 | struct module *mod; |
100 | |
101 | preempt_disable(); |
102 | mod = __module_address(addr: (unsigned long)addr); |
103 | if (mod && within_module_core(addr: (unsigned long)addr, mod)) |
104 | ret = true; |
105 | preempt_enable(); |
106 | #endif |
107 | return ret; |
108 | } |
109 | |
110 | static bool is_coretext(const struct core_text *ct, void *addr) |
111 | { |
112 | if (ct && within_coretext(ct, addr)) |
113 | return true; |
114 | if (within_coretext(ct: &builtin_coretext, addr)) |
115 | return true; |
116 | return within_module_coretext(addr); |
117 | } |
118 | |
119 | static bool skip_addr(void *dest) |
120 | { |
121 | if (dest == error_entry) |
122 | return true; |
123 | if (dest == paranoid_entry) |
124 | return true; |
125 | if (dest == xen_error_entry) |
126 | return true; |
127 | /* Does FILL_RSB... */ |
128 | if (dest == __switch_to_asm) |
129 | return true; |
130 | /* Accounts directly */ |
131 | if (dest == ret_from_fork) |
132 | return true; |
133 | #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT) |
134 | if (dest == soft_restart_cpu) |
135 | return true; |
136 | #endif |
137 | #ifdef CONFIG_FUNCTION_TRACER |
138 | if (dest == __fentry__) |
139 | return true; |
140 | #endif |
141 | #ifdef CONFIG_KEXEC_CORE |
142 | if (dest >= (void *)relocate_kernel && |
143 | dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE) |
144 | return true; |
145 | #endif |
146 | #ifdef CONFIG_XEN |
147 | if (dest >= (void *)hypercall_page && |
148 | dest < (void*)hypercall_page + PAGE_SIZE) |
149 | return true; |
150 | #endif |
151 | return false; |
152 | } |
153 | |
154 | static __init_or_module void *call_get_dest(void *addr) |
155 | { |
156 | struct insn insn; |
157 | void *dest; |
158 | int ret; |
159 | |
160 | ret = insn_decode_kernel(&insn, addr); |
161 | if (ret) |
162 | return ERR_PTR(error: ret); |
163 | |
164 | /* Patched out call? */ |
165 | if (insn.opcode.bytes[0] != CALL_INSN_OPCODE) |
166 | return NULL; |
167 | |
168 | dest = addr + insn.length + insn.immediate.value; |
169 | if (skip_addr(dest)) |
170 | return NULL; |
171 | return dest; |
172 | } |
173 | |
174 | static const u8 nops[] = { |
175 | 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, |
176 | 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, |
177 | 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, |
178 | 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, |
179 | }; |
180 | |
181 | static void *patch_dest(void *dest, bool direct) |
182 | { |
183 | unsigned int tsize = SKL_TMPL_SIZE; |
184 | u8 insn_buff[MAX_PATCH_LEN]; |
185 | u8 *pad = dest - tsize; |
186 | |
187 | memcpy(insn_buff, skl_call_thunk_template, tsize); |
188 | apply_relocation(buf: insn_buff, len: tsize, dest: pad, |
189 | src: skl_call_thunk_template, src_len: tsize); |
190 | |
191 | /* Already patched? */ |
192 | if (!bcmp(pad, insn_buff, tsize)) |
193 | return pad; |
194 | |
195 | /* Ensure there are nops */ |
196 | if (bcmp(pad, nops, tsize)) { |
197 | pr_warn_once("Invalid padding area for %pS\n" , dest); |
198 | return NULL; |
199 | } |
200 | |
201 | if (direct) |
202 | memcpy(pad, insn_buff, tsize); |
203 | else |
204 | text_poke_copy_locked(addr: pad, opcode: insn_buff, len: tsize, core_ok: true); |
205 | return pad; |
206 | } |
207 | |
208 | static __init_or_module void patch_call(void *addr, const struct core_text *ct) |
209 | { |
210 | void *pad, *dest; |
211 | u8 bytes[8]; |
212 | |
213 | if (!within_coretext(ct, addr)) |
214 | return; |
215 | |
216 | dest = call_get_dest(addr); |
217 | if (!dest || WARN_ON_ONCE(IS_ERR(dest))) |
218 | return; |
219 | |
220 | if (!is_coretext(ct, addr: dest)) |
221 | return; |
222 | |
223 | pad = patch_dest(dest, direct: within_coretext(ct, addr: dest)); |
224 | if (!pad) |
225 | return; |
226 | |
227 | prdbg("Patch call at: %pS %px to %pS %px -> %px \n" , addr, addr, |
228 | dest, dest, pad); |
229 | __text_gen_insn(buf: bytes, CALL_INSN_OPCODE, addr, dest: pad, CALL_INSN_SIZE); |
230 | text_poke_early(addr, opcode: bytes, CALL_INSN_SIZE); |
231 | } |
232 | |
233 | static __init_or_module void |
234 | patch_call_sites(s32 *start, s32 *end, const struct core_text *ct) |
235 | { |
236 | s32 *s; |
237 | |
238 | for (s = start; s < end; s++) |
239 | patch_call(addr: (void *)s + *s, ct); |
240 | } |
241 | |
242 | static __init_or_module void |
243 | patch_alt_call_sites(struct alt_instr *start, struct alt_instr *end, |
244 | const struct core_text *ct) |
245 | { |
246 | struct alt_instr *a; |
247 | |
248 | for (a = start; a < end; a++) |
249 | patch_call(addr: (void *)&a->instr_offset + a->instr_offset, ct); |
250 | } |
251 | |
252 | static __init_or_module void |
253 | callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct) |
254 | { |
255 | prdbg("Patching call sites %s\n" , ct->name); |
256 | patch_call_sites(start: cs->call_start, end: cs->call_end, ct); |
257 | patch_alt_call_sites(start: cs->alt_start, end: cs->alt_end, ct); |
258 | prdbg("Patching call sites done%s\n" , ct->name); |
259 | } |
260 | |
261 | void __init callthunks_patch_builtin_calls(void) |
262 | { |
263 | struct callthunk_sites cs = { |
264 | .call_start = __call_sites, |
265 | .call_end = __call_sites_end, |
266 | .alt_start = __alt_instructions, |
267 | .alt_end = __alt_instructions_end |
268 | }; |
269 | |
270 | if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) |
271 | return; |
272 | |
273 | pr_info("Setting up call depth tracking\n" ); |
274 | mutex_lock(&text_mutex); |
275 | callthunks_setup(cs: &cs, ct: &builtin_coretext); |
276 | thunks_initialized = true; |
277 | mutex_unlock(lock: &text_mutex); |
278 | } |
279 | |
280 | void *callthunks_translate_call_dest(void *dest) |
281 | { |
282 | void *target; |
283 | |
284 | lockdep_assert_held(&text_mutex); |
285 | |
286 | if (!thunks_initialized || skip_addr(dest)) |
287 | return dest; |
288 | |
289 | if (!is_coretext(NULL, addr: dest)) |
290 | return dest; |
291 | |
292 | target = patch_dest(dest, direct: false); |
293 | return target ? : dest; |
294 | } |
295 | |
296 | #ifdef CONFIG_BPF_JIT |
297 | static bool is_callthunk(void *addr) |
298 | { |
299 | unsigned int tmpl_size = SKL_TMPL_SIZE; |
300 | u8 insn_buff[MAX_PATCH_LEN]; |
301 | unsigned long dest; |
302 | u8 *pad; |
303 | |
304 | dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT); |
305 | if (!thunks_initialized || skip_addr(dest: (void *)dest)) |
306 | return false; |
307 | |
308 | pad = (void *)(dest - tmpl_size); |
309 | |
310 | memcpy(insn_buff, skl_call_thunk_template, tmpl_size); |
311 | apply_relocation(buf: insn_buff, len: tmpl_size, dest: pad, |
312 | src: skl_call_thunk_template, src_len: tmpl_size); |
313 | |
314 | return !bcmp(pad, insn_buff, tmpl_size); |
315 | } |
316 | |
317 | int x86_call_depth_emit_accounting(u8 **pprog, void *func, void *ip) |
318 | { |
319 | unsigned int tmpl_size = SKL_TMPL_SIZE; |
320 | u8 insn_buff[MAX_PATCH_LEN]; |
321 | |
322 | if (!thunks_initialized) |
323 | return 0; |
324 | |
325 | /* Is function call target a thunk? */ |
326 | if (func && is_callthunk(addr: func)) |
327 | return 0; |
328 | |
329 | memcpy(insn_buff, skl_call_thunk_template, tmpl_size); |
330 | apply_relocation(buf: insn_buff, len: tmpl_size, dest: ip, |
331 | src: skl_call_thunk_template, src_len: tmpl_size); |
332 | |
333 | memcpy(*pprog, insn_buff, tmpl_size); |
334 | *pprog += tmpl_size; |
335 | return tmpl_size; |
336 | } |
337 | #endif |
338 | |
339 | #ifdef CONFIG_MODULES |
340 | void noinline callthunks_patch_module_calls(struct callthunk_sites *cs, |
341 | struct module *mod) |
342 | { |
343 | struct core_text ct = { |
344 | .base = (unsigned long)mod->mem[MOD_TEXT].base, |
345 | .end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size, |
346 | .name = mod->name, |
347 | }; |
348 | |
349 | if (!thunks_initialized) |
350 | return; |
351 | |
352 | mutex_lock(&text_mutex); |
353 | callthunks_setup(cs, ct: &ct); |
354 | mutex_unlock(lock: &text_mutex); |
355 | } |
356 | #endif /* CONFIG_MODULES */ |
357 | |
358 | #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS) |
359 | static int callthunks_debug_show(struct seq_file *m, void *p) |
360 | { |
361 | unsigned long cpu = (unsigned long)m->private; |
362 | |
363 | seq_printf(m, fmt: "C: %16llu R: %16llu S: %16llu X: %16llu\n," , |
364 | per_cpu(__x86_call_count, cpu), |
365 | per_cpu(__x86_ret_count, cpu), |
366 | per_cpu(__x86_stuffs_count, cpu), |
367 | per_cpu(__x86_ctxsw_count, cpu)); |
368 | return 0; |
369 | } |
370 | |
371 | static int callthunks_debug_open(struct inode *inode, struct file *file) |
372 | { |
373 | return single_open(file, callthunks_debug_show, inode->i_private); |
374 | } |
375 | |
376 | static const struct file_operations dfs_ops = { |
377 | .open = callthunks_debug_open, |
378 | .read = seq_read, |
379 | .llseek = seq_lseek, |
380 | .release = single_release, |
381 | }; |
382 | |
383 | static int __init callthunks_debugfs_init(void) |
384 | { |
385 | struct dentry *dir; |
386 | unsigned long cpu; |
387 | |
388 | dir = debugfs_create_dir(name: "callthunks" , NULL); |
389 | for_each_possible_cpu(cpu) { |
390 | void *arg = (void *)cpu; |
391 | char name [10]; |
392 | |
393 | sprintf(buf: name, fmt: "cpu%lu" , cpu); |
394 | debugfs_create_file(name, mode: 0644, parent: dir, data: arg, fops: &dfs_ops); |
395 | } |
396 | return 0; |
397 | } |
398 | __initcall(callthunks_debugfs_init); |
399 | #endif |
400 | |