1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * arch/arm/kernel/kprobes.c |
4 | * |
5 | * Kprobes on ARM |
6 | * |
7 | * Abhishek Sagar <sagar.abhishek@gmail.com> |
8 | * Copyright (C) 2006, 2007 Motorola Inc. |
9 | * |
10 | * Nicolas Pitre <nico@marvell.com> |
11 | * Copyright (C) 2007 Marvell Ltd. |
12 | */ |
13 | |
14 | #define pr_fmt(fmt) "kprobes: " fmt |
15 | |
16 | #include <linux/kernel.h> |
17 | #include <linux/kprobes.h> |
18 | #include <linux/module.h> |
19 | #include <linux/slab.h> |
20 | #include <linux/stop_machine.h> |
21 | #include <linux/sched/debug.h> |
22 | #include <linux/stringify.h> |
23 | #include <asm/traps.h> |
24 | #include <asm/opcodes.h> |
25 | #include <asm/cacheflush.h> |
26 | #include <linux/percpu.h> |
27 | #include <linux/bug.h> |
28 | #include <asm/patch.h> |
29 | #include <asm/sections.h> |
30 | |
31 | #include "../decode-arm.h" |
32 | #include "../decode-thumb.h" |
33 | #include "core.h" |
34 | |
35 | #define MIN_STACK_SIZE(addr) \ |
36 | min((unsigned long)MAX_STACK_SIZE, \ |
37 | (unsigned long)current_thread_info() + THREAD_START_SP - (addr)) |
38 | |
39 | #define flush_insns(addr, size) \ |
40 | flush_icache_range((unsigned long)(addr), \ |
41 | (unsigned long)(addr) + \ |
42 | (size)) |
43 | |
44 | DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; |
45 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); |
46 | |
47 | |
48 | int __kprobes arch_prepare_kprobe(struct kprobe *p) |
49 | { |
50 | kprobe_opcode_t insn; |
51 | kprobe_opcode_t tmp_insn[MAX_INSN_SIZE]; |
52 | unsigned long addr = (unsigned long)p->addr; |
53 | bool thumb; |
54 | kprobe_decode_insn_t *decode_insn; |
55 | const union decode_action *actions; |
56 | int is; |
57 | const struct decode_checker **checkers; |
58 | |
59 | #ifdef CONFIG_THUMB2_KERNEL |
60 | thumb = true; |
61 | addr &= ~1; /* Bit 0 would normally be set to indicate Thumb code */ |
62 | insn = __mem_to_opcode_thumb16(((u16 *)addr)[0]); |
63 | if (is_wide_instruction(insn)) { |
64 | u16 inst2 = __mem_to_opcode_thumb16(((u16 *)addr)[1]); |
65 | insn = __opcode_thumb32_compose(insn, inst2); |
66 | decode_insn = thumb32_probes_decode_insn; |
67 | actions = kprobes_t32_actions; |
68 | checkers = kprobes_t32_checkers; |
69 | } else { |
70 | decode_insn = thumb16_probes_decode_insn; |
71 | actions = kprobes_t16_actions; |
72 | checkers = kprobes_t16_checkers; |
73 | } |
74 | #else /* !CONFIG_THUMB2_KERNEL */ |
75 | thumb = false; |
76 | if (addr & 0x3) |
77 | return -EINVAL; |
78 | insn = __mem_to_opcode_arm(*p->addr); |
79 | decode_insn = arm_probes_decode_insn; |
80 | actions = kprobes_arm_actions; |
81 | checkers = kprobes_arm_checkers; |
82 | #endif |
83 | |
84 | p->opcode = insn; |
85 | p->ainsn.insn = tmp_insn; |
86 | |
87 | switch ((*decode_insn)(insn, &p->ainsn, true, actions, checkers)) { |
88 | case INSN_REJECTED: /* not supported */ |
89 | return -EINVAL; |
90 | |
91 | case INSN_GOOD: /* instruction uses slot */ |
92 | p->ainsn.insn = get_insn_slot(); |
93 | if (!p->ainsn.insn) |
94 | return -ENOMEM; |
95 | for (is = 0; is < MAX_INSN_SIZE; ++is) |
96 | p->ainsn.insn[is] = tmp_insn[is]; |
97 | flush_insns(p->ainsn.insn, |
98 | sizeof(p->ainsn.insn[0]) * MAX_INSN_SIZE); |
99 | p->ainsn.insn_fn = (probes_insn_fn_t *) |
100 | ((uintptr_t)p->ainsn.insn | thumb); |
101 | break; |
102 | |
103 | case INSN_GOOD_NO_SLOT: /* instruction doesn't need insn slot */ |
104 | p->ainsn.insn = NULL; |
105 | break; |
106 | } |
107 | |
108 | /* |
109 | * Never instrument insn like 'str r0, [sp, +/-r1]'. Also, insn likes |
110 | * 'str r0, [sp, #-68]' should also be prohibited. |
111 | * See __und_svc. |
112 | */ |
113 | if ((p->ainsn.stack_space < 0) || |
114 | (p->ainsn.stack_space > MAX_STACK_SIZE)) |
115 | return -EINVAL; |
116 | |
117 | return 0; |
118 | } |
119 | |
120 | void __kprobes arch_arm_kprobe(struct kprobe *p) |
121 | { |
122 | unsigned int brkp; |
123 | void *addr; |
124 | |
125 | if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) { |
126 | /* Remove any Thumb flag */ |
127 | addr = (void *)((uintptr_t)p->addr & ~1); |
128 | |
129 | if (is_wide_instruction(p->opcode)) |
130 | brkp = KPROBE_THUMB32_BREAKPOINT_INSTRUCTION; |
131 | else |
132 | brkp = KPROBE_THUMB16_BREAKPOINT_INSTRUCTION; |
133 | } else { |
134 | kprobe_opcode_t insn = p->opcode; |
135 | |
136 | addr = p->addr; |
137 | brkp = KPROBE_ARM_BREAKPOINT_INSTRUCTION; |
138 | |
139 | if (insn >= 0xe0000000) |
140 | brkp |= 0xe0000000; /* Unconditional instruction */ |
141 | else |
142 | brkp |= insn & 0xf0000000; /* Copy condition from insn */ |
143 | } |
144 | |
145 | patch_text(addr, brkp); |
146 | } |
147 | |
148 | /* |
149 | * The actual disarming is done here on each CPU and synchronized using |
150 | * stop_machine. This synchronization is necessary on SMP to avoid removing |
151 | * a probe between the moment the 'Undefined Instruction' exception is raised |
152 | * and the moment the exception handler reads the faulting instruction from |
153 | * memory. It is also needed to atomically set the two half-words of a 32-bit |
154 | * Thumb breakpoint. |
155 | */ |
156 | struct patch { |
157 | void *addr; |
158 | unsigned int insn; |
159 | }; |
160 | |
161 | static int __kprobes_remove_breakpoint(void *data) |
162 | { |
163 | struct patch *p = data; |
164 | __patch_text(p->addr, p->insn); |
165 | return 0; |
166 | } |
167 | |
168 | void __kprobes kprobes_remove_breakpoint(void *addr, unsigned int insn) |
169 | { |
170 | struct patch p = { |
171 | .addr = addr, |
172 | .insn = insn, |
173 | }; |
174 | stop_machine_cpuslocked(fn: __kprobes_remove_breakpoint, data: &p, |
175 | cpu_online_mask); |
176 | } |
177 | |
178 | void __kprobes arch_disarm_kprobe(struct kprobe *p) |
179 | { |
180 | kprobes_remove_breakpoint(addr: (void *)((uintptr_t)p->addr & ~1), |
181 | insn: p->opcode); |
182 | } |
183 | |
184 | void __kprobes arch_remove_kprobe(struct kprobe *p) |
185 | { |
186 | if (p->ainsn.insn) { |
187 | free_insn_slot(slot: p->ainsn.insn, dirty: 0); |
188 | p->ainsn.insn = NULL; |
189 | } |
190 | } |
191 | |
192 | static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) |
193 | { |
194 | kcb->prev_kprobe.kp = kprobe_running(); |
195 | kcb->prev_kprobe.status = kcb->kprobe_status; |
196 | } |
197 | |
198 | static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) |
199 | { |
200 | __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); |
201 | kcb->kprobe_status = kcb->prev_kprobe.status; |
202 | } |
203 | |
204 | static void __kprobes set_current_kprobe(struct kprobe *p) |
205 | { |
206 | __this_cpu_write(current_kprobe, p); |
207 | } |
208 | |
209 | static void __kprobes |
210 | singlestep_skip(struct kprobe *p, struct pt_regs *regs) |
211 | { |
212 | #ifdef CONFIG_THUMB2_KERNEL |
213 | regs->ARM_cpsr = it_advance(regs->ARM_cpsr); |
214 | if (is_wide_instruction(p->opcode)) |
215 | regs->ARM_pc += 4; |
216 | else |
217 | regs->ARM_pc += 2; |
218 | #else |
219 | regs->ARM_pc += 4; |
220 | #endif |
221 | } |
222 | |
223 | static inline void __kprobes |
224 | singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) |
225 | { |
226 | p->ainsn.insn_singlestep(p->opcode, &p->ainsn, regs); |
227 | } |
228 | |
229 | /* |
230 | * Called with IRQs disabled. IRQs must remain disabled from that point |
231 | * all the way until processing this kprobe is complete. The current |
232 | * kprobes implementation cannot process more than one nested level of |
233 | * kprobe, and that level is reserved for user kprobe handlers, so we can't |
234 | * risk encountering a new kprobe in an interrupt handler. |
235 | */ |
236 | static void __kprobes kprobe_handler(struct pt_regs *regs) |
237 | { |
238 | struct kprobe *p, *cur; |
239 | struct kprobe_ctlblk *kcb; |
240 | |
241 | kcb = get_kprobe_ctlblk(); |
242 | cur = kprobe_running(); |
243 | |
244 | #ifdef CONFIG_THUMB2_KERNEL |
245 | /* |
246 | * First look for a probe which was registered using an address with |
247 | * bit 0 set, this is the usual situation for pointers to Thumb code. |
248 | * If not found, fallback to looking for one with bit 0 clear. |
249 | */ |
250 | p = get_kprobe((kprobe_opcode_t *)(regs->ARM_pc | 1)); |
251 | if (!p) |
252 | p = get_kprobe((kprobe_opcode_t *)regs->ARM_pc); |
253 | |
254 | #else /* ! CONFIG_THUMB2_KERNEL */ |
255 | p = get_kprobe(addr: (kprobe_opcode_t *)regs->ARM_pc); |
256 | #endif |
257 | |
258 | if (p) { |
259 | if (!p->ainsn.insn_check_cc(regs->ARM_cpsr)) { |
260 | /* |
261 | * Probe hit but conditional execution check failed, |
262 | * so just skip the instruction and continue as if |
263 | * nothing had happened. |
264 | * In this case, we can skip recursing check too. |
265 | */ |
266 | singlestep_skip(p, regs); |
267 | } else if (cur) { |
268 | /* Kprobe is pending, so we're recursing. */ |
269 | switch (kcb->kprobe_status) { |
270 | case KPROBE_HIT_ACTIVE: |
271 | case KPROBE_HIT_SSDONE: |
272 | case KPROBE_HIT_SS: |
273 | /* A pre- or post-handler probe got us here. */ |
274 | kprobes_inc_nmissed_count(p); |
275 | save_previous_kprobe(kcb); |
276 | set_current_kprobe(p); |
277 | kcb->kprobe_status = KPROBE_REENTER; |
278 | singlestep(p, regs, kcb); |
279 | restore_previous_kprobe(kcb); |
280 | break; |
281 | case KPROBE_REENTER: |
282 | /* A nested probe was hit in FIQ, it is a BUG */ |
283 | pr_warn("Failed to recover from reentered kprobes.\n" ); |
284 | dump_kprobe(kp: p); |
285 | fallthrough; |
286 | default: |
287 | /* impossible cases */ |
288 | BUG(); |
289 | } |
290 | } else { |
291 | /* Probe hit and conditional execution check ok. */ |
292 | set_current_kprobe(p); |
293 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
294 | |
295 | /* |
296 | * If we have no pre-handler or it returned 0, we |
297 | * continue with normal processing. If we have a |
298 | * pre-handler and it returned non-zero, it will |
299 | * modify the execution path and no need to single |
300 | * stepping. Let's just reset current kprobe and exit. |
301 | */ |
302 | if (!p->pre_handler || !p->pre_handler(p, regs)) { |
303 | kcb->kprobe_status = KPROBE_HIT_SS; |
304 | singlestep(p, regs, kcb); |
305 | if (p->post_handler) { |
306 | kcb->kprobe_status = KPROBE_HIT_SSDONE; |
307 | p->post_handler(p, regs, 0); |
308 | } |
309 | } |
310 | reset_current_kprobe(); |
311 | } |
312 | } else { |
313 | /* |
314 | * The probe was removed and a race is in progress. |
315 | * There is nothing we can do about it. Let's restart |
316 | * the instruction. By the time we can restart, the |
317 | * real instruction will be there. |
318 | */ |
319 | } |
320 | } |
321 | |
322 | static int __kprobes kprobe_trap_handler(struct pt_regs *regs, unsigned int instr) |
323 | { |
324 | unsigned long flags; |
325 | local_irq_save(flags); |
326 | kprobe_handler(regs); |
327 | local_irq_restore(flags); |
328 | return 0; |
329 | } |
330 | |
331 | int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) |
332 | { |
333 | struct kprobe *cur = kprobe_running(); |
334 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
335 | |
336 | switch (kcb->kprobe_status) { |
337 | case KPROBE_HIT_SS: |
338 | case KPROBE_REENTER: |
339 | /* |
340 | * We are here because the instruction being single |
341 | * stepped caused a page fault. We reset the current |
342 | * kprobe and the PC to point back to the probe address |
343 | * and allow the page fault handler to continue as a |
344 | * normal page fault. |
345 | */ |
346 | regs->ARM_pc = (long)cur->addr; |
347 | if (kcb->kprobe_status == KPROBE_REENTER) { |
348 | restore_previous_kprobe(kcb); |
349 | } else { |
350 | reset_current_kprobe(); |
351 | } |
352 | break; |
353 | } |
354 | |
355 | return 0; |
356 | } |
357 | |
358 | int __kprobes kprobe_exceptions_notify(struct notifier_block *self, |
359 | unsigned long val, void *data) |
360 | { |
361 | /* |
362 | * notify_die() is currently never called on ARM, |
363 | * so this callback is currently empty. |
364 | */ |
365 | return NOTIFY_DONE; |
366 | } |
367 | |
368 | /* |
369 | * When a retprobed function returns, trampoline_handler() is called, |
370 | * calling the kretprobe's handler. We construct a struct pt_regs to |
371 | * give a view of registers r0-r11, sp, lr, and pc to the user |
372 | * return-handler. This is not a complete pt_regs structure, but that |
373 | * should be enough for stacktrace from the return handler with or |
374 | * without pt_regs. |
375 | */ |
376 | void __naked __kprobes __kretprobe_trampoline(void) |
377 | { |
378 | __asm__ __volatile__ ( |
379 | #ifdef CONFIG_FRAME_POINTER |
380 | "ldr lr, =__kretprobe_trampoline \n\t" |
381 | /* __kretprobe_trampoline makes a framepointer on pt_regs. */ |
382 | #ifdef CONFIG_CC_IS_CLANG |
383 | "stmdb sp, {sp, lr, pc} \n\t" |
384 | "sub sp, sp, #12 \n\t" |
385 | /* In clang case, pt_regs->ip = lr. */ |
386 | "stmdb sp!, {r0 - r11, lr} \n\t" |
387 | /* fp points regs->r11 (fp) */ |
388 | "add fp, sp, #44 \n\t" |
389 | #else /* !CONFIG_CC_IS_CLANG */ |
390 | /* In gcc case, pt_regs->ip = fp. */ |
391 | "stmdb sp, {fp, sp, lr, pc} \n\t" |
392 | "sub sp, sp, #16 \n\t" |
393 | "stmdb sp!, {r0 - r11} \n\t" |
394 | /* fp points regs->r15 (pc) */ |
395 | "add fp, sp, #60 \n\t" |
396 | #endif /* CONFIG_CC_IS_CLANG */ |
397 | #else /* !CONFIG_FRAME_POINTER */ |
398 | "sub sp, sp, #16 \n\t" |
399 | "stmdb sp!, {r0 - r11} \n\t" |
400 | #endif /* CONFIG_FRAME_POINTER */ |
401 | "mov r0, sp \n\t" |
402 | "bl trampoline_handler \n\t" |
403 | "mov lr, r0 \n\t" |
404 | "ldmia sp!, {r0 - r11} \n\t" |
405 | "add sp, sp, #16 \n\t" |
406 | #ifdef CONFIG_THUMB2_KERNEL |
407 | "bx lr \n\t" |
408 | #else |
409 | "mov pc, lr \n\t" |
410 | #endif |
411 | : : : "memory" ); |
412 | } |
413 | |
414 | /* Called from __kretprobe_trampoline */ |
415 | static __used __kprobes void *trampoline_handler(struct pt_regs *regs) |
416 | { |
417 | return (void *)kretprobe_trampoline_handler(regs, (void *)regs->ARM_fp); |
418 | } |
419 | |
420 | void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, |
421 | struct pt_regs *regs) |
422 | { |
423 | ri->ret_addr = (kprobe_opcode_t *)regs->ARM_lr; |
424 | ri->fp = (void *)regs->ARM_fp; |
425 | |
426 | /* Replace the return addr with trampoline addr. */ |
427 | regs->ARM_lr = (unsigned long)&__kretprobe_trampoline; |
428 | } |
429 | |
430 | int __kprobes arch_trampoline_kprobe(struct kprobe *p) |
431 | { |
432 | return 0; |
433 | } |
434 | |
435 | #ifdef CONFIG_THUMB2_KERNEL |
436 | |
437 | static struct undef_hook kprobes_thumb16_break_hook = { |
438 | .instr_mask = 0xffff, |
439 | .instr_val = KPROBE_THUMB16_BREAKPOINT_INSTRUCTION, |
440 | .cpsr_mask = MODE_MASK, |
441 | .cpsr_val = SVC_MODE, |
442 | .fn = kprobe_trap_handler, |
443 | }; |
444 | |
445 | static struct undef_hook kprobes_thumb32_break_hook = { |
446 | .instr_mask = 0xffffffff, |
447 | .instr_val = KPROBE_THUMB32_BREAKPOINT_INSTRUCTION, |
448 | .cpsr_mask = MODE_MASK, |
449 | .cpsr_val = SVC_MODE, |
450 | .fn = kprobe_trap_handler, |
451 | }; |
452 | |
453 | #else /* !CONFIG_THUMB2_KERNEL */ |
454 | |
455 | static struct undef_hook kprobes_arm_break_hook = { |
456 | .instr_mask = 0x0fffffff, |
457 | .instr_val = KPROBE_ARM_BREAKPOINT_INSTRUCTION, |
458 | .cpsr_mask = MODE_MASK, |
459 | .cpsr_val = SVC_MODE, |
460 | .fn = kprobe_trap_handler, |
461 | }; |
462 | |
463 | #endif /* !CONFIG_THUMB2_KERNEL */ |
464 | |
465 | int __init arch_init_kprobes(void) |
466 | { |
467 | arm_probes_decode_init(); |
468 | #ifdef CONFIG_THUMB2_KERNEL |
469 | register_undef_hook(&kprobes_thumb16_break_hook); |
470 | register_undef_hook(&kprobes_thumb32_break_hook); |
471 | #else |
472 | register_undef_hook(&kprobes_arm_break_hook); |
473 | #endif |
474 | return 0; |
475 | } |
476 | |
477 | bool arch_within_kprobe_blacklist(unsigned long addr) |
478 | { |
479 | void *a = (void *)addr; |
480 | |
481 | return __in_irqentry_text(addr) || |
482 | in_entry_text(addr) || |
483 | in_idmap_text(addr) || |
484 | memory_contains(begin: __kprobes_text_start, end: __kprobes_text_end, virt: a, size: 1); |
485 | } |
486 | |