1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * x86 implementation of rethook. Mostly copied from arch/x86/kernel/kprobes/core.c. |
4 | */ |
5 | #include <linux/bug.h> |
6 | #include <linux/rethook.h> |
7 | #include <linux/kprobes.h> |
8 | #include <linux/objtool.h> |
9 | |
10 | #include "kprobes/common.h" |
11 | |
12 | __visible void arch_rethook_trampoline_callback(struct pt_regs *regs); |
13 | |
14 | #ifndef ANNOTATE_NOENDBR |
15 | #define ANNOTATE_NOENDBR |
16 | #endif |
17 | |
18 | /* |
19 | * When a target function returns, this code saves registers and calls |
20 | * arch_rethook_trampoline_callback(), which calls the rethook handler. |
21 | */ |
22 | asm( |
23 | ".text\n" |
24 | ".global arch_rethook_trampoline\n" |
25 | ".type arch_rethook_trampoline, @function\n" |
26 | "arch_rethook_trampoline:\n" |
27 | #ifdef CONFIG_X86_64 |
28 | ANNOTATE_NOENDBR /* This is only jumped from ret instruction */ |
29 | /* Push a fake return address to tell the unwinder it's a rethook. */ |
30 | " pushq $arch_rethook_trampoline\n" |
31 | UNWIND_HINT_FUNC |
32 | " pushq $" __stringify(__KERNEL_DS) "\n" |
33 | /* Save the 'sp - 16', this will be fixed later. */ |
34 | " pushq %rsp\n" |
35 | " pushfq\n" |
36 | SAVE_REGS_STRING |
37 | " movq %rsp, %rdi\n" |
38 | " call arch_rethook_trampoline_callback\n" |
39 | RESTORE_REGS_STRING |
40 | /* In the callback function, 'regs->flags' is copied to 'regs->ss'. */ |
41 | " addq $16, %rsp\n" |
42 | " popfq\n" |
43 | #else |
44 | /* Push a fake return address to tell the unwinder it's a rethook. */ |
45 | " pushl $arch_rethook_trampoline\n" |
46 | UNWIND_HINT_FUNC |
47 | " pushl %ss\n" |
48 | /* Save the 'sp - 8', this will be fixed later. */ |
49 | " pushl %esp\n" |
50 | " pushfl\n" |
51 | SAVE_REGS_STRING |
52 | " movl %esp, %eax\n" |
53 | " call arch_rethook_trampoline_callback\n" |
54 | RESTORE_REGS_STRING |
55 | /* In the callback function, 'regs->flags' is copied to 'regs->ss'. */ |
56 | " addl $8, %esp\n" |
57 | " popfl\n" |
58 | #endif |
59 | ASM_RET |
60 | ".size arch_rethook_trampoline, .-arch_rethook_trampoline\n" |
61 | ); |
62 | NOKPROBE_SYMBOL(arch_rethook_trampoline); |
63 | |
64 | /* |
65 | * Called from arch_rethook_trampoline |
66 | */ |
67 | __used __visible void arch_rethook_trampoline_callback(struct pt_regs *regs) |
68 | { |
69 | unsigned long *frame_pointer; |
70 | |
71 | /* fixup registers */ |
72 | regs->cs = __KERNEL_CS; |
73 | #ifdef CONFIG_X86_32 |
74 | regs->gs = 0; |
75 | #endif |
76 | regs->ip = (unsigned long)&arch_rethook_trampoline; |
77 | regs->orig_ax = ~0UL; |
78 | regs->sp += 2*sizeof(long); |
79 | frame_pointer = (long *)(regs + 1); |
80 | |
81 | /* |
82 | * The return address at 'frame_pointer' is recovered by the |
83 | * arch_rethook_fixup_return() which called from this |
84 | * rethook_trampoline_handler(). |
85 | */ |
86 | rethook_trampoline_handler(regs, frame: (unsigned long)frame_pointer); |
87 | |
88 | /* |
89 | * Copy FLAGS to 'pt_regs::ss' so that arch_rethook_trapmoline() |
90 | * can do RET right after POPF. |
91 | */ |
92 | *(unsigned long *)®s->ss = regs->flags; |
93 | } |
94 | NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); |
95 | |
96 | /* |
97 | * arch_rethook_trampoline() skips updating frame pointer. The frame pointer |
98 | * saved in arch_rethook_trampoline_callback() points to the real caller |
99 | * function's frame pointer. Thus the arch_rethook_trampoline() doesn't have |
100 | * a standard stack frame with CONFIG_FRAME_POINTER=y. |
101 | * Let's mark it non-standard function. Anyway, FP unwinder can correctly |
102 | * unwind without the hint. |
103 | */ |
104 | STACK_FRAME_NON_STANDARD_FP(arch_rethook_trampoline); |
105 | |
106 | /* This is called from rethook_trampoline_handler(). */ |
107 | void arch_rethook_fixup_return(struct pt_regs *regs, |
108 | unsigned long correct_ret_addr) |
109 | { |
110 | unsigned long *frame_pointer = (void *)(regs + 1); |
111 | |
112 | /* Replace fake return address with real one. */ |
113 | *frame_pointer = correct_ret_addr; |
114 | } |
115 | NOKPROBE_SYMBOL(arch_rethook_fixup_return); |
116 | |
117 | void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount) |
118 | { |
119 | unsigned long *stack = (unsigned long *)regs->sp; |
120 | |
121 | rh->ret_addr = stack[0]; |
122 | rh->frame = regs->sp; |
123 | |
124 | /* Replace the return addr with trampoline addr */ |
125 | stack[0] = (unsigned long) arch_rethook_trampoline; |
126 | } |
127 | NOKPROBE_SYMBOL(arch_rethook_prepare); |
128 | |