1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * Copyright (C) 2017 Steven Rostedt, VMware Inc. |
4 | */ |
5 | |
6 | #include <linux/export.h> |
7 | #include <linux/linkage.h> |
8 | #include <asm/page_types.h> |
9 | #include <asm/segment.h> |
10 | #include <asm/ftrace.h> |
11 | #include <asm/nospec-branch.h> |
12 | #include <asm/frame.h> |
13 | #include <asm/asm-offsets.h> |
14 | |
15 | #ifdef CONFIG_FRAME_POINTER |
16 | # define MCOUNT_FRAME 1 /* using frame = true */ |
17 | #else |
18 | # define MCOUNT_FRAME 0 /* using frame = false */ |
19 | #endif |
20 | |
21 | SYM_FUNC_START(__fentry__) |
22 | RET |
23 | SYM_FUNC_END(__fentry__) |
24 | EXPORT_SYMBOL(__fentry__) |
25 | |
26 | SYM_CODE_START(ftrace_caller) |
27 | |
28 | #ifdef CONFIG_FRAME_POINTER |
29 | /* |
30 | * Frame pointers are of ip followed by bp. |
31 | * Since fentry is an immediate jump, we are left with |
32 | * parent-ip, function-ip. We need to add a frame with |
33 | * parent-ip followed by ebp. |
34 | */ |
35 | pushl 4(%esp) /* parent ip */ |
36 | pushl %ebp |
37 | movl %esp, %ebp |
38 | pushl 2*4(%esp) /* function ip */ |
39 | |
40 | /* For mcount, the function ip is directly above */ |
41 | pushl %ebp |
42 | movl %esp, %ebp |
43 | #endif |
44 | pushl %eax |
45 | pushl %ecx |
46 | pushl %edx |
47 | pushl $0 /* Pass NULL as regs pointer */ |
48 | |
49 | #ifdef CONFIG_FRAME_POINTER |
50 | /* Load parent ebp into edx */ |
51 | movl 4*4(%esp), %edx |
52 | #else |
53 | /* There's no frame pointer, load the appropriate stack addr instead */ |
54 | lea 4*4(%esp), %edx |
55 | #endif |
56 | |
57 | movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */ |
58 | /* Get the parent ip */ |
59 | movl 4(%edx), %edx /* edx has ebp */ |
60 | |
61 | movl function_trace_op, %ecx |
62 | subl $MCOUNT_INSN_SIZE, %eax |
63 | |
64 | .globl ftrace_call |
65 | ftrace_call: |
66 | call ftrace_stub |
67 | |
68 | addl $4, %esp /* skip NULL pointer */ |
69 | popl %edx |
70 | popl %ecx |
71 | popl %eax |
72 | #ifdef CONFIG_FRAME_POINTER |
73 | popl %ebp |
74 | addl $4,%esp /* skip function ip */ |
75 | popl %ebp /* this is the orig bp */ |
76 | addl $4, %esp /* skip parent ip */ |
77 | #endif |
78 | .Lftrace_ret: |
79 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
80 | .globl ftrace_graph_call |
81 | ftrace_graph_call: |
82 | jmp ftrace_stub |
83 | #endif |
84 | |
85 | /* This is weak to keep gas from relaxing the jumps */ |
86 | SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) |
87 | RET |
88 | SYM_CODE_END(ftrace_caller) |
89 | |
90 | SYM_CODE_START(ftrace_regs_caller) |
91 | /* |
92 | * We're here from an mcount/fentry CALL, and the stack frame looks like: |
93 | * |
94 | * <previous context> |
95 | * RET-IP |
96 | * |
97 | * The purpose of this function is to call out in an emulated INT3 |
98 | * environment with a stack frame like: |
99 | * |
100 | * <previous context> |
101 | * gap / RET-IP |
102 | * gap |
103 | * gap |
104 | * gap |
105 | * pt_regs |
106 | * |
107 | * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds |
108 | */ |
109 | subl $3*4, %esp # RET-IP + 3 gaps |
110 | pushl %ss # ss |
111 | pushl %esp # points at ss |
112 | addl $5*4, (%esp) # make it point at <previous context> |
113 | pushfl # flags |
114 | pushl $__KERNEL_CS # cs |
115 | pushl 7*4(%esp) # ip <- RET-IP |
116 | pushl $0 # orig_eax |
117 | |
118 | pushl %gs |
119 | pushl %fs |
120 | pushl %es |
121 | pushl %ds |
122 | |
123 | pushl %eax |
124 | pushl %ebp |
125 | pushl %edi |
126 | pushl %esi |
127 | pushl %edx |
128 | pushl %ecx |
129 | pushl %ebx |
130 | |
131 | ENCODE_FRAME_POINTER |
132 | |
133 | movl PT_EIP(%esp), %eax # 1st argument: IP |
134 | subl $MCOUNT_INSN_SIZE, %eax |
135 | movl 21*4(%esp), %edx # 2nd argument: parent ip |
136 | movl function_trace_op, %ecx # 3rd argument: ftrace_pos |
137 | pushl %esp # 4th argument: pt_regs |
138 | |
139 | SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) |
140 | call ftrace_stub |
141 | |
142 | addl $4, %esp # skip 4th argument |
143 | |
144 | /* place IP below the new SP */ |
145 | movl PT_OLDESP(%esp), %eax |
146 | movl PT_EIP(%esp), %ecx |
147 | movl %ecx, -4(%eax) |
148 | |
149 | /* place EAX below that */ |
150 | movl PT_EAX(%esp), %ecx |
151 | movl %ecx, -8(%eax) |
152 | |
153 | popl %ebx |
154 | popl %ecx |
155 | popl %edx |
156 | popl %esi |
157 | popl %edi |
158 | popl %ebp |
159 | |
160 | lea -8(%eax), %esp |
161 | popl %eax |
162 | |
163 | jmp .Lftrace_ret |
164 | SYM_CODE_END(ftrace_regs_caller) |
165 | |
166 | SYM_FUNC_START(ftrace_stub_direct_tramp) |
167 | CALL_DEPTH_ACCOUNT |
168 | RET |
169 | SYM_FUNC_END(ftrace_stub_direct_tramp) |
170 | |
171 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
172 | SYM_CODE_START(ftrace_graph_caller) |
173 | pushl %eax |
174 | pushl %ecx |
175 | pushl %edx |
176 | movl 3*4(%esp), %eax |
177 | /* Even with frame pointers, fentry doesn't have one here */ |
178 | lea 4*4(%esp), %edx |
179 | movl $0, %ecx |
180 | subl $MCOUNT_INSN_SIZE, %eax |
181 | call prepare_ftrace_return |
182 | popl %edx |
183 | popl %ecx |
184 | popl %eax |
185 | RET |
186 | SYM_CODE_END(ftrace_graph_caller) |
187 | |
188 | .globl return_to_handler |
189 | return_to_handler: |
190 | pushl $0 |
191 | pushl %edx |
192 | pushl %eax |
193 | movl %esp, %eax |
194 | call ftrace_return_to_handler |
195 | movl %eax, %ecx |
196 | popl %eax |
197 | popl %edx |
198 | addl $4, %esp # skip ebp |
199 | JMP_NOSPEC ecx |
200 | #endif |
201 | |