1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/objtool.h> |
3 | #include <linux/module.h> |
4 | #include <linux/sort.h> |
5 | #include <asm/exception.h> |
6 | #include <asm/orc_header.h> |
7 | #include <asm/orc_lookup.h> |
8 | #include <asm/orc_types.h> |
9 | #include <asm/ptrace.h> |
10 | #include <asm/setup.h> |
11 | #include <asm/stacktrace.h> |
12 | #include <asm/tlb.h> |
13 | #include <asm/unwind.h> |
14 | |
15 | ORC_HEADER; |
16 | |
17 | #define orc_warn(fmt, ...) \ |
18 | printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__) |
19 | |
20 | extern int __start_orc_unwind_ip[]; |
21 | extern int __stop_orc_unwind_ip[]; |
22 | extern struct orc_entry __start_orc_unwind[]; |
23 | extern struct orc_entry __stop_orc_unwind[]; |
24 | |
25 | static bool orc_init __ro_after_init; |
26 | static unsigned int lookup_num_blocks __ro_after_init; |
27 | |
28 | /* Fake frame pointer entry -- used as a fallback for generated code */ |
29 | static struct orc_entry orc_fp_entry = { |
30 | .sp_reg = ORC_REG_FP, |
31 | .sp_offset = 16, |
32 | .fp_reg = ORC_REG_PREV_SP, |
33 | .fp_offset = -16, |
34 | .ra_reg = ORC_REG_PREV_SP, |
35 | .ra_offset = -8, |
36 | .type = ORC_TYPE_CALL |
37 | }; |
38 | |
39 | /* |
40 | * If we crash with IP==0, the last successfully executed instruction |
41 | * was probably an indirect function call with a NULL function pointer, |
42 | * and we don't have unwind information for NULL. |
43 | * This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function |
44 | * pointer into its parent and then continue normally from there. |
45 | */ |
46 | static struct orc_entry orc_null_entry = { |
47 | .sp_reg = ORC_REG_SP, |
48 | .sp_offset = sizeof(long), |
49 | .fp_reg = ORC_REG_UNDEFINED, |
50 | .type = ORC_TYPE_CALL |
51 | }; |
52 | |
53 | static inline unsigned long orc_ip(const int *ip) |
54 | { |
55 | return (unsigned long)ip + *ip; |
56 | } |
57 | |
58 | static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table, |
59 | unsigned int num_entries, unsigned long ip) |
60 | { |
61 | int *first = ip_table; |
62 | int *mid = first, *found = first; |
63 | int *last = ip_table + num_entries - 1; |
64 | |
65 | if (!num_entries) |
66 | return NULL; |
67 | |
68 | /* |
69 | * Do a binary range search to find the rightmost duplicate of a given |
70 | * starting address. Some entries are section terminators which are |
71 | * "weak" entries for ensuring there are no gaps. They should be |
72 | * ignored when they conflict with a real entry. |
73 | */ |
74 | while (first <= last) { |
75 | mid = first + ((last - first) / 2); |
76 | |
77 | if (orc_ip(ip: mid) <= ip) { |
78 | found = mid; |
79 | first = mid + 1; |
80 | } else |
81 | last = mid - 1; |
82 | } |
83 | |
84 | return u_table + (found - ip_table); |
85 | } |
86 | |
87 | #ifdef CONFIG_MODULES |
88 | static struct orc_entry *orc_module_find(unsigned long ip) |
89 | { |
90 | struct module *mod; |
91 | |
92 | mod = __module_address(addr: ip); |
93 | if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip) |
94 | return NULL; |
95 | |
96 | return __orc_find(ip_table: mod->arch.orc_unwind_ip, u_table: mod->arch.orc_unwind, num_entries: mod->arch.num_orcs, ip); |
97 | } |
98 | #else |
99 | static struct orc_entry *orc_module_find(unsigned long ip) |
100 | { |
101 | return NULL; |
102 | } |
103 | #endif |
104 | |
105 | #ifdef CONFIG_DYNAMIC_FTRACE |
106 | static struct orc_entry *orc_find(unsigned long ip); |
107 | |
108 | /* |
109 | * Ftrace dynamic trampolines do not have orc entries of their own. |
110 | * But they are copies of the ftrace entries that are static and |
111 | * defined in ftrace_*.S, which do have orc entries. |
112 | * |
113 | * If the unwinder comes across a ftrace trampoline, then find the |
114 | * ftrace function that was used to create it, and use that ftrace |
115 | * function's orc entry, as the placement of the return code in |
116 | * the stack will be identical. |
117 | */ |
118 | static struct orc_entry *orc_ftrace_find(unsigned long ip) |
119 | { |
120 | struct ftrace_ops *ops; |
121 | unsigned long tramp_addr, offset; |
122 | |
123 | ops = ftrace_ops_trampoline(addr: ip); |
124 | if (!ops) |
125 | return NULL; |
126 | |
127 | /* Set tramp_addr to the start of the code copied by the trampoline */ |
128 | if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) |
129 | tramp_addr = (unsigned long)ftrace_regs_caller; |
130 | else |
131 | tramp_addr = (unsigned long)ftrace_caller; |
132 | |
133 | /* Now place tramp_addr to the location within the trampoline ip is at */ |
134 | offset = ip - ops->trampoline; |
135 | tramp_addr += offset; |
136 | |
137 | /* Prevent unlikely recursion */ |
138 | if (ip == tramp_addr) |
139 | return NULL; |
140 | |
141 | return orc_find(ip: tramp_addr); |
142 | } |
143 | #else |
144 | static struct orc_entry *orc_ftrace_find(unsigned long ip) |
145 | { |
146 | return NULL; |
147 | } |
148 | #endif |
149 | |
150 | static struct orc_entry *orc_find(unsigned long ip) |
151 | { |
152 | static struct orc_entry *orc; |
153 | |
154 | if (ip == 0) |
155 | return &orc_null_entry; |
156 | |
157 | /* For non-init vmlinux addresses, use the fast lookup table: */ |
158 | if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) { |
159 | unsigned int idx, start, stop; |
160 | |
161 | idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE; |
162 | |
163 | if (unlikely((idx >= lookup_num_blocks-1))) { |
164 | orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n" , |
165 | idx, lookup_num_blocks, (void *)ip); |
166 | return NULL; |
167 | } |
168 | |
169 | start = orc_lookup[idx]; |
170 | stop = orc_lookup[idx + 1] + 1; |
171 | |
172 | if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) || |
173 | (__start_orc_unwind + stop > __stop_orc_unwind))) { |
174 | orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n" , |
175 | idx, lookup_num_blocks, start, stop, (void *)ip); |
176 | return NULL; |
177 | } |
178 | |
179 | return __orc_find(ip_table: __start_orc_unwind_ip + start, |
180 | u_table: __start_orc_unwind + start, num_entries: stop - start, ip); |
181 | } |
182 | |
183 | /* vmlinux .init slow lookup: */ |
184 | if (is_kernel_inittext(addr: ip)) |
185 | return __orc_find(ip_table: __start_orc_unwind_ip, u_table: __start_orc_unwind, |
186 | num_entries: __stop_orc_unwind_ip - __start_orc_unwind_ip, ip); |
187 | |
188 | /* Module lookup: */ |
189 | orc = orc_module_find(ip); |
190 | if (orc) |
191 | return orc; |
192 | |
193 | return orc_ftrace_find(ip); |
194 | } |
195 | |
196 | #ifdef CONFIG_MODULES |
197 | |
198 | static DEFINE_MUTEX(sort_mutex); |
199 | static int *cur_orc_ip_table = __start_orc_unwind_ip; |
200 | static struct orc_entry *cur_orc_table = __start_orc_unwind; |
201 | |
202 | static void orc_sort_swap(void *_a, void *_b, int size) |
203 | { |
204 | int delta = _b - _a; |
205 | int *a = _a, *b = _b, tmp; |
206 | struct orc_entry *orc_a, *orc_b; |
207 | |
208 | /* Swap the .orc_unwind_ip entries: */ |
209 | tmp = *a; |
210 | *a = *b + delta; |
211 | *b = tmp - delta; |
212 | |
213 | /* Swap the corresponding .orc_unwind entries: */ |
214 | orc_a = cur_orc_table + (a - cur_orc_ip_table); |
215 | orc_b = cur_orc_table + (b - cur_orc_ip_table); |
216 | swap(*orc_a, *orc_b); |
217 | } |
218 | |
219 | static int orc_sort_cmp(const void *_a, const void *_b) |
220 | { |
221 | const int *a = _a, *b = _b; |
222 | unsigned long a_val = orc_ip(ip: a); |
223 | unsigned long b_val = orc_ip(ip: b); |
224 | struct orc_entry *orc_a; |
225 | |
226 | if (a_val > b_val) |
227 | return 1; |
228 | if (a_val < b_val) |
229 | return -1; |
230 | |
231 | /* |
232 | * The "weak" section terminator entries need to always be first |
233 | * to ensure the lookup code skips them in favor of real entries. |
234 | * These terminator entries exist to handle any gaps created by |
235 | * whitelisted .o files which didn't get objtool generation. |
236 | */ |
237 | orc_a = cur_orc_table + (a - cur_orc_ip_table); |
238 | |
239 | return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; |
240 | } |
241 | |
242 | void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size, |
243 | void *_orc, size_t orc_size) |
244 | { |
245 | int *orc_ip = _orc_ip; |
246 | struct orc_entry *orc = _orc; |
247 | unsigned int num_entries = orc_ip_size / sizeof(int); |
248 | |
249 | WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 || |
250 | orc_size % sizeof(*orc) != 0 || |
251 | num_entries != orc_size / sizeof(*orc)); |
252 | |
253 | /* |
254 | * The 'cur_orc_*' globals allow the orc_sort_swap() callback to |
255 | * associate an .orc_unwind_ip table entry with its corresponding |
256 | * .orc_unwind entry so they can both be swapped. |
257 | */ |
258 | mutex_lock(&sort_mutex); |
259 | cur_orc_ip_table = orc_ip; |
260 | cur_orc_table = orc; |
261 | sort(base: orc_ip, num: num_entries, size: sizeof(int), cmp_func: orc_sort_cmp, swap_func: orc_sort_swap); |
262 | mutex_unlock(lock: &sort_mutex); |
263 | |
264 | mod->arch.orc_unwind_ip = orc_ip; |
265 | mod->arch.orc_unwind = orc; |
266 | mod->arch.num_orcs = num_entries; |
267 | } |
268 | #endif |
269 | |
270 | void __init unwind_init(void) |
271 | { |
272 | int i; |
273 | size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind; |
274 | size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip; |
275 | size_t num_entries = orc_ip_size / sizeof(int); |
276 | struct orc_entry *orc; |
277 | |
278 | if (!num_entries || orc_ip_size % sizeof(int) != 0 || |
279 | orc_size % sizeof(struct orc_entry) != 0 || |
280 | num_entries != orc_size / sizeof(struct orc_entry)) { |
281 | orc_warn("WARNING: Bad or missing .orc_unwind table. Disabling unwinder.\n" ); |
282 | return; |
283 | } |
284 | |
285 | /* |
286 | * Note, the orc_unwind and orc_unwind_ip tables were already |
287 | * sorted at build time via the 'sorttable' tool. |
288 | * It's ready for binary search straight away, no need to sort it. |
289 | */ |
290 | |
291 | /* Initialize the fast lookup table: */ |
292 | lookup_num_blocks = orc_lookup_end - orc_lookup; |
293 | for (i = 0; i < lookup_num_blocks-1; i++) { |
294 | orc = __orc_find(ip_table: __start_orc_unwind_ip, u_table: __start_orc_unwind, |
295 | num_entries, LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i)); |
296 | if (!orc) { |
297 | orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n" ); |
298 | return; |
299 | } |
300 | |
301 | orc_lookup[i] = orc - __start_orc_unwind; |
302 | } |
303 | |
304 | /* Initialize the ending block: */ |
305 | orc = __orc_find(ip_table: __start_orc_unwind_ip, u_table: __start_orc_unwind, num_entries, LOOKUP_STOP_IP); |
306 | if (!orc) { |
307 | orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n" ); |
308 | return; |
309 | } |
310 | orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind; |
311 | |
312 | orc_init = true; |
313 | } |
314 | |
315 | static inline bool on_stack(struct stack_info *info, unsigned long addr, size_t len) |
316 | { |
317 | unsigned long begin = info->begin; |
318 | unsigned long end = info->end; |
319 | |
320 | return (info->type != STACK_TYPE_UNKNOWN && |
321 | addr >= begin && addr < end && addr + len > begin && addr + len <= end); |
322 | } |
323 | |
324 | static bool stack_access_ok(struct unwind_state *state, unsigned long addr, size_t len) |
325 | { |
326 | struct stack_info *info = &state->stack_info; |
327 | |
328 | if (on_stack(info, addr, len)) |
329 | return true; |
330 | |
331 | return !get_stack_info(addr, state->task, info) && on_stack(info, addr, len); |
332 | } |
333 | |
334 | unsigned long unwind_get_return_address(struct unwind_state *state) |
335 | { |
336 | return __unwind_get_return_address(state); |
337 | } |
338 | EXPORT_SYMBOL_GPL(unwind_get_return_address); |
339 | |
340 | void unwind_start(struct unwind_state *state, struct task_struct *task, |
341 | struct pt_regs *regs) |
342 | { |
343 | __unwind_start(state, task, regs); |
344 | state->type = UNWINDER_ORC; |
345 | if (!unwind_done(state) && !__kernel_text_address(addr: state->pc)) |
346 | unwind_next_frame(state); |
347 | } |
348 | EXPORT_SYMBOL_GPL(unwind_start); |
349 | |
350 | static bool is_entry_func(unsigned long addr) |
351 | { |
352 | extern u32 kernel_entry; |
353 | extern u32 kernel_entry_end; |
354 | |
355 | return addr >= (unsigned long)&kernel_entry && addr < (unsigned long)&kernel_entry_end; |
356 | } |
357 | |
358 | static inline unsigned long bt_address(unsigned long ra) |
359 | { |
360 | extern unsigned long eentry; |
361 | |
362 | if (__kernel_text_address(addr: ra)) |
363 | return ra; |
364 | |
365 | if (__module_text_address(addr: ra)) |
366 | return ra; |
367 | |
368 | if (ra >= eentry && ra < eentry + EXCCODE_INT_END * VECSIZE) { |
369 | unsigned long func; |
370 | unsigned long type = (ra - eentry) / VECSIZE; |
371 | unsigned long offset = (ra - eentry) % VECSIZE; |
372 | |
373 | switch (type) { |
374 | case 0 ... EXCCODE_INT_START - 1: |
375 | func = (unsigned long)exception_table[type]; |
376 | break; |
377 | case EXCCODE_INT_START ... EXCCODE_INT_END: |
378 | func = (unsigned long)handle_vint; |
379 | break; |
380 | default: |
381 | func = (unsigned long)handle_reserved; |
382 | break; |
383 | } |
384 | |
385 | return func + offset; |
386 | } |
387 | |
388 | return ra; |
389 | } |
390 | |
391 | bool unwind_next_frame(struct unwind_state *state) |
392 | { |
393 | unsigned long *p, pc; |
394 | struct pt_regs *regs; |
395 | struct orc_entry *orc; |
396 | struct stack_info *info = &state->stack_info; |
397 | |
398 | if (unwind_done(state)) |
399 | return false; |
400 | |
401 | /* Don't let modules unload while we're reading their ORC data. */ |
402 | preempt_disable(); |
403 | |
404 | if (is_entry_func(addr: state->pc)) |
405 | goto end; |
406 | |
407 | orc = orc_find(ip: state->pc); |
408 | if (!orc) { |
409 | /* |
410 | * As a fallback, try to assume this code uses a frame pointer. |
411 | * This is useful for generated code, like BPF, which ORC |
412 | * doesn't know about. This is just a guess, so the rest of |
413 | * the unwind is no longer considered reliable. |
414 | */ |
415 | orc = &orc_fp_entry; |
416 | state->error = true; |
417 | } else { |
418 | if (orc->type == ORC_TYPE_UNDEFINED) |
419 | goto err; |
420 | |
421 | if (orc->type == ORC_TYPE_END_OF_STACK) |
422 | goto end; |
423 | } |
424 | |
425 | switch (orc->sp_reg) { |
426 | case ORC_REG_SP: |
427 | if (info->type == STACK_TYPE_IRQ && state->sp == info->end) |
428 | orc->type = ORC_TYPE_REGS; |
429 | else |
430 | state->sp = state->sp + orc->sp_offset; |
431 | break; |
432 | case ORC_REG_FP: |
433 | state->sp = state->fp; |
434 | break; |
435 | default: |
436 | orc_warn("unknown SP base reg %d at %pB\n" , orc->sp_reg, (void *)state->pc); |
437 | goto err; |
438 | } |
439 | |
440 | switch (orc->fp_reg) { |
441 | case ORC_REG_PREV_SP: |
442 | p = (unsigned long *)(state->sp + orc->fp_offset); |
443 | if (!stack_access_ok(state, addr: (unsigned long)p, len: sizeof(unsigned long))) |
444 | goto err; |
445 | |
446 | state->fp = *p; |
447 | break; |
448 | case ORC_REG_UNDEFINED: |
449 | /* Nothing. */ |
450 | break; |
451 | default: |
452 | orc_warn("unknown FP base reg %d at %pB\n" , orc->fp_reg, (void *)state->pc); |
453 | goto err; |
454 | } |
455 | |
456 | switch (orc->type) { |
457 | case ORC_TYPE_CALL: |
458 | if (orc->ra_reg == ORC_REG_PREV_SP) { |
459 | p = (unsigned long *)(state->sp + orc->ra_offset); |
460 | if (!stack_access_ok(state, addr: (unsigned long)p, len: sizeof(unsigned long))) |
461 | goto err; |
462 | |
463 | pc = unwind_graph_addr(state, *p, state->sp); |
464 | pc -= LOONGARCH_INSN_SIZE; |
465 | } else if (orc->ra_reg == ORC_REG_UNDEFINED) { |
466 | if (!state->ra || state->ra == state->pc) |
467 | goto err; |
468 | |
469 | pc = unwind_graph_addr(state, state->ra, state->sp); |
470 | pc -= LOONGARCH_INSN_SIZE; |
471 | state->ra = 0; |
472 | } else { |
473 | orc_warn("unknown ra base reg %d at %pB\n" , orc->ra_reg, (void *)state->pc); |
474 | goto err; |
475 | } |
476 | break; |
477 | case ORC_TYPE_REGS: |
478 | if (info->type == STACK_TYPE_IRQ && state->sp == info->end) |
479 | regs = (struct pt_regs *)info->next_sp; |
480 | else |
481 | regs = (struct pt_regs *)state->sp; |
482 | |
483 | if (!stack_access_ok(state, addr: (unsigned long)regs, len: sizeof(*regs))) |
484 | goto err; |
485 | |
486 | if ((info->end == (unsigned long)regs + sizeof(*regs)) && |
487 | !regs->regs[3] && !regs->regs[1]) |
488 | goto end; |
489 | |
490 | if (user_mode(regs)) |
491 | goto end; |
492 | |
493 | pc = regs->csr_era; |
494 | if (!__kernel_text_address(addr: pc)) |
495 | goto err; |
496 | |
497 | state->sp = regs->regs[3]; |
498 | state->ra = regs->regs[1]; |
499 | state->fp = regs->regs[22]; |
500 | get_stack_info(state->sp, state->task, info); |
501 | |
502 | break; |
503 | default: |
504 | orc_warn("unknown .orc_unwind entry type %d at %pB\n" , orc->type, (void *)state->pc); |
505 | goto err; |
506 | } |
507 | |
508 | state->pc = bt_address(ra: pc); |
509 | if (!state->pc) { |
510 | pr_err("cannot find unwind pc at %pK\n" , (void *)pc); |
511 | goto err; |
512 | } |
513 | |
514 | if (!__kernel_text_address(addr: state->pc)) |
515 | goto err; |
516 | |
517 | preempt_enable(); |
518 | return true; |
519 | |
520 | err: |
521 | state->error = true; |
522 | |
523 | end: |
524 | preempt_enable(); |
525 | state->stack_info.type = STACK_TYPE_UNKNOWN; |
526 | return false; |
527 | } |
528 | EXPORT_SYMBOL_GPL(unwind_next_frame); |
529 | |