1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/objtool.h> |
3 | #include <linux/module.h> |
4 | #include <linux/sort.h> |
5 | #include <asm/ptrace.h> |
6 | #include <asm/stacktrace.h> |
7 | #include <asm/unwind.h> |
8 | #include <asm/orc_types.h> |
9 | #include <asm/orc_lookup.h> |
10 | #include <asm/orc_header.h> |
11 | |
12 | ORC_HEADER; |
13 | |
14 | #define orc_warn(fmt, ...) \ |
15 | printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__) |
16 | |
17 | #define orc_warn_current(args...) \ |
18 | ({ \ |
19 | static bool dumped_before; \ |
20 | if (state->task == current && !state->error) { \ |
21 | orc_warn(args); \ |
22 | if (unwind_debug && !dumped_before) { \ |
23 | dumped_before = true; \ |
24 | unwind_dump(state); \ |
25 | } \ |
26 | } \ |
27 | }) |
28 | |
29 | extern int __start_orc_unwind_ip[]; |
30 | extern int __stop_orc_unwind_ip[]; |
31 | extern struct orc_entry __start_orc_unwind[]; |
32 | extern struct orc_entry __stop_orc_unwind[]; |
33 | |
34 | static bool orc_init __ro_after_init; |
35 | static bool unwind_debug __ro_after_init; |
36 | static unsigned int lookup_num_blocks __ro_after_init; |
37 | |
38 | static int __init unwind_debug_cmdline(char *str) |
39 | { |
40 | unwind_debug = true; |
41 | |
42 | return 0; |
43 | } |
44 | early_param("unwind_debug" , unwind_debug_cmdline); |
45 | |
46 | static void unwind_dump(struct unwind_state *state) |
47 | { |
48 | static bool dumped_before; |
49 | unsigned long word, *sp; |
50 | struct stack_info stack_info = {0}; |
51 | unsigned long visit_mask = 0; |
52 | |
53 | if (dumped_before) |
54 | return; |
55 | |
56 | dumped_before = true; |
57 | |
58 | printk_deferred("unwind stack type:%d next_sp:%p mask:0x%lx graph_idx:%d\n" , |
59 | state->stack_info.type, state->stack_info.next_sp, |
60 | state->stack_mask, state->graph_idx); |
61 | |
62 | for (sp = __builtin_frame_address(0); sp; |
63 | sp = PTR_ALIGN(stack_info.next_sp, sizeof(long))) { |
64 | if (get_stack_info(stack: sp, task: state->task, info: &stack_info, visit_mask: &visit_mask)) |
65 | break; |
66 | |
67 | for (; sp < stack_info.end; sp++) { |
68 | |
69 | word = READ_ONCE_NOCHECK(*sp); |
70 | |
71 | printk_deferred("%0*lx: %0*lx (%pB)\n" , BITS_PER_LONG/4, |
72 | (unsigned long)sp, BITS_PER_LONG/4, |
73 | word, (void *)word); |
74 | } |
75 | } |
76 | } |
77 | |
78 | static inline unsigned long orc_ip(const int *ip) |
79 | { |
80 | return (unsigned long)ip + *ip; |
81 | } |
82 | |
83 | static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table, |
84 | unsigned int num_entries, unsigned long ip) |
85 | { |
86 | int *first = ip_table; |
87 | int *last = ip_table + num_entries - 1; |
88 | int *mid, *found = first; |
89 | |
90 | if (!num_entries) |
91 | return NULL; |
92 | |
93 | /* |
94 | * Do a binary range search to find the rightmost duplicate of a given |
95 | * starting address. Some entries are section terminators which are |
96 | * "weak" entries for ensuring there are no gaps. They should be |
97 | * ignored when they conflict with a real entry. |
98 | */ |
99 | while (first <= last) { |
100 | mid = first + ((last - first) / 2); |
101 | |
102 | if (orc_ip(ip: mid) <= ip) { |
103 | found = mid; |
104 | first = mid + 1; |
105 | } else |
106 | last = mid - 1; |
107 | } |
108 | |
109 | return u_table + (found - ip_table); |
110 | } |
111 | |
112 | #ifdef CONFIG_MODULES |
113 | static struct orc_entry *orc_module_find(unsigned long ip) |
114 | { |
115 | struct module *mod; |
116 | |
117 | mod = __module_address(addr: ip); |
118 | if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip) |
119 | return NULL; |
120 | return __orc_find(ip_table: mod->arch.orc_unwind_ip, u_table: mod->arch.orc_unwind, |
121 | num_entries: mod->arch.num_orcs, ip); |
122 | } |
123 | #else |
124 | static struct orc_entry *orc_module_find(unsigned long ip) |
125 | { |
126 | return NULL; |
127 | } |
128 | #endif |
129 | |
130 | #ifdef CONFIG_DYNAMIC_FTRACE |
131 | static struct orc_entry *orc_find(unsigned long ip); |
132 | |
133 | /* |
134 | * Ftrace dynamic trampolines do not have orc entries of their own. |
135 | * But they are copies of the ftrace entries that are static and |
136 | * defined in ftrace_*.S, which do have orc entries. |
137 | * |
138 | * If the unwinder comes across a ftrace trampoline, then find the |
139 | * ftrace function that was used to create it, and use that ftrace |
140 | * function's orc entry, as the placement of the return code in |
141 | * the stack will be identical. |
142 | */ |
143 | static struct orc_entry *orc_ftrace_find(unsigned long ip) |
144 | { |
145 | struct ftrace_ops *ops; |
146 | unsigned long tramp_addr, offset; |
147 | |
148 | ops = ftrace_ops_trampoline(addr: ip); |
149 | if (!ops) |
150 | return NULL; |
151 | |
152 | /* Set tramp_addr to the start of the code copied by the trampoline */ |
153 | if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) |
154 | tramp_addr = (unsigned long)ftrace_regs_caller; |
155 | else |
156 | tramp_addr = (unsigned long)ftrace_caller; |
157 | |
158 | /* Now place tramp_addr to the location within the trampoline ip is at */ |
159 | offset = ip - ops->trampoline; |
160 | tramp_addr += offset; |
161 | |
162 | /* Prevent unlikely recursion */ |
163 | if (ip == tramp_addr) |
164 | return NULL; |
165 | |
166 | return orc_find(ip: tramp_addr); |
167 | } |
168 | #else |
169 | static struct orc_entry *orc_ftrace_find(unsigned long ip) |
170 | { |
171 | return NULL; |
172 | } |
173 | #endif |
174 | |
175 | /* |
176 | * If we crash with IP==0, the last successfully executed instruction |
177 | * was probably an indirect function call with a NULL function pointer, |
178 | * and we don't have unwind information for NULL. |
179 | * This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function |
180 | * pointer into its parent and then continue normally from there. |
181 | */ |
182 | static struct orc_entry null_orc_entry = { |
183 | .sp_offset = sizeof(long), |
184 | .sp_reg = ORC_REG_SP, |
185 | .bp_reg = ORC_REG_UNDEFINED, |
186 | .type = ORC_TYPE_CALL |
187 | }; |
188 | |
189 | /* Fake frame pointer entry -- used as a fallback for generated code */ |
190 | static struct orc_entry orc_fp_entry = { |
191 | .type = ORC_TYPE_CALL, |
192 | .sp_reg = ORC_REG_BP, |
193 | .sp_offset = 16, |
194 | .bp_reg = ORC_REG_PREV_SP, |
195 | .bp_offset = -16, |
196 | }; |
197 | |
198 | static struct orc_entry *orc_find(unsigned long ip) |
199 | { |
200 | static struct orc_entry *orc; |
201 | |
202 | if (ip == 0) |
203 | return &null_orc_entry; |
204 | |
205 | /* For non-init vmlinux addresses, use the fast lookup table: */ |
206 | if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) { |
207 | unsigned int idx, start, stop; |
208 | |
209 | idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE; |
210 | |
211 | if (unlikely((idx >= lookup_num_blocks-1))) { |
212 | orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n" , |
213 | idx, lookup_num_blocks, (void *)ip); |
214 | return NULL; |
215 | } |
216 | |
217 | start = orc_lookup[idx]; |
218 | stop = orc_lookup[idx + 1] + 1; |
219 | |
220 | if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) || |
221 | (__start_orc_unwind + stop > __stop_orc_unwind))) { |
222 | orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n" , |
223 | idx, lookup_num_blocks, start, stop, (void *)ip); |
224 | return NULL; |
225 | } |
226 | |
227 | return __orc_find(ip_table: __start_orc_unwind_ip + start, |
228 | u_table: __start_orc_unwind + start, num_entries: stop - start, ip); |
229 | } |
230 | |
231 | /* vmlinux .init slow lookup: */ |
232 | if (is_kernel_inittext(addr: ip)) |
233 | return __orc_find(ip_table: __start_orc_unwind_ip, u_table: __start_orc_unwind, |
234 | num_entries: __stop_orc_unwind_ip - __start_orc_unwind_ip, ip); |
235 | |
236 | /* Module lookup: */ |
237 | orc = orc_module_find(ip); |
238 | if (orc) |
239 | return orc; |
240 | |
241 | return orc_ftrace_find(ip); |
242 | } |
243 | |
244 | #ifdef CONFIG_MODULES |
245 | |
246 | static DEFINE_MUTEX(sort_mutex); |
247 | static int *cur_orc_ip_table = __start_orc_unwind_ip; |
248 | static struct orc_entry *cur_orc_table = __start_orc_unwind; |
249 | |
250 | static void orc_sort_swap(void *_a, void *_b, int size) |
251 | { |
252 | struct orc_entry *orc_a, *orc_b; |
253 | int *a = _a, *b = _b, tmp; |
254 | int delta = _b - _a; |
255 | |
256 | /* Swap the .orc_unwind_ip entries: */ |
257 | tmp = *a; |
258 | *a = *b + delta; |
259 | *b = tmp - delta; |
260 | |
261 | /* Swap the corresponding .orc_unwind entries: */ |
262 | orc_a = cur_orc_table + (a - cur_orc_ip_table); |
263 | orc_b = cur_orc_table + (b - cur_orc_ip_table); |
264 | swap(*orc_a, *orc_b); |
265 | } |
266 | |
267 | static int orc_sort_cmp(const void *_a, const void *_b) |
268 | { |
269 | struct orc_entry *orc_a; |
270 | const int *a = _a, *b = _b; |
271 | unsigned long a_val = orc_ip(ip: a); |
272 | unsigned long b_val = orc_ip(ip: b); |
273 | |
274 | if (a_val > b_val) |
275 | return 1; |
276 | if (a_val < b_val) |
277 | return -1; |
278 | |
279 | /* |
280 | * The "weak" section terminator entries need to always be first |
281 | * to ensure the lookup code skips them in favor of real entries. |
282 | * These terminator entries exist to handle any gaps created by |
283 | * whitelisted .o files which didn't get objtool generation. |
284 | */ |
285 | orc_a = cur_orc_table + (a - cur_orc_ip_table); |
286 | return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; |
287 | } |
288 | |
289 | void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size, |
290 | void *_orc, size_t orc_size) |
291 | { |
292 | int *orc_ip = _orc_ip; |
293 | struct orc_entry *orc = _orc; |
294 | unsigned int num_entries = orc_ip_size / sizeof(int); |
295 | |
296 | WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 || |
297 | orc_size % sizeof(*orc) != 0 || |
298 | num_entries != orc_size / sizeof(*orc)); |
299 | |
300 | /* |
301 | * The 'cur_orc_*' globals allow the orc_sort_swap() callback to |
302 | * associate an .orc_unwind_ip table entry with its corresponding |
303 | * .orc_unwind entry so they can both be swapped. |
304 | */ |
305 | mutex_lock(&sort_mutex); |
306 | cur_orc_ip_table = orc_ip; |
307 | cur_orc_table = orc; |
308 | sort(base: orc_ip, num: num_entries, size: sizeof(int), cmp_func: orc_sort_cmp, swap_func: orc_sort_swap); |
309 | mutex_unlock(lock: &sort_mutex); |
310 | |
311 | mod->arch.orc_unwind_ip = orc_ip; |
312 | mod->arch.orc_unwind = orc; |
313 | mod->arch.num_orcs = num_entries; |
314 | } |
315 | #endif |
316 | |
317 | void __init unwind_init(void) |
318 | { |
319 | size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip; |
320 | size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind; |
321 | size_t num_entries = orc_ip_size / sizeof(int); |
322 | struct orc_entry *orc; |
323 | int i; |
324 | |
325 | if (!num_entries || orc_ip_size % sizeof(int) != 0 || |
326 | orc_size % sizeof(struct orc_entry) != 0 || |
327 | num_entries != orc_size / sizeof(struct orc_entry)) { |
328 | orc_warn("WARNING: Bad or missing .orc_unwind table. Disabling unwinder.\n" ); |
329 | return; |
330 | } |
331 | |
332 | /* |
333 | * Note, the orc_unwind and orc_unwind_ip tables were already |
334 | * sorted at build time via the 'sorttable' tool. |
335 | * It's ready for binary search straight away, no need to sort it. |
336 | */ |
337 | |
338 | /* Initialize the fast lookup table: */ |
339 | lookup_num_blocks = orc_lookup_end - orc_lookup; |
340 | for (i = 0; i < lookup_num_blocks-1; i++) { |
341 | orc = __orc_find(ip_table: __start_orc_unwind_ip, u_table: __start_orc_unwind, |
342 | num_entries, |
343 | LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i)); |
344 | if (!orc) { |
345 | orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n" ); |
346 | return; |
347 | } |
348 | |
349 | orc_lookup[i] = orc - __start_orc_unwind; |
350 | } |
351 | |
352 | /* Initialize the ending block: */ |
353 | orc = __orc_find(ip_table: __start_orc_unwind_ip, u_table: __start_orc_unwind, num_entries, |
354 | LOOKUP_STOP_IP); |
355 | if (!orc) { |
356 | orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n" ); |
357 | return; |
358 | } |
359 | orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind; |
360 | |
361 | orc_init = true; |
362 | } |
363 | |
364 | unsigned long unwind_get_return_address(struct unwind_state *state) |
365 | { |
366 | if (unwind_done(state)) |
367 | return 0; |
368 | |
369 | return __kernel_text_address(addr: state->ip) ? state->ip : 0; |
370 | } |
371 | EXPORT_SYMBOL_GPL(unwind_get_return_address); |
372 | |
373 | unsigned long *unwind_get_return_address_ptr(struct unwind_state *state) |
374 | { |
375 | if (unwind_done(state)) |
376 | return NULL; |
377 | |
378 | if (state->regs) |
379 | return &state->regs->ip; |
380 | |
381 | if (state->sp) |
382 | return (unsigned long *)state->sp - 1; |
383 | |
384 | return NULL; |
385 | } |
386 | |
387 | static bool stack_access_ok(struct unwind_state *state, unsigned long _addr, |
388 | size_t len) |
389 | { |
390 | struct stack_info *info = &state->stack_info; |
391 | void *addr = (void *)_addr; |
392 | |
393 | if (on_stack(info, addr, len)) |
394 | return true; |
395 | |
396 | return !get_stack_info(stack: addr, task: state->task, info, visit_mask: &state->stack_mask) && |
397 | on_stack(info, addr, len); |
398 | } |
399 | |
400 | static bool deref_stack_reg(struct unwind_state *state, unsigned long addr, |
401 | unsigned long *val) |
402 | { |
403 | if (!stack_access_ok(state, addr: addr, len: sizeof(long))) |
404 | return false; |
405 | |
406 | *val = READ_ONCE_NOCHECK(*(unsigned long *)addr); |
407 | return true; |
408 | } |
409 | |
410 | static bool deref_stack_regs(struct unwind_state *state, unsigned long addr, |
411 | unsigned long *ip, unsigned long *sp) |
412 | { |
413 | struct pt_regs *regs = (struct pt_regs *)addr; |
414 | |
415 | /* x86-32 support will be more complicated due to the ®s->sp hack */ |
416 | BUILD_BUG_ON(IS_ENABLED(CONFIG_X86_32)); |
417 | |
418 | if (!stack_access_ok(state, addr: addr, len: sizeof(struct pt_regs))) |
419 | return false; |
420 | |
421 | *ip = READ_ONCE_NOCHECK(regs->ip); |
422 | *sp = READ_ONCE_NOCHECK(regs->sp); |
423 | return true; |
424 | } |
425 | |
426 | static bool deref_stack_iret_regs(struct unwind_state *state, unsigned long addr, |
427 | unsigned long *ip, unsigned long *sp) |
428 | { |
429 | struct pt_regs *regs = (void *)addr - IRET_FRAME_OFFSET; |
430 | |
431 | if (!stack_access_ok(state, addr: addr, IRET_FRAME_SIZE)) |
432 | return false; |
433 | |
434 | *ip = READ_ONCE_NOCHECK(regs->ip); |
435 | *sp = READ_ONCE_NOCHECK(regs->sp); |
436 | return true; |
437 | } |
438 | |
439 | /* |
440 | * If state->regs is non-NULL, and points to a full pt_regs, just get the reg |
441 | * value from state->regs. |
442 | * |
443 | * Otherwise, if state->regs just points to IRET regs, and the previous frame |
444 | * had full regs, it's safe to get the value from the previous regs. This can |
445 | * happen when early/late IRQ entry code gets interrupted by an NMI. |
446 | */ |
447 | static bool get_reg(struct unwind_state *state, unsigned int reg_off, |
448 | unsigned long *val) |
449 | { |
450 | unsigned int reg = reg_off/8; |
451 | |
452 | if (!state->regs) |
453 | return false; |
454 | |
455 | if (state->full_regs) { |
456 | *val = READ_ONCE_NOCHECK(((unsigned long *)state->regs)[reg]); |
457 | return true; |
458 | } |
459 | |
460 | if (state->prev_regs) { |
461 | *val = READ_ONCE_NOCHECK(((unsigned long *)state->prev_regs)[reg]); |
462 | return true; |
463 | } |
464 | |
465 | return false; |
466 | } |
467 | |
468 | bool unwind_next_frame(struct unwind_state *state) |
469 | { |
470 | unsigned long ip_p, sp, tmp, orig_ip = state->ip, prev_sp = state->sp; |
471 | enum stack_type prev_type = state->stack_info.type; |
472 | struct orc_entry *orc; |
473 | bool indirect = false; |
474 | |
475 | if (unwind_done(state)) |
476 | return false; |
477 | |
478 | /* Don't let modules unload while we're reading their ORC data. */ |
479 | preempt_disable(); |
480 | |
481 | /* End-of-stack check for user tasks: */ |
482 | if (state->regs && user_mode(regs: state->regs)) |
483 | goto the_end; |
484 | |
485 | /* |
486 | * Find the orc_entry associated with the text address. |
487 | * |
488 | * For a call frame (as opposed to a signal frame), state->ip points to |
489 | * the instruction after the call. That instruction's stack layout |
490 | * could be different from the call instruction's layout, for example |
491 | * if the call was to a noreturn function. So get the ORC data for the |
492 | * call instruction itself. |
493 | */ |
494 | orc = orc_find(ip: state->signal ? state->ip : state->ip - 1); |
495 | if (!orc) { |
496 | /* |
497 | * As a fallback, try to assume this code uses a frame pointer. |
498 | * This is useful for generated code, like BPF, which ORC |
499 | * doesn't know about. This is just a guess, so the rest of |
500 | * the unwind is no longer considered reliable. |
501 | */ |
502 | orc = &orc_fp_entry; |
503 | state->error = true; |
504 | } else { |
505 | if (orc->type == ORC_TYPE_UNDEFINED) |
506 | goto err; |
507 | |
508 | if (orc->type == ORC_TYPE_END_OF_STACK) |
509 | goto the_end; |
510 | } |
511 | |
512 | state->signal = orc->signal; |
513 | |
514 | /* Find the previous frame's stack: */ |
515 | switch (orc->sp_reg) { |
516 | case ORC_REG_SP: |
517 | sp = state->sp + orc->sp_offset; |
518 | break; |
519 | |
520 | case ORC_REG_BP: |
521 | sp = state->bp + orc->sp_offset; |
522 | break; |
523 | |
524 | case ORC_REG_SP_INDIRECT: |
525 | sp = state->sp; |
526 | indirect = true; |
527 | break; |
528 | |
529 | case ORC_REG_BP_INDIRECT: |
530 | sp = state->bp + orc->sp_offset; |
531 | indirect = true; |
532 | break; |
533 | |
534 | case ORC_REG_R10: |
535 | if (!get_reg(state, offsetof(struct pt_regs, r10), val: &sp)) { |
536 | orc_warn_current("missing R10 value at %pB\n" , |
537 | (void *)state->ip); |
538 | goto err; |
539 | } |
540 | break; |
541 | |
542 | case ORC_REG_R13: |
543 | if (!get_reg(state, offsetof(struct pt_regs, r13), val: &sp)) { |
544 | orc_warn_current("missing R13 value at %pB\n" , |
545 | (void *)state->ip); |
546 | goto err; |
547 | } |
548 | break; |
549 | |
550 | case ORC_REG_DI: |
551 | if (!get_reg(state, offsetof(struct pt_regs, di), val: &sp)) { |
552 | orc_warn_current("missing RDI value at %pB\n" , |
553 | (void *)state->ip); |
554 | goto err; |
555 | } |
556 | break; |
557 | |
558 | case ORC_REG_DX: |
559 | if (!get_reg(state, offsetof(struct pt_regs, dx), val: &sp)) { |
560 | orc_warn_current("missing DX value at %pB\n" , |
561 | (void *)state->ip); |
562 | goto err; |
563 | } |
564 | break; |
565 | |
566 | default: |
567 | orc_warn("unknown SP base reg %d at %pB\n" , |
568 | orc->sp_reg, (void *)state->ip); |
569 | goto err; |
570 | } |
571 | |
572 | if (indirect) { |
573 | if (!deref_stack_reg(state, addr: sp, val: &sp)) |
574 | goto err; |
575 | |
576 | if (orc->sp_reg == ORC_REG_SP_INDIRECT) |
577 | sp += orc->sp_offset; |
578 | } |
579 | |
580 | /* Find IP, SP and possibly regs: */ |
581 | switch (orc->type) { |
582 | case ORC_TYPE_CALL: |
583 | ip_p = sp - sizeof(long); |
584 | |
585 | if (!deref_stack_reg(state, addr: ip_p, val: &state->ip)) |
586 | goto err; |
587 | |
588 | state->ip = unwind_recover_ret_addr(state, addr: state->ip, |
589 | addr_p: (unsigned long *)ip_p); |
590 | state->sp = sp; |
591 | state->regs = NULL; |
592 | state->prev_regs = NULL; |
593 | break; |
594 | |
595 | case ORC_TYPE_REGS: |
596 | if (!deref_stack_regs(state, addr: sp, ip: &state->ip, sp: &state->sp)) { |
597 | orc_warn_current("can't access registers at %pB\n" , |
598 | (void *)orig_ip); |
599 | goto err; |
600 | } |
601 | /* |
602 | * There is a small chance to interrupt at the entry of |
603 | * arch_rethook_trampoline() where the ORC info doesn't exist. |
604 | * That point is right after the RET to arch_rethook_trampoline() |
605 | * which was modified return address. |
606 | * At that point, the @addr_p of the unwind_recover_rethook() |
607 | * (this has to point the address of the stack entry storing |
608 | * the modified return address) must be "SP - (a stack entry)" |
609 | * because SP is incremented by the RET. |
610 | */ |
611 | state->ip = unwind_recover_rethook(state, addr: state->ip, |
612 | addr_p: (unsigned long *)(state->sp - sizeof(long))); |
613 | state->regs = (struct pt_regs *)sp; |
614 | state->prev_regs = NULL; |
615 | state->full_regs = true; |
616 | break; |
617 | |
618 | case ORC_TYPE_REGS_PARTIAL: |
619 | if (!deref_stack_iret_regs(state, addr: sp, ip: &state->ip, sp: &state->sp)) { |
620 | orc_warn_current("can't access iret registers at %pB\n" , |
621 | (void *)orig_ip); |
622 | goto err; |
623 | } |
624 | /* See ORC_TYPE_REGS case comment. */ |
625 | state->ip = unwind_recover_rethook(state, addr: state->ip, |
626 | addr_p: (unsigned long *)(state->sp - sizeof(long))); |
627 | |
628 | if (state->full_regs) |
629 | state->prev_regs = state->regs; |
630 | state->regs = (void *)sp - IRET_FRAME_OFFSET; |
631 | state->full_regs = false; |
632 | break; |
633 | |
634 | default: |
635 | orc_warn("unknown .orc_unwind entry type %d at %pB\n" , |
636 | orc->type, (void *)orig_ip); |
637 | goto err; |
638 | } |
639 | |
640 | /* Find BP: */ |
641 | switch (orc->bp_reg) { |
642 | case ORC_REG_UNDEFINED: |
643 | if (get_reg(state, offsetof(struct pt_regs, bp), val: &tmp)) |
644 | state->bp = tmp; |
645 | break; |
646 | |
647 | case ORC_REG_PREV_SP: |
648 | if (!deref_stack_reg(state, addr: sp + orc->bp_offset, val: &state->bp)) |
649 | goto err; |
650 | break; |
651 | |
652 | case ORC_REG_BP: |
653 | if (!deref_stack_reg(state, addr: state->bp + orc->bp_offset, val: &state->bp)) |
654 | goto err; |
655 | break; |
656 | |
657 | default: |
658 | orc_warn("unknown BP base reg %d for ip %pB\n" , |
659 | orc->bp_reg, (void *)orig_ip); |
660 | goto err; |
661 | } |
662 | |
663 | /* Prevent a recursive loop due to bad ORC data: */ |
664 | if (state->stack_info.type == prev_type && |
665 | on_stack(info: &state->stack_info, addr: (void *)state->sp, len: sizeof(long)) && |
666 | state->sp <= prev_sp) { |
667 | orc_warn_current("stack going in the wrong direction? at %pB\n" , |
668 | (void *)orig_ip); |
669 | goto err; |
670 | } |
671 | |
672 | preempt_enable(); |
673 | return true; |
674 | |
675 | err: |
676 | state->error = true; |
677 | |
678 | the_end: |
679 | preempt_enable(); |
680 | state->stack_info.type = STACK_TYPE_UNKNOWN; |
681 | return false; |
682 | } |
683 | EXPORT_SYMBOL_GPL(unwind_next_frame); |
684 | |
685 | void __unwind_start(struct unwind_state *state, struct task_struct *task, |
686 | struct pt_regs *regs, unsigned long *first_frame) |
687 | { |
688 | memset(state, 0, sizeof(*state)); |
689 | state->task = task; |
690 | |
691 | if (!orc_init) |
692 | goto err; |
693 | |
694 | /* |
695 | * Refuse to unwind the stack of a task while it's executing on another |
696 | * CPU. This check is racy, but that's ok: the unwinder has other |
697 | * checks to prevent it from going off the rails. |
698 | */ |
699 | if (task_on_another_cpu(task)) |
700 | goto err; |
701 | |
702 | if (regs) { |
703 | if (user_mode(regs)) |
704 | goto the_end; |
705 | |
706 | state->ip = regs->ip; |
707 | state->sp = regs->sp; |
708 | state->bp = regs->bp; |
709 | state->regs = regs; |
710 | state->full_regs = true; |
711 | state->signal = true; |
712 | |
713 | } else if (task == current) { |
714 | asm volatile("lea (%%rip), %0\n\t" |
715 | "mov %%rsp, %1\n\t" |
716 | "mov %%rbp, %2\n\t" |
717 | : "=r" (state->ip), "=r" (state->sp), |
718 | "=r" (state->bp)); |
719 | |
720 | } else { |
721 | struct inactive_task_frame *frame = (void *)task->thread.sp; |
722 | |
723 | state->sp = task->thread.sp + sizeof(*frame); |
724 | state->bp = READ_ONCE_NOCHECK(frame->bp); |
725 | state->ip = READ_ONCE_NOCHECK(frame->ret_addr); |
726 | state->signal = (void *)state->ip == ret_from_fork; |
727 | } |
728 | |
729 | if (get_stack_info(stack: (unsigned long *)state->sp, task: state->task, |
730 | info: &state->stack_info, visit_mask: &state->stack_mask)) { |
731 | /* |
732 | * We weren't on a valid stack. It's possible that |
733 | * we overflowed a valid stack into a guard page. |
734 | * See if the next page up is valid so that we can |
735 | * generate some kind of backtrace if this happens. |
736 | */ |
737 | void *next_page = (void *)PAGE_ALIGN((unsigned long)state->sp); |
738 | state->error = true; |
739 | if (get_stack_info(stack: next_page, task: state->task, info: &state->stack_info, |
740 | visit_mask: &state->stack_mask)) |
741 | return; |
742 | } |
743 | |
744 | /* |
745 | * The caller can provide the address of the first frame directly |
746 | * (first_frame) or indirectly (regs->sp) to indicate which stack frame |
747 | * to start unwinding at. Skip ahead until we reach it. |
748 | */ |
749 | |
750 | /* When starting from regs, skip the regs frame: */ |
751 | if (regs) { |
752 | unwind_next_frame(state); |
753 | return; |
754 | } |
755 | |
756 | /* Otherwise, skip ahead to the user-specified starting frame: */ |
757 | while (!unwind_done(state) && |
758 | (!on_stack(info: &state->stack_info, addr: first_frame, len: sizeof(long)) || |
759 | state->sp <= (unsigned long)first_frame)) |
760 | unwind_next_frame(state); |
761 | |
762 | return; |
763 | |
764 | err: |
765 | state->error = true; |
766 | the_end: |
767 | state->stack_info.type = STACK_TYPE_UNKNOWN; |
768 | } |
769 | EXPORT_SYMBOL_GPL(__unwind_start); |
770 | |