1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4	*/
5
6	#include <string.h>
7	#include <stdlib.h>
8	#include <inttypes.h>
9	#include <sys/mman.h>
10
11	#include <objtool/builtin.h>
12	#include <objtool/cfi.h>
13	#include <objtool/arch.h>
14	#include <objtool/check.h>
15	#include <objtool/special.h>
16	#include <objtool/warn.h>
17	#include <objtool/endianness.h>
18
19	#include <linux/objtool_types.h>
20	#include <linux/hashtable.h>
21	#include <linux/kernel.h>
22	#include <linux/static_call_types.h>
23
24	struct alternative {
25	struct alternative *next;
26	struct instruction *insn;
27	bool skip_orig;
28	};
29
30	static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
31
32	static struct cfi_init_state initial_func_cfi;
33	static struct cfi_state init_cfi;
34	static struct cfi_state func_cfi;
35	static struct cfi_state force_undefined_cfi;
36
37	struct instruction *find_insn(struct objtool_file *file,
38	struct section *sec, unsigned long offset)
39	{
40	struct instruction *insn;
41
42	hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
43	if (insn->sec == sec && insn->offset == offset)
44	return insn;
45	}
46
47	return NULL;
48	}
49
50	struct instruction *next_insn_same_sec(struct objtool_file *file,
51	struct instruction *insn)
52	{
53	if (insn->idx == INSN_CHUNK_MAX)
54	return find_insn(file, sec: insn->sec, offset: insn->offset + insn->len);
55
56	insn++;
57	if (!insn->len)
58	return NULL;
59
60	return insn;
61	}
62
63	static struct instruction *next_insn_same_func(struct objtool_file *file,
64	struct instruction *insn)
65	{
66	struct instruction *next = next_insn_same_sec(file, insn);
67	struct symbol *func = insn_func(insn);
68
69	if (!func)
70	return NULL;
71
72	if (next && insn_func(next) == func)
73	return next;
74
75	/* Check if we're already in the subfunction: */
76	if (func == func->cfunc)
77	return NULL;
78
79	/* Move to the subfunction: */
80	return find_insn(file, sec: func->cfunc->sec, offset: func->cfunc->offset);
81	}
82
83	static struct instruction *prev_insn_same_sec(struct objtool_file *file,
84	struct instruction *insn)
85	{
86	if (insn->idx == 0) {
87	if (insn->prev_len)
88	return find_insn(file, sec: insn->sec, offset: insn->offset - insn->prev_len);
89	return NULL;
90	}
91
92	return insn - 1;
93	}
94
95	static struct instruction *prev_insn_same_sym(struct objtool_file *file,
96	struct instruction *insn)
97	{
98	struct instruction *prev = prev_insn_same_sec(file, insn);
99
100	if (prev && insn_func(prev) == insn_func(insn))
101	return prev;
102
103	return NULL;
104	}
105
106	#define for_each_insn(file, insn) \
107	for (struct section *__sec, *__fake = (struct section *)1; \
108	__fake; __fake = NULL) \
109	for_each_sec(file, __sec) \
110	sec_for_each_insn(file, __sec, insn)
111
112	#define func_for_each_insn(file, func, insn) \
113	for (insn = find_insn(file, func->sec, func->offset); \
114	insn; \
115	insn = next_insn_same_func(file, insn))
116
117	#define sym_for_each_insn(file, sym, insn) \
118	for (insn = find_insn(file, sym->sec, sym->offset); \
119	insn && insn->offset < sym->offset + sym->len; \
120	insn = next_insn_same_sec(file, insn))
121
122	#define sym_for_each_insn_continue_reverse(file, sym, insn) \
123	for (insn = prev_insn_same_sec(file, insn); \
124	insn && insn->offset >= sym->offset; \
125	insn = prev_insn_same_sec(file, insn))
126
127	#define sec_for_each_insn_from(file, insn) \
128	for (; insn; insn = next_insn_same_sec(file, insn))
129
130	#define sec_for_each_insn_continue(file, insn) \
131	for (insn = next_insn_same_sec(file, insn); insn; \
132	insn = next_insn_same_sec(file, insn))
133
134	static inline struct symbol *insn_call_dest(struct instruction *insn)
135	{
136	if (insn->type == INSN_JUMP_DYNAMIC ||
137	insn->type == INSN_CALL_DYNAMIC)
138	return NULL;
139
140	return insn->_call_dest;
141	}
142
143	static inline struct reloc *insn_jump_table(struct instruction *insn)
144	{
145	if (insn->type == INSN_JUMP_DYNAMIC ||
146	insn->type == INSN_CALL_DYNAMIC)
147	return insn->_jump_table;
148
149	return NULL;
150	}
151
152	static bool is_jump_table_jump(struct instruction *insn)
153	{
154	struct alt_group *alt_group = insn->alt_group;
155
156	if (insn_jump_table(insn))
157	return true;
158
159	/* Retpoline alternative for a jump table? */
160	return alt_group && alt_group->orig_group &&
161	insn_jump_table(insn: alt_group->orig_group->first_insn);
162	}
163
164	static bool is_sibling_call(struct instruction *insn)
165	{
166	/*
167	* Assume only STT_FUNC calls have jump-tables.
168	*/
169	if (insn_func(insn)) {
170	/* An indirect jump is either a sibling call or a jump to a table. */
171	if (insn->type == INSN_JUMP_DYNAMIC)
172	return !is_jump_table_jump(insn);
173	}
174
175	/* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
176	return (is_static_jump(insn) && insn_call_dest(insn));
177	}
178
179	/*
180	* This checks to see if the given function is a "noreturn" function.
181	*
182	* For global functions which are outside the scope of this object file, we
183	* have to keep a manual list of them.
184	*
185	* For local functions, we have to detect them manually by simply looking for
186	* the lack of a return instruction.
187	*/
188	static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
189	int recursion)
190	{
191	int i;
192	struct instruction *insn;
193	bool empty = true;
194
195	#define NORETURN(func) __stringify(func),
196	static const char * const global_noreturns[] = {
197	#include "noreturns.h"
198	};
199	#undef NORETURN
200
201	if (!func)
202	return false;
203
204	if (func->bind == STB_GLOBAL || func->bind == STB_WEAK)
205	for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
206	if (!strcmp(func->name, global_noreturns[i]))
207	return true;
208
209	if (func->bind == STB_WEAK)
210	return false;
211
212	if (!func->len)
213	return false;
214
215	insn = find_insn(file, sec: func->sec, offset: func->offset);
216	if (!insn || !insn_func(insn))
217	return false;
218
219	func_for_each_insn(file, func, insn) {
220	empty = false;
221
222	if (insn->type == INSN_RETURN)
223	return false;
224	}
225
226	if (empty)
227	return false;
228
229	/*
230	* A function can have a sibling call instead of a return. In that
231	* case, the function's dead-end status depends on whether the target
232	* of the sibling call returns.
233	*/
234	func_for_each_insn(file, func, insn) {
235	if (is_sibling_call(insn)) {
236	struct instruction *dest = insn->jump_dest;
237
238	if (!dest)
239	/* sibling call to another file */
240	return false;
241
242	/* local sibling call */
243	if (recursion == 5) {
244	/*
245	* Infinite recursion: two functions have
246	* sibling calls to each other. This is a very
247	* rare case. It means they aren't dead ends.
248	*/
249	return false;
250	}
251
252	return __dead_end_function(file, func: insn_func(dest), recursion: recursion+1);
253	}
254	}
255
256	return true;
257	}
258
259	static bool dead_end_function(struct objtool_file *file, struct symbol *func)
260	{
261	return __dead_end_function(file, func, recursion: 0);
262	}
263
264	static void init_cfi_state(struct cfi_state *cfi)
265	{
266	int i;
267
268	for (i = 0; i < CFI_NUM_REGS; i++) {
269	cfi->regs[i].base = CFI_UNDEFINED;
270	cfi->vals[i].base = CFI_UNDEFINED;
271	}
272	cfi->cfa.base = CFI_UNDEFINED;
273	cfi->drap_reg = CFI_UNDEFINED;
274	cfi->drap_offset = -1;
275	}
276
277	static void init_insn_state(struct objtool_file *file, struct insn_state *state,
278	struct section *sec)
279	{
280	memset(state, 0, sizeof(*state));
281	init_cfi_state(cfi: &state->cfi);
282
283	/*
284	* We need the full vmlinux for noinstr validation, otherwise we can
285	* not correctly determine insn_call_dest(insn)->sec (external symbols
286	* do not have a section).
287	*/
288	if (opts.link && opts.noinstr && sec)
289	state->noinstr = sec->noinstr;
290	}
291
292	static struct cfi_state *cfi_alloc(void)
293	{
294	struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
295	if (!cfi) {
296	WARN("calloc failed");
297	exit(1);
298	}
299	nr_cfi++;
300	return cfi;
301	}
302
303	static int cfi_bits;
304	static struct hlist_head *cfi_hash;
305
306	static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
307	{
308	return memcmp((void *)cfi1 + sizeof(cfi1->hash),
309	(void *)cfi2 + sizeof(cfi2->hash),
310	sizeof(struct cfi_state) - sizeof(struct hlist_node));
311	}
312
313	static inline u32 cfi_key(struct cfi_state *cfi)
314	{
315	return jhash((void *)cfi + sizeof(cfi->hash),
316	sizeof(*cfi) - sizeof(cfi->hash), 0);
317	}
318
319	static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
320	{
321	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
322	struct cfi_state *obj;
323
324	hlist_for_each_entry(obj, head, hash) {
325	if (!cficmp(cfi1: cfi, cfi2: obj)) {
326	nr_cfi_cache++;
327	return obj;
328	}
329	}
330
331	obj = cfi_alloc();
332	*obj = *cfi;
333	hlist_add_head(n: &obj->hash, h: head);
334
335	return obj;
336	}
337
338	static void cfi_hash_add(struct cfi_state *cfi)
339	{
340	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
341
342	hlist_add_head(n: &cfi->hash, h: head);
343	}
344
345	static void *cfi_hash_alloc(unsigned long size)
346	{
347	cfi_bits = max(10, ilog2(size));
348	cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
349	PROT_READ|PROT_WRITE,
350	MAP_PRIVATE|MAP_ANON, -1, 0);
351	if (cfi_hash == (void *)-1L) {
352	WARN("mmap fail cfi_hash");
353	cfi_hash = NULL;
354	} else if (opts.stats) {
355	printf("cfi_bits: %d\n", cfi_bits);
356	}
357
358	return cfi_hash;
359	}
360
361	static unsigned long nr_insns;
362	static unsigned long nr_insns_visited;
363
364	/*
365	* Call the arch-specific instruction decoder for all the instructions and add
366	* them to the global instruction list.
367	*/
368	static int decode_instructions(struct objtool_file *file)
369	{
370	struct section *sec;
371	struct symbol *func;
372	unsigned long offset;
373	struct instruction *insn;
374	int ret;
375
376	for_each_sec(file, sec) {
377	struct instruction *insns = NULL;
378	u8 prev_len = 0;
379	u8 idx = 0;
380
381	if (!(sec->sh.sh_flags & SHF_EXECINSTR))
382	continue;
383
384	if (strcmp(sec->name, ".altinstr_replacement") &&
385	strcmp(sec->name, ".altinstr_aux") &&
386	strncmp(sec->name, ".discard.", 9))
387	sec->text = true;
388
389	if (!strcmp(sec->name, ".noinstr.text") ||
390	!strcmp(sec->name, ".entry.text") ||
391	!strcmp(sec->name, ".cpuidle.text") ||
392	!strncmp(sec->name, ".text..__x86.", 13))
393	sec->noinstr = true;
394
395	/*
396	* .init.text code is ran before userspace and thus doesn't
397	* strictly need retpolines, except for modules which are
398	* loaded late, they very much do need retpoline in their
399	* .init.text
400	*/
401	if (!strcmp(sec->name, ".init.text") && !opts.module)
402	sec->init = true;
403
404	for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
405	if (!insns || idx == INSN_CHUNK_MAX) {
406	insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
407	if (!insns) {
408	WARN("malloc failed");
409	return -1;
410	}
411	idx = 0;
412	} else {
413	idx++;
414	}
415	insn = &insns[idx];
416	insn->idx = idx;
417
418	INIT_LIST_HEAD(list: &insn->call_node);
419	insn->sec = sec;
420	insn->offset = offset;
421	insn->prev_len = prev_len;
422
423	ret = arch_decode_instruction(file, sec, offset,
424	sec->sh.sh_size - offset,
425	insn);
426	if (ret)
427	return ret;
428
429	prev_len = insn->len;
430
431	/*
432	* By default, "ud2" is a dead end unless otherwise
433	* annotated, because GCC 7 inserts it for certain
434	* divide-by-zero cases.
435	*/
436	if (insn->type == INSN_BUG)
437	insn->dead_end = true;
438
439	hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
440	nr_insns++;
441	}
442
443	// printf("%s: last chunk used: %d\n", sec->name, (int)idx);
444
445	sec_for_each_sym(sec, func) {
446	if (func->type != STT_NOTYPE && func->type != STT_FUNC)
447	continue;
448
449	if (func->offset == sec->sh.sh_size) {
450	/* Heuristic: likely an "end" symbol */
451	if (func->type == STT_NOTYPE)
452	continue;
453	WARN("%s(): STT_FUNC at end of section",
454	func->name);
455	return -1;
456	}
457
458	if (func->embedded_insn || func->alias != func)
459	continue;
460
461	if (!find_insn(file, sec, offset: func->offset)) {
462	WARN("%s(): can't find starting instruction",
463	func->name);
464	return -1;
465	}
466
467	sym_for_each_insn(file, func, insn) {
468	insn->sym = func;
469	if (func->type == STT_FUNC &&
470	insn->type == INSN_ENDBR &&
471	list_empty(head: &insn->call_node)) {
472	if (insn->offset == func->offset) {
473	list_add_tail(new: &insn->call_node, head: &file->endbr_list);
474	file->nr_endbr++;
475	} else {
476	file->nr_endbr_int++;
477	}
478	}
479	}
480	}
481	}
482
483	if (opts.stats)
484	printf("nr_insns: %lu\n", nr_insns);
485
486	return 0;
487	}
488
489	/*
490	* Read the pv_ops[] .data table to find the static initialized values.
491	*/
492	static int add_pv_ops(struct objtool_file *file, const char *symname)
493	{
494	struct symbol *sym, *func;
495	unsigned long off, end;
496	struct reloc *reloc;
497	int idx;
498
499	sym = find_symbol_by_name(file->elf, symname);
500	if (!sym)
501	return 0;
502
503	off = sym->offset;
504	end = off + sym->len;
505	for (;;) {
506	reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
507	if (!reloc)
508	break;
509
510	func = reloc->sym;
511	if (func->type == STT_SECTION)
512	func = find_symbol_by_offset(reloc->sym->sec,
513	reloc_addend(reloc));
514
515	idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
516
517	objtool_pv_add(file, idx, func);
518
519	off = reloc_offset(reloc) + 1;
520	if (off > end)
521	break;
522	}
523
524	return 0;
525	}
526
527	/*
528	* Allocate and initialize file->pv_ops[].
529	*/
530	static int init_pv_ops(struct objtool_file *file)
531	{
532	static const char *pv_ops_tables[] = {
533	"pv_ops",
534	"xen_cpu_ops",
535	"xen_irq_ops",
536	"xen_mmu_ops",
537	NULL,
538	};
539	const char *pv_ops;
540	struct symbol *sym;
541	int idx, nr;
542
543	if (!opts.noinstr)
544	return 0;
545
546	file->pv_ops = NULL;
547
548	sym = find_symbol_by_name(file->elf, "pv_ops");
549	if (!sym)
550	return 0;
551
552	nr = sym->len / sizeof(unsigned long);
553	file->pv_ops = calloc(sizeof(struct pv_state), nr);
554	if (!file->pv_ops)
555	return -1;
556
557	for (idx = 0; idx < nr; idx++)
558	INIT_LIST_HEAD(list: &file->pv_ops[idx].targets);
559
560	for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
561	add_pv_ops(file, symname: pv_ops);
562
563	return 0;
564	}
565
566	static struct instruction *find_last_insn(struct objtool_file *file,
567	struct section *sec)
568	{
569	struct instruction *insn = NULL;
570	unsigned int offset;
571	unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
572
573	for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
574	insn = find_insn(file, sec, offset);
575
576	return insn;
577	}
578
579	/*
580	* Mark "ud2" instructions and manually annotated dead ends.
581	*/
582	static int add_dead_ends(struct objtool_file *file)
583	{
584	struct section *rsec;
585	struct reloc *reloc;
586	struct instruction *insn;
587	s64 addend;
588
589	/*
590	* Check for manually annotated dead ends.
591	*/
592	rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
593	if (!rsec)
594	goto reachable;
595
596	for_each_reloc(rsec, reloc) {
597
598	if (reloc->sym->type != STT_SECTION) {
599	WARN("unexpected relocation symbol type in %s", rsec->name);
600	return -1;
601	}
602
603	addend = reloc_addend(reloc);
604
605	insn = find_insn(file, sec: reloc->sym->sec, offset: addend);
606	if (insn)
607	insn = prev_insn_same_sec(file, insn);
608	else if (addend == reloc->sym->sec->sh.sh_size) {
609	insn = find_last_insn(file, sec: reloc->sym->sec);
610	if (!insn) {
611	WARN("can't find unreachable insn at %s+0x%" PRIx64,
612	reloc->sym->sec->name, addend);
613	return -1;
614	}
615	} else {
616	WARN("can't find unreachable insn at %s+0x%" PRIx64,
617	reloc->sym->sec->name, addend);
618	return -1;
619	}
620
621	insn->dead_end = true;
622	}
623
624	reachable:
625	/*
626	* These manually annotated reachable checks are needed for GCC 4.4,
627	* where the Linux unreachable() macro isn't supported. In that case
628	* GCC doesn't know the "ud2" is fatal, so it generates code as if it's
629	* not a dead end.
630	*/
631	rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
632	if (!rsec)
633	return 0;
634
635	for_each_reloc(rsec, reloc) {
636
637	if (reloc->sym->type != STT_SECTION) {
638	WARN("unexpected relocation symbol type in %s", rsec->name);
639	return -1;
640	}
641
642	addend = reloc_addend(reloc);
643
644	insn = find_insn(file, sec: reloc->sym->sec, offset: addend);
645	if (insn)
646	insn = prev_insn_same_sec(file, insn);
647	else if (addend == reloc->sym->sec->sh.sh_size) {
648	insn = find_last_insn(file, sec: reloc->sym->sec);
649	if (!insn) {
650	WARN("can't find reachable insn at %s+0x%" PRIx64,
651	reloc->sym->sec->name, addend);
652	return -1;
653	}
654	} else {
655	WARN("can't find reachable insn at %s+0x%" PRIx64,
656	reloc->sym->sec->name, addend);
657	return -1;
658	}
659
660	insn->dead_end = false;
661	}
662
663	return 0;
664	}
665
666	static int create_static_call_sections(struct objtool_file *file)
667	{
668	struct static_call_site *site;
669	struct section *sec;
670	struct instruction *insn;
671	struct symbol *key_sym;
672	char *key_name, *tmp;
673	int idx;
674
675	sec = find_section_by_name(file->elf, ".static_call_sites");
676	if (sec) {
677	INIT_LIST_HEAD(list: &file->static_call_list);
678	WARN("file already has .static_call_sites section, skipping");
679	return 0;
680	}
681
682	if (list_empty(head: &file->static_call_list))
683	return 0;
684
685	idx = 0;
686	list_for_each_entry(insn, &file->static_call_list, call_node)
687	idx++;
688
689	sec = elf_create_section_pair(file->elf, ".static_call_sites",
690	sizeof(*site), idx, idx * 2);
691	if (!sec)
692	return -1;
693
694	/* Allow modules to modify the low bits of static_call_site::key */
695	sec->sh.sh_flags |= SHF_WRITE;
696
697	idx = 0;
698	list_for_each_entry(insn, &file->static_call_list, call_node) {
699
700	/* populate reloc for 'addr' */
701	if (!elf_init_reloc_text_sym(file->elf, sec,
702	idx * sizeof(*site), idx * 2,
703	insn->sec, insn->offset))
704	return -1;
705
706	/* find key symbol */
707	key_name = strdup(insn_call_dest(insn)->name);
708	if (!key_name) {
709	perror("strdup");
710	return -1;
711	}
712	if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
713	STATIC_CALL_TRAMP_PREFIX_LEN)) {
714	WARN("static_call: trampoline name malformed: %s", key_name);
715	free(key_name);
716	return -1;
717	}
718	tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
719	memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
720
721	key_sym = find_symbol_by_name(file->elf, tmp);
722	if (!key_sym) {
723	if (!opts.module) {
724	WARN("static_call: can't find static_call_key symbol: %s", tmp);
725	free(key_name);
726	return -1;
727	}
728
729	/*
730	* For modules(), the key might not be exported, which
731	* means the module can make static calls but isn't
732	* allowed to change them.
733	*
734	* In that case we temporarily set the key to be the
735	* trampoline address. This is fixed up in
736	* static_call_add_module().
737	*/
738	key_sym = insn_call_dest(insn);
739	}
740	free(key_name);
741
742	/* populate reloc for 'key' */
743	if (!elf_init_reloc_data_sym(file->elf, sec,
744	idx * sizeof(*site) + 4,
745	(idx * 2) + 1, key_sym,
746	is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
747	return -1;
748
749	idx++;
750	}
751
752	return 0;
753	}
754
755	static int create_retpoline_sites_sections(struct objtool_file *file)
756	{
757	struct instruction *insn;
758	struct section *sec;
759	int idx;
760
761	sec = find_section_by_name(file->elf, ".retpoline_sites");
762	if (sec) {
763	WARN("file already has .retpoline_sites, skipping");
764	return 0;
765	}
766
767	idx = 0;
768	list_for_each_entry(insn, &file->retpoline_call_list, call_node)
769	idx++;
770
771	if (!idx)
772	return 0;
773
774	sec = elf_create_section_pair(file->elf, ".retpoline_sites",
775	sizeof(int), idx, idx);
776	if (!sec)
777	return -1;
778
779	idx = 0;
780	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
781
782	if (!elf_init_reloc_text_sym(file->elf, sec,
783	idx * sizeof(int), idx,
784	insn->sec, insn->offset))
785	return -1;
786
787	idx++;
788	}
789
790	return 0;
791	}
792
793	static int create_return_sites_sections(struct objtool_file *file)
794	{
795	struct instruction *insn;
796	struct section *sec;
797	int idx;
798
799	sec = find_section_by_name(file->elf, ".return_sites");
800	if (sec) {
801	WARN("file already has .return_sites, skipping");
802	return 0;
803	}
804
805	idx = 0;
806	list_for_each_entry(insn, &file->return_thunk_list, call_node)
807	idx++;
808
809	if (!idx)
810	return 0;
811
812	sec = elf_create_section_pair(file->elf, ".return_sites",
813	sizeof(int), idx, idx);
814	if (!sec)
815	return -1;
816
817	idx = 0;
818	list_for_each_entry(insn, &file->return_thunk_list, call_node) {
819
820	if (!elf_init_reloc_text_sym(file->elf, sec,
821	idx * sizeof(int), idx,
822	insn->sec, insn->offset))
823	return -1;
824
825	idx++;
826	}
827
828	return 0;
829	}
830
831	static int create_ibt_endbr_seal_sections(struct objtool_file *file)
832	{
833	struct instruction *insn;
834	struct section *sec;
835	int idx;
836
837	sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
838	if (sec) {
839	WARN("file already has .ibt_endbr_seal, skipping");
840	return 0;
841	}
842
843	idx = 0;
844	list_for_each_entry(insn, &file->endbr_list, call_node)
845	idx++;
846
847	if (opts.stats) {
848	printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
849	printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
850	printf("ibt: superfluous ENDBR: %d\n", idx);
851	}
852
853	if (!idx)
854	return 0;
855
856	sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
857	sizeof(int), idx, idx);
858	if (!sec)
859	return -1;
860
861	idx = 0;
862	list_for_each_entry(insn, &file->endbr_list, call_node) {
863
864	int *site = (int *)sec->data->d_buf + idx;
865	struct symbol *sym = insn->sym;
866	*site = 0;
867
868	if (opts.module && sym && sym->type == STT_FUNC &&
869	insn->offset == sym->offset &&
870	(!strcmp(sym->name, "init_module") ||
871	!strcmp(sym->name, "cleanup_module")))
872	WARN("%s(): not an indirect call target", sym->name);
873
874	if (!elf_init_reloc_text_sym(file->elf, sec,
875	idx * sizeof(int), idx,
876	insn->sec, insn->offset))
877	return -1;
878
879	idx++;
880	}
881
882	return 0;
883	}
884
885	static int create_cfi_sections(struct objtool_file *file)
886	{
887	struct section *sec;
888	struct symbol *sym;
889	int idx;
890
891	sec = find_section_by_name(file->elf, ".cfi_sites");
892	if (sec) {
893	INIT_LIST_HEAD(list: &file->call_list);
894	WARN("file already has .cfi_sites section, skipping");
895	return 0;
896	}
897
898	idx = 0;
899	for_each_sym(file, sym) {
900	if (sym->type != STT_FUNC)
901	continue;
902
903	if (strncmp(sym->name, "__cfi_", 6))
904	continue;
905
906	idx++;
907	}
908
909	sec = elf_create_section_pair(file->elf, ".cfi_sites",
910	sizeof(unsigned int), idx, idx);
911	if (!sec)
912	return -1;
913
914	idx = 0;
915	for_each_sym(file, sym) {
916	if (sym->type != STT_FUNC)
917	continue;
918
919	if (strncmp(sym->name, "__cfi_", 6))
920	continue;
921
922	if (!elf_init_reloc_text_sym(file->elf, sec,
923	idx * sizeof(unsigned int), idx,
924	sym->sec, sym->offset))
925	return -1;
926
927	idx++;
928	}
929
930	return 0;
931	}
932
933	static int create_mcount_loc_sections(struct objtool_file *file)
934	{
935	size_t addr_size = elf_addr_size(file->elf);
936	struct instruction *insn;
937	struct section *sec;
938	int idx;
939
940	sec = find_section_by_name(file->elf, "__mcount_loc");
941	if (sec) {
942	INIT_LIST_HEAD(list: &file->mcount_loc_list);
943	WARN("file already has __mcount_loc section, skipping");
944	return 0;
945	}
946
947	if (list_empty(head: &file->mcount_loc_list))
948	return 0;
949
950	idx = 0;
951	list_for_each_entry(insn, &file->mcount_loc_list, call_node)
952	idx++;
953
954	sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
955	idx, idx);
956	if (!sec)
957	return -1;
958
959	sec->sh.sh_addralign = addr_size;
960
961	idx = 0;
962	list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
963
964	struct reloc *reloc;
965
966	reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
967	insn->sec, insn->offset);
968	if (!reloc)
969	return -1;
970
971	set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
972
973	idx++;
974	}
975
976	return 0;
977	}
978
979	static int create_direct_call_sections(struct objtool_file *file)
980	{
981	struct instruction *insn;
982	struct section *sec;
983	int idx;
984
985	sec = find_section_by_name(file->elf, ".call_sites");
986	if (sec) {
987	INIT_LIST_HEAD(list: &file->call_list);
988	WARN("file already has .call_sites section, skipping");
989	return 0;
990	}
991
992	if (list_empty(head: &file->call_list))
993	return 0;
994
995	idx = 0;
996	list_for_each_entry(insn, &file->call_list, call_node)
997	idx++;
998
999	sec = elf_create_section_pair(file->elf, ".call_sites",
1000	sizeof(unsigned int), idx, idx);
1001	if (!sec)
1002	return -1;
1003
1004	idx = 0;
1005	list_for_each_entry(insn, &file->call_list, call_node) {
1006
1007	if (!elf_init_reloc_text_sym(file->elf, sec,
1008	idx * sizeof(unsigned int), idx,
1009	insn->sec, insn->offset))
1010	return -1;
1011
1012	idx++;
1013	}
1014
1015	return 0;
1016	}
1017
1018	/*
1019	* Warnings shouldn't be reported for ignored functions.
1020	*/
1021	static void add_ignores(struct objtool_file *file)
1022	{
1023	struct instruction *insn;
1024	struct section *rsec;
1025	struct symbol *func;
1026	struct reloc *reloc;
1027
1028	rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1029	if (!rsec)
1030	return;
1031
1032	for_each_reloc(rsec, reloc) {
1033	switch (reloc->sym->type) {
1034	case STT_FUNC:
1035	func = reloc->sym;
1036	break;
1037
1038	case STT_SECTION:
1039	func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1040	if (!func)
1041	continue;
1042	break;
1043
1044	default:
1045	WARN("unexpected relocation symbol type in %s: %d",
1046	rsec->name, reloc->sym->type);
1047	continue;
1048	}
1049
1050	func_for_each_insn(file, func, insn)
1051	insn->ignore = true;
1052	}
1053	}
1054
1055	/*
1056	* This is a whitelist of functions that is allowed to be called with AC set.
1057	* The list is meant to be minimal and only contains compiler instrumentation
1058	* ABI and a few functions used to implement *_{to,from}_user() functions.
1059	*
1060	* These functions must not directly change AC, but may PUSHF/POPF.
1061	*/
1062	static const char *uaccess_safe_builtin[] = {
1063	/* KASAN */
1064	"kasan_report",
1065	"kasan_check_range",
1066	/* KASAN out-of-line */
1067	"__asan_loadN_noabort",
1068	"__asan_load1_noabort",
1069	"__asan_load2_noabort",
1070	"__asan_load4_noabort",
1071	"__asan_load8_noabort",
1072	"__asan_load16_noabort",
1073	"__asan_storeN_noabort",
1074	"__asan_store1_noabort",
1075	"__asan_store2_noabort",
1076	"__asan_store4_noabort",
1077	"__asan_store8_noabort",
1078	"__asan_store16_noabort",
1079	"__kasan_check_read",
1080	"__kasan_check_write",
1081	/* KASAN in-line */
1082	"__asan_report_load_n_noabort",
1083	"__asan_report_load1_noabort",
1084	"__asan_report_load2_noabort",
1085	"__asan_report_load4_noabort",
1086	"__asan_report_load8_noabort",
1087	"__asan_report_load16_noabort",
1088	"__asan_report_store_n_noabort",
1089	"__asan_report_store1_noabort",
1090	"__asan_report_store2_noabort",
1091	"__asan_report_store4_noabort",
1092	"__asan_report_store8_noabort",
1093	"__asan_report_store16_noabort",
1094	/* KCSAN */
1095	"__kcsan_check_access",
1096	"__kcsan_mb",
1097	"__kcsan_wmb",
1098	"__kcsan_rmb",
1099	"__kcsan_release",
1100	"kcsan_found_watchpoint",
1101	"kcsan_setup_watchpoint",
1102	"kcsan_check_scoped_accesses",
1103	"kcsan_disable_current",
1104	"kcsan_enable_current_nowarn",
1105	/* KCSAN/TSAN */
1106	"__tsan_func_entry",
1107	"__tsan_func_exit",
1108	"__tsan_read_range",
1109	"__tsan_write_range",
1110	"__tsan_read1",
1111	"__tsan_read2",
1112	"__tsan_read4",
1113	"__tsan_read8",
1114	"__tsan_read16",
1115	"__tsan_write1",
1116	"__tsan_write2",
1117	"__tsan_write4",
1118	"__tsan_write8",
1119	"__tsan_write16",
1120	"__tsan_read_write1",
1121	"__tsan_read_write2",
1122	"__tsan_read_write4",
1123	"__tsan_read_write8",
1124	"__tsan_read_write16",
1125	"__tsan_volatile_read1",
1126	"__tsan_volatile_read2",
1127	"__tsan_volatile_read4",
1128	"__tsan_volatile_read8",
1129	"__tsan_volatile_read16",
1130	"__tsan_volatile_write1",
1131	"__tsan_volatile_write2",
1132	"__tsan_volatile_write4",
1133	"__tsan_volatile_write8",
1134	"__tsan_volatile_write16",
1135	"__tsan_atomic8_load",
1136	"__tsan_atomic16_load",
1137	"__tsan_atomic32_load",
1138	"__tsan_atomic64_load",
1139	"__tsan_atomic8_store",
1140	"__tsan_atomic16_store",
1141	"__tsan_atomic32_store",
1142	"__tsan_atomic64_store",
1143	"__tsan_atomic8_exchange",
1144	"__tsan_atomic16_exchange",
1145	"__tsan_atomic32_exchange",
1146	"__tsan_atomic64_exchange",
1147	"__tsan_atomic8_fetch_add",
1148	"__tsan_atomic16_fetch_add",
1149	"__tsan_atomic32_fetch_add",
1150	"__tsan_atomic64_fetch_add",
1151	"__tsan_atomic8_fetch_sub",
1152	"__tsan_atomic16_fetch_sub",
1153	"__tsan_atomic32_fetch_sub",
1154	"__tsan_atomic64_fetch_sub",
1155	"__tsan_atomic8_fetch_and",
1156	"__tsan_atomic16_fetch_and",
1157	"__tsan_atomic32_fetch_and",
1158	"__tsan_atomic64_fetch_and",
1159	"__tsan_atomic8_fetch_or",
1160	"__tsan_atomic16_fetch_or",
1161	"__tsan_atomic32_fetch_or",
1162	"__tsan_atomic64_fetch_or",
1163	"__tsan_atomic8_fetch_xor",
1164	"__tsan_atomic16_fetch_xor",
1165	"__tsan_atomic32_fetch_xor",
1166	"__tsan_atomic64_fetch_xor",
1167	"__tsan_atomic8_fetch_nand",
1168	"__tsan_atomic16_fetch_nand",
1169	"__tsan_atomic32_fetch_nand",
1170	"__tsan_atomic64_fetch_nand",
1171	"__tsan_atomic8_compare_exchange_strong",
1172	"__tsan_atomic16_compare_exchange_strong",
1173	"__tsan_atomic32_compare_exchange_strong",
1174	"__tsan_atomic64_compare_exchange_strong",
1175	"__tsan_atomic8_compare_exchange_weak",
1176	"__tsan_atomic16_compare_exchange_weak",
1177	"__tsan_atomic32_compare_exchange_weak",
1178	"__tsan_atomic64_compare_exchange_weak",
1179	"__tsan_atomic8_compare_exchange_val",
1180	"__tsan_atomic16_compare_exchange_val",
1181	"__tsan_atomic32_compare_exchange_val",
1182	"__tsan_atomic64_compare_exchange_val",
1183	"__tsan_atomic_thread_fence",
1184	"__tsan_atomic_signal_fence",
1185	"__tsan_unaligned_read16",
1186	"__tsan_unaligned_write16",
1187	/* KCOV */
1188	"write_comp_data",
1189	"check_kcov_mode",
1190	"__sanitizer_cov_trace_pc",
1191	"__sanitizer_cov_trace_const_cmp1",
1192	"__sanitizer_cov_trace_const_cmp2",
1193	"__sanitizer_cov_trace_const_cmp4",
1194	"__sanitizer_cov_trace_const_cmp8",
1195	"__sanitizer_cov_trace_cmp1",
1196	"__sanitizer_cov_trace_cmp2",
1197	"__sanitizer_cov_trace_cmp4",
1198	"__sanitizer_cov_trace_cmp8",
1199	"__sanitizer_cov_trace_switch",
1200	/* KMSAN */
1201	"kmsan_copy_to_user",
1202	"kmsan_report",
1203	"kmsan_unpoison_entry_regs",
1204	"kmsan_unpoison_memory",
1205	"__msan_chain_origin",
1206	"__msan_get_context_state",
1207	"__msan_instrument_asm_store",
1208	"__msan_metadata_ptr_for_load_1",
1209	"__msan_metadata_ptr_for_load_2",
1210	"__msan_metadata_ptr_for_load_4",
1211	"__msan_metadata_ptr_for_load_8",
1212	"__msan_metadata_ptr_for_load_n",
1213	"__msan_metadata_ptr_for_store_1",
1214	"__msan_metadata_ptr_for_store_2",
1215	"__msan_metadata_ptr_for_store_4",
1216	"__msan_metadata_ptr_for_store_8",
1217	"__msan_metadata_ptr_for_store_n",
1218	"__msan_poison_alloca",
1219	"__msan_warning",
1220	/* UBSAN */
1221	"ubsan_type_mismatch_common",
1222	"__ubsan_handle_type_mismatch",
1223	"__ubsan_handle_type_mismatch_v1",
1224	"__ubsan_handle_shift_out_of_bounds",
1225	"__ubsan_handle_load_invalid_value",
1226	/* STACKLEAK */
1227	"stackleak_track_stack",
1228	/* misc */
1229	"csum_partial_copy_generic",
1230	"copy_mc_fragile",
1231	"copy_mc_fragile_handle_tail",
1232	"copy_mc_enhanced_fast_string",
1233	"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1234	"rep_stos_alternative",
1235	"rep_movs_alternative",
1236	"__copy_user_nocache",
1237	NULL
1238	};
1239
1240	static void add_uaccess_safe(struct objtool_file *file)
1241	{
1242	struct symbol *func;
1243	const char **name;
1244
1245	if (!opts.uaccess)
1246	return;
1247
1248	for (name = uaccess_safe_builtin; *name; name++) {
1249	func = find_symbol_by_name(file->elf, *name);
1250	if (!func)
1251	continue;
1252
1253	func->uaccess_safe = true;
1254	}
1255	}
1256
1257	/*
1258	* FIXME: For now, just ignore any alternatives which add retpolines. This is
1259	* a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1260	* But it at least allows objtool to understand the control flow *around* the
1261	* retpoline.
1262	*/
1263	static int add_ignore_alternatives(struct objtool_file *file)
1264	{
1265	struct section *rsec;
1266	struct reloc *reloc;
1267	struct instruction *insn;
1268
1269	rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1270	if (!rsec)
1271	return 0;
1272
1273	for_each_reloc(rsec, reloc) {
1274	if (reloc->sym->type != STT_SECTION) {
1275	WARN("unexpected relocation symbol type in %s", rsec->name);
1276	return -1;
1277	}
1278
1279	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
1280	if (!insn) {
1281	WARN("bad .discard.ignore_alts entry");
1282	return -1;
1283	}
1284
1285	insn->ignore_alts = true;
1286	}
1287
1288	return 0;
1289	}
1290
1291	/*
1292	* Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1293	* will be added to the .retpoline_sites section.
1294	*/
1295	__weak bool arch_is_retpoline(struct symbol *sym)
1296	{
1297	return false;
1298	}
1299
1300	/*
1301	* Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1302	* will be added to the .return_sites section.
1303	*/
1304	__weak bool arch_is_rethunk(struct symbol *sym)
1305	{
1306	return false;
1307	}
1308
1309	/*
1310	* Symbols that are embedded inside other instructions, because sometimes crazy
1311	* code exists. These are mostly ignored for validation purposes.
1312	*/
1313	__weak bool arch_is_embedded_insn(struct symbol *sym)
1314	{
1315	return false;
1316	}
1317
1318	static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1319	{
1320	struct reloc *reloc;
1321
1322	if (insn->no_reloc)
1323	return NULL;
1324
1325	if (!file)
1326	return NULL;
1327
1328	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1329	insn->offset, insn->len);
1330	if (!reloc) {
1331	insn->no_reloc = 1;
1332	return NULL;
1333	}
1334
1335	return reloc;
1336	}
1337
1338	static void remove_insn_ops(struct instruction *insn)
1339	{
1340	struct stack_op *op, *next;
1341
1342	for (op = insn->stack_ops; op; op = next) {
1343	next = op->next;
1344	free(op);
1345	}
1346	insn->stack_ops = NULL;
1347	}
1348
1349	static void annotate_call_site(struct objtool_file *file,
1350	struct instruction *insn, bool sibling)
1351	{
1352	struct reloc *reloc = insn_reloc(file, insn);
1353	struct symbol *sym = insn_call_dest(insn);
1354
1355	if (!sym)
1356	sym = reloc->sym;
1357
1358	/*
1359	* Alternative replacement code is just template code which is
1360	* sometimes copied to the original instruction. For now, don't
1361	* annotate it. (In the future we might consider annotating the
1362	* original instruction if/when it ever makes sense to do so.)
1363	*/
1364	if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1365	return;
1366
1367	if (sym->static_call_tramp) {
1368	list_add_tail(new: &insn->call_node, head: &file->static_call_list);
1369	return;
1370	}
1371
1372	if (sym->retpoline_thunk) {
1373	list_add_tail(new: &insn->call_node, head: &file->retpoline_call_list);
1374	return;
1375	}
1376
1377	/*
1378	* Many compilers cannot disable KCOV or sanitizer calls with a function
1379	* attribute so they need a little help, NOP out any such calls from
1380	* noinstr text.
1381	*/
1382	if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1383	if (reloc)
1384	set_reloc_type(file->elf, reloc, R_NONE);
1385
1386	elf_write_insn(file->elf, insn->sec,
1387	insn->offset, insn->len,
1388	sibling ? arch_ret_insn(insn->len)
1389	: arch_nop_insn(insn->len));
1390
1391	insn->type = sibling ? INSN_RETURN : INSN_NOP;
1392
1393	if (sibling) {
1394	/*
1395	* We've replaced the tail-call JMP insn by two new
1396	* insn: RET; INT3, except we only have a single struct
1397	* insn here. Mark it retpoline_safe to avoid the SLS
1398	* warning, instead of adding another insn.
1399	*/
1400	insn->retpoline_safe = true;
1401	}
1402
1403	return;
1404	}
1405
1406	if (opts.mcount && sym->fentry) {
1407	if (sibling)
1408	WARN_INSN(insn, "tail call to __fentry__ !?!?");
1409	if (opts.mnop) {
1410	if (reloc)
1411	set_reloc_type(file->elf, reloc, R_NONE);
1412
1413	elf_write_insn(file->elf, insn->sec,
1414	insn->offset, insn->len,
1415	arch_nop_insn(insn->len));
1416
1417	insn->type = INSN_NOP;
1418	}
1419
1420	list_add_tail(new: &insn->call_node, head: &file->mcount_loc_list);
1421	return;
1422	}
1423
1424	if (insn->type == INSN_CALL && !insn->sec->init)
1425	list_add_tail(new: &insn->call_node, head: &file->call_list);
1426
1427	if (!sibling && dead_end_function(file, func: sym))
1428	insn->dead_end = true;
1429	}
1430
1431	static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1432	struct symbol *dest, bool sibling)
1433	{
1434	insn->_call_dest = dest;
1435	if (!dest)
1436	return;
1437
1438	/*
1439	* Whatever stack impact regular CALLs have, should be undone
1440	* by the RETURN of the called function.
1441	*
1442	* Annotated intra-function calls retain the stack_ops but
1443	* are converted to JUMP, see read_intra_function_calls().
1444	*/
1445	remove_insn_ops(insn);
1446
1447	annotate_call_site(file, insn, sibling);
1448	}
1449
1450	static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1451	{
1452	/*
1453	* Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1454	* so convert them accordingly.
1455	*/
1456	switch (insn->type) {
1457	case INSN_CALL:
1458	insn->type = INSN_CALL_DYNAMIC;
1459	break;
1460	case INSN_JUMP_UNCONDITIONAL:
1461	insn->type = INSN_JUMP_DYNAMIC;
1462	break;
1463	case INSN_JUMP_CONDITIONAL:
1464	insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1465	break;
1466	default:
1467	return;
1468	}
1469
1470	insn->retpoline_safe = true;
1471
1472	/*
1473	* Whatever stack impact regular CALLs have, should be undone
1474	* by the RETURN of the called function.
1475	*
1476	* Annotated intra-function calls retain the stack_ops but
1477	* are converted to JUMP, see read_intra_function_calls().
1478	*/
1479	remove_insn_ops(insn);
1480
1481	annotate_call_site(file, insn, sibling: false);
1482	}
1483
1484	static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1485	{
1486	/*
1487	* Return thunk tail calls are really just returns in disguise,
1488	* so convert them accordingly.
1489	*/
1490	insn->type = INSN_RETURN;
1491	insn->retpoline_safe = true;
1492
1493	if (add)
1494	list_add_tail(new: &insn->call_node, head: &file->return_thunk_list);
1495	}
1496
1497	static bool is_first_func_insn(struct objtool_file *file,
1498	struct instruction *insn, struct symbol *sym)
1499	{
1500	if (insn->offset == sym->offset)
1501	return true;
1502
1503	/* Allow direct CALL/JMP past ENDBR */
1504	if (opts.ibt) {
1505	struct instruction *prev = prev_insn_same_sym(file, insn);
1506
1507	if (prev && prev->type == INSN_ENDBR &&
1508	insn->offset == sym->offset + prev->len)
1509	return true;
1510	}
1511
1512	return false;
1513	}
1514
1515	/*
1516	* A sibling call is a tail-call to another symbol -- to differentiate from a
1517	* recursive tail-call which is to the same symbol.
1518	*/
1519	static bool jump_is_sibling_call(struct objtool_file *file,
1520	struct instruction *from, struct instruction *to)
1521	{
1522	struct symbol *fs = from->sym;
1523	struct symbol *ts = to->sym;
1524
1525	/* Not a sibling call if from/to a symbol hole */
1526	if (!fs || !ts)
1527	return false;
1528
1529	/* Not a sibling call if not targeting the start of a symbol. */
1530	if (!is_first_func_insn(file, insn: to, sym: ts))
1531	return false;
1532
1533	/* Disallow sibling calls into STT_NOTYPE */
1534	if (ts->type == STT_NOTYPE)
1535	return false;
1536
1537	/* Must not be self to be a sibling */
1538	return fs->pfunc != ts->pfunc;
1539	}
1540
1541	/*
1542	* Find the destination instructions for all jumps.
1543	*/
1544	static int add_jump_destinations(struct objtool_file *file)
1545	{
1546	struct instruction *insn, *jump_dest;
1547	struct reloc *reloc;
1548	struct section *dest_sec;
1549	unsigned long dest_off;
1550
1551	for_each_insn(file, insn) {
1552	if (insn->jump_dest) {
1553	/*
1554	* handle_group_alt() may have previously set
1555	* 'jump_dest' for some alternatives.
1556	*/
1557	continue;
1558	}
1559	if (!is_static_jump(insn))
1560	continue;
1561
1562	reloc = insn_reloc(file, insn);
1563	if (!reloc) {
1564	dest_sec = insn->sec;
1565	dest_off = arch_jump_destination(insn);
1566	} else if (reloc->sym->type == STT_SECTION) {
1567	dest_sec = reloc->sym->sec;
1568	dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1569	} else if (reloc->sym->retpoline_thunk) {
1570	add_retpoline_call(file, insn);
1571	continue;
1572	} else if (reloc->sym->return_thunk) {
1573	add_return_call(file, insn, true);
1574	continue;
1575	} else if (insn_func(insn)) {
1576	/*
1577	* External sibling call or internal sibling call with
1578	* STT_FUNC reloc.
1579	*/
1580	add_call_dest(file, insn, reloc->sym, true);
1581	continue;
1582	} else if (reloc->sym->sec->idx) {
1583	dest_sec = reloc->sym->sec;
1584	dest_off = reloc->sym->sym.st_value +
1585	arch_dest_reloc_offset(reloc_addend(reloc));
1586	} else {
1587	/* non-func asm code jumping to another file */
1588	continue;
1589	}
1590
1591	jump_dest = find_insn(file, dest_sec, dest_off);
1592	if (!jump_dest) {
1593	struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1594
1595	/*
1596	* This is a special case for retbleed_untrain_ret().
1597	* It jumps to __x86_return_thunk(), but objtool
1598	* can't find the thunk's starting RET
1599	* instruction, because the RET is also in the
1600	* middle of another instruction. Objtool only
1601	* knows about the outer instruction.
1602	*/
1603	if (sym && sym->embedded_insn) {
1604	add_return_call(file, insn, false);
1605	continue;
1606	}
1607
1608	WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1609	dest_sec->name, dest_off);
1610	return -1;
1611	}
1612
1613	/*
1614	* An intra-TU jump in retpoline.o might not have a relocation
1615	* for its jump dest, in which case the above
1616	* add_{retpoline,return}_call() didn't happen.
1617	*/
1618	if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1619	if (jump_dest->sym->retpoline_thunk) {
1620	add_retpoline_call(file, insn);
1621	continue;
1622	}
1623	if (jump_dest->sym->return_thunk) {
1624	add_return_call(file, insn, true);
1625	continue;
1626	}
1627	}
1628
1629	/*
1630	* Cross-function jump.
1631	*/
1632	if (insn_func(insn) && insn_func(jump_dest) &&
1633	insn_func(insn) != insn_func(jump_dest)) {
1634
1635	/*
1636	* For GCC 8+, create parent/child links for any cold
1637	* subfunctions. This is _mostly_ redundant with a
1638	* similar initialization in read_symbols().
1639	*
1640	* If a function has aliases, we want the *first* such
1641	* function in the symbol table to be the subfunction's
1642	* parent. In that case we overwrite the
1643	* initialization done in read_symbols().
1644	*
1645	* However this code can't completely replace the
1646	* read_symbols() code because this doesn't detect the
1647	* case where the parent function's only reference to a
1648	* subfunction is through a jump table.
1649	*/
1650	if (!strstr(insn_func(insn)->name, ".cold") &&
1651	strstr(insn_func(jump_dest)->name, ".cold")) {
1652	insn_func(insn)->cfunc = insn_func(jump_dest);
1653	insn_func(jump_dest)->pfunc = insn_func(insn);
1654	}
1655	}
1656
1657	if (jump_is_sibling_call(file, insn, jump_dest)) {
1658	/*
1659	* Internal sibling call without reloc or with
1660	* STT_SECTION reloc.
1661	*/
1662	add_call_dest(file, insn, insn_func(jump_dest), true);
1663	continue;
1664	}
1665
1666	insn->jump_dest = jump_dest;
1667	}
1668
1669	return 0;
1670	}
1671
1672	static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1673	{
1674	struct symbol *call_dest;
1675
1676	call_dest = find_func_by_offset(sec, offset);
1677	if (!call_dest)
1678	call_dest = find_symbol_by_offset(sec, offset);
1679
1680	return call_dest;
1681	}
1682
1683	/*
1684	* Find the destination instructions for all calls.
1685	*/
1686	static int add_call_destinations(struct objtool_file *file)
1687	{
1688	struct instruction *insn;
1689	unsigned long dest_off;
1690	struct symbol *dest;
1691	struct reloc *reloc;
1692
1693	for_each_insn(file, insn) {
1694	if (insn->type != INSN_CALL)
1695	continue;
1696
1697	reloc = insn_reloc(file, insn);
1698	if (!reloc) {
1699	dest_off = arch_jump_destination(insn);
1700	dest = find_call_destination(insn->sec, dest_off);
1701
1702	add_call_dest(file, insn, dest, false);
1703
1704	if (insn->ignore)
1705	continue;
1706
1707	if (!insn_call_dest(insn)) {
1708	WARN_INSN(insn, "unannotated intra-function call");
1709	return -1;
1710	}
1711
1712	if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1713	WARN_INSN(insn, "unsupported call to non-function");
1714	return -1;
1715	}
1716
1717	} else if (reloc->sym->type == STT_SECTION) {
1718	dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1719	dest = find_call_destination(reloc->sym->sec, dest_off);
1720	if (!dest) {
1721	WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1722	reloc->sym->sec->name, dest_off);
1723	return -1;
1724	}
1725
1726	add_call_dest(file, insn, dest, false);
1727
1728	} else if (reloc->sym->retpoline_thunk) {
1729	add_retpoline_call(file, insn);
1730
1731	} else
1732	add_call_dest(file, insn, reloc->sym, false);
1733	}
1734
1735	return 0;
1736	}
1737
1738	/*
1739	* The .alternatives section requires some extra special care over and above
1740	* other special sections because alternatives are patched in place.
1741	*/
1742	static int handle_group_alt(struct objtool_file *file,
1743	struct special_alt *special_alt,
1744	struct instruction *orig_insn,
1745	struct instruction **new_insn)
1746	{
1747	struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1748	struct alt_group *orig_alt_group, *new_alt_group;
1749	unsigned long dest_off;
1750
1751	orig_alt_group = orig_insn->alt_group;
1752	if (!orig_alt_group) {
1753	struct instruction *last_orig_insn = NULL;
1754
1755	orig_alt_group = malloc(sizeof(*orig_alt_group));
1756	if (!orig_alt_group) {
1757	WARN("malloc failed");
1758	return -1;
1759	}
1760	orig_alt_group->cfi = calloc(special_alt->orig_len,
1761	sizeof(struct cfi_state *));
1762	if (!orig_alt_group->cfi) {
1763	WARN("calloc failed");
1764	return -1;
1765	}
1766
1767	insn = orig_insn;
1768	sec_for_each_insn_from(file, insn) {
1769	if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1770	break;
1771
1772	insn->alt_group = orig_alt_group;
1773	last_orig_insn = insn;
1774	}
1775	orig_alt_group->orig_group = NULL;
1776	orig_alt_group->first_insn = orig_insn;
1777	orig_alt_group->last_insn = last_orig_insn;
1778	orig_alt_group->nop = NULL;
1779	} else {
1780	if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1781	orig_alt_group->first_insn->offset != special_alt->orig_len) {
1782	WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1783	orig_alt_group->last_insn->offset +
1784	orig_alt_group->last_insn->len -
1785	orig_alt_group->first_insn->offset,
1786	special_alt->orig_len);
1787	return -1;
1788	}
1789	}
1790
1791	new_alt_group = malloc(sizeof(*new_alt_group));
1792	if (!new_alt_group) {
1793	WARN("malloc failed");
1794	return -1;
1795	}
1796
1797	if (special_alt->new_len < special_alt->orig_len) {
1798	/*
1799	* Insert a fake nop at the end to make the replacement
1800	* alt_group the same size as the original. This is needed to
1801	* allow propagate_alt_cfi() to do its magic. When the last
1802	* instruction affects the stack, the instruction after it (the
1803	* nop) will propagate the new state to the shared CFI array.
1804	*/
1805	nop = malloc(sizeof(*nop));
1806	if (!nop) {
1807	WARN("malloc failed");
1808	return -1;
1809	}
1810	memset(nop, 0, sizeof(*nop));
1811
1812	nop->sec = special_alt->new_sec;
1813	nop->offset = special_alt->new_off + special_alt->new_len;
1814	nop->len = special_alt->orig_len - special_alt->new_len;
1815	nop->type = INSN_NOP;
1816	nop->sym = orig_insn->sym;
1817	nop->alt_group = new_alt_group;
1818	nop->ignore = orig_insn->ignore_alts;
1819	}
1820
1821	if (!special_alt->new_len) {
1822	*new_insn = nop;
1823	goto end;
1824	}
1825
1826	insn = *new_insn;
1827	sec_for_each_insn_from(file, insn) {
1828	struct reloc *alt_reloc;
1829
1830	if (insn->offset >= special_alt->new_off + special_alt->new_len)
1831	break;
1832
1833	last_new_insn = insn;
1834
1835	insn->ignore = orig_insn->ignore_alts;
1836	insn->sym = orig_insn->sym;
1837	insn->alt_group = new_alt_group;
1838
1839	/*
1840	* Since alternative replacement code is copy/pasted by the
1841	* kernel after applying relocations, generally such code can't
1842	* have relative-address relocation references to outside the
1843	* .altinstr_replacement section, unless the arch's
1844	* alternatives code can adjust the relative offsets
1845	* accordingly.
1846	*/
1847	alt_reloc = insn_reloc(file, insn);
1848	if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1849	!arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1850
1851	WARN_INSN(insn, "unsupported relocation in alternatives section");
1852	return -1;
1853	}
1854
1855	if (!is_static_jump(insn))
1856	continue;
1857
1858	if (!insn->immediate)
1859	continue;
1860
1861	dest_off = arch_jump_destination(insn);
1862	if (dest_off == special_alt->new_off + special_alt->new_len) {
1863	insn->jump_dest = next_insn_same_sec(file, insn: orig_alt_group->last_insn);
1864	if (!insn->jump_dest) {
1865	WARN_INSN(insn, "can't find alternative jump destination");
1866	return -1;
1867	}
1868	}
1869	}
1870
1871	if (!last_new_insn) {
1872	WARN_FUNC("can't find last new alternative instruction",
1873	special_alt->new_sec, special_alt->new_off);
1874	return -1;
1875	}
1876
1877	end:
1878	new_alt_group->orig_group = orig_alt_group;
1879	new_alt_group->first_insn = *new_insn;
1880	new_alt_group->last_insn = last_new_insn;
1881	new_alt_group->nop = nop;
1882	new_alt_group->cfi = orig_alt_group->cfi;
1883	return 0;
1884	}
1885
1886	/*
1887	* A jump table entry can either convert a nop to a jump or a jump to a nop.
1888	* If the original instruction is a jump, make the alt entry an effective nop
1889	* by just skipping the original instruction.
1890	*/
1891	static int handle_jump_alt(struct objtool_file *file,
1892	struct special_alt *special_alt,
1893	struct instruction *orig_insn,
1894	struct instruction **new_insn)
1895	{
1896	if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1897	orig_insn->type != INSN_NOP) {
1898
1899	WARN_INSN(orig_insn, "unsupported instruction at jump label");
1900	return -1;
1901	}
1902
1903	if (opts.hack_jump_label && special_alt->key_addend & 2) {
1904	struct reloc *reloc = insn_reloc(file, insn: orig_insn);
1905
1906	if (reloc)
1907	set_reloc_type(file->elf, reloc, R_NONE);
1908	elf_write_insn(file->elf, orig_insn->sec,
1909	orig_insn->offset, orig_insn->len,
1910	arch_nop_insn(orig_insn->len));
1911	orig_insn->type = INSN_NOP;
1912	}
1913
1914	if (orig_insn->type == INSN_NOP) {
1915	if (orig_insn->len == 2)
1916	file->jl_nop_short++;
1917	else
1918	file->jl_nop_long++;
1919
1920	return 0;
1921	}
1922
1923	if (orig_insn->len == 2)
1924	file->jl_short++;
1925	else
1926	file->jl_long++;
1927
1928	*new_insn = next_insn_same_sec(file, insn: orig_insn);
1929	return 0;
1930	}
1931
1932	/*
1933	* Read all the special sections which have alternate instructions which can be
1934	* patched in or redirected to at runtime. Each instruction having alternate
1935	* instruction(s) has them added to its insn->alts list, which will be
1936	* traversed in validate_branch().
1937	*/
1938	static int add_special_section_alts(struct objtool_file *file)
1939	{
1940	struct list_head special_alts;
1941	struct instruction *orig_insn, *new_insn;
1942	struct special_alt *special_alt, *tmp;
1943	struct alternative *alt;
1944	int ret;
1945
1946	ret = special_get_alts(file->elf, &special_alts);
1947	if (ret)
1948	return ret;
1949
1950	list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1951
1952	orig_insn = find_insn(file, sec: special_alt->orig_sec,
1953	offset: special_alt->orig_off);
1954	if (!orig_insn) {
1955	WARN_FUNC("special: can't find orig instruction",
1956	special_alt->orig_sec, special_alt->orig_off);
1957	ret = -1;
1958	goto out;
1959	}
1960
1961	new_insn = NULL;
1962	if (!special_alt->group || special_alt->new_len) {
1963	new_insn = find_insn(file, sec: special_alt->new_sec,
1964	offset: special_alt->new_off);
1965	if (!new_insn) {
1966	WARN_FUNC("special: can't find new instruction",
1967	special_alt->new_sec,
1968	special_alt->new_off);
1969	ret = -1;
1970	goto out;
1971	}
1972	}
1973
1974	if (special_alt->group) {
1975	if (!special_alt->orig_len) {
1976	WARN_INSN(orig_insn, "empty alternative entry");
1977	continue;
1978	}
1979
1980	ret = handle_group_alt(file, special_alt, orig_insn,
1981	new_insn: &new_insn);
1982	if (ret)
1983	goto out;
1984	} else if (special_alt->jump_or_nop) {
1985	ret = handle_jump_alt(file, special_alt, orig_insn,
1986	new_insn: &new_insn);
1987	if (ret)
1988	goto out;
1989	}
1990
1991	alt = malloc(sizeof(*alt));
1992	if (!alt) {
1993	WARN("malloc failed");
1994	ret = -1;
1995	goto out;
1996	}
1997
1998	alt->insn = new_insn;
1999	alt->skip_orig = special_alt->skip_orig;
2000	orig_insn->ignore_alts |= special_alt->skip_alt;
2001	alt->next = orig_insn->alts;
2002	orig_insn->alts = alt;
2003
2004	list_del(entry: &special_alt->list);
2005	free(special_alt);
2006	}
2007
2008	if (opts.stats) {
2009	printf("jl\\\tNOP\tJMP\n");
2010	printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2011	printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2012	}
2013
2014	out:
2015	return ret;
2016	}
2017
2018	static int add_jump_table(struct objtool_file *file, struct instruction *insn,
2019	struct reloc *next_table)
2020	{
2021	struct symbol *pfunc = insn_func(insn)->pfunc;
2022	struct reloc *table = insn_jump_table(insn);
2023	struct instruction *dest_insn;
2024	unsigned int prev_offset = 0;
2025	struct reloc *reloc = table;
2026	struct alternative *alt;
2027
2028	/*
2029	* Each @reloc is a switch table relocation which points to the target
2030	* instruction.
2031	*/
2032	for_each_reloc_from(table->sec, reloc) {
2033
2034	/* Check for the end of the table: */
2035	if (reloc != table && reloc == next_table)
2036	break;
2037
2038	/* Make sure the table entries are consecutive: */
2039	if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2040	break;
2041
2042	/* Detect function pointers from contiguous objects: */
2043	if (reloc->sym->sec == pfunc->sec &&
2044	reloc_addend(reloc) == pfunc->offset)
2045	break;
2046
2047	dest_insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2048	if (!dest_insn)
2049	break;
2050
2051	/* Make sure the destination is in the same function: */
2052	if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2053	break;
2054
2055	alt = malloc(sizeof(*alt));
2056	if (!alt) {
2057	WARN("malloc failed");
2058	return -1;
2059	}
2060
2061	alt->insn = dest_insn;
2062	alt->next = insn->alts;
2063	insn->alts = alt;
2064	prev_offset = reloc_offset(reloc);
2065	}
2066
2067	if (!prev_offset) {
2068	WARN_INSN(insn, "can't find switch jump table");
2069	return -1;
2070	}
2071
2072	return 0;
2073	}
2074
2075	/*
2076	* find_jump_table() - Given a dynamic jump, find the switch jump table
2077	* associated with it.
2078	*/
2079	static struct reloc *find_jump_table(struct objtool_file *file,
2080	struct symbol *func,
2081	struct instruction *insn)
2082	{
2083	struct reloc *table_reloc;
2084	struct instruction *dest_insn, *orig_insn = insn;
2085
2086	/*
2087	* Backward search using the @first_jump_src links, these help avoid
2088	* much of the 'in between' code. Which avoids us getting confused by
2089	* it.
2090	*/
2091	for (;
2092	insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2093	insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2094
2095	if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2096	break;
2097
2098	/* allow small jumps within the range */
2099	if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2100	insn->jump_dest &&
2101	(insn->jump_dest->offset <= insn->offset ||
2102	insn->jump_dest->offset > orig_insn->offset))
2103	break;
2104
2105	table_reloc = arch_find_switch_table(file, insn);
2106	if (!table_reloc)
2107	continue;
2108	dest_insn = find_insn(file, sec: table_reloc->sym->sec, offset: reloc_addend(table_reloc));
2109	if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2110	continue;
2111
2112	return table_reloc;
2113	}
2114
2115	return NULL;
2116	}
2117
2118	/*
2119	* First pass: Mark the head of each jump table so that in the next pass,
2120	* we know when a given jump table ends and the next one starts.
2121	*/
2122	static void mark_func_jump_tables(struct objtool_file *file,
2123	struct symbol *func)
2124	{
2125	struct instruction *insn, *last = NULL;
2126	struct reloc *reloc;
2127
2128	func_for_each_insn(file, func, insn) {
2129	if (!last)
2130	last = insn;
2131
2132	/*
2133	* Store back-pointers for unconditional forward jumps such
2134	* that find_jump_table() can back-track using those and
2135	* avoid some potentially confusing code.
2136	*/
2137	if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2138	insn->offset > last->offset &&
2139	insn->jump_dest->offset > insn->offset &&
2140	!insn->jump_dest->first_jump_src) {
2141
2142	insn->jump_dest->first_jump_src = insn;
2143	last = insn->jump_dest;
2144	}
2145
2146	if (insn->type != INSN_JUMP_DYNAMIC)
2147	continue;
2148
2149	reloc = find_jump_table(file, func, insn);
2150	if (reloc)
2151	insn->_jump_table = reloc;
2152	}
2153	}
2154
2155	static int add_func_jump_tables(struct objtool_file *file,
2156	struct symbol *func)
2157	{
2158	struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2159	int ret = 0;
2160
2161	func_for_each_insn(file, func, insn) {
2162	if (!insn_jump_table(insn))
2163	continue;
2164
2165	if (!insn_t1) {
2166	insn_t1 = insn;
2167	continue;
2168	}
2169
2170	insn_t2 = insn;
2171
2172	ret = add_jump_table(file, insn: insn_t1, next_table: insn_jump_table(insn: insn_t2));
2173	if (ret)
2174	return ret;
2175
2176	insn_t1 = insn_t2;
2177	}
2178
2179	if (insn_t1)
2180	ret = add_jump_table(file, insn: insn_t1, NULL);
2181
2182	return ret;
2183	}
2184
2185	/*
2186	* For some switch statements, gcc generates a jump table in the .rodata
2187	* section which contains a list of addresses within the function to jump to.
2188	* This finds these jump tables and adds them to the insn->alts lists.
2189	*/
2190	static int add_jump_table_alts(struct objtool_file *file)
2191	{
2192	struct symbol *func;
2193	int ret;
2194
2195	if (!file->rodata)
2196	return 0;
2197
2198	for_each_sym(file, func) {
2199	if (func->type != STT_FUNC)
2200	continue;
2201
2202	mark_func_jump_tables(file, func);
2203	ret = add_func_jump_tables(file, func);
2204	if (ret)
2205	return ret;
2206	}
2207
2208	return 0;
2209	}
2210
2211	static void set_func_state(struct cfi_state *state)
2212	{
2213	state->cfa = initial_func_cfi.cfa;
2214	memcpy(&state->regs, &initial_func_cfi.regs,
2215	CFI_NUM_REGS * sizeof(struct cfi_reg));
2216	state->stack_size = initial_func_cfi.cfa.offset;
2217	state->type = UNWIND_HINT_TYPE_CALL;
2218	}
2219
2220	static int read_unwind_hints(struct objtool_file *file)
2221	{
2222	struct cfi_state cfi = init_cfi;
2223	struct section *sec;
2224	struct unwind_hint *hint;
2225	struct instruction *insn;
2226	struct reloc *reloc;
2227	int i;
2228
2229	sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2230	if (!sec)
2231	return 0;
2232
2233	if (!sec->rsec) {
2234	WARN("missing .rela.discard.unwind_hints section");
2235	return -1;
2236	}
2237
2238	if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2239	WARN("struct unwind_hint size mismatch");
2240	return -1;
2241	}
2242
2243	file->hints = true;
2244
2245	for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2246	hint = (struct unwind_hint *)sec->data->d_buf + i;
2247
2248	reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2249	if (!reloc) {
2250	WARN("can't find reloc for unwind_hints[%d]", i);
2251	return -1;
2252	}
2253
2254	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2255	if (!insn) {
2256	WARN("can't find insn for unwind_hints[%d]", i);
2257	return -1;
2258	}
2259
2260	insn->hint = true;
2261
2262	if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2263	insn->cfi = &force_undefined_cfi;
2264	continue;
2265	}
2266
2267	if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2268	insn->hint = false;
2269	insn->save = true;
2270	continue;
2271	}
2272
2273	if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2274	insn->restore = true;
2275	continue;
2276	}
2277
2278	if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2279	struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2280
2281	if (sym && sym->bind == STB_GLOBAL) {
2282	if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2283	WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2284	}
2285	}
2286	}
2287
2288	if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2289	insn->cfi = &func_cfi;
2290	continue;
2291	}
2292
2293	if (insn->cfi)
2294	cfi = *(insn->cfi);
2295
2296	if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2297	WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2298	return -1;
2299	}
2300
2301	cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2302	cfi.type = hint->type;
2303	cfi.signal = hint->signal;
2304
2305	insn->cfi = cfi_hash_find_or_add(cfi: &cfi);
2306	}
2307
2308	return 0;
2309	}
2310
2311	static int read_noendbr_hints(struct objtool_file *file)
2312	{
2313	struct instruction *insn;
2314	struct section *rsec;
2315	struct reloc *reloc;
2316
2317	rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2318	if (!rsec)
2319	return 0;
2320
2321	for_each_reloc(rsec, reloc) {
2322	insn = find_insn(file, sec: reloc->sym->sec,
2323	offset: reloc->sym->offset + reloc_addend(reloc));
2324	if (!insn) {
2325	WARN("bad .discard.noendbr entry");
2326	return -1;
2327	}
2328
2329	insn->noendbr = 1;
2330	}
2331
2332	return 0;
2333	}
2334
2335	static int read_retpoline_hints(struct objtool_file *file)
2336	{
2337	struct section *rsec;
2338	struct instruction *insn;
2339	struct reloc *reloc;
2340
2341	rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2342	if (!rsec)
2343	return 0;
2344
2345	for_each_reloc(rsec, reloc) {
2346	if (reloc->sym->type != STT_SECTION) {
2347	WARN("unexpected relocation symbol type in %s", rsec->name);
2348	return -1;
2349	}
2350
2351	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2352	if (!insn) {
2353	WARN("bad .discard.retpoline_safe entry");
2354	return -1;
2355	}
2356
2357	if (insn->type != INSN_JUMP_DYNAMIC &&
2358	insn->type != INSN_CALL_DYNAMIC &&
2359	insn->type != INSN_RETURN &&
2360	insn->type != INSN_NOP) {
2361	WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2362	return -1;
2363	}
2364
2365	insn->retpoline_safe = true;
2366	}
2367
2368	return 0;
2369	}
2370
2371	static int read_instr_hints(struct objtool_file *file)
2372	{
2373	struct section *rsec;
2374	struct instruction *insn;
2375	struct reloc *reloc;
2376
2377	rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2378	if (!rsec)
2379	return 0;
2380
2381	for_each_reloc(rsec, reloc) {
2382	if (reloc->sym->type != STT_SECTION) {
2383	WARN("unexpected relocation symbol type in %s", rsec->name);
2384	return -1;
2385	}
2386
2387	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2388	if (!insn) {
2389	WARN("bad .discard.instr_end entry");
2390	return -1;
2391	}
2392
2393	insn->instr--;
2394	}
2395
2396	rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2397	if (!rsec)
2398	return 0;
2399
2400	for_each_reloc(rsec, reloc) {
2401	if (reloc->sym->type != STT_SECTION) {
2402	WARN("unexpected relocation symbol type in %s", rsec->name);
2403	return -1;
2404	}
2405
2406	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2407	if (!insn) {
2408	WARN("bad .discard.instr_begin entry");
2409	return -1;
2410	}
2411
2412	insn->instr++;
2413	}
2414
2415	return 0;
2416	}
2417
2418	static int read_validate_unret_hints(struct objtool_file *file)
2419	{
2420	struct section *rsec;
2421	struct instruction *insn;
2422	struct reloc *reloc;
2423
2424	rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2425	if (!rsec)
2426	return 0;
2427
2428	for_each_reloc(rsec, reloc) {
2429	if (reloc->sym->type != STT_SECTION) {
2430	WARN("unexpected relocation symbol type in %s", rsec->name);
2431	return -1;
2432	}
2433
2434	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2435	if (!insn) {
2436	WARN("bad .discard.instr_end entry");
2437	return -1;
2438	}
2439	insn->unret = 1;
2440	}
2441
2442	return 0;
2443	}
2444
2445
2446	static int read_intra_function_calls(struct objtool_file *file)
2447	{
2448	struct instruction *insn;
2449	struct section *rsec;
2450	struct reloc *reloc;
2451
2452	rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2453	if (!rsec)
2454	return 0;
2455
2456	for_each_reloc(rsec, reloc) {
2457	unsigned long dest_off;
2458
2459	if (reloc->sym->type != STT_SECTION) {
2460	WARN("unexpected relocation symbol type in %s",
2461	rsec->name);
2462	return -1;
2463	}
2464
2465	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2466	if (!insn) {
2467	WARN("bad .discard.intra_function_call entry");
2468	return -1;
2469	}
2470
2471	if (insn->type != INSN_CALL) {
2472	WARN_INSN(insn, "intra_function_call not a direct call");
2473	return -1;
2474	}
2475
2476	/*
2477	* Treat intra-function CALLs as JMPs, but with a stack_op.
2478	* See add_call_destinations(), which strips stack_ops from
2479	* normal CALLs.
2480	*/
2481	insn->type = INSN_JUMP_UNCONDITIONAL;
2482
2483	dest_off = arch_jump_destination(insn);
2484	insn->jump_dest = find_insn(file, sec: insn->sec, offset: dest_off);
2485	if (!insn->jump_dest) {
2486	WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2487	insn->sec->name, dest_off);
2488	return -1;
2489	}
2490	}
2491
2492	return 0;
2493	}
2494
2495	/*
2496	* Return true if name matches an instrumentation function, where calls to that
2497	* function from noinstr code can safely be removed, but compilers won't do so.
2498	*/
2499	static bool is_profiling_func(const char *name)
2500	{
2501	/*
2502	* Many compilers cannot disable KCOV with a function attribute.
2503	*/
2504	if (!strncmp(name, "__sanitizer_cov_", 16))
2505	return true;
2506
2507	/*
2508	* Some compilers currently do not remove __tsan_func_entry/exit nor
2509	* __tsan_atomic_signal_fence (used for barrier instrumentation) with
2510	* the __no_sanitize_thread attribute, remove them. Once the kernel's
2511	* minimum Clang version is 14.0, this can be removed.
2512	*/
2513	if (!strncmp(name, "__tsan_func_", 12) ||
2514	!strcmp(name, "__tsan_atomic_signal_fence"))
2515	return true;
2516
2517	return false;
2518	}
2519
2520	static int classify_symbols(struct objtool_file *file)
2521	{
2522	struct symbol *func;
2523
2524	for_each_sym(file, func) {
2525	if (func->bind != STB_GLOBAL)
2526	continue;
2527
2528	if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2529	strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2530	func->static_call_tramp = true;
2531
2532	if (arch_is_retpoline(sym: func))
2533	func->retpoline_thunk = true;
2534
2535	if (arch_is_rethunk(sym: func))
2536	func->return_thunk = true;
2537
2538	if (arch_is_embedded_insn(sym: func))
2539	func->embedded_insn = true;
2540
2541	if (arch_ftrace_match(func->name))
2542	func->fentry = true;
2543
2544	if (is_profiling_func(name: func->name))
2545	func->profiling_func = true;
2546	}
2547
2548	return 0;
2549	}
2550
2551	static void mark_rodata(struct objtool_file *file)
2552	{
2553	struct section *sec;
2554	bool found = false;
2555
2556	/*
2557	* Search for the following rodata sections, each of which can
2558	* potentially contain jump tables:
2559	*
2560	* - .rodata: can contain GCC switch tables
2561	* - .rodata.<func>: same, if -fdata-sections is being used
2562	* - .rodata..c_jump_table: contains C annotated jump tables
2563	*
2564	* .rodata.str1.* sections are ignored; they don't contain jump tables.
2565	*/
2566	for_each_sec(file, sec) {
2567	if (!strncmp(sec->name, ".rodata", 7) &&
2568	!strstr(sec->name, ".str1.")) {
2569	sec->rodata = true;
2570	found = true;
2571	}
2572	}
2573
2574	file->rodata = found;
2575	}
2576
2577	static int decode_sections(struct objtool_file *file)
2578	{
2579	int ret;
2580
2581	mark_rodata(file);
2582
2583	ret = init_pv_ops(file);
2584	if (ret)
2585	return ret;
2586
2587	/*
2588	* Must be before add_{jump_call}_destination.
2589	*/
2590	ret = classify_symbols(file);
2591	if (ret)
2592	return ret;
2593
2594	ret = decode_instructions(file);
2595	if (ret)
2596	return ret;
2597
2598	add_ignores(file);
2599	add_uaccess_safe(file);
2600
2601	ret = add_ignore_alternatives(file);
2602	if (ret)
2603	return ret;
2604
2605	/*
2606	* Must be before read_unwind_hints() since that needs insn->noendbr.
2607	*/
2608	ret = read_noendbr_hints(file);
2609	if (ret)
2610	return ret;
2611
2612	/*
2613	* Must be before add_jump_destinations(), which depends on 'func'
2614	* being set for alternatives, to enable proper sibling call detection.
2615	*/
2616	if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2617	ret = add_special_section_alts(file);
2618	if (ret)
2619	return ret;
2620	}
2621
2622	ret = add_jump_destinations(file);
2623	if (ret)
2624	return ret;
2625
2626	/*
2627	* Must be before add_call_destination(); it changes INSN_CALL to
2628	* INSN_JUMP.
2629	*/
2630	ret = read_intra_function_calls(file);
2631	if (ret)
2632	return ret;
2633
2634	ret = add_call_destinations(file);
2635	if (ret)
2636	return ret;
2637
2638	/*
2639	* Must be after add_call_destinations() such that it can override
2640	* dead_end_function() marks.
2641	*/
2642	ret = add_dead_ends(file);
2643	if (ret)
2644	return ret;
2645
2646	ret = add_jump_table_alts(file);
2647	if (ret)
2648	return ret;
2649
2650	ret = read_unwind_hints(file);
2651	if (ret)
2652	return ret;
2653
2654	ret = read_retpoline_hints(file);
2655	if (ret)
2656	return ret;
2657
2658	ret = read_instr_hints(file);
2659	if (ret)
2660	return ret;
2661
2662	ret = read_validate_unret_hints(file);
2663	if (ret)
2664	return ret;
2665
2666	return 0;
2667	}
2668
2669	static bool is_special_call(struct instruction *insn)
2670	{
2671	if (insn->type == INSN_CALL) {
2672	struct symbol *dest = insn_call_dest(insn);
2673
2674	if (!dest)
2675	return false;
2676
2677	if (dest->fentry || dest->embedded_insn)
2678	return true;
2679	}
2680
2681	return false;
2682	}
2683
2684	static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2685	{
2686	struct cfi_state *cfi = &state->cfi;
2687	int i;
2688
2689	if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2690	return true;
2691
2692	if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2693	return true;
2694
2695	if (cfi->stack_size != initial_func_cfi.cfa.offset)
2696	return true;
2697
2698	for (i = 0; i < CFI_NUM_REGS; i++) {
2699	if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2700	cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2701	return true;
2702	}
2703
2704	return false;
2705	}
2706
2707	static bool check_reg_frame_pos(const struct cfi_reg *reg,
2708	int expected_offset)
2709	{
2710	return reg->base == CFI_CFA &&
2711	reg->offset == expected_offset;
2712	}
2713
2714	static bool has_valid_stack_frame(struct insn_state *state)
2715	{
2716	struct cfi_state *cfi = &state->cfi;
2717
2718	if (cfi->cfa.base == CFI_BP &&
2719	check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2720	check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2721	return true;
2722
2723	if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2724	return true;
2725
2726	return false;
2727	}
2728
2729	static int update_cfi_state_regs(struct instruction *insn,
2730	struct cfi_state *cfi,
2731	struct stack_op *op)
2732	{
2733	struct cfi_reg *cfa = &cfi->cfa;
2734
2735	if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2736	return 0;
2737
2738	/* push */
2739	if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2740	cfa->offset += 8;
2741
2742	/* pop */
2743	if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2744	cfa->offset -= 8;
2745
2746	/* add immediate to sp */
2747	if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2748	op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2749	cfa->offset -= op->src.offset;
2750
2751	return 0;
2752	}
2753
2754	static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2755	{
2756	if (arch_callee_saved_reg(reg) &&
2757	cfi->regs[reg].base == CFI_UNDEFINED) {
2758	cfi->regs[reg].base = base;
2759	cfi->regs[reg].offset = offset;
2760	}
2761	}
2762
2763	static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2764	{
2765	cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2766	cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2767	}
2768
2769	/*
2770	* A note about DRAP stack alignment:
2771	*
2772	* GCC has the concept of a DRAP register, which is used to help keep track of
2773	* the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2774	* register. The typical DRAP pattern is:
2775	*
2776	* 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2777	* 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2778	* 41 ff 72 f8 pushq -0x8(%r10)
2779	* 55 push %rbp
2780	* 48 89 e5 mov %rsp,%rbp
2781	* (more pushes)
2782	* 41 52 push %r10
2783	* ...
2784	* 41 5a pop %r10
2785	* (more pops)
2786	* 5d pop %rbp
2787	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2788	* c3 retq
2789	*
2790	* There are some variations in the epilogues, like:
2791	*
2792	* 5b pop %rbx
2793	* 41 5a pop %r10
2794	* 41 5c pop %r12
2795	* 41 5d pop %r13
2796	* 41 5e pop %r14
2797	* c9 leaveq
2798	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2799	* c3 retq
2800	*
2801	* and:
2802	*
2803	* 4c 8b 55 e8 mov -0x18(%rbp),%r10
2804	* 48 8b 5d e0 mov -0x20(%rbp),%rbx
2805	* 4c 8b 65 f0 mov -0x10(%rbp),%r12
2806	* 4c 8b 6d f8 mov -0x8(%rbp),%r13
2807	* c9 leaveq
2808	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2809	* c3 retq
2810	*
2811	* Sometimes r13 is used as the DRAP register, in which case it's saved and
2812	* restored beforehand:
2813	*
2814	* 41 55 push %r13
2815	* 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2816	* 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2817	* ...
2818	* 49 8d 65 f0 lea -0x10(%r13),%rsp
2819	* 41 5d pop %r13
2820	* c3 retq
2821	*/
2822	static int update_cfi_state(struct instruction *insn,
2823	struct instruction *next_insn,
2824	struct cfi_state *cfi, struct stack_op *op)
2825	{
2826	struct cfi_reg *cfa = &cfi->cfa;
2827	struct cfi_reg *regs = cfi->regs;
2828
2829	/* ignore UNWIND_HINT_UNDEFINED regions */
2830	if (cfi->force_undefined)
2831	return 0;
2832
2833	/* stack operations don't make sense with an undefined CFA */
2834	if (cfa->base == CFI_UNDEFINED) {
2835	if (insn_func(insn)) {
2836	WARN_INSN(insn, "undefined stack state");
2837	return -1;
2838	}
2839	return 0;
2840	}
2841
2842	if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2843	cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2844	return update_cfi_state_regs(insn, cfi, op);
2845
2846	switch (op->dest.type) {
2847
2848	case OP_DEST_REG:
2849	switch (op->src.type) {
2850
2851	case OP_SRC_REG:
2852	if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2853	cfa->base == CFI_SP &&
2854	check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2855
2856	/* mov %rsp, %rbp */
2857	cfa->base = op->dest.reg;
2858	cfi->bp_scratch = false;
2859	}
2860
2861	else if (op->src.reg == CFI_SP &&
2862	op->dest.reg == CFI_BP && cfi->drap) {
2863
2864	/* drap: mov %rsp, %rbp */
2865	regs[CFI_BP].base = CFI_BP;
2866	regs[CFI_BP].offset = -cfi->stack_size;
2867	cfi->bp_scratch = false;
2868	}
2869
2870	else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2871
2872	/*
2873	* mov %rsp, %reg
2874	*
2875	* This is needed for the rare case where GCC
2876	* does:
2877	*
2878	* mov %rsp, %rax
2879	* ...
2880	* mov %rax, %rsp
2881	*/
2882	cfi->vals[op->dest.reg].base = CFI_CFA;
2883	cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2884	}
2885
2886	else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2887	(cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2888
2889	/*
2890	* mov %rbp, %rsp
2891	*
2892	* Restore the original stack pointer (Clang).
2893	*/
2894	cfi->stack_size = -cfi->regs[CFI_BP].offset;
2895	}
2896
2897	else if (op->dest.reg == cfa->base) {
2898
2899	/* mov %reg, %rsp */
2900	if (cfa->base == CFI_SP &&
2901	cfi->vals[op->src.reg].base == CFI_CFA) {
2902
2903	/*
2904	* This is needed for the rare case
2905	* where GCC does something dumb like:
2906	*
2907	* lea 0x8(%rsp), %rcx
2908	* ...
2909	* mov %rcx, %rsp
2910	*/
2911	cfa->offset = -cfi->vals[op->src.reg].offset;
2912	cfi->stack_size = cfa->offset;
2913
2914	} else if (cfa->base == CFI_SP &&
2915	cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2916	cfi->vals[op->src.reg].offset == cfa->offset) {
2917
2918	/*
2919	* Stack swizzle:
2920	*
2921	* 1: mov %rsp, (%[tos])
2922	* 2: mov %[tos], %rsp
2923	* ...
2924	* 3: pop %rsp
2925	*
2926	* Where:
2927	*
2928	* 1 - places a pointer to the previous
2929	* stack at the Top-of-Stack of the
2930	* new stack.
2931	*
2932	* 2 - switches to the new stack.
2933	*
2934	* 3 - pops the Top-of-Stack to restore
2935	* the original stack.
2936	*
2937	* Note: we set base to SP_INDIRECT
2938	* here and preserve offset. Therefore
2939	* when the unwinder reaches ToS it
2940	* will dereference SP and then add the
2941	* offset to find the next frame, IOW:
2942	* (%rsp) + offset.
2943	*/
2944	cfa->base = CFI_SP_INDIRECT;
2945
2946	} else {
2947	cfa->base = CFI_UNDEFINED;
2948	cfa->offset = 0;
2949	}
2950	}
2951
2952	else if (op->dest.reg == CFI_SP &&
2953	cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2954	cfi->vals[op->src.reg].offset == cfa->offset) {
2955
2956	/*
2957	* The same stack swizzle case 2) as above. But
2958	* because we can't change cfa->base, case 3)
2959	* will become a regular POP. Pretend we're a
2960	* PUSH so things don't go unbalanced.
2961	*/
2962	cfi->stack_size += 8;
2963	}
2964
2965
2966	break;
2967
2968	case OP_SRC_ADD:
2969	if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2970
2971	/* add imm, %rsp */
2972	cfi->stack_size -= op->src.offset;
2973	if (cfa->base == CFI_SP)
2974	cfa->offset -= op->src.offset;
2975	break;
2976	}
2977
2978	if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2979
2980	/* lea disp(%rbp), %rsp */
2981	cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2982	break;
2983	}
2984
2985	if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2986
2987	/* drap: lea disp(%rsp), %drap */
2988	cfi->drap_reg = op->dest.reg;
2989
2990	/*
2991	* lea disp(%rsp), %reg
2992	*
2993	* This is needed for the rare case where GCC
2994	* does something dumb like:
2995	*
2996	* lea 0x8(%rsp), %rcx
2997	* ...
2998	* mov %rcx, %rsp
2999	*/
3000	cfi->vals[op->dest.reg].base = CFI_CFA;
3001	cfi->vals[op->dest.reg].offset = \
3002	-cfi->stack_size + op->src.offset;
3003
3004	break;
3005	}
3006
3007	if (cfi->drap && op->dest.reg == CFI_SP &&
3008	op->src.reg == cfi->drap_reg) {
3009
3010	/* drap: lea disp(%drap), %rsp */
3011	cfa->base = CFI_SP;
3012	cfa->offset = cfi->stack_size = -op->src.offset;
3013	cfi->drap_reg = CFI_UNDEFINED;
3014	cfi->drap = false;
3015	break;
3016	}
3017
3018	if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3019	WARN_INSN(insn, "unsupported stack register modification");
3020	return -1;
3021	}
3022
3023	break;
3024
3025	case OP_SRC_AND:
3026	if (op->dest.reg != CFI_SP ||
3027	(cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
3028	(cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3029	WARN_INSN(insn, "unsupported stack pointer realignment");
3030	return -1;
3031	}
3032
3033	if (cfi->drap_reg != CFI_UNDEFINED) {
3034	/* drap: and imm, %rsp */
3035	cfa->base = cfi->drap_reg;
3036	cfa->offset = cfi->stack_size = 0;
3037	cfi->drap = true;
3038	}
3039
3040	/*
3041	* Older versions of GCC (4.8ish) realign the stack
3042	* without DRAP, with a frame pointer.
3043	*/
3044
3045	break;
3046
3047	case OP_SRC_POP:
3048	case OP_SRC_POPF:
3049	if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3050
3051	/* pop %rsp; # restore from a stack swizzle */
3052	cfa->base = CFI_SP;
3053	break;
3054	}
3055
3056	if (!cfi->drap && op->dest.reg == cfa->base) {
3057
3058	/* pop %rbp */
3059	cfa->base = CFI_SP;
3060	}
3061
3062	if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3063	op->dest.reg == cfi->drap_reg &&
3064	cfi->drap_offset == -cfi->stack_size) {
3065
3066	/* drap: pop %drap */
3067	cfa->base = cfi->drap_reg;
3068	cfa->offset = 0;
3069	cfi->drap_offset = -1;
3070
3071	} else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3072
3073	/* pop %reg */
3074	restore_reg(cfi, reg: op->dest.reg);
3075	}
3076
3077	cfi->stack_size -= 8;
3078	if (cfa->base == CFI_SP)
3079	cfa->offset -= 8;
3080
3081	break;
3082
3083	case OP_SRC_REG_INDIRECT:
3084	if (!cfi->drap && op->dest.reg == cfa->base &&
3085	op->dest.reg == CFI_BP) {
3086
3087	/* mov disp(%rsp), %rbp */
3088	cfa->base = CFI_SP;
3089	cfa->offset = cfi->stack_size;
3090	}
3091
3092	if (cfi->drap && op->src.reg == CFI_BP &&
3093	op->src.offset == cfi->drap_offset) {
3094
3095	/* drap: mov disp(%rbp), %drap */
3096	cfa->base = cfi->drap_reg;
3097	cfa->offset = 0;
3098	cfi->drap_offset = -1;
3099	}
3100
3101	if (cfi->drap && op->src.reg == CFI_BP &&
3102	op->src.offset == regs[op->dest.reg].offset) {
3103
3104	/* drap: mov disp(%rbp), %reg */
3105	restore_reg(cfi, reg: op->dest.reg);
3106
3107	} else if (op->src.reg == cfa->base &&
3108	op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3109
3110	/* mov disp(%rbp), %reg */
3111	/* mov disp(%rsp), %reg */
3112	restore_reg(cfi, reg: op->dest.reg);
3113
3114	} else if (op->src.reg == CFI_SP &&
3115	op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3116
3117	/* mov disp(%rsp), %reg */
3118	restore_reg(cfi, reg: op->dest.reg);
3119	}
3120
3121	break;
3122
3123	default:
3124	WARN_INSN(insn, "unknown stack-related instruction");
3125	return -1;
3126	}
3127
3128	break;
3129
3130	case OP_DEST_PUSH:
3131	case OP_DEST_PUSHF:
3132	cfi->stack_size += 8;
3133	if (cfa->base == CFI_SP)
3134	cfa->offset += 8;
3135
3136	if (op->src.type != OP_SRC_REG)
3137	break;
3138
3139	if (cfi->drap) {
3140	if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3141
3142	/* drap: push %drap */
3143	cfa->base = CFI_BP_INDIRECT;
3144	cfa->offset = -cfi->stack_size;
3145
3146	/* save drap so we know when to restore it */
3147	cfi->drap_offset = -cfi->stack_size;
3148
3149	} else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3150
3151	/* drap: push %rbp */
3152	cfi->stack_size = 0;
3153
3154	} else {
3155
3156	/* drap: push %reg */
3157	save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3158	}
3159
3160	} else {
3161
3162	/* push %reg */
3163	save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3164	}
3165
3166	/* detect when asm code uses rbp as a scratch register */
3167	if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3168	cfa->base != CFI_BP)
3169	cfi->bp_scratch = true;
3170	break;
3171
3172	case OP_DEST_REG_INDIRECT:
3173
3174	if (cfi->drap) {
3175	if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3176
3177	/* drap: mov %drap, disp(%rbp) */
3178	cfa->base = CFI_BP_INDIRECT;
3179	cfa->offset = op->dest.offset;
3180
3181	/* save drap offset so we know when to restore it */
3182	cfi->drap_offset = op->dest.offset;
3183	} else {
3184
3185	/* drap: mov reg, disp(%rbp) */
3186	save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3187	}
3188
3189	} else if (op->dest.reg == cfa->base) {
3190
3191	/* mov reg, disp(%rbp) */
3192	/* mov reg, disp(%rsp) */
3193	save_reg(cfi, op->src.reg, CFI_CFA,
3194	op->dest.offset - cfi->cfa.offset);
3195
3196	} else if (op->dest.reg == CFI_SP) {
3197
3198	/* mov reg, disp(%rsp) */
3199	save_reg(cfi, op->src.reg, CFI_CFA,
3200	op->dest.offset - cfi->stack_size);
3201
3202	} else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3203
3204	/* mov %rsp, (%reg); # setup a stack swizzle. */
3205	cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3206	cfi->vals[op->dest.reg].offset = cfa->offset;
3207	}
3208
3209	break;
3210
3211	case OP_DEST_MEM:
3212	if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3213	WARN_INSN(insn, "unknown stack-related memory operation");
3214	return -1;
3215	}
3216
3217	/* pop mem */
3218	cfi->stack_size -= 8;
3219	if (cfa->base == CFI_SP)
3220	cfa->offset -= 8;
3221
3222	break;
3223
3224	default:
3225	WARN_INSN(insn, "unknown stack-related instruction");
3226	return -1;
3227	}
3228
3229	return 0;
3230	}
3231
3232	/*
3233	* The stack layouts of alternatives instructions can sometimes diverge when
3234	* they have stack modifications. That's fine as long as the potential stack
3235	* layouts don't conflict at any given potential instruction boundary.
3236	*
3237	* Flatten the CFIs of the different alternative code streams (both original
3238	* and replacement) into a single shared CFI array which can be used to detect
3239	* conflicts and nicely feed a linear array of ORC entries to the unwinder.
3240	*/
3241	static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3242	{
3243	struct cfi_state **alt_cfi;
3244	int group_off;
3245
3246	if (!insn->alt_group)
3247	return 0;
3248
3249	if (!insn->cfi) {
3250	WARN("CFI missing");
3251	return -1;
3252	}
3253
3254	alt_cfi = insn->alt_group->cfi;
3255	group_off = insn->offset - insn->alt_group->first_insn->offset;
3256
3257	if (!alt_cfi[group_off]) {
3258	alt_cfi[group_off] = insn->cfi;
3259	} else {
3260	if (cficmp(cfi1: alt_cfi[group_off], cfi2: insn->cfi)) {
3261	struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3262	struct instruction *orig = orig_group->first_insn;
3263	char *where = offstr(insn->sec, insn->offset);
3264	WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3265	free(where);
3266	return -1;
3267	}
3268	}
3269
3270	return 0;
3271	}
3272
3273	static int handle_insn_ops(struct instruction *insn,
3274	struct instruction *next_insn,
3275	struct insn_state *state)
3276	{
3277	struct stack_op *op;
3278
3279	for (op = insn->stack_ops; op; op = op->next) {
3280
3281	if (update_cfi_state(insn, next_insn, cfi: &state->cfi, op))
3282	return 1;
3283
3284	if (!insn->alt_group)
3285	continue;
3286
3287	if (op->dest.type == OP_DEST_PUSHF) {
3288	if (!state->uaccess_stack) {
3289	state->uaccess_stack = 1;
3290	} else if (state->uaccess_stack >> 31) {
3291	WARN_INSN(insn, "PUSHF stack exhausted");
3292	return 1;
3293	}
3294	state->uaccess_stack <<= 1;
3295	state->uaccess_stack |= state->uaccess;
3296	}
3297
3298	if (op->src.type == OP_SRC_POPF) {
3299	if (state->uaccess_stack) {
3300	state->uaccess = state->uaccess_stack & 1;
3301	state->uaccess_stack >>= 1;
3302	if (state->uaccess_stack == 1)
3303	state->uaccess_stack = 0;
3304	}
3305	}
3306	}
3307
3308	return 0;
3309	}
3310
3311	static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3312	{
3313	struct cfi_state *cfi1 = insn->cfi;
3314	int i;
3315
3316	if (!cfi1) {
3317	WARN("CFI missing");
3318	return false;
3319	}
3320
3321	if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3322
3323	WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3324	cfi1->cfa.base, cfi1->cfa.offset,
3325	cfi2->cfa.base, cfi2->cfa.offset);
3326
3327	} else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3328	for (i = 0; i < CFI_NUM_REGS; i++) {
3329	if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3330	sizeof(struct cfi_reg)))
3331	continue;
3332
3333	WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3334	i, cfi1->regs[i].base, cfi1->regs[i].offset,
3335	i, cfi2->regs[i].base, cfi2->regs[i].offset);
3336	break;
3337	}
3338
3339	} else if (cfi1->type != cfi2->type) {
3340
3341	WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3342	cfi1->type, cfi2->type);
3343
3344	} else if (cfi1->drap != cfi2->drap ||
3345	(cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3346	(cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3347
3348	WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3349	cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3350	cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3351
3352	} else
3353	return true;
3354
3355	return false;
3356	}
3357
3358	static inline bool func_uaccess_safe(struct symbol *func)
3359	{
3360	if (func)
3361	return func->uaccess_safe;
3362
3363	return false;
3364	}
3365
3366	static inline const char *call_dest_name(struct instruction *insn)
3367	{
3368	static char pvname[19];
3369	struct reloc *reloc;
3370	int idx;
3371
3372	if (insn_call_dest(insn))
3373	return insn_call_dest(insn)->name;
3374
3375	reloc = insn_reloc(NULL, insn);
3376	if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3377	idx = (reloc_addend(reloc) / sizeof(void *));
3378	snprintf(buf: pvname, size: sizeof(pvname), fmt: "pv_ops[%d]", idx);
3379	return pvname;
3380	}
3381
3382	return "{dynamic}";
3383	}
3384
3385	static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3386	{
3387	struct symbol *target;
3388	struct reloc *reloc;
3389	int idx;
3390
3391	reloc = insn_reloc(file, insn);
3392	if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3393	return false;
3394
3395	idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3396
3397	if (file->pv_ops[idx].clean)
3398	return true;
3399
3400	file->pv_ops[idx].clean = true;
3401
3402	list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3403	if (!target->sec->noinstr) {
3404	WARN("pv_ops[%d]: %s", idx, target->name);
3405	file->pv_ops[idx].clean = false;
3406	}
3407	}
3408
3409	return file->pv_ops[idx].clean;
3410	}
3411
3412	static inline bool noinstr_call_dest(struct objtool_file *file,
3413	struct instruction *insn,
3414	struct symbol *func)
3415	{
3416	/*
3417	* We can't deal with indirect function calls at present;
3418	* assume they're instrumented.
3419	*/
3420	if (!func) {
3421	if (file->pv_ops)
3422	return pv_call_dest(file, insn);
3423
3424	return false;
3425	}
3426
3427	/*
3428	* If the symbol is from a noinstr section; we good.
3429	*/
3430	if (func->sec->noinstr)
3431	return true;
3432
3433	/*
3434	* If the symbol is a static_call trampoline, we can't tell.
3435	*/
3436	if (func->static_call_tramp)
3437	return true;
3438
3439	/*
3440	* The __ubsan_handle_*() calls are like WARN(), they only happen when
3441	* something 'BAD' happened. At the risk of taking the machine down,
3442	* let them proceed to get the message out.
3443	*/
3444	if (!strncmp(func->name, "__ubsan_handle_", 15))
3445	return true;
3446
3447	return false;
3448	}
3449
3450	static int validate_call(struct objtool_file *file,
3451	struct instruction *insn,
3452	struct insn_state *state)
3453	{
3454	if (state->noinstr && state->instr <= 0 &&
3455	!noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3456	WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3457	return 1;
3458	}
3459
3460	if (state->uaccess && !func_uaccess_safe(func: insn_call_dest(insn))) {
3461	WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3462	return 1;
3463	}
3464
3465	if (state->df) {
3466	WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3467	return 1;
3468	}
3469
3470	return 0;
3471	}
3472
3473	static int validate_sibling_call(struct objtool_file *file,
3474	struct instruction *insn,
3475	struct insn_state *state)
3476	{
3477	if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3478	WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3479	return 1;
3480	}
3481
3482	return validate_call(file, insn, state);
3483	}
3484
3485	static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3486	{
3487	if (state->noinstr && state->instr > 0) {
3488	WARN_INSN(insn, "return with instrumentation enabled");
3489	return 1;
3490	}
3491
3492	if (state->uaccess && !func_uaccess_safe(func)) {
3493	WARN_INSN(insn, "return with UACCESS enabled");
3494	return 1;
3495	}
3496
3497	if (!state->uaccess && func_uaccess_safe(func)) {
3498	WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3499	return 1;
3500	}
3501
3502	if (state->df) {
3503	WARN_INSN(insn, "return with DF set");
3504	return 1;
3505	}
3506
3507	if (func && has_modified_stack_frame(insn, state)) {
3508	WARN_INSN(insn, "return with modified stack frame");
3509	return 1;
3510	}
3511
3512	if (state->cfi.bp_scratch) {
3513	WARN_INSN(insn, "BP used as a scratch register");
3514	return 1;
3515	}
3516
3517	return 0;
3518	}
3519
3520	static struct instruction *next_insn_to_validate(struct objtool_file *file,
3521	struct instruction *insn)
3522	{
3523	struct alt_group *alt_group = insn->alt_group;
3524
3525	/*
3526	* Simulate the fact that alternatives are patched in-place. When the
3527	* end of a replacement alt_group is reached, redirect objtool flow to
3528	* the end of the original alt_group.
3529	*
3530	* insn->alts->insn -> alt_group->first_insn
3531	* ...
3532	* alt_group->last_insn
3533	* [alt_group->nop] -> next(orig_group->last_insn)
3534	*/
3535	if (alt_group) {
3536	if (alt_group->nop) {
3537	/* ->nop implies ->orig_group */
3538	if (insn == alt_group->last_insn)
3539	return alt_group->nop;
3540	if (insn == alt_group->nop)
3541	goto next_orig;
3542	}
3543	if (insn == alt_group->last_insn && alt_group->orig_group)
3544	goto next_orig;
3545	}
3546
3547	return next_insn_same_sec(file, insn);
3548
3549	next_orig:
3550	return next_insn_same_sec(file, insn: alt_group->orig_group->last_insn);
3551	}
3552
3553	/*
3554	* Follow the branch starting at the given instruction, and recursively follow
3555	* any other branches (jumps). Meanwhile, track the frame pointer state at
3556	* each instruction and validate all the rules described in
3557	* tools/objtool/Documentation/objtool.txt.
3558	*/
3559	static int validate_branch(struct objtool_file *file, struct symbol *func,
3560	struct instruction *insn, struct insn_state state)
3561	{
3562	struct alternative *alt;
3563	struct instruction *next_insn, *prev_insn = NULL;
3564	struct section *sec;
3565	u8 visited;
3566	int ret;
3567
3568	sec = insn->sec;
3569
3570	while (1) {
3571	next_insn = next_insn_to_validate(file, insn);
3572
3573	if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3574	/* Ignore KCFI type preambles, which always fall through */
3575	if (!strncmp(func->name, "__cfi_", 6) ||
3576	!strncmp(func->name, "__pfx_", 6))
3577	return 0;
3578
3579	WARN("%s() falls through to next function %s()",
3580	func->name, insn_func(insn)->name);
3581	return 1;
3582	}
3583
3584	if (func && insn->ignore) {
3585	WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3586	return 1;
3587	}
3588
3589	visited = VISITED_BRANCH << state.uaccess;
3590	if (insn->visited & VISITED_BRANCH_MASK) {
3591	if (!insn->hint && !insn_cfi_match(insn, cfi2: &state.cfi))
3592	return 1;
3593
3594	if (insn->visited & visited)
3595	return 0;
3596	} else {
3597	nr_insns_visited++;
3598	}
3599
3600	if (state.noinstr)
3601	state.instr += insn->instr;
3602
3603	if (insn->hint) {
3604	if (insn->restore) {
3605	struct instruction *save_insn, *i;
3606
3607	i = insn;
3608	save_insn = NULL;
3609
3610	sym_for_each_insn_continue_reverse(file, func, i) {
3611	if (i->save) {
3612	save_insn = i;
3613	break;
3614	}
3615	}
3616
3617	if (!save_insn) {
3618	WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3619	return 1;
3620	}
3621
3622	if (!save_insn->visited) {
3623	WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3624	return 1;
3625	}
3626
3627	insn->cfi = save_insn->cfi;
3628	nr_cfi_reused++;
3629	}
3630
3631	state.cfi = *insn->cfi;
3632	} else {
3633	/* XXX track if we actually changed state.cfi */
3634
3635	if (prev_insn && !cficmp(cfi1: prev_insn->cfi, cfi2: &state.cfi)) {
3636	insn->cfi = prev_insn->cfi;
3637	nr_cfi_reused++;
3638	} else {
3639	insn->cfi = cfi_hash_find_or_add(cfi: &state.cfi);
3640	}
3641	}
3642
3643	insn->visited |= visited;
3644
3645	if (propagate_alt_cfi(file, insn))
3646	return 1;
3647
3648	if (!insn->ignore_alts && insn->alts) {
3649	bool skip_orig = false;
3650
3651	for (alt = insn->alts; alt; alt = alt->next) {
3652	if (alt->skip_orig)
3653	skip_orig = true;
3654
3655	ret = validate_branch(file, func, insn: alt->insn, state: state);
3656	if (ret) {
3657	BT_INSN(insn, "(alt)");
3658	return ret;
3659	}
3660	}
3661
3662	if (skip_orig)
3663	return 0;
3664	}
3665
3666	if (handle_insn_ops(insn, next_insn, state: &state))
3667	return 1;
3668
3669	switch (insn->type) {
3670
3671	case INSN_RETURN:
3672	return validate_return(func, insn, state: &state);
3673
3674	case INSN_CALL:
3675	case INSN_CALL_DYNAMIC:
3676	ret = validate_call(file, insn, state: &state);
3677	if (ret)
3678	return ret;
3679
3680	if (opts.stackval && func && !is_special_call(insn) &&
3681	!has_valid_stack_frame(&state)) {
3682	WARN_INSN(insn, "call without frame pointer save/setup");
3683	return 1;
3684	}
3685
3686	if (insn->dead_end)
3687	return 0;
3688
3689	break;
3690
3691	case INSN_JUMP_CONDITIONAL:
3692	case INSN_JUMP_UNCONDITIONAL:
3693	if (is_sibling_call(insn)) {
3694	ret = validate_sibling_call(file, insn, state: &state);
3695	if (ret)
3696	return ret;
3697
3698	} else if (insn->jump_dest) {
3699	ret = validate_branch(file, func,
3700	insn: insn->jump_dest, state: state);
3701	if (ret) {
3702	BT_INSN(insn, "(branch)");
3703	return ret;
3704	}
3705	}
3706
3707	if (insn->type == INSN_JUMP_UNCONDITIONAL)
3708	return 0;
3709
3710	break;
3711
3712	case INSN_JUMP_DYNAMIC:
3713	case INSN_JUMP_DYNAMIC_CONDITIONAL:
3714	if (is_sibling_call(insn)) {
3715	ret = validate_sibling_call(file, insn, state: &state);
3716	if (ret)
3717	return ret;
3718	}
3719
3720	if (insn->type == INSN_JUMP_DYNAMIC)
3721	return 0;
3722
3723	break;
3724
3725	case INSN_CONTEXT_SWITCH:
3726	if (func && (!next_insn || !next_insn->hint)) {
3727	WARN_INSN(insn, "unsupported instruction in callable function");
3728	return 1;
3729	}
3730	return 0;
3731
3732	case INSN_STAC:
3733	if (state.uaccess) {
3734	WARN_INSN(insn, "recursive UACCESS enable");
3735	return 1;
3736	}
3737
3738	state.uaccess = true;
3739	break;
3740
3741	case INSN_CLAC:
3742	if (!state.uaccess && func) {
3743	WARN_INSN(insn, "redundant UACCESS disable");
3744	return 1;
3745	}
3746
3747	if (func_uaccess_safe(func) && !state.uaccess_stack) {
3748	WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3749	return 1;
3750	}
3751
3752	state.uaccess = false;
3753	break;
3754
3755	case INSN_STD:
3756	if (state.df) {
3757	WARN_INSN(insn, "recursive STD");
3758	return 1;
3759	}
3760
3761	state.df = true;
3762	break;
3763
3764	case INSN_CLD:
3765	if (!state.df && func) {
3766	WARN_INSN(insn, "redundant CLD");
3767	return 1;
3768	}
3769
3770	state.df = false;
3771	break;
3772
3773	default:
3774	break;
3775	}
3776
3777	if (insn->dead_end)
3778	return 0;
3779
3780	if (!next_insn) {
3781	if (state.cfi.cfa.base == CFI_UNDEFINED)
3782	return 0;
3783	WARN("%s: unexpected end of section", sec->name);
3784	return 1;
3785	}
3786
3787	prev_insn = insn;
3788	insn = next_insn;
3789	}
3790
3791	return 0;
3792	}
3793
3794	static int validate_unwind_hint(struct objtool_file *file,
3795	struct instruction *insn,
3796	struct insn_state *state)
3797	{
3798	if (insn->hint && !insn->visited && !insn->ignore) {
3799	int ret = validate_branch(file, insn_func(insn), insn, *state);
3800	if (ret)
3801	BT_INSN(insn, "<=== (hint)");
3802	return ret;
3803	}
3804
3805	return 0;
3806	}
3807
3808	static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3809	{
3810	struct instruction *insn;
3811	struct insn_state state;
3812	int warnings = 0;
3813
3814	if (!file->hints)
3815	return 0;
3816
3817	init_insn_state(file, state: &state, sec);
3818
3819	if (sec) {
3820	sec_for_each_insn(file, sec, insn)
3821	warnings += validate_unwind_hint(file, insn, state: &state);
3822	} else {
3823	for_each_insn(file, insn)
3824	warnings += validate_unwind_hint(file, insn, &state);
3825	}
3826
3827	return warnings;
3828	}
3829
3830	/*
3831	* Validate rethunk entry constraint: must untrain RET before the first RET.
3832	*
3833	* Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3834	* before an actual RET instruction.
3835	*/
3836	static int validate_unret(struct objtool_file *file, struct instruction *insn)
3837	{
3838	struct instruction *next, *dest;
3839	int ret;
3840
3841	for (;;) {
3842	next = next_insn_to_validate(file, insn);
3843
3844	if (insn->visited & VISITED_UNRET)
3845	return 0;
3846
3847	insn->visited |= VISITED_UNRET;
3848
3849	if (!insn->ignore_alts && insn->alts) {
3850	struct alternative *alt;
3851	bool skip_orig = false;
3852
3853	for (alt = insn->alts; alt; alt = alt->next) {
3854	if (alt->skip_orig)
3855	skip_orig = true;
3856
3857	ret = validate_unret(file, insn: alt->insn);
3858	if (ret) {
3859	BT_INSN(insn, "(alt)");
3860	return ret;
3861	}
3862	}
3863
3864	if (skip_orig)
3865	return 0;
3866	}
3867
3868	switch (insn->type) {
3869
3870	case INSN_CALL_DYNAMIC:
3871	case INSN_JUMP_DYNAMIC:
3872	case INSN_JUMP_DYNAMIC_CONDITIONAL:
3873	WARN_INSN(insn, "early indirect call");
3874	return 1;
3875
3876	case INSN_JUMP_UNCONDITIONAL:
3877	case INSN_JUMP_CONDITIONAL:
3878	if (!is_sibling_call(insn)) {
3879	if (!insn->jump_dest) {
3880	WARN_INSN(insn, "unresolved jump target after linking?!?");
3881	return -1;
3882	}
3883	ret = validate_unret(file, insn: insn->jump_dest);
3884	if (ret) {
3885	BT_INSN(insn, "(branch%s)",
3886	insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3887	return ret;
3888	}
3889
3890	if (insn->type == INSN_JUMP_UNCONDITIONAL)
3891	return 0;
3892
3893	break;
3894	}
3895
3896	/* fallthrough */
3897	case INSN_CALL:
3898	dest = find_insn(file, sec: insn_call_dest(insn)->sec,
3899	offset: insn_call_dest(insn)->offset);
3900	if (!dest) {
3901	WARN("Unresolved function after linking!?: %s",
3902	insn_call_dest(insn)->name);
3903	return -1;
3904	}
3905
3906	ret = validate_unret(file, insn: dest);
3907	if (ret) {
3908	BT_INSN(insn, "(call)");
3909	return ret;
3910	}
3911	/*
3912	* If a call returns without error, it must have seen UNTRAIN_RET.
3913	* Therefore any non-error return is a success.
3914	*/
3915	return 0;
3916
3917	case INSN_RETURN:
3918	WARN_INSN(insn, "RET before UNTRAIN");
3919	return 1;
3920
3921	case INSN_NOP:
3922	if (insn->retpoline_safe)
3923	return 0;
3924	break;
3925
3926	default:
3927	break;
3928	}
3929
3930	if (!next) {
3931	WARN_INSN(insn, "teh end!");
3932	return -1;
3933	}
3934	insn = next;
3935	}
3936
3937	return 0;
3938	}
3939
3940	/*
3941	* Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
3942	* VALIDATE_UNRET_END before RET.
3943	*/
3944	static int validate_unrets(struct objtool_file *file)
3945	{
3946	struct instruction *insn;
3947	int ret, warnings = 0;
3948
3949	for_each_insn(file, insn) {
3950	if (!insn->unret)
3951	continue;
3952
3953	ret = validate_unret(file, insn);
3954	if (ret < 0) {
3955	WARN_INSN(insn, "Failed UNRET validation");
3956	return ret;
3957	}
3958	warnings += ret;
3959	}
3960
3961	return warnings;
3962	}
3963
3964	static int validate_retpoline(struct objtool_file *file)
3965	{
3966	struct instruction *insn;
3967	int warnings = 0;
3968
3969	for_each_insn(file, insn) {
3970	if (insn->type != INSN_JUMP_DYNAMIC &&
3971	insn->type != INSN_CALL_DYNAMIC &&
3972	insn->type != INSN_RETURN)
3973	continue;
3974
3975	if (insn->retpoline_safe)
3976	continue;
3977
3978	if (insn->sec->init)
3979	continue;
3980
3981	if (insn->type == INSN_RETURN) {
3982	if (opts.rethunk) {
3983	WARN_INSN(insn, "'naked' return found in RETHUNK build");
3984	} else
3985	continue;
3986	} else {
3987	WARN_INSN(insn, "indirect %s found in RETPOLINE build",
3988	insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3989	}
3990
3991	warnings++;
3992	}
3993
3994	return warnings;
3995	}
3996
3997	static bool is_kasan_insn(struct instruction *insn)
3998	{
3999	return (insn->type == INSN_CALL &&
4000	!strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4001	}
4002
4003	static bool is_ubsan_insn(struct instruction *insn)
4004	{
4005	return (insn->type == INSN_CALL &&
4006	!strcmp(insn_call_dest(insn)->name,
4007	"__ubsan_handle_builtin_unreachable"));
4008	}
4009
4010	static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4011	{
4012	int i;
4013	struct instruction *prev_insn;
4014
4015	if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
4016	return true;
4017
4018	/*
4019	* Ignore alternative replacement instructions. This can happen
4020	* when a whitelisted function uses one of the ALTERNATIVE macros.
4021	*/
4022	if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4023	!strcmp(insn->sec->name, ".altinstr_aux"))
4024	return true;
4025
4026	/*
4027	* Whole archive runs might encounter dead code from weak symbols.
4028	* This is where the linker will have dropped the weak symbol in
4029	* favour of a regular symbol, but leaves the code in place.
4030	*
4031	* In this case we'll find a piece of code (whole function) that is not
4032	* covered by a !section symbol. Ignore them.
4033	*/
4034	if (opts.link && !insn_func(insn)) {
4035	int size = find_symbol_hole_containing(insn->sec, insn->offset);
4036	unsigned long end = insn->offset + size;
4037
4038	if (!size) /* not a hole */
4039	return false;
4040
4041	if (size < 0) /* hole until the end */
4042	return true;
4043
4044	sec_for_each_insn_continue(file, insn) {
4045	/*
4046	* If we reach a visited instruction at or before the
4047	* end of the hole, ignore the unreachable.
4048	*/
4049	if (insn->visited)
4050	return true;
4051
4052	if (insn->offset >= end)
4053	break;
4054
4055	/*
4056	* If this hole jumps to a .cold function, mark it ignore too.
4057	*/
4058	if (insn->jump_dest && insn_func(insn->jump_dest) &&
4059	strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4060	struct instruction *dest = insn->jump_dest;
4061	func_for_each_insn(file, insn_func(dest), dest)
4062	dest->ignore = true;
4063	}
4064	}
4065
4066	return false;
4067	}
4068
4069	if (!insn_func(insn))
4070	return false;
4071
4072	if (insn_func(insn)->static_call_tramp)
4073	return true;
4074
4075	/*
4076	* CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4077	* __builtin_unreachable(). The BUG() macro has an unreachable() after
4078	* the UD2, which causes GCC's undefined trap logic to emit another UD2
4079	* (or occasionally a JMP to UD2).
4080	*
4081	* It may also insert a UD2 after calling a __noreturn function.
4082	*/
4083	prev_insn = prev_insn_same_sec(file, insn);
4084	if (prev_insn->dead_end &&
4085	(insn->type == INSN_BUG ||
4086	(insn->type == INSN_JUMP_UNCONDITIONAL &&
4087	insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4088	return true;
4089
4090	/*
4091	* Check if this (or a subsequent) instruction is related to
4092	* CONFIG_UBSAN or CONFIG_KASAN.
4093	*
4094	* End the search at 5 instructions to avoid going into the weeds.
4095	*/
4096	for (i = 0; i < 5; i++) {
4097
4098	if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4099	return true;
4100
4101	if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4102	if (insn->jump_dest &&
4103	insn_func(insn->jump_dest) == insn_func(insn)) {
4104	insn = insn->jump_dest;
4105	continue;
4106	}
4107
4108	break;
4109	}
4110
4111	if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4112	break;
4113
4114	insn = next_insn_same_sec(file, insn);
4115	}
4116
4117	return false;
4118	}
4119
4120	static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4121	{
4122	struct instruction *insn, *prev;
4123	struct cfi_state *cfi;
4124
4125	insn = find_insn(file, sec: func->sec, offset: func->offset);
4126	if (!insn)
4127	return -1;
4128
4129	for (prev = prev_insn_same_sec(file, insn);
4130	prev;
4131	prev = prev_insn_same_sec(file, insn: prev)) {
4132	u64 offset;
4133
4134	if (prev->type != INSN_NOP)
4135	return -1;
4136
4137	offset = func->offset - prev->offset;
4138
4139	if (offset > opts.prefix)
4140	return -1;
4141
4142	if (offset < opts.prefix)
4143	continue;
4144
4145	elf_create_prefix_symbol(file->elf, func, opts.prefix);
4146	break;
4147	}
4148
4149	if (!prev)
4150	return -1;
4151
4152	if (!insn->cfi) {
4153	/*
4154	* This can happen if stack validation isn't enabled or the
4155	* function is annotated with STACK_FRAME_NON_STANDARD.
4156	*/
4157	return 0;
4158	}
4159
4160	/* Propagate insn->cfi to the prefix code */
4161	cfi = cfi_hash_find_or_add(cfi: insn->cfi);
4162	for (; prev != insn; prev = next_insn_same_sec(file, insn: prev))
4163	prev->cfi = cfi;
4164
4165	return 0;
4166	}
4167
4168	static int add_prefix_symbols(struct objtool_file *file)
4169	{
4170	struct section *sec;
4171	struct symbol *func;
4172
4173	for_each_sec(file, sec) {
4174	if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4175	continue;
4176
4177	sec_for_each_sym(sec, func) {
4178	if (func->type != STT_FUNC)
4179	continue;
4180
4181	add_prefix_symbol(file, func);
4182	}
4183	}
4184
4185	return 0;
4186	}
4187
4188	static int validate_symbol(struct objtool_file *file, struct section *sec,
4189	struct symbol *sym, struct insn_state *state)
4190	{
4191	struct instruction *insn;
4192	int ret;
4193
4194	if (!sym->len) {
4195	WARN("%s() is missing an ELF size annotation", sym->name);
4196	return 1;
4197	}
4198
4199	if (sym->pfunc != sym || sym->alias != sym)
4200	return 0;
4201
4202	insn = find_insn(file, sec, offset: sym->offset);
4203	if (!insn || insn->ignore || insn->visited)
4204	return 0;
4205
4206	state->uaccess = sym->uaccess_safe;
4207
4208	ret = validate_branch(file, insn_func(insn), insn, *state);
4209	if (ret)
4210	BT_INSN(insn, "<=== (sym)");
4211	return ret;
4212	}
4213
4214	static int validate_section(struct objtool_file *file, struct section *sec)
4215	{
4216	struct insn_state state;
4217	struct symbol *func;
4218	int warnings = 0;
4219
4220	sec_for_each_sym(sec, func) {
4221	if (func->type != STT_FUNC)
4222	continue;
4223
4224	init_insn_state(file, state: &state, sec);
4225	set_func_state(&state.cfi);
4226
4227	warnings += validate_symbol(file, sec, sym: func, state: &state);
4228	}
4229
4230	return warnings;
4231	}
4232
4233	static int validate_noinstr_sections(struct objtool_file *file)
4234	{
4235	struct section *sec;
4236	int warnings = 0;
4237
4238	sec = find_section_by_name(file->elf, ".noinstr.text");
4239	if (sec) {
4240	warnings += validate_section(file, sec);
4241	warnings += validate_unwind_hints(file, sec);
4242	}
4243
4244	sec = find_section_by_name(file->elf, ".entry.text");
4245	if (sec) {
4246	warnings += validate_section(file, sec);
4247	warnings += validate_unwind_hints(file, sec);
4248	}
4249
4250	sec = find_section_by_name(file->elf, ".cpuidle.text");
4251	if (sec) {
4252	warnings += validate_section(file, sec);
4253	warnings += validate_unwind_hints(file, sec);
4254	}
4255
4256	return warnings;
4257	}
4258
4259	static int validate_functions(struct objtool_file *file)
4260	{
4261	struct section *sec;
4262	int warnings = 0;
4263
4264	for_each_sec(file, sec) {
4265	if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4266	continue;
4267
4268	warnings += validate_section(file, sec);
4269	}
4270
4271	return warnings;
4272	}
4273
4274	static void mark_endbr_used(struct instruction *insn)
4275	{
4276	if (!list_empty(head: &insn->call_node))
4277	list_del_init(entry: &insn->call_node);
4278	}
4279
4280	static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4281	{
4282	struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4283	struct instruction *first;
4284
4285	if (!sym)
4286	return false;
4287
4288	first = find_insn(file, sec: sym->sec, offset: sym->offset);
4289	if (!first)
4290	return false;
4291
4292	if (first->type != INSN_ENDBR && !first->noendbr)
4293	return false;
4294
4295	return insn->offset == sym->offset + sym->len;
4296	}
4297
4298	static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4299	{
4300	struct instruction *dest;
4301	struct reloc *reloc;
4302	unsigned long off;
4303	int warnings = 0;
4304
4305	/*
4306	* Looking for function pointer load relocations. Ignore
4307	* direct/indirect branches:
4308	*/
4309	switch (insn->type) {
4310	case INSN_CALL:
4311	case INSN_CALL_DYNAMIC:
4312	case INSN_JUMP_CONDITIONAL:
4313	case INSN_JUMP_UNCONDITIONAL:
4314	case INSN_JUMP_DYNAMIC:
4315	case INSN_JUMP_DYNAMIC_CONDITIONAL:
4316	case INSN_RETURN:
4317	case INSN_NOP:
4318	return 0;
4319	default:
4320	break;
4321	}
4322
4323	for (reloc = insn_reloc(file, insn);
4324	reloc;
4325	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4326	reloc_offset(reloc) + 1,
4327	(insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4328
4329	/*
4330	* static_call_update() references the trampoline, which
4331	* doesn't have (or need) ENDBR. Skip warning in that case.
4332	*/
4333	if (reloc->sym->static_call_tramp)
4334	continue;
4335
4336	off = reloc->sym->offset;
4337	if (reloc_type(reloc) == R_X86_64_PC32 ||
4338	reloc_type(reloc) == R_X86_64_PLT32)
4339	off += arch_dest_reloc_offset(reloc_addend(reloc));
4340	else
4341	off += reloc_addend(reloc);
4342
4343	dest = find_insn(file, sec: reloc->sym->sec, offset: off);
4344	if (!dest)
4345	continue;
4346
4347	if (dest->type == INSN_ENDBR) {
4348	mark_endbr_used(insn: dest);
4349	continue;
4350	}
4351
4352	if (insn_func(dest) && insn_func(insn) &&
4353	insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4354	/*
4355	* Anything from->to self is either _THIS_IP_ or
4356	* IRET-to-self.
4357	*
4358	* There is no sane way to annotate _THIS_IP_ since the
4359	* compiler treats the relocation as a constant and is
4360	* happy to fold in offsets, skewing any annotation we
4361	* do, leading to vast amounts of false-positives.
4362	*
4363	* There's also compiler generated _THIS_IP_ through
4364	* KCOV and such which we have no hope of annotating.
4365	*
4366	* As such, blanket accept self-references without
4367	* issue.
4368	*/
4369	continue;
4370	}
4371
4372	/*
4373	* Accept anything ANNOTATE_NOENDBR.
4374	*/
4375	if (dest->noendbr)
4376	continue;
4377
4378	/*
4379	* Accept if this is the instruction after a symbol
4380	* that is (no)endbr -- typical code-range usage.
4381	*/
4382	if (noendbr_range(file, insn: dest))
4383	continue;
4384
4385	WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4386
4387	warnings++;
4388	}
4389
4390	return warnings;
4391	}
4392
4393	static int validate_ibt_data_reloc(struct objtool_file *file,
4394	struct reloc *reloc)
4395	{
4396	struct instruction *dest;
4397
4398	dest = find_insn(file, sec: reloc->sym->sec,
4399	offset: reloc->sym->offset + reloc_addend(reloc));
4400	if (!dest)
4401	return 0;
4402
4403	if (dest->type == INSN_ENDBR) {
4404	mark_endbr_used(insn: dest);
4405	return 0;
4406	}
4407
4408	if (dest->noendbr)
4409	return 0;
4410
4411	WARN_FUNC("data relocation to !ENDBR: %s",
4412	reloc->sec->base, reloc_offset(reloc),
4413	offstr(dest->sec, dest->offset));
4414
4415	return 1;
4416	}
4417
4418	/*
4419	* Validate IBT rules and remove used ENDBR instructions from the seal list.
4420	* Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4421	* NOPs) later, in create_ibt_endbr_seal_sections().
4422	*/
4423	static int validate_ibt(struct objtool_file *file)
4424	{
4425	struct section *sec;
4426	struct reloc *reloc;
4427	struct instruction *insn;
4428	int warnings = 0;
4429
4430	for_each_insn(file, insn)
4431	warnings += validate_ibt_insn(file, insn);
4432
4433	for_each_sec(file, sec) {
4434
4435	/* Already done by validate_ibt_insn() */
4436	if (sec->sh.sh_flags & SHF_EXECINSTR)
4437	continue;
4438
4439	if (!sec->rsec)
4440	continue;
4441
4442	/*
4443	* These sections can reference text addresses, but not with
4444	* the intent to indirect branch to them.
4445	*/
4446	if ((!strncmp(sec->name, ".discard", 8) &&
4447	strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
4448	!strncmp(sec->name, ".debug", 6) ||
4449	!strcmp(sec->name, ".altinstructions") ||
4450	!strcmp(sec->name, ".ibt_endbr_seal") ||
4451	!strcmp(sec->name, ".orc_unwind_ip") ||
4452	!strcmp(sec->name, ".parainstructions") ||
4453	!strcmp(sec->name, ".retpoline_sites") ||
4454	!strcmp(sec->name, ".smp_locks") ||
4455	!strcmp(sec->name, ".static_call_sites") ||
4456	!strcmp(sec->name, "_error_injection_whitelist") ||
4457	!strcmp(sec->name, "_kprobe_blacklist") ||
4458	!strcmp(sec->name, "__bug_table") ||
4459	!strcmp(sec->name, "__ex_table") ||
4460	!strcmp(sec->name, "__jump_table") ||
4461	!strcmp(sec->name, "__mcount_loc") ||
4462	!strcmp(sec->name, ".kcfi_traps") ||
4463	strstr(sec->name, "__patchable_function_entries"))
4464	continue;
4465
4466	for_each_reloc(sec->rsec, reloc)
4467	warnings += validate_ibt_data_reloc(file, reloc);
4468	}
4469
4470	return warnings;
4471	}
4472
4473	static int validate_sls(struct objtool_file *file)
4474	{
4475	struct instruction *insn, *next_insn;
4476	int warnings = 0;
4477
4478	for_each_insn(file, insn) {
4479	next_insn = next_insn_same_sec(file, insn);
4480
4481	if (insn->retpoline_safe)
4482	continue;
4483
4484	switch (insn->type) {
4485	case INSN_RETURN:
4486	if (!next_insn || next_insn->type != INSN_TRAP) {
4487	WARN_INSN(insn, "missing int3 after ret");
4488	warnings++;
4489	}
4490
4491	break;
4492	case INSN_JUMP_DYNAMIC:
4493	if (!next_insn || next_insn->type != INSN_TRAP) {
4494	WARN_INSN(insn, "missing int3 after indirect jump");
4495	warnings++;
4496	}
4497	break;
4498	default:
4499	break;
4500	}
4501	}
4502
4503	return warnings;
4504	}
4505
4506	static bool ignore_noreturn_call(struct instruction *insn)
4507	{
4508	struct symbol *call_dest = insn_call_dest(insn);
4509
4510	/*
4511	* FIXME: hack, we need a real noreturn solution
4512	*
4513	* Problem is, exc_double_fault() may or may not return, depending on
4514	* whether CONFIG_X86_ESPFIX64 is set. But objtool has no visibility
4515	* to the kernel config.
4516	*
4517	* Other potential ways to fix it:
4518	*
4519	* - have compiler communicate __noreturn functions somehow
4520	* - remove CONFIG_X86_ESPFIX64
4521	* - read the .config file
4522	* - add a cmdline option
4523	* - create a generic objtool annotation format (vs a bunch of custom
4524	* formats) and annotate it
4525	*/
4526	if (!strcmp(call_dest->name, "exc_double_fault")) {
4527	/* prevent further unreachable warnings for the caller */
4528	insn->sym->warned = 1;
4529	return true;
4530	}
4531
4532	return false;
4533	}
4534
4535	static int validate_reachable_instructions(struct objtool_file *file)
4536	{
4537	struct instruction *insn, *prev_insn;
4538	struct symbol *call_dest;
4539	int warnings = 0;
4540
4541	if (file->ignore_unreachables)
4542	return 0;
4543
4544	for_each_insn(file, insn) {
4545	if (insn->visited || ignore_unreachable_insn(file, insn))
4546	continue;
4547
4548	prev_insn = prev_insn_same_sec(file, insn);
4549	if (prev_insn && prev_insn->dead_end) {
4550	call_dest = insn_call_dest(prev_insn);
4551	if (call_dest && !ignore_noreturn_call(prev_insn)) {
4552	WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4553	call_dest->name);
4554	warnings++;
4555	continue;
4556	}
4557	}
4558
4559	WARN_INSN(insn, "unreachable instruction");
4560	warnings++;
4561	}
4562
4563	return warnings;
4564	}
4565
4566	/* 'funcs' is a space-separated list of function names */
4567	static int disas_funcs(const char *funcs)
4568	{
4569	const char *objdump_str, *cross_compile;
4570	int size, ret;
4571	char *cmd;
4572
4573	cross_compile = getenv("CROSS_COMPILE");
4574
4575	objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4576	"BEGIN { split(_funcs, funcs); }"
4577	"/^$/ { func_match = 0; }"
4578	"/<.*>:/ { "
4579	"f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4580	"for (i in funcs) {"
4581	"if (funcs[i] == f) {"
4582	"func_match = 1;"
4583	"base = strtonum(\"0x\" $1);"
4584	"break;"
4585	"}"
4586	"}"
4587	"}"
4588	"{"
4589	"if (func_match) {"
4590	"addr = strtonum(\"0x\" $1);"
4591	"printf(\"%%04x \", addr - base);"
4592	"print;"
4593	"}"
4594	"}' 1>&2";
4595
4596	/* fake snprintf() to calculate the size */
4597	size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4598	if (size <= 0) {
4599	WARN("objdump string size calculation failed");
4600	return -1;
4601	}
4602
4603	cmd = malloc(size);
4604
4605	/* real snprintf() */
4606	snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4607	ret = system(cmd);
4608	if (ret) {
4609	WARN("disassembly failed: %d", ret);
4610	return -1;
4611	}
4612
4613	return 0;
4614	}
4615
4616	static int disas_warned_funcs(struct objtool_file *file)
4617	{
4618	struct symbol *sym;
4619	char *funcs = NULL, *tmp;
4620
4621	for_each_sym(file, sym) {
4622	if (sym->warned) {
4623	if (!funcs) {
4624	funcs = malloc(strlen(sym->name) + 1);
4625	strcpy(funcs, sym->name);
4626	} else {
4627	tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4628	sprintf(buf: tmp, fmt: "%s %s", funcs, sym->name);
4629	free(funcs);
4630	funcs = tmp;
4631	}
4632	}
4633	}
4634
4635	if (funcs)
4636	disas_funcs(funcs);
4637
4638	return 0;
4639	}
4640
4641	struct insn_chunk {
4642	void *addr;
4643	struct insn_chunk *next;
4644	};
4645
4646	/*
4647	* Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4648	* which can trigger more allocations for .debug_* sections whose data hasn't
4649	* been read yet.
4650	*/
4651	static void free_insns(struct objtool_file *file)
4652	{
4653	struct instruction *insn;
4654	struct insn_chunk *chunks = NULL, *chunk;
4655
4656	for_each_insn(file, insn) {
4657	if (!insn->idx) {
4658	chunk = malloc(sizeof(*chunk));
4659	chunk->addr = insn;
4660	chunk->next = chunks;
4661	chunks = chunk;
4662	}
4663	}
4664
4665	for (chunk = chunks; chunk; chunk = chunk->next)
4666	free(chunk->addr);
4667	}
4668
4669	int check(struct objtool_file *file)
4670	{
4671	int ret, warnings = 0;
4672
4673	arch_initial_func_cfi_state(&initial_func_cfi);
4674	init_cfi_state(cfi: &init_cfi);
4675	init_cfi_state(cfi: &func_cfi);
4676	set_func_state(&func_cfi);
4677	init_cfi_state(cfi: &force_undefined_cfi);
4678	force_undefined_cfi.force_undefined = true;
4679
4680	if (!cfi_hash_alloc(size: 1UL << (file->elf->symbol_bits - 3)))
4681	goto out;
4682
4683	cfi_hash_add(cfi: &init_cfi);
4684	cfi_hash_add(cfi: &func_cfi);
4685
4686	ret = decode_sections(file);
4687	if (ret < 0)
4688	goto out;
4689
4690	warnings += ret;
4691
4692	if (!nr_insns)
4693	goto out;
4694
4695	if (opts.retpoline) {
4696	ret = validate_retpoline(file);
4697	if (ret < 0)
4698	return ret;
4699	warnings += ret;
4700	}
4701
4702	if (opts.stackval || opts.orc || opts.uaccess) {
4703	ret = validate_functions(file);
4704	if (ret < 0)
4705	goto out;
4706	warnings += ret;
4707
4708	ret = validate_unwind_hints(file, NULL);
4709	if (ret < 0)
4710	goto out;
4711	warnings += ret;
4712
4713	if (!warnings) {
4714	ret = validate_reachable_instructions(file);
4715	if (ret < 0)
4716	goto out;
4717	warnings += ret;
4718	}
4719
4720	} else if (opts.noinstr) {
4721	ret = validate_noinstr_sections(file);
4722	if (ret < 0)
4723	goto out;
4724	warnings += ret;
4725	}
4726
4727	if (opts.unret) {
4728	/*
4729	* Must be after validate_branch() and friends, it plays
4730	* further games with insn->visited.
4731	*/
4732	ret = validate_unrets(file);
4733	if (ret < 0)
4734	return ret;
4735	warnings += ret;
4736	}
4737
4738	if (opts.ibt) {
4739	ret = validate_ibt(file);
4740	if (ret < 0)
4741	goto out;
4742	warnings += ret;
4743	}
4744
4745	if (opts.sls) {
4746	ret = validate_sls(file);
4747	if (ret < 0)
4748	goto out;
4749	warnings += ret;
4750	}
4751
4752	if (opts.static_call) {
4753	ret = create_static_call_sections(file);
4754	if (ret < 0)
4755	goto out;
4756	warnings += ret;
4757	}
4758
4759	if (opts.retpoline) {
4760	ret = create_retpoline_sites_sections(file);
4761	if (ret < 0)
4762	goto out;
4763	warnings += ret;
4764	}
4765
4766	if (opts.cfi) {
4767	ret = create_cfi_sections(file);
4768	if (ret < 0)
4769	goto out;
4770	warnings += ret;
4771	}
4772
4773	if (opts.rethunk) {
4774	ret = create_return_sites_sections(file);
4775	if (ret < 0)
4776	goto out;
4777	warnings += ret;
4778
4779	if (opts.hack_skylake) {
4780	ret = create_direct_call_sections(file);
4781	if (ret < 0)
4782	goto out;
4783	warnings += ret;
4784	}
4785	}
4786
4787	if (opts.mcount) {
4788	ret = create_mcount_loc_sections(file);
4789	if (ret < 0)
4790	goto out;
4791	warnings += ret;
4792	}
4793
4794	if (opts.prefix) {
4795	ret = add_prefix_symbols(file);
4796	if (ret < 0)
4797	return ret;
4798	warnings += ret;
4799	}
4800
4801	if (opts.ibt) {
4802	ret = create_ibt_endbr_seal_sections(file);
4803	if (ret < 0)
4804	goto out;
4805	warnings += ret;
4806	}
4807
4808	if (opts.orc && nr_insns) {
4809	ret = orc_create(file);
4810	if (ret < 0)
4811	goto out;
4812	warnings += ret;
4813	}
4814
4815	free_insns(file);
4816
4817	if (opts.verbose)
4818	disas_warned_funcs(file);
4819
4820	if (opts.stats) {
4821	printf("nr_insns_visited: %ld\n", nr_insns_visited);
4822	printf("nr_cfi: %ld\n", nr_cfi);
4823	printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4824	printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4825	}
4826
4827	out:
4828	/*
4829	* For now, don't fail the kernel build on fatal warnings. These
4830	* errors are still fairly common due to the growing matrix of
4831	* supported toolchains and their recent pace of change.
4832	*/
4833	return 0;
4834	}
4835