1	/* AddressSanitizer, a fast memory error detector.
2	Copyright (C) 2012-2024 Free Software Foundation, Inc.
3	Contributed by Kostya Serebryany <kcc@google.com>
4
5	This file is part of GCC.
6
7	GCC is free software; you can redistribute it and/or modify it under
8	the terms of the GNU General Public License as published by the Free
9	Software Foundation; either version 3, or (at your option) any later
10	version.
11
12	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13	WARRANTY; without even the implied warranty of MERCHANTABILITY or
14	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15	for more details.
16
17	You should have received a copy of the GNU General Public License
18	along with GCC; see the file COPYING3. If not see
19	<http://www.gnu.org/licenses/>. */
20
21
22	#include "config.h"
23	#include "system.h"
24	#include "coretypes.h"
25	#include "backend.h"
26	#include "target.h"
27	#include "rtl.h"
28	#include "tree.h"
29	#include "gimple.h"
30	#include "cfghooks.h"
31	#include "alloc-pool.h"
32	#include "tree-pass.h"
33	#include "memmodel.h"
34	#include "tm_p.h"
35	#include "ssa.h"
36	#include "stringpool.h"
37	#include "tree-ssanames.h"
38	#include "optabs.h"
39	#include "emit-rtl.h"
40	#include "cgraph.h"
41	#include "gimple-pretty-print.h"
42	#include "alias.h"
43	#include "fold-const.h"
44	#include "cfganal.h"
45	#include "gimplify.h"
46	#include "gimple-iterator.h"
47	#include "varasm.h"
48	#include "stor-layout.h"
49	#include "tree-iterator.h"
50	#include "stringpool.h"
51	#include "attribs.h"
52	#include "asan.h"
53	#include "dojump.h"
54	#include "explow.h"
55	#include "expr.h"
56	#include "output.h"
57	#include "langhooks.h"
58	#include "cfgloop.h"
59	#include "gimple-builder.h"
60	#include "gimple-fold.h"
61	#include "ubsan.h"
62	#include "builtins.h"
63	#include "fnmatch.h"
64	#include "tree-inline.h"
65	#include "tree-ssa.h"
66	#include "tree-eh.h"
67	#include "diagnostic-core.h"
68
69	/* AddressSanitizer finds out-of-bounds and use-after-free bugs
70	with <2x slowdown on average.
71
72	The tool consists of two parts:
73	instrumentation module (this file) and a run-time library.
74	The instrumentation module adds a run-time check before every memory insn.
75	For a 8- or 16- byte load accessing address X:
76	ShadowAddr = (X >> 3) + Offset
77	ShadowValue = *(char*)ShadowAddr; // *(short*) for 16-byte access.
78	if (ShadowValue)
79	__asan_report_load8(X);
80	For a load of N bytes (N=1, 2 or 4) from address X:
81	ShadowAddr = (X >> 3) + Offset
82	ShadowValue = *(char*)ShadowAddr;
83	if (ShadowValue)
84	if ((X & 7) + N - 1 > ShadowValue)
85	__asan_report_loadN(X);
86	Stores are instrumented similarly, but using __asan_report_storeN functions.
87	A call too __asan_init_vN() is inserted to the list of module CTORs.
88	N is the version number of the AddressSanitizer API. The changes between the
89	API versions are listed in libsanitizer/asan/asan_interface_internal.h.
90
91	The run-time library redefines malloc (so that redzone are inserted around
92	the allocated memory) and free (so that reuse of free-ed memory is delayed),
93	provides __asan_report* and __asan_init_vN functions.
94
95	Read more:
96	http://code.google.com/p/address-sanitizer/wiki/AddressSanitizerAlgorithm
97
98	The current implementation supports detection of out-of-bounds and
99	use-after-free in the heap, on the stack and for global variables.
100
101	[Protection of stack variables]
102
103	To understand how detection of out-of-bounds and use-after-free works
104	for stack variables, lets look at this example on x86_64 where the
105	stack grows downward:
106
107	int
108	foo ()
109	{
110	char a[24] = {0};
111	int b[2] = {0};
112
113	a[5] = 1;
114	b[1] = 2;
115
116	return a[5] + b[1];
117	}
118
119	For this function, the stack protected by asan will be organized as
120	follows, from the top of the stack to the bottom:
121
122	Slot 1/ [red zone of 32 bytes called 'RIGHT RedZone']
123
124	Slot 2/ [8 bytes of red zone, that adds up to the space of 'a' to make
125	the next slot be 32 bytes aligned; this one is called Partial
126	Redzone; this 32 bytes alignment is an asan constraint]
127
128	Slot 3/ [24 bytes for variable 'a']
129
130	Slot 4/ [red zone of 32 bytes called 'Middle RedZone']
131
132	Slot 5/ [24 bytes of Partial Red Zone (similar to slot 2]
133
134	Slot 6/ [8 bytes for variable 'b']
135
136	Slot 7/ [32 bytes of Red Zone at the bottom of the stack, called
137	'LEFT RedZone']
138
139	The 32 bytes of LEFT red zone at the bottom of the stack can be
140	decomposed as such:
141
142	1/ The first 8 bytes contain a magical asan number that is always
143	0x41B58AB3.
144
145	2/ The following 8 bytes contains a pointer to a string (to be
146	parsed at runtime by the runtime asan library), which format is
147	the following:
148
149	"<function-name> <space> <num-of-variables-on-the-stack>
150	(<32-bytes-aligned-offset-in-bytes-of-variable> <space>
151	<length-of-var-in-bytes> ){n} "
152
153	where '(...){n}' means the content inside the parenthesis occurs 'n'
154	times, with 'n' being the number of variables on the stack.
155
156	3/ The following 8 bytes contain the PC of the current function which
157	will be used by the run-time library to print an error message.
158
159	4/ The following 8 bytes are reserved for internal use by the run-time.
160
161	The shadow memory for that stack layout is going to look like this:
162
163	- content of shadow memory 8 bytes for slot 7: 0xF1F1F1F1.
164	The F1 byte pattern is a magic number called
165	ASAN_STACK_MAGIC_LEFT and is a way for the runtime to know that
166	the memory for that shadow byte is part of a the LEFT red zone
167	intended to seat at the bottom of the variables on the stack.
168
169	- content of shadow memory 8 bytes for slots 6 and 5:
170	0xF4F4F400. The F4 byte pattern is a magic number
171	called ASAN_STACK_MAGIC_PARTIAL. It flags the fact that the
172	memory region for this shadow byte is a PARTIAL red zone
173	intended to pad a variable A, so that the slot following
174	{A,padding} is 32 bytes aligned.
175
176	Note that the fact that the least significant byte of this
177	shadow memory content is 00 means that 8 bytes of its
178	corresponding memory (which corresponds to the memory of
179	variable 'b') is addressable.
180
181	- content of shadow memory 8 bytes for slot 4: 0xF2F2F2F2.
182	The F2 byte pattern is a magic number called
183	ASAN_STACK_MAGIC_MIDDLE. It flags the fact that the memory
184	region for this shadow byte is a MIDDLE red zone intended to
185	seat between two 32 aligned slots of {variable,padding}.
186
187	- content of shadow memory 8 bytes for slot 3 and 2:
188	0xF4000000. This represents is the concatenation of
189	variable 'a' and the partial red zone following it, like what we
190	had for variable 'b'. The least significant 3 bytes being 00
191	means that the 3 bytes of variable 'a' are addressable.
192
193	- content of shadow memory 8 bytes for slot 1: 0xF3F3F3F3.
194	The F3 byte pattern is a magic number called
195	ASAN_STACK_MAGIC_RIGHT. It flags the fact that the memory
196	region for this shadow byte is a RIGHT red zone intended to seat
197	at the top of the variables of the stack.
198
199	Note that the real variable layout is done in expand_used_vars in
200	cfgexpand.cc. As far as Address Sanitizer is concerned, it lays out
201	stack variables as well as the different red zones, emits some
202	prologue code to populate the shadow memory as to poison (mark as
203	non-accessible) the regions of the red zones and mark the regions of
204	stack variables as accessible, and emit some epilogue code to
205	un-poison (mark as accessible) the regions of red zones right before
206	the function exits.
207
208	[Protection of global variables]
209
210	The basic idea is to insert a red zone between two global variables
211	and install a constructor function that calls the asan runtime to do
212	the populating of the relevant shadow memory regions at load time.
213
214	So the global variables are laid out as to insert a red zone between
215	them. The size of the red zones is so that each variable starts on a
216	32 bytes boundary.
217
218	Then a constructor function is installed so that, for each global
219	variable, it calls the runtime asan library function
220	__asan_register_globals_with an instance of this type:
221
222	struct __asan_global
223	{
224	// Address of the beginning of the global variable.
225	const void *__beg;
226
227	// Initial size of the global variable.
228	uptr __size;
229
230	// Size of the global variable + size of the red zone. This
231	// size is 32 bytes aligned.
232	uptr __size_with_redzone;
233
234	// Name of the global variable.
235	const void *__name;
236
237	// Name of the module where the global variable is declared.
238	const void *__module_name;
239
240	// 1 if it has dynamic initialization, 0 otherwise.
241	uptr __has_dynamic_init;
242
243	// A pointer to struct that contains source location, could be NULL.
244	__asan_global_source_location *__location;
245	}
246
247	A destructor function that calls the runtime asan library function
248	_asan_unregister_globals is also installed. */
249
250	static unsigned HOST_WIDE_INT asan_shadow_offset_value;
251	static bool asan_shadow_offset_computed;
252	static vec<char *> sanitized_sections;
253	static tree last_alloca_addr;
254
255	/* Set of variable declarations that are going to be guarded by
256	use-after-scope sanitizer. */
257
258	hash_set<tree> *asan_handled_variables = NULL;
259
260	hash_set <tree> *asan_used_labels = NULL;
261
262	/* Global variables for HWASAN stack tagging. */
263	/* hwasan_frame_tag_offset records the offset from the frame base tag that the
264	next object should have. */
265	static uint8_t hwasan_frame_tag_offset = 0;
266	/* hwasan_frame_base_ptr is a pointer with the same address as
267	`virtual_stack_vars_rtx` for the current frame, and with the frame base tag
268	stored in it. N.b. this global RTX does not need to be marked GTY, but is
269	done so anyway. The need is not there since all uses are in just one pass
270	(cfgexpand) and there are no calls to ggc_collect between the uses. We mark
271	it GTY(()) anyway to allow the use of the variable later on if needed by
272	future features. */
273	static GTY(()) rtx hwasan_frame_base_ptr = NULL_RTX;
274	/* hwasan_frame_base_init_seq is the sequence of RTL insns that will initialize
275	the hwasan_frame_base_ptr. When the hwasan_frame_base_ptr is requested, we
276	generate this sequence but do not emit it. If the sequence was created it
277	is emitted once the function body has been expanded.
278
279	This delay is because the frame base pointer may be needed anywhere in the
280	function body, or needed by the expand_used_vars function. Emitting once in
281	a known place is simpler than requiring the emission of the instructions to
282	be know where it should go depending on the first place the hwasan frame
283	base is needed. */
284	static GTY(()) rtx_insn *hwasan_frame_base_init_seq = NULL;
285
286	/* Structure defining the extent of one object on the stack that HWASAN needs
287	to tag in the corresponding shadow stack space.
288
289	The range this object spans on the stack is between `untagged_base +
290	nearest_offset` and `untagged_base + farthest_offset`.
291	`tagged_base` is an rtx containing the same value as `untagged_base` but
292	with a random tag stored in the top byte. We record both `untagged_base`
293	and `tagged_base` so that `hwasan_emit_prologue` can use both without having
294	to emit RTL into the instruction stream to re-calculate one from the other.
295	(`hwasan_emit_prologue` needs to use both bases since the
296	__hwasan_tag_memory call it emits uses an untagged value, and it calculates
297	the tag to store in shadow memory based on the tag_offset plus the tag in
298	tagged_base). */
299	struct hwasan_stack_var
300	{
301	rtx untagged_base;
302	rtx tagged_base;
303	poly_int64 nearest_offset;
304	poly_int64 farthest_offset;
305	uint8_t tag_offset;
306	};
307
308	/* Variable recording all stack variables that HWASAN needs to tag.
309	Does not need to be marked as GTY(()) since every use is in the cfgexpand
310	pass and gcc_collect is not called in the middle of that pass. */
311	static vec<hwasan_stack_var> hwasan_tagged_stack_vars;
312
313
314	/* Sets shadow offset to value in string VAL. */
315
316	bool
317	set_asan_shadow_offset (const char *val)
318	{
319	char *endp;
320
321	errno = 0;
322	#ifdef HAVE_LONG_LONG
323	asan_shadow_offset_value = strtoull (nptr: val, endptr: &endp, base: 0);
324	#else
325	asan_shadow_offset_value = strtoul (val, &endp, 0);
326	#endif
327	if (!(*val != '\0' && *endp == '\0' && errno == 0))
328	return false;
329
330	asan_shadow_offset_computed = true;
331
332	return true;
333	}
334
335	/* Set list of user-defined sections that need to be sanitized. */
336
337	void
338	set_sanitized_sections (const char *sections)
339	{
340	char *pat;
341	unsigned i;
342	FOR_EACH_VEC_ELT (sanitized_sections, i, pat)
343	free (ptr: pat);
344	sanitized_sections.truncate (size: 0);
345
346	for (const char *s = sections; *s; )
347	{
348	const char *end;
349	for (end = s; *end && *end != ','; ++end);
350	size_t len = end - s;
351	sanitized_sections.safe_push (obj: xstrndup (s, len));
352	s = *end ? end + 1 : end;
353	}
354	}
355
356	bool
357	asan_mark_p (gimple *stmt, enum asan_mark_flags flag)
358	{
359	return (gimple_call_internal_p (gs: stmt, fn: IFN_ASAN_MARK)
360	&& tree_to_uhwi (gimple_call_arg (gs: stmt, index: 0)) == flag);
361	}
362
363	bool
364	asan_sanitize_stack_p (void)
365	{
366	return (sanitize_flags_p (flag: SANITIZE_ADDRESS) && param_asan_stack);
367	}
368
369	bool
370	asan_sanitize_allocas_p (void)
371	{
372	return (asan_sanitize_stack_p () && param_asan_protect_allocas);
373	}
374
375	bool
376	asan_instrument_reads (void)
377	{
378	return (sanitize_flags_p (flag: SANITIZE_ADDRESS) && param_asan_instrument_reads);
379	}
380
381	bool
382	asan_instrument_writes (void)
383	{
384	return (sanitize_flags_p (flag: SANITIZE_ADDRESS) && param_asan_instrument_writes);
385	}
386
387	bool
388	asan_memintrin (void)
389	{
390	return (sanitize_flags_p (flag: SANITIZE_ADDRESS) && param_asan_memintrin);
391	}
392
393
394	/* Support for --param asan-kernel-mem-intrinsic-prefix=1. */
395	static GTY(()) rtx asan_memfn_rtls[3];
396
397	rtx
398	asan_memfn_rtl (tree fndecl)
399	{
400	int i;
401	const char *f, *p;
402	char buf[sizeof ("__hwasan_memmove")];
403
404	switch (DECL_FUNCTION_CODE (decl: fndecl))
405	{
406	case BUILT_IN_MEMCPY: i = 0; f = "memcpy"; break;
407	case BUILT_IN_MEMSET: i = 1; f = "memset"; break;
408	case BUILT_IN_MEMMOVE: i = 2; f = "memmove"; break;
409	default: gcc_unreachable ();
410	}
411	if (asan_memfn_rtls[i] == NULL_RTX)
412	{
413	tree save_name = DECL_NAME (fndecl);
414	tree save_assembler_name = DECL_ASSEMBLER_NAME (fndecl);
415	rtx save_rtl = DECL_RTL (fndecl);
416	if (flag_sanitize & SANITIZE_KERNEL_HWADDRESS)
417	p = "__hwasan_";
418	else
419	p = "__asan_";
420	strcpy (dest: buf, src: p);
421	strcat (dest: buf, src: f);
422	DECL_NAME (fndecl) = get_identifier (buf);
423	DECL_ASSEMBLER_NAME_RAW (fndecl) = NULL_TREE;
424	SET_DECL_RTL (fndecl, NULL_RTX);
425	asan_memfn_rtls[i] = DECL_RTL (fndecl);
426	DECL_NAME (fndecl) = save_name;
427	DECL_ASSEMBLER_NAME_RAW (fndecl) = save_assembler_name;
428	SET_DECL_RTL (fndecl, save_rtl);
429	}
430	return asan_memfn_rtls[i];
431	}
432
433
434	/* Checks whether section SEC should be sanitized. */
435
436	static bool
437	section_sanitized_p (const char *sec)
438	{
439	char *pat;
440	unsigned i;
441	FOR_EACH_VEC_ELT (sanitized_sections, i, pat)
442	if (fnmatch (pattern: pat, string: sec, FNM_PERIOD) == 0)
443	return true;
444	return false;
445	}
446
447	/* Returns Asan shadow offset. */
448
449	static unsigned HOST_WIDE_INT
450	asan_shadow_offset ()
451	{
452	if (!asan_shadow_offset_computed)
453	{
454	asan_shadow_offset_computed = true;
455	asan_shadow_offset_value = targetm.asan_shadow_offset ();
456	}
457	return asan_shadow_offset_value;
458	}
459
460	/* Returns Asan shadow offset has been set. */
461	bool
462	asan_shadow_offset_set_p ()
463	{
464	return asan_shadow_offset_computed;
465	}
466
467	alias_set_type asan_shadow_set = -1;
468
469	/* Pointer types to 1, 2 or 4 byte integers in shadow memory. A separate
470	alias set is used for all shadow memory accesses. */
471	static GTY(()) tree shadow_ptr_types[3];
472
473	/* Decl for __asan_option_detect_stack_use_after_return. */
474	static GTY(()) tree asan_detect_stack_use_after_return;
475
476	/* Hashtable support for memory references used by gimple
477	statements. */
478
479	/* This type represents a reference to a memory region. */
480	struct asan_mem_ref
481	{
482	/* The expression of the beginning of the memory region. */
483	tree start;
484
485	/* The size of the access. */
486	HOST_WIDE_INT access_size;
487	};
488
489	object_allocator <asan_mem_ref> asan_mem_ref_pool ("asan_mem_ref");
490
491	/* Initializes an instance of asan_mem_ref. */
492
493	static void
494	asan_mem_ref_init (asan_mem_ref *ref, tree start, HOST_WIDE_INT access_size)
495	{
496	ref->start = start;
497	ref->access_size = access_size;
498	}
499
500	/* Allocates memory for an instance of asan_mem_ref into the memory
501	pool returned by asan_mem_ref_get_alloc_pool and initialize it.
502	START is the address of (or the expression pointing to) the
503	beginning of memory reference. ACCESS_SIZE is the size of the
504	access to the referenced memory. */
505
506	static asan_mem_ref*
507	asan_mem_ref_new (tree start, HOST_WIDE_INT access_size)
508	{
509	asan_mem_ref *ref = asan_mem_ref_pool.allocate ();
510
511	asan_mem_ref_init (ref, start, access_size);
512	return ref;
513	}
514
515	/* This builds and returns a pointer to the end of the memory region
516	that starts at START and of length LEN. */
517
518	tree
519	asan_mem_ref_get_end (tree start, tree len)
520	{
521	if (len == NULL_TREE || integer_zerop (len))
522	return start;
523
524	if (!ptrofftype_p (type: len))
525	len = convert_to_ptrofftype (len);
526
527	return fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (start), start, len);
528	}
529
530	/* Return a tree expression that represents the end of the referenced
531	memory region. Beware that this function can actually build a new
532	tree expression. */
533
534	tree
535	asan_mem_ref_get_end (const asan_mem_ref *ref, tree len)
536	{
537	return asan_mem_ref_get_end (start: ref->start, len);
538	}
539
540	struct asan_mem_ref_hasher : nofree_ptr_hash <asan_mem_ref>
541	{
542	static inline hashval_t hash (const asan_mem_ref *);
543	static inline bool equal (const asan_mem_ref *, const asan_mem_ref *);
544	};
545
546	/* Hash a memory reference. */
547
548	inline hashval_t
549	asan_mem_ref_hasher::hash (const asan_mem_ref *mem_ref)
550	{
551	return iterative_hash_expr (tree: mem_ref->start, seed: 0);
552	}
553
554	/* Compare two memory references. We accept the length of either
555	memory references to be NULL_TREE. */
556
557	inline bool
558	asan_mem_ref_hasher::equal (const asan_mem_ref *m1,
559	const asan_mem_ref *m2)
560	{
561	return operand_equal_p (m1->start, m2->start, flags: 0);
562	}
563
564	static hash_table<asan_mem_ref_hasher> *asan_mem_ref_ht;
565
566	/* Returns a reference to the hash table containing memory references.
567	This function ensures that the hash table is created. Note that
568	this hash table is updated by the function
569	update_mem_ref_hash_table. */
570
571	static hash_table<asan_mem_ref_hasher> *
572	get_mem_ref_hash_table ()
573	{
574	if (!asan_mem_ref_ht)
575	asan_mem_ref_ht = new hash_table<asan_mem_ref_hasher> (10);
576
577	return asan_mem_ref_ht;
578	}
579
580	/* Clear all entries from the memory references hash table. */
581
582	static void
583	empty_mem_ref_hash_table ()
584	{
585	if (asan_mem_ref_ht)
586	asan_mem_ref_ht->empty ();
587	}
588
589	/* Free the memory references hash table. */
590
591	static void
592	free_mem_ref_resources ()
593	{
594	delete asan_mem_ref_ht;
595	asan_mem_ref_ht = NULL;
596
597	asan_mem_ref_pool.release ();
598	}
599
600	/* Return true iff the memory reference REF has been instrumented. */
601
602	static bool
603	has_mem_ref_been_instrumented (tree ref, HOST_WIDE_INT access_size)
604	{
605	asan_mem_ref r;
606	asan_mem_ref_init (ref: &r, start: ref, access_size);
607
608	asan_mem_ref *saved_ref = get_mem_ref_hash_table ()->find (value: &r);
609	return saved_ref && saved_ref->access_size >= access_size;
610	}
611
612	/* Return true iff the memory reference REF has been instrumented. */
613
614	static bool
615	has_mem_ref_been_instrumented (const asan_mem_ref *ref)
616	{
617	return has_mem_ref_been_instrumented (ref: ref->start, access_size: ref->access_size);
618	}
619
620	/* Return true iff access to memory region starting at REF and of
621	length LEN has been instrumented. */
622
623	static bool
624	has_mem_ref_been_instrumented (const asan_mem_ref *ref, tree len)
625	{
626	HOST_WIDE_INT size_in_bytes
627	= tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
628
629	return size_in_bytes != -1
630	&& has_mem_ref_been_instrumented (ref: ref->start, access_size: size_in_bytes);
631	}
632
633	/* Set REF to the memory reference present in a gimple assignment
634	ASSIGNMENT. Return true upon successful completion, false
635	otherwise. */
636
637	static bool
638	get_mem_ref_of_assignment (const gassign *assignment,
639	asan_mem_ref *ref,
640	bool *ref_is_store)
641	{
642	gcc_assert (gimple_assign_single_p (assignment));
643
644	if (gimple_store_p (gs: assignment)
645	&& !gimple_clobber_p (s: assignment))
646	{
647	ref->start = gimple_assign_lhs (gs: assignment);
648	*ref_is_store = true;
649	}
650	else if (gimple_assign_load_p (assignment))
651	{
652	ref->start = gimple_assign_rhs1 (gs: assignment);
653	*ref_is_store = false;
654	}
655	else
656	return false;
657
658	ref->access_size = int_size_in_bytes (TREE_TYPE (ref->start));
659	return true;
660	}
661
662	/* Return address of last allocated dynamic alloca. */
663
664	static tree
665	get_last_alloca_addr ()
666	{
667	if (last_alloca_addr)
668	return last_alloca_addr;
669
670	last_alloca_addr = create_tmp_reg (ptr_type_node, "last_alloca_addr");
671	gassign *g = gimple_build_assign (last_alloca_addr, null_pointer_node);
672	edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
673	gsi_insert_on_edge_immediate (e, g);
674	return last_alloca_addr;
675	}
676
677	/* Insert __asan_allocas_unpoison (top, bottom) call before
678	__builtin_stack_restore (new_sp) call.
679	The pseudocode of this routine should look like this:
680	top = last_alloca_addr;
681	bot = new_sp;
682	__asan_allocas_unpoison (top, bot);
683	last_alloca_addr = new_sp;
684	__builtin_stack_restore (new_sp);
685	In general, we can't use new_sp as bot parameter because on some
686	architectures SP has non zero offset from dynamic stack area. Moreover, on
687	some architectures this offset (STACK_DYNAMIC_OFFSET) becomes known for each
688	particular function only after all callees were expanded to rtl.
689	The most noticeable example is PowerPC{,64}, see
690	http://refspecs.linuxfoundation.org/ELF/ppc64/PPC-elf64abi.html#DYNAM-STACK.
691	To overcome the issue we use following trick: pass new_sp as a second
692	parameter to __asan_allocas_unpoison and rewrite it during expansion with
693	new_sp + (virtual_dynamic_stack_rtx - sp) later in
694	expand_asan_emit_allocas_unpoison function.
695
696	HWASAN needs to do very similar, the eventual pseudocode should be:
697	__hwasan_tag_memory (virtual_stack_dynamic_rtx,
698	0,
699	new_sp - sp);
700	__builtin_stack_restore (new_sp)
701
702	Need to use the same trick to handle STACK_DYNAMIC_OFFSET as described
703	above. */
704
705	static void
706	handle_builtin_stack_restore (gcall *call, gimple_stmt_iterator *iter)
707	{
708	if (!iter
709	|| !(asan_sanitize_allocas_p () || hwasan_sanitize_allocas_p ()))
710	return;
711
712	tree restored_stack = gimple_call_arg (gs: call, index: 0);
713
714	gimple *g;
715
716	if (hwasan_sanitize_allocas_p ())
717	{
718	enum internal_fn fn = IFN_HWASAN_ALLOCA_UNPOISON;
719	/* There is only one piece of information `expand_HWASAN_ALLOCA_UNPOISON`
720	needs to work. This is the length of the area that we're
721	deallocating. Since the stack pointer is known at expand time, the
722	position of the new stack pointer after deallocation is enough
723	information to calculate this length. */
724	g = gimple_build_call_internal (fn, 1, restored_stack);
725	}
726	else
727	{
728	tree last_alloca = get_last_alloca_addr ();
729	tree fn = builtin_decl_implicit (fncode: BUILT_IN_ASAN_ALLOCAS_UNPOISON);
730	g = gimple_build_call (fn, 2, last_alloca, restored_stack);
731	gsi_insert_before (iter, g, GSI_SAME_STMT);
732	g = gimple_build_assign (last_alloca, restored_stack);
733	}
734
735	gsi_insert_before (iter, g, GSI_SAME_STMT);
736	}
737
738	/* Deploy and poison redzones around __builtin_alloca call. To do this, we
739	should replace this call with another one with changed parameters and
740	replace all its uses with new address, so
741	addr = __builtin_alloca (old_size, align);
742	is replaced by
743	left_redzone_size = max (align, ASAN_RED_ZONE_SIZE);
744	Following two statements are optimized out if we know that
745	old_size & (ASAN_RED_ZONE_SIZE - 1) == 0, i.e. alloca doesn't need partial
746	redzone.
747	misalign = old_size & (ASAN_RED_ZONE_SIZE - 1);
748	partial_redzone_size = ASAN_RED_ZONE_SIZE - misalign;
749	right_redzone_size = ASAN_RED_ZONE_SIZE;
750	additional_size = left_redzone_size + partial_redzone_size +
751	right_redzone_size;
752	new_size = old_size + additional_size;
753	new_alloca = __builtin_alloca (new_size, max (align, 32))
754	__asan_alloca_poison (new_alloca, old_size)
755	addr = new_alloca + max (align, ASAN_RED_ZONE_SIZE);
756	last_alloca_addr = new_alloca;
757	ADDITIONAL_SIZE is added to make new memory allocation contain not only
758	requested memory, but also left, partial and right redzones as well as some
759	additional space, required by alignment. */
760
761	static void
762	handle_builtin_alloca (gcall *call, gimple_stmt_iterator *iter)
763	{
764	if (!iter
765	|| !(asan_sanitize_allocas_p () || hwasan_sanitize_allocas_p ()))
766	return;
767
768	gassign *g;
769	gcall *gg;
770	tree callee = gimple_call_fndecl (gs: call);
771	tree lhs = gimple_call_lhs (gs: call);
772	tree old_size = gimple_call_arg (gs: call, index: 0);
773	tree ptr_type = lhs ? TREE_TYPE (lhs) : ptr_type_node;
774	tree partial_size = NULL_TREE;
775	unsigned int align
776	= DECL_FUNCTION_CODE (decl: callee) == BUILT_IN_ALLOCA
777	? 0 : tree_to_uhwi (gimple_call_arg (gs: call, index: 1));
778
779	bool throws = false;
780	edge e = NULL;
781	if (stmt_can_throw_internal (cfun, call))
782	{
783	if (!lhs)
784	return;
785	throws = true;
786	e = find_fallthru_edge (edges: gsi_bb (i: *iter)->succs);
787	}
788
789	if (hwasan_sanitize_allocas_p ())
790	{
791	gimple_seq stmts = NULL;
792	location_t loc = gimple_location (g: gsi_stmt (i: *iter));
793	/*
794	HWASAN needs a different expansion.
795
796	addr = __builtin_alloca (size, align);
797
798	should be replaced by
799
800	new_size = size rounded up to HWASAN_TAG_GRANULE_SIZE byte alignment;
801	untagged_addr = __builtin_alloca (new_size, align);
802	tag = __hwasan_choose_alloca_tag ();
803	addr = ifn_HWASAN_SET_TAG (untagged_addr, tag);
804	__hwasan_tag_memory (untagged_addr, tag, new_size);
805	*/
806	/* Ensure alignment at least HWASAN_TAG_GRANULE_SIZE bytes so we start on
807	a tag granule. */
808	align = align > HWASAN_TAG_GRANULE_SIZE ? align : HWASAN_TAG_GRANULE_SIZE;
809
810	tree old_size = gimple_call_arg (gs: call, index: 0);
811	tree new_size = gimple_build_round_up (seq: &stmts, loc, size_type_node,
812	old_size,
813	HWASAN_TAG_GRANULE_SIZE);
814
815	/* Make the alloca call */
816	tree untagged_addr
817	= gimple_build (seq: &stmts, loc,
818	fn: as_combined_fn (fn: BUILT_IN_ALLOCA_WITH_ALIGN), type: ptr_type,
819	args: new_size, args: build_int_cst (size_type_node, align));
820
821	/* Choose the tag.
822	Here we use an internal function so we can choose the tag at expand
823	time. We need the decision to be made after stack variables have been
824	assigned their tag (i.e. once the hwasan_frame_tag_offset variable has
825	been set to one after the last stack variables tag). */
826	tree tag = gimple_build (seq: &stmts, loc, fn: CFN_HWASAN_CHOOSE_TAG,
827	unsigned_char_type_node);
828
829	/* Add tag to pointer. */
830	tree addr
831	= gimple_build (seq: &stmts, loc, fn: CFN_HWASAN_SET_TAG, type: ptr_type,
832	args: untagged_addr, args: tag);
833
834	/* Tag shadow memory.
835	NOTE: require using `untagged_addr` here for libhwasan API. */
836	gimple_build (seq: &stmts, loc, fn: as_combined_fn (fn: BUILT_IN_HWASAN_TAG_MEM),
837	void_type_node, args: untagged_addr, args: tag, args: new_size);
838
839	/* Insert the built up code sequence into the original instruction stream
840	the iterator points to. */
841	gsi_insert_seq_before (iter, stmts, GSI_SAME_STMT);
842
843	/* Finally, replace old alloca ptr with NEW_ALLOCA. */
844	replace_call_with_value (iter, addr);
845	return;
846	}
847
848	tree last_alloca = get_last_alloca_addr ();
849	const HOST_WIDE_INT redzone_mask = ASAN_RED_ZONE_SIZE - 1;
850
851	/* If ALIGN > ASAN_RED_ZONE_SIZE, we embed left redzone into first ALIGN
852	bytes of allocated space. Otherwise, align alloca to ASAN_RED_ZONE_SIZE
853	manually. */
854	align = MAX (align, ASAN_RED_ZONE_SIZE * BITS_PER_UNIT);
855
856	tree alloca_rz_mask = build_int_cst (size_type_node, redzone_mask);
857	tree redzone_size = build_int_cst (size_type_node, ASAN_RED_ZONE_SIZE);
858
859	/* Extract lower bits from old_size. */
860	wide_int size_nonzero_bits = get_nonzero_bits (old_size);
861	wide_int rz_mask
862	= wi::uhwi (val: redzone_mask, precision: wi::get_precision (x: size_nonzero_bits));
863	wide_int old_size_lower_bits = wi::bit_and (x: size_nonzero_bits, y: rz_mask);
864
865	/* If alloca size is aligned to ASAN_RED_ZONE_SIZE, we don't need partial
866	redzone. Otherwise, compute its size here. */
867	if (wi::ne_p (x: old_size_lower_bits, y: 0))
868	{
869	/* misalign = size & (ASAN_RED_ZONE_SIZE - 1)
870	partial_size = ASAN_RED_ZONE_SIZE - misalign. */
871	g = gimple_build_assign (make_ssa_name (size_type_node, NULL),
872	BIT_AND_EXPR, old_size, alloca_rz_mask);
873	gsi_insert_before (iter, g, GSI_SAME_STMT);
874	tree misalign = gimple_assign_lhs (gs: g);
875	g = gimple_build_assign (make_ssa_name (size_type_node, NULL), MINUS_EXPR,
876	redzone_size, misalign);
877	gsi_insert_before (iter, g, GSI_SAME_STMT);
878	partial_size = gimple_assign_lhs (gs: g);
879	}
880
881	/* additional_size = align + ASAN_RED_ZONE_SIZE. */
882	tree additional_size = build_int_cst (size_type_node, align / BITS_PER_UNIT
883	+ ASAN_RED_ZONE_SIZE);
884	/* If alloca has partial redzone, include it to additional_size too. */
885	if (partial_size)
886	{
887	/* additional_size += partial_size. */
888	g = gimple_build_assign (make_ssa_name (size_type_node), PLUS_EXPR,
889	partial_size, additional_size);
890	gsi_insert_before (iter, g, GSI_SAME_STMT);
891	additional_size = gimple_assign_lhs (gs: g);
892	}
893
894	/* new_size = old_size + additional_size. */
895	g = gimple_build_assign (make_ssa_name (size_type_node), PLUS_EXPR, old_size,
896	additional_size);
897	gsi_insert_before (iter, g, GSI_SAME_STMT);
898	tree new_size = gimple_assign_lhs (gs: g);
899
900	/* Build new __builtin_alloca call:
901	new_alloca_with_rz = __builtin_alloca (new_size, align). */
902	tree fn = builtin_decl_implicit (fncode: BUILT_IN_ALLOCA_WITH_ALIGN);
903	gg = gimple_build_call (fn, 2, new_size,
904	build_int_cst (size_type_node, align));
905	tree new_alloca_with_rz = make_ssa_name (var: ptr_type, stmt: gg);
906	gimple_call_set_lhs (gs: gg, lhs: new_alloca_with_rz);
907	if (throws)
908	{
909	gimple_call_set_lhs (gs: call, NULL);
910	gsi_replace (iter, gg, true);
911	}
912	else
913	gsi_insert_before (iter, gg, GSI_SAME_STMT);
914
915	/* new_alloca = new_alloca_with_rz + align. */
916	g = gimple_build_assign (make_ssa_name (var: ptr_type), POINTER_PLUS_EXPR,
917	new_alloca_with_rz,
918	build_int_cst (size_type_node,
919	align / BITS_PER_UNIT));
920	gimple_stmt_iterator gsi = gsi_none ();
921	if (throws)
922	{
923	gsi_insert_on_edge_immediate (e, g);
924	gsi = gsi_for_stmt (g);
925	}
926	else
927	gsi_insert_before (iter, g, GSI_SAME_STMT);
928	tree new_alloca = gimple_assign_lhs (gs: g);
929
930	/* Poison newly created alloca redzones:
931	__asan_alloca_poison (new_alloca, old_size). */
932	fn = builtin_decl_implicit (fncode: BUILT_IN_ASAN_ALLOCA_POISON);
933	gg = gimple_build_call (fn, 2, new_alloca, old_size);
934	if (throws)
935	gsi_insert_after (&gsi, gg, GSI_NEW_STMT);
936	else
937	gsi_insert_before (iter, gg, GSI_SAME_STMT);
938
939	/* Save new_alloca_with_rz value into last_alloca to use it during
940	allocas unpoisoning. */
941	g = gimple_build_assign (last_alloca, new_alloca_with_rz);
942	if (throws)
943	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
944	else
945	gsi_insert_before (iter, g, GSI_SAME_STMT);
946
947	/* Finally, replace old alloca ptr with NEW_ALLOCA. */
948	if (throws)
949	{
950	g = gimple_build_assign (lhs, new_alloca);
951	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
952	}
953	else
954	replace_call_with_value (iter, new_alloca);
955	}
956
957	/* Return the memory references contained in a gimple statement
958	representing a builtin call that has to do with memory access. */
959
960	static bool
961	get_mem_refs_of_builtin_call (gcall *call,
962	asan_mem_ref *src0,
963	tree *src0_len,
964	bool *src0_is_store,
965	asan_mem_ref *src1,
966	tree *src1_len,
967	bool *src1_is_store,
968	asan_mem_ref *dst,
969	tree *dst_len,
970	bool *dst_is_store,
971	bool *dest_is_deref,
972	bool *intercepted_p,
973	gimple_stmt_iterator *iter = NULL)
974	{
975	gcc_checking_assert (gimple_call_builtin_p (call, BUILT_IN_NORMAL));
976
977	tree callee = gimple_call_fndecl (gs: call);
978	tree source0 = NULL_TREE, source1 = NULL_TREE,
979	dest = NULL_TREE, len = NULL_TREE;
980	bool is_store = true, got_reference_p = false;
981	HOST_WIDE_INT access_size = 1;
982
983	*intercepted_p = asan_intercepted_p (fcode: (DECL_FUNCTION_CODE (decl: callee)));
984
985	switch (DECL_FUNCTION_CODE (decl: callee))
986	{
987	/* (s, s, n) style memops. */
988	case BUILT_IN_BCMP:
989	case BUILT_IN_MEMCMP:
990	source0 = gimple_call_arg (gs: call, index: 0);
991	source1 = gimple_call_arg (gs: call, index: 1);
992	len = gimple_call_arg (gs: call, index: 2);
993	break;
994
995	/* (src, dest, n) style memops. */
996	case BUILT_IN_BCOPY:
997	source0 = gimple_call_arg (gs: call, index: 0);
998	dest = gimple_call_arg (gs: call, index: 1);
999	len = gimple_call_arg (gs: call, index: 2);
1000	break;
1001
1002	/* (dest, src, n) style memops. */
1003	case BUILT_IN_MEMCPY:
1004	case BUILT_IN_MEMCPY_CHK:
1005	case BUILT_IN_MEMMOVE:
1006	case BUILT_IN_MEMMOVE_CHK:
1007	case BUILT_IN_MEMPCPY:
1008	case BUILT_IN_MEMPCPY_CHK:
1009	dest = gimple_call_arg (gs: call, index: 0);
1010	source0 = gimple_call_arg (gs: call, index: 1);
1011	len = gimple_call_arg (gs: call, index: 2);
1012	break;
1013
1014	/* (dest, n) style memops. */
1015	case BUILT_IN_BZERO:
1016	dest = gimple_call_arg (gs: call, index: 0);
1017	len = gimple_call_arg (gs: call, index: 1);
1018	break;
1019
1020	/* (dest, x, n) style memops*/
1021	case BUILT_IN_MEMSET:
1022	case BUILT_IN_MEMSET_CHK:
1023	dest = gimple_call_arg (gs: call, index: 0);
1024	len = gimple_call_arg (gs: call, index: 2);
1025	break;
1026
1027	case BUILT_IN_STRLEN:
1028	/* Special case strlen here since its length is taken from its return
1029	value.
1030
1031	The approach taken by the sanitizers is to check a memory access
1032	before it's taken. For ASAN strlen is intercepted by libasan, so no
1033	check is inserted by the compiler.
1034
1035	This function still returns `true` and provides a length to the rest
1036	of the ASAN pass in order to record what areas have been checked,
1037	avoiding superfluous checks later on.
1038
1039	HWASAN does not intercept any of these internal functions.
1040	This means that checks for memory accesses must be inserted by the
1041	compiler.
1042	strlen is a special case, because we can tell the length from the
1043	return of the function, but that is not known until after the function
1044	has returned.
1045
1046	Hence we can't check the memory access before it happens.
1047	We could check the memory access after it has already happened, but
1048	for now we choose to just ignore `strlen` calls.
1049	This decision was simply made because that means the special case is
1050	limited to this one case of this one function. */
1051	if (hwasan_sanitize_p ())
1052	return false;
1053	source0 = gimple_call_arg (gs: call, index: 0);
1054	len = gimple_call_lhs (gs: call);
1055	break;
1056
1057	case BUILT_IN_STACK_RESTORE:
1058	handle_builtin_stack_restore (call, iter);
1059	break;
1060
1061	CASE_BUILT_IN_ALLOCA:
1062	handle_builtin_alloca (call, iter);
1063	break;
1064	/* And now the __atomic* and __sync builtins.
1065	These are handled differently from the classical memory
1066	access builtins above. */
1067
1068	case BUILT_IN_ATOMIC_LOAD_1:
1069	is_store = false;
1070	/* FALLTHRU */
1071	case BUILT_IN_SYNC_FETCH_AND_ADD_1:
1072	case BUILT_IN_SYNC_FETCH_AND_SUB_1:
1073	case BUILT_IN_SYNC_FETCH_AND_OR_1:
1074	case BUILT_IN_SYNC_FETCH_AND_AND_1:
1075	case BUILT_IN_SYNC_FETCH_AND_XOR_1:
1076	case BUILT_IN_SYNC_FETCH_AND_NAND_1:
1077	case BUILT_IN_SYNC_ADD_AND_FETCH_1:
1078	case BUILT_IN_SYNC_SUB_AND_FETCH_1:
1079	case BUILT_IN_SYNC_OR_AND_FETCH_1:
1080	case BUILT_IN_SYNC_AND_AND_FETCH_1:
1081	case BUILT_IN_SYNC_XOR_AND_FETCH_1:
1082	case BUILT_IN_SYNC_NAND_AND_FETCH_1:
1083	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_1:
1084	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_1:
1085	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_1:
1086	case BUILT_IN_SYNC_LOCK_RELEASE_1:
1087	case BUILT_IN_ATOMIC_EXCHANGE_1:
1088	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1:
1089	case BUILT_IN_ATOMIC_STORE_1:
1090	case BUILT_IN_ATOMIC_ADD_FETCH_1:
1091	case BUILT_IN_ATOMIC_SUB_FETCH_1:
1092	case BUILT_IN_ATOMIC_AND_FETCH_1:
1093	case BUILT_IN_ATOMIC_NAND_FETCH_1:
1094	case BUILT_IN_ATOMIC_XOR_FETCH_1:
1095	case BUILT_IN_ATOMIC_OR_FETCH_1:
1096	case BUILT_IN_ATOMIC_FETCH_ADD_1:
1097	case BUILT_IN_ATOMIC_FETCH_SUB_1:
1098	case BUILT_IN_ATOMIC_FETCH_AND_1:
1099	case BUILT_IN_ATOMIC_FETCH_NAND_1:
1100	case BUILT_IN_ATOMIC_FETCH_XOR_1:
1101	case BUILT_IN_ATOMIC_FETCH_OR_1:
1102	access_size = 1;
1103	goto do_atomic;
1104
1105	case BUILT_IN_ATOMIC_LOAD_2:
1106	is_store = false;
1107	/* FALLTHRU */
1108	case BUILT_IN_SYNC_FETCH_AND_ADD_2:
1109	case BUILT_IN_SYNC_FETCH_AND_SUB_2:
1110	case BUILT_IN_SYNC_FETCH_AND_OR_2:
1111	case BUILT_IN_SYNC_FETCH_AND_AND_2:
1112	case BUILT_IN_SYNC_FETCH_AND_XOR_2:
1113	case BUILT_IN_SYNC_FETCH_AND_NAND_2:
1114	case BUILT_IN_SYNC_ADD_AND_FETCH_2:
1115	case BUILT_IN_SYNC_SUB_AND_FETCH_2:
1116	case BUILT_IN_SYNC_OR_AND_FETCH_2:
1117	case BUILT_IN_SYNC_AND_AND_FETCH_2:
1118	case BUILT_IN_SYNC_XOR_AND_FETCH_2:
1119	case BUILT_IN_SYNC_NAND_AND_FETCH_2:
1120	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_2:
1121	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_2:
1122	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_2:
1123	case BUILT_IN_SYNC_LOCK_RELEASE_2:
1124	case BUILT_IN_ATOMIC_EXCHANGE_2:
1125	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_2:
1126	case BUILT_IN_ATOMIC_STORE_2:
1127	case BUILT_IN_ATOMIC_ADD_FETCH_2:
1128	case BUILT_IN_ATOMIC_SUB_FETCH_2:
1129	case BUILT_IN_ATOMIC_AND_FETCH_2:
1130	case BUILT_IN_ATOMIC_NAND_FETCH_2:
1131	case BUILT_IN_ATOMIC_XOR_FETCH_2:
1132	case BUILT_IN_ATOMIC_OR_FETCH_2:
1133	case BUILT_IN_ATOMIC_FETCH_ADD_2:
1134	case BUILT_IN_ATOMIC_FETCH_SUB_2:
1135	case BUILT_IN_ATOMIC_FETCH_AND_2:
1136	case BUILT_IN_ATOMIC_FETCH_NAND_2:
1137	case BUILT_IN_ATOMIC_FETCH_XOR_2:
1138	case BUILT_IN_ATOMIC_FETCH_OR_2:
1139	access_size = 2;
1140	goto do_atomic;
1141
1142	case BUILT_IN_ATOMIC_LOAD_4:
1143	is_store = false;
1144	/* FALLTHRU */
1145	case BUILT_IN_SYNC_FETCH_AND_ADD_4:
1146	case BUILT_IN_SYNC_FETCH_AND_SUB_4:
1147	case BUILT_IN_SYNC_FETCH_AND_OR_4:
1148	case BUILT_IN_SYNC_FETCH_AND_AND_4:
1149	case BUILT_IN_SYNC_FETCH_AND_XOR_4:
1150	case BUILT_IN_SYNC_FETCH_AND_NAND_4:
1151	case BUILT_IN_SYNC_ADD_AND_FETCH_4:
1152	case BUILT_IN_SYNC_SUB_AND_FETCH_4:
1153	case BUILT_IN_SYNC_OR_AND_FETCH_4:
1154	case BUILT_IN_SYNC_AND_AND_FETCH_4:
1155	case BUILT_IN_SYNC_XOR_AND_FETCH_4:
1156	case BUILT_IN_SYNC_NAND_AND_FETCH_4:
1157	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_4:
1158	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_4:
1159	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_4:
1160	case BUILT_IN_SYNC_LOCK_RELEASE_4:
1161	case BUILT_IN_ATOMIC_EXCHANGE_4:
1162	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_4:
1163	case BUILT_IN_ATOMIC_STORE_4:
1164	case BUILT_IN_ATOMIC_ADD_FETCH_4:
1165	case BUILT_IN_ATOMIC_SUB_FETCH_4:
1166	case BUILT_IN_ATOMIC_AND_FETCH_4:
1167	case BUILT_IN_ATOMIC_NAND_FETCH_4:
1168	case BUILT_IN_ATOMIC_XOR_FETCH_4:
1169	case BUILT_IN_ATOMIC_OR_FETCH_4:
1170	case BUILT_IN_ATOMIC_FETCH_ADD_4:
1171	case BUILT_IN_ATOMIC_FETCH_SUB_4:
1172	case BUILT_IN_ATOMIC_FETCH_AND_4:
1173	case BUILT_IN_ATOMIC_FETCH_NAND_4:
1174	case BUILT_IN_ATOMIC_FETCH_XOR_4:
1175	case BUILT_IN_ATOMIC_FETCH_OR_4:
1176	access_size = 4;
1177	goto do_atomic;
1178
1179	case BUILT_IN_ATOMIC_LOAD_8:
1180	is_store = false;
1181	/* FALLTHRU */
1182	case BUILT_IN_SYNC_FETCH_AND_ADD_8:
1183	case BUILT_IN_SYNC_FETCH_AND_SUB_8:
1184	case BUILT_IN_SYNC_FETCH_AND_OR_8:
1185	case BUILT_IN_SYNC_FETCH_AND_AND_8:
1186	case BUILT_IN_SYNC_FETCH_AND_XOR_8:
1187	case BUILT_IN_SYNC_FETCH_AND_NAND_8:
1188	case BUILT_IN_SYNC_ADD_AND_FETCH_8:
1189	case BUILT_IN_SYNC_SUB_AND_FETCH_8:
1190	case BUILT_IN_SYNC_OR_AND_FETCH_8:
1191	case BUILT_IN_SYNC_AND_AND_FETCH_8:
1192	case BUILT_IN_SYNC_XOR_AND_FETCH_8:
1193	case BUILT_IN_SYNC_NAND_AND_FETCH_8:
1194	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_8:
1195	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_8:
1196	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_8:
1197	case BUILT_IN_SYNC_LOCK_RELEASE_8:
1198	case BUILT_IN_ATOMIC_EXCHANGE_8:
1199	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_8:
1200	case BUILT_IN_ATOMIC_STORE_8:
1201	case BUILT_IN_ATOMIC_ADD_FETCH_8:
1202	case BUILT_IN_ATOMIC_SUB_FETCH_8:
1203	case BUILT_IN_ATOMIC_AND_FETCH_8:
1204	case BUILT_IN_ATOMIC_NAND_FETCH_8:
1205	case BUILT_IN_ATOMIC_XOR_FETCH_8:
1206	case BUILT_IN_ATOMIC_OR_FETCH_8:
1207	case BUILT_IN_ATOMIC_FETCH_ADD_8:
1208	case BUILT_IN_ATOMIC_FETCH_SUB_8:
1209	case BUILT_IN_ATOMIC_FETCH_AND_8:
1210	case BUILT_IN_ATOMIC_FETCH_NAND_8:
1211	case BUILT_IN_ATOMIC_FETCH_XOR_8:
1212	case BUILT_IN_ATOMIC_FETCH_OR_8:
1213	access_size = 8;
1214	goto do_atomic;
1215
1216	case BUILT_IN_ATOMIC_LOAD_16:
1217	is_store = false;
1218	/* FALLTHRU */
1219	case BUILT_IN_SYNC_FETCH_AND_ADD_16:
1220	case BUILT_IN_SYNC_FETCH_AND_SUB_16:
1221	case BUILT_IN_SYNC_FETCH_AND_OR_16:
1222	case BUILT_IN_SYNC_FETCH_AND_AND_16:
1223	case BUILT_IN_SYNC_FETCH_AND_XOR_16:
1224	case BUILT_IN_SYNC_FETCH_AND_NAND_16:
1225	case BUILT_IN_SYNC_ADD_AND_FETCH_16:
1226	case BUILT_IN_SYNC_SUB_AND_FETCH_16:
1227	case BUILT_IN_SYNC_OR_AND_FETCH_16:
1228	case BUILT_IN_SYNC_AND_AND_FETCH_16:
1229	case BUILT_IN_SYNC_XOR_AND_FETCH_16:
1230	case BUILT_IN_SYNC_NAND_AND_FETCH_16:
1231	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_16:
1232	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_16:
1233	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_16:
1234	case BUILT_IN_SYNC_LOCK_RELEASE_16:
1235	case BUILT_IN_ATOMIC_EXCHANGE_16:
1236	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_16:
1237	case BUILT_IN_ATOMIC_STORE_16:
1238	case BUILT_IN_ATOMIC_ADD_FETCH_16:
1239	case BUILT_IN_ATOMIC_SUB_FETCH_16:
1240	case BUILT_IN_ATOMIC_AND_FETCH_16:
1241	case BUILT_IN_ATOMIC_NAND_FETCH_16:
1242	case BUILT_IN_ATOMIC_XOR_FETCH_16:
1243	case BUILT_IN_ATOMIC_OR_FETCH_16:
1244	case BUILT_IN_ATOMIC_FETCH_ADD_16:
1245	case BUILT_IN_ATOMIC_FETCH_SUB_16:
1246	case BUILT_IN_ATOMIC_FETCH_AND_16:
1247	case BUILT_IN_ATOMIC_FETCH_NAND_16:
1248	case BUILT_IN_ATOMIC_FETCH_XOR_16:
1249	case BUILT_IN_ATOMIC_FETCH_OR_16:
1250	access_size = 16;
1251	/* FALLTHRU */
1252	do_atomic:
1253	{
1254	dest = gimple_call_arg (gs: call, index: 0);
1255	/* DEST represents the address of a memory location.
1256	instrument_derefs wants the memory location, so lets
1257	dereference the address DEST before handing it to
1258	instrument_derefs. */
1259	tree type = build_nonstandard_integer_type (access_size
1260	* BITS_PER_UNIT, 1);
1261	dest = build2 (MEM_REF, type, dest,
1262	build_int_cst (build_pointer_type (char_type_node), 0));
1263	break;
1264	}
1265
1266	default:
1267	/* The other builtins memory access are not instrumented in this
1268	function because they either don't have any length parameter,
1269	or their length parameter is just a limit. */
1270	break;
1271	}
1272
1273	if (len != NULL_TREE)
1274	{
1275	if (source0 != NULL_TREE)
1276	{
1277	src0->start = source0;
1278	src0->access_size = access_size;
1279	*src0_len = len;
1280	*src0_is_store = false;
1281	}
1282
1283	if (source1 != NULL_TREE)
1284	{
1285	src1->start = source1;
1286	src1->access_size = access_size;
1287	*src1_len = len;
1288	*src1_is_store = false;
1289	}
1290
1291	if (dest != NULL_TREE)
1292	{
1293	dst->start = dest;
1294	dst->access_size = access_size;
1295	*dst_len = len;
1296	*dst_is_store = true;
1297	}
1298
1299	got_reference_p = true;
1300	}
1301	else if (dest)
1302	{
1303	dst->start = dest;
1304	dst->access_size = access_size;
1305	*dst_len = NULL_TREE;
1306	*dst_is_store = is_store;
1307	*dest_is_deref = true;
1308	got_reference_p = true;
1309	}
1310
1311	return got_reference_p;
1312	}
1313
1314	/* Return true iff a given gimple statement has been instrumented.
1315	Note that the statement is "defined" by the memory references it
1316	contains. */
1317
1318	static bool
1319	has_stmt_been_instrumented_p (gimple *stmt)
1320	{
1321	if (gimple_assign_single_p (gs: stmt))
1322	{
1323	bool r_is_store;
1324	asan_mem_ref r;
1325	asan_mem_ref_init (ref: &r, NULL, access_size: 1);
1326
1327	if (get_mem_ref_of_assignment (assignment: as_a <gassign *> (p: stmt), ref: &r,
1328	ref_is_store: &r_is_store))
1329	{
1330	if (!has_mem_ref_been_instrumented (ref: &r))
1331	return false;
1332	if (r_is_store && gimple_assign_load_p (stmt))
1333	{
1334	asan_mem_ref src;
1335	asan_mem_ref_init (ref: &src, NULL, access_size: 1);
1336	src.start = gimple_assign_rhs1 (gs: stmt);
1337	src.access_size = int_size_in_bytes (TREE_TYPE (src.start));
1338	if (!has_mem_ref_been_instrumented (ref: &src))
1339	return false;
1340	}
1341	return true;
1342	}
1343	}
1344	else if (gimple_call_builtin_p (stmt, BUILT_IN_NORMAL))
1345	{
1346	asan_mem_ref src0, src1, dest;
1347	asan_mem_ref_init (ref: &src0, NULL, access_size: 1);
1348	asan_mem_ref_init (ref: &src1, NULL, access_size: 1);
1349	asan_mem_ref_init (ref: &dest, NULL, access_size: 1);
1350
1351	tree src0_len = NULL_TREE, src1_len = NULL_TREE, dest_len = NULL_TREE;
1352	bool src0_is_store = false, src1_is_store = false,
1353	dest_is_store = false, dest_is_deref = false, intercepted_p = true;
1354	if (get_mem_refs_of_builtin_call (call: as_a <gcall *> (p: stmt),
1355	src0: &src0, src0_len: &src0_len, src0_is_store: &src0_is_store,
1356	src1: &src1, src1_len: &src1_len, src1_is_store: &src1_is_store,
1357	dst: &dest, dst_len: &dest_len, dst_is_store: &dest_is_store,
1358	dest_is_deref: &dest_is_deref, intercepted_p: &intercepted_p))
1359	{
1360	if (src0.start != NULL_TREE
1361	&& !has_mem_ref_been_instrumented (ref: &src0, len: src0_len))
1362	return false;
1363
1364	if (src1.start != NULL_TREE
1365	&& !has_mem_ref_been_instrumented (ref: &src1, len: src1_len))
1366	return false;
1367
1368	if (dest.start != NULL_TREE
1369	&& !has_mem_ref_been_instrumented (ref: &dest, len: dest_len))
1370	return false;
1371
1372	return true;
1373	}
1374	}
1375	else if (is_gimple_call (gs: stmt)
1376	&& gimple_store_p (gs: stmt)
1377	&& (gimple_call_builtin_p (stmt)
1378	|| gimple_call_internal_p (gs: stmt)
1379	|| !aggregate_value_p (TREE_TYPE (gimple_call_lhs (stmt)),
1380	gimple_call_fntype (gs: stmt))))
1381	{
1382	asan_mem_ref r;
1383	asan_mem_ref_init (ref: &r, NULL, access_size: 1);
1384
1385	r.start = gimple_call_lhs (gs: stmt);
1386	r.access_size = int_size_in_bytes (TREE_TYPE (r.start));
1387	return has_mem_ref_been_instrumented (ref: &r);
1388	}
1389
1390	return false;
1391	}
1392
1393	/* Insert a memory reference into the hash table. */
1394
1395	static void
1396	update_mem_ref_hash_table (tree ref, HOST_WIDE_INT access_size)
1397	{
1398	hash_table<asan_mem_ref_hasher> *ht = get_mem_ref_hash_table ();
1399
1400	asan_mem_ref r;
1401	asan_mem_ref_init (ref: &r, start: ref, access_size);
1402
1403	asan_mem_ref **slot = ht->find_slot (value: &r, insert: INSERT);
1404	if (*slot == NULL || (*slot)->access_size < access_size)
1405	*slot = asan_mem_ref_new (start: ref, access_size);
1406	}
1407
1408	/* Initialize shadow_ptr_types array. */
1409
1410	static void
1411	asan_init_shadow_ptr_types (void)
1412	{
1413	asan_shadow_set = new_alias_set ();
1414	tree types[3] = { signed_char_type_node, short_integer_type_node,
1415	integer_type_node };
1416
1417	for (unsigned i = 0; i < 3; i++)
1418	{
1419	shadow_ptr_types[i] = build_distinct_type_copy (types[i]);
1420	TYPE_ALIAS_SET (shadow_ptr_types[i]) = asan_shadow_set;
1421	shadow_ptr_types[i] = build_pointer_type (shadow_ptr_types[i]);
1422	}
1423
1424	initialize_sanitizer_builtins ();
1425	}
1426
1427	/* Create ADDR_EXPR of STRING_CST with the PP pretty printer text. */
1428
1429	static tree
1430	asan_pp_string (pretty_printer *pp)
1431	{
1432	const char *buf = pp_formatted_text (pp);
1433	size_t len = strlen (s: buf);
1434	tree ret = build_string (len + 1, buf);
1435	TREE_TYPE (ret)
1436	= build_array_type (TREE_TYPE (shadow_ptr_types[0]),
1437	build_index_type (size_int (len)));
1438	TREE_READONLY (ret) = 1;
1439	TREE_STATIC (ret) = 1;
1440	return build1 (ADDR_EXPR, shadow_ptr_types[0], ret);
1441	}
1442
1443	/* Clear shadow memory at SHADOW_MEM, LEN bytes. Can't call a library call here
1444	though. */
1445
1446	static void
1447	asan_clear_shadow (rtx shadow_mem, HOST_WIDE_INT len)
1448	{
1449	rtx_insn *insn, *insns, *jump;
1450	rtx_code_label *top_label;
1451	rtx end, addr, tmp;
1452
1453	gcc_assert ((len & 3) == 0);
1454	start_sequence ();
1455	clear_storage (shadow_mem, GEN_INT (len), BLOCK_OP_NORMAL);
1456	insns = get_insns ();
1457	end_sequence ();
1458	for (insn = insns; insn; insn = NEXT_INSN (insn))
1459	if (CALL_P (insn))
1460	break;
1461	if (insn == NULL_RTX)
1462	{
1463	emit_insn (insns);
1464	return;
1465	}
1466
1467	top_label = gen_label_rtx ();
1468	addr = copy_to_mode_reg (Pmode, XEXP (shadow_mem, 0));
1469	shadow_mem = adjust_automodify_address (shadow_mem, SImode, addr, 0);
1470	end = force_reg (Pmode, plus_constant (Pmode, addr, len));
1471	emit_label (top_label);
1472
1473	emit_move_insn (shadow_mem, const0_rtx);
1474	tmp = expand_simple_binop (Pmode, PLUS, addr, gen_int_mode (4, Pmode), addr,
1475	true, OPTAB_LIB_WIDEN);
1476	if (tmp != addr)
1477	emit_move_insn (addr, tmp);
1478	emit_cmp_and_jump_insns (addr, end, LT, NULL_RTX, Pmode, true, top_label);
1479	jump = get_last_insn ();
1480	gcc_assert (JUMP_P (jump));
1481	add_reg_br_prob_note (jump,
1482	profile_probability::guessed_always ()
1483	.apply_scale (num: 80, den: 100));
1484	}
1485
1486	void
1487	asan_function_start (void)
1488	{
1489	ASM_OUTPUT_DEBUG_LABEL (asm_out_file, "LASANPC", current_function_funcdef_no);
1490	}
1491
1492	/* Return number of shadow bytes that are occupied by a local variable
1493	of SIZE bytes. */
1494
1495	static unsigned HOST_WIDE_INT
1496	shadow_mem_size (unsigned HOST_WIDE_INT size)
1497	{
1498	/* It must be possible to align stack variables to granularity
1499	of shadow memory. */
1500	gcc_assert (BITS_PER_UNIT
1501	* ASAN_SHADOW_GRANULARITY <= MAX_SUPPORTED_STACK_ALIGNMENT);
1502
1503	return ROUND_UP (size, ASAN_SHADOW_GRANULARITY) / ASAN_SHADOW_GRANULARITY;
1504	}
1505
1506	/* Always emit 4 bytes at a time. */
1507	#define RZ_BUFFER_SIZE 4
1508
1509	/* ASAN redzone buffer container that handles emission of shadow bytes. */
1510	class asan_redzone_buffer
1511	{
1512	public:
1513	/* Constructor. */
1514	asan_redzone_buffer (rtx shadow_mem, HOST_WIDE_INT prev_offset):
1515	m_shadow_mem (shadow_mem), m_prev_offset (prev_offset),
1516	m_original_offset (prev_offset), m_shadow_bytes (RZ_BUFFER_SIZE)
1517	{}
1518
1519	/* Emit VALUE shadow byte at a given OFFSET. */
1520	void emit_redzone_byte (HOST_WIDE_INT offset, unsigned char value);
1521
1522	/* Emit RTX emission of the content of the buffer. */
1523	void flush_redzone_payload (void);
1524
1525	private:
1526	/* Flush if the content of the buffer is full
1527	(equal to RZ_BUFFER_SIZE). */
1528	void flush_if_full (void);
1529
1530	/* Memory where we last emitted a redzone payload. */
1531	rtx m_shadow_mem;
1532
1533	/* Relative offset where we last emitted a redzone payload. */
1534	HOST_WIDE_INT m_prev_offset;
1535
1536	/* Relative original offset. Used for checking only. */
1537	HOST_WIDE_INT m_original_offset;
1538
1539	public:
1540	/* Buffer with redzone payload. */
1541	auto_vec<unsigned char> m_shadow_bytes;
1542	};
1543
1544	/* Emit VALUE shadow byte at a given OFFSET. */
1545
1546	void
1547	asan_redzone_buffer::emit_redzone_byte (HOST_WIDE_INT offset,
1548	unsigned char value)
1549	{
1550	gcc_assert ((offset & (ASAN_SHADOW_GRANULARITY - 1)) == 0);
1551	gcc_assert (offset >= m_prev_offset);
1552
1553	HOST_WIDE_INT off
1554	= m_prev_offset + ASAN_SHADOW_GRANULARITY * m_shadow_bytes.length ();
1555	if (off == offset)
1556	/* Consecutive shadow memory byte. */;
1557	else if (offset < m_prev_offset + (HOST_WIDE_INT) (ASAN_SHADOW_GRANULARITY
1558	* RZ_BUFFER_SIZE)
1559	&& !m_shadow_bytes.is_empty ())
1560	{
1561	/* Shadow memory byte with a small gap. */
1562	for (; off < offset; off += ASAN_SHADOW_GRANULARITY)
1563	m_shadow_bytes.safe_push (obj: 0);
1564	}
1565	else
1566	{
1567	if (!m_shadow_bytes.is_empty ())
1568	flush_redzone_payload ();
1569
1570	/* Maybe start earlier in order to use aligned store. */
1571	HOST_WIDE_INT align = (offset - m_prev_offset) % ASAN_RED_ZONE_SIZE;
1572	if (align)
1573	{
1574	offset -= align;
1575	for (unsigned i = 0; i < align / BITS_PER_UNIT; i++)
1576	m_shadow_bytes.safe_push (obj: 0);
1577	}
1578
1579	/* Adjust m_prev_offset and m_shadow_mem. */
1580	HOST_WIDE_INT diff = offset - m_prev_offset;
1581	m_shadow_mem = adjust_address (m_shadow_mem, VOIDmode,
1582	diff >> ASAN_SHADOW_SHIFT);
1583	m_prev_offset = offset;
1584	}
1585	m_shadow_bytes.safe_push (obj: value);
1586	flush_if_full ();
1587	}
1588
1589	/* Emit RTX emission of the content of the buffer. */
1590
1591	void
1592	asan_redzone_buffer::flush_redzone_payload (void)
1593	{
1594	gcc_assert (WORDS_BIG_ENDIAN == BYTES_BIG_ENDIAN);
1595
1596	if (m_shadow_bytes.is_empty ())
1597	return;
1598
1599	/* Be sure we always emit to an aligned address. */
1600	gcc_assert (((m_prev_offset - m_original_offset)
1601	& (ASAN_RED_ZONE_SIZE - 1)) == 0);
1602
1603	/* Fill it to RZ_BUFFER_SIZE bytes with zeros if needed. */
1604	unsigned l = m_shadow_bytes.length ();
1605	for (unsigned i = 0; i <= RZ_BUFFER_SIZE - l; i++)
1606	m_shadow_bytes.safe_push (obj: 0);
1607
1608	if (dump_file && (dump_flags & TDF_DETAILS))
1609	fprintf (stream: dump_file,
1610	format: "Flushing rzbuffer at offset %" PRId64 " with: ", m_prev_offset);
1611
1612	unsigned HOST_WIDE_INT val = 0;
1613	for (unsigned i = 0; i < RZ_BUFFER_SIZE; i++)
1614	{
1615	unsigned char v
1616	= m_shadow_bytes[BYTES_BIG_ENDIAN ? RZ_BUFFER_SIZE - i - 1 : i];
1617	val |= (unsigned HOST_WIDE_INT)v << (BITS_PER_UNIT * i);
1618	if (dump_file && (dump_flags & TDF_DETAILS))
1619	fprintf (stream: dump_file, format: "%02x ", v);
1620	}
1621
1622	if (dump_file && (dump_flags & TDF_DETAILS))
1623	fprintf (stream: dump_file, format: "\n");
1624
1625	rtx c = gen_int_mode (val, SImode);
1626	m_shadow_mem = adjust_address (m_shadow_mem, SImode, 0);
1627	emit_move_insn (m_shadow_mem, c);
1628	m_shadow_bytes.truncate (size: 0);
1629	}
1630
1631	/* Flush if the content of the buffer is full
1632	(equal to RZ_BUFFER_SIZE). */
1633
1634	void
1635	asan_redzone_buffer::flush_if_full (void)
1636	{
1637	if (m_shadow_bytes.length () == RZ_BUFFER_SIZE)
1638	flush_redzone_payload ();
1639	}
1640
1641
1642	/* HWAddressSanitizer (hwasan) is a probabilistic method for detecting
1643	out-of-bounds and use-after-free bugs.
1644	Read more:
1645	http://code.google.com/p/address-sanitizer/
1646
1647	Similar to AddressSanitizer (asan) it consists of two parts: the
1648	instrumentation module in this file, and a run-time library.
1649
1650	The instrumentation module adds a run-time check before every memory insn in
1651	the same manner as asan (see the block comment for AddressSanitizer above).
1652	Currently, hwasan only adds out-of-line instrumentation, where each check is
1653	implemented as a function call to the run-time library. Hence a check for a
1654	load of N bytes from address X would be implemented with a function call to
1655	__hwasan_loadN(X), and checking a store of N bytes from address X would be
1656	implemented with a function call to __hwasan_storeN(X).
1657
1658	The main difference between hwasan and asan is in the information stored to
1659	help this checking. Both sanitizers use a shadow memory area which stores
1660	data recording the state of main memory at a corresponding address.
1661
1662	For hwasan, each 16 byte granule in main memory has a corresponding 1 byte
1663	in shadow memory. This shadow address can be calculated with equation:
1664	(addr >> log_2(HWASAN_TAG_GRANULE_SIZE))
1665	+ __hwasan_shadow_memory_dynamic_address;
1666	The conversion between real and shadow memory for asan is given in the block
1667	comment at the top of this file.
1668	The description of how this shadow memory is laid out for asan is in the
1669	block comment at the top of this file, here we describe how this shadow
1670	memory is used for hwasan.
1671
1672	For hwasan, each variable is assigned a byte-sized 'tag'. The extent of
1673	the shadow memory for that variable is filled with the assigned tag, and
1674	every pointer referencing that variable has its top byte set to the same
1675	tag. The run-time library redefines malloc so that every allocation returns
1676	a tagged pointer and tags the corresponding shadow memory with the same tag.
1677
1678	On each pointer dereference the tag found in the pointer is compared to the
1679	tag found in the shadow memory corresponding to the accessed memory address.
1680	If these tags are found to differ then this memory access is judged to be
1681	invalid and a report is generated.
1682
1683	This method of bug detection is not perfect -- it can not catch every bad
1684	access -- but catches them probabilistically instead. There is always the
1685	possibility that an invalid memory access will happen to access memory
1686	tagged with the same tag as the pointer that this access used.
1687	The chances of this are approx. 0.4% for any two uncorrelated objects.
1688
1689	Random tag generation can mitigate this problem by decreasing the
1690	probability that an invalid access will be missed in the same manner over
1691	multiple runs. i.e. if two objects are tagged the same in one run of the
1692	binary they are unlikely to be tagged the same in the next run.
1693	Both heap and stack allocated objects have random tags by default.
1694
1695	[16 byte granule implications]
1696	Since the shadow memory only has a resolution on real memory of 16 bytes,
1697	invalid accesses that are within the same 16 byte granule as a valid
1698	address will not be caught.
1699
1700	There is a "short-granule" feature in the runtime library which does catch
1701	such accesses, but this feature is not implemented for stack objects (since
1702	stack objects are allocated and tagged by compiler instrumentation, and
1703	this feature has not yet been implemented in GCC instrumentation).
1704
1705	Another outcome of this 16 byte resolution is that each tagged object must
1706	be 16 byte aligned. If two objects were to share any 16 byte granule in
1707	memory, then they both would have to be given the same tag, and invalid
1708	accesses to one using a pointer to the other would be undetectable.
1709
1710	[Compiler instrumentation]
1711	Compiler instrumentation ensures that two adjacent buffers on the stack are
1712	given different tags, this means an access to one buffer using a pointer
1713	generated from the other (e.g. through buffer overrun) will have mismatched
1714	tags and be caught by hwasan.
1715
1716	We don't randomly tag every object on the stack, since that would require
1717	keeping many registers to record each tag. Instead we randomly generate a
1718	tag for each function frame, and each new stack object uses a tag offset
1719	from that frame tag.
1720	i.e. each object is tagged as RFT + offset, where RFT is the "random frame
1721	tag" generated for this frame.
1722	This means that randomisation does not peturb the difference between tags
1723	on tagged stack objects within a frame, but this is mitigated by the fact
1724	that objects with the same tag within a frame are very far apart
1725	(approx. 2^HWASAN_TAG_SIZE objects apart).
1726
1727	As a demonstration, using the same example program as in the asan block
1728	comment above:
1729
1730	int
1731	foo ()
1732	{
1733	char a[24] = {0};
1734	int b[2] = {0};
1735
1736	a[5] = 1;
1737	b[1] = 2;
1738
1739	return a[5] + b[1];
1740	}
1741
1742	On AArch64 the stack will be ordered as follows for the above function:
1743
1744	Slot 1/ [24 bytes for variable 'a']
1745	Slot 2/ [8 bytes padding for alignment]
1746	Slot 3/ [8 bytes for variable 'b']
1747	Slot 4/ [8 bytes padding for alignment]
1748
1749	(The padding is there to ensure 16 byte alignment as described in the 16
1750	byte granule implications).
1751
1752	While the shadow memory will be ordered as follows:
1753
1754	- 2 bytes (representing 32 bytes in real memory) tagged with RFT + 1.
1755	- 1 byte (representing 16 bytes in real memory) tagged with RFT + 2.
1756
1757	And any pointer to "a" will have the tag RFT + 1, and any pointer to "b"
1758	will have the tag RFT + 2.
1759
1760	[Top Byte Ignore requirements]
1761	Hwasan requires the ability to store an 8 bit tag in every pointer. There
1762	is no instrumentation done to remove this tag from pointers before
1763	dereferencing, which means the hardware must ignore this tag during memory
1764	accesses.
1765
1766	Architectures where this feature is available should indicate this using
1767	the TARGET_MEMTAG_CAN_TAG_ADDRESSES hook.
1768
1769	[Stack requires cleanup on unwinding]
1770	During normal operation of a hwasan sanitized program more space in the
1771	shadow memory becomes tagged as the stack grows. As the stack shrinks this
1772	shadow memory space must become untagged. If it is not untagged then when
1773	the stack grows again (during other function calls later on in the program)
1774	objects on the stack that are usually not tagged (e.g. parameters passed on
1775	the stack) can be placed in memory whose shadow space is tagged with
1776	something else, and accesses can cause false positive reports.
1777
1778	Hence we place untagging code on every epilogue of functions which tag some
1779	stack objects.
1780
1781	Moreover, the run-time library intercepts longjmp & setjmp to untag when
1782	the stack is unwound this way.
1783
1784	C++ exceptions are not yet handled, which means this sanitizer can not
1785	handle C++ code that throws exceptions -- it will give false positives
1786	after an exception has been thrown. The implementation that the hwasan
1787	library has for handling these relies on the frame pointer being after any
1788	local variables. This is not generally the case for GCC. */
1789
1790
1791	/* Returns whether we are tagging pointers and checking those tags on memory
1792	access. */
1793	bool
1794	hwasan_sanitize_p ()
1795	{
1796	return sanitize_flags_p (flag: SANITIZE_HWADDRESS);
1797	}
1798
1799	/* Are we tagging the stack? */
1800	bool
1801	hwasan_sanitize_stack_p ()
1802	{
1803	return (hwasan_sanitize_p () && param_hwasan_instrument_stack);
1804	}
1805
1806	/* Are we tagging alloca objects? */
1807	bool
1808	hwasan_sanitize_allocas_p (void)
1809	{
1810	return (hwasan_sanitize_stack_p () && param_hwasan_instrument_allocas);
1811	}
1812
1813	/* Should we instrument reads? */
1814	bool
1815	hwasan_instrument_reads (void)
1816	{
1817	return (hwasan_sanitize_p () && param_hwasan_instrument_reads);
1818	}
1819
1820	/* Should we instrument writes? */
1821	bool
1822	hwasan_instrument_writes (void)
1823	{
1824	return (hwasan_sanitize_p () && param_hwasan_instrument_writes);
1825	}
1826
1827	/* Should we instrument builtin calls? */
1828	bool
1829	hwasan_memintrin (void)
1830	{
1831	return (hwasan_sanitize_p () && param_hwasan_instrument_mem_intrinsics);
1832	}
1833
1834	/* Insert code to protect stack vars. The prologue sequence should be emitted
1835	directly, epilogue sequence returned. BASE is the register holding the
1836	stack base, against which OFFSETS array offsets are relative to, OFFSETS
1837	array contains pairs of offsets in reverse order, always the end offset
1838	of some gap that needs protection followed by starting offset,
1839	and DECLS is an array of representative decls for each var partition.
1840	LENGTH is the length of the OFFSETS array, DECLS array is LENGTH / 2 - 1
1841	elements long (OFFSETS include gap before the first variable as well
1842	as gaps after each stack variable). PBASE is, if non-NULL, some pseudo
1843	register which stack vars DECL_RTLs are based on. Either BASE should be
1844	assigned to PBASE, when not doing use after return protection, or
1845	corresponding address based on __asan_stack_malloc* return value. */
1846
1847	rtx_insn *
1848	asan_emit_stack_protection (rtx base, rtx pbase, unsigned int alignb,
1849	HOST_WIDE_INT *offsets, tree *decls, int length)
1850	{
1851	rtx shadow_base, shadow_mem, ret, mem, orig_base;
1852	rtx_code_label *lab;
1853	rtx_insn *insns;
1854	char buf[32];
1855	HOST_WIDE_INT base_offset = offsets[length - 1];
1856	HOST_WIDE_INT base_align_bias = 0, offset, prev_offset;
1857	HOST_WIDE_INT asan_frame_size = offsets[0] - base_offset;
1858	HOST_WIDE_INT last_offset, last_size, last_size_aligned;
1859	int l;
1860	unsigned char cur_shadow_byte = ASAN_STACK_MAGIC_LEFT;
1861	tree str_cst, decl, id;
1862	int use_after_return_class = -1;
1863
1864	/* Don't emit anything when doing error recovery, the assertions
1865	might fail e.g. if a function had a frame offset overflow. */
1866	if (seen_error ())
1867	return NULL;
1868
1869	if (shadow_ptr_types[0] == NULL_TREE)
1870	asan_init_shadow_ptr_types ();
1871
1872	expanded_location cfun_xloc
1873	= expand_location (DECL_SOURCE_LOCATION (current_function_decl));
1874
1875	/* First of all, prepare the description string. */
1876	pretty_printer asan_pp;
1877
1878	pp_decimal_int (&asan_pp, length / 2 - 1);
1879	pp_space (&asan_pp);
1880	for (l = length - 2; l; l -= 2)
1881	{
1882	tree decl = decls[l / 2 - 1];
1883	pp_wide_integer (pp: &asan_pp, i: offsets[l] - base_offset);
1884	pp_space (&asan_pp);
1885	pp_wide_integer (pp: &asan_pp, i: offsets[l - 1] - offsets[l]);
1886	pp_space (&asan_pp);
1887
1888	expanded_location xloc
1889	= expand_location (DECL_SOURCE_LOCATION (decl));
1890	char location[32];
1891
1892	if (xloc.file == cfun_xloc.file)
1893	sprintf (s: location, format: ":%d", xloc.line);
1894	else
1895	location[0] = '\0';
1896
1897	if (DECL_P (decl) && DECL_NAME (decl))
1898	{
1899	unsigned idlen
1900	= IDENTIFIER_LENGTH (DECL_NAME (decl)) + strlen (s: location);
1901	pp_decimal_int (&asan_pp, idlen);
1902	pp_space (&asan_pp);
1903	pp_tree_identifier (&asan_pp, DECL_NAME (decl));
1904	pp_string (&asan_pp, location);
1905	}
1906	else
1907	pp_string (&asan_pp, "9 <unknown>");
1908
1909	if (l > 2)
1910	pp_space (&asan_pp);
1911	}
1912	str_cst = asan_pp_string (pp: &asan_pp);
1913
1914	gcc_checking_assert (offsets[0] == (crtl->stack_protect_guard
1915	? -ASAN_RED_ZONE_SIZE : 0));
1916	/* Emit the prologue sequence. */
1917	if (asan_frame_size > 32 && asan_frame_size <= 65536 && pbase
1918	&& param_asan_use_after_return)
1919	{
1920	HOST_WIDE_INT adjusted_frame_size = asan_frame_size;
1921	/* The stack protector guard is allocated at the top of the frame
1922	and cfgexpand.cc then uses align_frame_offset (ASAN_RED_ZONE_SIZE);
1923	while in that case we can still use asan_frame_size, we need to take
1924	that into account when computing base_align_bias. */
1925	if (alignb > ASAN_RED_ZONE_SIZE && crtl->stack_protect_guard)
1926	adjusted_frame_size += ASAN_RED_ZONE_SIZE;
1927	use_after_return_class = floor_log2 (x: asan_frame_size - 1) - 5;
1928	/* __asan_stack_malloc_N guarantees alignment
1929	N < 6 ? (64 << N) : 4096 bytes. */
1930	if (alignb > (use_after_return_class < 6
1931	? (64U << use_after_return_class) : 4096U))
1932	use_after_return_class = -1;
1933	else if (alignb > ASAN_RED_ZONE_SIZE
1934	&& (adjusted_frame_size & (alignb - 1)))
1935	{
1936	base_align_bias
1937	= ((adjusted_frame_size + alignb - 1)
1938	& ~(alignb - HOST_WIDE_INT_1)) - adjusted_frame_size;
1939	use_after_return_class
1940	= floor_log2 (x: asan_frame_size + base_align_bias - 1) - 5;
1941	if (use_after_return_class > 10)
1942	{
1943	base_align_bias = 0;
1944	use_after_return_class = -1;
1945	}
1946	}
1947	}
1948
1949	/* Align base if target is STRICT_ALIGNMENT. */
1950	if (STRICT_ALIGNMENT)
1951	{
1952	const HOST_WIDE_INT align
1953	= (GET_MODE_ALIGNMENT (SImode) / BITS_PER_UNIT) << ASAN_SHADOW_SHIFT;
1954	base = expand_binop (Pmode, and_optab, base, gen_int_mode (-align, Pmode),
1955	NULL_RTX, 1, OPTAB_DIRECT);
1956	}
1957
1958	if (use_after_return_class == -1 && pbase)
1959	emit_move_insn (pbase, base);
1960
1961	base = expand_binop (Pmode, add_optab, base,
1962	gen_int_mode (base_offset - base_align_bias, Pmode),
1963	NULL_RTX, 1, OPTAB_DIRECT);
1964	orig_base = NULL_RTX;
1965	if (use_after_return_class != -1)
1966	{
1967	if (asan_detect_stack_use_after_return == NULL_TREE)
1968	{
1969	id = get_identifier ("__asan_option_detect_stack_use_after_return");
1970	decl = build_decl (BUILTINS_LOCATION, VAR_DECL, id,
1971	integer_type_node);
1972	SET_DECL_ASSEMBLER_NAME (decl, id);
1973	TREE_ADDRESSABLE (decl) = 1;
1974	DECL_ARTIFICIAL (decl) = 1;
1975	DECL_IGNORED_P (decl) = 1;
1976	DECL_EXTERNAL (decl) = 1;
1977	TREE_STATIC (decl) = 1;
1978	TREE_PUBLIC (decl) = 1;
1979	TREE_USED (decl) = 1;
1980	asan_detect_stack_use_after_return = decl;
1981	}
1982	orig_base = gen_reg_rtx (Pmode);
1983	emit_move_insn (orig_base, base);
1984	ret = expand_normal (exp: asan_detect_stack_use_after_return);
1985	lab = gen_label_rtx ();
1986	emit_cmp_and_jump_insns (ret, const0_rtx, EQ, NULL_RTX,
1987	VOIDmode, 0, lab,
1988	prob: profile_probability::very_likely ());
1989	snprintf (s: buf, maxlen: sizeof buf, format: "__asan_stack_malloc_%d",
1990	use_after_return_class);
1991	ret = init_one_libfunc (buf);
1992	ret = emit_library_call_value (fun: ret, NULL_RTX, fn_type: LCT_NORMAL, outmode: ptr_mode,
1993	GEN_INT (asan_frame_size
1994	+ base_align_bias),
1995	TYPE_MODE (pointer_sized_int_node));
1996	/* __asan_stack_malloc_[n] returns a pointer to fake stack if succeeded
1997	and NULL otherwise. Check RET value is NULL here and jump over the
1998	BASE reassignment in this case. Otherwise, reassign BASE to RET. */
1999	emit_cmp_and_jump_insns (ret, const0_rtx, EQ, NULL_RTX,
2000	VOIDmode, 0, lab,
2001	prob: profile_probability:: very_unlikely ());
2002	ret = convert_memory_address (Pmode, ret);
2003	emit_move_insn (base, ret);
2004	emit_label (lab);
2005	emit_move_insn (pbase, expand_binop (Pmode, add_optab, base,
2006	gen_int_mode (base_align_bias
2007	- base_offset, Pmode),
2008	NULL_RTX, 1, OPTAB_DIRECT));
2009	}
2010	mem = gen_rtx_MEM (ptr_mode, base);
2011	mem = adjust_address (mem, VOIDmode, base_align_bias);
2012	emit_move_insn (mem, gen_int_mode (ASAN_STACK_FRAME_MAGIC, ptr_mode));
2013	mem = adjust_address (mem, VOIDmode, GET_MODE_SIZE (ptr_mode));
2014	emit_move_insn (mem, expand_normal (exp: str_cst));
2015	mem = adjust_address (mem, VOIDmode, GET_MODE_SIZE (ptr_mode));
2016	ASM_GENERATE_INTERNAL_LABEL (buf, "LASANPC", current_function_funcdef_no);
2017	id = get_identifier (buf);
2018	decl = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
2019	VAR_DECL, id, char_type_node);
2020	SET_DECL_ASSEMBLER_NAME (decl, id);
2021	TREE_ADDRESSABLE (decl) = 1;
2022	TREE_READONLY (decl) = 1;
2023	DECL_ARTIFICIAL (decl) = 1;
2024	DECL_IGNORED_P (decl) = 1;
2025	TREE_STATIC (decl) = 1;
2026	TREE_PUBLIC (decl) = 0;
2027	TREE_USED (decl) = 1;
2028	DECL_INITIAL (decl) = decl;
2029	TREE_ASM_WRITTEN (decl) = 1;
2030	TREE_ASM_WRITTEN (id) = 1;
2031	DECL_ALIGN_RAW (decl) = DECL_ALIGN_RAW (current_function_decl);
2032	emit_move_insn (mem, expand_normal (build_fold_addr_expr (decl)));
2033	shadow_base = expand_binop (Pmode, lshr_optab, base,
2034	gen_int_shift_amount (Pmode, ASAN_SHADOW_SHIFT),
2035	NULL_RTX, 1, OPTAB_DIRECT);
2036	shadow_base
2037	= plus_constant (Pmode, shadow_base,
2038	asan_shadow_offset ()
2039	+ (base_align_bias >> ASAN_SHADOW_SHIFT));
2040	gcc_assert (asan_shadow_set != -1
2041	&& (ASAN_RED_ZONE_SIZE >> ASAN_SHADOW_SHIFT) == 4);
2042	shadow_mem = gen_rtx_MEM (SImode, shadow_base);
2043	set_mem_alias_set (shadow_mem, asan_shadow_set);
2044	if (STRICT_ALIGNMENT)
2045	set_mem_align (shadow_mem, (GET_MODE_ALIGNMENT (SImode)));
2046	prev_offset = base_offset;
2047
2048	asan_redzone_buffer rz_buffer (shadow_mem, prev_offset);
2049	for (l = length; l; l -= 2)
2050	{
2051	if (l == 2)
2052	cur_shadow_byte = ASAN_STACK_MAGIC_RIGHT;
2053	offset = offsets[l - 1];
2054
2055	bool extra_byte = (offset - base_offset) & (ASAN_SHADOW_GRANULARITY - 1);
2056	/* If a red-zone is not aligned to ASAN_SHADOW_GRANULARITY then
2057	the previous stack variable has size % ASAN_SHADOW_GRANULARITY != 0.
2058	In that case we have to emit one extra byte that will describe
2059	how many bytes (our of ASAN_SHADOW_GRANULARITY) can be accessed. */
2060	if (extra_byte)
2061	{
2062	HOST_WIDE_INT aoff
2063	= base_offset + ((offset - base_offset)
2064	& ~(ASAN_SHADOW_GRANULARITY - HOST_WIDE_INT_1));
2065	rz_buffer.emit_redzone_byte (offset: aoff, value: offset - aoff);
2066	offset = aoff + ASAN_SHADOW_GRANULARITY;
2067	}
2068
2069	/* Calculate size of red zone payload. */
2070	while (offset < offsets[l - 2])
2071	{
2072	rz_buffer.emit_redzone_byte (offset, value: cur_shadow_byte);
2073	offset += ASAN_SHADOW_GRANULARITY;
2074	}
2075
2076	cur_shadow_byte = ASAN_STACK_MAGIC_MIDDLE;
2077	}
2078
2079	/* As the automatic variables are aligned to
2080	ASAN_RED_ZONE_SIZE / ASAN_SHADOW_GRANULARITY, the buffer should be
2081	flushed here. */
2082	gcc_assert (rz_buffer.m_shadow_bytes.is_empty ());
2083
2084	do_pending_stack_adjust ();
2085
2086	/* Construct epilogue sequence. */
2087	start_sequence ();
2088
2089	lab = NULL;
2090	if (use_after_return_class != -1)
2091	{
2092	rtx_code_label *lab2 = gen_label_rtx ();
2093	char c = (char) ASAN_STACK_MAGIC_USE_AFTER_RET;
2094	emit_cmp_and_jump_insns (orig_base, base, EQ, NULL_RTX,
2095	VOIDmode, 0, lab2,
2096	prob: profile_probability::very_likely ());
2097	shadow_mem = gen_rtx_MEM (BLKmode, shadow_base);
2098	set_mem_alias_set (shadow_mem, asan_shadow_set);
2099	mem = gen_rtx_MEM (ptr_mode, base);
2100	mem = adjust_address (mem, VOIDmode, base_align_bias);
2101	emit_move_insn (mem, gen_int_mode (ASAN_STACK_RETIRED_MAGIC, ptr_mode));
2102	unsigned HOST_WIDE_INT sz = asan_frame_size >> ASAN_SHADOW_SHIFT;
2103	if (use_after_return_class < 5
2104	&& can_store_by_pieces (sz, builtin_memset_read_str, &c,
2105	BITS_PER_UNIT, true))
2106	{
2107	/* Emit:
2108	memset(ShadowBase, kAsanStackAfterReturnMagic, ShadowSize);
2109	**SavedFlagPtr(FakeStack, class_id) = 0
2110	*/
2111	store_by_pieces (shadow_mem, sz, builtin_memset_read_str, &c,
2112	BITS_PER_UNIT, true, RETURN_BEGIN);
2113
2114	unsigned HOST_WIDE_INT offset
2115	= (1 << (use_after_return_class + 6));
2116	offset -= GET_MODE_SIZE (mode: ptr_mode);
2117	mem = gen_rtx_MEM (ptr_mode, base);
2118	mem = adjust_address (mem, ptr_mode, offset);
2119	rtx addr = gen_reg_rtx (ptr_mode);
2120	emit_move_insn (addr, mem);
2121	addr = convert_memory_address (Pmode, addr);
2122	mem = gen_rtx_MEM (QImode, addr);
2123	emit_move_insn (mem, const0_rtx);
2124	}
2125	else if (use_after_return_class >= 5
2126	|| !set_storage_via_setmem (shadow_mem,
2127	GEN_INT (sz),
2128	gen_int_mode (c, QImode),
2129	BITS_PER_UNIT, BITS_PER_UNIT,
2130	-1, sz, sz, sz))
2131	{
2132	snprintf (s: buf, maxlen: sizeof buf, format: "__asan_stack_free_%d",
2133	use_after_return_class);
2134	ret = init_one_libfunc (buf);
2135	rtx addr = convert_memory_address (ptr_mode, base);
2136	rtx orig_addr = convert_memory_address (ptr_mode, orig_base);
2137	emit_library_call (fun: ret, fn_type: LCT_NORMAL, outmode: ptr_mode, arg1: addr, arg1_mode: ptr_mode,
2138	GEN_INT (asan_frame_size + base_align_bias),
2139	TYPE_MODE (pointer_sized_int_node),
2140	arg3: orig_addr, arg3_mode: ptr_mode);
2141	}
2142	lab = gen_label_rtx ();
2143	emit_jump (lab);
2144	emit_label (lab2);
2145	}
2146
2147	shadow_mem = gen_rtx_MEM (BLKmode, shadow_base);
2148	set_mem_alias_set (shadow_mem, asan_shadow_set);
2149
2150	if (STRICT_ALIGNMENT)
2151	set_mem_align (shadow_mem, (GET_MODE_ALIGNMENT (SImode)));
2152
2153	prev_offset = base_offset;
2154	last_offset = base_offset;
2155	last_size = 0;
2156	last_size_aligned = 0;
2157	for (l = length; l; l -= 2)
2158	{
2159	offset = base_offset + ((offsets[l - 1] - base_offset)
2160	& ~(ASAN_RED_ZONE_SIZE - HOST_WIDE_INT_1));
2161	if (last_offset + last_size_aligned < offset)
2162	{
2163	shadow_mem = adjust_address (shadow_mem, VOIDmode,
2164	(last_offset - prev_offset)
2165	>> ASAN_SHADOW_SHIFT);
2166	prev_offset = last_offset;
2167	asan_clear_shadow (shadow_mem, len: last_size_aligned >> ASAN_SHADOW_SHIFT);
2168	last_offset = offset;
2169	last_size = 0;
2170	}
2171	else
2172	last_size = offset - last_offset;
2173	last_size += base_offset + ((offsets[l - 2] - base_offset)
2174	& ~(ASAN_MIN_RED_ZONE_SIZE - HOST_WIDE_INT_1))
2175	- offset;
2176
2177	/* Unpoison shadow memory that corresponds to a variable that is
2178	is subject of use-after-return sanitization. */
2179	if (l > 2)
2180	{
2181	decl = decls[l / 2 - 2];
2182	if (asan_handled_variables != NULL
2183	&& asan_handled_variables->contains (k: decl))
2184	{
2185	HOST_WIDE_INT size = offsets[l - 3] - offsets[l - 2];
2186	if (dump_file && (dump_flags & TDF_DETAILS))
2187	{
2188	const char *n = (DECL_NAME (decl)
2189	? IDENTIFIER_POINTER (DECL_NAME (decl))
2190	: "<unknown>");
2191	fprintf (stream: dump_file, format: "Unpoisoning shadow stack for variable: "
2192	"%s (%" PRId64 " B)\n", n, size);
2193	}
2194
2195	last_size += size & ~(ASAN_MIN_RED_ZONE_SIZE - HOST_WIDE_INT_1);
2196	}
2197	}
2198	last_size_aligned
2199	= ((last_size + (ASAN_RED_ZONE_SIZE - HOST_WIDE_INT_1))
2200	& ~(ASAN_RED_ZONE_SIZE - HOST_WIDE_INT_1));
2201	}
2202	if (last_size_aligned)
2203	{
2204	shadow_mem = adjust_address (shadow_mem, VOIDmode,
2205	(last_offset - prev_offset)
2206	>> ASAN_SHADOW_SHIFT);
2207	asan_clear_shadow (shadow_mem, len: last_size_aligned >> ASAN_SHADOW_SHIFT);
2208	}
2209
2210	/* Clean-up set with instrumented stack variables. */
2211	delete asan_handled_variables;
2212	asan_handled_variables = NULL;
2213	delete asan_used_labels;
2214	asan_used_labels = NULL;
2215
2216	do_pending_stack_adjust ();
2217	if (lab)
2218	emit_label (lab);
2219
2220	insns = get_insns ();
2221	end_sequence ();
2222	return insns;
2223	}
2224
2225	/* Emit __asan_allocas_unpoison (top, bot) call. The BASE parameter corresponds
2226	to BOT argument, for TOP virtual_stack_dynamic_rtx is used. NEW_SEQUENCE
2227	indicates whether we're emitting new instructions sequence or not. */
2228
2229	rtx_insn *
2230	asan_emit_allocas_unpoison (rtx top, rtx bot, rtx_insn *before)
2231	{
2232	if (before)
2233	push_to_sequence (before);
2234	else
2235	start_sequence ();
2236	rtx ret = init_one_libfunc ("__asan_allocas_unpoison");
2237	top = convert_memory_address (ptr_mode, top);
2238	bot = convert_memory_address (ptr_mode, bot);
2239	emit_library_call (fun: ret, fn_type: LCT_NORMAL, outmode: ptr_mode,
2240	arg1: top, arg1_mode: ptr_mode, arg2: bot, arg2_mode: ptr_mode);
2241
2242	do_pending_stack_adjust ();
2243	rtx_insn *insns = get_insns ();
2244	end_sequence ();
2245	return insns;
2246	}
2247
2248	/* Return true if DECL, a global var, might be overridden and needs
2249	therefore a local alias. */
2250
2251	static bool
2252	asan_needs_local_alias (tree decl)
2253	{
2254	return DECL_WEAK (decl) || !targetm.binds_local_p (decl);
2255	}
2256
2257	/* Return true if DECL, a global var, is an artificial ODR indicator symbol
2258	therefore doesn't need protection. */
2259
2260	static bool
2261	is_odr_indicator (tree decl)
2262	{
2263	return (DECL_ARTIFICIAL (decl)
2264	&& lookup_attribute (attr_name: "asan odr indicator", DECL_ATTRIBUTES (decl)));
2265	}
2266
2267	/* Return true if DECL is a VAR_DECL that should be protected
2268	by Address Sanitizer, by appending a red zone with protected
2269	shadow memory after it and aligning it to at least
2270	ASAN_RED_ZONE_SIZE bytes. */
2271
2272	bool
2273	asan_protect_global (tree decl, bool ignore_decl_rtl_set_p)
2274	{
2275	if (!param_asan_globals)
2276	return false;
2277
2278	rtx rtl, symbol;
2279
2280	if (TREE_CODE (decl) == STRING_CST)
2281	{
2282	/* Instrument all STRING_CSTs except those created
2283	by asan_pp_string here. */
2284	if (shadow_ptr_types[0] != NULL_TREE
2285	&& TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
2286	&& TREE_TYPE (TREE_TYPE (decl)) == TREE_TYPE (shadow_ptr_types[0]))
2287	return false;
2288	return true;
2289	}
2290	if (!VAR_P (decl)
2291	/* TLS vars aren't statically protectable. */
2292	|| DECL_THREAD_LOCAL_P (decl)
2293	/* Externs will be protected elsewhere. */
2294	|| DECL_EXTERNAL (decl)
2295	/* PR sanitizer/81697: For architectures that use section anchors first
2296	call to asan_protect_global may occur before DECL_RTL (decl) is set.
2297	We should ignore DECL_RTL_SET_P then, because otherwise the first call
2298	to asan_protect_global will return FALSE and the following calls on the
2299	same decl after setting DECL_RTL (decl) will return TRUE and we'll end
2300	up with inconsistency at runtime. */
2301	|| (!DECL_RTL_SET_P (decl) && !ignore_decl_rtl_set_p)
2302	/* Comdat vars pose an ABI problem, we can't know if
2303	the var that is selected by the linker will have
2304	padding or not. */
2305	|| DECL_ONE_ONLY (decl)
2306	/* Similarly for common vars. People can use -fno-common.
2307	Note: Linux kernel is built with -fno-common, so we do instrument
2308	globals there even if it is C. */
2309	|| (DECL_COMMON (decl) && TREE_PUBLIC (decl))
2310	/* Don't protect if using user section, often vars placed
2311	into user section from multiple TUs are then assumed
2312	to be an array of such vars, putting padding in there
2313	breaks this assumption. */
2314	|| (DECL_SECTION_NAME (decl) != NULL
2315	&& !symtab_node::get (decl)->implicit_section
2316	&& !section_sanitized_p (DECL_SECTION_NAME (decl)))
2317	/* Don't protect variables in non-generic address-space. */
2318	|| !ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (TREE_TYPE (decl)))
2319	|| DECL_SIZE (decl) == 0
2320	|| ASAN_RED_ZONE_SIZE * BITS_PER_UNIT > MAX_OFILE_ALIGNMENT
2321	|| TREE_CODE (DECL_SIZE_UNIT (decl)) != INTEGER_CST
2322	|| !valid_constant_size_p (DECL_SIZE_UNIT (decl))
2323	|| DECL_ALIGN_UNIT (decl) > 2 * ASAN_RED_ZONE_SIZE
2324	|| TREE_TYPE (decl) == ubsan_get_source_location_type ()
2325	|| is_odr_indicator (decl))
2326	return false;
2327
2328	if (!ignore_decl_rtl_set_p || DECL_RTL_SET_P (decl))
2329	{
2330
2331	rtl = DECL_RTL (decl);
2332	if (!MEM_P (rtl) || GET_CODE (XEXP (rtl, 0)) != SYMBOL_REF)
2333	return false;
2334	symbol = XEXP (rtl, 0);
2335
2336	if (CONSTANT_POOL_ADDRESS_P (symbol)
2337	|| TREE_CONSTANT_POOL_ADDRESS_P (symbol))
2338	return false;
2339	}
2340
2341	if (lookup_attribute (attr_name: "weakref", DECL_ATTRIBUTES (decl)))
2342	return false;
2343
2344	if (!TARGET_SUPPORTS_ALIASES && asan_needs_local_alias (decl))
2345	return false;
2346
2347	return true;
2348	}
2349
2350	/* Construct a function tree for __asan_report_{load,store}{1,2,4,8,16,_n}.
2351	IS_STORE is either 1 (for a store) or 0 (for a load). */
2352
2353	static tree
2354	report_error_func (bool is_store, bool recover_p, HOST_WIDE_INT size_in_bytes,
2355	int *nargs)
2356	{
2357	gcc_assert (!hwasan_sanitize_p ());
2358
2359	static enum built_in_function report[2][2][6]
2360	= { { { BUILT_IN_ASAN_REPORT_LOAD1, BUILT_IN_ASAN_REPORT_LOAD2,
2361	BUILT_IN_ASAN_REPORT_LOAD4, BUILT_IN_ASAN_REPORT_LOAD8,
2362	BUILT_IN_ASAN_REPORT_LOAD16, BUILT_IN_ASAN_REPORT_LOAD_N },
2363	{ BUILT_IN_ASAN_REPORT_STORE1, BUILT_IN_ASAN_REPORT_STORE2,
2364	BUILT_IN_ASAN_REPORT_STORE4, BUILT_IN_ASAN_REPORT_STORE8,
2365	BUILT_IN_ASAN_REPORT_STORE16, BUILT_IN_ASAN_REPORT_STORE_N } },
2366	{ { BUILT_IN_ASAN_REPORT_LOAD1_NOABORT,
2367	BUILT_IN_ASAN_REPORT_LOAD2_NOABORT,
2368	BUILT_IN_ASAN_REPORT_LOAD4_NOABORT,
2369	BUILT_IN_ASAN_REPORT_LOAD8_NOABORT,
2370	BUILT_IN_ASAN_REPORT_LOAD16_NOABORT,
2371	BUILT_IN_ASAN_REPORT_LOAD_N_NOABORT },
2372	{ BUILT_IN_ASAN_REPORT_STORE1_NOABORT,
2373	BUILT_IN_ASAN_REPORT_STORE2_NOABORT,
2374	BUILT_IN_ASAN_REPORT_STORE4_NOABORT,
2375	BUILT_IN_ASAN_REPORT_STORE8_NOABORT,
2376	BUILT_IN_ASAN_REPORT_STORE16_NOABORT,
2377	BUILT_IN_ASAN_REPORT_STORE_N_NOABORT } } };
2378	if (size_in_bytes == -1)
2379	{
2380	*nargs = 2;
2381	return builtin_decl_implicit (fncode: report[recover_p][is_store][5]);
2382	}
2383	*nargs = 1;
2384	int size_log2 = exact_log2 (x: size_in_bytes);
2385	return builtin_decl_implicit (fncode: report[recover_p][is_store][size_log2]);
2386	}
2387
2388	/* Construct a function tree for __asan_{load,store}{1,2,4,8,16,_n}.
2389	IS_STORE is either 1 (for a store) or 0 (for a load). */
2390
2391	static tree
2392	check_func (bool is_store, bool recover_p, HOST_WIDE_INT size_in_bytes,
2393	int *nargs)
2394	{
2395	static enum built_in_function check[2][2][6]
2396	= { { { BUILT_IN_ASAN_LOAD1, BUILT_IN_ASAN_LOAD2,
2397	BUILT_IN_ASAN_LOAD4, BUILT_IN_ASAN_LOAD8,
2398	BUILT_IN_ASAN_LOAD16, BUILT_IN_ASAN_LOADN },
2399	{ BUILT_IN_ASAN_STORE1, BUILT_IN_ASAN_STORE2,
2400	BUILT_IN_ASAN_STORE4, BUILT_IN_ASAN_STORE8,
2401	BUILT_IN_ASAN_STORE16, BUILT_IN_ASAN_STOREN } },
2402	{ { BUILT_IN_ASAN_LOAD1_NOABORT,
2403	BUILT_IN_ASAN_LOAD2_NOABORT,
2404	BUILT_IN_ASAN_LOAD4_NOABORT,
2405	BUILT_IN_ASAN_LOAD8_NOABORT,
2406	BUILT_IN_ASAN_LOAD16_NOABORT,
2407	BUILT_IN_ASAN_LOADN_NOABORT },
2408	{ BUILT_IN_ASAN_STORE1_NOABORT,
2409	BUILT_IN_ASAN_STORE2_NOABORT,
2410	BUILT_IN_ASAN_STORE4_NOABORT,
2411	BUILT_IN_ASAN_STORE8_NOABORT,
2412	BUILT_IN_ASAN_STORE16_NOABORT,
2413	BUILT_IN_ASAN_STOREN_NOABORT } } };
2414	if (size_in_bytes == -1)
2415	{
2416	*nargs = 2;
2417	return builtin_decl_implicit (fncode: check[recover_p][is_store][5]);
2418	}
2419	*nargs = 1;
2420	int size_log2 = exact_log2 (x: size_in_bytes);
2421	return builtin_decl_implicit (fncode: check[recover_p][is_store][size_log2]);
2422	}
2423
2424	/* Split the current basic block and create a condition statement
2425	insertion point right before or after the statement pointed to by
2426	ITER. Return an iterator to the point at which the caller might
2427	safely insert the condition statement.
2428
2429	THEN_BLOCK must be set to the address of an uninitialized instance
2430	of basic_block. The function will then set *THEN_BLOCK to the
2431	'then block' of the condition statement to be inserted by the
2432	caller.
2433
2434	If CREATE_THEN_FALLTHRU_EDGE is false, no edge will be created from
2435	*THEN_BLOCK to *FALLTHROUGH_BLOCK.
2436
2437	Similarly, the function will set *FALLTRHOUGH_BLOCK to the 'else
2438	block' of the condition statement to be inserted by the caller.
2439
2440	Note that *FALLTHROUGH_BLOCK is a new block that contains the
2441	statements starting from *ITER, and *THEN_BLOCK is a new empty
2442	block.
2443
2444	*ITER is adjusted to point to always point to the first statement
2445	of the basic block * FALLTHROUGH_BLOCK. That statement is the
2446	same as what ITER was pointing to prior to calling this function,
2447	if BEFORE_P is true; otherwise, it is its following statement. */
2448
2449	gimple_stmt_iterator
2450	create_cond_insert_point (gimple_stmt_iterator *iter,
2451	bool before_p,
2452	bool then_more_likely_p,
2453	bool create_then_fallthru_edge,
2454	basic_block *then_block,
2455	basic_block *fallthrough_block)
2456	{
2457	gimple_stmt_iterator gsi = *iter;
2458
2459	if (!gsi_end_p (i: gsi) && before_p)
2460	gsi_prev (i: &gsi);
2461
2462	basic_block cur_bb = gsi_bb (i: *iter);
2463
2464	edge e = split_block (cur_bb, gsi_stmt (i: gsi));
2465
2466	/* Get a hold on the 'condition block', the 'then block' and the
2467	'else block'. */
2468	basic_block cond_bb = e->src;
2469	basic_block fallthru_bb = e->dest;
2470	basic_block then_bb = create_empty_bb (cond_bb);
2471	if (current_loops)
2472	{
2473	add_bb_to_loop (then_bb, cond_bb->loop_father);
2474	loops_state_set (flags: LOOPS_NEED_FIXUP);
2475	}
2476
2477	/* Set up the newly created 'then block'. */
2478	e = make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
2479	profile_probability fallthrough_probability
2480	= then_more_likely_p
2481	? profile_probability::very_unlikely ()
2482	: profile_probability::very_likely ();
2483	e->probability = fallthrough_probability.invert ();
2484	then_bb->count = e->count ();
2485	if (create_then_fallthru_edge)
2486	make_single_succ_edge (then_bb, fallthru_bb, EDGE_FALLTHRU);
2487
2488	/* Set up the fallthrough basic block. */
2489	e = find_edge (cond_bb, fallthru_bb);
2490	e->flags = EDGE_FALSE_VALUE;
2491	e->probability = fallthrough_probability;
2492
2493	/* Update dominance info for the newly created then_bb; note that
2494	fallthru_bb's dominance info has already been updated by
2495	split_bock. */
2496	if (dom_info_available_p (CDI_DOMINATORS))
2497	set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
2498
2499	*then_block = then_bb;
2500	*fallthrough_block = fallthru_bb;
2501	*iter = gsi_start_bb (bb: fallthru_bb);
2502
2503	return gsi_last_bb (bb: cond_bb);
2504	}
2505
2506	/* Insert an if condition followed by a 'then block' right before the
2507	statement pointed to by ITER. The fallthrough block -- which is the
2508	else block of the condition as well as the destination of the
2509	outcoming edge of the 'then block' -- starts with the statement
2510	pointed to by ITER.
2511
2512	COND is the condition of the if.
2513
2514	If THEN_MORE_LIKELY_P is true, the probability of the edge to the
2515	'then block' is higher than the probability of the edge to the
2516	fallthrough block.
2517
2518	Upon completion of the function, *THEN_BB is set to the newly
2519	inserted 'then block' and similarly, *FALLTHROUGH_BB is set to the
2520	fallthrough block.
2521
2522	*ITER is adjusted to still point to the same statement it was
2523	pointing to initially. */
2524
2525	static void
2526	insert_if_then_before_iter (gcond *cond,
2527	gimple_stmt_iterator *iter,
2528	bool then_more_likely_p,
2529	basic_block *then_bb,
2530	basic_block *fallthrough_bb)
2531	{
2532	gimple_stmt_iterator cond_insert_point =
2533	create_cond_insert_point (iter,
2534	/*before_p=*/true,
2535	then_more_likely_p,
2536	/*create_then_fallthru_edge=*/true,
2537	then_block: then_bb,
2538	fallthrough_block: fallthrough_bb);
2539	gsi_insert_after (&cond_insert_point, cond, GSI_NEW_STMT);
2540	}
2541
2542	/* Build (base_addr >> ASAN_SHADOW_SHIFT) + asan_shadow_offset ().
2543	If RETURN_ADDRESS is set to true, return memory location instread
2544	of a value in the shadow memory. */
2545
2546	static tree
2547	build_shadow_mem_access (gimple_stmt_iterator *gsi, location_t location,
2548	tree base_addr, tree shadow_ptr_type,
2549	bool return_address = false)
2550	{
2551	tree t, uintptr_type = TREE_TYPE (base_addr);
2552	tree shadow_type = TREE_TYPE (shadow_ptr_type);
2553	gimple *g;
2554
2555	t = build_int_cst (uintptr_type, ASAN_SHADOW_SHIFT);
2556	g = gimple_build_assign (make_ssa_name (var: uintptr_type), RSHIFT_EXPR,
2557	base_addr, t);
2558	gimple_set_location (g, location);
2559	gsi_insert_after (gsi, g, GSI_NEW_STMT);
2560
2561	t = build_int_cst (uintptr_type, asan_shadow_offset ());
2562	g = gimple_build_assign (make_ssa_name (var: uintptr_type), PLUS_EXPR,
2563	gimple_assign_lhs (gs: g), t);
2564	gimple_set_location (g, location);
2565	gsi_insert_after (gsi, g, GSI_NEW_STMT);
2566
2567	g = gimple_build_assign (make_ssa_name (var: shadow_ptr_type), NOP_EXPR,
2568	gimple_assign_lhs (gs: g));
2569	gimple_set_location (g, location);
2570	gsi_insert_after (gsi, g, GSI_NEW_STMT);
2571
2572	if (!return_address)
2573	{
2574	t = build2 (MEM_REF, shadow_type, gimple_assign_lhs (gs: g),
2575	build_int_cst (shadow_ptr_type, 0));
2576	g = gimple_build_assign (make_ssa_name (var: shadow_type), MEM_REF, t);
2577	gimple_set_location (g, location);
2578	gsi_insert_after (gsi, g, GSI_NEW_STMT);
2579	}
2580
2581	return gimple_assign_lhs (gs: g);
2582	}
2583
2584	/* BASE can already be an SSA_NAME; in that case, do not create a
2585	new SSA_NAME for it. */
2586
2587	static tree
2588	maybe_create_ssa_name (location_t loc, tree base, gimple_stmt_iterator *iter,
2589	bool before_p)
2590	{
2591	STRIP_USELESS_TYPE_CONVERSION (base);
2592	if (TREE_CODE (base) == SSA_NAME)
2593	return base;
2594	gimple *g = gimple_build_assign (make_ssa_name (TREE_TYPE (base)), base);
2595	gimple_set_location (g, location: loc);
2596	if (before_p)
2597	gsi_safe_insert_before (iter, g);
2598	else
2599	gsi_insert_after (iter, g, GSI_NEW_STMT);
2600	return gimple_assign_lhs (gs: g);
2601	}
2602
2603	/* LEN can already have necessary size and precision;
2604	in that case, do not create a new variable. */
2605
2606	tree
2607	maybe_cast_to_ptrmode (location_t loc, tree len, gimple_stmt_iterator *iter,
2608	bool before_p)
2609	{
2610	if (ptrofftype_p (type: len))
2611	return len;
2612	gimple *g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
2613	NOP_EXPR, len);
2614	gimple_set_location (g, location: loc);
2615	if (before_p)
2616	gsi_safe_insert_before (iter, g);
2617	else
2618	gsi_insert_after (iter, g, GSI_NEW_STMT);
2619	return gimple_assign_lhs (gs: g);
2620	}
2621
2622	/* Instrument the memory access instruction BASE. Insert new
2623	statements before or after ITER.
2624
2625	Note that the memory access represented by BASE can be either an
2626	SSA_NAME, or a non-SSA expression. LOCATION is the source code
2627	location. IS_STORE is TRUE for a store, FALSE for a load.
2628	BEFORE_P is TRUE for inserting the instrumentation code before
2629	ITER, FALSE for inserting it after ITER. IS_SCALAR_ACCESS is TRUE
2630	for a scalar memory access and FALSE for memory region access.
2631	NON_ZERO_P is TRUE if memory region is guaranteed to have non-zero
2632	length. ALIGN tells alignment of accessed memory object.
2633
2634	START_INSTRUMENTED and END_INSTRUMENTED are TRUE if start/end of
2635	memory region have already been instrumented.
2636
2637	If BEFORE_P is TRUE, *ITER is arranged to still point to the
2638	statement it was pointing to prior to calling this function,
2639	otherwise, it points to the statement logically following it. */
2640
2641	static void
2642	build_check_stmt (location_t loc, tree base, tree len,
2643	HOST_WIDE_INT size_in_bytes, gimple_stmt_iterator *iter,
2644	bool is_non_zero_len, bool before_p, bool is_store,
2645	bool is_scalar_access, unsigned int align = 0)
2646	{
2647	gimple_stmt_iterator gsi = *iter;
2648	gimple *g;
2649
2650	gcc_assert (!(size_in_bytes > 0 && !is_non_zero_len));
2651	gcc_assert (size_in_bytes == -1 || size_in_bytes >= 1);
2652
2653	gsi = *iter;
2654
2655	base = unshare_expr (base);
2656	base = maybe_create_ssa_name (loc, base, iter: &gsi, before_p);
2657
2658	if (len)
2659	{
2660	len = unshare_expr (len);
2661	len = maybe_cast_to_ptrmode (loc, len, iter, before_p);
2662	}
2663	else
2664	{
2665	gcc_assert (size_in_bytes != -1);
2666	len = build_int_cst (pointer_sized_int_node, size_in_bytes);
2667	}
2668
2669	if (size_in_bytes > 1)
2670	{
2671	if ((size_in_bytes & (size_in_bytes - 1)) != 0
2672	|| size_in_bytes > 16)
2673	is_scalar_access = false;
2674	else if (align && align < size_in_bytes * BITS_PER_UNIT)
2675	{
2676	/* On non-strict alignment targets, if
2677	16-byte access is just 8-byte aligned,
2678	this will result in misaligned shadow
2679	memory 2 byte load, but otherwise can
2680	be handled using one read. */
2681	if (size_in_bytes != 16
2682	|| STRICT_ALIGNMENT
2683	|| align < 8 * BITS_PER_UNIT)
2684	is_scalar_access = false;
2685	}
2686	}
2687
2688	HOST_WIDE_INT flags = 0;
2689	if (is_store)
2690	flags |= ASAN_CHECK_STORE;
2691	if (is_non_zero_len)
2692	flags |= ASAN_CHECK_NON_ZERO_LEN;
2693	if (is_scalar_access)
2694	flags |= ASAN_CHECK_SCALAR_ACCESS;
2695
2696	enum internal_fn fn = hwasan_sanitize_p ()
2697	? IFN_HWASAN_CHECK
2698	: IFN_ASAN_CHECK;
2699
2700	g = gimple_build_call_internal (fn, 4,
2701	build_int_cst (integer_type_node, flags),
2702	base, len,
2703	build_int_cst (integer_type_node,
2704	align / BITS_PER_UNIT));
2705	gimple_set_location (g, location: loc);
2706	if (before_p)
2707	gsi_safe_insert_before (&gsi, g);
2708	else
2709	{
2710	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
2711	gsi_next (i: &gsi);
2712	*iter = gsi;
2713	}
2714	}
2715
2716	/* If T represents a memory access, add instrumentation code before ITER.
2717	LOCATION is source code location.
2718	IS_STORE is either TRUE (for a store) or FALSE (for a load). */
2719
2720	static void
2721	instrument_derefs (gimple_stmt_iterator *iter, tree t,
2722	location_t location, bool is_store)
2723	{
2724	if (is_store && !(asan_instrument_writes () || hwasan_instrument_writes ()))
2725	return;
2726	if (!is_store && !(asan_instrument_reads () || hwasan_instrument_reads ()))
2727	return;
2728
2729	tree type, base;
2730	HOST_WIDE_INT size_in_bytes;
2731	if (location == UNKNOWN_LOCATION)
2732	location = EXPR_LOCATION (t);
2733
2734	type = TREE_TYPE (t);
2735	switch (TREE_CODE (t))
2736	{
2737	case ARRAY_REF:
2738	case COMPONENT_REF:
2739	case INDIRECT_REF:
2740	case MEM_REF:
2741	case VAR_DECL:
2742	case BIT_FIELD_REF:
2743	break;
2744	/* FALLTHRU */
2745	default:
2746	return;
2747	}
2748
2749	size_in_bytes = int_size_in_bytes (type);
2750	if (size_in_bytes <= 0)
2751	return;
2752
2753	poly_int64 bitsize, bitpos;
2754	tree offset;
2755	machine_mode mode;
2756	int unsignedp, reversep, volatilep = 0;
2757	tree inner = get_inner_reference (t, &bitsize, &bitpos, &offset, &mode,
2758	&unsignedp, &reversep, &volatilep);
2759
2760	if (TREE_CODE (t) == COMPONENT_REF
2761	&& DECL_BIT_FIELD_REPRESENTATIVE (TREE_OPERAND (t, 1)) != NULL_TREE)
2762	{
2763	tree repr = DECL_BIT_FIELD_REPRESENTATIVE (TREE_OPERAND (t, 1));
2764	instrument_derefs (iter, t: build3 (COMPONENT_REF, TREE_TYPE (repr),
2765	TREE_OPERAND (t, 0), repr,
2766	TREE_OPERAND (t, 2)),
2767	location, is_store);
2768	return;
2769	}
2770
2771	if (!multiple_p (a: bitpos, BITS_PER_UNIT)
2772	|| maybe_ne (a: bitsize, b: size_in_bytes * BITS_PER_UNIT))
2773	return;
2774
2775	if (VAR_P (inner) && DECL_HARD_REGISTER (inner))
2776	return;
2777
2778	/* Accesses to non-generic address-spaces should not be instrumented. */
2779	if (!ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (TREE_TYPE (inner))))
2780	return;
2781
2782	poly_int64 decl_size;
2783	if ((VAR_P (inner)
2784	|| (TREE_CODE (inner) == RESULT_DECL
2785	&& !aggregate_value_p (inner, current_function_decl)))
2786	&& offset == NULL_TREE
2787	&& DECL_SIZE (inner)
2788	&& poly_int_tree_p (DECL_SIZE (inner), value: &decl_size)
2789	&& known_subrange_p (pos1: bitpos, size1: bitsize, pos2: 0, size2: decl_size))
2790	{
2791	if (VAR_P (inner) && DECL_THREAD_LOCAL_P (inner))
2792	return;
2793	/* If we're not sanitizing globals and we can tell statically that this
2794	access is inside a global variable, then there's no point adding
2795	instrumentation to check the access. N.b. hwasan currently never
2796	sanitizes globals. */
2797	if ((hwasan_sanitize_p () || !param_asan_globals)
2798	&& is_global_var (t: inner))
2799	return;
2800	if (!TREE_STATIC (inner))
2801	{
2802	/* Automatic vars in the current function will be always
2803	accessible. */
2804	if (decl_function_context (inner) == current_function_decl
2805	&& (!asan_sanitize_use_after_scope ()
2806	|| !TREE_ADDRESSABLE (inner)))
2807	return;
2808	}
2809	/* Always instrument external vars, they might be dynamically
2810	initialized. */
2811	else if (!DECL_EXTERNAL (inner))
2812	{
2813	/* For static vars if they are known not to be dynamically
2814	initialized, they will be always accessible. */
2815	varpool_node *vnode = varpool_node::get (decl: inner);
2816	if (vnode && !vnode->dynamically_initialized)
2817	return;
2818	}
2819	}
2820
2821	if (DECL_P (inner)
2822	&& decl_function_context (inner) == current_function_decl
2823	&& !TREE_ADDRESSABLE (inner))
2824	mark_addressable (inner);
2825
2826	base = build_fold_addr_expr (t);
2827	if (!has_mem_ref_been_instrumented (ref: base, access_size: size_in_bytes))
2828	{
2829	unsigned int align = get_object_alignment (t);
2830	build_check_stmt (loc: location, base, NULL_TREE, size_in_bytes, iter,
2831	/*is_non_zero_len*/size_in_bytes > 0, /*before_p=*/true,
2832	is_store, /*is_scalar_access*/true, align);
2833	update_mem_ref_hash_table (ref: base, access_size: size_in_bytes);
2834	update_mem_ref_hash_table (ref: t, access_size: size_in_bytes);
2835	}
2836
2837	}
2838
2839	/* Insert a memory reference into the hash table if access length
2840	can be determined in compile time. */
2841
2842	static void
2843	maybe_update_mem_ref_hash_table (tree base, tree len)
2844	{
2845	if (!POINTER_TYPE_P (TREE_TYPE (base))
2846	|| !INTEGRAL_TYPE_P (TREE_TYPE (len)))
2847	return;
2848
2849	HOST_WIDE_INT size_in_bytes = tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
2850
2851	if (size_in_bytes != -1)
2852	update_mem_ref_hash_table (ref: base, access_size: size_in_bytes);
2853	}
2854
2855	/* Instrument an access to a contiguous memory region that starts at
2856	the address pointed to by BASE, over a length of LEN (expressed in
2857	the sizeof (*BASE) bytes). ITER points to the instruction before
2858	which the instrumentation instructions must be inserted. LOCATION
2859	is the source location that the instrumentation instructions must
2860	have. If IS_STORE is true, then the memory access is a store;
2861	otherwise, it's a load. */
2862
2863	static void
2864	instrument_mem_region_access (tree base, tree len,
2865	gimple_stmt_iterator *iter,
2866	location_t location, bool is_store)
2867	{
2868	if (!POINTER_TYPE_P (TREE_TYPE (base))
2869	|| !INTEGRAL_TYPE_P (TREE_TYPE (len))
2870	|| integer_zerop (len))
2871	return;
2872
2873	HOST_WIDE_INT size_in_bytes = tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
2874
2875	if ((size_in_bytes == -1)
2876	|| !has_mem_ref_been_instrumented (ref: base, access_size: size_in_bytes))
2877	{
2878	build_check_stmt (loc: location, base, len, size_in_bytes, iter,
2879	/*is_non_zero_len*/size_in_bytes > 0, /*before_p*/true,
2880	is_store, /*is_scalar_access*/false, /*align*/0);
2881	}
2882
2883	maybe_update_mem_ref_hash_table (base, len);
2884	*iter = gsi_for_stmt (gsi_stmt (i: *iter));
2885	}
2886
2887	/* Instrument the call to a built-in memory access function that is
2888	pointed to by the iterator ITER.
2889
2890	Upon completion, return TRUE iff *ITER has been advanced to the
2891	statement following the one it was originally pointing to. */
2892
2893	static bool
2894	instrument_builtin_call (gimple_stmt_iterator *iter)
2895	{
2896	if (!(asan_memintrin () || hwasan_memintrin ()))
2897	return false;
2898
2899	bool iter_advanced_p = false;
2900	gcall *call = as_a <gcall *> (p: gsi_stmt (i: *iter));
2901
2902	gcc_checking_assert (gimple_call_builtin_p (call, BUILT_IN_NORMAL));
2903
2904	location_t loc = gimple_location (g: call);
2905
2906	asan_mem_ref src0, src1, dest;
2907	asan_mem_ref_init (ref: &src0, NULL, access_size: 1);
2908	asan_mem_ref_init (ref: &src1, NULL, access_size: 1);
2909	asan_mem_ref_init (ref: &dest, NULL, access_size: 1);
2910
2911	tree src0_len = NULL_TREE, src1_len = NULL_TREE, dest_len = NULL_TREE;
2912	bool src0_is_store = false, src1_is_store = false, dest_is_store = false,
2913	dest_is_deref = false, intercepted_p = true;
2914
2915	if (get_mem_refs_of_builtin_call (call,
2916	src0: &src0, src0_len: &src0_len, src0_is_store: &src0_is_store,
2917	src1: &src1, src1_len: &src1_len, src1_is_store: &src1_is_store,
2918	dst: &dest, dst_len: &dest_len, dst_is_store: &dest_is_store,
2919	dest_is_deref: &dest_is_deref, intercepted_p: &intercepted_p, iter))
2920	{
2921	if (dest_is_deref)
2922	{
2923	instrument_derefs (iter, t: dest.start, location: loc, is_store: dest_is_store);
2924	gsi_next (i: iter);
2925	iter_advanced_p = true;
2926	}
2927	else if (!intercepted_p
2928	&& (src0_len || src1_len || dest_len))
2929	{
2930	if (src0.start != NULL_TREE)
2931	instrument_mem_region_access (base: src0.start, len: src0_len,
2932	iter, location: loc, /*is_store=*/false);
2933	if (src1.start != NULL_TREE)
2934	instrument_mem_region_access (base: src1.start, len: src1_len,
2935	iter, location: loc, /*is_store=*/false);
2936	if (dest.start != NULL_TREE)
2937	instrument_mem_region_access (base: dest.start, len: dest_len,
2938	iter, location: loc, /*is_store=*/true);
2939
2940	*iter = gsi_for_stmt (call);
2941	gsi_next (i: iter);
2942	iter_advanced_p = true;
2943	}
2944	else
2945	{
2946	if (src0.start != NULL_TREE)
2947	maybe_update_mem_ref_hash_table (base: src0.start, len: src0_len);
2948	if (src1.start != NULL_TREE)
2949	maybe_update_mem_ref_hash_table (base: src1.start, len: src1_len);
2950	if (dest.start != NULL_TREE)
2951	maybe_update_mem_ref_hash_table (base: dest.start, len: dest_len);
2952	}
2953	}
2954	return iter_advanced_p;
2955	}
2956
2957	/* Instrument the assignment statement ITER if it is subject to
2958	instrumentation. Return TRUE iff instrumentation actually
2959	happened. In that case, the iterator ITER is advanced to the next
2960	logical expression following the one initially pointed to by ITER,
2961	and the relevant memory reference that which access has been
2962	instrumented is added to the memory references hash table. */
2963
2964	static bool
2965	maybe_instrument_assignment (gimple_stmt_iterator *iter)
2966	{
2967	gimple *s = gsi_stmt (i: *iter);
2968
2969	gcc_assert (gimple_assign_single_p (s));
2970
2971	tree ref_expr = NULL_TREE;
2972	bool is_store, is_instrumented = false;
2973
2974	if (gimple_store_p (gs: s))
2975	{
2976	ref_expr = gimple_assign_lhs (gs: s);
2977	is_store = true;
2978	instrument_derefs (iter, t: ref_expr,
2979	location: gimple_location (g: s),
2980	is_store);
2981	is_instrumented = true;
2982	}
2983
2984	if (gimple_assign_load_p (s))
2985	{
2986	ref_expr = gimple_assign_rhs1 (gs: s);
2987	is_store = false;
2988	instrument_derefs (iter, t: ref_expr,
2989	location: gimple_location (g: s),
2990	is_store);
2991	is_instrumented = true;
2992	}
2993
2994	if (is_instrumented)
2995	gsi_next (i: iter);
2996
2997	return is_instrumented;
2998	}
2999
3000	/* Instrument the function call pointed to by the iterator ITER, if it
3001	is subject to instrumentation. At the moment, the only function
3002	calls that are instrumented are some built-in functions that access
3003	memory. Look at instrument_builtin_call to learn more.
3004
3005	Upon completion return TRUE iff *ITER was advanced to the statement
3006	following the one it was originally pointing to. */
3007
3008	static bool
3009	maybe_instrument_call (gimple_stmt_iterator *iter)
3010	{
3011	gimple *stmt = gsi_stmt (i: *iter);
3012	bool is_builtin = gimple_call_builtin_p (stmt, BUILT_IN_NORMAL);
3013
3014	if (is_builtin && instrument_builtin_call (iter))
3015	return true;
3016
3017	if (gimple_call_noreturn_p (s: stmt))
3018	{
3019	if (is_builtin)
3020	{
3021	tree callee = gimple_call_fndecl (gs: stmt);
3022	switch (DECL_FUNCTION_CODE (decl: callee))
3023	{
3024	case BUILT_IN_UNREACHABLE:
3025	case BUILT_IN_UNREACHABLE_TRAP:
3026	case BUILT_IN_TRAP:
3027	/* Don't instrument these. */
3028	return false;
3029	default:
3030	break;
3031	}
3032	}
3033	if (gimple_call_internal_p (gs: stmt, fn: IFN_ABNORMAL_DISPATCHER))
3034	/* Don't instrument this. */
3035	return false;
3036	/* If a function does not return, then we must handle clearing up the
3037	shadow stack accordingly. For ASAN we can simply set the entire stack
3038	to "valid" for accesses by setting the shadow space to 0 and all
3039	accesses will pass checks. That means that some bad accesses may be
3040	missed, but we will not report any false positives.
3041
3042	This is not possible for HWASAN. Since there is no "always valid" tag
3043	we can not set any space to "always valid". If we were to clear the
3044	entire shadow stack then code resuming from `longjmp` or a caught
3045	exception would trigger false positives when correctly accessing
3046	variables on the stack. Hence we need to handle things like
3047	`longjmp`, thread exit, and exceptions in a different way. These
3048	problems must be handled externally to the compiler, e.g. in the
3049	language runtime. */
3050	if (! hwasan_sanitize_p ())
3051	{
3052	tree decl = builtin_decl_implicit (fncode: BUILT_IN_ASAN_HANDLE_NO_RETURN);
3053	gimple *g = gimple_build_call (decl, 0);
3054	gimple_set_location (g, location: gimple_location (g: stmt));
3055	gsi_safe_insert_before (iter, g);
3056	}
3057	}
3058
3059	bool instrumented = false;
3060	if (gimple_store_p (gs: stmt)
3061	&& (gimple_call_builtin_p (stmt)
3062	|| gimple_call_internal_p (gs: stmt)
3063	|| !aggregate_value_p (TREE_TYPE (gimple_call_lhs (stmt)),
3064	gimple_call_fntype (gs: stmt))))
3065	{
3066	tree ref_expr = gimple_call_lhs (gs: stmt);
3067	instrument_derefs (iter, t: ref_expr,
3068	location: gimple_location (g: stmt),
3069	/*is_store=*/true);
3070
3071	instrumented = true;
3072	}
3073
3074	/* Walk through gimple_call arguments and check them id needed. */
3075	unsigned args_num = gimple_call_num_args (gs: stmt);
3076	for (unsigned i = 0; i < args_num; ++i)
3077	{
3078	tree arg = gimple_call_arg (gs: stmt, index: i);
3079	/* If ARG is not a non-aggregate register variable, compiler in general
3080	creates temporary for it and pass it as argument to gimple call.
3081	But in some cases, e.g. when we pass by value a small structure that
3082	fits to register, compiler can avoid extra overhead by pulling out
3083	these temporaries. In this case, we should check the argument. */
3084	if (!is_gimple_reg (arg) && !is_gimple_min_invariant (arg))
3085	{
3086	instrument_derefs (iter, t: arg,
3087	location: gimple_location (g: stmt),
3088	/*is_store=*/false);
3089	instrumented = true;
3090	}
3091	}
3092	if (instrumented)
3093	gsi_next (i: iter);
3094	return instrumented;
3095	}
3096
3097	/* Walk each instruction of all basic block and instrument those that
3098	represent memory references: loads, stores, or function calls.
3099	In a given basic block, this function avoids instrumenting memory
3100	references that have already been instrumented. */
3101
3102	static void
3103	transform_statements (void)
3104	{
3105	basic_block bb, last_bb = NULL;
3106	gimple_stmt_iterator i;
3107	int saved_last_basic_block = last_basic_block_for_fn (cfun);
3108
3109	FOR_EACH_BB_FN (bb, cfun)
3110	{
3111	basic_block prev_bb = bb;
3112
3113	if (bb->index >= saved_last_basic_block) continue;
3114
3115	/* Flush the mem ref hash table, if current bb doesn't have
3116	exactly one predecessor, or if that predecessor (skipping
3117	over asan created basic blocks) isn't the last processed
3118	basic block. Thus we effectively flush on extended basic
3119	block boundaries. */
3120	while (single_pred_p (bb: prev_bb))
3121	{
3122	prev_bb = single_pred (bb: prev_bb);
3123	if (prev_bb->index < saved_last_basic_block)
3124	break;
3125	}
3126	if (prev_bb != last_bb)
3127	empty_mem_ref_hash_table ();
3128	last_bb = bb;
3129
3130	for (i = gsi_start_bb (bb); !gsi_end_p (i);)
3131	{
3132	gimple *s = gsi_stmt (i);
3133
3134	if (has_stmt_been_instrumented_p (stmt: s))
3135	gsi_next (i: &i);
3136	else if (gimple_assign_single_p (gs: s)
3137	&& !gimple_clobber_p (s)
3138	&& maybe_instrument_assignment (iter: &i))
3139	/* Nothing to do as maybe_instrument_assignment advanced
3140	the iterator I. */;
3141	else if (is_gimple_call (gs: s) && maybe_instrument_call (iter: &i))
3142	/* Nothing to do as maybe_instrument_call
3143	advanced the iterator I. */;
3144	else
3145	{
3146	/* No instrumentation happened.
3147
3148	If the current instruction is a function call that
3149	might free something, let's forget about the memory
3150	references that got instrumented. Otherwise we might
3151	miss some instrumentation opportunities. Do the same
3152	for a ASAN_MARK poisoning internal function. */
3153	if (is_gimple_call (gs: s)
3154	&& (!nonfreeing_call_p (s)
3155	|| asan_mark_p (stmt: s, flag: ASAN_MARK_POISON)))
3156	empty_mem_ref_hash_table ();
3157
3158	gsi_next (i: &i);
3159	}
3160	}
3161	}
3162	free_mem_ref_resources ();
3163	}
3164
3165	/* Build
3166	__asan_before_dynamic_init (module_name)
3167	or
3168	__asan_after_dynamic_init ()
3169	call. */
3170
3171	tree
3172	asan_dynamic_init_call (bool after_p)
3173	{
3174	if (shadow_ptr_types[0] == NULL_TREE)
3175	asan_init_shadow_ptr_types ();
3176
3177	tree fn = builtin_decl_implicit (fncode: after_p
3178	? BUILT_IN_ASAN_AFTER_DYNAMIC_INIT
3179	: BUILT_IN_ASAN_BEFORE_DYNAMIC_INIT);
3180	tree module_name_cst = NULL_TREE;
3181	if (!after_p)
3182	{
3183	pretty_printer module_name_pp;
3184	pp_string (&module_name_pp, main_input_filename);
3185
3186	module_name_cst = asan_pp_string (pp: &module_name_pp);
3187	module_name_cst = fold_convert (const_ptr_type_node,
3188	module_name_cst);
3189	}
3190
3191	return build_call_expr (fn, after_p ? 0 : 1, module_name_cst);
3192	}
3193
3194	/* Build
3195	struct __asan_global
3196	{
3197	const void *__beg;
3198	uptr __size;
3199	uptr __size_with_redzone;
3200	const void *__name;
3201	const void *__module_name;
3202	uptr __has_dynamic_init;
3203	__asan_global_source_location *__location;
3204	char *__odr_indicator;
3205	} type. */
3206
3207	static tree
3208	asan_global_struct (void)
3209	{
3210	static const char *field_names[]
3211	= { "__beg", "__size", "__size_with_redzone",
3212	"__name", "__module_name", "__has_dynamic_init", "__location",
3213	"__odr_indicator" };
3214	tree fields[ARRAY_SIZE (field_names)], ret;
3215	unsigned i;
3216
3217	ret = make_node (RECORD_TYPE);
3218	for (i = 0; i < ARRAY_SIZE (field_names); i++)
3219	{
3220	fields[i]
3221	= build_decl (UNKNOWN_LOCATION, FIELD_DECL,
3222	get_identifier (field_names[i]),
3223	(i == 0 || i == 3) ? const_ptr_type_node
3224	: pointer_sized_int_node);
3225	DECL_CONTEXT (fields[i]) = ret;
3226	if (i)
3227	DECL_CHAIN (fields[i - 1]) = fields[i];
3228	}
3229	tree type_decl = build_decl (input_location, TYPE_DECL,
3230	get_identifier ("__asan_global"), ret);
3231	DECL_IGNORED_P (type_decl) = 1;
3232	DECL_ARTIFICIAL (type_decl) = 1;
3233	TYPE_FIELDS (ret) = fields[0];
3234	TYPE_NAME (ret) = type_decl;
3235	TYPE_STUB_DECL (ret) = type_decl;
3236	TYPE_ARTIFICIAL (ret) = 1;
3237	layout_type (ret);
3238	return ret;
3239	}
3240
3241	/* Create and return odr indicator symbol for DECL.
3242	TYPE is __asan_global struct type as returned by asan_global_struct. */
3243
3244	static tree
3245	create_odr_indicator (tree decl, tree type)
3246	{
3247	char *name;
3248	tree uptr = TREE_TYPE (DECL_CHAIN (TYPE_FIELDS (type)));
3249	tree decl_name
3250	= (HAS_DECL_ASSEMBLER_NAME_P (decl) ? DECL_ASSEMBLER_NAME (decl)
3251	: DECL_NAME (decl));
3252	/* DECL_NAME theoretically might be NULL. Bail out with 0 in this case. */
3253	if (decl_name == NULL_TREE)
3254	return build_int_cst (uptr, 0);
3255	const char *dname = IDENTIFIER_POINTER (decl_name);
3256	if (HAS_DECL_ASSEMBLER_NAME_P (decl))
3257	dname = targetm.strip_name_encoding (dname);
3258	size_t len = strlen (s: dname) + sizeof ("__odr_asan_");
3259	name = XALLOCAVEC (char, len);
3260	snprintf (s: name, maxlen: len, format: "__odr_asan_%s", dname);
3261	#ifndef NO_DOT_IN_LABEL
3262	name[sizeof ("__odr_asan") - 1] = '.';
3263	#elif !defined(NO_DOLLAR_IN_LABEL)
3264	name[sizeof ("__odr_asan") - 1] = '$';
3265	#endif
3266	tree var = build_decl (UNKNOWN_LOCATION, VAR_DECL, get_identifier (name),
3267	char_type_node);
3268	TREE_ADDRESSABLE (var) = 1;
3269	TREE_READONLY (var) = 0;
3270	TREE_THIS_VOLATILE (var) = 1;
3271	DECL_ARTIFICIAL (var) = 1;
3272	DECL_IGNORED_P (var) = 1;
3273	TREE_STATIC (var) = 1;
3274	TREE_PUBLIC (var) = 1;
3275	DECL_VISIBILITY (var) = DECL_VISIBILITY (decl);
3276	DECL_VISIBILITY_SPECIFIED (var) = DECL_VISIBILITY_SPECIFIED (decl);
3277
3278	TREE_USED (var) = 1;
3279	tree ctor = build_constructor_va (TREE_TYPE (var), 1, NULL_TREE,
3280	build_int_cst (unsigned_type_node, 0));
3281	TREE_CONSTANT (ctor) = 1;
3282	TREE_STATIC (ctor) = 1;
3283	DECL_INITIAL (var) = ctor;
3284	DECL_ATTRIBUTES (var) = tree_cons (get_identifier ("asan odr indicator"),
3285	NULL, DECL_ATTRIBUTES (var));
3286	make_decl_rtl (var);
3287	varpool_node::finalize_decl (decl: var);
3288	return fold_convert (uptr, build_fold_addr_expr (var));
3289	}
3290
3291	/* Return true if DECL, a global var, might be overridden and needs
3292	an additional odr indicator symbol. */
3293
3294	static bool
3295	asan_needs_odr_indicator_p (tree decl)
3296	{
3297	/* Don't emit ODR indicators for kernel because:
3298	a) Kernel is written in C thus doesn't need ODR indicators.
3299	b) Some kernel code may have assumptions about symbols containing specific
3300	patterns in their names. Since ODR indicators contain original names
3301	of symbols they are emitted for, these assumptions would be broken for
3302	ODR indicator symbols. */
3303	return (!(flag_sanitize & SANITIZE_KERNEL_ADDRESS)
3304	&& !DECL_ARTIFICIAL (decl)
3305	&& !DECL_WEAK (decl)
3306	&& TREE_PUBLIC (decl));
3307	}
3308
3309	/* Append description of a single global DECL into vector V.
3310	TYPE is __asan_global struct type as returned by asan_global_struct. */
3311
3312	static void
3313	asan_add_global (tree decl, tree type, vec<constructor_elt, va_gc> *v)
3314	{
3315	tree init, uptr = TREE_TYPE (DECL_CHAIN (TYPE_FIELDS (type)));
3316	unsigned HOST_WIDE_INT size;
3317	tree str_cst, module_name_cst, refdecl = decl;
3318	vec<constructor_elt, va_gc> *vinner = NULL;
3319
3320	pretty_printer asan_pp, module_name_pp;
3321
3322	if (DECL_NAME (decl))
3323	pp_tree_identifier (&asan_pp, DECL_NAME (decl));
3324	else
3325	pp_string (&asan_pp, "<unknown>");
3326	str_cst = asan_pp_string (pp: &asan_pp);
3327
3328	if (!in_lto_p)
3329	pp_string (&module_name_pp, main_input_filename);
3330	else
3331	{
3332	const_tree tu = get_ultimate_context ((const_tree)decl);
3333	if (tu != NULL_TREE)
3334	pp_string (&module_name_pp, IDENTIFIER_POINTER (DECL_NAME (tu)));
3335	else
3336	pp_string (&module_name_pp, aux_base_name);
3337	}
3338
3339	module_name_cst = asan_pp_string (pp: &module_name_pp);
3340
3341	if (asan_needs_local_alias (decl))
3342	{
3343	char buf[20];
3344	ASM_GENERATE_INTERNAL_LABEL (buf, "LASAN", vec_safe_length (v) + 1);
3345	refdecl = build_decl (DECL_SOURCE_LOCATION (decl),
3346	VAR_DECL, get_identifier (buf), TREE_TYPE (decl));
3347	TREE_ADDRESSABLE (refdecl) = TREE_ADDRESSABLE (decl);
3348	TREE_READONLY (refdecl) = TREE_READONLY (decl);
3349	TREE_THIS_VOLATILE (refdecl) = TREE_THIS_VOLATILE (decl);
3350	DECL_NOT_GIMPLE_REG_P (refdecl) = DECL_NOT_GIMPLE_REG_P (decl);
3351	DECL_ARTIFICIAL (refdecl) = DECL_ARTIFICIAL (decl);
3352	DECL_IGNORED_P (refdecl) = DECL_IGNORED_P (decl);
3353	TREE_STATIC (refdecl) = 1;
3354	TREE_PUBLIC (refdecl) = 0;
3355	TREE_USED (refdecl) = 1;
3356	assemble_alias (refdecl, DECL_ASSEMBLER_NAME (decl));
3357	}
3358
3359	tree odr_indicator_ptr
3360	= (asan_needs_odr_indicator_p (decl) ? create_odr_indicator (decl, type)
3361	: build_int_cst (uptr, 0));
3362	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE,
3363	fold_convert (const_ptr_type_node,
3364	build_fold_addr_expr (refdecl)));
3365	size = tree_to_uhwi (DECL_SIZE_UNIT (decl));
3366	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE, build_int_cst (uptr, size));
3367	size += asan_red_zone_size (size);
3368	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE, build_int_cst (uptr, size));
3369	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE,
3370	fold_convert (const_ptr_type_node, str_cst));
3371	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE,
3372	fold_convert (const_ptr_type_node, module_name_cst));
3373	varpool_node *vnode = varpool_node::get (decl);
3374	int has_dynamic_init = 0;
3375	/* FIXME: Enable initialization order fiasco detection in LTO mode once
3376	proper fix for PR 79061 will be applied. */
3377	if (!in_lto_p)
3378	has_dynamic_init = vnode ? vnode->dynamically_initialized : 0;
3379	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE,
3380	build_int_cst (uptr, has_dynamic_init));
3381	tree locptr = NULL_TREE;
3382	location_t loc = DECL_SOURCE_LOCATION (decl);
3383	expanded_location xloc = expand_location (loc);
3384	if (xloc.file != NULL)
3385	{
3386	static int lasanloccnt = 0;
3387	char buf[25];
3388	ASM_GENERATE_INTERNAL_LABEL (buf, "LASANLOC", ++lasanloccnt);
3389	tree var = build_decl (UNKNOWN_LOCATION, VAR_DECL, get_identifier (buf),
3390	ubsan_get_source_location_type ());
3391	TREE_STATIC (var) = 1;
3392	TREE_PUBLIC (var) = 0;
3393	DECL_ARTIFICIAL (var) = 1;
3394	DECL_IGNORED_P (var) = 1;
3395	pretty_printer filename_pp;
3396	pp_string (&filename_pp, xloc.file);
3397	tree str = asan_pp_string (pp: &filename_pp);
3398	tree ctor = build_constructor_va (TREE_TYPE (var), 3,
3399	NULL_TREE, str, NULL_TREE,
3400	build_int_cst (unsigned_type_node,
3401	xloc.line), NULL_TREE,
3402	build_int_cst (unsigned_type_node,
3403	xloc.column));
3404	TREE_CONSTANT (ctor) = 1;
3405	TREE_STATIC (ctor) = 1;
3406	DECL_INITIAL (var) = ctor;
3407	varpool_node::finalize_decl (decl: var);
3408	locptr = fold_convert (uptr, build_fold_addr_expr (var));
3409	}
3410	else
3411	locptr = build_int_cst (uptr, 0);
3412	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE, locptr);
3413	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE, odr_indicator_ptr);
3414	init = build_constructor (type, vinner);
3415	CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, init);
3416	}
3417
3418	/* Initialize sanitizer.def builtins if the FE hasn't initialized them. */
3419	void
3420	initialize_sanitizer_builtins (void)
3421	{
3422	tree decl;
3423
3424	if (builtin_decl_implicit_p (fncode: BUILT_IN_ASAN_INIT))
3425	return;
3426
3427	tree BT_FN_VOID = build_function_type_list (void_type_node, NULL_TREE);
3428	tree BT_FN_VOID_PTR
3429	= build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
3430	tree BT_FN_VOID_CONST_PTR
3431	= build_function_type_list (void_type_node, const_ptr_type_node, NULL_TREE);
3432	tree BT_FN_VOID_PTR_PTR
3433	= build_function_type_list (void_type_node, ptr_type_node,
3434	ptr_type_node, NULL_TREE);
3435	tree BT_FN_VOID_PTR_PTR_PTR
3436	= build_function_type_list (void_type_node, ptr_type_node,
3437	ptr_type_node, ptr_type_node, NULL_TREE);
3438	tree BT_FN_VOID_PTR_PTRMODE
3439	= build_function_type_list (void_type_node, ptr_type_node,
3440	pointer_sized_int_node, NULL_TREE);
3441	tree BT_FN_VOID_INT
3442	= build_function_type_list (void_type_node, integer_type_node, NULL_TREE);
3443	tree BT_FN_SIZE_CONST_PTR_INT
3444	= build_function_type_list (size_type_node, const_ptr_type_node,
3445	integer_type_node, NULL_TREE);
3446
3447	tree BT_FN_VOID_UINT8_UINT8
3448	= build_function_type_list (void_type_node, unsigned_char_type_node,
3449	unsigned_char_type_node, NULL_TREE);
3450	tree BT_FN_VOID_UINT16_UINT16
3451	= build_function_type_list (void_type_node, uint16_type_node,
3452	uint16_type_node, NULL_TREE);
3453	tree BT_FN_VOID_UINT32_UINT32
3454	= build_function_type_list (void_type_node, uint32_type_node,
3455	uint32_type_node, NULL_TREE);
3456	tree BT_FN_VOID_UINT64_UINT64
3457	= build_function_type_list (void_type_node, uint64_type_node,
3458	uint64_type_node, NULL_TREE);
3459	tree BT_FN_VOID_FLOAT_FLOAT
3460	= build_function_type_list (void_type_node, float_type_node,
3461	float_type_node, NULL_TREE);
3462	tree BT_FN_VOID_DOUBLE_DOUBLE
3463	= build_function_type_list (void_type_node, double_type_node,
3464	double_type_node, NULL_TREE);
3465	tree BT_FN_VOID_UINT64_PTR
3466	= build_function_type_list (void_type_node, uint64_type_node,
3467	ptr_type_node, NULL_TREE);
3468
3469	tree BT_FN_PTR_CONST_PTR_UINT8
3470	= build_function_type_list (ptr_type_node, const_ptr_type_node,
3471	unsigned_char_type_node, NULL_TREE);
3472	tree BT_FN_VOID_PTR_UINT8_PTRMODE
3473	= build_function_type_list (void_type_node, ptr_type_node,
3474	unsigned_char_type_node,
3475	pointer_sized_int_node, NULL_TREE);
3476
3477	tree BT_FN_BOOL_VPTR_PTR_IX_INT_INT[5];
3478	tree BT_FN_IX_CONST_VPTR_INT[5];
3479	tree BT_FN_IX_VPTR_IX_INT[5];
3480	tree BT_FN_VOID_VPTR_IX_INT[5];
3481	tree vptr
3482	= build_pointer_type (build_qualified_type (void_type_node,
3483	TYPE_QUAL_VOLATILE));
3484	tree cvptr
3485	= build_pointer_type (build_qualified_type (void_type_node,
3486	TYPE_QUAL_VOLATILE
3487	|TYPE_QUAL_CONST));
3488	tree boolt
3489	= lang_hooks.types.type_for_size (BOOL_TYPE_SIZE, 1);
3490	int i;
3491	for (i = 0; i < 5; i++)
3492	{
3493	tree ix = build_nonstandard_integer_type (BITS_PER_UNIT * (1 << i), 1);
3494	BT_FN_BOOL_VPTR_PTR_IX_INT_INT[i]
3495	= build_function_type_list (boolt, vptr, ptr_type_node, ix,
3496	integer_type_node, integer_type_node,
3497	NULL_TREE);
3498	BT_FN_IX_CONST_VPTR_INT[i]
3499	= build_function_type_list (ix, cvptr, integer_type_node, NULL_TREE);
3500	BT_FN_IX_VPTR_IX_INT[i]
3501	= build_function_type_list (ix, vptr, ix, integer_type_node,
3502	NULL_TREE);
3503	BT_FN_VOID_VPTR_IX_INT[i]
3504	= build_function_type_list (void_type_node, vptr, ix,
3505	integer_type_node, NULL_TREE);
3506	}
3507	#define BT_FN_BOOL_VPTR_PTR_I1_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[0]
3508	#define BT_FN_I1_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[0]
3509	#define BT_FN_I1_VPTR_I1_INT BT_FN_IX_VPTR_IX_INT[0]
3510	#define BT_FN_VOID_VPTR_I1_INT BT_FN_VOID_VPTR_IX_INT[0]
3511	#define BT_FN_BOOL_VPTR_PTR_I2_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[1]
3512	#define BT_FN_I2_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[1]
3513	#define BT_FN_I2_VPTR_I2_INT BT_FN_IX_VPTR_IX_INT[1]
3514	#define BT_FN_VOID_VPTR_I2_INT BT_FN_VOID_VPTR_IX_INT[1]
3515	#define BT_FN_BOOL_VPTR_PTR_I4_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[2]
3516	#define BT_FN_I4_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[2]
3517	#define BT_FN_I4_VPTR_I4_INT BT_FN_IX_VPTR_IX_INT[2]
3518	#define BT_FN_VOID_VPTR_I4_INT BT_FN_VOID_VPTR_IX_INT[2]
3519	#define BT_FN_BOOL_VPTR_PTR_I8_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[3]
3520	#define BT_FN_I8_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[3]
3521	#define BT_FN_I8_VPTR_I8_INT BT_FN_IX_VPTR_IX_INT[3]
3522	#define BT_FN_VOID_VPTR_I8_INT BT_FN_VOID_VPTR_IX_INT[3]
3523	#define BT_FN_BOOL_VPTR_PTR_I16_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[4]
3524	#define BT_FN_I16_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[4]
3525	#define BT_FN_I16_VPTR_I16_INT BT_FN_IX_VPTR_IX_INT[4]
3526	#define BT_FN_VOID_VPTR_I16_INT BT_FN_VOID_VPTR_IX_INT[4]
3527	#undef ATTR_NOTHROW_LIST
3528	#define ATTR_NOTHROW_LIST ECF_NOTHROW
3529	#undef ATTR_NOTHROW_LEAF_LIST
3530	#define ATTR_NOTHROW_LEAF_LIST ECF_NOTHROW | ECF_LEAF
3531	#undef ATTR_TMPURE_NOTHROW_LEAF_LIST
3532	#define ATTR_TMPURE_NOTHROW_LEAF_LIST ECF_TM_PURE | ATTR_NOTHROW_LEAF_LIST
3533	#undef ATTR_NORETURN_NOTHROW_LEAF_LIST
3534	#define ATTR_NORETURN_NOTHROW_LEAF_LIST ECF_NORETURN | ATTR_NOTHROW_LEAF_LIST
3535	#undef ATTR_CONST_NORETURN_NOTHROW_LEAF_LIST
3536	#define ATTR_CONST_NORETURN_NOTHROW_LEAF_LIST \
3537	ECF_CONST | ATTR_NORETURN_NOTHROW_LEAF_LIST
3538	#undef ATTR_TMPURE_NORETURN_NOTHROW_LEAF_LIST
3539	#define ATTR_TMPURE_NORETURN_NOTHROW_LEAF_LIST \
3540	ECF_TM_PURE | ATTR_NORETURN_NOTHROW_LEAF_LIST
3541	#undef ATTR_COLD_NOTHROW_LEAF_LIST
3542	#define ATTR_COLD_NOTHROW_LEAF_LIST \
3543	/* ECF_COLD missing */ ATTR_NOTHROW_LEAF_LIST
3544	#undef ATTR_COLD_NORETURN_NOTHROW_LEAF_LIST
3545	#define ATTR_COLD_NORETURN_NOTHROW_LEAF_LIST \
3546	/* ECF_COLD missing */ ATTR_NORETURN_NOTHROW_LEAF_LIST
3547	#undef ATTR_COLD_CONST_NORETURN_NOTHROW_LEAF_LIST
3548	#define ATTR_COLD_CONST_NORETURN_NOTHROW_LEAF_LIST \
3549	/* ECF_COLD missing */ ATTR_CONST_NORETURN_NOTHROW_LEAF_LIST
3550	#undef ATTR_PURE_NOTHROW_LEAF_LIST
3551	#define ATTR_PURE_NOTHROW_LEAF_LIST ECF_PURE | ATTR_NOTHROW_LEAF_LIST
3552	#undef DEF_BUILTIN_STUB
3553	#define DEF_BUILTIN_STUB(ENUM, NAME)
3554	#undef DEF_SANITIZER_BUILTIN_1
3555	#define DEF_SANITIZER_BUILTIN_1(ENUM, NAME, TYPE, ATTRS) \
3556	do { \
3557	decl = add_builtin_function ("__builtin_" NAME, TYPE, ENUM, \
3558	BUILT_IN_NORMAL, NAME, NULL_TREE); \
3559	set_call_expr_flags (decl, ATTRS); \
3560	set_builtin_decl (ENUM, decl, true); \
3561	} while (0)
3562	#undef DEF_SANITIZER_BUILTIN
3563	#define DEF_SANITIZER_BUILTIN(ENUM, NAME, TYPE, ATTRS) \
3564	DEF_SANITIZER_BUILTIN_1 (ENUM, NAME, TYPE, ATTRS);
3565
3566	#include "sanitizer.def"
3567
3568	/* -fsanitize=object-size uses __builtin_dynamic_object_size and
3569	__builtin_object_size, but they might not be available for e.g. Fortran at
3570	this point. We use DEF_SANITIZER_BUILTIN here only as a convenience
3571	macro. */
3572	if (flag_sanitize & SANITIZE_OBJECT_SIZE)
3573	{
3574	if (!builtin_decl_implicit_p (fncode: BUILT_IN_OBJECT_SIZE))
3575	DEF_SANITIZER_BUILTIN_1 (BUILT_IN_OBJECT_SIZE, "object_size",
3576	BT_FN_SIZE_CONST_PTR_INT,
3577	ATTR_PURE_NOTHROW_LEAF_LIST);
3578	if (!builtin_decl_implicit_p (fncode: BUILT_IN_DYNAMIC_OBJECT_SIZE))
3579	DEF_SANITIZER_BUILTIN_1 (BUILT_IN_DYNAMIC_OBJECT_SIZE,
3580	"dynamic_object_size",
3581	BT_FN_SIZE_CONST_PTR_INT,
3582	ATTR_PURE_NOTHROW_LEAF_LIST);
3583	}
3584
3585	#undef DEF_SANITIZER_BUILTIN_1
3586	#undef DEF_SANITIZER_BUILTIN
3587	#undef DEF_BUILTIN_STUB
3588	}
3589
3590	/* Called via htab_traverse. Count number of emitted
3591	STRING_CSTs in the constant hash table. */
3592
3593	int
3594	count_string_csts (constant_descriptor_tree **slot,
3595	unsigned HOST_WIDE_INT *data)
3596	{
3597	struct constant_descriptor_tree *desc = *slot;
3598	if (TREE_CODE (desc->value) == STRING_CST
3599	&& TREE_ASM_WRITTEN (desc->value)
3600	&& asan_protect_global (decl: desc->value))
3601	++*data;
3602	return 1;
3603	}
3604
3605	/* Helper structure to pass two parameters to
3606	add_string_csts. */
3607
3608	struct asan_add_string_csts_data
3609	{
3610	tree type;
3611	vec<constructor_elt, va_gc> *v;
3612	};
3613
3614	/* Called via hash_table::traverse. Call asan_add_global
3615	on emitted STRING_CSTs from the constant hash table. */
3616
3617	int
3618	add_string_csts (constant_descriptor_tree **slot,
3619	asan_add_string_csts_data *aascd)
3620	{
3621	struct constant_descriptor_tree *desc = *slot;
3622	if (TREE_CODE (desc->value) == STRING_CST
3623	&& TREE_ASM_WRITTEN (desc->value)
3624	&& asan_protect_global (decl: desc->value))
3625	{
3626	asan_add_global (SYMBOL_REF_DECL (XEXP (desc->rtl, 0)),
3627	type: aascd->type, v: aascd->v);
3628	}
3629	return 1;
3630	}
3631
3632	/* Needs to be GTY(()), because cgraph_build_static_cdtor may
3633	invoke ggc_collect. */
3634	static GTY(()) tree asan_ctor_statements;
3635
3636	/* Module-level instrumentation.
3637	- Insert __asan_init_vN() into the list of CTORs.
3638	- TODO: insert redzones around globals.
3639	*/
3640
3641	void
3642	asan_finish_file (void)
3643	{
3644	varpool_node *vnode;
3645	unsigned HOST_WIDE_INT gcount = 0;
3646
3647	if (shadow_ptr_types[0] == NULL_TREE)
3648	asan_init_shadow_ptr_types ();
3649	/* Avoid instrumenting code in the asan ctors/dtors.
3650	We don't need to insert padding after the description strings,
3651	nor after .LASAN* array. */
3652	flag_sanitize &= ~SANITIZE_ADDRESS;
3653
3654	/* For user-space we want asan constructors to run first.
3655	Linux kernel does not support priorities other than default, and the only
3656	other user of constructors is coverage. So we run with the default
3657	priority. */
3658	int priority = flag_sanitize & SANITIZE_USER_ADDRESS
3659	? MAX_RESERVED_INIT_PRIORITY - 1 : DEFAULT_INIT_PRIORITY;
3660
3661	if (flag_sanitize & SANITIZE_USER_ADDRESS)
3662	{
3663	tree fn = builtin_decl_implicit (fncode: BUILT_IN_ASAN_INIT);
3664	append_to_statement_list (build_call_expr (fn, 0), &asan_ctor_statements);
3665	fn = builtin_decl_implicit (fncode: BUILT_IN_ASAN_VERSION_MISMATCH_CHECK);
3666	append_to_statement_list (build_call_expr (fn, 0), &asan_ctor_statements);
3667	}
3668	FOR_EACH_DEFINED_VARIABLE (vnode)
3669	if (TREE_ASM_WRITTEN (vnode->decl)
3670	&& asan_protect_global (decl: vnode->decl))
3671	++gcount;
3672	hash_table<tree_descriptor_hasher> *const_desc_htab = constant_pool_htab ();
3673	const_desc_htab->traverse<unsigned HOST_WIDE_INT *, count_string_csts>
3674	(argument: &gcount);
3675	if (gcount)
3676	{
3677	tree type = asan_global_struct (), var, ctor;
3678	tree dtor_statements = NULL_TREE;
3679	vec<constructor_elt, va_gc> *v;
3680	char buf[20];
3681
3682	type = build_array_type_nelts (type, gcount);
3683	ASM_GENERATE_INTERNAL_LABEL (buf, "LASAN", 0);
3684	var = build_decl (UNKNOWN_LOCATION, VAR_DECL, get_identifier (buf),
3685	type);
3686	TREE_STATIC (var) = 1;
3687	TREE_PUBLIC (var) = 0;
3688	DECL_ARTIFICIAL (var) = 1;
3689	DECL_IGNORED_P (var) = 1;
3690	vec_alloc (v, nelems: gcount);
3691	FOR_EACH_DEFINED_VARIABLE (vnode)
3692	if (TREE_ASM_WRITTEN (vnode->decl)
3693	&& asan_protect_global (decl: vnode->decl))
3694	asan_add_global (decl: vnode->decl, TREE_TYPE (type), v);
3695	struct asan_add_string_csts_data aascd;
3696	aascd.type = TREE_TYPE (type);
3697	aascd.v = v;
3698	const_desc_htab->traverse<asan_add_string_csts_data *, add_string_csts>
3699	(argument: &aascd);
3700	ctor = build_constructor (type, v);
3701	TREE_CONSTANT (ctor) = 1;
3702	TREE_STATIC (ctor) = 1;
3703	DECL_INITIAL (var) = ctor;
3704	SET_DECL_ALIGN (var, MAX (DECL_ALIGN (var),
3705	ASAN_SHADOW_GRANULARITY * BITS_PER_UNIT));
3706
3707	varpool_node::finalize_decl (decl: var);
3708
3709	tree fn = builtin_decl_implicit (fncode: BUILT_IN_ASAN_REGISTER_GLOBALS);
3710	tree gcount_tree = build_int_cst (pointer_sized_int_node, gcount);
3711	append_to_statement_list (build_call_expr (fn, 2,
3712	build_fold_addr_expr (var),
3713	gcount_tree),
3714	&asan_ctor_statements);
3715
3716	fn = builtin_decl_implicit (fncode: BUILT_IN_ASAN_UNREGISTER_GLOBALS);
3717	append_to_statement_list (build_call_expr (fn, 2,
3718	build_fold_addr_expr (var),
3719	gcount_tree),
3720	&dtor_statements);
3721	cgraph_build_static_cdtor (which: 'D', body: dtor_statements, priority);
3722	}
3723	if (asan_ctor_statements)
3724	cgraph_build_static_cdtor (which: 'I', body: asan_ctor_statements, priority);
3725	flag_sanitize |= SANITIZE_ADDRESS;
3726	}
3727
3728	/* Poison or unpoison (depending on IS_CLOBBER variable) shadow memory based
3729	on SHADOW address. Newly added statements will be added to ITER with
3730	given location LOC. We mark SIZE bytes in shadow memory, where
3731	LAST_CHUNK_SIZE is greater than zero in situation where we are at the
3732	end of a variable. */
3733
3734	static void
3735	asan_store_shadow_bytes (gimple_stmt_iterator *iter, location_t loc,
3736	tree shadow,
3737	unsigned HOST_WIDE_INT base_addr_offset,
3738	bool is_clobber, unsigned size,
3739	unsigned last_chunk_size)
3740	{
3741	tree shadow_ptr_type;
3742
3743	switch (size)
3744	{
3745	case 1:
3746	shadow_ptr_type = shadow_ptr_types[0];
3747	break;
3748	case 2:
3749	shadow_ptr_type = shadow_ptr_types[1];
3750	break;
3751	case 4:
3752	shadow_ptr_type = shadow_ptr_types[2];
3753	break;
3754	default:
3755	gcc_unreachable ();
3756	}
3757
3758	unsigned char c = (char) is_clobber ? ASAN_STACK_MAGIC_USE_AFTER_SCOPE : 0;
3759	unsigned HOST_WIDE_INT val = 0;
3760	unsigned last_pos = size;
3761	if (last_chunk_size && !is_clobber)
3762	last_pos = BYTES_BIG_ENDIAN ? 0 : size - 1;
3763	for (unsigned i = 0; i < size; ++i)
3764	{
3765	unsigned char shadow_c = c;
3766	if (i == last_pos)
3767	shadow_c = last_chunk_size;
3768	val |= (unsigned HOST_WIDE_INT) shadow_c << (BITS_PER_UNIT * i);
3769	}
3770
3771	/* Handle last chunk in unpoisoning. */
3772	tree magic = build_int_cst (TREE_TYPE (shadow_ptr_type), val);
3773
3774	tree dest = build2 (MEM_REF, TREE_TYPE (shadow_ptr_type), shadow,
3775	build_int_cst (shadow_ptr_type, base_addr_offset));
3776
3777	gimple *g = gimple_build_assign (dest, magic);
3778	gimple_set_location (g, location: loc);
3779	gsi_insert_after (iter, g, GSI_NEW_STMT);
3780	}
3781
3782	/* Expand the ASAN_MARK builtins. */
3783
3784	bool
3785	asan_expand_mark_ifn (gimple_stmt_iterator *iter)
3786	{
3787	gimple *g = gsi_stmt (i: *iter);
3788	location_t loc = gimple_location (g);
3789	HOST_WIDE_INT flag = tree_to_shwi (gimple_call_arg (gs: g, index: 0));
3790	bool is_poison = ((asan_mark_flags)flag) == ASAN_MARK_POISON;
3791
3792	tree base = gimple_call_arg (gs: g, index: 1);
3793	gcc_checking_assert (TREE_CODE (base) == ADDR_EXPR);
3794	tree decl = TREE_OPERAND (base, 0);
3795
3796	/* For a nested function, we can have: ASAN_MARK (2, &FRAME.2.fp_input, 4) */
3797	if (TREE_CODE (decl) == COMPONENT_REF
3798	&& DECL_NONLOCAL_FRAME (TREE_OPERAND (decl, 0)))
3799	decl = TREE_OPERAND (decl, 0);
3800
3801	gcc_checking_assert (TREE_CODE (decl) == VAR_DECL);
3802
3803	if (hwasan_sanitize_p ())
3804	{
3805	gcc_assert (param_hwasan_instrument_stack);
3806	gimple_seq stmts = NULL;
3807	/* Here we swap ASAN_MARK calls for HWASAN_MARK.
3808	This is because we are using the approach of using ASAN_MARK as a
3809	synonym until here.
3810	That approach means we don't yet have to duplicate all the special
3811	cases for ASAN_MARK and ASAN_POISON with the exact same handling but
3812	called HWASAN_MARK etc.
3813
3814	N.b. __asan_poison_stack_memory (which implements ASAN_MARK for ASAN)
3815	rounds the size up to its shadow memory granularity, while
3816	__hwasan_tag_memory (which implements the same for HWASAN) does not.
3817	Hence we emit HWASAN_MARK with an aligned size unlike ASAN_MARK. */
3818	tree len = gimple_call_arg (gs: g, index: 2);
3819	tree new_len = gimple_build_round_up (seq: &stmts, loc, size_type_node, old_size: len,
3820	HWASAN_TAG_GRANULE_SIZE);
3821	gimple_build (seq: &stmts, loc, fn: CFN_HWASAN_MARK,
3822	void_type_node, args: gimple_call_arg (gs: g, index: 0),
3823	args: base, args: new_len);
3824	gsi_replace_with_seq (iter, stmts, true);
3825	return false;
3826	}
3827
3828	if (is_poison)
3829	{
3830	if (asan_handled_variables == NULL)
3831	asan_handled_variables = new hash_set<tree> (16);
3832	asan_handled_variables->add (k: decl);
3833	}
3834	tree len = gimple_call_arg (gs: g, index: 2);
3835
3836	gcc_assert (poly_int_tree_p (len));
3837
3838	g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
3839	NOP_EXPR, base);
3840	gimple_set_location (g, location: loc);
3841	gsi_replace (iter, g, false);
3842	tree base_addr = gimple_assign_lhs (gs: g);
3843
3844	/* Generate direct emission if size_in_bytes is small. */
3845	unsigned threshold = param_use_after_scope_direct_emission_threshold;
3846	if (tree_fits_uhwi_p (len) && tree_to_uhwi (len) <= threshold)
3847	{
3848	unsigned HOST_WIDE_INT size_in_bytes = tree_to_uhwi (len);
3849	const unsigned HOST_WIDE_INT shadow_size
3850	= shadow_mem_size (size: size_in_bytes);
3851	const unsigned int shadow_align
3852	= (get_pointer_alignment (base) / BITS_PER_UNIT) >> ASAN_SHADOW_SHIFT;
3853
3854	tree shadow = build_shadow_mem_access (gsi: iter, location: loc, base_addr,
3855	shadow_ptr_type: shadow_ptr_types[0], return_address: true);
3856
3857	for (unsigned HOST_WIDE_INT offset = 0; offset < shadow_size;)
3858	{
3859	unsigned size = 1;
3860	if (shadow_size - offset >= 4
3861	&& (!STRICT_ALIGNMENT || shadow_align >= 4))
3862	size = 4;
3863	else if (shadow_size - offset >= 2
3864	&& (!STRICT_ALIGNMENT || shadow_align >= 2))
3865	size = 2;
3866
3867	unsigned HOST_WIDE_INT last_chunk_size = 0;
3868	unsigned HOST_WIDE_INT s = (offset + size) * ASAN_SHADOW_GRANULARITY;
3869	if (s > size_in_bytes)
3870	last_chunk_size = ASAN_SHADOW_GRANULARITY - (s - size_in_bytes);
3871
3872	asan_store_shadow_bytes (iter, loc, shadow, base_addr_offset: offset, is_clobber: is_poison,
3873	size, last_chunk_size);
3874	offset += size;
3875	}
3876	}
3877	else
3878	{
3879	g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
3880	NOP_EXPR, len);
3881	gimple_set_location (g, location: loc);
3882	gsi_safe_insert_before (iter, g);
3883	tree sz_arg = gimple_assign_lhs (gs: g);
3884
3885	tree fun
3886	= builtin_decl_implicit (fncode: is_poison ? BUILT_IN_ASAN_POISON_STACK_MEMORY
3887	: BUILT_IN_ASAN_UNPOISON_STACK_MEMORY);
3888	g = gimple_build_call (fun, 2, base_addr, sz_arg);
3889	gimple_set_location (g, location: loc);
3890	gsi_insert_after (iter, g, GSI_NEW_STMT);
3891	}
3892
3893	return false;
3894	}
3895
3896	/* Expand the ASAN_{LOAD,STORE} builtins. */
3897
3898	bool
3899	asan_expand_check_ifn (gimple_stmt_iterator *iter, bool use_calls)
3900	{
3901	gcc_assert (!hwasan_sanitize_p ());
3902	gimple *g = gsi_stmt (i: *iter);
3903	location_t loc = gimple_location (g);
3904	bool recover_p;
3905	if (flag_sanitize & SANITIZE_USER_ADDRESS)
3906	recover_p = (flag_sanitize_recover & SANITIZE_USER_ADDRESS) != 0;
3907	else
3908	recover_p = (flag_sanitize_recover & SANITIZE_KERNEL_ADDRESS) != 0;
3909
3910	HOST_WIDE_INT flags = tree_to_shwi (gimple_call_arg (gs: g, index: 0));
3911	gcc_assert (flags < ASAN_CHECK_LAST);
3912	bool is_scalar_access = (flags & ASAN_CHECK_SCALAR_ACCESS) != 0;
3913	bool is_store = (flags & ASAN_CHECK_STORE) != 0;
3914	bool is_non_zero_len = (flags & ASAN_CHECK_NON_ZERO_LEN) != 0;
3915
3916	tree base = gimple_call_arg (gs: g, index: 1);
3917	tree len = gimple_call_arg (gs: g, index: 2);
3918	HOST_WIDE_INT align = tree_to_shwi (gimple_call_arg (gs: g, index: 3));
3919
3920	HOST_WIDE_INT size_in_bytes
3921	= is_scalar_access && tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
3922
3923	if (use_calls)
3924	{
3925	/* Instrument using callbacks. */
3926	gimple *g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
3927	NOP_EXPR, base);
3928	gimple_set_location (g, location: loc);
3929	gsi_insert_before (iter, g, GSI_SAME_STMT);
3930	tree base_addr = gimple_assign_lhs (gs: g);
3931
3932	int nargs;
3933	tree fun = check_func (is_store, recover_p, size_in_bytes, nargs: &nargs);
3934	if (nargs == 1)
3935	g = gimple_build_call (fun, 1, base_addr);
3936	else
3937	{
3938	gcc_assert (nargs == 2);
3939	g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
3940	NOP_EXPR, len);
3941	gimple_set_location (g, location: loc);
3942	gsi_insert_before (iter, g, GSI_SAME_STMT);
3943	tree sz_arg = gimple_assign_lhs (gs: g);
3944	g = gimple_build_call (fun, nargs, base_addr, sz_arg);
3945	}
3946	gimple_set_location (g, location: loc);
3947	gsi_replace (iter, g, false);
3948	return false;
3949	}
3950
3951	HOST_WIDE_INT real_size_in_bytes = size_in_bytes == -1 ? 1 : size_in_bytes;
3952
3953	tree shadow_ptr_type = shadow_ptr_types[real_size_in_bytes == 16 ? 1 : 0];
3954	tree shadow_type = TREE_TYPE (shadow_ptr_type);
3955
3956	gimple_stmt_iterator gsi = *iter;
3957
3958	if (!is_non_zero_len)
3959	{
3960	/* So, the length of the memory area to asan-protect is
3961	non-constant. Let's guard the generated instrumentation code
3962	like:
3963
3964	if (len != 0)
3965	{
3966	//asan instrumentation code goes here.
3967	}
3968	// falltrough instructions, starting with *ITER. */
3969
3970	g = gimple_build_cond (NE_EXPR,
3971	len,
3972	build_int_cst (TREE_TYPE (len), 0),
3973	NULL_TREE, NULL_TREE);
3974	gimple_set_location (g, location: loc);
3975
3976	basic_block then_bb, fallthrough_bb;
3977	insert_if_then_before_iter (cond: as_a <gcond *> (p: g), iter,
3978	/*then_more_likely_p=*/true,
3979	then_bb: &then_bb, fallthrough_bb: &fallthrough_bb);
3980	/* Note that fallthrough_bb starts with the statement that was
3981	pointed to by ITER. */
3982
3983	/* The 'then block' of the 'if (len != 0) condition is where
3984	we'll generate the asan instrumentation code now. */
3985	gsi = gsi_last_bb (bb: then_bb);
3986	}
3987
3988	/* Get an iterator on the point where we can add the condition
3989	statement for the instrumentation. */
3990	basic_block then_bb, else_bb;
3991	gsi = create_cond_insert_point (iter: &gsi, /*before_p*/false,
3992	/*then_more_likely_p=*/false,
3993	/*create_then_fallthru_edge*/recover_p,
3994	then_block: &then_bb,
3995	fallthrough_block: &else_bb);
3996
3997	g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
3998	NOP_EXPR, base);
3999	gimple_set_location (g, location: loc);
4000	gsi_insert_before (&gsi, g, GSI_NEW_STMT);
4001	tree base_addr = gimple_assign_lhs (gs: g);
4002
4003	tree t = NULL_TREE;
4004	if (real_size_in_bytes >= 8)
4005	{
4006	tree shadow = build_shadow_mem_access (gsi: &gsi, location: loc, base_addr,
4007	shadow_ptr_type);
4008	t = shadow;
4009	}
4010	else
4011	{
4012	/* Slow path for 1, 2 and 4 byte accesses. */
4013	/* Test (shadow != 0)
4014	& ((base_addr & 7) + (real_size_in_bytes - 1)) >= shadow). */
4015	tree shadow = build_shadow_mem_access (gsi: &gsi, location: loc, base_addr,
4016	shadow_ptr_type);
4017	gimple *shadow_test = build_assign (NE_EXPR, shadow, 0);
4018	gimple_seq seq = NULL;
4019	gimple_seq_add_stmt (&seq, shadow_test);
4020	/* Aligned (>= 8 bytes) can test just
4021	(real_size_in_bytes - 1 >= shadow), as base_addr & 7 is known
4022	to be 0. */
4023	if (align < 8)
4024	{
4025	gimple_seq_add_stmt (&seq, build_assign (BIT_AND_EXPR,
4026	base_addr, 7));
4027	gimple_seq_add_stmt (&seq,
4028	build_type_cast (shadow_type,
4029	gimple_seq_last (s: seq)));
4030	if (real_size_in_bytes > 1)
4031	gimple_seq_add_stmt (&seq,
4032	build_assign (PLUS_EXPR,
4033	gimple_seq_last (s: seq),
4034	real_size_in_bytes - 1));
4035	t = gimple_assign_lhs (gs: gimple_seq_last_stmt (s: seq));
4036	}
4037	else
4038	t = build_int_cst (shadow_type, real_size_in_bytes - 1);
4039	gimple_seq_add_stmt (&seq, build_assign (GE_EXPR, t, shadow));
4040	gimple_seq_add_stmt (&seq, build_assign (BIT_AND_EXPR, shadow_test,
4041	gimple_seq_last (s: seq)));
4042	t = gimple_assign_lhs (gs: gimple_seq_last (s: seq));
4043	gimple_seq_set_location (seq, loc);
4044	gsi_insert_seq_after (&gsi, seq, GSI_CONTINUE_LINKING);
4045
4046	/* For non-constant, misaligned or otherwise weird access sizes,
4047	check first and last byte. */
4048	if (size_in_bytes == -1)
4049	{
4050	g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
4051	MINUS_EXPR, len,
4052	build_int_cst (pointer_sized_int_node, 1));
4053	gimple_set_location (g, location: loc);
4054	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
4055	tree last = gimple_assign_lhs (gs: g);
4056	g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
4057	PLUS_EXPR, base_addr, last);
4058	gimple_set_location (g, location: loc);
4059	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
4060	tree base_end_addr = gimple_assign_lhs (gs: g);
4061
4062	tree shadow = build_shadow_mem_access (gsi: &gsi, location: loc, base_addr: base_end_addr,
4063	shadow_ptr_type);
4064	gimple *shadow_test = build_assign (NE_EXPR, shadow, 0);
4065	gimple_seq seq = NULL;
4066	gimple_seq_add_stmt (&seq, shadow_test);
4067	gimple_seq_add_stmt (&seq, build_assign (BIT_AND_EXPR,
4068	base_end_addr, 7));
4069	gimple_seq_add_stmt (&seq, build_type_cast (shadow_type,
4070	gimple_seq_last (s: seq)));
4071	gimple_seq_add_stmt (&seq, build_assign (GE_EXPR,
4072	gimple_seq_last (s: seq),
4073	shadow));
4074	gimple_seq_add_stmt (&seq, build_assign (BIT_AND_EXPR, shadow_test,
4075	gimple_seq_last (s: seq)));
4076	gimple_seq_add_stmt (&seq, build_assign (BIT_IOR_EXPR, t,
4077	gimple_seq_last (s: seq)));
4078	t = gimple_assign_lhs (gs: gimple_seq_last (s: seq));
4079	gimple_seq_set_location (seq, loc);
4080	gsi_insert_seq_after (&gsi, seq, GSI_CONTINUE_LINKING);
4081	}
4082	}
4083
4084	g = gimple_build_cond (NE_EXPR, t, build_int_cst (TREE_TYPE (t), 0),
4085	NULL_TREE, NULL_TREE);
4086	gimple_set_location (g, location: loc);
4087	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
4088
4089	/* Generate call to the run-time library (e.g. __asan_report_load8). */
4090	gsi = gsi_start_bb (bb: then_bb);
4091	int nargs;
4092	tree fun = report_error_func (is_store, recover_p, size_in_bytes, nargs: &nargs);
4093	g = gimple_build_call (fun, nargs, base_addr, len);
4094	gimple_set_location (g, location: loc);
4095	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
4096
4097	gsi_remove (iter, true);
4098	*iter = gsi_start_bb (bb: else_bb);
4099
4100	return true;
4101	}
4102
4103	/* Create ASAN shadow variable for a VAR_DECL which has been rewritten
4104	into SSA. Already seen VAR_DECLs are stored in SHADOW_VARS_MAPPING. */
4105
4106	static tree
4107	create_asan_shadow_var (tree var_decl,
4108	hash_map<tree, tree> &shadow_vars_mapping)
4109	{
4110	tree *slot = shadow_vars_mapping.get (k: var_decl);
4111	if (slot == NULL)
4112	{
4113	tree shadow_var = copy_node (var_decl);
4114
4115	copy_body_data id;
4116	memset (s: &id, c: 0, n: sizeof (copy_body_data));
4117	id.src_fn = id.dst_fn = current_function_decl;
4118	copy_decl_for_dup_finish (id: &id, decl: var_decl, copy: shadow_var);
4119
4120	DECL_ARTIFICIAL (shadow_var) = 1;
4121	DECL_IGNORED_P (shadow_var) = 1;
4122	DECL_SEEN_IN_BIND_EXPR_P (shadow_var) = 0;
4123	gimple_add_tmp_var (shadow_var);
4124
4125	shadow_vars_mapping.put (k: var_decl, v: shadow_var);
4126	return shadow_var;
4127	}
4128	else
4129	return *slot;
4130	}
4131
4132	/* Expand ASAN_POISON ifn. */
4133
4134	bool
4135	asan_expand_poison_ifn (gimple_stmt_iterator *iter,
4136	bool *need_commit_edge_insert,
4137	hash_map<tree, tree> &shadow_vars_mapping)
4138	{
4139	gimple *g = gsi_stmt (i: *iter);
4140	tree poisoned_var = gimple_call_lhs (gs: g);
4141	if (!poisoned_var || has_zero_uses (var: poisoned_var))
4142	{
4143	gsi_remove (iter, true);
4144	return true;
4145	}
4146
4147	if (SSA_NAME_VAR (poisoned_var) == NULL_TREE)
4148	SET_SSA_NAME_VAR_OR_IDENTIFIER (poisoned_var,
4149	create_tmp_var (TREE_TYPE (poisoned_var)));
4150
4151	tree shadow_var = create_asan_shadow_var (SSA_NAME_VAR (poisoned_var),
4152	shadow_vars_mapping);
4153
4154	bool recover_p;
4155	if (flag_sanitize & SANITIZE_USER_ADDRESS)
4156	recover_p = (flag_sanitize_recover & SANITIZE_USER_ADDRESS) != 0;
4157	else
4158	recover_p = (flag_sanitize_recover & SANITIZE_KERNEL_ADDRESS) != 0;
4159	tree size = DECL_SIZE_UNIT (shadow_var);
4160	gimple *poison_call
4161	= gimple_build_call_internal (IFN_ASAN_MARK, 3,
4162	build_int_cst (integer_type_node,
4163	ASAN_MARK_POISON),
4164	build_fold_addr_expr (shadow_var), size);
4165
4166	gimple *use;
4167	imm_use_iterator imm_iter;
4168	FOR_EACH_IMM_USE_STMT (use, imm_iter, poisoned_var)
4169	{
4170	if (is_gimple_debug (gs: use))
4171	continue;
4172
4173	int nargs;
4174	bool store_p = gimple_call_internal_p (gs: use, fn: IFN_ASAN_POISON_USE);
4175	gcall *call;
4176	if (hwasan_sanitize_p ())
4177	{
4178	tree fun = builtin_decl_implicit (fncode: BUILT_IN_HWASAN_TAG_MISMATCH4);
4179	/* NOTE: hwasan has no __hwasan_report_* functions like asan does.
4180	We use __hwasan_tag_mismatch4 with arguments that tell it the
4181	size of access and load to report all tag mismatches.
4182
4183	The arguments to this function are:
4184	Address of invalid access.
4185	Bitfield containing information about the access
4186	(access_info)
4187	Pointer to a frame of registers
4188	(for use in printing the contents of registers in a dump)
4189	Not used yet -- to be used by inline instrumentation.
4190	Size of access.
4191
4192	The access_info bitfield encodes the following pieces of
4193	information:
4194	- Is this a store or load?
4195	access_info & 0x10 => store
4196	- Should the program continue after reporting the error?
4197	access_info & 0x20 => recover
4198	- What size access is this (not used here since we can always
4199	pass the size in the last argument)
4200
4201	if (access_info & 0xf == 0xf)
4202	size is taken from last argument.
4203	else
4204	size == 1 << (access_info & 0xf)
4205
4206	The last argument contains the size of the access iff the
4207	access_info size indicator is 0xf (we always use this argument
4208	rather than storing the size in the access_info bitfield).
4209
4210	See the function definition `__hwasan_tag_mismatch4` in
4211	libsanitizer/hwasan for the full definition.
4212	*/
4213	unsigned access_info = (0x20 * recover_p)
4214	+ (0x10 * store_p)
4215	+ (0xf);
4216	call = gimple_build_call (fun, 4,
4217	build_fold_addr_expr (shadow_var),
4218	build_int_cst (pointer_sized_int_node,
4219	access_info),
4220	build_int_cst (pointer_sized_int_node, 0),
4221	size);
4222	}
4223	else
4224	{
4225	tree fun = report_error_func (is_store: store_p, recover_p, size_in_bytes: tree_to_uhwi (size),
4226	nargs: &nargs);
4227	call = gimple_build_call (fun, 1,
4228	build_fold_addr_expr (shadow_var));
4229	}
4230	gimple_set_location (g: call, location: gimple_location (g: use));
4231	gimple *call_to_insert = call;
4232
4233	/* The USE can be a gimple PHI node. If so, insert the call on
4234	all edges leading to the PHI node. */
4235	if (is_a <gphi *> (p: use))
4236	{
4237	gphi *phi = dyn_cast<gphi *> (p: use);
4238	for (unsigned i = 0; i < gimple_phi_num_args (gs: phi); ++i)
4239	if (gimple_phi_arg_def (gs: phi, index: i) == poisoned_var)
4240	{
4241	edge e = gimple_phi_arg_edge (phi, i);
4242
4243	/* Do not insert on an edge we can't split. */
4244	if (e->flags & EDGE_ABNORMAL)
4245	continue;
4246
4247	if (call_to_insert == NULL)
4248	call_to_insert = gimple_copy (call);
4249
4250	gsi_insert_seq_on_edge (e, call_to_insert);
4251	*need_commit_edge_insert = true;
4252	call_to_insert = NULL;
4253	}
4254	}
4255	else
4256	{
4257	gimple_stmt_iterator gsi = gsi_for_stmt (use);
4258	if (store_p)
4259	gsi_replace (&gsi, call, true);
4260	else
4261	gsi_insert_before (&gsi, call, GSI_NEW_STMT);
4262	}
4263	}
4264
4265	SSA_NAME_IS_DEFAULT_DEF (poisoned_var) = true;
4266	SSA_NAME_DEF_STMT (poisoned_var) = gimple_build_nop ();
4267	gsi_replace (iter, poison_call, false);
4268
4269	return true;
4270	}
4271
4272	/* Instrument the current function. */
4273
4274	static unsigned int
4275	asan_instrument (void)
4276	{
4277	if (hwasan_sanitize_p ())
4278	{
4279	transform_statements ();
4280	return 0;
4281	}
4282
4283	if (shadow_ptr_types[0] == NULL_TREE)
4284	asan_init_shadow_ptr_types ();
4285	transform_statements ();
4286	last_alloca_addr = NULL_TREE;
4287	return 0;
4288	}
4289
4290	static bool
4291	gate_asan (void)
4292	{
4293	return sanitize_flags_p (flag: SANITIZE_ADDRESS);
4294	}
4295
4296	namespace {
4297
4298	const pass_data pass_data_asan =
4299	{
4300	.type: GIMPLE_PASS, /* type */
4301	.name: "asan", /* name */
4302	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
4303	.tv_id: TV_NONE, /* tv_id */
4304	.properties_required: ( PROP_ssa | PROP_cfg | PROP_gimple_leh ), /* properties_required */
4305	.properties_provided: 0, /* properties_provided */
4306	.properties_destroyed: 0, /* properties_destroyed */
4307	.todo_flags_start: 0, /* todo_flags_start */
4308	TODO_update_ssa, /* todo_flags_finish */
4309	};
4310
4311	class pass_asan : public gimple_opt_pass
4312	{
4313	public:
4314	pass_asan (gcc::context *ctxt)
4315	: gimple_opt_pass (pass_data_asan, ctxt)
4316	{}
4317
4318	/* opt_pass methods: */
4319	opt_pass * clone () final override { return new pass_asan (m_ctxt); }
4320	bool gate (function *) final override
4321	{
4322	return gate_asan () || gate_hwasan ();
4323	}
4324	unsigned int execute (function *) final override
4325	{
4326	return asan_instrument ();
4327	}
4328
4329	}; // class pass_asan
4330
4331	} // anon namespace
4332
4333	gimple_opt_pass *
4334	make_pass_asan (gcc::context *ctxt)
4335	{
4336	return new pass_asan (ctxt);
4337	}
4338
4339	namespace {
4340
4341	const pass_data pass_data_asan_O0 =
4342	{
4343	.type: GIMPLE_PASS, /* type */
4344	.name: "asan0", /* name */
4345	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
4346	.tv_id: TV_NONE, /* tv_id */
4347	.properties_required: ( PROP_ssa | PROP_cfg | PROP_gimple_leh ), /* properties_required */
4348	.properties_provided: 0, /* properties_provided */
4349	.properties_destroyed: 0, /* properties_destroyed */
4350	.todo_flags_start: 0, /* todo_flags_start */
4351	TODO_update_ssa, /* todo_flags_finish */
4352	};
4353
4354	class pass_asan_O0 : public gimple_opt_pass
4355	{
4356	public:
4357	pass_asan_O0 (gcc::context *ctxt)
4358	: gimple_opt_pass (pass_data_asan_O0, ctxt)
4359	{}
4360
4361	/* opt_pass methods: */
4362	bool gate (function *) final override
4363	{
4364	return !optimize && (gate_asan () || gate_hwasan ());
4365	}
4366	unsigned int execute (function *) final override
4367	{
4368	return asan_instrument ();
4369	}
4370
4371	}; // class pass_asan_O0
4372
4373	} // anon namespace
4374
4375	gimple_opt_pass *
4376	make_pass_asan_O0 (gcc::context *ctxt)
4377	{
4378	return new pass_asan_O0 (ctxt);
4379	}
4380
4381	/* HWASAN */
4382
4383	/* For stack tagging:
4384
4385	Return the offset from the frame base tag that the "next" expanded object
4386	should have. */
4387	uint8_t
4388	hwasan_current_frame_tag ()
4389	{
4390	return hwasan_frame_tag_offset;
4391	}
4392
4393	/* For stack tagging:
4394
4395	Return the 'base pointer' for this function. If that base pointer has not
4396	yet been created then we create a register to hold it and record the insns
4397	to initialize the register in `hwasan_frame_base_init_seq` for later
4398	emission. */
4399	rtx
4400	hwasan_frame_base ()
4401	{
4402	if (! hwasan_frame_base_ptr)
4403	{
4404	start_sequence ();
4405	hwasan_frame_base_ptr
4406	= force_reg (Pmode,
4407	targetm.memtag.insert_random_tag (virtual_stack_vars_rtx,
4408	NULL_RTX));
4409	hwasan_frame_base_init_seq = get_insns ();
4410	end_sequence ();
4411	}
4412
4413	return hwasan_frame_base_ptr;
4414	}
4415
4416	/* For stack tagging:
4417
4418	Check whether this RTX is a standard pointer addressing the base of the
4419	stack variables for this frame. Returns true if the RTX is either
4420	virtual_stack_vars_rtx or hwasan_frame_base_ptr. */
4421	bool
4422	stack_vars_base_reg_p (rtx base)
4423	{
4424	return base == virtual_stack_vars_rtx || base == hwasan_frame_base_ptr;
4425	}
4426
4427	/* For stack tagging:
4428
4429	Emit frame base initialisation.
4430	If hwasan_frame_base has been used before here then
4431	hwasan_frame_base_init_seq contains the sequence of instructions to
4432	initialize it. This must be put just before the hwasan prologue, so we emit
4433	the insns before parm_birth_insn (which will point to the first instruction
4434	of the hwasan prologue if it exists).
4435
4436	We update `parm_birth_insn` to point to the start of this initialisation
4437	since that represents the end of the initialisation done by
4438	expand_function_{start,end} functions and we want to maintain that. */
4439	void
4440	hwasan_maybe_emit_frame_base_init ()
4441	{
4442	if (! hwasan_frame_base_init_seq)
4443	return;
4444	emit_insn_before (hwasan_frame_base_init_seq, parm_birth_insn);
4445	parm_birth_insn = hwasan_frame_base_init_seq;
4446	}
4447
4448	/* Record a compile-time constant size stack variable that HWASAN will need to
4449	tag. This record of the range of a stack variable will be used by
4450	`hwasan_emit_prologue` to emit the RTL at the start of each frame which will
4451	set tags in the shadow memory according to the assigned tag for each object.
4452
4453	The range that the object spans in stack space should be described by the
4454	bounds `untagged_base + nearest_offset` and
4455	`untagged_base + farthest_offset`.
4456	`tagged_base` is the base address which contains the "base frame tag" for
4457	this frame, and from which the value to address this object with will be
4458	calculated.
4459
4460	We record the `untagged_base` since the functions in the hwasan library we
4461	use to tag memory take pointers without a tag. */
4462	void
4463	hwasan_record_stack_var (rtx untagged_base, rtx tagged_base,
4464	poly_int64 nearest_offset, poly_int64 farthest_offset)
4465	{
4466	hwasan_stack_var cur_var;
4467	cur_var.untagged_base = untagged_base;
4468	cur_var.tagged_base = tagged_base;
4469	cur_var.nearest_offset = nearest_offset;
4470	cur_var.farthest_offset = farthest_offset;
4471	cur_var.tag_offset = hwasan_current_frame_tag ();
4472
4473	hwasan_tagged_stack_vars.safe_push (obj: cur_var);
4474	}
4475
4476	/* Return the RTX representing the farthest extent of the statically allocated
4477	stack objects for this frame. If hwasan_frame_base_ptr has not been
4478	initialized then we are not storing any static variables on the stack in
4479	this frame. In this case we return NULL_RTX to represent that.
4480
4481	Otherwise simply return virtual_stack_vars_rtx + frame_offset. */
4482	rtx
4483	hwasan_get_frame_extent ()
4484	{
4485	return (hwasan_frame_base_ptr
4486	? plus_constant (Pmode, virtual_stack_vars_rtx, frame_offset)
4487	: NULL_RTX);
4488	}
4489
4490	/* For stack tagging:
4491
4492	Increment the frame tag offset modulo the size a tag can represent. */
4493	void
4494	hwasan_increment_frame_tag ()
4495	{
4496	uint8_t tag_bits = HWASAN_TAG_SIZE;
4497	gcc_assert (HWASAN_TAG_SIZE
4498	<= sizeof (hwasan_frame_tag_offset) * CHAR_BIT);
4499	hwasan_frame_tag_offset = (hwasan_frame_tag_offset + 1) % (1 << tag_bits);
4500	/* The "background tag" of the stack is zero by definition.
4501	This is the tag that objects like parameters passed on the stack and
4502	spilled registers are given. It is handy to avoid this tag for objects
4503	whose tags we decide ourselves, partly to ensure that buffer overruns
4504	can't affect these important variables (e.g. saved link register, saved
4505	stack pointer etc) and partly to make debugging easier (everything with a
4506	tag of zero is space allocated automatically by the compiler).
4507
4508	This is not feasible when using random frame tags (the default
4509	configuration for hwasan) since the tag for the given frame is randomly
4510	chosen at runtime. In order to avoid any tags matching the stack
4511	background we would need to decide tag offsets at runtime instead of
4512	compile time (and pay the resulting performance cost).
4513
4514	When not using random base tags for each frame (i.e. when compiled with
4515	`--param hwasan-random-frame-tag=0`) the base tag for each frame is zero.
4516	This means the tag that each object gets is equal to the
4517	hwasan_frame_tag_offset used in determining it.
4518	When this is the case we *can* ensure no object gets the tag of zero by
4519	simply ensuring no object has the hwasan_frame_tag_offset of zero.
4520
4521	There is the extra complication that we only record the
4522	hwasan_frame_tag_offset here (which is the offset from the tag stored in
4523	the stack pointer). In the kernel, the tag in the stack pointer is 0xff
4524	rather than zero. This does not cause problems since tags of 0xff are
4525	never checked in the kernel. As mentioned at the beginning of this
4526	comment the background tag of the stack is zero by definition, which means
4527	that for the kernel we should skip offsets of both 0 and 1 from the stack
4528	pointer. Avoiding the offset of 0 ensures we use a tag which will be
4529	checked, avoiding the offset of 1 ensures we use a tag that is not the
4530	same as the background. */
4531	if (hwasan_frame_tag_offset == 0 && ! param_hwasan_random_frame_tag)
4532	hwasan_frame_tag_offset += 1;
4533	if (hwasan_frame_tag_offset == 1 && ! param_hwasan_random_frame_tag
4534	&& sanitize_flags_p (flag: SANITIZE_KERNEL_HWADDRESS))
4535	hwasan_frame_tag_offset += 1;
4536	}
4537
4538	/* Clear internal state for the next function.
4539	This function is called before variables on the stack get expanded, in
4540	`init_vars_expansion`. */
4541	void
4542	hwasan_record_frame_init ()
4543	{
4544	delete asan_used_labels;
4545	asan_used_labels = NULL;
4546
4547	/* If this isn't the case then some stack variable was recorded *before*
4548	hwasan_record_frame_init is called, yet *after* the hwasan prologue for
4549	the previous frame was emitted. Such stack variables would not have
4550	their shadow stack filled in. */
4551	gcc_assert (hwasan_tagged_stack_vars.is_empty ());
4552	hwasan_frame_base_ptr = NULL_RTX;
4553	hwasan_frame_base_init_seq = NULL;
4554
4555	/* When not using a random frame tag we can avoid the background stack
4556	color which gives the user a little better debug output upon a crash.
4557	Meanwhile, when using a random frame tag it will be nice to avoid adding
4558	tags for the first object since that is unnecessary extra work.
4559	Hence set the initial hwasan_frame_tag_offset to be 0 if using a random
4560	frame tag and 1 otherwise.
4561
4562	As described in hwasan_increment_frame_tag, in the kernel the stack
4563	pointer has the tag 0xff. That means that to avoid 0xff and 0 (the tag
4564	which the kernel does not check and the background tag respectively) we
4565	start with a tag offset of 2. */
4566	hwasan_frame_tag_offset = param_hwasan_random_frame_tag
4567	? 0
4568	: sanitize_flags_p (flag: SANITIZE_KERNEL_HWADDRESS) ? 2 : 1;
4569	}
4570
4571	/* For stack tagging:
4572	(Emits HWASAN equivalent of what is emitted by
4573	`asan_emit_stack_protection`).
4574
4575	Emits the extra prologue code to set the shadow stack as required for HWASAN
4576	stack instrumentation.
4577
4578	Uses the vector of recorded stack variables hwasan_tagged_stack_vars. When
4579	this function has completed hwasan_tagged_stack_vars is empty and all
4580	objects it had pointed to are deallocated. */
4581	void
4582	hwasan_emit_prologue ()
4583	{
4584	/* We need untagged base pointers since libhwasan only accepts untagged
4585	pointers in __hwasan_tag_memory. We need the tagged base pointer to obtain
4586	the base tag for an offset. */
4587
4588	if (hwasan_tagged_stack_vars.is_empty ())
4589	return;
4590
4591	poly_int64 bot = 0, top = 0;
4592	for (hwasan_stack_var &cur : hwasan_tagged_stack_vars)
4593	{
4594	poly_int64 nearest = cur.nearest_offset;
4595	poly_int64 farthest = cur.farthest_offset;
4596
4597	if (known_ge (nearest, farthest))
4598	{
4599	top = nearest;
4600	bot = farthest;
4601	}
4602	else
4603	{
4604	/* Given how these values are calculated, one must be known greater
4605	than the other. */
4606	gcc_assert (known_le (nearest, farthest));
4607	top = farthest;
4608	bot = nearest;
4609	}
4610	poly_int64 size = (top - bot);
4611
4612	/* Assert the edge of each variable is aligned to the HWASAN tag granule
4613	size. */
4614	gcc_assert (multiple_p (top, HWASAN_TAG_GRANULE_SIZE));
4615	gcc_assert (multiple_p (bot, HWASAN_TAG_GRANULE_SIZE));
4616	gcc_assert (multiple_p (size, HWASAN_TAG_GRANULE_SIZE));
4617
4618	rtx fn = init_one_libfunc ("__hwasan_tag_memory");
4619	rtx base_tag = targetm.memtag.extract_tag (cur.tagged_base, NULL_RTX);
4620	rtx tag = plus_constant (QImode, base_tag, cur.tag_offset);
4621	tag = hwasan_truncate_to_tag_size (tag, NULL_RTX);
4622
4623	rtx bottom = convert_memory_address (ptr_mode,
4624	plus_constant (Pmode,
4625	cur.untagged_base,
4626	bot));
4627	emit_library_call (fun: fn, fn_type: LCT_NORMAL, VOIDmode,
4628	arg1: bottom, arg1_mode: ptr_mode,
4629	arg2: tag, QImode,
4630	arg3: gen_int_mode (size, ptr_mode), arg3_mode: ptr_mode);
4631	}
4632	/* Clear the stack vars, we've emitted the prologue for them all now. */
4633	hwasan_tagged_stack_vars.truncate (size: 0);
4634	}
4635
4636	/* For stack tagging:
4637
4638	Return RTL insns to clear the tags between DYNAMIC and VARS pointers
4639	into the stack. These instructions should be emitted at the end of
4640	every function.
4641
4642	If `dynamic` is NULL_RTX then no insns are returned. */
4643	rtx_insn *
4644	hwasan_emit_untag_frame (rtx dynamic, rtx vars)
4645	{
4646	if (! dynamic)
4647	return NULL;
4648
4649	start_sequence ();
4650
4651	dynamic = convert_memory_address (ptr_mode, dynamic);
4652	vars = convert_memory_address (ptr_mode, vars);
4653
4654	rtx top_rtx;
4655	rtx bot_rtx;
4656	if (FRAME_GROWS_DOWNWARD)
4657	{
4658	top_rtx = vars;
4659	bot_rtx = dynamic;
4660	}
4661	else
4662	{
4663	top_rtx = dynamic;
4664	bot_rtx = vars;
4665	}
4666
4667	rtx size_rtx = expand_simple_binop (ptr_mode, MINUS, top_rtx, bot_rtx,
4668	NULL_RTX, /* unsignedp = */0,
4669	OPTAB_DIRECT);
4670
4671	rtx fn = init_one_libfunc ("__hwasan_tag_memory");
4672	emit_library_call (fun: fn, fn_type: LCT_NORMAL, VOIDmode,
4673	arg1: bot_rtx, arg1_mode: ptr_mode,
4674	HWASAN_STACK_BACKGROUND, QImode,
4675	arg3: size_rtx, arg3_mode: ptr_mode);
4676
4677	do_pending_stack_adjust ();
4678	rtx_insn *insns = get_insns ();
4679	end_sequence ();
4680	return insns;
4681	}
4682
4683	/* Needs to be GTY(()), because cgraph_build_static_cdtor may
4684	invoke ggc_collect. */
4685	static GTY(()) tree hwasan_ctor_statements;
4686
4687	/* Insert module initialization into this TU. This initialization calls the
4688	initialization code for libhwasan. */
4689	void
4690	hwasan_finish_file (void)
4691	{
4692	/* Do not emit constructor initialization for the kernel.
4693	(the kernel has its own initialization already). */
4694	if (flag_sanitize & SANITIZE_KERNEL_HWADDRESS)
4695	return;
4696
4697	/* Avoid instrumenting code in the hwasan constructors/destructors. */
4698	flag_sanitize &= ~SANITIZE_HWADDRESS;
4699	int priority = MAX_RESERVED_INIT_PRIORITY - 1;
4700	tree fn = builtin_decl_implicit (fncode: BUILT_IN_HWASAN_INIT);
4701	append_to_statement_list (build_call_expr (fn, 0), &hwasan_ctor_statements);
4702	cgraph_build_static_cdtor (which: 'I', body: hwasan_ctor_statements, priority);
4703	flag_sanitize |= SANITIZE_HWADDRESS;
4704	}
4705
4706	/* For stack tagging:
4707
4708	Truncate `tag` to the number of bits that a tag uses (i.e. to
4709	HWASAN_TAG_SIZE). Store the result in `target` if it's convenient. */
4710	rtx
4711	hwasan_truncate_to_tag_size (rtx tag, rtx target)
4712	{
4713	gcc_assert (GET_MODE (tag) == QImode);
4714	if (HWASAN_TAG_SIZE != GET_MODE_PRECISION (QImode))
4715	{
4716	gcc_assert (GET_MODE_PRECISION (QImode) > HWASAN_TAG_SIZE);
4717	rtx mask = gen_int_mode ((HOST_WIDE_INT_1U << HWASAN_TAG_SIZE) - 1,
4718	QImode);
4719	tag = expand_simple_binop (QImode, AND, tag, mask, target,
4720	/* unsignedp = */1, OPTAB_WIDEN);
4721	gcc_assert (tag);
4722	}
4723	return tag;
4724	}
4725
4726	/* Construct a function tree for __hwasan_{load,store}{1,2,4,8,16,_n}.
4727	IS_STORE is either 1 (for a store) or 0 (for a load). */
4728	static combined_fn
4729	hwasan_check_func (bool is_store, bool recover_p, HOST_WIDE_INT size_in_bytes,
4730	int *nargs)
4731	{
4732	static enum built_in_function check[2][2][6]
4733	= { { { BUILT_IN_HWASAN_LOAD1, BUILT_IN_HWASAN_LOAD2,
4734	BUILT_IN_HWASAN_LOAD4, BUILT_IN_HWASAN_LOAD8,
4735	BUILT_IN_HWASAN_LOAD16, BUILT_IN_HWASAN_LOADN },
4736	{ BUILT_IN_HWASAN_STORE1, BUILT_IN_HWASAN_STORE2,
4737	BUILT_IN_HWASAN_STORE4, BUILT_IN_HWASAN_STORE8,
4738	BUILT_IN_HWASAN_STORE16, BUILT_IN_HWASAN_STOREN } },
4739	{ { BUILT_IN_HWASAN_LOAD1_NOABORT,
4740	BUILT_IN_HWASAN_LOAD2_NOABORT,
4741	BUILT_IN_HWASAN_LOAD4_NOABORT,
4742	BUILT_IN_HWASAN_LOAD8_NOABORT,
4743	BUILT_IN_HWASAN_LOAD16_NOABORT,
4744	BUILT_IN_HWASAN_LOADN_NOABORT },
4745	{ BUILT_IN_HWASAN_STORE1_NOABORT,
4746	BUILT_IN_HWASAN_STORE2_NOABORT,
4747	BUILT_IN_HWASAN_STORE4_NOABORT,
4748	BUILT_IN_HWASAN_STORE8_NOABORT,
4749	BUILT_IN_HWASAN_STORE16_NOABORT,
4750	BUILT_IN_HWASAN_STOREN_NOABORT } } };
4751	if (size_in_bytes == -1)
4752	{
4753	*nargs = 2;
4754	return as_combined_fn (fn: check[recover_p][is_store][5]);
4755	}
4756	*nargs = 1;
4757	int size_log2 = exact_log2 (x: size_in_bytes);
4758	gcc_assert (size_log2 >= 0 && size_log2 <= 5);
4759	return as_combined_fn (fn: check[recover_p][is_store][size_log2]);
4760	}
4761
4762	/* Expand the HWASAN_{LOAD,STORE} builtins. */
4763	bool
4764	hwasan_expand_check_ifn (gimple_stmt_iterator *iter, bool)
4765	{
4766	gimple *g = gsi_stmt (i: *iter);
4767	location_t loc = gimple_location (g);
4768	bool recover_p;
4769	if (flag_sanitize & SANITIZE_USER_HWADDRESS)
4770	recover_p = (flag_sanitize_recover & SANITIZE_USER_HWADDRESS) != 0;
4771	else
4772	recover_p = (flag_sanitize_recover & SANITIZE_KERNEL_HWADDRESS) != 0;
4773
4774	HOST_WIDE_INT flags = tree_to_shwi (gimple_call_arg (gs: g, index: 0));
4775	gcc_assert (flags < ASAN_CHECK_LAST);
4776	bool is_scalar_access = (flags & ASAN_CHECK_SCALAR_ACCESS) != 0;
4777	bool is_store = (flags & ASAN_CHECK_STORE) != 0;
4778	bool is_non_zero_len = (flags & ASAN_CHECK_NON_ZERO_LEN) != 0;
4779
4780	tree base = gimple_call_arg (gs: g, index: 1);
4781	tree len = gimple_call_arg (gs: g, index: 2);
4782
4783	/* `align` is unused for HWASAN_CHECK, but we pass the argument anyway
4784	since that way the arguments match ASAN_CHECK. */
4785	/* HOST_WIDE_INT align = tree_to_shwi (gimple_call_arg (g, 3)); */
4786
4787	unsigned HOST_WIDE_INT size_in_bytes
4788	= is_scalar_access ? tree_to_shwi (len) : -1;
4789
4790	gimple_stmt_iterator gsi = *iter;
4791
4792	if (!is_non_zero_len)
4793	{
4794	/* So, the length of the memory area to hwasan-protect is
4795	non-constant. Let's guard the generated instrumentation code
4796	like:
4797
4798	if (len != 0)
4799	{
4800	// hwasan instrumentation code goes here.
4801	}
4802	// falltrough instructions, starting with *ITER. */
4803
4804	g = gimple_build_cond (NE_EXPR,
4805	len,
4806	build_int_cst (TREE_TYPE (len), 0),
4807	NULL_TREE, NULL_TREE);
4808	gimple_set_location (g, location: loc);
4809
4810	basic_block then_bb, fallthrough_bb;
4811	insert_if_then_before_iter (cond: as_a <gcond *> (p: g), iter,
4812	/*then_more_likely_p=*/true,
4813	then_bb: &then_bb, fallthrough_bb: &fallthrough_bb);
4814	/* Note that fallthrough_bb starts with the statement that was
4815	pointed to by ITER. */
4816
4817	/* The 'then block' of the 'if (len != 0) condition is where
4818	we'll generate the hwasan instrumentation code now. */
4819	gsi = gsi_last_bb (bb: then_bb);
4820	}
4821
4822	gimple_seq stmts = NULL;
4823	tree base_addr = gimple_build (seq: &stmts, loc, code: NOP_EXPR,
4824	pointer_sized_int_node, ops: base);
4825
4826	int nargs = 0;
4827	combined_fn fn
4828	= hwasan_check_func (is_store, recover_p, size_in_bytes, nargs: &nargs);
4829	if (nargs == 1)
4830	gimple_build (seq: &stmts, loc, fn, void_type_node, args: base_addr);
4831	else
4832	{
4833	gcc_assert (nargs == 2);
4834	tree sz_arg = gimple_build (seq: &stmts, loc, code: NOP_EXPR,
4835	pointer_sized_int_node, ops: len);
4836	gimple_build (seq: &stmts, loc, fn, void_type_node, args: base_addr, args: sz_arg);
4837	}
4838
4839	gsi_insert_seq_after (&gsi, stmts, GSI_NEW_STMT);
4840	gsi_remove (iter, true);
4841	*iter = gsi;
4842	return false;
4843	}
4844
4845	/* For stack tagging:
4846
4847	Dummy: the HWASAN_MARK internal function should only ever be in the code
4848	after the sanopt pass. */
4849	bool
4850	hwasan_expand_mark_ifn (gimple_stmt_iterator *)
4851	{
4852	gcc_unreachable ();
4853	}
4854
4855	bool
4856	gate_hwasan ()
4857	{
4858	return hwasan_sanitize_p ();
4859	}
4860
4861	#include "gt-asan.h"
4862