1	/* A pass for lowering trees to RTL.
2	Copyright (C) 2004-2024 Free Software Foundation, Inc.
3
4	This file is part of GCC.
5
6	GCC is free software; you can redistribute it and/or modify
7	it under the terms of the GNU General Public License as published by
8	the Free Software Foundation; either version 3, or (at your option)
9	any later version.
10
11	GCC is distributed in the hope that it will be useful,
12	but WITHOUT ANY WARRANTY; without even the implied warranty of
13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	GNU General Public License for more details.
15
16	You should have received a copy of the GNU General Public License
17	along with GCC; see the file COPYING3. If not see
18	<http://www.gnu.org/licenses/>. */
19
20	#include "config.h"
21	#include "system.h"
22	#include "coretypes.h"
23	#include "backend.h"
24	#include "target.h"
25	#include "rtl.h"
26	#include "tree.h"
27	#include "gimple.h"
28	#include "cfghooks.h"
29	#include "tree-pass.h"
30	#include "memmodel.h"
31	#include "tm_p.h"
32	#include "ssa.h"
33	#include "optabs.h"
34	#include "regs.h" /* For reg_renumber. */
35	#include "emit-rtl.h"
36	#include "recog.h"
37	#include "cgraph.h"
38	#include "diagnostic.h"
39	#include "fold-const.h"
40	#include "varasm.h"
41	#include "stor-layout.h"
42	#include "stmt.h"
43	#include "print-tree.h"
44	#include "cfgrtl.h"
45	#include "cfganal.h"
46	#include "cfgbuild.h"
47	#include "cfgcleanup.h"
48	#include "dojump.h"
49	#include "explow.h"
50	#include "calls.h"
51	#include "expr.h"
52	#include "internal-fn.h"
53	#include "tree-eh.h"
54	#include "gimple-iterator.h"
55	#include "gimple-expr.h"
56	#include "gimple-walk.h"
57	#include "tree-cfg.h"
58	#include "tree-dfa.h"
59	#include "tree-ssa.h"
60	#include "except.h"
61	#include "gimple-pretty-print.h"
62	#include "toplev.h"
63	#include "debug.h"
64	#include "tree-inline.h"
65	#include "value-prof.h"
66	#include "tree-ssa-live.h"
67	#include "tree-outof-ssa.h"
68	#include "cfgloop.h"
69	#include "insn-attr.h" /* For INSN_SCHEDULING. */
70	#include "stringpool.h"
71	#include "attribs.h"
72	#include "asan.h"
73	#include "tree-ssa-address.h"
74	#include "output.h"
75	#include "builtins.h"
76	#include "opts.h"
77
78	/* Some systems use __main in a way incompatible with its use in gcc, in these
79	cases use the macros NAME__MAIN to give a quoted symbol and SYMBOL__MAIN to
80	give the same symbol without quotes for an alternative entry point. You
81	must define both, or neither. */
82	#ifndef NAME__MAIN
83	#define NAME__MAIN "__main"
84	#endif
85
86	/* This variable holds information helping the rewriting of SSA trees
87	into RTL. */
88	struct ssaexpand SA;
89
90	/* This variable holds the currently expanded gimple statement for purposes
91	of comminucating the profile info to the builtin expanders. */
92	gimple *currently_expanding_gimple_stmt;
93
94	static rtx expand_debug_expr (tree);
95
96	static bool defer_stack_allocation (tree, bool);
97
98	static void record_alignment_for_reg_var (unsigned int);
99
100	/* Return an expression tree corresponding to the RHS of GIMPLE
101	statement STMT. */
102
103	tree
104	gimple_assign_rhs_to_tree (gimple *stmt)
105	{
106	tree t;
107	switch (gimple_assign_rhs_class (gs: stmt))
108	{
109	case GIMPLE_TERNARY_RHS:
110	t = build3 (gimple_assign_rhs_code (gs: stmt),
111	TREE_TYPE (gimple_assign_lhs (stmt)),
112	gimple_assign_rhs1 (gs: stmt), gimple_assign_rhs2 (gs: stmt),
113	gimple_assign_rhs3 (gs: stmt));
114	break;
115	case GIMPLE_BINARY_RHS:
116	t = build2 (gimple_assign_rhs_code (gs: stmt),
117	TREE_TYPE (gimple_assign_lhs (stmt)),
118	gimple_assign_rhs1 (gs: stmt), gimple_assign_rhs2 (gs: stmt));
119	break;
120	case GIMPLE_UNARY_RHS:
121	t = build1 (gimple_assign_rhs_code (gs: stmt),
122	TREE_TYPE (gimple_assign_lhs (stmt)),
123	gimple_assign_rhs1 (gs: stmt));
124	break;
125	case GIMPLE_SINGLE_RHS:
126	{
127	t = gimple_assign_rhs1 (gs: stmt);
128	/* Avoid modifying this tree in place below. */
129	if ((gimple_has_location (g: stmt) && CAN_HAVE_LOCATION_P (t)
130	&& gimple_location (g: stmt) != EXPR_LOCATION (t))
131	|| (gimple_block (g: stmt) && currently_expanding_to_rtl
132	&& EXPR_P (t)))
133	t = copy_node (t);
134	break;
135	}
136	default:
137	gcc_unreachable ();
138	}
139
140	if (gimple_has_location (g: stmt) && CAN_HAVE_LOCATION_P (t))
141	SET_EXPR_LOCATION (t, gimple_location (stmt));
142
143	return t;
144	}
145
146
147	#ifndef STACK_ALIGNMENT_NEEDED
148	#define STACK_ALIGNMENT_NEEDED 1
149	#endif
150
151	#define SSAVAR(x) (TREE_CODE (x) == SSA_NAME ? SSA_NAME_VAR (x) : x)
152
153	/* Choose either CUR or NEXT as the leader DECL for a partition.
154	Prefer ignored decls, to simplify debug dumps and reduce ambiguity
155	out of the same user variable being in multiple partitions (this is
156	less likely for compiler-introduced temps). */
157
158	static tree
159	leader_merge (tree cur, tree next)
160	{
161	if (cur == NULL || cur == next)
162	return next;
163
164	if (DECL_P (cur) && DECL_IGNORED_P (cur))
165	return cur;
166
167	if (DECL_P (next) && DECL_IGNORED_P (next))
168	return next;
169
170	return cur;
171	}
172
173	/* Associate declaration T with storage space X. If T is no
174	SSA name this is exactly SET_DECL_RTL, otherwise make the
175	partition of T associated with X. */
176	static inline void
177	set_rtl (tree t, rtx x)
178	{
179	gcc_checking_assert (!x
180	|| !(TREE_CODE (t) == SSA_NAME || is_gimple_reg (t))
181	|| (use_register_for_decl (t)
182	? (REG_P (x)
183	|| (GET_CODE (x) == CONCAT
184	&& (REG_P (XEXP (x, 0))
185	|| SUBREG_P (XEXP (x, 0)))
186	&& (REG_P (XEXP (x, 1))
187	|| SUBREG_P (XEXP (x, 1))))
188	/* We need to accept PARALLELs for RESUT_DECLs
189	because of vector types with BLKmode returned
190	in multiple registers, but they are supposed
191	to be uncoalesced. */
192	|| (GET_CODE (x) == PARALLEL
193	&& SSAVAR (t)
194	&& TREE_CODE (SSAVAR (t)) == RESULT_DECL
195	&& (GET_MODE (x) == BLKmode
196	|| !flag_tree_coalesce_vars)))
197	: (MEM_P (x) || x == pc_rtx
198	|| (GET_CODE (x) == CONCAT
199	&& MEM_P (XEXP (x, 0))
200	&& MEM_P (XEXP (x, 1))))));
201	/* Check that the RTL for SSA_NAMEs and gimple-reg PARM_DECLs and
202	RESULT_DECLs has the expected mode. For memory, we accept
203	unpromoted modes, since that's what we're likely to get. For
204	PARM_DECLs and RESULT_DECLs, we'll have been called by
205	set_parm_rtl, which will give us the default def, so we don't
206	have to compute it ourselves. For RESULT_DECLs, we accept mode
207	mismatches too, as long as we have BLKmode or are not coalescing
208	across variables, so that we don't reject BLKmode PARALLELs or
209	unpromoted REGs. */
210	gcc_checking_assert (!x || x == pc_rtx || TREE_CODE (t) != SSA_NAME
211	|| (SSAVAR (t)
212	&& TREE_CODE (SSAVAR (t)) == RESULT_DECL
213	&& (promote_ssa_mode (t, NULL) == BLKmode
214	|| !flag_tree_coalesce_vars))
215	|| !use_register_for_decl (t)
216	|| GET_MODE (x) == promote_ssa_mode (t, NULL));
217
218	if (x)
219	{
220	bool skip = false;
221	tree cur = NULL_TREE;
222	rtx xm = x;
223
224	retry:
225	if (MEM_P (xm))
226	cur = MEM_EXPR (xm);
227	else if (REG_P (xm))
228	cur = REG_EXPR (xm);
229	else if (SUBREG_P (xm))
230	{
231	gcc_assert (subreg_lowpart_p (xm));
232	xm = SUBREG_REG (xm);
233	goto retry;
234	}
235	else if (GET_CODE (xm) == CONCAT)
236	{
237	xm = XEXP (xm, 0);
238	goto retry;
239	}
240	else if (GET_CODE (xm) == PARALLEL)
241	{
242	xm = XVECEXP (xm, 0, 0);
243	gcc_assert (GET_CODE (xm) == EXPR_LIST);
244	xm = XEXP (xm, 0);
245	goto retry;
246	}
247	else if (xm == pc_rtx)
248	skip = true;
249	else
250	gcc_unreachable ();
251
252	tree next = skip ? cur : leader_merge (cur, SSAVAR (t) ? SSAVAR (t) : t);
253
254	if (cur != next)
255	{
256	if (MEM_P (x))
257	set_mem_attributes (x,
258	next && TREE_CODE (next) == SSA_NAME
259	? TREE_TYPE (next)
260	: next, true);
261	else
262	set_reg_attrs_for_decl_rtl (t: next, x);
263	}
264	}
265
266	if (TREE_CODE (t) == SSA_NAME)
267	{
268	int part = var_to_partition (map: SA.map, var: t);
269	if (part != NO_PARTITION)
270	{
271	if (SA.partition_to_pseudo[part])
272	gcc_assert (SA.partition_to_pseudo[part] == x);
273	else if (x != pc_rtx)
274	SA.partition_to_pseudo[part] = x;
275	}
276	/* For the benefit of debug information at -O0 (where
277	vartracking doesn't run) record the place also in the base
278	DECL. For PARMs and RESULTs, do so only when setting the
279	default def. */
280	if (x && x != pc_rtx && SSA_NAME_VAR (t)
281	&& (VAR_P (SSA_NAME_VAR (t))
282	|| SSA_NAME_IS_DEFAULT_DEF (t)))
283	{
284	tree var = SSA_NAME_VAR (t);
285	/* If we don't yet have something recorded, just record it now. */
286	if (!DECL_RTL_SET_P (var))
287	SET_DECL_RTL (var, x);
288	/* If we have it set already to "multiple places" don't
289	change this. */
290	else if (DECL_RTL (var) == pc_rtx)
291	;
292	/* If we have something recorded and it's not the same place
293	as we want to record now, we have multiple partitions for the
294	same base variable, with different places. We can't just
295	randomly chose one, hence we have to say that we don't know.
296	This only happens with optimization, and there var-tracking
297	will figure out the right thing. */
298	else if (DECL_RTL (var) != x)
299	SET_DECL_RTL (var, pc_rtx);
300	}
301	}
302	else
303	SET_DECL_RTL (t, x);
304	}
305
306	/* This structure holds data relevant to one variable that will be
307	placed in a stack slot. */
308	class stack_var
309	{
310	public:
311	/* The Variable. */
312	tree decl;
313
314	/* Initially, the size of the variable. Later, the size of the partition,
315	if this variable becomes it's partition's representative. */
316	poly_uint64 size;
317
318	/* The *byte* alignment required for this variable. Or as, with the
319	size, the alignment for this partition. */
320	unsigned int alignb;
321
322	/* The partition representative. */
323	size_t representative;
324
325	/* The next stack variable in the partition, or EOC. */
326	size_t next;
327
328	/* The numbers of conflicting stack variables. */
329	bitmap conflicts;
330	};
331
332	#define EOC ((size_t)-1)
333
334	/* We have an array of such objects while deciding allocation. */
335	static class stack_var *stack_vars;
336	static size_t stack_vars_alloc;
337	static size_t stack_vars_num;
338	static hash_map<tree, size_t> *decl_to_stack_part;
339
340	/* Conflict bitmaps go on this obstack. This allows us to destroy
341	all of them in one big sweep. */
342	static bitmap_obstack stack_var_bitmap_obstack;
343
344	/* An array of indices such that stack_vars[stack_vars_sorted[i]].size
345	is non-decreasing. */
346	static size_t *stack_vars_sorted;
347
348	/* The phase of the stack frame. This is the known misalignment of
349	virtual_stack_vars_rtx from PREFERRED_STACK_BOUNDARY. That is,
350	(frame_offset+frame_phase) % PREFERRED_STACK_BOUNDARY == 0. */
351	static int frame_phase;
352
353	/* Used during expand_used_vars to remember if we saw any decls for
354	which we'd like to enable stack smashing protection. */
355	static bool has_protected_decls;
356
357	/* Used during expand_used_vars. Remember if we say a character buffer
358	smaller than our cutoff threshold. Used for -Wstack-protector. */
359	static bool has_short_buffer;
360
361	/* Compute the byte alignment to use for DECL. Ignore alignment
362	we can't do with expected alignment of the stack boundary. */
363
364	static unsigned int
365	align_local_variable (tree decl, bool really_expand)
366	{
367	unsigned int align;
368
369	if (TREE_CODE (decl) == SSA_NAME)
370	{
371	tree type = TREE_TYPE (decl);
372	machine_mode mode = TYPE_MODE (type);
373
374	align = TYPE_ALIGN (type);
375	if (mode != BLKmode
376	&& align < GET_MODE_ALIGNMENT (mode))
377	align = GET_MODE_ALIGNMENT (mode);
378	}
379	else
380	align = LOCAL_DECL_ALIGNMENT (decl);
381
382	if (hwasan_sanitize_stack_p ())
383	align = MAX (align, (unsigned) HWASAN_TAG_GRANULE_SIZE * BITS_PER_UNIT);
384
385	if (TREE_CODE (decl) != SSA_NAME && really_expand)
386	/* Don't change DECL_ALIGN when called from estimated_stack_frame_size.
387	That is done before IPA and could bump alignment based on host
388	backend even for offloaded code which wants different
389	LOCAL_DECL_ALIGNMENT. */
390	SET_DECL_ALIGN (decl, align);
391
392	return align / BITS_PER_UNIT;
393	}
394
395	/* Align given offset BASE with ALIGN. Truncate up if ALIGN_UP is true,
396	down otherwise. Return truncated BASE value. */
397
398	static inline unsigned HOST_WIDE_INT
399	align_base (HOST_WIDE_INT base, unsigned HOST_WIDE_INT align, bool align_up)
400	{
401	return align_up ? (base + align - 1) & -align : base & -align;
402	}
403
404	/* Allocate SIZE bytes at byte alignment ALIGN from the stack frame.
405	Return the frame offset. */
406
407	static poly_int64
408	alloc_stack_frame_space (poly_int64 size, unsigned HOST_WIDE_INT align)
409	{
410	poly_int64 offset, new_frame_offset;
411
412	if (FRAME_GROWS_DOWNWARD)
413	{
414	new_frame_offset
415	= aligned_lower_bound (frame_offset - frame_phase - size,
416	align) + frame_phase;
417	offset = new_frame_offset;
418	}
419	else
420	{
421	new_frame_offset
422	= aligned_upper_bound (frame_offset - frame_phase,
423	align) + frame_phase;
424	offset = new_frame_offset;
425	new_frame_offset += size;
426	}
427	frame_offset = new_frame_offset;
428
429	if (frame_offset_overflow (frame_offset, cfun->decl))
430	frame_offset = offset = 0;
431
432	return offset;
433	}
434
435	/* Ensure that the stack is aligned to ALIGN bytes.
436	Return the new frame offset. */
437	static poly_int64
438	align_frame_offset (unsigned HOST_WIDE_INT align)
439	{
440	return alloc_stack_frame_space (size: 0, align);
441	}
442
443	/* Accumulate DECL into STACK_VARS. */
444
445	static void
446	add_stack_var (tree decl, bool really_expand)
447	{
448	class stack_var *v;
449
450	if (stack_vars_num >= stack_vars_alloc)
451	{
452	if (stack_vars_alloc)
453	stack_vars_alloc = stack_vars_alloc * 3 / 2;
454	else
455	stack_vars_alloc = 32;
456	stack_vars
457	= XRESIZEVEC (class stack_var, stack_vars, stack_vars_alloc);
458	}
459	if (!decl_to_stack_part)
460	decl_to_stack_part = new hash_map<tree, size_t>;
461
462	v = &stack_vars[stack_vars_num];
463	decl_to_stack_part->put (k: decl, v: stack_vars_num);
464
465	v->decl = decl;
466	tree size = TREE_CODE (decl) == SSA_NAME
467	? TYPE_SIZE_UNIT (TREE_TYPE (decl))
468	: DECL_SIZE_UNIT (decl);
469	v->size = tree_to_poly_uint64 (size);
470	/* Ensure that all variables have size, so that &a != &b for any two
471	variables that are simultaneously live. */
472	if (known_eq (v->size, 0U))
473	v->size = 1;
474	v->alignb = align_local_variable (decl, really_expand);
475	/* An alignment of zero can mightily confuse us later. */
476	gcc_assert (v->alignb != 0);
477
478	/* All variables are initially in their own partition. */
479	v->representative = stack_vars_num;
480	v->next = EOC;
481
482	/* All variables initially conflict with no other. */
483	v->conflicts = NULL;
484
485	/* Ensure that this decl doesn't get put onto the list twice. */
486	set_rtl (t: decl, x: pc_rtx);
487
488	stack_vars_num++;
489	}
490
491	/* Make the decls associated with luid's X and Y conflict. */
492
493	static void
494	add_stack_var_conflict (size_t x, size_t y)
495	{
496	class stack_var *a = &stack_vars[x];
497	class stack_var *b = &stack_vars[y];
498	if (x == y)
499	return;
500	if (!a->conflicts)
501	a->conflicts = BITMAP_ALLOC (obstack: &stack_var_bitmap_obstack);
502	if (!b->conflicts)
503	b->conflicts = BITMAP_ALLOC (obstack: &stack_var_bitmap_obstack);
504	bitmap_set_bit (a->conflicts, y);
505	bitmap_set_bit (b->conflicts, x);
506	}
507
508	/* Check whether the decls associated with luid's X and Y conflict. */
509
510	static bool
511	stack_var_conflict_p (size_t x, size_t y)
512	{
513	class stack_var *a = &stack_vars[x];
514	class stack_var *b = &stack_vars[y];
515	if (x == y)
516	return false;
517	/* Partitions containing an SSA name result from gimple registers
518	with things like unsupported modes. They are top-level and
519	hence conflict with everything else. */
520	if (TREE_CODE (a->decl) == SSA_NAME || TREE_CODE (b->decl) == SSA_NAME)
521	return true;
522
523	if (!a->conflicts || !b->conflicts)
524	return false;
525	return bitmap_bit_p (a->conflicts, y);
526	}
527
528	/* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
529	enter its partition number into bitmap DATA. */
530
531	static bool
532	visit_op (gimple *, tree op, tree, void *data)
533	{
534	bitmap active = (bitmap)data;
535	op = get_base_address (t: op);
536	if (op
537	&& DECL_P (op)
538	&& DECL_RTL_IF_SET (op) == pc_rtx)
539	{
540	size_t *v = decl_to_stack_part->get (k: op);
541	if (v)
542	bitmap_set_bit (active, *v);
543	}
544	return false;
545	}
546
547	/* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
548	record conflicts between it and all currently active other partitions
549	from bitmap DATA. */
550
551	static bool
552	visit_conflict (gimple *, tree op, tree, void *data)
553	{
554	bitmap active = (bitmap)data;
555	op = get_base_address (t: op);
556	if (op
557	&& DECL_P (op)
558	&& DECL_RTL_IF_SET (op) == pc_rtx)
559	{
560	size_t *v = decl_to_stack_part->get (k: op);
561	if (v && bitmap_set_bit (active, *v))
562	{
563	size_t num = *v;
564	bitmap_iterator bi;
565	unsigned i;
566	gcc_assert (num < stack_vars_num);
567	EXECUTE_IF_SET_IN_BITMAP (active, 0, i, bi)
568	add_stack_var_conflict (x: num, y: i);
569	}
570	}
571	return false;
572	}
573
574	/* Helper function for add_scope_conflicts_1. For USE on
575	a stmt, if it is a SSA_NAME and in its SSA_NAME_DEF_STMT is known to be
576	based on some ADDR_EXPR, invoke VISIT on that ADDR_EXPR. */
577
578	static inline void
579	add_scope_conflicts_2 (tree use, bitmap work,
580	walk_stmt_load_store_addr_fn visit)
581	{
582	if (TREE_CODE (use) == SSA_NAME
583	&& (POINTER_TYPE_P (TREE_TYPE (use))
584	|| INTEGRAL_TYPE_P (TREE_TYPE (use))))
585	{
586	gimple *g = SSA_NAME_DEF_STMT (use);
587	if (is_gimple_assign (gs: g))
588	if (tree op = gimple_assign_rhs1 (gs: g))
589	if (TREE_CODE (op) == ADDR_EXPR)
590	visit (g, TREE_OPERAND (op, 0), op, work);
591	}
592	}
593
594	/* Helper routine for add_scope_conflicts, calculating the active partitions
595	at the end of BB, leaving the result in WORK. We're called to generate
596	conflicts when FOR_CONFLICT is true, otherwise we're just tracking
597	liveness. */
598
599	static void
600	add_scope_conflicts_1 (basic_block bb, bitmap work, bool for_conflict)
601	{
602	edge e;
603	edge_iterator ei;
604	gimple_stmt_iterator gsi;
605	walk_stmt_load_store_addr_fn visit;
606	use_operand_p use_p;
607	ssa_op_iter iter;
608
609	bitmap_clear (work);
610	FOR_EACH_EDGE (e, ei, bb->preds)
611	bitmap_ior_into (work, (bitmap)e->src->aux);
612
613	visit = visit_op;
614
615	for (gsi = gsi_start_phis (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
616	{
617	gimple *stmt = gsi_stmt (i: gsi);
618	gphi *phi = as_a <gphi *> (p: stmt);
619	walk_stmt_load_store_addr_ops (stmt, work, NULL, NULL, visit);
620	FOR_EACH_PHI_ARG (use_p, phi, iter, SSA_OP_USE)
621	add_scope_conflicts_2 (USE_FROM_PTR (use_p), work, visit);
622	}
623	for (gsi = gsi_after_labels (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
624	{
625	gimple *stmt = gsi_stmt (i: gsi);
626
627	if (gimple_clobber_p (s: stmt))
628	{
629	tree lhs = gimple_assign_lhs (gs: stmt);
630	size_t *v;
631	/* Nested function lowering might introduce LHSs
632	that are COMPONENT_REFs. */
633	if (!VAR_P (lhs))
634	continue;
635	if (DECL_RTL_IF_SET (lhs) == pc_rtx
636	&& (v = decl_to_stack_part->get (k: lhs)))
637	bitmap_clear_bit (work, *v);
638	}
639	else if (!is_gimple_debug (gs: stmt))
640	{
641	if (for_conflict && visit == visit_op)
642	{
643	/* If this is the first real instruction in this BB we need
644	to add conflicts for everything live at this point now.
645	Unlike classical liveness for named objects we can't
646	rely on seeing a def/use of the names we're interested in.
647	There might merely be indirect loads/stores. We'd not add any
648	conflicts for such partitions. */
649	bitmap_iterator bi;
650	unsigned i;
651	EXECUTE_IF_SET_IN_BITMAP (work, 0, i, bi)
652	{
653	class stack_var *a = &stack_vars[i];
654	if (!a->conflicts)
655	a->conflicts = BITMAP_ALLOC (obstack: &stack_var_bitmap_obstack);
656	bitmap_ior_into (a->conflicts, work);
657	}
658	visit = visit_conflict;
659	}
660	walk_stmt_load_store_addr_ops (stmt, work, visit, visit, visit);
661	FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
662	add_scope_conflicts_2 (USE_FROM_PTR (use_p), work, visit);
663	}
664	}
665	}
666
667	/* Generate stack partition conflicts between all partitions that are
668	simultaneously live. */
669
670	static void
671	add_scope_conflicts (void)
672	{
673	basic_block bb;
674	bool changed;
675	bitmap work = BITMAP_ALLOC (NULL);
676	int *rpo;
677	int n_bbs;
678
679	/* We approximate the live range of a stack variable by taking the first
680	mention of its name as starting point(s), and by the end-of-scope
681	death clobber added by gimplify as ending point(s) of the range.
682	This overapproximates in the case we for instance moved an address-taken
683	operation upward, without also moving a dereference to it upwards.
684	But it's conservatively correct as a variable never can hold values
685	before its name is mentioned at least once.
686
687	We then do a mostly classical bitmap liveness algorithm. */
688
689	FOR_ALL_BB_FN (bb, cfun)
690	bb->aux = BITMAP_ALLOC (obstack: &stack_var_bitmap_obstack);
691
692	rpo = XNEWVEC (int, last_basic_block_for_fn (cfun));
693	n_bbs = pre_and_rev_post_order_compute (NULL, rpo, false);
694
695	changed = true;
696	while (changed)
697	{
698	int i;
699	changed = false;
700	for (i = 0; i < n_bbs; i++)
701	{
702	bitmap active;
703	bb = BASIC_BLOCK_FOR_FN (cfun, rpo[i]);
704	active = (bitmap)bb->aux;
705	add_scope_conflicts_1 (bb, work, for_conflict: false);
706	if (bitmap_ior_into (active, work))
707	changed = true;
708	}
709	}
710
711	FOR_EACH_BB_FN (bb, cfun)
712	add_scope_conflicts_1 (bb, work, for_conflict: true);
713
714	free (ptr: rpo);
715	BITMAP_FREE (work);
716	FOR_ALL_BB_FN (bb, cfun)
717	BITMAP_FREE (bb->aux);
718	}
719
720	/* A subroutine of partition_stack_vars. A comparison function for qsort,
721	sorting an array of indices by the properties of the object. */
722
723	static int
724	stack_var_cmp (const void *a, const void *b)
725	{
726	size_t ia = *(const size_t *)a;
727	size_t ib = *(const size_t *)b;
728	unsigned int aligna = stack_vars[ia].alignb;
729	unsigned int alignb = stack_vars[ib].alignb;
730	poly_int64 sizea = stack_vars[ia].size;
731	poly_int64 sizeb = stack_vars[ib].size;
732	tree decla = stack_vars[ia].decl;
733	tree declb = stack_vars[ib].decl;
734	bool largea, largeb;
735	unsigned int uida, uidb;
736
737	/* Primary compare on "large" alignment. Large comes first. */
738	largea = (aligna * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
739	largeb = (alignb * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
740	if (largea != largeb)
741	return (int)largeb - (int)largea;
742
743	/* Secondary compare on size, decreasing */
744	int diff = compare_sizes_for_sort (a: sizeb, b: sizea);
745	if (diff != 0)
746	return diff;
747
748	/* Tertiary compare on true alignment, decreasing. */
749	if (aligna < alignb)
750	return -1;
751	if (aligna > alignb)
752	return 1;
753
754	/* Final compare on ID for sort stability, increasing.
755	Two SSA names are compared by their version, SSA names come before
756	non-SSA names, and two normal decls are compared by their DECL_UID. */
757	if (TREE_CODE (decla) == SSA_NAME)
758	{
759	if (TREE_CODE (declb) == SSA_NAME)
760	uida = SSA_NAME_VERSION (decla), uidb = SSA_NAME_VERSION (declb);
761	else
762	return -1;
763	}
764	else if (TREE_CODE (declb) == SSA_NAME)
765	return 1;
766	else
767	uida = DECL_UID (decla), uidb = DECL_UID (declb);
768	if (uida < uidb)
769	return 1;
770	if (uida > uidb)
771	return -1;
772	return 0;
773	}
774
775	struct part_traits : unbounded_int_hashmap_traits <size_t, bitmap> {};
776	typedef hash_map<size_t, bitmap, part_traits> part_hashmap;
777
778	/* If the points-to solution *PI points to variables that are in a partition
779	together with other variables add all partition members to the pointed-to
780	variables bitmap. */
781
782	static void
783	add_partitioned_vars_to_ptset (struct pt_solution *pt,
784	part_hashmap *decls_to_partitions,
785	hash_set<bitmap> *visited, bitmap temp)
786	{
787	bitmap_iterator bi;
788	unsigned i;
789	bitmap *part;
790
791	if (pt->anything
792	|| pt->vars == NULL
793	/* The pointed-to vars bitmap is shared, it is enough to
794	visit it once. */
795	|| visited->add (k: pt->vars))
796	return;
797
798	bitmap_clear (temp);
799
800	/* By using a temporary bitmap to store all members of the partitions
801	we have to add we make sure to visit each of the partitions only
802	once. */
803	EXECUTE_IF_SET_IN_BITMAP (pt->vars, 0, i, bi)
804	if ((!temp
805	|| !bitmap_bit_p (temp, i))
806	&& (part = decls_to_partitions->get (k: i)))
807	bitmap_ior_into (temp, *part);
808	if (!bitmap_empty_p (map: temp))
809	bitmap_ior_into (pt->vars, temp);
810	}
811
812	/* Update points-to sets based on partition info, so we can use them on RTL.
813	The bitmaps representing stack partitions will be saved until expand,
814	where partitioned decls used as bases in memory expressions will be
815	rewritten.
816
817	It is not necessary to update TBAA info on accesses to the coalesced
818	storage since our memory model doesn't allow TBAA to be used for
819	WAW or WAR dependences. For RAW when the write is to an old object
820	the new object would not have been initialized at the point of the
821	read, invoking undefined behavior. */
822
823	static void
824	update_alias_info_with_stack_vars (void)
825	{
826	part_hashmap *decls_to_partitions = NULL;
827	size_t i, j;
828	tree var = NULL_TREE;
829
830	for (i = 0; i < stack_vars_num; i++)
831	{
832	bitmap part = NULL;
833	tree name;
834	struct ptr_info_def *pi;
835
836	/* Not interested in partitions with single variable. */
837	if (stack_vars[i].representative != i
838	|| stack_vars[i].next == EOC)
839	continue;
840
841	if (!decls_to_partitions)
842	{
843	decls_to_partitions = new part_hashmap;
844	cfun->gimple_df->decls_to_pointers = new hash_map<tree, tree>;
845	}
846
847	/* Create an SSA_NAME that points to the partition for use
848	as base during alias-oracle queries on RTL for bases that
849	have been partitioned. */
850	if (var == NULL_TREE)
851	var = create_tmp_var (ptr_type_node);
852	name = make_ssa_name (var);
853
854	/* Create bitmaps representing partitions. They will be used for
855	points-to sets later, so use GGC alloc. */
856	part = BITMAP_GGC_ALLOC ();
857	for (j = i; j != EOC; j = stack_vars[j].next)
858	{
859	tree decl = stack_vars[j].decl;
860	unsigned int uid = DECL_PT_UID (decl);
861	bitmap_set_bit (part, uid);
862	decls_to_partitions->put (k: uid, v: part);
863	cfun->gimple_df->decls_to_pointers->put (k: decl, v: name);
864	if (TREE_ADDRESSABLE (decl))
865	TREE_ADDRESSABLE (name) = 1;
866	}
867
868	/* Make the SSA name point to all partition members. */
869	pi = get_ptr_info (name);
870	pt_solution_set (&pi->pt, part, false);
871	}
872
873	/* Make all points-to sets that contain one member of a partition
874	contain all members of the partition. */
875	if (decls_to_partitions)
876	{
877	unsigned i;
878	tree name;
879	hash_set<bitmap> visited;
880	bitmap temp = BITMAP_ALLOC (obstack: &stack_var_bitmap_obstack);
881
882	FOR_EACH_SSA_NAME (i, name, cfun)
883	{
884	struct ptr_info_def *pi;
885
886	if (POINTER_TYPE_P (TREE_TYPE (name))
887	&& ((pi = SSA_NAME_PTR_INFO (name)) != NULL))
888	add_partitioned_vars_to_ptset (pt: &pi->pt, decls_to_partitions,
889	visited: &visited, temp);
890	}
891
892	add_partitioned_vars_to_ptset (pt: &cfun->gimple_df->escaped,
893	decls_to_partitions, visited: &visited, temp);
894	add_partitioned_vars_to_ptset (pt: &cfun->gimple_df->escaped_return,
895	decls_to_partitions, visited: &visited, temp);
896	delete decls_to_partitions;
897	BITMAP_FREE (temp);
898	}
899	}
900
901	/* A subroutine of partition_stack_vars. The UNION portion of a UNION/FIND
902	partitioning algorithm. Partitions A and B are known to be non-conflicting.
903	Merge them into a single partition A. */
904
905	static void
906	union_stack_vars (size_t a, size_t b)
907	{
908	class stack_var *vb = &stack_vars[b];
909	bitmap_iterator bi;
910	unsigned u;
911
912	gcc_assert (stack_vars[b].next == EOC);
913	/* Add B to A's partition. */
914	stack_vars[b].next = stack_vars[a].next;
915	stack_vars[b].representative = a;
916	stack_vars[a].next = b;
917
918	/* Make sure A is big enough to hold B. */
919	stack_vars[a].size = upper_bound (a: stack_vars[a].size, b: stack_vars[b].size);
920
921	/* Update the required alignment of partition A to account for B. */
922	if (stack_vars[a].alignb < stack_vars[b].alignb)
923	stack_vars[a].alignb = stack_vars[b].alignb;
924
925	/* Update the interference graph and merge the conflicts. */
926	if (vb->conflicts)
927	{
928	EXECUTE_IF_SET_IN_BITMAP (vb->conflicts, 0, u, bi)
929	add_stack_var_conflict (x: a, y: stack_vars[u].representative);
930	BITMAP_FREE (vb->conflicts);
931	}
932	}
933
934	/* A subroutine of expand_used_vars. Binpack the variables into
935	partitions constrained by the interference graph. The overall
936	algorithm used is as follows:
937
938	Sort the objects by size in descending order.
939	For each object A {
940	S = size(A)
941	O = 0
942	loop {
943	Look for the largest non-conflicting object B with size <= S.
944	UNION (A, B)
945	}
946	}
947	*/
948
949	static void
950	partition_stack_vars (void)
951	{
952	size_t si, sj, n = stack_vars_num;
953
954	stack_vars_sorted = XNEWVEC (size_t, stack_vars_num);
955	for (si = 0; si < n; ++si)
956	stack_vars_sorted[si] = si;
957
958	if (n == 1)
959	return;
960
961	qsort (stack_vars_sorted, n, sizeof (size_t), stack_var_cmp);
962
963	for (si = 0; si < n; ++si)
964	{
965	size_t i = stack_vars_sorted[si];
966	unsigned int ialign = stack_vars[i].alignb;
967	poly_int64 isize = stack_vars[i].size;
968
969	/* Ignore objects that aren't partition representatives. If we
970	see a var that is not a partition representative, it must
971	have been merged earlier. */
972	if (stack_vars[i].representative != i)
973	continue;
974
975	for (sj = si + 1; sj < n; ++sj)
976	{
977	size_t j = stack_vars_sorted[sj];
978	unsigned int jalign = stack_vars[j].alignb;
979	poly_int64 jsize = stack_vars[j].size;
980
981	/* Ignore objects that aren't partition representatives. */
982	if (stack_vars[j].representative != j)
983	continue;
984
985	/* Do not mix objects of "small" (supported) alignment
986	and "large" (unsupported) alignment. */
987	if ((ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
988	!= (jalign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT))
989	break;
990
991	/* For Address Sanitizer do not mix objects with different
992	sizes, as the shorter vars wouldn't be adequately protected.
993	Don't do that for "large" (unsupported) alignment objects,
994	those aren't protected anyway. */
995	if (asan_sanitize_stack_p ()
996	&& maybe_ne (a: isize, b: jsize)
997	&& ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
998	break;
999
1000	/* Ignore conflicting objects. */
1001	if (stack_var_conflict_p (x: i, y: j))
1002	continue;
1003
1004	/* UNION the objects, placing J at OFFSET. */
1005	union_stack_vars (a: i, b: j);
1006	}
1007	}
1008
1009	update_alias_info_with_stack_vars ();
1010	}
1011
1012	/* A debugging aid for expand_used_vars. Dump the generated partitions. */
1013
1014	static void
1015	dump_stack_var_partition (void)
1016	{
1017	size_t si, i, j, n = stack_vars_num;
1018
1019	for (si = 0; si < n; ++si)
1020	{
1021	i = stack_vars_sorted[si];
1022
1023	/* Skip variables that aren't partition representatives, for now. */
1024	if (stack_vars[i].representative != i)
1025	continue;
1026
1027	fprintf (stream: dump_file, format: "Partition " HOST_SIZE_T_PRINT_UNSIGNED ": size ",
1028	(fmt_size_t) i);
1029	print_dec (value: stack_vars[i].size, file: dump_file);
1030	fprintf (stream: dump_file, format: " align %u\n", stack_vars[i].alignb);
1031
1032	for (j = i; j != EOC; j = stack_vars[j].next)
1033	{
1034	fputc (c: '\t', stream: dump_file);
1035	print_generic_expr (dump_file, stack_vars[j].decl, dump_flags);
1036	}
1037	fputc (c: '\n', stream: dump_file);
1038	}
1039	}
1040
1041	/* Assign rtl to DECL at BASE + OFFSET. */
1042
1043	static void
1044	expand_one_stack_var_at (tree decl, rtx base, unsigned base_align,
1045	poly_int64 offset)
1046	{
1047	unsigned align;
1048	rtx x;
1049
1050	/* If this fails, we've overflowed the stack frame. Error nicely? */
1051	gcc_assert (known_eq (offset, trunc_int_for_mode (offset, Pmode)));
1052
1053	if (hwasan_sanitize_stack_p ())
1054	x = targetm.memtag.add_tag (base, offset,
1055	hwasan_current_frame_tag ());
1056	else
1057	x = plus_constant (Pmode, base, offset);
1058
1059	x = gen_rtx_MEM (TREE_CODE (decl) == SSA_NAME
1060	? TYPE_MODE (TREE_TYPE (decl))
1061	: DECL_MODE (decl), x);
1062
1063	/* Set alignment we actually gave this decl if it isn't an SSA name.
1064	If it is we generate stack slots only accidentally so it isn't as
1065	important, we'll simply set the alignment directly on the MEM. */
1066
1067	if (stack_vars_base_reg_p (base))
1068	offset -= frame_phase;
1069	align = known_alignment (a: offset);
1070	align *= BITS_PER_UNIT;
1071	if (align == 0 || align > base_align)
1072	align = base_align;
1073
1074	if (TREE_CODE (decl) != SSA_NAME)
1075	{
1076	/* One would think that we could assert that we're not decreasing
1077	alignment here, but (at least) the i386 port does exactly this
1078	via the MINIMUM_ALIGNMENT hook. */
1079
1080	SET_DECL_ALIGN (decl, align);
1081	DECL_USER_ALIGN (decl) = 0;
1082	}
1083
1084	set_rtl (t: decl, x);
1085
1086	set_mem_align (x, align);
1087	}
1088
1089	class stack_vars_data
1090	{
1091	public:
1092	/* Vector of offset pairs, always end of some padding followed
1093	by start of the padding that needs Address Sanitizer protection.
1094	The vector is in reversed, highest offset pairs come first. */
1095	auto_vec<HOST_WIDE_INT> asan_vec;
1096
1097	/* Vector of partition representative decls in between the paddings. */
1098	auto_vec<tree> asan_decl_vec;
1099
1100	/* Base pseudo register for Address Sanitizer protected automatic vars. */
1101	rtx asan_base;
1102
1103	/* Alignment needed for the Address Sanitizer protected automatic vars. */
1104	unsigned int asan_alignb;
1105	};
1106
1107	/* A subroutine of expand_used_vars. Give each partition representative
1108	a unique location within the stack frame. Update each partition member
1109	with that location. */
1110	static void
1111	expand_stack_vars (bool (*pred) (size_t), class stack_vars_data *data)
1112	{
1113	size_t si, i, j, n = stack_vars_num;
1114	poly_uint64 large_size = 0, large_alloc = 0;
1115	rtx large_base = NULL;
1116	rtx large_untagged_base = NULL;
1117	unsigned large_align = 0;
1118	bool large_allocation_done = false;
1119	tree decl;
1120
1121	/* Determine if there are any variables requiring "large" alignment.
1122	Since these are dynamically allocated, we only process these if
1123	no predicate involved. */
1124	large_align = stack_vars[stack_vars_sorted[0]].alignb * BITS_PER_UNIT;
1125	if (pred == NULL && large_align > MAX_SUPPORTED_STACK_ALIGNMENT)
1126	{
1127	/* Find the total size of these variables. */
1128	for (si = 0; si < n; ++si)
1129	{
1130	unsigned alignb;
1131
1132	i = stack_vars_sorted[si];
1133	alignb = stack_vars[i].alignb;
1134
1135	/* All "large" alignment decls come before all "small" alignment
1136	decls, but "large" alignment decls are not sorted based on
1137	their alignment. Increase large_align to track the largest
1138	required alignment. */
1139	if ((alignb * BITS_PER_UNIT) > large_align)
1140	large_align = alignb * BITS_PER_UNIT;
1141
1142	/* Stop when we get to the first decl with "small" alignment. */
1143	if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1144	break;
1145
1146	/* Skip variables that aren't partition representatives. */
1147	if (stack_vars[i].representative != i)
1148	continue;
1149
1150	/* Skip variables that have already had rtl assigned. See also
1151	add_stack_var where we perpetrate this pc_rtx hack. */
1152	decl = stack_vars[i].decl;
1153	if (TREE_CODE (decl) == SSA_NAME
1154	? SA.partition_to_pseudo[var_to_partition (map: SA.map, var: decl)] != NULL_RTX
1155	: DECL_RTL (decl) != pc_rtx)
1156	continue;
1157
1158	large_size = aligned_upper_bound (value: large_size, align: alignb);
1159	large_size += stack_vars[i].size;
1160	}
1161	}
1162
1163	for (si = 0; si < n; ++si)
1164	{
1165	rtx base;
1166	unsigned base_align, alignb;
1167	poly_int64 offset = 0;
1168
1169	i = stack_vars_sorted[si];
1170
1171	/* Skip variables that aren't partition representatives, for now. */
1172	if (stack_vars[i].representative != i)
1173	continue;
1174
1175	/* Skip variables that have already had rtl assigned. See also
1176	add_stack_var where we perpetrate this pc_rtx hack. */
1177	decl = stack_vars[i].decl;
1178	if (TREE_CODE (decl) == SSA_NAME
1179	? SA.partition_to_pseudo[var_to_partition (map: SA.map, var: decl)] != NULL_RTX
1180	: DECL_RTL (decl) != pc_rtx)
1181	continue;
1182
1183	/* Check the predicate to see whether this variable should be
1184	allocated in this pass. */
1185	if (pred && !pred (i))
1186	continue;
1187
1188	base = (hwasan_sanitize_stack_p ()
1189	? hwasan_frame_base ()
1190	: virtual_stack_vars_rtx);
1191	alignb = stack_vars[i].alignb;
1192	if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1193	{
1194	poly_int64 hwasan_orig_offset;
1195	if (hwasan_sanitize_stack_p ())
1196	{
1197	/* There must be no tag granule "shared" between different
1198	objects. This means that no HWASAN_TAG_GRANULE_SIZE byte
1199	chunk can have more than one object in it.
1200
1201	We ensure this by forcing the end of the last bit of data to
1202	be aligned to HWASAN_TAG_GRANULE_SIZE bytes here, and setting
1203	the start of each variable to be aligned to
1204	HWASAN_TAG_GRANULE_SIZE bytes in `align_local_variable`.
1205
1206	We can't align just one of the start or end, since there are
1207	untagged things stored on the stack which we do not align to
1208	HWASAN_TAG_GRANULE_SIZE bytes. If we only aligned the start
1209	or the end of tagged objects then untagged objects could end
1210	up sharing the first granule of a tagged object or sharing the
1211	last granule of a tagged object respectively. */
1212	hwasan_orig_offset = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1213	gcc_assert (stack_vars[i].alignb >= HWASAN_TAG_GRANULE_SIZE);
1214	}
1215	/* ASAN description strings don't yet have a syntax for expressing
1216	polynomial offsets. */
1217	HOST_WIDE_INT prev_offset;
1218	if (asan_sanitize_stack_p ()
1219	&& pred
1220	&& frame_offset.is_constant (const_value: &prev_offset)
1221	&& stack_vars[i].size.is_constant ())
1222	{
1223	if (data->asan_vec.is_empty ())
1224	{
1225	align_frame_offset (ASAN_RED_ZONE_SIZE);
1226	prev_offset = frame_offset.to_constant ();
1227	}
1228	prev_offset = align_base (base: prev_offset,
1229	ASAN_MIN_RED_ZONE_SIZE,
1230	align_up: !FRAME_GROWS_DOWNWARD);
1231	tree repr_decl = NULL_TREE;
1232	unsigned HOST_WIDE_INT size
1233	= asan_var_and_redzone_size (size: stack_vars[i].size.to_constant ());
1234	if (data->asan_vec.is_empty ())
1235	size = MAX (size, ASAN_RED_ZONE_SIZE);
1236
1237	unsigned HOST_WIDE_INT alignment = MAX (alignb,
1238	ASAN_MIN_RED_ZONE_SIZE);
1239	offset = alloc_stack_frame_space (size, align: alignment);
1240
1241	data->asan_vec.safe_push (obj: prev_offset);
1242	/* Allocating a constant amount of space from a constant
1243	starting offset must give a constant result. */
1244	data->asan_vec.safe_push (obj: (offset + stack_vars[i].size)
1245	.to_constant ());
1246	/* Find best representative of the partition.
1247	Prefer those with DECL_NAME, even better
1248	satisfying asan_protect_stack_decl predicate. */
1249	for (j = i; j != EOC; j = stack_vars[j].next)
1250	if (asan_protect_stack_decl (decl: stack_vars[j].decl)
1251	&& DECL_NAME (stack_vars[j].decl))
1252	{
1253	repr_decl = stack_vars[j].decl;
1254	break;
1255	}
1256	else if (repr_decl == NULL_TREE
1257	&& DECL_P (stack_vars[j].decl)
1258	&& DECL_NAME (stack_vars[j].decl))
1259	repr_decl = stack_vars[j].decl;
1260	if (repr_decl == NULL_TREE)
1261	repr_decl = stack_vars[i].decl;
1262	data->asan_decl_vec.safe_push (obj: repr_decl);
1263
1264	/* Make sure a representative is unpoison if another
1265	variable in the partition is handled by
1266	use-after-scope sanitization. */
1267	if (asan_handled_variables != NULL
1268	&& !asan_handled_variables->contains (k: repr_decl))
1269	{
1270	for (j = i; j != EOC; j = stack_vars[j].next)
1271	if (asan_handled_variables->contains (k: stack_vars[j].decl))
1272	break;
1273	if (j != EOC)
1274	asan_handled_variables->add (k: repr_decl);
1275	}
1276
1277	data->asan_alignb = MAX (data->asan_alignb, alignb);
1278	if (data->asan_base == NULL)
1279	data->asan_base = gen_reg_rtx (Pmode);
1280	base = data->asan_base;
1281
1282	if (!STRICT_ALIGNMENT)
1283	base_align = crtl->max_used_stack_slot_alignment;
1284	else
1285	base_align = MAX (crtl->max_used_stack_slot_alignment,
1286	GET_MODE_ALIGNMENT (SImode)
1287	<< ASAN_SHADOW_SHIFT);
1288	}
1289	else
1290	{
1291	offset = alloc_stack_frame_space (size: stack_vars[i].size, align: alignb);
1292	base_align = crtl->max_used_stack_slot_alignment;
1293
1294	if (hwasan_sanitize_stack_p ())
1295	{
1296	/* Align again since the point of this alignment is to handle
1297	the "end" of the object (i.e. smallest address after the
1298	stack object). For FRAME_GROWS_DOWNWARD that requires
1299	aligning the stack before allocating, but for a frame that
1300	grows upwards that requires aligning the stack after
1301	allocation.
1302
1303	Use `frame_offset` to record the offset value rather than
1304	`offset` since the `frame_offset` describes the extent
1305	allocated for this particular variable while `offset`
1306	describes the address that this variable starts at. */
1307	align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1308	hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1309	hwasan_orig_offset, frame_offset);
1310	}
1311	}
1312	}
1313	else
1314	{
1315	/* Large alignment is only processed in the last pass. */
1316	if (pred)
1317	continue;
1318
1319	/* If there were any variables requiring "large" alignment, allocate
1320	space. */
1321	if (maybe_ne (a: large_size, b: 0U) && ! large_allocation_done)
1322	{
1323	poly_int64 loffset;
1324	rtx large_allocsize;
1325
1326	large_allocsize = gen_int_mode (large_size, Pmode);
1327	get_dynamic_stack_size (&large_allocsize, 0, large_align, NULL);
1328	loffset = alloc_stack_frame_space
1329	(size: rtx_to_poly_int64 (x: large_allocsize),
1330	PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT);
1331	large_base = get_dynamic_stack_base (loffset, large_align, base);
1332	large_allocation_done = true;
1333	}
1334
1335	gcc_assert (large_base != NULL);
1336	large_alloc = aligned_upper_bound (value: large_alloc, align: alignb);
1337	offset = large_alloc;
1338	large_alloc += stack_vars[i].size;
1339	if (hwasan_sanitize_stack_p ())
1340	{
1341	/* An object with a large alignment requirement means that the
1342	alignment requirement is greater than the required alignment
1343	for tags. */
1344	if (!large_untagged_base)
1345	large_untagged_base
1346	= targetm.memtag.untagged_pointer (large_base, NULL_RTX);
1347	/* Ensure the end of the variable is also aligned correctly. */
1348	poly_int64 align_again
1349	= aligned_upper_bound (value: large_alloc, HWASAN_TAG_GRANULE_SIZE);
1350	/* For large allocations we always allocate a chunk of space
1351	(which is addressed by large_untagged_base/large_base) and
1352	then use positive offsets from that. Hence the farthest
1353	offset is `align_again` and the nearest offset from the base
1354	is `offset`. */
1355	hwasan_record_stack_var (large_untagged_base, large_base,
1356	offset, align_again);
1357	}
1358
1359	base = large_base;
1360	base_align = large_align;
1361	}
1362
1363	/* Create rtl for each variable based on their location within the
1364	partition. */
1365	for (j = i; j != EOC; j = stack_vars[j].next)
1366	{
1367	expand_one_stack_var_at (decl: stack_vars[j].decl,
1368	base, base_align, offset);
1369	}
1370	if (hwasan_sanitize_stack_p ())
1371	hwasan_increment_frame_tag ();
1372	}
1373
1374	gcc_assert (known_eq (large_alloc, large_size));
1375	}
1376
1377	/* Take into account all sizes of partitions and reset DECL_RTLs. */
1378	static poly_uint64
1379	account_stack_vars (void)
1380	{
1381	size_t si, j, i, n = stack_vars_num;
1382	poly_uint64 size = 0;
1383
1384	for (si = 0; si < n; ++si)
1385	{
1386	i = stack_vars_sorted[si];
1387
1388	/* Skip variables that aren't partition representatives, for now. */
1389	if (stack_vars[i].representative != i)
1390	continue;
1391
1392	size += stack_vars[i].size;
1393	for (j = i; j != EOC; j = stack_vars[j].next)
1394	set_rtl (t: stack_vars[j].decl, NULL);
1395	}
1396	return size;
1397	}
1398
1399	/* Record the RTL assignment X for the default def of PARM. */
1400
1401	extern void
1402	set_parm_rtl (tree parm, rtx x)
1403	{
1404	gcc_assert (TREE_CODE (parm) == PARM_DECL
1405	|| TREE_CODE (parm) == RESULT_DECL);
1406
1407	if (x && !MEM_P (x))
1408	{
1409	unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (parm),
1410	TYPE_MODE (TREE_TYPE (parm)),
1411	TYPE_ALIGN (TREE_TYPE (parm)));
1412
1413	/* If the variable alignment is very large we'll dynamicaly
1414	allocate it, which means that in-frame portion is just a
1415	pointer. ??? We've got a pseudo for sure here, do we
1416	actually dynamically allocate its spilling area if needed?
1417	??? Isn't it a problem when Pmode alignment also exceeds
1418	MAX_SUPPORTED_STACK_ALIGNMENT, as can happen on cris and lm32? */
1419	if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1420	align = GET_MODE_ALIGNMENT (Pmode);
1421
1422	record_alignment_for_reg_var (align);
1423	}
1424
1425	tree ssa = ssa_default_def (cfun, parm);
1426	if (!ssa)
1427	return set_rtl (t: parm, x);
1428
1429	int part = var_to_partition (map: SA.map, var: ssa);
1430	gcc_assert (part != NO_PARTITION);
1431
1432	bool changed = bitmap_bit_p (SA.partitions_for_parm_default_defs, part);
1433	gcc_assert (changed);
1434
1435	set_rtl (t: ssa, x);
1436	gcc_assert (DECL_RTL (parm) == x);
1437	}
1438
1439	/* A subroutine of expand_one_var. Called to immediately assign rtl
1440	to a variable to be allocated in the stack frame. */
1441
1442	static void
1443	expand_one_stack_var_1 (tree var)
1444	{
1445	poly_uint64 size;
1446	poly_int64 offset;
1447	unsigned byte_align;
1448
1449	if (TREE_CODE (var) == SSA_NAME)
1450	{
1451	tree type = TREE_TYPE (var);
1452	size = tree_to_poly_uint64 (TYPE_SIZE_UNIT (type));
1453	}
1454	else
1455	size = tree_to_poly_uint64 (DECL_SIZE_UNIT (var));
1456
1457	byte_align = align_local_variable (decl: var, really_expand: true);
1458
1459	/* We handle highly aligned variables in expand_stack_vars. */
1460	gcc_assert (byte_align * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT);
1461
1462	rtx base;
1463	if (hwasan_sanitize_stack_p ())
1464	{
1465	/* Allocate zero bytes to align the stack. */
1466	poly_int64 hwasan_orig_offset
1467	= align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1468	offset = alloc_stack_frame_space (size, align: byte_align);
1469	align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1470	base = hwasan_frame_base ();
1471	/* Use `frame_offset` to automatically account for machines where the
1472	frame grows upwards.
1473
1474	`offset` will always point to the "start" of the stack object, which
1475	will be the smallest address, for ! FRAME_GROWS_DOWNWARD this is *not*
1476	the "furthest" offset from the base delimiting the current stack
1477	object. `frame_offset` will always delimit the extent that the frame.
1478	*/
1479	hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1480	hwasan_orig_offset, frame_offset);
1481	}
1482	else
1483	{
1484	offset = alloc_stack_frame_space (size, align: byte_align);
1485	base = virtual_stack_vars_rtx;
1486	}
1487
1488	expand_one_stack_var_at (decl: var, base,
1489	crtl->max_used_stack_slot_alignment, offset);
1490
1491	if (hwasan_sanitize_stack_p ())
1492	hwasan_increment_frame_tag ();
1493	}
1494
1495	/* Wrapper for expand_one_stack_var_1 that checks SSA_NAMEs are
1496	already assigned some MEM. */
1497
1498	static void
1499	expand_one_stack_var (tree var)
1500	{
1501	if (TREE_CODE (var) == SSA_NAME)
1502	{
1503	int part = var_to_partition (map: SA.map, var);
1504	if (part != NO_PARTITION)
1505	{
1506	rtx x = SA.partition_to_pseudo[part];
1507	gcc_assert (x);
1508	gcc_assert (MEM_P (x));
1509	return;
1510	}
1511	}
1512
1513	return expand_one_stack_var_1 (var);
1514	}
1515
1516	/* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1517	that will reside in a hard register. */
1518
1519	static void
1520	expand_one_hard_reg_var (tree var)
1521	{
1522	rest_of_decl_compilation (var, 0, 0);
1523	}
1524
1525	/* Record the alignment requirements of some variable assigned to a
1526	pseudo. */
1527
1528	static void
1529	record_alignment_for_reg_var (unsigned int align)
1530	{
1531	if (SUPPORTS_STACK_ALIGNMENT
1532	&& crtl->stack_alignment_estimated < align)
1533	{
1534	/* stack_alignment_estimated shouldn't change after stack
1535	realign decision made */
1536	gcc_assert (!crtl->stack_realign_processed);
1537	crtl->stack_alignment_estimated = align;
1538	}
1539
1540	/* stack_alignment_needed > PREFERRED_STACK_BOUNDARY is permitted.
1541	So here we only make sure stack_alignment_needed >= align. */
1542	if (crtl->stack_alignment_needed < align)
1543	crtl->stack_alignment_needed = align;
1544	if (crtl->max_used_stack_slot_alignment < align)
1545	crtl->max_used_stack_slot_alignment = align;
1546	}
1547
1548	/* Create RTL for an SSA partition. */
1549
1550	static void
1551	expand_one_ssa_partition (tree var)
1552	{
1553	int part = var_to_partition (map: SA.map, var);
1554	gcc_assert (part != NO_PARTITION);
1555
1556	if (SA.partition_to_pseudo[part])
1557	return;
1558
1559	unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
1560	TYPE_MODE (TREE_TYPE (var)),
1561	TYPE_ALIGN (TREE_TYPE (var)));
1562
1563	/* If the variable alignment is very large we'll dynamicaly allocate
1564	it, which means that in-frame portion is just a pointer. */
1565	if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1566	align = GET_MODE_ALIGNMENT (Pmode);
1567
1568	record_alignment_for_reg_var (align);
1569
1570	if (!use_register_for_decl (var))
1571	{
1572	if (defer_stack_allocation (var, true))
1573	add_stack_var (decl: var, really_expand: true);
1574	else
1575	expand_one_stack_var_1 (var);
1576	return;
1577	}
1578
1579	machine_mode reg_mode = promote_ssa_mode (var, NULL);
1580	rtx x = gen_reg_rtx (reg_mode);
1581
1582	set_rtl (t: var, x);
1583
1584	/* For a promoted variable, X will not be used directly but wrapped in a
1585	SUBREG with SUBREG_PROMOTED_VAR_P set, which means that the RTL land
1586	will assume that its upper bits can be inferred from its lower bits.
1587	Therefore, if X isn't initialized on every path from the entry, then
1588	we must do it manually in order to fulfill the above assumption. */
1589	if (reg_mode != TYPE_MODE (TREE_TYPE (var))
1590	&& bitmap_bit_p (SA.partitions_for_undefined_values, part))
1591	emit_move_insn (x, CONST0_RTX (reg_mode));
1592	}
1593
1594	/* Record the association between the RTL generated for partition PART
1595	and the underlying variable of the SSA_NAME VAR. */
1596
1597	static void
1598	adjust_one_expanded_partition_var (tree var)
1599	{
1600	if (!var)
1601	return;
1602
1603	tree decl = SSA_NAME_VAR (var);
1604
1605	int part = var_to_partition (map: SA.map, var);
1606	if (part == NO_PARTITION)
1607	return;
1608
1609	rtx x = SA.partition_to_pseudo[part];
1610
1611	gcc_assert (x);
1612
1613	set_rtl (t: var, x);
1614
1615	if (!REG_P (x))
1616	return;
1617
1618	/* Note if the object is a user variable. */
1619	if (decl && !DECL_ARTIFICIAL (decl))
1620	mark_user_reg (x);
1621
1622	if (POINTER_TYPE_P (decl ? TREE_TYPE (decl) : TREE_TYPE (var)))
1623	mark_reg_pointer (x, get_pointer_alignment (var));
1624	}
1625
1626	/* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1627	that will reside in a pseudo register. */
1628
1629	static void
1630	expand_one_register_var (tree var)
1631	{
1632	if (TREE_CODE (var) == SSA_NAME)
1633	{
1634	int part = var_to_partition (map: SA.map, var);
1635	if (part != NO_PARTITION)
1636	{
1637	rtx x = SA.partition_to_pseudo[part];
1638	gcc_assert (x);
1639	gcc_assert (REG_P (x));
1640	return;
1641	}
1642	gcc_unreachable ();
1643	}
1644
1645	tree decl = var;
1646	tree type = TREE_TYPE (decl);
1647	machine_mode reg_mode = promote_decl_mode (decl, NULL);
1648	rtx x = gen_reg_rtx (reg_mode);
1649
1650	set_rtl (t: var, x);
1651
1652	/* Note if the object is a user variable. */
1653	if (!DECL_ARTIFICIAL (decl))
1654	mark_user_reg (x);
1655
1656	if (POINTER_TYPE_P (type))
1657	mark_reg_pointer (x, get_pointer_alignment (var));
1658	}
1659
1660	/* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL that
1661	has some associated error, e.g. its type is error-mark. We just need
1662	to pick something that won't crash the rest of the compiler. */
1663
1664	static void
1665	expand_one_error_var (tree var)
1666	{
1667	machine_mode mode = DECL_MODE (var);
1668	rtx x;
1669
1670	if (mode == BLKmode)
1671	x = gen_rtx_MEM (BLKmode, const0_rtx);
1672	else if (mode == VOIDmode)
1673	x = const0_rtx;
1674	else
1675	x = gen_reg_rtx (mode);
1676
1677	SET_DECL_RTL (var, x);
1678	}
1679
1680	/* A subroutine of expand_one_var. VAR is a variable that will be
1681	allocated to the local stack frame. Return true if we wish to
1682	add VAR to STACK_VARS so that it will be coalesced with other
1683	variables. Return false to allocate VAR immediately.
1684
1685	This function is used to reduce the number of variables considered
1686	for coalescing, which reduces the size of the quadratic problem. */
1687
1688	static bool
1689	defer_stack_allocation (tree var, bool toplevel)
1690	{
1691	tree size_unit = TREE_CODE (var) == SSA_NAME
1692	? TYPE_SIZE_UNIT (TREE_TYPE (var))
1693	: DECL_SIZE_UNIT (var);
1694	poly_uint64 size;
1695
1696	/* Whether the variable is small enough for immediate allocation not to be
1697	a problem with regard to the frame size. */
1698	bool smallish
1699	= (poly_int_tree_p (t: size_unit, value: &size)
1700	&& (estimated_poly_value (x: size)
1701	< param_min_size_for_stack_sharing));
1702
1703	/* If stack protection is enabled, *all* stack variables must be deferred,
1704	so that we can re-order the strings to the top of the frame.
1705	Similarly for Address Sanitizer. */
1706	if (flag_stack_protect || asan_sanitize_stack_p ())
1707	return true;
1708
1709	unsigned int align = TREE_CODE (var) == SSA_NAME
1710	? TYPE_ALIGN (TREE_TYPE (var))
1711	: DECL_ALIGN (var);
1712
1713	/* We handle "large" alignment via dynamic allocation. We want to handle
1714	this extra complication in only one place, so defer them. */
1715	if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1716	return true;
1717
1718	bool ignored = TREE_CODE (var) == SSA_NAME
1719	? !SSAVAR (var) || DECL_IGNORED_P (SSA_NAME_VAR (var))
1720	: DECL_IGNORED_P (var);
1721
1722	/* When optimization is enabled, DECL_IGNORED_P variables originally scoped
1723	might be detached from their block and appear at toplevel when we reach
1724	here. We want to coalesce them with variables from other blocks when
1725	the immediate contribution to the frame size would be noticeable. */
1726	if (toplevel && optimize > 0 && ignored && !smallish)
1727	return true;
1728
1729	/* Variables declared in the outermost scope automatically conflict
1730	with every other variable. The only reason to want to defer them
1731	at all is that, after sorting, we can more efficiently pack
1732	small variables in the stack frame. Continue to defer at -O2. */
1733	if (toplevel && optimize < 2)
1734	return false;
1735
1736	/* Without optimization, *most* variables are allocated from the
1737	stack, which makes the quadratic problem large exactly when we
1738	want compilation to proceed as quickly as possible. On the
1739	other hand, we don't want the function's stack frame size to
1740	get completely out of hand. So we avoid adding scalars and
1741	"small" aggregates to the list at all. */
1742	if (optimize == 0 && smallish)
1743	return false;
1744
1745	return true;
1746	}
1747
1748	/* A subroutine of expand_used_vars. Expand one variable according to
1749	its flavor. Variables to be placed on the stack are not actually
1750	expanded yet, merely recorded.
1751	When REALLY_EXPAND is false, only add stack values to be allocated.
1752	Return stack usage this variable is supposed to take.
1753	*/
1754
1755	static poly_uint64
1756	expand_one_var (tree var, bool toplevel, bool really_expand,
1757	bitmap forced_stack_var = NULL)
1758	{
1759	unsigned int align = BITS_PER_UNIT;
1760	tree origvar = var;
1761
1762	var = SSAVAR (var);
1763
1764	if (TREE_TYPE (var) != error_mark_node && VAR_P (var))
1765	{
1766	if (is_global_var (t: var))
1767	return 0;
1768
1769	/* Because we don't know if VAR will be in register or on stack,
1770	we conservatively assume it will be on stack even if VAR is
1771	eventually put into register after RA pass. For non-automatic
1772	variables, which won't be on stack, we collect alignment of
1773	type and ignore user specified alignment. Similarly for
1774	SSA_NAMEs for which use_register_for_decl returns true. */
1775	if (TREE_STATIC (var)
1776	|| DECL_EXTERNAL (var)
1777	|| (TREE_CODE (origvar) == SSA_NAME && use_register_for_decl (var)))
1778	align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
1779	TYPE_MODE (TREE_TYPE (var)),
1780	TYPE_ALIGN (TREE_TYPE (var)));
1781	else if (DECL_HAS_VALUE_EXPR_P (var)
1782	|| (DECL_RTL_SET_P (var) && MEM_P (DECL_RTL (var))))
1783	/* Don't consider debug only variables with DECL_HAS_VALUE_EXPR_P set
1784	or variables which were assigned a stack slot already by
1785	expand_one_stack_var_at - in the latter case DECL_ALIGN has been
1786	changed from the offset chosen to it. */
1787	align = crtl->stack_alignment_estimated;
1788	else
1789	align = MINIMUM_ALIGNMENT (var, DECL_MODE (var), DECL_ALIGN (var));
1790
1791	/* If the variable alignment is very large we'll dynamicaly allocate
1792	it, which means that in-frame portion is just a pointer. */
1793	if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1794	align = GET_MODE_ALIGNMENT (Pmode);
1795	}
1796
1797	record_alignment_for_reg_var (align);
1798
1799	poly_uint64 size;
1800	if (TREE_CODE (origvar) == SSA_NAME)
1801	{
1802	gcc_assert (!VAR_P (var)
1803	|| (!DECL_EXTERNAL (var)
1804	&& !DECL_HAS_VALUE_EXPR_P (var)
1805	&& !TREE_STATIC (var)
1806	&& TREE_TYPE (var) != error_mark_node
1807	&& !DECL_HARD_REGISTER (var)
1808	&& really_expand));
1809	}
1810	if (!VAR_P (var) && TREE_CODE (origvar) != SSA_NAME)
1811	;
1812	else if (DECL_EXTERNAL (var))
1813	;
1814	else if (DECL_HAS_VALUE_EXPR_P (var))
1815	;
1816	else if (TREE_STATIC (var))
1817	;
1818	else if (TREE_CODE (origvar) != SSA_NAME && DECL_RTL_SET_P (var))
1819	;
1820	else if (TREE_TYPE (var) == error_mark_node)
1821	{
1822	if (really_expand)
1823	expand_one_error_var (var);
1824	}
1825	else if (VAR_P (var) && DECL_HARD_REGISTER (var))
1826	{
1827	if (really_expand)
1828	{
1829	expand_one_hard_reg_var (var);
1830	if (!DECL_HARD_REGISTER (var))
1831	/* Invalid register specification. */
1832	expand_one_error_var (var);
1833	}
1834	}
1835	else if (use_register_for_decl (var)
1836	&& (!forced_stack_var
1837	|| !bitmap_bit_p (forced_stack_var, DECL_UID (var))))
1838	{
1839	if (really_expand)
1840	expand_one_register_var (var: origvar);
1841	}
1842	else if (!poly_int_tree_p (DECL_SIZE_UNIT (var), value: &size)
1843	|| !valid_constant_size_p (DECL_SIZE_UNIT (var)))
1844	{
1845	/* Reject variables which cover more than half of the address-space. */
1846	if (really_expand)
1847	{
1848	if (DECL_NONLOCAL_FRAME (var))
1849	error_at (DECL_SOURCE_LOCATION (current_function_decl),
1850	"total size of local objects is too large");
1851	else
1852	error_at (DECL_SOURCE_LOCATION (var),
1853	"size of variable %q+D is too large", var);
1854	expand_one_error_var (var);
1855	}
1856	}
1857	else if (defer_stack_allocation (var, toplevel))
1858	add_stack_var (decl: origvar, really_expand);
1859	else
1860	{
1861	if (really_expand)
1862	{
1863	if (lookup_attribute (attr_name: "naked",
1864	DECL_ATTRIBUTES (current_function_decl)))
1865	error ("cannot allocate stack for variable %q+D, naked function",
1866	var);
1867
1868	expand_one_stack_var (var: origvar);
1869	}
1870	return size;
1871	}
1872	return 0;
1873	}
1874
1875	/* A subroutine of expand_used_vars. Walk down through the BLOCK tree
1876	expanding variables. Those variables that can be put into registers
1877	are allocated pseudos; those that can't are put on the stack.
1878
1879	TOPLEVEL is true if this is the outermost BLOCK. */
1880
1881	static void
1882	expand_used_vars_for_block (tree block, bool toplevel, bitmap forced_stack_vars)
1883	{
1884	tree t;
1885
1886	/* Expand all variables at this level. */
1887	for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
1888	if (TREE_USED (t)
1889	&& ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
1890	|| !DECL_NONSHAREABLE (t)))
1891	expand_one_var (var: t, toplevel, really_expand: true, forced_stack_var: forced_stack_vars);
1892
1893	/* Expand all variables at containing levels. */
1894	for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
1895	expand_used_vars_for_block (block: t, toplevel: false, forced_stack_vars);
1896	}
1897
1898	/* A subroutine of expand_used_vars. Walk down through the BLOCK tree
1899	and clear TREE_USED on all local variables. */
1900
1901	static void
1902	clear_tree_used (tree block)
1903	{
1904	tree t;
1905
1906	for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
1907	/* if (!TREE_STATIC (t) && !DECL_EXTERNAL (t)) */
1908	if ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
1909	|| !DECL_NONSHAREABLE (t))
1910	TREE_USED (t) = 0;
1911
1912	for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
1913	clear_tree_used (block: t);
1914	}
1915
1916	/* Examine TYPE and determine a bit mask of the following features. */
1917
1918	#define SPCT_HAS_LARGE_CHAR_ARRAY 1
1919	#define SPCT_HAS_SMALL_CHAR_ARRAY 2
1920	#define SPCT_HAS_ARRAY 4
1921	#define SPCT_HAS_AGGREGATE 8
1922
1923	static unsigned int
1924	stack_protect_classify_type (tree type)
1925	{
1926	unsigned int ret = 0;
1927	tree t;
1928
1929	switch (TREE_CODE (type))
1930	{
1931	case ARRAY_TYPE:
1932	t = TYPE_MAIN_VARIANT (TREE_TYPE (type));
1933	if (t == char_type_node
1934	|| t == signed_char_type_node
1935	|| t == unsigned_char_type_node)
1936	{
1937	unsigned HOST_WIDE_INT max = param_ssp_buffer_size;
1938	unsigned HOST_WIDE_INT len;
1939
1940	if (!TYPE_SIZE_UNIT (type)
1941	|| !tree_fits_uhwi_p (TYPE_SIZE_UNIT (type)))
1942	len = max;
1943	else
1944	len = tree_to_uhwi (TYPE_SIZE_UNIT (type));
1945
1946	if (len < max)
1947	ret = SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_ARRAY;
1948	else
1949	ret = SPCT_HAS_LARGE_CHAR_ARRAY | SPCT_HAS_ARRAY;
1950	}
1951	else
1952	ret = SPCT_HAS_ARRAY;
1953	break;
1954
1955	case UNION_TYPE:
1956	case QUAL_UNION_TYPE:
1957	case RECORD_TYPE:
1958	ret = SPCT_HAS_AGGREGATE;
1959	for (t = TYPE_FIELDS (type); t ; t = TREE_CHAIN (t))
1960	if (TREE_CODE (t) == FIELD_DECL)
1961	ret |= stack_protect_classify_type (TREE_TYPE (t));
1962	break;
1963
1964	default:
1965	break;
1966	}
1967
1968	return ret;
1969	}
1970
1971	/* Return nonzero if DECL should be segregated into the "vulnerable" upper
1972	part of the local stack frame. Remember if we ever return nonzero for
1973	any variable in this function. The return value is the phase number in
1974	which the variable should be allocated. */
1975
1976	static int
1977	stack_protect_decl_phase (tree decl)
1978	{
1979	unsigned int bits = stack_protect_classify_type (TREE_TYPE (decl));
1980	int ret = 0;
1981
1982	if (bits & SPCT_HAS_SMALL_CHAR_ARRAY)
1983	has_short_buffer = true;
1984
1985	tree attribs = DECL_ATTRIBUTES (current_function_decl);
1986	if (!lookup_attribute (attr_name: "no_stack_protector", list: attribs)
1987	&& (flag_stack_protect == SPCT_FLAG_ALL
1988	|| flag_stack_protect == SPCT_FLAG_STRONG
1989	|| (flag_stack_protect == SPCT_FLAG_EXPLICIT
1990	&& lookup_attribute (attr_name: "stack_protect", list: attribs))))
1991	{
1992	if ((bits & (SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_LARGE_CHAR_ARRAY))
1993	&& !(bits & SPCT_HAS_AGGREGATE))
1994	ret = 1;
1995	else if (bits & SPCT_HAS_ARRAY)
1996	ret = 2;
1997	}
1998	else
1999	ret = (bits & SPCT_HAS_LARGE_CHAR_ARRAY) != 0;
2000
2001	if (ret)
2002	has_protected_decls = true;
2003
2004	return ret;
2005	}
2006
2007	/* Two helper routines that check for phase 1 and phase 2. These are used
2008	as callbacks for expand_stack_vars. */
2009
2010	static bool
2011	stack_protect_decl_phase_1 (size_t i)
2012	{
2013	return stack_protect_decl_phase (decl: stack_vars[i].decl) == 1;
2014	}
2015
2016	static bool
2017	stack_protect_decl_phase_2 (size_t i)
2018	{
2019	return stack_protect_decl_phase (decl: stack_vars[i].decl) == 2;
2020	}
2021
2022	/* And helper function that checks for asan phase (with stack protector
2023	it is phase 3). This is used as callback for expand_stack_vars.
2024	Returns true if any of the vars in the partition need to be protected. */
2025
2026	static bool
2027	asan_decl_phase_3 (size_t i)
2028	{
2029	while (i != EOC)
2030	{
2031	if (asan_protect_stack_decl (decl: stack_vars[i].decl))
2032	return true;
2033	i = stack_vars[i].next;
2034	}
2035	return false;
2036	}
2037
2038	/* Ensure that variables in different stack protection phases conflict
2039	so that they are not merged and share the same stack slot.
2040	Return true if there are any address taken variables. */
2041
2042	static bool
2043	add_stack_protection_conflicts (void)
2044	{
2045	size_t i, j, n = stack_vars_num;
2046	unsigned char *phase;
2047	bool ret = false;
2048
2049	phase = XNEWVEC (unsigned char, n);
2050	for (i = 0; i < n; ++i)
2051	{
2052	phase[i] = stack_protect_decl_phase (decl: stack_vars[i].decl);
2053	if (TREE_ADDRESSABLE (stack_vars[i].decl))
2054	ret = true;
2055	}
2056
2057	for (i = 0; i < n; ++i)
2058	{
2059	unsigned char ph_i = phase[i];
2060	for (j = i + 1; j < n; ++j)
2061	if (ph_i != phase[j])
2062	add_stack_var_conflict (x: i, y: j);
2063	}
2064
2065	XDELETEVEC (phase);
2066	return ret;
2067	}
2068
2069	/* Create a decl for the guard at the top of the stack frame. */
2070
2071	static void
2072	create_stack_guard (void)
2073	{
2074	tree guard = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
2075	VAR_DECL, NULL, ptr_type_node);
2076	TREE_THIS_VOLATILE (guard) = 1;
2077	TREE_USED (guard) = 1;
2078	expand_one_stack_var (var: guard);
2079	crtl->stack_protect_guard = guard;
2080	}
2081
2082	/* Prepare for expanding variables. */
2083	static void
2084	init_vars_expansion (void)
2085	{
2086	/* Conflict bitmaps, and a few related temporary bitmaps, go here. */
2087	bitmap_obstack_initialize (&stack_var_bitmap_obstack);
2088
2089	/* A map from decl to stack partition. */
2090	decl_to_stack_part = new hash_map<tree, size_t>;
2091
2092	/* Initialize local stack smashing state. */
2093	has_protected_decls = false;
2094	has_short_buffer = false;
2095	if (hwasan_sanitize_stack_p ())
2096	hwasan_record_frame_init ();
2097	}
2098
2099	/* Free up stack variable graph data. */
2100	static void
2101	fini_vars_expansion (void)
2102	{
2103	bitmap_obstack_release (&stack_var_bitmap_obstack);
2104	if (stack_vars)
2105	XDELETEVEC (stack_vars);
2106	if (stack_vars_sorted)
2107	XDELETEVEC (stack_vars_sorted);
2108	stack_vars = NULL;
2109	stack_vars_sorted = NULL;
2110	stack_vars_alloc = stack_vars_num = 0;
2111	delete decl_to_stack_part;
2112	decl_to_stack_part = NULL;
2113	}
2114
2115	/* Make a fair guess for the size of the stack frame of the function
2116	in NODE. This doesn't have to be exact, the result is only used in
2117	the inline heuristics. So we don't want to run the full stack var
2118	packing algorithm (which is quadratic in the number of stack vars).
2119	Instead, we calculate the total size of all stack vars. This turns
2120	out to be a pretty fair estimate -- packing of stack vars doesn't
2121	happen very often. */
2122
2123	HOST_WIDE_INT
2124	estimated_stack_frame_size (struct cgraph_node *node)
2125	{
2126	poly_int64 size = 0;
2127	size_t i;
2128	tree var;
2129	struct function *fn = DECL_STRUCT_FUNCTION (node->decl);
2130
2131	push_cfun (new_cfun: fn);
2132
2133	init_vars_expansion ();
2134
2135	FOR_EACH_LOCAL_DECL (fn, i, var)
2136	if (auto_var_in_fn_p (var, fn->decl))
2137	size += expand_one_var (var, toplevel: true, really_expand: false);
2138
2139	if (stack_vars_num > 0)
2140	{
2141	/* Fake sorting the stack vars for account_stack_vars (). */
2142	stack_vars_sorted = XNEWVEC (size_t, stack_vars_num);
2143	for (i = 0; i < stack_vars_num; ++i)
2144	stack_vars_sorted[i] = i;
2145	size += account_stack_vars ();
2146	}
2147
2148	fini_vars_expansion ();
2149	pop_cfun ();
2150	return estimated_poly_value (x: size);
2151	}
2152
2153	/* Check if the current function has calls that use a return slot. */
2154
2155	static bool
2156	stack_protect_return_slot_p ()
2157	{
2158	basic_block bb;
2159
2160	FOR_ALL_BB_FN (bb, cfun)
2161	for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2162	!gsi_end_p (i: gsi); gsi_next (i: &gsi))
2163	{
2164	gimple *stmt = gsi_stmt (i: gsi);
2165	/* This assumes that calls to internal-only functions never
2166	use a return slot. */
2167	if (is_gimple_call (gs: stmt)
2168	&& !gimple_call_internal_p (gs: stmt)
2169	&& aggregate_value_p (TREE_TYPE (gimple_call_fntype (stmt)),
2170	gimple_call_fndecl (gs: stmt)))
2171	return true;
2172	}
2173	return false;
2174	}
2175
2176	/* Expand all variables used in the function. */
2177
2178	static rtx_insn *
2179	expand_used_vars (bitmap forced_stack_vars)
2180	{
2181	tree var, outer_block = DECL_INITIAL (current_function_decl);
2182	auto_vec<tree> maybe_local_decls;
2183	rtx_insn *var_end_seq = NULL;
2184	unsigned i;
2185	unsigned len;
2186	bool gen_stack_protect_signal = false;
2187
2188	/* Compute the phase of the stack frame for this function. */
2189	{
2190	int align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2191	int off = targetm.starting_frame_offset () % align;
2192	frame_phase = off ? align - off : 0;
2193	}
2194
2195	/* Set TREE_USED on all variables in the local_decls. */
2196	FOR_EACH_LOCAL_DECL (cfun, i, var)
2197	TREE_USED (var) = 1;
2198	/* Clear TREE_USED on all variables associated with a block scope. */
2199	clear_tree_used (DECL_INITIAL (current_function_decl));
2200
2201	init_vars_expansion ();
2202
2203	if (targetm.use_pseudo_pic_reg ())
2204	pic_offset_table_rtx = gen_reg_rtx (Pmode);
2205
2206	for (i = 0; i < SA.map->num_partitions; i++)
2207	{
2208	if (bitmap_bit_p (SA.partitions_for_parm_default_defs, i))
2209	continue;
2210
2211	tree var = partition_to_var (map: SA.map, i);
2212
2213	gcc_assert (!virtual_operand_p (var));
2214
2215	expand_one_ssa_partition (var);
2216	}
2217
2218	if (flag_stack_protect == SPCT_FLAG_STRONG)
2219	gen_stack_protect_signal = stack_protect_return_slot_p ();
2220
2221	/* At this point all variables on the local_decls with TREE_USED
2222	set are not associated with any block scope. Lay them out. */
2223
2224	len = vec_safe_length (cfun->local_decls);
2225	FOR_EACH_LOCAL_DECL (cfun, i, var)
2226	{
2227	bool expand_now = false;
2228
2229	/* Expanded above already. */
2230	if (is_gimple_reg (var))
2231	{
2232	TREE_USED (var) = 0;
2233	goto next;
2234	}
2235	/* We didn't set a block for static or extern because it's hard
2236	to tell the difference between a global variable (re)declared
2237	in a local scope, and one that's really declared there to
2238	begin with. And it doesn't really matter much, since we're
2239	not giving them stack space. Expand them now. */
2240	else if (TREE_STATIC (var) || DECL_EXTERNAL (var))
2241	expand_now = true;
2242
2243	/* Expand variables not associated with any block now. Those created by
2244	the optimizers could be live anywhere in the function. Those that
2245	could possibly have been scoped originally and detached from their
2246	block will have their allocation deferred so we coalesce them with
2247	others when optimization is enabled. */
2248	else if (TREE_USED (var))
2249	expand_now = true;
2250
2251	/* Finally, mark all variables on the list as used. We'll use
2252	this in a moment when we expand those associated with scopes. */
2253	TREE_USED (var) = 1;
2254
2255	if (expand_now)
2256	expand_one_var (var, toplevel: true, really_expand: true, forced_stack_var: forced_stack_vars);
2257
2258	next:
2259	if (DECL_ARTIFICIAL (var) && !DECL_IGNORED_P (var))
2260	{
2261	rtx rtl = DECL_RTL_IF_SET (var);
2262
2263	/* Keep artificial non-ignored vars in cfun->local_decls
2264	chain until instantiate_decls. */
2265	if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2266	add_local_decl (cfun, d: var);
2267	else if (rtl == NULL_RTX)
2268	/* If rtl isn't set yet, which can happen e.g. with
2269	-fstack-protector, retry before returning from this
2270	function. */
2271	maybe_local_decls.safe_push (obj: var);
2272	}
2273	}
2274
2275	/* We duplicated some of the decls in CFUN->LOCAL_DECLS.
2276
2277	+-----------------+-----------------+
2278	| ...processed... | ...duplicates...|
2279	+-----------------+-----------------+
2280	^
2281	+-- LEN points here.
2282
2283	We just want the duplicates, as those are the artificial
2284	non-ignored vars that we want to keep until instantiate_decls.
2285	Move them down and truncate the array. */
2286	if (!vec_safe_is_empty (cfun->local_decls))
2287	cfun->local_decls->block_remove (ix: 0, len);
2288
2289	/* At this point, all variables within the block tree with TREE_USED
2290	set are actually used by the optimized function. Lay them out. */
2291	expand_used_vars_for_block (block: outer_block, toplevel: true, forced_stack_vars);
2292
2293	tree attribs = DECL_ATTRIBUTES (current_function_decl);
2294	if (stack_vars_num > 0)
2295	{
2296	bool has_addressable_vars = false;
2297
2298	add_scope_conflicts ();
2299
2300	/* If stack protection is enabled, we don't share space between
2301	vulnerable data and non-vulnerable data. */
2302	if (flag_stack_protect != 0
2303	&& !lookup_attribute (attr_name: "no_stack_protector", list: attribs)
2304	&& (flag_stack_protect != SPCT_FLAG_EXPLICIT
2305	|| (flag_stack_protect == SPCT_FLAG_EXPLICIT
2306	&& lookup_attribute (attr_name: "stack_protect", list: attribs))))
2307	has_addressable_vars = add_stack_protection_conflicts ();
2308
2309	if (flag_stack_protect == SPCT_FLAG_STRONG && has_addressable_vars)
2310	gen_stack_protect_signal = true;
2311
2312	/* Now that we have collected all stack variables, and have computed a
2313	minimal interference graph, attempt to save some stack space. */
2314	partition_stack_vars ();
2315	if (dump_file)
2316	dump_stack_var_partition ();
2317	}
2318
2319
2320	if (!lookup_attribute (attr_name: "no_stack_protector", list: attribs))
2321	switch (flag_stack_protect)
2322	{
2323	case SPCT_FLAG_ALL:
2324	create_stack_guard ();
2325	break;
2326
2327	case SPCT_FLAG_STRONG:
2328	if (gen_stack_protect_signal
2329	|| cfun->calls_alloca
2330	|| has_protected_decls
2331	|| lookup_attribute (attr_name: "stack_protect", list: attribs))
2332	create_stack_guard ();
2333	break;
2334
2335	case SPCT_FLAG_DEFAULT:
2336	if (cfun->calls_alloca
2337	|| has_protected_decls
2338	|| lookup_attribute (attr_name: "stack_protect", list: attribs))
2339	create_stack_guard ();
2340	break;
2341
2342	case SPCT_FLAG_EXPLICIT:
2343	if (lookup_attribute (attr_name: "stack_protect", list: attribs))
2344	create_stack_guard ();
2345	break;
2346
2347	default:
2348	break;
2349	}
2350
2351	/* Assign rtl to each variable based on these partitions. */
2352	if (stack_vars_num > 0)
2353	{
2354	class stack_vars_data data;
2355
2356	data.asan_base = NULL_RTX;
2357	data.asan_alignb = 0;
2358
2359	/* Reorder decls to be protected by iterating over the variables
2360	array multiple times, and allocating out of each phase in turn. */
2361	/* ??? We could probably integrate this into the qsort we did
2362	earlier, such that we naturally see these variables first,
2363	and thus naturally allocate things in the right order. */
2364	if (has_protected_decls)
2365	{
2366	/* Phase 1 contains only character arrays. */
2367	expand_stack_vars (pred: stack_protect_decl_phase_1, data: &data);
2368
2369	/* Phase 2 contains other kinds of arrays. */
2370	if (!lookup_attribute (attr_name: "no_stack_protector", list: attribs)
2371	&& (flag_stack_protect == SPCT_FLAG_ALL
2372	|| flag_stack_protect == SPCT_FLAG_STRONG
2373	|| (flag_stack_protect == SPCT_FLAG_EXPLICIT
2374	&& lookup_attribute (attr_name: "stack_protect", list: attribs))))
2375	expand_stack_vars (pred: stack_protect_decl_phase_2, data: &data);
2376	}
2377
2378	if (asan_sanitize_stack_p ())
2379	/* Phase 3, any partitions that need asan protection
2380	in addition to phase 1 and 2. */
2381	expand_stack_vars (pred: asan_decl_phase_3, data: &data);
2382
2383	/* ASAN description strings don't yet have a syntax for expressing
2384	polynomial offsets. */
2385	HOST_WIDE_INT prev_offset;
2386	if (!data.asan_vec.is_empty ()
2387	&& frame_offset.is_constant (const_value: &prev_offset))
2388	{
2389	HOST_WIDE_INT offset, sz, redzonesz;
2390	redzonesz = ASAN_RED_ZONE_SIZE;
2391	sz = data.asan_vec[0] - prev_offset;
2392	if (data.asan_alignb > ASAN_RED_ZONE_SIZE
2393	&& data.asan_alignb <= 4096
2394	&& sz + ASAN_RED_ZONE_SIZE >= (int) data.asan_alignb)
2395	redzonesz = ((sz + ASAN_RED_ZONE_SIZE + data.asan_alignb - 1)
2396	& ~(data.asan_alignb - HOST_WIDE_INT_1)) - sz;
2397	/* Allocating a constant amount of space from a constant
2398	starting offset must give a constant result. */
2399	offset = (alloc_stack_frame_space (size: redzonesz, ASAN_RED_ZONE_SIZE)
2400	.to_constant ());
2401	data.asan_vec.safe_push (obj: prev_offset);
2402	data.asan_vec.safe_push (obj: offset);
2403	/* Leave space for alignment if STRICT_ALIGNMENT. */
2404	if (STRICT_ALIGNMENT)
2405	alloc_stack_frame_space (size: (GET_MODE_ALIGNMENT (SImode)
2406	<< ASAN_SHADOW_SHIFT)
2407	/ BITS_PER_UNIT, align: 1);
2408
2409	var_end_seq
2410	= asan_emit_stack_protection (virtual_stack_vars_rtx,
2411	data.asan_base,
2412	data.asan_alignb,
2413	data.asan_vec.address (),
2414	data.asan_decl_vec.address (),
2415	data.asan_vec.length ());
2416	}
2417
2418	expand_stack_vars (NULL, data: &data);
2419	}
2420
2421	if (hwasan_sanitize_stack_p ())
2422	hwasan_emit_prologue ();
2423	if (asan_sanitize_allocas_p () && cfun->calls_alloca)
2424	var_end_seq = asan_emit_allocas_unpoison (virtual_stack_dynamic_rtx,
2425	virtual_stack_vars_rtx,
2426	var_end_seq);
2427	else if (hwasan_sanitize_allocas_p () && cfun->calls_alloca)
2428	/* When using out-of-line instrumentation we only want to emit one function
2429	call for clearing the tags in a region of shadow stack. When there are
2430	alloca calls in this frame we want to emit a call using the
2431	virtual_stack_dynamic_rtx, but when not we use the hwasan_frame_extent
2432	rtx we created in expand_stack_vars. */
2433	var_end_seq = hwasan_emit_untag_frame (virtual_stack_dynamic_rtx,
2434	virtual_stack_vars_rtx);
2435	else if (hwasan_sanitize_stack_p ())
2436	/* If no variables were stored on the stack, `hwasan_get_frame_extent`
2437	will return NULL_RTX and hence `hwasan_emit_untag_frame` will return
2438	NULL (i.e. an empty sequence). */
2439	var_end_seq = hwasan_emit_untag_frame (hwasan_get_frame_extent (),
2440	virtual_stack_vars_rtx);
2441
2442	fini_vars_expansion ();
2443
2444	/* If there were any artificial non-ignored vars without rtl
2445	found earlier, see if deferred stack allocation hasn't assigned
2446	rtl to them. */
2447	FOR_EACH_VEC_ELT_REVERSE (maybe_local_decls, i, var)
2448	{
2449	rtx rtl = DECL_RTL_IF_SET (var);
2450
2451	/* Keep artificial non-ignored vars in cfun->local_decls
2452	chain until instantiate_decls. */
2453	if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2454	add_local_decl (cfun, d: var);
2455	}
2456
2457	/* If the target requires that FRAME_OFFSET be aligned, do it. */
2458	if (STACK_ALIGNMENT_NEEDED)
2459	{
2460	HOST_WIDE_INT align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2461	if (FRAME_GROWS_DOWNWARD)
2462	frame_offset = aligned_lower_bound (frame_offset, align);
2463	else
2464	frame_offset = aligned_upper_bound (frame_offset, align);
2465	}
2466
2467	return var_end_seq;
2468	}
2469
2470
2471	/* If we need to produce a detailed dump, print the tree representation
2472	for STMT to the dump file. SINCE is the last RTX after which the RTL
2473	generated for STMT should have been appended. */
2474
2475	static void
2476	maybe_dump_rtl_for_gimple_stmt (gimple *stmt, rtx_insn *since)
2477	{
2478	if (dump_file && (dump_flags & TDF_DETAILS))
2479	{
2480	fprintf (stream: dump_file, format: "\n;; ");
2481	print_gimple_stmt (dump_file, stmt, 0,
2482	TDF_SLIM | (dump_flags & TDF_LINENO));
2483	fprintf (stream: dump_file, format: "\n");
2484
2485	print_rtl (dump_file, since ? NEXT_INSN (insn: since) : since);
2486	}
2487	}
2488
2489	/* Maps the blocks that do not contain tree labels to rtx labels. */
2490
2491	static hash_map<basic_block, rtx_code_label *> *lab_rtx_for_bb;
2492
2493	/* Returns the label_rtx expression for a label starting basic block BB. */
2494
2495	static rtx_code_label *
2496	label_rtx_for_bb (basic_block bb ATTRIBUTE_UNUSED)
2497	{
2498	if (bb->flags & BB_RTL)
2499	return block_label (bb);
2500
2501	rtx_code_label **elt = lab_rtx_for_bb->get (k: bb);
2502	if (elt)
2503	return *elt;
2504
2505	/* Find the tree label if it is present. */
2506	gimple_stmt_iterator gsi = gsi_start_bb (bb);
2507	glabel *lab_stmt;
2508	if (!gsi_end_p (i: gsi)
2509	&& (lab_stmt = dyn_cast <glabel *> (p: gsi_stmt (i: gsi)))
2510	&& !DECL_NONLOCAL (gimple_label_label (lab_stmt)))
2511	return jump_target_rtx (gimple_label_label (gs: lab_stmt));
2512
2513	rtx_code_label *l = gen_label_rtx ();
2514	lab_rtx_for_bb->put (k: bb, v: l);
2515	return l;
2516	}
2517
2518
2519	/* A subroutine of expand_gimple_cond. Given E, a fallthrough edge
2520	of a basic block where we just expanded the conditional at the end,
2521	possibly clean up the CFG and instruction sequence. LAST is the
2522	last instruction before the just emitted jump sequence. */
2523
2524	static void
2525	maybe_cleanup_end_of_block (edge e, rtx_insn *last)
2526	{
2527	/* Special case: when jumpif decides that the condition is
2528	trivial it emits an unconditional jump (and the necessary
2529	barrier). But we still have two edges, the fallthru one is
2530	wrong. purge_dead_edges would clean this up later. Unfortunately
2531	we have to insert insns (and split edges) before
2532	find_many_sub_basic_blocks and hence before purge_dead_edges.
2533	But splitting edges might create new blocks which depend on the
2534	fact that if there are two edges there's no barrier. So the
2535	barrier would get lost and verify_flow_info would ICE. Instead
2536	of auditing all edge splitters to care for the barrier (which
2537	normally isn't there in a cleaned CFG), fix it here. */
2538	if (BARRIER_P (get_last_insn ()))
2539	{
2540	rtx_insn *insn;
2541	remove_edge (e);
2542	/* Now, we have a single successor block, if we have insns to
2543	insert on the remaining edge we potentially will insert
2544	it at the end of this block (if the dest block isn't feasible)
2545	in order to avoid splitting the edge. This insertion will take
2546	place in front of the last jump. But we might have emitted
2547	multiple jumps (conditional and one unconditional) to the
2548	same destination. Inserting in front of the last one then
2549	is a problem. See PR 40021. We fix this by deleting all
2550	jumps except the last unconditional one. */
2551	insn = PREV_INSN (insn: get_last_insn ());
2552	/* Make sure we have an unconditional jump. Otherwise we're
2553	confused. */
2554	gcc_assert (JUMP_P (insn) && !any_condjump_p (insn));
2555	for (insn = PREV_INSN (insn); insn != last;)
2556	{
2557	insn = PREV_INSN (insn);
2558	if (JUMP_P (NEXT_INSN (insn)))
2559	{
2560	if (!any_condjump_p (NEXT_INSN (insn)))
2561	{
2562	gcc_assert (BARRIER_P (NEXT_INSN (NEXT_INSN (insn))));
2563	delete_insn (NEXT_INSN (insn: NEXT_INSN (insn)));
2564	}
2565	delete_insn (NEXT_INSN (insn));
2566	}
2567	}
2568	}
2569	}
2570
2571	/* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_COND.
2572	Returns a new basic block if we've terminated the current basic
2573	block and created a new one. */
2574
2575	static basic_block
2576	expand_gimple_cond (basic_block bb, gcond *stmt)
2577	{
2578	basic_block new_bb, dest;
2579	edge true_edge;
2580	edge false_edge;
2581	rtx_insn *last2, *last;
2582	enum tree_code code;
2583	tree op0, op1;
2584
2585	code = gimple_cond_code (gs: stmt);
2586	op0 = gimple_cond_lhs (gs: stmt);
2587	op1 = gimple_cond_rhs (gs: stmt);
2588	/* We're sometimes presented with such code:
2589	D.123_1 = x < y;
2590	if (D.123_1 != 0)
2591	...
2592	This would expand to two comparisons which then later might
2593	be cleaned up by combine. But some pattern matchers like if-conversion
2594	work better when there's only one compare, so make up for this
2595	here as special exception if TER would have made the same change. */
2596	if (SA.values
2597	&& TREE_CODE (op0) == SSA_NAME
2598	&& TREE_CODE (TREE_TYPE (op0)) == BOOLEAN_TYPE
2599	&& TREE_CODE (op1) == INTEGER_CST
2600	&& ((gimple_cond_code (gs: stmt) == NE_EXPR
2601	&& integer_zerop (op1))
2602	|| (gimple_cond_code (gs: stmt) == EQ_EXPR
2603	&& integer_onep (op1)))
2604	&& bitmap_bit_p (SA.values, SSA_NAME_VERSION (op0)))
2605	{
2606	gimple *second = SSA_NAME_DEF_STMT (op0);
2607	if (gimple_code (g: second) == GIMPLE_ASSIGN)
2608	{
2609	enum tree_code code2 = gimple_assign_rhs_code (gs: second);
2610	if (TREE_CODE_CLASS (code2) == tcc_comparison)
2611	{
2612	code = code2;
2613	op0 = gimple_assign_rhs1 (gs: second);
2614	op1 = gimple_assign_rhs2 (gs: second);
2615	}
2616	/* If jumps are cheap and the target does not support conditional
2617	compare, turn some more codes into jumpy sequences. */
2618	else if (BRANCH_COST (optimize_insn_for_speed_p (), false) < 4
2619	&& targetm.gen_ccmp_first == NULL)
2620	{
2621	if ((code2 == BIT_AND_EXPR
2622	&& TYPE_PRECISION (TREE_TYPE (op0)) == 1
2623	&& TREE_CODE (gimple_assign_rhs2 (second)) != INTEGER_CST)
2624	|| code2 == TRUTH_AND_EXPR)
2625	{
2626	code = TRUTH_ANDIF_EXPR;
2627	op0 = gimple_assign_rhs1 (gs: second);
2628	op1 = gimple_assign_rhs2 (gs: second);
2629	}
2630	else if (code2 == BIT_IOR_EXPR || code2 == TRUTH_OR_EXPR)
2631	{
2632	code = TRUTH_ORIF_EXPR;
2633	op0 = gimple_assign_rhs1 (gs: second);
2634	op1 = gimple_assign_rhs2 (gs: second);
2635	}
2636	}
2637	}
2638	}
2639
2640	/* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
2641	into (x - C2) * C3 < C4. */
2642	if ((code == EQ_EXPR || code == NE_EXPR)
2643	&& TREE_CODE (op0) == SSA_NAME
2644	&& TREE_CODE (op1) == INTEGER_CST)
2645	code = maybe_optimize_mod_cmp (code, &op0, &op1);
2646
2647	/* Optimize (x - y) < 0 into x < y if x - y has undefined overflow. */
2648	if (!TYPE_UNSIGNED (TREE_TYPE (op0))
2649	&& (code == LT_EXPR || code == LE_EXPR
2650	|| code == GT_EXPR || code == GE_EXPR)
2651	&& integer_zerop (op1)
2652	&& TREE_CODE (op0) == SSA_NAME)
2653	maybe_optimize_sub_cmp_0 (code, &op0, &op1);
2654
2655	last2 = last = get_last_insn ();
2656
2657	extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
2658	set_curr_insn_location (gimple_location (g: stmt));
2659
2660	/* These flags have no purpose in RTL land. */
2661	true_edge->flags &= ~EDGE_TRUE_VALUE;
2662	false_edge->flags &= ~EDGE_FALSE_VALUE;
2663
2664	/* We can either have a pure conditional jump with one fallthru edge or
2665	two-way jump that needs to be decomposed into two basic blocks. */
2666	if (false_edge->dest == bb->next_bb)
2667	{
2668	jumpif_1 (code, op0, op1, label_rtx_for_bb (bb: true_edge->dest),
2669	true_edge->probability);
2670	maybe_dump_rtl_for_gimple_stmt (stmt, since: last);
2671	if (true_edge->goto_locus != UNKNOWN_LOCATION)
2672	set_curr_insn_location (true_edge->goto_locus);
2673	false_edge->flags |= EDGE_FALLTHRU;
2674	maybe_cleanup_end_of_block (e: false_edge, last);
2675	return NULL;
2676	}
2677	if (true_edge->dest == bb->next_bb)
2678	{
2679	jumpifnot_1 (code, op0, op1, label_rtx_for_bb (bb: false_edge->dest),
2680	false_edge->probability);
2681	maybe_dump_rtl_for_gimple_stmt (stmt, since: last);
2682	if (false_edge->goto_locus != UNKNOWN_LOCATION)
2683	set_curr_insn_location (false_edge->goto_locus);
2684	true_edge->flags |= EDGE_FALLTHRU;
2685	maybe_cleanup_end_of_block (e: true_edge, last);
2686	return NULL;
2687	}
2688
2689	jumpif_1 (code, op0, op1, label_rtx_for_bb (bb: true_edge->dest),
2690	true_edge->probability);
2691	last = get_last_insn ();
2692	if (false_edge->goto_locus != UNKNOWN_LOCATION)
2693	set_curr_insn_location (false_edge->goto_locus);
2694	emit_jump (label_rtx_for_bb (bb: false_edge->dest));
2695
2696	BB_END (bb) = last;
2697	if (BARRIER_P (BB_END (bb)))
2698	BB_END (bb) = PREV_INSN (BB_END (bb));
2699	update_bb_for_insn (bb);
2700
2701	new_bb = create_basic_block (NEXT_INSN (insn: last), get_last_insn (), bb);
2702	dest = false_edge->dest;
2703	redirect_edge_succ (false_edge, new_bb);
2704	false_edge->flags |= EDGE_FALLTHRU;
2705	new_bb->count = false_edge->count ();
2706	loop_p loop = find_common_loop (bb->loop_father, dest->loop_father);
2707	add_bb_to_loop (new_bb, loop);
2708	if (loop->latch == bb
2709	&& loop->header == dest)
2710	loop->latch = new_bb;
2711	make_single_succ_edge (new_bb, dest, 0);
2712	if (BARRIER_P (BB_END (new_bb)))
2713	BB_END (new_bb) = PREV_INSN (BB_END (new_bb));
2714	update_bb_for_insn (new_bb);
2715
2716	maybe_dump_rtl_for_gimple_stmt (stmt, since: last2);
2717
2718	if (true_edge->goto_locus != UNKNOWN_LOCATION)
2719	{
2720	set_curr_insn_location (true_edge->goto_locus);
2721	true_edge->goto_locus = curr_insn_location ();
2722	}
2723
2724	return new_bb;
2725	}
2726
2727	/* Mark all calls that can have a transaction restart. */
2728
2729	static void
2730	mark_transaction_restart_calls (gimple *stmt)
2731	{
2732	struct tm_restart_node dummy;
2733	tm_restart_node **slot;
2734
2735	if (!cfun->gimple_df->tm_restart)
2736	return;
2737
2738	dummy.stmt = stmt;
2739	slot = cfun->gimple_df->tm_restart->find_slot (value: &dummy, insert: NO_INSERT);
2740	if (slot)
2741	{
2742	struct tm_restart_node *n = *slot;
2743	tree list = n->label_or_list;
2744	rtx_insn *insn;
2745
2746	for (insn = next_real_insn (get_last_insn ());
2747	!CALL_P (insn);
2748	insn = next_real_insn (insn))
2749	continue;
2750
2751	if (TREE_CODE (list) == LABEL_DECL)
2752	add_reg_note (insn, REG_TM, label_rtx (list));
2753	else
2754	for (; list ; list = TREE_CHAIN (list))
2755	add_reg_note (insn, REG_TM, label_rtx (TREE_VALUE (list)));
2756	}
2757	}
2758
2759	/* A subroutine of expand_gimple_stmt_1, expanding one GIMPLE_CALL
2760	statement STMT. */
2761
2762	static void
2763	expand_call_stmt (gcall *stmt)
2764	{
2765	tree exp, decl, lhs;
2766	bool builtin_p;
2767	size_t i;
2768
2769	if (gimple_call_internal_p (gs: stmt))
2770	{
2771	expand_internal_call (stmt);
2772	return;
2773	}
2774
2775	/* If this is a call to a built-in function and it has no effect other
2776	than setting the lhs, try to implement it using an internal function
2777	instead. */
2778	decl = gimple_call_fndecl (gs: stmt);
2779	if (gimple_call_lhs (gs: stmt)
2780	&& !gimple_has_side_effects (stmt)
2781	&& (optimize || (decl && called_as_built_in (decl))))
2782	{
2783	internal_fn ifn = replacement_internal_fn (stmt);
2784	if (ifn != IFN_LAST)
2785	{
2786	expand_internal_call (ifn, stmt);
2787	return;
2788	}
2789	}
2790
2791	exp = build_vl_exp (CALL_EXPR, gimple_call_num_args (gs: stmt) + 3);
2792
2793	CALL_EXPR_FN (exp) = gimple_call_fn (gs: stmt);
2794	builtin_p = decl && fndecl_built_in_p (node: decl);
2795
2796	/* If this is not a builtin function, the function type through which the
2797	call is made may be different from the type of the function. */
2798	if (!builtin_p)
2799	CALL_EXPR_FN (exp)
2800	= fold_convert (build_pointer_type (gimple_call_fntype (stmt)),
2801	CALL_EXPR_FN (exp));
2802
2803	TREE_TYPE (exp) = gimple_call_return_type (gs: stmt);
2804	CALL_EXPR_STATIC_CHAIN (exp) = gimple_call_chain (gs: stmt);
2805
2806	for (i = 0; i < gimple_call_num_args (gs: stmt); i++)
2807	{
2808	tree arg = gimple_call_arg (gs: stmt, index: i);
2809	gimple *def;
2810	/* TER addresses into arguments of builtin functions so we have a
2811	chance to infer more correct alignment information. See PR39954. */
2812	if (builtin_p
2813	&& TREE_CODE (arg) == SSA_NAME
2814	&& (def = get_gimple_for_ssa_name (exp: arg))
2815	&& gimple_assign_rhs_code (gs: def) == ADDR_EXPR)
2816	arg = gimple_assign_rhs1 (gs: def);
2817	CALL_EXPR_ARG (exp, i) = arg;
2818	}
2819
2820	if (gimple_has_side_effects (stmt)
2821	/* ??? Downstream in expand_expr_real_1 we assume that expressions
2822	w/o side-effects do not throw so work around this here. */
2823	|| stmt_could_throw_p (cfun, stmt))
2824	TREE_SIDE_EFFECTS (exp) = 1;
2825
2826	if (gimple_call_nothrow_p (s: stmt))
2827	TREE_NOTHROW (exp) = 1;
2828
2829	CALL_EXPR_TAILCALL (exp) = gimple_call_tail_p (s: stmt);
2830	CALL_EXPR_MUST_TAIL_CALL (exp) = gimple_call_must_tail_p (s: stmt);
2831	CALL_EXPR_RETURN_SLOT_OPT (exp) = gimple_call_return_slot_opt_p (s: stmt);
2832	if (decl
2833	&& fndecl_built_in_p (node: decl, klass: BUILT_IN_NORMAL)
2834	&& ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (decl)))
2835	CALL_ALLOCA_FOR_VAR_P (exp) = gimple_call_alloca_for_var_p (s: stmt);
2836	else
2837	CALL_FROM_THUNK_P (exp) = gimple_call_from_thunk_p (s: stmt);
2838	CALL_EXPR_VA_ARG_PACK (exp) = gimple_call_va_arg_pack_p (s: stmt);
2839	CALL_EXPR_BY_DESCRIPTOR (exp) = gimple_call_by_descriptor_p (s: stmt);
2840	SET_EXPR_LOCATION (exp, gimple_location (stmt));
2841
2842	/* Must come after copying location. */
2843	copy_warning (exp, stmt);
2844
2845	/* Ensure RTL is created for debug args. */
2846	if (decl && DECL_HAS_DEBUG_ARGS_P (decl))
2847	{
2848	vec<tree, va_gc> **debug_args = decl_debug_args_lookup (decl);
2849	unsigned int ix;
2850	tree dtemp;
2851
2852	if (debug_args)
2853	for (ix = 1; (*debug_args)->iterate (ix, ptr: &dtemp); ix += 2)
2854	{
2855	gcc_assert (TREE_CODE (dtemp) == DEBUG_EXPR_DECL);
2856	expand_debug_expr (dtemp);
2857	}
2858	}
2859
2860	rtx_insn *before_call = get_last_insn ();
2861	lhs = gimple_call_lhs (gs: stmt);
2862	if (lhs)
2863	expand_assignment (lhs, exp, false);
2864	else
2865	expand_expr (exp, const0_rtx, VOIDmode, modifier: EXPAND_NORMAL);
2866
2867	/* If the gimple call is an indirect call and has 'nocf_check'
2868	attribute find a generated CALL insn to mark it as no
2869	control-flow verification is needed. */
2870	if (gimple_call_nocf_check_p (gs: stmt)
2871	&& !gimple_call_fndecl (gs: stmt))
2872	{
2873	rtx_insn *last = get_last_insn ();
2874	while (!CALL_P (last)
2875	&& last != before_call)
2876	last = PREV_INSN (insn: last);
2877
2878	if (last != before_call)
2879	add_reg_note (last, REG_CALL_NOCF_CHECK, const0_rtx);
2880	}
2881
2882	mark_transaction_restart_calls (stmt);
2883	}
2884
2885
2886	/* Generate RTL for an asm statement (explicit assembler code).
2887	STRING is a STRING_CST node containing the assembler code text,
2888	or an ADDR_EXPR containing a STRING_CST. VOL nonzero means the
2889	insn is volatile; don't optimize it. */
2890
2891	static void
2892	expand_asm_loc (tree string, int vol, location_t locus)
2893	{
2894	rtx body;
2895
2896	body = gen_rtx_ASM_INPUT_loc (VOIDmode,
2897	ggc_strdup (TREE_STRING_POINTER (string)),
2898	locus);
2899
2900	MEM_VOLATILE_P (body) = vol;
2901
2902	/* Non-empty basic ASM implicitly clobbers memory. */
2903	if (TREE_STRING_LENGTH (string) != 0)
2904	{
2905	rtx asm_op, clob;
2906	unsigned i, nclobbers;
2907	auto_vec<rtx> input_rvec, output_rvec;
2908	auto_vec<machine_mode> input_mode;
2909	auto_vec<const char *> constraints;
2910	auto_vec<rtx> use_rvec;
2911	auto_vec<rtx> clobber_rvec;
2912	HARD_REG_SET clobbered_regs;
2913	CLEAR_HARD_REG_SET (set&: clobbered_regs);
2914
2915	clob = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
2916	clobber_rvec.safe_push (obj: clob);
2917
2918	if (targetm.md_asm_adjust)
2919	targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
2920	constraints, use_rvec, clobber_rvec,
2921	clobbered_regs, locus);
2922
2923	asm_op = body;
2924	nclobbers = clobber_rvec.length ();
2925	auto nuses = use_rvec.length ();
2926	body = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (1 + nuses + nclobbers));
2927
2928	i = 0;
2929	XVECEXP (body, 0, i++) = asm_op;
2930	for (rtx use : use_rvec)
2931	XVECEXP (body, 0, i++) = gen_rtx_USE (VOIDmode, use);
2932	for (rtx clobber : clobber_rvec)
2933	XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobber);
2934	}
2935
2936	emit_insn (body);
2937	}
2938
2939	/* Return the number of times character C occurs in string S. */
2940	static int
2941	n_occurrences (int c, const char *s)
2942	{
2943	int n = 0;
2944	while (*s)
2945	n += (*s++ == c);
2946	return n;
2947	}
2948
2949	/* A subroutine of expand_asm_operands. Check that all operands have
2950	the same number of alternatives. Return true if so. */
2951
2952	static bool
2953	check_operand_nalternatives (const vec<const char *> &constraints)
2954	{
2955	unsigned len = constraints.length();
2956	if (len > 0)
2957	{
2958	int nalternatives = n_occurrences (c: ',', s: constraints[0]);
2959
2960	if (nalternatives + 1 > MAX_RECOG_ALTERNATIVES)
2961	{
2962	error ("too many alternatives in %<asm%>");
2963	return false;
2964	}
2965
2966	for (unsigned i = 1; i < len; ++i)
2967	if (n_occurrences (c: ',', s: constraints[i]) != nalternatives)
2968	{
2969	error ("operand constraints for %<asm%> differ "
2970	"in number of alternatives");
2971	return false;
2972	}
2973	}
2974	return true;
2975	}
2976
2977	/* Check for overlap between registers marked in CLOBBERED_REGS and
2978	anything inappropriate in T. Emit error and return the register
2979	variable definition for error, NULL_TREE for ok. */
2980
2981	static bool
2982	tree_conflicts_with_clobbers_p (tree t, HARD_REG_SET *clobbered_regs,
2983	location_t loc)
2984	{
2985	/* Conflicts between asm-declared register variables and the clobber
2986	list are not allowed. */
2987	tree overlap = tree_overlaps_hard_reg_set (t, clobbered_regs);
2988
2989	if (overlap)
2990	{
2991	error_at (loc, "%<asm%> specifier for variable %qE conflicts with "
2992	"%<asm%> clobber list", DECL_NAME (overlap));
2993
2994	/* Reset registerness to stop multiple errors emitted for a single
2995	variable. */
2996	DECL_REGISTER (overlap) = 0;
2997	return true;
2998	}
2999
3000	return false;
3001	}
3002
3003	/* Check that the given REGNO spanning NREGS is a valid
3004	asm clobber operand. Some HW registers cannot be
3005	saved/restored, hence they should not be clobbered by
3006	asm statements. */
3007	static bool
3008	asm_clobber_reg_is_valid (int regno, int nregs, const char *regname)
3009	{
3010	bool is_valid = true;
3011	HARD_REG_SET regset;
3012
3013	CLEAR_HARD_REG_SET (set&: regset);
3014
3015	add_range_to_hard_reg_set (regs: &regset, regno, nregs);
3016
3017	/* Clobbering the PIC register is an error. */
3018	if (PIC_OFFSET_TABLE_REGNUM != INVALID_REGNUM
3019	&& overlaps_hard_reg_set_p (regs: regset, Pmode, PIC_OFFSET_TABLE_REGNUM))
3020	{
3021	/* ??? Diagnose during gimplification? */
3022	error ("PIC register clobbered by %qs in %<asm%>", regname);
3023	is_valid = false;
3024	}
3025	else if (!in_hard_reg_set_p
3026	(accessible_reg_set, reg_raw_mode[regno], regno))
3027	{
3028	/* ??? Diagnose during gimplification? */
3029	error ("the register %qs cannot be clobbered in %<asm%>"
3030	" for the current target", regname);
3031	is_valid = false;
3032	}
3033
3034	/* Clobbering the stack pointer register is deprecated. GCC expects
3035	the value of the stack pointer after an asm statement to be the same
3036	as it was before, so no asm can validly clobber the stack pointer in
3037	the usual sense. Adding the stack pointer to the clobber list has
3038	traditionally had some undocumented and somewhat obscure side-effects. */
3039	if (overlaps_hard_reg_set_p (regs: regset, Pmode, STACK_POINTER_REGNUM))
3040	{
3041	crtl->sp_is_clobbered_by_asm = true;
3042	if (warning (OPT_Wdeprecated, "listing the stack pointer register"
3043	" %qs in a clobber list is deprecated", regname))
3044	inform (input_location, "the value of the stack pointer after"
3045	" an %<asm%> statement must be the same as it was before"
3046	" the statement");
3047	}
3048
3049	return is_valid;
3050	}
3051
3052	/* Generate RTL for an asm statement with arguments.
3053	STRING is the instruction template.
3054	OUTPUTS is a list of output arguments (lvalues); INPUTS a list of inputs.
3055	Each output or input has an expression in the TREE_VALUE and
3056	a tree list in TREE_PURPOSE which in turn contains a constraint
3057	name in TREE_VALUE (or NULL_TREE) and a constraint string
3058	in TREE_PURPOSE.
3059	CLOBBERS is a list of STRING_CST nodes each naming a hard register
3060	that is clobbered by this insn.
3061
3062	LABELS is a list of labels, and if LABELS is non-NULL, FALLTHRU_BB
3063	should be the fallthru basic block of the asm goto.
3064
3065	Not all kinds of lvalue that may appear in OUTPUTS can be stored directly.
3066	Some elements of OUTPUTS may be replaced with trees representing temporary
3067	values. The caller should copy those temporary values to the originally
3068	specified lvalues.
3069
3070	VOL nonzero means the insn is volatile; don't optimize it. */
3071
3072	static void
3073	expand_asm_stmt (gasm *stmt)
3074	{
3075	class save_input_location
3076	{
3077	location_t old;
3078
3079	public:
3080	explicit save_input_location(location_t where)
3081	{
3082	old = input_location;
3083	input_location = where;
3084	}
3085
3086	~save_input_location()
3087	{
3088	input_location = old;
3089	}
3090	};
3091
3092	location_t locus = gimple_location (g: stmt);
3093
3094	if (gimple_asm_input_p (asm_stmt: stmt))
3095	{
3096	const char *s = gimple_asm_string (asm_stmt: stmt);
3097	tree string = build_string (strlen (s: s), s);
3098	expand_asm_loc (string, vol: gimple_asm_volatile_p (asm_stmt: stmt), locus);
3099	return;
3100	}
3101
3102	/* There are some legacy diagnostics in here. */
3103	save_input_location s_i_l(locus);
3104
3105	unsigned noutputs = gimple_asm_noutputs (asm_stmt: stmt);
3106	unsigned ninputs = gimple_asm_ninputs (asm_stmt: stmt);
3107	unsigned nlabels = gimple_asm_nlabels (asm_stmt: stmt);
3108	unsigned i;
3109	bool error_seen = false;
3110
3111	/* ??? Diagnose during gimplification? */
3112	if (ninputs + noutputs + nlabels > MAX_RECOG_OPERANDS)
3113	{
3114	error_at (locus, "more than %d operands in %<asm%>", MAX_RECOG_OPERANDS);
3115	return;
3116	}
3117
3118	auto_vec<tree, MAX_RECOG_OPERANDS> output_tvec;
3119	auto_vec<tree, MAX_RECOG_OPERANDS> input_tvec;
3120	auto_vec<const char *, MAX_RECOG_OPERANDS> constraints;
3121
3122	/* Copy the gimple vectors into new vectors that we can manipulate. */
3123
3124	output_tvec.safe_grow (len: noutputs, exact: true);
3125	input_tvec.safe_grow (len: ninputs, exact: true);
3126	constraints.safe_grow (len: noutputs + ninputs, exact: true);
3127
3128	for (i = 0; i < noutputs; ++i)
3129	{
3130	tree t = gimple_asm_output_op (asm_stmt: stmt, index: i);
3131	output_tvec[i] = TREE_VALUE (t);
3132	constraints[i] = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3133	}
3134	for (i = 0; i < ninputs; i++)
3135	{
3136	tree t = gimple_asm_input_op (asm_stmt: stmt, index: i);
3137	input_tvec[i] = TREE_VALUE (t);
3138	constraints[i + noutputs]
3139	= TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3140	}
3141
3142	/* ??? Diagnose during gimplification? */
3143	if (! check_operand_nalternatives (constraints))
3144	return;
3145
3146	/* Count the number of meaningful clobbered registers, ignoring what
3147	we would ignore later. */
3148	auto_vec<rtx> clobber_rvec;
3149	HARD_REG_SET clobbered_regs;
3150	CLEAR_HARD_REG_SET (set&: clobbered_regs);
3151
3152	if (unsigned n = gimple_asm_nclobbers (asm_stmt: stmt))
3153	{
3154	clobber_rvec.reserve (nelems: n);
3155	for (i = 0; i < n; i++)
3156	{
3157	tree t = gimple_asm_clobber_op (asm_stmt: stmt, index: i);
3158	const char *regname = TREE_STRING_POINTER (TREE_VALUE (t));
3159	int nregs, j;
3160
3161	j = decode_reg_name_and_count (regname, &nregs);
3162	if (j < 0)
3163	{
3164	if (j == -2)
3165	{
3166	/* ??? Diagnose during gimplification? */
3167	error_at (locus, "unknown register name %qs in %<asm%>",
3168	regname);
3169	error_seen = true;
3170	}
3171	else if (j == -4)
3172	{
3173	rtx x = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
3174	clobber_rvec.safe_push (obj: x);
3175	}
3176	else
3177	{
3178	/* Otherwise we should have -1 == empty string
3179	or -3 == cc, which is not a register. */
3180	gcc_assert (j == -1 || j == -3);
3181	}
3182	}
3183	else
3184	for (int reg = j; reg < j + nregs; reg++)
3185	{
3186	if (!asm_clobber_reg_is_valid (regno: reg, nregs, regname))
3187	return;
3188
3189	SET_HARD_REG_BIT (set&: clobbered_regs, bit: reg);
3190	rtx x = gen_rtx_REG (reg_raw_mode[reg], reg);
3191	clobber_rvec.safe_push (obj: x);
3192	}
3193	}
3194	}
3195
3196	/* First pass over inputs and outputs checks validity and sets
3197	mark_addressable if needed. */
3198	/* ??? Diagnose during gimplification? */
3199
3200	for (i = 0; i < noutputs; ++i)
3201	{
3202	tree val = output_tvec[i];
3203	tree type = TREE_TYPE (val);
3204	const char *constraint;
3205	bool is_inout;
3206	bool allows_reg;
3207	bool allows_mem;
3208
3209	/* Try to parse the output constraint. If that fails, there's
3210	no point in going further. */
3211	constraint = constraints[i];
3212	if (!parse_output_constraint (&constraint, i, ninputs, noutputs,
3213	&allows_mem, &allows_reg, &is_inout))
3214	return;
3215
3216	/* If the output is a hard register, verify it doesn't conflict with
3217	any other operand's possible hard register use. */
3218	if (DECL_P (val)
3219	&& REG_P (DECL_RTL (val))
3220	&& HARD_REGISTER_P (DECL_RTL (val)))
3221	{
3222	unsigned j, output_hregno = REGNO (DECL_RTL (val));
3223	bool early_clobber_p = strchr (s: constraints[i], c: '&') != NULL;
3224	unsigned long match;
3225
3226	/* Verify the other outputs do not use the same hard register. */
3227	for (j = i + 1; j < noutputs; ++j)
3228	if (DECL_P (output_tvec[j])
3229	&& REG_P (DECL_RTL (output_tvec[j]))
3230	&& HARD_REGISTER_P (DECL_RTL (output_tvec[j]))
3231	&& output_hregno == REGNO (DECL_RTL (output_tvec[j])))
3232	{
3233	error_at (locus, "invalid hard register usage between output "
3234	"operands");
3235	error_seen = true;
3236	}
3237
3238	/* Verify matching constraint operands use the same hard register
3239	and that the non-matching constraint operands do not use the same
3240	hard register if the output is an early clobber operand. */
3241	for (j = 0; j < ninputs; ++j)
3242	if (DECL_P (input_tvec[j])
3243	&& REG_P (DECL_RTL (input_tvec[j]))
3244	&& HARD_REGISTER_P (DECL_RTL (input_tvec[j])))
3245	{
3246	unsigned input_hregno = REGNO (DECL_RTL (input_tvec[j]));
3247	switch (*constraints[j + noutputs])
3248	{
3249	case '0': case '1': case '2': case '3': case '4':
3250	case '5': case '6': case '7': case '8': case '9':
3251	match = strtoul (nptr: constraints[j + noutputs], NULL, base: 10);
3252	break;
3253	default:
3254	match = ULONG_MAX;
3255	break;
3256	}
3257	if (i == match
3258	&& output_hregno != input_hregno)
3259	{
3260	error_at (locus, "invalid hard register usage between "
3261	"output operand and matching constraint operand");
3262	error_seen = true;
3263	}
3264	else if (early_clobber_p
3265	&& i != match
3266	&& output_hregno == input_hregno)
3267	{
3268	error_at (locus, "invalid hard register usage between "
3269	"earlyclobber operand and input operand");
3270	error_seen = true;
3271	}
3272	}
3273	}
3274
3275	if (! allows_reg
3276	&& (allows_mem
3277	|| is_inout
3278	|| (DECL_P (val)
3279	&& REG_P (DECL_RTL (val))
3280	&& GET_MODE (DECL_RTL (val)) != TYPE_MODE (type))))
3281	mark_addressable (val);
3282	}
3283
3284	for (i = 0; i < ninputs; ++i)
3285	{
3286	bool allows_reg, allows_mem;
3287	const char *constraint;
3288
3289	constraint = constraints[i + noutputs];
3290	if (! parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3291	constraints.address (),
3292	&allows_mem, &allows_reg))
3293	return;
3294
3295	if (! allows_reg && allows_mem)
3296	mark_addressable (input_tvec[i]);
3297	}
3298
3299	/* Second pass evaluates arguments. */
3300
3301	/* Make sure stack is consistent for asm goto. */
3302	if (nlabels > 0)
3303	do_pending_stack_adjust ();
3304	int old_generating_concat_p = generating_concat_p;
3305
3306	/* Vector of RTX's of evaluated output operands. */
3307	auto_vec<rtx, MAX_RECOG_OPERANDS> output_rvec;
3308	auto_vec<int, MAX_RECOG_OPERANDS> inout_opnum;
3309	rtx_insn *after_rtl_seq = NULL, *after_rtl_end = NULL;
3310
3311	output_rvec.safe_grow (len: noutputs, exact: true);
3312
3313	for (i = 0; i < noutputs; ++i)
3314	{
3315	tree val = output_tvec[i];
3316	tree type = TREE_TYPE (val);
3317	bool is_inout, allows_reg, allows_mem, ok;
3318	rtx op;
3319
3320	ok = parse_output_constraint (&constraints[i], i, ninputs,
3321	noutputs, &allows_mem, &allows_reg,
3322	&is_inout);
3323	gcc_assert (ok);
3324
3325	/* If an output operand is not a decl or indirect ref and our constraint
3326	allows a register, make a temporary to act as an intermediate.
3327	Make the asm insn write into that, then we will copy it to
3328	the real output operand. Likewise for promoted variables. */
3329
3330	generating_concat_p = 0;
3331
3332	gcc_assert (TREE_CODE (val) != INDIRECT_REF);
3333	if (((TREE_CODE (val) == MEM_REF
3334	&& TREE_CODE (TREE_OPERAND (val, 0)) != ADDR_EXPR)
3335	&& allows_mem)
3336	|| (DECL_P (val)
3337	&& (allows_mem || REG_P (DECL_RTL (val)))
3338	&& ! (REG_P (DECL_RTL (val))
3339	&& GET_MODE (DECL_RTL (val)) != TYPE_MODE (type)))
3340	|| ! allows_reg
3341	|| is_inout
3342	|| TREE_ADDRESSABLE (type)
3343	|| (!tree_fits_poly_int64_p (TYPE_SIZE (type))
3344	&& !known_size_p (a: max_int_size_in_bytes (type))))
3345	{
3346	op = expand_expr (exp: val, NULL_RTX, VOIDmode,
3347	modifier: !allows_reg ? EXPAND_MEMORY : EXPAND_WRITE);
3348	if (MEM_P (op))
3349	op = validize_mem (op);
3350
3351	if (! allows_reg && !MEM_P (op))
3352	{
3353	error_at (locus, "output number %d not directly addressable", i);
3354	error_seen = true;
3355	}
3356	if ((! allows_mem && MEM_P (op) && GET_MODE (op) != BLKmode)
3357	|| GET_CODE (op) == CONCAT)
3358	{
3359	rtx old_op = op;
3360	op = gen_reg_rtx (GET_MODE (op));
3361
3362	generating_concat_p = old_generating_concat_p;
3363
3364	if (is_inout)
3365	emit_move_insn (op, old_op);
3366
3367	push_to_sequence2 (after_rtl_seq, after_rtl_end);
3368	emit_move_insn (old_op, op);
3369	after_rtl_seq = get_insns ();
3370	after_rtl_end = get_last_insn ();
3371	end_sequence ();
3372	}
3373	}
3374	else
3375	{
3376	op = assign_temp (type, 0, 1);
3377	op = validize_mem (op);
3378	if (!MEM_P (op) && TREE_CODE (val) == SSA_NAME)
3379	set_reg_attrs_for_decl_rtl (SSA_NAME_VAR (val), x: op);
3380
3381	generating_concat_p = old_generating_concat_p;
3382
3383	push_to_sequence2 (after_rtl_seq, after_rtl_end);
3384	expand_assignment (val, make_tree (type, op), false);
3385	after_rtl_seq = get_insns ();
3386	after_rtl_end = get_last_insn ();
3387	end_sequence ();
3388	}
3389	output_rvec[i] = op;
3390
3391	if (is_inout)
3392	inout_opnum.safe_push (obj: i);
3393	}
3394
3395	const char *str = gimple_asm_string (asm_stmt: stmt);
3396	if (error_seen)
3397	{
3398	ninputs = 0;
3399	noutputs = 0;
3400	inout_opnum.truncate (size: 0);
3401	output_rvec.truncate (size: 0);
3402	clobber_rvec.truncate (size: 0);
3403	constraints.truncate (size: 0);
3404	CLEAR_HARD_REG_SET (set&: clobbered_regs);
3405	str = "";
3406	}
3407
3408	auto_vec<rtx, MAX_RECOG_OPERANDS> input_rvec;
3409	auto_vec<machine_mode, MAX_RECOG_OPERANDS> input_mode;
3410
3411	input_rvec.safe_grow (len: ninputs, exact: true);
3412	input_mode.safe_grow (len: ninputs, exact: true);
3413
3414	generating_concat_p = 0;
3415
3416	for (i = 0; i < ninputs; ++i)
3417	{
3418	tree val = input_tvec[i];
3419	tree type = TREE_TYPE (val);
3420	bool allows_reg, allows_mem, ok;
3421	const char *constraint;
3422	rtx op;
3423
3424	constraint = constraints[i + noutputs];
3425	ok = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3426	constraints.address (),
3427	&allows_mem, &allows_reg);
3428	gcc_assert (ok);
3429
3430	/* EXPAND_INITIALIZER will not generate code for valid initializer
3431	constants, but will still generate code for other types of operand.
3432	This is the behavior we want for constant constraints. */
3433	op = expand_expr (exp: val, NULL_RTX, VOIDmode,
3434	modifier: allows_reg ? EXPAND_NORMAL
3435	: allows_mem ? EXPAND_MEMORY
3436	: EXPAND_INITIALIZER);
3437
3438	/* Never pass a CONCAT to an ASM. */
3439	if (GET_CODE (op) == CONCAT)
3440	op = force_reg (GET_MODE (op), op);
3441	else if (MEM_P (op))
3442	op = validize_mem (op);
3443
3444	if (asm_operand_ok (op, constraint, NULL) <= 0)
3445	{
3446	if (allows_reg && TYPE_MODE (type) != BLKmode)
3447	op = force_reg (TYPE_MODE (type), op);
3448	else if (!allows_mem)
3449	warning_at (locus, 0, "%<asm%> operand %d probably does not match "
3450	"constraints", i + noutputs);
3451	else if (MEM_P (op))
3452	{
3453	/* We won't recognize either volatile memory or memory
3454	with a queued address as available a memory_operand
3455	at this point. Ignore it: clearly this *is* a memory. */
3456	}
3457	else
3458	gcc_unreachable ();
3459	}
3460	input_rvec[i] = op;
3461	input_mode[i] = TYPE_MODE (type);
3462	}
3463
3464	/* For in-out operands, copy output rtx to input rtx. */
3465	unsigned ninout = inout_opnum.length ();
3466	for (i = 0; i < ninout; i++)
3467	{
3468	int j = inout_opnum[i];
3469	rtx o = output_rvec[j];
3470
3471	input_rvec.safe_push (obj: o);
3472	input_mode.safe_push (GET_MODE (o));
3473
3474	char buffer[16];
3475	sprintf (s: buffer, format: "%d", j);
3476	constraints.safe_push (ggc_strdup (buffer));
3477	}
3478	ninputs += ninout;
3479
3480	/* Sometimes we wish to automatically clobber registers across an asm.
3481	Case in point is when the i386 backend moved from cc0 to a hard reg --
3482	maintaining source-level compatibility means automatically clobbering
3483	the flags register. */
3484	rtx_insn *after_md_seq = NULL;
3485	auto_vec<rtx> use_rvec;
3486	if (targetm.md_asm_adjust)
3487	after_md_seq
3488	= targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
3489	constraints, use_rvec, clobber_rvec,
3490	clobbered_regs, locus);
3491
3492	/* Do not allow the hook to change the output and input count,
3493	lest it mess up the operand numbering. */
3494	gcc_assert (output_rvec.length() == noutputs);
3495	gcc_assert (input_rvec.length() == ninputs);
3496	gcc_assert (constraints.length() == noutputs + ninputs);
3497
3498	/* But it certainly can adjust the uses and clobbers. */
3499	unsigned nuses = use_rvec.length ();
3500	unsigned nclobbers = clobber_rvec.length ();
3501
3502	/* Third pass checks for easy conflicts. */
3503	/* ??? Why are we doing this on trees instead of rtx. */
3504
3505	bool clobber_conflict_found = 0;
3506	for (i = 0; i < noutputs; ++i)
3507	if (tree_conflicts_with_clobbers_p (t: output_tvec[i], clobbered_regs: &clobbered_regs, loc: locus))
3508	clobber_conflict_found = 1;
3509	for (i = 0; i < ninputs - ninout; ++i)
3510	if (tree_conflicts_with_clobbers_p (t: input_tvec[i], clobbered_regs: &clobbered_regs, loc: locus))
3511	clobber_conflict_found = 1;
3512
3513	/* Make vectors for the expression-rtx, constraint strings,
3514	and named operands. */
3515
3516	rtvec argvec = rtvec_alloc (ninputs);
3517	rtvec constraintvec = rtvec_alloc (ninputs);
3518	rtvec labelvec = rtvec_alloc (nlabels);
3519
3520	rtx body = gen_rtx_ASM_OPERANDS ((noutputs == 0 ? VOIDmode
3521	: GET_MODE (output_rvec[0])),
3522	ggc_strdup (str),
3523	"", 0, argvec, constraintvec,
3524	labelvec, locus);
3525	MEM_VOLATILE_P (body) = gimple_asm_volatile_p (asm_stmt: stmt);
3526
3527	for (i = 0; i < ninputs; ++i)
3528	{
3529	ASM_OPERANDS_INPUT (body, i) = input_rvec[i];
3530	ASM_OPERANDS_INPUT_CONSTRAINT_EXP (body, i)
3531	= gen_rtx_ASM_INPUT_loc (input_mode[i],
3532	constraints[i + noutputs],
3533	locus);
3534	}
3535
3536	/* Copy labels to the vector. */
3537	rtx_code_label *fallthru_label = NULL;
3538	if (nlabels > 0)
3539	{
3540	basic_block fallthru_bb = NULL;
3541	edge fallthru = find_fallthru_edge (edges: gimple_bb (g: stmt)->succs);
3542	if (fallthru)
3543	fallthru_bb = fallthru->dest;
3544
3545	for (i = 0; i < nlabels; ++i)
3546	{
3547	tree label = TREE_VALUE (gimple_asm_label_op (stmt, i));
3548	rtx_insn *r;
3549	/* If asm goto has any labels in the fallthru basic block, use
3550	a label that we emit immediately after the asm goto. Expansion
3551	may insert further instructions into the same basic block after
3552	asm goto and if we don't do this, insertion of instructions on
3553	the fallthru edge might misbehave. See PR58670. */
3554	if (fallthru_bb && label_to_block (cfun, label) == fallthru_bb)
3555	{
3556	if (fallthru_label == NULL_RTX)
3557	fallthru_label = gen_label_rtx ();
3558	r = fallthru_label;
3559	}
3560	else
3561	r = label_rtx (label);
3562	ASM_OPERANDS_LABEL (body, i) = gen_rtx_LABEL_REF (Pmode, r);
3563	}
3564	}
3565
3566	/* Now, for each output, construct an rtx
3567	(set OUTPUT (asm_operands INSN OUTPUTCONSTRAINT OUTPUTNUMBER
3568	ARGVEC CONSTRAINTS OPNAMES))
3569	If there is more than one, put them inside a PARALLEL. */
3570
3571	if (noutputs == 0 && nuses == 0 && nclobbers == 0)
3572	{
3573	/* No output operands: put in a raw ASM_OPERANDS rtx. */
3574	if (nlabels > 0)
3575	emit_jump_insn (body);
3576	else
3577	emit_insn (body);
3578	}
3579	else if (noutputs == 1 && nuses == 0 && nclobbers == 0)
3580	{
3581	ASM_OPERANDS_OUTPUT_CONSTRAINT (body) = constraints[0];
3582	if (nlabels > 0)
3583	emit_jump_insn (gen_rtx_SET (output_rvec[0], body));
3584	else
3585	emit_insn (gen_rtx_SET (output_rvec[0], body));
3586	}
3587	else
3588	{
3589	rtx obody = body;
3590	int num = noutputs;
3591
3592	if (num == 0)
3593	num = 1;
3594
3595	body = gen_rtx_PARALLEL (VOIDmode,
3596	rtvec_alloc (num + nuses + nclobbers));
3597
3598	/* For each output operand, store a SET. */
3599	for (i = 0; i < noutputs; ++i)
3600	{
3601	rtx src, o = output_rvec[i];
3602	if (i == 0)
3603	{
3604	ASM_OPERANDS_OUTPUT_CONSTRAINT (obody) = constraints[0];
3605	src = obody;
3606	}
3607	else
3608	{
3609	src = gen_rtx_ASM_OPERANDS (GET_MODE (o),
3610	ASM_OPERANDS_TEMPLATE (obody),
3611	constraints[i], i, argvec,
3612	constraintvec, labelvec, locus);
3613	MEM_VOLATILE_P (src) = gimple_asm_volatile_p (asm_stmt: stmt);
3614	}
3615	XVECEXP (body, 0, i) = gen_rtx_SET (o, src);
3616	}
3617
3618	/* If there are no outputs (but there are some clobbers)
3619	store the bare ASM_OPERANDS into the PARALLEL. */
3620	if (i == 0)
3621	XVECEXP (body, 0, i++) = obody;
3622
3623	/* Add the uses specified by the target hook. No checking should
3624	be needed since this doesn't come directly from user code. */
3625	for (rtx use : use_rvec)
3626	XVECEXP (body, 0, i++) = gen_rtx_USE (VOIDmode, use);
3627
3628	/* Store (clobber REG) for each clobbered register specified. */
3629	for (unsigned j = 0; j < nclobbers; ++j)
3630	{
3631	rtx clobbered_reg = clobber_rvec[j];
3632
3633	/* Do sanity check for overlap between clobbers and respectively
3634	input and outputs that hasn't been handled. Such overlap
3635	should have been detected and reported above. */
3636	if (!clobber_conflict_found && REG_P (clobbered_reg))
3637	{
3638	/* We test the old body (obody) contents to avoid
3639	tripping over the under-construction body. */
3640	for (unsigned k = 0; k < noutputs; ++k)
3641	if (reg_overlap_mentioned_p (clobbered_reg, output_rvec[k]))
3642	internal_error ("%<asm%> clobber conflict with "
3643	"output operand");
3644
3645	for (unsigned k = 0; k < ninputs - ninout; ++k)
3646	if (reg_overlap_mentioned_p (clobbered_reg, input_rvec[k]))
3647	internal_error ("%<asm%> clobber conflict with "
3648	"input operand");
3649	}
3650
3651	XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobbered_reg);
3652	}
3653
3654	if (nlabels > 0)
3655	emit_jump_insn (body);
3656	else
3657	emit_insn (body);
3658	}
3659
3660	generating_concat_p = old_generating_concat_p;
3661
3662	if (fallthru_label)
3663	emit_label (fallthru_label);
3664
3665	if (after_md_seq)
3666	emit_insn (after_md_seq);
3667	if (after_rtl_seq)
3668	{
3669	if (nlabels == 0)
3670	emit_insn (after_rtl_seq);
3671	else
3672	{
3673	edge e;
3674	edge_iterator ei;
3675	unsigned int cnt = EDGE_COUNT (gimple_bb (stmt)->succs);
3676
3677	FOR_EACH_EDGE (e, ei, gimple_bb (stmt)->succs)
3678	{
3679	rtx_insn *copy;
3680	if (--cnt == 0)
3681	copy = after_rtl_seq;
3682	else
3683	{
3684	start_sequence ();
3685	duplicate_insn_chain (after_rtl_seq, after_rtl_end,
3686	NULL, NULL);
3687	copy = get_insns ();
3688	end_sequence ();
3689	}
3690	prepend_insn_to_edge (copy, e);
3691	}
3692	}
3693	}
3694
3695	free_temp_slots ();
3696	crtl->has_asm_statement = 1;
3697	}
3698
3699	/* Emit code to jump to the address
3700	specified by the pointer expression EXP. */
3701
3702	static void
3703	expand_computed_goto (tree exp)
3704	{
3705	rtx x = expand_normal (exp);
3706
3707	do_pending_stack_adjust ();
3708	emit_indirect_jump (x);
3709	}
3710
3711	/* Generate RTL code for a `goto' statement with target label LABEL.
3712	LABEL should be a LABEL_DECL tree node that was or will later be
3713	defined with `expand_label'. */
3714
3715	static void
3716	expand_goto (tree label)
3717	{
3718	if (flag_checking)
3719	{
3720	/* Check for a nonlocal goto to a containing function. Should have
3721	gotten translated to __builtin_nonlocal_goto. */
3722	tree context = decl_function_context (label);
3723	gcc_assert (!context || context == current_function_decl);
3724	}
3725
3726	emit_jump (jump_target_rtx (label));
3727	}
3728
3729	/* Output a return with no value. */
3730
3731	static void
3732	expand_null_return_1 (void)
3733	{
3734	clear_pending_stack_adjust ();
3735	do_pending_stack_adjust ();
3736	emit_jump (return_label);
3737	}
3738
3739	/* Generate RTL to return from the current function, with no value.
3740	(That is, we do not do anything about returning any value.) */
3741
3742	void
3743	expand_null_return (void)
3744	{
3745	/* If this function was declared to return a value, but we
3746	didn't, clobber the return registers so that they are not
3747	propagated live to the rest of the function. */
3748	clobber_return_register ();
3749
3750	expand_null_return_1 ();
3751	}
3752
3753	/* Generate RTL to return from the current function, with value VAL. */
3754
3755	static void
3756	expand_value_return (rtx val)
3757	{
3758	/* Copy the value to the return location unless it's already there. */
3759
3760	tree decl = DECL_RESULT (current_function_decl);
3761	rtx return_reg = DECL_RTL (decl);
3762	if (return_reg != val)
3763	{
3764	tree funtype = TREE_TYPE (current_function_decl);
3765	tree type = TREE_TYPE (decl);
3766	int unsignedp = TYPE_UNSIGNED (type);
3767	machine_mode old_mode = DECL_MODE (decl);
3768	machine_mode mode;
3769	if (DECL_BY_REFERENCE (decl))
3770	mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 2);
3771	else
3772	mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 1);
3773
3774	if (mode != old_mode)
3775	{
3776	/* Some ABIs require scalar floating point modes to be returned
3777	in a wider scalar integer mode. We need to explicitly
3778	reinterpret to an integer mode of the correct precision
3779	before extending to the desired result. */
3780	if (SCALAR_INT_MODE_P (mode)
3781	&& SCALAR_FLOAT_MODE_P (old_mode)
3782	&& known_gt (GET_MODE_SIZE (mode), GET_MODE_SIZE (old_mode)))
3783	val = convert_float_to_wider_int (mode, fmode: old_mode, x: val);
3784	else
3785	val = convert_modes (mode, oldmode: old_mode, x: val, unsignedp);
3786	}
3787
3788	if (GET_CODE (return_reg) == PARALLEL)
3789	emit_group_load (return_reg, val, type, int_size_in_bytes (type));
3790	else
3791	emit_move_insn (return_reg, val);
3792	}
3793
3794	expand_null_return_1 ();
3795	}
3796
3797	/* Generate RTL to evaluate the expression RETVAL and return it
3798	from the current function. */
3799
3800	static void
3801	expand_return (tree retval)
3802	{
3803	rtx result_rtl;
3804	rtx val = 0;
3805	tree retval_rhs;
3806
3807	/* If function wants no value, give it none. */
3808	if (VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
3809	{
3810	expand_normal (exp: retval);
3811	expand_null_return ();
3812	return;
3813	}
3814
3815	if (retval == error_mark_node)
3816	{
3817	/* Treat this like a return of no value from a function that
3818	returns a value. */
3819	expand_null_return ();
3820	return;
3821	}
3822	else if ((TREE_CODE (retval) == MODIFY_EXPR
3823	|| TREE_CODE (retval) == INIT_EXPR)
3824	&& TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
3825	retval_rhs = TREE_OPERAND (retval, 1);
3826	else
3827	retval_rhs = retval;
3828
3829	result_rtl = DECL_RTL (DECL_RESULT (current_function_decl));
3830
3831	/* If we are returning the RESULT_DECL, then the value has already
3832	been stored into it, so we don't have to do anything special. */
3833	if (TREE_CODE (retval_rhs) == RESULT_DECL)
3834	expand_value_return (val: result_rtl);
3835
3836	/* If the result is an aggregate that is being returned in one (or more)
3837	registers, load the registers here. */
3838
3839	else if (retval_rhs != 0
3840	&& TYPE_MODE (TREE_TYPE (retval_rhs)) == BLKmode
3841	&& REG_P (result_rtl))
3842	{
3843	val = copy_blkmode_to_reg (GET_MODE (result_rtl), retval_rhs);
3844	if (val)
3845	{
3846	/* Use the mode of the result value on the return register. */
3847	PUT_MODE (x: result_rtl, GET_MODE (val));
3848	expand_value_return (val);
3849	}
3850	else
3851	expand_null_return ();
3852	}
3853	else if (retval_rhs != 0
3854	&& !VOID_TYPE_P (TREE_TYPE (retval_rhs))
3855	&& (REG_P (result_rtl)
3856	|| (GET_CODE (result_rtl) == PARALLEL)))
3857	{
3858	/* Compute the return value into a temporary (usually a pseudo reg). */
3859	val
3860	= assign_temp (TREE_TYPE (DECL_RESULT (current_function_decl)), 0, 1);
3861	val = expand_expr (exp: retval_rhs, target: val, GET_MODE (val), modifier: EXPAND_NORMAL);
3862	val = force_not_mem (val);
3863	expand_value_return (val);
3864	}
3865	else
3866	{
3867	/* No hard reg used; calculate value into hard return reg. */
3868	expand_expr (exp: retval, const0_rtx, VOIDmode, modifier: EXPAND_NORMAL);
3869	expand_value_return (val: result_rtl);
3870	}
3871	}
3872
3873	/* Expand a clobber of LHS. If LHS is stored it in a multi-part
3874	register, tell the rtl optimizers that its value is no longer
3875	needed. */
3876
3877	static void
3878	expand_clobber (tree lhs)
3879	{
3880	if (DECL_P (lhs))
3881	{
3882	rtx decl_rtl = DECL_RTL_IF_SET (lhs);
3883	if (decl_rtl && REG_P (decl_rtl))
3884	{
3885	machine_mode decl_mode = GET_MODE (decl_rtl);
3886	if (maybe_gt (GET_MODE_SIZE (decl_mode),
3887	REGMODE_NATURAL_SIZE (decl_mode)))
3888	emit_clobber (decl_rtl);
3889	}
3890	}
3891	}
3892
3893	/* A subroutine of expand_gimple_stmt, expanding one gimple statement
3894	STMT that doesn't require special handling for outgoing edges. That
3895	is no tailcalls and no GIMPLE_COND. */
3896
3897	static void
3898	expand_gimple_stmt_1 (gimple *stmt)
3899	{
3900	tree op0;
3901
3902	set_curr_insn_location (gimple_location (g: stmt));
3903
3904	switch (gimple_code (g: stmt))
3905	{
3906	case GIMPLE_GOTO:
3907	op0 = gimple_goto_dest (gs: stmt);
3908	if (TREE_CODE (op0) == LABEL_DECL)
3909	expand_goto (label: op0);
3910	else
3911	expand_computed_goto (exp: op0);
3912	break;
3913	case GIMPLE_LABEL:
3914	expand_label (gimple_label_label (gs: as_a <glabel *> (p: stmt)));
3915	break;
3916	case GIMPLE_NOP:
3917	case GIMPLE_PREDICT:
3918	break;
3919	case GIMPLE_SWITCH:
3920	{
3921	gswitch *swtch = as_a <gswitch *> (p: stmt);
3922	if (gimple_switch_num_labels (gs: swtch) == 1)
3923	expand_goto (CASE_LABEL (gimple_switch_default_label (swtch)));
3924	else
3925	expand_case (swtch);
3926	}
3927	break;
3928	case GIMPLE_ASM:
3929	expand_asm_stmt (stmt: as_a <gasm *> (p: stmt));
3930	break;
3931	case GIMPLE_CALL:
3932	expand_call_stmt (stmt: as_a <gcall *> (p: stmt));
3933	break;
3934
3935	case GIMPLE_RETURN:
3936	{
3937	op0 = gimple_return_retval (gs: as_a <greturn *> (p: stmt));
3938
3939	/* If a return doesn't have a location, it very likely represents
3940	multiple user returns so we cannot let it inherit the location
3941	of the last statement of the previous basic block in RTL. */
3942	if (!gimple_has_location (g: stmt))
3943	set_curr_insn_location (cfun->function_end_locus);
3944
3945	if (op0 && op0 != error_mark_node)
3946	{
3947	tree result = DECL_RESULT (current_function_decl);
3948
3949	/* If we are not returning the current function's RESULT_DECL,
3950	build an assignment to it. */
3951	if (op0 != result)
3952	{
3953	/* I believe that a function's RESULT_DECL is unique. */
3954	gcc_assert (TREE_CODE (op0) != RESULT_DECL);
3955
3956	/* ??? We'd like to use simply expand_assignment here,
3957	but this fails if the value is of BLKmode but the return
3958	decl is a register. expand_return has special handling
3959	for this combination, which eventually should move
3960	to common code. See comments there. Until then, let's
3961	build a modify expression :-/ */
3962	op0 = build2 (MODIFY_EXPR, TREE_TYPE (result),
3963	result, op0);
3964	}
3965	}
3966
3967	if (!op0)
3968	expand_null_return ();
3969	else
3970	expand_return (retval: op0);
3971	}
3972	break;
3973
3974	case GIMPLE_ASSIGN:
3975	{
3976	gassign *assign_stmt = as_a <gassign *> (p: stmt);
3977	tree lhs = gimple_assign_lhs (gs: assign_stmt);
3978
3979	/* Tree expand used to fiddle with |= and &= of two bitfield
3980	COMPONENT_REFs here. This can't happen with gimple, the LHS
3981	of binary assigns must be a gimple reg. */
3982
3983	if (TREE_CODE (lhs) != SSA_NAME
3984	|| gimple_assign_rhs_class (gs: assign_stmt) == GIMPLE_SINGLE_RHS)
3985	{
3986	tree rhs = gimple_assign_rhs1 (gs: assign_stmt);
3987	gcc_assert (gimple_assign_rhs_class (assign_stmt)
3988	== GIMPLE_SINGLE_RHS);
3989	if (gimple_has_location (g: stmt) && CAN_HAVE_LOCATION_P (rhs)
3990	/* Do not put locations on possibly shared trees. */
3991	&& !is_gimple_min_invariant (rhs))
3992	SET_EXPR_LOCATION (rhs, gimple_location (stmt));
3993	if (TREE_CLOBBER_P (rhs))
3994	/* This is a clobber to mark the going out of scope for
3995	this LHS. */
3996	expand_clobber (lhs);
3997	else
3998	expand_assignment (lhs, rhs,
3999	gimple_assign_nontemporal_move_p (
4000	gs: assign_stmt));
4001	}
4002	else
4003	{
4004	rtx target, temp;
4005	bool nontemporal = gimple_assign_nontemporal_move_p (gs: assign_stmt);
4006	bool promoted = false;
4007
4008	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
4009	if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
4010	promoted = true;
4011
4012	/* If we want to use a nontemporal store, force the value to
4013	register first. If we store into a promoted register,
4014	don't directly expand to target. */
4015	temp = nontemporal || promoted ? NULL_RTX : target;
4016	temp = expand_expr_real_gassign (assign_stmt, temp,
4017	GET_MODE (target), modifier: EXPAND_NORMAL);
4018
4019	if (temp == target)
4020	;
4021	else if (promoted)
4022	{
4023	int unsignedp = SUBREG_PROMOTED_SIGN (target);
4024	/* If TEMP is a VOIDmode constant, use convert_modes to make
4025	sure that we properly convert it. */
4026	if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode)
4027	{
4028	temp = convert_modes (GET_MODE (target),
4029	TYPE_MODE (TREE_TYPE (lhs)),
4030	x: temp, unsignedp);
4031	temp = convert_modes (GET_MODE (SUBREG_REG (target)),
4032	GET_MODE (target), x: temp, unsignedp);
4033	}
4034
4035	convert_move (SUBREG_REG (target), temp, unsignedp);
4036	}
4037	else if (nontemporal && emit_storent_insn (to: target, from: temp))
4038	;
4039	else
4040	{
4041	temp = force_operand (temp, target);
4042	if (temp != target)
4043	emit_move_insn (target, temp);
4044	}
4045	}
4046	}
4047	break;
4048
4049	default:
4050	gcc_unreachable ();
4051	}
4052	}
4053
4054	/* Expand one gimple statement STMT and return the last RTL instruction
4055	before any of the newly generated ones.
4056
4057	In addition to generating the necessary RTL instructions this also
4058	sets REG_EH_REGION notes if necessary and sets the current source
4059	location for diagnostics. */
4060
4061	static rtx_insn *
4062	expand_gimple_stmt (gimple *stmt)
4063	{
4064	location_t saved_location = input_location;
4065	rtx_insn *last = get_last_insn ();
4066	int lp_nr;
4067
4068	gcc_assert (cfun);
4069
4070	/* We need to save and restore the current source location so that errors
4071	discovered during expansion are emitted with the right location. But
4072	it would be better if the diagnostic routines used the source location
4073	embedded in the tree nodes rather than globals. */
4074	if (gimple_has_location (g: stmt))
4075	input_location = gimple_location (g: stmt);
4076
4077	expand_gimple_stmt_1 (stmt);
4078
4079	/* Free any temporaries used to evaluate this statement. */
4080	free_temp_slots ();
4081
4082	input_location = saved_location;
4083
4084	/* Mark all insns that may trap. */
4085	lp_nr = lookup_stmt_eh_lp (stmt);
4086	if (lp_nr)
4087	{
4088	rtx_insn *insn;
4089	for (insn = next_real_insn (last); insn;
4090	insn = next_real_insn (insn))
4091	{
4092	if (! find_reg_note (insn, REG_EH_REGION, NULL_RTX)
4093	/* If we want exceptions for non-call insns, any
4094	may_trap_p instruction may throw. */
4095	&& GET_CODE (PATTERN (insn)) != CLOBBER
4096	&& GET_CODE (PATTERN (insn)) != USE
4097	&& insn_could_throw_p (insn))
4098	make_reg_eh_region_note (insn, ecf_flags: 0, lp_nr);
4099	}
4100	}
4101
4102	return last;
4103	}
4104
4105	/* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_CALL
4106	that has CALL_EXPR_TAILCALL set. Returns non-null if we actually
4107	generated a tail call (something that might be denied by the ABI
4108	rules governing the call; see calls.cc).
4109
4110	Sets CAN_FALLTHRU if we generated a *conditional* tail call, and
4111	can still reach the rest of BB. The case here is __builtin_sqrt,
4112	where the NaN result goes through the external function (with a
4113	tailcall) and the normal result happens via a sqrt instruction. */
4114
4115	static basic_block
4116	expand_gimple_tailcall (basic_block bb, gcall *stmt, bool *can_fallthru)
4117	{
4118	rtx_insn *last2, *last;
4119	edge e;
4120	edge_iterator ei;
4121	profile_probability probability;
4122
4123	last2 = last = expand_gimple_stmt (stmt);
4124
4125	for (last = NEXT_INSN (insn: last); last; last = NEXT_INSN (insn: last))
4126	if (CALL_P (last) && SIBLING_CALL_P (last))
4127	goto found;
4128
4129	maybe_dump_rtl_for_gimple_stmt (stmt, since: last2);
4130
4131	*can_fallthru = true;
4132	return NULL;
4133
4134	found:
4135	/* ??? Wouldn't it be better to just reset any pending stack adjust?
4136	Any instructions emitted here are about to be deleted. */
4137	do_pending_stack_adjust ();
4138
4139	/* Remove any non-eh, non-abnormal edges that don't go to exit. */
4140	/* ??? I.e. the fallthrough edge. HOWEVER! If there were to be
4141	EH or abnormal edges, we shouldn't have created a tail call in
4142	the first place. So it seems to me we should just be removing
4143	all edges here, or redirecting the existing fallthru edge to
4144	the exit block. */
4145
4146	probability = profile_probability::never ();
4147
4148	for (ei = ei_start (bb->succs); (e = ei_safe_edge (i: ei)); )
4149	{
4150	if (!(e->flags & (EDGE_ABNORMAL | EDGE_EH)))
4151	{
4152	if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
4153	e->dest->count -= e->count ();
4154	probability += e->probability;
4155	remove_edge (e);
4156	}
4157	else
4158	ei_next (i: &ei);
4159	}
4160
4161	/* This is somewhat ugly: the call_expr expander often emits instructions
4162	after the sibcall (to perform the function return). These confuse the
4163	find_many_sub_basic_blocks code, so we need to get rid of these. */
4164	last = NEXT_INSN (insn: last);
4165	gcc_assert (BARRIER_P (last));
4166
4167	*can_fallthru = false;
4168	while (NEXT_INSN (insn: last))
4169	{
4170	/* For instance an sqrt builtin expander expands if with
4171	sibcall in the then and label for `else`. */
4172	if (LABEL_P (NEXT_INSN (last)))
4173	{
4174	*can_fallthru = true;
4175	break;
4176	}
4177	delete_insn (NEXT_INSN (insn: last));
4178	}
4179
4180	e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_ABNORMAL
4181	| EDGE_SIBCALL);
4182	e->probability = probability;
4183	BB_END (bb) = last;
4184	update_bb_for_insn (bb);
4185
4186	if (NEXT_INSN (insn: last))
4187	{
4188	bb = create_basic_block (NEXT_INSN (insn: last), get_last_insn (), bb);
4189
4190	last = BB_END (bb);
4191	if (BARRIER_P (last))
4192	BB_END (bb) = PREV_INSN (insn: last);
4193	}
4194
4195	maybe_dump_rtl_for_gimple_stmt (stmt, since: last2);
4196
4197	return bb;
4198	}
4199
4200	/* Return the difference between the floor and the truncated result of
4201	a signed division by OP1 with remainder MOD. */
4202	static rtx
4203	floor_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4204	{
4205	/* (mod != 0 ? (op1 / mod < 0 ? -1 : 0) : 0) */
4206	return gen_rtx_IF_THEN_ELSE
4207	(mode, gen_rtx_NE (BImode, mod, const0_rtx),
4208	gen_rtx_IF_THEN_ELSE
4209	(mode, gen_rtx_LT (BImode,
4210	gen_rtx_DIV (mode, op1, mod),
4211	const0_rtx),
4212	constm1_rtx, const0_rtx),
4213	const0_rtx);
4214	}
4215
4216	/* Return the difference between the ceil and the truncated result of
4217	a signed division by OP1 with remainder MOD. */
4218	static rtx
4219	ceil_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4220	{
4221	/* (mod != 0 ? (op1 / mod > 0 ? 1 : 0) : 0) */
4222	return gen_rtx_IF_THEN_ELSE
4223	(mode, gen_rtx_NE (BImode, mod, const0_rtx),
4224	gen_rtx_IF_THEN_ELSE
4225	(mode, gen_rtx_GT (BImode,
4226	gen_rtx_DIV (mode, op1, mod),
4227	const0_rtx),
4228	const1_rtx, const0_rtx),
4229	const0_rtx);
4230	}
4231
4232	/* Return the difference between the ceil and the truncated result of
4233	an unsigned division by OP1 with remainder MOD. */
4234	static rtx
4235	ceil_udiv_adjust (machine_mode mode, rtx mod, rtx op1 ATTRIBUTE_UNUSED)
4236	{
4237	/* (mod != 0 ? 1 : 0) */
4238	return gen_rtx_IF_THEN_ELSE
4239	(mode, gen_rtx_NE (BImode, mod, const0_rtx),
4240	const1_rtx, const0_rtx);
4241	}
4242
4243	/* Return the difference between the rounded and the truncated result
4244	of a signed division by OP1 with remainder MOD. Halfway cases are
4245	rounded away from zero, rather than to the nearest even number. */
4246	static rtx
4247	round_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4248	{
4249	/* (abs (mod) >= abs (op1) - abs (mod)
4250	? (op1 / mod > 0 ? 1 : -1)
4251	: 0) */
4252	return gen_rtx_IF_THEN_ELSE
4253	(mode, gen_rtx_GE (BImode, gen_rtx_ABS (mode, mod),
4254	gen_rtx_MINUS (mode,
4255	gen_rtx_ABS (mode, op1),
4256	gen_rtx_ABS (mode, mod))),
4257	gen_rtx_IF_THEN_ELSE
4258	(mode, gen_rtx_GT (BImode,
4259	gen_rtx_DIV (mode, op1, mod),
4260	const0_rtx),
4261	const1_rtx, constm1_rtx),
4262	const0_rtx);
4263	}
4264
4265	/* Return the difference between the rounded and the truncated result
4266	of a unsigned division by OP1 with remainder MOD. Halfway cases
4267	are rounded away from zero, rather than to the nearest even
4268	number. */
4269	static rtx
4270	round_udiv_adjust (machine_mode mode, rtx mod, rtx op1)
4271	{
4272	/* (mod >= op1 - mod ? 1 : 0) */
4273	return gen_rtx_IF_THEN_ELSE
4274	(mode, gen_rtx_GE (BImode, mod,
4275	gen_rtx_MINUS (mode, op1, mod)),
4276	const1_rtx, const0_rtx);
4277	}
4278
4279	/* Convert X to MODE, that must be Pmode or ptr_mode, without emitting
4280	any rtl. */
4281
4282	static rtx
4283	convert_debug_memory_address (scalar_int_mode mode, rtx x,
4284	addr_space_t as)
4285	{
4286	#ifndef POINTERS_EXTEND_UNSIGNED
4287	gcc_assert (mode == Pmode
4288	|| mode == targetm.addr_space.address_mode (as));
4289	gcc_assert (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode);
4290	#else
4291	rtx temp;
4292
4293	gcc_assert (targetm.addr_space.valid_pointer_mode (mode, as));
4294
4295	if (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode)
4296	return x;
4297
4298	/* X must have some form of address mode already. */
4299	scalar_int_mode xmode = as_a <scalar_int_mode> (GET_MODE (x));
4300	if (GET_MODE_PRECISION (mode) < GET_MODE_PRECISION (mode: xmode))
4301	x = lowpart_subreg (outermode: mode, op: x, innermode: xmode);
4302	else if (POINTERS_EXTEND_UNSIGNED > 0)
4303	x = gen_rtx_ZERO_EXTEND (mode, x);
4304	else if (!POINTERS_EXTEND_UNSIGNED)
4305	x = gen_rtx_SIGN_EXTEND (mode, x);
4306	else
4307	{
4308	switch (GET_CODE (x))
4309	{
4310	case SUBREG:
4311	if ((SUBREG_PROMOTED_VAR_P (x)
4312	|| (REG_P (SUBREG_REG (x)) && REG_POINTER (SUBREG_REG (x)))
4313	|| (GET_CODE (SUBREG_REG (x)) == PLUS
4314	&& REG_P (XEXP (SUBREG_REG (x), 0))
4315	&& REG_POINTER (XEXP (SUBREG_REG (x), 0))
4316	&& CONST_INT_P (XEXP (SUBREG_REG (x), 1))))
4317	&& GET_MODE (SUBREG_REG (x)) == mode)
4318	return SUBREG_REG (x);
4319	break;
4320	case LABEL_REF:
4321	temp = gen_rtx_LABEL_REF (mode, label_ref_label (x));
4322	LABEL_REF_NONLOCAL_P (temp) = LABEL_REF_NONLOCAL_P (x);
4323	return temp;
4324	case SYMBOL_REF:
4325	temp = shallow_copy_rtx (x);
4326	PUT_MODE (x: temp, mode);
4327	return temp;
4328	case CONST:
4329	temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4330	if (temp)
4331	temp = gen_rtx_CONST (mode, temp);
4332	return temp;
4333	case PLUS:
4334	case MINUS:
4335	if (CONST_INT_P (XEXP (x, 1)))
4336	{
4337	temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4338	if (temp)
4339	return gen_rtx_fmt_ee (GET_CODE (x), mode, temp, XEXP (x, 1));
4340	}
4341	break;
4342	default:
4343	break;
4344	}
4345	/* Don't know how to express ptr_extend as operation in debug info. */
4346	return NULL;
4347	}
4348	#endif /* POINTERS_EXTEND_UNSIGNED */
4349
4350	return x;
4351	}
4352
4353	/* Map from SSA_NAMEs to corresponding DEBUG_EXPR_DECLs created
4354	by avoid_deep_ter_for_debug. */
4355
4356	static hash_map<tree, tree> *deep_ter_debug_map;
4357
4358	/* Split too deep TER chains for debug stmts using debug temporaries. */
4359
4360	static void
4361	avoid_deep_ter_for_debug (gimple *stmt, int depth)
4362	{
4363	use_operand_p use_p;
4364	ssa_op_iter iter;
4365	FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
4366	{
4367	tree use = USE_FROM_PTR (use_p);
4368	if (TREE_CODE (use) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (use))
4369	continue;
4370	gimple *g = get_gimple_for_ssa_name (exp: use);
4371	if (g == NULL)
4372	continue;
4373	if (depth > 6 && !stmt_ends_bb_p (g))
4374	{
4375	if (deep_ter_debug_map == NULL)
4376	deep_ter_debug_map = new hash_map<tree, tree>;
4377
4378	tree &vexpr = deep_ter_debug_map->get_or_insert (k: use);
4379	if (vexpr != NULL)
4380	continue;
4381	vexpr = build_debug_expr_decl (TREE_TYPE (use));
4382	gimple *def_temp = gimple_build_debug_bind (vexpr, use, g);
4383	gimple_stmt_iterator gsi = gsi_for_stmt (g);
4384	gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
4385	avoid_deep_ter_for_debug (stmt: def_temp, depth: 0);
4386	}
4387	else
4388	avoid_deep_ter_for_debug (stmt: g, depth: depth + 1);
4389	}
4390	}
4391
4392	/* Return an RTX equivalent to the value of the parameter DECL. */
4393
4394	static rtx
4395	expand_debug_parm_decl (tree decl)
4396	{
4397	rtx incoming = DECL_INCOMING_RTL (decl);
4398
4399	if (incoming
4400	&& GET_MODE (incoming) != BLKmode
4401	&& ((REG_P (incoming) && HARD_REGISTER_P (incoming))
4402	|| (MEM_P (incoming)
4403	&& REG_P (XEXP (incoming, 0))
4404	&& HARD_REGISTER_P (XEXP (incoming, 0)))))
4405	{
4406	rtx rtl = gen_rtx_ENTRY_VALUE (GET_MODE (incoming));
4407
4408	#ifdef HAVE_window_save
4409	/* DECL_INCOMING_RTL uses the INCOMING_REGNO of parameter registers.
4410	If the target machine has an explicit window save instruction, the
4411	actual entry value is the corresponding OUTGOING_REGNO instead. */
4412	if (REG_P (incoming)
4413	&& OUTGOING_REGNO (REGNO (incoming)) != REGNO (incoming))
4414	incoming
4415	= gen_rtx_REG_offset (incoming, GET_MODE (incoming),
4416	OUTGOING_REGNO (REGNO (incoming)), 0);
4417	else if (MEM_P (incoming))
4418	{
4419	rtx reg = XEXP (incoming, 0);
4420	if (OUTGOING_REGNO (REGNO (reg)) != REGNO (reg))
4421	{
4422	reg = gen_raw_REG (GET_MODE (reg), OUTGOING_REGNO (REGNO (reg)));
4423	incoming = replace_equiv_address_nv (incoming, reg);
4424	}
4425	else
4426	incoming = copy_rtx (incoming);
4427	}
4428	#endif
4429
4430	ENTRY_VALUE_EXP (rtl) = incoming;
4431	return rtl;
4432	}
4433
4434	if (incoming
4435	&& GET_MODE (incoming) != BLKmode
4436	&& !TREE_ADDRESSABLE (decl)
4437	&& MEM_P (incoming)
4438	&& (XEXP (incoming, 0) == virtual_incoming_args_rtx
4439	|| (GET_CODE (XEXP (incoming, 0)) == PLUS
4440	&& XEXP (XEXP (incoming, 0), 0) == virtual_incoming_args_rtx
4441	&& CONST_INT_P (XEXP (XEXP (incoming, 0), 1)))))
4442	return copy_rtx (incoming);
4443
4444	return NULL_RTX;
4445	}
4446
4447	/* Return an RTX equivalent to the value of the tree expression EXP. */
4448
4449	static rtx
4450	expand_debug_expr (tree exp)
4451	{
4452	rtx op0 = NULL_RTX, op1 = NULL_RTX, op2 = NULL_RTX;
4453	machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
4454	machine_mode inner_mode = VOIDmode;
4455	int unsignedp = TYPE_UNSIGNED (TREE_TYPE (exp));
4456	addr_space_t as;
4457	scalar_int_mode op0_mode, op1_mode, addr_mode;
4458
4459	switch (TREE_CODE_CLASS (TREE_CODE (exp)))
4460	{
4461	case tcc_expression:
4462	switch (TREE_CODE (exp))
4463	{
4464	case COND_EXPR:
4465	case DOT_PROD_EXPR:
4466	case SAD_EXPR:
4467	case WIDEN_MULT_PLUS_EXPR:
4468	case WIDEN_MULT_MINUS_EXPR:
4469	goto ternary;
4470
4471	case TRUTH_ANDIF_EXPR:
4472	case TRUTH_ORIF_EXPR:
4473	case TRUTH_AND_EXPR:
4474	case TRUTH_OR_EXPR:
4475	case TRUTH_XOR_EXPR:
4476	goto binary;
4477
4478	case TRUTH_NOT_EXPR:
4479	goto unary;
4480
4481	default:
4482	break;
4483	}
4484	break;
4485
4486	ternary:
4487	op2 = expand_debug_expr (TREE_OPERAND (exp, 2));
4488	if (!op2)
4489	return NULL_RTX;
4490	/* Fall through. */
4491
4492	binary:
4493	case tcc_binary:
4494	if (mode == BLKmode)
4495	return NULL_RTX;
4496	op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
4497	if (!op1)
4498	return NULL_RTX;
4499	switch (TREE_CODE (exp))
4500	{
4501	case LSHIFT_EXPR:
4502	case RSHIFT_EXPR:
4503	case LROTATE_EXPR:
4504	case RROTATE_EXPR:
4505	case WIDEN_LSHIFT_EXPR:
4506	/* Ensure second operand isn't wider than the first one. */
4507	inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 1)));
4508	if (is_a <scalar_int_mode> (m: inner_mode, result: &op1_mode)
4509	&& (GET_MODE_UNIT_PRECISION (mode)
4510	< GET_MODE_PRECISION (mode: op1_mode)))
4511	op1 = lowpart_subreg (GET_MODE_INNER (mode), op: op1, innermode: op1_mode);
4512	break;
4513	default:
4514	break;
4515	}
4516	/* Fall through. */
4517
4518	unary:
4519	case tcc_unary:
4520	if (mode == BLKmode)
4521	return NULL_RTX;
4522	inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4523	op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
4524	if (!op0)
4525	return NULL_RTX;
4526	break;
4527
4528	case tcc_comparison:
4529	unsignedp = TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)));
4530	goto binary;
4531
4532	case tcc_type:
4533	case tcc_statement:
4534	gcc_unreachable ();
4535
4536	case tcc_constant:
4537	case tcc_exceptional:
4538	case tcc_declaration:
4539	case tcc_reference:
4540	case tcc_vl_exp:
4541	break;
4542	}
4543
4544	switch (TREE_CODE (exp))
4545	{
4546	case STRING_CST:
4547	if (!lookup_constant_def (exp))
4548	{
4549	if (strlen (TREE_STRING_POINTER (exp)) + 1
4550	!= (size_t) TREE_STRING_LENGTH (exp))
4551	return NULL_RTX;
4552	op0 = gen_rtx_CONST_STRING (Pmode, TREE_STRING_POINTER (exp));
4553	op0 = gen_rtx_MEM (BLKmode, op0);
4554	set_mem_attributes (op0, exp, 0);
4555	return op0;
4556	}
4557	/* Fall through. */
4558
4559	case INTEGER_CST:
4560	if (TREE_CODE (TREE_TYPE (exp)) == BITINT_TYPE
4561	&& TYPE_MODE (TREE_TYPE (exp)) == BLKmode)
4562	return NULL;
4563	/* FALLTHRU */
4564	case REAL_CST:
4565	case FIXED_CST:
4566	op0 = expand_expr (exp, NULL_RTX, mode, modifier: EXPAND_INITIALIZER);
4567	return op0;
4568
4569	case POLY_INT_CST:
4570	return immed_wide_int_const (poly_int_cst_value (x: exp), mode);
4571
4572	case COMPLEX_CST:
4573	gcc_assert (COMPLEX_MODE_P (mode));
4574	op0 = expand_debug_expr (TREE_REALPART (exp));
4575	op1 = expand_debug_expr (TREE_IMAGPART (exp));
4576	return gen_rtx_CONCAT (mode, op0, op1);
4577
4578	case DEBUG_EXPR_DECL:
4579	op0 = DECL_RTL_IF_SET (exp);
4580
4581	if (op0)
4582	{
4583	if (GET_MODE (op0) != mode)
4584	gcc_assert (VECTOR_TYPE_P (TREE_TYPE (exp)));
4585	else
4586	return op0;
4587	}
4588
4589	op0 = gen_rtx_DEBUG_EXPR (mode);
4590	DEBUG_EXPR_TREE_DECL (op0) = exp;
4591	SET_DECL_RTL (exp, op0);
4592
4593	return op0;
4594
4595	case VAR_DECL:
4596	case PARM_DECL:
4597	case FUNCTION_DECL:
4598	case LABEL_DECL:
4599	case CONST_DECL:
4600	case RESULT_DECL:
4601	op0 = DECL_RTL_IF_SET (exp);
4602
4603	/* This decl was probably optimized away. */
4604	if (!op0
4605	/* At least label RTXen are sometimes replaced by
4606	NOTE_INSN_DELETED_LABEL. Any notes here are not
4607	handled by copy_rtx. */
4608	|| NOTE_P (op0))
4609	{
4610	if (!VAR_P (exp)
4611	|| DECL_EXTERNAL (exp)
4612	|| !TREE_STATIC (exp)
4613	|| !DECL_NAME (exp)
4614	|| DECL_HARD_REGISTER (exp)
4615	|| DECL_IN_CONSTANT_POOL (exp)
4616	|| mode == VOIDmode
4617	|| symtab_node::get (decl: exp) == NULL)
4618	return NULL;
4619
4620	op0 = make_decl_rtl_for_debug (exp);
4621	if (!MEM_P (op0)
4622	|| GET_CODE (XEXP (op0, 0)) != SYMBOL_REF
4623	|| SYMBOL_REF_DECL (XEXP (op0, 0)) != exp)
4624	return NULL;
4625	}
4626	else if (VAR_P (exp)
4627	&& is_global_var (t: exp)
4628	&& symtab_node::get (decl: exp) == NULL)
4629	return NULL;
4630	else
4631	op0 = copy_rtx (op0);
4632
4633	if (GET_MODE (op0) == BLKmode
4634	/* If op0 is not BLKmode, but mode is, adjust_mode
4635	below would ICE. While it is likely a FE bug,
4636	try to be robust here. See PR43166. */
4637	|| mode == BLKmode
4638	|| (mode == VOIDmode && GET_MODE (op0) != VOIDmode))
4639	{
4640	gcc_assert (MEM_P (op0));
4641	op0 = adjust_address_nv (op0, mode, 0);
4642	return op0;
4643	}
4644
4645	/* Fall through. */
4646
4647	adjust_mode:
4648	case PAREN_EXPR:
4649	CASE_CONVERT:
4650	{
4651	inner_mode = GET_MODE (op0);
4652
4653	if (mode == inner_mode)
4654	return op0;
4655
4656	if (inner_mode == VOIDmode)
4657	{
4658	if (TREE_CODE (exp) == SSA_NAME)
4659	inner_mode = TYPE_MODE (TREE_TYPE (exp));
4660	else
4661	inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4662	if (mode == inner_mode)
4663	return op0;
4664	}
4665
4666	if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
4667	{
4668	if (GET_MODE_UNIT_BITSIZE (mode)
4669	== GET_MODE_UNIT_BITSIZE (inner_mode))
4670	op0 = simplify_gen_subreg (outermode: mode, op: op0, innermode: inner_mode, byte: 0);
4671	else if (GET_MODE_UNIT_BITSIZE (mode)
4672	< GET_MODE_UNIT_BITSIZE (inner_mode))
4673	op0 = simplify_gen_unary (code: FLOAT_TRUNCATE, mode, op: op0, op_mode: inner_mode);
4674	else
4675	op0 = simplify_gen_unary (code: FLOAT_EXTEND, mode, op: op0, op_mode: inner_mode);
4676	}
4677	else if (FLOAT_MODE_P (mode))
4678	{
4679	gcc_assert (TREE_CODE (exp) != SSA_NAME);
4680	if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
4681	op0 = simplify_gen_unary (code: UNSIGNED_FLOAT, mode, op: op0, op_mode: inner_mode);
4682	else
4683	op0 = simplify_gen_unary (code: FLOAT, mode, op: op0, op_mode: inner_mode);
4684	}
4685	else if (FLOAT_MODE_P (inner_mode))
4686	{
4687	if (unsignedp)
4688	op0 = simplify_gen_unary (code: UNSIGNED_FIX, mode, op: op0, op_mode: inner_mode);
4689	else
4690	op0 = simplify_gen_unary (code: FIX, mode, op: op0, op_mode: inner_mode);
4691	}
4692	else if (GET_MODE_UNIT_PRECISION (mode)
4693	== GET_MODE_UNIT_PRECISION (inner_mode))
4694	op0 = lowpart_subreg (outermode: mode, op: op0, innermode: inner_mode);
4695	else if (GET_MODE_UNIT_PRECISION (mode)
4696	< GET_MODE_UNIT_PRECISION (inner_mode))
4697	op0 = simplify_gen_unary (code: TRUNCATE, mode, op: op0, op_mode: inner_mode);
4698	else if (UNARY_CLASS_P (exp)
4699	? TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)))
4700	: unsignedp)
4701	op0 = simplify_gen_unary (code: ZERO_EXTEND, mode, op: op0, op_mode: inner_mode);
4702	else
4703	op0 = simplify_gen_unary (code: SIGN_EXTEND, mode, op: op0, op_mode: inner_mode);
4704
4705	return op0;
4706	}
4707
4708	case MEM_REF:
4709	if (!is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
4710	{
4711	tree newexp = fold_binary (MEM_REF, TREE_TYPE (exp),
4712	TREE_OPERAND (exp, 0),
4713	TREE_OPERAND (exp, 1));
4714	if (newexp)
4715	return expand_debug_expr (exp: newexp);
4716	}
4717	/* FALLTHROUGH */
4718	case INDIRECT_REF:
4719	inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4720	op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
4721	if (!op0)
4722	return NULL;
4723
4724	if (TREE_CODE (exp) == MEM_REF)
4725	{
4726	if (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
4727	|| (GET_CODE (op0) == PLUS
4728	&& GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR))
4729	/* (mem (debug_implicit_ptr)) might confuse aliasing.
4730	Instead just use get_inner_reference. */
4731	goto component_ref;
4732
4733	op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
4734	poly_int64 offset;
4735	if (!op1 || !poly_int_rtx_p (x: op1, res: &offset))
4736	return NULL;
4737
4738	op0 = plus_constant (inner_mode, op0, offset);
4739	}
4740
4741	as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
4742
4743	op0 = convert_debug_memory_address (mode: targetm.addr_space.address_mode (as),
4744	x: op0, as);
4745	if (op0 == NULL_RTX)
4746	return NULL;
4747
4748	op0 = gen_rtx_MEM (mode, op0);
4749	set_mem_attributes (op0, exp, 0);
4750	if (TREE_CODE (exp) == MEM_REF
4751	&& !is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
4752	set_mem_expr (op0, NULL_TREE);
4753	set_mem_addr_space (op0, as);
4754
4755	return op0;
4756
4757	case TARGET_MEM_REF:
4758	if (TREE_CODE (TMR_BASE (exp)) == ADDR_EXPR
4759	&& !DECL_RTL_SET_P (TREE_OPERAND (TMR_BASE (exp), 0)))
4760	return NULL;
4761
4762	op0 = expand_debug_expr
4763	(exp: tree_mem_ref_addr (build_pointer_type (TREE_TYPE (exp)), exp));
4764	if (!op0)
4765	return NULL;
4766
4767	as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
4768	op0 = convert_debug_memory_address (mode: targetm.addr_space.address_mode (as),
4769	x: op0, as);
4770	if (op0 == NULL_RTX)
4771	return NULL;
4772
4773	op0 = gen_rtx_MEM (mode, op0);
4774
4775	set_mem_attributes (op0, exp, 0);
4776	set_mem_addr_space (op0, as);
4777
4778	return op0;
4779
4780	component_ref:
4781	case ARRAY_REF:
4782	case ARRAY_RANGE_REF:
4783	case COMPONENT_REF:
4784	case BIT_FIELD_REF:
4785	case REALPART_EXPR:
4786	case IMAGPART_EXPR:
4787	case VIEW_CONVERT_EXPR:
4788	{
4789	machine_mode mode1;
4790	poly_int64 bitsize, bitpos;
4791	tree offset;
4792	int reversep, volatilep = 0;
4793	tree tem
4794	= get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
4795	&unsignedp, &reversep, &volatilep);
4796	rtx orig_op0;
4797
4798	if (known_eq (bitsize, 0))
4799	return NULL;
4800
4801	orig_op0 = op0 = expand_debug_expr (exp: tem);
4802
4803	if (!op0)
4804	return NULL;
4805
4806	if (offset)
4807	{
4808	machine_mode addrmode, offmode;
4809
4810	if (!MEM_P (op0))
4811	return NULL;
4812
4813	op0 = XEXP (op0, 0);
4814	addrmode = GET_MODE (op0);
4815	if (addrmode == VOIDmode)
4816	addrmode = Pmode;
4817
4818	op1 = expand_debug_expr (exp: offset);
4819	if (!op1)
4820	return NULL;
4821
4822	offmode = GET_MODE (op1);
4823	if (offmode == VOIDmode)
4824	offmode = TYPE_MODE (TREE_TYPE (offset));
4825
4826	if (addrmode != offmode)
4827	op1 = lowpart_subreg (outermode: addrmode, op: op1, innermode: offmode);
4828
4829	/* Don't use offset_address here, we don't need a
4830	recognizable address, and we don't want to generate
4831	code. */
4832	op0 = gen_rtx_MEM (mode, simplify_gen_binary (code: PLUS, mode: addrmode,
4833	op0, op1));
4834	}
4835
4836	if (MEM_P (op0))
4837	{
4838	if (mode1 == VOIDmode)
4839	{
4840	if (maybe_gt (bitsize, MAX_BITSIZE_MODE_ANY_INT))
4841	return NULL;
4842	/* Bitfield. */
4843	mode1 = smallest_int_mode_for_size (size: bitsize);
4844	}
4845	poly_int64 bytepos = bits_to_bytes_round_down (bitpos);
4846	if (maybe_ne (a: bytepos, b: 0))
4847	{
4848	op0 = adjust_address_nv (op0, mode1, bytepos);
4849	bitpos = num_trailing_bits (bitpos);
4850	}
4851	else if (known_eq (bitpos, 0)
4852	&& known_eq (bitsize, GET_MODE_BITSIZE (mode)))
4853	op0 = adjust_address_nv (op0, mode, 0);
4854	else if (GET_MODE (op0) != mode1)
4855	op0 = adjust_address_nv (op0, mode1, 0);
4856	else
4857	op0 = copy_rtx (op0);
4858	if (op0 == orig_op0)
4859	op0 = shallow_copy_rtx (op0);
4860	if (TREE_CODE (tem) != SSA_NAME)
4861	set_mem_attributes (op0, exp, 0);
4862	}
4863
4864	if (known_eq (bitpos, 0) && mode == GET_MODE (op0))
4865	return op0;
4866
4867	if (maybe_lt (a: bitpos, b: 0))
4868	return NULL;
4869
4870	if (GET_MODE (op0) == BLKmode || mode == BLKmode)
4871	return NULL;
4872
4873	poly_int64 bytepos;
4874	if (multiple_p (a: bitpos, BITS_PER_UNIT, multiple: &bytepos)
4875	&& known_eq (bitsize, GET_MODE_BITSIZE (mode1)))
4876	{
4877	machine_mode opmode = GET_MODE (op0);
4878
4879	if (opmode == VOIDmode)
4880	opmode = TYPE_MODE (TREE_TYPE (tem));
4881
4882	/* This condition may hold if we're expanding the address
4883	right past the end of an array that turned out not to
4884	be addressable (i.e., the address was only computed in
4885	debug stmts). The gen_subreg below would rightfully
4886	crash, and the address doesn't really exist, so just
4887	drop it. */
4888	if (known_ge (bitpos, GET_MODE_BITSIZE (opmode)))
4889	return NULL;
4890
4891	if (multiple_p (a: bitpos, b: GET_MODE_BITSIZE (mode)))
4892	return simplify_gen_subreg (outermode: mode, op: op0, innermode: opmode, byte: bytepos);
4893	}
4894
4895	return simplify_gen_ternary (SCALAR_INT_MODE_P (GET_MODE (op0))
4896	&& TYPE_UNSIGNED (TREE_TYPE (exp))
4897	? SIGN_EXTRACT
4898	: ZERO_EXTRACT, mode,
4899	GET_MODE (op0) != VOIDmode
4900	? GET_MODE (op0)
4901	: TYPE_MODE (TREE_TYPE (tem)),
4902	op0, op1: gen_int_mode (bitsize, word_mode),
4903	op2: gen_int_mode (bitpos, word_mode));
4904	}
4905
4906	case ABS_EXPR:
4907	case ABSU_EXPR:
4908	return simplify_gen_unary (code: ABS, mode, op: op0, op_mode: mode);
4909
4910	case NEGATE_EXPR:
4911	return simplify_gen_unary (code: NEG, mode, op: op0, op_mode: mode);
4912
4913	case BIT_NOT_EXPR:
4914	return simplify_gen_unary (code: NOT, mode, op: op0, op_mode: mode);
4915
4916	case FLOAT_EXPR:
4917	return simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
4918	0)))
4919	? UNSIGNED_FLOAT : FLOAT, mode, op: op0,
4920	op_mode: inner_mode);
4921
4922	case FIX_TRUNC_EXPR:
4923	return simplify_gen_unary (code: unsignedp ? UNSIGNED_FIX : FIX, mode, op: op0,
4924	op_mode: inner_mode);
4925
4926	case POINTER_PLUS_EXPR:
4927	/* For the rare target where pointers are not the same size as
4928	size_t, we need to check for mis-matched modes and correct
4929	the addend. */
4930	if (op0 && op1
4931	&& is_a <scalar_int_mode> (GET_MODE (op0), result: &op0_mode)
4932	&& is_a <scalar_int_mode> (GET_MODE (op1), result: &op1_mode)
4933	&& op0_mode != op1_mode)
4934	{
4935	if (GET_MODE_BITSIZE (mode: op0_mode) < GET_MODE_BITSIZE (mode: op1_mode)
4936	/* If OP0 is a partial mode, then we must truncate, even
4937	if it has the same bitsize as OP1 as GCC's
4938	representation of partial modes is opaque. */
4939	|| (GET_MODE_CLASS (op0_mode) == MODE_PARTIAL_INT
4940	&& (GET_MODE_BITSIZE (mode: op0_mode)
4941	== GET_MODE_BITSIZE (mode: op1_mode))))
4942	op1 = simplify_gen_unary (code: TRUNCATE, mode: op0_mode, op: op1, op_mode: op1_mode);
4943	else
4944	/* We always sign-extend, regardless of the signedness of
4945	the operand, because the operand is always unsigned
4946	here even if the original C expression is signed. */
4947	op1 = simplify_gen_unary (code: SIGN_EXTEND, mode: op0_mode, op: op1, op_mode: op1_mode);
4948	}
4949	/* Fall through. */
4950	case PLUS_EXPR:
4951	return simplify_gen_binary (code: PLUS, mode, op0, op1);
4952
4953	case MINUS_EXPR:
4954	case POINTER_DIFF_EXPR:
4955	return simplify_gen_binary (code: MINUS, mode, op0, op1);
4956
4957	case MULT_EXPR:
4958	return simplify_gen_binary (code: MULT, mode, op0, op1);
4959
4960	case RDIV_EXPR:
4961	case TRUNC_DIV_EXPR:
4962	case EXACT_DIV_EXPR:
4963	if (unsignedp)
4964	return simplify_gen_binary (code: UDIV, mode, op0, op1);
4965	else
4966	return simplify_gen_binary (code: DIV, mode, op0, op1);
4967
4968	case TRUNC_MOD_EXPR:
4969	return simplify_gen_binary (code: unsignedp ? UMOD : MOD, mode, op0, op1);
4970
4971	case FLOOR_DIV_EXPR:
4972	if (unsignedp)
4973	return simplify_gen_binary (code: UDIV, mode, op0, op1);
4974	else
4975	{
4976	rtx div = simplify_gen_binary (code: DIV, mode, op0, op1);
4977	rtx mod = simplify_gen_binary (code: MOD, mode, op0, op1);
4978	rtx adj = floor_sdiv_adjust (mode, mod, op1);
4979	return simplify_gen_binary (code: PLUS, mode, op0: div, op1: adj);
4980	}
4981
4982	case FLOOR_MOD_EXPR:
4983	if (unsignedp)
4984	return simplify_gen_binary (code: UMOD, mode, op0, op1);
4985	else
4986	{
4987	rtx mod = simplify_gen_binary (code: MOD, mode, op0, op1);
4988	rtx adj = floor_sdiv_adjust (mode, mod, op1);
4989	adj = simplify_gen_unary (code: NEG, mode,
4990	op: simplify_gen_binary (code: MULT, mode, op0: adj, op1),
4991	op_mode: mode);
4992	return simplify_gen_binary (code: PLUS, mode, op0: mod, op1: adj);
4993	}
4994
4995	case CEIL_DIV_EXPR:
4996	if (unsignedp)
4997	{
4998	rtx div = simplify_gen_binary (code: UDIV, mode, op0, op1);
4999	rtx mod = simplify_gen_binary (code: UMOD, mode, op0, op1);
5000	rtx adj = ceil_udiv_adjust (mode, mod, op1);
5001	return simplify_gen_binary (code: PLUS, mode, op0: div, op1: adj);
5002	}
5003	else
5004	{
5005	rtx div = simplify_gen_binary (code: DIV, mode, op0, op1);
5006	rtx mod = simplify_gen_binary (code: MOD, mode, op0, op1);
5007	rtx adj = ceil_sdiv_adjust (mode, mod, op1);
5008	return simplify_gen_binary (code: PLUS, mode, op0: div, op1: adj);
5009	}
5010
5011	case CEIL_MOD_EXPR:
5012	if (unsignedp)
5013	{
5014	rtx mod = simplify_gen_binary (code: UMOD, mode, op0, op1);
5015	rtx adj = ceil_udiv_adjust (mode, mod, op1);
5016	adj = simplify_gen_unary (code: NEG, mode,
5017	op: simplify_gen_binary (code: MULT, mode, op0: adj, op1),
5018	op_mode: mode);
5019	return simplify_gen_binary (code: PLUS, mode, op0: mod, op1: adj);
5020	}
5021	else
5022	{
5023	rtx mod = simplify_gen_binary (code: MOD, mode, op0, op1);
5024	rtx adj = ceil_sdiv_adjust (mode, mod, op1);
5025	adj = simplify_gen_unary (code: NEG, mode,
5026	op: simplify_gen_binary (code: MULT, mode, op0: adj, op1),
5027	op_mode: mode);
5028	return simplify_gen_binary (code: PLUS, mode, op0: mod, op1: adj);
5029	}
5030
5031	case ROUND_DIV_EXPR:
5032	if (unsignedp)
5033	{
5034	rtx div = simplify_gen_binary (code: UDIV, mode, op0, op1);
5035	rtx mod = simplify_gen_binary (code: UMOD, mode, op0, op1);
5036	rtx adj = round_udiv_adjust (mode, mod, op1);
5037	return simplify_gen_binary (code: PLUS, mode, op0: div, op1: adj);
5038	}
5039	else
5040	{
5041	rtx div = simplify_gen_binary (code: DIV, mode, op0, op1);
5042	rtx mod = simplify_gen_binary (code: MOD, mode, op0, op1);
5043	rtx adj = round_sdiv_adjust (mode, mod, op1);
5044	return simplify_gen_binary (code: PLUS, mode, op0: div, op1: adj);
5045	}
5046
5047	case ROUND_MOD_EXPR:
5048	if (unsignedp)
5049	{
5050	rtx mod = simplify_gen_binary (code: UMOD, mode, op0, op1);
5051	rtx adj = round_udiv_adjust (mode, mod, op1);
5052	adj = simplify_gen_unary (code: NEG, mode,
5053	op: simplify_gen_binary (code: MULT, mode, op0: adj, op1),
5054	op_mode: mode);
5055	return simplify_gen_binary (code: PLUS, mode, op0: mod, op1: adj);
5056	}
5057	else
5058	{
5059	rtx mod = simplify_gen_binary (code: MOD, mode, op0, op1);
5060	rtx adj = round_sdiv_adjust (mode, mod, op1);
5061	adj = simplify_gen_unary (code: NEG, mode,
5062	op: simplify_gen_binary (code: MULT, mode, op0: adj, op1),
5063	op_mode: mode);
5064	return simplify_gen_binary (code: PLUS, mode, op0: mod, op1: adj);
5065	}
5066
5067	case LSHIFT_EXPR:
5068	return simplify_gen_binary (code: ASHIFT, mode, op0, op1);
5069
5070	case RSHIFT_EXPR:
5071	if (unsignedp)
5072	return simplify_gen_binary (code: LSHIFTRT, mode, op0, op1);
5073	else
5074	return simplify_gen_binary (code: ASHIFTRT, mode, op0, op1);
5075
5076	case LROTATE_EXPR:
5077	return simplify_gen_binary (code: ROTATE, mode, op0, op1);
5078
5079	case RROTATE_EXPR:
5080	return simplify_gen_binary (code: ROTATERT, mode, op0, op1);
5081
5082	case MIN_EXPR:
5083	return simplify_gen_binary (code: unsignedp ? UMIN : SMIN, mode, op0, op1);
5084
5085	case MAX_EXPR:
5086	return simplify_gen_binary (code: unsignedp ? UMAX : SMAX, mode, op0, op1);
5087
5088	case BIT_AND_EXPR:
5089	case TRUTH_AND_EXPR:
5090	return simplify_gen_binary (code: AND, mode, op0, op1);
5091
5092	case BIT_IOR_EXPR:
5093	case TRUTH_OR_EXPR:
5094	return simplify_gen_binary (code: IOR, mode, op0, op1);
5095
5096	case BIT_XOR_EXPR:
5097	case TRUTH_XOR_EXPR:
5098	return simplify_gen_binary (code: XOR, mode, op0, op1);
5099
5100	case TRUTH_ANDIF_EXPR:
5101	return gen_rtx_IF_THEN_ELSE (mode, op0, op1, const0_rtx);
5102
5103	case TRUTH_ORIF_EXPR:
5104	return gen_rtx_IF_THEN_ELSE (mode, op0, const_true_rtx, op1);
5105
5106	case TRUTH_NOT_EXPR:
5107	return simplify_gen_relational (code: EQ, mode, op_mode: inner_mode, op0, const0_rtx);
5108
5109	case LT_EXPR:
5110	return simplify_gen_relational (code: unsignedp ? LTU : LT, mode, op_mode: inner_mode,
5111	op0, op1);
5112
5113	case LE_EXPR:
5114	return simplify_gen_relational (code: unsignedp ? LEU : LE, mode, op_mode: inner_mode,
5115	op0, op1);
5116
5117	case GT_EXPR:
5118	return simplify_gen_relational (code: unsignedp ? GTU : GT, mode, op_mode: inner_mode,
5119	op0, op1);
5120
5121	case GE_EXPR:
5122	return simplify_gen_relational (code: unsignedp ? GEU : GE, mode, op_mode: inner_mode,
5123	op0, op1);
5124
5125	case EQ_EXPR:
5126	return simplify_gen_relational (code: EQ, mode, op_mode: inner_mode, op0, op1);
5127
5128	case NE_EXPR:
5129	return simplify_gen_relational (code: NE, mode, op_mode: inner_mode, op0, op1);
5130
5131	case UNORDERED_EXPR:
5132	return simplify_gen_relational (code: UNORDERED, mode, op_mode: inner_mode, op0, op1);
5133
5134	case ORDERED_EXPR:
5135	return simplify_gen_relational (code: ORDERED, mode, op_mode: inner_mode, op0, op1);
5136
5137	case UNLT_EXPR:
5138	return simplify_gen_relational (code: UNLT, mode, op_mode: inner_mode, op0, op1);
5139
5140	case UNLE_EXPR:
5141	return simplify_gen_relational (code: UNLE, mode, op_mode: inner_mode, op0, op1);
5142
5143	case UNGT_EXPR:
5144	return simplify_gen_relational (code: UNGT, mode, op_mode: inner_mode, op0, op1);
5145
5146	case UNGE_EXPR:
5147	return simplify_gen_relational (code: UNGE, mode, op_mode: inner_mode, op0, op1);
5148
5149	case UNEQ_EXPR:
5150	return simplify_gen_relational (code: UNEQ, mode, op_mode: inner_mode, op0, op1);
5151
5152	case LTGT_EXPR:
5153	return simplify_gen_relational (code: LTGT, mode, op_mode: inner_mode, op0, op1);
5154
5155	case COND_EXPR:
5156	return gen_rtx_IF_THEN_ELSE (mode, op0, op1, op2);
5157
5158	case COMPLEX_EXPR:
5159	gcc_assert (COMPLEX_MODE_P (mode));
5160	if (GET_MODE (op0) == VOIDmode)
5161	op0 = gen_rtx_CONST (GET_MODE_INNER (mode), op0);
5162	if (GET_MODE (op1) == VOIDmode)
5163	op1 = gen_rtx_CONST (GET_MODE_INNER (mode), op1);
5164	return gen_rtx_CONCAT (mode, op0, op1);
5165
5166	case CONJ_EXPR:
5167	if (GET_CODE (op0) == CONCAT)
5168	return gen_rtx_CONCAT (mode, XEXP (op0, 0),
5169	simplify_gen_unary (NEG, GET_MODE_INNER (mode),
5170	XEXP (op0, 1),
5171	GET_MODE_INNER (mode)));
5172	else
5173	{
5174	scalar_mode imode = GET_MODE_INNER (mode);
5175	rtx re, im;
5176
5177	if (MEM_P (op0))
5178	{
5179	re = adjust_address_nv (op0, imode, 0);
5180	im = adjust_address_nv (op0, imode, GET_MODE_SIZE (imode));
5181	}
5182	else
5183	{
5184	scalar_int_mode ifmode;
5185	scalar_int_mode ihmode;
5186	rtx halfsize;
5187	if (!int_mode_for_mode (mode).exists (mode: &ifmode)
5188	|| !int_mode_for_mode (imode).exists (mode: &ihmode))
5189	return NULL;
5190	halfsize = GEN_INT (GET_MODE_BITSIZE (ihmode));
5191	re = op0;
5192	if (mode != ifmode)
5193	re = gen_rtx_SUBREG (ifmode, re, 0);
5194	re = gen_rtx_ZERO_EXTRACT (ihmode, re, halfsize, const0_rtx);
5195	if (imode != ihmode)
5196	re = gen_rtx_SUBREG (imode, re, 0);
5197	im = copy_rtx (op0);
5198	if (mode != ifmode)
5199	im = gen_rtx_SUBREG (ifmode, im, 0);
5200	im = gen_rtx_ZERO_EXTRACT (ihmode, im, halfsize, halfsize);
5201	if (imode != ihmode)
5202	im = gen_rtx_SUBREG (imode, im, 0);
5203	}
5204	im = gen_rtx_NEG (imode, im);
5205	return gen_rtx_CONCAT (mode, re, im);
5206	}
5207
5208	case ADDR_EXPR:
5209	op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
5210	if (!op0 || !MEM_P (op0))
5211	{
5212	if ((TREE_CODE (TREE_OPERAND (exp, 0)) == VAR_DECL
5213	|| TREE_CODE (TREE_OPERAND (exp, 0)) == PARM_DECL
5214	|| TREE_CODE (TREE_OPERAND (exp, 0)) == RESULT_DECL)
5215	&& (!TREE_ADDRESSABLE (TREE_OPERAND (exp, 0))
5216	|| target_for_debug_bind (TREE_OPERAND (exp, 0))))
5217	return gen_rtx_DEBUG_IMPLICIT_PTR (mode, TREE_OPERAND (exp, 0));
5218
5219	if (handled_component_p (TREE_OPERAND (exp, 0)))
5220	{
5221	poly_int64 bitoffset, bitsize, maxsize, byteoffset;
5222	bool reverse;
5223	tree decl
5224	= get_ref_base_and_extent (TREE_OPERAND (exp, 0), &bitoffset,
5225	&bitsize, &maxsize, &reverse);
5226	if ((VAR_P (decl)
5227	|| TREE_CODE (decl) == PARM_DECL
5228	|| TREE_CODE (decl) == RESULT_DECL)
5229	&& (!TREE_ADDRESSABLE (decl)
5230	|| target_for_debug_bind (decl))
5231	&& multiple_p (a: bitoffset, BITS_PER_UNIT, multiple: &byteoffset)
5232	&& known_gt (bitsize, 0)
5233	&& known_eq (bitsize, maxsize))
5234	{
5235	rtx base = gen_rtx_DEBUG_IMPLICIT_PTR (mode, decl);
5236	return plus_constant (mode, base, byteoffset);
5237	}
5238	}
5239
5240	if (TREE_CODE (TREE_OPERAND (exp, 0)) == MEM_REF
5241	&& TREE_CODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
5242	== ADDR_EXPR)
5243	{
5244	op0 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5245	0));
5246	if (op0 != NULL
5247	&& (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
5248	|| (GET_CODE (op0) == PLUS
5249	&& GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR
5250	&& CONST_INT_P (XEXP (op0, 1)))))
5251	{
5252	op1 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5253	1));
5254	poly_int64 offset;
5255	if (!op1 || !poly_int_rtx_p (x: op1, res: &offset))
5256	return NULL;
5257
5258	return plus_constant (mode, op0, offset);
5259	}
5260	}
5261
5262	return NULL;
5263	}
5264
5265	as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)));
5266	addr_mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (exp));
5267	op0 = convert_debug_memory_address (mode: addr_mode, XEXP (op0, 0), as);
5268
5269	return op0;
5270
5271	case VECTOR_CST:
5272	{
5273	unsigned HOST_WIDE_INT i, nelts;
5274
5275	if (!VECTOR_CST_NELTS (exp).is_constant (const_value: &nelts))
5276	return NULL;
5277
5278	op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5279
5280	for (i = 0; i < nelts; ++i)
5281	{
5282	op1 = expand_debug_expr (VECTOR_CST_ELT (exp, i));
5283	if (!op1)
5284	return NULL;
5285	XVECEXP (op0, 0, i) = op1;
5286	}
5287
5288	return op0;
5289	}
5290
5291	case CONSTRUCTOR:
5292	if (TREE_CLOBBER_P (exp))
5293	return NULL;
5294	else if (TREE_CODE (TREE_TYPE (exp)) == VECTOR_TYPE)
5295	{
5296	unsigned i;
5297	unsigned HOST_WIDE_INT nelts;
5298	tree val;
5299
5300	if (!TYPE_VECTOR_SUBPARTS (TREE_TYPE (exp)).is_constant (const_value: &nelts))
5301	goto flag_unsupported;
5302
5303	op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5304
5305	FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), i, val)
5306	{
5307	op1 = expand_debug_expr (exp: val);
5308	if (!op1)
5309	return NULL;
5310	XVECEXP (op0, 0, i) = op1;
5311	}
5312
5313	if (i < nelts)
5314	{
5315	op1 = expand_debug_expr
5316	(exp: build_zero_cst (TREE_TYPE (TREE_TYPE (exp))));
5317
5318	if (!op1)
5319	return NULL;
5320
5321	for (; i < nelts; i++)
5322	XVECEXP (op0, 0, i) = op1;
5323	}
5324
5325	return op0;
5326	}
5327	else
5328	goto flag_unsupported;
5329
5330	case CALL_EXPR:
5331	/* ??? Maybe handle some builtins? */
5332	return NULL;
5333
5334	case SSA_NAME:
5335	{
5336	gimple *g = get_gimple_for_ssa_name (exp);
5337	if (g)
5338	{
5339	tree t = NULL_TREE;
5340	if (deep_ter_debug_map)
5341	{
5342	tree *slot = deep_ter_debug_map->get (k: exp);
5343	if (slot)
5344	t = *slot;
5345	}
5346	if (t == NULL_TREE)
5347	t = gimple_assign_rhs_to_tree (stmt: g);
5348	op0 = expand_debug_expr (exp: t);
5349	if (!op0)
5350	return NULL;
5351	}
5352	else
5353	{
5354	/* If this is a reference to an incoming value of
5355	parameter that is never used in the code or where the
5356	incoming value is never used in the code, use
5357	PARM_DECL's DECL_RTL if set. */
5358	if (SSA_NAME_IS_DEFAULT_DEF (exp)
5359	&& SSA_NAME_VAR (exp)
5360	&& TREE_CODE (SSA_NAME_VAR (exp)) == PARM_DECL
5361	&& has_zero_uses (var: exp))
5362	{
5363	op0 = expand_debug_parm_decl (SSA_NAME_VAR (exp));
5364	if (op0)
5365	goto adjust_mode;
5366	op0 = expand_debug_expr (SSA_NAME_VAR (exp));
5367	if (op0)
5368	goto adjust_mode;
5369	}
5370
5371	int part = var_to_partition (map: SA.map, var: exp);
5372
5373	if (part == NO_PARTITION)
5374	return NULL;
5375
5376	gcc_assert (part >= 0 && (unsigned)part < SA.map->num_partitions);
5377
5378	op0 = copy_rtx (SA.partition_to_pseudo[part]);
5379	}
5380	goto adjust_mode;
5381	}
5382
5383	case ERROR_MARK:
5384	return NULL;
5385
5386	/* Vector stuff. For most of the codes we don't have rtl codes. */
5387	case REALIGN_LOAD_EXPR:
5388	case VEC_COND_EXPR:
5389	case VEC_PACK_FIX_TRUNC_EXPR:
5390	case VEC_PACK_FLOAT_EXPR:
5391	case VEC_PACK_SAT_EXPR:
5392	case VEC_PACK_TRUNC_EXPR:
5393	case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
5394	case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
5395	case VEC_UNPACK_FLOAT_HI_EXPR:
5396	case VEC_UNPACK_FLOAT_LO_EXPR:
5397	case VEC_UNPACK_HI_EXPR:
5398	case VEC_UNPACK_LO_EXPR:
5399	case VEC_WIDEN_MULT_HI_EXPR:
5400	case VEC_WIDEN_MULT_LO_EXPR:
5401	case VEC_WIDEN_MULT_EVEN_EXPR:
5402	case VEC_WIDEN_MULT_ODD_EXPR:
5403	case VEC_WIDEN_LSHIFT_HI_EXPR:
5404	case VEC_WIDEN_LSHIFT_LO_EXPR:
5405	case VEC_PERM_EXPR:
5406	case VEC_DUPLICATE_EXPR:
5407	case VEC_SERIES_EXPR:
5408	case SAD_EXPR:
5409	return NULL;
5410
5411	/* Misc codes. */
5412	case ADDR_SPACE_CONVERT_EXPR:
5413	case FIXED_CONVERT_EXPR:
5414	case OBJ_TYPE_REF:
5415	case WITH_SIZE_EXPR:
5416	case BIT_INSERT_EXPR:
5417	return NULL;
5418
5419	case DOT_PROD_EXPR:
5420	if (SCALAR_INT_MODE_P (GET_MODE (op0))
5421	&& SCALAR_INT_MODE_P (mode))
5422	{
5423	op0
5424	= simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5425	0)))
5426	? ZERO_EXTEND : SIGN_EXTEND, mode, op: op0,
5427	op_mode: inner_mode);
5428	op1
5429	= simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5430	1)))
5431	? ZERO_EXTEND : SIGN_EXTEND, mode, op: op1,
5432	op_mode: inner_mode);
5433	op0 = simplify_gen_binary (code: MULT, mode, op0, op1);
5434	return simplify_gen_binary (code: PLUS, mode, op0, op1: op2);
5435	}
5436	return NULL;
5437
5438	case WIDEN_MULT_EXPR:
5439	case WIDEN_MULT_PLUS_EXPR:
5440	case WIDEN_MULT_MINUS_EXPR:
5441	if (SCALAR_INT_MODE_P (GET_MODE (op0))
5442	&& SCALAR_INT_MODE_P (mode))
5443	{
5444	inner_mode = GET_MODE (op0);
5445	if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
5446	op0 = simplify_gen_unary (code: ZERO_EXTEND, mode, op: op0, op_mode: inner_mode);
5447	else
5448	op0 = simplify_gen_unary (code: SIGN_EXTEND, mode, op: op0, op_mode: inner_mode);
5449	if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 1))))
5450	op1 = simplify_gen_unary (code: ZERO_EXTEND, mode, op: op1, op_mode: inner_mode);
5451	else
5452	op1 = simplify_gen_unary (code: SIGN_EXTEND, mode, op: op1, op_mode: inner_mode);
5453	op0 = simplify_gen_binary (code: MULT, mode, op0, op1);
5454	if (TREE_CODE (exp) == WIDEN_MULT_EXPR)
5455	return op0;
5456	else if (TREE_CODE (exp) == WIDEN_MULT_PLUS_EXPR)
5457	return simplify_gen_binary (code: PLUS, mode, op0, op1: op2);
5458	else
5459	return simplify_gen_binary (code: MINUS, mode, op0: op2, op1: op0);
5460	}
5461	return NULL;
5462
5463	case MULT_HIGHPART_EXPR:
5464	/* ??? Similar to the above. */
5465	return NULL;
5466
5467	case WIDEN_SUM_EXPR:
5468	case WIDEN_LSHIFT_EXPR:
5469	if (SCALAR_INT_MODE_P (GET_MODE (op0))
5470	&& SCALAR_INT_MODE_P (mode))
5471	{
5472	op0
5473	= simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5474	0)))
5475	? ZERO_EXTEND : SIGN_EXTEND, mode, op: op0,
5476	op_mode: inner_mode);
5477	return simplify_gen_binary (TREE_CODE (exp) == WIDEN_LSHIFT_EXPR
5478	? ASHIFT : PLUS, mode, op0, op1);
5479	}
5480	return NULL;
5481
5482	default:
5483	flag_unsupported:
5484	if (flag_checking)
5485	{
5486	debug_tree (exp);
5487	gcc_unreachable ();
5488	}
5489	return NULL;
5490	}
5491	}
5492
5493	/* Return an RTX equivalent to the source bind value of the tree expression
5494	EXP. */
5495
5496	static rtx
5497	expand_debug_source_expr (tree exp)
5498	{
5499	rtx op0 = NULL_RTX;
5500	machine_mode mode = VOIDmode, inner_mode;
5501
5502	switch (TREE_CODE (exp))
5503	{
5504	case VAR_DECL:
5505	if (DECL_ABSTRACT_ORIGIN (exp))
5506	return expand_debug_source_expr (DECL_ABSTRACT_ORIGIN (exp));
5507	break;
5508	case PARM_DECL:
5509	{
5510	mode = DECL_MODE (exp);
5511	op0 = expand_debug_parm_decl (decl: exp);
5512	if (op0)
5513	break;
5514	/* See if this isn't an argument that has been completely
5515	optimized out. */
5516	if (!DECL_RTL_SET_P (exp)
5517	&& !DECL_INCOMING_RTL (exp)
5518	&& DECL_ABSTRACT_ORIGIN (current_function_decl))
5519	{
5520	tree aexp = DECL_ORIGIN (exp);
5521	if (DECL_CONTEXT (aexp)
5522	== DECL_ABSTRACT_ORIGIN (current_function_decl))
5523	{
5524	vec<tree, va_gc> **debug_args;
5525	unsigned int ix;
5526	tree ddecl;
5527	debug_args = decl_debug_args_lookup (current_function_decl);
5528	if (debug_args != NULL)
5529	{
5530	for (ix = 0; vec_safe_iterate (v: *debug_args, ix, ptr: &ddecl);
5531	ix += 2)
5532	if (ddecl == aexp)
5533	return gen_rtx_DEBUG_PARAMETER_REF (mode, aexp);
5534	}
5535	}
5536	}
5537	break;
5538	}
5539	default:
5540	break;
5541	}
5542
5543	if (op0 == NULL_RTX)
5544	return NULL_RTX;
5545
5546	inner_mode = GET_MODE (op0);
5547	if (mode == inner_mode)
5548	return op0;
5549
5550	if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
5551	{
5552	if (GET_MODE_UNIT_BITSIZE (mode)
5553	== GET_MODE_UNIT_BITSIZE (inner_mode))
5554	op0 = simplify_gen_subreg (outermode: mode, op: op0, innermode: inner_mode, byte: 0);
5555	else if (GET_MODE_UNIT_BITSIZE (mode)
5556	< GET_MODE_UNIT_BITSIZE (inner_mode))
5557	op0 = simplify_gen_unary (code: FLOAT_TRUNCATE, mode, op: op0, op_mode: inner_mode);
5558	else
5559	op0 = simplify_gen_unary (code: FLOAT_EXTEND, mode, op: op0, op_mode: inner_mode);
5560	}
5561	else if (FLOAT_MODE_P (mode))
5562	gcc_unreachable ();
5563	else if (FLOAT_MODE_P (inner_mode))
5564	{
5565	if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5566	op0 = simplify_gen_unary (code: UNSIGNED_FIX, mode, op: op0, op_mode: inner_mode);
5567	else
5568	op0 = simplify_gen_unary (code: FIX, mode, op: op0, op_mode: inner_mode);
5569	}
5570	else if (GET_MODE_UNIT_PRECISION (mode)
5571	== GET_MODE_UNIT_PRECISION (inner_mode))
5572	op0 = lowpart_subreg (outermode: mode, op: op0, innermode: inner_mode);
5573	else if (GET_MODE_UNIT_PRECISION (mode)
5574	< GET_MODE_UNIT_PRECISION (inner_mode))
5575	op0 = simplify_gen_unary (code: TRUNCATE, mode, op: op0, op_mode: inner_mode);
5576	else if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5577	op0 = simplify_gen_unary (code: ZERO_EXTEND, mode, op: op0, op_mode: inner_mode);
5578	else
5579	op0 = simplify_gen_unary (code: SIGN_EXTEND, mode, op: op0, op_mode: inner_mode);
5580
5581	return op0;
5582	}
5583
5584	/* Ensure INSN_VAR_LOCATION_LOC (insn) doesn't have unbound complexity.
5585	Allow 4 levels of rtl nesting for most rtl codes, and if we see anything
5586	deeper than that, create DEBUG_EXPRs and emit DEBUG_INSNs before INSN. */
5587
5588	static void
5589	avoid_complex_debug_insns (rtx_insn *insn, rtx *exp_p, int depth)
5590	{
5591	rtx exp = *exp_p;
5592
5593	if (exp == NULL_RTX)
5594	return;
5595
5596	if ((OBJECT_P (exp) && !MEM_P (exp)) || GET_CODE (exp) == CLOBBER)
5597	return;
5598
5599	if (depth == 4)
5600	{
5601	/* Create DEBUG_EXPR (and DEBUG_EXPR_DECL). */
5602	rtx dval = make_debug_expr_from_rtl (exp);
5603
5604	/* Emit a debug bind insn before INSN. */
5605	rtx bind = gen_rtx_VAR_LOCATION (GET_MODE (exp),
5606	DEBUG_EXPR_TREE_DECL (dval), exp,
5607	VAR_INIT_STATUS_INITIALIZED);
5608
5609	emit_debug_insn_before (bind, insn);
5610	*exp_p = dval;
5611	return;
5612	}
5613
5614	const char *format_ptr = GET_RTX_FORMAT (GET_CODE (exp));
5615	int i, j;
5616	for (i = 0; i < GET_RTX_LENGTH (GET_CODE (exp)); i++)
5617	switch (*format_ptr++)
5618	{
5619	case 'e':
5620	avoid_complex_debug_insns (insn, exp_p: &XEXP (exp, i), depth: depth + 1);
5621	break;
5622
5623	case 'E':
5624	case 'V':
5625	for (j = 0; j < XVECLEN (exp, i); j++)
5626	avoid_complex_debug_insns (insn, exp_p: &XVECEXP (exp, i, j), depth: depth + 1);
5627	break;
5628
5629	default:
5630	break;
5631	}
5632	}
5633
5634	/* Expand the _LOCs in debug insns. We run this after expanding all
5635	regular insns, so that any variables referenced in the function
5636	will have their DECL_RTLs set. */
5637
5638	static void
5639	expand_debug_locations (void)
5640	{
5641	rtx_insn *insn;
5642	rtx_insn *last = get_last_insn ();
5643	int save_strict_alias = flag_strict_aliasing;
5644
5645	/* New alias sets while setting up memory attributes cause
5646	-fcompare-debug failures, even though it doesn't bring about any
5647	codegen changes. */
5648	flag_strict_aliasing = 0;
5649
5650	for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
5651	if (DEBUG_BIND_INSN_P (insn))
5652	{
5653	tree value = (tree)INSN_VAR_LOCATION_LOC (insn);
5654	rtx val;
5655	rtx_insn *prev_insn, *insn2;
5656	machine_mode mode;
5657
5658	if (value == NULL_TREE)
5659	val = NULL_RTX;
5660	else
5661	{
5662	if (INSN_VAR_LOCATION_STATUS (insn)
5663	== VAR_INIT_STATUS_UNINITIALIZED)
5664	val = expand_debug_source_expr (exp: value);
5665	/* The avoid_deep_ter_for_debug function inserts
5666	debug bind stmts after SSA_NAME definition, with the
5667	SSA_NAME as the whole bind location. Disable temporarily
5668	expansion of that SSA_NAME into the DEBUG_EXPR_DECL
5669	being defined in this DEBUG_INSN. */
5670	else if (deep_ter_debug_map && TREE_CODE (value) == SSA_NAME)
5671	{
5672	tree *slot = deep_ter_debug_map->get (k: value);
5673	if (slot)
5674	{
5675	if (*slot == INSN_VAR_LOCATION_DECL (insn))
5676	*slot = NULL_TREE;
5677	else
5678	slot = NULL;
5679	}
5680	val = expand_debug_expr (exp: value);
5681	if (slot)
5682	*slot = INSN_VAR_LOCATION_DECL (insn);
5683	}
5684	else
5685	val = expand_debug_expr (exp: value);
5686	gcc_assert (last == get_last_insn ());
5687	}
5688
5689	if (!val)
5690	val = gen_rtx_UNKNOWN_VAR_LOC ();
5691	else
5692	{
5693	mode = GET_MODE (INSN_VAR_LOCATION (insn));
5694
5695	gcc_assert (mode == GET_MODE (val)
5696	|| (GET_MODE (val) == VOIDmode
5697	&& (CONST_SCALAR_INT_P (val)
5698	|| GET_CODE (val) == CONST_FIXED
5699	|| GET_CODE (val) == LABEL_REF)));
5700	}
5701
5702	INSN_VAR_LOCATION_LOC (insn) = val;
5703	prev_insn = PREV_INSN (insn);
5704	for (insn2 = insn; insn2 != prev_insn; insn2 = PREV_INSN (insn: insn2))
5705	avoid_complex_debug_insns (insn: insn2, exp_p: &INSN_VAR_LOCATION_LOC (insn2), depth: 0);
5706	}
5707
5708	flag_strict_aliasing = save_strict_alias;
5709	}
5710
5711	/* Performs swapping operands of commutative operations to expand
5712	the expensive one first. */
5713
5714	static void
5715	reorder_operands (basic_block bb)
5716	{
5717	unsigned int *lattice; /* Hold cost of each statement. */
5718	unsigned int i = 0, n = 0;
5719	gimple_stmt_iterator gsi;
5720	gimple_seq stmts;
5721	gimple *stmt;
5722	bool swap;
5723	tree op0, op1;
5724	ssa_op_iter iter;
5725	use_operand_p use_p;
5726	gimple *def0, *def1;
5727
5728	/* Compute cost of each statement using estimate_num_insns. */
5729	stmts = bb_seq (bb);
5730	for (gsi = gsi_start (seq&: stmts); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
5731	{
5732	stmt = gsi_stmt (i: gsi);
5733	if (!is_gimple_debug (gs: stmt))
5734	gimple_set_uid (g: stmt, uid: n++);
5735	}
5736	lattice = XNEWVEC (unsigned int, n);
5737	for (gsi = gsi_start (seq&: stmts); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
5738	{
5739	unsigned cost;
5740	stmt = gsi_stmt (i: gsi);
5741	if (is_gimple_debug (gs: stmt))
5742	continue;
5743	cost = estimate_num_insns (stmt, &eni_size_weights);
5744	lattice[i] = cost;
5745	FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
5746	{
5747	tree use = USE_FROM_PTR (use_p);
5748	gimple *def_stmt;
5749	if (TREE_CODE (use) != SSA_NAME)
5750	continue;
5751	def_stmt = get_gimple_for_ssa_name (exp: use);
5752	if (!def_stmt)
5753	continue;
5754	lattice[i] += lattice[gimple_uid (g: def_stmt)];
5755	}
5756	i++;
5757	if (!is_gimple_assign (gs: stmt)
5758	|| !commutative_tree_code (gimple_assign_rhs_code (gs: stmt)))
5759	continue;
5760	op0 = gimple_op (gs: stmt, i: 1);
5761	op1 = gimple_op (gs: stmt, i: 2);
5762	if (TREE_CODE (op0) != SSA_NAME
5763	|| TREE_CODE (op1) != SSA_NAME)
5764	continue;
5765	/* Swap operands if the second one is more expensive. */
5766	def0 = get_gimple_for_ssa_name (exp: op0);
5767	def1 = get_gimple_for_ssa_name (exp: op1);
5768	if (!def1)
5769	continue;
5770	swap = false;
5771	if (!def0 || lattice[gimple_uid (g: def1)] > lattice[gimple_uid (g: def0)])
5772	swap = true;
5773	if (swap)
5774	{
5775	if (dump_file && (dump_flags & TDF_DETAILS))
5776	{
5777	fprintf (stream: dump_file, format: "Swap operands in stmt:\n");
5778	print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
5779	fprintf (stream: dump_file, format: "Cost left opnd=%d, right opnd=%d\n",
5780	def0 ? lattice[gimple_uid (g: def0)] : 0,
5781	lattice[gimple_uid (g: def1)]);
5782	}
5783	swap_ssa_operands (stmt, gimple_assign_rhs1_ptr (gs: stmt),
5784	gimple_assign_rhs2_ptr (gs: stmt));
5785	}
5786	}
5787	XDELETE (lattice);
5788	}
5789
5790	/* Expand basic block BB from GIMPLE trees to RTL. */
5791
5792	static basic_block
5793	expand_gimple_basic_block (basic_block bb, bool disable_tail_calls)
5794	{
5795	gimple_stmt_iterator gsi;
5796	gimple_seq stmts;
5797	gimple *stmt = NULL;
5798	rtx_note *note = NULL;
5799	rtx_insn *last;
5800	edge e;
5801	edge_iterator ei;
5802	bool nondebug_stmt_seen = false;
5803
5804	if (dump_file)
5805	fprintf (stream: dump_file, format: "\n;; Generating RTL for gimple basic block %d\n",
5806	bb->index);
5807
5808	/* Note that since we are now transitioning from GIMPLE to RTL, we
5809	cannot use the gsi_*_bb() routines because they expect the basic
5810	block to be in GIMPLE, instead of RTL. Therefore, we need to
5811	access the BB sequence directly. */
5812	if (optimize)
5813	reorder_operands (bb);
5814	stmts = bb_seq (bb);
5815	bb->il.gimple.seq = NULL;
5816	bb->il.gimple.phi_nodes = NULL;
5817	rtl_profile_for_bb (bb);
5818	init_rtl_bb_info (bb);
5819	bb->flags |= BB_RTL;
5820
5821	/* Remove the RETURN_EXPR if we may fall though to the exit
5822	instead. */
5823	gsi = gsi_last (seq&: stmts);
5824	if (!gsi_end_p (i: gsi)
5825	&& gimple_code (g: gsi_stmt (i: gsi)) == GIMPLE_RETURN)
5826	{
5827	greturn *ret_stmt = as_a <greturn *> (p: gsi_stmt (i: gsi));
5828
5829	gcc_assert (single_succ_p (bb));
5830	gcc_assert (single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun));
5831
5832	if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
5833	&& !gimple_return_retval (gs: ret_stmt))
5834	{
5835	gsi_remove (&gsi, false);
5836	single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
5837	}
5838	}
5839
5840	gsi = gsi_start (seq&: stmts);
5841	if (!gsi_end_p (i: gsi))
5842	{
5843	stmt = gsi_stmt (i: gsi);
5844	if (gimple_code (g: stmt) != GIMPLE_LABEL)
5845	stmt = NULL;
5846	}
5847
5848	rtx_code_label **elt = lab_rtx_for_bb->get (k: bb);
5849
5850	if (stmt || elt)
5851	{
5852	gcc_checking_assert (!note);
5853	last = get_last_insn ();
5854
5855	if (stmt)
5856	{
5857	expand_gimple_stmt (stmt);
5858	gsi_next (i: &gsi);
5859	}
5860
5861	if (elt)
5862	emit_label (*elt);
5863
5864	BB_HEAD (bb) = NEXT_INSN (insn: last);
5865	if (NOTE_P (BB_HEAD (bb)))
5866	BB_HEAD (bb) = NEXT_INSN (BB_HEAD (bb));
5867	gcc_assert (LABEL_P (BB_HEAD (bb)));
5868	note = emit_note_after (NOTE_INSN_BASIC_BLOCK, BB_HEAD (bb));
5869
5870	maybe_dump_rtl_for_gimple_stmt (stmt, since: last);
5871	}
5872	else
5873	BB_HEAD (bb) = note = emit_note (NOTE_INSN_BASIC_BLOCK);
5874
5875	if (note)
5876	NOTE_BASIC_BLOCK (note) = bb;
5877
5878	for (; !gsi_end_p (i: gsi); gsi_next (i: &gsi))
5879	{
5880	basic_block new_bb;
5881
5882	stmt = gsi_stmt (i: gsi);
5883	if (!is_gimple_debug (gs: stmt))
5884	nondebug_stmt_seen = true;
5885
5886	/* If this statement is a non-debug one, and we generate debug
5887	insns, then this one might be the last real use of a TERed
5888	SSA_NAME, but where there are still some debug uses further
5889	down. Expanding the current SSA name in such further debug
5890	uses by their RHS might lead to wrong debug info, as coalescing
5891	might make the operands of such RHS be placed into the same
5892	pseudo as something else. Like so:
5893	a_1 = a_0 + 1; // Assume a_1 is TERed and a_0 is dead
5894	use(a_1);
5895	a_2 = ...
5896	#DEBUG ... => a_1
5897	As a_0 and a_2 don't overlap in lifetime, assume they are coalesced.
5898	If we now would expand a_1 by it's RHS (a_0 + 1) in the debug use,
5899	the write to a_2 would actually have clobbered the place which
5900	formerly held a_0.
5901
5902	So, instead of that, we recognize the situation, and generate
5903	debug temporaries at the last real use of TERed SSA names:
5904	a_1 = a_0 + 1;
5905	#DEBUG #D1 => a_1
5906	use(a_1);
5907	a_2 = ...
5908	#DEBUG ... => #D1
5909	*/
5910	if (MAY_HAVE_DEBUG_BIND_INSNS
5911	&& SA.values
5912	&& !is_gimple_debug (gs: stmt))
5913	{
5914	ssa_op_iter iter;
5915	tree op;
5916	gimple *def;
5917
5918	location_t sloc = curr_insn_location ();
5919
5920	/* Look for SSA names that have their last use here (TERed
5921	names always have only one real use). */
5922	FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
5923	if ((def = get_gimple_for_ssa_name (exp: op)))
5924	{
5925	imm_use_iterator imm_iter;
5926	use_operand_p use_p;
5927	bool have_debug_uses = false;
5928
5929	FOR_EACH_IMM_USE_FAST (use_p, imm_iter, op)
5930	{
5931	if (gimple_debug_bind_p (USE_STMT (use_p)))
5932	{
5933	have_debug_uses = true;
5934	break;
5935	}
5936	}
5937
5938	if (have_debug_uses)
5939	{
5940	/* OP is a TERed SSA name, with DEF its defining
5941	statement, and where OP is used in further debug
5942	instructions. Generate a debug temporary, and
5943	replace all uses of OP in debug insns with that
5944	temporary. */
5945	gimple *debugstmt;
5946	tree value = gimple_assign_rhs_to_tree (stmt: def);
5947	tree vexpr = build_debug_expr_decl (TREE_TYPE (value));
5948	rtx val;
5949	machine_mode mode;
5950
5951	set_curr_insn_location (gimple_location (g: def));
5952
5953	if (DECL_P (value))
5954	mode = DECL_MODE (value);
5955	else
5956	mode = TYPE_MODE (TREE_TYPE (value));
5957	/* FIXME: Is setting the mode really necessary? */
5958	SET_DECL_MODE (vexpr, mode);
5959
5960	val = gen_rtx_VAR_LOCATION
5961	(mode, vexpr, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
5962
5963	emit_debug_insn (val);
5964
5965	FOR_EACH_IMM_USE_STMT (debugstmt, imm_iter, op)
5966	{
5967	if (!gimple_debug_bind_p (s: debugstmt))
5968	continue;
5969
5970	FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
5971	SET_USE (use_p, vexpr);
5972
5973	update_stmt (s: debugstmt);
5974	}
5975	}
5976	}
5977	set_curr_insn_location (sloc);
5978	}
5979
5980	currently_expanding_gimple_stmt = stmt;
5981
5982	/* Expand this statement, then evaluate the resulting RTL and
5983	fixup the CFG accordingly. */
5984	if (gimple_code (g: stmt) == GIMPLE_COND)
5985	{
5986	new_bb = expand_gimple_cond (bb, stmt: as_a <gcond *> (p: stmt));
5987	if (new_bb)
5988	{
5989	currently_expanding_gimple_stmt = NULL;
5990	return new_bb;
5991	}
5992	}
5993	else if (is_gimple_debug (gs: stmt))
5994	{
5995	location_t sloc = curr_insn_location ();
5996	gimple_stmt_iterator nsi = gsi;
5997
5998	for (;;)
5999	{
6000	tree var;
6001	tree value = NULL_TREE;
6002	rtx val = NULL_RTX;
6003	machine_mode mode;
6004
6005	if (!gimple_debug_nonbind_marker_p (s: stmt))
6006	{
6007	if (gimple_debug_bind_p (s: stmt))
6008	{
6009	var = gimple_debug_bind_get_var (dbg: stmt);
6010
6011	if (TREE_CODE (var) != DEBUG_EXPR_DECL
6012	&& TREE_CODE (var) != LABEL_DECL
6013	&& !target_for_debug_bind (var))
6014	goto delink_debug_stmt;
6015
6016	if (DECL_P (var) && !VECTOR_TYPE_P (TREE_TYPE (var)))
6017	mode = DECL_MODE (var);
6018	else
6019	mode = TYPE_MODE (TREE_TYPE (var));
6020
6021	if (gimple_debug_bind_has_value_p (dbg: stmt))
6022	value = gimple_debug_bind_get_value (dbg: stmt);
6023
6024	val = gen_rtx_VAR_LOCATION
6025	(mode, var, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
6026	}
6027	else if (gimple_debug_source_bind_p (s: stmt))
6028	{
6029	var = gimple_debug_source_bind_get_var (dbg: stmt);
6030
6031	value = gimple_debug_source_bind_get_value (dbg: stmt);
6032
6033	if (!VECTOR_TYPE_P (TREE_TYPE (var)))
6034	mode = DECL_MODE (var);
6035	else
6036	mode = TYPE_MODE (TREE_TYPE (var));
6037
6038	val = gen_rtx_VAR_LOCATION (mode, var, (rtx)value,
6039	VAR_INIT_STATUS_UNINITIALIZED);
6040	}
6041	else
6042	gcc_unreachable ();
6043	}
6044	/* If this function was first compiled with markers
6045	enabled, but they're now disable (e.g. LTO), drop
6046	them on the floor. */
6047	else if (gimple_debug_nonbind_marker_p (s: stmt)
6048	&& !MAY_HAVE_DEBUG_MARKER_INSNS)
6049	goto delink_debug_stmt;
6050	else if (gimple_debug_begin_stmt_p (s: stmt))
6051	val = GEN_RTX_DEBUG_MARKER_BEGIN_STMT_PAT ();
6052	else if (gimple_debug_inline_entry_p (s: stmt))
6053	val = GEN_RTX_DEBUG_MARKER_INLINE_ENTRY_PAT ();
6054	else
6055	gcc_unreachable ();
6056
6057	last = get_last_insn ();
6058
6059	set_curr_insn_location (gimple_location (g: stmt));
6060
6061	emit_debug_insn (val);
6062
6063	if (dump_file && (dump_flags & TDF_DETAILS))
6064	{
6065	/* We can't dump the insn with a TREE where an RTX
6066	is expected. */
6067	if (GET_CODE (val) == VAR_LOCATION)
6068	{
6069	gcc_checking_assert (PAT_VAR_LOCATION_LOC (val) == (rtx)value);
6070	PAT_VAR_LOCATION_LOC (val) = const0_rtx;
6071	}
6072	maybe_dump_rtl_for_gimple_stmt (stmt, since: last);
6073	if (GET_CODE (val) == VAR_LOCATION)
6074	PAT_VAR_LOCATION_LOC (val) = (rtx)value;
6075	}
6076
6077	delink_debug_stmt:
6078	/* In order not to generate too many debug temporaries,
6079	we delink all uses of debug statements we already expanded.
6080	Therefore debug statements between definition and real
6081	use of TERed SSA names will continue to use the SSA name,
6082	and not be replaced with debug temps. */
6083	delink_stmt_imm_use (stmt);
6084
6085	gsi = nsi;
6086	gsi_next (i: &nsi);
6087	if (gsi_end_p (i: nsi))
6088	break;
6089	stmt = gsi_stmt (i: nsi);
6090	if (!is_gimple_debug (gs: stmt))
6091	break;
6092	}
6093
6094	set_curr_insn_location (sloc);
6095	}
6096	else
6097	{
6098	gcall *call_stmt = dyn_cast <gcall *> (p: stmt);
6099	if (call_stmt
6100	&& gimple_call_tail_p (s: call_stmt)
6101	&& disable_tail_calls)
6102	gimple_call_set_tail (s: call_stmt, tail_p: false);
6103
6104	if (call_stmt && gimple_call_tail_p (s: call_stmt))
6105	{
6106	bool can_fallthru;
6107	new_bb = expand_gimple_tailcall (bb, stmt: call_stmt, can_fallthru: &can_fallthru);
6108	if (new_bb)
6109	{
6110	if (can_fallthru)
6111	bb = new_bb;
6112	else
6113	{
6114	currently_expanding_gimple_stmt = NULL;
6115	return new_bb;
6116	}
6117	}
6118	}
6119	else
6120	{
6121	def_operand_p def_p;
6122	def_p = SINGLE_SSA_DEF_OPERAND (stmt, SSA_OP_DEF);
6123
6124	if (def_p != NULL)
6125	{
6126	/* Ignore this stmt if it is in the list of
6127	replaceable expressions. */
6128	if (SA.values
6129	&& bitmap_bit_p (SA.values,
6130	SSA_NAME_VERSION (DEF_FROM_PTR (def_p))))
6131	continue;
6132	}
6133	last = expand_gimple_stmt (stmt);
6134	maybe_dump_rtl_for_gimple_stmt (stmt, since: last);
6135	}
6136	}
6137	}
6138
6139	currently_expanding_gimple_stmt = NULL;
6140
6141	/* Expand implicit goto and convert goto_locus. */
6142	FOR_EACH_EDGE (e, ei, bb->succs)
6143	{
6144	if (e->goto_locus != UNKNOWN_LOCATION || !nondebug_stmt_seen)
6145	set_curr_insn_location (e->goto_locus);
6146	if ((e->flags & EDGE_FALLTHRU) && e->dest != bb->next_bb)
6147	{
6148	emit_jump (label_rtx_for_bb (bb: e->dest));
6149	e->flags &= ~EDGE_FALLTHRU;
6150	}
6151	}
6152
6153	/* Expanded RTL can create a jump in the last instruction of block.
6154	This later might be assumed to be a jump to successor and break edge insertion.
6155	We need to insert dummy move to prevent this. PR41440. */
6156	if (single_succ_p (bb)
6157	&& (single_succ_edge (bb)->flags & EDGE_FALLTHRU)
6158	&& (last = get_last_insn ())
6159	&& (JUMP_P (last)
6160	|| (DEBUG_INSN_P (last)
6161	&& JUMP_P (prev_nondebug_insn (last)))))
6162	{
6163	rtx dummy = gen_reg_rtx (SImode);
6164	emit_insn_after_noloc (gen_move_insn (dummy, dummy), last, NULL);
6165	}
6166
6167	do_pending_stack_adjust ();
6168
6169	/* Find the block tail. The last insn in the block is the insn
6170	before a barrier and/or table jump insn. */
6171	last = get_last_insn ();
6172	if (BARRIER_P (last))
6173	last = PREV_INSN (insn: last);
6174	if (JUMP_TABLE_DATA_P (last))
6175	last = PREV_INSN (insn: PREV_INSN (insn: last));
6176	if (BARRIER_P (last))
6177	last = PREV_INSN (insn: last);
6178	BB_END (bb) = last;
6179
6180	update_bb_for_insn (bb);
6181
6182	return bb;
6183	}
6184
6185
6186	/* Create a basic block for initialization code. */
6187
6188	static basic_block
6189	construct_init_block (void)
6190	{
6191	basic_block init_block, first_block;
6192	edge e = NULL;
6193	int flags;
6194
6195	/* Multiple entry points not supported yet. */
6196	gcc_assert (EDGE_COUNT (ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs) == 1);
6197	init_rtl_bb_info (ENTRY_BLOCK_PTR_FOR_FN (cfun));
6198	init_rtl_bb_info (EXIT_BLOCK_PTR_FOR_FN (cfun));
6199	ENTRY_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6200	EXIT_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6201
6202	e = EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun), 0);
6203
6204	/* When entry edge points to first basic block, we don't need jump,
6205	otherwise we have to jump into proper target. */
6206	if (e && e->dest != ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb)
6207	{
6208	tree label = gimple_block_label (e->dest);
6209
6210	emit_jump (jump_target_rtx (label));
6211	flags = 0;
6212	}
6213	else
6214	flags = EDGE_FALLTHRU;
6215
6216	init_block = create_basic_block (NEXT_INSN (insn: get_insns ()),
6217	get_last_insn (),
6218	ENTRY_BLOCK_PTR_FOR_FN (cfun));
6219	init_block->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
6220	add_bb_to_loop (init_block, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6221	if (e)
6222	{
6223	first_block = e->dest;
6224	redirect_edge_succ (e, init_block);
6225	make_single_succ_edge (init_block, first_block, flags);
6226	}
6227	else
6228	make_single_succ_edge (init_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6229	EDGE_FALLTHRU);
6230
6231	update_bb_for_insn (init_block);
6232	return init_block;
6233	}
6234
6235	/* For each lexical block, set BLOCK_NUMBER to the depth at which it is
6236	found in the block tree. */
6237
6238	static void
6239	set_block_levels (tree block, int level)
6240	{
6241	while (block)
6242	{
6243	BLOCK_NUMBER (block) = level;
6244	set_block_levels (BLOCK_SUBBLOCKS (block), level: level + 1);
6245	block = BLOCK_CHAIN (block);
6246	}
6247	}
6248
6249	/* Create a block containing landing pads and similar stuff. */
6250
6251	static void
6252	construct_exit_block (void)
6253	{
6254	rtx_insn *head = get_last_insn ();
6255	rtx_insn *end;
6256	basic_block exit_block;
6257	edge e, e2;
6258	unsigned ix;
6259	edge_iterator ei;
6260	basic_block prev_bb = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
6261	rtx_insn *orig_end = BB_END (prev_bb);
6262
6263	rtl_profile_for_bb (EXIT_BLOCK_PTR_FOR_FN (cfun));
6264
6265	/* Make sure the locus is set to the end of the function, so that
6266	epilogue line numbers and warnings are set properly. */
6267	if (LOCATION_LOCUS (cfun->function_end_locus) != UNKNOWN_LOCATION)
6268	input_location = cfun->function_end_locus;
6269
6270	/* Generate rtl for function exit. */
6271	expand_function_end ();
6272
6273	end = get_last_insn ();
6274	if (head == end)
6275	return;
6276	/* While emitting the function end we could move end of the last basic
6277	block. */
6278	BB_END (prev_bb) = orig_end;
6279	while (NEXT_INSN (insn: head) && NOTE_P (NEXT_INSN (head)))
6280	head = NEXT_INSN (insn: head);
6281	/* But make sure exit_block starts with RETURN_LABEL, otherwise the
6282	bb count counting will be confused. Any instructions before that
6283	label are emitted for the case where PREV_BB falls through into the
6284	exit block, so append those instructions to prev_bb in that case. */
6285	if (NEXT_INSN (insn: head) != return_label)
6286	{
6287	while (NEXT_INSN (insn: head) != return_label)
6288	{
6289	if (!NOTE_P (NEXT_INSN (head)))
6290	BB_END (prev_bb) = NEXT_INSN (insn: head);
6291	head = NEXT_INSN (insn: head);
6292	}
6293	}
6294	exit_block = create_basic_block (NEXT_INSN (insn: head), end, prev_bb);
6295	exit_block->count = EXIT_BLOCK_PTR_FOR_FN (cfun)->count;
6296	add_bb_to_loop (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6297
6298	ix = 0;
6299	while (ix < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
6300	{
6301	e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), ix);
6302	if (!(e->flags & EDGE_ABNORMAL))
6303	redirect_edge_succ (e, exit_block);
6304	else
6305	ix++;
6306	}
6307
6308	e = make_single_succ_edge (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6309	EDGE_FALLTHRU);
6310	FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
6311	if (e2 != e)
6312	{
6313	exit_block->count -= e2->count ();
6314	}
6315	update_bb_for_insn (exit_block);
6316	}
6317
6318	/* Helper function for discover_nonconstant_array_refs.
6319	Look for ARRAY_REF nodes with non-constant indexes and mark them
6320	addressable. */
6321
6322	static tree
6323	discover_nonconstant_array_refs_r (tree * tp, int *walk_subtrees,
6324	void *data)
6325	{
6326	tree t = *tp;
6327	bitmap forced_stack_vars = (bitmap)((walk_stmt_info *)data)->info;
6328
6329	if (IS_TYPE_OR_DECL_P (t))
6330	*walk_subtrees = 0;
6331	else if (REFERENCE_CLASS_P (t) && TREE_THIS_VOLATILE (t))
6332	{
6333	t = get_base_address (t);
6334	if (t && DECL_P (t)
6335	&& DECL_MODE (t) != BLKmode
6336	&& !TREE_ADDRESSABLE (t))
6337	bitmap_set_bit (forced_stack_vars, DECL_UID (t));
6338	*walk_subtrees = 0;
6339	}
6340	else if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6341	{
6342	while (((TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6343	&& is_gimple_min_invariant (TREE_OPERAND (t, 1))
6344	&& (!TREE_OPERAND (t, 2)
6345	|| is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6346	|| (TREE_CODE (t) == COMPONENT_REF
6347	&& (!TREE_OPERAND (t,2)
6348	|| is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6349	|| TREE_CODE (t) == BIT_FIELD_REF
6350	|| TREE_CODE (t) == REALPART_EXPR
6351	|| TREE_CODE (t) == IMAGPART_EXPR
6352	|| TREE_CODE (t) == VIEW_CONVERT_EXPR
6353	|| CONVERT_EXPR_P (t))
6354	t = TREE_OPERAND (t, 0);
6355
6356	if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6357	{
6358	t = get_base_address (t);
6359	if (t && DECL_P (t)
6360	&& DECL_MODE (t) != BLKmode
6361	&& !TREE_ADDRESSABLE (t))
6362	bitmap_set_bit (forced_stack_vars, DECL_UID (t));
6363	}
6364
6365	*walk_subtrees = 0;
6366	}
6367	/* References of size POLY_INT_CST to a fixed-size object must go
6368	through memory. It's more efficient to force that here than
6369	to create temporary slots on the fly.
6370	RTL expansion expectes TARGET_MEM_REF to always address actual memory.
6371	Also, force to stack non-BLKmode vars accessed through VIEW_CONVERT_EXPR
6372	to BLKmode type. */
6373	else if (TREE_CODE (t) == TARGET_MEM_REF
6374	|| (TREE_CODE (t) == MEM_REF
6375	&& TYPE_SIZE (TREE_TYPE (t))
6376	&& POLY_INT_CST_P (TYPE_SIZE (TREE_TYPE (t))))
6377	|| (TREE_CODE (t) == VIEW_CONVERT_EXPR
6378	&& TYPE_MODE (TREE_TYPE (t)) == BLKmode))
6379	{
6380	tree base = get_base_address (t);
6381	if (base
6382	&& DECL_P (base)
6383	&& !TREE_ADDRESSABLE (base)
6384	&& DECL_MODE (base) != BLKmode
6385	&& GET_MODE_SIZE (DECL_MODE (base)).is_constant ())
6386	bitmap_set_bit (forced_stack_vars, DECL_UID (base));
6387	*walk_subtrees = 0;
6388	}
6389
6390	return NULL_TREE;
6391	}
6392
6393	/* If there's a chance to get a pseudo for t then if it would be of float mode
6394	and the actual access is via an integer mode (lowered memcpy or similar
6395	access) then avoid the register expansion if the mode likely is not storage
6396	suitable for raw bits processing (like XFmode on i?86). */
6397
6398	static void
6399	avoid_type_punning_on_regs (tree t, bitmap forced_stack_vars)
6400	{
6401	machine_mode access_mode = TYPE_MODE (TREE_TYPE (t));
6402	if (access_mode != BLKmode
6403	&& !SCALAR_INT_MODE_P (access_mode))
6404	return;
6405	tree base = get_base_address (t);
6406	if (DECL_P (base)
6407	&& !TREE_ADDRESSABLE (base)
6408	&& FLOAT_MODE_P (DECL_MODE (base))
6409	&& maybe_lt (a: GET_MODE_PRECISION (DECL_MODE (base)),
6410	b: GET_MODE_BITSIZE (GET_MODE_INNER (DECL_MODE (base))))
6411	/* Double check in the expensive way we really would get a pseudo. */
6412	&& use_register_for_decl (base))
6413	bitmap_set_bit (forced_stack_vars, DECL_UID (base));
6414	}
6415
6416	/* RTL expansion is not able to compile array references with variable
6417	offsets for arrays stored in single register. Discover such
6418	expressions and mark variables as addressable to avoid this
6419	scenario. */
6420
6421	static void
6422	discover_nonconstant_array_refs (bitmap forced_stack_vars)
6423	{
6424	basic_block bb;
6425	gimple_stmt_iterator gsi;
6426
6427	walk_stmt_info wi = {};
6428	wi.info = forced_stack_vars;
6429	FOR_EACH_BB_FN (bb, cfun)
6430	for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
6431	{
6432	gimple *stmt = gsi_stmt (i: gsi);
6433	if (!is_gimple_debug (gs: stmt))
6434	{
6435	walk_gimple_op (stmt, discover_nonconstant_array_refs_r, &wi);
6436	gcall *call = dyn_cast <gcall *> (p: stmt);
6437	if (call && gimple_call_internal_p (gs: call))
6438	{
6439	tree cand = NULL_TREE;
6440	switch (gimple_call_internal_fn (gs: call))
6441	{
6442	case IFN_LOAD_LANES:
6443	/* The source must be a MEM. */
6444	cand = gimple_call_arg (gs: call, index: 0);
6445	break;
6446	case IFN_STORE_LANES:
6447	/* The destination must be a MEM. */
6448	cand = gimple_call_lhs (gs: call);
6449	break;
6450	default:
6451	break;
6452	}
6453	if (cand)
6454	cand = get_base_address (t: cand);
6455	if (cand
6456	&& DECL_P (cand)
6457	&& use_register_for_decl (cand))
6458	bitmap_set_bit (forced_stack_vars, DECL_UID (cand));
6459	}
6460	if (gimple_vdef (g: stmt))
6461	{
6462	tree t = gimple_get_lhs (stmt);
6463	if (t && REFERENCE_CLASS_P (t))
6464	avoid_type_punning_on_regs (t, forced_stack_vars);
6465	}
6466	}
6467	}
6468	}
6469
6470	/* This function sets crtl->args.internal_arg_pointer to a virtual
6471	register if DRAP is needed. Local register allocator will replace
6472	virtual_incoming_args_rtx with the virtual register. */
6473
6474	static void
6475	expand_stack_alignment (void)
6476	{
6477	rtx drap_rtx;
6478	unsigned int preferred_stack_boundary;
6479
6480	if (! SUPPORTS_STACK_ALIGNMENT)
6481	return;
6482
6483	if (cfun->calls_alloca
6484	|| cfun->has_nonlocal_label
6485	|| crtl->has_nonlocal_goto)
6486	crtl->need_drap = true;
6487
6488	/* Call update_stack_boundary here again to update incoming stack
6489	boundary. It may set incoming stack alignment to a different
6490	value after RTL expansion. TARGET_FUNCTION_OK_FOR_SIBCALL may
6491	use the minimum incoming stack alignment to check if it is OK
6492	to perform sibcall optimization since sibcall optimization will
6493	only align the outgoing stack to incoming stack boundary. */
6494	if (targetm.calls.update_stack_boundary)
6495	targetm.calls.update_stack_boundary ();
6496
6497	/* The incoming stack frame has to be aligned at least at
6498	parm_stack_boundary. */
6499	gcc_assert (crtl->parm_stack_boundary <= INCOMING_STACK_BOUNDARY);
6500
6501	/* Update crtl->stack_alignment_estimated and use it later to align
6502	stack. We check PREFERRED_STACK_BOUNDARY if there may be non-call
6503	exceptions since callgraph doesn't collect incoming stack alignment
6504	in this case. */
6505	if (cfun->can_throw_non_call_exceptions
6506	&& PREFERRED_STACK_BOUNDARY > crtl->preferred_stack_boundary)
6507	preferred_stack_boundary = PREFERRED_STACK_BOUNDARY;
6508	else
6509	preferred_stack_boundary = crtl->preferred_stack_boundary;
6510	if (preferred_stack_boundary > crtl->stack_alignment_estimated)
6511	crtl->stack_alignment_estimated = preferred_stack_boundary;
6512	if (preferred_stack_boundary > crtl->stack_alignment_needed)
6513	crtl->stack_alignment_needed = preferred_stack_boundary;
6514
6515	gcc_assert (crtl->stack_alignment_needed
6516	<= crtl->stack_alignment_estimated);
6517
6518	crtl->stack_realign_needed
6519	= INCOMING_STACK_BOUNDARY < crtl->stack_alignment_estimated;
6520	crtl->stack_realign_tried = crtl->stack_realign_needed;
6521
6522	crtl->stack_realign_processed = true;
6523
6524	/* Target has to redefine TARGET_GET_DRAP_RTX to support stack
6525	alignment. */
6526	gcc_assert (targetm.calls.get_drap_rtx != NULL);
6527	drap_rtx = targetm.calls.get_drap_rtx ();
6528
6529	/* stack_realign_drap and drap_rtx must match. */
6530	gcc_assert ((stack_realign_drap != 0) == (drap_rtx != NULL));
6531
6532	/* Do nothing if NULL is returned, which means DRAP is not needed. */
6533	if (drap_rtx != NULL)
6534	{
6535	crtl->args.internal_arg_pointer = drap_rtx;
6536
6537	/* Call fixup_tail_calls to clean up REG_EQUIV note if DRAP is
6538	needed. */
6539	fixup_tail_calls ();
6540	}
6541	}
6542
6543
6544	static void
6545	expand_main_function (void)
6546	{
6547	#if (defined(INVOKE__main) \
6548	|| (!defined(HAS_INIT_SECTION) \
6549	&& !defined(INIT_SECTION_ASM_OP) \
6550	&& !defined(INIT_ARRAY_SECTION_ASM_OP)))
6551	emit_library_call (init_one_libfunc (NAME__MAIN), LCT_NORMAL, VOIDmode);
6552	#endif
6553	}
6554
6555
6556	/* Expand code to initialize the stack_protect_guard. This is invoked at
6557	the beginning of a function to be protected. */
6558
6559	static void
6560	stack_protect_prologue (void)
6561	{
6562	tree guard_decl = targetm.stack_protect_guard ();
6563	rtx x, y;
6564
6565	crtl->stack_protect_guard_decl = guard_decl;
6566	x = expand_normal (crtl->stack_protect_guard);
6567
6568	if (targetm.have_stack_protect_combined_set () && guard_decl)
6569	{
6570	gcc_assert (DECL_P (guard_decl));
6571	y = DECL_RTL (guard_decl);
6572
6573	/* Allow the target to compute address of Y and copy it to X without
6574	leaking Y into a register. This combined address + copy pattern
6575	allows the target to prevent spilling of any intermediate results by
6576	splitting it after register allocator. */
6577	if (rtx_insn *insn = targetm.gen_stack_protect_combined_set (x, y))
6578	{
6579	emit_insn (insn);
6580	return;
6581	}
6582	}
6583
6584	if (guard_decl)
6585	y = expand_normal (exp: guard_decl);
6586	else
6587	y = const0_rtx;
6588
6589	/* Allow the target to copy from Y to X without leaking Y into a
6590	register. */
6591	if (targetm.have_stack_protect_set ())
6592	if (rtx_insn *insn = targetm.gen_stack_protect_set (x, y))
6593	{
6594	emit_insn (insn);
6595	return;
6596	}
6597
6598	/* Otherwise do a straight move. */
6599	emit_move_insn (x, y);
6600	}
6601
6602	/* Translate the intermediate representation contained in the CFG
6603	from GIMPLE trees to RTL.
6604
6605	We do conversion per basic block and preserve/update the tree CFG.
6606	This implies we have to do some magic as the CFG can simultaneously
6607	consist of basic blocks containing RTL and GIMPLE trees. This can
6608	confuse the CFG hooks, so be careful to not manipulate CFG during
6609	the expansion. */
6610
6611	namespace {
6612
6613	const pass_data pass_data_expand =
6614	{
6615	.type: RTL_PASS, /* type */
6616	.name: "expand", /* name */
6617	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
6618	.tv_id: TV_EXPAND, /* tv_id */
6619	.properties_required: ( PROP_ssa | PROP_gimple_leh | PROP_cfg
6620	| PROP_gimple_lcx
6621	| PROP_gimple_lvec
6622	| PROP_gimple_lva), /* properties_required */
6623	PROP_rtl, /* properties_provided */
6624	.properties_destroyed: ( PROP_ssa | PROP_gimple ), /* properties_destroyed */
6625	.todo_flags_start: 0, /* todo_flags_start */
6626	.todo_flags_finish: 0, /* todo_flags_finish */
6627	};
6628
6629	class pass_expand : public rtl_opt_pass
6630	{
6631	public:
6632	pass_expand (gcc::context *ctxt)
6633	: rtl_opt_pass (pass_data_expand, ctxt)
6634	{}
6635
6636	/* opt_pass methods: */
6637	unsigned int execute (function *) final override;
6638
6639	}; // class pass_expand
6640
6641	unsigned int
6642	pass_expand::execute (function *fun)
6643	{
6644	basic_block bb, init_block;
6645	edge_iterator ei;
6646	edge e;
6647	rtx_insn *var_seq, *var_ret_seq;
6648	unsigned i;
6649
6650	timevar_push (tv: TV_OUT_OF_SSA);
6651	rewrite_out_of_ssa (sa: &SA);
6652	timevar_pop (tv: TV_OUT_OF_SSA);
6653	SA.partition_to_pseudo = XCNEWVEC (rtx, SA.map->num_partitions);
6654
6655	if (MAY_HAVE_DEBUG_BIND_STMTS && flag_tree_ter)
6656	{
6657	gimple_stmt_iterator gsi;
6658	FOR_EACH_BB_FN (bb, cfun)
6659	for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
6660	if (gimple_debug_bind_p (s: gsi_stmt (i: gsi)))
6661	avoid_deep_ter_for_debug (stmt: gsi_stmt (i: gsi), depth: 0);
6662	}
6663
6664	/* Mark arrays indexed with non-constant indices with TREE_ADDRESSABLE. */
6665	auto_bitmap forced_stack_vars;
6666	discover_nonconstant_array_refs (forced_stack_vars);
6667
6668	/* Make sure all values used by the optimization passes have sane
6669	defaults. */
6670	reg_renumber = 0;
6671
6672	/* Some backends want to know that we are expanding to RTL. */
6673	currently_expanding_to_rtl = 1;
6674	/* Dominators are not kept up-to-date as we may create new basic-blocks. */
6675	free_dominance_info (CDI_DOMINATORS);
6676
6677	rtl_profile_for_bb (ENTRY_BLOCK_PTR_FOR_FN (fun));
6678
6679	insn_locations_init ();
6680	if (!DECL_IS_UNDECLARED_BUILTIN (current_function_decl))
6681	{
6682	/* Eventually, all FEs should explicitly set function_start_locus. */
6683	if (LOCATION_LOCUS (fun->function_start_locus) == UNKNOWN_LOCATION)
6684	set_curr_insn_location
6685	(DECL_SOURCE_LOCATION (current_function_decl));
6686	else
6687	set_curr_insn_location (fun->function_start_locus);
6688	}
6689	else
6690	set_curr_insn_location (UNKNOWN_LOCATION);
6691	prologue_location = curr_insn_location ();
6692
6693	#ifdef INSN_SCHEDULING
6694	init_sched_attrs ();
6695	#endif
6696
6697	/* Make sure first insn is a note even if we don't want linenums.
6698	This makes sure the first insn will never be deleted.
6699	Also, final expects a note to appear there. */
6700	emit_note (NOTE_INSN_DELETED);
6701
6702	targetm.expand_to_rtl_hook ();
6703	crtl->init_stack_alignment ();
6704	fun->cfg->max_jumptable_ents = 0;
6705
6706	/* Resovle the function section. Some targets, like ARM EABI rely on knowledge
6707	of the function section at exapnsion time to predict distance of calls. */
6708	resolve_unique_section (current_function_decl, 0, flag_function_sections);
6709
6710	/* Expand the variables recorded during gimple lowering. */
6711	timevar_push (tv: TV_VAR_EXPAND);
6712	start_sequence ();
6713
6714	var_ret_seq = expand_used_vars (forced_stack_vars);
6715
6716	var_seq = get_insns ();
6717	end_sequence ();
6718	timevar_pop (tv: TV_VAR_EXPAND);
6719
6720	/* Honor stack protection warnings. */
6721	if (warn_stack_protect)
6722	{
6723	if (fun->calls_alloca)
6724	warning (OPT_Wstack_protector,
6725	"stack protector not protecting local variables: "
6726	"variable length buffer");
6727	if (has_short_buffer && !crtl->stack_protect_guard)
6728	warning (OPT_Wstack_protector,
6729	"stack protector not protecting function: "
6730	"all local arrays are less than %d bytes long",
6731	(int) param_ssp_buffer_size);
6732	}
6733
6734	/* Temporarily mark PARM_DECLs and RESULT_DECLs we need to expand to
6735	memory addressable so expand_function_start can emit the required
6736	copies. */
6737	auto_vec<tree, 16> marked_parms;
6738	for (tree parm = DECL_ARGUMENTS (current_function_decl); parm;
6739	parm = DECL_CHAIN (parm))
6740	if (!TREE_ADDRESSABLE (parm)
6741	&& bitmap_bit_p (forced_stack_vars, DECL_UID (parm)))
6742	{
6743	TREE_ADDRESSABLE (parm) = 1;
6744	marked_parms.safe_push (obj: parm);
6745	}
6746	if (DECL_RESULT (current_function_decl)
6747	&& !TREE_ADDRESSABLE (DECL_RESULT (current_function_decl))
6748	&& bitmap_bit_p (forced_stack_vars,
6749	DECL_UID (DECL_RESULT (current_function_decl))))
6750	{
6751	TREE_ADDRESSABLE (DECL_RESULT (current_function_decl)) = 1;
6752	marked_parms.safe_push (DECL_RESULT (current_function_decl));
6753	}
6754
6755	/* Set up parameters and prepare for return, for the function. */
6756	expand_function_start (current_function_decl);
6757
6758	/* Clear TREE_ADDRESSABLE again. */
6759	while (!marked_parms.is_empty ())
6760	TREE_ADDRESSABLE (marked_parms.pop ()) = 0;
6761
6762	/* If we emitted any instructions for setting up the variables,
6763	emit them before the FUNCTION_START note. */
6764	if (var_seq)
6765	{
6766	emit_insn_before (var_seq, parm_birth_insn);
6767
6768	/* In expand_function_end we'll insert the alloca save/restore
6769	before parm_birth_insn. We've just insertted an alloca call.
6770	Adjust the pointer to match. */
6771	parm_birth_insn = var_seq;
6772	}
6773
6774	/* Now propagate the RTL assignment of each partition to the
6775	underlying var of each SSA_NAME. */
6776	tree name;
6777
6778	FOR_EACH_SSA_NAME (i, name, cfun)
6779	{
6780	/* We might have generated new SSA names in
6781	update_alias_info_with_stack_vars. They will have a NULL
6782	defining statements, and won't be part of the partitioning,
6783	so ignore those. */
6784	if (!SSA_NAME_DEF_STMT (name))
6785	continue;
6786
6787	adjust_one_expanded_partition_var (var: name);
6788	}
6789
6790	/* Clean up RTL of variables that straddle across multiple
6791	partitions, and check that the rtl of any PARM_DECLs that are not
6792	cleaned up is that of their default defs. */
6793	FOR_EACH_SSA_NAME (i, name, cfun)
6794	{
6795	int part;
6796
6797	/* We might have generated new SSA names in
6798	update_alias_info_with_stack_vars. They will have a NULL
6799	defining statements, and won't be part of the partitioning,
6800	so ignore those. */
6801	if (!SSA_NAME_DEF_STMT (name))
6802	continue;
6803	part = var_to_partition (map: SA.map, var: name);
6804	if (part == NO_PARTITION)
6805	continue;
6806
6807	/* If this decl was marked as living in multiple places, reset
6808	this now to NULL. */
6809	tree var = SSA_NAME_VAR (name);
6810	if (var && DECL_RTL_IF_SET (var) == pc_rtx)
6811	SET_DECL_RTL (var, NULL);
6812	/* Check that the pseudos chosen by assign_parms are those of
6813	the corresponding default defs. */
6814	else if (SSA_NAME_IS_DEFAULT_DEF (name)
6815	&& (TREE_CODE (var) == PARM_DECL
6816	|| TREE_CODE (var) == RESULT_DECL))
6817	{
6818	rtx in = DECL_RTL_IF_SET (var);
6819	gcc_assert (in);
6820	rtx out = SA.partition_to_pseudo[part];
6821	gcc_assert (in == out);
6822
6823	/* Now reset VAR's RTL to IN, so that the _EXPR attrs match
6824	those expected by debug backends for each parm and for
6825	the result. This is particularly important for stabs,
6826	whose register elimination from parm's DECL_RTL may cause
6827	-fcompare-debug differences as SET_DECL_RTL changes reg's
6828	attrs. So, make sure the RTL already has the parm as the
6829	EXPR, so that it won't change. */
6830	SET_DECL_RTL (var, NULL_RTX);
6831	if (MEM_P (in))
6832	set_mem_attributes (in, var, true);
6833	SET_DECL_RTL (var, in);
6834	}
6835	}
6836
6837	/* If this function is `main', emit a call to `__main'
6838	to run global initializers, etc. */
6839	if (DECL_NAME (current_function_decl)
6840	&& MAIN_NAME_P (DECL_NAME (current_function_decl))
6841	&& DECL_FILE_SCOPE_P (current_function_decl))
6842	expand_main_function ();
6843
6844	/* Initialize the stack_protect_guard field. This must happen after the
6845	call to __main (if any) so that the external decl is initialized. */
6846	if (crtl->stack_protect_guard && targetm.stack_protect_runtime_enabled_p ())
6847	stack_protect_prologue ();
6848
6849	expand_phi_nodes (sa: &SA);
6850
6851	/* Release any stale SSA redirection data. */
6852	redirect_edge_var_map_empty ();
6853
6854	/* Register rtl specific functions for cfg. */
6855	rtl_register_cfg_hooks ();
6856
6857	init_block = construct_init_block ();
6858
6859	/* Clear EDGE_EXECUTABLE on the entry edge(s). It is cleaned from the
6860	remaining edges later. */
6861	FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (fun)->succs)
6862	e->flags &= ~EDGE_EXECUTABLE;
6863
6864	/* If the function has too many markers, drop them while expanding. */
6865	if (cfun->debug_marker_count
6866	>= param_max_debug_marker_count)
6867	cfun->debug_nonbind_markers = false;
6868
6869	lab_rtx_for_bb = new hash_map<basic_block, rtx_code_label *>;
6870	FOR_BB_BETWEEN (bb, init_block->next_bb, EXIT_BLOCK_PTR_FOR_FN (fun),
6871	next_bb)
6872	bb = expand_gimple_basic_block (bb, disable_tail_calls: var_ret_seq != NULL_RTX);
6873
6874	if (MAY_HAVE_DEBUG_BIND_INSNS)
6875	expand_debug_locations ();
6876
6877	if (deep_ter_debug_map)
6878	{
6879	delete deep_ter_debug_map;
6880	deep_ter_debug_map = NULL;
6881	}
6882
6883	/* Free stuff we no longer need after GIMPLE optimizations. */
6884	free_dominance_info (CDI_DOMINATORS);
6885	free_dominance_info (CDI_POST_DOMINATORS);
6886	delete_tree_cfg_annotations (fun);
6887
6888	timevar_push (tv: TV_OUT_OF_SSA);
6889	finish_out_of_ssa (sa: &SA);
6890	timevar_pop (tv: TV_OUT_OF_SSA);
6891
6892	timevar_push (tv: TV_POST_EXPAND);
6893	/* We are no longer in SSA form. */
6894	fun->gimple_df->in_ssa_p = false;
6895	loops_state_clear (flags: LOOP_CLOSED_SSA);
6896
6897	/* Expansion is used by optimization passes too, set maybe_hot_insn_p
6898	conservatively to true until they are all profile aware. */
6899	delete lab_rtx_for_bb;
6900	free_histograms (fun);
6901
6902	construct_exit_block ();
6903	insn_locations_finalize ();
6904
6905	if (var_ret_seq)
6906	{
6907	rtx_insn *after = return_label;
6908	rtx_insn *next = NEXT_INSN (insn: after);
6909	if (next && NOTE_INSN_BASIC_BLOCK_P (next))
6910	after = next;
6911	emit_insn_after (var_ret_seq, after);
6912	}
6913
6914	if (hwasan_sanitize_stack_p ())
6915	hwasan_maybe_emit_frame_base_init ();
6916
6917	/* Zap the tree EH table. */
6918	set_eh_throw_stmt_table (fun, NULL);
6919
6920	/* We need JUMP_LABEL be set in order to redirect jumps, and hence
6921	split edges which edge insertions might do. */
6922	rebuild_jump_labels (get_insns ());
6923
6924	/* If we have a single successor to the entry block, put the pending insns
6925	after parm birth, but before NOTE_INSNS_FUNCTION_BEG. */
6926	if (single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (fun)))
6927	{
6928	edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
6929	if (e->insns.r)
6930	{
6931	rtx_insn *insns = e->insns.r;
6932	e->insns.r = NULL;
6933	rebuild_jump_labels_chain (insns);
6934	if (NOTE_P (parm_birth_insn)
6935	&& NOTE_KIND (parm_birth_insn) == NOTE_INSN_FUNCTION_BEG)
6936	emit_insn_before_noloc (insns, parm_birth_insn, e->dest);
6937	else
6938	emit_insn_after_noloc (insns, parm_birth_insn, e->dest);
6939	}
6940	}
6941
6942	/* Otherwise, as well as for other edges, take the usual way. */
6943	commit_edge_insertions ();
6944
6945	/* We're done expanding trees to RTL. */
6946	currently_expanding_to_rtl = 0;
6947
6948	flush_mark_addressable_queue ();
6949
6950	FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun)->next_bb,
6951	EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
6952	{
6953	edge e;
6954	edge_iterator ei;
6955	for (ei = ei_start (bb->succs); (e = ei_safe_edge (i: ei)); )
6956	{
6957	/* Clear EDGE_EXECUTABLE. This flag is never used in the backend. */
6958	e->flags &= ~EDGE_EXECUTABLE;
6959
6960	/* At the moment not all abnormal edges match the RTL
6961	representation. It is safe to remove them here as
6962	find_many_sub_basic_blocks will rediscover them.
6963	In the future we should get this fixed properly. */
6964	if ((e->flags & EDGE_ABNORMAL)
6965	&& !(e->flags & EDGE_SIBCALL))
6966	remove_edge (e);
6967	else
6968	ei_next (i: &ei);
6969	}
6970	}
6971
6972	auto_sbitmap blocks (last_basic_block_for_fn (fun));
6973	bitmap_ones (blocks);
6974	find_many_sub_basic_blocks (blocks);
6975	purge_all_dead_edges ();
6976
6977	/* After initial rtl generation, call back to finish generating
6978	exception support code. We need to do this before cleaning up
6979	the CFG as the code does not expect dead landing pads. */
6980	if (fun->eh->region_tree != NULL)
6981	finish_eh_generation ();
6982
6983	/* Call expand_stack_alignment after finishing all
6984	updates to crtl->preferred_stack_boundary. */
6985	expand_stack_alignment ();
6986
6987	/* Fixup REG_EQUIV notes in the prologue if there are tailcalls in this
6988	function. */
6989	if (crtl->tail_call_emit)
6990	fixup_tail_calls ();
6991
6992	HOST_WIDE_INT patch_area_size, patch_area_entry;
6993	parse_and_check_patch_area (flag_patchable_function_entry, report_error: false,
6994	patch_area_size: &patch_area_size, patch_area_start: &patch_area_entry);
6995
6996	tree patchable_function_entry_attr
6997	= lookup_attribute (attr_name: "patchable_function_entry",
6998	DECL_ATTRIBUTES (cfun->decl));
6999	if (patchable_function_entry_attr)
7000	{
7001	tree pp_val = TREE_VALUE (patchable_function_entry_attr);
7002	tree patchable_function_entry_value1 = TREE_VALUE (pp_val);
7003
7004	patch_area_size = tree_to_uhwi (patchable_function_entry_value1);
7005	patch_area_entry = 0;
7006	if (TREE_CHAIN (pp_val) != NULL_TREE)
7007	{
7008	tree patchable_function_entry_value2
7009	= TREE_VALUE (TREE_CHAIN (pp_val));
7010	patch_area_entry = tree_to_uhwi (patchable_function_entry_value2);
7011	}
7012	}
7013
7014	if (patch_area_entry > patch_area_size)
7015	{
7016	if (patch_area_size > 0)
7017	warning (OPT_Wattributes,
7018	"patchable function entry %wu exceeds size %wu",
7019	patch_area_entry, patch_area_size);
7020	patch_area_entry = 0;
7021	}
7022
7023	crtl->patch_area_size = patch_area_size;
7024	crtl->patch_area_entry = patch_area_entry;
7025
7026	/* BB subdivision may have created basic blocks that are only reachable
7027	from unlikely bbs but not marked as such in the profile. */
7028	if (optimize)
7029	propagate_unlikely_bbs_forward ();
7030
7031	/* Remove unreachable blocks, otherwise we cannot compute dominators
7032	which are needed for loop state verification. As a side-effect
7033	this also compacts blocks.
7034	??? We cannot remove trivially dead insns here as for example
7035	the DRAP reg on i?86 is not magically live at this point.
7036	gcc.c-torture/execute/ipa-sra-2.c execution, -Os -m32 fails otherwise. */
7037	cleanup_cfg (CLEANUP_NO_INSN_DEL);
7038
7039	checking_verify_flow_info ();
7040
7041	/* Initialize pseudos allocated for hard registers. */
7042	emit_initial_value_sets ();
7043
7044	/* And finally unshare all RTL. */
7045	unshare_all_rtl ();
7046
7047	/* There's no need to defer outputting this function any more; we
7048	know we want to output it. */
7049	DECL_DEFER_OUTPUT (current_function_decl) = 0;
7050
7051	/* Now that we're done expanding trees to RTL, we shouldn't have any
7052	more CONCATs anywhere. */
7053	generating_concat_p = 0;
7054
7055	if (dump_file)
7056	{
7057	fprintf (stream: dump_file,
7058	format: "\n\n;;\n;; Full RTL generated for this function:\n;;\n");
7059	/* And the pass manager will dump RTL for us. */
7060	}
7061
7062	/* If we're emitting a nested function, make sure its parent gets
7063	emitted as well. Doing otherwise confuses debug info. */
7064	{
7065	tree parent;
7066	for (parent = DECL_CONTEXT (current_function_decl);
7067	parent != NULL_TREE;
7068	parent = get_containing_scope (parent))
7069	if (TREE_CODE (parent) == FUNCTION_DECL)
7070	TREE_SYMBOL_REFERENCED (DECL_ASSEMBLER_NAME (parent)) = 1;
7071	}
7072
7073	TREE_ASM_WRITTEN (current_function_decl) = 1;
7074
7075	/* After expanding, the return labels are no longer needed. */
7076	return_label = NULL;
7077	naked_return_label = NULL;
7078
7079	/* After expanding, the tm_restart map is no longer needed. */
7080	if (fun->gimple_df->tm_restart)
7081	fun->gimple_df->tm_restart = NULL;
7082
7083	/* Tag the blocks with a depth number so that change_scope can find
7084	the common parent easily. */
7085	set_block_levels (DECL_INITIAL (fun->decl), level: 0);
7086	default_rtl_profile ();
7087
7088	/* For -dx discard loops now, otherwise IL verify in clean_state will
7089	ICE. */
7090	if (rtl_dump_and_exit)
7091	{
7092	cfun->curr_properties &= ~PROP_loops;
7093	loop_optimizer_finalize ();
7094	}
7095
7096	timevar_pop (tv: TV_POST_EXPAND);
7097
7098	return 0;
7099	}
7100
7101	} // anon namespace
7102
7103	rtl_opt_pass *
7104	make_pass_expand (gcc::context *ctxt)
7105	{
7106	return new pass_expand (ctxt);
7107	}
7108