1	/* Tree inlining.
2	Copyright (C) 2001-2024 Free Software Foundation, Inc.
3	Contributed by Alexandre Oliva <aoliva@redhat.com>
4
5	This file is part of GCC.
6
7	GCC is free software; you can redistribute it and/or modify
8	it under the terms of the GNU General Public License as published by
9	the Free Software Foundation; either version 3, or (at your option)
10	any later version.
11
12	GCC is distributed in the hope that it will be useful,
13	but WITHOUT ANY WARRANTY; without even the implied warranty of
14	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	GNU General Public License for more details.
16
17	You should have received a copy of the GNU General Public License
18	along with GCC; see the file COPYING3. If not see
19	<http://www.gnu.org/licenses/>. */
20
21	#include "config.h"
22	#include "system.h"
23	#include "coretypes.h"
24	#include "backend.h"
25	#include "target.h"
26	#include "rtl.h"
27	#include "tree.h"
28	#include "gimple.h"
29	#include "cfghooks.h"
30	#include "tree-pass.h"
31	#include "ssa.h"
32	#include "cgraph.h"
33	#include "tree-pretty-print.h"
34	#include "diagnostic-core.h"
35	#include "gimple-predict.h"
36	#include "fold-const.h"
37	#include "stor-layout.h"
38	#include "calls.h"
39	#include "tree-inline.h"
40	#include "langhooks.h"
41	#include "cfganal.h"
42	#include "tree-iterator.h"
43	#include "intl.h"
44	#include "gimple-iterator.h"
45	#include "gimple-fold.h"
46	#include "tree-eh.h"
47	#include "gimplify.h"
48	#include "gimplify-me.h"
49	#include "gimple-walk.h"
50	#include "tree-cfg.h"
51	#include "tree-into-ssa.h"
52	#include "tree-dfa.h"
53	#include "tree-ssa.h"
54	#include "except.h"
55	#include "debug.h"
56	#include "value-prof.h"
57	#include "cfgloop.h"
58	#include "builtins.h"
59	#include "stringpool.h"
60	#include "attribs.h"
61	#include "sreal.h"
62	#include "tree-cfgcleanup.h"
63	#include "tree-ssa-live.h"
64	#include "alloc-pool.h"
65	#include "symbol-summary.h"
66	#include "symtab-thunks.h"
67	#include "symtab-clones.h"
68
69	/* I'm not real happy about this, but we need to handle gimple and
70	non-gimple trees. */
71
72	/* Inlining, Cloning, Versioning, Parallelization
73
74	Inlining: a function body is duplicated, but the PARM_DECLs are
75	remapped into VAR_DECLs, and non-void RETURN_EXPRs become
76	MODIFY_EXPRs that store to a dedicated returned-value variable.
77	The duplicated eh_region info of the copy will later be appended
78	to the info for the caller; the eh_region info in copied throwing
79	statements and RESX statements are adjusted accordingly.
80
81	Cloning: (only in C++) We have one body for a con/de/structor, and
82	multiple function decls, each with a unique parameter list.
83	Duplicate the body, using the given splay tree; some parameters
84	will become constants (like 0 or 1).
85
86	Versioning: a function body is duplicated and the result is a new
87	function rather than into blocks of an existing function as with
88	inlining. Some parameters will become constants.
89
90	Parallelization: a region of a function is duplicated resulting in
91	a new function. Variables may be replaced with complex expressions
92	to enable shared variable semantics.
93
94	All of these will simultaneously lookup any callgraph edges. If
95	we're going to inline the duplicated function body, and the given
96	function has some cloned callgraph nodes (one for each place this
97	function will be inlined) those callgraph edges will be duplicated.
98	If we're cloning the body, those callgraph edges will be
99	updated to point into the new body. (Note that the original
100	callgraph node and edge list will not be altered.)
101
102	See the CALL_EXPR handling case in copy_tree_body_r (). */
103
104	/* To Do:
105
106	o In order to make inlining-on-trees work, we pessimized
107	function-local static constants. In particular, they are now
108	always output, even when not addressed. Fix this by treating
109	function-local static constants just like global static
110	constants; the back-end already knows not to output them if they
111	are not needed.
112
113	o Provide heuristics to clamp inlining of recursive template
114	calls? */
115
116
117	/* Weights that estimate_num_insns uses to estimate the size of the
118	produced code. */
119
120	eni_weights eni_size_weights;
121
122	/* Weights that estimate_num_insns uses to estimate the time necessary
123	to execute the produced code. */
124
125	eni_weights eni_time_weights;
126
127	/* Prototypes. */
128
129	static tree declare_return_variable (copy_body_data *, tree, tree,
130	basic_block);
131	static void remap_block (tree *, copy_body_data *);
132	static void copy_bind_expr (tree *, int *, copy_body_data *);
133	static void declare_inline_vars (tree, tree);
134	static void remap_save_expr (tree *, hash_map<tree, tree> *, int *);
135	static void prepend_lexical_block (tree current_block, tree new_block);
136	static tree copy_result_decl_to_var (tree, copy_body_data *);
137	static tree copy_decl_maybe_to_var (tree, copy_body_data *);
138	static gimple_seq remap_gimple_stmt (gimple *, copy_body_data *);
139	static void insert_init_stmt (copy_body_data *, basic_block, gimple *);
140
141	/* Insert a tree->tree mapping for ID. Despite the name suggests
142	that the trees should be variables, it is used for more than that. */
143
144	void
145	insert_decl_map (copy_body_data *id, tree key, tree value)
146	{
147	id->decl_map->put (k: key, v: value);
148
149	/* Always insert an identity map as well. If we see this same new
150	node again, we won't want to duplicate it a second time. */
151	if (key != value && value)
152	id->decl_map->put (k: value, v: value);
153	}
154
155	/* If nonzero, we're remapping the contents of inlined debug
156	statements. If negative, an error has occurred, such as a
157	reference to a variable that isn't available in the inlined
158	context. */
159	static int processing_debug_stmt = 0;
160
161	/* Construct new SSA name for old NAME. ID is the inline context. */
162
163	static tree
164	remap_ssa_name (tree name, copy_body_data *id)
165	{
166	tree new_tree, var;
167	tree *n;
168
169	gcc_assert (TREE_CODE (name) == SSA_NAME);
170
171	n = id->decl_map->get (k: name);
172	if (n)
173	{
174	/* When we perform edge redirection as part of CFG copy, IPA-SRA can
175	remove an unused LHS from a call statement. Such LHS can however
176	still appear in debug statements, but their value is lost in this
177	function and we do not want to map them. */
178	if (id->killed_new_ssa_names
179	&& id->killed_new_ssa_names->contains (k: *n))
180	{
181	gcc_assert (processing_debug_stmt);
182	processing_debug_stmt = -1;
183	return name;
184	}
185
186	return unshare_expr (*n);
187	}
188
189	if (processing_debug_stmt)
190	{
191	if (SSA_NAME_IS_DEFAULT_DEF (name)
192	&& TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL
193	&& id->entry_bb == NULL
194	&& single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)))
195	{
196	gimple *def_temp;
197	gimple_stmt_iterator gsi;
198	tree val = SSA_NAME_VAR (name);
199
200	n = id->decl_map->get (k: val);
201	if (n != NULL)
202	val = *n;
203	if (TREE_CODE (val) != PARM_DECL
204	&& !(VAR_P (val) && DECL_ABSTRACT_ORIGIN (val)))
205	{
206	processing_debug_stmt = -1;
207	return name;
208	}
209	n = id->decl_map->get (k: val);
210	if (n && TREE_CODE (*n) == DEBUG_EXPR_DECL)
211	return *n;
212	tree vexpr = build_debug_expr_decl (TREE_TYPE (name));
213	/* FIXME: Is setting the mode really necessary? */
214	SET_DECL_MODE (vexpr, DECL_MODE (SSA_NAME_VAR (name)));
215	def_temp = gimple_build_debug_source_bind (vexpr, val, NULL);
216	gsi = gsi_after_labels (bb: single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
217	gsi_insert_before (&gsi, def_temp, GSI_SAME_STMT);
218	insert_decl_map (id, key: val, value: vexpr);
219	return vexpr;
220	}
221
222	processing_debug_stmt = -1;
223	return name;
224	}
225
226	/* Remap anonymous SSA names or SSA names of anonymous decls. */
227	var = SSA_NAME_VAR (name);
228	if (!var
229	|| (!SSA_NAME_IS_DEFAULT_DEF (name)
230	&& VAR_P (var)
231	&& !VAR_DECL_IS_VIRTUAL_OPERAND (var)
232	&& DECL_ARTIFICIAL (var)
233	&& DECL_IGNORED_P (var)
234	&& !DECL_NAME (var)))
235	{
236	struct ptr_info_def *pi;
237	new_tree = make_ssa_name (var: remap_type (TREE_TYPE (name), id));
238	if (!var && SSA_NAME_IDENTIFIER (name))
239	SET_SSA_NAME_VAR_OR_IDENTIFIER (new_tree, SSA_NAME_IDENTIFIER (name));
240	insert_decl_map (id, key: name, value: new_tree);
241	SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree)
242	= SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name);
243	/* At least IPA points-to info can be directly transferred. */
244	if (id->src_cfun->gimple_df
245	&& id->src_cfun->gimple_df->ipa_pta
246	&& POINTER_TYPE_P (TREE_TYPE (name))
247	&& (pi = SSA_NAME_PTR_INFO (name))
248	&& !pi->pt.anything)
249	{
250	struct ptr_info_def *new_pi = get_ptr_info (new_tree);
251	new_pi->pt = pi->pt;
252	}
253	/* So can range-info. */
254	if (!POINTER_TYPE_P (TREE_TYPE (name))
255	&& SSA_NAME_RANGE_INFO (name))
256	duplicate_ssa_name_range_info (dest: new_tree, src: name);
257	return new_tree;
258	}
259
260	/* Do not set DEF_STMT yet as statement is not copied yet. We do that
261	in copy_bb. */
262	new_tree = remap_decl (decl: var, id);
263
264	/* We might've substituted constant or another SSA_NAME for
265	the variable.
266
267	Replace the SSA name representing RESULT_DECL by variable during
268	inlining: this saves us from need to introduce PHI node in a case
269	return value is just partly initialized. */
270	if ((VAR_P (new_tree) || TREE_CODE (new_tree) == PARM_DECL)
271	&& (!SSA_NAME_VAR (name)
272	|| TREE_CODE (SSA_NAME_VAR (name)) != RESULT_DECL
273	|| !id->transform_return_to_modify))
274	{
275	struct ptr_info_def *pi;
276	new_tree = make_ssa_name (var: new_tree);
277	insert_decl_map (id, key: name, value: new_tree);
278	SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree)
279	= SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name);
280	/* At least IPA points-to info can be directly transferred. */
281	if (id->src_cfun->gimple_df
282	&& id->src_cfun->gimple_df->ipa_pta
283	&& POINTER_TYPE_P (TREE_TYPE (name))
284	&& (pi = SSA_NAME_PTR_INFO (name))
285	&& !pi->pt.anything)
286	{
287	struct ptr_info_def *new_pi = get_ptr_info (new_tree);
288	new_pi->pt = pi->pt;
289	}
290	/* So can range-info. */
291	if (!POINTER_TYPE_P (TREE_TYPE (name))
292	&& SSA_NAME_RANGE_INFO (name))
293	duplicate_ssa_name_range_info (dest: new_tree, src: name);
294	if (SSA_NAME_IS_DEFAULT_DEF (name))
295	{
296	/* By inlining function having uninitialized variable, we might
297	extend the lifetime (variable might get reused). This cause
298	ICE in the case we end up extending lifetime of SSA name across
299	abnormal edge, but also increase register pressure.
300
301	We simply initialize all uninitialized vars by 0 except
302	for case we are inlining to very first BB. We can avoid
303	this for all BBs that are not inside strongly connected
304	regions of the CFG, but this is expensive to test. */
305	if (id->entry_bb
306	&& SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name)
307	&& (!SSA_NAME_VAR (name)
308	|| TREE_CODE (SSA_NAME_VAR (name)) != PARM_DECL)
309	&& (id->entry_bb != EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun),
310	0)->dest
311	|| EDGE_COUNT (id->entry_bb->preds) != 1))
312	{
313	gimple_stmt_iterator gsi = gsi_last_bb (bb: id->entry_bb);
314	gimple *init_stmt;
315	tree zero = build_zero_cst (TREE_TYPE (new_tree));
316
317	init_stmt = gimple_build_assign (new_tree, zero);
318	gsi_insert_after (&gsi, init_stmt, GSI_NEW_STMT);
319	SSA_NAME_IS_DEFAULT_DEF (new_tree) = 0;
320	}
321	else
322	{
323	SSA_NAME_DEF_STMT (new_tree) = gimple_build_nop ();
324	set_ssa_default_def (cfun, SSA_NAME_VAR (new_tree), new_tree);
325	}
326	}
327	}
328	else
329	insert_decl_map (id, key: name, value: new_tree);
330	return new_tree;
331	}
332
333	/* Remap DECL during the copying of the BLOCK tree for the function. */
334
335	tree
336	remap_decl (tree decl, copy_body_data *id)
337	{
338	tree *n;
339
340	/* We only remap local variables in the current function. */
341
342	/* See if we have remapped this declaration. */
343
344	n = id->decl_map->get (k: decl);
345
346	if (!n && processing_debug_stmt)
347	{
348	processing_debug_stmt = -1;
349	return decl;
350	}
351
352	/* When remapping a type within copy_gimple_seq_and_replace_locals, all
353	necessary DECLs have already been remapped and we do not want to duplicate
354	a decl coming from outside of the sequence we are copying. */
355	if (!n
356	&& id->prevent_decl_creation_for_types
357	&& id->remapping_type_depth > 0
358	&& (VAR_P (decl) || TREE_CODE (decl) == PARM_DECL))
359	return decl;
360
361	/* If we didn't already have an equivalent for this declaration, create one
362	now. */
363	if (!n)
364	{
365	/* Make a copy of the variable or label. */
366	tree t = id->copy_decl (decl, id);
367
368	/* Remember it, so that if we encounter this local entity again
369	we can reuse this copy. Do this early because remap_type may
370	need this decl for TYPE_STUB_DECL. */
371	insert_decl_map (id, key: decl, value: t);
372
373	if (!DECL_P (t) || t == decl)
374	return t;
375
376	/* Remap types, if necessary. */
377	TREE_TYPE (t) = remap_type (TREE_TYPE (t), id);
378	if (TREE_CODE (t) == TYPE_DECL)
379	{
380	DECL_ORIGINAL_TYPE (t) = remap_type (DECL_ORIGINAL_TYPE (t), id);
381
382	/* Preserve the invariant that DECL_ORIGINAL_TYPE != TREE_TYPE,
383	which is enforced in gen_typedef_die when DECL_ABSTRACT_ORIGIN
384	is not set on the TYPE_DECL, for example in LTO mode. */
385	if (DECL_ORIGINAL_TYPE (t) == TREE_TYPE (t))
386	{
387	tree x = build_variant_type_copy (TREE_TYPE (t));
388	TYPE_STUB_DECL (x) = TYPE_STUB_DECL (TREE_TYPE (t));
389	TYPE_NAME (x) = TYPE_NAME (TREE_TYPE (t));
390	DECL_ORIGINAL_TYPE (t) = x;
391	}
392	}
393
394	/* Remap sizes as necessary. */
395	walk_tree (&DECL_SIZE (t), copy_tree_body_r, id, NULL);
396	walk_tree (&DECL_SIZE_UNIT (t), copy_tree_body_r, id, NULL);
397
398	/* If fields, do likewise for offset and qualifier. */
399	if (TREE_CODE (t) == FIELD_DECL)
400	{
401	walk_tree (&DECL_FIELD_OFFSET (t), copy_tree_body_r, id, NULL);
402	if (TREE_CODE (DECL_CONTEXT (t)) == QUAL_UNION_TYPE)
403	walk_tree (&DECL_QUALIFIER (t), copy_tree_body_r, id, NULL);
404	}
405
406	return t;
407	}
408
409	if (id->do_not_unshare)
410	return *n;
411	else
412	return unshare_expr (*n);
413	}
414
415	static tree
416	remap_type_1 (tree type, copy_body_data *id)
417	{
418	tree new_tree, t;
419
420	/* We do need a copy. build and register it now. If this is a pointer or
421	reference type, remap the designated type and make a new pointer or
422	reference type. */
423	if (TREE_CODE (type) == POINTER_TYPE)
424	{
425	new_tree = build_pointer_type_for_mode (remap_type (TREE_TYPE (type), id),
426	TYPE_MODE (type),
427	TYPE_REF_CAN_ALIAS_ALL (type));
428	if (TYPE_ATTRIBUTES (type) || TYPE_QUALS (type))
429	new_tree = build_type_attribute_qual_variant (new_tree,
430	TYPE_ATTRIBUTES (type),
431	TYPE_QUALS (type));
432	insert_decl_map (id, key: type, value: new_tree);
433	return new_tree;
434	}
435	else if (TREE_CODE (type) == REFERENCE_TYPE)
436	{
437	new_tree = build_reference_type_for_mode (remap_type (TREE_TYPE (type), id),
438	TYPE_MODE (type),
439	TYPE_REF_CAN_ALIAS_ALL (type));
440	if (TYPE_ATTRIBUTES (type) || TYPE_QUALS (type))
441	new_tree = build_type_attribute_qual_variant (new_tree,
442	TYPE_ATTRIBUTES (type),
443	TYPE_QUALS (type));
444	insert_decl_map (id, key: type, value: new_tree);
445	return new_tree;
446	}
447	else
448	new_tree = copy_node (type);
449
450	insert_decl_map (id, key: type, value: new_tree);
451
452	/* This is a new type, not a copy of an old type. Need to reassociate
453	variants. We can handle everything except the main variant lazily. */
454	t = TYPE_MAIN_VARIANT (type);
455	if (type != t)
456	{
457	t = remap_type (type: t, id);
458	TYPE_MAIN_VARIANT (new_tree) = t;
459	TYPE_NEXT_VARIANT (new_tree) = TYPE_NEXT_VARIANT (t);
460	TYPE_NEXT_VARIANT (t) = new_tree;
461	}
462	else
463	{
464	TYPE_MAIN_VARIANT (new_tree) = new_tree;
465	TYPE_NEXT_VARIANT (new_tree) = NULL;
466	}
467
468	if (TYPE_STUB_DECL (type))
469	TYPE_STUB_DECL (new_tree) = remap_decl (TYPE_STUB_DECL (type), id);
470
471	/* Lazily create pointer and reference types. */
472	TYPE_POINTER_TO (new_tree) = NULL;
473	TYPE_REFERENCE_TO (new_tree) = NULL;
474
475	/* Copy all types that may contain references to local variables; be sure to
476	preserve sharing in between type and its main variant when possible. */
477	switch (TREE_CODE (new_tree))
478	{
479	case INTEGER_TYPE:
480	case REAL_TYPE:
481	case FIXED_POINT_TYPE:
482	case ENUMERAL_TYPE:
483	case BOOLEAN_TYPE:
484	if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
485	{
486	gcc_checking_assert (TYPE_MIN_VALUE (type) == TYPE_MIN_VALUE (TYPE_MAIN_VARIANT (type)));
487	gcc_checking_assert (TYPE_MAX_VALUE (type) == TYPE_MAX_VALUE (TYPE_MAIN_VARIANT (type)));
488
489	TYPE_MIN_VALUE (new_tree) = TYPE_MIN_VALUE (TYPE_MAIN_VARIANT (new_tree));
490	TYPE_MAX_VALUE (new_tree) = TYPE_MAX_VALUE (TYPE_MAIN_VARIANT (new_tree));
491	}
492	else
493	{
494	t = TYPE_MIN_VALUE (new_tree);
495	if (t && TREE_CODE (t) != INTEGER_CST)
496	walk_tree (&TYPE_MIN_VALUE (new_tree), copy_tree_body_r, id, NULL);
497
498	t = TYPE_MAX_VALUE (new_tree);
499	if (t && TREE_CODE (t) != INTEGER_CST)
500	walk_tree (&TYPE_MAX_VALUE (new_tree), copy_tree_body_r, id, NULL);
501	}
502	return new_tree;
503
504	case FUNCTION_TYPE:
505	if (TYPE_MAIN_VARIANT (new_tree) != new_tree
506	&& TREE_TYPE (type) == TREE_TYPE (TYPE_MAIN_VARIANT (type)))
507	TREE_TYPE (new_tree) = TREE_TYPE (TYPE_MAIN_VARIANT (new_tree));
508	else
509	TREE_TYPE (new_tree) = remap_type (TREE_TYPE (new_tree), id);
510	if (TYPE_MAIN_VARIANT (new_tree) != new_tree
511	&& TYPE_ARG_TYPES (type) == TYPE_ARG_TYPES (TYPE_MAIN_VARIANT (type)))
512	TYPE_ARG_TYPES (new_tree) = TYPE_ARG_TYPES (TYPE_MAIN_VARIANT (new_tree));
513	else
514	walk_tree (&TYPE_ARG_TYPES (new_tree), copy_tree_body_r, id, NULL);
515	return new_tree;
516
517	case ARRAY_TYPE:
518	if (TYPE_MAIN_VARIANT (new_tree) != new_tree
519	&& TREE_TYPE (type) == TREE_TYPE (TYPE_MAIN_VARIANT (type)))
520	TREE_TYPE (new_tree) = TREE_TYPE (TYPE_MAIN_VARIANT (new_tree));
521	else
522	TREE_TYPE (new_tree) = remap_type (TREE_TYPE (new_tree), id);
523
524	if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
525	{
526	gcc_checking_assert (TYPE_DOMAIN (type)
527	== TYPE_DOMAIN (TYPE_MAIN_VARIANT (type)));
528	TYPE_DOMAIN (new_tree) = TYPE_DOMAIN (TYPE_MAIN_VARIANT (new_tree));
529	}
530	else
531	{
532	TYPE_DOMAIN (new_tree) = remap_type (TYPE_DOMAIN (new_tree), id);
533	/* For array bounds where we have decided not to copy over the bounds
534	variable which isn't used in OpenMP/OpenACC region, change them to
535	an uninitialized VAR_DECL temporary. */
536	if (id->adjust_array_error_bounds
537	&& TYPE_DOMAIN (new_tree)
538	&& TYPE_MAX_VALUE (TYPE_DOMAIN (new_tree)) == error_mark_node
539	&& TYPE_MAX_VALUE (TYPE_DOMAIN (type)) != error_mark_node)
540	{
541	tree v = create_tmp_var (TREE_TYPE (TYPE_DOMAIN (new_tree)));
542	DECL_ATTRIBUTES (v)
543	= tree_cons (get_identifier ("omp dummy var"), NULL_TREE,
544	DECL_ATTRIBUTES (v));
545	TYPE_MAX_VALUE (TYPE_DOMAIN (new_tree)) = v;
546	}
547	}
548	break;
549
550	case RECORD_TYPE:
551	case UNION_TYPE:
552	case QUAL_UNION_TYPE:
553	if (TYPE_MAIN_VARIANT (type) != type
554	&& TYPE_FIELDS (type) == TYPE_FIELDS (TYPE_MAIN_VARIANT (type)))
555	TYPE_FIELDS (new_tree) = TYPE_FIELDS (TYPE_MAIN_VARIANT (new_tree));
556	else
557	{
558	tree f, nf = NULL;
559
560	for (f = TYPE_FIELDS (new_tree); f ; f = DECL_CHAIN (f))
561	{
562	t = remap_decl (decl: f, id);
563	DECL_CONTEXT (t) = new_tree;
564	DECL_CHAIN (t) = nf;
565	nf = t;
566	}
567	TYPE_FIELDS (new_tree) = nreverse (nf);
568	}
569	break;
570
571	case OFFSET_TYPE:
572	default:
573	/* Shouldn't have been thought variable sized. */
574	gcc_unreachable ();
575	}
576
577	/* All variants of type share the same size, so use the already remaped data. */
578	if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
579	{
580	tree s = TYPE_SIZE (type);
581	tree mvs = TYPE_SIZE (TYPE_MAIN_VARIANT (type));
582	tree su = TYPE_SIZE_UNIT (type);
583	tree mvsu = TYPE_SIZE_UNIT (TYPE_MAIN_VARIANT (type));
584	gcc_checking_assert ((TREE_CODE (s) == PLACEHOLDER_EXPR
585	&& (TREE_CODE (mvs) == PLACEHOLDER_EXPR))
586	|| s == mvs);
587	gcc_checking_assert ((TREE_CODE (su) == PLACEHOLDER_EXPR
588	&& (TREE_CODE (mvsu) == PLACEHOLDER_EXPR))
589	|| su == mvsu);
590	TYPE_SIZE (new_tree) = TYPE_SIZE (TYPE_MAIN_VARIANT (new_tree));
591	TYPE_SIZE_UNIT (new_tree) = TYPE_SIZE_UNIT (TYPE_MAIN_VARIANT (new_tree));
592	}
593	else
594	{
595	walk_tree (&TYPE_SIZE (new_tree), copy_tree_body_r, id, NULL);
596	walk_tree (&TYPE_SIZE_UNIT (new_tree), copy_tree_body_r, id, NULL);
597	}
598
599	return new_tree;
600	}
601
602	/* Helper function for remap_type_2, called through walk_tree. */
603
604	static tree
605	remap_type_3 (tree *tp, int *walk_subtrees, void *data)
606	{
607	copy_body_data *id = (copy_body_data *) data;
608
609	if (TYPE_P (*tp))
610	*walk_subtrees = 0;
611
612	else if (DECL_P (*tp) && remap_decl (decl: *tp, id) != *tp)
613	return *tp;
614
615	return NULL_TREE;
616	}
617
618	/* Return true if TYPE needs to be remapped because remap_decl on any
619	needed embedded decl returns something other than that decl. */
620
621	static bool
622	remap_type_2 (tree type, copy_body_data *id)
623	{
624	tree t;
625
626	#define RETURN_TRUE_IF_VAR(T) \
627	do \
628	{ \
629	tree _t = (T); \
630	if (_t) \
631	{ \
632	if (DECL_P (_t) && remap_decl (_t, id) != _t) \
633	return true; \
634	if (!TYPE_SIZES_GIMPLIFIED (type) \
635	&& walk_tree (&_t, remap_type_3, id, NULL)) \
636	return true; \
637	} \
638	} \
639	while (0)
640
641	switch (TREE_CODE (type))
642	{
643	case POINTER_TYPE:
644	case REFERENCE_TYPE:
645	case FUNCTION_TYPE:
646	case METHOD_TYPE:
647	return remap_type_2 (TREE_TYPE (type), id);
648
649	case INTEGER_TYPE:
650	case REAL_TYPE:
651	case FIXED_POINT_TYPE:
652	case ENUMERAL_TYPE:
653	case BOOLEAN_TYPE:
654	RETURN_TRUE_IF_VAR (TYPE_MIN_VALUE (type));
655	RETURN_TRUE_IF_VAR (TYPE_MAX_VALUE (type));
656	return false;
657
658	case ARRAY_TYPE:
659	if (remap_type_2 (TREE_TYPE (type), id)
660	|| (TYPE_DOMAIN (type) && remap_type_2 (TYPE_DOMAIN (type), id)))
661	return true;
662	break;
663
664	case RECORD_TYPE:
665	case UNION_TYPE:
666	case QUAL_UNION_TYPE:
667	for (t = TYPE_FIELDS (type); t; t = DECL_CHAIN (t))
668	if (TREE_CODE (t) == FIELD_DECL)
669	{
670	RETURN_TRUE_IF_VAR (DECL_FIELD_OFFSET (t));
671	RETURN_TRUE_IF_VAR (DECL_SIZE (t));
672	RETURN_TRUE_IF_VAR (DECL_SIZE_UNIT (t));
673	if (TREE_CODE (type) == QUAL_UNION_TYPE)
674	RETURN_TRUE_IF_VAR (DECL_QUALIFIER (t));
675	}
676	break;
677
678	default:
679	return false;
680	}
681
682	RETURN_TRUE_IF_VAR (TYPE_SIZE (type));
683	RETURN_TRUE_IF_VAR (TYPE_SIZE_UNIT (type));
684	return false;
685	#undef RETURN_TRUE_IF_VAR
686	}
687
688	tree
689	remap_type (tree type, copy_body_data *id)
690	{
691	tree *node;
692	tree tmp;
693
694	if (type == NULL)
695	return type;
696
697	/* See if we have remapped this type. */
698	node = id->decl_map->get (k: type);
699	if (node)
700	return *node;
701
702	/* The type only needs remapping if it's variably modified. */
703	if (! variably_modified_type_p (type, id->src_fn)
704	/* Don't remap if copy_decl method doesn't always return a new
705	decl and for all embedded decls returns the passed in decl. */
706	|| (id->dont_remap_vla_if_no_change && !remap_type_2 (type, id)))
707	{
708	insert_decl_map (id, key: type, value: type);
709	return type;
710	}
711
712	id->remapping_type_depth++;
713	tmp = remap_type_1 (type, id);
714	id->remapping_type_depth--;
715
716	return tmp;
717	}
718
719	/* Decide if DECL can be put into BLOCK_NONLOCAL_VARs. */
720
721	static bool
722	can_be_nonlocal (tree decl, copy_body_data *id)
723	{
724	/* We cannot duplicate function decls. */
725	if (TREE_CODE (decl) == FUNCTION_DECL)
726	return true;
727
728	/* Local static vars must be non-local or we get multiple declaration
729	problems. */
730	if (VAR_P (decl) && !auto_var_in_fn_p (decl, id->src_fn))
731	return true;
732
733	return false;
734	}
735
736	static tree
737	remap_decls (tree decls, vec<tree, va_gc> **nonlocalized_list,
738	copy_body_data *id)
739	{
740	tree old_var;
741	tree new_decls = NULL_TREE;
742
743	/* Remap its variables. */
744	for (old_var = decls; old_var; old_var = DECL_CHAIN (old_var))
745	{
746	tree new_var;
747
748	if (can_be_nonlocal (decl: old_var, id))
749	{
750	/* We need to add this variable to the local decls as otherwise
751	nothing else will do so. */
752	if (VAR_P (old_var) && ! DECL_EXTERNAL (old_var) && cfun)
753	add_local_decl (cfun, d: old_var);
754	if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE)
755	&& !DECL_IGNORED_P (old_var)
756	&& nonlocalized_list)
757	vec_safe_push (v&: *nonlocalized_list, obj: old_var);
758	continue;
759	}
760
761	/* Remap the variable. */
762	new_var = remap_decl (decl: old_var, id);
763
764	/* If we didn't remap this variable, we can't mess with its
765	TREE_CHAIN. If we remapped this variable to the return slot, it's
766	already declared somewhere else, so don't declare it here. */
767
768	if (new_var == old_var || new_var == id->retvar)
769	;
770	else if (!new_var)
771	{
772	if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE)
773	&& !DECL_IGNORED_P (old_var)
774	&& nonlocalized_list)
775	vec_safe_push (v&: *nonlocalized_list, obj: old_var);
776	}
777	else
778	{
779	gcc_assert (DECL_P (new_var));
780	DECL_CHAIN (new_var) = new_decls;
781	new_decls = new_var;
782
783	/* Also copy value-expressions. */
784	if (VAR_P (new_var) && DECL_HAS_VALUE_EXPR_P (new_var))
785	{
786	tree tem = DECL_VALUE_EXPR (new_var);
787	bool old_regimplify = id->regimplify;
788	id->remapping_type_depth++;
789	walk_tree (&tem, copy_tree_body_r, id, NULL);
790	id->remapping_type_depth--;
791	id->regimplify = old_regimplify;
792	SET_DECL_VALUE_EXPR (new_var, tem);
793	}
794	}
795	}
796
797	return nreverse (new_decls);
798	}
799
800	/* Copy the BLOCK to contain remapped versions of the variables
801	therein. And hook the new block into the block-tree. */
802
803	static void
804	remap_block (tree *block, copy_body_data *id)
805	{
806	tree old_block;
807	tree new_block;
808
809	/* Make the new block. */
810	old_block = *block;
811	new_block = make_node (BLOCK);
812	TREE_USED (new_block) = TREE_USED (old_block);
813	BLOCK_ABSTRACT_ORIGIN (new_block) = BLOCK_ORIGIN (old_block);
814	BLOCK_SOURCE_LOCATION (new_block) = BLOCK_SOURCE_LOCATION (old_block);
815	BLOCK_NONLOCALIZED_VARS (new_block)
816	= vec_safe_copy (BLOCK_NONLOCALIZED_VARS (old_block));
817	*block = new_block;
818
819	/* Remap its variables. */
820	BLOCK_VARS (new_block) = remap_decls (BLOCK_VARS (old_block),
821	nonlocalized_list: &BLOCK_NONLOCALIZED_VARS (new_block),
822	id);
823
824	/* Remember the remapped block. */
825	insert_decl_map (id, key: old_block, value: new_block);
826	}
827
828	/* Copy the whole block tree and root it in id->block. */
829
830	static tree
831	remap_blocks (tree block, copy_body_data *id)
832	{
833	tree t;
834	tree new_tree = block;
835
836	if (!block)
837	return NULL;
838
839	remap_block (block: &new_tree, id);
840	gcc_assert (new_tree != block);
841	for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
842	prepend_lexical_block (current_block: new_tree, new_block: remap_blocks (block: t, id));
843	/* Blocks are in arbitrary order, but make things slightly prettier and do
844	not swap order when producing a copy. */
845	BLOCK_SUBBLOCKS (new_tree) = blocks_nreverse (BLOCK_SUBBLOCKS (new_tree));
846	return new_tree;
847	}
848
849	/* Remap the block tree rooted at BLOCK to nothing. */
850
851	static void
852	remap_blocks_to_null (tree block, copy_body_data *id)
853	{
854	tree t;
855	insert_decl_map (id, key: block, NULL_TREE);
856	for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
857	remap_blocks_to_null (block: t, id);
858	}
859
860	/* Remap the location info pointed to by LOCUS. */
861
862	static location_t
863	remap_location (location_t locus, copy_body_data *id)
864	{
865	if (LOCATION_BLOCK (locus))
866	{
867	tree *n = id->decl_map->get (LOCATION_BLOCK (locus));
868	gcc_assert (n);
869	if (*n)
870	return set_block (loc: locus, block: *n);
871	}
872
873	locus = LOCATION_LOCUS (locus);
874
875	if (locus != UNKNOWN_LOCATION && id->block)
876	return set_block (loc: locus, block: id->block);
877
878	return locus;
879	}
880
881	static void
882	copy_statement_list (tree *tp)
883	{
884	tree_stmt_iterator oi, ni;
885	tree new_tree;
886
887	new_tree = alloc_stmt_list ();
888	ni = tsi_start (t: new_tree);
889	oi = tsi_start (t: *tp);
890	TREE_TYPE (new_tree) = TREE_TYPE (*tp);
891	*tp = new_tree;
892
893	for (; !tsi_end_p (i: oi); tsi_next (i: &oi))
894	{
895	tree stmt = tsi_stmt (i: oi);
896	if (TREE_CODE (stmt) == STATEMENT_LIST)
897	/* This copy is not redundant; tsi_link_after will smash this
898	STATEMENT_LIST into the end of the one we're building, and we
899	don't want to do that with the original. */
900	copy_statement_list (tp: &stmt);
901	tsi_link_after (&ni, stmt, TSI_CONTINUE_LINKING);
902	}
903	}
904
905	static void
906	copy_bind_expr (tree *tp, int *walk_subtrees, copy_body_data *id)
907	{
908	tree block = BIND_EXPR_BLOCK (*tp);
909	/* Copy (and replace) the statement. */
910	copy_tree_r (tp, walk_subtrees, NULL);
911	if (block)
912	{
913	remap_block (block: &block, id);
914	BIND_EXPR_BLOCK (*tp) = block;
915	}
916
917	if (BIND_EXPR_VARS (*tp))
918	/* This will remap a lot of the same decls again, but this should be
919	harmless. */
920	BIND_EXPR_VARS (*tp) = remap_decls (BIND_EXPR_VARS (*tp), NULL, id);
921	}
922
923
924	/* Create a new gimple_seq by remapping all the statements in BODY
925	using the inlining information in ID. */
926
927	static gimple_seq
928	remap_gimple_seq (gimple_seq body, copy_body_data *id)
929	{
930	gimple_stmt_iterator si;
931	gimple_seq new_body = NULL;
932
933	for (si = gsi_start (seq&: body); !gsi_end_p (i: si); gsi_next (i: &si))
934	{
935	gimple_seq new_stmts = remap_gimple_stmt (gsi_stmt (i: si), id);
936	gimple_seq_add_seq (&new_body, new_stmts);
937	}
938
939	return new_body;
940	}
941
942
943	/* Copy a GIMPLE_BIND statement STMT, remapping all the symbols in its
944	block using the mapping information in ID. */
945
946	static gimple *
947	copy_gimple_bind (gbind *stmt, copy_body_data *id)
948	{
949	gimple *new_bind;
950	tree new_block, new_vars;
951	gimple_seq body, new_body;
952
953	/* Copy the statement. Note that we purposely don't use copy_stmt
954	here because we need to remap statements as we copy. */
955	body = gimple_bind_body (gs: stmt);
956	new_body = remap_gimple_seq (body, id);
957
958	new_block = gimple_bind_block (bind_stmt: stmt);
959	if (new_block)
960	remap_block (block: &new_block, id);
961
962	/* This will remap a lot of the same decls again, but this should be
963	harmless. */
964	new_vars = gimple_bind_vars (bind_stmt: stmt);
965	if (new_vars)
966	new_vars = remap_decls (decls: new_vars, NULL, id);
967
968	new_bind = gimple_build_bind (new_vars, new_body, new_block);
969
970	return new_bind;
971	}
972
973	/* Return true if DECL is a parameter or a SSA_NAME for a parameter. */
974
975	static bool
976	is_parm (tree decl)
977	{
978	if (TREE_CODE (decl) == SSA_NAME)
979	{
980	decl = SSA_NAME_VAR (decl);
981	if (!decl)
982	return false;
983	}
984
985	return (TREE_CODE (decl) == PARM_DECL);
986	}
987
988	/* Remap the dependence CLIQUE from the source to the destination function
989	as specified in ID. */
990
991	static unsigned short
992	remap_dependence_clique (copy_body_data *id, unsigned short clique)
993	{
994	if (clique == 0 || processing_debug_stmt)
995	return 0;
996	if (!id->dependence_map)
997	id->dependence_map = new hash_map<dependence_hash, unsigned short>;
998	bool existed;
999	unsigned short &newc = id->dependence_map->get_or_insert (k: clique, existed: &existed);
1000	if (!existed)
1001	{
1002	/* Clique 1 is reserved for local ones set by PTA. */
1003	if (cfun->last_clique == 0)
1004	cfun->last_clique = 1;
1005	newc = get_new_clique (cfun);
1006	}
1007	return newc;
1008	}
1009
1010	/* Remap the GIMPLE operand pointed to by *TP. DATA is really a
1011	'struct walk_stmt_info *'. DATA->INFO is a 'copy_body_data *'.
1012	WALK_SUBTREES is used to indicate walk_gimple_op whether to keep
1013	recursing into the children nodes of *TP. */
1014
1015	static tree
1016	remap_gimple_op_r (tree *tp, int *walk_subtrees, void *data)
1017	{
1018	struct walk_stmt_info *wi_p = (struct walk_stmt_info *) data;
1019	copy_body_data *id = (copy_body_data *) wi_p->info;
1020	tree fn = id->src_fn;
1021
1022	/* For recursive invocations this is no longer the LHS itself. */
1023	bool is_lhs = wi_p->is_lhs;
1024	wi_p->is_lhs = false;
1025
1026	if (TREE_CODE (*tp) == SSA_NAME)
1027	{
1028	*tp = remap_ssa_name (name: *tp, id);
1029	*walk_subtrees = 0;
1030	if (is_lhs)
1031	SSA_NAME_DEF_STMT (*tp) = wi_p->stmt;
1032	return NULL;
1033	}
1034	else if (auto_var_in_fn_p (*tp, fn))
1035	{
1036	/* Local variables and labels need to be replaced by equivalent
1037	variables. We don't want to copy static variables; there's
1038	only one of those, no matter how many times we inline the
1039	containing function. Similarly for globals from an outer
1040	function. */
1041	tree new_decl;
1042
1043	/* Remap the declaration. */
1044	new_decl = remap_decl (decl: *tp, id);
1045	gcc_assert (new_decl);
1046	/* Replace this variable with the copy. */
1047	STRIP_TYPE_NOPS (new_decl);
1048	/* ??? The C++ frontend uses void * pointer zero to initialize
1049	any other type. This confuses the middle-end type verification.
1050	As cloned bodies do not go through gimplification again the fixup
1051	there doesn't trigger. */
1052	if (TREE_CODE (new_decl) == INTEGER_CST
1053	&& !useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (new_decl)))
1054	new_decl = fold_convert (TREE_TYPE (*tp), new_decl);
1055	*tp = new_decl;
1056	*walk_subtrees = 0;
1057	}
1058	else if (TREE_CODE (*tp) == STATEMENT_LIST)
1059	gcc_unreachable ();
1060	else if (TREE_CODE (*tp) == SAVE_EXPR)
1061	gcc_unreachable ();
1062	else if (TREE_CODE (*tp) == LABEL_DECL
1063	&& (!DECL_CONTEXT (*tp)
1064	|| decl_function_context (*tp) == id->src_fn))
1065	/* These may need to be remapped for EH handling. */
1066	*tp = remap_decl (decl: *tp, id);
1067	else if (TREE_CODE (*tp) == FIELD_DECL)
1068	{
1069	/* If the enclosing record type is variably_modified_type_p, the field
1070	has already been remapped. Otherwise, it need not be. */
1071	tree *n = id->decl_map->get (k: *tp);
1072	if (n)
1073	*tp = *n;
1074	*walk_subtrees = 0;
1075	}
1076	else if (TYPE_P (*tp))
1077	/* Types may need remapping as well. */
1078	*tp = remap_type (type: *tp, id);
1079	else if (CONSTANT_CLASS_P (*tp))
1080	{
1081	/* If this is a constant, we have to copy the node iff the type
1082	will be remapped. copy_tree_r will not copy a constant. */
1083	tree new_type = remap_type (TREE_TYPE (*tp), id);
1084
1085	if (new_type == TREE_TYPE (*tp))
1086	*walk_subtrees = 0;
1087
1088	else if (TREE_CODE (*tp) == INTEGER_CST)
1089	*tp = wide_int_to_tree (type: new_type, cst: wi::to_wide (t: *tp));
1090	else
1091	{
1092	*tp = copy_node (*tp);
1093	TREE_TYPE (*tp) = new_type;
1094	}
1095	}
1096	else
1097	{
1098	/* Otherwise, just copy the node. Note that copy_tree_r already
1099	knows not to copy VAR_DECLs, etc., so this is safe. */
1100
1101	if (TREE_CODE (*tp) == MEM_REF && !id->do_not_fold)
1102	{
1103	/* We need to re-canonicalize MEM_REFs from inline substitutions
1104	that can happen when a pointer argument is an ADDR_EXPR.
1105	Recurse here manually to allow that. */
1106	tree ptr = TREE_OPERAND (*tp, 0);
1107	tree type = remap_type (TREE_TYPE (*tp), id);
1108	tree old = *tp;
1109	walk_tree (&ptr, remap_gimple_op_r, data, NULL);
1110	*tp = fold_build2 (MEM_REF, type, ptr, TREE_OPERAND (*tp, 1));
1111	TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
1112	TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
1113	copy_warning (*tp, old);
1114	if (MR_DEPENDENCE_CLIQUE (old) != 0)
1115	{
1116	MR_DEPENDENCE_CLIQUE (*tp)
1117	= remap_dependence_clique (id, MR_DEPENDENCE_CLIQUE (old));
1118	MR_DEPENDENCE_BASE (*tp) = MR_DEPENDENCE_BASE (old);
1119	}
1120	/* We cannot propagate the TREE_THIS_NOTRAP flag if we have
1121	remapped a parameter as the property might be valid only
1122	for the parameter itself. */
1123	if (TREE_THIS_NOTRAP (old)
1124	&& (!is_parm (TREE_OPERAND (old, 0))
1125	|| (!id->transform_parameter && is_parm (decl: ptr))))
1126	TREE_THIS_NOTRAP (*tp) = 1;
1127	REF_REVERSE_STORAGE_ORDER (*tp) = REF_REVERSE_STORAGE_ORDER (old);
1128	*walk_subtrees = 0;
1129	return NULL;
1130	}
1131
1132	/* Here is the "usual case". Copy this tree node, and then
1133	tweak some special cases. */
1134	copy_tree_r (tp, walk_subtrees, NULL);
1135
1136	if (TREE_CODE (*tp) != OMP_CLAUSE)
1137	TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id);
1138
1139	if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3))
1140	{
1141	/* The copied TARGET_EXPR has never been expanded, even if the
1142	original node was expanded already. */
1143	TREE_OPERAND (*tp, 1) = TREE_OPERAND (*tp, 3);
1144	TREE_OPERAND (*tp, 3) = NULL_TREE;
1145	}
1146	else if (TREE_CODE (*tp) == ADDR_EXPR)
1147	{
1148	/* Variable substitution need not be simple. In particular,
1149	the MEM_REF substitution above. Make sure that
1150	TREE_CONSTANT and friends are up-to-date. */
1151	int invariant = is_gimple_min_invariant (*tp);
1152	walk_tree (&TREE_OPERAND (*tp, 0), remap_gimple_op_r, data, NULL);
1153	recompute_tree_invariant_for_addr_expr (*tp);
1154
1155	/* If this used to be invariant, but is not any longer,
1156	then regimplification is probably needed. */
1157	if (invariant && !is_gimple_min_invariant (*tp))
1158	id->regimplify = true;
1159
1160	*walk_subtrees = 0;
1161	}
1162	}
1163
1164	/* Update the TREE_BLOCK for the cloned expr. */
1165	if (EXPR_P (*tp))
1166	{
1167	tree new_block = id->remapping_type_depth == 0 ? id->block : NULL;
1168	tree old_block = TREE_BLOCK (*tp);
1169	if (old_block)
1170	{
1171	tree *n;
1172	n = id->decl_map->get (TREE_BLOCK (*tp));
1173	if (n)
1174	new_block = *n;
1175	}
1176	TREE_SET_BLOCK (*tp, new_block);
1177	}
1178
1179	/* Keep iterating. */
1180	return NULL_TREE;
1181	}
1182
1183
1184	/* Called from copy_body_id via walk_tree. DATA is really a
1185	`copy_body_data *'. */
1186
1187	tree
1188	copy_tree_body_r (tree *tp, int *walk_subtrees, void *data)
1189	{
1190	copy_body_data *id = (copy_body_data *) data;
1191	tree fn = id->src_fn;
1192	tree new_block;
1193
1194	/* Begin by recognizing trees that we'll completely rewrite for the
1195	inlining context. Our output for these trees is completely
1196	different from out input (e.g. RETURN_EXPR is deleted, and morphs
1197	into an edge). Further down, we'll handle trees that get
1198	duplicated and/or tweaked. */
1199
1200	/* When requested, RETURN_EXPRs should be transformed to just the
1201	contained MODIFY_EXPR. The branch semantics of the return will
1202	be handled elsewhere by manipulating the CFG rather than a statement. */
1203	if (TREE_CODE (*tp) == RETURN_EXPR && id->transform_return_to_modify)
1204	{
1205	tree assignment = TREE_OPERAND (*tp, 0);
1206
1207	/* If we're returning something, just turn that into an
1208	assignment into the equivalent of the original RESULT_DECL.
1209	If the "assignment" is just the result decl, the result
1210	decl has already been set (e.g. a recent "foo (&result_decl,
1211	...)"); just toss the entire RETURN_EXPR. */
1212	if (assignment && TREE_CODE (assignment) == MODIFY_EXPR)
1213	{
1214	/* Replace the RETURN_EXPR with (a copy of) the
1215	MODIFY_EXPR hanging underneath. */
1216	*tp = copy_node (assignment);
1217	}
1218	else /* Else the RETURN_EXPR returns no value. */
1219	{
1220	*tp = NULL;
1221	return (tree) (void *)1;
1222	}
1223	}
1224	else if (TREE_CODE (*tp) == SSA_NAME)
1225	{
1226	*tp = remap_ssa_name (name: *tp, id);
1227	*walk_subtrees = 0;
1228	return NULL;
1229	}
1230
1231	/* Local variables and labels need to be replaced by equivalent
1232	variables. We don't want to copy static variables; there's only
1233	one of those, no matter how many times we inline the containing
1234	function. Similarly for globals from an outer function. */
1235	else if (auto_var_in_fn_p (*tp, fn))
1236	{
1237	tree new_decl;
1238
1239	/* Remap the declaration. */
1240	new_decl = remap_decl (decl: *tp, id);
1241	gcc_assert (new_decl);
1242	/* Replace this variable with the copy. */
1243	STRIP_TYPE_NOPS (new_decl);
1244	*tp = new_decl;
1245	*walk_subtrees = 0;
1246	}
1247	else if (TREE_CODE (*tp) == STATEMENT_LIST)
1248	copy_statement_list (tp);
1249	else if (TREE_CODE (*tp) == SAVE_EXPR
1250	|| TREE_CODE (*tp) == TARGET_EXPR)
1251	remap_save_expr (tp, id->decl_map, walk_subtrees);
1252	else if (TREE_CODE (*tp) == LABEL_DECL
1253	&& (! DECL_CONTEXT (*tp)
1254	|| decl_function_context (*tp) == id->src_fn))
1255	/* These may need to be remapped for EH handling. */
1256	*tp = remap_decl (decl: *tp, id);
1257	else if (TREE_CODE (*tp) == BIND_EXPR)
1258	copy_bind_expr (tp, walk_subtrees, id);
1259	/* Types may need remapping as well. */
1260	else if (TYPE_P (*tp))
1261	*tp = remap_type (type: *tp, id);
1262
1263	/* If this is a constant, we have to copy the node iff the type will be
1264	remapped. copy_tree_r will not copy a constant. */
1265	else if (CONSTANT_CLASS_P (*tp))
1266	{
1267	tree new_type = remap_type (TREE_TYPE (*tp), id);
1268
1269	if (new_type == TREE_TYPE (*tp))
1270	*walk_subtrees = 0;
1271
1272	else if (TREE_CODE (*tp) == INTEGER_CST)
1273	*tp = wide_int_to_tree (type: new_type, cst: wi::to_wide (t: *tp));
1274	else
1275	{
1276	*tp = copy_node (*tp);
1277	TREE_TYPE (*tp) = new_type;
1278	}
1279	}
1280
1281	/* Otherwise, just copy the node. Note that copy_tree_r already
1282	knows not to copy VAR_DECLs, etc., so this is safe. */
1283	else
1284	{
1285	/* Here we handle trees that are not completely rewritten.
1286	First we detect some inlining-induced bogosities for
1287	discarding. */
1288	if (TREE_CODE (*tp) == MODIFY_EXPR
1289	&& TREE_OPERAND (*tp, 0) == TREE_OPERAND (*tp, 1)
1290	&& (auto_var_in_fn_p (TREE_OPERAND (*tp, 0), fn)))
1291	{
1292	/* Some assignments VAR = VAR; don't generate any rtl code
1293	and thus don't count as variable modification. Avoid
1294	keeping bogosities like 0 = 0. */
1295	tree decl = TREE_OPERAND (*tp, 0), value;
1296	tree *n;
1297
1298	n = id->decl_map->get (k: decl);
1299	if (n)
1300	{
1301	value = *n;
1302	STRIP_TYPE_NOPS (value);
1303	if (TREE_CONSTANT (value) || TREE_READONLY (value))
1304	{
1305	*tp = build_empty_stmt (EXPR_LOCATION (*tp));
1306	return copy_tree_body_r (tp, walk_subtrees, data);
1307	}
1308	}
1309	}
1310	else if (INDIRECT_REF_P (*tp))
1311	{
1312	/* Get rid of *& from inline substitutions that can happen when a
1313	pointer argument is an ADDR_EXPR. */
1314	tree decl = TREE_OPERAND (*tp, 0);
1315	tree *n = id->decl_map->get (k: decl);
1316	if (n)
1317	{
1318	/* If we happen to get an ADDR_EXPR in n->value, strip
1319	it manually here as we'll eventually get ADDR_EXPRs
1320	which lie about their types pointed to. In this case
1321	build_fold_indirect_ref wouldn't strip the INDIRECT_REF,
1322	but we absolutely rely on that. As fold_indirect_ref
1323	does other useful transformations, try that first, though. */
1324	tree type = TREE_TYPE (*tp);
1325	tree ptr = id->do_not_unshare ? *n : unshare_expr (*n);
1326	tree old = *tp;
1327	*tp = id->do_not_fold ? NULL : gimple_fold_indirect_ref (ptr);
1328	if (! *tp)
1329	{
1330	type = remap_type (type, id);
1331	if (TREE_CODE (ptr) == ADDR_EXPR && !id->do_not_fold)
1332	{
1333	*tp
1334	= fold_indirect_ref_1 (EXPR_LOCATION (ptr), type, ptr);
1335	/* ??? We should either assert here or build
1336	a VIEW_CONVERT_EXPR instead of blindly leaking
1337	incompatible types to our IL. */
1338	if (! *tp)
1339	*tp = TREE_OPERAND (ptr, 0);
1340	}
1341	else
1342	{
1343	*tp = build1 (INDIRECT_REF, type, ptr);
1344	TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
1345	TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
1346	TREE_READONLY (*tp) = TREE_READONLY (old);
1347	/* We cannot propagate the TREE_THIS_NOTRAP flag if we
1348	have remapped a parameter as the property might be
1349	valid only for the parameter itself. */
1350	if (TREE_THIS_NOTRAP (old)
1351	&& (!is_parm (TREE_OPERAND (old, 0))
1352	|| (!id->transform_parameter && is_parm (decl: ptr))))
1353	TREE_THIS_NOTRAP (*tp) = 1;
1354	}
1355	}
1356	*walk_subtrees = 0;
1357	return NULL;
1358	}
1359	}
1360	else if (TREE_CODE (*tp) == MEM_REF && !id->do_not_fold)
1361	{
1362	/* We need to re-canonicalize MEM_REFs from inline substitutions
1363	that can happen when a pointer argument is an ADDR_EXPR.
1364	Recurse here manually to allow that. */
1365	tree ptr = TREE_OPERAND (*tp, 0);
1366	tree type = remap_type (TREE_TYPE (*tp), id);
1367	tree old = *tp;
1368	walk_tree (&ptr, copy_tree_body_r, data, NULL);
1369	*tp = fold_build2 (MEM_REF, type, ptr, TREE_OPERAND (*tp, 1));
1370	TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
1371	TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
1372	copy_warning (*tp, old);
1373	if (MR_DEPENDENCE_CLIQUE (old) != 0)
1374	{
1375	MR_DEPENDENCE_CLIQUE (*tp)
1376	= remap_dependence_clique (id, MR_DEPENDENCE_CLIQUE (old));
1377	MR_DEPENDENCE_BASE (*tp) = MR_DEPENDENCE_BASE (old);
1378	}
1379	/* We cannot propagate the TREE_THIS_NOTRAP flag if we have
1380	remapped a parameter as the property might be valid only
1381	for the parameter itself. */
1382	if (TREE_THIS_NOTRAP (old)
1383	&& (!is_parm (TREE_OPERAND (old, 0))
1384	|| (!id->transform_parameter && is_parm (decl: ptr))))
1385	TREE_THIS_NOTRAP (*tp) = 1;
1386	REF_REVERSE_STORAGE_ORDER (*tp) = REF_REVERSE_STORAGE_ORDER (old);
1387	*walk_subtrees = 0;
1388	return NULL;
1389	}
1390
1391	/* Here is the "usual case". Copy this tree node, and then
1392	tweak some special cases. */
1393	copy_tree_r (tp, walk_subtrees, NULL);
1394
1395	/* If EXPR has block defined, map it to newly constructed block.
1396	When inlining we want EXPRs without block appear in the block
1397	of function call if we are not remapping a type. */
1398	if (EXPR_P (*tp))
1399	{
1400	new_block = id->remapping_type_depth == 0 ? id->block : NULL;
1401	if (TREE_BLOCK (*tp))
1402	{
1403	tree *n;
1404	n = id->decl_map->get (TREE_BLOCK (*tp));
1405	if (n)
1406	new_block = *n;
1407	}
1408	TREE_SET_BLOCK (*tp, new_block);
1409	}
1410
1411	if (TREE_CODE (*tp) != OMP_CLAUSE)
1412	TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id);
1413
1414	/* The copied TARGET_EXPR has never been expanded, even if the
1415	original node was expanded already. */
1416	if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3))
1417	{
1418	TREE_OPERAND (*tp, 1) = TREE_OPERAND (*tp, 3);
1419	TREE_OPERAND (*tp, 3) = NULL_TREE;
1420	}
1421
1422	/* Variable substitution need not be simple. In particular, the
1423	INDIRECT_REF substitution above. Make sure that TREE_CONSTANT
1424	and friends are up-to-date. */
1425	else if (TREE_CODE (*tp) == ADDR_EXPR)
1426	{
1427	int invariant = is_gimple_min_invariant (*tp);
1428	walk_tree (&TREE_OPERAND (*tp, 0), copy_tree_body_r, id, NULL);
1429
1430	/* Handle the case where we substituted an INDIRECT_REF
1431	into the operand of the ADDR_EXPR. */
1432	if (INDIRECT_REF_P (TREE_OPERAND (*tp, 0))
1433	&& !id->do_not_fold)
1434	{
1435	tree t = TREE_OPERAND (TREE_OPERAND (*tp, 0), 0);
1436	if (TREE_TYPE (t) != TREE_TYPE (*tp))
1437	t = fold_convert (remap_type (TREE_TYPE (*tp), id), t);
1438	*tp = t;
1439	}
1440	else
1441	recompute_tree_invariant_for_addr_expr (*tp);
1442
1443	/* If this used to be invariant, but is not any longer,
1444	then regimplification is probably needed. */
1445	if (invariant && !is_gimple_min_invariant (*tp))
1446	id->regimplify = true;
1447
1448	*walk_subtrees = 0;
1449	}
1450	else if (TREE_CODE (*tp) == OMP_CLAUSE
1451	&& (OMP_CLAUSE_CODE (*tp) == OMP_CLAUSE_AFFINITY
1452	|| OMP_CLAUSE_CODE (*tp) == OMP_CLAUSE_DEPEND))
1453	{
1454	tree t = OMP_CLAUSE_DECL (*tp);
1455	if (t
1456	&& TREE_CODE (t) == TREE_LIST
1457	&& TREE_PURPOSE (t)
1458	&& TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
1459	{
1460	*walk_subtrees = 0;
1461	OMP_CLAUSE_DECL (*tp) = copy_node (t);
1462	t = OMP_CLAUSE_DECL (*tp);
1463	TREE_PURPOSE (t) = copy_node (TREE_PURPOSE (t));
1464	for (int i = 0; i <= 4; i++)
1465	walk_tree (&TREE_VEC_ELT (TREE_PURPOSE (t), i),
1466	copy_tree_body_r, id, NULL);
1467	if (TREE_VEC_ELT (TREE_PURPOSE (t), 5))
1468	remap_block (block: &TREE_VEC_ELT (TREE_PURPOSE (t), 5), id);
1469	walk_tree (&TREE_VALUE (t), copy_tree_body_r, id, NULL);
1470	}
1471	}
1472	}
1473
1474	/* Keep iterating. */
1475	return NULL_TREE;
1476	}
1477
1478	/* Helper for remap_gimple_stmt. Given an EH region number for the
1479	source function, map that to the duplicate EH region number in
1480	the destination function. */
1481
1482	static int
1483	remap_eh_region_nr (int old_nr, copy_body_data *id)
1484	{
1485	eh_region old_r, new_r;
1486
1487	old_r = get_eh_region_from_number_fn (id->src_cfun, old_nr);
1488	new_r = static_cast<eh_region> (*id->eh_map->get (k: old_r));
1489
1490	return new_r->index;
1491	}
1492
1493	/* Similar, but operate on INTEGER_CSTs. */
1494
1495	static tree
1496	remap_eh_region_tree_nr (tree old_t_nr, copy_body_data *id)
1497	{
1498	int old_nr, new_nr;
1499
1500	old_nr = tree_to_shwi (old_t_nr);
1501	new_nr = remap_eh_region_nr (old_nr, id);
1502
1503	return build_int_cst (integer_type_node, new_nr);
1504	}
1505
1506	/* Helper for copy_bb. Remap statement STMT using the inlining
1507	information in ID. Return the new statement copy. */
1508
1509	static gimple_seq
1510	remap_gimple_stmt (gimple *stmt, copy_body_data *id)
1511	{
1512	gimple *copy = NULL;
1513	struct walk_stmt_info wi;
1514	bool skip_first = false;
1515	gimple_seq stmts = NULL;
1516
1517	if (is_gimple_debug (gs: stmt)
1518	&& (gimple_debug_nonbind_marker_p (s: stmt)
1519	? !DECL_STRUCT_FUNCTION (id->dst_fn)->debug_nonbind_markers
1520	: !opt_for_fn (id->dst_fn, flag_var_tracking_assignments)))
1521	return NULL;
1522
1523	if (!is_gimple_debug (gs: stmt)
1524	&& id->param_body_adjs
1525	&& id->param_body_adjs->m_dead_stmts.contains (k: stmt))
1526	{
1527	tree *dval = id->param_body_adjs->m_dead_stmt_debug_equiv.get (k: stmt);
1528	if (!dval)
1529	return NULL;
1530
1531	gcc_assert (is_gimple_assign (stmt));
1532	tree lhs = gimple_assign_lhs (gs: stmt);
1533	tree *dvar = id->param_body_adjs->m_dead_ssa_debug_equiv.get (k: lhs);
1534	gdebug *bind = gimple_build_debug_bind (*dvar, *dval, stmt);
1535	if (id->reset_location)
1536	gimple_set_location (g: bind, location: input_location);
1537	id->debug_stmts.safe_push (obj: bind);
1538	gimple_seq_add_stmt_without_update (&stmts, bind);
1539	return stmts;
1540	}
1541
1542	/* Begin by recognizing trees that we'll completely rewrite for the
1543	inlining context. Our output for these trees is completely
1544	different from our input (e.g. RETURN_EXPR is deleted and morphs
1545	into an edge). Further down, we'll handle trees that get
1546	duplicated and/or tweaked. */
1547
1548	/* When requested, GIMPLE_RETURN should be transformed to just the
1549	contained GIMPLE_ASSIGN. The branch semantics of the return will
1550	be handled elsewhere by manipulating the CFG rather than the
1551	statement. */
1552	if (gimple_code (g: stmt) == GIMPLE_RETURN && id->transform_return_to_modify)
1553	{
1554	tree retval = gimple_return_retval (gs: as_a <greturn *> (p: stmt));
1555
1556	/* If we're returning something, just turn that into an
1557	assignment to the equivalent of the original RESULT_DECL.
1558	If RETVAL is just the result decl, the result decl has
1559	already been set (e.g. a recent "foo (&result_decl, ...)");
1560	just toss the entire GIMPLE_RETURN. Likewise for when the
1561	call doesn't want the return value. */
1562	if (retval
1563	&& (TREE_CODE (retval) != RESULT_DECL
1564	&& (!id->call_stmt
1565	|| gimple_call_lhs (gs: id->call_stmt) != NULL_TREE)
1566	&& (TREE_CODE (retval) != SSA_NAME
1567	|| ! SSA_NAME_VAR (retval)
1568	|| TREE_CODE (SSA_NAME_VAR (retval)) != RESULT_DECL)))
1569	{
1570	copy = gimple_build_assign (id->do_not_unshare
1571	? id->retvar : unshare_expr (id->retvar),
1572	retval);
1573	/* id->retvar is already substituted. Skip it on later remapping. */
1574	skip_first = true;
1575	}
1576	else
1577	return NULL;
1578	}
1579	else if (gimple_has_substatements (g: stmt))
1580	{
1581	gimple_seq s1, s2;
1582
1583	/* When cloning bodies from the C++ front end, we will be handed bodies
1584	in High GIMPLE form. Handle here all the High GIMPLE statements that
1585	have embedded statements. */
1586	switch (gimple_code (g: stmt))
1587	{
1588	case GIMPLE_BIND:
1589	copy = copy_gimple_bind (stmt: as_a <gbind *> (p: stmt), id);
1590	break;
1591
1592	case GIMPLE_CATCH:
1593	{
1594	gcatch *catch_stmt = as_a <gcatch *> (p: stmt);
1595	s1 = remap_gimple_seq (body: gimple_catch_handler (catch_stmt), id);
1596	copy = gimple_build_catch (gimple_catch_types (catch_stmt), s1);
1597	}
1598	break;
1599
1600	case GIMPLE_EH_FILTER:
1601	s1 = remap_gimple_seq (body: gimple_eh_filter_failure (gs: stmt), id);
1602	copy = gimple_build_eh_filter (gimple_eh_filter_types (gs: stmt), s1);
1603	break;
1604
1605	case GIMPLE_TRY:
1606	s1 = remap_gimple_seq (body: gimple_try_eval (gs: stmt), id);
1607	s2 = remap_gimple_seq (body: gimple_try_cleanup (gs: stmt), id);
1608	copy = gimple_build_try (s1, s2, gimple_try_kind (gs: stmt));
1609	break;
1610
1611	case GIMPLE_WITH_CLEANUP_EXPR:
1612	s1 = remap_gimple_seq (body: gimple_wce_cleanup (gs: stmt), id);
1613	copy = gimple_build_wce (s1);
1614	break;
1615
1616	case GIMPLE_OMP_PARALLEL:
1617	{
1618	gomp_parallel *omp_par_stmt = as_a <gomp_parallel *> (p: stmt);
1619	s1 = remap_gimple_seq (body: gimple_omp_body (gs: omp_par_stmt), id);
1620	copy = gimple_build_omp_parallel
1621	(s1,
1622	gimple_omp_parallel_clauses (gs: omp_par_stmt),
1623	gimple_omp_parallel_child_fn (omp_parallel_stmt: omp_par_stmt),
1624	gimple_omp_parallel_data_arg (omp_parallel_stmt: omp_par_stmt));
1625	}
1626	break;
1627
1628	case GIMPLE_OMP_TASK:
1629	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1630	copy = gimple_build_omp_task
1631	(s1,
1632	gimple_omp_task_clauses (gs: stmt),
1633	gimple_omp_task_child_fn (gs: stmt),
1634	gimple_omp_task_data_arg (gs: stmt),
1635	gimple_omp_task_copy_fn (gs: stmt),
1636	gimple_omp_task_arg_size (gs: stmt),
1637	gimple_omp_task_arg_align (gs: stmt));
1638	break;
1639
1640	case GIMPLE_OMP_FOR:
1641	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1642	s2 = remap_gimple_seq (body: gimple_omp_for_pre_body (gs: stmt), id);
1643	copy = gimple_build_omp_for (s1, gimple_omp_for_kind (g: stmt),
1644	gimple_omp_for_clauses (gs: stmt),
1645	gimple_omp_for_collapse (gs: stmt), s2);
1646	{
1647	size_t i;
1648	for (i = 0; i < gimple_omp_for_collapse (gs: stmt); i++)
1649	{
1650	gimple_omp_for_set_index (gs: copy, i,
1651	index: gimple_omp_for_index (gs: stmt, i));
1652	gimple_omp_for_set_initial (gs: copy, i,
1653	initial: gimple_omp_for_initial (gs: stmt, i));
1654	gimple_omp_for_set_final (gs: copy, i,
1655	final: gimple_omp_for_final (gs: stmt, i));
1656	gimple_omp_for_set_incr (gs: copy, i,
1657	incr: gimple_omp_for_incr (gs: stmt, i));
1658	gimple_omp_for_set_cond (gs: copy, i,
1659	cond: gimple_omp_for_cond (gs: stmt, i));
1660	}
1661	}
1662	break;
1663
1664	case GIMPLE_OMP_MASTER:
1665	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1666	copy = gimple_build_omp_master (s1);
1667	break;
1668
1669	case GIMPLE_OMP_MASKED:
1670	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1671	copy = gimple_build_omp_masked
1672	(s1, gimple_omp_masked_clauses (gs: stmt));
1673	break;
1674
1675	case GIMPLE_OMP_SCOPE:
1676	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1677	copy = gimple_build_omp_scope
1678	(s1, gimple_omp_scope_clauses (gs: stmt));
1679	break;
1680
1681	case GIMPLE_OMP_TASKGROUP:
1682	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1683	copy = gimple_build_omp_taskgroup
1684	(s1, gimple_omp_taskgroup_clauses (gs: stmt));
1685	break;
1686
1687	case GIMPLE_OMP_ORDERED:
1688	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1689	copy = gimple_build_omp_ordered
1690	(s1,
1691	gimple_omp_ordered_clauses (ord_stmt: as_a <gomp_ordered *> (p: stmt)));
1692	break;
1693
1694	case GIMPLE_OMP_SCAN:
1695	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1696	copy = gimple_build_omp_scan
1697	(s1, gimple_omp_scan_clauses (scan_stmt: as_a <gomp_scan *> (p: stmt)));
1698	break;
1699
1700	case GIMPLE_OMP_SECTION:
1701	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1702	copy = gimple_build_omp_section (s1);
1703	break;
1704
1705	case GIMPLE_OMP_SECTIONS:
1706	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1707	copy = gimple_build_omp_sections
1708	(s1, gimple_omp_sections_clauses (gs: stmt));
1709	break;
1710
1711	case GIMPLE_OMP_STRUCTURED_BLOCK:
1712	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1713	copy = gimple_build_omp_structured_block (s1);
1714	break;
1715
1716	case GIMPLE_OMP_SINGLE:
1717	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1718	copy = gimple_build_omp_single
1719	(s1, gimple_omp_single_clauses (gs: stmt));
1720	break;
1721
1722	case GIMPLE_OMP_TARGET:
1723	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1724	copy = gimple_build_omp_target
1725	(s1, gimple_omp_target_kind (g: stmt),
1726	gimple_omp_target_clauses (gs: stmt));
1727	break;
1728
1729	case GIMPLE_OMP_TEAMS:
1730	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1731	copy = gimple_build_omp_teams
1732	(s1, gimple_omp_teams_clauses (gs: stmt));
1733	break;
1734
1735	case GIMPLE_OMP_CRITICAL:
1736	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1737	copy = gimple_build_omp_critical (s1,
1738	gimple_omp_critical_name
1739	(crit_stmt: as_a <gomp_critical *> (p: stmt)),
1740	gimple_omp_critical_clauses
1741	(crit_stmt: as_a <gomp_critical *> (p: stmt)));
1742	break;
1743
1744	case GIMPLE_ASSUME:
1745	s1 = remap_gimple_seq (body: gimple_assume_body (gs: stmt), id);
1746	copy = gimple_build_assume (gimple_assume_guard (gs: stmt), s1);
1747	break;
1748
1749	case GIMPLE_TRANSACTION:
1750	{
1751	gtransaction *old_trans_stmt = as_a <gtransaction *> (p: stmt);
1752	gtransaction *new_trans_stmt;
1753	s1 = remap_gimple_seq (body: gimple_transaction_body (transaction_stmt: old_trans_stmt),
1754	id);
1755	copy = new_trans_stmt = gimple_build_transaction (s1);
1756	gimple_transaction_set_subcode (transaction_stmt: new_trans_stmt,
1757	subcode: gimple_transaction_subcode (transaction_stmt: old_trans_stmt));
1758	gimple_transaction_set_label_norm (transaction_stmt: new_trans_stmt,
1759	label: gimple_transaction_label_norm (transaction_stmt: old_trans_stmt));
1760	gimple_transaction_set_label_uninst (transaction_stmt: new_trans_stmt,
1761	label: gimple_transaction_label_uninst (transaction_stmt: old_trans_stmt));
1762	gimple_transaction_set_label_over (transaction_stmt: new_trans_stmt,
1763	label: gimple_transaction_label_over (transaction_stmt: old_trans_stmt));
1764	}
1765	break;
1766
1767	default:
1768	gcc_unreachable ();
1769	}
1770	}
1771	else
1772	{
1773	if (gimple_assign_single_p (gs: stmt)
1774	&& gimple_assign_lhs (gs: stmt) == gimple_assign_rhs1 (gs: stmt)
1775	&& auto_var_in_fn_p (gimple_assign_lhs (gs: stmt), id->src_fn))
1776	{
1777	/* Here we handle statements that are not completely rewritten.
1778	First we detect some inlining-induced bogosities for
1779	discarding. */
1780
1781	/* Some assignments VAR = VAR; don't generate any rtl code
1782	and thus don't count as variable modification. Avoid
1783	keeping bogosities like 0 = 0. */
1784	tree decl = gimple_assign_lhs (gs: stmt), value;
1785	tree *n;
1786
1787	n = id->decl_map->get (k: decl);
1788	if (n)
1789	{
1790	value = *n;
1791	STRIP_TYPE_NOPS (value);
1792	if (TREE_CONSTANT (value) || TREE_READONLY (value))
1793	return NULL;
1794	}
1795	}
1796
1797	/* For *ptr_N ={v} {CLOBBER}, if ptr_N is SSA_NAME defined
1798	in a block that we aren't copying during tree_function_versioning,
1799	just drop the clobber stmt. */
1800	if (id->blocks_to_copy && gimple_clobber_p (s: stmt))
1801	{
1802	tree lhs = gimple_assign_lhs (gs: stmt);
1803	if (TREE_CODE (lhs) == MEM_REF
1804	&& TREE_CODE (TREE_OPERAND (lhs, 0)) == SSA_NAME)
1805	{
1806	gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (lhs, 0));
1807	if (gimple_bb (g: def_stmt)
1808	&& !bitmap_bit_p (id->blocks_to_copy,
1809	gimple_bb (g: def_stmt)->index))
1810	return NULL;
1811	}
1812	}
1813
1814	/* We do not allow CLOBBERs of handled components. In case
1815	returned value is stored via such handled component, remove
1816	the clobber so stmt verifier is happy. */
1817	if (gimple_clobber_p (s: stmt)
1818	&& TREE_CODE (gimple_assign_lhs (stmt)) == RESULT_DECL)
1819	{
1820	tree remapped = remap_decl (decl: gimple_assign_lhs (gs: stmt), id);
1821	if (!DECL_P (remapped)
1822	&& TREE_CODE (remapped) != MEM_REF)
1823	return NULL;
1824	}
1825
1826	if (gimple_debug_bind_p (s: stmt))
1827	{
1828	tree var = gimple_debug_bind_get_var (dbg: stmt);
1829	tree value = gimple_debug_bind_get_value (dbg: stmt);
1830	if (id->param_body_adjs
1831	&& id->param_body_adjs->m_dead_stmts.contains (k: stmt))
1832	{
1833	value = unshare_expr_without_location (value);
1834	id->param_body_adjs->remap_with_debug_expressions (t: &value);
1835	}
1836
1837	gdebug *copy = gimple_build_debug_bind (var, value, stmt);
1838	if (id->reset_location)
1839	gimple_set_location (g: copy, location: input_location);
1840	id->debug_stmts.safe_push (obj: copy);
1841	gimple_seq_add_stmt_without_update (&stmts, copy);
1842	return stmts;
1843	}
1844	if (gimple_debug_source_bind_p (s: stmt))
1845	{
1846	gdebug *copy = gimple_build_debug_source_bind
1847	(gimple_debug_source_bind_get_var (dbg: stmt),
1848	gimple_debug_source_bind_get_value (dbg: stmt),
1849	stmt);
1850	if (id->reset_location)
1851	gimple_set_location (g: copy, location: input_location);
1852	id->debug_stmts.safe_push (obj: copy);
1853	gimple_seq_add_stmt_without_update (&stmts, copy);
1854	return stmts;
1855	}
1856	if (gimple_debug_nonbind_marker_p (s: stmt))
1857	{
1858	/* If the inlined function has too many debug markers,
1859	don't copy them. */
1860	if (id->src_cfun->debug_marker_count
1861	> param_max_debug_marker_count
1862	|| id->reset_location)
1863	return stmts;
1864
1865	gdebug *copy = as_a <gdebug *> (p: gimple_copy (stmt));
1866	id->debug_stmts.safe_push (obj: copy);
1867	gimple_seq_add_stmt_without_update (&stmts, copy);
1868	return stmts;
1869	}
1870
1871	/* Create a new deep copy of the statement. */
1872	copy = gimple_copy (stmt);
1873
1874	/* Clear flags that need revisiting. */
1875	if (gcall *call_stmt = dyn_cast <gcall *> (p: copy))
1876	{
1877	if (gimple_call_tail_p (s: call_stmt))
1878	gimple_call_set_tail (s: call_stmt, tail_p: false);
1879	if (gimple_call_from_thunk_p (s: call_stmt))
1880	gimple_call_set_from_thunk (s: call_stmt, from_thunk_p: false);
1881	if (gimple_call_internal_p (gs: call_stmt))
1882	switch (gimple_call_internal_fn (gs: call_stmt))
1883	{
1884	case IFN_GOMP_SIMD_LANE:
1885	case IFN_GOMP_SIMD_VF:
1886	case IFN_GOMP_SIMD_LAST_LANE:
1887	case IFN_GOMP_SIMD_ORDERED_START:
1888	case IFN_GOMP_SIMD_ORDERED_END:
1889	DECL_STRUCT_FUNCTION (id->dst_fn)->has_simduid_loops = true;
1890	break;
1891	default:
1892	break;
1893	}
1894	}
1895
1896	/* Remap the region numbers for __builtin_eh_{pointer,filter},
1897	RESX and EH_DISPATCH. */
1898	if (id->eh_map)
1899	switch (gimple_code (g: copy))
1900	{
1901	case GIMPLE_CALL:
1902	{
1903	tree r, fndecl = gimple_call_fndecl (gs: copy);
1904	if (fndecl && fndecl_built_in_p (node: fndecl, klass: BUILT_IN_NORMAL))
1905	switch (DECL_FUNCTION_CODE (decl: fndecl))
1906	{
1907	case BUILT_IN_EH_COPY_VALUES:
1908	r = gimple_call_arg (gs: copy, index: 1);
1909	r = remap_eh_region_tree_nr (old_t_nr: r, id);
1910	gimple_call_set_arg (gs: copy, index: 1, arg: r);
1911	/* FALLTHRU */
1912
1913	case BUILT_IN_EH_POINTER:
1914	case BUILT_IN_EH_FILTER:
1915	r = gimple_call_arg (gs: copy, index: 0);
1916	r = remap_eh_region_tree_nr (old_t_nr: r, id);
1917	gimple_call_set_arg (gs: copy, index: 0, arg: r);
1918	break;
1919
1920	default:
1921	break;
1922	}
1923
1924	/* Reset alias info if we didn't apply measures to
1925	keep it valid over inlining by setting DECL_PT_UID. */
1926	if (!id->src_cfun->gimple_df
1927	|| !id->src_cfun->gimple_df->ipa_pta)
1928	gimple_call_reset_alias_info (as_a <gcall *> (p: copy));
1929	}
1930	break;
1931
1932	case GIMPLE_RESX:
1933	{
1934	gresx *resx_stmt = as_a <gresx *> (p: copy);
1935	int r = gimple_resx_region (resx_stmt);
1936	r = remap_eh_region_nr (old_nr: r, id);
1937	gimple_resx_set_region (resx_stmt, region: r);
1938	}
1939	break;
1940
1941	case GIMPLE_EH_DISPATCH:
1942	{
1943	geh_dispatch *eh_dispatch = as_a <geh_dispatch *> (p: copy);
1944	int r = gimple_eh_dispatch_region (eh_dispatch_stmt: eh_dispatch);
1945	r = remap_eh_region_nr (old_nr: r, id);
1946	gimple_eh_dispatch_set_region (eh_dispatch_stmt: eh_dispatch, region: r);
1947	}
1948	break;
1949
1950	default:
1951	break;
1952	}
1953	}
1954
1955	/* If STMT has a block defined, map it to the newly constructed block. */
1956	if (tree block = gimple_block (g: copy))
1957	{
1958	tree *n;
1959	n = id->decl_map->get (k: block);
1960	gcc_assert (n);
1961	gimple_set_block (g: copy, block: *n);
1962	}
1963	if (id->param_body_adjs)
1964	{
1965	gimple_seq extra_stmts = NULL;
1966	id->param_body_adjs->modify_gimple_stmt (stmt: &copy, extra_stmts: &extra_stmts, orig_stmt: stmt);
1967	if (!gimple_seq_empty_p (s: extra_stmts))
1968	{
1969	memset (s: &wi, c: 0, n: sizeof (wi));
1970	wi.info = id;
1971	for (gimple_stmt_iterator egsi = gsi_start (seq&: extra_stmts);
1972	!gsi_end_p (i: egsi);
1973	gsi_next (i: &egsi))
1974	walk_gimple_op (gsi_stmt (i: egsi), remap_gimple_op_r, &wi);
1975	gimple_seq_add_seq_without_update (&stmts, extra_stmts);
1976	}
1977	}
1978
1979	if (id->reset_location)
1980	gimple_set_location (g: copy, location: input_location);
1981
1982	/* Debug statements ought to be rebuilt and not copied. */
1983	gcc_checking_assert (!is_gimple_debug (copy));
1984
1985	/* Remap all the operands in COPY. */
1986	memset (s: &wi, c: 0, n: sizeof (wi));
1987	wi.info = id;
1988	if (skip_first)
1989	walk_tree (gimple_op_ptr (copy, 1), remap_gimple_op_r, &wi, NULL);
1990	else
1991	walk_gimple_op (copy, remap_gimple_op_r, &wi);
1992
1993	/* Clear the copied virtual operands. We are not remapping them here
1994	but are going to recreate them from scratch. */
1995	if (gimple_has_mem_ops (g: copy))
1996	{
1997	gimple_set_vdef (g: copy, NULL_TREE);
1998	gimple_set_vuse (g: copy, NULL_TREE);
1999	}
2000
2001	if (cfun->can_throw_non_call_exceptions)
2002	{
2003	/* When inlining a function which does not have non-call exceptions
2004	enabled into a function that has (which only happens with
2005	always-inline) we have to fixup stmts that cannot throw. */
2006	if (gcond *cond = dyn_cast <gcond *> (p: copy))
2007	if (gimple_could_trap_p (cond))
2008	{
2009	gassign *cmp
2010	= gimple_build_assign (make_ssa_name (boolean_type_node),
2011	gimple_cond_code (gs: cond),
2012	gimple_cond_lhs (gs: cond),
2013	gimple_cond_rhs (gs: cond));
2014	gimple_seq_add_stmt_without_update (&stmts, cmp);
2015	gimple_cond_set_code (gs: cond, code: NE_EXPR);
2016	gimple_cond_set_lhs (gs: cond, lhs: gimple_assign_lhs (gs: cmp));
2017	gimple_cond_set_rhs (gs: cond, boolean_false_node);
2018	}
2019	}
2020
2021	gimple_seq_add_stmt_without_update (&stmts, copy);
2022	return stmts;
2023	}
2024
2025
2026	/* Copy basic block, scale profile accordingly. Edges will be taken care of
2027	later */
2028
2029	static basic_block
2030	copy_bb (copy_body_data *id, basic_block bb,
2031	profile_count num, profile_count den)
2032	{
2033	gimple_stmt_iterator gsi, copy_gsi, seq_gsi;
2034	basic_block copy_basic_block;
2035	tree decl;
2036	basic_block prev;
2037
2038	profile_count::adjust_for_ipa_scaling (num: &num, den: &den);
2039
2040	/* Search for previous copied basic block. */
2041	prev = bb->prev_bb;
2042	while (!prev->aux)
2043	prev = prev->prev_bb;
2044
2045	/* create_basic_block() will append every new block to
2046	basic_block_info automatically. */
2047	copy_basic_block = create_basic_block (NULL, (basic_block) prev->aux);
2048	copy_basic_block->count = bb->count.apply_scale (num, den);
2049
2050	copy_gsi = gsi_start_bb (bb: copy_basic_block);
2051
2052	unsigned min_cond_uid = 0;
2053	if (id->src_cfun->cond_uids)
2054	{
2055	if (!cfun->cond_uids)
2056	cfun->cond_uids = new hash_map <gcond*, unsigned> ();
2057
2058	for (auto itr : *id->src_cfun->cond_uids)
2059	if (itr.second >= min_cond_uid)
2060	min_cond_uid = itr.second + 1;
2061	}
2062
2063	for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
2064	{
2065	gimple_seq stmts;
2066	gimple *stmt = gsi_stmt (i: gsi);
2067	gimple *orig_stmt = stmt;
2068	gimple_stmt_iterator stmts_gsi;
2069	bool stmt_added = false;
2070
2071	id->regimplify = false;
2072	stmts = remap_gimple_stmt (stmt, id);
2073
2074	if (gimple_seq_empty_p (s: stmts))
2075	continue;
2076
2077	seq_gsi = copy_gsi;
2078
2079	for (stmts_gsi = gsi_start (seq&: stmts);
2080	!gsi_end_p (i: stmts_gsi); )
2081	{
2082	stmt = gsi_stmt (i: stmts_gsi);
2083
2084	/* Advance iterator now before stmt is moved to seq_gsi. */
2085	gsi_next (i: &stmts_gsi);
2086
2087	if (gimple_nop_p (g: stmt))
2088	continue;
2089
2090	/* If -fcondition-coverage is used, register the inlined conditions
2091	in the cond->expression mapping of the caller. The expression tag
2092	is shifted conditions from the two bodies are not mixed. */
2093	if (id->src_cfun->cond_uids && is_a <gcond*> (p: stmt))
2094	{
2095	gcond *orig_cond = as_a <gcond*> (p: orig_stmt);
2096	gcond *cond = as_a <gcond*> (p: stmt);
2097	unsigned *v = id->src_cfun->cond_uids->get (k: orig_cond);
2098	if (v)
2099	cfun->cond_uids->put (k: cond, v: *v + min_cond_uid);
2100	}
2101
2102	gimple_duplicate_stmt_histograms (cfun, stmt, id->src_cfun,
2103	orig_stmt);
2104
2105	gsi_insert_after (&seq_gsi, stmt, GSI_NEW_STMT);
2106
2107	if (id->regimplify)
2108	gimple_regimplify_operands (stmt, &seq_gsi);
2109
2110	stmt_added = true;
2111	}
2112
2113	if (!stmt_added)
2114	continue;
2115
2116	/* If copy_basic_block has been empty at the start of this iteration,
2117	call gsi_start_bb again to get at the newly added statements. */
2118	if (gsi_end_p (i: copy_gsi))
2119	copy_gsi = gsi_start_bb (bb: copy_basic_block);
2120	else
2121	gsi_next (i: &copy_gsi);
2122
2123	/* Process the new statement. The call to gimple_regimplify_operands
2124	possibly turned the statement into multiple statements, we
2125	need to process all of them. */
2126	do
2127	{
2128	tree fn;
2129	gcall *call_stmt;
2130
2131	stmt = gsi_stmt (i: copy_gsi);
2132	call_stmt = dyn_cast <gcall *> (p: stmt);
2133	if (call_stmt
2134	&& gimple_call_va_arg_pack_p (s: call_stmt)
2135	&& id->call_stmt
2136	&& ! gimple_call_va_arg_pack_p (s: id->call_stmt))
2137	{
2138	/* __builtin_va_arg_pack () should be replaced by
2139	all arguments corresponding to ... in the caller. */
2140	tree p;
2141	gcall *new_call;
2142	vec<tree> argarray;
2143	size_t nargs_caller = gimple_call_num_args (gs: id->call_stmt);
2144	size_t nargs = nargs_caller;
2145
2146	for (p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
2147	{
2148	/* Avoid crashing on invalid IL that doesn't have a
2149	varargs function or that passes not enough arguments. */
2150	if (nargs == 0)
2151	break;
2152	nargs--;
2153	}
2154
2155	/* Create the new array of arguments. */
2156	size_t nargs_callee = gimple_call_num_args (gs: call_stmt);
2157	size_t n = nargs + nargs_callee;
2158	argarray.create (nelems: n);
2159	argarray.safe_grow_cleared (len: n, exact: true);
2160
2161	/* Copy all the arguments before '...' */
2162	if (nargs_callee)
2163	memcpy (dest: argarray.address (),
2164	src: gimple_call_arg_ptr (gs: call_stmt, index: 0),
2165	n: nargs_callee * sizeof (tree));
2166
2167	/* Append the arguments passed in '...' */
2168	if (nargs)
2169	memcpy (dest: argarray.address () + nargs_callee,
2170	src: gimple_call_arg_ptr (gs: id->call_stmt, index: 0)
2171	+ (nargs_caller - nargs), n: nargs * sizeof (tree));
2172
2173	new_call = gimple_build_call_vec (gimple_call_fn (gs: call_stmt),
2174	argarray);
2175
2176	argarray.release ();
2177
2178	/* Copy all GIMPLE_CALL flags, location and block, except
2179	GF_CALL_VA_ARG_PACK. */
2180	gimple_call_copy_flags (dest_call: new_call, orig_call: call_stmt);
2181	gimple_call_set_va_arg_pack (s: new_call, pass_arg_pack_p: false);
2182	gimple_call_set_fntype (call_stmt: new_call, fntype: gimple_call_fntype (gs: call_stmt));
2183	/* location includes block. */
2184	gimple_set_location (g: new_call, location: gimple_location (g: stmt));
2185	gimple_call_set_lhs (gs: new_call, lhs: gimple_call_lhs (gs: call_stmt));
2186
2187	gsi_replace (&copy_gsi, new_call, false);
2188	stmt = new_call;
2189	}
2190	else if (call_stmt
2191	&& id->call_stmt
2192	&& (decl = gimple_call_fndecl (gs: stmt))
2193	&& fndecl_built_in_p (node: decl, name1: BUILT_IN_VA_ARG_PACK_LEN))
2194	{
2195	/* __builtin_va_arg_pack_len () should be replaced by
2196	the number of anonymous arguments. */
2197	size_t nargs = gimple_call_num_args (gs: id->call_stmt);
2198	tree count, p;
2199	gimple *new_stmt;
2200
2201	for (p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
2202	nargs--;
2203
2204	if (!gimple_call_lhs (gs: stmt))
2205	{
2206	/* Drop unused calls. */
2207	gsi_remove (&copy_gsi, false);
2208	continue;
2209	}
2210	else if (!gimple_call_va_arg_pack_p (s: id->call_stmt))
2211	{
2212	count = build_int_cst (integer_type_node, nargs);
2213	new_stmt = gimple_build_assign (gimple_call_lhs (gs: stmt), count);
2214	gsi_replace (&copy_gsi, new_stmt, false);
2215	stmt = new_stmt;
2216	}
2217	else if (nargs != 0)
2218	{
2219	tree newlhs = create_tmp_reg_or_ssa_name (integer_type_node);
2220	count = build_int_cst (integer_type_node, nargs);
2221	new_stmt = gimple_build_assign (gimple_call_lhs (gs: stmt),
2222	PLUS_EXPR, newlhs, count);
2223	gimple_call_set_lhs (gs: stmt, lhs: newlhs);
2224	gsi_insert_after (&copy_gsi, new_stmt, GSI_NEW_STMT);
2225	}
2226	}
2227	else if (call_stmt
2228	&& id->call_stmt
2229	&& gimple_call_internal_p (gs: stmt)
2230	&& gimple_call_internal_fn (gs: stmt) == IFN_TSAN_FUNC_EXIT)
2231	{
2232	/* Drop TSAN_FUNC_EXIT () internal calls during inlining. */
2233	gsi_remove (&copy_gsi, false);
2234	continue;
2235	}
2236
2237	/* Statements produced by inlining can be unfolded, especially
2238	when we constant propagated some operands. We can't fold
2239	them right now for two reasons:
2240	1) folding require SSA_NAME_DEF_STMTs to be correct
2241	2) we can't change function calls to builtins.
2242	So we just mark statement for later folding. We mark
2243	all new statements, instead just statements that has changed
2244	by some nontrivial substitution so even statements made
2245	foldable indirectly are updated. If this turns out to be
2246	expensive, copy_body can be told to watch for nontrivial
2247	changes. */
2248	if (id->statements_to_fold)
2249	id->statements_to_fold->add (k: stmt);
2250
2251	/* We're duplicating a CALL_EXPR. Find any corresponding
2252	callgraph edges and update or duplicate them. */
2253	if (gcall *call_stmt = dyn_cast <gcall *> (p: stmt))
2254	{
2255	struct cgraph_edge *edge;
2256
2257	switch (id->transform_call_graph_edges)
2258	{
2259	case CB_CGE_DUPLICATE:
2260	edge = id->src_node->get_edge (call_stmt: orig_stmt);
2261	if (edge)
2262	{
2263	struct cgraph_edge *old_edge = edge;
2264
2265	/* A speculative call is consist of multiple
2266	edges - indirect edge and one or more direct edges
2267	Duplicate the whole thing and distribute frequencies
2268	accordingly. */
2269	if (edge->speculative)
2270	{
2271	int n = 0;
2272	profile_count direct_cnt
2273	= profile_count::zero ();
2274
2275	/* First figure out the distribution of counts
2276	so we can re-scale BB profile accordingly. */
2277	for (cgraph_edge *e = old_edge; e;
2278	e = e->next_speculative_call_target ())
2279	direct_cnt = direct_cnt + e->count;
2280
2281	cgraph_edge *indirect
2282	= old_edge->speculative_call_indirect_edge ();
2283	profile_count indir_cnt = indirect->count;
2284
2285	/* Next iterate all direct edges, clone it and its
2286	corresponding reference and update profile. */
2287	for (cgraph_edge *e = old_edge;
2288	e;
2289	e = e->next_speculative_call_target ())
2290	{
2291	profile_count cnt = e->count;
2292
2293	id->dst_node->clone_reference
2294	(ref: e->speculative_call_target_ref (), stmt);
2295	edge = e->clone (n: id->dst_node, call_stmt,
2296	stmt_uid: gimple_uid (g: stmt), num, den,
2297	update_original: true);
2298	profile_probability prob
2299	= cnt.probability_in (overall: direct_cnt
2300	+ indir_cnt);
2301	edge->count
2302	= copy_basic_block->count.apply_probability
2303	(prob);
2304	n++;
2305	}
2306	gcc_checking_assert
2307	(indirect->num_speculative_call_targets_p ()
2308	== n);
2309
2310	/* Duplicate the indirect edge after all direct edges
2311	cloned. */
2312	indirect = indirect->clone (n: id->dst_node, call_stmt,
2313	stmt_uid: gimple_uid (g: stmt),
2314	num, den,
2315	update_original: true);
2316
2317	profile_probability prob
2318	= indir_cnt.probability_in (overall: direct_cnt
2319	+ indir_cnt);
2320	indirect->count
2321	= copy_basic_block->count.apply_probability (prob);
2322	}
2323	else
2324	{
2325	edge = edge->clone (n: id->dst_node, call_stmt,
2326	stmt_uid: gimple_uid (g: stmt),
2327	num, den,
2328	update_original: true);
2329	edge->count = copy_basic_block->count;
2330	}
2331	}
2332	break;
2333
2334	case CB_CGE_MOVE_CLONES:
2335	id->dst_node->set_call_stmt_including_clones (old_stmt: orig_stmt,
2336	new_stmt: call_stmt);
2337	edge = id->dst_node->get_edge (call_stmt: stmt);
2338	break;
2339
2340	case CB_CGE_MOVE:
2341	edge = id->dst_node->get_edge (call_stmt: orig_stmt);
2342	if (edge)
2343	edge = cgraph_edge::set_call_stmt (e: edge, new_stmt: call_stmt);
2344	break;
2345
2346	default:
2347	gcc_unreachable ();
2348	}
2349
2350	/* Constant propagation on argument done during inlining
2351	may create new direct call. Produce an edge for it. */
2352	if ((!edge
2353	|| (edge->indirect_inlining_edge
2354	&& id->transform_call_graph_edges == CB_CGE_MOVE_CLONES))
2355	&& id->dst_node->definition
2356	&& (fn = gimple_call_fndecl (gs: stmt)) != NULL)
2357	{
2358	struct cgraph_node *dest = cgraph_node::get_create (fn);
2359
2360	/* We have missing edge in the callgraph. This can happen
2361	when previous inlining turned an indirect call into a
2362	direct call by constant propagating arguments or we are
2363	producing dead clone (for further cloning). In all
2364	other cases we hit a bug (incorrect node sharing is the
2365	most common reason for missing edges). */
2366	gcc_assert (!dest->definition
2367	|| dest->address_taken
2368	|| !id->src_node->definition
2369	|| !id->dst_node->definition);
2370	if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES)
2371	id->dst_node->create_edge_including_clones
2372	(callee: dest, old_stmt: orig_stmt, stmt: call_stmt, count: bb->count,
2373	reason: CIF_ORIGINALLY_INDIRECT_CALL);
2374	else
2375	id->dst_node->create_edge (callee: dest, call_stmt,
2376	count: bb->count)->inline_failed
2377	= CIF_ORIGINALLY_INDIRECT_CALL;
2378	if (dump_file)
2379	{
2380	fprintf (stream: dump_file, format: "Created new direct edge to %s\n",
2381	dest->dump_name ());
2382	}
2383	}
2384
2385	notice_special_calls (as_a <gcall *> (p: stmt));
2386	}
2387
2388	maybe_duplicate_eh_stmt_fn (cfun, stmt, id->src_cfun, orig_stmt,
2389	id->eh_map, id->eh_lp_nr);
2390
2391	gsi_next (i: &copy_gsi);
2392	}
2393	while (!gsi_end_p (i: copy_gsi));
2394
2395	copy_gsi = gsi_last_bb (bb: copy_basic_block);
2396	}
2397
2398	return copy_basic_block;
2399	}
2400
2401	/* Inserting Single Entry Multiple Exit region in SSA form into code in SSA
2402	form is quite easy, since dominator relationship for old basic blocks does
2403	not change.
2404
2405	There is however exception where inlining might change dominator relation
2406	across EH edges from basic block within inlined functions destinating
2407	to landing pads in function we inline into.
2408
2409	The function fills in PHI_RESULTs of such PHI nodes if they refer
2410	to gimple regs. Otherwise, the function mark PHI_RESULT of such
2411	PHI nodes for renaming. For non-gimple regs, renaming is safe: the
2412	EH edges are abnormal and SSA_NAME_OCCURS_IN_ABNORMAL_PHI must be
2413	set, and this means that there will be no overlapping live ranges
2414	for the underlying symbol.
2415
2416	This might change in future if we allow redirecting of EH edges and
2417	we might want to change way build CFG pre-inlining to include
2418	all the possible edges then. */
2419	static void
2420	update_ssa_across_abnormal_edges (basic_block bb, basic_block ret_bb,
2421	bool can_throw, bool nonlocal_goto)
2422	{
2423	edge e;
2424	edge_iterator ei;
2425
2426	FOR_EACH_EDGE (e, ei, bb->succs)
2427	if (!e->dest->aux
2428	|| ((basic_block)e->dest->aux)->index == ENTRY_BLOCK)
2429	{
2430	gphi *phi;
2431	gphi_iterator si;
2432
2433	if (!nonlocal_goto)
2434	gcc_assert (e->flags & EDGE_EH);
2435
2436	if (!can_throw)
2437	gcc_assert (!(e->flags & EDGE_EH));
2438
2439	for (si = gsi_start_phis (e->dest); !gsi_end_p (i: si); gsi_next (i: &si))
2440	{
2441	edge re;
2442
2443	phi = si.phi ();
2444
2445	/* For abnormal goto/call edges the receiver can be the
2446	ENTRY_BLOCK. Do not assert this cannot happen. */
2447
2448	gcc_assert ((e->flags & EDGE_EH)
2449	|| SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi)));
2450
2451	re = find_edge (ret_bb, e->dest);
2452	gcc_checking_assert (re);
2453	gcc_assert ((re->flags & (EDGE_EH | EDGE_ABNORMAL))
2454	== (e->flags & (EDGE_EH | EDGE_ABNORMAL)));
2455
2456	SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e),
2457	USE_FROM_PTR (PHI_ARG_DEF_PTR_FROM_EDGE (phi, re)));
2458	}
2459	}
2460	}
2461
2462	/* Insert clobbers for automatic variables of inlined ID->src_fn
2463	function at the start of basic block ID->eh_landing_pad_dest. */
2464
2465	static void
2466	add_clobbers_to_eh_landing_pad (copy_body_data *id)
2467	{
2468	tree var;
2469	basic_block bb = id->eh_landing_pad_dest;
2470	live_vars_map *vars = NULL;
2471	unsigned int cnt = 0;
2472	unsigned int i;
2473	FOR_EACH_VEC_SAFE_ELT (id->src_cfun->local_decls, i, var)
2474	if (VAR_P (var)
2475	&& !DECL_HARD_REGISTER (var)
2476	&& !TREE_THIS_VOLATILE (var)
2477	&& !DECL_HAS_VALUE_EXPR_P (var)
2478	&& !is_gimple_reg (var)
2479	&& auto_var_in_fn_p (var, id->src_fn)
2480	&& !lookup_attribute (attr_name: "omp simd array", DECL_ATTRIBUTES (var)))
2481	{
2482	tree *t = id->decl_map->get (k: var);
2483	if (!t)
2484	continue;
2485	tree new_var = *t;
2486	if (VAR_P (new_var)
2487	&& !DECL_HARD_REGISTER (new_var)
2488	&& !TREE_THIS_VOLATILE (new_var)
2489	&& !DECL_HAS_VALUE_EXPR_P (new_var)
2490	&& !is_gimple_reg (new_var)
2491	&& auto_var_in_fn_p (new_var, id->dst_fn))
2492	{
2493	if (vars == NULL)
2494	vars = new live_vars_map;
2495	vars->put (DECL_UID (var), v: cnt++);
2496	}
2497	}
2498	if (vars == NULL)
2499	return;
2500
2501	vec<bitmap_head> live = compute_live_vars (id->src_cfun, vars);
2502	FOR_EACH_VEC_SAFE_ELT (id->src_cfun->local_decls, i, var)
2503	if (VAR_P (var))
2504	{
2505	edge e;
2506	edge_iterator ei;
2507	bool needed = false;
2508	unsigned int *v = vars->get (DECL_UID (var));
2509	if (v == NULL)
2510	continue;
2511	FOR_EACH_EDGE (e, ei, bb->preds)
2512	if ((e->flags & EDGE_EH) != 0
2513	&& e->src->index >= id->add_clobbers_to_eh_landing_pads)
2514	{
2515	basic_block src_bb = (basic_block) e->src->aux;
2516
2517	if (bitmap_bit_p (&live[src_bb->index], *v))
2518	{
2519	needed = true;
2520	break;
2521	}
2522	}
2523	if (needed)
2524	{
2525	tree new_var = *id->decl_map->get (k: var);
2526	gimple_stmt_iterator gsi = gsi_after_labels (bb);
2527	tree clobber = build_clobber (TREE_TYPE (new_var));
2528	gimple *clobber_stmt = gimple_build_assign (new_var, clobber);
2529	gsi_insert_before (&gsi, clobber_stmt, GSI_NEW_STMT);
2530	}
2531	}
2532	destroy_live_vars (live);
2533	delete vars;
2534	}
2535
2536	/* Copy edges from BB into its copy constructed earlier, scale profile
2537	accordingly. Edges will be taken care of later. Assume aux
2538	pointers to point to the copies of each BB. Return true if any
2539	debug stmts are left after a statement that must end the basic block. */
2540
2541	static bool
2542	copy_edges_for_bb (basic_block bb, profile_count num, profile_count den,
2543	basic_block ret_bb, basic_block abnormal_goto_dest,
2544	copy_body_data *id)
2545	{
2546	basic_block new_bb = (basic_block) bb->aux;
2547	edge_iterator ei;
2548	edge old_edge;
2549	gimple_stmt_iterator si;
2550	bool need_debug_cleanup = false;
2551
2552	/* Use the indices from the original blocks to create edges for the
2553	new ones. */
2554	FOR_EACH_EDGE (old_edge, ei, bb->succs)
2555	if (!(old_edge->flags & EDGE_EH))
2556	{
2557	edge new_edge;
2558	int flags = old_edge->flags;
2559	location_t locus = old_edge->goto_locus;
2560
2561	/* Return edges do get a FALLTHRU flag when they get inlined. */
2562	if (old_edge->dest->index == EXIT_BLOCK
2563	&& !(flags & (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE|EDGE_FAKE))
2564	&& old_edge->dest->aux != EXIT_BLOCK_PTR_FOR_FN (cfun))
2565	flags |= EDGE_FALLTHRU;
2566
2567	new_edge
2568	= make_edge (new_bb, (basic_block) old_edge->dest->aux, flags);
2569	new_edge->probability = old_edge->probability;
2570	if (!id->reset_location)
2571	new_edge->goto_locus = remap_location (locus, id);
2572	}
2573
2574	if (bb->index == ENTRY_BLOCK || bb->index == EXIT_BLOCK)
2575	return false;
2576
2577	/* When doing function splitting, we must decrease count of the return block
2578	which was previously reachable by block we did not copy. */
2579	if (single_succ_p (bb) && single_succ_edge (bb)->dest->index == EXIT_BLOCK)
2580	FOR_EACH_EDGE (old_edge, ei, bb->preds)
2581	if (old_edge->src->index != ENTRY_BLOCK
2582	&& !old_edge->src->aux)
2583	new_bb->count -= old_edge->count ().apply_scale (num, den);
2584
2585	/* Walk stmts from end to start so that splitting will adjust the BB
2586	pointer for each stmt at most once, even when we split the block
2587	multiple times. */
2588	bool seen_nondebug = false;
2589	for (si = gsi_last_bb (bb: new_bb); !gsi_end_p (i: si);)
2590	{
2591	bool can_throw, nonlocal_goto;
2592	gimple *copy_stmt = gsi_stmt (i: si);
2593
2594	/* Do this before the possible split_block. */
2595	gsi_prev (i: &si);
2596
2597	/* If this tree could throw an exception, there are two
2598	cases where we need to add abnormal edge(s): the
2599	tree wasn't in a region and there is a "current
2600	region" in the caller; or the original tree had
2601	EH edges. In both cases split the block after the tree,
2602	and add abnormal edge(s) as needed; we need both
2603	those from the callee and the caller.
2604	We check whether the copy can throw, because the const
2605	propagation can change an INDIRECT_REF which throws
2606	into a COMPONENT_REF which doesn't. If the copy
2607	can throw, the original could also throw. */
2608	can_throw = stmt_can_throw_internal (cfun, copy_stmt);
2609	nonlocal_goto
2610	= (stmt_can_make_abnormal_goto (copy_stmt)
2611	&& !computed_goto_p (copy_stmt));
2612
2613	if (can_throw || nonlocal_goto)
2614	{
2615	/* If there's only debug insns after copy_stmt don't split
2616	the block but instead mark the block for cleanup. */
2617	if (!seen_nondebug)
2618	need_debug_cleanup = true;
2619	else
2620	{
2621	/* Note that bb's predecessor edges aren't necessarily
2622	right at this point; split_block doesn't care. */
2623	edge e = split_block (new_bb, copy_stmt);
2624	e->dest->aux = new_bb->aux;
2625	seen_nondebug = false;
2626	}
2627	}
2628
2629	if (!is_gimple_debug (gs: copy_stmt))
2630	seen_nondebug = true;
2631
2632	bool update_probs = false;
2633
2634	if (gimple_code (g: copy_stmt) == GIMPLE_EH_DISPATCH)
2635	{
2636	make_eh_dispatch_edges (as_a <geh_dispatch *> (p: copy_stmt));
2637	update_probs = true;
2638	}
2639	else if (can_throw)
2640	{
2641	make_eh_edge (copy_stmt);
2642	update_probs = true;
2643	}
2644
2645	/* EH edges may not match old edges. Copy as much as possible. */
2646	if (update_probs)
2647	{
2648	edge e;
2649	edge_iterator ei;
2650	basic_block copy_stmt_bb = gimple_bb (g: copy_stmt);
2651
2652	FOR_EACH_EDGE (old_edge, ei, bb->succs)
2653	if ((old_edge->flags & EDGE_EH)
2654	&& (e = find_edge (copy_stmt_bb,
2655	(basic_block) old_edge->dest->aux))
2656	&& (e->flags & EDGE_EH))
2657	e->probability = old_edge->probability;
2658
2659	FOR_EACH_EDGE (e, ei, copy_stmt_bb->succs)
2660	if (e->flags & EDGE_EH)
2661	{
2662	if (!e->probability.initialized_p ())
2663	e->probability = profile_probability::never ();
2664	if (e->dest->index < id->add_clobbers_to_eh_landing_pads)
2665	{
2666	if (id->eh_landing_pad_dest == NULL)
2667	id->eh_landing_pad_dest = e->dest;
2668	else
2669	gcc_assert (id->eh_landing_pad_dest == e->dest);
2670	}
2671	}
2672	}
2673
2674
2675	/* If the call we inline cannot make abnormal goto do not add
2676	additional abnormal edges but only retain those already present
2677	in the original function body. */
2678	if (abnormal_goto_dest == NULL)
2679	nonlocal_goto = false;
2680	if (nonlocal_goto)
2681	{
2682	basic_block copy_stmt_bb = gimple_bb (g: copy_stmt);
2683
2684	if (get_abnormal_succ_dispatcher (copy_stmt_bb))
2685	nonlocal_goto = false;
2686	/* ABNORMAL_DISPATCHER (1) is for longjmp/setjmp or nonlocal gotos
2687	in OpenMP regions which aren't allowed to be left abnormally.
2688	So, no need to add abnormal edge in that case. */
2689	else if (is_gimple_call (gs: copy_stmt)
2690	&& gimple_call_internal_p (gs: copy_stmt)
2691	&& (gimple_call_internal_fn (gs: copy_stmt)
2692	== IFN_ABNORMAL_DISPATCHER)
2693	&& gimple_call_arg (gs: copy_stmt, index: 0) == boolean_true_node)
2694	nonlocal_goto = false;
2695	else
2696	make_single_succ_edge (copy_stmt_bb, abnormal_goto_dest,
2697	EDGE_ABNORMAL);
2698	}
2699
2700	if ((can_throw || nonlocal_goto)
2701	&& gimple_in_ssa_p (cfun))
2702	update_ssa_across_abnormal_edges (bb: gimple_bb (g: copy_stmt), ret_bb,
2703	can_throw, nonlocal_goto);
2704	}
2705	return need_debug_cleanup;
2706	}
2707
2708	/* Copy the PHIs. All blocks and edges are copied, some blocks
2709	was possibly split and new outgoing EH edges inserted.
2710	BB points to the block of original function and AUX pointers links
2711	the original and newly copied blocks. */
2712
2713	static void
2714	copy_phis_for_bb (basic_block bb, copy_body_data *id)
2715	{
2716	basic_block const new_bb = (basic_block) bb->aux;
2717	edge_iterator ei;
2718	gphi *phi;
2719	gphi_iterator si;
2720	edge new_edge;
2721	bool inserted = false;
2722
2723	for (si = gsi_start_phis (bb); !gsi_end_p (i: si); gsi_next (i: &si))
2724	{
2725	tree res, new_res;
2726	gphi *new_phi;
2727
2728	phi = si.phi ();
2729	res = PHI_RESULT (phi);
2730	new_res = res;
2731	if (!virtual_operand_p (op: res)
2732	&& (!id->param_body_adjs
2733	|| !id->param_body_adjs->m_dead_stmts.contains (k: phi)))
2734	{
2735	walk_tree (&new_res, copy_tree_body_r, id, NULL);
2736	if (EDGE_COUNT (new_bb->preds) == 0)
2737	{
2738	/* Technically we'd want a SSA_DEFAULT_DEF here... */
2739	SSA_NAME_DEF_STMT (new_res) = gimple_build_nop ();
2740	}
2741	else
2742	{
2743	new_phi = create_phi_node (new_res, new_bb);
2744	FOR_EACH_EDGE (new_edge, ei, new_bb->preds)
2745	{
2746	edge old_edge = find_edge ((basic_block) new_edge->src->aux,
2747	bb);
2748	tree arg;
2749	tree new_arg;
2750	edge_iterator ei2;
2751	location_t locus;
2752
2753	/* When doing partial cloning, we allow PHIs on the entry
2754	block as long as all the arguments are the same.
2755	Find any input edge to see argument to copy. */
2756	if (!old_edge)
2757	FOR_EACH_EDGE (old_edge, ei2, bb->preds)
2758	if (!old_edge->src->aux)
2759	break;
2760
2761	arg = PHI_ARG_DEF_FROM_EDGE (phi, old_edge);
2762	new_arg = arg;
2763	walk_tree (&new_arg, copy_tree_body_r, id, NULL);
2764	gcc_assert (new_arg);
2765	/* With return slot optimization we can end up with
2766	non-gimple (foo *)&this->m, fix that here. */
2767	if (TREE_CODE (new_arg) != SSA_NAME
2768	&& TREE_CODE (new_arg) != FUNCTION_DECL
2769	&& !is_gimple_val (new_arg))
2770	{
2771	gimple_seq stmts = NULL;
2772	new_arg = force_gimple_operand (new_arg, &stmts, true,
2773	NULL);
2774	gsi_insert_seq_on_edge (new_edge, stmts);
2775	inserted = true;
2776	}
2777	locus = gimple_phi_arg_location_from_edge (phi, e: old_edge);
2778	if (id->reset_location)
2779	locus = input_location;
2780	else
2781	locus = remap_location (locus, id);
2782	add_phi_arg (new_phi, new_arg, new_edge, locus);
2783	}
2784	}
2785	}
2786	}
2787
2788	/* Commit the delayed edge insertions. */
2789	if (inserted)
2790	FOR_EACH_EDGE (new_edge, ei, new_bb->preds)
2791	gsi_commit_one_edge_insert (new_edge, NULL);
2792	}
2793
2794
2795	/* Wrapper for remap_decl so it can be used as a callback. */
2796
2797	static tree
2798	remap_decl_1 (tree decl, void *data)
2799	{
2800	return remap_decl (decl, id: (copy_body_data *) data);
2801	}
2802
2803	/* Build struct function and associated datastructures for the new clone
2804	NEW_FNDECL to be build. CALLEE_FNDECL is the original. Function changes
2805	the cfun to the function of new_fndecl (and current_function_decl too). */
2806
2807	static void
2808	initialize_cfun (tree new_fndecl, tree callee_fndecl, profile_count count)
2809	{
2810	struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
2811
2812	/* Register specific tree functions. */
2813	gimple_register_cfg_hooks ();
2814
2815	/* Get clean struct function. */
2816	push_struct_function (fndecl: new_fndecl, true);
2817	targetm.target_option.relayout_function (new_fndecl);
2818
2819	/* We will rebuild these, so just sanity check that they are empty. */
2820	gcc_assert (VALUE_HISTOGRAMS (cfun) == NULL);
2821	gcc_assert (cfun->local_decls == NULL);
2822	gcc_assert (cfun->cfg == NULL);
2823	gcc_assert (cfun->decl == new_fndecl);
2824
2825	/* Copy items we preserve during cloning. */
2826	cfun->static_chain_decl = src_cfun->static_chain_decl;
2827	cfun->nonlocal_goto_save_area = src_cfun->nonlocal_goto_save_area;
2828	cfun->function_end_locus = src_cfun->function_end_locus;
2829	cfun->curr_properties = src_cfun->curr_properties;
2830	cfun->last_verified = src_cfun->last_verified;
2831	cfun->va_list_gpr_size = src_cfun->va_list_gpr_size;
2832	cfun->va_list_fpr_size = src_cfun->va_list_fpr_size;
2833	cfun->has_nonlocal_label = src_cfun->has_nonlocal_label;
2834	cfun->calls_eh_return = src_cfun->calls_eh_return;
2835	cfun->stdarg = src_cfun->stdarg;
2836	cfun->after_inlining = src_cfun->after_inlining;
2837	cfun->can_throw_non_call_exceptions
2838	= src_cfun->can_throw_non_call_exceptions;
2839	cfun->can_delete_dead_exceptions = src_cfun->can_delete_dead_exceptions;
2840	cfun->returns_struct = src_cfun->returns_struct;
2841	cfun->returns_pcc_struct = src_cfun->returns_pcc_struct;
2842
2843	init_empty_tree_cfg ();
2844
2845	profile_status_for_fn (cfun) = profile_status_for_fn (src_cfun);
2846	cfun->cfg->full_profile = src_cfun->cfg->full_profile;
2847
2848	profile_count num = count;
2849	profile_count den = ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count;
2850	profile_count::adjust_for_ipa_scaling (num: &num, den: &den);
2851
2852	ENTRY_BLOCK_PTR_FOR_FN (cfun)->count =
2853	ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (num: count,
2854	ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count);
2855	EXIT_BLOCK_PTR_FOR_FN (cfun)->count =
2856	EXIT_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (num: count,
2857	ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count);
2858	if (src_cfun->eh)
2859	init_eh_for_function ();
2860
2861	if (src_cfun->gimple_df)
2862	{
2863	init_tree_ssa (cfun);
2864	cfun->gimple_df->in_ssa_p = src_cfun->gimple_df->in_ssa_p;
2865	if (cfun->gimple_df->in_ssa_p)
2866	init_ssa_operands (cfun);
2867	}
2868	}
2869
2870	/* Helper function for copy_cfg_body. Move debug stmts from the end
2871	of NEW_BB to the beginning of successor basic blocks when needed. If the
2872	successor has multiple predecessors, reset them, otherwise keep
2873	their value. */
2874
2875	static void
2876	maybe_move_debug_stmts_to_successors (copy_body_data *id, basic_block new_bb)
2877	{
2878	edge e;
2879	edge_iterator ei;
2880	gimple_stmt_iterator si = gsi_last_nondebug_bb (bb: new_bb);
2881
2882	if (gsi_end_p (i: si)
2883	|| gsi_one_before_end_p (i: si)
2884	|| !(stmt_can_throw_internal (cfun, gsi_stmt (i: si))
2885	|| stmt_can_make_abnormal_goto (gsi_stmt (i: si))))
2886	return;
2887
2888	FOR_EACH_EDGE (e, ei, new_bb->succs)
2889	{
2890	gimple_stmt_iterator ssi = gsi_last_bb (bb: new_bb);
2891	gimple_stmt_iterator dsi = gsi_after_labels (bb: e->dest);
2892	while (is_gimple_debug (gs: gsi_stmt (i: ssi)))
2893	{
2894	gimple *stmt = gsi_stmt (i: ssi);
2895	gdebug *new_stmt;
2896	tree var;
2897	tree value;
2898
2899	/* For the last edge move the debug stmts instead of copying
2900	them. */
2901	if (ei_one_before_end_p (i: ei))
2902	{
2903	si = ssi;
2904	gsi_prev (i: &ssi);
2905	if (!single_pred_p (bb: e->dest) && gimple_debug_bind_p (s: stmt))
2906	{
2907	gimple_debug_bind_reset_value (dbg: stmt);
2908	gimple_set_location (g: stmt, UNKNOWN_LOCATION);
2909	}
2910	gsi_remove (&si, false);
2911	gsi_insert_before (&dsi, stmt, GSI_NEW_STMT);
2912	continue;
2913	}
2914
2915	if (gimple_debug_bind_p (s: stmt))
2916	{
2917	var = gimple_debug_bind_get_var (dbg: stmt);
2918	if (single_pred_p (bb: e->dest))
2919	{
2920	value = gimple_debug_bind_get_value (dbg: stmt);
2921	value = unshare_expr (value);
2922	new_stmt = gimple_build_debug_bind (var, value, stmt);
2923	}
2924	else
2925	new_stmt = gimple_build_debug_bind (var, NULL_TREE, NULL);
2926	}
2927	else if (gimple_debug_source_bind_p (s: stmt))
2928	{
2929	var = gimple_debug_source_bind_get_var (dbg: stmt);
2930	value = gimple_debug_source_bind_get_value (dbg: stmt);
2931	new_stmt = gimple_build_debug_source_bind (var, value, stmt);
2932	}
2933	else if (gimple_debug_nonbind_marker_p (s: stmt))
2934	new_stmt = as_a <gdebug *> (p: gimple_copy (stmt));
2935	else
2936	gcc_unreachable ();
2937	gsi_insert_before (&dsi, new_stmt, GSI_NEW_STMT);
2938	id->debug_stmts.safe_push (obj: new_stmt);
2939	gsi_prev (i: &ssi);
2940	}
2941	}
2942	}
2943
2944	/* Make a copy of the sub-loops of SRC_PARENT and place them
2945	as siblings of DEST_PARENT. */
2946
2947	static void
2948	copy_loops (copy_body_data *id,
2949	class loop *dest_parent, class loop *src_parent)
2950	{
2951	class loop *src_loop = src_parent->inner;
2952	while (src_loop)
2953	{
2954	if (!id->blocks_to_copy
2955	|| bitmap_bit_p (id->blocks_to_copy, src_loop->header->index))
2956	{
2957	class loop *dest_loop = alloc_loop ();
2958
2959	/* Assign the new loop its header and latch and associate
2960	those with the new loop. */
2961	dest_loop->header = (basic_block)src_loop->header->aux;
2962	dest_loop->header->loop_father = dest_loop;
2963	if (src_loop->latch != NULL)
2964	{
2965	dest_loop->latch = (basic_block)src_loop->latch->aux;
2966	dest_loop->latch->loop_father = dest_loop;
2967	}
2968
2969	/* Copy loop meta-data. */
2970	copy_loop_info (loop: src_loop, target: dest_loop);
2971	if (dest_loop->unroll)
2972	cfun->has_unroll = true;
2973	if (dest_loop->force_vectorize)
2974	cfun->has_force_vectorize_loops = true;
2975	if (id->src_cfun->last_clique != 0)
2976	dest_loop->owned_clique
2977	= remap_dependence_clique (id,
2978	clique: src_loop->owned_clique
2979	? src_loop->owned_clique : 1);
2980
2981	/* Finally place it into the loop array and the loop tree. */
2982	place_new_loop (cfun, dest_loop);
2983	flow_loop_tree_node_add (dest_parent, dest_loop);
2984
2985	if (src_loop->simduid)
2986	{
2987	dest_loop->simduid = remap_decl (decl: src_loop->simduid, id);
2988	cfun->has_simduid_loops = true;
2989	}
2990
2991	/* Recurse. */
2992	copy_loops (id, dest_parent: dest_loop, src_parent: src_loop);
2993	}
2994	src_loop = src_loop->next;
2995	}
2996	}
2997
2998	/* Call redirect_call_stmt_to_callee on all calls in BB. */
2999
3000	void
3001	redirect_all_calls (copy_body_data * id, basic_block bb)
3002	{
3003	gimple_stmt_iterator si;
3004	gimple *last = last_nondebug_stmt (bb);
3005	for (si = gsi_start_bb (bb); !gsi_end_p (i: si); gsi_next (i: &si))
3006	{
3007	gimple *stmt = gsi_stmt (i: si);
3008	if (is_gimple_call (gs: stmt))
3009	{
3010	struct cgraph_edge *edge = id->dst_node->get_edge (call_stmt: stmt);
3011	if (edge)
3012	{
3013	if (!id->killed_new_ssa_names)
3014	id->killed_new_ssa_names = new hash_set<tree> (16);
3015	cgraph_edge::redirect_call_stmt_to_callee (e: edge,
3016	killed_ssas: id->killed_new_ssa_names);
3017
3018	if (stmt == last && id->call_stmt && maybe_clean_eh_stmt (stmt))
3019	gimple_purge_dead_eh_edges (bb);
3020	}
3021	}
3022	}
3023	}
3024
3025	/* Make a copy of the body of FN so that it can be inserted inline in
3026	another function. Walks FN via CFG, returns new fndecl. */
3027
3028	static tree
3029	copy_cfg_body (copy_body_data * id,
3030	basic_block entry_block_map, basic_block exit_block_map,
3031	basic_block new_entry)
3032	{
3033	tree callee_fndecl = id->src_fn;
3034	/* Original cfun for the callee, doesn't change. */
3035	struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
3036	struct function *cfun_to_copy;
3037	basic_block bb;
3038	tree new_fndecl = NULL;
3039	bool need_debug_cleanup = false;
3040	int last;
3041	profile_count den = ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count;
3042	profile_count num = entry_block_map->count;
3043
3044	cfun_to_copy = id->src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
3045
3046	/* Register specific tree functions. */
3047	gimple_register_cfg_hooks ();
3048
3049	/* If we are inlining just region of the function, make sure to connect
3050	new entry to ENTRY_BLOCK_PTR_FOR_FN (cfun). Since new entry can be
3051	part of loop, we must compute frequency and probability of
3052	ENTRY_BLOCK_PTR_FOR_FN (cfun) based on the frequencies and
3053	probabilities of edges incoming from nonduplicated region. */
3054	if (new_entry)
3055	{
3056	edge e;
3057	edge_iterator ei;
3058	den = profile_count::zero ();
3059
3060	FOR_EACH_EDGE (e, ei, new_entry->preds)
3061	if (!e->src->aux)
3062	den += e->count ();
3063	ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = den;
3064	}
3065
3066	profile_count::adjust_for_ipa_scaling (num: &num, den: &den);
3067
3068	/* Must have a CFG here at this point. */
3069	gcc_assert (ENTRY_BLOCK_PTR_FOR_FN
3070	(DECL_STRUCT_FUNCTION (callee_fndecl)));
3071
3072
3073	ENTRY_BLOCK_PTR_FOR_FN (cfun_to_copy)->aux = entry_block_map;
3074	EXIT_BLOCK_PTR_FOR_FN (cfun_to_copy)->aux = exit_block_map;
3075	entry_block_map->aux = ENTRY_BLOCK_PTR_FOR_FN (cfun_to_copy);
3076	exit_block_map->aux = EXIT_BLOCK_PTR_FOR_FN (cfun_to_copy);
3077
3078	/* Duplicate any exception-handling regions. */
3079	if (cfun->eh)
3080	id->eh_map = duplicate_eh_regions (cfun_to_copy, NULL, id->eh_lp_nr,
3081	remap_decl_1, id);
3082
3083	/* Use aux pointers to map the original blocks to copy. */
3084	FOR_EACH_BB_FN (bb, cfun_to_copy)
3085	if (!id->blocks_to_copy || bitmap_bit_p (id->blocks_to_copy, bb->index))
3086	{
3087	basic_block new_bb = copy_bb (id, bb, num, den);
3088	bb->aux = new_bb;
3089	new_bb->aux = bb;
3090	new_bb->loop_father = entry_block_map->loop_father;
3091	}
3092
3093	last = last_basic_block_for_fn (cfun);
3094
3095	/* Now that we've duplicated the blocks, duplicate their edges. */
3096	basic_block abnormal_goto_dest = NULL;
3097	if (id->call_stmt
3098	&& stmt_can_make_abnormal_goto (id->call_stmt))
3099	{
3100	gimple_stmt_iterator gsi = gsi_for_stmt (id->call_stmt);
3101
3102	bb = gimple_bb (g: id->call_stmt);
3103	gsi_next (i: &gsi);
3104	if (gsi_end_p (i: gsi))
3105	abnormal_goto_dest = get_abnormal_succ_dispatcher (bb);
3106	}
3107	FOR_ALL_BB_FN (bb, cfun_to_copy)
3108	if (!id->blocks_to_copy
3109	|| (bb->index > 0 && bitmap_bit_p (id->blocks_to_copy, bb->index)))
3110	need_debug_cleanup |= copy_edges_for_bb (bb, num, den, ret_bb: exit_block_map,
3111	abnormal_goto_dest, id);
3112
3113	if (id->eh_landing_pad_dest)
3114	{
3115	add_clobbers_to_eh_landing_pad (id);
3116	id->eh_landing_pad_dest = NULL;
3117	}
3118
3119	if (new_entry)
3120	{
3121	edge e = make_edge (entry_block_map, (basic_block)new_entry->aux,
3122	EDGE_FALLTHRU);
3123	e->probability = profile_probability::always ();
3124	}
3125
3126	/* Duplicate the loop tree, if available and wanted. */
3127	if (loops_for_fn (fn: src_cfun) != NULL
3128	&& current_loops != NULL)
3129	{
3130	copy_loops (id, dest_parent: entry_block_map->loop_father,
3131	src_parent: get_loop (fn: src_cfun, num: 0));
3132	/* Defer to cfgcleanup to update loop-father fields of basic-blocks. */
3133	loops_state_set (flags: LOOPS_NEED_FIXUP);
3134	}
3135
3136	/* If the loop tree in the source function needed fixup, mark the
3137	destination loop tree for fixup, too. */
3138	if (loops_for_fn (fn: src_cfun)->state & LOOPS_NEED_FIXUP)
3139	loops_state_set (flags: LOOPS_NEED_FIXUP);
3140
3141	if (gimple_in_ssa_p (cfun))
3142	FOR_ALL_BB_FN (bb, cfun_to_copy)
3143	if (!id->blocks_to_copy
3144	|| (bb->index > 0 && bitmap_bit_p (id->blocks_to_copy, bb->index)))
3145	copy_phis_for_bb (bb, id);
3146
3147	FOR_ALL_BB_FN (bb, cfun_to_copy)
3148	if (bb->aux)
3149	{
3150	if (need_debug_cleanup
3151	&& bb->index != ENTRY_BLOCK
3152	&& bb->index != EXIT_BLOCK)
3153	maybe_move_debug_stmts_to_successors (id, new_bb: (basic_block) bb->aux);
3154	/* Update call edge destinations. This cannot be done before loop
3155	info is updated, because we may split basic blocks. */
3156	if (id->transform_call_graph_edges == CB_CGE_DUPLICATE
3157	&& bb->index != ENTRY_BLOCK
3158	&& bb->index != EXIT_BLOCK)
3159	redirect_all_calls (id, bb: (basic_block)bb->aux);
3160	((basic_block)bb->aux)->aux = NULL;
3161	bb->aux = NULL;
3162	}
3163
3164	/* Zero out AUX fields of newly created block during EH edge
3165	insertion. */
3166	for (; last < last_basic_block_for_fn (cfun); last++)
3167	{
3168	if (need_debug_cleanup)
3169	maybe_move_debug_stmts_to_successors (id,
3170	BASIC_BLOCK_FOR_FN (cfun, last));
3171	BASIC_BLOCK_FOR_FN (cfun, last)->aux = NULL;
3172	/* Update call edge destinations. This cannot be done before loop
3173	info is updated, because we may split basic blocks. */
3174	if (id->transform_call_graph_edges == CB_CGE_DUPLICATE)
3175	redirect_all_calls (id, BASIC_BLOCK_FOR_FN (cfun, last));
3176	}
3177	entry_block_map->aux = NULL;
3178	exit_block_map->aux = NULL;
3179
3180	if (id->eh_map)
3181	{
3182	delete id->eh_map;
3183	id->eh_map = NULL;
3184	}
3185	if (id->dependence_map)
3186	{
3187	delete id->dependence_map;
3188	id->dependence_map = NULL;
3189	}
3190
3191	return new_fndecl;
3192	}
3193
3194	/* Copy the debug STMT using ID. We deal with these statements in a
3195	special way: if any variable in their VALUE expression wasn't
3196	remapped yet, we won't remap it, because that would get decl uids
3197	out of sync, causing codegen differences between -g and -g0. If
3198	this arises, we drop the VALUE expression altogether. */
3199
3200	static void
3201	copy_debug_stmt (gdebug *stmt, copy_body_data *id)
3202	{
3203	tree t, *n;
3204	struct walk_stmt_info wi;
3205
3206	if (tree block = gimple_block (g: stmt))
3207	{
3208	n = id->decl_map->get (k: block);
3209	gimple_set_block (g: stmt, block: n ? *n : id->block);
3210	}
3211
3212	if (gimple_debug_nonbind_marker_p (s: stmt))
3213	{
3214	if (id->call_stmt && !gimple_block (g: stmt))
3215	{
3216	gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
3217	gsi_remove (&gsi, true);
3218	}
3219	return;
3220	}
3221
3222	/* Remap all the operands in COPY. */
3223	memset (s: &wi, c: 0, n: sizeof (wi));
3224	wi.info = id;
3225
3226	processing_debug_stmt = 1;
3227
3228	if (gimple_debug_source_bind_p (s: stmt))
3229	t = gimple_debug_source_bind_get_var (dbg: stmt);
3230	else if (gimple_debug_bind_p (s: stmt))
3231	t = gimple_debug_bind_get_var (dbg: stmt);
3232	else
3233	gcc_unreachable ();
3234
3235	if (TREE_CODE (t) == PARM_DECL
3236	&& id->debug_map
3237	&& (n = id->debug_map->get (k: t)))
3238	{
3239	gcc_assert (VAR_P (*n));
3240	t = *n;
3241	}
3242	else if (VAR_P (t) && !is_global_var (t) && !id->decl_map->get (k: t))
3243	/* T is a non-localized variable. */;
3244	else
3245	walk_tree (&t, remap_gimple_op_r, &wi, NULL);
3246
3247	if (gimple_debug_bind_p (s: stmt))
3248	{
3249	gimple_debug_bind_set_var (dbg: stmt, var: t);
3250
3251	if (gimple_debug_bind_has_value_p (dbg: stmt))
3252	walk_tree (gimple_debug_bind_get_value_ptr (stmt),
3253	remap_gimple_op_r, &wi, NULL);
3254
3255	/* Punt if any decl couldn't be remapped. */
3256	if (processing_debug_stmt < 0)
3257	gimple_debug_bind_reset_value (dbg: stmt);
3258	}
3259	else if (gimple_debug_source_bind_p (s: stmt))
3260	{
3261	gimple_debug_source_bind_set_var (dbg: stmt, var: t);
3262	/* When inlining and source bind refers to one of the optimized
3263	away parameters, change the source bind into normal debug bind
3264	referring to the corresponding DEBUG_EXPR_DECL that should have
3265	been bound before the call stmt. */
3266	t = gimple_debug_source_bind_get_value (dbg: stmt);
3267	if (t != NULL_TREE
3268	&& TREE_CODE (t) == PARM_DECL
3269	&& id->call_stmt)
3270	{
3271	vec<tree, va_gc> **debug_args = decl_debug_args_lookup (id->src_fn);
3272	unsigned int i;
3273	if (debug_args != NULL)
3274	{
3275	for (i = 0; i < vec_safe_length (v: *debug_args); i += 2)
3276	if ((**debug_args)[i] == DECL_ORIGIN (t)
3277	&& TREE_CODE ((**debug_args)[i + 1]) == DEBUG_EXPR_DECL)
3278	{
3279	t = (**debug_args)[i + 1];
3280	stmt->subcode = GIMPLE_DEBUG_BIND;
3281	gimple_debug_bind_set_value (dbg: stmt, value: t);
3282	break;
3283	}
3284	}
3285	}
3286	if (gimple_debug_source_bind_p (s: stmt))
3287	walk_tree (gimple_debug_source_bind_get_value_ptr (stmt),
3288	remap_gimple_op_r, &wi, NULL);
3289	}
3290
3291	processing_debug_stmt = 0;
3292
3293	update_stmt (s: stmt);
3294	}
3295
3296	/* Process deferred debug stmts. In order to give values better odds
3297	of being successfully remapped, we delay the processing of debug
3298	stmts until all other stmts that might require remapping are
3299	processed. */
3300
3301	static void
3302	copy_debug_stmts (copy_body_data *id)
3303	{
3304	if (!id->debug_stmts.exists ())
3305	return;
3306
3307	for (gdebug *stmt : id->debug_stmts)
3308	copy_debug_stmt (stmt, id);
3309
3310	id->debug_stmts.release ();
3311	}
3312
3313	/* Make a copy of the body of SRC_FN so that it can be inserted inline in
3314	another function. */
3315
3316	static tree
3317	copy_tree_body (copy_body_data *id)
3318	{
3319	tree fndecl = id->src_fn;
3320	tree body = DECL_SAVED_TREE (fndecl);
3321
3322	walk_tree (&body, copy_tree_body_r, id, NULL);
3323
3324	return body;
3325	}
3326
3327	/* Make a copy of the body of FN so that it can be inserted inline in
3328	another function. */
3329
3330	static tree
3331	copy_body (copy_body_data *id,
3332	basic_block entry_block_map, basic_block exit_block_map,
3333	basic_block new_entry)
3334	{
3335	tree fndecl = id->src_fn;
3336	tree body;
3337
3338	/* If this body has a CFG, walk CFG and copy. */
3339	gcc_assert (ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (fndecl)));
3340	body = copy_cfg_body (id, entry_block_map, exit_block_map,
3341	new_entry);
3342	copy_debug_stmts (id);
3343	if (id->killed_new_ssa_names)
3344	{
3345	ipa_release_ssas_in_hash (killed_ssas: id->killed_new_ssa_names);
3346	delete id->killed_new_ssa_names;
3347	id->killed_new_ssa_names = NULL;
3348	}
3349
3350	return body;
3351	}
3352
3353	/* Return true if VALUE is an ADDR_EXPR of an automatic variable
3354	defined in function FN, or of a data member thereof. */
3355
3356	static bool
3357	self_inlining_addr_expr (tree value, tree fn)
3358	{
3359	tree var;
3360
3361	if (TREE_CODE (value) != ADDR_EXPR)
3362	return false;
3363
3364	var = get_base_address (TREE_OPERAND (value, 0));
3365
3366	return var && auto_var_in_fn_p (var, fn);
3367	}
3368
3369	/* Append to BB a debug annotation that binds VAR to VALUE, inheriting
3370	lexical block and line number information from base_stmt, if given,
3371	or from the last stmt of the block otherwise. */
3372
3373	static gimple *
3374	insert_init_debug_bind (copy_body_data *id,
3375	basic_block bb, tree var, tree value,
3376	gimple *base_stmt)
3377	{
3378	gimple *note;
3379	gimple_stmt_iterator gsi;
3380	tree tracked_var;
3381
3382	if (!gimple_in_ssa_p (fun: id->src_cfun))
3383	return NULL;
3384
3385	if (!opt_for_fn (id->dst_fn, flag_var_tracking_assignments))
3386	return NULL;
3387
3388	tracked_var = target_for_debug_bind (var);
3389	if (!tracked_var)
3390	return NULL;
3391
3392	if (bb)
3393	{
3394	gsi = gsi_last_bb (bb);
3395	if (!base_stmt && !gsi_end_p (i: gsi))
3396	base_stmt = gsi_stmt (i: gsi);
3397	}
3398
3399	note = gimple_build_debug_bind (tracked_var,
3400	value == error_mark_node
3401	? NULL_TREE : unshare_expr (value),
3402	base_stmt);
3403
3404	if (bb)
3405	{
3406	if (!gsi_end_p (i: gsi))
3407	gsi_insert_after (&gsi, note, GSI_SAME_STMT);
3408	else
3409	gsi_insert_before (&gsi, note, GSI_SAME_STMT);
3410	}
3411
3412	return note;
3413	}
3414
3415	static void
3416	insert_init_stmt (copy_body_data *id, basic_block bb, gimple *init_stmt)
3417	{
3418	/* If VAR represents a zero-sized variable, it's possible that the
3419	assignment statement may result in no gimple statements. */
3420	if (init_stmt)
3421	{
3422	gimple_stmt_iterator si = gsi_last_bb (bb);
3423
3424	/* We can end up with init statements that store to a non-register
3425	from a rhs with a conversion. Handle that here by forcing the
3426	rhs into a temporary. gimple_regimplify_operands is not
3427	prepared to do this for us. */
3428	if (!is_gimple_debug (gs: init_stmt)
3429	&& !is_gimple_reg (gimple_assign_lhs (gs: init_stmt))
3430	&& is_gimple_reg_type (TREE_TYPE (gimple_assign_lhs (init_stmt)))
3431	&& gimple_assign_rhs_class (gs: init_stmt) == GIMPLE_UNARY_RHS)
3432	{
3433	tree rhs = build1 (gimple_assign_rhs_code (gs: init_stmt),
3434	TREE_TYPE (gimple_assign_lhs (init_stmt)),
3435	gimple_assign_rhs1 (gs: init_stmt));
3436	rhs = force_gimple_operand_gsi (&si, rhs, true, NULL_TREE, false,
3437	GSI_NEW_STMT);
3438	gimple_assign_set_rhs_code (s: init_stmt, TREE_CODE (rhs));
3439	gimple_assign_set_rhs1 (gs: init_stmt, rhs);
3440	}
3441	gsi_insert_after (&si, init_stmt, GSI_NEW_STMT);
3442	if (!is_gimple_debug (gs: init_stmt))
3443	{
3444	gimple_regimplify_operands (init_stmt, &si);
3445
3446	tree def = gimple_assign_lhs (gs: init_stmt);
3447	insert_init_debug_bind (id, bb, var: def, value: def, base_stmt: init_stmt);
3448	}
3449	}
3450	}
3451
3452	/* Deal with mismatched formal/actual parameters, in a rather brute-force way
3453	if need be (which should only be necessary for invalid programs). Attempt
3454	to convert VAL to TYPE and return the result if it is possible, just return
3455	a zero constant of the given type if it fails. */
3456
3457	tree
3458	force_value_to_type (tree type, tree value)
3459	{
3460	/* If we can match up types by promotion/demotion do so. */
3461	if (fold_convertible_p (type, value))
3462	return fold_convert (type, value);
3463
3464	/* ??? For valid programs we should not end up here.
3465	Still if we end up with truly mismatched types here, fall back
3466	to using a VIEW_CONVERT_EXPR or a literal zero to not leak invalid
3467	GIMPLE to the following passes. */
3468	if (TREE_CODE (value) == WITH_SIZE_EXPR)
3469	return error_mark_node;
3470	else if (!is_gimple_reg_type (TREE_TYPE (value))
3471	|| TYPE_SIZE (type) == TYPE_SIZE (TREE_TYPE (value)))
3472	return fold_build1 (VIEW_CONVERT_EXPR, type, value);
3473	else
3474	return build_zero_cst (type);
3475	}
3476
3477	/* Initialize parameter P with VALUE. If needed, produce init statement
3478	at the end of BB. When BB is NULL, we return init statement to be
3479	output later. */
3480	static gimple *
3481	setup_one_parameter (copy_body_data *id, tree p, tree value, tree fn,
3482	basic_block bb, tree *vars)
3483	{
3484	gimple *init_stmt = NULL;
3485	tree var;
3486	tree def = (gimple_in_ssa_p (cfun)
3487	? ssa_default_def (id->src_cfun, p) : NULL);
3488
3489	/* Make an equivalent VAR_DECL. Note that we must NOT remap the type
3490	here since the type of this decl must be visible to the calling
3491	function. */
3492	var = copy_decl_to_var (p, id);
3493
3494	/* Declare this new variable. */
3495	DECL_CHAIN (var) = *vars;
3496	*vars = var;
3497
3498	/* Make gimplifier happy about this variable. */
3499	DECL_SEEN_IN_BIND_EXPR_P (var) = 1;
3500
3501	/* If the parameter is never assigned to, has no SSA_NAMEs created,
3502	we would not need to create a new variable here at all, if it
3503	weren't for debug info. Still, we can just use the argument
3504	value. */
3505	if (TREE_READONLY (p)
3506	&& !TREE_ADDRESSABLE (p)
3507	&& value
3508	&& !TREE_SIDE_EFFECTS (value)
3509	&& !def)
3510	{
3511	/* We may produce non-gimple trees by adding NOPs or introduce invalid
3512	sharing when the value is not constant or DECL. And we need to make
3513	sure that it cannot be modified from another path in the callee. */
3514	if (((is_gimple_min_invariant (value)
3515	/* When the parameter is used in a context that forces it to
3516	not be a GIMPLE register avoid substituting something that
3517	is not a decl there. */
3518	&& ! DECL_NOT_GIMPLE_REG_P (p))
3519	|| (DECL_P (value) && TREE_READONLY (value))
3520	|| (auto_var_in_fn_p (value, id->dst_fn)
3521	&& !TREE_ADDRESSABLE (value)))
3522	&& useless_type_conversion_p (TREE_TYPE (p), TREE_TYPE (value))
3523	/* We have to be very careful about ADDR_EXPR. Make sure
3524	the base variable isn't a local variable of the inlined
3525	function, e.g., when doing recursive inlining, direct or
3526	mutually-recursive or whatever, which is why we don't
3527	just test whether fn == current_function_decl. */
3528	&& ! self_inlining_addr_expr (value, fn))
3529	{
3530	insert_decl_map (id, key: p, value);
3531	if (!id->debug_map)
3532	id->debug_map = new hash_map<tree, tree>;
3533	id->debug_map->put (k: p, v: var);
3534	return insert_init_debug_bind (id, bb, var, value, NULL);
3535	}
3536	}
3537
3538	/* Register the VAR_DECL as the equivalent for the PARM_DECL;
3539	that way, when the PARM_DECL is encountered, it will be
3540	automatically replaced by the VAR_DECL. */
3541	insert_decl_map (id, key: p, value: var);
3542
3543	/* Even if P was TREE_READONLY, the new VAR should not be. In the original
3544	code, we would have constructed a temporary, and then the function body
3545	would have never changed the value of P. However, now, we will be
3546	constructing VAR directly. Therefore, it must not be TREE_READONLY. */
3547	TREE_READONLY (var) = 0;
3548
3549	tree rhs = value;
3550	if (value
3551	&& value != error_mark_node
3552	&& !useless_type_conversion_p (TREE_TYPE (p), TREE_TYPE (value)))
3553	rhs = force_value_to_type (TREE_TYPE (p), value);
3554
3555	/* If there is no setup required and we are in SSA, take the easy route
3556	replacing all SSA names representing the function parameter by the
3557	SSA name passed to function.
3558
3559	We need to construct map for the variable anyway as it might be used
3560	in different SSA names when parameter is set in function.
3561
3562	Do replacement at -O0 for const arguments replaced by constant.
3563	This is important for builtin_constant_p and other construct requiring
3564	constant argument to be visible in inlined function body. */
3565	if (gimple_in_ssa_p (cfun) && rhs && def && is_gimple_reg (p)
3566	&& (optimize
3567	|| (TREE_READONLY (p)
3568	&& is_gimple_min_invariant (rhs)))
3569	&& (TREE_CODE (rhs) == SSA_NAME
3570	|| is_gimple_min_invariant (rhs))
3571	&& !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (def))
3572	{
3573	insert_decl_map (id, key: def, value: rhs);
3574	return insert_init_debug_bind (id, bb, var, value: rhs, NULL);
3575	}
3576
3577	/* If the value of argument is never used, don't care about initializing
3578	it. */
3579	if (optimize && gimple_in_ssa_p (cfun) && !def && is_gimple_reg (p))
3580	{
3581	/* When there's a gross type mismatch between the passed value
3582	and the declared argument type drop it on the floor and do
3583	not bother to insert a debug bind. */
3584	if (value && !is_gimple_reg_type (TREE_TYPE (value)))
3585	return NULL;
3586	return insert_init_debug_bind (id, bb, var, value: rhs, NULL);
3587	}
3588
3589	/* Initialize this VAR_DECL from the equivalent argument. Convert
3590	the argument to the proper type in case it was promoted. */
3591	if (value)
3592	{
3593	if (rhs == error_mark_node)
3594	{
3595	insert_decl_map (id, key: p, value: var);
3596	return insert_init_debug_bind (id, bb, var, value: rhs, NULL);
3597	}
3598
3599	STRIP_USELESS_TYPE_CONVERSION (rhs);
3600
3601	/* If we are in SSA form properly remap the default definition. */
3602	if (gimple_in_ssa_p (cfun) && is_gimple_reg (p))
3603	{
3604	if (def)
3605	{
3606	def = remap_ssa_name (name: def, id);
3607	init_stmt = gimple_build_assign (def, rhs);
3608	SSA_NAME_IS_DEFAULT_DEF (def) = 0;
3609	set_ssa_default_def (cfun, var, NULL);
3610	}
3611	}
3612	else if (!is_empty_type (TREE_TYPE (var)))
3613	init_stmt = gimple_build_assign (var, rhs);
3614
3615	if (bb && init_stmt)
3616	insert_init_stmt (id, bb, init_stmt);
3617	}
3618	return init_stmt;
3619	}
3620
3621	/* Generate code to initialize the parameters of the function at the
3622	top of the stack in ID from the GIMPLE_CALL STMT. */
3623
3624	static void
3625	initialize_inlined_parameters (copy_body_data *id, gimple *stmt,
3626	tree fn, basic_block bb)
3627	{
3628	tree parms;
3629	size_t i;
3630	tree p;
3631	tree vars = NULL_TREE;
3632	tree static_chain = gimple_call_chain (gs: stmt);
3633
3634	/* Figure out what the parameters are. */
3635	parms = DECL_ARGUMENTS (fn);
3636
3637	/* Loop through the parameter declarations, replacing each with an
3638	equivalent VAR_DECL, appropriately initialized. */
3639	for (p = parms, i = 0; p; p = DECL_CHAIN (p), i++)
3640	{
3641	tree val;
3642	val = i < gimple_call_num_args (gs: stmt) ? gimple_call_arg (gs: stmt, index: i) : NULL;
3643	setup_one_parameter (id, p, value: val, fn, bb, vars: &vars);
3644	}
3645	/* After remapping parameters remap their types. This has to be done
3646	in a second loop over all parameters to appropriately remap
3647	variable sized arrays when the size is specified in a
3648	parameter following the array. */
3649	for (p = parms, i = 0; p; p = DECL_CHAIN (p), i++)
3650	{
3651	tree *varp = id->decl_map->get (k: p);
3652	if (varp && VAR_P (*varp))
3653	{
3654	tree def = (gimple_in_ssa_p (cfun) && is_gimple_reg (p)
3655	? ssa_default_def (id->src_cfun, p) : NULL);
3656	tree var = *varp;
3657	TREE_TYPE (var) = remap_type (TREE_TYPE (var), id);
3658	/* Also remap the default definition if it was remapped
3659	to the default definition of the parameter replacement
3660	by the parameter setup. */
3661	if (def)
3662	{
3663	tree *defp = id->decl_map->get (k: def);
3664	if (defp
3665	&& TREE_CODE (*defp) == SSA_NAME
3666	&& SSA_NAME_VAR (*defp) == var)
3667	TREE_TYPE (*defp) = TREE_TYPE (var);
3668	}
3669	/* When not optimizing and the parameter is unused, assign to
3670	a dummy SSA name. Do this after remapping the type above. */
3671	else if (!optimize
3672	&& is_gimple_reg (p)
3673	&& i < gimple_call_num_args (gs: stmt))
3674	{
3675	tree val = gimple_call_arg (gs: stmt, index: i);
3676	if (val != error_mark_node)
3677	{
3678	if (!useless_type_conversion_p (TREE_TYPE (p),
3679	TREE_TYPE (val)))
3680	val = force_value_to_type (TREE_TYPE (p), value: val);
3681	def = make_ssa_name (var);
3682	gimple *init_stmt = gimple_build_assign (def, val);
3683	insert_init_stmt (id, bb, init_stmt);
3684	}
3685	}
3686	}
3687	}
3688
3689	/* Initialize the static chain. */
3690	p = DECL_STRUCT_FUNCTION (fn)->static_chain_decl;
3691	gcc_assert (fn != current_function_decl);
3692	if (p)
3693	{
3694	/* No static chain? Seems like a bug in tree-nested.cc. */
3695	gcc_assert (static_chain);
3696
3697	setup_one_parameter (id, p, value: static_chain, fn, bb, vars: &vars);
3698	}
3699
3700	/* Reverse so the variables appear in the correct order in DWARF
3701	debug info. */
3702	vars = nreverse (vars);
3703
3704	declare_inline_vars (id->block, vars);
3705	}
3706
3707
3708	/* Declare a return variable to replace the RESULT_DECL for the
3709	function we are calling. An appropriate DECL_STMT is returned.
3710	The USE_STMT is filled to contain a use of the declaration to
3711	indicate the return value of the function.
3712
3713	RETURN_SLOT, if non-null is place where to store the result. It
3714	is set only for CALL_EXPR_RETURN_SLOT_OPT. MODIFY_DEST, if non-null,
3715	was the LHS of the MODIFY_EXPR to which this call is the RHS.
3716
3717	The return value is a (possibly null) value that holds the result
3718	as seen by the caller. */
3719
3720	static tree
3721	declare_return_variable (copy_body_data *id, tree return_slot, tree modify_dest,
3722	basic_block entry_bb)
3723	{
3724	tree callee = id->src_fn;
3725	tree result = DECL_RESULT (callee);
3726	tree callee_type = TREE_TYPE (result);
3727	tree caller_type;
3728	tree var, use;
3729
3730	/* Handle type-mismatches in the function declaration return type
3731	vs. the call expression. */
3732	if (modify_dest)
3733	caller_type = TREE_TYPE (modify_dest);
3734	else if (return_slot)
3735	caller_type = TREE_TYPE (return_slot);
3736	else /* No LHS on the call. */
3737	caller_type = TREE_TYPE (TREE_TYPE (callee));
3738
3739	/* We don't need to do anything for functions that don't return anything. */
3740	if (VOID_TYPE_P (callee_type))
3741	return NULL_TREE;
3742
3743	/* If there was a return slot, then the return value is the
3744	dereferenced address of that object. */
3745	if (return_slot)
3746	{
3747	/* The front end shouldn't have used both return_slot and
3748	a modify expression. */
3749	gcc_assert (!modify_dest);
3750	if (DECL_BY_REFERENCE (result))
3751	{
3752	tree return_slot_addr = build_fold_addr_expr (return_slot);
3753	STRIP_USELESS_TYPE_CONVERSION (return_slot_addr);
3754
3755	/* We are going to construct *&return_slot and we can't do that
3756	for variables believed to be not addressable.
3757
3758	FIXME: This check possibly can match, because values returned
3759	via return slot optimization are not believed to have address
3760	taken by alias analysis. */
3761	gcc_assert (TREE_CODE (return_slot) != SSA_NAME);
3762	var = return_slot_addr;
3763	mark_addressable (return_slot);
3764	}
3765	else
3766	{
3767	var = return_slot;
3768	gcc_assert (TREE_CODE (var) != SSA_NAME);
3769	if (TREE_ADDRESSABLE (result))
3770	mark_addressable (var);
3771	}
3772	if (DECL_NOT_GIMPLE_REG_P (result)
3773	&& DECL_P (var))
3774	DECL_NOT_GIMPLE_REG_P (var) = 1;
3775
3776	if (!useless_type_conversion_p (callee_type, caller_type))
3777	var = build1 (VIEW_CONVERT_EXPR, callee_type, var);
3778
3779	use = NULL;
3780	goto done;
3781	}
3782
3783	/* All types requiring non-trivial constructors should have been handled. */
3784	gcc_assert (!TREE_ADDRESSABLE (callee_type));
3785
3786	/* Attempt to avoid creating a new temporary variable. */
3787	if (modify_dest
3788	&& TREE_CODE (modify_dest) != SSA_NAME)
3789	{
3790	bool use_it = false;
3791
3792	/* We can't use MODIFY_DEST if there's type promotion involved. */
3793	if (!useless_type_conversion_p (callee_type, caller_type))
3794	use_it = false;
3795
3796	/* ??? If we're assigning to a variable sized type, then we must
3797	reuse the destination variable, because we've no good way to
3798	create variable sized temporaries at this point. */
3799	else if (!poly_int_tree_p (TYPE_SIZE_UNIT (caller_type)))
3800	use_it = true;
3801
3802	/* If the callee cannot possibly modify MODIFY_DEST, then we can
3803	reuse it as the result of the call directly. Don't do this if
3804	it would promote MODIFY_DEST to addressable. */
3805	else if (TREE_ADDRESSABLE (result))
3806	use_it = false;
3807	else
3808	{
3809	tree base_m = get_base_address (t: modify_dest);
3810
3811	/* If the base isn't a decl, then it's a pointer, and we don't
3812	know where that's going to go. */
3813	if (!DECL_P (base_m))
3814	use_it = false;
3815	else if (is_global_var (t: base_m))
3816	use_it = false;
3817	else if (DECL_NOT_GIMPLE_REG_P (result)
3818	&& !DECL_NOT_GIMPLE_REG_P (base_m))
3819	use_it = false;
3820	else if (!TREE_ADDRESSABLE (base_m))
3821	use_it = true;
3822	}
3823
3824	if (use_it)
3825	{
3826	var = modify_dest;
3827	use = NULL;
3828	goto done;
3829	}
3830	}
3831
3832	gcc_assert (poly_int_tree_p (TYPE_SIZE_UNIT (callee_type)));
3833
3834	var = copy_result_decl_to_var (result, id);
3835	DECL_SEEN_IN_BIND_EXPR_P (var) = 1;
3836
3837	/* Do not have the rest of GCC warn about this variable as it should
3838	not be visible to the user. */
3839	suppress_warning (var /* OPT_Wuninitialized? */);
3840
3841	declare_inline_vars (id->block, var);
3842
3843	/* Build the use expr. If the return type of the function was
3844	promoted, convert it back to the expected type. */
3845	use = var;
3846	if (!useless_type_conversion_p (caller_type, TREE_TYPE (var)))
3847	{
3848	/* If we can match up types by promotion/demotion do so. */
3849	if (fold_convertible_p (caller_type, var))
3850	use = fold_convert (caller_type, var);
3851	else
3852	{
3853	/* ??? For valid programs we should not end up here.
3854	Still if we end up with truly mismatched types here, fall back
3855	to using a MEM_REF to not leak invalid GIMPLE to the following
3856	passes. */
3857	/* Prevent var from being written into SSA form. */
3858	if (is_gimple_reg_type (TREE_TYPE (var)))
3859	DECL_NOT_GIMPLE_REG_P (var) = true;
3860	use = fold_build2 (MEM_REF, caller_type,
3861	build_fold_addr_expr (var),
3862	build_int_cst (ptr_type_node, 0));
3863	}
3864	}
3865
3866	STRIP_USELESS_TYPE_CONVERSION (use);
3867
3868	if (DECL_BY_REFERENCE (result))
3869	{
3870	TREE_ADDRESSABLE (var) = 1;
3871	var = build_fold_addr_expr (var);
3872	}
3873
3874	done:
3875	/* Register the VAR_DECL as the equivalent for the RESULT_DECL; that
3876	way, when the RESULT_DECL is encountered, it will be
3877	automatically replaced by the VAR_DECL.
3878
3879	When returning by reference, ensure that RESULT_DECL remaps to
3880	gimple_val. */
3881	if (DECL_BY_REFERENCE (result)
3882	&& !is_gimple_val (var))
3883	{
3884	tree temp = create_tmp_var (TREE_TYPE (result), "retvalptr");
3885	insert_decl_map (id, key: result, value: temp);
3886	/* When RESULT_DECL is in SSA form, we need to remap and initialize
3887	it's default_def SSA_NAME. */
3888	if (gimple_in_ssa_p (fun: id->src_cfun)
3889	&& is_gimple_reg (result))
3890	if (tree default_def = ssa_default_def (id->src_cfun, result))
3891	{
3892	temp = make_ssa_name (var: temp);
3893	insert_decl_map (id, key: default_def, value: temp);
3894	}
3895	insert_init_stmt (id, bb: entry_bb, init_stmt: gimple_build_assign (temp, var));
3896	}
3897	else
3898	insert_decl_map (id, key: result, value: var);
3899
3900	/* Remember this so we can ignore it in remap_decls. */
3901	id->retvar = var;
3902	return use;
3903	}
3904
3905	/* Determine if the function can be copied. If so return NULL. If
3906	not return a string describng the reason for failure. */
3907
3908	const char *
3909	copy_forbidden (struct function *fun)
3910	{
3911	const char *reason = fun->cannot_be_copied_reason;
3912
3913	/* Only examine the function once. */
3914	if (fun->cannot_be_copied_set)
3915	return reason;
3916
3917	/* We cannot copy a function that receives a non-local goto
3918	because we cannot remap the destination label used in the
3919	function that is performing the non-local goto. */
3920	/* ??? Actually, this should be possible, if we work at it.
3921	No doubt there's just a handful of places that simply
3922	assume it doesn't happen and don't substitute properly. */
3923	if (fun->has_nonlocal_label)
3924	{
3925	reason = G_("function %q+F can never be copied "
3926	"because it receives a non-local goto");
3927	goto fail;
3928	}
3929
3930	if (fun->has_forced_label_in_static)
3931	{
3932	reason = G_("function %q+F can never be copied because it saves "
3933	"address of local label in a static variable");
3934	goto fail;
3935	}
3936
3937	fail:
3938	fun->cannot_be_copied_reason = reason;
3939	fun->cannot_be_copied_set = true;
3940	return reason;
3941	}
3942
3943
3944	static const char *inline_forbidden_reason;
3945
3946	/* A callback for walk_gimple_seq to handle statements. Returns non-null
3947	iff a function cannot be inlined. Also sets the reason why. */
3948
3949	static tree
3950	inline_forbidden_p_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
3951	struct walk_stmt_info *wip)
3952	{
3953	tree fn = (tree) wip->info;
3954	tree t;
3955	gimple *stmt = gsi_stmt (i: *gsi);
3956
3957	switch (gimple_code (g: stmt))
3958	{
3959	case GIMPLE_CALL:
3960	/* Refuse to inline alloca call unless user explicitly forced so as
3961	this may change program's memory overhead drastically when the
3962	function using alloca is called in loop. In GCC present in
3963	SPEC2000 inlining into schedule_block cause it to require 2GB of
3964	RAM instead of 256MB. Don't do so for alloca calls emitted for
3965	VLA objects as those can't cause unbounded growth (they're always
3966	wrapped inside stack_save/stack_restore regions. */
3967	if (gimple_maybe_alloca_call_p (stmt)
3968	&& !gimple_call_alloca_for_var_p (s: as_a <gcall *> (p: stmt))
3969	&& !lookup_attribute (attr_name: "always_inline", DECL_ATTRIBUTES (fn)))
3970	{
3971	inline_forbidden_reason
3972	= G_("function %q+F can never be inlined because it uses "
3973	"alloca (override using the always_inline attribute)");
3974	*handled_ops_p = true;
3975	return fn;
3976	}
3977
3978	t = gimple_call_fndecl (gs: stmt);
3979	if (t == NULL_TREE)
3980	break;
3981
3982	/* We cannot inline functions that call setjmp. */
3983	if (setjmp_call_p (t))
3984	{
3985	inline_forbidden_reason
3986	= G_("function %q+F can never be inlined because it uses setjmp");
3987	*handled_ops_p = true;
3988	return t;
3989	}
3990
3991	if (DECL_BUILT_IN_CLASS (t) == BUILT_IN_NORMAL)
3992	switch (DECL_FUNCTION_CODE (decl: t))
3993	{
3994	/* We cannot inline functions that take a variable number of
3995	arguments. */
3996	case BUILT_IN_VA_START:
3997	case BUILT_IN_NEXT_ARG:
3998	case BUILT_IN_VA_END:
3999	inline_forbidden_reason
4000	= G_("function %q+F can never be inlined because it "
4001	"uses variable argument lists");
4002	*handled_ops_p = true;
4003	return t;
4004
4005	case BUILT_IN_LONGJMP:
4006	/* We can't inline functions that call __builtin_longjmp at
4007	all. The non-local goto machinery really requires the
4008	destination be in a different function. If we allow the
4009	function calling __builtin_longjmp to be inlined into the
4010	function calling __builtin_setjmp, Things will Go Awry. */
4011	inline_forbidden_reason
4012	= G_("function %q+F can never be inlined because "
4013	"it uses setjmp-longjmp exception handling");
4014	*handled_ops_p = true;
4015	return t;
4016
4017	case BUILT_IN_NONLOCAL_GOTO:
4018	/* Similarly. */
4019	inline_forbidden_reason
4020	= G_("function %q+F can never be inlined because "
4021	"it uses non-local goto");
4022	*handled_ops_p = true;
4023	return t;
4024
4025	case BUILT_IN_RETURN:
4026	case BUILT_IN_APPLY_ARGS:
4027	/* If a __builtin_apply_args caller would be inlined,
4028	it would be saving arguments of the function it has
4029	been inlined into. Similarly __builtin_return would
4030	return from the function the inline has been inlined into. */
4031	inline_forbidden_reason
4032	= G_("function %q+F can never be inlined because "
4033	"it uses %<__builtin_return%> or %<__builtin_apply_args%>");
4034	*handled_ops_p = true;
4035	return t;
4036
4037	default:
4038	break;
4039	}
4040	break;
4041
4042	case GIMPLE_GOTO:
4043	t = gimple_goto_dest (gs: stmt);
4044
4045	/* We will not inline a function which uses computed goto. The
4046	addresses of its local labels, which may be tucked into
4047	global storage, are of course not constant across
4048	instantiations, which causes unexpected behavior. */
4049	if (TREE_CODE (t) != LABEL_DECL)
4050	{
4051	inline_forbidden_reason
4052	= G_("function %q+F can never be inlined "
4053	"because it contains a computed goto");
4054	*handled_ops_p = true;
4055	return t;
4056	}
4057	break;
4058
4059	default:
4060	break;
4061	}
4062
4063	*handled_ops_p = false;
4064	return NULL_TREE;
4065	}
4066
4067	/* Return true if FNDECL is a function that cannot be inlined into
4068	another one. */
4069
4070	static bool
4071	inline_forbidden_p (tree fndecl)
4072	{
4073	struct function *fun = DECL_STRUCT_FUNCTION (fndecl);
4074	struct walk_stmt_info wi;
4075	basic_block bb;
4076	bool forbidden_p = false;
4077
4078	/* First check for shared reasons not to copy the code. */
4079	inline_forbidden_reason = copy_forbidden (fun);
4080	if (inline_forbidden_reason != NULL)
4081	return true;
4082
4083	/* Next, walk the statements of the function looking for
4084	constraucts we can't handle, or are non-optimal for inlining. */
4085	hash_set<tree> visited_nodes;
4086	memset (s: &wi, c: 0, n: sizeof (wi));
4087	wi.info = (void *) fndecl;
4088	wi.pset = &visited_nodes;
4089
4090	/* We cannot inline a function with a variable-sized parameter because we
4091	cannot materialize a temporary of such a type in the caller if need be.
4092	Note that the return case is not symmetrical because we can guarantee
4093	that a temporary is not needed by means of CALL_EXPR_RETURN_SLOT_OPT. */
4094	for (tree parm = DECL_ARGUMENTS (fndecl); parm; parm = DECL_CHAIN (parm))
4095	if (!poly_int_tree_p (DECL_SIZE (parm)))
4096	{
4097	inline_forbidden_reason
4098	= G_("function %q+F can never be inlined because "
4099	"it has a VLA argument");
4100	return true;
4101	}
4102
4103	FOR_EACH_BB_FN (bb, fun)
4104	{
4105	gimple *ret;
4106	gimple_seq seq = bb_seq (bb);
4107	ret = walk_gimple_seq (seq, inline_forbidden_p_stmt, NULL, &wi);
4108	forbidden_p = (ret != NULL);
4109	if (forbidden_p)
4110	break;
4111	}
4112
4113	return forbidden_p;
4114	}
4115
4116	/* Return false if the function FNDECL cannot be inlined on account of its
4117	attributes, true otherwise. */
4118	static bool
4119	function_attribute_inlinable_p (const_tree fndecl)
4120	{
4121	for (auto scoped_attributes : targetm.attribute_table)
4122	{
4123	const_tree a;
4124
4125	for (a = DECL_ATTRIBUTES (fndecl); a; a = TREE_CHAIN (a))
4126	{
4127	const_tree name = get_attribute_name (a);
4128
4129	for (const attribute_spec &attribute : scoped_attributes->attributes)
4130	if (is_attribute_p (attr_name: attribute.name, ident: name))
4131	return targetm.function_attribute_inlinable_p (fndecl);
4132	}
4133	}
4134
4135	return true;
4136	}
4137
4138	/* Returns nonzero if FN is a function that does not have any
4139	fundamental inline blocking properties. */
4140
4141	bool
4142	tree_inlinable_function_p (tree fn)
4143	{
4144	bool inlinable = true;
4145	bool do_warning;
4146	tree always_inline;
4147
4148	/* If we've already decided this function shouldn't be inlined,
4149	there's no need to check again. */
4150	if (DECL_UNINLINABLE (fn))
4151	return false;
4152
4153	/* We only warn for functions declared `inline' by the user. */
4154	do_warning = (opt_for_fn (fn, warn_inline)
4155	&& DECL_DECLARED_INLINE_P (fn)
4156	&& !DECL_NO_INLINE_WARNING_P (fn)
4157	&& !DECL_IN_SYSTEM_HEADER (fn));
4158
4159	always_inline = lookup_attribute (attr_name: "always_inline", DECL_ATTRIBUTES (fn));
4160
4161	if (flag_no_inline
4162	&& always_inline == NULL)
4163	{
4164	if (do_warning)
4165	warning (OPT_Winline, "function %q+F can never be inlined because it "
4166	"is suppressed using %<-fno-inline%>", fn);
4167	inlinable = false;
4168	}
4169
4170	else if (!function_attribute_inlinable_p (fndecl: fn))
4171	{
4172	if (do_warning)
4173	warning (OPT_Winline, "function %q+F can never be inlined because it "
4174	"uses attributes conflicting with inlining", fn);
4175	inlinable = false;
4176	}
4177
4178	else if (inline_forbidden_p (fndecl: fn))
4179	{
4180	/* See if we should warn about uninlinable functions. Previously,
4181	some of these warnings would be issued while trying to expand
4182	the function inline, but that would cause multiple warnings
4183	about functions that would for example call alloca. But since
4184	this a property of the function, just one warning is enough.
4185	As a bonus we can now give more details about the reason why a
4186	function is not inlinable. */
4187	if (always_inline)
4188	error (inline_forbidden_reason, fn);
4189	else if (do_warning)
4190	warning (OPT_Winline, inline_forbidden_reason, fn);
4191
4192	inlinable = false;
4193	}
4194
4195	/* Squirrel away the result so that we don't have to check again. */
4196	DECL_UNINLINABLE (fn) = !inlinable;
4197
4198	return inlinable;
4199	}
4200
4201	/* Estimate the cost of a memory move of type TYPE. Use machine dependent
4202	word size and take possible memcpy call into account and return
4203	cost based on whether optimizing for size or speed according to SPEED_P. */
4204
4205	int
4206	estimate_move_cost (tree type, bool ARG_UNUSED (speed_p))
4207	{
4208	HOST_WIDE_INT size;
4209
4210	gcc_assert (!VOID_TYPE_P (type));
4211
4212	if (VECTOR_TYPE_P (type))
4213	{
4214	scalar_mode inner = SCALAR_TYPE_MODE (TREE_TYPE (type));
4215	machine_mode simd = targetm.vectorize.preferred_simd_mode (inner);
4216	int orig_mode_size
4217	= estimated_poly_value (x: GET_MODE_SIZE (TYPE_MODE (type)));
4218	int simd_mode_size = estimated_poly_value (x: GET_MODE_SIZE (mode: simd));
4219	return ((orig_mode_size + simd_mode_size - 1)
4220	/ simd_mode_size);
4221	}
4222
4223	size = int_size_in_bytes (type);
4224
4225	if (size < 0 || size > MOVE_MAX_PIECES * MOVE_RATIO (speed_p))
4226	/* Cost of a memcpy call, 3 arguments and the call. */
4227	return 4;
4228	else
4229	return ((size + MOVE_MAX_PIECES - 1) / MOVE_MAX_PIECES);
4230	}
4231
4232	/* Returns cost of operation CODE, according to WEIGHTS */
4233
4234	static int
4235	estimate_operator_cost (enum tree_code code, eni_weights *weights,
4236	tree op1 ATTRIBUTE_UNUSED, tree op2)
4237	{
4238	switch (code)
4239	{
4240	/* These are "free" conversions, or their presumed cost
4241	is folded into other operations. */
4242	case RANGE_EXPR:
4243	CASE_CONVERT:
4244	case COMPLEX_EXPR:
4245	case PAREN_EXPR:
4246	case VIEW_CONVERT_EXPR:
4247	return 0;
4248
4249	/* Assign cost of 1 to usual operations.
4250	??? We may consider mapping RTL costs to this. */
4251	case COND_EXPR:
4252	case VEC_COND_EXPR:
4253	case VEC_PERM_EXPR:
4254
4255	case PLUS_EXPR:
4256	case POINTER_PLUS_EXPR:
4257	case POINTER_DIFF_EXPR:
4258	case MINUS_EXPR:
4259	case MULT_EXPR:
4260	case MULT_HIGHPART_EXPR:
4261
4262	case ADDR_SPACE_CONVERT_EXPR:
4263	case FIXED_CONVERT_EXPR:
4264	case FIX_TRUNC_EXPR:
4265
4266	case NEGATE_EXPR:
4267	case FLOAT_EXPR:
4268	case MIN_EXPR:
4269	case MAX_EXPR:
4270	case ABS_EXPR:
4271	case ABSU_EXPR:
4272
4273	case LSHIFT_EXPR:
4274	case RSHIFT_EXPR:
4275	case LROTATE_EXPR:
4276	case RROTATE_EXPR:
4277
4278	case BIT_IOR_EXPR:
4279	case BIT_XOR_EXPR:
4280	case BIT_AND_EXPR:
4281	case BIT_NOT_EXPR:
4282
4283	case TRUTH_ANDIF_EXPR:
4284	case TRUTH_ORIF_EXPR:
4285	case TRUTH_AND_EXPR:
4286	case TRUTH_OR_EXPR:
4287	case TRUTH_XOR_EXPR:
4288	case TRUTH_NOT_EXPR:
4289
4290	case LT_EXPR:
4291	case LE_EXPR:
4292	case GT_EXPR:
4293	case GE_EXPR:
4294	case EQ_EXPR:
4295	case NE_EXPR:
4296	case ORDERED_EXPR:
4297	case UNORDERED_EXPR:
4298
4299	case UNLT_EXPR:
4300	case UNLE_EXPR:
4301	case UNGT_EXPR:
4302	case UNGE_EXPR:
4303	case UNEQ_EXPR:
4304	case LTGT_EXPR:
4305
4306	case CONJ_EXPR:
4307
4308	case PREDECREMENT_EXPR:
4309	case PREINCREMENT_EXPR:
4310	case POSTDECREMENT_EXPR:
4311	case POSTINCREMENT_EXPR:
4312
4313	case REALIGN_LOAD_EXPR:
4314
4315	case WIDEN_SUM_EXPR:
4316	case WIDEN_MULT_EXPR:
4317	case DOT_PROD_EXPR:
4318	case SAD_EXPR:
4319	case WIDEN_MULT_PLUS_EXPR:
4320	case WIDEN_MULT_MINUS_EXPR:
4321	case WIDEN_LSHIFT_EXPR:
4322
4323	case VEC_WIDEN_MULT_HI_EXPR:
4324	case VEC_WIDEN_MULT_LO_EXPR:
4325	case VEC_WIDEN_MULT_EVEN_EXPR:
4326	case VEC_WIDEN_MULT_ODD_EXPR:
4327	case VEC_UNPACK_HI_EXPR:
4328	case VEC_UNPACK_LO_EXPR:
4329	case VEC_UNPACK_FLOAT_HI_EXPR:
4330	case VEC_UNPACK_FLOAT_LO_EXPR:
4331	case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
4332	case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
4333	case VEC_PACK_TRUNC_EXPR:
4334	case VEC_PACK_SAT_EXPR:
4335	case VEC_PACK_FIX_TRUNC_EXPR:
4336	case VEC_PACK_FLOAT_EXPR:
4337	case VEC_WIDEN_LSHIFT_HI_EXPR:
4338	case VEC_WIDEN_LSHIFT_LO_EXPR:
4339	case VEC_DUPLICATE_EXPR:
4340	case VEC_SERIES_EXPR:
4341
4342	return 1;
4343
4344	/* Few special cases of expensive operations. This is useful
4345	to avoid inlining on functions having too many of these. */
4346	case TRUNC_DIV_EXPR:
4347	case CEIL_DIV_EXPR:
4348	case FLOOR_DIV_EXPR:
4349	case ROUND_DIV_EXPR:
4350	case EXACT_DIV_EXPR:
4351	case TRUNC_MOD_EXPR:
4352	case CEIL_MOD_EXPR:
4353	case FLOOR_MOD_EXPR:
4354	case ROUND_MOD_EXPR:
4355	case RDIV_EXPR:
4356	if (TREE_CODE (op2) != INTEGER_CST)
4357	return weights->div_mod_cost;
4358	return 1;
4359
4360	/* Bit-field insertion needs several shift and mask operations. */
4361	case BIT_INSERT_EXPR:
4362	return 3;
4363
4364	default:
4365	/* We expect a copy assignment with no operator. */
4366	gcc_assert (get_gimple_rhs_class (code) == GIMPLE_SINGLE_RHS);
4367	return 0;
4368	}
4369	}
4370
4371
4372	/* Estimate number of instructions that will be created by expanding
4373	the statements in the statement sequence STMTS.
4374	WEIGHTS contains weights attributed to various constructs. */
4375
4376	int
4377	estimate_num_insns_seq (gimple_seq stmts, eni_weights *weights)
4378	{
4379	int cost;
4380	gimple_stmt_iterator gsi;
4381
4382	cost = 0;
4383	for (gsi = gsi_start (seq&: stmts); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
4384	cost += estimate_num_insns (gsi_stmt (i: gsi), weights);
4385
4386	return cost;
4387	}
4388
4389
4390	/* Estimate number of instructions that will be created by expanding STMT.
4391	WEIGHTS contains weights attributed to various constructs. */
4392
4393	int
4394	estimate_num_insns (gimple *stmt, eni_weights *weights)
4395	{
4396	unsigned cost, i;
4397	enum gimple_code code = gimple_code (g: stmt);
4398	tree lhs;
4399	tree rhs;
4400
4401	switch (code)
4402	{
4403	case GIMPLE_ASSIGN:
4404	/* Try to estimate the cost of assignments. We have three cases to
4405	deal with:
4406	1) Simple assignments to registers;
4407	2) Stores to things that must live in memory. This includes
4408	"normal" stores to scalars, but also assignments of large
4409	structures, or constructors of big arrays;
4410
4411	Let us look at the first two cases, assuming we have "a = b + C":
4412	<GIMPLE_ASSIGN <var_decl "a">
4413	<plus_expr <var_decl "b"> <constant C>>
4414	If "a" is a GIMPLE register, the assignment to it is free on almost
4415	any target, because "a" usually ends up in a real register. Hence
4416	the only cost of this expression comes from the PLUS_EXPR, and we
4417	can ignore the GIMPLE_ASSIGN.
4418	If "a" is not a GIMPLE register, the assignment to "a" will most
4419	likely be a real store, so the cost of the GIMPLE_ASSIGN is the cost
4420	of moving something into "a", which we compute using the function
4421	estimate_move_cost. */
4422	if (gimple_clobber_p (s: stmt))
4423	return 0; /* ={v} {CLOBBER} stmt expands to nothing. */
4424
4425	lhs = gimple_assign_lhs (gs: stmt);
4426	rhs = gimple_assign_rhs1 (gs: stmt);
4427
4428	cost = 0;
4429
4430	/* Account for the cost of moving to / from memory. */
4431	if (gimple_store_p (gs: stmt))
4432	cost += estimate_move_cost (TREE_TYPE (lhs), speed_p: weights->time_based);
4433	if (gimple_assign_load_p (stmt))
4434	cost += estimate_move_cost (TREE_TYPE (rhs), speed_p: weights->time_based);
4435
4436	cost += estimate_operator_cost (code: gimple_assign_rhs_code (gs: stmt), weights,
4437	op1: gimple_assign_rhs1 (gs: stmt),
4438	op2: get_gimple_rhs_class (code: gimple_assign_rhs_code (gs: stmt))
4439	== GIMPLE_BINARY_RHS
4440	? gimple_assign_rhs2 (gs: stmt) : NULL);
4441	break;
4442
4443	case GIMPLE_COND:
4444	cost = 1 + estimate_operator_cost (code: gimple_cond_code (gs: stmt), weights,
4445	op1: gimple_op (gs: stmt, i: 0),
4446	op2: gimple_op (gs: stmt, i: 1));
4447	break;
4448
4449	case GIMPLE_SWITCH:
4450	{
4451	gswitch *switch_stmt = as_a <gswitch *> (p: stmt);
4452	/* Take into account cost of the switch + guess 2 conditional jumps for
4453	each case label.
4454
4455	TODO: once the switch expansion logic is sufficiently separated, we can
4456	do better job on estimating cost of the switch. */
4457	if (weights->time_based)
4458	cost = floor_log2 (x: gimple_switch_num_labels (gs: switch_stmt)) * 2;
4459	else
4460	cost = gimple_switch_num_labels (gs: switch_stmt) * 2;
4461	}
4462	break;
4463
4464	case GIMPLE_CALL:
4465	{
4466	tree decl;
4467
4468	if (gimple_call_internal_p (gs: stmt))
4469	return 0;
4470	else if ((decl = gimple_call_fndecl (gs: stmt))
4471	&& fndecl_built_in_p (node: decl))
4472	{
4473	/* Do not special case builtins where we see the body.
4474	This just confuse inliner. */
4475	struct cgraph_node *node;
4476	if ((node = cgraph_node::get (decl))
4477	&& node->definition)
4478	;
4479	/* For buitins that are likely expanded to nothing or
4480	inlined do not account operand costs. */
4481	else if (is_simple_builtin (decl))
4482	return 0;
4483	else if (is_inexpensive_builtin (decl))
4484	return weights->target_builtin_call_cost;
4485	else if (gimple_call_builtin_p (stmt, BUILT_IN_NORMAL))
4486	{
4487	/* We canonicalize x * x to pow (x, 2.0) with -ffast-math, so
4488	specialize the cheap expansion we do here.
4489	??? This asks for a more general solution. */
4490	switch (DECL_FUNCTION_CODE (decl))
4491	{
4492	case BUILT_IN_POW:
4493	case BUILT_IN_POWF:
4494	case BUILT_IN_POWL:
4495	if (TREE_CODE (gimple_call_arg (stmt, 1)) == REAL_CST
4496	&& (real_equal
4497	(&TREE_REAL_CST (gimple_call_arg (stmt, 1)),
4498	&dconst2)))
4499	return estimate_operator_cost
4500	(code: MULT_EXPR, weights, op1: gimple_call_arg (gs: stmt, index: 0),
4501	op2: gimple_call_arg (gs: stmt, index: 0));
4502	break;
4503
4504	default:
4505	break;
4506	}
4507	}
4508	}
4509
4510	cost = decl ? weights->call_cost : weights->indirect_call_cost;
4511	if (gimple_call_lhs (gs: stmt))
4512	cost += estimate_move_cost (TREE_TYPE (gimple_call_lhs (stmt)),
4513	speed_p: weights->time_based);
4514	for (i = 0; i < gimple_call_num_args (gs: stmt); i++)
4515	{
4516	tree arg = gimple_call_arg (gs: stmt, index: i);
4517	cost += estimate_move_cost (TREE_TYPE (arg),
4518	speed_p: weights->time_based);
4519	}
4520	break;
4521	}
4522
4523	case GIMPLE_RETURN:
4524	return weights->return_cost;
4525
4526	case GIMPLE_GOTO:
4527	case GIMPLE_LABEL:
4528	case GIMPLE_NOP:
4529	case GIMPLE_PHI:
4530	case GIMPLE_PREDICT:
4531	case GIMPLE_DEBUG:
4532	return 0;
4533
4534	case GIMPLE_ASM:
4535	{
4536	int count = asm_str_count (templ: gimple_asm_string (asm_stmt: as_a <gasm *> (p: stmt)));
4537	/* 1000 means infinity. This avoids overflows later
4538	with very long asm statements. */
4539	if (count > 1000)
4540	count = 1000;
4541	/* If this asm is asm inline, count anything as minimum size. */
4542	if (gimple_asm_inline_p (asm_stmt: as_a <gasm *> (p: stmt)))
4543	count = MIN (1, count);
4544	return MAX (1, count);
4545	}
4546
4547	case GIMPLE_RESX:
4548	/* This is either going to be an external function call with one
4549	argument, or two register copy statements plus a goto. */
4550	return 2;
4551
4552	case GIMPLE_EH_DISPATCH:
4553	/* ??? This is going to turn into a switch statement. Ideally
4554	we'd have a look at the eh region and estimate the number of
4555	edges involved. */
4556	return 10;
4557
4558	case GIMPLE_BIND:
4559	return estimate_num_insns_seq (
4560	stmts: gimple_bind_body (gs: as_a <gbind *> (p: stmt)),
4561	weights);
4562
4563	case GIMPLE_EH_FILTER:
4564	return estimate_num_insns_seq (stmts: gimple_eh_filter_failure (gs: stmt), weights);
4565
4566	case GIMPLE_CATCH:
4567	return estimate_num_insns_seq (stmts: gimple_catch_handler (
4568	catch_stmt: as_a <gcatch *> (p: stmt)),
4569	weights);
4570
4571	case GIMPLE_TRY:
4572	return (estimate_num_insns_seq (stmts: gimple_try_eval (gs: stmt), weights)
4573	+ estimate_num_insns_seq (stmts: gimple_try_cleanup (gs: stmt), weights));
4574
4575	/* OMP directives are generally very expensive. */
4576
4577	case GIMPLE_OMP_RETURN:
4578	case GIMPLE_OMP_SECTIONS_SWITCH:
4579	case GIMPLE_OMP_ATOMIC_STORE:
4580	case GIMPLE_OMP_CONTINUE:
4581	/* ...except these, which are cheap. */
4582	return 0;
4583
4584	case GIMPLE_OMP_ATOMIC_LOAD:
4585	return weights->omp_cost;
4586
4587	case GIMPLE_OMP_FOR:
4588	return (weights->omp_cost
4589	+ estimate_num_insns_seq (stmts: gimple_omp_body (gs: stmt), weights)
4590	+ estimate_num_insns_seq (stmts: gimple_omp_for_pre_body (gs: stmt), weights));
4591
4592	case GIMPLE_OMP_PARALLEL:
4593	case GIMPLE_OMP_TASK:
4594	case GIMPLE_OMP_CRITICAL:
4595	case GIMPLE_OMP_MASTER:
4596	case GIMPLE_OMP_MASKED:
4597	case GIMPLE_OMP_SCOPE:
4598	case GIMPLE_OMP_TASKGROUP:
4599	case GIMPLE_OMP_ORDERED:
4600	case GIMPLE_OMP_SCAN:
4601	case GIMPLE_OMP_SECTION:
4602	case GIMPLE_OMP_SECTIONS:
4603	case GIMPLE_OMP_STRUCTURED_BLOCK:
4604	case GIMPLE_OMP_SINGLE:
4605	case GIMPLE_OMP_TARGET:
4606	case GIMPLE_OMP_TEAMS:
4607	return (weights->omp_cost
4608	+ estimate_num_insns_seq (stmts: gimple_omp_body (gs: stmt), weights));
4609
4610	case GIMPLE_TRANSACTION:
4611	return (weights->tm_cost
4612	+ estimate_num_insns_seq (stmts: gimple_transaction_body (
4613	transaction_stmt: as_a <gtransaction *> (p: stmt)),
4614	weights));
4615
4616	default:
4617	gcc_unreachable ();
4618	}
4619
4620	return cost;
4621	}
4622
4623	/* Estimate number of instructions that will be created by expanding
4624	function FNDECL. WEIGHTS contains weights attributed to various
4625	constructs. */
4626
4627	int
4628	estimate_num_insns_fn (tree fndecl, eni_weights *weights)
4629	{
4630	struct function *my_function = DECL_STRUCT_FUNCTION (fndecl);
4631	gimple_stmt_iterator bsi;
4632	basic_block bb;
4633	int n = 0;
4634
4635	gcc_assert (my_function && my_function->cfg);
4636	FOR_EACH_BB_FN (bb, my_function)
4637	{
4638	for (bsi = gsi_start_bb (bb); !gsi_end_p (i: bsi); gsi_next (i: &bsi))
4639	n += estimate_num_insns (stmt: gsi_stmt (i: bsi), weights);
4640	}
4641
4642	return n;
4643	}
4644
4645
4646	/* Initializes weights used by estimate_num_insns. */
4647
4648	void
4649	init_inline_once (void)
4650	{
4651	eni_size_weights.call_cost = 1;
4652	eni_size_weights.indirect_call_cost = 3;
4653	eni_size_weights.target_builtin_call_cost = 1;
4654	eni_size_weights.div_mod_cost = 1;
4655	eni_size_weights.omp_cost = 40;
4656	eni_size_weights.tm_cost = 10;
4657	eni_size_weights.time_based = false;
4658	eni_size_weights.return_cost = 1;
4659
4660	/* Estimating time for call is difficult, since we have no idea what the
4661	called function does. In the current uses of eni_time_weights,
4662	underestimating the cost does less harm than overestimating it, so
4663	we choose a rather small value here. */
4664	eni_time_weights.call_cost = 10;
4665	eni_time_weights.indirect_call_cost = 15;
4666	eni_time_weights.target_builtin_call_cost = 1;
4667	eni_time_weights.div_mod_cost = 10;
4668	eni_time_weights.omp_cost = 40;
4669	eni_time_weights.tm_cost = 40;
4670	eni_time_weights.time_based = true;
4671	eni_time_weights.return_cost = 2;
4672	}
4673
4674
4675	/* Install new lexical TREE_BLOCK underneath 'current_block'. */
4676
4677	static void
4678	prepend_lexical_block (tree current_block, tree new_block)
4679	{
4680	BLOCK_CHAIN (new_block) = BLOCK_SUBBLOCKS (current_block);
4681	BLOCK_SUBBLOCKS (current_block) = new_block;
4682	BLOCK_SUPERCONTEXT (new_block) = current_block;
4683	}
4684
4685	/* Add local variables from CALLEE to CALLER. */
4686
4687	static inline void
4688	add_local_variables (struct function *callee, struct function *caller,
4689	copy_body_data *id)
4690	{
4691	tree var;
4692	unsigned ix;
4693
4694	FOR_EACH_LOCAL_DECL (callee, ix, var)
4695	if (!can_be_nonlocal (decl: var, id))
4696	{
4697	tree new_var = remap_decl (decl: var, id);
4698
4699	/* Remap debug-expressions. */
4700	if (VAR_P (new_var)
4701	&& DECL_HAS_DEBUG_EXPR_P (var)
4702	&& new_var != var)
4703	{
4704	tree tem = DECL_DEBUG_EXPR (var);
4705	bool old_regimplify = id->regimplify;
4706	id->remapping_type_depth++;
4707	walk_tree (&tem, copy_tree_body_r, id, NULL);
4708	id->remapping_type_depth--;
4709	id->regimplify = old_regimplify;
4710	SET_DECL_DEBUG_EXPR (new_var, tem);
4711	DECL_HAS_DEBUG_EXPR_P (new_var) = 1;
4712	}
4713	add_local_decl (fun: caller, d: new_var);
4714	}
4715	}
4716
4717	/* Add to BINDINGS a debug stmt resetting SRCVAR if inlining might
4718	have brought in or introduced any debug stmts for SRCVAR. */
4719
4720	static inline void
4721	reset_debug_binding (copy_body_data *id, tree srcvar, gimple_seq *bindings)
4722	{
4723	tree *remappedvarp = id->decl_map->get (k: srcvar);
4724
4725	if (!remappedvarp)
4726	return;
4727
4728	if (!VAR_P (*remappedvarp))
4729	return;
4730
4731	if (*remappedvarp == id->retvar)
4732	return;
4733
4734	tree tvar = target_for_debug_bind (*remappedvarp);
4735	if (!tvar)
4736	return;
4737
4738	gdebug *stmt = gimple_build_debug_bind (tvar, NULL_TREE,
4739	id->call_stmt);
4740	gimple_seq_add_stmt (bindings, stmt);
4741	}
4742
4743	/* For each inlined variable for which we may have debug bind stmts,
4744	add before GSI a final debug stmt resetting it, marking the end of
4745	its life, so that var-tracking knows it doesn't have to compute
4746	further locations for it. */
4747
4748	static inline void
4749	reset_debug_bindings (copy_body_data *id, gimple_stmt_iterator gsi)
4750	{
4751	tree var;
4752	unsigned ix;
4753	gimple_seq bindings = NULL;
4754
4755	if (!gimple_in_ssa_p (fun: id->src_cfun))
4756	return;
4757
4758	if (!opt_for_fn (id->dst_fn, flag_var_tracking_assignments))
4759	return;
4760
4761	for (var = DECL_ARGUMENTS (id->src_fn);
4762	var; var = DECL_CHAIN (var))
4763	reset_debug_binding (id, srcvar: var, bindings: &bindings);
4764
4765	FOR_EACH_LOCAL_DECL (id->src_cfun, ix, var)
4766	reset_debug_binding (id, srcvar: var, bindings: &bindings);
4767
4768	gsi_insert_seq_before_without_update (&gsi, bindings, GSI_SAME_STMT);
4769	}
4770
4771	/* If STMT is a GIMPLE_CALL, replace it with its inline expansion. */
4772
4773	static bool
4774	expand_call_inline (basic_block bb, gimple *stmt, copy_body_data *id,
4775	bitmap to_purge)
4776	{
4777	tree use_retvar;
4778	tree fn;
4779	hash_map<tree, tree> *dst;
4780	hash_map<tree, tree> *st = NULL;
4781	tree return_slot;
4782	tree modify_dest;
4783	struct cgraph_edge *cg_edge;
4784	cgraph_inline_failed_t reason;
4785	basic_block return_block;
4786	edge e;
4787	gimple_stmt_iterator gsi, stmt_gsi;
4788	bool successfully_inlined = false;
4789	bool purge_dead_abnormal_edges;
4790	gcall *call_stmt;
4791	unsigned int prop_mask, src_properties;
4792	struct function *dst_cfun;
4793	tree simduid;
4794	use_operand_p use;
4795	gimple *simtenter_stmt = NULL;
4796	vec<tree> *simtvars_save;
4797
4798	/* The gimplifier uses input_location in too many places, such as
4799	internal_get_tmp_var (). */
4800	location_t saved_location = input_location;
4801	input_location = gimple_location (g: stmt);
4802
4803	/* From here on, we're only interested in CALL_EXPRs. */
4804	call_stmt = dyn_cast <gcall *> (p: stmt);
4805	if (!call_stmt)
4806	goto egress;
4807
4808	cg_edge = id->dst_node->get_edge (call_stmt: stmt);
4809	gcc_checking_assert (cg_edge);
4810	/* First, see if we can figure out what function is being called.
4811	If we cannot, then there is no hope of inlining the function. */
4812	if (cg_edge->indirect_unknown_callee)
4813	goto egress;
4814	fn = cg_edge->callee->decl;
4815	gcc_checking_assert (fn);
4816
4817	/* If FN is a declaration of a function in a nested scope that was
4818	globally declared inline, we don't set its DECL_INITIAL.
4819	However, we can't blindly follow DECL_ABSTRACT_ORIGIN because the
4820	C++ front-end uses it for cdtors to refer to their internal
4821	declarations, that are not real functions. Fortunately those
4822	don't have trees to be saved, so we can tell by checking their
4823	gimple_body. */
4824	if (!DECL_INITIAL (fn)
4825	&& DECL_ABSTRACT_ORIGIN (fn)
4826	&& gimple_has_body_p (DECL_ABSTRACT_ORIGIN (fn)))
4827	fn = DECL_ABSTRACT_ORIGIN (fn);
4828
4829	/* Don't try to inline functions that are not well-suited to inlining. */
4830	if (cg_edge->inline_failed)
4831	{
4832	reason = cg_edge->inline_failed;
4833	/* If this call was originally indirect, we do not want to emit any
4834	inlining related warnings or sorry messages because there are no
4835	guarantees regarding those. */
4836	if (cg_edge->indirect_inlining_edge)
4837	goto egress;
4838
4839	if (lookup_attribute (attr_name: "always_inline", DECL_ATTRIBUTES (fn))
4840	/* For extern inline functions that get redefined we always
4841	silently ignored always_inline flag. Better behavior would
4842	be to be able to keep both bodies and use extern inline body
4843	for inlining, but we can't do that because frontends overwrite
4844	the body. */
4845	&& !cg_edge->callee->redefined_extern_inline
4846	/* During early inline pass, report only when optimization is
4847	not turned on. */
4848	&& (symtab->global_info_ready
4849	|| !optimize
4850	|| cgraph_inline_failed_type (reason) == CIF_FINAL_ERROR)
4851	/* PR 20090218-1_0.c. Body can be provided by another module. */
4852	&& (reason != CIF_BODY_NOT_AVAILABLE || !flag_generate_lto))
4853	{
4854	error ("inlining failed in call to %<always_inline%> %q+F: %s", fn,
4855	cgraph_inline_failed_string (reason));
4856	if (gimple_location (g: stmt) != UNKNOWN_LOCATION)
4857	inform (gimple_location (g: stmt), "called from here");
4858	else if (DECL_SOURCE_LOCATION (cfun->decl) != UNKNOWN_LOCATION)
4859	inform (DECL_SOURCE_LOCATION (cfun->decl),
4860	"called from this function");
4861	}
4862	else if (opt_for_fn (fn, warn_inline)
4863	&& DECL_DECLARED_INLINE_P (fn)
4864	&& !DECL_NO_INLINE_WARNING_P (fn)
4865	&& !DECL_IN_SYSTEM_HEADER (fn)
4866	&& reason != CIF_UNSPECIFIED
4867	&& !lookup_attribute (attr_name: "noinline", DECL_ATTRIBUTES (fn))
4868	/* Do not warn about not inlined recursive calls. */
4869	&& !cg_edge->recursive_p ()
4870	/* Avoid warnings during early inline pass. */
4871	&& symtab->global_info_ready)
4872	{
4873	auto_diagnostic_group d;
4874	if (warning (OPT_Winline, "inlining failed in call to %q+F: %s",
4875	fn, _(cgraph_inline_failed_string (reason))))
4876	{
4877	if (gimple_location (g: stmt) != UNKNOWN_LOCATION)
4878	inform (gimple_location (g: stmt), "called from here");
4879	else if (DECL_SOURCE_LOCATION (cfun->decl) != UNKNOWN_LOCATION)
4880	inform (DECL_SOURCE_LOCATION (cfun->decl),
4881	"called from this function");
4882	}
4883	}
4884	goto egress;
4885	}
4886	id->src_node = cg_edge->callee;
4887
4888	/* If callee is thunk, all we need is to adjust the THIS pointer
4889	and redirect to function being thunked. */
4890	if (id->src_node->thunk)
4891	{
4892	cgraph_edge *edge;
4893	tree virtual_offset = NULL;
4894	profile_count count = cg_edge->count;
4895	tree op;
4896	gimple_stmt_iterator iter = gsi_for_stmt (stmt);
4897	thunk_info *info = thunk_info::get (node: id->src_node);
4898
4899	cgraph_edge::remove (edge: cg_edge);
4900	edge = id->src_node->callees->clone (n: id->dst_node, call_stmt,
4901	stmt_uid: gimple_uid (g: stmt),
4902	num: profile_count::one (),
4903	den: profile_count::one (),
4904	update_original: true);
4905	edge->count = count;
4906	if (info->virtual_offset_p)
4907	virtual_offset = size_int (info->virtual_value);
4908	op = create_tmp_reg_fn (cfun, TREE_TYPE (gimple_call_arg (stmt, 0)),
4909	NULL);
4910	gsi_insert_before (&iter, gimple_build_assign (op,
4911	gimple_call_arg (gs: stmt, index: 0)),
4912	GSI_NEW_STMT);
4913	gcc_assert (info->this_adjusting);
4914	op = thunk_adjust (&iter, op, 1, info->fixed_offset,
4915	virtual_offset, info->indirect_offset);
4916
4917	gimple_call_set_arg (gs: stmt, index: 0, arg: op);
4918	gimple_call_set_fndecl (gs: stmt, decl: edge->callee->decl);
4919	update_stmt (s: stmt);
4920	id->src_node->remove ();
4921	successfully_inlined = expand_call_inline (bb, stmt, id, to_purge);
4922	maybe_remove_unused_call_args (cfun, stmt);
4923	/* This used to return true even though we do fail to inline in
4924	some cases. See PR98525. */
4925	goto egress;
4926	}
4927	fn = cg_edge->callee->decl;
4928	cg_edge->callee->get_untransformed_body ();
4929
4930	if (flag_checking && cg_edge->callee->decl != id->dst_node->decl)
4931	cg_edge->callee->verify ();
4932
4933	/* We will be inlining this callee. */
4934	id->eh_lp_nr = lookup_stmt_eh_lp (stmt);
4935
4936	/* Update the callers EH personality. */
4937	if (DECL_FUNCTION_PERSONALITY (fn))
4938	DECL_FUNCTION_PERSONALITY (cg_edge->caller->decl)
4939	= DECL_FUNCTION_PERSONALITY (fn);
4940
4941	/* Split the block before the GIMPLE_CALL. */
4942	stmt_gsi = gsi_for_stmt (stmt);
4943	gsi_prev (i: &stmt_gsi);
4944	e = split_block (bb, gsi_end_p (i: stmt_gsi) ? NULL : gsi_stmt (i: stmt_gsi));
4945	bb = e->src;
4946	return_block = e->dest;
4947	remove_edge (e);
4948
4949	/* If the GIMPLE_CALL was in the last statement of BB, it may have
4950	been the source of abnormal edges. In this case, schedule
4951	the removal of dead abnormal edges. */
4952	gsi = gsi_start_bb (bb: return_block);
4953	gsi_next (i: &gsi);
4954	purge_dead_abnormal_edges = gsi_end_p (i: gsi);
4955
4956	stmt_gsi = gsi_start_bb (bb: return_block);
4957
4958	/* Build a block containing code to initialize the arguments, the
4959	actual inline expansion of the body, and a label for the return
4960	statements within the function to jump to. The type of the
4961	statement expression is the return type of the function call.
4962	??? If the call does not have an associated block then we will
4963	remap all callee blocks to NULL, effectively dropping most of
4964	its debug information. This should only happen for calls to
4965	artificial decls inserted by the compiler itself. We need to
4966	either link the inlined blocks into the caller block tree or
4967	not refer to them in any way to not break GC for locations. */
4968	if (tree block = gimple_block (g: stmt))
4969	{
4970	/* We do want to assign a not UNKNOWN_LOCATION BLOCK_SOURCE_LOCATION
4971	to make inlined_function_outer_scope_p return true on this BLOCK. */
4972	location_t loc = LOCATION_LOCUS (gimple_location (stmt));
4973	if (loc == UNKNOWN_LOCATION)
4974	loc = LOCATION_LOCUS (DECL_SOURCE_LOCATION (fn));
4975	if (loc == UNKNOWN_LOCATION)
4976	loc = BUILTINS_LOCATION;
4977	id->block = make_node (BLOCK);
4978	BLOCK_ABSTRACT_ORIGIN (id->block) = DECL_ORIGIN (fn);
4979	BLOCK_SOURCE_LOCATION (id->block) = loc;
4980	prepend_lexical_block (current_block: block, new_block: id->block);
4981	}
4982
4983	/* Local declarations will be replaced by their equivalents in this map. */
4984	st = id->decl_map;
4985	id->decl_map = new hash_map<tree, tree>;
4986	dst = id->debug_map;
4987	id->debug_map = NULL;
4988	if (flag_stack_reuse != SR_NONE)
4989	id->add_clobbers_to_eh_landing_pads = last_basic_block_for_fn (cfun);
4990
4991	/* Record the function we are about to inline. */
4992	id->src_fn = fn;
4993	id->src_cfun = DECL_STRUCT_FUNCTION (fn);
4994	id->reset_location = DECL_IGNORED_P (fn);
4995	id->call_stmt = call_stmt;
4996	cfun->cfg->full_profile &= id->src_cfun->cfg->full_profile;
4997
4998	/* When inlining into an OpenMP SIMD-on-SIMT loop, arrange for new automatic
4999	variables to be added to IFN_GOMP_SIMT_ENTER argument list. */
5000	dst_cfun = DECL_STRUCT_FUNCTION (id->dst_fn);
5001	simtvars_save = id->dst_simt_vars;
5002	if (!(dst_cfun->curr_properties & PROP_gimple_lomp_dev)
5003	&& (simduid = bb->loop_father->simduid) != NULL_TREE
5004	&& (simduid = ssa_default_def (dst_cfun, simduid)) != NULL_TREE
5005	&& single_imm_use (var: simduid, use_p: &use, stmt: &simtenter_stmt)
5006	&& is_gimple_call (gs: simtenter_stmt)
5007	&& gimple_call_internal_p (gs: simtenter_stmt, fn: IFN_GOMP_SIMT_ENTER))
5008	vec_alloc (v&: id->dst_simt_vars, nelems: 0);
5009	else
5010	id->dst_simt_vars = NULL;
5011
5012	if (profile_status_for_fn (id->src_cfun) == PROFILE_ABSENT)
5013	profile_status_for_fn (dst_cfun) = PROFILE_ABSENT;
5014
5015	/* If the src function contains an IFN_VA_ARG, then so will the dst
5016	function after inlining. Likewise for IFN_GOMP_USE_SIMT. */
5017	prop_mask = PROP_gimple_lva | PROP_gimple_lomp_dev;
5018	src_properties = id->src_cfun->curr_properties & prop_mask;
5019	if (src_properties != prop_mask)
5020	dst_cfun->curr_properties &= src_properties | ~prop_mask;
5021	dst_cfun->calls_eh_return |= id->src_cfun->calls_eh_return;
5022	id->dst_node->calls_declare_variant_alt
5023	|= id->src_node->calls_declare_variant_alt;
5024
5025	gcc_assert (!id->src_cfun->after_inlining);
5026
5027	id->entry_bb = bb;
5028	if (lookup_attribute (attr_name: "cold", DECL_ATTRIBUTES (fn)))
5029	{
5030	gimple_stmt_iterator si = gsi_last_bb (bb);
5031	gsi_insert_after (&si, gimple_build_predict (predictor: PRED_COLD_FUNCTION,
5032	outcome: NOT_TAKEN),
5033	GSI_NEW_STMT);
5034	}
5035	initialize_inlined_parameters (id, stmt, fn, bb);
5036	if (debug_nonbind_markers_p && debug_inline_points && id->block
5037	&& inlined_function_outer_scope_p (block: id->block))
5038	{
5039	gimple_stmt_iterator si = gsi_last_bb (bb);
5040	gsi_insert_after (&si, gimple_build_debug_inline_entry
5041	(id->block, DECL_SOURCE_LOCATION (id->src_fn)),
5042	GSI_NEW_STMT);
5043	}
5044
5045	if (DECL_INITIAL (fn))
5046	{
5047	if (gimple_block (g: stmt))
5048	{
5049	tree *var;
5050
5051	prepend_lexical_block (current_block: id->block,
5052	new_block: remap_blocks (DECL_INITIAL (fn), id));
5053	gcc_checking_assert (BLOCK_SUBBLOCKS (id->block)
5054	&& (BLOCK_CHAIN (BLOCK_SUBBLOCKS (id->block))
5055	== NULL_TREE));
5056	/* Move vars for PARM_DECLs from DECL_INITIAL block to id->block,
5057	otherwise for DWARF DW_TAG_formal_parameter will not be children of
5058	DW_TAG_inlined_subroutine, but of a DW_TAG_lexical_block
5059	under it. The parameters can be then evaluated in the debugger,
5060	but don't show in backtraces. */
5061	for (var = &BLOCK_VARS (BLOCK_SUBBLOCKS (id->block)); *var; )
5062	if (TREE_CODE (DECL_ORIGIN (*var)) == PARM_DECL)
5063	{
5064	tree v = *var;
5065	*var = TREE_CHAIN (v);
5066	TREE_CHAIN (v) = BLOCK_VARS (id->block);
5067	BLOCK_VARS (id->block) = v;
5068	}
5069	else
5070	var = &TREE_CHAIN (*var);
5071	}
5072	else
5073	remap_blocks_to_null (DECL_INITIAL (fn), id);
5074	}
5075
5076	/* Return statements in the function body will be replaced by jumps
5077	to the RET_LABEL. */
5078	gcc_assert (DECL_INITIAL (fn));
5079	gcc_assert (TREE_CODE (DECL_INITIAL (fn)) == BLOCK);
5080
5081	/* Find the LHS to which the result of this call is assigned. */
5082	return_slot = NULL;
5083	if (gimple_call_lhs (gs: stmt))
5084	{
5085	modify_dest = gimple_call_lhs (gs: stmt);
5086
5087	/* The function which we are inlining might not return a value,
5088	in which case we should issue a warning that the function
5089	does not return a value. In that case the optimizers will
5090	see that the variable to which the value is assigned was not
5091	initialized. We do not want to issue a warning about that
5092	uninitialized variable. */
5093	if (DECL_P (modify_dest))
5094	suppress_warning (modify_dest, OPT_Wuninitialized);
5095
5096	if (gimple_call_return_slot_opt_p (s: call_stmt))
5097	{
5098	return_slot = modify_dest;
5099	modify_dest = NULL;
5100	}
5101	}
5102	else
5103	modify_dest = NULL;
5104
5105	/* If we are inlining a call to the C++ operator new, we don't want
5106	to use type based alias analysis on the return value. Otherwise
5107	we may get confused if the compiler sees that the inlined new
5108	function returns a pointer which was just deleted. See bug
5109	33407. */
5110	if (DECL_IS_OPERATOR_NEW_P (fn))
5111	{
5112	return_slot = NULL;
5113	modify_dest = NULL;
5114	}
5115
5116	/* Declare the return variable for the function. */
5117	use_retvar = declare_return_variable (id, return_slot, modify_dest, entry_bb: bb);
5118
5119	/* Add local vars in this inlined callee to caller. */
5120	add_local_variables (callee: id->src_cfun, cfun, id);
5121
5122	if (dump_enabled_p ())
5123	{
5124	char buf[128];
5125	snprintf (s: buf, maxlen: sizeof(buf), format: "%4.2f",
5126	cg_edge->sreal_frequency ().to_double ());
5127	dump_printf_loc (MSG_NOTE | MSG_PRIORITY_INTERNALS,
5128	call_stmt,
5129	"Inlining %C to %C with frequency %s\n",
5130	id->src_node, id->dst_node, buf);
5131	if (dump_file && (dump_flags & TDF_DETAILS))
5132	{
5133	id->src_node->dump (f: dump_file);
5134	id->dst_node->dump (f: dump_file);
5135	}
5136	}
5137
5138	/* This is it. Duplicate the callee body. Assume callee is
5139	pre-gimplified. Note that we must not alter the caller
5140	function in any way before this point, as this CALL_EXPR may be
5141	a self-referential call; if we're calling ourselves, we need to
5142	duplicate our body before altering anything. */
5143	copy_body (id, entry_block_map: bb, exit_block_map: return_block, NULL);
5144
5145	reset_debug_bindings (id, gsi: stmt_gsi);
5146
5147	if (flag_stack_reuse != SR_NONE)
5148	for (tree p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
5149	if (!TREE_THIS_VOLATILE (p))
5150	{
5151	/* The value associated with P is a local temporary only if
5152	there is no value associated with P in the debug map. */
5153	tree *varp = id->decl_map->get (k: p);
5154	if (varp
5155	&& VAR_P (*varp)
5156	&& !is_gimple_reg (*varp)
5157	&& !(id->debug_map && id->debug_map->get (k: p)))
5158	{
5159	tree clobber = build_clobber (TREE_TYPE (*varp),
5160	CLOBBER_STORAGE_END);
5161	gimple *clobber_stmt;
5162	clobber_stmt = gimple_build_assign (*varp, clobber);
5163	gimple_set_location (g: clobber_stmt, location: gimple_location (g: stmt));
5164	gsi_insert_before (&stmt_gsi, clobber_stmt, GSI_SAME_STMT);
5165	}
5166	}
5167
5168	/* Reset the escaped solution. */
5169	if (cfun->gimple_df)
5170	{
5171	pt_solution_reset (&cfun->gimple_df->escaped);
5172	pt_solution_reset (&cfun->gimple_df->escaped_return);
5173	}
5174
5175	/* Add new automatic variables to IFN_GOMP_SIMT_ENTER arguments. */
5176	if (id->dst_simt_vars && id->dst_simt_vars->length () > 0)
5177	{
5178	size_t nargs = gimple_call_num_args (gs: simtenter_stmt);
5179	vec<tree> *vars = id->dst_simt_vars;
5180	auto_vec<tree> newargs (nargs + vars->length ());
5181	for (size_t i = 0; i < nargs; i++)
5182	newargs.quick_push (obj: gimple_call_arg (gs: simtenter_stmt, index: i));
5183	for (tree *pvar = vars->begin (); pvar != vars->end (); pvar++)
5184	{
5185	tree ptrtype = build_pointer_type (TREE_TYPE (*pvar));
5186	newargs.quick_push (obj: build1 (ADDR_EXPR, ptrtype, *pvar));
5187	}
5188	gcall *g = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, newargs);
5189	gimple_call_set_lhs (gs: g, lhs: gimple_call_lhs (gs: simtenter_stmt));
5190	gimple_stmt_iterator gsi = gsi_for_stmt (simtenter_stmt);
5191	gsi_replace (&gsi, g, false);
5192	}
5193	vec_free (v&: id->dst_simt_vars);
5194	id->dst_simt_vars = simtvars_save;
5195
5196	/* Clean up. */
5197	if (id->debug_map)
5198	{
5199	delete id->debug_map;
5200	id->debug_map = dst;
5201	}
5202	delete id->decl_map;
5203	id->decl_map = st;
5204
5205	/* Unlink the calls virtual operands before replacing it. */
5206	unlink_stmt_vdef (stmt);
5207	if (gimple_vdef (g: stmt)
5208	&& TREE_CODE (gimple_vdef (stmt)) == SSA_NAME)
5209	release_ssa_name (name: gimple_vdef (g: stmt));
5210
5211	/* If the inlined function returns a result that we care about,
5212	substitute the GIMPLE_CALL with an assignment of the return
5213	variable to the LHS of the call. That is, if STMT was
5214	'a = foo (...)', substitute the call with 'a = USE_RETVAR'. */
5215	if (use_retvar && gimple_call_lhs (gs: stmt))
5216	{
5217	gimple *old_stmt = stmt;
5218	stmt = gimple_build_assign (gimple_call_lhs (gs: stmt), use_retvar);
5219	gimple_set_location (g: stmt, location: gimple_location (g: old_stmt));
5220	gsi_replace (&stmt_gsi, stmt, false);
5221	maybe_clean_or_replace_eh_stmt (old_stmt, stmt);
5222	/* Append a clobber for id->retvar if easily possible. */
5223	if (flag_stack_reuse != SR_NONE
5224	&& id->retvar
5225	&& VAR_P (id->retvar)
5226	&& id->retvar != return_slot
5227	&& id->retvar != modify_dest
5228	&& !TREE_THIS_VOLATILE (id->retvar)
5229	&& !is_gimple_reg (id->retvar)
5230	&& !stmt_ends_bb_p (stmt))
5231	{
5232	tree clobber = build_clobber (TREE_TYPE (id->retvar),
5233	CLOBBER_STORAGE_END);
5234	gimple *clobber_stmt;
5235	clobber_stmt = gimple_build_assign (id->retvar, clobber);
5236	gimple_set_location (g: clobber_stmt, location: gimple_location (g: old_stmt));
5237	gsi_insert_after (&stmt_gsi, clobber_stmt, GSI_SAME_STMT);
5238	}
5239	}
5240	else
5241	{
5242	/* Handle the case of inlining a function with no return
5243	statement, which causes the return value to become undefined. */
5244	if (gimple_call_lhs (gs: stmt)
5245	&& TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
5246	{
5247	tree name = gimple_call_lhs (gs: stmt);
5248	tree var = SSA_NAME_VAR (name);
5249	tree def = var ? ssa_default_def (cfun, var) : NULL;
5250
5251	if (def)
5252	{
5253	/* If the variable is used undefined, make this name
5254	undefined via a move. */
5255	stmt = gimple_build_assign (gimple_call_lhs (gs: stmt), def);
5256	gsi_replace (&stmt_gsi, stmt, true);
5257	}
5258	else
5259	{
5260	if (!var)
5261	{
5262	var = create_tmp_reg_fn (cfun, TREE_TYPE (name), NULL);
5263	SET_SSA_NAME_VAR_OR_IDENTIFIER (name, var);
5264	}
5265	/* Otherwise make this variable undefined. */
5266	gsi_remove (&stmt_gsi, true);
5267	set_ssa_default_def (cfun, var, name);
5268	SSA_NAME_DEF_STMT (name) = gimple_build_nop ();
5269	}
5270	}
5271	/* Replace with a clobber for id->retvar. */
5272	else if (flag_stack_reuse != SR_NONE
5273	&& id->retvar
5274	&& VAR_P (id->retvar)
5275	&& id->retvar != return_slot
5276	&& id->retvar != modify_dest
5277	&& !TREE_THIS_VOLATILE (id->retvar)
5278	&& !is_gimple_reg (id->retvar))
5279	{
5280	tree clobber = build_clobber (TREE_TYPE (id->retvar));
5281	gimple *clobber_stmt;
5282	clobber_stmt = gimple_build_assign (id->retvar, clobber);
5283	gimple_set_location (g: clobber_stmt, location: gimple_location (g: stmt));
5284	gsi_replace (&stmt_gsi, clobber_stmt, false);
5285	maybe_clean_or_replace_eh_stmt (stmt, clobber_stmt);
5286	}
5287	else
5288	gsi_remove (&stmt_gsi, true);
5289	}
5290
5291	if (purge_dead_abnormal_edges)
5292	bitmap_set_bit (to_purge, return_block->index);
5293
5294	/* If the value of the new expression is ignored, that's OK. We
5295	don't warn about this for CALL_EXPRs, so we shouldn't warn about
5296	the equivalent inlined version either. */
5297	if (is_gimple_assign (gs: stmt))
5298	{
5299	gcc_assert (gimple_assign_single_p (stmt)
5300	|| CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt)));
5301	TREE_USED (gimple_assign_rhs1 (stmt)) = 1;
5302	}
5303
5304	id->add_clobbers_to_eh_landing_pads = 0;
5305
5306	/* Output the inlining info for this abstract function, since it has been
5307	inlined. If we don't do this now, we can lose the information about the
5308	variables in the function when the blocks get blown away as soon as we
5309	remove the cgraph node. */
5310	if (gimple_block (g: stmt))
5311	(*debug_hooks->outlining_inline_function) (fn);
5312
5313	/* Update callgraph if needed. */
5314	cg_edge->callee->remove ();
5315
5316	id->block = NULL_TREE;
5317	id->retvar = NULL_TREE;
5318	successfully_inlined = true;
5319
5320	egress:
5321	input_location = saved_location;
5322	return successfully_inlined;
5323	}
5324
5325	/* Expand call statements reachable from STMT_P.
5326	We can only have CALL_EXPRs as the "toplevel" tree code or nested
5327	in a MODIFY_EXPR. */
5328
5329	static bool
5330	gimple_expand_calls_inline (basic_block bb, copy_body_data *id,
5331	bitmap to_purge)
5332	{
5333	gimple_stmt_iterator gsi;
5334	bool inlined = false;
5335
5336	for (gsi = gsi_last_bb (bb); !gsi_end_p (i: gsi);)
5337	{
5338	gimple *stmt = gsi_stmt (i: gsi);
5339	gsi_prev (i: &gsi);
5340
5341	if (is_gimple_call (gs: stmt)
5342	&& !gimple_call_internal_p (gs: stmt))
5343	inlined |= expand_call_inline (bb, stmt, id, to_purge);
5344	}
5345
5346	return inlined;
5347	}
5348
5349
5350	/* Walk all basic blocks created after FIRST and try to fold every statement
5351	in the STATEMENTS pointer set. */
5352
5353	static void
5354	fold_marked_statements (int first, hash_set<gimple *> *statements)
5355	{
5356	auto_bitmap to_purge;
5357
5358	auto_vec<edge, 20> stack (n_basic_blocks_for_fn (cfun) + 2);
5359	auto_sbitmap visited (last_basic_block_for_fn (cfun));
5360	bitmap_clear (visited);
5361
5362	stack.quick_push (obj: single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
5363	while (!stack.is_empty ())
5364	{
5365	/* Look at the edge on the top of the stack. */
5366	edge e = stack.pop ();
5367	basic_block dest = e->dest;
5368
5369	if (dest == EXIT_BLOCK_PTR_FOR_FN (cfun)
5370	|| bitmap_bit_p (map: visited, bitno: dest->index))
5371	continue;
5372
5373	bitmap_set_bit (map: visited, bitno: dest->index);
5374
5375	if (dest->index >= first)
5376	for (gimple_stmt_iterator gsi = gsi_start_bb (bb: dest);
5377	!gsi_end_p (i: gsi); gsi_next (i: &gsi))
5378	{
5379	if (!statements->contains (k: gsi_stmt (i: gsi)))
5380	continue;
5381
5382	gimple *old_stmt = gsi_stmt (i: gsi);
5383	tree old_decl = (is_gimple_call (gs: old_stmt)
5384	? gimple_call_fndecl (gs: old_stmt) : 0);
5385	if (old_decl && fndecl_built_in_p (node: old_decl))
5386	{
5387	/* Folding builtins can create multiple instructions,
5388	we need to look at all of them. */
5389	gimple_stmt_iterator i2 = gsi;
5390	gsi_prev (i: &i2);
5391	if (fold_stmt (&gsi))
5392	{
5393	gimple *new_stmt;
5394	/* If a builtin at the end of a bb folded into nothing,
5395	the following loop won't work. */
5396	if (gsi_end_p (i: gsi))
5397	{
5398	cgraph_update_edges_for_call_stmt (old_stmt,
5399	old_decl, NULL);
5400	break;
5401	}
5402	if (gsi_end_p (i: i2))
5403	i2 = gsi_start_bb (bb: dest);
5404	else
5405	gsi_next (i: &i2);
5406	while (1)
5407	{
5408	new_stmt = gsi_stmt (i: i2);
5409	update_stmt (s: new_stmt);
5410	cgraph_update_edges_for_call_stmt (old_stmt, old_decl,
5411	new_stmt);
5412
5413	if (new_stmt == gsi_stmt (i: gsi))
5414	{
5415	/* It is okay to check only for the very last
5416	of these statements. If it is a throwing
5417	statement nothing will change. If it isn't
5418	this can remove EH edges. If that weren't
5419	correct then because some intermediate stmts
5420	throw, but not the last one. That would mean
5421	we'd have to split the block, which we can't
5422	here and we'd loose anyway. And as builtins
5423	probably never throw, this all
5424	is mood anyway. */
5425	if (maybe_clean_or_replace_eh_stmt (old_stmt,
5426	new_stmt))
5427	bitmap_set_bit (to_purge, dest->index);
5428	break;
5429	}
5430	gsi_next (i: &i2);
5431	}
5432	}
5433	}
5434	else if (fold_stmt (&gsi))
5435	{
5436	/* Re-read the statement from GSI as fold_stmt() may
5437	have changed it. */
5438	gimple *new_stmt = gsi_stmt (i: gsi);
5439	update_stmt (s: new_stmt);
5440
5441	if (is_gimple_call (gs: old_stmt)
5442	|| is_gimple_call (gs: new_stmt))
5443	cgraph_update_edges_for_call_stmt (old_stmt, old_decl,
5444	new_stmt);
5445
5446	if (maybe_clean_or_replace_eh_stmt (old_stmt, new_stmt))
5447	bitmap_set_bit (to_purge, dest->index);
5448	}
5449	}
5450
5451	if (EDGE_COUNT (dest->succs) > 0)
5452	{
5453	/* Avoid warnings emitted from folding statements that
5454	became unreachable because of inlined function parameter
5455	propagation. */
5456	e = find_taken_edge (dest, NULL_TREE);
5457	if (e)
5458	stack.quick_push (obj: e);
5459	else
5460	{
5461	edge_iterator ei;
5462	FOR_EACH_EDGE (e, ei, dest->succs)
5463	stack.safe_push (obj: e);
5464	}
5465	}
5466	}
5467
5468	gimple_purge_all_dead_eh_edges (to_purge);
5469	}
5470
5471	/* Expand calls to inline functions in the body of FN. */
5472
5473	unsigned int
5474	optimize_inline_calls (tree fn)
5475	{
5476	copy_body_data id;
5477	basic_block bb;
5478	int last = n_basic_blocks_for_fn (cfun);
5479	bool inlined_p = false;
5480
5481	/* Clear out ID. */
5482	memset (s: &id, c: 0, n: sizeof (id));
5483
5484	id.src_node = id.dst_node = cgraph_node::get (decl: fn);
5485	gcc_assert (id.dst_node->definition);
5486	id.dst_fn = fn;
5487	/* Or any functions that aren't finished yet. */
5488	if (current_function_decl)
5489	id.dst_fn = current_function_decl;
5490
5491	id.copy_decl = copy_decl_maybe_to_var;
5492	id.transform_call_graph_edges = CB_CGE_DUPLICATE;
5493	id.transform_new_cfg = false;
5494	id.transform_return_to_modify = true;
5495	id.transform_parameter = true;
5496	id.statements_to_fold = new hash_set<gimple *>;
5497
5498	push_gimplify_context ();
5499
5500	/* We make no attempts to keep dominance info up-to-date. */
5501	free_dominance_info (CDI_DOMINATORS);
5502	free_dominance_info (CDI_POST_DOMINATORS);
5503
5504	/* Register specific gimple functions. */
5505	gimple_register_cfg_hooks ();
5506
5507	/* Reach the trees by walking over the CFG, and note the
5508	enclosing basic-blocks in the call edges. */
5509	/* We walk the blocks going forward, because inlined function bodies
5510	will split id->current_basic_block, and the new blocks will
5511	follow it; we'll trudge through them, processing their CALL_EXPRs
5512	along the way. */
5513	auto_bitmap to_purge;
5514	FOR_EACH_BB_FN (bb, cfun)
5515	inlined_p |= gimple_expand_calls_inline (bb, id: &id, to_purge);
5516
5517	pop_gimplify_context (NULL);
5518
5519	if (flag_checking)
5520	{
5521	struct cgraph_edge *e;
5522
5523	id.dst_node->verify ();
5524
5525	/* Double check that we inlined everything we are supposed to inline. */
5526	for (e = id.dst_node->callees; e; e = e->next_callee)
5527	gcc_assert (e->inline_failed);
5528	}
5529
5530	/* If we didn't inline into the function there is nothing to do. */
5531	if (!inlined_p)
5532	{
5533	delete id.statements_to_fold;
5534	return 0;
5535	}
5536
5537	/* Fold queued statements. */
5538	update_max_bb_count ();
5539	fold_marked_statements (first: last, statements: id.statements_to_fold);
5540	delete id.statements_to_fold;
5541
5542	/* Finally purge EH and abnormal edges from the call stmts we inlined.
5543	We need to do this after fold_marked_statements since that may walk
5544	the SSA use-def chain. */
5545	unsigned i;
5546	bitmap_iterator bi;
5547	EXECUTE_IF_SET_IN_BITMAP (to_purge, 0, i, bi)
5548	{
5549	basic_block bb = BASIC_BLOCK_FOR_FN (cfun, i);
5550	if (bb)
5551	{
5552	gimple_purge_dead_eh_edges (bb);
5553	gimple_purge_dead_abnormal_call_edges (bb);
5554	}
5555	}
5556
5557	gcc_assert (!id.debug_stmts.exists ());
5558
5559	/* Renumber the lexical scoping (non-code) blocks consecutively. */
5560	number_blocks (fn);
5561
5562	delete_unreachable_blocks_update_callgraph (dst_node: id.dst_node, update_clones: false);
5563	id.dst_node->calls_comdat_local = id.dst_node->check_calls_comdat_local_p ();
5564
5565	if (flag_checking)
5566	id.dst_node->verify ();
5567
5568	/* It would be nice to check SSA/CFG/statement consistency here, but it is
5569	not possible yet - the IPA passes might make various functions to not
5570	throw and they don't care to proactively update local EH info. This is
5571	done later in fixup_cfg pass that also execute the verification. */
5572	return (TODO_update_ssa
5573	| TODO_cleanup_cfg
5574	| (gimple_in_ssa_p (cfun) ? TODO_remove_unused_locals : 0)
5575	| (gimple_in_ssa_p (cfun) ? TODO_update_address_taken : 0));
5576	}
5577
5578	/* Passed to walk_tree. Copies the node pointed to, if appropriate. */
5579
5580	tree
5581	copy_tree_r (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED)
5582	{
5583	enum tree_code code = TREE_CODE (*tp);
5584	enum tree_code_class cl = TREE_CODE_CLASS (code);
5585
5586	/* We make copies of most nodes. */
5587	if (IS_EXPR_CODE_CLASS (cl)
5588	|| code == TREE_LIST
5589	|| code == TREE_VEC
5590	|| code == TYPE_DECL
5591	|| code == OMP_CLAUSE)
5592	{
5593	/* Because the chain gets clobbered when we make a copy, we save it
5594	here. */
5595	tree chain = NULL_TREE, new_tree;
5596
5597	if (CODE_CONTAINS_STRUCT (code, TS_COMMON))
5598	chain = TREE_CHAIN (*tp);
5599
5600	/* Copy the node. */
5601	new_tree = copy_node (*tp);
5602
5603	*tp = new_tree;
5604
5605	/* Now, restore the chain, if appropriate. That will cause
5606	walk_tree to walk into the chain as well. */
5607	if (code == PARM_DECL
5608	|| code == TREE_LIST
5609	|| code == OMP_CLAUSE)
5610	TREE_CHAIN (*tp) = chain;
5611
5612	/* For now, we don't update BLOCKs when we make copies. So, we
5613	have to nullify all BIND_EXPRs. */
5614	if (TREE_CODE (*tp) == BIND_EXPR)
5615	BIND_EXPR_BLOCK (*tp) = NULL_TREE;
5616	}
5617	else if (code == CONSTRUCTOR)
5618	{
5619	/* CONSTRUCTOR nodes need special handling because
5620	we need to duplicate the vector of elements. */
5621	tree new_tree;
5622
5623	new_tree = copy_node (*tp);
5624	CONSTRUCTOR_ELTS (new_tree) = vec_safe_copy (CONSTRUCTOR_ELTS (*tp));
5625	*tp = new_tree;
5626	}
5627	else if (code == STATEMENT_LIST)
5628	/* We used to just abort on STATEMENT_LIST, but we can run into them
5629	with statement-expressions (c++/40975). */
5630	copy_statement_list (tp);
5631	else if (TREE_CODE_CLASS (code) == tcc_type)
5632	*walk_subtrees = 0;
5633	else if (TREE_CODE_CLASS (code) == tcc_declaration)
5634	*walk_subtrees = 0;
5635	else if (TREE_CODE_CLASS (code) == tcc_constant)
5636	*walk_subtrees = 0;
5637	return NULL_TREE;
5638	}
5639
5640	/* The SAVE_EXPR pointed to by TP is being copied. If ST contains
5641	information indicating to what new SAVE_EXPR this one should be mapped,
5642	use that one. Otherwise, create a new node and enter it in ST. FN is
5643	the function into which the copy will be placed. */
5644
5645	static void
5646	remap_save_expr (tree *tp, hash_map<tree, tree> *st, int *walk_subtrees)
5647	{
5648	tree *n;
5649	tree t;
5650
5651	/* See if we already encountered this SAVE_EXPR. */
5652	n = st->get (k: *tp);
5653
5654	/* If we didn't already remap this SAVE_EXPR, do so now. */
5655	if (!n)
5656	{
5657	t = copy_node (*tp);
5658
5659	/* Remember this SAVE_EXPR. */
5660	st->put (k: *tp, v: t);
5661	/* Make sure we don't remap an already-remapped SAVE_EXPR. */
5662	st->put (k: t, v: t);
5663	}
5664	else
5665	{
5666	/* We've already walked into this SAVE_EXPR; don't do it again. */
5667	*walk_subtrees = 0;
5668	t = *n;
5669	}
5670
5671	/* Replace this SAVE_EXPR with the copy. */
5672	*tp = t;
5673	}
5674
5675	/* Called via walk_gimple_seq. If *GSIP points to a GIMPLE_LABEL for a local
5676	label, copies the declaration and enters it in the splay_tree in DATA (which
5677	is really a 'copy_body_data *'. */
5678
5679	static tree
5680	mark_local_labels_stmt (gimple_stmt_iterator *gsip,
5681	bool *handled_ops_p ATTRIBUTE_UNUSED,
5682	struct walk_stmt_info *wi)
5683	{
5684	copy_body_data *id = (copy_body_data *) wi->info;
5685	glabel *stmt = dyn_cast <glabel *> (p: gsi_stmt (i: *gsip));
5686
5687	if (stmt)
5688	{
5689	tree decl = gimple_label_label (gs: stmt);
5690
5691	/* Copy the decl and remember the copy. */
5692	insert_decl_map (id, key: decl, value: id->copy_decl (decl, id));
5693	}
5694
5695	return NULL_TREE;
5696	}
5697
5698	static gimple_seq duplicate_remap_omp_clause_seq (gimple_seq seq,
5699	struct walk_stmt_info *wi);
5700
5701	/* Called via walk_gimple_seq by copy_gimple_seq_and_replace_local.
5702	Using the splay_tree pointed to by ST (which is really a `splay_tree'),
5703	remaps all local declarations to appropriate replacements in gimple
5704	operands. */
5705
5706	static tree
5707	replace_locals_op (tree *tp, int *walk_subtrees, void *data)
5708	{
5709	struct walk_stmt_info *wi = (struct walk_stmt_info*) data;
5710	copy_body_data *id = (copy_body_data *) wi->info;
5711	hash_map<tree, tree> *st = id->decl_map;
5712	tree *n;
5713	tree expr = *tp;
5714
5715	/* For recursive invocations this is no longer the LHS itself. */
5716	bool is_lhs = wi->is_lhs;
5717	wi->is_lhs = false;
5718
5719	if (TREE_CODE (expr) == SSA_NAME)
5720	{
5721	*tp = remap_ssa_name (name: *tp, id);
5722	*walk_subtrees = 0;
5723	if (is_lhs)
5724	SSA_NAME_DEF_STMT (*tp) = gsi_stmt (i: wi->gsi);
5725	}
5726	/* Only a local declaration (variable or label). */
5727	else if ((VAR_P (expr) && !TREE_STATIC (expr))
5728	|| TREE_CODE (expr) == LABEL_DECL)
5729	{
5730	/* Lookup the declaration. */
5731	n = st->get (k: expr);
5732
5733	/* If it's there, remap it. */
5734	if (n)
5735	*tp = *n;
5736	*walk_subtrees = 0;
5737	}
5738	else if (TREE_CODE (expr) == STATEMENT_LIST
5739	|| TREE_CODE (expr) == BIND_EXPR
5740	|| TREE_CODE (expr) == SAVE_EXPR)
5741	gcc_unreachable ();
5742	else if (TREE_CODE (expr) == TARGET_EXPR)
5743	{
5744	/* Don't mess with a TARGET_EXPR that hasn't been expanded.
5745	It's OK for this to happen if it was part of a subtree that
5746	isn't immediately expanded, such as operand 2 of another
5747	TARGET_EXPR. */
5748	if (!TREE_OPERAND (expr, 1))
5749	{
5750	TREE_OPERAND (expr, 1) = TREE_OPERAND (expr, 3);
5751	TREE_OPERAND (expr, 3) = NULL_TREE;
5752	}
5753	}
5754	else if (TREE_CODE (expr) == OMP_CLAUSE)
5755	{
5756	/* Before the omplower pass completes, some OMP clauses can contain
5757	sequences that are neither copied by gimple_seq_copy nor walked by
5758	walk_gimple_seq. To make copy_gimple_seq_and_replace_locals work even
5759	in those situations, we have to copy and process them explicitely. */
5760
5761	if (OMP_CLAUSE_CODE (expr) == OMP_CLAUSE_LASTPRIVATE)
5762	{
5763	gimple_seq seq = OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (expr);
5764	seq = duplicate_remap_omp_clause_seq (seq, wi);
5765	OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (expr) = seq;
5766	}
5767	else if (OMP_CLAUSE_CODE (expr) == OMP_CLAUSE_LINEAR)
5768	{
5769	gimple_seq seq = OMP_CLAUSE_LINEAR_GIMPLE_SEQ (expr);
5770	seq = duplicate_remap_omp_clause_seq (seq, wi);
5771	OMP_CLAUSE_LINEAR_GIMPLE_SEQ (expr) = seq;
5772	}
5773	else if (OMP_CLAUSE_CODE (expr) == OMP_CLAUSE_REDUCTION)
5774	{
5775	gimple_seq seq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (expr);
5776	seq = duplicate_remap_omp_clause_seq (seq, wi);
5777	OMP_CLAUSE_REDUCTION_GIMPLE_INIT (expr) = seq;
5778	seq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (expr);
5779	seq = duplicate_remap_omp_clause_seq (seq, wi);
5780	OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (expr) = seq;
5781	}
5782	}
5783
5784	/* Keep iterating. */
5785	return NULL_TREE;
5786	}
5787
5788
5789	/* Called via walk_gimple_seq by copy_gimple_seq_and_replace_local.
5790	Using the splay_tree pointed to by ST (which is really a `splay_tree'),
5791	remaps all local declarations to appropriate replacements in gimple
5792	statements. */
5793
5794	static tree
5795	replace_locals_stmt (gimple_stmt_iterator *gsip,
5796	bool *handled_ops_p ATTRIBUTE_UNUSED,
5797	struct walk_stmt_info *wi)
5798	{
5799	copy_body_data *id = (copy_body_data *) wi->info;
5800	gimple *gs = gsi_stmt (i: *gsip);
5801
5802	if (gbind *stmt = dyn_cast <gbind *> (p: gs))
5803	{
5804	tree block = gimple_bind_block (bind_stmt: stmt);
5805
5806	if (block)
5807	{
5808	remap_block (block: &block, id);
5809	gimple_bind_set_block (bind_stmt: stmt, block);
5810	}
5811
5812	/* This will remap a lot of the same decls again, but this should be
5813	harmless. */
5814	if (gimple_bind_vars (bind_stmt: stmt))
5815	{
5816	tree old_var, decls = gimple_bind_vars (bind_stmt: stmt);
5817
5818	for (old_var = decls; old_var; old_var = DECL_CHAIN (old_var))
5819	if (!can_be_nonlocal (decl: old_var, id)
5820	&& ! variably_modified_type_p (TREE_TYPE (old_var), id->src_fn))
5821	remap_decl (decl: old_var, id);
5822
5823	gcc_checking_assert (!id->prevent_decl_creation_for_types);
5824	id->prevent_decl_creation_for_types = true;
5825	gimple_bind_set_vars (bind_stmt: stmt, vars: remap_decls (decls, NULL, id));
5826	id->prevent_decl_creation_for_types = false;
5827	}
5828	}
5829
5830	/* Keep iterating. */
5831	return NULL_TREE;
5832	}
5833
5834	/* Create a copy of SEQ and remap all decls in it. */
5835
5836	static gimple_seq
5837	duplicate_remap_omp_clause_seq (gimple_seq seq, struct walk_stmt_info *wi)
5838	{
5839	if (!seq)
5840	return NULL;
5841
5842	/* If there are any labels in OMP sequences, they can be only referred to in
5843	the sequence itself and therefore we can do both here. */
5844	walk_gimple_seq (seq, mark_local_labels_stmt, NULL, wi);
5845	gimple_seq copy = gimple_seq_copy (seq);
5846	walk_gimple_seq (copy, replace_locals_stmt, replace_locals_op, wi);
5847	return copy;
5848	}
5849
5850	/* Copies everything in SEQ and replaces variables and labels local to
5851	current_function_decl. */
5852
5853	gimple_seq
5854	copy_gimple_seq_and_replace_locals (gimple_seq seq)
5855	{
5856	copy_body_data id;
5857	struct walk_stmt_info wi;
5858	gimple_seq copy;
5859
5860	/* There's nothing to do for NULL_TREE. */
5861	if (seq == NULL)
5862	return seq;
5863
5864	/* Set up ID. */
5865	memset (s: &id, c: 0, n: sizeof (id));
5866	id.src_fn = current_function_decl;
5867	id.dst_fn = current_function_decl;
5868	id.src_cfun = cfun;
5869	id.decl_map = new hash_map<tree, tree>;
5870	id.debug_map = NULL;
5871
5872	id.copy_decl = copy_decl_no_change;
5873	id.transform_call_graph_edges = CB_CGE_DUPLICATE;
5874	id.transform_new_cfg = false;
5875	id.transform_return_to_modify = false;
5876	id.transform_parameter = false;
5877
5878	/* Walk the tree once to find local labels. */
5879	memset (s: &wi, c: 0, n: sizeof (wi));
5880	hash_set<tree> visited;
5881	wi.info = &id;
5882	wi.pset = &visited;
5883	walk_gimple_seq (seq, mark_local_labels_stmt, NULL, &wi);
5884
5885	copy = gimple_seq_copy (seq);
5886
5887	/* Walk the copy, remapping decls. */
5888	memset (s: &wi, c: 0, n: sizeof (wi));
5889	wi.info = &id;
5890	walk_gimple_seq (copy, replace_locals_stmt, replace_locals_op, &wi);
5891
5892	/* Clean up. */
5893	delete id.decl_map;
5894	if (id.debug_map)
5895	delete id.debug_map;
5896	if (id.dependence_map)
5897	{
5898	delete id.dependence_map;
5899	id.dependence_map = NULL;
5900	}
5901
5902	return copy;
5903	}
5904
5905
5906	/* Allow someone to determine if SEARCH is a child of TOP from gdb. */
5907
5908	static tree
5909	debug_find_tree_1 (tree *tp, int *walk_subtrees ATTRIBUTE_UNUSED, void *data)
5910	{
5911	if (*tp == data)
5912	return (tree) data;
5913	else
5914	return NULL;
5915	}
5916
5917	DEBUG_FUNCTION bool
5918	debug_find_tree (tree top, tree search)
5919	{
5920	return walk_tree_without_duplicates (&top, debug_find_tree_1, search) != 0;
5921	}
5922
5923
5924	/* Declare the variables created by the inliner. Add all the variables in
5925	VARS to BIND_EXPR. */
5926
5927	static void
5928	declare_inline_vars (tree block, tree vars)
5929	{
5930	tree t;
5931	for (t = vars; t; t = DECL_CHAIN (t))
5932	{
5933	DECL_SEEN_IN_BIND_EXPR_P (t) = 1;
5934	gcc_assert (!TREE_STATIC (t) && !TREE_ASM_WRITTEN (t));
5935	add_local_decl (cfun, d: t);
5936	}
5937
5938	if (block)
5939	BLOCK_VARS (block) = chainon (BLOCK_VARS (block), vars);
5940	}
5941
5942	/* Copy NODE (which must be a DECL). The DECL originally was in the FROM_FN,
5943	but now it will be in the TO_FN. PARM_TO_VAR means enable PARM_DECL to
5944	VAR_DECL translation. */
5945
5946	tree
5947	copy_decl_for_dup_finish (copy_body_data *id, tree decl, tree copy)
5948	{
5949	/* Don't generate debug information for the copy if we wouldn't have
5950	generated it for the copy either. */
5951	DECL_ARTIFICIAL (copy) = DECL_ARTIFICIAL (decl);
5952	DECL_IGNORED_P (copy) = DECL_IGNORED_P (decl);
5953
5954	/* Set the DECL_ABSTRACT_ORIGIN so the debugging routines know what
5955	declaration inspired this copy. */
5956	DECL_ABSTRACT_ORIGIN (copy) = DECL_ORIGIN (decl);
5957
5958	/* The new variable/label has no RTL, yet. */
5959	if (HAS_RTL_P (copy)
5960	&& !TREE_STATIC (copy) && !DECL_EXTERNAL (copy))
5961	SET_DECL_RTL (copy, 0);
5962	/* For vector typed decls make sure to update DECL_MODE according
5963	to the new function context. */
5964	if (VECTOR_TYPE_P (TREE_TYPE (copy)))
5965	SET_DECL_MODE (copy, TYPE_MODE (TREE_TYPE (copy)));
5966
5967	/* These args would always appear unused, if not for this. */
5968	TREE_USED (copy) = 1;
5969
5970	/* Set the context for the new declaration. */
5971	if (!DECL_CONTEXT (decl))
5972	/* Globals stay global. */
5973	;
5974	else if (DECL_CONTEXT (decl) != id->src_fn)
5975	/* Things that weren't in the scope of the function we're inlining
5976	from aren't in the scope we're inlining to, either. */
5977	;
5978	else if (TREE_STATIC (decl))
5979	/* Function-scoped static variables should stay in the original
5980	function. */
5981	;
5982	else
5983	{
5984	/* Ordinary automatic local variables are now in the scope of the
5985	new function. */
5986	DECL_CONTEXT (copy) = id->dst_fn;
5987	if (VAR_P (copy) && id->dst_simt_vars && !is_gimple_reg (copy))
5988	{
5989	if (!lookup_attribute (attr_name: "omp simt private", DECL_ATTRIBUTES (copy)))
5990	DECL_ATTRIBUTES (copy)
5991	= tree_cons (get_identifier ("omp simt private"), NULL,
5992	DECL_ATTRIBUTES (copy));
5993	id->dst_simt_vars->safe_push (obj: copy);
5994	}
5995	}
5996
5997	return copy;
5998	}
5999
6000	/* Create a new VAR_DECL that is indentical in all respect to DECL except that
6001	DECL can be either a VAR_DECL, a PARM_DECL or RESULT_DECL. The original
6002	DECL must come from ID->src_fn and the copy will be part of ID->dst_fn. */
6003
6004	tree
6005	copy_decl_to_var (tree decl, copy_body_data *id)
6006	{
6007	tree copy, type;
6008
6009	gcc_assert (TREE_CODE (decl) == PARM_DECL
6010	|| TREE_CODE (decl) == RESULT_DECL);
6011
6012	type = TREE_TYPE (decl);
6013
6014	copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn),
6015	VAR_DECL, DECL_NAME (decl), type);
6016	if (DECL_PT_UID_SET_P (decl))
6017	SET_DECL_PT_UID (copy, DECL_PT_UID (decl));
6018	TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl);
6019	TREE_READONLY (copy) = TREE_READONLY (decl);
6020	TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl);
6021	DECL_NOT_GIMPLE_REG_P (copy) = DECL_NOT_GIMPLE_REG_P (decl);
6022	DECL_BY_REFERENCE (copy) = DECL_BY_REFERENCE (decl);
6023
6024	return copy_decl_for_dup_finish (id, decl, copy);
6025	}
6026
6027	/* Like copy_decl_to_var, but create a return slot object instead of a
6028	pointer variable for return by invisible reference. */
6029
6030	static tree
6031	copy_result_decl_to_var (tree decl, copy_body_data *id)
6032	{
6033	tree copy, type;
6034
6035	gcc_assert (TREE_CODE (decl) == PARM_DECL
6036	|| TREE_CODE (decl) == RESULT_DECL);
6037
6038	type = TREE_TYPE (decl);
6039	if (DECL_BY_REFERENCE (decl))
6040	type = TREE_TYPE (type);
6041
6042	copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn),
6043	VAR_DECL, DECL_NAME (decl), type);
6044	if (DECL_PT_UID_SET_P (decl))
6045	SET_DECL_PT_UID (copy, DECL_PT_UID (decl));
6046	TREE_READONLY (copy) = TREE_READONLY (decl);
6047	TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl);
6048	if (!DECL_BY_REFERENCE (decl))
6049	{
6050	TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl);
6051	DECL_NOT_GIMPLE_REG_P (copy)
6052	= (DECL_NOT_GIMPLE_REG_P (decl)
6053	/* RESULT_DECLs are treated special by needs_to_live_in_memory,
6054	mirror that to the created VAR_DECL. */
6055	|| (TREE_CODE (decl) == RESULT_DECL
6056	&& aggregate_value_p (decl, id->src_fn)));
6057	}
6058
6059	return copy_decl_for_dup_finish (id, decl, copy);
6060	}
6061
6062	tree
6063	copy_decl_no_change (tree decl, copy_body_data *id)
6064	{
6065	tree copy;
6066
6067	copy = copy_node (decl);
6068
6069	/* The COPY is not abstract; it will be generated in DST_FN. */
6070	DECL_ABSTRACT_P (copy) = false;
6071	lang_hooks.dup_lang_specific_decl (copy);
6072
6073	/* TREE_ADDRESSABLE isn't used to indicate that a label's address has
6074	been taken; it's for internal bookkeeping in expand_goto_internal. */
6075	if (TREE_CODE (copy) == LABEL_DECL)
6076	{
6077	TREE_ADDRESSABLE (copy) = 0;
6078	LABEL_DECL_UID (copy) = -1;
6079	}
6080
6081	return copy_decl_for_dup_finish (id, decl, copy);
6082	}
6083
6084	static tree
6085	copy_decl_maybe_to_var (tree decl, copy_body_data *id)
6086	{
6087	if (TREE_CODE (decl) == PARM_DECL || TREE_CODE (decl) == RESULT_DECL)
6088	return copy_decl_to_var (decl, id);
6089	else
6090	return copy_decl_no_change (decl, id);
6091	}
6092
6093	/* Return a copy of the function's argument tree without any modifications. */
6094
6095	static tree
6096	copy_arguments_nochange (tree orig_parm, copy_body_data * id)
6097	{
6098	tree arg, *parg;
6099	tree new_parm = NULL;
6100
6101	parg = &new_parm;
6102	for (arg = orig_parm; arg; arg = DECL_CHAIN (arg))
6103	{
6104	tree new_tree = remap_decl (decl: arg, id);
6105	if (TREE_CODE (new_tree) != PARM_DECL)
6106	new_tree = id->copy_decl (arg, id);
6107	lang_hooks.dup_lang_specific_decl (new_tree);
6108	*parg = new_tree;
6109	parg = &DECL_CHAIN (new_tree);
6110	}
6111	return new_parm;
6112	}
6113
6114	/* Return a copy of the function's static chain. */
6115	static tree
6116	copy_static_chain (tree static_chain, copy_body_data * id)
6117	{
6118	tree *chain_copy, *pvar;
6119
6120	chain_copy = &static_chain;
6121	for (pvar = chain_copy; *pvar; pvar = &DECL_CHAIN (*pvar))
6122	{
6123	tree new_tree = remap_decl (decl: *pvar, id);
6124	lang_hooks.dup_lang_specific_decl (new_tree);
6125	DECL_CHAIN (new_tree) = DECL_CHAIN (*pvar);
6126	*pvar = new_tree;
6127	}
6128	return static_chain;
6129	}
6130
6131	/* Return true if the function is allowed to be versioned.
6132	This is a guard for the versioning functionality. */
6133
6134	bool
6135	tree_versionable_function_p (tree fndecl)
6136	{
6137	return (!lookup_attribute (attr_name: "noclone", DECL_ATTRIBUTES (fndecl))
6138	&& copy_forbidden (DECL_STRUCT_FUNCTION (fndecl)) == NULL);
6139	}
6140
6141	/* Update clone info after duplication. */
6142
6143	static void
6144	update_clone_info (copy_body_data * id)
6145	{
6146	struct cgraph_node *this_node = id->dst_node;
6147	if (!this_node->clones)
6148	return;
6149	for (cgraph_node *node = this_node->clones; node != this_node;)
6150	{
6151	/* First update replace maps to match the new body. */
6152	clone_info *info = clone_info::get (node);
6153	if (info && info->tree_map)
6154	{
6155	unsigned int i;
6156	for (i = 0; i < vec_safe_length (v: info->tree_map); i++)
6157	{
6158	struct ipa_replace_map *replace_info;
6159	replace_info = (*info->tree_map)[i];
6160	walk_tree (&replace_info->new_tree, copy_tree_body_r, id, NULL);
6161	}
6162	}
6163
6164	if (node->clones)
6165	node = node->clones;
6166	else if (node->next_sibling_clone)
6167	node = node->next_sibling_clone;
6168	else
6169	{
6170	while (node != id->dst_node && !node->next_sibling_clone)
6171	node = node->clone_of;
6172	if (node != id->dst_node)
6173	node = node->next_sibling_clone;
6174	}
6175	}
6176	}
6177
6178	/* Create a copy of a function's tree.
6179	OLD_DECL and NEW_DECL are FUNCTION_DECL tree nodes
6180	of the original function and the new copied function
6181	respectively. In case we want to replace a DECL
6182	tree with another tree while duplicating the function's
6183	body, TREE_MAP represents the mapping between these
6184	trees. If UPDATE_CLONES is set, the call_stmt fields
6185	of edges of clones of the function will be updated.
6186
6187	If non-NULL PARAM_ADJUSTMENTS determines how function prototype (i.e. the
6188	function parameters and return value) should be modified).
6189	If non-NULL BLOCKS_TO_COPY determine what basic blocks to copy.
6190	If non_NULL NEW_ENTRY determine new entry BB of the clone.
6191	*/
6192	void
6193	tree_function_versioning (tree old_decl, tree new_decl,
6194	vec<ipa_replace_map *, va_gc> *tree_map,
6195	ipa_param_adjustments *param_adjustments,
6196	bool update_clones, bitmap blocks_to_copy,
6197	basic_block new_entry)
6198	{
6199	struct cgraph_node *old_version_node;
6200	struct cgraph_node *new_version_node;
6201	copy_body_data id;
6202	tree p;
6203	unsigned i;
6204	struct ipa_replace_map *replace_info;
6205	basic_block old_entry_block, bb;
6206	auto_vec<gimple *, 10> init_stmts;
6207	tree vars = NULL_TREE;
6208
6209	/* We can get called recursively from expand_call_inline via clone
6210	materialization. While expand_call_inline maintains input_location
6211	we cannot tolerate it to leak into the materialized clone. */
6212	location_t saved_location = input_location;
6213	input_location = UNKNOWN_LOCATION;
6214
6215	gcc_assert (TREE_CODE (old_decl) == FUNCTION_DECL
6216	&& TREE_CODE (new_decl) == FUNCTION_DECL);
6217	DECL_POSSIBLY_INLINED (old_decl) = 1;
6218
6219	old_version_node = cgraph_node::get (decl: old_decl);
6220	gcc_checking_assert (old_version_node);
6221	new_version_node = cgraph_node::get (decl: new_decl);
6222	gcc_checking_assert (new_version_node);
6223
6224	/* Copy over debug args. */
6225	if (DECL_HAS_DEBUG_ARGS_P (old_decl))
6226	{
6227	vec<tree, va_gc> **new_debug_args, **old_debug_args;
6228	gcc_checking_assert (decl_debug_args_lookup (new_decl) == NULL);
6229	DECL_HAS_DEBUG_ARGS_P (new_decl) = 0;
6230	old_debug_args = decl_debug_args_lookup (old_decl);
6231	if (old_debug_args)
6232	{
6233	new_debug_args = decl_debug_args_insert (new_decl);
6234	*new_debug_args = vec_safe_copy (src: *old_debug_args);
6235	}
6236	}
6237
6238	/* Output the inlining info for this abstract function, since it has been
6239	inlined. If we don't do this now, we can lose the information about the
6240	variables in the function when the blocks get blown away as soon as we
6241	remove the cgraph node. */
6242	(*debug_hooks->outlining_inline_function) (old_decl);
6243
6244	DECL_ARTIFICIAL (new_decl) = 1;
6245	DECL_ABSTRACT_ORIGIN (new_decl) = DECL_ORIGIN (old_decl);
6246	if (DECL_ORIGIN (old_decl) == old_decl)
6247	old_version_node->used_as_abstract_origin = true;
6248	DECL_FUNCTION_PERSONALITY (new_decl) = DECL_FUNCTION_PERSONALITY (old_decl);
6249
6250	/* Prepare the data structures for the tree copy. */
6251	memset (s: &id, c: 0, n: sizeof (id));
6252
6253	/* Generate a new name for the new version. */
6254	id.statements_to_fold = new hash_set<gimple *>;
6255
6256	id.decl_map = new hash_map<tree, tree>;
6257	id.debug_map = NULL;
6258	id.src_fn = old_decl;
6259	id.dst_fn = new_decl;
6260	id.src_node = old_version_node;
6261	id.dst_node = new_version_node;
6262	id.src_cfun = DECL_STRUCT_FUNCTION (old_decl);
6263	id.blocks_to_copy = blocks_to_copy;
6264
6265	id.copy_decl = copy_decl_no_change;
6266	id.transform_call_graph_edges
6267	= update_clones ? CB_CGE_MOVE_CLONES : CB_CGE_MOVE;
6268	id.transform_new_cfg = true;
6269	id.transform_return_to_modify = false;
6270	id.transform_parameter = false;
6271
6272	old_entry_block = ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (old_decl));
6273	DECL_RESULT (new_decl) = DECL_RESULT (old_decl);
6274	DECL_ARGUMENTS (new_decl) = DECL_ARGUMENTS (old_decl);
6275	initialize_cfun (new_fndecl: new_decl, callee_fndecl: old_decl,
6276	count: new_entry ? new_entry->count : old_entry_block->count);
6277	new_version_node->calls_declare_variant_alt
6278	= old_version_node->calls_declare_variant_alt;
6279	if (DECL_STRUCT_FUNCTION (new_decl)->gimple_df)
6280	DECL_STRUCT_FUNCTION (new_decl)->gimple_df->ipa_pta
6281	= id.src_cfun->gimple_df->ipa_pta;
6282
6283	/* Copy the function's static chain. */
6284	p = DECL_STRUCT_FUNCTION (old_decl)->static_chain_decl;
6285	if (p)
6286	DECL_STRUCT_FUNCTION (new_decl)->static_chain_decl
6287	= copy_static_chain (static_chain: p, id: &id);
6288
6289	auto_vec<int, 16> new_param_indices;
6290	clone_info *info = clone_info::get (node: old_version_node);
6291	ipa_param_adjustments *old_param_adjustments
6292	= info ? info->param_adjustments : NULL;
6293	if (old_param_adjustments)
6294	old_param_adjustments->get_updated_indices (new_indices: &new_param_indices);
6295
6296	/* If there's a tree_map, prepare for substitution. */
6297	if (tree_map)
6298	for (i = 0; i < tree_map->length (); i++)
6299	{
6300	gimple *init;
6301	replace_info = (*tree_map)[i];
6302
6303	int p = replace_info->parm_num;
6304	if (old_param_adjustments)
6305	p = new_param_indices[p];
6306
6307	tree parm;
6308	for (parm = DECL_ARGUMENTS (old_decl); p;
6309	parm = DECL_CHAIN (parm))
6310	p--;
6311	gcc_assert (parm);
6312	init = setup_one_parameter (id: &id, p: parm, value: replace_info->new_tree,
6313	fn: id.src_fn, NULL, vars: &vars);
6314	if (init)
6315	init_stmts.safe_push (obj: init);
6316	}
6317
6318	ipa_param_body_adjustments *param_body_adjs = NULL;
6319	if (param_adjustments)
6320	{
6321	param_body_adjs = new ipa_param_body_adjustments (param_adjustments,
6322	new_decl, old_decl,
6323	&id, &vars, tree_map);
6324	id.param_body_adjs = param_body_adjs;
6325	DECL_ARGUMENTS (new_decl) = param_body_adjs->get_new_param_chain ();
6326	}
6327	else if (DECL_ARGUMENTS (old_decl) != NULL_TREE)
6328	DECL_ARGUMENTS (new_decl)
6329	= copy_arguments_nochange (DECL_ARGUMENTS (old_decl), id: &id);
6330
6331	DECL_INITIAL (new_decl) = remap_blocks (DECL_INITIAL (id.src_fn), id: &id);
6332	BLOCK_SUPERCONTEXT (DECL_INITIAL (new_decl)) = new_decl;
6333
6334	declare_inline_vars (DECL_INITIAL (new_decl), vars);
6335
6336	if (!vec_safe_is_empty (DECL_STRUCT_FUNCTION (old_decl)->local_decls))
6337	/* Add local vars. */
6338	add_local_variables (DECL_STRUCT_FUNCTION (old_decl), cfun, id: &id);
6339
6340	if (DECL_RESULT (old_decl) == NULL_TREE)
6341	;
6342	else if (param_adjustments && param_adjustments->m_skip_return
6343	&& !VOID_TYPE_P (TREE_TYPE (DECL_RESULT (old_decl))))
6344	{
6345	tree resdecl_repl = copy_result_decl_to_var (DECL_RESULT (old_decl),
6346	id: &id);
6347	declare_inline_vars (NULL, vars: resdecl_repl);
6348	if (DECL_BY_REFERENCE (DECL_RESULT (old_decl)))
6349	resdecl_repl = build_fold_addr_expr (resdecl_repl);
6350	insert_decl_map (id: &id, DECL_RESULT (old_decl), value: resdecl_repl);
6351
6352	DECL_RESULT (new_decl)
6353	= build_decl (DECL_SOURCE_LOCATION (DECL_RESULT (old_decl)),
6354	RESULT_DECL, NULL_TREE, void_type_node);
6355	DECL_CONTEXT (DECL_RESULT (new_decl)) = new_decl;
6356	DECL_IS_MALLOC (new_decl) = false;
6357	cfun->returns_struct = 0;
6358	cfun->returns_pcc_struct = 0;
6359	}
6360	else
6361	{
6362	tree old_name;
6363	DECL_RESULT (new_decl) = remap_decl (DECL_RESULT (old_decl), id: &id);
6364	lang_hooks.dup_lang_specific_decl (DECL_RESULT (new_decl));
6365	if (gimple_in_ssa_p (fun: id.src_cfun)
6366	&& DECL_BY_REFERENCE (DECL_RESULT (old_decl))
6367	&& (old_name = ssa_default_def (id.src_cfun, DECL_RESULT (old_decl))))
6368	{
6369	tree new_name = make_ssa_name (DECL_RESULT (new_decl));
6370	insert_decl_map (id: &id, key: old_name, value: new_name);
6371	SSA_NAME_DEF_STMT (new_name) = gimple_build_nop ();
6372	set_ssa_default_def (cfun, DECL_RESULT (new_decl), new_name);
6373	}
6374	}
6375
6376	/* Set up the destination functions loop tree. */
6377	if (loops_for_fn (DECL_STRUCT_FUNCTION (old_decl)) != NULL)
6378	{
6379	cfun->curr_properties &= ~PROP_loops;
6380	loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
6381	cfun->curr_properties |= PROP_loops;
6382	}
6383
6384	/* Copy the Function's body. */
6385	copy_body (id: &id, ENTRY_BLOCK_PTR_FOR_FN (cfun), EXIT_BLOCK_PTR_FOR_FN (cfun),
6386	new_entry);
6387
6388	/* Renumber the lexical scoping (non-code) blocks consecutively. */
6389	number_blocks (new_decl);
6390
6391	/* We want to create the BB unconditionally, so that the addition of
6392	debug stmts doesn't affect BB count, which may in the end cause
6393	codegen differences. */
6394	bb = split_edge (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
6395	while (init_stmts.length ())
6396	insert_init_stmt (id: &id, bb, init_stmt: init_stmts.pop ());
6397	if (param_body_adjs)
6398	param_body_adjs->append_init_stmts (bb);
6399	update_clone_info (id: &id);
6400
6401	/* Remap the nonlocal_goto_save_area, if any. */
6402	if (cfun->nonlocal_goto_save_area)
6403	{
6404	struct walk_stmt_info wi;
6405
6406	memset (s: &wi, c: 0, n: sizeof (wi));
6407	wi.info = &id;
6408	walk_tree (&cfun->nonlocal_goto_save_area, remap_gimple_op_r, &wi, NULL);
6409	}
6410
6411	/* Clean up. */
6412	delete id.decl_map;
6413	if (id.debug_map)
6414	delete id.debug_map;
6415	free_dominance_info (CDI_DOMINATORS);
6416	free_dominance_info (CDI_POST_DOMINATORS);
6417
6418	update_max_bb_count ();
6419	fold_marked_statements (first: 0, statements: id.statements_to_fold);
6420	delete id.statements_to_fold;
6421	delete_unreachable_blocks_update_callgraph (dst_node: id.dst_node, update_clones);
6422	if (id.dst_node->definition)
6423	cgraph_edge::rebuild_references ();
6424	if (loops_state_satisfies_p (flags: LOOPS_NEED_FIXUP))
6425	{
6426	calculate_dominance_info (CDI_DOMINATORS);
6427	fix_loop_structure (NULL);
6428	}
6429	update_ssa (TODO_update_ssa);
6430
6431	/* After partial cloning we need to rescale frequencies, so they are
6432	within proper range in the cloned function. */
6433	if (new_entry)
6434	{
6435	struct cgraph_edge *e;
6436	rebuild_frequencies ();
6437
6438	new_version_node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
6439	for (e = new_version_node->callees; e; e = e->next_callee)
6440	{
6441	basic_block bb = gimple_bb (g: e->call_stmt);
6442	e->count = bb->count;
6443	}
6444	for (e = new_version_node->indirect_calls; e; e = e->next_callee)
6445	{
6446	basic_block bb = gimple_bb (g: e->call_stmt);
6447	e->count = bb->count;
6448	}
6449	}
6450
6451	if (param_body_adjs && MAY_HAVE_DEBUG_BIND_STMTS)
6452	{
6453	vec<tree, va_gc> **debug_args = NULL;
6454	unsigned int len = 0;
6455	unsigned reset_len = param_body_adjs->m_reset_debug_decls.length ();
6456
6457	for (i = 0; i < reset_len; i++)
6458	{
6459	tree parm = param_body_adjs->m_reset_debug_decls[i];
6460	gcc_assert (is_gimple_reg (parm));
6461	tree ddecl;
6462
6463	if (debug_args == NULL)
6464	{
6465	debug_args = decl_debug_args_insert (new_decl);
6466	len = vec_safe_length (v: *debug_args);
6467	}
6468	ddecl = build_debug_expr_decl (TREE_TYPE (parm));
6469	/* FIXME: Is setting the mode really necessary? */
6470	SET_DECL_MODE (ddecl, DECL_MODE (parm));
6471	vec_safe_push (v&: *debug_args, DECL_ORIGIN (parm));
6472	vec_safe_push (v&: *debug_args, obj: ddecl);
6473	}
6474	if (debug_args != NULL)
6475	{
6476	/* On the callee side, add
6477	DEBUG D#Y s=> parm
6478	DEBUG var => D#Y
6479	stmts to the first bb where var is a VAR_DECL created for the
6480	optimized away parameter in DECL_INITIAL block. This hints
6481	in the debug info that var (whole DECL_ORIGIN is the parm
6482	PARM_DECL) is optimized away, but could be looked up at the
6483	call site as value of D#X there. */
6484	gimple_stmt_iterator cgsi
6485	= gsi_after_labels (bb: single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
6486	gimple *def_temp;
6487	tree var = vars;
6488	i = vec_safe_length (v: *debug_args);
6489	do
6490	{
6491	tree vexpr = NULL_TREE;
6492	i -= 2;
6493	while (var != NULL_TREE
6494	&& DECL_ABSTRACT_ORIGIN (var) != (**debug_args)[i])
6495	var = TREE_CHAIN (var);
6496	if (var == NULL_TREE)
6497	break;
6498	tree parm = (**debug_args)[i];
6499	if (tree parm_ddef = ssa_default_def (id.src_cfun, parm))
6500	if (tree *d
6501	= param_body_adjs->m_dead_ssa_debug_equiv.get (k: parm_ddef))
6502	vexpr = *d;
6503	if (!vexpr)
6504	{
6505	vexpr = build_debug_expr_decl (TREE_TYPE (parm));
6506	/* FIXME: Is setting the mode really necessary? */
6507	SET_DECL_MODE (vexpr, DECL_MODE (parm));
6508	}
6509	def_temp = gimple_build_debug_bind (var, vexpr, NULL);
6510	gsi_insert_before (&cgsi, def_temp, GSI_NEW_STMT);
6511	def_temp = gimple_build_debug_source_bind (vexpr, parm, NULL);
6512	gsi_insert_before (&cgsi, def_temp, GSI_NEW_STMT);
6513	}
6514	while (i > len);
6515	}
6516	}
6517	delete param_body_adjs;
6518	free_dominance_info (CDI_DOMINATORS);
6519	free_dominance_info (CDI_POST_DOMINATORS);
6520
6521	gcc_assert (!id.debug_stmts.exists ());
6522	pop_cfun ();
6523	input_location = saved_location;
6524	return;
6525	}
6526
6527	/* EXP is CALL_EXPR present in a GENERIC expression tree. Try to integrate
6528	the callee and return the inlined body on success. */
6529
6530	tree
6531	maybe_inline_call_in_expr (tree exp)
6532	{
6533	tree fn = get_callee_fndecl (exp);
6534
6535	/* We can only try to inline "const" functions. */
6536	if (fn && TREE_READONLY (fn) && DECL_SAVED_TREE (fn))
6537	{
6538	call_expr_arg_iterator iter;
6539	copy_body_data id;
6540	tree param, arg, t;
6541	hash_map<tree, tree> decl_map;
6542
6543	/* Remap the parameters. */
6544	for (param = DECL_ARGUMENTS (fn), arg = first_call_expr_arg (exp, iter: &iter);
6545	param;
6546	param = DECL_CHAIN (param), arg = next_call_expr_arg (iter: &iter))
6547	decl_map.put (k: param, v: arg);
6548
6549	memset (s: &id, c: 0, n: sizeof (id));
6550	id.src_fn = fn;
6551	id.dst_fn = current_function_decl;
6552	id.src_cfun = DECL_STRUCT_FUNCTION (fn);
6553	id.decl_map = &decl_map;
6554
6555	id.copy_decl = copy_decl_no_change;
6556	id.transform_call_graph_edges = CB_CGE_DUPLICATE;
6557	id.transform_new_cfg = false;
6558	id.transform_return_to_modify = true;
6559	id.transform_parameter = true;
6560
6561	/* Make sure not to unshare trees behind the front-end's back
6562	since front-end specific mechanisms may rely on sharing. */
6563	id.regimplify = false;
6564	id.do_not_unshare = true;
6565
6566	/* We're not inside any EH region. */
6567	id.eh_lp_nr = 0;
6568
6569	t = copy_tree_body (id: &id);
6570
6571	/* We can only return something suitable for use in a GENERIC
6572	expression tree. */
6573	if (TREE_CODE (t) == MODIFY_EXPR)
6574	return TREE_OPERAND (t, 1);
6575	}
6576
6577	return NULL_TREE;
6578	}
6579
6580	/* Duplicate a type, fields and all. */
6581
6582	tree
6583	build_duplicate_type (tree type)
6584	{
6585	struct copy_body_data id;
6586
6587	memset (s: &id, c: 0, n: sizeof (id));
6588	id.src_fn = current_function_decl;
6589	id.dst_fn = current_function_decl;
6590	id.src_cfun = cfun;
6591	id.decl_map = new hash_map<tree, tree>;
6592	id.debug_map = NULL;
6593	id.copy_decl = copy_decl_no_change;
6594
6595	type = remap_type_1 (type, id: &id);
6596
6597	delete id.decl_map;
6598	if (id.debug_map)
6599	delete id.debug_map;
6600
6601	TYPE_CANONICAL (type) = type;
6602
6603	return type;
6604	}
6605
6606	/* Unshare the entire DECL_SAVED_TREE of FN and return the remapped
6607	parameters and RESULT_DECL in PARMS and RESULT. Used by C++ constexpr
6608	evaluation. */
6609
6610	tree
6611	copy_fn (tree fn, tree& parms, tree& result)
6612	{
6613	copy_body_data id;
6614	tree param;
6615	hash_map<tree, tree> decl_map;
6616
6617	tree *p = &parms;
6618	*p = NULL_TREE;
6619
6620	memset (s: &id, c: 0, n: sizeof (id));
6621	id.src_fn = fn;
6622	id.dst_fn = current_function_decl;
6623	id.src_cfun = DECL_STRUCT_FUNCTION (fn);
6624	id.decl_map = &decl_map;
6625
6626	id.copy_decl = [] (tree decl, copy_body_data *id)
6627	{
6628	if (TREE_CODE (decl) == TYPE_DECL || TREE_CODE (decl) == CONST_DECL)
6629	/* Don't make copies of local types or injected enumerators,
6630	the C++ constexpr evaluator doesn't need them and they
6631	confuse modules streaming. */
6632	return decl;
6633	return copy_decl_no_change (decl, id);
6634	};
6635	id.transform_call_graph_edges = CB_CGE_DUPLICATE;
6636	id.transform_new_cfg = false;
6637	id.transform_return_to_modify = false;
6638	id.transform_parameter = true;
6639
6640	/* Make sure not to unshare trees behind the front-end's back
6641	since front-end specific mechanisms may rely on sharing. */
6642	id.regimplify = false;
6643	id.do_not_unshare = true;
6644	id.do_not_fold = true;
6645
6646	/* We're not inside any EH region. */
6647	id.eh_lp_nr = 0;
6648
6649	/* Remap the parameters and result and return them to the caller. */
6650	for (param = DECL_ARGUMENTS (fn);
6651	param;
6652	param = DECL_CHAIN (param))
6653	{
6654	*p = remap_decl (decl: param, id: &id);
6655	p = &DECL_CHAIN (*p);
6656	}
6657
6658	if (DECL_RESULT (fn))
6659	result = remap_decl (DECL_RESULT (fn), id: &id);
6660	else
6661	result = NULL_TREE;
6662
6663	return copy_tree_body (id: &id);
6664	}
6665