1	/* Control flow functions for trees.
2	Copyright (C) 2001-2024 Free Software Foundation, Inc.
3	Contributed by Diego Novillo <dnovillo@redhat.com>
4
5	This file is part of GCC.
6
7	GCC is free software; you can redistribute it and/or modify
8	it under the terms of the GNU General Public License as published by
9	the Free Software Foundation; either version 3, or (at your option)
10	any later version.
11
12	GCC is distributed in the hope that it will be useful,
13	but WITHOUT ANY WARRANTY; without even the implied warranty of
14	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	GNU General Public License for more details.
16
17	You should have received a copy of the GNU General Public License
18	along with GCC; see the file COPYING3. If not see
19	<http://www.gnu.org/licenses/>. */
20
21	#include "config.h"
22	#include "system.h"
23	#include "coretypes.h"
24	#include "backend.h"
25	#include "target.h"
26	#include "rtl.h"
27	#include "tree.h"
28	#include "gimple.h"
29	#include "cfghooks.h"
30	#include "tree-pass.h"
31	#include "ssa.h"
32	#include "cgraph.h"
33	#include "gimple-pretty-print.h"
34	#include "diagnostic-core.h"
35	#include "fold-const.h"
36	#include "trans-mem.h"
37	#include "stor-layout.h"
38	#include "print-tree.h"
39	#include "cfganal.h"
40	#include "gimple-iterator.h"
41	#include "gimple-fold.h"
42	#include "tree-eh.h"
43	#include "gimplify-me.h"
44	#include "gimple-walk.h"
45	#include "tree-cfg.h"
46	#include "tree-ssa-loop-manip.h"
47	#include "tree-ssa-loop-niter.h"
48	#include "tree-into-ssa.h"
49	#include "tree-dfa.h"
50	#include "tree-ssa.h"
51	#include "except.h"
52	#include "cfgloop.h"
53	#include "tree-ssa-propagate.h"
54	#include "value-prof.h"
55	#include "tree-inline.h"
56	#include "tree-ssa-live.h"
57	#include "tree-ssa-dce.h"
58	#include "omp-general.h"
59	#include "omp-expand.h"
60	#include "tree-cfgcleanup.h"
61	#include "gimplify.h"
62	#include "attribs.h"
63	#include "selftest.h"
64	#include "opts.h"
65	#include "asan.h"
66	#include "profile.h"
67	#include "sreal.h"
68
69	/* This file contains functions for building the Control Flow Graph (CFG)
70	for a function tree. */
71
72	/* Local declarations. */
73
74	/* Initial capacity for the basic block array. */
75	static const int initial_cfg_capacity = 20;
76
77	/* This hash table allows us to efficiently lookup all CASE_LABEL_EXPRs
78	which use a particular edge. The CASE_LABEL_EXPRs are chained together
79	via their CASE_CHAIN field, which we clear after we're done with the
80	hash table to prevent problems with duplication of GIMPLE_SWITCHes.
81
82	Access to this list of CASE_LABEL_EXPRs allows us to efficiently
83	update the case vector in response to edge redirections.
84
85	Right now this table is set up and torn down at key points in the
86	compilation process. It would be nice if we could make the table
87	more persistent. The key is getting notification of changes to
88	the CFG (particularly edge removal, creation and redirection). */
89
90	static hash_map<edge, tree> *edge_to_cases;
91
92	/* If we record edge_to_cases, this bitmap will hold indexes
93	of basic blocks that end in a GIMPLE_SWITCH which we touched
94	due to edge manipulations. */
95
96	static bitmap touched_switch_bbs;
97
98	/* OpenMP region idxs for blocks during cfg pass. */
99	static vec<int> bb_to_omp_idx;
100
101	/* CFG statistics. */
102	struct cfg_stats_d
103	{
104	long num_merged_labels;
105	};
106
107	static struct cfg_stats_d cfg_stats;
108
109	/* Data to pass to replace_block_vars_by_duplicates_1. */
110	struct replace_decls_d
111	{
112	hash_map<tree, tree> *vars_map;
113	tree to_context;
114	};
115
116	/* Hash table to store last discriminator assigned for each locus. */
117	struct locus_discrim_map
118	{
119	int location_line;
120	int discriminator;
121	};
122
123	/* Hashtable helpers. */
124
125	struct locus_discrim_hasher : free_ptr_hash <locus_discrim_map>
126	{
127	static inline hashval_t hash (const locus_discrim_map *);
128	static inline bool equal (const locus_discrim_map *,
129	const locus_discrim_map *);
130	};
131
132	/* Trivial hash function for a location_t. ITEM is a pointer to
133	a hash table entry that maps a location_t to a discriminator. */
134
135	inline hashval_t
136	locus_discrim_hasher::hash (const locus_discrim_map *item)
137	{
138	return item->location_line;
139	}
140
141	/* Equality function for the locus-to-discriminator map. A and B
142	point to the two hash table entries to compare. */
143
144	inline bool
145	locus_discrim_hasher::equal (const locus_discrim_map *a,
146	const locus_discrim_map *b)
147	{
148	return a->location_line == b->location_line;
149	}
150
151	static hash_table<locus_discrim_hasher> *discriminator_per_locus;
152
153	/* Basic blocks and flowgraphs. */
154	static void make_blocks (gimple_seq);
155
156	/* Edges. */
157	static void make_edges (void);
158	static void assign_discriminators (void);
159	static void make_cond_expr_edges (basic_block);
160	static void make_gimple_switch_edges (gswitch *, basic_block);
161	static bool make_goto_expr_edges (basic_block);
162	static void make_gimple_asm_edges (basic_block);
163	static edge gimple_redirect_edge_and_branch (edge, basic_block);
164	static edge gimple_try_redirect_by_replacing_jump (edge, basic_block);
165
166	/* Various helpers. */
167	static inline bool stmt_starts_bb_p (gimple *, gimple *);
168	static bool gimple_verify_flow_info (void);
169	static void gimple_make_forwarder_block (edge);
170	static gimple *first_non_label_stmt (basic_block);
171	static bool verify_gimple_transaction (gtransaction *);
172	static bool call_can_make_abnormal_goto (gimple *);
173
174	/* Flowgraph optimization and cleanup. */
175	static void gimple_merge_blocks (basic_block, basic_block);
176	static bool gimple_can_merge_blocks_p (basic_block, basic_block);
177	static void remove_bb (basic_block);
178	static edge find_taken_edge_computed_goto (basic_block, tree);
179	static edge find_taken_edge_cond_expr (const gcond *, tree);
180
181	void
182	init_empty_tree_cfg_for_function (struct function *fn)
183	{
184	/* Initialize the basic block array. */
185	init_flow (fn);
186	profile_status_for_fn (fn) = PROFILE_ABSENT;
187	n_basic_blocks_for_fn (fn) = NUM_FIXED_BLOCKS;
188	last_basic_block_for_fn (fn) = NUM_FIXED_BLOCKS;
189	vec_safe_grow_cleared (basic_block_info_for_fn (fn),
190	len: initial_cfg_capacity, exact: true);
191
192	/* Build a mapping of labels to their associated blocks. */
193	vec_safe_grow_cleared (label_to_block_map_for_fn (fn),
194	len: initial_cfg_capacity, exact: true);
195
196	SET_BASIC_BLOCK_FOR_FN (fn, ENTRY_BLOCK, ENTRY_BLOCK_PTR_FOR_FN (fn));
197	SET_BASIC_BLOCK_FOR_FN (fn, EXIT_BLOCK, EXIT_BLOCK_PTR_FOR_FN (fn));
198
199	ENTRY_BLOCK_PTR_FOR_FN (fn)->next_bb
200	= EXIT_BLOCK_PTR_FOR_FN (fn);
201	EXIT_BLOCK_PTR_FOR_FN (fn)->prev_bb
202	= ENTRY_BLOCK_PTR_FOR_FN (fn);
203	}
204
205	void
206	init_empty_tree_cfg (void)
207	{
208	init_empty_tree_cfg_for_function (cfun);
209	}
210
211	/*---------------------------------------------------------------------------
212	Create basic blocks
213	---------------------------------------------------------------------------*/
214
215	/* Entry point to the CFG builder for trees. SEQ is the sequence of
216	statements to be added to the flowgraph. */
217
218	static void
219	build_gimple_cfg (gimple_seq seq)
220	{
221	/* Register specific gimple functions. */
222	gimple_register_cfg_hooks ();
223
224	memset (s: (void *) &cfg_stats, c: 0, n: sizeof (cfg_stats));
225
226	init_empty_tree_cfg ();
227
228	make_blocks (seq);
229
230	/* Make sure there is always at least one block, even if it's empty. */
231	if (n_basic_blocks_for_fn (cfun) == NUM_FIXED_BLOCKS)
232	create_empty_bb (ENTRY_BLOCK_PTR_FOR_FN (cfun));
233
234	/* Adjust the size of the array. */
235	if (basic_block_info_for_fn (cfun)->length ()
236	< (size_t) n_basic_blocks_for_fn (cfun))
237	vec_safe_grow_cleared (basic_block_info_for_fn (cfun),
238	n_basic_blocks_for_fn (cfun));
239
240	/* To speed up statement iterator walks, we first purge dead labels. */
241	cleanup_dead_labels ();
242
243	/* Group case nodes to reduce the number of edges.
244	We do this after cleaning up dead labels because otherwise we miss
245	a lot of obvious case merging opportunities. */
246	group_case_labels ();
247
248	/* Create the edges of the flowgraph. */
249	discriminator_per_locus = new hash_table<locus_discrim_hasher> (13);
250	make_edges ();
251	assign_discriminators ();
252	cleanup_dead_labels ();
253	delete discriminator_per_locus;
254	discriminator_per_locus = NULL;
255	}
256
257	/* Look for ANNOTATE calls with loop annotation kind in BB; if found, remove
258	them and propagate the information to LOOP. We assume that the annotations
259	come immediately before the condition in BB, if any. */
260
261	static void
262	replace_loop_annotate_in_block (basic_block bb, class loop *loop)
263	{
264	gimple_stmt_iterator gsi = gsi_last_bb (bb);
265	gimple *stmt = gsi_stmt (i: gsi);
266
267	if (!(stmt && gimple_code (g: stmt) == GIMPLE_COND))
268	return;
269
270	for (gsi_prev_nondebug (i: &gsi); !gsi_end_p (i: gsi); gsi_prev (i: &gsi))
271	{
272	stmt = gsi_stmt (i: gsi);
273	if (gimple_code (g: stmt) != GIMPLE_CALL)
274	break;
275	if (!gimple_call_internal_p (gs: stmt)
276	|| gimple_call_internal_fn (gs: stmt) != IFN_ANNOTATE)
277	break;
278
279	switch ((annot_expr_kind) tree_to_shwi (gimple_call_arg (gs: stmt, index: 1)))
280	{
281	case annot_expr_ivdep_kind:
282	loop->safelen = INT_MAX;
283	break;
284	case annot_expr_unroll_kind:
285	loop->unroll
286	= (unsigned short) tree_to_shwi (gimple_call_arg (gs: stmt, index: 2));
287	cfun->has_unroll = true;
288	break;
289	case annot_expr_no_vector_kind:
290	loop->dont_vectorize = true;
291	break;
292	case annot_expr_vector_kind:
293	loop->force_vectorize = true;
294	cfun->has_force_vectorize_loops = true;
295	break;
296	case annot_expr_parallel_kind:
297	loop->can_be_parallel = true;
298	loop->safelen = INT_MAX;
299	break;
300	case annot_expr_maybe_infinite_kind:
301	loop->finite_p = false;
302	break;
303	default:
304	gcc_unreachable ();
305	}
306
307	stmt = gimple_build_assign (gimple_call_lhs (gs: stmt),
308	gimple_call_arg (gs: stmt, index: 0));
309	gsi_replace (&gsi, stmt, true);
310	}
311	}
312
313	/* Look for ANNOTATE calls with loop annotation kind; if found, remove
314	them and propagate the information to the loop. We assume that the
315	annotations come immediately before the condition of the loop. */
316
317	static void
318	replace_loop_annotate (void)
319	{
320	basic_block bb;
321	gimple_stmt_iterator gsi;
322	gimple *stmt;
323
324	for (auto loop : loops_list (cfun, 0))
325	{
326	/* Push the global flag_finite_loops state down to individual loops. */
327	loop->finite_p = flag_finite_loops;
328
329	/* Check all exit source blocks for annotations. */
330	for (auto e : get_loop_exit_edges (loop))
331	replace_loop_annotate_in_block (bb: e->src, loop);
332	}
333
334	/* Remove IFN_ANNOTATE. Safeguard for the case loop->latch == NULL. */
335	FOR_EACH_BB_FN (bb, cfun)
336	{
337	for (gsi = gsi_last_bb (bb); !gsi_end_p (i: gsi); gsi_prev (i: &gsi))
338	{
339	stmt = gsi_stmt (i: gsi);
340	if (gimple_code (g: stmt) != GIMPLE_CALL)
341	continue;
342	if (!gimple_call_internal_p (gs: stmt)
343	|| gimple_call_internal_fn (gs: stmt) != IFN_ANNOTATE)
344	continue;
345
346	switch ((annot_expr_kind) tree_to_shwi (gimple_call_arg (gs: stmt, index: 1)))
347	{
348	case annot_expr_ivdep_kind:
349	case annot_expr_unroll_kind:
350	case annot_expr_no_vector_kind:
351	case annot_expr_vector_kind:
352	case annot_expr_parallel_kind:
353	case annot_expr_maybe_infinite_kind:
354	break;
355	default:
356	gcc_unreachable ();
357	}
358
359	warning_at (gimple_location (g: stmt), 0, "ignoring loop annotation");
360	stmt = gimple_build_assign (gimple_call_lhs (gs: stmt),
361	gimple_call_arg (gs: stmt, index: 0));
362	gsi_replace (&gsi, stmt, true);
363	}
364	}
365	}
366
367	static unsigned int
368	execute_build_cfg (void)
369	{
370	gimple_seq body = gimple_body (current_function_decl);
371
372	build_gimple_cfg (seq: body);
373	gimple_set_body (current_function_decl, NULL);
374	if (dump_file && (dump_flags & TDF_DETAILS))
375	{
376	fprintf (stream: dump_file, format: "Scope blocks:\n");
377	dump_scope_blocks (dump_file, dump_flags);
378	}
379	cleanup_tree_cfg ();
380
381	bb_to_omp_idx.release ();
382
383	loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
384	replace_loop_annotate ();
385	return 0;
386	}
387
388	namespace {
389
390	const pass_data pass_data_build_cfg =
391	{
392	.type: GIMPLE_PASS, /* type */
393	.name: "cfg", /* name */
394	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
395	.tv_id: TV_TREE_CFG, /* tv_id */
396	PROP_gimple_leh, /* properties_required */
397	.properties_provided: ( PROP_cfg | PROP_loops ), /* properties_provided */
398	.properties_destroyed: 0, /* properties_destroyed */
399	.todo_flags_start: 0, /* todo_flags_start */
400	.todo_flags_finish: 0, /* todo_flags_finish */
401	};
402
403	class pass_build_cfg : public gimple_opt_pass
404	{
405	public:
406	pass_build_cfg (gcc::context *ctxt)
407	: gimple_opt_pass (pass_data_build_cfg, ctxt)
408	{}
409
410	/* opt_pass methods: */
411	unsigned int execute (function *) final override
412	{
413	return execute_build_cfg ();
414	}
415
416	}; // class pass_build_cfg
417
418	} // anon namespace
419
420	gimple_opt_pass *
421	make_pass_build_cfg (gcc::context *ctxt)
422	{
423	return new pass_build_cfg (ctxt);
424	}
425
426
427	/* Return true if T is a computed goto. */
428
429	bool
430	computed_goto_p (gimple *t)
431	{
432	return (gimple_code (g: t) == GIMPLE_GOTO
433	&& TREE_CODE (gimple_goto_dest (t)) != LABEL_DECL);
434	}
435
436	/* Returns true if the sequence of statements STMTS only contains
437	a call to __builtin_unreachable (). */
438
439	bool
440	gimple_seq_unreachable_p (gimple_seq stmts)
441	{
442	if (stmts == NULL
443	/* Return false if -fsanitize=unreachable, we don't want to
444	optimize away those calls, but rather turn them into
445	__ubsan_handle_builtin_unreachable () or __builtin_trap ()
446	later. */
447	|| sanitize_flags_p (flag: SANITIZE_UNREACHABLE))
448	return false;
449
450	gimple_stmt_iterator gsi = gsi_last (seq&: stmts);
451
452	if (!gimple_call_builtin_p (gsi_stmt (i: gsi), BUILT_IN_UNREACHABLE))
453	return false;
454
455	for (gsi_prev (i: &gsi); !gsi_end_p (i: gsi); gsi_prev (i: &gsi))
456	{
457	gimple *stmt = gsi_stmt (i: gsi);
458	if (gimple_code (g: stmt) != GIMPLE_LABEL
459	&& !is_gimple_debug (gs: stmt)
460	&& !gimple_clobber_p (s: stmt))
461	return false;
462	}
463	return true;
464	}
465
466	/* Returns true for edge E where e->src ends with a GIMPLE_COND and
467	the other edge points to a bb with just __builtin_unreachable ().
468	I.e. return true for C->M edge in:
469	<bb C>:
470	...
471	if (something)
472	goto <bb N>;
473	else
474	goto <bb M>;
475	<bb N>:
476	__builtin_unreachable ();
477	<bb M>: */
478
479	bool
480	assert_unreachable_fallthru_edge_p (edge e)
481	{
482	basic_block pred_bb = e->src;
483	if (safe_is_a <gcond *> (p: *gsi_last_bb (bb: pred_bb)))
484	{
485	basic_block other_bb = EDGE_SUCC (pred_bb, 0)->dest;
486	if (other_bb == e->dest)
487	other_bb = EDGE_SUCC (pred_bb, 1)->dest;
488	if (EDGE_COUNT (other_bb->succs) == 0)
489	return gimple_seq_unreachable_p (stmts: bb_seq (bb: other_bb));
490	}
491	return false;
492	}
493
494
495	/* Initialize GF_CALL_CTRL_ALTERING flag, which indicates the call
496	could alter control flow except via eh. We initialize the flag at
497	CFG build time and only ever clear it later. */
498
499	static void
500	gimple_call_initialize_ctrl_altering (gimple *stmt)
501	{
502	int flags = gimple_call_flags (stmt);
503
504	/* A call alters control flow if it can make an abnormal goto. */
505	if (call_can_make_abnormal_goto (stmt)
506	/* A call also alters control flow if it does not return. */
507	|| flags & ECF_NORETURN
508	/* TM ending statements have backedges out of the transaction.
509	Return true so we split the basic block containing them.
510	Note that the TM_BUILTIN test is merely an optimization. */
511	|| ((flags & ECF_TM_BUILTIN)
512	&& is_tm_ending_fndecl (gimple_call_fndecl (gs: stmt)))
513	/* BUILT_IN_RETURN call is same as return statement. */
514	|| gimple_call_builtin_p (stmt, BUILT_IN_RETURN)
515	/* IFN_UNIQUE should be the last insn, to make checking for it
516	as cheap as possible. */
517	|| (gimple_call_internal_p (gs: stmt)
518	&& gimple_call_internal_unique_p (gs: stmt)))
519	gimple_call_set_ctrl_altering (s: stmt, ctrl_altering_p: true);
520	else
521	gimple_call_set_ctrl_altering (s: stmt, ctrl_altering_p: false);
522	}
523
524
525	/* Insert SEQ after BB and build a flowgraph. */
526
527	static basic_block
528	make_blocks_1 (gimple_seq seq, basic_block bb)
529	{
530	gimple_stmt_iterator i = gsi_start (seq);
531	gimple *stmt = NULL;
532	gimple *prev_stmt = NULL;
533	bool start_new_block = true;
534	bool first_stmt_of_seq = true;
535
536	while (!gsi_end_p (i))
537	{
538	/* PREV_STMT should only be set to a debug stmt if the debug
539	stmt is before nondebug stmts. Once stmt reaches a nondebug
540	nonlabel, prev_stmt will be set to it, so that
541	stmt_starts_bb_p will know to start a new block if a label is
542	found. However, if stmt was a label after debug stmts only,
543	keep the label in prev_stmt even if we find further debug
544	stmts, for there may be other labels after them, and they
545	should land in the same block. */
546	if (!prev_stmt || !stmt || !is_gimple_debug (gs: stmt))
547	prev_stmt = stmt;
548	stmt = gsi_stmt (i);
549
550	if (stmt && is_gimple_call (gs: stmt))
551	gimple_call_initialize_ctrl_altering (stmt);
552
553	/* If the statement starts a new basic block or if we have determined
554	in a previous pass that we need to create a new block for STMT, do
555	so now. */
556	if (start_new_block || stmt_starts_bb_p (stmt, prev_stmt))
557	{
558	if (!first_stmt_of_seq)
559	gsi_split_seq_before (&i, &seq);
560	bb = create_basic_block (seq, bb);
561	start_new_block = false;
562	prev_stmt = NULL;
563	}
564
565	/* Now add STMT to BB and create the subgraphs for special statement
566	codes. */
567	gimple_set_bb (stmt, bb);
568
569	/* If STMT is a basic block terminator, set START_NEW_BLOCK for the
570	next iteration. */
571	if (stmt_ends_bb_p (stmt))
572	{
573	/* If the stmt can make abnormal goto use a new temporary
574	for the assignment to the LHS. This makes sure the old value
575	of the LHS is available on the abnormal edge. Otherwise
576	we will end up with overlapping life-ranges for abnormal
577	SSA names. */
578	if (gimple_has_lhs (stmt)
579	&& stmt_can_make_abnormal_goto (stmt)
580	&& is_gimple_reg_type (TREE_TYPE (gimple_get_lhs (stmt))))
581	{
582	tree lhs = gimple_get_lhs (stmt);
583	tree tmp = create_tmp_var (TREE_TYPE (lhs));
584	gimple *s = gimple_build_assign (lhs, tmp);
585	gimple_set_location (g: s, location: gimple_location (g: stmt));
586	gimple_set_block (g: s, block: gimple_block (g: stmt));
587	gimple_set_lhs (stmt, tmp);
588	gsi_insert_after (&i, s, GSI_SAME_STMT);
589	}
590	start_new_block = true;
591	}
592
593	gsi_next (i: &i);
594	first_stmt_of_seq = false;
595	}
596	return bb;
597	}
598
599	/* Build a flowgraph for the sequence of stmts SEQ. */
600
601	static void
602	make_blocks (gimple_seq seq)
603	{
604	/* Look for debug markers right before labels, and move the debug
605	stmts after the labels. Accepting labels among debug markers
606	adds no value, just complexity; if we wanted to annotate labels
607	with view numbers (so sequencing among markers would matter) or
608	somesuch, we're probably better off still moving the labels, but
609	adding other debug annotations in their original positions or
610	emitting nonbind or bind markers associated with the labels in
611	the original position of the labels.
612
613	Moving labels would probably be simpler, but we can't do that:
614	moving labels assigns label ids to them, and doing so because of
615	debug markers makes for -fcompare-debug and possibly even codegen
616	differences. So, we have to move the debug stmts instead. To
617	that end, we scan SEQ backwards, marking the position of the
618	latest (earliest we find) label, and moving debug stmts that are
619	not separated from it by nondebug nonlabel stmts after the
620	label. */
621	if (MAY_HAVE_DEBUG_MARKER_STMTS)
622	{
623	gimple_stmt_iterator label = gsi_none ();
624
625	for (gimple_stmt_iterator i = gsi_last (seq); !gsi_end_p (i); gsi_prev (i: &i))
626	{
627	gimple *stmt = gsi_stmt (i);
628
629	/* If this is the first label we encounter (latest in SEQ)
630	before nondebug stmts, record its position. */
631	if (is_a <glabel *> (p: stmt))
632	{
633	if (gsi_end_p (i: label))
634	label = i;
635	continue;
636	}
637
638	/* Without a recorded label position to move debug stmts to,
639	there's nothing to do. */
640	if (gsi_end_p (i: label))
641	continue;
642
643	/* Move the debug stmt at I after LABEL. */
644	if (is_gimple_debug (gs: stmt))
645	{
646	gcc_assert (gimple_debug_nonbind_marker_p (stmt));
647	/* As STMT is removed, I advances to the stmt after
648	STMT, so the gsi_prev in the for "increment"
649	expression gets us to the stmt we're to visit after
650	STMT. LABEL, however, would advance to the moved
651	stmt if we passed it to gsi_move_after, so pass it a
652	copy instead, so as to keep LABEL pointing to the
653	LABEL. */
654	gimple_stmt_iterator copy = label;
655	gsi_move_after (&i, &copy);
656	continue;
657	}
658
659	/* There aren't any (more?) debug stmts before label, so
660	there isn't anything else to move after it. */
661	label = gsi_none ();
662	}
663	}
664
665	make_blocks_1 (seq, ENTRY_BLOCK_PTR_FOR_FN (cfun));
666	}
667
668	/* Create and return a new empty basic block after bb AFTER. */
669
670	static basic_block
671	create_bb (void *h, void *e, basic_block after)
672	{
673	basic_block bb;
674
675	gcc_assert (!e);
676
677	/* Create and initialize a new basic block. Since alloc_block uses
678	GC allocation that clears memory to allocate a basic block, we do
679	not have to clear the newly allocated basic block here. */
680	bb = alloc_block ();
681
682	bb->index = last_basic_block_for_fn (cfun);
683	bb->flags = BB_NEW;
684	set_bb_seq (bb, seq: h ? (gimple_seq) h : NULL);
685
686	/* Add the new block to the linked list of blocks. */
687	link_block (bb, after);
688
689	/* Grow the basic block array if needed. */
690	if ((size_t) last_basic_block_for_fn (cfun)
691	== basic_block_info_for_fn (cfun)->length ())
692	vec_safe_grow_cleared (basic_block_info_for_fn (cfun),
693	last_basic_block_for_fn (cfun) + 1);
694
695	/* Add the newly created block to the array. */
696	SET_BASIC_BLOCK_FOR_FN (cfun, last_basic_block_for_fn (cfun), bb);
697
698	n_basic_blocks_for_fn (cfun)++;
699	last_basic_block_for_fn (cfun)++;
700
701	return bb;
702	}
703
704
705	/*---------------------------------------------------------------------------
706	Edge creation
707	---------------------------------------------------------------------------*/
708
709	/* If basic block BB has an abnormal edge to a basic block
710	containing IFN_ABNORMAL_DISPATCHER internal call, return
711	that the dispatcher's basic block, otherwise return NULL. */
712
713	basic_block
714	get_abnormal_succ_dispatcher (basic_block bb)
715	{
716	edge e;
717	edge_iterator ei;
718
719	FOR_EACH_EDGE (e, ei, bb->succs)
720	if ((e->flags & (EDGE_ABNORMAL | EDGE_EH)) == EDGE_ABNORMAL)
721	{
722	gimple_stmt_iterator gsi
723	= gsi_start_nondebug_after_labels_bb (bb: e->dest);
724	gimple *g = gsi_stmt (i: gsi);
725	if (g && gimple_call_internal_p (gs: g, fn: IFN_ABNORMAL_DISPATCHER))
726	return e->dest;
727	}
728	return NULL;
729	}
730
731	/* Helper function for make_edges. Create a basic block with
732	with ABNORMAL_DISPATCHER internal call in it if needed, and
733	create abnormal edges from BBS to it and from it to FOR_BB
734	if COMPUTED_GOTO is false, otherwise factor the computed gotos. */
735
736	static void
737	handle_abnormal_edges (basic_block *dispatcher_bbs, basic_block for_bb,
738	auto_vec<basic_block> *bbs, bool computed_goto)
739	{
740	basic_block *dispatcher = dispatcher_bbs + (computed_goto ? 1 : 0);
741	unsigned int idx = 0;
742	basic_block bb;
743	bool inner = false;
744
745	if (!bb_to_omp_idx.is_empty ())
746	{
747	dispatcher = dispatcher_bbs + 2 * bb_to_omp_idx[for_bb->index];
748	if (bb_to_omp_idx[for_bb->index] != 0)
749	inner = true;
750	}
751
752	/* If the dispatcher has been created already, then there are basic
753	blocks with abnormal edges to it, so just make a new edge to
754	for_bb. */
755	if (*dispatcher == NULL)
756	{
757	/* Check if there are any basic blocks that need to have
758	abnormal edges to this dispatcher. If there are none, return
759	early. */
760	if (bb_to_omp_idx.is_empty ())
761	{
762	if (bbs->is_empty ())
763	return;
764	}
765	else
766	{
767	FOR_EACH_VEC_ELT (*bbs, idx, bb)
768	if (bb_to_omp_idx[bb->index] == bb_to_omp_idx[for_bb->index])
769	break;
770	if (bb == NULL)
771	return;
772	}
773
774	/* Create the dispatcher bb. */
775	*dispatcher = create_basic_block (NULL, for_bb);
776	if (computed_goto)
777	{
778	/* Factor computed gotos into a common computed goto site. Also
779	record the location of that site so that we can un-factor the
780	gotos after we have converted back to normal form. */
781	gimple_stmt_iterator gsi = gsi_start_bb (bb: *dispatcher);
782
783	/* Create the destination of the factored goto. Each original
784	computed goto will put its desired destination into this
785	variable and jump to the label we create immediately below. */
786	tree var = create_tmp_var (ptr_type_node, "gotovar");
787
788	/* Build a label for the new block which will contain the
789	factored computed goto. */
790	tree factored_label_decl
791	= create_artificial_label (UNKNOWN_LOCATION);
792	gimple *factored_computed_goto_label
793	= gimple_build_label (label: factored_label_decl);
794	gsi_insert_after (&gsi, factored_computed_goto_label, GSI_NEW_STMT);
795
796	/* Build our new computed goto. */
797	gimple *factored_computed_goto = gimple_build_goto (dest: var);
798	gsi_insert_after (&gsi, factored_computed_goto, GSI_NEW_STMT);
799
800	FOR_EACH_VEC_ELT (*bbs, idx, bb)
801	{
802	if (!bb_to_omp_idx.is_empty ()
803	&& bb_to_omp_idx[bb->index] != bb_to_omp_idx[for_bb->index])
804	continue;
805
806	gsi = gsi_last_bb (bb);
807	gimple *last = gsi_stmt (i: gsi);
808
809	gcc_assert (computed_goto_p (last));
810
811	/* Copy the original computed goto's destination into VAR. */
812	gimple *assignment
813	= gimple_build_assign (var, gimple_goto_dest (gs: last));
814	gsi_insert_before (&gsi, assignment, GSI_SAME_STMT);
815
816	edge e = make_edge (bb, *dispatcher, EDGE_FALLTHRU);
817	e->goto_locus = gimple_location (g: last);
818	gsi_remove (&gsi, true);
819	}
820	}
821	else
822	{
823	tree arg = inner ? boolean_true_node : boolean_false_node;
824	gcall *g = gimple_build_call_internal (IFN_ABNORMAL_DISPATCHER,
825	1, arg);
826	gimple_call_set_ctrl_altering (s: g, ctrl_altering_p: true);
827	gimple_stmt_iterator gsi = gsi_after_labels (bb: *dispatcher);
828	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
829
830	/* Create predecessor edges of the dispatcher. */
831	FOR_EACH_VEC_ELT (*bbs, idx, bb)
832	{
833	if (!bb_to_omp_idx.is_empty ()
834	&& bb_to_omp_idx[bb->index] != bb_to_omp_idx[for_bb->index])
835	continue;
836	make_edge (bb, *dispatcher, EDGE_ABNORMAL);
837	}
838	}
839	}
840
841	make_edge (*dispatcher, for_bb, EDGE_ABNORMAL);
842	}
843
844	/* Creates outgoing edges for BB. Returns 1 when it ends with an
845	computed goto, returns 2 when it ends with a statement that
846	might return to this function via an nonlocal goto, otherwise
847	return 0. Updates *PCUR_REGION with the OMP region this BB is in. */
848
849	static int
850	make_edges_bb (basic_block bb, struct omp_region **pcur_region, int *pomp_index)
851	{
852	gimple *last = *gsi_last_bb (bb);
853	bool fallthru = false;
854	int ret = 0;
855
856	if (!last)
857	return ret;
858
859	switch (gimple_code (g: last))
860	{
861	case GIMPLE_GOTO:
862	if (make_goto_expr_edges (bb))
863	ret = 1;
864	fallthru = false;
865	break;
866	case GIMPLE_RETURN:
867	{
868	edge e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
869	e->goto_locus = gimple_location (g: last);
870	fallthru = false;
871	}
872	break;
873	case GIMPLE_COND:
874	make_cond_expr_edges (bb);
875	fallthru = false;
876	break;
877	case GIMPLE_SWITCH:
878	make_gimple_switch_edges (as_a <gswitch *> (p: last), bb);
879	fallthru = false;
880	break;
881	case GIMPLE_RESX:
882	make_eh_edge (last);
883	fallthru = false;
884	break;
885	case GIMPLE_EH_DISPATCH:
886	fallthru = make_eh_dispatch_edges (as_a <geh_dispatch *> (p: last));
887	break;
888
889	case GIMPLE_CALL:
890	/* If this function receives a nonlocal goto, then we need to
891	make edges from this call site to all the nonlocal goto
892	handlers. */
893	if (stmt_can_make_abnormal_goto (last))
894	ret = 2;
895
896	/* If this statement has reachable exception handlers, then
897	create abnormal edges to them. */
898	make_eh_edge (last);
899
900	/* BUILTIN_RETURN is really a return statement. */
901	if (gimple_call_builtin_p (last, BUILT_IN_RETURN))
902	{
903	make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
904	fallthru = false;
905	}
906	/* Some calls are known not to return. */
907	else
908	fallthru = !gimple_call_noreturn_p (s: last);
909	break;
910
911	case GIMPLE_ASSIGN:
912	/* A GIMPLE_ASSIGN may throw internally and thus be considered
913	control-altering. */
914	if (is_ctrl_altering_stmt (last))
915	make_eh_edge (last);
916	fallthru = true;
917	break;
918
919	case GIMPLE_ASM:
920	make_gimple_asm_edges (bb);
921	fallthru = true;
922	break;
923
924	CASE_GIMPLE_OMP:
925	fallthru = omp_make_gimple_edges (bb, region: pcur_region, region_idx: pomp_index);
926	break;
927
928	case GIMPLE_TRANSACTION:
929	{
930	gtransaction *txn = as_a <gtransaction *> (p: last);
931	tree label1 = gimple_transaction_label_norm (transaction_stmt: txn);
932	tree label2 = gimple_transaction_label_uninst (transaction_stmt: txn);
933
934	if (label1)
935	make_edge (bb, label_to_block (cfun, label1), EDGE_FALLTHRU);
936	if (label2)
937	make_edge (bb, label_to_block (cfun, label2),
938	EDGE_TM_UNINSTRUMENTED | (label1 ? 0 : EDGE_FALLTHRU));
939
940	tree label3 = gimple_transaction_label_over (transaction_stmt: txn);
941	if (gimple_transaction_subcode (transaction_stmt: txn)
942	& (GTMA_HAVE_ABORT | GTMA_IS_OUTER))
943	make_edge (bb, label_to_block (cfun, label3), EDGE_TM_ABORT);
944
945	fallthru = false;
946	}
947	break;
948
949	default:
950	gcc_assert (!stmt_ends_bb_p (last));
951	fallthru = true;
952	break;
953	}
954
955	if (fallthru)
956	make_edge (bb, bb->next_bb, EDGE_FALLTHRU);
957
958	return ret;
959	}
960
961	/* Join all the blocks in the flowgraph. */
962
963	static void
964	make_edges (void)
965	{
966	basic_block bb;
967	struct omp_region *cur_region = NULL;
968	auto_vec<basic_block> ab_edge_goto;
969	auto_vec<basic_block> ab_edge_call;
970	int cur_omp_region_idx = 0;
971
972	/* Create an edge from entry to the first block with executable
973	statements in it. */
974	make_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun),
975	BASIC_BLOCK_FOR_FN (cfun, NUM_FIXED_BLOCKS),
976	EDGE_FALLTHRU);
977
978	/* Traverse the basic block array placing edges. */
979	FOR_EACH_BB_FN (bb, cfun)
980	{
981	int mer;
982
983	if (!bb_to_omp_idx.is_empty ())
984	bb_to_omp_idx[bb->index] = cur_omp_region_idx;
985
986	mer = make_edges_bb (bb, pcur_region: &cur_region, pomp_index: &cur_omp_region_idx);
987	if (mer == 1)
988	ab_edge_goto.safe_push (obj: bb);
989	else if (mer == 2)
990	ab_edge_call.safe_push (obj: bb);
991
992	if (cur_region && bb_to_omp_idx.is_empty ())
993	bb_to_omp_idx.safe_grow_cleared (n_basic_blocks_for_fn (cfun), exact: true);
994	}
995
996	/* Computed gotos are hell to deal with, especially if there are
997	lots of them with a large number of destinations. So we factor
998	them to a common computed goto location before we build the
999	edge list. After we convert back to normal form, we will un-factor
1000	the computed gotos since factoring introduces an unwanted jump.
1001	For non-local gotos and abnormal edges from calls to calls that return
1002	twice or forced labels, factor the abnormal edges too, by having all
1003	abnormal edges from the calls go to a common artificial basic block
1004	with ABNORMAL_DISPATCHER internal call and abnormal edges from that
1005	basic block to all forced labels and calls returning twice.
1006	We do this per-OpenMP structured block, because those regions
1007	are guaranteed to be single entry single exit by the standard,
1008	so it is not allowed to enter or exit such regions abnormally this way,
1009	thus all computed gotos, non-local gotos and setjmp/longjmp calls
1010	must not transfer control across SESE region boundaries. */
1011	if (!ab_edge_goto.is_empty () || !ab_edge_call.is_empty ())
1012	{
1013	gimple_stmt_iterator gsi;
1014	basic_block dispatcher_bb_array[2] = { NULL, NULL };
1015	basic_block *dispatcher_bbs = dispatcher_bb_array;
1016	int count = n_basic_blocks_for_fn (cfun);
1017
1018	if (!bb_to_omp_idx.is_empty ())
1019	dispatcher_bbs = XCNEWVEC (basic_block, 2 * count);
1020
1021	FOR_EACH_BB_FN (bb, cfun)
1022	{
1023	for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
1024	{
1025	glabel *label_stmt = dyn_cast <glabel *> (p: gsi_stmt (i: gsi));
1026	tree target;
1027
1028	if (!label_stmt)
1029	break;
1030
1031	target = gimple_label_label (gs: label_stmt);
1032
1033	/* Make an edge to every label block that has been marked as a
1034	potential target for a computed goto or a non-local goto. */
1035	if (FORCED_LABEL (target))
1036	handle_abnormal_edges (dispatcher_bbs, for_bb: bb, bbs: &ab_edge_goto,
1037	computed_goto: true);
1038	if (DECL_NONLOCAL (target))
1039	{
1040	handle_abnormal_edges (dispatcher_bbs, for_bb: bb, bbs: &ab_edge_call,
1041	computed_goto: false);
1042	break;
1043	}
1044	}
1045
1046	if (!gsi_end_p (i: gsi) && is_gimple_debug (gs: gsi_stmt (i: gsi)))
1047	gsi_next_nondebug (i: &gsi);
1048	if (!gsi_end_p (i: gsi))
1049	{
1050	/* Make an edge to every setjmp-like call. */
1051	gimple *call_stmt = gsi_stmt (i: gsi);
1052	if (is_gimple_call (gs: call_stmt)
1053	&& ((gimple_call_flags (call_stmt) & ECF_RETURNS_TWICE)
1054	|| gimple_call_builtin_p (call_stmt,
1055	BUILT_IN_SETJMP_RECEIVER)))
1056	handle_abnormal_edges (dispatcher_bbs, for_bb: bb, bbs: &ab_edge_call,
1057	computed_goto: false);
1058	}
1059	}
1060
1061	if (!bb_to_omp_idx.is_empty ())
1062	XDELETE (dispatcher_bbs);
1063	}
1064
1065	omp_free_regions ();
1066	}
1067
1068	/* Add SEQ after GSI. Start new bb after GSI, and created further bbs as
1069	needed. Returns true if new bbs were created.
1070	Note: This is transitional code, and should not be used for new code. We
1071	should be able to get rid of this by rewriting all target va-arg
1072	gimplification hooks to use an interface gimple_build_cond_value as described
1073	in https://gcc.gnu.org/ml/gcc-patches/2015-02/msg01194.html. */
1074
1075	bool
1076	gimple_find_sub_bbs (gimple_seq seq, gimple_stmt_iterator *gsi)
1077	{
1078	gimple *stmt = gsi_stmt (i: *gsi);
1079	basic_block bb = gimple_bb (g: stmt);
1080	basic_block lastbb, afterbb;
1081	int old_num_bbs = n_basic_blocks_for_fn (cfun);
1082	edge e;
1083	lastbb = make_blocks_1 (seq, bb);
1084	if (old_num_bbs == n_basic_blocks_for_fn (cfun))
1085	return false;
1086	e = split_block (bb, stmt);
1087	/* Move e->dest to come after the new basic blocks. */
1088	afterbb = e->dest;
1089	unlink_block (afterbb);
1090	link_block (afterbb, lastbb);
1091	redirect_edge_succ (e, bb->next_bb);
1092	bb = bb->next_bb;
1093	while (bb != afterbb)
1094	{
1095	struct omp_region *cur_region = NULL;
1096	profile_count cnt = profile_count::zero ();
1097	bool all = true;
1098
1099	int cur_omp_region_idx = 0;
1100	int mer = make_edges_bb (bb, pcur_region: &cur_region, pomp_index: &cur_omp_region_idx);
1101	gcc_assert (!mer && !cur_region);
1102	add_bb_to_loop (bb, afterbb->loop_father);
1103
1104	edge e;
1105	edge_iterator ei;
1106	FOR_EACH_EDGE (e, ei, bb->preds)
1107	{
1108	if (e->count ().initialized_p ())
1109	cnt += e->count ();
1110	else
1111	all = false;
1112	}
1113	tree_guess_outgoing_edge_probabilities (bb);
1114	if (all || profile_status_for_fn (cfun) == PROFILE_READ)
1115	bb->count = cnt;
1116
1117	bb = bb->next_bb;
1118	}
1119	return true;
1120	}
1121
1122	/* Find the next available discriminator value for LOCUS. The
1123	discriminator distinguishes among several basic blocks that
1124	share a common locus, allowing for more accurate sample-based
1125	profiling. */
1126
1127	static int
1128	next_discriminator_for_locus (int line)
1129	{
1130	struct locus_discrim_map item;
1131	struct locus_discrim_map **slot;
1132
1133	item.location_line = line;
1134	item.discriminator = 0;
1135	slot = discriminator_per_locus->find_slot_with_hash (comparable: &item, hash: line, insert: INSERT);
1136	gcc_assert (slot);
1137	if (*slot == HTAB_EMPTY_ENTRY)
1138	{
1139	*slot = XNEW (struct locus_discrim_map);
1140	gcc_assert (*slot);
1141	(*slot)->location_line = line;
1142	(*slot)->discriminator = 0;
1143	}
1144	(*slot)->discriminator++;
1145	return (*slot)->discriminator;
1146	}
1147
1148	/* Return TRUE if LOCUS1 and LOCUS2 refer to the same source line. */
1149
1150	static bool
1151	same_line_p (location_t locus1, expanded_location *from, location_t locus2)
1152	{
1153	expanded_location to;
1154
1155	if (locus1 == locus2)
1156	return true;
1157
1158	to = expand_location (locus2);
1159
1160	if (from->line != to.line)
1161	return false;
1162	if (from->file == to.file)
1163	return true;
1164	return (from->file != NULL
1165	&& to.file != NULL
1166	&& filename_cmp (s1: from->file, s2: to.file) == 0);
1167	}
1168
1169	/* Assign a unique discriminator value to all statements in block bb that
1170	have the same line number as locus. */
1171
1172	static void
1173	assign_discriminator (location_t locus, basic_block bb)
1174	{
1175	gimple_stmt_iterator gsi;
1176	int discriminator;
1177
1178	if (locus == UNKNOWN_LOCATION)
1179	return;
1180
1181	expanded_location locus_e = expand_location (locus);
1182
1183	discriminator = next_discriminator_for_locus (line: locus_e.line);
1184
1185	for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
1186	{
1187	gimple *stmt = gsi_stmt (i: gsi);
1188	location_t stmt_locus = gimple_location (g: stmt);
1189	if (same_line_p (locus1: locus, from: &locus_e, locus2: stmt_locus))
1190	gimple_set_location (g: stmt,
1191	location: location_with_discriminator (stmt_locus, discriminator));
1192	}
1193	}
1194
1195	/* Assign discriminators to statement locations. */
1196
1197	static void
1198	assign_discriminators (void)
1199	{
1200	basic_block bb;
1201
1202	FOR_EACH_BB_FN (bb, cfun)
1203	{
1204	edge e;
1205	edge_iterator ei;
1206	gimple_stmt_iterator gsi;
1207	location_t curr_locus = UNKNOWN_LOCATION;
1208	expanded_location curr_locus_e = {};
1209	int curr_discr = 0;
1210
1211	/* Traverse the basic block, if two function calls within a basic block
1212	are mapped to the same line, assign a new discriminator because a call
1213	stmt could be a split point of a basic block. */
1214	for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
1215	{
1216	gimple *stmt = gsi_stmt (i: gsi);
1217
1218	/* Don't allow debug stmts to affect discriminators, but
1219	allow them to take discriminators when they're on the
1220	same line as the preceding nondebug stmt. */
1221	if (is_gimple_debug (gs: stmt))
1222	{
1223	if (curr_locus != UNKNOWN_LOCATION
1224	&& same_line_p (locus1: curr_locus, from: &curr_locus_e,
1225	locus2: gimple_location (g: stmt)))
1226	{
1227	location_t loc = gimple_location (g: stmt);
1228	location_t dloc = location_with_discriminator (loc,
1229	curr_discr);
1230	gimple_set_location (g: stmt, location: dloc);
1231	}
1232	continue;
1233	}
1234	if (curr_locus == UNKNOWN_LOCATION)
1235	{
1236	curr_locus = gimple_location (g: stmt);
1237	curr_locus_e = expand_location (curr_locus);
1238	}
1239	else if (!same_line_p (locus1: curr_locus, from: &curr_locus_e, locus2: gimple_location (g: stmt)))
1240	{
1241	curr_locus = gimple_location (g: stmt);
1242	curr_locus_e = expand_location (curr_locus);
1243	curr_discr = 0;
1244	}
1245	else if (curr_discr != 0)
1246	{
1247	location_t loc = gimple_location (g: stmt);
1248	location_t dloc = location_with_discriminator (loc, curr_discr);
1249	gimple_set_location (g: stmt, location: dloc);
1250	}
1251	/* Allocate a new discriminator for CALL stmt. */
1252	if (gimple_code (g: stmt) == GIMPLE_CALL)
1253	curr_discr = next_discriminator_for_locus (line: curr_locus);
1254	}
1255
1256	gimple *last = last_nondebug_stmt (bb);
1257	location_t locus = last ? gimple_location (g: last) : UNKNOWN_LOCATION;
1258	if (locus == UNKNOWN_LOCATION)
1259	continue;
1260
1261	expanded_location locus_e = expand_location (locus);
1262
1263	FOR_EACH_EDGE (e, ei, bb->succs)
1264	{
1265	gimple *first = first_non_label_stmt (e->dest);
1266	gimple *last = last_nondebug_stmt (e->dest);
1267
1268	gimple *stmt_on_same_line = NULL;
1269	if (first && same_line_p (locus1: locus, from: &locus_e,
1270	locus2: gimple_location (g: first)))
1271	stmt_on_same_line = first;
1272	else if (last && same_line_p (locus1: locus, from: &locus_e,
1273	locus2: gimple_location (g: last)))
1274	stmt_on_same_line = last;
1275
1276	if (stmt_on_same_line)
1277	{
1278	if (has_discriminator (gimple_location (g: stmt_on_same_line))
1279	&& !has_discriminator (locus))
1280	assign_discriminator (locus, bb);
1281	else
1282	assign_discriminator (locus, bb: e->dest);
1283	}
1284	}
1285	}
1286	}
1287
1288	/* Create the edges for a GIMPLE_COND starting at block BB. */
1289
1290	static void
1291	make_cond_expr_edges (basic_block bb)
1292	{
1293	gcond *entry = as_a <gcond *> (p: *gsi_last_bb (bb));
1294	gimple *then_stmt, *else_stmt;
1295	basic_block then_bb, else_bb;
1296	tree then_label, else_label;
1297	edge e;
1298
1299	gcc_assert (entry);
1300
1301	/* Entry basic blocks for each component. */
1302	then_label = gimple_cond_true_label (gs: entry);
1303	else_label = gimple_cond_false_label (gs: entry);
1304	then_bb = label_to_block (cfun, then_label);
1305	else_bb = label_to_block (cfun, else_label);
1306	then_stmt = first_stmt (then_bb);
1307	else_stmt = first_stmt (else_bb);
1308
1309	e = make_edge (bb, then_bb, EDGE_TRUE_VALUE);
1310	e->goto_locus = gimple_location (g: then_stmt);
1311	e = make_edge (bb, else_bb, EDGE_FALSE_VALUE);
1312	if (e)
1313	e->goto_locus = gimple_location (g: else_stmt);
1314
1315	/* We do not need the labels anymore. */
1316	gimple_cond_set_true_label (gs: entry, NULL_TREE);
1317	gimple_cond_set_false_label (gs: entry, NULL_TREE);
1318	}
1319
1320
1321	/* Called for each element in the hash table (P) as we delete the
1322	edge to cases hash table.
1323
1324	Clear all the CASE_CHAINs to prevent problems with copying of
1325	SWITCH_EXPRs and structure sharing rules, then free the hash table
1326	element. */
1327
1328	bool
1329	edge_to_cases_cleanup (edge const &, tree const &value, void *)
1330	{
1331	tree t, next;
1332
1333	for (t = value; t; t = next)
1334	{
1335	next = CASE_CHAIN (t);
1336	CASE_CHAIN (t) = NULL;
1337	}
1338
1339	return true;
1340	}
1341
1342	/* Start recording information mapping edges to case labels. */
1343
1344	void
1345	start_recording_case_labels (void)
1346	{
1347	gcc_assert (edge_to_cases == NULL);
1348	edge_to_cases = new hash_map<edge, tree>;
1349	touched_switch_bbs = BITMAP_ALLOC (NULL);
1350	}
1351
1352	/* Return nonzero if we are recording information for case labels. */
1353
1354	static bool
1355	recording_case_labels_p (void)
1356	{
1357	return (edge_to_cases != NULL);
1358	}
1359
1360	/* Stop recording information mapping edges to case labels and
1361	remove any information we have recorded. */
1362	void
1363	end_recording_case_labels (void)
1364	{
1365	bitmap_iterator bi;
1366	unsigned i;
1367	edge_to_cases->traverse<void *, edge_to_cases_cleanup> (NULL);
1368	delete edge_to_cases;
1369	edge_to_cases = NULL;
1370	EXECUTE_IF_SET_IN_BITMAP (touched_switch_bbs, 0, i, bi)
1371	{
1372	basic_block bb = BASIC_BLOCK_FOR_FN (cfun, i);
1373	if (bb)
1374	{
1375	if (gswitch *stmt = safe_dyn_cast <gswitch *> (p: *gsi_last_bb (bb)))
1376	group_case_labels_stmt (stmt);
1377	}
1378	}
1379	BITMAP_FREE (touched_switch_bbs);
1380	}
1381
1382	/* If we are inside a {start,end}_recording_cases block, then return
1383	a chain of CASE_LABEL_EXPRs from T which reference E.
1384
1385	Otherwise return NULL. */
1386
1387	tree
1388	get_cases_for_edge (edge e, gswitch *t)
1389	{
1390	tree *slot;
1391	size_t i, n;
1392
1393	/* If we are not recording cases, then we do not have CASE_LABEL_EXPR
1394	chains available. Return NULL so the caller can detect this case. */
1395	if (!recording_case_labels_p ())
1396	return NULL;
1397
1398	slot = edge_to_cases->get (k: e);
1399	if (slot)
1400	return *slot;
1401
1402	/* If we did not find E in the hash table, then this must be the first
1403	time we have been queried for information about E & T. Add all the
1404	elements from T to the hash table then perform the query again. */
1405
1406	n = gimple_switch_num_labels (gs: t);
1407	for (i = 0; i < n; i++)
1408	{
1409	tree elt = gimple_switch_label (gs: t, index: i);
1410	tree lab = CASE_LABEL (elt);
1411	basic_block label_bb = label_to_block (cfun, lab);
1412	edge this_edge = find_edge (e->src, label_bb);
1413
1414	/* Add it to the chain of CASE_LABEL_EXPRs referencing E, or create
1415	a new chain. */
1416	tree &s = edge_to_cases->get_or_insert (k: this_edge);
1417	CASE_CHAIN (elt) = s;
1418	s = elt;
1419	}
1420
1421	return *edge_to_cases->get (k: e);
1422	}
1423
1424	/* Create the edges for a GIMPLE_SWITCH starting at block BB. */
1425
1426	static void
1427	make_gimple_switch_edges (gswitch *entry, basic_block bb)
1428	{
1429	size_t i, n;
1430
1431	n = gimple_switch_num_labels (gs: entry);
1432
1433	for (i = 0; i < n; ++i)
1434	{
1435	basic_block label_bb = gimple_switch_label_bb (cfun, entry, i);
1436	make_edge (bb, label_bb, 0);
1437	}
1438	}
1439
1440
1441	/* Return the basic block holding label DEST. */
1442
1443	basic_block
1444	label_to_block (struct function *ifun, tree dest)
1445	{
1446	int uid = LABEL_DECL_UID (dest);
1447
1448	/* We would die hard when faced by an undefined label. Emit a label to
1449	the very first basic block. This will hopefully make even the dataflow
1450	and undefined variable warnings quite right. */
1451	if (seen_error () && uid < 0)
1452	{
1453	gimple_stmt_iterator gsi =
1454	gsi_start_bb (BASIC_BLOCK_FOR_FN (cfun, NUM_FIXED_BLOCKS));
1455	gimple *stmt;
1456
1457	stmt = gimple_build_label (label: dest);
1458	gsi_insert_before (&gsi, stmt, GSI_NEW_STMT);
1459	uid = LABEL_DECL_UID (dest);
1460	}
1461	if (vec_safe_length (v: ifun->cfg->x_label_to_block_map) <= (unsigned int) uid)
1462	return NULL;
1463	return (*ifun->cfg->x_label_to_block_map)[uid];
1464	}
1465
1466	/* Create edges for a goto statement at block BB. Returns true
1467	if abnormal edges should be created. */
1468
1469	static bool
1470	make_goto_expr_edges (basic_block bb)
1471	{
1472	gimple_stmt_iterator last = gsi_last_bb (bb);
1473	gimple *goto_t = gsi_stmt (i: last);
1474
1475	/* A simple GOTO creates normal edges. */
1476	if (simple_goto_p (goto_t))
1477	{
1478	tree dest = gimple_goto_dest (gs: goto_t);
1479	basic_block label_bb = label_to_block (cfun, dest);
1480	edge e = make_edge (bb, label_bb, EDGE_FALLTHRU);
1481	e->goto_locus = gimple_location (g: goto_t);
1482	gsi_remove (&last, true);
1483	return false;
1484	}
1485
1486	/* A computed GOTO creates abnormal edges. */
1487	return true;
1488	}
1489
1490	/* Create edges for an asm statement with labels at block BB. */
1491
1492	static void
1493	make_gimple_asm_edges (basic_block bb)
1494	{
1495	gasm *stmt = as_a <gasm *> (p: *gsi_last_bb (bb));
1496	int i, n = gimple_asm_nlabels (asm_stmt: stmt);
1497
1498	for (i = 0; i < n; ++i)
1499	{
1500	tree label = TREE_VALUE (gimple_asm_label_op (stmt, i));
1501	basic_block label_bb = label_to_block (cfun, dest: label);
1502	make_edge (bb, label_bb, 0);
1503	}
1504	}
1505
1506	/*---------------------------------------------------------------------------
1507	Flowgraph analysis
1508	---------------------------------------------------------------------------*/
1509
1510	/* Cleanup useless labels in basic blocks. This is something we wish
1511	to do early because it allows us to group case labels before creating
1512	the edges for the CFG, and it speeds up block statement iterators in
1513	all passes later on.
1514	We rerun this pass after CFG is created, to get rid of the labels that
1515	are no longer referenced. After then we do not run it any more, since
1516	(almost) no new labels should be created. */
1517
1518	/* A map from basic block index to the leading label of that block. */
1519	struct label_record
1520	{
1521	/* The label. */
1522	tree label;
1523
1524	/* True if the label is referenced from somewhere. */
1525	bool used;
1526	};
1527
1528	/* Given LABEL return the first label in the same basic block. */
1529
1530	static tree
1531	main_block_label (tree label, label_record *label_for_bb)
1532	{
1533	basic_block bb = label_to_block (cfun, dest: label);
1534	tree main_label = label_for_bb[bb->index].label;
1535
1536	/* label_to_block possibly inserted undefined label into the chain. */
1537	if (!main_label)
1538	{
1539	label_for_bb[bb->index].label = label;
1540	main_label = label;
1541	}
1542
1543	label_for_bb[bb->index].used = true;
1544	return main_label;
1545	}
1546
1547	/* Clean up redundant labels within the exception tree. */
1548
1549	static void
1550	cleanup_dead_labels_eh (label_record *label_for_bb)
1551	{
1552	eh_landing_pad lp;
1553	eh_region r;
1554	tree lab;
1555	int i;
1556
1557	if (cfun->eh == NULL)
1558	return;
1559
1560	for (i = 1; vec_safe_iterate (cfun->eh->lp_array, ix: i, ptr: &lp); ++i)
1561	if (lp && lp->post_landing_pad)
1562	{
1563	lab = main_block_label (label: lp->post_landing_pad, label_for_bb);
1564	if (lab != lp->post_landing_pad)
1565	{
1566	EH_LANDING_PAD_NR (lp->post_landing_pad) = 0;
1567	lp->post_landing_pad = lab;
1568	EH_LANDING_PAD_NR (lab) = lp->index;
1569	}
1570	}
1571
1572	FOR_ALL_EH_REGION (r)
1573	switch (r->type)
1574	{
1575	case ERT_CLEANUP:
1576	case ERT_MUST_NOT_THROW:
1577	break;
1578
1579	case ERT_TRY:
1580	{
1581	eh_catch c;
1582	for (c = r->u.eh_try.first_catch; c ; c = c->next_catch)
1583	{
1584	lab = c->label;
1585	if (lab)
1586	c->label = main_block_label (label: lab, label_for_bb);
1587	}
1588	}
1589	break;
1590
1591	case ERT_ALLOWED_EXCEPTIONS:
1592	lab = r->u.allowed.label;
1593	if (lab)
1594	r->u.allowed.label = main_block_label (label: lab, label_for_bb);
1595	break;
1596	}
1597	}
1598
1599
1600	/* Cleanup redundant labels. This is a three-step process:
1601	1) Find the leading label for each block.
1602	2) Redirect all references to labels to the leading labels.
1603	3) Cleanup all useless labels. */
1604
1605	void
1606	cleanup_dead_labels (void)
1607	{
1608	basic_block bb;
1609	label_record *label_for_bb = XCNEWVEC (struct label_record,
1610	last_basic_block_for_fn (cfun));
1611
1612	/* Find a suitable label for each block. We use the first user-defined
1613	label if there is one, or otherwise just the first label we see. */
1614	FOR_EACH_BB_FN (bb, cfun)
1615	{
1616	gimple_stmt_iterator i;
1617
1618	for (i = gsi_start_bb (bb); !gsi_end_p (i); gsi_next (i: &i))
1619	{
1620	tree label;
1621	glabel *label_stmt = dyn_cast <glabel *> (p: gsi_stmt (i));
1622
1623	if (!label_stmt)
1624	break;
1625
1626	label = gimple_label_label (gs: label_stmt);
1627
1628	/* If we have not yet seen a label for the current block,
1629	remember this one and see if there are more labels. */
1630	if (!label_for_bb[bb->index].label)
1631	{
1632	label_for_bb[bb->index].label = label;
1633	continue;
1634	}
1635
1636	/* If we did see a label for the current block already, but it
1637	is an artificially created label, replace it if the current
1638	label is a user defined label. */
1639	if (!DECL_ARTIFICIAL (label)
1640	&& DECL_ARTIFICIAL (label_for_bb[bb->index].label))
1641	{
1642	label_for_bb[bb->index].label = label;
1643	break;
1644	}
1645	}
1646	}
1647
1648	/* Now redirect all jumps/branches to the selected label.
1649	First do so for each block ending in a control statement. */
1650	FOR_EACH_BB_FN (bb, cfun)
1651	{
1652	gimple *stmt = *gsi_last_bb (bb);
1653	tree label, new_label;
1654
1655	if (!stmt)
1656	continue;
1657
1658	switch (gimple_code (g: stmt))
1659	{
1660	case GIMPLE_COND:
1661	{
1662	gcond *cond_stmt = as_a <gcond *> (p: stmt);
1663	label = gimple_cond_true_label (gs: cond_stmt);
1664	if (label)
1665	{
1666	new_label = main_block_label (label, label_for_bb);
1667	if (new_label != label)
1668	gimple_cond_set_true_label (gs: cond_stmt, label: new_label);
1669	}
1670
1671	label = gimple_cond_false_label (gs: cond_stmt);
1672	if (label)
1673	{
1674	new_label = main_block_label (label, label_for_bb);
1675	if (new_label != label)
1676	gimple_cond_set_false_label (gs: cond_stmt, label: new_label);
1677	}
1678	}
1679	break;
1680
1681	case GIMPLE_SWITCH:
1682	{
1683	gswitch *switch_stmt = as_a <gswitch *> (p: stmt);
1684	size_t i, n = gimple_switch_num_labels (gs: switch_stmt);
1685
1686	/* Replace all destination labels. */
1687	for (i = 0; i < n; ++i)
1688	{
1689	tree case_label = gimple_switch_label (gs: switch_stmt, index: i);
1690	label = CASE_LABEL (case_label);
1691	new_label = main_block_label (label, label_for_bb);
1692	if (new_label != label)
1693	CASE_LABEL (case_label) = new_label;
1694	}
1695	break;
1696	}
1697
1698	case GIMPLE_ASM:
1699	{
1700	gasm *asm_stmt = as_a <gasm *> (p: stmt);
1701	int i, n = gimple_asm_nlabels (asm_stmt);
1702
1703	for (i = 0; i < n; ++i)
1704	{
1705	tree cons = gimple_asm_label_op (asm_stmt, index: i);
1706	tree label = main_block_label (TREE_VALUE (cons), label_for_bb);
1707	TREE_VALUE (cons) = label;
1708	}
1709	break;
1710	}
1711
1712	/* We have to handle gotos until they're removed, and we don't
1713	remove them until after we've created the CFG edges. */
1714	case GIMPLE_GOTO:
1715	if (!computed_goto_p (t: stmt))
1716	{
1717	ggoto *goto_stmt = as_a <ggoto *> (p: stmt);
1718	label = gimple_goto_dest (gs: goto_stmt);
1719	new_label = main_block_label (label, label_for_bb);
1720	if (new_label != label)
1721	gimple_goto_set_dest (gs: goto_stmt, dest: new_label);
1722	}
1723	break;
1724
1725	case GIMPLE_TRANSACTION:
1726	{
1727	gtransaction *txn = as_a <gtransaction *> (p: stmt);
1728
1729	label = gimple_transaction_label_norm (transaction_stmt: txn);
1730	if (label)
1731	{
1732	new_label = main_block_label (label, label_for_bb);
1733	if (new_label != label)
1734	gimple_transaction_set_label_norm (transaction_stmt: txn, label: new_label);
1735	}
1736
1737	label = gimple_transaction_label_uninst (transaction_stmt: txn);
1738	if (label)
1739	{
1740	new_label = main_block_label (label, label_for_bb);
1741	if (new_label != label)
1742	gimple_transaction_set_label_uninst (transaction_stmt: txn, label: new_label);
1743	}
1744
1745	label = gimple_transaction_label_over (transaction_stmt: txn);
1746	if (label)
1747	{
1748	new_label = main_block_label (label, label_for_bb);
1749	if (new_label != label)
1750	gimple_transaction_set_label_over (transaction_stmt: txn, label: new_label);
1751	}
1752	}
1753	break;
1754
1755	default:
1756	break;
1757	}
1758	}
1759
1760	/* Do the same for the exception region tree labels. */
1761	cleanup_dead_labels_eh (label_for_bb);
1762
1763	/* Finally, purge dead labels. All user-defined labels and labels that
1764	can be the target of non-local gotos and labels which have their
1765	address taken are preserved. */
1766	FOR_EACH_BB_FN (bb, cfun)
1767	{
1768	gimple_stmt_iterator i;
1769	tree label_for_this_bb = label_for_bb[bb->index].label;
1770
1771	if (!label_for_this_bb)
1772	continue;
1773
1774	/* If the main label of the block is unused, we may still remove it. */
1775	if (!label_for_bb[bb->index].used)
1776	label_for_this_bb = NULL;
1777
1778	for (i = gsi_start_bb (bb); !gsi_end_p (i); )
1779	{
1780	tree label;
1781	glabel *label_stmt = dyn_cast <glabel *> (p: gsi_stmt (i));
1782
1783	if (!label_stmt)
1784	break;
1785
1786	label = gimple_label_label (gs: label_stmt);
1787
1788	if (label == label_for_this_bb
1789	|| !DECL_ARTIFICIAL (label)
1790	|| DECL_NONLOCAL (label)
1791	|| FORCED_LABEL (label))
1792	gsi_next (i: &i);
1793	else
1794	{
1795	gcc_checking_assert (EH_LANDING_PAD_NR (label) == 0);
1796	gsi_remove (&i, true);
1797	}
1798	}
1799	}
1800
1801	free (ptr: label_for_bb);
1802	}
1803
1804	/* Scan the sorted vector of cases in STMT (a GIMPLE_SWITCH) and combine
1805	the ones jumping to the same label.
1806	Eg. three separate entries 1: 2: 3: become one entry 1..3: */
1807
1808	bool
1809	group_case_labels_stmt (gswitch *stmt)
1810	{
1811	int old_size = gimple_switch_num_labels (gs: stmt);
1812	int i, next_index, new_size;
1813	basic_block default_bb = NULL;
1814	hash_set<tree> *removed_labels = NULL;
1815
1816	default_bb = gimple_switch_default_bb (cfun, stmt);
1817
1818	/* Look for possible opportunities to merge cases. */
1819	new_size = i = 1;
1820	while (i < old_size)
1821	{
1822	tree base_case, base_high;
1823	basic_block base_bb;
1824
1825	base_case = gimple_switch_label (gs: stmt, index: i);
1826
1827	gcc_assert (base_case);
1828	base_bb = label_to_block (cfun, CASE_LABEL (base_case));
1829
1830	/* Discard cases that have the same destination as the default case or
1831	whose destination blocks have already been removed as unreachable. */
1832	if (base_bb == NULL
1833	|| base_bb == default_bb
1834	|| (removed_labels
1835	&& removed_labels->contains (CASE_LABEL (base_case))))
1836	{
1837	i++;
1838	continue;
1839	}
1840
1841	base_high = CASE_HIGH (base_case)
1842	? CASE_HIGH (base_case)
1843	: CASE_LOW (base_case);
1844	next_index = i + 1;
1845
1846	/* Try to merge case labels. Break out when we reach the end
1847	of the label vector or when we cannot merge the next case
1848	label with the current one. */
1849	while (next_index < old_size)
1850	{
1851	tree merge_case = gimple_switch_label (gs: stmt, index: next_index);
1852	basic_block merge_bb = label_to_block (cfun, CASE_LABEL (merge_case));
1853	wide_int bhp1 = wi::to_wide (t: base_high) + 1;
1854
1855	/* Merge the cases if they jump to the same place,
1856	and their ranges are consecutive. */
1857	if (merge_bb == base_bb
1858	&& (removed_labels == NULL
1859	|| !removed_labels->contains (CASE_LABEL (merge_case)))
1860	&& wi::to_wide (CASE_LOW (merge_case)) == bhp1)
1861	{
1862	base_high
1863	= (CASE_HIGH (merge_case)
1864	? CASE_HIGH (merge_case) : CASE_LOW (merge_case));
1865	CASE_HIGH (base_case) = base_high;
1866	next_index++;
1867	}
1868	else
1869	break;
1870	}
1871
1872	/* Discard cases that have an unreachable destination block. */
1873	if (EDGE_COUNT (base_bb->succs) == 0
1874	&& gimple_seq_unreachable_p (stmts: bb_seq (bb: base_bb))
1875	/* Don't optimize this if __builtin_unreachable () is the
1876	implicitly added one by the C++ FE too early, before
1877	-Wreturn-type can be diagnosed. We'll optimize it later
1878	during switchconv pass or any other cfg cleanup. */
1879	&& (gimple_in_ssa_p (cfun)
1880	|| (LOCATION_LOCUS (gimple_location (last_nondebug_stmt (base_bb)))
1881	!= BUILTINS_LOCATION)))
1882	{
1883	edge base_edge = find_edge (gimple_bb (g: stmt), base_bb);
1884	if (base_edge != NULL)
1885	{
1886	for (gimple_stmt_iterator gsi = gsi_start_bb (bb: base_bb);
1887	!gsi_end_p (i: gsi); gsi_next (i: &gsi))
1888	if (glabel *stmt = dyn_cast <glabel *> (p: gsi_stmt (i: gsi)))
1889	{
1890	if (FORCED_LABEL (gimple_label_label (stmt))
1891	|| DECL_NONLOCAL (gimple_label_label (stmt)))
1892	{
1893	/* Forced/non-local labels aren't going to be removed,
1894	but they will be moved to some neighbouring basic
1895	block. If some later case label refers to one of
1896	those labels, we should throw that case away rather
1897	than keeping it around and refering to some random
1898	other basic block without an edge to it. */
1899	if (removed_labels == NULL)
1900	removed_labels = new hash_set<tree>;
1901	removed_labels->add (k: gimple_label_label (gs: stmt));
1902	}
1903	}
1904	else
1905	break;
1906	remove_edge_and_dominated_blocks (base_edge);
1907	}
1908	i = next_index;
1909	continue;
1910	}
1911
1912	if (new_size < i)
1913	gimple_switch_set_label (gs: stmt, index: new_size,
1914	label: gimple_switch_label (gs: stmt, index: i));
1915	i = next_index;
1916	new_size++;
1917	}
1918
1919	gcc_assert (new_size <= old_size);
1920
1921	if (new_size < old_size)
1922	gimple_switch_set_num_labels (g: stmt, nlabels: new_size);
1923
1924	delete removed_labels;
1925	return new_size < old_size;
1926	}
1927
1928	/* Look for blocks ending in a multiway branch (a GIMPLE_SWITCH),
1929	and scan the sorted vector of cases. Combine the ones jumping to the
1930	same label. */
1931
1932	bool
1933	group_case_labels (void)
1934	{
1935	basic_block bb;
1936	bool changed = false;
1937
1938	FOR_EACH_BB_FN (bb, cfun)
1939	{
1940	if (gswitch *stmt = safe_dyn_cast <gswitch *> (p: *gsi_last_bb (bb)))
1941	changed |= group_case_labels_stmt (stmt);
1942	}
1943
1944	return changed;
1945	}
1946
1947	/* Checks whether we can merge block B into block A. */
1948
1949	static bool
1950	gimple_can_merge_blocks_p (basic_block a, basic_block b)
1951	{
1952	gimple *stmt;
1953
1954	if (!single_succ_p (bb: a))
1955	return false;
1956
1957	if (single_succ_edge (bb: a)->flags & EDGE_COMPLEX)
1958	return false;
1959
1960	if (single_succ (bb: a) != b)
1961	return false;
1962
1963	if (!single_pred_p (bb: b))
1964	return false;
1965
1966	if (a == ENTRY_BLOCK_PTR_FOR_FN (cfun)
1967	|| b == EXIT_BLOCK_PTR_FOR_FN (cfun))
1968	return false;
1969
1970	/* If A ends by a statement causing exceptions or something similar, we
1971	cannot merge the blocks. */
1972	stmt = *gsi_last_bb (bb: a);
1973	if (stmt && stmt_ends_bb_p (stmt))
1974	return false;
1975
1976	/* Examine the labels at the beginning of B. */
1977	for (gimple_stmt_iterator gsi = gsi_start_bb (bb: b); !gsi_end_p (i: gsi);
1978	gsi_next (i: &gsi))
1979	{
1980	tree lab;
1981	glabel *label_stmt = dyn_cast <glabel *> (p: gsi_stmt (i: gsi));
1982	if (!label_stmt)
1983	break;
1984	lab = gimple_label_label (gs: label_stmt);
1985
1986	/* Do not remove user forced labels or for -O0 any user labels. */
1987	if (!DECL_ARTIFICIAL (lab) && (!optimize || FORCED_LABEL (lab)))
1988	return false;
1989	}
1990
1991	/* Protect simple loop latches. We only want to avoid merging
1992	the latch with the loop header or with a block in another
1993	loop in this case. */
1994	if (current_loops
1995	&& b->loop_father->latch == b
1996	&& loops_state_satisfies_p (flags: LOOPS_HAVE_SIMPLE_LATCHES)
1997	&& (b->loop_father->header == a
1998	|| b->loop_father != a->loop_father))
1999	return false;
2000
2001	/* It must be possible to eliminate all phi nodes in B. If ssa form
2002	is not up-to-date and a name-mapping is registered, we cannot eliminate
2003	any phis. Symbols marked for renaming are never a problem though. */
2004	for (gphi_iterator gsi = gsi_start_phis (b); !gsi_end_p (i: gsi);
2005	gsi_next (i: &gsi))
2006	{
2007	gphi *phi = gsi.phi ();
2008	/* Technically only new names matter. */
2009	if (name_registered_for_update_p (PHI_RESULT (phi)))
2010	return false;
2011	}
2012
2013	/* When not optimizing, don't merge if we'd lose goto_locus. */
2014	if (!optimize
2015	&& single_succ_edge (bb: a)->goto_locus != UNKNOWN_LOCATION)
2016	{
2017	location_t goto_locus = single_succ_edge (bb: a)->goto_locus;
2018	gimple_stmt_iterator prev, next;
2019	prev = gsi_last_nondebug_bb (bb: a);
2020	next = gsi_after_labels (bb: b);
2021	if (!gsi_end_p (i: next) && is_gimple_debug (gs: gsi_stmt (i: next)))
2022	gsi_next_nondebug (i: &next);
2023	if ((gsi_end_p (i: prev)
2024	|| gimple_location (g: gsi_stmt (i: prev)) != goto_locus)
2025	&& (gsi_end_p (i: next)
2026	|| gimple_location (g: gsi_stmt (i: next)) != goto_locus))
2027	return false;
2028	}
2029
2030	return true;
2031	}
2032
2033	/* Replaces all uses of NAME by VAL. */
2034
2035	void
2036	replace_uses_by (tree name, tree val)
2037	{
2038	imm_use_iterator imm_iter;
2039	use_operand_p use;
2040	gimple *stmt;
2041	edge e;
2042
2043	FOR_EACH_IMM_USE_STMT (stmt, imm_iter, name)
2044	{
2045	/* Mark the block if we change the last stmt in it. */
2046	if (cfgcleanup_altered_bbs
2047	&& stmt_ends_bb_p (stmt))
2048	bitmap_set_bit (cfgcleanup_altered_bbs, gimple_bb (g: stmt)->index);
2049
2050	FOR_EACH_IMM_USE_ON_STMT (use, imm_iter)
2051	{
2052	replace_exp (use, val);
2053
2054	if (gimple_code (g: stmt) == GIMPLE_PHI)
2055	{
2056	e = gimple_phi_arg_edge (phi: as_a <gphi *> (p: stmt),
2057	PHI_ARG_INDEX_FROM_USE (use));
2058	if (e->flags & EDGE_ABNORMAL
2059	&& !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (val))
2060	{
2061	/* This can only occur for virtual operands, since
2062	for the real ones SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name))
2063	would prevent replacement. */
2064	gcc_checking_assert (virtual_operand_p (name));
2065	SSA_NAME_OCCURS_IN_ABNORMAL_PHI (val) = 1;
2066	}
2067	}
2068	}
2069
2070	if (gimple_code (g: stmt) != GIMPLE_PHI)
2071	{
2072	gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
2073	gimple *orig_stmt = stmt;
2074	size_t i;
2075
2076	/* FIXME. It shouldn't be required to keep TREE_CONSTANT
2077	on ADDR_EXPRs up-to-date on GIMPLE. Propagation will
2078	only change sth from non-invariant to invariant, and only
2079	when propagating constants. */
2080	if (is_gimple_min_invariant (val))
2081	for (i = 0; i < gimple_num_ops (gs: stmt); i++)
2082	{
2083	tree op = gimple_op (gs: stmt, i);
2084	/* Operands may be empty here. For example, the labels
2085	of a GIMPLE_COND are nulled out following the creation
2086	of the corresponding CFG edges. */
2087	if (op && TREE_CODE (op) == ADDR_EXPR)
2088	recompute_tree_invariant_for_addr_expr (op);
2089	}
2090
2091	if (fold_stmt (&gsi))
2092	stmt = gsi_stmt (i: gsi);
2093
2094	if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
2095	gimple_purge_dead_eh_edges (gimple_bb (g: stmt));
2096
2097	update_stmt (s: stmt);
2098	}
2099	}
2100
2101	gcc_checking_assert (has_zero_uses (name));
2102
2103	/* Also update the trees stored in loop structures. */
2104	if (current_loops)
2105	{
2106	for (auto loop : loops_list (cfun, 0))
2107	substitute_in_loop_info (loop, name, val);
2108	}
2109	}
2110
2111	/* Merge block B into block A. */
2112
2113	static void
2114	gimple_merge_blocks (basic_block a, basic_block b)
2115	{
2116	gimple_stmt_iterator last, gsi;
2117	gphi_iterator psi;
2118
2119	if (dump_file)
2120	fprintf (stream: dump_file, format: "Merging blocks %d and %d\n", a->index, b->index);
2121
2122	/* Remove all single-valued PHI nodes from block B of the form
2123	V_i = PHI <V_j> by propagating V_j to all the uses of V_i. */
2124	gsi = gsi_last_bb (bb: a);
2125	for (psi = gsi_start_phis (b); !gsi_end_p (i: psi); )
2126	{
2127	gimple *phi = gsi_stmt (i: psi);
2128	tree def = gimple_phi_result (gs: phi), use = gimple_phi_arg_def (gs: phi, index: 0);
2129	gimple *copy;
2130	bool may_replace_uses = (virtual_operand_p (op: def)
2131	|| may_propagate_copy (def, use));
2132
2133	/* In case we maintain loop closed ssa form, do not propagate arguments
2134	of loop exit phi nodes. */
2135	if (current_loops
2136	&& loops_state_satisfies_p (flags: LOOP_CLOSED_SSA)
2137	&& !virtual_operand_p (op: def)
2138	&& TREE_CODE (use) == SSA_NAME
2139	&& a->loop_father != b->loop_father)
2140	may_replace_uses = false;
2141
2142	if (!may_replace_uses)
2143	{
2144	gcc_assert (!virtual_operand_p (def));
2145
2146	/* Note that just emitting the copies is fine -- there is no problem
2147	with ordering of phi nodes. This is because A is the single
2148	predecessor of B, therefore results of the phi nodes cannot
2149	appear as arguments of the phi nodes. */
2150	copy = gimple_build_assign (def, use);
2151	gsi_insert_after (&gsi, copy, GSI_NEW_STMT);
2152	remove_phi_node (&psi, false);
2153	}
2154	else
2155	{
2156	/* If we deal with a PHI for virtual operands, we can simply
2157	propagate these without fussing with folding or updating
2158	the stmt. */
2159	if (virtual_operand_p (op: def))
2160	{
2161	imm_use_iterator iter;
2162	use_operand_p use_p;
2163	gimple *stmt;
2164
2165	FOR_EACH_IMM_USE_STMT (stmt, iter, def)
2166	FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
2167	SET_USE (use_p, use);
2168
2169	if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (def))
2170	SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use) = 1;
2171	}
2172	else
2173	replace_uses_by (name: def, val: use);
2174
2175	remove_phi_node (&psi, true);
2176	}
2177	}
2178
2179	/* Ensure that B follows A. */
2180	move_block_after (b, a);
2181
2182	gcc_assert (single_succ_edge (a)->flags & EDGE_FALLTHRU);
2183	gcc_assert (!*gsi_last_bb (a)
2184	|| !stmt_ends_bb_p (*gsi_last_bb (a)));
2185
2186	/* Remove labels from B and set gimple_bb to A for other statements. */
2187	for (gsi = gsi_start_bb (bb: b); !gsi_end_p (i: gsi);)
2188	{
2189	gimple *stmt = gsi_stmt (i: gsi);
2190	if (glabel *label_stmt = dyn_cast <glabel *> (p: stmt))
2191	{
2192	tree label = gimple_label_label (gs: label_stmt);
2193	int lp_nr;
2194
2195	gsi_remove (&gsi, false);
2196
2197	/* Now that we can thread computed gotos, we might have
2198	a situation where we have a forced label in block B
2199	However, the label at the start of block B might still be
2200	used in other ways (think about the runtime checking for
2201	Fortran assigned gotos). So we cannot just delete the
2202	label. Instead we move the label to the start of block A. */
2203	if (FORCED_LABEL (label))
2204	{
2205	gimple_stmt_iterator dest_gsi = gsi_start_bb (bb: a);
2206	tree first_label = NULL_TREE;
2207	if (!gsi_end_p (i: dest_gsi))
2208	if (glabel *first_label_stmt
2209	= dyn_cast <glabel *> (p: gsi_stmt (i: dest_gsi)))
2210	first_label = gimple_label_label (gs: first_label_stmt);
2211	if (first_label
2212	&& (DECL_NONLOCAL (first_label)
2213	|| EH_LANDING_PAD_NR (first_label) != 0))
2214	gsi_insert_after (&dest_gsi, stmt, GSI_NEW_STMT);
2215	else
2216	gsi_insert_before (&dest_gsi, stmt, GSI_NEW_STMT);
2217	}
2218	/* Other user labels keep around in a form of a debug stmt. */
2219	else if (!DECL_ARTIFICIAL (label) && MAY_HAVE_DEBUG_BIND_STMTS)
2220	{
2221	gimple *dbg = gimple_build_debug_bind (label,
2222	integer_zero_node,
2223	stmt);
2224	gimple_debug_bind_reset_value (dbg);
2225	gsi_insert_before (&gsi, dbg, GSI_SAME_STMT);
2226	}
2227
2228	lp_nr = EH_LANDING_PAD_NR (label);
2229	if (lp_nr)
2230	{
2231	eh_landing_pad lp = get_eh_landing_pad_from_number (lp_nr);
2232	lp->post_landing_pad = NULL;
2233	}
2234	}
2235	else
2236	{
2237	gimple_set_bb (stmt, a);
2238	gsi_next (i: &gsi);
2239	}
2240	}
2241
2242	/* When merging two BBs, if their counts are different, the larger count
2243	is selected as the new bb count. This is to handle inconsistent
2244	profiles. */
2245	if (a->loop_father == b->loop_father)
2246	{
2247	a->count = a->count.merge (other: b->count);
2248	}
2249
2250	/* Merge the sequences. */
2251	last = gsi_last_bb (bb: a);
2252	gsi_insert_seq_after (&last, bb_seq (bb: b), GSI_NEW_STMT);
2253	set_bb_seq (bb: b, NULL);
2254
2255	if (cfgcleanup_altered_bbs)
2256	bitmap_set_bit (cfgcleanup_altered_bbs, a->index);
2257	}
2258
2259
2260	/* Return the one of two successors of BB that is not reachable by a
2261	complex edge, if there is one. Else, return BB. We use
2262	this in optimizations that use post-dominators for their heuristics,
2263	to catch the cases in C++ where function calls are involved. */
2264
2265	basic_block
2266	single_noncomplex_succ (basic_block bb)
2267	{
2268	edge e0, e1;
2269	if (EDGE_COUNT (bb->succs) != 2)
2270	return bb;
2271
2272	e0 = EDGE_SUCC (bb, 0);
2273	e1 = EDGE_SUCC (bb, 1);
2274	if (e0->flags & EDGE_COMPLEX)
2275	return e1->dest;
2276	if (e1->flags & EDGE_COMPLEX)
2277	return e0->dest;
2278
2279	return bb;
2280	}
2281
2282	/* T is CALL_EXPR. Set current_function_calls_* flags. */
2283
2284	void
2285	notice_special_calls (gcall *call)
2286	{
2287	int flags = gimple_call_flags (call);
2288
2289	if (flags & ECF_MAY_BE_ALLOCA)
2290	cfun->calls_alloca = true;
2291	if (flags & ECF_RETURNS_TWICE)
2292	cfun->calls_setjmp = true;
2293	}
2294
2295
2296	/* Clear flags set by notice_special_calls. Used by dead code removal
2297	to update the flags. */
2298
2299	void
2300	clear_special_calls (void)
2301	{
2302	cfun->calls_alloca = false;
2303	cfun->calls_setjmp = false;
2304	}
2305
2306	/* Remove PHI nodes associated with basic block BB and all edges out of BB. */
2307
2308	static void
2309	remove_phi_nodes_and_edges_for_unreachable_block (basic_block bb)
2310	{
2311	/* Since this block is no longer reachable, we can just delete all
2312	of its PHI nodes. */
2313	remove_phi_nodes (bb);
2314
2315	/* Remove edges to BB's successors. */
2316	while (EDGE_COUNT (bb->succs) > 0)
2317	remove_edge (EDGE_SUCC (bb, 0));
2318	}
2319
2320
2321	/* Remove statements of basic block BB. */
2322
2323	static void
2324	remove_bb (basic_block bb)
2325	{
2326	gimple_stmt_iterator i;
2327
2328	if (dump_file)
2329	{
2330	fprintf (stream: dump_file, format: "Removing basic block %d\n", bb->index);
2331	if (dump_flags & TDF_DETAILS)
2332	{
2333	dump_bb (dump_file, bb, 0, TDF_BLOCKS);
2334	fprintf (stream: dump_file, format: "\n");
2335	}
2336	}
2337
2338	if (current_loops)
2339	{
2340	class loop *loop = bb->loop_father;
2341
2342	/* If a loop gets removed, clean up the information associated
2343	with it. */
2344	if (loop->latch == bb
2345	|| loop->header == bb)
2346	free_numbers_of_iterations_estimates (loop);
2347	}
2348
2349	/* Remove all the instructions in the block. */
2350	if (bb_seq (bb) != NULL)
2351	{
2352	/* Walk backwards so as to get a chance to substitute all
2353	released DEFs into debug stmts. See
2354	eliminate_unnecessary_stmts() in tree-ssa-dce.cc for more
2355	details. */
2356	for (i = gsi_last_bb (bb); !gsi_end_p (i);)
2357	{
2358	gimple *stmt = gsi_stmt (i);
2359	glabel *label_stmt = dyn_cast <glabel *> (p: stmt);
2360	if (label_stmt
2361	&& (FORCED_LABEL (gimple_label_label (label_stmt))
2362	|| DECL_NONLOCAL (gimple_label_label (label_stmt))))
2363	{
2364	basic_block new_bb;
2365	gimple_stmt_iterator new_gsi;
2366
2367	/* A non-reachable non-local label may still be referenced.
2368	But it no longer needs to carry the extra semantics of
2369	non-locality. */
2370	if (DECL_NONLOCAL (gimple_label_label (label_stmt)))
2371	{
2372	DECL_NONLOCAL (gimple_label_label (label_stmt)) = 0;
2373	FORCED_LABEL (gimple_label_label (label_stmt)) = 1;
2374	}
2375
2376	new_bb = bb->prev_bb;
2377	/* Don't move any labels into ENTRY block. */
2378	if (new_bb == ENTRY_BLOCK_PTR_FOR_FN (cfun))
2379	{
2380	new_bb = single_succ (bb: new_bb);
2381	gcc_assert (new_bb != bb);
2382	}
2383	if ((unsigned) bb->index < bb_to_omp_idx.length ()
2384	&& ((unsigned) new_bb->index >= bb_to_omp_idx.length ()
2385	|| (bb_to_omp_idx[bb->index]
2386	!= bb_to_omp_idx[new_bb->index])))
2387	{
2388	/* During cfg pass make sure to put orphaned labels
2389	into the right OMP region. */
2390	unsigned int i;
2391	int idx;
2392	new_bb = NULL;
2393	FOR_EACH_VEC_ELT (bb_to_omp_idx, i, idx)
2394	if (i >= NUM_FIXED_BLOCKS
2395	&& idx == bb_to_omp_idx[bb->index]
2396	&& i != (unsigned) bb->index)
2397	{
2398	new_bb = BASIC_BLOCK_FOR_FN (cfun, i);
2399	break;
2400	}
2401	if (new_bb == NULL)
2402	{
2403	new_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
2404	gcc_assert (new_bb != bb);
2405	}
2406	}
2407	new_gsi = gsi_after_labels (bb: new_bb);
2408	gsi_remove (&i, false);
2409	gsi_insert_before (&new_gsi, stmt, GSI_NEW_STMT);
2410	}
2411	else
2412	{
2413	/* Release SSA definitions. */
2414	release_defs (stmt);
2415	gsi_remove (&i, true);
2416	}
2417
2418	if (gsi_end_p (i))
2419	i = gsi_last_bb (bb);
2420	else
2421	gsi_prev (i: &i);
2422	}
2423	}
2424
2425	if ((unsigned) bb->index < bb_to_omp_idx.length ())
2426	bb_to_omp_idx[bb->index] = -1;
2427	remove_phi_nodes_and_edges_for_unreachable_block (bb);
2428	bb->il.gimple.seq = NULL;
2429	bb->il.gimple.phi_nodes = NULL;
2430	}
2431
2432
2433	/* Given a basic block BB and a value VAL for use in the final statement
2434	of the block (if a GIMPLE_COND, GIMPLE_SWITCH, or computed goto), return
2435	the edge that will be taken out of the block.
2436	If VAL is NULL_TREE, then the current value of the final statement's
2437	predicate or index is used.
2438	If the value does not match a unique edge, NULL is returned. */
2439
2440	edge
2441	find_taken_edge (basic_block bb, tree val)
2442	{
2443	gimple *stmt;
2444
2445	stmt = *gsi_last_bb (bb);
2446
2447	/* Handle ENTRY and EXIT. */
2448	if (!stmt)
2449	;
2450
2451	else if (gimple_code (g: stmt) == GIMPLE_COND)
2452	return find_taken_edge_cond_expr (as_a <gcond *> (p: stmt), val);
2453
2454	else if (gimple_code (g: stmt) == GIMPLE_SWITCH)
2455	return find_taken_edge_switch_expr (switch_stmt: as_a <gswitch *> (p: stmt), val);
2456
2457	else if (computed_goto_p (t: stmt))
2458	{
2459	/* Only optimize if the argument is a label, if the argument is
2460	not a label then we cannot construct a proper CFG.
2461
2462	It may be the case that we only need to allow the LABEL_REF to
2463	appear inside an ADDR_EXPR, but we also allow the LABEL_REF to
2464	appear inside a LABEL_EXPR just to be safe. */
2465	if (val
2466	&& (TREE_CODE (val) == ADDR_EXPR || TREE_CODE (val) == LABEL_EXPR)
2467	&& TREE_CODE (TREE_OPERAND (val, 0)) == LABEL_DECL)
2468	return find_taken_edge_computed_goto (bb, TREE_OPERAND (val, 0));
2469	}
2470
2471	/* Otherwise we only know the taken successor edge if it's unique. */
2472	return single_succ_p (bb) ? single_succ_edge (bb) : NULL;
2473	}
2474
2475	/* Given a constant value VAL and the entry block BB to a GOTO_EXPR
2476	statement, determine which of the outgoing edges will be taken out of the
2477	block. Return NULL if either edge may be taken. */
2478
2479	static edge
2480	find_taken_edge_computed_goto (basic_block bb, tree val)
2481	{
2482	basic_block dest;
2483	edge e = NULL;
2484
2485	dest = label_to_block (cfun, dest: val);
2486	if (dest)
2487	e = find_edge (bb, dest);
2488
2489	/* It's possible for find_edge to return NULL here on invalid code
2490	that abuses the labels-as-values extension (e.g. code that attempts to
2491	jump *between* functions via stored labels-as-values; PR 84136).
2492	If so, then we simply return that NULL for the edge.
2493	We don't currently have a way of detecting such invalid code, so we
2494	can't assert that it was the case when a NULL edge occurs here. */
2495
2496	return e;
2497	}
2498
2499	/* Given COND_STMT and a constant value VAL for use as the predicate,
2500	determine which of the two edges will be taken out of
2501	the statement's block. Return NULL if either edge may be taken.
2502	If VAL is NULL_TREE, then the current value of COND_STMT's predicate
2503	is used. */
2504
2505	static edge
2506	find_taken_edge_cond_expr (const gcond *cond_stmt, tree val)
2507	{
2508	edge true_edge, false_edge;
2509
2510	if (val == NULL_TREE)
2511	{
2512	/* Use the current value of the predicate. */
2513	if (gimple_cond_true_p (gs: cond_stmt))
2514	val = integer_one_node;
2515	else if (gimple_cond_false_p (gs: cond_stmt))
2516	val = integer_zero_node;
2517	else
2518	return NULL;
2519	}
2520	else if (TREE_CODE (val) != INTEGER_CST)
2521	return NULL;
2522
2523	extract_true_false_edges_from_block (gimple_bb (g: cond_stmt),
2524	&true_edge, &false_edge);
2525
2526	return (integer_zerop (val) ? false_edge : true_edge);
2527	}
2528
2529	/* Given SWITCH_STMT and an INTEGER_CST VAL for use as the index, determine
2530	which edge will be taken out of the statement's block. Return NULL if any
2531	edge may be taken.
2532	If VAL is NULL_TREE, then the current value of SWITCH_STMT's index
2533	is used. */
2534
2535	edge
2536	find_taken_edge_switch_expr (const gswitch *switch_stmt, tree val)
2537	{
2538	basic_block dest_bb;
2539	edge e;
2540	tree taken_case;
2541
2542	if (gimple_switch_num_labels (gs: switch_stmt) == 1)
2543	taken_case = gimple_switch_default_label (gs: switch_stmt);
2544	else
2545	{
2546	if (val == NULL_TREE)
2547	val = gimple_switch_index (gs: switch_stmt);
2548	if (TREE_CODE (val) != INTEGER_CST)
2549	return NULL;
2550	else
2551	taken_case = find_case_label_for_value (switch_stmt, val);
2552	}
2553	dest_bb = label_to_block (cfun, CASE_LABEL (taken_case));
2554
2555	e = find_edge (gimple_bb (g: switch_stmt), dest_bb);
2556	gcc_assert (e);
2557	return e;
2558	}
2559
2560
2561	/* Return the CASE_LABEL_EXPR that SWITCH_STMT will take for VAL.
2562	We can make optimal use here of the fact that the case labels are
2563	sorted: We can do a binary search for a case matching VAL. */
2564
2565	tree
2566	find_case_label_for_value (const gswitch *switch_stmt, tree val)
2567	{
2568	size_t low, high, n = gimple_switch_num_labels (gs: switch_stmt);
2569	tree default_case = gimple_switch_default_label (gs: switch_stmt);
2570
2571	for (low = 0, high = n; high - low > 1; )
2572	{
2573	size_t i = (high + low) / 2;
2574	tree t = gimple_switch_label (gs: switch_stmt, index: i);
2575	int cmp;
2576
2577	/* Cache the result of comparing CASE_LOW and val. */
2578	cmp = tree_int_cst_compare (CASE_LOW (t), t2: val);
2579
2580	if (cmp > 0)
2581	high = i;
2582	else
2583	low = i;
2584
2585	if (CASE_HIGH (t) == NULL)
2586	{
2587	/* A singe-valued case label. */
2588	if (cmp == 0)
2589	return t;
2590	}
2591	else
2592	{
2593	/* A case range. We can only handle integer ranges. */
2594	if (cmp <= 0 && tree_int_cst_compare (CASE_HIGH (t), t2: val) >= 0)
2595	return t;
2596	}
2597	}
2598
2599	return default_case;
2600	}
2601
2602
2603	/* Dump a basic block on stderr. */
2604
2605	void
2606	gimple_debug_bb (basic_block bb)
2607	{
2608	dump_bb (stderr, bb, 0, TDF_VOPS|TDF_MEMSYMS|TDF_BLOCKS);
2609	}
2610
2611
2612	/* Dump basic block with index N on stderr. */
2613
2614	basic_block
2615	gimple_debug_bb_n (int n)
2616	{
2617	gimple_debug_bb (BASIC_BLOCK_FOR_FN (cfun, n));
2618	return BASIC_BLOCK_FOR_FN (cfun, n);
2619	}
2620
2621
2622	/* Dump the CFG on stderr.
2623
2624	FLAGS are the same used by the tree dumping functions
2625	(see TDF_* in dumpfile.h). */
2626
2627	void
2628	gimple_debug_cfg (dump_flags_t flags)
2629	{
2630	gimple_dump_cfg (stderr, flags);
2631	}
2632
2633
2634	/* Dump the program showing basic block boundaries on the given FILE.
2635
2636	FLAGS are the same used by the tree dumping functions (see TDF_* in
2637	tree.h). */
2638
2639	void
2640	gimple_dump_cfg (FILE *file, dump_flags_t flags)
2641	{
2642	if (flags & TDF_DETAILS)
2643	{
2644	dump_function_header (file, current_function_decl, flags);
2645	fprintf (stream: file, format: ";; \n%d basic blocks, %d edges, last basic block %d.\n\n",
2646	n_basic_blocks_for_fn (cfun), n_edges_for_fn (cfun),
2647	last_basic_block_for_fn (cfun));
2648
2649	brief_dump_cfg (file, flags);
2650	fprintf (stream: file, format: "\n");
2651	}
2652
2653	if (flags & TDF_STATS)
2654	dump_cfg_stats (file);
2655
2656	dump_function_to_file (current_function_decl, file, flags | TDF_BLOCKS);
2657	}
2658
2659
2660	/* Dump CFG statistics on FILE. */
2661
2662	void
2663	dump_cfg_stats (FILE *file)
2664	{
2665	static long max_num_merged_labels = 0;
2666	unsigned long size, total = 0;
2667	long num_edges;
2668	basic_block bb;
2669	const char * const fmt_str = "%-30s%-13s%12s\n";
2670	const char * const fmt_str_1 = "%-30s%13d" PRsa (11) "\n";
2671	const char * const fmt_str_2 = "%-30s%13ld" PRsa (11) "\n";
2672	const char * const fmt_str_3 = "%-43s" PRsa (11) "\n";
2673	const char *funcname = current_function_name ();
2674
2675	fprintf (stream: file, format: "\nCFG Statistics for %s\n\n", funcname);
2676
2677	fprintf (stream: file, format: "---------------------------------------------------------\n");
2678	fprintf (stream: file, format: fmt_str, "", " Number of ", "Memory");
2679	fprintf (stream: file, format: fmt_str, "", " instances ", "used ");
2680	fprintf (stream: file, format: "---------------------------------------------------------\n");
2681
2682	size = n_basic_blocks_for_fn (cfun) * sizeof (struct basic_block_def);
2683	total += size;
2684	fprintf (stream: file, format: fmt_str_1, "Basic blocks", n_basic_blocks_for_fn (cfun),
2685	SIZE_AMOUNT (size));
2686
2687	num_edges = 0;
2688	FOR_EACH_BB_FN (bb, cfun)
2689	num_edges += EDGE_COUNT (bb->succs);
2690	size = num_edges * sizeof (class edge_def);
2691	total += size;
2692	fprintf (stream: file, format: fmt_str_2, "Edges", num_edges, SIZE_AMOUNT (size));
2693
2694	fprintf (stream: file, format: "---------------------------------------------------------\n");
2695	fprintf (stream: file, format: fmt_str_3, "Total memory used by CFG data",
2696	SIZE_AMOUNT (total));
2697	fprintf (stream: file, format: "---------------------------------------------------------\n");
2698	fprintf (stream: file, format: "\n");
2699
2700	if (cfg_stats.num_merged_labels > max_num_merged_labels)
2701	max_num_merged_labels = cfg_stats.num_merged_labels;
2702
2703	fprintf (stream: file, format: "Coalesced label blocks: %ld (Max so far: %ld)\n",
2704	cfg_stats.num_merged_labels, max_num_merged_labels);
2705
2706	fprintf (stream: file, format: "\n");
2707	}
2708
2709
2710	/* Dump CFG statistics on stderr. Keep extern so that it's always
2711	linked in the final executable. */
2712
2713	DEBUG_FUNCTION void
2714	debug_cfg_stats (void)
2715	{
2716	dump_cfg_stats (stderr);
2717	}
2718
2719	/*---------------------------------------------------------------------------
2720	Miscellaneous helpers
2721	---------------------------------------------------------------------------*/
2722
2723	/* Return true if T, a GIMPLE_CALL, can make an abnormal transfer of control
2724	flow. Transfers of control flow associated with EH are excluded. */
2725
2726	static bool
2727	call_can_make_abnormal_goto (gimple *t)
2728	{
2729	/* If the function has no non-local labels, then a call cannot make an
2730	abnormal transfer of control. */
2731	if (!cfun->has_nonlocal_label
2732	&& !cfun->calls_setjmp)
2733	return false;
2734
2735	/* Likewise if the call has no side effects. */
2736	if (!gimple_has_side_effects (t))
2737	return false;
2738
2739	/* Likewise if the called function is leaf. */
2740	if (gimple_call_flags (t) & ECF_LEAF)
2741	return false;
2742
2743	return true;
2744	}
2745
2746
2747	/* Return true if T can make an abnormal transfer of control flow.
2748	Transfers of control flow associated with EH are excluded. */
2749
2750	bool
2751	stmt_can_make_abnormal_goto (gimple *t)
2752	{
2753	if (computed_goto_p (t))
2754	return true;
2755	if (is_gimple_call (gs: t))
2756	return call_can_make_abnormal_goto (t);
2757	return false;
2758	}
2759
2760
2761	/* Return true if T represents a stmt that always transfers control. */
2762
2763	bool
2764	is_ctrl_stmt (gimple *t)
2765	{
2766	switch (gimple_code (g: t))
2767	{
2768	case GIMPLE_COND:
2769	case GIMPLE_SWITCH:
2770	case GIMPLE_GOTO:
2771	case GIMPLE_RETURN:
2772	case GIMPLE_RESX:
2773	return true;
2774	default:
2775	return false;
2776	}
2777	}
2778
2779
2780	/* Return true if T is a statement that may alter the flow of control
2781	(e.g., a call to a non-returning function). */
2782
2783	bool
2784	is_ctrl_altering_stmt (gimple *t)
2785	{
2786	gcc_assert (t);
2787
2788	switch (gimple_code (g: t))
2789	{
2790	case GIMPLE_CALL:
2791	/* Per stmt call flag indicates whether the call could alter
2792	controlflow. */
2793	if (gimple_call_ctrl_altering_p (gs: t))
2794	return true;
2795	break;
2796
2797	case GIMPLE_EH_DISPATCH:
2798	/* EH_DISPATCH branches to the individual catch handlers at
2799	this level of a try or allowed-exceptions region. It can
2800	fallthru to the next statement as well. */
2801	return true;
2802
2803	case GIMPLE_ASM:
2804	if (gimple_asm_nlabels (asm_stmt: as_a <gasm *> (p: t)) > 0)
2805	return true;
2806	break;
2807
2808	CASE_GIMPLE_OMP:
2809	/* OpenMP directives alter control flow. */
2810	return true;
2811
2812	case GIMPLE_TRANSACTION:
2813	/* A transaction start alters control flow. */
2814	return true;
2815
2816	default:
2817	break;
2818	}
2819
2820	/* If a statement can throw, it alters control flow. */
2821	return stmt_can_throw_internal (cfun, t);
2822	}
2823
2824
2825	/* Return true if T is a simple local goto. */
2826
2827	bool
2828	simple_goto_p (gimple *t)
2829	{
2830	return (gimple_code (g: t) == GIMPLE_GOTO
2831	&& TREE_CODE (gimple_goto_dest (t)) == LABEL_DECL);
2832	}
2833
2834
2835	/* Return true if STMT should start a new basic block. PREV_STMT is
2836	the statement preceding STMT. It is used when STMT is a label or a
2837	case label. Labels should only start a new basic block if their
2838	previous statement wasn't a label. Otherwise, sequence of labels
2839	would generate unnecessary basic blocks that only contain a single
2840	label. */
2841
2842	static inline bool
2843	stmt_starts_bb_p (gimple *stmt, gimple *prev_stmt)
2844	{
2845	if (stmt == NULL)
2846	return false;
2847
2848	/* PREV_STMT is only set to a debug stmt if the debug stmt is before
2849	any nondebug stmts in the block. We don't want to start another
2850	block in this case: the debug stmt will already have started the
2851	one STMT would start if we weren't outputting debug stmts. */
2852	if (prev_stmt && is_gimple_debug (gs: prev_stmt))
2853	return false;
2854
2855	/* Labels start a new basic block only if the preceding statement
2856	wasn't a label of the same type. This prevents the creation of
2857	consecutive blocks that have nothing but a single label. */
2858	if (glabel *label_stmt = dyn_cast <glabel *> (p: stmt))
2859	{
2860	/* Nonlocal and computed GOTO targets always start a new block. */
2861	if (DECL_NONLOCAL (gimple_label_label (label_stmt))
2862	|| FORCED_LABEL (gimple_label_label (label_stmt)))
2863	return true;
2864
2865	if (glabel *plabel = safe_dyn_cast <glabel *> (p: prev_stmt))
2866	{
2867	if (DECL_NONLOCAL (gimple_label_label (plabel))
2868	|| !DECL_ARTIFICIAL (gimple_label_label (plabel)))
2869	return true;
2870
2871	cfg_stats.num_merged_labels++;
2872	return false;
2873	}
2874	else
2875	return true;
2876	}
2877	else if (gimple_code (g: stmt) == GIMPLE_CALL)
2878	{
2879	if (gimple_call_flags (stmt) & ECF_RETURNS_TWICE)
2880	/* setjmp acts similar to a nonlocal GOTO target and thus should
2881	start a new block. */
2882	return true;
2883	if (gimple_call_internal_p (gs: stmt, fn: IFN_PHI)
2884	&& prev_stmt
2885	&& gimple_code (g: prev_stmt) != GIMPLE_LABEL
2886	&& (gimple_code (g: prev_stmt) != GIMPLE_CALL
2887	|| ! gimple_call_internal_p (gs: prev_stmt, fn: IFN_PHI)))
2888	/* PHI nodes start a new block unless preceeded by a label
2889	or another PHI. */
2890	return true;
2891	}
2892
2893	return false;
2894	}
2895
2896
2897	/* Return true if T should end a basic block. */
2898
2899	bool
2900	stmt_ends_bb_p (gimple *t)
2901	{
2902	return is_ctrl_stmt (t) || is_ctrl_altering_stmt (t);
2903	}
2904
2905	/* Remove block annotations and other data structures. */
2906
2907	void
2908	delete_tree_cfg_annotations (struct function *fn)
2909	{
2910	vec_free (label_to_block_map_for_fn (fn));
2911	}
2912
2913	/* Return the virtual phi in BB. */
2914
2915	gphi *
2916	get_virtual_phi (basic_block bb)
2917	{
2918	for (gphi_iterator gsi = gsi_start_phis (bb);
2919	!gsi_end_p (i: gsi);
2920	gsi_next (i: &gsi))
2921	{
2922	gphi *phi = gsi.phi ();
2923
2924	if (virtual_operand_p (PHI_RESULT (phi)))
2925	return phi;
2926	}
2927
2928	return NULL;
2929	}
2930
2931	/* Return the first statement in basic block BB. */
2932
2933	gimple *
2934	first_stmt (basic_block bb)
2935	{
2936	gimple_stmt_iterator i = gsi_start_bb (bb);
2937	gimple *stmt = NULL;
2938
2939	while (!gsi_end_p (i) && is_gimple_debug (gs: (stmt = gsi_stmt (i))))
2940	{
2941	gsi_next (i: &i);
2942	stmt = NULL;
2943	}
2944	return stmt;
2945	}
2946
2947	/* Return the first non-label statement in basic block BB. */
2948
2949	static gimple *
2950	first_non_label_stmt (basic_block bb)
2951	{
2952	gimple_stmt_iterator i = gsi_start_bb (bb);
2953	while (!gsi_end_p (i) && gimple_code (g: gsi_stmt (i)) == GIMPLE_LABEL)
2954	gsi_next (i: &i);
2955	return !gsi_end_p (i) ? gsi_stmt (i) : NULL;
2956	}
2957
2958	/* Return the last statement in basic block BB. */
2959
2960	gimple *
2961	last_nondebug_stmt (basic_block bb)
2962	{
2963	gimple_stmt_iterator i = gsi_last_bb (bb);
2964	gimple *stmt = NULL;
2965
2966	while (!gsi_end_p (i) && is_gimple_debug (gs: (stmt = gsi_stmt (i))))
2967	{
2968	gsi_prev (i: &i);
2969	stmt = NULL;
2970	}
2971	return stmt;
2972	}
2973
2974	/* Return the last statement of an otherwise empty block. Return NULL
2975	if the block is totally empty, or if it contains more than one
2976	statement. */
2977
2978	gimple *
2979	last_and_only_stmt (basic_block bb)
2980	{
2981	gimple_stmt_iterator i = gsi_last_nondebug_bb (bb);
2982	gimple *last, *prev;
2983
2984	if (gsi_end_p (i))
2985	return NULL;
2986
2987	last = gsi_stmt (i);
2988	gsi_prev_nondebug (i: &i);
2989	if (gsi_end_p (i))
2990	return last;
2991
2992	/* Empty statements should no longer appear in the instruction stream.
2993	Everything that might have appeared before should be deleted by
2994	remove_useless_stmts, and the optimizers should just gsi_remove
2995	instead of smashing with build_empty_stmt.
2996
2997	Thus the only thing that should appear here in a block containing
2998	one executable statement is a label. */
2999	prev = gsi_stmt (i);
3000	if (gimple_code (g: prev) == GIMPLE_LABEL)
3001	return last;
3002	else
3003	return NULL;
3004	}
3005
3006	/* Returns the basic block after which the new basic block created
3007	by splitting edge EDGE_IN should be placed. Tries to keep the new block
3008	near its "logical" location. This is of most help to humans looking
3009	at debugging dumps. */
3010
3011	basic_block
3012	split_edge_bb_loc (edge edge_in)
3013	{
3014	basic_block dest = edge_in->dest;
3015	basic_block dest_prev = dest->prev_bb;
3016
3017	if (dest_prev)
3018	{
3019	edge e = find_edge (dest_prev, dest);
3020	if (e && !(e->flags & EDGE_COMPLEX))
3021	return edge_in->src;
3022	}
3023	return dest_prev;
3024	}
3025
3026	/* Split a (typically critical) edge EDGE_IN. Return the new block.
3027	Abort on abnormal edges. */
3028
3029	static basic_block
3030	gimple_split_edge (edge edge_in)
3031	{
3032	basic_block new_bb, after_bb, dest;
3033	edge new_edge, e;
3034
3035	/* Abnormal edges cannot be split. */
3036	gcc_assert (!(edge_in->flags & EDGE_ABNORMAL));
3037
3038	dest = edge_in->dest;
3039
3040	after_bb = split_edge_bb_loc (edge_in);
3041
3042	new_bb = create_empty_bb (after_bb);
3043	new_bb->count = edge_in->count ();
3044
3045	/* We want to avoid re-allocating PHIs when we first
3046	add the fallthru edge from new_bb to dest but we also
3047	want to avoid changing PHI argument order when
3048	first redirecting edge_in away from dest. The former
3049	avoids changing PHI argument order by adding them
3050	last and then the redirection swapping it back into
3051	place by means of unordered remove.
3052	So hack around things by temporarily removing all PHIs
3053	from the destination during the edge redirection and then
3054	making sure the edges stay in order. */
3055	gimple_seq saved_phis = phi_nodes (bb: dest);
3056	unsigned old_dest_idx = edge_in->dest_idx;
3057	set_phi_nodes (dest, NULL);
3058	new_edge = make_single_succ_edge (new_bb, dest, EDGE_FALLTHRU);
3059	e = redirect_edge_and_branch (edge_in, new_bb);
3060	gcc_assert (e == edge_in && new_edge->dest_idx == old_dest_idx);
3061	/* set_phi_nodes sets the BB of the PHI nodes, so do it manually here. */
3062	dest->il.gimple.phi_nodes = saved_phis;
3063
3064	return new_bb;
3065	}
3066
3067
3068	/* Verify properties of the address expression T whose base should be
3069	TREE_ADDRESSABLE if VERIFY_ADDRESSABLE is true. */
3070
3071	static bool
3072	verify_address (tree t, bool verify_addressable)
3073	{
3074	bool old_constant;
3075	bool old_side_effects;
3076	bool new_constant;
3077	bool new_side_effects;
3078
3079	old_constant = TREE_CONSTANT (t);
3080	old_side_effects = TREE_SIDE_EFFECTS (t);
3081
3082	recompute_tree_invariant_for_addr_expr (t);
3083	new_side_effects = TREE_SIDE_EFFECTS (t);
3084	new_constant = TREE_CONSTANT (t);
3085
3086	if (old_constant != new_constant)
3087	{
3088	error ("constant not recomputed when %<ADDR_EXPR%> changed");
3089	return true;
3090	}
3091	if (old_side_effects != new_side_effects)
3092	{
3093	error ("side effects not recomputed when %<ADDR_EXPR%> changed");
3094	return true;
3095	}
3096
3097	tree base = TREE_OPERAND (t, 0);
3098	while (handled_component_p (t: base))
3099	base = TREE_OPERAND (base, 0);
3100
3101	if (!(VAR_P (base)
3102	|| TREE_CODE (base) == PARM_DECL
3103	|| TREE_CODE (base) == RESULT_DECL))
3104	return false;
3105
3106	if (verify_addressable && !TREE_ADDRESSABLE (base))
3107	{
3108	error ("address taken but %<TREE_ADDRESSABLE%> bit not set");
3109	return true;
3110	}
3111
3112	return false;
3113	}
3114
3115
3116	/* Verify if EXPR is a valid GIMPLE reference expression. If
3117	REQUIRE_LVALUE is true verifies it is an lvalue. Returns true
3118	if there is an error, otherwise false. */
3119
3120	static bool
3121	verify_types_in_gimple_reference (tree expr, bool require_lvalue)
3122	{
3123	const char *code_name = get_tree_code_name (TREE_CODE (expr));
3124
3125	if (TREE_CODE (expr) == REALPART_EXPR
3126	|| TREE_CODE (expr) == IMAGPART_EXPR
3127	|| TREE_CODE (expr) == BIT_FIELD_REF
3128	|| TREE_CODE (expr) == VIEW_CONVERT_EXPR)
3129	{
3130	tree op = TREE_OPERAND (expr, 0);
3131	if (TREE_CODE (expr) != VIEW_CONVERT_EXPR
3132	&& !is_gimple_reg_type (TREE_TYPE (expr)))
3133	{
3134	error ("non-scalar %qs", code_name);
3135	return true;
3136	}
3137
3138	if (TREE_CODE (expr) == BIT_FIELD_REF)
3139	{
3140	tree t1 = TREE_OPERAND (expr, 1);
3141	tree t2 = TREE_OPERAND (expr, 2);
3142	poly_uint64 size, bitpos;
3143	if (!poly_int_tree_p (t: t1, value: &size)
3144	|| !poly_int_tree_p (t: t2, value: &bitpos)
3145	|| !types_compatible_p (bitsizetype, TREE_TYPE (t1))
3146	|| !types_compatible_p (bitsizetype, TREE_TYPE (t2)))
3147	{
3148	error ("invalid position or size operand to %qs", code_name);
3149	return true;
3150	}
3151	if (INTEGRAL_TYPE_P (TREE_TYPE (expr))
3152	&& maybe_ne (TYPE_PRECISION (TREE_TYPE (expr)), b: size))
3153	{
3154	error ("integral result type precision does not match "
3155	"field size of %qs", code_name);
3156	return true;
3157	}
3158	else if (!INTEGRAL_TYPE_P (TREE_TYPE (expr))
3159	&& TYPE_MODE (TREE_TYPE (expr)) != BLKmode
3160	&& maybe_ne (a: GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (expr))),
3161	b: size))
3162	{
3163	error ("mode size of non-integral result does not "
3164	"match field size of %qs",
3165	code_name);
3166	return true;
3167	}
3168	if (INTEGRAL_TYPE_P (TREE_TYPE (op))
3169	&& !type_has_mode_precision_p (TREE_TYPE (op)))
3170	{
3171	error ("%qs of non-mode-precision operand", code_name);
3172	return true;
3173	}
3174	if (!AGGREGATE_TYPE_P (TREE_TYPE (op))
3175	&& maybe_gt (size + bitpos,
3176	tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (op)))))
3177	{
3178	error ("position plus size exceeds size of referenced object in "
3179	"%qs", code_name);
3180	return true;
3181	}
3182	}
3183
3184	if ((TREE_CODE (expr) == REALPART_EXPR
3185	|| TREE_CODE (expr) == IMAGPART_EXPR)
3186	&& !useless_type_conversion_p (TREE_TYPE (expr),
3187	TREE_TYPE (TREE_TYPE (op))))
3188	{
3189	error ("type mismatch in %qs reference", code_name);
3190	debug_generic_stmt (TREE_TYPE (expr));
3191	debug_generic_stmt (TREE_TYPE (TREE_TYPE (op)));
3192	return true;
3193	}
3194
3195	if (TREE_CODE (expr) == VIEW_CONVERT_EXPR)
3196	{
3197	/* For VIEW_CONVERT_EXPRs which are allowed here too, we only check
3198	that their operand is not a register an invariant when
3199	requiring an lvalue (this usually means there is a SRA or IPA-SRA
3200	bug). Otherwise there is nothing to verify, gross mismatches at
3201	most invoke undefined behavior. */
3202	if (require_lvalue
3203	&& (is_gimple_reg (op) || is_gimple_min_invariant (op)))
3204	{
3205	error ("conversion of %qs on the left hand side of %qs",
3206	get_tree_code_name (TREE_CODE (op)), code_name);
3207	debug_generic_stmt (expr);
3208	return true;
3209	}
3210	else if (is_gimple_reg (op)
3211	&& TYPE_SIZE (TREE_TYPE (expr)) != TYPE_SIZE (TREE_TYPE (op)))
3212	{
3213	error ("conversion of register to a different size in %qs",
3214	code_name);
3215	debug_generic_stmt (expr);
3216	return true;
3217	}
3218	}
3219
3220	expr = op;
3221	}
3222
3223	bool require_non_reg = false;
3224	while (handled_component_p (t: expr))
3225	{
3226	require_non_reg = true;
3227	code_name = get_tree_code_name (TREE_CODE (expr));
3228
3229	if (TREE_CODE (expr) == REALPART_EXPR
3230	|| TREE_CODE (expr) == IMAGPART_EXPR
3231	|| TREE_CODE (expr) == BIT_FIELD_REF)
3232	{
3233	error ("non-top-level %qs", code_name);
3234	return true;
3235	}
3236
3237	tree op = TREE_OPERAND (expr, 0);
3238
3239	if (TREE_CODE (expr) == ARRAY_REF
3240	|| TREE_CODE (expr) == ARRAY_RANGE_REF)
3241	{
3242	if (!is_gimple_val (TREE_OPERAND (expr, 1))
3243	|| (TREE_OPERAND (expr, 2)
3244	&& !is_gimple_val (TREE_OPERAND (expr, 2)))
3245	|| (TREE_OPERAND (expr, 3)
3246	&& !is_gimple_val (TREE_OPERAND (expr, 3))))
3247	{
3248	error ("invalid operands to %qs", code_name);
3249	debug_generic_stmt (expr);
3250	return true;
3251	}
3252	}
3253
3254	/* Verify if the reference array element types are compatible. */
3255	if (TREE_CODE (expr) == ARRAY_REF
3256	&& !useless_type_conversion_p (TREE_TYPE (expr),
3257	TREE_TYPE (TREE_TYPE (op))))
3258	{
3259	error ("type mismatch in %qs", code_name);
3260	debug_generic_stmt (TREE_TYPE (expr));
3261	debug_generic_stmt (TREE_TYPE (TREE_TYPE (op)));
3262	return true;
3263	}
3264	if (TREE_CODE (expr) == ARRAY_RANGE_REF
3265	&& !useless_type_conversion_p (TREE_TYPE (TREE_TYPE (expr)),
3266	TREE_TYPE (TREE_TYPE (op))))
3267	{
3268	error ("type mismatch in %qs", code_name);
3269	debug_generic_stmt (TREE_TYPE (TREE_TYPE (expr)));
3270	debug_generic_stmt (TREE_TYPE (TREE_TYPE (op)));
3271	return true;
3272	}
3273
3274	if (TREE_CODE (expr) == COMPONENT_REF)
3275	{
3276	if (TREE_OPERAND (expr, 2)
3277	&& !is_gimple_val (TREE_OPERAND (expr, 2)))
3278	{
3279	error ("invalid %qs offset operator", code_name);
3280	return true;
3281	}
3282	if (!useless_type_conversion_p (TREE_TYPE (expr),
3283	TREE_TYPE (TREE_OPERAND (expr, 1))))
3284	{
3285	error ("type mismatch in %qs", code_name);
3286	debug_generic_stmt (TREE_TYPE (expr));
3287	debug_generic_stmt (TREE_TYPE (TREE_OPERAND (expr, 1)));
3288	return true;
3289	}
3290	}
3291
3292	expr = op;
3293	}
3294
3295	code_name = get_tree_code_name (TREE_CODE (expr));
3296
3297	if (TREE_CODE (expr) == MEM_REF)
3298	{
3299	if (!is_gimple_mem_ref_addr (TREE_OPERAND (expr, 0))
3300	|| (TREE_CODE (TREE_OPERAND (expr, 0)) == ADDR_EXPR
3301	&& verify_address (TREE_OPERAND (expr, 0), verify_addressable: false)))
3302	{
3303	error ("invalid address operand in %qs", code_name);
3304	debug_generic_stmt (expr);
3305	return true;
3306	}
3307	if (!poly_int_tree_p (TREE_OPERAND (expr, 1))
3308	|| !POINTER_TYPE_P (TREE_TYPE (TREE_OPERAND (expr, 1))))
3309	{
3310	error ("invalid offset operand in %qs", code_name);
3311	debug_generic_stmt (expr);
3312	return true;
3313	}
3314	if (MR_DEPENDENCE_CLIQUE (expr) != 0
3315	&& MR_DEPENDENCE_CLIQUE (expr) > cfun->last_clique)
3316	{
3317	error ("invalid clique in %qs", code_name);
3318	debug_generic_stmt (expr);
3319	return true;
3320	}
3321	}
3322	else if (TREE_CODE (expr) == TARGET_MEM_REF)
3323	{
3324	if (!TMR_BASE (expr)
3325	|| !is_gimple_mem_ref_addr (TMR_BASE (expr))
3326	|| (TREE_CODE (TMR_BASE (expr)) == ADDR_EXPR
3327	&& verify_address (TMR_BASE (expr), verify_addressable: false)))
3328	{
3329	error ("invalid address operand in %qs", code_name);
3330	return true;
3331	}
3332	if (!TMR_OFFSET (expr)
3333	|| !poly_int_tree_p (TMR_OFFSET (expr))
3334	|| !POINTER_TYPE_P (TREE_TYPE (TMR_OFFSET (expr))))
3335	{
3336	error ("invalid offset operand in %qs", code_name);
3337	debug_generic_stmt (expr);
3338	return true;
3339	}
3340	if (MR_DEPENDENCE_CLIQUE (expr) != 0
3341	&& MR_DEPENDENCE_CLIQUE (expr) > cfun->last_clique)
3342	{
3343	error ("invalid clique in %qs", code_name);
3344	debug_generic_stmt (expr);
3345	return true;
3346	}
3347	}
3348	else if (INDIRECT_REF_P (expr))
3349	{
3350	error ("%qs in gimple IL", code_name);
3351	debug_generic_stmt (expr);
3352	return true;
3353	}
3354	else if (require_non_reg
3355	&& (is_gimple_reg (expr)
3356	|| (is_gimple_min_invariant (expr)
3357	/* STRING_CSTs are representatives of the string table
3358	entry which lives in memory. */
3359	&& TREE_CODE (expr) != STRING_CST)))
3360	{
3361	error ("%qs as base where non-register is required", code_name);
3362	debug_generic_stmt (expr);
3363	return true;
3364	}
3365
3366	if (!require_lvalue
3367	&& (is_gimple_reg (expr) || is_gimple_min_invariant (expr)))
3368	return false;
3369
3370	if (TREE_CODE (expr) != SSA_NAME && is_gimple_id (t: expr))
3371	return false;
3372
3373	if (TREE_CODE (expr) != TARGET_MEM_REF
3374	&& TREE_CODE (expr) != MEM_REF)
3375	{
3376	error ("invalid expression for min lvalue");
3377	return true;
3378	}
3379
3380	return false;
3381	}
3382
3383	/* Returns true if there is one pointer type in TYPE_POINTER_TO (SRC_OBJ)
3384	list of pointer-to types that is trivially convertible to DEST. */
3385
3386	static bool
3387	one_pointer_to_useless_type_conversion_p (tree dest, tree src_obj)
3388	{
3389	tree src;
3390
3391	if (!TYPE_POINTER_TO (src_obj))
3392	return true;
3393
3394	for (src = TYPE_POINTER_TO (src_obj); src; src = TYPE_NEXT_PTR_TO (src))
3395	if (useless_type_conversion_p (dest, src))
3396	return true;
3397
3398	return false;
3399	}
3400
3401	/* Return true if TYPE1 is a fixed-point type and if conversions to and
3402	from TYPE2 can be handled by FIXED_CONVERT_EXPR. */
3403
3404	static bool
3405	valid_fixed_convert_types_p (tree type1, tree type2)
3406	{
3407	return (FIXED_POINT_TYPE_P (type1)
3408	&& (INTEGRAL_TYPE_P (type2)
3409	|| SCALAR_FLOAT_TYPE_P (type2)
3410	|| FIXED_POINT_TYPE_P (type2)));
3411	}
3412
3413	/* Verify the contents of a GIMPLE_CALL STMT. Returns true when there
3414	is a problem, otherwise false. */
3415
3416	static bool
3417	verify_gimple_call (gcall *stmt)
3418	{
3419	tree fn = gimple_call_fn (gs: stmt);
3420	tree fntype, fndecl;
3421	unsigned i;
3422
3423	if (gimple_call_internal_p (gs: stmt))
3424	{
3425	if (fn)
3426	{
3427	error ("gimple call has two targets");
3428	debug_generic_stmt (fn);
3429	return true;
3430	}
3431	}
3432	else
3433	{
3434	if (!fn)
3435	{
3436	error ("gimple call has no target");
3437	return true;
3438	}
3439	}
3440
3441	if (fn && !is_gimple_call_addr (fn))
3442	{
3443	error ("invalid function in gimple call");
3444	debug_generic_stmt (fn);
3445	return true;
3446	}
3447
3448	if (fn
3449	&& (!POINTER_TYPE_P (TREE_TYPE (fn))
3450	|| (TREE_CODE (TREE_TYPE (TREE_TYPE (fn))) != FUNCTION_TYPE
3451	&& TREE_CODE (TREE_TYPE (TREE_TYPE (fn))) != METHOD_TYPE)))
3452	{
3453	error ("non-function in gimple call");
3454	return true;
3455	}
3456
3457	fndecl = gimple_call_fndecl (gs: stmt);
3458	if (fndecl
3459	&& TREE_CODE (fndecl) == FUNCTION_DECL
3460	&& DECL_LOOPING_CONST_OR_PURE_P (fndecl)
3461	&& !DECL_PURE_P (fndecl)
3462	&& !TREE_READONLY (fndecl))
3463	{
3464	error ("invalid pure const state for function");
3465	return true;
3466	}
3467
3468	tree lhs = gimple_call_lhs (gs: stmt);
3469	if (lhs
3470	&& (!is_gimple_reg (lhs)
3471	&& (!is_gimple_lvalue (lhs)
3472	|| verify_types_in_gimple_reference
3473	(TREE_CODE (lhs) == WITH_SIZE_EXPR
3474	? TREE_OPERAND (lhs, 0) : lhs, require_lvalue: true))))
3475	{
3476	error ("invalid LHS in gimple call");
3477	return true;
3478	}
3479
3480	if (gimple_call_ctrl_altering_p (gs: stmt)
3481	&& gimple_call_noreturn_p (s: stmt)
3482	&& should_remove_lhs_p (lhs))
3483	{
3484	error ("LHS in %<noreturn%> call");
3485	return true;
3486	}
3487
3488	fntype = gimple_call_fntype (gs: stmt);
3489	if (fntype
3490	&& lhs
3491	&& !useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (fntype))
3492	/* ??? At least C++ misses conversions at assignments from
3493	void * call results.
3494	For now simply allow arbitrary pointer type conversions. */
3495	&& !(POINTER_TYPE_P (TREE_TYPE (lhs))
3496	&& POINTER_TYPE_P (TREE_TYPE (fntype))))
3497	{
3498	error ("invalid conversion in gimple call");
3499	debug_generic_stmt (TREE_TYPE (lhs));
3500	debug_generic_stmt (TREE_TYPE (fntype));
3501	return true;
3502	}
3503
3504	if (gimple_call_chain (gs: stmt)
3505	&& !is_gimple_val (gimple_call_chain (gs: stmt)))
3506	{
3507	error ("invalid static chain in gimple call");
3508	debug_generic_stmt (gimple_call_chain (gs: stmt));
3509	return true;
3510	}
3511
3512	/* If there is a static chain argument, the call should either be
3513	indirect, or the decl should have DECL_STATIC_CHAIN set. */
3514	if (gimple_call_chain (gs: stmt)
3515	&& fndecl
3516	&& !DECL_STATIC_CHAIN (fndecl))
3517	{
3518	error ("static chain with function that doesn%'t use one");
3519	return true;
3520	}
3521
3522	if (fndecl && fndecl_built_in_p (node: fndecl, klass: BUILT_IN_NORMAL))
3523	{
3524	switch (DECL_FUNCTION_CODE (decl: fndecl))
3525	{
3526	case BUILT_IN_UNREACHABLE:
3527	case BUILT_IN_UNREACHABLE_TRAP:
3528	case BUILT_IN_TRAP:
3529	if (gimple_call_num_args (gs: stmt) > 0)
3530	{
3531	/* Built-in unreachable with parameters might not be caught by
3532	undefined behavior sanitizer. Front-ends do check users do not
3533	call them that way but we also produce calls to
3534	__builtin_unreachable internally, for example when IPA figures
3535	out a call cannot happen in a legal program. In such cases,
3536	we must make sure arguments are stripped off. */
3537	error ("%<__builtin_unreachable%> or %<__builtin_trap%> call "
3538	"with arguments");
3539	return true;
3540	}
3541	break;
3542	default:
3543	break;
3544	}
3545	}
3546
3547	/* For a call to .DEFERRED_INIT,
3548	LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL)
3549	we should guarantee that when the 1st argument is a constant, it should
3550	be the same as the size of the LHS. */
3551
3552	if (gimple_call_internal_p (gs: stmt, fn: IFN_DEFERRED_INIT))
3553	{
3554	tree size_of_arg0 = gimple_call_arg (gs: stmt, index: 0);
3555	tree size_of_lhs = TYPE_SIZE_UNIT (TREE_TYPE (lhs));
3556
3557	if (TREE_CODE (lhs) == SSA_NAME)
3558	lhs = SSA_NAME_VAR (lhs);
3559
3560	poly_uint64 size_from_arg0, size_from_lhs;
3561	bool is_constant_size_arg0 = poly_int_tree_p (t: size_of_arg0,
3562	value: &size_from_arg0);
3563	bool is_constant_size_lhs = poly_int_tree_p (t: size_of_lhs,
3564	value: &size_from_lhs);
3565	if (is_constant_size_arg0 && is_constant_size_lhs)
3566	if (maybe_ne (a: size_from_arg0, b: size_from_lhs))
3567	{
3568	error ("%<DEFERRED_INIT%> calls should have same "
3569	"constant size for the first argument and LHS");
3570	return true;
3571	}
3572	}
3573
3574	/* ??? The C frontend passes unpromoted arguments in case it
3575	didn't see a function declaration before the call. So for now
3576	leave the call arguments mostly unverified. Once we gimplify
3577	unit-at-a-time we have a chance to fix this. */
3578	for (i = 0; i < gimple_call_num_args (gs: stmt); ++i)
3579	{
3580	tree arg = gimple_call_arg (gs: stmt, index: i);
3581	if ((is_gimple_reg_type (TREE_TYPE (arg))
3582	&& !is_gimple_val (arg))
3583	|| (!is_gimple_reg_type (TREE_TYPE (arg))
3584	&& !is_gimple_lvalue (arg)))
3585	{
3586	error ("invalid argument to gimple call");
3587	debug_generic_expr (arg);
3588	return true;
3589	}
3590	if (!is_gimple_reg (arg))
3591	{
3592	if (TREE_CODE (arg) == WITH_SIZE_EXPR)
3593	arg = TREE_OPERAND (arg, 0);
3594	if (verify_types_in_gimple_reference (expr: arg, require_lvalue: false))
3595	return true;
3596	}
3597	}
3598
3599	return false;
3600	}
3601
3602	/* Verifies the gimple comparison with the result type TYPE and
3603	the operands OP0 and OP1, comparison code is CODE. */
3604
3605	static bool
3606	verify_gimple_comparison (tree type, tree op0, tree op1, enum tree_code code)
3607	{
3608	tree op0_type = TREE_TYPE (op0);
3609	tree op1_type = TREE_TYPE (op1);
3610
3611	if (!is_gimple_val (op0) || !is_gimple_val (op1))
3612	{
3613	error ("invalid operands in gimple comparison");
3614	return true;
3615	}
3616
3617	/* For comparisons we do not have the operations type as the
3618	effective type the comparison is carried out in. Instead
3619	we require that either the first operand is trivially
3620	convertible into the second, or the other way around. */
3621	if (!useless_type_conversion_p (op0_type, op1_type)
3622	&& !useless_type_conversion_p (op1_type, op0_type))
3623	{
3624	error ("mismatching comparison operand types");
3625	debug_generic_expr (op0_type);
3626	debug_generic_expr (op1_type);
3627	return true;
3628	}
3629
3630	/* The resulting type of a comparison may be an effective boolean type. */
3631	if (INTEGRAL_TYPE_P (type)
3632	&& (TREE_CODE (type) == BOOLEAN_TYPE
3633	|| TYPE_PRECISION (type) == 1))
3634	{
3635	if ((VECTOR_TYPE_P (op0_type)
3636	|| VECTOR_TYPE_P (op1_type))
3637	&& code != EQ_EXPR && code != NE_EXPR
3638	&& !VECTOR_BOOLEAN_TYPE_P (op0_type)
3639	&& !VECTOR_INTEGER_TYPE_P (op0_type))
3640	{
3641	error ("unsupported operation or type for vector comparison"
3642	" returning a boolean");
3643	debug_generic_expr (op0_type);
3644	debug_generic_expr (op1_type);
3645	return true;
3646	}
3647	}
3648	/* Or a boolean vector type with the same element count
3649	as the comparison operand types. */
3650	else if (VECTOR_TYPE_P (type)
3651	&& TREE_CODE (TREE_TYPE (type)) == BOOLEAN_TYPE)
3652	{
3653	if (TREE_CODE (op0_type) != VECTOR_TYPE
3654	|| TREE_CODE (op1_type) != VECTOR_TYPE)
3655	{
3656	error ("non-vector operands in vector comparison");
3657	debug_generic_expr (op0_type);
3658	debug_generic_expr (op1_type);
3659	return true;
3660	}
3661
3662	if (maybe_ne (a: TYPE_VECTOR_SUBPARTS (node: type),
3663	b: TYPE_VECTOR_SUBPARTS (node: op0_type)))
3664	{
3665	error ("invalid vector comparison resulting type");
3666	debug_generic_expr (type);
3667	return true;
3668	}
3669	}
3670	else
3671	{
3672	error ("bogus comparison result type");
3673	debug_generic_expr (type);
3674	return true;
3675	}
3676
3677	return false;
3678	}
3679
3680	/* Verify a gimple assignment statement STMT with an unary rhs.
3681	Returns true if anything is wrong. */
3682
3683	static bool
3684	verify_gimple_assign_unary (gassign *stmt)
3685	{
3686	enum tree_code rhs_code = gimple_assign_rhs_code (gs: stmt);
3687	tree lhs = gimple_assign_lhs (gs: stmt);
3688	tree lhs_type = TREE_TYPE (lhs);
3689	tree rhs1 = gimple_assign_rhs1 (gs: stmt);
3690	tree rhs1_type = TREE_TYPE (rhs1);
3691
3692	if (!is_gimple_reg (lhs))
3693	{
3694	error ("non-register as LHS of unary operation");
3695	return true;
3696	}
3697
3698	if (!is_gimple_val (rhs1))
3699	{
3700	error ("invalid operand in unary operation");
3701	return true;
3702	}
3703
3704	const char* const code_name = get_tree_code_name (rhs_code);
3705
3706	/* First handle conversions. */
3707	switch (rhs_code)
3708	{
3709	CASE_CONVERT:
3710	{
3711	/* Allow conversions between vectors with the same number of elements,
3712	provided that the conversion is OK for the element types too. */
3713	if (VECTOR_TYPE_P (lhs_type)
3714	&& VECTOR_TYPE_P (rhs1_type)
3715	&& known_eq (TYPE_VECTOR_SUBPARTS (lhs_type),
3716	TYPE_VECTOR_SUBPARTS (rhs1_type)))
3717	{
3718	lhs_type = TREE_TYPE (lhs_type);
3719	rhs1_type = TREE_TYPE (rhs1_type);
3720	}
3721	else if (VECTOR_TYPE_P (lhs_type) || VECTOR_TYPE_P (rhs1_type))
3722	{
3723	error ("invalid vector types in nop conversion");
3724	debug_generic_expr (lhs_type);
3725	debug_generic_expr (rhs1_type);
3726	return true;
3727	}
3728
3729	/* Allow conversions from pointer type to integral type only if
3730	there is no sign or zero extension involved.
3731	For targets were the precision of ptrofftype doesn't match that
3732	of pointers we allow conversions to types where
3733	POINTERS_EXTEND_UNSIGNED specifies how that works. */
3734	if ((POINTER_TYPE_P (lhs_type)
3735	&& INTEGRAL_TYPE_P (rhs1_type))
3736	|| (POINTER_TYPE_P (rhs1_type)
3737	&& INTEGRAL_TYPE_P (lhs_type)
3738	&& (TYPE_PRECISION (rhs1_type) >= TYPE_PRECISION (lhs_type)
3739	#if defined(POINTERS_EXTEND_UNSIGNED)
3740	|| (TYPE_MODE (rhs1_type) == ptr_mode
3741	&& (TYPE_PRECISION (lhs_type)
3742	== BITS_PER_WORD /* word_mode */
3743	|| (TYPE_PRECISION (lhs_type)
3744	== GET_MODE_PRECISION (Pmode))))
3745	#endif
3746	)))
3747	return false;
3748
3749	/* Allow conversion from integral to offset type and vice versa. */
3750	if ((TREE_CODE (lhs_type) == OFFSET_TYPE
3751	&& INTEGRAL_TYPE_P (rhs1_type))
3752	|| (INTEGRAL_TYPE_P (lhs_type)
3753	&& TREE_CODE (rhs1_type) == OFFSET_TYPE))
3754	return false;
3755
3756	/* Otherwise assert we are converting between types of the
3757	same kind. */
3758	if (INTEGRAL_TYPE_P (lhs_type) != INTEGRAL_TYPE_P (rhs1_type))
3759	{
3760	error ("invalid types in nop conversion");
3761	debug_generic_expr (lhs_type);
3762	debug_generic_expr (rhs1_type);
3763	return true;
3764	}
3765
3766	return false;
3767	}
3768
3769	case ADDR_SPACE_CONVERT_EXPR:
3770	{
3771	if (!POINTER_TYPE_P (rhs1_type) || !POINTER_TYPE_P (lhs_type)
3772	|| (TYPE_ADDR_SPACE (TREE_TYPE (rhs1_type))
3773	== TYPE_ADDR_SPACE (TREE_TYPE (lhs_type))))
3774	{
3775	error ("invalid types in address space conversion");
3776	debug_generic_expr (lhs_type);
3777	debug_generic_expr (rhs1_type);
3778	return true;
3779	}
3780
3781	return false;
3782	}
3783
3784	case FIXED_CONVERT_EXPR:
3785	{
3786	if (!valid_fixed_convert_types_p (type1: lhs_type, type2: rhs1_type)
3787	&& !valid_fixed_convert_types_p (type1: rhs1_type, type2: lhs_type))
3788	{
3789	error ("invalid types in fixed-point conversion");
3790	debug_generic_expr (lhs_type);
3791	debug_generic_expr (rhs1_type);
3792	return true;
3793	}
3794
3795	return false;
3796	}
3797
3798	case FLOAT_EXPR:
3799	{
3800	if ((!INTEGRAL_TYPE_P (rhs1_type) || !SCALAR_FLOAT_TYPE_P (lhs_type))
3801	&& (!VECTOR_INTEGER_TYPE_P (rhs1_type)
3802	|| !VECTOR_FLOAT_TYPE_P (lhs_type)))
3803	{
3804	error ("invalid types in conversion to floating-point");
3805	debug_generic_expr (lhs_type);
3806	debug_generic_expr (rhs1_type);
3807	return true;
3808	}
3809
3810	return false;
3811	}
3812
3813	case FIX_TRUNC_EXPR:
3814	{
3815	if ((!INTEGRAL_TYPE_P (lhs_type) || !SCALAR_FLOAT_TYPE_P (rhs1_type))
3816	&& (!VECTOR_INTEGER_TYPE_P (lhs_type)
3817	|| !VECTOR_FLOAT_TYPE_P (rhs1_type)))
3818	{
3819	error ("invalid types in conversion to integer");
3820	debug_generic_expr (lhs_type);
3821	debug_generic_expr (rhs1_type);
3822	return true;
3823	}
3824
3825	return false;
3826	}
3827
3828	case VEC_UNPACK_HI_EXPR:
3829	case VEC_UNPACK_LO_EXPR:
3830	case VEC_UNPACK_FLOAT_HI_EXPR:
3831	case VEC_UNPACK_FLOAT_LO_EXPR:
3832	case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
3833	case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
3834	if (TREE_CODE (rhs1_type) != VECTOR_TYPE
3835	|| TREE_CODE (lhs_type) != VECTOR_TYPE
3836	|| (!INTEGRAL_TYPE_P (TREE_TYPE (lhs_type))
3837	&& !SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs_type)))
3838	|| (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
3839	&& !SCALAR_FLOAT_TYPE_P (TREE_TYPE (rhs1_type)))
3840	|| ((rhs_code == VEC_UNPACK_HI_EXPR
3841	|| rhs_code == VEC_UNPACK_LO_EXPR)
3842	&& (INTEGRAL_TYPE_P (TREE_TYPE (lhs_type))
3843	!= INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))))
3844	|| ((rhs_code == VEC_UNPACK_FLOAT_HI_EXPR
3845	|| rhs_code == VEC_UNPACK_FLOAT_LO_EXPR)
3846	&& (INTEGRAL_TYPE_P (TREE_TYPE (lhs_type))
3847	|| SCALAR_FLOAT_TYPE_P (TREE_TYPE (rhs1_type))))
3848	|| ((rhs_code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
3849	|| rhs_code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
3850	&& (INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
3851	|| SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs_type))))
3852	|| (maybe_ne (a: GET_MODE_SIZE (mode: element_mode (lhs_type)),
3853	b: 2 * GET_MODE_SIZE (mode: element_mode (rhs1_type)))
3854	&& (!VECTOR_BOOLEAN_TYPE_P (lhs_type)
3855	|| !VECTOR_BOOLEAN_TYPE_P (rhs1_type)))
3856	|| maybe_ne (a: 2 * TYPE_VECTOR_SUBPARTS (node: lhs_type),
3857	b: TYPE_VECTOR_SUBPARTS (node: rhs1_type)))
3858	{
3859	error ("type mismatch in %qs expression", code_name);
3860	debug_generic_expr (lhs_type);
3861	debug_generic_expr (rhs1_type);
3862	return true;
3863	}
3864
3865	return false;
3866
3867	case NEGATE_EXPR:
3868	case ABS_EXPR:
3869	case BIT_NOT_EXPR:
3870	case PAREN_EXPR:
3871	case CONJ_EXPR:
3872	/* Disallow pointer and offset types for many of the unary gimple. */
3873	if (POINTER_TYPE_P (lhs_type)
3874	|| TREE_CODE (lhs_type) == OFFSET_TYPE)
3875	{
3876	error ("invalid types for %qs", code_name);
3877	debug_generic_expr (lhs_type);
3878	debug_generic_expr (rhs1_type);
3879	return true;
3880	}
3881	break;
3882
3883	case ABSU_EXPR:
3884	if (!ANY_INTEGRAL_TYPE_P (lhs_type)
3885	|| !TYPE_UNSIGNED (lhs_type)
3886	|| !ANY_INTEGRAL_TYPE_P (rhs1_type)
3887	|| TYPE_UNSIGNED (rhs1_type)
3888	|| element_precision (lhs_type) != element_precision (rhs1_type))
3889	{
3890	error ("invalid types for %qs", code_name);
3891	debug_generic_expr (lhs_type);
3892	debug_generic_expr (rhs1_type);
3893	return true;
3894	}
3895	return false;
3896
3897	case VEC_DUPLICATE_EXPR:
3898	if (TREE_CODE (lhs_type) != VECTOR_TYPE
3899	|| !useless_type_conversion_p (TREE_TYPE (lhs_type), rhs1_type))
3900	{
3901	error ("%qs should be from a scalar to a like vector", code_name);
3902	debug_generic_expr (lhs_type);
3903	debug_generic_expr (rhs1_type);
3904	return true;
3905	}
3906	return false;
3907
3908	default:
3909	gcc_unreachable ();
3910	}
3911
3912	/* For the remaining codes assert there is no conversion involved. */
3913	if (!useless_type_conversion_p (lhs_type, rhs1_type))
3914	{
3915	error ("non-trivial conversion in unary operation");
3916	debug_generic_expr (lhs_type);
3917	debug_generic_expr (rhs1_type);
3918	return true;
3919	}
3920
3921	return false;
3922	}
3923
3924	/* Verify a gimple assignment statement STMT with a binary rhs.
3925	Returns true if anything is wrong. */
3926
3927	static bool
3928	verify_gimple_assign_binary (gassign *stmt)
3929	{
3930	enum tree_code rhs_code = gimple_assign_rhs_code (gs: stmt);
3931	tree lhs = gimple_assign_lhs (gs: stmt);
3932	tree lhs_type = TREE_TYPE (lhs);
3933	tree rhs1 = gimple_assign_rhs1 (gs: stmt);
3934	tree rhs1_type = TREE_TYPE (rhs1);
3935	tree rhs2 = gimple_assign_rhs2 (gs: stmt);
3936	tree rhs2_type = TREE_TYPE (rhs2);
3937
3938	if (!is_gimple_reg (lhs))
3939	{
3940	error ("non-register as LHS of binary operation");
3941	return true;
3942	}
3943
3944	if (!is_gimple_val (rhs1)
3945	|| !is_gimple_val (rhs2))
3946	{
3947	error ("invalid operands in binary operation");
3948	return true;
3949	}
3950
3951	const char* const code_name = get_tree_code_name (rhs_code);
3952
3953	/* First handle operations that involve different types. */
3954	switch (rhs_code)
3955	{
3956	case COMPLEX_EXPR:
3957	{
3958	if (TREE_CODE (lhs_type) != COMPLEX_TYPE
3959	|| !(INTEGRAL_TYPE_P (rhs1_type)
3960	|| SCALAR_FLOAT_TYPE_P (rhs1_type))
3961	|| !(INTEGRAL_TYPE_P (rhs2_type)
3962	|| SCALAR_FLOAT_TYPE_P (rhs2_type)))
3963	{
3964	error ("type mismatch in %qs", code_name);
3965	debug_generic_expr (lhs_type);
3966	debug_generic_expr (rhs1_type);
3967	debug_generic_expr (rhs2_type);
3968	return true;
3969	}
3970
3971	return false;
3972	}
3973
3974	case LSHIFT_EXPR:
3975	case RSHIFT_EXPR:
3976	case LROTATE_EXPR:
3977	case RROTATE_EXPR:
3978	{
3979	/* Shifts and rotates are ok on integral types, fixed point
3980	types and integer vector types. */
3981	if ((!INTEGRAL_TYPE_P (rhs1_type)
3982	&& !FIXED_POINT_TYPE_P (rhs1_type)
3983	&& ! (VECTOR_TYPE_P (rhs1_type)
3984	&& INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))))
3985	|| (!INTEGRAL_TYPE_P (rhs2_type)
3986	/* Vector shifts of vectors are also ok. */
3987	&& ! (VECTOR_TYPE_P (rhs1_type)
3988	&& INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
3989	&& VECTOR_TYPE_P (rhs2_type)
3990	&& INTEGRAL_TYPE_P (TREE_TYPE (rhs2_type))))
3991	|| !useless_type_conversion_p (lhs_type, rhs1_type))
3992	{
3993	error ("type mismatch in %qs", code_name);
3994	debug_generic_expr (lhs_type);
3995	debug_generic_expr (rhs1_type);
3996	debug_generic_expr (rhs2_type);
3997	return true;
3998	}
3999
4000	return false;
4001	}
4002
4003	case WIDEN_LSHIFT_EXPR:
4004	{
4005	if (!INTEGRAL_TYPE_P (lhs_type)
4006	|| !INTEGRAL_TYPE_P (rhs1_type)
4007	|| TREE_CODE (rhs2) != INTEGER_CST
4008	|| (2 * TYPE_PRECISION (rhs1_type) > TYPE_PRECISION (lhs_type)))
4009	{
4010	error ("type mismatch in %qs", code_name);
4011	debug_generic_expr (lhs_type);
4012	debug_generic_expr (rhs1_type);
4013	debug_generic_expr (rhs2_type);
4014	return true;
4015	}
4016
4017	return false;
4018	}
4019
4020	case VEC_WIDEN_LSHIFT_HI_EXPR:
4021	case VEC_WIDEN_LSHIFT_LO_EXPR:
4022	{
4023	if (TREE_CODE (rhs1_type) != VECTOR_TYPE
4024	|| TREE_CODE (lhs_type) != VECTOR_TYPE
4025	|| !INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
4026	|| !INTEGRAL_TYPE_P (TREE_TYPE (lhs_type))
4027	|| TREE_CODE (rhs2) != INTEGER_CST
4028	|| (2 * TYPE_PRECISION (TREE_TYPE (rhs1_type))
4029	> TYPE_PRECISION (TREE_TYPE (lhs_type))))
4030	{
4031	error ("type mismatch in %qs", code_name);
4032	debug_generic_expr (lhs_type);
4033	debug_generic_expr (rhs1_type);
4034	debug_generic_expr (rhs2_type);
4035	return true;
4036	}
4037
4038	return false;
4039	}
4040
4041	case PLUS_EXPR:
4042	case MINUS_EXPR:
4043	{
4044	tree lhs_etype = lhs_type;
4045	tree rhs1_etype = rhs1_type;
4046	tree rhs2_etype = rhs2_type;
4047	if (VECTOR_TYPE_P (lhs_type))
4048	{
4049	if (TREE_CODE (rhs1_type) != VECTOR_TYPE
4050	|| TREE_CODE (rhs2_type) != VECTOR_TYPE)
4051	{
4052	error ("invalid non-vector operands to %qs", code_name);
4053	return true;
4054	}
4055	lhs_etype = TREE_TYPE (lhs_type);
4056	rhs1_etype = TREE_TYPE (rhs1_type);
4057	rhs2_etype = TREE_TYPE (rhs2_type);
4058	}
4059	if (POINTER_TYPE_P (lhs_etype)
4060	|| POINTER_TYPE_P (rhs1_etype)
4061	|| POINTER_TYPE_P (rhs2_etype))
4062	{
4063	error ("invalid (pointer) operands %qs", code_name);
4064	return true;
4065	}
4066
4067	/* Continue with generic binary expression handling. */
4068	break;
4069	}
4070
4071	case POINTER_PLUS_EXPR:
4072	{
4073	if (!POINTER_TYPE_P (rhs1_type)
4074	|| !useless_type_conversion_p (lhs_type, rhs1_type)
4075	|| !ptrofftype_p (type: rhs2_type))
4076	{
4077	error ("type mismatch in %qs", code_name);
4078	debug_generic_stmt (lhs_type);
4079	debug_generic_stmt (rhs1_type);
4080	debug_generic_stmt (rhs2_type);
4081	return true;
4082	}
4083
4084	return false;
4085	}
4086
4087	case POINTER_DIFF_EXPR:
4088	{
4089	if (!POINTER_TYPE_P (rhs1_type)
4090	|| !POINTER_TYPE_P (rhs2_type)
4091	/* Because we special-case pointers to void we allow difference
4092	of arbitrary pointers with the same mode. */
4093	|| TYPE_MODE (rhs1_type) != TYPE_MODE (rhs2_type)
4094	|| !INTEGRAL_TYPE_P (lhs_type)
4095	|| TYPE_UNSIGNED (lhs_type)
4096	|| TYPE_PRECISION (lhs_type) != TYPE_PRECISION (rhs1_type))
4097	{
4098	error ("type mismatch in %qs", code_name);
4099	debug_generic_stmt (lhs_type);
4100	debug_generic_stmt (rhs1_type);
4101	debug_generic_stmt (rhs2_type);
4102	return true;
4103	}
4104
4105	return false;
4106	}
4107
4108	case TRUTH_ANDIF_EXPR:
4109	case TRUTH_ORIF_EXPR:
4110	case TRUTH_AND_EXPR:
4111	case TRUTH_OR_EXPR:
4112	case TRUTH_XOR_EXPR:
4113
4114	gcc_unreachable ();
4115
4116	case LT_EXPR:
4117	case LE_EXPR:
4118	case GT_EXPR:
4119	case GE_EXPR:
4120	case EQ_EXPR:
4121	case NE_EXPR:
4122	case UNORDERED_EXPR:
4123	case ORDERED_EXPR:
4124	case UNLT_EXPR:
4125	case UNLE_EXPR:
4126	case UNGT_EXPR:
4127	case UNGE_EXPR:
4128	case UNEQ_EXPR:
4129	case LTGT_EXPR:
4130	/* Comparisons are also binary, but the result type is not
4131	connected to the operand types. */
4132	return verify_gimple_comparison (type: lhs_type, op0: rhs1, op1: rhs2, code: rhs_code);
4133
4134	case WIDEN_MULT_EXPR:
4135	if (TREE_CODE (lhs_type) != INTEGER_TYPE)
4136	return true;
4137	return ((2 * TYPE_PRECISION (rhs1_type) > TYPE_PRECISION (lhs_type))
4138	|| (TYPE_PRECISION (rhs1_type) != TYPE_PRECISION (rhs2_type)));
4139
4140	case WIDEN_SUM_EXPR:
4141	{
4142	if (((TREE_CODE (rhs1_type) != VECTOR_TYPE
4143	|| TREE_CODE (lhs_type) != VECTOR_TYPE)
4144	&& ((!INTEGRAL_TYPE_P (rhs1_type)
4145	&& !SCALAR_FLOAT_TYPE_P (rhs1_type))
4146	|| (!INTEGRAL_TYPE_P (lhs_type)
4147	&& !SCALAR_FLOAT_TYPE_P (lhs_type))))
4148	|| !useless_type_conversion_p (lhs_type, rhs2_type)
4149	|| maybe_lt (a: GET_MODE_SIZE (mode: element_mode (rhs2_type)),
4150	b: 2 * GET_MODE_SIZE (mode: element_mode (rhs1_type))))
4151	{
4152	error ("type mismatch in %qs", code_name);
4153	debug_generic_expr (lhs_type);
4154	debug_generic_expr (rhs1_type);
4155	debug_generic_expr (rhs2_type);
4156	return true;
4157	}
4158	return false;
4159	}
4160
4161	case VEC_WIDEN_MULT_HI_EXPR:
4162	case VEC_WIDEN_MULT_LO_EXPR:
4163	case VEC_WIDEN_MULT_EVEN_EXPR:
4164	case VEC_WIDEN_MULT_ODD_EXPR:
4165	{
4166	if (TREE_CODE (rhs1_type) != VECTOR_TYPE
4167	|| TREE_CODE (lhs_type) != VECTOR_TYPE
4168	|| !types_compatible_p (type1: rhs1_type, type2: rhs2_type)
4169	|| maybe_ne (a: GET_MODE_SIZE (mode: element_mode (lhs_type)),
4170	b: 2 * GET_MODE_SIZE (mode: element_mode (rhs1_type))))
4171	{
4172	error ("type mismatch in %qs", code_name);
4173	debug_generic_expr (lhs_type);
4174	debug_generic_expr (rhs1_type);
4175	debug_generic_expr (rhs2_type);
4176	return true;
4177	}
4178	return false;
4179	}
4180
4181	case VEC_PACK_TRUNC_EXPR:
4182	/* ??? We currently use VEC_PACK_TRUNC_EXPR to simply concat
4183	vector boolean types. */
4184	if (VECTOR_BOOLEAN_TYPE_P (lhs_type)
4185	&& VECTOR_BOOLEAN_TYPE_P (rhs1_type)
4186	&& types_compatible_p (type1: rhs1_type, type2: rhs2_type)
4187	&& known_eq (TYPE_VECTOR_SUBPARTS (lhs_type),
4188	2 * TYPE_VECTOR_SUBPARTS (rhs1_type)))
4189	return false;
4190
4191	/* Fallthru. */
4192	case VEC_PACK_SAT_EXPR:
4193	case VEC_PACK_FIX_TRUNC_EXPR:
4194	{
4195	if (TREE_CODE (rhs1_type) != VECTOR_TYPE
4196	|| TREE_CODE (lhs_type) != VECTOR_TYPE
4197	|| !((rhs_code == VEC_PACK_FIX_TRUNC_EXPR
4198	&& SCALAR_FLOAT_TYPE_P (TREE_TYPE (rhs1_type))
4199	&& INTEGRAL_TYPE_P (TREE_TYPE (lhs_type)))
4200	|| (INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
4201	== INTEGRAL_TYPE_P (TREE_TYPE (lhs_type))))
4202	|| !types_compatible_p (type1: rhs1_type, type2: rhs2_type)
4203	|| maybe_ne (a: GET_MODE_SIZE (mode: element_mode (rhs1_type)),
4204	b: 2 * GET_MODE_SIZE (mode: element_mode (lhs_type)))
4205	|| maybe_ne (a: 2 * TYPE_VECTOR_SUBPARTS (node: rhs1_type),
4206	b: TYPE_VECTOR_SUBPARTS (node: lhs_type)))
4207	{
4208	error ("type mismatch in %qs", code_name);
4209	debug_generic_expr (lhs_type);
4210	debug_generic_expr (rhs1_type);
4211	debug_generic_expr (rhs2_type);
4212	return true;
4213	}
4214
4215	return false;
4216	}
4217
4218	case VEC_PACK_FLOAT_EXPR:
4219	if (TREE_CODE (rhs1_type) != VECTOR_TYPE
4220	|| TREE_CODE (lhs_type) != VECTOR_TYPE
4221	|| !INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
4222	|| !SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs_type))
4223	|| !types_compatible_p (type1: rhs1_type, type2: rhs2_type)
4224	|| maybe_ne (a: GET_MODE_SIZE (mode: element_mode (rhs1_type)),
4225	b: 2 * GET_MODE_SIZE (mode: element_mode (lhs_type)))
4226	|| maybe_ne (a: 2 * TYPE_VECTOR_SUBPARTS (node: rhs1_type),
4227	b: TYPE_VECTOR_SUBPARTS (node: lhs_type)))
4228	{
4229	error ("type mismatch in %qs", code_name);
4230	debug_generic_expr (lhs_type);
4231	debug_generic_expr (rhs1_type);
4232	debug_generic_expr (rhs2_type);
4233	return true;
4234	}
4235
4236	return false;
4237
4238	case MULT_EXPR:
4239	case MULT_HIGHPART_EXPR:
4240	case TRUNC_DIV_EXPR:
4241	case CEIL_DIV_EXPR:
4242	case FLOOR_DIV_EXPR:
4243	case ROUND_DIV_EXPR:
4244	case TRUNC_MOD_EXPR:
4245	case CEIL_MOD_EXPR:
4246	case FLOOR_MOD_EXPR:
4247	case ROUND_MOD_EXPR:
4248	case RDIV_EXPR:
4249	case EXACT_DIV_EXPR:
4250	case BIT_IOR_EXPR:
4251	case BIT_XOR_EXPR:
4252	/* Disallow pointer and offset types for many of the binary gimple. */
4253	if (POINTER_TYPE_P (lhs_type)
4254	|| TREE_CODE (lhs_type) == OFFSET_TYPE)
4255	{
4256	error ("invalid types for %qs", code_name);
4257	debug_generic_expr (lhs_type);
4258	debug_generic_expr (rhs1_type);
4259	debug_generic_expr (rhs2_type);
4260	return true;
4261	}
4262	/* Continue with generic binary expression handling. */
4263	break;
4264
4265	case MIN_EXPR:
4266	case MAX_EXPR:
4267	/* Continue with generic binary expression handling. */
4268	break;
4269
4270	case BIT_AND_EXPR:
4271	if (POINTER_TYPE_P (lhs_type)
4272	&& TREE_CODE (rhs2) == INTEGER_CST)
4273	break;
4274	/* Disallow pointer and offset types for many of the binary gimple. */
4275	if (POINTER_TYPE_P (lhs_type)
4276	|| TREE_CODE (lhs_type) == OFFSET_TYPE)
4277	{
4278	error ("invalid types for %qs", code_name);
4279	debug_generic_expr (lhs_type);
4280	debug_generic_expr (rhs1_type);
4281	debug_generic_expr (rhs2_type);
4282	return true;
4283	}
4284	/* Continue with generic binary expression handling. */
4285	break;
4286
4287	case VEC_SERIES_EXPR:
4288	if (!useless_type_conversion_p (rhs1_type, rhs2_type))
4289	{
4290	error ("type mismatch in %qs", code_name);
4291	debug_generic_expr (rhs1_type);
4292	debug_generic_expr (rhs2_type);
4293	return true;
4294	}
4295	if (TREE_CODE (lhs_type) != VECTOR_TYPE
4296	|| !useless_type_conversion_p (TREE_TYPE (lhs_type), rhs1_type))
4297	{
4298	error ("vector type expected in %qs", code_name);
4299	debug_generic_expr (lhs_type);
4300	return true;
4301	}
4302	return false;
4303
4304	default:
4305	gcc_unreachable ();
4306	}
4307
4308	if (!useless_type_conversion_p (lhs_type, rhs1_type)
4309	|| !useless_type_conversion_p (lhs_type, rhs2_type))
4310	{
4311	error ("type mismatch in binary expression");
4312	debug_generic_stmt (lhs_type);
4313	debug_generic_stmt (rhs1_type);
4314	debug_generic_stmt (rhs2_type);
4315	return true;
4316	}
4317
4318	return false;
4319	}
4320
4321	/* Verify a gimple assignment statement STMT with a ternary rhs.
4322	Returns true if anything is wrong. */
4323
4324	static bool
4325	verify_gimple_assign_ternary (gassign *stmt)
4326	{
4327	enum tree_code rhs_code = gimple_assign_rhs_code (gs: stmt);
4328	tree lhs = gimple_assign_lhs (gs: stmt);
4329	tree lhs_type = TREE_TYPE (lhs);
4330	tree rhs1 = gimple_assign_rhs1 (gs: stmt);
4331	tree rhs1_type = TREE_TYPE (rhs1);
4332	tree rhs2 = gimple_assign_rhs2 (gs: stmt);
4333	tree rhs2_type = TREE_TYPE (rhs2);
4334	tree rhs3 = gimple_assign_rhs3 (gs: stmt);
4335	tree rhs3_type = TREE_TYPE (rhs3);
4336
4337	if (!is_gimple_reg (lhs))
4338	{
4339	error ("non-register as LHS of ternary operation");
4340	return true;
4341	}
4342
4343	if (!is_gimple_val (rhs1)
4344	|| !is_gimple_val (rhs2)
4345	|| !is_gimple_val (rhs3))
4346	{
4347	error ("invalid operands in ternary operation");
4348	return true;
4349	}
4350
4351	const char* const code_name = get_tree_code_name (rhs_code);
4352
4353	/* First handle operations that involve different types. */
4354	switch (rhs_code)
4355	{
4356	case WIDEN_MULT_PLUS_EXPR:
4357	case WIDEN_MULT_MINUS_EXPR:
4358	if ((!INTEGRAL_TYPE_P (rhs1_type)
4359	&& !FIXED_POINT_TYPE_P (rhs1_type))
4360	|| !useless_type_conversion_p (rhs1_type, rhs2_type)
4361	|| !useless_type_conversion_p (lhs_type, rhs3_type)
4362	|| 2 * TYPE_PRECISION (rhs1_type) > TYPE_PRECISION (lhs_type)
4363	|| TYPE_PRECISION (rhs1_type) != TYPE_PRECISION (rhs2_type))
4364	{
4365	error ("type mismatch in %qs", code_name);
4366	debug_generic_expr (lhs_type);
4367	debug_generic_expr (rhs1_type);
4368	debug_generic_expr (rhs2_type);
4369	debug_generic_expr (rhs3_type);
4370	return true;
4371	}
4372	break;
4373
4374	case VEC_COND_EXPR:
4375	if (!VECTOR_BOOLEAN_TYPE_P (rhs1_type)
4376	|| maybe_ne (a: TYPE_VECTOR_SUBPARTS (node: rhs1_type),
4377	b: TYPE_VECTOR_SUBPARTS (node: lhs_type)))
4378	{
4379	error ("the first argument of a %qs must be of a "
4380	"boolean vector type of the same number of elements "
4381	"as the result", code_name);
4382	debug_generic_expr (lhs_type);
4383	debug_generic_expr (rhs1_type);
4384	return true;
4385	}
4386	/* Fallthrough. */
4387	case COND_EXPR:
4388	if (!useless_type_conversion_p (lhs_type, rhs2_type)
4389	|| !useless_type_conversion_p (lhs_type, rhs3_type))
4390	{
4391	error ("type mismatch in %qs", code_name);
4392	debug_generic_expr (lhs_type);
4393	debug_generic_expr (rhs2_type);
4394	debug_generic_expr (rhs3_type);
4395	return true;
4396	}
4397	break;
4398
4399	case VEC_PERM_EXPR:
4400	/* If permute is constant, then we allow for lhs and rhs
4401	to have different vector types, provided:
4402	(1) lhs, rhs1, rhs2 have same element type.
4403	(2) rhs3 vector is constant and has integer element type.
4404	(3) len(lhs) == len(rhs3) && len(rhs1) == len(rhs2). */
4405
4406	if (TREE_CODE (lhs_type) != VECTOR_TYPE
4407	|| TREE_CODE (rhs1_type) != VECTOR_TYPE
4408	|| TREE_CODE (rhs2_type) != VECTOR_TYPE
4409	|| TREE_CODE (rhs3_type) != VECTOR_TYPE)
4410	{
4411	error ("vector types expected in %qs", code_name);
4412	debug_generic_expr (lhs_type);
4413	debug_generic_expr (rhs1_type);
4414	debug_generic_expr (rhs2_type);
4415	debug_generic_expr (rhs3_type);
4416	return true;
4417	}
4418
4419	/* If rhs3 is constant, we allow lhs, rhs1 and rhs2 to be different vector types,
4420	as long as lhs, rhs1 and rhs2 have same element type. */
4421	if (TREE_CONSTANT (rhs3)
4422	? (!useless_type_conversion_p (TREE_TYPE (lhs_type), TREE_TYPE (rhs1_type))
4423	|| !useless_type_conversion_p (TREE_TYPE (lhs_type), TREE_TYPE (rhs2_type)))
4424	: (!useless_type_conversion_p (lhs_type, rhs1_type)
4425	|| !useless_type_conversion_p (lhs_type, rhs2_type)))
4426	{
4427	error ("type mismatch in %qs", code_name);
4428	debug_generic_expr (lhs_type);
4429	debug_generic_expr (rhs1_type);
4430	debug_generic_expr (rhs2_type);
4431	debug_generic_expr (rhs3_type);
4432	return true;
4433	}
4434
4435	/* If rhs3 is constant, relax the check len(rhs2) == len(rhs3). */
4436	if (maybe_ne (a: TYPE_VECTOR_SUBPARTS (node: rhs1_type),
4437	b: TYPE_VECTOR_SUBPARTS (node: rhs2_type))
4438	|| (!TREE_CONSTANT(rhs3)
4439	&& maybe_ne (a: TYPE_VECTOR_SUBPARTS (node: rhs2_type),
4440	b: TYPE_VECTOR_SUBPARTS (node: rhs3_type)))
4441	|| maybe_ne (a: TYPE_VECTOR_SUBPARTS (node: rhs3_type),
4442	b: TYPE_VECTOR_SUBPARTS (node: lhs_type)))
4443	{
4444	error ("vectors with different element number found in %qs",
4445	code_name);
4446	debug_generic_expr (lhs_type);
4447	debug_generic_expr (rhs1_type);
4448	debug_generic_expr (rhs2_type);
4449	debug_generic_expr (rhs3_type);
4450	return true;
4451	}
4452
4453	if (TREE_CODE (TREE_TYPE (rhs3_type)) != INTEGER_TYPE
4454	|| (TREE_CODE (rhs3) != VECTOR_CST
4455	&& (GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE
4456	(TREE_TYPE (rhs3_type)))
4457	!= GET_MODE_BITSIZE (SCALAR_TYPE_MODE
4458	(TREE_TYPE (rhs1_type))))))
4459	{
4460	error ("invalid mask type in %qs", code_name);
4461	debug_generic_expr (lhs_type);
4462	debug_generic_expr (rhs1_type);
4463	debug_generic_expr (rhs2_type);
4464	debug_generic_expr (rhs3_type);
4465	return true;
4466	}
4467
4468	return false;
4469
4470	case SAD_EXPR:
4471	if (!useless_type_conversion_p (rhs1_type, rhs2_type)
4472	|| !useless_type_conversion_p (lhs_type, rhs3_type)
4473	|| 2 * GET_MODE_UNIT_BITSIZE (TYPE_MODE (TREE_TYPE (rhs1_type)))
4474	> GET_MODE_UNIT_BITSIZE (TYPE_MODE (TREE_TYPE (lhs_type))))
4475	{
4476	error ("type mismatch in %qs", code_name);
4477	debug_generic_expr (lhs_type);
4478	debug_generic_expr (rhs1_type);
4479	debug_generic_expr (rhs2_type);
4480	debug_generic_expr (rhs3_type);
4481	return true;
4482	}
4483
4484	if (TREE_CODE (rhs1_type) != VECTOR_TYPE
4485	|| TREE_CODE (rhs2_type) != VECTOR_TYPE
4486	|| TREE_CODE (rhs3_type) != VECTOR_TYPE)
4487	{
4488	error ("vector types expected in %qs", code_name);
4489	debug_generic_expr (lhs_type);
4490	debug_generic_expr (rhs1_type);
4491	debug_generic_expr (rhs2_type);
4492	debug_generic_expr (rhs3_type);
4493	return true;
4494	}
4495
4496	return false;
4497
4498	case BIT_INSERT_EXPR:
4499	if (! useless_type_conversion_p (lhs_type, rhs1_type))
4500	{
4501	error ("type mismatch in %qs", code_name);
4502	debug_generic_expr (lhs_type);
4503	debug_generic_expr (rhs1_type);
4504	return true;
4505	}
4506	if (! ((INTEGRAL_TYPE_P (rhs1_type)
4507	&& INTEGRAL_TYPE_P (rhs2_type))
4508	/* Vector element insert. */
4509	|| (VECTOR_TYPE_P (rhs1_type)
4510	&& types_compatible_p (TREE_TYPE (rhs1_type), type2: rhs2_type))
4511	/* Aligned sub-vector insert. */
4512	|| (VECTOR_TYPE_P (rhs1_type)
4513	&& VECTOR_TYPE_P (rhs2_type)
4514	&& types_compatible_p (TREE_TYPE (rhs1_type),
4515	TREE_TYPE (rhs2_type))
4516	&& multiple_p (a: TYPE_VECTOR_SUBPARTS (node: rhs1_type),
4517	b: TYPE_VECTOR_SUBPARTS (node: rhs2_type))
4518	&& multiple_p (a: wi::to_poly_offset (t: rhs3),
4519	b: wi::to_poly_offset (TYPE_SIZE (rhs2_type))))))
4520	{
4521	error ("not allowed type combination in %qs", code_name);
4522	debug_generic_expr (rhs1_type);
4523	debug_generic_expr (rhs2_type);
4524	return true;
4525	}
4526	if (! tree_fits_uhwi_p (rhs3)
4527	|| ! types_compatible_p (bitsizetype, TREE_TYPE (rhs3))
4528	|| ! tree_fits_uhwi_p (TYPE_SIZE (rhs2_type)))
4529	{
4530	error ("invalid position or size in %qs", code_name);
4531	return true;
4532	}
4533	if (INTEGRAL_TYPE_P (rhs1_type)
4534	&& !type_has_mode_precision_p (t: rhs1_type))
4535	{
4536	error ("%qs into non-mode-precision operand", code_name);
4537	return true;
4538	}
4539	if (INTEGRAL_TYPE_P (rhs1_type))
4540	{
4541	unsigned HOST_WIDE_INT bitpos = tree_to_uhwi (rhs3);
4542	if (bitpos >= TYPE_PRECISION (rhs1_type)
4543	|| (bitpos + TYPE_PRECISION (rhs2_type)
4544	> TYPE_PRECISION (rhs1_type)))
4545	{
4546	error ("insertion out of range in %qs", code_name);
4547	return true;
4548	}
4549	}
4550	else if (VECTOR_TYPE_P (rhs1_type))
4551	{
4552	unsigned HOST_WIDE_INT bitpos = tree_to_uhwi (rhs3);
4553	unsigned HOST_WIDE_INT bitsize = tree_to_uhwi (TYPE_SIZE (rhs2_type));
4554	if (bitpos % bitsize != 0)
4555	{
4556	error ("%qs not at element boundary", code_name);
4557	return true;
4558	}
4559	}
4560	return false;
4561
4562	case DOT_PROD_EXPR:
4563	{
4564	if (((TREE_CODE (rhs1_type) != VECTOR_TYPE
4565	|| TREE_CODE (lhs_type) != VECTOR_TYPE)
4566	&& ((!INTEGRAL_TYPE_P (rhs1_type)
4567	&& !SCALAR_FLOAT_TYPE_P (rhs1_type))
4568	|| (!INTEGRAL_TYPE_P (lhs_type)
4569	&& !SCALAR_FLOAT_TYPE_P (lhs_type))))
4570	/* rhs1_type and rhs2_type may differ in sign. */
4571	|| !tree_nop_conversion_p (rhs1_type, rhs2_type)
4572	|| !useless_type_conversion_p (lhs_type, rhs3_type)
4573	|| maybe_lt (a: GET_MODE_SIZE (mode: element_mode (rhs3_type)),
4574	b: 2 * GET_MODE_SIZE (mode: element_mode (rhs1_type))))
4575	{
4576	error ("type mismatch in %qs", code_name);
4577	debug_generic_expr (lhs_type);
4578	debug_generic_expr (rhs1_type);
4579	debug_generic_expr (rhs2_type);
4580	return true;
4581	}
4582	return false;
4583	}
4584
4585	case REALIGN_LOAD_EXPR:
4586	/* FIXME. */
4587	return false;
4588
4589	default:
4590	gcc_unreachable ();
4591	}
4592	return false;
4593	}
4594
4595	/* Verify a gimple assignment statement STMT with a single rhs.
4596	Returns true if anything is wrong. */
4597
4598	static bool
4599	verify_gimple_assign_single (gassign *stmt)
4600	{
4601	enum tree_code rhs_code = gimple_assign_rhs_code (gs: stmt);
4602	tree lhs = gimple_assign_lhs (gs: stmt);
4603	tree lhs_type = TREE_TYPE (lhs);
4604	tree rhs1 = gimple_assign_rhs1 (gs: stmt);
4605	tree rhs1_type = TREE_TYPE (rhs1);
4606	bool res = false;
4607
4608	const char* const code_name = get_tree_code_name (rhs_code);
4609
4610	if (!useless_type_conversion_p (lhs_type, rhs1_type))
4611	{
4612	error ("non-trivial conversion in %qs", code_name);
4613	debug_generic_expr (lhs_type);
4614	debug_generic_expr (rhs1_type);
4615	return true;
4616	}
4617
4618	if (gimple_clobber_p (s: stmt)
4619	&& !(DECL_P (lhs) || TREE_CODE (lhs) == MEM_REF))
4620	{
4621	error ("%qs LHS in clobber statement",
4622	get_tree_code_name (TREE_CODE (lhs)));
4623	debug_generic_expr (lhs);
4624	return true;
4625	}
4626
4627	if (TREE_CODE (lhs) == WITH_SIZE_EXPR)
4628	{
4629	error ("%qs LHS in assignment statement",
4630	get_tree_code_name (TREE_CODE (lhs)));
4631	debug_generic_expr (lhs);
4632	return true;
4633	}
4634
4635	if (handled_component_p (t: lhs)
4636	|| TREE_CODE (lhs) == MEM_REF
4637	|| TREE_CODE (lhs) == TARGET_MEM_REF)
4638	res |= verify_types_in_gimple_reference (expr: lhs, require_lvalue: true);
4639
4640	/* Special codes we cannot handle via their class. */
4641	switch (rhs_code)
4642	{
4643	case ADDR_EXPR:
4644	{
4645	tree op = TREE_OPERAND (rhs1, 0);
4646	if (!is_gimple_addressable (t: op))
4647	{
4648	error ("invalid operand in %qs", code_name);
4649	return true;
4650	}
4651
4652	/* Technically there is no longer a need for matching types, but
4653	gimple hygiene asks for this check. In LTO we can end up
4654	combining incompatible units and thus end up with addresses
4655	of globals that change their type to a common one. */
4656	if (!in_lto_p
4657	&& !types_compatible_p (TREE_TYPE (op),
4658	TREE_TYPE (TREE_TYPE (rhs1)))
4659	&& !one_pointer_to_useless_type_conversion_p (TREE_TYPE (rhs1),
4660	TREE_TYPE (op)))
4661	{
4662	error ("type mismatch in %qs", code_name);
4663	debug_generic_stmt (TREE_TYPE (rhs1));
4664	debug_generic_stmt (TREE_TYPE (op));
4665	return true;
4666	}
4667
4668	return (verify_address (t: rhs1, verify_addressable: true)
4669	|| verify_types_in_gimple_reference (expr: op, require_lvalue: true));
4670	}
4671
4672	/* tcc_reference */
4673	case INDIRECT_REF:
4674	error ("%qs in gimple IL", code_name);
4675	return true;
4676
4677	case WITH_SIZE_EXPR:
4678	if (!is_gimple_val (TREE_OPERAND (rhs1, 1)))
4679	{
4680	error ("invalid %qs size argument in load", code_name);
4681	debug_generic_stmt (lhs);
4682	debug_generic_stmt (rhs1);
4683	return true;
4684	}
4685	rhs1 = TREE_OPERAND (rhs1, 0);
4686	/* Fallthru. */
4687	case COMPONENT_REF:
4688	case BIT_FIELD_REF:
4689	case ARRAY_REF:
4690	case ARRAY_RANGE_REF:
4691	case VIEW_CONVERT_EXPR:
4692	case REALPART_EXPR:
4693	case IMAGPART_EXPR:
4694	case TARGET_MEM_REF:
4695	case MEM_REF:
4696	if (!is_gimple_reg (lhs)
4697	&& is_gimple_reg_type (TREE_TYPE (lhs)))
4698	{
4699	error ("invalid RHS for gimple memory store: %qs", code_name);
4700	debug_generic_stmt (lhs);
4701	debug_generic_stmt (rhs1);
4702	return true;
4703	}
4704	return res || verify_types_in_gimple_reference (expr: rhs1, require_lvalue: false);
4705
4706	/* tcc_constant */
4707	case SSA_NAME:
4708	case INTEGER_CST:
4709	case REAL_CST:
4710	case FIXED_CST:
4711	case COMPLEX_CST:
4712	case VECTOR_CST:
4713	case STRING_CST:
4714	return res;
4715
4716	/* tcc_declaration */
4717	case CONST_DECL:
4718	return res;
4719	case VAR_DECL:
4720	case PARM_DECL:
4721	if (!is_gimple_reg (lhs)
4722	&& !is_gimple_reg (rhs1)
4723	&& is_gimple_reg_type (TREE_TYPE (lhs)))
4724	{
4725	error ("invalid RHS for gimple memory store: %qs", code_name);
4726	debug_generic_stmt (lhs);
4727	debug_generic_stmt (rhs1);
4728	return true;
4729	}
4730	return res;
4731
4732	case CONSTRUCTOR:
4733	if (VECTOR_TYPE_P (rhs1_type))
4734	{
4735	unsigned int i;
4736	tree elt_i, elt_v, elt_t = NULL_TREE;
4737
4738	if (CONSTRUCTOR_NELTS (rhs1) == 0)
4739	return res;
4740	/* For vector CONSTRUCTORs we require that either it is empty
4741	CONSTRUCTOR, or it is a CONSTRUCTOR of smaller vector elements
4742	(then the element count must be correct to cover the whole
4743	outer vector and index must be NULL on all elements, or it is
4744	a CONSTRUCTOR of scalar elements, where we as an exception allow
4745	smaller number of elements (assuming zero filling) and
4746	consecutive indexes as compared to NULL indexes (such
4747	CONSTRUCTORs can appear in the IL from FEs). */
4748	FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (rhs1), i, elt_i, elt_v)
4749	{
4750	if (elt_t == NULL_TREE)
4751	{
4752	elt_t = TREE_TYPE (elt_v);
4753	if (VECTOR_TYPE_P (elt_t))
4754	{
4755	tree elt_t = TREE_TYPE (elt_v);
4756	if (!useless_type_conversion_p (TREE_TYPE (rhs1_type),
4757	TREE_TYPE (elt_t)))
4758	{
4759	error ("incorrect type of vector %qs elements",
4760	code_name);
4761	debug_generic_stmt (rhs1);
4762	return true;
4763	}
4764	else if (maybe_ne (CONSTRUCTOR_NELTS (rhs1)
4765	* TYPE_VECTOR_SUBPARTS (node: elt_t),
4766	b: TYPE_VECTOR_SUBPARTS (node: rhs1_type)))
4767	{
4768	error ("incorrect number of vector %qs elements",
4769	code_name);
4770	debug_generic_stmt (rhs1);
4771	return true;
4772	}
4773	}
4774	else if (!useless_type_conversion_p (TREE_TYPE (rhs1_type),
4775	elt_t))
4776	{
4777	error ("incorrect type of vector %qs elements",
4778	code_name);
4779	debug_generic_stmt (rhs1);
4780	return true;
4781	}
4782	else if (maybe_gt (CONSTRUCTOR_NELTS (rhs1),
4783	TYPE_VECTOR_SUBPARTS (rhs1_type)))
4784	{
4785	error ("incorrect number of vector %qs elements",
4786	code_name);
4787	debug_generic_stmt (rhs1);
4788	return true;
4789	}
4790	}
4791	else if (!useless_type_conversion_p (elt_t, TREE_TYPE (elt_v)))
4792	{
4793	error ("incorrect type of vector CONSTRUCTOR elements");
4794	debug_generic_stmt (rhs1);
4795	return true;
4796	}
4797	if (elt_i != NULL_TREE
4798	&& (VECTOR_TYPE_P (elt_t)
4799	|| TREE_CODE (elt_i) != INTEGER_CST
4800	|| compare_tree_int (elt_i, i) != 0))
4801	{
4802	error ("vector %qs with non-NULL element index",
4803	code_name);
4804	debug_generic_stmt (rhs1);
4805	return true;
4806	}
4807	if (!is_gimple_val (elt_v))
4808	{
4809	error ("vector %qs element is not a GIMPLE value",
4810	code_name);
4811	debug_generic_stmt (rhs1);
4812	return true;
4813	}
4814	}
4815	}
4816	else if (CONSTRUCTOR_NELTS (rhs1) != 0)
4817	{
4818	error ("non-vector %qs with elements", code_name);
4819	debug_generic_stmt (rhs1);
4820	return true;
4821	}
4822	return res;
4823
4824	case OBJ_TYPE_REF:
4825	/* FIXME. */
4826	return res;
4827
4828	default:;
4829	}
4830
4831	return res;
4832	}
4833
4834	/* Verify the contents of a GIMPLE_ASSIGN STMT. Returns true when there
4835	is a problem, otherwise false. */
4836
4837	static bool
4838	verify_gimple_assign (gassign *stmt)
4839	{
4840	switch (gimple_assign_rhs_class (gs: stmt))
4841	{
4842	case GIMPLE_SINGLE_RHS:
4843	return verify_gimple_assign_single (stmt);
4844
4845	case GIMPLE_UNARY_RHS:
4846	return verify_gimple_assign_unary (stmt);
4847
4848	case GIMPLE_BINARY_RHS:
4849	return verify_gimple_assign_binary (stmt);
4850
4851	case GIMPLE_TERNARY_RHS:
4852	return verify_gimple_assign_ternary (stmt);
4853
4854	default:
4855	gcc_unreachable ();
4856	}
4857	}
4858
4859	/* Verify the contents of a GIMPLE_RETURN STMT. Returns true when there
4860	is a problem, otherwise false. */
4861
4862	static bool
4863	verify_gimple_return (greturn *stmt)
4864	{
4865	tree op = gimple_return_retval (gs: stmt);
4866	tree restype = TREE_TYPE (TREE_TYPE (cfun->decl));
4867
4868	/* We cannot test for present return values as we do not fix up missing
4869	return values from the original source. */
4870	if (op == NULL)
4871	return false;
4872
4873	if (!is_gimple_val (op)
4874	&& TREE_CODE (op) != RESULT_DECL)
4875	{
4876	error ("invalid operand in return statement");
4877	debug_generic_stmt (op);
4878	return true;
4879	}
4880
4881	if ((TREE_CODE (op) == RESULT_DECL
4882	&& DECL_BY_REFERENCE (op))
4883	|| (TREE_CODE (op) == SSA_NAME
4884	&& SSA_NAME_VAR (op)
4885	&& TREE_CODE (SSA_NAME_VAR (op)) == RESULT_DECL
4886	&& DECL_BY_REFERENCE (SSA_NAME_VAR (op))))
4887	op = TREE_TYPE (op);
4888
4889	if (!useless_type_conversion_p (restype, TREE_TYPE (op)))
4890	{
4891	error ("invalid conversion in return statement");
4892	debug_generic_stmt (restype);
4893	debug_generic_stmt (TREE_TYPE (op));
4894	return true;
4895	}
4896
4897	return false;
4898	}
4899
4900
4901	/* Verify the contents of a GIMPLE_GOTO STMT. Returns true when there
4902	is a problem, otherwise false. */
4903
4904	static bool
4905	verify_gimple_goto (ggoto *stmt)
4906	{
4907	tree dest = gimple_goto_dest (gs: stmt);
4908
4909	/* ??? We have two canonical forms of direct goto destinations, a
4910	bare LABEL_DECL and an ADDR_EXPR of a LABEL_DECL. */
4911	if (TREE_CODE (dest) != LABEL_DECL
4912	&& (!is_gimple_val (dest)
4913	|| !POINTER_TYPE_P (TREE_TYPE (dest))))
4914	{
4915	error ("goto destination is neither a label nor a pointer");
4916	return true;
4917	}
4918
4919	return false;
4920	}
4921
4922	/* Verify the contents of a GIMPLE_SWITCH STMT. Returns true when there
4923	is a problem, otherwise false. */
4924
4925	static bool
4926	verify_gimple_switch (gswitch *stmt)
4927	{
4928	unsigned int i, n;
4929	tree elt, prev_upper_bound = NULL_TREE;
4930	tree index_type, elt_type = NULL_TREE;
4931
4932	if (!is_gimple_val (gimple_switch_index (gs: stmt)))
4933	{
4934	error ("invalid operand to switch statement");
4935	debug_generic_stmt (gimple_switch_index (gs: stmt));
4936	return true;
4937	}
4938
4939	index_type = TREE_TYPE (gimple_switch_index (stmt));
4940	if (! INTEGRAL_TYPE_P (index_type))
4941	{
4942	error ("non-integral type switch statement");
4943	debug_generic_expr (index_type);
4944	return true;
4945	}
4946
4947	elt = gimple_switch_label (gs: stmt, index: 0);
4948	if (CASE_LOW (elt) != NULL_TREE
4949	|| CASE_HIGH (elt) != NULL_TREE
4950	|| CASE_CHAIN (elt) != NULL_TREE)
4951	{
4952	error ("invalid default case label in switch statement");
4953	debug_generic_expr (elt);
4954	return true;
4955	}
4956
4957	n = gimple_switch_num_labels (gs: stmt);
4958	for (i = 1; i < n; i++)
4959	{
4960	elt = gimple_switch_label (gs: stmt, index: i);
4961
4962	if (CASE_CHAIN (elt))
4963	{
4964	error ("invalid %<CASE_CHAIN%>");
4965	debug_generic_expr (elt);
4966	return true;
4967	}
4968	if (! CASE_LOW (elt))
4969	{
4970	error ("invalid case label in switch statement");
4971	debug_generic_expr (elt);
4972	return true;
4973	}
4974	if (CASE_HIGH (elt)
4975	&& ! tree_int_cst_lt (CASE_LOW (elt), CASE_HIGH (elt)))
4976	{
4977	error ("invalid case range in switch statement");
4978	debug_generic_expr (elt);
4979	return true;
4980	}
4981
4982	if (! elt_type)
4983	{
4984	elt_type = TREE_TYPE (CASE_LOW (elt));
4985	if (TYPE_PRECISION (index_type) < TYPE_PRECISION (elt_type))
4986	{
4987	error ("type precision mismatch in switch statement");
4988	return true;
4989	}
4990	}
4991	if (TREE_TYPE (CASE_LOW (elt)) != elt_type
4992	|| (CASE_HIGH (elt) && TREE_TYPE (CASE_HIGH (elt)) != elt_type))
4993	{
4994	error ("type mismatch for case label in switch statement");
4995	debug_generic_expr (elt);
4996	return true;
4997	}
4998
4999	if (prev_upper_bound)
5000	{
5001	if (! tree_int_cst_lt (t1: prev_upper_bound, CASE_LOW (elt)))
5002	{
5003	error ("case labels not sorted in switch statement");
5004	return true;
5005	}
5006	}
5007
5008	prev_upper_bound = CASE_HIGH (elt);
5009	if (! prev_upper_bound)
5010	prev_upper_bound = CASE_LOW (elt);
5011	}
5012
5013	return false;
5014	}
5015
5016	/* Verify a gimple debug statement STMT.
5017	Returns true if anything is wrong. */
5018
5019	static bool
5020	verify_gimple_debug (gimple *stmt ATTRIBUTE_UNUSED)
5021	{
5022	/* There isn't much that could be wrong in a gimple debug stmt. A
5023	gimple debug bind stmt, for example, maps a tree, that's usually
5024	a VAR_DECL or a PARM_DECL, but that could also be some scalarized
5025	component or member of an aggregate type, to another tree, that
5026	can be an arbitrary expression. These stmts expand into debug
5027	insns, and are converted to debug notes by var-tracking.cc. */
5028	return false;
5029	}
5030
5031	/* Verify a gimple label statement STMT.
5032	Returns true if anything is wrong. */
5033
5034	static bool
5035	verify_gimple_label (glabel *stmt)
5036	{
5037	tree decl = gimple_label_label (gs: stmt);
5038	int uid;
5039	bool err = false;
5040
5041	if (TREE_CODE (decl) != LABEL_DECL)
5042	return true;
5043	if (!DECL_NONLOCAL (decl) && !FORCED_LABEL (decl)
5044	&& DECL_CONTEXT (decl) != current_function_decl)
5045	{
5046	error ("label context is not the current function declaration");
5047	err |= true;
5048	}
5049
5050	uid = LABEL_DECL_UID (decl);
5051	if (cfun->cfg
5052	&& (uid == -1
5053	|| (*label_to_block_map_for_fn (cfun))[uid] != gimple_bb (g: stmt)))
5054	{
5055	error ("incorrect entry in %<label_to_block_map%>");
5056	err |= true;
5057	}
5058
5059	uid = EH_LANDING_PAD_NR (decl);
5060	if (uid)
5061	{
5062	eh_landing_pad lp = get_eh_landing_pad_from_number (uid);
5063	if (decl != lp->post_landing_pad)
5064	{
5065	error ("incorrect setting of landing pad number");
5066	err |= true;
5067	}
5068	}
5069
5070	return err;
5071	}
5072
5073	/* Verify a gimple cond statement STMT.
5074	Returns true if anything is wrong. */
5075
5076	static bool
5077	verify_gimple_cond (gcond *stmt)
5078	{
5079	if (TREE_CODE_CLASS (gimple_cond_code (stmt)) != tcc_comparison)
5080	{
5081	error ("invalid comparison code in gimple cond");
5082	return true;
5083	}
5084	if (!(!gimple_cond_true_label (gs: stmt)
5085	|| TREE_CODE (gimple_cond_true_label (stmt)) == LABEL_DECL)
5086	|| !(!gimple_cond_false_label (gs: stmt)
5087	|| TREE_CODE (gimple_cond_false_label (stmt)) == LABEL_DECL))
5088	{
5089	error ("invalid labels in gimple cond");
5090	return true;
5091	}
5092
5093	return verify_gimple_comparison (boolean_type_node,
5094	op0: gimple_cond_lhs (gs: stmt),
5095	op1: gimple_cond_rhs (gs: stmt),
5096	code: gimple_cond_code (gs: stmt));
5097	}
5098
5099	/* Verify the GIMPLE statement STMT. Returns true if there is an
5100	error, otherwise false. */
5101
5102	static bool
5103	verify_gimple_stmt (gimple *stmt)
5104	{
5105	switch (gimple_code (g: stmt))
5106	{
5107	case GIMPLE_ASSIGN:
5108	return verify_gimple_assign (stmt: as_a <gassign *> (p: stmt));
5109
5110	case GIMPLE_LABEL:
5111	return verify_gimple_label (stmt: as_a <glabel *> (p: stmt));
5112
5113	case GIMPLE_CALL:
5114	return verify_gimple_call (stmt: as_a <gcall *> (p: stmt));
5115
5116	case GIMPLE_COND:
5117	return verify_gimple_cond (stmt: as_a <gcond *> (p: stmt));
5118
5119	case GIMPLE_GOTO:
5120	return verify_gimple_goto (stmt: as_a <ggoto *> (p: stmt));
5121
5122	case GIMPLE_SWITCH:
5123	return verify_gimple_switch (stmt: as_a <gswitch *> (p: stmt));
5124
5125	case GIMPLE_RETURN:
5126	return verify_gimple_return (stmt: as_a <greturn *> (p: stmt));
5127
5128	case GIMPLE_ASM:
5129	return false;
5130
5131	case GIMPLE_TRANSACTION:
5132	return verify_gimple_transaction (as_a <gtransaction *> (p: stmt));
5133
5134	/* Tuples that do not have tree operands. */
5135	case GIMPLE_NOP:
5136	case GIMPLE_PREDICT:
5137	case GIMPLE_RESX:
5138	case GIMPLE_EH_DISPATCH:
5139	case GIMPLE_EH_MUST_NOT_THROW:
5140	return false;
5141
5142	CASE_GIMPLE_OMP:
5143	/* OpenMP directives are validated by the FE and never operated
5144	on by the optimizers. Furthermore, GIMPLE_OMP_FOR may contain
5145	non-gimple expressions when the main index variable has had
5146	its address taken. This does not affect the loop itself
5147	because the header of an GIMPLE_OMP_FOR is merely used to determine
5148	how to setup the parallel iteration. */
5149	return false;
5150
5151	case GIMPLE_ASSUME:
5152	return false;
5153
5154	case GIMPLE_DEBUG:
5155	return verify_gimple_debug (stmt);
5156
5157	default:
5158	gcc_unreachable ();
5159	}
5160	}
5161
5162	/* Verify the contents of a GIMPLE_PHI. Returns true if there is a problem,
5163	and false otherwise. */
5164
5165	static bool
5166	verify_gimple_phi (gphi *phi)
5167	{
5168	bool err = false;
5169	unsigned i;
5170	tree phi_result = gimple_phi_result (gs: phi);
5171	bool virtual_p;
5172
5173	if (!phi_result)
5174	{
5175	error ("invalid %<PHI%> result");
5176	return true;
5177	}
5178
5179	virtual_p = virtual_operand_p (op: phi_result);
5180	if (TREE_CODE (phi_result) != SSA_NAME
5181	|| (virtual_p
5182	&& SSA_NAME_VAR (phi_result) != gimple_vop (cfun)))
5183	{
5184	error ("invalid %<PHI%> result");
5185	err = true;
5186	}
5187
5188	for (i = 0; i < gimple_phi_num_args (gs: phi); i++)
5189	{
5190	tree t = gimple_phi_arg_def (gs: phi, index: i);
5191
5192	if (!t)
5193	{
5194	error ("missing %<PHI%> def");
5195	err |= true;
5196	continue;
5197	}
5198	/* Addressable variables do have SSA_NAMEs but they
5199	are not considered gimple values. */
5200	else if ((TREE_CODE (t) == SSA_NAME
5201	&& virtual_p != virtual_operand_p (op: t))
5202	|| (virtual_p
5203	&& (TREE_CODE (t) != SSA_NAME
5204	|| SSA_NAME_VAR (t) != gimple_vop (cfun)))
5205	|| (!virtual_p
5206	&& !is_gimple_val (t)))
5207	{
5208	error ("invalid %<PHI%> argument");
5209	debug_generic_expr (t);
5210	err |= true;
5211	}
5212	#ifdef ENABLE_TYPES_CHECKING
5213	if (!useless_type_conversion_p (TREE_TYPE (phi_result), TREE_TYPE (t)))
5214	{
5215	error ("incompatible types in %<PHI%> argument %u", i);
5216	debug_generic_stmt (TREE_TYPE (phi_result));
5217	debug_generic_stmt (TREE_TYPE (t));
5218	err |= true;
5219	}
5220	#endif
5221	}
5222
5223	return err;
5224	}
5225
5226	/* Verify the GIMPLE statements inside the sequence STMTS. */
5227
5228	static bool
5229	verify_gimple_in_seq_2 (gimple_seq stmts)
5230	{
5231	gimple_stmt_iterator ittr;
5232	bool err = false;
5233
5234	for (ittr = gsi_start (seq&: stmts); !gsi_end_p (i: ittr); gsi_next (i: &ittr))
5235	{
5236	gimple *stmt = gsi_stmt (i: ittr);
5237
5238	switch (gimple_code (g: stmt))
5239	{
5240	case GIMPLE_BIND:
5241	err |= verify_gimple_in_seq_2 (
5242	stmts: gimple_bind_body (gs: as_a <gbind *> (p: stmt)));
5243	break;
5244
5245	case GIMPLE_TRY:
5246	err |= verify_gimple_in_seq_2 (stmts: gimple_try_eval (gs: stmt));
5247	err |= verify_gimple_in_seq_2 (stmts: gimple_try_cleanup (gs: stmt));
5248	break;
5249
5250	case GIMPLE_EH_FILTER:
5251	err |= verify_gimple_in_seq_2 (stmts: gimple_eh_filter_failure (gs: stmt));
5252	break;
5253
5254	case GIMPLE_EH_ELSE:
5255	{
5256	geh_else *eh_else = as_a <geh_else *> (p: stmt);
5257	err |= verify_gimple_in_seq_2 (stmts: gimple_eh_else_n_body (eh_else_stmt: eh_else));
5258	err |= verify_gimple_in_seq_2 (stmts: gimple_eh_else_e_body (eh_else_stmt: eh_else));
5259	}
5260	break;
5261
5262	case GIMPLE_CATCH:
5263	err |= verify_gimple_in_seq_2 (stmts: gimple_catch_handler (
5264	catch_stmt: as_a <gcatch *> (p: stmt)));
5265	break;
5266
5267	case GIMPLE_ASSUME:
5268	err |= verify_gimple_in_seq_2 (stmts: gimple_assume_body (gs: stmt));
5269	break;
5270
5271	case GIMPLE_TRANSACTION:
5272	err |= verify_gimple_transaction (as_a <gtransaction *> (p: stmt));
5273	break;
5274
5275	default:
5276	{
5277	bool err2 = verify_gimple_stmt (stmt);
5278	if (err2)
5279	debug_gimple_stmt (stmt);
5280	err |= err2;
5281	}
5282	}
5283	}
5284
5285	return err;
5286	}
5287
5288	/* Verify the contents of a GIMPLE_TRANSACTION. Returns true if there
5289	is a problem, otherwise false. */
5290
5291	static bool
5292	verify_gimple_transaction (gtransaction *stmt)
5293	{
5294	tree lab;
5295
5296	lab = gimple_transaction_label_norm (transaction_stmt: stmt);
5297	if (lab != NULL && TREE_CODE (lab) != LABEL_DECL)
5298	return true;
5299	lab = gimple_transaction_label_uninst (transaction_stmt: stmt);
5300	if (lab != NULL && TREE_CODE (lab) != LABEL_DECL)
5301	return true;
5302	lab = gimple_transaction_label_over (transaction_stmt: stmt);
5303	if (lab != NULL && TREE_CODE (lab) != LABEL_DECL)
5304	return true;
5305
5306	return verify_gimple_in_seq_2 (stmts: gimple_transaction_body (transaction_stmt: stmt));
5307	}
5308
5309
5310	/* Verify the GIMPLE statements inside the statement list STMTS. */
5311
5312	DEBUG_FUNCTION bool
5313	verify_gimple_in_seq (gimple_seq stmts, bool ice)
5314	{
5315	timevar_push (tv: TV_TREE_STMT_VERIFY);
5316	bool res = verify_gimple_in_seq_2 (stmts);
5317	if (res && ice)
5318	internal_error ("%<verify_gimple%> failed");
5319	timevar_pop (tv: TV_TREE_STMT_VERIFY);
5320	return res;
5321	}
5322
5323	/* Return true when the T can be shared. */
5324
5325	static bool
5326	tree_node_can_be_shared (tree t)
5327	{
5328	if (IS_TYPE_OR_DECL_P (t)
5329	|| TREE_CODE (t) == SSA_NAME
5330	|| TREE_CODE (t) == IDENTIFIER_NODE
5331	|| TREE_CODE (t) == CASE_LABEL_EXPR
5332	|| is_gimple_min_invariant (t))
5333	return true;
5334
5335	if (t == error_mark_node)
5336	return true;
5337
5338	return false;
5339	}
5340
5341	/* Called via walk_tree. Verify tree sharing. */
5342
5343	static tree
5344	verify_node_sharing_1 (tree *tp, int *walk_subtrees, void *data)
5345	{
5346	hash_set<void *> *visited = (hash_set<void *> *) data;
5347
5348	if (tree_node_can_be_shared (t: *tp))
5349	{
5350	*walk_subtrees = false;
5351	return NULL;
5352	}
5353
5354	if (visited->add (k: *tp))
5355	return *tp;
5356
5357	return NULL;
5358	}
5359
5360	/* Called via walk_gimple_stmt. Verify tree sharing. */
5361
5362	static tree
5363	verify_node_sharing (tree *tp, int *walk_subtrees, void *data)
5364	{
5365	struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
5366	return verify_node_sharing_1 (tp, walk_subtrees, data: wi->info);
5367	}
5368
5369	static bool eh_error_found;
5370	bool
5371	verify_eh_throw_stmt_node (gimple *const &stmt, const int &,
5372	hash_set<gimple *> *visited)
5373	{
5374	if (!visited->contains (k: stmt))
5375	{
5376	error ("dead statement in EH table");
5377	debug_gimple_stmt (stmt);
5378	eh_error_found = true;
5379	}
5380	return true;
5381	}
5382
5383	/* Verify if the location LOCs block is in BLOCKS. */
5384
5385	static bool
5386	verify_location (hash_set<tree> *blocks, location_t loc)
5387	{
5388	tree block = LOCATION_BLOCK (loc);
5389	if (block != NULL_TREE
5390	&& !blocks->contains (k: block))
5391	{
5392	error ("location references block not in block tree");
5393	return true;
5394	}
5395	if (block != NULL_TREE)
5396	return verify_location (blocks, BLOCK_SOURCE_LOCATION (block));
5397	return false;
5398	}
5399
5400	/* Called via walk_tree. Verify that expressions have no blocks. */
5401
5402	static tree
5403	verify_expr_no_block (tree *tp, int *walk_subtrees, void *)
5404	{
5405	if (!EXPR_P (*tp))
5406	{
5407	*walk_subtrees = false;
5408	return NULL;
5409	}
5410
5411	location_t loc = EXPR_LOCATION (*tp);
5412	if (LOCATION_BLOCK (loc) != NULL)
5413	return *tp;
5414
5415	return NULL;
5416	}
5417
5418	/* Called via walk_tree. Verify locations of expressions. */
5419
5420	static tree
5421	verify_expr_location_1 (tree *tp, int *walk_subtrees, void *data)
5422	{
5423	hash_set<tree> *blocks = (hash_set<tree> *) data;
5424	tree t = *tp;
5425
5426	/* ??? This doesn't really belong here but there's no good place to
5427	stick this remainder of old verify_expr. */
5428	/* ??? This barfs on debug stmts which contain binds to vars with
5429	different function context. */
5430	#if 0
5431	if (VAR_P (t)
5432	|| TREE_CODE (t) == PARM_DECL
5433	|| TREE_CODE (t) == RESULT_DECL)
5434	{
5435	tree context = decl_function_context (t);
5436	if (context != cfun->decl
5437	&& !SCOPE_FILE_SCOPE_P (context)
5438	&& !TREE_STATIC (t)
5439	&& !DECL_EXTERNAL (t))
5440	{
5441	error ("local declaration from a different function");
5442	return t;
5443	}
5444	}
5445	#endif
5446
5447	if (VAR_P (t) && DECL_HAS_DEBUG_EXPR_P (t))
5448	{
5449	tree x = DECL_DEBUG_EXPR (t);
5450	tree addr = walk_tree (&x, verify_expr_no_block, NULL, NULL);
5451	if (addr)
5452	return addr;
5453	}
5454	if ((VAR_P (t)
5455	|| TREE_CODE (t) == PARM_DECL
5456	|| TREE_CODE (t) == RESULT_DECL)
5457	&& DECL_HAS_VALUE_EXPR_P (t))
5458	{
5459	tree x = DECL_VALUE_EXPR (t);
5460	tree addr = walk_tree (&x, verify_expr_no_block, NULL, NULL);
5461	if (addr)
5462	return addr;
5463	}
5464
5465	if (!EXPR_P (t))
5466	{
5467	*walk_subtrees = false;
5468	return NULL;
5469	}
5470
5471	location_t loc = EXPR_LOCATION (t);
5472	if (verify_location (blocks, loc))
5473	return t;
5474
5475	return NULL;
5476	}
5477
5478	/* Called via walk_gimple_op. Verify locations of expressions. */
5479
5480	static tree
5481	verify_expr_location (tree *tp, int *walk_subtrees, void *data)
5482	{
5483	struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
5484	return verify_expr_location_1 (tp, walk_subtrees, data: wi->info);
5485	}
5486
5487	/* Insert all subblocks of BLOCK into BLOCKS and recurse. */
5488
5489	static void
5490	collect_subblocks (hash_set<tree> *blocks, tree block)
5491	{
5492	tree t;
5493	for (t = BLOCK_SUBBLOCKS (block); t; t = BLOCK_CHAIN (t))
5494	{
5495	blocks->add (k: t);
5496	collect_subblocks (blocks, block: t);
5497	}
5498	}
5499
5500	/* Disable warnings about missing quoting in GCC diagnostics for
5501	the verification errors. Their format strings don't follow
5502	GCC diagnostic conventions and trigger an ICE in the end. */
5503	#if __GNUC__ >= 10
5504	# pragma GCC diagnostic push
5505	# pragma GCC diagnostic ignored "-Wformat-diag"
5506	#endif
5507
5508	/* Verify the GIMPLE statements in the CFG of FN. */
5509
5510	DEBUG_FUNCTION bool
5511	verify_gimple_in_cfg (struct function *fn, bool verify_nothrow, bool ice)
5512	{
5513	basic_block bb;
5514	bool err = false;
5515
5516	timevar_push (tv: TV_TREE_STMT_VERIFY);
5517	hash_set<void *> visited;
5518	hash_set<gimple *> visited_throwing_stmts;
5519
5520	/* Collect all BLOCKs referenced by the BLOCK tree of FN. */
5521	hash_set<tree> blocks;
5522	if (DECL_INITIAL (fn->decl))
5523	{
5524	blocks.add (DECL_INITIAL (fn->decl));
5525	collect_subblocks (blocks: &blocks, DECL_INITIAL (fn->decl));
5526	}
5527
5528	FOR_EACH_BB_FN (bb, fn)
5529	{
5530	gimple_stmt_iterator gsi;
5531	edge_iterator ei;
5532	edge e;
5533
5534	for (gphi_iterator gpi = gsi_start_phis (bb);
5535	!gsi_end_p (i: gpi);
5536	gsi_next (i: &gpi))
5537	{
5538	gphi *phi = gpi.phi ();
5539	bool err2 = false;
5540	unsigned i;
5541
5542	if (gimple_bb (g: phi) != bb)
5543	{
5544	error ("gimple_bb (phi) is set to a wrong basic block");
5545	err2 = true;
5546	}
5547
5548	err2 |= verify_gimple_phi (phi);
5549
5550	/* Only PHI arguments have locations. */
5551	if (gimple_location (g: phi) != UNKNOWN_LOCATION)
5552	{
5553	error ("PHI node with location");
5554	err2 = true;
5555	}
5556
5557	for (i = 0; i < gimple_phi_num_args (gs: phi); i++)
5558	{
5559	tree arg = gimple_phi_arg_def (gs: phi, index: i);
5560	tree addr = walk_tree (&arg, verify_node_sharing_1,
5561	&visited, NULL);
5562	if (addr)
5563	{
5564	error ("incorrect sharing of tree nodes");
5565	debug_generic_expr (addr);
5566	err2 |= true;
5567	}
5568	location_t loc = gimple_phi_arg_location (phi, i);
5569	if (virtual_operand_p (op: gimple_phi_result (gs: phi))
5570	&& loc != UNKNOWN_LOCATION)
5571	{
5572	error ("virtual PHI with argument locations");
5573	err2 = true;
5574	}
5575	addr = walk_tree (&arg, verify_expr_location_1, &blocks, NULL);
5576	if (addr)
5577	{
5578	debug_generic_expr (addr);
5579	err2 = true;
5580	}
5581	err2 |= verify_location (blocks: &blocks, loc);
5582	}
5583
5584	if (err2)
5585	debug_gimple_stmt (phi);
5586	err |= err2;
5587	}
5588
5589	for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
5590	{
5591	gimple *stmt = gsi_stmt (i: gsi);
5592	bool err2 = false;
5593	struct walk_stmt_info wi;
5594	tree addr;
5595	int lp_nr;
5596
5597	if (gimple_bb (g: stmt) != bb)
5598	{
5599	error ("gimple_bb (stmt) is set to a wrong basic block");
5600	err2 = true;
5601	}
5602
5603	err2 |= verify_gimple_stmt (stmt);
5604	err2 |= verify_location (blocks: &blocks, loc: gimple_location (g: stmt));
5605
5606	memset (s: &wi, c: 0, n: sizeof (wi));
5607	wi.info = (void *) &visited;
5608	addr = walk_gimple_op (stmt, verify_node_sharing, &wi);
5609	if (addr)
5610	{
5611	error ("incorrect sharing of tree nodes");
5612	debug_generic_expr (addr);
5613	err2 |= true;
5614	}
5615
5616	memset (s: &wi, c: 0, n: sizeof (wi));
5617	wi.info = (void *) &blocks;
5618	addr = walk_gimple_op (stmt, verify_expr_location, &wi);
5619	if (addr)
5620	{
5621	debug_generic_expr (addr);
5622	err2 |= true;
5623	}
5624
5625	/* If the statement is marked as part of an EH region, then it is
5626	expected that the statement could throw. Verify that when we
5627	have optimizations that simplify statements such that we prove
5628	that they cannot throw, that we update other data structures
5629	to match. */
5630	lp_nr = lookup_stmt_eh_lp (stmt);
5631	if (lp_nr != 0)
5632	visited_throwing_stmts.add (k: stmt);
5633	if (lp_nr > 0)
5634	{
5635	if (!stmt_could_throw_p (cfun, stmt))
5636	{
5637	if (verify_nothrow)
5638	{
5639	error ("statement marked for throw, but doesn%'t");
5640	err2 |= true;
5641	}
5642	}
5643	else if (!gsi_one_before_end_p (i: gsi))
5644	{
5645	error ("statement marked for throw in middle of block");
5646	err2 |= true;
5647	}
5648	}
5649
5650	if (err2)
5651	debug_gimple_stmt (stmt);
5652	err |= err2;
5653	}
5654
5655	FOR_EACH_EDGE (e, ei, bb->succs)
5656	if (e->goto_locus != UNKNOWN_LOCATION)
5657	err |= verify_location (blocks: &blocks, loc: e->goto_locus);
5658	}
5659
5660	hash_map<gimple *, int> *eh_table = get_eh_throw_stmt_table (cfun);
5661	eh_error_found = false;
5662	if (eh_table)
5663	eh_table->traverse<hash_set<gimple *> *, verify_eh_throw_stmt_node>
5664	(a: &visited_throwing_stmts);
5665
5666	if (ice && (err || eh_error_found))
5667	internal_error ("verify_gimple failed");
5668
5669	verify_histograms ();
5670	timevar_pop (tv: TV_TREE_STMT_VERIFY);
5671
5672	return (err || eh_error_found);
5673	}
5674
5675
5676	/* Verifies that the flow information is OK. */
5677
5678	static bool
5679	gimple_verify_flow_info (void)
5680	{
5681	bool err = false;
5682	basic_block bb;
5683	gimple_stmt_iterator gsi;
5684	gimple *stmt;
5685	edge e;
5686	edge_iterator ei;
5687
5688	if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->il.gimple.seq
5689	|| ENTRY_BLOCK_PTR_FOR_FN (cfun)->il.gimple.phi_nodes)
5690	{
5691	error ("ENTRY_BLOCK has IL associated with it");
5692	err = true;
5693	}
5694
5695	if (EXIT_BLOCK_PTR_FOR_FN (cfun)->il.gimple.seq
5696	|| EXIT_BLOCK_PTR_FOR_FN (cfun)->il.gimple.phi_nodes)
5697	{
5698	error ("EXIT_BLOCK has IL associated with it");
5699	err = true;
5700	}
5701
5702	FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
5703	if (e->flags & EDGE_FALLTHRU)
5704	{
5705	error ("fallthru to exit from bb %d", e->src->index);
5706	err = true;
5707	}
5708	if (cfun->cfg->full_profile
5709	&& !ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.initialized_p ())
5710	{
5711	error ("entry block count not initialized");
5712	err = true;
5713	}
5714	if (cfun->cfg->full_profile
5715	&& !EXIT_BLOCK_PTR_FOR_FN (cfun)->count.initialized_p ())
5716	{
5717	error ("exit block count not initialized");
5718	err = true;
5719	}
5720	if (cfun->cfg->full_profile
5721	&& !single_succ_edge
5722	(ENTRY_BLOCK_PTR_FOR_FN (cfun))->probability.initialized_p ())
5723	{
5724	error ("probability of edge from entry block not initialized");
5725	err = true;
5726	}
5727
5728
5729	FOR_EACH_BB_FN (bb, cfun)
5730	{
5731	bool found_ctrl_stmt = false;
5732
5733	stmt = NULL;
5734
5735	if (cfun->cfg->full_profile)
5736	{
5737	if (!bb->count.initialized_p ())
5738	{
5739	error ("count of bb %d not initialized", bb->index);
5740	err = true;
5741	}
5742	FOR_EACH_EDGE (e, ei, bb->succs)
5743	if (!e->probability.initialized_p ())
5744	{
5745	error ("probability of edge %d->%d not initialized",
5746	bb->index, e->dest->index);
5747	err = true;
5748	}
5749	}
5750
5751	/* Skip labels on the start of basic block. */
5752	for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
5753	{
5754	tree label;
5755	gimple *prev_stmt = stmt;
5756
5757	stmt = gsi_stmt (i: gsi);
5758
5759	if (gimple_code (g: stmt) != GIMPLE_LABEL)
5760	break;
5761
5762	label = gimple_label_label (gs: as_a <glabel *> (p: stmt));
5763	if (prev_stmt && DECL_NONLOCAL (label))
5764	{
5765	error ("nonlocal label %qD is not first in a sequence "
5766	"of labels in bb %d", label, bb->index);
5767	err = true;
5768	}
5769
5770	if (prev_stmt && EH_LANDING_PAD_NR (label) != 0)
5771	{
5772	error ("EH landing pad label %qD is not first in a sequence "
5773	"of labels in bb %d", label, bb->index);
5774	err = true;
5775	}
5776
5777	if (label_to_block (cfun, dest: label) != bb)
5778	{
5779	error ("label %qD to block does not match in bb %d",
5780	label, bb->index);
5781	err = true;
5782	}
5783
5784	if (decl_function_context (label) != current_function_decl)
5785	{
5786	error ("label %qD has incorrect context in bb %d",
5787	label, bb->index);
5788	err = true;
5789	}
5790	}
5791
5792	/* Verify that body of basic block BB is free of control flow. */
5793	bool seen_nondebug_stmt = false;
5794	for (; !gsi_end_p (i: gsi); gsi_next (i: &gsi))
5795	{
5796	gimple *stmt = gsi_stmt (i: gsi);
5797
5798	/* Do NOT disregard debug stmts after found_ctrl_stmt. */
5799	if (found_ctrl_stmt)
5800	{
5801	error ("control flow in the middle of basic block %d",
5802	bb->index);
5803	err = true;
5804	}
5805
5806	if (stmt_ends_bb_p (t: stmt))
5807	found_ctrl_stmt = true;
5808
5809	if (glabel *label_stmt = dyn_cast <glabel *> (p: stmt))
5810	{
5811	error ("label %qD in the middle of basic block %d",
5812	gimple_label_label (gs: label_stmt), bb->index);
5813	err = true;
5814	}
5815
5816	/* Check that no statements appear between a returns_twice call
5817	and its associated abnormal edge. */
5818	if (gimple_code (g: stmt) == GIMPLE_CALL
5819	&& gimple_call_flags (stmt) & ECF_RETURNS_TWICE)
5820	{
5821	bool misplaced = false;
5822	/* TM is an exception: it points abnormal edges just after the
5823	call that starts a transaction, i.e. it must end the BB. */
5824	if (gimple_call_builtin_p (stmt, BUILT_IN_TM_START))
5825	{
5826	if (single_succ_p (bb)
5827	&& bb_has_abnormal_pred (bb: single_succ (bb))
5828	&& !gsi_one_nondebug_before_end_p (i: gsi))
5829	{
5830	error ("returns_twice call is not last in basic block "
5831	"%d", bb->index);
5832	misplaced = true;
5833	}
5834	}
5835	else
5836	{
5837	if (seen_nondebug_stmt && bb_has_abnormal_pred (bb))
5838	{
5839	error ("returns_twice call is not first in basic block "
5840	"%d", bb->index);
5841	misplaced = true;
5842	}
5843	}
5844	if (misplaced)
5845	{
5846	print_gimple_stmt (stderr, stmt, 0, TDF_SLIM);
5847	err = true;
5848	}
5849	}
5850	if (!is_gimple_debug (gs: stmt))
5851	seen_nondebug_stmt = true;
5852	}
5853
5854	gsi = gsi_last_nondebug_bb (bb);
5855	if (gsi_end_p (i: gsi))
5856	continue;
5857
5858	stmt = gsi_stmt (i: gsi);
5859
5860	if (gimple_code (g: stmt) == GIMPLE_LABEL)
5861	continue;
5862
5863	if (verify_eh_edges (stmt))
5864	err = true;
5865
5866	if (is_ctrl_stmt (t: stmt))
5867	{
5868	FOR_EACH_EDGE (e, ei, bb->succs)
5869	if (e->flags & EDGE_FALLTHRU)
5870	{
5871	error ("fallthru edge after a control statement in bb %d",
5872	bb->index);
5873	err = true;
5874	}
5875	}
5876
5877	if (gimple_code (g: stmt) != GIMPLE_COND)
5878	{
5879	/* Verify that there are no edges with EDGE_TRUE/FALSE_FLAG set
5880	after anything else but if statement. */
5881	FOR_EACH_EDGE (e, ei, bb->succs)
5882	if (e->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE))
5883	{
5884	error ("true/false edge after a non-GIMPLE_COND in bb %d",
5885	bb->index);
5886	err = true;
5887	}
5888	}
5889
5890	switch (gimple_code (g: stmt))
5891	{
5892	case GIMPLE_COND:
5893	{
5894	edge true_edge;
5895	edge false_edge;
5896
5897	extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
5898
5899	if (!true_edge
5900	|| !false_edge
5901	|| !(true_edge->flags & EDGE_TRUE_VALUE)
5902	|| !(false_edge->flags & EDGE_FALSE_VALUE)
5903	|| (true_edge->flags & (EDGE_FALLTHRU | EDGE_ABNORMAL))
5904	|| (false_edge->flags & (EDGE_FALLTHRU | EDGE_ABNORMAL))
5905	|| EDGE_COUNT (bb->succs) >= 3)
5906	{
5907	error ("wrong outgoing edge flags at end of bb %d",
5908	bb->index);
5909	err = true;
5910	}
5911	}
5912	break;
5913
5914	case GIMPLE_GOTO:
5915	if (simple_goto_p (t: stmt))
5916	{
5917	error ("explicit goto at end of bb %d", bb->index);
5918	err = true;
5919	}
5920	else
5921	{
5922	/* FIXME. We should double check that the labels in the
5923	destination blocks have their address taken. */
5924	FOR_EACH_EDGE (e, ei, bb->succs)
5925	if ((e->flags & (EDGE_FALLTHRU | EDGE_TRUE_VALUE
5926	| EDGE_FALSE_VALUE))
5927	|| !(e->flags & EDGE_ABNORMAL))
5928	{
5929	error ("wrong outgoing edge flags at end of bb %d",
5930	bb->index);
5931	err = true;
5932	}
5933	}
5934	break;
5935
5936	case GIMPLE_CALL:
5937	if (!gimple_call_builtin_p (stmt, BUILT_IN_RETURN))
5938	break;
5939	/* fallthru */
5940	case GIMPLE_RETURN:
5941	if (!single_succ_p (bb)
5942	|| (single_succ_edge (bb)->flags
5943	& (EDGE_FALLTHRU | EDGE_ABNORMAL
5944	| EDGE_TRUE_VALUE | EDGE_FALSE_VALUE)))
5945	{
5946	error ("wrong outgoing edge flags at end of bb %d", bb->index);
5947	err = true;
5948	}
5949	if (single_succ (bb) != EXIT_BLOCK_PTR_FOR_FN (cfun))
5950	{
5951	error ("return edge does not point to exit in bb %d",
5952	bb->index);
5953	err = true;
5954	}
5955	break;
5956
5957	case GIMPLE_SWITCH:
5958	{
5959	gswitch *switch_stmt = as_a <gswitch *> (p: stmt);
5960	tree prev;
5961	edge e;
5962	size_t i, n;
5963
5964	n = gimple_switch_num_labels (gs: switch_stmt);
5965
5966	/* Mark all the destination basic blocks. */
5967	for (i = 0; i < n; ++i)
5968	{
5969	basic_block label_bb = gimple_switch_label_bb (cfun, switch_stmt, i);
5970	gcc_assert (!label_bb->aux || label_bb->aux == (void *)1);
5971	label_bb->aux = (void *)1;
5972	}
5973
5974	/* Verify that the case labels are sorted. */
5975	prev = gimple_switch_label (gs: switch_stmt, index: 0);
5976	for (i = 1; i < n; ++i)
5977	{
5978	tree c = gimple_switch_label (gs: switch_stmt, index: i);
5979	if (!CASE_LOW (c))
5980	{
5981	error ("found default case not at the start of "
5982	"case vector");
5983	err = true;
5984	continue;
5985	}
5986	if (CASE_LOW (prev)
5987	&& !tree_int_cst_lt (CASE_LOW (prev), CASE_LOW (c)))
5988	{
5989	error ("case labels not sorted: ");
5990	print_generic_expr (stderr, prev);
5991	fprintf (stderr,format: " is greater than ");
5992	print_generic_expr (stderr, c);
5993	fprintf (stderr,format: " but comes before it.\n");
5994	err = true;
5995	}
5996	prev = c;
5997	}
5998	/* VRP will remove the default case if it can prove it will
5999	never be executed. So do not verify there always exists
6000	a default case here. */
6001
6002	FOR_EACH_EDGE (e, ei, bb->succs)
6003	{
6004	if (!e->dest->aux)
6005	{
6006	error ("extra outgoing edge %d->%d",
6007	bb->index, e->dest->index);
6008	err = true;
6009	}
6010
6011	e->dest->aux = (void *)2;
6012	if ((e->flags & (EDGE_FALLTHRU | EDGE_ABNORMAL
6013	| EDGE_TRUE_VALUE | EDGE_FALSE_VALUE)))
6014	{
6015	error ("wrong outgoing edge flags at end of bb %d",
6016	bb->index);
6017	err = true;
6018	}
6019	}
6020
6021	/* Check that we have all of them. */
6022	for (i = 0; i < n; ++i)
6023	{
6024	basic_block label_bb = gimple_switch_label_bb (cfun,
6025	switch_stmt, i);
6026
6027	if (label_bb->aux != (void *)2)
6028	{
6029	error ("missing edge %i->%i", bb->index, label_bb->index);
6030	err = true;
6031	}
6032	}
6033
6034	FOR_EACH_EDGE (e, ei, bb->succs)
6035	e->dest->aux = (void *)0;
6036	}
6037	break;
6038
6039	case GIMPLE_EH_DISPATCH:
6040	if (verify_eh_dispatch_edge (as_a <geh_dispatch *> (p: stmt)))
6041	err = true;
6042	break;
6043
6044	default:
6045	break;
6046	}
6047	}
6048
6049	if (dom_info_state (CDI_DOMINATORS) >= DOM_NO_FAST_QUERY)
6050	verify_dominators (CDI_DOMINATORS);
6051
6052	return err;
6053	}
6054
6055	#if __GNUC__ >= 10
6056	# pragma GCC diagnostic pop
6057	#endif
6058
6059	/* Updates phi nodes after creating a forwarder block joined
6060	by edge FALLTHRU. */
6061
6062	static void
6063	gimple_make_forwarder_block (edge fallthru)
6064	{
6065	edge e;
6066	edge_iterator ei;
6067	basic_block dummy, bb;
6068	tree var;
6069	gphi_iterator gsi;
6070	bool forward_location_p;
6071
6072	dummy = fallthru->src;
6073	bb = fallthru->dest;
6074
6075	if (single_pred_p (bb))
6076	return;
6077
6078	/* We can forward location info if we have only one predecessor. */
6079	forward_location_p = single_pred_p (bb: dummy);
6080
6081	/* If we redirected a branch we must create new PHI nodes at the
6082	start of BB. */
6083	for (gsi = gsi_start_phis (dummy); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
6084	{
6085	gphi *phi, *new_phi;
6086
6087	phi = gsi.phi ();
6088	var = gimple_phi_result (gs: phi);
6089	new_phi = create_phi_node (var, bb);
6090	gimple_phi_set_result (phi, result: copy_ssa_name (var, stmt: phi));
6091	add_phi_arg (new_phi, gimple_phi_result (gs: phi), fallthru,
6092	forward_location_p
6093	? gimple_phi_arg_location (phi, i: 0) : UNKNOWN_LOCATION);
6094	}
6095
6096	/* Add the arguments we have stored on edges. */
6097	FOR_EACH_EDGE (e, ei, bb->preds)
6098	{
6099	if (e == fallthru)
6100	continue;
6101
6102	flush_pending_stmts (e);
6103	}
6104	}
6105
6106
6107	/* Return a non-special label in the head of basic block BLOCK.
6108	Create one if it doesn't exist. */
6109
6110	tree
6111	gimple_block_label (basic_block bb)
6112	{
6113	gimple_stmt_iterator i, s = gsi_start_bb (bb);
6114	bool first = true;
6115	tree label;
6116	glabel *stmt;
6117
6118	for (i = s; !gsi_end_p (i); first = false, gsi_next (i: &i))
6119	{
6120	stmt = dyn_cast <glabel *> (p: gsi_stmt (i));
6121	if (!stmt)
6122	break;
6123	label = gimple_label_label (gs: stmt);
6124	if (!DECL_NONLOCAL (label))
6125	{
6126	if (!first)
6127	gsi_move_before (&i, &s);
6128	return label;
6129	}
6130	}
6131
6132	label = create_artificial_label (UNKNOWN_LOCATION);
6133	stmt = gimple_build_label (label);
6134	gsi_insert_before (&s, stmt, GSI_NEW_STMT);
6135	return label;
6136	}
6137
6138
6139	/* Attempt to perform edge redirection by replacing a possibly complex
6140	jump instruction by a goto or by removing the jump completely.
6141	This can apply only if all edges now point to the same block. The
6142	parameters and return values are equivalent to
6143	redirect_edge_and_branch. */
6144
6145	static edge
6146	gimple_try_redirect_by_replacing_jump (edge e, basic_block target)
6147	{
6148	basic_block src = e->src;
6149	gimple_stmt_iterator i;
6150	gimple *stmt;
6151
6152	/* We can replace or remove a complex jump only when we have exactly
6153	two edges. */
6154	if (EDGE_COUNT (src->succs) != 2
6155	/* Verify that all targets will be TARGET. Specifically, the
6156	edge that is not E must also go to TARGET. */
6157	|| EDGE_SUCC (src, EDGE_SUCC (src, 0) == e)->dest != target)
6158	return NULL;
6159
6160	i = gsi_last_bb (bb: src);
6161	if (gsi_end_p (i))
6162	return NULL;
6163
6164	stmt = gsi_stmt (i);
6165
6166	if (gimple_code (g: stmt) == GIMPLE_COND || gimple_code (g: stmt) == GIMPLE_SWITCH)
6167	{
6168	gsi_remove (&i, true);
6169	e = ssa_redirect_edge (e, target);
6170	e->flags = EDGE_FALLTHRU;
6171	return e;
6172	}
6173
6174	return NULL;
6175	}
6176
6177
6178	/* Redirect E to DEST. Return NULL on failure. Otherwise, return the
6179	edge representing the redirected branch. */
6180
6181	static edge
6182	gimple_redirect_edge_and_branch (edge e, basic_block dest)
6183	{
6184	basic_block bb = e->src;
6185	gimple_stmt_iterator gsi;
6186	edge ret;
6187	gimple *stmt;
6188
6189	if (e->flags & EDGE_ABNORMAL)
6190	return NULL;
6191
6192	if (e->dest == dest)
6193	return NULL;
6194
6195	if (e->flags & EDGE_EH)
6196	return redirect_eh_edge (e, dest);
6197
6198	if (e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun))
6199	{
6200	ret = gimple_try_redirect_by_replacing_jump (e, target: dest);
6201	if (ret)
6202	return ret;
6203	}
6204
6205	gsi = gsi_last_nondebug_bb (bb);
6206	stmt = gsi_end_p (i: gsi) ? NULL : gsi_stmt (i: gsi);
6207
6208	switch (stmt ? gimple_code (g: stmt) : GIMPLE_ERROR_MARK)
6209	{
6210	case GIMPLE_COND:
6211	/* For COND_EXPR, we only need to redirect the edge. */
6212	break;
6213
6214	case GIMPLE_GOTO:
6215	/* No non-abnormal edges should lead from a non-simple goto, and
6216	simple ones should be represented implicitly. */
6217	gcc_unreachable ();
6218
6219	case GIMPLE_SWITCH:
6220	{
6221	gswitch *switch_stmt = as_a <gswitch *> (p: stmt);
6222	tree label = gimple_block_label (bb: dest);
6223	tree cases = get_cases_for_edge (e, t: switch_stmt);
6224
6225	/* If we have a list of cases associated with E, then use it
6226	as it's a lot faster than walking the entire case vector. */
6227	if (cases)
6228	{
6229	edge e2 = find_edge (e->src, dest);
6230	tree last, first;
6231
6232	first = cases;
6233	while (cases)
6234	{
6235	last = cases;
6236	CASE_LABEL (cases) = label;
6237	cases = CASE_CHAIN (cases);
6238	}
6239
6240	/* If there was already an edge in the CFG, then we need
6241	to move all the cases associated with E to E2. */
6242	if (e2)
6243	{
6244	tree cases2 = get_cases_for_edge (e: e2, t: switch_stmt);
6245
6246	CASE_CHAIN (last) = CASE_CHAIN (cases2);
6247	CASE_CHAIN (cases2) = first;
6248	}
6249	bitmap_set_bit (touched_switch_bbs, gimple_bb (g: stmt)->index);
6250	}
6251	else
6252	{
6253	size_t i, n = gimple_switch_num_labels (gs: switch_stmt);
6254
6255	for (i = 0; i < n; i++)
6256	{
6257	tree elt = gimple_switch_label (gs: switch_stmt, index: i);
6258	if (label_to_block (cfun, CASE_LABEL (elt)) == e->dest)
6259	CASE_LABEL (elt) = label;
6260	}
6261	}
6262	}
6263	break;
6264
6265	case GIMPLE_ASM:
6266	{
6267	gasm *asm_stmt = as_a <gasm *> (p: stmt);
6268	int i, n = gimple_asm_nlabels (asm_stmt);
6269	tree label = NULL;
6270
6271	for (i = 0; i < n; ++i)
6272	{
6273	tree cons = gimple_asm_label_op (asm_stmt, index: i);
6274	if (label_to_block (cfun, TREE_VALUE (cons)) == e->dest)
6275	{
6276	if (!label)
6277	label = gimple_block_label (bb: dest);
6278	TREE_VALUE (cons) = label;
6279	}
6280	}
6281
6282	/* If we didn't find any label matching the former edge in the
6283	asm labels, we must be redirecting the fallthrough
6284	edge. */
6285	gcc_assert (label || (e->flags & EDGE_FALLTHRU));
6286	}
6287	break;
6288
6289	case GIMPLE_RETURN:
6290	gsi_remove (&gsi, true);
6291	e->flags |= EDGE_FALLTHRU;
6292	break;
6293
6294	case GIMPLE_OMP_RETURN:
6295	case GIMPLE_OMP_CONTINUE:
6296	case GIMPLE_OMP_SECTIONS_SWITCH:
6297	case GIMPLE_OMP_FOR:
6298	/* The edges from OMP constructs can be simply redirected. */
6299	break;
6300
6301	case GIMPLE_EH_DISPATCH:
6302	if (!(e->flags & EDGE_FALLTHRU))
6303	redirect_eh_dispatch_edge (as_a <geh_dispatch *> (p: stmt), e, dest);
6304	break;
6305
6306	case GIMPLE_TRANSACTION:
6307	if (e->flags & EDGE_TM_ABORT)
6308	gimple_transaction_set_label_over (transaction_stmt: as_a <gtransaction *> (p: stmt),
6309	label: gimple_block_label (bb: dest));
6310	else if (e->flags & EDGE_TM_UNINSTRUMENTED)
6311	gimple_transaction_set_label_uninst (transaction_stmt: as_a <gtransaction *> (p: stmt),
6312	label: gimple_block_label (bb: dest));
6313	else
6314	gimple_transaction_set_label_norm (transaction_stmt: as_a <gtransaction *> (p: stmt),
6315	label: gimple_block_label (bb: dest));
6316	break;
6317
6318	default:
6319	/* Otherwise it must be a fallthru edge, and we don't need to
6320	do anything besides redirecting it. */
6321	gcc_assert (e->flags & EDGE_FALLTHRU);
6322	break;
6323	}
6324
6325	/* Update/insert PHI nodes as necessary. */
6326
6327	/* Now update the edges in the CFG. */
6328	e = ssa_redirect_edge (e, dest);
6329
6330	return e;
6331	}
6332
6333	/* Returns true if it is possible to remove edge E by redirecting
6334	it to the destination of the other edge from E->src. */
6335
6336	static bool
6337	gimple_can_remove_branch_p (const_edge e)
6338	{
6339	if (e->flags & (EDGE_ABNORMAL | EDGE_EH))
6340	return false;
6341
6342	return true;
6343	}
6344
6345	/* Simple wrapper, as we can always redirect fallthru edges. */
6346
6347	static basic_block
6348	gimple_redirect_edge_and_branch_force (edge e, basic_block dest)
6349	{
6350	e = gimple_redirect_edge_and_branch (e, dest);
6351	gcc_assert (e);
6352
6353	return NULL;
6354	}
6355
6356
6357	/* Splits basic block BB after statement STMT (but at least after the
6358	labels). If STMT is NULL, BB is split just after the labels. */
6359
6360	static basic_block
6361	gimple_split_block (basic_block bb, void *stmt)
6362	{
6363	gimple_stmt_iterator gsi;
6364	gimple_stmt_iterator gsi_tgt;
6365	gimple_seq list;
6366	basic_block new_bb;
6367	edge e;
6368	edge_iterator ei;
6369
6370	new_bb = create_empty_bb (bb);
6371
6372	/* Redirect the outgoing edges. */
6373	new_bb->succs = bb->succs;
6374	bb->succs = NULL;
6375	FOR_EACH_EDGE (e, ei, new_bb->succs)
6376	e->src = new_bb;
6377
6378	/* Get a stmt iterator pointing to the first stmt to move. */
6379	if (!stmt || gimple_code (g: (gimple *) stmt) == GIMPLE_LABEL)
6380	gsi = gsi_after_labels (bb);
6381	else
6382	{
6383	gsi = gsi_for_stmt ((gimple *) stmt);
6384	gsi_next (i: &gsi);
6385	}
6386
6387	/* Move everything from GSI to the new basic block. */
6388	if (gsi_end_p (i: gsi))
6389	return new_bb;
6390
6391	/* Split the statement list - avoid re-creating new containers as this
6392	brings ugly quadratic memory consumption in the inliner.
6393	(We are still quadratic since we need to update stmt BB pointers,
6394	sadly.) */
6395	gsi_split_seq_before (&gsi, &list);
6396	set_bb_seq (bb: new_bb, seq: list);
6397	for (gsi_tgt = gsi_start (seq&: list);
6398	!gsi_end_p (i: gsi_tgt); gsi_next (i: &gsi_tgt))
6399	gimple_set_bb (gsi_stmt (i: gsi_tgt), new_bb);
6400
6401	return new_bb;
6402	}
6403
6404
6405	/* Moves basic block BB after block AFTER. */
6406
6407	static bool
6408	gimple_move_block_after (basic_block bb, basic_block after)
6409	{
6410	if (bb->prev_bb == after)
6411	return true;
6412
6413	unlink_block (bb);
6414	link_block (bb, after);
6415
6416	return true;
6417	}
6418
6419
6420	/* Return TRUE if block BB has no executable statements, otherwise return
6421	FALSE. */
6422
6423	static bool
6424	gimple_empty_block_p (basic_block bb)
6425	{
6426	/* BB must have no executable statements. */
6427	gimple_stmt_iterator gsi = gsi_after_labels (bb);
6428	if (phi_nodes (bb))
6429	return false;
6430	while (!gsi_end_p (i: gsi))
6431	{
6432	gimple *stmt = gsi_stmt (i: gsi);
6433	if (is_gimple_debug (gs: stmt))
6434	;
6435	else if (gimple_code (g: stmt) == GIMPLE_NOP
6436	|| gimple_code (g: stmt) == GIMPLE_PREDICT)
6437	;
6438	else
6439	return false;
6440	gsi_next (i: &gsi);
6441	}
6442	return true;
6443	}
6444
6445
6446	/* Split a basic block if it ends with a conditional branch and if the
6447	other part of the block is not empty. */
6448
6449	static basic_block
6450	gimple_split_block_before_cond_jump (basic_block bb)
6451	{
6452	gimple *last, *split_point;
6453	gimple_stmt_iterator gsi = gsi_last_nondebug_bb (bb);
6454	if (gsi_end_p (i: gsi))
6455	return NULL;
6456	last = gsi_stmt (i: gsi);
6457	if (gimple_code (g: last) != GIMPLE_COND
6458	&& gimple_code (g: last) != GIMPLE_SWITCH)
6459	return NULL;
6460	gsi_prev (i: &gsi);
6461	split_point = gsi_stmt (i: gsi);
6462	return split_block (bb, split_point)->dest;
6463	}
6464
6465
6466	/* Return true if basic_block can be duplicated. */
6467
6468	static bool
6469	gimple_can_duplicate_bb_p (const_basic_block bb)
6470	{
6471	gimple *last = last_nondebug_stmt (CONST_CAST_BB (bb));
6472
6473	/* Do checks that can only fail for the last stmt, to minimize the work in the
6474	stmt loop. */
6475	if (last) {
6476	/* A transaction is a single entry multiple exit region. It
6477	must be duplicated in its entirety or not at all. */
6478	if (gimple_code (g: last) == GIMPLE_TRANSACTION)
6479	return false;
6480
6481	/* An IFN_UNIQUE call must be duplicated as part of its group,
6482	or not at all. */
6483	if (is_gimple_call (gs: last)
6484	&& gimple_call_internal_p (gs: last)
6485	&& gimple_call_internal_unique_p (gs: last))
6486	return false;
6487	}
6488
6489	for (gimple_stmt_iterator gsi = gsi_start_bb (CONST_CAST_BB (bb));
6490	!gsi_end_p (i: gsi); gsi_next (i: &gsi))
6491	{
6492	gimple *g = gsi_stmt (i: gsi);
6493
6494	/* Prohibit duplication of returns_twice calls, otherwise associated
6495	abnormal edges also need to be duplicated properly.
6496	An IFN_GOMP_SIMT_ENTER_ALLOC/IFN_GOMP_SIMT_EXIT call must be
6497	duplicated as part of its group, or not at all.
6498	The IFN_GOMP_SIMT_VOTE_ANY and IFN_GOMP_SIMT_XCHG_* are part of such a
6499	group, so the same holds there. */
6500	if (is_gimple_call (gs: g)
6501	&& (gimple_call_flags (g) & ECF_RETURNS_TWICE
6502	|| gimple_call_internal_p (gs: g, fn: IFN_GOMP_SIMT_ENTER_ALLOC)
6503	|| gimple_call_internal_p (gs: g, fn: IFN_GOMP_SIMT_EXIT)
6504	|| gimple_call_internal_p (gs: g, fn: IFN_GOMP_SIMT_VOTE_ANY)
6505	|| gimple_call_internal_p (gs: g, fn: IFN_GOMP_SIMT_XCHG_BFLY)
6506	|| gimple_call_internal_p (gs: g, fn: IFN_GOMP_SIMT_XCHG_IDX)))
6507	return false;
6508	}
6509
6510	return true;
6511	}
6512
6513	/* Create a duplicate of the basic block BB. NOTE: This does not
6514	preserve SSA form. */
6515
6516	static basic_block
6517	gimple_duplicate_bb (basic_block bb, copy_bb_data *id)
6518	{
6519	basic_block new_bb;
6520	gimple_stmt_iterator gsi_tgt;
6521
6522	new_bb = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb);
6523
6524	/* Copy the PHI nodes. We ignore PHI node arguments here because
6525	the incoming edges have not been setup yet. */
6526	for (gphi_iterator gpi = gsi_start_phis (bb);
6527	!gsi_end_p (i: gpi);
6528	gsi_next (i: &gpi))
6529	{
6530	gphi *phi, *copy;
6531	phi = gpi.phi ();
6532	copy = create_phi_node (NULL_TREE, new_bb);
6533	create_new_def_for (gimple_phi_result (gs: phi), copy,
6534	gimple_phi_result_ptr (gs: copy));
6535	gimple_set_uid (g: copy, uid: gimple_uid (g: phi));
6536	}
6537
6538	gsi_tgt = gsi_start_bb (bb: new_bb);
6539	for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
6540	!gsi_end_p (i: gsi);
6541	gsi_next (i: &gsi))
6542	{
6543	def_operand_p def_p;
6544	ssa_op_iter op_iter;
6545	tree lhs;
6546	gimple *stmt, *copy;
6547
6548	stmt = gsi_stmt (i: gsi);
6549	if (gimple_code (g: stmt) == GIMPLE_LABEL)
6550	continue;
6551
6552	/* Don't duplicate label debug stmts. */
6553	if (gimple_debug_bind_p (s: stmt)
6554	&& TREE_CODE (gimple_debug_bind_get_var (stmt))
6555	== LABEL_DECL)
6556	continue;
6557
6558	/* Create a new copy of STMT and duplicate STMT's virtual
6559	operands. */
6560	copy = gimple_copy (stmt);
6561	gsi_insert_after (&gsi_tgt, copy, GSI_NEW_STMT);
6562
6563	maybe_duplicate_eh_stmt (copy, stmt);
6564	gimple_duplicate_stmt_histograms (cfun, copy, cfun, stmt);
6565
6566	/* When copying around a stmt writing into a local non-user
6567	aggregate, make sure it won't share stack slot with other
6568	vars. */
6569	lhs = gimple_get_lhs (stmt);
6570	if (lhs && TREE_CODE (lhs) != SSA_NAME)
6571	{
6572	tree base = get_base_address (t: lhs);
6573	if (base
6574	&& (VAR_P (base) || TREE_CODE (base) == RESULT_DECL)
6575	&& DECL_IGNORED_P (base)
6576	&& !TREE_STATIC (base)
6577	&& !DECL_EXTERNAL (base)
6578	&& (!VAR_P (base) || !DECL_HAS_VALUE_EXPR_P (base)))
6579	DECL_NONSHAREABLE (base) = 1;
6580	}
6581
6582	/* If requested remap dependence info of cliques brought in
6583	via inlining. */
6584	if (id)
6585	for (unsigned i = 0; i < gimple_num_ops (gs: copy); ++i)
6586	{
6587	tree op = gimple_op (gs: copy, i);
6588	if (!op)
6589	continue;
6590	if (TREE_CODE (op) == ADDR_EXPR
6591	|| TREE_CODE (op) == WITH_SIZE_EXPR)
6592	op = TREE_OPERAND (op, 0);
6593	while (handled_component_p (t: op))
6594	op = TREE_OPERAND (op, 0);
6595	if ((TREE_CODE (op) == MEM_REF
6596	|| TREE_CODE (op) == TARGET_MEM_REF)
6597	&& MR_DEPENDENCE_CLIQUE (op) > 1
6598	&& MR_DEPENDENCE_CLIQUE (op) != bb->loop_father->owned_clique)
6599	{
6600	if (!id->dependence_map)
6601	id->dependence_map = new hash_map<dependence_hash,
6602	unsigned short>;
6603	bool existed;
6604	unsigned short &newc = id->dependence_map->get_or_insert
6605	(MR_DEPENDENCE_CLIQUE (op), existed: &existed);
6606	if (!existed)
6607	{
6608	gcc_assert (MR_DEPENDENCE_CLIQUE (op) <= cfun->last_clique);
6609	newc = get_new_clique (cfun);
6610	}
6611	MR_DEPENDENCE_CLIQUE (op) = newc;
6612	}
6613	}
6614
6615	/* Create new names for all the definitions created by COPY and
6616	add replacement mappings for each new name. */
6617	FOR_EACH_SSA_DEF_OPERAND (def_p, copy, op_iter, SSA_OP_ALL_DEFS)
6618	create_new_def_for (DEF_FROM_PTR (def_p), copy, def_p);
6619	}
6620
6621	return new_bb;
6622	}
6623
6624	/* Adds phi node arguments for edge E_COPY after basic block duplication. */
6625
6626	static void
6627	add_phi_args_after_copy_edge (edge e_copy)
6628	{
6629	basic_block bb, bb_copy = e_copy->src, dest;
6630	edge e;
6631	edge_iterator ei;
6632	gphi *phi, *phi_copy;
6633	tree def;
6634	gphi_iterator psi, psi_copy;
6635
6636	if (gimple_seq_empty_p (s: phi_nodes (bb: e_copy->dest)))
6637	return;
6638
6639	bb = bb_copy->flags & BB_DUPLICATED ? get_bb_original (bb_copy) : bb_copy;
6640
6641	if (e_copy->dest->flags & BB_DUPLICATED)
6642	dest = get_bb_original (e_copy->dest);
6643	else
6644	dest = e_copy->dest;
6645
6646	e = find_edge (bb, dest);
6647	if (!e)
6648	{
6649	/* During loop unrolling the target of the latch edge is copied.
6650	In this case we are not looking for edge to dest, but to
6651	duplicated block whose original was dest. */
6652	FOR_EACH_EDGE (e, ei, bb->succs)
6653	{
6654	if ((e->dest->flags & BB_DUPLICATED)
6655	&& get_bb_original (e->dest) == dest)
6656	break;
6657	}
6658
6659	gcc_assert (e != NULL);
6660	}
6661
6662	for (psi = gsi_start_phis (e->dest),
6663	psi_copy = gsi_start_phis (e_copy->dest);
6664	!gsi_end_p (i: psi);
6665	gsi_next (i: &psi), gsi_next (i: &psi_copy))
6666	{
6667	phi = psi.phi ();
6668	phi_copy = psi_copy.phi ();
6669	def = PHI_ARG_DEF_FROM_EDGE (phi, e);
6670	add_phi_arg (phi_copy, def, e_copy,
6671	gimple_phi_arg_location_from_edge (phi, e));
6672	}
6673	}
6674
6675
6676	/* Basic block BB_COPY was created by code duplication. Add phi node
6677	arguments for edges going out of BB_COPY. The blocks that were
6678	duplicated have BB_DUPLICATED set. */
6679
6680	void
6681	add_phi_args_after_copy_bb (basic_block bb_copy)
6682	{
6683	edge e_copy;
6684	edge_iterator ei;
6685
6686	FOR_EACH_EDGE (e_copy, ei, bb_copy->succs)
6687	{
6688	add_phi_args_after_copy_edge (e_copy);
6689	}
6690	}
6691
6692	/* Blocks in REGION_COPY array of length N_REGION were created by
6693	duplication of basic blocks. Add phi node arguments for edges
6694	going from these blocks. If E_COPY is not NULL, also add
6695	phi node arguments for its destination.*/
6696
6697	void
6698	add_phi_args_after_copy (basic_block *region_copy, unsigned n_region,
6699	edge e_copy)
6700	{
6701	unsigned i;
6702
6703	for (i = 0; i < n_region; i++)
6704	region_copy[i]->flags |= BB_DUPLICATED;
6705
6706	for (i = 0; i < n_region; i++)
6707	add_phi_args_after_copy_bb (bb_copy: region_copy[i]);
6708	if (e_copy)
6709	add_phi_args_after_copy_edge (e_copy);
6710
6711	for (i = 0; i < n_region; i++)
6712	region_copy[i]->flags &= ~BB_DUPLICATED;
6713	}
6714
6715	/* Duplicates a REGION (set of N_REGION basic blocks) with just a single
6716	important exit edge EXIT. By important we mean that no SSA name defined
6717	inside region is live over the other exit edges of the region. All entry
6718	edges to the region must go to ENTRY->dest. The edge ENTRY is redirected
6719	to the duplicate of the region. Dominance and loop information is
6720	updated if UPDATE_DOMINANCE is true, but not the SSA web. If
6721	UPDATE_DOMINANCE is false then we assume that the caller will update the
6722	dominance information after calling this function. The new basic
6723	blocks are stored to REGION_COPY in the same order as they had in REGION,
6724	provided that REGION_COPY is not NULL.
6725	The function returns false if it is unable to copy the region,
6726	true otherwise.
6727
6728	It is callers responsibility to update profile. */
6729
6730	bool
6731	gimple_duplicate_seme_region (edge entry, edge exit,
6732	basic_block *region, unsigned n_region,
6733	basic_block *region_copy,
6734	bool update_dominance)
6735	{
6736	unsigned i;
6737	bool free_region_copy = false, copying_header = false;
6738	class loop *loop = entry->dest->loop_father;
6739	edge exit_copy;
6740	edge redirected;
6741
6742	if (!can_copy_bbs_p (region, n_region))
6743	return false;
6744
6745	/* Some sanity checking. Note that we do not check for all possible
6746	missuses of the functions. I.e. if you ask to copy something weird,
6747	it will work, but the state of structures probably will not be
6748	correct. */
6749	for (i = 0; i < n_region; i++)
6750	{
6751	/* We do not handle subloops, i.e. all the blocks must belong to the
6752	same loop. */
6753	if (region[i]->loop_father != loop)
6754	return false;
6755
6756	if (region[i] != entry->dest
6757	&& region[i] == loop->header)
6758	return false;
6759	}
6760
6761	/* In case the function is used for loop header copying (which is the primary
6762	use), ensure that EXIT and its copy will be new latch and entry edges. */
6763	if (loop->header == entry->dest)
6764	{
6765	copying_header = true;
6766
6767	if (!dominated_by_p (CDI_DOMINATORS, loop->latch, exit->src))
6768	return false;
6769
6770	for (i = 0; i < n_region; i++)
6771	if (region[i] != exit->src
6772	&& dominated_by_p (CDI_DOMINATORS, region[i], exit->src))
6773	return false;
6774	}
6775
6776	initialize_original_copy_tables ();
6777
6778	if (copying_header)
6779	set_loop_copy (loop, loop_outer (loop));
6780	else
6781	set_loop_copy (loop, loop);
6782
6783	if (!region_copy)
6784	{
6785	region_copy = XNEWVEC (basic_block, n_region);
6786	free_region_copy = true;
6787	}
6788
6789	/* Record blocks outside the region that are dominated by something
6790	inside. */
6791	auto_vec<basic_block> doms;
6792	if (update_dominance)
6793	doms = get_dominated_by_region (CDI_DOMINATORS, region, n_region);
6794
6795	copy_bbs (region, n_region, region_copy, &exit, 1, &exit_copy, loop,
6796	split_edge_bb_loc (edge_in: entry), update_dominance);
6797
6798	if (copying_header)
6799	{
6800	loop->header = exit->dest;
6801	loop->latch = exit->src;
6802	}
6803
6804	/* Redirect the entry and add the phi node arguments. */
6805	redirected = redirect_edge_and_branch (entry, get_bb_copy (entry->dest));
6806	gcc_assert (redirected != NULL);
6807	flush_pending_stmts (entry);
6808
6809	/* Concerning updating of dominators: We must recount dominators
6810	for entry block and its copy. Anything that is outside of the
6811	region, but was dominated by something inside needs recounting as
6812	well. */
6813	if (update_dominance)
6814	{
6815	set_immediate_dominator (CDI_DOMINATORS, entry->dest, entry->src);
6816	doms.safe_push (obj: get_bb_original (entry->dest));
6817	iterate_fix_dominators (CDI_DOMINATORS, doms, false);
6818	}
6819
6820	/* Add the other PHI node arguments. */
6821	add_phi_args_after_copy (region_copy, n_region, NULL);
6822
6823	if (free_region_copy)
6824	free (ptr: region_copy);
6825
6826	free_original_copy_tables ();
6827	return true;
6828	}
6829
6830	/* Checks if BB is part of the region defined by N_REGION BBS. */
6831	static bool
6832	bb_part_of_region_p (basic_block bb, basic_block* bbs, unsigned n_region)
6833	{
6834	unsigned int n;
6835
6836	for (n = 0; n < n_region; n++)
6837	{
6838	if (bb == bbs[n])
6839	return true;
6840	}
6841	return false;
6842	}
6843
6844
6845	/* For each PHI in BB, copy the argument associated with SRC_E to TGT_E.
6846	Assuming the argument exists, just does not have a value. */
6847
6848	void
6849	copy_phi_arg_into_existing_phi (edge src_e, edge tgt_e)
6850	{
6851	int src_idx = src_e->dest_idx;
6852	int tgt_idx = tgt_e->dest_idx;
6853
6854	/* Iterate over each PHI in e->dest. */
6855	for (gphi_iterator gsi = gsi_start_phis (src_e->dest),
6856	gsi2 = gsi_start_phis (tgt_e->dest);
6857	!gsi_end_p (i: gsi);
6858	gsi_next (i: &gsi), gsi_next (i: &gsi2))
6859	{
6860	gphi *src_phi = gsi.phi ();
6861	gphi *dest_phi = gsi2.phi ();
6862	tree val = gimple_phi_arg_def (gs: src_phi, index: src_idx);
6863	location_t locus = gimple_phi_arg_location (phi: src_phi, i: src_idx);
6864
6865	SET_PHI_ARG_DEF (dest_phi, tgt_idx, val);
6866	gimple_phi_arg_set_location (phi: dest_phi, i: tgt_idx, loc: locus);
6867	}
6868	}
6869
6870	/* Duplicates REGION consisting of N_REGION blocks. The new blocks
6871	are stored to REGION_COPY in the same order in that they appear
6872	in REGION, if REGION_COPY is not NULL. ENTRY is the entry to
6873	the region, EXIT an exit from it. The condition guarding EXIT
6874	is moved to ENTRY. Returns true if duplication succeeds, false
6875	otherwise.
6876
6877	For example,
6878
6879	some_code;
6880	if (cond)
6881	A;
6882	else
6883	B;
6884
6885	is transformed to
6886
6887	if (cond)
6888	{
6889	some_code;
6890	A;
6891	}
6892	else
6893	{
6894	some_code;
6895	B;
6896	}
6897	*/
6898
6899	bool
6900	gimple_duplicate_sese_tail (edge entry, edge exit,
6901	basic_block *region, unsigned n_region,
6902	basic_block *region_copy)
6903	{
6904	unsigned i;
6905	bool free_region_copy = false;
6906	class loop *loop = exit->dest->loop_father;
6907	class loop *orig_loop = entry->dest->loop_father;
6908	basic_block switch_bb, entry_bb, nentry_bb;
6909	profile_count total_count = profile_count::uninitialized (),
6910	exit_count = profile_count::uninitialized ();
6911	edge exits[2], nexits[2], e;
6912	gimple_stmt_iterator gsi;
6913	edge sorig, snew;
6914	basic_block exit_bb;
6915	class loop *target, *aloop, *cloop;
6916
6917	gcc_assert (EDGE_COUNT (exit->src->succs) == 2);
6918	exits[0] = exit;
6919	exits[1] = EDGE_SUCC (exit->src, EDGE_SUCC (exit->src, 0) == exit);
6920
6921	if (!can_copy_bbs_p (region, n_region))
6922	return false;
6923
6924	initialize_original_copy_tables ();
6925	set_loop_copy (orig_loop, loop);
6926
6927	target= loop;
6928	for (aloop = orig_loop->inner; aloop; aloop = aloop->next)
6929	{
6930	if (bb_part_of_region_p (bb: aloop->header, bbs: region, n_region))
6931	{
6932	cloop = duplicate_loop (aloop, target);
6933	duplicate_subloops (aloop, cloop);
6934	}
6935	}
6936
6937	if (!region_copy)
6938	{
6939	region_copy = XNEWVEC (basic_block, n_region);
6940	free_region_copy = true;
6941	}
6942
6943	gcc_assert (!need_ssa_update_p (cfun));
6944
6945	/* Record blocks outside the region that are dominated by something
6946	inside. */
6947	auto_vec<basic_block> doms = get_dominated_by_region (CDI_DOMINATORS, region,
6948	n_region);
6949
6950	total_count = exit->src->count;
6951	exit_count = exit->count ();
6952	/* Fix up corner cases, to avoid division by zero or creation of negative
6953	frequencies. */
6954	if (exit_count > total_count)
6955	exit_count = total_count;
6956
6957	copy_bbs (region, n_region, region_copy, exits, 2, nexits, orig_loop,
6958	split_edge_bb_loc (edge_in: exit), true);
6959	if (total_count.initialized_p () && exit_count.initialized_p ())
6960	{
6961	scale_bbs_frequencies_profile_count (region, n_region,
6962	total_count - exit_count,
6963	total_count);
6964	scale_bbs_frequencies_profile_count (region_copy, n_region, exit_count,
6965	total_count);
6966	}
6967
6968	/* Create the switch block, and put the exit condition to it. */
6969	entry_bb = entry->dest;
6970	nentry_bb = get_bb_copy (entry_bb);
6971	if (!*gsi_last_bb (bb: entry->src)
6972	|| !stmt_ends_bb_p (t: *gsi_last_bb (bb: entry->src)))
6973	switch_bb = entry->src;
6974	else
6975	switch_bb = split_edge (entry);
6976	set_immediate_dominator (CDI_DOMINATORS, nentry_bb, switch_bb);
6977
6978	gcond *cond_stmt = as_a <gcond *> (p: *gsi_last_bb (bb: exit->src));
6979	cond_stmt = as_a <gcond *> (p: gimple_copy (cond_stmt));
6980
6981	gsi = gsi_last_bb (bb: switch_bb);
6982	gsi_insert_after (&gsi, cond_stmt, GSI_NEW_STMT);
6983
6984	sorig = single_succ_edge (bb: switch_bb);
6985	sorig->flags = exits[1]->flags;
6986	sorig->probability = exits[1]->probability;
6987	snew = make_edge (switch_bb, nentry_bb, exits[0]->flags);
6988	snew->probability = exits[0]->probability;
6989
6990
6991	/* Register the new edge from SWITCH_BB in loop exit lists. */
6992	rescan_loop_exit (snew, true, false);
6993
6994	/* Add the PHI node arguments. */
6995	add_phi_args_after_copy (region_copy, n_region, e_copy: snew);
6996
6997	/* Get rid of now superfluous conditions and associated edges (and phi node
6998	arguments). */
6999	exit_bb = exit->dest;
7000
7001	e = redirect_edge_and_branch (exits[0], exits[1]->dest);
7002	PENDING_STMT (e) = NULL;
7003
7004	/* The latch of ORIG_LOOP was copied, and so was the backedge
7005	to the original header. We redirect this backedge to EXIT_BB. */
7006	for (i = 0; i < n_region; i++)
7007	if (get_bb_original (region_copy[i]) == orig_loop->latch)
7008	{
7009	gcc_assert (single_succ_edge (region_copy[i]));
7010	e = redirect_edge_and_branch (single_succ_edge (bb: region_copy[i]), exit_bb);
7011	PENDING_STMT (e) = NULL;
7012	copy_phi_arg_into_existing_phi (src_e: nexits[0], tgt_e: e);
7013	}
7014	e = redirect_edge_and_branch (nexits[1], nexits[0]->dest);
7015	PENDING_STMT (e) = NULL;
7016
7017	/* Anything that is outside of the region, but was dominated by something
7018	inside needs to update dominance info. */
7019	iterate_fix_dominators (CDI_DOMINATORS, doms, false);
7020
7021	if (free_region_copy)
7022	free (ptr: region_copy);
7023
7024	free_original_copy_tables ();
7025	return true;
7026	}
7027
7028	/* Add all the blocks dominated by ENTRY to the array BBS_P. Stop
7029	adding blocks when the dominator traversal reaches EXIT. This
7030	function silently assumes that ENTRY strictly dominates EXIT. */
7031
7032	void
7033	gather_blocks_in_sese_region (basic_block entry, basic_block exit,
7034	vec<basic_block> *bbs_p)
7035	{
7036	basic_block son;
7037
7038	for (son = first_dom_son (CDI_DOMINATORS, entry);
7039	son;
7040	son = next_dom_son (CDI_DOMINATORS, son))
7041	{
7042	bbs_p->safe_push (obj: son);
7043	if (son != exit)
7044	gather_blocks_in_sese_region (entry: son, exit, bbs_p);
7045	}
7046	}
7047
7048	/* Replaces *TP with a duplicate (belonging to function TO_CONTEXT).
7049	The duplicates are recorded in VARS_MAP. */
7050
7051	static void
7052	replace_by_duplicate_decl (tree *tp, hash_map<tree, tree> *vars_map,
7053	tree to_context)
7054	{
7055	tree t = *tp, new_t;
7056	struct function *f = DECL_STRUCT_FUNCTION (to_context);
7057
7058	if (DECL_CONTEXT (t) == to_context)
7059	return;
7060
7061	bool existed;
7062	tree &loc = vars_map->get_or_insert (k: t, existed: &existed);
7063
7064	if (!existed)
7065	{
7066	if (SSA_VAR_P (t))
7067	{
7068	new_t = copy_var_decl (t, DECL_NAME (t), TREE_TYPE (t));
7069	add_local_decl (fun: f, d: new_t);
7070	}
7071	else
7072	{
7073	gcc_assert (TREE_CODE (t) == CONST_DECL);
7074	new_t = copy_node (t);
7075	}
7076	DECL_CONTEXT (new_t) = to_context;
7077
7078	loc = new_t;
7079	}
7080	else
7081	new_t = loc;
7082
7083	*tp = new_t;
7084	}
7085
7086
7087	/* Creates an ssa name in TO_CONTEXT equivalent to NAME.
7088	VARS_MAP maps old ssa names and var_decls to the new ones. */
7089
7090	static tree
7091	replace_ssa_name (tree name, hash_map<tree, tree> *vars_map,
7092	tree to_context)
7093	{
7094	tree new_name;
7095
7096	gcc_assert (!virtual_operand_p (name));
7097
7098	tree *loc = vars_map->get (k: name);
7099
7100	if (!loc)
7101	{
7102	tree decl = SSA_NAME_VAR (name);
7103	if (decl)
7104	{
7105	gcc_assert (!SSA_NAME_IS_DEFAULT_DEF (name));
7106	replace_by_duplicate_decl (tp: &decl, vars_map, to_context);
7107	new_name = make_ssa_name_fn (DECL_STRUCT_FUNCTION (to_context),
7108	decl, SSA_NAME_DEF_STMT (name));
7109	}
7110	else
7111	new_name = copy_ssa_name_fn (DECL_STRUCT_FUNCTION (to_context),
7112	name, SSA_NAME_DEF_STMT (name));
7113
7114	/* Now that we've used the def stmt to define new_name, make sure it
7115	doesn't define name anymore. */
7116	SSA_NAME_DEF_STMT (name) = NULL;
7117
7118	vars_map->put (k: name, v: new_name);
7119	}
7120	else
7121	new_name = *loc;
7122
7123	return new_name;
7124	}
7125
7126	struct move_stmt_d
7127	{
7128	tree orig_block;
7129	tree new_block;
7130	tree from_context;
7131	tree to_context;
7132	hash_map<tree, tree> *vars_map;
7133	htab_t new_label_map;
7134	hash_map<void *, void *> *eh_map;
7135	bool remap_decls_p;
7136	};
7137
7138	/* Helper for move_block_to_fn. Set TREE_BLOCK in every expression
7139	contained in *TP if it has been ORIG_BLOCK previously and change the
7140	DECL_CONTEXT of every local variable referenced in *TP. */
7141
7142	static tree
7143	move_stmt_op (tree *tp, int *walk_subtrees, void *data)
7144	{
7145	struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
7146	struct move_stmt_d *p = (struct move_stmt_d *) wi->info;
7147	tree t = *tp;
7148
7149	if (EXPR_P (t))
7150	{
7151	tree block = TREE_BLOCK (t);
7152	if (block == NULL_TREE)
7153	;
7154	else if (block == p->orig_block
7155	|| p->orig_block == NULL_TREE)
7156	{
7157	/* tree_node_can_be_shared says we can share invariant
7158	addresses but unshare_expr copies them anyways. Make sure
7159	to unshare before adjusting the block in place - we do not
7160	always see a copy here. */
7161	if (TREE_CODE (t) == ADDR_EXPR
7162	&& is_gimple_min_invariant (t))
7163	*tp = t = unshare_expr (t);
7164	TREE_SET_BLOCK (t, p->new_block);
7165	}
7166	else if (flag_checking)
7167	{
7168	while (block && TREE_CODE (block) == BLOCK && block != p->orig_block)
7169	block = BLOCK_SUPERCONTEXT (block);
7170	gcc_assert (block == p->orig_block);
7171	}
7172	}
7173	else if (DECL_P (t) || TREE_CODE (t) == SSA_NAME)
7174	{
7175	if (TREE_CODE (t) == SSA_NAME)
7176	*tp = replace_ssa_name (name: t, vars_map: p->vars_map, to_context: p->to_context);
7177	else if (TREE_CODE (t) == PARM_DECL
7178	&& gimple_in_ssa_p (cfun))
7179	*tp = *(p->vars_map->get (k: t));
7180	else if (TREE_CODE (t) == LABEL_DECL)
7181	{
7182	if (p->new_label_map)
7183	{
7184	struct tree_map in, *out;
7185	in.base.from = t;
7186	out = (struct tree_map *)
7187	htab_find_with_hash (p->new_label_map, &in, DECL_UID (t));
7188	if (out)
7189	*tp = t = out->to;
7190	}
7191
7192	/* For FORCED_LABELs we can end up with references from other
7193	functions if some SESE regions are outlined. It is UB to
7194	jump in between them, but they could be used just for printing
7195	addresses etc. In that case, DECL_CONTEXT on the label should
7196	be the function containing the glabel stmt with that LABEL_DECL,
7197	rather than whatever function a reference to the label was seen
7198	last time. */
7199	if (!FORCED_LABEL (t) && !DECL_NONLOCAL (t))
7200	DECL_CONTEXT (t) = p->to_context;
7201	}
7202	else if (p->remap_decls_p)
7203	{
7204	/* Replace T with its duplicate. T should no longer appear in the
7205	parent function, so this looks wasteful; however, it may appear
7206	in referenced_vars, and more importantly, as virtual operands of
7207	statements, and in alias lists of other variables. It would be
7208	quite difficult to expunge it from all those places. ??? It might
7209	suffice to do this for addressable variables. */
7210	if ((VAR_P (t) && !is_global_var (t))
7211	|| TREE_CODE (t) == CONST_DECL)
7212	replace_by_duplicate_decl (tp, vars_map: p->vars_map, to_context: p->to_context);
7213	}
7214	*walk_subtrees = 0;
7215	}
7216	else if (TYPE_P (t))
7217	*walk_subtrees = 0;
7218
7219	return NULL_TREE;
7220	}
7221
7222	/* Helper for move_stmt_r. Given an EH region number for the source
7223	function, map that to the duplicate EH regio number in the dest. */
7224
7225	static int
7226	move_stmt_eh_region_nr (int old_nr, struct move_stmt_d *p)
7227	{
7228	eh_region old_r, new_r;
7229
7230	old_r = get_eh_region_from_number (old_nr);
7231	new_r = static_cast<eh_region> (*p->eh_map->get (k: old_r));
7232
7233	return new_r->index;
7234	}
7235
7236	/* Similar, but operate on INTEGER_CSTs. */
7237
7238	static tree
7239	move_stmt_eh_region_tree_nr (tree old_t_nr, struct move_stmt_d *p)
7240	{
7241	int old_nr, new_nr;
7242
7243	old_nr = tree_to_shwi (old_t_nr);
7244	new_nr = move_stmt_eh_region_nr (old_nr, p);
7245
7246	return build_int_cst (integer_type_node, new_nr);
7247	}
7248
7249	/* Like move_stmt_op, but for gimple statements.
7250
7251	Helper for move_block_to_fn. Set GIMPLE_BLOCK in every expression
7252	contained in the current statement in *GSI_P and change the
7253	DECL_CONTEXT of every local variable referenced in the current
7254	statement. */
7255
7256	static tree
7257	move_stmt_r (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
7258	struct walk_stmt_info *wi)
7259	{
7260	struct move_stmt_d *p = (struct move_stmt_d *) wi->info;
7261	gimple *stmt = gsi_stmt (i: *gsi_p);
7262	tree block = gimple_block (g: stmt);
7263
7264	if (block == p->orig_block
7265	|| (p->orig_block == NULL_TREE
7266	&& block != NULL_TREE))
7267	gimple_set_block (g: stmt, block: p->new_block);
7268
7269	switch (gimple_code (g: stmt))
7270	{
7271	case GIMPLE_CALL:
7272	/* Remap the region numbers for __builtin_eh_{pointer,filter}. */
7273	{
7274	tree r, fndecl = gimple_call_fndecl (gs: stmt);
7275	if (fndecl && fndecl_built_in_p (node: fndecl, klass: BUILT_IN_NORMAL))
7276	switch (DECL_FUNCTION_CODE (decl: fndecl))
7277	{
7278	case BUILT_IN_EH_COPY_VALUES:
7279	r = gimple_call_arg (gs: stmt, index: 1);
7280	r = move_stmt_eh_region_tree_nr (old_t_nr: r, p);
7281	gimple_call_set_arg (gs: stmt, index: 1, arg: r);
7282	/* FALLTHRU */
7283
7284	case BUILT_IN_EH_POINTER:
7285	case BUILT_IN_EH_FILTER:
7286	r = gimple_call_arg (gs: stmt, index: 0);
7287	r = move_stmt_eh_region_tree_nr (old_t_nr: r, p);
7288	gimple_call_set_arg (gs: stmt, index: 0, arg: r);
7289	break;
7290
7291	default:
7292	break;
7293	}
7294	}
7295	break;
7296
7297	case GIMPLE_RESX:
7298	{
7299	gresx *resx_stmt = as_a <gresx *> (p: stmt);
7300	int r = gimple_resx_region (resx_stmt);
7301	r = move_stmt_eh_region_nr (old_nr: r, p);
7302	gimple_resx_set_region (resx_stmt, region: r);
7303	}
7304	break;
7305
7306	case GIMPLE_EH_DISPATCH:
7307	{
7308	geh_dispatch *eh_dispatch_stmt = as_a <geh_dispatch *> (p: stmt);
7309	int r = gimple_eh_dispatch_region (eh_dispatch_stmt);
7310	r = move_stmt_eh_region_nr (old_nr: r, p);
7311	gimple_eh_dispatch_set_region (eh_dispatch_stmt, region: r);
7312	}
7313	break;
7314
7315	case GIMPLE_OMP_RETURN:
7316	case GIMPLE_OMP_CONTINUE:
7317	break;
7318
7319	case GIMPLE_LABEL:
7320	{
7321	/* For FORCED_LABEL, move_stmt_op doesn't adjust DECL_CONTEXT,
7322	so that such labels can be referenced from other regions.
7323	Make sure to update it when seeing a GIMPLE_LABEL though,
7324	that is the owner of the label. */
7325	walk_gimple_op (stmt, move_stmt_op, wi);
7326	*handled_ops_p = true;
7327	tree label = gimple_label_label (gs: as_a <glabel *> (p: stmt));
7328	if (FORCED_LABEL (label) || DECL_NONLOCAL (label))
7329	DECL_CONTEXT (label) = p->to_context;
7330	}
7331	break;
7332
7333	default:
7334	if (is_gimple_omp (stmt))
7335	{
7336	/* Do not remap variables inside OMP directives. Variables
7337	referenced in clauses and directive header belong to the
7338	parent function and should not be moved into the child
7339	function. */
7340	bool save_remap_decls_p = p->remap_decls_p;
7341	p->remap_decls_p = false;
7342	*handled_ops_p = true;
7343
7344	walk_gimple_seq_mod (gimple_omp_body_ptr (gs: stmt), move_stmt_r,
7345	move_stmt_op, wi);
7346
7347	p->remap_decls_p = save_remap_decls_p;
7348	}
7349	break;
7350	}
7351
7352	return NULL_TREE;
7353	}
7354
7355	/* Move basic block BB from function CFUN to function DEST_FN. The
7356	block is moved out of the original linked list and placed after
7357	block AFTER in the new list. Also, the block is removed from the
7358	original array of blocks and placed in DEST_FN's array of blocks.
7359	If UPDATE_EDGE_COUNT_P is true, the edge counts on both CFGs is
7360	updated to reflect the moved edges.
7361
7362	The local variables are remapped to new instances, VARS_MAP is used
7363	to record the mapping. */
7364
7365	static void
7366	move_block_to_fn (struct function *dest_cfun, basic_block bb,
7367	basic_block after, bool update_edge_count_p,
7368	struct move_stmt_d *d)
7369	{
7370	struct control_flow_graph *cfg;
7371	edge_iterator ei;
7372	edge e;
7373	gimple_stmt_iterator si;
7374	unsigned old_len;
7375
7376	/* Remove BB from dominance structures. */
7377	delete_from_dominance_info (CDI_DOMINATORS, bb);
7378
7379	/* Move BB from its current loop to the copy in the new function. */
7380	if (current_loops)
7381	{
7382	class loop *new_loop = (class loop *)bb->loop_father->aux;
7383	if (new_loop)
7384	bb->loop_father = new_loop;
7385	}
7386
7387	/* Link BB to the new linked list. */
7388	move_block_after (bb, after);
7389
7390	/* Update the edge count in the corresponding flowgraphs. */
7391	if (update_edge_count_p)
7392	FOR_EACH_EDGE (e, ei, bb->succs)
7393	{
7394	cfun->cfg->x_n_edges--;
7395	dest_cfun->cfg->x_n_edges++;
7396	}
7397
7398	/* Remove BB from the original basic block array. */
7399	(*cfun->cfg->x_basic_block_info)[bb->index] = NULL;
7400	cfun->cfg->x_n_basic_blocks--;
7401
7402	/* Grow DEST_CFUN's basic block array if needed. */
7403	cfg = dest_cfun->cfg;
7404	cfg->x_n_basic_blocks++;
7405	if (bb->index >= cfg->x_last_basic_block)
7406	cfg->x_last_basic_block = bb->index + 1;
7407
7408	old_len = vec_safe_length (v: cfg->x_basic_block_info);
7409	if ((unsigned) cfg->x_last_basic_block >= old_len)
7410	vec_safe_grow_cleared (v&: cfg->x_basic_block_info,
7411	len: cfg->x_last_basic_block + 1);
7412
7413	(*cfg->x_basic_block_info)[bb->index] = bb;
7414
7415	/* Remap the variables in phi nodes. */
7416	for (gphi_iterator psi = gsi_start_phis (bb);
7417	!gsi_end_p (i: psi); )
7418	{
7419	gphi *phi = psi.phi ();
7420	use_operand_p use;
7421	tree op = PHI_RESULT (phi);
7422	ssa_op_iter oi;
7423	unsigned i;
7424
7425	if (virtual_operand_p (op))
7426	{
7427	/* Remove the phi nodes for virtual operands (alias analysis will be
7428	run for the new function, anyway). But replace all uses that
7429	might be outside of the region we move. */
7430	use_operand_p use_p;
7431	imm_use_iterator iter;
7432	gimple *use_stmt;
7433	FOR_EACH_IMM_USE_STMT (use_stmt, iter, op)
7434	FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
7435	SET_USE (use_p, SSA_NAME_VAR (op));
7436	remove_phi_node (&psi, true);
7437	continue;
7438	}
7439
7440	SET_PHI_RESULT (phi,
7441	replace_ssa_name (op, d->vars_map, dest_cfun->decl));
7442	FOR_EACH_PHI_ARG (use, phi, oi, SSA_OP_USE)
7443	{
7444	op = USE_FROM_PTR (use);
7445	if (TREE_CODE (op) == SSA_NAME)
7446	SET_USE (use, replace_ssa_name (op, d->vars_map, dest_cfun->decl));
7447	}
7448
7449	for (i = 0; i < EDGE_COUNT (bb->preds); i++)
7450	{
7451	location_t locus = gimple_phi_arg_location (phi, i);
7452	tree block = LOCATION_BLOCK (locus);
7453
7454	if (locus == UNKNOWN_LOCATION)
7455	continue;
7456	if (d->orig_block == NULL_TREE || block == d->orig_block)
7457	{
7458	locus = set_block (loc: locus, block: d->new_block);
7459	gimple_phi_arg_set_location (phi, i, loc: locus);
7460	}
7461	}
7462
7463	gsi_next (i: &psi);
7464	}
7465
7466	for (si = gsi_start_bb (bb); !gsi_end_p (i: si); gsi_next (i: &si))
7467	{
7468	gimple *stmt = gsi_stmt (i: si);
7469	struct walk_stmt_info wi;
7470
7471	memset (s: &wi, c: 0, n: sizeof (wi));
7472	wi.info = d;
7473	walk_gimple_stmt (&si, move_stmt_r, move_stmt_op, &wi);
7474
7475	if (glabel *label_stmt = dyn_cast <glabel *> (p: stmt))
7476	{
7477	tree label = gimple_label_label (gs: label_stmt);
7478	int uid = LABEL_DECL_UID (label);
7479
7480	gcc_assert (uid > -1);
7481
7482	old_len = vec_safe_length (v: cfg->x_label_to_block_map);
7483	if (old_len <= (unsigned) uid)
7484	vec_safe_grow_cleared (v&: cfg->x_label_to_block_map, len: uid + 1);
7485
7486	(*cfg->x_label_to_block_map)[uid] = bb;
7487	(*cfun->cfg->x_label_to_block_map)[uid] = NULL;
7488
7489	gcc_assert (DECL_CONTEXT (label) == dest_cfun->decl);
7490
7491	if (uid >= dest_cfun->cfg->last_label_uid)
7492	dest_cfun->cfg->last_label_uid = uid + 1;
7493	}
7494
7495	maybe_duplicate_eh_stmt_fn (dest_cfun, stmt, cfun, stmt, d->eh_map, 0);
7496	remove_stmt_from_eh_lp_fn (cfun, stmt);
7497
7498	gimple_duplicate_stmt_histograms (dest_cfun, stmt, cfun, stmt);
7499	gimple_remove_stmt_histograms (cfun, stmt);
7500
7501	/* We cannot leave any operands allocated from the operand caches of
7502	the current function. */
7503	free_stmt_operands (cfun, stmt);
7504	push_cfun (new_cfun: dest_cfun);
7505	update_stmt (s: stmt);
7506	if (is_gimple_call (gs: stmt))
7507	notice_special_calls (call: as_a <gcall *> (p: stmt));
7508	pop_cfun ();
7509	}
7510
7511	FOR_EACH_EDGE (e, ei, bb->succs)
7512	if (e->goto_locus != UNKNOWN_LOCATION)
7513	{
7514	tree block = LOCATION_BLOCK (e->goto_locus);
7515	if (d->orig_block == NULL_TREE
7516	|| block == d->orig_block)
7517	e->goto_locus = set_block (loc: e->goto_locus, block: d->new_block);
7518	}
7519	}
7520
7521	/* Examine the statements in BB (which is in SRC_CFUN); find and return
7522	the outermost EH region. Use REGION as the incoming base EH region.
7523	If there is no single outermost region, return NULL and set *ALL to
7524	true. */
7525
7526	static eh_region
7527	find_outermost_region_in_block (struct function *src_cfun,
7528	basic_block bb, eh_region region,
7529	bool *all)
7530	{
7531	gimple_stmt_iterator si;
7532
7533	for (si = gsi_start_bb (bb); !gsi_end_p (i: si); gsi_next (i: &si))
7534	{
7535	gimple *stmt = gsi_stmt (i: si);
7536	eh_region stmt_region;
7537	int lp_nr;
7538
7539	lp_nr = lookup_stmt_eh_lp_fn (src_cfun, stmt);
7540	stmt_region = get_eh_region_from_lp_number_fn (src_cfun, lp_nr);
7541	if (stmt_region)
7542	{
7543	if (region == NULL)
7544	region = stmt_region;
7545	else if (stmt_region != region)
7546	{
7547	region = eh_region_outermost (src_cfun, stmt_region, region);
7548	if (region == NULL)
7549	{
7550	*all = true;
7551	return NULL;
7552	}
7553	}
7554	}
7555	}
7556
7557	return region;
7558	}
7559
7560	static tree
7561	new_label_mapper (tree decl, void *data)
7562	{
7563	htab_t hash = (htab_t) data;
7564	struct tree_map *m;
7565	void **slot;
7566
7567	gcc_assert (TREE_CODE (decl) == LABEL_DECL);
7568
7569	m = XNEW (struct tree_map);
7570	m->hash = DECL_UID (decl);
7571	m->base.from = decl;
7572	m->to = create_artificial_label (UNKNOWN_LOCATION);
7573	LABEL_DECL_UID (m->to) = LABEL_DECL_UID (decl);
7574	if (LABEL_DECL_UID (m->to) >= cfun->cfg->last_label_uid)
7575	cfun->cfg->last_label_uid = LABEL_DECL_UID (m->to) + 1;
7576
7577	slot = htab_find_slot_with_hash (hash, m, m->hash, INSERT);
7578	gcc_assert (*slot == NULL);
7579
7580	*slot = m;
7581
7582	return m->to;
7583	}
7584
7585	/* Tree walker to replace the decls used inside value expressions by
7586	duplicates. */
7587
7588	static tree
7589	replace_block_vars_by_duplicates_1 (tree *tp, int *walk_subtrees, void *data)
7590	{
7591	struct replace_decls_d *rd = (struct replace_decls_d *)data;
7592
7593	switch (TREE_CODE (*tp))
7594	{
7595	case VAR_DECL:
7596	case PARM_DECL:
7597	case RESULT_DECL:
7598	replace_by_duplicate_decl (tp, vars_map: rd->vars_map, to_context: rd->to_context);
7599	break;
7600	default:
7601	break;
7602	}
7603
7604	if (IS_TYPE_OR_DECL_P (*tp))
7605	*walk_subtrees = false;
7606
7607	return NULL;
7608	}
7609
7610	/* Change DECL_CONTEXT of all BLOCK_VARS in block, including
7611	subblocks. */
7612
7613	static void
7614	replace_block_vars_by_duplicates (tree block, hash_map<tree, tree> *vars_map,
7615	tree to_context)
7616	{
7617	tree *tp, t;
7618
7619	for (tp = &BLOCK_VARS (block); *tp; tp = &DECL_CHAIN (*tp))
7620	{
7621	t = *tp;
7622	if (!VAR_P (t) && TREE_CODE (t) != CONST_DECL)
7623	continue;
7624	replace_by_duplicate_decl (tp: &t, vars_map, to_context);
7625	if (t != *tp)
7626	{
7627	if (VAR_P (*tp) && DECL_HAS_VALUE_EXPR_P (*tp))
7628	{
7629	tree x = DECL_VALUE_EXPR (*tp);
7630	struct replace_decls_d rd = { .vars_map: vars_map, .to_context: to_context };
7631	unshare_expr (x);
7632	walk_tree (&x, replace_block_vars_by_duplicates_1, &rd, NULL);
7633	SET_DECL_VALUE_EXPR (t, x);
7634	DECL_HAS_VALUE_EXPR_P (t) = 1;
7635	}
7636	DECL_CHAIN (t) = DECL_CHAIN (*tp);
7637	*tp = t;
7638	}
7639	}
7640
7641	for (block = BLOCK_SUBBLOCKS (block); block; block = BLOCK_CHAIN (block))
7642	replace_block_vars_by_duplicates (block, vars_map, to_context);
7643	}
7644
7645	/* Fixup the loop arrays and numbers after moving LOOP and its subloops
7646	from FN1 to FN2. */
7647
7648	static void
7649	fixup_loop_arrays_after_move (struct function *fn1, struct function *fn2,
7650	class loop *loop)
7651	{
7652	/* Discard it from the old loop array. */
7653	(*get_loops (fn: fn1))[loop->num] = NULL;
7654
7655	/* Place it in the new loop array, assigning it a new number. */
7656	loop->num = number_of_loops (fn: fn2);
7657	vec_safe_push (v&: loops_for_fn (fn: fn2)->larray, obj: loop);
7658
7659	/* Recurse to children. */
7660	for (loop = loop->inner; loop; loop = loop->next)
7661	fixup_loop_arrays_after_move (fn1, fn2, loop);
7662	}
7663
7664	/* Verify that the blocks in BBS_P are a single-entry, single-exit region
7665	delimited by ENTRY_BB and EXIT_BB, possibly containing noreturn blocks. */
7666
7667	DEBUG_FUNCTION void
7668	verify_sese (basic_block entry, basic_block exit, vec<basic_block> *bbs_p)
7669	{
7670	basic_block bb;
7671	edge_iterator ei;
7672	edge e;
7673	bitmap bbs = BITMAP_ALLOC (NULL);
7674	int i;
7675
7676	gcc_assert (entry != NULL);
7677	gcc_assert (entry != exit);
7678	gcc_assert (bbs_p != NULL);
7679
7680	gcc_assert (bbs_p->length () > 0);
7681
7682	FOR_EACH_VEC_ELT (*bbs_p, i, bb)
7683	bitmap_set_bit (bbs, bb->index);
7684
7685	gcc_assert (bitmap_bit_p (bbs, entry->index));
7686	gcc_assert (exit == NULL || bitmap_bit_p (bbs, exit->index));
7687
7688	FOR_EACH_VEC_ELT (*bbs_p, i, bb)
7689	{
7690	if (bb == entry)
7691	{
7692	gcc_assert (single_pred_p (entry));
7693	gcc_assert (!bitmap_bit_p (bbs, single_pred (entry)->index));
7694	}
7695	else
7696	for (ei = ei_start (bb->preds); !ei_end_p (i: ei); ei_next (i: &ei))
7697	{
7698	e = ei_edge (i: ei);
7699	gcc_assert (bitmap_bit_p (bbs, e->src->index));
7700	}
7701
7702	if (bb == exit)
7703	{
7704	gcc_assert (single_succ_p (exit));
7705	gcc_assert (!bitmap_bit_p (bbs, single_succ (exit)->index));
7706	}
7707	else
7708	for (ei = ei_start (bb->succs); !ei_end_p (i: ei); ei_next (i: &ei))
7709	{
7710	e = ei_edge (i: ei);
7711	gcc_assert (bitmap_bit_p (bbs, e->dest->index));
7712	}
7713	}
7714
7715	BITMAP_FREE (bbs);
7716	}
7717
7718	/* If FROM is an SSA_NAME, mark the version in bitmap DATA. */
7719
7720	bool
7721	gather_ssa_name_hash_map_from (tree const &from, tree const &, void *data)
7722	{
7723	bitmap release_names = (bitmap)data;
7724
7725	if (TREE_CODE (from) != SSA_NAME)
7726	return true;
7727
7728	bitmap_set_bit (release_names, SSA_NAME_VERSION (from));
7729	return true;
7730	}
7731
7732	/* Return LOOP_DIST_ALIAS call if present in BB. */
7733
7734	static gimple *
7735	find_loop_dist_alias (basic_block bb)
7736	{
7737	gimple_stmt_iterator gsi = gsi_last_bb (bb);
7738	if (!safe_is_a <gcond *> (p: *gsi))
7739	return NULL;
7740
7741	gsi_prev (i: &gsi);
7742	if (gsi_end_p (i: gsi))
7743	return NULL;
7744
7745	gimple *g = gsi_stmt (i: gsi);
7746	if (gimple_call_internal_p (gs: g, fn: IFN_LOOP_DIST_ALIAS))
7747	return g;
7748	return NULL;
7749	}
7750
7751	/* Fold loop internal call G like IFN_LOOP_VECTORIZED/IFN_LOOP_DIST_ALIAS
7752	to VALUE and update any immediate uses of it's LHS. */
7753
7754	void
7755	fold_loop_internal_call (gimple *g, tree value)
7756	{
7757	tree lhs = gimple_call_lhs (gs: g);
7758	use_operand_p use_p;
7759	imm_use_iterator iter;
7760	gimple *use_stmt;
7761	gimple_stmt_iterator gsi = gsi_for_stmt (g);
7762
7763	replace_call_with_value (&gsi, value);
7764	FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
7765	{
7766	FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
7767	SET_USE (use_p, value);
7768	update_stmt (s: use_stmt);
7769	/* If we turn conditional to constant, scale profile counts.
7770	We know that the conditional was created by loop distribution
7771	and all basic blocks dominated by the taken edge are part of
7772	the loop distributed. */
7773	if (gimple_code (g: use_stmt) == GIMPLE_COND)
7774	{
7775	edge true_edge, false_edge;
7776	extract_true_false_edges_from_block (gimple_bb (g: use_stmt),
7777	&true_edge, &false_edge);
7778	edge taken_edge = NULL, other_edge = NULL;
7779	if (gimple_cond_true_p (gs: as_a <gcond *>(p: use_stmt)))
7780	{
7781	taken_edge = true_edge;
7782	other_edge = false_edge;
7783	}
7784	else if (gimple_cond_false_p (gs: as_a <gcond *>(p: use_stmt)))
7785	{
7786	taken_edge = false_edge;
7787	other_edge = true_edge;
7788	}
7789	if (taken_edge
7790	&& !(taken_edge->probability == profile_probability::always ()))
7791	{
7792	profile_count old_count = taken_edge->count ();
7793	profile_count new_count = taken_edge->src->count;
7794	taken_edge->probability = profile_probability::always ();
7795	other_edge->probability = profile_probability::never ();
7796	/* If we have multiple predecessors, we can't use the dominance
7797	test. This should not happen as the guarded code should
7798	start with pre-header. */
7799	gcc_assert (single_pred_edge (taken_edge->dest));
7800	if (old_count.nonzero_p ())
7801	{
7802	taken_edge->dest->count
7803	= taken_edge->dest->count.apply_scale (num: new_count,
7804	den: old_count);
7805	scale_strictly_dominated_blocks (taken_edge->dest,
7806	new_count, old_count);
7807	}
7808	}
7809	}
7810	}
7811	}
7812
7813	/* Move a single-entry, single-exit region delimited by ENTRY_BB and
7814	EXIT_BB to function DEST_CFUN. The whole region is replaced by a
7815	single basic block in the original CFG and the new basic block is
7816	returned. DEST_CFUN must not have a CFG yet.
7817
7818	Note that the region need not be a pure SESE region. Blocks inside
7819	the region may contain calls to abort/exit. The only restriction
7820	is that ENTRY_BB should be the only entry point and it must
7821	dominate EXIT_BB.
7822
7823	Change TREE_BLOCK of all statements in ORIG_BLOCK to the new
7824	functions outermost BLOCK, move all subblocks of ORIG_BLOCK
7825	to the new function.
7826
7827	All local variables referenced in the region are assumed to be in
7828	the corresponding BLOCK_VARS and unexpanded variable lists
7829	associated with DEST_CFUN.
7830
7831	TODO: investigate whether we can reuse gimple_duplicate_sese_region to
7832	reimplement move_sese_region_to_fn by duplicating the region rather than
7833	moving it. */
7834
7835	basic_block
7836	move_sese_region_to_fn (struct function *dest_cfun, basic_block entry_bb,
7837	basic_block exit_bb, tree orig_block)
7838	{
7839	vec<basic_block> bbs;
7840	basic_block dom_entry = get_immediate_dominator (CDI_DOMINATORS, entry_bb);
7841	basic_block after, bb, *entry_pred, *exit_succ, abb;
7842	struct function *saved_cfun = cfun;
7843	int *entry_flag, *exit_flag;
7844	profile_probability *entry_prob, *exit_prob;
7845	unsigned i, num_entry_edges, num_exit_edges, num_nodes;
7846	edge e;
7847	edge_iterator ei;
7848	htab_t new_label_map;
7849	hash_map<void *, void *> *eh_map;
7850	class loop *loop = entry_bb->loop_father;
7851	class loop *loop0 = get_loop (fn: saved_cfun, num: 0);
7852	struct move_stmt_d d;
7853
7854	/* If ENTRY does not strictly dominate EXIT, this cannot be an SESE
7855	region. */
7856	gcc_assert (entry_bb != exit_bb
7857	&& (!exit_bb
7858	|| dominated_by_p (CDI_DOMINATORS, exit_bb, entry_bb)));
7859
7860	/* Collect all the blocks in the region. Manually add ENTRY_BB
7861	because it won't be added by dfs_enumerate_from. */
7862	bbs.create (nelems: 0);
7863	bbs.safe_push (obj: entry_bb);
7864	gather_blocks_in_sese_region (entry: entry_bb, exit: exit_bb, bbs_p: &bbs);
7865
7866	if (flag_checking)
7867	verify_sese (entry: entry_bb, exit: exit_bb, bbs_p: &bbs);
7868
7869	/* The blocks that used to be dominated by something in BBS will now be
7870	dominated by the new block. */
7871	auto_vec<basic_block> dom_bbs = get_dominated_by_region (CDI_DOMINATORS,
7872	bbs.address (),
7873	bbs.length ());
7874
7875	/* Detach ENTRY_BB and EXIT_BB from CFUN->CFG. We need to remember
7876	the predecessor edges to ENTRY_BB and the successor edges to
7877	EXIT_BB so that we can re-attach them to the new basic block that
7878	will replace the region. */
7879	num_entry_edges = EDGE_COUNT (entry_bb->preds);
7880	entry_pred = XNEWVEC (basic_block, num_entry_edges);
7881	entry_flag = XNEWVEC (int, num_entry_edges);
7882	entry_prob = XNEWVEC (profile_probability, num_entry_edges);
7883	i = 0;
7884	for (ei = ei_start (entry_bb->preds); (e = ei_safe_edge (i: ei)) != NULL;)
7885	{
7886	entry_prob[i] = e->probability;
7887	entry_flag[i] = e->flags;
7888	entry_pred[i++] = e->src;
7889	remove_edge (e);
7890	}
7891
7892	if (exit_bb)
7893	{
7894	num_exit_edges = EDGE_COUNT (exit_bb->succs);
7895	exit_succ = XNEWVEC (basic_block, num_exit_edges);
7896	exit_flag = XNEWVEC (int, num_exit_edges);
7897	exit_prob = XNEWVEC (profile_probability, num_exit_edges);
7898	i = 0;
7899	for (ei = ei_start (exit_bb->succs); (e = ei_safe_edge (i: ei)) != NULL;)
7900	{
7901	exit_prob[i] = e->probability;
7902	exit_flag[i] = e->flags;
7903	exit_succ[i++] = e->dest;
7904	remove_edge (e);
7905	}
7906	}
7907	else
7908	{
7909	num_exit_edges = 0;
7910	exit_succ = NULL;
7911	exit_flag = NULL;
7912	exit_prob = NULL;
7913	}
7914
7915	/* Switch context to the child function to initialize DEST_FN's CFG. */
7916	gcc_assert (dest_cfun->cfg == NULL);
7917	push_cfun (new_cfun: dest_cfun);
7918
7919	init_empty_tree_cfg ();
7920
7921	/* Initialize EH information for the new function. */
7922	eh_map = NULL;
7923	new_label_map = NULL;
7924	if (saved_cfun->eh)
7925	{
7926	eh_region region = NULL;
7927	bool all = false;
7928
7929	FOR_EACH_VEC_ELT (bbs, i, bb)
7930	{
7931	region = find_outermost_region_in_block (src_cfun: saved_cfun, bb, region, all: &all);
7932	if (all)
7933	break;
7934	}
7935
7936	init_eh_for_function ();
7937	if (region != NULL || all)
7938	{
7939	new_label_map = htab_create (17, tree_map_hash, tree_map_eq, free);
7940	eh_map = duplicate_eh_regions (saved_cfun, region, 0,
7941	new_label_mapper, new_label_map);
7942	}
7943	}
7944
7945	/* Initialize an empty loop tree. */
7946	struct loops *loops = ggc_cleared_alloc<struct loops> ();
7947	init_loops_structure (dest_cfun, loops, 1);
7948	loops->state = LOOPS_MAY_HAVE_MULTIPLE_LATCHES;
7949	set_loops_for_fn (fn: dest_cfun, loops);
7950
7951	vec<loop_p, va_gc> *larray = get_loops (fn: saved_cfun)->copy ();
7952
7953	/* Move the outlined loop tree part. */
7954	num_nodes = bbs.length ();
7955	FOR_EACH_VEC_ELT (bbs, i, bb)
7956	{
7957	if (bb->loop_father->header == bb)
7958	{
7959	class loop *this_loop = bb->loop_father;
7960	/* Avoid the need to remap SSA names used in nb_iterations. */
7961	free_numbers_of_iterations_estimates (this_loop);
7962	class loop *outer = loop_outer (loop: this_loop);
7963	if (outer == loop
7964	/* If the SESE region contains some bbs ending with
7965	a noreturn call, those are considered to belong
7966	to the outermost loop in saved_cfun, rather than
7967	the entry_bb's loop_father. */
7968	|| outer == loop0)
7969	{
7970	if (outer != loop)
7971	num_nodes -= this_loop->num_nodes;
7972	flow_loop_tree_node_remove (bb->loop_father);
7973	flow_loop_tree_node_add (get_loop (fn: dest_cfun, num: 0), this_loop);
7974	fixup_loop_arrays_after_move (fn1: saved_cfun, cfun, loop: this_loop);
7975	}
7976	}
7977	else if (bb->loop_father == loop0 && loop0 != loop)
7978	num_nodes--;
7979
7980	/* Remove loop exits from the outlined region. */
7981	if (loops_for_fn (fn: saved_cfun)->exits)
7982	FOR_EACH_EDGE (e, ei, bb->succs)
7983	{
7984	struct loops *l = loops_for_fn (fn: saved_cfun);
7985	loop_exit **slot
7986	= l->exits->find_slot_with_hash (comparable: e, hash: htab_hash_pointer (e),
7987	insert: NO_INSERT);
7988	if (slot)
7989	l->exits->clear_slot (slot);
7990	}
7991	}
7992
7993	/* Adjust the number of blocks in the tree root of the outlined part. */
7994	get_loop (fn: dest_cfun, num: 0)->num_nodes = bbs.length () + 2;
7995
7996	/* Setup a mapping to be used by move_block_to_fn. */
7997	loop->aux = current_loops->tree_root;
7998	loop0->aux = current_loops->tree_root;
7999
8000	/* Fix up orig_loop_num. If the block referenced in it has been moved
8001	to dest_cfun, update orig_loop_num field, otherwise clear it. */
8002	signed char *moved_orig_loop_num = NULL;
8003	for (auto dloop : loops_list (dest_cfun, 0))
8004	if (dloop->orig_loop_num)
8005	{
8006	if (moved_orig_loop_num == NULL)
8007	moved_orig_loop_num
8008	= XCNEWVEC (signed char, vec_safe_length (larray));
8009	if ((*larray)[dloop->orig_loop_num] != NULL
8010	&& get_loop (fn: saved_cfun, num: dloop->orig_loop_num) == NULL)
8011	{
8012	if (moved_orig_loop_num[dloop->orig_loop_num] >= 0
8013	&& moved_orig_loop_num[dloop->orig_loop_num] < 2)
8014	moved_orig_loop_num[dloop->orig_loop_num]++;
8015	dloop->orig_loop_num = (*larray)[dloop->orig_loop_num]->num;
8016	}
8017	else
8018	{
8019	moved_orig_loop_num[dloop->orig_loop_num] = -1;
8020	dloop->orig_loop_num = 0;
8021	}
8022	}
8023	pop_cfun ();
8024
8025	if (moved_orig_loop_num)
8026	{
8027	FOR_EACH_VEC_ELT (bbs, i, bb)
8028	{
8029	gimple *g = find_loop_dist_alias (bb);
8030	if (g == NULL)
8031	continue;
8032
8033	int orig_loop_num = tree_to_shwi (gimple_call_arg (gs: g, index: 0));
8034	gcc_assert (orig_loop_num
8035	&& (unsigned) orig_loop_num < vec_safe_length (larray));
8036	if (moved_orig_loop_num[orig_loop_num] == 2)
8037	{
8038	/* If we have moved both loops with this orig_loop_num into
8039	dest_cfun and the LOOP_DIST_ALIAS call is being moved there
8040	too, update the first argument. */
8041	gcc_assert ((*larray)[orig_loop_num] != NULL
8042	&& (get_loop (saved_cfun, orig_loop_num) == NULL));
8043	tree t = build_int_cst (integer_type_node,
8044	(*larray)[orig_loop_num]->num);
8045	gimple_call_set_arg (gs: g, index: 0, arg: t);
8046	update_stmt (s: g);
8047	/* Make sure the following loop will not update it. */
8048	moved_orig_loop_num[orig_loop_num] = 0;
8049	}
8050	else
8051	/* Otherwise at least one of the loops stayed in saved_cfun.
8052	Remove the LOOP_DIST_ALIAS call. */
8053	fold_loop_internal_call (g, value: gimple_call_arg (gs: g, index: 1));
8054	}
8055	FOR_EACH_BB_FN (bb, saved_cfun)
8056	{
8057	gimple *g = find_loop_dist_alias (bb);
8058	if (g == NULL)
8059	continue;
8060	int orig_loop_num = tree_to_shwi (gimple_call_arg (gs: g, index: 0));
8061	gcc_assert (orig_loop_num
8062	&& (unsigned) orig_loop_num < vec_safe_length (larray));
8063	if (moved_orig_loop_num[orig_loop_num])
8064	/* LOOP_DIST_ALIAS call remained in saved_cfun, if at least one
8065	of the corresponding loops was moved, remove it. */
8066	fold_loop_internal_call (g, value: gimple_call_arg (gs: g, index: 1));
8067	}
8068	XDELETEVEC (moved_orig_loop_num);
8069	}
8070	ggc_free (larray);
8071
8072	/* Move blocks from BBS into DEST_CFUN. */
8073	gcc_assert (bbs.length () >= 2);
8074	after = dest_cfun->cfg->x_entry_block_ptr;
8075	hash_map<tree, tree> vars_map;
8076
8077	memset (s: &d, c: 0, n: sizeof (d));
8078	d.orig_block = orig_block;
8079	d.new_block = DECL_INITIAL (dest_cfun->decl);
8080	d.from_context = cfun->decl;
8081	d.to_context = dest_cfun->decl;
8082	d.vars_map = &vars_map;
8083	d.new_label_map = new_label_map;
8084	d.eh_map = eh_map;
8085	d.remap_decls_p = true;
8086
8087	if (gimple_in_ssa_p (cfun))
8088	for (tree arg = DECL_ARGUMENTS (d.to_context); arg; arg = DECL_CHAIN (arg))
8089	{
8090	tree narg = make_ssa_name_fn (dest_cfun, arg, gimple_build_nop ());
8091	set_ssa_default_def (dest_cfun, arg, narg);
8092	vars_map.put (k: arg, v: narg);
8093	}
8094
8095	FOR_EACH_VEC_ELT (bbs, i, bb)
8096	{
8097	/* No need to update edge counts on the last block. It has
8098	already been updated earlier when we detached the region from
8099	the original CFG. */
8100	move_block_to_fn (dest_cfun, bb, after, update_edge_count_p: bb != exit_bb, d: &d);
8101	after = bb;
8102	}
8103
8104	/* Adjust the maximum clique used. */
8105	dest_cfun->last_clique = saved_cfun->last_clique;
8106
8107	loop->aux = NULL;
8108	loop0->aux = NULL;
8109	/* Loop sizes are no longer correct, fix them up. */
8110	loop->num_nodes -= num_nodes;
8111	for (class loop *outer = loop_outer (loop);
8112	outer; outer = loop_outer (loop: outer))
8113	outer->num_nodes -= num_nodes;
8114	loop0->num_nodes -= bbs.length () - num_nodes;
8115
8116	if (saved_cfun->has_simduid_loops || saved_cfun->has_force_vectorize_loops)
8117	{
8118	class loop *aloop;
8119	for (i = 0; vec_safe_iterate (v: loops->larray, ix: i, ptr: &aloop); i++)
8120	if (aloop != NULL)
8121	{
8122	if (aloop->simduid)
8123	{
8124	replace_by_duplicate_decl (tp: &aloop->simduid, vars_map: d.vars_map,
8125	to_context: d.to_context);
8126	dest_cfun->has_simduid_loops = true;
8127	}
8128	if (aloop->force_vectorize)
8129	dest_cfun->has_force_vectorize_loops = true;
8130	}
8131	}
8132
8133	/* Rewire BLOCK_SUBBLOCKS of orig_block. */
8134	if (orig_block)
8135	{
8136	tree block;
8137	gcc_assert (BLOCK_SUBBLOCKS (DECL_INITIAL (dest_cfun->decl))
8138	== NULL_TREE);
8139	BLOCK_SUBBLOCKS (DECL_INITIAL (dest_cfun->decl))
8140	= BLOCK_SUBBLOCKS (orig_block);
8141	for (block = BLOCK_SUBBLOCKS (orig_block);
8142	block; block = BLOCK_CHAIN (block))
8143	BLOCK_SUPERCONTEXT (block) = DECL_INITIAL (dest_cfun->decl);
8144	BLOCK_SUBBLOCKS (orig_block) = NULL_TREE;
8145	}
8146
8147	replace_block_vars_by_duplicates (DECL_INITIAL (dest_cfun->decl),
8148	vars_map: &vars_map, to_context: dest_cfun->decl);
8149
8150	if (new_label_map)
8151	htab_delete (new_label_map);
8152	if (eh_map)
8153	delete eh_map;
8154
8155	/* We need to release ssa-names in a defined order, so first find them,
8156	and then iterate in ascending version order. */
8157	bitmap release_names = BITMAP_ALLOC (NULL);
8158	vars_map.traverse<void *, gather_ssa_name_hash_map_from> (a: release_names);
8159	bitmap_iterator bi;
8160	EXECUTE_IF_SET_IN_BITMAP (release_names, 0, i, bi)
8161	release_ssa_name (ssa_name (i));
8162	BITMAP_FREE (release_names);
8163
8164	/* Rewire the entry and exit blocks. The successor to the entry
8165	block turns into the successor of DEST_FN's ENTRY_BLOCK_PTR in
8166	the child function. Similarly, the predecessor of DEST_FN's
8167	EXIT_BLOCK_PTR turns into the predecessor of EXIT_BLOCK_PTR. We
8168	need to switch CFUN between DEST_CFUN and SAVED_CFUN so that the
8169	various CFG manipulation function get to the right CFG.
8170
8171	FIXME, this is silly. The CFG ought to become a parameter to
8172	these helpers. */
8173	push_cfun (new_cfun: dest_cfun);
8174	ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = entry_bb->count;
8175	make_single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun), entry_bb, EDGE_FALLTHRU);
8176	if (exit_bb)
8177	{
8178	make_single_succ_edge (exit_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
8179	EXIT_BLOCK_PTR_FOR_FN (cfun)->count = exit_bb->count;
8180	}
8181	else
8182	EXIT_BLOCK_PTR_FOR_FN (cfun)->count = profile_count::zero ();
8183	pop_cfun ();
8184
8185	/* Back in the original function, the SESE region has disappeared,
8186	create a new basic block in its place. */
8187	bb = create_empty_bb (entry_pred[0]);
8188	if (current_loops)
8189	add_bb_to_loop (bb, loop);
8190	profile_count count = profile_count::zero ();
8191	for (i = 0; i < num_entry_edges; i++)
8192	{
8193	e = make_edge (entry_pred[i], bb, entry_flag[i]);
8194	e->probability = entry_prob[i];
8195	count += e->count ();
8196	}
8197	bb->count = count;
8198
8199	for (i = 0; i < num_exit_edges; i++)
8200	{
8201	e = make_edge (bb, exit_succ[i], exit_flag[i]);
8202	e->probability = exit_prob[i];
8203	}
8204
8205	set_immediate_dominator (CDI_DOMINATORS, bb, dom_entry);
8206	FOR_EACH_VEC_ELT (dom_bbs, i, abb)
8207	set_immediate_dominator (CDI_DOMINATORS, abb, bb);
8208
8209	if (exit_bb)
8210	{
8211	free (ptr: exit_prob);
8212	free (ptr: exit_flag);
8213	free (ptr: exit_succ);
8214	}
8215	free (ptr: entry_prob);
8216	free (ptr: entry_flag);
8217	free (ptr: entry_pred);
8218	bbs.release ();
8219
8220	return bb;
8221	}
8222
8223	/* Dump default def DEF to file FILE using FLAGS and indentation
8224	SPC. */
8225
8226	static void
8227	dump_default_def (FILE *file, tree def, int spc, dump_flags_t flags)
8228	{
8229	for (int i = 0; i < spc; ++i)
8230	fprintf (stream: file, format: " ");
8231	dump_ssaname_info_to_file (file, def, spc);
8232
8233	print_generic_expr (file, TREE_TYPE (def), flags);
8234	fprintf (stream: file, format: " ");
8235	print_generic_expr (file, def, flags);
8236	fprintf (stream: file, format: " = ");
8237	print_generic_expr (file, SSA_NAME_VAR (def), flags);
8238	fprintf (stream: file, format: ";\n");
8239	}
8240
8241	/* Print no_sanitize attribute to FILE for a given attribute VALUE. */
8242
8243	static void
8244	print_no_sanitize_attr_value (FILE *file, tree value)
8245	{
8246	unsigned int flags = tree_to_uhwi (value);
8247	bool first = true;
8248	for (int i = 0; sanitizer_opts[i].name != NULL; ++i)
8249	{
8250	if ((sanitizer_opts[i].flag & flags) == sanitizer_opts[i].flag)
8251	{
8252	if (!first)
8253	fprintf (stream: file, format: " | ");
8254	fprintf (stream: file, format: "%s", sanitizer_opts[i].name);
8255	first = false;
8256	}
8257	}
8258	}
8259
8260	/* Dump FUNCTION_DECL FN to file FILE using FLAGS (see TDF_* in dumpfile.h)
8261	*/
8262
8263	void
8264	dump_function_to_file (tree fndecl, FILE *file, dump_flags_t flags)
8265	{
8266	tree arg, var, old_current_fndecl = current_function_decl;
8267	struct function *dsf;
8268	bool ignore_topmost_bind = false, any_var = false;
8269	basic_block bb;
8270	tree chain;
8271	bool tmclone = (TREE_CODE (fndecl) == FUNCTION_DECL
8272	&& decl_is_tm_clone (fndecl));
8273	struct function *fun = DECL_STRUCT_FUNCTION (fndecl);
8274
8275	tree fntype = TREE_TYPE (fndecl);
8276	tree attrs[] = { DECL_ATTRIBUTES (fndecl), TYPE_ATTRIBUTES (fntype) };
8277
8278	for (int i = 0; i != 2; ++i)
8279	{
8280	if (!attrs[i])
8281	continue;
8282
8283	fprintf (stream: file, format: "__attribute__((");
8284
8285	bool first = true;
8286	tree chain;
8287	for (chain = attrs[i]; chain; first = false, chain = TREE_CHAIN (chain))
8288	{
8289	if (!first)
8290	fprintf (stream: file, format: ", ");
8291
8292	tree name = get_attribute_name (chain);
8293	print_generic_expr (file, name, dump_flags);
8294	if (TREE_VALUE (chain) != NULL_TREE)
8295	{
8296	fprintf (stream: file, format: " (");
8297
8298	if (strstr (IDENTIFIER_POINTER (name), needle: "no_sanitize"))
8299	print_no_sanitize_attr_value (file, TREE_VALUE (chain));
8300	else if (!strcmp (IDENTIFIER_POINTER (name),
8301	s2: "omp declare variant base"))
8302	{
8303	tree a = TREE_VALUE (chain);
8304	print_generic_expr (file, TREE_PURPOSE (a), dump_flags);
8305	fprintf (stream: file, format: " match ");
8306	print_omp_context_selector (file, TREE_VALUE (a),
8307	dump_flags);
8308	}
8309	else
8310	print_generic_expr (file, TREE_VALUE (chain), dump_flags);
8311	fprintf (stream: file, format: ")");
8312	}
8313	}
8314
8315	fprintf (stream: file, format: "))\n");
8316	}
8317
8318	current_function_decl = fndecl;
8319	if (flags & TDF_GIMPLE)
8320	{
8321	static bool hotness_bb_param_printed = false;
8322	if (profile_info != NULL
8323	&& !hotness_bb_param_printed)
8324	{
8325	hotness_bb_param_printed = true;
8326	fprintf (stream: file,
8327	format: "/* --param=gimple-fe-computed-hot-bb-threshold=%" PRId64
8328	" */\n", get_hot_bb_threshold ());
8329	}
8330
8331	print_generic_expr (file, TREE_TYPE (TREE_TYPE (fndecl)),
8332	dump_flags | TDF_SLIM);
8333	fprintf (stream: file, format: " __GIMPLE (%s",
8334	(fun->curr_properties & PROP_ssa) ? "ssa"
8335	: (fun->curr_properties & PROP_cfg) ? "cfg"
8336	: "");
8337
8338	if (fun && fun->cfg)
8339	{
8340	basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (fun);
8341	if (bb->count.initialized_p ())
8342	fprintf (stream: file, format: ",%s(%" PRIu64 ")",
8343	profile_quality_as_string (bb->count.quality ()),
8344	bb->count.value ());
8345	if (dump_flags & TDF_UID)
8346	fprintf (stream: file, format: ")\n%sD_%u (", function_name (fun),
8347	DECL_UID (fndecl));
8348	else
8349	fprintf (stream: file, format: ")\n%s (", function_name (fun));
8350	}
8351	}
8352	else
8353	{
8354	print_generic_expr (file, TREE_TYPE (fntype), dump_flags);
8355	if (dump_flags & TDF_UID)
8356	fprintf (stream: file, format: " %sD.%u %s(", function_name (fun), DECL_UID (fndecl),
8357	tmclone ? "[tm-clone] " : "");
8358	else
8359	fprintf (stream: file, format: " %s %s(", function_name (fun),
8360	tmclone ? "[tm-clone] " : "");
8361	}
8362
8363	arg = DECL_ARGUMENTS (fndecl);
8364	while (arg)
8365	{
8366	print_generic_expr (file, TREE_TYPE (arg), dump_flags);
8367	fprintf (stream: file, format: " ");
8368	print_generic_expr (file, arg, dump_flags);
8369	if (DECL_CHAIN (arg))
8370	fprintf (stream: file, format: ", ");
8371	arg = DECL_CHAIN (arg);
8372	}
8373	fprintf (stream: file, format: ")\n");
8374
8375	dsf = DECL_STRUCT_FUNCTION (fndecl);
8376	if (dsf && (flags & TDF_EH))
8377	dump_eh_tree (file, dsf);
8378
8379	if (flags & TDF_RAW && !gimple_has_body_p (fndecl))
8380	{
8381	dump_node (fndecl, TDF_SLIM | flags, file);
8382	current_function_decl = old_current_fndecl;
8383	return;
8384	}
8385
8386	/* When GIMPLE is lowered, the variables are no longer available in
8387	BIND_EXPRs, so display them separately. */
8388	if (fun && fun->decl == fndecl && (fun->curr_properties & PROP_gimple_lcf))
8389	{
8390	unsigned ix;
8391	ignore_topmost_bind = true;
8392
8393	fprintf (stream: file, format: "{\n");
8394	if (gimple_in_ssa_p (fun)
8395	&& (flags & TDF_ALIAS))
8396	{
8397	for (arg = DECL_ARGUMENTS (fndecl); arg != NULL;
8398	arg = DECL_CHAIN (arg))
8399	{
8400	tree def = ssa_default_def (fun, arg);
8401	if (def)
8402	dump_default_def (file, def, spc: 2, flags);
8403	}
8404
8405	tree res = DECL_RESULT (fun->decl);
8406	if (res != NULL_TREE
8407	&& DECL_BY_REFERENCE (res))
8408	{
8409	tree def = ssa_default_def (fun, res);
8410	if (def)
8411	dump_default_def (file, def, spc: 2, flags);
8412	}
8413
8414	tree static_chain = fun->static_chain_decl;
8415	if (static_chain != NULL_TREE)
8416	{
8417	tree def = ssa_default_def (fun, static_chain);
8418	if (def)
8419	dump_default_def (file, def, spc: 2, flags);
8420	}
8421	}
8422
8423	if (!vec_safe_is_empty (v: fun->local_decls))
8424	FOR_EACH_LOCAL_DECL (fun, ix, var)
8425	{
8426	print_generic_decl (file, var, flags);
8427	fprintf (stream: file, format: "\n");
8428
8429	any_var = true;
8430	}
8431
8432	tree name;
8433
8434	if (gimple_in_ssa_p (fun))
8435	FOR_EACH_SSA_NAME (ix, name, fun)
8436	{
8437	if (!SSA_NAME_VAR (name)
8438	/* SSA name with decls without a name still get
8439	dumped as _N, list those explicitely as well even
8440	though we've dumped the decl declaration as D.xxx
8441	above. */
8442	|| !SSA_NAME_IDENTIFIER (name))
8443	{
8444	fprintf (stream: file, format: " ");
8445	print_generic_expr (file, TREE_TYPE (name), flags);
8446	fprintf (stream: file, format: " ");
8447	print_generic_expr (file, name, flags);
8448	fprintf (stream: file, format: ";\n");
8449
8450	any_var = true;
8451	}
8452	}
8453	}
8454
8455	if (fun && fun->decl == fndecl
8456	&& fun->cfg
8457	&& basic_block_info_for_fn (fun))
8458	{
8459	/* If the CFG has been built, emit a CFG-based dump. */
8460	if (!ignore_topmost_bind)
8461	fprintf (stream: file, format: "{\n");
8462
8463	if (any_var && n_basic_blocks_for_fn (fun))
8464	fprintf (stream: file, format: "\n");
8465
8466	FOR_EACH_BB_FN (bb, fun)
8467	dump_bb (file, bb, 2, flags);
8468
8469	fprintf (stream: file, format: "}\n");
8470	}
8471	else if (fun && (fun->curr_properties & PROP_gimple_any))
8472	{
8473	/* The function is now in GIMPLE form but the CFG has not been
8474	built yet. Emit the single sequence of GIMPLE statements
8475	that make up its body. */
8476	gimple_seq body = gimple_body (fndecl);
8477
8478	if (gimple_seq_first_stmt (s: body)
8479	&& gimple_seq_first_stmt (s: body) == gimple_seq_last_stmt (s: body)
8480	&& gimple_code (g: gimple_seq_first_stmt (s: body)) == GIMPLE_BIND)
8481	print_gimple_seq (file, body, 0, flags);
8482	else
8483	{
8484	if (!ignore_topmost_bind)
8485	fprintf (stream: file, format: "{\n");
8486
8487	if (any_var)
8488	fprintf (stream: file, format: "\n");
8489
8490	print_gimple_seq (file, body, 2, flags);
8491	fprintf (stream: file, format: "}\n");
8492	}
8493	}
8494	else
8495	{
8496	int indent;
8497
8498	/* Make a tree based dump. */
8499	chain = DECL_SAVED_TREE (fndecl);
8500	if (chain && TREE_CODE (chain) == BIND_EXPR)
8501	{
8502	if (ignore_topmost_bind)
8503	{
8504	chain = BIND_EXPR_BODY (chain);
8505	indent = 2;
8506	}
8507	else
8508	indent = 0;
8509	}
8510	else
8511	{
8512	if (!ignore_topmost_bind)
8513	{
8514	fprintf (stream: file, format: "{\n");
8515	/* No topmost bind, pretend it's ignored for later. */
8516	ignore_topmost_bind = true;
8517	}
8518	indent = 2;
8519	}
8520
8521	if (any_var)
8522	fprintf (stream: file, format: "\n");
8523
8524	print_generic_stmt_indented (file, chain, flags, indent);
8525	if (ignore_topmost_bind)
8526	fprintf (stream: file, format: "}\n");
8527	}
8528
8529	if (flags & TDF_ENUMERATE_LOCALS)
8530	dump_enumerated_decls (file, flags);
8531	fprintf (stream: file, format: "\n\n");
8532
8533	current_function_decl = old_current_fndecl;
8534	}
8535
8536	/* Dump FUNCTION_DECL FN to stderr using FLAGS (see TDF_* in tree.h) */
8537
8538	DEBUG_FUNCTION void
8539	debug_function (tree fn, dump_flags_t flags)
8540	{
8541	dump_function_to_file (fndecl: fn, stderr, flags);
8542	}
8543
8544
8545	/* Print on FILE the indexes for the predecessors of basic_block BB. */
8546
8547	static void
8548	print_pred_bbs (FILE *file, basic_block bb)
8549	{
8550	edge e;
8551	edge_iterator ei;
8552
8553	FOR_EACH_EDGE (e, ei, bb->preds)
8554	fprintf (stream: file, format: "bb_%d ", e->src->index);
8555	}
8556
8557
8558	/* Print on FILE the indexes for the successors of basic_block BB. */
8559
8560	static void
8561	print_succ_bbs (FILE *file, basic_block bb)
8562	{
8563	edge e;
8564	edge_iterator ei;
8565
8566	FOR_EACH_EDGE (e, ei, bb->succs)
8567	fprintf (stream: file, format: "bb_%d ", e->dest->index);
8568	}
8569
8570	/* Print to FILE the basic block BB following the VERBOSITY level. */
8571
8572	void
8573	print_loops_bb (FILE *file, basic_block bb, int indent, int verbosity)
8574	{
8575	char *s_indent = (char *) alloca ((size_t) indent + 1);
8576	memset (s: (void *) s_indent, c: ' ', n: (size_t) indent);
8577	s_indent[indent] = '\0';
8578
8579	/* Print basic_block's header. */
8580	if (verbosity >= 2)
8581	{
8582	fprintf (stream: file, format: "%s bb_%d (preds = {", s_indent, bb->index);
8583	print_pred_bbs (file, bb);
8584	fprintf (stream: file, format: "}, succs = {");
8585	print_succ_bbs (file, bb);
8586	fprintf (stream: file, format: "})\n");
8587	}
8588
8589	/* Print basic_block's body. */
8590	if (verbosity >= 3)
8591	{
8592	fprintf (stream: file, format: "%s {\n", s_indent);
8593	dump_bb (file, bb, indent + 4, TDF_VOPS|TDF_MEMSYMS);
8594	fprintf (stream: file, format: "%s }\n", s_indent);
8595	}
8596	}
8597
8598	/* Print loop information. */
8599
8600	void
8601	print_loop_info (FILE *file, const class loop *loop, const char *prefix)
8602	{
8603	if (loop->can_be_parallel)
8604	fprintf (stream: file, format: ", can_be_parallel");
8605	if (loop->warned_aggressive_loop_optimizations)
8606	fprintf (stream: file, format: ", warned_aggressive_loop_optimizations");
8607	if (loop->dont_vectorize)
8608	fprintf (stream: file, format: ", dont_vectorize");
8609	if (loop->force_vectorize)
8610	fprintf (stream: file, format: ", force_vectorize");
8611	if (loop->in_oacc_kernels_region)
8612	fprintf (stream: file, format: ", in_oacc_kernels_region");
8613	if (loop->finite_p)
8614	fprintf (stream: file, format: ", finite_p");
8615	if (loop->unroll)
8616	fprintf (stream: file, format: "\n%sunroll %d", prefix, loop->unroll);
8617	if (loop->nb_iterations)
8618	{
8619	fprintf (stream: file, format: "\n%sniter ", prefix);
8620	print_generic_expr (file, loop->nb_iterations);
8621	}
8622
8623	if (loop->any_upper_bound)
8624	{
8625	fprintf (stream: file, format: "\n%supper_bound ", prefix);
8626	print_decu (wi: loop->nb_iterations_upper_bound, file);
8627	}
8628	if (loop->any_likely_upper_bound)
8629	{
8630	fprintf (stream: file, format: "\n%slikely_upper_bound ", prefix);
8631	print_decu (wi: loop->nb_iterations_likely_upper_bound, file);
8632	}
8633
8634	if (loop->any_estimate)
8635	{
8636	fprintf (stream: file, format: "\n%sestimate ", prefix);
8637	print_decu (wi: loop->nb_iterations_estimate, file);
8638	}
8639	bool reliable;
8640	sreal iterations;
8641	if (loop->num && expected_loop_iterations_by_profile (loop, ret: &iterations, reliable: &reliable))
8642	{
8643	fprintf (stream: file, format: "\n%siterations by profile: %f (%s%s) entry count:", prefix,
8644	iterations.to_double (), reliable ? "reliable" : "unreliable",
8645	maybe_flat_loop_profile (loop) ? ", maybe flat" : "");
8646	loop_count_in (loop).dump (f: file, cfun);
8647	}
8648
8649	}
8650
8651	static void print_loop_and_siblings (FILE *, class loop *, int, int);
8652
8653	/* Pretty print LOOP on FILE, indented INDENT spaces. Following
8654	VERBOSITY level this outputs the contents of the loop, or just its
8655	structure. */
8656
8657	static void
8658	print_loop (FILE *file, class loop *loop, int indent, int verbosity)
8659	{
8660	char *s_indent;
8661	basic_block bb;
8662
8663	if (loop == NULL)
8664	return;
8665
8666	s_indent = (char *) alloca ((size_t) indent + 1);
8667	memset (s: (void *) s_indent, c: ' ', n: (size_t) indent);
8668	s_indent[indent] = '\0';
8669
8670	/* Print loop's header. */
8671	fprintf (stream: file, format: "%sloop_%d (", s_indent, loop->num);
8672	if (loop->header)
8673	fprintf (stream: file, format: "header = %d", loop->header->index);
8674	else
8675	{
8676	fprintf (stream: file, format: "deleted)\n");
8677	return;
8678	}
8679	if (loop->latch)
8680	fprintf (stream: file, format: ", latch = %d", loop->latch->index);
8681	else
8682	fprintf (stream: file, format: ", multiple latches");
8683	print_loop_info (file, loop, prefix: s_indent);
8684	fprintf (stream: file, format: ")\n");
8685
8686	/* Print loop's body. */
8687	if (verbosity >= 1)
8688	{
8689	fprintf (stream: file, format: "%s{\n", s_indent);
8690	FOR_EACH_BB_FN (bb, cfun)
8691	if (bb->loop_father == loop)
8692	print_loops_bb (file, bb, indent, verbosity);
8693
8694	print_loop_and_siblings (file, loop->inner, indent + 2, verbosity);
8695	fprintf (stream: file, format: "%s}\n", s_indent);
8696	}
8697	}
8698
8699	/* Print the LOOP and its sibling loops on FILE, indented INDENT
8700	spaces. Following VERBOSITY level this outputs the contents of the
8701	loop, or just its structure. */
8702
8703	static void
8704	print_loop_and_siblings (FILE *file, class loop *loop, int indent,
8705	int verbosity)
8706	{
8707	if (loop == NULL)
8708	return;
8709
8710	print_loop (file, loop, indent, verbosity);
8711	print_loop_and_siblings (file, loop: loop->next, indent, verbosity);
8712	}
8713
8714	/* Follow a CFG edge from the entry point of the program, and on entry
8715	of a loop, pretty print the loop structure on FILE. */
8716
8717	void
8718	print_loops (FILE *file, int verbosity)
8719	{
8720	basic_block bb;
8721
8722	bb = ENTRY_BLOCK_PTR_FOR_FN (cfun);
8723	fprintf (stream: file, format: "\nLoops in function: %s\n", current_function_name ());
8724	if (bb && bb->loop_father)
8725	print_loop_and_siblings (file, loop: bb->loop_father, indent: 0, verbosity);
8726	}
8727
8728	/* Dump a loop. */
8729
8730	DEBUG_FUNCTION void
8731	debug (class loop &ref)
8732	{
8733	print_loop (stderr, loop: &ref, indent: 0, /*verbosity*/0);
8734	}
8735
8736	DEBUG_FUNCTION void
8737	debug (class loop *ptr)
8738	{
8739	if (ptr)
8740	debug (ref&: *ptr);
8741	else
8742	fprintf (stderr, format: "<nil>\n");
8743	}
8744
8745	/* Dump a loop verbosely. */
8746
8747	DEBUG_FUNCTION void
8748	debug_verbose (class loop &ref)
8749	{
8750	print_loop (stderr, loop: &ref, indent: 0, /*verbosity*/3);
8751	}
8752
8753	DEBUG_FUNCTION void
8754	debug_verbose (class loop *ptr)
8755	{
8756	if (ptr)
8757	debug (ref&: *ptr);
8758	else
8759	fprintf (stderr, format: "<nil>\n");
8760	}
8761
8762
8763	/* Debugging loops structure at tree level, at some VERBOSITY level. */
8764
8765	DEBUG_FUNCTION void
8766	debug_loops (int verbosity)
8767	{
8768	print_loops (stderr, verbosity);
8769	}
8770
8771	/* Print on stderr the code of LOOP, at some VERBOSITY level. */
8772
8773	DEBUG_FUNCTION void
8774	debug_loop (class loop *loop, int verbosity)
8775	{
8776	print_loop (stderr, loop, indent: 0, verbosity);
8777	}
8778
8779	/* Print on stderr the code of loop number NUM, at some VERBOSITY
8780	level. */
8781
8782	DEBUG_FUNCTION void
8783	debug_loop_num (unsigned num, int verbosity)
8784	{
8785	debug_loop (loop: get_loop (cfun, num), verbosity);
8786	}
8787
8788	/* Return true if BB ends with a call, possibly followed by some
8789	instructions that must stay with the call. Return false,
8790	otherwise. */
8791
8792	static bool
8793	gimple_block_ends_with_call_p (basic_block bb)
8794	{
8795	gimple_stmt_iterator gsi = gsi_last_nondebug_bb (bb);
8796	return !gsi_end_p (i: gsi) && is_gimple_call (gs: gsi_stmt (i: gsi));
8797	}
8798
8799
8800	/* Return true if BB ends with a conditional branch. Return false,
8801	otherwise. */
8802
8803	static bool
8804	gimple_block_ends_with_condjump_p (const_basic_block bb)
8805	{
8806	return safe_is_a <gcond *> (p: *gsi_last_bb (bb: const_cast <basic_block> (bb)));
8807	}
8808
8809
8810	/* Return true if statement T may terminate execution of BB in ways not
8811	explicitly represtented in the CFG. */
8812
8813	bool
8814	stmt_can_terminate_bb_p (gimple *t)
8815	{
8816	tree fndecl = NULL_TREE;
8817	int call_flags = 0;
8818
8819	/* Eh exception not handled internally terminates execution of the whole
8820	function. */
8821	if (stmt_can_throw_external (cfun, t))
8822	return true;
8823
8824	/* NORETURN and LONGJMP calls already have an edge to exit.
8825	CONST and PURE calls do not need one.
8826	We don't currently check for CONST and PURE here, although
8827	it would be a good idea, because those attributes are
8828	figured out from the RTL in mark_constant_function, and
8829	the counter incrementation code from -fprofile-arcs
8830	leads to different results from -fbranch-probabilities. */
8831	if (is_gimple_call (gs: t))
8832	{
8833	fndecl = gimple_call_fndecl (gs: t);
8834	call_flags = gimple_call_flags (t);
8835	}
8836
8837	if (is_gimple_call (gs: t)
8838	&& fndecl
8839	&& fndecl_built_in_p (node: fndecl)
8840	&& (call_flags & ECF_NOTHROW)
8841	&& !(call_flags & ECF_RETURNS_TWICE)
8842	/* fork() doesn't really return twice, but the effect of
8843	wrapping it in __gcov_fork() which calls __gcov_dump() and
8844	__gcov_reset() and clears the counters before forking has the same
8845	effect as returning twice. Force a fake edge. */
8846	&& !fndecl_built_in_p (node: fndecl, name1: BUILT_IN_FORK))
8847	return false;
8848
8849	if (is_gimple_call (gs: t))
8850	{
8851	edge_iterator ei;
8852	edge e;
8853	basic_block bb;
8854
8855	if (call_flags & (ECF_PURE | ECF_CONST)
8856	&& !(call_flags & ECF_LOOPING_CONST_OR_PURE))
8857	return false;
8858
8859	/* Function call may do longjmp, terminate program or do other things.
8860	Special case noreturn that have non-abnormal edges out as in this case
8861	the fact is sufficiently represented by lack of edges out of T. */
8862	if (!(call_flags & ECF_NORETURN))
8863	return true;
8864
8865	bb = gimple_bb (g: t);
8866	FOR_EACH_EDGE (e, ei, bb->succs)
8867	if ((e->flags & EDGE_FAKE) == 0)
8868	return true;
8869	}
8870
8871	if (gasm *asm_stmt = dyn_cast <gasm *> (p: t))
8872	if (gimple_asm_volatile_p (asm_stmt) || gimple_asm_input_p (asm_stmt))
8873	return true;
8874
8875	return false;
8876	}
8877
8878
8879	/* Add fake edges to the function exit for any non constant and non
8880	noreturn calls (or noreturn calls with EH/abnormal edges),
8881	volatile inline assembly in the bitmap of blocks specified by BLOCKS
8882	or to the whole CFG if BLOCKS is zero. Return the number of blocks
8883	that were split.
8884
8885	The goal is to expose cases in which entering a basic block does
8886	not imply that all subsequent instructions must be executed. */
8887
8888	static int
8889	gimple_flow_call_edges_add (sbitmap blocks)
8890	{
8891	int i;
8892	int blocks_split = 0;
8893	int last_bb = last_basic_block_for_fn (cfun);
8894	bool check_last_block = false;
8895
8896	if (n_basic_blocks_for_fn (cfun) == NUM_FIXED_BLOCKS)
8897	return 0;
8898
8899	if (! blocks)
8900	check_last_block = true;
8901	else
8902	check_last_block = bitmap_bit_p (map: blocks,
8903	EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb->index);
8904
8905	/* In the last basic block, before epilogue generation, there will be
8906	a fallthru edge to EXIT. Special care is required if the last insn
8907	of the last basic block is a call because make_edge folds duplicate
8908	edges, which would result in the fallthru edge also being marked
8909	fake, which would result in the fallthru edge being removed by
8910	remove_fake_edges, which would result in an invalid CFG.
8911
8912	Moreover, we can't elide the outgoing fake edge, since the block
8913	profiler needs to take this into account in order to solve the minimal
8914	spanning tree in the case that the call doesn't return.
8915
8916	Handle this by adding a dummy instruction in a new last basic block. */
8917	if (check_last_block)
8918	{
8919	basic_block bb = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
8920	gimple_stmt_iterator gsi = gsi_last_nondebug_bb (bb);
8921	gimple *t = NULL;
8922
8923	if (!gsi_end_p (i: gsi))
8924	t = gsi_stmt (i: gsi);
8925
8926	if (t && stmt_can_terminate_bb_p (t))
8927	{
8928	edge e;
8929
8930	e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun));
8931	if (e)
8932	{
8933	gsi_insert_on_edge (e, gimple_build_nop ());
8934	gsi_commit_edge_inserts ();
8935	}
8936	}
8937	}
8938
8939	/* Now add fake edges to the function exit for any non constant
8940	calls since there is no way that we can determine if they will
8941	return or not... */
8942	for (i = 0; i < last_bb; i++)
8943	{
8944	basic_block bb = BASIC_BLOCK_FOR_FN (cfun, i);
8945	gimple_stmt_iterator gsi;
8946	gimple *stmt, *last_stmt;
8947
8948	if (!bb)
8949	continue;
8950
8951	if (blocks && !bitmap_bit_p (map: blocks, bitno: i))
8952	continue;
8953
8954	gsi = gsi_last_nondebug_bb (bb);
8955	if (!gsi_end_p (i: gsi))
8956	{
8957	last_stmt = gsi_stmt (i: gsi);
8958	do
8959	{
8960	stmt = gsi_stmt (i: gsi);
8961	if (stmt_can_terminate_bb_p (t: stmt))
8962	{
8963	edge e;
8964
8965	/* The handling above of the final block before the
8966	epilogue should be enough to verify that there is
8967	no edge to the exit block in CFG already.
8968	Calling make_edge in such case would cause us to
8969	mark that edge as fake and remove it later. */
8970	if (flag_checking && stmt == last_stmt)
8971	{
8972	e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun));
8973	gcc_assert (e == NULL);
8974	}
8975
8976	/* Note that the following may create a new basic block
8977	and renumber the existing basic blocks. */
8978	if (stmt != last_stmt)
8979	{
8980	e = split_block (bb, stmt);
8981	if (e)
8982	blocks_split++;
8983	}
8984	e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_FAKE);
8985	e->probability = profile_probability::guessed_never ();
8986	}
8987	gsi_prev (i: &gsi);
8988	}
8989	while (!gsi_end_p (i: gsi));
8990	}
8991	}
8992
8993	if (blocks_split)
8994	checking_verify_flow_info ();
8995
8996	return blocks_split;
8997	}
8998
8999	/* Removes edge E and all the blocks dominated by it, and updates dominance
9000	information. The IL in E->src needs to be updated separately.
9001	If dominance info is not available, only the edge E is removed.*/
9002
9003	void
9004	remove_edge_and_dominated_blocks (edge e)
9005	{
9006	vec<basic_block> bbs_to_fix_dom = vNULL;
9007	edge f;
9008	edge_iterator ei;
9009	bool none_removed = false;
9010	unsigned i;
9011	basic_block bb, dbb;
9012	bitmap_iterator bi;
9013
9014	/* If we are removing a path inside a non-root loop that may change
9015	loop ownership of blocks or remove loops. Mark loops for fixup. */
9016	class loop *src_loop = e->src->loop_father;
9017	if (current_loops
9018	&& loop_outer (loop: src_loop) != NULL
9019	&& src_loop == e->dest->loop_father)
9020	{
9021	loops_state_set (flags: LOOPS_NEED_FIXUP);
9022	/* If we are removing a backedge clear the number of iterations
9023	and estimates. */
9024	class loop *dest_loop = e->dest->loop_father;
9025	if (e->dest == src_loop->header
9026	|| (e->dest == dest_loop->header
9027	&& flow_loop_nested_p (dest_loop, src_loop)))
9028	{
9029	free_numbers_of_iterations_estimates (dest_loop);
9030	/* If we removed the last backedge mark the loop for removal. */
9031	FOR_EACH_EDGE (f, ei, dest_loop->header->preds)
9032	if (f != e
9033	&& (f->src->loop_father == dest_loop
9034	|| flow_loop_nested_p (dest_loop, f->src->loop_father)))
9035	break;
9036	if (!f)
9037	mark_loop_for_removal (dest_loop);
9038	}
9039	}
9040
9041	if (!dom_info_available_p (CDI_DOMINATORS))
9042	{
9043	remove_edge (e);
9044	return;
9045	}
9046
9047	/* No updating is needed for edges to exit. */
9048	if (e->dest == EXIT_BLOCK_PTR_FOR_FN (cfun))
9049	{
9050	if (cfgcleanup_altered_bbs)
9051	bitmap_set_bit (cfgcleanup_altered_bbs, e->src->index);
9052	remove_edge (e);
9053	return;
9054	}
9055
9056	/* First, we find the basic blocks to remove. If E->dest has a predecessor
9057	that is not dominated by E->dest, then this set is empty. Otherwise,
9058	all the basic blocks dominated by E->dest are removed.
9059
9060	Also, to DF_IDOM we store the immediate dominators of the blocks in
9061	the dominance frontier of E (i.e., of the successors of the
9062	removed blocks, if there are any, and of E->dest otherwise). */
9063	FOR_EACH_EDGE (f, ei, e->dest->preds)
9064	{
9065	if (f == e)
9066	continue;
9067
9068	if (!dominated_by_p (CDI_DOMINATORS, f->src, e->dest))
9069	{
9070	none_removed = true;
9071	break;
9072	}
9073	}
9074
9075	auto_bitmap df, df_idom;
9076	auto_vec<basic_block> bbs_to_remove;
9077	if (none_removed)
9078	bitmap_set_bit (df_idom,
9079	get_immediate_dominator (CDI_DOMINATORS, e->dest)->index);
9080	else
9081	{
9082	bbs_to_remove = get_all_dominated_blocks (CDI_DOMINATORS, e->dest);
9083	FOR_EACH_VEC_ELT (bbs_to_remove, i, bb)
9084	{
9085	FOR_EACH_EDGE (f, ei, bb->succs)
9086	{
9087	if (f->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
9088	bitmap_set_bit (df, f->dest->index);
9089	}
9090	}
9091	FOR_EACH_VEC_ELT (bbs_to_remove, i, bb)
9092	bitmap_clear_bit (df, bb->index);
9093
9094	EXECUTE_IF_SET_IN_BITMAP (df, 0, i, bi)
9095	{
9096	bb = BASIC_BLOCK_FOR_FN (cfun, i);
9097	bitmap_set_bit (df_idom,
9098	get_immediate_dominator (CDI_DOMINATORS, bb)->index);
9099	}
9100	}
9101
9102	if (cfgcleanup_altered_bbs)
9103	{
9104	/* Record the set of the altered basic blocks. */
9105	bitmap_set_bit (cfgcleanup_altered_bbs, e->src->index);
9106	bitmap_ior_into (cfgcleanup_altered_bbs, df);
9107	}
9108
9109	/* Remove E and the cancelled blocks. */
9110	if (none_removed)
9111	remove_edge (e);
9112	else
9113	{
9114	/* Walk backwards so as to get a chance to substitute all
9115	released DEFs into debug stmts. See
9116	eliminate_unnecessary_stmts() in tree-ssa-dce.cc for more
9117	details. */
9118	for (i = bbs_to_remove.length (); i-- > 0; )
9119	delete_basic_block (bbs_to_remove[i]);
9120	}
9121
9122	/* Update the dominance information. The immediate dominator may change only
9123	for blocks whose immediate dominator belongs to DF_IDOM:
9124
9125	Suppose that idom(X) = Y before removal of E and idom(X) != Y after the
9126	removal. Let Z the arbitrary block such that idom(Z) = Y and
9127	Z dominates X after the removal. Before removal, there exists a path P
9128	from Y to X that avoids Z. Let F be the last edge on P that is
9129	removed, and let W = F->dest. Before removal, idom(W) = Y (since Y
9130	dominates W, and because of P, Z does not dominate W), and W belongs to
9131	the dominance frontier of E. Therefore, Y belongs to DF_IDOM. */
9132	EXECUTE_IF_SET_IN_BITMAP (df_idom, 0, i, bi)
9133	{
9134	bb = BASIC_BLOCK_FOR_FN (cfun, i);
9135	for (dbb = first_dom_son (CDI_DOMINATORS, bb);
9136	dbb;
9137	dbb = next_dom_son (CDI_DOMINATORS, dbb))
9138	bbs_to_fix_dom.safe_push (obj: dbb);
9139	}
9140
9141	iterate_fix_dominators (CDI_DOMINATORS, bbs_to_fix_dom, true);
9142
9143	bbs_to_fix_dom.release ();
9144	}
9145
9146	/* Purge dead EH edges from basic block BB. */
9147
9148	bool
9149	gimple_purge_dead_eh_edges (basic_block bb)
9150	{
9151	bool changed = false;
9152	edge e;
9153	edge_iterator ei;
9154	gimple *stmt = *gsi_last_bb (bb);
9155
9156	if (stmt && stmt_can_throw_internal (cfun, stmt))
9157	return false;
9158
9159	for (ei = ei_start (bb->succs); (e = ei_safe_edge (i: ei)); )
9160	{
9161	if (e->flags & EDGE_EH)
9162	{
9163	remove_edge_and_dominated_blocks (e);
9164	changed = true;
9165	}
9166	else
9167	ei_next (i: &ei);
9168	}
9169
9170	return changed;
9171	}
9172
9173	/* Purge dead EH edges from basic block listed in BLOCKS. */
9174
9175	bool
9176	gimple_purge_all_dead_eh_edges (const_bitmap blocks)
9177	{
9178	bool changed = false;
9179	unsigned i;
9180	bitmap_iterator bi;
9181
9182	EXECUTE_IF_SET_IN_BITMAP (blocks, 0, i, bi)
9183	{
9184	basic_block bb = BASIC_BLOCK_FOR_FN (cfun, i);
9185
9186	/* Earlier gimple_purge_dead_eh_edges could have removed
9187	this basic block already. */
9188	gcc_assert (bb || changed);
9189	if (bb != NULL)
9190	changed |= gimple_purge_dead_eh_edges (bb);
9191	}
9192
9193	return changed;
9194	}
9195
9196	/* Purge dead abnormal call edges from basic block BB. */
9197
9198	bool
9199	gimple_purge_dead_abnormal_call_edges (basic_block bb)
9200	{
9201	bool changed = false;
9202	edge e;
9203	edge_iterator ei;
9204	gimple *stmt = *gsi_last_bb (bb);
9205
9206	if (stmt && stmt_can_make_abnormal_goto (t: stmt))
9207	return false;
9208
9209	for (ei = ei_start (bb->succs); (e = ei_safe_edge (i: ei)); )
9210	{
9211	if (e->flags & EDGE_ABNORMAL)
9212	{
9213	if (e->flags & EDGE_FALLTHRU)
9214	e->flags &= ~EDGE_ABNORMAL;
9215	else
9216	remove_edge_and_dominated_blocks (e);
9217	changed = true;
9218	}
9219	else
9220	ei_next (i: &ei);
9221	}
9222
9223	return changed;
9224	}
9225
9226	/* Purge dead abnormal call edges from basic block listed in BLOCKS. */
9227
9228	bool
9229	gimple_purge_all_dead_abnormal_call_edges (const_bitmap blocks)
9230	{
9231	bool changed = false;
9232	unsigned i;
9233	bitmap_iterator bi;
9234
9235	EXECUTE_IF_SET_IN_BITMAP (blocks, 0, i, bi)
9236	{
9237	basic_block bb = BASIC_BLOCK_FOR_FN (cfun, i);
9238
9239	/* Earlier gimple_purge_dead_abnormal_call_edges could have removed
9240	this basic block already. */
9241	gcc_assert (bb || changed);
9242	if (bb != NULL)
9243	changed |= gimple_purge_dead_abnormal_call_edges (bb);
9244	}
9245
9246	return changed;
9247	}
9248
9249	/* This function is called whenever a new edge is created or
9250	redirected. */
9251
9252	static void
9253	gimple_execute_on_growing_pred (edge e)
9254	{
9255	basic_block bb = e->dest;
9256
9257	if (!gimple_seq_empty_p (s: phi_nodes (bb)))
9258	reserve_phi_args_for_new_edge (bb);
9259	}
9260
9261	/* This function is called immediately before edge E is removed from
9262	the edge vector E->dest->preds. */
9263
9264	static void
9265	gimple_execute_on_shrinking_pred (edge e)
9266	{
9267	if (!gimple_seq_empty_p (s: phi_nodes (bb: e->dest)))
9268	remove_phi_args (e);
9269	}
9270
9271	/*---------------------------------------------------------------------------
9272	Helper functions for Loop versioning
9273	---------------------------------------------------------------------------*/
9274
9275	/* Adjust phi nodes for 'first' basic block. 'second' basic block is a copy
9276	of 'first'. Both of them are dominated by 'new_head' basic block. When
9277	'new_head' was created by 'second's incoming edge it received phi arguments
9278	on the edge by split_edge(). Later, additional edge 'e' was created to
9279	connect 'new_head' and 'first'. Now this routine adds phi args on this
9280	additional edge 'e' that new_head to second edge received as part of edge
9281	splitting. */
9282
9283	static void
9284	gimple_lv_adjust_loop_header_phi (basic_block first, basic_block second,
9285	basic_block new_head, edge e)
9286	{
9287	gphi *phi1, *phi2;
9288	gphi_iterator psi1, psi2;
9289	tree def;
9290	edge e2 = find_edge (new_head, second);
9291
9292	/* Because NEW_HEAD has been created by splitting SECOND's incoming
9293	edge, we should always have an edge from NEW_HEAD to SECOND. */
9294	gcc_assert (e2 != NULL);
9295
9296	/* Browse all 'second' basic block phi nodes and add phi args to
9297	edge 'e' for 'first' head. PHI args are always in correct order. */
9298
9299	for (psi2 = gsi_start_phis (second),
9300	psi1 = gsi_start_phis (first);
9301	!gsi_end_p (i: psi2) && !gsi_end_p (i: psi1);
9302	gsi_next (i: &psi2), gsi_next (i: &psi1))
9303	{
9304	phi1 = psi1.phi ();
9305	phi2 = psi2.phi ();
9306	def = PHI_ARG_DEF (phi2, e2->dest_idx);
9307	add_phi_arg (phi1, def, e, gimple_phi_arg_location_from_edge (phi: phi2, e: e2));
9308	}
9309	}
9310
9311
9312	/* Adds a if else statement to COND_BB with condition COND_EXPR.
9313	SECOND_HEAD is the destination of the THEN and FIRST_HEAD is
9314	the destination of the ELSE part. */
9315
9316	static void
9317	gimple_lv_add_condition_to_bb (basic_block first_head ATTRIBUTE_UNUSED,
9318	basic_block second_head ATTRIBUTE_UNUSED,
9319	basic_block cond_bb, void *cond_e)
9320	{
9321	gimple_stmt_iterator gsi;
9322	gimple *new_cond_expr;
9323	tree cond_expr = (tree) cond_e;
9324	edge e0;
9325
9326	/* Build new conditional expr */
9327	gsi = gsi_last_bb (bb: cond_bb);
9328
9329	cond_expr = force_gimple_operand_gsi_1 (&gsi, cond_expr,
9330	is_gimple_condexpr_for_cond,
9331	NULL_TREE, false,
9332	GSI_CONTINUE_LINKING);
9333	new_cond_expr = gimple_build_cond_from_tree (cond_expr,
9334	NULL_TREE, NULL_TREE);
9335
9336	/* Add new cond in cond_bb. */
9337	gsi_insert_after (&gsi, new_cond_expr, GSI_NEW_STMT);
9338
9339	/* Adjust edges appropriately to connect new head with first head
9340	as well as second head. */
9341	e0 = single_succ_edge (bb: cond_bb);
9342	e0->flags &= ~EDGE_FALLTHRU;
9343	e0->flags |= EDGE_FALSE_VALUE;
9344	}
9345
9346
9347	/* Do book-keeping of basic block BB for the profile consistency checker.
9348	Store the counting in RECORD. */
9349	static void
9350	gimple_account_profile_record (basic_block bb,
9351	struct profile_record *record)
9352	{
9353	gimple_stmt_iterator i;
9354	for (i = gsi_start_nondebug_after_labels_bb (bb); !gsi_end_p (i);
9355	gsi_next_nondebug (i: &i))
9356	{
9357	record->size
9358	+= estimate_num_insns (gsi_stmt (i), &eni_size_weights);
9359	if (profile_info)
9360	{
9361	if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa ().initialized_p ()
9362	&& ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa ().nonzero_p ()
9363	&& bb->count.ipa ().initialized_p ())
9364	record->time
9365	+= estimate_num_insns (gsi_stmt (i),
9366	&eni_time_weights)
9367	* bb->count.ipa ().to_gcov_type ();
9368	}
9369	else if (bb->count.initialized_p ()
9370	&& ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.initialized_p ())
9371	record->time
9372	+= estimate_num_insns
9373	(gsi_stmt (i),
9374	&eni_time_weights)
9375	* bb->count.to_sreal_scale
9376	(ENTRY_BLOCK_PTR_FOR_FN (cfun)->count).to_double ();
9377	else
9378	record->time
9379	+= estimate_num_insns (gsi_stmt (i), &eni_time_weights);
9380	}
9381	}
9382
9383	struct cfg_hooks gimple_cfg_hooks = {
9384	.name: "gimple",
9385	.verify_flow_info: gimple_verify_flow_info,
9386	.dump_bb: gimple_dump_bb, /* dump_bb */
9387	.dump_bb_for_graph: gimple_dump_bb_for_graph, /* dump_bb_for_graph */
9388	.create_basic_block: create_bb, /* create_basic_block */
9389	.redirect_edge_and_branch: gimple_redirect_edge_and_branch, /* redirect_edge_and_branch */
9390	.redirect_edge_and_branch_force: gimple_redirect_edge_and_branch_force, /* redirect_edge_and_branch_force */
9391	.can_remove_branch_p: gimple_can_remove_branch_p, /* can_remove_branch_p */
9392	.delete_basic_block: remove_bb, /* delete_basic_block */
9393	.split_block: gimple_split_block, /* split_block */
9394	.move_block_after: gimple_move_block_after, /* move_block_after */
9395	.can_merge_blocks_p: gimple_can_merge_blocks_p, /* can_merge_blocks_p */
9396	.merge_blocks: gimple_merge_blocks, /* merge_blocks */
9397	.predict_edge: gimple_predict_edge, /* predict_edge */
9398	.predicted_by_p: gimple_predicted_by_p, /* predicted_by_p */
9399	.can_duplicate_block_p: gimple_can_duplicate_bb_p, /* can_duplicate_block_p */
9400	.duplicate_block: gimple_duplicate_bb, /* duplicate_block */
9401	.split_edge: gimple_split_edge, /* split_edge */
9402	.make_forwarder_block: gimple_make_forwarder_block, /* make_forward_block */
9403	NULL, /* tidy_fallthru_edge */
9404	NULL, /* force_nonfallthru */
9405	.block_ends_with_call_p: gimple_block_ends_with_call_p,/* block_ends_with_call_p */
9406	.block_ends_with_condjump_p: gimple_block_ends_with_condjump_p, /* block_ends_with_condjump_p */
9407	.flow_call_edges_add: gimple_flow_call_edges_add, /* flow_call_edges_add */
9408	.execute_on_growing_pred: gimple_execute_on_growing_pred, /* execute_on_growing_pred */
9409	.execute_on_shrinking_pred: gimple_execute_on_shrinking_pred, /* execute_on_shrinking_pred */
9410	.cfg_hook_duplicate_loop_body_to_header_edge: gimple_duplicate_loop_body_to_header_edge, /* duplicate loop for trees */
9411	.lv_add_condition_to_bb: gimple_lv_add_condition_to_bb, /* lv_add_condition_to_bb */
9412	.lv_adjust_loop_header_phi: gimple_lv_adjust_loop_header_phi, /* lv_adjust_loop_header_phi*/
9413	.extract_cond_bb_edges: extract_true_false_edges_from_block, /* extract_cond_bb_edges */
9414	.flush_pending_stmts: flush_pending_stmts, /* flush_pending_stmts */
9415	.empty_block_p: gimple_empty_block_p, /* block_empty_p */
9416	.split_block_before_cond_jump: gimple_split_block_before_cond_jump, /* split_block_before_cond_jump */
9417	.account_profile_record: gimple_account_profile_record,
9418	};
9419
9420
9421	/* Split all critical edges. Split some extra (not necessarily critical) edges
9422	if FOR_EDGE_INSERTION_P is true. */
9423
9424	unsigned int
9425	split_critical_edges (bool for_edge_insertion_p /* = false */)
9426	{
9427	basic_block bb;
9428	edge e;
9429	edge_iterator ei;
9430
9431	/* split_edge can redirect edges out of SWITCH_EXPRs, which can get
9432	expensive. So we want to enable recording of edge to CASE_LABEL_EXPR
9433	mappings around the calls to split_edge. */
9434	start_recording_case_labels ();
9435	FOR_ALL_BB_FN (bb, cfun)
9436	{
9437	FOR_EACH_EDGE (e, ei, bb->succs)
9438	{
9439	if (EDGE_CRITICAL_P (e) && !(e->flags & EDGE_ABNORMAL))
9440	split_edge (e);
9441	/* PRE inserts statements to edges and expects that
9442	since split_critical_edges was done beforehand, committing edge
9443	insertions will not split more edges. In addition to critical
9444	edges we must split edges that have multiple successors and
9445	end by control flow statements, such as RESX.
9446	Go ahead and split them too. This matches the logic in
9447	gimple_find_edge_insert_loc. */
9448	else if (for_edge_insertion_p
9449	&& (!single_pred_p (bb: e->dest)
9450	|| !gimple_seq_empty_p (s: phi_nodes (bb: e->dest))
9451	|| e->dest == EXIT_BLOCK_PTR_FOR_FN (cfun))
9452	&& e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun)
9453	&& !(e->flags & EDGE_ABNORMAL))
9454	{
9455	gimple_stmt_iterator gsi;
9456
9457	gsi = gsi_last_bb (bb: e->src);
9458	if (!gsi_end_p (i: gsi)
9459	&& stmt_ends_bb_p (t: gsi_stmt (i: gsi))
9460	&& (gimple_code (g: gsi_stmt (i: gsi)) != GIMPLE_RETURN
9461	&& !gimple_call_builtin_p (gsi_stmt (i: gsi),
9462	BUILT_IN_RETURN)))
9463	split_edge (e);
9464	}
9465	}
9466	}
9467	end_recording_case_labels ();
9468	return 0;
9469	}
9470
9471	namespace {
9472
9473	const pass_data pass_data_split_crit_edges =
9474	{
9475	.type: GIMPLE_PASS, /* type */
9476	.name: "crited", /* name */
9477	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
9478	.tv_id: TV_TREE_SPLIT_EDGES, /* tv_id */
9479	PROP_cfg, /* properties_required */
9480	PROP_no_crit_edges, /* properties_provided */
9481	.properties_destroyed: 0, /* properties_destroyed */
9482	.todo_flags_start: 0, /* todo_flags_start */
9483	.todo_flags_finish: 0, /* todo_flags_finish */
9484	};
9485
9486	class pass_split_crit_edges : public gimple_opt_pass
9487	{
9488	public:
9489	pass_split_crit_edges (gcc::context *ctxt)
9490	: gimple_opt_pass (pass_data_split_crit_edges, ctxt)
9491	{}
9492
9493	/* opt_pass methods: */
9494	unsigned int execute (function *) final override
9495	{
9496	return split_critical_edges ();
9497	}
9498
9499	opt_pass * clone () final override
9500	{
9501	return new pass_split_crit_edges (m_ctxt);
9502	}
9503	}; // class pass_split_crit_edges
9504
9505	} // anon namespace
9506
9507	gimple_opt_pass *
9508	make_pass_split_crit_edges (gcc::context *ctxt)
9509	{
9510	return new pass_split_crit_edges (ctxt);
9511	}
9512
9513
9514	/* Insert COND expression which is GIMPLE_COND after STMT
9515	in basic block BB with appropriate basic block split
9516	and creation of a new conditionally executed basic block.
9517	Update profile so the new bb is visited with probability PROB.
9518	Return created basic block. */
9519	basic_block
9520	insert_cond_bb (basic_block bb, gimple *stmt, gimple *cond,
9521	profile_probability prob)
9522	{
9523	edge fall = split_block (bb, stmt);
9524	gimple_stmt_iterator iter = gsi_last_bb (bb);
9525	basic_block new_bb;
9526
9527	/* Insert cond statement. */
9528	gcc_assert (gimple_code (cond) == GIMPLE_COND);
9529	if (gsi_end_p (i: iter))
9530	gsi_insert_before (&iter, cond, GSI_CONTINUE_LINKING);
9531	else
9532	gsi_insert_after (&iter, cond, GSI_CONTINUE_LINKING);
9533
9534	/* Create conditionally executed block. */
9535	new_bb = create_empty_bb (bb);
9536	edge e = make_edge (bb, new_bb, EDGE_TRUE_VALUE);
9537	e->probability = prob;
9538	new_bb->count = e->count ();
9539	make_single_succ_edge (new_bb, fall->dest, EDGE_FALLTHRU);
9540
9541	/* Fix edge for split bb. */
9542	fall->flags = EDGE_FALSE_VALUE;
9543	fall->probability -= e->probability;
9544
9545	/* Update dominance info. */
9546	if (dom_info_available_p (CDI_DOMINATORS))
9547	{
9548	set_immediate_dominator (CDI_DOMINATORS, new_bb, bb);
9549	set_immediate_dominator (CDI_DOMINATORS, fall->dest, bb);
9550	}
9551
9552	/* Update loop info. */
9553	if (current_loops)
9554	add_bb_to_loop (new_bb, bb->loop_father);
9555
9556	return new_bb;
9557	}
9558
9559
9560
9561	/* Given a basic block B which ends with a conditional and has
9562	precisely two successors, determine which of the edges is taken if
9563	the conditional is true and which is taken if the conditional is
9564	false. Set TRUE_EDGE and FALSE_EDGE appropriately. */
9565
9566	void
9567	extract_true_false_edges_from_block (basic_block b,
9568	edge *true_edge,
9569	edge *false_edge)
9570	{
9571	edge e = EDGE_SUCC (b, 0);
9572
9573	if (e->flags & EDGE_TRUE_VALUE)
9574	{
9575	*true_edge = e;
9576	*false_edge = EDGE_SUCC (b, 1);
9577	}
9578	else
9579	{
9580	*false_edge = e;
9581	*true_edge = EDGE_SUCC (b, 1);
9582	}
9583	}
9584
9585
9586	/* From a controlling predicate in the immediate dominator DOM of
9587	PHIBLOCK determine the edges into PHIBLOCK that are chosen if the
9588	predicate evaluates to true and false and store them to
9589	*TRUE_CONTROLLED_EDGE and *FALSE_CONTROLLED_EDGE if
9590	they are non-NULL. Returns true if the edges can be determined,
9591	else return false. */
9592
9593	bool
9594	extract_true_false_controlled_edges (basic_block dom, basic_block phiblock,
9595	edge *true_controlled_edge,
9596	edge *false_controlled_edge)
9597	{
9598	basic_block bb = phiblock;
9599	edge true_edge, false_edge, tem;
9600	edge e0 = NULL, e1 = NULL;
9601
9602	/* We have to verify that one edge into the PHI node is dominated
9603	by the true edge of the predicate block and the other edge
9604	dominated by the false edge. This ensures that the PHI argument
9605	we are going to take is completely determined by the path we
9606	take from the predicate block.
9607	We can only use BB dominance checks below if the destination of
9608	the true/false edges are dominated by their edge, thus only
9609	have a single predecessor. */
9610	extract_true_false_edges_from_block (b: dom, true_edge: &true_edge, false_edge: &false_edge);
9611	tem = EDGE_PRED (bb, 0);
9612	if (tem == true_edge
9613	|| (single_pred_p (bb: true_edge->dest)
9614	&& (tem->src == true_edge->dest
9615	|| dominated_by_p (CDI_DOMINATORS,
9616	tem->src, true_edge->dest))))
9617	e0 = tem;
9618	else if (tem == false_edge
9619	|| (single_pred_p (bb: false_edge->dest)
9620	&& (tem->src == false_edge->dest
9621	|| dominated_by_p (CDI_DOMINATORS,
9622	tem->src, false_edge->dest))))
9623	e1 = tem;
9624	else
9625	return false;
9626	tem = EDGE_PRED (bb, 1);
9627	if (tem == true_edge
9628	|| (single_pred_p (bb: true_edge->dest)
9629	&& (tem->src == true_edge->dest
9630	|| dominated_by_p (CDI_DOMINATORS,
9631	tem->src, true_edge->dest))))
9632	e0 = tem;
9633	else if (tem == false_edge
9634	|| (single_pred_p (bb: false_edge->dest)
9635	&& (tem->src == false_edge->dest
9636	|| dominated_by_p (CDI_DOMINATORS,
9637	tem->src, false_edge->dest))))
9638	e1 = tem;
9639	else
9640	return false;
9641	if (!e0 || !e1)
9642	return false;
9643
9644	if (true_controlled_edge)
9645	*true_controlled_edge = e0;
9646	if (false_controlled_edge)
9647	*false_controlled_edge = e1;
9648
9649	return true;
9650	}
9651
9652	/* Generate a range test LHS CODE RHS that determines whether INDEX is in the
9653	range [low, high]. Place associated stmts before *GSI. */
9654
9655	void
9656	generate_range_test (basic_block bb, tree index, tree low, tree high,
9657	tree *lhs, tree *rhs)
9658	{
9659	tree type = TREE_TYPE (index);
9660	tree utype = range_check_type (type);
9661
9662	low = fold_convert (utype, low);
9663	high = fold_convert (utype, high);
9664
9665	gimple_seq seq = NULL;
9666	index = gimple_convert (seq: &seq, type: utype, op: index);
9667	*lhs = gimple_build (seq: &seq, code: MINUS_EXPR, type: utype, ops: index, ops: low);
9668	*rhs = const_binop (MINUS_EXPR, utype, high, low);
9669
9670	gimple_stmt_iterator gsi = gsi_last_bb (bb);
9671	gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
9672	}
9673
9674	/* Return the basic block that belongs to label numbered INDEX
9675	of a switch statement. */
9676
9677	basic_block
9678	gimple_switch_label_bb (function *ifun, gswitch *gs, unsigned index)
9679	{
9680	return label_to_block (ifun, CASE_LABEL (gimple_switch_label (gs, index)));
9681	}
9682
9683	/* Return the default basic block of a switch statement. */
9684
9685	basic_block
9686	gimple_switch_default_bb (function *ifun, gswitch *gs)
9687	{
9688	return gimple_switch_label_bb (ifun, gs, index: 0);
9689	}
9690
9691	/* Return the edge that belongs to label numbered INDEX
9692	of a switch statement. */
9693
9694	edge
9695	gimple_switch_edge (function *ifun, gswitch *gs, unsigned index)
9696	{
9697	return find_edge (gimple_bb (g: gs), gimple_switch_label_bb (ifun, gs, index));
9698	}
9699
9700	/* Return the default edge of a switch statement. */
9701
9702	edge
9703	gimple_switch_default_edge (function *ifun, gswitch *gs)
9704	{
9705	return gimple_switch_edge (ifun, gs, index: 0);
9706	}
9707
9708	/* Return true if the only executable statement in BB is a GIMPLE_COND. */
9709
9710	bool
9711	cond_only_block_p (basic_block bb)
9712	{
9713	/* BB must have no executable statements. */
9714	gimple_stmt_iterator gsi = gsi_after_labels (bb);
9715	if (phi_nodes (bb))
9716	return false;
9717	while (!gsi_end_p (i: gsi))
9718	{
9719	gimple *stmt = gsi_stmt (i: gsi);
9720	if (is_gimple_debug (gs: stmt))
9721	;
9722	else if (gimple_code (g: stmt) == GIMPLE_NOP
9723	|| gimple_code (g: stmt) == GIMPLE_PREDICT
9724	|| gimple_code (g: stmt) == GIMPLE_COND)
9725	;
9726	else
9727	return false;
9728	gsi_next (i: &gsi);
9729	}
9730	return true;
9731	}
9732
9733
9734	/* Emit return warnings. */
9735
9736	namespace {
9737
9738	const pass_data pass_data_warn_function_return =
9739	{
9740	.type: GIMPLE_PASS, /* type */
9741	.name: "*warn_function_return", /* name */
9742	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
9743	.tv_id: TV_NONE, /* tv_id */
9744	PROP_cfg, /* properties_required */
9745	.properties_provided: 0, /* properties_provided */
9746	.properties_destroyed: 0, /* properties_destroyed */
9747	.todo_flags_start: 0, /* todo_flags_start */
9748	.todo_flags_finish: 0, /* todo_flags_finish */
9749	};
9750
9751	class pass_warn_function_return : public gimple_opt_pass
9752	{
9753	public:
9754	pass_warn_function_return (gcc::context *ctxt)
9755	: gimple_opt_pass (pass_data_warn_function_return, ctxt)
9756	{}
9757
9758	/* opt_pass methods: */
9759	unsigned int execute (function *) final override;
9760
9761	}; // class pass_warn_function_return
9762
9763	unsigned int
9764	pass_warn_function_return::execute (function *fun)
9765	{
9766	location_t location;
9767	gimple *last;
9768	edge e;
9769	edge_iterator ei;
9770
9771	if (!targetm.warn_func_return (fun->decl))
9772	return 0;
9773
9774	/* If we have a path to EXIT, then we do return. */
9775	if (TREE_THIS_VOLATILE (fun->decl)
9776	&& EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (fun)->preds) > 0)
9777	{
9778	location = UNKNOWN_LOCATION;
9779	for (ei = ei_start (EXIT_BLOCK_PTR_FOR_FN (fun)->preds);
9780	(e = ei_safe_edge (i: ei)); )
9781	{
9782	last = *gsi_last_bb (bb: e->src);
9783	if ((gimple_code (g: last) == GIMPLE_RETURN
9784	|| gimple_call_builtin_p (last, BUILT_IN_RETURN))
9785	&& location == UNKNOWN_LOCATION
9786	&& ((location = LOCATION_LOCUS (gimple_location (last)))
9787	!= UNKNOWN_LOCATION)
9788	&& !optimize)
9789	break;
9790	/* When optimizing, replace return stmts in noreturn functions
9791	with __builtin_unreachable () call. */
9792	if (optimize && gimple_code (g: last) == GIMPLE_RETURN)
9793	{
9794	location_t loc = gimple_location (g: last);
9795	gimple *new_stmt = gimple_build_builtin_unreachable (loc);
9796	gimple_stmt_iterator gsi = gsi_for_stmt (last);
9797	gsi_replace (&gsi, new_stmt, true);
9798	remove_edge (e);
9799	}
9800	else
9801	ei_next (i: &ei);
9802	}
9803	if (location == UNKNOWN_LOCATION)
9804	location = cfun->function_end_locus;
9805	warning_at (location, 0, "%<noreturn%> function does return");
9806	}
9807
9808	/* If we see "return;" in some basic block, then we do reach the end
9809	without returning a value. */
9810	else if (warn_return_type > 0
9811	&& !warning_suppressed_p (fun->decl, OPT_Wreturn_type)
9812	&& !VOID_TYPE_P (TREE_TYPE (TREE_TYPE (fun->decl))))
9813	{
9814	FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (fun)->preds)
9815	{
9816	greturn *return_stmt = dyn_cast <greturn *> (p: *gsi_last_bb (bb: e->src));
9817	if (return_stmt
9818	&& gimple_return_retval (gs: return_stmt) == NULL
9819	&& !warning_suppressed_p (return_stmt, OPT_Wreturn_type))
9820	{
9821	location = gimple_location (g: return_stmt);
9822	if (LOCATION_LOCUS (location) == UNKNOWN_LOCATION)
9823	location = fun->function_end_locus;
9824	if (warning_at (location, OPT_Wreturn_type,
9825	"control reaches end of non-void function"))
9826	suppress_warning (fun->decl, OPT_Wreturn_type);
9827	break;
9828	}
9829	}
9830	/* The C++ FE turns fallthrough from the end of non-void function
9831	into __builtin_unreachable () call with BUILTINS_LOCATION.
9832	Recognize those as well as calls from ubsan_instrument_return. */
9833	basic_block bb;
9834	if (!warning_suppressed_p (fun->decl, OPT_Wreturn_type))
9835	FOR_EACH_BB_FN (bb, fun)
9836	if (EDGE_COUNT (bb->succs) == 0)
9837	{
9838	gimple *last = *gsi_last_bb (bb);
9839	const enum built_in_function ubsan_missing_ret
9840	= BUILT_IN_UBSAN_HANDLE_MISSING_RETURN;
9841	if (last
9842	&& ((LOCATION_LOCUS (gimple_location (last))
9843	== BUILTINS_LOCATION
9844	&& (gimple_call_builtin_p (last, BUILT_IN_UNREACHABLE)
9845	|| gimple_call_builtin_p (last,
9846	BUILT_IN_UNREACHABLE_TRAP)
9847	|| gimple_call_builtin_p (last, BUILT_IN_TRAP)))
9848	|| gimple_call_builtin_p (last, ubsan_missing_ret)))
9849	{
9850	gimple_stmt_iterator gsi = gsi_for_stmt (last);
9851	gsi_prev_nondebug (i: &gsi);
9852	gimple *prev = gsi_stmt (i: gsi);
9853	if (prev == NULL)
9854	location = UNKNOWN_LOCATION;
9855	else
9856	location = gimple_location (g: prev);
9857	if (LOCATION_LOCUS (location) == UNKNOWN_LOCATION)
9858	location = fun->function_end_locus;
9859	if (warning_at (location, OPT_Wreturn_type,
9860	"control reaches end of non-void function"))
9861	suppress_warning (fun->decl, OPT_Wreturn_type);
9862	break;
9863	}
9864	}
9865	}
9866	return 0;
9867	}
9868
9869	} // anon namespace
9870
9871	gimple_opt_pass *
9872	make_pass_warn_function_return (gcc::context *ctxt)
9873	{
9874	return new pass_warn_function_return (ctxt);
9875	}
9876
9877	/* Walk a gimplified function and warn for functions whose return value is
9878	ignored and attribute((warn_unused_result)) is set. This is done before
9879	inlining, so we don't have to worry about that. */
9880
9881	static void
9882	do_warn_unused_result (gimple_seq seq)
9883	{
9884	tree fdecl, ftype;
9885	gimple_stmt_iterator i;
9886
9887	for (i = gsi_start (seq); !gsi_end_p (i); gsi_next (i: &i))
9888	{
9889	gimple *g = gsi_stmt (i);
9890
9891	switch (gimple_code (g))
9892	{
9893	case GIMPLE_BIND:
9894	do_warn_unused_result (seq: gimple_bind_body (gs: as_a <gbind *>(p: g)));
9895	break;
9896	case GIMPLE_TRY:
9897	do_warn_unused_result (seq: gimple_try_eval (gs: g));
9898	do_warn_unused_result (seq: gimple_try_cleanup (gs: g));
9899	break;
9900	case GIMPLE_CATCH:
9901	do_warn_unused_result (seq: gimple_catch_handler (
9902	catch_stmt: as_a <gcatch *> (p: g)));
9903	break;
9904	case GIMPLE_EH_FILTER:
9905	do_warn_unused_result (seq: gimple_eh_filter_failure (gs: g));
9906	break;
9907
9908	case GIMPLE_CALL:
9909	if (gimple_call_lhs (gs: g))
9910	break;
9911	if (gimple_call_internal_p (gs: g))
9912	break;
9913
9914	/* This is a naked call, as opposed to a GIMPLE_CALL with an
9915	LHS. All calls whose value is ignored should be
9916	represented like this. Look for the attribute. */
9917	fdecl = gimple_call_fndecl (gs: g);
9918	ftype = gimple_call_fntype (gs: g);
9919
9920	if (lookup_attribute (attr_name: "warn_unused_result", TYPE_ATTRIBUTES (ftype)))
9921	{
9922	location_t loc = gimple_location (g);
9923
9924	if (fdecl)
9925	warning_at (loc, OPT_Wunused_result,
9926	"ignoring return value of %qD "
9927	"declared with attribute %<warn_unused_result%>",
9928	fdecl);
9929	else
9930	warning_at (loc, OPT_Wunused_result,
9931	"ignoring return value of function "
9932	"declared with attribute %<warn_unused_result%>");
9933	}
9934	break;
9935
9936	default:
9937	/* Not a container, not a call, or a call whose value is used. */
9938	break;
9939	}
9940	}
9941	}
9942
9943	namespace {
9944
9945	const pass_data pass_data_warn_unused_result =
9946	{
9947	.type: GIMPLE_PASS, /* type */
9948	.name: "*warn_unused_result", /* name */
9949	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
9950	.tv_id: TV_NONE, /* tv_id */
9951	PROP_gimple_any, /* properties_required */
9952	.properties_provided: 0, /* properties_provided */
9953	.properties_destroyed: 0, /* properties_destroyed */
9954	.todo_flags_start: 0, /* todo_flags_start */
9955	.todo_flags_finish: 0, /* todo_flags_finish */
9956	};
9957
9958	class pass_warn_unused_result : public gimple_opt_pass
9959	{
9960	public:
9961	pass_warn_unused_result (gcc::context *ctxt)
9962	: gimple_opt_pass (pass_data_warn_unused_result, ctxt)
9963	{}
9964
9965	/* opt_pass methods: */
9966	bool gate (function *) final override { return flag_warn_unused_result; }
9967	unsigned int execute (function *) final override
9968	{
9969	do_warn_unused_result (seq: gimple_body (current_function_decl));
9970	return 0;
9971	}
9972
9973	}; // class pass_warn_unused_result
9974
9975	} // anon namespace
9976
9977	gimple_opt_pass *
9978	make_pass_warn_unused_result (gcc::context *ctxt)
9979	{
9980	return new pass_warn_unused_result (ctxt);
9981	}
9982
9983	/* Maybe Remove stores to variables we marked write-only.
9984	Return true if a store was removed. */
9985	static bool
9986	maybe_remove_writeonly_store (gimple_stmt_iterator &gsi, gimple *stmt,
9987	bitmap dce_ssa_names)
9988	{
9989	/* Keep access when store has side effect, i.e. in case when source
9990	is volatile. */
9991	if (!gimple_store_p (gs: stmt)
9992	|| gimple_has_side_effects (stmt)
9993	|| optimize_debug)
9994	return false;
9995
9996	tree lhs = get_base_address (t: gimple_get_lhs (stmt));
9997
9998	if (!VAR_P (lhs)
9999	|| (!TREE_STATIC (lhs) && !DECL_EXTERNAL (lhs))
10000	|| !varpool_node::get (decl: lhs)->writeonly)
10001	return false;
10002
10003	if (dump_file && (dump_flags & TDF_DETAILS))
10004	{
10005	fprintf (stream: dump_file, format: "Removing statement, writes"
10006	" to write only var:\n");
10007	print_gimple_stmt (dump_file, stmt, 0,
10008	TDF_VOPS|TDF_MEMSYMS);
10009	}
10010
10011	/* Mark ssa name defining to be checked for simple dce. */
10012	if (gimple_assign_single_p (gs: stmt))
10013	{
10014	tree rhs = gimple_assign_rhs1 (gs: stmt);
10015	if (TREE_CODE (rhs) == SSA_NAME
10016	&& !SSA_NAME_IS_DEFAULT_DEF (rhs))
10017	bitmap_set_bit (dce_ssa_names, SSA_NAME_VERSION (rhs));
10018	}
10019	unlink_stmt_vdef (stmt);
10020	gsi_remove (&gsi, true);
10021	release_defs (stmt);
10022	return true;
10023	}
10024
10025	/* IPA passes, compilation of earlier functions or inlining
10026	might have changed some properties, such as marked functions nothrow,
10027	pure, const or noreturn.
10028	Remove redundant edges and basic blocks, and create new ones if necessary. */
10029
10030	unsigned int
10031	execute_fixup_cfg (void)
10032	{
10033	basic_block bb;
10034	gimple_stmt_iterator gsi;
10035	int todo = 0;
10036	cgraph_node *node = cgraph_node::get (decl: current_function_decl);
10037	/* Same scaling is also done by ipa_merge_profiles. */
10038	profile_count num = node->count;
10039	profile_count den = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
10040	bool scale = num.initialized_p () && !(num == den);
10041	auto_bitmap dce_ssa_names;
10042
10043	if (scale)
10044	{
10045	profile_count::adjust_for_ipa_scaling (num: &num, den: &den);
10046	ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = node->count;
10047	EXIT_BLOCK_PTR_FOR_FN (cfun)->count
10048	= EXIT_BLOCK_PTR_FOR_FN (cfun)->count.apply_scale (num, den);
10049	}
10050
10051	FOR_EACH_BB_FN (bb, cfun)
10052	{
10053	if (scale)
10054	bb->count = bb->count.apply_scale (num, den);
10055	for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi);)
10056	{
10057	gimple *stmt = gsi_stmt (i: gsi);
10058	tree decl = is_gimple_call (gs: stmt)
10059	? gimple_call_fndecl (gs: stmt)
10060	: NULL;
10061	if (decl)
10062	{
10063	int flags = gimple_call_flags (stmt);
10064	if (flags & (ECF_CONST | ECF_PURE | ECF_LOOPING_CONST_OR_PURE))
10065	{
10066	if (gimple_in_ssa_p (cfun))
10067	{
10068	todo |= TODO_update_ssa | TODO_cleanup_cfg;
10069	update_stmt (s: stmt);
10070	}
10071	}
10072	if (flags & ECF_NORETURN
10073	&& fixup_noreturn_call (stmt))
10074	todo |= TODO_cleanup_cfg;
10075	}
10076
10077	/* Remove stores to variables we marked write-only. */
10078	if (maybe_remove_writeonly_store (gsi, stmt, dce_ssa_names))
10079	{
10080	todo |= TODO_update_ssa | TODO_cleanup_cfg;
10081	continue;
10082	}
10083
10084	/* For calls we can simply remove LHS when it is known
10085	to be write-only. */
10086	if (is_gimple_call (gs: stmt)
10087	&& gimple_get_lhs (stmt))
10088	{
10089	tree lhs = get_base_address (t: gimple_get_lhs (stmt));
10090
10091	if (VAR_P (lhs)
10092	&& (TREE_STATIC (lhs) || DECL_EXTERNAL (lhs))
10093	&& varpool_node::get (decl: lhs)->writeonly)
10094	{
10095	gimple_call_set_lhs (gs: stmt, NULL);
10096	update_stmt (s: stmt);
10097	todo |= TODO_update_ssa | TODO_cleanup_cfg;
10098	}
10099	}
10100
10101	gsi_next (i: &gsi);
10102	}
10103	if (gimple *last = *gsi_last_bb (bb))
10104	{
10105	if (maybe_clean_eh_stmt (last)
10106	&& gimple_purge_dead_eh_edges (bb))
10107	todo |= TODO_cleanup_cfg;
10108	if (gimple_purge_dead_abnormal_call_edges (bb))
10109	todo |= TODO_cleanup_cfg;
10110	}
10111
10112	/* If we have a basic block with no successors that does not
10113	end with a control statement or a noreturn call end it with
10114	a call to __builtin_unreachable. This situation can occur
10115	when inlining a noreturn call that does in fact return. */
10116	if (EDGE_COUNT (bb->succs) == 0)
10117	{
10118	gimple *stmt = last_nondebug_stmt (bb);
10119	if (!stmt
10120	|| (!is_ctrl_stmt (t: stmt)
10121	&& (!is_gimple_call (gs: stmt)
10122	|| !gimple_call_noreturn_p (s: stmt))))
10123	{
10124	if (stmt && is_gimple_call (gs: stmt))
10125	gimple_call_set_ctrl_altering (s: stmt, ctrl_altering_p: false);
10126	stmt = gimple_build_builtin_unreachable (UNKNOWN_LOCATION);
10127	gimple_stmt_iterator gsi = gsi_last_bb (bb);
10128	gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
10129	if (!cfun->after_inlining)
10130	if (tree fndecl = gimple_call_fndecl (gs: stmt))
10131	{
10132	gcall *call_stmt = dyn_cast <gcall *> (p: stmt);
10133	node->create_edge (callee: cgraph_node::get_create (fndecl),
10134	call_stmt, count: bb->count);
10135	}
10136	}
10137	}
10138	}
10139	if (scale)
10140	{
10141	update_max_bb_count ();
10142	compute_function_frequency ();
10143	}
10144
10145	if (current_loops
10146	&& (todo & TODO_cleanup_cfg))
10147	loops_state_set (flags: LOOPS_NEED_FIXUP);
10148
10149	simple_dce_from_worklist (dce_ssa_names);
10150
10151	return todo;
10152	}
10153
10154	namespace {
10155
10156	const pass_data pass_data_fixup_cfg =
10157	{
10158	.type: GIMPLE_PASS, /* type */
10159	.name: "fixup_cfg", /* name */
10160	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
10161	.tv_id: TV_NONE, /* tv_id */
10162	PROP_cfg, /* properties_required */
10163	.properties_provided: 0, /* properties_provided */
10164	.properties_destroyed: 0, /* properties_destroyed */
10165	.todo_flags_start: 0, /* todo_flags_start */
10166	.todo_flags_finish: 0, /* todo_flags_finish */
10167	};
10168
10169	class pass_fixup_cfg : public gimple_opt_pass
10170	{
10171	public:
10172	pass_fixup_cfg (gcc::context *ctxt)
10173	: gimple_opt_pass (pass_data_fixup_cfg, ctxt)
10174	{}
10175
10176	/* opt_pass methods: */
10177	opt_pass * clone () final override { return new pass_fixup_cfg (m_ctxt); }
10178	unsigned int execute (function *) final override
10179	{
10180	return execute_fixup_cfg ();
10181	}
10182
10183	}; // class pass_fixup_cfg
10184
10185	} // anon namespace
10186
10187	gimple_opt_pass *
10188	make_pass_fixup_cfg (gcc::context *ctxt)
10189	{
10190	return new pass_fixup_cfg (ctxt);
10191	}
10192
10193	/* Garbage collection support for edge_def. */
10194
10195	extern void gt_ggc_mx (tree&);
10196	extern void gt_ggc_mx (gimple *&);
10197	extern void gt_ggc_mx (rtx&);
10198	extern void gt_ggc_mx (basic_block&);
10199
10200	static void
10201	gt_ggc_mx (rtx_insn *& x)
10202	{
10203	if (x)
10204	gt_ggc_mx_rtx_def ((void *) x);
10205	}
10206
10207	void
10208	gt_ggc_mx (edge_def *e)
10209	{
10210	tree block = LOCATION_BLOCK (e->goto_locus);
10211	gt_ggc_mx (e->src);
10212	gt_ggc_mx (e->dest);
10213	if (current_ir_type () == IR_GIMPLE)
10214	gt_ggc_mx (e->insns.g);
10215	else
10216	gt_ggc_mx (x&: e->insns.r);
10217	gt_ggc_mx (block);
10218	}
10219
10220	/* PCH support for edge_def. */
10221
10222	extern void gt_pch_nx (tree&);
10223	extern void gt_pch_nx (gimple *&);
10224	extern void gt_pch_nx (rtx&);
10225	extern void gt_pch_nx (basic_block&);
10226
10227	static void
10228	gt_pch_nx (rtx_insn *& x)
10229	{
10230	if (x)
10231	gt_pch_nx_rtx_def ((void *) x);
10232	}
10233
10234	void
10235	gt_pch_nx (edge_def *e)
10236	{
10237	tree block = LOCATION_BLOCK (e->goto_locus);
10238	gt_pch_nx (e->src);
10239	gt_pch_nx (e->dest);
10240	if (current_ir_type () == IR_GIMPLE)
10241	gt_pch_nx (e->insns.g);
10242	else
10243	gt_pch_nx (x&: e->insns.r);
10244	gt_pch_nx (block);
10245	}
10246
10247	void
10248	gt_pch_nx (edge_def *e, gt_pointer_operator op, void *cookie)
10249	{
10250	tree block = LOCATION_BLOCK (e->goto_locus);
10251	op (&(e->src), NULL, cookie);
10252	op (&(e->dest), NULL, cookie);
10253	if (current_ir_type () == IR_GIMPLE)
10254	op (&(e->insns.g), NULL, cookie);
10255	else
10256	op (&(e->insns.r), NULL, cookie);
10257	op (&(block), &(block), cookie);
10258	}
10259
10260	#if CHECKING_P
10261
10262	namespace selftest {
10263
10264	/* Helper function for CFG selftests: create a dummy function decl
10265	and push it as cfun. */
10266
10267	static tree
10268	push_fndecl (const char *name)
10269	{
10270	tree fn_type = build_function_type_array (integer_type_node, 0, NULL);
10271	/* FIXME: this uses input_location: */
10272	tree fndecl = build_fn_decl (name, fn_type);
10273	tree retval = build_decl (UNKNOWN_LOCATION, RESULT_DECL,
10274	NULL_TREE, integer_type_node);
10275	DECL_RESULT (fndecl) = retval;
10276	push_struct_function (fndecl);
10277	function *fun = DECL_STRUCT_FUNCTION (fndecl);
10278	ASSERT_TRUE (fun != NULL);
10279	init_empty_tree_cfg_for_function (fn: fun);
10280	ASSERT_EQ (2, n_basic_blocks_for_fn (fun));
10281	ASSERT_EQ (0, n_edges_for_fn (fun));
10282	return fndecl;
10283	}
10284
10285	/* These tests directly create CFGs.
10286	Compare with the static fns within tree-cfg.cc:
10287	- build_gimple_cfg
10288	- make_blocks: calls create_basic_block (seq, bb);
10289	- make_edges. */
10290
10291	/* Verify a simple cfg of the form:
10292	ENTRY -> A -> B -> C -> EXIT. */
10293
10294	static void
10295	test_linear_chain ()
10296	{
10297	gimple_register_cfg_hooks ();
10298
10299	tree fndecl = push_fndecl (name: "cfg_test_linear_chain");
10300	function *fun = DECL_STRUCT_FUNCTION (fndecl);
10301
10302	/* Create some empty blocks. */
10303	basic_block bb_a = create_empty_bb (ENTRY_BLOCK_PTR_FOR_FN (fun));
10304	basic_block bb_b = create_empty_bb (bb_a);
10305	basic_block bb_c = create_empty_bb (bb_b);
10306
10307	ASSERT_EQ (5, n_basic_blocks_for_fn (fun));
10308	ASSERT_EQ (0, n_edges_for_fn (fun));
10309
10310	/* Create some edges: a simple linear chain of BBs. */
10311	make_edge (ENTRY_BLOCK_PTR_FOR_FN (fun), bb_a, EDGE_FALLTHRU);
10312	make_edge (bb_a, bb_b, 0);
10313	make_edge (bb_b, bb_c, 0);
10314	make_edge (bb_c, EXIT_BLOCK_PTR_FOR_FN (fun), 0);
10315
10316	/* Verify the edges. */
10317	ASSERT_EQ (4, n_edges_for_fn (fun));
10318	ASSERT_EQ (NULL, ENTRY_BLOCK_PTR_FOR_FN (fun)->preds);
10319	ASSERT_EQ (1, ENTRY_BLOCK_PTR_FOR_FN (fun)->succs->length ());
10320	ASSERT_EQ (1, bb_a->preds->length ());
10321	ASSERT_EQ (1, bb_a->succs->length ());
10322	ASSERT_EQ (1, bb_b->preds->length ());
10323	ASSERT_EQ (1, bb_b->succs->length ());
10324	ASSERT_EQ (1, bb_c->preds->length ());
10325	ASSERT_EQ (1, bb_c->succs->length ());
10326	ASSERT_EQ (1, EXIT_BLOCK_PTR_FOR_FN (fun)->preds->length ());
10327	ASSERT_EQ (NULL, EXIT_BLOCK_PTR_FOR_FN (fun)->succs);
10328
10329	/* Verify the dominance information
10330	Each BB in our simple chain should be dominated by the one before
10331	it. */
10332	calculate_dominance_info (CDI_DOMINATORS);
10333	ASSERT_EQ (bb_a, get_immediate_dominator (CDI_DOMINATORS, bb_b));
10334	ASSERT_EQ (bb_b, get_immediate_dominator (CDI_DOMINATORS, bb_c));
10335	auto_vec<basic_block> dom_by_b = get_dominated_by (CDI_DOMINATORS, bb_b);
10336	ASSERT_EQ (1, dom_by_b.length ());
10337	ASSERT_EQ (bb_c, dom_by_b[0]);
10338	free_dominance_info (CDI_DOMINATORS);
10339
10340	/* Similarly for post-dominance: each BB in our chain is post-dominated
10341	by the one after it. */
10342	calculate_dominance_info (CDI_POST_DOMINATORS);
10343	ASSERT_EQ (bb_b, get_immediate_dominator (CDI_POST_DOMINATORS, bb_a));
10344	ASSERT_EQ (bb_c, get_immediate_dominator (CDI_POST_DOMINATORS, bb_b));
10345	auto_vec<basic_block> postdom_by_b = get_dominated_by (CDI_POST_DOMINATORS, bb_b);
10346	ASSERT_EQ (1, postdom_by_b.length ());
10347	ASSERT_EQ (bb_a, postdom_by_b[0]);
10348	free_dominance_info (CDI_POST_DOMINATORS);
10349
10350	pop_cfun ();
10351	}
10352
10353	/* Verify a simple CFG of the form:
10354	ENTRY
10355	|
10356	A
10357	/ \
10358	/t \f
10359	B C
10360	\ /
10361	\ /
10362	D
10363	|
10364	EXIT. */
10365
10366	static void
10367	test_diamond ()
10368	{
10369	gimple_register_cfg_hooks ();
10370
10371	tree fndecl = push_fndecl (name: "cfg_test_diamond");
10372	function *fun = DECL_STRUCT_FUNCTION (fndecl);
10373
10374	/* Create some empty blocks. */
10375	basic_block bb_a = create_empty_bb (ENTRY_BLOCK_PTR_FOR_FN (fun));
10376	basic_block bb_b = create_empty_bb (bb_a);
10377	basic_block bb_c = create_empty_bb (bb_a);
10378	basic_block bb_d = create_empty_bb (bb_b);
10379
10380	ASSERT_EQ (6, n_basic_blocks_for_fn (fun));
10381	ASSERT_EQ (0, n_edges_for_fn (fun));
10382
10383	/* Create the edges. */
10384	make_edge (ENTRY_BLOCK_PTR_FOR_FN (fun), bb_a, EDGE_FALLTHRU);
10385	make_edge (bb_a, bb_b, EDGE_TRUE_VALUE);
10386	make_edge (bb_a, bb_c, EDGE_FALSE_VALUE);
10387	make_edge (bb_b, bb_d, 0);
10388	make_edge (bb_c, bb_d, 0);
10389	make_edge (bb_d, EXIT_BLOCK_PTR_FOR_FN (fun), 0);
10390
10391	/* Verify the edges. */
10392	ASSERT_EQ (6, n_edges_for_fn (fun));
10393	ASSERT_EQ (1, bb_a->preds->length ());
10394	ASSERT_EQ (2, bb_a->succs->length ());
10395	ASSERT_EQ (1, bb_b->preds->length ());
10396	ASSERT_EQ (1, bb_b->succs->length ());
10397	ASSERT_EQ (1, bb_c->preds->length ());
10398	ASSERT_EQ (1, bb_c->succs->length ());
10399	ASSERT_EQ (2, bb_d->preds->length ());
10400	ASSERT_EQ (1, bb_d->succs->length ());
10401
10402	/* Verify the dominance information. */
10403	calculate_dominance_info (CDI_DOMINATORS);
10404	ASSERT_EQ (bb_a, get_immediate_dominator (CDI_DOMINATORS, bb_b));
10405	ASSERT_EQ (bb_a, get_immediate_dominator (CDI_DOMINATORS, bb_c));
10406	ASSERT_EQ (bb_a, get_immediate_dominator (CDI_DOMINATORS, bb_d));
10407	auto_vec<basic_block> dom_by_a = get_dominated_by (CDI_DOMINATORS, bb_a);
10408	ASSERT_EQ (3, dom_by_a.length ()); /* B, C, D, in some order. */
10409	dom_by_a.release ();
10410	auto_vec<basic_block> dom_by_b = get_dominated_by (CDI_DOMINATORS, bb_b);
10411	ASSERT_EQ (0, dom_by_b.length ());
10412	dom_by_b.release ();
10413	free_dominance_info (CDI_DOMINATORS);
10414
10415	/* Similarly for post-dominance. */
10416	calculate_dominance_info (CDI_POST_DOMINATORS);
10417	ASSERT_EQ (bb_d, get_immediate_dominator (CDI_POST_DOMINATORS, bb_a));
10418	ASSERT_EQ (bb_d, get_immediate_dominator (CDI_POST_DOMINATORS, bb_b));
10419	ASSERT_EQ (bb_d, get_immediate_dominator (CDI_POST_DOMINATORS, bb_c));
10420	auto_vec<basic_block> postdom_by_d = get_dominated_by (CDI_POST_DOMINATORS, bb_d);
10421	ASSERT_EQ (3, postdom_by_d.length ()); /* A, B, C in some order. */
10422	postdom_by_d.release ();
10423	auto_vec<basic_block> postdom_by_b = get_dominated_by (CDI_POST_DOMINATORS, bb_b);
10424	ASSERT_EQ (0, postdom_by_b.length ());
10425	postdom_by_b.release ();
10426	free_dominance_info (CDI_POST_DOMINATORS);
10427
10428	pop_cfun ();
10429	}
10430
10431	/* Verify that we can handle a CFG containing a "complete" aka
10432	fully-connected subgraph (where A B C D below all have edges
10433	pointing to each other node, also to themselves).
10434	e.g.:
10435	ENTRY EXIT
10436	| ^
10437	| /
10438	| /
10439	| /
10440	V/
10441	A<--->B
10442	^^ ^^
10443	| \ / |
10444	| X |
10445	| / \ |
10446	VV VV
10447	C<--->D
10448	*/
10449
10450	static void
10451	test_fully_connected ()
10452	{
10453	gimple_register_cfg_hooks ();
10454
10455	tree fndecl = push_fndecl (name: "cfg_fully_connected");
10456	function *fun = DECL_STRUCT_FUNCTION (fndecl);
10457
10458	const int n = 4;
10459
10460	/* Create some empty blocks. */
10461	auto_vec <basic_block> subgraph_nodes;
10462	for (int i = 0; i < n; i++)
10463	subgraph_nodes.safe_push (obj: create_empty_bb (ENTRY_BLOCK_PTR_FOR_FN (fun)));
10464
10465	ASSERT_EQ (n + 2, n_basic_blocks_for_fn (fun));
10466	ASSERT_EQ (0, n_edges_for_fn (fun));
10467
10468	/* Create the edges. */
10469	make_edge (ENTRY_BLOCK_PTR_FOR_FN (fun), subgraph_nodes[0], EDGE_FALLTHRU);
10470	make_edge (subgraph_nodes[0], EXIT_BLOCK_PTR_FOR_FN (fun), 0);
10471	for (int i = 0; i < n; i++)
10472	for (int j = 0; j < n; j++)
10473	make_edge (subgraph_nodes[i], subgraph_nodes[j], 0);
10474
10475	/* Verify the edges. */
10476	ASSERT_EQ (2 + (n * n), n_edges_for_fn (fun));
10477	/* The first one is linked to ENTRY/EXIT as well as itself and
10478	everything else. */
10479	ASSERT_EQ (n + 1, subgraph_nodes[0]->preds->length ());
10480	ASSERT_EQ (n + 1, subgraph_nodes[0]->succs->length ());
10481	/* The other ones in the subgraph are linked to everything in
10482	the subgraph (including themselves). */
10483	for (int i = 1; i < n; i++)
10484	{
10485	ASSERT_EQ (n, subgraph_nodes[i]->preds->length ());
10486	ASSERT_EQ (n, subgraph_nodes[i]->succs->length ());
10487	}
10488
10489	/* Verify the dominance information. */
10490	calculate_dominance_info (CDI_DOMINATORS);
10491	/* The initial block in the subgraph should be dominated by ENTRY. */
10492	ASSERT_EQ (ENTRY_BLOCK_PTR_FOR_FN (fun),
10493	get_immediate_dominator (CDI_DOMINATORS,
10494	subgraph_nodes[0]));
10495	/* Every other block in the subgraph should be dominated by the
10496	initial block. */
10497	for (int i = 1; i < n; i++)
10498	ASSERT_EQ (subgraph_nodes[0],
10499	get_immediate_dominator (CDI_DOMINATORS,
10500	subgraph_nodes[i]));
10501	free_dominance_info (CDI_DOMINATORS);
10502
10503	/* Similarly for post-dominance. */
10504	calculate_dominance_info (CDI_POST_DOMINATORS);
10505	/* The initial block in the subgraph should be postdominated by EXIT. */
10506	ASSERT_EQ (EXIT_BLOCK_PTR_FOR_FN (fun),
10507	get_immediate_dominator (CDI_POST_DOMINATORS,
10508	subgraph_nodes[0]));
10509	/* Every other block in the subgraph should be postdominated by the
10510	initial block, since that leads to EXIT. */
10511	for (int i = 1; i < n; i++)
10512	ASSERT_EQ (subgraph_nodes[0],
10513	get_immediate_dominator (CDI_POST_DOMINATORS,
10514	subgraph_nodes[i]));
10515	free_dominance_info (CDI_POST_DOMINATORS);
10516
10517	pop_cfun ();
10518	}
10519
10520	/* Run all of the selftests within this file. */
10521
10522	void
10523	tree_cfg_cc_tests ()
10524	{
10525	test_linear_chain ();
10526	test_diamond ();
10527	test_fully_connected ();
10528	}
10529
10530	} // namespace selftest
10531
10532	/* TODO: test the dominator/postdominator logic with various graphs/nodes:
10533	- loop
10534	- nested loops
10535	- switch statement (a block with many out-edges)
10536	- something that jumps to itself
10537	- etc */
10538
10539	#endif /* CHECKING_P */
10540