1	/* Instruction scheduling pass. Selective scheduler and pipeliner.
2	Copyright (C) 2006-2024 Free Software Foundation, Inc.
3
4	This file is part of GCC.
5
6	GCC is free software; you can redistribute it and/or modify it under
7	the terms of the GNU General Public License as published by the Free
8	Software Foundation; either version 3, or (at your option) any later
9	version.
10
11	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12	WARRANTY; without even the implied warranty of MERCHANTABILITY or
13	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14	for more details.
15
16	You should have received a copy of the GNU General Public License
17	along with GCC; see the file COPYING3. If not see
18	<http://www.gnu.org/licenses/>. */
19
20	#include "config.h"
21	#include "system.h"
22	#include "coretypes.h"
23	#include "backend.h"
24	#include "tree.h"
25	#include "rtl.h"
26	#include "df.h"
27	#include "memmodel.h"
28	#include "tm_p.h"
29	#include "regs.h"
30	#include "cfgbuild.h"
31	#include "cfgcleanup.h"
32	#include "insn-config.h"
33	#include "insn-attr.h"
34	#include "target.h"
35	#include "sched-int.h"
36	#include "rtlhooks-def.h"
37	#include "ira.h"
38	#include "ira-int.h"
39	#include "rtl-iter.h"
40
41	#ifdef INSN_SCHEDULING
42	#include "regset.h"
43	#include "cfgloop.h"
44	#include "sel-sched-ir.h"
45	#include "sel-sched-dump.h"
46	#include "sel-sched.h"
47	#include "dbgcnt.h"
48	#include "function-abi.h"
49
50	/* Implementation of selective scheduling approach.
51	The below implementation follows the original approach with the following
52	changes:
53
54	o the scheduler works after register allocation (but can be also tuned
55	to work before RA);
56	o some instructions are not copied or register renamed;
57	o conditional jumps are not moved with code duplication;
58	o several jumps in one parallel group are not supported;
59	o when pipelining outer loops, code motion through inner loops
60	is not supported;
61	o control and data speculation are supported;
62	o some improvements for better compile time/performance were made.
63
64	Terminology
65	===========
66
67	A vinsn, or virtual insn, is an insn with additional data characterizing
68	insn pattern, such as LHS, RHS, register sets used/set/clobbered, etc.
69	Vinsns also act as smart pointers to save memory by reusing them in
70	different expressions. A vinsn is described by vinsn_t type.
71
72	An expression is a vinsn with additional data characterizing its properties
73	at some point in the control flow graph. The data may be its usefulness,
74	priority, speculative status, whether it was renamed/subsituted, etc.
75	An expression is described by expr_t type.
76
77	Availability set (av_set) is a set of expressions at a given control flow
78	point. It is represented as av_set_t. The expressions in av sets are kept
79	sorted in the terms of expr_greater_p function. It allows to truncate
80	the set while leaving the best expressions.
81
82	A fence is a point through which code motion is prohibited. On each step,
83	we gather a parallel group of insns at a fence. It is possible to have
84	multiple fences. A fence is represented via fence_t.
85
86	A boundary is the border between the fence group and the rest of the code.
87	Currently, we never have more than one boundary per fence, as we finalize
88	the fence group when a jump is scheduled. A boundary is represented
89	via bnd_t.
90
91	High-level overview
92	===================
93
94	The scheduler finds regions to schedule, schedules each one, and finalizes.
95	The regions are formed starting from innermost loops, so that when the inner
96	loop is pipelined, its prologue can be scheduled together with yet unprocessed
97	outer loop. The rest of acyclic regions are found using extend_rgns:
98	the blocks that are not yet allocated to any regions are traversed in top-down
99	order, and a block is added to a region to which all its predecessors belong;
100	otherwise, the block starts its own region.
101
102	The main scheduling loop (sel_sched_region_2) consists of just
103	scheduling on each fence and updating fences. For each fence,
104	we fill a parallel group of insns (fill_insns) until some insns can be added.
105	First, we compute available exprs (av-set) at the boundary of the current
106	group. Second, we choose the best expression from it. If the stall is
107	required to schedule any of the expressions, we advance the current cycle
108	appropriately. So, the final group does not exactly correspond to a VLIW
109	word. Third, we move the chosen expression to the boundary (move_op)
110	and update the intermediate av sets and liveness sets. We quit fill_insns
111	when either no insns left for scheduling or we have scheduled enough insns
112	so we feel like advancing a scheduling point.
113
114	Computing available expressions
115	===============================
116
117	The computation (compute_av_set) is a bottom-up traversal. At each insn,
118	we're moving the union of its successors' sets through it via
119	moveup_expr_set. The dependent expressions are removed. Local
120	transformations (substitution, speculation) are applied to move more
121	exprs. Then the expr corresponding to the current insn is added.
122	The result is saved on each basic block header.
123
124	When traversing the CFG, we're moving down for no more than max_ws insns.
125	Also, we do not move down to ineligible successors (is_ineligible_successor),
126	which include moving along a back-edge, moving to already scheduled code,
127	and moving to another fence. The first two restrictions are lifted during
128	pipelining, which allows us to move insns along a back-edge. We always have
129	an acyclic region for scheduling because we forbid motion through fences.
130
131	Choosing the best expression
132	============================
133
134	We sort the final availability set via sel_rank_for_schedule, then we remove
135	expressions which are not yet ready (tick_check_p) or which dest registers
136	cannot be used. For some of them, we choose another register via
137	find_best_reg. To do this, we run find_used_regs to calculate the set of
138	registers which cannot be used. The find_used_regs function performs
139	a traversal of code motion paths for an expr. We consider for renaming
140	only registers which are from the same regclass as the original one and
141	using which does not interfere with any live ranges. Finally, we convert
142	the resulting set to the ready list format and use max_issue and reorder*
143	hooks similarly to the Haifa scheduler.
144
145	Scheduling the best expression
146	==============================
147
148	We run the move_op routine to perform the same type of code motion paths
149	traversal as in find_used_regs. (These are working via the same driver,
150	code_motion_path_driver.) When moving down the CFG, we look for original
151	instruction that gave birth to a chosen expression. We undo
152	the transformations performed on an expression via the history saved in it.
153	When found, we remove the instruction or leave a reg-reg copy/speculation
154	check if needed. On a way up, we insert bookkeeping copies at each join
155	point. If a copy is not needed, it will be removed later during this
156	traversal. We update the saved av sets and liveness sets on the way up, too.
157
158	Finalizing the schedule
159	=======================
160
161	When pipelining, we reschedule the blocks from which insns were pipelined
162	to get a tighter schedule. On Itanium, we also perform bundling via
163	the same routine from ia64.cc.
164
165	Dependence analysis changes
166	===========================
167
168	We augmented the sched-deps.cc with hooks that get called when a particular
169	dependence is found in a particular part of an insn. Using these hooks, we
170	can do several actions such as: determine whether an insn can be moved through
171	another (has_dependence_p, moveup_expr); find out whether an insn can be
172	scheduled on the current cycle (tick_check_p); find out registers that
173	are set/used/clobbered by an insn and find out all the strange stuff that
174	restrict its movement, like SCHED_GROUP_P or CANT_MOVE (done in
175	init_global_and_expr_for_insn).
176
177	Initialization changes
178	======================
179
180	There are parts of haifa-sched.cc, sched-deps.cc, and sched-rgn.cc that are
181	reused in all of the schedulers. We have split up the initialization of data
182	of such parts into different functions prefixed with scheduler type and
183	postfixed with the type of data initialized: {,sel_,haifa_}sched_{init,finish},
184	sched_rgn_init/finish, sched_deps_init/finish, sched_init_{luids/bbs}, etc.
185	The same splitting is done with current_sched_info structure:
186	dependence-related parts are in sched_deps_info, common part is in
187	common_sched_info, and haifa/sel/etc part is in current_sched_info.
188
189	Target contexts
190	===============
191
192	As we now have multiple-point scheduling, this would not work with backends
193	which save some of the scheduler state to use it in the target hooks.
194	For this purpose, we introduce a concept of target contexts, which
195	encapsulate such information. The backend should implement simple routines
196	of allocating/freeing/setting such a context. The scheduler calls these
197	as target hooks and handles the target context as an opaque pointer (similar
198	to the DFA state type, state_t).
199
200	Various speedups
201	================
202
203	As the correct data dependence graph is not supported during scheduling (which
204	is to be changed in mid-term), we cache as much of the dependence analysis
205	results as possible to avoid reanalyzing. This includes: bitmap caches on
206	each insn in stream of the region saying yes/no for a query with a pair of
207	UIDs; hashtables with the previously done transformations on each insn in
208	stream; a vector keeping a history of transformations on each expr.
209
210	Also, we try to minimize the dependence context used on each fence to check
211	whether the given expression is ready for scheduling by removing from it
212	insns that are definitely completed the execution. The results of
213	tick_check_p checks are also cached in a vector on each fence.
214
215	We keep a valid liveness set on each insn in a region to avoid the high
216	cost of recomputation on large basic blocks.
217
218	Finally, we try to minimize the number of needed updates to the availability
219	sets. The updates happen in two cases: when fill_insns terminates,
220	we advance all fences and increase the stage number to show that the region
221	has changed and the sets are to be recomputed; and when the next iteration
222	of a loop in fill_insns happens (but this one reuses the saved av sets
223	on bb headers.) Thus, we try to break the fill_insns loop only when
224	"significant" number of insns from the current scheduling window was
225	scheduled. This should be made a target param.
226
227
228	TODO: correctly support the data dependence graph at all stages and get rid
229	of all caches. This should speed up the scheduler.
230	TODO: implement moving cond jumps with bookkeeping copies on both targets.
231	TODO: tune the scheduler before RA so it does not create too much pseudos.
232
233
234	References:
235	S.-M. Moon and K. Ebcioglu. Parallelizing nonnumerical code with
236	selective scheduling and software pipelining.
237	ACM TOPLAS, Vol 19, No. 6, pages 853--898, Nov. 1997.
238
239	Andrey Belevantsev, Maxim Kuvyrkov, Vladimir Makarov, Dmitry Melnik,
240	and Dmitry Zhurikhin. An interblock VLIW-targeted instruction scheduler
241	for GCC. In Proceedings of GCC Developers' Summit 2006.
242
243	Arutyun Avetisyan, Andrey Belevantsev, and Dmitry Melnik. GCC Instruction
244	Scheduler and Software Pipeliner on the Itanium Platform. EPIC-7 Workshop.
245	http://rogue.colorado.edu/EPIC7/.
246
247	*/
248
249	/* True when pipelining is enabled. */
250	bool pipelining_p;
251
252	/* True if bookkeeping is enabled. */
253	bool bookkeeping_p;
254
255	/* Maximum number of insns that are eligible for renaming. */
256	int max_insns_to_rename;
257
258
259	/* Definitions of local types and macros. */
260
261	/* Represents possible outcomes of moving an expression through an insn. */
262	enum MOVEUP_EXPR_CODE
263	{
264	/* The expression is not changed. */
265	MOVEUP_EXPR_SAME,
266
267	/* Not changed, but requires a new destination register. */
268	MOVEUP_EXPR_AS_RHS,
269
270	/* Cannot be moved. */
271	MOVEUP_EXPR_NULL,
272
273	/* Changed (substituted or speculated). */
274	MOVEUP_EXPR_CHANGED
275	};
276
277	/* The container to be passed into rtx search & replace functions. */
278	struct rtx_search_arg
279	{
280	/* What we are searching for. */
281	rtx x;
282
283	/* The occurrence counter. */
284	int n;
285	};
286
287	typedef struct rtx_search_arg *rtx_search_arg_p;
288
289	/* This struct contains precomputed hard reg sets that are needed when
290	computing registers available for renaming. */
291	struct hard_regs_data
292	{
293	/* For every mode, this stores registers available for use with
294	that mode. */
295	HARD_REG_SET regs_for_mode[NUM_MACHINE_MODES];
296
297	/* True when regs_for_mode[mode] is initialized. */
298	bool regs_for_mode_ok[NUM_MACHINE_MODES];
299
300	/* For every register, it has regs that are ok to rename into it.
301	The register in question is always set. If not, this means
302	that the whole set is not computed yet. */
303	HARD_REG_SET regs_for_rename[FIRST_PSEUDO_REGISTER];
304
305	/* All registers that are used or call used. */
306	HARD_REG_SET regs_ever_used;
307
308	#ifdef STACK_REGS
309	/* Stack registers. */
310	HARD_REG_SET stack_regs;
311	#endif
312	};
313
314	/* Holds the results of computation of available for renaming and
315	unavailable hard registers. */
316	struct reg_rename
317	{
318	/* These are unavailable due to calls crossing, globalness, etc. */
319	HARD_REG_SET unavailable_hard_regs;
320
321	/* These are *available* for renaming. */
322	HARD_REG_SET available_for_renaming;
323
324	/* The set of ABIs used by calls that the code motion path crosses. */
325	unsigned int crossed_call_abis : NUM_ABI_IDS;
326	};
327
328	/* A global structure that contains the needed information about harg
329	regs. */
330	static struct hard_regs_data sel_hrd;
331
332
333	/* This structure holds local data used in code_motion_path_driver hooks on
334	the same or adjacent levels of recursion. Here we keep those parameters
335	that are not used in code_motion_path_driver routine itself, but only in
336	its hooks. Moreover, all parameters that can be modified in hooks are
337	in this structure, so all other parameters passed explicitly to hooks are
338	read-only. */
339	struct cmpd_local_params
340	{
341	/* Local params used in move_op_* functions. */
342
343	/* Edges for bookkeeping generation. */
344	edge e1, e2;
345
346	/* C_EXPR merged from all successors and locally allocated temporary C_EXPR. */
347	expr_t c_expr_merged, c_expr_local;
348
349	/* Local params used in fur_* functions. */
350	/* Copy of the ORIGINAL_INSN list, stores the original insns already
351	found before entering the current level of code_motion_path_driver. */
352	def_list_t old_original_insns;
353
354	/* Local params used in move_op_* functions. */
355	/* True when we have removed last insn in the block which was
356	also a boundary. Do not update anything or create bookkeeping copies. */
357	BOOL_BITFIELD removed_last_insn : 1;
358	};
359
360	/* Stores the static parameters for move_op_* calls. */
361	struct moveop_static_params
362	{
363	/* Destination register. */
364	rtx dest;
365
366	/* Current C_EXPR. */
367	expr_t c_expr;
368
369	/* An UID of expr_vliw which is to be moved up. If we find other exprs,
370	they are to be removed. */
371	int uid;
372
373	/* This is initialized to the insn on which the driver stopped its traversal. */
374	insn_t failed_insn;
375
376	/* True if we scheduled an insn with different register. */
377	bool was_renamed;
378	};
379
380	/* Stores the static parameters for fur_* calls. */
381	struct fur_static_params
382	{
383	/* Set of registers unavailable on the code motion path. */
384	regset used_regs;
385
386	/* Pointer to the list of original insns definitions. */
387	def_list_t *original_insns;
388
389	/* The set of ABIs used by calls that the code motion path crosses. */
390	unsigned int crossed_call_abis : NUM_ABI_IDS;
391	};
392
393	typedef struct fur_static_params *fur_static_params_p;
394	typedef struct cmpd_local_params *cmpd_local_params_p;
395	typedef struct moveop_static_params *moveop_static_params_p;
396
397	/* Set of hooks and parameters that determine behavior specific to
398	move_op or find_used_regs functions. */
399	struct code_motion_path_driver_info_def
400	{
401	/* Called on enter to the basic block. */
402	int (*on_enter) (insn_t, cmpd_local_params_p, void *, bool);
403
404	/* Called when original expr is found. */
405	void (*orig_expr_found) (insn_t, expr_t, cmpd_local_params_p, void *);
406
407	/* Called while descending current basic block if current insn is not
408	the original EXPR we're searching for. */
409	bool (*orig_expr_not_found) (insn_t, av_set_t, void *);
410
411	/* Function to merge C_EXPRes from different successors. */
412	void (*merge_succs) (insn_t, insn_t, int, cmpd_local_params_p, void *);
413
414	/* Function to finalize merge from different successors and possibly
415	deallocate temporary data structures used for merging. */
416	void (*after_merge_succs) (cmpd_local_params_p, void *);
417
418	/* Called on the backward stage of recursion to do moveup_expr.
419	Used only with move_op_*. */
420	void (*ascend) (insn_t, void *);
421
422	/* Called on the ascending pass, before returning from the current basic
423	block or from the whole traversal. */
424	void (*at_first_insn) (insn_t, cmpd_local_params_p, void *);
425
426	/* When processing successors in move_op we need only descend into
427	SUCCS_NORMAL successors, while in find_used_regs we need SUCCS_ALL. */
428	int succ_flags;
429
430	/* The routine name to print in dumps ("move_op" of "find_used_regs"). */
431	const char *routine_name;
432	};
433
434	/* Global pointer to current hooks, either points to MOVE_OP_HOOKS or
435	FUR_HOOKS. */
436	struct code_motion_path_driver_info_def *code_motion_path_driver_info;
437
438	/* Set of hooks for performing move_op and find_used_regs routines with
439	code_motion_path_driver. */
440	extern struct code_motion_path_driver_info_def move_op_hooks, fur_hooks;
441
442	/* True if/when we want to emulate Haifa scheduler in the common code.
443	This is used in sched_rgn_local_init and in various places in
444	sched-deps.cc. */
445	int sched_emulate_haifa_p;
446
447	/* GLOBAL_LEVEL is used to discard information stored in basic block headers
448	av_sets. Av_set of bb header is valid if its (bb header's) level is equal
449	to GLOBAL_LEVEL. And invalid if lesser. This is primarily used to advance
450	scheduling window. */
451	int global_level;
452
453	/* Current fences. */
454	flist_t fences;
455
456	/* True when separable insns should be scheduled as RHSes. */
457	static bool enable_schedule_as_rhs_p;
458
459	/* Used in verify_target_availability to assert that target reg is reported
460	unavailabile by both TARGET_UNAVAILABLE and find_used_regs only if
461	we haven't scheduled anything on the previous fence.
462	if scheduled_something_on_previous_fence is true, TARGET_UNAVAILABLE can
463	have more conservative value than the one returned by the
464	find_used_regs, thus we shouldn't assert that these values are equal. */
465	static bool scheduled_something_on_previous_fence;
466
467	/* All newly emitted insns will have their uids greater than this value. */
468	static int first_emitted_uid;
469
470	/* Set of basic blocks that are forced to start new ebbs. This is a subset
471	of all the ebb heads. */
472	bitmap forced_ebb_heads;
473
474	/* Blocks that need to be rescheduled after pipelining. */
475	bitmap blocks_to_reschedule = NULL;
476
477	/* True when the first lv set should be ignored when updating liveness. */
478	static bool ignore_first = false;
479
480	/* Number of insns max_issue has initialized data structures for. */
481	static int max_issue_size = 0;
482
483	/* Whether we can issue more instructions. */
484	static int can_issue_more;
485
486	/* Maximum software lookahead window size, reduced when rescheduling after
487	pipelining. */
488	static int max_ws;
489
490	/* Number of insns scheduled in current region. */
491	static int num_insns_scheduled;
492
493	/* A vector of expressions is used to be able to sort them. */
494	static vec<expr_t> vec_av_set;
495
496	/* A vector of vinsns is used to hold temporary lists of vinsns. */
497	typedef vec<vinsn_t> vinsn_vec_t;
498
499	/* This vector has the exprs which may still present in av_sets, but actually
500	can't be moved up due to bookkeeping created during code motion to another
501	fence. See comment near the call to update_and_record_unavailable_insns
502	for the detailed explanations. */
503	static vinsn_vec_t vec_bookkeeping_blocked_vinsns = vinsn_vec_t ();
504
505	/* This vector has vinsns which are scheduled with renaming on the first fence
506	and then seen on the second. For expressions with such vinsns, target
507	availability information may be wrong. */
508	static vinsn_vec_t vec_target_unavailable_vinsns = vinsn_vec_t ();
509
510	/* Vector to store temporary nops inserted in move_op to prevent removal
511	of empty bbs. */
512	static vec<insn_t> vec_temp_moveop_nops;
513
514	/* These bitmaps record original instructions scheduled on the current
515	iteration and bookkeeping copies created by them. */
516	static bitmap current_originators = NULL;
517	static bitmap current_copies = NULL;
518
519	/* This bitmap marks the blocks visited by code_motion_path_driver so we don't
520	visit them afterwards. */
521	static bitmap code_motion_visited_blocks = NULL;
522
523	/* Variables to accumulate different statistics. */
524
525	/* The number of bookkeeping copies created. */
526	static int stat_bookkeeping_copies;
527
528	/* The number of insns that required bookkeeiping for their scheduling. */
529	static int stat_insns_needed_bookkeeping;
530
531	/* The number of insns that got renamed. */
532	static int stat_renamed_scheduled;
533
534	/* The number of substitutions made during scheduling. */
535	static int stat_substitutions_total;
536
537
538	/* Forward declarations of static functions. */
539	static bool rtx_ok_for_substitution_p (rtx, rtx);
540	static int sel_rank_for_schedule (const void *, const void *);
541	static av_set_t find_sequential_best_exprs (bnd_t, expr_t, bool);
542	static basic_block find_block_for_bookkeeping (edge e1, edge e2, bool lax);
543
544	static rtx get_dest_from_orig_ops (av_set_t);
545	static basic_block generate_bookkeeping_insn (expr_t, edge, edge);
546	static bool find_used_regs (insn_t, av_set_t, regset, struct reg_rename *,
547	def_list_t *);
548	static bool move_op (insn_t, av_set_t, expr_t, rtx, expr_t, bool*);
549	static int code_motion_path_driver (insn_t, av_set_t, ilist_t,
550	cmpd_local_params_p, void *);
551	static void sel_sched_region_1 (void);
552	static void sel_sched_region_2 (int);
553	static av_set_t compute_av_set_inside_bb (insn_t, ilist_t, int, bool);
554
555	static void debug_state (state_t);
556
557
558	/* Functions that work with fences. */
559
560	/* Advance one cycle on FENCE. */
561	static void
562	advance_one_cycle (fence_t fence)
563	{
564	unsigned i;
565	int cycle;
566	rtx_insn *insn;
567
568	advance_state (FENCE_STATE (fence));
569	cycle = ++FENCE_CYCLE (fence);
570	FENCE_ISSUED_INSNS (fence) = 0;
571	FENCE_STARTS_CYCLE_P (fence) = 1;
572	can_issue_more = issue_rate;
573	FENCE_ISSUE_MORE (fence) = can_issue_more;
574
575	for (i = 0; vec_safe_iterate (FENCE_EXECUTING_INSNS (fence), ix: i, ptr: &insn); )
576	{
577	if (INSN_READY_CYCLE (insn) < cycle)
578	{
579	remove_from_deps (FENCE_DC (fence), insn);
580	FENCE_EXECUTING_INSNS (fence)->unordered_remove (ix: i);
581	continue;
582	}
583	i++;
584	}
585	if (sched_verbose >= 2)
586	{
587	sel_print (fmt: "Finished a cycle. Current cycle = %d\n", FENCE_CYCLE (fence));
588	debug_state (FENCE_STATE (fence));
589	}
590	}
591
592	/* Returns true when SUCC in a fallthru bb of INSN, possibly
593	skipping empty basic blocks. */
594	static bool
595	in_fallthru_bb_p (rtx_insn *insn, rtx succ)
596	{
597	basic_block bb = BLOCK_FOR_INSN (insn);
598	edge e;
599
600	if (bb == BLOCK_FOR_INSN (insn: succ))
601	return true;
602
603	e = find_fallthru_edge_from (bb);
604	if (e)
605	bb = e->dest;
606	else
607	return false;
608
609	while (sel_bb_empty_p (bb))
610	bb = bb->next_bb;
611
612	return bb == BLOCK_FOR_INSN (insn: succ);
613	}
614
615	/* Construct successor fences from OLD_FENCEs and put them in NEW_FENCES.
616	When a successor will continue a ebb, transfer all parameters of a fence
617	to the new fence. ORIG_MAX_SEQNO is the maximal seqno before this round
618	of scheduling helping to distinguish between the old and the new code. */
619	static void
620	extract_new_fences_from (flist_t old_fences, flist_tail_t new_fences,
621	int orig_max_seqno)
622	{
623	bool was_here_p = false;
624	insn_t insn = NULL;
625	insn_t succ;
626	succ_iterator si;
627	ilist_iterator ii;
628	fence_t fence = FLIST_FENCE (old_fences);
629	basic_block bb;
630
631	/* Get the only element of FENCE_BNDS (fence). */
632	FOR_EACH_INSN (insn, ii, FENCE_BNDS (fence))
633	{
634	gcc_assert (!was_here_p);
635	was_here_p = true;
636	}
637	gcc_assert (was_here_p && insn != NULL_RTX);
638
639	/* When in the "middle" of the block, just move this fence
640	to the new list. */
641	bb = BLOCK_FOR_INSN (insn);
642	if (! sel_bb_end_p (insn)
643	|| (single_succ_p (bb)
644	&& single_pred_p (bb: single_succ (bb))))
645	{
646	insn_t succ;
647
648	succ = (sel_bb_end_p (insn)
649	? sel_bb_head (single_succ (bb))
650	: NEXT_INSN (insn));
651
652	if (INSN_SEQNO (succ) > 0
653	&& INSN_SEQNO (succ) <= orig_max_seqno
654	&& INSN_SCHED_TIMES (succ) <= 0)
655	{
656	FENCE_INSN (fence) = succ;
657	move_fence_to_fences (old_fences, new_fences);
658
659	if (sched_verbose >= 1)
660	sel_print (fmt: "Fence %d continues as %d[%d] (state continue)\n",
661	INSN_UID (insn), INSN_UID (insn: succ), BLOCK_NUM (succ));
662	}
663	return;
664	}
665
666	/* Otherwise copy fence's structures to (possibly) multiple successors. */
667	FOR_EACH_SUCC_1 (succ, si, insn, SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
668	{
669	int seqno = INSN_SEQNO (succ);
670
671	if (seqno > 0 && seqno <= orig_max_seqno
672	&& (pipelining_p || INSN_SCHED_TIMES (succ) <= 0))
673	{
674	bool b = (in_same_ebb_p (insn, succ)
675	|| in_fallthru_bb_p (insn, succ));
676
677	if (sched_verbose >= 1)
678	sel_print (fmt: "Fence %d continues as %d[%d] (state %s)\n",
679	INSN_UID (insn), INSN_UID (insn: succ),
680	BLOCK_NUM (succ), b ? "continue" : "reset");
681
682	if (b)
683	add_dirty_fence_to_fences (new_fences, succ, fence);
684	else
685	{
686	/* Mark block of the SUCC as head of the new ebb. */
687	bitmap_set_bit (forced_ebb_heads, BLOCK_NUM (succ));
688	add_clean_fence_to_fences (new_fences, succ, fence);
689	}
690	}
691	}
692	}
693
694
695	/* Functions to support substitution. */
696
697	/* Returns whether INSN with dependence status DS is eligible for
698	substitution, i.e. it's a copy operation x := y, and RHS that is
699	moved up through this insn should be substituted. */
700	static bool
701	can_substitute_through_p (insn_t insn, ds_t ds)
702	{
703	/* We can substitute only true dependencies. */
704	if ((ds & DEP_OUTPUT)
705	|| (ds & DEP_ANTI)
706	|| ! INSN_RHS (insn)
707	|| ! INSN_LHS (insn))
708	return false;
709
710	/* Now we just need to make sure the INSN_RHS consists of only one
711	simple REG rtx. */
712	if (REG_P (INSN_LHS (insn))
713	&& REG_P (INSN_RHS (insn)))
714	return true;
715	return false;
716	}
717
718	/* Substitute all occurrences of INSN's destination in EXPR' vinsn with INSN's
719	source (if INSN is eligible for substitution). Returns TRUE if
720	substitution was actually performed, FALSE otherwise. Substitution might
721	be not performed because it's either EXPR' vinsn doesn't contain INSN's
722	destination or the resulting insn is invalid for the target machine.
723	When UNDO is true, perform unsubstitution instead (the difference is in
724	the part of rtx on which validate_replace_rtx is called). */
725	static bool
726	substitute_reg_in_expr (expr_t expr, insn_t insn, bool undo)
727	{
728	rtx *where;
729	bool new_insn_valid;
730	vinsn_t *vi = &EXPR_VINSN (expr);
731	bool has_rhs = VINSN_RHS (*vi) != NULL;
732	rtx old, new_rtx;
733
734	/* Do not try to replace in SET_DEST. Although we'll choose new
735	register for the RHS, we don't want to change RHS' original reg.
736	If the insn is not SET, we may still be able to substitute something
737	in it, and if we're here (don't have deps), it doesn't write INSN's
738	dest. */
739	where = (has_rhs
740	? &VINSN_RHS (*vi)
741	: &PATTERN (VINSN_INSN_RTX (*vi)));
742	old = undo ? INSN_RHS (insn) : INSN_LHS (insn);
743
744	/* Substitute if INSN has a form of x:=y and LHS(INSN) occurs in *VI. */
745	if (rtx_ok_for_substitution_p (old, *where))
746	{
747	rtx_insn *new_insn;
748	rtx *where_replace;
749
750	/* We should copy these rtxes before substitution. */
751	new_rtx = copy_rtx (undo ? INSN_LHS (insn) : INSN_RHS (insn));
752	new_insn = create_copy_of_insn_rtx (VINSN_INSN_RTX (*vi));
753
754	/* Where we'll replace.
755	WHERE_REPLACE should point inside NEW_INSN, so INSN_RHS couldn't be
756	used instead of SET_SRC. */
757	where_replace = (has_rhs
758	? &SET_SRC (PATTERN (new_insn))
759	: &PATTERN (insn: new_insn));
760
761	new_insn_valid
762	= validate_replace_rtx_part_nosimplify (old, new_rtx, where_replace,
763	new_insn);
764
765	/* ??? Actually, constrain_operands result depends upon choice of
766	destination register. E.g. if we allow single register to be an rhs,
767	and if we try to move dx=ax(as rhs) through ax=dx, we'll result
768	in invalid insn dx=dx, so we'll loose this rhs here.
769	Just can't come up with significant testcase for this, so just
770	leaving it for now. */
771	if (new_insn_valid)
772	{
773	change_vinsn_in_expr (expr,
774	create_vinsn_from_insn_rtx (new_insn, false));
775
776	/* Do not allow clobbering the address register of speculative
777	insns. */
778	if ((EXPR_SPEC_DONE_DS (expr) & SPECULATIVE)
779	&& register_unavailable_p (VINSN_REG_USES (EXPR_VINSN (expr)),
780	expr_dest_reg (expr)))
781	EXPR_TARGET_AVAILABLE (expr) = false;
782
783	return true;
784	}
785	else
786	return false;
787	}
788	else
789	return false;
790	}
791
792	/* Return the number of places WHAT appears within WHERE.
793	Bail out when we found a reference occupying several hard registers. */
794	static int
795	count_occurrences_equiv (const_rtx what, const_rtx where)
796	{
797	int count = 0;
798	subrtx_iterator::array_type array;
799	FOR_EACH_SUBRTX (iter, array, where, NONCONST)
800	{
801	const_rtx x = *iter;
802	if (REG_P (x) && REGNO (x) == REGNO (what))
803	{
804	/* Bail out if mode is different or more than one register is
805	used. */
806	if (GET_MODE (x) != GET_MODE (what) || REG_NREGS (x) > 1)
807	return 0;
808	count += 1;
809	}
810	else if (GET_CODE (x) == SUBREG
811	&& (!REG_P (SUBREG_REG (x))
812	|| REGNO (SUBREG_REG (x)) == REGNO (what)))
813	/* ??? Do not support substituting regs inside subregs. In that case,
814	simplify_subreg will be called by validate_replace_rtx, and
815	unsubstitution will fail later. */
816	return 0;
817	}
818	return count;
819	}
820
821	/* Returns TRUE if WHAT is found in WHERE rtx tree. */
822	static bool
823	rtx_ok_for_substitution_p (rtx what, rtx where)
824	{
825	return (count_occurrences_equiv (what, where) > 0);
826	}
827
828
829	/* Functions to support register renaming. */
830
831	/* Substitute VI's set source with REGNO. Returns newly created pattern
832	that has REGNO as its source. */
833	static rtx_insn *
834	create_insn_rtx_with_rhs (vinsn_t vi, rtx rhs_rtx)
835	{
836	rtx lhs_rtx;
837	rtx pattern;
838	rtx_insn *insn_rtx;
839
840	lhs_rtx = copy_rtx (VINSN_LHS (vi));
841
842	pattern = gen_rtx_SET (lhs_rtx, rhs_rtx);
843	insn_rtx = create_insn_rtx_from_pattern (pattern, NULL_RTX);
844
845	return insn_rtx;
846	}
847
848	/* Returns whether INSN's src can be replaced with register number
849	NEW_SRC_REG. E.g. the following insn is valid for i386:
850
851	(insn:HI 2205 6585 2207 727 ../../gcc/libiberty/regex.c:3337
852	(set (mem/s:QI (plus:SI (plus:SI (reg/f:SI 7 sp)
853	(reg:SI 0 ax [orig:770 c1 ] [770]))
854	(const_int 288 [0x120])) [0 str S1 A8])
855	(const_int 0 [0x0])) 43 {*movqi_1} (nil)
856	(nil))
857
858	But if we change (const_int 0 [0x0]) to (reg:QI 4 si), it will be invalid
859	because of operand constraints:
860
861	(define_insn "*movqi_1"
862	[(set (match_operand:QI 0 "nonimmediate_operand" "=q,q ,q ,r,r ,?r,m")
863	(match_operand:QI 1 "general_operand" " q,qn,qm,q,rn,qm,qn")
864	)]
865
866	So do constrain_operands here, before choosing NEW_SRC_REG as best
867	reg for rhs. */
868
869	static bool
870	replace_src_with_reg_ok_p (insn_t insn, rtx new_src_reg)
871	{
872	vinsn_t vi = INSN_VINSN (insn);
873	machine_mode mode;
874	rtx dst_loc;
875	bool res;
876
877	gcc_assert (VINSN_SEPARABLE_P (vi));
878
879	get_dest_and_mode (insn, &dst_loc, &mode);
880	gcc_assert (mode == GET_MODE (new_src_reg));
881
882	if (REG_P (dst_loc) && REGNO (new_src_reg) == REGNO (dst_loc))
883	return true;
884
885	/* See whether SET_SRC can be replaced with this register. */
886	validate_change (insn, &SET_SRC (PATTERN (insn)), new_src_reg, 1);
887	res = verify_changes (0);
888	cancel_changes (0);
889
890	return res;
891	}
892
893	/* Returns whether INSN still be valid after replacing it's DEST with
894	register NEW_REG. */
895	static bool
896	replace_dest_with_reg_ok_p (insn_t insn, rtx new_reg)
897	{
898	vinsn_t vi = INSN_VINSN (insn);
899	bool res;
900
901	/* We should deal here only with separable insns. */
902	gcc_assert (VINSN_SEPARABLE_P (vi));
903	gcc_assert (GET_MODE (VINSN_LHS (vi)) == GET_MODE (new_reg));
904
905	/* See whether SET_DEST can be replaced with this register. */
906	validate_change (insn, &SET_DEST (PATTERN (insn)), new_reg, 1);
907	res = verify_changes (0);
908	cancel_changes (0);
909
910	return res;
911	}
912
913	/* Create a pattern with rhs of VI and lhs of LHS_RTX. */
914	static rtx_insn *
915	create_insn_rtx_with_lhs (vinsn_t vi, rtx lhs_rtx)
916	{
917	rtx rhs_rtx;
918	rtx pattern;
919	rtx_insn *insn_rtx;
920
921	rhs_rtx = copy_rtx (VINSN_RHS (vi));
922
923	pattern = gen_rtx_SET (lhs_rtx, rhs_rtx);
924	insn_rtx = create_insn_rtx_from_pattern (pattern, NULL_RTX);
925
926	return insn_rtx;
927	}
928
929	/* Substitute lhs in the given expression EXPR for the register with number
930	NEW_REGNO. SET_DEST may be arbitrary rtx, not only register. */
931	static void
932	replace_dest_with_reg_in_expr (expr_t expr, rtx new_reg)
933	{
934	rtx_insn *insn_rtx;
935	vinsn_t vinsn;
936
937	insn_rtx = create_insn_rtx_with_lhs (EXPR_VINSN (expr), lhs_rtx: new_reg);
938	vinsn = create_vinsn_from_insn_rtx (insn_rtx, false);
939
940	change_vinsn_in_expr (expr, vinsn);
941	EXPR_WAS_RENAMED (expr) = 1;
942	EXPR_TARGET_AVAILABLE (expr) = 1;
943	}
944
945	/* Returns whether VI writes either one of the USED_REGS registers or,
946	if a register is a hard one, one of the UNAVAILABLE_HARD_REGS registers. */
947	static bool
948	vinsn_writes_one_of_regs_p (vinsn_t vi, regset used_regs,
949	HARD_REG_SET unavailable_hard_regs)
950	{
951	unsigned regno;
952	reg_set_iterator rsi;
953
954	EXECUTE_IF_SET_IN_REG_SET (VINSN_REG_SETS (vi), 0, regno, rsi)
955	{
956	if (REGNO_REG_SET_P (used_regs, regno))
957	return true;
958	if (HARD_REGISTER_NUM_P (regno)
959	&& TEST_HARD_REG_BIT (set: unavailable_hard_regs, bit: regno))
960	return true;
961	}
962
963	EXECUTE_IF_SET_IN_REG_SET (VINSN_REG_CLOBBERS (vi), 0, regno, rsi)
964	{
965	if (REGNO_REG_SET_P (used_regs, regno))
966	return true;
967	if (HARD_REGISTER_NUM_P (regno)
968	&& TEST_HARD_REG_BIT (set: unavailable_hard_regs, bit: regno))
969	return true;
970	}
971
972	return false;
973	}
974
975	/* Returns register class of the output register in INSN.
976	Returns NO_REGS for call insns because some targets have constraints on
977	destination register of a call insn.
978
979	Code adopted from regrename.cc::build_def_use. */
980	static enum reg_class
981	get_reg_class (rtx_insn *insn)
982	{
983	int i, n_ops;
984
985	extract_constrain_insn (insn);
986	preprocess_constraints (insn);
987	n_ops = recog_data.n_operands;
988
989	const operand_alternative *op_alt = which_op_alt ();
990	if (asm_noperands (PATTERN (insn)) > 0)
991	{
992	for (i = 0; i < n_ops; i++)
993	if (recog_data.operand_type[i] == OP_OUT)
994	{
995	rtx *loc = recog_data.operand_loc[i];
996	rtx op = *loc;
997	enum reg_class cl = alternative_class (alt: op_alt, i);
998
999	if (REG_P (op)
1000	&& REGNO (op) == ORIGINAL_REGNO (op))
1001	continue;
1002
1003	return cl;
1004	}
1005	}
1006	else if (!CALL_P (insn))
1007	{
1008	for (i = 0; i < n_ops + recog_data.n_dups; i++)
1009	{
1010	int opn = i < n_ops ? i : recog_data.dup_num[i - n_ops];
1011	enum reg_class cl = alternative_class (alt: op_alt, i: opn);
1012
1013	if (recog_data.operand_type[opn] == OP_OUT ||
1014	recog_data.operand_type[opn] == OP_INOUT)
1015	return cl;
1016	}
1017	}
1018
1019	/* Insns like
1020	(insn (set (reg:CCZ 17 flags) (compare:CCZ ...)))
1021	may result in returning NO_REGS, cause flags is written implicitly through
1022	CMP insn, which has no OP_OUT | OP_INOUT operands. */
1023	return NO_REGS;
1024	}
1025
1026	/* Calculate HARD_REGNO_RENAME_OK data for REGNO. */
1027	static void
1028	init_hard_regno_rename (int regno)
1029	{
1030	int cur_reg;
1031
1032	SET_HARD_REG_BIT (set&: sel_hrd.regs_for_rename[regno], bit: regno);
1033
1034	for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1035	{
1036	/* We are not interested in renaming in other regs. */
1037	if (!TEST_HARD_REG_BIT (set: sel_hrd.regs_ever_used, bit: cur_reg))
1038	continue;
1039
1040	if (HARD_REGNO_RENAME_OK (regno, cur_reg))
1041	SET_HARD_REG_BIT (set&: sel_hrd.regs_for_rename[regno], bit: cur_reg);
1042	}
1043	}
1044
1045	/* A wrapper around HARD_REGNO_RENAME_OK that will look into the hard regs
1046	data first. */
1047	static inline bool
1048	sel_hard_regno_rename_ok (int from ATTRIBUTE_UNUSED, int to ATTRIBUTE_UNUSED)
1049	{
1050	/* Check whether this is all calculated. */
1051	if (TEST_HARD_REG_BIT (set: sel_hrd.regs_for_rename[from], bit: from))
1052	return TEST_HARD_REG_BIT (set: sel_hrd.regs_for_rename[from], bit: to);
1053
1054	init_hard_regno_rename (regno: from);
1055
1056	return TEST_HARD_REG_BIT (set: sel_hrd.regs_for_rename[from], bit: to);
1057	}
1058
1059	/* Calculate set of registers that are capable of holding MODE. */
1060	static void
1061	init_regs_for_mode (machine_mode mode)
1062	{
1063	int cur_reg;
1064
1065	CLEAR_HARD_REG_SET (set&: sel_hrd.regs_for_mode[mode]);
1066
1067	for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1068	{
1069	int nregs;
1070	int i;
1071
1072	/* See whether it accepts all modes that occur in
1073	original insns. */
1074	if (!targetm.hard_regno_mode_ok (cur_reg, mode))
1075	continue;
1076
1077	nregs = hard_regno_nregs (regno: cur_reg, mode);
1078
1079	for (i = nregs - 1; i >= 0; --i)
1080	if (fixed_regs[cur_reg + i]
1081	|| global_regs[cur_reg + i]
1082	/* Can't use regs which aren't saved by
1083	the prologue. */
1084	|| !TEST_HARD_REG_BIT (set: sel_hrd.regs_ever_used, bit: cur_reg + i)
1085	/* Can't use regs with non-null REG_BASE_VALUE, because adjusting
1086	it affects aliasing globally and invalidates all AV sets. */
1087	|| get_reg_base_value (cur_reg + i)
1088	#ifdef LEAF_REGISTERS
1089	/* We can't use a non-leaf register if we're in a
1090	leaf function. */
1091	|| (crtl->is_leaf
1092	&& !LEAF_REGISTERS[cur_reg + i])
1093	#endif
1094	)
1095	break;
1096
1097	if (i >= 0)
1098	continue;
1099
1100	/* If the CUR_REG passed all the checks above,
1101	then it's ok. */
1102	SET_HARD_REG_BIT (set&: sel_hrd.regs_for_mode[mode], bit: cur_reg);
1103	}
1104
1105	sel_hrd.regs_for_mode_ok[mode] = true;
1106	}
1107
1108	/* Init all register sets gathered in HRD. */
1109	static void
1110	init_hard_regs_data (void)
1111	{
1112	int cur_reg = 0;
1113	int cur_mode = 0;
1114
1115	CLEAR_HARD_REG_SET (set&: sel_hrd.regs_ever_used);
1116	for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1117	if (df_regs_ever_live_p (cur_reg)
1118	|| crtl->abi->clobbers_full_reg_p (regno: cur_reg))
1119	SET_HARD_REG_BIT (set&: sel_hrd.regs_ever_used, bit: cur_reg);
1120
1121	/* Initialize registers that are valid based on mode when this is
1122	really needed. */
1123	for (cur_mode = 0; cur_mode < NUM_MACHINE_MODES; cur_mode++)
1124	sel_hrd.regs_for_mode_ok[cur_mode] = false;
1125
1126	/* Mark that all HARD_REGNO_RENAME_OK is not calculated. */
1127	for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1128	CLEAR_HARD_REG_SET (set&: sel_hrd.regs_for_rename[cur_reg]);
1129
1130	#ifdef STACK_REGS
1131	CLEAR_HARD_REG_SET (set&: sel_hrd.stack_regs);
1132
1133	for (cur_reg = FIRST_STACK_REG; cur_reg <= LAST_STACK_REG; cur_reg++)
1134	SET_HARD_REG_BIT (set&: sel_hrd.stack_regs, bit: cur_reg);
1135	#endif
1136	}
1137
1138	/* Mark hardware regs in REG_RENAME_P that are not suitable
1139	for renaming rhs in INSN due to hardware restrictions (register class,
1140	modes compatibility etc). This doesn't affect original insn's dest reg,
1141	if it isn't in USED_REGS. DEF is a definition insn of rhs for which the
1142	destination register is sought. LHS (DEF->ORIG_INSN) may be REG or MEM.
1143	Registers that are in used_regs are always marked in
1144	unavailable_hard_regs as well. */
1145
1146	static void
1147	mark_unavailable_hard_regs (def_t def, struct reg_rename *reg_rename_p,
1148	regset used_regs ATTRIBUTE_UNUSED)
1149	{
1150	machine_mode mode;
1151	enum reg_class cl = NO_REGS;
1152	rtx orig_dest;
1153	unsigned cur_reg, regno;
1154	hard_reg_set_iterator hrsi;
1155
1156	gcc_assert (GET_CODE (PATTERN (def->orig_insn)) == SET);
1157	gcc_assert (reg_rename_p);
1158
1159	orig_dest = SET_DEST (PATTERN (def->orig_insn));
1160
1161	/* We have decided not to rename 'mem = something;' insns, as 'something'
1162	is usually a register. */
1163	if (!REG_P (orig_dest))
1164	return;
1165
1166	regno = REGNO (orig_dest);
1167
1168	/* If before reload, don't try to work with pseudos. */
1169	if (!reload_completed && !HARD_REGISTER_NUM_P (regno))
1170	return;
1171
1172	if (reload_completed)
1173	cl = get_reg_class (insn: def->orig_insn);
1174
1175	/* Stop if the original register is one of the fixed_regs, global_regs or
1176	frame pointer, or we could not discover its class. */
1177	if (fixed_regs[regno]
1178	|| global_regs[regno]
1179	|| (!HARD_FRAME_POINTER_IS_FRAME_POINTER && frame_pointer_needed
1180	&& regno == HARD_FRAME_POINTER_REGNUM)
1181	|| (HARD_FRAME_POINTER_IS_FRAME_POINTER && frame_pointer_needed
1182	&& regno == FRAME_POINTER_REGNUM)
1183	|| (reload_completed && cl == NO_REGS))
1184	{
1185	SET_HARD_REG_SET (reg_rename_p->unavailable_hard_regs);
1186
1187	/* Give a chance for original register, if it isn't in used_regs. */
1188	if (!def->crossed_call_abis)
1189	CLEAR_HARD_REG_BIT (set&: reg_rename_p->unavailable_hard_regs, bit: regno);
1190
1191	return;
1192	}
1193
1194	/* If something allocated on stack in this function, mark frame pointer
1195	register unavailable, considering also modes.
1196	FIXME: it is enough to do this once per all original defs. */
1197	if (frame_pointer_needed)
1198	{
1199	add_to_hard_reg_set (regs: &reg_rename_p->unavailable_hard_regs,
1200	Pmode, FRAME_POINTER_REGNUM);
1201
1202	if (!HARD_FRAME_POINTER_IS_FRAME_POINTER)
1203	add_to_hard_reg_set (regs: &reg_rename_p->unavailable_hard_regs,
1204	Pmode, HARD_FRAME_POINTER_REGNUM);
1205	}
1206
1207	#ifdef STACK_REGS
1208	/* For the stack registers the presence of FIRST_STACK_REG in USED_REGS
1209	is equivalent to as if all stack regs were in this set.
1210	I.e. no stack register can be renamed, and even if it's an original
1211	register here we make sure it won't be lifted over it's previous def
1212	(it's previous def will appear as if it's a FIRST_STACK_REG def.
1213	The HARD_REGNO_RENAME_OK covers other cases in condition below. */
1214	if (IN_RANGE (REGNO (orig_dest), FIRST_STACK_REG, LAST_STACK_REG)
1215	&& REGNO_REG_SET_P (used_regs, FIRST_STACK_REG))
1216	reg_rename_p->unavailable_hard_regs |= sel_hrd.stack_regs;
1217	#endif
1218
1219	mode = GET_MODE (orig_dest);
1220
1221	/* If there's a call on this path, make regs from full_reg_clobbers
1222	unavailable.
1223
1224	??? It would be better to track the set of clobbered registers
1225	directly, but that would be quite expensive in a def_t. */
1226	if (def->crossed_call_abis)
1227	reg_rename_p->unavailable_hard_regs
1228	|= call_clobbers_in_region (def->crossed_call_abis,
1229	reg_class_contents[ALL_REGS], mode);
1230
1231	/* Stop here before reload: we need FRAME_REGS, STACK_REGS, and
1232	crossed_call_abis, but not register classes. */
1233	if (!reload_completed)
1234	return;
1235
1236	/* Leave regs as 'available' only from the current
1237	register class. */
1238	reg_rename_p->available_for_renaming = reg_class_contents[cl];
1239
1240	/* Leave only registers available for this mode. */
1241	if (!sel_hrd.regs_for_mode_ok[mode])
1242	init_regs_for_mode (mode);
1243	reg_rename_p->available_for_renaming &= sel_hrd.regs_for_mode[mode];
1244
1245	/* Leave only those that are ok to rename. */
1246	EXECUTE_IF_SET_IN_HARD_REG_SET (reg_rename_p->available_for_renaming,
1247	0, cur_reg, hrsi)
1248	{
1249	int nregs;
1250	int i;
1251
1252	nregs = hard_regno_nregs (regno: cur_reg, mode);
1253	gcc_assert (nregs > 0);
1254
1255	for (i = nregs - 1; i >= 0; --i)
1256	if (! sel_hard_regno_rename_ok (from: regno + i, to: cur_reg + i))
1257	break;
1258
1259	if (i >= 0)
1260	CLEAR_HARD_REG_BIT (set&: reg_rename_p->available_for_renaming,
1261	bit: cur_reg);
1262	}
1263
1264	reg_rename_p->available_for_renaming &= ~reg_rename_p->unavailable_hard_regs;
1265
1266	/* Regno is always ok from the renaming part of view, but it really
1267	could be in *unavailable_hard_regs already, so set it here instead
1268	of there. */
1269	SET_HARD_REG_BIT (set&: reg_rename_p->available_for_renaming, bit: regno);
1270	}
1271
1272	/* reg_rename_tick[REG1] > reg_rename_tick[REG2] if REG1 was chosen as the
1273	best register more recently than REG2. */
1274	static int reg_rename_tick[FIRST_PSEUDO_REGISTER];
1275
1276	/* Indicates the number of times renaming happened before the current one. */
1277	static int reg_rename_this_tick;
1278
1279	/* Choose the register among free, that is suitable for storing
1280	the rhs value.
1281
1282	ORIGINAL_INSNS is the list of insns where the operation (rhs)
1283	originally appears. There could be multiple original operations
1284	for single rhs since we moving it up and merging along different
1285	paths.
1286
1287	Some code is adapted from regrename.cc (regrename_optimize).
1288	If original register is available, function returns it.
1289	Otherwise it performs the checks, so the new register should
1290	comply with the following:
1291	- it should not violate any live ranges (such registers are in
1292	REG_RENAME_P->available_for_renaming set);
1293	- it should not be in the HARD_REGS_USED regset;
1294	- it should be in the class compatible with original uses;
1295	- it should not be clobbered through reference with different mode;
1296	- if we're in the leaf function, then the new register should
1297	not be in the LEAF_REGISTERS;
1298	- etc.
1299
1300	If several registers meet the conditions, the register with smallest
1301	tick is returned to achieve more even register allocation.
1302
1303	If original register seems to be ok, we set *IS_ORIG_REG_P_PTR to true.
1304
1305	If no register satisfies the above conditions, NULL_RTX is returned. */
1306	static rtx
1307	choose_best_reg_1 (HARD_REG_SET hard_regs_used,
1308	struct reg_rename *reg_rename_p,
1309	def_list_t original_insns, bool *is_orig_reg_p_ptr)
1310	{
1311	int best_new_reg;
1312	unsigned cur_reg;
1313	machine_mode mode = VOIDmode;
1314	unsigned regno, i, n;
1315	hard_reg_set_iterator hrsi;
1316	def_list_iterator di;
1317	def_t def;
1318
1319	/* If original register is available, return it. */
1320	*is_orig_reg_p_ptr = true;
1321
1322	FOR_EACH_DEF (def, di, original_insns)
1323	{
1324	rtx orig_dest = SET_DEST (PATTERN (def->orig_insn));
1325
1326	gcc_assert (REG_P (orig_dest));
1327
1328	/* Check that all original operations have the same mode.
1329	This is done for the next loop; if we'd return from this
1330	loop, we'd check only part of them, but in this case
1331	it doesn't matter. */
1332	if (mode == VOIDmode)
1333	mode = GET_MODE (orig_dest);
1334	gcc_assert (mode == GET_MODE (orig_dest));
1335
1336	regno = REGNO (orig_dest);
1337	for (i = 0, n = REG_NREGS (orig_dest); i < n; i++)
1338	if (TEST_HARD_REG_BIT (set: hard_regs_used, bit: regno + i))
1339	break;
1340
1341	/* All hard registers are available. */
1342	if (i == n)
1343	{
1344	gcc_assert (mode != VOIDmode);
1345
1346	/* Hard registers should not be shared. */
1347	return gen_rtx_REG (mode, regno);
1348	}
1349	}
1350
1351	*is_orig_reg_p_ptr = false;
1352	best_new_reg = -1;
1353
1354	/* Among all available regs choose the register that was
1355	allocated earliest. */
1356	EXECUTE_IF_SET_IN_HARD_REG_SET (reg_rename_p->available_for_renaming,
1357	0, cur_reg, hrsi)
1358	if (! TEST_HARD_REG_BIT (set: hard_regs_used, bit: cur_reg))
1359	{
1360	/* Check that all hard regs for mode are available. */
1361	for (i = 1, n = hard_regno_nregs (regno: cur_reg, mode); i < n; i++)
1362	if (TEST_HARD_REG_BIT (set: hard_regs_used, bit: cur_reg + i)
1363	|| !TEST_HARD_REG_BIT (set: reg_rename_p->available_for_renaming,
1364	bit: cur_reg + i))
1365	break;
1366
1367	if (i < n)
1368	continue;
1369
1370	/* All hard registers are available. */
1371	if (best_new_reg < 0
1372	|| reg_rename_tick[cur_reg] < reg_rename_tick[best_new_reg])
1373	{
1374	best_new_reg = cur_reg;
1375
1376	/* Return immediately when we know there's no better reg. */
1377	if (! reg_rename_tick[best_new_reg])
1378	break;
1379	}
1380	}
1381
1382	if (best_new_reg >= 0)
1383	{
1384	/* Use the check from the above loop. */
1385	gcc_assert (mode != VOIDmode);
1386	return gen_rtx_REG (mode, best_new_reg);
1387	}
1388
1389	return NULL_RTX;
1390	}
1391
1392	/* A wrapper around choose_best_reg_1 () to verify that we make correct
1393	assumptions about available registers in the function. */
1394	static rtx
1395	choose_best_reg (HARD_REG_SET hard_regs_used, struct reg_rename *reg_rename_p,
1396	def_list_t original_insns, bool *is_orig_reg_p_ptr)
1397	{
1398	rtx best_reg = choose_best_reg_1 (hard_regs_used, reg_rename_p,
1399	original_insns, is_orig_reg_p_ptr);
1400
1401	/* FIXME loop over hard_regno_nregs here. */
1402	gcc_assert (best_reg == NULL_RTX
1403	|| TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, REGNO (best_reg)));
1404
1405	return best_reg;
1406	}
1407
1408	/* Choose the pseudo register for storing rhs value. As this is supposed
1409	to work before reload, we return either the original register or make
1410	the new one. The parameters are the same that in choose_nest_reg_1
1411	functions, except that USED_REGS may contain pseudos.
1412	If we work with hard regs, check also REG_RENAME_P->UNAVAILABLE_HARD_REGS.
1413
1414	TODO: take into account register pressure while doing this. Up to this
1415	moment, this function would never return NULL for pseudos, but we should
1416	not rely on this. */
1417	static rtx
1418	choose_best_pseudo_reg (regset used_regs,
1419	struct reg_rename *reg_rename_p,
1420	def_list_t original_insns, bool *is_orig_reg_p_ptr)
1421	{
1422	def_list_iterator i;
1423	def_t def;
1424	machine_mode mode = VOIDmode;
1425	bool bad_hard_regs = false;
1426
1427	/* We should not use this after reload. */
1428	gcc_assert (!reload_completed);
1429
1430	/* If original register is available, return it. */
1431	*is_orig_reg_p_ptr = true;
1432
1433	FOR_EACH_DEF (def, i, original_insns)
1434	{
1435	rtx dest = SET_DEST (PATTERN (def->orig_insn));
1436	int orig_regno;
1437
1438	gcc_assert (REG_P (dest));
1439
1440	/* Check that all original operations have the same mode. */
1441	if (mode == VOIDmode)
1442	mode = GET_MODE (dest);
1443	else
1444	gcc_assert (mode == GET_MODE (dest));
1445	orig_regno = REGNO (dest);
1446
1447	/* Check that nothing in used_regs intersects with orig_regno. When
1448	we have a hard reg here, still loop over hard_regno_nregs. */
1449	if (HARD_REGISTER_NUM_P (orig_regno))
1450	{
1451	int j, n;
1452	for (j = 0, n = REG_NREGS (dest); j < n; j++)
1453	if (REGNO_REG_SET_P (used_regs, orig_regno + j))
1454	break;
1455	if (j < n)
1456	continue;
1457	}
1458	else
1459	{
1460	if (REGNO_REG_SET_P (used_regs, orig_regno))
1461	continue;
1462	}
1463	if (HARD_REGISTER_NUM_P (orig_regno))
1464	{
1465	gcc_assert (df_regs_ever_live_p (orig_regno));
1466
1467	/* For hard registers, we have to check hardware imposed
1468	limitations (frame/stack registers, calls crossed). */
1469	if (!TEST_HARD_REG_BIT (set: reg_rename_p->unavailable_hard_regs,
1470	bit: orig_regno))
1471	{
1472	/* Don't let register cross a call if it doesn't already
1473	cross one. This condition is written in accordance with
1474	that in sched-deps.cc sched_analyze_reg(). */
1475	if (!reg_rename_p->crossed_call_abis
1476	|| REG_N_CALLS_CROSSED (orig_regno) > 0)
1477	return gen_rtx_REG (mode, orig_regno);
1478	}
1479
1480	bad_hard_regs = true;
1481	}
1482	else
1483	return dest;
1484	}
1485
1486	*is_orig_reg_p_ptr = false;
1487
1488	/* We had some original hard registers that couldn't be used.
1489	Those were likely special. Don't try to create a pseudo. */
1490	if (bad_hard_regs)
1491	return NULL_RTX;
1492
1493	/* We haven't found a register from original operations. Get a new one.
1494	FIXME: control register pressure somehow. */
1495	{
1496	rtx new_reg = gen_reg_rtx (mode);
1497
1498	gcc_assert (mode != VOIDmode);
1499
1500	max_regno = max_reg_num ();
1501	maybe_extend_reg_info_p ();
1502	REG_N_CALLS_CROSSED (REGNO (new_reg))
1503	= reg_rename_p->crossed_call_abis ? 1 : 0;
1504
1505	return new_reg;
1506	}
1507	}
1508
1509	/* True when target of EXPR is available due to EXPR_TARGET_AVAILABLE,
1510	USED_REGS and REG_RENAME_P->UNAVAILABLE_HARD_REGS. */
1511	static void
1512	verify_target_availability (expr_t expr, regset used_regs,
1513	struct reg_rename *reg_rename_p)
1514	{
1515	unsigned n, i, regno;
1516	machine_mode mode;
1517	bool target_available, live_available, hard_available;
1518
1519	if (!REG_P (EXPR_LHS (expr)) || EXPR_TARGET_AVAILABLE (expr) < 0)
1520	return;
1521
1522	regno = expr_dest_regno (expr);
1523	mode = GET_MODE (EXPR_LHS (expr));
1524	target_available = EXPR_TARGET_AVAILABLE (expr) == 1;
1525	n = HARD_REGISTER_NUM_P (regno) ? hard_regno_nregs (regno, mode) : 1;
1526
1527	live_available = hard_available = true;
1528	for (i = 0; i < n; i++)
1529	{
1530	if (bitmap_bit_p (used_regs, regno + i))
1531	live_available = false;
1532	if (TEST_HARD_REG_BIT (set: reg_rename_p->unavailable_hard_regs, bit: regno + i))
1533	hard_available = false;
1534	}
1535
1536	/* When target is not available, it may be due to hard register
1537	restrictions, e.g. crosses calls, so we check hard_available too. */
1538	if (target_available)
1539	gcc_assert (live_available);
1540	else
1541	/* Check only if we haven't scheduled something on the previous fence,
1542	cause due to MAX_SOFTWARE_LOOKAHEAD_WINDOW_SIZE issues
1543	and having more than one fence, we may end having targ_un in a block
1544	in which successors target register is actually available.
1545
1546	The last condition handles the case when a dependence from a call insn
1547	was created in sched-deps.cc for insns with destination registers that
1548	never crossed a call before, but do cross one after our code motion.
1549
1550	FIXME: in the latter case, we just uselessly called find_used_regs,
1551	because we can't move this expression with any other register
1552	as well. */
1553	gcc_assert (scheduled_something_on_previous_fence || !live_available
1554	|| !hard_available
1555	|| (!reload_completed
1556	&& reg_rename_p->crossed_call_abis
1557	&& REG_N_CALLS_CROSSED (regno) == 0));
1558	}
1559
1560	/* Collect unavailable registers due to liveness for EXPR from BNDS
1561	into USED_REGS. Save additional information about available
1562	registers and unavailable due to hardware restriction registers
1563	into REG_RENAME_P structure. Save original insns into ORIGINAL_INSNS
1564	list. */
1565	static void
1566	collect_unavailable_regs_from_bnds (expr_t expr, blist_t bnds, regset used_regs,
1567	struct reg_rename *reg_rename_p,
1568	def_list_t *original_insns)
1569	{
1570	for (; bnds; bnds = BLIST_NEXT (bnds))
1571	{
1572	bool res;
1573	av_set_t orig_ops = NULL;
1574	bnd_t bnd = BLIST_BND (bnds);
1575
1576	/* If the chosen best expr doesn't belong to current boundary,
1577	skip it. */
1578	if (!av_set_is_in_p (BND_AV1 (bnd), EXPR_VINSN (expr)))
1579	continue;
1580
1581	/* Put in ORIG_OPS all exprs from this boundary that became
1582	RES on top. */
1583	orig_ops = find_sequential_best_exprs (bnd, expr, false);
1584
1585	/* Compute used regs and OR it into the USED_REGS. */
1586	res = find_used_regs (BND_TO (bnd), orig_ops, used_regs,
1587	reg_rename_p, original_insns);
1588
1589	/* FIXME: the assert is true until we'd have several boundaries. */
1590	gcc_assert (res);
1591	av_set_clear (&orig_ops);
1592	}
1593	}
1594
1595	/* Return TRUE if it is possible to replace LHSes of ORIG_INSNS with BEST_REG.
1596	If BEST_REG is valid, replace LHS of EXPR with it. */
1597	static bool
1598	try_replace_dest_reg (ilist_t orig_insns, rtx best_reg, expr_t expr)
1599	{
1600	/* Try whether we'll be able to generate the insn
1601	'dest := best_reg' at the place of the original operation. */
1602	for (; orig_insns; orig_insns = ILIST_NEXT (orig_insns))
1603	{
1604	insn_t orig_insn = DEF_LIST_DEF (orig_insns)->orig_insn;
1605
1606	gcc_assert (EXPR_SEPARABLE_P (INSN_EXPR (orig_insn)));
1607
1608	if (REGNO (best_reg) != REGNO (INSN_LHS (orig_insn))
1609	&& (! replace_src_with_reg_ok_p (insn: orig_insn, new_src_reg: best_reg)
1610	|| ! replace_dest_with_reg_ok_p (insn: orig_insn, new_reg: best_reg)))
1611	return false;
1612	}
1613
1614	/* Make sure that EXPR has the right destination
1615	register. */
1616	if (expr_dest_regno (expr) != REGNO (best_reg))
1617	{
1618	rtx_insn *vinsn = EXPR_INSN_RTX (expr);
1619	validate_change (vinsn, &SET_DEST (PATTERN (vinsn)), best_reg, 1);
1620	bool res = verify_changes (0);
1621	cancel_changes (0);
1622	if (!res)
1623	return false;
1624	replace_dest_with_reg_in_expr (expr, new_reg: best_reg);
1625	}
1626	else
1627	EXPR_TARGET_AVAILABLE (expr) = 1;
1628
1629	return true;
1630	}
1631
1632	/* Select and assign best register to EXPR searching from BNDS.
1633	Set *IS_ORIG_REG_P to TRUE if original register was selected.
1634	Return FALSE if no register can be chosen, which could happen when:
1635	* EXPR_SEPARABLE_P is true but we were unable to find suitable register;
1636	* EXPR_SEPARABLE_P is false but the insn sets/clobbers one of the registers
1637	that are used on the moving path. */
1638	static bool
1639	find_best_reg_for_expr (expr_t expr, blist_t bnds, bool *is_orig_reg_p)
1640	{
1641	static struct reg_rename reg_rename_data;
1642
1643	regset used_regs;
1644	def_list_t original_insns = NULL;
1645	bool reg_ok;
1646
1647	*is_orig_reg_p = false;
1648
1649	/* Don't bother to do anything if this insn doesn't set any registers. */
1650	if (bitmap_empty_p (VINSN_REG_SETS (EXPR_VINSN (expr)))
1651	&& bitmap_empty_p (VINSN_REG_CLOBBERS (EXPR_VINSN (expr))))
1652	return true;
1653
1654	used_regs = get_clear_regset_from_pool ();
1655	CLEAR_HARD_REG_SET (set&: reg_rename_data.unavailable_hard_regs);
1656
1657	collect_unavailable_regs_from_bnds (expr, bnds, used_regs, reg_rename_p: &reg_rename_data,
1658	original_insns: &original_insns);
1659
1660	/* If after reload, make sure we're working with hard regs here. */
1661	if (flag_checking && reload_completed)
1662	{
1663	reg_set_iterator rsi;
1664	unsigned i;
1665
1666	EXECUTE_IF_SET_IN_REG_SET (used_regs, FIRST_PSEUDO_REGISTER, i, rsi)
1667	gcc_unreachable ();
1668	}
1669
1670	if (EXPR_SEPARABLE_P (expr))
1671	{
1672	rtx best_reg = NULL_RTX;
1673	/* Check that we have computed availability of a target register
1674	correctly. */
1675	verify_target_availability (expr, used_regs, reg_rename_p: &reg_rename_data);
1676
1677	/* Turn everything in hard regs after reload. */
1678	if (reload_completed)
1679	{
1680	HARD_REG_SET hard_regs_used;
1681	REG_SET_TO_HARD_REG_SET (hard_regs_used, used_regs);
1682
1683	/* Join hard registers unavailable due to register class
1684	restrictions and live range intersection. */
1685	hard_regs_used |= reg_rename_data.unavailable_hard_regs;
1686
1687	best_reg = choose_best_reg (hard_regs_used, reg_rename_p: &reg_rename_data,
1688	original_insns, is_orig_reg_p_ptr: is_orig_reg_p);
1689	}
1690	else
1691	best_reg = choose_best_pseudo_reg (used_regs, reg_rename_p: &reg_rename_data,
1692	original_insns, is_orig_reg_p_ptr: is_orig_reg_p);
1693
1694	if (!best_reg)
1695	reg_ok = false;
1696	else if (*is_orig_reg_p)
1697	{
1698	/* In case of unification BEST_REG may be different from EXPR's LHS
1699	when EXPR's LHS is unavailable, and there is another LHS among
1700	ORIGINAL_INSNS. */
1701	reg_ok = try_replace_dest_reg (orig_insns: original_insns, best_reg, expr);
1702	}
1703	else
1704	{
1705	/* Forbid renaming of low-cost insns. */
1706	if (sel_vinsn_cost (EXPR_VINSN (expr)) < 2)
1707	reg_ok = false;
1708	else
1709	reg_ok = try_replace_dest_reg (orig_insns: original_insns, best_reg, expr);
1710	}
1711	}
1712	else
1713	{
1714	/* If !EXPR_SCHEDULE_AS_RHS (EXPR), just make sure INSN doesn't set
1715	any of the HARD_REGS_USED set. */
1716	if (vinsn_writes_one_of_regs_p (EXPR_VINSN (expr), used_regs,
1717	unavailable_hard_regs: reg_rename_data.unavailable_hard_regs))
1718	{
1719	reg_ok = false;
1720	gcc_assert (EXPR_TARGET_AVAILABLE (expr) <= 0);
1721	}
1722	else
1723	{
1724	reg_ok = true;
1725	gcc_assert (EXPR_TARGET_AVAILABLE (expr) != 0);
1726	}
1727	}
1728
1729	ilist_clear (&original_insns);
1730	return_regset_to_pool (used_regs);
1731
1732	return reg_ok;
1733	}
1734
1735
1736	/* Return true if dependence described by DS can be overcomed. */
1737	static bool
1738	can_speculate_dep_p (ds_t ds)
1739	{
1740	if (spec_info == NULL)
1741	return false;
1742
1743	/* Leave only speculative data. */
1744	ds &= SPECULATIVE;
1745
1746	if (ds == 0)
1747	return false;
1748
1749	{
1750	/* FIXME: make sched-deps.cc produce only those non-hard dependencies,
1751	that we can overcome. */
1752	ds_t spec_mask = spec_info->mask;
1753
1754	if ((ds & spec_mask) != ds)
1755	return false;
1756	}
1757
1758	if (ds_weak (ds) < spec_info->data_weakness_cutoff)
1759	return false;
1760
1761	return true;
1762	}
1763
1764	/* Get a speculation check instruction.
1765	C_EXPR is a speculative expression,
1766	CHECK_DS describes speculations that should be checked,
1767	ORIG_INSN is the original non-speculative insn in the stream. */
1768	static insn_t
1769	create_speculation_check (expr_t c_expr, ds_t check_ds, insn_t orig_insn)
1770	{
1771	rtx check_pattern;
1772	rtx_insn *insn_rtx;
1773	insn_t insn;
1774	basic_block recovery_block;
1775	rtx_insn *label;
1776
1777	/* Create a recovery block if target is going to emit branchy check, or if
1778	ORIG_INSN was speculative already. */
1779	if (targetm.sched.needs_block_p (check_ds)
1780	|| EXPR_SPEC_DONE_DS (INSN_EXPR (orig_insn)) != 0)
1781	{
1782	recovery_block = sel_create_recovery_block (orig_insn);
1783	label = BB_HEAD (recovery_block);
1784	}
1785	else
1786	{
1787	recovery_block = NULL;
1788	label = NULL;
1789	}
1790
1791	/* Get pattern of the check. */
1792	check_pattern = targetm.sched.gen_spec_check (EXPR_INSN_RTX (c_expr), label,
1793	check_ds);
1794
1795	gcc_assert (check_pattern != NULL);
1796
1797	/* Emit check. */
1798	insn_rtx = create_insn_rtx_from_pattern (check_pattern, label);
1799
1800	insn = sel_gen_insn_from_rtx_after (insn_rtx, INSN_EXPR (orig_insn),
1801	INSN_SEQNO (orig_insn), orig_insn);
1802
1803	/* Make check to be non-speculative. */
1804	EXPR_SPEC_DONE_DS (INSN_EXPR (insn)) = 0;
1805	INSN_SPEC_CHECKED_DS (insn) = check_ds;
1806
1807	/* Decrease priority of check by difference of load/check instruction
1808	latencies. */
1809	EXPR_PRIORITY (INSN_EXPR (insn)) -= (sel_vinsn_cost (INSN_VINSN (orig_insn))
1810	- sel_vinsn_cost (INSN_VINSN (insn)));
1811
1812	/* Emit copy of original insn (though with replaced target register,
1813	if needed) to the recovery block. */
1814	if (recovery_block != NULL)
1815	{
1816	rtx twin_rtx;
1817
1818	twin_rtx = copy_rtx (PATTERN (EXPR_INSN_RTX (c_expr)));
1819	twin_rtx = create_insn_rtx_from_pattern (twin_rtx, NULL_RTX);
1820	sel_gen_recovery_insn_from_rtx_after (twin_rtx,
1821	INSN_EXPR (orig_insn),
1822	INSN_SEQNO (insn),
1823	bb_note (recovery_block));
1824	}
1825
1826	/* If we've generated a data speculation check, make sure
1827	that all the bookkeeping instruction we'll create during
1828	this move_op () will allocate an ALAT entry so that the
1829	check won't fail.
1830	In case of control speculation we must convert C_EXPR to control
1831	speculative mode, because failing to do so will bring us an exception
1832	thrown by the non-control-speculative load. */
1833	check_ds = ds_get_max_dep_weak (check_ds);
1834	speculate_expr (c_expr, check_ds);
1835
1836	return insn;
1837	}
1838
1839	/* True when INSN is a "regN = regN" copy. */
1840	static bool
1841	identical_copy_p (rtx_insn *insn)
1842	{
1843	rtx lhs, rhs, pat;
1844
1845	pat = PATTERN (insn);
1846
1847	if (GET_CODE (pat) != SET)
1848	return false;
1849
1850	lhs = SET_DEST (pat);
1851	if (!REG_P (lhs))
1852	return false;
1853
1854	rhs = SET_SRC (pat);
1855	if (!REG_P (rhs))
1856	return false;
1857
1858	return REGNO (lhs) == REGNO (rhs);
1859	}
1860
1861	/* Undo all transformations on *AV_PTR that were done when
1862	moving through INSN. */
1863	static void
1864	undo_transformations (av_set_t *av_ptr, rtx_insn *insn)
1865	{
1866	av_set_iterator av_iter;
1867	expr_t expr;
1868	av_set_t new_set = NULL;
1869
1870	/* First, kill any EXPR that uses registers set by an insn. This is
1871	required for correctness. */
1872	FOR_EACH_EXPR_1 (expr, av_iter, av_ptr)
1873	if (!sched_insns_conditions_mutex_p (insn, EXPR_INSN_RTX (expr))
1874	&& bitmap_intersect_p (INSN_REG_SETS (insn),
1875	VINSN_REG_USES (EXPR_VINSN (expr)))
1876	/* When an insn looks like 'r1 = r1', we could substitute through
1877	it, but the above condition will still hold. This happened with
1878	gcc.c-torture/execute/961125-1.c. */
1879	&& !identical_copy_p (insn))
1880	{
1881	if (sched_verbose >= 6)
1882	sel_print (fmt: "Expr %d removed due to use/set conflict\n",
1883	INSN_UID (EXPR_INSN_RTX (expr)));
1884	av_set_iter_remove (&av_iter);
1885	}
1886
1887	/* Undo transformations looking at the history vector. */
1888	FOR_EACH_EXPR (expr, av_iter, *av_ptr)
1889	{
1890	int index = find_in_history_vect (EXPR_HISTORY_OF_CHANGES (expr),
1891	insn, EXPR_VINSN (expr), true);
1892
1893	if (index >= 0)
1894	{
1895	expr_history_def *phist;
1896
1897	phist = &EXPR_HISTORY_OF_CHANGES (expr)[index];
1898
1899	switch (phist->type)
1900	{
1901	case TRANS_SPECULATION:
1902	{
1903	ds_t old_ds, new_ds;
1904
1905	/* Compute the difference between old and new speculative
1906	statuses: that's what we need to check.
1907	Earlier we used to assert that the status will really
1908	change. This no longer works because only the probability
1909	bits in the status may have changed during compute_av_set,
1910	and in the case of merging different probabilities of the
1911	same speculative status along different paths we do not
1912	record this in the history vector. */
1913	old_ds = phist->spec_ds;
1914	new_ds = EXPR_SPEC_DONE_DS (expr);
1915
1916	old_ds &= SPECULATIVE;
1917	new_ds &= SPECULATIVE;
1918	new_ds &= ~old_ds;
1919
1920	EXPR_SPEC_TO_CHECK_DS (expr) |= new_ds;
1921	break;
1922	}
1923	case TRANS_SUBSTITUTION:
1924	{
1925	expr_def _tmp_expr, *tmp_expr = &_tmp_expr;
1926	vinsn_t new_vi;
1927	bool add = true;
1928
1929	new_vi = phist->old_expr_vinsn;
1930
1931	gcc_assert (VINSN_SEPARABLE_P (new_vi)
1932	== EXPR_SEPARABLE_P (expr));
1933	copy_expr (tmp_expr, expr);
1934
1935	if (vinsn_equal_p (phist->new_expr_vinsn,
1936	EXPR_VINSN (tmp_expr)))
1937	change_vinsn_in_expr (tmp_expr, new_vi);
1938	else
1939	/* This happens when we're unsubstituting on a bookkeeping
1940	copy, which was in turn substituted. The history is wrong
1941	in this case. Do it the hard way. */
1942	add = substitute_reg_in_expr (expr: tmp_expr, insn, undo: true);
1943	if (add)
1944	av_set_add (&new_set, tmp_expr);
1945	clear_expr (tmp_expr);
1946	break;
1947	}
1948	default:
1949	gcc_unreachable ();
1950	}
1951	}
1952
1953	}
1954
1955	av_set_union_and_clear (av_ptr, &new_set, NULL);
1956	}
1957
1958
1959	/* Moveup_* helpers for code motion and computing av sets. */
1960
1961	/* Propagates EXPR inside an insn group through THROUGH_INSN.
1962	The difference from the below function is that only substitution is
1963	performed. */
1964	static enum MOVEUP_EXPR_CODE
1965	moveup_expr_inside_insn_group (expr_t expr, insn_t through_insn)
1966	{
1967	vinsn_t vi = EXPR_VINSN (expr);
1968	ds_t *has_dep_p;
1969	ds_t full_ds;
1970
1971	/* Do this only inside insn group. */
1972	gcc_assert (INSN_SCHED_CYCLE (through_insn) > 0);
1973
1974	full_ds = has_dependence_p (expr, through_insn, &has_dep_p);
1975	if (full_ds == 0)
1976	return MOVEUP_EXPR_SAME;
1977
1978	/* Substitution is the possible choice in this case. */
1979	if (has_dep_p[DEPS_IN_RHS])
1980	{
1981	/* Can't substitute UNIQUE VINSNs. */
1982	gcc_assert (!VINSN_UNIQUE_P (vi));
1983
1984	if (can_substitute_through_p (insn: through_insn,
1985	ds: has_dep_p[DEPS_IN_RHS])
1986	&& substitute_reg_in_expr (expr, insn: through_insn, undo: false))
1987	{
1988	EXPR_WAS_SUBSTITUTED (expr) = true;
1989	return MOVEUP_EXPR_CHANGED;
1990	}
1991
1992	/* Don't care about this, as even true dependencies may be allowed
1993	in an insn group. */
1994	return MOVEUP_EXPR_SAME;
1995	}
1996
1997	/* This can catch output dependencies in COND_EXECs. */
1998	if (has_dep_p[DEPS_IN_INSN])
1999	return MOVEUP_EXPR_NULL;
2000
2001	/* This is either an output or an anti dependence, which usually have
2002	a zero latency. Allow this here, if we'd be wrong, tick_check_p
2003	will fix this. */
2004	gcc_assert (has_dep_p[DEPS_IN_LHS]);
2005	return MOVEUP_EXPR_AS_RHS;
2006	}
2007
2008	/* True when a trapping EXPR cannot be moved through THROUGH_INSN. */
2009	#define CANT_MOVE_TRAPPING(expr, through_insn) \
2010	(VINSN_MAY_TRAP_P (EXPR_VINSN (expr)) \
2011	&& !sel_insn_has_single_succ_p ((through_insn), SUCCS_ALL) \
2012	&& !sel_insn_is_speculation_check (through_insn))
2013
2014	/* True when a conflict on a target register was found during moveup_expr. */
2015	static bool was_target_conflict = false;
2016
2017	/* Return true when moving a debug INSN across THROUGH_INSN will
2018	create a bookkeeping block. We don't want to create such blocks,
2019	for they would cause codegen differences between compilations with
2020	and without debug info. */
2021
2022	static bool
2023	moving_insn_creates_bookkeeping_block_p (insn_t insn,
2024	insn_t through_insn)
2025	{
2026	basic_block bbi, bbt;
2027	edge e1, e2;
2028	edge_iterator ei1, ei2;
2029
2030	if (!bookkeeping_can_be_created_if_moved_through_p (through_insn))
2031	{
2032	if (sched_verbose >= 9)
2033	sel_print (fmt: "no bookkeeping required: ");
2034	return false;
2035	}
2036
2037	bbi = BLOCK_FOR_INSN (insn);
2038
2039	if (EDGE_COUNT (bbi->preds) == 1)
2040	{
2041	if (sched_verbose >= 9)
2042	sel_print (fmt: "only one pred edge: ");
2043	return true;
2044	}
2045
2046	bbt = BLOCK_FOR_INSN (insn: through_insn);
2047
2048	FOR_EACH_EDGE (e1, ei1, bbt->succs)
2049	{
2050	FOR_EACH_EDGE (e2, ei2, bbi->preds)
2051	{
2052	if (find_block_for_bookkeeping (e1, e2, lax: true))
2053	{
2054	if (sched_verbose >= 9)
2055	sel_print (fmt: "found existing block: ");
2056	return false;
2057	}
2058	}
2059	}
2060
2061	if (sched_verbose >= 9)
2062	sel_print (fmt: "would create bookkeeping block: ");
2063
2064	return true;
2065	}
2066
2067	/* Return true when the conflict with newly created implicit clobbers
2068	between EXPR and THROUGH_INSN is found because of renaming. */
2069	static bool
2070	implicit_clobber_conflict_p (insn_t through_insn, expr_t expr)
2071	{
2072	HARD_REG_SET temp;
2073	rtx_insn *insn;
2074	rtx reg, rhs, pat;
2075	hard_reg_set_iterator hrsi;
2076	unsigned regno;
2077	bool valid;
2078
2079	/* Make a new pseudo register. */
2080	reg = gen_reg_rtx (GET_MODE (EXPR_LHS (expr)));
2081	max_regno = max_reg_num ();
2082	maybe_extend_reg_info_p ();
2083
2084	/* Validate a change and bail out early. */
2085	insn = EXPR_INSN_RTX (expr);
2086	validate_change (insn, &SET_DEST (PATTERN (insn)), reg, true);
2087	valid = verify_changes (0);
2088	cancel_changes (0);
2089	if (!valid)
2090	{
2091	if (sched_verbose >= 6)
2092	sel_print (fmt: "implicit clobbers failed validation, ");
2093	return true;
2094	}
2095
2096	/* Make a new insn with it. */
2097	rhs = copy_rtx (VINSN_RHS (EXPR_VINSN (expr)));
2098	pat = gen_rtx_SET (reg, rhs);
2099	start_sequence ();
2100	insn = emit_insn (pat);
2101	end_sequence ();
2102
2103	/* Calculate implicit clobbers. */
2104	extract_insn (insn);
2105	preprocess_constraints (insn);
2106	alternative_mask prefrred = get_preferred_alternatives (insn);
2107	ira_implicitly_set_insn_hard_regs (&temp, prefrred);
2108	temp &= ~ira_no_alloc_regs;
2109
2110	/* If any implicit clobber registers intersect with regular ones in
2111	through_insn, we have a dependency and thus bail out. */
2112	EXECUTE_IF_SET_IN_HARD_REG_SET (temp, 0, regno, hrsi)
2113	{
2114	vinsn_t vi = INSN_VINSN (through_insn);
2115	if (bitmap_bit_p (VINSN_REG_SETS (vi), regno)
2116	|| bitmap_bit_p (VINSN_REG_CLOBBERS (vi), regno)
2117	|| bitmap_bit_p (VINSN_REG_USES (vi), regno))
2118	return true;
2119	}
2120
2121	return false;
2122	}
2123
2124	/* Modifies EXPR so it can be moved through the THROUGH_INSN,
2125	performing necessary transformations. Record the type of transformation
2126	made in PTRANS_TYPE, when it is not NULL. When INSIDE_INSN_GROUP,
2127	permit all dependencies except true ones, and try to remove those
2128	too via forward substitution. All cases when a non-eliminable
2129	non-zero cost dependency exists inside an insn group will be fixed
2130	in tick_check_p instead. */
2131	static enum MOVEUP_EXPR_CODE
2132	moveup_expr (expr_t expr, insn_t through_insn, bool inside_insn_group,
2133	enum local_trans_type *ptrans_type)
2134	{
2135	vinsn_t vi = EXPR_VINSN (expr);
2136	insn_t insn = VINSN_INSN_RTX (vi);
2137	bool was_changed = false;
2138	bool as_rhs = false;
2139	ds_t *has_dep_p;
2140	ds_t full_ds;
2141
2142	/* ??? We use dependencies of non-debug insns on debug insns to
2143	indicate that the debug insns need to be reset if the non-debug
2144	insn is pulled ahead of it. It's hard to figure out how to
2145	introduce such a notion in sel-sched, but it already fails to
2146	support debug insns in other ways, so we just go ahead and
2147	let the deug insns go corrupt for now. */
2148	if (DEBUG_INSN_P (through_insn) && !DEBUG_INSN_P (insn))
2149	return MOVEUP_EXPR_SAME;
2150
2151	/* When inside_insn_group, delegate to the helper. */
2152	if (inside_insn_group)
2153	return moveup_expr_inside_insn_group (expr, through_insn);
2154
2155	/* Deal with unique insns and control dependencies. */
2156	if (VINSN_UNIQUE_P (vi))
2157	{
2158	/* We can move jumps without side-effects or jumps that are
2159	mutually exclusive with instruction THROUGH_INSN (all in cases
2160	dependencies allow to do so and jump is not speculative). */
2161	if (control_flow_insn_p (insn))
2162	{
2163	basic_block fallthru_bb;
2164
2165	/* Do not move checks and do not move jumps through other
2166	jumps. */
2167	if (control_flow_insn_p (through_insn)
2168	|| sel_insn_is_speculation_check (insn))
2169	return MOVEUP_EXPR_NULL;
2170
2171	/* Don't move jumps through CFG joins. */
2172	if (bookkeeping_can_be_created_if_moved_through_p (through_insn))
2173	return MOVEUP_EXPR_NULL;
2174
2175	/* The jump should have a clear fallthru block, and
2176	this block should be in the current region. */
2177	if ((fallthru_bb = fallthru_bb_of_jump (insn)) == NULL
2178	|| ! in_current_region_p (fallthru_bb))
2179	return MOVEUP_EXPR_NULL;
2180
2181	/* And it should be mutually exclusive with through_insn. */
2182	if (! sched_insns_conditions_mutex_p (insn, through_insn)
2183	&& ! DEBUG_INSN_P (through_insn))
2184	return MOVEUP_EXPR_NULL;
2185	}
2186
2187	/* Don't move what we can't move. */
2188	if (EXPR_CANT_MOVE (expr)
2189	&& BLOCK_FOR_INSN (insn: through_insn) != BLOCK_FOR_INSN (insn))
2190	return MOVEUP_EXPR_NULL;
2191
2192	/* Don't move SCHED_GROUP instruction through anything.
2193	If we don't force this, then it will be possible to start
2194	scheduling a sched_group before all its dependencies are
2195	resolved.
2196	??? Haifa deals with this issue by delaying the SCHED_GROUP
2197	as late as possible through rank_for_schedule. */
2198	if (SCHED_GROUP_P (insn))
2199	return MOVEUP_EXPR_NULL;
2200	}
2201	else
2202	gcc_assert (!control_flow_insn_p (insn));
2203
2204	/* Don't move debug insns if this would require bookkeeping. */
2205	if (DEBUG_INSN_P (insn)
2206	&& BLOCK_FOR_INSN (insn: through_insn) != BLOCK_FOR_INSN (insn)
2207	&& moving_insn_creates_bookkeeping_block_p (insn, through_insn))
2208	return MOVEUP_EXPR_NULL;
2209
2210	/* Deal with data dependencies. */
2211	was_target_conflict = false;
2212	full_ds = has_dependence_p (expr, through_insn, &has_dep_p);
2213	if (full_ds == 0)
2214	{
2215	if (!CANT_MOVE_TRAPPING (expr, through_insn))
2216	return MOVEUP_EXPR_SAME;
2217	}
2218	else
2219	{
2220	/* We can move UNIQUE insn up only as a whole and unchanged,
2221	so it shouldn't have any dependencies. */
2222	if (VINSN_UNIQUE_P (vi))
2223	return MOVEUP_EXPR_NULL;
2224	}
2225
2226	if (full_ds != 0 && can_speculate_dep_p (ds: full_ds))
2227	{
2228	int res;
2229
2230	res = speculate_expr (expr, full_ds);
2231	if (res >= 0)
2232	{
2233	/* Speculation was successful. */
2234	full_ds = 0;
2235	was_changed = (res > 0);
2236	if (res == 2)
2237	was_target_conflict = true;
2238	if (ptrans_type)
2239	*ptrans_type = TRANS_SPECULATION;
2240	sel_clear_has_dependence ();
2241	}
2242	}
2243
2244	if (has_dep_p[DEPS_IN_INSN])
2245	/* We have some dependency that cannot be discarded. */
2246	return MOVEUP_EXPR_NULL;
2247
2248	if (has_dep_p[DEPS_IN_LHS])
2249	{
2250	/* Only separable insns can be moved up with the new register.
2251	Anyways, we should mark that the original register is
2252	unavailable. */
2253	if (!enable_schedule_as_rhs_p || !EXPR_SEPARABLE_P (expr))
2254	return MOVEUP_EXPR_NULL;
2255
2256	/* When renaming a hard register to a pseudo before reload, extra
2257	dependencies can occur from the implicit clobbers of the insn.
2258	Filter out such cases here. */
2259	if (!reload_completed && REG_P (EXPR_LHS (expr))
2260	&& HARD_REGISTER_P (EXPR_LHS (expr))
2261	&& implicit_clobber_conflict_p (through_insn, expr))
2262	{
2263	if (sched_verbose >= 6)
2264	sel_print (fmt: "implicit clobbers conflict detected, ");
2265	return MOVEUP_EXPR_NULL;
2266	}
2267	EXPR_TARGET_AVAILABLE (expr) = false;
2268	was_target_conflict = true;
2269	as_rhs = true;
2270	}
2271
2272	/* At this point we have either separable insns, that will be lifted
2273	up only as RHSes, or non-separable insns with no dependency in lhs.
2274	If dependency is in RHS, then try to perform substitution and move up
2275	substituted RHS:
2276
2277	Ex. 1: Ex.2
2278	y = x; y = x;
2279	z = y*2; y = y*2;
2280
2281	In Ex.1 y*2 can be substituted for x*2 and the whole operation can be
2282	moved above y=x assignment as z=x*2.
2283
2284	In Ex.2 y*2 also can be substituted for x*2, but only the right hand
2285	side can be moved because of the output dependency. The operation was
2286	cropped to its rhs above. */
2287	if (has_dep_p[DEPS_IN_RHS])
2288	{
2289	ds_t *rhs_dsp = &has_dep_p[DEPS_IN_RHS];
2290
2291	/* Can't substitute UNIQUE VINSNs. */
2292	gcc_assert (!VINSN_UNIQUE_P (vi));
2293
2294	if (can_speculate_dep_p (ds: *rhs_dsp))
2295	{
2296	int res;
2297
2298	res = speculate_expr (expr, *rhs_dsp);
2299	if (res >= 0)
2300	{
2301	/* Speculation was successful. */
2302	*rhs_dsp = 0;
2303	was_changed = (res > 0);
2304	if (res == 2)
2305	was_target_conflict = true;
2306	if (ptrans_type)
2307	*ptrans_type = TRANS_SPECULATION;
2308	}
2309	else
2310	return MOVEUP_EXPR_NULL;
2311	}
2312	else if (can_substitute_through_p (insn: through_insn,
2313	ds: *rhs_dsp)
2314	&& substitute_reg_in_expr (expr, insn: through_insn, undo: false))
2315	{
2316	/* ??? We cannot perform substitution AND speculation on the same
2317	insn. */
2318	gcc_assert (!was_changed);
2319	was_changed = true;
2320	if (ptrans_type)
2321	*ptrans_type = TRANS_SUBSTITUTION;
2322	EXPR_WAS_SUBSTITUTED (expr) = true;
2323	}
2324	else
2325	return MOVEUP_EXPR_NULL;
2326	}
2327
2328	/* Don't move trapping insns through jumps.
2329	This check should be at the end to give a chance to control speculation
2330	to perform its duties. */
2331	if (CANT_MOVE_TRAPPING (expr, through_insn))
2332	return MOVEUP_EXPR_NULL;
2333
2334	return (was_changed
2335	? MOVEUP_EXPR_CHANGED
2336	: (as_rhs
2337	? MOVEUP_EXPR_AS_RHS
2338	: MOVEUP_EXPR_SAME));
2339	}
2340
2341	/* Try to look at bitmap caches for EXPR and INSN pair, return true
2342	if successful. When INSIDE_INSN_GROUP, also try ignore dependencies
2343	that can exist within a parallel group. Write to RES the resulting
2344	code for moveup_expr. */
2345	static bool
2346	try_bitmap_cache (expr_t expr, insn_t insn,
2347	bool inside_insn_group,
2348	enum MOVEUP_EXPR_CODE *res)
2349	{
2350	int expr_uid = INSN_UID (EXPR_INSN_RTX (expr));
2351
2352	/* First check whether we've analyzed this situation already. */
2353	if (bitmap_bit_p (INSN_ANALYZED_DEPS (insn), expr_uid))
2354	{
2355	if (bitmap_bit_p (INSN_FOUND_DEPS (insn), expr_uid))
2356	{
2357	if (sched_verbose >= 6)
2358	sel_print (fmt: "removed (cached)\n");
2359	*res = MOVEUP_EXPR_NULL;
2360	return true;
2361	}
2362	else
2363	{
2364	if (sched_verbose >= 6)
2365	sel_print (fmt: "unchanged (cached)\n");
2366	*res = MOVEUP_EXPR_SAME;
2367	return true;
2368	}
2369	}
2370	else if (bitmap_bit_p (INSN_FOUND_DEPS (insn), expr_uid))
2371	{
2372	if (inside_insn_group)
2373	{
2374	if (sched_verbose >= 6)
2375	sel_print (fmt: "unchanged (as RHS, cached, inside insn group)\n");
2376	*res = MOVEUP_EXPR_SAME;
2377	return true;
2378
2379	}
2380	else
2381	EXPR_TARGET_AVAILABLE (expr) = false;
2382
2383	/* This is the only case when propagation result can change over time,
2384	as we can dynamically switch off scheduling as RHS. In this case,
2385	just check the flag to reach the correct decision. */
2386	if (enable_schedule_as_rhs_p)
2387	{
2388	if (sched_verbose >= 6)
2389	sel_print (fmt: "unchanged (as RHS, cached)\n");
2390	*res = MOVEUP_EXPR_AS_RHS;
2391	return true;
2392	}
2393	else
2394	{
2395	if (sched_verbose >= 6)
2396	sel_print (fmt: "removed (cached as RHS, but renaming"
2397	" is now disabled)\n");
2398	*res = MOVEUP_EXPR_NULL;
2399	return true;
2400	}
2401	}
2402
2403	return false;
2404	}
2405
2406	/* Try to look at bitmap caches for EXPR and INSN pair, return true
2407	if successful. Write to RES the resulting code for moveup_expr. */
2408	static bool
2409	try_transformation_cache (expr_t expr, insn_t insn,
2410	enum MOVEUP_EXPR_CODE *res)
2411	{
2412	struct transformed_insns *pti
2413	= (struct transformed_insns *)
2414	htab_find_with_hash (INSN_TRANSFORMED_INSNS (insn),
2415	&EXPR_VINSN (expr),
2416	VINSN_HASH_RTX (EXPR_VINSN (expr)));
2417	if (pti)
2418	{
2419	/* This EXPR was already moved through this insn and was
2420	changed as a result. Fetch the proper data from
2421	the hashtable. */
2422	insert_in_history_vect (&EXPR_HISTORY_OF_CHANGES (expr),
2423	INSN_UID (insn), pti->type,
2424	pti->vinsn_old, pti->vinsn_new,
2425	EXPR_SPEC_DONE_DS (expr));
2426
2427	if (INSN_IN_STREAM_P (VINSN_INSN_RTX (pti->vinsn_new)))
2428	pti->vinsn_new = vinsn_copy (pti->vinsn_new, true);
2429	change_vinsn_in_expr (expr, pti->vinsn_new);
2430	if (pti->was_target_conflict)
2431	EXPR_TARGET_AVAILABLE (expr) = false;
2432	if (pti->type == TRANS_SPECULATION)
2433	{
2434	EXPR_SPEC_DONE_DS (expr) = pti->ds;
2435	EXPR_NEEDS_SPEC_CHECK_P (expr) |= pti->needs_check;
2436	}
2437
2438	if (sched_verbose >= 6)
2439	{
2440	sel_print (fmt: "changed (cached): ");
2441	dump_expr (expr);
2442	sel_print (fmt: "\n");
2443	}
2444
2445	*res = MOVEUP_EXPR_CHANGED;
2446	return true;
2447	}
2448
2449	return false;
2450	}
2451
2452	/* Update bitmap caches on INSN with result RES of propagating EXPR. */
2453	static void
2454	update_bitmap_cache (expr_t expr, insn_t insn, bool inside_insn_group,
2455	enum MOVEUP_EXPR_CODE res)
2456	{
2457	int expr_uid = INSN_UID (EXPR_INSN_RTX (expr));
2458
2459	/* Do not cache result of propagating jumps through an insn group,
2460	as it is always true, which is not useful outside the group. */
2461	if (inside_insn_group)
2462	return;
2463
2464	if (res == MOVEUP_EXPR_NULL)
2465	{
2466	bitmap_set_bit (INSN_ANALYZED_DEPS (insn), expr_uid);
2467	bitmap_set_bit (INSN_FOUND_DEPS (insn), expr_uid);
2468	}
2469	else if (res == MOVEUP_EXPR_SAME)
2470	{
2471	bitmap_set_bit (INSN_ANALYZED_DEPS (insn), expr_uid);
2472	bitmap_clear_bit (INSN_FOUND_DEPS (insn), expr_uid);
2473	}
2474	else if (res == MOVEUP_EXPR_AS_RHS)
2475	{
2476	bitmap_clear_bit (INSN_ANALYZED_DEPS (insn), expr_uid);
2477	bitmap_set_bit (INSN_FOUND_DEPS (insn), expr_uid);
2478	}
2479	else
2480	gcc_unreachable ();
2481	}
2482
2483	/* Update hashtable on INSN with changed EXPR, old EXPR_OLD_VINSN
2484	and transformation type TRANS_TYPE. */
2485	static void
2486	update_transformation_cache (expr_t expr, insn_t insn,
2487	bool inside_insn_group,
2488	enum local_trans_type trans_type,
2489	vinsn_t expr_old_vinsn)
2490	{
2491	struct transformed_insns *pti;
2492
2493	if (inside_insn_group)
2494	return;
2495
2496	pti = XNEW (struct transformed_insns);
2497	pti->vinsn_old = expr_old_vinsn;
2498	pti->vinsn_new = EXPR_VINSN (expr);
2499	pti->type = trans_type;
2500	pti->was_target_conflict = was_target_conflict;
2501	pti->ds = EXPR_SPEC_DONE_DS (expr);
2502	pti->needs_check = EXPR_NEEDS_SPEC_CHECK_P (expr);
2503	vinsn_attach (pti->vinsn_old);
2504	vinsn_attach (pti->vinsn_new);
2505	*((struct transformed_insns **)
2506	htab_find_slot_with_hash (INSN_TRANSFORMED_INSNS (insn),
2507	pti, VINSN_HASH_RTX (expr_old_vinsn),
2508	INSERT)) = pti;
2509	}
2510
2511	/* Same as moveup_expr, but first looks up the result of
2512	transformation in caches. */
2513	static enum MOVEUP_EXPR_CODE
2514	moveup_expr_cached (expr_t expr, insn_t insn, bool inside_insn_group)
2515	{
2516	enum MOVEUP_EXPR_CODE res;
2517	bool got_answer = false;
2518
2519	if (sched_verbose >= 6)
2520	{
2521	sel_print (fmt: "Moving ");
2522	dump_expr (expr);
2523	sel_print (fmt: " through %d: ", INSN_UID (insn));
2524	}
2525
2526	if (DEBUG_INSN_P (EXPR_INSN_RTX (expr))
2527	&& BLOCK_FOR_INSN (EXPR_INSN_RTX (expr))
2528	&& (sel_bb_head (BLOCK_FOR_INSN (EXPR_INSN_RTX (expr)))
2529	== EXPR_INSN_RTX (expr)))
2530	/* Don't use cached information for debug insns that are heads of
2531	basic blocks. */;
2532	else if (try_bitmap_cache (expr, insn, inside_insn_group, res: &res))
2533	/* When inside insn group, we do not want remove stores conflicting
2534	with previosly issued loads. */
2535	got_answer = ! inside_insn_group || res != MOVEUP_EXPR_NULL;
2536	else if (try_transformation_cache (expr, insn, res: &res))
2537	got_answer = true;
2538
2539	if (! got_answer)
2540	{
2541	/* Invoke moveup_expr and record the results. */
2542	vinsn_t expr_old_vinsn = EXPR_VINSN (expr);
2543	ds_t expr_old_spec_ds = EXPR_SPEC_DONE_DS (expr);
2544	int expr_uid = INSN_UID (VINSN_INSN_RTX (expr_old_vinsn));
2545	bool unique_p = VINSN_UNIQUE_P (expr_old_vinsn);
2546	enum local_trans_type trans_type = TRANS_SUBSTITUTION;
2547
2548	/* ??? Invent something better than this. We can't allow old_vinsn
2549	to go, we need it for the history vector. */
2550	vinsn_attach (expr_old_vinsn);
2551
2552	res = moveup_expr (expr, through_insn: insn, inside_insn_group,
2553	ptrans_type: &trans_type);
2554	switch (res)
2555	{
2556	case MOVEUP_EXPR_NULL:
2557	update_bitmap_cache (expr, insn, inside_insn_group, res);
2558	if (sched_verbose >= 6)
2559	sel_print (fmt: "removed\n");
2560	break;
2561
2562	case MOVEUP_EXPR_SAME:
2563	update_bitmap_cache (expr, insn, inside_insn_group, res);
2564	if (sched_verbose >= 6)
2565	sel_print (fmt: "unchanged\n");
2566	break;
2567
2568	case MOVEUP_EXPR_AS_RHS:
2569	gcc_assert (!unique_p || inside_insn_group);
2570	update_bitmap_cache (expr, insn, inside_insn_group, res);
2571	if (sched_verbose >= 6)
2572	sel_print (fmt: "unchanged (as RHS)\n");
2573	break;
2574
2575	case MOVEUP_EXPR_CHANGED:
2576	gcc_assert (INSN_UID (EXPR_INSN_RTX (expr)) != expr_uid
2577	|| EXPR_SPEC_DONE_DS (expr) != expr_old_spec_ds);
2578	insert_in_history_vect (&EXPR_HISTORY_OF_CHANGES (expr),
2579	INSN_UID (insn), trans_type,
2580	expr_old_vinsn, EXPR_VINSN (expr),
2581	expr_old_spec_ds);
2582	update_transformation_cache (expr, insn, inside_insn_group,
2583	trans_type, expr_old_vinsn);
2584	if (sched_verbose >= 6)
2585	{
2586	sel_print (fmt: "changed: ");
2587	dump_expr (expr);
2588	sel_print (fmt: "\n");
2589	}
2590	break;
2591	default:
2592	gcc_unreachable ();
2593	}
2594
2595	vinsn_detach (expr_old_vinsn);
2596	}
2597
2598	return res;
2599	}
2600
2601	/* Moves an av set AVP up through INSN, performing necessary
2602	transformations. */
2603	static void
2604	moveup_set_expr (av_set_t *avp, insn_t insn, bool inside_insn_group)
2605	{
2606	av_set_iterator i;
2607	expr_t expr;
2608
2609	FOR_EACH_EXPR_1 (expr, i, avp)
2610	{
2611
2612	switch (moveup_expr_cached (expr, insn, inside_insn_group))
2613	{
2614	case MOVEUP_EXPR_SAME:
2615	case MOVEUP_EXPR_AS_RHS:
2616	break;
2617
2618	case MOVEUP_EXPR_NULL:
2619	av_set_iter_remove (&i);
2620	break;
2621
2622	case MOVEUP_EXPR_CHANGED:
2623	expr = merge_with_other_exprs (avp, &i, expr);
2624	break;
2625
2626	default:
2627	gcc_unreachable ();
2628	}
2629	}
2630	}
2631
2632	/* Moves AVP set along PATH. */
2633	static void
2634	moveup_set_inside_insn_group (av_set_t *avp, ilist_t path)
2635	{
2636	int last_cycle;
2637
2638	if (sched_verbose >= 6)
2639	sel_print (fmt: "Moving expressions up in the insn group...\n");
2640	if (! path)
2641	return;
2642	last_cycle = INSN_SCHED_CYCLE (ILIST_INSN (path));
2643	while (path
2644	&& INSN_SCHED_CYCLE (ILIST_INSN (path)) == last_cycle)
2645	{
2646	moveup_set_expr (avp, ILIST_INSN (path), inside_insn_group: true);
2647	path = ILIST_NEXT (path);
2648	}
2649	}
2650
2651	/* Returns true if after moving EXPR along PATH it equals to EXPR_VLIW. */
2652	static bool
2653	equal_after_moveup_path_p (expr_t expr, ilist_t path, expr_t expr_vliw)
2654	{
2655	expr_def _tmp, *tmp = &_tmp;
2656	int last_cycle;
2657	bool res = true;
2658
2659	copy_expr_onside (tmp, expr);
2660	last_cycle = path ? INSN_SCHED_CYCLE (ILIST_INSN (path)) : 0;
2661	while (path
2662	&& res
2663	&& INSN_SCHED_CYCLE (ILIST_INSN (path)) == last_cycle)
2664	{
2665	res = (moveup_expr_cached (expr: tmp, ILIST_INSN (path), inside_insn_group: true)
2666	!= MOVEUP_EXPR_NULL);
2667	path = ILIST_NEXT (path);
2668	}
2669
2670	if (res)
2671	{
2672	vinsn_t tmp_vinsn = EXPR_VINSN (tmp);
2673	vinsn_t expr_vliw_vinsn = EXPR_VINSN (expr_vliw);
2674
2675	if (tmp_vinsn != expr_vliw_vinsn)
2676	res = vinsn_equal_p (tmp_vinsn, expr_vliw_vinsn);
2677	}
2678
2679	clear_expr (tmp);
2680	return res;
2681	}
2682
2683
2684	/* Functions that compute av and lv sets. */
2685
2686	/* Returns true if INSN is not a downward continuation of the given path P in
2687	the current stage. */
2688	static bool
2689	is_ineligible_successor (insn_t insn, ilist_t p)
2690	{
2691	insn_t prev_insn;
2692
2693	/* Check if insn is not deleted. */
2694	if (PREV_INSN (insn) && NEXT_INSN (insn: PREV_INSN (insn)) != insn)
2695	gcc_unreachable ();
2696	else if (NEXT_INSN (insn) && PREV_INSN (insn: NEXT_INSN (insn)) != insn)
2697	gcc_unreachable ();
2698
2699	/* If it's the first insn visited, then the successor is ok. */
2700	if (!p)
2701	return false;
2702
2703	prev_insn = ILIST_INSN (p);
2704
2705	if (/* a backward edge. */
2706	INSN_SEQNO (insn) < INSN_SEQNO (prev_insn)
2707	/* is already visited. */
2708	|| (INSN_SEQNO (insn) == INSN_SEQNO (prev_insn)
2709	&& (ilist_is_in_p (l: p, insn)
2710	/* We can reach another fence here and still seqno of insn
2711	would be equal to seqno of prev_insn. This is possible
2712	when prev_insn is a previously created bookkeeping copy.
2713	In that case it'd get a seqno of insn. Thus, check here
2714	whether insn is in current fence too. */
2715	|| IN_CURRENT_FENCE_P (insn)))
2716	/* Was already scheduled on this round. */
2717	|| (INSN_SEQNO (insn) > INSN_SEQNO (prev_insn)
2718	&& IN_CURRENT_FENCE_P (insn))
2719	/* An insn from another fence could also be
2720	scheduled earlier even if this insn is not in
2721	a fence list right now. Check INSN_SCHED_CYCLE instead. */
2722	|| (!pipelining_p
2723	&& INSN_SCHED_TIMES (insn) > 0))
2724	return true;
2725	else
2726	return false;
2727	}
2728
2729	/* Computes the av_set below the last bb insn INSN, doing all the 'dirty work'
2730	of handling multiple successors and properly merging its av_sets. P is
2731	the current path traversed. WS is the size of lookahead window.
2732	Return the av set computed. */
2733	static av_set_t
2734	compute_av_set_at_bb_end (insn_t insn, ilist_t p, int ws)
2735	{
2736	struct succs_info *sinfo;
2737	av_set_t expr_in_all_succ_branches = NULL;
2738	int is;
2739	insn_t succ, zero_succ = NULL;
2740	av_set_t av1 = NULL;
2741
2742	gcc_assert (sel_bb_end_p (insn));
2743
2744	/* Find different kind of successors needed for correct computing of
2745	SPEC and TARGET_AVAILABLE attributes. */
2746	sinfo = compute_succs_info (insn, SUCCS_NORMAL);
2747
2748	/* Debug output. */
2749	if (sched_verbose >= 6)
2750	{
2751	sel_print (fmt: "successors of bb end (%d): ", INSN_UID (insn));
2752	dump_insn_vector (sinfo->succs_ok);
2753	sel_print (fmt: "\n");
2754	if (sinfo->succs_ok_n != sinfo->all_succs_n)
2755	sel_print (fmt: "real successors num: %d\n", sinfo->all_succs_n);
2756	}
2757
2758	/* Add insn to the tail of current path. */
2759	ilist_add (lp: &p, insn);
2760
2761	FOR_EACH_VEC_ELT (sinfo->succs_ok, is, succ)
2762	{
2763	av_set_t succ_set;
2764
2765	/* We will edit SUCC_SET and EXPR_SPEC field of its elements. */
2766	succ_set = compute_av_set_inside_bb (succ, p, ws, true);
2767
2768	av_set_split_usefulness (succ_set,
2769	sinfo->probs_ok[is],
2770	sinfo->all_prob);
2771
2772	if (sinfo->all_succs_n > 1)
2773	{
2774	/* Find EXPR'es that came from *all* successors and save them
2775	into expr_in_all_succ_branches. This set will be used later
2776	for calculating speculation attributes of EXPR'es. */
2777	if (is == 0)
2778	{
2779	expr_in_all_succ_branches = av_set_copy (succ_set);
2780
2781	/* Remember the first successor for later. */
2782	zero_succ = succ;
2783	}
2784	else
2785	{
2786	av_set_iterator i;
2787	expr_t expr;
2788
2789	FOR_EACH_EXPR_1 (expr, i, &expr_in_all_succ_branches)
2790	if (!av_set_is_in_p (succ_set, EXPR_VINSN (expr)))
2791	av_set_iter_remove (&i);
2792	}
2793	}
2794
2795	/* Union the av_sets. Check liveness restrictions on target registers
2796	in special case of two successors. */
2797	if (sinfo->succs_ok_n == 2 && is == 1)
2798	{
2799	basic_block bb0 = BLOCK_FOR_INSN (insn: zero_succ);
2800	basic_block bb1 = BLOCK_FOR_INSN (insn: succ);
2801
2802	gcc_assert (BB_LV_SET_VALID_P (bb0) && BB_LV_SET_VALID_P (bb1));
2803	av_set_union_and_live (&av1, &succ_set,
2804	BB_LV_SET (bb0),
2805	BB_LV_SET (bb1),
2806	insn);
2807	}
2808	else
2809	av_set_union_and_clear (&av1, &succ_set, insn);
2810	}
2811
2812	/* Check liveness restrictions via hard way when there are more than
2813	two successors. */
2814	if (sinfo->succs_ok_n > 2)
2815	FOR_EACH_VEC_ELT (sinfo->succs_ok, is, succ)
2816	{
2817	basic_block succ_bb = BLOCK_FOR_INSN (insn: succ);
2818	av_set_t av_succ = (is_ineligible_successor (insn: succ, p)
2819	? NULL
2820	: BB_AV_SET (succ_bb));
2821
2822	gcc_assert (BB_LV_SET_VALID_P (succ_bb));
2823	mark_unavailable_targets (av1, av_succ, BB_LV_SET (succ_bb));
2824	}
2825
2826	/* Finally, check liveness restrictions on paths leaving the region. */
2827	if (sinfo->all_succs_n > sinfo->succs_ok_n)
2828	FOR_EACH_VEC_ELT (sinfo->succs_other, is, succ)
2829	mark_unavailable_targets
2830	(av1, NULL, BB_LV_SET (BLOCK_FOR_INSN (succ)));
2831
2832	if (sinfo->all_succs_n > 1)
2833	{
2834	av_set_iterator i;
2835	expr_t expr;
2836
2837	/* Increase the spec attribute of all EXPR'es that didn't come
2838	from all successors. */
2839	FOR_EACH_EXPR (expr, i, av1)
2840	if (!av_set_is_in_p (expr_in_all_succ_branches, EXPR_VINSN (expr)))
2841	EXPR_SPEC (expr)++;
2842
2843	av_set_clear (&expr_in_all_succ_branches);
2844
2845	/* Do not move conditional branches through other
2846	conditional branches. So, remove all conditional
2847	branches from av_set if current operator is a conditional
2848	branch. */
2849	av_set_substract_cond_branches (&av1);
2850	}
2851
2852	ilist_remove (&p);
2853	free_succs_info (sinfo);
2854
2855	if (sched_verbose >= 6)
2856	{
2857	sel_print (fmt: "av_succs (%d): ", INSN_UID (insn));
2858	dump_av_set (av1);
2859	sel_print (fmt: "\n");
2860	}
2861
2862	return av1;
2863	}
2864
2865	/* This function computes av_set for the FIRST_INSN by dragging valid
2866	av_set through all basic block insns either from the end of basic block
2867	(computed using compute_av_set_at_bb_end) or from the insn on which
2868	MAX_WS was exceeded. It uses compute_av_set_at_bb_end to compute av_set
2869	below the basic block and handling conditional branches.
2870	FIRST_INSN - the basic block head, P - path consisting of the insns
2871	traversed on the way to the FIRST_INSN (the path is sparse, only bb heads
2872	and bb ends are added to the path), WS - current window size,
2873	NEED_COPY_P - true if we'll make a copy of av_set before returning it. */
2874	static av_set_t
2875	compute_av_set_inside_bb (insn_t first_insn, ilist_t p, int ws,
2876	bool need_copy_p)
2877	{
2878	insn_t cur_insn;
2879	int end_ws = ws;
2880	insn_t bb_end = sel_bb_end (BLOCK_FOR_INSN (insn: first_insn));
2881	insn_t after_bb_end = NEXT_INSN (insn: bb_end);
2882	insn_t last_insn;
2883	av_set_t av = NULL;
2884	basic_block cur_bb = BLOCK_FOR_INSN (insn: first_insn);
2885
2886	/* Return NULL if insn is not on the legitimate downward path. */
2887	if (is_ineligible_successor (insn: first_insn, p))
2888	{
2889	if (sched_verbose >= 6)
2890	sel_print (fmt: "Insn %d is ineligible_successor\n", INSN_UID (insn: first_insn));
2891
2892	return NULL;
2893	}
2894
2895	/* If insn already has valid av(insn) computed, just return it. */
2896	if (AV_SET_VALID_P (first_insn))
2897	{
2898	av_set_t av_set;
2899
2900	if (sel_bb_head_p (first_insn))
2901	av_set = BB_AV_SET (BLOCK_FOR_INSN (first_insn));
2902	else
2903	av_set = NULL;
2904
2905	if (sched_verbose >= 6)
2906	{
2907	sel_print (fmt: "Insn %d has a valid av set: ", INSN_UID (insn: first_insn));
2908	dump_av_set (av_set);
2909	sel_print (fmt: "\n");
2910	}
2911
2912	return need_copy_p ? av_set_copy (av_set) : av_set;
2913	}
2914
2915	ilist_add (lp: &p, insn: first_insn);
2916
2917	/* As the result after this loop have completed, in LAST_INSN we'll
2918	have the insn which has valid av_set to start backward computation
2919	from: it either will be NULL because on it the window size was exceeded
2920	or other valid av_set as returned by compute_av_set for the last insn
2921	of the basic block. */
2922	for (last_insn = first_insn; last_insn != after_bb_end;
2923	last_insn = NEXT_INSN (insn: last_insn))
2924	{
2925	/* We may encounter valid av_set not only on bb_head, but also on
2926	those insns on which previously MAX_WS was exceeded. */
2927	if (AV_SET_VALID_P (last_insn))
2928	{
2929	if (sched_verbose >= 6)
2930	sel_print (fmt: "Insn %d has a valid empty av set\n", INSN_UID (insn: last_insn));
2931	break;
2932	}
2933
2934	/* The special case: the last insn of the BB may be an
2935	ineligible_successor due to its SEQ_NO that was set on
2936	it as a bookkeeping. */
2937	if (last_insn != first_insn
2938	&& is_ineligible_successor (insn: last_insn, p))
2939	{
2940	if (sched_verbose >= 6)
2941	sel_print (fmt: "Insn %d is ineligible_successor\n", INSN_UID (insn: last_insn));
2942	break;
2943	}
2944
2945	if (DEBUG_INSN_P (last_insn))
2946	continue;
2947
2948	if (end_ws > max_ws)
2949	{
2950	/* We can reach max lookahead size at bb_header, so clean av_set
2951	first. */
2952	INSN_WS_LEVEL (last_insn) = global_level;
2953
2954	if (sched_verbose >= 6)
2955	sel_print (fmt: "Insn %d is beyond the software lookahead window size\n",
2956	INSN_UID (insn: last_insn));
2957	break;
2958	}
2959
2960	end_ws++;
2961	}
2962
2963	/* Get the valid av_set into AV above the LAST_INSN to start backward
2964	computation from. It either will be empty av_set or av_set computed from
2965	the successors on the last insn of the current bb. */
2966	if (last_insn != after_bb_end)
2967	{
2968	av = NULL;
2969
2970	/* This is needed only to obtain av_sets that are identical to
2971	those computed by the old compute_av_set version. */
2972	if (last_insn == first_insn && !INSN_NOP_P (last_insn))
2973	av_set_add (&av, INSN_EXPR (last_insn));
2974	}
2975	else
2976	/* END_WS is always already increased by 1 if LAST_INSN == AFTER_BB_END. */
2977	av = compute_av_set_at_bb_end (insn: bb_end, p, ws: end_ws);
2978
2979	/* Compute av_set in AV starting from below the LAST_INSN up to
2980	location above the FIRST_INSN. */
2981	for (cur_insn = PREV_INSN (insn: last_insn); cur_insn != PREV_INSN (insn: first_insn);
2982	cur_insn = PREV_INSN (insn: cur_insn))
2983	if (!INSN_NOP_P (cur_insn))
2984	{
2985	expr_t expr;
2986
2987	moveup_set_expr (avp: &av, insn: cur_insn, inside_insn_group: false);
2988
2989	/* If the expression for CUR_INSN is already in the set,
2990	replace it by the new one. */
2991	expr = av_set_lookup (av, INSN_VINSN (cur_insn));
2992	if (expr != NULL)
2993	{
2994	clear_expr (expr);
2995	copy_expr (expr, INSN_EXPR (cur_insn));
2996	}
2997	else
2998	av_set_add (&av, INSN_EXPR (cur_insn));
2999	}
3000
3001	/* Clear stale bb_av_set. */
3002	if (sel_bb_head_p (first_insn))
3003	{
3004	av_set_clear (&BB_AV_SET (cur_bb));
3005	BB_AV_SET (cur_bb) = need_copy_p ? av_set_copy (av) : av;
3006	BB_AV_LEVEL (cur_bb) = global_level;
3007	}
3008
3009	if (sched_verbose >= 6)
3010	{
3011	sel_print (fmt: "Computed av set for insn %d: ", INSN_UID (insn: first_insn));
3012	dump_av_set (av);
3013	sel_print (fmt: "\n");
3014	}
3015
3016	ilist_remove (&p);
3017	return av;
3018	}
3019
3020	/* Compute av set before INSN.
3021	INSN - the current operation (actual rtx INSN)
3022	P - the current path, which is list of insns visited so far
3023	WS - software lookahead window size.
3024	UNIQUE_P - TRUE, if returned av_set will be changed, hence
3025	if we want to save computed av_set in s_i_d, we should make a copy of it.
3026
3027	In the resulting set we will have only expressions that don't have delay
3028	stalls and nonsubstitutable dependences. */
3029	static av_set_t
3030	compute_av_set (insn_t insn, ilist_t p, int ws, bool unique_p)
3031	{
3032	return compute_av_set_inside_bb (first_insn: insn, p, ws, need_copy_p: unique_p);
3033	}
3034
3035	/* Propagate a liveness set LV through INSN. */
3036	static void
3037	propagate_lv_set (regset lv, insn_t insn)
3038	{
3039	gcc_assert (INSN_P (insn));
3040
3041	if (INSN_NOP_P (insn))
3042	return;
3043
3044	df_simulate_one_insn_backwards (BLOCK_FOR_INSN (insn), insn, lv);
3045	}
3046
3047	/* Return livness set at the end of BB. */
3048	static regset
3049	compute_live_after_bb (basic_block bb)
3050	{
3051	edge e;
3052	edge_iterator ei;
3053	regset lv = get_clear_regset_from_pool ();
3054
3055	gcc_assert (!ignore_first);
3056
3057	FOR_EACH_EDGE (e, ei, bb->succs)
3058	if (sel_bb_empty_p (e->dest))
3059	{
3060	if (! BB_LV_SET_VALID_P (e->dest))
3061	{
3062	gcc_unreachable ();
3063	gcc_assert (BB_LV_SET (e->dest) == NULL);
3064	BB_LV_SET (e->dest) = compute_live_after_bb (bb: e->dest);
3065	BB_LV_SET_VALID_P (e->dest) = true;
3066	}
3067	IOR_REG_SET (lv, BB_LV_SET (e->dest));
3068	}
3069	else
3070	IOR_REG_SET (lv, compute_live (sel_bb_head (e->dest)));
3071
3072	return lv;
3073	}
3074
3075	/* Compute the set of all live registers at the point before INSN and save
3076	it at INSN if INSN is bb header. */
3077	regset
3078	compute_live (insn_t insn)
3079	{
3080	basic_block bb = BLOCK_FOR_INSN (insn);
3081	insn_t final, temp;
3082	regset lv;
3083
3084	/* Return the valid set if we're already on it. */
3085	if (!ignore_first)
3086	{
3087	regset src = NULL;
3088
3089	if (sel_bb_head_p (insn) && BB_LV_SET_VALID_P (bb))
3090	src = BB_LV_SET (bb);
3091	else
3092	{
3093	gcc_assert (in_current_region_p (bb));
3094	if (INSN_LIVE_VALID_P (insn))
3095	src = INSN_LIVE (insn);
3096	}
3097
3098	if (src)
3099	{
3100	lv = get_regset_from_pool ();
3101	COPY_REG_SET (lv, src);
3102
3103	if (sel_bb_head_p (insn) && ! BB_LV_SET_VALID_P (bb))
3104	{
3105	COPY_REG_SET (BB_LV_SET (bb), lv);
3106	BB_LV_SET_VALID_P (bb) = true;
3107	}
3108
3109	return_regset_to_pool (lv);
3110	return lv;
3111	}
3112	}
3113
3114	/* We've skipped the wrong lv_set. Don't skip the right one. */
3115	ignore_first = false;
3116	gcc_assert (in_current_region_p (bb));
3117
3118	/* Find a valid LV set in this block or below, if needed.
3119	Start searching from the next insn: either ignore_first is true, or
3120	INSN doesn't have a correct live set. */
3121	temp = NEXT_INSN (insn);
3122	final = NEXT_INSN (BB_END (bb));
3123	while (temp != final && ! INSN_LIVE_VALID_P (temp))
3124	temp = NEXT_INSN (insn: temp);
3125	if (temp == final)
3126	{
3127	lv = compute_live_after_bb (bb);
3128	temp = PREV_INSN (insn: temp);
3129	}
3130	else
3131	{
3132	lv = get_regset_from_pool ();
3133	COPY_REG_SET (lv, INSN_LIVE (temp));
3134	}
3135
3136	/* Put correct lv sets on the insns which have bad sets. */
3137	final = PREV_INSN (insn);
3138	while (temp != final)
3139	{
3140	propagate_lv_set (lv, insn: temp);
3141	COPY_REG_SET (INSN_LIVE (temp), lv);
3142	INSN_LIVE_VALID_P (temp) = true;
3143	temp = PREV_INSN (insn: temp);
3144	}
3145
3146	/* Also put it in a BB. */
3147	if (sel_bb_head_p (insn))
3148	{
3149	basic_block bb = BLOCK_FOR_INSN (insn);
3150
3151	COPY_REG_SET (BB_LV_SET (bb), lv);
3152	BB_LV_SET_VALID_P (bb) = true;
3153	}
3154
3155	/* We return LV to the pool, but will not clear it there. Thus we can
3156	legimatelly use LV till the next use of regset_pool_get (). */
3157	return_regset_to_pool (lv);
3158	return lv;
3159	}
3160
3161	/* Update liveness sets for INSN. */
3162	static inline void
3163	update_liveness_on_insn (rtx_insn *insn)
3164	{
3165	ignore_first = true;
3166	compute_live (insn);
3167	}
3168
3169	/* Compute liveness below INSN and write it into REGS. */
3170	static inline void
3171	compute_live_below_insn (rtx_insn *insn, regset regs)
3172	{
3173	rtx_insn *succ;
3174	succ_iterator si;
3175
3176	FOR_EACH_SUCC_1 (succ, si, insn, SUCCS_ALL)
3177	IOR_REG_SET (regs, compute_live (succ));
3178	}
3179
3180	/* Update the data gathered in av and lv sets starting from INSN. */
3181	static void
3182	update_data_sets (rtx_insn *insn)
3183	{
3184	update_liveness_on_insn (insn);
3185	if (sel_bb_head_p (insn))
3186	{
3187	gcc_assert (AV_LEVEL (insn) != 0);
3188	BB_AV_LEVEL (BLOCK_FOR_INSN (insn)) = -1;
3189	compute_av_set (insn, NULL, ws: 0, unique_p: 0);
3190	}
3191	}
3192
3193
3194	/* Helper for move_op () and find_used_regs ().
3195	Return speculation type for which a check should be created on the place
3196	of INSN. EXPR is one of the original ops we are searching for. */
3197	static ds_t
3198	get_spec_check_type_for_insn (insn_t insn, expr_t expr)
3199	{
3200	ds_t to_check_ds;
3201	ds_t already_checked_ds = EXPR_SPEC_DONE_DS (INSN_EXPR (insn));
3202
3203	to_check_ds = EXPR_SPEC_TO_CHECK_DS (expr);
3204
3205	if (targetm.sched.get_insn_checked_ds)
3206	already_checked_ds |= targetm.sched.get_insn_checked_ds (insn);
3207
3208	if (spec_info != NULL
3209	&& (spec_info->flags & SEL_SCHED_SPEC_DONT_CHECK_CONTROL))
3210	already_checked_ds |= BEGIN_CONTROL;
3211
3212	already_checked_ds = ds_get_speculation_types (already_checked_ds);
3213
3214	to_check_ds &= ~already_checked_ds;
3215
3216	return to_check_ds;
3217	}
3218
3219	/* Find the set of registers that are unavailable for storing expres
3220	while moving ORIG_OPS up on the path starting from INSN due to
3221	liveness (USED_REGS) or hardware restrictions (REG_RENAME_P).
3222
3223	All the original operations found during the traversal are saved in the
3224	ORIGINAL_INSNS list.
3225
3226	REG_RENAME_P denotes the set of hardware registers that
3227	cannot be used with renaming due to the register class restrictions,
3228	mode restrictions and other (the register we'll choose should be
3229	compatible class with the original uses, shouldn't be in call_used_regs,
3230	should be HARD_REGNO_RENAME_OK etc).
3231
3232	Returns TRUE if we've found all original insns, FALSE otherwise.
3233
3234	This function utilizes code_motion_path_driver (formerly find_used_regs_1)
3235	to traverse the code motion paths. This helper function finds registers
3236	that are not available for storing expres while moving ORIG_OPS up on the
3237	path starting from INSN. A register considered as used on the moving path,
3238	if one of the following conditions is not satisfied:
3239
3240	(1) a register not set or read on any path from xi to an instance of
3241	the original operation,
3242	(2) not among the live registers of the point immediately following the
3243	first original operation on a given downward path, except for the
3244	original target register of the operation,
3245	(3) not live on the other path of any conditional branch that is passed
3246	by the operation, in case original operations are not present on
3247	both paths of the conditional branch.
3248
3249	All the original operations found during the traversal are saved in the
3250	ORIGINAL_INSNS list.
3251
3252	REG_RENAME_P->CROSSED_CALL_ABIS is true, if there is a call insn on the path
3253	from INSN to original insn. In this case CALL_USED_REG_SET will be added
3254	to unavailable hard regs at the point original operation is found. */
3255
3256	static bool
3257	find_used_regs (insn_t insn, av_set_t orig_ops, regset used_regs,
3258	struct reg_rename *reg_rename_p, def_list_t *original_insns)
3259	{
3260	def_list_iterator i;
3261	def_t def;
3262	int res;
3263	bool needs_spec_check_p = false;
3264	expr_t expr;
3265	av_set_iterator expr_iter;
3266	struct fur_static_params sparams;
3267	struct cmpd_local_params lparams;
3268
3269	/* We haven't visited any blocks yet. */
3270	bitmap_clear (code_motion_visited_blocks);
3271
3272	/* Init parameters for code_motion_path_driver. */
3273	sparams.crossed_call_abis = 0;
3274	sparams.original_insns = original_insns;
3275	sparams.used_regs = used_regs;
3276
3277	/* Set the appropriate hooks and data. */
3278	code_motion_path_driver_info = &fur_hooks;
3279
3280	res = code_motion_path_driver (insn, orig_ops, NULL, &lparams, &sparams);
3281
3282	reg_rename_p->crossed_call_abis |= sparams.crossed_call_abis;
3283
3284	gcc_assert (res == 1);
3285	gcc_assert (original_insns && *original_insns);
3286
3287	/* ??? We calculate whether an expression needs a check when computing
3288	av sets. This information is not as precise as it could be due to
3289	merging this bit in merge_expr. We can do better in find_used_regs,
3290	but we want to avoid multiple traversals of the same code motion
3291	paths. */
3292	FOR_EACH_EXPR (expr, expr_iter, orig_ops)
3293	needs_spec_check_p |= EXPR_NEEDS_SPEC_CHECK_P (expr);
3294
3295	/* Mark hardware regs in REG_RENAME_P that are not suitable
3296	for renaming expr in INSN due to hardware restrictions (register class,
3297	modes compatibility etc). */
3298	FOR_EACH_DEF (def, i, *original_insns)
3299	{
3300	vinsn_t vinsn = INSN_VINSN (def->orig_insn);
3301
3302	if (VINSN_SEPARABLE_P (vinsn))
3303	mark_unavailable_hard_regs (def, reg_rename_p, used_regs);
3304
3305	/* Do not allow clobbering of ld.[sa] address in case some of the
3306	original operations need a check. */
3307	if (needs_spec_check_p)
3308	IOR_REG_SET (used_regs, VINSN_REG_USES (vinsn));
3309	}
3310
3311	return true;
3312	}
3313
3314
3315	/* Functions to choose the best insn from available ones. */
3316
3317	/* Adjusts the priority for EXPR using the backend *_adjust_priority hook. */
3318	static int
3319	sel_target_adjust_priority (expr_t expr)
3320	{
3321	int priority = EXPR_PRIORITY (expr);
3322	int new_priority;
3323
3324	if (targetm.sched.adjust_priority)
3325	new_priority = targetm.sched.adjust_priority (EXPR_INSN_RTX (expr), priority);
3326	else
3327	new_priority = priority;
3328
3329	/* If the priority has changed, adjust EXPR_PRIORITY_ADJ accordingly. */
3330	EXPR_PRIORITY_ADJ (expr) = new_priority - EXPR_PRIORITY (expr);
3331
3332	if (sched_verbose >= 4)
3333	sel_print (fmt: "sel_target_adjust_priority: insn %d, %d+%d = %d.\n",
3334	INSN_UID (EXPR_INSN_RTX (expr)), EXPR_PRIORITY (expr),
3335	EXPR_PRIORITY_ADJ (expr), new_priority);
3336
3337	return new_priority;
3338	}
3339
3340	/* Rank two available exprs for schedule. Never return 0 here. */
3341	static int
3342	sel_rank_for_schedule (const void *x, const void *y)
3343	{
3344	expr_t tmp = *(const expr_t *) y;
3345	expr_t tmp2 = *(const expr_t *) x;
3346	insn_t tmp_insn, tmp2_insn;
3347	vinsn_t tmp_vinsn, tmp2_vinsn;
3348	int val;
3349
3350	tmp_vinsn = EXPR_VINSN (tmp);
3351	tmp2_vinsn = EXPR_VINSN (tmp2);
3352	tmp_insn = EXPR_INSN_RTX (tmp);
3353	tmp2_insn = EXPR_INSN_RTX (tmp2);
3354
3355	/* Schedule debug insns as early as possible. */
3356	if (DEBUG_INSN_P (tmp_insn) && !DEBUG_INSN_P (tmp2_insn))
3357	return -1;
3358	else if (DEBUG_INSN_P (tmp2_insn))
3359	return 1;
3360
3361	/* Prefer SCHED_GROUP_P insns to any others. */
3362	if (SCHED_GROUP_P (tmp_insn) != SCHED_GROUP_P (tmp2_insn))
3363	{
3364	if (VINSN_UNIQUE_P (tmp_vinsn) && VINSN_UNIQUE_P (tmp2_vinsn))
3365	return SCHED_GROUP_P (tmp2_insn) ? 1 : -1;
3366
3367	/* Now uniqueness means SCHED_GROUP_P is set, because schedule groups
3368	cannot be cloned. */
3369	if (VINSN_UNIQUE_P (tmp2_vinsn))
3370	return 1;
3371	return -1;
3372	}
3373
3374	/* Discourage scheduling of speculative checks. */
3375	val = (sel_insn_is_speculation_check (tmp_insn)
3376	- sel_insn_is_speculation_check (tmp2_insn));
3377	if (val)
3378	return val;
3379
3380	/* Prefer not scheduled insn over scheduled one. */
3381	if (EXPR_SCHED_TIMES (tmp) > 0 || EXPR_SCHED_TIMES (tmp2) > 0)
3382	{
3383	val = EXPR_SCHED_TIMES (tmp) - EXPR_SCHED_TIMES (tmp2);
3384	if (val)
3385	return val;
3386	}
3387
3388	/* Prefer jump over non-jump instruction. */
3389	if (control_flow_insn_p (tmp_insn) && !control_flow_insn_p (tmp2_insn))
3390	return -1;
3391	else if (control_flow_insn_p (tmp2_insn) && !control_flow_insn_p (tmp_insn))
3392	return 1;
3393
3394	/* Prefer an expr with non-zero usefulness. */
3395	int u1 = EXPR_USEFULNESS (tmp), u2 = EXPR_USEFULNESS (tmp2);
3396
3397	if (u1 == 0)
3398	{
3399	if (u2 == 0)
3400	u1 = u2 = 1;
3401	else
3402	return 1;
3403	}
3404	else if (u2 == 0)
3405	return -1;
3406
3407	/* Prefer an expr with greater priority. */
3408	val = (u2 * (EXPR_PRIORITY (tmp2) + EXPR_PRIORITY_ADJ (tmp2))
3409	- u1 * (EXPR_PRIORITY (tmp) + EXPR_PRIORITY_ADJ (tmp)));
3410	if (val)
3411	return val;
3412
3413	if (spec_info != NULL && spec_info->mask != 0)
3414	/* This code was taken from haifa-sched.cc: rank_for_schedule (). */
3415	{
3416	ds_t ds1, ds2;
3417	dw_t dw1, dw2;
3418	int dw;
3419
3420	ds1 = EXPR_SPEC_DONE_DS (tmp);
3421	if (ds1)
3422	dw1 = ds_weak (ds1);
3423	else
3424	dw1 = NO_DEP_WEAK;
3425
3426	ds2 = EXPR_SPEC_DONE_DS (tmp2);
3427	if (ds2)
3428	dw2 = ds_weak (ds2);
3429	else
3430	dw2 = NO_DEP_WEAK;
3431
3432	dw = dw2 - dw1;
3433	if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8))
3434	return dw;
3435	}
3436
3437	/* Prefer an old insn to a bookkeeping insn. */
3438	if (INSN_UID (insn: tmp_insn) < first_emitted_uid
3439	&& INSN_UID (insn: tmp2_insn) >= first_emitted_uid)
3440	return -1;
3441	if (INSN_UID (insn: tmp_insn) >= first_emitted_uid
3442	&& INSN_UID (insn: tmp2_insn) < first_emitted_uid)
3443	return 1;
3444
3445	/* Prefer an insn with smaller UID, as a last resort.
3446	We can't safely use INSN_LUID as it is defined only for those insns
3447	that are in the stream. */
3448	return INSN_UID (insn: tmp_insn) - INSN_UID (insn: tmp2_insn);
3449	}
3450
3451	/* Filter out expressions from av set pointed to by AV_PTR
3452	that are pipelined too many times. */
3453	static void
3454	process_pipelined_exprs (av_set_t *av_ptr)
3455	{
3456	expr_t expr;
3457	av_set_iterator si;
3458
3459	/* Don't pipeline already pipelined code as that would increase
3460	number of unnecessary register moves. */
3461	FOR_EACH_EXPR_1 (expr, si, av_ptr)
3462	{
3463	if (EXPR_SCHED_TIMES (expr)
3464	>= param_selsched_max_sched_times)
3465	av_set_iter_remove (&si);
3466	}
3467	}
3468
3469	/* Filter speculative insns from AV_PTR if we don't want them. */
3470	static void
3471	process_spec_exprs (av_set_t *av_ptr)
3472	{
3473	expr_t expr;
3474	av_set_iterator si;
3475
3476	if (spec_info == NULL)
3477	return;
3478
3479	/* Scan *AV_PTR to find out if we want to consider speculative
3480	instructions for scheduling. */
3481	FOR_EACH_EXPR_1 (expr, si, av_ptr)
3482	{
3483	ds_t ds;
3484
3485	ds = EXPR_SPEC_DONE_DS (expr);
3486
3487	/* The probability of a success is too low - don't speculate. */
3488	if ((ds & SPECULATIVE)
3489	&& (ds_weak (ds) < spec_info->data_weakness_cutoff
3490	|| EXPR_USEFULNESS (expr) < spec_info->control_weakness_cutoff
3491	|| (pipelining_p && false
3492	&& (ds & DATA_SPEC)
3493	&& (ds & CONTROL_SPEC))))
3494	{
3495	av_set_iter_remove (&si);
3496	continue;
3497	}
3498	}
3499	}
3500
3501	/* Search for any use-like insns in AV_PTR and decide on scheduling
3502	them. Return one when found, and NULL otherwise.
3503	Note that we check here whether a USE could be scheduled to avoid
3504	an infinite loop later. */
3505	static expr_t
3506	process_use_exprs (av_set_t *av_ptr)
3507	{
3508	expr_t expr;
3509	av_set_iterator si;
3510	bool uses_present_p = false;
3511	bool try_uses_p = true;
3512
3513	FOR_EACH_EXPR_1 (expr, si, av_ptr)
3514	{
3515	/* This will also initialize INSN_CODE for later use. */
3516	if (recog_memoized (EXPR_INSN_RTX (expr)) < 0)
3517	{
3518	/* If we have a USE in *AV_PTR that was not scheduled yet,
3519	do so because it will do good only. */
3520	if (EXPR_SCHED_TIMES (expr) <= 0)
3521	{
3522	if (EXPR_TARGET_AVAILABLE (expr) == 1)
3523	return expr;
3524
3525	av_set_iter_remove (&si);
3526	}
3527	else
3528	{
3529	gcc_assert (pipelining_p);
3530
3531	uses_present_p = true;
3532	}
3533	}
3534	else
3535	try_uses_p = false;
3536	}
3537
3538	if (uses_present_p)
3539	{
3540	/* If we don't want to schedule any USEs right now and we have some
3541	in *AV_PTR, remove them, else just return the first one found. */
3542	if (!try_uses_p)
3543	{
3544	FOR_EACH_EXPR_1 (expr, si, av_ptr)
3545	if (INSN_CODE (EXPR_INSN_RTX (expr)) < 0)
3546	av_set_iter_remove (&si);
3547	}
3548	else
3549	{
3550	FOR_EACH_EXPR_1 (expr, si, av_ptr)
3551	{
3552	gcc_assert (INSN_CODE (EXPR_INSN_RTX (expr)) < 0);
3553
3554	if (EXPR_TARGET_AVAILABLE (expr) == 1)
3555	return expr;
3556
3557	av_set_iter_remove (&si);
3558	}
3559	}
3560	}
3561
3562	return NULL;
3563	}
3564
3565	/* Lookup EXPR in VINSN_VEC and return TRUE if found. Also check patterns from
3566	EXPR's history of changes. */
3567	static bool
3568	vinsn_vec_has_expr_p (vinsn_vec_t vinsn_vec, expr_t expr)
3569	{
3570	vinsn_t vinsn, expr_vinsn;
3571	int n;
3572	unsigned i;
3573
3574	/* Start with checking expr itself and then proceed with all the old forms
3575	of expr taken from its history vector. */
3576	for (i = 0, expr_vinsn = EXPR_VINSN (expr);
3577	expr_vinsn;
3578	expr_vinsn = (i < EXPR_HISTORY_OF_CHANGES (expr).length ()
3579	? EXPR_HISTORY_OF_CHANGES (expr)[i++].old_expr_vinsn
3580	: NULL))
3581	FOR_EACH_VEC_ELT (vinsn_vec, n, vinsn)
3582	if (VINSN_SEPARABLE_P (vinsn))
3583	{
3584	if (vinsn_equal_p (vinsn, expr_vinsn))
3585	return true;
3586	}
3587	else
3588	{
3589	/* For non-separable instructions, the blocking insn can have
3590	another pattern due to substitution, and we can't choose
3591	different register as in the above case. Check all registers
3592	being written instead. */
3593	if (bitmap_intersect_p (VINSN_REG_SETS (vinsn),
3594	VINSN_REG_SETS (expr_vinsn)))
3595	return true;
3596	}
3597
3598	return false;
3599	}
3600
3601	/* Return true if either of expressions from ORIG_OPS can be blocked
3602	by previously created bookkeeping code. STATIC_PARAMS points to static
3603	parameters of move_op. */
3604	static bool
3605	av_set_could_be_blocked_by_bookkeeping_p (av_set_t orig_ops, void *static_params)
3606	{
3607	expr_t expr;
3608	av_set_iterator iter;
3609	moveop_static_params_p sparams;
3610
3611	/* This checks that expressions in ORIG_OPS are not blocked by bookkeeping
3612	created while scheduling on another fence. */
3613	FOR_EACH_EXPR (expr, iter, orig_ops)
3614	if (vinsn_vec_has_expr_p (vinsn_vec: vec_bookkeeping_blocked_vinsns, expr))
3615	return true;
3616
3617	gcc_assert (code_motion_path_driver_info == &move_op_hooks);
3618	sparams = (moveop_static_params_p) static_params;
3619
3620	/* Expressions can be also blocked by bookkeeping created during current
3621	move_op. */
3622	if (bitmap_bit_p (current_copies, INSN_UID (insn: sparams->failed_insn)))
3623	FOR_EACH_EXPR (expr, iter, orig_ops)
3624	if (moveup_expr_cached (expr, insn: sparams->failed_insn, inside_insn_group: false) != MOVEUP_EXPR_NULL)
3625	return true;
3626
3627	/* Expressions in ORIG_OPS may have wrong destination register due to
3628	renaming. Check with the right register instead. */
3629	if (sparams->dest && REG_P (sparams->dest))
3630	{
3631	rtx reg = sparams->dest;
3632	vinsn_t failed_vinsn = INSN_VINSN (sparams->failed_insn);
3633
3634	if (register_unavailable_p (VINSN_REG_SETS (failed_vinsn), reg)
3635	|| register_unavailable_p (VINSN_REG_USES (failed_vinsn), reg)
3636	|| register_unavailable_p (VINSN_REG_CLOBBERS (failed_vinsn), reg))
3637	return true;
3638	}
3639
3640	return false;
3641	}
3642
3643	/* Clear VINSN_VEC and detach vinsns. */
3644	static void
3645	vinsn_vec_clear (vinsn_vec_t *vinsn_vec)
3646	{
3647	unsigned len = vinsn_vec->length ();
3648	if (len > 0)
3649	{
3650	vinsn_t vinsn;
3651	int n;
3652
3653	FOR_EACH_VEC_ELT (*vinsn_vec, n, vinsn)
3654	vinsn_detach (vinsn);
3655	vinsn_vec->block_remove (ix: 0, len);
3656	}
3657	}
3658
3659	/* Add the vinsn of EXPR to the VINSN_VEC. */
3660	static void
3661	vinsn_vec_add (vinsn_vec_t *vinsn_vec, expr_t expr)
3662	{
3663	vinsn_attach (EXPR_VINSN (expr));
3664	vinsn_vec->safe_push (EXPR_VINSN (expr));
3665	}
3666
3667	/* Free the vector representing blocked expressions. */
3668	static void
3669	vinsn_vec_free (vinsn_vec_t &vinsn_vec)
3670	{
3671	vinsn_vec.release ();
3672	}
3673
3674	/* Increase EXPR_PRIORITY_ADJ for INSN by AMOUNT. */
3675
3676	void sel_add_to_insn_priority (rtx insn, int amount)
3677	{
3678	EXPR_PRIORITY_ADJ (INSN_EXPR (insn)) += amount;
3679
3680	if (sched_verbose >= 2)
3681	sel_print (fmt: "sel_add_to_insn_priority: insn %d, by %d (now %d+%d).\n",
3682	INSN_UID (insn), amount, EXPR_PRIORITY (INSN_EXPR (insn)),
3683	EXPR_PRIORITY_ADJ (INSN_EXPR (insn)));
3684	}
3685
3686	/* Turn AV into a vector, filter inappropriate insns and sort it. Return
3687	true if there is something to schedule. BNDS and FENCE are current
3688	boundaries and fence, respectively. If we need to stall for some cycles
3689	before an expr from AV would become available, write this number to
3690	*PNEED_STALL. */
3691	static bool
3692	fill_vec_av_set (av_set_t av, blist_t bnds, fence_t fence,
3693	int *pneed_stall)
3694	{
3695	av_set_iterator si;
3696	expr_t expr;
3697	int sched_next_worked = 0, stalled, n;
3698	static int av_max_prio, est_ticks_till_branch;
3699	int min_need_stall = -1;
3700	deps_t dc = BND_DC (BLIST_BND (bnds));
3701
3702	/* Bail out early when the ready list contained only USEs/CLOBBERs that are
3703	already scheduled. */
3704	if (av == NULL)
3705	return false;
3706
3707	/* Empty vector from the previous stuff. */
3708	if (vec_av_set.length () > 0)
3709	vec_av_set.block_remove (ix: 0, len: vec_av_set.length ());
3710
3711	/* Turn the set into a vector for sorting and call sel_target_adjust_priority
3712	for each insn. */
3713	gcc_assert (vec_av_set.is_empty ());
3714	FOR_EACH_EXPR (expr, si, av)
3715	{
3716	vec_av_set.safe_push (obj: expr);
3717
3718	gcc_assert (EXPR_PRIORITY_ADJ (expr) == 0 || *pneed_stall);
3719
3720	/* Adjust priority using target backend hook. */
3721	sel_target_adjust_priority (expr);
3722	}
3723
3724	/* Sort the vector. */
3725	vec_av_set.qsort (sel_rank_for_schedule);
3726
3727	/* We record maximal priority of insns in av set for current instruction
3728	group. */
3729	if (FENCE_STARTS_CYCLE_P (fence))
3730	av_max_prio = est_ticks_till_branch = INT_MIN;
3731
3732	/* Filter out inappropriate expressions. Loop's direction is reversed to
3733	visit "best" instructions first. We assume that vec::unordered_remove
3734	moves last element in place of one being deleted. */
3735	for (n = vec_av_set.length () - 1, stalled = 0; n >= 0; n--)
3736	{
3737	expr_t expr = vec_av_set[n];
3738	insn_t insn = EXPR_INSN_RTX (expr);
3739	signed char target_available;
3740	bool is_orig_reg_p = true;
3741	int need_cycles, new_prio;
3742	bool fence_insn_p = INSN_UID (insn) == INSN_UID (FENCE_INSN (fence));
3743
3744	/* Don't allow any insns other than from SCHED_GROUP if we have one. */
3745	if (FENCE_SCHED_NEXT (fence) && insn != FENCE_SCHED_NEXT (fence))
3746	{
3747	vec_av_set.unordered_remove (ix: n);
3748	continue;
3749	}
3750
3751	/* Set number of sched_next insns (just in case there
3752	could be several). */
3753	if (FENCE_SCHED_NEXT (fence))
3754	sched_next_worked++;
3755
3756	/* Check all liveness requirements and try renaming.
3757	FIXME: try to minimize calls to this. */
3758	target_available = EXPR_TARGET_AVAILABLE (expr);
3759
3760	/* If insn was already scheduled on the current fence,
3761	set TARGET_AVAILABLE to -1 no matter what expr's attribute says. */
3762	if (vinsn_vec_has_expr_p (vinsn_vec: vec_target_unavailable_vinsns, expr)
3763	&& !fence_insn_p)
3764	target_available = -1;
3765
3766	/* If the availability of the EXPR is invalidated by the insertion of
3767	bookkeeping earlier, make sure that we won't choose this expr for
3768	scheduling if it's not separable, and if it is separable, then
3769	we have to recompute the set of available registers for it. */
3770	if (vinsn_vec_has_expr_p (vinsn_vec: vec_bookkeeping_blocked_vinsns, expr))
3771	{
3772	vec_av_set.unordered_remove (ix: n);
3773	if (sched_verbose >= 4)
3774	sel_print (fmt: "Expr %d is blocked by bookkeeping inserted earlier\n",
3775	INSN_UID (insn));
3776	continue;
3777	}
3778
3779	if (target_available == true)
3780	{
3781	/* Do nothing -- we can use an existing register. */
3782	is_orig_reg_p = EXPR_SEPARABLE_P (expr);
3783	}
3784	else if (/* Non-separable instruction will never
3785	get another register. */
3786	(target_available == false
3787	&& !EXPR_SEPARABLE_P (expr))
3788	/* Don't try to find a register for low-priority expression. */
3789	|| (int) vec_av_set.length () - 1 - n >= max_insns_to_rename
3790	/* ??? FIXME: Don't try to rename data speculation. */
3791	|| (EXPR_SPEC_DONE_DS (expr) & BEGIN_DATA)
3792	|| ! find_best_reg_for_expr (expr, bnds, is_orig_reg_p: &is_orig_reg_p))
3793	{
3794	vec_av_set.unordered_remove (ix: n);
3795	if (sched_verbose >= 4)
3796	sel_print (fmt: "Expr %d has no suitable target register\n",
3797	INSN_UID (insn));
3798
3799	/* A fence insn should not get here. */
3800	gcc_assert (!fence_insn_p);
3801	continue;
3802	}
3803
3804	/* At this point a fence insn should always be available. */
3805	gcc_assert (!fence_insn_p
3806	|| INSN_UID (FENCE_INSN (fence)) == INSN_UID (EXPR_INSN_RTX (expr)));
3807
3808	/* Filter expressions that need to be renamed or speculated when
3809	pipelining, because compensating register copies or speculation
3810	checks are likely to be placed near the beginning of the loop,
3811	causing a stall. */
3812	if (pipelining_p && EXPR_ORIG_SCHED_CYCLE (expr) > 0
3813	&& (!is_orig_reg_p || EXPR_SPEC_DONE_DS (expr) != 0))
3814	{
3815	/* Estimation of number of cycles until loop branch for
3816	renaming/speculation to be successful. */
3817	int need_n_ticks_till_branch = sel_vinsn_cost (EXPR_VINSN (expr));
3818
3819	if ((int) current_loop_nest->ninsns < 9)
3820	{
3821	vec_av_set.unordered_remove (ix: n);
3822	if (sched_verbose >= 4)
3823	sel_print (fmt: "Pipelining expr %d will likely cause stall\n",
3824	INSN_UID (insn));
3825	continue;
3826	}
3827
3828	if ((int) current_loop_nest->ninsns - num_insns_scheduled
3829	< need_n_ticks_till_branch * issue_rate / 2
3830	&& est_ticks_till_branch < need_n_ticks_till_branch)
3831	{
3832	vec_av_set.unordered_remove (ix: n);
3833	if (sched_verbose >= 4)
3834	sel_print (fmt: "Pipelining expr %d will likely cause stall\n",
3835	INSN_UID (insn));
3836	continue;
3837	}
3838	}
3839
3840	/* We want to schedule speculation checks as late as possible. Discard
3841	them from av set if there are instructions with higher priority. */
3842	if (sel_insn_is_speculation_check (insn)
3843	&& EXPR_PRIORITY (expr) < av_max_prio)
3844	{
3845	stalled++;
3846	min_need_stall = min_need_stall < 0 ? 1 : MIN (min_need_stall, 1);
3847	vec_av_set.unordered_remove (ix: n);
3848	if (sched_verbose >= 4)
3849	sel_print (fmt: "Delaying speculation check %d until its first use\n",
3850	INSN_UID (insn));
3851	continue;
3852	}
3853
3854	/* Ignore EXPRs available from pipelining to update AV_MAX_PRIO. */
3855	if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0)
3856	av_max_prio = MAX (av_max_prio, EXPR_PRIORITY (expr));
3857
3858	/* Don't allow any insns whose data is not yet ready.
3859	Check first whether we've already tried them and failed. */
3860	if (INSN_UID (insn) < FENCE_READY_TICKS_SIZE (fence))
3861	{
3862	need_cycles = (FENCE_READY_TICKS (fence)[INSN_UID (insn)]
3863	- FENCE_CYCLE (fence));
3864	if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0)
3865	est_ticks_till_branch = MAX (est_ticks_till_branch,
3866	EXPR_PRIORITY (expr) + need_cycles);
3867
3868	if (need_cycles > 0)
3869	{
3870	stalled++;
3871	min_need_stall = (min_need_stall < 0
3872	? need_cycles
3873	: MIN (min_need_stall, need_cycles));
3874	vec_av_set.unordered_remove (ix: n);
3875
3876	if (sched_verbose >= 4)
3877	sel_print (fmt: "Expr %d is not ready until cycle %d (cached)\n",
3878	INSN_UID (insn),
3879	FENCE_READY_TICKS (fence)[INSN_UID (insn)]);
3880	continue;
3881	}
3882	}
3883
3884	/* Now resort to dependence analysis to find whether EXPR might be
3885	stalled due to dependencies from FENCE's context. */
3886	need_cycles = tick_check_p (expr, dc, fence);
3887	new_prio = EXPR_PRIORITY (expr) + EXPR_PRIORITY_ADJ (expr) + need_cycles;
3888
3889	if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0)
3890	est_ticks_till_branch = MAX (est_ticks_till_branch,
3891	new_prio);
3892
3893	if (need_cycles > 0)
3894	{
3895	if (INSN_UID (insn) >= FENCE_READY_TICKS_SIZE (fence))
3896	{
3897	int new_size = INSN_UID (insn) * 3 / 2;
3898
3899	FENCE_READY_TICKS (fence)
3900	= (int *) xrecalloc (FENCE_READY_TICKS (fence),
3901	new_size, FENCE_READY_TICKS_SIZE (fence),
3902	sizeof (int));
3903	}
3904	FENCE_READY_TICKS (fence)[INSN_UID (insn)]
3905	= FENCE_CYCLE (fence) + need_cycles;
3906
3907	stalled++;
3908	min_need_stall = (min_need_stall < 0
3909	? need_cycles
3910	: MIN (min_need_stall, need_cycles));
3911
3912	vec_av_set.unordered_remove (ix: n);
3913
3914	if (sched_verbose >= 4)
3915	sel_print (fmt: "Expr %d is not ready yet until cycle %d\n",
3916	INSN_UID (insn),
3917	FENCE_READY_TICKS (fence)[INSN_UID (insn)]);
3918	continue;
3919	}
3920
3921	if (sched_verbose >= 4)
3922	sel_print (fmt: "Expr %d is ok\n", INSN_UID (insn));
3923	min_need_stall = 0;
3924	}
3925
3926	/* Clear SCHED_NEXT. */
3927	if (FENCE_SCHED_NEXT (fence))
3928	{
3929	gcc_assert (sched_next_worked == 1);
3930	FENCE_SCHED_NEXT (fence) = NULL;
3931	}
3932
3933	/* No need to stall if this variable was not initialized. */
3934	if (min_need_stall < 0)
3935	min_need_stall = 0;
3936
3937	if (vec_av_set.is_empty ())
3938	{
3939	/* We need to set *pneed_stall here, because later we skip this code
3940	when ready list is empty. */
3941	*pneed_stall = min_need_stall;
3942	return false;
3943	}
3944	else
3945	gcc_assert (min_need_stall == 0);
3946
3947	/* Sort the vector. */
3948	vec_av_set.qsort (sel_rank_for_schedule);
3949
3950	if (sched_verbose >= 4)
3951	{
3952	sel_print (fmt: "Total ready exprs: %d, stalled: %d\n",
3953	vec_av_set.length (), stalled);
3954	sel_print (fmt: "Sorted av set (%d): ", vec_av_set.length ());
3955	FOR_EACH_VEC_ELT (vec_av_set, n, expr)
3956	dump_expr (expr);
3957	sel_print (fmt: "\n");
3958	}
3959
3960	*pneed_stall = 0;
3961	return true;
3962	}
3963
3964	/* Convert a vectored and sorted av set to the ready list that
3965	the rest of the backend wants to see. */
3966	static void
3967	convert_vec_av_set_to_ready (void)
3968	{
3969	int n;
3970	expr_t expr;
3971
3972	/* Allocate and fill the ready list from the sorted vector. */
3973	ready.n_ready = vec_av_set.length ();
3974	ready.first = ready.n_ready - 1;
3975
3976	gcc_assert (ready.n_ready > 0);
3977
3978	if (ready.n_ready > max_issue_size)
3979	{
3980	max_issue_size = ready.n_ready;
3981	sched_extend_ready_list (ready.n_ready);
3982	}
3983
3984	FOR_EACH_VEC_ELT (vec_av_set, n, expr)
3985	{
3986	vinsn_t vi = EXPR_VINSN (expr);
3987	insn_t insn = VINSN_INSN_RTX (vi);
3988
3989	ready_try[n] = 0;
3990	ready.vec[n] = insn;
3991	}
3992	}
3993
3994	/* Initialize ready list from *AV_PTR for the max_issue () call.
3995	If any unrecognizable insn found in *AV_PTR, return it (and skip
3996	max_issue). BND and FENCE are current boundary and fence,
3997	respectively. If we need to stall for some cycles before an expr
3998	from *AV_PTR would become available, write this number to *PNEED_STALL. */
3999	static expr_t
4000	fill_ready_list (av_set_t *av_ptr, blist_t bnds, fence_t fence,
4001	int *pneed_stall)
4002	{
4003	expr_t expr;
4004
4005	/* We do not support multiple boundaries per fence. */
4006	gcc_assert (BLIST_NEXT (bnds) == NULL);
4007
4008	/* Process expressions required special handling, i.e. pipelined,
4009	speculative and recog() < 0 expressions first. */
4010	process_pipelined_exprs (av_ptr);
4011	process_spec_exprs (av_ptr);
4012
4013	/* A USE could be scheduled immediately. */
4014	expr = process_use_exprs (av_ptr);
4015	if (expr)
4016	{
4017	*pneed_stall = 0;
4018	return expr;
4019	}
4020
4021	/* Turn the av set to a vector for sorting. */
4022	if (! fill_vec_av_set (av: *av_ptr, bnds, fence, pneed_stall))
4023	{
4024	ready.n_ready = 0;
4025	return NULL;
4026	}
4027
4028	/* Build the final ready list. */
4029	convert_vec_av_set_to_ready ();
4030	return NULL;
4031	}
4032
4033	/* Wrapper for dfa_new_cycle (). Returns TRUE if cycle was advanced. */
4034	static bool
4035	sel_dfa_new_cycle (insn_t insn, fence_t fence)
4036	{
4037	int last_scheduled_cycle = FENCE_LAST_SCHEDULED_INSN (fence)
4038	? INSN_SCHED_CYCLE (FENCE_LAST_SCHEDULED_INSN (fence))
4039	: FENCE_CYCLE (fence) - 1;
4040	bool res = false;
4041	int sort_p = 0;
4042
4043	if (!targetm.sched.dfa_new_cycle)
4044	return false;
4045
4046	memcpy (dest: curr_state, FENCE_STATE (fence), n: dfa_state_size);
4047
4048	while (!sort_p && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose,
4049	insn, last_scheduled_cycle,
4050	FENCE_CYCLE (fence), &sort_p))
4051	{
4052	memcpy (FENCE_STATE (fence), src: curr_state, n: dfa_state_size);
4053	advance_one_cycle (fence);
4054	memcpy (dest: curr_state, FENCE_STATE (fence), n: dfa_state_size);
4055	res = true;
4056	}
4057
4058	return res;
4059	}
4060
4061	/* Invoke reorder* target hooks on the ready list. Return the number of insns
4062	we can issue. FENCE is the current fence. */
4063	static int
4064	invoke_reorder_hooks (fence_t fence)
4065	{
4066	int issue_more;
4067	bool ran_hook = false;
4068
4069	/* Call the reorder hook at the beginning of the cycle, and call
4070	the reorder2 hook in the middle of the cycle. */
4071	if (FENCE_ISSUED_INSNS (fence) == 0)
4072	{
4073	if (targetm.sched.reorder
4074	&& !SCHED_GROUP_P (ready_element (&ready, 0))
4075	&& ready.n_ready > 1)
4076	{
4077	/* Don't give reorder the most prioritized insn as it can break
4078	pipelining. */
4079	if (pipelining_p)
4080	--ready.n_ready;
4081
4082	issue_more
4083	= targetm.sched.reorder (sched_dump, sched_verbose,
4084	ready_lastpos (&ready),
4085	&ready.n_ready, FENCE_CYCLE (fence));
4086
4087	if (pipelining_p)
4088	++ready.n_ready;
4089
4090	ran_hook = true;
4091	}
4092	else
4093	/* Initialize can_issue_more for variable_issue. */
4094	issue_more = issue_rate;
4095	}
4096	else if (targetm.sched.reorder2
4097	&& !SCHED_GROUP_P (ready_element (&ready, 0)))
4098	{
4099	if (ready.n_ready == 1)
4100	issue_more =
4101	targetm.sched.reorder2 (sched_dump, sched_verbose,
4102	ready_lastpos (&ready),
4103	&ready.n_ready, FENCE_CYCLE (fence));
4104	else
4105	{
4106	if (pipelining_p)
4107	--ready.n_ready;
4108
4109	issue_more =
4110	targetm.sched.reorder2 (sched_dump, sched_verbose,
4111	ready.n_ready
4112	? ready_lastpos (&ready) : NULL,
4113	&ready.n_ready, FENCE_CYCLE (fence));
4114
4115	if (pipelining_p)
4116	++ready.n_ready;
4117	}
4118
4119	ran_hook = true;
4120	}
4121	else
4122	issue_more = FENCE_ISSUE_MORE (fence);
4123
4124	/* Ensure that ready list and vec_av_set are in line with each other,
4125	i.e. vec_av_set[i] == ready_element (&ready, i). */
4126	if (issue_more && ran_hook)
4127	{
4128	int i, j, n;
4129	rtx_insn **arr = ready.vec;
4130	expr_t *vec = vec_av_set.address ();
4131
4132	for (i = 0, n = ready.n_ready; i < n; i++)
4133	if (EXPR_INSN_RTX (vec[i]) != arr[i])
4134	{
4135	for (j = i; j < n; j++)
4136	if (EXPR_INSN_RTX (vec[j]) == arr[i])
4137	break;
4138	gcc_assert (j < n);
4139
4140	std::swap (a&: vec[i], b&: vec[j]);
4141	}
4142	}
4143
4144	return issue_more;
4145	}
4146
4147	/* Return an EXPR corresponding to INDEX element of ready list, if
4148	FOLLOW_READY_ELEMENT is true (i.e., an expr of
4149	ready_element (&ready, INDEX) will be returned), and to INDEX element of
4150	ready.vec otherwise. */
4151	static inline expr_t
4152	find_expr_for_ready (int index, bool follow_ready_element)
4153	{
4154	expr_t expr;
4155	int real_index;
4156
4157	real_index = follow_ready_element ? ready.first - index : index;
4158
4159	expr = vec_av_set[real_index];
4160	gcc_assert (ready.vec[real_index] == EXPR_INSN_RTX (expr));
4161
4162	return expr;
4163	}
4164
4165	/* Calculate insns worth trying via lookahead_guard hook. Return a number
4166	of such insns found. */
4167	static int
4168	invoke_dfa_lookahead_guard (void)
4169	{
4170	int i, n;
4171	bool have_hook
4172	= targetm.sched.first_cycle_multipass_dfa_lookahead_guard != NULL;
4173
4174	if (sched_verbose >= 2)
4175	sel_print (fmt: "ready after reorder: ");
4176
4177	for (i = 0, n = 0; i < ready.n_ready; i++)
4178	{
4179	expr_t expr;
4180	insn_t insn;
4181	int r;
4182
4183	/* In this loop insn is Ith element of the ready list given by
4184	ready_element, not Ith element of ready.vec. */
4185	insn = ready_element (&ready, i);
4186
4187	if (! have_hook || i == 0)
4188	r = 0;
4189	else
4190	r = targetm.sched.first_cycle_multipass_dfa_lookahead_guard (insn, i);
4191
4192	gcc_assert (INSN_CODE (insn) >= 0);
4193
4194	/* Only insns with ready_try = 0 can get here
4195	from fill_ready_list. */
4196	gcc_assert (ready_try [i] == 0);
4197	ready_try[i] = r;
4198	if (!r)
4199	n++;
4200
4201	expr = find_expr_for_ready (index: i, follow_ready_element: true);
4202
4203	if (sched_verbose >= 2)
4204	{
4205	dump_vinsn (EXPR_VINSN (expr));
4206	sel_print (fmt: ":%d; ", ready_try[i]);
4207	}
4208	}
4209
4210	if (sched_verbose >= 2)
4211	sel_print (fmt: "\n");
4212	return n;
4213	}
4214
4215	/* Calculate the number of privileged insns and return it. */
4216	static int
4217	calculate_privileged_insns (void)
4218	{
4219	expr_t cur_expr, min_spec_expr = NULL;
4220	int privileged_n = 0, i;
4221
4222	for (i = 0; i < ready.n_ready; i++)
4223	{
4224	if (ready_try[i])
4225	continue;
4226
4227	if (! min_spec_expr)
4228	min_spec_expr = find_expr_for_ready (index: i, follow_ready_element: true);
4229
4230	cur_expr = find_expr_for_ready (index: i, follow_ready_element: true);
4231
4232	if (EXPR_SPEC (cur_expr) > EXPR_SPEC (min_spec_expr))
4233	break;
4234
4235	++privileged_n;
4236	}
4237
4238	if (i == ready.n_ready)
4239	privileged_n = 0;
4240
4241	if (sched_verbose >= 2)
4242	sel_print (fmt: "privileged_n: %d insns with SPEC %d\n",
4243	privileged_n, privileged_n ? EXPR_SPEC (min_spec_expr) : -1);
4244	return privileged_n;
4245	}
4246
4247	/* Call the rest of the hooks after the choice was made. Return
4248	the number of insns that still can be issued given that the current
4249	number is ISSUE_MORE. FENCE and BEST_INSN are the current fence
4250	and the insn chosen for scheduling, respectively. */
4251	static int
4252	invoke_aftermath_hooks (fence_t fence, rtx_insn *best_insn, int issue_more)
4253	{
4254	gcc_assert (INSN_P (best_insn));
4255
4256	/* First, call dfa_new_cycle, and then variable_issue, if available. */
4257	sel_dfa_new_cycle (insn: best_insn, fence);
4258
4259	if (targetm.sched.variable_issue)
4260	{
4261	memcpy (dest: curr_state, FENCE_STATE (fence), n: dfa_state_size);
4262	issue_more =
4263	targetm.sched.variable_issue (sched_dump, sched_verbose, best_insn,
4264	issue_more);
4265	memcpy (FENCE_STATE (fence), src: curr_state, n: dfa_state_size);
4266	}
4267	else if (!DEBUG_INSN_P (best_insn)
4268	&& GET_CODE (PATTERN (best_insn)) != USE
4269	&& GET_CODE (PATTERN (best_insn)) != CLOBBER)
4270	issue_more--;
4271
4272	return issue_more;
4273	}
4274
4275	/* Estimate the cost of issuing INSN on DFA state STATE. */
4276	static int
4277	estimate_insn_cost (rtx_insn *insn, state_t state)
4278	{
4279	static state_t temp = NULL;
4280	int cost;
4281
4282	if (!temp)
4283	temp = xmalloc (dfa_state_size);
4284
4285	memcpy (dest: temp, src: state, n: dfa_state_size);
4286	cost = state_transition (temp, insn);
4287
4288	if (cost < 0)
4289	return 0;
4290	else if (cost == 0)
4291	return 1;
4292	return cost;
4293	}
4294
4295	/* Return the cost of issuing EXPR on the FENCE as estimated by DFA.
4296	This function properly handles ASMs, USEs etc. */
4297	static int
4298	get_expr_cost (expr_t expr, fence_t fence)
4299	{
4300	rtx_insn *insn = EXPR_INSN_RTX (expr);
4301
4302	if (recog_memoized (insn) < 0)
4303	{
4304	if (!FENCE_STARTS_CYCLE_P (fence)
4305	&& INSN_ASM_P (insn))
4306	/* This is asm insn which is tryed to be issued on the
4307	cycle not first. Issue it on the next cycle. */
4308	return 1;
4309	else
4310	/* A USE insn, or something else we don't need to
4311	understand. We can't pass these directly to
4312	state_transition because it will trigger a
4313	fatal error for unrecognizable insns. */
4314	return 0;
4315	}
4316	else
4317	return estimate_insn_cost (insn, FENCE_STATE (fence));
4318	}
4319
4320	/* Find the best insn for scheduling, either via max_issue or just take
4321	the most prioritized available. */
4322	static int
4323	choose_best_insn (fence_t fence, int privileged_n, int *index)
4324	{
4325	int can_issue = 0;
4326
4327	if (dfa_lookahead > 0)
4328	{
4329	cycle_issued_insns = FENCE_ISSUED_INSNS (fence);
4330	/* TODO: pass equivalent of first_cycle_insn_p to max_issue (). */
4331	can_issue = max_issue (&ready, privileged_n,
4332	FENCE_STATE (fence), true, index);
4333	if (sched_verbose >= 2)
4334	sel_print (fmt: "max_issue: we can issue %d insns, already did %d insns\n",
4335	can_issue, FENCE_ISSUED_INSNS (fence));
4336	}
4337	else
4338	{
4339	/* We can't use max_issue; just return the first available element. */
4340	int i;
4341
4342	for (i = 0; i < ready.n_ready; i++)
4343	{
4344	expr_t expr = find_expr_for_ready (index: i, follow_ready_element: true);
4345
4346	if (get_expr_cost (expr, fence) < 1)
4347	{
4348	can_issue = can_issue_more;
4349	*index = i;
4350
4351	if (sched_verbose >= 2)
4352	sel_print (fmt: "using %dth insn from the ready list\n", i + 1);
4353
4354	break;
4355	}
4356	}
4357
4358	if (i == ready.n_ready)
4359	{
4360	can_issue = 0;
4361	*index = -1;
4362	}
4363	}
4364
4365	return can_issue;
4366	}
4367
4368	/* Choose the best expr from *AV_VLIW_PTR and a suitable register for it.
4369	BNDS and FENCE are current boundaries and scheduling fence respectively.
4370	Return the expr found and NULL if nothing can be issued atm.
4371	Write to PNEED_STALL the number of cycles to stall if no expr was found. */
4372	static expr_t
4373	find_best_expr (av_set_t *av_vliw_ptr, blist_t bnds, fence_t fence,
4374	int *pneed_stall)
4375	{
4376	expr_t best;
4377
4378	/* Choose the best insn for scheduling via:
4379	1) sorting the ready list based on priority;
4380	2) calling the reorder hook;
4381	3) calling max_issue. */
4382	best = fill_ready_list (av_ptr: av_vliw_ptr, bnds, fence, pneed_stall);
4383	if (best == NULL && ready.n_ready > 0)
4384	{
4385	int privileged_n, index;
4386
4387	can_issue_more = invoke_reorder_hooks (fence);
4388	if (can_issue_more > 0)
4389	{
4390	/* Try choosing the best insn until we find one that is could be
4391	scheduled due to liveness restrictions on its destination register.
4392	In the future, we'd like to choose once and then just probe insns
4393	in the order of their priority. */
4394	invoke_dfa_lookahead_guard ();
4395	privileged_n = calculate_privileged_insns ();
4396	can_issue_more = choose_best_insn (fence, privileged_n, index: &index);
4397	if (can_issue_more)
4398	best = find_expr_for_ready (index, follow_ready_element: true);
4399	}
4400	/* We had some available insns, so if we can't issue them,
4401	we have a stall. */
4402	if (can_issue_more == 0)
4403	{
4404	best = NULL;
4405	*pneed_stall = 1;
4406	}
4407	}
4408
4409	if (best != NULL)
4410	{
4411	can_issue_more = invoke_aftermath_hooks (fence, EXPR_INSN_RTX (best),
4412	issue_more: can_issue_more);
4413	if (targetm.sched.variable_issue
4414	&& can_issue_more == 0)
4415	*pneed_stall = 1;
4416	}
4417
4418	if (sched_verbose >= 2)
4419	{
4420	if (best != NULL)
4421	{
4422	sel_print (fmt: "Best expression (vliw form): ");
4423	dump_expr (best);
4424	sel_print (fmt: "; cycle %d\n", FENCE_CYCLE (fence));
4425	}
4426	else
4427	sel_print (fmt: "No best expr found!\n");
4428	}
4429
4430	return best;
4431	}
4432
4433
4434	/* Functions that implement the core of the scheduler. */
4435
4436
4437	/* Emit an instruction from EXPR with SEQNO and VINSN after
4438	PLACE_TO_INSERT. */
4439	static insn_t
4440	emit_insn_from_expr_after (expr_t expr, vinsn_t vinsn, int seqno,
4441	insn_t place_to_insert)
4442	{
4443	/* This assert fails when we have identical instructions
4444	one of which dominates the other. In this case move_op ()
4445	finds the first instruction and doesn't search for second one.
4446	The solution would be to compute av_set after the first found
4447	insn and, if insn present in that set, continue searching.
4448	For now we workaround this issue in move_op. */
4449	gcc_assert (!INSN_IN_STREAM_P (EXPR_INSN_RTX (expr)));
4450
4451	if (EXPR_WAS_RENAMED (expr))
4452	{
4453	unsigned regno = expr_dest_regno (expr);
4454
4455	if (HARD_REGISTER_NUM_P (regno))
4456	{
4457	df_set_regs_ever_live (regno, true);
4458	reg_rename_tick[regno] = ++reg_rename_this_tick;
4459	}
4460	}
4461
4462	return sel_gen_insn_from_expr_after (expr, vinsn, seqno,
4463	place_to_insert);
4464	}
4465
4466	/* Return TRUE if BB can hold bookkeeping code. */
4467	static bool
4468	block_valid_for_bookkeeping_p (basic_block bb)
4469	{
4470	insn_t bb_end = BB_END (bb);
4471
4472	if (!in_current_region_p (bb) || EDGE_COUNT (bb->succs) > 1)
4473	return false;
4474
4475	if (INSN_P (bb_end))
4476	{
4477	if (INSN_SCHED_TIMES (bb_end) > 0)
4478	return false;
4479	}
4480	else
4481	gcc_assert (NOTE_INSN_BASIC_BLOCK_P (bb_end));
4482
4483	return true;
4484	}
4485
4486	/* Attempt to find a block that can hold bookkeeping code for path(s) incoming
4487	into E2->dest, except from E1->src (there may be a sequence of empty basic
4488	blocks between E1->src and E2->dest). Return found block, or NULL if new
4489	one must be created. If LAX holds, don't assume there is a simple path
4490	from E1->src to E2->dest. */
4491	static basic_block
4492	find_block_for_bookkeeping (edge e1, edge e2, bool lax)
4493	{
4494	basic_block candidate_block = NULL;
4495	edge e;
4496
4497	/* Loop over edges from E1 to E2, inclusive. */
4498	for (e = e1; !lax || e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun); e =
4499	EDGE_SUCC (e->dest, 0))
4500	{
4501	if (EDGE_COUNT (e->dest->preds) == 2)
4502	{
4503	if (candidate_block == NULL)
4504	candidate_block = (EDGE_PRED (e->dest, 0) == e
4505	? EDGE_PRED (e->dest, 1)->src
4506	: EDGE_PRED (e->dest, 0)->src);
4507	else
4508	/* Found additional edge leading to path from e1 to e2
4509	from aside. */
4510	return NULL;
4511	}
4512	else if (EDGE_COUNT (e->dest->preds) > 2)
4513	/* Several edges leading to path from e1 to e2 from aside. */
4514	return NULL;
4515
4516	if (e == e2)
4517	return ((!lax || candidate_block)
4518	&& block_valid_for_bookkeeping_p (bb: candidate_block)
4519	? candidate_block
4520	: NULL);
4521
4522	if (lax && EDGE_COUNT (e->dest->succs) != 1)
4523	return NULL;
4524	}
4525
4526	if (lax)
4527	return NULL;
4528
4529	gcc_unreachable ();
4530	}
4531
4532	/* Create new basic block for bookkeeping code for path(s) incoming into
4533	E2->dest, except from E1->src. Return created block. */
4534	static basic_block
4535	create_block_for_bookkeeping (edge e1, edge e2)
4536	{
4537	basic_block new_bb, bb = e2->dest;
4538
4539	/* Check that we don't spoil the loop structure. */
4540	if (current_loop_nest)
4541	{
4542	basic_block latch = current_loop_nest->latch;
4543
4544	/* We do not split header. */
4545	gcc_assert (e2->dest != current_loop_nest->header);
4546
4547	/* We do not redirect the only edge to the latch block. */
4548	gcc_assert (e1->dest != latch
4549	|| !single_pred_p (latch)
4550	|| e1 != single_pred_edge (latch));
4551	}
4552
4553	/* Split BB to insert BOOK_INSN there. */
4554	new_bb = sched_split_block (bb, NULL);
4555
4556	/* Move note_list from the upper bb. */
4557	gcc_assert (BB_NOTE_LIST (new_bb) == NULL_RTX);
4558	BB_NOTE_LIST (new_bb) = BB_NOTE_LIST (bb);
4559	BB_NOTE_LIST (bb) = NULL;
4560
4561	gcc_assert (e2->dest == bb);
4562
4563	/* Skip block for bookkeeping copy when leaving E1->src. */
4564	if (e1->flags & EDGE_FALLTHRU)
4565	sel_redirect_edge_and_branch_force (e1, new_bb);
4566	else
4567	sel_redirect_edge_and_branch (e1, new_bb);
4568
4569	gcc_assert (e1->dest == new_bb);
4570	gcc_assert (sel_bb_empty_p (bb));
4571
4572	/* To keep basic block numbers in sync between debug and non-debug
4573	compilations, we have to rotate blocks here. Consider that we
4574	started from (a,b)->d, (c,d)->e, and d contained only debug
4575	insns. It would have been removed before if the debug insns
4576	weren't there, so we'd have split e rather than d. So what we do
4577	now is to swap the block numbers of new_bb and
4578	single_succ(new_bb) == e, so that the insns that were in e before
4579	get the new block number. */
4580
4581	if (MAY_HAVE_DEBUG_INSNS)
4582	{
4583	basic_block succ;
4584	insn_t insn = sel_bb_head (new_bb);
4585	insn_t last;
4586
4587	if (DEBUG_INSN_P (insn)
4588	&& single_succ_p (bb: new_bb)
4589	&& (succ = single_succ (bb: new_bb))
4590	&& succ != EXIT_BLOCK_PTR_FOR_FN (cfun)
4591	&& DEBUG_INSN_P ((last = sel_bb_end (new_bb))))
4592	{
4593	while (insn != last && (DEBUG_INSN_P (insn) || NOTE_P (insn)))
4594	insn = NEXT_INSN (insn);
4595
4596	if (insn == last)
4597	{
4598	sel_global_bb_info_def gbi;
4599	sel_region_bb_info_def rbi;
4600
4601	if (sched_verbose >= 2)
4602	sel_print (fmt: "Swapping block ids %i and %i\n",
4603	new_bb->index, succ->index);
4604
4605	std::swap (a&: new_bb->index, b&: succ->index);
4606
4607	SET_BASIC_BLOCK_FOR_FN (cfun, new_bb->index, new_bb);
4608	SET_BASIC_BLOCK_FOR_FN (cfun, succ->index, succ);
4609
4610	memcpy (dest: &gbi, SEL_GLOBAL_BB_INFO (new_bb), n: sizeof (gbi));
4611	memcpy (SEL_GLOBAL_BB_INFO (new_bb), SEL_GLOBAL_BB_INFO (succ),
4612	n: sizeof (gbi));
4613	memcpy (SEL_GLOBAL_BB_INFO (succ), src: &gbi, n: sizeof (gbi));
4614
4615	memcpy (dest: &rbi, SEL_REGION_BB_INFO (new_bb), n: sizeof (rbi));
4616	memcpy (SEL_REGION_BB_INFO (new_bb), SEL_REGION_BB_INFO (succ),
4617	n: sizeof (rbi));
4618	memcpy (SEL_REGION_BB_INFO (succ), src: &rbi, n: sizeof (rbi));
4619
4620	std::swap (BLOCK_TO_BB (new_bb->index),
4621	BLOCK_TO_BB (succ->index));
4622
4623	std::swap (CONTAINING_RGN (new_bb->index),
4624	CONTAINING_RGN (succ->index));
4625
4626	for (int i = 0; i < current_nr_blocks; i++)
4627	if (BB_TO_BLOCK (i) == succ->index)
4628	BB_TO_BLOCK (i) = new_bb->index;
4629	else if (BB_TO_BLOCK (i) == new_bb->index)
4630	BB_TO_BLOCK (i) = succ->index;
4631
4632	FOR_BB_INSNS (new_bb, insn)
4633	if (INSN_P (insn))
4634	EXPR_ORIG_BB_INDEX (INSN_EXPR (insn)) = new_bb->index;
4635
4636	FOR_BB_INSNS (succ, insn)
4637	if (INSN_P (insn))
4638	EXPR_ORIG_BB_INDEX (INSN_EXPR (insn)) = succ->index;
4639
4640	if (bitmap_clear_bit (code_motion_visited_blocks, new_bb->index))
4641	bitmap_set_bit (code_motion_visited_blocks, succ->index);
4642
4643	gcc_assert (LABEL_P (BB_HEAD (new_bb))
4644	&& LABEL_P (BB_HEAD (succ)));
4645
4646	if (sched_verbose >= 4)
4647	sel_print (fmt: "Swapping code labels %i and %i\n",
4648	CODE_LABEL_NUMBER (BB_HEAD (new_bb)),
4649	CODE_LABEL_NUMBER (BB_HEAD (succ)));
4650
4651	std::swap (CODE_LABEL_NUMBER (BB_HEAD (new_bb)),
4652	CODE_LABEL_NUMBER (BB_HEAD (succ)));
4653	}
4654	}
4655	}
4656
4657	return bb;
4658	}
4659
4660	/* Return insn after which we must insert bookkeeping code for path(s) incoming
4661	into E2->dest, except from E1->src. If the returned insn immediately
4662	precedes a fence, assign that fence to *FENCE_TO_REWIND. */
4663	static insn_t
4664	find_place_for_bookkeeping (edge e1, edge e2, fence_t *fence_to_rewind)
4665	{
4666	insn_t place_to_insert;
4667	/* Find a basic block that can hold bookkeeping. If it can be found, do not
4668	create new basic block, but insert bookkeeping there. */
4669	basic_block book_block = find_block_for_bookkeeping (e1, e2, lax: false);
4670
4671	if (book_block)
4672	{
4673	place_to_insert = BB_END (book_block);
4674
4675	/* Don't use a block containing only debug insns for
4676	bookkeeping, this causes scheduling differences between debug
4677	and non-debug compilations, for the block would have been
4678	removed already. */
4679	if (DEBUG_INSN_P (place_to_insert))
4680	{
4681	rtx_insn *insn = sel_bb_head (book_block);
4682
4683	while (insn != place_to_insert &&
4684	(DEBUG_INSN_P (insn) || NOTE_P (insn)))
4685	insn = NEXT_INSN (insn);
4686
4687	if (insn == place_to_insert)
4688	book_block = NULL;
4689	}
4690	}
4691
4692	if (!book_block)
4693	{
4694	book_block = create_block_for_bookkeeping (e1, e2);
4695	place_to_insert = BB_END (book_block);
4696	if (sched_verbose >= 9)
4697	sel_print (fmt: "New block is %i, split from bookkeeping block %i\n",
4698	EDGE_SUCC (book_block, 0)->dest->index, book_block->index);
4699	}
4700	else
4701	{
4702	if (sched_verbose >= 9)
4703	sel_print (fmt: "Pre-existing bookkeeping block is %i\n", book_block->index);
4704	}
4705
4706	*fence_to_rewind = NULL;
4707	/* If basic block ends with a jump, insert bookkeeping code right before it.
4708	Notice if we are crossing a fence when taking PREV_INSN. */
4709	if (INSN_P (place_to_insert) && control_flow_insn_p (place_to_insert))
4710	{
4711	*fence_to_rewind = flist_lookup (fences, place_to_insert);
4712	place_to_insert = PREV_INSN (insn: place_to_insert);
4713	}
4714
4715	return place_to_insert;
4716	}
4717
4718	/* Find a proper seqno for bookkeeing insn inserted at PLACE_TO_INSERT
4719	for JOIN_POINT. */
4720	static int
4721	find_seqno_for_bookkeeping (insn_t place_to_insert, insn_t join_point)
4722	{
4723	int seqno;
4724
4725	/* Check if we are about to insert bookkeeping copy before a jump, and use
4726	jump's seqno for the copy; otherwise, use JOIN_POINT's seqno. */
4727	rtx_insn *next = NEXT_INSN (insn: place_to_insert);
4728	if (INSN_P (next)
4729	&& JUMP_P (next)
4730	&& BLOCK_FOR_INSN (insn: next) == BLOCK_FOR_INSN (insn: place_to_insert))
4731	{
4732	gcc_assert (INSN_SCHED_TIMES (next) == 0);
4733	seqno = INSN_SEQNO (next);
4734	}
4735	else if (INSN_SEQNO (join_point) > 0)
4736	seqno = INSN_SEQNO (join_point);
4737	else
4738	{
4739	seqno = get_seqno_by_preds (place_to_insert);
4740
4741	/* Sometimes the fences can move in such a way that there will be
4742	no instructions with positive seqno around this bookkeeping.
4743	This means that there will be no way to get to it by a regular
4744	fence movement. Never mind because we pick up such pieces for
4745	rescheduling anyways, so any positive value will do for now. */
4746	if (seqno < 0)
4747	{
4748	gcc_assert (pipelining_p);
4749	seqno = 1;
4750	}
4751	}
4752
4753	gcc_assert (seqno > 0);
4754	return seqno;
4755	}
4756
4757	/* Insert bookkeeping copy of C_EXPS's insn after PLACE_TO_INSERT, assigning
4758	NEW_SEQNO to it. Return created insn. */
4759	static insn_t
4760	emit_bookkeeping_insn (insn_t place_to_insert, expr_t c_expr, int new_seqno)
4761	{
4762	rtx_insn *new_insn_rtx = create_copy_of_insn_rtx (EXPR_INSN_RTX (c_expr));
4763
4764	vinsn_t new_vinsn
4765	= create_vinsn_from_insn_rtx (new_insn_rtx,
4766	VINSN_UNIQUE_P (EXPR_VINSN (c_expr)));
4767
4768	insn_t new_insn = emit_insn_from_expr_after (expr: c_expr, vinsn: new_vinsn, seqno: new_seqno,
4769	place_to_insert);
4770
4771	INSN_SCHED_TIMES (new_insn) = 0;
4772	bitmap_set_bit (current_copies, INSN_UID (insn: new_insn));
4773
4774	return new_insn;
4775	}
4776
4777	/* Generate a bookkeeping copy of C_EXPR's insn for path(s) incoming into to
4778	E2->dest, except from E1->src (there may be a sequence of empty blocks
4779	between E1->src and E2->dest). Return block containing the copy.
4780	All scheduler data is initialized for the newly created insn. */
4781	static basic_block
4782	generate_bookkeeping_insn (expr_t c_expr, edge e1, edge e2)
4783	{
4784	insn_t join_point, place_to_insert, new_insn;
4785	int new_seqno;
4786	bool need_to_exchange_data_sets;
4787	fence_t fence_to_rewind;
4788
4789	if (sched_verbose >= 4)
4790	sel_print (fmt: "Generating bookkeeping insn (%d->%d)\n", e1->src->index,
4791	e2->dest->index);
4792
4793	join_point = sel_bb_head (e2->dest);
4794	place_to_insert = find_place_for_bookkeeping (e1, e2, fence_to_rewind: &fence_to_rewind);
4795	new_seqno = find_seqno_for_bookkeeping (place_to_insert, join_point);
4796	need_to_exchange_data_sets
4797	= sel_bb_empty_p (BLOCK_FOR_INSN (insn: place_to_insert));
4798
4799	new_insn = emit_bookkeeping_insn (place_to_insert, c_expr, new_seqno);
4800
4801	if (fence_to_rewind)
4802	FENCE_INSN (fence_to_rewind) = new_insn;
4803
4804	/* When inserting bookkeeping insn in new block, av sets should be
4805	following: old basic block (that now holds bookkeeping) data sets are
4806	the same as was before generation of bookkeeping, and new basic block
4807	(that now hold all other insns of old basic block) data sets are
4808	invalid. So exchange data sets for these basic blocks as sel_split_block
4809	mistakenly exchanges them in this case. Cannot do it earlier because
4810	when single instruction is added to new basic block it should hold NULL
4811	lv_set. */
4812	if (need_to_exchange_data_sets)
4813	exchange_data_sets (BLOCK_FOR_INSN (insn: new_insn),
4814	BLOCK_FOR_INSN (insn: join_point));
4815
4816	stat_bookkeeping_copies++;
4817	return BLOCK_FOR_INSN (insn: new_insn);
4818	}
4819
4820	/* Remove from AV_PTR all insns that may need bookkeeping when scheduling
4821	on FENCE, but we are unable to copy them. */
4822	static void
4823	remove_insns_that_need_bookkeeping (fence_t fence, av_set_t *av_ptr)
4824	{
4825	expr_t expr;
4826	av_set_iterator i;
4827
4828	/* An expression does not need bookkeeping if it is available on all paths
4829	from current block to original block and current block dominates
4830	original block. We check availability on all paths by examining
4831	EXPR_SPEC; this is not equivalent, because it may be positive even
4832	if expr is available on all paths (but if expr is not available on
4833	any path, EXPR_SPEC will be positive). */
4834
4835	FOR_EACH_EXPR_1 (expr, i, av_ptr)
4836	{
4837	if (!control_flow_insn_p (EXPR_INSN_RTX (expr))
4838	&& (!bookkeeping_p || VINSN_UNIQUE_P (EXPR_VINSN (expr)))
4839	&& (EXPR_SPEC (expr)
4840	|| !EXPR_ORIG_BB_INDEX (expr)
4841	|| !dominated_by_p (CDI_DOMINATORS,
4842	BASIC_BLOCK_FOR_FN (cfun,
4843	EXPR_ORIG_BB_INDEX (expr)),
4844	BLOCK_FOR_INSN (FENCE_INSN (fence)))))
4845	{
4846	if (sched_verbose >= 4)
4847	sel_print (fmt: "Expr %d removed because it would need bookkeeping, which "
4848	"cannot be created\n", INSN_UID (EXPR_INSN_RTX (expr)));
4849	av_set_iter_remove (&i);
4850	}
4851	}
4852	}
4853
4854	/* Moving conditional jump through some instructions.
4855
4856	Consider example:
4857
4858	... <- current scheduling point
4859	NOTE BASIC BLOCK: <- bb header
4860	(p8) add r14=r14+0x9;;
4861	(p8) mov [r14]=r23
4862	(!p8) jump L1;;
4863	NOTE BASIC BLOCK:
4864	...
4865
4866	We can schedule jump one cycle earlier, than mov, because they cannot be
4867	executed together as their predicates are mutually exclusive.
4868
4869	This is done in this way: first, new fallthrough basic block is created
4870	after jump (it is always can be done, because there already should be a
4871	fallthrough block, where control flow goes in case of predicate being true -
4872	in our example; otherwise there should be a dependence between those
4873	instructions and jump and we cannot schedule jump right now);
4874	next, all instructions between jump and current scheduling point are moved
4875	to this new block. And the result is this:
4876
4877	NOTE BASIC BLOCK:
4878	(!p8) jump L1 <- current scheduling point
4879	NOTE BASIC BLOCK: <- bb header
4880	(p8) add r14=r14+0x9;;
4881	(p8) mov [r14]=r23
4882	NOTE BASIC BLOCK:
4883	...
4884	*/
4885	static void
4886	move_cond_jump (rtx_insn *insn, bnd_t bnd)
4887	{
4888	edge ft_edge;
4889	basic_block block_from, block_next, block_new, block_bnd, bb;
4890	rtx_insn *next, *prev, *link, *head;
4891
4892	block_from = BLOCK_FOR_INSN (insn);
4893	block_bnd = BLOCK_FOR_INSN (BND_TO (bnd));
4894	prev = BND_TO (bnd);
4895
4896	/* Moving of jump should not cross any other jumps or beginnings of new
4897	basic blocks. The only exception is when we move a jump through
4898	mutually exclusive insns along fallthru edges. */
4899	if (flag_checking && block_from != block_bnd)
4900	{
4901	bb = block_from;
4902	for (link = PREV_INSN (insn); link != PREV_INSN (insn: prev);
4903	link = PREV_INSN (insn: link))
4904	{
4905	if (INSN_P (link))
4906	gcc_assert (sched_insns_conditions_mutex_p (insn, link));
4907	if (BLOCK_FOR_INSN (insn: link) && BLOCK_FOR_INSN (insn: link) != bb)
4908	{
4909	gcc_assert (single_pred (bb) == BLOCK_FOR_INSN (link));
4910	bb = BLOCK_FOR_INSN (insn: link);
4911	}
4912	}
4913	}
4914
4915	/* Jump is moved to the boundary. */
4916	next = PREV_INSN (insn);
4917	BND_TO (bnd) = insn;
4918
4919	ft_edge = find_fallthru_edge_from (block_from);
4920	block_next = ft_edge->dest;
4921	/* There must be a fallthrough block (or where should go
4922	control flow in case of false jump predicate otherwise?). */
4923	gcc_assert (block_next);
4924
4925	/* Create new empty basic block after source block. */
4926	block_new = sel_split_edge (ft_edge);
4927	gcc_assert (block_new->next_bb == block_next
4928	&& block_from->next_bb == block_new);
4929
4930	/* Move all instructions except INSN to BLOCK_NEW. */
4931	bb = block_bnd;
4932	head = BB_HEAD (block_new);
4933	while (bb != block_from->next_bb)
4934	{
4935	rtx_insn *from, *to;
4936	from = bb == block_bnd ? prev : sel_bb_head (bb);
4937	to = bb == block_from ? next : sel_bb_end (bb);
4938
4939	/* The jump being moved can be the first insn in the block.
4940	In this case we don't have to move anything in this block. */
4941	if (NEXT_INSN (insn: to) != from)
4942	{
4943	reorder_insns (from, to, head);
4944
4945	for (link = to; link != head; link = PREV_INSN (insn: link))
4946	EXPR_ORIG_BB_INDEX (INSN_EXPR (link)) = block_new->index;
4947	head = to;
4948	}
4949
4950	/* Cleanup possibly empty blocks left. */
4951	block_next = bb->next_bb;
4952	if (bb != block_from)
4953	tidy_control_flow (bb, false);
4954	bb = block_next;
4955	}
4956
4957	/* Assert there is no jump to BLOCK_NEW, only fallthrough edge. */
4958	gcc_assert (NOTE_INSN_BASIC_BLOCK_P (BB_HEAD (block_new)));
4959
4960	gcc_assert (!sel_bb_empty_p (block_from)
4961	&& !sel_bb_empty_p (block_new));
4962
4963	/* Update data sets for BLOCK_NEW to represent that INSN and
4964	instructions from the other branch of INSN is no longer
4965	available at BLOCK_NEW. */
4966	BB_AV_LEVEL (block_new) = global_level;
4967	gcc_assert (BB_LV_SET (block_new) == NULL);
4968	BB_LV_SET (block_new) = get_clear_regset_from_pool ();
4969	update_data_sets (insn: sel_bb_head (block_new));
4970
4971	/* INSN is a new basic block header - so prepare its data
4972	structures and update availability and liveness sets. */
4973	update_data_sets (insn);
4974
4975	if (sched_verbose >= 4)
4976	sel_print (fmt: "Moving jump %d\n", INSN_UID (insn));
4977	}
4978
4979	/* Remove nops generated during move_op for preventing removal of empty
4980	basic blocks. */
4981	static void
4982	remove_temp_moveop_nops (bool full_tidying)
4983	{
4984	int i;
4985	insn_t insn;
4986
4987	FOR_EACH_VEC_ELT (vec_temp_moveop_nops, i, insn)
4988	{
4989	gcc_assert (INSN_NOP_P (insn));
4990	return_nop_to_pool (insn, full_tidying);
4991	}
4992
4993	/* Empty the vector. */
4994	if (vec_temp_moveop_nops.length () > 0)
4995	vec_temp_moveop_nops.block_remove (ix: 0, len: vec_temp_moveop_nops.length ());
4996	}
4997
4998	/* Records the maximal UID before moving up an instruction. Used for
4999	distinguishing between bookkeeping copies and original insns. */
5000	static int max_uid_before_move_op = 0;
5001
5002	/* When true, we're always scheduling next insn on the already scheduled code
5003	to get the right insn data for the following bundling or other passes. */
5004	static int force_next_insn = 0;
5005
5006	/* Remove from AV_VLIW_P all instructions but next when debug counter
5007	tells us so. Next instruction is fetched from BNDS. */
5008	static void
5009	remove_insns_for_debug (blist_t bnds, av_set_t *av_vliw_p)
5010	{
5011	if (! dbg_cnt (index: sel_sched_insn_cnt) || force_next_insn)
5012	/* Leave only the next insn in av_vliw. */
5013	{
5014	av_set_iterator av_it;
5015	expr_t expr;
5016	bnd_t bnd = BLIST_BND (bnds);
5017	insn_t next = BND_TO (bnd);
5018
5019	gcc_assert (BLIST_NEXT (bnds) == NULL);
5020
5021	FOR_EACH_EXPR_1 (expr, av_it, av_vliw_p)
5022	if (EXPR_INSN_RTX (expr) != next)
5023	av_set_iter_remove (&av_it);
5024	}
5025	}
5026
5027	/* Compute available instructions on BNDS. FENCE is the current fence. Write
5028	the computed set to *AV_VLIW_P. */
5029	static void
5030	compute_av_set_on_boundaries (fence_t fence, blist_t bnds, av_set_t *av_vliw_p)
5031	{
5032	if (sched_verbose >= 2)
5033	{
5034	sel_print (fmt: "Boundaries: ");
5035	dump_blist (bnds);
5036	sel_print (fmt: "\n");
5037	}
5038
5039	for (; bnds; bnds = BLIST_NEXT (bnds))
5040	{
5041	bnd_t bnd = BLIST_BND (bnds);
5042	av_set_t av1_copy;
5043	insn_t bnd_to = BND_TO (bnd);
5044
5045	/* Rewind BND->TO to the basic block header in case some bookkeeping
5046	instructions were inserted before BND->TO and it needs to be
5047	adjusted. */
5048	if (sel_bb_head_p (bnd_to))
5049	gcc_assert (INSN_SCHED_TIMES (bnd_to) == 0);
5050	else
5051	while (INSN_SCHED_TIMES (PREV_INSN (bnd_to)) == 0)
5052	{
5053	bnd_to = PREV_INSN (insn: bnd_to);
5054	if (sel_bb_head_p (bnd_to))
5055	break;
5056	}
5057
5058	if (BND_TO (bnd) != bnd_to)
5059	{
5060	gcc_assert (FENCE_INSN (fence) == BND_TO (bnd));
5061	FENCE_INSN (fence) = bnd_to;
5062	BND_TO (bnd) = bnd_to;
5063	}
5064
5065	av_set_clear (&BND_AV (bnd));
5066	BND_AV (bnd) = compute_av_set (BND_TO (bnd), NULL, ws: 0, unique_p: true);
5067
5068	av_set_clear (&BND_AV1 (bnd));
5069	BND_AV1 (bnd) = av_set_copy (BND_AV (bnd));
5070
5071	moveup_set_inside_insn_group (avp: &BND_AV1 (bnd), NULL);
5072
5073	av1_copy = av_set_copy (BND_AV1 (bnd));
5074	av_set_union_and_clear (av_vliw_p, &av1_copy, NULL);
5075	}
5076
5077	if (sched_verbose >= 2)
5078	{
5079	sel_print (fmt: "Available exprs (vliw form): ");
5080	dump_av_set (*av_vliw_p);
5081	sel_print (fmt: "\n");
5082	}
5083	}
5084
5085	/* Calculate the sequential av set on BND corresponding to the EXPR_VLIW
5086	expression. When FOR_MOVEOP is true, also replace the register of
5087	expressions found with the register from EXPR_VLIW. */
5088	static av_set_t
5089	find_sequential_best_exprs (bnd_t bnd, expr_t expr_vliw, bool for_moveop)
5090	{
5091	av_set_t expr_seq = NULL;
5092	expr_t expr;
5093	av_set_iterator i;
5094
5095	FOR_EACH_EXPR (expr, i, BND_AV (bnd))
5096	{
5097	if (equal_after_moveup_path_p (expr, NULL, expr_vliw))
5098	{
5099	if (for_moveop)
5100	{
5101	/* The sequential expression has the right form to pass
5102	to move_op except when renaming happened. Put the
5103	correct register in EXPR then. */
5104	if (EXPR_SEPARABLE_P (expr) && REG_P (EXPR_LHS (expr)))
5105	{
5106	if (expr_dest_regno (expr) != expr_dest_regno (expr_vliw))
5107	{
5108	replace_dest_with_reg_in_expr (expr, EXPR_LHS (expr_vliw));
5109	stat_renamed_scheduled++;
5110	}
5111	/* Also put the correct TARGET_AVAILABLE bit on the expr.
5112	This is needed when renaming came up with original
5113	register. */
5114	else if (EXPR_TARGET_AVAILABLE (expr)
5115	!= EXPR_TARGET_AVAILABLE (expr_vliw))
5116	{
5117	gcc_assert (EXPR_TARGET_AVAILABLE (expr_vliw) == 1);
5118	EXPR_TARGET_AVAILABLE (expr) = 1;
5119	}
5120	}
5121	if (EXPR_WAS_SUBSTITUTED (expr))
5122	stat_substitutions_total++;
5123	}
5124
5125	av_set_add (&expr_seq, expr);
5126
5127	/* With substitution inside insn group, it is possible
5128	that more than one expression in expr_seq will correspond
5129	to expr_vliw. In this case, choose one as the attempt to
5130	move both leads to miscompiles. */
5131	break;
5132	}
5133	}
5134
5135	if (for_moveop && sched_verbose >= 2)
5136	{
5137	sel_print (fmt: "Best expression(s) (sequential form): ");
5138	dump_av_set (expr_seq);
5139	sel_print (fmt: "\n");
5140	}
5141
5142	return expr_seq;
5143	}
5144
5145
5146	/* Move nop to previous block. */
5147	static void ATTRIBUTE_UNUSED
5148	move_nop_to_previous_block (insn_t nop, basic_block prev_bb)
5149	{
5150	insn_t prev_insn, next_insn;
5151
5152	gcc_assert (sel_bb_head_p (nop)
5153	&& prev_bb == BLOCK_FOR_INSN (nop)->prev_bb);
5154	rtx_note *note = bb_note (BLOCK_FOR_INSN (insn: nop));
5155	prev_insn = sel_bb_end (prev_bb);
5156	next_insn = NEXT_INSN (insn: nop);
5157	gcc_assert (prev_insn != NULL_RTX
5158	&& PREV_INSN (note) == prev_insn);
5159
5160	SET_NEXT_INSN (prev_insn) = nop;
5161	SET_PREV_INSN (nop) = prev_insn;
5162
5163	SET_PREV_INSN (note) = nop;
5164	SET_NEXT_INSN (note) = next_insn;
5165
5166	SET_NEXT_INSN (nop) = note;
5167	SET_PREV_INSN (next_insn) = note;
5168
5169	BB_END (prev_bb) = nop;
5170	BLOCK_FOR_INSN (insn: nop) = prev_bb;
5171	}
5172
5173	/* Prepare a place to insert the chosen expression on BND. */
5174	static insn_t
5175	prepare_place_to_insert (bnd_t bnd)
5176	{
5177	insn_t place_to_insert;
5178
5179	/* Init place_to_insert before calling move_op, as the later
5180	can possibly remove BND_TO (bnd). */
5181	if (/* If this is not the first insn scheduled. */
5182	BND_PTR (bnd))
5183	{
5184	/* Add it after last scheduled. */
5185	place_to_insert = ILIST_INSN (BND_PTR (bnd));
5186	if (DEBUG_INSN_P (place_to_insert))
5187	{
5188	ilist_t l = BND_PTR (bnd);
5189	while ((l = ILIST_NEXT (l)) &&
5190	DEBUG_INSN_P (ILIST_INSN (l)))
5191	;
5192	if (!l)
5193	place_to_insert = NULL;
5194	}
5195	}
5196	else
5197	place_to_insert = NULL;
5198
5199	if (!place_to_insert)
5200	{
5201	/* Add it before BND_TO. The difference is in the
5202	basic block, where INSN will be added. */
5203	place_to_insert = get_nop_from_pool (BND_TO (bnd));
5204	gcc_assert (BLOCK_FOR_INSN (place_to_insert)
5205	== BLOCK_FOR_INSN (BND_TO (bnd)));
5206	}
5207
5208	return place_to_insert;
5209	}
5210
5211	/* Find original instructions for EXPR_SEQ and move it to BND boundary.
5212	Return the expression to emit in C_EXPR. */
5213	static bool
5214	move_exprs_to_boundary (bnd_t bnd, expr_t expr_vliw,
5215	av_set_t expr_seq, expr_t c_expr)
5216	{
5217	bool b, should_move;
5218	unsigned book_uid;
5219	bitmap_iterator bi;
5220	int n_bookkeeping_copies_before_moveop;
5221
5222	/* Make a move. This call will remove the original operation,
5223	insert all necessary bookkeeping instructions and update the
5224	data sets. After that all we have to do is add the operation
5225	at before BND_TO (BND). */
5226	n_bookkeeping_copies_before_moveop = stat_bookkeeping_copies;
5227	max_uid_before_move_op = get_max_uid ();
5228	bitmap_clear (current_copies);
5229	bitmap_clear (current_originators);
5230
5231	b = move_op (BND_TO (bnd), expr_seq, expr_vliw,
5232	get_dest_from_orig_ops (expr_seq), c_expr, &should_move);
5233
5234	/* We should be able to find the expression we've chosen for
5235	scheduling. */
5236	gcc_assert (b);
5237
5238	if (stat_bookkeeping_copies > n_bookkeeping_copies_before_moveop)
5239	stat_insns_needed_bookkeeping++;
5240
5241	EXECUTE_IF_SET_IN_BITMAP (current_copies, 0, book_uid, bi)
5242	{
5243	unsigned uid;
5244	bitmap_iterator bi;
5245
5246	/* We allocate these bitmaps lazily. */
5247	if (! INSN_ORIGINATORS_BY_UID (book_uid))
5248	INSN_ORIGINATORS_BY_UID (book_uid) = BITMAP_ALLOC (NULL);
5249
5250	bitmap_copy (INSN_ORIGINATORS_BY_UID (book_uid),
5251	current_originators);
5252
5253	/* Transitively add all originators' originators. */
5254	EXECUTE_IF_SET_IN_BITMAP (current_originators, 0, uid, bi)
5255	if (INSN_ORIGINATORS_BY_UID (uid))
5256	bitmap_ior_into (INSN_ORIGINATORS_BY_UID (book_uid),
5257	INSN_ORIGINATORS_BY_UID (uid));
5258	}
5259
5260	return should_move;
5261	}
5262
5263
5264	/* Debug a DFA state as an array of bytes. */
5265	static void
5266	debug_state (state_t state)
5267	{
5268	unsigned char *p;
5269	unsigned int i, size = dfa_state_size;
5270
5271	sel_print (fmt: "state (%u):", size);
5272	for (i = 0, p = (unsigned char *) state; i < size; i++)
5273	sel_print (fmt: " %d", p[i]);
5274	sel_print (fmt: "\n");
5275	}
5276
5277	/* Advance state on FENCE with INSN. Return true if INSN is
5278	an ASM, and we should advance state once more. */
5279	static bool
5280	advance_state_on_fence (fence_t fence, insn_t insn)
5281	{
5282	bool asm_p;
5283
5284	if (recog_memoized (insn) >= 0)
5285	{
5286	int res;
5287	state_t temp_state = alloca (dfa_state_size);
5288
5289	gcc_assert (!INSN_ASM_P (insn));
5290	asm_p = false;
5291
5292	memcpy (dest: temp_state, FENCE_STATE (fence), n: dfa_state_size);
5293	res = state_transition (FENCE_STATE (fence), insn);
5294	gcc_assert (res < 0);
5295
5296	if (memcmp (s1: temp_state, FENCE_STATE (fence), n: dfa_state_size))
5297	{
5298	FENCE_ISSUED_INSNS (fence)++;
5299
5300	/* We should never issue more than issue_rate insns. */
5301	if (FENCE_ISSUED_INSNS (fence) > issue_rate)
5302	gcc_unreachable ();
5303	}
5304	}
5305	else
5306	{
5307	/* This could be an ASM insn which we'd like to schedule
5308	on the next cycle. */
5309	asm_p = INSN_ASM_P (insn);
5310	if (!FENCE_STARTS_CYCLE_P (fence) && asm_p)
5311	advance_one_cycle (fence);
5312	}
5313
5314	if (sched_verbose >= 2)
5315	debug_state (FENCE_STATE (fence));
5316	if (!DEBUG_INSN_P (insn))
5317	FENCE_STARTS_CYCLE_P (fence) = 0;
5318	FENCE_ISSUE_MORE (fence) = can_issue_more;
5319	return asm_p;
5320	}
5321
5322	/* Update FENCE on which INSN was scheduled and this INSN, too. NEED_STALL
5323	is nonzero if we need to stall after issuing INSN. */
5324	static void
5325	update_fence_and_insn (fence_t fence, insn_t insn, int need_stall)
5326	{
5327	bool asm_p;
5328
5329	/* First, reflect that something is scheduled on this fence. */
5330	asm_p = advance_state_on_fence (fence, insn);
5331	FENCE_LAST_SCHEDULED_INSN (fence) = insn;
5332	vec_safe_push (FENCE_EXECUTING_INSNS (fence), obj: insn);
5333	if (SCHED_GROUP_P (insn))
5334	{
5335	FENCE_SCHED_NEXT (fence) = INSN_SCHED_NEXT (insn);
5336	SCHED_GROUP_P (insn) = 0;
5337	}
5338	else
5339	FENCE_SCHED_NEXT (fence) = NULL;
5340	if (INSN_UID (insn) < FENCE_READY_TICKS_SIZE (fence))
5341	FENCE_READY_TICKS (fence) [INSN_UID (insn)] = 0;
5342
5343	/* Set instruction scheduling info. This will be used in bundling,
5344	pipelining, tick computations etc. */
5345	++INSN_SCHED_TIMES (insn);
5346	EXPR_TARGET_AVAILABLE (INSN_EXPR (insn)) = true;
5347	EXPR_ORIG_SCHED_CYCLE (INSN_EXPR (insn)) = FENCE_CYCLE (fence);
5348	INSN_AFTER_STALL_P (insn) = FENCE_AFTER_STALL_P (fence);
5349	INSN_SCHED_CYCLE (insn) = FENCE_CYCLE (fence);
5350
5351	/* This does not account for adjust_cost hooks, just add the biggest
5352	constant the hook may add to the latency. TODO: make this
5353	a target dependent constant. */
5354	INSN_READY_CYCLE (insn)
5355	= INSN_SCHED_CYCLE (insn) + (INSN_CODE (insn) < 0
5356	? 1
5357	: maximal_insn_latency (insn) + 1);
5358
5359	/* Change these fields last, as they're used above. */
5360	FENCE_AFTER_STALL_P (fence) = 0;
5361	if (asm_p || need_stall)
5362	advance_one_cycle (fence);
5363
5364	/* Indicate that we've scheduled something on this fence. */
5365	FENCE_SCHEDULED_P (fence) = true;
5366	scheduled_something_on_previous_fence = true;
5367
5368	/* Print debug information when insn's fields are updated. */
5369	if (sched_verbose >= 2)
5370	{
5371	sel_print (fmt: "Scheduling insn: ");
5372	dump_insn_1 (insn, 1);
5373	sel_print (fmt: "\n");
5374	}
5375	}
5376
5377	/* Update boundary BND (and, if needed, FENCE) with INSN, remove the
5378	old boundary from BNDSP, add new boundaries to BNDS_TAIL_P and
5379	return it. */
5380	static blist_t *
5381	update_boundaries (fence_t fence, bnd_t bnd, insn_t insn, blist_t *bndsp,
5382	blist_t *bnds_tailp)
5383	{
5384	succ_iterator si;
5385	insn_t succ;
5386
5387	advance_deps_context (BND_DC (bnd), insn);
5388	FOR_EACH_SUCC_1 (succ, si, insn,
5389	SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
5390	{
5391	ilist_t ptr = ilist_copy (BND_PTR (bnd));
5392
5393	ilist_add (lp: &ptr, insn);
5394
5395	if (DEBUG_INSN_P (insn) && sel_bb_end_p (insn)
5396	&& is_ineligible_successor (insn: succ, p: ptr))
5397	{
5398	ilist_clear (&ptr);
5399	continue;
5400	}
5401
5402	if (FENCE_INSN (fence) == insn && !sel_bb_end_p (insn))
5403	{
5404	if (sched_verbose >= 9)
5405	sel_print (fmt: "Updating fence insn from %i to %i\n",
5406	INSN_UID (insn), INSN_UID (insn: succ));
5407	FENCE_INSN (fence) = succ;
5408	}
5409	blist_add (bnds_tailp, succ, ptr, BND_DC (bnd));
5410	bnds_tailp = &BLIST_NEXT (*bnds_tailp);
5411	}
5412
5413	blist_remove (bndsp);
5414	return bnds_tailp;
5415	}
5416
5417	/* Schedule EXPR_VLIW on BND. Return the insn emitted. */
5418	static insn_t
5419	schedule_expr_on_boundary (bnd_t bnd, expr_t expr_vliw, int seqno)
5420	{
5421	av_set_t expr_seq;
5422	expr_t c_expr = XALLOCA (expr_def);
5423	insn_t place_to_insert;
5424	insn_t insn;
5425	bool should_move;
5426
5427	expr_seq = find_sequential_best_exprs (bnd, expr_vliw, for_moveop: true);
5428
5429	/* In case of scheduling a jump skipping some other instructions,
5430	prepare CFG. After this, jump is at the boundary and can be
5431	scheduled as usual insn by MOVE_OP. */
5432	if (vinsn_cond_branch_p (EXPR_VINSN (expr_vliw)))
5433	{
5434	insn = EXPR_INSN_RTX (expr_vliw);
5435
5436	/* Speculative jumps are not handled. */
5437	if (insn != BND_TO (bnd)
5438	&& !sel_insn_is_speculation_check (insn))
5439	move_cond_jump (insn, bnd);
5440	}
5441
5442	/* Find a place for C_EXPR to schedule. */
5443	place_to_insert = prepare_place_to_insert (bnd);
5444	should_move = move_exprs_to_boundary (bnd, expr_vliw, expr_seq, c_expr);
5445	clear_expr (c_expr);
5446
5447	/* Add the instruction. The corner case to care about is when
5448	the expr_seq set has more than one expr, and we chose the one that
5449	is not equal to expr_vliw. Then expr_vliw may be insn in stream, and
5450	we can't use it. Generate the new vinsn. */
5451	if (INSN_IN_STREAM_P (EXPR_INSN_RTX (expr_vliw)))
5452	{
5453	vinsn_t vinsn_new;
5454
5455	vinsn_new = vinsn_copy (EXPR_VINSN (expr_vliw), false);
5456	change_vinsn_in_expr (expr_vliw, vinsn_new);
5457	should_move = false;
5458	}
5459	if (should_move)
5460	insn = sel_move_insn (expr_vliw, seqno, place_to_insert);
5461	else
5462	insn = emit_insn_from_expr_after (expr: expr_vliw, NULL, seqno,
5463	place_to_insert);
5464
5465	/* Return the nops generated for preserving of data sets back
5466	into pool. */
5467	if (INSN_NOP_P (place_to_insert))
5468	return_nop_to_pool (place_to_insert, !DEBUG_INSN_P (insn));
5469	remove_temp_moveop_nops (full_tidying: !DEBUG_INSN_P (insn));
5470
5471	av_set_clear (&expr_seq);
5472
5473	/* Save the expression scheduled so to reset target availability if we'll
5474	meet it later on the same fence. */
5475	if (EXPR_WAS_RENAMED (expr_vliw))
5476	vinsn_vec_add (vinsn_vec: &vec_target_unavailable_vinsns, INSN_EXPR (insn));
5477
5478	/* Check that the recent movement didn't destroyed loop
5479	structure. */
5480	gcc_assert (!pipelining_p
5481	|| current_loop_nest == NULL
5482	|| loop_latch_edge (current_loop_nest));
5483	return insn;
5484	}
5485
5486	/* Stall for N cycles on FENCE. */
5487	static void
5488	stall_for_cycles (fence_t fence, int n)
5489	{
5490	int could_more;
5491
5492	could_more = n > 1 || FENCE_ISSUED_INSNS (fence) < issue_rate;
5493	while (n--)
5494	advance_one_cycle (fence);
5495	if (could_more)
5496	FENCE_AFTER_STALL_P (fence) = 1;
5497	}
5498
5499	/* Gather a parallel group of insns at FENCE and assign their seqno
5500	to SEQNO. All scheduled insns are gathered in SCHEDULED_INSNS_TAILPP
5501	list for later recalculation of seqnos. */
5502	static void
5503	fill_insns (fence_t fence, int seqno, ilist_t **scheduled_insns_tailpp)
5504	{
5505	blist_t bnds = NULL, *bnds_tailp;
5506	av_set_t av_vliw = NULL;
5507	insn_t insn = FENCE_INSN (fence);
5508
5509	if (sched_verbose >= 2)
5510	sel_print (fmt: "Starting fill_insns for insn %d, cycle %d\n",
5511	INSN_UID (insn), FENCE_CYCLE (fence));
5512
5513	blist_add (&bnds, insn, NULL, FENCE_DC (fence));
5514	bnds_tailp = &BLIST_NEXT (bnds);
5515	set_target_context (FENCE_TC (fence));
5516	can_issue_more = FENCE_ISSUE_MORE (fence);
5517	target_bb = INSN_BB (insn);
5518
5519	/* Do while we can add any operation to the current group. */
5520	do
5521	{
5522	blist_t *bnds_tailp1, *bndsp;
5523	expr_t expr_vliw;
5524	int need_stall = false;
5525	int was_stall = 0, scheduled_insns = 0;
5526	int max_insns = pipelining_p ? issue_rate : 2 * issue_rate;
5527	int max_stall = pipelining_p ? 1 : 3;
5528	bool last_insn_was_debug = false;
5529	bool was_debug_bb_end_p = false;
5530
5531	compute_av_set_on_boundaries (fence, bnds, av_vliw_p: &av_vliw);
5532	remove_insns_that_need_bookkeeping (fence, av_ptr: &av_vliw);
5533	remove_insns_for_debug (bnds, av_vliw_p: &av_vliw);
5534
5535	/* Return early if we have nothing to schedule. */
5536	if (av_vliw == NULL)
5537	break;
5538
5539	/* Choose the best expression and, if needed, destination register
5540	for it. */
5541	do
5542	{
5543	expr_vliw = find_best_expr (av_vliw_ptr: &av_vliw, bnds, fence, pneed_stall: &need_stall);
5544	if (! expr_vliw && need_stall)
5545	{
5546	/* All expressions required a stall. Do not recompute av sets
5547	as we'll get the same answer (modulo the insns between
5548	the fence and its boundary, which will not be available for
5549	pipelining).
5550	If we are going to stall for too long, break to recompute av
5551	sets and bring more insns for pipelining. */
5552	was_stall++;
5553	if (need_stall <= 3)
5554	stall_for_cycles (fence, n: need_stall);
5555	else
5556	{
5557	stall_for_cycles (fence, n: 1);
5558	break;
5559	}
5560	}
5561	}
5562	while (! expr_vliw && need_stall);
5563
5564	/* Now either we've selected expr_vliw or we have nothing to schedule. */
5565	if (!expr_vliw)
5566	{
5567	av_set_clear (&av_vliw);
5568	break;
5569	}
5570
5571	bndsp = &bnds;
5572	bnds_tailp1 = bnds_tailp;
5573
5574	do
5575	/* This code will be executed only once until we'd have several
5576	boundaries per fence. */
5577	{
5578	bnd_t bnd = BLIST_BND (*bndsp);
5579
5580	if (!av_set_is_in_p (BND_AV1 (bnd), EXPR_VINSN (expr_vliw)))
5581	{
5582	bndsp = &BLIST_NEXT (*bndsp);
5583	continue;
5584	}
5585
5586	insn = schedule_expr_on_boundary (bnd, expr_vliw, seqno);
5587	last_insn_was_debug = DEBUG_INSN_P (insn);
5588	if (last_insn_was_debug)
5589	was_debug_bb_end_p = (insn == BND_TO (bnd) && sel_bb_end_p (insn));
5590	update_fence_and_insn (fence, insn, need_stall);
5591	bnds_tailp = update_boundaries (fence, bnd, insn, bndsp, bnds_tailp);
5592
5593	/* Add insn to the list of scheduled on this cycle instructions. */
5594	ilist_add (lp: *scheduled_insns_tailpp, insn);
5595	*scheduled_insns_tailpp = &ILIST_NEXT (**scheduled_insns_tailpp);
5596	}
5597	while (*bndsp != *bnds_tailp1);
5598
5599	av_set_clear (&av_vliw);
5600	if (!last_insn_was_debug)
5601	scheduled_insns++;
5602
5603	/* We currently support information about candidate blocks only for
5604	one 'target_bb' block. Hence we can't schedule after jump insn,
5605	as this will bring two boundaries and, hence, necessity to handle
5606	information for two or more blocks concurrently. */
5607	if ((last_insn_was_debug ? was_debug_bb_end_p : sel_bb_end_p (insn))
5608	|| (was_stall
5609	&& (was_stall >= max_stall
5610	|| scheduled_insns >= max_insns)))
5611	break;
5612	}
5613	while (bnds);
5614
5615	gcc_assert (!FENCE_BNDS (fence));
5616
5617	/* Update boundaries of the FENCE. */
5618	while (bnds)
5619	{
5620	ilist_t ptr = BND_PTR (BLIST_BND (bnds));
5621
5622	if (ptr)
5623	{
5624	insn = ILIST_INSN (ptr);
5625
5626	if (!ilist_is_in_p (FENCE_BNDS (fence), insn))
5627	ilist_add (lp: &FENCE_BNDS (fence), insn);
5628	}
5629
5630	blist_remove (&bnds);
5631	}
5632
5633	/* Update target context on the fence. */
5634	reset_target_context (FENCE_TC (fence), false);
5635	}
5636
5637	/* All exprs in ORIG_OPS must have the same destination register or memory.
5638	Return that destination. */
5639	static rtx
5640	get_dest_from_orig_ops (av_set_t orig_ops)
5641	{
5642	rtx dest = NULL_RTX;
5643	av_set_iterator av_it;
5644	expr_t expr;
5645	bool first_p = true;
5646
5647	FOR_EACH_EXPR (expr, av_it, orig_ops)
5648	{
5649	rtx x = EXPR_LHS (expr);
5650
5651	if (first_p)
5652	{
5653	first_p = false;
5654	dest = x;
5655	}
5656	else
5657	gcc_assert (dest == x
5658	|| (dest != NULL_RTX && x != NULL_RTX
5659	&& rtx_equal_p (dest, x)));
5660	}
5661
5662	return dest;
5663	}
5664
5665	/* Update data sets for the bookkeeping block and record those expressions
5666	which become no longer available after inserting this bookkeeping. */
5667	static void
5668	update_and_record_unavailable_insns (basic_block book_block)
5669	{
5670	av_set_iterator i;
5671	av_set_t old_av_set = NULL;
5672	expr_t cur_expr;
5673	rtx_insn *bb_end = sel_bb_end (book_block);
5674
5675	/* First, get correct liveness in the bookkeeping block. The problem is
5676	the range between the bookeeping insn and the end of block. */
5677	update_liveness_on_insn (insn: bb_end);
5678	if (control_flow_insn_p (bb_end))
5679	update_liveness_on_insn (insn: PREV_INSN (insn: bb_end));
5680
5681	/* If there's valid av_set on BOOK_BLOCK, then there might exist another
5682	fence above, where we may choose to schedule an insn which is
5683	actually blocked from moving up with the bookkeeping we create here. */
5684	if (AV_SET_VALID_P (sel_bb_head (book_block)))
5685	{
5686	old_av_set = av_set_copy (BB_AV_SET (book_block));
5687	update_data_sets (insn: sel_bb_head (book_block));
5688
5689	/* Traverse all the expressions in the old av_set and check whether
5690	CUR_EXPR is in new AV_SET. */
5691	FOR_EACH_EXPR (cur_expr, i, old_av_set)
5692	{
5693	expr_t new_expr = av_set_lookup (BB_AV_SET (book_block),
5694	EXPR_VINSN (cur_expr));
5695
5696	if (! new_expr
5697	/* In this case, we can just turn off the E_T_A bit, but we can't
5698	represent this information with the current vector. */
5699	|| EXPR_TARGET_AVAILABLE (new_expr)
5700	!= EXPR_TARGET_AVAILABLE (cur_expr))
5701	/* Unfortunately, the below code could be also fired up on
5702	separable insns, e.g. when moving insns through the new
5703	speculation check as in PR 53701. */
5704	vinsn_vec_add (vinsn_vec: &vec_bookkeeping_blocked_vinsns, expr: cur_expr);
5705	}
5706
5707	av_set_clear (&old_av_set);
5708	}
5709	}
5710
5711	/* The main effect of this function is that sparams->c_expr is merged
5712	with (or copied to) lparams->c_expr_merged. If there's only one successor,
5713	we avoid merging anything by copying sparams->c_expr to lparams->c_expr_merged.
5714	lparams->c_expr_merged is copied back to sparams->c_expr after all
5715	successors has been traversed. lparams->c_expr_local is an expr allocated
5716	on stack in the caller function, and is used if there is more than one
5717	successor.
5718
5719	SUCC is one of the SUCCS_NORMAL successors of INSN,
5720	MOVEOP_DRV_CALL_RES is the result of call code_motion_path_driver on succ,
5721	LPARAMS and STATIC_PARAMS contain the parameters described above. */
5722	static void
5723	move_op_merge_succs (insn_t insn ATTRIBUTE_UNUSED,
5724	insn_t succ ATTRIBUTE_UNUSED,
5725	int moveop_drv_call_res,
5726	cmpd_local_params_p lparams, void *static_params)
5727	{
5728	moveop_static_params_p sparams = (moveop_static_params_p) static_params;
5729
5730	/* Nothing to do, if original expr wasn't found below. */
5731	if (moveop_drv_call_res != 1)
5732	return;
5733
5734	/* If this is a first successor. */
5735	if (!lparams->c_expr_merged)
5736	{
5737	lparams->c_expr_merged = sparams->c_expr;
5738	sparams->c_expr = lparams->c_expr_local;
5739	}
5740	else
5741	{
5742	/* We must merge all found expressions to get reasonable
5743	EXPR_SPEC_DONE_DS for the resulting insn. If we don't
5744	do so then we can first find the expr with epsilon
5745	speculation success probability and only then with the
5746	good probability. As a result the insn will get epsilon
5747	probability and will never be scheduled because of
5748	weakness_cutoff in find_best_expr.
5749
5750	We call merge_expr_data here instead of merge_expr
5751	because due to speculation C_EXPR and X may have the
5752	same insns with different speculation types. And as of
5753	now such insns are considered non-equal.
5754
5755	However, EXPR_SCHED_TIMES is different -- we must get
5756	SCHED_TIMES from a real insn, not a bookkeeping copy.
5757	We force this here. Instead, we may consider merging
5758	SCHED_TIMES to the maximum instead of minimum in the
5759	below function. */
5760	int old_times = EXPR_SCHED_TIMES (lparams->c_expr_merged);
5761
5762	merge_expr_data (lparams->c_expr_merged, sparams->c_expr, NULL);
5763	if (EXPR_SCHED_TIMES (sparams->c_expr) == 0)
5764	EXPR_SCHED_TIMES (lparams->c_expr_merged) = old_times;
5765
5766	clear_expr (sparams->c_expr);
5767	}
5768	}
5769
5770	/* Add used regs for the successor SUCC into SPARAMS->USED_REGS.
5771
5772	SUCC is one of the SUCCS_NORMAL successors of INSN,
5773	MOVEOP_DRV_CALL_RES is the result of call code_motion_path_driver on succ or 0,
5774	if SUCC is one of SUCCS_BACK or SUCCS_OUT.
5775	STATIC_PARAMS contain USED_REGS set. */
5776	static void
5777	fur_merge_succs (insn_t insn ATTRIBUTE_UNUSED, insn_t succ,
5778	int moveop_drv_call_res,
5779	cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
5780	void *static_params)
5781	{
5782	regset succ_live;
5783	fur_static_params_p sparams = (fur_static_params_p) static_params;
5784
5785	/* Here we compute live regsets only for branches that do not lie
5786	on the code motion paths. These branches correspond to value
5787	MOVEOP_DRV_CALL_RES==0 and include SUCCS_BACK and SUCCS_OUT, though
5788	for such branches code_motion_path_driver is not called. */
5789	if (moveop_drv_call_res != 0)
5790	return;
5791
5792	/* Mark all registers that do not meet the following condition:
5793	(3) not live on the other path of any conditional branch
5794	that is passed by the operation, in case original
5795	operations are not present on both paths of the
5796	conditional branch. */
5797	succ_live = compute_live (insn: succ);
5798	IOR_REG_SET (sparams->used_regs, succ_live);
5799	}
5800
5801	/* This function is called after the last successor. Copies LP->C_EXPR_MERGED
5802	into SP->CEXPR. */
5803	static void
5804	move_op_after_merge_succs (cmpd_local_params_p lp, void *sparams)
5805	{
5806	moveop_static_params_p sp = (moveop_static_params_p) sparams;
5807
5808	sp->c_expr = lp->c_expr_merged;
5809	}
5810
5811	/* Track bookkeeping copies created, insns scheduled, and blocks for
5812	rescheduling when INSN is found by move_op. */
5813	static void
5814	track_scheduled_insns_and_blocks (rtx_insn *insn)
5815	{
5816	/* Even if this insn can be a copy that will be removed during current move_op,
5817	we still need to count it as an originator. */
5818	bitmap_set_bit (current_originators, INSN_UID (insn));
5819
5820	if (!bitmap_clear_bit (current_copies, INSN_UID (insn)))
5821	{
5822	/* Note that original block needs to be rescheduled, as we pulled an
5823	instruction out of it. */
5824	if (INSN_SCHED_TIMES (insn) > 0)
5825	bitmap_set_bit (blocks_to_reschedule, BLOCK_FOR_INSN (insn)->index);
5826	else if (INSN_UID (insn) < first_emitted_uid && !DEBUG_INSN_P (insn))
5827	num_insns_scheduled++;
5828	}
5829
5830	/* For instructions we must immediately remove insn from the
5831	stream, so subsequent update_data_sets () won't include this
5832	insn into av_set.
5833	For expr we must make insn look like "INSN_REG (insn) := c_expr". */
5834	if (INSN_UID (insn) > max_uid_before_move_op)
5835	stat_bookkeeping_copies--;
5836	}
5837
5838	/* Emit a register-register copy for INSN if needed. Return true if
5839	emitted one. PARAMS is the move_op static parameters. */
5840	static bool
5841	maybe_emit_renaming_copy (rtx_insn *insn,
5842	moveop_static_params_p params)
5843	{
5844	bool insn_emitted = false;
5845	rtx cur_reg;
5846
5847	/* Bail out early when expression cannot be renamed at all. */
5848	if (!EXPR_SEPARABLE_P (params->c_expr))
5849	return false;
5850
5851	cur_reg = expr_dest_reg (params->c_expr);
5852	gcc_assert (cur_reg && params->dest && REG_P (params->dest));
5853
5854	/* If original operation has expr and the register chosen for
5855	that expr is not original operation's dest reg, substitute
5856	operation's right hand side with the register chosen. */
5857	if (REGNO (params->dest) != REGNO (cur_reg))
5858	{
5859	insn_t reg_move_insn, reg_move_insn_rtx;
5860
5861	reg_move_insn_rtx = create_insn_rtx_with_rhs (INSN_VINSN (insn),
5862	rhs_rtx: params->dest);
5863	reg_move_insn = sel_gen_insn_from_rtx_after (reg_move_insn_rtx,
5864	INSN_EXPR (insn),
5865	INSN_SEQNO (insn),
5866	insn);
5867	EXPR_SPEC_DONE_DS (INSN_EXPR (reg_move_insn)) = 0;
5868	replace_dest_with_reg_in_expr (expr: params->c_expr, new_reg: params->dest);
5869
5870	insn_emitted = true;
5871	params->was_renamed = true;
5872	}
5873
5874	return insn_emitted;
5875	}
5876
5877	/* Emit a speculative check for INSN speculated as EXPR if needed.
5878	Return true if we've emitted one. PARAMS is the move_op static
5879	parameters. */
5880	static bool
5881	maybe_emit_speculative_check (rtx_insn *insn, expr_t expr,
5882	moveop_static_params_p params)
5883	{
5884	bool insn_emitted = false;
5885	insn_t x;
5886	ds_t check_ds;
5887
5888	check_ds = get_spec_check_type_for_insn (insn, expr);
5889	if (check_ds != 0)
5890	{
5891	/* A speculation check should be inserted. */
5892	x = create_speculation_check (c_expr: params->c_expr, check_ds, orig_insn: insn);
5893	insn_emitted = true;
5894	}
5895	else
5896	{
5897	EXPR_SPEC_DONE_DS (INSN_EXPR (insn)) = 0;
5898	x = insn;
5899	}
5900
5901	gcc_assert (EXPR_SPEC_DONE_DS (INSN_EXPR (x)) == 0
5902	&& EXPR_SPEC_TO_CHECK_DS (INSN_EXPR (x)) == 0);
5903	return insn_emitted;
5904	}
5905
5906	/* Handle transformations that leave an insn in place of original
5907	insn such as renaming/speculation. Return true if one of such
5908	transformations actually happened, and we have emitted this insn. */
5909	static bool
5910	handle_emitting_transformations (rtx_insn *insn, expr_t expr,
5911	moveop_static_params_p params)
5912	{
5913	bool insn_emitted = false;
5914
5915	insn_emitted = maybe_emit_renaming_copy (insn, params);
5916	insn_emitted |= maybe_emit_speculative_check (insn, expr, params);
5917
5918	return insn_emitted;
5919	}
5920
5921	/* If INSN is the only insn in the basic block (not counting JUMP,
5922	which may be a jump to next insn, and DEBUG_INSNs), we want to
5923	leave a NOP there till the return to fill_insns. */
5924
5925	static bool
5926	need_nop_to_preserve_insn_bb (rtx_insn *insn)
5927	{
5928	insn_t bb_head, bb_end, bb_next, in_next;
5929	basic_block bb = BLOCK_FOR_INSN (insn);
5930
5931	bb_head = sel_bb_head (bb);
5932	bb_end = sel_bb_end (bb);
5933
5934	if (bb_head == bb_end)
5935	return true;
5936
5937	while (bb_head != bb_end && DEBUG_INSN_P (bb_head))
5938	bb_head = NEXT_INSN (insn: bb_head);
5939
5940	if (bb_head == bb_end)
5941	return true;
5942
5943	while (bb_head != bb_end && DEBUG_INSN_P (bb_end))
5944	bb_end = PREV_INSN (insn: bb_end);
5945
5946	if (bb_head == bb_end)
5947	return true;
5948
5949	bb_next = NEXT_INSN (insn: bb_head);
5950	while (bb_next != bb_end && DEBUG_INSN_P (bb_next))
5951	bb_next = NEXT_INSN (insn: bb_next);
5952
5953	if (bb_next == bb_end && JUMP_P (bb_end))
5954	return true;
5955
5956	in_next = NEXT_INSN (insn);
5957	while (DEBUG_INSN_P (in_next))
5958	in_next = NEXT_INSN (insn: in_next);
5959
5960	if (IN_CURRENT_FENCE_P (in_next))
5961	return true;
5962
5963	return false;
5964	}
5965
5966	/* Remove INSN from stream. When ONLY_DISCONNECT is true, its data
5967	is not removed but reused when INSN is re-emitted. */
5968	static void
5969	remove_insn_from_stream (rtx_insn *insn, bool only_disconnect)
5970	{
5971	/* If there's only one insn in the BB, make sure that a nop is
5972	inserted into it, so the basic block won't disappear when we'll
5973	delete INSN below with sel_remove_insn. It should also survive
5974	till the return to fill_insns. */
5975	if (need_nop_to_preserve_insn_bb (insn))
5976	{
5977	insn_t nop = get_nop_from_pool (insn);
5978	gcc_assert (INSN_NOP_P (nop));
5979	vec_temp_moveop_nops.safe_push (obj: nop);
5980	}
5981
5982	sel_remove_insn (insn, only_disconnect, false);
5983	}
5984
5985	/* This function is called when original expr is found.
5986	INSN - current insn traversed, EXPR - the corresponding expr found.
5987	LPARAMS is the local parameters of code modion driver, STATIC_PARAMS
5988	is static parameters of move_op. */
5989	static void
5990	move_op_orig_expr_found (insn_t insn, expr_t expr,
5991	cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
5992	void *static_params)
5993	{
5994	bool only_disconnect;
5995	moveop_static_params_p params = (moveop_static_params_p) static_params;
5996
5997	copy_expr_onside (params->c_expr, INSN_EXPR (insn));
5998	track_scheduled_insns_and_blocks (insn);
5999	handle_emitting_transformations (insn, expr, params);
6000	only_disconnect = params->uid == INSN_UID (insn);
6001
6002	/* Mark that we've disconnected an insn. */
6003	if (only_disconnect)
6004	params->uid = -1;
6005	remove_insn_from_stream (insn, only_disconnect);
6006	}
6007
6008	/* The function is called when original expr is found.
6009	INSN - current insn traversed, EXPR - the corresponding expr found,
6010	crossed_call_abis and original_insns in STATIC_PARAMS are updated. */
6011	static void
6012	fur_orig_expr_found (insn_t insn, expr_t expr ATTRIBUTE_UNUSED,
6013	cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
6014	void *static_params)
6015	{
6016	fur_static_params_p params = (fur_static_params_p) static_params;
6017	regset tmp;
6018
6019	if (CALL_P (insn))
6020	params->crossed_call_abis |= 1 << insn_callee_abi (insn).id ();
6021
6022	def_list_add (params->original_insns, insn, params->crossed_call_abis);
6023
6024	/* Mark the registers that do not meet the following condition:
6025	(2) not among the live registers of the point
6026	immediately following the first original operation on
6027	a given downward path, except for the original target
6028	register of the operation. */
6029	tmp = get_clear_regset_from_pool ();
6030	compute_live_below_insn (insn, regs: tmp);
6031	AND_COMPL_REG_SET (tmp, INSN_REG_SETS (insn));
6032	AND_COMPL_REG_SET (tmp, INSN_REG_CLOBBERS (insn));
6033	IOR_REG_SET (params->used_regs, tmp);
6034	return_regset_to_pool (tmp);
6035
6036	/* (*1) We need to add to USED_REGS registers that are read by
6037	INSN's lhs. This may lead to choosing wrong src register.
6038	E.g. (scheduling const expr enabled):
6039
6040	429: ax=0x0 <- Can't use AX for this expr (0x0)
6041	433: dx=[bp-0x18]
6042	427: [ax+dx+0x1]=ax
6043	REG_DEAD: ax
6044	168: di=dx
6045	REG_DEAD: dx
6046	*/
6047	/* FIXME: see comment above and enable MEM_P
6048	in vinsn_separable_p. */
6049	gcc_assert (!VINSN_SEPARABLE_P (INSN_VINSN (insn))
6050	|| !MEM_P (INSN_LHS (insn)));
6051	}
6052
6053	/* This function is called on the ascending pass, before returning from
6054	current basic block. */
6055	static void
6056	move_op_at_first_insn (insn_t insn, cmpd_local_params_p lparams,
6057	void *static_params)
6058	{
6059	moveop_static_params_p sparams = (moveop_static_params_p) static_params;
6060	basic_block book_block = NULL;
6061
6062	/* When we have removed the boundary insn for scheduling, which also
6063	happened to be the end insn in its bb, we don't need to update sets. */
6064	if (!lparams->removed_last_insn
6065	&& lparams->e1
6066	&& sel_bb_head_p (insn))
6067	{
6068	/* We should generate bookkeeping code only if we are not at the
6069	top level of the move_op. */
6070	if (sel_num_cfg_preds_gt_1 (insn))
6071	book_block = generate_bookkeeping_insn (c_expr: sparams->c_expr,
6072	e1: lparams->e1, e2: lparams->e2);
6073	/* Update data sets for the current insn. */
6074	update_data_sets (insn);
6075	}
6076
6077	/* If bookkeeping code was inserted, we need to update av sets of basic
6078	block that received bookkeeping. After generation of bookkeeping insn,
6079	bookkeeping block does not contain valid av set because we are not following
6080	the original algorithm in every detail with regards to e.g. renaming
6081	simple reg-reg copies. Consider example:
6082
6083	bookkeeping block scheduling fence
6084	\ /
6085	\ join /
6086	----------
6087	| |
6088	----------
6089	/ \
6090	/ \
6091	r1 := r2 r1 := r3
6092
6093	We try to schedule insn "r1 := r3" on the current
6094	scheduling fence. Also, note that av set of bookkeeping block
6095	contain both insns "r1 := r2" and "r1 := r3". When the insn has
6096	been scheduled, the CFG is as follows:
6097
6098	r1 := r3 r1 := r3
6099	bookkeeping block scheduling fence
6100	\ /
6101	\ join /
6102	----------
6103	| |
6104	----------
6105	/ \
6106	/ \
6107	r1 := r2
6108
6109	Here, insn "r1 := r3" was scheduled at the current scheduling point
6110	and bookkeeping code was generated at the bookeeping block. This
6111	way insn "r1 := r2" is no longer available as a whole instruction
6112	(but only as expr) ahead of insn "r1 := r3" in bookkeeping block.
6113	This situation is handled by calling update_data_sets.
6114
6115	Since update_data_sets is called only on the bookkeeping block, and
6116	it also may have predecessors with av_sets, containing instructions that
6117	are no longer available, we save all such expressions that become
6118	unavailable during data sets update on the bookkeeping block in
6119	VEC_BOOKKEEPING_BLOCKED_VINSNS. Later we avoid selecting such
6120	expressions for scheduling. This allows us to avoid recomputation of
6121	av_sets outside the code motion path. */
6122
6123	if (book_block)
6124	update_and_record_unavailable_insns (book_block);
6125
6126	/* If INSN was previously marked for deletion, it's time to do it. */
6127	if (lparams->removed_last_insn)
6128	insn = PREV_INSN (insn);
6129
6130	/* Do not tidy control flow at the topmost moveop, as we can erroneously
6131	kill a block with a single nop in which the insn should be emitted. */
6132	if (lparams->e1)
6133	tidy_control_flow (BLOCK_FOR_INSN (insn), true);
6134	}
6135
6136	/* This function is called on the ascending pass, before returning from the
6137	current basic block. */
6138	static void
6139	fur_at_first_insn (insn_t insn,
6140	cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
6141	void *static_params ATTRIBUTE_UNUSED)
6142	{
6143	gcc_assert (!sel_bb_head_p (insn) || AV_SET_VALID_P (insn)
6144	|| AV_LEVEL (insn) == -1);
6145	}
6146
6147	/* Called on the backward stage of recursion to call moveup_expr for insn
6148	and sparams->c_expr. */
6149	static void
6150	move_op_ascend (insn_t insn, void *static_params)
6151	{
6152	enum MOVEUP_EXPR_CODE res;
6153	moveop_static_params_p sparams = (moveop_static_params_p) static_params;
6154
6155	if (! INSN_NOP_P (insn))
6156	{
6157	res = moveup_expr_cached (expr: sparams->c_expr, insn, inside_insn_group: false);
6158	gcc_assert (res != MOVEUP_EXPR_NULL);
6159	}
6160
6161	/* Update liveness for this insn as it was invalidated. */
6162	update_liveness_on_insn (insn);
6163	}
6164
6165	/* This function is called on enter to the basic block.
6166	Returns TRUE if this block already have been visited and
6167	code_motion_path_driver should return 1, FALSE otherwise. */
6168	static int
6169	fur_on_enter (insn_t insn ATTRIBUTE_UNUSED, cmpd_local_params_p local_params,
6170	void *static_params, bool visited_p)
6171	{
6172	fur_static_params_p sparams = (fur_static_params_p) static_params;
6173
6174	if (visited_p)
6175	{
6176	/* If we have found something below this block, there should be at
6177	least one insn in ORIGINAL_INSNS. */
6178	gcc_assert (*sparams->original_insns);
6179
6180	/* Adjust CROSSED_CALL_ABIS, since we may have come to this block along
6181	different path. */
6182	DEF_LIST_DEF (*sparams->original_insns)->crossed_call_abis
6183	|= sparams->crossed_call_abis;
6184	}
6185	else
6186	local_params->old_original_insns = *sparams->original_insns;
6187
6188	return 1;
6189	}
6190
6191	/* Same as above but for move_op. */
6192	static int
6193	move_op_on_enter (insn_t insn ATTRIBUTE_UNUSED,
6194	cmpd_local_params_p local_params ATTRIBUTE_UNUSED,
6195	void *static_params ATTRIBUTE_UNUSED, bool visited_p)
6196	{
6197	if (visited_p)
6198	return -1;
6199	return 1;
6200	}
6201
6202	/* This function is called while descending current basic block if current
6203	insn is not the original EXPR we're searching for.
6204
6205	Return value: FALSE, if code_motion_path_driver should perform a local
6206	cleanup and return 0 itself;
6207	TRUE, if code_motion_path_driver should continue. */
6208	static bool
6209	move_op_orig_expr_not_found (insn_t insn, av_set_t orig_ops ATTRIBUTE_UNUSED,
6210	void *static_params)
6211	{
6212	moveop_static_params_p sparams = (moveop_static_params_p) static_params;
6213
6214	sparams->failed_insn = insn;
6215
6216	/* If we're scheduling separate expr, in order to generate correct code
6217	we need to stop the search at bookkeeping code generated with the
6218	same destination register or memory. */
6219	if (lhs_of_insn_equals_to_dest_p (insn, sparams->dest))
6220	return false;
6221	return true;
6222	}
6223
6224	/* This function is called while descending current basic block if current
6225	insn is not the original EXPR we're searching for.
6226
6227	Return value: TRUE (code_motion_path_driver should continue). */
6228	static bool
6229	fur_orig_expr_not_found (insn_t insn, av_set_t orig_ops, void *static_params)
6230	{
6231	bool mutexed;
6232	expr_t r;
6233	av_set_iterator avi;
6234	fur_static_params_p sparams = (fur_static_params_p) static_params;
6235
6236	if (CALL_P (insn))
6237	sparams->crossed_call_abis |= 1 << insn_callee_abi (insn).id ();
6238	else if (DEBUG_INSN_P (insn))
6239	return true;
6240
6241	/* If current insn we are looking at cannot be executed together
6242	with original insn, then we can skip it safely.
6243
6244	Example: ORIG_OPS = { (p6) r14 = sign_extend (r15); }
6245	INSN = (!p6) r14 = r14 + 1;
6246
6247	Here we can schedule ORIG_OP with lhs = r14, though only
6248	looking at the set of used and set registers of INSN we must
6249	forbid it. So, add set/used in INSN registers to the
6250	untouchable set only if there is an insn in ORIG_OPS that can
6251	affect INSN. */
6252	mutexed = true;
6253	FOR_EACH_EXPR (r, avi, orig_ops)
6254	if (!sched_insns_conditions_mutex_p (insn, EXPR_INSN_RTX (r)))
6255	{
6256	mutexed = false;
6257	break;
6258	}
6259
6260	/* Mark all registers that do not meet the following condition:
6261	(1) Not set or read on any path from xi to an instance of the
6262	original operation. */
6263	if (!mutexed)
6264	{
6265	IOR_REG_SET (sparams->used_regs, INSN_REG_SETS (insn));
6266	IOR_REG_SET (sparams->used_regs, INSN_REG_USES (insn));
6267	IOR_REG_SET (sparams->used_regs, INSN_REG_CLOBBERS (insn));
6268	}
6269
6270	return true;
6271	}
6272
6273	/* Hooks and data to perform move_op operations with code_motion_path_driver. */
6274	struct code_motion_path_driver_info_def move_op_hooks = {
6275	.on_enter: move_op_on_enter,
6276	.orig_expr_found: move_op_orig_expr_found,
6277	.orig_expr_not_found: move_op_orig_expr_not_found,
6278	.merge_succs: move_op_merge_succs,
6279	.after_merge_succs: move_op_after_merge_succs,
6280	.ascend: move_op_ascend,
6281	.at_first_insn: move_op_at_first_insn,
6282	SUCCS_NORMAL,
6283	.routine_name: "move_op"
6284	};
6285
6286	/* Hooks and data to perform find_used_regs operations
6287	with code_motion_path_driver. */
6288	struct code_motion_path_driver_info_def fur_hooks = {
6289	.on_enter: fur_on_enter,
6290	.orig_expr_found: fur_orig_expr_found,
6291	.orig_expr_not_found: fur_orig_expr_not_found,
6292	.merge_succs: fur_merge_succs,
6293	NULL, /* fur_after_merge_succs */
6294	NULL, /* fur_ascend */
6295	.at_first_insn: fur_at_first_insn,
6296	SUCCS_ALL,
6297	.routine_name: "find_used_regs"
6298	};
6299
6300	/* Traverse all successors of INSN. For each successor that is SUCCS_NORMAL
6301	code_motion_path_driver is called recursively. Original operation
6302	was found at least on one path that is starting with one of INSN's
6303	successors (this fact is asserted). ORIG_OPS is expressions we're looking
6304	for, PATH is the path we've traversed, STATIC_PARAMS is the parameters
6305	of either move_op or find_used_regs depending on the caller.
6306
6307	Return 0 if we haven't found expression, 1 if we found it, -1 if we don't
6308	know for sure at this point. */
6309	static int
6310	code_motion_process_successors (insn_t insn, av_set_t orig_ops,
6311	ilist_t path, void *static_params)
6312	{
6313	int res = 0;
6314	succ_iterator succ_i;
6315	insn_t succ;
6316	basic_block bb;
6317	int old_index;
6318	unsigned old_succs;
6319
6320	struct cmpd_local_params lparams;
6321	expr_def _x;
6322
6323	lparams.c_expr_local = &_x;
6324	lparams.c_expr_merged = NULL;
6325
6326	/* We need to process only NORMAL succs for move_op, and collect live
6327	registers from ALL branches (including those leading out of the
6328	region) for find_used_regs.
6329
6330	In move_op, there can be a case when insn's bb number has changed
6331	due to created bookkeeping. This happens very rare, as we need to
6332	move expression from the beginning to the end of the same block.
6333	Rescan successors in this case. */
6334
6335	rescan:
6336	bb = BLOCK_FOR_INSN (insn);
6337	old_index = bb->index;
6338	old_succs = EDGE_COUNT (bb->succs);
6339
6340	FOR_EACH_SUCC_1 (succ, succ_i, insn, code_motion_path_driver_info->succ_flags)
6341	{
6342	int b;
6343
6344	lparams.e1 = succ_i.e1;
6345	lparams.e2 = succ_i.e2;
6346
6347	/* Go deep into recursion only for NORMAL edges (non-backedges within the
6348	current region). */
6349	if (succ_i.current_flags == SUCCS_NORMAL)
6350	b = code_motion_path_driver (succ, orig_ops, path, &lparams,
6351	static_params);
6352	else
6353	b = 0;
6354
6355	/* Merge c_expres found or unify live register sets from different
6356	successors. */
6357	code_motion_path_driver_info->merge_succs (insn, succ, b, &lparams,
6358	static_params);
6359	if (b == 1)
6360	res = b;
6361	else if (b == -1 && res != 1)
6362	res = b;
6363
6364	/* We have simplified the control flow below this point. In this case,
6365	the iterator becomes invalid. We need to try again.
6366	If we have removed the insn itself, it could be only an
6367	unconditional jump. Thus, do not rescan but break immediately --
6368	we have already visited the only successor block. */
6369	if (!BLOCK_FOR_INSN (insn))
6370	{
6371	if (sched_verbose >= 6)
6372	sel_print (fmt: "Not doing rescan: already visited the only successor"
6373	" of block %d\n", old_index);
6374	break;
6375	}
6376	if (BLOCK_FOR_INSN (insn)->index != old_index
6377	|| EDGE_COUNT (bb->succs) != old_succs)
6378	{
6379	if (sched_verbose >= 6)
6380	sel_print (fmt: "Rescan: CFG was simplified below insn %d, block %d\n",
6381	INSN_UID (insn), BLOCK_FOR_INSN (insn)->index);
6382	insn = sel_bb_end (BLOCK_FOR_INSN (insn));
6383	goto rescan;
6384	}
6385	}
6386
6387	/* Here, RES==1 if original expr was found at least for one of the
6388	successors. After the loop, RES may happen to have zero value
6389	only if at some point the expr searched is present in av_set, but is
6390	not found below. In most cases, this situation is an error.
6391	The exception is when the original operation is blocked by
6392	bookkeeping generated for another fence or for another path in current
6393	move_op. */
6394	gcc_checking_assert (res == 1
6395	|| (res == 0
6396	&& av_set_could_be_blocked_by_bookkeeping_p (orig_ops, static_params))
6397	|| res == -1);
6398
6399	/* Merge data, clean up, etc. */
6400	if (res != -1 && code_motion_path_driver_info->after_merge_succs)
6401	code_motion_path_driver_info->after_merge_succs (&lparams, static_params);
6402
6403	return res;
6404	}
6405
6406
6407	/* Perform a cleanup when the driver is about to terminate. ORIG_OPS_P
6408	is the pointer to the av set with expressions we were looking for,
6409	PATH_P is the pointer to the traversed path. */
6410	static inline void
6411	code_motion_path_driver_cleanup (av_set_t *orig_ops_p, ilist_t *path_p)
6412	{
6413	ilist_remove (path_p);
6414	av_set_clear (orig_ops_p);
6415	}
6416
6417	/* The driver function that implements move_op or find_used_regs
6418	functionality dependent whether code_motion_path_driver_INFO is set to
6419	&MOVE_OP_HOOKS or &FUR_HOOKS. This function implements the common parts
6420	of code (CFG traversal etc) that are shared among both functions. INSN
6421	is the insn we're starting the search from, ORIG_OPS are the expressions
6422	we're searching for, PATH is traversed path, LOCAL_PARAMS_IN are local
6423	parameters of the driver, and STATIC_PARAMS are static parameters of
6424	the caller.
6425
6426	Returns whether original instructions were found. Note that top-level
6427	code_motion_path_driver always returns true. */
6428	static int
6429	code_motion_path_driver (insn_t insn, av_set_t orig_ops, ilist_t path,
6430	cmpd_local_params_p local_params_in,
6431	void *static_params)
6432	{
6433	expr_t expr = NULL;
6434	basic_block bb = BLOCK_FOR_INSN (insn);
6435	insn_t first_insn, original_insn, bb_tail, before_first;
6436	bool removed_last_insn = false;
6437
6438	if (sched_verbose >= 6)
6439	{
6440	sel_print (fmt: "%s (", code_motion_path_driver_info->routine_name);
6441	dump_insn (insn);
6442	sel_print (fmt: ",");
6443	dump_av_set (orig_ops);
6444	sel_print (fmt: ")\n");
6445	}
6446
6447	gcc_assert (orig_ops);
6448
6449	/* If no original operations exist below this insn, return immediately. */
6450	if (is_ineligible_successor (insn, p: path))
6451	{
6452	if (sched_verbose >= 6)
6453	sel_print (fmt: "Insn %d is ineligible successor\n", INSN_UID (insn));
6454	return false;
6455	}
6456
6457	/* The block can have invalid av set, in which case it was created earlier
6458	during move_op. Return immediately. */
6459	if (sel_bb_head_p (insn))
6460	{
6461	if (! AV_SET_VALID_P (insn))
6462	{
6463	if (sched_verbose >= 6)
6464	sel_print (fmt: "Returned from block %d as it had invalid av set\n",
6465	bb->index);
6466	return false;
6467	}
6468
6469	if (bitmap_bit_p (code_motion_visited_blocks, bb->index))
6470	{
6471	/* We have already found an original operation on this branch, do not
6472	go any further and just return TRUE here. If we don't stop here,
6473	function can have exponential behavior even on the small code
6474	with many different paths (e.g. with data speculation and
6475	recovery blocks). */
6476	if (sched_verbose >= 6)
6477	sel_print (fmt: "Block %d already visited in this traversal\n", bb->index);
6478	if (code_motion_path_driver_info->on_enter)
6479	return code_motion_path_driver_info->on_enter (insn,
6480	local_params_in,
6481	static_params,
6482	true);
6483	}
6484	}
6485
6486	if (code_motion_path_driver_info->on_enter)
6487	code_motion_path_driver_info->on_enter (insn, local_params_in,
6488	static_params, false);
6489	orig_ops = av_set_copy (orig_ops);
6490
6491	/* Filter the orig_ops set. */
6492	if (AV_SET_VALID_P (insn))
6493	av_set_code_motion_filter (&orig_ops, AV_SET (insn));
6494
6495	/* If no more original ops, return immediately. */
6496	if (!orig_ops)
6497	{
6498	if (sched_verbose >= 6)
6499	sel_print (fmt: "No intersection with av set of block %d\n", bb->index);
6500	return false;
6501	}
6502
6503	/* For non-speculative insns we have to leave only one form of the
6504	original operation, because if we don't, we may end up with
6505	different C_EXPRes and, consequently, with bookkeepings for different
6506	expression forms along the same code motion path. That may lead to
6507	generation of incorrect code. So for each code motion we stick to
6508	the single form of the instruction, except for speculative insns
6509	which we need to keep in different forms with all speculation
6510	types. */
6511	av_set_leave_one_nonspec (&orig_ops);
6512
6513	/* It is not possible that all ORIG_OPS are filtered out. */
6514	gcc_assert (orig_ops);
6515
6516	/* It is enough to place only heads and tails of visited basic blocks into
6517	the PATH. */
6518	ilist_add (lp: &path, insn);
6519	first_insn = original_insn = insn;
6520	bb_tail = sel_bb_end (bb);
6521
6522	/* Descend the basic block in search of the original expr; this part
6523	corresponds to the part of the original move_op procedure executed
6524	before the recursive call. */
6525	for (;;)
6526	{
6527	/* Look at the insn and decide if it could be an ancestor of currently
6528	scheduling operation. If it is so, then the insn "dest = op" could
6529	either be replaced with "dest = reg", because REG now holds the result
6530	of OP, or just removed, if we've scheduled the insn as a whole.
6531
6532	If this insn doesn't contain currently scheduling OP, then proceed
6533	with searching and look at its successors. Operations we're searching
6534	for could have changed when moving up through this insn via
6535	substituting. In this case, perform unsubstitution on them first.
6536
6537	When traversing the DAG below this insn is finished, insert
6538	bookkeeping code, if the insn is a joint point, and remove
6539	leftovers. */
6540
6541	expr = av_set_lookup (orig_ops, INSN_VINSN (insn));
6542	if (expr)
6543	{
6544	insn_t last_insn = PREV_INSN (insn);
6545
6546	/* We have found the original operation. */
6547	if (sched_verbose >= 6)
6548	sel_print (fmt: "Found original operation at insn %d\n", INSN_UID (insn));
6549
6550	code_motion_path_driver_info->orig_expr_found
6551	(insn, expr, local_params_in, static_params);
6552
6553	/* Step back, so on the way back we'll start traversing from the
6554	previous insn (or we'll see that it's bb_note and skip that
6555	loop). */
6556	if (insn == first_insn)
6557	{
6558	first_insn = NEXT_INSN (insn: last_insn);
6559	removed_last_insn = sel_bb_end_p (last_insn);
6560	}
6561	insn = last_insn;
6562	break;
6563	}
6564	else
6565	{
6566	/* We haven't found the original expr, continue descending the basic
6567	block. */
6568	if (code_motion_path_driver_info->orig_expr_not_found
6569	(insn, orig_ops, static_params))
6570	{
6571	/* Av set ops could have been changed when moving through this
6572	insn. To find them below it, we have to un-substitute them. */
6573	undo_transformations (av_ptr: &orig_ops, insn);
6574	}
6575	else
6576	{
6577	/* Clean up and return, if the hook tells us to do so. It may
6578	happen if we've encountered the previously created
6579	bookkeeping. */
6580	code_motion_path_driver_cleanup (orig_ops_p: &orig_ops, path_p: &path);
6581	return -1;
6582	}
6583
6584	gcc_assert (orig_ops);
6585	}
6586
6587	/* Stop at insn if we got to the end of BB. */
6588	if (insn == bb_tail)
6589	break;
6590
6591	insn = NEXT_INSN (insn);
6592	}
6593
6594	/* Here INSN either points to the insn before the original insn (may be
6595	bb_note, if original insn was a bb_head) or to the bb_end. */
6596	if (!expr)
6597	{
6598	int res;
6599	rtx_insn *last_insn = PREV_INSN (insn);
6600	bool added_to_path;
6601
6602	gcc_assert (insn == sel_bb_end (bb));
6603
6604	/* Add bb tail to PATH (but it doesn't make any sense if it's a bb_head -
6605	it's already in PATH then). */
6606	if (insn != first_insn)
6607	{
6608	ilist_add (lp: &path, insn);
6609	added_to_path = true;
6610	}
6611	else
6612	added_to_path = false;
6613
6614	/* Process_successors should be able to find at least one
6615	successor for which code_motion_path_driver returns TRUE. */
6616	res = code_motion_process_successors (insn, orig_ops,
6617	path, static_params);
6618
6619	/* Jump in the end of basic block could have been removed or replaced
6620	during code_motion_process_successors, so recompute insn as the
6621	last insn in bb. */
6622	if (NEXT_INSN (insn: last_insn) != insn)
6623	{
6624	insn = sel_bb_end (bb);
6625	first_insn = sel_bb_head (bb);
6626	if (first_insn != original_insn)
6627	first_insn = original_insn;
6628	}
6629
6630	/* Remove bb tail from path. */
6631	if (added_to_path)
6632	ilist_remove (&path);
6633
6634	if (res != 1)
6635	{
6636	/* This is the case when one of the original expr is no longer available
6637	due to bookkeeping created on this branch with the same register.
6638	In the original algorithm, which doesn't have update_data_sets call
6639	on a bookkeeping block, it would simply result in returning
6640	FALSE when we've encountered a previously generated bookkeeping
6641	insn in moveop_orig_expr_not_found. */
6642	code_motion_path_driver_cleanup (orig_ops_p: &orig_ops, path_p: &path);
6643	return res;
6644	}
6645	}
6646
6647	/* Don't need it any more. */
6648	av_set_clear (&orig_ops);
6649
6650	/* Backward pass: now, when we have C_EXPR computed, we'll drag it to
6651	the beginning of the basic block. */
6652	before_first = PREV_INSN (insn: first_insn);
6653	while (insn != before_first)
6654	{
6655	if (code_motion_path_driver_info->ascend)
6656	code_motion_path_driver_info->ascend (insn, static_params);
6657
6658	insn = PREV_INSN (insn);
6659	}
6660
6661	/* Now we're at the bb head. */
6662	insn = first_insn;
6663	ilist_remove (&path);
6664	local_params_in->removed_last_insn = removed_last_insn;
6665	code_motion_path_driver_info->at_first_insn (insn, local_params_in, static_params);
6666
6667	/* This should be the very last operation as at bb head we could change
6668	the numbering by creating bookkeeping blocks. */
6669	if (removed_last_insn)
6670	insn = PREV_INSN (insn);
6671
6672	/* If we have simplified the control flow and removed the first jump insn,
6673	there's no point in marking this block in the visited blocks bitmap. */
6674	if (BLOCK_FOR_INSN (insn))
6675	bitmap_set_bit (code_motion_visited_blocks, BLOCK_FOR_INSN (insn)->index);
6676	return true;
6677	}
6678
6679	/* Move up the operations from ORIG_OPS set traversing the dag starting
6680	from INSN. PATH represents the edges traversed so far.
6681	DEST is the register chosen for scheduling the current expr. Insert
6682	bookkeeping code in the join points. EXPR_VLIW is the chosen expression,
6683	C_EXPR is how it looks like at the given cfg point.
6684	Set *SHOULD_MOVE to indicate whether we have only disconnected
6685	one of the insns found.
6686
6687	Returns whether original instructions were found, which is asserted
6688	to be true in the caller. */
6689	static bool
6690	move_op (insn_t insn, av_set_t orig_ops, expr_t expr_vliw,
6691	rtx dest, expr_t c_expr, bool *should_move)
6692	{
6693	struct moveop_static_params sparams;
6694	struct cmpd_local_params lparams;
6695	int res;
6696
6697	/* Init params for code_motion_path_driver. */
6698	sparams.dest = dest;
6699	sparams.c_expr = c_expr;
6700	sparams.uid = INSN_UID (EXPR_INSN_RTX (expr_vliw));
6701	sparams.failed_insn = NULL;
6702	sparams.was_renamed = false;
6703	lparams.e1 = NULL;
6704
6705	/* We haven't visited any blocks yet. */
6706	bitmap_clear (code_motion_visited_blocks);
6707
6708	/* Set appropriate hooks and data. */
6709	code_motion_path_driver_info = &move_op_hooks;
6710	res = code_motion_path_driver (insn, orig_ops, NULL, local_params_in: &lparams, static_params: &sparams);
6711
6712	gcc_assert (res != -1);
6713
6714	if (sparams.was_renamed)
6715	EXPR_WAS_RENAMED (expr_vliw) = true;
6716
6717	*should_move = (sparams.uid == -1);
6718
6719	return res;
6720	}
6721
6722
6723	/* Functions that work with regions. */
6724
6725	/* Current number of seqno used in init_seqno and init_seqno_1. */
6726	static int cur_seqno;
6727
6728	/* A helper for init_seqno. Traverse the region starting from BB and
6729	compute seqnos for visited insns, marking visited bbs in VISITED_BBS.
6730	Clear visited blocks from BLOCKS_TO_RESCHEDULE. */
6731	static void
6732	init_seqno_1 (basic_block bb, sbitmap visited_bbs, bitmap blocks_to_reschedule)
6733	{
6734	int bbi = BLOCK_TO_BB (bb->index);
6735	insn_t insn;
6736	insn_t succ_insn;
6737	succ_iterator si;
6738
6739	rtx_note *note = bb_note (bb);
6740	bitmap_set_bit (map: visited_bbs, bitno: bbi);
6741	if (blocks_to_reschedule)
6742	bitmap_clear_bit (blocks_to_reschedule, bb->index);
6743
6744	FOR_EACH_SUCC_1 (succ_insn, si, BB_END (bb),
6745	SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
6746	{
6747	basic_block succ = BLOCK_FOR_INSN (insn: succ_insn);
6748	int succ_bbi = BLOCK_TO_BB (succ->index);
6749
6750	gcc_assert (in_current_region_p (succ));
6751
6752	if (!bitmap_bit_p (map: visited_bbs, bitno: succ_bbi))
6753	{
6754	gcc_assert (succ_bbi > bbi);
6755
6756	init_seqno_1 (bb: succ, visited_bbs, blocks_to_reschedule);
6757	}
6758	else if (blocks_to_reschedule)
6759	bitmap_set_bit (forced_ebb_heads, succ->index);
6760	}
6761
6762	for (insn = BB_END (bb); insn != note; insn = PREV_INSN (insn))
6763	INSN_SEQNO (insn) = cur_seqno--;
6764	}
6765
6766	/* Initialize seqnos for the current region. BLOCKS_TO_RESCHEDULE contains
6767	blocks on which we're rescheduling when pipelining, FROM is the block where
6768	traversing region begins (it may not be the head of the region when
6769	pipelining, but the head of the loop instead).
6770
6771	Returns the maximal seqno found. */
6772	static int
6773	init_seqno (bitmap blocks_to_reschedule, basic_block from)
6774	{
6775	bitmap_iterator bi;
6776	unsigned bbi;
6777
6778	auto_sbitmap visited_bbs (current_nr_blocks);
6779
6780	if (blocks_to_reschedule)
6781	{
6782	bitmap_ones (visited_bbs);
6783	EXECUTE_IF_SET_IN_BITMAP (blocks_to_reschedule, 0, bbi, bi)
6784	{
6785	gcc_assert (BLOCK_TO_BB (bbi) < current_nr_blocks);
6786	bitmap_clear_bit (map: visited_bbs, BLOCK_TO_BB (bbi));
6787	}
6788	}
6789	else
6790	{
6791	bitmap_clear (visited_bbs);
6792	from = EBB_FIRST_BB (0);
6793	}
6794
6795	cur_seqno = sched_max_luid - 1;
6796	init_seqno_1 (bb: from, visited_bbs, blocks_to_reschedule);
6797
6798	/* cur_seqno may be positive if the number of instructions is less than
6799	sched_max_luid - 1 (when rescheduling or if some instructions have been
6800	removed by the call to purge_empty_blocks in sel_sched_region_1). */
6801	gcc_assert (cur_seqno >= 0);
6802
6803	return sched_max_luid - 1;
6804	}
6805
6806	/* Initialize scheduling parameters for current region. */
6807	static void
6808	sel_setup_region_sched_flags (void)
6809	{
6810	enable_schedule_as_rhs_p = 1;
6811	bookkeeping_p = 1;
6812	pipelining_p = (bookkeeping_p
6813	&& (flag_sel_sched_pipelining != 0)
6814	&& current_loop_nest != NULL
6815	&& loop_has_exit_edges (loop: current_loop_nest));
6816	max_insns_to_rename = param_selsched_insns_to_rename;
6817	max_ws = MAX_WS;
6818	}
6819
6820	/* Return true if all basic blocks of current region are empty. */
6821	static bool
6822	current_region_empty_p (void)
6823	{
6824	int i;
6825	for (i = 0; i < current_nr_blocks; i++)
6826	if (! sel_bb_empty_p (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (i))))
6827	return false;
6828
6829	return true;
6830	}
6831
6832	/* Prepare and verify loop nest for pipelining. */
6833	static void
6834	setup_current_loop_nest (int rgn, bb_vec_t *bbs)
6835	{
6836	current_loop_nest = get_loop_nest_for_rgn (rgn);
6837
6838	if (!current_loop_nest)
6839	return;
6840
6841	/* If this loop has any saved loop preheaders from nested loops,
6842	add these basic blocks to the current region. */
6843	sel_add_loop_preheaders (bbs);
6844
6845	/* Check that we're starting with a valid information. */
6846	gcc_assert (loop_latch_edge (current_loop_nest));
6847	gcc_assert (LOOP_MARKED_FOR_PIPELINING_P (current_loop_nest));
6848	}
6849
6850	/* Compute instruction priorities for current region. */
6851	static void
6852	sel_compute_priorities (int rgn)
6853	{
6854	sched_rgn_compute_dependencies (rgn);
6855
6856	/* Compute insn priorities in haifa style. Then free haifa style
6857	dependencies that we've calculated for this. */
6858	compute_priorities ();
6859
6860	if (sched_verbose >= 5)
6861	debug_rgn_dependencies (0);
6862
6863	free_rgn_deps ();
6864	}
6865
6866	/* Init scheduling data for RGN. Returns true when this region should not
6867	be scheduled. */
6868	static bool
6869	sel_region_init (int rgn)
6870	{
6871	int i;
6872	bb_vec_t bbs;
6873
6874	rgn_setup_region (rgn);
6875
6876	/* Even if sched_is_disabled_for_current_region_p() is true, we still
6877	do region initialization here so the region can be bundled correctly,
6878	but we'll skip the scheduling in sel_sched_region (). */
6879	if (current_region_empty_p ())
6880	return true;
6881
6882	bbs.create (nelems: current_nr_blocks);
6883
6884	for (i = 0; i < current_nr_blocks; i++)
6885	bbs.quick_push (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (i)));
6886
6887	sel_init_bbs (bbs);
6888
6889	if (flag_sel_sched_pipelining)
6890	setup_current_loop_nest (rgn, bbs: &bbs);
6891
6892	sel_setup_region_sched_flags ();
6893
6894	/* Initialize luids and dependence analysis which both sel-sched and haifa
6895	need. */
6896	sched_init_luids (bbs);
6897	sched_deps_init (false);
6898
6899	/* Initialize haifa data. */
6900	rgn_setup_sched_infos ();
6901	sel_set_sched_flags ();
6902	haifa_init_h_i_d (bbs);
6903
6904	sel_compute_priorities (rgn);
6905	init_deps_global ();
6906
6907	/* Main initialization. */
6908	sel_setup_sched_infos ();
6909	sel_init_global_and_expr (bbs);
6910
6911	bbs.release ();
6912
6913	blocks_to_reschedule = BITMAP_ALLOC (NULL);
6914
6915	/* Init correct liveness sets on each instruction of a single-block loop.
6916	This is the only situation when we can't update liveness when calling
6917	compute_live for the first insn of the loop. */
6918	if (current_loop_nest)
6919	{
6920	int header =
6921	(sel_is_loop_preheader_p (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (0)))
6922	? 1
6923	: 0);
6924
6925	if (current_nr_blocks == header + 1)
6926	update_liveness_on_insn
6927	(insn: sel_bb_head (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (header))));
6928	}
6929
6930	/* Set hooks so that no newly generated insn will go out unnoticed. */
6931	sel_register_cfg_hooks ();
6932
6933	/* !!! We call target.sched.init () for the whole region, but we invoke
6934	targetm.sched.finish () for every ebb. */
6935	if (targetm.sched.init)
6936	/* None of the arguments are actually used in any target. */
6937	targetm.sched.init (sched_dump, sched_verbose, -1);
6938
6939	first_emitted_uid = get_max_uid () + 1;
6940	preheader_removed = false;
6941
6942	/* Reset register allocation ticks array. */
6943	memset (s: reg_rename_tick, c: 0, n: sizeof reg_rename_tick);
6944	reg_rename_this_tick = 0;
6945
6946	forced_ebb_heads = BITMAP_ALLOC (NULL);
6947
6948	setup_nop_vinsn ();
6949	current_copies = BITMAP_ALLOC (NULL);
6950	current_originators = BITMAP_ALLOC (NULL);
6951	code_motion_visited_blocks = BITMAP_ALLOC (NULL);
6952
6953	return false;
6954	}
6955
6956	/* Simplify insns after the scheduling. */
6957	static void
6958	simplify_changed_insns (void)
6959	{
6960	int i;
6961
6962	for (i = 0; i < current_nr_blocks; i++)
6963	{
6964	basic_block bb = BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (i));
6965	rtx_insn *insn;
6966
6967	FOR_BB_INSNS (bb, insn)
6968	if (INSN_P (insn))
6969	{
6970	expr_t expr = INSN_EXPR (insn);
6971
6972	if (EXPR_WAS_SUBSTITUTED (expr))
6973	validate_simplify_insn (insn);
6974	}
6975	}
6976	}
6977
6978	/* Find boundaries of the EBB starting from basic block BB, marking blocks of
6979	this EBB in SCHEDULED_BLOCKS and appropriately filling in HEAD, TAIL,
6980	PREV_HEAD, and NEXT_TAIL fields of CURRENT_SCHED_INFO structure. */
6981	static void
6982	find_ebb_boundaries (basic_block bb, bitmap scheduled_blocks)
6983	{
6984	rtx_insn *head, *tail;
6985	basic_block bb1 = bb;
6986	if (sched_verbose >= 2)
6987	sel_print (fmt: "Finishing schedule in bbs: ");
6988
6989	do
6990	{
6991	bitmap_set_bit (scheduled_blocks, BLOCK_TO_BB (bb1->index));
6992
6993	if (sched_verbose >= 2)
6994	sel_print (fmt: "%d; ", bb1->index);
6995	}
6996	while (!bb_ends_ebb_p (bb1) && (bb1 = bb_next_bb (bb: bb1)));
6997
6998	if (sched_verbose >= 2)
6999	sel_print (fmt: "\n");
7000
7001	get_ebb_head_tail (bb, bb1, &head, &tail);
7002
7003	current_sched_info->head = head;
7004	current_sched_info->tail = tail;
7005	current_sched_info->prev_head = PREV_INSN (insn: head);
7006	current_sched_info->next_tail = NEXT_INSN (insn: tail);
7007	}
7008
7009	/* Regenerate INSN_SCHED_CYCLEs for insns of current EBB. */
7010	static void
7011	reset_sched_cycles_in_current_ebb (void)
7012	{
7013	int last_clock = 0;
7014	int haifa_last_clock = -1;
7015	int haifa_clock = 0;
7016	int issued_insns = 0;
7017	insn_t insn;
7018
7019	if (targetm.sched.init)
7020	{
7021	/* None of the arguments are actually used in any target.
7022	NB: We should have md_reset () hook for cases like this. */
7023	targetm.sched.init (sched_dump, sched_verbose, -1);
7024	}
7025
7026	state_reset (curr_state);
7027	advance_state (curr_state);
7028
7029	for (insn = current_sched_info->head;
7030	insn != current_sched_info->next_tail;
7031	insn = NEXT_INSN (insn))
7032	{
7033	int cost, haifa_cost;
7034	int sort_p;
7035	bool asm_p, real_insn, after_stall, all_issued;
7036	int clock;
7037
7038	if (!INSN_P (insn))
7039	continue;
7040
7041	asm_p = false;
7042	real_insn = recog_memoized (insn) >= 0;
7043	clock = INSN_SCHED_CYCLE (insn);
7044
7045	cost = clock - last_clock;
7046
7047	/* Initialize HAIFA_COST. */
7048	if (! real_insn)
7049	{
7050	asm_p = INSN_ASM_P (insn);
7051
7052	if (asm_p)
7053	/* This is asm insn which *had* to be scheduled first
7054	on the cycle. */
7055	haifa_cost = 1;
7056	else
7057	/* This is a use/clobber insn. It should not change
7058	cost. */
7059	haifa_cost = 0;
7060	}
7061	else
7062	haifa_cost = estimate_insn_cost (insn, state: curr_state);
7063
7064	/* Stall for whatever cycles we've stalled before. */
7065	after_stall = 0;
7066	if (INSN_AFTER_STALL_P (insn) && cost > haifa_cost)
7067	{
7068	haifa_cost = cost;
7069	after_stall = 1;
7070	}
7071	all_issued = issued_insns == issue_rate;
7072	if (haifa_cost == 0 && all_issued)
7073	haifa_cost = 1;
7074	if (haifa_cost > 0)
7075	{
7076	int i = 0;
7077
7078	while (haifa_cost--)
7079	{
7080	advance_state (curr_state);
7081	issued_insns = 0;
7082	i++;
7083
7084	if (sched_verbose >= 2)
7085	{
7086	sel_print (fmt: "advance_state (state_transition)\n");
7087	debug_state (state: curr_state);
7088	}
7089
7090	/* The DFA may report that e.g. insn requires 2 cycles to be
7091	issued, but on the next cycle it says that insn is ready
7092	to go. Check this here. */
7093	if (!after_stall
7094	&& real_insn
7095	&& haifa_cost > 0
7096	&& estimate_insn_cost (insn, state: curr_state) == 0)
7097	break;
7098
7099	/* When the data dependency stall is longer than the DFA stall,
7100	and when we have issued exactly issue_rate insns and stalled,
7101	it could be that after this longer stall the insn will again
7102	become unavailable to the DFA restrictions. Looks strange
7103	but happens e.g. on x86-64. So recheck DFA on the last
7104	iteration. */
7105	if ((after_stall || all_issued)
7106	&& real_insn
7107	&& haifa_cost == 0)
7108	haifa_cost = estimate_insn_cost (insn, state: curr_state);
7109	}
7110
7111	haifa_clock += i;
7112	if (sched_verbose >= 2)
7113	sel_print (fmt: "haifa clock: %d\n", haifa_clock);
7114	}
7115	else
7116	gcc_assert (haifa_cost == 0);
7117
7118	if (sched_verbose >= 2)
7119	sel_print (fmt: "Haifa cost for insn %d: %d\n", INSN_UID (insn), haifa_cost);
7120
7121	if (targetm.sched.dfa_new_cycle)
7122	while (targetm.sched.dfa_new_cycle (sched_dump, sched_verbose, insn,
7123	haifa_last_clock, haifa_clock,
7124	&sort_p))
7125	{
7126	advance_state (curr_state);
7127	issued_insns = 0;
7128	haifa_clock++;
7129	if (sched_verbose >= 2)
7130	{
7131	sel_print (fmt: "advance_state (dfa_new_cycle)\n");
7132	debug_state (state: curr_state);
7133	sel_print (fmt: "haifa clock: %d\n", haifa_clock + 1);
7134	}
7135	}
7136
7137	if (real_insn)
7138	{
7139	static state_t temp = NULL;
7140
7141	if (!temp)
7142	temp = xmalloc (dfa_state_size);
7143	memcpy (dest: temp, src: curr_state, n: dfa_state_size);
7144
7145	cost = state_transition (curr_state, insn);
7146	if (memcmp (s1: temp, s2: curr_state, n: dfa_state_size))
7147	issued_insns++;
7148
7149	if (sched_verbose >= 2)
7150	{
7151	sel_print (fmt: "scheduled insn %d, clock %d\n", INSN_UID (insn),
7152	haifa_clock + 1);
7153	debug_state (state: curr_state);
7154	}
7155	gcc_assert (cost < 0);
7156	}
7157
7158	if (targetm.sched.variable_issue)
7159	targetm.sched.variable_issue (sched_dump, sched_verbose, insn, 0);
7160
7161	INSN_SCHED_CYCLE (insn) = haifa_clock;
7162
7163	last_clock = clock;
7164	haifa_last_clock = haifa_clock;
7165	}
7166	}
7167
7168	/* Put TImode markers on insns starting a new issue group. */
7169	static void
7170	put_TImodes (void)
7171	{
7172	int last_clock = -1;
7173	insn_t insn;
7174
7175	for (insn = current_sched_info->head; insn != current_sched_info->next_tail;
7176	insn = NEXT_INSN (insn))
7177	{
7178	int cost, clock;
7179
7180	if (!INSN_P (insn))
7181	continue;
7182
7183	clock = INSN_SCHED_CYCLE (insn);
7184	cost = (last_clock == -1) ? 1 : clock - last_clock;
7185
7186	gcc_assert (cost >= 0);
7187
7188	if (issue_rate > 1
7189	&& GET_CODE (PATTERN (insn)) != USE
7190	&& GET_CODE (PATTERN (insn)) != CLOBBER)
7191	{
7192	if (reload_completed && cost > 0)
7193	PUT_MODE (x: insn, TImode);
7194
7195	last_clock = clock;
7196	}
7197
7198	if (sched_verbose >= 2)
7199	sel_print (fmt: "Cost for insn %d is %d\n", INSN_UID (insn), cost);
7200	}
7201	}
7202
7203	/* Perform MD_FINISH on EBBs comprising current region. When
7204	RESET_SCHED_CYCLES_P is true, run a pass emulating the scheduler
7205	to produce correct sched cycles on insns. */
7206	static void
7207	sel_region_target_finish (bool reset_sched_cycles_p)
7208	{
7209	int i;
7210	bitmap scheduled_blocks = BITMAP_ALLOC (NULL);
7211
7212	for (i = 0; i < current_nr_blocks; i++)
7213	{
7214	if (bitmap_bit_p (scheduled_blocks, i))
7215	continue;
7216
7217	/* While pipelining outer loops, skip bundling for loop
7218	preheaders. Those will be rescheduled in the outer loop. */
7219	if (sel_is_loop_preheader_p (EBB_FIRST_BB (i)))
7220	continue;
7221
7222	find_ebb_boundaries (EBB_FIRST_BB (i), scheduled_blocks);
7223
7224	if (no_real_insns_p (current_sched_info->head, current_sched_info->tail))
7225	continue;
7226
7227	if (reset_sched_cycles_p)
7228	reset_sched_cycles_in_current_ebb ();
7229
7230	if (targetm.sched.init)
7231	targetm.sched.init (sched_dump, sched_verbose, -1);
7232
7233	put_TImodes ();
7234
7235	if (targetm.sched.finish)
7236	{
7237	targetm.sched.finish (sched_dump, sched_verbose);
7238
7239	/* Extend luids so that insns generated by the target will
7240	get zero luid. */
7241	sched_extend_luids ();
7242	}
7243	}
7244
7245	BITMAP_FREE (scheduled_blocks);
7246	}
7247
7248	/* Free the scheduling data for the current region. When RESET_SCHED_CYCLES_P
7249	is true, make an additional pass emulating scheduler to get correct insn
7250	cycles for md_finish calls. */
7251	static void
7252	sel_region_finish (bool reset_sched_cycles_p)
7253	{
7254	simplify_changed_insns ();
7255	sched_finish_ready_list ();
7256	free_nop_pool ();
7257
7258	/* Free the vectors. */
7259	vec_av_set.release ();
7260	BITMAP_FREE (current_copies);
7261	BITMAP_FREE (current_originators);
7262	BITMAP_FREE (code_motion_visited_blocks);
7263	vinsn_vec_free (vinsn_vec&: vec_bookkeeping_blocked_vinsns);
7264	vinsn_vec_free (vinsn_vec&: vec_target_unavailable_vinsns);
7265
7266	/* If LV_SET of the region head should be updated, do it now because
7267	there will be no other chance. */
7268	{
7269	succ_iterator si;
7270	insn_t insn;
7271
7272	FOR_EACH_SUCC_1 (insn, si, bb_note (EBB_FIRST_BB (0)),
7273	SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
7274	{
7275	basic_block bb = BLOCK_FOR_INSN (insn);
7276
7277	if (!BB_LV_SET_VALID_P (bb))
7278	compute_live (insn);
7279	}
7280	}
7281
7282	/* Emulate the Haifa scheduler for bundling. */
7283	if (reload_completed)
7284	sel_region_target_finish (reset_sched_cycles_p);
7285
7286	sel_finish_global_and_expr ();
7287
7288	BITMAP_FREE (forced_ebb_heads);
7289
7290	free_nop_vinsn ();
7291
7292	finish_deps_global ();
7293	sched_finish_luids ();
7294	h_d_i_d.release ();
7295
7296	sel_finish_bbs ();
7297	BITMAP_FREE (blocks_to_reschedule);
7298
7299	sel_unregister_cfg_hooks ();
7300
7301	max_issue_size = 0;
7302	}
7303
7304
7305	/* Functions that implement the scheduler driver. */
7306
7307	/* Schedule a parallel instruction group on each of FENCES. MAX_SEQNO
7308	is the current maximum seqno. SCHEDULED_INSNS_TAILPP is the list
7309	of insns scheduled -- these would be postprocessed later. */
7310	static void
7311	schedule_on_fences (flist_t fences, int max_seqno,
7312	ilist_t **scheduled_insns_tailpp)
7313	{
7314	flist_t old_fences = fences;
7315
7316	if (sched_verbose >= 1)
7317	{
7318	sel_print (fmt: "\nScheduling on fences: ");
7319	dump_flist (fences);
7320	sel_print (fmt: "\n");
7321	}
7322
7323	scheduled_something_on_previous_fence = false;
7324	for (; fences; fences = FLIST_NEXT (fences))
7325	{
7326	fence_t fence = NULL;
7327	int seqno = 0;
7328	flist_t fences2;
7329	bool first_p = true;
7330
7331	/* Choose the next fence group to schedule.
7332	The fact that insn can be scheduled only once
7333	on the cycle is guaranteed by two properties:
7334	1. seqnos of parallel groups decrease with each iteration.
7335	2. If is_ineligible_successor () sees the larger seqno, it
7336	checks if candidate insn is_in_current_fence_p (). */
7337	for (fences2 = old_fences; fences2; fences2 = FLIST_NEXT (fences2))
7338	{
7339	fence_t f = FLIST_FENCE (fences2);
7340
7341	if (!FENCE_PROCESSED_P (f))
7342	{
7343	int i = INSN_SEQNO (FENCE_INSN (f));
7344
7345	if (first_p || i > seqno)
7346	{
7347	seqno = i;
7348	fence = f;
7349	first_p = false;
7350	}
7351	else
7352	/* ??? Seqnos of different groups should be different. */
7353	gcc_assert (1 || i != seqno);
7354	}
7355	}
7356
7357	gcc_assert (fence);
7358
7359	/* As FENCE is nonnull, SEQNO is initialized. */
7360	seqno -= max_seqno + 1;
7361	fill_insns (fence, seqno, scheduled_insns_tailpp);
7362	FENCE_PROCESSED_P (fence) = true;
7363	}
7364
7365	/* All av_sets are invalidated by GLOBAL_LEVEL increase, thus we
7366	don't need to keep bookkeeping-invalidated and target-unavailable
7367	vinsns any more. */
7368	vinsn_vec_clear (vinsn_vec: &vec_bookkeeping_blocked_vinsns);
7369	vinsn_vec_clear (vinsn_vec: &vec_target_unavailable_vinsns);
7370	}
7371
7372	/* Calculate MIN_SEQNO and MAX_SEQNO. */
7373	static void
7374	find_min_max_seqno (flist_t fences, int *min_seqno, int *max_seqno)
7375	{
7376	*min_seqno = *max_seqno = INSN_SEQNO (FENCE_INSN (FLIST_FENCE (fences)));
7377
7378	/* The first element is already processed. */
7379	while ((fences = FLIST_NEXT (fences)))
7380	{
7381	int seqno = INSN_SEQNO (FENCE_INSN (FLIST_FENCE (fences)));
7382
7383	if (*min_seqno > seqno)
7384	*min_seqno = seqno;
7385	else if (*max_seqno < seqno)
7386	*max_seqno = seqno;
7387	}
7388	}
7389
7390	/* Calculate new fences from FENCES. Write the current time to PTIME. */
7391	static flist_t
7392	calculate_new_fences (flist_t fences, int orig_max_seqno, int *ptime)
7393	{
7394	flist_t old_fences = fences;
7395	struct flist_tail_def _new_fences, *new_fences = &_new_fences;
7396	int max_time = 0;
7397
7398	flist_tail_init (new_fences);
7399	for (; fences; fences = FLIST_NEXT (fences))
7400	{
7401	fence_t fence = FLIST_FENCE (fences);
7402	insn_t insn;
7403
7404	if (!FENCE_BNDS (fence))
7405	{
7406	/* This fence doesn't have any successors. */
7407	if (!FENCE_SCHEDULED_P (fence))
7408	{
7409	/* Nothing was scheduled on this fence. */
7410	int seqno;
7411
7412	insn = FENCE_INSN (fence);
7413	seqno = INSN_SEQNO (insn);
7414	gcc_assert (seqno > 0 && seqno <= orig_max_seqno);
7415
7416	if (sched_verbose >= 1)
7417	sel_print (fmt: "Fence %d[%d] has not changed\n",
7418	INSN_UID (insn),
7419	BLOCK_NUM (insn));
7420	move_fence_to_fences (fences, new_fences);
7421	}
7422	}
7423	else
7424	extract_new_fences_from (old_fences: fences, new_fences, orig_max_seqno);
7425	max_time = MAX (max_time, FENCE_CYCLE (fence));
7426	}
7427
7428	flist_clear (&old_fences);
7429	*ptime = max_time;
7430	return FLIST_TAIL_HEAD (new_fences);
7431	}
7432
7433	/* Update seqnos of insns given by PSCHEDULED_INSNS. MIN_SEQNO and MAX_SEQNO
7434	are the miminum and maximum seqnos of the group, HIGHEST_SEQNO_IN_USE is
7435	the highest seqno used in a region. Return the updated highest seqno. */
7436	static int
7437	update_seqnos_and_stage (int min_seqno, int max_seqno,
7438	int highest_seqno_in_use,
7439	ilist_t *pscheduled_insns)
7440	{
7441	int new_hs;
7442	ilist_iterator ii;
7443	insn_t insn;
7444
7445	/* Actually, new_hs is the seqno of the instruction, that was
7446	scheduled first (i.e. it is the first one in SCHEDULED_INSNS). */
7447	if (*pscheduled_insns)
7448	{
7449	new_hs = (INSN_SEQNO (ILIST_INSN (*pscheduled_insns))
7450	+ highest_seqno_in_use + max_seqno - min_seqno + 2);
7451	gcc_assert (new_hs > highest_seqno_in_use);
7452	}
7453	else
7454	new_hs = highest_seqno_in_use;
7455
7456	FOR_EACH_INSN (insn, ii, *pscheduled_insns)
7457	{
7458	gcc_assert (INSN_SEQNO (insn) < 0);
7459	INSN_SEQNO (insn) += highest_seqno_in_use + max_seqno - min_seqno + 2;
7460	gcc_assert (INSN_SEQNO (insn) <= new_hs);
7461
7462	/* When not pipelining, purge unneeded insn info on the scheduled insns.
7463	For example, having reg_last array of INSN_DEPS_CONTEXT in memory may
7464	require > 1GB of memory e.g. on limit-fnargs.c. */
7465	if (! pipelining_p)
7466	free_data_for_scheduled_insn (insn);
7467	}
7468
7469	ilist_clear (pscheduled_insns);
7470	global_level++;
7471
7472	return new_hs;
7473	}
7474
7475	/* The main driver for scheduling a region. This function is responsible
7476	for correct propagation of fences (i.e. scheduling points) and creating
7477	a group of parallel insns at each of them. It also supports
7478	pipelining. ORIG_MAX_SEQNO is the maximal seqno before this pass
7479	of scheduling. */
7480	static void
7481	sel_sched_region_2 (int orig_max_seqno)
7482	{
7483	int highest_seqno_in_use = orig_max_seqno;
7484	int max_time = 0;
7485
7486	stat_bookkeeping_copies = 0;
7487	stat_insns_needed_bookkeeping = 0;
7488	stat_renamed_scheduled = 0;
7489	stat_substitutions_total = 0;
7490	num_insns_scheduled = 0;
7491
7492	while (fences)
7493	{
7494	int min_seqno, max_seqno;
7495	ilist_t scheduled_insns = NULL;
7496	ilist_t *scheduled_insns_tailp = &scheduled_insns;
7497
7498	find_min_max_seqno (fences, min_seqno: &min_seqno, max_seqno: &max_seqno);
7499	schedule_on_fences (fences, max_seqno, scheduled_insns_tailpp: &scheduled_insns_tailp);
7500	fences = calculate_new_fences (fences, orig_max_seqno, ptime: &max_time);
7501	highest_seqno_in_use = update_seqnos_and_stage (min_seqno, max_seqno,
7502	highest_seqno_in_use,
7503	pscheduled_insns: &scheduled_insns);
7504	}
7505
7506	if (sched_verbose >= 1)
7507	{
7508	sel_print (fmt: "Total scheduling time: %d cycles\n", max_time);
7509	sel_print (fmt: "Scheduled %d bookkeeping copies, %d insns needed "
7510	"bookkeeping, %d insns renamed, %d insns substituted\n",
7511	stat_bookkeeping_copies,
7512	stat_insns_needed_bookkeeping,
7513	stat_renamed_scheduled,
7514	stat_substitutions_total);
7515	}
7516	}
7517
7518	/* Schedule a region. When pipelining, search for possibly never scheduled
7519	bookkeeping code and schedule it. Reschedule pipelined code without
7520	pipelining after. */
7521	static void
7522	sel_sched_region_1 (void)
7523	{
7524	int orig_max_seqno;
7525
7526	/* Remove empty blocks that might be in the region from the beginning. */
7527	purge_empty_blocks ();
7528
7529	orig_max_seqno = init_seqno (NULL, NULL);
7530	gcc_assert (orig_max_seqno >= 1);
7531
7532	/* When pipelining outer loops, create fences on the loop header,
7533	not preheader. */
7534	fences = NULL;
7535	if (current_loop_nest)
7536	init_fences (BB_END (EBB_FIRST_BB (0)));
7537	else
7538	init_fences (bb_note (EBB_FIRST_BB (0)));
7539	global_level = 1;
7540
7541	sel_sched_region_2 (orig_max_seqno);
7542
7543	gcc_assert (fences == NULL);
7544
7545	if (pipelining_p)
7546	{
7547	int i;
7548	basic_block bb;
7549	struct flist_tail_def _new_fences;
7550	flist_tail_t new_fences = &_new_fences;
7551	bool do_p = true;
7552
7553	pipelining_p = false;
7554	max_ws = MIN (max_ws, issue_rate * 3 / 2);
7555	bookkeeping_p = false;
7556	enable_schedule_as_rhs_p = false;
7557
7558	/* Schedule newly created code, that has not been scheduled yet. */
7559	do_p = true;
7560
7561	while (do_p)
7562	{
7563	do_p = false;
7564
7565	for (i = 0; i < current_nr_blocks; i++)
7566	{
7567	basic_block bb = EBB_FIRST_BB (i);
7568
7569	if (bitmap_bit_p (blocks_to_reschedule, bb->index))
7570	{
7571	if (! bb_ends_ebb_p (bb))
7572	bitmap_set_bit (blocks_to_reschedule, bb_next_bb (bb)->index);
7573	if (sel_bb_empty_p (bb))
7574	{
7575	bitmap_clear_bit (blocks_to_reschedule, bb->index);
7576	continue;
7577	}
7578	clear_outdated_rtx_info (bb);
7579	if (sel_insn_is_speculation_check (BB_END (bb))
7580	&& JUMP_P (BB_END (bb)))
7581	bitmap_set_bit (blocks_to_reschedule,
7582	BRANCH_EDGE (bb)->dest->index);
7583	}
7584	else if (! sel_bb_empty_p (bb)
7585	&& INSN_SCHED_TIMES (sel_bb_head (bb)) <= 0)
7586	bitmap_set_bit (blocks_to_reschedule, bb->index);
7587	}
7588
7589	for (i = 0; i < current_nr_blocks; i++)
7590	{
7591	bb = EBB_FIRST_BB (i);
7592
7593	/* While pipelining outer loops, skip bundling for loop
7594	preheaders. Those will be rescheduled in the outer
7595	loop. */
7596	if (sel_is_loop_preheader_p (bb))
7597	{
7598	clear_outdated_rtx_info (bb);
7599	continue;
7600	}
7601
7602	if (bitmap_bit_p (blocks_to_reschedule, bb->index))
7603	{
7604	flist_tail_init (new_fences);
7605
7606	orig_max_seqno = init_seqno (blocks_to_reschedule, from: bb);
7607
7608	/* Mark BB as head of the new ebb. */
7609	bitmap_set_bit (forced_ebb_heads, bb->index);
7610
7611	gcc_assert (fences == NULL);
7612
7613	init_fences (bb_note (bb));
7614
7615	sel_sched_region_2 (orig_max_seqno);
7616
7617	do_p = true;
7618	break;
7619	}
7620	}
7621	}
7622	}
7623	}
7624
7625	/* Schedule the RGN region. */
7626	void
7627	sel_sched_region (int rgn)
7628	{
7629	bool schedule_p;
7630	bool reset_sched_cycles_p;
7631
7632	if (sel_region_init (rgn))
7633	return;
7634
7635	if (sched_verbose >= 1)
7636	sel_print (fmt: "Scheduling region %d\n", rgn);
7637
7638	schedule_p = (!sched_is_disabled_for_current_region_p ()
7639	&& dbg_cnt (index: sel_sched_region_cnt));
7640	reset_sched_cycles_p = pipelining_p;
7641	if (schedule_p)
7642	sel_sched_region_1 ();
7643	else
7644	{
7645	/* Schedule always selecting the next insn to make the correct data
7646	for bundling or other later passes. */
7647	pipelining_p = false;
7648	reset_sched_cycles_p = false;
7649	force_next_insn = 1;
7650	sel_sched_region_1 ();
7651	force_next_insn = 0;
7652	}
7653	sel_region_finish (reset_sched_cycles_p);
7654	}
7655
7656	/* Perform global init for the scheduler. */
7657	static void
7658	sel_global_init (void)
7659	{
7660	/* Remove empty blocks: their presence can break assumptions elsewhere,
7661	e.g. the logic to invoke update_liveness_on_insn in sel_region_init. */
7662	cleanup_cfg (0);
7663
7664	calculate_dominance_info (CDI_DOMINATORS);
7665	alloc_sched_pools ();
7666
7667	/* Setup the infos for sched_init. */
7668	sel_setup_sched_infos ();
7669	setup_sched_dump ();
7670
7671	sched_rgn_init (false);
7672	sched_init ();
7673
7674	sched_init_bbs ();
7675	/* Reset AFTER_RECOVERY if it has been set by the 1st scheduler pass. */
7676	after_recovery = 0;
7677	can_issue_more = issue_rate;
7678
7679	sched_extend_target ();
7680	sched_deps_init (true);
7681	setup_nop_and_exit_insns ();
7682	sel_extend_global_bb_info ();
7683	init_lv_sets ();
7684	init_hard_regs_data ();
7685	}
7686
7687	/* Free the global data of the scheduler. */
7688	static void
7689	sel_global_finish (void)
7690	{
7691	free_bb_note_pool ();
7692	free_lv_sets ();
7693	sel_finish_global_bb_info ();
7694
7695	free_regset_pool ();
7696	free_nop_and_exit_insns ();
7697
7698	sched_rgn_finish ();
7699	sched_deps_finish ();
7700	sched_finish ();
7701
7702	if (current_loops)
7703	sel_finish_pipelining ();
7704
7705	free_sched_pools ();
7706	free_dominance_info (CDI_DOMINATORS);
7707	}
7708
7709	/* Return true when we need to skip selective scheduling. Used for debugging. */
7710	bool
7711	maybe_skip_selective_scheduling (void)
7712	{
7713	return ! dbg_cnt (index: sel_sched_cnt);
7714	}
7715
7716	/* The entry point. */
7717	void
7718	run_selective_scheduling (void)
7719	{
7720	int rgn;
7721
7722	if (n_basic_blocks_for_fn (cfun) == NUM_FIXED_BLOCKS)
7723	return;
7724
7725	sel_global_init ();
7726
7727	for (rgn = 0; rgn < nr_regions; rgn++)
7728	sel_sched_region (rgn);
7729
7730	sel_global_finish ();
7731	}
7732
7733	#endif
7734