haifa-sched.cc source code [gcc/haifa-sched.cc]

1	/ Instruction scheduling pass.*
2	Copyright (C) 1992-2024 Free Software Foundation, Inc.
3	Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
4	and currently maintained by, Jim Wilson (wilson@cygnus.com)
5
6	This file is part of GCC.
7
8	GCC is free software; you can redistribute it and/or modify it under
9	the terms of the GNU General Public License as published by the Free
10	Software Foundation; either version 3, or (at your option) any later
11	version.
12
13	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14	WARRANTY; without even the implied warranty of MERCHANTABILITY or
15	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16	for more details.
17
18	You should have received a copy of the GNU General Public License
19	along with GCC; see the file COPYING3. If not see
20	<http://www.gnu.org/licenses/>. /*
21
22	/ Instruction scheduling pass. This file, along with sched-deps.cc,*
23	contains the generic parts. The actual entry point for
24	the normal instruction scheduling pass is found in sched-rgn.cc.
25
26	We compute insn priorities based on data dependencies. Flow
27	analysis only creates a fraction of the data-dependencies we must
28	observe: namely, only those dependencies which the combiner can be
29	expected to use. For this pass, we must therefore create the
30	remaining dependencies we need to observe: register dependencies,
31	memory dependencies, dependencies to keep function calls in order,
32	and the dependence between a conditional branch and the setting of
33	condition codes are all dealt with here.
34
35	The scheduler first traverses the data flow graph, starting with
36	the last instruction, and proceeding to the first, assigning values
37	to insn_priority as it goes. This sorts the instructions
38	topologically by data dependence.
39
40	Once priorities have been established, we order the insns using
41	list scheduling. This works as follows: starting with a list of
42	all the ready insns, and sorted according to priority number, we
43	schedule the insn from the end of the list by placing its
44	predecessors in the list according to their priority order. We
45	consider this insn scheduled by setting the pointer to the "end" of
46	the list to point to the previous insn. When an insn has no
47	predecessors, we either queue it until sufficient time has elapsed
48	or add it to the ready list. As the instructions are scheduled or
49	when stalls are introduced, the queue advances and dumps insns into
50	the ready list. When all insns down to the lowest priority have
51	been scheduled, the critical path of the basic block has been made
52	as short as possible. The remaining insns are then scheduled in
53	remaining slots.
54
55	The following list shows the order in which we want to break ties
56	among insns in the ready list:
57
58	1. choose insn with the longest path to end of bb, ties
59	broken by
60	2. choose insn with least contribution to register pressure,
61	ties broken by
62	3. prefer in-block upon interblock motion, ties broken by
63	4. prefer useful upon speculative motion, ties broken by
64	5. choose insn with largest control flow probability, ties
65	broken by
66	6. choose insn with the least dependences upon the previously
67	scheduled insn, or finally
68	7 choose the insn which has the most insns dependent on it.
69	8. choose insn with lowest UID.
70
71	Memory references complicate matters. Only if we can be certain
72	that memory references are not part of the data dependency graph
73	(via true, anti, or output dependence), can we move operations past
74	memory references. To first approximation, reads can be done
75	independently, while writes introduce dependencies. Better
76	approximations will yield fewer dependencies.
77
78	Before reload, an extended analysis of interblock data dependences
79	is required for interblock scheduling. This is performed in
80	compute_block_dependences ().
81
82	Dependencies set up by memory references are treated in exactly the
83	same way as other dependencies, by using insn backward dependences
84	INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences
85	INSN_FORW_DEPS for the purpose of forward list scheduling.
86
87	Having optimized the critical path, we may have also unduly
88	extended the lifetimes of some registers. If an operation requires
89	that constants be loaded into registers, it is certainly desirable
90	to load those constants as early as necessary, but no earlier.
91	I.e., it will not do to load up a bunch of registers at the
92	beginning of a basic block only to use them at the end, if they
93	could be loaded later, since this may result in excessive register
94	utilization.
95
96	Note that since branches are never in basic blocks, but only end
97	basic blocks, this pass will not move branches. But that is ok,
98	since we can use GNU's delayed branch scheduling pass to take care
99	of this case.
100
101	Also note that no further optimizations based on algebraic
102	identities are performed, so this pass would be a good one to
103	perform instruction splitting, such as breaking up a multiply
104	instruction into shifts and adds where that is profitable.
105
106	Given the memory aliasing analysis that this pass should perform,
107	it should be possible to remove redundant stores to memory, and to
108	load values from registers instead of hitting memory.
109
110	Before reload, speculative insns are moved only if a 'proof' exists
111	that no exception will be caused by this, and if no live registers
112	exist that inhibit the motion (live registers constraints are not
113	represented by data dependence edges).
114
115	This pass must update information that subsequent passes expect to
116	be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
117	reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END.
118
119	The information in the line number notes is carefully retained by
120	this pass. Notes that refer to the starting and ending of
121	exception regions are also carefully retained by this pass. All
122	other NOTE insns are grouped in their same relative order at the
123	beginning of basic blocks and regions that have been scheduled. /*
124
125	#include "config.h"
126	#include "system.h"
127	#include "coretypes.h"
128	#include "backend.h"
129	#include "target.h"
130	#include "rtl.h"
131	#include "cfghooks.h"
132	#include "df.h"
133	#include "memmodel.h"
134	#include "tm_p.h"
135	#include "insn-config.h"
136	#include "regs.h"
137	#include "ira.h"
138	#include "recog.h"
139	#include "insn-attr.h"
140	#include "cfgrtl.h"
141	#include "cfgbuild.h"
142	#include "sched-int.h"
143	#include "common/common-target.h"
144	#include "dbgcnt.h"
145	#include "cfgloop.h"
146	#include "dumpfile.h"
147	#include "print-rtl.h"
148	#include "function-abi.h"
149
150	#ifdef INSN_SCHEDULING
151
152	/ True if we do register pressure relief through live-range*
153	shrinkage. /*
154	static bool live_range_shrinkage_p;
155
156	/ Switch on live range shrinkage. /
157	void
158	initialize_live_range_shrinkage (void)
159	{
160	live_range_shrinkage_p = true;
161	}
162
163	/ Switch off live range shrinkage. /
164	void
165	finish_live_range_shrinkage (void)
166	{
167	live_range_shrinkage_p = false;
168	}
169
170	/ issue_rate is the number of insns that can be scheduled in the same*
171	machine cycle. It can be defined in the config/mach/mach.h file,
172	otherwise we set it to 1. /*
173
174	int issue_rate;
175
176	/ This can be set to true by a backend if the scheduler should not*
177	enable a DCE pass. /*
178	bool sched_no_dce;
179
180	/ The current initiation interval used when modulo scheduling. /
181	static int modulo_ii;
182
183	/ The maximum number of stages we are prepared to handle. /
184	static int modulo_max_stages;
185
186	/ The number of insns that exist in each iteration of the loop. We use this*
187	to detect when we've scheduled all insns from the first iteration. /*
188	static int modulo_n_insns;
189
190	/ The current count of insns in the first iteration of the loop that have*
191	already been scheduled. /*
192	static int modulo_insns_scheduled;
193
194	/ The maximum uid of insns from the first iteration of the loop. /
195	static int modulo_iter0_max_uid;
196
197	/ The number of times we should attempt to backtrack when modulo scheduling.*
198	Decreased each time we have to backtrack. /*
199	static int modulo_backtracks_left;
200
201	/ The stage in which the last insn from the original loop was*
202	scheduled. /*
203	static int modulo_last_stage;
204
205	/ sched-verbose controls the amount of debugging output the*
206	scheduler prints. It is controlled by -fsched-verbose=N:
207	N=0: no debugging output.
208	N=1: default value.
209	N=2: bb's probabilities, detailed ready list info, unit/insn info.
210	N=3: rtl at abort point, control-flow, regions info.
211	N=5: dependences info. /*
212	int sched_verbose = `0`;
213
214	/ Debugging file. All printouts are sent to dump. /
215	FILE *sched_dump = `0`;
216
217	/ This is a placeholder for the scheduler parameters common*
218	to all schedulers. /*
219	struct common_sched_info_def *common_sched_info;
220
221	#define INSN_TICK(INSN) (HID (INSN)->tick)
222	#define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
223	#define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
224	#define INTER_TICK(INSN) (HID (INSN)->inter_tick)
225	#define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
226	#define SHADOW_P(INSN) (HID (INSN)->shadow_p)
227	#define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
228	/ Cached cost of the instruction. Use insn_sched_cost to get cost of the*
229	insn. -1 here means that the field is not initialized. /*
230	#define INSN_COST(INSN) (HID (INSN)->cost)
231
232	/ If INSN_TICK of an instruction is equal to INVALID_TICK,*
233	then it should be recalculated from scratch. /*
234	#define INVALID_TICK (-(max_insn_queue_index + 1))
235	/ The minimal value of the INSN_TICK of an instruction. /
236	#define MIN_TICK (-max_insn_queue_index)
237
238	/ Original order of insns in the ready list.*
239	Used to keep order of normal insns while separating DEBUG_INSNs. /*
240	#define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
241
242	/ The deciding reason for INSN's place in the ready list. /
243	#define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
244
245	/ List of important notes we must keep around. This is a pointer to the*
246	last element in the list. /*
247	rtx_insn *note_list;
248
249	static struct spec_info_def spec_info_var;
250	/ Description of the speculative part of the scheduling.*
251	If NULL - no speculation. /*
252	spec_info_t spec_info = NULL;
253
254	/ True, if recovery block was added during scheduling of current block.*
255	Used to determine, if we need to fix INSN_TICKs. /*
256	static bool haifa_recovery_bb_recently_added_p;
257
258	/ True, if recovery block was added during this scheduling pass.*
259	Used to determine if we should have empty memory pools of dependencies
260	after finishing current region. /*
261	bool haifa_recovery_bb_ever_added_p;
262
263	/ Counters of different types of speculative instructions. /
264	static int nr_begin_data, nr_be_in_data, nr_begin_control, nr_be_in_control;
265
266	/ Array used in {unlink, restore}_bb_notes. /
267	static rtx_insn **bb_header = `0`;
268
269	/ Basic block after which recovery blocks will be created. /
270	static basic_block before_recovery;
271
272	/ Basic block just before the EXIT_BLOCK and after recovery, if we have*
273	created it. /*
274	basic_block after_recovery;
275
276	/ FALSE if we add bb to another region, so we don't need to initialize it. /
277	bool adding_bb_to_current_region_p = true;
278
279	/ Queues, etc. /
280
281	/ An instruction is ready to be scheduled when all insns preceding it*
282	have already been scheduled. It is important to ensure that all
283	insns which use its result will not be executed until its result
284	has been computed. An insn is maintained in one of four structures:
285
286	(P) the "Pending" set of insns which cannot be scheduled until
287	their dependencies have been satisfied.
288	(Q) the "Queued" set of insns that can be scheduled when sufficient
289	time has passed.
290	(R) the "Ready" list of unscheduled, uncommitted insns.
291	(S) the "Scheduled" list of insns.
292
293	Initially, all insns are either "Pending" or "Ready" depending on
294	whether their dependencies are satisfied.
295
296	Insns move from the "Ready" list to the "Scheduled" list as they
297	are committed to the schedule. As this occurs, the insns in the
298	"Pending" list have their dependencies satisfied and move to either
299	the "Ready" list or the "Queued" set depending on whether
300	sufficient time has passed to make them ready. As time passes,
301	insns move from the "Queued" set to the "Ready" list.
302
303	The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
304	unscheduled insns, i.e., those that are ready, queued, and pending.
305	The "Queued" set (Q) is implemented by the variable `insn_queue'.
306	The "Ready" list (R) is implemented by the variables `ready' and
307	`n_ready'.
308	The "Scheduled" list (S) is the new insn chain built by this pass.
309
310	The transition (R->S) is implemented in the scheduling loop in
311	`schedule_block' when the best insn to schedule is chosen.
312	The transitions (P->R and P->Q) are implemented in `schedule_insn' as
313	insns move from the ready list to the scheduled list.
314	The transition (Q->R) is implemented in 'queue_to_insn' as time
315	passes or stalls are introduced. /*
316
317	/ Implement a circular buffer to delay instructions until sufficient*
318	time has passed. For the new pipeline description interface,
319	MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
320	than maximal time of instruction execution computed by genattr.cc on
321	the base maximal time of functional unit reservations and getting a
322	result. This is the longest time an insn may be queued. /*
323
324	static rtx_insn_list **insn_queue;
325	static int q_ptr = `0`;
326	static int q_size = `0`;
327	#define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
328	#define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
329
330	#define QUEUE_SCHEDULED (-3)
331	#define QUEUE_NOWHERE (-2)
332	#define QUEUE_READY (-1)
333	/ QUEUE_SCHEDULED - INSN is scheduled.*
334	QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in
335	queue or ready list.
336	QUEUE_READY - INSN is in ready list.
337	N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. /*
338
339	#define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
340
341	/ The following variable value refers for all current and future*
342	reservations of the processor units. /*
343	state_t curr_state;
344
345	/ The following variable value is size of memory representing all*
346	current and future reservations of the processor units. /*
347	size_t dfa_state_size;
348
349	/ The following array is used to find the best insn from ready when*
350	the automaton pipeline interface is used. /*
351	signed char *ready_try = NULL;
352
353	/ The ready list. /
354	struct ready_list ready = {NULL, .veclen: `0`, .first: `0`, .n_ready: `0`, .n_debug: `0`};
355
356	/ The pointer to the ready list (to be removed). /
357	static struct ready_list *readyp = &ready;
358
359	/ Scheduling clock. /
360	static int clock_var;
361
362	/ Clock at which the previous instruction was issued. /
363	static int last_clock_var;
364
365	/ Set to true if, when queuing a shadow insn, we discover that it would be*
366	scheduled too late. /*
367	static bool must_backtrack;
368
369	/ The following variable value is number of essential insns issued on*
370	the current cycle. An insn is essential one if it changes the
371	processors state. /*
372	int cycle_issued_insns;
373
374	/ This records the actual schedule. It is built up during the main phase*
375	of schedule_block, and afterwards used to reorder the insns in the RTL. /*
376	static vec<rtx_insn *> scheduled_insns;
377
378	static int may_trap_exp (const_rtx, int);
379
380	/ Nonzero iff the address is comprised from at most 1 register. /
381	#define CONST_BASED_ADDRESS_P(x) \
382	(REG_P (x) \
383	\|\| ((GET_CODE (x) == PLUS \|\| GET_CODE (x) == MINUS \
384	\|\| (GET_CODE (x) == LO_SUM)) \
385	&& (CONSTANT_P (XEXP (x, 0)) \
386	\|\| CONSTANT_P (XEXP (x, 1)))))
387
388	/ Returns a class that insn with GET_DEST(insn)=x may belong to,*
389	as found by analyzing insn's expression. /*
390
391
392	static int haifa_luid_for_non_insn (rtx x);
393
394	/ Haifa version of sched_info hooks common to all headers. /
395	const struct common_sched_info_def haifa_common_sched_info =
396	{
397	NULL, / fix_recovery_cfg /
398	NULL, / add_block /
399	NULL, / estimate_number_of_insns /
400	.luid_for_non_insn: haifa_luid_for_non_insn, / luid_for_non_insn /
401	.sched_pass_id: SCHED_PASS_UNKNOWN / sched_pass_id /
402	};
403
404	/ Mapping from instruction UID to its Logical UID. /
405	vec<int> sched_luids;
406
407	/ Next LUID to assign to an instruction. /
408	int sched_max_luid = `1`;
409
410	/ Haifa Instruction Data. /
411	vec<haifa_insn_data_def> h_i_d;
412
413	void (* sched_init_only_bb) (basic_block, basic_block);
414
415	/ Split block function. Different schedulers might use different functions*
416	to handle their internal data consistent. /*
417	basic_block (* sched_split_block) (basic_block, rtx);
418
419	/ Create empty basic block after the specified block. /
420	basic_block (* sched_create_empty_bb) (basic_block);
421
422	/ Return the number of cycles until INSN is expected to be ready.*
423	Return zero if it already is. /*
424	static int
425	insn_delay (rtx_insn *insn)
426	{
427	return MAX (INSN_TICK (insn) - clock_var, `0`);
428	}
429
430	static int
431	may_trap_exp (const_rtx x, int is_store)
432	{
433	enum rtx_code code;
434
435	if (x == `0`)
436	return TRAP_FREE;
437	code = GET_CODE (x);
438	if (is_store)
439	{
440	if (code == MEM && may_trap_p (x))
441	return TRAP_RISKY;
442	else
443	return TRAP_FREE;
444	}
445	if (code == MEM)
446	{
447	/ The insn uses memory: a volatile load. /
448	if (MEM_VOLATILE_P (x))
449	return IRISKY;
450	/ An exception-free load. /
451	if (!may_trap_p (x))
452	return IFREE;
453	/ A load with 1 base register, to be further checked. /
454	if (CONST_BASED_ADDRESS_P (XEXP (x, `0`)))
455	return PFREE_CANDIDATE;
456	/ No info on the load, to be further checked. /
457	return PRISKY_CANDIDATE;
458	}
459	else
460	{
461	const char *fmt;
462	int i, insn_class = TRAP_FREE;
463
464	/ Neither store nor load, check if it may cause a trap. /
465	if (may_trap_p (x))
466	return TRAP_RISKY;
467	/ Recursive step: walk the insn... /
468	fmt = GET_RTX_FORMAT (code);
469	for (i = GET_RTX_LENGTH (code) - `1`; i >= `0`; i--)
470	{
471	if (fmt[i] == `'e'`)
472	{
473	int tmp_class = may_trap_exp (XEXP (x, i), is_store);
474	insn_class = WORST_CLASS (insn_class, tmp_class);
475	}
476	else if (fmt[i] == `'E'`)
477	{
478	int j;
479	for (j = `0`; j < XVECLEN (x, i); j++)
480	{
481	int tmp_class = may_trap_exp (XVECEXP (x, i, j), is_store);
482	insn_class = WORST_CLASS (insn_class, tmp_class);
483	if (insn_class == TRAP_RISKY \|\| insn_class == IRISKY)
484	break;
485	}
486	}
487	if (insn_class == TRAP_RISKY \|\| insn_class == IRISKY)
488	break;
489	}
490	return insn_class;
491	}
492	}
493
494	/ Classifies rtx X of an insn for the purpose of verifying that X can be*
495	executed speculatively (and consequently the insn can be moved
496	speculatively), by examining X, returning:
497	TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
498	TRAP_FREE: non-load insn.
499	IFREE: load from a globally safe location.
500	IRISKY: volatile load.
501	PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
502	being either PFREE or PRISKY. /*
503
504	static int
505	haifa_classify_rtx (const_rtx x)
506	{
507	int tmp_class = TRAP_FREE;
508	int insn_class = TRAP_FREE;
509	enum rtx_code code;
510
511	if (GET_CODE (x) == PARALLEL)
512	{
513	int i, len = XVECLEN (x, `0`);
514
515	for (i = len - `1`; i >= `0`; i--)
516	{
517	tmp_class = haifa_classify_rtx (XVECEXP (x, `0`, i));
518	insn_class = WORST_CLASS (insn_class, tmp_class);
519	if (insn_class == TRAP_RISKY \|\| insn_class == IRISKY)
520	break;
521	}
522	}
523	else
524	{
525	code = GET_CODE (x);
526	switch (code)
527	{
528	case CLOBBER:
529	/ Test if it is a 'store'. /
530	tmp_class = may_trap_exp (XEXP (x, `0`), is_store: `1`);
531	break;
532	case SET:
533	/ Test if it is a store. /
534	tmp_class = may_trap_exp (SET_DEST (x), is_store: `1`);
535	if (tmp_class == TRAP_RISKY)
536	break;
537	/ Test if it is a load. /
538	tmp_class =
539	WORST_CLASS (tmp_class,
540	may_trap_exp (SET_SRC (x), `0`));
541	break;
542	case COND_EXEC:
543	tmp_class = haifa_classify_rtx (COND_EXEC_CODE (x));
544	if (tmp_class == TRAP_RISKY)
545	break;
546	tmp_class = WORST_CLASS (tmp_class,
547	may_trap_exp (COND_EXEC_TEST (x), `0`));
548	break;
549	case TRAP_IF:
550	tmp_class = TRAP_RISKY;
551	break;
552	default:;
553	}
554	insn_class = tmp_class;
555	}
556
557	return insn_class;
558	}
559
560	int
561	haifa_classify_insn (const_rtx insn)
562	{
563	return haifa_classify_rtx (x: PATTERN (insn));
564	}
565
566	/ After the scheduler initialization function has been called, this function*
567	can be called to enable modulo scheduling. II is the initiation interval
568	we should use, it affects the delays for delay_pairs that were recorded as
569	separated by a given number of stages.
570
571	MAX_STAGES provides us with a limit
572	after which we give up scheduling; the caller must have unrolled at least
573	as many copies of the loop body and recorded delay_pairs for them.
574
575	INSNS is the number of real (non-debug) insns in one iteration of
576	the loop. MAX_UID can be used to test whether an insn belongs to
577	the first iteration of the loop; all of them have a uid lower than
578	MAX_UID. /*
579	void
580	set_modulo_params (int ii, int max_stages, int insns, int max_uid)
581	{
582	modulo_ii = ii;
583	modulo_max_stages = max_stages;
584	modulo_n_insns = insns;
585	modulo_iter0_max_uid = max_uid;
586	modulo_backtracks_left = param_max_modulo_backtrack_attempts;
587	}
588
589	/ A structure to record a pair of insns where the first one is a real*
590	insn that has delay slots, and the second is its delayed shadow.
591	I1 is scheduled normally and will emit an assembly instruction,
592	while I2 describes the side effect that takes place at the
593	transition between cycles CYCLES and (CYCLES + 1) after I1. /*
594	struct delay_pair
595	{
596	struct delay_pair *next_same_i1;
597	rtx_insn i1, i2;
598	int cycles;
599	/ When doing modulo scheduling, we a delay_pair can also be used to*
600	show that I1 and I2 are the same insn in a different stage. If that
601	is the case, STAGES will be nonzero. /*
602	int stages;
603	};
604
605	/ Helpers for delay hashing. /
606
607	struct delay_i1_hasher : nofree_ptr_hash <delay_pair>
608	{
609	typedef void *compare_type;
610	static inline hashval_t hash (const delay_pair *);
611	static inline bool equal (const delay_pair , const* void *);
612	};
613
614	/ Returns a hash value for X, based on hashing just I1. /
615
616	inline hashval_t
617	delay_i1_hasher::hash (const delay_pair *x)
618	{
619	return htab_hash_pointer (x->i1);
620	}
621
622	/ Return true if I1 of pair X is the same as that of pair Y. /
623
624	inline bool
625	delay_i1_hasher::equal (const delay_pair x, const* void *y)
626	{
627	return x->i1 == y;
628	}
629
630	struct delay_i2_hasher : free_ptr_hash <delay_pair>
631	{
632	typedef void *compare_type;
633	static inline hashval_t hash (const delay_pair *);
634	static inline bool equal (const delay_pair , const* void *);
635	};
636
637	/ Returns a hash value for X, based on hashing just I2. /
638
639	inline hashval_t
640	delay_i2_hasher::hash (const delay_pair *x)
641	{
642	return htab_hash_pointer (x->i2);
643	}
644
645	/ Return true if I2 of pair X is the same as that of pair Y. /
646
647	inline bool
648	delay_i2_hasher::equal (const delay_pair x, const* void *y)
649	{
650	return x->i2 == y;
651	}
652
653	/ Two hash tables to record delay_pairs, one indexed by I1 and the other*
654	indexed by I2. /*
655	static hash_table<delay_i1_hasher> *delay_htab;
656	static hash_table<delay_i2_hasher> *delay_htab_i2;
657
658	/ Called through htab_traverse. Walk the hashtable using I2 as*
659	index, and delete all elements involving an UID higher than
660	that pointed to by DATA. /
661	int
662	haifa_htab_i2_traverse (delay_pair *slot, int* *data)
663	{
664	int maxuid = *data;
665	struct delay_pair p = slot;
666	if (INSN_UID (insn: p->i2) >= maxuid \|\| INSN_UID (insn: p->i1) >= maxuid)
667	{
668	delay_htab_i2->clear_slot (slot);
669	}
670	return `1`;
671	}
672
673	/ Called through htab_traverse. Walk the hashtable using I2 as*
674	index, and delete all elements involving an UID higher than
675	that pointed to by DATA. /
676	int
677	haifa_htab_i1_traverse (delay_pair *pslot, int* *data)
678	{
679	int maxuid = *data;
680	struct delay_pair p, first, **pprev;
681
682	if (INSN_UID (insn: (*pslot)->i1) >= maxuid)
683	{
684	delay_htab->clear_slot (slot: pslot);
685	return `1`;
686	}
687	pprev = &first;
688	for (p = *pslot; p; p = p->next_same_i1)
689	{
690	if (INSN_UID (insn: p->i2) < maxuid)
691	{
692	*pprev = p;
693	pprev = &p->next_same_i1;
694	}
695	}
696	*pprev = NULL;
697	if (first == NULL)
698	delay_htab->clear_slot (slot: pslot);
699	else
700	*pslot = first;
701	return `1`;
702	}
703
704	/ Discard all delay pairs which involve an insn with an UID higher*
705	than MAX_UID. /*
706	void
707	discard_delay_pairs_above (int max_uid)
708	{
709	delay_htab->traverse <int *, haifa_htab_i1_traverse> (argument: &max_uid);
710	delay_htab_i2->traverse <int *, haifa_htab_i2_traverse> (argument: &max_uid);
711	}
712
713	/ This function can be called by a port just before it starts the final*
714	scheduling pass. It records the fact that an instruction with delay
715	slots has been split into two insns, I1 and I2. The first one will be
716	scheduled normally and initiates the operation. The second one is a
717	shadow which must follow a specific number of cycles after I1; its only
718	purpose is to show the side effect that occurs at that cycle in the RTL.
719	If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
720	while I2 retains the original insn type.
721
722	There are two ways in which the number of cycles can be specified,
723	involving the CYCLES and STAGES arguments to this function. If STAGES
724	is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor
725	which is multiplied by MODULO_II to give the number of cycles. This is
726	only useful if the caller also calls set_modulo_params to enable modulo
727	scheduling. /*
728
729	void
730	record_delay_slot_pair (rtx_insn i1, rtx_insn i2, int cycles, int stages)
731	{
732	struct delay_pair p = XNEW (struct* delay_pair);
733	struct delay_pair **slot;
734
735	p->i1 = i1;
736	p->i2 = i2;
737	p->cycles = cycles;
738	p->stages = stages;
739
740	if (!delay_htab)
741	{
742	delay_htab = new hash_table<delay_i1_hasher> (`10`);
743	delay_htab_i2 = new hash_table<delay_i2_hasher> (`10`);
744	}
745	slot = delay_htab->find_slot_with_hash (comparable: i1, hash: htab_hash_pointer (i1), insert: INSERT);
746	p->next_same_i1 = *slot;
747	*slot = p;
748	slot = delay_htab_i2->find_slot (value: p, insert: INSERT);
749	*slot = p;
750	}
751
752	/ Examine the delay pair hashtable to see if INSN is a shadow for another,*
753	and return the other insn if so. Return NULL otherwise. /*
754	rtx_insn *
755	real_insn_for_shadow (rtx_insn *insn)
756	{
757	struct delay_pair *pair;
758
759	if (!delay_htab)
760	return NULL;
761
762	pair = delay_htab_i2->find_with_hash (comparable: insn, hash: htab_hash_pointer (insn));
763	if (!pair \|\| pair->stages > `0`)
764	return NULL;
765	return pair->i1;
766	}
767
768	/ For a pair P of insns, return the fixed distance in cycles from the first*
769	insn after which the second must be scheduled. /*
770	static int
771	pair_delay (struct delay_pair *p)
772	{
773	if (p->stages == `0`)
774	return p->cycles;
775	else
776	return p->stages * modulo_ii;
777	}
778
779	/ Given an insn INSN, add a dependence on its delayed shadow if it*
780	has one. Also try to find situations where shadows depend on each other
781	and add dependencies to the real insns to limit the amount of backtracking
782	needed. /*
783	void
784	add_delay_dependencies (rtx_insn *insn)
785	{
786	struct delay_pair *pair;
787	sd_iterator_def sd_it;
788	dep_t dep;
789
790	if (!delay_htab)
791	return;
792
793	pair = delay_htab_i2->find_with_hash (comparable: insn, hash: htab_hash_pointer (insn));
794	if (!pair)
795	return;
796	add_dependence (insn, pair->i1, REG_DEP_ANTI);
797	if (pair->stages)
798	return;
799
800	FOR_EACH_DEP (pair->i2, SD_LIST_BACK, sd_it, dep)
801	{
802	rtx_insn *pro = DEP_PRO (dep);
803	struct delay_pair *other_pair
804	= delay_htab_i2->find_with_hash (comparable: pro, hash: htab_hash_pointer (pro));
805	if (!other_pair \|\| other_pair->stages)
806	continue;
807	if (pair_delay (p: other_pair) >= pair_delay (p: pair))
808	{
809	if (sched_verbose >= `4`)
810	{
811	fprintf (stream: sched_dump, format: ";;\tadding dependence %d <- %d\n",
812	INSN_UID (insn: other_pair->i1),
813	INSN_UID (insn: pair->i1));
814	fprintf (stream: sched_dump, format: ";;\tpair1 %d <- %d, cost %d\n",
815	INSN_UID (insn: pair->i1),
816	INSN_UID (insn: pair->i2),
817	pair_delay (p: pair));
818	fprintf (stream: sched_dump, format: ";;\tpair2 %d <- %d, cost %d\n",
819	INSN_UID (insn: other_pair->i1),
820	INSN_UID (insn: other_pair->i2),
821	pair_delay (p: other_pair));
822	}
823	add_dependence (pair->i1, other_pair->i1, REG_DEP_ANTI);
824	}
825	}
826	}
827
828	/ Forward declarations. /
829
830	static int priority (rtx_insn , bool* force_recompute = false);
831	static int autopref_rank_for_schedule (const rtx_insn , const* rtx_insn *);
832	static int rank_for_schedule (const void , const* void *);
833	static void swap_sort (rtx_insn *, int*);
834	static void queue_insn (rtx_insn , int, const* char *);
835	static int schedule_insn (rtx_insn *);
836	static void adjust_priority (rtx_insn *);
837	static void advance_one_cycle (void);
838	static void extend_h_i_d (void);
839
840
841	/ Notes handling mechanism:*
842	=========================
843	Generally, NOTES are saved before scheduling and restored after scheduling.
844	The scheduler distinguishes between two types of notes:
845
846	(1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
847	Before scheduling a region, a pointer to the note is added to the insn
848	that follows or precedes it. (This happens as part of the data dependence
849	computation). After scheduling an insn, the pointer contained in it is
850	used for regenerating the corresponding note (in reemit_notes).
851
852	(2) All other notes (e.g. INSN_DELETED): Before scheduling a block,
853	these notes are put in a list (in rm_other_notes() and
854	unlink_other_notes ()). After scheduling the block, these notes are
855	inserted at the beginning of the block (in schedule_block()). /*
856
857	static void ready_add (struct ready_list , rtx_insn , bool);
858	static rtx_insn ready_remove_first (struct* ready_list *);
859	static rtx_insn ready_remove_first_dispatch (struct* ready_list *ready);
860
861	static void queue_to_ready (struct ready_list *);
862	static int early_queue_to_ready (state_t, struct ready_list *);
863
864	/ The following functions are used to implement multi-pass scheduling*
865	on the first cycle. /*
866	static rtx_insn ready_remove (struct* ready_list , int*);
867	static void ready_remove_insn (rtx_insn *);
868
869	static void fix_inter_tick (rtx_insn , rtx_insn );
870	static int fix_tick_ready (rtx_insn *);
871	static void change_queue_index (rtx_insn , int*);
872
873	/ The following functions are used to implement scheduling of data/control*
874	speculative instructions. /*
875
876	static void extend_h_i_d (void);
877	static void init_h_i_d (rtx_insn *);
878	static int haifa_speculate_insn (rtx_insn , ds_t, rtx );
879	static void generate_recovery_code (rtx_insn *);
880	static void process_insn_forw_deps_be_in_spec (rtx_insn , rtx_insn , ds_t);
881	static void begin_speculative_block (rtx_insn *);
882	static void add_to_speculative_block (rtx_insn *);
883	static void init_before_recovery (basic_block *);
884	static void create_check_block_twin (rtx_insn , bool*);
885	static void fix_recovery_deps (basic_block);
886	static bool haifa_change_pattern (rtx_insn *, rtx);
887	static void dump_new_block_header (int, basic_block, rtx_insn , rtx_insn );
888	static void restore_bb_notes (basic_block);
889	static void fix_jump_move (rtx_insn *);
890	static void move_block_after_check (rtx_insn *);
891	static void move_succs (vec<edge, va_gc> **, basic_block);
892	static void sched_remove_insn (rtx_insn *);
893	static void clear_priorities (rtx_insn , rtx_vec_t );
894	static void calc_priorities (const rtx_vec_t &);
895	static void add_jump_dependencies (rtx_insn , rtx_insn );
896
897	#endif /* INSN_SCHEDULING */
898
899	/ Point to state used for the current scheduling pass. /
900	struct haifa_sched_info *current_sched_info;
901
902	#ifndef INSN_SCHEDULING
903	void
904	schedule_insns (void)
905	{
906	}
907	#else
908
909	/ Do register pressure sensitive insn scheduling if the flag is set*
910	up. /*
911	enum sched_pressure_algorithm sched_pressure;
912
913	/ Map regno -> its pressure class. The map defined only when*
914	SCHED_PRESSURE != SCHED_PRESSURE_NONE. /*
915	enum reg_class *sched_regno_pressure_class;
916
917	/ The current register pressure. Only elements corresponding pressure*
918	classes are defined. /*
919	static int curr_reg_pressure[N_REG_CLASSES];
920
921	/ Saved value of the previous array. /
922	static int saved_reg_pressure[N_REG_CLASSES];
923
924	/ Register living at given scheduling point. /
925	static bitmap curr_reg_live;
926
927	/ Saved value of the previous array. /
928	static bitmap saved_reg_live;
929
930	/ Registers mentioned in the current region. /
931	static bitmap region_ref_regs;
932
933	/ Temporary bitmap used for SCHED_PRESSURE_MODEL. /
934	static bitmap tmp_bitmap;
935
936	/ Effective number of available registers of a given class (see comment*
937	in sched_pressure_start_bb). /*
938	static int sched_class_regs_num[N_REG_CLASSES];
939	/ The number of registers that the function would need to save before it*
940	uses them, and the number of fixed_regs. Helpers for calculating of
941	sched_class_regs_num. /*
942	static int call_saved_regs_num[N_REG_CLASSES];
943	static int fixed_regs_num[N_REG_CLASSES];
944
945	/ Initiate register pressure relative info for scheduling the current*
946	region. Currently it is only clearing register mentioned in the
947	current region. /*
948	void
949	sched_init_region_reg_pressure_info (void)
950	{
951	bitmap_clear (region_ref_regs);
952	}
953
954	/ PRESSURE[CL] describes the pressure on register class CL. Update it*
955	for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
956	LIVE tracks the set of live registers; if it is null, assume that
957	every birth or death is genuine. /*
958	static inline void
959	mark_regno_birth_or_death (bitmap live, int pressure, int* regno, bool birth_p)
960	{
961	enum reg_class pressure_class;
962
963	pressure_class = sched_regno_pressure_class[regno];
964	if (regno >= FIRST_PSEUDO_REGISTER)
965	{
966	if (pressure_class != NO_REGS)
967	{
968	if (birth_p)
969	{
970	if (!live \|\| bitmap_set_bit (live, regno))
971	pressure[pressure_class]
972	+= (ira_reg_class_max_nregs
973	[pressure_class][PSEUDO_REGNO_MODE (regno)]);
974	}
975	else
976	{
977	if (!live \|\| bitmap_clear_bit (live, regno))
978	pressure[pressure_class]
979	-= (ira_reg_class_max_nregs
980	[pressure_class][PSEUDO_REGNO_MODE (regno)]);
981	}
982	}
983	}
984	else if (pressure_class != NO_REGS
985	&& ! TEST_HARD_REG_BIT (ira_no_alloc_regs, bit: regno))
986	{
987	if (birth_p)
988	{
989	if (!live \|\| bitmap_set_bit (live, regno))
990	pressure[pressure_class]++;
991	}
992	else
993	{
994	if (!live \|\| bitmap_clear_bit (live, regno))
995	pressure[pressure_class]--;
996	}
997	}
998	}
999
1000	/ Initiate current register pressure related info from living*
1001	registers given by LIVE. /*
1002	static void
1003	initiate_reg_pressure_info (bitmap live)
1004	{
1005	int i;
1006	unsigned int j;
1007	bitmap_iterator bi;
1008
1009	for (i = `0`; i < ira_pressure_classes_num; i++)
1010	curr_reg_pressure[ira_pressure_classes[i]] = `0`;
1011	bitmap_clear (curr_reg_live);
1012	EXECUTE_IF_SET_IN_BITMAP (live, `0`, j, bi)
1013	if (sched_pressure == SCHED_PRESSURE_MODEL
1014	\|\| current_nr_blocks == `1`
1015	\|\| bitmap_bit_p (region_ref_regs, j))
1016	mark_regno_birth_or_death (live: curr_reg_live, pressure: curr_reg_pressure, regno: j, birth_p: true);
1017	}
1018
1019	/ Mark registers in X as mentioned in the current region. /
1020	static void
1021	setup_ref_regs (rtx x)
1022	{
1023	int i, j;
1024	const RTX_CODE code = GET_CODE (x);
1025	const char *fmt;
1026
1027	if (REG_P (x))
1028	{
1029	bitmap_set_range (region_ref_regs, REGNO (x), REG_NREGS (x));
1030	return;
1031	}
1032	fmt = GET_RTX_FORMAT (code);
1033	for (i = GET_RTX_LENGTH (code) - `1`; i >= `0`; i--)
1034	if (fmt[i] == `'e'`)
1035	setup_ref_regs (XEXP (x, i));
1036	else if (fmt[i] == `'E'`)
1037	{
1038	for (j = `0`; j < XVECLEN (x, i); j++)
1039	setup_ref_regs (XVECEXP (x, i, j));
1040	}
1041	}
1042
1043	/ Initiate current register pressure related info at the start of*
1044	basic block BB. /*
1045	static void
1046	initiate_bb_reg_pressure_info (basic_block bb)
1047	{
1048	unsigned int i ATTRIBUTE_UNUSED;
1049	rtx_insn *insn;
1050
1051	if (current_nr_blocks > `1`)
1052	FOR_BB_INSNS (bb, insn)
1053	if (NONDEBUG_INSN_P (insn))
1054	setup_ref_regs (PATTERN (insn));
1055	initiate_reg_pressure_info (live: df_get_live_in (bb));
1056	if (bb_has_eh_pred (bb))
1057	for (i = `0`; ; ++i)
1058	{
1059	unsigned int regno = EH_RETURN_DATA_REGNO (i);
1060
1061	if (regno == INVALID_REGNUM)
1062	break;
1063	if (! bitmap_bit_p (df_get_live_in (bb), regno))
1064	mark_regno_birth_or_death (live: curr_reg_live, pressure: curr_reg_pressure,
1065	regno, birth_p: true);
1066	}
1067	}
1068
1069	/ Save current register pressure related info. /
1070	static void
1071	save_reg_pressure (void)
1072	{
1073	int i;
1074
1075	for (i = `0`; i < ira_pressure_classes_num; i++)
1076	saved_reg_pressure[ira_pressure_classes[i]]
1077	= curr_reg_pressure[ira_pressure_classes[i]];
1078	bitmap_copy (saved_reg_live, curr_reg_live);
1079	}
1080
1081	/ Restore saved register pressure related info. /
1082	static void
1083	restore_reg_pressure (void)
1084	{
1085	int i;
1086
1087	for (i = `0`; i < ira_pressure_classes_num; i++)
1088	curr_reg_pressure[ira_pressure_classes[i]]
1089	= saved_reg_pressure[ira_pressure_classes[i]];
1090	bitmap_copy (curr_reg_live, saved_reg_live);
1091	}
1092
1093	/ Return TRUE if the register is dying after its USE. /
1094	static bool
1095	dying_use_p (struct reg_use_data *use)
1096	{
1097	struct reg_use_data *next;
1098
1099	for (next = use->next_regno_use; next != use; next = next->next_regno_use)
1100	if (NONDEBUG_INSN_P (next->insn)
1101	&& QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED)
1102	return false;
1103	return true;
1104	}
1105
1106	/ Print info about the current register pressure and its excess for*
1107	each pressure class. /*
1108	static void
1109	print_curr_reg_pressure (void)
1110	{
1111	int i;
1112	enum reg_class cl;
1113
1114	fprintf (stream: sched_dump, format: ";;\t");
1115	for (i = `0`; i < ira_pressure_classes_num; i++)
1116	{
1117	cl = ira_pressure_classes[i];
1118	gcc_assert (curr_reg_pressure[cl] >= `0`);
1119	fprintf (stream: sched_dump, format: " %s:%d(%d)", reg_class_names[cl],
1120	curr_reg_pressure[cl],
1121	curr_reg_pressure[cl] - sched_class_regs_num[cl]);
1122	}
1123	fprintf (stream: sched_dump, format: "\n");
1124	}
1125
1126	/ Determine if INSN has a condition that is clobbered if a register*
1127	in SET_REGS is modified. /*
1128	static bool
1129	cond_clobbered_p (rtx_insn *insn, HARD_REG_SET set_regs)
1130	{
1131	rtx pat = PATTERN (insn);
1132	gcc_assert (GET_CODE (pat) == COND_EXEC);
1133	if (TEST_HARD_REG_BIT (set: set_regs, REGNO (XEXP (COND_EXEC_TEST (pat), `0`))))
1134	{
1135	sd_iterator_def sd_it;
1136	dep_t dep;
1137	haifa_change_pattern (insn, ORIG_PAT (insn));
1138	FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
1139	DEP_STATUS (dep) &= ~DEP_CANCELLED;
1140	TODO_SPEC (insn) = HARD_DEP;
1141	if (sched_verbose >= `2`)
1142	fprintf (stream: sched_dump,
1143	format: ";;\t\tdequeue insn %s because of clobbered condition\n",
1144	(*current_sched_info->print_insn) (insn, `0`));
1145	return true;
1146	}
1147
1148	return false;
1149	}
1150
1151	/ This function should be called after modifying the pattern of INSN,*
1152	to update scheduler data structures as needed. /*
1153	static void
1154	update_insn_after_change (rtx_insn *insn)
1155	{
1156	sd_iterator_def sd_it;
1157	dep_t dep;
1158
1159	dfa_clear_single_insn_cache (insn);
1160
1161	sd_it = sd_iterator_start (insn,
1162	SD_LIST_FORW \| SD_LIST_BACK \| SD_LIST_RES_BACK);
1163	while (sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep))
1164	{
1165	DEP_COST (dep) = UNKNOWN_DEP_COST;
1166	sd_iterator_next (it_ptr: &sd_it);
1167	}
1168
1169	/ Invalidate INSN_COST, so it'll be recalculated. /
1170	INSN_COST (insn) = -`1`;
1171	/ Invalidate INSN_TICK, so it'll be recalculated. /
1172	INSN_TICK (insn) = INVALID_TICK;
1173
1174	/ Invalidate autoprefetch data entry. /
1175	INSN_AUTOPREF_MULTIPASS_DATA (insn)[`0`].status
1176	= AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
1177	INSN_AUTOPREF_MULTIPASS_DATA (insn)[`1`].status
1178	= AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
1179	}
1180
1181
1182	/ Two VECs, one to hold dependencies for which pattern replacements*
1183	need to be applied or restored at the start of the next cycle, and
1184	another to hold an integer that is either one, to apply the
1185	corresponding replacement, or zero to restore it. /*
1186	static vec<dep_t> next_cycle_replace_deps;
1187	static vec<int> next_cycle_apply;
1188
1189	static void apply_replacement (dep_t, bool);
1190	static void restore_pattern (dep_t, bool);
1191
1192	/ Look at the remaining dependencies for insn NEXT, and compute and return*
1193	the TODO_SPEC value we should use for it. This is called after one of
1194	NEXT's dependencies has been resolved.
1195	We also perform pattern replacements for predication, and for broken
1196	replacement dependencies. The latter is only done if FOR_BACKTRACK is
1197	false. /*
1198
1199	static ds_t
1200	recompute_todo_spec (rtx_insn next, bool* for_backtrack)
1201	{
1202	ds_t new_ds;
1203	sd_iterator_def sd_it;
1204	dep_t dep, modify_dep = NULL;
1205	int n_spec = `0`;
1206	int n_control = `0`;
1207	int n_replace = `0`;
1208	bool first_p = true;
1209
1210	if (sd_lists_empty_p (next, SD_LIST_BACK))
1211	/ NEXT has all its dependencies resolved. /
1212	return `0`;
1213
1214	if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
1215	return HARD_DEP;
1216
1217	/ If NEXT is intended to sit adjacent to this instruction, we don't*
1218	want to try to break any dependencies. Treat it as a HARD_DEP. /*
1219	if (SCHED_GROUP_P (next))
1220	return HARD_DEP;
1221
1222	/ Now we've got NEXT with speculative deps only.*
1223	1. Look at the deps to see what we have to do.
1224	2. Check if we can do 'todo'. /*
1225	new_ds = `0`;
1226
1227	FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
1228	{
1229	rtx_insn *pro = DEP_PRO (dep);
1230	ds_t ds = DEP_STATUS (dep) & SPECULATIVE;
1231
1232	if (DEBUG_INSN_P (pro) && !DEBUG_INSN_P (next))
1233	continue;
1234
1235	if (ds)
1236	{
1237	n_spec++;
1238	if (first_p)
1239	{
1240	first_p = false;
1241
1242	new_ds = ds;
1243	}
1244	else
1245	new_ds = ds_merge (new_ds, ds);
1246	}
1247	else if (DEP_TYPE (dep) == REG_DEP_CONTROL)
1248	{
1249	if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED)
1250	{
1251	n_control++;
1252	modify_dep = dep;
1253	}
1254	DEP_STATUS (dep) &= ~DEP_CANCELLED;
1255	}
1256	else if (DEP_REPLACE (dep) != NULL)
1257	{
1258	if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED)
1259	{
1260	n_replace++;
1261	modify_dep = dep;
1262	}
1263	DEP_STATUS (dep) &= ~DEP_CANCELLED;
1264	}
1265	}
1266
1267	if (n_replace > `0` && n_control == `0` && n_spec == `0`)
1268	{
1269	if (!dbg_cnt (index: sched_breakdep))
1270	return HARD_DEP;
1271	FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
1272	{
1273	struct dep_replacement *desc = DEP_REPLACE (dep);
1274	if (desc != NULL)
1275	{
1276	if (desc->insn == next && !for_backtrack)
1277	{
1278	gcc_assert (n_replace == `1`);
1279	apply_replacement (dep, true);
1280	}
1281	DEP_STATUS (dep) \|= DEP_CANCELLED;
1282	}
1283	}
1284	return `0`;
1285	}
1286
1287	else if (n_control == `1` && n_replace == `0` && n_spec == `0`)
1288	{
1289	rtx_insn pro, other;
1290	rtx new_pat;
1291	rtx cond = NULL_RTX;
1292	bool success;
1293	rtx_insn *prev = NULL;
1294	int i;
1295	unsigned regno;
1296
1297	if ((current_sched_info->flags & DO_PREDICATION) == `0`
1298	\|\| (ORIG_PAT (next) != NULL_RTX
1299	&& PREDICATED_PAT (next) == NULL_RTX))
1300	return HARD_DEP;
1301
1302	pro = DEP_PRO (modify_dep);
1303	other = real_insn_for_shadow (insn: pro);
1304	if (other != NULL_RTX)
1305	pro = other;
1306
1307	cond = sched_get_reverse_condition_uncached (pro);
1308	regno = REGNO (XEXP (cond, `0`));
1309
1310	/ Find the last scheduled insn that modifies the condition register.*
1311	We can stop looking once we find the insn we depend on through the
1312	REG_DEP_CONTROL; if the condition register isn't modified after it,
1313	we know that it still has the right value. /*
1314	if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED)
1315	FOR_EACH_VEC_ELT_REVERSE (scheduled_insns, i, prev)
1316	{
1317	HARD_REG_SET t;
1318
1319	find_all_hard_reg_sets (prev, &t, true);
1320	if (TEST_HARD_REG_BIT (set: t, bit: regno))
1321	return HARD_DEP;
1322	if (prev == pro)
1323	break;
1324	}
1325	if (ORIG_PAT (next) == NULL_RTX)
1326	{
1327	ORIG_PAT (next) = PATTERN (insn: next);
1328
1329	new_pat = gen_rtx_COND_EXEC (VOIDmode, cond, PATTERN (next));
1330	success = haifa_change_pattern (next, new_pat);
1331	if (!success)
1332	return HARD_DEP;
1333	PREDICATED_PAT (next) = new_pat;
1334	}
1335	else if (PATTERN (insn: next) != PREDICATED_PAT (next))
1336	{
1337	bool success = haifa_change_pattern (next,
1338	PREDICATED_PAT (next));
1339	gcc_assert (success);
1340	}
1341	DEP_STATUS (modify_dep) \|= DEP_CANCELLED;
1342	return DEP_CONTROL;
1343	}
1344
1345	if (PREDICATED_PAT (next) != NULL_RTX)
1346	{
1347	int tick = INSN_TICK (next);
1348	bool success = haifa_change_pattern (next,
1349	ORIG_PAT (next));
1350	INSN_TICK (next) = tick;
1351	gcc_assert (success);
1352	}
1353
1354	/ We can't handle the case where there are both speculative and control*
1355	dependencies, so we return HARD_DEP in such a case. Also fail if
1356	we have speculative dependencies with not enough points, or more than
1357	one control dependency. /*
1358	if ((n_spec > `0` && (n_control > `0` \|\| n_replace > `0`))
1359	\|\| (n_spec > `0`
1360	/ Too few points? /
1361	&& ds_weak (new_ds) < spec_info->data_weakness_cutoff)
1362	\|\| n_control > `0`
1363	\|\| n_replace > `0`)
1364	return HARD_DEP;
1365
1366	return new_ds;
1367	}
1368
1369	/ Pointer to the last instruction scheduled. /
1370	static rtx_insn *last_scheduled_insn;
1371
1372	/ Pointer to the last nondebug instruction scheduled within the*
1373	block, or the prev_head of the scheduling block. Used by
1374	rank_for_schedule, so that insns independent of the last scheduled
1375	insn will be preferred over dependent instructions. /*
1376	static rtx_insn *last_nondebug_scheduled_insn;
1377
1378	/ Pointer that iterates through the list of unscheduled insns if we*
1379	have a dbg_cnt enabled. It always points at an insn prior to the
1380	first unscheduled one. /*
1381	static rtx_insn *nonscheduled_insns_begin;
1382
1383	/ Compute cost of executing INSN.*
1384	This is the number of cycles between instruction issue and
1385	instruction results. /*
1386	int
1387	insn_sched_cost (rtx_insn *insn)
1388	{
1389	int cost;
1390
1391	if (sched_fusion)
1392	return `0`;
1393
1394	if (sel_sched_p ())
1395	{
1396	if (recog_memoized (insn) < `0`)
1397	return `0`;
1398
1399	cost = insn_default_latency (insn);
1400	if (cost < `0`)
1401	cost = `0`;
1402
1403	return cost;
1404	}
1405
1406	cost = INSN_COST (insn);
1407
1408	if (cost < `0`)
1409	{
1410	/ A USE insn, or something else we don't need to*
1411	understand. We can't pass these directly to
1412	result_ready_cost or insn_default_latency because it will
1413	trigger a fatal error for unrecognizable insns. /*
1414	if (recog_memoized (insn) < `0`)
1415	{
1416	INSN_COST (insn) = `0`;
1417	return `0`;
1418	}
1419	else
1420	{
1421	cost = insn_default_latency (insn);
1422	if (cost < `0`)
1423	cost = `0`;
1424
1425	INSN_COST (insn) = cost;
1426	}
1427	}
1428
1429	return cost;
1430	}
1431
1432	/ Compute cost of dependence LINK.*
1433	This is the number of cycles between instruction issue and
1434	instruction results.
1435	??? We also use this function to call recog_memoized on all insns. /*
1436	int
1437	dep_cost_1 (dep_t link, dw_t dw)
1438	{
1439	rtx_insn *insn = DEP_PRO (link);
1440	rtx_insn *used = DEP_CON (link);
1441	int cost;
1442
1443	if (DEP_COST (link) != UNKNOWN_DEP_COST)
1444	return DEP_COST (link);
1445
1446	if (delay_htab)
1447	{
1448	struct delay_pair *delay_entry;
1449	delay_entry
1450	= delay_htab_i2->find_with_hash (comparable: used, hash: htab_hash_pointer (used));
1451	if (delay_entry)
1452	{
1453	if (delay_entry->i1 == insn)
1454	{
1455	DEP_COST (link) = pair_delay (p: delay_entry);
1456	return DEP_COST (link);
1457	}
1458	}
1459	}
1460
1461	/ A USE insn should never require the value used to be computed.*
1462	This allows the computation of a function's result and parameter
1463	values to overlap the return and call. We don't care about the
1464	dependence cost when only decreasing register pressure. /*
1465	if (recog_memoized (insn: used) < `0`)
1466	{
1467	cost = `0`;
1468	recog_memoized (insn);
1469	}
1470	else
1471	{
1472	enum reg_note dep_type = DEP_TYPE (link);
1473
1474	cost = insn_sched_cost (insn);
1475
1476	if (INSN_CODE (insn) >= `0`)
1477	{
1478	if (dep_type == REG_DEP_ANTI)
1479	cost = `0`;
1480	else if (dep_type == REG_DEP_OUTPUT)
1481	{
1482	cost = (insn_default_latency (insn)
1483	- insn_default_latency (used));
1484	if (cost <= `0`)
1485	cost = `1`;
1486	}
1487	else if (bypass_p (insn))
1488	cost = insn_latency (insn, used);
1489	}
1490
1491
1492	if (targetm.sched.adjust_cost)
1493	cost = targetm.sched.adjust_cost (used, (int) dep_type, insn, cost,
1494	dw);
1495
1496	if (cost < `0`)
1497	cost = `0`;
1498	}
1499
1500	DEP_COST (link) = cost;
1501	return cost;
1502	}
1503
1504	/ Compute cost of dependence LINK.*
1505	This is the number of cycles between instruction issue and
1506	instruction results. /*
1507	int
1508	dep_cost (dep_t link)
1509	{
1510	return dep_cost_1 (link, dw: `0`);
1511	}
1512
1513	/ Use this sel-sched.cc friendly function in reorder2 instead of increasing*
1514	INSN_PRIORITY explicitly. /*
1515	void
1516	increase_insn_priority (rtx_insn insn, int* amount)
1517	{
1518	if (!sel_sched_p ())
1519	{
1520	/ We're dealing with haifa-sched.cc INSN_PRIORITY. /
1521	if (INSN_PRIORITY_KNOWN (insn))
1522	INSN_PRIORITY (insn) += amount;
1523	}
1524	else
1525	{
1526	/ In sel-sched.cc INSN_PRIORITY is not kept up to date.*
1527	Use EXPR_PRIORITY instead. /*
1528	sel_add_to_insn_priority (insn, amount);
1529	}
1530	}
1531
1532	/ Return 'true' if DEP should be included in priority calculations. /
1533	static bool
1534	contributes_to_priority_p (dep_t dep)
1535	{
1536	if (DEBUG_INSN_P (DEP_CON (dep))
1537	\|\| DEBUG_INSN_P (DEP_PRO (dep)))
1538	return false;
1539
1540	/ Critical path is meaningful in block boundaries only. /
1541	if (!current_sched_info->contributes_to_priority (DEP_CON (dep),
1542	DEP_PRO (dep)))
1543	return false;
1544
1545	if (DEP_REPLACE (dep) != NULL)
1546	return false;
1547
1548	/ If flag COUNT_SPEC_IN_CRITICAL_PATH is set,*
1549	then speculative instructions will less likely be
1550	scheduled. That is because the priority of
1551	their producers will increase, and, thus, the
1552	producers will more likely be scheduled, thus,
1553	resolving the dependence. /*
1554	if (sched_deps_info->generate_spec_deps
1555	&& !(spec_info->flags & COUNT_SPEC_IN_CRITICAL_PATH)
1556	&& (DEP_STATUS (dep) & SPECULATIVE))
1557	return false;
1558
1559	return true;
1560	}
1561
1562	/ Compute the number of nondebug deps in list LIST for INSN. /
1563	int
1564	dep_list_size (rtx_insn *insn, sd_list_types_def list)
1565	{
1566	sd_iterator_def sd_it;
1567	dep_t dep;
1568	int dbgcount = `0`, nodbgcount = `0`;
1569
1570	if (!MAY_HAVE_DEBUG_INSNS)
1571	return sd_lists_size (insn, list);
1572
1573	/* TODO: We should split normal and debug insns into separate SD_LIST_*
1574	sub-lists, and then we'll be able to use something like
1575	sd_lists_size(insn, list & SD_LIST_NON_DEBUG)
1576	instead of walking dependencies below. /*
1577
1578	FOR_EACH_DEP (insn, list, sd_it, dep)
1579	{
1580	if (DEBUG_INSN_P (DEP_CON (dep)))
1581	dbgcount++;
1582	else if (!DEBUG_INSN_P (DEP_PRO (dep)))
1583	nodbgcount++;
1584	}
1585
1586	gcc_assert (dbgcount + nodbgcount == sd_lists_size (insn, list));
1587
1588	return nodbgcount;
1589	}
1590
1591	bool sched_fusion;
1592
1593	/ Compute the priority number for INSN. /
1594	static int
1595	priority (rtx_insn insn, bool* force_recompute)
1596	{
1597	if (! INSN_P (insn))
1598	return `0`;
1599
1600	/ We should not be interested in priority of an already scheduled insn. /
1601	gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
1602
1603	if (force_recompute \|\| !INSN_PRIORITY_KNOWN (insn))
1604	{
1605	int this_priority = -`1`;
1606
1607	if (sched_fusion)
1608	{
1609	int this_fusion_priority;
1610
1611	targetm.sched.fusion_priority (insn, FUSION_MAX_PRIORITY,
1612	&this_fusion_priority, &this_priority);
1613	INSN_FUSION_PRIORITY (insn) = this_fusion_priority;
1614	}
1615	else if (dep_list_size (insn, SD_LIST_FORW) == `0`)
1616	/ ??? We should set INSN_PRIORITY to insn_sched_cost when and insn*
1617	has some forward deps but all of them are ignored by
1618	contributes_to_priority hook. At the moment we set priority of
1619	such insn to 0. /*
1620	this_priority = insn_sched_cost (insn);
1621	else
1622	{
1623	rtx_insn prev_first, twin;
1624	basic_block rec;
1625
1626	/ For recovery check instructions we calculate priority slightly*
1627	different than that of normal instructions. Instead of walking
1628	through INSN_FORW_DEPS (check) list, we walk through
1629	INSN_FORW_DEPS list of each instruction in the corresponding
1630	recovery block. /*
1631
1632	/ Selective scheduling does not define RECOVERY_BLOCK macro. /
1633	rec = sel_sched_p () ? NULL : RECOVERY_BLOCK (insn);
1634	if (!rec \|\| rec == EXIT_BLOCK_PTR_FOR_FN (cfun))
1635	{
1636	prev_first = PREV_INSN (insn);
1637	twin = insn;
1638	}
1639	else
1640	{
1641	prev_first = NEXT_INSN (BB_HEAD (rec));
1642	twin = PREV_INSN (BB_END (rec));
1643	}
1644
1645	do
1646	{
1647	sd_iterator_def sd_it;
1648	dep_t dep;
1649
1650	FOR_EACH_DEP (twin, SD_LIST_FORW, sd_it, dep)
1651	{
1652	rtx_insn *next;
1653	int next_priority;
1654
1655	next = DEP_CON (dep);
1656
1657	if (BLOCK_FOR_INSN (insn: next) != rec)
1658	{
1659	int cost;
1660
1661	if (!contributes_to_priority_p (dep))
1662	continue;
1663
1664	if (twin == insn)
1665	cost = dep_cost (link: dep);
1666	else
1667	{
1668	struct _dep _dep1, *dep1 = &_dep1;
1669
1670	init_dep (dep1, insn, next, REG_DEP_ANTI);
1671
1672	cost = dep_cost (link: dep1);
1673	}
1674
1675	next_priority = cost + priority (insn: next);
1676
1677	if (next_priority > this_priority)
1678	this_priority = next_priority;
1679	}
1680	}
1681
1682	twin = PREV_INSN (insn: twin);
1683	}
1684	while (twin != prev_first);
1685	}
1686
1687	if (this_priority < `0`)
1688	{
1689	gcc_assert (this_priority == -`1`);
1690
1691	this_priority = insn_sched_cost (insn);
1692	}
1693
1694	INSN_PRIORITY (insn) = this_priority;
1695	INSN_PRIORITY_STATUS (insn) = `1`;
1696	}
1697
1698	return INSN_PRIORITY (insn);
1699	}
1700
1701	/ Macros and functions for keeping the priority queue sorted, and*
1702	dealing with queuing and dequeuing of instructions. /*
1703
1704	/ For each pressure class CL, set DEATH[CL] to the number of registers*
1705	in that class that die in INSN. /*
1706
1707	static void
1708	calculate_reg_deaths (rtx_insn insn, int* *death)
1709	{
1710	int i;
1711	struct reg_use_data *use;
1712
1713	for (i = `0`; i < ira_pressure_classes_num; i++)
1714	death[ira_pressure_classes[i]] = `0`;
1715	for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
1716	if (dying_use_p (use))
1717	mark_regno_birth_or_death (live: `0`, pressure: death, regno: use->regno, birth_p: true);
1718	}
1719
1720	/ Setup info about the current register pressure impact of scheduling*
1721	INSN at the current scheduling point. /*
1722	static void
1723	setup_insn_reg_pressure_info (rtx_insn *insn)
1724	{
1725	int i, change, before, after, hard_regno;
1726	int excess_cost_change;
1727	machine_mode mode;
1728	enum reg_class cl;
1729	struct reg_pressure_data *pressure_info;
1730	int *max_reg_pressure;
1731	static int death[N_REG_CLASSES];
1732
1733	gcc_checking_assert (!DEBUG_INSN_P (insn));
1734
1735	excess_cost_change = `0`;
1736	calculate_reg_deaths (insn, death);
1737	pressure_info = INSN_REG_PRESSURE (insn);
1738	max_reg_pressure = INSN_MAX_REG_PRESSURE (insn);
1739	gcc_assert (pressure_info != NULL && max_reg_pressure != NULL);
1740	for (i = `0`; i < ira_pressure_classes_num; i++)
1741	{
1742	cl = ira_pressure_classes[i];
1743	gcc_assert (curr_reg_pressure[cl] >= `0`);
1744	change = (int) pressure_info[i].set_increase - death[cl];
1745	before = MAX (`0`, max_reg_pressure[i] - sched_class_regs_num[cl]);
1746	after = MAX (`0`, max_reg_pressure[i] + change
1747	- sched_class_regs_num[cl]);
1748	hard_regno = ira_class_hard_regs[cl][`0`];
1749	gcc_assert (hard_regno >= `0`);
1750	mode = reg_raw_mode[hard_regno];
1751	excess_cost_change += ((after - before)
1752	* (ira_memory_move_cost[mode][cl][`0`]
1753	+ ira_memory_move_cost[mode][cl][`1`]));
1754	}
1755	INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn) = excess_cost_change;
1756	}
1757
1758	/ This is the first page of code related to SCHED_PRESSURE_MODEL.*
1759	It tries to make the scheduler take register pressure into account
1760	without introducing too many unnecessary stalls. It hooks into the
1761	main scheduling algorithm at several points:
1762
1763	- Before scheduling starts, model_start_schedule constructs a
1764	"model schedule" for the current block. This model schedule is
1765	chosen solely to keep register pressure down. It does not take the
1766	target's pipeline or the original instruction order into account,
1767	except as a tie-breaker. It also doesn't work to a particular
1768	pressure limit.
1769
1770	This model schedule gives us an idea of what pressure can be
1771	achieved for the block and gives us an example of a schedule that
1772	keeps to that pressure. It also makes the final schedule less
1773	dependent on the original instruction order. This is important
1774	because the original order can either be "wide" (many values live
1775	at once, such as in user-scheduled code) or "narrow" (few values
1776	live at once, such as after loop unrolling, where several
1777	iterations are executed sequentially).
1778
1779	We do not apply this model schedule to the rtx stream. We simply
1780	record it in model_schedule. We also compute the maximum pressure,
1781	MP, that was seen during this schedule.
1782
1783	- Instructions are added to the ready queue even if they require
1784	a stall. The length of the stall is instead computed as:
1785
1786	MAX (INSN_TICK (INSN) - clock_var, 0)
1787
1788	(= insn_delay). This allows rank_for_schedule to choose between
1789	introducing a deliberate stall or increasing pressure.
1790
1791	- Before sorting the ready queue, model_set_excess_costs assigns
1792	a pressure-based cost to each ready instruction in the queue.
1793	This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
1794	(ECC for short) and is effectively measured in cycles.
1795
1796	- rank_for_schedule ranks instructions based on:
1797
1798	ECC (insn) + insn_delay (insn)
1799
1800	then as:
1801
1802	insn_delay (insn)
1803
1804	So, for example, an instruction X1 with an ECC of 1 that can issue
1805	now will win over an instruction X0 with an ECC of zero that would
1806	introduce a stall of one cycle. However, an instruction X2 with an
1807	ECC of 2 that can issue now will lose to both X0 and X1.
1808
1809	- When an instruction is scheduled, model_recompute updates the model
1810	schedule with the new pressures (some of which might now exceed the
1811	original maximum pressure MP). model_update_limit_points then searches
1812	for the new point of maximum pressure, if not already known. /*
1813
1814	/ Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL*
1815	from surrounding debug information. /*
1816	#define MODEL_BAR \
1817	";;\t\t+------------------------------------------------------\n"
1818
1819	/ Information about the pressure on a particular register class at a*
1820	particular point of the model schedule. /*
1821	struct model_pressure_data {
1822	/ The pressure at this point of the model schedule, or -1 if the*
1823	point is associated with an instruction that has already been
1824	scheduled. /*
1825	int ref_pressure;
1826
1827	/ The maximum pressure during or after this point of the model schedule. /
1828	int max_pressure;
1829	};
1830
1831	/ Per-instruction information that is used while building the model*
1832	schedule. Here, "schedule" refers to the model schedule rather
1833	than the main schedule. /*
1834	struct model_insn_info {
1835	/ The instruction itself. /
1836	rtx_insn *insn;
1837
1838	/ If this instruction is in model_worklist, these fields link to the*
1839	previous (higher-priority) and next (lower-priority) instructions
1840	in the list. /*
1841	struct model_insn_info *prev;
1842	struct model_insn_info *next;
1843
1844	/ While constructing the schedule, QUEUE_INDEX describes whether an*
1845	instruction has already been added to the schedule (QUEUE_SCHEDULED),
1846	is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
1847	old_queue records the value that QUEUE_INDEX had before scheduling
1848	started, so that we can restore it once the schedule is complete. /*
1849	int old_queue;
1850
1851	/ The relative importance of an unscheduled instruction. Higher*
1852	values indicate greater importance. /*
1853	unsigned int model_priority;
1854
1855	/ The length of the longest path of satisfied true dependencies*
1856	that leads to this instruction. /*
1857	unsigned int depth;
1858
1859	/ The length of the longest path of dependencies of any kind*
1860	that leads from this instruction. /*
1861	unsigned int alap;
1862
1863	/ The number of predecessor nodes that must still be scheduled. /
1864	int unscheduled_preds;
1865	};
1866
1867	/ Information about the pressure limit for a particular register class.*
1868	This structure is used when applying a model schedule to the main
1869	schedule. /*
1870	struct model_pressure_limit {
1871	/ The maximum register pressure seen in the original model schedule. /
1872	int orig_pressure;
1873
1874	/ The maximum register pressure seen in the current model schedule*
1875	(which excludes instructions that have already been scheduled). /*
1876	int pressure;
1877
1878	/ The point of the current model schedule at which PRESSURE is first*
1879	reached. It is set to -1 if the value needs to be recomputed. /*
1880	int point;
1881	};
1882
1883	/ Describes a particular way of measuring register pressure. /
1884	struct model_pressure_group {
1885	/ Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. /
1886	struct model_pressure_limit limits[N_REG_CLASSES];
1887
1888	/ Index (POINT * ira_num_pressure_classes + PCI) describes the pressure*
1889	on register class ira_pressure_classes[PCI] at point POINT of the
1890	current model schedule. A POINT of model_num_insns describes the
1891	pressure at the end of the schedule. /*
1892	struct model_pressure_data *model;
1893	};
1894
1895	/ Index POINT gives the instruction at point POINT of the model schedule.*
1896	This array doesn't change during main scheduling. /*
1897	static vec<rtx_insn *> model_schedule;
1898
1899	/ The list of instructions in the model worklist, sorted in order of*
1900	decreasing priority. /*
1901	static struct model_insn_info *model_worklist;
1902
1903	/ Index I describes the instruction with INSN_LUID I. /
1904	static struct model_insn_info *model_insns;
1905
1906	/ The number of instructions in the model schedule. /
1907	static int model_num_insns;
1908
1909	/ The index of the first instruction in model_schedule that hasn't yet been*
1910	added to the main schedule, or model_num_insns if all of them have. /*
1911	static int model_curr_point;
1912
1913	/ Describes the pressure before each instruction in the model schedule. /
1914	static struct model_pressure_group model_before_pressure;
1915
1916	/ The first unused model_priority value (as used in model_insn_info). /
1917	static unsigned int model_next_priority;
1918
1919
1920	/ The model_pressure_data for ira_pressure_classes[PCI] in GROUP*
1921	at point POINT of the model schedule. /*
1922	#define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
1923	(&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
1924
1925	/ The maximum pressure on ira_pressure_classes[PCI] in GROUP at or*
1926	after point POINT of the model schedule. /*
1927	#define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
1928	(MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
1929
1930	/ The pressure on ira_pressure_classes[PCI] in GROUP at point POINT*
1931	of the model schedule. /*
1932	#define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
1933	(MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
1934
1935	/ Information about INSN that is used when creating the model schedule. /
1936	#define MODEL_INSN_INFO(INSN) \
1937	(&model_insns[INSN_LUID (INSN)])
1938
1939	/ The instruction at point POINT of the model schedule. /
1940	#define MODEL_INSN(POINT) \
1941	(model_schedule[POINT])
1942
1943
1944	/ Return INSN's index in the model schedule, or model_num_insns if it*
1945	doesn't belong to that schedule. /*
1946
1947	static int
1948	model_index (rtx_insn *insn)
1949	{
1950	if (INSN_MODEL_INDEX (insn) == `0`)
1951	return model_num_insns;
1952	return INSN_MODEL_INDEX (insn) - `1`;
1953	}
1954
1955	/ Make sure that GROUP->limits is up-to-date for the current point*
1956	of the model schedule. /*
1957
1958	static void
1959	model_update_limit_points_in_group (struct model_pressure_group *group)
1960	{
1961	int pci, max_pressure, point;
1962
1963	for (pci = `0`; pci < ira_pressure_classes_num; pci++)
1964	{
1965	/ We may have passed the final point at which the pressure in*
1966	group->limits[pci].pressure was reached. Update the limit if so. /*
1967	max_pressure = MODEL_MAX_PRESSURE (group, model_curr_point, pci);
1968	group->limits[pci].pressure = max_pressure;
1969
1970	/ Find the point at which MAX_PRESSURE is first reached. We need*
1971	to search in three cases:
1972
1973	- We've already moved past the previous pressure point.
1974	In this case we search forward from model_curr_point.
1975
1976	- We scheduled the previous point of maximum pressure ahead of
1977	its position in the model schedule, but doing so didn't bring
1978	the pressure point earlier. In this case we search forward
1979	from that previous pressure point.
1980
1981	- Scheduling an instruction early caused the maximum pressure
1982	to decrease. In this case we will have set the pressure
1983	point to -1, and we search forward from model_curr_point. /*
1984	point = MAX (group->limits[pci].point, model_curr_point);
1985	while (point < model_num_insns
1986	&& MODEL_REF_PRESSURE (group, point, pci) < max_pressure)
1987	point++;
1988	group->limits[pci].point = point;
1989
1990	gcc_assert (MODEL_REF_PRESSURE (group, point, pci) == max_pressure);
1991	gcc_assert (MODEL_MAX_PRESSURE (group, point, pci) == max_pressure);
1992	}
1993	}
1994
1995	/ Make sure that all register-pressure limits are up-to-date for the*
1996	current position in the model schedule. /*
1997
1998	static void
1999	model_update_limit_points (void)
2000	{
2001	model_update_limit_points_in_group (group: &model_before_pressure);
2002	}
2003
2004	/ Return the model_index of the last unscheduled use in chain USE*
2005	outside of USE's instruction. Return -1 if there are no other uses,
2006	or model_num_insns if the register is live at the end of the block. /*
2007
2008	static int
2009	model_last_use_except (struct reg_use_data *use)
2010	{
2011	struct reg_use_data *next;
2012	int last, index;
2013
2014	last = -`1`;
2015	for (next = use->next_regno_use; next != use; next = next->next_regno_use)
2016	if (NONDEBUG_INSN_P (next->insn)
2017	&& QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED)
2018	{
2019	index = model_index (insn: next->insn);
2020	if (index == model_num_insns)
2021	return model_num_insns;
2022	if (last < index)
2023	last = index;
2024	}
2025	return last;
2026	}
2027
2028	/ An instruction with model_index POINT has just been scheduled, and it*
2029	adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
2030	Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
2031	MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. /*
2032
2033	static void
2034	model_start_update_pressure (struct model_pressure_group *group,
2035	int point, int pci, int delta)
2036	{
2037	int next_max_pressure;
2038
2039	if (point == model_num_insns)
2040	{
2041	/ The instruction wasn't part of the model schedule; it was moved*
2042	from a different block. Update the pressure for the end of
2043	the model schedule. /*
2044	MODEL_REF_PRESSURE (group, point, pci) += delta;
2045	MODEL_MAX_PRESSURE (group, point, pci) += delta;
2046	}
2047	else
2048	{
2049	/ Record that this instruction has been scheduled. Nothing now*
2050	changes between POINT and POINT + 1, so get the maximum pressure
2051	from the latter. If the maximum pressure decreases, the new
2052	pressure point may be before POINT. /*
2053	MODEL_REF_PRESSURE (group, point, pci) = -`1`;
2054	next_max_pressure = MODEL_MAX_PRESSURE (group, point + `1`, pci);
2055	if (MODEL_MAX_PRESSURE (group, point, pci) > next_max_pressure)
2056	{
2057	MODEL_MAX_PRESSURE (group, point, pci) = next_max_pressure;
2058	if (group->limits[pci].point == point)
2059	group->limits[pci].point = -`1`;
2060	}
2061	}
2062	}
2063
2064	/ Record that scheduling a later instruction has changed the pressure*
2065	at point POINT of the model schedule by DELTA (which might be 0).
2066	Update GROUP accordingly. Return nonzero if these changes might
2067	trigger changes to previous points as well. /*
2068
2069	static int
2070	model_update_pressure (struct model_pressure_group *group,
2071	int point, int pci, int delta)
2072	{
2073	int ref_pressure, max_pressure, next_max_pressure;
2074
2075	/ If POINT hasn't yet been scheduled, update its pressure. /
2076	ref_pressure = MODEL_REF_PRESSURE (group, point, pci);
2077	if (ref_pressure >= `0` && delta != `0`)
2078	{
2079	ref_pressure += delta;
2080	MODEL_REF_PRESSURE (group, point, pci) = ref_pressure;
2081
2082	/ Check whether the maximum pressure in the overall schedule*
2083	has increased. (This means that the MODEL_MAX_PRESSURE of
2084	every point <= POINT will need to increase too; see below.) /*
2085	if (group->limits[pci].pressure < ref_pressure)
2086	group->limits[pci].pressure = ref_pressure;
2087
2088	/ If we are at maximum pressure, and the maximum pressure*
2089	point was previously unknown or later than POINT,
2090	bring it forward. /*
2091	if (group->limits[pci].pressure == ref_pressure
2092	&& !IN_RANGE (group->limits[pci].point, `0`, point))
2093	group->limits[pci].point = point;
2094
2095	/ If POINT used to be the point of maximum pressure, but isn't*
2096	any longer, we need to recalculate it using a forward walk. /*
2097	if (group->limits[pci].pressure > ref_pressure
2098	&& group->limits[pci].point == point)
2099	group->limits[pci].point = -`1`;
2100	}
2101
2102	/ Update the maximum pressure at POINT. Changes here might also*
2103	affect the maximum pressure at POINT - 1. /*
2104	next_max_pressure = MODEL_MAX_PRESSURE (group, point + `1`, pci);
2105	max_pressure = MAX (ref_pressure, next_max_pressure);
2106	if (MODEL_MAX_PRESSURE (group, point, pci) != max_pressure)
2107	{
2108	MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
2109	return `1`;
2110	}
2111	return `0`;
2112	}
2113
2114	/ INSN has just been scheduled. Update the model schedule accordingly. /
2115
2116	static void
2117	model_recompute (rtx_insn *insn)
2118	{
2119	struct {
2120	int last_use;
2121	int regno;
2122	} uses[FIRST_PSEUDO_REGISTER + MAX_RECOG_OPERANDS];
2123	struct reg_use_data *use;
2124	struct reg_pressure_data *reg_pressure;
2125	int delta[N_REG_CLASSES];
2126	int pci, point, mix, new_last, cl, ref_pressure, queue;
2127	unsigned int i, num_uses, num_pending_births;
2128	bool print_p;
2129
2130	/ The destinations of INSN were previously live from POINT onwards, but are*
2131	now live from model_curr_point onwards. Set up DELTA accordingly. /*
2132	point = model_index (insn);
2133	reg_pressure = INSN_REG_PRESSURE (insn);
2134	for (pci = `0`; pci < ira_pressure_classes_num; pci++)
2135	{
2136	cl = ira_pressure_classes[pci];
2137	delta[cl] = reg_pressure[pci].set_increase;
2138	}
2139
2140	/ Record which registers previously died at POINT, but which now die*
2141	before POINT. Adjust DELTA so that it represents the effect of
2142	this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of
2143	registers that will be born in the range [model_curr_point, POINT). /*
2144	num_uses = `0`;
2145	num_pending_births = `0`;
2146	bitmap_clear (tmp_bitmap);
2147	for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
2148	{
2149	new_last = model_last_use_except (use);
2150	if (new_last < point && bitmap_set_bit (tmp_bitmap, use->regno))
2151	{
2152	gcc_assert (num_uses < ARRAY_SIZE (uses));
2153	uses[num_uses].last_use = new_last;
2154	uses[num_uses].regno = use->regno;
2155	/ This register is no longer live after POINT - 1. /
2156	mark_regno_birth_or_death (NULL, pressure: delta, regno: use->regno, birth_p: false);
2157	num_uses++;
2158	if (new_last >= `0`)
2159	num_pending_births++;
2160	}
2161	}
2162
2163	/ Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.*
2164	Also set each group pressure limit for POINT. /*
2165	for (pci = `0`; pci < ira_pressure_classes_num; pci++)
2166	{
2167	cl = ira_pressure_classes[pci];
2168	model_start_update_pressure (group: &model_before_pressure,
2169	point, pci, delta: delta[cl]);
2170	}
2171
2172	/ Walk the model schedule backwards, starting immediately before POINT. /
2173	print_p = false;
2174	if (point != model_curr_point)
2175	do
2176	{
2177	point--;
2178	insn = MODEL_INSN (point);
2179	queue = QUEUE_INDEX (insn);
2180
2181	if (queue != QUEUE_SCHEDULED)
2182	{
2183	/ DELTA describes the effect of the move on the register pressure*
2184	after POINT. Make it describe the effect on the pressure
2185	before POINT. /*
2186	i = `0`;
2187	while (i < num_uses)
2188	{
2189	if (uses[i].last_use == point)
2190	{
2191	/ This register is now live again. /
2192	mark_regno_birth_or_death (NULL, pressure: delta,
2193	regno: uses[i].regno, birth_p: true);
2194
2195	/ Remove this use from the array. /
2196	uses[i] = uses[num_uses - `1`];
2197	num_uses--;
2198	num_pending_births--;
2199	}
2200	else
2201	i++;
2202	}
2203
2204	if (sched_verbose >= `5`)
2205	{
2206	if (!print_p)
2207	{
2208	fprintf (stream: sched_dump, MODEL_BAR);
2209	fprintf (stream: sched_dump, format: ";;\t\t\| New pressure for model"
2210	" schedule\n");
2211	fprintf (stream: sched_dump, MODEL_BAR);
2212	print_p = true;
2213	}
2214
2215	fprintf (stream: sched_dump, format: ";;\t\t\| %3d %4d %-30s ",
2216	point, INSN_UID (insn),
2217	str_pattern_slim (PATTERN (insn)));
2218	for (pci = `0`; pci < ira_pressure_classes_num; pci++)
2219	{
2220	cl = ira_pressure_classes[pci];
2221	ref_pressure = MODEL_REF_PRESSURE (&model_before_pressure,
2222	point, pci);
2223	fprintf (stream: sched_dump, format: " %s:[%d->%d]",
2224	reg_class_names[ira_pressure_classes[pci]],
2225	ref_pressure, ref_pressure + delta[cl]);
2226	}
2227	fprintf (stream: sched_dump, format: "\n");
2228	}
2229	}
2230
2231	/ Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1*
2232	might have changed as well. /*
2233	mix = num_pending_births;
2234	for (pci = `0`; pci < ira_pressure_classes_num; pci++)
2235	{
2236	cl = ira_pressure_classes[pci];
2237	mix \|= delta[cl];
2238	mix \|= model_update_pressure (group: &model_before_pressure,
2239	point, pci, delta: delta[cl]);
2240	}
2241	}
2242	while (mix && point > model_curr_point);
2243
2244	if (print_p)
2245	fprintf (stream: sched_dump, MODEL_BAR);
2246	}
2247
2248	/ After DEP, which was cancelled, has been resolved for insn NEXT,*
2249	check whether the insn's pattern needs restoring. /*
2250	static bool
2251	must_restore_pattern_p (rtx_insn *next, dep_t dep)
2252	{
2253	if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
2254	return false;
2255
2256	if (DEP_TYPE (dep) == REG_DEP_CONTROL)
2257	{
2258	gcc_assert (ORIG_PAT (next) != NULL_RTX);
2259	gcc_assert (next == DEP_CON (dep));
2260	}
2261	else
2262	{
2263	struct dep_replacement *desc = DEP_REPLACE (dep);
2264	if (desc->insn != next)
2265	{
2266	gcc_assert (*desc->loc == desc->orig);
2267	return false;
2268	}
2269	}
2270	return true;
2271	}
2272
2273	/ model_spill_cost (CL, P, P') returns the cost of increasing the*
2274	pressure on CL from P to P'. We use this to calculate a "base ECC",
2275	baseECC (CL, X), for each pressure class CL and each instruction X.
2276	Supposing X changes the pressure on CL from P to P', and that the
2277	maximum pressure on CL in the current model schedule is MP', then:
2278
2279	* if X occurs before or at the next point of maximum pressure in
2280	the model schedule and P' > MP', then:
2281
2282	baseECC (CL, X) = model_spill_cost (CL, MP, P')
2283
2284	The idea is that the pressure after scheduling a fixed set of
2285	instructions -- in this case, the set up to and including the
2286	next maximum pressure point -- is going to be the same regardless
2287	of the order; we simply want to keep the intermediate pressure
2288	under control. Thus X has a cost of zero unless scheduling it
2289	now would exceed MP'.
2290
2291	If all increases in the set are by the same amount, no zero-cost
2292	instruction will ever cause the pressure to exceed MP'. However,
2293	if X is instead moved past an instruction X' with pressure in the
2294	range (MP' - (P' - P), MP'), the pressure at X' will increase
2295	beyond MP'. Since baseECC is very much a heuristic anyway,
2296	it doesn't seem worth the overhead of tracking cases like these.
2297
2298	The cost of exceeding MP' is always based on the original maximum
2299	pressure MP. This is so that going 2 registers over the original
2300	limit has the same cost regardless of whether it comes from two
2301	separate +1 deltas or from a single +2 delta.
2302
2303	* if X occurs after the next point of maximum pressure in the model
2304	schedule and P' > P, then:
2305
2306	baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
2307
2308	That is, if we move X forward across a point of maximum pressure,
2309	and if X increases the pressure by P' - P, then we conservatively
2310	assume that scheduling X next would increase the maximum pressure
2311	by P' - P. Again, the cost of doing this is based on the original
2312	maximum pressure MP, for the same reason as above.
2313
2314	* if P' < P, P > MP, and X occurs at or after the next point of
2315	maximum pressure, then:
2316
2317	baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
2318
2319	That is, if we have already exceeded the original maximum pressure MP,
2320	and if X might reduce the maximum pressure again -- or at least push
2321	it further back, and thus allow more scheduling freedom -- it is given
2322	a negative cost to reflect the improvement.
2323
2324	* otherwise,
2325
2326	baseECC (CL, X) = 0
2327
2328	In this case, X is not expected to affect the maximum pressure MP',
2329	so it has zero cost.
2330
2331	We then create a combined value baseECC (X) that is the sum of
2332	baseECC (CL, X) for each pressure class CL.
2333
2334	baseECC (X) could itself be used as the ECC value described above.
2335	However, this is often too conservative, in the sense that it
2336	tends to make high-priority instructions that increase pressure
2337	wait too long in cases where introducing a spill would be better.
2338	For this reason the final ECC is a priority-adjusted form of
2339	baseECC (X). Specifically, we calculate:
2340
2341	P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
2342	baseP = MAX { P (X) \| baseECC (X) <= 0 }
2343
2344	Then:
2345
2346	ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
2347
2348	Thus an instruction's effect on pressure is ignored if it has a high
2349	enough priority relative to the ones that don't increase pressure.
2350	Negative values of baseECC (X) do not increase the priority of X
2351	itself, but they do make it harder for other instructions to
2352	increase the pressure further.
2353
2354	This pressure cost is deliberately timid. The intention has been
2355	to choose a heuristic that rarely interferes with the normal list
2356	scheduler in cases where that scheduler would produce good code.
2357	We simply want to curb some of its worst excesses. /*
2358
2359	/ Return the cost of increasing the pressure in class CL from FROM to TO.*
2360
2361	Here we use the very simplistic cost model that every register above
2362	sched_class_regs_num[CL] has a spill cost of 1. We could use other
2363	measures instead, such as one based on MEMORY_MOVE_COST. However:
2364
2365	(1) In order for an instruction to be scheduled, the higher cost
2366	would need to be justified in a single saving of that many stalls.
2367	This is overly pessimistic, because the benefit of spilling is
2368	often to avoid a sequence of several short stalls rather than
2369	a single long one.
2370
2371	(2) The cost is still arbitrary. Because we are not allocating
2372	registers during scheduling, we have no way of knowing for
2373	sure how many memory accesses will be required by each spill,
2374	where the spills will be placed within the block, or even
2375	which block(s) will contain the spills.
2376
2377	So a higher cost than 1 is often too conservative in practice,
2378	forcing blocks to contain unnecessary stalls instead of spill code.
2379	The simple cost below seems to be the best compromise. It reduces
2380	the interference with the normal list scheduler, which helps make
2381	it more suitable for a default-on option. /*
2382
2383	static int
2384	model_spill_cost (int cl, int from, int to)
2385	{
2386	from = MAX (from, sched_class_regs_num[cl]);
2387	return MAX (to, from) - from;
2388	}
2389
2390	/ Return baseECC (ira_pressure_classes[PCI], POINT), given that*
2391	P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
2392	P' = P + DELTA. /*
2393
2394	static int
2395	model_excess_group_cost (struct model_pressure_group *group,
2396	int point, int pci, int delta)
2397	{
2398	int pressure, cl;
2399
2400	cl = ira_pressure_classes[pci];
2401	if (delta < `0` && point >= group->limits[pci].point)
2402	{
2403	pressure = MAX (group->limits[pci].orig_pressure,
2404	curr_reg_pressure[cl] + delta);
2405	return -model_spill_cost (cl, from: pressure, to: curr_reg_pressure[cl]);
2406	}
2407
2408	if (delta > `0`)
2409	{
2410	if (point > group->limits[pci].point)
2411	pressure = group->limits[pci].pressure + delta;
2412	else
2413	pressure = curr_reg_pressure[cl] + delta;
2414
2415	if (pressure > group->limits[pci].pressure)
2416	return model_spill_cost (cl, from: group->limits[pci].orig_pressure,
2417	to: pressure);
2418	}
2419
2420	return `0`;
2421	}
2422
2423	/ Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump*
2424	if PRINT_P. /*
2425
2426	static int
2427	model_excess_cost (rtx_insn insn, bool* print_p)
2428	{
2429	int point, pci, cl, cost, this_cost, delta;
2430	struct reg_pressure_data *insn_reg_pressure;
2431	int insn_death[N_REG_CLASSES];
2432
2433	calculate_reg_deaths (insn, death: insn_death);
2434	point = model_index (insn);
2435	insn_reg_pressure = INSN_REG_PRESSURE (insn);
2436	cost = `0`;
2437
2438	if (print_p)
2439	fprintf (stream: sched_dump, format: ";;\t\t\| %3d %4d \| %4d %+3d \|", point,
2440	INSN_UID (insn), INSN_PRIORITY (insn), insn_delay (insn));
2441
2442	/ Sum up the individual costs for each register class. /
2443	for (pci = `0`; pci < ira_pressure_classes_num; pci++)
2444	{
2445	cl = ira_pressure_classes[pci];
2446	delta = insn_reg_pressure[pci].set_increase - insn_death[cl];
2447	this_cost = model_excess_group_cost (group: &model_before_pressure,
2448	point, pci, delta);
2449	cost += this_cost;
2450	if (print_p)
2451	fprintf (stream: sched_dump, format: " %s:[%d base cost %d]",
2452	reg_class_names[cl], delta, this_cost);
2453	}
2454
2455	if (print_p)
2456	fprintf (stream: sched_dump, format: "\n");
2457
2458	return cost;
2459	}
2460
2461	/ Dump the next points of maximum pressure for GROUP. /
2462
2463	static void
2464	model_dump_pressure_points (struct model_pressure_group *group)
2465	{
2466	int pci, cl;
2467
2468	fprintf (stream: sched_dump, format: ";;\t\t\| pressure points");
2469	for (pci = `0`; pci < ira_pressure_classes_num; pci++)
2470	{
2471	cl = ira_pressure_classes[pci];
2472	fprintf (stream: sched_dump, format: " %s:[%d->%d at ", reg_class_names[cl],
2473	curr_reg_pressure[cl], group->limits[pci].pressure);
2474	if (group->limits[pci].point < model_num_insns)
2475	fprintf (stream: sched_dump, format: "%d:%d]", group->limits[pci].point,
2476	INSN_UID (MODEL_INSN (group->limits[pci].point)));
2477	else
2478	fprintf (stream: sched_dump, format: "end]");
2479	}
2480	fprintf (stream: sched_dump, format: "\n");
2481	}
2482
2483	/ Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. /
2484
2485	static void
2486	model_set_excess_costs (rtx_insn *insns, int* count)
2487	{
2488	int i, cost, priority_base, priority;
2489	bool print_p;
2490
2491	/ Record the baseECC value for each instruction in the model schedule,*
2492	except that negative costs are converted to zero ones now rather than
2493	later. Do not assign a cost to debug instructions, since they must
2494	not change code-generation decisions. Experiments suggest we also
2495	get better results by not assigning a cost to instructions from
2496	a different block.
2497
2498	Set PRIORITY_BASE to baseP in the block comment above. This is the
2499	maximum priority of the "cheap" instructions, which should always
2500	include the next model instruction. /*
2501	priority_base = `0`;
2502	print_p = false;
2503	for (i = `0`; i < count; i++)
2504	if (INSN_MODEL_INDEX (insns[i]))
2505	{
2506	if (sched_verbose >= `6` && !print_p)
2507	{
2508	fprintf (stream: sched_dump, MODEL_BAR);
2509	fprintf (stream: sched_dump, format: ";;\t\t\| Pressure costs for ready queue\n");
2510	model_dump_pressure_points (group: &model_before_pressure);
2511	fprintf (stream: sched_dump, MODEL_BAR);
2512	print_p = true;
2513	}
2514	cost = model_excess_cost (insn: insns[i], print_p);
2515	if (cost <= `0`)
2516	{
2517	priority = INSN_PRIORITY (insns[i]) - insn_delay (insn: insns[i]) - cost;
2518	priority_base = MAX (priority_base, priority);
2519	cost = `0`;
2520	}
2521	INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = cost;
2522	}
2523	if (print_p)
2524	fprintf (stream: sched_dump, MODEL_BAR);
2525
2526	/ Use MAX (baseECC, 0) and baseP to calculcate ECC for each*
2527	instruction. /*
2528	for (i = `0`; i < count; i++)
2529	{
2530	cost = INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]);
2531	priority = INSN_PRIORITY (insns[i]) - insn_delay (insn: insns[i]);
2532	if (cost > `0` && priority > priority_base)
2533	{
2534	cost += priority_base - priority;
2535	INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = MAX (cost, `0`);
2536	}
2537	}
2538	}
2539
2540
2541	/ Enum of rank_for_schedule heuristic decisions. /
2542	enum rfs_decision {
2543	RFS_LIVE_RANGE_SHRINK1, RFS_LIVE_RANGE_SHRINK2,
2544	RFS_SCHED_GROUP, RFS_PRESSURE_DELAY, RFS_PRESSURE_TICK,
2545	RFS_FEEDS_BACKTRACK_INSN, RFS_PRIORITY, RFS_AUTOPREF, RFS_SPECULATION,
2546	RFS_SCHED_RANK, RFS_LAST_INSN, RFS_PRESSURE_INDEX,
2547	RFS_DEP_COUNT, RFS_TIE, RFS_FUSION, RFS_COST, RFS_N };
2548
2549	/ Corresponding strings for print outs. /
2550	static const char *rfs_str[RFS_N] = {
2551	"RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
2552	"RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
2553	"RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_AUTOPREF", "RFS_SPECULATION",
2554	"RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
2555	"RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION", "RFS_COST" };
2556
2557	/ Statistical breakdown of rank_for_schedule decisions. /
2558	struct rank_for_schedule_stats_t { unsigned stats[RFS_N]; };
2559	static rank_for_schedule_stats_t rank_for_schedule_stats;
2560
2561	/ Return the result of comparing insns TMP and TMP2 and update*
2562	Rank_For_Schedule statistics. /*
2563	static int
2564	rfs_result (enum rfs_decision decision, int result, rtx tmp, rtx tmp2)
2565	{
2566	++rank_for_schedule_stats.stats[decision];
2567	if (result < `0`)
2568	INSN_LAST_RFS_WIN (tmp) = decision;
2569	else if (result > `0`)
2570	INSN_LAST_RFS_WIN (tmp2) = decision;
2571	else
2572	gcc_unreachable ();
2573	return result;
2574	}
2575
2576	/ Sorting predicate to move DEBUG_INSNs to the top of ready list, while*
2577	keeping normal insns in original order. /*
2578
2579	static int
2580	rank_for_schedule_debug (const void x, const* void *y)
2581	{
2582	rtx_insn tmp = (rtx_insn * const *) y;
2583	rtx_insn tmp2 = (rtx_insn * const *) x;
2584
2585	/ Schedule debug insns as early as possible. /
2586	if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2))
2587	return -`1`;
2588	else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
2589	return `1`;
2590	else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
2591	return INSN_LUID (tmp) - INSN_LUID (tmp2);
2592	else
2593	return INSN_RFS_DEBUG_ORIG_ORDER (tmp2) - INSN_RFS_DEBUG_ORIG_ORDER (tmp);
2594	}
2595
2596	/ Returns a positive value if x is preferred; returns a negative value if*
2597	y is preferred. Should never return 0, since that will make the sort
2598	unstable. /*
2599
2600	static int
2601	rank_for_schedule (const void x, const* void *y)
2602	{
2603	rtx_insn tmp = (rtx_insn * const *) y;
2604	rtx_insn tmp2 = (rtx_insn * const *) x;
2605	int tmp_class, tmp2_class;
2606	int val, priority_val, info_val, diff;
2607
2608	if (live_range_shrinkage_p)
2609	{
2610	/ Don't use SCHED_PRESSURE_MODEL -- it results in much worse*
2611	code. /*
2612	gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
2613	if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) < `0`
2614	\|\| INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) < `0`)
2615	&& (diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
2616	- INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2))) != `0`)
2617	return rfs_result (decision: RFS_LIVE_RANGE_SHRINK1, result: diff, tmp, tmp2);
2618	/ Sort by INSN_LUID (original insn order), so that we make the*
2619	sort stable. This minimizes instruction movement, thus
2620	minimizing sched's effect on debugging and cross-jumping. /*
2621	return rfs_result (decision: RFS_LIVE_RANGE_SHRINK2,
2622	INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2623	}
2624
2625	/ The insn in a schedule group should be issued the first. /
2626	if (flag_sched_group_heuristic &&
2627	SCHED_GROUP_P (tmp) != SCHED_GROUP_P (tmp2))
2628	return rfs_result (decision: RFS_SCHED_GROUP, SCHED_GROUP_P (tmp2) ? `1` : -`1`,
2629	tmp, tmp2);
2630
2631	/ Make sure that priority of TMP and TMP2 are initialized. /
2632	gcc_assert (INSN_PRIORITY_KNOWN (tmp) && INSN_PRIORITY_KNOWN (tmp2));
2633
2634	if (sched_fusion)
2635	{
2636	/ The instruction that has the same fusion priority as the last*
2637	instruction is the instruction we picked next. If that is not
2638	the case, we sort ready list firstly by fusion priority, then
2639	by priority, and at last by INSN_LUID. /*
2640	int a = INSN_FUSION_PRIORITY (tmp);
2641	int b = INSN_FUSION_PRIORITY (tmp2);
2642	int last = -`1`;
2643
2644	if (last_nondebug_scheduled_insn
2645	&& !NOTE_P (last_nondebug_scheduled_insn)
2646	&& BLOCK_FOR_INSN (insn: tmp)
2647	== BLOCK_FOR_INSN (insn: last_nondebug_scheduled_insn))
2648	last = INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn);
2649
2650	if (a != last && b != last)
2651	{
2652	if (a == b)
2653	{
2654	a = INSN_PRIORITY (tmp);
2655	b = INSN_PRIORITY (tmp2);
2656	}
2657	if (a != b)
2658	return rfs_result (decision: RFS_FUSION, result: b - a, tmp, tmp2);
2659	else
2660	return rfs_result (decision: RFS_FUSION,
2661	INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2662	}
2663	else if (a == b)
2664	{
2665	gcc_assert (last_nondebug_scheduled_insn
2666	&& !NOTE_P (last_nondebug_scheduled_insn));
2667	last = INSN_PRIORITY (last_nondebug_scheduled_insn);
2668
2669	a = abs (INSN_PRIORITY (tmp) - last);
2670	b = abs (INSN_PRIORITY (tmp2) - last);
2671	if (a != b)
2672	return rfs_result (decision: RFS_FUSION, result: a - b, tmp, tmp2);
2673	else
2674	return rfs_result (decision: RFS_FUSION,
2675	INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2676	}
2677	else if (a == last)
2678	return rfs_result (decision: RFS_FUSION, result: -`1`, tmp, tmp2);
2679	else
2680	return rfs_result (decision: RFS_FUSION, result: `1`, tmp, tmp2);
2681	}
2682
2683	if (sched_pressure != SCHED_PRESSURE_NONE)
2684	{
2685	/ Prefer insn whose scheduling results in the smallest register*
2686	pressure excess. /*
2687	if ((diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
2688	+ insn_delay (insn: tmp)
2689	- INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2)
2690	- insn_delay (insn: tmp2))))
2691	return rfs_result (decision: RFS_PRESSURE_DELAY, result: diff, tmp, tmp2);
2692	}
2693
2694	if (sched_pressure != SCHED_PRESSURE_NONE
2695	&& (INSN_TICK (tmp2) > clock_var \|\| INSN_TICK (tmp) > clock_var)
2696	&& INSN_TICK (tmp2) != INSN_TICK (tmp))
2697	{
2698	diff = INSN_TICK (tmp) - INSN_TICK (tmp2);
2699	return rfs_result (decision: RFS_PRESSURE_TICK, result: diff, tmp, tmp2);
2700	}
2701
2702	/ If we are doing backtracking in this schedule, prefer insns that*
2703	have forward dependencies with negative cost against an insn that
2704	was already scheduled. /*
2705	if (current_sched_info->flags & DO_BACKTRACKING)
2706	{
2707	priority_val = FEEDS_BACKTRACK_INSN (tmp2) - FEEDS_BACKTRACK_INSN (tmp);
2708	if (priority_val)
2709	return rfs_result (decision: RFS_FEEDS_BACKTRACK_INSN, result: priority_val, tmp, tmp2);
2710	}
2711
2712	/ Prefer insn with higher priority. /
2713	priority_val = INSN_PRIORITY (tmp2) - INSN_PRIORITY (tmp);
2714
2715	if (flag_sched_critical_path_heuristic && priority_val)
2716	return rfs_result (decision: RFS_PRIORITY, result: priority_val, tmp, tmp2);
2717
2718	if (param_sched_autopref_queue_depth >= `0`)
2719	{
2720	int autopref = autopref_rank_for_schedule (tmp, tmp2);
2721	if (autopref != `0`)
2722	return rfs_result (decision: RFS_AUTOPREF, result: autopref, tmp, tmp2);
2723	}
2724
2725	/ Prefer speculative insn with greater dependencies weakness. /
2726	if (flag_sched_spec_insn_heuristic && spec_info)
2727	{
2728	ds_t ds1, ds2;
2729	dw_t dw1, dw2;
2730	int dw;
2731
2732	ds1 = TODO_SPEC (tmp) & SPECULATIVE;
2733	if (ds1)
2734	dw1 = ds_weak (ds1);
2735	else
2736	dw1 = NO_DEP_WEAK;
2737
2738	ds2 = TODO_SPEC (tmp2) & SPECULATIVE;
2739	if (ds2)
2740	dw2 = ds_weak (ds2);
2741	else
2742	dw2 = NO_DEP_WEAK;
2743
2744	dw = dw2 - dw1;
2745	if (dw > (NO_DEP_WEAK / `8`) \|\| dw < -(NO_DEP_WEAK / `8`))
2746	return rfs_result (decision: RFS_SPECULATION, result: dw, tmp, tmp2);
2747	}
2748
2749	info_val = (*current_sched_info->rank) (tmp, tmp2);
2750	if (flag_sched_rank_heuristic && info_val)
2751	return rfs_result (decision: RFS_SCHED_RANK, result: info_val, tmp, tmp2);
2752
2753	/ Compare insns based on their relation to the last scheduled*
2754	non-debug insn. /*
2755	if (flag_sched_last_insn_heuristic && last_nondebug_scheduled_insn)
2756	{
2757	dep_t dep1;
2758	dep_t dep2;
2759	rtx_insn *last = last_nondebug_scheduled_insn;
2760
2761	/ Classify the instructions into three classes:*
2762	1) Data dependent on last schedule insn.
2763	2) Anti/Output dependent on last scheduled insn.
2764	3) Independent of last scheduled insn, or has latency of one.
2765	Choose the insn from the highest numbered class if different. /*
2766	dep1 = sd_find_dep_between (last, tmp, true);
2767
2768	if (dep1 == NULL \|\| dep_cost (link: dep1) == `1`)
2769	tmp_class = `3`;
2770	else if (/ Data dependence. /
2771	DEP_TYPE (dep1) == REG_DEP_TRUE)
2772	tmp_class = `1`;
2773	else
2774	tmp_class = `2`;
2775
2776	dep2 = sd_find_dep_between (last, tmp2, true);
2777
2778	if (dep2 == NULL \|\| dep_cost (link: dep2) == `1`)
2779	tmp2_class = `3`;
2780	else if (/ Data dependence. /
2781	DEP_TYPE (dep2) == REG_DEP_TRUE)
2782	tmp2_class = `1`;
2783	else
2784	tmp2_class = `2`;
2785
2786	if ((val = tmp2_class - tmp_class))
2787	return rfs_result (decision: RFS_LAST_INSN, result: val, tmp, tmp2);
2788	}
2789
2790	/ Prefer instructions that occur earlier in the model schedule. /
2791	if (sched_pressure == SCHED_PRESSURE_MODEL)
2792	{
2793	diff = model_index (insn: tmp) - model_index (insn: tmp2);
2794	if (diff != `0`)
2795	return rfs_result (decision: RFS_PRESSURE_INDEX, result: diff, tmp, tmp2);
2796	}
2797
2798	/ Prefer the insn which has more later insns that depend on it.*
2799	This gives the scheduler more freedom when scheduling later
2800	instructions at the expense of added register pressure. /*
2801
2802	val = (dep_list_size (insn: tmp2, SD_LIST_FORW)
2803	- dep_list_size (insn: tmp, SD_LIST_FORW));
2804
2805	if (flag_sched_dep_count_heuristic && val != `0`)
2806	return rfs_result (decision: RFS_DEP_COUNT, result: val, tmp, tmp2);
2807
2808	/ Sort by INSN_COST rather than INSN_LUID. This means that instructions*
2809	which take longer to execute are prioritised and it leads to more
2810	dual-issue opportunities on in-order cores which have this feature. /*
2811
2812	if (INSN_COST (tmp) != INSN_COST (tmp2))
2813	return rfs_result (decision: RFS_COST, INSN_COST (tmp2) - INSN_COST (tmp),
2814	tmp, tmp2);
2815
2816	/ If insns are equally good, sort by INSN_LUID (original insn order),*
2817	so that we make the sort stable. This minimizes instruction movement,
2818	thus minimizing sched's effect on debugging and cross-jumping. /*
2819	return rfs_result (decision: RFS_TIE, INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2820	}
2821
2822	/ Resort the array A in which only element at index N may be out of order. /
2823
2824	HAIFA_INLINE static void
2825	swap_sort (rtx_insn *a, int* n)
2826	{
2827	rtx_insn *insn = a[n - `1`];
2828	int i = n - `2`;
2829
2830	while (i >= `0` && rank_for_schedule (x: a + i, y: &insn) >= `0`)
2831	{
2832	a[i + `1`] = a[i];
2833	i -= `1`;
2834	}
2835	a[i + `1`] = insn;
2836	}
2837
2838	/ Add INSN to the insn queue so that it can be executed at least*
2839	N_CYCLES after the currently executing insn. Preserve insns
2840	chain for debugging purposes. REASON will be printed in debugging
2841	output. /*
2842
2843	HAIFA_INLINE static void
2844	queue_insn (rtx_insn insn, int* n_cycles, const char *reason)
2845	{
2846	int next_q = NEXT_Q_AFTER (q_ptr, n_cycles);
2847	rtx_insn_list *link = alloc_INSN_LIST (insn, insn_queue[next_q]);
2848	int new_tick;
2849
2850	gcc_assert (n_cycles <= max_insn_queue_index);
2851	gcc_assert (!DEBUG_INSN_P (insn));
2852
2853	insn_queue[next_q] = link;
2854	q_size += `1`;
2855
2856	if (sched_verbose >= `2`)
2857	{
2858	fprintf (stream: sched_dump, format: ";;\t\tReady-->Q: insn %s: ",
2859	(*current_sched_info->print_insn) (insn, `0`));
2860
2861	fprintf (stream: sched_dump, format: "queued for %d cycles (%s).\n", n_cycles, reason);
2862	}
2863
2864	QUEUE_INDEX (insn) = next_q;
2865
2866	if (current_sched_info->flags & DO_BACKTRACKING)
2867	{
2868	new_tick = clock_var + n_cycles;
2869	if (INSN_TICK (insn) == INVALID_TICK \|\| INSN_TICK (insn) < new_tick)
2870	INSN_TICK (insn) = new_tick;
2871
2872	if (INSN_EXACT_TICK (insn) != INVALID_TICK
2873	&& INSN_EXACT_TICK (insn) < clock_var + n_cycles)
2874	{
2875	must_backtrack = true;
2876	if (sched_verbose >= `2`)
2877	fprintf (stream: sched_dump, format: ";;\t\tcausing a backtrack.\n");
2878	}
2879	}
2880	}
2881
2882	/ Remove INSN from queue. /
2883	static void
2884	queue_remove (rtx_insn *insn)
2885	{
2886	gcc_assert (QUEUE_INDEX (insn) >= `0`);
2887	remove_free_INSN_LIST_elem (insn, &insn_queue[QUEUE_INDEX (insn)]);
2888	q_size--;
2889	QUEUE_INDEX (insn) = QUEUE_NOWHERE;
2890	}
2891
2892	/ Return a pointer to the bottom of the ready list, i.e. the insn*
2893	with the lowest priority. /*
2894
2895	rtx_insn **
2896	ready_lastpos (struct ready_list *ready)
2897	{
2898	gcc_assert (ready->n_ready >= `1`);
2899	return ready->vec + ready->first - ready->n_ready + `1`;
2900	}
2901
2902	/ Add an element INSN to the ready list so that it ends up with the*
2903	lowest/highest priority depending on FIRST_P. /*
2904
2905	HAIFA_INLINE static void
2906	ready_add (struct ready_list ready, rtx_insn insn, bool first_p)
2907	{
2908	if (!first_p)
2909	{
2910	if (ready->first == ready->n_ready)
2911	{
2912	memmove (dest: ready->vec + ready->veclen - ready->n_ready,
2913	src: ready_lastpos (ready),
2914	n: ready->n_ready * sizeof (rtx));
2915	ready->first = ready->veclen - `1`;
2916	}
2917	ready->vec[ready->first - ready->n_ready] = insn;
2918	}
2919	else
2920	{
2921	if (ready->first == ready->veclen - `1`)
2922	{
2923	if (ready->n_ready)
2924	/ ready_lastpos() fails when called with (ready->n_ready == 0). /
2925	memmove (dest: ready->vec + ready->veclen - ready->n_ready - `1`,
2926	src: ready_lastpos (ready),
2927	n: ready->n_ready * sizeof (rtx));
2928	ready->first = ready->veclen - `2`;
2929	}
2930	ready->vec[++(ready->first)] = insn;
2931	}
2932
2933	ready->n_ready++;
2934	if (DEBUG_INSN_P (insn))
2935	ready->n_debug++;
2936
2937	gcc_assert (QUEUE_INDEX (insn) != QUEUE_READY);
2938	QUEUE_INDEX (insn) = QUEUE_READY;
2939
2940	if (INSN_EXACT_TICK (insn) != INVALID_TICK
2941	&& INSN_EXACT_TICK (insn) < clock_var)
2942	{
2943	must_backtrack = true;
2944	}
2945	}
2946
2947	/ Remove the element with the highest priority from the ready list and*
2948	return it. /*
2949
2950	HAIFA_INLINE static rtx_insn *
2951	ready_remove_first (struct ready_list *ready)
2952	{
2953	rtx_insn *t;
2954
2955	gcc_assert (ready->n_ready);
2956	t = ready->vec[ready->first--];
2957	ready->n_ready--;
2958	if (DEBUG_INSN_P (t))
2959	ready->n_debug--;
2960	/ If the queue becomes empty, reset it. /
2961	if (ready->n_ready == `0`)
2962	ready->first = ready->veclen - `1`;
2963
2964	gcc_assert (QUEUE_INDEX (t) == QUEUE_READY);
2965	QUEUE_INDEX (t) = QUEUE_NOWHERE;
2966
2967	return t;
2968	}
2969
2970	/ The following code implements multi-pass scheduling for the first*
2971	cycle. In other words, we will try to choose ready insn which
2972	permits to start maximum number of insns on the same cycle. /*
2973
2974	/ Return a pointer to the element INDEX from the ready. INDEX for*
2975	insn with the highest priority is 0, and the lowest priority has
2976	N_READY - 1. /*
2977
2978	rtx_insn *
2979	ready_element (struct ready_list ready, int* index)
2980	{
2981	gcc_assert (ready->n_ready && index < ready->n_ready);
2982
2983	return ready->vec[ready->first - index];
2984	}
2985
2986	/ Remove the element INDEX from the ready list and return it. INDEX*
2987	for insn with the highest priority is 0, and the lowest priority
2988	has N_READY - 1. /*
2989
2990	HAIFA_INLINE static rtx_insn *
2991	ready_remove (struct ready_list ready, int* index)
2992	{
2993	rtx_insn *t;
2994	int i;
2995
2996	if (index == `0`)
2997	return ready_remove_first (ready);
2998	gcc_assert (ready->n_ready && index < ready->n_ready);
2999	t = ready->vec[ready->first - index];
3000	ready->n_ready--;
3001	if (DEBUG_INSN_P (t))
3002	ready->n_debug--;
3003	for (i = index; i < ready->n_ready; i++)
3004	ready->vec[ready->first - i] = ready->vec[ready->first - i - `1`];
3005	QUEUE_INDEX (t) = QUEUE_NOWHERE;
3006	return t;
3007	}
3008
3009	/ Remove INSN from the ready list. /
3010	static void
3011	ready_remove_insn (rtx_insn *insn)
3012	{
3013	int i;
3014
3015	for (i = `0`; i < readyp->n_ready; i++)
3016	if (ready_element (ready: readyp, index: i) == insn)
3017	{
3018	ready_remove (ready: readyp, index: i);
3019	return;
3020	}
3021	gcc_unreachable ();
3022	}
3023
3024	/ Calculate difference of two statistics set WAS and NOW.*
3025	Result returned in WAS. /*
3026	static void
3027	rank_for_schedule_stats_diff (rank_for_schedule_stats_t *was,
3028	const rank_for_schedule_stats_t *now)
3029	{
3030	for (int i = `0`; i < RFS_N; ++i)
3031	was->stats[i] = now->stats[i] - was->stats[i];
3032	}
3033
3034	/ Print rank_for_schedule statistics. /
3035	static void
3036	print_rank_for_schedule_stats (const char *prefix,
3037	const rank_for_schedule_stats_t *stats,
3038	struct ready_list *ready)
3039	{
3040	for (int i = `0`; i < RFS_N; ++i)
3041	if (stats->stats[i])
3042	{
3043	fprintf (stream: sched_dump, format: "%s%20s: %u", prefix, rfs_str[i], stats->stats[i]);
3044
3045	if (ready != NULL)
3046	/ Print out insns that won due to RFS_<I>. /
3047	{
3048	rtx_insn **p = ready_lastpos (ready);
3049
3050	fprintf (stream: sched_dump, format: ":");
3051	/ Start with 1 since least-priority insn didn't have any wins. /
3052	for (int j = `1`; j < ready->n_ready; ++j)
3053	if (INSN_LAST_RFS_WIN (p[j]) == i)
3054	fprintf (stream: sched_dump, format: " %s",
3055	(*current_sched_info->print_insn) (p[j], `0`));
3056	}
3057	fprintf (stream: sched_dump, format: "\n");
3058	}
3059	}
3060
3061	/ Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end*
3062	of array. /*
3063	static void
3064	ready_sort_debug (struct ready_list *ready)
3065	{
3066	int i;
3067	rtx_insn **first = ready_lastpos (ready);
3068
3069	for (i = `0`; i < ready->n_ready; ++i)
3070	if (!DEBUG_INSN_P (first[i]))
3071	INSN_RFS_DEBUG_ORIG_ORDER (first[i]) = i;
3072
3073	qsort (first, ready->n_ready, sizeof (rtx), rank_for_schedule_debug);
3074	}
3075
3076	/ Sort non-debug insns in the ready list READY by ascending priority.*
3077	Assumes that all debug insns are separated from the real insns. /*
3078	static void
3079	ready_sort_real (struct ready_list *ready)
3080	{
3081	int i;
3082	rtx_insn **first = ready_lastpos (ready);
3083	int n_ready_real = ready->n_ready - ready->n_debug;
3084
3085	if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
3086	for (i = `0`; i < n_ready_real; ++i)
3087	setup_insn_reg_pressure_info (first[i]);
3088	else if (sched_pressure == SCHED_PRESSURE_MODEL
3089	&& model_curr_point < model_num_insns)
3090	model_set_excess_costs (insns: first, count: n_ready_real);
3091
3092	rank_for_schedule_stats_t stats1;
3093	if (sched_verbose >= `4`)
3094	stats1 = rank_for_schedule_stats;
3095
3096	if (n_ready_real == `2`)
3097	swap_sort (a: first, n: n_ready_real);
3098	else if (n_ready_real > `2`)
3099	qsort (first, n_ready_real, sizeof (rtx), rank_for_schedule);
3100
3101	if (sched_verbose >= `4`)
3102	{
3103	rank_for_schedule_stats_diff (was: &stats1, now: &rank_for_schedule_stats);
3104	print_rank_for_schedule_stats (prefix: ";;\t\t", stats: &stats1, ready);
3105	}
3106	}
3107
3108	/ Sort the ready list READY by ascending priority. /
3109	static void
3110	ready_sort (struct ready_list *ready)
3111	{
3112	if (ready->n_debug > `0`)
3113	ready_sort_debug (ready);
3114	else
3115	ready_sort_real (ready);
3116	}
3117
3118	/ PREV is an insn that is ready to execute. Adjust its priority if that*
3119	will help shorten or lengthen register lifetimes as appropriate. Also
3120	provide a hook for the target to tweak itself. /*
3121
3122	HAIFA_INLINE static void
3123	adjust_priority (rtx_insn *prev)
3124	{
3125	/ ??? There used to be code here to try and estimate how an insn*
3126	affected register lifetimes, but it did it by looking at REG_DEAD
3127	notes, which we removed in schedule_region. Nor did it try to
3128	take into account register pressure or anything useful like that.
3129
3130	Revisit when we have a machine model to work with and not before. /*
3131
3132	if (targetm.sched.adjust_priority)
3133	INSN_PRIORITY (prev) =
3134	targetm.sched.adjust_priority (prev, INSN_PRIORITY (prev));
3135	}
3136
3137	/ Advance DFA state STATE on one cycle. /
3138	void
3139	advance_state (state_t state)
3140	{
3141	if (targetm.sched.dfa_pre_advance_cycle)
3142	targetm.sched.dfa_pre_advance_cycle ();
3143
3144	if (targetm.sched.dfa_pre_cycle_insn)
3145	state_transition (state,
3146	targetm.sched.dfa_pre_cycle_insn ());
3147
3148	state_transition (state, NULL);
3149
3150	if (targetm.sched.dfa_post_cycle_insn)
3151	state_transition (state,
3152	targetm.sched.dfa_post_cycle_insn ());
3153
3154	if (targetm.sched.dfa_post_advance_cycle)
3155	targetm.sched.dfa_post_advance_cycle ();
3156	}
3157
3158	/ Advance time on one cycle. /
3159	HAIFA_INLINE static void
3160	advance_one_cycle (void)
3161	{
3162	int i;
3163
3164	advance_state (state: curr_state);
3165	for (i = `4`; i <= sched_verbose; ++i)
3166	fprintf (stream: sched_dump, format: ";;\tAdvance the current state: %d.\n", clock_var);
3167	}
3168
3169	/ Update register pressure after scheduling INSN. /
3170	static void
3171	update_register_pressure (rtx_insn *insn)
3172	{
3173	struct reg_use_data *use;
3174	struct reg_set_data *set;
3175
3176	gcc_checking_assert (!DEBUG_INSN_P (insn));
3177
3178	for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
3179	if (dying_use_p (use))
3180	mark_regno_birth_or_death (live: curr_reg_live, pressure: curr_reg_pressure,
3181	regno: use->regno, birth_p: false);
3182	for (set = INSN_REG_SET_LIST (insn); set != NULL; set = set->next_insn_set)
3183	mark_regno_birth_or_death (live: curr_reg_live, pressure: curr_reg_pressure,
3184	regno: set->regno, birth_p: true);
3185	}
3186
3187	/ Set up or update (if UPDATE_P) max register pressure (see its*
3188	meaning in sched-int.h::_haifa_insn_data) for all current BB insns
3189	after insn AFTER. /*
3190	static void
3191	setup_insn_max_reg_pressure (rtx_insn after, bool* update_p)
3192	{
3193	int i, p;
3194	bool eq_p;
3195	rtx_insn *insn;
3196	static int max_reg_pressure[N_REG_CLASSES];
3197
3198	save_reg_pressure ();
3199	for (i = `0`; i < ira_pressure_classes_num; i++)
3200	max_reg_pressure[ira_pressure_classes[i]]
3201	= curr_reg_pressure[ira_pressure_classes[i]];
3202	for (insn = NEXT_INSN (insn: after);
3203	insn != NULL_RTX && ! BARRIER_P (insn)
3204	&& BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (insn: after);
3205	insn = NEXT_INSN (insn))
3206	if (NONDEBUG_INSN_P (insn))
3207	{
3208	eq_p = true;
3209	for (i = `0`; i < ira_pressure_classes_num; i++)
3210	{
3211	p = max_reg_pressure[ira_pressure_classes[i]];
3212	if (INSN_MAX_REG_PRESSURE (insn)[i] != p)
3213	{
3214	eq_p = false;
3215	INSN_MAX_REG_PRESSURE (insn)[i]
3216	= max_reg_pressure[ira_pressure_classes[i]];
3217	}
3218	}
3219	if (update_p && eq_p)
3220	break;
3221	update_register_pressure (insn);
3222	for (i = `0`; i < ira_pressure_classes_num; i++)
3223	if (max_reg_pressure[ira_pressure_classes[i]]
3224	< curr_reg_pressure[ira_pressure_classes[i]])
3225	max_reg_pressure[ira_pressure_classes[i]]
3226	= curr_reg_pressure[ira_pressure_classes[i]];
3227	}
3228	restore_reg_pressure ();
3229	}
3230
3231	/ Update the current register pressure after scheduling INSN. Update*
3232	also max register pressure for unscheduled insns of the current
3233	BB. /*
3234	static void
3235	update_reg_and_insn_max_reg_pressure (rtx_insn *insn)
3236	{
3237	int i;
3238	int before[N_REG_CLASSES];
3239
3240	for (i = `0`; i < ira_pressure_classes_num; i++)
3241	before[i] = curr_reg_pressure[ira_pressure_classes[i]];
3242	update_register_pressure (insn);
3243	for (i = `0`; i < ira_pressure_classes_num; i++)
3244	if (curr_reg_pressure[ira_pressure_classes[i]] != before[i])
3245	break;
3246	if (i < ira_pressure_classes_num)
3247	setup_insn_max_reg_pressure (after: insn, update_p: true);
3248	}
3249
3250	/ Set up register pressure at the beginning of basic block BB whose*
3251	insns starting after insn AFTER. Set up also max register pressure
3252	for all insns of the basic block. /*
3253	void
3254	sched_setup_bb_reg_pressure_info (basic_block bb, rtx_insn *after)
3255	{
3256	gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
3257	initiate_bb_reg_pressure_info (bb);
3258	setup_insn_max_reg_pressure (after, update_p: false);
3259	}
3260
3261	/ If doing predication while scheduling, verify whether INSN, which*
3262	has just been scheduled, clobbers the conditions of any
3263	instructions that must be predicated in order to break their
3264	dependencies. If so, remove them from the queues so that they will
3265	only be scheduled once their control dependency is resolved. /*
3266
3267	static void
3268	check_clobbered_conditions (rtx_insn *insn)
3269	{
3270	HARD_REG_SET t;
3271	int i;
3272
3273	if ((current_sched_info->flags & DO_PREDICATION) == `0`)
3274	return;
3275
3276	find_all_hard_reg_sets (insn, &t, true);
3277
3278	restart:
3279	for (i = `0`; i < ready.n_ready; i++)
3280	{
3281	rtx_insn *x = ready_element (ready: &ready, index: i);
3282	if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (insn: x, set_regs: t))
3283	{
3284	ready_remove_insn (insn: x);
3285	goto restart;
3286	}
3287	}
3288	for (i = `0`; i <= max_insn_queue_index; i++)
3289	{
3290	rtx_insn_list *link;
3291	int q = NEXT_Q_AFTER (q_ptr, i);
3292
3293	restart_queue:
3294	for (link = insn_queue[q]; link; link = link->next ())
3295	{
3296	rtx_insn *x = link->insn ();
3297	if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (insn: x, set_regs: t))
3298	{
3299	queue_remove (insn: x);
3300	goto restart_queue;
3301	}
3302	}
3303	}
3304	}
3305
3306	/ Return (in order):*
3307
3308	- positive if INSN adversely affects the pressure on one
3309	register class
3310
3311	- negative if INSN reduces the pressure on one register class
3312
3313	- 0 if INSN doesn't affect the pressure on any register class. /*
3314
3315	static int
3316	model_classify_pressure (struct model_insn_info *insn)
3317	{
3318	struct reg_pressure_data *reg_pressure;
3319	int death[N_REG_CLASSES];
3320	int pci, cl, sum;
3321
3322	calculate_reg_deaths (insn: insn->insn, death);
3323	reg_pressure = INSN_REG_PRESSURE (insn->insn);
3324	sum = `0`;
3325	for (pci = `0`; pci < ira_pressure_classes_num; pci++)
3326	{
3327	cl = ira_pressure_classes[pci];
3328	if (death[cl] < reg_pressure[pci].set_increase)
3329	return `1`;
3330	sum += reg_pressure[pci].set_increase - death[cl];
3331	}
3332	return sum;
3333	}
3334
3335	/ Return true if INSN1 should come before INSN2 in the model schedule. /
3336
3337	static int
3338	model_order_p (struct model_insn_info insn1, struct* model_insn_info *insn2)
3339	{
3340	unsigned int height1, height2;
3341	unsigned int priority1, priority2;
3342
3343	/ Prefer instructions with a higher model priority. /
3344	if (insn1->model_priority != insn2->model_priority)
3345	return insn1->model_priority > insn2->model_priority;
3346
3347	/ Combine the length of the longest path of satisfied true dependencies*
3348	that leads to each instruction (depth) with the length of the longest
3349	path of any dependencies that leads from the instruction (alap).
3350	Prefer instructions with the greatest combined length. If the combined
3351	lengths are equal, prefer instructions with the greatest depth.
3352
3353	The idea is that, if we have a set S of "equal" instructions that each
3354	have ALAP value X, and we pick one such instruction I, any true-dependent
3355	successors of I that have ALAP value X - 1 should be preferred over S.
3356	This encourages the schedule to be "narrow" rather than "wide".
3357	However, if I is a low-priority instruction that we decided to
3358	schedule because of its model_classify_pressure, and if there
3359	is a set of higher-priority instructions T, the aforementioned
3360	successors of I should not have the edge over T. /*
3361	height1 = insn1->depth + insn1->alap;
3362	height2 = insn2->depth + insn2->alap;
3363	if (height1 != height2)
3364	return height1 > height2;
3365	if (insn1->depth != insn2->depth)
3366	return insn1->depth > insn2->depth;
3367
3368	/ We have no real preference between INSN1 an INSN2 as far as attempts*
3369	to reduce pressure go. Prefer instructions with higher priorities. /*
3370	priority1 = INSN_PRIORITY (insn1->insn);
3371	priority2 = INSN_PRIORITY (insn2->insn);
3372	if (priority1 != priority2)
3373	return priority1 > priority2;
3374
3375	/ Use the original rtl sequence as a tie-breaker. /
3376	return insn1 < insn2;
3377	}
3378
3379	/ Add INSN to the model worklist immediately after PREV. Add it to the*
3380	beginning of the list if PREV is null. /*
3381
3382	static void
3383	model_add_to_worklist_at (struct model_insn_info *insn,
3384	struct model_insn_info *prev)
3385	{
3386	gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_NOWHERE);
3387	QUEUE_INDEX (insn->insn) = QUEUE_READY;
3388
3389	insn->prev = prev;
3390	if (prev)
3391	{
3392	insn->next = prev->next;
3393	prev->next = insn;
3394	}
3395	else
3396	{
3397	insn->next = model_worklist;
3398	model_worklist = insn;
3399	}
3400	if (insn->next)
3401	insn->next->prev = insn;
3402	}
3403
3404	/ Remove INSN from the model worklist. /
3405
3406	static void
3407	model_remove_from_worklist (struct model_insn_info *insn)
3408	{
3409	gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_READY);
3410	QUEUE_INDEX (insn->insn) = QUEUE_NOWHERE;
3411
3412	if (insn->prev)
3413	insn->prev->next = insn->next;
3414	else
3415	model_worklist = insn->next;
3416	if (insn->next)
3417	insn->next->prev = insn->prev;
3418	}
3419
3420	/ Add INSN to the model worklist. Start looking for a suitable position*
3421	between neighbors PREV and NEXT, testing at most param_max_sched_ready_insns
3422	insns either side. A null PREV indicates the beginning of the list and
3423	a null NEXT indicates the end. /*
3424
3425	static void
3426	model_add_to_worklist (struct model_insn_info *insn,
3427	struct model_insn_info *prev,
3428	struct model_insn_info *next)
3429	{
3430	int count;
3431
3432	count = param_max_sched_ready_insns;
3433	if (count > `0` && prev && model_order_p (insn1: insn, insn2: prev))
3434	do
3435	{
3436	count--;
3437	prev = prev->prev;
3438	}
3439	while (count > `0` && prev && model_order_p (insn1: insn, insn2: prev));
3440	else
3441	while (count > `0` && next && model_order_p (insn1: next, insn2: insn))
3442	{
3443	count--;
3444	prev = next;
3445	next = next->next;
3446	}
3447	model_add_to_worklist_at (insn, prev);
3448	}
3449
3450	/ INSN may now have a higher priority (in the model_order_p sense)*
3451	than before. Move it up the worklist if necessary. /*
3452
3453	static void
3454	model_promote_insn (struct model_insn_info *insn)
3455	{
3456	struct model_insn_info *prev;
3457	int count;
3458
3459	prev = insn->prev;
3460	count = param_max_sched_ready_insns;
3461	while (count > `0` && prev && model_order_p (insn1: insn, insn2: prev))
3462	{
3463	count--;
3464	prev = prev->prev;
3465	}
3466	if (prev != insn->prev)
3467	{
3468	model_remove_from_worklist (insn);
3469	model_add_to_worklist_at (insn, prev);
3470	}
3471	}
3472
3473	/ Add INSN to the end of the model schedule. /
3474
3475	static void
3476	model_add_to_schedule (rtx_insn *insn)
3477	{
3478	unsigned int point;
3479
3480	gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE);
3481	QUEUE_INDEX (insn) = QUEUE_SCHEDULED;
3482
3483	point = model_schedule.length ();
3484	model_schedule.quick_push (obj: insn);
3485	INSN_MODEL_INDEX (insn) = point + `1`;
3486	}
3487
3488	/ Analyze the instructions that are to be scheduled, setting up*
3489	MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready
3490	instructions to model_worklist. /*
3491
3492	static void
3493	model_analyze_insns (void)
3494	{
3495	rtx_insn start, end, *iter;
3496	sd_iterator_def sd_it;
3497	dep_t dep;
3498	struct model_insn_info insn, con;
3499
3500	model_num_insns = `0`;
3501	start = PREV_INSN (insn: current_sched_info->next_tail);
3502	end = current_sched_info->prev_head;
3503	for (iter = start; iter != end; iter = PREV_INSN (insn: iter))
3504	if (NONDEBUG_INSN_P (iter))
3505	{
3506	insn = MODEL_INSN_INFO (iter);
3507	insn->insn = iter;
3508	FOR_EACH_DEP (iter, SD_LIST_FORW, sd_it, dep)
3509	{
3510	con = MODEL_INSN_INFO (DEP_CON (dep));
3511	if (con->insn && insn->alap < con->alap + `1`)
3512	insn->alap = con->alap + `1`;
3513	}
3514
3515	insn->old_queue = QUEUE_INDEX (iter);
3516	QUEUE_INDEX (iter) = QUEUE_NOWHERE;
3517
3518	insn->unscheduled_preds = dep_list_size (insn: iter, SD_LIST_HARD_BACK);
3519	if (insn->unscheduled_preds == `0`)
3520	model_add_to_worklist (insn, NULL, next: model_worklist);
3521
3522	model_num_insns++;
3523	}
3524	}
3525
3526	/ The global state describes the register pressure at the start of the*
3527	model schedule. Initialize GROUP accordingly. /*
3528
3529	static void
3530	model_init_pressure_group (struct model_pressure_group *group)
3531	{
3532	int pci, cl;
3533
3534	for (pci = `0`; pci < ira_pressure_classes_num; pci++)
3535	{
3536	cl = ira_pressure_classes[pci];
3537	group->limits[pci].pressure = curr_reg_pressure[cl];
3538	group->limits[pci].point = `0`;
3539	}
3540	/ Use index model_num_insns to record the state after the last*
3541	instruction in the model schedule. /*
3542	group->model = XNEWVEC (struct model_pressure_data,
3543	(model_num_insns + `1`) * ira_pressure_classes_num);
3544	}
3545
3546	/ Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.*
3547	Update the maximum pressure for the whole schedule. /*
3548
3549	static void
3550	model_record_pressure (struct model_pressure_group *group,
3551	int point, int pci, int pressure)
3552	{
3553	MODEL_REF_PRESSURE (group, point, pci) = pressure;
3554	if (group->limits[pci].pressure < pressure)
3555	{
3556	group->limits[pci].pressure = pressure;
3557	group->limits[pci].point = point;
3558	}
3559	}
3560
3561	/ INSN has just been added to the end of the model schedule. Record its*
3562	register-pressure information. /*
3563
3564	static void
3565	model_record_pressures (struct model_insn_info *insn)
3566	{
3567	struct reg_pressure_data *reg_pressure;
3568	int point, pci, cl, delta;
3569	int death[N_REG_CLASSES];
3570
3571	point = model_index (insn: insn->insn);
3572	if (sched_verbose >= `2`)
3573	{
3574	if (point == `0`)
3575	{
3576	fprintf (stream: sched_dump, format: "\n;;\tModel schedule:\n;;\n");
3577	fprintf (stream: sched_dump, format: ";;\t\| idx insn \| mpri hght dpth prio \|\n");
3578	}
3579	fprintf (stream: sched_dump, format: ";;\t\| %3d %4d \| %4d %4d %4d %4d \| %-30s ",
3580	point, INSN_UID (insn: insn->insn), insn->model_priority,
3581	insn->depth + insn->alap, insn->depth,
3582	INSN_PRIORITY (insn->insn),
3583	str_pattern_slim (PATTERN (insn: insn->insn)));
3584	}
3585	calculate_reg_deaths (insn: insn->insn, death);
3586	reg_pressure = INSN_REG_PRESSURE (insn->insn);
3587	for (pci = `0`; pci < ira_pressure_classes_num; pci++)
3588	{
3589	cl = ira_pressure_classes[pci];
3590	delta = reg_pressure[pci].set_increase - death[cl];
3591	if (sched_verbose >= `2`)
3592	fprintf (stream: sched_dump, format: " %s:[%d,%+d]", reg_class_names[cl],
3593	curr_reg_pressure[cl], delta);
3594	model_record_pressure (group: &model_before_pressure, point, pci,
3595	pressure: curr_reg_pressure[cl]);
3596	}
3597	if (sched_verbose >= `2`)
3598	fprintf (stream: sched_dump, format: "\n");
3599	}
3600
3601	/ All instructions have been added to the model schedule. Record the*
3602	final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. /*
3603
3604	static void
3605	model_record_final_pressures (struct model_pressure_group *group)
3606	{
3607	int point, pci, max_pressure, ref_pressure, cl;
3608
3609	for (pci = `0`; pci < ira_pressure_classes_num; pci++)
3610	{
3611	/ Record the final pressure for this class. /
3612	cl = ira_pressure_classes[pci];
3613	point = model_num_insns;
3614	ref_pressure = curr_reg_pressure[cl];
3615	model_record_pressure (group, point, pci, pressure: ref_pressure);
3616
3617	/ Record the original maximum pressure. /
3618	group->limits[pci].orig_pressure = group->limits[pci].pressure;
3619
3620	/ Update the MODEL_MAX_PRESSURE for every point of the schedule. /
3621	max_pressure = ref_pressure;
3622	MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
3623	while (point > `0`)
3624	{
3625	point--;
3626	ref_pressure = MODEL_REF_PRESSURE (group, point, pci);
3627	max_pressure = MAX (max_pressure, ref_pressure);
3628	MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
3629	}
3630	}
3631	}
3632
3633	/ Update all successors of INSN, given that INSN has just been scheduled. /
3634
3635	static void
3636	model_add_successors_to_worklist (struct model_insn_info *insn)
3637	{
3638	sd_iterator_def sd_it;
3639	struct model_insn_info *con;
3640	dep_t dep;
3641
3642	FOR_EACH_DEP (insn->insn, SD_LIST_FORW, sd_it, dep)
3643	{
3644	con = MODEL_INSN_INFO (DEP_CON (dep));
3645	/ Ignore debug instructions, and instructions from other blocks. /
3646	if (con->insn)
3647	{
3648	con->unscheduled_preds--;
3649
3650	/ Update the depth field of each true-dependent successor.*
3651	Increasing the depth gives them a higher priority than
3652	before. /*
3653	if (DEP_TYPE (dep) == REG_DEP_TRUE && con->depth < insn->depth + `1`)
3654	{
3655	con->depth = insn->depth + `1`;
3656	if (QUEUE_INDEX (con->insn) == QUEUE_READY)
3657	model_promote_insn (insn: con);
3658	}
3659
3660	/ If this is a true dependency, or if there are no remaining*
3661	dependencies for CON (meaning that CON only had non-true
3662	dependencies), make sure that CON is on the worklist.
3663	We don't bother otherwise because it would tend to fill the
3664	worklist with a lot of low-priority instructions that are not
3665	yet ready to issue. /*
3666	if ((con->depth > `0` \|\| con->unscheduled_preds == `0`)
3667	&& QUEUE_INDEX (con->insn) == QUEUE_NOWHERE)
3668	model_add_to_worklist (insn: con, prev: insn, next: insn->next);
3669	}
3670	}
3671	}
3672
3673	/ Give INSN a higher priority than any current instruction, then give*
3674	unscheduled predecessors of INSN a higher priority still. If any of
3675	those predecessors are not on the model worklist, do the same for its
3676	predecessors, and so on. /*
3677
3678	static void
3679	model_promote_predecessors (struct model_insn_info *insn)
3680	{
3681	struct model_insn_info pro, first;
3682	sd_iterator_def sd_it;
3683	dep_t dep;
3684
3685	if (sched_verbose >= `7`)
3686	fprintf (stream: sched_dump, format: ";;\t+--- priority of %d = %d, priority of",
3687	INSN_UID (insn: insn->insn), model_next_priority);
3688	insn->model_priority = model_next_priority++;
3689	model_remove_from_worklist (insn);
3690	model_add_to_worklist_at (insn, NULL);
3691
3692	first = NULL;
3693	for (;;)
3694	{
3695	FOR_EACH_DEP (insn->insn, SD_LIST_HARD_BACK, sd_it, dep)
3696	{
3697	pro = MODEL_INSN_INFO (DEP_PRO (dep));
3698	/ The first test is to ignore debug instructions, and instructions*
3699	from other blocks. /*
3700	if (pro->insn
3701	&& pro->model_priority != model_next_priority
3702	&& QUEUE_INDEX (pro->insn) != QUEUE_SCHEDULED)
3703	{
3704	pro->model_priority = model_next_priority;
3705	if (sched_verbose >= `7`)
3706	fprintf (stream: sched_dump, format: " %d", INSN_UID (insn: pro->insn));
3707	if (QUEUE_INDEX (pro->insn) == QUEUE_READY)
3708	{
3709	/ PRO is already in the worklist, but it now has*
3710	a higher priority than before. Move it at the
3711	appropriate place. /*
3712	model_remove_from_worklist (insn: pro);
3713	model_add_to_worklist (insn: pro, NULL, next: model_worklist);
3714	}
3715	else
3716	{
3717	/ PRO isn't in the worklist. Recursively process*
3718	its predecessors until we find one that is. /*
3719	pro->next = first;
3720	first = pro;
3721	}
3722	}
3723	}
3724	if (!first)
3725	break;
3726	insn = first;
3727	first = insn->next;
3728	}
3729	if (sched_verbose >= `7`)
3730	fprintf (stream: sched_dump, format: " = %d\n", model_next_priority);
3731	model_next_priority++;
3732	}
3733
3734	/ Pick one instruction from model_worklist and process it. /
3735
3736	static void
3737	model_choose_insn (void)
3738	{
3739	struct model_insn_info insn, fallback;
3740	int count;
3741
3742	if (sched_verbose >= `7`)
3743	{
3744	fprintf (stream: sched_dump, format: ";;\t+--- worklist:\n");
3745	insn = model_worklist;
3746	count = param_max_sched_ready_insns;
3747	while (count > `0` && insn)
3748	{
3749	fprintf (stream: sched_dump, format: ";;\t+--- %d [%d, %d, %d, %d]\n",
3750	INSN_UID (insn: insn->insn), insn->model_priority,
3751	insn->depth + insn->alap, insn->depth,
3752	INSN_PRIORITY (insn->insn));
3753	count--;
3754	insn = insn->next;
3755	}
3756	}
3757
3758	/ Look for a ready instruction whose model_classify_priority is zero*
3759	or negative, picking the highest-priority one. Adding such an
3760	instruction to the schedule now should do no harm, and may actually
3761	do some good.
3762
3763	Failing that, see whether there is an instruction with the highest
3764	extant model_priority that is not yet ready, but which would reduce
3765	pressure if it became ready. This is designed to catch cases like:
3766
3767	(set (mem (reg R1)) (reg R2))
3768
3769	where the instruction is the last remaining use of R1 and where the
3770	value of R2 is not yet available (or vice versa). The death of R1
3771	means that this instruction already reduces pressure. It is of
3772	course possible that the computation of R2 involves other registers
3773	that are hard to kill, but such cases are rare enough for this
3774	heuristic to be a win in general.
3775
3776	Failing that, just pick the highest-priority instruction in the
3777	worklist. /*
3778	count = param_max_sched_ready_insns;
3779	insn = model_worklist;
3780	fallback = `0`;
3781	for (;;)
3782	{
3783	if (count == `0` \|\| !insn)
3784	{
3785	insn = fallback ? fallback : model_worklist;
3786	break;
3787	}
3788	if (insn->unscheduled_preds)
3789	{
3790	if (model_worklist->model_priority == insn->model_priority
3791	&& !fallback
3792	&& model_classify_pressure (insn) < `0`)
3793	fallback = insn;
3794	}
3795	else
3796	{
3797	if (model_classify_pressure (insn) <= `0`)
3798	break;
3799	}
3800	count--;
3801	insn = insn->next;
3802	}
3803
3804	if (sched_verbose >= `7` && insn != model_worklist)
3805	{
3806	if (insn->unscheduled_preds)
3807	fprintf (stream: sched_dump, format: ";;\t+--- promoting insn %d, with dependencies\n",
3808	INSN_UID (insn: insn->insn));
3809	else
3810	fprintf (stream: sched_dump, format: ";;\t+--- promoting insn %d, which is ready\n",
3811	INSN_UID (insn: insn->insn));
3812	}
3813	if (insn->unscheduled_preds)
3814	/ INSN isn't yet ready to issue. Give all its predecessors the*
3815	highest priority. /*
3816	model_promote_predecessors (insn);
3817	else
3818	{
3819	/ INSN is ready. Add it to the end of model_schedule and*
3820	process its successors. /*
3821	model_add_successors_to_worklist (insn);
3822	model_remove_from_worklist (insn);
3823	model_add_to_schedule (insn: insn->insn);
3824	model_record_pressures (insn);
3825	update_register_pressure (insn: insn->insn);
3826	}
3827	}
3828
3829	/ Restore all QUEUE_INDEXs to the values that they had before*
3830	model_start_schedule was called. /*
3831
3832	static void
3833	model_reset_queue_indices (void)
3834	{
3835	unsigned int i;
3836	rtx_insn *insn;
3837
3838	FOR_EACH_VEC_ELT (model_schedule, i, insn)
3839	QUEUE_INDEX (insn) = MODEL_INSN_INFO (insn)->old_queue;
3840	}
3841
3842	/ We have calculated the model schedule and spill costs. Print a summary*
3843	to sched_dump. /*
3844
3845	static void
3846	model_dump_pressure_summary (void)
3847	{
3848	int pci, cl;
3849
3850	fprintf (stream: sched_dump, format: ";; Pressure summary:");
3851	for (pci = `0`; pci < ira_pressure_classes_num; pci++)
3852	{
3853	cl = ira_pressure_classes[pci];
3854	fprintf (stream: sched_dump, format: " %s:%d", reg_class_names[cl],
3855	model_before_pressure.limits[pci].pressure);
3856	}
3857	fprintf (stream: sched_dump, format: "\n\n");
3858	}
3859
3860	/ Initialize the SCHED_PRESSURE_MODEL information for the current*
3861	scheduling region. /*
3862
3863	static void
3864	model_start_schedule (basic_block bb)
3865	{
3866	model_next_priority = `1`;
3867	model_schedule.create (nelems: sched_max_luid);
3868	model_insns = XCNEWVEC (struct model_insn_info, sched_max_luid);
3869
3870	gcc_assert (bb == BLOCK_FOR_INSN (NEXT_INSN (current_sched_info->prev_head)));
3871	initiate_reg_pressure_info (live: df_get_live_in (bb));
3872
3873	model_analyze_insns ();
3874	model_init_pressure_group (group: &model_before_pressure);
3875	while (model_worklist)
3876	model_choose_insn ();
3877	gcc_assert (model_num_insns == (int) model_schedule.length ());
3878	if (sched_verbose >= `2`)
3879	fprintf (stream: sched_dump, format: "\n");
3880
3881	model_record_final_pressures (group: &model_before_pressure);
3882	model_reset_queue_indices ();
3883
3884	XDELETEVEC (model_insns);
3885
3886	model_curr_point = `0`;
3887	initiate_reg_pressure_info (live: df_get_live_in (bb));
3888	if (sched_verbose >= `1`)
3889	model_dump_pressure_summary ();
3890	}
3891
3892	/ Free the information associated with GROUP. /
3893
3894	static void
3895	model_finalize_pressure_group (struct model_pressure_group *group)
3896	{
3897	XDELETEVEC (group->model);
3898	}
3899
3900	/ Free the information created by model_start_schedule. /
3901
3902	static void
3903	model_end_schedule (void)
3904	{
3905	model_finalize_pressure_group (group: &model_before_pressure);
3906	model_schedule.release ();
3907	}
3908
3909	/ Prepare reg pressure scheduling for basic block BB. /
3910	static void
3911	sched_pressure_start_bb (basic_block bb)
3912	{
3913	/ Set the number of available registers for each class taking into account*
3914	relative probability of current basic block versus function prologue and
3915	epilogue.
3916	* If the basic block executes much more often than the prologue/epilogue
3917	(e.g., inside a hot loop), then cost of spill in the prologue is close to
3918	nil, so the effective number of available registers is
3919	(ira_class_hard_regs_num[cl] - fixed_regs_num[cl] - 0).
3920	* If the basic block executes as often as the prologue/epilogue,
3921	then spill in the block is as costly as in the prologue, so the effective
3922	number of available registers is
3923	(ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
3924	- call_saved_regs_num[cl]).
3925	Note that all-else-equal, we prefer to spill in the prologue, since that
3926	allows "extra" registers for other basic blocks of the function.
3927	* If the basic block is on the cold path of the function and executes
3928	rarely, then we should always prefer to spill in the block, rather than
3929	in the prologue/epilogue. The effective number of available register is
3930	(ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
3931	- call_saved_regs_num[cl]). /*
3932	{
3933	int i;
3934	int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun);
3935	int bb_freq = bb->count.to_frequency (cfun);
3936
3937	if (bb_freq == `0`)
3938	{
3939	if (entry_freq == `0`)
3940	entry_freq = bb_freq = `1`;
3941	}
3942	if (bb_freq < entry_freq)
3943	bb_freq = entry_freq;
3944
3945	for (i = `0`; i < ira_pressure_classes_num; ++i)
3946	{
3947	enum reg_class cl = ira_pressure_classes[i];
3948	sched_class_regs_num[cl] = ira_class_hard_regs_num[cl]
3949	- fixed_regs_num[cl];
3950	sched_class_regs_num[cl]
3951	-= (call_saved_regs_num[cl] * entry_freq) / bb_freq;
3952	}
3953	}
3954
3955	if (sched_pressure == SCHED_PRESSURE_MODEL)
3956	model_start_schedule (bb);
3957	}
3958
3959	/ A structure that holds local state for the loop in schedule_block. /
3960	struct sched_block_state
3961	{
3962	/ True if no real insns have been scheduled in the current cycle. /
3963	bool first_cycle_insn_p;
3964	/ True if a shadow insn has been scheduled in the current cycle, which*
3965	means that no more normal insns can be issued. /*
3966	bool shadows_only_p;
3967	/ True if we're winding down a modulo schedule, which means that we only*
3968	issue insns with INSN_EXACT_TICK set. /*
3969	bool modulo_epilogue;
3970	/ Initialized with the machine's issue rate every cycle, and updated*
3971	by calls to the variable_issue hook. /*
3972	int can_issue_more;
3973	};
3974
3975	/ INSN is the "currently executing insn". Launch each insn which was*
3976	waiting on INSN. READY is the ready list which contains the insns
3977	that are ready to fire. CLOCK is the current cycle. The function
3978	returns necessary cycle advance after issuing the insn (it is not
3979	zero for insns in a schedule group). /*
3980
3981	static int
3982	schedule_insn (rtx_insn *insn)
3983	{
3984	sd_iterator_def sd_it;
3985	dep_t dep;
3986	int i;
3987	int advance = `0`;
3988
3989	if (sched_verbose >= `1`)
3990	{
3991	struct reg_pressure_data *pressure_info;
3992	fprintf (stream: sched_dump, format: ";;\t%3i--> %s %-40s:",
3993	clock_var, (*current_sched_info->print_insn) (insn, `1`),
3994	str_pattern_slim (PATTERN (insn)));
3995
3996	if (recog_memoized (insn) < `0`)
3997	fprintf (stream: sched_dump, format: "nothing");
3998	else
3999	print_reservation (sched_dump, insn);
4000	pressure_info = INSN_REG_PRESSURE (insn);
4001	if (pressure_info != NULL)
4002	{
4003	fputc (c: `':'`, stream: sched_dump);
4004	for (i = `0`; i < ira_pressure_classes_num; i++)
4005	fprintf (stream: sched_dump, format: "%s%s%+d(%d)",
4006	scheduled_insns.length () > `1`
4007	&& INSN_LUID (insn)
4008	< INSN_LUID (scheduled_insns[scheduled_insns.length () - `2`]) ? "@" : "",
4009	reg_class_names[ira_pressure_classes[i]],
4010	pressure_info[i].set_increase, pressure_info[i].change);
4011	}
4012	if (sched_pressure == SCHED_PRESSURE_MODEL
4013	&& model_curr_point < model_num_insns
4014	&& model_index (insn) == model_curr_point)
4015	fprintf (stream: sched_dump, format: ":model %d", model_curr_point);
4016	fputc (c: `'\n'`, stream: sched_dump);
4017	}
4018
4019	if (sched_pressure == SCHED_PRESSURE_WEIGHTED && !DEBUG_INSN_P (insn))
4020	update_reg_and_insn_max_reg_pressure (insn);
4021
4022	/ Scheduling instruction should have all its dependencies resolved and*
4023	should have been removed from the ready list. /*
4024	gcc_assert (sd_lists_empty_p (insn, SD_LIST_HARD_BACK));
4025
4026	/ Reset debug insns invalidated by moving this insn. /
4027	if (MAY_HAVE_DEBUG_BIND_INSNS && !DEBUG_INSN_P (insn))
4028	for (sd_it = sd_iterator_start (insn, SD_LIST_BACK);
4029	sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep);)
4030	{
4031	rtx_insn *dbg = DEP_PRO (dep);
4032	struct reg_use_data use, next;
4033
4034	if (DEP_STATUS (dep) & DEP_CANCELLED)
4035	{
4036	sd_iterator_next (it_ptr: &sd_it);
4037	continue;
4038	}
4039
4040	gcc_assert (DEBUG_BIND_INSN_P (dbg));
4041
4042	if (sched_verbose >= `6`)
4043	fprintf (stream: sched_dump, format: ";;\t\tresetting: debug insn %d\n",
4044	INSN_UID (insn: dbg));
4045
4046	/ ??? Rather than resetting the debug insn, we might be able*
4047	to emit a debug temp before the just-scheduled insn, but
4048	this would involve checking that the expression at the
4049	point of the debug insn is equivalent to the expression
4050	before the just-scheduled insn. They might not be: the
4051	expression in the debug insn may depend on other insns not
4052	yet scheduled that set MEMs, REGs or even other debug
4053	insns. It's not clear that attempting to preserve debug
4054	information in these cases is worth the effort, given how
4055	uncommon these resets are and the likelihood that the debug
4056	temps introduced won't survive the schedule change. /*
4057	INSN_VAR_LOCATION_LOC (dbg) = gen_rtx_UNKNOWN_VAR_LOC ();
4058	df_insn_rescan (dbg);
4059
4060	/ Unknown location doesn't use any registers. /
4061	for (use = INSN_REG_USE_LIST (dbg); use != NULL; use = next)
4062	{
4063	struct reg_use_data *prev = use;
4064
4065	/ Remove use from the cyclic next_regno_use chain first. /
4066	while (prev->next_regno_use != use)
4067	prev = prev->next_regno_use;
4068	prev->next_regno_use = use->next_regno_use;
4069	next = use->next_insn_use;
4070	free (ptr: use);
4071	}
4072	INSN_REG_USE_LIST (dbg) = NULL;
4073
4074	/ We delete rather than resolve these deps, otherwise we*
4075	crash in sched_free_deps(), because forward deps are
4076	expected to be released before backward deps. /*
4077	sd_delete_dep (sd_it);
4078	}
4079
4080	gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE);
4081	QUEUE_INDEX (insn) = QUEUE_SCHEDULED;
4082
4083	if (sched_pressure == SCHED_PRESSURE_MODEL
4084	&& model_curr_point < model_num_insns
4085	&& NONDEBUG_INSN_P (insn))
4086	{
4087	if (model_index (insn) == model_curr_point)
4088	do
4089	model_curr_point++;
4090	while (model_curr_point < model_num_insns
4091	&& (QUEUE_INDEX (MODEL_INSN (model_curr_point))
4092	== QUEUE_SCHEDULED));
4093	else
4094	model_recompute (insn);
4095	model_update_limit_points ();
4096	update_register_pressure (insn);
4097	if (sched_verbose >= `2`)
4098	print_curr_reg_pressure ();
4099	}
4100
4101	gcc_assert (INSN_TICK (insn) >= MIN_TICK);
4102	if (INSN_TICK (insn) > clock_var)
4103	/ INSN has been prematurely moved from the queue to the ready list.*
4104	This is possible only if following flags are set. /*
4105	gcc_assert (flag_sched_stalled_insns \|\| sched_fusion);
4106
4107	/ ??? Probably, if INSN is scheduled prematurely, we should leave*
4108	INSN_TICK untouched. This is a machine-dependent issue, actually. /*
4109	INSN_TICK (insn) = clock_var;
4110
4111	check_clobbered_conditions (insn);
4112
4113	/ Update dependent instructions. First, see if by scheduling this insn*
4114	now we broke a dependence in a way that requires us to change another
4115	insn. /*
4116	for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
4117	sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep); sd_iterator_next (it_ptr: &sd_it))
4118	{
4119	struct dep_replacement *desc = DEP_REPLACE (dep);
4120	rtx_insn *pro = DEP_PRO (dep);
4121	if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED
4122	&& desc != NULL && desc->insn == pro)
4123	apply_replacement (dep, false);
4124	}
4125
4126	/ Go through and resolve forward dependencies. /
4127	for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
4128	sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep);)
4129	{
4130	rtx_insn *next = DEP_CON (dep);
4131	bool cancelled = (DEP_STATUS (dep) & DEP_CANCELLED) != `0`;
4132
4133	/ Resolve the dependence between INSN and NEXT.*
4134	sd_resolve_dep () moves current dep to another list thus
4135	advancing the iterator. /*
4136	sd_resolve_dep (sd_it);
4137
4138	if (cancelled)
4139	{
4140	if (must_restore_pattern_p (next, dep))
4141	restore_pattern (dep, false);
4142	continue;
4143	}
4144
4145	/ Don't bother trying to mark next as ready if insn is a debug*
4146	insn. If insn is the last hard dependency, it will have
4147	already been discounted. /*
4148	if (DEBUG_INSN_P (insn) && !DEBUG_INSN_P (next))
4149	continue;
4150
4151	if (!IS_SPECULATION_BRANCHY_CHECK_P (insn))
4152	{
4153	int effective_cost;
4154
4155	effective_cost = try_ready (next);
4156
4157	if (effective_cost >= `0`
4158	&& SCHED_GROUP_P (next)
4159	&& advance < effective_cost)
4160	advance = effective_cost;
4161	}
4162	else
4163	/ Check always has only one forward dependence (to the first insn in*
4164	the recovery block), therefore, this will be executed only once. /*
4165	{
4166	gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW));
4167	fix_recovery_deps (RECOVERY_BLOCK (insn));
4168	}
4169	}
4170
4171	/ Annotate the instruction with issue information -- TImode*
4172	indicates that the instruction is expected not to be able
4173	to issue on the same cycle as the previous insn. A machine
4174	may use this information to decide how the instruction should
4175	be aligned. /*
4176	if (issue_rate > `1`
4177	&& GET_CODE (PATTERN (insn)) != USE
4178	&& GET_CODE (PATTERN (insn)) != CLOBBER
4179	&& !DEBUG_INSN_P (insn))
4180	{
4181	if (reload_completed)
4182	PUT_MODE (x: insn, mode: clock_var > last_clock_var ? TImode : VOIDmode);
4183	last_clock_var = clock_var;
4184	}
4185
4186	if (nonscheduled_insns_begin != NULL_RTX)
4187	/ Indicate to debug counters that INSN is scheduled. /
4188	nonscheduled_insns_begin = insn;
4189
4190	return advance;
4191	}
4192
4193	/ Functions for handling of notes. /
4194
4195	/ Add note list that ends on FROM_END to the end of TO_ENDP. /
4196	void
4197	concat_note_lists (rtx_insn from_end, rtx_insn *to_endp)
4198	{
4199	rtx_insn *from_start;
4200
4201	/ It's easy when have nothing to concat. /
4202	if (from_end == NULL)
4203	return;
4204
4205	/ It's also easy when destination is empty. /
4206	if (*to_endp == NULL)
4207	{
4208	*to_endp = from_end;
4209	return;
4210	}
4211
4212	from_start = from_end;
4213	while (PREV_INSN (insn: from_start) != NULL)
4214	from_start = PREV_INSN (insn: from_start);
4215
4216	SET_PREV_INSN (from_start) = *to_endp;
4217	SET_NEXT_INSN (*to_endp) = from_start;
4218	*to_endp = from_end;
4219	}
4220
4221	/ Delete notes between HEAD and TAIL and put them in the chain*
4222	of notes ended by NOTE_LIST. /*
4223	void
4224	remove_notes (rtx_insn head, rtx_insn tail)
4225	{
4226	rtx_insn next_tail, insn, *next;
4227
4228	note_list = `0`;
4229	if (head == tail && !INSN_P (head))
4230	return;
4231
4232	next_tail = NEXT_INSN (insn: tail);
4233	for (insn = head; insn != next_tail; insn = next)
4234	{
4235	next = NEXT_INSN (insn);
4236	if (!NOTE_P (insn))
4237	continue;
4238
4239	switch (NOTE_KIND (insn))
4240	{
4241	case NOTE_INSN_BASIC_BLOCK:
4242	continue;
4243
4244	case NOTE_INSN_EPILOGUE_BEG:
4245	if (insn != tail)
4246	{
4247	remove_insn (insn);
4248	/ If an insn was split just before the EPILOGUE_BEG note and*
4249	that split created new basic blocks, we could have a
4250	BASIC_BLOCK note here. Safely advance over it in that case
4251	and assert that we land on a real insn. /*
4252	if (NOTE_P (next)
4253	&& NOTE_KIND (next) == NOTE_INSN_BASIC_BLOCK
4254	&& next != next_tail)
4255	next = NEXT_INSN (insn: next);
4256	gcc_assert (INSN_P (next));
4257	add_reg_note (next, REG_SAVE_NOTE,
4258	GEN_INT (NOTE_INSN_EPILOGUE_BEG));
4259	break;
4260	}
4261	/ FALLTHRU /
4262
4263	default:
4264	remove_insn (insn);
4265
4266	/ Add the note to list that ends at NOTE_LIST. /
4267	SET_PREV_INSN (insn) = note_list;
4268	SET_NEXT_INSN (insn) = NULL_RTX;
4269	if (note_list)
4270	SET_NEXT_INSN (note_list) = insn;
4271	note_list = insn;
4272	break;
4273	}
4274
4275	gcc_assert ((sel_sched_p () \|\| insn != tail) && insn != head);
4276	}
4277	}
4278
4279	/ A structure to record enough data to allow us to backtrack the scheduler to*
4280	a previous state. /*
4281	struct haifa_saved_data
4282	{
4283	/ Next entry on the list. /
4284	struct haifa_saved_data *next;
4285
4286	/ Backtracking is associated with scheduling insns that have delay slots.*
4287	DELAY_PAIR points to the structure that contains the insns involved, and
4288	the number of cycles between them. /*
4289	struct delay_pair *delay_pair;
4290
4291	/ Data used by the frontend (e.g. sched-ebb or sched-rgn). /
4292	void *fe_saved_data;
4293	/ Data used by the backend. /
4294	void *be_saved_data;
4295
4296	/ Copies of global state. /
4297	int clock_var, last_clock_var;
4298	struct ready_list ready;
4299	state_t curr_state;
4300
4301	rtx_insn *last_scheduled_insn;
4302	rtx_insn *last_nondebug_scheduled_insn;
4303	rtx_insn *nonscheduled_insns_begin;
4304	int cycle_issued_insns;
4305
4306	/ Copies of state used in the inner loop of schedule_block. /
4307	struct sched_block_state sched_block;
4308
4309	/ We don't need to save q_ptr, as its value is arbitrary and we can set it*
4310	to 0 when restoring. /*
4311	int q_size;
4312	rtx_insn_list **insn_queue;
4313
4314	/ Describe pattern replacements that occurred since this backtrack point*
4315	was queued. /*
4316	vec<dep_t> replacement_deps;
4317	vec<int> replace_apply;
4318
4319	/ A copy of the next-cycle replacement vectors at the time of the backtrack*
4320	point. /*
4321	vec<dep_t> next_cycle_deps;
4322	vec<int> next_cycle_apply;
4323	};
4324
4325	/ A record, in reverse order, of all scheduled insns which have delay slots*
4326	and may require backtracking. /*
4327	static struct haifa_saved_data *backtrack_queue;
4328
4329	/ For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according*
4330	to SET_P. /*
4331	static void
4332	mark_backtrack_feeds (rtx_insn insn, int* set_p)
4333	{
4334	sd_iterator_def sd_it;
4335	dep_t dep;
4336	FOR_EACH_DEP (insn, SD_LIST_HARD_BACK, sd_it, dep)
4337	{
4338	FEEDS_BACKTRACK_INSN (DEP_PRO (dep)) = set_p;
4339	}
4340	}
4341
4342	/ Save the current scheduler state so that we can backtrack to it*
4343	later if necessary. PAIR gives the insns that make it necessary to
4344	save this point. SCHED_BLOCK is the local state of schedule_block
4345	that need to be saved. /*
4346	static void
4347	save_backtrack_point (struct delay_pair *pair,
4348	struct sched_block_state sched_block)
4349	{
4350	int i;
4351	struct haifa_saved_data save = XNEW (struct* haifa_saved_data);
4352
4353	save->curr_state = xmalloc (dfa_state_size);
4354	memcpy (dest: save->curr_state, src: curr_state, n: dfa_state_size);
4355
4356	save->ready.first = ready.first;
4357	save->ready.n_ready = ready.n_ready;
4358	save->ready.n_debug = ready.n_debug;
4359	save->ready.veclen = ready.veclen;
4360	save->ready.vec = XNEWVEC (rtx_insn *, ready.veclen);
4361	memcpy (dest: save->ready.vec, src: ready.vec, n: ready.veclen * sizeof (rtx));
4362
4363	save->insn_queue = XNEWVEC (rtx_insn_list *, max_insn_queue_index + `1`);
4364	save->q_size = q_size;
4365	for (i = `0`; i <= max_insn_queue_index; i++)
4366	{
4367	int q = NEXT_Q_AFTER (q_ptr, i);
4368	save->insn_queue[i] = copy_INSN_LIST (insn_queue[q]);
4369	}
4370
4371	save->clock_var = clock_var;
4372	save->last_clock_var = last_clock_var;
4373	save->cycle_issued_insns = cycle_issued_insns;
4374	save->last_scheduled_insn = last_scheduled_insn;
4375	save->last_nondebug_scheduled_insn = last_nondebug_scheduled_insn;
4376	save->nonscheduled_insns_begin = nonscheduled_insns_begin;
4377
4378	save->sched_block = sched_block;
4379
4380	save->replacement_deps.create (nelems: `0`);
4381	save->replace_apply.create (nelems: `0`);
4382	save->next_cycle_deps = next_cycle_replace_deps.copy ();
4383	save->next_cycle_apply = next_cycle_apply.copy ();
4384
4385	if (current_sched_info->save_state)
4386	save->fe_saved_data = (*current_sched_info->save_state) ();
4387
4388	if (targetm.sched.alloc_sched_context)
4389	{
4390	save->be_saved_data = targetm.sched.alloc_sched_context ();
4391	targetm.sched.init_sched_context (save->be_saved_data, false);
4392	}
4393	else
4394	save->be_saved_data = NULL;
4395
4396	save->delay_pair = pair;
4397
4398	save->next = backtrack_queue;
4399	backtrack_queue = save;
4400
4401	while (pair)
4402	{
4403	mark_backtrack_feeds (insn: pair->i2, set_p: `1`);
4404	INSN_TICK (pair->i2) = INVALID_TICK;
4405	INSN_EXACT_TICK (pair->i2) = clock_var + pair_delay (p: pair);
4406	SHADOW_P (pair->i2) = pair->stages == `0`;
4407	pair = pair->next_same_i1;
4408	}
4409	}
4410
4411	/ Walk the ready list and all queues. If any insns have unresolved backwards*
4412	dependencies, these must be cancelled deps, broken by predication. Set or
4413	clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. /*
4414
4415	static void
4416	toggle_cancelled_flags (bool set)
4417	{
4418	int i;
4419	sd_iterator_def sd_it;
4420	dep_t dep;
4421
4422	if (ready.n_ready > `0`)
4423	{
4424	rtx_insn **first = ready_lastpos (ready: &ready);
4425	for (i = `0`; i < ready.n_ready; i++)
4426	FOR_EACH_DEP (first[i], SD_LIST_BACK, sd_it, dep)
4427	if (!DEBUG_INSN_P (DEP_PRO (dep)))
4428	{
4429	if (set)
4430	DEP_STATUS (dep) \|= DEP_CANCELLED;
4431	else
4432	DEP_STATUS (dep) &= ~DEP_CANCELLED;
4433	}
4434	}
4435	for (i = `0`; i <= max_insn_queue_index; i++)
4436	{
4437	int q = NEXT_Q_AFTER (q_ptr, i);
4438	rtx_insn_list *link;
4439	for (link = insn_queue[q]; link; link = link->next ())
4440	{
4441	rtx_insn *insn = link->insn ();
4442	FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
4443	if (!DEBUG_INSN_P (DEP_PRO (dep)))
4444	{
4445	if (set)
4446	DEP_STATUS (dep) \|= DEP_CANCELLED;
4447	else
4448	DEP_STATUS (dep) &= ~DEP_CANCELLED;
4449	}
4450	}
4451	}
4452	}
4453
4454	/ Undo the replacements that have occurred after backtrack point SAVE*
4455	was placed. /*
4456	static void
4457	undo_replacements_for_backtrack (struct haifa_saved_data *save)
4458	{
4459	while (!save->replacement_deps.is_empty ())
4460	{
4461	dep_t dep = save->replacement_deps.pop ();
4462	int apply_p = save->replace_apply.pop ();
4463
4464	if (apply_p)
4465	restore_pattern (dep, true);
4466	else
4467	apply_replacement (dep, true);
4468	}
4469	save->replacement_deps.release ();
4470	save->replace_apply.release ();
4471	}
4472
4473	/ Pop entries from the SCHEDULED_INSNS vector up to and including INSN.*
4474	Restore their dependencies to an unresolved state, and mark them as
4475	queued nowhere. /*
4476
4477	static void
4478	unschedule_insns_until (rtx_insn *insn)
4479	{
4480	auto_vec<rtx_insn *> recompute_vec;
4481
4482	/ Make two passes over the insns to be unscheduled. First, we clear out*
4483	dependencies and other trivial bookkeeping. /*
4484	for (;;)
4485	{
4486	rtx_insn *last;
4487	sd_iterator_def sd_it;
4488	dep_t dep;
4489
4490	last = scheduled_insns.pop ();
4491
4492	/ This will be changed by restore_backtrack_point if the insn is in*
4493	any queue. /*
4494	QUEUE_INDEX (last) = QUEUE_NOWHERE;
4495	if (last != insn)
4496	INSN_TICK (last) = INVALID_TICK;
4497
4498	if (modulo_ii > `0` && INSN_UID (insn: last) < modulo_iter0_max_uid)
4499	modulo_insns_scheduled--;
4500
4501	for (sd_it = sd_iterator_start (insn: last, SD_LIST_RES_FORW);
4502	sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep);)
4503	{
4504	rtx_insn *con = DEP_CON (dep);
4505	sd_unresolve_dep (sd_it);
4506	if (!MUST_RECOMPUTE_SPEC_P (con))
4507	{
4508	MUST_RECOMPUTE_SPEC_P (con) = `1`;
4509	recompute_vec.safe_push (obj: con);
4510	}
4511	}
4512
4513	if (last == insn)
4514	break;
4515	}
4516
4517	/ A second pass, to update ready and speculation status for insns*
4518	depending on the unscheduled ones. The first pass must have
4519	popped the scheduled_insns vector up to the point where we
4520	restart scheduling, as recompute_todo_spec requires it to be
4521	up-to-date. /*
4522	while (!recompute_vec.is_empty ())
4523	{
4524	rtx_insn *con;
4525
4526	con = recompute_vec.pop ();
4527	MUST_RECOMPUTE_SPEC_P (con) = `0`;
4528	if (!sd_lists_empty_p (con, SD_LIST_HARD_BACK))
4529	{
4530	TODO_SPEC (con) = HARD_DEP;
4531	INSN_TICK (con) = INVALID_TICK;
4532	if (PREDICATED_PAT (con) != NULL_RTX)
4533	haifa_change_pattern (con, ORIG_PAT (con));
4534	}
4535	else if (QUEUE_INDEX (con) != QUEUE_SCHEDULED)
4536	TODO_SPEC (con) = recompute_todo_spec (next: con, for_backtrack: true);
4537	}
4538	}
4539
4540	/ Restore scheduler state from the topmost entry on the backtracking queue.*
4541	PSCHED_BLOCK_P points to the local data of schedule_block that we must
4542	overwrite with the saved data.
4543	The caller must already have called unschedule_insns_until. /*
4544
4545	static void
4546	restore_last_backtrack_point (struct sched_block_state *psched_block)
4547	{
4548	int i;
4549	struct haifa_saved_data *save = backtrack_queue;
4550
4551	backtrack_queue = save->next;
4552
4553	if (current_sched_info->restore_state)
4554	(*current_sched_info->restore_state) (save->fe_saved_data);
4555
4556	if (targetm.sched.alloc_sched_context)
4557	{
4558	targetm.sched.set_sched_context (save->be_saved_data);
4559	targetm.sched.free_sched_context (save->be_saved_data);
4560	}
4561
4562	/ Do this first since it clobbers INSN_TICK of the involved*
4563	instructions. /*
4564	undo_replacements_for_backtrack (save);
4565
4566	/ Clear the QUEUE_INDEX of everything in the ready list or one*
4567	of the queues. /*
4568	if (ready.n_ready > `0`)
4569	{
4570	rtx_insn **first = ready_lastpos (ready: &ready);
4571	for (i = `0`; i < ready.n_ready; i++)
4572	{
4573	rtx_insn *insn = first[i];
4574	QUEUE_INDEX (insn) = QUEUE_NOWHERE;
4575	INSN_TICK (insn) = INVALID_TICK;
4576	}
4577	}
4578	for (i = `0`; i <= max_insn_queue_index; i++)
4579	{
4580	int q = NEXT_Q_AFTER (q_ptr, i);
4581
4582	for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
4583	{
4584	rtx_insn *x = link->insn ();
4585	QUEUE_INDEX (x) = QUEUE_NOWHERE;
4586	INSN_TICK (x) = INVALID_TICK;
4587	}
4588	free_INSN_LIST_list (&insn_queue[q]);
4589	}
4590
4591	free (ptr: ready.vec);
4592	ready = save->ready;
4593
4594	if (ready.n_ready > `0`)
4595	{
4596	rtx_insn **first = ready_lastpos (ready: &ready);
4597	for (i = `0`; i < ready.n_ready; i++)
4598	{
4599	rtx_insn *insn = first[i];
4600	QUEUE_INDEX (insn) = QUEUE_READY;
4601	TODO_SPEC (insn) = recompute_todo_spec (next: insn, for_backtrack: true);
4602	INSN_TICK (insn) = save->clock_var;
4603	}
4604	}
4605
4606	q_ptr = `0`;
4607	q_size = save->q_size;
4608	for (i = `0`; i <= max_insn_queue_index; i++)
4609	{
4610	int q = NEXT_Q_AFTER (q_ptr, i);
4611
4612	insn_queue[q] = save->insn_queue[q];
4613
4614	for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
4615	{
4616	rtx_insn *x = link->insn ();
4617	QUEUE_INDEX (x) = i;
4618	TODO_SPEC (x) = recompute_todo_spec (next: x, for_backtrack: true);
4619	INSN_TICK (x) = save->clock_var + i;
4620	}
4621	}
4622	free (ptr: save->insn_queue);
4623
4624	toggle_cancelled_flags (set: true);
4625
4626	clock_var = save->clock_var;
4627	last_clock_var = save->last_clock_var;
4628	cycle_issued_insns = save->cycle_issued_insns;
4629	last_scheduled_insn = save->last_scheduled_insn;
4630	last_nondebug_scheduled_insn = save->last_nondebug_scheduled_insn;
4631	nonscheduled_insns_begin = save->nonscheduled_insns_begin;
4632
4633	*psched_block = save->sched_block;
4634
4635	memcpy (dest: curr_state, src: save->curr_state, n: dfa_state_size);
4636	free (ptr: save->curr_state);
4637
4638	mark_backtrack_feeds (insn: save->delay_pair->i2, set_p: `0`);
4639
4640	gcc_assert (next_cycle_replace_deps.is_empty ());
4641	next_cycle_replace_deps = save->next_cycle_deps.copy ();
4642	next_cycle_apply = save->next_cycle_apply.copy ();
4643
4644	free (ptr: save);
4645
4646	for (save = backtrack_queue; save; save = save->next)
4647	{
4648	mark_backtrack_feeds (insn: save->delay_pair->i2, set_p: `1`);
4649	}
4650	}
4651
4652	/ Discard all data associated with the topmost entry in the backtrack*
4653	queue. If RESET_TICK is false, we just want to free the data. If true,
4654	we are doing this because we discovered a reason to backtrack. In the
4655	latter case, also reset the INSN_TICK for the shadow insn. /*
4656	static void
4657	free_topmost_backtrack_point (bool reset_tick)
4658	{
4659	struct haifa_saved_data *save = backtrack_queue;
4660	int i;
4661
4662	backtrack_queue = save->next;
4663
4664	if (reset_tick)
4665	{
4666	struct delay_pair *pair = save->delay_pair;
4667	while (pair)
4668	{
4669	INSN_TICK (pair->i2) = INVALID_TICK;
4670	INSN_EXACT_TICK (pair->i2) = INVALID_TICK;
4671	pair = pair->next_same_i1;
4672	}
4673	undo_replacements_for_backtrack (save);
4674	}
4675	else
4676	{
4677	save->replacement_deps.release ();
4678	save->replace_apply.release ();
4679	}
4680
4681	if (targetm.sched.free_sched_context)
4682	targetm.sched.free_sched_context (save->be_saved_data);
4683	if (current_sched_info->restore_state)
4684	free (ptr: save->fe_saved_data);
4685	for (i = `0`; i <= max_insn_queue_index; i++)
4686	free_INSN_LIST_list (&save->insn_queue[i]);
4687	free (ptr: save->insn_queue);
4688	free (ptr: save->curr_state);
4689	free (ptr: save->ready.vec);
4690	free (ptr: save);
4691	}
4692
4693	/ Free the entire backtrack queue. /
4694	static void
4695	free_backtrack_queue (void)
4696	{
4697	while (backtrack_queue)
4698	free_topmost_backtrack_point (reset_tick: false);
4699	}
4700
4701	/ Apply a replacement described by DESC. If IMMEDIATELY is false, we*
4702	may have to postpone the replacement until the start of the next cycle,
4703	at which point we will be called again with IMMEDIATELY true. This is
4704	only done for machines which have instruction packets with explicit
4705	parallelism however. /*
4706	static void
4707	apply_replacement (dep_t dep, bool immediately)
4708	{
4709	struct dep_replacement *desc = DEP_REPLACE (dep);
4710	if (!immediately && targetm.sched.exposed_pipeline && reload_completed)
4711	{
4712	next_cycle_replace_deps.safe_push (obj: dep);
4713	next_cycle_apply.safe_push (obj: `1`);
4714	}
4715	else
4716	{
4717	bool success;
4718
4719	if (QUEUE_INDEX (desc->insn) == QUEUE_SCHEDULED)
4720	return;
4721
4722	if (sched_verbose >= `5`)
4723	fprintf (stream: sched_dump, format: "applying replacement for insn %d\n",
4724	INSN_UID (insn: desc->insn));
4725
4726	success = validate_change (desc->insn, desc->loc, desc->newval, `0`);
4727	gcc_assert (success);
4728
4729	rtx_insn *insn = DEP_PRO (dep);
4730
4731	/ Recompute priority since dependent priorities may have changed. /
4732	priority (insn, force_recompute: true);
4733	update_insn_after_change (insn: desc->insn);
4734
4735	if ((TODO_SPEC (desc->insn) & (HARD_DEP \| DEP_POSTPONED)) == `0`)
4736	fix_tick_ready (desc->insn);
4737
4738	if (backtrack_queue != NULL)
4739	{
4740	backtrack_queue->replacement_deps.safe_push (obj: dep);
4741	backtrack_queue->replace_apply.safe_push (obj: `1`);
4742	}
4743	}
4744	}
4745
4746	/ We have determined that a pattern involved in DEP must be restored.*
4747	If IMMEDIATELY is false, we may have to postpone the replacement
4748	until the start of the next cycle, at which point we will be called
4749	again with IMMEDIATELY true. /*
4750	static void
4751	restore_pattern (dep_t dep, bool immediately)
4752	{
4753	rtx_insn *next = DEP_CON (dep);
4754	int tick = INSN_TICK (next);
4755
4756	/ If we already scheduled the insn, the modified version is*
4757	correct. /*
4758	if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
4759	return;
4760
4761	if (!immediately && targetm.sched.exposed_pipeline && reload_completed)
4762	{
4763	next_cycle_replace_deps.safe_push (obj: dep);
4764	next_cycle_apply.safe_push (obj: `0`);
4765	return;
4766	}
4767
4768
4769	if (DEP_TYPE (dep) == REG_DEP_CONTROL)
4770	{
4771	if (sched_verbose >= `5`)
4772	fprintf (stream: sched_dump, format: "restoring pattern for insn %d\n",
4773	INSN_UID (insn: next));
4774	haifa_change_pattern (next, ORIG_PAT (next));
4775	}
4776	else
4777	{
4778	struct dep_replacement *desc = DEP_REPLACE (dep);
4779	bool success;
4780
4781	if (sched_verbose >= `5`)
4782	fprintf (stream: sched_dump, format: "restoring pattern for insn %d\n",
4783	INSN_UID (insn: desc->insn));
4784	tick = INSN_TICK (desc->insn);
4785
4786	success = validate_change (desc->insn, desc->loc, desc->orig, `0`);
4787	gcc_assert (success);
4788
4789	rtx_insn *insn = DEP_PRO (dep);
4790
4791	if (QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
4792	{
4793	/ Recompute priority since dependent priorities may have changed. /
4794	priority (insn, force_recompute: true);
4795	}
4796
4797	update_insn_after_change (insn: desc->insn);
4798
4799	if (backtrack_queue != NULL)
4800	{
4801	backtrack_queue->replacement_deps.safe_push (obj: dep);
4802	backtrack_queue->replace_apply.safe_push (obj: `0`);
4803	}
4804	}
4805	INSN_TICK (next) = tick;
4806	if (TODO_SPEC (next) == DEP_POSTPONED)
4807	return;
4808
4809	if (sd_lists_empty_p (next, SD_LIST_BACK))
4810	TODO_SPEC (next) = `0`;
4811	else if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
4812	TODO_SPEC (next) = HARD_DEP;
4813	}
4814
4815	/ Perform pattern replacements that were queued up until the next*
4816	cycle. /*
4817	static void
4818	perform_replacements_new_cycle (void)
4819	{
4820	int i;
4821	dep_t dep;
4822	FOR_EACH_VEC_ELT (next_cycle_replace_deps, i, dep)
4823	{
4824	int apply_p = next_cycle_apply [i];
4825	if (apply_p)
4826	apply_replacement (dep, immediately: true);
4827	else
4828	restore_pattern (dep, immediately: true);
4829	}
4830	next_cycle_replace_deps.truncate (size: `0`);
4831	next_cycle_apply.truncate (size: `0`);
4832	}
4833
4834	/ Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of*
4835	instructions we've previously encountered, a set bit prevents
4836	recursion. BUDGET is a limit on how far ahead we look, it is
4837	reduced on recursive calls. Return true if we produced a good
4838	estimate, or false if we exceeded the budget. /*
4839	static bool
4840	estimate_insn_tick (bitmap processed, rtx_insn insn, int* budget)
4841	{
4842	sd_iterator_def sd_it;
4843	dep_t dep;
4844	int earliest = INSN_TICK (insn);
4845
4846	FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
4847	{
4848	rtx_insn *pro = DEP_PRO (dep);
4849	int t;
4850
4851	if (DEP_STATUS (dep) & DEP_CANCELLED)
4852	continue;
4853
4854	if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED)
4855	gcc_assert (INSN_TICK (pro) + dep_cost (dep) <= INSN_TICK (insn));
4856	else
4857	{
4858	int cost = dep_cost (link: dep);
4859	if (cost >= budget)
4860	return false;
4861	if (!bitmap_bit_p (processed, INSN_LUID (pro)))
4862	{
4863	if (!estimate_insn_tick (processed, insn: pro, budget: budget - cost))
4864	return false;
4865	}
4866	gcc_assert (INSN_TICK_ESTIMATE (pro) != INVALID_TICK);
4867	t = INSN_TICK_ESTIMATE (pro) + cost;
4868	if (earliest == INVALID_TICK \|\| t > earliest)
4869	earliest = t;
4870	}
4871	}
4872	bitmap_set_bit (processed, INSN_LUID (insn));
4873	INSN_TICK_ESTIMATE (insn) = earliest;
4874	return true;
4875	}
4876
4877	/ Examine the pair of insns in P, and estimate (optimistically, assuming*
4878	infinite resources) the cycle in which the delayed shadow can be issued.
4879	Return the number of cycles that must pass before the real insn can be
4880	issued in order to meet this constraint. /*
4881	static int
4882	estimate_shadow_tick (struct delay_pair *p)
4883	{
4884	auto_bitmap processed;
4885	int t;
4886	bool cutoff;
4887
4888	cutoff = !estimate_insn_tick (processed, insn: p->i2,
4889	budget: max_insn_queue_index + pair_delay (p));
4890	if (cutoff)
4891	return max_insn_queue_index;
4892	t = INSN_TICK_ESTIMATE (p->i2) - (clock_var + pair_delay (p) + `1`);
4893	if (t > `0`)
4894	return t;
4895	return `0`;
4896	}
4897
4898	/ If INSN has no unresolved backwards dependencies, add it to the schedule and*
4899	recursively resolve all its forward dependencies. /*
4900	static void
4901	resolve_dependencies (rtx_insn *insn)
4902	{
4903	sd_iterator_def sd_it;
4904	dep_t dep;
4905
4906	/ Don't use sd_lists_empty_p; it ignores debug insns. /
4907	if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn)) != NULL
4908	\|\| DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn)) != NULL)
4909	return;
4910
4911	if (sched_verbose >= `4`)
4912	fprintf (stream: sched_dump, format: ";;\tquickly resolving %d\n", INSN_UID (insn));
4913
4914	if (QUEUE_INDEX (insn) >= `0`)
4915	queue_remove (insn);
4916
4917	scheduled_insns.safe_push (obj: insn);
4918
4919	/ Update dependent instructions. /
4920	for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
4921	sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep);)
4922	{
4923	rtx_insn *next = DEP_CON (dep);
4924
4925	if (sched_verbose >= `4`)
4926	fprintf (stream: sched_dump, format: ";;\t\tdep %d against %d\n", INSN_UID (insn),
4927	INSN_UID (insn: next));
4928
4929	/ Resolve the dependence between INSN and NEXT.*
4930	sd_resolve_dep () moves current dep to another list thus
4931	advancing the iterator. /*
4932	sd_resolve_dep (sd_it);
4933
4934	if (!IS_SPECULATION_BRANCHY_CHECK_P (insn))
4935	{
4936	resolve_dependencies (insn: next);
4937	}
4938	else
4939	/ Check always has only one forward dependence (to the first insn in*
4940	the recovery block), therefore, this will be executed only once. /*
4941	{
4942	gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW));
4943	}
4944	}
4945	}
4946
4947
4948	/ Return the head and tail pointers of ebb starting at BEG and ending*
4949	at END. /*
4950	void
4951	get_ebb_head_tail (basic_block beg, basic_block end,
4952	rtx_insn headp, rtx_insn tailp)
4953	{
4954	rtx_insn *beg_head = BB_HEAD (beg);
4955	rtx_insn * beg_tail = BB_END (beg);
4956	rtx_insn * end_head = BB_HEAD (end);
4957	rtx_insn * end_tail = BB_END (end);
4958
4959	/ Don't include any notes or labels at the beginning of the BEG*
4960	basic block, or notes at the end of the END basic blocks. /*
4961
4962	if (LABEL_P (beg_head))
4963	beg_head = NEXT_INSN (insn: beg_head);
4964
4965	while (beg_head != beg_tail)
4966	if (NOTE_P (beg_head))
4967	beg_head = NEXT_INSN (insn: beg_head);
4968	else if (DEBUG_INSN_P (beg_head))
4969	{
4970	rtx_insn * note, *next;
4971
4972	for (note = NEXT_INSN (insn: beg_head);
4973	note != beg_tail;
4974	note = next)
4975	{
4976	next = NEXT_INSN (insn: note);
4977	if (NOTE_P (note))
4978	{
4979	if (sched_verbose >= `9`)
4980	fprintf (stream: sched_dump, format: "reorder %i\n", INSN_UID (insn: note));
4981
4982	reorder_insns_nobb (note, note, PREV_INSN (insn: beg_head));
4983
4984	if (BLOCK_FOR_INSN (insn: note) != beg)
4985	df_insn_change_bb (note, beg);
4986	}
4987	else if (!DEBUG_INSN_P (note))
4988	break;
4989	}
4990
4991	break;
4992	}
4993	else
4994	break;
4995
4996	*headp = beg_head;
4997
4998	if (beg == end)
4999	end_head = beg_head;
5000	else if (LABEL_P (end_head))
5001	end_head = NEXT_INSN (insn: end_head);
5002
5003	while (end_head != end_tail)
5004	if (NOTE_P (end_tail))
5005	end_tail = PREV_INSN (insn: end_tail);
5006	else if (DEBUG_INSN_P (end_tail))
5007	{
5008	rtx_insn * note, *prev;
5009
5010	for (note = PREV_INSN (insn: end_tail);
5011	note != end_head;
5012	note = prev)
5013	{
5014	prev = PREV_INSN (insn: note);
5015	if (NOTE_P (note))
5016	{
5017	if (sched_verbose >= `9`)
5018	fprintf (stream: sched_dump, format: "reorder %i\n", INSN_UID (insn: note));
5019
5020	reorder_insns_nobb (note, note, end_tail);
5021
5022	if (end_tail == BB_END (end))
5023	BB_END (end) = note;
5024
5025	if (BLOCK_FOR_INSN (insn: note) != end)
5026	df_insn_change_bb (note, end);
5027	}
5028	else if (!DEBUG_INSN_P (note))
5029	break;
5030	}
5031
5032	break;
5033	}
5034	else
5035	break;
5036
5037	*tailp = end_tail;
5038	}
5039
5040	/ Return true if there are no real insns in the range [ HEAD, TAIL ]. /
5041
5042	bool
5043	no_real_insns_p (const rtx_insn head, const* rtx_insn *tail)
5044	{
5045	while (head != NEXT_INSN (insn: tail))
5046	{
5047	if (!NOTE_P (head) && !LABEL_P (head))
5048	return false;
5049	head = NEXT_INSN (insn: head);
5050	}
5051	return true;
5052	}
5053
5054	/ Restore-other-notes: NOTE_LIST is the end of a chain of notes*
5055	previously found among the insns. Insert them just before HEAD. /*
5056	rtx_insn *
5057	restore_other_notes (rtx_insn *head, basic_block head_bb)
5058	{
5059	if (note_list != `0`)
5060	{
5061	rtx_insn *note_head = note_list;
5062
5063	if (head)
5064	head_bb = BLOCK_FOR_INSN (insn: head);
5065	else
5066	head = NEXT_INSN (insn: bb_note (head_bb));
5067
5068	while (PREV_INSN (insn: note_head))
5069	{
5070	set_block_for_insn (insn: note_head, bb: head_bb);
5071	note_head = PREV_INSN (insn: note_head);
5072	}
5073	/ In the above cycle we've missed this note. /
5074	set_block_for_insn (insn: note_head, bb: head_bb);
5075
5076	SET_PREV_INSN (note_head) = PREV_INSN (insn: head);
5077	SET_NEXT_INSN (PREV_INSN (insn: head)) = note_head;
5078	SET_PREV_INSN (head) = note_list;
5079	SET_NEXT_INSN (note_list) = head;
5080
5081	if (BLOCK_FOR_INSN (insn: head) != head_bb)
5082	BB_END (head_bb) = note_list;
5083
5084	head = note_head;
5085	}
5086
5087	return head;
5088	}
5089
5090	/ When we know we are going to discard the schedule due to a failed attempt*
5091	at modulo scheduling, undo all replacements. /*
5092	static void
5093	undo_all_replacements (void)
5094	{
5095	rtx_insn *insn;
5096	int i;
5097
5098	FOR_EACH_VEC_ELT (scheduled_insns, i, insn)
5099	{
5100	sd_iterator_def sd_it;
5101	dep_t dep;
5102
5103	/ See if we must undo a replacement. /
5104	for (sd_it = sd_iterator_start (insn, SD_LIST_RES_FORW);
5105	sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep); sd_iterator_next (it_ptr: &sd_it))
5106	{
5107	struct dep_replacement *desc = DEP_REPLACE (dep);
5108	if (desc != NULL)
5109	validate_change (desc->insn, desc->loc, desc->orig, `0`);
5110	}
5111	}
5112	}
5113
5114	/ Return first non-scheduled insn in the current scheduling block.*
5115	This is mostly used for debug-counter purposes. /*
5116	static rtx_insn *
5117	first_nonscheduled_insn (void)
5118	{
5119	rtx_insn *insn = (nonscheduled_insns_begin != NULL_RTX
5120	? nonscheduled_insns_begin
5121	: current_sched_info->prev_head);
5122
5123	do
5124	{
5125	insn = next_nonnote_nondebug_insn (insn);
5126	}
5127	while (QUEUE_INDEX (insn) == QUEUE_SCHEDULED);
5128
5129	return insn;
5130	}
5131
5132	/ Move insns that became ready to fire from queue to ready list. /
5133
5134	static void
5135	queue_to_ready (struct ready_list *ready)
5136	{
5137	rtx_insn *insn;
5138	rtx_insn_list *link;
5139	rtx_insn *skip_insn;
5140
5141	q_ptr = NEXT_Q (q_ptr);
5142
5143	if (dbg_cnt (index: sched_insn) == false)
5144	/ If debug counter is activated do not requeue the first*
5145	nonscheduled insn. /*
5146	skip_insn = first_nonscheduled_insn ();
5147	else
5148	skip_insn = NULL;
5149
5150	/ Add all pending insns that can be scheduled without stalls to the*
5151	ready list. /*
5152	for (link = insn_queue[q_ptr]; link; link = link->next ())
5153	{
5154	insn = link->insn ();
5155	q_size -= `1`;
5156
5157	if (sched_verbose >= `2`)
5158	fprintf (stream: sched_dump, format: ";;\t\tQ-->Ready: insn %s: ",
5159	(*current_sched_info->print_insn) (insn, `0`));
5160
5161	/ If the ready list is full, delay the insn for 1 cycle.*
5162	See the comment in schedule_block for the rationale. /*
5163	if (!reload_completed
5164	&& (ready->n_ready - ready->n_debug > param_max_sched_ready_insns
5165	\|\| (sched_pressure == SCHED_PRESSURE_MODEL
5166	/ Limit pressure recalculations to*
5167	param_max_sched_ready_insns instructions too. /*
5168	&& model_index (insn) > (model_curr_point
5169	+ param_max_sched_ready_insns)))
5170	&& !(sched_pressure == SCHED_PRESSURE_MODEL
5171	&& model_curr_point < model_num_insns
5172	/ Always allow the next model instruction to issue. /
5173	&& model_index (insn) == model_curr_point)
5174	&& !SCHED_GROUP_P (insn)
5175	&& insn != skip_insn)
5176	{
5177	if (sched_verbose >= `2`)
5178	fprintf (stream: sched_dump, format: "keeping in queue, ready full\n");
5179	queue_insn (insn, n_cycles: `1`, reason: "ready full");
5180	}
5181	else
5182	{
5183	ready_add (ready, insn, first_p: false);
5184	if (sched_verbose >= `2`)
5185	fprintf (stream: sched_dump, format: "moving to ready without stalls\n");
5186	}
5187	}
5188	free_INSN_LIST_list (&insn_queue[q_ptr]);
5189
5190	/ If there are no ready insns, stall until one is ready and add all*
5191	of the pending insns at that point to the ready list. /*
5192	if (ready->n_ready == `0`)
5193	{
5194	int stalls;
5195
5196	for (stalls = `1`; stalls <= max_insn_queue_index; stalls++)
5197	{
5198	if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
5199	{
5200	for (; link; link = link->next ())
5201	{
5202	insn = link->insn ();
5203	q_size -= `1`;
5204
5205	if (sched_verbose >= `2`)
5206	fprintf (stream: sched_dump, format: ";;\t\tQ-->Ready: insn %s: ",
5207	(*current_sched_info->print_insn) (insn, `0`));
5208
5209	ready_add (ready, insn, first_p: false);
5210	if (sched_verbose >= `2`)
5211	fprintf (stream: sched_dump, format: "moving to ready with %d stalls\n", stalls);
5212	}
5213	free_INSN_LIST_list (&insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]);
5214
5215	advance_one_cycle ();
5216
5217	break;
5218	}
5219
5220	advance_one_cycle ();
5221	}
5222
5223	q_ptr = NEXT_Q_AFTER (q_ptr, stalls);
5224	clock_var += stalls;
5225	if (sched_verbose >= `2`)
5226	fprintf (stream: sched_dump, format: ";;\tAdvancing clock by %d cycle[s] to %d\n",
5227	stalls, clock_var);
5228	}
5229	}
5230
5231	/ Used by early_queue_to_ready. Determines whether it is "ok" to*
5232	prematurely move INSN from the queue to the ready list. Currently,
5233	if a target defines the hook 'is_costly_dependence', this function
5234	uses the hook to check whether there exist any dependences which are
5235	considered costly by the target, between INSN and other insns that
5236	have already been scheduled. Dependences are checked up to Y cycles
5237	back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
5238	controlling this value.
5239	(Other considerations could be taken into account instead (or in
5240	addition) depending on user flags and target hooks. /*
5241
5242	static bool
5243	ok_for_early_queue_removal (rtx_insn *insn)
5244	{
5245	if (targetm.sched.is_costly_dependence)
5246	{
5247	int n_cycles;
5248	int i = scheduled_insns.length ();
5249	for (n_cycles = flag_sched_stalled_insns_dep; n_cycles; n_cycles--)
5250	{
5251	while (i-- > `0`)
5252	{
5253	int cost;
5254
5255	rtx_insn *prev_insn = scheduled_insns [i];
5256
5257	if (!NOTE_P (prev_insn))
5258	{
5259	dep_t dep;
5260
5261	dep = sd_find_dep_between (prev_insn, insn, true);
5262
5263	if (dep != NULL)
5264	{
5265	cost = dep_cost (link: dep);
5266
5267	if (targetm.sched.is_costly_dependence (dep, cost,
5268	flag_sched_stalled_insns_dep - n_cycles))
5269	return false;
5270	}
5271	}
5272
5273	if (GET_MODE (prev_insn) == TImode) / end of dispatch group /
5274	break;
5275	}
5276
5277	if (i == `0`)
5278	break;
5279	}
5280	}
5281
5282	return true;
5283	}
5284
5285
5286	/ Remove insns from the queue, before they become "ready" with respect*
5287	to FU latency considerations. /*
5288
5289	static int
5290	early_queue_to_ready (state_t state, struct ready_list *ready)
5291	{
5292	rtx_insn *insn;
5293	rtx_insn_list *link;
5294	rtx_insn_list *next_link;
5295	rtx_insn_list *prev_link;
5296	bool move_to_ready;
5297	int cost;
5298	state_t temp_state = alloca (dfa_state_size);
5299	int stalls;
5300	int insns_removed = `0`;
5301
5302	/*
5303	Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
5304	function:
5305
5306	X == 0: There is no limit on how many queued insns can be removed
5307	prematurely. (flag_sched_stalled_insns = -1).
5308
5309	X >= 1: Only X queued insns can be removed prematurely in each
5310	invocation. (flag_sched_stalled_insns = X).
5311
5312	Otherwise: Early queue removal is disabled.
5313	(flag_sched_stalled_insns = 0)
5314	*/
5315
5316	if (! flag_sched_stalled_insns)
5317	return `0`;
5318
5319	for (stalls = `0`; stalls <= max_insn_queue_index; stalls++)
5320	{
5321	if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
5322	{
5323	if (sched_verbose > `6`)
5324	fprintf (stream: sched_dump, format: ";; look at index %d + %d\n", q_ptr, stalls);
5325
5326	prev_link = `0`;
5327	while (link)
5328	{
5329	next_link = link->next ();
5330	insn = link->insn ();
5331	if (insn && sched_verbose > `6`)
5332	print_rtl_single (sched_dump, insn);
5333
5334	memcpy (dest: temp_state, src: state, n: dfa_state_size);
5335	if (recog_memoized (insn) < `0`)
5336	/ non-negative to indicate that it's not ready*
5337	to avoid infinite Q->R->Q->R... /*
5338	cost = `0`;
5339	else
5340	cost = state_transition (temp_state, insn);
5341
5342	if (sched_verbose >= `6`)
5343	fprintf (stream: sched_dump, format: "transition cost = %d\n", cost);
5344
5345	move_to_ready = false;
5346	if (cost < `0`)
5347	{
5348	move_to_ready = ok_for_early_queue_removal (insn);
5349	if (move_to_ready == true)
5350	{
5351	/ move from Q to R /
5352	q_size -= `1`;
5353	ready_add (ready, insn, first_p: false);
5354
5355	if (prev_link)
5356	XEXP (prev_link, `1`) = next_link;
5357	else
5358	insn_queue[NEXT_Q_AFTER (q_ptr, stalls)] = next_link;
5359
5360	free_INSN_LIST_node (link);
5361
5362	if (sched_verbose >= `2`)
5363	fprintf (stream: sched_dump, format: ";;\t\tEarly Q-->Ready: insn %s\n",
5364	(*current_sched_info->print_insn) (insn, `0`));
5365
5366	insns_removed++;
5367	if (insns_removed == flag_sched_stalled_insns)
5368	/ Remove no more than flag_sched_stalled_insns insns*
5369	from Q at a time. /*
5370	return insns_removed;
5371	}
5372	}
5373
5374	if (move_to_ready == false)
5375	prev_link = link;
5376
5377	link = next_link;
5378	} / while link /
5379	} / if link /
5380
5381	} / for stalls.. /
5382
5383	return insns_removed;
5384	}
5385
5386
5387	/ Print the ready list for debugging purposes.*
5388	If READY_TRY is non-zero then only print insns that max_issue
5389	will consider. /*
5390	static void
5391	debug_ready_list_1 (struct ready_list ready, signed* char *ready_try)
5392	{
5393	rtx_insn **p;
5394	int i;
5395
5396	if (ready->n_ready == `0`)
5397	{
5398	fprintf (stream: sched_dump, format: "\n");
5399	return;
5400	}
5401
5402	p = ready_lastpos (ready);
5403	for (i = `0`; i < ready->n_ready; i++)
5404	{
5405	if (ready_try != NULL && ready_try[ready->n_ready - i - `1`])
5406	continue;
5407
5408	fprintf (stream: sched_dump, format: " %s:%d",
5409	(*current_sched_info->print_insn) (p[i], `0`),
5410	INSN_LUID (p[i]));
5411	if (sched_pressure != SCHED_PRESSURE_NONE)
5412	fprintf (stream: sched_dump, format: "(cost=%d",
5413	INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p[i]));
5414	fprintf (stream: sched_dump, format: ":prio=%d", INSN_PRIORITY (p[i]));
5415	if (INSN_TICK (p[i]) > clock_var)
5416	fprintf (stream: sched_dump, format: ":delay=%d", INSN_TICK (p[i]) - clock_var);
5417	if (sched_pressure == SCHED_PRESSURE_MODEL)
5418	fprintf (stream: sched_dump, format: ":idx=%d",
5419	model_index (insn: p[i]));
5420	if (sched_pressure != SCHED_PRESSURE_NONE)
5421	fprintf (stream: sched_dump, format: ")");
5422	}
5423	fprintf (stream: sched_dump, format: "\n");
5424	}
5425
5426	/ Print the ready list. Callable from debugger. /
5427	static void
5428	debug_ready_list (struct ready_list *ready)
5429	{
5430	debug_ready_list_1 (ready, NULL);
5431	}
5432
5433	/ Search INSN for REG_SAVE_NOTE notes and convert them back into insn*
5434	NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
5435	replaces the epilogue note in the correct basic block. /*
5436	void
5437	reemit_notes (rtx_insn *insn)
5438	{
5439	rtx note;
5440	rtx_insn *last = insn;
5441
5442	for (note = REG_NOTES (insn); note; note = XEXP (note, `1`))
5443	{
5444	if (REG_NOTE_KIND (note) == REG_SAVE_NOTE)
5445	{
5446	enum insn_note note_type = (enum insn_note) INTVAL (XEXP (note, `0`));
5447
5448	last = emit_note_before (note_type, last);
5449	remove_note (insn, note);
5450	df_insn_create_insn_record (last);
5451	}
5452	}
5453	}
5454
5455	/ Move INSN. Reemit notes if needed. Update CFG, if needed. /
5456	static void
5457	move_insn (rtx_insn insn, rtx_insn last, rtx nt)
5458	{
5459	if (PREV_INSN (insn) != last)
5460	{
5461	basic_block bb;
5462	rtx_insn *note;
5463	int jump_p = `0`;
5464
5465	bb = BLOCK_FOR_INSN (insn);
5466
5467	/ BB_HEAD is either LABEL or NOTE. /
5468	gcc_assert (BB_HEAD (bb) != insn);
5469
5470	if (BB_END (bb) == insn)
5471	/ If this is last instruction in BB, move end marker one*
5472	instruction up. /*
5473	{
5474	/ Jumps are always placed at the end of basic block. /
5475	jump_p = control_flow_insn_p (insn);
5476
5477	gcc_assert (!jump_p
5478	\|\| ((common_sched_info->sched_pass_id == SCHED_RGN_PASS)
5479	&& IS_SPECULATION_BRANCHY_CHECK_P (insn))
5480	\|\| (common_sched_info->sched_pass_id
5481	== SCHED_EBB_PASS));
5482
5483	gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn)) == bb);
5484
5485	BB_END (bb) = PREV_INSN (insn);
5486	}
5487
5488	gcc_assert (BB_END (bb) != last);
5489
5490	if (jump_p)
5491	/ We move the block note along with jump. /
5492	{
5493	gcc_assert (nt);
5494
5495	note = NEXT_INSN (insn);
5496	while (NOTE_NOT_BB_P (note) && note != nt)
5497	note = NEXT_INSN (insn: note);
5498
5499	if (note != nt
5500	&& (LABEL_P (note)
5501	\|\| BARRIER_P (note)))
5502	note = NEXT_INSN (insn: note);
5503
5504	gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
5505	}
5506	else
5507	note = insn;
5508
5509	SET_NEXT_INSN (PREV_INSN (insn)) = NEXT_INSN (insn: note);
5510	SET_PREV_INSN (NEXT_INSN (insn: note)) = PREV_INSN (insn);
5511
5512	SET_NEXT_INSN (note) = NEXT_INSN (insn: last);
5513	SET_PREV_INSN (NEXT_INSN (insn: last)) = note;
5514
5515	SET_NEXT_INSN (last) = insn;
5516	SET_PREV_INSN (insn) = last;
5517
5518	bb = BLOCK_FOR_INSN (insn: last);
5519
5520	if (jump_p)
5521	{
5522	fix_jump_move (insn);
5523
5524	if (BLOCK_FOR_INSN (insn) != bb)
5525	move_block_after_check (insn);
5526
5527	gcc_assert (BB_END (bb) == last);
5528	}
5529
5530	df_insn_change_bb (insn, bb);
5531
5532	/ Update BB_END, if needed. /
5533	if (BB_END (bb) == last)
5534	BB_END (bb) = insn;
5535	}
5536
5537	SCHED_GROUP_P (insn) = `0`;
5538	}
5539
5540	/ Return true if scheduling INSN will finish current clock cycle. /
5541	static bool
5542	insn_finishes_cycle_p (rtx_insn *insn)
5543	{
5544	if (SCHED_GROUP_P (insn))
5545	/ After issuing INSN, rest of the sched_group will be forced to issue*
5546	in order. Don't make any plans for the rest of cycle. /*
5547	return true;
5548
5549	/ Finishing the block will, apparently, finish the cycle. /
5550	if (current_sched_info->insn_finishes_block_p
5551	&& current_sched_info->insn_finishes_block_p (insn))
5552	return true;
5553
5554	return false;
5555	}
5556
5557	/ Helper for autopref_multipass_init. Given a SET in PAT and whether*
5558	we're expecting a memory WRITE or not, check that the insn is relevant to
5559	the autoprefetcher modelling code. Return true iff that is the case.
5560	If it is relevant, record the base register of the memory op in BASE and
5561	the offset in OFFSET. /*
5562
5563	static bool
5564	analyze_set_insn_for_autopref (rtx pat, bool write, rtx base, int* *offset)
5565	{
5566	if (GET_CODE (pat) != SET)
5567	return false;
5568
5569	rtx mem = write ? SET_DEST (pat) : SET_SRC (pat);
5570	if (!MEM_P (mem))
5571	return false;
5572
5573	struct address_info info;
5574	decompose_mem_address (&info, mem);
5575
5576	/ TODO: Currently only (base+const) addressing is supported. /
5577	if (info.base == NULL \|\| !REG_P (*info.base)
5578	\|\| (info.disp != NULL && !CONST_INT_P (*info.disp)))
5579	return false;
5580
5581	base = info.base;
5582	offset = info.disp ? INTVAL (info.disp) : `0`;
5583	return true;
5584	}
5585
5586	/ Functions to model cache auto-prefetcher.*
5587
5588	Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
5589	memory prefetches if it sees instructions with consequitive memory accesses
5590	in the instruction stream. Details of such hardware units are not published,
5591	so we can only guess what exactly is going on there.
5592	In the scheduler, we model abstract auto-prefetcher. If there are memory
5593	insns in the ready list (or the queue) that have same memory base, but
5594	different offsets, then we delay the insns with larger offsets until insns
5595	with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
5596	is "1", then we look at the ready list; if it is N>1, then we also look
5597	through N-1 queue entries.
5598	If the param is N>=0, then rank_for_schedule will consider auto-prefetching
5599	among its heuristics.
5600	Param value of "-1" disables modelling of the auto-prefetcher. /*
5601
5602	/ Initialize autoprefetcher model data for INSN. /
5603	static void
5604	autopref_multipass_init (const rtx_insn insn, int* write)
5605	{
5606	autopref_multipass_data_t data = &INSN_AUTOPREF_MULTIPASS_DATA (insn)[write];
5607
5608	gcc_assert (data->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED);
5609	data->base = NULL_RTX;
5610	data->offset = `0`;
5611	/ Set insn entry initialized, but not relevant for auto-prefetcher. /
5612	data->status = AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5613
5614	rtx pat = PATTERN (insn);
5615
5616	/ We have a multi-set insn like a load-multiple or store-multiple.*
5617	We care about these as long as all the memory ops inside the PARALLEL
5618	have the same base register. We care about the minimum and maximum
5619	offsets from that base but don't check for the order of those offsets
5620	within the PARALLEL insn itself. /*
5621	if (GET_CODE (pat) == PARALLEL)
5622	{
5623	int n_elems = XVECLEN (pat, `0`);
5624
5625	int i, offset;
5626	rtx base, prev_base = NULL_RTX;
5627	int min_offset = INT_MAX;
5628
5629	for (i = `0`; i < n_elems; i++)
5630	{
5631	rtx set = XVECEXP (pat, `0`, i);
5632	if (GET_CODE (set) != SET)
5633	return;
5634
5635	if (!analyze_set_insn_for_autopref (pat: set, write, base: &base, offset: &offset))
5636	return;
5637
5638	/ Ensure that all memory operations in the PARALLEL use the same*
5639	base register. /*
5640	if (i > `0` && REGNO (base) != REGNO (prev_base))
5641	return;
5642	prev_base = base;
5643	min_offset = MIN (min_offset, offset);
5644	}
5645
5646	/ If we reached here then we have a valid PARALLEL of multiple memory ops*
5647	with prev_base as the base and min_offset containing the offset. /*
5648	gcc_assert (prev_base);
5649	data->base = prev_base;
5650	data->offset = min_offset;
5651	data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
5652	return;
5653	}
5654
5655	/ Otherwise this is a single set memory operation. /
5656	rtx set = single_set (insn);
5657	if (set == NULL_RTX)
5658	return;
5659
5660	if (!analyze_set_insn_for_autopref (pat: set, write, base: &data->base,
5661	offset: &data->offset))
5662	return;
5663
5664	/ This insn is relevant for the auto-prefetcher.*
5665	The base and offset fields will have been filled in the
5666	analyze_set_insn_for_autopref call above. /*
5667	data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
5668	}
5669
5670	/ Helper function for rank_for_schedule sorting. /
5671	static int
5672	autopref_rank_for_schedule (const rtx_insn insn1, const* rtx_insn *insn2)
5673	{
5674	int r = `0`;
5675	for (int write = `0`; write < `2` && !r; ++write)
5676	{
5677	autopref_multipass_data_t data1
5678	= &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5679	autopref_multipass_data_t data2
5680	= &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
5681
5682	if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5683	autopref_multipass_init (insn: insn1, write);
5684
5685	if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5686	autopref_multipass_init (insn: insn2, write);
5687
5688	int irrel1 = data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5689	int irrel2 = data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5690
5691	if (!irrel1 && !irrel2)
5692	/ Sort memory references from lowest offset to the largest. /
5693	r = (data1->offset > data2->offset) - (data1->offset < data2->offset);
5694	else if (write)
5695	/ Schedule "irrelevant" insns before memory stores to resolve*
5696	as many producer dependencies of stores as possible. /*
5697	r = irrel2 - irrel1;
5698	else
5699	/ Schedule "irrelevant" insns after memory reads to avoid breaking*
5700	memory read sequences. /*
5701	r = irrel1 - irrel2;
5702	}
5703
5704	return r;
5705	}
5706
5707	/ True if header of debug dump was printed. /
5708	static bool autopref_multipass_dfa_lookahead_guard_started_dump_p;
5709
5710	/ Helper for autopref_multipass_dfa_lookahead_guard.*
5711	Return "1" if INSN1 should be delayed in favor of INSN2. /*
5712	static int
5713	autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn *insn1,
5714	const rtx_insn insn2, int* write)
5715	{
5716	autopref_multipass_data_t data1
5717	= &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5718	autopref_multipass_data_t data2
5719	= &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
5720
5721	if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5722	autopref_multipass_init (insn: insn2, write);
5723	if (data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
5724	return `0`;
5725
5726	if (rtx_equal_p (data1->base, data2->base)
5727	&& data1->offset > data2->offset)
5728	{
5729	if (sched_verbose >= `2`)
5730	{
5731	if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
5732	{
5733	fprintf (stream: sched_dump,
5734	format: ";;\t\tnot trying in max_issue due to autoprefetch "
5735	"model: ");
5736	autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
5737	}
5738
5739	fprintf (stream: sched_dump, format: " %d(%d)", INSN_UID (insn: insn1), INSN_UID (insn: insn2));
5740	}
5741
5742	return `1`;
5743	}
5744
5745	return `0`;
5746	}
5747
5748	/ General note:*
5749
5750	We could have also hooked autoprefetcher model into
5751	first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
5752	to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
5753	(e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
5754	unblocked). We don't bother about this yet because target of interest
5755	(ARM Cortex-A15) can issue only 1 memory operation per cycle. /*
5756
5757	/ Implementation of first_cycle_multipass_dfa_lookahead_guard hook.*
5758	Return "1" if INSN1 should not be considered in max_issue due to
5759	auto-prefetcher considerations. /*
5760	int
5761	autopref_multipass_dfa_lookahead_guard (rtx_insn insn1, int* ready_index)
5762	{
5763	int r = `0`;
5764
5765	/ Exit early if the param forbids this or if we're not entering here through*
5766	normal haifa scheduling. This can happen if selective scheduling is
5767	explicitly enabled. /*
5768	if (!insn_queue \|\| param_sched_autopref_queue_depth <= `0`)
5769	return `0`;
5770
5771	if (sched_verbose >= `2` && ready_index == `0`)
5772	autopref_multipass_dfa_lookahead_guard_started_dump_p = false;
5773
5774	for (int write = `0`; write < `2`; ++write)
5775	{
5776	autopref_multipass_data_t data1
5777	= &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5778
5779	if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5780	autopref_multipass_init (insn: insn1, write);
5781	if (data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
5782	continue;
5783
5784	if (ready_index == `0`
5785	&& data1->status == AUTOPREF_MULTIPASS_DATA_DONT_DELAY)
5786	/ We allow only a single delay on priviledged instructions.*
5787	Doing otherwise would cause infinite loop. /*
5788	{
5789	if (sched_verbose >= `2`)
5790	{
5791	if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
5792	{
5793	fprintf (stream: sched_dump,
5794	format: ";;\t\tnot trying in max_issue due to autoprefetch "
5795	"model: ");
5796	autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
5797	}
5798
5799	fprintf (stream: sched_dump, format: " %d", INSN_UID (insn: insn1));
5800	}
5801	continue;
5802	}
5803
5804	for (int i2 = `0`; i2 < ready.n_ready; ++i2)
5805	{
5806	rtx_insn *insn2 = get_ready_element (i2);
5807	if (insn1 == insn2)
5808	continue;
5809	r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2, write);
5810	if (r)
5811	{
5812	if (ready_index == `0`)
5813	{
5814	r = -`1`;
5815	data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
5816	}
5817	goto finish;
5818	}
5819	}
5820
5821	if (param_sched_autopref_queue_depth == `1`)
5822	continue;
5823
5824	/ Everything from the current queue slot should have been moved to*
5825	the ready list. /*
5826	gcc_assert (insn_queue[NEXT_Q_AFTER (q_ptr, `0`)] == NULL_RTX);
5827
5828	int n_stalls = param_sched_autopref_queue_depth - `1`;
5829	if (n_stalls > max_insn_queue_index)
5830	n_stalls = max_insn_queue_index;
5831
5832	for (int stalls = `1`; stalls <= n_stalls; ++stalls)
5833	{
5834	for (rtx_insn_list *link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)];
5835	link != NULL_RTX;
5836	link = link->next ())
5837	{
5838	rtx_insn *insn2 = link->insn ();
5839	r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2,
5840	write);
5841	if (r)
5842	{
5843	/ Queue INSN1 until INSN2 can issue. /
5844	r = -stalls;
5845	if (ready_index == `0`)
5846	data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
5847	goto finish;
5848	}
5849	}
5850	}
5851	}
5852
5853	finish:
5854	if (sched_verbose >= `2`
5855	&& autopref_multipass_dfa_lookahead_guard_started_dump_p
5856	&& (ready_index == ready.n_ready - `1` \|\| r < `0`))
5857	/ This does not /always/ trigger. We don't output EOL if the last*
5858	insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
5859	called. We can live with this. /*
5860	fprintf (stream: sched_dump, format: "\n");
5861
5862	return r;
5863	}
5864
5865	/ Define type for target data used in multipass scheduling. /
5866	#ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
5867	# define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
5868	#endif
5869	typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t;
5870
5871	/ The following structure describe an entry of the stack of choices. /
5872	struct choice_entry
5873	{
5874	/ Ordinal number of the issued insn in the ready queue. /
5875	int index;
5876	/ The number of the rest insns whose issues we should try. /
5877	int rest;
5878	/ The number of issued essential insns. /
5879	int n;
5880	/ State after issuing the insn. /
5881	state_t state;
5882	/ Target-specific data. /
5883	first_cycle_multipass_data_t target_data;
5884	};
5885
5886	/ The following array is used to implement a stack of choices used in*
5887	function max_issue. /*
5888	static struct choice_entry *choice_stack;
5889
5890	/ This holds the value of the target dfa_lookahead hook. /
5891	int dfa_lookahead;
5892
5893	/ The following variable value is maximal number of tries of issuing*
5894	insns for the first cycle multipass insn scheduling. We define
5895	this value as constant(DFA_LOOKAHEAD*ISSUE_RATE). We would not
5896	need this constraint if all real insns (with non-negative codes)
5897	had reservations because in this case the algorithm complexity is
5898	O(DFA_LOOKAHEADISSUE_RATE). Unfortunately, the dfa descriptions
5899	might be incomplete and such insn might occur. For such
5900	descriptions, the complexity of algorithm (without the constraint)
5901	could achieve DFA_LOOKAHEAD * N , where N is the queue length. /
5902	static int max_lookahead_tries;
5903
5904	/ The following function returns maximal (or close to maximal) number*
5905	of insns which can be issued on the same cycle and one of which
5906	insns is insns with the best rank (the first insn in READY). To
5907	make this function tries different samples of ready insns. READY
5908	is current queue `ready'. Global array READY_TRY reflects what
5909	insns are already issued in this try. The function stops immediately,
5910	if it reached the such a solution, that all instruction can be issued.
5911	INDEX will contain index of the best insn in READY. The following
5912	function is used only for first cycle multipass scheduling.
5913
5914	PRIVILEGED_N >= 0
5915
5916	This function expects recognized insns only. All USEs,
5917	CLOBBERs, etc must be filtered elsewhere. /*
5918	int
5919	max_issue (struct ready_list ready, int* privileged_n, state_t state,
5920	bool first_cycle_insn_p, int *index)
5921	{
5922	int n, i, all, n_ready, best, delay, tries_num;
5923	int more_issue;
5924	struct choice_entry *top;
5925	rtx_insn *insn;
5926
5927	if (sched_fusion)
5928	return `0`;
5929
5930	n_ready = ready->n_ready;
5931	gcc_assert (dfa_lookahead >= `1` && privileged_n >= `0`
5932	&& privileged_n <= n_ready);
5933
5934	/ Init MAX_LOOKAHEAD_TRIES. /
5935	if (max_lookahead_tries == `0`)
5936	{
5937	max_lookahead_tries = `100`;
5938	for (i = `0`; i < issue_rate; i++)
5939	max_lookahead_tries *= dfa_lookahead;
5940	}
5941
5942	/ Init max_points. /
5943	more_issue = issue_rate - cycle_issued_insns;
5944	gcc_assert (more_issue >= `0`);
5945
5946	/ The number of the issued insns in the best solution. /
5947	best = `0`;
5948
5949	top = choice_stack;
5950
5951	/ Set initial state of the search. /
5952	memcpy (dest: top->state, src: state, n: dfa_state_size);
5953	top->rest = dfa_lookahead;
5954	top->n = `0`;
5955	if (targetm.sched.first_cycle_multipass_begin)
5956	targetm.sched.first_cycle_multipass_begin (&top->target_data,
5957	ready_try, n_ready,
5958	first_cycle_insn_p);
5959
5960	/ Count the number of the insns to search among. /
5961	for (all = i = `0`; i < n_ready; i++)
5962	if (!ready_try [i])
5963	all++;
5964
5965	if (sched_verbose >= `2`)
5966	{
5967	fprintf (stream: sched_dump, format: ";;\t\tmax_issue among %d insns:", all);
5968	debug_ready_list_1 (ready, ready_try);
5969	}
5970
5971	/ I is the index of the insn to try next. /
5972	i = `0`;
5973	tries_num = `0`;
5974	for (;;)
5975	{
5976	if (/ If we've reached a dead end or searched enough of what we have*
5977	been asked... /*
5978	top->rest == `0`
5979	/ or have nothing else to try... /
5980	\|\| i >= n_ready
5981	/ or should not issue more. /
5982	\|\| top->n >= more_issue)
5983	{
5984	/ ??? (... \|\| i == n_ready). /
5985	gcc_assert (i <= n_ready);
5986
5987	/ We should not issue more than issue_rate instructions. /
5988	gcc_assert (top->n <= more_issue);
5989
5990	if (top == choice_stack)
5991	break;
5992
5993	if (best < top - choice_stack)
5994	{
5995	if (privileged_n)
5996	{
5997	n = privileged_n;
5998	/ Try to find issued privileged insn. /
5999	while (n && !ready_try[--n])
6000	;
6001	}
6002
6003	if (/ If all insns are equally good... /
6004	privileged_n == `0`
6005	/ Or a privileged insn will be issued. /
6006	\|\| ready_try[n])
6007	/ Then we have a solution. /
6008	{
6009	best = top - choice_stack;
6010	/ This is the index of the insn issued first in this*
6011	solution. /*
6012	*index = choice_stack [`1`].index;
6013	if (top->n == more_issue \|\| best == all)
6014	break;
6015	}
6016	}
6017
6018	/ Set ready-list index to point to the last insn*
6019	('i++' below will advance it to the next insn). /*
6020	i = top->index;
6021
6022	/ Backtrack. /
6023	ready_try [i] = `0`;
6024
6025	if (targetm.sched.first_cycle_multipass_backtrack)
6026	targetm.sched.first_cycle_multipass_backtrack (&top->target_data,
6027	ready_try, n_ready);
6028
6029	top--;
6030	memcpy (dest: state, src: top->state, n: dfa_state_size);
6031	}
6032	else if (!ready_try [i])
6033	{
6034	tries_num++;
6035	if (tries_num > max_lookahead_tries)
6036	break;
6037	insn = ready_element (ready, index: i);
6038	delay = state_transition (state, insn);
6039	if (delay < `0`)
6040	{
6041	if (state_dead_lock_p (state)
6042	\|\| insn_finishes_cycle_p (insn))
6043	/ We won't issue any more instructions in the next*
6044	choice_state. /*
6045	top->rest = `0`;
6046	else
6047	top->rest--;
6048
6049	n = top->n;
6050	if (memcmp (s1: top->state, s2: state, n: dfa_state_size) != `0`)
6051	n++;
6052
6053	/ Advance to the next choice_entry. /
6054	top++;
6055	/ Initialize it. /
6056	top->rest = dfa_lookahead;
6057	top->index = i;
6058	top->n = n;
6059	memcpy (dest: top->state, src: state, n: dfa_state_size);
6060	ready_try [i] = `1`;
6061
6062	if (targetm.sched.first_cycle_multipass_issue)
6063	targetm.sched.first_cycle_multipass_issue (&top->target_data,
6064	ready_try, n_ready,
6065	insn,
6066	&((top - `1`)
6067	->target_data));
6068
6069	i = -`1`;
6070	}
6071	}
6072
6073	/ Increase ready-list index. /
6074	i++;
6075	}
6076
6077	if (targetm.sched.first_cycle_multipass_end)
6078	targetm.sched.first_cycle_multipass_end (best != `0`
6079	? &choice_stack[`1`].target_data
6080	: NULL);
6081
6082	/ Restore the original state of the DFA. /
6083	memcpy (dest: state, src: choice_stack->state, n: dfa_state_size);
6084
6085	return best;
6086	}
6087
6088	/ The following function chooses insn from READY and modifies*
6089	READY. The following function is used only for first
6090	cycle multipass scheduling.
6091	Return:
6092	-1 if cycle should be advanced,
6093	0 if INSN_PTR is set to point to the desirable insn,
6094	1 if choose_ready () should be restarted without advancing the cycle. /*
6095	static int
6096	choose_ready (struct ready_list ready, bool* first_cycle_insn_p,
6097	rtx_insn **insn_ptr)
6098	{
6099	if (dbg_cnt (index: sched_insn) == false)
6100	{
6101	if (nonscheduled_insns_begin == NULL_RTX)
6102	nonscheduled_insns_begin = current_sched_info->prev_head;
6103
6104	rtx_insn *insn = first_nonscheduled_insn ();
6105
6106	if (QUEUE_INDEX (insn) == QUEUE_READY)
6107	/ INSN is in the ready_list. /
6108	{
6109	ready_remove_insn (insn);
6110	*insn_ptr = insn;
6111	return `0`;
6112	}
6113
6114	/ INSN is in the queue. Advance cycle to move it to the ready list. /
6115	gcc_assert (QUEUE_INDEX (insn) >= `0`);
6116	return -`1`;
6117	}
6118
6119	if (dfa_lookahead <= `0` \|\| SCHED_GROUP_P (ready_element (ready, `0`))
6120	\|\| DEBUG_INSN_P (ready_element (ready, `0`)))
6121	{
6122	if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
6123	*insn_ptr = ready_remove_first_dispatch (ready);
6124	else
6125	*insn_ptr = ready_remove_first (ready);
6126
6127	return `0`;
6128	}
6129	else
6130	{
6131	/ Try to choose the best insn. /
6132	int index = `0`, i;
6133	rtx_insn *insn;
6134
6135	insn = ready_element (ready, index: `0`);
6136	if (INSN_CODE (insn) < `0`)
6137	{
6138	*insn_ptr = ready_remove_first (ready);
6139	return `0`;
6140	}
6141
6142	/ Filter the search space. /
6143	for (i = `0`; i < ready->n_ready; i++)
6144	{
6145	ready_try[i] = `0`;
6146
6147	insn = ready_element (ready, index: i);
6148
6149	/ If this insn is recognizable we should have already*
6150	recognized it earlier.
6151	??? Not very clear where this is supposed to be done.
6152	See dep_cost_1. /*
6153	gcc_checking_assert (INSN_CODE (insn) >= `0`
6154	\|\| recog_memoized (insn) < `0`);
6155	if (INSN_CODE (insn) < `0`)
6156	{
6157	/ Non-recognized insns at position 0 are handled above. /
6158	gcc_assert (i > `0`);
6159	ready_try[i] = `1`;
6160	continue;
6161	}
6162
6163	if (targetm.sched.first_cycle_multipass_dfa_lookahead_guard)
6164	{
6165	ready_try[i]
6166	= (targetm.sched.first_cycle_multipass_dfa_lookahead_guard
6167	(insn, i));
6168
6169	if (ready_try[i] < `0`)
6170	/ Queue instruction for several cycles.*
6171	We need to restart choose_ready as we have changed
6172	the ready list. /*
6173	{
6174	change_queue_index (insn, -ready_try[i]);
6175	return `1`;
6176	}
6177
6178	/ Make sure that we didn't end up with 0'th insn filtered out.*
6179	Don't be tempted to make life easier for backends and just
6180	requeue 0'th insn if (ready_try[0] == 0) and restart
6181	choose_ready. Backends should be very considerate about
6182	requeueing instructions -- especially the highest priority
6183	one at position 0. /*
6184	gcc_assert (ready_try[i] == `0` \|\| i > `0`);
6185	if (ready_try[i])
6186	continue;
6187	}
6188
6189	gcc_assert (ready_try[i] == `0`);
6190	/ INSN made it through the scrutiny of filters! /
6191	}
6192
6193	if (max_issue (ready, privileged_n: `1`, state: curr_state, first_cycle_insn_p, index: &index) == `0`)
6194	{
6195	*insn_ptr = ready_remove_first (ready);
6196	if (sched_verbose >= `4`)
6197	fprintf (stream: sched_dump, format: ";;\t\tChosen insn (but can't issue) : %s \n",
6198	(current_sched_info->print_insn) (insn_ptr, `0`));
6199	return `0`;
6200	}
6201	else
6202	{
6203	if (sched_verbose >= `4`)
6204	fprintf (stream: sched_dump, format: ";;\t\tChosen insn : %s\n",
6205	(*current_sched_info->print_insn)
6206	(ready_element (ready, index), `0`));
6207
6208	*insn_ptr = ready_remove (ready, index);
6209	return `0`;
6210	}
6211	}
6212	}
6213
6214	/ This function is called when we have successfully scheduled a*
6215	block. It uses the schedule stored in the scheduled_insns vector
6216	to rearrange the RTL. PREV_HEAD is used as the anchor to which we
6217	append the scheduled insns; TAIL is the insn after the scheduled
6218	block. TARGET_BB is the argument passed to schedule_block. /*
6219
6220	static void
6221	commit_schedule (rtx_insn prev_head, rtx_insn tail, basic_block *target_bb)
6222	{
6223	unsigned int i;
6224	rtx_insn *insn;
6225
6226	last_scheduled_insn = prev_head;
6227	for (i = `0`;
6228	scheduled_insns.iterate (ix: i, ptr: &insn);
6229	i++)
6230	{
6231	if (control_flow_insn_p (last_scheduled_insn)
6232	\|\| current_sched_info->advance_target_bb (*target_bb, insn))
6233	{
6234	target_bb = current_sched_info->advance_target_bb (target_bb, `0`);
6235
6236	if (sched_verbose)
6237	{
6238	rtx_insn *x;
6239
6240	x = next_real_insn (last_scheduled_insn);
6241	gcc_assert (x);
6242	dump_new_block_header (`1`, *target_bb, x, tail);
6243	}
6244
6245	last_scheduled_insn = bb_note (*target_bb);
6246	}
6247
6248	if (current_sched_info->begin_move_insn)
6249	(*current_sched_info->begin_move_insn) (insn, last_scheduled_insn);
6250	move_insn (insn, last: last_scheduled_insn,
6251	nt: current_sched_info->next_tail);
6252	if (!DEBUG_INSN_P (insn))
6253	reemit_notes (insn);
6254	last_scheduled_insn = insn;
6255	}
6256
6257	scheduled_insns.truncate (size: `0`);
6258	}
6259
6260	/ Examine all insns on the ready list and queue those which can't be*
6261	issued in this cycle. TEMP_STATE is temporary scheduler state we
6262	can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns
6263	have been issued for the current cycle, which means it is valid to
6264	issue an asm statement.
6265
6266	If SHADOWS_ONLY_P is true, we eliminate all real insns and only
6267	leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true,
6268	we only leave insns which have an INSN_EXACT_TICK. /*
6269
6270	static void
6271	prune_ready_list (state_t temp_state, bool first_cycle_insn_p,
6272	bool shadows_only_p, bool modulo_epilogue_p)
6273	{
6274	int i, pass;
6275	bool sched_group_found = false;
6276	int min_cost_group = `0`;
6277
6278	if (sched_fusion)
6279	return;
6280
6281	for (i = `0`; i < ready.n_ready; i++)
6282	{
6283	rtx_insn *insn = ready_element (ready: &ready, index: i);
6284	if (SCHED_GROUP_P (insn))
6285	{
6286	sched_group_found = true;
6287	break;
6288	}
6289	}
6290
6291	/ Make two passes if there's a SCHED_GROUP_P insn; make sure to handle*
6292	such an insn first and note its cost. If at least one SCHED_GROUP_P insn
6293	gets queued, then all other insns get queued for one cycle later. /*
6294	for (pass = sched_group_found ? `0` : `1`; pass < `2`; )
6295	{
6296	int n = ready.n_ready;
6297	for (i = `0`; i < n; i++)
6298	{
6299	rtx_insn *insn = ready_element (ready: &ready, index: i);
6300	int cost = `0`;
6301	const char *reason = "resource conflict";
6302
6303	if (DEBUG_INSN_P (insn))
6304	continue;
6305
6306	if (sched_group_found && !SCHED_GROUP_P (insn)
6307	&& ((pass == `0`) \|\| (min_cost_group >= `1`)))
6308	{
6309	if (pass == `0`)
6310	continue;
6311	cost = min_cost_group;
6312	reason = "not in sched group";
6313	}
6314	else if (modulo_epilogue_p
6315	&& INSN_EXACT_TICK (insn) == INVALID_TICK)
6316	{
6317	cost = max_insn_queue_index;
6318	reason = "not an epilogue insn";
6319	}
6320	else if (shadows_only_p && !SHADOW_P (insn))
6321	{
6322	cost = `1`;
6323	reason = "not a shadow";
6324	}
6325	else if (recog_memoized (insn) < `0`)
6326	{
6327	if (!first_cycle_insn_p
6328	&& (GET_CODE (PATTERN (insn)) == ASM_INPUT
6329	\|\| asm_noperands (PATTERN (insn)) >= `0`))
6330	cost = `1`;
6331	reason = "asm";
6332	}
6333	else if (sched_pressure != SCHED_PRESSURE_NONE)
6334	{
6335	if (sched_pressure == SCHED_PRESSURE_MODEL
6336	&& INSN_TICK (insn) <= clock_var)
6337	{
6338	memcpy (dest: temp_state, src: curr_state, n: dfa_state_size);
6339	if (state_transition (temp_state, insn) >= `0`)
6340	INSN_TICK (insn) = clock_var + `1`;
6341	}
6342	cost = `0`;
6343	}
6344	else
6345	{
6346	int delay_cost = `0`;
6347
6348	if (delay_htab)
6349	{
6350	struct delay_pair *delay_entry;
6351	delay_entry
6352	= delay_htab->find_with_hash (comparable: insn,
6353	hash: htab_hash_pointer (insn));
6354	while (delay_entry && delay_cost == `0`)
6355	{
6356	delay_cost = estimate_shadow_tick (p: delay_entry);
6357	if (delay_cost > max_insn_queue_index)
6358	delay_cost = max_insn_queue_index;
6359	delay_entry = delay_entry->next_same_i1;
6360	}
6361	}
6362
6363	memcpy (dest: temp_state, src: curr_state, n: dfa_state_size);
6364	cost = state_transition (temp_state, insn);
6365	if (cost < `0`)
6366	cost = `0`;
6367	else if (cost == `0`)
6368	cost = `1`;
6369	if (cost < delay_cost)
6370	{
6371	cost = delay_cost;
6372	reason = "shadow tick";
6373	}
6374	}
6375	if (cost >= `1`)
6376	{
6377	if (SCHED_GROUP_P (insn) && cost > min_cost_group)
6378	min_cost_group = cost;
6379	ready_remove (ready: &ready, index: i);
6380	/ Normally we'd want to queue INSN for COST cycles. However,*
6381	if SCHED_GROUP_P is set, then we must ensure that nothing
6382	else comes between INSN and its predecessor. If there is
6383	some other insn ready to fire on the next cycle, then that
6384	invariant would be broken.
6385
6386	So when SCHED_GROUP_P is set, just queue this insn for a
6387	single cycle. /*
6388	queue_insn (insn, SCHED_GROUP_P (insn) ? `1` : cost, reason);
6389	if (i + `1` < n)
6390	break;
6391	}
6392	}
6393	if (i == n)
6394	pass++;
6395	}
6396	}
6397
6398	/ Called when we detect that the schedule is impossible. We examine the*
6399	backtrack queue to find the earliest insn that caused this condition. /*
6400
6401	static struct haifa_saved_data *
6402	verify_shadows (void)
6403	{
6404	struct haifa_saved_data save, earliest_fail = NULL;
6405	for (save = backtrack_queue; save; save = save->next)
6406	{
6407	int t;
6408	struct delay_pair *pair = save->delay_pair;
6409	rtx_insn *i1 = pair->i1;
6410
6411	for (; pair; pair = pair->next_same_i1)
6412	{
6413	rtx_insn *i2 = pair->i2;
6414
6415	if (QUEUE_INDEX (i2) == QUEUE_SCHEDULED)
6416	continue;
6417
6418	t = INSN_TICK (i1) + pair_delay (p: pair);
6419	if (t < clock_var)
6420	{
6421	if (sched_verbose >= `2`)
6422	fprintf (stream: sched_dump,
6423	format: ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
6424	", not ready\n",
6425	INSN_UID (insn: pair->i1), INSN_UID (insn: pair->i2),
6426	INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2));
6427	earliest_fail = save;
6428	break;
6429	}
6430	if (QUEUE_INDEX (i2) >= `0`)
6431	{
6432	int queued_for = INSN_TICK (i2);
6433
6434	if (t < queued_for)
6435	{
6436	if (sched_verbose >= `2`)
6437	fprintf (stream: sched_dump,
6438	format: ";;\t\tfailed delay requirements for %d/%d"
6439	" (%d->%d), queued too late\n",
6440	INSN_UID (insn: pair->i1), INSN_UID (insn: pair->i2),
6441	INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2));
6442	earliest_fail = save;
6443	break;
6444	}
6445	}
6446	}
6447	}
6448
6449	return earliest_fail;
6450	}
6451
6452	/ Print instructions together with useful scheduling information between*
6453	HEAD and TAIL (inclusive). /*
6454	static void
6455	dump_insn_stream (rtx_insn head, rtx_insn tail)
6456	{
6457	fprintf (stream: sched_dump, format: ";;\t\| insn \| prio \|\n");
6458
6459	rtx_insn *next_tail = NEXT_INSN (insn: tail);
6460	for (rtx_insn *insn = head; insn != next_tail; insn = NEXT_INSN (insn))
6461	{
6462	int priority = NOTE_P (insn) ? `0` : INSN_PRIORITY (insn);
6463	const char *pattern = (NOTE_P (insn)
6464	? "note"
6465	: str_pattern_slim (PATTERN (insn)));
6466
6467	fprintf (stream: sched_dump, format: ";;\t\| %4d \| %4d \| %-30s ",
6468	INSN_UID (insn), priority, pattern);
6469
6470	if (sched_verbose >= `4`)
6471	{
6472	if (NOTE_P (insn) \|\| LABEL_P (insn) \|\| recog_memoized (insn) < `0`)
6473	fprintf (stream: sched_dump, format: "nothing");
6474	else
6475	print_reservation (sched_dump, insn);
6476	}
6477	fprintf (stream: sched_dump, format: "\n");
6478	}
6479	}
6480
6481	/ Use forward list scheduling to rearrange insns of block pointed to by*
6482	TARGET_BB, possibly bringing insns from subsequent blocks in the same
6483	region. /*
6484
6485	bool
6486	schedule_block (basic_block *target_bb, state_t init_state)
6487	{
6488	int i;
6489	bool success = modulo_ii == `0`;
6490	struct sched_block_state ls;
6491	state_t temp_state = NULL; / It is used for multipass scheduling. /
6492	int sort_p, advance, start_clock_var;
6493
6494	/ Head/tail info for this block. /
6495	rtx_insn *prev_head = current_sched_info->prev_head;
6496	rtx_insn *next_tail = current_sched_info->next_tail;
6497	rtx_insn *head = NEXT_INSN (insn: prev_head);
6498	rtx_insn *tail = PREV_INSN (insn: next_tail);
6499
6500	if ((current_sched_info->flags & DONT_BREAK_DEPENDENCIES) == `0`
6501	&& sched_pressure != SCHED_PRESSURE_MODEL && !sched_fusion)
6502	find_modifiable_mems (head, tail);
6503
6504	/ We used to have code to avoid getting parameters moved from hard*
6505	argument registers into pseudos.
6506
6507	However, it was removed when it proved to be of marginal benefit
6508	and caused problems because schedule_block and compute_forward_dependences
6509	had different notions of what the "head" insn was. /*
6510
6511	gcc_assert (head != tail \|\| INSN_P (head));
6512
6513	haifa_recovery_bb_recently_added_p = false;
6514
6515	backtrack_queue = NULL;
6516
6517	/ Debug info. /
6518	if (sched_verbose)
6519	{
6520	dump_new_block_header (`0`, *target_bb, head, tail);
6521
6522	if (sched_verbose >= `2`)
6523	{
6524	dump_insn_stream (head, tail);
6525	memset (s: &rank_for_schedule_stats, c: `0`,
6526	n: sizeof (rank_for_schedule_stats));
6527	}
6528	}
6529
6530	if (init_state == NULL)
6531	state_reset (curr_state);
6532	else
6533	memcpy (dest: curr_state, src: init_state, n: dfa_state_size);
6534
6535	/ Clear the ready list. /
6536	ready.first = ready.veclen - `1`;
6537	ready.n_ready = `0`;
6538	ready.n_debug = `0`;
6539
6540	/ It is used for first cycle multipass scheduling. /
6541	temp_state = alloca (dfa_state_size);
6542
6543	if (targetm.sched.init)
6544	targetm.sched.init (sched_dump, sched_verbose, ready.veclen);
6545
6546	/ We start inserting insns after PREV_HEAD. /
6547	last_scheduled_insn = prev_head;
6548	last_nondebug_scheduled_insn = NULL;
6549	nonscheduled_insns_begin = NULL;
6550
6551	gcc_assert ((NOTE_P (last_scheduled_insn)
6552	\|\| DEBUG_INSN_P (last_scheduled_insn))
6553	&& BLOCK_FOR_INSN (last_scheduled_insn) == *target_bb);
6554
6555	/ Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the*
6556	queue. /*
6557	q_ptr = `0`;
6558	q_size = `0`;
6559
6560	insn_queue = XALLOCAVEC (rtx_insn_list *, max_insn_queue_index + `1`);
6561	memset (s: insn_queue, c: `0`, n: (max_insn_queue_index + `1`) * sizeof (rtx));
6562
6563	/ Start just before the beginning of time. /
6564	clock_var = -`1`;
6565
6566	/ We need queue and ready lists and clock_var be initialized*
6567	in try_ready () (which is called through init_ready_list ()). /*
6568	(*current_sched_info->init_ready_list) ();
6569
6570	if (sched_pressure)
6571	sched_pressure_start_bb (bb: *target_bb);
6572
6573	/ The algorithm is O(n^2) in the number of ready insns at any given*
6574	time in the worst case. Before reload we are more likely to have
6575	big lists so truncate them to a reasonable size. /*
6576	if (!reload_completed
6577	&& ready.n_ready - ready.n_debug > param_max_sched_ready_insns)
6578	{
6579	ready_sort_debug (ready: &ready);
6580	ready_sort_real (ready: &ready);
6581
6582	/ Find first free-standing insn past param_max_sched_ready_insns.*
6583	If there are debug insns, we know they're first. /*
6584	for (i = param_max_sched_ready_insns + ready.n_debug; i < ready.n_ready;
6585	i++)
6586	if (!SCHED_GROUP_P (ready_element (&ready, i)))
6587	break;
6588
6589	if (sched_verbose >= `2`)
6590	{
6591	fprintf (stream: sched_dump,
6592	format: ";;\t\tReady list on entry: %d insns: ", ready.n_ready);
6593	debug_ready_list (ready: &ready);
6594	fprintf (stream: sched_dump,
6595	format: ";;\t\t before reload => truncated to %d insns\n", i);
6596	}
6597
6598	/ Delay all insns past it for 1 cycle. If debug counter is*
6599	activated make an exception for the insn right after
6600	nonscheduled_insns_begin. /*
6601	{
6602	rtx_insn *skip_insn;
6603
6604	if (dbg_cnt (index: sched_insn) == false)
6605	skip_insn = first_nonscheduled_insn ();
6606	else
6607	skip_insn = NULL;
6608
6609	while (i < ready.n_ready)
6610	{
6611	rtx_insn *insn;
6612
6613	insn = ready_remove (ready: &ready, index: i);
6614
6615	if (insn != skip_insn)
6616	queue_insn (insn, n_cycles: `1`, reason: "list truncated");
6617	}
6618	if (skip_insn)
6619	ready_add (ready: &ready, insn: skip_insn, first_p: true);
6620	}
6621	}
6622
6623	/ Now we can restore basic block notes and maintain precise cfg. /
6624	restore_bb_notes (*target_bb);
6625
6626	last_clock_var = -`1`;
6627
6628	advance = `0`;
6629
6630	gcc_assert (scheduled_insns.length () == `0`);
6631	sort_p = true;
6632	must_backtrack = false;
6633	modulo_insns_scheduled = `0`;
6634
6635	ls.modulo_epilogue = false;
6636	ls.first_cycle_insn_p = true;
6637
6638	/ Loop until all the insns in BB are scheduled. /
6639	while ((*current_sched_info->schedule_more_p) ())
6640	{
6641	perform_replacements_new_cycle ();
6642	do
6643	{
6644	start_clock_var = clock_var;
6645
6646	clock_var++;
6647
6648	advance_one_cycle ();
6649
6650	/ Add to the ready list all pending insns that can be issued now.*
6651	If there are no ready insns, increment clock until one
6652	is ready and add all pending insns at that point to the ready
6653	list. /*
6654	queue_to_ready (ready: &ready);
6655
6656	gcc_assert (ready.n_ready);
6657
6658	if (sched_verbose >= `2`)
6659	{
6660	fprintf (stream: sched_dump, format: ";;\t\tReady list after queue_to_ready:");
6661	debug_ready_list (ready: &ready);
6662	}
6663	advance -= clock_var - start_clock_var;
6664	}
6665	while (advance > `0`);
6666
6667	if (ls.modulo_epilogue)
6668	{
6669	int stage = clock_var / modulo_ii;
6670	if (stage > modulo_last_stage * `2` + `2`)
6671	{
6672	if (sched_verbose >= `2`)
6673	fprintf (stream: sched_dump,
6674	format: ";;\t\tmodulo scheduled succeeded at II %d\n",
6675	modulo_ii);
6676	success = true;
6677	goto end_schedule;
6678	}
6679	}
6680	else if (modulo_ii > `0`)
6681	{
6682	int stage = clock_var / modulo_ii;
6683	if (stage > modulo_max_stages)
6684	{
6685	if (sched_verbose >= `2`)
6686	fprintf (stream: sched_dump,
6687	format: ";;\t\tfailing schedule due to excessive stages\n");
6688	goto end_schedule;
6689	}
6690	if (modulo_n_insns == modulo_insns_scheduled
6691	&& stage > modulo_last_stage)
6692	{
6693	if (sched_verbose >= `2`)
6694	fprintf (stream: sched_dump,
6695	format: ";;\t\tfound kernel after %d stages, II %d\n",
6696	stage, modulo_ii);
6697	ls.modulo_epilogue = true;
6698	}
6699	}
6700
6701	prune_ready_list (temp_state, first_cycle_insn_p: true, shadows_only_p: false, modulo_epilogue_p: ls.modulo_epilogue);
6702	if (ready.n_ready == `0`)
6703	continue;
6704	if (must_backtrack)
6705	goto do_backtrack;
6706
6707	ls.shadows_only_p = false;
6708	cycle_issued_insns = `0`;
6709	ls.can_issue_more = issue_rate;
6710	for (;;)
6711	{
6712	rtx_insn *insn;
6713	int cost;
6714	bool asm_p;
6715
6716	if (sort_p && ready.n_ready > `0`)
6717	{
6718	/ Sort the ready list based on priority. This must be*
6719	done every iteration through the loop, as schedule_insn
6720	may have readied additional insns that will not be
6721	sorted correctly. /*
6722	ready_sort (ready: &ready);
6723
6724	if (sched_verbose >= `2`)
6725	{
6726	fprintf (stream: sched_dump,
6727	format: ";;\t\tReady list after ready_sort: ");
6728	debug_ready_list (ready: &ready);
6729	}
6730	}
6731
6732	/ We don't want md sched reorder to even see debug isns, so put*
6733	them out right away. /*
6734	if (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, `0`))
6735	&& (*current_sched_info->schedule_more_p) ())
6736	{
6737	while (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, `0`)))
6738	{
6739	rtx_insn *insn = ready_remove_first (ready: &ready);
6740	gcc_assert (DEBUG_INSN_P (insn));
6741	(*current_sched_info->begin_schedule_ready) (insn);
6742	scheduled_insns.safe_push (obj: insn);
6743	last_scheduled_insn = insn;
6744	advance = schedule_insn (insn);
6745	gcc_assert (advance == `0`);
6746	if (ready.n_ready > `0`)
6747	ready_sort (ready: &ready);
6748	}
6749	}
6750
6751	if (ls.first_cycle_insn_p && !ready.n_ready)
6752	break;
6753
6754	resume_after_backtrack:
6755	/ Allow the target to reorder the list, typically for*
6756	better instruction bundling. /*
6757	if (sort_p
6758	&& (ready.n_ready == `0`
6759	\|\| !SCHED_GROUP_P (ready_element (&ready, `0`))))
6760	{
6761	if (ls.first_cycle_insn_p && targetm.sched.reorder)
6762	ls.can_issue_more
6763	= targetm.sched.reorder (sched_dump, sched_verbose,
6764	ready_lastpos (ready: &ready),
6765	&ready.n_ready, clock_var);
6766	else if (!ls.first_cycle_insn_p && targetm.sched.reorder2)
6767	ls.can_issue_more
6768	= targetm.sched.reorder2 (sched_dump, sched_verbose,
6769	ready.n_ready
6770	? ready_lastpos (ready: &ready) : NULL,
6771	&ready.n_ready, clock_var);
6772	}
6773
6774	restart_choose_ready:
6775	if (sched_verbose >= `2`)
6776	{
6777	fprintf (stream: sched_dump, format: ";;\tReady list (t = %3d): ",
6778	clock_var);
6779	debug_ready_list (ready: &ready);
6780	if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
6781	print_curr_reg_pressure ();
6782	}
6783
6784	if (ready.n_ready == `0`
6785	&& ls.can_issue_more
6786	&& reload_completed)
6787	{
6788	/ Allow scheduling insns directly from the queue in case*
6789	there's nothing better to do (ready list is empty) but
6790	there are still vacant dispatch slots in the current cycle. /*
6791	if (sched_verbose >= `6`)
6792	fprintf (stream: sched_dump,format: ";;\t\tSecond chance\n");
6793	memcpy (dest: temp_state, src: curr_state, n: dfa_state_size);
6794	if (early_queue_to_ready (state: temp_state, ready: &ready))
6795	ready_sort (ready: &ready);
6796	}
6797
6798	if (ready.n_ready == `0`
6799	\|\| !ls.can_issue_more
6800	\|\| state_dead_lock_p (curr_state)
6801	\|\| !(*current_sched_info->schedule_more_p) ())
6802	break;
6803
6804	/ Select and remove the insn from the ready list. /
6805	if (sort_p)
6806	{
6807	int res;
6808
6809	insn = NULL;
6810	res = choose_ready (ready: &ready, first_cycle_insn_p: ls.first_cycle_insn_p, insn_ptr: &insn);
6811
6812	if (res < `0`)
6813	/ Finish cycle. /
6814	break;
6815	if (res > `0`)
6816	goto restart_choose_ready;
6817
6818	gcc_assert (insn != NULL_RTX);
6819	}
6820	else
6821	insn = ready_remove_first (ready: &ready);
6822
6823	if (sched_pressure != SCHED_PRESSURE_NONE
6824	&& INSN_TICK (insn) > clock_var)
6825	{
6826	ready_add (ready: &ready, insn, first_p: true);
6827	advance = `1`;
6828	break;
6829	}
6830
6831	if (targetm.sched.dfa_new_cycle
6832	&& targetm.sched.dfa_new_cycle (sched_dump, sched_verbose,
6833	insn, last_clock_var,
6834	clock_var, &sort_p))
6835	/ SORT_P is used by the target to override sorting*
6836	of the ready list. This is needed when the target
6837	has modified its internal structures expecting that
6838	the insn will be issued next. As we need the insn
6839	to have the highest priority (so it will be returned by
6840	the ready_remove_first call above), we invoke
6841	ready_add (&ready, insn, true).
6842	But, still, there is one issue: INSN can be later
6843	discarded by scheduler's front end through
6844	current_sched_info->can_schedule_ready_p, hence, won't
6845	be issued next. /*
6846	{
6847	ready_add (ready: &ready, insn, first_p: true);
6848	break;
6849	}
6850
6851	sort_p = true;
6852
6853	if (current_sched_info->can_schedule_ready_p
6854	&& ! (*current_sched_info->can_schedule_ready_p) (insn))
6855	/ We normally get here only if we don't want to move*
6856	insn from the split block. /*
6857	{
6858	TODO_SPEC (insn) = DEP_POSTPONED;
6859	goto restart_choose_ready;
6860	}
6861
6862	if (delay_htab)
6863	{
6864	/ If this insn is the first part of a delay-slot pair, record a*
6865	backtrack point. /*
6866	struct delay_pair *delay_entry;
6867	delay_entry
6868	= delay_htab->find_with_hash (comparable: insn, hash: htab_hash_pointer (insn));
6869	if (delay_entry)
6870	{
6871	save_backtrack_point (pair: delay_entry, sched_block: ls);
6872	if (sched_verbose >= `2`)
6873	fprintf (stream: sched_dump, format: ";;\t\tsaving backtrack point\n");
6874	}
6875	}
6876
6877	/ DECISION is made. /
6878
6879	if (modulo_ii > `0` && INSN_UID (insn) < modulo_iter0_max_uid)
6880	{
6881	modulo_insns_scheduled++;
6882	modulo_last_stage = clock_var / modulo_ii;
6883	}
6884	if (TODO_SPEC (insn) & SPECULATIVE)
6885	generate_recovery_code (insn);
6886
6887	if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
6888	targetm.sched.dispatch_do (insn, ADD_TO_DISPATCH_WINDOW);
6889
6890	/ Update counters, etc in the scheduler's front end. /
6891	(*current_sched_info->begin_schedule_ready) (insn);
6892	scheduled_insns.safe_push (obj: insn);
6893	gcc_assert (NONDEBUG_INSN_P (insn));
6894	last_nondebug_scheduled_insn = last_scheduled_insn = insn;
6895
6896	if (recog_memoized (insn) >= `0`)
6897	{
6898	memcpy (dest: temp_state, src: curr_state, n: dfa_state_size);
6899	cost = state_transition (curr_state, insn);
6900	if (sched_pressure != SCHED_PRESSURE_WEIGHTED && !sched_fusion)
6901	gcc_assert (cost < `0`);
6902	if (memcmp (s1: temp_state, s2: curr_state, n: dfa_state_size) != `0`)
6903	cycle_issued_insns++;
6904	asm_p = false;
6905	}
6906	else
6907	asm_p = (GET_CODE (PATTERN (insn)) == ASM_INPUT
6908	\|\| asm_noperands (PATTERN (insn)) >= `0`);
6909
6910	if (targetm.sched.variable_issue)
6911	ls.can_issue_more =
6912	targetm.sched.variable_issue (sched_dump, sched_verbose,
6913	insn, ls.can_issue_more);
6914	/ A naked CLOBBER or USE generates no instruction, so do*
6915	not count them against the issue rate. /*
6916	else if (GET_CODE (PATTERN (insn)) != USE
6917	&& GET_CODE (PATTERN (insn)) != CLOBBER)
6918	ls.can_issue_more--;
6919	advance = schedule_insn (insn);
6920
6921	if (SHADOW_P (insn))
6922	ls.shadows_only_p = true;
6923
6924	/ After issuing an asm insn we should start a new cycle. /
6925	if (advance == `0` && asm_p)
6926	advance = `1`;
6927
6928	if (must_backtrack)
6929	break;
6930
6931	if (advance != `0`)
6932	break;
6933
6934	ls.first_cycle_insn_p = false;
6935	if (ready.n_ready > `0`)
6936	prune_ready_list (temp_state, first_cycle_insn_p: false, shadows_only_p: ls.shadows_only_p,
6937	modulo_epilogue_p: ls.modulo_epilogue);
6938	}
6939
6940	do_backtrack:
6941	if (!must_backtrack)
6942	for (i = `0`; i < ready.n_ready; i++)
6943	{
6944	rtx_insn *insn = ready_element (ready: &ready, index: i);
6945	if (INSN_EXACT_TICK (insn) == clock_var)
6946	{
6947	must_backtrack = true;
6948	clock_var++;
6949	break;
6950	}
6951	}
6952	if (must_backtrack && modulo_ii > `0`)
6953	{
6954	if (modulo_backtracks_left == `0`)
6955	goto end_schedule;
6956	modulo_backtracks_left--;
6957	}
6958	while (must_backtrack)
6959	{
6960	struct haifa_saved_data *failed;
6961	rtx_insn *failed_insn;
6962
6963	must_backtrack = false;
6964	failed = verify_shadows ();
6965	gcc_assert (failed);
6966
6967	failed_insn = failed->delay_pair->i1;
6968	/ Clear these queues. /
6969	perform_replacements_new_cycle ();
6970	toggle_cancelled_flags (set: false);
6971	unschedule_insns_until (insn: failed_insn);
6972	while (failed != backtrack_queue)
6973	free_topmost_backtrack_point (reset_tick: true);
6974	restore_last_backtrack_point (psched_block: &ls);
6975	if (sched_verbose >= `2`)
6976	fprintf (stream: sched_dump, format: ";;\t\trewind to cycle %d\n", clock_var);
6977	/ Delay by at least a cycle. This could cause additional*
6978	backtracking. /*
6979	queue_insn (insn: failed_insn, n_cycles: `1`, reason: "backtracked");
6980	advance = `0`;
6981	if (must_backtrack)
6982	continue;
6983	if (ready.n_ready > `0`)
6984	goto resume_after_backtrack;
6985	else
6986	{
6987	if (clock_var == `0` && ls.first_cycle_insn_p)
6988	goto end_schedule;
6989	advance = `1`;
6990	break;
6991	}
6992	}
6993	ls.first_cycle_insn_p = true;
6994	}
6995	if (ls.modulo_epilogue)
6996	success = true;
6997	end_schedule:
6998	if (!ls.first_cycle_insn_p \|\| advance)
6999	advance_one_cycle ();
7000	perform_replacements_new_cycle ();
7001	if (modulo_ii > `0`)
7002	{
7003	/ Once again, debug insn suckiness: they can be on the ready list*
7004	even if they have unresolved dependencies. To make our view
7005	of the world consistent, remove such "ready" insns. /*
7006	restart_debug_insn_loop:
7007	for (i = ready.n_ready - `1`; i >= `0`; i--)
7008	{
7009	rtx_insn *x;
7010
7011	x = ready_element (ready: &ready, index: i);
7012	if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x)) != NULL
7013	\|\| DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x)) != NULL)
7014	{
7015	ready_remove (ready: &ready, index: i);
7016	goto restart_debug_insn_loop;
7017	}
7018	}
7019	for (i = ready.n_ready - `1`; i >= `0`; i--)
7020	{
7021	rtx_insn *x;
7022
7023	x = ready_element (ready: &ready, index: i);
7024	resolve_dependencies (insn: x);
7025	}
7026	for (i = `0`; i <= max_insn_queue_index; i++)
7027	{
7028	rtx_insn_list *link;
7029	while ((link = insn_queue[i]) != NULL)
7030	{
7031	rtx_insn *x = link->insn ();
7032	insn_queue[i] = link->next ();
7033	QUEUE_INDEX (x) = QUEUE_NOWHERE;
7034	free_INSN_LIST_node (link);
7035	resolve_dependencies (insn: x);
7036	}
7037	}
7038	}
7039
7040	if (!success)
7041	undo_all_replacements ();
7042
7043	/ Debug info. /
7044	if (sched_verbose)
7045	{
7046	fprintf (stream: sched_dump, format: ";;\tReady list (final): ");
7047	debug_ready_list (ready: &ready);
7048	}
7049
7050	if (modulo_ii == `0` && current_sched_info->queue_must_finish_empty)
7051	/ Sanity check -- queue must be empty now. Meaningless if region has*
7052	multiple bbs. /*
7053	gcc_assert (!q_size && !ready.n_ready && !ready.n_debug);
7054	else if (modulo_ii == `0`)
7055	{
7056	/ We must maintain QUEUE_INDEX between blocks in region. /
7057	for (i = ready.n_ready - `1`; i >= `0`; i--)
7058	{
7059	rtx_insn *x;
7060
7061	x = ready_element (ready: &ready, index: i);
7062	QUEUE_INDEX (x) = QUEUE_NOWHERE;
7063	TODO_SPEC (x) = HARD_DEP;
7064	}
7065
7066	if (q_size)
7067	for (i = `0`; i <= max_insn_queue_index; i++)
7068	{
7069	rtx_insn_list *link;
7070	for (link = insn_queue[i]; link; link = link->next ())
7071	{
7072	rtx_insn *x;
7073
7074	x = link->insn ();
7075	QUEUE_INDEX (x) = QUEUE_NOWHERE;
7076	TODO_SPEC (x) = HARD_DEP;
7077	}
7078	free_INSN_LIST_list (&insn_queue[i]);
7079	}
7080	}
7081
7082	if (sched_pressure == SCHED_PRESSURE_MODEL)
7083	model_end_schedule ();
7084
7085	if (success)
7086	{
7087	commit_schedule (prev_head, tail, target_bb);
7088	if (sched_verbose)
7089	fprintf (stream: sched_dump, format: ";; total time = %d\n", clock_var);
7090	}
7091	else
7092	last_scheduled_insn = tail;
7093
7094	scheduled_insns.truncate (size: `0`);
7095
7096	if (!current_sched_info->queue_must_finish_empty
7097	\|\| haifa_recovery_bb_recently_added_p)
7098	{
7099	/ INSN_TICK (minimum clock tick at which the insn becomes*
7100	ready) may be not correct for the insn in the subsequent
7101	blocks of the region. We should use a correct value of
7102	`clock_var' or modify INSN_TICK. It is better to keep
7103	clock_var value equal to 0 at the start of a basic block.
7104	Therefore we modify INSN_TICK here. /*
7105	fix_inter_tick (NEXT_INSN (insn: prev_head), last_scheduled_insn);
7106	}
7107
7108	if (targetm.sched.finish)
7109	{
7110	targetm.sched.finish (sched_dump, sched_verbose);
7111	/ Target might have added some instructions to the scheduled block*
7112	in its md_finish () hook. These new insns don't have any data
7113	initialized and to identify them we extend h_i_d so that they'll
7114	get zero luids. /*
7115	sched_extend_luids ();
7116	}
7117
7118	/ Update head/tail boundaries. /
7119	head = NEXT_INSN (insn: prev_head);
7120	tail = last_scheduled_insn;
7121
7122	if (sched_verbose)
7123	{
7124	fprintf (stream: sched_dump, format: ";; new head = %d\n;; new tail = %d\n",
7125	INSN_UID (insn: head), INSN_UID (insn: tail));
7126
7127	if (sched_verbose >= `2`)
7128	{
7129	dump_insn_stream (head, tail);
7130	print_rank_for_schedule_stats (prefix: ";; TOTAL ", stats: &rank_for_schedule_stats,
7131	NULL);
7132	}
7133
7134	fprintf (stream: sched_dump, format: "\n");
7135	}
7136
7137	head = restore_other_notes (head, NULL);
7138
7139	current_sched_info->head = head;
7140	current_sched_info->tail = tail;
7141
7142	free_backtrack_queue ();
7143
7144	return success;
7145	}
7146
7147	/ Set_priorities: compute priority of each insn in the block. /
7148
7149	int
7150	set_priorities (rtx_insn head, rtx_insn tail)
7151	{
7152	rtx_insn *insn;
7153	int n_insn;
7154	int sched_max_insns_priority =
7155	current_sched_info->sched_max_insns_priority;
7156	rtx_insn *prev_head;
7157
7158	if (head == tail && ! INSN_P (head))
7159	gcc_unreachable ();
7160
7161	n_insn = `0`;
7162
7163	prev_head = PREV_INSN (insn: head);
7164	for (insn = tail; insn != prev_head; insn = PREV_INSN (insn))
7165	{
7166	if (!INSN_P (insn))
7167	continue;
7168
7169	n_insn++;
7170	(void) priority (insn);
7171
7172	gcc_assert (INSN_PRIORITY_KNOWN (insn));
7173
7174	sched_max_insns_priority = MAX (sched_max_insns_priority,
7175	INSN_PRIORITY (insn));
7176	}
7177
7178	current_sched_info->sched_max_insns_priority = sched_max_insns_priority;
7179
7180	return n_insn;
7181	}
7182
7183	/ Set sched_dump and sched_verbose for the desired debugging output. /
7184	void
7185	setup_sched_dump (void)
7186	{
7187	sched_verbose = sched_verbose_param;
7188	sched_dump = dump_file;
7189	if (!dump_file)
7190	sched_verbose = `0`;
7191	}
7192
7193	/ Allocate data for register pressure sensitive scheduling. /
7194	static void
7195	alloc_global_sched_pressure_data (void)
7196	{
7197	if (sched_pressure != SCHED_PRESSURE_NONE)
7198	{
7199	int i, max_regno = max_reg_num ();
7200
7201	if (sched_dump != NULL)
7202	/ We need info about pseudos for rtl dumps about pseudo*
7203	classes and costs. /*
7204	regstat_init_n_sets_and_refs ();
7205	ira_set_pseudo_classes (true, sched_verbose ? sched_dump : NULL);
7206	sched_regno_pressure_class
7207	= (enum reg_class ) xmalloc (max_regno sizeof (enum reg_class));
7208	for (i = `0`; i < max_regno; i++)
7209	sched_regno_pressure_class[i]
7210	= (i < FIRST_PSEUDO_REGISTER
7211	? ira_pressure_class_translate[REGNO_REG_CLASS (i)]
7212	: ira_pressure_class_translate[reg_allocno_class (i)]);
7213	curr_reg_live = BITMAP_ALLOC (NULL);
7214	if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
7215	{
7216	saved_reg_live = BITMAP_ALLOC (NULL);
7217	region_ref_regs = BITMAP_ALLOC (NULL);
7218	}
7219	if (sched_pressure == SCHED_PRESSURE_MODEL)
7220	tmp_bitmap = BITMAP_ALLOC (NULL);
7221
7222	/ Calculate number of CALL_SAVED_REGS and FIXED_REGS in register classes*
7223	that we calculate register pressure for. /*
7224	for (int c = `0`; c < ira_pressure_classes_num; ++c)
7225	{
7226	enum reg_class cl = ira_pressure_classes[c];
7227
7228	call_saved_regs_num[cl] = `0`;
7229	fixed_regs_num[cl] = `0`;
7230
7231	for (int i = `0`; i < ira_class_hard_regs_num[cl]; ++i)
7232	{
7233	unsigned int regno = ira_class_hard_regs[cl][i];
7234	if (fixed_regs[regno])
7235	++fixed_regs_num[cl];
7236	else if (!crtl->abi->clobbers_full_reg_p (regno))
7237	++call_saved_regs_num[cl];
7238	}
7239	}
7240	}
7241	}
7242
7243	/ Free data for register pressure sensitive scheduling. Also called*
7244	from schedule_region when stopping sched-pressure early. /*
7245	void
7246	free_global_sched_pressure_data (void)
7247	{
7248	if (sched_pressure != SCHED_PRESSURE_NONE)
7249	{
7250	if (regstat_n_sets_and_refs != NULL)
7251	regstat_free_n_sets_and_refs ();
7252	if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
7253	{
7254	BITMAP_FREE (region_ref_regs);
7255	BITMAP_FREE (saved_reg_live);
7256	}
7257	if (sched_pressure == SCHED_PRESSURE_MODEL)
7258	BITMAP_FREE (tmp_bitmap);
7259	BITMAP_FREE (curr_reg_live);
7260	free (ptr: sched_regno_pressure_class);
7261	}
7262	}
7263
7264	/ Initialize some global state for the scheduler. This function works*
7265	with the common data shared between all the schedulers. It is called
7266	from the scheduler specific initialization routine. /*
7267
7268	void
7269	sched_init (void)
7270	{
7271	if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
7272	targetm.sched.dispatch_do (NULL, DISPATCH_INIT);
7273
7274	if (live_range_shrinkage_p)
7275	sched_pressure = SCHED_PRESSURE_WEIGHTED;
7276	else if (flag_sched_pressure
7277	&& !reload_completed
7278	&& common_sched_info->sched_pass_id == SCHED_RGN_PASS)
7279	sched_pressure = ((enum sched_pressure_algorithm)
7280	param_sched_pressure_algorithm);
7281	else
7282	sched_pressure = SCHED_PRESSURE_NONE;
7283
7284	if (sched_pressure != SCHED_PRESSURE_NONE)
7285	ira_setup_eliminable_regset ();
7286
7287	/ Initialize SPEC_INFO. /
7288	if (targetm.sched.set_sched_flags)
7289	{
7290	spec_info = &spec_info_var;
7291	targetm.sched.set_sched_flags (spec_info);
7292
7293	if (spec_info->mask != `0`)
7294	{
7295	spec_info->data_weakness_cutoff
7296	= (param_sched_spec_prob_cutoff * MAX_DEP_WEAK) / `100`;
7297	spec_info->control_weakness_cutoff
7298	= (param_sched_spec_prob_cutoff * REG_BR_PROB_BASE) / `100`;
7299	}
7300	else
7301	/ So we won't read anything accidentally. /
7302	spec_info = NULL;
7303
7304	}
7305	else
7306	/ So we won't read anything accidentally. /
7307	spec_info = `0`;
7308
7309	/ Initialize issue_rate. /
7310	if (targetm.sched.issue_rate)
7311	issue_rate = targetm.sched.issue_rate ();
7312	else
7313	issue_rate = `1`;
7314
7315	if (targetm.sched.first_cycle_multipass_dfa_lookahead
7316	/ Don't use max_issue with reg_pressure scheduling. Multipass*
7317	scheduling and reg_pressure scheduling undo each other's decisions. /*
7318	&& sched_pressure == SCHED_PRESSURE_NONE)
7319	dfa_lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead ();
7320	else
7321	dfa_lookahead = `0`;
7322
7323	/ Set to "0" so that we recalculate. /
7324	max_lookahead_tries = `0`;
7325
7326	if (targetm.sched.init_dfa_pre_cycle_insn)
7327	targetm.sched.init_dfa_pre_cycle_insn ();
7328
7329	if (targetm.sched.init_dfa_post_cycle_insn)
7330	targetm.sched.init_dfa_post_cycle_insn ();
7331
7332	dfa_start ();
7333	dfa_state_size = state_size ();
7334
7335	init_alias_analysis ();
7336
7337	if (!sched_no_dce)
7338	df_set_flags (DF_LR_RUN_DCE);
7339	df_note_add_problem ();
7340
7341	/ More problems needed for interloop dep calculation in SMS. /
7342	if (common_sched_info->sched_pass_id == SCHED_SMS_PASS)
7343	{
7344	df_rd_add_problem ();
7345	df_chain_add_problem (DF_DU_CHAIN + DF_UD_CHAIN);
7346	}
7347
7348	df_analyze ();
7349
7350	/ Do not run DCE after reload, as this can kill nops inserted*
7351	by bundling. /*
7352	if (reload_completed)
7353	df_clear_flags (DF_LR_RUN_DCE);
7354
7355	regstat_compute_calls_crossed ();
7356
7357	if (targetm.sched.init_global)
7358	targetm.sched.init_global (sched_dump, sched_verbose, get_max_uid () + `1`);
7359
7360	alloc_global_sched_pressure_data ();
7361
7362	curr_state = xmalloc (dfa_state_size);
7363	}
7364
7365	static void haifa_init_only_bb (basic_block, basic_block);
7366
7367	/ Initialize data structures specific to the Haifa scheduler. /
7368	void
7369	haifa_sched_init (void)
7370	{
7371	setup_sched_dump ();
7372	sched_init ();
7373
7374	scheduled_insns.create (nelems: `0`);
7375
7376	if (spec_info != NULL)
7377	{
7378	sched_deps_info->use_deps_list = `1`;
7379	sched_deps_info->generate_spec_deps = `1`;
7380	}
7381
7382	/ Initialize luids, dependency caches, target and h_i_d for the*
7383	whole function. /*
7384	{
7385	sched_init_bbs ();
7386
7387	auto_vec<basic_block> bbs (n_basic_blocks_for_fn (cfun));
7388	basic_block bb;
7389	FOR_EACH_BB_FN (bb, cfun)
7390	bbs.quick_push (obj: bb);
7391	sched_init_luids (bbs);
7392	sched_deps_init (true);
7393	sched_extend_target ();
7394	haifa_init_h_i_d (bbs);
7395	}
7396
7397	sched_init_only_bb = haifa_init_only_bb;
7398	sched_split_block = sched_split_block_1;
7399	sched_create_empty_bb = sched_create_empty_bb_1;
7400	haifa_recovery_bb_ever_added_p = false;
7401
7402	nr_begin_data = nr_begin_control = nr_be_in_data = nr_be_in_control = `0`;
7403	before_recovery = `0`;
7404	after_recovery = `0`;
7405
7406	modulo_ii = `0`;
7407	}
7408
7409	/ Finish work with the data specific to the Haifa scheduler. /
7410	void
7411	haifa_sched_finish (void)
7412	{
7413	sched_create_empty_bb = NULL;
7414	sched_split_block = NULL;
7415	sched_init_only_bb = NULL;
7416
7417	if (spec_info && spec_info->dump)
7418	{
7419	char c = reload_completed ? `'a'` : `'b'`;
7420
7421	fprintf (stream: spec_info->dump,
7422	format: ";; %s:\n", current_function_name ());
7423
7424	fprintf (stream: spec_info->dump,
7425	format: ";; Procedure %cr-begin-data-spec motions == %d\n",
7426	c, nr_begin_data);
7427	fprintf (stream: spec_info->dump,
7428	format: ";; Procedure %cr-be-in-data-spec motions == %d\n",
7429	c, nr_be_in_data);
7430	fprintf (stream: spec_info->dump,
7431	format: ";; Procedure %cr-begin-control-spec motions == %d\n",
7432	c, nr_begin_control);
7433	fprintf (stream: spec_info->dump,
7434	format: ";; Procedure %cr-be-in-control-spec motions == %d\n",
7435	c, nr_be_in_control);
7436	}
7437
7438	scheduled_insns.release ();
7439
7440	/ Finalize h_i_d, dependency caches, and luids for the whole*
7441	function. Target will be finalized in md_global_finish (). /*
7442	sched_deps_finish ();
7443	sched_finish_luids ();
7444	current_sched_info = NULL;
7445	insn_queue = NULL;
7446	sched_finish ();
7447	}
7448
7449	/ Free global data used during insn scheduling. This function works with*
7450	the common data shared between the schedulers. /*
7451
7452	void
7453	sched_finish (void)
7454	{
7455	haifa_finish_h_i_d ();
7456	free_global_sched_pressure_data ();
7457	free (ptr: curr_state);
7458
7459	if (targetm.sched.finish_global)
7460	targetm.sched.finish_global (sched_dump, sched_verbose);
7461
7462	end_alias_analysis ();
7463
7464	regstat_free_calls_crossed ();
7465
7466	dfa_finish ();
7467	}
7468
7469	/ Free all delay_pair structures that were recorded. /
7470	void
7471	free_delay_pairs (void)
7472	{
7473	if (delay_htab)
7474	{
7475	delay_htab->empty ();
7476	delay_htab_i2->empty ();
7477	}
7478	}
7479
7480	/ Fix INSN_TICKs of the instructions in the current block as well as*
7481	INSN_TICKs of their dependents.
7482	HEAD and TAIL are the begin and the end of the current scheduled block. /*
7483	static void
7484	fix_inter_tick (rtx_insn head, rtx_insn tail)
7485	{
7486	/ Set of instructions with corrected INSN_TICK. /
7487	auto_bitmap processed;
7488	/ ??? It is doubtful if we should assume that cycle advance happens on*
7489	basic block boundaries. Basically insns that are unconditionally ready
7490	on the start of the block are more preferable then those which have
7491	a one cycle dependency over insn from the previous block. /*
7492	int next_clock = clock_var + `1`;
7493
7494	/ Iterates over scheduled instructions and fix their INSN_TICKs and*
7495	INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
7496	across different blocks. /*
7497	for (tail = NEXT_INSN (insn: tail); head != tail; head = NEXT_INSN (insn: head))
7498	{
7499	if (INSN_P (head))
7500	{
7501	int tick;
7502	sd_iterator_def sd_it;
7503	dep_t dep;
7504
7505	tick = INSN_TICK (head);
7506	gcc_assert (tick >= MIN_TICK);
7507
7508	/ Fix INSN_TICK of instruction from just scheduled block. /
7509	if (bitmap_set_bit (processed, INSN_LUID (head)))
7510	{
7511	tick -= next_clock;
7512
7513	if (tick < MIN_TICK)
7514	tick = MIN_TICK;
7515
7516	INSN_TICK (head) = tick;
7517	}
7518
7519	if (DEBUG_INSN_P (head))
7520	continue;
7521
7522	FOR_EACH_DEP (head, SD_LIST_RES_FORW, sd_it, dep)
7523	{
7524	rtx_insn *next;
7525
7526	next = DEP_CON (dep);
7527	tick = INSN_TICK (next);
7528
7529	if (tick != INVALID_TICK
7530	/ If NEXT has its INSN_TICK calculated, fix it.*
7531	If not - it will be properly calculated from
7532	scratch later in fix_tick_ready. /*
7533	&& bitmap_set_bit (processed, INSN_LUID (next)))
7534	{
7535	tick -= next_clock;
7536
7537	if (tick < MIN_TICK)
7538	tick = MIN_TICK;
7539
7540	if (tick > INTER_TICK (next))
7541	INTER_TICK (next) = tick;
7542	else
7543	tick = INTER_TICK (next);
7544
7545	INSN_TICK (next) = tick;
7546	}
7547	}
7548	}
7549	}
7550	}
7551
7552	/ Check if NEXT is ready to be added to the ready or queue list.*
7553	If "yes", add it to the proper list.
7554	Returns:
7555	-1 - is not ready yet,
7556	0 - added to the ready list,
7557	0 < N - queued for N cycles. /*
7558	int
7559	try_ready (rtx_insn *next)
7560	{
7561	ds_t old_ts, new_ts;
7562
7563	old_ts = TODO_SPEC (next);
7564
7565	gcc_assert (!(old_ts & ~(SPECULATIVE \| HARD_DEP \| DEP_CONTROL \| DEP_POSTPONED))
7566	&& (old_ts == HARD_DEP
7567	\|\| old_ts == DEP_POSTPONED
7568	\|\| (old_ts & SPECULATIVE)
7569	\|\| old_ts == DEP_CONTROL));
7570
7571	new_ts = recompute_todo_spec (next, for_backtrack: false);
7572
7573	if (new_ts & (HARD_DEP \| DEP_POSTPONED))
7574	gcc_assert (new_ts == old_ts
7575	&& QUEUE_INDEX (next) == QUEUE_NOWHERE);
7576	else if (current_sched_info->new_ready)
7577	new_ts = current_sched_info->new_ready (next, new_ts);
7578
7579	/ * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might*
7580	have its original pattern or changed (speculative) one. This is due
7581	to changing ebb in region scheduling.
7582	* But if (old_ts & SPECULATIVE), then we are pretty sure that insn
7583	has speculative pattern.
7584
7585	We can't assert (!(new_ts & HARD_DEP) \|\| new_ts == old_ts) here because
7586	control-speculative NEXT could have been discarded by sched-rgn.cc
7587	(the same case as when discarded by can_schedule_ready_p ()). /*
7588
7589	if ((new_ts & SPECULATIVE)
7590	/ If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't*
7591	need to change anything. /*
7592	&& new_ts != old_ts)
7593	{
7594	int res;
7595	rtx new_pat;
7596
7597	gcc_assert ((new_ts & SPECULATIVE) && !(new_ts & ~SPECULATIVE));
7598
7599	res = haifa_speculate_insn (next, new_ts, &new_pat);
7600
7601	switch (res)
7602	{
7603	case -`1`:
7604	/ It would be nice to change DEP_STATUS of all dependences,*
7605	which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
7606	so we won't reanalyze anything. /*
7607	new_ts = HARD_DEP;
7608	break;
7609
7610	case `0`:
7611	/ We follow the rule, that every speculative insn*
7612	has non-null ORIG_PAT. /*
7613	if (!ORIG_PAT (next))
7614	ORIG_PAT (next) = PATTERN (insn: next);
7615	break;
7616
7617	case `1`:
7618	if (!ORIG_PAT (next))
7619	/ If we gonna to overwrite the original pattern of insn,*
7620	save it. /*
7621	ORIG_PAT (next) = PATTERN (insn: next);
7622
7623	res = haifa_change_pattern (next, new_pat);
7624	gcc_assert (res);
7625	break;
7626
7627	default:
7628	gcc_unreachable ();
7629	}
7630	}
7631
7632	/ We need to restore pattern only if (new_ts == 0), because otherwise it is*
7633	either correct (new_ts & SPECULATIVE),
7634	or we simply don't care (new_ts & HARD_DEP). /*
7635
7636	gcc_assert (!ORIG_PAT (next)
7637	\|\| !IS_SPECULATION_BRANCHY_CHECK_P (next));
7638
7639	TODO_SPEC (next) = new_ts;
7640
7641	if (new_ts & (HARD_DEP \| DEP_POSTPONED))
7642	{
7643	/ We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because*
7644	control-speculative NEXT could have been discarded by sched-rgn.cc
7645	(the same case as when discarded by can_schedule_ready_p ()). /*
7646	/gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);/
7647
7648	change_queue_index (next, QUEUE_NOWHERE);
7649
7650	return -`1`;
7651	}
7652	else if (!(new_ts & BEGIN_SPEC)
7653	&& ORIG_PAT (next) && PREDICATED_PAT (next) == NULL_RTX
7654	&& !IS_SPECULATION_CHECK_P (next))
7655	/ We should change pattern of every previously speculative*
7656	instruction - and we determine if NEXT was speculative by using
7657	ORIG_PAT field. Except one case - speculation checks have ORIG_PAT
7658	pat too, so skip them. /*
7659	{
7660	bool success = haifa_change_pattern (next, ORIG_PAT (next));
7661	gcc_assert (success);
7662	ORIG_PAT (next) = `0`;
7663	}
7664
7665	if (sched_verbose >= `2`)
7666	{
7667	fprintf (stream: sched_dump, format: ";;\t\tdependencies resolved: insn %s",
7668	(*current_sched_info->print_insn) (next, `0`));
7669
7670	if (spec_info && spec_info->dump)
7671	{
7672	if (new_ts & BEGIN_DATA)
7673	fprintf (stream: spec_info->dump, format: "; data-spec;");
7674	if (new_ts & BEGIN_CONTROL)
7675	fprintf (stream: spec_info->dump, format: "; control-spec;");
7676	if (new_ts & BE_IN_CONTROL)
7677	fprintf (stream: spec_info->dump, format: "; in-control-spec;");
7678	}
7679	if (TODO_SPEC (next) & DEP_CONTROL)
7680	fprintf (stream: sched_dump, format: " predicated");
7681	fprintf (stream: sched_dump, format: "\n");
7682	}
7683
7684	adjust_priority (prev: next);
7685
7686	return fix_tick_ready (next);
7687	}
7688
7689	/ Calculate INSN_TICK of NEXT and add it to either ready or queue list. /
7690	static int
7691	fix_tick_ready (rtx_insn *next)
7692	{
7693	int tick, delay;
7694
7695	if (!DEBUG_INSN_P (next) && !sd_lists_empty_p (next, SD_LIST_RES_BACK))
7696	{
7697	int full_p;
7698	sd_iterator_def sd_it;
7699	dep_t dep;
7700
7701	tick = INSN_TICK (next);
7702	/ if tick is not equal to INVALID_TICK, then update*
7703	INSN_TICK of NEXT with the most recent resolved dependence
7704	cost. Otherwise, recalculate from scratch. /*
7705	full_p = (tick == INVALID_TICK);
7706
7707	FOR_EACH_DEP (next, SD_LIST_RES_BACK, sd_it, dep)
7708	{
7709	rtx_insn *pro = DEP_PRO (dep);
7710	int tick1;
7711
7712	gcc_assert (INSN_TICK (pro) >= MIN_TICK);
7713
7714	tick1 = INSN_TICK (pro) + dep_cost (link: dep);
7715	if (tick1 > tick)
7716	tick = tick1;
7717
7718	if (!full_p)
7719	break;
7720	}
7721	}
7722	else
7723	tick = -`1`;
7724
7725	INSN_TICK (next) = tick;
7726
7727	delay = tick - clock_var;
7728	if (delay <= `0` \|\| sched_pressure != SCHED_PRESSURE_NONE \|\| sched_fusion)
7729	delay = QUEUE_READY;
7730
7731	change_queue_index (next, delay);
7732
7733	return delay;
7734	}
7735
7736	/ Move NEXT to the proper queue list with (DELAY >= 1),*
7737	or add it to the ready list (DELAY == QUEUE_READY),
7738	or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). /*
7739	static void
7740	change_queue_index (rtx_insn next, int* delay)
7741	{
7742	int i = QUEUE_INDEX (next);
7743
7744	gcc_assert (QUEUE_NOWHERE <= delay && delay <= max_insn_queue_index
7745	&& delay != `0`);
7746	gcc_assert (i != QUEUE_SCHEDULED);
7747
7748	if ((delay > `0` && NEXT_Q_AFTER (q_ptr, delay) == i)
7749	\|\| (delay < `0` && delay == i))
7750	/ We have nothing to do. /
7751	return;
7752
7753	/ Remove NEXT from wherever it is now. /
7754	if (i == QUEUE_READY)
7755	ready_remove_insn (insn: next);
7756	else if (i >= `0`)
7757	queue_remove (insn: next);
7758
7759	/ Add it to the proper place. /
7760	if (delay == QUEUE_READY)
7761	ready_add (ready: readyp, insn: next, first_p: false);
7762	else if (delay >= `1`)
7763	queue_insn (insn: next, n_cycles: delay, reason: "change queue index");
7764
7765	if (sched_verbose >= `2`)
7766	{
7767	fprintf (stream: sched_dump, format: ";;\t\ttick updated: insn %s",
7768	(*current_sched_info->print_insn) (next, `0`));
7769
7770	if (delay == QUEUE_READY)
7771	fprintf (stream: sched_dump, format: " into ready\n");
7772	else if (delay >= `1`)
7773	fprintf (stream: sched_dump, format: " into queue with cost=%d\n", delay);
7774	else
7775	fprintf (stream: sched_dump, format: " removed from ready or queue lists\n");
7776	}
7777	}
7778
7779	static int sched_ready_n_insns = -`1`;
7780
7781	/ Initialize per region data structures. /
7782	void
7783	sched_extend_ready_list (int new_sched_ready_n_insns)
7784	{
7785	int i;
7786
7787	if (sched_ready_n_insns == -`1`)
7788	/ At the first call we need to initialize one more choice_stack*
7789	entry. /*
7790	{
7791	i = `0`;
7792	sched_ready_n_insns = `0`;
7793	scheduled_insns.reserve (nelems: new_sched_ready_n_insns);
7794	}
7795	else
7796	i = sched_ready_n_insns + `1`;
7797
7798	ready.veclen = new_sched_ready_n_insns + issue_rate;
7799	ready.vec = XRESIZEVEC (rtx_insn *, ready.vec, ready.veclen);
7800
7801	gcc_assert (new_sched_ready_n_insns >= sched_ready_n_insns);
7802
7803	ready_try = (signed char *) xrecalloc (ready_try, new_sched_ready_n_insns,
7804	sched_ready_n_insns,
7805	sizeof (*ready_try));
7806
7807	/ We allocate +1 element to save initial state in the choice_stack[0]*
7808	entry. /*
7809	choice_stack = XRESIZEVEC (struct choice_entry, choice_stack,
7810	new_sched_ready_n_insns + `1`);
7811
7812	for (; i <= new_sched_ready_n_insns; i++)
7813	{
7814	choice_stack[i].state = xmalloc (dfa_state_size);
7815
7816	if (targetm.sched.first_cycle_multipass_init)
7817	targetm.sched.first_cycle_multipass_init (&(choice_stack[i]
7818	.target_data));
7819	}
7820
7821	sched_ready_n_insns = new_sched_ready_n_insns;
7822	}
7823
7824	/ Free per region data structures. /
7825	void
7826	sched_finish_ready_list (void)
7827	{
7828	int i;
7829
7830	free (ptr: ready.vec);
7831	ready.vec = NULL;
7832	ready.veclen = `0`;
7833
7834	free (ptr: ready_try);
7835	ready_try = NULL;
7836
7837	for (i = `0`; i <= sched_ready_n_insns; i++)
7838	{
7839	if (targetm.sched.first_cycle_multipass_fini)
7840	targetm.sched.first_cycle_multipass_fini (&(choice_stack[i]
7841	.target_data));
7842
7843	free (ptr: choice_stack [i].state);
7844	}
7845	free (ptr: choice_stack);
7846	choice_stack = NULL;
7847
7848	sched_ready_n_insns = -`1`;
7849	}
7850
7851	static int
7852	haifa_luid_for_non_insn (rtx x)
7853	{
7854	gcc_assert (NOTE_P (x) \|\| LABEL_P (x));
7855
7856	return `0`;
7857	}
7858
7859	/ Generates recovery code for INSN. /
7860	static void
7861	generate_recovery_code (rtx_insn *insn)
7862	{
7863	if (TODO_SPEC (insn) & BEGIN_SPEC)
7864	begin_speculative_block (insn);
7865
7866	/ Here we have insn with no dependencies to*
7867	instructions other then CHECK_SPEC ones. /*
7868
7869	if (TODO_SPEC (insn) & BE_IN_SPEC)
7870	add_to_speculative_block (insn);
7871	}
7872
7873	/ Helper function.*
7874	Tries to add speculative dependencies of type FS between instructions
7875	in deps_list L and TWIN. /*
7876	static void
7877	process_insn_forw_deps_be_in_spec (rtx_insn insn, rtx_insn twin, ds_t fs)
7878	{
7879	sd_iterator_def sd_it;
7880	dep_t dep;
7881
7882	FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep)
7883	{
7884	ds_t ds;
7885	rtx_insn *consumer;
7886
7887	consumer = DEP_CON (dep);
7888
7889	ds = DEP_STATUS (dep);
7890
7891	if (/ If we want to create speculative dep. /
7892	fs
7893	/ And we can do that because this is a true dep. /
7894	&& (ds & DEP_TYPES) == DEP_TRUE)
7895	{
7896	gcc_assert (!(ds & BE_IN_SPEC));
7897
7898	if (/ If this dep can be overcome with 'begin speculation'. /
7899	ds & BEGIN_SPEC)
7900	/ Then we have a choice: keep the dep 'begin speculative'*
7901	or transform it into 'be in speculative'. /*
7902	{
7903	if (/ In try_ready we assert that if insn once became ready*
7904	it can be removed from the ready (or queue) list only
7905	due to backend decision. Hence we can't let the
7906	probability of the speculative dep to decrease. /*
7907	ds_weak (ds) <= ds_weak (fs))
7908	{
7909	ds_t new_ds;
7910
7911	new_ds = (ds & ~BEGIN_SPEC) \| fs;
7912
7913	if (/ consumer can 'be in speculative'. /
7914	sched_insn_is_legitimate_for_speculation_p (consumer,
7915	new_ds))
7916	/ Transform it to be in speculative. /
7917	ds = new_ds;
7918	}
7919	}
7920	else
7921	/ Mark the dep as 'be in speculative'. /
7922	ds \|= fs;
7923	}
7924
7925	{
7926	dep_def _new_dep, *new_dep = &_new_dep;
7927
7928	init_dep_1 (new_dep, twin, consumer, DEP_TYPE (dep), ds);
7929	sd_add_dep (new_dep, false);
7930	}
7931	}
7932	}
7933
7934	/ Generates recovery code for BEGIN speculative INSN. /
7935	static void
7936	begin_speculative_block (rtx_insn *insn)
7937	{
7938	if (TODO_SPEC (insn) & BEGIN_DATA)
7939	nr_begin_data++;
7940	if (TODO_SPEC (insn) & BEGIN_CONTROL)
7941	nr_begin_control++;
7942
7943	create_check_block_twin (insn, false);
7944
7945	TODO_SPEC (insn) &= ~BEGIN_SPEC;
7946	}
7947
7948	static void haifa_init_insn (rtx_insn *);
7949
7950	/ Generates recovery code for BE_IN speculative INSN. /
7951	static void
7952	add_to_speculative_block (rtx_insn *insn)
7953	{
7954	ds_t ts;
7955	sd_iterator_def sd_it;
7956	dep_t dep;
7957	auto_vec<rtx_insn *, `10`> twins;
7958
7959	ts = TODO_SPEC (insn);
7960	gcc_assert (!(ts & ~BE_IN_SPEC));
7961
7962	if (ts & BE_IN_DATA)
7963	nr_be_in_data++;
7964	if (ts & BE_IN_CONTROL)
7965	nr_be_in_control++;
7966
7967	TODO_SPEC (insn) &= ~BE_IN_SPEC;
7968	gcc_assert (!TODO_SPEC (insn));
7969
7970	DONE_SPEC (insn) \|= ts;
7971
7972	/ First we convert all simple checks to branchy. /
7973	for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
7974	sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep);)
7975	{
7976	rtx_insn *check = DEP_PRO (dep);
7977
7978	if (IS_SPECULATION_SIMPLE_CHECK_P (check))
7979	{
7980	create_check_block_twin (check, true);
7981
7982	/ Restart search. /
7983	sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
7984	}
7985	else
7986	/ Continue search. /
7987	sd_iterator_next (it_ptr: &sd_it);
7988	}
7989
7990	auto_vec<rtx_insn *> priorities_roots;
7991	clear_priorities (insn, &priorities_roots);
7992
7993	while (`1`)
7994	{
7995	rtx_insn check, twin;
7996	basic_block rec;
7997
7998	/ Get the first backward dependency of INSN. /
7999	sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8000	if (!sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep))
8001	/ INSN has no backward dependencies left. /
8002	break;
8003
8004	gcc_assert ((DEP_STATUS (dep) & BEGIN_SPEC) == `0`
8005	&& (DEP_STATUS (dep) & BE_IN_SPEC) != `0`
8006	&& (DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE);
8007
8008	check = DEP_PRO (dep);
8009
8010	gcc_assert (!IS_SPECULATION_CHECK_P (check) && !ORIG_PAT (check)
8011	&& QUEUE_INDEX (check) == QUEUE_NOWHERE);
8012
8013	rec = BLOCK_FOR_INSN (insn: check);
8014
8015	twin = emit_insn_before (copy_insn (PATTERN (insn)), BB_END (rec));
8016	haifa_init_insn (twin);
8017
8018	sd_copy_back_deps (twin, insn, true);
8019
8020	if (sched_verbose && spec_info->dump)
8021	/ INSN_BB (insn) isn't determined for twin insns yet.*
8022	So we can't use current_sched_info->print_insn. /*
8023	fprintf (stream: spec_info->dump, format: ";;\t\tGenerated twin insn : %d/rec%d\n",
8024	INSN_UID (insn: twin), rec->index);
8025
8026	twins.safe_push (obj: twin);
8027
8028	/ Add dependences between TWIN and all appropriate*
8029	instructions from REC. /*
8030	FOR_EACH_DEP (insn, SD_LIST_SPEC_BACK, sd_it, dep)
8031	{
8032	rtx_insn *pro = DEP_PRO (dep);
8033
8034	gcc_assert (DEP_TYPE (dep) == REG_DEP_TRUE);
8035
8036	/ INSN might have dependencies from the instructions from*
8037	several recovery blocks. At this iteration we process those
8038	producers that reside in REC. /*
8039	if (BLOCK_FOR_INSN (insn: pro) == rec)
8040	{
8041	dep_def _new_dep, *new_dep = &_new_dep;
8042
8043	init_dep (new_dep, pro, twin, REG_DEP_TRUE);
8044	sd_add_dep (new_dep, false);
8045	}
8046	}
8047
8048	process_insn_forw_deps_be_in_spec (insn, twin, fs: ts);
8049
8050	/ Remove all dependencies between INSN and insns in REC. /
8051	for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8052	sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep);)
8053	{
8054	rtx_insn *pro = DEP_PRO (dep);
8055
8056	if (BLOCK_FOR_INSN (insn: pro) == rec)
8057	sd_delete_dep (sd_it);
8058	else
8059	sd_iterator_next (it_ptr: &sd_it);
8060	}
8061	}
8062
8063	/ We couldn't have added the dependencies between INSN and TWINS earlier*
8064	because that would make TWINS appear in the INSN_BACK_DEPS (INSN). /*
8065	unsigned int i;
8066	rtx_insn *twin;
8067	FOR_EACH_VEC_ELT_REVERSE (twins, i, twin)
8068	{
8069	dep_def _new_dep, *new_dep = &_new_dep;
8070
8071	init_dep (new_dep, insn, twin, REG_DEP_OUTPUT);
8072	sd_add_dep (new_dep, false);
8073	}
8074
8075	calc_priorities (priorities_roots);
8076	}
8077
8078	/ Extends and fills with zeros (only the new part) array pointed to by P. /
8079	void *
8080	xrecalloc (void *p, size_t new_nmemb, size_t old_nmemb, size_t size)
8081	{
8082	gcc_assert (new_nmemb >= old_nmemb);
8083	p = XRESIZEVAR (void, p, new_nmemb * size);
8084	memset (s: ((char ) p) + old_nmemb size, c: `0`, n: (new_nmemb - old_nmemb) * size);
8085	return p;
8086	}
8087
8088	/ Helper function.*
8089	Find fallthru edge from PRED. /*
8090	edge
8091	find_fallthru_edge_from (basic_block pred)
8092	{
8093	edge e;
8094	basic_block succ;
8095
8096	succ = pred->next_bb;
8097	gcc_assert (succ->prev_bb == pred);
8098
8099	if (EDGE_COUNT (pred->succs) <= EDGE_COUNT (succ->preds))
8100	{
8101	e = find_fallthru_edge (edges: pred->succs);
8102
8103	if (e)
8104	{
8105	gcc_assert (e->dest == succ \|\| e->dest->index == EXIT_BLOCK);
8106	return e;
8107	}
8108	}
8109	else
8110	{
8111	e = find_fallthru_edge (edges: succ->preds);
8112
8113	if (e)
8114	{
8115	gcc_assert (e->src == pred);
8116	return e;
8117	}
8118	}
8119
8120	return NULL;
8121	}
8122
8123	/ Extend per basic block data structures. /
8124	static void
8125	sched_extend_bb (void)
8126	{
8127	/ The following is done to keep current_sched_info->next_tail non null. /
8128	rtx_insn *end = BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb);
8129	rtx_insn *insn = DEBUG_INSN_P (end) ? prev_nondebug_insn (end) : end;
8130	if (NEXT_INSN (insn: end) == `0`
8131	\|\| (!NOTE_P (insn)
8132	&& !LABEL_P (insn)
8133	/ Don't emit a NOTE if it would end up before a BARRIER. /
8134	&& !BARRIER_P (next_nondebug_insn (end))))
8135	{
8136	rtx_note *note = emit_note_after (NOTE_INSN_DELETED, end);
8137	/ Make note appear outside BB. /
8138	set_block_for_insn (insn: note, NULL);
8139	BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb) = end;
8140	}
8141	}
8142
8143	/ Init per basic block data structures. /
8144	void
8145	sched_init_bbs (void)
8146	{
8147	sched_extend_bb ();
8148	}
8149
8150	/ Initialize BEFORE_RECOVERY variable. /
8151	static void
8152	init_before_recovery (basic_block *before_recovery_ptr)
8153	{
8154	basic_block last;
8155	edge e;
8156
8157	last = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
8158	e = find_fallthru_edge_from (pred: last);
8159
8160	if (e)
8161	{
8162	/ We create two basic blocks:*
8163	1. Single instruction block is inserted right after E->SRC
8164	and has jump to
8165	2. Empty block right before EXIT_BLOCK.
8166	Between these two blocks recovery blocks will be emitted. /*
8167
8168	basic_block single, empty;
8169
8170	/ If the fallthrough edge to exit we've found is from the block we've*
8171	created before, don't do anything more. /*
8172	if (last == after_recovery)
8173	return;
8174
8175	adding_bb_to_current_region_p = false;
8176
8177	single = sched_create_empty_bb (last);
8178	empty = sched_create_empty_bb (single);
8179
8180	/ Add new blocks to the root loop. /
8181	if (current_loops != NULL)
8182	{
8183	add_bb_to_loop (single, (*current_loops->larray)[`0`]);
8184	add_bb_to_loop (empty, (*current_loops->larray)[`0`]);
8185	}
8186
8187	single->count = last->count;
8188	empty->count = last->count;
8189	BB_COPY_PARTITION (single, last);
8190	BB_COPY_PARTITION (empty, last);
8191
8192	redirect_edge_succ (e, single);
8193	make_single_succ_edge (single, empty, `0`);
8194	make_single_succ_edge (empty, EXIT_BLOCK_PTR_FOR_FN (cfun),
8195	EDGE_FALLTHRU);
8196
8197	rtx_code_label *label = block_label (empty);
8198	rtx_jump_insn *x = emit_jump_insn_after (targetm.gen_jump (label),
8199	BB_END (single));
8200	JUMP_LABEL (x) = label;
8201	LABEL_NUSES (label)++;
8202	haifa_init_insn (x);
8203
8204	emit_barrier_after (x);
8205
8206	sched_init_only_bb (empty, NULL);
8207	sched_init_only_bb (single, NULL);
8208	sched_extend_bb ();
8209
8210	adding_bb_to_current_region_p = true;
8211	before_recovery = single;
8212	after_recovery = empty;
8213
8214	if (before_recovery_ptr)
8215	*before_recovery_ptr = before_recovery;
8216
8217	if (sched_verbose >= `2` && spec_info->dump)
8218	fprintf (stream: spec_info->dump,
8219	format: ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n",
8220	last->index, single->index, empty->index);
8221	}
8222	else
8223	before_recovery = last;
8224	}
8225
8226	/ Returns new recovery block. /
8227	basic_block
8228	sched_create_recovery_block (basic_block *before_recovery_ptr)
8229	{
8230	rtx_insn *barrier;
8231	basic_block rec;
8232
8233	haifa_recovery_bb_recently_added_p = true;
8234	haifa_recovery_bb_ever_added_p = true;
8235
8236	init_before_recovery (before_recovery_ptr);
8237
8238	barrier = get_last_bb_insn (before_recovery);
8239	gcc_assert (BARRIER_P (barrier));
8240
8241	rtx_insn *label = emit_label_after (gen_label_rtx (), barrier);
8242
8243	rec = create_basic_block (label, label, before_recovery);
8244
8245	/ A recovery block always ends with an unconditional jump. /
8246	emit_barrier_after (BB_END (rec));
8247
8248	if (BB_PARTITION (before_recovery) != BB_UNPARTITIONED)
8249	BB_SET_PARTITION (rec, BB_COLD_PARTITION);
8250
8251	if (sched_verbose && spec_info->dump)
8252	fprintf (stream: spec_info->dump, format: ";;\t\tGenerated recovery block rec%d\n",
8253	rec->index);
8254
8255	return rec;
8256	}
8257
8258	/ Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB*
8259	and emit necessary jumps. /*
8260	void
8261	sched_create_recovery_edges (basic_block first_bb, basic_block rec,
8262	basic_block second_bb)
8263	{
8264	int edge_flags;
8265
8266	/ This is fixing of incoming edge. /
8267	/ ??? Which other flags should be specified? /
8268	if (BB_PARTITION (first_bb) != BB_PARTITION (rec))
8269	/ Partition type is the same, if it is "unpartitioned". /
8270	edge_flags = EDGE_CROSSING;
8271	else
8272	edge_flags = `0`;
8273
8274	edge e2 = single_succ_edge (bb: first_bb);
8275	edge e = make_edge (first_bb, rec, edge_flags);
8276
8277	/ TODO: The actual probability can be determined and is computed as*
8278	'todo_spec' variable in create_check_block_twin and
8279	in sel-sched.cc `check_ds' in create_speculation_check. /*
8280	e->probability = profile_probability::very_unlikely ();
8281	rec->count = e->count ();
8282	e2->probability = e->probability.invert ();
8283
8284	rtx_code_label *label = block_label (second_bb);
8285	rtx_jump_insn *jump = emit_jump_insn_after (targetm.gen_jump (label),
8286	BB_END (rec));
8287	JUMP_LABEL (jump) = label;
8288	LABEL_NUSES (label)++;
8289
8290	if (BB_PARTITION (second_bb) != BB_PARTITION (rec))
8291	/ Partition type is the same, if it is "unpartitioned". /
8292	{
8293	/ Rewritten from cfgrtl.cc. /
8294	if (crtl->has_bb_partition && targetm_common.have_named_sections)
8295	{
8296	/ We don't need the same note for the check because*
8297	any_condjump_p (check) == true. /*
8298	CROSSING_JUMP_P (jump) = `1`;
8299	}
8300	edge_flags = EDGE_CROSSING;
8301	}
8302	else
8303	edge_flags = `0`;
8304
8305	make_single_succ_edge (rec, second_bb, edge_flags);
8306	if (dom_info_available_p (CDI_DOMINATORS))
8307	set_immediate_dominator (CDI_DOMINATORS, rec, first_bb);
8308	}
8309
8310	/ This function creates recovery code for INSN. If MUTATE_P is nonzero,*
8311	INSN is a simple check, that should be converted to branchy one. /*
8312	static void
8313	create_check_block_twin (rtx_insn insn, bool* mutate_p)
8314	{
8315	basic_block rec;
8316	rtx_insn label, check, *twin;
8317	rtx check_pat;
8318	ds_t fs;
8319	sd_iterator_def sd_it;
8320	dep_t dep;
8321	dep_def _new_dep, *new_dep = &_new_dep;
8322	ds_t todo_spec;
8323
8324	gcc_assert (ORIG_PAT (insn) != NULL_RTX);
8325
8326	if (!mutate_p)
8327	todo_spec = TODO_SPEC (insn);
8328	else
8329	{
8330	gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn)
8331	&& (TODO_SPEC (insn) & SPECULATIVE) == `0`);
8332
8333	todo_spec = CHECK_SPEC (insn);
8334	}
8335
8336	todo_spec &= SPECULATIVE;
8337
8338	/ Create recovery block. /
8339	if (mutate_p \|\| targetm.sched.needs_block_p (todo_spec))
8340	{
8341	rec = sched_create_recovery_block (NULL);
8342	label = BB_HEAD (rec);
8343	}
8344	else
8345	{
8346	rec = EXIT_BLOCK_PTR_FOR_FN (cfun);
8347	label = NULL;
8348	}
8349
8350	/ Emit CHECK. /
8351	check_pat = targetm.sched.gen_spec_check (insn, label, todo_spec);
8352
8353	if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8354	{
8355	/ To have mem_reg alive at the beginning of second_bb,*
8356	we emit check BEFORE insn, so insn after splitting
8357	insn will be at the beginning of second_bb, which will
8358	provide us with the correct life information. /*
8359	check = emit_jump_insn_before (check_pat, insn);
8360	JUMP_LABEL (check) = label;
8361	LABEL_NUSES (label)++;
8362	}
8363	else
8364	check = emit_insn_before (check_pat, insn);
8365
8366	/ Extend data structures. /
8367	haifa_init_insn (check);
8368
8369	/ CHECK is being added to current region. Extend ready list. /
8370	gcc_assert (sched_ready_n_insns != -`1`);
8371	sched_extend_ready_list (new_sched_ready_n_insns: sched_ready_n_insns + `1`);
8372
8373	if (current_sched_info->add_remove_insn)
8374	current_sched_info->add_remove_insn (insn, `0`);
8375
8376	RECOVERY_BLOCK (check) = rec;
8377
8378	if (sched_verbose && spec_info->dump)
8379	fprintf (stream: spec_info->dump, format: ";;\t\tGenerated check insn : %s\n",
8380	(*current_sched_info->print_insn) (check, `0`));
8381
8382	gcc_assert (ORIG_PAT (insn));
8383
8384	/ Initialize TWIN (twin is a duplicate of original instruction*
8385	in the recovery block). /*
8386	if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8387	{
8388	sd_iterator_def sd_it;
8389	dep_t dep;
8390
8391	FOR_EACH_DEP (insn, SD_LIST_RES_BACK, sd_it, dep)
8392	if ((DEP_STATUS (dep) & DEP_OUTPUT) != `0`)
8393	{
8394	struct _dep _dep2, *dep2 = &_dep2;
8395
8396	init_dep (dep2, DEP_PRO (dep), check, REG_DEP_TRUE);
8397
8398	sd_add_dep (dep2, true);
8399	}
8400
8401	twin = emit_insn_after (ORIG_PAT (insn), BB_END (rec));
8402	haifa_init_insn (twin);
8403
8404	if (sched_verbose && spec_info->dump)
8405	/ INSN_BB (insn) isn't determined for twin insns yet.*
8406	So we can't use current_sched_info->print_insn. /*
8407	fprintf (stream: spec_info->dump, format: ";;\t\tGenerated twin insn : %d/rec%d\n",
8408	INSN_UID (insn: twin), rec->index);
8409	}
8410	else
8411	{
8412	ORIG_PAT (check) = ORIG_PAT (insn);
8413	HAS_INTERNAL_DEP (check) = `1`;
8414	twin = check;
8415	/ ??? We probably should change all OUTPUT dependencies to*
8416	(TRUE \| OUTPUT). /*
8417	}
8418
8419	/ Copy all resolved back dependencies of INSN to TWIN. This will*
8420	provide correct value for INSN_TICK (TWIN). /*
8421	sd_copy_back_deps (twin, insn, true);
8422
8423	if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8424	/ In case of branchy check, fix CFG. /
8425	{
8426	basic_block first_bb, second_bb;
8427	rtx_insn *jump;
8428
8429	first_bb = BLOCK_FOR_INSN (insn: check);
8430	second_bb = sched_split_block (first_bb, check);
8431
8432	sched_create_recovery_edges (first_bb, rec, second_bb);
8433
8434	sched_init_only_bb (second_bb, first_bb);
8435	sched_init_only_bb (rec, EXIT_BLOCK_PTR_FOR_FN (cfun));
8436
8437	jump = BB_END (rec);
8438	haifa_init_insn (jump);
8439	}
8440
8441	/ Move backward dependences from INSN to CHECK and*
8442	move forward dependences from INSN to TWIN. /*
8443
8444	/ First, create dependencies between INSN's producers and CHECK & TWIN. /
8445	FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
8446	{
8447	rtx_insn *pro = DEP_PRO (dep);
8448	ds_t ds;
8449
8450	/ If BEGIN_DATA: [insn ~~TRUE~~> producer]:*
8451	check --TRUE--> producer ??? or ANTI ???
8452	twin --TRUE--> producer
8453	twin --ANTI--> check
8454
8455	If BEGIN_CONTROL: [insn ~~ANTI~~> producer]:
8456	check --ANTI--> producer
8457	twin --ANTI--> producer
8458	twin --ANTI--> check
8459
8460	If BE_IN_SPEC: [insn ~~TRUE~~> producer]:
8461	check ~~TRUE~~> producer
8462	twin ~~TRUE~~> producer
8463	twin --ANTI--> check /*
8464
8465	ds = DEP_STATUS (dep);
8466
8467	if (ds & BEGIN_SPEC)
8468	{
8469	gcc_assert (!mutate_p);
8470	ds &= ~BEGIN_SPEC;
8471	}
8472
8473	init_dep_1 (new_dep, pro, check, DEP_TYPE (dep), ds);
8474	sd_add_dep (new_dep, false);
8475
8476	if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8477	{
8478	DEP_CON (new_dep) = twin;
8479	sd_add_dep (new_dep, false);
8480	}
8481	}
8482
8483	/ Second, remove backward dependencies of INSN. /
8484	for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8485	sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep);)
8486	{
8487	if ((DEP_STATUS (dep) & BEGIN_SPEC)
8488	\|\| mutate_p)
8489	/ We can delete this dep because we overcome it with*
8490	BEGIN_SPECULATION. /*
8491	sd_delete_dep (sd_it);
8492	else
8493	sd_iterator_next (it_ptr: &sd_it);
8494	}
8495
8496	/ Future Speculations. Determine what BE_IN speculations will be like. /
8497	fs = `0`;
8498
8499	/ Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only*
8500	here. /*
8501
8502	gcc_assert (!DONE_SPEC (insn));
8503
8504	if (!mutate_p)
8505	{
8506	ds_t ts = TODO_SPEC (insn);
8507
8508	DONE_SPEC (insn) = ts & BEGIN_SPEC;
8509	CHECK_SPEC (check) = ts & BEGIN_SPEC;
8510
8511	/ Luckiness of future speculations solely depends upon initial*
8512	BEGIN speculation. /*
8513	if (ts & BEGIN_DATA)
8514	fs = set_dep_weak (fs, BE_IN_DATA, get_dep_weak (ts, BEGIN_DATA));
8515	if (ts & BEGIN_CONTROL)
8516	fs = set_dep_weak (fs, BE_IN_CONTROL,
8517	get_dep_weak (ts, BEGIN_CONTROL));
8518	}
8519	else
8520	CHECK_SPEC (check) = CHECK_SPEC (insn);
8521
8522	/ Future speculations: call the helper. /
8523	process_insn_forw_deps_be_in_spec (insn, twin, fs);
8524
8525	if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8526	{
8527	/ Which types of dependencies should we use here is,*
8528	generally, machine-dependent question... But, for now,
8529	it is not. /*
8530
8531	if (!mutate_p)
8532	{
8533	init_dep (new_dep, insn, check, REG_DEP_TRUE);
8534	sd_add_dep (new_dep, false);
8535
8536	init_dep (new_dep, insn, twin, REG_DEP_OUTPUT);
8537	sd_add_dep (new_dep, false);
8538	}
8539	else
8540	{
8541	if (spec_info->dump)
8542	fprintf (stream: spec_info->dump, format: ";;\t\tRemoved simple check : %s\n",
8543	(*current_sched_info->print_insn) (insn, `0`));
8544
8545	/ Remove all dependencies of the INSN. /
8546	{
8547	sd_it = sd_iterator_start (insn, types: (SD_LIST_FORW
8548	\| SD_LIST_BACK
8549	\| SD_LIST_RES_BACK));
8550	while (sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep))
8551	sd_delete_dep (sd_it);
8552	}
8553
8554	/ If former check (INSN) already was moved to the ready (or queue)*
8555	list, add new check (CHECK) there too. /*
8556	if (QUEUE_INDEX (insn) != QUEUE_NOWHERE)
8557	try_ready (next: check);
8558
8559	/ Remove old check from instruction stream and free its*
8560	data. /*
8561	sched_remove_insn (insn);
8562	}
8563
8564	init_dep (new_dep, check, twin, REG_DEP_ANTI);
8565	sd_add_dep (new_dep, false);
8566	}
8567	else
8568	{
8569	init_dep_1 (new_dep, insn, check, REG_DEP_TRUE, DEP_TRUE \| DEP_OUTPUT);
8570	sd_add_dep (new_dep, false);
8571	}
8572
8573	if (!mutate_p)
8574	/ Fix priorities. If MUTATE_P is nonzero, this is not necessary,*
8575	because it'll be done later in add_to_speculative_block. /*
8576	{
8577	auto_vec<rtx_insn *> priorities_roots;
8578
8579	clear_priorities (twin, &priorities_roots);
8580	calc_priorities (priorities_roots);
8581	}
8582	}
8583
8584	/ Removes dependency between instructions in the recovery block REC*
8585	and usual region instructions. It keeps inner dependences so it
8586	won't be necessary to recompute them. /*
8587	static void
8588	fix_recovery_deps (basic_block rec)
8589	{
8590	rtx_insn note, insn, *jump;
8591	auto_vec<rtx_insn *, `10`> ready_list;
8592	auto_bitmap in_ready;
8593
8594	/ NOTE - a basic block note. /
8595	note = NEXT_INSN (BB_HEAD (rec));
8596	gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8597	insn = BB_END (rec);
8598	gcc_assert (JUMP_P (insn));
8599	insn = PREV_INSN (insn);
8600
8601	do
8602	{
8603	sd_iterator_def sd_it;
8604	dep_t dep;
8605
8606	for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
8607	sd_iterator_cond (it_ptr: &sd_it, dep_ptr: &dep);)
8608	{
8609	rtx_insn *consumer = DEP_CON (dep);
8610
8611	if (BLOCK_FOR_INSN (insn: consumer) != rec)
8612	{
8613	sd_delete_dep (sd_it);
8614
8615	if (bitmap_set_bit (in_ready, INSN_LUID (consumer)))
8616	ready_list.safe_push (obj: consumer);
8617	}
8618	else
8619	{
8620	gcc_assert ((DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE);
8621
8622	sd_iterator_next (it_ptr: &sd_it);
8623	}
8624	}
8625
8626	insn = PREV_INSN (insn);
8627	}
8628	while (insn != note);
8629
8630	/ Try to add instructions to the ready or queue list. /
8631	unsigned int i;
8632	rtx_insn *temp;
8633	FOR_EACH_VEC_ELT_REVERSE (ready_list, i, temp)
8634	try_ready (next: temp);
8635
8636	/ Fixing jump's dependences. /
8637	insn = BB_HEAD (rec);
8638	jump = BB_END (rec);
8639
8640	gcc_assert (LABEL_P (insn));
8641	insn = NEXT_INSN (insn);
8642
8643	gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn));
8644	add_jump_dependencies (insn, jump);
8645	}
8646
8647	/ Change pattern of INSN to NEW_PAT. Invalidate cached haifa*
8648	instruction data. /*
8649	static bool
8650	haifa_change_pattern (rtx_insn *insn, rtx new_pat)
8651	{
8652	int t;
8653
8654	t = validate_change (insn, &PATTERN (insn), new_pat, `0`);
8655	if (!t)
8656	return false;
8657
8658	update_insn_after_change (insn);
8659	return true;
8660	}
8661
8662	/ -1 - can't speculate,*
8663	0 - for speculation with REQUEST mode it is OK to use
8664	current instruction pattern,
8665	1 - need to change pattern for NEW_PAT to be speculative. /
8666	int
8667	sched_speculate_insn (rtx_insn insn, ds_t request, rtx new_pat)
8668	{
8669	gcc_assert (current_sched_info->flags & DO_SPECULATION
8670	&& (request & SPECULATIVE)
8671	&& sched_insn_is_legitimate_for_speculation_p (insn, request));
8672
8673	if ((request & spec_info->mask) != request)
8674	return -`1`;
8675
8676	if (request & BE_IN_SPEC
8677	&& !(request & BEGIN_SPEC))
8678	return `0`;
8679
8680	return targetm.sched.speculate_insn (insn, request, new_pat);
8681	}
8682
8683	static int
8684	haifa_speculate_insn (rtx_insn insn, ds_t request, rtx new_pat)
8685	{
8686	gcc_assert (sched_deps_info->generate_spec_deps
8687	&& !IS_SPECULATION_CHECK_P (insn));
8688
8689	if (HAS_INTERNAL_DEP (insn)
8690	\|\| SCHED_GROUP_P (insn))
8691	return -`1`;
8692
8693	return sched_speculate_insn (insn, request, new_pat);
8694	}
8695
8696	/ Print some information about block BB, which starts with HEAD and*
8697	ends with TAIL, before scheduling it.
8698	I is zero, if scheduler is about to start with the fresh ebb. /*
8699	static void
8700	dump_new_block_header (int i, basic_block bb, rtx_insn head, rtx_insn tail)
8701	{
8702	if (!i)
8703	fprintf (stream: sched_dump,
8704	format: ";; ======================================================\n");
8705	else
8706	fprintf (stream: sched_dump,
8707	format: ";; =====================ADVANCING TO=====================\n");
8708	fprintf (stream: sched_dump,
8709	format: ";; -- basic block %d from %d to %d -- %s reload\n",
8710	bb->index, INSN_UID (insn: head), INSN_UID (insn: tail),
8711	(reload_completed ? "after" : "before"));
8712	fprintf (stream: sched_dump,
8713	format: ";; ======================================================\n");
8714	fprintf (stream: sched_dump, format: "\n");
8715	}
8716
8717	/ Unlink basic block notes and labels and saves them, so they*
8718	can be easily restored. We unlink basic block notes in EBB to
8719	provide back-compatibility with the previous code, as target backends
8720	assume, that there'll be only instructions between
8721	current_sched_info->{head and tail}. We restore these notes as soon
8722	as we can.
8723	FIRST (LAST) is the first (last) basic block in the ebb.
8724	NB: In usual case (FIRST == LAST) nothing is really done. /*
8725	void
8726	unlink_bb_notes (basic_block first, basic_block last)
8727	{
8728	/ We DON'T unlink basic block notes of the first block in the ebb. /
8729	if (first == last)
8730	return;
8731
8732	bb_header = XNEWVEC (rtx_insn *, last_basic_block_for_fn (cfun));
8733
8734	/ Make a sentinel. /
8735	if (last->next_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
8736	bb_header[last->next_bb->index] = `0`;
8737
8738	first = first->next_bb;
8739	do
8740	{
8741	rtx_insn prev, label, note, next;
8742
8743	label = BB_HEAD (last);
8744	if (LABEL_P (label))
8745	note = NEXT_INSN (insn: label);
8746	else
8747	note = label;
8748	gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8749
8750	prev = PREV_INSN (insn: label);
8751	next = NEXT_INSN (insn: note);
8752	gcc_assert (prev && next);
8753
8754	SET_NEXT_INSN (prev) = next;
8755	SET_PREV_INSN (next) = prev;
8756
8757	bb_header[last->index] = label;
8758
8759	if (last == first)
8760	break;
8761
8762	last = last->prev_bb;
8763	}
8764	while (`1`);
8765	}
8766
8767	/ Restore basic block notes.*
8768	FIRST is the first basic block in the ebb. /*
8769	static void
8770	restore_bb_notes (basic_block first)
8771	{
8772	if (!bb_header)
8773	return;
8774
8775	/ We DON'T unlink basic block notes of the first block in the ebb. /
8776	first = first->next_bb;
8777	/ Remember: FIRST is actually a second basic block in the ebb. /
8778
8779	while (first != EXIT_BLOCK_PTR_FOR_FN (cfun)
8780	&& bb_header[first->index])
8781	{
8782	rtx_insn prev, label, note, next;
8783
8784	label = bb_header[first->index];
8785	prev = PREV_INSN (insn: label);
8786	next = NEXT_INSN (insn: prev);
8787
8788	if (LABEL_P (label))
8789	note = NEXT_INSN (insn: label);
8790	else
8791	note = label;
8792	gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8793
8794	bb_header[first->index] = `0`;
8795
8796	SET_NEXT_INSN (prev) = label;
8797	SET_NEXT_INSN (note) = next;
8798	SET_PREV_INSN (next) = note;
8799
8800	first = first->next_bb;
8801	}
8802
8803	free (ptr: bb_header);
8804	bb_header = `0`;
8805	}
8806
8807	/ Helper function.*
8808	Fix CFG after both in- and inter-block movement of
8809	control_flow_insn_p JUMP. /*
8810	static void
8811	fix_jump_move (rtx_insn *jump)
8812	{
8813	basic_block bb, jump_bb, jump_bb_next;
8814
8815	bb = BLOCK_FOR_INSN (insn: PREV_INSN (insn: jump));
8816	jump_bb = BLOCK_FOR_INSN (insn: jump);
8817	jump_bb_next = jump_bb->next_bb;
8818
8819	gcc_assert (common_sched_info->sched_pass_id == SCHED_EBB_PASS
8820	\|\| IS_SPECULATION_BRANCHY_CHECK_P (jump));
8821
8822	if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next)))
8823	/ if jump_bb_next is not empty. /
8824	BB_END (jump_bb) = BB_END (jump_bb_next);
8825
8826	if (BB_END (bb) != PREV_INSN (insn: jump))
8827	/ Then there are instruction after jump that should be placed*
8828	to jump_bb_next. /*
8829	BB_END (jump_bb_next) = BB_END (bb);
8830	else
8831	/ Otherwise jump_bb_next is empty. /
8832	BB_END (jump_bb_next) = NEXT_INSN (BB_HEAD (jump_bb_next));
8833
8834	/ To make assertion in move_insn happy. /
8835	BB_END (bb) = PREV_INSN (insn: jump);
8836
8837	update_bb_for_insn (jump_bb_next);
8838	}
8839
8840	/ Fix CFG after interblock movement of control_flow_insn_p JUMP. /
8841	static void
8842	move_block_after_check (rtx_insn *jump)
8843	{
8844	basic_block bb, jump_bb, jump_bb_next;
8845	vec<edge, va_gc> *t;
8846
8847	bb = BLOCK_FOR_INSN (insn: PREV_INSN (insn: jump));
8848	jump_bb = BLOCK_FOR_INSN (insn: jump);
8849	jump_bb_next = jump_bb->next_bb;
8850
8851	update_bb_for_insn (jump_bb);
8852
8853	gcc_assert (IS_SPECULATION_CHECK_P (jump)
8854	\|\| IS_SPECULATION_CHECK_P (BB_END (jump_bb_next)));
8855
8856	unlink_block (jump_bb_next);
8857	link_block (jump_bb_next, bb);
8858
8859	t = bb->succs;
8860	bb->succs = `0`;
8861	move_succs (&(jump_bb->succs), bb);
8862	move_succs (&(jump_bb_next->succs), jump_bb);
8863	move_succs (&t, jump_bb_next);
8864
8865	df_mark_solutions_dirty ();
8866
8867	common_sched_info->fix_recovery_cfg
8868	(bb->index, jump_bb->index, jump_bb_next->index);
8869	}
8870
8871	/ Helper function for move_block_after_check.*
8872	This functions attaches edge vector pointed to by SUCCSP to
8873	block TO. /*
8874	static void
8875	move_succs (vec<edge, va_gc> **succsp, basic_block to)
8876	{
8877	edge e;
8878	edge_iterator ei;
8879
8880	gcc_assert (to->succs == `0`);
8881
8882	to->succs = *succsp;
8883
8884	FOR_EACH_EDGE (e, ei, to->succs)
8885	e->src = to;
8886
8887	*succsp = `0`;
8888	}
8889
8890	/ Remove INSN from the instruction stream.*
8891	INSN should have any dependencies. /*
8892	static void
8893	sched_remove_insn (rtx_insn *insn)
8894	{
8895	sd_finish_insn (insn);
8896
8897	change_queue_index (next: insn, QUEUE_NOWHERE);
8898	current_sched_info->add_remove_insn (insn, `1`);
8899	delete_insn (insn);
8900	}
8901
8902	/ Clear priorities of all instructions, that are forward dependent on INSN.*
8903	Store in vector pointed to by ROOTS_PTR insns on which priority () should
8904	be invoked to initialize all cleared priorities. /*
8905	static void
8906	clear_priorities (rtx_insn insn, rtx_vec_t roots_ptr)
8907	{
8908	sd_iterator_def sd_it;
8909	dep_t dep;
8910	bool insn_is_root_p = true;
8911
8912	gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
8913
8914	FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
8915	{
8916	rtx_insn *pro = DEP_PRO (dep);
8917
8918	if (INSN_PRIORITY_STATUS (pro) >= `0`
8919	&& QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
8920	{
8921	/ If DEP doesn't contribute to priority then INSN itself should*
8922	be added to priority roots. /*
8923	if (contributes_to_priority_p (dep))
8924	insn_is_root_p = false;
8925
8926	INSN_PRIORITY_STATUS (pro) = -`1`;
8927	clear_priorities (insn: pro, roots_ptr);
8928	}
8929	}
8930
8931	if (insn_is_root_p)
8932	roots_ptr->safe_push (obj: insn);
8933	}
8934
8935	/ Recompute priorities of instructions, whose priorities might have been*
8936	changed. ROOTS is a vector of instructions whose priority computation will
8937	trigger initialization of all cleared priorities. /*
8938	static void
8939	calc_priorities (const rtx_vec_t &roots)
8940	{
8941	int i;
8942	rtx_insn *insn;
8943
8944	FOR_EACH_VEC_ELT (roots, i, insn)
8945	priority (insn);
8946	}
8947
8948
8949	/ Add dependences between JUMP and other instructions in the recovery*
8950	block. INSN is the first insn the recovery block. /*
8951	static void
8952	add_jump_dependencies (rtx_insn insn, rtx_insn jump)
8953	{
8954	do
8955	{
8956	insn = NEXT_INSN (insn);
8957	if (insn == jump)
8958	break;
8959
8960	if (dep_list_size (insn, SD_LIST_FORW) == `0`)
8961	{
8962	dep_def _new_dep, *new_dep = &_new_dep;
8963
8964	init_dep (new_dep, insn, jump, REG_DEP_ANTI);
8965	sd_add_dep (new_dep, false);
8966	}
8967	}
8968	while (`1`);
8969
8970	gcc_assert (!sd_lists_empty_p (jump, SD_LIST_BACK));
8971	}
8972
8973	/ Extend data structures for logical insn UID. /
8974	void
8975	sched_extend_luids (void)
8976	{
8977	int new_luids_max_uid = get_max_uid () + `1`;
8978
8979	sched_luids.safe_grow_cleared (len: new_luids_max_uid, exact: true);
8980	}
8981
8982	/ Initialize LUID for INSN. /
8983	void
8984	sched_init_insn_luid (rtx_insn *insn)
8985	{
8986	int i = INSN_P (insn) ? `1` : common_sched_info->luid_for_non_insn (insn);
8987	int luid;
8988
8989	if (i >= `0`)
8990	{
8991	luid = sched_max_luid;
8992	sched_max_luid += i;
8993	}
8994	else
8995	luid = -`1`;
8996
8997	SET_INSN_LUID (insn, luid);
8998	}
8999
9000	/ Initialize luids for BBS.*
9001	The hook common_sched_info->luid_for_non_insn () is used to determine
9002	if notes, labels, etc. need luids. /*
9003	void
9004	sched_init_luids (const bb_vec_t &bbs)
9005	{
9006	int i;
9007	basic_block bb;
9008
9009	sched_extend_luids ();
9010	FOR_EACH_VEC_ELT (bbs, i, bb)
9011	{
9012	rtx_insn *insn;
9013
9014	FOR_BB_INSNS (bb, insn)
9015	sched_init_insn_luid (insn);
9016	}
9017	}
9018
9019	/ Free LUIDs. /
9020	void
9021	sched_finish_luids (void)
9022	{
9023	sched_luids.release ();
9024	sched_max_luid = `1`;
9025	}
9026
9027	/ Return logical uid of INSN. Helpful while debugging. /
9028	int
9029	insn_luid (rtx_insn *insn)
9030	{
9031	return INSN_LUID (insn);
9032	}
9033
9034	/ Extend per insn data in the target. /
9035	void
9036	sched_extend_target (void)
9037	{
9038	if (targetm.sched.h_i_d_extended)
9039	targetm.sched.h_i_d_extended ();
9040	}
9041
9042	/ Extend global scheduler structures (those, that live across calls to*
9043	schedule_block) to include information about just emitted INSN. /*
9044	static void
9045	extend_h_i_d (void)
9046	{
9047	int reserve = (get_max_uid () + `1` - h_i_d.length ());
9048	if (reserve > `0`
9049	&& ! h_i_d.space (nelems: reserve))
9050	{
9051	h_i_d.safe_grow_cleared (len: `3U` * get_max_uid () / `2`, exact: true);
9052	sched_extend_target ();
9053	}
9054	}
9055
9056	/ Initialize h_i_d entry of the INSN with default values.*
9057	Values, that are not explicitly initialized here, hold zero. /*
9058	static void
9059	init_h_i_d (rtx_insn *insn)
9060	{
9061	if (INSN_LUID (insn) > `0`)
9062	{
9063	INSN_COST (insn) = -`1`;
9064	QUEUE_INDEX (insn) = QUEUE_NOWHERE;
9065	INSN_TICK (insn) = INVALID_TICK;
9066	INSN_EXACT_TICK (insn) = INVALID_TICK;
9067	INTER_TICK (insn) = INVALID_TICK;
9068	TODO_SPEC (insn) = HARD_DEP;
9069	INSN_AUTOPREF_MULTIPASS_DATA (insn)[`0`].status
9070	= AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
9071	INSN_AUTOPREF_MULTIPASS_DATA (insn)[`1`].status
9072	= AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
9073	}
9074	}
9075
9076	/ Initialize haifa_insn_data for BBS. /
9077	void
9078	haifa_init_h_i_d (const bb_vec_t &bbs)
9079	{
9080	int i;
9081	basic_block bb;
9082
9083	extend_h_i_d ();
9084	FOR_EACH_VEC_ELT (bbs, i, bb)
9085	{
9086	rtx_insn *insn;
9087
9088	FOR_BB_INSNS (bb, insn)
9089	init_h_i_d (insn);
9090	}
9091	}
9092
9093	/ Finalize haifa_insn_data. /
9094	void
9095	haifa_finish_h_i_d (void)
9096	{
9097	int i;
9098	haifa_insn_data_t data;
9099	reg_use_data use, next_use;
9100	reg_set_data set, next_set;
9101
9102	FOR_EACH_VEC_ELT (h_i_d, i, data)
9103	{
9104	free (ptr: data->max_reg_pressure);
9105	free (ptr: data->reg_pressure);
9106	for (use = data->reg_use_list; use != NULL; use = next_use)
9107	{
9108	next_use = use->next_insn_use;
9109	free (ptr: use);
9110	}
9111	for (set = data->reg_set_list; set != NULL; set = next_set)
9112	{
9113	next_set = set->next_insn_set;
9114	free (ptr: set);
9115	}
9116
9117	}
9118	h_i_d.release ();
9119	}
9120
9121	/ Init data for the new insn INSN. /
9122	static void
9123	haifa_init_insn (rtx_insn *insn)
9124	{
9125	gcc_assert (insn != NULL);
9126
9127	sched_extend_luids ();
9128	sched_init_insn_luid (insn);
9129	sched_extend_target ();
9130	sched_deps_init (false);
9131	extend_h_i_d ();
9132	init_h_i_d (insn);
9133
9134	if (adding_bb_to_current_region_p)
9135	{
9136	sd_init_insn (insn);
9137
9138	/ Extend dependency caches by one element. /
9139	extend_dependency_caches (`1`, false);
9140	}
9141	if (sched_pressure != SCHED_PRESSURE_NONE)
9142	init_insn_reg_pressure_info (insn);
9143	}
9144
9145	/ Init data for the new basic block BB which comes after AFTER. /
9146	static void
9147	haifa_init_only_bb (basic_block bb, basic_block after)
9148	{
9149	gcc_assert (bb != NULL);
9150
9151	sched_init_bbs ();
9152
9153	if (common_sched_info->add_block)
9154	/ This changes only data structures of the front-end. /
9155	common_sched_info->add_block (bb, after);
9156	}
9157
9158	/ A generic version of sched_split_block (). /
9159	basic_block
9160	sched_split_block_1 (basic_block first_bb, rtx after)
9161	{
9162	edge e;
9163
9164	e = split_block (first_bb, after);
9165	gcc_assert (e->src == first_bb);
9166
9167	/ sched_split_block emits note if check == BB_END. Probably it
9168	is better to rip that note off. /*
9169
9170	return e->dest;
9171	}
9172
9173	/ A generic version of sched_create_empty_bb (). /
9174	basic_block
9175	sched_create_empty_bb_1 (basic_block after)
9176	{
9177	return create_empty_bb (after);
9178	}
9179
9180	/ Insert PAT as an INSN into the schedule and update the necessary data*
9181	structures to account for it. /*
9182	rtx_insn *
9183	sched_emit_insn (rtx pat)
9184	{
9185	rtx_insn *insn = emit_insn_before (pat, first_nonscheduled_insn ());
9186	haifa_init_insn (insn);
9187
9188	if (current_sched_info->add_remove_insn)
9189	current_sched_info->add_remove_insn (insn, `0`);
9190
9191	(*current_sched_info->begin_schedule_ready) (insn);
9192	scheduled_insns.safe_push (obj: insn);
9193
9194	last_scheduled_insn = insn;
9195	return insn;
9196	}
9197
9198	/ This function returns a candidate satisfying dispatch constraints from*
9199	the ready list. /*
9200
9201	static rtx_insn *
9202	ready_remove_first_dispatch (struct ready_list *ready)
9203	{
9204	int i;
9205	rtx_insn *insn = ready_element (ready, index: `0`);
9206
9207	if (ready->n_ready == `1`
9208	\|\| !INSN_P (insn)
9209	\|\| INSN_CODE (insn) < `0`
9210	\|\| !active_insn_p (insn)
9211	\|\| targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW))
9212	return ready_remove_first (ready);
9213
9214	for (i = `1`; i < ready->n_ready; i++)
9215	{
9216	insn = ready_element (ready, index: i);
9217
9218	if (!INSN_P (insn)
9219	\|\| INSN_CODE (insn) < `0`
9220	\|\| !active_insn_p (insn))
9221	continue;
9222
9223	if (targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW))
9224	{
9225	/ Return ith element of ready. /
9226	insn = ready_remove (ready, index: i);
9227	return insn;
9228	}
9229	}
9230
9231	if (targetm.sched.dispatch (NULL, DISPATCH_VIOLATION))
9232	return ready_remove_first (ready);
9233
9234	for (i = `1`; i < ready->n_ready; i++)
9235	{
9236	insn = ready_element (ready, index: i);
9237
9238	if (!INSN_P (insn)
9239	\|\| INSN_CODE (insn) < `0`
9240	\|\| !active_insn_p (insn))
9241	continue;
9242
9243	/ Return i-th element of ready. /
9244	if (targetm.sched.dispatch (insn, IS_CMP))
9245	return ready_remove (ready, index: i);
9246	}
9247
9248	return ready_remove_first (ready);
9249	}
9250
9251	/ Get number of ready insn in the ready list. /
9252
9253	int
9254	number_in_ready (void)
9255	{
9256	return ready.n_ready;
9257	}
9258
9259	/ Get number of ready's in the ready list. /
9260
9261	rtx_insn *
9262	get_ready_element (int i)
9263	{
9264	return ready_element (ready: &ready, index: i);
9265	}
9266
9267	#endif /* INSN_SCHEDULING */
9268

source code of gcc/haifa-sched.cc