1	/* Decompose multiword subregs.
2	Copyright (C) 2007-2024 Free Software Foundation, Inc.
3	Contributed by Richard Henderson <rth@redhat.com>
4	Ian Lance Taylor <iant@google.com>
5
6	This file is part of GCC.
7
8	GCC is free software; you can redistribute it and/or modify it under
9	the terms of the GNU General Public License as published by the Free
10	Software Foundation; either version 3, or (at your option) any later
11	version.
12
13	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14	WARRANTY; without even the implied warranty of MERCHANTABILITY or
15	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16	for more details.
17
18	You should have received a copy of the GNU General Public License
19	along with GCC; see the file COPYING3. If not see
20	<http://www.gnu.org/licenses/>. */
21
22	#include "config.h"
23	#include "system.h"
24	#include "coretypes.h"
25	#include "backend.h"
26	#include "rtl.h"
27	#include "tree.h"
28	#include "cfghooks.h"
29	#include "df.h"
30	#include "memmodel.h"
31	#include "tm_p.h"
32	#include "expmed.h"
33	#include "insn-config.h"
34	#include "emit-rtl.h"
35	#include "recog.h"
36	#include "cfgrtl.h"
37	#include "cfgbuild.h"
38	#include "dce.h"
39	#include "expr.h"
40	#include "explow.h"
41	#include "tree-pass.h"
42	#include "lower-subreg.h"
43	#include "rtl-iter.h"
44	#include "target.h"
45
46
47	/* Decompose multi-word pseudo-registers into individual
48	pseudo-registers when possible and profitable. This is possible
49	when all the uses of a multi-word register are via SUBREG, or are
50	copies of the register to another location. Breaking apart the
51	register permits more CSE and permits better register allocation.
52	This is profitable if the machine does not have move instructions
53	to do this.
54
55	This pass only splits moves with modes that are wider than
56	word_mode and ASHIFTs, LSHIFTRTs, ASHIFTRTs and ZERO_EXTENDs with
57	integer modes that are twice the width of word_mode. The latter
58	could be generalized if there was a need to do this, but the trend in
59	architectures is to not need this.
60
61	There are two useful preprocessor defines for use by maintainers:
62
63	#define LOG_COSTS 1
64
65	if you wish to see the actual cost estimates that are being used
66	for each mode wider than word mode and the cost estimates for zero
67	extension and the shifts. This can be useful when port maintainers
68	are tuning insn rtx costs.
69
70	#define FORCE_LOWERING 1
71
72	if you wish to test the pass with all the transformation forced on.
73	This can be useful for finding bugs in the transformations. */
74
75	#define LOG_COSTS 0
76	#define FORCE_LOWERING 0
77
78	/* Bit N in this bitmap is set if regno N is used in a context in
79	which we can decompose it. */
80	static bitmap decomposable_context;
81
82	/* Bit N in this bitmap is set if regno N is used in a context in
83	which it cannot be decomposed. */
84	static bitmap non_decomposable_context;
85
86	/* Bit N in this bitmap is set if regno N is used in a subreg
87	which changes the mode but not the size. This typically happens
88	when the register accessed as a floating-point value; we want to
89	avoid generating accesses to its subwords in integer modes. */
90	static bitmap subreg_context;
91
92	/* Bit N in the bitmap in element M of this array is set if there is a
93	copy from reg M to reg N. */
94	static vec<bitmap> reg_copy_graph;
95
96	struct target_lower_subreg default_target_lower_subreg;
97	#if SWITCHABLE_TARGET
98	struct target_lower_subreg *this_target_lower_subreg
99	= &default_target_lower_subreg;
100	#endif
101
102	#define twice_word_mode \
103	this_target_lower_subreg->x_twice_word_mode
104	#define choices \
105	this_target_lower_subreg->x_choices
106
107	/* Return true if MODE is a mode we know how to lower. When returning true,
108	store its byte size in *BYTES and its word size in *WORDS. */
109
110	static inline bool
111	interesting_mode_p (machine_mode mode, unsigned int *bytes,
112	unsigned int *words)
113	{
114	if (!GET_MODE_SIZE (mode).is_constant (const_value: bytes))
115	return false;
116	*words = CEIL (*bytes, UNITS_PER_WORD);
117	return true;
118	}
119
120	/* RTXes used while computing costs. */
121	struct cost_rtxes {
122	/* Source and target registers. */
123	rtx source;
124	rtx target;
125
126	/* A twice_word_mode ZERO_EXTEND of SOURCE. */
127	rtx zext;
128
129	/* A shift of SOURCE. */
130	rtx shift;
131
132	/* A SET of TARGET. */
133	rtx set;
134	};
135
136	/* Return the cost of a CODE shift in mode MODE by OP1 bits, using the
137	rtxes in RTXES. SPEED_P selects between the speed and size cost. */
138
139	static int
140	shift_cost (bool speed_p, struct cost_rtxes *rtxes, enum rtx_code code,
141	machine_mode mode, int op1)
142	{
143	PUT_CODE (rtxes->shift, code);
144	PUT_MODE (x: rtxes->shift, mode);
145	PUT_MODE (x: rtxes->source, mode);
146	XEXP (rtxes->shift, 1) = gen_int_shift_amount (mode, op1);
147	return set_src_cost (x: rtxes->shift, mode, speed_p);
148	}
149
150	/* For each X in the range [0, BITS_PER_WORD), set SPLITTING[X]
151	to true if it is profitable to split a double-word CODE shift
152	of X + BITS_PER_WORD bits. SPEED_P says whether we are testing
153	for speed or size profitability.
154
155	Use the rtxes in RTXES to calculate costs. WORD_MOVE_ZERO_COST is
156	the cost of moving zero into a word-mode register. WORD_MOVE_COST
157	is the cost of moving between word registers. */
158
159	static void
160	compute_splitting_shift (bool speed_p, struct cost_rtxes *rtxes,
161	bool *splitting, enum rtx_code code,
162	int word_move_zero_cost, int word_move_cost)
163	{
164	int wide_cost, narrow_cost, upper_cost, i;
165
166	for (i = 0; i < BITS_PER_WORD; i++)
167	{
168	wide_cost = shift_cost (speed_p, rtxes, code, twice_word_mode,
169	op1: i + BITS_PER_WORD);
170	if (i == 0)
171	narrow_cost = word_move_cost;
172	else
173	narrow_cost = shift_cost (speed_p, rtxes, code, mode: word_mode, op1: i);
174
175	if (code != ASHIFTRT)
176	upper_cost = word_move_zero_cost;
177	else if (i == BITS_PER_WORD - 1)
178	upper_cost = word_move_cost;
179	else
180	upper_cost = shift_cost (speed_p, rtxes, code, mode: word_mode,
181	BITS_PER_WORD - 1);
182
183	if (LOG_COSTS)
184	fprintf (stderr, format: "%s %s by %d: original cost %d, split cost %d + %d\n",
185	GET_MODE_NAME (twice_word_mode), GET_RTX_NAME (code),
186	i + BITS_PER_WORD, wide_cost, narrow_cost, upper_cost);
187
188	if (FORCE_LOWERING || wide_cost >= narrow_cost + upper_cost)
189	splitting[i] = true;
190	}
191	}
192
193	/* Compute what we should do when optimizing for speed or size; SPEED_P
194	selects which. Use RTXES for computing costs. */
195
196	static void
197	compute_costs (bool speed_p, struct cost_rtxes *rtxes)
198	{
199	unsigned int i;
200	int word_move_zero_cost, word_move_cost;
201
202	PUT_MODE (x: rtxes->target, mode: word_mode);
203	SET_SRC (rtxes->set) = CONST0_RTX (word_mode);
204	word_move_zero_cost = set_rtx_cost (x: rtxes->set, speed_p);
205
206	SET_SRC (rtxes->set) = rtxes->source;
207	word_move_cost = set_rtx_cost (x: rtxes->set, speed_p);
208
209	if (LOG_COSTS)
210	fprintf (stderr, format: "%s move: from zero cost %d, from reg cost %d\n",
211	GET_MODE_NAME (word_mode), word_move_zero_cost, word_move_cost);
212
213	for (i = 0; i < MAX_MACHINE_MODE; i++)
214	{
215	machine_mode mode = (machine_mode) i;
216	unsigned int size, factor;
217	if (interesting_mode_p (mode, bytes: &size, words: &factor) && factor > 1)
218	{
219	unsigned int mode_move_cost;
220
221	PUT_MODE (x: rtxes->target, mode);
222	PUT_MODE (x: rtxes->source, mode);
223	mode_move_cost = set_rtx_cost (x: rtxes->set, speed_p);
224
225	if (LOG_COSTS)
226	fprintf (stderr, format: "%s move: original cost %d, split cost %d * %d\n",
227	GET_MODE_NAME (mode), mode_move_cost,
228	word_move_cost, factor);
229
230	if (FORCE_LOWERING || mode_move_cost >= word_move_cost * factor)
231	{
232	choices[speed_p].move_modes_to_split[i] = true;
233	choices[speed_p].something_to_do = true;
234	}
235	}
236	}
237
238	/* For the moves and shifts, the only case that is checked is one
239	where the mode of the target is an integer mode twice the width
240	of the word_mode.
241
242	If it is not profitable to split a double word move then do not
243	even consider the shifts or the zero extension. */
244	if (choices[speed_p].move_modes_to_split[(int) twice_word_mode])
245	{
246	int zext_cost;
247
248	/* The only case here to check to see if moving the upper part with a
249	zero is cheaper than doing the zext itself. */
250	PUT_MODE (x: rtxes->source, mode: word_mode);
251	zext_cost = set_src_cost (x: rtxes->zext, twice_word_mode, speed_p);
252
253	if (LOG_COSTS)
254	fprintf (stderr, format: "%s %s: original cost %d, split cost %d + %d\n",
255	GET_MODE_NAME (twice_word_mode), GET_RTX_NAME (ZERO_EXTEND),
256	zext_cost, word_move_cost, word_move_zero_cost);
257
258	if (FORCE_LOWERING || zext_cost >= word_move_cost + word_move_zero_cost)
259	choices[speed_p].splitting_zext = true;
260
261	compute_splitting_shift (speed_p, rtxes,
262	choices[speed_p].splitting_ashift, code: ASHIFT,
263	word_move_zero_cost, word_move_cost);
264	compute_splitting_shift (speed_p, rtxes,
265	choices[speed_p].splitting_lshiftrt, code: LSHIFTRT,
266	word_move_zero_cost, word_move_cost);
267	compute_splitting_shift (speed_p, rtxes,
268	choices[speed_p].splitting_ashiftrt, code: ASHIFTRT,
269	word_move_zero_cost, word_move_cost);
270	}
271	}
272
273	/* Do one-per-target initialisation. This involves determining
274	which operations on the machine are profitable. If none are found,
275	then the pass just returns when called. */
276
277	void
278	init_lower_subreg (void)
279	{
280	struct cost_rtxes rtxes;
281
282	memset (s: this_target_lower_subreg, c: 0, n: sizeof (*this_target_lower_subreg));
283
284	twice_word_mode = GET_MODE_2XWIDER_MODE (m: word_mode).require ();
285
286	rtxes.target = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
287	rtxes.source = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 2);
288	rtxes.set = gen_rtx_SET (rtxes.target, rtxes.source);
289	rtxes.zext = gen_rtx_ZERO_EXTEND (twice_word_mode, rtxes.source);
290	rtxes.shift = gen_rtx_ASHIFT (twice_word_mode, rtxes.source, const0_rtx);
291
292	if (LOG_COSTS)
293	fprintf (stderr, format: "\nSize costs\n==========\n\n");
294	compute_costs (speed_p: false, rtxes: &rtxes);
295
296	if (LOG_COSTS)
297	fprintf (stderr, format: "\nSpeed costs\n===========\n\n");
298	compute_costs (speed_p: true, rtxes: &rtxes);
299	}
300
301	static bool
302	simple_move_operand (rtx x)
303	{
304	if (GET_CODE (x) == SUBREG)
305	x = SUBREG_REG (x);
306
307	if (!OBJECT_P (x))
308	return false;
309
310	if (GET_CODE (x) == LABEL_REF
311	|| GET_CODE (x) == SYMBOL_REF
312	|| GET_CODE (x) == HIGH
313	|| GET_CODE (x) == CONST)
314	return false;
315
316	if (MEM_P (x)
317	&& (MEM_VOLATILE_P (x)
318	|| mode_dependent_address_p (XEXP (x, 0), MEM_ADDR_SPACE (x))))
319	return false;
320
321	return true;
322	}
323
324	/* If X is an operator that can be treated as a simple move that we
325	can split, then return the operand that is operated on. */
326
327	static rtx
328	operand_for_swap_move_operator (rtx x)
329	{
330	/* A word sized rotate of a register pair is equivalent to swapping
331	the registers in the register pair. */
332	if (GET_CODE (x) == ROTATE
333	&& GET_MODE (x) == twice_word_mode
334	&& simple_move_operand (XEXP (x, 0))
335	&& CONST_INT_P (XEXP (x, 1))
336	&& INTVAL (XEXP (x, 1)) == BITS_PER_WORD)
337	return XEXP (x, 0);
338
339	return NULL_RTX;
340	}
341
342	/* If INSN is a single set between two objects that we want to split,
343	return the single set. SPEED_P says whether we are optimizing
344	INSN for speed or size.
345
346	INSN should have been passed to recog and extract_insn before this
347	is called. */
348
349	static rtx
350	simple_move (rtx_insn *insn, bool speed_p)
351	{
352	rtx x, op;
353	rtx set;
354	machine_mode mode;
355
356	if (recog_data.n_operands != 2)
357	return NULL_RTX;
358
359	set = single_set (insn);
360	if (!set)
361	return NULL_RTX;
362
363	x = SET_DEST (set);
364	if (x != recog_data.operand[0] && x != recog_data.operand[1])
365	return NULL_RTX;
366	if (!simple_move_operand (x))
367	return NULL_RTX;
368
369	x = SET_SRC (set);
370	if ((op = operand_for_swap_move_operator (x)) != NULL_RTX)
371	x = op;
372
373	if (x != recog_data.operand[0] && x != recog_data.operand[1])
374	return NULL_RTX;
375	/* For the src we can handle ASM_OPERANDS, and it is beneficial for
376	things like x86 rdtsc which returns a DImode value. */
377	if (GET_CODE (x) != ASM_OPERANDS
378	&& !simple_move_operand (x))
379	return NULL_RTX;
380
381	/* We try to decompose in integer modes, to avoid generating
382	inefficient code copying between integer and floating point
383	registers. That means that we can't decompose if this is a
384	non-integer mode for which there is no integer mode of the same
385	size. */
386	mode = GET_MODE (SET_DEST (set));
387	scalar_int_mode int_mode;
388	if (!SCALAR_INT_MODE_P (mode)
389	&& (!int_mode_for_size (size: GET_MODE_BITSIZE (mode), limit: 0).exists (mode: &int_mode)
390	|| !targetm.modes_tieable_p (mode, int_mode)))
391	return NULL_RTX;
392
393	/* Reject PARTIAL_INT modes. They are used for processor specific
394	purposes and it's probably best not to tamper with them. */
395	if (GET_MODE_CLASS (mode) == MODE_PARTIAL_INT)
396	return NULL_RTX;
397
398	if (!choices[speed_p].move_modes_to_split[(int) mode])
399	return NULL_RTX;
400
401	return set;
402	}
403
404	/* If SET is a copy from one multi-word pseudo-register to another,
405	record that in reg_copy_graph. Return whether it is such a
406	copy. */
407
408	static bool
409	find_pseudo_copy (rtx set)
410	{
411	rtx dest = SET_DEST (set);
412	rtx src = SET_SRC (set);
413	rtx op;
414	unsigned int rd, rs;
415	bitmap b;
416
417	if ((op = operand_for_swap_move_operator (x: src)) != NULL_RTX)
418	src = op;
419
420	if (!REG_P (dest) || !REG_P (src))
421	return false;
422
423	rd = REGNO (dest);
424	rs = REGNO (src);
425	if (HARD_REGISTER_NUM_P (rd) || HARD_REGISTER_NUM_P (rs))
426	return false;
427
428	b = reg_copy_graph[rs];
429	if (b == NULL)
430	{
431	b = BITMAP_ALLOC (NULL);
432	reg_copy_graph[rs] = b;
433	}
434
435	bitmap_set_bit (b, rd);
436
437	return true;
438	}
439
440	/* Look through the registers in DECOMPOSABLE_CONTEXT. For each case
441	where they are copied to another register, add the register to
442	which they are copied to DECOMPOSABLE_CONTEXT. Use
443	NON_DECOMPOSABLE_CONTEXT to limit this--we don't bother to track
444	copies of registers which are in NON_DECOMPOSABLE_CONTEXT. */
445
446	static void
447	propagate_pseudo_copies (void)
448	{
449	auto_bitmap queue, propagate;
450
451	bitmap_copy (queue, decomposable_context);
452	do
453	{
454	bitmap_iterator iter;
455	unsigned int i;
456
457	bitmap_clear (propagate);
458
459	EXECUTE_IF_SET_IN_BITMAP (queue, 0, i, iter)
460	{
461	bitmap b = reg_copy_graph[i];
462	if (b)
463	bitmap_ior_and_compl_into (A: propagate, B: b, C: non_decomposable_context);
464	}
465
466	bitmap_and_compl (queue, propagate, decomposable_context);
467	bitmap_ior_into (decomposable_context, propagate);
468	}
469	while (!bitmap_empty_p (map: queue));
470	}
471
472	/* A pointer to one of these values is passed to
473	find_decomposable_subregs. */
474
475	enum classify_move_insn
476	{
477	/* Not a simple move from one location to another. */
478	NOT_SIMPLE_MOVE,
479	/* A simple move we want to decompose. */
480	DECOMPOSABLE_SIMPLE_MOVE,
481	/* Any other simple move. */
482	SIMPLE_MOVE
483	};
484
485	/* If we find a SUBREG in *LOC which we could use to decompose a
486	pseudo-register, set a bit in DECOMPOSABLE_CONTEXT. If we find an
487	unadorned register which is not a simple pseudo-register copy,
488	DATA will point at the type of move, and we set a bit in
489	DECOMPOSABLE_CONTEXT or NON_DECOMPOSABLE_CONTEXT as appropriate. */
490
491	static void
492	find_decomposable_subregs (rtx *loc, enum classify_move_insn *pcmi)
493	{
494	subrtx_var_iterator::array_type array;
495	FOR_EACH_SUBRTX_VAR (iter, array, *loc, NONCONST)
496	{
497	rtx x = *iter;
498	if (GET_CODE (x) == SUBREG)
499	{
500	rtx inner = SUBREG_REG (x);
501	unsigned int regno, outer_size, inner_size, outer_words, inner_words;
502
503	if (!REG_P (inner))
504	continue;
505
506	regno = REGNO (inner);
507	if (HARD_REGISTER_NUM_P (regno))
508	{
509	iter.skip_subrtxes ();
510	continue;
511	}
512
513	if (!interesting_mode_p (GET_MODE (x), bytes: &outer_size, words: &outer_words)
514	|| !interesting_mode_p (GET_MODE (inner), bytes: &inner_size,
515	words: &inner_words))
516	continue;
517
518	/* We only try to decompose single word subregs of multi-word
519	registers. When we find one, we return -1 to avoid iterating
520	over the inner register.
521
522	??? This doesn't allow, e.g., DImode subregs of TImode values
523	on 32-bit targets. We would need to record the way the
524	pseudo-register was used, and only decompose if all the uses
525	were the same number and size of pieces. Hopefully this
526	doesn't happen much. */
527
528	if (outer_words == 1
529	&& inner_words > 1
530	/* Don't allow to decompose floating point subregs of
531	multi-word pseudos if the floating point mode does
532	not have word size, because otherwise we'd generate
533	a subreg with that floating mode from a different
534	sized integral pseudo which is not allowed by
535	validate_subreg. */
536	&& (!FLOAT_MODE_P (GET_MODE (x))
537	|| outer_size == UNITS_PER_WORD))
538	{
539	bitmap_set_bit (decomposable_context, regno);
540	iter.skip_subrtxes ();
541	continue;
542	}
543
544	/* If this is a cast from one mode to another, where the modes
545	have the same size, and they are not tieable, then mark this
546	register as non-decomposable. If we decompose it we are
547	likely to mess up whatever the backend is trying to do. */
548	if (outer_words > 1
549	&& outer_size == inner_size
550	&& !targetm.modes_tieable_p (GET_MODE (x), GET_MODE (inner)))
551	{
552	bitmap_set_bit (non_decomposable_context, regno);
553	bitmap_set_bit (subreg_context, regno);
554	iter.skip_subrtxes ();
555	continue;
556	}
557	}
558	else if (REG_P (x))
559	{
560	unsigned int regno, size, words;
561
562	/* We will see an outer SUBREG before we see the inner REG, so
563	when we see a plain REG here it means a direct reference to
564	the register.
565
566	If this is not a simple copy from one location to another,
567	then we cannot decompose this register. If this is a simple
568	copy we want to decompose, and the mode is right,
569	then we mark the register as decomposable.
570	Otherwise we don't say anything about this register --
571	it could be decomposed, but whether that would be
572	profitable depends upon how it is used elsewhere.
573
574	We only set bits in the bitmap for multi-word
575	pseudo-registers, since those are the only ones we care about
576	and it keeps the size of the bitmaps down. */
577
578	regno = REGNO (x);
579	if (!HARD_REGISTER_NUM_P (regno)
580	&& interesting_mode_p (GET_MODE (x), bytes: &size, words: &words)
581	&& words > 1)
582	{
583	switch (*pcmi)
584	{
585	case NOT_SIMPLE_MOVE:
586	bitmap_set_bit (non_decomposable_context, regno);
587	break;
588	case DECOMPOSABLE_SIMPLE_MOVE:
589	if (targetm.modes_tieable_p (GET_MODE (x), word_mode))
590	bitmap_set_bit (decomposable_context, regno);
591	break;
592	case SIMPLE_MOVE:
593	break;
594	default:
595	gcc_unreachable ();
596	}
597	}
598	}
599	else if (MEM_P (x))
600	{
601	enum classify_move_insn cmi_mem = NOT_SIMPLE_MOVE;
602
603	/* Any registers used in a MEM do not participate in a
604	SIMPLE_MOVE or DECOMPOSABLE_SIMPLE_MOVE. Do our own recursion
605	here, and return -1 to block the parent's recursion. */
606	find_decomposable_subregs (loc: &XEXP (x, 0), pcmi: &cmi_mem);
607	iter.skip_subrtxes ();
608	}
609	}
610	}
611
612	/* Decompose REGNO into word-sized components. We smash the REG node
613	in place. This ensures that (1) something goes wrong quickly if we
614	fail to make some replacement, and (2) the debug information inside
615	the symbol table is automatically kept up to date. */
616
617	static void
618	decompose_register (unsigned int regno)
619	{
620	rtx reg;
621	unsigned int size, words, i;
622	rtvec v;
623
624	reg = regno_reg_rtx[regno];
625
626	regno_reg_rtx[regno] = NULL_RTX;
627
628	if (!interesting_mode_p (GET_MODE (reg), bytes: &size, words: &words))
629	gcc_unreachable ();
630
631	v = rtvec_alloc (words);
632	for (i = 0; i < words; ++i)
633	RTVEC_ELT (v, i) = gen_reg_rtx_offset (reg, word_mode, i * UNITS_PER_WORD);
634
635	PUT_CODE (reg, CONCATN);
636	XVEC (reg, 0) = v;
637
638	if (dump_file)
639	{
640	fprintf (stream: dump_file, format: "; Splitting reg %u ->", regno);
641	for (i = 0; i < words; ++i)
642	fprintf (stream: dump_file, format: " %u", REGNO (XVECEXP (reg, 0, i)));
643	fputc (c: '\n', stream: dump_file);
644	}
645	}
646
647	/* Get a SUBREG of a CONCATN. */
648
649	static rtx
650	simplify_subreg_concatn (machine_mode outermode, rtx op, poly_uint64 orig_byte)
651	{
652	unsigned int outer_size, outer_words, inner_size, inner_words;
653	machine_mode innermode, partmode;
654	rtx part;
655	unsigned int final_offset;
656	unsigned int byte;
657
658	innermode = GET_MODE (op);
659	if (!interesting_mode_p (mode: outermode, bytes: &outer_size, words: &outer_words)
660	|| !interesting_mode_p (mode: innermode, bytes: &inner_size, words: &inner_words))
661	gcc_unreachable ();
662
663	/* Must be constant if interesting_mode_p passes. */
664	byte = orig_byte.to_constant ();
665	gcc_assert (GET_CODE (op) == CONCATN);
666	gcc_assert (byte % outer_size == 0);
667
668	gcc_assert (byte < inner_size);
669	if (outer_size > inner_size)
670	return NULL_RTX;
671
672	inner_size /= XVECLEN (op, 0);
673	part = XVECEXP (op, 0, byte / inner_size);
674	partmode = GET_MODE (part);
675
676	final_offset = byte % inner_size;
677	if (final_offset + outer_size > inner_size)
678	return NULL_RTX;
679
680	/* VECTOR_CSTs in debug expressions are expanded into CONCATN instead of
681	regular CONST_VECTORs. They have vector or integer modes, depending
682	on the capabilities of the target. Cope with them. */
683	if (partmode == VOIDmode && VECTOR_MODE_P (innermode))
684	partmode = GET_MODE_INNER (innermode);
685	else if (partmode == VOIDmode)
686	partmode = mode_for_size (inner_size * BITS_PER_UNIT,
687	GET_MODE_CLASS (innermode), 0).require ();
688
689	return simplify_gen_subreg (outermode, op: part, innermode: partmode, byte: final_offset);
690	}
691
692	/* Wrapper around simplify_gen_subreg which handles CONCATN. */
693
694	static rtx
695	simplify_gen_subreg_concatn (machine_mode outermode, rtx op,
696	machine_mode innermode, unsigned int byte)
697	{
698	rtx ret;
699
700	/* We have to handle generating a SUBREG of a SUBREG of a CONCATN.
701	If OP is a SUBREG of a CONCATN, then it must be a simple mode
702	change with the same size and offset 0, or it must extract a
703	part. We shouldn't see anything else here. */
704	if (GET_CODE (op) == SUBREG && GET_CODE (SUBREG_REG (op)) == CONCATN)
705	{
706	rtx op2;
707
708	if (known_eq (GET_MODE_SIZE (GET_MODE (op)),
709	GET_MODE_SIZE (GET_MODE (SUBREG_REG (op))))
710	&& known_eq (SUBREG_BYTE (op), 0))
711	return simplify_gen_subreg_concatn (outermode, SUBREG_REG (op),
712	GET_MODE (SUBREG_REG (op)), byte);
713
714	op2 = simplify_subreg_concatn (GET_MODE (op), SUBREG_REG (op),
715	SUBREG_BYTE (op));
716	if (op2 == NULL_RTX)
717	{
718	/* We don't handle paradoxical subregs here. */
719	gcc_assert (!paradoxical_subreg_p (outermode, GET_MODE (op)));
720	gcc_assert (!paradoxical_subreg_p (op));
721	op2 = simplify_subreg_concatn (outermode, SUBREG_REG (op),
722	orig_byte: byte + SUBREG_BYTE (op));
723	gcc_assert (op2 != NULL_RTX);
724	return op2;
725	}
726
727	op = op2;
728	gcc_assert (op != NULL_RTX);
729	gcc_assert (innermode == GET_MODE (op));
730	}
731
732	if (GET_CODE (op) == CONCATN)
733	return simplify_subreg_concatn (outermode, op, orig_byte: byte);
734
735	ret = simplify_gen_subreg (outermode, op, innermode, byte);
736
737	/* If we see an insn like (set (reg:DI) (subreg:DI (reg:SI) 0)) then
738	resolve_simple_move will ask for the high part of the paradoxical
739	subreg, which does not have a value. Just return a zero. */
740	if (ret == NULL_RTX
741	&& paradoxical_subreg_p (x: op))
742	return CONST0_RTX (outermode);
743
744	gcc_assert (ret != NULL_RTX);
745	return ret;
746	}
747
748	/* Return whether we should resolve X into the registers into which it
749	was decomposed. */
750
751	static bool
752	resolve_reg_p (rtx x)
753	{
754	return GET_CODE (x) == CONCATN;
755	}
756
757	/* Return whether X is a SUBREG of a register which we need to
758	resolve. */
759
760	static bool
761	resolve_subreg_p (rtx x)
762	{
763	if (GET_CODE (x) != SUBREG)
764	return false;
765	return resolve_reg_p (SUBREG_REG (x));
766	}
767
768	/* Look for SUBREGs in *LOC which need to be decomposed. */
769
770	static bool
771	resolve_subreg_use (rtx *loc, rtx insn)
772	{
773	subrtx_ptr_iterator::array_type array;
774	FOR_EACH_SUBRTX_PTR (iter, array, loc, NONCONST)
775	{
776	rtx *loc = *iter;
777	rtx x = *loc;
778	if (resolve_subreg_p (x))
779	{
780	x = simplify_subreg_concatn (GET_MODE (x), SUBREG_REG (x),
781	SUBREG_BYTE (x));
782
783	/* It is possible for a note to contain a reference which we can
784	decompose. In this case, return 1 to the caller to indicate
785	that the note must be removed. */
786	if (!x)
787	{
788	gcc_assert (!insn);
789	return true;
790	}
791
792	validate_change (insn, loc, x, 1);
793	iter.skip_subrtxes ();
794	}
795	else if (resolve_reg_p (x))
796	/* Return 1 to the caller to indicate that we found a direct
797	reference to a register which is being decomposed. This can
798	happen inside notes, multiword shift or zero-extend
799	instructions. */
800	return true;
801	}
802
803	return false;
804	}
805
806	/* Resolve any decomposed registers which appear in register notes on
807	INSN. */
808
809	static void
810	resolve_reg_notes (rtx_insn *insn)
811	{
812	rtx *pnote, note;
813
814	note = find_reg_equal_equiv_note (insn);
815	if (note)
816	{
817	int old_count = num_validated_changes ();
818	if (resolve_subreg_use (loc: &XEXP (note, 0), NULL_RTX))
819	remove_note (insn, note);
820	else
821	if (old_count != num_validated_changes ())
822	df_notes_rescan (insn);
823	}
824
825	pnote = &REG_NOTES (insn);
826	while (*pnote != NULL_RTX)
827	{
828	bool del = false;
829
830	note = *pnote;
831	switch (REG_NOTE_KIND (note))
832	{
833	case REG_DEAD:
834	case REG_UNUSED:
835	if (resolve_reg_p (XEXP (note, 0)))
836	del = true;
837	break;
838
839	default:
840	break;
841	}
842
843	if (del)
844	*pnote = XEXP (note, 1);
845	else
846	pnote = &XEXP (note, 1);
847	}
848	}
849
850	/* Return whether X can be decomposed into subwords. */
851
852	static bool
853	can_decompose_p (rtx x)
854	{
855	if (REG_P (x))
856	{
857	unsigned int regno = REGNO (x);
858
859	if (HARD_REGISTER_NUM_P (regno))
860	{
861	unsigned int byte, num_bytes, num_words;
862
863	if (!interesting_mode_p (GET_MODE (x), bytes: &num_bytes, words: &num_words))
864	return false;
865	for (byte = 0; byte < num_bytes; byte += UNITS_PER_WORD)
866	if (simplify_subreg_regno (regno, GET_MODE (x), byte, word_mode) < 0)
867	return false;
868	return true;
869	}
870	else
871	return !bitmap_bit_p (subreg_context, regno);
872	}
873
874	return true;
875	}
876
877	/* OPND is a concatn operand this is used with a simple move operator.
878	Return a new rtx with the concatn's operands swapped. */
879
880	static rtx
881	resolve_operand_for_swap_move_operator (rtx opnd)
882	{
883	gcc_assert (GET_CODE (opnd) == CONCATN);
884	rtx concatn = copy_rtx (opnd);
885	rtx op0 = XVECEXP (concatn, 0, 0);
886	rtx op1 = XVECEXP (concatn, 0, 1);
887	XVECEXP (concatn, 0, 0) = op1;
888	XVECEXP (concatn, 0, 1) = op0;
889	return concatn;
890	}
891
892	/* Decompose the registers used in a simple move SET within INSN. If
893	we don't change anything, return INSN, otherwise return the start
894	of the sequence of moves. */
895
896	static rtx_insn *
897	resolve_simple_move (rtx set, rtx_insn *insn)
898	{
899	rtx src, dest, real_dest, src_op;
900	rtx_insn *insns;
901	machine_mode orig_mode, dest_mode;
902	unsigned int orig_size, words;
903	bool pushing;
904
905	src = SET_SRC (set);
906	dest = SET_DEST (set);
907	orig_mode = GET_MODE (dest);
908
909	if (!interesting_mode_p (mode: orig_mode, bytes: &orig_size, words: &words))
910	gcc_unreachable ();
911	gcc_assert (words > 1);
912
913	start_sequence ();
914
915	/* We have to handle copying from a SUBREG of a decomposed reg where
916	the SUBREG is larger than word size. Rather than assume that we
917	can take a word_mode SUBREG of the destination, we copy to a new
918	register and then copy that to the destination. */
919
920	real_dest = NULL_RTX;
921
922	if ((src_op = operand_for_swap_move_operator (x: src)) != NULL_RTX)
923	{
924	if (resolve_reg_p (x: dest))
925	{
926	/* DEST is a CONCATN, so swap its operands and strip
927	SRC's operator. */
928	dest = resolve_operand_for_swap_move_operator (opnd: dest);
929	src = src_op;
930	if (resolve_reg_p (x: src))
931	{
932	gcc_assert (GET_CODE (src) == CONCATN);
933	if (reg_overlap_mentioned_p (XVECEXP (dest, 0, 0),
934	XVECEXP (src, 0, 1)))
935	{
936	/* If there is overlap between the first half of the
937	destination and what will be stored to the second one,
938	use a temporary pseudo. See PR114211. */
939	rtx tem = gen_reg_rtx (GET_MODE (XVECEXP (src, 0, 1)));
940	emit_move_insn (tem, XVECEXP (src, 0, 1));
941	src = copy_rtx (src);
942	XVECEXP (src, 0, 1) = tem;
943	}
944	}
945	}
946	else if (resolve_reg_p (x: src_op))
947	{
948	/* SRC is an operation on a CONCATN, so strip the operator and
949	swap the CONCATN's operands. */
950	src = resolve_operand_for_swap_move_operator (opnd: src_op);
951	}
952	}
953
954	if (GET_CODE (src) == SUBREG
955	&& resolve_reg_p (SUBREG_REG (src))
956	&& (maybe_ne (SUBREG_BYTE (src), b: 0)
957	|| maybe_ne (a: orig_size, b: GET_MODE_SIZE (GET_MODE (SUBREG_REG (src))))))
958	{
959	real_dest = dest;
960	dest = gen_reg_rtx (orig_mode);
961	if (REG_P (real_dest))
962	REG_ATTRS (dest) = REG_ATTRS (real_dest);
963	}
964
965	/* Similarly if we are copying to a SUBREG of a decomposed reg where
966	the SUBREG is larger than word size. */
967
968	if (GET_CODE (dest) == SUBREG
969	&& resolve_reg_p (SUBREG_REG (dest))
970	&& (maybe_ne (SUBREG_BYTE (dest), b: 0)
971	|| maybe_ne (a: orig_size,
972	b: GET_MODE_SIZE (GET_MODE (SUBREG_REG (dest))))))
973	{
974	rtx reg, smove;
975	rtx_insn *minsn;
976
977	reg = gen_reg_rtx (orig_mode);
978	minsn = emit_move_insn (reg, src);
979	smove = single_set (insn: minsn);
980	gcc_assert (smove != NULL_RTX);
981	resolve_simple_move (set: smove, insn: minsn);
982	src = reg;
983	}
984
985	/* If we didn't have any big SUBREGS of decomposed registers, and
986	neither side of the move is a register we are decomposing, then
987	we don't have to do anything here. */
988
989	if (src == SET_SRC (set)
990	&& dest == SET_DEST (set)
991	&& !resolve_reg_p (x: src)
992	&& !resolve_subreg_p (x: src)
993	&& !resolve_reg_p (x: dest)
994	&& !resolve_subreg_p (x: dest))
995	{
996	end_sequence ();
997	return insn;
998	}
999
1000	/* It's possible for the code to use a subreg of a decomposed
1001	register while forming an address. We need to handle that before
1002	passing the address to emit_move_insn. We pass NULL_RTX as the
1003	insn parameter to resolve_subreg_use because we cannot validate
1004	the insn yet. */
1005	if (MEM_P (src) || MEM_P (dest))
1006	{
1007	int acg;
1008
1009	if (MEM_P (src))
1010	resolve_subreg_use (loc: &XEXP (src, 0), NULL_RTX);
1011	if (MEM_P (dest))
1012	resolve_subreg_use (loc: &XEXP (dest, 0), NULL_RTX);
1013	acg = apply_change_group ();
1014	gcc_assert (acg);
1015	}
1016
1017	/* If SRC is a register which we can't decompose, or has side
1018	effects, we need to move via a temporary register. */
1019
1020	if (!can_decompose_p (x: src)
1021	|| side_effects_p (src)
1022	|| GET_CODE (src) == ASM_OPERANDS)
1023	{
1024	rtx reg;
1025
1026	reg = gen_reg_rtx (orig_mode);
1027
1028	if (AUTO_INC_DEC)
1029	{
1030	rtx_insn *move = emit_move_insn (reg, src);
1031	if (MEM_P (src))
1032	{
1033	rtx note = find_reg_note (insn, REG_INC, NULL_RTX);
1034	if (note)
1035	add_reg_note (move, REG_INC, XEXP (note, 0));
1036	}
1037	}
1038	else
1039	emit_move_insn (reg, src);
1040
1041	src = reg;
1042	}
1043
1044	/* If DEST is a register which we can't decompose, or has side
1045	effects, we need to first move to a temporary register. We
1046	handle the common case of pushing an operand directly. We also
1047	go through a temporary register if it holds a floating point
1048	value. This gives us better code on systems which can't move
1049	data easily between integer and floating point registers. */
1050
1051	dest_mode = orig_mode;
1052	pushing = push_operand (dest, dest_mode);
1053	if (!can_decompose_p (x: dest)
1054	|| (side_effects_p (dest) && !pushing)
1055	|| (!SCALAR_INT_MODE_P (dest_mode)
1056	&& !resolve_reg_p (x: dest)
1057	&& !resolve_subreg_p (x: dest)))
1058	{
1059	if (real_dest == NULL_RTX)
1060	real_dest = dest;
1061	if (!SCALAR_INT_MODE_P (dest_mode))
1062	dest_mode = int_mode_for_mode (dest_mode).require ();
1063	dest = gen_reg_rtx (dest_mode);
1064	if (REG_P (real_dest))
1065	REG_ATTRS (dest) = REG_ATTRS (real_dest);
1066	}
1067
1068	if (pushing)
1069	{
1070	unsigned int i, j, jinc;
1071
1072	gcc_assert (orig_size % UNITS_PER_WORD == 0);
1073	gcc_assert (GET_CODE (XEXP (dest, 0)) != PRE_MODIFY);
1074	gcc_assert (GET_CODE (XEXP (dest, 0)) != POST_MODIFY);
1075
1076	if (WORDS_BIG_ENDIAN == STACK_GROWS_DOWNWARD)
1077	{
1078	j = 0;
1079	jinc = 1;
1080	}
1081	else
1082	{
1083	j = words - 1;
1084	jinc = -1;
1085	}
1086
1087	for (i = 0; i < words; ++i, j += jinc)
1088	{
1089	rtx temp;
1090
1091	temp = copy_rtx (XEXP (dest, 0));
1092	temp = adjust_automodify_address_nv (dest, word_mode, temp,
1093	j * UNITS_PER_WORD);
1094	emit_move_insn (temp,
1095	simplify_gen_subreg_concatn (outermode: word_mode, op: src,
1096	innermode: orig_mode,
1097	byte: j * UNITS_PER_WORD));
1098	}
1099	}
1100	else
1101	{
1102	unsigned int i;
1103
1104	for (i = 0; i < words; ++i)
1105	{
1106	rtx t = simplify_gen_subreg_concatn (outermode: word_mode, op: dest,
1107	innermode: dest_mode,
1108	byte: i * UNITS_PER_WORD);
1109	/* simplify_gen_subreg_concatn can return (const_int 0) for
1110	some sub-objects of paradoxical subregs. As a source operand,
1111	that's fine. As a destination it must be avoided. Those are
1112	supposed to be don't care bits, so we can just drop that store
1113	on the floor. */
1114	if (t != CONST0_RTX (word_mode))
1115	emit_move_insn (t,
1116	simplify_gen_subreg_concatn (outermode: word_mode, op: src,
1117	innermode: orig_mode,
1118	byte: i * UNITS_PER_WORD));
1119	}
1120	}
1121
1122	if (real_dest != NULL_RTX)
1123	{
1124	rtx mdest, smove;
1125	rtx_insn *minsn;
1126
1127	if (dest_mode == orig_mode)
1128	mdest = dest;
1129	else
1130	mdest = simplify_gen_subreg (outermode: orig_mode, op: dest, GET_MODE (dest), byte: 0);
1131	minsn = emit_move_insn (real_dest, mdest);
1132
1133	if (AUTO_INC_DEC && MEM_P (real_dest)
1134	&& !(resolve_reg_p (x: real_dest) || resolve_subreg_p (x: real_dest)))
1135	{
1136	rtx note = find_reg_note (insn, REG_INC, NULL_RTX);
1137	if (note)
1138	add_reg_note (minsn, REG_INC, XEXP (note, 0));
1139	}
1140
1141	smove = single_set (insn: minsn);
1142	gcc_assert (smove != NULL_RTX);
1143
1144	resolve_simple_move (set: smove, insn: minsn);
1145	}
1146
1147	insns = get_insns ();
1148	end_sequence ();
1149
1150	copy_reg_eh_region_note_forward (insn, insns, NULL_RTX);
1151
1152	emit_insn_before (insns, insn);
1153
1154	/* If we get here via self-recursion, then INSN is not yet in the insns
1155	chain and delete_insn will fail. We only want to remove INSN from the
1156	current sequence. See PR56738. */
1157	if (in_sequence_p ())
1158	remove_insn (insn);
1159	else
1160	delete_insn (insn);
1161
1162	return insns;
1163	}
1164
1165	/* Change a CLOBBER of a decomposed register into a CLOBBER of the
1166	component registers. Return whether we changed something. */
1167
1168	static bool
1169	resolve_clobber (rtx pat, rtx_insn *insn)
1170	{
1171	rtx reg;
1172	machine_mode orig_mode;
1173	unsigned int orig_size, words, i;
1174	int ret;
1175
1176	reg = XEXP (pat, 0);
1177	/* For clobbers we can look through paradoxical subregs which
1178	we do not handle in simplify_gen_subreg_concatn. */
1179	if (paradoxical_subreg_p (x: reg))
1180	reg = SUBREG_REG (reg);
1181	if (!resolve_reg_p (x: reg) && !resolve_subreg_p (x: reg))
1182	return false;
1183
1184	orig_mode = GET_MODE (reg);
1185	if (!interesting_mode_p (mode: orig_mode, bytes: &orig_size, words: &words))
1186	gcc_unreachable ();
1187
1188	ret = validate_change (NULL_RTX, &XEXP (pat, 0),
1189	simplify_gen_subreg_concatn (outermode: word_mode, op: reg,
1190	innermode: orig_mode, byte: 0),
1191	0);
1192	df_insn_rescan (insn);
1193	gcc_assert (ret != 0);
1194
1195	for (i = words - 1; i > 0; --i)
1196	{
1197	rtx x;
1198
1199	x = simplify_gen_subreg_concatn (outermode: word_mode, op: reg, innermode: orig_mode,
1200	byte: i * UNITS_PER_WORD);
1201	x = gen_rtx_CLOBBER (VOIDmode, x);
1202	emit_insn_after (x, insn);
1203	}
1204
1205	resolve_reg_notes (insn);
1206
1207	return true;
1208	}
1209
1210	/* A USE of a decomposed register is no longer meaningful. Return
1211	whether we changed something. */
1212
1213	static bool
1214	resolve_use (rtx pat, rtx_insn *insn)
1215	{
1216	if (resolve_reg_p (XEXP (pat, 0)) || resolve_subreg_p (XEXP (pat, 0)))
1217	{
1218	delete_insn (insn);
1219	return true;
1220	}
1221
1222	resolve_reg_notes (insn);
1223
1224	return false;
1225	}
1226
1227	/* A VAR_LOCATION can be simplified. */
1228
1229	static void
1230	resolve_debug (rtx_insn *insn)
1231	{
1232	subrtx_ptr_iterator::array_type array;
1233	FOR_EACH_SUBRTX_PTR (iter, array, &PATTERN (insn), NONCONST)
1234	{
1235	rtx *loc = *iter;
1236	rtx x = *loc;
1237	if (resolve_subreg_p (x))
1238	{
1239	x = simplify_subreg_concatn (GET_MODE (x), SUBREG_REG (x),
1240	SUBREG_BYTE (x));
1241
1242	if (x)
1243	*loc = x;
1244	else
1245	x = copy_rtx (*loc);
1246	}
1247	if (resolve_reg_p (x))
1248	*loc = copy_rtx (x);
1249	}
1250
1251	df_insn_rescan (insn);
1252
1253	resolve_reg_notes (insn);
1254	}
1255
1256	/* Check if INSN is a decomposable multiword-shift or zero-extend and
1257	set the decomposable_context bitmap accordingly. SPEED_P is true
1258	if we are optimizing INSN for speed rather than size. Return true
1259	if INSN is decomposable. */
1260
1261	static bool
1262	find_decomposable_shift_zext (rtx_insn *insn, bool speed_p)
1263	{
1264	rtx set;
1265	rtx op;
1266	rtx op_operand;
1267
1268	set = single_set (insn);
1269	if (!set)
1270	return false;
1271
1272	op = SET_SRC (set);
1273	if (GET_CODE (op) != ASHIFT
1274	&& GET_CODE (op) != LSHIFTRT
1275	&& GET_CODE (op) != ASHIFTRT
1276	&& GET_CODE (op) != ZERO_EXTEND)
1277	return false;
1278
1279	op_operand = XEXP (op, 0);
1280	if (!REG_P (SET_DEST (set)) || !REG_P (op_operand)
1281	|| HARD_REGISTER_NUM_P (REGNO (SET_DEST (set)))
1282	|| HARD_REGISTER_NUM_P (REGNO (op_operand))
1283	|| GET_MODE (op) != twice_word_mode)
1284	return false;
1285
1286	if (GET_CODE (op) == ZERO_EXTEND)
1287	{
1288	if (GET_MODE (op_operand) != word_mode
1289	|| !choices[speed_p].splitting_zext)
1290	return false;
1291	}
1292	else /* left or right shift */
1293	{
1294	bool *splitting = (GET_CODE (op) == ASHIFT
1295	? choices[speed_p].splitting_ashift
1296	: GET_CODE (op) == ASHIFTRT
1297	? choices[speed_p].splitting_ashiftrt
1298	: choices[speed_p].splitting_lshiftrt);
1299	if (!CONST_INT_P (XEXP (op, 1))
1300	|| !IN_RANGE (INTVAL (XEXP (op, 1)), BITS_PER_WORD,
1301	2 * BITS_PER_WORD - 1)
1302	|| !splitting[INTVAL (XEXP (op, 1)) - BITS_PER_WORD])
1303	return false;
1304
1305	bitmap_set_bit (decomposable_context, REGNO (op_operand));
1306	}
1307
1308	bitmap_set_bit (decomposable_context, REGNO (SET_DEST (set)));
1309
1310	return true;
1311	}
1312
1313	/* Decompose a more than word wide shift (in INSN) of a multiword
1314	pseudo or a multiword zero-extend of a wordmode pseudo into a move
1315	and 'set to zero' insn. SPEED_P says whether we are optimizing
1316	for speed or size, when checking if a ZERO_EXTEND is preferable.
1317	Return a pointer to the new insn when a replacement was done. */
1318
1319	static rtx_insn *
1320	resolve_shift_zext (rtx_insn *insn, bool speed_p)
1321	{
1322	rtx set;
1323	rtx op;
1324	rtx op_operand;
1325	rtx_insn *insns;
1326	rtx src_reg, dest_reg, dest_upper, upper_src = NULL_RTX;
1327	int src_reg_num, dest_reg_num, offset1, offset2, src_offset;
1328	scalar_int_mode inner_mode;
1329
1330	set = single_set (insn);
1331	if (!set)
1332	return NULL;
1333
1334	op = SET_SRC (set);
1335	if (GET_CODE (op) != ASHIFT
1336	&& GET_CODE (op) != LSHIFTRT
1337	&& GET_CODE (op) != ASHIFTRT
1338	&& GET_CODE (op) != ZERO_EXTEND)
1339	return NULL;
1340
1341	op_operand = XEXP (op, 0);
1342	if (!is_a <scalar_int_mode> (GET_MODE (op_operand), result: &inner_mode))
1343	return NULL;
1344
1345	/* We can tear this operation apart only if the regs were already
1346	torn apart. */
1347	if (!resolve_reg_p (SET_DEST (set)) && !resolve_reg_p (x: op_operand))
1348	return NULL;
1349
1350	/* src_reg_num is the number of the word mode register which we
1351	are operating on. For a left shift and a zero_extend on little
1352	endian machines this is register 0. */
1353	src_reg_num = (GET_CODE (op) == LSHIFTRT || GET_CODE (op) == ASHIFTRT)
1354	? 1 : 0;
1355
1356	if (WORDS_BIG_ENDIAN && GET_MODE_SIZE (mode: inner_mode) > UNITS_PER_WORD)
1357	src_reg_num = 1 - src_reg_num;
1358
1359	if (GET_CODE (op) == ZERO_EXTEND)
1360	dest_reg_num = WORDS_BIG_ENDIAN ? 1 : 0;
1361	else
1362	dest_reg_num = 1 - src_reg_num;
1363
1364	offset1 = UNITS_PER_WORD * dest_reg_num;
1365	offset2 = UNITS_PER_WORD * (1 - dest_reg_num);
1366	src_offset = UNITS_PER_WORD * src_reg_num;
1367
1368	start_sequence ();
1369
1370	dest_reg = simplify_gen_subreg_concatn (outermode: word_mode, SET_DEST (set),
1371	GET_MODE (SET_DEST (set)),
1372	byte: offset1);
1373	dest_upper = simplify_gen_subreg_concatn (outermode: word_mode, SET_DEST (set),
1374	GET_MODE (SET_DEST (set)),
1375	byte: offset2);
1376	src_reg = simplify_gen_subreg_concatn (outermode: word_mode, op: op_operand,
1377	GET_MODE (op_operand),
1378	byte: src_offset);
1379	if (GET_CODE (op) == ASHIFTRT
1380	&& INTVAL (XEXP (op, 1)) != 2 * BITS_PER_WORD - 1)
1381	upper_src = expand_shift (RSHIFT_EXPR, word_mode, copy_rtx (src_reg),
1382	BITS_PER_WORD - 1, NULL_RTX, 0);
1383
1384	if (GET_CODE (op) != ZERO_EXTEND)
1385	{
1386	int shift_count = INTVAL (XEXP (op, 1));
1387	if (shift_count > BITS_PER_WORD)
1388	src_reg = expand_shift (GET_CODE (op) == ASHIFT ?
1389	LSHIFT_EXPR : RSHIFT_EXPR,
1390	word_mode, src_reg,
1391	shift_count - BITS_PER_WORD,
1392	dest_reg, GET_CODE (op) != ASHIFTRT);
1393	}
1394
1395	/* Consider using ZERO_EXTEND instead of setting DEST_UPPER to zero
1396	if this is considered reasonable. */
1397	if (GET_CODE (op) == LSHIFTRT
1398	&& GET_MODE (op) == twice_word_mode
1399	&& REG_P (SET_DEST (set))
1400	&& !choices[speed_p].splitting_zext)
1401	{
1402	rtx tmp = force_reg (word_mode, copy_rtx (src_reg));
1403	tmp = simplify_gen_unary (code: ZERO_EXTEND, twice_word_mode, op: tmp, op_mode: word_mode);
1404	emit_move_insn (SET_DEST (set), tmp);
1405	}
1406	else
1407	{
1408	if (dest_reg != src_reg)
1409	emit_move_insn (dest_reg, src_reg);
1410	if (GET_CODE (op) != ASHIFTRT)
1411	emit_move_insn (dest_upper, CONST0_RTX (word_mode));
1412	else if (INTVAL (XEXP (op, 1)) == 2 * BITS_PER_WORD - 1)
1413	emit_move_insn (dest_upper, copy_rtx (src_reg));
1414	else
1415	emit_move_insn (dest_upper, upper_src);
1416	}
1417
1418	insns = get_insns ();
1419
1420	end_sequence ();
1421
1422	emit_insn_before (insns, insn);
1423
1424	if (dump_file)
1425	{
1426	rtx_insn *in;
1427	fprintf (stream: dump_file, format: "; Replacing insn: %d with insns: ", INSN_UID (insn));
1428	for (in = insns; in != insn; in = NEXT_INSN (insn: in))
1429	fprintf (stream: dump_file, format: "%d ", INSN_UID (insn: in));
1430	fprintf (stream: dump_file, format: "\n");
1431	}
1432
1433	delete_insn (insn);
1434	return insns;
1435	}
1436
1437	/* Print to dump_file a description of what we're doing with shift code CODE.
1438	SPLITTING[X] is true if we are splitting shifts by X + BITS_PER_WORD. */
1439
1440	static void
1441	dump_shift_choices (enum rtx_code code, bool *splitting)
1442	{
1443	int i;
1444	const char *sep;
1445
1446	fprintf (stream: dump_file,
1447	format: " Splitting mode %s for %s lowering with shift amounts = ",
1448	GET_MODE_NAME (twice_word_mode), GET_RTX_NAME (code));
1449	sep = "";
1450	for (i = 0; i < BITS_PER_WORD; i++)
1451	if (splitting[i])
1452	{
1453	fprintf (stream: dump_file, format: "%s%d", sep, i + BITS_PER_WORD);
1454	sep = ",";
1455	}
1456	fprintf (stream: dump_file, format: "\n");
1457	}
1458
1459	/* Print to dump_file a description of what we're doing when optimizing
1460	for speed or size; SPEED_P says which. DESCRIPTION is a description
1461	of the SPEED_P choice. */
1462
1463	static void
1464	dump_choices (bool speed_p, const char *description)
1465	{
1466	unsigned int size, factor, i;
1467
1468	fprintf (stream: dump_file, format: "Choices when optimizing for %s:\n", description);
1469
1470	for (i = 0; i < MAX_MACHINE_MODE; i++)
1471	if (interesting_mode_p (mode: (machine_mode) i, bytes: &size, words: &factor)
1472	&& factor > 1)
1473	fprintf (stream: dump_file, format: " %s mode %s for copy lowering.\n",
1474	choices[speed_p].move_modes_to_split[i]
1475	? "Splitting"
1476	: "Skipping",
1477	GET_MODE_NAME ((machine_mode) i));
1478
1479	fprintf (stream: dump_file, format: " %s mode %s for zero_extend lowering.\n",
1480	choices[speed_p].splitting_zext ? "Splitting" : "Skipping",
1481	GET_MODE_NAME (twice_word_mode));
1482
1483	dump_shift_choices (code: ASHIFT, choices[speed_p].splitting_ashift);
1484	dump_shift_choices (code: LSHIFTRT, choices[speed_p].splitting_lshiftrt);
1485	dump_shift_choices (code: ASHIFTRT, choices[speed_p].splitting_ashiftrt);
1486	fprintf (stream: dump_file, format: "\n");
1487	}
1488
1489	/* Look for registers which are always accessed via word-sized SUBREGs
1490	or -if DECOMPOSE_COPIES is true- via copies. Decompose these
1491	registers into several word-sized pseudo-registers. */
1492
1493	static void
1494	decompose_multiword_subregs (bool decompose_copies)
1495	{
1496	unsigned int max;
1497	basic_block bb;
1498	bool speed_p;
1499
1500	if (dump_file)
1501	{
1502	dump_choices (speed_p: false, description: "size");
1503	dump_choices (speed_p: true, description: "speed");
1504	}
1505
1506	/* Check if this target even has any modes to consider lowering. */
1507	if (!choices[false].something_to_do && !choices[true].something_to_do)
1508	{
1509	if (dump_file)
1510	fprintf (stream: dump_file, format: "Nothing to do!\n");
1511	return;
1512	}
1513
1514	max = max_reg_num ();
1515
1516	/* First see if there are any multi-word pseudo-registers. If there
1517	aren't, there is nothing we can do. This should speed up this
1518	pass in the normal case, since it should be faster than scanning
1519	all the insns. */
1520	{
1521	unsigned int i;
1522	bool useful_modes_seen = false;
1523
1524	for (i = FIRST_PSEUDO_REGISTER; i < max; ++i)
1525	if (regno_reg_rtx[i] != NULL)
1526	{
1527	machine_mode mode = GET_MODE (regno_reg_rtx[i]);
1528	if (choices[false].move_modes_to_split[(int) mode]
1529	|| choices[true].move_modes_to_split[(int) mode])
1530	{
1531	useful_modes_seen = true;
1532	break;
1533	}
1534	}
1535
1536	if (!useful_modes_seen)
1537	{
1538	if (dump_file)
1539	fprintf (stream: dump_file, format: "Nothing to lower in this function.\n");
1540	return;
1541	}
1542	}
1543
1544	if (df)
1545	{
1546	df_set_flags (DF_DEFER_INSN_RESCAN);
1547	run_word_dce ();
1548	}
1549
1550	/* FIXME: It may be possible to change this code to look for each
1551	multi-word pseudo-register and to find each insn which sets or
1552	uses that register. That should be faster than scanning all the
1553	insns. */
1554
1555	decomposable_context = BITMAP_ALLOC (NULL);
1556	non_decomposable_context = BITMAP_ALLOC (NULL);
1557	subreg_context = BITMAP_ALLOC (NULL);
1558
1559	reg_copy_graph.create (nelems: max);
1560	reg_copy_graph.safe_grow_cleared (len: max, exact: true);
1561	memset (s: reg_copy_graph.address (), c: 0, n: sizeof (bitmap) * max);
1562
1563	speed_p = optimize_function_for_speed_p (cfun);
1564	FOR_EACH_BB_FN (bb, cfun)
1565	{
1566	rtx_insn *insn;
1567
1568	FOR_BB_INSNS (bb, insn)
1569	{
1570	rtx set;
1571	enum classify_move_insn cmi;
1572	int i, n;
1573
1574	if (!INSN_P (insn)
1575	|| GET_CODE (PATTERN (insn)) == CLOBBER
1576	|| GET_CODE (PATTERN (insn)) == USE)
1577	continue;
1578
1579	recog_memoized (insn);
1580
1581	if (find_decomposable_shift_zext (insn, speed_p))
1582	continue;
1583
1584	extract_insn (insn);
1585
1586	set = simple_move (insn, speed_p);
1587
1588	if (!set)
1589	cmi = NOT_SIMPLE_MOVE;
1590	else
1591	{
1592	/* We mark pseudo-to-pseudo copies as decomposable during the
1593	second pass only. The first pass is so early that there is
1594	good chance such moves will be optimized away completely by
1595	subsequent optimizations anyway.
1596
1597	However, we call find_pseudo_copy even during the first pass
1598	so as to properly set up the reg_copy_graph. */
1599	if (find_pseudo_copy (set))
1600	cmi = decompose_copies? DECOMPOSABLE_SIMPLE_MOVE : SIMPLE_MOVE;
1601	else
1602	cmi = SIMPLE_MOVE;
1603	}
1604
1605	n = recog_data.n_operands;
1606	for (i = 0; i < n; ++i)
1607	{
1608	find_decomposable_subregs (loc: &recog_data.operand[i], pcmi: &cmi);
1609
1610	/* We handle ASM_OPERANDS as a special case to support
1611	things like x86 rdtsc which returns a DImode value.
1612	We can decompose the output, which will certainly be
1613	operand 0, but not the inputs. */
1614
1615	if (cmi == SIMPLE_MOVE
1616	&& GET_CODE (SET_SRC (set)) == ASM_OPERANDS)
1617	{
1618	gcc_assert (i == 0);
1619	cmi = NOT_SIMPLE_MOVE;
1620	}
1621	}
1622	}
1623	}
1624
1625	bitmap_and_compl_into (decomposable_context, non_decomposable_context);
1626	if (!bitmap_empty_p (map: decomposable_context))
1627	{
1628	unsigned int i;
1629	sbitmap_iterator sbi;
1630	bitmap_iterator iter;
1631	unsigned int regno;
1632
1633	propagate_pseudo_copies ();
1634
1635	auto_sbitmap sub_blocks (last_basic_block_for_fn (cfun));
1636	bitmap_clear (sub_blocks);
1637
1638	EXECUTE_IF_SET_IN_BITMAP (decomposable_context, 0, regno, iter)
1639	decompose_register (regno);
1640
1641	FOR_EACH_BB_FN (bb, cfun)
1642	{
1643	rtx_insn *insn;
1644
1645	FOR_BB_INSNS (bb, insn)
1646	{
1647	rtx pat;
1648
1649	if (!INSN_P (insn))
1650	continue;
1651
1652	pat = PATTERN (insn);
1653	if (GET_CODE (pat) == CLOBBER)
1654	resolve_clobber (pat, insn);
1655	else if (GET_CODE (pat) == USE)
1656	resolve_use (pat, insn);
1657	else if (DEBUG_INSN_P (insn))
1658	resolve_debug (insn);
1659	else
1660	{
1661	rtx set;
1662	int i;
1663
1664	recog_memoized (insn);
1665	extract_insn (insn);
1666
1667	set = simple_move (insn, speed_p);
1668	if (set)
1669	{
1670	rtx_insn *orig_insn = insn;
1671	bool cfi = control_flow_insn_p (insn);
1672
1673	/* We can end up splitting loads to multi-word pseudos
1674	into separate loads to machine word size pseudos.
1675	When this happens, we first had one load that can
1676	throw, and after resolve_simple_move we'll have a
1677	bunch of loads (at least two). All those loads may
1678	trap if we can have non-call exceptions, so they
1679	all will end the current basic block. We split the
1680	block after the outer loop over all insns, but we
1681	make sure here that we will be able to split the
1682	basic block and still produce the correct control
1683	flow graph for it. */
1684	gcc_assert (!cfi
1685	|| (cfun->can_throw_non_call_exceptions
1686	&& can_throw_internal (insn)));
1687
1688	insn = resolve_simple_move (set, insn);
1689	if (insn != orig_insn)
1690	{
1691	recog_memoized (insn);
1692	extract_insn (insn);
1693
1694	if (cfi)
1695	bitmap_set_bit (map: sub_blocks, bitno: bb->index);
1696	}
1697	}
1698	else
1699	{
1700	rtx_insn *decomposed_shift;
1701
1702	decomposed_shift = resolve_shift_zext (insn, speed_p);
1703	if (decomposed_shift != NULL_RTX)
1704	{
1705	insn = decomposed_shift;
1706	recog_memoized (insn);
1707	extract_insn (insn);
1708	}
1709	}
1710
1711	for (i = recog_data.n_operands - 1; i >= 0; --i)
1712	resolve_subreg_use (loc: recog_data.operand_loc[i], insn);
1713
1714	resolve_reg_notes (insn);
1715
1716	if (num_validated_changes () > 0)
1717	{
1718	for (i = recog_data.n_dups - 1; i >= 0; --i)
1719	{
1720	rtx *pl = recog_data.dup_loc[i];
1721	int dup_num = recog_data.dup_num[i];
1722	rtx *px = recog_data.operand_loc[dup_num];
1723
1724	validate_unshare_change (insn, pl, *px, 1);
1725	}
1726
1727	i = apply_change_group ();
1728	gcc_assert (i);
1729	}
1730	}
1731	}
1732	}
1733
1734	/* If we had insns to split that caused control flow insns in the middle
1735	of a basic block, split those blocks now. Note that we only handle
1736	the case where splitting a load has caused multiple possibly trapping
1737	loads to appear. */
1738	EXECUTE_IF_SET_IN_BITMAP (sub_blocks, 0, i, sbi)
1739	{
1740	rtx_insn *insn, *end;
1741	edge fallthru;
1742
1743	bb = BASIC_BLOCK_FOR_FN (cfun, i);
1744	insn = BB_HEAD (bb);
1745	end = BB_END (bb);
1746
1747	while (insn != end)
1748	{
1749	if (control_flow_insn_p (insn))
1750	{
1751	/* Split the block after insn. There will be a fallthru
1752	edge, which is OK so we keep it. We have to create the
1753	exception edges ourselves. */
1754	fallthru = split_block (bb, insn);
1755	rtl_make_eh_edge (NULL, bb, BB_END (bb));
1756	bb = fallthru->dest;
1757	insn = BB_HEAD (bb);
1758	}
1759	else
1760	insn = NEXT_INSN (insn);
1761	}
1762	}
1763	}
1764
1765	for (bitmap b : reg_copy_graph)
1766	if (b)
1767	BITMAP_FREE (b);
1768
1769	reg_copy_graph.release ();
1770
1771	BITMAP_FREE (decomposable_context);
1772	BITMAP_FREE (non_decomposable_context);
1773	BITMAP_FREE (subreg_context);
1774	}
1775
1776	/* Implement first lower subreg pass. */
1777
1778	namespace {
1779
1780	const pass_data pass_data_lower_subreg =
1781	{
1782	.type: RTL_PASS, /* type */
1783	.name: "subreg1", /* name */
1784	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
1785	.tv_id: TV_LOWER_SUBREG, /* tv_id */
1786	.properties_required: 0, /* properties_required */
1787	.properties_provided: 0, /* properties_provided */
1788	.properties_destroyed: 0, /* properties_destroyed */
1789	.todo_flags_start: 0, /* todo_flags_start */
1790	.todo_flags_finish: 0, /* todo_flags_finish */
1791	};
1792
1793	class pass_lower_subreg : public rtl_opt_pass
1794	{
1795	public:
1796	pass_lower_subreg (gcc::context *ctxt)
1797	: rtl_opt_pass (pass_data_lower_subreg, ctxt)
1798	{}
1799
1800	/* opt_pass methods: */
1801	bool gate (function *) final override { return flag_split_wide_types != 0; }
1802	unsigned int execute (function *) final override
1803	{
1804	decompose_multiword_subregs (decompose_copies: false);
1805	return 0;
1806	}
1807
1808	}; // class pass_lower_subreg
1809
1810	} // anon namespace
1811
1812	rtl_opt_pass *
1813	make_pass_lower_subreg (gcc::context *ctxt)
1814	{
1815	return new pass_lower_subreg (ctxt);
1816	}
1817
1818	/* Implement second lower subreg pass. */
1819
1820	namespace {
1821
1822	const pass_data pass_data_lower_subreg2 =
1823	{
1824	.type: RTL_PASS, /* type */
1825	.name: "subreg2", /* name */
1826	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
1827	.tv_id: TV_LOWER_SUBREG, /* tv_id */
1828	.properties_required: 0, /* properties_required */
1829	.properties_provided: 0, /* properties_provided */
1830	.properties_destroyed: 0, /* properties_destroyed */
1831	.todo_flags_start: 0, /* todo_flags_start */
1832	TODO_df_finish, /* todo_flags_finish */
1833	};
1834
1835	class pass_lower_subreg2 : public rtl_opt_pass
1836	{
1837	public:
1838	pass_lower_subreg2 (gcc::context *ctxt)
1839	: rtl_opt_pass (pass_data_lower_subreg2, ctxt)
1840	{}
1841
1842	/* opt_pass methods: */
1843	bool gate (function *) final override
1844	{
1845	return flag_split_wide_types && flag_split_wide_types_early;
1846	}
1847	unsigned int execute (function *) final override
1848	{
1849	decompose_multiword_subregs (decompose_copies: true);
1850	return 0;
1851	}
1852
1853	}; // class pass_lower_subreg2
1854
1855	} // anon namespace
1856
1857	rtl_opt_pass *
1858	make_pass_lower_subreg2 (gcc::context *ctxt)
1859	{
1860	return new pass_lower_subreg2 (ctxt);
1861	}
1862
1863	/* Implement third lower subreg pass. */
1864
1865	namespace {
1866
1867	const pass_data pass_data_lower_subreg3 =
1868	{
1869	.type: RTL_PASS, /* type */
1870	.name: "subreg3", /* name */
1871	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
1872	.tv_id: TV_LOWER_SUBREG, /* tv_id */
1873	.properties_required: 0, /* properties_required */
1874	.properties_provided: 0, /* properties_provided */
1875	.properties_destroyed: 0, /* properties_destroyed */
1876	.todo_flags_start: 0, /* todo_flags_start */
1877	TODO_df_finish, /* todo_flags_finish */
1878	};
1879
1880	class pass_lower_subreg3 : public rtl_opt_pass
1881	{
1882	public:
1883	pass_lower_subreg3 (gcc::context *ctxt)
1884	: rtl_opt_pass (pass_data_lower_subreg3, ctxt)
1885	{}
1886
1887	/* opt_pass methods: */
1888	bool gate (function *) final override { return flag_split_wide_types; }
1889	unsigned int execute (function *) final override
1890	{
1891	decompose_multiword_subregs (decompose_copies: true);
1892	return 0;
1893	}
1894
1895	}; // class pass_lower_subreg3
1896
1897	} // anon namespace
1898
1899	rtl_opt_pass *
1900	make_pass_lower_subreg3 (gcc::context *ctxt)
1901	{
1902	return new pass_lower_subreg3 (ctxt);
1903	}
1904