1	/* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2	Copyright (C) 1987-2024 Free Software Foundation, Inc.
3
4	This file is part of GCC.
5
6	GCC is free software; you can redistribute it and/or modify it under
7	the terms of the GNU General Public License as published by the Free
8	Software Foundation; either version 3, or (at your option) any later
9	version.
10
11	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12	WARRANTY; without even the implied warranty of MERCHANTABILITY or
13	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14	for more details.
15
16	You should have received a copy of the GNU General Public License
17	along with GCC; see the file COPYING3. If not see
18	<http://www.gnu.org/licenses/>. */
19
20
21	#include "config.h"
22	#include "system.h"
23	#include "coretypes.h"
24	#include "backend.h"
25	#include "target.h"
26	#include "rtl.h"
27	#include "tree.h"
28	#include "memmodel.h"
29	#include "predict.h"
30	#include "tm_p.h"
31	#include "optabs.h"
32	#include "expmed.h"
33	#include "emit-rtl.h"
34	#include "recog.h"
35	#include "diagnostic-core.h"
36	#include "rtx-vector-builder.h"
37
38	/* Include insn-config.h before expr.h so that HAVE_conditional_move
39	is properly defined. */
40	#include "stor-layout.h"
41	#include "except.h"
42	#include "dojump.h"
43	#include "explow.h"
44	#include "expr.h"
45	#include "optabs-tree.h"
46	#include "libfuncs.h"
47	#include "internal-fn.h"
48	#include "langhooks.h"
49	#include "gimple.h"
50	#include "ssa.h"
51
52	static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
53	machine_mode *);
54	static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
55	static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
56
57	static rtx emit_conditional_move_1 (rtx, rtx, rtx, rtx, machine_mode);
58
59	/* Debug facility for use in GDB. */
60	void debug_optab_libfuncs (void);
61
62	/* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
63	the result of operation CODE applied to OP0 (and OP1 if it is a binary
64	operation). OP0_MODE is OP0's mode.
65
66	If the last insn does not set TARGET, don't do anything, but return true.
67
68	If the last insn or a previous insn sets TARGET and TARGET is one of OP0
69	or OP1, don't add the REG_EQUAL note but return false. Our caller can then
70	try again, ensuring that TARGET is not one of the operands. */
71
72	static bool
73	add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
74	rtx op1, machine_mode op0_mode)
75	{
76	rtx_insn *last_insn;
77	rtx set;
78	rtx note;
79
80	gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
81
82	if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
83	&& GET_RTX_CLASS (code) != RTX_BIN_ARITH
84	&& GET_RTX_CLASS (code) != RTX_COMM_COMPARE
85	&& GET_RTX_CLASS (code) != RTX_COMPARE
86	&& GET_RTX_CLASS (code) != RTX_UNARY)
87	return true;
88
89	if (GET_CODE (target) == ZERO_EXTRACT)
90	return true;
91
92	for (last_insn = insns;
93	NEXT_INSN (insn: last_insn) != NULL_RTX;
94	last_insn = NEXT_INSN (insn: last_insn))
95	;
96
97	/* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
98	a value changing in the insn, so the note would be invalid for CSE. */
99	if (reg_overlap_mentioned_p (target, op0)
100	|| (op1 && reg_overlap_mentioned_p (target, op1)))
101	{
102	if (MEM_P (target)
103	&& (rtx_equal_p (target, op0)
104	|| (op1 && rtx_equal_p (target, op1))))
105	{
106	/* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
107	over expanding it as temp = MEM op X, MEM = temp. If the target
108	supports MEM = MEM op X instructions, it is sometimes too hard
109	to reconstruct that form later, especially if X is also a memory,
110	and due to multiple occurrences of addresses the address might
111	be forced into register unnecessarily.
112	Note that not emitting the REG_EQUIV note might inhibit
113	CSE in some cases. */
114	set = single_set (insn: last_insn);
115	if (set
116	&& GET_CODE (SET_SRC (set)) == code
117	&& MEM_P (SET_DEST (set))
118	&& (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
119	|| (op1 && rtx_equal_p (SET_DEST (set),
120	XEXP (SET_SRC (set), 1)))))
121	return true;
122	}
123	return false;
124	}
125
126	set = set_for_reg_notes (last_insn);
127	if (set == NULL_RTX)
128	return true;
129
130	if (! rtx_equal_p (SET_DEST (set), target)
131	/* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
132	&& (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
133	|| ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
134	return true;
135
136	if (GET_RTX_CLASS (code) == RTX_UNARY)
137	switch (code)
138	{
139	case FFS:
140	case CLZ:
141	case CTZ:
142	case CLRSB:
143	case POPCOUNT:
144	case PARITY:
145	case BSWAP:
146	if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
147	{
148	note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
149	if (GET_MODE_UNIT_SIZE (op0_mode)
150	> GET_MODE_UNIT_SIZE (GET_MODE (target)))
151	note = simplify_gen_unary (code: TRUNCATE, GET_MODE (target),
152	op: note, op_mode: op0_mode);
153	else
154	note = simplify_gen_unary (code: ZERO_EXTEND, GET_MODE (target),
155	op: note, op_mode: op0_mode);
156	break;
157	}
158	/* FALLTHRU */
159	default:
160	note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
161	break;
162	}
163	else
164	note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
165
166	set_unique_reg_note (last_insn, REG_EQUAL, note);
167
168	return true;
169	}
170
171	/* Given two input operands, OP0 and OP1, determine what the correct from_mode
172	for a widening operation would be. In most cases this would be OP0, but if
173	that's a constant it'll be VOIDmode, which isn't useful. */
174
175	static machine_mode
176	widened_mode (machine_mode to_mode, rtx op0, rtx op1)
177	{
178	machine_mode m0 = GET_MODE (op0);
179	machine_mode m1 = GET_MODE (op1);
180	machine_mode result;
181
182	if (m0 == VOIDmode && m1 == VOIDmode)
183	return to_mode;
184	else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
185	result = m1;
186	else
187	result = m0;
188
189	if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
190	return to_mode;
191
192	return result;
193	}
194
195	/* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
196	says whether OP is signed or unsigned. NO_EXTEND is true if we need
197	not actually do a sign-extend or zero-extend, but can leave the
198	higher-order bits of the result rtx undefined, for example, in the case
199	of logical operations, but not right shifts. */
200
201	static rtx
202	widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
203	int unsignedp, bool no_extend)
204	{
205	rtx result;
206	scalar_int_mode int_mode;
207
208	/* If we don't have to extend and this is a constant, return it. */
209	if (no_extend && GET_MODE (op) == VOIDmode)
210	return op;
211
212	/* If we must extend do so. If OP is a SUBREG for a promoted object, also
213	extend since it will be more efficient to do so unless the signedness of
214	a promoted object differs from our extension. */
215	if (! no_extend
216	|| !is_a <scalar_int_mode> (m: mode, result: &int_mode)
217	|| (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
218	&& SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
219	return convert_modes (mode, oldmode, x: op, unsignedp);
220
221	/* If MODE is no wider than a single word, we return a lowpart or paradoxical
222	SUBREG. */
223	if (GET_MODE_SIZE (mode: int_mode) <= UNITS_PER_WORD)
224	return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
225
226	/* Otherwise, get an object of MODE, clobber it, and set the low-order
227	part to OP. */
228
229	result = gen_reg_rtx (int_mode);
230	emit_clobber (result);
231	emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
232	return result;
233	}
234
235	/* Expand vector widening operations.
236
237	There are two different classes of operations handled here:
238	1) Operations whose result is wider than all the arguments to the operation.
239	Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
240	In this case OP0 and optionally OP1 would be initialized,
241	but WIDE_OP wouldn't (not relevant for this case).
242	2) Operations whose result is of the same size as the last argument to the
243	operation, but wider than all the other arguments to the operation.
244	Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
245	In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
246
247	E.g, when called to expand the following operations, this is how
248	the arguments will be initialized:
249	nops OP0 OP1 WIDE_OP
250	widening-sum 2 oprnd0 - oprnd1
251	widening-dot-product 3 oprnd0 oprnd1 oprnd2
252	widening-mult 2 oprnd0 oprnd1 -
253	type-promotion (vec-unpack) 1 oprnd0 - - */
254
255	rtx
256	expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op,
257	rtx target, int unsignedp)
258	{
259	class expand_operand eops[4];
260	tree oprnd0, oprnd1, oprnd2;
261	machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
262	optab widen_pattern_optab;
263	enum insn_code icode;
264	int nops = TREE_CODE_LENGTH (ops->code);
265	int op;
266	bool sbool = false;
267
268	oprnd0 = ops->op0;
269	oprnd1 = nops >= 2 ? ops->op1 : NULL_TREE;
270	oprnd2 = nops >= 3 ? ops->op2 : NULL_TREE;
271
272	tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
273	if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
274	|| ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
275	/* The sign is from the result type rather than operand's type
276	for these ops. */
277	widen_pattern_optab
278	= optab_for_tree_code (ops->code, ops->type, optab_default);
279	else if ((ops->code == VEC_UNPACK_HI_EXPR
280	|| ops->code == VEC_UNPACK_LO_EXPR)
281	&& VECTOR_BOOLEAN_TYPE_P (ops->type)
282	&& VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0))
283	&& TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0))
284	&& SCALAR_INT_MODE_P (TYPE_MODE (ops->type)))
285	{
286	/* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
287	the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
288	vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
289	the pattern number of elements in the wider vector. */
290	widen_pattern_optab
291	= (ops->code == VEC_UNPACK_HI_EXPR
292	? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab);
293	sbool = true;
294	}
295	else if (ops->code == DOT_PROD_EXPR)
296	{
297	enum optab_subtype subtype = optab_default;
298	signop sign1 = TYPE_SIGN (TREE_TYPE (oprnd0));
299	signop sign2 = TYPE_SIGN (TREE_TYPE (oprnd1));
300	if (sign1 == sign2)
301	;
302	else if (sign1 == SIGNED && sign2 == UNSIGNED)
303	{
304	subtype = optab_vector_mixed_sign;
305	/* Same as optab_vector_mixed_sign but flip the operands. */
306	std::swap (a&: op0, b&: op1);
307	}
308	else if (sign1 == UNSIGNED && sign2 == SIGNED)
309	subtype = optab_vector_mixed_sign;
310	else
311	gcc_unreachable ();
312
313	widen_pattern_optab
314	= optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), subtype);
315	}
316	else
317	widen_pattern_optab
318	= optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
319	if (ops->code == WIDEN_MULT_PLUS_EXPR
320	|| ops->code == WIDEN_MULT_MINUS_EXPR)
321	icode = find_widening_optab_handler (widen_pattern_optab,
322	TYPE_MODE (TREE_TYPE (ops->op2)),
323	tmode0);
324	else
325	icode = optab_handler (op: widen_pattern_optab, mode: tmode0);
326	gcc_assert (icode != CODE_FOR_nothing);
327
328	if (nops >= 2)
329	tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
330	else if (sbool)
331	{
332	nops = 2;
333	op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ());
334	tmode1 = tmode0;
335	}
336
337	/* The last operand is of a wider mode than the rest of the operands. */
338	if (nops == 2)
339	wmode = tmode1;
340	else if (nops == 3)
341	{
342	gcc_assert (tmode1 == tmode0);
343	gcc_assert (op1);
344	wmode = TYPE_MODE (TREE_TYPE (oprnd2));
345	}
346
347	op = 0;
348	create_output_operand (op: &eops[op++], x: target, TYPE_MODE (ops->type));
349	create_convert_operand_from (op: &eops[op++], value: op0, mode: tmode0, unsigned_p: unsignedp);
350	if (op1)
351	create_convert_operand_from (op: &eops[op++], value: op1, mode: tmode1, unsigned_p: unsignedp);
352	if (wide_op)
353	create_convert_operand_from (op: &eops[op++], value: wide_op, mode: wmode, unsigned_p: unsignedp);
354	expand_insn (icode, nops: op, ops: eops);
355	return eops[0].value;
356	}
357
358	/* Generate code to perform an operation specified by TERNARY_OPTAB
359	on operands OP0, OP1 and OP2, with result having machine-mode MODE.
360
361	UNSIGNEDP is for the case where we have to widen the operands
362	to perform the operation. It says to use zero-extension.
363
364	If TARGET is nonzero, the value
365	is generated there, if it is convenient to do so.
366	In all cases an rtx is returned for the locus of the value;
367	this may or may not be TARGET. */
368
369	rtx
370	expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
371	rtx op1, rtx op2, rtx target, int unsignedp)
372	{
373	class expand_operand ops[4];
374	enum insn_code icode = optab_handler (op: ternary_optab, mode);
375
376	gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
377
378	create_output_operand (op: &ops[0], x: target, mode);
379	create_convert_operand_from (op: &ops[1], value: op0, mode, unsigned_p: unsignedp);
380	create_convert_operand_from (op: &ops[2], value: op1, mode, unsigned_p: unsignedp);
381	create_convert_operand_from (op: &ops[3], value: op2, mode, unsigned_p: unsignedp);
382	expand_insn (icode, nops: 4, ops);
383	return ops[0].value;
384	}
385
386
387	/* Like expand_binop, but return a constant rtx if the result can be
388	calculated at compile time. The arguments and return value are
389	otherwise the same as for expand_binop. */
390
391	rtx
392	simplify_expand_binop (machine_mode mode, optab binoptab,
393	rtx op0, rtx op1, rtx target, int unsignedp,
394	enum optab_methods methods)
395	{
396	if (CONSTANT_P (op0) && CONSTANT_P (op1))
397	{
398	rtx x = simplify_binary_operation (code: optab_to_code (op: binoptab),
399	mode, op0, op1);
400	if (x)
401	return x;
402	}
403
404	return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
405	}
406
407	/* Like simplify_expand_binop, but always put the result in TARGET.
408	Return true if the expansion succeeded. */
409
410	bool
411	force_expand_binop (machine_mode mode, optab binoptab,
412	rtx op0, rtx op1, rtx target, int unsignedp,
413	enum optab_methods methods)
414	{
415	rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
416	target, unsignedp, methods);
417	if (x == 0)
418	return false;
419	if (x != target)
420	emit_move_insn (target, x);
421	return true;
422	}
423
424	/* Create a new vector value in VMODE with all elements set to OP. The
425	mode of OP must be the element mode of VMODE. If OP is a constant,
426	then the return value will be a constant. */
427
428	rtx
429	expand_vector_broadcast (machine_mode vmode, rtx op)
430	{
431	int n;
432	rtvec vec;
433
434	gcc_checking_assert (VECTOR_MODE_P (vmode));
435
436	if (valid_for_const_vector_p (vmode, op))
437	return gen_const_vec_duplicate (vmode, op);
438
439	insn_code icode = optab_handler (op: vec_duplicate_optab, mode: vmode);
440	if (icode != CODE_FOR_nothing)
441	{
442	class expand_operand ops[2];
443	create_output_operand (op: &ops[0], NULL_RTX, mode: vmode);
444	create_input_operand (op: &ops[1], value: op, GET_MODE (op));
445	expand_insn (icode, nops: 2, ops);
446	return ops[0].value;
447	}
448
449	if (!GET_MODE_NUNITS (mode: vmode).is_constant (const_value: &n))
450	return NULL;
451
452	/* ??? If the target doesn't have a vec_init, then we have no easy way
453	of performing this operation. Most of this sort of generic support
454	is hidden away in the vector lowering support in gimple. */
455	icode = convert_optab_handler (op: vec_init_optab, to_mode: vmode,
456	GET_MODE_INNER (vmode));
457	if (icode == CODE_FOR_nothing)
458	return NULL;
459
460	vec = rtvec_alloc (n);
461	for (int i = 0; i < n; ++i)
462	RTVEC_ELT (vec, i) = op;
463	rtx ret = gen_reg_rtx (vmode);
464	emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
465
466	return ret;
467	}
468
469	/* This subroutine of expand_doubleword_shift handles the cases in which
470	the effective shift value is >= BITS_PER_WORD. The arguments and return
471	value are the same as for the parent routine, except that SUPERWORD_OP1
472	is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
473	INTO_TARGET may be null if the caller has decided to calculate it. */
474
475	static bool
476	expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
477	rtx outof_target, rtx into_target,
478	int unsignedp, enum optab_methods methods)
479	{
480	if (into_target != 0)
481	if (!force_expand_binop (mode: word_mode, binoptab, op0: outof_input, op1: superword_op1,
482	target: into_target, unsignedp, methods))
483	return false;
484
485	if (outof_target != 0)
486	{
487	/* For a signed right shift, we must fill OUTOF_TARGET with copies
488	of the sign bit, otherwise we must fill it with zeros. */
489	if (binoptab != ashr_optab)
490	emit_move_insn (outof_target, CONST0_RTX (word_mode));
491	else
492	if (!force_expand_binop (mode: word_mode, binoptab, op0: outof_input,
493	op1: gen_int_shift_amount (word_mode,
494	BITS_PER_WORD - 1),
495	target: outof_target, unsignedp, methods))
496	return false;
497	}
498	return true;
499	}
500
501	/* This subroutine of expand_doubleword_shift handles the cases in which
502	the effective shift value is < BITS_PER_WORD. The arguments and return
503	value are the same as for the parent routine. */
504
505	static bool
506	expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
507	rtx outof_input, rtx into_input, rtx op1,
508	rtx outof_target, rtx into_target,
509	int unsignedp, enum optab_methods methods,
510	unsigned HOST_WIDE_INT shift_mask)
511	{
512	optab reverse_unsigned_shift, unsigned_shift;
513	rtx tmp, carries;
514
515	reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
516	unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
517
518	/* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
519	We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
520	the opposite direction to BINOPTAB. */
521	if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
522	{
523	carries = outof_input;
524	tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
525	mode: op1_mode), op1_mode);
526	tmp = simplify_expand_binop (mode: op1_mode, binoptab: sub_optab, op0: tmp, op1,
527	target: 0, unsignedp: true, methods);
528	}
529	else
530	{
531	/* We must avoid shifting by BITS_PER_WORD bits since that is either
532	the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
533	has unknown behavior. Do a single shift first, then shift by the
534	remainder. It's OK to use ~OP1 as the remainder if shift counts
535	are truncated to the mode size. */
536	carries = simplify_expand_binop (mode: word_mode, binoptab: reverse_unsigned_shift,
537	op0: outof_input, const1_rtx, target: 0,
538	unsignedp, methods);
539	if (carries == const0_rtx)
540	tmp = const0_rtx;
541	else if (shift_mask == BITS_PER_WORD - 1)
542	tmp = expand_unop (op1_mode, one_cmpl_optab, op1, 0, true);
543	else
544	{
545	tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
546	mode: op1_mode), op1_mode);
547	tmp = simplify_expand_binop (mode: op1_mode, binoptab: sub_optab, op0: tmp, op1,
548	target: 0, unsignedp: true, methods);
549	}
550	}
551	if (tmp == 0 || carries == 0)
552	return false;
553	if (carries != const0_rtx && tmp != const0_rtx)
554	carries = simplify_expand_binop (mode: word_mode, binoptab: reverse_unsigned_shift,
555	op0: carries, op1: tmp, target: 0, unsignedp, methods);
556	if (carries == 0)
557	return false;
558
559	if (into_input != const0_rtx)
560	{
561	/* Shift INTO_INPUT logically by OP1. This is the last use of
562	INTO_INPUT so the result can go directly into INTO_TARGET if
563	convenient. */
564	tmp = simplify_expand_binop (mode: word_mode, binoptab: unsigned_shift, op0: into_input,
565	op1, target: into_target, unsignedp, methods);
566	if (tmp == 0)
567	return false;
568
569	/* Now OR in the bits carried over from OUTOF_INPUT. */
570	if (!force_expand_binop (mode: word_mode, binoptab: ior_optab, op0: tmp, op1: carries,
571	target: into_target, unsignedp, methods))
572	return false;
573	}
574	else
575	emit_move_insn (into_target, carries);
576
577	/* Use a standard word_mode shift for the out-of half. */
578	if (outof_target != 0)
579	if (!force_expand_binop (mode: word_mode, binoptab, op0: outof_input, op1,
580	target: outof_target, unsignedp, methods))
581	return false;
582
583	return true;
584	}
585
586
587	/* Try implementing expand_doubleword_shift using conditional moves.
588	The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
589	otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
590	are the shift counts to use in the former and latter case. All other
591	arguments are the same as the parent routine. */
592
593	static bool
594	expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
595	enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
596	rtx outof_input, rtx into_input,
597	rtx subword_op1, rtx superword_op1,
598	rtx outof_target, rtx into_target,
599	int unsignedp, enum optab_methods methods,
600	unsigned HOST_WIDE_INT shift_mask)
601	{
602	rtx outof_superword, into_superword;
603
604	/* Put the superword version of the output into OUTOF_SUPERWORD and
605	INTO_SUPERWORD. */
606	outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
607	if (outof_target != 0 && subword_op1 == superword_op1)
608	{
609	/* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
610	OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
611	into_superword = outof_target;
612	if (!expand_superword_shift (binoptab, outof_input, superword_op1,
613	outof_target: outof_superword, into_target: 0, unsignedp, methods))
614	return false;
615	}
616	else
617	{
618	into_superword = gen_reg_rtx (word_mode);
619	if (!expand_superword_shift (binoptab, outof_input, superword_op1,
620	outof_target: outof_superword, into_target: into_superword,
621	unsignedp, methods))
622	return false;
623	}
624
625	/* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
626	if (!expand_subword_shift (op1_mode, binoptab,
627	outof_input, into_input, op1: subword_op1,
628	outof_target, into_target,
629	unsignedp, methods, shift_mask))
630	return false;
631
632	/* Select between them. Do the INTO half first because INTO_SUPERWORD
633	might be the current value of OUTOF_TARGET. */
634	if (!emit_conditional_move (into_target, { .code: cmp_code, .op0: cmp1, .op1: cmp2, .mode: op1_mode },
635	into_target, into_superword, word_mode, false))
636	return false;
637
638	if (outof_target != 0)
639	if (!emit_conditional_move (outof_target,
640	{ .code: cmp_code, .op0: cmp1, .op1: cmp2, .mode: op1_mode },
641	outof_target, outof_superword,
642	word_mode, false))
643	return false;
644
645	return true;
646	}
647
648	/* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
649	OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
650	input operand; the shift moves bits in the direction OUTOF_INPUT->
651	INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
652	of the target. OP1 is the shift count and OP1_MODE is its mode.
653	If OP1 is constant, it will have been truncated as appropriate
654	and is known to be nonzero.
655
656	If SHIFT_MASK is zero, the result of word shifts is undefined when the
657	shift count is outside the range [0, BITS_PER_WORD). This routine must
658	avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
659
660	If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
661	masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
662	fill with zeros or sign bits as appropriate.
663
664	If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
665	a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
666	Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
667	In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
668	are undefined.
669
670	BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
671	may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
672	OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
673	function wants to calculate it itself.
674
675	Return true if the shift could be successfully synthesized. */
676
677	static bool
678	expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
679	rtx outof_input, rtx into_input, rtx op1,
680	rtx outof_target, rtx into_target,
681	int unsignedp, enum optab_methods methods,
682	unsigned HOST_WIDE_INT shift_mask)
683	{
684	rtx superword_op1, tmp, cmp1, cmp2;
685	enum rtx_code cmp_code;
686
687	/* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
688	fill the result with sign or zero bits as appropriate. If so, the value
689	of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
690	this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
691	and INTO_INPUT), then emit code to set up OUTOF_TARGET.
692
693	This isn't worthwhile for constant shifts since the optimizers will
694	cope better with in-range shift counts. */
695	if (shift_mask >= BITS_PER_WORD
696	&& outof_target != 0
697	&& !CONSTANT_P (op1))
698	{
699	if (!expand_doubleword_shift (op1_mode, binoptab,
700	outof_input, into_input, op1,
701	outof_target: 0, into_target,
702	unsignedp, methods, shift_mask))
703	return false;
704	if (!force_expand_binop (mode: word_mode, binoptab, op0: outof_input, op1,
705	target: outof_target, unsignedp, methods))
706	return false;
707	return true;
708	}
709
710	/* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
711	is true when the effective shift value is less than BITS_PER_WORD.
712	Set SUPERWORD_OP1 to the shift count that should be used to shift
713	OUTOF_INPUT into INTO_TARGET when the condition is false. */
714	tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, mode: op1_mode), op1_mode);
715	if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
716	{
717	/* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
718	is a subword shift count. */
719	cmp1 = simplify_expand_binop (mode: op1_mode, binoptab: and_optab, op0: op1, op1: tmp,
720	target: 0, unsignedp: true, methods);
721	cmp2 = CONST0_RTX (op1_mode);
722	cmp_code = EQ;
723	superword_op1 = op1;
724	}
725	else
726	{
727	/* Set CMP1 to OP1 - BITS_PER_WORD. */
728	cmp1 = simplify_expand_binop (mode: op1_mode, binoptab: sub_optab, op0: op1, op1: tmp,
729	target: 0, unsignedp: true, methods);
730	cmp2 = CONST0_RTX (op1_mode);
731	cmp_code = LT;
732	superword_op1 = cmp1;
733	}
734	if (cmp1 == 0)
735	return false;
736
737	/* If we can compute the condition at compile time, pick the
738	appropriate subroutine. */
739	tmp = simplify_relational_operation (code: cmp_code, SImode, op_mode: op1_mode, op0: cmp1, op1: cmp2);
740	if (tmp != 0 && CONST_INT_P (tmp))
741	{
742	if (tmp == const0_rtx)
743	return expand_superword_shift (binoptab, outof_input, superword_op1,
744	outof_target, into_target,
745	unsignedp, methods);
746	else
747	return expand_subword_shift (op1_mode, binoptab,
748	outof_input, into_input, op1,
749	outof_target, into_target,
750	unsignedp, methods, shift_mask);
751	}
752
753	/* Try using conditional moves to generate straight-line code. */
754	if (HAVE_conditional_move)
755	{
756	rtx_insn *start = get_last_insn ();
757	if (expand_doubleword_shift_condmove (op1_mode, binoptab,
758	cmp_code, cmp1, cmp2,
759	outof_input, into_input,
760	subword_op1: op1, superword_op1,
761	outof_target, into_target,
762	unsignedp, methods, shift_mask))
763	return true;
764	delete_insns_since (start);
765	}
766
767	/* As a last resort, use branches to select the correct alternative. */
768	rtx_code_label *subword_label = gen_label_rtx ();
769	rtx_code_label *done_label = gen_label_rtx ();
770
771	NO_DEFER_POP;
772	do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
773	0, 0, subword_label,
774	profile_probability::uninitialized ());
775	OK_DEFER_POP;
776
777	if (!expand_superword_shift (binoptab, outof_input, superword_op1,
778	outof_target, into_target,
779	unsignedp, methods))
780	return false;
781
782	emit_jump_insn (targetm.gen_jump (done_label));
783	emit_barrier ();
784	emit_label (subword_label);
785
786	if (!expand_subword_shift (op1_mode, binoptab,
787	outof_input, into_input, op1,
788	outof_target, into_target,
789	unsignedp, methods, shift_mask))
790	return false;
791
792	emit_label (done_label);
793	return true;
794	}
795
796	/* Subroutine of expand_binop. Perform a double word multiplication of
797	operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
798	as the target's word_mode. This function return NULL_RTX if anything
799	goes wrong, in which case it may have already emitted instructions
800	which need to be deleted.
801
802	If we want to multiply two two-word values and have normal and widening
803	multiplies of single-word values, we can do this with three smaller
804	multiplications.
805
806	The multiplication proceeds as follows:
807	_______________________
808	[__op0_high_|__op0_low__]
809	_______________________
810	* [__op1_high_|__op1_low__]
811	_______________________________________________
812	_______________________
813	(1) [__op0_low__*__op1_low__]
814	_______________________
815	(2a) [__op0_low__*__op1_high_]
816	_______________________
817	(2b) [__op0_high_*__op1_low__]
818	_______________________
819	(3) [__op0_high_*__op1_high_]
820
821
822	This gives a 4-word result. Since we are only interested in the
823	lower 2 words, partial result (3) and the upper words of (2a) and
824	(2b) don't need to be calculated. Hence (2a) and (2b) can be
825	calculated using non-widening multiplication.
826
827	(1), however, needs to be calculated with an unsigned widening
828	multiplication. If this operation is not directly supported we
829	try using a signed widening multiplication and adjust the result.
830	This adjustment works as follows:
831
832	If both operands are positive then no adjustment is needed.
833
834	If the operands have different signs, for example op0_low < 0 and
835	op1_low >= 0, the instruction treats the most significant bit of
836	op0_low as a sign bit instead of a bit with significance
837	2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
838	with 2**BITS_PER_WORD - op0_low, and two's complements the
839	result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
840	the result.
841
842	Similarly, if both operands are negative, we need to add
843	(op0_low + op1_low) * 2**BITS_PER_WORD.
844
845	We use a trick to adjust quickly. We logically shift op0_low right
846	(op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
847	op0_high (op1_high) before it is used to calculate 2b (2a). If no
848	logical shift exists, we do an arithmetic right shift and subtract
849	the 0 or -1. */
850
851	static rtx
852	expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
853	bool umulp, enum optab_methods methods)
854	{
855	int low = (WORDS_BIG_ENDIAN ? 1 : 0);
856	int high = (WORDS_BIG_ENDIAN ? 0 : 1);
857	rtx wordm1 = (umulp ? NULL_RTX
858	: gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
859	rtx product, adjust, product_high, temp;
860
861	rtx op0_high = operand_subword_force (op0, high, mode);
862	rtx op0_low = operand_subword_force (op0, low, mode);
863	rtx op1_high = operand_subword_force (op1, high, mode);
864	rtx op1_low = operand_subword_force (op1, low, mode);
865
866	/* If we're using an unsigned multiply to directly compute the product
867	of the low-order words of the operands and perform any required
868	adjustments of the operands, we begin by trying two more multiplications
869	and then computing the appropriate sum.
870
871	We have checked above that the required addition is provided.
872	Full-word addition will normally always succeed, especially if
873	it is provided at all, so we don't worry about its failure. The
874	multiplication may well fail, however, so we do handle that. */
875
876	if (!umulp)
877	{
878	/* ??? This could be done with emit_store_flag where available. */
879	temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
880	NULL_RTX, 1, methods);
881	if (temp)
882	op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
883	NULL_RTX, 0, OPTAB_DIRECT);
884	else
885	{
886	temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
887	NULL_RTX, 0, methods);
888	if (!temp)
889	return NULL_RTX;
890	op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
891	NULL_RTX, 0, OPTAB_DIRECT);
892	}
893
894	if (!op0_high)
895	return NULL_RTX;
896	}
897
898	adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
899	NULL_RTX, 0, OPTAB_DIRECT);
900	if (!adjust)
901	return NULL_RTX;
902
903	/* OP0_HIGH should now be dead. */
904
905	if (!umulp)
906	{
907	/* ??? This could be done with emit_store_flag where available. */
908	temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
909	NULL_RTX, 1, methods);
910	if (temp)
911	op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
912	NULL_RTX, 0, OPTAB_DIRECT);
913	else
914	{
915	temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
916	NULL_RTX, 0, methods);
917	if (!temp)
918	return NULL_RTX;
919	op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
920	NULL_RTX, 0, OPTAB_DIRECT);
921	}
922
923	if (!op1_high)
924	return NULL_RTX;
925	}
926
927	temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
928	NULL_RTX, 0, OPTAB_DIRECT);
929	if (!temp)
930	return NULL_RTX;
931
932	/* OP1_HIGH should now be dead. */
933
934	adjust = expand_binop (word_mode, add_optab, adjust, temp,
935	NULL_RTX, 0, OPTAB_DIRECT);
936
937	if (target && !REG_P (target))
938	target = NULL_RTX;
939
940	/* *_widen_optab needs to determine operand mode, make sure at least
941	one operand has non-VOID mode. */
942	if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
943	op0_low = force_reg (word_mode, op0_low);
944
945	if (umulp)
946	product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
947	target, 1, OPTAB_DIRECT);
948	else
949	product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
950	target, 1, OPTAB_DIRECT);
951
952	if (!product)
953	return NULL_RTX;
954
955	product_high = operand_subword (product, high, 1, mode);
956	adjust = expand_binop (word_mode, add_optab, product_high, adjust,
957	NULL_RTX, 0, OPTAB_DIRECT);
958	emit_move_insn (product_high, adjust);
959	return product;
960	}
961
962	/* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for
963	constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
964	(prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
965	computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
966	+ (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values
967	depends on the bit value, if 2, then carry from the addition needs to be
968	added too, i.e. like:
969	sum += __builtin_add_overflow (low, high, &sum)
970
971	Optimize signed double-word OP0 % OP1 similarly, just apply some correction
972	factor to the sum before doing unsigned remainder, in the form of
973	sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
974	then perform unsigned
975	remainder = sum % OP1;
976	and finally
977	remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */
978
979	static rtx
980	expand_doubleword_mod (machine_mode mode, rtx op0, rtx op1, bool unsignedp)
981	{
982	if (INTVAL (op1) <= 1 || (INTVAL (op1) & 1) == 0)
983	return NULL_RTX;
984
985	rtx_insn *last = get_last_insn ();
986	for (int bit = BITS_PER_WORD; bit >= BITS_PER_WORD / 2; bit--)
987	{
988	wide_int w = wi::shifted_mask (start: bit, width: 1, negate_p: false, precision: 2 * BITS_PER_WORD);
989	if (wi::ne_p (x: wi::umod_trunc (x: w, INTVAL (op1)), y: 1))
990	continue;
991	rtx sum = NULL_RTX, mask = NULL_RTX;
992	if (bit == BITS_PER_WORD)
993	{
994	/* For signed modulo we need to add correction to the sum
995	and that might again overflow. */
996	if (!unsignedp)
997	continue;
998	if (optab_handler (op: uaddv4_optab, mode: word_mode) == CODE_FOR_nothing)
999	continue;
1000	tree wtype = lang_hooks.types.type_for_mode (word_mode, 1);
1001	if (wtype == NULL_TREE)
1002	continue;
1003	tree ctype = build_complex_type (wtype);
1004	if (TYPE_MODE (ctype) != GET_MODE_COMPLEX_MODE (word_mode))
1005	continue;
1006	machine_mode cmode = TYPE_MODE (ctype);
1007	rtx op00 = operand_subword_force (op0, 0, mode);
1008	rtx op01 = operand_subword_force (op0, 1, mode);
1009	rtx cres = gen_rtx_CONCAT (cmode, gen_reg_rtx (word_mode),
1010	gen_reg_rtx (word_mode));
1011	tree lhs = make_tree (ctype, cres);
1012	tree arg0 = make_tree (wtype, op00);
1013	tree arg1 = make_tree (wtype, op01);
1014	expand_addsub_overflow (UNKNOWN_LOCATION, PLUS_EXPR, lhs, arg0,
1015	arg1, true, true, true, false, NULL);
1016	sum = expand_simple_binop (word_mode, PLUS, XEXP (cres, 0),
1017	XEXP (cres, 1), NULL_RTX, 1,
1018	OPTAB_DIRECT);
1019	if (sum == NULL_RTX)
1020	return NULL_RTX;
1021	}
1022	else
1023	{
1024	/* Code below uses GEN_INT, so we need the masks to be representable
1025	in HOST_WIDE_INTs. */
1026	if (bit >= HOST_BITS_PER_WIDE_INT)
1027	continue;
1028	/* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
1029	overflow. Consider 64-bit -1ULL for word size 32, if we add
1030	0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */
1031	if (bit == BITS_PER_WORD - 1)
1032	continue;
1033
1034	int count = (2 * BITS_PER_WORD + bit - 1) / bit;
1035	rtx sum_corr = NULL_RTX;
1036
1037	if (!unsignedp)
1038	{
1039	/* For signed modulo, compute it as unsigned modulo of
1040	sum with a correction added to it if OP0 is negative,
1041	such that the result can be computed as unsigned
1042	remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */
1043	w = wi::min_value (2 * BITS_PER_WORD, SIGNED);
1044	wide_int wmod1 = wi::umod_trunc (x: w, INTVAL (op1));
1045	wide_int wmod2 = wi::smod_trunc (x: w, INTVAL (op1));
1046	/* wmod2 == -wmod1. */
1047	wmod2 = wmod2 + (INTVAL (op1) - 1);
1048	if (wi::ne_p (x: wmod1, y: wmod2))
1049	{
1050	wide_int wcorr = wmod2 - wmod1;
1051	if (wi::neg_p (x: w))
1052	wcorr = wcorr + INTVAL (op1);
1053	/* Now verify if the count sums can't overflow, and punt
1054	if they could. */
1055	w = wi::mask (width: bit, negate_p: false, precision: 2 * BITS_PER_WORD);
1056	w = w * (count - 1);
1057	w = w + wi::mask (width: 2 * BITS_PER_WORD - (count - 1) * bit,
1058	negate_p: false, precision: 2 * BITS_PER_WORD);
1059	w = w + wcorr;
1060	w = wi::lrshift (x: w, BITS_PER_WORD);
1061	if (wi::ne_p (x: w, y: 0))
1062	continue;
1063
1064	mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1065	mode);
1066	mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1067	GEN_INT (BITS_PER_WORD - 1),
1068	NULL_RTX, 0, OPTAB_DIRECT);
1069	if (mask == NULL_RTX)
1070	return NULL_RTX;
1071	sum_corr = immed_wide_int_const (wcorr, word_mode);
1072	sum_corr = expand_simple_binop (word_mode, AND, mask,
1073	sum_corr, NULL_RTX, 1,
1074	OPTAB_DIRECT);
1075	if (sum_corr == NULL_RTX)
1076	return NULL_RTX;
1077	}
1078	}
1079
1080	for (int i = 0; i < count; i++)
1081	{
1082	rtx v = op0;
1083	if (i)
1084	v = expand_simple_binop (mode, LSHIFTRT, v, GEN_INT (i * bit),
1085	NULL_RTX, 1, OPTAB_DIRECT);
1086	if (v == NULL_RTX)
1087	return NULL_RTX;
1088	v = lowpart_subreg (outermode: word_mode, op: v, innermode: mode);
1089	if (v == NULL_RTX)
1090	return NULL_RTX;
1091	if (i != count - 1)
1092	v = expand_simple_binop (word_mode, AND, v,
1093	GEN_INT ((HOST_WIDE_INT_1U << bit)
1094	- 1), NULL_RTX, 1,
1095	OPTAB_DIRECT);
1096	if (v == NULL_RTX)
1097	return NULL_RTX;
1098	if (sum == NULL_RTX)
1099	sum = v;
1100	else
1101	sum = expand_simple_binop (word_mode, PLUS, sum, v, NULL_RTX,
1102	1, OPTAB_DIRECT);
1103	if (sum == NULL_RTX)
1104	return NULL_RTX;
1105	}
1106	if (sum_corr)
1107	{
1108	sum = expand_simple_binop (word_mode, PLUS, sum, sum_corr,
1109	NULL_RTX, 1, OPTAB_DIRECT);
1110	if (sum == NULL_RTX)
1111	return NULL_RTX;
1112	}
1113	}
1114	rtx remainder = expand_divmod (1, TRUNC_MOD_EXPR, word_mode, sum,
1115	gen_int_mode (INTVAL (op1), word_mode),
1116	NULL_RTX, 1, OPTAB_DIRECT);
1117	if (remainder == NULL_RTX)
1118	return NULL_RTX;
1119
1120	if (!unsignedp)
1121	{
1122	if (mask == NULL_RTX)
1123	{
1124	mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1125	mode);
1126	mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1127	GEN_INT (BITS_PER_WORD - 1),
1128	NULL_RTX, 0, OPTAB_DIRECT);
1129	if (mask == NULL_RTX)
1130	return NULL_RTX;
1131	}
1132	mask = expand_simple_binop (word_mode, AND, mask,
1133	gen_int_mode (1 - INTVAL (op1),
1134	word_mode),
1135	NULL_RTX, 1, OPTAB_DIRECT);
1136	if (mask == NULL_RTX)
1137	return NULL_RTX;
1138	remainder = expand_simple_binop (word_mode, PLUS, remainder,
1139	mask, NULL_RTX, 1, OPTAB_DIRECT);
1140	if (remainder == NULL_RTX)
1141	return NULL_RTX;
1142	}
1143
1144	remainder = convert_modes (mode, oldmode: word_mode, x: remainder, unsignedp);
1145	/* Punt if we need any library calls. */
1146	if (last)
1147	last = NEXT_INSN (insn: last);
1148	else
1149	last = get_insns ();
1150	for (; last; last = NEXT_INSN (insn: last))
1151	if (CALL_P (last))
1152	return NULL_RTX;
1153	return remainder;
1154	}
1155	return NULL_RTX;
1156	}
1157
1158	/* Similarly to the above function, but compute both quotient and remainder.
1159	Quotient can be computed from the remainder as:
1160	rem = op0 % op1; // Handled using expand_doubleword_mod
1161	quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo
1162	// 2 * BITS_PER_WORD
1163
1164	We can also handle cases where op1 is a multiple of power of two constant
1165	and constant handled by expand_doubleword_mod.
1166	op11 = 1 << __builtin_ctz (op1);
1167	op12 = op1 / op11;
1168	rem1 = op0 % op12; // Handled using expand_doubleword_mod
1169	quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo
1170	// 2 * BITS_PER_WORD
1171	rem = (quot1 % op11) * op12 + rem1;
1172	quot = quot1 / op11; */
1173
1174	rtx
1175	expand_doubleword_divmod (machine_mode mode, rtx op0, rtx op1, rtx *rem,
1176	bool unsignedp)
1177	{
1178	*rem = NULL_RTX;
1179
1180	/* Negative dividend should have been optimized into positive,
1181	similarly modulo by 1 and modulo by power of two is optimized
1182	differently too. */
1183	if (INTVAL (op1) <= 1 || pow2p_hwi (INTVAL (op1)))
1184	return NULL_RTX;
1185
1186	rtx op11 = const1_rtx;
1187	rtx op12 = op1;
1188	if ((INTVAL (op1) & 1) == 0)
1189	{
1190	int bit = ctz_hwi (INTVAL (op1));
1191	op11 = GEN_INT (HOST_WIDE_INT_1 << bit);
1192	op12 = GEN_INT (INTVAL (op1) >> bit);
1193	}
1194
1195	rtx rem1 = expand_doubleword_mod (mode, op0, op1: op12, unsignedp);
1196	if (rem1 == NULL_RTX)
1197	return NULL_RTX;
1198
1199	int prec = 2 * BITS_PER_WORD;
1200	wide_int a = wide_int::from (INTVAL (op12), precision: prec + 1, sgn: UNSIGNED);
1201	wide_int b = wi::shifted_mask (start: prec, width: 1, negate_p: false, precision: prec + 1);
1202	wide_int m = wide_int::from (x: wi::mod_inv (a, b), precision: prec, sgn: UNSIGNED);
1203	rtx inv = immed_wide_int_const (m, mode);
1204
1205	rtx_insn *last = get_last_insn ();
1206	rtx quot1 = expand_simple_binop (mode, MINUS, op0, rem1,
1207	NULL_RTX, unsignedp, OPTAB_DIRECT);
1208	if (quot1 == NULL_RTX)
1209	return NULL_RTX;
1210
1211	quot1 = expand_simple_binop (mode, MULT, quot1, inv,
1212	NULL_RTX, unsignedp, OPTAB_DIRECT);
1213	if (quot1 == NULL_RTX)
1214	return NULL_RTX;
1215
1216	if (op11 != const1_rtx)
1217	{
1218	rtx rem2 = expand_divmod (1, TRUNC_MOD_EXPR, mode, quot1, op11,
1219	NULL_RTX, unsignedp, OPTAB_DIRECT);
1220	if (rem2 == NULL_RTX)
1221	return NULL_RTX;
1222
1223	rem2 = expand_simple_binop (mode, MULT, rem2, op12, NULL_RTX,
1224	unsignedp, OPTAB_DIRECT);
1225	if (rem2 == NULL_RTX)
1226	return NULL_RTX;
1227
1228	rem2 = expand_simple_binop (mode, PLUS, rem2, rem1, NULL_RTX,
1229	unsignedp, OPTAB_DIRECT);
1230	if (rem2 == NULL_RTX)
1231	return NULL_RTX;
1232
1233	rtx quot2 = expand_divmod (0, TRUNC_DIV_EXPR, mode, quot1, op11,
1234	NULL_RTX, unsignedp, OPTAB_DIRECT);
1235	if (quot2 == NULL_RTX)
1236	return NULL_RTX;
1237
1238	rem1 = rem2;
1239	quot1 = quot2;
1240	}
1241
1242	/* Punt if we need any library calls. */
1243	if (last)
1244	last = NEXT_INSN (insn: last);
1245	else
1246	last = get_insns ();
1247	for (; last; last = NEXT_INSN (insn: last))
1248	if (CALL_P (last))
1249	return NULL_RTX;
1250
1251	*rem = rem1;
1252	return quot1;
1253	}
1254
1255	/* Wrapper around expand_binop which takes an rtx code to specify
1256	the operation to perform, not an optab pointer. All other
1257	arguments are the same. */
1258	rtx
1259	expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
1260	rtx op1, rtx target, int unsignedp,
1261	enum optab_methods methods)
1262	{
1263	optab binop = code_to_optab (code);
1264	gcc_assert (binop);
1265
1266	return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
1267	}
1268
1269	/* Return whether OP0 and OP1 should be swapped when expanding a commutative
1270	binop. Order them according to commutative_operand_precedence and, if
1271	possible, try to put TARGET or a pseudo first. */
1272	static bool
1273	swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
1274	{
1275	int op0_prec = commutative_operand_precedence (op0);
1276	int op1_prec = commutative_operand_precedence (op1);
1277
1278	if (op0_prec < op1_prec)
1279	return true;
1280
1281	if (op0_prec > op1_prec)
1282	return false;
1283
1284	/* With equal precedence, both orders are ok, but it is better if the
1285	first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1286	if (target == 0 || REG_P (target))
1287	return (REG_P (op1) && !REG_P (op0)) || target == op1;
1288	else
1289	return rtx_equal_p (op1, target);
1290	}
1291
1292	/* Return true if BINOPTAB implements a shift operation. */
1293
1294	static bool
1295	shift_optab_p (optab binoptab)
1296	{
1297	switch (optab_to_code (op: binoptab))
1298	{
1299	case ASHIFT:
1300	case SS_ASHIFT:
1301	case US_ASHIFT:
1302	case ASHIFTRT:
1303	case LSHIFTRT:
1304	case ROTATE:
1305	case ROTATERT:
1306	return true;
1307
1308	default:
1309	return false;
1310	}
1311	}
1312
1313	/* Return true if BINOPTAB implements a commutative binary operation. */
1314
1315	static bool
1316	commutative_optab_p (optab binoptab)
1317	{
1318	return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
1319	|| binoptab == smul_widen_optab
1320	|| binoptab == umul_widen_optab
1321	|| binoptab == smul_highpart_optab
1322	|| binoptab == umul_highpart_optab
1323	|| binoptab == vec_widen_sadd_optab
1324	|| binoptab == vec_widen_uadd_optab
1325	|| binoptab == vec_widen_sadd_hi_optab
1326	|| binoptab == vec_widen_sadd_lo_optab
1327	|| binoptab == vec_widen_uadd_hi_optab
1328	|| binoptab == vec_widen_uadd_lo_optab
1329	|| binoptab == vec_widen_sadd_even_optab
1330	|| binoptab == vec_widen_sadd_odd_optab
1331	|| binoptab == vec_widen_uadd_even_optab
1332	|| binoptab == vec_widen_uadd_odd_optab);
1333	}
1334
1335	/* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1336	optimizing, and if the operand is a constant that costs more than
1337	1 instruction, force the constant into a register and return that
1338	register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1339
1340	static rtx
1341	avoid_expensive_constant (machine_mode mode, optab binoptab,
1342	int opn, rtx x, bool unsignedp)
1343	{
1344	bool speed = optimize_insn_for_speed_p ();
1345
1346	if (mode != VOIDmode
1347	&& optimize
1348	&& CONSTANT_P (x)
1349	&& (rtx_cost (x, mode, optab_to_code (op: binoptab), opn, speed)
1350	> set_src_cost (x, mode, speed_p: speed)))
1351	{
1352	if (CONST_INT_P (x))
1353	{
1354	HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1355	if (intval != INTVAL (x))
1356	x = GEN_INT (intval);
1357	}
1358	else
1359	x = convert_modes (mode, VOIDmode, x, unsignedp);
1360	x = force_reg (mode, x);
1361	}
1362	return x;
1363	}
1364
1365	/* Helper function for expand_binop: handle the case where there
1366	is an insn ICODE that directly implements the indicated operation.
1367	Returns null if this is not possible. */
1368	static rtx
1369	expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
1370	rtx op0, rtx op1,
1371	rtx target, int unsignedp, enum optab_methods methods,
1372	rtx_insn *last)
1373	{
1374	machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1375	machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1376	machine_mode mode0, mode1, tmp_mode;
1377	class expand_operand ops[3];
1378	bool commutative_p;
1379	rtx_insn *pat;
1380	rtx xop0 = op0, xop1 = op1;
1381	bool canonicalize_op1 = false;
1382
1383	/* If it is a commutative operator and the modes would match
1384	if we would swap the operands, we can save the conversions. */
1385	commutative_p = commutative_optab_p (binoptab);
1386	if (commutative_p
1387	&& GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1388	&& GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode0)
1389	std::swap (a&: xop0, b&: xop1);
1390
1391	/* If we are optimizing, force expensive constants into a register. */
1392	xop0 = avoid_expensive_constant (mode: xmode0, binoptab, opn: 0, x: xop0, unsignedp);
1393	if (!shift_optab_p (binoptab))
1394	xop1 = avoid_expensive_constant (mode: xmode1, binoptab, opn: 1, x: xop1, unsignedp);
1395	else
1396	/* Shifts and rotates often use a different mode for op1 from op0;
1397	for VOIDmode constants we don't know the mode, so force it
1398	to be canonicalized using convert_modes. */
1399	canonicalize_op1 = true;
1400
1401	/* In case the insn wants input operands in modes different from
1402	those of the actual operands, convert the operands. It would
1403	seem that we don't need to convert CONST_INTs, but we do, so
1404	that they're properly zero-extended, sign-extended or truncated
1405	for their mode. */
1406
1407	mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1408	if (xmode0 != VOIDmode && xmode0 != mode0)
1409	{
1410	xop0 = convert_modes (mode: xmode0, oldmode: mode0, x: xop0, unsignedp);
1411	mode0 = xmode0;
1412	}
1413
1414	mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1415	? GET_MODE (xop1) : mode);
1416	if (xmode1 != VOIDmode && xmode1 != mode1)
1417	{
1418	xop1 = convert_modes (mode: xmode1, oldmode: mode1, x: xop1, unsignedp);
1419	mode1 = xmode1;
1420	}
1421
1422	/* If operation is commutative,
1423	try to make the first operand a register.
1424	Even better, try to make it the same as the target.
1425	Also try to make the last operand a constant. */
1426	if (commutative_p
1427	&& swap_commutative_operands_with_target (target, op0: xop0, op1: xop1))
1428	std::swap (a&: xop0, b&: xop1);
1429
1430	/* Now, if insn's predicates don't allow our operands, put them into
1431	pseudo regs. */
1432
1433	if (binoptab == vec_pack_trunc_optab
1434	|| binoptab == vec_pack_usat_optab
1435	|| binoptab == vec_pack_ssat_optab
1436	|| binoptab == vec_pack_ufix_trunc_optab
1437	|| binoptab == vec_pack_sfix_trunc_optab
1438	|| binoptab == vec_packu_float_optab
1439	|| binoptab == vec_packs_float_optab)
1440	{
1441	/* The mode of the result is different then the mode of the
1442	arguments. */
1443	tmp_mode = insn_data[(int) icode].operand[0].mode;
1444	if (VECTOR_MODE_P (mode)
1445	&& maybe_ne (a: GET_MODE_NUNITS (mode: tmp_mode), b: 2 * GET_MODE_NUNITS (mode)))
1446	{
1447	delete_insns_since (last);
1448	return NULL_RTX;
1449	}
1450	}
1451	else
1452	tmp_mode = mode;
1453
1454	create_output_operand (op: &ops[0], x: target, mode: tmp_mode);
1455	create_input_operand (op: &ops[1], value: xop0, mode: mode0);
1456	create_input_operand (op: &ops[2], value: xop1, mode: mode1);
1457	pat = maybe_gen_insn (icode, nops: 3, ops);
1458	if (pat)
1459	{
1460	/* If PAT is composed of more than one insn, try to add an appropriate
1461	REG_EQUAL note to it. If we can't because TEMP conflicts with an
1462	operand, call expand_binop again, this time without a target. */
1463	if (INSN_P (pat) && NEXT_INSN (insn: pat) != NULL_RTX
1464	&& ! add_equal_note (insns: pat, target: ops[0].value,
1465	code: optab_to_code (op: binoptab),
1466	op0: ops[1].value, op1: ops[2].value, op0_mode: mode0))
1467	{
1468	delete_insns_since (last);
1469	return expand_binop (mode, binoptab, op0, op1, NULL_RTX,
1470	unsignedp, methods);
1471	}
1472
1473	emit_insn (pat);
1474	return ops[0].value;
1475	}
1476	delete_insns_since (last);
1477	return NULL_RTX;
1478	}
1479
1480	/* Generate code to perform an operation specified by BINOPTAB
1481	on operands OP0 and OP1, with result having machine-mode MODE.
1482
1483	UNSIGNEDP is for the case where we have to widen the operands
1484	to perform the operation. It says to use zero-extension.
1485
1486	If TARGET is nonzero, the value
1487	is generated there, if it is convenient to do so.
1488	In all cases an rtx is returned for the locus of the value;
1489	this may or may not be TARGET. */
1490
1491	rtx
1492	expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1493	rtx target, int unsignedp, enum optab_methods methods)
1494	{
1495	enum optab_methods next_methods
1496	= (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1497	? OPTAB_WIDEN : methods);
1498	enum mode_class mclass;
1499	enum insn_code icode;
1500	machine_mode wider_mode;
1501	scalar_int_mode int_mode;
1502	rtx libfunc;
1503	rtx temp;
1504	rtx_insn *entry_last = get_last_insn ();
1505	rtx_insn *last;
1506
1507	mclass = GET_MODE_CLASS (mode);
1508
1509	/* If subtracting an integer constant, convert this into an addition of
1510	the negated constant. */
1511
1512	if (binoptab == sub_optab && CONST_INT_P (op1))
1513	{
1514	op1 = negate_rtx (mode, op1);
1515	binoptab = add_optab;
1516	}
1517	/* For shifts, constant invalid op1 might be expanded from different
1518	mode than MODE. As those are invalid, force them to a register
1519	to avoid further problems during expansion. */
1520	else if (CONST_INT_P (op1)
1521	&& shift_optab_p (binoptab)
1522	&& UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1523	{
1524	op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1525	op1 = force_reg (GET_MODE_INNER (mode), op1);
1526	}
1527
1528	/* Record where to delete back to if we backtrack. */
1529	last = get_last_insn ();
1530
1531	/* If we can do it with a three-operand insn, do so. */
1532
1533	if (methods != OPTAB_MUST_WIDEN)
1534	{
1535	if (convert_optab_p (op: binoptab))
1536	{
1537	machine_mode from_mode = widened_mode (to_mode: mode, op0, op1);
1538	icode = find_widening_optab_handler (binoptab, mode, from_mode);
1539	}
1540	else
1541	icode = optab_handler (op: binoptab, mode);
1542	if (icode != CODE_FOR_nothing)
1543	{
1544	temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1545	target, unsignedp, methods, last);
1546	if (temp)
1547	return temp;
1548	}
1549	}
1550
1551	/* If we were trying to rotate, and that didn't work, try rotating
1552	the other direction before falling back to shifts and bitwise-or. */
1553	if (((binoptab == rotl_optab
1554	&& (icode = optab_handler (op: rotr_optab, mode)) != CODE_FOR_nothing)
1555	|| (binoptab == rotr_optab
1556	&& (icode = optab_handler (op: rotl_optab, mode)) != CODE_FOR_nothing))
1557	&& is_int_mode (mode, int_mode: &int_mode))
1558	{
1559	optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1560	rtx newop1;
1561	unsigned int bits = GET_MODE_PRECISION (mode: int_mode);
1562
1563	if (CONST_INT_P (op1))
1564	newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
1565	else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1566	newop1 = negate_rtx (GET_MODE (op1), op1);
1567	else
1568	newop1 = expand_binop (GET_MODE (op1), binoptab: sub_optab,
1569	op0: gen_int_mode (bits, GET_MODE (op1)), op1,
1570	NULL_RTX, unsignedp, methods: OPTAB_DIRECT);
1571
1572	temp = expand_binop_directly (icode, mode: int_mode, binoptab: otheroptab, op0, op1: newop1,
1573	target, unsignedp, methods, last);
1574	if (temp)
1575	return temp;
1576	}
1577
1578	/* If this is a multiply, see if we can do a widening operation that
1579	takes operands of this mode and makes a wider mode. */
1580
1581	if (binoptab == smul_optab
1582	&& GET_MODE_2XWIDER_MODE (m: mode).exists (mode: &wider_mode)
1583	&& (convert_optab_handler (op: (unsignedp
1584	? umul_widen_optab
1585	: smul_widen_optab),
1586	to_mode: wider_mode, from_mode: mode) != CODE_FOR_nothing))
1587	{
1588	/* *_widen_optab needs to determine operand mode, make sure at least
1589	one operand has non-VOID mode. */
1590	if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1591	op0 = force_reg (mode, op0);
1592	temp = expand_binop (mode: wider_mode,
1593	binoptab: unsignedp ? umul_widen_optab : smul_widen_optab,
1594	op0, op1, NULL_RTX, unsignedp, methods: OPTAB_DIRECT);
1595
1596	if (temp != 0)
1597	{
1598	if (GET_MODE_CLASS (mode) == MODE_INT
1599	&& TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1600	return gen_lowpart (mode, temp);
1601	else
1602	return convert_to_mode (mode, temp, unsignedp);
1603	}
1604	}
1605
1606	/* If this is a vector shift by a scalar, see if we can do a vector
1607	shift by a vector. If so, broadcast the scalar into a vector. */
1608	if (mclass == MODE_VECTOR_INT)
1609	{
1610	optab otheroptab = unknown_optab;
1611
1612	if (binoptab == ashl_optab)
1613	otheroptab = vashl_optab;
1614	else if (binoptab == ashr_optab)
1615	otheroptab = vashr_optab;
1616	else if (binoptab == lshr_optab)
1617	otheroptab = vlshr_optab;
1618	else if (binoptab == rotl_optab)
1619	otheroptab = vrotl_optab;
1620	else if (binoptab == rotr_optab)
1621	otheroptab = vrotr_optab;
1622
1623	if (otheroptab
1624	&& (icode = optab_handler (op: otheroptab, mode)) != CODE_FOR_nothing)
1625	{
1626	/* The scalar may have been extended to be too wide. Truncate
1627	it back to the proper size to fit in the broadcast vector. */
1628	scalar_mode inner_mode = GET_MODE_INNER (mode);
1629	if (!CONST_INT_P (op1)
1630	&& (GET_MODE_BITSIZE (mode: as_a <scalar_int_mode> (GET_MODE (op1)))
1631	> GET_MODE_BITSIZE (mode: inner_mode)))
1632	op1 = force_reg (inner_mode,
1633	simplify_gen_unary (code: TRUNCATE, mode: inner_mode, op: op1,
1634	GET_MODE (op1)));
1635	rtx vop1 = expand_vector_broadcast (vmode: mode, op: op1);
1636	if (vop1)
1637	{
1638	temp = expand_binop_directly (icode, mode, binoptab: otheroptab, op0, op1: vop1,
1639	target, unsignedp, methods, last);
1640	if (temp)
1641	return temp;
1642	}
1643	}
1644	}
1645
1646	/* Look for a wider mode of the same class for which we think we
1647	can open-code the operation. Check for a widening multiply at the
1648	wider mode as well. */
1649
1650	if (CLASS_HAS_WIDER_MODES_P (mclass)
1651	&& methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1652	FOR_EACH_WIDER_MODE (wider_mode, mode)
1653	{
1654	machine_mode next_mode;
1655	if (optab_handler (op: binoptab, mode: wider_mode) != CODE_FOR_nothing
1656	|| (binoptab == smul_optab
1657	&& GET_MODE_WIDER_MODE (m: wider_mode).exists (mode: &next_mode)
1658	&& (find_widening_optab_handler ((unsignedp
1659	? umul_widen_optab
1660	: smul_widen_optab),
1661	next_mode, mode)
1662	!= CODE_FOR_nothing)))
1663	{
1664	rtx xop0 = op0, xop1 = op1;
1665	bool no_extend = false;
1666
1667	/* For certain integer operations, we need not actually extend
1668	the narrow operands, as long as we will truncate
1669	the results to the same narrowness. */
1670
1671	if ((binoptab == ior_optab || binoptab == and_optab
1672	|| binoptab == xor_optab
1673	|| binoptab == add_optab || binoptab == sub_optab
1674	|| binoptab == smul_optab || binoptab == ashl_optab)
1675	&& mclass == MODE_INT)
1676	{
1677	no_extend = true;
1678	xop0 = avoid_expensive_constant (mode, binoptab, opn: 0,
1679	x: xop0, unsignedp);
1680	if (binoptab != ashl_optab)
1681	xop1 = avoid_expensive_constant (mode, binoptab, opn: 1,
1682	x: xop1, unsignedp);
1683	}
1684
1685	xop0 = widen_operand (op: xop0, mode: wider_mode, oldmode: mode, unsignedp, no_extend);
1686
1687	/* The second operand of a shift must always be extended. */
1688	xop1 = widen_operand (op: xop1, mode: wider_mode, oldmode: mode, unsignedp,
1689	no_extend: no_extend && binoptab != ashl_optab);
1690
1691	temp = expand_binop (mode: wider_mode, binoptab, op0: xop0, op1: xop1, NULL_RTX,
1692	unsignedp, methods: OPTAB_DIRECT);
1693	if (temp)
1694	{
1695	if (mclass != MODE_INT
1696	|| !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1697	{
1698	if (target == 0)
1699	target = gen_reg_rtx (mode);
1700	convert_move (target, temp, 0);
1701	return target;
1702	}
1703	else
1704	return gen_lowpart (mode, temp);
1705	}
1706	else
1707	delete_insns_since (last);
1708	}
1709	}
1710
1711	/* If operation is commutative,
1712	try to make the first operand a register.
1713	Even better, try to make it the same as the target.
1714	Also try to make the last operand a constant. */
1715	if (commutative_optab_p (binoptab)
1716	&& swap_commutative_operands_with_target (target, op0, op1))
1717	std::swap (a&: op0, b&: op1);
1718
1719	/* These can be done a word at a time. */
1720	if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1721	&& is_int_mode (mode, int_mode: &int_mode)
1722	&& GET_MODE_SIZE (mode: int_mode) > UNITS_PER_WORD
1723	&& optab_handler (op: binoptab, mode: word_mode) != CODE_FOR_nothing)
1724	{
1725	int i;
1726	rtx_insn *insns;
1727
1728	/* If TARGET is the same as one of the operands, the REG_EQUAL note
1729	won't be accurate, so use a new target. */
1730	if (target == 0
1731	|| target == op0
1732	|| target == op1
1733	|| reg_overlap_mentioned_p (target, op0)
1734	|| reg_overlap_mentioned_p (target, op1)
1735	|| !valid_multiword_target_p (target))
1736	target = gen_reg_rtx (int_mode);
1737
1738	start_sequence ();
1739
1740	/* Do the actual arithmetic. */
1741	machine_mode op0_mode = GET_MODE (op0);
1742	machine_mode op1_mode = GET_MODE (op1);
1743	if (op0_mode == VOIDmode)
1744	op0_mode = int_mode;
1745	if (op1_mode == VOIDmode)
1746	op1_mode = int_mode;
1747	for (i = 0; i < GET_MODE_BITSIZE (mode: int_mode) / BITS_PER_WORD; i++)
1748	{
1749	rtx target_piece = operand_subword (target, i, 1, int_mode);
1750	rtx x = expand_binop (mode: word_mode, binoptab,
1751	op0: operand_subword_force (op0, i, op0_mode),
1752	op1: operand_subword_force (op1, i, op1_mode),
1753	target: target_piece, unsignedp, methods: next_methods);
1754
1755	if (x == 0)
1756	break;
1757
1758	if (target_piece != x)
1759	emit_move_insn (target_piece, x);
1760	}
1761
1762	insns = get_insns ();
1763	end_sequence ();
1764
1765	if (i == GET_MODE_BITSIZE (mode: int_mode) / BITS_PER_WORD)
1766	{
1767	emit_insn (insns);
1768	return target;
1769	}
1770	}
1771
1772	/* Synthesize double word shifts from single word shifts. */
1773	if ((binoptab == lshr_optab || binoptab == ashl_optab
1774	|| binoptab == ashr_optab)
1775	&& is_int_mode (mode, int_mode: &int_mode)
1776	&& (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1777	&& GET_MODE_SIZE (mode: int_mode) == 2 * UNITS_PER_WORD
1778	&& GET_MODE_PRECISION (mode: int_mode) == GET_MODE_BITSIZE (mode: int_mode)
1779	&& optab_handler (op: binoptab, mode: word_mode) != CODE_FOR_nothing
1780	&& optab_handler (op: ashl_optab, mode: word_mode) != CODE_FOR_nothing
1781	&& optab_handler (op: lshr_optab, mode: word_mode) != CODE_FOR_nothing)
1782	{
1783	unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1784	scalar_int_mode op1_mode;
1785
1786	double_shift_mask = targetm.shift_truncation_mask (int_mode);
1787	shift_mask = targetm.shift_truncation_mask (word_mode);
1788	op1_mode = (GET_MODE (op1) != VOIDmode
1789	? as_a <scalar_int_mode> (GET_MODE (op1))
1790	: word_mode);
1791
1792	/* Apply the truncation to constant shifts. */
1793	if (double_shift_mask > 0 && CONST_INT_P (op1))
1794	op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
1795
1796	if (op1 == CONST0_RTX (op1_mode))
1797	return op0;
1798
1799	/* Make sure that this is a combination that expand_doubleword_shift
1800	can handle. See the comments there for details. */
1801	if (double_shift_mask == 0
1802	|| (shift_mask == BITS_PER_WORD - 1
1803	&& double_shift_mask == BITS_PER_WORD * 2 - 1))
1804	{
1805	rtx_insn *insns;
1806	rtx into_target, outof_target;
1807	rtx into_input, outof_input;
1808	int left_shift, outof_word;
1809
1810	/* If TARGET is the same as one of the operands, the REG_EQUAL note
1811	won't be accurate, so use a new target. */
1812	if (target == 0
1813	|| target == op0
1814	|| target == op1
1815	|| reg_overlap_mentioned_p (target, op0)
1816	|| reg_overlap_mentioned_p (target, op1)
1817	|| !valid_multiword_target_p (target))
1818	target = gen_reg_rtx (int_mode);
1819
1820	start_sequence ();
1821
1822	/* OUTOF_* is the word we are shifting bits away from, and
1823	INTO_* is the word that we are shifting bits towards, thus
1824	they differ depending on the direction of the shift and
1825	WORDS_BIG_ENDIAN. */
1826
1827	left_shift = binoptab == ashl_optab;
1828	outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1829
1830	outof_target = operand_subword (target, outof_word, 1, int_mode);
1831	into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1832
1833	outof_input = operand_subword_force (op0, outof_word, int_mode);
1834	into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1835
1836	if (expand_doubleword_shift (op1_mode, binoptab,
1837	outof_input, into_input, op1,
1838	outof_target, into_target,
1839	unsignedp, methods: next_methods, shift_mask))
1840	{
1841	insns = get_insns ();
1842	end_sequence ();
1843
1844	emit_insn (insns);
1845	return target;
1846	}
1847	end_sequence ();
1848	}
1849	}
1850
1851	/* Synthesize double word rotates from single word shifts. */
1852	if ((binoptab == rotl_optab || binoptab == rotr_optab)
1853	&& is_int_mode (mode, int_mode: &int_mode)
1854	&& CONST_INT_P (op1)
1855	&& GET_MODE_PRECISION (mode: int_mode) == 2 * BITS_PER_WORD
1856	&& optab_handler (op: ashl_optab, mode: word_mode) != CODE_FOR_nothing
1857	&& optab_handler (op: lshr_optab, mode: word_mode) != CODE_FOR_nothing)
1858	{
1859	rtx_insn *insns;
1860	rtx into_target, outof_target;
1861	rtx into_input, outof_input;
1862	rtx inter;
1863	int shift_count, left_shift, outof_word;
1864
1865	/* If TARGET is the same as one of the operands, the REG_EQUAL note
1866	won't be accurate, so use a new target. Do this also if target is not
1867	a REG, first because having a register instead may open optimization
1868	opportunities, and second because if target and op0 happen to be MEMs
1869	designating the same location, we would risk clobbering it too early
1870	in the code sequence we generate below. */
1871	if (target == 0
1872	|| target == op0
1873	|| target == op1
1874	|| !REG_P (target)
1875	|| reg_overlap_mentioned_p (target, op0)
1876	|| reg_overlap_mentioned_p (target, op1)
1877	|| !valid_multiword_target_p (target))
1878	target = gen_reg_rtx (int_mode);
1879
1880	start_sequence ();
1881
1882	shift_count = INTVAL (op1);
1883
1884	/* OUTOF_* is the word we are shifting bits away from, and
1885	INTO_* is the word that we are shifting bits towards, thus
1886	they differ depending on the direction of the shift and
1887	WORDS_BIG_ENDIAN. */
1888
1889	left_shift = (binoptab == rotl_optab);
1890	outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1891
1892	outof_target = operand_subword (target, outof_word, 1, int_mode);
1893	into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1894
1895	outof_input = operand_subword_force (op0, outof_word, int_mode);
1896	into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1897
1898	if (shift_count == BITS_PER_WORD)
1899	{
1900	/* This is just a word swap. */
1901	emit_move_insn (outof_target, into_input);
1902	emit_move_insn (into_target, outof_input);
1903	inter = const0_rtx;
1904	}
1905	else
1906	{
1907	rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1908	HOST_WIDE_INT first_shift_count, second_shift_count;
1909	optab reverse_unsigned_shift, unsigned_shift;
1910
1911	reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1912	? lshr_optab : ashl_optab);
1913
1914	unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1915	? ashl_optab : lshr_optab);
1916
1917	if (shift_count > BITS_PER_WORD)
1918	{
1919	first_shift_count = shift_count - BITS_PER_WORD;
1920	second_shift_count = 2 * BITS_PER_WORD - shift_count;
1921	}
1922	else
1923	{
1924	first_shift_count = BITS_PER_WORD - shift_count;
1925	second_shift_count = shift_count;
1926	}
1927	rtx first_shift_count_rtx
1928	= gen_int_shift_amount (word_mode, first_shift_count);
1929	rtx second_shift_count_rtx
1930	= gen_int_shift_amount (word_mode, second_shift_count);
1931
1932	into_temp1 = expand_binop (mode: word_mode, binoptab: unsigned_shift,
1933	op0: outof_input, op1: first_shift_count_rtx,
1934	NULL_RTX, unsignedp, methods: next_methods);
1935	into_temp2 = expand_binop (mode: word_mode, binoptab: reverse_unsigned_shift,
1936	op0: into_input, op1: second_shift_count_rtx,
1937	NULL_RTX, unsignedp, methods: next_methods);
1938
1939	if (into_temp1 != 0 && into_temp2 != 0)
1940	inter = expand_binop (mode: word_mode, binoptab: ior_optab, op0: into_temp1, op1: into_temp2,
1941	target: into_target, unsignedp, methods: next_methods);
1942	else
1943	inter = 0;
1944
1945	if (inter != 0 && inter != into_target)
1946	emit_move_insn (into_target, inter);
1947
1948	outof_temp1 = expand_binop (mode: word_mode, binoptab: unsigned_shift,
1949	op0: into_input, op1: first_shift_count_rtx,
1950	NULL_RTX, unsignedp, methods: next_methods);
1951	outof_temp2 = expand_binop (mode: word_mode, binoptab: reverse_unsigned_shift,
1952	op0: outof_input, op1: second_shift_count_rtx,
1953	NULL_RTX, unsignedp, methods: next_methods);
1954
1955	if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1956	inter = expand_binop (mode: word_mode, binoptab: ior_optab,
1957	op0: outof_temp1, op1: outof_temp2,
1958	target: outof_target, unsignedp, methods: next_methods);
1959
1960	if (inter != 0 && inter != outof_target)
1961	emit_move_insn (outof_target, inter);
1962	}
1963
1964	insns = get_insns ();
1965	end_sequence ();
1966
1967	if (inter != 0)
1968	{
1969	emit_insn (insns);
1970	return target;
1971	}
1972	}
1973
1974	/* These can be done a word at a time by propagating carries. */
1975	if ((binoptab == add_optab || binoptab == sub_optab)
1976	&& is_int_mode (mode, int_mode: &int_mode)
1977	&& GET_MODE_SIZE (mode: int_mode) >= 2 * UNITS_PER_WORD
1978	&& optab_handler (op: binoptab, mode: word_mode) != CODE_FOR_nothing)
1979	{
1980	unsigned int i;
1981	optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
1982	const unsigned int nwords = GET_MODE_BITSIZE (mode: int_mode) / BITS_PER_WORD;
1983	rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
1984	rtx xop0, xop1, xtarget;
1985
1986	/* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1987	value is one of those, use it. Otherwise, use 1 since it is the
1988	one easiest to get. */
1989	#if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1990	int normalizep = STORE_FLAG_VALUE;
1991	#else
1992	int normalizep = 1;
1993	#endif
1994
1995	/* Prepare the operands. */
1996	xop0 = force_reg (int_mode, op0);
1997	xop1 = force_reg (int_mode, op1);
1998
1999	xtarget = gen_reg_rtx (int_mode);
2000
2001	if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
2002	target = xtarget;
2003
2004	/* Indicate for flow that the entire target reg is being set. */
2005	if (REG_P (target))
2006	emit_clobber (xtarget);
2007
2008	/* Do the actual arithmetic. */
2009	for (i = 0; i < nwords; i++)
2010	{
2011	int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
2012	rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
2013	rtx op0_piece = operand_subword_force (xop0, index, int_mode);
2014	rtx op1_piece = operand_subword_force (xop1, index, int_mode);
2015	rtx x;
2016
2017	/* Main add/subtract of the input operands. */
2018	x = expand_binop (mode: word_mode, binoptab,
2019	op0: op0_piece, op1: op1_piece,
2020	target: target_piece, unsignedp, methods: next_methods);
2021	if (x == 0)
2022	break;
2023
2024	if (i + 1 < nwords)
2025	{
2026	/* Store carry from main add/subtract. */
2027	carry_out = gen_reg_rtx (word_mode);
2028	carry_out = emit_store_flag_force (carry_out,
2029	(binoptab == add_optab
2030	? LT : GT),
2031	x, op0_piece,
2032	word_mode, 1, normalizep);
2033	}
2034
2035	if (i > 0)
2036	{
2037	rtx newx;
2038
2039	/* Add/subtract previous carry to main result. */
2040	newx = expand_binop (mode: word_mode,
2041	binoptab: normalizep == 1 ? binoptab : otheroptab,
2042	op0: x, op1: carry_in,
2043	NULL_RTX, unsignedp: 1, methods: next_methods);
2044
2045	if (i + 1 < nwords)
2046	{
2047	/* Get out carry from adding/subtracting carry in. */
2048	rtx carry_tmp = gen_reg_rtx (word_mode);
2049	carry_tmp = emit_store_flag_force (carry_tmp,
2050	(binoptab == add_optab
2051	? LT : GT),
2052	newx, x,
2053	word_mode, 1, normalizep);
2054
2055	/* Logical-ior the two poss. carry together. */
2056	carry_out = expand_binop (mode: word_mode, binoptab: ior_optab,
2057	op0: carry_out, op1: carry_tmp,
2058	target: carry_out, unsignedp: 0, methods: next_methods);
2059	if (carry_out == 0)
2060	break;
2061	}
2062	emit_move_insn (target_piece, newx);
2063	}
2064	else
2065	{
2066	if (x != target_piece)
2067	emit_move_insn (target_piece, x);
2068	}
2069
2070	carry_in = carry_out;
2071	}
2072
2073	if (i == GET_MODE_BITSIZE (mode: int_mode) / (unsigned) BITS_PER_WORD)
2074	{
2075	if (optab_handler (op: mov_optab, mode: int_mode) != CODE_FOR_nothing
2076	|| ! rtx_equal_p (target, xtarget))
2077	{
2078	rtx_insn *temp = emit_move_insn (target, xtarget);
2079
2080	set_dst_reg_note (temp, REG_EQUAL,
2081	gen_rtx_fmt_ee (optab_to_code (binoptab),
2082	int_mode, copy_rtx (xop0),
2083	copy_rtx (xop1)),
2084	target);
2085	}
2086	else
2087	target = xtarget;
2088
2089	return target;
2090	}
2091
2092	else
2093	delete_insns_since (last);
2094	}
2095
2096	/* Attempt to synthesize double word multiplies using a sequence of word
2097	mode multiplications. We first attempt to generate a sequence using a
2098	more efficient unsigned widening multiply, and if that fails we then
2099	try using a signed widening multiply. */
2100
2101	if (binoptab == smul_optab
2102	&& is_int_mode (mode, int_mode: &int_mode)
2103	&& GET_MODE_SIZE (mode: int_mode) == 2 * UNITS_PER_WORD
2104	&& optab_handler (op: smul_optab, mode: word_mode) != CODE_FOR_nothing
2105	&& optab_handler (op: add_optab, mode: word_mode) != CODE_FOR_nothing)
2106	{
2107	rtx product = NULL_RTX;
2108	if (convert_optab_handler (op: umul_widen_optab, to_mode: int_mode, from_mode: word_mode)
2109	!= CODE_FOR_nothing)
2110	{
2111	product = expand_doubleword_mult (mode: int_mode, op0, op1, target,
2112	umulp: true, methods);
2113	if (!product)
2114	delete_insns_since (last);
2115	}
2116
2117	if (product == NULL_RTX
2118	&& (convert_optab_handler (op: smul_widen_optab, to_mode: int_mode, from_mode: word_mode)
2119	!= CODE_FOR_nothing))
2120	{
2121	product = expand_doubleword_mult (mode: int_mode, op0, op1, target,
2122	umulp: false, methods);
2123	if (!product)
2124	delete_insns_since (last);
2125	}
2126
2127	if (product != NULL_RTX)
2128	{
2129	if (optab_handler (op: mov_optab, mode: int_mode) != CODE_FOR_nothing)
2130	{
2131	rtx_insn *move = emit_move_insn (target ? target : product,
2132	product);
2133	set_dst_reg_note (move,
2134	REG_EQUAL,
2135	gen_rtx_fmt_ee (MULT, int_mode,
2136	copy_rtx (op0),
2137	copy_rtx (op1)),
2138	target ? target : product);
2139	}
2140	return product;
2141	}
2142	}
2143
2144	/* Attempt to synthetize double word modulo by constant divisor. */
2145	if ((binoptab == umod_optab
2146	|| binoptab == smod_optab
2147	|| binoptab == udiv_optab
2148	|| binoptab == sdiv_optab)
2149	&& optimize
2150	&& CONST_INT_P (op1)
2151	&& is_int_mode (mode, int_mode: &int_mode)
2152	&& GET_MODE_SIZE (mode: int_mode) == 2 * UNITS_PER_WORD
2153	&& optab_handler (op: (binoptab == umod_optab || binoptab == udiv_optab)
2154	? udivmod_optab : sdivmod_optab,
2155	mode: int_mode) == CODE_FOR_nothing
2156	&& optab_handler (op: and_optab, mode: word_mode) != CODE_FOR_nothing
2157	&& optab_handler (op: add_optab, mode: word_mode) != CODE_FOR_nothing
2158	&& optimize_insn_for_speed_p ())
2159	{
2160	rtx res = NULL_RTX;
2161	if ((binoptab == umod_optab || binoptab == smod_optab)
2162	&& (INTVAL (op1) & 1) == 0)
2163	res = expand_doubleword_mod (mode: int_mode, op0, op1,
2164	unsignedp: binoptab == umod_optab);
2165	else
2166	{
2167	rtx quot = expand_doubleword_divmod (mode: int_mode, op0, op1, rem: &res,
2168	unsignedp: binoptab == umod_optab
2169	|| binoptab == udiv_optab);
2170	if (quot == NULL_RTX)
2171	res = NULL_RTX;
2172	else if (binoptab == udiv_optab || binoptab == sdiv_optab)
2173	res = quot;
2174	}
2175	if (res != NULL_RTX)
2176	{
2177	if (optab_handler (op: mov_optab, mode: int_mode) != CODE_FOR_nothing)
2178	{
2179	rtx_insn *move = emit_move_insn (target ? target : res,
2180	res);
2181	set_dst_reg_note (move, REG_EQUAL,
2182	gen_rtx_fmt_ee (optab_to_code (binoptab),
2183	int_mode, copy_rtx (op0), op1),
2184	target ? target : res);
2185	}
2186	return res;
2187	}
2188	else
2189	delete_insns_since (last);
2190	}
2191
2192	/* It can't be open-coded in this mode.
2193	Use a library call if one is available and caller says that's ok. */
2194
2195	libfunc = optab_libfunc (binoptab, mode);
2196	if (libfunc
2197	&& (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
2198	{
2199	rtx_insn *insns;
2200	rtx op1x = op1;
2201	machine_mode op1_mode = mode;
2202	rtx value;
2203
2204	start_sequence ();
2205
2206	if (shift_optab_p (binoptab))
2207	{
2208	op1_mode = targetm.libgcc_shift_count_mode ();
2209	/* Specify unsigned here,
2210	since negative shift counts are meaningless. */
2211	op1x = convert_to_mode (op1_mode, op1, 1);
2212	}
2213
2214	if (GET_MODE (op0) != VOIDmode
2215	&& GET_MODE (op0) != mode)
2216	op0 = convert_to_mode (mode, op0, unsignedp);
2217
2218	/* Pass 1 for NO_QUEUE so we don't lose any increments
2219	if the libcall is cse'd or moved. */
2220	value = emit_library_call_value (fun: libfunc,
2221	NULL_RTX, fn_type: LCT_CONST, outmode: mode,
2222	arg1: op0, arg1_mode: mode, arg2: op1x, arg2_mode: op1_mode);
2223
2224	insns = get_insns ();
2225	end_sequence ();
2226
2227	bool trapv = trapv_binoptab_p (binoptab);
2228	target = gen_reg_rtx (mode);
2229	emit_libcall_block_1 (insns, target, value,
2230	trapv ? NULL_RTX
2231	: gen_rtx_fmt_ee (optab_to_code (binoptab),
2232	mode, op0, op1), trapv);
2233
2234	return target;
2235	}
2236
2237	delete_insns_since (last);
2238
2239	/* It can't be done in this mode. Can we do it in a wider mode? */
2240
2241	if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
2242	|| methods == OPTAB_MUST_WIDEN))
2243	{
2244	/* Caller says, don't even try. */
2245	delete_insns_since (entry_last);
2246	return 0;
2247	}
2248
2249	/* Compute the value of METHODS to pass to recursive calls.
2250	Don't allow widening to be tried recursively. */
2251
2252	methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
2253
2254	/* Look for a wider mode of the same class for which it appears we can do
2255	the operation. */
2256
2257	if (CLASS_HAS_WIDER_MODES_P (mclass))
2258	{
2259	/* This code doesn't make sense for conversion optabs, since we
2260	wouldn't then want to extend the operands to be the same size
2261	as the result. */
2262	gcc_assert (!convert_optab_p (binoptab));
2263	FOR_EACH_WIDER_MODE (wider_mode, mode)
2264	{
2265	if (optab_handler (op: binoptab, mode: wider_mode)
2266	|| (methods == OPTAB_LIB
2267	&& optab_libfunc (binoptab, wider_mode)))
2268	{
2269	rtx xop0 = op0, xop1 = op1;
2270	bool no_extend = false;
2271
2272	/* For certain integer operations, we need not actually extend
2273	the narrow operands, as long as we will truncate
2274	the results to the same narrowness. */
2275
2276	if ((binoptab == ior_optab || binoptab == and_optab
2277	|| binoptab == xor_optab
2278	|| binoptab == add_optab || binoptab == sub_optab
2279	|| binoptab == smul_optab || binoptab == ashl_optab)
2280	&& mclass == MODE_INT)
2281	no_extend = true;
2282
2283	xop0 = widen_operand (op: xop0, mode: wider_mode, oldmode: mode,
2284	unsignedp, no_extend);
2285
2286	/* The second operand of a shift must always be extended. */
2287	xop1 = widen_operand (op: xop1, mode: wider_mode, oldmode: mode, unsignedp,
2288	no_extend: no_extend && binoptab != ashl_optab);
2289
2290	temp = expand_binop (mode: wider_mode, binoptab, op0: xop0, op1: xop1, NULL_RTX,
2291	unsignedp, methods);
2292	if (temp)
2293	{
2294	if (mclass != MODE_INT
2295	|| !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2296	{
2297	if (target == 0)
2298	target = gen_reg_rtx (mode);
2299	convert_move (target, temp, 0);
2300	return target;
2301	}
2302	else
2303	return gen_lowpart (mode, temp);
2304	}
2305	else
2306	delete_insns_since (last);
2307	}
2308	}
2309	}
2310
2311	delete_insns_since (entry_last);
2312	return 0;
2313	}
2314
2315	/* Expand a binary operator which has both signed and unsigned forms.
2316	UOPTAB is the optab for unsigned operations, and SOPTAB is for
2317	signed operations.
2318
2319	If we widen unsigned operands, we may use a signed wider operation instead
2320	of an unsigned wider operation, since the result would be the same. */
2321
2322	rtx
2323	sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
2324	rtx op0, rtx op1, rtx target, int unsignedp,
2325	enum optab_methods methods)
2326	{
2327	rtx temp;
2328	optab direct_optab = unsignedp ? uoptab : soptab;
2329	bool save_enable;
2330
2331	/* Do it without widening, if possible. */
2332	temp = expand_binop (mode, binoptab: direct_optab, op0, op1, target,
2333	unsignedp, methods: OPTAB_DIRECT);
2334	if (temp || methods == OPTAB_DIRECT)
2335	return temp;
2336
2337	/* Try widening to a signed int. Disable any direct use of any
2338	signed insn in the current mode. */
2339	save_enable = swap_optab_enable (soptab, mode, false);
2340
2341	temp = expand_binop (mode, binoptab: soptab, op0, op1, target,
2342	unsignedp, methods: OPTAB_WIDEN);
2343
2344	/* For unsigned operands, try widening to an unsigned int. */
2345	if (!temp && unsignedp)
2346	temp = expand_binop (mode, binoptab: uoptab, op0, op1, target,
2347	unsignedp, methods: OPTAB_WIDEN);
2348	if (temp || methods == OPTAB_WIDEN)
2349	goto egress;
2350
2351	/* Use the right width libcall if that exists. */
2352	temp = expand_binop (mode, binoptab: direct_optab, op0, op1, target,
2353	unsignedp, methods: OPTAB_LIB);
2354	if (temp || methods == OPTAB_LIB)
2355	goto egress;
2356
2357	/* Must widen and use a libcall, use either signed or unsigned. */
2358	temp = expand_binop (mode, binoptab: soptab, op0, op1, target,
2359	unsignedp, methods);
2360	if (!temp && unsignedp)
2361	temp = expand_binop (mode, binoptab: uoptab, op0, op1, target,
2362	unsignedp, methods);
2363
2364	egress:
2365	/* Undo the fiddling above. */
2366	if (save_enable)
2367	swap_optab_enable (soptab, mode, true);
2368	return temp;
2369	}
2370
2371	/* Generate code to perform an operation specified by UNOPPTAB
2372	on operand OP0, with two results to TARG0 and TARG1.
2373	We assume that the order of the operands for the instruction
2374	is TARG0, TARG1, OP0.
2375
2376	Either TARG0 or TARG1 may be zero, but what that means is that
2377	the result is not actually wanted. We will generate it into
2378	a dummy pseudo-reg and discard it. They may not both be zero.
2379
2380	Returns true if this operation can be performed; false if not. */
2381
2382	bool
2383	expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2384	int unsignedp)
2385	{
2386	machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2387	enum mode_class mclass;
2388	machine_mode wider_mode;
2389	rtx_insn *entry_last = get_last_insn ();
2390	rtx_insn *last;
2391
2392	mclass = GET_MODE_CLASS (mode);
2393
2394	if (!targ0)
2395	targ0 = gen_reg_rtx (mode);
2396	if (!targ1)
2397	targ1 = gen_reg_rtx (mode);
2398
2399	/* Record where to go back to if we fail. */
2400	last = get_last_insn ();
2401
2402	if (optab_handler (op: unoptab, mode) != CODE_FOR_nothing)
2403	{
2404	class expand_operand ops[3];
2405	enum insn_code icode = optab_handler (op: unoptab, mode);
2406
2407	create_fixed_operand (op: &ops[0], x: targ0);
2408	create_fixed_operand (op: &ops[1], x: targ1);
2409	create_convert_operand_from (op: &ops[2], value: op0, mode, unsigned_p: unsignedp);
2410	if (maybe_expand_insn (icode, nops: 3, ops))
2411	return true;
2412	}
2413
2414	/* It can't be done in this mode. Can we do it in a wider mode? */
2415
2416	if (CLASS_HAS_WIDER_MODES_P (mclass))
2417	{
2418	FOR_EACH_WIDER_MODE (wider_mode, mode)
2419	{
2420	if (optab_handler (op: unoptab, mode: wider_mode) != CODE_FOR_nothing)
2421	{
2422	rtx t0 = gen_reg_rtx (wider_mode);
2423	rtx t1 = gen_reg_rtx (wider_mode);
2424	rtx cop0 = convert_modes (mode: wider_mode, oldmode: mode, x: op0, unsignedp);
2425
2426	if (expand_twoval_unop (unoptab, op0: cop0, targ0: t0, targ1: t1, unsignedp))
2427	{
2428	convert_move (targ0, t0, unsignedp);
2429	convert_move (targ1, t1, unsignedp);
2430	return true;
2431	}
2432	else
2433	delete_insns_since (last);
2434	}
2435	}
2436	}
2437
2438	delete_insns_since (entry_last);
2439	return false;
2440	}
2441
2442	/* Generate code to perform an operation specified by BINOPTAB
2443	on operands OP0 and OP1, with two results to TARG1 and TARG2.
2444	We assume that the order of the operands for the instruction
2445	is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2446	[(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2447
2448	Either TARG0 or TARG1 may be zero, but what that means is that
2449	the result is not actually wanted. We will generate it into
2450	a dummy pseudo-reg and discard it. They may not both be zero.
2451
2452	Returns true if this operation can be performed; false if not. */
2453
2454	bool
2455	expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2456	int unsignedp)
2457	{
2458	machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2459	enum mode_class mclass;
2460	machine_mode wider_mode;
2461	rtx_insn *entry_last = get_last_insn ();
2462	rtx_insn *last;
2463
2464	mclass = GET_MODE_CLASS (mode);
2465
2466	if (!targ0)
2467	targ0 = gen_reg_rtx (mode);
2468	if (!targ1)
2469	targ1 = gen_reg_rtx (mode);
2470
2471	/* Record where to go back to if we fail. */
2472	last = get_last_insn ();
2473
2474	if (optab_handler (op: binoptab, mode) != CODE_FOR_nothing)
2475	{
2476	class expand_operand ops[4];
2477	enum insn_code icode = optab_handler (op: binoptab, mode);
2478	machine_mode mode0 = insn_data[icode].operand[1].mode;
2479	machine_mode mode1 = insn_data[icode].operand[2].mode;
2480	rtx xop0 = op0, xop1 = op1;
2481
2482	/* If we are optimizing, force expensive constants into a register. */
2483	xop0 = avoid_expensive_constant (mode: mode0, binoptab, opn: 0, x: xop0, unsignedp);
2484	xop1 = avoid_expensive_constant (mode: mode1, binoptab, opn: 1, x: xop1, unsignedp);
2485
2486	create_fixed_operand (op: &ops[0], x: targ0);
2487	create_convert_operand_from (op: &ops[1], value: xop0, mode, unsigned_p: unsignedp);
2488	create_convert_operand_from (op: &ops[2], value: xop1, mode, unsigned_p: unsignedp);
2489	create_fixed_operand (op: &ops[3], x: targ1);
2490	if (maybe_expand_insn (icode, nops: 4, ops))
2491	return true;
2492	delete_insns_since (last);
2493	}
2494
2495	/* It can't be done in this mode. Can we do it in a wider mode? */
2496
2497	if (CLASS_HAS_WIDER_MODES_P (mclass))
2498	{
2499	FOR_EACH_WIDER_MODE (wider_mode, mode)
2500	{
2501	if (optab_handler (op: binoptab, mode: wider_mode) != CODE_FOR_nothing)
2502	{
2503	rtx t0 = gen_reg_rtx (wider_mode);
2504	rtx t1 = gen_reg_rtx (wider_mode);
2505	rtx cop0 = convert_modes (mode: wider_mode, oldmode: mode, x: op0, unsignedp);
2506	rtx cop1 = convert_modes (mode: wider_mode, oldmode: mode, x: op1, unsignedp);
2507
2508	if (expand_twoval_binop (binoptab, op0: cop0, op1: cop1,
2509	targ0: t0, targ1: t1, unsignedp))
2510	{
2511	convert_move (targ0, t0, unsignedp);
2512	convert_move (targ1, t1, unsignedp);
2513	return true;
2514	}
2515	else
2516	delete_insns_since (last);
2517	}
2518	}
2519	}
2520
2521	delete_insns_since (entry_last);
2522	return false;
2523	}
2524
2525	/* Expand the two-valued library call indicated by BINOPTAB, but
2526	preserve only one of the values. If TARG0 is non-NULL, the first
2527	value is placed into TARG0; otherwise the second value is placed
2528	into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2529	value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2530	This routine assumes that the value returned by the library call is
2531	as if the return value was of an integral mode twice as wide as the
2532	mode of OP0. Returns 1 if the call was successful. */
2533
2534	bool
2535	expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2536	rtx targ0, rtx targ1, enum rtx_code code)
2537	{
2538	machine_mode mode;
2539	machine_mode libval_mode;
2540	rtx libval;
2541	rtx_insn *insns;
2542	rtx libfunc;
2543
2544	/* Exactly one of TARG0 or TARG1 should be non-NULL. */
2545	gcc_assert (!targ0 != !targ1);
2546
2547	mode = GET_MODE (op0);
2548	libfunc = optab_libfunc (binoptab, mode);
2549	if (!libfunc)
2550	return false;
2551
2552	/* The value returned by the library function will have twice as
2553	many bits as the nominal MODE. */
2554	libval_mode = smallest_int_mode_for_size (size: 2 * GET_MODE_BITSIZE (mode));
2555	start_sequence ();
2556	libval = emit_library_call_value (fun: libfunc, NULL_RTX, fn_type: LCT_CONST,
2557	outmode: libval_mode,
2558	arg1: op0, arg1_mode: mode,
2559	arg2: op1, arg2_mode: mode);
2560	/* Get the part of VAL containing the value that we want. */
2561	libval = simplify_gen_subreg (outermode: mode, op: libval, innermode: libval_mode,
2562	byte: targ0 ? 0 : GET_MODE_SIZE (mode));
2563	insns = get_insns ();
2564	end_sequence ();
2565	/* Move the into the desired location. */
2566	emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2567	gen_rtx_fmt_ee (code, mode, op0, op1));
2568
2569	return true;
2570	}
2571
2572
2573	/* Wrapper around expand_unop which takes an rtx code to specify
2574	the operation to perform, not an optab pointer. All other
2575	arguments are the same. */
2576	rtx
2577	expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2578	rtx target, int unsignedp)
2579	{
2580	optab unop = code_to_optab (code);
2581	gcc_assert (unop);
2582
2583	return expand_unop (mode, unop, op0, target, unsignedp);
2584	}
2585
2586	/* Try calculating
2587	(clz:narrow x)
2588	as
2589	(clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2590
2591	A similar operation can be used for clrsb. UNOPTAB says which operation
2592	we are trying to expand. */
2593	static rtx
2594	widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2595	{
2596	opt_scalar_int_mode wider_mode_iter;
2597	FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2598	{
2599	scalar_int_mode wider_mode = wider_mode_iter.require ();
2600	if (optab_handler (op: unoptab, mode: wider_mode) != CODE_FOR_nothing)
2601	{
2602	rtx xop0, temp;
2603	rtx_insn *last;
2604
2605	last = get_last_insn ();
2606
2607	if (target == 0)
2608	target = gen_reg_rtx (mode);
2609	xop0 = widen_operand (op: op0, mode: wider_mode, oldmode: mode,
2610	unsignedp: unoptab != clrsb_optab, no_extend: false);
2611	temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2612	unoptab != clrsb_optab);
2613	if (temp != 0)
2614	temp = expand_binop
2615	(mode: wider_mode, binoptab: sub_optab, op0: temp,
2616	op1: gen_int_mode (GET_MODE_PRECISION (mode: wider_mode)
2617	- GET_MODE_PRECISION (mode),
2618	wider_mode),
2619	target, unsignedp: true, methods: OPTAB_DIRECT);
2620	if (temp == 0)
2621	delete_insns_since (last);
2622
2623	return temp;
2624	}
2625	}
2626	return 0;
2627	}
2628
2629	/* Attempt to emit (clrsb:mode op0) as
2630	(plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1))))
2631	(const_int -1))
2632	if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec,
2633	or as
2634	(clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1))
2635	(ashr:mode op0 (const_int prec-1)))
2636	(const_int 1)))
2637	otherwise. */
2638
2639	static rtx
2640	expand_clrsb_using_clz (scalar_int_mode mode, rtx op0, rtx target)
2641	{
2642	if (optimize_insn_for_size_p ()
2643	|| optab_handler (op: clz_optab, mode) == CODE_FOR_nothing)
2644	return NULL_RTX;
2645
2646	start_sequence ();
2647	HOST_WIDE_INT val = 0;
2648	if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) != 2
2649	|| val != GET_MODE_PRECISION (mode))
2650	val = 0;
2651	else
2652	val = 1;
2653
2654	rtx temp2 = op0;
2655	if (!val)
2656	{
2657	temp2 = expand_binop (mode, binoptab: ashl_optab, op0, const1_rtx,
2658	NULL_RTX, unsignedp: 0, methods: OPTAB_DIRECT);
2659	if (!temp2)
2660	{
2661	fail:
2662	end_sequence ();
2663	return NULL_RTX;
2664	}
2665	}
2666
2667	rtx temp = expand_binop (mode, binoptab: ashr_optab, op0,
2668	GEN_INT (GET_MODE_PRECISION (mode) - 1),
2669	NULL_RTX, unsignedp: 0, methods: OPTAB_DIRECT);
2670	if (!temp)
2671	goto fail;
2672
2673	temp = expand_binop (mode, binoptab: xor_optab, op0: temp2, op1: temp, NULL_RTX, unsignedp: 0,
2674	methods: OPTAB_DIRECT);
2675	if (!temp)
2676	goto fail;
2677
2678	if (!val)
2679	{
2680	temp = expand_binop (mode, binoptab: ior_optab, op0: temp, const1_rtx,
2681	NULL_RTX, unsignedp: 0, methods: OPTAB_DIRECT);
2682	if (!temp)
2683	goto fail;
2684	}
2685	temp = expand_unop_direct (mode, clz_optab, temp, val ? NULL_RTX : target,
2686	true);
2687	if (!temp)
2688	goto fail;
2689	if (val)
2690	{
2691	temp = expand_binop (mode, binoptab: add_optab, op0: temp, constm1_rtx,
2692	target, unsignedp: 0, methods: OPTAB_DIRECT);
2693	if (!temp)
2694	goto fail;
2695	}
2696
2697	rtx_insn *seq = get_insns ();
2698	end_sequence ();
2699
2700	add_equal_note (insns: seq, target: temp, code: CLRSB, op0, NULL_RTX, op0_mode: mode);
2701	emit_insn (seq);
2702	return temp;
2703	}
2704
2705	static rtx expand_ffs (scalar_int_mode, rtx, rtx);
2706
2707	/* Try calculating clz, ctz or ffs of a double-word quantity as two clz, ctz or
2708	ffs operations on word-sized quantities, choosing which based on whether the
2709	high (for clz) or low (for ctz and ffs) word is nonzero. */
2710	static rtx
2711	expand_doubleword_clz_ctz_ffs (scalar_int_mode mode, rtx op0, rtx target,
2712	optab unoptab)
2713	{
2714	rtx xop0 = force_reg (mode, op0);
2715	rtx subhi = gen_highpart (word_mode, xop0);
2716	rtx sublo = gen_lowpart (word_mode, xop0);
2717	rtx_code_label *hi0_label = gen_label_rtx ();
2718	rtx_code_label *after_label = gen_label_rtx ();
2719	rtx_insn *seq;
2720	rtx temp, result;
2721	int addend = 0;
2722
2723	/* If we were not given a target, use a word_mode register, not a
2724	'mode' register. The result will fit, and nobody is expecting
2725	anything bigger (the return type of __builtin_clz* is int). */
2726	if (!target)
2727	target = gen_reg_rtx (word_mode);
2728
2729	/* In any case, write to a word_mode scratch in both branches of the
2730	conditional, so we can ensure there is a single move insn setting
2731	'target' to tag a REG_EQUAL note on. */
2732	result = gen_reg_rtx (word_mode);
2733
2734	if (unoptab != clz_optab)
2735	std::swap (a&: subhi, b&: sublo);
2736
2737	start_sequence ();
2738
2739	/* If the high word is not equal to zero,
2740	then clz of the full value is clz of the high word. */
2741	emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2742	word_mode, true, hi0_label);
2743
2744	if (optab_handler (op: unoptab, mode: word_mode) != CODE_FOR_nothing)
2745	temp = expand_unop_direct (word_mode, unoptab, subhi, result, true);
2746	else
2747	{
2748	gcc_assert (unoptab == ffs_optab);
2749	temp = expand_ffs (word_mode, subhi, result);
2750	}
2751	if (!temp)
2752	goto fail;
2753
2754	if (temp != result)
2755	convert_move (result, temp, true);
2756
2757	emit_jump_insn (targetm.gen_jump (after_label));
2758	emit_barrier ();
2759
2760	/* Else clz of the full value is clz of the low word plus the number
2761	of bits in the high word. Similarly for ctz/ffs of the high word,
2762	except that ffs should be 0 when both words are zero. */
2763	emit_label (hi0_label);
2764
2765	if (unoptab == ffs_optab)
2766	{
2767	convert_move (result, const0_rtx, true);
2768	emit_cmp_and_jump_insns (sublo, CONST0_RTX (word_mode), EQ, 0,
2769	word_mode, true, after_label);
2770	}
2771
2772	if (optab_handler (op: unoptab, mode: word_mode) != CODE_FOR_nothing)
2773	temp = expand_unop_direct (word_mode, unoptab, sublo, NULL_RTX, true);
2774	else
2775	{
2776	gcc_assert (unoptab == ffs_optab);
2777	temp = expand_unop_direct (word_mode, ctz_optab, sublo, NULL_RTX, true);
2778	addend = 1;
2779	}
2780
2781	if (!temp)
2782	goto fail;
2783
2784	temp = expand_binop (mode: word_mode, binoptab: add_optab, op0: temp,
2785	op1: gen_int_mode (GET_MODE_BITSIZE (mode: word_mode) + addend,
2786	word_mode),
2787	target: result, unsignedp: true, methods: OPTAB_DIRECT);
2788	if (!temp)
2789	goto fail;
2790	if (temp != result)
2791	convert_move (result, temp, true);
2792
2793	emit_label (after_label);
2794	convert_move (target, result, true);
2795
2796	seq = get_insns ();
2797	end_sequence ();
2798
2799	add_equal_note (insns: seq, target, code: optab_to_code (op: unoptab), op0: xop0, NULL_RTX, op0_mode: mode);
2800	emit_insn (seq);
2801	return target;
2802
2803	fail:
2804	end_sequence ();
2805	return 0;
2806	}
2807
2808	/* Try calculating popcount of a double-word quantity as two popcount's of
2809	word-sized quantities and summing up the results. */
2810	static rtx
2811	expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2812	{
2813	rtx t0, t1, t;
2814	rtx_insn *seq;
2815
2816	start_sequence ();
2817
2818	t0 = expand_unop_direct (word_mode, popcount_optab,
2819	operand_subword_force (op0, 0, mode), NULL_RTX,
2820	true);
2821	t1 = expand_unop_direct (word_mode, popcount_optab,
2822	operand_subword_force (op0, 1, mode), NULL_RTX,
2823	true);
2824	if (!t0 || !t1)
2825	{
2826	end_sequence ();
2827	return NULL_RTX;
2828	}
2829
2830	/* If we were not given a target, use a word_mode register, not a
2831	'mode' register. The result will fit, and nobody is expecting
2832	anything bigger (the return type of __builtin_popcount* is int). */
2833	if (!target)
2834	target = gen_reg_rtx (word_mode);
2835
2836	t = expand_binop (mode: word_mode, binoptab: add_optab, op0: t0, op1: t1, target, unsignedp: 0, methods: OPTAB_DIRECT);
2837
2838	seq = get_insns ();
2839	end_sequence ();
2840
2841	add_equal_note (insns: seq, target: t, code: POPCOUNT, op0, NULL_RTX, op0_mode: mode);
2842	emit_insn (seq);
2843	return t;
2844	}
2845
2846	/* Try calculating
2847	(parity:wide x)
2848	as
2849	(parity:narrow (low (x) ^ high (x))) */
2850	static rtx
2851	expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2852	{
2853	rtx t = expand_binop (mode: word_mode, binoptab: xor_optab,
2854	op0: operand_subword_force (op0, 0, mode),
2855	op1: operand_subword_force (op0, 1, mode),
2856	NULL_RTX, unsignedp: 0, methods: OPTAB_DIRECT);
2857	return expand_unop (word_mode, parity_optab, t, target, true);
2858	}
2859
2860	/* Try calculating
2861	(bswap:narrow x)
2862	as
2863	(lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2864	static rtx
2865	widen_bswap (scalar_int_mode mode, rtx op0, rtx target)
2866	{
2867	rtx x;
2868	rtx_insn *last;
2869	opt_scalar_int_mode wider_mode_iter;
2870
2871	FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2872	if (optab_handler (op: bswap_optab, mode: wider_mode_iter.require ())
2873	!= CODE_FOR_nothing)
2874	break;
2875
2876	if (!wider_mode_iter.exists ())
2877	return NULL_RTX;
2878
2879	scalar_int_mode wider_mode = wider_mode_iter.require ();
2880	last = get_last_insn ();
2881
2882	x = widen_operand (op: op0, mode: wider_mode, oldmode: mode, unsignedp: true, no_extend: true);
2883	x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2884
2885	gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2886	&& GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2887	if (x != 0)
2888	x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2889	GET_MODE_BITSIZE (mode: wider_mode)
2890	- GET_MODE_BITSIZE (mode),
2891	NULL_RTX, true);
2892
2893	if (x != 0)
2894	{
2895	if (target == 0)
2896	target = gen_reg_rtx (mode);
2897	emit_move_insn (target, gen_lowpart (mode, x));
2898	}
2899	else
2900	delete_insns_since (last);
2901
2902	return target;
2903	}
2904
2905	/* Try calculating bswap as two bswaps of two word-sized operands. */
2906
2907	static rtx
2908	expand_doubleword_bswap (machine_mode mode, rtx op, rtx target)
2909	{
2910	rtx t0, t1;
2911
2912	t1 = expand_unop (word_mode, bswap_optab,
2913	operand_subword_force (op, 0, mode), NULL_RTX, true);
2914	t0 = expand_unop (word_mode, bswap_optab,
2915	operand_subword_force (op, 1, mode), NULL_RTX, true);
2916
2917	if (target == 0 || !valid_multiword_target_p (target))
2918	target = gen_reg_rtx (mode);
2919	if (REG_P (target))
2920	emit_clobber (target);
2921	emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2922	emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2923
2924	return target;
2925	}
2926
2927	/* Try calculating (parity x) as (and (popcount x) 1), where
2928	popcount can also be done in a wider mode. */
2929	static rtx
2930	expand_parity (scalar_int_mode mode, rtx op0, rtx target)
2931	{
2932	enum mode_class mclass = GET_MODE_CLASS (mode);
2933	opt_scalar_int_mode wider_mode_iter;
2934	FOR_EACH_MODE_FROM (wider_mode_iter, mode)
2935	{
2936	scalar_int_mode wider_mode = wider_mode_iter.require ();
2937	if (optab_handler (op: popcount_optab, mode: wider_mode) != CODE_FOR_nothing)
2938	{
2939	rtx xop0, temp;
2940	rtx_insn *last;
2941
2942	last = get_last_insn ();
2943
2944	if (target == 0 || GET_MODE (target) != wider_mode)
2945	target = gen_reg_rtx (wider_mode);
2946
2947	xop0 = widen_operand (op: op0, mode: wider_mode, oldmode: mode, unsignedp: true, no_extend: false);
2948	temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2949	true);
2950	if (temp != 0)
2951	temp = expand_binop (mode: wider_mode, binoptab: and_optab, op0: temp, const1_rtx,
2952	target, unsignedp: true, methods: OPTAB_DIRECT);
2953
2954	if (temp)
2955	{
2956	if (mclass != MODE_INT
2957	|| !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2958	return convert_to_mode (mode, temp, 0);
2959	else
2960	return gen_lowpart (mode, temp);
2961	}
2962	else
2963	delete_insns_since (last);
2964	}
2965	}
2966	return 0;
2967	}
2968
2969	/* Try calculating ctz(x) as K - clz(x & -x) ,
2970	where K is GET_MODE_PRECISION(mode) - 1.
2971
2972	Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2973	don't have to worry about what the hardware does in that case. (If
2974	the clz instruction produces the usual value at 0, which is K, the
2975	result of this code sequence will be -1; expand_ffs, below, relies
2976	on this. It might be nice to have it be K instead, for consistency
2977	with the (very few) processors that provide a ctz with a defined
2978	value, but that would take one more instruction, and it would be
2979	less convenient for expand_ffs anyway. */
2980
2981	static rtx
2982	expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
2983	{
2984	rtx_insn *seq;
2985	rtx temp;
2986
2987	if (optab_handler (op: clz_optab, mode) == CODE_FOR_nothing)
2988	return 0;
2989
2990	start_sequence ();
2991
2992	temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
2993	if (temp)
2994	temp = expand_binop (mode, binoptab: and_optab, op0, op1: temp, NULL_RTX,
2995	unsignedp: true, methods: OPTAB_DIRECT);
2996	if (temp)
2997	temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
2998	if (temp)
2999	temp = expand_binop (mode, binoptab: sub_optab,
3000	op0: gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
3001	op1: temp, target,
3002	unsignedp: true, methods: OPTAB_DIRECT);
3003	if (temp == 0)
3004	{
3005	end_sequence ();
3006	return 0;
3007	}
3008
3009	seq = get_insns ();
3010	end_sequence ();
3011
3012	add_equal_note (insns: seq, target: temp, code: CTZ, op0, NULL_RTX, op0_mode: mode);
3013	emit_insn (seq);
3014	return temp;
3015	}
3016
3017
3018	/* Try calculating ffs(x) using ctz(x) if we have that instruction, or
3019	else with the sequence used by expand_clz.
3020
3021	The ffs builtin promises to return zero for a zero value and ctz/clz
3022	may have an undefined value in that case. If they do not give us a
3023	convenient value, we have to generate a test and branch. */
3024	static rtx
3025	expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
3026	{
3027	HOST_WIDE_INT val = 0;
3028	bool defined_at_zero = false;
3029	rtx temp;
3030	rtx_insn *seq;
3031
3032	if (optab_handler (op: ctz_optab, mode) != CODE_FOR_nothing)
3033	{
3034	start_sequence ();
3035
3036	temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
3037	if (!temp)
3038	goto fail;
3039
3040	defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
3041	}
3042	else if (optab_handler (op: clz_optab, mode) != CODE_FOR_nothing)
3043	{
3044	start_sequence ();
3045	temp = expand_ctz (mode, op0, target: 0);
3046	if (!temp)
3047	goto fail;
3048
3049	if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
3050	{
3051	defined_at_zero = true;
3052	val = (GET_MODE_PRECISION (mode) - 1) - val;
3053	}
3054	}
3055	else
3056	return 0;
3057
3058	if (defined_at_zero && val == -1)
3059	/* No correction needed at zero. */;
3060	else
3061	{
3062	/* We don't try to do anything clever with the situation found
3063	on some processors (eg Alpha) where ctz(0:mode) ==
3064	bitsize(mode). If someone can think of a way to send N to -1
3065	and leave alone all values in the range 0..N-1 (where N is a
3066	power of two), cheaper than this test-and-branch, please add it.
3067
3068	The test-and-branch is done after the operation itself, in case
3069	the operation sets condition codes that can be recycled for this.
3070	(This is true on i386, for instance.) */
3071
3072	rtx_code_label *nonzero_label = gen_label_rtx ();
3073	emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
3074	mode, true, nonzero_label);
3075
3076	convert_move (temp, GEN_INT (-1), false);
3077	emit_label (nonzero_label);
3078	}
3079
3080	/* temp now has a value in the range -1..bitsize-1. ffs is supposed
3081	to produce a value in the range 0..bitsize. */
3082	temp = expand_binop (mode, binoptab: add_optab, op0: temp, op1: gen_int_mode (1, mode),
3083	target, unsignedp: false, methods: OPTAB_DIRECT);
3084	if (!temp)
3085	goto fail;
3086
3087	seq = get_insns ();
3088	end_sequence ();
3089
3090	add_equal_note (insns: seq, target: temp, code: FFS, op0, NULL_RTX, op0_mode: mode);
3091	emit_insn (seq);
3092	return temp;
3093
3094	fail:
3095	end_sequence ();
3096	return 0;
3097	}
3098
3099	/* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
3100	conditions, VAL may already be a SUBREG against which we cannot generate
3101	a further SUBREG. In this case, we expect forcing the value into a
3102	register will work around the situation. */
3103
3104	static rtx
3105	lowpart_subreg_maybe_copy (machine_mode omode, rtx val,
3106	machine_mode imode)
3107	{
3108	rtx ret;
3109	ret = lowpart_subreg (outermode: omode, op: val, innermode: imode);
3110	if (ret == NULL)
3111	{
3112	val = force_reg (imode, val);
3113	ret = lowpart_subreg (outermode: omode, op: val, innermode: imode);
3114	gcc_assert (ret != NULL);
3115	}
3116	return ret;
3117	}
3118
3119	/* Expand a floating point absolute value or negation operation via a
3120	logical operation on the sign bit. */
3121
3122	static rtx
3123	expand_absneg_bit (enum rtx_code code, scalar_float_mode mode,
3124	rtx op0, rtx target)
3125	{
3126	const struct real_format *fmt;
3127	int bitpos, word, nwords, i;
3128	scalar_int_mode imode;
3129	rtx temp;
3130	rtx_insn *insns;
3131
3132	/* The format has to have a simple sign bit. */
3133	fmt = REAL_MODE_FORMAT (mode);
3134	if (fmt == NULL)
3135	return NULL_RTX;
3136
3137	bitpos = fmt->signbit_rw;
3138	if (bitpos < 0)
3139	return NULL_RTX;
3140
3141	/* Don't create negative zeros if the format doesn't support them. */
3142	if (code == NEG && !fmt->has_signed_zero)
3143	return NULL_RTX;
3144
3145	if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3146	{
3147	if (!int_mode_for_mode (mode).exists (mode: &imode))
3148	return NULL_RTX;
3149	word = 0;
3150	nwords = 1;
3151	}
3152	else
3153	{
3154	imode = word_mode;
3155
3156	if (FLOAT_WORDS_BIG_ENDIAN)
3157	word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3158	else
3159	word = bitpos / BITS_PER_WORD;
3160	bitpos = bitpos % BITS_PER_WORD;
3161	nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3162	}
3163
3164	wide_int mask = wi::set_bit_in_zero (bit: bitpos, precision: GET_MODE_PRECISION (mode: imode));
3165	if (code == ABS)
3166	mask = ~mask;
3167
3168	if (target == 0
3169	|| target == op0
3170	|| reg_overlap_mentioned_p (target, op0)
3171	|| (nwords > 1 && !valid_multiword_target_p (target)))
3172	target = gen_reg_rtx (mode);
3173
3174	if (nwords > 1)
3175	{
3176	start_sequence ();
3177
3178	for (i = 0; i < nwords; ++i)
3179	{
3180	rtx targ_piece = operand_subword (target, i, 1, mode);
3181	rtx op0_piece = operand_subword_force (op0, i, mode);
3182
3183	if (i == word)
3184	{
3185	temp = expand_binop (mode: imode, binoptab: code == ABS ? and_optab : xor_optab,
3186	op0: op0_piece,
3187	op1: immed_wide_int_const (mask, imode),
3188	target: targ_piece, unsignedp: 1, methods: OPTAB_LIB_WIDEN);
3189	if (temp != targ_piece)
3190	emit_move_insn (targ_piece, temp);
3191	}
3192	else
3193	emit_move_insn (targ_piece, op0_piece);
3194	}
3195
3196	insns = get_insns ();
3197	end_sequence ();
3198
3199	emit_insn (insns);
3200	}
3201	else
3202	{
3203	temp = expand_binop (mode: imode, binoptab: code == ABS ? and_optab : xor_optab,
3204	gen_lowpart (imode, op0),
3205	op1: immed_wide_int_const (mask, imode),
3206	gen_lowpart (imode, target), unsignedp: 1, methods: OPTAB_LIB_WIDEN);
3207	target = lowpart_subreg_maybe_copy (omode: mode, val: temp, imode);
3208
3209	set_dst_reg_note (get_last_insn (), REG_EQUAL,
3210	gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
3211	target);
3212	}
3213
3214	return target;
3215	}
3216
3217	/* As expand_unop, but will fail rather than attempt the operation in a
3218	different mode or with a libcall. */
3219	static rtx
3220	expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
3221	int unsignedp)
3222	{
3223	if (optab_handler (op: unoptab, mode) != CODE_FOR_nothing)
3224	{
3225	class expand_operand ops[2];
3226	enum insn_code icode = optab_handler (op: unoptab, mode);
3227	rtx_insn *last = get_last_insn ();
3228	rtx_insn *pat;
3229
3230	create_output_operand (op: &ops[0], x: target, mode);
3231	create_convert_operand_from (op: &ops[1], value: op0, mode, unsigned_p: unsignedp);
3232	pat = maybe_gen_insn (icode, nops: 2, ops);
3233	if (pat)
3234	{
3235	if (INSN_P (pat) && NEXT_INSN (insn: pat) != NULL_RTX
3236	&& ! add_equal_note (insns: pat, target: ops[0].value,
3237	code: optab_to_code (op: unoptab),
3238	op0: ops[1].value, NULL_RTX, op0_mode: mode))
3239	{
3240	delete_insns_since (last);
3241	return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
3242	}
3243
3244	emit_insn (pat);
3245
3246	return ops[0].value;
3247	}
3248	}
3249	return 0;
3250	}
3251
3252	/* Generate code to perform an operation specified by UNOPTAB
3253	on operand OP0, with result having machine-mode MODE.
3254
3255	UNSIGNEDP is for the case where we have to widen the operands
3256	to perform the operation. It says to use zero-extension.
3257
3258	If TARGET is nonzero, the value
3259	is generated there, if it is convenient to do so.
3260	In all cases an rtx is returned for the locus of the value;
3261	this may or may not be TARGET. */
3262
3263	rtx
3264	expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
3265	int unsignedp)
3266	{
3267	enum mode_class mclass = GET_MODE_CLASS (mode);
3268	machine_mode wider_mode;
3269	scalar_int_mode int_mode;
3270	scalar_float_mode float_mode;
3271	rtx temp;
3272	rtx libfunc;
3273
3274	temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
3275	if (temp)
3276	return temp;
3277
3278	/* It can't be done in this mode. Can we open-code it in a wider mode? */
3279
3280	/* Widening (or narrowing) clz needs special treatment. */
3281	if (unoptab == clz_optab)
3282	{
3283	if (is_a <scalar_int_mode> (m: mode, result: &int_mode))
3284	{
3285	temp = widen_leading (mode: int_mode, op0, target, unoptab);
3286	if (temp)
3287	return temp;
3288
3289	if (GET_MODE_SIZE (mode: int_mode) == 2 * UNITS_PER_WORD
3290	&& optab_handler (op: unoptab, mode: word_mode) != CODE_FOR_nothing)
3291	{
3292	temp = expand_doubleword_clz_ctz_ffs (mode: int_mode, op0, target,
3293	unoptab);
3294	if (temp)
3295	return temp;
3296	}
3297	}
3298
3299	goto try_libcall;
3300	}
3301
3302	if (unoptab == clrsb_optab)
3303	{
3304	if (is_a <scalar_int_mode> (m: mode, result: &int_mode))
3305	{
3306	temp = widen_leading (mode: int_mode, op0, target, unoptab);
3307	if (temp)
3308	return temp;
3309	temp = expand_clrsb_using_clz (mode: int_mode, op0, target);
3310	if (temp)
3311	return temp;
3312	}
3313	goto try_libcall;
3314	}
3315
3316	if (unoptab == popcount_optab
3317	&& is_a <scalar_int_mode> (m: mode, result: &int_mode)
3318	&& GET_MODE_SIZE (mode: int_mode) == 2 * UNITS_PER_WORD
3319	&& optab_handler (op: unoptab, mode: word_mode) != CODE_FOR_nothing
3320	&& optimize_insn_for_speed_p ())
3321	{
3322	temp = expand_doubleword_popcount (mode: int_mode, op0, target);
3323	if (temp)
3324	return temp;
3325	}
3326
3327	if (unoptab == parity_optab
3328	&& is_a <scalar_int_mode> (m: mode, result: &int_mode)
3329	&& GET_MODE_SIZE (mode: int_mode) == 2 * UNITS_PER_WORD
3330	&& (optab_handler (op: unoptab, mode: word_mode) != CODE_FOR_nothing
3331	|| optab_handler (op: popcount_optab, mode: word_mode) != CODE_FOR_nothing)
3332	&& optimize_insn_for_speed_p ())
3333	{
3334	temp = expand_doubleword_parity (mode: int_mode, op0, target);
3335	if (temp)
3336	return temp;
3337	}
3338
3339	/* Widening (or narrowing) bswap needs special treatment. */
3340	if (unoptab == bswap_optab)
3341	{
3342	/* HImode is special because in this mode BSWAP is equivalent to ROTATE
3343	or ROTATERT. First try these directly; if this fails, then try the
3344	obvious pair of shifts with allowed widening, as this will probably
3345	be always more efficient than the other fallback methods. */
3346	if (mode == HImode)
3347	{
3348	rtx_insn *last;
3349	rtx temp1, temp2;
3350
3351	if (optab_handler (op: rotl_optab, mode) != CODE_FOR_nothing)
3352	{
3353	temp = expand_binop (mode, binoptab: rotl_optab, op0,
3354	op1: gen_int_shift_amount (mode, 8),
3355	target, unsignedp, methods: OPTAB_DIRECT);
3356	if (temp)
3357	return temp;
3358	}
3359
3360	if (optab_handler (op: rotr_optab, mode) != CODE_FOR_nothing)
3361	{
3362	temp = expand_binop (mode, binoptab: rotr_optab, op0,
3363	op1: gen_int_shift_amount (mode, 8),
3364	target, unsignedp, methods: OPTAB_DIRECT);
3365	if (temp)
3366	return temp;
3367	}
3368
3369	last = get_last_insn ();
3370
3371	temp1 = expand_binop (mode, binoptab: ashl_optab, op0,
3372	op1: gen_int_shift_amount (mode, 8), NULL_RTX,
3373	unsignedp, methods: OPTAB_WIDEN);
3374	temp2 = expand_binop (mode, binoptab: lshr_optab, op0,
3375	op1: gen_int_shift_amount (mode, 8), NULL_RTX,
3376	unsignedp, methods: OPTAB_WIDEN);
3377	if (temp1 && temp2)
3378	{
3379	temp = expand_binop (mode, binoptab: ior_optab, op0: temp1, op1: temp2, target,
3380	unsignedp, methods: OPTAB_WIDEN);
3381	if (temp)
3382	return temp;
3383	}
3384
3385	delete_insns_since (last);
3386	}
3387
3388	if (is_a <scalar_int_mode> (m: mode, result: &int_mode))
3389	{
3390	temp = widen_bswap (mode: int_mode, op0, target);
3391	if (temp)
3392	return temp;
3393
3394	/* We do not provide a 128-bit bswap in libgcc so force the use of
3395	a double bswap for 64-bit targets. */
3396	if (GET_MODE_SIZE (mode: int_mode) == 2 * UNITS_PER_WORD
3397	&& (UNITS_PER_WORD == 8
3398	|| optab_handler (op: unoptab, mode: word_mode) != CODE_FOR_nothing))
3399	{
3400	temp = expand_doubleword_bswap (mode, op: op0, target);
3401	if (temp)
3402	return temp;
3403	}
3404	}
3405
3406	goto try_libcall;
3407	}
3408
3409	if (CLASS_HAS_WIDER_MODES_P (mclass))
3410	FOR_EACH_WIDER_MODE (wider_mode, mode)
3411	{
3412	if (optab_handler (op: unoptab, mode: wider_mode) != CODE_FOR_nothing)
3413	{
3414	rtx xop0 = op0;
3415	rtx_insn *last = get_last_insn ();
3416
3417	/* For certain operations, we need not actually extend
3418	the narrow operand, as long as we will truncate the
3419	results to the same narrowness. */
3420
3421	xop0 = widen_operand (op: xop0, mode: wider_mode, oldmode: mode, unsignedp,
3422	no_extend: (unoptab == neg_optab
3423	|| unoptab == one_cmpl_optab)
3424	&& mclass == MODE_INT);
3425
3426	temp = expand_unop (mode: wider_mode, unoptab, op0: xop0, NULL_RTX,
3427	unsignedp);
3428
3429	if (temp)
3430	{
3431	if (mclass != MODE_INT
3432	|| !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3433	{
3434	if (target == 0)
3435	target = gen_reg_rtx (mode);
3436	convert_move (target, temp, 0);
3437	return target;
3438	}
3439	else
3440	return gen_lowpart (mode, temp);
3441	}
3442	else
3443	delete_insns_since (last);
3444	}
3445	}
3446
3447	/* These can be done a word at a time. */
3448	if (unoptab == one_cmpl_optab
3449	&& is_int_mode (mode, int_mode: &int_mode)
3450	&& GET_MODE_SIZE (mode: int_mode) > UNITS_PER_WORD
3451	&& optab_handler (op: unoptab, mode: word_mode) != CODE_FOR_nothing)
3452	{
3453	int i;
3454	rtx_insn *insns;
3455
3456	if (target == 0
3457	|| target == op0
3458	|| reg_overlap_mentioned_p (target, op0)
3459	|| !valid_multiword_target_p (target))
3460	target = gen_reg_rtx (int_mode);
3461
3462	start_sequence ();
3463
3464	/* Do the actual arithmetic. */
3465	for (i = 0; i < GET_MODE_BITSIZE (mode: int_mode) / BITS_PER_WORD; i++)
3466	{
3467	rtx target_piece = operand_subword (target, i, 1, int_mode);
3468	rtx x = expand_unop (mode: word_mode, unoptab,
3469	op0: operand_subword_force (op0, i, int_mode),
3470	target: target_piece, unsignedp);
3471
3472	if (target_piece != x)
3473	emit_move_insn (target_piece, x);
3474	}
3475
3476	insns = get_insns ();
3477	end_sequence ();
3478
3479	emit_insn (insns);
3480	return target;
3481	}
3482
3483	/* Emit ~op0 as op0 ^ -1. */
3484	if (unoptab == one_cmpl_optab
3485	&& (SCALAR_INT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
3486	&& optab_handler (op: xor_optab, mode) != CODE_FOR_nothing)
3487	{
3488	temp = expand_binop (mode, binoptab: xor_optab, op0, CONSTM1_RTX (mode),
3489	target, unsignedp, methods: OPTAB_DIRECT);
3490	if (temp)
3491	return temp;
3492	}
3493
3494	if (optab_to_code (op: unoptab) == NEG)
3495	{
3496	/* Try negating floating point values by flipping the sign bit. */
3497	if (is_a <scalar_float_mode> (m: mode, result: &float_mode))
3498	{
3499	temp = expand_absneg_bit (code: NEG, mode: float_mode, op0, target);
3500	if (temp)
3501	return temp;
3502	}
3503
3504	/* If there is no negation pattern, and we have no negative zero,
3505	try subtracting from zero. */
3506	if (!HONOR_SIGNED_ZEROS (mode))
3507	{
3508	temp = expand_binop (mode, binoptab: (unoptab == negv_optab
3509	? subv_optab : sub_optab),
3510	CONST0_RTX (mode), op1: op0, target,
3511	unsignedp, methods: OPTAB_DIRECT);
3512	if (temp)
3513	return temp;
3514	}
3515	}
3516
3517	/* Try calculating parity (x) as popcount (x) % 2. */
3518	if (unoptab == parity_optab && is_a <scalar_int_mode> (m: mode, result: &int_mode))
3519	{
3520	temp = expand_parity (mode: int_mode, op0, target);
3521	if (temp)
3522	return temp;
3523	}
3524
3525	/* Try implementing ffs (x) in terms of clz (x). */
3526	if (unoptab == ffs_optab && is_a <scalar_int_mode> (m: mode, result: &int_mode))
3527	{
3528	temp = expand_ffs (mode: int_mode, op0, target);
3529	if (temp)
3530	return temp;
3531	}
3532
3533	/* Try implementing ctz (x) in terms of clz (x). */
3534	if (unoptab == ctz_optab && is_a <scalar_int_mode> (m: mode, result: &int_mode))
3535	{
3536	temp = expand_ctz (mode: int_mode, op0, target);
3537	if (temp)
3538	return temp;
3539	}
3540
3541	if ((unoptab == ctz_optab || unoptab == ffs_optab)
3542	&& optimize_insn_for_speed_p ()
3543	&& is_a <scalar_int_mode> (m: mode, result: &int_mode)
3544	&& GET_MODE_SIZE (mode: int_mode) == 2 * UNITS_PER_WORD
3545	&& (optab_handler (op: unoptab, mode: word_mode) != CODE_FOR_nothing
3546	|| optab_handler (op: ctz_optab, mode: word_mode) != CODE_FOR_nothing))
3547	{
3548	temp = expand_doubleword_clz_ctz_ffs (mode: int_mode, op0, target, unoptab);
3549	if (temp)
3550	return temp;
3551	}
3552
3553	try_libcall:
3554	/* Now try a library call in this mode. */
3555	libfunc = optab_libfunc (unoptab, mode);
3556	if (libfunc)
3557	{
3558	rtx_insn *insns;
3559	rtx value;
3560	rtx eq_value;
3561	machine_mode outmode = mode;
3562
3563	/* All of these functions return small values. Thus we choose to
3564	have them return something that isn't a double-word. */
3565	if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3566	|| unoptab == clrsb_optab || unoptab == popcount_optab
3567	|| unoptab == parity_optab)
3568	outmode
3569	= GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3570	optab_libfunc (unoptab, mode)));
3571
3572	start_sequence ();
3573
3574	/* Pass 1 for NO_QUEUE so we don't lose any increments
3575	if the libcall is cse'd or moved. */
3576	value = emit_library_call_value (fun: libfunc, NULL_RTX, fn_type: LCT_CONST, outmode,
3577	arg1: op0, arg1_mode: mode);
3578	insns = get_insns ();
3579	end_sequence ();
3580
3581	target = gen_reg_rtx (outmode);
3582	bool trapv = trapv_unoptab_p (unoptab);
3583	if (trapv)
3584	eq_value = NULL_RTX;
3585	else
3586	{
3587	eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3588	if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
3589	eq_value = simplify_gen_unary (code: TRUNCATE, mode: outmode, op: eq_value, op_mode: mode);
3590	else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3591	eq_value = simplify_gen_unary (code: ZERO_EXTEND,
3592	mode: outmode, op: eq_value, op_mode: mode);
3593	}
3594	emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3595
3596	return target;
3597	}
3598
3599	/* It can't be done in this mode. Can we do it in a wider mode? */
3600
3601	if (CLASS_HAS_WIDER_MODES_P (mclass))
3602	{
3603	FOR_EACH_WIDER_MODE (wider_mode, mode)
3604	{
3605	if (optab_handler (op: unoptab, mode: wider_mode) != CODE_FOR_nothing
3606	|| optab_libfunc (unoptab, wider_mode))
3607	{
3608	rtx xop0 = op0;
3609	rtx_insn *last = get_last_insn ();
3610
3611	/* For certain operations, we need not actually extend
3612	the narrow operand, as long as we will truncate the
3613	results to the same narrowness. */
3614	xop0 = widen_operand (op: xop0, mode: wider_mode, oldmode: mode, unsignedp,
3615	no_extend: (unoptab == neg_optab
3616	|| unoptab == one_cmpl_optab
3617	|| unoptab == bswap_optab)
3618	&& mclass == MODE_INT);
3619
3620	temp = expand_unop (mode: wider_mode, unoptab, op0: xop0, NULL_RTX,
3621	unsignedp);
3622
3623	/* If we are generating clz using wider mode, adjust the
3624	result. Similarly for clrsb. */
3625	if ((unoptab == clz_optab || unoptab == clrsb_optab)
3626	&& temp != 0)
3627	{
3628	scalar_int_mode wider_int_mode
3629	= as_a <scalar_int_mode> (m: wider_mode);
3630	int_mode = as_a <scalar_int_mode> (m: mode);
3631	temp = expand_binop
3632	(mode: wider_mode, binoptab: sub_optab, op0: temp,
3633	op1: gen_int_mode (GET_MODE_PRECISION (mode: wider_int_mode)
3634	- GET_MODE_PRECISION (mode: int_mode),
3635	wider_int_mode),
3636	target, unsignedp: true, methods: OPTAB_DIRECT);
3637	}
3638
3639	/* Likewise for bswap. */
3640	if (unoptab == bswap_optab && temp != 0)
3641	{
3642	scalar_int_mode wider_int_mode
3643	= as_a <scalar_int_mode> (m: wider_mode);
3644	int_mode = as_a <scalar_int_mode> (m: mode);
3645	gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3646	== GET_MODE_BITSIZE (wider_int_mode)
3647	&& GET_MODE_PRECISION (int_mode)
3648	== GET_MODE_BITSIZE (int_mode));
3649
3650	temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3651	GET_MODE_BITSIZE (mode: wider_int_mode)
3652	- GET_MODE_BITSIZE (mode: int_mode),
3653	NULL_RTX, true);
3654	}
3655
3656	if (temp)
3657	{
3658	if (mclass != MODE_INT)
3659	{
3660	if (target == 0)
3661	target = gen_reg_rtx (mode);
3662	convert_move (target, temp, 0);
3663	return target;
3664	}
3665	else
3666	return gen_lowpart (mode, temp);
3667	}
3668	else
3669	delete_insns_since (last);
3670	}
3671	}
3672	}
3673
3674	/* One final attempt at implementing negation via subtraction,
3675	this time allowing widening of the operand. */
3676	if (optab_to_code (op: unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3677	{
3678	rtx temp;
3679	temp = expand_binop (mode,
3680	binoptab: unoptab == negv_optab ? subv_optab : sub_optab,
3681	CONST0_RTX (mode), op1: op0,
3682	target, unsignedp, methods: OPTAB_LIB_WIDEN);
3683	if (temp)
3684	return temp;
3685	}
3686
3687	return 0;
3688	}
3689
3690	/* Emit code to compute the absolute value of OP0, with result to
3691	TARGET if convenient. (TARGET may be 0.) The return value says
3692	where the result actually is to be found.
3693
3694	MODE is the mode of the operand; the mode of the result is
3695	different but can be deduced from MODE.
3696
3697	*/
3698
3699	rtx
3700	expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3701	int result_unsignedp)
3702	{
3703	rtx temp;
3704
3705	if (GET_MODE_CLASS (mode) != MODE_INT
3706	|| ! flag_trapv)
3707	result_unsignedp = 1;
3708
3709	/* First try to do it with a special abs instruction. */
3710	temp = expand_unop (mode, unoptab: result_unsignedp ? abs_optab : absv_optab,
3711	op0, target, unsignedp: 0);
3712	if (temp != 0)
3713	return temp;
3714
3715	/* For floating point modes, try clearing the sign bit. */
3716	scalar_float_mode float_mode;
3717	if (is_a <scalar_float_mode> (m: mode, result: &float_mode))
3718	{
3719	temp = expand_absneg_bit (code: ABS, mode: float_mode, op0, target);
3720	if (temp)
3721	return temp;
3722	}
3723
3724	/* If we have a MAX insn, we can do this as MAX (x, -x). */
3725	if (optab_handler (op: smax_optab, mode) != CODE_FOR_nothing
3726	&& !HONOR_SIGNED_ZEROS (mode))
3727	{
3728	rtx_insn *last = get_last_insn ();
3729
3730	temp = expand_unop (mode, unoptab: result_unsignedp ? neg_optab : negv_optab,
3731	op0, NULL_RTX, unsignedp: 0);
3732	if (temp != 0)
3733	temp = expand_binop (mode, binoptab: smax_optab, op0, op1: temp, target, unsignedp: 0,
3734	methods: OPTAB_WIDEN);
3735
3736	if (temp != 0)
3737	return temp;
3738
3739	delete_insns_since (last);
3740	}
3741
3742	/* If this machine has expensive jumps, we can do integer absolute
3743	value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3744	where W is the width of MODE. */
3745
3746	scalar_int_mode int_mode;
3747	if (is_int_mode (mode, int_mode: &int_mode)
3748	&& BRANCH_COST (optimize_insn_for_speed_p (),
3749	false) >= 2)
3750	{
3751	rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3752	GET_MODE_PRECISION (mode: int_mode) - 1,
3753	NULL_RTX, 0);
3754
3755	temp = expand_binop (mode: int_mode, binoptab: xor_optab, op0: extended, op1: op0, target, unsignedp: 0,
3756	methods: OPTAB_LIB_WIDEN);
3757	if (temp != 0)
3758	temp = expand_binop (mode: int_mode,
3759	binoptab: result_unsignedp ? sub_optab : subv_optab,
3760	op0: temp, op1: extended, target, unsignedp: 0, methods: OPTAB_LIB_WIDEN);
3761
3762	if (temp != 0)
3763	return temp;
3764	}
3765
3766	return NULL_RTX;
3767	}
3768
3769	rtx
3770	expand_abs (machine_mode mode, rtx op0, rtx target,
3771	int result_unsignedp, int safe)
3772	{
3773	rtx temp;
3774	rtx_code_label *op1;
3775
3776	if (GET_MODE_CLASS (mode) != MODE_INT
3777	|| ! flag_trapv)
3778	result_unsignedp = 1;
3779
3780	temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3781	if (temp != 0)
3782	return temp;
3783
3784	/* If that does not win, use conditional jump and negate. */
3785
3786	/* It is safe to use the target if it is the same
3787	as the source if this is also a pseudo register */
3788	if (op0 == target && REG_P (op0)
3789	&& REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3790	safe = 1;
3791
3792	op1 = gen_label_rtx ();
3793	if (target == 0 || ! safe
3794	|| GET_MODE (target) != mode
3795	|| (MEM_P (target) && MEM_VOLATILE_P (target))
3796	|| (REG_P (target)
3797	&& REGNO (target) < FIRST_PSEUDO_REGISTER))
3798	target = gen_reg_rtx (mode);
3799
3800	emit_move_insn (target, op0);
3801	NO_DEFER_POP;
3802
3803	do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3804	NULL_RTX, NULL, op1,
3805	profile_probability::uninitialized ());
3806
3807	op0 = expand_unop (mode, unoptab: result_unsignedp ? neg_optab : negv_optab,
3808	op0: target, target, unsignedp: 0);
3809	if (op0 != target)
3810	emit_move_insn (target, op0);
3811	emit_label (op1);
3812	OK_DEFER_POP;
3813	return target;
3814	}
3815
3816	/* Emit code to compute the one's complement absolute value of OP0
3817	(if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3818	(TARGET may be NULL_RTX.) The return value says where the result
3819	actually is to be found.
3820
3821	MODE is the mode of the operand; the mode of the result is
3822	different but can be deduced from MODE. */
3823
3824	rtx
3825	expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3826	{
3827	rtx temp;
3828
3829	/* Not applicable for floating point modes. */
3830	if (FLOAT_MODE_P (mode))
3831	return NULL_RTX;
3832
3833	/* If we have a MAX insn, we can do this as MAX (x, ~x). */
3834	if (optab_handler (op: smax_optab, mode) != CODE_FOR_nothing)
3835	{
3836	rtx_insn *last = get_last_insn ();
3837
3838	temp = expand_unop (mode, unoptab: one_cmpl_optab, op0, NULL_RTX, unsignedp: 0);
3839	if (temp != 0)
3840	temp = expand_binop (mode, binoptab: smax_optab, op0, op1: temp, target, unsignedp: 0,
3841	methods: OPTAB_WIDEN);
3842
3843	if (temp != 0)
3844	return temp;
3845
3846	delete_insns_since (last);
3847	}
3848
3849	/* If this machine has expensive jumps, we can do one's complement
3850	absolute value of X as (((signed) x >> (W-1)) ^ x). */
3851
3852	scalar_int_mode int_mode;
3853	if (is_int_mode (mode, int_mode: &int_mode)
3854	&& BRANCH_COST (optimize_insn_for_speed_p (),
3855	false) >= 2)
3856	{
3857	rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3858	GET_MODE_PRECISION (mode: int_mode) - 1,
3859	NULL_RTX, 0);
3860
3861	temp = expand_binop (mode: int_mode, binoptab: xor_optab, op0: extended, op1: op0, target, unsignedp: 0,
3862	methods: OPTAB_LIB_WIDEN);
3863
3864	if (temp != 0)
3865	return temp;
3866	}
3867
3868	return NULL_RTX;
3869	}
3870
3871	/* A subroutine of expand_copysign, perform the copysign operation using the
3872	abs and neg primitives advertised to exist on the target. The assumption
3873	is that we have a split register file, and leaving op0 in fp registers,
3874	and not playing with subregs so much, will help the register allocator. */
3875
3876	static rtx
3877	expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3878	int bitpos, bool op0_is_abs)
3879	{
3880	scalar_int_mode imode;
3881	enum insn_code icode;
3882	rtx sign;
3883	rtx_code_label *label;
3884
3885	if (target == op1)
3886	target = NULL_RTX;
3887
3888	/* Check if the back end provides an insn that handles signbit for the
3889	argument's mode. */
3890	icode = optab_handler (op: signbit_optab, mode);
3891	if (icode != CODE_FOR_nothing)
3892	{
3893	imode = as_a <scalar_int_mode> (m: insn_data[(int) icode].operand[0].mode);
3894	sign = gen_reg_rtx (imode);
3895	emit_unop_insn (icode, sign, op1, UNKNOWN);
3896	}
3897	else
3898	{
3899	if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3900	{
3901	if (!int_mode_for_mode (mode).exists (mode: &imode))
3902	return NULL_RTX;
3903	op1 = gen_lowpart (imode, op1);
3904	}
3905	else
3906	{
3907	int word;
3908
3909	imode = word_mode;
3910	if (FLOAT_WORDS_BIG_ENDIAN)
3911	word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3912	else
3913	word = bitpos / BITS_PER_WORD;
3914	bitpos = bitpos % BITS_PER_WORD;
3915	op1 = operand_subword_force (op1, word, mode);
3916	}
3917
3918	wide_int mask = wi::set_bit_in_zero (bit: bitpos, precision: GET_MODE_PRECISION (mode: imode));
3919	sign = expand_binop (mode: imode, binoptab: and_optab, op0: op1,
3920	op1: immed_wide_int_const (mask, imode),
3921	NULL_RTX, unsignedp: 1, methods: OPTAB_LIB_WIDEN);
3922	}
3923
3924	if (!op0_is_abs)
3925	{
3926	op0 = expand_unop (mode, unoptab: abs_optab, op0, target, unsignedp: 0);
3927	if (op0 == NULL)
3928	return NULL_RTX;
3929	target = op0;
3930	}
3931	else
3932	{
3933	if (target == NULL_RTX)
3934	target = copy_to_reg (op0);
3935	else
3936	emit_move_insn (target, op0);
3937	}
3938
3939	label = gen_label_rtx ();
3940	emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3941
3942	if (CONST_DOUBLE_AS_FLOAT_P (op0))
3943	op0 = simplify_unary_operation (code: NEG, mode, op: op0, op_mode: mode);
3944	else
3945	op0 = expand_unop (mode, unoptab: neg_optab, op0, target, unsignedp: 0);
3946	if (op0 != target)
3947	emit_move_insn (target, op0);
3948
3949	emit_label (label);
3950
3951	return target;
3952	}
3953
3954
3955	/* A subroutine of expand_copysign, perform the entire copysign operation
3956	with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3957	is true if op0 is known to have its sign bit clear. */
3958
3959	static rtx
3960	expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3961	int bitpos, bool op0_is_abs)
3962	{
3963	scalar_int_mode imode;
3964	int word, nwords, i;
3965	rtx temp;
3966	rtx_insn *insns;
3967
3968	if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3969	{
3970	if (!int_mode_for_mode (mode).exists (mode: &imode))
3971	return NULL_RTX;
3972	word = 0;
3973	nwords = 1;
3974	}
3975	else
3976	{
3977	imode = word_mode;
3978
3979	if (FLOAT_WORDS_BIG_ENDIAN)
3980	word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3981	else
3982	word = bitpos / BITS_PER_WORD;
3983	bitpos = bitpos % BITS_PER_WORD;
3984	nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3985	}
3986
3987	wide_int mask = wi::set_bit_in_zero (bit: bitpos, precision: GET_MODE_PRECISION (mode: imode));
3988
3989	if (target == 0
3990	|| target == op0
3991	|| target == op1
3992	|| reg_overlap_mentioned_p (target, op0)
3993	|| reg_overlap_mentioned_p (target, op1)
3994	|| (nwords > 1 && !valid_multiword_target_p (target)))
3995	target = gen_reg_rtx (mode);
3996
3997	if (nwords > 1)
3998	{
3999	start_sequence ();
4000
4001	for (i = 0; i < nwords; ++i)
4002	{
4003	rtx targ_piece = operand_subword (target, i, 1, mode);
4004	rtx op0_piece = operand_subword_force (op0, i, mode);
4005
4006	if (i == word)
4007	{
4008	if (!op0_is_abs)
4009	op0_piece
4010	= expand_binop (mode: imode, binoptab: and_optab, op0: op0_piece,
4011	op1: immed_wide_int_const (~mask, imode),
4012	NULL_RTX, unsignedp: 1, methods: OPTAB_LIB_WIDEN);
4013	op1 = expand_binop (mode: imode, binoptab: and_optab,
4014	op0: operand_subword_force (op1, i, mode),
4015	op1: immed_wide_int_const (mask, imode),
4016	NULL_RTX, unsignedp: 1, methods: OPTAB_LIB_WIDEN);
4017
4018	temp = expand_binop (mode: imode, binoptab: ior_optab, op0: op0_piece, op1,
4019	target: targ_piece, unsignedp: 1, methods: OPTAB_LIB_WIDEN);
4020	if (temp != targ_piece)
4021	emit_move_insn (targ_piece, temp);
4022	}
4023	else
4024	emit_move_insn (targ_piece, op0_piece);
4025	}
4026
4027	insns = get_insns ();
4028	end_sequence ();
4029
4030	emit_insn (insns);
4031	}
4032	else
4033	{
4034	op1 = expand_binop (mode: imode, binoptab: and_optab, gen_lowpart (imode, op1),
4035	op1: immed_wide_int_const (mask, imode),
4036	NULL_RTX, unsignedp: 1, methods: OPTAB_LIB_WIDEN);
4037
4038	op0 = gen_lowpart (imode, op0);
4039	if (!op0_is_abs)
4040	op0 = expand_binop (mode: imode, binoptab: and_optab, op0,
4041	op1: immed_wide_int_const (~mask, imode),
4042	NULL_RTX, unsignedp: 1, methods: OPTAB_LIB_WIDEN);
4043
4044	temp = expand_binop (mode: imode, binoptab: ior_optab, op0, op1,
4045	gen_lowpart (imode, target), unsignedp: 1, methods: OPTAB_LIB_WIDEN);
4046	target = lowpart_subreg_maybe_copy (omode: mode, val: temp, imode);
4047	}
4048
4049	return target;
4050	}
4051
4052	/* Expand the C99 copysign operation. OP0 and OP1 must be the same
4053	scalar floating point mode. Return NULL if we do not know how to
4054	expand the operation inline. */
4055
4056	rtx
4057	expand_copysign (rtx op0, rtx op1, rtx target)
4058	{
4059	scalar_float_mode mode;
4060	const struct real_format *fmt;
4061	bool op0_is_abs;
4062	rtx temp;
4063
4064	mode = as_a <scalar_float_mode> (GET_MODE (op0));
4065	gcc_assert (GET_MODE (op1) == mode);
4066
4067	/* First try to do it with a special instruction. */
4068	temp = expand_binop (mode, binoptab: copysign_optab, op0, op1,
4069	target, unsignedp: 0, methods: OPTAB_DIRECT);
4070	if (temp)
4071	return temp;
4072
4073	fmt = REAL_MODE_FORMAT (mode);
4074	if (fmt == NULL || !fmt->has_signed_zero)
4075	return NULL_RTX;
4076
4077	op0_is_abs = false;
4078	if (CONST_DOUBLE_AS_FLOAT_P (op0))
4079	{
4080	if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
4081	op0 = simplify_unary_operation (code: ABS, mode, op: op0, op_mode: mode);
4082	op0_is_abs = true;
4083	}
4084
4085	if (fmt->signbit_ro >= 0
4086	&& (CONST_DOUBLE_AS_FLOAT_P (op0)
4087	|| (optab_handler (op: neg_optab, mode) != CODE_FOR_nothing
4088	&& optab_handler (op: abs_optab, mode) != CODE_FOR_nothing)))
4089	{
4090	temp = expand_copysign_absneg (mode, op0, op1, target,
4091	bitpos: fmt->signbit_ro, op0_is_abs);
4092	if (temp)
4093	return temp;
4094	}
4095
4096	if (fmt->signbit_rw < 0)
4097	return NULL_RTX;
4098	return expand_copysign_bit (mode, op0, op1, target,
4099	bitpos: fmt->signbit_rw, op0_is_abs);
4100	}
4101
4102	/* Generate an instruction whose insn-code is INSN_CODE,
4103	with two operands: an output TARGET and an input OP0.
4104	TARGET *must* be nonzero, and the output is always stored there.
4105	CODE is an rtx code such that (CODE OP0) is an rtx that describes
4106	the value that is stored into TARGET.
4107
4108	Return false if expansion failed. */
4109
4110	bool
4111	maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
4112	enum rtx_code code)
4113	{
4114	class expand_operand ops[2];
4115	rtx_insn *pat;
4116
4117	create_output_operand (op: &ops[0], x: target, GET_MODE (target));
4118	create_input_operand (op: &ops[1], value: op0, GET_MODE (op0));
4119	pat = maybe_gen_insn (icode, nops: 2, ops);
4120	if (!pat)
4121	return false;
4122
4123	if (INSN_P (pat) && NEXT_INSN (insn: pat) != NULL_RTX
4124	&& code != UNKNOWN)
4125	add_equal_note (insns: pat, target: ops[0].value, code, op0: ops[1].value, NULL_RTX,
4126	GET_MODE (op0));
4127
4128	emit_insn (pat);
4129
4130	if (ops[0].value != target)
4131	emit_move_insn (target, ops[0].value);
4132	return true;
4133	}
4134	/* Generate an instruction whose insn-code is INSN_CODE,
4135	with two operands: an output TARGET and an input OP0.
4136	TARGET *must* be nonzero, and the output is always stored there.
4137	CODE is an rtx code such that (CODE OP0) is an rtx that describes
4138	the value that is stored into TARGET. */
4139
4140	void
4141	emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
4142	{
4143	bool ok = maybe_emit_unop_insn (icode, target, op0, code);
4144	gcc_assert (ok);
4145	}
4146
4147	struct no_conflict_data
4148	{
4149	rtx target;
4150	rtx_insn *first, *insn;
4151	bool must_stay;
4152	};
4153
4154	/* Called via note_stores by emit_libcall_block. Set P->must_stay if
4155	the currently examined clobber / store has to stay in the list of
4156	insns that constitute the actual libcall block. */
4157	static void
4158	no_conflict_move_test (rtx dest, const_rtx set, void *p0)
4159	{
4160	struct no_conflict_data *p= (struct no_conflict_data *) p0;
4161
4162	/* If this inns directly contributes to setting the target, it must stay. */
4163	if (reg_overlap_mentioned_p (p->target, dest))
4164	p->must_stay = true;
4165	/* If we haven't committed to keeping any other insns in the list yet,
4166	there is nothing more to check. */
4167	else if (p->insn == p->first)
4168	return;
4169	/* If this insn sets / clobbers a register that feeds one of the insns
4170	already in the list, this insn has to stay too. */
4171	else if (reg_overlap_mentioned_p (dest, PATTERN (insn: p->first))
4172	|| (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
4173	|| reg_used_between_p (dest, p->first, p->insn)
4174	/* Likewise if this insn depends on a register set by a previous
4175	insn in the list, or if it sets a result (presumably a hard
4176	register) that is set or clobbered by a previous insn.
4177	N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
4178	SET_DEST perform the former check on the address, and the latter
4179	check on the MEM. */
4180	|| (GET_CODE (set) == SET
4181	&& (modified_in_p (SET_SRC (set), p->first)
4182	|| modified_in_p (SET_DEST (set), p->first)
4183	|| modified_between_p (SET_SRC (set), p->first, p->insn)
4184	|| modified_between_p (SET_DEST (set), p->first, p->insn))))
4185	p->must_stay = true;
4186	}
4187
4188
4189	/* Emit code to make a call to a constant function or a library call.
4190
4191	INSNS is a list containing all insns emitted in the call.
4192	These insns leave the result in RESULT. Our block is to copy RESULT
4193	to TARGET, which is logically equivalent to EQUIV.
4194
4195	We first emit any insns that set a pseudo on the assumption that these are
4196	loading constants into registers; doing so allows them to be safely cse'ed
4197	between blocks. Then we emit all the other insns in the block, followed by
4198	an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
4199	note with an operand of EQUIV. */
4200
4201	static void
4202	emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
4203	bool equiv_may_trap)
4204	{
4205	rtx final_dest = target;
4206	rtx_insn *next, *last, *insn;
4207
4208	/* If this is a reg with REG_USERVAR_P set, then it could possibly turn
4209	into a MEM later. Protect the libcall block from this change. */
4210	if (! REG_P (target) || REG_USERVAR_P (target))
4211	target = gen_reg_rtx (GET_MODE (target));
4212
4213	/* If we're using non-call exceptions, a libcall corresponding to an
4214	operation that may trap may also trap. */
4215	/* ??? See the comment in front of make_reg_eh_region_note. */
4216	if (cfun->can_throw_non_call_exceptions
4217	&& (equiv_may_trap || may_trap_p (equiv)))
4218	{
4219	for (insn = insns; insn; insn = NEXT_INSN (insn))
4220	if (CALL_P (insn))
4221	{
4222	rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
4223	if (note)
4224	{
4225	int lp_nr = INTVAL (XEXP (note, 0));
4226	if (lp_nr == 0 || lp_nr == INT_MIN)
4227	remove_note (insn, note);
4228	}
4229	}
4230	}
4231	else
4232	{
4233	/* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
4234	reg note to indicate that this call cannot throw or execute a nonlocal
4235	goto (unless there is already a REG_EH_REGION note, in which case
4236	we update it). */
4237	for (insn = insns; insn; insn = NEXT_INSN (insn))
4238	if (CALL_P (insn))
4239	make_reg_eh_region_note_nothrow_nononlocal (insn);
4240	}
4241
4242	/* First emit all insns that set pseudos. Remove them from the list as
4243	we go. Avoid insns that set pseudos which were referenced in previous
4244	insns. These can be generated by move_by_pieces, for example,
4245	to update an address. Similarly, avoid insns that reference things
4246	set in previous insns. */
4247
4248	for (insn = insns; insn; insn = next)
4249	{
4250	rtx set = single_set (insn);
4251
4252	next = NEXT_INSN (insn);
4253
4254	if (set != 0 && REG_P (SET_DEST (set))
4255	&& REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
4256	{
4257	struct no_conflict_data data;
4258
4259	data.target = const0_rtx;
4260	data.first = insns;
4261	data.insn = insn;
4262	data.must_stay = 0;
4263	note_stores (insn, no_conflict_move_test, &data);
4264	if (! data.must_stay)
4265	{
4266	if (PREV_INSN (insn))
4267	SET_NEXT_INSN (PREV_INSN (insn)) = next;
4268	else
4269	insns = next;
4270
4271	if (next)
4272	SET_PREV_INSN (next) = PREV_INSN (insn);
4273
4274	add_insn (insn);
4275	}
4276	}
4277
4278	/* Some ports use a loop to copy large arguments onto the stack.
4279	Don't move anything outside such a loop. */
4280	if (LABEL_P (insn))
4281	break;
4282	}
4283
4284	/* Write the remaining insns followed by the final copy. */
4285	for (insn = insns; insn; insn = next)
4286	{
4287	next = NEXT_INSN (insn);
4288
4289	add_insn (insn);
4290	}
4291
4292	last = emit_move_insn (target, result);
4293	if (equiv)
4294	set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
4295
4296	if (final_dest != target)
4297	emit_move_insn (final_dest, target);
4298	}
4299
4300	void
4301	emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
4302	{
4303	emit_libcall_block_1 (insns, target, result, equiv, equiv_may_trap: false);
4304	}
4305
4306	/* True if we can perform a comparison of mode MODE straightforwardly.
4307	PURPOSE describes how this comparison will be used. CODE is the rtx
4308	comparison code we will be using.
4309
4310	??? Actually, CODE is slightly weaker than that. A target is still
4311	required to implement all of the normal bcc operations, but not
4312	required to implement all (or any) of the unordered bcc operations. */
4313
4314	bool
4315	can_compare_p (enum rtx_code code, machine_mode mode,
4316	enum can_compare_purpose purpose)
4317	{
4318	rtx test;
4319	test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
4320	do
4321	{
4322	enum insn_code icode;
4323
4324	if (purpose == ccp_jump
4325	&& (icode = optab_handler (op: cbranch_optab, mode)) != CODE_FOR_nothing
4326	&& insn_operand_matches (icode, opno: 0, operand: test))
4327	return true;
4328	if (purpose == ccp_store_flag
4329	&& (icode = optab_handler (op: cstore_optab, mode)) != CODE_FOR_nothing
4330	&& insn_operand_matches (icode, opno: 1, operand: test))
4331	return true;
4332	if (purpose == ccp_cmov
4333	&& optab_handler (op: cmov_optab, mode) != CODE_FOR_nothing)
4334	return true;
4335
4336	mode = GET_MODE_WIDER_MODE (m: mode).else_void ();
4337	PUT_MODE (x: test, mode);
4338	}
4339	while (mode != VOIDmode);
4340
4341	return false;
4342	}
4343
4344	/* Return whether RTL code CODE corresponds to an unsigned optab. */
4345
4346	static bool
4347	unsigned_optab_p (enum rtx_code code)
4348	{
4349	return code == LTU || code == LEU || code == GTU || code == GEU;
4350	}
4351
4352	/* Return whether the backend-emitted comparison for code CODE, comparing
4353	operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
4354	operand OPNO of pattern ICODE. */
4355
4356	static bool
4357	insn_predicate_matches_p (enum insn_code icode, unsigned int opno,
4358	enum rtx_code code, machine_mode mask_mode,
4359	machine_mode value_mode)
4360	{
4361	rtx reg1 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 1);
4362	rtx reg2 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 2);
4363	rtx test = alloca_rtx_fmt_ee (code, mask_mode, reg1, reg2);
4364	return insn_operand_matches (icode, opno, operand: test);
4365	}
4366
4367	/* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
4368	for code CODE, comparing operands of mode VALUE_MODE and producing a result
4369	with MASK_MODE. */
4370
4371	bool
4372	can_vec_cmp_compare_p (enum rtx_code code, machine_mode value_mode,
4373	machine_mode mask_mode)
4374	{
4375	enum insn_code icode
4376	= get_vec_cmp_icode (vmode: value_mode, mask_mode, uns: unsigned_optab_p (code));
4377	if (icode == CODE_FOR_nothing)
4378	return false;
4379
4380	return insn_predicate_matches_p (icode, opno: 1, code, mask_mode, value_mode);
4381	}
4382
4383	/* Return whether the backend can emit a vector comparison (vcond/vcondu) for
4384	code CODE, comparing operands of mode CMP_OP_MODE and producing a result
4385	with VALUE_MODE. */
4386
4387	bool
4388	can_vcond_compare_p (enum rtx_code code, machine_mode value_mode,
4389	machine_mode cmp_op_mode)
4390	{
4391	enum insn_code icode
4392	= get_vcond_icode (vmode: value_mode, cmode: cmp_op_mode, uns: unsigned_optab_p (code));
4393	if (icode == CODE_FOR_nothing)
4394	return false;
4395
4396	return insn_predicate_matches_p (icode, opno: 3, code, mask_mode: value_mode, value_mode: cmp_op_mode);
4397	}
4398
4399	/* Return whether the backend can emit vector set instructions for inserting
4400	element into vector at variable index position. */
4401
4402	bool
4403	can_vec_set_var_idx_p (machine_mode vec_mode)
4404	{
4405	if (!VECTOR_MODE_P (vec_mode))
4406	return false;
4407
4408	machine_mode inner_mode = GET_MODE_INNER (vec_mode);
4409
4410	rtx reg1 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 1);
4411	rtx reg2 = alloca_raw_REG (inner_mode, LAST_VIRTUAL_REGISTER + 2);
4412
4413	enum insn_code icode = optab_handler (op: vec_set_optab, mode: vec_mode);
4414
4415	const struct insn_data_d *data = &insn_data[icode];
4416	machine_mode idx_mode = data->operand[2].mode;
4417
4418	rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
4419
4420	return icode != CODE_FOR_nothing && insn_operand_matches (icode, opno: 0, operand: reg1)
4421	&& insn_operand_matches (icode, opno: 1, operand: reg2)
4422	&& insn_operand_matches (icode, opno: 2, operand: reg3);
4423	}
4424
4425	/* Return whether the backend can emit a vec_extract instruction with
4426	a non-constant index. */
4427	bool
4428	can_vec_extract_var_idx_p (machine_mode vec_mode, machine_mode extr_mode)
4429	{
4430	if (!VECTOR_MODE_P (vec_mode))
4431	return false;
4432
4433	rtx reg1 = alloca_raw_REG (extr_mode, LAST_VIRTUAL_REGISTER + 1);
4434	rtx reg2 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 2);
4435
4436	enum insn_code icode = convert_optab_handler (op: vec_extract_optab,
4437	to_mode: vec_mode, from_mode: extr_mode);
4438
4439	const struct insn_data_d *data = &insn_data[icode];
4440	machine_mode idx_mode = data->operand[2].mode;
4441
4442	rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
4443
4444	return icode != CODE_FOR_nothing && insn_operand_matches (icode, opno: 0, operand: reg1)
4445	&& insn_operand_matches (icode, opno: 1, operand: reg2)
4446	&& insn_operand_matches (icode, opno: 2, operand: reg3);
4447	}
4448
4449	/* This function is called when we are going to emit a compare instruction that
4450	compares the values found in X and Y, using the rtl operator COMPARISON.
4451
4452	If they have mode BLKmode, then SIZE specifies the size of both operands.
4453
4454	UNSIGNEDP nonzero says that the operands are unsigned;
4455	this matters if they need to be widened (as given by METHODS).
4456
4457	*PTEST is where the resulting comparison RTX is returned or NULL_RTX
4458	if we failed to produce one.
4459
4460	*PMODE is the mode of the inputs (in case they are const_int).
4461
4462	This function performs all the setup necessary so that the caller only has
4463	to emit a single comparison insn. This setup can involve doing a BLKmode
4464	comparison or emitting a library call to perform the comparison if no insn
4465	is available to handle it.
4466	The values which are passed in through pointers can be modified; the caller
4467	should perform the comparison on the modified values. Constant
4468	comparisons must have already been folded. */
4469
4470	static void
4471	prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size,
4472	int unsignedp, enum optab_methods methods,
4473	rtx *ptest, machine_mode *pmode)
4474	{
4475	machine_mode mode = *pmode;
4476	rtx libfunc, test;
4477	machine_mode cmp_mode;
4478
4479	/* The other methods are not needed. */
4480	gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
4481	|| methods == OPTAB_LIB_WIDEN);
4482
4483	if (CONST_SCALAR_INT_P (y))
4484	canonicalize_comparison (mode, &comparison, &y);
4485
4486	/* If we are optimizing, force expensive constants into a register. */
4487	if (CONSTANT_P (x) && optimize
4488	&& (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
4489	> COSTS_N_INSNS (1))
4490	&& can_create_pseudo_p ())
4491	x = force_reg (mode, x);
4492
4493	if (CONSTANT_P (y) && optimize
4494	&& (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
4495	> COSTS_N_INSNS (1))
4496	&& can_create_pseudo_p ())
4497	y = force_reg (mode, y);
4498
4499	/* Don't let both operands fail to indicate the mode. */
4500	if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
4501	x = force_reg (mode, x);
4502	if (mode == VOIDmode)
4503	mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
4504
4505	/* Handle all BLKmode compares. */
4506
4507	if (mode == BLKmode)
4508	{
4509	machine_mode result_mode;
4510	enum insn_code cmp_code;
4511	rtx result;
4512	rtx opalign
4513	= GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
4514
4515	gcc_assert (size);
4516
4517	/* Try to use a memory block compare insn - either cmpstr
4518	or cmpmem will do. */
4519	opt_scalar_int_mode cmp_mode_iter;
4520	FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
4521	{
4522	scalar_int_mode cmp_mode = cmp_mode_iter.require ();
4523	cmp_code = direct_optab_handler (op: cmpmem_optab, mode: cmp_mode);
4524	if (cmp_code == CODE_FOR_nothing)
4525	cmp_code = direct_optab_handler (op: cmpstr_optab, mode: cmp_mode);
4526	if (cmp_code == CODE_FOR_nothing)
4527	cmp_code = direct_optab_handler (op: cmpstrn_optab, mode: cmp_mode);
4528	if (cmp_code == CODE_FOR_nothing)
4529	continue;
4530
4531	/* Must make sure the size fits the insn's mode. */
4532	if (CONST_INT_P (size)
4533	? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
4534	: (GET_MODE_BITSIZE (mode: as_a <scalar_int_mode> (GET_MODE (size)))
4535	> GET_MODE_BITSIZE (mode: cmp_mode)))
4536	continue;
4537
4538	result_mode = insn_data[cmp_code].operand[0].mode;
4539	result = gen_reg_rtx (result_mode);
4540	size = convert_to_mode (cmp_mode, size, 1);
4541	emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
4542
4543	*ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
4544	*pmode = result_mode;
4545	return;
4546	}
4547
4548	if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
4549	goto fail;
4550
4551	/* Otherwise call a library function. */
4552	result = emit_block_comp_via_libcall (dst: x, src: y, size);
4553
4554	x = result;
4555	y = const0_rtx;
4556	mode = TYPE_MODE (integer_type_node);
4557	methods = OPTAB_LIB_WIDEN;
4558	unsignedp = false;
4559	}
4560
4561	/* Don't allow operands to the compare to trap, as that can put the
4562	compare and branch in different basic blocks. */
4563	if (cfun->can_throw_non_call_exceptions)
4564	{
4565	if (!can_create_pseudo_p () && (may_trap_p (x) || may_trap_p (y)))
4566	goto fail;
4567	if (may_trap_p (x))
4568	x = copy_to_reg (x);
4569	if (may_trap_p (y))
4570	y = copy_to_reg (y);
4571	}
4572
4573	if (GET_MODE_CLASS (mode) == MODE_CC)
4574	{
4575	enum insn_code icode = optab_handler (op: cbranch_optab, CCmode);
4576	test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4577	if (icode != CODE_FOR_nothing
4578	&& insn_operand_matches (icode, opno: 0, operand: test))
4579	{
4580	*ptest = test;
4581	return;
4582	}
4583	else
4584	goto fail;
4585	}
4586
4587	test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4588	FOR_EACH_WIDER_MODE_FROM (cmp_mode, mode)
4589	{
4590	enum insn_code icode;
4591	icode = optab_handler (op: cbranch_optab, mode: cmp_mode);
4592	if (icode != CODE_FOR_nothing
4593	&& insn_operand_matches (icode, opno: 0, operand: test))
4594	{
4595	rtx_insn *last = get_last_insn ();
4596	rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
4597	rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
4598	if (op0 && op1
4599	&& insn_operand_matches (icode, opno: 1, operand: op0)
4600	&& insn_operand_matches (icode, opno: 2, operand: op1))
4601	{
4602	XEXP (test, 0) = op0;
4603	XEXP (test, 1) = op1;
4604	*ptest = test;
4605	*pmode = cmp_mode;
4606	return;
4607	}
4608	delete_insns_since (last);
4609	}
4610
4611	if (methods == OPTAB_DIRECT)
4612	break;
4613	}
4614
4615	if (methods != OPTAB_LIB_WIDEN)
4616	goto fail;
4617
4618	if (SCALAR_FLOAT_MODE_P (mode))
4619	{
4620	/* Small trick if UNORDERED isn't implemented by the hardware. */
4621	if (comparison == UNORDERED && rtx_equal_p (x, y))
4622	{
4623	prepare_cmp_insn (x, y, comparison: UNLT, NULL_RTX, unsignedp, methods: OPTAB_WIDEN,
4624	ptest, pmode);
4625	if (*ptest)
4626	return;
4627	}
4628
4629	prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4630	}
4631	else
4632	{
4633	rtx result;
4634	machine_mode ret_mode;
4635
4636	/* Handle a libcall just for the mode we are using. */
4637	libfunc = optab_libfunc (cmp_optab, mode);
4638	gcc_assert (libfunc);
4639
4640	/* If we want unsigned, and this mode has a distinct unsigned
4641	comparison routine, use that. */
4642	if (unsignedp)
4643	{
4644	rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4645	if (ulibfunc)
4646	libfunc = ulibfunc;
4647	}
4648
4649	ret_mode = targetm.libgcc_cmp_return_mode ();
4650	result = emit_library_call_value (fun: libfunc, NULL_RTX, fn_type: LCT_CONST,
4651	outmode: ret_mode, arg1: x, arg1_mode: mode, arg2: y, arg2_mode: mode);
4652
4653	/* There are two kinds of comparison routines. Biased routines
4654	return 0/1/2, and unbiased routines return -1/0/1. Other parts
4655	of gcc expect that the comparison operation is equivalent
4656	to the modified comparison. For signed comparisons compare the
4657	result against 1 in the biased case, and zero in the unbiased
4658	case. For unsigned comparisons always compare against 1 after
4659	biasing the unbiased result by adding 1. This gives us a way to
4660	represent LTU.
4661	The comparisons in the fixed-point helper library are always
4662	biased. */
4663	x = result;
4664	y = const1_rtx;
4665
4666	if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4667	{
4668	if (unsignedp)
4669	x = plus_constant (ret_mode, result, 1);
4670	else
4671	y = const0_rtx;
4672	}
4673
4674	*pmode = ret_mode;
4675	prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods,
4676	ptest, pmode);
4677	}
4678
4679	return;
4680
4681	fail:
4682	*ptest = NULL_RTX;
4683	}
4684
4685	/* Before emitting an insn with code ICODE, make sure that X, which is going
4686	to be used for operand OPNUM of the insn, is converted from mode MODE to
4687	WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4688	that it is accepted by the operand predicate. Return the new value. */
4689
4690	rtx
4691	prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4692	machine_mode wider_mode, int unsignedp)
4693	{
4694	if (mode != wider_mode)
4695	x = convert_modes (mode: wider_mode, oldmode: mode, x, unsignedp);
4696
4697	if (!insn_operand_matches (icode, opno: opnum, operand: x))
4698	{
4699	machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4700	if (reload_completed)
4701	return NULL_RTX;
4702	if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4703	return NULL_RTX;
4704	x = copy_to_mode_reg (op_mode, x);
4705	}
4706
4707	return x;
4708	}
4709
4710	/* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4711	we can do the branch. */
4712
4713	static void
4714	emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label,
4715	direct_optab cmp_optab, profile_probability prob,
4716	bool test_branch)
4717	{
4718	machine_mode optab_mode;
4719	enum mode_class mclass;
4720	enum insn_code icode;
4721	rtx_insn *insn;
4722
4723	mclass = GET_MODE_CLASS (mode);
4724	optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4725	icode = optab_handler (op: cmp_optab, mode: optab_mode);
4726
4727	gcc_assert (icode != CODE_FOR_nothing);
4728	gcc_assert (test_branch || insn_operand_matches (icode, 0, test));
4729	if (test_branch)
4730	insn = emit_jump_insn (GEN_FCN (icode) (XEXP (test, 0),
4731	XEXP (test, 1), label));
4732	else
4733	insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4734	XEXP (test, 1), label));
4735
4736	if (prob.initialized_p ()
4737	&& profile_status_for_fn (cfun) != PROFILE_ABSENT
4738	&& insn
4739	&& JUMP_P (insn)
4740	&& any_condjump_p (insn)
4741	&& !find_reg_note (insn, REG_BR_PROB, 0))
4742	add_reg_br_prob_note (insn, prob);
4743	}
4744
4745	/* PTEST points to a comparison that compares its first operand with zero.
4746	Check to see if it can be performed as a bit-test-and-branch instead.
4747	On success, return the instruction that performs the bit-test-and-branch
4748	and replace the second operand of *PTEST with the bit number to test.
4749	On failure, return CODE_FOR_nothing and leave *PTEST unchanged.
4750
4751	Note that the comparison described by *PTEST should not be taken
4752	literally after a successful return. *PTEST is just a convenient
4753	place to store the two operands of the bit-and-test.
4754
4755	VAL must contain the original tree expression for the first operand
4756	of *PTEST. */
4757
4758	static enum insn_code
4759	validate_test_and_branch (tree val, rtx *ptest, machine_mode *pmode, optab *res)
4760	{
4761	if (!val || TREE_CODE (val) != SSA_NAME)
4762	return CODE_FOR_nothing;
4763
4764	machine_mode mode = TYPE_MODE (TREE_TYPE (val));
4765	rtx test = *ptest;
4766	direct_optab optab;
4767
4768	if (GET_CODE (test) == EQ)
4769	optab = tbranch_eq_optab;
4770	else if (GET_CODE (test) == NE)
4771	optab = tbranch_ne_optab;
4772	else
4773	return CODE_FOR_nothing;
4774
4775	*res = optab;
4776
4777	/* If the target supports the testbit comparison directly, great. */
4778	auto icode = direct_optab_handler (op: optab, mode);
4779	if (icode == CODE_FOR_nothing)
4780	return icode;
4781
4782	if (tree_zero_one_valued_p (val))
4783	{
4784	auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 : 0;
4785	XEXP (test, 1) = gen_int_mode (pos, mode);
4786	*ptest = test;
4787	*pmode = mode;
4788	return icode;
4789	}
4790
4791	wide_int wcst = get_nonzero_bits (val);
4792	if (wcst == -1)
4793	return CODE_FOR_nothing;
4794
4795	int bitpos;
4796
4797	if ((bitpos = wi::exact_log2 (wcst)) == -1)
4798	return CODE_FOR_nothing;
4799
4800	auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 - bitpos : bitpos;
4801	XEXP (test, 1) = gen_int_mode (pos, mode);
4802	*ptest = test;
4803	*pmode = mode;
4804	return icode;
4805	}
4806
4807	/* Generate code to compare X with Y so that the condition codes are
4808	set and to jump to LABEL if the condition is true. If X is a
4809	constant and Y is not a constant, then the comparison is swapped to
4810	ensure that the comparison RTL has the canonical form.
4811
4812	UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4813	need to be widened. UNSIGNEDP is also used to select the proper
4814	branch condition code.
4815
4816	If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4817
4818	MODE is the mode of the inputs (in case they are const_int).
4819
4820	COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4821	It will be potentially converted into an unsigned variant based on
4822	UNSIGNEDP to select a proper jump instruction.
4823
4824	PROB is the probability of jumping to LABEL. If the comparison is against
4825	zero then VAL contains the expression from which the non-zero RTL is
4826	derived. */
4827
4828	void
4829	emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4830	machine_mode mode, int unsignedp, tree val, rtx label,
4831	profile_probability prob)
4832	{
4833	rtx op0 = x, op1 = y;
4834	rtx test;
4835
4836	/* Swap operands and condition to ensure canonical RTL. */
4837	if (swap_commutative_operands_p (x, y)
4838	&& can_compare_p (code: swap_condition (comparison), mode, purpose: ccp_jump))
4839	{
4840	op0 = y, op1 = x;
4841	comparison = swap_condition (comparison);
4842	}
4843
4844	/* If OP0 is still a constant, then both X and Y must be constants
4845	or the opposite comparison is not supported. Force X into a register
4846	to create canonical RTL. */
4847	if (CONSTANT_P (op0))
4848	op0 = force_reg (mode, op0);
4849
4850	if (unsignedp)
4851	comparison = unsigned_condition (comparison);
4852
4853	prepare_cmp_insn (x: op0, y: op1, comparison, size, unsignedp, methods: OPTAB_LIB_WIDEN,
4854	ptest: &test, pmode: &mode);
4855
4856	/* Check if we're comparing a truth type with 0, and if so check if
4857	the target supports tbranch. */
4858	machine_mode tmode = mode;
4859	direct_optab optab;
4860	if (op1 == CONST0_RTX (GET_MODE (op1))
4861	&& validate_test_and_branch (val, ptest: &test, pmode: &tmode,
4862	res: &optab) != CODE_FOR_nothing)
4863	{
4864	emit_cmp_and_jump_insn_1 (test, mode: tmode, label, cmp_optab: optab, prob, test_branch: true);
4865	return;
4866	}
4867
4868	emit_cmp_and_jump_insn_1 (test, mode, label, cmp_optab: cbranch_optab, prob, test_branch: false);
4869	}
4870
4871	/* Overloaded version of emit_cmp_and_jump_insns in which VAL is unknown. */
4872
4873	void
4874	emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4875	machine_mode mode, int unsignedp, rtx label,
4876	profile_probability prob)
4877	{
4878	emit_cmp_and_jump_insns (x, y, comparison, size, mode, unsignedp, NULL,
4879	label, prob);
4880	}
4881
4882
4883	/* Emit a library call comparison between floating point X and Y.
4884	COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4885
4886	static void
4887	prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
4888	rtx *ptest, machine_mode *pmode)
4889	{
4890	enum rtx_code swapped = swap_condition (comparison);
4891	enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
4892	machine_mode orig_mode = GET_MODE (x);
4893	machine_mode mode;
4894	rtx true_rtx, false_rtx;
4895	rtx value, target, equiv;
4896	rtx_insn *insns;
4897	rtx libfunc = 0;
4898	bool reversed_p = false;
4899	scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
4900
4901	FOR_EACH_WIDER_MODE_FROM (mode, orig_mode)
4902	{
4903	if (code_to_optab (code: comparison)
4904	&& (libfunc = optab_libfunc (code_to_optab (code: comparison), mode)))
4905	break;
4906
4907	if (code_to_optab (code: swapped)
4908	&& (libfunc = optab_libfunc (code_to_optab (code: swapped), mode)))
4909	{
4910	std::swap (a&: x, b&: y);
4911	comparison = swapped;
4912	break;
4913	}
4914
4915	if (code_to_optab (code: reversed)
4916	&& (libfunc = optab_libfunc (code_to_optab (code: reversed), mode)))
4917	{
4918	comparison = reversed;
4919	reversed_p = true;
4920	break;
4921	}
4922	}
4923
4924	gcc_assert (mode != VOIDmode);
4925
4926	if (mode != orig_mode)
4927	{
4928	x = convert_to_mode (mode, x, 0);
4929	y = convert_to_mode (mode, y, 0);
4930	}
4931
4932	/* Attach a REG_EQUAL note describing the semantics of the libcall to
4933	the RTL. The allows the RTL optimizers to delete the libcall if the
4934	condition can be determined at compile-time. */
4935	if (comparison == UNORDERED
4936	|| FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4937	{
4938	true_rtx = const_true_rtx;
4939	false_rtx = const0_rtx;
4940	}
4941	else
4942	{
4943	switch (comparison)
4944	{
4945	case EQ:
4946	true_rtx = const0_rtx;
4947	false_rtx = const_true_rtx;
4948	break;
4949
4950	case NE:
4951	true_rtx = const_true_rtx;
4952	false_rtx = const0_rtx;
4953	break;
4954
4955	case GT:
4956	true_rtx = const1_rtx;
4957	false_rtx = const0_rtx;
4958	break;
4959
4960	case GE:
4961	true_rtx = const0_rtx;
4962	false_rtx = constm1_rtx;
4963	break;
4964
4965	case LT:
4966	true_rtx = constm1_rtx;
4967	false_rtx = const0_rtx;
4968	break;
4969
4970	case LE:
4971	true_rtx = const0_rtx;
4972	false_rtx = const1_rtx;
4973	break;
4974
4975	default:
4976	gcc_unreachable ();
4977	}
4978	}
4979
4980	if (comparison == UNORDERED)
4981	{
4982	rtx temp = simplify_gen_relational (code: NE, mode: cmp_mode, op_mode: mode, op0: x, op1: x);
4983	equiv = simplify_gen_relational (code: NE, mode: cmp_mode, op_mode: mode, op0: y, op1: y);
4984	equiv = simplify_gen_ternary (code: IF_THEN_ELSE, mode: cmp_mode, op0_mode: cmp_mode,
4985	op0: temp, op1: const_true_rtx, op2: equiv);
4986	}
4987	else
4988	{
4989	equiv = simplify_gen_relational (code: comparison, mode: cmp_mode, op_mode: mode, op0: x, op1: y);
4990	if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4991	equiv = simplify_gen_ternary (code: IF_THEN_ELSE, mode: cmp_mode, op0_mode: cmp_mode,
4992	op0: equiv, op1: true_rtx, op2: false_rtx);
4993	}
4994
4995	start_sequence ();
4996	value = emit_library_call_value (fun: libfunc, NULL_RTX, fn_type: LCT_CONST,
4997	outmode: cmp_mode, arg1: x, arg1_mode: mode, arg2: y, arg2_mode: mode);
4998	insns = get_insns ();
4999	end_sequence ();
5000
5001	target = gen_reg_rtx (cmp_mode);
5002	emit_libcall_block (insns, target, result: value, equiv);
5003
5004	if (comparison == UNORDERED
5005	|| FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
5006	|| reversed_p)
5007	*ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
5008	else
5009	*ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
5010
5011	*pmode = cmp_mode;
5012	}
5013
5014	/* Generate code to indirectly jump to a location given in the rtx LOC. */
5015
5016	void
5017	emit_indirect_jump (rtx loc)
5018	{
5019	if (!targetm.have_indirect_jump ())
5020	sorry ("indirect jumps are not available on this target");
5021	else
5022	{
5023	class expand_operand ops[1];
5024	create_address_operand (op: &ops[0], value: loc);
5025	expand_jump_insn (icode: targetm.code_for_indirect_jump, nops: 1, ops);
5026	emit_barrier ();
5027	}
5028	}
5029
5030
5031	/* Emit a conditional move instruction if the machine supports one for that
5032	condition and machine mode.
5033
5034	OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5035	the mode to use should they be constants. If it is VOIDmode, they cannot
5036	both be constants.
5037
5038	OP2 should be stored in TARGET if the comparison is true, otherwise OP3
5039	should be stored there. MODE is the mode to use should they be constants.
5040	If it is VOIDmode, they cannot both be constants.
5041
5042	The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5043	is not supported. */
5044
5045	rtx
5046	emit_conditional_move (rtx target, struct rtx_comparison comp,
5047	rtx op2, rtx op3,
5048	machine_mode mode, int unsignedp)
5049	{
5050	rtx comparison;
5051	rtx_insn *last;
5052	enum insn_code icode;
5053	enum rtx_code reversed;
5054
5055	/* If the two source operands are identical, that's just a move. */
5056
5057	if (rtx_equal_p (op2, op3))
5058	{
5059	if (!target)
5060	target = gen_reg_rtx (mode);
5061
5062	emit_move_insn (target, op3);
5063	return target;
5064	}
5065
5066	/* If one operand is constant, make it the second one. Only do this
5067	if the other operand is not constant as well. */
5068
5069	if (swap_commutative_operands_p (comp.op0, comp.op1))
5070	{
5071	std::swap (a&: comp.op0, b&: comp.op1);
5072	comp.code = swap_condition (comp.code);
5073	}
5074
5075	/* get_condition will prefer to generate LT and GT even if the old
5076	comparison was against zero, so undo that canonicalization here since
5077	comparisons against zero are cheaper. */
5078
5079	if (comp.code == LT && comp.op1 == const1_rtx)
5080	comp.code = LE, comp.op1 = const0_rtx;
5081	else if (comp.code == GT && comp.op1 == constm1_rtx)
5082	comp.code = GE, comp.op1 = const0_rtx;
5083
5084	if (comp.mode == VOIDmode)
5085	comp.mode = GET_MODE (comp.op0);
5086
5087	enum rtx_code orig_code = comp.code;
5088	bool swapped = false;
5089	if (swap_commutative_operands_p (op2, op3)
5090	&& ((reversed =
5091	reversed_comparison_code_parts (comp.code, comp.op0, comp.op1, NULL))
5092	!= UNKNOWN))
5093	{
5094	std::swap (a&: op2, b&: op3);
5095	comp.code = reversed;
5096	swapped = true;
5097	}
5098
5099	if (mode == VOIDmode)
5100	mode = GET_MODE (op2);
5101
5102	icode = direct_optab_handler (op: movcc_optab, mode);
5103
5104	if (icode == CODE_FOR_nothing)
5105	return NULL_RTX;
5106
5107	if (!target)
5108	target = gen_reg_rtx (mode);
5109
5110	for (int pass = 0; ; pass++)
5111	{
5112	comp.code = unsignedp ? unsigned_condition (comp.code) : comp.code;
5113	comparison =
5114	simplify_gen_relational (code: comp.code, VOIDmode,
5115	op_mode: comp.mode, op0: comp.op0, op1: comp.op1);
5116
5117	/* We can get const0_rtx or const_true_rtx in some circumstances. Just
5118	punt and let the caller figure out how best to deal with this
5119	situation. */
5120	if (COMPARISON_P (comparison))
5121	{
5122	saved_pending_stack_adjust save;
5123	save_pending_stack_adjust (&save);
5124	last = get_last_insn ();
5125	do_pending_stack_adjust ();
5126	machine_mode cmpmode = comp.mode;
5127	rtx orig_op0 = XEXP (comparison, 0);
5128	rtx orig_op1 = XEXP (comparison, 1);
5129	rtx op2p = op2;
5130	rtx op3p = op3;
5131	/* If we are optimizing, force expensive constants into a register
5132	but preserve an eventual equality with op2/op3. */
5133	if (CONSTANT_P (orig_op0) && optimize
5134	&& cmpmode == mode
5135	&& (rtx_cost (orig_op0, mode, COMPARE, 0,
5136	optimize_insn_for_speed_p ())
5137	> COSTS_N_INSNS (1))
5138	&& can_create_pseudo_p ())
5139	{
5140	if (rtx_equal_p (orig_op0, op2))
5141	op2p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
5142	else if (rtx_equal_p (orig_op0, op3))
5143	op3p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
5144	}
5145	if (CONSTANT_P (orig_op1) && optimize
5146	&& cmpmode == mode
5147	&& (rtx_cost (orig_op1, mode, COMPARE, 0,
5148	optimize_insn_for_speed_p ())
5149	> COSTS_N_INSNS (1))
5150	&& can_create_pseudo_p ())
5151	{
5152	if (rtx_equal_p (orig_op1, op2))
5153	op2p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
5154	else if (rtx_equal_p (orig_op1, op3))
5155	op3p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
5156	}
5157	prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
5158	GET_CODE (comparison), NULL_RTX, unsignedp,
5159	methods: OPTAB_WIDEN, ptest: &comparison, pmode: &cmpmode);
5160	if (comparison)
5161	{
5162	rtx res = emit_conditional_move_1 (target, comparison,
5163	op2p, op3p, mode);
5164	if (res != NULL_RTX)
5165	return res;
5166	}
5167	delete_insns_since (last);
5168	restore_pending_stack_adjust (&save);
5169	}
5170
5171	if (pass == 1)
5172	return NULL_RTX;
5173
5174	/* If the preferred op2/op3 order is not usable, retry with other
5175	operand order, perhaps it will expand successfully. */
5176	if (swapped)
5177	comp.code = orig_code;
5178	else if ((reversed =
5179	reversed_comparison_code_parts (orig_code, comp.op0, comp.op1,
5180	NULL))
5181	!= UNKNOWN)
5182	comp.code = reversed;
5183	else
5184	return NULL_RTX;
5185	std::swap (a&: op2, b&: op3);
5186	}
5187	}
5188
5189	/* Helper function that, in addition to COMPARISON, also tries
5190	the reversed REV_COMPARISON with swapped OP2 and OP3. As opposed
5191	to when we pass the specific constituents of a comparison, no
5192	additional insns are emitted for it. It might still be necessary
5193	to emit more than one insn for the final conditional move, though. */
5194
5195	rtx
5196	emit_conditional_move (rtx target, rtx comparison, rtx rev_comparison,
5197	rtx op2, rtx op3, machine_mode mode)
5198	{
5199	rtx res = emit_conditional_move_1 (target, comparison, op2, op3, mode);
5200
5201	if (res != NULL_RTX)
5202	return res;
5203
5204	return emit_conditional_move_1 (target, rev_comparison, op3, op2, mode);
5205	}
5206
5207	/* Helper for emitting a conditional move. */
5208
5209	static rtx
5210	emit_conditional_move_1 (rtx target, rtx comparison,
5211	rtx op2, rtx op3, machine_mode mode)
5212	{
5213	enum insn_code icode;
5214
5215	if (comparison == NULL_RTX || !COMPARISON_P (comparison))
5216	return NULL_RTX;
5217
5218	/* If the two source operands are identical, that's just a move.
5219	As the comparison comes in non-canonicalized, we must make
5220	sure not to discard any possible side effects. If there are
5221	side effects, just let the target handle it. */
5222	if (!side_effects_p (comparison) && rtx_equal_p (op2, op3))
5223	{
5224	if (!target)
5225	target = gen_reg_rtx (mode);
5226
5227	emit_move_insn (target, op3);
5228	return target;
5229	}
5230
5231	if (mode == VOIDmode)
5232	mode = GET_MODE (op2);
5233
5234	icode = direct_optab_handler (op: movcc_optab, mode);
5235
5236	if (icode == CODE_FOR_nothing)
5237	return NULL_RTX;
5238
5239	if (!target)
5240	target = gen_reg_rtx (mode);
5241
5242	class expand_operand ops[4];
5243
5244	create_output_operand (op: &ops[0], x: target, mode);
5245	create_fixed_operand (op: &ops[1], x: comparison);
5246	create_input_operand (op: &ops[2], value: op2, mode);
5247	create_input_operand (op: &ops[3], value: op3, mode);
5248
5249	if (maybe_expand_insn (icode, nops: 4, ops))
5250	{
5251	if (ops[0].value != target)
5252	convert_move (target, ops[0].value, false);
5253	return target;
5254	}
5255
5256	return NULL_RTX;
5257	}
5258
5259
5260	/* Emit a conditional negate or bitwise complement using the
5261	negcc or notcc optabs if available. Return NULL_RTX if such operations
5262	are not available. Otherwise return the RTX holding the result.
5263	TARGET is the desired destination of the result. COMP is the comparison
5264	on which to negate. If COND is true move into TARGET the negation
5265	or bitwise complement of OP1. Otherwise move OP2 into TARGET.
5266	CODE is either NEG or NOT. MODE is the machine mode in which the
5267	operation is performed. */
5268
5269	rtx
5270	emit_conditional_neg_or_complement (rtx target, rtx_code code,
5271	machine_mode mode, rtx cond, rtx op1,
5272	rtx op2)
5273	{
5274	optab op = unknown_optab;
5275	if (code == NEG)
5276	op = negcc_optab;
5277	else if (code == NOT)
5278	op = notcc_optab;
5279	else
5280	gcc_unreachable ();
5281
5282	insn_code icode = direct_optab_handler (op, mode);
5283
5284	if (icode == CODE_FOR_nothing)
5285	return NULL_RTX;
5286
5287	if (!target)
5288	target = gen_reg_rtx (mode);
5289
5290	rtx_insn *last = get_last_insn ();
5291	class expand_operand ops[4];
5292
5293	create_output_operand (op: &ops[0], x: target, mode);
5294	create_fixed_operand (op: &ops[1], x: cond);
5295	create_input_operand (op: &ops[2], value: op1, mode);
5296	create_input_operand (op: &ops[3], value: op2, mode);
5297
5298	if (maybe_expand_insn (icode, nops: 4, ops))
5299	{
5300	if (ops[0].value != target)
5301	convert_move (target, ops[0].value, false);
5302
5303	return target;
5304	}
5305	delete_insns_since (last);
5306	return NULL_RTX;
5307	}
5308
5309	/* Emit a conditional addition instruction if the machine supports one for that
5310	condition and machine mode.
5311
5312	OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5313	the mode to use should they be constants. If it is VOIDmode, they cannot
5314	both be constants.
5315
5316	OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
5317	should be stored there. MODE is the mode to use should they be constants.
5318	If it is VOIDmode, they cannot both be constants.
5319
5320	The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5321	is not supported. */
5322
5323	rtx
5324	emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
5325	machine_mode cmode, rtx op2, rtx op3,
5326	machine_mode mode, int unsignedp)
5327	{
5328	rtx comparison;
5329	rtx_insn *last;
5330	enum insn_code icode;
5331
5332	/* If one operand is constant, make it the second one. Only do this
5333	if the other operand is not constant as well. */
5334
5335	if (swap_commutative_operands_p (op0, op1))
5336	{
5337	std::swap (a&: op0, b&: op1);
5338	code = swap_condition (code);
5339	}
5340
5341	/* get_condition will prefer to generate LT and GT even if the old
5342	comparison was against zero, so undo that canonicalization here since
5343	comparisons against zero are cheaper. */
5344	if (code == LT && op1 == const1_rtx)
5345	code = LE, op1 = const0_rtx;
5346	else if (code == GT && op1 == constm1_rtx)
5347	code = GE, op1 = const0_rtx;
5348
5349	if (cmode == VOIDmode)
5350	cmode = GET_MODE (op0);
5351
5352	if (mode == VOIDmode)
5353	mode = GET_MODE (op2);
5354
5355	icode = optab_handler (op: addcc_optab, mode);
5356
5357	if (icode == CODE_FOR_nothing)
5358	return 0;
5359
5360	if (!target)
5361	target = gen_reg_rtx (mode);
5362
5363	code = unsignedp ? unsigned_condition (code) : code;
5364	comparison = simplify_gen_relational (code, VOIDmode, op_mode: cmode, op0, op1);
5365
5366	/* We can get const0_rtx or const_true_rtx in some circumstances. Just
5367	return NULL and let the caller figure out how best to deal with this
5368	situation. */
5369	if (!COMPARISON_P (comparison))
5370	return NULL_RTX;
5371
5372	do_pending_stack_adjust ();
5373	last = get_last_insn ();
5374	prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
5375	GET_CODE (comparison), NULL_RTX, unsignedp, methods: OPTAB_WIDEN,
5376	ptest: &comparison, pmode: &cmode);
5377	if (comparison)
5378	{
5379	class expand_operand ops[4];
5380
5381	create_output_operand (op: &ops[0], x: target, mode);
5382	create_fixed_operand (op: &ops[1], x: comparison);
5383	create_input_operand (op: &ops[2], value: op2, mode);
5384	create_input_operand (op: &ops[3], value: op3, mode);
5385	if (maybe_expand_insn (icode, nops: 4, ops))
5386	{
5387	if (ops[0].value != target)
5388	convert_move (target, ops[0].value, false);
5389	return target;
5390	}
5391	}
5392	delete_insns_since (last);
5393	return NULL_RTX;
5394	}
5395
5396	/* These functions attempt to generate an insn body, rather than
5397	emitting the insn, but if the gen function already emits them, we
5398	make no attempt to turn them back into naked patterns. */
5399
5400	/* Generate and return an insn body to add Y to X. */
5401
5402	rtx_insn *
5403	gen_add2_insn (rtx x, rtx y)
5404	{
5405	enum insn_code icode = optab_handler (op: add_optab, GET_MODE (x));
5406
5407	gcc_assert (insn_operand_matches (icode, 0, x));
5408	gcc_assert (insn_operand_matches (icode, 1, x));
5409	gcc_assert (insn_operand_matches (icode, 2, y));
5410
5411	return GEN_FCN (icode) (x, x, y);
5412	}
5413
5414	/* Generate and return an insn body to add r1 and c,
5415	storing the result in r0. */
5416
5417	rtx_insn *
5418	gen_add3_insn (rtx r0, rtx r1, rtx c)
5419	{
5420	enum insn_code icode = optab_handler (op: add_optab, GET_MODE (r0));
5421
5422	if (icode == CODE_FOR_nothing
5423	|| !insn_operand_matches (icode, opno: 0, operand: r0)
5424	|| !insn_operand_matches (icode, opno: 1, operand: r1)
5425	|| !insn_operand_matches (icode, opno: 2, operand: c))
5426	return NULL;
5427
5428	return GEN_FCN (icode) (r0, r1, c);
5429	}
5430
5431	bool
5432	have_add2_insn (rtx x, rtx y)
5433	{
5434	enum insn_code icode;
5435
5436	gcc_assert (GET_MODE (x) != VOIDmode);
5437
5438	icode = optab_handler (op: add_optab, GET_MODE (x));
5439
5440	if (icode == CODE_FOR_nothing)
5441	return false;
5442
5443	if (!insn_operand_matches (icode, opno: 0, operand: x)
5444	|| !insn_operand_matches (icode, opno: 1, operand: x)
5445	|| !insn_operand_matches (icode, opno: 2, operand: y))
5446	return false;
5447
5448	return true;
5449	}
5450
5451	/* Generate and return an insn body to add Y to X. */
5452
5453	rtx_insn *
5454	gen_addptr3_insn (rtx x, rtx y, rtx z)
5455	{
5456	enum insn_code icode = optab_handler (op: addptr3_optab, GET_MODE (x));
5457
5458	gcc_assert (insn_operand_matches (icode, 0, x));
5459	gcc_assert (insn_operand_matches (icode, 1, y));
5460	gcc_assert (insn_operand_matches (icode, 2, z));
5461
5462	return GEN_FCN (icode) (x, y, z);
5463	}
5464
5465	/* Return true if the target implements an addptr pattern and X, Y,
5466	and Z are valid for the pattern predicates. */
5467
5468	bool
5469	have_addptr3_insn (rtx x, rtx y, rtx z)
5470	{
5471	enum insn_code icode;
5472
5473	gcc_assert (GET_MODE (x) != VOIDmode);
5474
5475	icode = optab_handler (op: addptr3_optab, GET_MODE (x));
5476
5477	if (icode == CODE_FOR_nothing)
5478	return false;
5479
5480	if (!insn_operand_matches (icode, opno: 0, operand: x)
5481	|| !insn_operand_matches (icode, opno: 1, operand: y)
5482	|| !insn_operand_matches (icode, opno: 2, operand: z))
5483	return false;
5484
5485	return true;
5486	}
5487
5488	/* Generate and return an insn body to subtract Y from X. */
5489
5490	rtx_insn *
5491	gen_sub2_insn (rtx x, rtx y)
5492	{
5493	enum insn_code icode = optab_handler (op: sub_optab, GET_MODE (x));
5494
5495	gcc_assert (insn_operand_matches (icode, 0, x));
5496	gcc_assert (insn_operand_matches (icode, 1, x));
5497	gcc_assert (insn_operand_matches (icode, 2, y));
5498
5499	return GEN_FCN (icode) (x, x, y);
5500	}
5501
5502	/* Generate and return an insn body to subtract r1 and c,
5503	storing the result in r0. */
5504
5505	rtx_insn *
5506	gen_sub3_insn (rtx r0, rtx r1, rtx c)
5507	{
5508	enum insn_code icode = optab_handler (op: sub_optab, GET_MODE (r0));
5509
5510	if (icode == CODE_FOR_nothing
5511	|| !insn_operand_matches (icode, opno: 0, operand: r0)
5512	|| !insn_operand_matches (icode, opno: 1, operand: r1)
5513	|| !insn_operand_matches (icode, opno: 2, operand: c))
5514	return NULL;
5515
5516	return GEN_FCN (icode) (r0, r1, c);
5517	}
5518
5519	bool
5520	have_sub2_insn (rtx x, rtx y)
5521	{
5522	enum insn_code icode;
5523
5524	gcc_assert (GET_MODE (x) != VOIDmode);
5525
5526	icode = optab_handler (op: sub_optab, GET_MODE (x));
5527
5528	if (icode == CODE_FOR_nothing)
5529	return false;
5530
5531	if (!insn_operand_matches (icode, opno: 0, operand: x)
5532	|| !insn_operand_matches (icode, opno: 1, operand: x)
5533	|| !insn_operand_matches (icode, opno: 2, operand: y))
5534	return false;
5535
5536	return true;
5537	}
5538
5539	/* Generate the body of an insn to extend Y (with mode MFROM)
5540	into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
5541
5542	rtx_insn *
5543	gen_extend_insn (rtx x, rtx y, machine_mode mto,
5544	machine_mode mfrom, int unsignedp)
5545	{
5546	enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
5547	return GEN_FCN (icode) (x, y);
5548	}
5549
5550	/* Generate code to convert FROM to floating point
5551	and store in TO. FROM must be fixed point and not VOIDmode.
5552	UNSIGNEDP nonzero means regard FROM as unsigned.
5553	Normally this is done by correcting the final value
5554	if it is negative. */
5555
5556	void
5557	expand_float (rtx to, rtx from, int unsignedp)
5558	{
5559	enum insn_code icode;
5560	rtx target = to;
5561	scalar_mode from_mode, to_mode;
5562	machine_mode fmode, imode;
5563	bool can_do_signed = false;
5564
5565	/* Crash now, because we won't be able to decide which mode to use. */
5566	gcc_assert (GET_MODE (from) != VOIDmode);
5567
5568	/* Look for an insn to do the conversion. Do it in the specified
5569	modes if possible; otherwise convert either input, output or both to
5570	wider mode. If the integer mode is wider than the mode of FROM,
5571	we can do the conversion signed even if the input is unsigned. */
5572
5573	FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
5574	FOR_EACH_MODE_FROM (imode, GET_MODE (from))
5575	{
5576	int doing_unsigned = unsignedp;
5577
5578	if (fmode != GET_MODE (to)
5579	&& (significand_size (fmode)
5580	< GET_MODE_UNIT_PRECISION (GET_MODE (from))))
5581	continue;
5582
5583	icode = can_float_p (fmode, imode, unsignedp);
5584	if (icode == CODE_FOR_nothing && unsignedp)
5585	{
5586	enum insn_code scode = can_float_p (fmode, imode, 0);
5587	if (scode != CODE_FOR_nothing)
5588	can_do_signed = true;
5589	if (imode != GET_MODE (from))
5590	icode = scode, doing_unsigned = 0;
5591	}
5592
5593	if (icode != CODE_FOR_nothing)
5594	{
5595	if (imode != GET_MODE (from))
5596	from = convert_to_mode (imode, from, unsignedp);
5597
5598	if (fmode != GET_MODE (to))
5599	target = gen_reg_rtx (fmode);
5600
5601	emit_unop_insn (icode, target, op0: from,
5602	code: doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
5603
5604	if (target != to)
5605	convert_move (to, target, 0);
5606	return;
5607	}
5608	}
5609
5610	/* Unsigned integer, and no way to convert directly. Convert as signed,
5611	then unconditionally adjust the result. */
5612	if (unsignedp
5613	&& can_do_signed
5614	&& is_a <scalar_mode> (GET_MODE (to), result: &to_mode)
5615	&& is_a <scalar_mode> (GET_MODE (from), result: &from_mode))
5616	{
5617	opt_scalar_mode fmode_iter;
5618	rtx_code_label *label = gen_label_rtx ();
5619	rtx temp;
5620	REAL_VALUE_TYPE offset;
5621
5622	/* Look for a usable floating mode FMODE wider than the source and at
5623	least as wide as the target. Using FMODE will avoid rounding woes
5624	with unsigned values greater than the signed maximum value. */
5625
5626	FOR_EACH_MODE_FROM (fmode_iter, to_mode)
5627	{
5628	scalar_mode fmode = fmode_iter.require ();
5629	if (GET_MODE_PRECISION (mode: from_mode) < GET_MODE_BITSIZE (mode: fmode)
5630	&& can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
5631	break;
5632	}
5633
5634	if (!fmode_iter.exists (mode: &fmode))
5635	{
5636	/* There is no such mode. Pretend the target is wide enough. */
5637	fmode = to_mode;
5638
5639	/* Avoid double-rounding when TO is narrower than FROM. */
5640	if ((significand_size (fmode) + 1)
5641	< GET_MODE_PRECISION (mode: from_mode))
5642	{
5643	rtx temp1;
5644	rtx_code_label *neglabel = gen_label_rtx ();
5645
5646	/* Don't use TARGET if it isn't a register, is a hard register,
5647	or is the wrong mode. */
5648	if (!REG_P (target)
5649	|| REGNO (target) < FIRST_PSEUDO_REGISTER
5650	|| GET_MODE (target) != fmode)
5651	target = gen_reg_rtx (fmode);
5652
5653	imode = from_mode;
5654	do_pending_stack_adjust ();
5655
5656	/* Test whether the sign bit is set. */
5657	emit_cmp_and_jump_insns (x: from, const0_rtx, comparison: LT, NULL_RTX, mode: imode,
5658	unsignedp: 0, label: neglabel);
5659
5660	/* The sign bit is not set. Convert as signed. */
5661	expand_float (to: target, from, unsignedp: 0);
5662	emit_jump_insn (targetm.gen_jump (label));
5663	emit_barrier ();
5664
5665	/* The sign bit is set.
5666	Convert to a usable (positive signed) value by shifting right
5667	one bit, while remembering if a nonzero bit was shifted
5668	out; i.e., compute (from & 1) | (from >> 1). */
5669
5670	emit_label (neglabel);
5671	temp = expand_binop (mode: imode, binoptab: and_optab, op0: from, const1_rtx,
5672	NULL_RTX, unsignedp: 1, methods: OPTAB_LIB_WIDEN);
5673	temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
5674	temp = expand_binop (mode: imode, binoptab: ior_optab, op0: temp, op1: temp1, target: temp, unsignedp: 1,
5675	methods: OPTAB_LIB_WIDEN);
5676	expand_float (to: target, from: temp, unsignedp: 0);
5677
5678	/* Multiply by 2 to undo the shift above. */
5679	temp = expand_binop (mode: fmode, binoptab: add_optab, op0: target, op1: target,
5680	target, unsignedp: 0, methods: OPTAB_LIB_WIDEN);
5681	if (temp != target)
5682	emit_move_insn (target, temp);
5683
5684	do_pending_stack_adjust ();
5685	emit_label (label);
5686	goto done;
5687	}
5688	}
5689
5690	/* If we are about to do some arithmetic to correct for an
5691	unsigned operand, do it in a pseudo-register. */
5692
5693	if (to_mode != fmode
5694	|| !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
5695	target = gen_reg_rtx (fmode);
5696
5697	/* Convert as signed integer to floating. */
5698	expand_float (to: target, from, unsignedp: 0);
5699
5700	/* If FROM is negative (and therefore TO is negative),
5701	correct its value by 2**bitwidth. */
5702
5703	do_pending_stack_adjust ();
5704	emit_cmp_and_jump_insns (x: from, const0_rtx, comparison: GE, NULL_RTX, mode: from_mode,
5705	unsignedp: 0, label);
5706
5707
5708	real_2expN (&offset, GET_MODE_PRECISION (mode: from_mode), fmode);
5709	temp = expand_binop (mode: fmode, binoptab: add_optab, op0: target,
5710	op1: const_double_from_real_value (offset, fmode),
5711	target, unsignedp: 0, methods: OPTAB_LIB_WIDEN);
5712	if (temp != target)
5713	emit_move_insn (target, temp);
5714
5715	do_pending_stack_adjust ();
5716	emit_label (label);
5717	goto done;
5718	}
5719
5720	/* No hardware instruction available; call a library routine. */
5721	{
5722	rtx libfunc;
5723	rtx_insn *insns;
5724	rtx value;
5725	convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
5726
5727	if (is_narrower_int_mode (GET_MODE (from), SImode))
5728	from = convert_to_mode (SImode, from, unsignedp);
5729
5730	libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5731	gcc_assert (libfunc);
5732
5733	start_sequence ();
5734
5735	value = emit_library_call_value (fun: libfunc, NULL_RTX, fn_type: LCT_CONST,
5736	GET_MODE (to), arg1: from, GET_MODE (from));
5737	insns = get_insns ();
5738	end_sequence ();
5739
5740	emit_libcall_block (insns, target, result: value,
5741	gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
5742	GET_MODE (to), from));
5743	}
5744
5745	done:
5746
5747	/* Copy result to requested destination
5748	if we have been computing in a temp location. */
5749
5750	if (target != to)
5751	{
5752	if (GET_MODE (target) == GET_MODE (to))
5753	emit_move_insn (to, target);
5754	else
5755	convert_move (to, target, 0);
5756	}
5757	}
5758
5759	/* Generate code to convert FROM to fixed point and store in TO. FROM
5760	must be floating point. */
5761
5762	void
5763	expand_fix (rtx to, rtx from, int unsignedp)
5764	{
5765	enum insn_code icode;
5766	rtx target = to;
5767	machine_mode fmode, imode;
5768	opt_scalar_mode fmode_iter;
5769	bool must_trunc = false;
5770
5771	/* We first try to find a pair of modes, one real and one integer, at
5772	least as wide as FROM and TO, respectively, in which we can open-code
5773	this conversion. If the integer mode is wider than the mode of TO,
5774	we can do the conversion either signed or unsigned. */
5775
5776	FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
5777	FOR_EACH_MODE_FROM (imode, GET_MODE (to))
5778	{
5779	int doing_unsigned = unsignedp;
5780
5781	icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
5782	if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
5783	icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
5784
5785	if (icode != CODE_FOR_nothing)
5786	{
5787	rtx_insn *last = get_last_insn ();
5788	rtx from1 = from;
5789	if (fmode != GET_MODE (from))
5790	{
5791	if (REAL_MODE_FORMAT (GET_MODE (from))
5792	== &arm_bfloat_half_format
5793	&& REAL_MODE_FORMAT (fmode) == &ieee_single_format)
5794	/* The BF -> SF conversions can be just a shift, doesn't
5795	need to handle sNANs. */
5796	{
5797	int save_flag_finite_math_only = flag_finite_math_only;
5798	flag_finite_math_only = true;
5799	from1 = convert_to_mode (fmode, from, 0);
5800	flag_finite_math_only = save_flag_finite_math_only;
5801	}
5802	else
5803	from1 = convert_to_mode (fmode, from, 0);
5804	}
5805
5806	if (must_trunc)
5807	{
5808	rtx temp = gen_reg_rtx (GET_MODE (from1));
5809	from1 = expand_unop (GET_MODE (from1), unoptab: ftrunc_optab, op0: from1,
5810	target: temp, unsignedp: 0);
5811	}
5812
5813	if (imode != GET_MODE (to))
5814	target = gen_reg_rtx (imode);
5815
5816	if (maybe_emit_unop_insn (icode, target, op0: from1,
5817	code: doing_unsigned ? UNSIGNED_FIX : FIX))
5818	{
5819	if (target != to)
5820	convert_move (to, target, unsignedp);
5821	return;
5822	}
5823	delete_insns_since (last);
5824	}
5825	}
5826
5827	/* For an unsigned conversion, there is one more way to do it.
5828	If we have a signed conversion, we generate code that compares
5829	the real value to the largest representable positive number. If if
5830	is smaller, the conversion is done normally. Otherwise, subtract
5831	one plus the highest signed number, convert, and add it back.
5832
5833	We only need to check all real modes, since we know we didn't find
5834	anything with a wider integer mode.
5835
5836	This code used to extend FP value into mode wider than the destination.
5837	This is needed for decimal float modes which cannot accurately
5838	represent one plus the highest signed number of the same size, but
5839	not for binary modes. Consider, for instance conversion from SFmode
5840	into DImode.
5841
5842	The hot path through the code is dealing with inputs smaller than 2^63
5843	and doing just the conversion, so there is no bits to lose.
5844
5845	In the other path we know the value is positive in the range 2^63..2^64-1
5846	inclusive. (as for other input overflow happens and result is undefined)
5847	So we know that the most important bit set in mantissa corresponds to
5848	2^63. The subtraction of 2^63 should not generate any rounding as it
5849	simply clears out that bit. The rest is trivial. */
5850
5851	scalar_int_mode to_mode;
5852	if (unsignedp
5853	&& is_a <scalar_int_mode> (GET_MODE (to), result: &to_mode)
5854	&& HWI_COMPUTABLE_MODE_P (mode: to_mode))
5855	FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
5856	{
5857	scalar_mode fmode = fmode_iter.require ();
5858	if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
5859	0, &must_trunc)
5860	&& (!DECIMAL_FLOAT_MODE_P (fmode)
5861	|| (GET_MODE_BITSIZE (mode: fmode) > GET_MODE_PRECISION (mode: to_mode))))
5862	{
5863	int bitsize;
5864	REAL_VALUE_TYPE offset;
5865	rtx limit;
5866	rtx_code_label *lab1, *lab2;
5867	rtx_insn *insn;
5868
5869	bitsize = GET_MODE_PRECISION (mode: to_mode);
5870	real_2expN (&offset, bitsize - 1, fmode);
5871	limit = const_double_from_real_value (offset, fmode);
5872	lab1 = gen_label_rtx ();
5873	lab2 = gen_label_rtx ();
5874
5875	if (fmode != GET_MODE (from))
5876	{
5877	if (REAL_MODE_FORMAT (GET_MODE (from))
5878	== &arm_bfloat_half_format
5879	&& REAL_MODE_FORMAT (fmode) == &ieee_single_format)
5880	/* The BF -> SF conversions can be just a shift, doesn't
5881	need to handle sNANs. */
5882	{
5883	int save_flag_finite_math_only = flag_finite_math_only;
5884	flag_finite_math_only = true;
5885	from = convert_to_mode (fmode, from, 0);
5886	flag_finite_math_only = save_flag_finite_math_only;
5887	}
5888	else
5889	from = convert_to_mode (fmode, from, 0);
5890	}
5891
5892	/* See if we need to do the subtraction. */
5893	do_pending_stack_adjust ();
5894	emit_cmp_and_jump_insns (x: from, y: limit, comparison: GE, NULL_RTX,
5895	GET_MODE (from), unsignedp: 0, label: lab1);
5896
5897	/* If not, do the signed "fix" and branch around fixup code. */
5898	expand_fix (to, from, unsignedp: 0);
5899	emit_jump_insn (targetm.gen_jump (lab2));
5900	emit_barrier ();
5901
5902	/* Otherwise, subtract 2**(N-1), convert to signed number,
5903	then add 2**(N-1). Do the addition using XOR since this
5904	will often generate better code. */
5905	emit_label (lab1);
5906	target = expand_binop (GET_MODE (from), binoptab: sub_optab, op0: from, op1: limit,
5907	NULL_RTX, unsignedp: 0, methods: OPTAB_LIB_WIDEN);
5908	expand_fix (to, from: target, unsignedp: 0);
5909	target = expand_binop (mode: to_mode, binoptab: xor_optab, op0: to,
5910	op1: gen_int_mode
5911	(HOST_WIDE_INT_1 << (bitsize - 1),
5912	to_mode),
5913	target: to, unsignedp: 1, methods: OPTAB_LIB_WIDEN);
5914
5915	if (target != to)
5916	emit_move_insn (to, target);
5917
5918	emit_label (lab2);
5919
5920	if (optab_handler (op: mov_optab, mode: to_mode) != CODE_FOR_nothing)
5921	{
5922	/* Make a place for a REG_NOTE and add it. */
5923	insn = emit_move_insn (to, to);
5924	set_dst_reg_note (insn, REG_EQUAL,
5925	gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
5926	copy_rtx (from)),
5927	to);
5928	}
5929
5930	return;
5931	}
5932	}
5933
5934	#ifdef HAVE_SFmode
5935	if (REAL_MODE_FORMAT (GET_MODE (from)) == &arm_bfloat_half_format
5936	&& REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
5937	/* We don't have BF -> TI library functions, use BF -> SF -> TI
5938	instead but the BF -> SF conversion can be just a shift, doesn't
5939	need to handle sNANs. */
5940	{
5941	int save_flag_finite_math_only = flag_finite_math_only;
5942	flag_finite_math_only = true;
5943	from = convert_to_mode (SFmode, from, 0);
5944	flag_finite_math_only = save_flag_finite_math_only;
5945	expand_fix (to, from, unsignedp);
5946	return;
5947	}
5948	#endif
5949
5950	/* We can't do it with an insn, so use a library call. But first ensure
5951	that the mode of TO is at least as wide as SImode, since those are the
5952	only library calls we know about. */
5953
5954	if (is_narrower_int_mode (GET_MODE (to), SImode))
5955	{
5956	target = gen_reg_rtx (SImode);
5957
5958	expand_fix (to: target, from, unsignedp);
5959	}
5960	else
5961	{
5962	rtx_insn *insns;
5963	rtx value;
5964	rtx libfunc;
5965
5966	convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
5967	libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5968	gcc_assert (libfunc);
5969
5970	start_sequence ();
5971
5972	value = emit_library_call_value (fun: libfunc, NULL_RTX, fn_type: LCT_CONST,
5973	GET_MODE (to), arg1: from, GET_MODE (from));
5974	insns = get_insns ();
5975	end_sequence ();
5976
5977	emit_libcall_block (insns, target, result: value,
5978	gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
5979	GET_MODE (to), from));
5980	}
5981
5982	if (target != to)
5983	{
5984	if (GET_MODE (to) == GET_MODE (target))
5985	emit_move_insn (to, target);
5986	else
5987	convert_move (to, target, 0);
5988	}
5989	}
5990
5991
5992	/* Promote integer arguments for a libcall if necessary.
5993	emit_library_call_value cannot do the promotion because it does not
5994	know if it should do a signed or unsigned promotion. This is because
5995	there are no tree types defined for libcalls. */
5996
5997	static rtx
5998	prepare_libcall_arg (rtx arg, int uintp)
5999	{
6000	scalar_int_mode mode;
6001	machine_mode arg_mode;
6002	if (is_a <scalar_int_mode> (GET_MODE (arg), result: &mode))
6003	{
6004	/* If we need to promote the integer function argument we need to do
6005	it here instead of inside emit_library_call_value because in
6006	emit_library_call_value we don't know if we should do a signed or
6007	unsigned promotion. */
6008
6009	int unsigned_p = 0;
6010	arg_mode = promote_function_mode (NULL_TREE, mode,
6011	&unsigned_p, NULL_TREE, 0);
6012	if (arg_mode != mode)
6013	return convert_to_mode (arg_mode, arg, uintp);
6014	}
6015	return arg;
6016	}
6017
6018	/* Generate code to convert FROM or TO a fixed-point.
6019	If UINTP is true, either TO or FROM is an unsigned integer.
6020	If SATP is true, we need to saturate the result. */
6021
6022	void
6023	expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
6024	{
6025	machine_mode to_mode = GET_MODE (to);
6026	machine_mode from_mode = GET_MODE (from);
6027	convert_optab tab;
6028	enum rtx_code this_code;
6029	enum insn_code code;
6030	rtx_insn *insns;
6031	rtx value;
6032	rtx libfunc;
6033
6034	if (to_mode == from_mode)
6035	{
6036	emit_move_insn (to, from);
6037	return;
6038	}
6039
6040	if (uintp)
6041	{
6042	tab = satp ? satfractuns_optab : fractuns_optab;
6043	this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
6044	}
6045	else
6046	{
6047	tab = satp ? satfract_optab : fract_optab;
6048	this_code = satp ? SAT_FRACT : FRACT_CONVERT;
6049	}
6050	code = convert_optab_handler (op: tab, to_mode, from_mode);
6051	if (code != CODE_FOR_nothing)
6052	{
6053	emit_unop_insn (icode: code, target: to, op0: from, code: this_code);
6054	return;
6055	}
6056
6057	libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
6058	gcc_assert (libfunc);
6059
6060	from = prepare_libcall_arg (arg: from, uintp);
6061	from_mode = GET_MODE (from);
6062
6063	start_sequence ();
6064	value = emit_library_call_value (fun: libfunc, NULL_RTX, fn_type: LCT_CONST, outmode: to_mode,
6065	arg1: from, arg1_mode: from_mode);
6066	insns = get_insns ();
6067	end_sequence ();
6068
6069	emit_libcall_block (insns, target: to, result: value,
6070	gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
6071	}
6072
6073	/* Generate code to convert FROM to fixed point and store in TO. FROM
6074	must be floating point, TO must be signed. Use the conversion optab
6075	TAB to do the conversion. */
6076
6077	bool
6078	expand_sfix_optab (rtx to, rtx from, convert_optab tab)
6079	{
6080	enum insn_code icode;
6081	rtx target = to;
6082	machine_mode fmode, imode;
6083
6084	/* We first try to find a pair of modes, one real and one integer, at
6085	least as wide as FROM and TO, respectively, in which we can open-code
6086	this conversion. If the integer mode is wider than the mode of TO,
6087	we can do the conversion either signed or unsigned. */
6088
6089	FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
6090	FOR_EACH_MODE_FROM (imode, GET_MODE (to))
6091	{
6092	icode = convert_optab_handler (tab, imode, fmode,
6093	insn_optimization_type ());
6094	if (icode != CODE_FOR_nothing)
6095	{
6096	rtx_insn *last = get_last_insn ();
6097	if (fmode != GET_MODE (from))
6098	from = convert_to_mode (fmode, from, 0);
6099
6100	if (imode != GET_MODE (to))
6101	target = gen_reg_rtx (imode);
6102
6103	if (!maybe_emit_unop_insn (icode, target, op0: from, code: UNKNOWN))
6104	{
6105	delete_insns_since (last);
6106	continue;
6107	}
6108	if (target != to)
6109	convert_move (to, target, 0);
6110	return true;
6111	}
6112	}
6113
6114	return false;
6115	}
6116
6117	/* Report whether we have an instruction to perform the operation
6118	specified by CODE on operands of mode MODE. */
6119	bool
6120	have_insn_for (enum rtx_code code, machine_mode mode)
6121	{
6122	return (code_to_optab (code)
6123	&& (optab_handler (op: code_to_optab (code), mode)
6124	!= CODE_FOR_nothing));
6125	}
6126
6127	/* Print information about the current contents of the optabs on
6128	STDERR. */
6129
6130	DEBUG_FUNCTION void
6131	debug_optab_libfuncs (void)
6132	{
6133	int i, j, k;
6134
6135	/* Dump the arithmetic optabs. */
6136	for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
6137	for (j = 0; j < NUM_MACHINE_MODES; ++j)
6138	{
6139	rtx l = optab_libfunc ((optab) i, (machine_mode) j);
6140	if (l)
6141	{
6142	gcc_assert (GET_CODE (l) == SYMBOL_REF);
6143	fprintf (stderr, format: "%s\t%s:\t%s\n",
6144	GET_RTX_NAME (optab_to_code ((optab) i)),
6145	GET_MODE_NAME (j),
6146	XSTR (l, 0));
6147	}
6148	}
6149
6150	/* Dump the conversion optabs. */
6151	for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
6152	for (j = 0; j < NUM_MACHINE_MODES; ++j)
6153	for (k = 0; k < NUM_MACHINE_MODES; ++k)
6154	{
6155	rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
6156	(machine_mode) k);
6157	if (l)
6158	{
6159	gcc_assert (GET_CODE (l) == SYMBOL_REF);
6160	fprintf (stderr, format: "%s\t%s\t%s:\t%s\n",
6161	GET_RTX_NAME (optab_to_code ((optab) i)),
6162	GET_MODE_NAME (j),
6163	GET_MODE_NAME (k),
6164	XSTR (l, 0));
6165	}
6166	}
6167	}
6168
6169	/* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6170	CODE. Return 0 on failure. */
6171
6172	rtx_insn *
6173	gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
6174	{
6175	machine_mode mode = GET_MODE (op1);
6176	enum insn_code icode;
6177	rtx_insn *insn;
6178	rtx trap_rtx;
6179
6180	if (mode == VOIDmode)
6181	return 0;
6182
6183	icode = optab_handler (op: ctrap_optab, mode);
6184	if (icode == CODE_FOR_nothing)
6185	return 0;
6186
6187	/* Some targets only accept a zero trap code. */
6188	if (!insn_operand_matches (icode, opno: 3, operand: tcode))
6189	return 0;
6190
6191	do_pending_stack_adjust ();
6192	start_sequence ();
6193	prepare_cmp_insn (x: op1, y: op2, comparison: code, NULL_RTX, unsignedp: false, methods: OPTAB_DIRECT,
6194	ptest: &trap_rtx, pmode: &mode);
6195	if (!trap_rtx)
6196	insn = NULL;
6197	else
6198	insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
6199	tcode);
6200
6201	/* If that failed, then give up. */
6202	if (insn == 0)
6203	{
6204	end_sequence ();
6205	return 0;
6206	}
6207
6208	emit_insn (insn);
6209	insn = get_insns ();
6210	end_sequence ();
6211	return insn;
6212	}
6213
6214	/* Return rtx code for TCODE or UNKNOWN. Use UNSIGNEDP to select signed
6215	or unsigned operation code. */
6216
6217	enum rtx_code
6218	get_rtx_code_1 (enum tree_code tcode, bool unsignedp)
6219	{
6220	enum rtx_code code;
6221	switch (tcode)
6222	{
6223	case EQ_EXPR:
6224	code = EQ;
6225	break;
6226	case NE_EXPR:
6227	code = NE;
6228	break;
6229	case LT_EXPR:
6230	code = unsignedp ? LTU : LT;
6231	break;
6232	case LE_EXPR:
6233	code = unsignedp ? LEU : LE;
6234	break;
6235	case GT_EXPR:
6236	code = unsignedp ? GTU : GT;
6237	break;
6238	case GE_EXPR:
6239	code = unsignedp ? GEU : GE;
6240	break;
6241
6242	case UNORDERED_EXPR:
6243	code = UNORDERED;
6244	break;
6245	case ORDERED_EXPR:
6246	code = ORDERED;
6247	break;
6248	case UNLT_EXPR:
6249	code = UNLT;
6250	break;
6251	case UNLE_EXPR:
6252	code = UNLE;
6253	break;
6254	case UNGT_EXPR:
6255	code = UNGT;
6256	break;
6257	case UNGE_EXPR:
6258	code = UNGE;
6259	break;
6260	case UNEQ_EXPR:
6261	code = UNEQ;
6262	break;
6263	case LTGT_EXPR:
6264	code = LTGT;
6265	break;
6266
6267	case BIT_AND_EXPR:
6268	code = AND;
6269	break;
6270
6271	case BIT_IOR_EXPR:
6272	code = IOR;
6273	break;
6274
6275	default:
6276	code = UNKNOWN;
6277	break;
6278	}
6279	return code;
6280	}
6281
6282	/* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6283	or unsigned operation code. */
6284
6285	enum rtx_code
6286	get_rtx_code (enum tree_code tcode, bool unsignedp)
6287	{
6288	enum rtx_code code = get_rtx_code_1 (tcode, unsignedp);
6289	gcc_assert (code != UNKNOWN);
6290	return code;
6291	}
6292
6293	/* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
6294	select signed or unsigned operators. OPNO holds the index of the
6295	first comparison operand for insn ICODE. Do not generate the
6296	compare instruction itself. */
6297
6298	rtx
6299	vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
6300	tree t_op0, tree t_op1, bool unsignedp,
6301	enum insn_code icode, unsigned int opno)
6302	{
6303	class expand_operand ops[2];
6304	rtx rtx_op0, rtx_op1;
6305	machine_mode m0, m1;
6306	enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
6307
6308	gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
6309
6310	/* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6311	has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6312	cases, use the original mode. */
6313	rtx_op0 = expand_expr (exp: t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
6314	modifier: EXPAND_STACK_PARM);
6315	m0 = GET_MODE (rtx_op0);
6316	if (m0 == VOIDmode)
6317	m0 = TYPE_MODE (TREE_TYPE (t_op0));
6318
6319	rtx_op1 = expand_expr (exp: t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
6320	modifier: EXPAND_STACK_PARM);
6321	m1 = GET_MODE (rtx_op1);
6322	if (m1 == VOIDmode)
6323	m1 = TYPE_MODE (TREE_TYPE (t_op1));
6324
6325	create_input_operand (op: &ops[0], value: rtx_op0, mode: m0);
6326	create_input_operand (op: &ops[1], value: rtx_op1, mode: m1);
6327	if (!maybe_legitimize_operands (icode, opno, nops: 2, ops))
6328	gcc_unreachable ();
6329	return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
6330	}
6331
6332	/* Check if vec_perm mask SEL is a constant equivalent to a shift of
6333	the first vec_perm operand, assuming the second operand (for left shift
6334	first operand) is a constant vector of zeros. Return the shift distance
6335	in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
6336	mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
6337	shift or vec_shl_optab for left shift. */
6338	static rtx
6339	shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel,
6340	optab shift_optab)
6341	{
6342	unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
6343	poly_int64 first = sel[0];
6344	if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
6345	return NULL_RTX;
6346
6347	if (shift_optab == vec_shl_optab)
6348	{
6349	unsigned int nelt;
6350	if (!GET_MODE_NUNITS (mode).is_constant (const_value: &nelt))
6351	return NULL_RTX;
6352	unsigned firstidx = 0;
6353	for (unsigned int i = 0; i < nelt; i++)
6354	{
6355	if (known_eq (sel[i], nelt))
6356	{
6357	if (i == 0 || firstidx)
6358	return NULL_RTX;
6359	firstidx = i;
6360	}
6361	else if (firstidx
6362	? maybe_ne (a: sel[i], b: nelt + i - firstidx)
6363	: maybe_ge (sel[i], nelt))
6364	return NULL_RTX;
6365	}
6366
6367	if (firstidx == 0)
6368	return NULL_RTX;
6369	first = firstidx;
6370	}
6371	else if (!sel.series_p (0, 1, first, 1))
6372	{
6373	unsigned int nelt;
6374	if (!GET_MODE_NUNITS (mode).is_constant (const_value: &nelt))
6375	return NULL_RTX;
6376	for (unsigned int i = 1; i < nelt; i++)
6377	{
6378	poly_int64 expected = i + first;
6379	/* Indices into the second vector are all equivalent. */
6380	if (maybe_lt (a: sel[i], b: nelt)
6381	? maybe_ne (a: sel[i], b: expected)
6382	: maybe_lt (a: expected, b: nelt))
6383	return NULL_RTX;
6384	}
6385	}
6386
6387	return gen_int_shift_amount (mode, first * bitsize);
6388	}
6389
6390	/* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
6391
6392	static rtx
6393	expand_vec_perm_1 (enum insn_code icode, rtx target,
6394	rtx v0, rtx v1, rtx sel)
6395	{
6396	machine_mode tmode = GET_MODE (target);
6397	machine_mode smode = GET_MODE (sel);
6398	class expand_operand ops[4];
6399
6400	gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
6401	|| related_int_vector_mode (tmode).require () == smode);
6402	create_output_operand (op: &ops[0], x: target, mode: tmode);
6403	create_input_operand (op: &ops[3], value: sel, mode: smode);
6404
6405	/* Make an effort to preserve v0 == v1. The target expander is able to
6406	rely on this to determine if we're permuting a single input operand. */
6407	if (rtx_equal_p (v0, v1))
6408	{
6409	if (!insn_operand_matches (icode, opno: 1, operand: v0))
6410	v0 = force_reg (tmode, v0);
6411	gcc_checking_assert (insn_operand_matches (icode, 1, v0));
6412	gcc_checking_assert (insn_operand_matches (icode, 2, v0));
6413
6414	create_fixed_operand (op: &ops[1], x: v0);
6415	create_fixed_operand (op: &ops[2], x: v0);
6416	}
6417	else
6418	{
6419	create_input_operand (op: &ops[1], value: v0, mode: tmode);
6420	create_input_operand (op: &ops[2], value: v1, mode: tmode);
6421	}
6422
6423	if (maybe_expand_insn (icode, nops: 4, ops))
6424	return ops[0].value;
6425	return NULL_RTX;
6426	}
6427
6428	/* Implement a permutation of vectors v0 and v1 using the permutation
6429	vector in SEL and return the result. Use TARGET to hold the result
6430	if nonnull and convenient.
6431
6432	MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
6433	is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
6434	to have a particular mode. */
6435
6436	rtx
6437	expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
6438	const vec_perm_builder &sel, machine_mode sel_mode,
6439	rtx target)
6440	{
6441	if (!target || !register_operand (target, mode))
6442	target = gen_reg_rtx (mode);
6443
6444	/* Set QIMODE to a different vector mode with byte elements.
6445	If no such mode, or if MODE already has byte elements, use VOIDmode. */
6446	machine_mode qimode;
6447	if (!qimode_for_vec_perm (mode).exists (mode: &qimode))
6448	qimode = VOIDmode;
6449
6450	rtx_insn *last = get_last_insn ();
6451
6452	bool single_arg_p = rtx_equal_p (v0, v1);
6453	/* Always specify two input vectors here and leave the target to handle
6454	cases in which the inputs are equal. Not all backends can cope with
6455	the single-input representation when testing for a double-input
6456	target instruction. */
6457	vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode));
6458
6459	/* See if this can be handled with a vec_shr or vec_shl. We only do this
6460	if the second (for vec_shr) or first (for vec_shl) vector is all
6461	zeroes. */
6462	insn_code shift_code = CODE_FOR_nothing;
6463	insn_code shift_code_qi = CODE_FOR_nothing;
6464	optab shift_optab = unknown_optab;
6465	rtx v2 = v0;
6466	if (v1 == CONST0_RTX (GET_MODE (v1)))
6467	shift_optab = vec_shr_optab;
6468	else if (v0 == CONST0_RTX (GET_MODE (v0)))
6469	{
6470	shift_optab = vec_shl_optab;
6471	v2 = v1;
6472	}
6473	if (shift_optab != unknown_optab)
6474	{
6475	shift_code = optab_handler (op: shift_optab, mode);
6476	shift_code_qi = ((qimode != VOIDmode && qimode != mode)
6477	? optab_handler (op: shift_optab, mode: qimode)
6478	: CODE_FOR_nothing);
6479	}
6480	if (shift_code != CODE_FOR_nothing || shift_code_qi != CODE_FOR_nothing)
6481	{
6482	rtx shift_amt = shift_amt_for_vec_perm_mask (mode, sel: indices, shift_optab);
6483	if (shift_amt)
6484	{
6485	class expand_operand ops[3];
6486	if (shift_amt == const0_rtx)
6487	return v2;
6488	if (shift_code != CODE_FOR_nothing)
6489	{
6490	create_output_operand (op: &ops[0], x: target, mode);
6491	create_input_operand (op: &ops[1], value: v2, mode);
6492	create_convert_operand_from_type (op: &ops[2], value: shift_amt, sizetype);
6493	if (maybe_expand_insn (icode: shift_code, nops: 3, ops))
6494	return ops[0].value;
6495	}
6496	if (shift_code_qi != CODE_FOR_nothing)
6497	{
6498	rtx tmp = gen_reg_rtx (qimode);
6499	create_output_operand (op: &ops[0], x: tmp, mode: qimode);
6500	create_input_operand (op: &ops[1], gen_lowpart (qimode, v2), mode: qimode);
6501	create_convert_operand_from_type (op: &ops[2], value: shift_amt, sizetype);
6502	if (maybe_expand_insn (icode: shift_code_qi, nops: 3, ops))
6503	return gen_lowpart (mode, ops[0].value);
6504	}
6505	}
6506	}
6507
6508	if (targetm.vectorize.vec_perm_const != NULL)
6509	{
6510	if (single_arg_p)
6511	v1 = v0;
6512
6513	gcc_checking_assert (GET_MODE (v0) == GET_MODE (v1));
6514	machine_mode op_mode = GET_MODE (v0);
6515	if (targetm.vectorize.vec_perm_const (mode, op_mode, target, v0, v1,
6516	indices))
6517	return target;
6518	}
6519
6520	/* Fall back to a constant byte-based permutation. */
6521	vec_perm_indices qimode_indices;
6522	rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
6523	if (qimode != VOIDmode)
6524	{
6525	qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
6526	target_qi = gen_reg_rtx (qimode);
6527	v0_qi = gen_lowpart (qimode, v0);
6528	v1_qi = gen_lowpart (qimode, v1);
6529	if (targetm.vectorize.vec_perm_const != NULL
6530	&& targetm.vectorize.vec_perm_const (qimode, qimode, target_qi, v0_qi,
6531	v1_qi, qimode_indices))
6532	return gen_lowpart (mode, target_qi);
6533	}
6534
6535	v0 = force_reg (mode, v0);
6536	if (single_arg_p)
6537	v1 = v0;
6538	v1 = force_reg (mode, v1);
6539
6540	/* Otherwise expand as a fully variable permuation. */
6541
6542	/* The optabs are only defined for selectors with the same width
6543	as the values being permuted. */
6544	machine_mode required_sel_mode;
6545	if (!related_int_vector_mode (mode).exists (mode: &required_sel_mode))
6546	{
6547	delete_insns_since (last);
6548	return NULL_RTX;
6549	}
6550
6551	/* We know that it is semantically valid to treat SEL as having SEL_MODE.
6552	If that isn't the mode we want then we need to prove that using
6553	REQUIRED_SEL_MODE is OK. */
6554	if (sel_mode != required_sel_mode)
6555	{
6556	if (!selector_fits_mode_p (required_sel_mode, indices))
6557	{
6558	delete_insns_since (last);
6559	return NULL_RTX;
6560	}
6561	sel_mode = required_sel_mode;
6562	}
6563
6564	insn_code icode = direct_optab_handler (op: vec_perm_optab, mode);
6565	if (icode != CODE_FOR_nothing)
6566	{
6567	rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
6568	rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel: sel_rtx);
6569	if (tmp)
6570	return tmp;
6571	}
6572
6573	if (qimode != VOIDmode
6574	&& selector_fits_mode_p (qimode, qimode_indices))
6575	{
6576	icode = direct_optab_handler (op: vec_perm_optab, mode: qimode);
6577	if (icode != CODE_FOR_nothing)
6578	{
6579	rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
6580	rtx tmp = expand_vec_perm_1 (icode, target: target_qi, v0: v0_qi, v1: v1_qi, sel: sel_qi);
6581	if (tmp)
6582	return gen_lowpart (mode, tmp);
6583	}
6584	}
6585
6586	delete_insns_since (last);
6587	return NULL_RTX;
6588	}
6589
6590	/* Implement a permutation of vectors v0 and v1 using the permutation
6591	vector in SEL and return the result. Use TARGET to hold the result
6592	if nonnull and convenient.
6593
6594	MODE is the mode of the vectors being permuted (V0 and V1).
6595	SEL must have the integer equivalent of MODE and is known to be
6596	unsuitable for permutes with a constant permutation vector. */
6597
6598	rtx
6599	expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
6600	{
6601	enum insn_code icode;
6602	unsigned int i, u;
6603	rtx tmp, sel_qi;
6604
6605	u = GET_MODE_UNIT_SIZE (mode);
6606
6607	if (!target || GET_MODE (target) != mode)
6608	target = gen_reg_rtx (mode);
6609
6610	icode = direct_optab_handler (op: vec_perm_optab, mode);
6611	if (icode != CODE_FOR_nothing)
6612	{
6613	tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6614	if (tmp)
6615	return tmp;
6616	}
6617
6618	/* As a special case to aid several targets, lower the element-based
6619	permutation to a byte-based permutation and try again. */
6620	machine_mode qimode;
6621	if (!qimode_for_vec_perm (mode).exists (mode: &qimode)
6622	|| maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
6623	return NULL_RTX;
6624	icode = direct_optab_handler (op: vec_perm_optab, mode: qimode);
6625	if (icode == CODE_FOR_nothing)
6626	return NULL_RTX;
6627
6628	/* Multiply each element by its byte size. */
6629	machine_mode selmode = GET_MODE (sel);
6630	if (u == 2)
6631	sel = expand_simple_binop (mode: selmode, code: PLUS, op0: sel, op1: sel,
6632	NULL, unsignedp: 0, methods: OPTAB_DIRECT);
6633	else
6634	sel = expand_simple_binop (mode: selmode, code: ASHIFT, op0: sel,
6635	op1: gen_int_shift_amount (selmode, exact_log2 (x: u)),
6636	NULL, unsignedp: 0, methods: OPTAB_DIRECT);
6637	gcc_assert (sel != NULL);
6638
6639	/* Broadcast the low byte each element into each of its bytes.
6640	The encoding has U interleaved stepped patterns, one for each
6641	byte of an element. */
6642	vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
6643	unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
6644	for (i = 0; i < 3; ++i)
6645	for (unsigned int j = 0; j < u; ++j)
6646	const_sel.quick_push (obj: i * u + low_byte_in_u);
6647	sel = gen_lowpart (qimode, sel);
6648	sel = expand_vec_perm_const (mode: qimode, v0: sel, v1: sel, sel: const_sel, sel_mode: qimode, NULL);
6649	gcc_assert (sel != NULL);
6650
6651	/* Add the byte offset to each byte element. */
6652	/* Note that the definition of the indicies here is memory ordering,
6653	so there should be no difference between big and little endian. */
6654	rtx_vector_builder byte_indices (qimode, u, 1);
6655	for (i = 0; i < u; ++i)
6656	byte_indices.quick_push (GEN_INT (i));
6657	tmp = byte_indices.build ();
6658	sel_qi = expand_simple_binop (mode: qimode, code: PLUS, op0: sel, op1: tmp,
6659	target: sel, unsignedp: 0, methods: OPTAB_DIRECT);
6660	gcc_assert (sel_qi != NULL);
6661
6662	tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
6663	tmp = expand_vec_perm_1 (icode, target: tmp, gen_lowpart (qimode, v0),
6664	gen_lowpart (qimode, v1), sel: sel_qi);
6665	if (tmp)
6666	tmp = gen_lowpart (mode, tmp);
6667	return tmp;
6668	}
6669
6670	/* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
6671	Use TARGET for the result if nonnull and convenient. */
6672
6673	rtx
6674	expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
6675	{
6676	class expand_operand ops[3];
6677	enum insn_code icode;
6678	machine_mode emode = GET_MODE_INNER (vmode);
6679
6680	icode = direct_optab_handler (op: vec_series_optab, mode: vmode);
6681	gcc_assert (icode != CODE_FOR_nothing);
6682
6683	create_output_operand (op: &ops[0], x: target, mode: vmode);
6684	create_input_operand (op: &ops[1], value: op0, mode: emode);
6685	create_input_operand (op: &ops[2], value: op1, mode: emode);
6686
6687	expand_insn (icode, nops: 3, ops);
6688	return ops[0].value;
6689	}
6690
6691	/* Generate insns for a vector comparison into a mask. */
6692
6693	rtx
6694	expand_vec_cmp_expr (tree type, tree exp, rtx target)
6695	{
6696	class expand_operand ops[4];
6697	enum insn_code icode;
6698	rtx comparison;
6699	machine_mode mask_mode = TYPE_MODE (type);
6700	machine_mode vmode;
6701	bool unsignedp;
6702	tree op0a, op0b;
6703	enum tree_code tcode;
6704
6705	op0a = TREE_OPERAND (exp, 0);
6706	op0b = TREE_OPERAND (exp, 1);
6707	tcode = TREE_CODE (exp);
6708
6709	unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
6710	vmode = TYPE_MODE (TREE_TYPE (op0a));
6711
6712	icode = get_vec_cmp_icode (vmode, mask_mode, uns: unsignedp);
6713	if (icode == CODE_FOR_nothing)
6714	{
6715	if (tcode == EQ_EXPR || tcode == NE_EXPR)
6716	icode = get_vec_cmp_eq_icode (vmode, mask_mode);
6717	if (icode == CODE_FOR_nothing)
6718	return 0;
6719	}
6720
6721	comparison = vector_compare_rtx (cmp_mode: mask_mode, tcode, t_op0: op0a, t_op1: op0b,
6722	unsignedp, icode, opno: 2);
6723	create_output_operand (op: &ops[0], x: target, mode: mask_mode);
6724	create_fixed_operand (op: &ops[1], x: comparison);
6725	create_fixed_operand (op: &ops[2], XEXP (comparison, 0));
6726	create_fixed_operand (op: &ops[3], XEXP (comparison, 1));
6727	expand_insn (icode, nops: 4, ops);
6728	return ops[0].value;
6729	}
6730
6731	/* Expand a highpart multiply. */
6732
6733	rtx
6734	expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
6735	rtx target, bool uns_p)
6736	{
6737	class expand_operand eops[3];
6738	enum insn_code icode;
6739	int method, i;
6740	machine_mode wmode;
6741	rtx m1, m2;
6742	optab tab1, tab2;
6743
6744	method = can_mult_highpart_p (mode, uns_p);
6745	switch (method)
6746	{
6747	case 0:
6748	return NULL_RTX;
6749	case 1:
6750	tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
6751	return expand_binop (mode, binoptab: tab1, op0, op1, target, unsignedp: uns_p,
6752	methods: OPTAB_LIB_WIDEN);
6753	case 2:
6754	tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6755	tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6756	break;
6757	case 3:
6758	tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6759	tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6760	if (BYTES_BIG_ENDIAN)
6761	std::swap (a&: tab1, b&: tab2);
6762	break;
6763	default:
6764	gcc_unreachable ();
6765	}
6766
6767	icode = optab_handler (op: tab1, mode);
6768	wmode = insn_data[icode].operand[0].mode;
6769	gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
6770	GET_MODE_NUNITS (mode)));
6771	gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
6772
6773	create_output_operand (op: &eops[0], x: gen_reg_rtx (wmode), mode: wmode);
6774	create_input_operand (op: &eops[1], value: op0, mode);
6775	create_input_operand (op: &eops[2], value: op1, mode);
6776	expand_insn (icode, nops: 3, ops: eops);
6777	m1 = gen_lowpart (mode, eops[0].value);
6778
6779	create_output_operand (op: &eops[0], x: gen_reg_rtx (wmode), mode: wmode);
6780	create_input_operand (op: &eops[1], value: op0, mode);
6781	create_input_operand (op: &eops[2], value: op1, mode);
6782	expand_insn (icode: optab_handler (op: tab2, mode), nops: 3, ops: eops);
6783	m2 = gen_lowpart (mode, eops[0].value);
6784
6785	vec_perm_builder sel;
6786	if (method == 2)
6787	{
6788	/* The encoding has 2 interleaved stepped patterns. */
6789	sel.new_vector (full_nelts: GET_MODE_NUNITS (mode), npatterns: 2, nelts_per_pattern: 3);
6790	for (i = 0; i < 6; ++i)
6791	sel.quick_push (obj: !BYTES_BIG_ENDIAN + (i & ~1)
6792	+ ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
6793	}
6794	else
6795	{
6796	/* The encoding has a single interleaved stepped pattern. */
6797	sel.new_vector (full_nelts: GET_MODE_NUNITS (mode), npatterns: 1, nelts_per_pattern: 3);
6798	for (i = 0; i < 3; ++i)
6799	sel.quick_push (obj: 2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
6800	}
6801
6802	return expand_vec_perm_const (mode, v0: m1, v1: m2, sel, BLKmode, target);
6803	}
6804
6805	/* Helper function to find the MODE_CC set in a sync_compare_and_swap
6806	pattern. */
6807
6808	static void
6809	find_cc_set (rtx x, const_rtx pat, void *data)
6810	{
6811	if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
6812	&& GET_CODE (pat) == SET)
6813	{
6814	rtx *p_cc_reg = (rtx *) data;
6815	gcc_assert (!*p_cc_reg);
6816	*p_cc_reg = x;
6817	}
6818	}
6819
6820	/* This is a helper function for the other atomic operations. This function
6821	emits a loop that contains SEQ that iterates until a compare-and-swap
6822	operation at the end succeeds. MEM is the memory to be modified. SEQ is
6823	a set of instructions that takes a value from OLD_REG as an input and
6824	produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
6825	set to the current contents of MEM. After SEQ, a compare-and-swap will
6826	attempt to update MEM with NEW_REG. The function returns true when the
6827	loop was generated successfully. */
6828
6829	static bool
6830	expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
6831	{
6832	machine_mode mode = GET_MODE (mem);
6833	rtx_code_label *label;
6834	rtx cmp_reg, success, oldval;
6835
6836	/* The loop we want to generate looks like
6837
6838	cmp_reg = mem;
6839	label:
6840	old_reg = cmp_reg;
6841	seq;
6842	(success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6843	if (success)
6844	goto label;
6845
6846	Note that we only do the plain load from memory once. Subsequent
6847	iterations use the value loaded by the compare-and-swap pattern. */
6848
6849	label = gen_label_rtx ();
6850	cmp_reg = gen_reg_rtx (mode);
6851
6852	emit_move_insn (cmp_reg, mem);
6853	emit_label (label);
6854	emit_move_insn (old_reg, cmp_reg);
6855	if (seq)
6856	emit_insn (seq);
6857
6858	success = NULL_RTX;
6859	oldval = cmp_reg;
6860	if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
6861	new_reg, false, MEMMODEL_SYNC_SEQ_CST,
6862	MEMMODEL_RELAXED))
6863	return false;
6864
6865	if (oldval != cmp_reg)
6866	emit_move_insn (cmp_reg, oldval);
6867
6868	/* Mark this jump predicted not taken. */
6869	emit_cmp_and_jump_insns (x: success, const0_rtx, comparison: EQ, const0_rtx,
6870	GET_MODE (success), unsignedp: 1, label,
6871	prob: profile_probability::guessed_never ());
6872	return true;
6873	}
6874
6875
6876	/* This function tries to emit an atomic_exchange intruction. VAL is written
6877	to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6878	using TARGET if possible. */
6879
6880	static rtx
6881	maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6882	{
6883	machine_mode mode = GET_MODE (mem);
6884	enum insn_code icode;
6885
6886	/* If the target supports the exchange directly, great. */
6887	icode = direct_optab_handler (op: atomic_exchange_optab, mode);
6888	if (icode != CODE_FOR_nothing)
6889	{
6890	class expand_operand ops[4];
6891
6892	create_output_operand (op: &ops[0], x: target, mode);
6893	create_fixed_operand (op: &ops[1], x: mem);
6894	create_input_operand (op: &ops[2], value: val, mode);
6895	create_integer_operand (&ops[3], model);
6896	if (maybe_expand_insn (icode, nops: 4, ops))
6897	return ops[0].value;
6898	}
6899
6900	return NULL_RTX;
6901	}
6902
6903	/* This function tries to implement an atomic exchange operation using
6904	__sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6905	The previous contents of *MEM are returned, using TARGET if possible.
6906	Since this instructionn is an acquire barrier only, stronger memory
6907	models may require additional barriers to be emitted. */
6908
6909	static rtx
6910	maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
6911	enum memmodel model)
6912	{
6913	machine_mode mode = GET_MODE (mem);
6914	enum insn_code icode;
6915	rtx_insn *last_insn = get_last_insn ();
6916
6917	icode = optab_handler (op: sync_lock_test_and_set_optab, mode);
6918
6919	/* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
6920	exists, and the memory model is stronger than acquire, add a release
6921	barrier before the instruction. */
6922
6923	if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
6924	expand_mem_thread_fence (model);
6925
6926	if (icode != CODE_FOR_nothing)
6927	{
6928	class expand_operand ops[3];
6929	create_output_operand (op: &ops[0], x: target, mode);
6930	create_fixed_operand (op: &ops[1], x: mem);
6931	create_input_operand (op: &ops[2], value: val, mode);
6932	if (maybe_expand_insn (icode, nops: 3, ops))
6933	return ops[0].value;
6934	}
6935
6936	/* If an external test-and-set libcall is provided, use that instead of
6937	any external compare-and-swap that we might get from the compare-and-
6938	swap-loop expansion later. */
6939	if (!can_compare_and_swap_p (mode, false))
6940	{
6941	rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
6942	if (libfunc != NULL)
6943	{
6944	rtx addr;
6945
6946	addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6947	return emit_library_call_value (fun: libfunc, NULL_RTX, fn_type: LCT_NORMAL,
6948	outmode: mode, arg1: addr, arg1_mode: ptr_mode,
6949	arg2: val, arg2_mode: mode);
6950	}
6951	}
6952
6953	/* If the test_and_set can't be emitted, eliminate any barrier that might
6954	have been emitted. */
6955	delete_insns_since (last_insn);
6956	return NULL_RTX;
6957	}
6958
6959	/* This function tries to implement an atomic exchange operation using a
6960	compare_and_swap loop. VAL is written to *MEM. The previous contents of
6961	*MEM are returned, using TARGET if possible. No memory model is required
6962	since a compare_and_swap loop is seq-cst. */
6963
6964	static rtx
6965	maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
6966	{
6967	machine_mode mode = GET_MODE (mem);
6968
6969	if (can_compare_and_swap_p (mode, true))
6970	{
6971	if (!target || !register_operand (target, mode))
6972	target = gen_reg_rtx (mode);
6973	if (expand_compare_and_swap_loop (mem, old_reg: target, new_reg: val, NULL_RTX))
6974	return target;
6975	}
6976
6977	return NULL_RTX;
6978	}
6979
6980	/* This function tries to implement an atomic test-and-set operation
6981	using the atomic_test_and_set instruction pattern. A boolean value
6982	is returned from the operation, using TARGET if possible. */
6983
6984	static rtx
6985	maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6986	{
6987	machine_mode pat_bool_mode;
6988	class expand_operand ops[3];
6989
6990	if (!targetm.have_atomic_test_and_set ())
6991	return NULL_RTX;
6992
6993	/* While we always get QImode from __atomic_test_and_set, we get
6994	other memory modes from __sync_lock_test_and_set. Note that we
6995	use no endian adjustment here. This matches the 4.6 behavior
6996	in the Sparc backend. */
6997	enum insn_code icode = targetm.code_for_atomic_test_and_set;
6998	gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
6999	if (GET_MODE (mem) != QImode)
7000	mem = adjust_address_nv (mem, QImode, 0);
7001
7002	pat_bool_mode = insn_data[icode].operand[0].mode;
7003	create_output_operand (op: &ops[0], x: target, mode: pat_bool_mode);
7004	create_fixed_operand (op: &ops[1], x: mem);
7005	create_integer_operand (&ops[2], model);
7006
7007	if (maybe_expand_insn (icode, nops: 3, ops))
7008	return ops[0].value;
7009	return NULL_RTX;
7010	}
7011
7012	/* This function expands the legacy _sync_lock test_and_set operation which is
7013	generally an atomic exchange. Some limited targets only allow the
7014	constant 1 to be stored. This is an ACQUIRE operation.
7015
7016	TARGET is an optional place to stick the return value.
7017	MEM is where VAL is stored. */
7018
7019	rtx
7020	expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
7021	{
7022	rtx ret;
7023
7024	/* Try an atomic_exchange first. */
7025	ret = maybe_emit_atomic_exchange (target, mem, val, model: MEMMODEL_SYNC_ACQUIRE);
7026	if (ret)
7027	return ret;
7028
7029	ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
7030	model: MEMMODEL_SYNC_ACQUIRE);
7031	if (ret)
7032	return ret;
7033
7034	ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7035	if (ret)
7036	return ret;
7037
7038	/* If there are no other options, try atomic_test_and_set if the value
7039	being stored is 1. */
7040	if (val == const1_rtx)
7041	ret = maybe_emit_atomic_test_and_set (target, mem, model: MEMMODEL_SYNC_ACQUIRE);
7042
7043	return ret;
7044	}
7045
7046	/* This function expands the atomic test_and_set operation:
7047	atomically store a boolean TRUE into MEM and return the previous value.
7048
7049	MEMMODEL is the memory model variant to use.
7050	TARGET is an optional place to stick the return value. */
7051
7052	rtx
7053	expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7054	{
7055	machine_mode mode = GET_MODE (mem);
7056	rtx ret, trueval, subtarget;
7057
7058	ret = maybe_emit_atomic_test_and_set (target, mem, model);
7059	if (ret)
7060	return ret;
7061
7062	/* Be binary compatible with non-default settings of trueval, and different
7063	cpu revisions. E.g. one revision may have atomic-test-and-set, but
7064	another only has atomic-exchange. */
7065	if (targetm.atomic_test_and_set_trueval == 1)
7066	{
7067	trueval = const1_rtx;
7068	subtarget = target ? target : gen_reg_rtx (mode);
7069	}
7070	else
7071	{
7072	trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
7073	subtarget = gen_reg_rtx (mode);
7074	}
7075
7076	/* Try the atomic-exchange optab... */
7077	ret = maybe_emit_atomic_exchange (target: subtarget, mem, val: trueval, model);
7078
7079	/* ... then an atomic-compare-and-swap loop ... */
7080	if (!ret)
7081	ret = maybe_emit_compare_and_swap_exchange_loop (target: subtarget, mem, val: trueval);
7082
7083	/* ... before trying the vaguely defined legacy lock_test_and_set. */
7084	if (!ret)
7085	ret = maybe_emit_sync_lock_test_and_set (target: subtarget, mem, val: trueval, model);
7086
7087	/* Recall that the legacy lock_test_and_set optab was allowed to do magic
7088	things with the value 1. Thus we try again without trueval. */
7089	if (!ret && targetm.atomic_test_and_set_trueval != 1)
7090	{
7091	ret = maybe_emit_sync_lock_test_and_set (target: subtarget, mem, const1_rtx, model);
7092
7093	if (ret)
7094	{
7095	/* Rectify the not-one trueval. */
7096	ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
7097	gcc_assert (ret);
7098	}
7099	}
7100
7101	return ret;
7102	}
7103
7104	/* This function expands the atomic exchange operation:
7105	atomically store VAL in MEM and return the previous value in MEM.
7106
7107	MEMMODEL is the memory model variant to use.
7108	TARGET is an optional place to stick the return value. */
7109
7110	rtx
7111	expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7112	{
7113	machine_mode mode = GET_MODE (mem);
7114	rtx ret;
7115
7116	/* If loads are not atomic for the required size and we are not called to
7117	provide a __sync builtin, do not do anything so that we stay consistent
7118	with atomic loads of the same size. */
7119	if (!can_atomic_load_p (mode) && !is_mm_sync (model))
7120	return NULL_RTX;
7121
7122	ret = maybe_emit_atomic_exchange (target, mem, val, model);
7123
7124	/* Next try a compare-and-swap loop for the exchange. */
7125	if (!ret)
7126	ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7127
7128	return ret;
7129	}
7130
7131	/* This function expands the atomic compare exchange operation:
7132
7133	*PTARGET_BOOL is an optional place to store the boolean success/failure.
7134	*PTARGET_OVAL is an optional place to store the old value from memory.
7135	Both target parameters may be NULL or const0_rtx to indicate that we do
7136	not care about that return value. Both target parameters are updated on
7137	success to the actual location of the corresponding result.
7138
7139	MEMMODEL is the memory model variant to use.
7140
7141	The return value of the function is true for success. */
7142
7143	bool
7144	expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
7145	rtx mem, rtx expected, rtx desired,
7146	bool is_weak, enum memmodel succ_model,
7147	enum memmodel fail_model)
7148	{
7149	machine_mode mode = GET_MODE (mem);
7150	class expand_operand ops[8];
7151	enum insn_code icode;
7152	rtx target_oval, target_bool = NULL_RTX;
7153	rtx libfunc;
7154
7155	/* If loads are not atomic for the required size and we are not called to
7156	provide a __sync builtin, do not do anything so that we stay consistent
7157	with atomic loads of the same size. */
7158	if (!can_atomic_load_p (mode) && !is_mm_sync (model: succ_model))
7159	return false;
7160
7161	/* Load expected into a register for the compare and swap. */
7162	if (MEM_P (expected))
7163	expected = copy_to_reg (expected);
7164
7165	/* Make sure we always have some place to put the return oldval.
7166	Further, make sure that place is distinct from the input expected,
7167	just in case we need that path down below. */
7168	if (ptarget_oval && *ptarget_oval == const0_rtx)
7169	ptarget_oval = NULL;
7170
7171	if (ptarget_oval == NULL
7172	|| (target_oval = *ptarget_oval) == NULL
7173	|| reg_overlap_mentioned_p (expected, target_oval))
7174	target_oval = gen_reg_rtx (mode);
7175
7176	icode = direct_optab_handler (op: atomic_compare_and_swap_optab, mode);
7177	if (icode != CODE_FOR_nothing)
7178	{
7179	machine_mode bool_mode = insn_data[icode].operand[0].mode;
7180
7181	if (ptarget_bool && *ptarget_bool == const0_rtx)
7182	ptarget_bool = NULL;
7183
7184	/* Make sure we always have a place for the bool operand. */
7185	if (ptarget_bool == NULL
7186	|| (target_bool = *ptarget_bool) == NULL
7187	|| GET_MODE (target_bool) != bool_mode)
7188	target_bool = gen_reg_rtx (bool_mode);
7189
7190	/* Emit the compare_and_swap. */
7191	create_output_operand (op: &ops[0], x: target_bool, mode: bool_mode);
7192	create_output_operand (op: &ops[1], x: target_oval, mode);
7193	create_fixed_operand (op: &ops[2], x: mem);
7194	create_input_operand (op: &ops[3], value: expected, mode);
7195	create_input_operand (op: &ops[4], value: desired, mode);
7196	create_integer_operand (&ops[5], is_weak);
7197	create_integer_operand (&ops[6], succ_model);
7198	create_integer_operand (&ops[7], fail_model);
7199	if (maybe_expand_insn (icode, nops: 8, ops))
7200	{
7201	/* Return success/failure. */
7202	target_bool = ops[0].value;
7203	target_oval = ops[1].value;
7204	goto success;
7205	}
7206	}
7207
7208	/* Otherwise fall back to the original __sync_val_compare_and_swap
7209	which is always seq-cst. */
7210	icode = optab_handler (op: sync_compare_and_swap_optab, mode);
7211	if (icode != CODE_FOR_nothing)
7212	{
7213	rtx cc_reg;
7214
7215	create_output_operand (op: &ops[0], x: target_oval, mode);
7216	create_fixed_operand (op: &ops[1], x: mem);
7217	create_input_operand (op: &ops[2], value: expected, mode);
7218	create_input_operand (op: &ops[3], value: desired, mode);
7219	if (!maybe_expand_insn (icode, nops: 4, ops))
7220	return false;
7221
7222	target_oval = ops[0].value;
7223
7224	/* If the caller isn't interested in the boolean return value,
7225	skip the computation of it. */
7226	if (ptarget_bool == NULL)
7227	goto success;
7228
7229	/* Otherwise, work out if the compare-and-swap succeeded. */
7230	cc_reg = NULL_RTX;
7231	if (have_insn_for (code: COMPARE, CCmode))
7232	note_stores (get_last_insn (), find_cc_set, &cc_reg);
7233	if (cc_reg)
7234	{
7235	target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
7236	const0_rtx, VOIDmode, 0, 1);
7237	goto success;
7238	}
7239	goto success_bool_from_val;
7240	}
7241
7242	/* Also check for library support for __sync_val_compare_and_swap. */
7243	libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
7244	if (libfunc != NULL)
7245	{
7246	rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7247	rtx target = emit_library_call_value (fun: libfunc, NULL_RTX, fn_type: LCT_NORMAL,
7248	outmode: mode, arg1: addr, arg1_mode: ptr_mode,
7249	arg2: expected, arg2_mode: mode, arg3: desired, arg3_mode: mode);
7250	emit_move_insn (target_oval, target);
7251
7252	/* Compute the boolean return value only if requested. */
7253	if (ptarget_bool)
7254	goto success_bool_from_val;
7255	else
7256	goto success;
7257	}
7258
7259	/* Failure. */
7260	return false;
7261
7262	success_bool_from_val:
7263	target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
7264	expected, VOIDmode, 1, 1);
7265	success:
7266	/* Make sure that the oval output winds up where the caller asked. */
7267	if (ptarget_oval)
7268	*ptarget_oval = target_oval;
7269	if (ptarget_bool)
7270	*ptarget_bool = target_bool;
7271	return true;
7272	}
7273
7274	/* Generate asm volatile("" : : : "memory") as the memory blockage. */
7275
7276	static void
7277	expand_asm_memory_blockage (void)
7278	{
7279	rtx asm_op, clob;
7280
7281	asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7282	rtvec_alloc (0), rtvec_alloc (0),
7283	rtvec_alloc (0), UNKNOWN_LOCATION);
7284	MEM_VOLATILE_P (asm_op) = 1;
7285
7286	clob = gen_rtx_SCRATCH (VOIDmode);
7287	clob = gen_rtx_MEM (BLKmode, clob);
7288	clob = gen_rtx_CLOBBER (VOIDmode, clob);
7289
7290	emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
7291	}
7292
7293	/* Do not propagate memory accesses across this point. */
7294
7295	static void
7296	expand_memory_blockage (void)
7297	{
7298	if (targetm.have_memory_blockage ())
7299	emit_insn (targetm.gen_memory_blockage ());
7300	else
7301	expand_asm_memory_blockage ();
7302	}
7303
7304	/* Generate asm volatile("" : : : "memory") as a memory blockage, at the
7305	same time clobbering the register set specified by REGS. */
7306
7307	void
7308	expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs)
7309	{
7310	rtx asm_op, clob_mem;
7311
7312	unsigned int num_of_regs = 0;
7313	for (unsigned int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
7314	if (TEST_HARD_REG_BIT (set: regs, bit: i))
7315	num_of_regs++;
7316
7317	asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7318	rtvec_alloc (0), rtvec_alloc (0),
7319	rtvec_alloc (0), UNKNOWN_LOCATION);
7320	MEM_VOLATILE_P (asm_op) = 1;
7321
7322	rtvec v = rtvec_alloc (num_of_regs + 2);
7323
7324	clob_mem = gen_rtx_SCRATCH (VOIDmode);
7325	clob_mem = gen_rtx_MEM (BLKmode, clob_mem);
7326	clob_mem = gen_rtx_CLOBBER (VOIDmode, clob_mem);
7327
7328	RTVEC_ELT (v, 0) = asm_op;
7329	RTVEC_ELT (v, 1) = clob_mem;
7330
7331	if (num_of_regs > 0)
7332	{
7333	unsigned int j = 2;
7334	for (unsigned int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
7335	if (TEST_HARD_REG_BIT (set: regs, bit: i))
7336	{
7337	RTVEC_ELT (v, j) = gen_rtx_CLOBBER (VOIDmode, regno_reg_rtx[i]);
7338	j++;
7339	}
7340	gcc_assert (j == (num_of_regs + 2));
7341	}
7342
7343	emit_insn (gen_rtx_PARALLEL (VOIDmode, v));
7344	}
7345
7346	/* This routine will either emit the mem_thread_fence pattern or issue a
7347	sync_synchronize to generate a fence for memory model MEMMODEL. */
7348
7349	void
7350	expand_mem_thread_fence (enum memmodel model)
7351	{
7352	if (is_mm_relaxed (model))
7353	return;
7354	if (targetm.have_mem_thread_fence ())
7355	{
7356	emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
7357	expand_memory_blockage ();
7358	}
7359	else if (targetm.have_memory_barrier ())
7360	emit_insn (targetm.gen_memory_barrier ());
7361	else if (synchronize_libfunc != NULL_RTX)
7362	emit_library_call (synchronize_libfunc, fn_type: LCT_NORMAL, VOIDmode);
7363	else
7364	expand_memory_blockage ();
7365	}
7366
7367	/* Emit a signal fence with given memory model. */
7368
7369	void
7370	expand_mem_signal_fence (enum memmodel model)
7371	{
7372	/* No machine barrier is required to implement a signal fence, but
7373	a compiler memory barrier must be issued, except for relaxed MM. */
7374	if (!is_mm_relaxed (model))
7375	expand_memory_blockage ();
7376	}
7377
7378	/* This function expands the atomic load operation:
7379	return the atomically loaded value in MEM.
7380
7381	MEMMODEL is the memory model variant to use.
7382	TARGET is an option place to stick the return value. */
7383
7384	rtx
7385	expand_atomic_load (rtx target, rtx mem, enum memmodel model)
7386	{
7387	machine_mode mode = GET_MODE (mem);
7388	enum insn_code icode;
7389
7390	/* If the target supports the load directly, great. */
7391	icode = direct_optab_handler (op: atomic_load_optab, mode);
7392	if (icode != CODE_FOR_nothing)
7393	{
7394	class expand_operand ops[3];
7395	rtx_insn *last = get_last_insn ();
7396	if (is_mm_seq_cst (model))
7397	expand_memory_blockage ();
7398
7399	create_output_operand (op: &ops[0], x: target, mode);
7400	create_fixed_operand (op: &ops[1], x: mem);
7401	create_integer_operand (&ops[2], model);
7402	if (maybe_expand_insn (icode, nops: 3, ops))
7403	{
7404	if (!is_mm_relaxed (model))
7405	expand_memory_blockage ();
7406	return ops[0].value;
7407	}
7408	delete_insns_since (last);
7409	}
7410
7411	/* If the size of the object is greater than word size on this target,
7412	then we assume that a load will not be atomic. We could try to
7413	emulate a load with a compare-and-swap operation, but the store that
7414	doing this could result in would be incorrect if this is a volatile
7415	atomic load or targetting read-only-mapped memory. */
7416	if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7417	/* If there is no atomic load, leave the library call. */
7418	return NULL_RTX;
7419
7420	/* Otherwise assume loads are atomic, and emit the proper barriers. */
7421	if (!target || target == const0_rtx)
7422	target = gen_reg_rtx (mode);
7423
7424	/* For SEQ_CST, emit a barrier before the load. */
7425	if (is_mm_seq_cst (model))
7426	expand_mem_thread_fence (model);
7427
7428	emit_move_insn (target, mem);
7429
7430	/* Emit the appropriate barrier after the load. */
7431	expand_mem_thread_fence (model);
7432
7433	return target;
7434	}
7435
7436	/* This function expands the atomic store operation:
7437	Atomically store VAL in MEM.
7438	MEMMODEL is the memory model variant to use.
7439	USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7440	function returns const0_rtx if a pattern was emitted. */
7441
7442	rtx
7443	expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
7444	{
7445	machine_mode mode = GET_MODE (mem);
7446	enum insn_code icode;
7447	class expand_operand ops[3];
7448
7449	/* If the target supports the store directly, great. */
7450	icode = direct_optab_handler (op: atomic_store_optab, mode);
7451	if (icode != CODE_FOR_nothing)
7452	{
7453	rtx_insn *last = get_last_insn ();
7454	if (!is_mm_relaxed (model))
7455	expand_memory_blockage ();
7456	create_fixed_operand (op: &ops[0], x: mem);
7457	create_input_operand (op: &ops[1], value: val, mode);
7458	create_integer_operand (&ops[2], model);
7459	if (maybe_expand_insn (icode, nops: 3, ops))
7460	{
7461	if (is_mm_seq_cst (model))
7462	expand_memory_blockage ();
7463	return const0_rtx;
7464	}
7465	delete_insns_since (last);
7466	}
7467
7468	/* If using __sync_lock_release is a viable alternative, try it.
7469	Note that this will not be set to true if we are expanding a generic
7470	__atomic_store_n. */
7471	if (use_release)
7472	{
7473	icode = direct_optab_handler (op: sync_lock_release_optab, mode);
7474	if (icode != CODE_FOR_nothing)
7475	{
7476	create_fixed_operand (op: &ops[0], x: mem);
7477	create_input_operand (op: &ops[1], const0_rtx, mode);
7478	if (maybe_expand_insn (icode, nops: 2, ops))
7479	{
7480	/* lock_release is only a release barrier. */
7481	if (is_mm_seq_cst (model))
7482	expand_mem_thread_fence (model);
7483	return const0_rtx;
7484	}
7485	}
7486	}
7487
7488	/* If the size of the object is greater than word size on this target,
7489	a default store will not be atomic. */
7490	if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7491	{
7492	/* If loads are atomic or we are called to provide a __sync builtin,
7493	we can try a atomic_exchange and throw away the result. Otherwise,
7494	don't do anything so that we do not create an inconsistency between
7495	loads and stores. */
7496	if (can_atomic_load_p (mode) || is_mm_sync (model))
7497	{
7498	rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
7499	if (!target)
7500	target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
7501	val);
7502	if (target)
7503	return const0_rtx;
7504	}
7505	return NULL_RTX;
7506	}
7507
7508	/* Otherwise assume stores are atomic, and emit the proper barriers. */
7509	expand_mem_thread_fence (model);
7510
7511	emit_move_insn (mem, val);
7512
7513	/* For SEQ_CST, also emit a barrier after the store. */
7514	if (is_mm_seq_cst (model))
7515	expand_mem_thread_fence (model);
7516
7517	return const0_rtx;
7518	}
7519
7520
7521	/* Structure containing the pointers and values required to process the
7522	various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7523
7524	struct atomic_op_functions
7525	{
7526	direct_optab mem_fetch_before;
7527	direct_optab mem_fetch_after;
7528	direct_optab mem_no_result;
7529	optab fetch_before;
7530	optab fetch_after;
7531	direct_optab no_result;
7532	enum rtx_code reverse_code;
7533	};
7534
7535
7536	/* Fill in structure pointed to by OP with the various optab entries for an
7537	operation of type CODE. */
7538
7539	static void
7540	get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
7541	{
7542	gcc_assert (op!= NULL);
7543
7544	/* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7545	in the source code during compilation, and the optab entries are not
7546	computable until runtime. Fill in the values at runtime. */
7547	switch (code)
7548	{
7549	case PLUS:
7550	op->mem_fetch_before = atomic_fetch_add_optab;
7551	op->mem_fetch_after = atomic_add_fetch_optab;
7552	op->mem_no_result = atomic_add_optab;
7553	op->fetch_before = sync_old_add_optab;
7554	op->fetch_after = sync_new_add_optab;
7555	op->no_result = sync_add_optab;
7556	op->reverse_code = MINUS;
7557	break;
7558	case MINUS:
7559	op->mem_fetch_before = atomic_fetch_sub_optab;
7560	op->mem_fetch_after = atomic_sub_fetch_optab;
7561	op->mem_no_result = atomic_sub_optab;
7562	op->fetch_before = sync_old_sub_optab;
7563	op->fetch_after = sync_new_sub_optab;
7564	op->no_result = sync_sub_optab;
7565	op->reverse_code = PLUS;
7566	break;
7567	case XOR:
7568	op->mem_fetch_before = atomic_fetch_xor_optab;
7569	op->mem_fetch_after = atomic_xor_fetch_optab;
7570	op->mem_no_result = atomic_xor_optab;
7571	op->fetch_before = sync_old_xor_optab;
7572	op->fetch_after = sync_new_xor_optab;
7573	op->no_result = sync_xor_optab;
7574	op->reverse_code = XOR;
7575	break;
7576	case AND:
7577	op->mem_fetch_before = atomic_fetch_and_optab;
7578	op->mem_fetch_after = atomic_and_fetch_optab;
7579	op->mem_no_result = atomic_and_optab;
7580	op->fetch_before = sync_old_and_optab;
7581	op->fetch_after = sync_new_and_optab;
7582	op->no_result = sync_and_optab;
7583	op->reverse_code = UNKNOWN;
7584	break;
7585	case IOR:
7586	op->mem_fetch_before = atomic_fetch_or_optab;
7587	op->mem_fetch_after = atomic_or_fetch_optab;
7588	op->mem_no_result = atomic_or_optab;
7589	op->fetch_before = sync_old_ior_optab;
7590	op->fetch_after = sync_new_ior_optab;
7591	op->no_result = sync_ior_optab;
7592	op->reverse_code = UNKNOWN;
7593	break;
7594	case NOT:
7595	op->mem_fetch_before = atomic_fetch_nand_optab;
7596	op->mem_fetch_after = atomic_nand_fetch_optab;
7597	op->mem_no_result = atomic_nand_optab;
7598	op->fetch_before = sync_old_nand_optab;
7599	op->fetch_after = sync_new_nand_optab;
7600	op->no_result = sync_nand_optab;
7601	op->reverse_code = UNKNOWN;
7602	break;
7603	default:
7604	gcc_unreachable ();
7605	}
7606	}
7607
7608	/* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7609	using memory order MODEL. If AFTER is true the operation needs to return
7610	the value of *MEM after the operation, otherwise the previous value.
7611	TARGET is an optional place to place the result. The result is unused if
7612	it is const0_rtx.
7613	Return the result if there is a better sequence, otherwise NULL_RTX. */
7614
7615	static rtx
7616	maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7617	enum memmodel model, bool after)
7618	{
7619	/* If the value is prefetched, or not used, it may be possible to replace
7620	the sequence with a native exchange operation. */
7621	if (!after || target == const0_rtx)
7622	{
7623	/* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7624	if (code == AND && val == const0_rtx)
7625	{
7626	if (target == const0_rtx)
7627	target = gen_reg_rtx (GET_MODE (mem));
7628	return maybe_emit_atomic_exchange (target, mem, val, model);
7629	}
7630
7631	/* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7632	if (code == IOR && val == constm1_rtx)
7633	{
7634	if (target == const0_rtx)
7635	target = gen_reg_rtx (GET_MODE (mem));
7636	return maybe_emit_atomic_exchange (target, mem, val, model);
7637	}
7638	}
7639
7640	return NULL_RTX;
7641	}
7642
7643	/* Try to emit an instruction for a specific operation varaition.
7644	OPTAB contains the OP functions.
7645	TARGET is an optional place to return the result. const0_rtx means unused.
7646	MEM is the memory location to operate on.
7647	VAL is the value to use in the operation.
7648	USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7649	MODEL is the memory model, if used.
7650	AFTER is true if the returned result is the value after the operation. */
7651
7652	static rtx
7653	maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
7654	rtx val, bool use_memmodel, enum memmodel model, bool after)
7655	{
7656	machine_mode mode = GET_MODE (mem);
7657	class expand_operand ops[4];
7658	enum insn_code icode;
7659	int op_counter = 0;
7660	int num_ops;
7661
7662	/* Check to see if there is a result returned. */
7663	if (target == const0_rtx)
7664	{
7665	if (use_memmodel)
7666	{
7667	icode = direct_optab_handler (op: optab->mem_no_result, mode);
7668	create_integer_operand (&ops[2], model);
7669	num_ops = 3;
7670	}
7671	else
7672	{
7673	icode = direct_optab_handler (op: optab->no_result, mode);
7674	num_ops = 2;
7675	}
7676	}
7677	/* Otherwise, we need to generate a result. */
7678	else
7679	{
7680	if (use_memmodel)
7681	{
7682	icode = direct_optab_handler (op: after ? optab->mem_fetch_after
7683	: optab->mem_fetch_before, mode);
7684	create_integer_operand (&ops[3], model);
7685	num_ops = 4;
7686	}
7687	else
7688	{
7689	icode = optab_handler (op: after ? optab->fetch_after
7690	: optab->fetch_before, mode);
7691	num_ops = 3;
7692	}
7693	create_output_operand (op: &ops[op_counter++], x: target, mode);
7694	}
7695	if (icode == CODE_FOR_nothing)
7696	return NULL_RTX;
7697
7698	create_fixed_operand (op: &ops[op_counter++], x: mem);
7699	/* VAL may have been promoted to a wider mode. Shrink it if so. */
7700	create_convert_operand_to (op: &ops[op_counter++], value: val, mode, unsigned_p: true);
7701
7702	if (maybe_expand_insn (icode, nops: num_ops, ops))
7703	return (target == const0_rtx ? const0_rtx : ops[0].value);
7704
7705	return NULL_RTX;
7706	}
7707
7708
7709	/* This function expands an atomic fetch_OP or OP_fetch operation:
7710	TARGET is an option place to stick the return value. const0_rtx indicates
7711	the result is unused.
7712	atomically fetch MEM, perform the operation with VAL and return it to MEM.
7713	CODE is the operation being performed (OP)
7714	MEMMODEL is the memory model variant to use.
7715	AFTER is true to return the result of the operation (OP_fetch).
7716	AFTER is false to return the value before the operation (fetch_OP).
7717
7718	This function will *only* generate instructions if there is a direct
7719	optab. No compare and swap loops or libcalls will be generated. */
7720
7721	static rtx
7722	expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
7723	enum rtx_code code, enum memmodel model,
7724	bool after)
7725	{
7726	machine_mode mode = GET_MODE (mem);
7727	struct atomic_op_functions optab;
7728	rtx result;
7729	bool unused_result = (target == const0_rtx);
7730
7731	get_atomic_op_for_code (op: &optab, code);
7732
7733	/* Check to see if there are any better instructions. */
7734	result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
7735	if (result)
7736	return result;
7737
7738	/* Check for the case where the result isn't used and try those patterns. */
7739	if (unused_result)
7740	{
7741	/* Try the memory model variant first. */
7742	result = maybe_emit_op (optab: &optab, target, mem, val, use_memmodel: true, model, after: true);
7743	if (result)
7744	return result;
7745
7746	/* Next try the old style withuot a memory model. */
7747	result = maybe_emit_op (optab: &optab, target, mem, val, use_memmodel: false, model, after: true);
7748	if (result)
7749	return result;
7750
7751	/* There is no no-result pattern, so try patterns with a result. */
7752	target = NULL_RTX;
7753	}
7754
7755	/* Try the __atomic version. */
7756	result = maybe_emit_op (optab: &optab, target, mem, val, use_memmodel: true, model, after);
7757	if (result)
7758	return result;
7759
7760	/* Try the older __sync version. */
7761	result = maybe_emit_op (optab: &optab, target, mem, val, use_memmodel: false, model, after);
7762	if (result)
7763	return result;
7764
7765	/* If the fetch value can be calculated from the other variation of fetch,
7766	try that operation. */
7767	if (after || unused_result || optab.reverse_code != UNKNOWN)
7768	{
7769	/* Try the __atomic version, then the older __sync version. */
7770	result = maybe_emit_op (optab: &optab, target, mem, val, use_memmodel: true, model, after: !after);
7771	if (!result)
7772	result = maybe_emit_op (optab: &optab, target, mem, val, use_memmodel: false, model, after: !after);
7773
7774	if (result)
7775	{
7776	/* If the result isn't used, no need to do compensation code. */
7777	if (unused_result)
7778	return result;
7779
7780	/* Issue compensation code. Fetch_after == fetch_before OP val.
7781	Fetch_before == after REVERSE_OP val. */
7782	if (!after)
7783	code = optab.reverse_code;
7784	if (code == NOT)
7785	{
7786	result = expand_simple_binop (mode, code: AND, op0: result, op1: val, NULL_RTX,
7787	unsignedp: true, methods: OPTAB_LIB_WIDEN);
7788	result = expand_simple_unop (mode, code: NOT, op0: result, target, unsignedp: true);
7789	}
7790	else
7791	result = expand_simple_binop (mode, code, op0: result, op1: val, target,
7792	unsignedp: true, methods: OPTAB_LIB_WIDEN);
7793	return result;
7794	}
7795	}
7796
7797	/* No direct opcode can be generated. */
7798	return NULL_RTX;
7799	}
7800
7801
7802
7803	/* This function expands an atomic fetch_OP or OP_fetch operation:
7804	TARGET is an option place to stick the return value. const0_rtx indicates
7805	the result is unused.
7806	atomically fetch MEM, perform the operation with VAL and return it to MEM.
7807	CODE is the operation being performed (OP)
7808	MEMMODEL is the memory model variant to use.
7809	AFTER is true to return the result of the operation (OP_fetch).
7810	AFTER is false to return the value before the operation (fetch_OP). */
7811	rtx
7812	expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7813	enum memmodel model, bool after)
7814	{
7815	machine_mode mode = GET_MODE (mem);
7816	rtx result;
7817	bool unused_result = (target == const0_rtx);
7818
7819	/* If loads are not atomic for the required size and we are not called to
7820	provide a __sync builtin, do not do anything so that we stay consistent
7821	with atomic loads of the same size. */
7822	if (!can_atomic_load_p (mode) && !is_mm_sync (model))
7823	return NULL_RTX;
7824
7825	result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
7826	after);
7827
7828	if (result)
7829	return result;
7830
7831	/* Add/sub can be implemented by doing the reverse operation with -(val). */
7832	if (code == PLUS || code == MINUS)
7833	{
7834	rtx tmp;
7835	enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
7836
7837	start_sequence ();
7838	tmp = expand_simple_unop (mode, code: NEG, op0: val, NULL_RTX, unsignedp: true);
7839	result = expand_atomic_fetch_op_no_fallback (target, mem, val: tmp, code: reverse,
7840	model, after);
7841	if (result)
7842	{
7843	/* PLUS worked so emit the insns and return. */
7844	tmp = get_insns ();
7845	end_sequence ();
7846	emit_insn (tmp);
7847	return result;
7848	}
7849
7850	/* PLUS did not work, so throw away the negation code and continue. */
7851	end_sequence ();
7852	}
7853
7854	/* Try the __sync libcalls only if we can't do compare-and-swap inline. */
7855	if (!can_compare_and_swap_p (mode, false))
7856	{
7857	rtx libfunc;
7858	bool fixup = false;
7859	enum rtx_code orig_code = code;
7860	struct atomic_op_functions optab;
7861
7862	get_atomic_op_for_code (op: &optab, code);
7863	libfunc = optab_libfunc (after ? optab.fetch_after
7864	: optab.fetch_before, mode);
7865	if (libfunc == NULL
7866	&& (after || unused_result || optab.reverse_code != UNKNOWN))
7867	{
7868	fixup = true;
7869	if (!after)
7870	code = optab.reverse_code;
7871	libfunc = optab_libfunc (after ? optab.fetch_before
7872	: optab.fetch_after, mode);
7873	}
7874	if (libfunc != NULL)
7875	{
7876	rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7877	result = emit_library_call_value (fun: libfunc, NULL, fn_type: LCT_NORMAL, outmode: mode,
7878	arg1: addr, arg1_mode: ptr_mode, arg2: val, arg2_mode: mode);
7879
7880	if (!unused_result && fixup)
7881	result = expand_simple_binop (mode, code, op0: result, op1: val, target,
7882	unsignedp: true, methods: OPTAB_LIB_WIDEN);
7883	return result;
7884	}
7885
7886	/* We need the original code for any further attempts. */
7887	code = orig_code;
7888	}
7889
7890	/* If nothing else has succeeded, default to a compare and swap loop. */
7891	if (can_compare_and_swap_p (mode, true))
7892	{
7893	rtx_insn *insn;
7894	rtx t0 = gen_reg_rtx (mode), t1;
7895
7896	start_sequence ();
7897
7898	/* If the result is used, get a register for it. */
7899	if (!unused_result)
7900	{
7901	if (!target || !register_operand (target, mode))
7902	target = gen_reg_rtx (mode);
7903	/* If fetch_before, copy the value now. */
7904	if (!after)
7905	emit_move_insn (target, t0);
7906	}
7907	else
7908	target = const0_rtx;
7909
7910	t1 = t0;
7911	if (code == NOT)
7912	{
7913	t1 = expand_simple_binop (mode, code: AND, op0: t1, op1: val, NULL_RTX,
7914	unsignedp: true, methods: OPTAB_LIB_WIDEN);
7915	t1 = expand_simple_unop (mode, code, op0: t1, NULL_RTX, unsignedp: true);
7916	}
7917	else
7918	t1 = expand_simple_binop (mode, code, op0: t1, op1: val, NULL_RTX, unsignedp: true,
7919	methods: OPTAB_LIB_WIDEN);
7920
7921	/* For after, copy the value now. */
7922	if (!unused_result && after)
7923	emit_move_insn (target, t1);
7924	insn = get_insns ();
7925	end_sequence ();
7926
7927	if (t1 != NULL && expand_compare_and_swap_loop (mem, old_reg: t0, new_reg: t1, seq: insn))
7928	return target;
7929	}
7930
7931	return NULL_RTX;
7932	}
7933
7934	/* Return true if OPERAND is suitable for operand number OPNO of
7935	instruction ICODE. */
7936
7937	bool
7938	insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
7939	{
7940	return (!insn_data[(int) icode].operand[opno].predicate
7941	|| (insn_data[(int) icode].operand[opno].predicate
7942	(operand, insn_data[(int) icode].operand[opno].mode)));
7943	}
7944
7945	/* TARGET is a target of a multiword operation that we are going to
7946	implement as a series of word-mode operations. Return true if
7947	TARGET is suitable for this purpose. */
7948
7949	bool
7950	valid_multiword_target_p (rtx target)
7951	{
7952	machine_mode mode;
7953	int i, size;
7954
7955	mode = GET_MODE (target);
7956	if (!GET_MODE_SIZE (mode).is_constant (const_value: &size))
7957	return false;
7958	for (i = 0; i < size; i += UNITS_PER_WORD)
7959	if (!validate_subreg (word_mode, mode, target, i))
7960	return false;
7961	return true;
7962	}
7963
7964	/* Make OP describe an input operand that has value INTVAL and that has
7965	no inherent mode. This function should only be used for operands that
7966	are always expand-time constants. The backend may request that INTVAL
7967	be copied into a different kind of rtx, but it must specify the mode
7968	of that rtx if so. */
7969
7970	void
7971	create_integer_operand (class expand_operand *op, poly_int64 intval)
7972	{
7973	create_expand_operand (op, type: EXPAND_INTEGER,
7974	value: gen_int_mode (intval, MAX_MODE_INT),
7975	VOIDmode, unsigned_p: false, int_value: intval);
7976	}
7977
7978	/* Like maybe_legitimize_operand, but do not change the code of the
7979	current rtx value. */
7980
7981	static bool
7982	maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
7983	class expand_operand *op)
7984	{
7985	/* See if the operand matches in its current form. */
7986	if (insn_operand_matches (icode, opno, operand: op->value))
7987	return true;
7988
7989	/* If the operand is a memory whose address has no side effects,
7990	try forcing the address into a non-virtual pseudo register.
7991	The check for side effects is important because copy_to_mode_reg
7992	cannot handle things like auto-modified addresses. */
7993	if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
7994	{
7995	rtx addr, mem;
7996
7997	mem = op->value;
7998	addr = XEXP (mem, 0);
7999	if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
8000	&& !side_effects_p (addr))
8001	{
8002	rtx_insn *last;
8003	machine_mode mode;
8004
8005	last = get_last_insn ();
8006	mode = get_address_mode (mem);
8007	mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
8008	if (insn_operand_matches (icode, opno, operand: mem))
8009	{
8010	op->value = mem;
8011	return true;
8012	}
8013	delete_insns_since (last);
8014	}
8015	}
8016
8017	return false;
8018	}
8019
8020	/* Try to make OP match operand OPNO of instruction ICODE. Return true
8021	on success, storing the new operand value back in OP. */
8022
8023	static bool
8024	maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
8025	class expand_operand *op)
8026	{
8027	machine_mode mode, imode, tmode;
8028
8029	mode = op->mode;
8030	switch (op->type)
8031	{
8032	case EXPAND_FIXED:
8033	{
8034	temporary_volatile_ok v (true);
8035	return maybe_legitimize_operand_same_code (icode, opno, op);
8036	}
8037
8038	case EXPAND_OUTPUT:
8039	gcc_assert (mode != VOIDmode);
8040	if (op->value
8041	&& op->value != const0_rtx
8042	&& GET_MODE (op->value) == mode
8043	&& maybe_legitimize_operand_same_code (icode, opno, op))
8044	return true;
8045
8046	op->value = gen_reg_rtx (mode);
8047	op->target = 0;
8048	break;
8049
8050	case EXPAND_INPUT:
8051	input:
8052	gcc_assert (mode != VOIDmode);
8053	gcc_assert (GET_MODE (op->value) == VOIDmode
8054	|| GET_MODE (op->value) == mode);
8055	if (maybe_legitimize_operand_same_code (icode, opno, op))
8056	return true;
8057
8058	op->value = copy_to_mode_reg (mode, op->value);
8059	break;
8060
8061	case EXPAND_CONVERT_TO:
8062	gcc_assert (mode != VOIDmode);
8063	op->value = convert_to_mode (mode, op->value, op->unsigned_p);
8064	goto input;
8065
8066	case EXPAND_CONVERT_FROM:
8067	if (GET_MODE (op->value) != VOIDmode)
8068	mode = GET_MODE (op->value);
8069	else
8070	/* The caller must tell us what mode this value has. */
8071	gcc_assert (mode != VOIDmode);
8072
8073	imode = insn_data[(int) icode].operand[opno].mode;
8074	tmode = (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode)
8075	? GET_MODE_INNER (imode) : imode);
8076	if (tmode != VOIDmode && tmode != mode)
8077	{
8078	op->value = convert_modes (mode: tmode, oldmode: mode, x: op->value, unsignedp: op->unsigned_p);
8079	mode = tmode;
8080	}
8081	if (imode != VOIDmode && imode != mode)
8082	{
8083	gcc_assert (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode));
8084	op->value = expand_vector_broadcast (vmode: imode, op: op->value);
8085	mode = imode;
8086	}
8087	goto input;
8088
8089	case EXPAND_ADDRESS:
8090	op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
8091	op->value);
8092	goto input;
8093
8094	case EXPAND_INTEGER:
8095	mode = insn_data[(int) icode].operand[opno].mode;
8096	if (mode != VOIDmode
8097	&& known_eq (trunc_int_for_mode (op->int_value, mode),
8098	op->int_value))
8099	{
8100	op->value = gen_int_mode (op->int_value, mode);
8101	goto input;
8102	}
8103	break;
8104
8105	case EXPAND_UNDEFINED_INPUT:
8106	/* See if the predicate accepts a SCRATCH rtx, which in this context
8107	indicates an undefined value. Use an uninitialized register if not. */
8108	if (!insn_operand_matches (icode, opno, operand: op->value))
8109	{
8110	op->value = gen_reg_rtx (op->mode);
8111	goto input;
8112	}
8113	return true;
8114	}
8115	return insn_operand_matches (icode, opno, operand: op->value);
8116	}
8117
8118	/* Make OP describe an input operand that should have the same value
8119	as VALUE, after any mode conversion that the target might request.
8120	TYPE is the type of VALUE. */
8121
8122	void
8123	create_convert_operand_from_type (class expand_operand *op,
8124	rtx value, tree type)
8125	{
8126	create_convert_operand_from (op, value, TYPE_MODE (type),
8127	TYPE_UNSIGNED (type));
8128	}
8129
8130	/* Return true if the requirements on operands OP1 and OP2 of instruction
8131	ICODE are similar enough for the result of legitimizing OP1 to be
8132	reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
8133	with OP1 and OP2 respectively. */
8134
8135	static inline bool
8136	can_reuse_operands_p (enum insn_code icode,
8137	unsigned int opno1, unsigned int opno2,
8138	const class expand_operand *op1,
8139	const class expand_operand *op2)
8140	{
8141	/* Check requirements that are common to all types. */
8142	if (op1->type != op2->type
8143	|| op1->mode != op2->mode
8144	|| (insn_data[(int) icode].operand[opno1].mode
8145	!= insn_data[(int) icode].operand[opno2].mode))
8146	return false;
8147
8148	/* Check the requirements for specific types. */
8149	switch (op1->type)
8150	{
8151	case EXPAND_OUTPUT:
8152	case EXPAND_UNDEFINED_INPUT:
8153	/* Outputs and undefined intputs must remain distinct. */
8154	return false;
8155
8156	case EXPAND_FIXED:
8157	case EXPAND_INPUT:
8158	case EXPAND_ADDRESS:
8159	case EXPAND_INTEGER:
8160	return true;
8161
8162	case EXPAND_CONVERT_TO:
8163	case EXPAND_CONVERT_FROM:
8164	return op1->unsigned_p == op2->unsigned_p;
8165	}
8166	gcc_unreachable ();
8167	}
8168
8169	/* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8170	of instruction ICODE. Return true on success, leaving the new operand
8171	values in the OPS themselves. Emit no code on failure. */
8172
8173	bool
8174	maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
8175	unsigned int nops, class expand_operand *ops)
8176	{
8177	rtx_insn *last = get_last_insn ();
8178	rtx *orig_values = XALLOCAVEC (rtx, nops);
8179	for (unsigned int i = 0; i < nops; i++)
8180	{
8181	orig_values[i] = ops[i].value;
8182
8183	/* First try reusing the result of an earlier legitimization.
8184	This avoids duplicate rtl and ensures that tied operands
8185	remain tied.
8186
8187	This search is linear, but NOPS is bounded at compile time
8188	to a small number (current a single digit). */
8189	unsigned int j = 0;
8190	for (; j < i; ++j)
8191	if (can_reuse_operands_p (icode, opno1: opno + j, opno2: opno + i, op1: &ops[j], op2: &ops[i])
8192	&& rtx_equal_p (orig_values[j], orig_values[i])
8193	&& ops[j].value
8194	&& insn_operand_matches (icode, opno: opno + i, operand: ops[j].value))
8195	{
8196	ops[i].value = copy_rtx (ops[j].value);
8197	break;
8198	}
8199
8200	/* Otherwise try legitimizing the operand on its own. */
8201	if (j == i && !maybe_legitimize_operand (icode, opno: opno + i, op: &ops[i]))
8202	{
8203	delete_insns_since (last);
8204	return false;
8205	}
8206	}
8207	return true;
8208	}
8209
8210	/* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8211	as its operands. Return the instruction pattern on success,
8212	and emit any necessary set-up code. Return null and emit no
8213	code on failure. */
8214
8215	rtx_insn *
8216	maybe_gen_insn (enum insn_code icode, unsigned int nops,
8217	class expand_operand *ops)
8218	{
8219	gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
8220	if (!maybe_legitimize_operands (icode, opno: 0, nops, ops))
8221	return NULL;
8222
8223	switch (nops)
8224	{
8225	case 0:
8226	return GEN_FCN (icode) ();
8227	case 1:
8228	return GEN_FCN (icode) (ops[0].value);
8229	case 2:
8230	return GEN_FCN (icode) (ops[0].value, ops[1].value);
8231	case 3:
8232	return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
8233	case 4:
8234	return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8235	ops[3].value);
8236	case 5:
8237	return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8238	ops[3].value, ops[4].value);
8239	case 6:
8240	return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8241	ops[3].value, ops[4].value, ops[5].value);
8242	case 7:
8243	return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8244	ops[3].value, ops[4].value, ops[5].value,
8245	ops[6].value);
8246	case 8:
8247	return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8248	ops[3].value, ops[4].value, ops[5].value,
8249	ops[6].value, ops[7].value);
8250	case 9:
8251	return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8252	ops[3].value, ops[4].value, ops[5].value,
8253	ops[6].value, ops[7].value, ops[8].value);
8254	case 10:
8255	return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8256	ops[3].value, ops[4].value, ops[5].value,
8257	ops[6].value, ops[7].value, ops[8].value,
8258	ops[9].value);
8259	case 11:
8260	return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8261	ops[3].value, ops[4].value, ops[5].value,
8262	ops[6].value, ops[7].value, ops[8].value,
8263	ops[9].value, ops[10].value);
8264	}
8265	gcc_unreachable ();
8266	}
8267
8268	/* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8269	as its operands. Return true on success and emit no code on failure. */
8270
8271	bool
8272	maybe_expand_insn (enum insn_code icode, unsigned int nops,
8273	class expand_operand *ops)
8274	{
8275	rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8276	if (pat)
8277	{
8278	emit_insn (pat);
8279	return true;
8280	}
8281	return false;
8282	}
8283
8284	/* Like maybe_expand_insn, but for jumps. */
8285
8286	bool
8287	maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
8288	class expand_operand *ops)
8289	{
8290	rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8291	if (pat)
8292	{
8293	emit_jump_insn (pat);
8294	return true;
8295	}
8296	return false;
8297	}
8298
8299	/* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8300	as its operands. */
8301
8302	void
8303	expand_insn (enum insn_code icode, unsigned int nops,
8304	class expand_operand *ops)
8305	{
8306	if (!maybe_expand_insn (icode, nops, ops))
8307	gcc_unreachable ();
8308	}
8309
8310	/* Like expand_insn, but for jumps. */
8311
8312	void
8313	expand_jump_insn (enum insn_code icode, unsigned int nops,
8314	class expand_operand *ops)
8315	{
8316	if (!maybe_expand_jump_insn (icode, nops, ops))
8317	gcc_unreachable ();
8318	}
8319