1	/* Internal functions.
2	Copyright (C) 2011-2024 Free Software Foundation, Inc.
3
4	This file is part of GCC.
5
6	GCC is free software; you can redistribute it and/or modify it under
7	the terms of the GNU General Public License as published by the Free
8	Software Foundation; either version 3, or (at your option) any later
9	version.
10
11	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12	WARRANTY; without even the implied warranty of MERCHANTABILITY or
13	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14	for more details.
15
16	You should have received a copy of the GNU General Public License
17	along with GCC; see the file COPYING3. If not see
18	<http://www.gnu.org/licenses/>. */
19
20	#include "config.h"
21	#include "system.h"
22	#include "coretypes.h"
23	#include "backend.h"
24	#include "target.h"
25	#include "rtl.h"
26	#include "tree.h"
27	#include "gimple.h"
28	#include "predict.h"
29	#include "stringpool.h"
30	#include "tree-vrp.h"
31	#include "tree-ssanames.h"
32	#include "expmed.h"
33	#include "memmodel.h"
34	#include "optabs.h"
35	#include "emit-rtl.h"
36	#include "diagnostic-core.h"
37	#include "fold-const.h"
38	#include "internal-fn.h"
39	#include "stor-layout.h"
40	#include "dojump.h"
41	#include "expr.h"
42	#include "stringpool.h"
43	#include "attribs.h"
44	#include "asan.h"
45	#include "ubsan.h"
46	#include "recog.h"
47	#include "builtins.h"
48	#include "optabs-tree.h"
49	#include "gimple-ssa.h"
50	#include "tree-phinodes.h"
51	#include "ssa-iterators.h"
52	#include "explow.h"
53	#include "rtl-iter.h"
54	#include "gimple-range.h"
55	#include "fold-const-call.h"
56
57	/* For lang_hooks.types.type_for_mode. */
58	#include "langhooks.h"
59
60	/* The names of each internal function, indexed by function number. */
61	const char *const internal_fn_name_array[] = {
62	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
63	#include "internal-fn.def"
64	"<invalid-fn>"
65	};
66
67	/* The ECF_* flags of each internal function, indexed by function number. */
68	const int internal_fn_flags_array[] = {
69	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
70	#include "internal-fn.def"
71	0
72	};
73
74	/* Return the internal function called NAME, or IFN_LAST if there's
75	no such function. */
76
77	internal_fn
78	lookup_internal_fn (const char *name)
79	{
80	typedef hash_map<nofree_string_hash, internal_fn> name_to_fn_map_type;
81	static name_to_fn_map_type *name_to_fn_map;
82
83	if (!name_to_fn_map)
84	{
85	name_to_fn_map = new name_to_fn_map_type (IFN_LAST);
86	for (unsigned int i = 0; i < IFN_LAST; ++i)
87	name_to_fn_map->put (k: internal_fn_name (fn: internal_fn (i)),
88	v: internal_fn (i));
89	}
90	internal_fn *entry = name_to_fn_map->get (k: name);
91	return entry ? *entry : IFN_LAST;
92	}
93
94	/* Geven an internal_fn IFN that is a widening function, return its
95	corresponding LO and HI internal_fns. */
96
97	extern void
98	lookup_hilo_internal_fn (internal_fn ifn, internal_fn *lo, internal_fn *hi)
99	{
100	gcc_assert (widening_fn_p (ifn));
101
102	switch (ifn)
103	{
104	default:
105	gcc_unreachable ();
106	#define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
107	#define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
108	case IFN_##NAME: \
109	*lo = internal_fn (IFN_##NAME##_LO); \
110	*hi = internal_fn (IFN_##NAME##_HI); \
111	break;
112	#include "internal-fn.def"
113	}
114	}
115
116	/* Given an internal_fn IFN that is a widening function, return its
117	corresponding _EVEN and _ODD internal_fns in *EVEN and *ODD. */
118
119	extern void
120	lookup_evenodd_internal_fn (internal_fn ifn, internal_fn *even,
121	internal_fn *odd)
122	{
123	gcc_assert (widening_fn_p (ifn));
124
125	switch (ifn)
126	{
127	default:
128	gcc_unreachable ();
129	#define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
130	#define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
131	case IFN_##NAME: \
132	*even = internal_fn (IFN_##NAME##_EVEN); \
133	*odd = internal_fn (IFN_##NAME##_ODD); \
134	break;
135	#include "internal-fn.def"
136	}
137	}
138
139
140	/* Fnspec of each internal function, indexed by function number. */
141	const_tree internal_fn_fnspec_array[IFN_LAST + 1];
142
143	void
144	init_internal_fns ()
145	{
146	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
147	if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
148	build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
149	#include "internal-fn.def"
150	internal_fn_fnspec_array[IFN_LAST] = 0;
151	}
152
153	/* Create static initializers for the information returned by
154	direct_internal_fn. */
155	#define not_direct { -2, -2, false }
156	#define mask_load_direct { -1, 2, false }
157	#define load_lanes_direct { -1, -1, false }
158	#define mask_load_lanes_direct { -1, -1, false }
159	#define gather_load_direct { 3, 1, false }
160	#define len_load_direct { -1, -1, false }
161	#define mask_len_load_direct { -1, 4, false }
162	#define mask_store_direct { 3, 2, false }
163	#define store_lanes_direct { 0, 0, false }
164	#define mask_store_lanes_direct { 0, 0, false }
165	#define vec_cond_mask_direct { 1, 0, false }
166	#define vec_cond_mask_len_direct { 1, 1, false }
167	#define vec_cond_direct { 2, 0, false }
168	#define scatter_store_direct { 3, 1, false }
169	#define len_store_direct { 3, 3, false }
170	#define mask_len_store_direct { 4, 5, false }
171	#define vec_set_direct { 3, 3, false }
172	#define vec_extract_direct { 0, -1, false }
173	#define unary_direct { 0, 0, true }
174	#define unary_convert_direct { -1, 0, true }
175	#define binary_direct { 0, 0, true }
176	#define ternary_direct { 0, 0, true }
177	#define cond_unary_direct { 1, 1, true }
178	#define cond_binary_direct { 1, 1, true }
179	#define cond_ternary_direct { 1, 1, true }
180	#define cond_len_unary_direct { 1, 1, true }
181	#define cond_len_binary_direct { 1, 1, true }
182	#define cond_len_ternary_direct { 1, 1, true }
183	#define while_direct { 0, 2, false }
184	#define fold_extract_direct { 2, 2, false }
185	#define fold_len_extract_direct { 2, 2, false }
186	#define fold_left_direct { 1, 1, false }
187	#define mask_fold_left_direct { 1, 1, false }
188	#define mask_len_fold_left_direct { 1, 1, false }
189	#define check_ptrs_direct { 0, 0, false }
190
191	const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
192	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
193	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
194	#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
195	UNSIGNED_OPTAB, TYPE) TYPE##_direct,
196	#include "internal-fn.def"
197	not_direct
198	};
199
200	/* Expand STMT using instruction ICODE. The instruction has NOUTPUTS
201	output operands and NINPUTS input operands, where NOUTPUTS is either
202	0 or 1. The output operand (if any) comes first, followed by the
203	NINPUTS input operands. */
204
205	static void
206	expand_fn_using_insn (gcall *stmt, insn_code icode, unsigned int noutputs,
207	unsigned int ninputs)
208	{
209	gcc_assert (icode != CODE_FOR_nothing);
210
211	expand_operand *ops = XALLOCAVEC (expand_operand, noutputs + ninputs);
212	unsigned int opno = 0;
213	rtx lhs_rtx = NULL_RTX;
214	tree lhs = gimple_call_lhs (gs: stmt);
215
216	if (noutputs)
217	{
218	gcc_assert (noutputs == 1);
219	if (lhs)
220	lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
221
222	/* Do not assign directly to a promoted subreg, since there is no
223	guarantee that the instruction will leave the upper bits of the
224	register in the state required by SUBREG_PROMOTED_SIGN. */
225	rtx dest = lhs_rtx;
226	if (dest && GET_CODE (dest) == SUBREG && SUBREG_PROMOTED_VAR_P (dest))
227	dest = NULL_RTX;
228	create_output_operand (op: &ops[opno], x: dest,
229	mode: insn_data[icode].operand[opno].mode);
230	opno += 1;
231	}
232	else
233	gcc_assert (!lhs);
234
235	for (unsigned int i = 0; i < ninputs; ++i)
236	{
237	tree rhs = gimple_call_arg (gs: stmt, index: i);
238	tree rhs_type = TREE_TYPE (rhs);
239	rtx rhs_rtx = expand_normal (exp: rhs);
240	if (INTEGRAL_TYPE_P (rhs_type))
241	create_convert_operand_from (op: &ops[opno], value: rhs_rtx,
242	TYPE_MODE (rhs_type),
243	TYPE_UNSIGNED (rhs_type));
244	else if (TREE_CODE (rhs) == SSA_NAME
245	&& SSA_NAME_IS_DEFAULT_DEF (rhs)
246	&& VAR_P (SSA_NAME_VAR (rhs)))
247	create_undefined_input_operand (op: &ops[opno], TYPE_MODE (rhs_type));
248	else if (VECTOR_BOOLEAN_TYPE_P (rhs_type)
249	&& SCALAR_INT_MODE_P (TYPE_MODE (rhs_type))
250	&& maybe_ne (a: GET_MODE_PRECISION (TYPE_MODE (rhs_type)),
251	b: TYPE_VECTOR_SUBPARTS (node: rhs_type).to_constant ()))
252	{
253	/* Ensure that the vector bitmasks do not have excess bits. */
254	int nunits = TYPE_VECTOR_SUBPARTS (node: rhs_type).to_constant ();
255	rtx tmp = expand_binop (TYPE_MODE (rhs_type), and_optab, rhs_rtx,
256	GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
257	NULL_RTX, true, OPTAB_WIDEN);
258	create_input_operand (op: &ops[opno], value: tmp, TYPE_MODE (rhs_type));
259	}
260	else
261	create_input_operand (op: &ops[opno], value: rhs_rtx, TYPE_MODE (rhs_type));
262	opno += 1;
263	}
264
265	gcc_assert (opno == noutputs + ninputs);
266	expand_insn (icode, nops: opno, ops);
267	if (lhs_rtx && !rtx_equal_p (lhs_rtx, ops[0].value))
268	{
269	/* If the return value has an integral type, convert the instruction
270	result to that type. This is useful for things that return an
271	int regardless of the size of the input. If the instruction result
272	is smaller than required, assume that it is signed.
273
274	If the return value has a nonintegral type, its mode must match
275	the instruction result. */
276	if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
277	{
278	/* If this is a scalar in a register that is stored in a wider
279	mode than the declared mode, compute the result into its
280	declared mode and then convert to the wider mode. */
281	gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
282	rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), ops[0].value, 0);
283	convert_move (SUBREG_REG (lhs_rtx), tmp,
284	SUBREG_PROMOTED_SIGN (lhs_rtx));
285	}
286	else if (GET_MODE (lhs_rtx) == GET_MODE (ops[0].value))
287	emit_move_insn (lhs_rtx, ops[0].value);
288	else
289	{
290	gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
291	convert_move (lhs_rtx, ops[0].value, 0);
292	}
293	}
294	}
295
296	/* ARRAY_TYPE is an array of vector modes. Return the associated insn
297	for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
298
299	static enum insn_code
300	get_multi_vector_move (tree array_type, convert_optab optab)
301	{
302	machine_mode imode;
303	machine_mode vmode;
304
305	gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
306	imode = TYPE_MODE (array_type);
307	vmode = TYPE_MODE (TREE_TYPE (array_type));
308
309	return convert_optab_handler (op: optab, to_mode: imode, from_mode: vmode);
310	}
311
312	/* Add mask and len arguments according to the STMT. */
313
314	static unsigned int
315	add_mask_and_len_args (expand_operand *ops, unsigned int opno, gcall *stmt)
316	{
317	internal_fn ifn = gimple_call_internal_fn (gs: stmt);
318	int len_index = internal_fn_len_index (ifn);
319	/* BIAS is always consecutive next of LEN. */
320	int bias_index = len_index + 1;
321	int mask_index = internal_fn_mask_index (ifn);
322	/* The order of arguments are always {len,bias,mask}. */
323	if (mask_index >= 0)
324	{
325	tree mask = gimple_call_arg (gs: stmt, index: mask_index);
326	rtx mask_rtx = expand_normal (exp: mask);
327
328	tree mask_type = TREE_TYPE (mask);
329	if (VECTOR_BOOLEAN_TYPE_P (mask_type)
330	&& SCALAR_INT_MODE_P (TYPE_MODE (mask_type))
331	&& maybe_ne (a: GET_MODE_PRECISION (TYPE_MODE (mask_type)),
332	b: TYPE_VECTOR_SUBPARTS (node: mask_type).to_constant ()))
333	{
334	/* Ensure that the vector bitmasks do not have excess bits. */
335	int nunits = TYPE_VECTOR_SUBPARTS (node: mask_type).to_constant ();
336	mask_rtx = expand_binop (TYPE_MODE (mask_type), and_optab, mask_rtx,
337	GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
338	NULL_RTX, true, OPTAB_WIDEN);
339	}
340
341	create_input_operand (op: &ops[opno++], value: mask_rtx,
342	TYPE_MODE (TREE_TYPE (mask)));
343	}
344	if (len_index >= 0)
345	{
346	tree len = gimple_call_arg (gs: stmt, index: len_index);
347	rtx len_rtx = expand_normal (exp: len);
348	create_convert_operand_from (op: &ops[opno++], value: len_rtx,
349	TYPE_MODE (TREE_TYPE (len)),
350	TYPE_UNSIGNED (TREE_TYPE (len)));
351	tree biast = gimple_call_arg (gs: stmt, index: bias_index);
352	rtx bias = expand_normal (exp: biast);
353	create_input_operand (op: &ops[opno++], value: bias, QImode);
354	}
355	return opno;
356	}
357
358	/* Expand LOAD_LANES call STMT using optab OPTAB. */
359
360	static void
361	expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
362	{
363	class expand_operand ops[2];
364	tree type, lhs, rhs;
365	rtx target, mem;
366
367	lhs = gimple_call_lhs (gs: stmt);
368	rhs = gimple_call_arg (gs: stmt, index: 0);
369	type = TREE_TYPE (lhs);
370
371	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
372	mem = expand_normal (exp: rhs);
373
374	gcc_assert (MEM_P (mem));
375	PUT_MODE (x: mem, TYPE_MODE (type));
376
377	create_output_operand (op: &ops[0], x: target, TYPE_MODE (type));
378	create_fixed_operand (op: &ops[1], x: mem);
379	expand_insn (icode: get_multi_vector_move (array_type: type, optab), nops: 2, ops);
380	if (!rtx_equal_p (target, ops[0].value))
381	emit_move_insn (target, ops[0].value);
382	}
383
384	/* Expand STORE_LANES call STMT using optab OPTAB. */
385
386	static void
387	expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
388	{
389	class expand_operand ops[2];
390	tree type, lhs, rhs;
391	rtx target, reg;
392
393	lhs = gimple_call_lhs (gs: stmt);
394	rhs = gimple_call_arg (gs: stmt, index: 0);
395	type = TREE_TYPE (rhs);
396
397	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
398	reg = expand_normal (exp: rhs);
399
400	gcc_assert (MEM_P (target));
401	PUT_MODE (x: target, TYPE_MODE (type));
402
403	create_fixed_operand (op: &ops[0], x: target);
404	create_input_operand (op: &ops[1], value: reg, TYPE_MODE (type));
405	expand_insn (icode: get_multi_vector_move (array_type: type, optab), nops: 2, ops);
406	}
407
408	static void
409	expand_ANNOTATE (internal_fn, gcall *)
410	{
411	gcc_unreachable ();
412	}
413
414	/* This should get expanded in omp_device_lower pass. */
415
416	static void
417	expand_GOMP_USE_SIMT (internal_fn, gcall *)
418	{
419	gcc_unreachable ();
420	}
421
422	/* This should get expanded in omp_device_lower pass. */
423
424	static void
425	expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
426	{
427	gcc_unreachable ();
428	}
429
430	/* Allocate per-lane storage and begin non-uniform execution region. */
431
432	static void
433	expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
434	{
435	rtx target;
436	tree lhs = gimple_call_lhs (gs: stmt);
437	if (lhs)
438	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
439	else
440	target = gen_reg_rtx (Pmode);
441	rtx size = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
442	rtx align = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
443	class expand_operand ops[3];
444	create_output_operand (op: &ops[0], x: target, Pmode);
445	create_input_operand (op: &ops[1], value: size, Pmode);
446	create_input_operand (op: &ops[2], value: align, Pmode);
447	gcc_assert (targetm.have_omp_simt_enter ());
448	expand_insn (icode: targetm.code_for_omp_simt_enter, nops: 3, ops);
449	if (!rtx_equal_p (target, ops[0].value))
450	emit_move_insn (target, ops[0].value);
451	}
452
453	/* Deallocate per-lane storage and leave non-uniform execution region. */
454
455	static void
456	expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
457	{
458	gcc_checking_assert (!gimple_call_lhs (stmt));
459	rtx arg = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
460	class expand_operand ops[1];
461	create_input_operand (op: &ops[0], value: arg, Pmode);
462	gcc_assert (targetm.have_omp_simt_exit ());
463	expand_insn (icode: targetm.code_for_omp_simt_exit, nops: 1, ops);
464	}
465
466	/* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
467	without SIMT execution this should be expanded in omp_device_lower pass. */
468
469	static void
470	expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
471	{
472	tree lhs = gimple_call_lhs (gs: stmt);
473	if (!lhs)
474	return;
475
476	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
477	gcc_assert (targetm.have_omp_simt_lane ());
478	emit_insn (targetm.gen_omp_simt_lane (target));
479	}
480
481	/* This should get expanded in omp_device_lower pass. */
482
483	static void
484	expand_GOMP_SIMT_VF (internal_fn, gcall *)
485	{
486	gcc_unreachable ();
487	}
488
489	/* This should get expanded in omp_device_lower pass. */
490
491	static void
492	expand_GOMP_TARGET_REV (internal_fn, gcall *)
493	{
494	gcc_unreachable ();
495	}
496
497	/* Lane index of the first SIMT lane that supplies a non-zero argument.
498	This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
499	lane that executed the last iteration for handling OpenMP lastprivate. */
500
501	static void
502	expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
503	{
504	tree lhs = gimple_call_lhs (gs: stmt);
505	if (!lhs)
506	return;
507
508	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
509	rtx cond = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
510	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
511	class expand_operand ops[2];
512	create_output_operand (op: &ops[0], x: target, mode);
513	create_input_operand (op: &ops[1], value: cond, mode);
514	gcc_assert (targetm.have_omp_simt_last_lane ());
515	expand_insn (icode: targetm.code_for_omp_simt_last_lane, nops: 2, ops);
516	if (!rtx_equal_p (target, ops[0].value))
517	emit_move_insn (target, ops[0].value);
518	}
519
520	/* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
521
522	static void
523	expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
524	{
525	tree lhs = gimple_call_lhs (gs: stmt);
526	if (!lhs)
527	return;
528
529	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
530	rtx ctr = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
531	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
532	class expand_operand ops[2];
533	create_output_operand (op: &ops[0], x: target, mode);
534	create_input_operand (op: &ops[1], value: ctr, mode);
535	gcc_assert (targetm.have_omp_simt_ordered ());
536	expand_insn (icode: targetm.code_for_omp_simt_ordered, nops: 2, ops);
537	if (!rtx_equal_p (target, ops[0].value))
538	emit_move_insn (target, ops[0].value);
539	}
540
541	/* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
542	any lane supplies a non-zero argument. */
543
544	static void
545	expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
546	{
547	tree lhs = gimple_call_lhs (gs: stmt);
548	if (!lhs)
549	return;
550
551	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
552	rtx cond = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
553	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
554	class expand_operand ops[2];
555	create_output_operand (op: &ops[0], x: target, mode);
556	create_input_operand (op: &ops[1], value: cond, mode);
557	gcc_assert (targetm.have_omp_simt_vote_any ());
558	expand_insn (icode: targetm.code_for_omp_simt_vote_any, nops: 2, ops);
559	if (!rtx_equal_p (target, ops[0].value))
560	emit_move_insn (target, ops[0].value);
561	}
562
563	/* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
564	is destination lane index XOR given offset. */
565
566	static void
567	expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
568	{
569	tree lhs = gimple_call_lhs (gs: stmt);
570	if (!lhs)
571	return;
572
573	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
574	rtx src = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
575	rtx idx = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
576	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
577	class expand_operand ops[3];
578	create_output_operand (op: &ops[0], x: target, mode);
579	create_input_operand (op: &ops[1], value: src, mode);
580	create_input_operand (op: &ops[2], value: idx, SImode);
581	gcc_assert (targetm.have_omp_simt_xchg_bfly ());
582	expand_insn (icode: targetm.code_for_omp_simt_xchg_bfly, nops: 3, ops);
583	if (!rtx_equal_p (target, ops[0].value))
584	emit_move_insn (target, ops[0].value);
585	}
586
587	/* Exchange between SIMT lanes according to given source lane index. */
588
589	static void
590	expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
591	{
592	tree lhs = gimple_call_lhs (gs: stmt);
593	if (!lhs)
594	return;
595
596	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
597	rtx src = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
598	rtx idx = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
599	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
600	class expand_operand ops[3];
601	create_output_operand (op: &ops[0], x: target, mode);
602	create_input_operand (op: &ops[1], value: src, mode);
603	create_input_operand (op: &ops[2], value: idx, SImode);
604	gcc_assert (targetm.have_omp_simt_xchg_idx ());
605	expand_insn (icode: targetm.code_for_omp_simt_xchg_idx, nops: 3, ops);
606	if (!rtx_equal_p (target, ops[0].value))
607	emit_move_insn (target, ops[0].value);
608	}
609
610	/* This should get expanded in adjust_simduid_builtins. */
611
612	static void
613	expand_GOMP_SIMD_LANE (internal_fn, gcall *)
614	{
615	gcc_unreachable ();
616	}
617
618	/* This should get expanded in adjust_simduid_builtins. */
619
620	static void
621	expand_GOMP_SIMD_VF (internal_fn, gcall *)
622	{
623	gcc_unreachable ();
624	}
625
626	/* This should get expanded in adjust_simduid_builtins. */
627
628	static void
629	expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
630	{
631	gcc_unreachable ();
632	}
633
634	/* This should get expanded in adjust_simduid_builtins. */
635
636	static void
637	expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
638	{
639	gcc_unreachable ();
640	}
641
642	/* This should get expanded in adjust_simduid_builtins. */
643
644	static void
645	expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
646	{
647	gcc_unreachable ();
648	}
649
650	/* This should get expanded in the sanopt pass. */
651
652	static void
653	expand_UBSAN_NULL (internal_fn, gcall *)
654	{
655	gcc_unreachable ();
656	}
657
658	/* This should get expanded in the sanopt pass. */
659
660	static void
661	expand_UBSAN_BOUNDS (internal_fn, gcall *)
662	{
663	gcc_unreachable ();
664	}
665
666	/* This should get expanded in the sanopt pass. */
667
668	static void
669	expand_UBSAN_VPTR (internal_fn, gcall *)
670	{
671	gcc_unreachable ();
672	}
673
674	/* This should get expanded in the sanopt pass. */
675
676	static void
677	expand_UBSAN_PTR (internal_fn, gcall *)
678	{
679	gcc_unreachable ();
680	}
681
682	/* This should get expanded in the sanopt pass. */
683
684	static void
685	expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
686	{
687	gcc_unreachable ();
688	}
689
690	/* This should get expanded in the sanopt pass. */
691
692	static void
693	expand_HWASAN_CHECK (internal_fn, gcall *)
694	{
695	gcc_unreachable ();
696	}
697
698	/* For hwasan stack tagging:
699	Clear tags on the dynamically allocated space.
700	For use after an object dynamically allocated on the stack goes out of
701	scope. */
702	static void
703	expand_HWASAN_ALLOCA_UNPOISON (internal_fn, gcall *gc)
704	{
705	gcc_assert (Pmode == ptr_mode);
706	tree restored_position = gimple_call_arg (gs: gc, index: 0);
707	rtx restored_rtx = expand_expr (exp: restored_position, NULL_RTX, VOIDmode,
708	modifier: EXPAND_NORMAL);
709	rtx func = init_one_libfunc ("__hwasan_tag_memory");
710	rtx off = expand_simple_binop (Pmode, MINUS, restored_rtx,
711	stack_pointer_rtx, NULL_RTX, 0,
712	OPTAB_WIDEN);
713	emit_library_call_value (fun: func, NULL_RTX, fn_type: LCT_NORMAL, VOIDmode,
714	virtual_stack_dynamic_rtx, Pmode,
715	HWASAN_STACK_BACKGROUND, QImode,
716	arg3: off, Pmode);
717	}
718
719	/* For hwasan stack tagging:
720	Return a tag to be used for a dynamic allocation. */
721	static void
722	expand_HWASAN_CHOOSE_TAG (internal_fn, gcall *gc)
723	{
724	tree tag = gimple_call_lhs (gs: gc);
725	rtx target = expand_expr (exp: tag, NULL_RTX, VOIDmode, modifier: EXPAND_NORMAL);
726	machine_mode mode = GET_MODE (target);
727	gcc_assert (mode == QImode);
728
729	rtx base_tag = targetm.memtag.extract_tag (hwasan_frame_base (), NULL_RTX);
730	gcc_assert (base_tag);
731	rtx tag_offset = gen_int_mode (hwasan_current_frame_tag (), QImode);
732	rtx chosen_tag = expand_simple_binop (QImode, PLUS, base_tag, tag_offset,
733	target, /* unsignedp = */1,
734	OPTAB_WIDEN);
735	chosen_tag = hwasan_truncate_to_tag_size (chosen_tag, target);
736
737	/* Really need to put the tag into the `target` RTX. */
738	if (chosen_tag != target)
739	{
740	rtx temp = chosen_tag;
741	gcc_assert (GET_MODE (chosen_tag) == mode);
742	emit_move_insn (target, temp);
743	}
744
745	hwasan_increment_frame_tag ();
746	}
747
748	/* For hwasan stack tagging:
749	Tag a region of space in the shadow stack according to the base pointer of
750	an object on the stack. N.b. the length provided in the internal call is
751	required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
752	static void
753	expand_HWASAN_MARK (internal_fn, gcall *gc)
754	{
755	gcc_assert (ptr_mode == Pmode);
756	HOST_WIDE_INT flag = tree_to_shwi (gimple_call_arg (gs: gc, index: 0));
757	bool is_poison = ((asan_mark_flags)flag) == ASAN_MARK_POISON;
758
759	tree base = gimple_call_arg (gs: gc, index: 1);
760	gcc_checking_assert (TREE_CODE (base) == ADDR_EXPR);
761	rtx base_rtx = expand_normal (exp: base);
762
763	rtx tag = is_poison ? HWASAN_STACK_BACKGROUND
764	: targetm.memtag.extract_tag (base_rtx, NULL_RTX);
765	rtx address = targetm.memtag.untagged_pointer (base_rtx, NULL_RTX);
766
767	tree len = gimple_call_arg (gs: gc, index: 2);
768	rtx r_len = expand_normal (exp: len);
769
770	rtx func = init_one_libfunc ("__hwasan_tag_memory");
771	emit_library_call (fun: func, fn_type: LCT_NORMAL, VOIDmode, arg1: address, Pmode,
772	arg2: tag, QImode, arg3: r_len, Pmode);
773	}
774
775	/* For hwasan stack tagging:
776	Store a tag into a pointer. */
777	static void
778	expand_HWASAN_SET_TAG (internal_fn, gcall *gc)
779	{
780	gcc_assert (ptr_mode == Pmode);
781	tree g_target = gimple_call_lhs (gs: gc);
782	tree g_ptr = gimple_call_arg (gs: gc, index: 0);
783	tree g_tag = gimple_call_arg (gs: gc, index: 1);
784
785	rtx ptr = expand_normal (exp: g_ptr);
786	rtx tag = expand_expr (exp: g_tag, NULL_RTX, QImode, modifier: EXPAND_NORMAL);
787	rtx target = expand_normal (exp: g_target);
788
789	rtx untagged = targetm.memtag.untagged_pointer (ptr, target);
790	rtx tagged_value = targetm.memtag.set_tag (untagged, tag, target);
791	if (tagged_value != target)
792	emit_move_insn (target, tagged_value);
793	}
794
795	/* This should get expanded in the sanopt pass. */
796
797	static void
798	expand_ASAN_CHECK (internal_fn, gcall *)
799	{
800	gcc_unreachable ();
801	}
802
803	/* This should get expanded in the sanopt pass. */
804
805	static void
806	expand_ASAN_MARK (internal_fn, gcall *)
807	{
808	gcc_unreachable ();
809	}
810
811	/* This should get expanded in the sanopt pass. */
812
813	static void
814	expand_ASAN_POISON (internal_fn, gcall *)
815	{
816	gcc_unreachable ();
817	}
818
819	/* This should get expanded in the sanopt pass. */
820
821	static void
822	expand_ASAN_POISON_USE (internal_fn, gcall *)
823	{
824	gcc_unreachable ();
825	}
826
827	/* This should get expanded in the tsan pass. */
828
829	static void
830	expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
831	{
832	gcc_unreachable ();
833	}
834
835	/* This should get expanded in the lower pass. */
836
837	static void
838	expand_FALLTHROUGH (internal_fn, gcall *call)
839	{
840	error_at (gimple_location (g: call),
841	"invalid use of attribute %<fallthrough%>");
842	}
843
844	/* Return minimum precision needed to represent all values
845	of ARG in SIGNed integral type. */
846
847	static int
848	get_min_precision (tree arg, signop sign)
849	{
850	int prec = TYPE_PRECISION (TREE_TYPE (arg));
851	int cnt = 0;
852	signop orig_sign = sign;
853	if (TREE_CODE (arg) == INTEGER_CST)
854	{
855	int p;
856	if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
857	{
858	widest_int w = wi::to_widest (t: arg);
859	w = wi::ext (x: w, offset: prec, sgn: sign);
860	p = wi::min_precision (x: w, sgn: sign);
861	}
862	else
863	p = wi::min_precision (x: wi::to_wide (t: arg), sgn: sign);
864	return MIN (p, prec);
865	}
866	while (CONVERT_EXPR_P (arg)
867	&& INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
868	&& TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
869	{
870	arg = TREE_OPERAND (arg, 0);
871	if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
872	{
873	if (TYPE_UNSIGNED (TREE_TYPE (arg)))
874	sign = UNSIGNED;
875	else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
876	return prec + (orig_sign != sign);
877	prec = TYPE_PRECISION (TREE_TYPE (arg));
878	}
879	if (++cnt > 30)
880	return prec + (orig_sign != sign);
881	}
882	if (CONVERT_EXPR_P (arg)
883	&& INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
884	&& TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) > prec)
885	{
886	/* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
887	If y_2's min precision is smaller than prec, return that. */
888	int oprec = get_min_precision (TREE_OPERAND (arg, 0), sign);
889	if (oprec < prec)
890	return oprec + (orig_sign != sign);
891	}
892	if (TREE_CODE (arg) != SSA_NAME)
893	return prec + (orig_sign != sign);
894	value_range r;
895	while (!get_global_range_query ()->range_of_expr (r, expr: arg)
896	|| r.varying_p ()
897	|| r.undefined_p ())
898	{
899	gimple *g = SSA_NAME_DEF_STMT (arg);
900	if (is_gimple_assign (gs: g)
901	&& CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
902	{
903	tree t = gimple_assign_rhs1 (gs: g);
904	if (INTEGRAL_TYPE_P (TREE_TYPE (t))
905	&& TYPE_PRECISION (TREE_TYPE (t)) <= prec)
906	{
907	arg = t;
908	if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
909	{
910	if (TYPE_UNSIGNED (TREE_TYPE (arg)))
911	sign = UNSIGNED;
912	else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
913	return prec + (orig_sign != sign);
914	prec = TYPE_PRECISION (TREE_TYPE (arg));
915	}
916	if (++cnt > 30)
917	return prec + (orig_sign != sign);
918	continue;
919	}
920	}
921	return prec + (orig_sign != sign);
922	}
923	if (sign == TYPE_SIGN (TREE_TYPE (arg)))
924	{
925	int p1 = wi::min_precision (x: r.lower_bound (), sgn: sign);
926	int p2 = wi::min_precision (x: r.upper_bound (), sgn: sign);
927	p1 = MAX (p1, p2);
928	prec = MIN (prec, p1);
929	}
930	else if (sign == UNSIGNED && !wi::neg_p (x: r.lower_bound (), sgn: SIGNED))
931	{
932	int p = wi::min_precision (x: r.upper_bound (), sgn: UNSIGNED);
933	prec = MIN (prec, p);
934	}
935	return prec + (orig_sign != sign);
936	}
937
938	/* Helper for expand_*_overflow. Set the __imag__ part to true
939	(1 except for signed:1 type, in which case store -1). */
940
941	static void
942	expand_arith_set_overflow (tree lhs, rtx target)
943	{
944	if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
945	&& !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
946	write_complex_part (target, constm1_rtx, true, false);
947	else
948	write_complex_part (target, const1_rtx, true, false);
949	}
950
951	/* Helper for expand_*_overflow. Store RES into the __real__ part
952	of TARGET. If RES has larger MODE than __real__ part of TARGET,
953	set the __imag__ part to 1 if RES doesn't fit into it. Similarly
954	if LHS has smaller precision than its mode. */
955
956	static void
957	expand_arith_overflow_result_store (tree lhs, rtx target,
958	scalar_int_mode mode, rtx res)
959	{
960	scalar_int_mode tgtmode
961	= as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
962	rtx lres = res;
963	if (tgtmode != mode)
964	{
965	rtx_code_label *done_label = gen_label_rtx ();
966	int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
967	lres = convert_modes (mode: tgtmode, oldmode: mode, x: res, unsignedp: uns);
968	gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
969	do_compare_rtx_and_jump (res, convert_modes (mode, oldmode: tgtmode, x: lres, unsignedp: uns),
970	EQ, true, mode, NULL_RTX, NULL, done_label,
971	profile_probability::very_likely ());
972	expand_arith_set_overflow (lhs, target);
973	emit_label (done_label);
974	}
975	int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
976	int tgtprec = GET_MODE_PRECISION (mode: tgtmode);
977	if (prec < tgtprec)
978	{
979	rtx_code_label *done_label = gen_label_rtx ();
980	int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
981	res = lres;
982	if (uns)
983	{
984	rtx mask
985	= immed_wide_int_const (wi::shifted_mask (start: 0, width: prec, negate_p: false, precision: tgtprec),
986	tgtmode);
987	lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
988	true, OPTAB_LIB_WIDEN);
989	}
990	else
991	{
992	lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
993	NULL_RTX, 1);
994	lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
995	NULL_RTX, 0);
996	}
997	do_compare_rtx_and_jump (res, lres,
998	EQ, true, tgtmode, NULL_RTX, NULL, done_label,
999	profile_probability::very_likely ());
1000	expand_arith_set_overflow (lhs, target);
1001	emit_label (done_label);
1002	}
1003	write_complex_part (target, lres, false, false);
1004	}
1005
1006	/* Helper for expand_*_overflow. Store RES into TARGET. */
1007
1008	static void
1009	expand_ubsan_result_store (tree lhs, rtx target, scalar_int_mode mode,
1010	rtx res, rtx_code_label *do_error)
1011	{
1012	if (TREE_CODE (TREE_TYPE (lhs)) == BITINT_TYPE
1013	&& TYPE_PRECISION (TREE_TYPE (lhs)) < GET_MODE_PRECISION (mode))
1014	{
1015	int uns = TYPE_UNSIGNED (TREE_TYPE (lhs));
1016	int prec = TYPE_PRECISION (TREE_TYPE (lhs));
1017	int tgtprec = GET_MODE_PRECISION (mode);
1018	rtx resc = gen_reg_rtx (mode), lres;
1019	emit_move_insn (resc, res);
1020	if (uns)
1021	{
1022	rtx mask
1023	= immed_wide_int_const (wi::shifted_mask (start: 0, width: prec, negate_p: false, precision: tgtprec),
1024	mode);
1025	lres = expand_simple_binop (mode, AND, res, mask, NULL_RTX,
1026	true, OPTAB_LIB_WIDEN);
1027	}
1028	else
1029	{
1030	lres = expand_shift (LSHIFT_EXPR, mode, res, tgtprec - prec,
1031	NULL_RTX, 1);
1032	lres = expand_shift (RSHIFT_EXPR, mode, lres, tgtprec - prec,
1033	NULL_RTX, 0);
1034	}
1035	if (lres != res)
1036	emit_move_insn (res, lres);
1037	do_compare_rtx_and_jump (res, resc,
1038	NE, true, mode, NULL_RTX, NULL, do_error,
1039	profile_probability::very_unlikely ());
1040	}
1041	if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
1042	/* If this is a scalar in a register that is stored in a wider mode
1043	than the declared mode, compute the result into its declared mode
1044	and then convert to the wider mode. Our value is the computed
1045	expression. */
1046	convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
1047	else
1048	emit_move_insn (target, res);
1049	}
1050
1051	/* Add sub/add overflow checking to the statement STMT.
1052	CODE says whether the operation is +, or -. */
1053
1054	void
1055	expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
1056	tree arg0, tree arg1, bool unsr_p, bool uns0_p,
1057	bool uns1_p, bool is_ubsan, tree *datap)
1058	{
1059	rtx res, target = NULL_RTX;
1060	tree fn;
1061	rtx_code_label *done_label = gen_label_rtx ();
1062	rtx_code_label *do_error = gen_label_rtx ();
1063	do_pending_stack_adjust ();
1064	rtx op0 = expand_normal (exp: arg0);
1065	rtx op1 = expand_normal (exp: arg1);
1066	scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
1067	int prec = GET_MODE_PRECISION (mode);
1068	rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1069	bool do_xor = false;
1070
1071	if (is_ubsan)
1072	gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1073
1074	if (lhs)
1075	{
1076	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
1077	if (!is_ubsan)
1078	write_complex_part (target, const0_rtx, true, false);
1079	}
1080
1081	/* We assume both operands and result have the same precision
1082	here (GET_MODE_BITSIZE (mode)), S stands for signed type
1083	with that precision, U for unsigned type with that precision,
1084	sgn for unsigned most significant bit in that precision.
1085	s1 is signed first operand, u1 is unsigned first operand,
1086	s2 is signed second operand, u2 is unsigned second operand,
1087	sr is signed result, ur is unsigned result and the following
1088	rules say how to compute result (which is always result of
1089	the operands as if both were unsigned, cast to the right
1090	signedness) and how to compute whether operation overflowed.
1091
1092	s1 + s2 -> sr
1093	res = (S) ((U) s1 + (U) s2)
1094	ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
1095	s1 - s2 -> sr
1096	res = (S) ((U) s1 - (U) s2)
1097	ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
1098	u1 + u2 -> ur
1099	res = u1 + u2
1100	ovf = res < u1 (or jump on carry, but RTL opts will handle it)
1101	u1 - u2 -> ur
1102	res = u1 - u2
1103	ovf = res > u1 (or jump on carry, but RTL opts will handle it)
1104	s1 + u2 -> sr
1105	res = (S) ((U) s1 + u2)
1106	ovf = ((U) res ^ sgn) < u2
1107	s1 + u2 -> ur
1108	t1 = (S) (u2 ^ sgn)
1109	t2 = s1 + t1
1110	res = (U) t2 ^ sgn
1111	ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
1112	s1 - u2 -> sr
1113	res = (S) ((U) s1 - u2)
1114	ovf = u2 > ((U) s1 ^ sgn)
1115	s1 - u2 -> ur
1116	res = (U) s1 - u2
1117	ovf = s1 < 0 || u2 > (U) s1
1118	u1 - s2 -> sr
1119	res = u1 - (U) s2
1120	ovf = u1 >= ((U) s2 ^ sgn)
1121	u1 - s2 -> ur
1122	t1 = u1 ^ sgn
1123	t2 = t1 - (U) s2
1124	res = t2 ^ sgn
1125	ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
1126	s1 + s2 -> ur
1127	res = (U) s1 + (U) s2
1128	ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
1129	u1 + u2 -> sr
1130	res = (S) (u1 + u2)
1131	ovf = (U) res < u2 || res < 0
1132	u1 - u2 -> sr
1133	res = (S) (u1 - u2)
1134	ovf = u1 >= u2 ? res < 0 : res >= 0
1135	s1 - s2 -> ur
1136	res = (U) s1 - (U) s2
1137	ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
1138
1139	if (code == PLUS_EXPR && uns0_p && !uns1_p)
1140	{
1141	/* PLUS_EXPR is commutative, if operand signedness differs,
1142	canonicalize to the first operand being signed and second
1143	unsigned to simplify following code. */
1144	std::swap (a&: op0, b&: op1);
1145	std::swap (a&: arg0, b&: arg1);
1146	uns0_p = false;
1147	uns1_p = true;
1148	}
1149
1150	/* u1 +- u2 -> ur */
1151	if (uns0_p && uns1_p && unsr_p)
1152	{
1153	insn_code icode = optab_handler (op: code == PLUS_EXPR ? uaddv4_optab
1154	: usubv4_optab, mode);
1155	if (icode != CODE_FOR_nothing)
1156	{
1157	class expand_operand ops[4];
1158	rtx_insn *last = get_last_insn ();
1159
1160	res = gen_reg_rtx (mode);
1161	create_output_operand (op: &ops[0], x: res, mode);
1162	create_input_operand (op: &ops[1], value: op0, mode);
1163	create_input_operand (op: &ops[2], value: op1, mode);
1164	create_fixed_operand (op: &ops[3], x: do_error);
1165	if (maybe_expand_insn (icode, nops: 4, ops))
1166	{
1167	last = get_last_insn ();
1168	if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1169	&& JUMP_P (last)
1170	&& any_condjump_p (last)
1171	&& !find_reg_note (last, REG_BR_PROB, 0))
1172	add_reg_br_prob_note (last,
1173	profile_probability::very_unlikely ());
1174	emit_jump (done_label);
1175	goto do_error_label;
1176	}
1177
1178	delete_insns_since (last);
1179	}
1180
1181	/* Compute the operation. On RTL level, the addition is always
1182	unsigned. */
1183	res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1184	op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1185	rtx tem = op0;
1186	/* For PLUS_EXPR, the operation is commutative, so we can pick
1187	operand to compare against. For prec <= BITS_PER_WORD, I think
1188	preferring REG operand is better over CONST_INT, because
1189	the CONST_INT might enlarge the instruction or CSE would need
1190	to figure out we'd already loaded it into a register before.
1191	For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
1192	as then the multi-word comparison can be perhaps simplified. */
1193	if (code == PLUS_EXPR
1194	&& (prec <= BITS_PER_WORD
1195	? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
1196	: CONST_SCALAR_INT_P (op1)))
1197	tem = op1;
1198	do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
1199	true, mode, NULL_RTX, NULL, done_label,
1200	profile_probability::very_likely ());
1201	goto do_error_label;
1202	}
1203
1204	/* s1 +- u2 -> sr */
1205	if (!uns0_p && uns1_p && !unsr_p)
1206	{
1207	/* Compute the operation. On RTL level, the addition is always
1208	unsigned. */
1209	res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1210	op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1211	rtx tem = expand_binop (mode, add_optab,
1212	code == PLUS_EXPR ? res : op0, sgn,
1213	NULL_RTX, false, OPTAB_LIB_WIDEN);
1214	do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
1215	done_label, profile_probability::very_likely ());
1216	goto do_error_label;
1217	}
1218
1219	/* s1 + u2 -> ur */
1220	if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
1221	{
1222	op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
1223	OPTAB_LIB_WIDEN);
1224	/* As we've changed op1, we have to avoid using the value range
1225	for the original argument. */
1226	arg1 = error_mark_node;
1227	do_xor = true;
1228	goto do_signed;
1229	}
1230
1231	/* u1 - s2 -> ur */
1232	if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
1233	{
1234	op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
1235	OPTAB_LIB_WIDEN);
1236	/* As we've changed op0, we have to avoid using the value range
1237	for the original argument. */
1238	arg0 = error_mark_node;
1239	do_xor = true;
1240	goto do_signed;
1241	}
1242
1243	/* s1 - u2 -> ur */
1244	if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
1245	{
1246	/* Compute the operation. On RTL level, the addition is always
1247	unsigned. */
1248	res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1249	OPTAB_LIB_WIDEN);
1250	int pos_neg = get_range_pos_neg (arg0);
1251	if (pos_neg == 2)
1252	/* If ARG0 is known to be always negative, this is always overflow. */
1253	emit_jump (do_error);
1254	else if (pos_neg == 3)
1255	/* If ARG0 is not known to be always positive, check at runtime. */
1256	do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
1257	NULL, do_error, profile_probability::very_unlikely ());
1258	do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
1259	done_label, profile_probability::very_likely ());
1260	goto do_error_label;
1261	}
1262
1263	/* u1 - s2 -> sr */
1264	if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
1265	{
1266	/* Compute the operation. On RTL level, the addition is always
1267	unsigned. */
1268	res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1269	OPTAB_LIB_WIDEN);
1270	rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
1271	OPTAB_LIB_WIDEN);
1272	do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
1273	done_label, profile_probability::very_likely ());
1274	goto do_error_label;
1275	}
1276
1277	/* u1 + u2 -> sr */
1278	if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
1279	{
1280	/* Compute the operation. On RTL level, the addition is always
1281	unsigned. */
1282	res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
1283	OPTAB_LIB_WIDEN);
1284	do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1285	NULL, do_error, profile_probability::very_unlikely ());
1286	rtx tem = op1;
1287	/* The operation is commutative, so we can pick operand to compare
1288	against. For prec <= BITS_PER_WORD, I think preferring REG operand
1289	is better over CONST_INT, because the CONST_INT might enlarge the
1290	instruction or CSE would need to figure out we'd already loaded it
1291	into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1292	might be more beneficial, as then the multi-word comparison can be
1293	perhaps simplified. */
1294	if (prec <= BITS_PER_WORD
1295	? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
1296	: CONST_SCALAR_INT_P (op0))
1297	tem = op0;
1298	do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
1299	done_label, profile_probability::very_likely ());
1300	goto do_error_label;
1301	}
1302
1303	/* s1 +- s2 -> ur */
1304	if (!uns0_p && !uns1_p && unsr_p)
1305	{
1306	/* Compute the operation. On RTL level, the addition is always
1307	unsigned. */
1308	res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1309	op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1310	int pos_neg = get_range_pos_neg (arg1);
1311	if (code == PLUS_EXPR)
1312	{
1313	int pos_neg0 = get_range_pos_neg (arg0);
1314	if (pos_neg0 != 3 && pos_neg == 3)
1315	{
1316	std::swap (a&: op0, b&: op1);
1317	pos_neg = pos_neg0;
1318	}
1319	}
1320	rtx tem;
1321	if (pos_neg != 3)
1322	{
1323	tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
1324	? and_optab : ior_optab,
1325	op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
1326	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
1327	NULL, done_label, profile_probability::very_likely ());
1328	}
1329	else
1330	{
1331	rtx_code_label *do_ior_label = gen_label_rtx ();
1332	do_compare_rtx_and_jump (op1, const0_rtx,
1333	code == MINUS_EXPR ? GE : LT, false, mode,
1334	NULL_RTX, NULL, do_ior_label,
1335	profile_probability::even ());
1336	tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
1337	OPTAB_LIB_WIDEN);
1338	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1339	NULL, done_label, profile_probability::very_likely ());
1340	emit_jump (do_error);
1341	emit_label (do_ior_label);
1342	tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
1343	OPTAB_LIB_WIDEN);
1344	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1345	NULL, done_label, profile_probability::very_likely ());
1346	}
1347	goto do_error_label;
1348	}
1349
1350	/* u1 - u2 -> sr */
1351	if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
1352	{
1353	/* Compute the operation. On RTL level, the addition is always
1354	unsigned. */
1355	res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1356	OPTAB_LIB_WIDEN);
1357	rtx_code_label *op0_geu_op1 = gen_label_rtx ();
1358	do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
1359	op0_geu_op1, profile_probability::even ());
1360	do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1361	NULL, done_label, profile_probability::very_likely ());
1362	emit_jump (do_error);
1363	emit_label (op0_geu_op1);
1364	do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
1365	NULL, done_label, profile_probability::very_likely ());
1366	goto do_error_label;
1367	}
1368
1369	gcc_assert (!uns0_p && !uns1_p && !unsr_p);
1370
1371	/* s1 +- s2 -> sr */
1372	do_signed:
1373	{
1374	insn_code icode = optab_handler (op: code == PLUS_EXPR ? addv4_optab
1375	: subv4_optab, mode);
1376	if (icode != CODE_FOR_nothing)
1377	{
1378	class expand_operand ops[4];
1379	rtx_insn *last = get_last_insn ();
1380
1381	res = gen_reg_rtx (mode);
1382	create_output_operand (op: &ops[0], x: res, mode);
1383	create_input_operand (op: &ops[1], value: op0, mode);
1384	create_input_operand (op: &ops[2], value: op1, mode);
1385	create_fixed_operand (op: &ops[3], x: do_error);
1386	if (maybe_expand_insn (icode, nops: 4, ops))
1387	{
1388	last = get_last_insn ();
1389	if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1390	&& JUMP_P (last)
1391	&& any_condjump_p (last)
1392	&& !find_reg_note (last, REG_BR_PROB, 0))
1393	add_reg_br_prob_note (last,
1394	profile_probability::very_unlikely ());
1395	emit_jump (done_label);
1396	goto do_error_label;
1397	}
1398
1399	delete_insns_since (last);
1400	}
1401
1402	/* Compute the operation. On RTL level, the addition is always
1403	unsigned. */
1404	res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1405	op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1406
1407	/* If we can prove that one of the arguments (for MINUS_EXPR only
1408	the second operand, as subtraction is not commutative) is always
1409	non-negative or always negative, we can do just one comparison
1410	and conditional jump. */
1411	int pos_neg = get_range_pos_neg (arg1);
1412	if (code == PLUS_EXPR)
1413	{
1414	int pos_neg0 = get_range_pos_neg (arg0);
1415	if (pos_neg0 != 3 && pos_neg == 3)
1416	{
1417	std::swap (a&: op0, b&: op1);
1418	pos_neg = pos_neg0;
1419	}
1420	}
1421
1422	/* Addition overflows if and only if the two operands have the same sign,
1423	and the result has the opposite sign. Subtraction overflows if and
1424	only if the two operands have opposite sign, and the subtrahend has
1425	the same sign as the result. Here 0 is counted as positive. */
1426	if (pos_neg == 3)
1427	{
1428	/* Compute op0 ^ op1 (operands have opposite sign). */
1429	rtx op_xor = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1430	OPTAB_LIB_WIDEN);
1431
1432	/* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1433	rtx res_xor = expand_binop (mode, xor_optab, res, op1, NULL_RTX, false,
1434	OPTAB_LIB_WIDEN);
1435
1436	rtx tem;
1437	if (code == PLUS_EXPR)
1438	{
1439	/* Compute (res ^ op1) & ~(op0 ^ op1). */
1440	tem = expand_unop (mode, one_cmpl_optab, op_xor, NULL_RTX, false);
1441	tem = expand_binop (mode, and_optab, res_xor, tem, NULL_RTX, false,
1442	OPTAB_LIB_WIDEN);
1443	}
1444	else
1445	{
1446	/* Compute (op0 ^ op1) & ~(res ^ op1). */
1447	tem = expand_unop (mode, one_cmpl_optab, res_xor, NULL_RTX, false);
1448	tem = expand_binop (mode, and_optab, op_xor, tem, NULL_RTX, false,
1449	OPTAB_LIB_WIDEN);
1450	}
1451
1452	/* No overflow if the result has bit sign cleared. */
1453	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1454	NULL, done_label, profile_probability::very_likely ());
1455	}
1456
1457	/* Compare the result of the operation with the first operand.
1458	No overflow for addition if second operand is positive and result
1459	is larger or second operand is negative and result is smaller.
1460	Likewise for subtraction with sign of second operand flipped. */
1461	else
1462	do_compare_rtx_and_jump (res, op0,
1463	(pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
1464	false, mode, NULL_RTX, NULL, done_label,
1465	profile_probability::very_likely ());
1466	}
1467
1468	do_error_label:
1469	emit_label (do_error);
1470	if (is_ubsan)
1471	{
1472	/* Expand the ubsan builtin call. */
1473	push_temp_slots ();
1474	fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
1475	arg0, arg1, datap);
1476	expand_normal (exp: fn);
1477	pop_temp_slots ();
1478	do_pending_stack_adjust ();
1479	}
1480	else if (lhs)
1481	expand_arith_set_overflow (lhs, target);
1482
1483	/* We're done. */
1484	emit_label (done_label);
1485
1486	if (lhs)
1487	{
1488	if (is_ubsan)
1489	expand_ubsan_result_store (lhs, target, mode, res, do_error);
1490	else
1491	{
1492	if (do_xor)
1493	res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
1494	OPTAB_LIB_WIDEN);
1495
1496	expand_arith_overflow_result_store (lhs, target, mode, res);
1497	}
1498	}
1499	}
1500
1501	/* Add negate overflow checking to the statement STMT. */
1502
1503	static void
1504	expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan,
1505	tree *datap)
1506	{
1507	rtx res, op1;
1508	tree fn;
1509	rtx_code_label *done_label, *do_error;
1510	rtx target = NULL_RTX;
1511
1512	done_label = gen_label_rtx ();
1513	do_error = gen_label_rtx ();
1514
1515	do_pending_stack_adjust ();
1516	op1 = expand_normal (exp: arg1);
1517
1518	scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1));
1519	if (lhs)
1520	{
1521	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
1522	if (!is_ubsan)
1523	write_complex_part (target, const0_rtx, true, false);
1524	}
1525
1526	enum insn_code icode = optab_handler (op: negv3_optab, mode);
1527	if (icode != CODE_FOR_nothing)
1528	{
1529	class expand_operand ops[3];
1530	rtx_insn *last = get_last_insn ();
1531
1532	res = gen_reg_rtx (mode);
1533	create_output_operand (op: &ops[0], x: res, mode);
1534	create_input_operand (op: &ops[1], value: op1, mode);
1535	create_fixed_operand (op: &ops[2], x: do_error);
1536	if (maybe_expand_insn (icode, nops: 3, ops))
1537	{
1538	last = get_last_insn ();
1539	if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1540	&& JUMP_P (last)
1541	&& any_condjump_p (last)
1542	&& !find_reg_note (last, REG_BR_PROB, 0))
1543	add_reg_br_prob_note (last,
1544	profile_probability::very_unlikely ());
1545	emit_jump (done_label);
1546	}
1547	else
1548	{
1549	delete_insns_since (last);
1550	icode = CODE_FOR_nothing;
1551	}
1552	}
1553
1554	if (icode == CODE_FOR_nothing)
1555	{
1556	/* Compute the operation. On RTL level, the addition is always
1557	unsigned. */
1558	res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1559
1560	/* Compare the operand with the most negative value. */
1561	rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
1562	do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
1563	done_label, profile_probability::very_likely ());
1564	}
1565
1566	emit_label (do_error);
1567	if (is_ubsan)
1568	{
1569	/* Expand the ubsan builtin call. */
1570	push_temp_slots ();
1571	fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
1572	arg1, NULL_TREE, datap);
1573	expand_normal (exp: fn);
1574	pop_temp_slots ();
1575	do_pending_stack_adjust ();
1576	}
1577	else if (lhs)
1578	expand_arith_set_overflow (lhs, target);
1579
1580	/* We're done. */
1581	emit_label (done_label);
1582
1583	if (lhs)
1584	{
1585	if (is_ubsan)
1586	expand_ubsan_result_store (lhs, target, mode, res, do_error);
1587	else
1588	expand_arith_overflow_result_store (lhs, target, mode, res);
1589	}
1590	}
1591
1592	/* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1593	mode MODE can be expanded without using a libcall. */
1594
1595	static bool
1596	can_widen_mult_without_libcall (scalar_int_mode wmode, scalar_int_mode mode,
1597	rtx op0, rtx op1, bool uns)
1598	{
1599	if (find_widening_optab_handler (umul_widen_optab, wmode, mode)
1600	!= CODE_FOR_nothing)
1601	return true;
1602
1603	if (find_widening_optab_handler (smul_widen_optab, wmode, mode)
1604	!= CODE_FOR_nothing)
1605	return true;
1606
1607	rtx_insn *last = get_last_insn ();
1608	if (CONSTANT_P (op0))
1609	op0 = convert_modes (mode: wmode, oldmode: mode, x: op0, unsignedp: uns);
1610	else
1611	op0 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 1);
1612	if (CONSTANT_P (op1))
1613	op1 = convert_modes (mode: wmode, oldmode: mode, x: op1, unsignedp: uns);
1614	else
1615	op1 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 2);
1616	rtx ret = expand_mult (wmode, op0, op1, NULL_RTX, uns, true);
1617	delete_insns_since (last);
1618	return ret != NULL_RTX;
1619	}
1620
1621	/* Add mul overflow checking to the statement STMT. */
1622
1623	static void
1624	expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
1625	bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan,
1626	tree *datap)
1627	{
1628	rtx res, op0, op1;
1629	tree fn, type;
1630	rtx_code_label *done_label, *do_error;
1631	rtx target = NULL_RTX;
1632	signop sign;
1633	enum insn_code icode;
1634	int save_flag_trapv = flag_trapv;
1635
1636	/* We don't want any __mulv?i3 etc. calls from the expansion of
1637	these internal functions, so disable -ftrapv temporarily. */
1638	flag_trapv = 0;
1639	done_label = gen_label_rtx ();
1640	do_error = gen_label_rtx ();
1641
1642	do_pending_stack_adjust ();
1643	op0 = expand_normal (exp: arg0);
1644	op1 = expand_normal (exp: arg1);
1645
1646	scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
1647	bool uns = unsr_p;
1648	if (lhs)
1649	{
1650	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
1651	if (!is_ubsan)
1652	write_complex_part (target, const0_rtx, true, false);
1653	}
1654
1655	if (is_ubsan)
1656	gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1657
1658	/* We assume both operands and result have the same precision
1659	here (GET_MODE_BITSIZE (mode)), S stands for signed type
1660	with that precision, U for unsigned type with that precision,
1661	sgn for unsigned most significant bit in that precision.
1662	s1 is signed first operand, u1 is unsigned first operand,
1663	s2 is signed second operand, u2 is unsigned second operand,
1664	sr is signed result, ur is unsigned result and the following
1665	rules say how to compute result (which is always result of
1666	the operands as if both were unsigned, cast to the right
1667	signedness) and how to compute whether operation overflowed.
1668	main_ovf (false) stands for jump on signed multiplication
1669	overflow or the main algorithm with uns == false.
1670	main_ovf (true) stands for jump on unsigned multiplication
1671	overflow or the main algorithm with uns == true.
1672
1673	s1 * s2 -> sr
1674	res = (S) ((U) s1 * (U) s2)
1675	ovf = main_ovf (false)
1676	u1 * u2 -> ur
1677	res = u1 * u2
1678	ovf = main_ovf (true)
1679	s1 * u2 -> ur
1680	res = (U) s1 * u2
1681	ovf = (s1 < 0 && u2) || main_ovf (true)
1682	u1 * u2 -> sr
1683	res = (S) (u1 * u2)
1684	ovf = res < 0 || main_ovf (true)
1685	s1 * u2 -> sr
1686	res = (S) ((U) s1 * u2)
1687	ovf = (S) u2 >= 0 ? main_ovf (false)
1688	: (s1 != 0 && (s1 != -1 || u2 != (U) res))
1689	s1 * s2 -> ur
1690	t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1691	t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1692	res = t1 * t2
1693	ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1694
1695	if (uns0_p && !uns1_p)
1696	{
1697	/* Multiplication is commutative, if operand signedness differs,
1698	canonicalize to the first operand being signed and second
1699	unsigned to simplify following code. */
1700	std::swap (a&: op0, b&: op1);
1701	std::swap (a&: arg0, b&: arg1);
1702	uns0_p = false;
1703	uns1_p = true;
1704	}
1705
1706	int pos_neg0 = get_range_pos_neg (arg0);
1707	int pos_neg1 = get_range_pos_neg (arg1);
1708	/* Unsigned types with smaller than mode precision, even if they have most
1709	significant bit set, are still zero-extended. */
1710	if (uns0_p && TYPE_PRECISION (TREE_TYPE (arg0)) < GET_MODE_PRECISION (mode))
1711	pos_neg0 = 1;
1712	if (uns1_p && TYPE_PRECISION (TREE_TYPE (arg1)) < GET_MODE_PRECISION (mode))
1713	pos_neg1 = 1;
1714
1715	/* s1 * u2 -> ur */
1716	if (!uns0_p && uns1_p && unsr_p)
1717	{
1718	switch (pos_neg0)
1719	{
1720	case 1:
1721	/* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1722	goto do_main;
1723	case 2:
1724	/* If s1 is negative, avoid the main code, just multiply and
1725	signal overflow if op1 is not 0. */
1726	struct separate_ops ops;
1727	ops.code = MULT_EXPR;
1728	ops.type = TREE_TYPE (arg1);
1729	ops.op0 = make_tree (ops.type, op0);
1730	ops.op1 = make_tree (ops.type, op1);
1731	ops.op2 = NULL_TREE;
1732	ops.location = loc;
1733	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1734	do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1735	NULL, done_label, profile_probability::very_likely ());
1736	goto do_error_label;
1737	case 3:
1738	if (get_min_precision (arg: arg1, sign: UNSIGNED)
1739	+ get_min_precision (arg: arg0, sign: SIGNED) <= GET_MODE_PRECISION (mode))
1740	{
1741	/* If the first operand is sign extended from narrower type, the
1742	second operand is zero extended from narrower type and
1743	the sum of the two precisions is smaller or equal to the
1744	result precision: if the first argument is at runtime
1745	non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1746	and there will be no overflow, if the first argument is
1747	negative and the second argument zero, the result will be
1748	0 and there will be no overflow, if the first argument is
1749	negative and the second argument positive, the result when
1750	treated as signed will be negative (minimum -0x7f80 or
1751	-0x7f..f80..0) there will be always overflow. So, do
1752	res = (U) (s1 * u2)
1753	ovf = (S) res < 0 */
1754	struct separate_ops ops;
1755	ops.code = MULT_EXPR;
1756	ops.type
1757	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1758	1);
1759	ops.op0 = make_tree (ops.type, op0);
1760	ops.op1 = make_tree (ops.type, op1);
1761	ops.op2 = NULL_TREE;
1762	ops.location = loc;
1763	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1764	do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1765	mode, NULL_RTX, NULL, done_label,
1766	profile_probability::very_likely ());
1767	goto do_error_label;
1768	}
1769	rtx_code_label *do_main_label;
1770	do_main_label = gen_label_rtx ();
1771	do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
1772	NULL, do_main_label, profile_probability::very_likely ());
1773	do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1774	NULL, do_main_label, profile_probability::very_likely ());
1775	expand_arith_set_overflow (lhs, target);
1776	emit_label (do_main_label);
1777	goto do_main;
1778	default:
1779	gcc_unreachable ();
1780	}
1781	}
1782
1783	/* u1 * u2 -> sr */
1784	if (uns0_p && uns1_p && !unsr_p)
1785	{
1786	if ((pos_neg0 | pos_neg1) == 1)
1787	{
1788	/* If both arguments are zero extended from narrower types,
1789	the MSB will be clear on both and so we can pretend it is
1790	a normal s1 * s2 -> sr multiplication. */
1791	uns0_p = false;
1792	uns1_p = false;
1793	}
1794	else
1795	uns = true;
1796	/* Rest of handling of this case after res is computed. */
1797	goto do_main;
1798	}
1799
1800	/* s1 * u2 -> sr */
1801	if (!uns0_p && uns1_p && !unsr_p)
1802	{
1803	switch (pos_neg1)
1804	{
1805	case 1:
1806	goto do_main;
1807	case 2:
1808	/* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1809	avoid the main code, just multiply and signal overflow
1810	unless 0 * u2 or -1 * ((U) Smin). */
1811	struct separate_ops ops;
1812	ops.code = MULT_EXPR;
1813	ops.type = TREE_TYPE (arg1);
1814	ops.op0 = make_tree (ops.type, op0);
1815	ops.op1 = make_tree (ops.type, op1);
1816	ops.op2 = NULL_TREE;
1817	ops.location = loc;
1818	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1819	do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1820	NULL, done_label, profile_probability::very_likely ());
1821	do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
1822	NULL, do_error, profile_probability::very_unlikely ());
1823	int prec;
1824	prec = GET_MODE_PRECISION (mode);
1825	rtx sgn;
1826	sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1827	do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
1828	NULL, done_label, profile_probability::very_likely ());
1829	goto do_error_label;
1830	case 3:
1831	/* Rest of handling of this case after res is computed. */
1832	goto do_main;
1833	default:
1834	gcc_unreachable ();
1835	}
1836	}
1837
1838	/* s1 * s2 -> ur */
1839	if (!uns0_p && !uns1_p && unsr_p)
1840	{
1841	rtx tem;
1842	switch (pos_neg0 | pos_neg1)
1843	{
1844	case 1: /* Both operands known to be non-negative. */
1845	goto do_main;
1846	case 2: /* Both operands known to be negative. */
1847	op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
1848	op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1849	/* Avoid looking at arg0/arg1 ranges, as we've changed
1850	the arguments. */
1851	arg0 = error_mark_node;
1852	arg1 = error_mark_node;
1853	goto do_main;
1854	case 3:
1855	if ((pos_neg0 ^ pos_neg1) == 3)
1856	{
1857	/* If one operand is known to be negative and the other
1858	non-negative, this overflows always, unless the non-negative
1859	one is 0. Just do normal multiply and set overflow
1860	unless one of the operands is 0. */
1861	struct separate_ops ops;
1862	ops.code = MULT_EXPR;
1863	ops.type
1864	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1865	1);
1866	ops.op0 = make_tree (ops.type, op0);
1867	ops.op1 = make_tree (ops.type, op1);
1868	ops.op2 = NULL_TREE;
1869	ops.location = loc;
1870	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1871	do_compare_rtx_and_jump (pos_neg0 == 1 ? op0 : op1, const0_rtx, EQ,
1872	true, mode, NULL_RTX, NULL, done_label,
1873	profile_probability::very_likely ());
1874	goto do_error_label;
1875	}
1876	if (get_min_precision (arg: arg0, sign: SIGNED)
1877	+ get_min_precision (arg: arg1, sign: SIGNED) <= GET_MODE_PRECISION (mode))
1878	{
1879	/* If both operands are sign extended from narrower types and
1880	the sum of the two precisions is smaller or equal to the
1881	result precision: if both arguments are at runtime
1882	non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1883	and there will be no overflow, if both arguments are negative,
1884	maximum result will be 0x40..00 and there will be no overflow
1885	either, if one argument is positive and the other argument
1886	negative, the result when treated as signed will be negative
1887	and there will be always overflow, and if one argument is
1888	zero and the other negative the result will be zero and no
1889	overflow. So, do
1890	res = (U) (s1 * s2)
1891	ovf = (S) res < 0 */
1892	struct separate_ops ops;
1893	ops.code = MULT_EXPR;
1894	ops.type
1895	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1896	1);
1897	ops.op0 = make_tree (ops.type, op0);
1898	ops.op1 = make_tree (ops.type, op1);
1899	ops.op2 = NULL_TREE;
1900	ops.location = loc;
1901	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1902	do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1903	mode, NULL_RTX, NULL, done_label,
1904	profile_probability::very_likely ());
1905	goto do_error_label;
1906	}
1907	/* The general case, do all the needed comparisons at runtime. */
1908	rtx_code_label *do_main_label, *after_negate_label;
1909	rtx rop0, rop1;
1910	rop0 = gen_reg_rtx (mode);
1911	rop1 = gen_reg_rtx (mode);
1912	emit_move_insn (rop0, op0);
1913	emit_move_insn (rop1, op1);
1914	op0 = rop0;
1915	op1 = rop1;
1916	do_main_label = gen_label_rtx ();
1917	after_negate_label = gen_label_rtx ();
1918	tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1919	OPTAB_LIB_WIDEN);
1920	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1921	NULL, after_negate_label, profile_probability::very_likely ());
1922	/* Both arguments negative here, negate them and continue with
1923	normal unsigned overflow checking multiplication. */
1924	emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
1925	NULL_RTX, false));
1926	emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
1927	NULL_RTX, false));
1928	/* Avoid looking at arg0/arg1 ranges, as we might have changed
1929	the arguments. */
1930	arg0 = error_mark_node;
1931	arg1 = error_mark_node;
1932	emit_jump (do_main_label);
1933	emit_label (after_negate_label);
1934	tem = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1935	OPTAB_LIB_WIDEN);
1936	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1937	NULL, do_main_label,
1938	profile_probability::very_likely ());
1939	/* One argument is negative here, the other positive. This
1940	overflows always, unless one of the arguments is 0. But
1941	if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1942	is, thus we can keep do_main code oring in overflow as is. */
1943	if (pos_neg0 != 2)
1944	do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1945	NULL, do_main_label,
1946	profile_probability::very_unlikely ());
1947	if (pos_neg1 != 2)
1948	do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1949	NULL, do_main_label,
1950	profile_probability::very_unlikely ());
1951	expand_arith_set_overflow (lhs, target);
1952	emit_label (do_main_label);
1953	goto do_main;
1954	default:
1955	gcc_unreachable ();
1956	}
1957	}
1958
1959	do_main:
1960	type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
1961	sign = uns ? UNSIGNED : SIGNED;
1962	icode = optab_handler (op: uns ? umulv4_optab : mulv4_optab, mode);
1963	if (uns
1964	&& (integer_pow2p (arg0) || integer_pow2p (arg1))
1965	&& (optimize_insn_for_speed_p () || icode == CODE_FOR_nothing))
1966	{
1967	/* Optimize unsigned multiplication by power of 2 constant
1968	using 2 shifts, one for result, one to extract the shifted
1969	out bits to see if they are all zero.
1970	Don't do this if optimizing for size and we have umulv4_optab,
1971	in that case assume multiplication will be shorter.
1972	This is heuristics based on the single target that provides
1973	umulv4 right now (i?86/x86_64), if further targets add it, this
1974	might need to be revisited.
1975	Cases where both operands are constant should be folded already
1976	during GIMPLE, and cases where one operand is constant but not
1977	power of 2 are questionable, either the WIDEN_MULT_EXPR case
1978	below can be done without multiplication, just by shifts and adds,
1979	or we'd need to divide the result (and hope it actually doesn't
1980	really divide nor multiply) and compare the result of the division
1981	with the original operand. */
1982	rtx opn0 = op0;
1983	rtx opn1 = op1;
1984	tree argn0 = arg0;
1985	tree argn1 = arg1;
1986	if (integer_pow2p (arg0))
1987	{
1988	std::swap (a&: opn0, b&: opn1);
1989	std::swap (a&: argn0, b&: argn1);
1990	}
1991	int cnt = tree_log2 (argn1);
1992	if (cnt >= 0 && cnt < GET_MODE_PRECISION (mode))
1993	{
1994	rtx upper = const0_rtx;
1995	res = expand_shift (LSHIFT_EXPR, mode, opn0, cnt, NULL_RTX, uns);
1996	if (cnt != 0)
1997	upper = expand_shift (RSHIFT_EXPR, mode, opn0,
1998	GET_MODE_PRECISION (mode) - cnt,
1999	NULL_RTX, uns);
2000	do_compare_rtx_and_jump (upper, const0_rtx, EQ, true, mode,
2001	NULL_RTX, NULL, done_label,
2002	profile_probability::very_likely ());
2003	goto do_error_label;
2004	}
2005	}
2006	if (icode != CODE_FOR_nothing)
2007	{
2008	class expand_operand ops[4];
2009	rtx_insn *last = get_last_insn ();
2010
2011	res = gen_reg_rtx (mode);
2012	create_output_operand (op: &ops[0], x: res, mode);
2013	create_input_operand (op: &ops[1], value: op0, mode);
2014	create_input_operand (op: &ops[2], value: op1, mode);
2015	create_fixed_operand (op: &ops[3], x: do_error);
2016	if (maybe_expand_insn (icode, nops: 4, ops))
2017	{
2018	last = get_last_insn ();
2019	if (profile_status_for_fn (cfun) != PROFILE_ABSENT
2020	&& JUMP_P (last)
2021	&& any_condjump_p (last)
2022	&& !find_reg_note (last, REG_BR_PROB, 0))
2023	add_reg_br_prob_note (last,
2024	profile_probability::very_unlikely ());
2025	emit_jump (done_label);
2026	}
2027	else
2028	{
2029	delete_insns_since (last);
2030	icode = CODE_FOR_nothing;
2031	}
2032	}
2033
2034	if (icode == CODE_FOR_nothing)
2035	{
2036	struct separate_ops ops;
2037	int prec = GET_MODE_PRECISION (mode);
2038	scalar_int_mode hmode, wmode;
2039	ops.op0 = make_tree (type, op0);
2040	ops.op1 = make_tree (type, op1);
2041	ops.op2 = NULL_TREE;
2042	ops.location = loc;
2043
2044	/* Optimize unsigned overflow check where we don't use the
2045	multiplication result, just whether overflow happened.
2046	If we can do MULT_HIGHPART_EXPR, that followed by
2047	comparison of the result against zero is cheapest.
2048	We'll still compute res, but it should be DCEd later. */
2049	use_operand_p use;
2050	gimple *use_stmt;
2051	if (!is_ubsan
2052	&& lhs
2053	&& uns
2054	&& !(uns0_p && uns1_p && !unsr_p)
2055	&& can_mult_highpart_p (mode, uns) == 1
2056	&& single_imm_use (var: lhs, use_p: &use, stmt: &use_stmt)
2057	&& is_gimple_assign (gs: use_stmt)
2058	&& gimple_assign_rhs_code (gs: use_stmt) == IMAGPART_EXPR)
2059	goto highpart;
2060
2061	if (GET_MODE_2XWIDER_MODE (m: mode).exists (mode: &wmode)
2062	&& targetm.scalar_mode_supported_p (wmode)
2063	&& can_widen_mult_without_libcall (wmode, mode, op0, op1, uns))
2064	{
2065	twoxwider:
2066	ops.code = WIDEN_MULT_EXPR;
2067	ops.type
2068	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode: wmode), uns);
2069
2070	res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
2071	rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
2072	NULL_RTX, uns);
2073	hipart = convert_modes (mode, oldmode: wmode, x: hipart, unsignedp: uns);
2074	res = convert_modes (mode, oldmode: wmode, x: res, unsignedp: uns);
2075	if (uns)
2076	/* For the unsigned multiplication, there was overflow if
2077	HIPART is non-zero. */
2078	do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
2079	NULL_RTX, NULL, done_label,
2080	profile_probability::very_likely ());
2081	else
2082	{
2083	/* RES is used more than once, place it in a pseudo. */
2084	res = force_reg (mode, res);
2085
2086	rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
2087	NULL_RTX, 0);
2088	/* RES is low half of the double width result, HIPART
2089	the high half. There was overflow if
2090	HIPART is different from RES < 0 ? -1 : 0. */
2091	do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
2092	NULL_RTX, NULL, done_label,
2093	profile_probability::very_likely ());
2094	}
2095	}
2096	else if (can_mult_highpart_p (mode, uns) == 1)
2097	{
2098	highpart:
2099	ops.code = MULT_HIGHPART_EXPR;
2100	ops.type = type;
2101
2102	rtx hipart = expand_expr_real_2 (&ops, NULL_RTX, mode,
2103	EXPAND_NORMAL);
2104	ops.code = MULT_EXPR;
2105	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2106	if (uns)
2107	/* For the unsigned multiplication, there was overflow if
2108	HIPART is non-zero. */
2109	do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
2110	NULL_RTX, NULL, done_label,
2111	profile_probability::very_likely ());
2112	else
2113	{
2114	rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
2115	NULL_RTX, 0);
2116	/* RES is low half of the double width result, HIPART
2117	the high half. There was overflow if
2118	HIPART is different from RES < 0 ? -1 : 0. */
2119	do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
2120	NULL_RTX, NULL, done_label,
2121	profile_probability::very_likely ());
2122	}
2123
2124	}
2125	else if (int_mode_for_size (size: prec / 2, limit: 1).exists (mode: &hmode)
2126	&& 2 * GET_MODE_PRECISION (mode: hmode) == prec)
2127	{
2128	rtx_code_label *large_op0 = gen_label_rtx ();
2129	rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
2130	rtx_code_label *one_small_one_large = gen_label_rtx ();
2131	rtx_code_label *both_ops_large = gen_label_rtx ();
2132	rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
2133	rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
2134	rtx_code_label *do_overflow = gen_label_rtx ();
2135	rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
2136
2137	unsigned int hprec = GET_MODE_PRECISION (mode: hmode);
2138	rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
2139	NULL_RTX, uns);
2140	hipart0 = convert_modes (mode: hmode, oldmode: mode, x: hipart0, unsignedp: uns);
2141	rtx lopart0 = convert_modes (mode: hmode, oldmode: mode, x: op0, unsignedp: uns);
2142	rtx signbit0 = const0_rtx;
2143	if (!uns)
2144	signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
2145	NULL_RTX, 0);
2146	rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
2147	NULL_RTX, uns);
2148	hipart1 = convert_modes (mode: hmode, oldmode: mode, x: hipart1, unsignedp: uns);
2149	rtx lopart1 = convert_modes (mode: hmode, oldmode: mode, x: op1, unsignedp: uns);
2150	rtx signbit1 = const0_rtx;
2151	if (!uns)
2152	signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
2153	NULL_RTX, 0);
2154
2155	res = gen_reg_rtx (mode);
2156
2157	/* True if op0 resp. op1 are known to be in the range of
2158	halfstype. */
2159	bool op0_small_p = false;
2160	bool op1_small_p = false;
2161	/* True if op0 resp. op1 are known to have all zeros or all ones
2162	in the upper half of bits, but are not known to be
2163	op{0,1}_small_p. */
2164	bool op0_medium_p = false;
2165	bool op1_medium_p = false;
2166	/* -1 if op{0,1} is known to be negative, 0 if it is known to be
2167	nonnegative, 1 if unknown. */
2168	int op0_sign = 1;
2169	int op1_sign = 1;
2170
2171	if (pos_neg0 == 1)
2172	op0_sign = 0;
2173	else if (pos_neg0 == 2)
2174	op0_sign = -1;
2175	if (pos_neg1 == 1)
2176	op1_sign = 0;
2177	else if (pos_neg1 == 2)
2178	op1_sign = -1;
2179
2180	unsigned int mprec0 = prec;
2181	if (arg0 != error_mark_node)
2182	mprec0 = get_min_precision (arg: arg0, sign);
2183	if (mprec0 <= hprec)
2184	op0_small_p = true;
2185	else if (!uns && mprec0 <= hprec + 1)
2186	op0_medium_p = true;
2187	unsigned int mprec1 = prec;
2188	if (arg1 != error_mark_node)
2189	mprec1 = get_min_precision (arg: arg1, sign);
2190	if (mprec1 <= hprec)
2191	op1_small_p = true;
2192	else if (!uns && mprec1 <= hprec + 1)
2193	op1_medium_p = true;
2194
2195	int smaller_sign = 1;
2196	int larger_sign = 1;
2197	if (op0_small_p)
2198	{
2199	smaller_sign = op0_sign;
2200	larger_sign = op1_sign;
2201	}
2202	else if (op1_small_p)
2203	{
2204	smaller_sign = op1_sign;
2205	larger_sign = op0_sign;
2206	}
2207	else if (op0_sign == op1_sign)
2208	{
2209	smaller_sign = op0_sign;
2210	larger_sign = op0_sign;
2211	}
2212
2213	if (!op0_small_p)
2214	do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
2215	NULL_RTX, NULL, large_op0,
2216	profile_probability::unlikely ());
2217
2218	if (!op1_small_p)
2219	do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
2220	NULL_RTX, NULL, small_op0_large_op1,
2221	profile_probability::unlikely ());
2222
2223	/* If both op0 and op1 are sign (!uns) or zero (uns) extended from
2224	hmode to mode, the multiplication will never overflow. We can
2225	do just one hmode x hmode => mode widening multiplication. */
2226	tree halfstype = build_nonstandard_integer_type (hprec, uns);
2227	ops.op0 = make_tree (halfstype, lopart0);
2228	ops.op1 = make_tree (halfstype, lopart1);
2229	ops.code = WIDEN_MULT_EXPR;
2230	ops.type = type;
2231	rtx thisres
2232	= expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2233	emit_move_insn (res, thisres);
2234	emit_jump (done_label);
2235
2236	emit_label (small_op0_large_op1);
2237
2238	/* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
2239	but op1 is not, just swap the arguments and handle it as op1
2240	sign/zero extended, op0 not. */
2241	rtx larger = gen_reg_rtx (mode);
2242	rtx hipart = gen_reg_rtx (hmode);
2243	rtx lopart = gen_reg_rtx (hmode);
2244	emit_move_insn (larger, op1);
2245	emit_move_insn (hipart, hipart1);
2246	emit_move_insn (lopart, lopart0);
2247	emit_jump (one_small_one_large);
2248
2249	emit_label (large_op0);
2250
2251	if (!op1_small_p)
2252	do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
2253	NULL_RTX, NULL, both_ops_large,
2254	profile_probability::unlikely ());
2255
2256	/* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2257	but op0 is not, prepare larger, hipart and lopart pseudos and
2258	handle it together with small_op0_large_op1. */
2259	emit_move_insn (larger, op0);
2260	emit_move_insn (hipart, hipart0);
2261	emit_move_insn (lopart, lopart1);
2262
2263	emit_label (one_small_one_large);
2264
2265	/* lopart is the low part of the operand that is sign extended
2266	to mode, larger is the other operand, hipart is the
2267	high part of larger and lopart0 and lopart1 are the low parts
2268	of both operands.
2269	We perform lopart0 * lopart1 and lopart * hipart widening
2270	multiplications. */
2271	tree halfutype = build_nonstandard_integer_type (hprec, 1);
2272	ops.op0 = make_tree (halfutype, lopart0);
2273	ops.op1 = make_tree (halfutype, lopart1);
2274	rtx lo0xlo1
2275	= expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2276
2277	ops.op0 = make_tree (halfutype, lopart);
2278	ops.op1 = make_tree (halfutype, hipart);
2279	rtx loxhi = gen_reg_rtx (mode);
2280	rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2281	emit_move_insn (loxhi, tem);
2282
2283	if (!uns)
2284	{
2285	/* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2286	if (larger_sign == 0)
2287	emit_jump (after_hipart_neg);
2288	else if (larger_sign != -1)
2289	do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
2290	NULL_RTX, NULL, after_hipart_neg,
2291	profile_probability::even ());
2292
2293	tem = convert_modes (mode, oldmode: hmode, x: lopart, unsignedp: 1);
2294	tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
2295	tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
2296	1, OPTAB_WIDEN);
2297	emit_move_insn (loxhi, tem);
2298
2299	emit_label (after_hipart_neg);
2300
2301	/* if (lopart < 0) loxhi -= larger; */
2302	if (smaller_sign == 0)
2303	emit_jump (after_lopart_neg);
2304	else if (smaller_sign != -1)
2305	do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
2306	NULL_RTX, NULL, after_lopart_neg,
2307	profile_probability::even ());
2308
2309	tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
2310	1, OPTAB_WIDEN);
2311	emit_move_insn (loxhi, tem);
2312
2313	emit_label (after_lopart_neg);
2314	}
2315
2316	/* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2317	tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
2318	tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
2319	1, OPTAB_WIDEN);
2320	emit_move_insn (loxhi, tem);
2321
2322	/* if (loxhi >> (bitsize / 2)
2323	== (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2324	if (loxhi >> (bitsize / 2) == 0 (if uns). */
2325	rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
2326	NULL_RTX, 0);
2327	hipartloxhi = convert_modes (mode: hmode, oldmode: mode, x: hipartloxhi, unsignedp: 0);
2328	rtx signbitloxhi = const0_rtx;
2329	if (!uns)
2330	signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
2331	convert_modes (mode: hmode, oldmode: mode,
2332	x: loxhi, unsignedp: 0),
2333	hprec - 1, NULL_RTX, 0);
2334
2335	do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
2336	NULL_RTX, NULL, do_overflow,
2337	profile_probability::very_unlikely ());
2338
2339	/* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2340	rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
2341	NULL_RTX, 1);
2342	tem = convert_modes (mode, oldmode: hmode,
2343	x: convert_modes (mode: hmode, oldmode: mode, x: lo0xlo1, unsignedp: 1), unsignedp: 1);
2344
2345	tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
2346	1, OPTAB_WIDEN);
2347	if (tem != res)
2348	emit_move_insn (res, tem);
2349	emit_jump (done_label);
2350
2351	emit_label (both_ops_large);
2352
2353	/* If both operands are large (not sign (!uns) or zero (uns)
2354	extended from hmode), then perform the full multiplication
2355	which will be the result of the operation.
2356	The only cases which don't overflow are for signed multiplication
2357	some cases where both hipart0 and highpart1 are 0 or -1.
2358	For unsigned multiplication when high parts are both non-zero
2359	this overflows always. */
2360	ops.code = MULT_EXPR;
2361	ops.op0 = make_tree (type, op0);
2362	ops.op1 = make_tree (type, op1);
2363	tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2364	emit_move_insn (res, tem);
2365
2366	if (!uns)
2367	{
2368	if (!op0_medium_p)
2369	{
2370	tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
2371	NULL_RTX, 1, OPTAB_WIDEN);
2372	do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2373	NULL_RTX, NULL, do_error,
2374	profile_probability::very_unlikely ());
2375	}
2376
2377	if (!op1_medium_p)
2378	{
2379	tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
2380	NULL_RTX, 1, OPTAB_WIDEN);
2381	do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2382	NULL_RTX, NULL, do_error,
2383	profile_probability::very_unlikely ());
2384	}
2385
2386	/* At this point hipart{0,1} are both in [-1, 0]. If they are
2387	the same, overflow happened if res is non-positive, if they
2388	are different, overflow happened if res is positive. */
2389	if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
2390	emit_jump (hipart_different);
2391	else if (op0_sign == 1 || op1_sign == 1)
2392	do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
2393	NULL_RTX, NULL, hipart_different,
2394	profile_probability::even ());
2395
2396	do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
2397	NULL_RTX, NULL, do_error,
2398	profile_probability::very_unlikely ());
2399	emit_jump (done_label);
2400
2401	emit_label (hipart_different);
2402
2403	do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
2404	NULL_RTX, NULL, do_error,
2405	profile_probability::very_unlikely ());
2406	emit_jump (done_label);
2407	}
2408
2409	emit_label (do_overflow);
2410
2411	/* Overflow, do full multiplication and fallthru into do_error. */
2412	ops.op0 = make_tree (type, op0);
2413	ops.op1 = make_tree (type, op1);
2414	tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2415	emit_move_insn (res, tem);
2416	}
2417	else if (GET_MODE_2XWIDER_MODE (m: mode).exists (mode: &wmode)
2418	&& targetm.scalar_mode_supported_p (wmode))
2419	/* Even emitting a libcall is better than not detecting overflow
2420	at all. */
2421	goto twoxwider;
2422	else
2423	{
2424	gcc_assert (!is_ubsan);
2425	ops.code = MULT_EXPR;
2426	ops.type = type;
2427	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2428	emit_jump (done_label);
2429	}
2430	}
2431
2432	do_error_label:
2433	emit_label (do_error);
2434	if (is_ubsan)
2435	{
2436	/* Expand the ubsan builtin call. */
2437	push_temp_slots ();
2438	fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
2439	arg0, arg1, datap);
2440	expand_normal (exp: fn);
2441	pop_temp_slots ();
2442	do_pending_stack_adjust ();
2443	}
2444	else if (lhs)
2445	expand_arith_set_overflow (lhs, target);
2446
2447	/* We're done. */
2448	emit_label (done_label);
2449
2450	/* u1 * u2 -> sr */
2451	if (uns0_p && uns1_p && !unsr_p)
2452	{
2453	rtx_code_label *all_done_label = gen_label_rtx ();
2454	do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
2455	NULL, all_done_label, profile_probability::very_likely ());
2456	expand_arith_set_overflow (lhs, target);
2457	emit_label (all_done_label);
2458	}
2459
2460	/* s1 * u2 -> sr */
2461	if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
2462	{
2463	rtx_code_label *all_done_label = gen_label_rtx ();
2464	rtx_code_label *set_noovf = gen_label_rtx ();
2465	do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
2466	NULL, all_done_label, profile_probability::very_likely ());
2467	expand_arith_set_overflow (lhs, target);
2468	do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
2469	NULL, set_noovf, profile_probability::very_likely ());
2470	do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
2471	NULL, all_done_label, profile_probability::very_unlikely ());
2472	do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
2473	all_done_label, profile_probability::very_unlikely ());
2474	emit_label (set_noovf);
2475	write_complex_part (target, const0_rtx, true, false);
2476	emit_label (all_done_label);
2477	}
2478
2479	if (lhs)
2480	{
2481	if (is_ubsan)
2482	expand_ubsan_result_store (lhs, target, mode, res, do_error);
2483	else
2484	expand_arith_overflow_result_store (lhs, target, mode, res);
2485	}
2486	flag_trapv = save_flag_trapv;
2487	}
2488
2489	/* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2490
2491	static void
2492	expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs,
2493	tree arg0, tree arg1)
2494	{
2495	poly_uint64 cnt = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
2496	rtx_code_label *loop_lab = NULL;
2497	rtx cntvar = NULL_RTX;
2498	tree cntv = NULL_TREE;
2499	tree eltype = TREE_TYPE (TREE_TYPE (arg0));
2500	tree sz = TYPE_SIZE (eltype);
2501	tree data = NULL_TREE;
2502	tree resv = NULL_TREE;
2503	rtx lhsr = NULL_RTX;
2504	rtx resvr = NULL_RTX;
2505	unsigned HOST_WIDE_INT const_cnt = 0;
2506	bool use_loop_p = (!cnt.is_constant (const_value: &const_cnt) || const_cnt > 4);
2507	int save_flag_trapv = flag_trapv;
2508
2509	/* We don't want any __mulv?i3 etc. calls from the expansion of
2510	these internal functions, so disable -ftrapv temporarily. */
2511	flag_trapv = 0;
2512	if (lhs)
2513	{
2514	optab op;
2515	lhsr = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
2516	if (!VECTOR_MODE_P (GET_MODE (lhsr))
2517	|| (op = optab_for_tree_code (code, TREE_TYPE (arg0),
2518	optab_default)) == unknown_optab
2519	|| (optab_handler (op, TYPE_MODE (TREE_TYPE (arg0)))
2520	== CODE_FOR_nothing))
2521	{
2522	if (MEM_P (lhsr))
2523	resv = make_tree (TREE_TYPE (lhs), lhsr);
2524	else
2525	{
2526	resvr = assign_temp (TREE_TYPE (lhs), 1, 1);
2527	resv = make_tree (TREE_TYPE (lhs), resvr);
2528	}
2529	}
2530	}
2531	if (use_loop_p)
2532	{
2533	do_pending_stack_adjust ();
2534	loop_lab = gen_label_rtx ();
2535	cntvar = gen_reg_rtx (TYPE_MODE (sizetype));
2536	cntv = make_tree (sizetype, cntvar);
2537	emit_move_insn (cntvar, const0_rtx);
2538	emit_label (loop_lab);
2539	}
2540	if (TREE_CODE (arg0) != VECTOR_CST)
2541	{
2542	rtx arg0r = expand_normal (exp: arg0);
2543	arg0 = make_tree (TREE_TYPE (arg0), arg0r);
2544	}
2545	if (TREE_CODE (arg1) != VECTOR_CST)
2546	{
2547	rtx arg1r = expand_normal (exp: arg1);
2548	arg1 = make_tree (TREE_TYPE (arg1), arg1r);
2549	}
2550	for (unsigned int i = 0; i < (use_loop_p ? 1 : const_cnt); i++)
2551	{
2552	tree op0, op1, res = NULL_TREE;
2553	if (use_loop_p)
2554	{
2555	tree atype = build_array_type_nelts (eltype, cnt);
2556	op0 = uniform_vector_p (arg0);
2557	if (op0 == NULL_TREE)
2558	{
2559	op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg0);
2560	op0 = build4_loc (loc, code: ARRAY_REF, type: eltype, arg0: op0, arg1: cntv,
2561	NULL_TREE, NULL_TREE);
2562	}
2563	op1 = uniform_vector_p (arg1);
2564	if (op1 == NULL_TREE)
2565	{
2566	op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg1);
2567	op1 = build4_loc (loc, code: ARRAY_REF, type: eltype, arg0: op1, arg1: cntv,
2568	NULL_TREE, NULL_TREE);
2569	}
2570	if (resv)
2571	{
2572	res = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, resv);
2573	res = build4_loc (loc, code: ARRAY_REF, type: eltype, arg0: res, arg1: cntv,
2574	NULL_TREE, NULL_TREE);
2575	}
2576	}
2577	else
2578	{
2579	tree bitpos = bitsize_int (tree_to_uhwi (sz) * i);
2580	op0 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg0, sz, bitpos);
2581	op1 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg1, sz, bitpos);
2582	if (resv)
2583	res = fold_build3_loc (loc, BIT_FIELD_REF, eltype, resv, sz,
2584	bitpos);
2585	}
2586	switch (code)
2587	{
2588	case PLUS_EXPR:
2589	expand_addsub_overflow (loc, code: PLUS_EXPR, lhs: res, arg0: op0, arg1: op1,
2590	unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true, datap: &data);
2591	break;
2592	case MINUS_EXPR:
2593	if (use_loop_p ? integer_zerop (arg0) : integer_zerop (op0))
2594	expand_neg_overflow (loc, lhs: res, arg1: op1, is_ubsan: true, datap: &data);
2595	else
2596	expand_addsub_overflow (loc, code: MINUS_EXPR, lhs: res, arg0: op0, arg1: op1,
2597	unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true, datap: &data);
2598	break;
2599	case MULT_EXPR:
2600	expand_mul_overflow (loc, lhs: res, arg0: op0, arg1: op1, unsr_p: false, uns0_p: false, uns1_p: false,
2601	is_ubsan: true, datap: &data);
2602	break;
2603	default:
2604	gcc_unreachable ();
2605	}
2606	}
2607	if (use_loop_p)
2608	{
2609	struct separate_ops ops;
2610	ops.code = PLUS_EXPR;
2611	ops.type = TREE_TYPE (cntv);
2612	ops.op0 = cntv;
2613	ops.op1 = build_int_cst (TREE_TYPE (cntv), 1);
2614	ops.op2 = NULL_TREE;
2615	ops.location = loc;
2616	rtx ret = expand_expr_real_2 (&ops, cntvar, TYPE_MODE (sizetype),
2617	EXPAND_NORMAL);
2618	if (ret != cntvar)
2619	emit_move_insn (cntvar, ret);
2620	rtx cntrtx = gen_int_mode (cnt, TYPE_MODE (sizetype));
2621	do_compare_rtx_and_jump (cntvar, cntrtx, NE, false,
2622	TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
2623	profile_probability::very_likely ());
2624	}
2625	if (lhs && resv == NULL_TREE)
2626	{
2627	struct separate_ops ops;
2628	ops.code = code;
2629	ops.type = TREE_TYPE (arg0);
2630	ops.op0 = arg0;
2631	ops.op1 = arg1;
2632	ops.op2 = NULL_TREE;
2633	ops.location = loc;
2634	rtx ret = expand_expr_real_2 (&ops, lhsr, TYPE_MODE (TREE_TYPE (arg0)),
2635	EXPAND_NORMAL);
2636	if (ret != lhsr)
2637	emit_move_insn (lhsr, ret);
2638	}
2639	else if (resvr)
2640	emit_move_insn (lhsr, resvr);
2641	flag_trapv = save_flag_trapv;
2642	}
2643
2644	/* Expand UBSAN_CHECK_ADD call STMT. */
2645
2646	static void
2647	expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
2648	{
2649	location_t loc = gimple_location (g: stmt);
2650	tree lhs = gimple_call_lhs (gs: stmt);
2651	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
2652	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
2653	if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2654	expand_vector_ubsan_overflow (loc, code: PLUS_EXPR, lhs, arg0, arg1);
2655	else
2656	expand_addsub_overflow (loc, code: PLUS_EXPR, lhs, arg0, arg1,
2657	unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true, NULL);
2658	}
2659
2660	/* Expand UBSAN_CHECK_SUB call STMT. */
2661
2662	static void
2663	expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
2664	{
2665	location_t loc = gimple_location (g: stmt);
2666	tree lhs = gimple_call_lhs (gs: stmt);
2667	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
2668	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
2669	if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2670	expand_vector_ubsan_overflow (loc, code: MINUS_EXPR, lhs, arg0, arg1);
2671	else if (integer_zerop (arg0))
2672	expand_neg_overflow (loc, lhs, arg1, is_ubsan: true, NULL);
2673	else
2674	expand_addsub_overflow (loc, code: MINUS_EXPR, lhs, arg0, arg1,
2675	unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true, NULL);
2676	}
2677
2678	/* Expand UBSAN_CHECK_MUL call STMT. */
2679
2680	static void
2681	expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
2682	{
2683	location_t loc = gimple_location (g: stmt);
2684	tree lhs = gimple_call_lhs (gs: stmt);
2685	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
2686	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
2687	if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2688	expand_vector_ubsan_overflow (loc, code: MULT_EXPR, lhs, arg0, arg1);
2689	else
2690	expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true,
2691	NULL);
2692	}
2693
2694	/* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2695
2696	static void
2697	expand_arith_overflow (enum tree_code code, gimple *stmt)
2698	{
2699	tree lhs = gimple_call_lhs (gs: stmt);
2700	if (lhs == NULL_TREE)
2701	return;
2702	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
2703	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
2704	tree type = TREE_TYPE (TREE_TYPE (lhs));
2705	int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
2706	int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
2707	int unsr_p = TYPE_UNSIGNED (type);
2708	int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
2709	int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
2710	int precres = TYPE_PRECISION (type);
2711	location_t loc = gimple_location (g: stmt);
2712	if (!uns0_p && get_range_pos_neg (arg0) == 1)
2713	uns0_p = true;
2714	if (!uns1_p && get_range_pos_neg (arg1) == 1)
2715	uns1_p = true;
2716	int pr = get_min_precision (arg: arg0, sign: uns0_p ? UNSIGNED : SIGNED);
2717	prec0 = MIN (prec0, pr);
2718	pr = get_min_precision (arg: arg1, sign: uns1_p ? UNSIGNED : SIGNED);
2719	prec1 = MIN (prec1, pr);
2720	int save_flag_trapv = flag_trapv;
2721
2722	/* We don't want any __mulv?i3 etc. calls from the expansion of
2723	these internal functions, so disable -ftrapv temporarily. */
2724	flag_trapv = 0;
2725	/* If uns0_p && uns1_p, precop is minimum needed precision
2726	of unsigned type to hold the exact result, otherwise
2727	precop is minimum needed precision of signed type to
2728	hold the exact result. */
2729	int precop;
2730	if (code == MULT_EXPR)
2731	precop = prec0 + prec1 + (uns0_p != uns1_p);
2732	else
2733	{
2734	if (uns0_p == uns1_p)
2735	precop = MAX (prec0, prec1) + 1;
2736	else if (uns0_p)
2737	precop = MAX (prec0 + 1, prec1) + 1;
2738	else
2739	precop = MAX (prec0, prec1 + 1) + 1;
2740	}
2741	int orig_precres = precres;
2742
2743	do
2744	{
2745	if ((uns0_p && uns1_p)
2746	? ((precop + !unsr_p) <= precres
2747	/* u1 - u2 -> ur can overflow, no matter what precision
2748	the result has. */
2749	&& (code != MINUS_EXPR || !unsr_p))
2750	: (!unsr_p && precop <= precres))
2751	{
2752	/* The infinity precision result will always fit into result. */
2753	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
2754	write_complex_part (target, const0_rtx, true, false);
2755	scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
2756	struct separate_ops ops;
2757	ops.code = code;
2758	ops.type = type;
2759	ops.op0 = fold_convert_loc (loc, type, arg0);
2760	ops.op1 = fold_convert_loc (loc, type, arg1);
2761	ops.op2 = NULL_TREE;
2762	ops.location = loc;
2763	rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2764	expand_arith_overflow_result_store (lhs, target, mode, res: tem);
2765	flag_trapv = save_flag_trapv;
2766	return;
2767	}
2768
2769	/* For operations with low precision, if target doesn't have them, start
2770	with precres widening right away, otherwise do it only if the most
2771	simple cases can't be used. */
2772	const int min_precision = targetm.min_arithmetic_precision ();
2773	if (orig_precres == precres && precres < min_precision)
2774	;
2775	else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
2776	&& prec1 <= precres)
2777	|| ((!uns0_p || !uns1_p) && !unsr_p
2778	&& prec0 + uns0_p <= precres
2779	&& prec1 + uns1_p <= precres))
2780	{
2781	arg0 = fold_convert_loc (loc, type, arg0);
2782	arg1 = fold_convert_loc (loc, type, arg1);
2783	switch (code)
2784	{
2785	case MINUS_EXPR:
2786	if (integer_zerop (arg0) && !unsr_p)
2787	{
2788	expand_neg_overflow (loc, lhs, arg1, is_ubsan: false, NULL);
2789	flag_trapv = save_flag_trapv;
2790	return;
2791	}
2792	/* FALLTHRU */
2793	case PLUS_EXPR:
2794	expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2795	uns0_p: unsr_p, uns1_p: unsr_p, is_ubsan: false, NULL);
2796	flag_trapv = save_flag_trapv;
2797	return;
2798	case MULT_EXPR:
2799	expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2800	uns0_p: unsr_p, uns1_p: unsr_p, is_ubsan: false, NULL);
2801	flag_trapv = save_flag_trapv;
2802	return;
2803	default:
2804	gcc_unreachable ();
2805	}
2806	}
2807
2808	/* For sub-word operations, retry with a wider type first. */
2809	if (orig_precres == precres && precop <= BITS_PER_WORD)
2810	{
2811	int p = MAX (min_precision, precop);
2812	scalar_int_mode m = smallest_int_mode_for_size (size: p);
2813	tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (mode: m),
2814	uns0_p && uns1_p
2815	&& unsr_p);
2816	p = TYPE_PRECISION (optype);
2817	if (p > precres)
2818	{
2819	precres = p;
2820	unsr_p = TYPE_UNSIGNED (optype);
2821	type = optype;
2822	continue;
2823	}
2824	}
2825
2826	if (prec0 <= precres && prec1 <= precres)
2827	{
2828	tree types[2];
2829	if (unsr_p)
2830	{
2831	types[0] = build_nonstandard_integer_type (precres, 0);
2832	types[1] = type;
2833	}
2834	else
2835	{
2836	types[0] = type;
2837	types[1] = build_nonstandard_integer_type (precres, 1);
2838	}
2839	arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
2840	arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
2841	if (code != MULT_EXPR)
2842	expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2843	uns0_p, uns1_p, is_ubsan: false, NULL);
2844	else
2845	expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2846	uns0_p, uns1_p, is_ubsan: false, NULL);
2847	flag_trapv = save_flag_trapv;
2848	return;
2849	}
2850
2851	/* Retry with a wider type. */
2852	if (orig_precres == precres)
2853	{
2854	int p = MAX (prec0, prec1);
2855	scalar_int_mode m = smallest_int_mode_for_size (size: p);
2856	tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (mode: m),
2857	uns0_p && uns1_p
2858	&& unsr_p);
2859	p = TYPE_PRECISION (optype);
2860	if (p > precres)
2861	{
2862	precres = p;
2863	unsr_p = TYPE_UNSIGNED (optype);
2864	type = optype;
2865	continue;
2866	}
2867	}
2868
2869	gcc_unreachable ();
2870	}
2871	while (1);
2872	}
2873
2874	/* Expand ADD_OVERFLOW STMT. */
2875
2876	static void
2877	expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
2878	{
2879	expand_arith_overflow (code: PLUS_EXPR, stmt);
2880	}
2881
2882	/* Expand SUB_OVERFLOW STMT. */
2883
2884	static void
2885	expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
2886	{
2887	expand_arith_overflow (code: MINUS_EXPR, stmt);
2888	}
2889
2890	/* Expand MUL_OVERFLOW STMT. */
2891
2892	static void
2893	expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
2894	{
2895	expand_arith_overflow (code: MULT_EXPR, stmt);
2896	}
2897
2898	/* Expand UADDC STMT. */
2899
2900	static void
2901	expand_UADDC (internal_fn ifn, gcall *stmt)
2902	{
2903	tree lhs = gimple_call_lhs (gs: stmt);
2904	tree arg1 = gimple_call_arg (gs: stmt, index: 0);
2905	tree arg2 = gimple_call_arg (gs: stmt, index: 1);
2906	tree arg3 = gimple_call_arg (gs: stmt, index: 2);
2907	tree type = TREE_TYPE (arg1);
2908	machine_mode mode = TYPE_MODE (type);
2909	insn_code icode = optab_handler (op: ifn == IFN_UADDC
2910	? uaddc5_optab : usubc5_optab, mode);
2911	rtx op1 = expand_normal (exp: arg1);
2912	rtx op2 = expand_normal (exp: arg2);
2913	rtx op3 = expand_normal (exp: arg3);
2914	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
2915	rtx re = gen_reg_rtx (mode);
2916	rtx im = gen_reg_rtx (mode);
2917	class expand_operand ops[5];
2918	create_output_operand (op: &ops[0], x: re, mode);
2919	create_output_operand (op: &ops[1], x: im, mode);
2920	create_input_operand (op: &ops[2], value: op1, mode);
2921	create_input_operand (op: &ops[3], value: op2, mode);
2922	create_input_operand (op: &ops[4], value: op3, mode);
2923	expand_insn (icode, nops: 5, ops);
2924	write_complex_part (target, re, false, false);
2925	write_complex_part (target, im, true, false);
2926	}
2927
2928	/* Expand USUBC STMT. */
2929
2930	static void
2931	expand_USUBC (internal_fn ifn, gcall *stmt)
2932	{
2933	expand_UADDC (ifn, stmt);
2934	}
2935
2936	/* This should get folded in tree-vectorizer.cc. */
2937
2938	static void
2939	expand_LOOP_VECTORIZED (internal_fn, gcall *)
2940	{
2941	gcc_unreachable ();
2942	}
2943
2944	/* This should get folded in tree-vectorizer.cc. */
2945
2946	static void
2947	expand_LOOP_DIST_ALIAS (internal_fn, gcall *)
2948	{
2949	gcc_unreachable ();
2950	}
2951
2952	/* Return a memory reference of type TYPE for argument INDEX of STMT.
2953	Use argument INDEX + 1 to derive the second (TBAA) operand. */
2954
2955	static tree
2956	expand_call_mem_ref (tree type, gcall *stmt, int index)
2957	{
2958	tree addr = gimple_call_arg (gs: stmt, index);
2959	tree alias_ptr_type = TREE_TYPE (gimple_call_arg (stmt, index + 1));
2960	unsigned int align = tree_to_shwi (gimple_call_arg (gs: stmt, index: index + 1));
2961	if (TYPE_ALIGN (type) != align)
2962	type = build_aligned_type (type, align);
2963
2964	tree tmp = addr;
2965	if (TREE_CODE (tmp) == SSA_NAME)
2966	{
2967	gimple *def = SSA_NAME_DEF_STMT (tmp);
2968	if (gimple_assign_single_p (gs: def))
2969	tmp = gimple_assign_rhs1 (gs: def);
2970	}
2971
2972	if (TREE_CODE (tmp) == ADDR_EXPR)
2973	{
2974	tree mem = TREE_OPERAND (tmp, 0);
2975	if (TREE_CODE (mem) == TARGET_MEM_REF
2976	&& types_compatible_p (TREE_TYPE (mem), type2: type))
2977	{
2978	tree offset = TMR_OFFSET (mem);
2979	if (type != TREE_TYPE (mem)
2980	|| alias_ptr_type != TREE_TYPE (offset)
2981	|| !integer_zerop (offset))
2982	{
2983	mem = copy_node (mem);
2984	TMR_OFFSET (mem) = wide_int_to_tree (type: alias_ptr_type,
2985	cst: wi::to_poly_wide (t: offset));
2986	TREE_TYPE (mem) = type;
2987	}
2988	return mem;
2989	}
2990	}
2991
2992	return fold_build2 (MEM_REF, type, addr, build_int_cst (alias_ptr_type, 0));
2993	}
2994
2995	/* Expand MASK_LOAD{,_LANES}, MASK_LEN_LOAD or LEN_LOAD call STMT using optab
2996	* OPTAB. */
2997
2998	static void
2999	expand_partial_load_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
3000	{
3001	int i = 0;
3002	class expand_operand ops[5];
3003	tree type, lhs, rhs, maskt;
3004	rtx mem, target;
3005	insn_code icode;
3006
3007	maskt = gimple_call_arg (gs: stmt, index: internal_fn_mask_index (ifn));
3008	lhs = gimple_call_lhs (gs: stmt);
3009	if (lhs == NULL_TREE)
3010	return;
3011	type = TREE_TYPE (lhs);
3012	rhs = expand_call_mem_ref (type, stmt, index: 0);
3013
3014	if (optab == vec_mask_load_lanes_optab
3015	|| optab == vec_mask_len_load_lanes_optab)
3016	icode = get_multi_vector_move (array_type: type, optab);
3017	else if (optab == len_load_optab)
3018	icode = direct_optab_handler (op: optab, TYPE_MODE (type));
3019	else
3020	icode = convert_optab_handler (op: optab, TYPE_MODE (type),
3021	TYPE_MODE (TREE_TYPE (maskt)));
3022
3023	mem = expand_expr (exp: rhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3024	gcc_assert (MEM_P (mem));
3025	/* The built MEM_REF does not accurately reflect that the load
3026	is only partial. Clear it. */
3027	set_mem_expr (mem, NULL_TREE);
3028	clear_mem_offset (mem);
3029	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3030	create_output_operand (op: &ops[i++], x: target, TYPE_MODE (type));
3031	create_fixed_operand (op: &ops[i++], x: mem);
3032	i = add_mask_and_len_args (ops, opno: i, stmt);
3033	expand_insn (icode, nops: i, ops);
3034
3035	if (!rtx_equal_p (target, ops[0].value))
3036	emit_move_insn (target, ops[0].value);
3037	}
3038
3039	#define expand_mask_load_optab_fn expand_partial_load_optab_fn
3040	#define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
3041	#define expand_len_load_optab_fn expand_partial_load_optab_fn
3042	#define expand_mask_len_load_optab_fn expand_partial_load_optab_fn
3043
3044	/* Expand MASK_STORE{,_LANES}, MASK_LEN_STORE or LEN_STORE call STMT using optab
3045	* OPTAB. */
3046
3047	static void
3048	expand_partial_store_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
3049	{
3050	int i = 0;
3051	class expand_operand ops[5];
3052	tree type, lhs, rhs, maskt;
3053	rtx mem, reg;
3054	insn_code icode;
3055
3056	maskt = gimple_call_arg (gs: stmt, index: internal_fn_mask_index (ifn));
3057	rhs = gimple_call_arg (gs: stmt, index: internal_fn_stored_value_index (ifn));
3058	type = TREE_TYPE (rhs);
3059	lhs = expand_call_mem_ref (type, stmt, index: 0);
3060
3061	if (optab == vec_mask_store_lanes_optab
3062	|| optab == vec_mask_len_store_lanes_optab)
3063	icode = get_multi_vector_move (array_type: type, optab);
3064	else if (optab == len_store_optab)
3065	icode = direct_optab_handler (op: optab, TYPE_MODE (type));
3066	else
3067	icode = convert_optab_handler (op: optab, TYPE_MODE (type),
3068	TYPE_MODE (TREE_TYPE (maskt)));
3069
3070	mem = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3071	gcc_assert (MEM_P (mem));
3072	/* The built MEM_REF does not accurately reflect that the store
3073	is only partial. Clear it. */
3074	set_mem_expr (mem, NULL_TREE);
3075	clear_mem_offset (mem);
3076	reg = expand_normal (exp: rhs);
3077	create_fixed_operand (op: &ops[i++], x: mem);
3078	create_input_operand (op: &ops[i++], value: reg, TYPE_MODE (type));
3079	i = add_mask_and_len_args (ops, opno: i, stmt);
3080	expand_insn (icode, nops: i, ops);
3081	}
3082
3083	#define expand_mask_store_optab_fn expand_partial_store_optab_fn
3084	#define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
3085	#define expand_len_store_optab_fn expand_partial_store_optab_fn
3086	#define expand_mask_len_store_optab_fn expand_partial_store_optab_fn
3087
3088	/* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
3089	The expansion of STMT happens based on OPTAB table associated. */
3090
3091	static void
3092	expand_vec_cond_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3093	{
3094	class expand_operand ops[6];
3095	insn_code icode;
3096	tree lhs = gimple_call_lhs (gs: stmt);
3097	tree op0a = gimple_call_arg (gs: stmt, index: 0);
3098	tree op0b = gimple_call_arg (gs: stmt, index: 1);
3099	tree op1 = gimple_call_arg (gs: stmt, index: 2);
3100	tree op2 = gimple_call_arg (gs: stmt, index: 3);
3101	enum tree_code tcode = (tree_code) int_cst_value (gimple_call_arg (gs: stmt, index: 4));
3102
3103	tree vec_cond_type = TREE_TYPE (lhs);
3104	tree op_mode = TREE_TYPE (op0a);
3105	bool unsignedp = TYPE_UNSIGNED (op_mode);
3106
3107	machine_mode mode = TYPE_MODE (vec_cond_type);
3108	machine_mode cmp_op_mode = TYPE_MODE (op_mode);
3109
3110	icode = convert_optab_handler (op: optab, to_mode: mode, from_mode: cmp_op_mode);
3111	rtx comparison
3112	= vector_compare_rtx (VOIDmode, tcode, t_op0: op0a, t_op1: op0b, unsignedp, icode, opno: 4);
3113	/* vector_compare_rtx legitimizes operands, preserve equality when
3114	expanding op1/op2. */
3115	rtx rtx_op1, rtx_op2;
3116	if (operand_equal_p (op1, op0a))
3117	rtx_op1 = XEXP (comparison, 0);
3118	else if (operand_equal_p (op1, op0b))
3119	rtx_op1 = XEXP (comparison, 1);
3120	else
3121	rtx_op1 = expand_normal (exp: op1);
3122	if (operand_equal_p (op2, op0a))
3123	rtx_op2 = XEXP (comparison, 0);
3124	else if (operand_equal_p (op2, op0b))
3125	rtx_op2 = XEXP (comparison, 1);
3126	else
3127	rtx_op2 = expand_normal (exp: op2);
3128
3129	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3130	create_output_operand (op: &ops[0], x: target, mode);
3131	create_input_operand (op: &ops[1], value: rtx_op1, mode);
3132	create_input_operand (op: &ops[2], value: rtx_op2, mode);
3133	create_fixed_operand (op: &ops[3], x: comparison);
3134	create_fixed_operand (op: &ops[4], XEXP (comparison, 0));
3135	create_fixed_operand (op: &ops[5], XEXP (comparison, 1));
3136	expand_insn (icode, nops: 6, ops);
3137	if (!rtx_equal_p (ops[0].value, target))
3138	emit_move_insn (target, ops[0].value);
3139	}
3140
3141	/* Expand VCOND_MASK optab internal function.
3142	The expansion of STMT happens based on OPTAB table associated. */
3143
3144	static void
3145	expand_vec_cond_mask_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3146	{
3147	class expand_operand ops[4];
3148
3149	tree lhs = gimple_call_lhs (gs: stmt);
3150	tree op0 = gimple_call_arg (gs: stmt, index: 0);
3151	tree op1 = gimple_call_arg (gs: stmt, index: 1);
3152	tree op2 = gimple_call_arg (gs: stmt, index: 2);
3153	tree vec_cond_type = TREE_TYPE (lhs);
3154
3155	machine_mode mode = TYPE_MODE (vec_cond_type);
3156	machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
3157	enum insn_code icode = convert_optab_handler (op: optab, to_mode: mode, from_mode: mask_mode);
3158	rtx mask, rtx_op1, rtx_op2;
3159
3160	gcc_assert (icode != CODE_FOR_nothing);
3161
3162	mask = expand_normal (exp: op0);
3163	rtx_op1 = expand_normal (exp: op1);
3164	rtx_op2 = expand_normal (exp: op2);
3165
3166	mask = force_reg (mask_mode, mask);
3167	rtx_op1 = force_reg (mode, rtx_op1);
3168
3169	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3170	create_output_operand (op: &ops[0], x: target, mode);
3171	create_input_operand (op: &ops[1], value: rtx_op1, mode);
3172	create_input_operand (op: &ops[2], value: rtx_op2, mode);
3173	create_input_operand (op: &ops[3], value: mask, mode: mask_mode);
3174	expand_insn (icode, nops: 4, ops);
3175	if (!rtx_equal_p (ops[0].value, target))
3176	emit_move_insn (target, ops[0].value);
3177	}
3178
3179	/* Expand VEC_SET internal functions. */
3180
3181	static void
3182	expand_vec_set_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3183	{
3184	tree lhs = gimple_call_lhs (gs: stmt);
3185	tree op0 = gimple_call_arg (gs: stmt, index: 0);
3186	tree op1 = gimple_call_arg (gs: stmt, index: 1);
3187	tree op2 = gimple_call_arg (gs: stmt, index: 2);
3188	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3189	rtx src = expand_normal (exp: op0);
3190
3191	machine_mode outermode = TYPE_MODE (TREE_TYPE (op0));
3192	scalar_mode innermode = GET_MODE_INNER (outermode);
3193
3194	rtx value = expand_normal (exp: op1);
3195	rtx pos = expand_normal (exp: op2);
3196
3197	class expand_operand ops[3];
3198	enum insn_code icode = optab_handler (op: optab, mode: outermode);
3199
3200	if (icode != CODE_FOR_nothing)
3201	{
3202	rtx temp = gen_reg_rtx (outermode);
3203	emit_move_insn (temp, src);
3204
3205	create_fixed_operand (op: &ops[0], x: temp);
3206	create_input_operand (op: &ops[1], value, mode: innermode);
3207	create_convert_operand_from (op: &ops[2], value: pos, TYPE_MODE (TREE_TYPE (op2)),
3208	unsigned_p: true);
3209	if (maybe_expand_insn (icode, nops: 3, ops))
3210	{
3211	emit_move_insn (target, temp);
3212	return;
3213	}
3214	}
3215	gcc_unreachable ();
3216	}
3217
3218	static void
3219	expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
3220	{
3221	}
3222
3223	static void
3224	expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
3225	{
3226	/* When guessing was done, the hints should be already stripped away. */
3227	gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
3228
3229	rtx target;
3230	tree lhs = gimple_call_lhs (gs: stmt);
3231	if (lhs)
3232	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3233	else
3234	target = const0_rtx;
3235	rtx val = expand_expr (exp: gimple_call_arg (gs: stmt, index: 0), target, VOIDmode, modifier: EXPAND_NORMAL);
3236	if (lhs && val != target)
3237	emit_move_insn (target, val);
3238	}
3239
3240	/* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
3241	should never be called. */
3242
3243	static void
3244	expand_VA_ARG (internal_fn, gcall *)
3245	{
3246	gcc_unreachable ();
3247	}
3248
3249	/* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
3250	dummy function should never be called. */
3251
3252	static void
3253	expand_VEC_CONVERT (internal_fn, gcall *)
3254	{
3255	gcc_unreachable ();
3256	}
3257
3258	/* Expand IFN_RAWMEMCHR internal function. */
3259
3260	void
3261	expand_RAWMEMCHR (internal_fn, gcall *stmt)
3262	{
3263	expand_operand ops[3];
3264
3265	tree lhs = gimple_call_lhs (gs: stmt);
3266	if (!lhs)
3267	return;
3268	machine_mode lhs_mode = TYPE_MODE (TREE_TYPE (lhs));
3269	rtx lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3270	create_output_operand (op: &ops[0], x: lhs_rtx, mode: lhs_mode);
3271
3272	tree mem = gimple_call_arg (gs: stmt, index: 0);
3273	rtx mem_rtx = get_memory_rtx (exp: mem, NULL);
3274	create_fixed_operand (op: &ops[1], x: mem_rtx);
3275
3276	tree pattern = gimple_call_arg (gs: stmt, index: 1);
3277	machine_mode mode = TYPE_MODE (TREE_TYPE (pattern));
3278	rtx pattern_rtx = expand_normal (exp: pattern);
3279	create_input_operand (op: &ops[2], value: pattern_rtx, mode);
3280
3281	insn_code icode = direct_optab_handler (op: rawmemchr_optab, mode);
3282
3283	expand_insn (icode, nops: 3, ops);
3284	if (!rtx_equal_p (lhs_rtx, ops[0].value))
3285	emit_move_insn (lhs_rtx, ops[0].value);
3286	}
3287
3288	/* Expand the IFN_UNIQUE function according to its first argument. */
3289
3290	static void
3291	expand_UNIQUE (internal_fn, gcall *stmt)
3292	{
3293	rtx pattern = NULL_RTX;
3294	enum ifn_unique_kind kind
3295	= (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
3296
3297	switch (kind)
3298	{
3299	default:
3300	gcc_unreachable ();
3301
3302	case IFN_UNIQUE_UNSPEC:
3303	if (targetm.have_unique ())
3304	pattern = targetm.gen_unique ();
3305	break;
3306
3307	case IFN_UNIQUE_OACC_FORK:
3308	case IFN_UNIQUE_OACC_JOIN:
3309	if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
3310	{
3311	tree lhs = gimple_call_lhs (gs: stmt);
3312	rtx target = const0_rtx;
3313
3314	if (lhs)
3315	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3316
3317	rtx data_dep = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
3318	rtx axis = expand_normal (exp: gimple_call_arg (gs: stmt, index: 2));
3319
3320	if (kind == IFN_UNIQUE_OACC_FORK)
3321	pattern = targetm.gen_oacc_fork (target, data_dep, axis);
3322	else
3323	pattern = targetm.gen_oacc_join (target, data_dep, axis);
3324	}
3325	else
3326	gcc_unreachable ();
3327	break;
3328	}
3329
3330	if (pattern)
3331	emit_insn (pattern);
3332	}
3333
3334	/* Expand the IFN_DEFERRED_INIT function:
3335	LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3336
3337	Initialize the LHS with zero/pattern according to its second argument
3338	INIT_TYPE:
3339	if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3340	if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3341	to initialize;
3342	The LHS variable is initialized including paddings.
3343	The reasons to choose 0xFE for pattern initialization are:
3344	1. It is a non-canonical virtual address on x86_64, and at the
3345	high end of the i386 kernel address space.
3346	2. It is a very large float value (-1.694739530317379e+38).
3347	3. It is also an unusual number for integers. */
3348	#define INIT_PATTERN_VALUE 0xFE
3349	static void
3350	expand_DEFERRED_INIT (internal_fn, gcall *stmt)
3351	{
3352	tree lhs = gimple_call_lhs (gs: stmt);
3353	tree var_size = gimple_call_arg (gs: stmt, index: 0);
3354	enum auto_init_type init_type
3355	= (enum auto_init_type) TREE_INT_CST_LOW (gimple_call_arg (stmt, 1));
3356	bool reg_lhs = true;
3357
3358	tree var_type = TREE_TYPE (lhs);
3359	gcc_assert (init_type > AUTO_INIT_UNINITIALIZED);
3360
3361	if (TREE_CODE (lhs) == SSA_NAME)
3362	reg_lhs = true;
3363	else
3364	{
3365	tree lhs_base = lhs;
3366	while (handled_component_p (t: lhs_base))
3367	lhs_base = TREE_OPERAND (lhs_base, 0);
3368	reg_lhs = (mem_ref_refers_to_non_mem_p (lhs_base)
3369	|| non_mem_decl_p (lhs_base));
3370	/* If this expands to a register and the underlying decl is wrapped in
3371	a MEM_REF that just serves as an access type change expose the decl
3372	if it is of correct size. This avoids a situation as in PR103271
3373	if the target does not support a direct move to the registers mode. */
3374	if (reg_lhs
3375	&& TREE_CODE (lhs_base) == MEM_REF
3376	&& TREE_CODE (TREE_OPERAND (lhs_base, 0)) == ADDR_EXPR
3377	&& DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))
3378	&& integer_zerop (TREE_OPERAND (lhs_base, 1))
3379	&& tree_fits_uhwi_p (var_size)
3380	&& tree_int_cst_equal
3381	(var_size,
3382	DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))))
3383	{
3384	lhs = TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0);
3385	var_type = TREE_TYPE (lhs);
3386	}
3387	}
3388
3389	if (!reg_lhs)
3390	{
3391	/* If the variable is not in register, expand to a memset
3392	to initialize it. */
3393	mark_addressable (lhs);
3394	tree var_addr = build_fold_addr_expr (lhs);
3395
3396	tree value = (init_type == AUTO_INIT_PATTERN)
3397	? build_int_cst (integer_type_node,
3398	INIT_PATTERN_VALUE)
3399	: integer_zero_node;
3400	tree m_call = build_call_expr (builtin_decl_implicit (fncode: BUILT_IN_MEMSET),
3401	3, var_addr, value, var_size);
3402	/* Expand this memset call. */
3403	expand_builtin_memset (m_call, NULL_RTX, TYPE_MODE (var_type));
3404	}
3405	else
3406	{
3407	/* If this variable is in a register use expand_assignment.
3408	For boolean scalars force zero-init. */
3409	tree init;
3410	scalar_int_mode var_mode;
3411	if (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE
3412	&& tree_fits_uhwi_p (var_size)
3413	&& (init_type == AUTO_INIT_PATTERN
3414	|| !is_gimple_reg_type (type: var_type))
3415	&& int_mode_for_size (size: tree_to_uhwi (var_size) * BITS_PER_UNIT,
3416	limit: 0).exists (mode: &var_mode)
3417	&& have_insn_for (SET, var_mode))
3418	{
3419	unsigned HOST_WIDE_INT total_bytes = tree_to_uhwi (var_size);
3420	unsigned char *buf = XALLOCAVEC (unsigned char, total_bytes);
3421	memset (s: buf, c: (init_type == AUTO_INIT_PATTERN
3422	? INIT_PATTERN_VALUE : 0), n: total_bytes);
3423	tree itype = build_nonstandard_integer_type
3424	(total_bytes * BITS_PER_UNIT, 1);
3425	wide_int w = wi::from_buffer (buf, total_bytes);
3426	init = wide_int_to_tree (type: itype, cst: w);
3427	/* Pun the LHS to make sure its type has constant size
3428	unless it is an SSA name where that's already known. */
3429	if (TREE_CODE (lhs) != SSA_NAME)
3430	lhs = build1 (VIEW_CONVERT_EXPR, itype, lhs);
3431	else
3432	init = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), init);
3433	}
3434	else
3435	/* Use zero-init also for variable-length sizes. */
3436	init = build_zero_cst (var_type);
3437
3438	expand_assignment (lhs, init, false);
3439	}
3440	}
3441
3442	/* The size of an OpenACC compute dimension. */
3443
3444	static void
3445	expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
3446	{
3447	tree lhs = gimple_call_lhs (gs: stmt);
3448
3449	if (!lhs)
3450	return;
3451
3452	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3453	if (targetm.have_oacc_dim_size ())
3454	{
3455	rtx dim = expand_expr (exp: gimple_call_arg (gs: stmt, index: 0), NULL_RTX,
3456	VOIDmode, modifier: EXPAND_NORMAL);
3457	emit_insn (targetm.gen_oacc_dim_size (target, dim));
3458	}
3459	else
3460	emit_move_insn (target, GEN_INT (1));
3461	}
3462
3463	/* The position of an OpenACC execution engine along one compute axis. */
3464
3465	static void
3466	expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
3467	{
3468	tree lhs = gimple_call_lhs (gs: stmt);
3469
3470	if (!lhs)
3471	return;
3472
3473	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3474	if (targetm.have_oacc_dim_pos ())
3475	{
3476	rtx dim = expand_expr (exp: gimple_call_arg (gs: stmt, index: 0), NULL_RTX,
3477	VOIDmode, modifier: EXPAND_NORMAL);
3478	emit_insn (targetm.gen_oacc_dim_pos (target, dim));
3479	}
3480	else
3481	emit_move_insn (target, const0_rtx);
3482	}
3483
3484	/* This is expanded by oacc_device_lower pass. */
3485
3486	static void
3487	expand_GOACC_LOOP (internal_fn, gcall *)
3488	{
3489	gcc_unreachable ();
3490	}
3491
3492	/* This is expanded by oacc_device_lower pass. */
3493
3494	static void
3495	expand_GOACC_REDUCTION (internal_fn, gcall *)
3496	{
3497	gcc_unreachable ();
3498	}
3499
3500	/* This is expanded by oacc_device_lower pass. */
3501
3502	static void
3503	expand_GOACC_TILE (internal_fn, gcall *)
3504	{
3505	gcc_unreachable ();
3506	}
3507
3508	/* Set errno to EDOM. */
3509
3510	static void
3511	expand_SET_EDOM (internal_fn, gcall *)
3512	{
3513	#ifdef TARGET_EDOM
3514	#ifdef GEN_ERRNO_RTX
3515	rtx errno_rtx = GEN_ERRNO_RTX;
3516	#else
3517	rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
3518	#endif
3519	emit_move_insn (errno_rtx,
3520	gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
3521	#else
3522	gcc_unreachable ();
3523	#endif
3524	}
3525
3526	/* Expand atomic bit test and set. */
3527
3528	static void
3529	expand_ATOMIC_BIT_TEST_AND_SET (internal_fn, gcall *call)
3530	{
3531	expand_ifn_atomic_bit_test_and (call);
3532	}
3533
3534	/* Expand atomic bit test and complement. */
3535
3536	static void
3537	expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn, gcall *call)
3538	{
3539	expand_ifn_atomic_bit_test_and (call);
3540	}
3541
3542	/* Expand atomic bit test and reset. */
3543
3544	static void
3545	expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn, gcall *call)
3546	{
3547	expand_ifn_atomic_bit_test_and (call);
3548	}
3549
3550	/* Expand atomic bit test and set. */
3551
3552	static void
3553	expand_ATOMIC_COMPARE_EXCHANGE (internal_fn, gcall *call)
3554	{
3555	expand_ifn_atomic_compare_exchange (call);
3556	}
3557
3558	/* Expand atomic add fetch and cmp with 0. */
3559
3560	static void
3561	expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn, gcall *call)
3562	{
3563	expand_ifn_atomic_op_fetch_cmp_0 (call);
3564	}
3565
3566	/* Expand atomic sub fetch and cmp with 0. */
3567
3568	static void
3569	expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn, gcall *call)
3570	{
3571	expand_ifn_atomic_op_fetch_cmp_0 (call);
3572	}
3573
3574	/* Expand atomic and fetch and cmp with 0. */
3575
3576	static void
3577	expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn, gcall *call)
3578	{
3579	expand_ifn_atomic_op_fetch_cmp_0 (call);
3580	}
3581
3582	/* Expand atomic or fetch and cmp with 0. */
3583
3584	static void
3585	expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn, gcall *call)
3586	{
3587	expand_ifn_atomic_op_fetch_cmp_0 (call);
3588	}
3589
3590	/* Expand atomic xor fetch and cmp with 0. */
3591
3592	static void
3593	expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn, gcall *call)
3594	{
3595	expand_ifn_atomic_op_fetch_cmp_0 (call);
3596	}
3597
3598	/* Expand LAUNDER to assignment, lhs = arg0. */
3599
3600	static void
3601	expand_LAUNDER (internal_fn, gcall *call)
3602	{
3603	tree lhs = gimple_call_lhs (gs: call);
3604
3605	if (!lhs)
3606	return;
3607
3608	expand_assignment (lhs, gimple_call_arg (gs: call, index: 0), false);
3609	}
3610
3611	/* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3612
3613	static void
3614	expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3615	{
3616	internal_fn ifn = gimple_call_internal_fn (gs: stmt);
3617	int rhs_index = internal_fn_stored_value_index (ifn);
3618	tree base = gimple_call_arg (gs: stmt, index: 0);
3619	tree offset = gimple_call_arg (gs: stmt, index: 1);
3620	tree scale = gimple_call_arg (gs: stmt, index: 2);
3621	tree rhs = gimple_call_arg (gs: stmt, index: rhs_index);
3622
3623	rtx base_rtx = expand_normal (exp: base);
3624	rtx offset_rtx = expand_normal (exp: offset);
3625	HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3626	rtx rhs_rtx = expand_normal (exp: rhs);
3627
3628	class expand_operand ops[8];
3629	int i = 0;
3630	create_address_operand (op: &ops[i++], value: base_rtx);
3631	create_input_operand (op: &ops[i++], value: offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3632	create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3633	create_integer_operand (&ops[i++], scale_int);
3634	create_input_operand (op: &ops[i++], value: rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
3635	i = add_mask_and_len_args (ops, opno: i, stmt);
3636
3637	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (TREE_TYPE (rhs)),
3638	TYPE_MODE (TREE_TYPE (offset)));
3639	expand_insn (icode, nops: i, ops);
3640	}
3641
3642	/* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3643
3644	static void
3645	expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3646	{
3647	tree lhs = gimple_call_lhs (gs: stmt);
3648	tree base = gimple_call_arg (gs: stmt, index: 0);
3649	tree offset = gimple_call_arg (gs: stmt, index: 1);
3650	tree scale = gimple_call_arg (gs: stmt, index: 2);
3651
3652	rtx lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3653	rtx base_rtx = expand_normal (exp: base);
3654	rtx offset_rtx = expand_normal (exp: offset);
3655	HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3656
3657	int i = 0;
3658	class expand_operand ops[8];
3659	create_output_operand (op: &ops[i++], x: lhs_rtx, TYPE_MODE (TREE_TYPE (lhs)));
3660	create_address_operand (op: &ops[i++], value: base_rtx);
3661	create_input_operand (op: &ops[i++], value: offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3662	create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3663	create_integer_operand (&ops[i++], scale_int);
3664	i = add_mask_and_len_args (ops, opno: i, stmt);
3665	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (TREE_TYPE (lhs)),
3666	TYPE_MODE (TREE_TYPE (offset)));
3667	expand_insn (icode, nops: i, ops);
3668	if (!rtx_equal_p (lhs_rtx, ops[0].value))
3669	emit_move_insn (lhs_rtx, ops[0].value);
3670	}
3671
3672	/* Helper for expand_DIVMOD. Return true if the sequence starting with
3673	INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3674
3675	static bool
3676	contains_call_div_mod (rtx_insn *insn)
3677	{
3678	subrtx_iterator::array_type array;
3679	for (; insn; insn = NEXT_INSN (insn))
3680	if (CALL_P (insn))
3681	return true;
3682	else if (INSN_P (insn))
3683	FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
3684	switch (GET_CODE (*iter))
3685	{
3686	case CALL:
3687	case DIV:
3688	case UDIV:
3689	case MOD:
3690	case UMOD:
3691	return true;
3692	default:
3693	break;
3694	}
3695	return false;
3696	}
3697
3698	/* Expand DIVMOD() using:
3699	a) optab handler for udivmod/sdivmod if it is available.
3700	b) If optab_handler doesn't exist, generate call to
3701	target-specific divmod libfunc. */
3702
3703	static void
3704	expand_DIVMOD (internal_fn, gcall *call_stmt)
3705	{
3706	tree lhs = gimple_call_lhs (gs: call_stmt);
3707	tree arg0 = gimple_call_arg (gs: call_stmt, index: 0);
3708	tree arg1 = gimple_call_arg (gs: call_stmt, index: 1);
3709
3710	gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
3711	tree type = TREE_TYPE (TREE_TYPE (lhs));
3712	machine_mode mode = TYPE_MODE (type);
3713	bool unsignedp = TYPE_UNSIGNED (type);
3714	optab tab = (unsignedp) ? udivmod_optab : sdivmod_optab;
3715
3716	rtx op0 = expand_normal (exp: arg0);
3717	rtx op1 = expand_normal (exp: arg1);
3718	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3719
3720	rtx quotient = NULL_RTX, remainder = NULL_RTX;
3721	rtx_insn *insns = NULL;
3722
3723	if (TREE_CODE (arg1) == INTEGER_CST)
3724	{
3725	/* For DIVMOD by integral constants, there could be efficient code
3726	expanded inline e.g. using shifts and plus/minus. Try to expand
3727	the division and modulo and if it emits any library calls or any
3728	{,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3729	divmod libcall. */
3730	scalar_int_mode int_mode;
3731	if (remainder == NULL_RTX
3732	&& optimize
3733	&& CONST_INT_P (op1)
3734	&& !pow2p_hwi (INTVAL (op1))
3735	&& is_int_mode (TYPE_MODE (type), int_mode: &int_mode)
3736	&& GET_MODE_SIZE (mode: int_mode) == 2 * UNITS_PER_WORD
3737	&& optab_handler (op: and_optab, mode: word_mode) != CODE_FOR_nothing
3738	&& optab_handler (op: add_optab, mode: word_mode) != CODE_FOR_nothing
3739	&& optimize_insn_for_speed_p ())
3740	{
3741	rtx_insn *last = get_last_insn ();
3742	remainder = NULL_RTX;
3743	quotient = expand_doubleword_divmod (int_mode, op0, op1, &remainder,
3744	TYPE_UNSIGNED (type));
3745	if (quotient != NULL_RTX)
3746	{
3747	if (optab_handler (op: mov_optab, mode: int_mode) != CODE_FOR_nothing)
3748	{
3749	rtx_insn *move = emit_move_insn (quotient, quotient);
3750	set_dst_reg_note (move, REG_EQUAL,
3751	gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3752	? UDIV : DIV, int_mode,
3753	copy_rtx (op0), op1),
3754	quotient);
3755	move = emit_move_insn (remainder, remainder);
3756	set_dst_reg_note (move, REG_EQUAL,
3757	gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3758	? UMOD : MOD, int_mode,
3759	copy_rtx (op0), op1),
3760	quotient);
3761	}
3762	}
3763	else
3764	delete_insns_since (last);
3765	}
3766
3767	if (remainder == NULL_RTX)
3768	{
3769	struct separate_ops ops;
3770	ops.code = TRUNC_DIV_EXPR;
3771	ops.type = type;
3772	ops.op0 = make_tree (ops.type, op0);
3773	ops.op1 = arg1;
3774	ops.op2 = NULL_TREE;
3775	ops.location = gimple_location (g: call_stmt);
3776	start_sequence ();
3777	quotient = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
3778	if (contains_call_div_mod (insn: get_insns ()))
3779	quotient = NULL_RTX;
3780	else
3781	{
3782	ops.code = TRUNC_MOD_EXPR;
3783	remainder = expand_expr_real_2 (&ops, NULL_RTX, mode,
3784	EXPAND_NORMAL);
3785	if (contains_call_div_mod (insn: get_insns ()))
3786	remainder = NULL_RTX;
3787	}
3788	if (remainder)
3789	insns = get_insns ();
3790	end_sequence ();
3791	}
3792	}
3793
3794	if (remainder)
3795	emit_insn (insns);
3796
3797	/* Check if optab_handler exists for divmod_optab for given mode. */
3798	else if (optab_handler (op: tab, mode) != CODE_FOR_nothing)
3799	{
3800	quotient = gen_reg_rtx (mode);
3801	remainder = gen_reg_rtx (mode);
3802	expand_twoval_binop (tab, op0, op1, quotient, remainder, unsignedp);
3803	}
3804
3805	/* Generate call to divmod libfunc if it exists. */
3806	else if (rtx libfunc = optab_libfunc (tab, mode))
3807	targetm.expand_divmod_libfunc (libfunc, mode, op0, op1,
3808	&quotient, &remainder);
3809
3810	else
3811	gcc_unreachable ();
3812
3813	/* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3814	expand_expr (exp: build2 (COMPLEX_EXPR, TREE_TYPE (lhs),
3815	make_tree (TREE_TYPE (arg0), quotient),
3816	make_tree (TREE_TYPE (arg1), remainder)),
3817	target, VOIDmode, modifier: EXPAND_NORMAL);
3818	}
3819
3820	/* Expand a NOP. */
3821
3822	static void
3823	expand_NOP (internal_fn, gcall *)
3824	{
3825	/* Nothing. But it shouldn't really prevail. */
3826	}
3827
3828	/* Coroutines, all should have been processed at this stage. */
3829
3830	static void
3831	expand_CO_FRAME (internal_fn, gcall *)
3832	{
3833	gcc_unreachable ();
3834	}
3835
3836	static void
3837	expand_CO_YIELD (internal_fn, gcall *)
3838	{
3839	gcc_unreachable ();
3840	}
3841
3842	static void
3843	expand_CO_SUSPN (internal_fn, gcall *)
3844	{
3845	gcc_unreachable ();
3846	}
3847
3848	static void
3849	expand_CO_ACTOR (internal_fn, gcall *)
3850	{
3851	gcc_unreachable ();
3852	}
3853
3854	/* Expand a call to FN using the operands in STMT. FN has a single
3855	output operand and NARGS input operands. */
3856
3857	static void
3858	expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
3859	unsigned int nargs)
3860	{
3861	tree_pair types = direct_internal_fn_types (fn, stmt);
3862	insn_code icode = direct_optab_handler (op: optab, TYPE_MODE (types.first));
3863	expand_fn_using_insn (stmt, icode, noutputs: 1, ninputs: nargs);
3864	}
3865
3866	/* Expand WHILE_ULT call STMT using optab OPTAB. */
3867
3868	static void
3869	expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3870	{
3871	expand_operand ops[4];
3872	tree rhs_type[2];
3873
3874	tree lhs = gimple_call_lhs (gs: stmt);
3875	tree lhs_type = TREE_TYPE (lhs);
3876	rtx lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3877	create_output_operand (op: &ops[0], x: lhs_rtx, TYPE_MODE (lhs_type));
3878
3879	for (unsigned int i = 0; i < 2; ++i)
3880	{
3881	tree rhs = gimple_call_arg (gs: stmt, index: i);
3882	rhs_type[i] = TREE_TYPE (rhs);
3883	rtx rhs_rtx = expand_normal (exp: rhs);
3884	create_input_operand (op: &ops[i + 1], value: rhs_rtx, TYPE_MODE (rhs_type[i]));
3885	}
3886
3887	int opcnt;
3888	if (!VECTOR_MODE_P (TYPE_MODE (lhs_type)))
3889	{
3890	/* When the mask is an integer mode the exact vector length may not
3891	be clear to the backend, so we pass it in operand[3].
3892	Use the vector in arg2 for the most reliable intended size. */
3893	tree type = TREE_TYPE (gimple_call_arg (stmt, 2));
3894	create_integer_operand (&ops[3], TYPE_VECTOR_SUBPARTS (node: type));
3895	opcnt = 4;
3896	}
3897	else
3898	/* The mask has a vector type so the length operand is unnecessary. */
3899	opcnt = 3;
3900
3901	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (rhs_type[0]),
3902	TYPE_MODE (lhs_type));
3903
3904	expand_insn (icode, nops: opcnt, ops);
3905	if (!rtx_equal_p (lhs_rtx, ops[0].value))
3906	emit_move_insn (lhs_rtx, ops[0].value);
3907	}
3908
3909	/* Expand a call to a convert-like optab using the operands in STMT.
3910	FN has a single output operand and NARGS input operands. */
3911
3912	static void
3913	expand_convert_optab_fn (internal_fn fn, gcall *stmt, convert_optab optab,
3914	unsigned int nargs)
3915	{
3916	tree_pair types = direct_internal_fn_types (fn, stmt);
3917	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (types.first),
3918	TYPE_MODE (types.second));
3919	expand_fn_using_insn (stmt, icode, noutputs: 1, ninputs: nargs);
3920	}
3921
3922	/* Expanders for optabs that can use expand_direct_optab_fn. */
3923
3924	#define expand_unary_optab_fn(FN, STMT, OPTAB) \
3925	expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3926
3927	#define expand_binary_optab_fn(FN, STMT, OPTAB) \
3928	expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3929
3930	#define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3931	expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3932
3933	#define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3934	expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3935
3936	#define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3937	expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3938
3939	#define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3940	expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3941
3942	#define expand_cond_len_unary_optab_fn(FN, STMT, OPTAB) \
3943	expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3944
3945	#define expand_cond_len_binary_optab_fn(FN, STMT, OPTAB) \
3946	expand_direct_optab_fn (FN, STMT, OPTAB, 6)
3947
3948	#define expand_cond_len_ternary_optab_fn(FN, STMT, OPTAB) \
3949	expand_direct_optab_fn (FN, STMT, OPTAB, 7)
3950
3951	#define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3952	expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3953
3954	#define expand_fold_len_extract_optab_fn(FN, STMT, OPTAB) \
3955	expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3956
3957	#define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3958	expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3959
3960	#define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3961	expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3962
3963	#define expand_mask_len_fold_left_optab_fn(FN, STMT, OPTAB) \
3964	expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3965
3966	#define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3967	expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3968
3969	/* Expanders for optabs that can use expand_convert_optab_fn. */
3970
3971	#define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
3972	expand_convert_optab_fn (FN, STMT, OPTAB, 1)
3973
3974	#define expand_vec_extract_optab_fn(FN, STMT, OPTAB) \
3975	expand_convert_optab_fn (FN, STMT, OPTAB, 2)
3976
3977	/* RETURN_TYPE and ARGS are a return type and argument list that are
3978	in principle compatible with FN (which satisfies direct_internal_fn_p).
3979	Return the types that should be used to determine whether the
3980	target supports FN. */
3981
3982	tree_pair
3983	direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
3984	{
3985	const direct_internal_fn_info &info = direct_internal_fn (fn);
3986	tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
3987	tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
3988	return tree_pair (type0, type1);
3989	}
3990
3991	/* CALL is a call whose return type and arguments are in principle
3992	compatible with FN (which satisfies direct_internal_fn_p). Return the
3993	types that should be used to determine whether the target supports FN. */
3994
3995	tree_pair
3996	direct_internal_fn_types (internal_fn fn, gcall *call)
3997	{
3998	const direct_internal_fn_info &info = direct_internal_fn (fn);
3999	tree op0 = (info.type0 < 0
4000	? gimple_call_lhs (gs: call)
4001	: gimple_call_arg (gs: call, index: info.type0));
4002	tree op1 = (info.type1 < 0
4003	? gimple_call_lhs (gs: call)
4004	: gimple_call_arg (gs: call, index: info.type1));
4005	return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
4006	}
4007
4008	/* Return true if OPTAB is supported for TYPES (whose modes should be
4009	the same) when the optimization type is OPT_TYPE. Used for simple
4010	direct optabs. */
4011
4012	static bool
4013	direct_optab_supported_p (direct_optab optab, tree_pair types,
4014	optimization_type opt_type)
4015	{
4016	machine_mode mode = TYPE_MODE (types.first);
4017	gcc_checking_assert (mode == TYPE_MODE (types.second));
4018	return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
4019	}
4020
4021	/* Return true if OPTAB is supported for TYPES, where the first type
4022	is the destination and the second type is the source. Used for
4023	convert optabs. */
4024
4025	static bool
4026	convert_optab_supported_p (convert_optab optab, tree_pair types,
4027	optimization_type opt_type)
4028	{
4029	return (convert_optab_handler (optab, TYPE_MODE (types.first),
4030	TYPE_MODE (types.second), opt_type)
4031	!= CODE_FOR_nothing);
4032	}
4033
4034	/* Return true if load/store lanes optab OPTAB is supported for
4035	array type TYPES.first when the optimization type is OPT_TYPE. */
4036
4037	static bool
4038	multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
4039	optimization_type opt_type)
4040	{
4041	gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
4042	machine_mode imode = TYPE_MODE (types.first);
4043	machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
4044	return (convert_optab_handler (optab, imode, vmode, opt_type)
4045	!= CODE_FOR_nothing);
4046	}
4047
4048	#define direct_unary_optab_supported_p direct_optab_supported_p
4049	#define direct_unary_convert_optab_supported_p convert_optab_supported_p
4050	#define direct_binary_optab_supported_p direct_optab_supported_p
4051	#define direct_ternary_optab_supported_p direct_optab_supported_p
4052	#define direct_cond_unary_optab_supported_p direct_optab_supported_p
4053	#define direct_cond_binary_optab_supported_p direct_optab_supported_p
4054	#define direct_cond_ternary_optab_supported_p direct_optab_supported_p
4055	#define direct_cond_len_unary_optab_supported_p direct_optab_supported_p
4056	#define direct_cond_len_binary_optab_supported_p direct_optab_supported_p
4057	#define direct_cond_len_ternary_optab_supported_p direct_optab_supported_p
4058	#define direct_mask_load_optab_supported_p convert_optab_supported_p
4059	#define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
4060	#define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
4061	#define direct_gather_load_optab_supported_p convert_optab_supported_p
4062	#define direct_len_load_optab_supported_p direct_optab_supported_p
4063	#define direct_mask_len_load_optab_supported_p convert_optab_supported_p
4064	#define direct_mask_store_optab_supported_p convert_optab_supported_p
4065	#define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
4066	#define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
4067	#define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
4068	#define direct_vec_cond_optab_supported_p convert_optab_supported_p
4069	#define direct_scatter_store_optab_supported_p convert_optab_supported_p
4070	#define direct_len_store_optab_supported_p direct_optab_supported_p
4071	#define direct_mask_len_store_optab_supported_p convert_optab_supported_p
4072	#define direct_while_optab_supported_p convert_optab_supported_p
4073	#define direct_fold_extract_optab_supported_p direct_optab_supported_p
4074	#define direct_fold_len_extract_optab_supported_p direct_optab_supported_p
4075	#define direct_fold_left_optab_supported_p direct_optab_supported_p
4076	#define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
4077	#define direct_mask_len_fold_left_optab_supported_p direct_optab_supported_p
4078	#define direct_check_ptrs_optab_supported_p direct_optab_supported_p
4079	#define direct_vec_set_optab_supported_p direct_optab_supported_p
4080	#define direct_vec_extract_optab_supported_p convert_optab_supported_p
4081
4082	/* Return the optab used by internal function FN. */
4083
4084	optab
4085	direct_internal_fn_optab (internal_fn fn, tree_pair types)
4086	{
4087	switch (fn)
4088	{
4089	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4090	case IFN_##CODE: break;
4091	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4092	case IFN_##CODE: return OPTAB##_optab;
4093	#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4094	UNSIGNED_OPTAB, TYPE) \
4095	case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
4096	? UNSIGNED_OPTAB ## _optab \
4097	: SIGNED_OPTAB ## _optab);
4098	#include "internal-fn.def"
4099
4100	case IFN_LAST:
4101	break;
4102	}
4103	gcc_unreachable ();
4104	}
4105
4106	/* Return the optab used by internal function FN. */
4107
4108	static optab
4109	direct_internal_fn_optab (internal_fn fn)
4110	{
4111	switch (fn)
4112	{
4113	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4114	case IFN_##CODE: break;
4115	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4116	case IFN_##CODE: return OPTAB##_optab;
4117	#include "internal-fn.def"
4118
4119	case IFN_LAST:
4120	break;
4121	}
4122	gcc_unreachable ();
4123	}
4124
4125	/* Return true if FN is supported for the types in TYPES when the
4126	optimization type is OPT_TYPE. The types are those associated with
4127	the "type0" and "type1" fields of FN's direct_internal_fn_info
4128	structure. */
4129
4130	bool
4131	direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
4132	optimization_type opt_type)
4133	{
4134	switch (fn)
4135	{
4136	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4137	case IFN_##CODE: break;
4138	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4139	case IFN_##CODE: \
4140	return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
4141	opt_type);
4142	#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4143	UNSIGNED_OPTAB, TYPE) \
4144	case IFN_##CODE: \
4145	{ \
4146	optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
4147	? UNSIGNED_OPTAB ## _optab \
4148	: SIGNED_OPTAB ## _optab); \
4149	return direct_##TYPE##_optab_supported_p (which_optab, types, \
4150	opt_type); \
4151	}
4152	#include "internal-fn.def"
4153
4154	case IFN_LAST:
4155	break;
4156	}
4157	gcc_unreachable ();
4158	}
4159
4160	/* Return true if FN is supported for type TYPE when the optimization
4161	type is OPT_TYPE. The caller knows that the "type0" and "type1"
4162	fields of FN's direct_internal_fn_info structure are the same. */
4163
4164	bool
4165	direct_internal_fn_supported_p (internal_fn fn, tree type,
4166	optimization_type opt_type)
4167	{
4168	const direct_internal_fn_info &info = direct_internal_fn (fn);
4169	gcc_checking_assert (info.type0 == info.type1);
4170	return direct_internal_fn_supported_p (fn, types: tree_pair (type, type), opt_type);
4171	}
4172
4173	/* Return true if the STMT is supported when the optimization type is OPT_TYPE,
4174	given that STMT is a call to a direct internal function. */
4175
4176	bool
4177	direct_internal_fn_supported_p (gcall *stmt, optimization_type opt_type)
4178	{
4179	internal_fn fn = gimple_call_internal_fn (gs: stmt);
4180	tree_pair types = direct_internal_fn_types (fn, call: stmt);
4181	return direct_internal_fn_supported_p (fn, types, opt_type);
4182	}
4183
4184	/* Return true if FN is a binary operation and if FN is commutative. */
4185
4186	bool
4187	commutative_binary_fn_p (internal_fn fn)
4188	{
4189	switch (fn)
4190	{
4191	case IFN_AVG_FLOOR:
4192	case IFN_AVG_CEIL:
4193	case IFN_MULH:
4194	case IFN_MULHS:
4195	case IFN_MULHRS:
4196	case IFN_FMIN:
4197	case IFN_FMAX:
4198	case IFN_COMPLEX_MUL:
4199	case IFN_UBSAN_CHECK_ADD:
4200	case IFN_UBSAN_CHECK_MUL:
4201	case IFN_ADD_OVERFLOW:
4202	case IFN_MUL_OVERFLOW:
4203	case IFN_VEC_WIDEN_PLUS:
4204	case IFN_VEC_WIDEN_PLUS_LO:
4205	case IFN_VEC_WIDEN_PLUS_HI:
4206	case IFN_VEC_WIDEN_PLUS_EVEN:
4207	case IFN_VEC_WIDEN_PLUS_ODD:
4208	return true;
4209
4210	default:
4211	return false;
4212	}
4213	}
4214
4215	/* Return true if FN is a ternary operation and if its first two arguments
4216	are commutative. */
4217
4218	bool
4219	commutative_ternary_fn_p (internal_fn fn)
4220	{
4221	switch (fn)
4222	{
4223	case IFN_FMA:
4224	case IFN_FMS:
4225	case IFN_FNMA:
4226	case IFN_FNMS:
4227	case IFN_UADDC:
4228	return true;
4229
4230	default:
4231	return false;
4232	}
4233	}
4234
4235	/* Return true if FN is an associative binary operation. */
4236
4237	bool
4238	associative_binary_fn_p (internal_fn fn)
4239	{
4240	switch (fn)
4241	{
4242	case IFN_FMIN:
4243	case IFN_FMAX:
4244	return true;
4245
4246	default:
4247	return false;
4248	}
4249	}
4250
4251	/* If FN is commutative in two consecutive arguments, return the
4252	index of the first, otherwise return -1. */
4253
4254	int
4255	first_commutative_argument (internal_fn fn)
4256	{
4257	switch (fn)
4258	{
4259	case IFN_COND_ADD:
4260	case IFN_COND_MUL:
4261	case IFN_COND_MIN:
4262	case IFN_COND_MAX:
4263	case IFN_COND_FMIN:
4264	case IFN_COND_FMAX:
4265	case IFN_COND_AND:
4266	case IFN_COND_IOR:
4267	case IFN_COND_XOR:
4268	case IFN_COND_FMA:
4269	case IFN_COND_FMS:
4270	case IFN_COND_FNMA:
4271	case IFN_COND_FNMS:
4272	case IFN_COND_LEN_ADD:
4273	case IFN_COND_LEN_MUL:
4274	case IFN_COND_LEN_MIN:
4275	case IFN_COND_LEN_MAX:
4276	case IFN_COND_LEN_FMIN:
4277	case IFN_COND_LEN_FMAX:
4278	case IFN_COND_LEN_AND:
4279	case IFN_COND_LEN_IOR:
4280	case IFN_COND_LEN_XOR:
4281	case IFN_COND_LEN_FMA:
4282	case IFN_COND_LEN_FMS:
4283	case IFN_COND_LEN_FNMA:
4284	case IFN_COND_LEN_FNMS:
4285	return 1;
4286
4287	default:
4288	if (commutative_binary_fn_p (fn)
4289	|| commutative_ternary_fn_p (fn))
4290	return 0;
4291	return -1;
4292	}
4293	}
4294
4295	/* Return true if this CODE describes an internal_fn that returns a vector with
4296	elements twice as wide as the element size of the input vectors. */
4297
4298	bool
4299	widening_fn_p (code_helper code)
4300	{
4301	if (!code.is_fn_code ())
4302	return false;
4303
4304	if (!internal_fn_p (code: (combined_fn) code))
4305	return false;
4306
4307	internal_fn fn = as_internal_fn (code: (combined_fn) code);
4308	switch (fn)
4309	{
4310	#define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
4311	case IFN_##NAME: \
4312	case IFN_##NAME##_HI: \
4313	case IFN_##NAME##_LO: \
4314	case IFN_##NAME##_EVEN: \
4315	case IFN_##NAME##_ODD: \
4316	return true;
4317	#include "internal-fn.def"
4318
4319	default:
4320	return false;
4321	}
4322	}
4323
4324	/* Return true if IFN_SET_EDOM is supported. */
4325
4326	bool
4327	set_edom_supported_p (void)
4328	{
4329	#ifdef TARGET_EDOM
4330	return true;
4331	#else
4332	return false;
4333	#endif
4334	}
4335
4336	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4337	static void \
4338	expand_##CODE (internal_fn fn, gcall *stmt) \
4339	{ \
4340	expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
4341	}
4342	#define DEF_INTERNAL_INT_EXT_FN(CODE, FLAGS, OPTAB, TYPE)
4343	#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4344	UNSIGNED_OPTAB, TYPE) \
4345	static void \
4346	expand_##CODE (internal_fn fn, gcall *stmt) \
4347	{ \
4348	tree_pair types = direct_internal_fn_types (fn, stmt); \
4349	optab which_optab = direct_internal_fn_optab (fn, types); \
4350	expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
4351	}
4352	#include "internal-fn.def"
4353
4354	/* Routines to expand each internal function, indexed by function number.
4355	Each routine has the prototype:
4356
4357	expand_<NAME> (gcall *stmt)
4358
4359	where STMT is the statement that performs the call. */
4360	static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
4361
4362	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4363	#include "internal-fn.def"
4364	0
4365	};
4366
4367	/* Invoke T(CODE, SUFFIX) for each conditional function IFN_COND_##SUFFIX
4368	that maps to a tree code CODE. There is also an IFN_COND_LEN_##SUFFIX
4369	for each such IFN_COND_##SUFFIX. */
4370	#define FOR_EACH_CODE_MAPPING(T) \
4371	T (PLUS_EXPR, ADD) \
4372	T (MINUS_EXPR, SUB) \
4373	T (MULT_EXPR, MUL) \
4374	T (TRUNC_DIV_EXPR, DIV) \
4375	T (TRUNC_MOD_EXPR, MOD) \
4376	T (RDIV_EXPR, RDIV) \
4377	T (MIN_EXPR, MIN) \
4378	T (MAX_EXPR, MAX) \
4379	T (BIT_AND_EXPR, AND) \
4380	T (BIT_IOR_EXPR, IOR) \
4381	T (BIT_XOR_EXPR, XOR) \
4382	T (LSHIFT_EXPR, SHL) \
4383	T (RSHIFT_EXPR, SHR) \
4384	T (NEGATE_EXPR, NEG)
4385
4386	/* Return a function that only performs CODE when a certain condition is met
4387	and that uses a given fallback value otherwise. For example, if CODE is
4388	a binary operation associated with conditional function FN:
4389
4390	LHS = FN (COND, A, B, ELSE)
4391
4392	is equivalent to the C expression:
4393
4394	LHS = COND ? A CODE B : ELSE;
4395
4396	operating elementwise if the operands are vectors.
4397
4398	Return IFN_LAST if no such function exists. */
4399
4400	internal_fn
4401	get_conditional_internal_fn (tree_code code)
4402	{
4403	switch (code)
4404	{
4405	#define CASE(CODE, IFN) case CODE: return IFN_COND_##IFN;
4406	FOR_EACH_CODE_MAPPING(CASE)
4407	#undef CASE
4408	default:
4409	return IFN_LAST;
4410	}
4411	}
4412
4413	/* If IFN implements the conditional form of a tree code, return that
4414	tree code, otherwise return ERROR_MARK. */
4415
4416	tree_code
4417	conditional_internal_fn_code (internal_fn ifn)
4418	{
4419	switch (ifn)
4420	{
4421	#define CASE(CODE, IFN) \
4422	case IFN_COND_##IFN: \
4423	case IFN_COND_LEN_##IFN: \
4424	return CODE;
4425	FOR_EACH_CODE_MAPPING (CASE)
4426	#undef CASE
4427	default:
4428	return ERROR_MARK;
4429	}
4430	}
4431
4432	/* Like get_conditional_internal_fn, but return a function that
4433	additionally restricts the operation to the leading elements
4434	of a vector. The number of elements to process is given by a length
4435	and bias pair, as for IFN_LOAD_LEN. The values of the remaining
4436	elements are taken from the fallback ("else") argument.
4437
4438	For example, if CODE is a binary operation associated with FN:
4439
4440	LHS = FN (COND, A, B, ELSE, LEN, BIAS)
4441
4442	is equivalent to the C code:
4443
4444	for (int i = 0; i < NUNITS; i++)
4445	{
4446	if (i < LEN + BIAS && COND[i])
4447	LHS[i] = A[i] CODE B[i];
4448	else
4449	LHS[i] = ELSE[i];
4450	}
4451	*/
4452
4453	internal_fn
4454	get_conditional_len_internal_fn (tree_code code)
4455	{
4456	switch (code)
4457	{
4458	#define CASE(CODE, IFN) case CODE: return IFN_COND_LEN_##IFN;
4459	FOR_EACH_CODE_MAPPING(CASE)
4460	#undef CASE
4461	default:
4462	return IFN_LAST;
4463	}
4464	}
4465
4466	/* Invoke T(IFN) for each internal function IFN that also has an
4467	IFN_COND_* form. */
4468	#define FOR_EACH_COND_FN_PAIR(T) \
4469	T (FMAX) \
4470	T (FMIN) \
4471	T (FMA) \
4472	T (FMS) \
4473	T (FNMA) \
4474	T (FNMS)
4475
4476	/* Return a function that only performs internal function FN when a
4477	certain condition is met and that uses a given fallback value otherwise.
4478	In other words, the returned function FN' is such that:
4479
4480	LHS = FN' (COND, A1, ... An, ELSE)
4481
4482	is equivalent to the C expression:
4483
4484	LHS = COND ? FN (A1, ..., An) : ELSE;
4485
4486	operating elementwise if the operands are vectors.
4487
4488	Return IFN_LAST if no such function exists. */
4489
4490	internal_fn
4491	get_conditional_internal_fn (internal_fn fn)
4492	{
4493	switch (fn)
4494	{
4495	#define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4496	FOR_EACH_COND_FN_PAIR(CASE)
4497	#undef CASE
4498	default:
4499	return IFN_LAST;
4500	}
4501	}
4502
4503	/* If there exists an internal function like IFN that operates on vectors,
4504	but with additional length and bias parameters, return the internal_fn
4505	for that function, otherwise return IFN_LAST. */
4506	internal_fn
4507	get_len_internal_fn (internal_fn fn)
4508	{
4509	switch (fn)
4510	{
4511	#define DEF_INTERNAL_COND_FN(NAME, ...) \
4512	case IFN_COND_##NAME: \
4513	return IFN_COND_LEN_##NAME;
4514	#define DEF_INTERNAL_SIGNED_COND_FN(NAME, ...) \
4515	case IFN_COND_##NAME: \
4516	return IFN_COND_LEN_##NAME;
4517	#include "internal-fn.def"
4518	default:
4519	return IFN_LAST;
4520	}
4521	}
4522
4523	/* If IFN implements the conditional form of an unconditional internal
4524	function, return that unconditional function, otherwise return IFN_LAST. */
4525
4526	internal_fn
4527	get_unconditional_internal_fn (internal_fn ifn)
4528	{
4529	switch (ifn)
4530	{
4531	#define CASE(NAME) \
4532	case IFN_COND_##NAME: \
4533	case IFN_COND_LEN_##NAME: \
4534	return IFN_##NAME;
4535	FOR_EACH_COND_FN_PAIR (CASE)
4536	#undef CASE
4537	default:
4538	return IFN_LAST;
4539	}
4540	}
4541
4542	/* Return true if STMT can be interpreted as a conditional tree code
4543	operation of the form:
4544
4545	LHS = COND ? OP (RHS1, ...) : ELSE;
4546
4547	operating elementwise if the operands are vectors. This includes
4548	the case of an all-true COND, so that the operation always happens.
4549
4550	There is an alternative approach to interpret the STMT when the operands
4551	are vectors which is the operation predicated by both conditional mask
4552	and loop control length, the equivalent C code:
4553
4554	for (int i = 0; i < NUNTIS; i++)
4555	{
4556	if (i < LEN + BIAS && COND[i])
4557	LHS[i] = A[i] CODE B[i];
4558	else
4559	LHS[i] = ELSE[i];
4560	}
4561
4562	When returning true, set:
4563
4564	- *COND_OUT to the condition COND, or to NULL_TREE if the condition
4565	is known to be all-true
4566	- *CODE_OUT to the tree code
4567	- OPS[I] to operand I of *CODE_OUT
4568	- *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4569	condition is known to be all true.
4570	- *LEN to the len argument if it COND_LEN_* operations or to NULL_TREE.
4571	- *BIAS to the bias argument if it COND_LEN_* operations or to NULL_TREE. */
4572
4573	bool
4574	can_interpret_as_conditional_op_p (gimple *stmt, tree *cond_out,
4575	tree_code *code_out,
4576	tree (&ops)[3], tree *else_out,
4577	tree *len, tree *bias)
4578	{
4579	*len = NULL_TREE;
4580	*bias = NULL_TREE;
4581	if (gassign *assign = dyn_cast <gassign *> (p: stmt))
4582	{
4583	*cond_out = NULL_TREE;
4584	*code_out = gimple_assign_rhs_code (gs: assign);
4585	ops[0] = gimple_assign_rhs1 (gs: assign);
4586	ops[1] = gimple_assign_rhs2 (gs: assign);
4587	ops[2] = gimple_assign_rhs3 (gs: assign);
4588	*else_out = NULL_TREE;
4589	return true;
4590	}
4591	if (gcall *call = dyn_cast <gcall *> (p: stmt))
4592	if (gimple_call_internal_p (gs: call))
4593	{
4594	internal_fn ifn = gimple_call_internal_fn (gs: call);
4595	tree_code code = conditional_internal_fn_code (ifn);
4596	int len_index = internal_fn_len_index (ifn);
4597	int cond_nargs = len_index >= 0 ? 4 : 2;
4598	if (code != ERROR_MARK)
4599	{
4600	*cond_out = gimple_call_arg (gs: call, index: 0);
4601	*code_out = code;
4602	unsigned int nops = gimple_call_num_args (gs: call) - cond_nargs;
4603	for (unsigned int i = 0; i < 3; ++i)
4604	ops[i] = i < nops ? gimple_call_arg (gs: call, index: i + 1) : NULL_TREE;
4605	*else_out = gimple_call_arg (gs: call, index: nops + 1);
4606	if (len_index < 0)
4607	{
4608	if (integer_truep (*cond_out))
4609	{
4610	*cond_out = NULL_TREE;
4611	*else_out = NULL_TREE;
4612	}
4613	}
4614	else
4615	{
4616	*len = gimple_call_arg (gs: call, index: len_index);
4617	*bias = gimple_call_arg (gs: call, index: len_index + 1);
4618	}
4619	return true;
4620	}
4621	}
4622	return false;
4623	}
4624
4625	/* Return true if IFN is some form of load from memory. */
4626
4627	bool
4628	internal_load_fn_p (internal_fn fn)
4629	{
4630	switch (fn)
4631	{
4632	case IFN_MASK_LOAD:
4633	case IFN_LOAD_LANES:
4634	case IFN_MASK_LOAD_LANES:
4635	case IFN_MASK_LEN_LOAD_LANES:
4636	case IFN_GATHER_LOAD:
4637	case IFN_MASK_GATHER_LOAD:
4638	case IFN_MASK_LEN_GATHER_LOAD:
4639	case IFN_LEN_LOAD:
4640	case IFN_MASK_LEN_LOAD:
4641	return true;
4642
4643	default:
4644	return false;
4645	}
4646	}
4647
4648	/* Return true if IFN is some form of store to memory. */
4649
4650	bool
4651	internal_store_fn_p (internal_fn fn)
4652	{
4653	switch (fn)
4654	{
4655	case IFN_MASK_STORE:
4656	case IFN_STORE_LANES:
4657	case IFN_MASK_STORE_LANES:
4658	case IFN_MASK_LEN_STORE_LANES:
4659	case IFN_SCATTER_STORE:
4660	case IFN_MASK_SCATTER_STORE:
4661	case IFN_MASK_LEN_SCATTER_STORE:
4662	case IFN_LEN_STORE:
4663	case IFN_MASK_LEN_STORE:
4664	return true;
4665
4666	default:
4667	return false;
4668	}
4669	}
4670
4671	/* Return true if IFN is some form of gather load or scatter store. */
4672
4673	bool
4674	internal_gather_scatter_fn_p (internal_fn fn)
4675	{
4676	switch (fn)
4677	{
4678	case IFN_GATHER_LOAD:
4679	case IFN_MASK_GATHER_LOAD:
4680	case IFN_MASK_LEN_GATHER_LOAD:
4681	case IFN_SCATTER_STORE:
4682	case IFN_MASK_SCATTER_STORE:
4683	case IFN_MASK_LEN_SCATTER_STORE:
4684	return true;
4685
4686	default:
4687	return false;
4688	}
4689	}
4690
4691	/* If FN takes a vector len argument, return the index of that argument,
4692	otherwise return -1. */
4693
4694	int
4695	internal_fn_len_index (internal_fn fn)
4696	{
4697	switch (fn)
4698	{
4699	case IFN_LEN_LOAD:
4700	case IFN_LEN_STORE:
4701	return 2;
4702
4703	case IFN_MASK_LEN_GATHER_LOAD:
4704	case IFN_MASK_LEN_SCATTER_STORE:
4705	case IFN_COND_LEN_FMA:
4706	case IFN_COND_LEN_FMS:
4707	case IFN_COND_LEN_FNMA:
4708	case IFN_COND_LEN_FNMS:
4709	return 5;
4710
4711	case IFN_COND_LEN_ADD:
4712	case IFN_COND_LEN_SUB:
4713	case IFN_COND_LEN_MUL:
4714	case IFN_COND_LEN_DIV:
4715	case IFN_COND_LEN_MOD:
4716	case IFN_COND_LEN_RDIV:
4717	case IFN_COND_LEN_MIN:
4718	case IFN_COND_LEN_MAX:
4719	case IFN_COND_LEN_FMIN:
4720	case IFN_COND_LEN_FMAX:
4721	case IFN_COND_LEN_AND:
4722	case IFN_COND_LEN_IOR:
4723	case IFN_COND_LEN_XOR:
4724	case IFN_COND_LEN_SHL:
4725	case IFN_COND_LEN_SHR:
4726	return 4;
4727
4728	case IFN_COND_LEN_NEG:
4729	case IFN_MASK_LEN_LOAD:
4730	case IFN_MASK_LEN_STORE:
4731	case IFN_MASK_LEN_LOAD_LANES:
4732	case IFN_MASK_LEN_STORE_LANES:
4733	case IFN_VCOND_MASK_LEN:
4734	return 3;
4735
4736	default:
4737	return -1;
4738	}
4739	}
4740
4741	/* If FN is an IFN_COND_* or IFN_COND_LEN_* function, return the index of the
4742	argument that is used when the condition is false. Return -1 otherwise. */
4743
4744	int
4745	internal_fn_else_index (internal_fn fn)
4746	{
4747	switch (fn)
4748	{
4749	case IFN_COND_NEG:
4750	case IFN_COND_NOT:
4751	case IFN_COND_LEN_NEG:
4752	case IFN_COND_LEN_NOT:
4753	return 2;
4754
4755	case IFN_COND_ADD:
4756	case IFN_COND_SUB:
4757	case IFN_COND_MUL:
4758	case IFN_COND_DIV:
4759	case IFN_COND_MOD:
4760	case IFN_COND_MIN:
4761	case IFN_COND_MAX:
4762	case IFN_COND_FMIN:
4763	case IFN_COND_FMAX:
4764	case IFN_COND_AND:
4765	case IFN_COND_IOR:
4766	case IFN_COND_XOR:
4767	case IFN_COND_SHL:
4768	case IFN_COND_SHR:
4769	case IFN_COND_LEN_ADD:
4770	case IFN_COND_LEN_SUB:
4771	case IFN_COND_LEN_MUL:
4772	case IFN_COND_LEN_DIV:
4773	case IFN_COND_LEN_MOD:
4774	case IFN_COND_LEN_MIN:
4775	case IFN_COND_LEN_MAX:
4776	case IFN_COND_LEN_FMIN:
4777	case IFN_COND_LEN_FMAX:
4778	case IFN_COND_LEN_AND:
4779	case IFN_COND_LEN_IOR:
4780	case IFN_COND_LEN_XOR:
4781	case IFN_COND_LEN_SHL:
4782	case IFN_COND_LEN_SHR:
4783	return 3;
4784
4785	case IFN_COND_FMA:
4786	case IFN_COND_FMS:
4787	case IFN_COND_FNMA:
4788	case IFN_COND_FNMS:
4789	case IFN_COND_LEN_FMA:
4790	case IFN_COND_LEN_FMS:
4791	case IFN_COND_LEN_FNMA:
4792	case IFN_COND_LEN_FNMS:
4793	return 4;
4794
4795	default:
4796	return -1;
4797	}
4798
4799	return -1;
4800	}
4801
4802	/* If FN takes a vector mask argument, return the index of that argument,
4803	otherwise return -1. */
4804
4805	int
4806	internal_fn_mask_index (internal_fn fn)
4807	{
4808	switch (fn)
4809	{
4810	case IFN_MASK_LOAD:
4811	case IFN_MASK_LOAD_LANES:
4812	case IFN_MASK_LEN_LOAD_LANES:
4813	case IFN_MASK_STORE:
4814	case IFN_MASK_STORE_LANES:
4815	case IFN_MASK_LEN_STORE_LANES:
4816	case IFN_MASK_LEN_LOAD:
4817	case IFN_MASK_LEN_STORE:
4818	return 2;
4819
4820	case IFN_MASK_GATHER_LOAD:
4821	case IFN_MASK_SCATTER_STORE:
4822	case IFN_MASK_LEN_GATHER_LOAD:
4823	case IFN_MASK_LEN_SCATTER_STORE:
4824	return 4;
4825
4826	case IFN_VCOND_MASK_LEN:
4827	return 0;
4828
4829	default:
4830	return (conditional_internal_fn_code (ifn: fn) != ERROR_MARK
4831	|| get_unconditional_internal_fn (ifn: fn) != IFN_LAST ? 0 : -1);
4832	}
4833	}
4834
4835	/* If FN takes a value that should be stored to memory, return the index
4836	of that argument, otherwise return -1. */
4837
4838	int
4839	internal_fn_stored_value_index (internal_fn fn)
4840	{
4841	switch (fn)
4842	{
4843	case IFN_MASK_STORE:
4844	case IFN_MASK_STORE_LANES:
4845	case IFN_SCATTER_STORE:
4846	case IFN_MASK_SCATTER_STORE:
4847	case IFN_MASK_LEN_SCATTER_STORE:
4848	return 3;
4849
4850	case IFN_LEN_STORE:
4851	return 4;
4852
4853	case IFN_MASK_LEN_STORE:
4854	case IFN_MASK_LEN_STORE_LANES:
4855	return 5;
4856
4857	default:
4858	return -1;
4859	}
4860	}
4861
4862	/* Return true if the target supports gather load or scatter store function
4863	IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4864	while for stores it is the vector type of the stored data argument.
4865	MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4866	or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4867	offset from the shared base address of each loaded or stored element.
4868	SCALE is the amount by which these offsets should be multiplied
4869	*after* they have been extended to address width. */
4870
4871	bool
4872	internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
4873	tree memory_element_type,
4874	tree offset_vector_type, int scale)
4875	{
4876	if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)),
4877	TYPE_SIZE (memory_element_type)))
4878	return false;
4879	if (maybe_ne (a: TYPE_VECTOR_SUBPARTS (node: vector_type),
4880	b: TYPE_VECTOR_SUBPARTS (node: offset_vector_type)))
4881	return false;
4882	optab optab = direct_internal_fn_optab (fn: ifn);
4883	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (vector_type),
4884	TYPE_MODE (offset_vector_type));
4885	int output_ops = internal_load_fn_p (fn: ifn) ? 1 : 0;
4886	bool unsigned_p = TYPE_UNSIGNED (TREE_TYPE (offset_vector_type));
4887	return (icode != CODE_FOR_nothing
4888	&& insn_operand_matches (icode, opno: 2 + output_ops, GEN_INT (unsigned_p))
4889	&& insn_operand_matches (icode, opno: 3 + output_ops, GEN_INT (scale)));
4890	}
4891
4892	/* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4893	for pointers of type TYPE when the accesses have LENGTH bytes and their
4894	common byte alignment is ALIGN. */
4895
4896	bool
4897	internal_check_ptrs_fn_supported_p (internal_fn ifn, tree type,
4898	poly_uint64 length, unsigned int align)
4899	{
4900	machine_mode mode = TYPE_MODE (type);
4901	optab optab = direct_internal_fn_optab (fn: ifn);
4902	insn_code icode = direct_optab_handler (op: optab, mode);
4903	if (icode == CODE_FOR_nothing)
4904	return false;
4905	rtx length_rtx = immed_wide_int_const (length, mode);
4906	return (insn_operand_matches (icode, opno: 3, operand: length_rtx)
4907	&& insn_operand_matches (icode, opno: 4, GEN_INT (align)));
4908	}
4909
4910	/* Return the supported bias for IFN which is either IFN_{LEN_,MASK_LEN_,}LOAD
4911	or IFN_{LEN_,MASK_LEN_,}STORE. For now we only support the biases of 0 and
4912	-1 (in case 0 is not an allowable length for {len_,mask_len_}load or
4913	{len_,mask_len_}store). If none of the biases match what the backend
4914	provides, return VECT_PARTIAL_BIAS_UNSUPPORTED. */
4915
4916	signed char
4917	internal_len_load_store_bias (internal_fn ifn, machine_mode mode)
4918	{
4919	optab optab = direct_internal_fn_optab (fn: ifn);
4920	insn_code icode = direct_optab_handler (op: optab, mode);
4921	int bias_no = 3;
4922
4923	if (icode == CODE_FOR_nothing)
4924	{
4925	machine_mode mask_mode;
4926	if (!targetm.vectorize.get_mask_mode (mode).exists (mode: &mask_mode))
4927	return VECT_PARTIAL_BIAS_UNSUPPORTED;
4928	if (ifn == IFN_LEN_LOAD)
4929	{
4930	/* Try MASK_LEN_LOAD. */
4931	optab = direct_internal_fn_optab (fn: IFN_MASK_LEN_LOAD);
4932	}
4933	else
4934	{
4935	/* Try MASK_LEN_STORE. */
4936	optab = direct_internal_fn_optab (fn: IFN_MASK_LEN_STORE);
4937	}
4938	icode = convert_optab_handler (op: optab, to_mode: mode, from_mode: mask_mode);
4939	bias_no = 4;
4940	}
4941
4942	if (icode != CODE_FOR_nothing)
4943	{
4944	/* For now we only support biases of 0 or -1. Try both of them. */
4945	if (insn_operand_matches (icode, opno: bias_no, GEN_INT (0)))
4946	return 0;
4947	if (insn_operand_matches (icode, opno: bias_no, GEN_INT (-1)))
4948	return -1;
4949	}
4950
4951	return VECT_PARTIAL_BIAS_UNSUPPORTED;
4952	}
4953
4954	/* Expand STMT as though it were a call to internal function FN. */
4955
4956	void
4957	expand_internal_call (internal_fn fn, gcall *stmt)
4958	{
4959	internal_fn_expanders[fn] (fn, stmt);
4960	}
4961
4962	/* Expand STMT, which is a call to internal function FN. */
4963
4964	void
4965	expand_internal_call (gcall *stmt)
4966	{
4967	expand_internal_call (fn: gimple_call_internal_fn (gs: stmt), stmt);
4968	}
4969
4970	/* If TYPE is a vector type, return true if IFN is a direct internal
4971	function that is supported for that type. If TYPE is a scalar type,
4972	return true if IFN is a direct internal function that is supported for
4973	the target's preferred vector version of TYPE. */
4974
4975	bool
4976	vectorized_internal_fn_supported_p (internal_fn ifn, tree type)
4977	{
4978	if (VECTOR_MODE_P (TYPE_MODE (type)))
4979	return direct_internal_fn_supported_p (fn: ifn, type, opt_type: OPTIMIZE_FOR_SPEED);
4980
4981	scalar_mode smode;
4982	if (VECTOR_TYPE_P (type)
4983	|| !is_a <scalar_mode> (TYPE_MODE (type), result: &smode))
4984	return false;
4985
4986	machine_mode vmode = targetm.vectorize.preferred_simd_mode (smode);
4987	if (VECTOR_MODE_P (vmode))
4988	{
4989	tree vectype = build_vector_type_for_mode (type, vmode);
4990	if (direct_internal_fn_supported_p (fn: ifn, type: vectype, opt_type: OPTIMIZE_FOR_SPEED))
4991	return true;
4992	}
4993
4994	auto_vector_modes vector_modes;
4995	targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
4996	for (machine_mode base_mode : vector_modes)
4997	if (related_vector_mode (base_mode, smode).exists (mode: &vmode))
4998	{
4999	tree vectype = build_vector_type_for_mode (type, vmode);
5000	if (direct_internal_fn_supported_p (fn: ifn, type: vectype, opt_type: OPTIMIZE_FOR_SPEED))
5001	return true;
5002	}
5003
5004	return false;
5005	}
5006
5007	void
5008	expand_SHUFFLEVECTOR (internal_fn, gcall *)
5009	{
5010	gcc_unreachable ();
5011	}
5012
5013	void
5014	expand_PHI (internal_fn, gcall *)
5015	{
5016	gcc_unreachable ();
5017	}
5018
5019	void
5020	expand_SPACESHIP (internal_fn, gcall *stmt)
5021	{
5022	tree lhs = gimple_call_lhs (gs: stmt);
5023	tree rhs1 = gimple_call_arg (gs: stmt, index: 0);
5024	tree rhs2 = gimple_call_arg (gs: stmt, index: 1);
5025	tree type = TREE_TYPE (rhs1);
5026
5027	do_pending_stack_adjust ();
5028
5029	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
5030	rtx op1 = expand_normal (exp: rhs1);
5031	rtx op2 = expand_normal (exp: rhs2);
5032
5033	class expand_operand ops[3];
5034	create_output_operand (op: &ops[0], x: target, TYPE_MODE (TREE_TYPE (lhs)));
5035	create_input_operand (op: &ops[1], value: op1, TYPE_MODE (type));
5036	create_input_operand (op: &ops[2], value: op2, TYPE_MODE (type));
5037	insn_code icode = optab_handler (op: spaceship_optab, TYPE_MODE (type));
5038	expand_insn (icode, nops: 3, ops);
5039	if (!rtx_equal_p (target, ops[0].value))
5040	emit_move_insn (target, ops[0].value);
5041	}
5042
5043	void
5044	expand_ASSUME (internal_fn, gcall *)
5045	{
5046	}
5047
5048	void
5049	expand_MASK_CALL (internal_fn, gcall *)
5050	{
5051	/* This IFN should only exist between ifcvt and vect passes. */
5052	gcc_unreachable ();
5053	}
5054
5055	void
5056	expand_MULBITINT (internal_fn, gcall *stmt)
5057	{
5058	rtx_mode_t args[6];
5059	for (int i = 0; i < 6; i++)
5060	args[i] = rtx_mode_t (expand_normal (exp: gimple_call_arg (gs: stmt, index: i)),
5061	(i & 1) ? SImode : ptr_mode);
5062	rtx fun = init_one_libfunc ("__mulbitint3");
5063	emit_library_call_value_1 (0, fun, NULL_RTX, LCT_NORMAL, VOIDmode, 6, args);
5064	}
5065
5066	void
5067	expand_DIVMODBITINT (internal_fn, gcall *stmt)
5068	{
5069	rtx_mode_t args[8];
5070	for (int i = 0; i < 8; i++)
5071	args[i] = rtx_mode_t (expand_normal (exp: gimple_call_arg (gs: stmt, index: i)),
5072	(i & 1) ? SImode : ptr_mode);
5073	rtx fun = init_one_libfunc ("__divmodbitint4");
5074	emit_library_call_value_1 (0, fun, NULL_RTX, LCT_NORMAL, VOIDmode, 8, args);
5075	}
5076
5077	void
5078	expand_FLOATTOBITINT (internal_fn, gcall *stmt)
5079	{
5080	machine_mode mode = TYPE_MODE (TREE_TYPE (gimple_call_arg (stmt, 2)));
5081	rtx arg0 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
5082	rtx arg1 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
5083	rtx arg2 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 2));
5084	const char *mname = GET_MODE_NAME (mode);
5085	unsigned mname_len = strlen (s: mname);
5086	int len = 12 + mname_len;
5087	if (DECIMAL_FLOAT_MODE_P (mode))
5088	len += 4;
5089	char *libfunc_name = XALLOCAVEC (char, len);
5090	char *p = libfunc_name;
5091	const char *q;
5092	if (DECIMAL_FLOAT_MODE_P (mode))
5093	{
5094	#if ENABLE_DECIMAL_BID_FORMAT
5095	memcpy (dest: p, src: "__bid_fix", n: 9);
5096	#else
5097	memcpy (p, "__dpd_fix", 9);
5098	#endif
5099	p += 9;
5100	}
5101	else
5102	{
5103	memcpy (dest: p, src: "__fix", n: 5);
5104	p += 5;
5105	}
5106	for (q = mname; *q; q++)
5107	*p++ = TOLOWER (*q);
5108	memcpy (dest: p, src: "bitint", n: 7);
5109	rtx fun = init_one_libfunc (libfunc_name);
5110	emit_library_call (fun, fn_type: LCT_NORMAL, VOIDmode, arg1: arg0, arg1_mode: ptr_mode, arg2: arg1,
5111	SImode, arg3: arg2, arg3_mode: mode);
5112	}
5113
5114	void
5115	expand_BITINTTOFLOAT (internal_fn, gcall *stmt)
5116	{
5117	tree lhs = gimple_call_lhs (gs: stmt);
5118	if (!lhs)
5119	return;
5120	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
5121	rtx arg0 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
5122	rtx arg1 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
5123	const char *mname = GET_MODE_NAME (mode);
5124	unsigned mname_len = strlen (s: mname);
5125	int len = 14 + mname_len;
5126	if (DECIMAL_FLOAT_MODE_P (mode))
5127	len += 4;
5128	char *libfunc_name = XALLOCAVEC (char, len);
5129	char *p = libfunc_name;
5130	const char *q;
5131	if (DECIMAL_FLOAT_MODE_P (mode))
5132	{
5133	#if ENABLE_DECIMAL_BID_FORMAT
5134	memcpy (dest: p, src: "__bid_floatbitint", n: 17);
5135	#else
5136	memcpy (p, "__dpd_floatbitint", 17);
5137	#endif
5138	p += 17;
5139	}
5140	else
5141	{
5142	memcpy (dest: p, src: "__floatbitint", n: 13);
5143	p += 13;
5144	}
5145	for (q = mname; *q; q++)
5146	*p++ = TOLOWER (*q);
5147	*p = '\0';
5148	rtx fun = init_one_libfunc (libfunc_name);
5149	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
5150	rtx val = emit_library_call_value (fun, value: target, fn_type: LCT_PURE, outmode: mode,
5151	arg1: arg0, arg1_mode: ptr_mode, arg2: arg1, SImode);
5152	if (val != target)
5153	emit_move_insn (target, val);
5154	}
5155
5156	static bool
5157	expand_bitquery (internal_fn fn, gcall *stmt)
5158	{
5159	tree lhs = gimple_call_lhs (gs: stmt);
5160	if (lhs == NULL_TREE)
5161	return false;
5162	tree arg = gimple_call_arg (gs: stmt, index: 0);
5163	if (TREE_CODE (arg) == INTEGER_CST)
5164	{
5165	tree ret = fold_const_call (as_combined_fn (fn), TREE_TYPE (arg), arg);
5166	gcc_checking_assert (ret && TREE_CODE (ret) == INTEGER_CST);
5167	expand_assignment (lhs, ret, false);
5168	return false;
5169	}
5170	return true;
5171	}
5172
5173	void
5174	expand_CLRSB (internal_fn fn, gcall *stmt)
5175	{
5176	if (expand_bitquery (fn, stmt))
5177	expand_unary_optab_fn (fn, stmt, clrsb_optab);
5178	}
5179
5180	void
5181	expand_CLZ (internal_fn fn, gcall *stmt)
5182	{
5183	if (expand_bitquery (fn, stmt))
5184	expand_unary_optab_fn (fn, stmt, clz_optab);
5185	}
5186
5187	void
5188	expand_CTZ (internal_fn fn, gcall *stmt)
5189	{
5190	if (expand_bitquery (fn, stmt))
5191	expand_unary_optab_fn (fn, stmt, ctz_optab);
5192	}
5193
5194	void
5195	expand_FFS (internal_fn fn, gcall *stmt)
5196	{
5197	if (expand_bitquery (fn, stmt))
5198	expand_unary_optab_fn (fn, stmt, ffs_optab);
5199	}
5200
5201	void
5202	expand_PARITY (internal_fn fn, gcall *stmt)
5203	{
5204	if (expand_bitquery (fn, stmt))
5205	expand_unary_optab_fn (fn, stmt, parity_optab);
5206	}
5207
5208	void
5209	expand_POPCOUNT (internal_fn fn, gcall *stmt)
5210	{
5211	if (!expand_bitquery (fn, stmt))
5212	return;
5213	if (gimple_call_num_args (gs: stmt) == 1)
5214	{
5215	expand_unary_optab_fn (fn, stmt, popcount_optab);
5216	return;
5217	}
5218	/* If .POPCOUNT call has 2 arguments, match_single_bit_test marked it
5219	because the result is only used in an equality comparison against 1.
5220	Use rtx costs in that case to determine if .POPCOUNT (arg) == 1
5221	or (arg ^ (arg - 1)) > arg - 1 is cheaper.
5222	If .POPCOUNT second argument is 0, we additionally know that arg
5223	is non-zero, so use arg & (arg - 1) == 0 instead. */
5224	bool speed_p = optimize_insn_for_speed_p ();
5225	tree lhs = gimple_call_lhs (gs: stmt);
5226	tree arg = gimple_call_arg (gs: stmt, index: 0);
5227	bool nonzero_arg = integer_zerop (gimple_call_arg (gs: stmt, index: 1));
5228	tree type = TREE_TYPE (arg);
5229	machine_mode mode = TYPE_MODE (type);
5230	do_pending_stack_adjust ();
5231	start_sequence ();
5232	expand_unary_optab_fn (fn, stmt, popcount_optab);
5233	rtx_insn *popcount_insns = get_insns ();
5234	end_sequence ();
5235	start_sequence ();
5236	rtx plhs = expand_normal (exp: lhs);
5237	rtx pcmp = emit_store_flag (NULL_RTX, EQ, plhs, const1_rtx, mode, 0, 0);
5238	if (pcmp == NULL_RTX)
5239	{
5240	fail:
5241	end_sequence ();
5242	emit_insn (popcount_insns);
5243	return;
5244	}
5245	rtx_insn *popcount_cmp_insns = get_insns ();
5246	end_sequence ();
5247	start_sequence ();
5248	rtx op0 = expand_normal (exp: arg);
5249	rtx argm1 = expand_simple_binop (mode, PLUS, op0, constm1_rtx, NULL_RTX,
5250	1, OPTAB_DIRECT);
5251	if (argm1 == NULL_RTX)
5252	goto fail;
5253	rtx argxorargm1 = expand_simple_binop (mode, nonzero_arg ? AND : XOR, op0,
5254	argm1, NULL_RTX, 1, OPTAB_DIRECT);
5255	if (argxorargm1 == NULL_RTX)
5256	goto fail;
5257	rtx cmp;
5258	if (nonzero_arg)
5259	cmp = emit_store_flag (NULL_RTX, EQ, argxorargm1, const0_rtx, mode, 1, 1);
5260	else
5261	cmp = emit_store_flag (NULL_RTX, GTU, argxorargm1, argm1, mode, 1, 1);
5262	if (cmp == NULL_RTX)
5263	goto fail;
5264	rtx_insn *cmp_insns = get_insns ();
5265	end_sequence ();
5266	unsigned popcount_cost = (seq_cost (popcount_insns, speed_p)
5267	+ seq_cost (popcount_cmp_insns, speed_p));
5268	unsigned cmp_cost = seq_cost (cmp_insns, speed_p);
5269	if (popcount_cost <= cmp_cost)
5270	emit_insn (popcount_insns);
5271	else
5272	{
5273	emit_insn (cmp_insns);
5274	plhs = expand_normal (exp: lhs);
5275	if (GET_MODE (cmp) != GET_MODE (plhs))
5276	cmp = convert_to_mode (GET_MODE (plhs), cmp, 1);
5277	emit_move_insn (plhs, cmp);
5278	}
5279	}
5280