1/* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2017 Free Software Foundation, Inc.
3
4This file is part of GCC.
5
6GCC is free software; you can redistribute it and/or modify it under
7the terms of the GNU General Public License as published by the Free
8Software Foundation; either version 3, or (at your option) any later
9version.
10
11GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12WARRANTY; without even the implied warranty of MERCHANTABILITY or
13FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14for more details.
15
16You should have received a copy of the GNU General Public License
17along with GCC; see the file COPYING3. If not see
18<http://www.gnu.org/licenses/>. */
19
20
21#include "config.h"
22#include "system.h"
23#include "coretypes.h"
24#include "backend.h"
25#include "target.h"
26#include "rtl.h"
27#include "tree.h"
28#include "memmodel.h"
29#include "predict.h"
30#include "tm_p.h"
31#include "expmed.h"
32#include "optabs.h"
33#include "emit-rtl.h"
34#include "recog.h"
35#include "diagnostic-core.h"
36
37/* Include insn-config.h before expr.h so that HAVE_conditional_move
38 is properly defined. */
39#include "stor-layout.h"
40#include "except.h"
41#include "dojump.h"
42#include "explow.h"
43#include "expr.h"
44#include "optabs-tree.h"
45#include "libfuncs.h"
46
47static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
48 machine_mode *);
49static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
50static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
51
52/* Debug facility for use in GDB. */
53void debug_optab_libfuncs (void);
54
55/* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
56 the result of operation CODE applied to OP0 (and OP1 if it is a binary
57 operation).
58
59 If the last insn does not set TARGET, don't do anything, but return 1.
60
61 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
62 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
63 try again, ensuring that TARGET is not one of the operands. */
64
65static int
66add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0, rtx op1)
67{
68 rtx_insn *last_insn;
69 rtx set;
70 rtx note;
71
72 gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
73
74 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
75 && GET_RTX_CLASS (code) != RTX_BIN_ARITH
76 && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
77 && GET_RTX_CLASS (code) != RTX_COMPARE
78 && GET_RTX_CLASS (code) != RTX_UNARY)
79 return 1;
80
81 if (GET_CODE (target) == ZERO_EXTRACT)
82 return 1;
83
84 for (last_insn = insns;
85 NEXT_INSN (last_insn) != NULL_RTX;
86 last_insn = NEXT_INSN (last_insn))
87 ;
88
89 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
90 a value changing in the insn, so the note would be invalid for CSE. */
91 if (reg_overlap_mentioned_p (target, op0)
92 || (op1 && reg_overlap_mentioned_p (target, op1)))
93 {
94 if (MEM_P (target)
95 && (rtx_equal_p (target, op0)
96 || (op1 && rtx_equal_p (target, op1))))
97 {
98 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
99 over expanding it as temp = MEM op X, MEM = temp. If the target
100 supports MEM = MEM op X instructions, it is sometimes too hard
101 to reconstruct that form later, especially if X is also a memory,
102 and due to multiple occurrences of addresses the address might
103 be forced into register unnecessarily.
104 Note that not emitting the REG_EQUIV note might inhibit
105 CSE in some cases. */
106 set = single_set (last_insn);
107 if (set
108 && GET_CODE (SET_SRC (set)) == code
109 && MEM_P (SET_DEST (set))
110 && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
111 || (op1 && rtx_equal_p (SET_DEST (set),
112 XEXP (SET_SRC (set), 1)))))
113 return 1;
114 }
115 return 0;
116 }
117
118 set = set_for_reg_notes (last_insn);
119 if (set == NULL_RTX)
120 return 1;
121
122 if (! rtx_equal_p (SET_DEST (set), target)
123 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
124 && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
125 || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
126 return 1;
127
128 if (GET_RTX_CLASS (code) == RTX_UNARY)
129 switch (code)
130 {
131 case FFS:
132 case CLZ:
133 case CTZ:
134 case CLRSB:
135 case POPCOUNT:
136 case PARITY:
137 case BSWAP:
138 if (GET_MODE (op0) != VOIDmode && GET_MODE (target) != GET_MODE (op0))
139 {
140 note = gen_rtx_fmt_e (code, GET_MODE (op0), copy_rtx (op0));
141 if (GET_MODE_UNIT_SIZE (GET_MODE (op0))
142 > GET_MODE_UNIT_SIZE (GET_MODE (target)))
143 note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
144 note, GET_MODE (op0));
145 else
146 note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
147 note, GET_MODE (op0));
148 break;
149 }
150 /* FALLTHRU */
151 default:
152 note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
153 break;
154 }
155 else
156 note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
157
158 set_unique_reg_note (last_insn, REG_EQUAL, note);
159
160 return 1;
161}
162
163/* Given two input operands, OP0 and OP1, determine what the correct from_mode
164 for a widening operation would be. In most cases this would be OP0, but if
165 that's a constant it'll be VOIDmode, which isn't useful. */
166
167static machine_mode
168widened_mode (machine_mode to_mode, rtx op0, rtx op1)
169{
170 machine_mode m0 = GET_MODE (op0);
171 machine_mode m1 = GET_MODE (op1);
172 machine_mode result;
173
174 if (m0 == VOIDmode && m1 == VOIDmode)
175 return to_mode;
176 else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
177 result = m1;
178 else
179 result = m0;
180
181 if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
182 return to_mode;
183
184 return result;
185}
186
187/* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
188 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
189 not actually do a sign-extend or zero-extend, but can leave the
190 higher-order bits of the result rtx undefined, for example, in the case
191 of logical operations, but not right shifts. */
192
193static rtx
194widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
195 int unsignedp, int no_extend)
196{
197 rtx result;
198 scalar_int_mode int_mode;
199
200 /* If we don't have to extend and this is a constant, return it. */
201 if (no_extend && GET_MODE (op) == VOIDmode)
202 return op;
203
204 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
205 extend since it will be more efficient to do so unless the signedness of
206 a promoted object differs from our extension. */
207 if (! no_extend
208 || !is_a <scalar_int_mode> (mode, &int_mode)
209 || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
210 && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
211 return convert_modes (mode, oldmode, op, unsignedp);
212
213 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
214 SUBREG. */
215 if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
216 return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
217
218 /* Otherwise, get an object of MODE, clobber it, and set the low-order
219 part to OP. */
220
221 result = gen_reg_rtx (int_mode);
222 emit_clobber (result);
223 emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
224 return result;
225}
226
227/* Expand vector widening operations.
228
229 There are two different classes of operations handled here:
230 1) Operations whose result is wider than all the arguments to the operation.
231 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
232 In this case OP0 and optionally OP1 would be initialized,
233 but WIDE_OP wouldn't (not relevant for this case).
234 2) Operations whose result is of the same size as the last argument to the
235 operation, but wider than all the other arguments to the operation.
236 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
237 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
238
239 E.g, when called to expand the following operations, this is how
240 the arguments will be initialized:
241 nops OP0 OP1 WIDE_OP
242 widening-sum 2 oprnd0 - oprnd1
243 widening-dot-product 3 oprnd0 oprnd1 oprnd2
244 widening-mult 2 oprnd0 oprnd1 -
245 type-promotion (vec-unpack) 1 oprnd0 - - */
246
247rtx
248expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op,
249 rtx target, int unsignedp)
250{
251 struct expand_operand eops[4];
252 tree oprnd0, oprnd1, oprnd2;
253 machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
254 optab widen_pattern_optab;
255 enum insn_code icode;
256 int nops = TREE_CODE_LENGTH (ops->code);
257 int op;
258
259 oprnd0 = ops->op0;
260 tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
261 widen_pattern_optab =
262 optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
263 if (ops->code == WIDEN_MULT_PLUS_EXPR
264 || ops->code == WIDEN_MULT_MINUS_EXPR)
265 icode = find_widening_optab_handler (widen_pattern_optab,
266 TYPE_MODE (TREE_TYPE (ops->op2)),
267 tmode0);
268 else
269 icode = optab_handler (widen_pattern_optab, tmode0);
270 gcc_assert (icode != CODE_FOR_nothing);
271
272 if (nops >= 2)
273 {
274 oprnd1 = ops->op1;
275 tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
276 }
277
278 /* The last operand is of a wider mode than the rest of the operands. */
279 if (nops == 2)
280 wmode = tmode1;
281 else if (nops == 3)
282 {
283 gcc_assert (tmode1 == tmode0);
284 gcc_assert (op1);
285 oprnd2 = ops->op2;
286 wmode = TYPE_MODE (TREE_TYPE (oprnd2));
287 }
288
289 op = 0;
290 create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
291 create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
292 if (op1)
293 create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
294 if (wide_op)
295 create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
296 expand_insn (icode, op, eops);
297 return eops[0].value;
298}
299
300/* Generate code to perform an operation specified by TERNARY_OPTAB
301 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
302
303 UNSIGNEDP is for the case where we have to widen the operands
304 to perform the operation. It says to use zero-extension.
305
306 If TARGET is nonzero, the value
307 is generated there, if it is convenient to do so.
308 In all cases an rtx is returned for the locus of the value;
309 this may or may not be TARGET. */
310
311rtx
312expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
313 rtx op1, rtx op2, rtx target, int unsignedp)
314{
315 struct expand_operand ops[4];
316 enum insn_code icode = optab_handler (ternary_optab, mode);
317
318 gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
319
320 create_output_operand (&ops[0], target, mode);
321 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
322 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
323 create_convert_operand_from (&ops[3], op2, mode, unsignedp);
324 expand_insn (icode, 4, ops);
325 return ops[0].value;
326}
327
328
329/* Like expand_binop, but return a constant rtx if the result can be
330 calculated at compile time. The arguments and return value are
331 otherwise the same as for expand_binop. */
332
333rtx
334simplify_expand_binop (machine_mode mode, optab binoptab,
335 rtx op0, rtx op1, rtx target, int unsignedp,
336 enum optab_methods methods)
337{
338 if (CONSTANT_P (op0) && CONSTANT_P (op1))
339 {
340 rtx x = simplify_binary_operation (optab_to_code (binoptab),
341 mode, op0, op1);
342 if (x)
343 return x;
344 }
345
346 return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
347}
348
349/* Like simplify_expand_binop, but always put the result in TARGET.
350 Return true if the expansion succeeded. */
351
352bool
353force_expand_binop (machine_mode mode, optab binoptab,
354 rtx op0, rtx op1, rtx target, int unsignedp,
355 enum optab_methods methods)
356{
357 rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
358 target, unsignedp, methods);
359 if (x == 0)
360 return false;
361 if (x != target)
362 emit_move_insn (target, x);
363 return true;
364}
365
366/* Create a new vector value in VMODE with all elements set to OP. The
367 mode of OP must be the element mode of VMODE. If OP is a constant,
368 then the return value will be a constant. */
369
370static rtx
371expand_vector_broadcast (machine_mode vmode, rtx op)
372{
373 enum insn_code icode;
374 rtvec vec;
375 rtx ret;
376 int i, n;
377
378 gcc_checking_assert (VECTOR_MODE_P (vmode));
379
380 if (valid_for_const_vec_duplicate_p (vmode, op))
381 return gen_const_vec_duplicate (vmode, op);
382
383 /* ??? If the target doesn't have a vec_init, then we have no easy way
384 of performing this operation. Most of this sort of generic support
385 is hidden away in the vector lowering support in gimple. */
386 icode = convert_optab_handler (vec_init_optab, vmode,
387 GET_MODE_INNER (vmode));
388 if (icode == CODE_FOR_nothing)
389 return NULL;
390
391 n = GET_MODE_NUNITS (vmode);
392 vec = rtvec_alloc (n);
393 for (i = 0; i < n; ++i)
394 RTVEC_ELT (vec, i) = op;
395 ret = gen_reg_rtx (vmode);
396 emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
397
398 return ret;
399}
400
401/* This subroutine of expand_doubleword_shift handles the cases in which
402 the effective shift value is >= BITS_PER_WORD. The arguments and return
403 value are the same as for the parent routine, except that SUPERWORD_OP1
404 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
405 INTO_TARGET may be null if the caller has decided to calculate it. */
406
407static bool
408expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
409 rtx outof_target, rtx into_target,
410 int unsignedp, enum optab_methods methods)
411{
412 if (into_target != 0)
413 if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
414 into_target, unsignedp, methods))
415 return false;
416
417 if (outof_target != 0)
418 {
419 /* For a signed right shift, we must fill OUTOF_TARGET with copies
420 of the sign bit, otherwise we must fill it with zeros. */
421 if (binoptab != ashr_optab)
422 emit_move_insn (outof_target, CONST0_RTX (word_mode));
423 else
424 if (!force_expand_binop (word_mode, binoptab,
425 outof_input, GEN_INT (BITS_PER_WORD - 1),
426 outof_target, unsignedp, methods))
427 return false;
428 }
429 return true;
430}
431
432/* This subroutine of expand_doubleword_shift handles the cases in which
433 the effective shift value is < BITS_PER_WORD. The arguments and return
434 value are the same as for the parent routine. */
435
436static bool
437expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
438 rtx outof_input, rtx into_input, rtx op1,
439 rtx outof_target, rtx into_target,
440 int unsignedp, enum optab_methods methods,
441 unsigned HOST_WIDE_INT shift_mask)
442{
443 optab reverse_unsigned_shift, unsigned_shift;
444 rtx tmp, carries;
445
446 reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
447 unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
448
449 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
450 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
451 the opposite direction to BINOPTAB. */
452 if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
453 {
454 carries = outof_input;
455 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
456 op1_mode), op1_mode);
457 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
458 0, true, methods);
459 }
460 else
461 {
462 /* We must avoid shifting by BITS_PER_WORD bits since that is either
463 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
464 has unknown behavior. Do a single shift first, then shift by the
465 remainder. It's OK to use ~OP1 as the remainder if shift counts
466 are truncated to the mode size. */
467 carries = expand_binop (word_mode, reverse_unsigned_shift,
468 outof_input, const1_rtx, 0, unsignedp, methods);
469 if (shift_mask == BITS_PER_WORD - 1)
470 {
471 tmp = immed_wide_int_const
472 (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode);
473 tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp,
474 0, true, methods);
475 }
476 else
477 {
478 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
479 op1_mode), op1_mode);
480 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
481 0, true, methods);
482 }
483 }
484 if (tmp == 0 || carries == 0)
485 return false;
486 carries = expand_binop (word_mode, reverse_unsigned_shift,
487 carries, tmp, 0, unsignedp, methods);
488 if (carries == 0)
489 return false;
490
491 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
492 so the result can go directly into INTO_TARGET if convenient. */
493 tmp = expand_binop (word_mode, unsigned_shift, into_input, op1,
494 into_target, unsignedp, methods);
495 if (tmp == 0)
496 return false;
497
498 /* Now OR in the bits carried over from OUTOF_INPUT. */
499 if (!force_expand_binop (word_mode, ior_optab, tmp, carries,
500 into_target, unsignedp, methods))
501 return false;
502
503 /* Use a standard word_mode shift for the out-of half. */
504 if (outof_target != 0)
505 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
506 outof_target, unsignedp, methods))
507 return false;
508
509 return true;
510}
511
512
513/* Try implementing expand_doubleword_shift using conditional moves.
514 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
515 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
516 are the shift counts to use in the former and latter case. All other
517 arguments are the same as the parent routine. */
518
519static bool
520expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
521 enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
522 rtx outof_input, rtx into_input,
523 rtx subword_op1, rtx superword_op1,
524 rtx outof_target, rtx into_target,
525 int unsignedp, enum optab_methods methods,
526 unsigned HOST_WIDE_INT shift_mask)
527{
528 rtx outof_superword, into_superword;
529
530 /* Put the superword version of the output into OUTOF_SUPERWORD and
531 INTO_SUPERWORD. */
532 outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
533 if (outof_target != 0 && subword_op1 == superword_op1)
534 {
535 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
536 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
537 into_superword = outof_target;
538 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
539 outof_superword, 0, unsignedp, methods))
540 return false;
541 }
542 else
543 {
544 into_superword = gen_reg_rtx (word_mode);
545 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
546 outof_superword, into_superword,
547 unsignedp, methods))
548 return false;
549 }
550
551 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
552 if (!expand_subword_shift (op1_mode, binoptab,
553 outof_input, into_input, subword_op1,
554 outof_target, into_target,
555 unsignedp, methods, shift_mask))
556 return false;
557
558 /* Select between them. Do the INTO half first because INTO_SUPERWORD
559 might be the current value of OUTOF_TARGET. */
560 if (!emit_conditional_move (into_target, cmp_code, cmp1, cmp2, op1_mode,
561 into_target, into_superword, word_mode, false))
562 return false;
563
564 if (outof_target != 0)
565 if (!emit_conditional_move (outof_target, cmp_code, cmp1, cmp2, op1_mode,
566 outof_target, outof_superword,
567 word_mode, false))
568 return false;
569
570 return true;
571}
572
573/* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
574 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
575 input operand; the shift moves bits in the direction OUTOF_INPUT->
576 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
577 of the target. OP1 is the shift count and OP1_MODE is its mode.
578 If OP1 is constant, it will have been truncated as appropriate
579 and is known to be nonzero.
580
581 If SHIFT_MASK is zero, the result of word shifts is undefined when the
582 shift count is outside the range [0, BITS_PER_WORD). This routine must
583 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
584
585 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
586 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
587 fill with zeros or sign bits as appropriate.
588
589 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
590 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
591 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
592 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
593 are undefined.
594
595 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
596 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
597 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
598 function wants to calculate it itself.
599
600 Return true if the shift could be successfully synthesized. */
601
602static bool
603expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
604 rtx outof_input, rtx into_input, rtx op1,
605 rtx outof_target, rtx into_target,
606 int unsignedp, enum optab_methods methods,
607 unsigned HOST_WIDE_INT shift_mask)
608{
609 rtx superword_op1, tmp, cmp1, cmp2;
610 enum rtx_code cmp_code;
611
612 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
613 fill the result with sign or zero bits as appropriate. If so, the value
614 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
615 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
616 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
617
618 This isn't worthwhile for constant shifts since the optimizers will
619 cope better with in-range shift counts. */
620 if (shift_mask >= BITS_PER_WORD
621 && outof_target != 0
622 && !CONSTANT_P (op1))
623 {
624 if (!expand_doubleword_shift (op1_mode, binoptab,
625 outof_input, into_input, op1,
626 0, into_target,
627 unsignedp, methods, shift_mask))
628 return false;
629 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
630 outof_target, unsignedp, methods))
631 return false;
632 return true;
633 }
634
635 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
636 is true when the effective shift value is less than BITS_PER_WORD.
637 Set SUPERWORD_OP1 to the shift count that should be used to shift
638 OUTOF_INPUT into INTO_TARGET when the condition is false. */
639 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
640 if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
641 {
642 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
643 is a subword shift count. */
644 cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
645 0, true, methods);
646 cmp2 = CONST0_RTX (op1_mode);
647 cmp_code = EQ;
648 superword_op1 = op1;
649 }
650 else
651 {
652 /* Set CMP1 to OP1 - BITS_PER_WORD. */
653 cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
654 0, true, methods);
655 cmp2 = CONST0_RTX (op1_mode);
656 cmp_code = LT;
657 superword_op1 = cmp1;
658 }
659 if (cmp1 == 0)
660 return false;
661
662 /* If we can compute the condition at compile time, pick the
663 appropriate subroutine. */
664 tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
665 if (tmp != 0 && CONST_INT_P (tmp))
666 {
667 if (tmp == const0_rtx)
668 return expand_superword_shift (binoptab, outof_input, superword_op1,
669 outof_target, into_target,
670 unsignedp, methods);
671 else
672 return expand_subword_shift (op1_mode, binoptab,
673 outof_input, into_input, op1,
674 outof_target, into_target,
675 unsignedp, methods, shift_mask);
676 }
677
678 /* Try using conditional moves to generate straight-line code. */
679 if (HAVE_conditional_move)
680 {
681 rtx_insn *start = get_last_insn ();
682 if (expand_doubleword_shift_condmove (op1_mode, binoptab,
683 cmp_code, cmp1, cmp2,
684 outof_input, into_input,
685 op1, superword_op1,
686 outof_target, into_target,
687 unsignedp, methods, shift_mask))
688 return true;
689 delete_insns_since (start);
690 }
691
692 /* As a last resort, use branches to select the correct alternative. */
693 rtx_code_label *subword_label = gen_label_rtx ();
694 rtx_code_label *done_label = gen_label_rtx ();
695
696 NO_DEFER_POP;
697 do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
698 0, 0, subword_label,
699 profile_probability::uninitialized ());
700 OK_DEFER_POP;
701
702 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
703 outof_target, into_target,
704 unsignedp, methods))
705 return false;
706
707 emit_jump_insn (targetm.gen_jump (done_label));
708 emit_barrier ();
709 emit_label (subword_label);
710
711 if (!expand_subword_shift (op1_mode, binoptab,
712 outof_input, into_input, op1,
713 outof_target, into_target,
714 unsignedp, methods, shift_mask))
715 return false;
716
717 emit_label (done_label);
718 return true;
719}
720
721/* Subroutine of expand_binop. Perform a double word multiplication of
722 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
723 as the target's word_mode. This function return NULL_RTX if anything
724 goes wrong, in which case it may have already emitted instructions
725 which need to be deleted.
726
727 If we want to multiply two two-word values and have normal and widening
728 multiplies of single-word values, we can do this with three smaller
729 multiplications.
730
731 The multiplication proceeds as follows:
732 _______________________
733 [__op0_high_|__op0_low__]
734 _______________________
735 * [__op1_high_|__op1_low__]
736 _______________________________________________
737 _______________________
738 (1) [__op0_low__*__op1_low__]
739 _______________________
740 (2a) [__op0_low__*__op1_high_]
741 _______________________
742 (2b) [__op0_high_*__op1_low__]
743 _______________________
744 (3) [__op0_high_*__op1_high_]
745
746
747 This gives a 4-word result. Since we are only interested in the
748 lower 2 words, partial result (3) and the upper words of (2a) and
749 (2b) don't need to be calculated. Hence (2a) and (2b) can be
750 calculated using non-widening multiplication.
751
752 (1), however, needs to be calculated with an unsigned widening
753 multiplication. If this operation is not directly supported we
754 try using a signed widening multiplication and adjust the result.
755 This adjustment works as follows:
756
757 If both operands are positive then no adjustment is needed.
758
759 If the operands have different signs, for example op0_low < 0 and
760 op1_low >= 0, the instruction treats the most significant bit of
761 op0_low as a sign bit instead of a bit with significance
762 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
763 with 2**BITS_PER_WORD - op0_low, and two's complements the
764 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
765 the result.
766
767 Similarly, if both operands are negative, we need to add
768 (op0_low + op1_low) * 2**BITS_PER_WORD.
769
770 We use a trick to adjust quickly. We logically shift op0_low right
771 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
772 op0_high (op1_high) before it is used to calculate 2b (2a). If no
773 logical shift exists, we do an arithmetic right shift and subtract
774 the 0 or -1. */
775
776static rtx
777expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
778 bool umulp, enum optab_methods methods)
779{
780 int low = (WORDS_BIG_ENDIAN ? 1 : 0);
781 int high = (WORDS_BIG_ENDIAN ? 0 : 1);
782 rtx wordm1 = umulp ? NULL_RTX : GEN_INT (BITS_PER_WORD - 1);
783 rtx product, adjust, product_high, temp;
784
785 rtx op0_high = operand_subword_force (op0, high, mode);
786 rtx op0_low = operand_subword_force (op0, low, mode);
787 rtx op1_high = operand_subword_force (op1, high, mode);
788 rtx op1_low = operand_subword_force (op1, low, mode);
789
790 /* If we're using an unsigned multiply to directly compute the product
791 of the low-order words of the operands and perform any required
792 adjustments of the operands, we begin by trying two more multiplications
793 and then computing the appropriate sum.
794
795 We have checked above that the required addition is provided.
796 Full-word addition will normally always succeed, especially if
797 it is provided at all, so we don't worry about its failure. The
798 multiplication may well fail, however, so we do handle that. */
799
800 if (!umulp)
801 {
802 /* ??? This could be done with emit_store_flag where available. */
803 temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
804 NULL_RTX, 1, methods);
805 if (temp)
806 op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
807 NULL_RTX, 0, OPTAB_DIRECT);
808 else
809 {
810 temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
811 NULL_RTX, 0, methods);
812 if (!temp)
813 return NULL_RTX;
814 op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
815 NULL_RTX, 0, OPTAB_DIRECT);
816 }
817
818 if (!op0_high)
819 return NULL_RTX;
820 }
821
822 adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
823 NULL_RTX, 0, OPTAB_DIRECT);
824 if (!adjust)
825 return NULL_RTX;
826
827 /* OP0_HIGH should now be dead. */
828
829 if (!umulp)
830 {
831 /* ??? This could be done with emit_store_flag where available. */
832 temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
833 NULL_RTX, 1, methods);
834 if (temp)
835 op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
836 NULL_RTX, 0, OPTAB_DIRECT);
837 else
838 {
839 temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
840 NULL_RTX, 0, methods);
841 if (!temp)
842 return NULL_RTX;
843 op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
844 NULL_RTX, 0, OPTAB_DIRECT);
845 }
846
847 if (!op1_high)
848 return NULL_RTX;
849 }
850
851 temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
852 NULL_RTX, 0, OPTAB_DIRECT);
853 if (!temp)
854 return NULL_RTX;
855
856 /* OP1_HIGH should now be dead. */
857
858 adjust = expand_binop (word_mode, add_optab, adjust, temp,
859 NULL_RTX, 0, OPTAB_DIRECT);
860
861 if (target && !REG_P (target))
862 target = NULL_RTX;
863
864 /* *_widen_optab needs to determine operand mode, make sure at least
865 one operand has non-VOID mode. */
866 if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
867 op0_low = force_reg (word_mode, op0_low);
868
869 if (umulp)
870 product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
871 target, 1, OPTAB_DIRECT);
872 else
873 product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
874 target, 1, OPTAB_DIRECT);
875
876 if (!product)
877 return NULL_RTX;
878
879 product_high = operand_subword (product, high, 1, mode);
880 adjust = expand_binop (word_mode, add_optab, product_high, adjust,
881 NULL_RTX, 0, OPTAB_DIRECT);
882 emit_move_insn (product_high, adjust);
883 return product;
884}
885
886/* Wrapper around expand_binop which takes an rtx code to specify
887 the operation to perform, not an optab pointer. All other
888 arguments are the same. */
889rtx
890expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
891 rtx op1, rtx target, int unsignedp,
892 enum optab_methods methods)
893{
894 optab binop = code_to_optab (code);
895 gcc_assert (binop);
896
897 return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
898}
899
900/* Return whether OP0 and OP1 should be swapped when expanding a commutative
901 binop. Order them according to commutative_operand_precedence and, if
902 possible, try to put TARGET or a pseudo first. */
903static bool
904swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
905{
906 int op0_prec = commutative_operand_precedence (op0);
907 int op1_prec = commutative_operand_precedence (op1);
908
909 if (op0_prec < op1_prec)
910 return true;
911
912 if (op0_prec > op1_prec)
913 return false;
914
915 /* With equal precedence, both orders are ok, but it is better if the
916 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
917 if (target == 0 || REG_P (target))
918 return (REG_P (op1) && !REG_P (op0)) || target == op1;
919 else
920 return rtx_equal_p (op1, target);
921}
922
923/* Return true if BINOPTAB implements a shift operation. */
924
925static bool
926shift_optab_p (optab binoptab)
927{
928 switch (optab_to_code (binoptab))
929 {
930 case ASHIFT:
931 case SS_ASHIFT:
932 case US_ASHIFT:
933 case ASHIFTRT:
934 case LSHIFTRT:
935 case ROTATE:
936 case ROTATERT:
937 return true;
938
939 default:
940 return false;
941 }
942}
943
944/* Return true if BINOPTAB implements a commutative binary operation. */
945
946static bool
947commutative_optab_p (optab binoptab)
948{
949 return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
950 || binoptab == smul_widen_optab
951 || binoptab == umul_widen_optab
952 || binoptab == smul_highpart_optab
953 || binoptab == umul_highpart_optab);
954}
955
956/* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
957 optimizing, and if the operand is a constant that costs more than
958 1 instruction, force the constant into a register and return that
959 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
960
961static rtx
962avoid_expensive_constant (machine_mode mode, optab binoptab,
963 int opn, rtx x, bool unsignedp)
964{
965 bool speed = optimize_insn_for_speed_p ();
966
967 if (mode != VOIDmode
968 && optimize
969 && CONSTANT_P (x)
970 && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
971 > set_src_cost (x, mode, speed)))
972 {
973 if (CONST_INT_P (x))
974 {
975 HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
976 if (intval != INTVAL (x))
977 x = GEN_INT (intval);
978 }
979 else
980 x = convert_modes (mode, VOIDmode, x, unsignedp);
981 x = force_reg (mode, x);
982 }
983 return x;
984}
985
986/* Helper function for expand_binop: handle the case where there
987 is an insn ICODE that directly implements the indicated operation.
988 Returns null if this is not possible. */
989static rtx
990expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
991 rtx op0, rtx op1,
992 rtx target, int unsignedp, enum optab_methods methods,
993 rtx_insn *last)
994{
995 machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
996 machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
997 machine_mode mode0, mode1, tmp_mode;
998 struct expand_operand ops[3];
999 bool commutative_p;
1000 rtx_insn *pat;
1001 rtx xop0 = op0, xop1 = op1;
1002 bool canonicalize_op1 = false;
1003
1004 /* If it is a commutative operator and the modes would match
1005 if we would swap the operands, we can save the conversions. */
1006 commutative_p = commutative_optab_p (binoptab);
1007 if (commutative_p
1008 && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1009 && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode1)
1010 std::swap (xop0, xop1);
1011
1012 /* If we are optimizing, force expensive constants into a register. */
1013 xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1014 if (!shift_optab_p (binoptab))
1015 xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1016 else
1017 /* Shifts and rotates often use a different mode for op1 from op0;
1018 for VOIDmode constants we don't know the mode, so force it
1019 to be canonicalized using convert_modes. */
1020 canonicalize_op1 = true;
1021
1022 /* In case the insn wants input operands in modes different from
1023 those of the actual operands, convert the operands. It would
1024 seem that we don't need to convert CONST_INTs, but we do, so
1025 that they're properly zero-extended, sign-extended or truncated
1026 for their mode. */
1027
1028 mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1029 if (xmode0 != VOIDmode && xmode0 != mode0)
1030 {
1031 xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1032 mode0 = xmode0;
1033 }
1034
1035 mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1036 ? GET_MODE (xop1) : mode);
1037 if (xmode1 != VOIDmode && xmode1 != mode1)
1038 {
1039 xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1040 mode1 = xmode1;
1041 }
1042
1043 /* If operation is commutative,
1044 try to make the first operand a register.
1045 Even better, try to make it the same as the target.
1046 Also try to make the last operand a constant. */
1047 if (commutative_p
1048 && swap_commutative_operands_with_target (target, xop0, xop1))
1049 std::swap (xop0, xop1);
1050
1051 /* Now, if insn's predicates don't allow our operands, put them into
1052 pseudo regs. */
1053
1054 if (binoptab == vec_pack_trunc_optab
1055 || binoptab == vec_pack_usat_optab
1056 || binoptab == vec_pack_ssat_optab
1057 || binoptab == vec_pack_ufix_trunc_optab
1058 || binoptab == vec_pack_sfix_trunc_optab)
1059 {
1060 /* The mode of the result is different then the mode of the
1061 arguments. */
1062 tmp_mode = insn_data[(int) icode].operand[0].mode;
1063 if (VECTOR_MODE_P (mode)
1064 && GET_MODE_NUNITS (tmp_mode) != 2 * GET_MODE_NUNITS (mode))
1065 {
1066 delete_insns_since (last);
1067 return NULL_RTX;
1068 }
1069 }
1070 else
1071 tmp_mode = mode;
1072
1073 create_output_operand (&ops[0], target, tmp_mode);
1074 create_input_operand (&ops[1], xop0, mode0);
1075 create_input_operand (&ops[2], xop1, mode1);
1076 pat = maybe_gen_insn (icode, 3, ops);
1077 if (pat)
1078 {
1079 /* If PAT is composed of more than one insn, try to add an appropriate
1080 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1081 operand, call expand_binop again, this time without a target. */
1082 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1083 && ! add_equal_note (pat, ops[0].value,
1084 optab_to_code (binoptab),
1085 ops[1].value, ops[2].value))
1086 {
1087 delete_insns_since (last);
1088 return expand_binop (mode, binoptab, op0, op1, NULL_RTX,
1089 unsignedp, methods);
1090 }
1091
1092 emit_insn (pat);
1093 return ops[0].value;
1094 }
1095 delete_insns_since (last);
1096 return NULL_RTX;
1097}
1098
1099/* Generate code to perform an operation specified by BINOPTAB
1100 on operands OP0 and OP1, with result having machine-mode MODE.
1101
1102 UNSIGNEDP is for the case where we have to widen the operands
1103 to perform the operation. It says to use zero-extension.
1104
1105 If TARGET is nonzero, the value
1106 is generated there, if it is convenient to do so.
1107 In all cases an rtx is returned for the locus of the value;
1108 this may or may not be TARGET. */
1109
1110rtx
1111expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1112 rtx target, int unsignedp, enum optab_methods methods)
1113{
1114 enum optab_methods next_methods
1115 = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1116 ? OPTAB_WIDEN : methods);
1117 enum mode_class mclass;
1118 enum insn_code icode;
1119 machine_mode wider_mode;
1120 scalar_int_mode int_mode;
1121 rtx libfunc;
1122 rtx temp;
1123 rtx_insn *entry_last = get_last_insn ();
1124 rtx_insn *last;
1125
1126 mclass = GET_MODE_CLASS (mode);
1127
1128 /* If subtracting an integer constant, convert this into an addition of
1129 the negated constant. */
1130
1131 if (binoptab == sub_optab && CONST_INT_P (op1))
1132 {
1133 op1 = negate_rtx (mode, op1);
1134 binoptab = add_optab;
1135 }
1136 /* For shifts, constant invalid op1 might be expanded from different
1137 mode than MODE. As those are invalid, force them to a register
1138 to avoid further problems during expansion. */
1139 else if (CONST_INT_P (op1)
1140 && shift_optab_p (binoptab)
1141 && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1142 {
1143 op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1144 op1 = force_reg (GET_MODE_INNER (mode), op1);
1145 }
1146
1147 /* Record where to delete back to if we backtrack. */
1148 last = get_last_insn ();
1149
1150 /* If we can do it with a three-operand insn, do so. */
1151
1152 if (methods != OPTAB_MUST_WIDEN)
1153 {
1154 if (convert_optab_p (binoptab))
1155 {
1156 machine_mode from_mode = widened_mode (mode, op0, op1);
1157 icode = find_widening_optab_handler (binoptab, mode, from_mode);
1158 }
1159 else
1160 icode = optab_handler (binoptab, mode);
1161 if (icode != CODE_FOR_nothing)
1162 {
1163 temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1164 target, unsignedp, methods, last);
1165 if (temp)
1166 return temp;
1167 }
1168 }
1169
1170 /* If we were trying to rotate, and that didn't work, try rotating
1171 the other direction before falling back to shifts and bitwise-or. */
1172 if (((binoptab == rotl_optab
1173 && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
1174 || (binoptab == rotr_optab
1175 && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
1176 && is_int_mode (mode, &int_mode))
1177 {
1178 optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1179 rtx newop1;
1180 unsigned int bits = GET_MODE_PRECISION (int_mode);
1181
1182 if (CONST_INT_P (op1))
1183 newop1 = GEN_INT (bits - INTVAL (op1));
1184 else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1185 newop1 = negate_rtx (GET_MODE (op1), op1);
1186 else
1187 newop1 = expand_binop (GET_MODE (op1), sub_optab,
1188 gen_int_mode (bits, GET_MODE (op1)), op1,
1189 NULL_RTX, unsignedp, OPTAB_DIRECT);
1190
1191 temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
1192 target, unsignedp, methods, last);
1193 if (temp)
1194 return temp;
1195 }
1196
1197 /* If this is a multiply, see if we can do a widening operation that
1198 takes operands of this mode and makes a wider mode. */
1199
1200 if (binoptab == smul_optab
1201 && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
1202 && (convert_optab_handler ((unsignedp
1203 ? umul_widen_optab
1204 : smul_widen_optab),
1205 wider_mode, mode) != CODE_FOR_nothing))
1206 {
1207 /* *_widen_optab needs to determine operand mode, make sure at least
1208 one operand has non-VOID mode. */
1209 if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1210 op0 = force_reg (mode, op0);
1211 temp = expand_binop (wider_mode,
1212 unsignedp ? umul_widen_optab : smul_widen_optab,
1213 op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1214
1215 if (temp != 0)
1216 {
1217 if (GET_MODE_CLASS (mode) == MODE_INT
1218 && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1219 return gen_lowpart (mode, temp);
1220 else
1221 return convert_to_mode (mode, temp, unsignedp);
1222 }
1223 }
1224
1225 /* If this is a vector shift by a scalar, see if we can do a vector
1226 shift by a vector. If so, broadcast the scalar into a vector. */
1227 if (mclass == MODE_VECTOR_INT)
1228 {
1229 optab otheroptab = unknown_optab;
1230
1231 if (binoptab == ashl_optab)
1232 otheroptab = vashl_optab;
1233 else if (binoptab == ashr_optab)
1234 otheroptab = vashr_optab;
1235 else if (binoptab == lshr_optab)
1236 otheroptab = vlshr_optab;
1237 else if (binoptab == rotl_optab)
1238 otheroptab = vrotl_optab;
1239 else if (binoptab == rotr_optab)
1240 otheroptab = vrotr_optab;
1241
1242 if (otheroptab
1243 && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
1244 {
1245 /* The scalar may have been extended to be too wide. Truncate
1246 it back to the proper size to fit in the broadcast vector. */
1247 scalar_mode inner_mode = GET_MODE_INNER (mode);
1248 if (!CONST_INT_P (op1)
1249 && (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (op1)))
1250 > GET_MODE_BITSIZE (inner_mode)))
1251 op1 = force_reg (inner_mode,
1252 simplify_gen_unary (TRUNCATE, inner_mode, op1,
1253 GET_MODE (op1)));
1254 rtx vop1 = expand_vector_broadcast (mode, op1);
1255 if (vop1)
1256 {
1257 temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
1258 target, unsignedp, methods, last);
1259 if (temp)
1260 return temp;
1261 }
1262 }
1263 }
1264
1265 /* Look for a wider mode of the same class for which we think we
1266 can open-code the operation. Check for a widening multiply at the
1267 wider mode as well. */
1268
1269 if (CLASS_HAS_WIDER_MODES_P (mclass)
1270 && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1271 FOR_EACH_WIDER_MODE (wider_mode, mode)
1272 {
1273 machine_mode next_mode;
1274 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1275 || (binoptab == smul_optab
1276 && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
1277 && (find_widening_optab_handler ((unsignedp
1278 ? umul_widen_optab
1279 : smul_widen_optab),
1280 next_mode, mode)
1281 != CODE_FOR_nothing)))
1282 {
1283 rtx xop0 = op0, xop1 = op1;
1284 int no_extend = 0;
1285
1286 /* For certain integer operations, we need not actually extend
1287 the narrow operands, as long as we will truncate
1288 the results to the same narrowness. */
1289
1290 if ((binoptab == ior_optab || binoptab == and_optab
1291 || binoptab == xor_optab
1292 || binoptab == add_optab || binoptab == sub_optab
1293 || binoptab == smul_optab || binoptab == ashl_optab)
1294 && mclass == MODE_INT)
1295 {
1296 no_extend = 1;
1297 xop0 = avoid_expensive_constant (mode, binoptab, 0,
1298 xop0, unsignedp);
1299 if (binoptab != ashl_optab)
1300 xop1 = avoid_expensive_constant (mode, binoptab, 1,
1301 xop1, unsignedp);
1302 }
1303
1304 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1305
1306 /* The second operand of a shift must always be extended. */
1307 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1308 no_extend && binoptab != ashl_optab);
1309
1310 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1311 unsignedp, OPTAB_DIRECT);
1312 if (temp)
1313 {
1314 if (mclass != MODE_INT
1315 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1316 {
1317 if (target == 0)
1318 target = gen_reg_rtx (mode);
1319 convert_move (target, temp, 0);
1320 return target;
1321 }
1322 else
1323 return gen_lowpart (mode, temp);
1324 }
1325 else
1326 delete_insns_since (last);
1327 }
1328 }
1329
1330 /* If operation is commutative,
1331 try to make the first operand a register.
1332 Even better, try to make it the same as the target.
1333 Also try to make the last operand a constant. */
1334 if (commutative_optab_p (binoptab)
1335 && swap_commutative_operands_with_target (target, op0, op1))
1336 std::swap (op0, op1);
1337
1338 /* These can be done a word at a time. */
1339 if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1340 && is_int_mode (mode, &int_mode)
1341 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
1342 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1343 {
1344 int i;
1345 rtx_insn *insns;
1346
1347 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1348 won't be accurate, so use a new target. */
1349 if (target == 0
1350 || target == op0
1351 || target == op1
1352 || !valid_multiword_target_p (target))
1353 target = gen_reg_rtx (int_mode);
1354
1355 start_sequence ();
1356
1357 /* Do the actual arithmetic. */
1358 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
1359 {
1360 rtx target_piece = operand_subword (target, i, 1, int_mode);
1361 rtx x = expand_binop (word_mode, binoptab,
1362 operand_subword_force (op0, i, int_mode),
1363 operand_subword_force (op1, i, int_mode),
1364 target_piece, unsignedp, next_methods);
1365
1366 if (x == 0)
1367 break;
1368
1369 if (target_piece != x)
1370 emit_move_insn (target_piece, x);
1371 }
1372
1373 insns = get_insns ();
1374 end_sequence ();
1375
1376 if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
1377 {
1378 emit_insn (insns);
1379 return target;
1380 }
1381 }
1382
1383 /* Synthesize double word shifts from single word shifts. */
1384 if ((binoptab == lshr_optab || binoptab == ashl_optab
1385 || binoptab == ashr_optab)
1386 && is_int_mode (mode, &int_mode)
1387 && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1388 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1389 && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
1390 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1391 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1392 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1393 {
1394 unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1395 scalar_int_mode op1_mode;
1396
1397 double_shift_mask = targetm.shift_truncation_mask (int_mode);
1398 shift_mask = targetm.shift_truncation_mask (word_mode);
1399 op1_mode = (GET_MODE (op1) != VOIDmode
1400 ? as_a <scalar_int_mode> (GET_MODE (op1))
1401 : word_mode);
1402
1403 /* Apply the truncation to constant shifts. */
1404 if (double_shift_mask > 0 && CONST_INT_P (op1))
1405 op1 = GEN_INT (INTVAL (op1) & double_shift_mask);
1406
1407 if (op1 == CONST0_RTX (op1_mode))
1408 return op0;
1409
1410 /* Make sure that this is a combination that expand_doubleword_shift
1411 can handle. See the comments there for details. */
1412 if (double_shift_mask == 0
1413 || (shift_mask == BITS_PER_WORD - 1
1414 && double_shift_mask == BITS_PER_WORD * 2 - 1))
1415 {
1416 rtx_insn *insns;
1417 rtx into_target, outof_target;
1418 rtx into_input, outof_input;
1419 int left_shift, outof_word;
1420
1421 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1422 won't be accurate, so use a new target. */
1423 if (target == 0
1424 || target == op0
1425 || target == op1
1426 || !valid_multiword_target_p (target))
1427 target = gen_reg_rtx (int_mode);
1428
1429 start_sequence ();
1430
1431 /* OUTOF_* is the word we are shifting bits away from, and
1432 INTO_* is the word that we are shifting bits towards, thus
1433 they differ depending on the direction of the shift and
1434 WORDS_BIG_ENDIAN. */
1435
1436 left_shift = binoptab == ashl_optab;
1437 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1438
1439 outof_target = operand_subword (target, outof_word, 1, int_mode);
1440 into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1441
1442 outof_input = operand_subword_force (op0, outof_word, int_mode);
1443 into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1444
1445 if (expand_doubleword_shift (op1_mode, binoptab,
1446 outof_input, into_input, op1,
1447 outof_target, into_target,
1448 unsignedp, next_methods, shift_mask))
1449 {
1450 insns = get_insns ();
1451 end_sequence ();
1452
1453 emit_insn (insns);
1454 return target;
1455 }
1456 end_sequence ();
1457 }
1458 }
1459
1460 /* Synthesize double word rotates from single word shifts. */
1461 if ((binoptab == rotl_optab || binoptab == rotr_optab)
1462 && is_int_mode (mode, &int_mode)
1463 && CONST_INT_P (op1)
1464 && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
1465 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1466 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1467 {
1468 rtx_insn *insns;
1469 rtx into_target, outof_target;
1470 rtx into_input, outof_input;
1471 rtx inter;
1472 int shift_count, left_shift, outof_word;
1473
1474 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1475 won't be accurate, so use a new target. Do this also if target is not
1476 a REG, first because having a register instead may open optimization
1477 opportunities, and second because if target and op0 happen to be MEMs
1478 designating the same location, we would risk clobbering it too early
1479 in the code sequence we generate below. */
1480 if (target == 0
1481 || target == op0
1482 || target == op1
1483 || !REG_P (target)
1484 || !valid_multiword_target_p (target))
1485 target = gen_reg_rtx (int_mode);
1486
1487 start_sequence ();
1488
1489 shift_count = INTVAL (op1);
1490
1491 /* OUTOF_* is the word we are shifting bits away from, and
1492 INTO_* is the word that we are shifting bits towards, thus
1493 they differ depending on the direction of the shift and
1494 WORDS_BIG_ENDIAN. */
1495
1496 left_shift = (binoptab == rotl_optab);
1497 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1498
1499 outof_target = operand_subword (target, outof_word, 1, int_mode);
1500 into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1501
1502 outof_input = operand_subword_force (op0, outof_word, int_mode);
1503 into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1504
1505 if (shift_count == BITS_PER_WORD)
1506 {
1507 /* This is just a word swap. */
1508 emit_move_insn (outof_target, into_input);
1509 emit_move_insn (into_target, outof_input);
1510 inter = const0_rtx;
1511 }
1512 else
1513 {
1514 rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1515 rtx first_shift_count, second_shift_count;
1516 optab reverse_unsigned_shift, unsigned_shift;
1517
1518 reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1519 ? lshr_optab : ashl_optab);
1520
1521 unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1522 ? ashl_optab : lshr_optab);
1523
1524 if (shift_count > BITS_PER_WORD)
1525 {
1526 first_shift_count = GEN_INT (shift_count - BITS_PER_WORD);
1527 second_shift_count = GEN_INT (2 * BITS_PER_WORD - shift_count);
1528 }
1529 else
1530 {
1531 first_shift_count = GEN_INT (BITS_PER_WORD - shift_count);
1532 second_shift_count = GEN_INT (shift_count);
1533 }
1534
1535 into_temp1 = expand_binop (word_mode, unsigned_shift,
1536 outof_input, first_shift_count,
1537 NULL_RTX, unsignedp, next_methods);
1538 into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1539 into_input, second_shift_count,
1540 NULL_RTX, unsignedp, next_methods);
1541
1542 if (into_temp1 != 0 && into_temp2 != 0)
1543 inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2,
1544 into_target, unsignedp, next_methods);
1545 else
1546 inter = 0;
1547
1548 if (inter != 0 && inter != into_target)
1549 emit_move_insn (into_target, inter);
1550
1551 outof_temp1 = expand_binop (word_mode, unsigned_shift,
1552 into_input, first_shift_count,
1553 NULL_RTX, unsignedp, next_methods);
1554 outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1555 outof_input, second_shift_count,
1556 NULL_RTX, unsignedp, next_methods);
1557
1558 if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1559 inter = expand_binop (word_mode, ior_optab,
1560 outof_temp1, outof_temp2,
1561 outof_target, unsignedp, next_methods);
1562
1563 if (inter != 0 && inter != outof_target)
1564 emit_move_insn (outof_target, inter);
1565 }
1566
1567 insns = get_insns ();
1568 end_sequence ();
1569
1570 if (inter != 0)
1571 {
1572 emit_insn (insns);
1573 return target;
1574 }
1575 }
1576
1577 /* These can be done a word at a time by propagating carries. */
1578 if ((binoptab == add_optab || binoptab == sub_optab)
1579 && is_int_mode (mode, &int_mode)
1580 && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
1581 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1582 {
1583 unsigned int i;
1584 optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
1585 const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
1586 rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
1587 rtx xop0, xop1, xtarget;
1588
1589 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1590 value is one of those, use it. Otherwise, use 1 since it is the
1591 one easiest to get. */
1592#if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1593 int normalizep = STORE_FLAG_VALUE;
1594#else
1595 int normalizep = 1;
1596#endif
1597
1598 /* Prepare the operands. */
1599 xop0 = force_reg (int_mode, op0);
1600 xop1 = force_reg (int_mode, op1);
1601
1602 xtarget = gen_reg_rtx (int_mode);
1603
1604 if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1605 target = xtarget;
1606
1607 /* Indicate for flow that the entire target reg is being set. */
1608 if (REG_P (target))
1609 emit_clobber (xtarget);
1610
1611 /* Do the actual arithmetic. */
1612 for (i = 0; i < nwords; i++)
1613 {
1614 int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
1615 rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
1616 rtx op0_piece = operand_subword_force (xop0, index, int_mode);
1617 rtx op1_piece = operand_subword_force (xop1, index, int_mode);
1618 rtx x;
1619
1620 /* Main add/subtract of the input operands. */
1621 x = expand_binop (word_mode, binoptab,
1622 op0_piece, op1_piece,
1623 target_piece, unsignedp, next_methods);
1624 if (x == 0)
1625 break;
1626
1627 if (i + 1 < nwords)
1628 {
1629 /* Store carry from main add/subtract. */
1630 carry_out = gen_reg_rtx (word_mode);
1631 carry_out = emit_store_flag_force (carry_out,
1632 (binoptab == add_optab
1633 ? LT : GT),
1634 x, op0_piece,
1635 word_mode, 1, normalizep);
1636 }
1637
1638 if (i > 0)
1639 {
1640 rtx newx;
1641
1642 /* Add/subtract previous carry to main result. */
1643 newx = expand_binop (word_mode,
1644 normalizep == 1 ? binoptab : otheroptab,
1645 x, carry_in,
1646 NULL_RTX, 1, next_methods);
1647
1648 if (i + 1 < nwords)
1649 {
1650 /* Get out carry from adding/subtracting carry in. */
1651 rtx carry_tmp = gen_reg_rtx (word_mode);
1652 carry_tmp = emit_store_flag_force (carry_tmp,
1653 (binoptab == add_optab
1654 ? LT : GT),
1655 newx, x,
1656 word_mode, 1, normalizep);
1657
1658 /* Logical-ior the two poss. carry together. */
1659 carry_out = expand_binop (word_mode, ior_optab,
1660 carry_out, carry_tmp,
1661 carry_out, 0, next_methods);
1662 if (carry_out == 0)
1663 break;
1664 }
1665 emit_move_insn (target_piece, newx);
1666 }
1667 else
1668 {
1669 if (x != target_piece)
1670 emit_move_insn (target_piece, x);
1671 }
1672
1673 carry_in = carry_out;
1674 }
1675
1676 if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
1677 {
1678 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
1679 || ! rtx_equal_p (target, xtarget))
1680 {
1681 rtx_insn *temp = emit_move_insn (target, xtarget);
1682
1683 set_dst_reg_note (temp, REG_EQUAL,
1684 gen_rtx_fmt_ee (optab_to_code (binoptab),
1685 int_mode, copy_rtx (xop0),
1686 copy_rtx (xop1)),
1687 target);
1688 }
1689 else
1690 target = xtarget;
1691
1692 return target;
1693 }
1694
1695 else
1696 delete_insns_since (last);
1697 }
1698
1699 /* Attempt to synthesize double word multiplies using a sequence of word
1700 mode multiplications. We first attempt to generate a sequence using a
1701 more efficient unsigned widening multiply, and if that fails we then
1702 try using a signed widening multiply. */
1703
1704 if (binoptab == smul_optab
1705 && is_int_mode (mode, &int_mode)
1706 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1707 && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
1708 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
1709 {
1710 rtx product = NULL_RTX;
1711 if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
1712 != CODE_FOR_nothing)
1713 {
1714 product = expand_doubleword_mult (int_mode, op0, op1, target,
1715 true, methods);
1716 if (!product)
1717 delete_insns_since (last);
1718 }
1719
1720 if (product == NULL_RTX
1721 && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
1722 != CODE_FOR_nothing))
1723 {
1724 product = expand_doubleword_mult (int_mode, op0, op1, target,
1725 false, methods);
1726 if (!product)
1727 delete_insns_since (last);
1728 }
1729
1730 if (product != NULL_RTX)
1731 {
1732 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
1733 {
1734 rtx_insn *move = emit_move_insn (target ? target : product,
1735 product);
1736 set_dst_reg_note (move,
1737 REG_EQUAL,
1738 gen_rtx_fmt_ee (MULT, int_mode,
1739 copy_rtx (op0),
1740 copy_rtx (op1)),
1741 target ? target : product);
1742 }
1743 return product;
1744 }
1745 }
1746
1747 /* It can't be open-coded in this mode.
1748 Use a library call if one is available and caller says that's ok. */
1749
1750 libfunc = optab_libfunc (binoptab, mode);
1751 if (libfunc
1752 && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
1753 {
1754 rtx_insn *insns;
1755 rtx op1x = op1;
1756 machine_mode op1_mode = mode;
1757 rtx value;
1758
1759 start_sequence ();
1760
1761 if (shift_optab_p (binoptab))
1762 {
1763 op1_mode = targetm.libgcc_shift_count_mode ();
1764 /* Specify unsigned here,
1765 since negative shift counts are meaningless. */
1766 op1x = convert_to_mode (op1_mode, op1, 1);
1767 }
1768
1769 if (GET_MODE (op0) != VOIDmode
1770 && GET_MODE (op0) != mode)
1771 op0 = convert_to_mode (mode, op0, unsignedp);
1772
1773 /* Pass 1 for NO_QUEUE so we don't lose any increments
1774 if the libcall is cse'd or moved. */
1775 value = emit_library_call_value (libfunc,
1776 NULL_RTX, LCT_CONST, mode,
1777 op0, mode, op1x, op1_mode);
1778
1779 insns = get_insns ();
1780 end_sequence ();
1781
1782 bool trapv = trapv_binoptab_p (binoptab);
1783 target = gen_reg_rtx (mode);
1784 emit_libcall_block_1 (insns, target, value,
1785 trapv ? NULL_RTX
1786 : gen_rtx_fmt_ee (optab_to_code (binoptab),
1787 mode, op0, op1), trapv);
1788
1789 return target;
1790 }
1791
1792 delete_insns_since (last);
1793
1794 /* It can't be done in this mode. Can we do it in a wider mode? */
1795
1796 if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
1797 || methods == OPTAB_MUST_WIDEN))
1798 {
1799 /* Caller says, don't even try. */
1800 delete_insns_since (entry_last);
1801 return 0;
1802 }
1803
1804 /* Compute the value of METHODS to pass to recursive calls.
1805 Don't allow widening to be tried recursively. */
1806
1807 methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
1808
1809 /* Look for a wider mode of the same class for which it appears we can do
1810 the operation. */
1811
1812 if (CLASS_HAS_WIDER_MODES_P (mclass))
1813 {
1814 /* This code doesn't make sense for conversion optabs, since we
1815 wouldn't then want to extend the operands to be the same size
1816 as the result. */
1817 gcc_assert (!convert_optab_p (binoptab));
1818 FOR_EACH_WIDER_MODE (wider_mode, mode)
1819 {
1820 if (optab_handler (binoptab, wider_mode)
1821 || (methods == OPTAB_LIB
1822 && optab_libfunc (binoptab, wider_mode)))
1823 {
1824 rtx xop0 = op0, xop1 = op1;
1825 int no_extend = 0;
1826
1827 /* For certain integer operations, we need not actually extend
1828 the narrow operands, as long as we will truncate
1829 the results to the same narrowness. */
1830
1831 if ((binoptab == ior_optab || binoptab == and_optab
1832 || binoptab == xor_optab
1833 || binoptab == add_optab || binoptab == sub_optab
1834 || binoptab == smul_optab || binoptab == ashl_optab)
1835 && mclass == MODE_INT)
1836 no_extend = 1;
1837
1838 xop0 = widen_operand (xop0, wider_mode, mode,
1839 unsignedp, no_extend);
1840
1841 /* The second operand of a shift must always be extended. */
1842 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1843 no_extend && binoptab != ashl_optab);
1844
1845 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1846 unsignedp, methods);
1847 if (temp)
1848 {
1849 if (mclass != MODE_INT
1850 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1851 {
1852 if (target == 0)
1853 target = gen_reg_rtx (mode);
1854 convert_move (target, temp, 0);
1855 return target;
1856 }
1857 else
1858 return gen_lowpart (mode, temp);
1859 }
1860 else
1861 delete_insns_since (last);
1862 }
1863 }
1864 }
1865
1866 delete_insns_since (entry_last);
1867 return 0;
1868}
1869
1870/* Expand a binary operator which has both signed and unsigned forms.
1871 UOPTAB is the optab for unsigned operations, and SOPTAB is for
1872 signed operations.
1873
1874 If we widen unsigned operands, we may use a signed wider operation instead
1875 of an unsigned wider operation, since the result would be the same. */
1876
1877rtx
1878sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
1879 rtx op0, rtx op1, rtx target, int unsignedp,
1880 enum optab_methods methods)
1881{
1882 rtx temp;
1883 optab direct_optab = unsignedp ? uoptab : soptab;
1884 bool save_enable;
1885
1886 /* Do it without widening, if possible. */
1887 temp = expand_binop (mode, direct_optab, op0, op1, target,
1888 unsignedp, OPTAB_DIRECT);
1889 if (temp || methods == OPTAB_DIRECT)
1890 return temp;
1891
1892 /* Try widening to a signed int. Disable any direct use of any
1893 signed insn in the current mode. */
1894 save_enable = swap_optab_enable (soptab, mode, false);
1895
1896 temp = expand_binop (mode, soptab, op0, op1, target,
1897 unsignedp, OPTAB_WIDEN);
1898
1899 /* For unsigned operands, try widening to an unsigned int. */
1900 if (!temp && unsignedp)
1901 temp = expand_binop (mode, uoptab, op0, op1, target,
1902 unsignedp, OPTAB_WIDEN);
1903 if (temp || methods == OPTAB_WIDEN)
1904 goto egress;
1905
1906 /* Use the right width libcall if that exists. */
1907 temp = expand_binop (mode, direct_optab, op0, op1, target,
1908 unsignedp, OPTAB_LIB);
1909 if (temp || methods == OPTAB_LIB)
1910 goto egress;
1911
1912 /* Must widen and use a libcall, use either signed or unsigned. */
1913 temp = expand_binop (mode, soptab, op0, op1, target,
1914 unsignedp, methods);
1915 if (!temp && unsignedp)
1916 temp = expand_binop (mode, uoptab, op0, op1, target,
1917 unsignedp, methods);
1918
1919 egress:
1920 /* Undo the fiddling above. */
1921 if (save_enable)
1922 swap_optab_enable (soptab, mode, true);
1923 return temp;
1924}
1925
1926/* Generate code to perform an operation specified by UNOPPTAB
1927 on operand OP0, with two results to TARG0 and TARG1.
1928 We assume that the order of the operands for the instruction
1929 is TARG0, TARG1, OP0.
1930
1931 Either TARG0 or TARG1 may be zero, but what that means is that
1932 the result is not actually wanted. We will generate it into
1933 a dummy pseudo-reg and discard it. They may not both be zero.
1934
1935 Returns 1 if this operation can be performed; 0 if not. */
1936
1937int
1938expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
1939 int unsignedp)
1940{
1941 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
1942 enum mode_class mclass;
1943 machine_mode wider_mode;
1944 rtx_insn *entry_last = get_last_insn ();
1945 rtx_insn *last;
1946
1947 mclass = GET_MODE_CLASS (mode);
1948
1949 if (!targ0)
1950 targ0 = gen_reg_rtx (mode);
1951 if (!targ1)
1952 targ1 = gen_reg_rtx (mode);
1953
1954 /* Record where to go back to if we fail. */
1955 last = get_last_insn ();
1956
1957 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
1958 {
1959 struct expand_operand ops[3];
1960 enum insn_code icode = optab_handler (unoptab, mode);
1961
1962 create_fixed_operand (&ops[0], targ0);
1963 create_fixed_operand (&ops[1], targ1);
1964 create_convert_operand_from (&ops[2], op0, mode, unsignedp);
1965 if (maybe_expand_insn (icode, 3, ops))
1966 return 1;
1967 }
1968
1969 /* It can't be done in this mode. Can we do it in a wider mode? */
1970
1971 if (CLASS_HAS_WIDER_MODES_P (mclass))
1972 {
1973 FOR_EACH_WIDER_MODE (wider_mode, mode)
1974 {
1975 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
1976 {
1977 rtx t0 = gen_reg_rtx (wider_mode);
1978 rtx t1 = gen_reg_rtx (wider_mode);
1979 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
1980
1981 if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
1982 {
1983 convert_move (targ0, t0, unsignedp);
1984 convert_move (targ1, t1, unsignedp);
1985 return 1;
1986 }
1987 else
1988 delete_insns_since (last);
1989 }
1990 }
1991 }
1992
1993 delete_insns_since (entry_last);
1994 return 0;
1995}
1996
1997/* Generate code to perform an operation specified by BINOPTAB
1998 on operands OP0 and OP1, with two results to TARG1 and TARG2.
1999 We assume that the order of the operands for the instruction
2000 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2001 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2002
2003 Either TARG0 or TARG1 may be zero, but what that means is that
2004 the result is not actually wanted. We will generate it into
2005 a dummy pseudo-reg and discard it. They may not both be zero.
2006
2007 Returns 1 if this operation can be performed; 0 if not. */
2008
2009int
2010expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2011 int unsignedp)
2012{
2013 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2014 enum mode_class mclass;
2015 machine_mode wider_mode;
2016 rtx_insn *entry_last = get_last_insn ();
2017 rtx_insn *last;
2018
2019 mclass = GET_MODE_CLASS (mode);
2020
2021 if (!targ0)
2022 targ0 = gen_reg_rtx (mode);
2023 if (!targ1)
2024 targ1 = gen_reg_rtx (mode);
2025
2026 /* Record where to go back to if we fail. */
2027 last = get_last_insn ();
2028
2029 if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2030 {
2031 struct expand_operand ops[4];
2032 enum insn_code icode = optab_handler (binoptab, mode);
2033 machine_mode mode0 = insn_data[icode].operand[1].mode;
2034 machine_mode mode1 = insn_data[icode].operand[2].mode;
2035 rtx xop0 = op0, xop1 = op1;
2036
2037 /* If we are optimizing, force expensive constants into a register. */
2038 xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2039 xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2040
2041 create_fixed_operand (&ops[0], targ0);
2042 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2043 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
2044 create_fixed_operand (&ops[3], targ1);
2045 if (maybe_expand_insn (icode, 4, ops))
2046 return 1;
2047 delete_insns_since (last);
2048 }
2049
2050 /* It can't be done in this mode. Can we do it in a wider mode? */
2051
2052 if (CLASS_HAS_WIDER_MODES_P (mclass))
2053 {
2054 FOR_EACH_WIDER_MODE (wider_mode, mode)
2055 {
2056 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2057 {
2058 rtx t0 = gen_reg_rtx (wider_mode);
2059 rtx t1 = gen_reg_rtx (wider_mode);
2060 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2061 rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2062
2063 if (expand_twoval_binop (binoptab, cop0, cop1,
2064 t0, t1, unsignedp))
2065 {
2066 convert_move (targ0, t0, unsignedp);
2067 convert_move (targ1, t1, unsignedp);
2068 return 1;
2069 }
2070 else
2071 delete_insns_since (last);
2072 }
2073 }
2074 }
2075
2076 delete_insns_since (entry_last);
2077 return 0;
2078}
2079
2080/* Expand the two-valued library call indicated by BINOPTAB, but
2081 preserve only one of the values. If TARG0 is non-NULL, the first
2082 value is placed into TARG0; otherwise the second value is placed
2083 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2084 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2085 This routine assumes that the value returned by the library call is
2086 as if the return value was of an integral mode twice as wide as the
2087 mode of OP0. Returns 1 if the call was successful. */
2088
2089bool
2090expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2091 rtx targ0, rtx targ1, enum rtx_code code)
2092{
2093 machine_mode mode;
2094 machine_mode libval_mode;
2095 rtx libval;
2096 rtx_insn *insns;
2097 rtx libfunc;
2098
2099 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2100 gcc_assert (!targ0 != !targ1);
2101
2102 mode = GET_MODE (op0);
2103 libfunc = optab_libfunc (binoptab, mode);
2104 if (!libfunc)
2105 return false;
2106
2107 /* The value returned by the library function will have twice as
2108 many bits as the nominal MODE. */
2109 libval_mode = smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode));
2110 start_sequence ();
2111 libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2112 libval_mode,
2113 op0, mode,
2114 op1, mode);
2115 /* Get the part of VAL containing the value that we want. */
2116 libval = simplify_gen_subreg (mode, libval, libval_mode,
2117 targ0 ? 0 : GET_MODE_SIZE (mode));
2118 insns = get_insns ();
2119 end_sequence ();
2120 /* Move the into the desired location. */
2121 emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2122 gen_rtx_fmt_ee (code, mode, op0, op1));
2123
2124 return true;
2125}
2126
2127
2128/* Wrapper around expand_unop which takes an rtx code to specify
2129 the operation to perform, not an optab pointer. All other
2130 arguments are the same. */
2131rtx
2132expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2133 rtx target, int unsignedp)
2134{
2135 optab unop = code_to_optab (code);
2136 gcc_assert (unop);
2137
2138 return expand_unop (mode, unop, op0, target, unsignedp);
2139}
2140
2141/* Try calculating
2142 (clz:narrow x)
2143 as
2144 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2145
2146 A similar operation can be used for clrsb. UNOPTAB says which operation
2147 we are trying to expand. */
2148static rtx
2149widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2150{
2151 opt_scalar_int_mode wider_mode_iter;
2152 FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2153 {
2154 scalar_int_mode wider_mode = wider_mode_iter.require ();
2155 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2156 {
2157 rtx xop0, temp;
2158 rtx_insn *last;
2159
2160 last = get_last_insn ();
2161
2162 if (target == 0)
2163 target = gen_reg_rtx (mode);
2164 xop0 = widen_operand (op0, wider_mode, mode,
2165 unoptab != clrsb_optab, false);
2166 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2167 unoptab != clrsb_optab);
2168 if (temp != 0)
2169 temp = expand_binop
2170 (wider_mode, sub_optab, temp,
2171 gen_int_mode (GET_MODE_PRECISION (wider_mode)
2172 - GET_MODE_PRECISION (mode),
2173 wider_mode),
2174 target, true, OPTAB_DIRECT);
2175 if (temp == 0)
2176 delete_insns_since (last);
2177
2178 return temp;
2179 }
2180 }
2181 return 0;
2182}
2183
2184/* Try calculating clz of a double-word quantity as two clz's of word-sized
2185 quantities, choosing which based on whether the high word is nonzero. */
2186static rtx
2187expand_doubleword_clz (scalar_int_mode mode, rtx op0, rtx target)
2188{
2189 rtx xop0 = force_reg (mode, op0);
2190 rtx subhi = gen_highpart (word_mode, xop0);
2191 rtx sublo = gen_lowpart (word_mode, xop0);
2192 rtx_code_label *hi0_label = gen_label_rtx ();
2193 rtx_code_label *after_label = gen_label_rtx ();
2194 rtx_insn *seq;
2195 rtx temp, result;
2196
2197 /* If we were not given a target, use a word_mode register, not a
2198 'mode' register. The result will fit, and nobody is expecting
2199 anything bigger (the return type of __builtin_clz* is int). */
2200 if (!target)
2201 target = gen_reg_rtx (word_mode);
2202
2203 /* In any case, write to a word_mode scratch in both branches of the
2204 conditional, so we can ensure there is a single move insn setting
2205 'target' to tag a REG_EQUAL note on. */
2206 result = gen_reg_rtx (word_mode);
2207
2208 start_sequence ();
2209
2210 /* If the high word is not equal to zero,
2211 then clz of the full value is clz of the high word. */
2212 emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2213 word_mode, true, hi0_label);
2214
2215 temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true);
2216 if (!temp)
2217 goto fail;
2218
2219 if (temp != result)
2220 convert_move (result, temp, true);
2221
2222 emit_jump_insn (targetm.gen_jump (after_label));
2223 emit_barrier ();
2224
2225 /* Else clz of the full value is clz of the low word plus the number
2226 of bits in the high word. */
2227 emit_label (hi0_label);
2228
2229 temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true);
2230 if (!temp)
2231 goto fail;
2232 temp = expand_binop (word_mode, add_optab, temp,
2233 gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode),
2234 result, true, OPTAB_DIRECT);
2235 if (!temp)
2236 goto fail;
2237 if (temp != result)
2238 convert_move (result, temp, true);
2239
2240 emit_label (after_label);
2241 convert_move (target, result, true);
2242
2243 seq = get_insns ();
2244 end_sequence ();
2245
2246 add_equal_note (seq, target, CLZ, xop0, 0);
2247 emit_insn (seq);
2248 return target;
2249
2250 fail:
2251 end_sequence ();
2252 return 0;
2253}
2254
2255/* Try calculating popcount of a double-word quantity as two popcount's of
2256 word-sized quantities and summing up the results. */
2257static rtx
2258expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2259{
2260 rtx t0, t1, t;
2261 rtx_insn *seq;
2262
2263 start_sequence ();
2264
2265 t0 = expand_unop_direct (word_mode, popcount_optab,
2266 operand_subword_force (op0, 0, mode), NULL_RTX,
2267 true);
2268 t1 = expand_unop_direct (word_mode, popcount_optab,
2269 operand_subword_force (op0, 1, mode), NULL_RTX,
2270 true);
2271 if (!t0 || !t1)
2272 {
2273 end_sequence ();
2274 return NULL_RTX;
2275 }
2276
2277 /* If we were not given a target, use a word_mode register, not a
2278 'mode' register. The result will fit, and nobody is expecting
2279 anything bigger (the return type of __builtin_popcount* is int). */
2280 if (!target)
2281 target = gen_reg_rtx (word_mode);
2282
2283 t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
2284
2285 seq = get_insns ();
2286 end_sequence ();
2287
2288 add_equal_note (seq, t, POPCOUNT, op0, 0);
2289 emit_insn (seq);
2290 return t;
2291}
2292
2293/* Try calculating
2294 (parity:wide x)
2295 as
2296 (parity:narrow (low (x) ^ high (x))) */
2297static rtx
2298expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2299{
2300 rtx t = expand_binop (word_mode, xor_optab,
2301 operand_subword_force (op0, 0, mode),
2302 operand_subword_force (op0, 1, mode),
2303 NULL_RTX, 0, OPTAB_DIRECT);
2304 return expand_unop (word_mode, parity_optab, t, target, true);
2305}
2306
2307/* Try calculating
2308 (bswap:narrow x)
2309 as
2310 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2311static rtx
2312widen_bswap (scalar_int_mode mode, rtx op0, rtx target)
2313{
2314 rtx x;
2315 rtx_insn *last;
2316 opt_scalar_int_mode wider_mode_iter;
2317
2318 FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2319 if (optab_handler (bswap_optab, wider_mode_iter.require ())
2320 != CODE_FOR_nothing)
2321 break;
2322
2323 if (!wider_mode_iter.exists ())
2324 return NULL_RTX;
2325
2326 scalar_int_mode wider_mode = wider_mode_iter.require ();
2327 last = get_last_insn ();
2328
2329 x = widen_operand (op0, wider_mode, mode, true, true);
2330 x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2331
2332 gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2333 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2334 if (x != 0)
2335 x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2336 GET_MODE_BITSIZE (wider_mode)
2337 - GET_MODE_BITSIZE (mode),
2338 NULL_RTX, true);
2339
2340 if (x != 0)
2341 {
2342 if (target == 0)
2343 target = gen_reg_rtx (mode);
2344 emit_move_insn (target, gen_lowpart (mode, x));
2345 }
2346 else
2347 delete_insns_since (last);
2348
2349 return target;
2350}
2351
2352/* Try calculating bswap as two bswaps of two word-sized operands. */
2353
2354static rtx
2355expand_doubleword_bswap (machine_mode mode, rtx op, rtx target)
2356{
2357 rtx t0, t1;
2358
2359 t1 = expand_unop (word_mode, bswap_optab,
2360 operand_subword_force (op, 0, mode), NULL_RTX, true);
2361 t0 = expand_unop (word_mode, bswap_optab,
2362 operand_subword_force (op, 1, mode), NULL_RTX, true);
2363
2364 if (target == 0 || !valid_multiword_target_p (target))
2365 target = gen_reg_rtx (mode);
2366 if (REG_P (target))
2367 emit_clobber (target);
2368 emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2369 emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2370
2371 return target;
2372}
2373
2374/* Try calculating (parity x) as (and (popcount x) 1), where
2375 popcount can also be done in a wider mode. */
2376static rtx
2377expand_parity (scalar_int_mode mode, rtx op0, rtx target)
2378{
2379 enum mode_class mclass = GET_MODE_CLASS (mode);
2380 opt_scalar_int_mode wider_mode_iter;
2381 FOR_EACH_MODE_FROM (wider_mode_iter, mode)
2382 {
2383 scalar_int_mode wider_mode = wider_mode_iter.require ();
2384 if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2385 {
2386 rtx xop0, temp;
2387 rtx_insn *last;
2388
2389 last = get_last_insn ();
2390
2391 if (target == 0 || GET_MODE (target) != wider_mode)
2392 target = gen_reg_rtx (wider_mode);
2393
2394 xop0 = widen_operand (op0, wider_mode, mode, true, false);
2395 temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2396 true);
2397 if (temp != 0)
2398 temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2399 target, true, OPTAB_DIRECT);
2400
2401 if (temp)
2402 {
2403 if (mclass != MODE_INT
2404 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2405 return convert_to_mode (mode, temp, 0);
2406 else
2407 return gen_lowpart (mode, temp);
2408 }
2409 else
2410 delete_insns_since (last);
2411 }
2412 }
2413 return 0;
2414}
2415
2416/* Try calculating ctz(x) as K - clz(x & -x) ,
2417 where K is GET_MODE_PRECISION(mode) - 1.
2418
2419 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2420 don't have to worry about what the hardware does in that case. (If
2421 the clz instruction produces the usual value at 0, which is K, the
2422 result of this code sequence will be -1; expand_ffs, below, relies
2423 on this. It might be nice to have it be K instead, for consistency
2424 with the (very few) processors that provide a ctz with a defined
2425 value, but that would take one more instruction, and it would be
2426 less convenient for expand_ffs anyway. */
2427
2428static rtx
2429expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
2430{
2431 rtx_insn *seq;
2432 rtx temp;
2433
2434 if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2435 return 0;
2436
2437 start_sequence ();
2438
2439 temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
2440 if (temp)
2441 temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
2442 true, OPTAB_DIRECT);
2443 if (temp)
2444 temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
2445 if (temp)
2446 temp = expand_binop (mode, sub_optab,
2447 gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
2448 temp, target,
2449 true, OPTAB_DIRECT);
2450 if (temp == 0)
2451 {
2452 end_sequence ();
2453 return 0;
2454 }
2455
2456 seq = get_insns ();
2457 end_sequence ();
2458
2459 add_equal_note (seq, temp, CTZ, op0, 0);
2460 emit_insn (seq);
2461 return temp;
2462}
2463
2464
2465/* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2466 else with the sequence used by expand_clz.
2467
2468 The ffs builtin promises to return zero for a zero value and ctz/clz
2469 may have an undefined value in that case. If they do not give us a
2470 convenient value, we have to generate a test and branch. */
2471static rtx
2472expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
2473{
2474 HOST_WIDE_INT val = 0;
2475 bool defined_at_zero = false;
2476 rtx temp;
2477 rtx_insn *seq;
2478
2479 if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
2480 {
2481 start_sequence ();
2482
2483 temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
2484 if (!temp)
2485 goto fail;
2486
2487 defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
2488 }
2489 else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
2490 {
2491 start_sequence ();
2492 temp = expand_ctz (mode, op0, 0);
2493 if (!temp)
2494 goto fail;
2495
2496 if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
2497 {
2498 defined_at_zero = true;
2499 val = (GET_MODE_PRECISION (mode) - 1) - val;
2500 }
2501 }
2502 else
2503 return 0;
2504
2505 if (defined_at_zero && val == -1)
2506 /* No correction needed at zero. */;
2507 else
2508 {
2509 /* We don't try to do anything clever with the situation found
2510 on some processors (eg Alpha) where ctz(0:mode) ==
2511 bitsize(mode). If someone can think of a way to send N to -1
2512 and leave alone all values in the range 0..N-1 (where N is a
2513 power of two), cheaper than this test-and-branch, please add it.
2514
2515 The test-and-branch is done after the operation itself, in case
2516 the operation sets condition codes that can be recycled for this.
2517 (This is true on i386, for instance.) */
2518
2519 rtx_code_label *nonzero_label = gen_label_rtx ();
2520 emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
2521 mode, true, nonzero_label);
2522
2523 convert_move (temp, GEN_INT (-1), false);
2524 emit_label (nonzero_label);
2525 }
2526
2527 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2528 to produce a value in the range 0..bitsize. */
2529 temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
2530 target, false, OPTAB_DIRECT);
2531 if (!temp)
2532 goto fail;
2533
2534 seq = get_insns ();
2535 end_sequence ();
2536
2537 add_equal_note (seq, temp, FFS, op0, 0);
2538 emit_insn (seq);
2539 return temp;
2540
2541 fail:
2542 end_sequence ();
2543 return 0;
2544}
2545
2546/* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2547 conditions, VAL may already be a SUBREG against which we cannot generate
2548 a further SUBREG. In this case, we expect forcing the value into a
2549 register will work around the situation. */
2550
2551static rtx
2552lowpart_subreg_maybe_copy (machine_mode omode, rtx val,
2553 machine_mode imode)
2554{
2555 rtx ret;
2556 ret = lowpart_subreg (omode, val, imode);
2557 if (ret == NULL)
2558 {
2559 val = force_reg (imode, val);
2560 ret = lowpart_subreg (omode, val, imode);
2561 gcc_assert (ret != NULL);
2562 }
2563 return ret;
2564}
2565
2566/* Expand a floating point absolute value or negation operation via a
2567 logical operation on the sign bit. */
2568
2569static rtx
2570expand_absneg_bit (enum rtx_code code, scalar_float_mode mode,
2571 rtx op0, rtx target)
2572{
2573 const struct real_format *fmt;
2574 int bitpos, word, nwords, i;
2575 scalar_int_mode imode;
2576 rtx temp;
2577 rtx_insn *insns;
2578
2579 /* The format has to have a simple sign bit. */
2580 fmt = REAL_MODE_FORMAT (mode);
2581 if (fmt == NULL)
2582 return NULL_RTX;
2583
2584 bitpos = fmt->signbit_rw;
2585 if (bitpos < 0)
2586 return NULL_RTX;
2587
2588 /* Don't create negative zeros if the format doesn't support them. */
2589 if (code == NEG && !fmt->has_signed_zero)
2590 return NULL_RTX;
2591
2592 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
2593 {
2594 if (!int_mode_for_mode (mode).exists (&imode))
2595 return NULL_RTX;
2596 word = 0;
2597 nwords = 1;
2598 }
2599 else
2600 {
2601 imode = word_mode;
2602
2603 if (FLOAT_WORDS_BIG_ENDIAN)
2604 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
2605 else
2606 word = bitpos / BITS_PER_WORD;
2607 bitpos = bitpos % BITS_PER_WORD;
2608 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
2609 }
2610
2611 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
2612 if (code == ABS)
2613 mask = ~mask;
2614
2615 if (target == 0
2616 || target == op0
2617 || (nwords > 1 && !valid_multiword_target_p (target)))
2618 target = gen_reg_rtx (mode);
2619
2620 if (nwords > 1)
2621 {
2622 start_sequence ();
2623
2624 for (i = 0; i < nwords; ++i)
2625 {
2626 rtx targ_piece = operand_subword (target, i, 1, mode);
2627 rtx op0_piece = operand_subword_force (op0, i, mode);
2628
2629 if (i == word)
2630 {
2631 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2632 op0_piece,
2633 immed_wide_int_const (mask, imode),
2634 targ_piece, 1, OPTAB_LIB_WIDEN);
2635 if (temp != targ_piece)
2636 emit_move_insn (targ_piece, temp);
2637 }
2638 else
2639 emit_move_insn (targ_piece, op0_piece);
2640 }
2641
2642 insns = get_insns ();
2643 end_sequence ();
2644
2645 emit_insn (insns);
2646 }
2647 else
2648 {
2649 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2650 gen_lowpart (imode, op0),
2651 immed_wide_int_const (mask, imode),
2652 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
2653 target = lowpart_subreg_maybe_copy (mode, temp, imode);
2654
2655 set_dst_reg_note (get_last_insn (), REG_EQUAL,
2656 gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
2657 target);
2658 }
2659
2660 return target;
2661}
2662
2663/* As expand_unop, but will fail rather than attempt the operation in a
2664 different mode or with a libcall. */
2665static rtx
2666expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
2667 int unsignedp)
2668{
2669 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2670 {
2671 struct expand_operand ops[2];
2672 enum insn_code icode = optab_handler (unoptab, mode);
2673 rtx_insn *last = get_last_insn ();
2674 rtx_insn *pat;
2675
2676 create_output_operand (&ops[0], target, mode);
2677 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2678 pat = maybe_gen_insn (icode, 2, ops);
2679 if (pat)
2680 {
2681 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
2682 && ! add_equal_note (pat, ops[0].value,
2683 optab_to_code (unoptab),
2684 ops[1].value, NULL_RTX))
2685 {
2686 delete_insns_since (last);
2687 return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
2688 }
2689
2690 emit_insn (pat);
2691
2692 return ops[0].value;
2693 }
2694 }
2695 return 0;
2696}
2697
2698/* Generate code to perform an operation specified by UNOPTAB
2699 on operand OP0, with result having machine-mode MODE.
2700
2701 UNSIGNEDP is for the case where we have to widen the operands
2702 to perform the operation. It says to use zero-extension.
2703
2704 If TARGET is nonzero, the value
2705 is generated there, if it is convenient to do so.
2706 In all cases an rtx is returned for the locus of the value;
2707 this may or may not be TARGET. */
2708
2709rtx
2710expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
2711 int unsignedp)
2712{
2713 enum mode_class mclass = GET_MODE_CLASS (mode);
2714 machine_mode wider_mode;
2715 scalar_int_mode int_mode;
2716 scalar_float_mode float_mode;
2717 rtx temp;
2718 rtx libfunc;
2719
2720 temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
2721 if (temp)
2722 return temp;
2723
2724 /* It can't be done in this mode. Can we open-code it in a wider mode? */
2725
2726 /* Widening (or narrowing) clz needs special treatment. */
2727 if (unoptab == clz_optab)
2728 {
2729 if (is_a <scalar_int_mode> (mode, &int_mode))
2730 {
2731 temp = widen_leading (int_mode, op0, target, unoptab);
2732 if (temp)
2733 return temp;
2734
2735 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2736 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2737 {
2738 temp = expand_doubleword_clz (int_mode, op0, target);
2739 if (temp)
2740 return temp;
2741 }
2742 }
2743
2744 goto try_libcall;
2745 }
2746
2747 if (unoptab == clrsb_optab)
2748 {
2749 if (is_a <scalar_int_mode> (mode, &int_mode))
2750 {
2751 temp = widen_leading (int_mode, op0, target, unoptab);
2752 if (temp)
2753 return temp;
2754 }
2755 goto try_libcall;
2756 }
2757
2758 if (unoptab == popcount_optab
2759 && is_a <scalar_int_mode> (mode, &int_mode)
2760 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2761 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
2762 && optimize_insn_for_speed_p ())
2763 {
2764 temp = expand_doubleword_popcount (int_mode, op0, target);
2765 if (temp)
2766 return temp;
2767 }
2768
2769 if (unoptab == parity_optab
2770 && is_a <scalar_int_mode> (mode, &int_mode)
2771 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2772 && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
2773 || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
2774 && optimize_insn_for_speed_p ())
2775 {
2776 temp = expand_doubleword_parity (int_mode, op0, target);
2777 if (temp)
2778 return temp;
2779 }
2780
2781 /* Widening (or narrowing) bswap needs special treatment. */
2782 if (unoptab == bswap_optab)
2783 {
2784 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
2785 or ROTATERT. First try these directly; if this fails, then try the
2786 obvious pair of shifts with allowed widening, as this will probably
2787 be always more efficient than the other fallback methods. */
2788 if (mode == HImode)
2789 {
2790 rtx_insn *last;
2791 rtx temp1, temp2;
2792
2793 if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
2794 {
2795 temp = expand_binop (mode, rotl_optab, op0, GEN_INT (8), target,
2796 unsignedp, OPTAB_DIRECT);
2797 if (temp)
2798 return temp;
2799 }
2800
2801 if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
2802 {
2803 temp = expand_binop (mode, rotr_optab, op0, GEN_INT (8), target,
2804 unsignedp, OPTAB_DIRECT);
2805 if (temp)
2806 return temp;
2807 }
2808
2809 last = get_last_insn ();
2810
2811 temp1 = expand_binop (mode, ashl_optab, op0, GEN_INT (8), NULL_RTX,
2812 unsignedp, OPTAB_WIDEN);
2813 temp2 = expand_binop (mode, lshr_optab, op0, GEN_INT (8), NULL_RTX,
2814 unsignedp, OPTAB_WIDEN);
2815 if (temp1 && temp2)
2816 {
2817 temp = expand_binop (mode, ior_optab, temp1, temp2, target,
2818 unsignedp, OPTAB_WIDEN);
2819 if (temp)
2820 return temp;
2821 }
2822
2823 delete_insns_since (last);
2824 }
2825
2826 if (is_a <scalar_int_mode> (mode, &int_mode))
2827 {
2828 temp = widen_bswap (int_mode, op0, target);
2829 if (temp)
2830 return temp;
2831
2832 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2833 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2834 {
2835 temp = expand_doubleword_bswap (mode, op0, target);
2836 if (temp)
2837 return temp;
2838 }
2839 }
2840
2841 goto try_libcall;
2842 }
2843
2844 if (CLASS_HAS_WIDER_MODES_P (mclass))
2845 FOR_EACH_WIDER_MODE (wider_mode, mode)
2846 {
2847 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2848 {
2849 rtx xop0 = op0;
2850 rtx_insn *last = get_last_insn ();
2851
2852 /* For certain operations, we need not actually extend
2853 the narrow operand, as long as we will truncate the
2854 results to the same narrowness. */
2855
2856 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
2857 (unoptab == neg_optab
2858 || unoptab == one_cmpl_optab)
2859 && mclass == MODE_INT);
2860
2861 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2862 unsignedp);
2863
2864 if (temp)
2865 {
2866 if (mclass != MODE_INT
2867 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2868 {
2869 if (target == 0)
2870 target = gen_reg_rtx (mode);
2871 convert_move (target, temp, 0);
2872 return target;
2873 }
2874 else
2875 return gen_lowpart (mode, temp);
2876 }
2877 else
2878 delete_insns_since (last);
2879 }
2880 }
2881
2882 /* These can be done a word at a time. */
2883 if (unoptab == one_cmpl_optab
2884 && is_int_mode (mode, &int_mode)
2885 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
2886 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2887 {
2888 int i;
2889 rtx_insn *insns;
2890
2891 if (target == 0 || target == op0 || !valid_multiword_target_p (target))
2892 target = gen_reg_rtx (int_mode);
2893
2894 start_sequence ();
2895
2896 /* Do the actual arithmetic. */
2897 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
2898 {
2899 rtx target_piece = operand_subword (target, i, 1, int_mode);
2900 rtx x = expand_unop (word_mode, unoptab,
2901 operand_subword_force (op0, i, int_mode),
2902 target_piece, unsignedp);
2903
2904 if (target_piece != x)
2905 emit_move_insn (target_piece, x);
2906 }
2907
2908 insns = get_insns ();
2909 end_sequence ();
2910
2911 emit_insn (insns);
2912 return target;
2913 }
2914
2915 if (optab_to_code (unoptab) == NEG)
2916 {
2917 /* Try negating floating point values by flipping the sign bit. */
2918 if (is_a <scalar_float_mode> (mode, &float_mode))
2919 {
2920 temp = expand_absneg_bit (NEG, float_mode, op0, target);
2921 if (temp)
2922 return temp;
2923 }
2924
2925 /* If there is no negation pattern, and we have no negative zero,
2926 try subtracting from zero. */
2927 if (!HONOR_SIGNED_ZEROS (mode))
2928 {
2929 temp = expand_binop (mode, (unoptab == negv_optab
2930 ? subv_optab : sub_optab),
2931 CONST0_RTX (mode), op0, target,
2932 unsignedp, OPTAB_DIRECT);
2933 if (temp)
2934 return temp;
2935 }
2936 }
2937
2938 /* Try calculating parity (x) as popcount (x) % 2. */
2939 if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
2940 {
2941 temp = expand_parity (int_mode, op0, target);
2942 if (temp)
2943 return temp;
2944 }
2945
2946 /* Try implementing ffs (x) in terms of clz (x). */
2947 if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
2948 {
2949 temp = expand_ffs (int_mode, op0, target);
2950 if (temp)
2951 return temp;
2952 }
2953
2954 /* Try implementing ctz (x) in terms of clz (x). */
2955 if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
2956 {
2957 temp = expand_ctz (int_mode, op0, target);
2958 if (temp)
2959 return temp;
2960 }
2961
2962 try_libcall:
2963 /* Now try a library call in this mode. */
2964 libfunc = optab_libfunc (unoptab, mode);
2965 if (libfunc)
2966 {
2967 rtx_insn *insns;
2968 rtx value;
2969 rtx eq_value;
2970 machine_mode outmode = mode;
2971
2972 /* All of these functions return small values. Thus we choose to
2973 have them return something that isn't a double-word. */
2974 if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
2975 || unoptab == clrsb_optab || unoptab == popcount_optab
2976 || unoptab == parity_optab)
2977 outmode
2978 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
2979 optab_libfunc (unoptab, mode)));
2980
2981 start_sequence ();
2982
2983 /* Pass 1 for NO_QUEUE so we don't lose any increments
2984 if the libcall is cse'd or moved. */
2985 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
2986 op0, mode);
2987 insns = get_insns ();
2988 end_sequence ();
2989
2990 target = gen_reg_rtx (outmode);
2991 bool trapv = trapv_unoptab_p (unoptab);
2992 if (trapv)
2993 eq_value = NULL_RTX;
2994 else
2995 {
2996 eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
2997 if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
2998 eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
2999 else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3000 eq_value = simplify_gen_unary (ZERO_EXTEND,
3001 outmode, eq_value, mode);
3002 }
3003 emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3004
3005 return target;
3006 }
3007
3008 /* It can't be done in this mode. Can we do it in a wider mode? */
3009
3010 if (CLASS_HAS_WIDER_MODES_P (mclass))
3011 {
3012 FOR_EACH_WIDER_MODE (wider_mode, mode)
3013 {
3014 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3015 || optab_libfunc (unoptab, wider_mode))
3016 {
3017 rtx xop0 = op0;
3018 rtx_insn *last = get_last_insn ();
3019
3020 /* For certain operations, we need not actually extend
3021 the narrow operand, as long as we will truncate the
3022 results to the same narrowness. */
3023 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3024 (unoptab == neg_optab
3025 || unoptab == one_cmpl_optab
3026 || unoptab == bswap_optab)
3027 && mclass == MODE_INT);
3028
3029 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3030 unsignedp);
3031
3032 /* If we are generating clz using wider mode, adjust the
3033 result. Similarly for clrsb. */
3034 if ((unoptab == clz_optab || unoptab == clrsb_optab)
3035 && temp != 0)
3036 {
3037 scalar_int_mode wider_int_mode
3038 = as_a <scalar_int_mode> (wider_mode);
3039 int_mode = as_a <scalar_int_mode> (mode);
3040 temp = expand_binop
3041 (wider_mode, sub_optab, temp,
3042 gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
3043 - GET_MODE_PRECISION (int_mode),
3044 wider_int_mode),
3045 target, true, OPTAB_DIRECT);
3046 }
3047
3048 /* Likewise for bswap. */
3049 if (unoptab == bswap_optab && temp != 0)
3050 {
3051 scalar_int_mode wider_int_mode
3052 = as_a <scalar_int_mode> (wider_mode);
3053 int_mode = as_a <scalar_int_mode> (mode);
3054 gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3055 == GET_MODE_BITSIZE (wider_int_mode)
3056 && GET_MODE_PRECISION (int_mode)
3057 == GET_MODE_BITSIZE (int_mode));
3058
3059 temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3060 GET_MODE_BITSIZE (wider_int_mode)
3061 - GET_MODE_BITSIZE (int_mode),
3062 NULL_RTX, true);
3063 }
3064
3065 if (temp)
3066 {
3067 if (mclass != MODE_INT)
3068 {
3069 if (target == 0)
3070 target = gen_reg_rtx (mode);
3071 convert_move (target, temp, 0);
3072 return target;
3073 }
3074 else
3075 return gen_lowpart (mode, temp);
3076 }
3077 else
3078 delete_insns_since (last);
3079 }
3080 }
3081 }
3082
3083 /* One final attempt at implementing negation via subtraction,
3084 this time allowing widening of the operand. */
3085 if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3086 {
3087 rtx temp;
3088 temp = expand_binop (mode,
3089 unoptab == negv_optab ? subv_optab : sub_optab,
3090 CONST0_RTX (mode), op0,
3091 target, unsignedp, OPTAB_LIB_WIDEN);
3092 if (temp)
3093 return temp;
3094 }
3095
3096 return 0;
3097}
3098
3099/* Emit code to compute the absolute value of OP0, with result to
3100 TARGET if convenient. (TARGET may be 0.) The return value says
3101 where the result actually is to be found.
3102
3103 MODE is the mode of the operand; the mode of the result is
3104 different but can be deduced from MODE.
3105
3106 */
3107
3108rtx
3109expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3110 int result_unsignedp)
3111{
3112 rtx temp;
3113
3114 if (GET_MODE_CLASS (mode) != MODE_INT
3115 || ! flag_trapv)
3116 result_unsignedp = 1;
3117
3118 /* First try to do it with a special abs instruction. */
3119 temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3120 op0, target, 0);
3121 if (temp != 0)
3122 return temp;
3123
3124 /* For floating point modes, try clearing the sign bit. */
3125 scalar_float_mode float_mode;
3126 if (is_a <scalar_float_mode> (mode, &float_mode))
3127 {
3128 temp = expand_absneg_bit (ABS, float_mode, op0, target);
3129 if (temp)
3130 return temp;
3131 }
3132
3133 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3134 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3135 && !HONOR_SIGNED_ZEROS (mode))
3136 {
3137 rtx_insn *last = get_last_insn ();
3138
3139 temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3140 op0, NULL_RTX, 0);
3141 if (temp != 0)
3142 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3143 OPTAB_WIDEN);
3144
3145 if (temp != 0)
3146 return temp;
3147
3148 delete_insns_since (last);
3149 }
3150
3151 /* If this machine has expensive jumps, we can do integer absolute
3152 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3153 where W is the width of MODE. */
3154
3155 scalar_int_mode int_mode;
3156 if (is_int_mode (mode, &int_mode)
3157 && BRANCH_COST (optimize_insn_for_speed_p (),
3158 false) >= 2)
3159 {
3160 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3161 GET_MODE_PRECISION (int_mode) - 1,
3162 NULL_RTX, 0);
3163
3164 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3165 OPTAB_LIB_WIDEN);
3166 if (temp != 0)
3167 temp = expand_binop (int_mode,
3168 result_unsignedp ? sub_optab : subv_optab,
3169 temp, extended, target, 0, OPTAB_LIB_WIDEN);
3170
3171 if (temp != 0)
3172 return temp;
3173 }
3174
3175 return NULL_RTX;
3176}
3177
3178rtx
3179expand_abs (machine_mode mode, rtx op0, rtx target,
3180 int result_unsignedp, int safe)
3181{
3182 rtx temp;
3183 rtx_code_label *op1;
3184
3185 if (GET_MODE_CLASS (mode) != MODE_INT
3186 || ! flag_trapv)
3187 result_unsignedp = 1;
3188
3189 temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3190 if (temp != 0)
3191 return temp;
3192
3193 /* If that does not win, use conditional jump and negate. */
3194
3195 /* It is safe to use the target if it is the same
3196 as the source if this is also a pseudo register */
3197 if (op0 == target && REG_P (op0)
3198 && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3199 safe = 1;
3200
3201 op1 = gen_label_rtx ();
3202 if (target == 0 || ! safe
3203 || GET_MODE (target) != mode
3204 || (MEM_P (target) && MEM_VOLATILE_P (target))
3205 || (REG_P (target)
3206 && REGNO (target) < FIRST_PSEUDO_REGISTER))
3207 target = gen_reg_rtx (mode);
3208
3209 emit_move_insn (target, op0);
3210 NO_DEFER_POP;
3211
3212 do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3213 NULL_RTX, NULL, op1,
3214 profile_probability::uninitialized ());
3215
3216 op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3217 target, target, 0);
3218 if (op0 != target)
3219 emit_move_insn (target, op0);
3220 emit_label (op1);
3221 OK_DEFER_POP;
3222 return target;
3223}
3224
3225/* Emit code to compute the one's complement absolute value of OP0
3226 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3227 (TARGET may be NULL_RTX.) The return value says where the result
3228 actually is to be found.
3229
3230 MODE is the mode of the operand; the mode of the result is
3231 different but can be deduced from MODE. */
3232
3233rtx
3234expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3235{
3236 rtx temp;
3237
3238 /* Not applicable for floating point modes. */
3239 if (FLOAT_MODE_P (mode))
3240 return NULL_RTX;
3241
3242 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3243 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3244 {
3245 rtx_insn *last = get_last_insn ();
3246
3247 temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3248 if (temp != 0)
3249 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3250 OPTAB_WIDEN);
3251
3252 if (temp != 0)
3253 return temp;
3254
3255 delete_insns_since (last);
3256 }
3257
3258 /* If this machine has expensive jumps, we can do one's complement
3259 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3260
3261 scalar_int_mode int_mode;
3262 if (is_int_mode (mode, &int_mode)
3263 && BRANCH_COST (optimize_insn_for_speed_p (),
3264 false) >= 2)
3265 {
3266 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3267 GET_MODE_PRECISION (int_mode) - 1,
3268 NULL_RTX, 0);
3269
3270 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3271 OPTAB_LIB_WIDEN);
3272
3273 if (temp != 0)
3274 return temp;
3275 }
3276
3277 return NULL_RTX;
3278}
3279
3280/* A subroutine of expand_copysign, perform the copysign operation using the
3281 abs and neg primitives advertised to exist on the target. The assumption
3282 is that we have a split register file, and leaving op0 in fp registers,
3283 and not playing with subregs so much, will help the register allocator. */
3284
3285static rtx
3286expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3287 int bitpos, bool op0_is_abs)
3288{
3289 scalar_int_mode imode;
3290 enum insn_code icode;
3291 rtx sign;
3292 rtx_code_label *label;
3293
3294 if (target == op1)
3295 target = NULL_RTX;
3296
3297 /* Check if the back end provides an insn that handles signbit for the
3298 argument's mode. */
3299 icode = optab_handler (signbit_optab, mode);
3300 if (icode != CODE_FOR_nothing)
3301 {
3302 imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
3303 sign = gen_reg_rtx (imode);
3304 emit_unop_insn (icode, sign, op1, UNKNOWN);
3305 }
3306 else
3307 {
3308 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3309 {
3310 if (!int_mode_for_mode (mode).exists (&imode))
3311 return NULL_RTX;
3312 op1 = gen_lowpart (imode, op1);
3313 }
3314 else
3315 {
3316 int word;
3317
3318 imode = word_mode;
3319 if (FLOAT_WORDS_BIG_ENDIAN)
3320 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3321 else
3322 word = bitpos / BITS_PER_WORD;
3323 bitpos = bitpos % BITS_PER_WORD;
3324 op1 = operand_subword_force (op1, word, mode);
3325 }
3326
3327 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3328 sign = expand_binop (imode, and_optab, op1,
3329 immed_wide_int_const (mask, imode),
3330 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3331 }
3332
3333 if (!op0_is_abs)
3334 {
3335 op0 = expand_unop (mode, abs_optab, op0, target, 0);
3336 if (op0 == NULL)
3337 return NULL_RTX;
3338 target = op0;
3339 }
3340 else
3341 {
3342 if (target == NULL_RTX)
3343 target = copy_to_reg (op0);
3344 else
3345 emit_move_insn (target, op0);
3346 }
3347
3348 label = gen_label_rtx ();
3349 emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3350
3351 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3352 op0 = simplify_unary_operation (NEG, mode, op0, mode);
3353 else
3354 op0 = expand_unop (mode, neg_optab, op0, target, 0);
3355 if (op0 != target)
3356 emit_move_insn (target, op0);
3357
3358 emit_label (label);
3359
3360 return target;
3361}
3362
3363
3364/* A subroutine of expand_copysign, perform the entire copysign operation
3365 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3366 is true if op0 is known to have its sign bit clear. */
3367
3368static rtx
3369expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3370 int bitpos, bool op0_is_abs)
3371{
3372 scalar_int_mode imode;
3373 int word, nwords, i;
3374 rtx temp;
3375 rtx_insn *insns;
3376
3377 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3378 {
3379 if (!int_mode_for_mode (mode).exists (&imode))
3380 return NULL_RTX;
3381 word = 0;
3382 nwords = 1;
3383 }
3384 else
3385 {
3386 imode = word_mode;
3387
3388 if (FLOAT_WORDS_BIG_ENDIAN)
3389 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3390 else
3391 word = bitpos / BITS_PER_WORD;
3392 bitpos = bitpos % BITS_PER_WORD;
3393 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3394 }
3395
3396 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3397
3398 if (target == 0
3399 || target == op0
3400 || target == op1
3401 || (nwords > 1 && !valid_multiword_target_p (target)))
3402 target = gen_reg_rtx (mode);
3403
3404 if (nwords > 1)
3405 {
3406 start_sequence ();
3407
3408 for (i = 0; i < nwords; ++i)
3409 {
3410 rtx targ_piece = operand_subword (target, i, 1, mode);
3411 rtx op0_piece = operand_subword_force (op0, i, mode);
3412
3413 if (i == word)
3414 {
3415 if (!op0_is_abs)
3416 op0_piece
3417 = expand_binop (imode, and_optab, op0_piece,
3418 immed_wide_int_const (~mask, imode),
3419 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3420 op1 = expand_binop (imode, and_optab,
3421 operand_subword_force (op1, i, mode),
3422 immed_wide_int_const (mask, imode),
3423 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3424
3425 temp = expand_binop (imode, ior_optab, op0_piece, op1,
3426 targ_piece, 1, OPTAB_LIB_WIDEN);
3427 if (temp != targ_piece)
3428 emit_move_insn (targ_piece, temp);
3429 }
3430 else
3431 emit_move_insn (targ_piece, op0_piece);
3432 }
3433
3434 insns = get_insns ();
3435 end_sequence ();
3436
3437 emit_insn (insns);
3438 }
3439 else
3440 {
3441 op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
3442 immed_wide_int_const (mask, imode),
3443 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3444
3445 op0 = gen_lowpart (imode, op0);
3446 if (!op0_is_abs)
3447 op0 = expand_binop (imode, and_optab, op0,
3448 immed_wide_int_const (~mask, imode),
3449 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3450
3451 temp = expand_binop (imode, ior_optab, op0, op1,
3452 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
3453 target = lowpart_subreg_maybe_copy (mode, temp, imode);
3454 }
3455
3456 return target;
3457}
3458
3459/* Expand the C99 copysign operation. OP0 and OP1 must be the same
3460 scalar floating point mode. Return NULL if we do not know how to
3461 expand the operation inline. */
3462
3463rtx
3464expand_copysign (rtx op0, rtx op1, rtx target)
3465{
3466 scalar_float_mode mode;
3467 const struct real_format *fmt;
3468 bool op0_is_abs;
3469 rtx temp;
3470
3471 mode = as_a <scalar_float_mode> (GET_MODE (op0));
3472 gcc_assert (GET_MODE (op1) == mode);
3473
3474 /* First try to do it with a special instruction. */
3475 temp = expand_binop (mode, copysign_optab, op0, op1,
3476 target, 0, OPTAB_DIRECT);
3477 if (temp)
3478 return temp;
3479
3480 fmt = REAL_MODE_FORMAT (mode);
3481 if (fmt == NULL || !fmt->has_signed_zero)
3482 return NULL_RTX;
3483
3484 op0_is_abs = false;
3485 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3486 {
3487 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
3488 op0 = simplify_unary_operation (ABS, mode, op0, mode);
3489 op0_is_abs = true;
3490 }
3491
3492 if (fmt->signbit_ro >= 0
3493 && (CONST_DOUBLE_AS_FLOAT_P (op0)
3494 || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
3495 && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
3496 {
3497 temp = expand_copysign_absneg (mode, op0, op1, target,
3498 fmt->signbit_ro, op0_is_abs);
3499 if (temp)
3500 return temp;
3501 }
3502
3503 if (fmt->signbit_rw < 0)
3504 return NULL_RTX;
3505 return expand_copysign_bit (mode, op0, op1, target,
3506 fmt->signbit_rw, op0_is_abs);
3507}
3508
3509/* Generate an instruction whose insn-code is INSN_CODE,
3510 with two operands: an output TARGET and an input OP0.
3511 TARGET *must* be nonzero, and the output is always stored there.
3512 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3513 the value that is stored into TARGET.
3514
3515 Return false if expansion failed. */
3516
3517bool
3518maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
3519 enum rtx_code code)
3520{
3521 struct expand_operand ops[2];
3522 rtx_insn *pat;
3523
3524 create_output_operand (&ops[0], target, GET_MODE (target));
3525 create_input_operand (&ops[1], op0, GET_MODE (op0));
3526 pat = maybe_gen_insn (icode, 2, ops);
3527 if (!pat)
3528 return false;
3529
3530 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3531 && code != UNKNOWN)
3532 add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX);
3533
3534 emit_insn (pat);
3535
3536 if (ops[0].value != target)
3537 emit_move_insn (target, ops[0].value);
3538 return true;
3539}
3540/* Generate an instruction whose insn-code is INSN_CODE,
3541 with two operands: an output TARGET and an input OP0.
3542 TARGET *must* be nonzero, and the output is always stored there.
3543 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3544 the value that is stored into TARGET. */
3545
3546void
3547emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
3548{
3549 bool ok = maybe_emit_unop_insn (icode, target, op0, code);
3550 gcc_assert (ok);
3551}
3552
3553struct no_conflict_data
3554{
3555 rtx target;
3556 rtx_insn *first, *insn;
3557 bool must_stay;
3558};
3559
3560/* Called via note_stores by emit_libcall_block. Set P->must_stay if
3561 the currently examined clobber / store has to stay in the list of
3562 insns that constitute the actual libcall block. */
3563static void
3564no_conflict_move_test (rtx dest, const_rtx set, void *p0)
3565{
3566 struct no_conflict_data *p= (struct no_conflict_data *) p0;
3567
3568 /* If this inns directly contributes to setting the target, it must stay. */
3569 if (reg_overlap_mentioned_p (p->target, dest))
3570 p->must_stay = true;
3571 /* If we haven't committed to keeping any other insns in the list yet,
3572 there is nothing more to check. */
3573 else if (p->insn == p->first)
3574 return;
3575 /* If this insn sets / clobbers a register that feeds one of the insns
3576 already in the list, this insn has to stay too. */
3577 else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
3578 || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
3579 || reg_used_between_p (dest, p->first, p->insn)
3580 /* Likewise if this insn depends on a register set by a previous
3581 insn in the list, or if it sets a result (presumably a hard
3582 register) that is set or clobbered by a previous insn.
3583 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3584 SET_DEST perform the former check on the address, and the latter
3585 check on the MEM. */
3586 || (GET_CODE (set) == SET
3587 && (modified_in_p (SET_SRC (set), p->first)
3588 || modified_in_p (SET_DEST (set), p->first)
3589 || modified_between_p (SET_SRC (set), p->first, p->insn)
3590 || modified_between_p (SET_DEST (set), p->first, p->insn))))
3591 p->must_stay = true;
3592}
3593
3594
3595/* Emit code to make a call to a constant function or a library call.
3596
3597 INSNS is a list containing all insns emitted in the call.
3598 These insns leave the result in RESULT. Our block is to copy RESULT
3599 to TARGET, which is logically equivalent to EQUIV.
3600
3601 We first emit any insns that set a pseudo on the assumption that these are
3602 loading constants into registers; doing so allows them to be safely cse'ed
3603 between blocks. Then we emit all the other insns in the block, followed by
3604 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3605 note with an operand of EQUIV. */
3606
3607static void
3608emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
3609 bool equiv_may_trap)
3610{
3611 rtx final_dest = target;
3612 rtx_insn *next, *last, *insn;
3613
3614 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3615 into a MEM later. Protect the libcall block from this change. */
3616 if (! REG_P (target) || REG_USERVAR_P (target))
3617 target = gen_reg_rtx (GET_MODE (target));
3618
3619 /* If we're using non-call exceptions, a libcall corresponding to an
3620 operation that may trap may also trap. */
3621 /* ??? See the comment in front of make_reg_eh_region_note. */
3622 if (cfun->can_throw_non_call_exceptions
3623 && (equiv_may_trap || may_trap_p (equiv)))
3624 {
3625 for (insn = insns; insn; insn = NEXT_INSN (insn))
3626 if (CALL_P (insn))
3627 {
3628 rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3629 if (note)
3630 {
3631 int lp_nr = INTVAL (XEXP (note, 0));
3632 if (lp_nr == 0 || lp_nr == INT_MIN)
3633 remove_note (insn, note);
3634 }
3635 }
3636 }
3637 else
3638 {
3639 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3640 reg note to indicate that this call cannot throw or execute a nonlocal
3641 goto (unless there is already a REG_EH_REGION note, in which case
3642 we update it). */
3643 for (insn = insns; insn; insn = NEXT_INSN (insn))
3644 if (CALL_P (insn))
3645 make_reg_eh_region_note_nothrow_nononlocal (insn);
3646 }
3647
3648 /* First emit all insns that set pseudos. Remove them from the list as
3649 we go. Avoid insns that set pseudos which were referenced in previous
3650 insns. These can be generated by move_by_pieces, for example,
3651 to update an address. Similarly, avoid insns that reference things
3652 set in previous insns. */
3653
3654 for (insn = insns; insn; insn = next)
3655 {
3656 rtx set = single_set (insn);
3657
3658 next = NEXT_INSN (insn);
3659
3660 if (set != 0 && REG_P (SET_DEST (set))
3661 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
3662 {
3663 struct no_conflict_data data;
3664
3665 data.target = const0_rtx;
3666 data.first = insns;
3667 data.insn = insn;
3668 data.must_stay = 0;
3669 note_stores (PATTERN (insn), no_conflict_move_test, &data);
3670 if (! data.must_stay)
3671 {
3672 if (PREV_INSN (insn))
3673 SET_NEXT_INSN (PREV_INSN (insn)) = next;
3674 else
3675 insns = next;
3676
3677 if (next)
3678 SET_PREV_INSN (next) = PREV_INSN (insn);
3679
3680 add_insn (insn);
3681 }
3682 }
3683
3684 /* Some ports use a loop to copy large arguments onto the stack.
3685 Don't move anything outside such a loop. */
3686 if (LABEL_P (insn))
3687 break;
3688 }
3689
3690 /* Write the remaining insns followed by the final copy. */
3691 for (insn = insns; insn; insn = next)
3692 {
3693 next = NEXT_INSN (insn);
3694
3695 add_insn (insn);
3696 }
3697
3698 last = emit_move_insn (target, result);
3699 if (equiv)
3700 set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
3701
3702 if (final_dest != target)
3703 emit_move_insn (final_dest, target);
3704}
3705
3706void
3707emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
3708{
3709 emit_libcall_block_1 (insns, target, result, equiv, false);
3710}
3711
3712/* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3713 PURPOSE describes how this comparison will be used. CODE is the rtx
3714 comparison code we will be using.
3715
3716 ??? Actually, CODE is slightly weaker than that. A target is still
3717 required to implement all of the normal bcc operations, but not
3718 required to implement all (or any) of the unordered bcc operations. */
3719
3720int
3721can_compare_p (enum rtx_code code, machine_mode mode,
3722 enum can_compare_purpose purpose)
3723{
3724 rtx test;
3725 test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
3726 do
3727 {
3728 enum insn_code icode;
3729
3730 if (purpose == ccp_jump
3731 && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
3732 && insn_operand_matches (icode, 0, test))
3733 return 1;
3734 if (purpose == ccp_store_flag
3735 && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
3736 && insn_operand_matches (icode, 1, test))
3737 return 1;
3738 if (purpose == ccp_cmov
3739 && optab_handler (cmov_optab, mode) != CODE_FOR_nothing)
3740 return 1;
3741
3742 mode = GET_MODE_WIDER_MODE (mode).else_void ();
3743 PUT_MODE (test, mode);
3744 }
3745 while (mode != VOIDmode);
3746
3747 return 0;
3748}
3749
3750/* This function is called when we are going to emit a compare instruction that
3751 compares the values found in X and Y, using the rtl operator COMPARISON.
3752
3753 If they have mode BLKmode, then SIZE specifies the size of both operands.
3754
3755 UNSIGNEDP nonzero says that the operands are unsigned;
3756 this matters if they need to be widened (as given by METHODS).
3757
3758 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
3759 if we failed to produce one.
3760
3761 *PMODE is the mode of the inputs (in case they are const_int).
3762
3763 This function performs all the setup necessary so that the caller only has
3764 to emit a single comparison insn. This setup can involve doing a BLKmode
3765 comparison or emitting a library call to perform the comparison if no insn
3766 is available to handle it.
3767 The values which are passed in through pointers can be modified; the caller
3768 should perform the comparison on the modified values. Constant
3769 comparisons must have already been folded. */
3770
3771static void
3772prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size,
3773 int unsignedp, enum optab_methods methods,
3774 rtx *ptest, machine_mode *pmode)
3775{
3776 machine_mode mode = *pmode;
3777 rtx libfunc, test;
3778 machine_mode cmp_mode;
3779 enum mode_class mclass;
3780
3781 /* The other methods are not needed. */
3782 gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
3783 || methods == OPTAB_LIB_WIDEN);
3784
3785 /* If we are optimizing, force expensive constants into a register. */
3786 if (CONSTANT_P (x) && optimize
3787 && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
3788 > COSTS_N_INSNS (1)))
3789 x = force_reg (mode, x);
3790
3791 if (CONSTANT_P (y) && optimize
3792 && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
3793 > COSTS_N_INSNS (1)))
3794 y = force_reg (mode, y);
3795
3796#if HAVE_cc0
3797 /* Make sure if we have a canonical comparison. The RTL
3798 documentation states that canonical comparisons are required only
3799 for targets which have cc0. */
3800 gcc_assert (!CONSTANT_P (x) || CONSTANT_P (y));
3801#endif
3802
3803 /* Don't let both operands fail to indicate the mode. */
3804 if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
3805 x = force_reg (mode, x);
3806 if (mode == VOIDmode)
3807 mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
3808
3809 /* Handle all BLKmode compares. */
3810
3811 if (mode == BLKmode)
3812 {
3813 machine_mode result_mode;
3814 enum insn_code cmp_code;
3815 rtx result;
3816 rtx opalign
3817 = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
3818
3819 gcc_assert (size);
3820
3821 /* Try to use a memory block compare insn - either cmpstr
3822 or cmpmem will do. */
3823 opt_scalar_int_mode cmp_mode_iter;
3824 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
3825 {
3826 scalar_int_mode cmp_mode = cmp_mode_iter.require ();
3827 cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
3828 if (cmp_code == CODE_FOR_nothing)
3829 cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
3830 if (cmp_code == CODE_FOR_nothing)
3831 cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
3832 if (cmp_code == CODE_FOR_nothing)
3833 continue;
3834
3835 /* Must make sure the size fits the insn's mode. */
3836 if (CONST_INT_P (size)
3837 ? INTVAL (size) >= (1 << GET_MODE_BITSIZE (cmp_mode))
3838 : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
3839 > GET_MODE_BITSIZE (cmp_mode)))
3840 continue;
3841
3842 result_mode = insn_data[cmp_code].operand[0].mode;
3843 result = gen_reg_rtx (result_mode);
3844 size = convert_to_mode (cmp_mode, size, 1);
3845 emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
3846
3847 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
3848 *pmode = result_mode;
3849 return;
3850 }
3851
3852 if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
3853 goto fail;
3854
3855 /* Otherwise call a library function. */
3856 result = emit_block_comp_via_libcall (XEXP (x, 0), XEXP (y, 0), size);
3857
3858 x = result;
3859 y = const0_rtx;
3860 mode = TYPE_MODE (integer_type_node);
3861 methods = OPTAB_LIB_WIDEN;
3862 unsignedp = false;
3863 }
3864
3865 /* Don't allow operands to the compare to trap, as that can put the
3866 compare and branch in different basic blocks. */
3867 if (cfun->can_throw_non_call_exceptions)
3868 {
3869 if (may_trap_p (x))
3870 x = copy_to_reg (x);
3871 if (may_trap_p (y))
3872 y = copy_to_reg (y);
3873 }
3874
3875 if (GET_MODE_CLASS (mode) == MODE_CC)
3876 {
3877 enum insn_code icode = optab_handler (cbranch_optab, CCmode);
3878 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
3879 gcc_assert (icode != CODE_FOR_nothing
3880 && insn_operand_matches (icode, 0, test));
3881 *ptest = test;
3882 return;
3883 }
3884
3885 mclass = GET_MODE_CLASS (mode);
3886 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
3887 FOR_EACH_MODE_FROM (cmp_mode, mode)
3888 {
3889 enum insn_code icode;
3890 icode = optab_handler (cbranch_optab, cmp_mode);
3891 if (icode != CODE_FOR_nothing
3892 && insn_operand_matches (icode, 0, test))
3893 {
3894 rtx_insn *last = get_last_insn ();
3895 rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
3896 rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
3897 if (op0 && op1
3898 && insn_operand_matches (icode, 1, op0)
3899 && insn_operand_matches (icode, 2, op1))
3900 {
3901 XEXP (test, 0) = op0;
3902 XEXP (test, 1) = op1;
3903 *ptest = test;
3904 *pmode = cmp_mode;
3905 return;
3906 }
3907 delete_insns_since (last);
3908 }
3909
3910 if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass))
3911 break;
3912 }
3913
3914 if (methods != OPTAB_LIB_WIDEN)
3915 goto fail;
3916
3917 if (!SCALAR_FLOAT_MODE_P (mode))
3918 {
3919 rtx result;
3920 machine_mode ret_mode;
3921
3922 /* Handle a libcall just for the mode we are using. */
3923 libfunc = optab_libfunc (cmp_optab, mode);
3924 gcc_assert (libfunc);
3925
3926 /* If we want unsigned, and this mode has a distinct unsigned
3927 comparison routine, use that. */
3928 if (unsignedp)
3929 {
3930 rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
3931 if (ulibfunc)
3932