1/* OMP constructs' SIMD clone supporting code.
2
3Copyright (C) 2005-2017 Free Software Foundation, Inc.
4
5This file is part of GCC.
6
7GCC is free software; you can redistribute it and/or modify it under
8the terms of the GNU General Public License as published by the Free
9Software Foundation; either version 3, or (at your option) any later
10version.
11
12GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13WARRANTY; without even the implied warranty of MERCHANTABILITY or
14FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15for more details.
16
17You should have received a copy of the GNU General Public License
18along with GCC; see the file COPYING3. If not see
19<http://www.gnu.org/licenses/>. */
20
21#include "config.h"
22#include "system.h"
23#include "coretypes.h"
24#include "backend.h"
25#include "target.h"
26#include "tree.h"
27#include "gimple.h"
28#include "cfghooks.h"
29#include "alloc-pool.h"
30#include "tree-pass.h"
31#include "ssa.h"
32#include "cgraph.h"
33#include "pretty-print.h"
34#include "diagnostic-core.h"
35#include "fold-const.h"
36#include "stor-layout.h"
37#include "cfganal.h"
38#include "gimplify.h"
39#include "gimple-iterator.h"
40#include "gimplify-me.h"
41#include "gimple-walk.h"
42#include "langhooks.h"
43#include "tree-cfg.h"
44#include "tree-into-ssa.h"
45#include "tree-dfa.h"
46#include "cfgloop.h"
47#include "symbol-summary.h"
48#include "ipa-param-manipulation.h"
49#include "tree-eh.h"
50#include "varasm.h"
51#include "stringpool.h"
52#include "attribs.h"
53
54/* Allocate a fresh `simd_clone' and return it. NARGS is the number
55 of arguments to reserve space for. */
56
57static struct cgraph_simd_clone *
58simd_clone_struct_alloc (int nargs)
59{
60 struct cgraph_simd_clone *clone_info;
61 size_t len = (sizeof (struct cgraph_simd_clone)
62 + nargs * sizeof (struct cgraph_simd_clone_arg));
63 clone_info = (struct cgraph_simd_clone *)
64 ggc_internal_cleared_alloc (len);
65 return clone_info;
66}
67
68/* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
69
70static inline void
71simd_clone_struct_copy (struct cgraph_simd_clone *to,
72 struct cgraph_simd_clone *from)
73{
74 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
75 + ((from->nargs - from->inbranch)
76 * sizeof (struct cgraph_simd_clone_arg))));
77}
78
79/* Return vector of parameter types of function FNDECL. This uses
80 TYPE_ARG_TYPES if available, otherwise falls back to types of
81 DECL_ARGUMENTS types. */
82
83static vec<tree>
84simd_clone_vector_of_formal_parm_types (tree fndecl)
85{
86 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
87 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
88 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
89 unsigned int i;
90 tree arg;
91 FOR_EACH_VEC_ELT (args, i, arg)
92 args[i] = TREE_TYPE (args[i]);
93 return args;
94}
95
96/* Given a simd function in NODE, extract the simd specific
97 information from the OMP clauses passed in CLAUSES, and return
98 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
99 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
100 otherwise set to FALSE. */
101
102static struct cgraph_simd_clone *
103simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
104 bool *inbranch_specified)
105{
106 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
107 tree t;
108 int n;
109 *inbranch_specified = false;
110
111 n = args.length ();
112 if (n > 0 && args.last () == void_type_node)
113 n--;
114
115 /* Allocate one more than needed just in case this is an in-branch
116 clone which will require a mask argument. */
117 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
118 clone_info->nargs = n;
119
120 if (!clauses)
121 goto out;
122
123 clauses = TREE_VALUE (clauses);
124 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
125 goto out;
126
127 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
128 {
129 switch (OMP_CLAUSE_CODE (t))
130 {
131 case OMP_CLAUSE_INBRANCH:
132 clone_info->inbranch = 1;
133 *inbranch_specified = true;
134 break;
135 case OMP_CLAUSE_NOTINBRANCH:
136 clone_info->inbranch = 0;
137 *inbranch_specified = true;
138 break;
139 case OMP_CLAUSE_SIMDLEN:
140 clone_info->simdlen
141 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
142 break;
143 case OMP_CLAUSE_LINEAR:
144 {
145 tree decl = OMP_CLAUSE_DECL (t);
146 tree step = OMP_CLAUSE_LINEAR_STEP (t);
147 int argno = TREE_INT_CST_LOW (decl);
148 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
149 {
150 enum cgraph_simd_clone_arg_type arg_type;
151 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
152 switch (OMP_CLAUSE_LINEAR_KIND (t))
153 {
154 case OMP_CLAUSE_LINEAR_REF:
155 arg_type
156 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
157 break;
158 case OMP_CLAUSE_LINEAR_UVAL:
159 arg_type
160 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
161 break;
162 case OMP_CLAUSE_LINEAR_VAL:
163 case OMP_CLAUSE_LINEAR_DEFAULT:
164 arg_type
165 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
166 break;
167 default:
168 gcc_unreachable ();
169 }
170 else
171 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
172 clone_info->args[argno].arg_type = arg_type;
173 clone_info->args[argno].linear_step = tree_to_shwi (step);
174 gcc_assert (clone_info->args[argno].linear_step >= 0
175 && clone_info->args[argno].linear_step < n);
176 }
177 else
178 {
179 if (POINTER_TYPE_P (args[argno]))
180 step = fold_convert (ssizetype, step);
181 if (!tree_fits_shwi_p (step))
182 {
183 warning_at (OMP_CLAUSE_LOCATION (t), 0,
184 "ignoring large linear step");
185 args.release ();
186 return NULL;
187 }
188 else if (integer_zerop (step))
189 {
190 warning_at (OMP_CLAUSE_LOCATION (t), 0,
191 "ignoring zero linear step");
192 args.release ();
193 return NULL;
194 }
195 else
196 {
197 enum cgraph_simd_clone_arg_type arg_type;
198 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
199 switch (OMP_CLAUSE_LINEAR_KIND (t))
200 {
201 case OMP_CLAUSE_LINEAR_REF:
202 arg_type
203 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
204 break;
205 case OMP_CLAUSE_LINEAR_UVAL:
206 arg_type
207 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
208 break;
209 case OMP_CLAUSE_LINEAR_VAL:
210 case OMP_CLAUSE_LINEAR_DEFAULT:
211 arg_type
212 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
213 break;
214 default:
215 gcc_unreachable ();
216 }
217 else
218 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
219 clone_info->args[argno].arg_type = arg_type;
220 clone_info->args[argno].linear_step = tree_to_shwi (step);
221 }
222 }
223 break;
224 }
225 case OMP_CLAUSE_UNIFORM:
226 {
227 tree decl = OMP_CLAUSE_DECL (t);
228 int argno = tree_to_uhwi (decl);
229 clone_info->args[argno].arg_type
230 = SIMD_CLONE_ARG_TYPE_UNIFORM;
231 break;
232 }
233 case OMP_CLAUSE_ALIGNED:
234 {
235 tree decl = OMP_CLAUSE_DECL (t);
236 int argno = tree_to_uhwi (decl);
237 clone_info->args[argno].alignment
238 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
239 break;
240 }
241 default:
242 break;
243 }
244 }
245
246 out:
247 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl))))
248 {
249 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
250 "ignoring %<#pragma omp declare simd%> on function "
251 "with %<_Atomic%> qualified return type");
252 args.release ();
253 return NULL;
254 }
255
256 for (unsigned int argno = 0; argno < clone_info->nargs; argno++)
257 if (TYPE_ATOMIC (args[argno])
258 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM)
259 {
260 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
261 "ignoring %<#pragma omp declare simd%> on function "
262 "with %<_Atomic%> qualified non-%<uniform%> argument");
263 args.release ();
264 return NULL;
265 }
266
267 args.release ();
268 return clone_info;
269}
270
271/* Given a SIMD clone in NODE, calculate the characteristic data
272 type and return the coresponding type. The characteristic data
273 type is computed as described in the Intel Vector ABI. */
274
275static tree
276simd_clone_compute_base_data_type (struct cgraph_node *node,
277 struct cgraph_simd_clone *clone_info)
278{
279 tree type = integer_type_node;
280 tree fndecl = node->decl;
281
282 /* a) For non-void function, the characteristic data type is the
283 return type. */
284 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
285 type = TREE_TYPE (TREE_TYPE (fndecl));
286
287 /* b) If the function has any non-uniform, non-linear parameters,
288 then the characteristic data type is the type of the first
289 such parameter. */
290 else
291 {
292 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
293 for (unsigned int i = 0; i < clone_info->nargs; ++i)
294 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
295 {
296 type = map[i];
297 break;
298 }
299 map.release ();
300 }
301
302 /* c) If the characteristic data type determined by a) or b) above
303 is struct, union, or class type which is pass-by-value (except
304 for the type that maps to the built-in complex data type), the
305 characteristic data type is int. */
306 if (RECORD_OR_UNION_TYPE_P (type)
307 && !aggregate_value_p (type, NULL)
308 && TREE_CODE (type) != COMPLEX_TYPE)
309 return integer_type_node;
310
311 /* d) If none of the above three classes is applicable, the
312 characteristic data type is int. */
313
314 return type;
315
316 /* e) For Intel Xeon Phi native and offload compilation, if the
317 resulting characteristic data type is 8-bit or 16-bit integer
318 data type, the characteristic data type is int. */
319 /* Well, we don't handle Xeon Phi yet. */
320}
321
322static tree
323simd_clone_mangle (struct cgraph_node *node,
324 struct cgraph_simd_clone *clone_info)
325{
326 char vecsize_mangle = clone_info->vecsize_mangle;
327 char mask = clone_info->inbranch ? 'M' : 'N';
328 unsigned int simdlen = clone_info->simdlen;
329 unsigned int n;
330 pretty_printer pp;
331
332 gcc_assert (vecsize_mangle && simdlen);
333
334 pp_string (&pp, "_ZGV");
335 pp_character (&pp, vecsize_mangle);
336 pp_character (&pp, mask);
337 pp_decimal_int (&pp, simdlen);
338
339 for (n = 0; n < clone_info->nargs; ++n)
340 {
341 struct cgraph_simd_clone_arg arg = clone_info->args[n];
342
343 switch (arg.arg_type)
344 {
345 case SIMD_CLONE_ARG_TYPE_UNIFORM:
346 pp_character (&pp, 'u');
347 break;
348 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
349 pp_character (&pp, 'l');
350 goto mangle_linear;
351 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
352 pp_character (&pp, 'R');
353 goto mangle_linear;
354 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
355 pp_character (&pp, 'L');
356 goto mangle_linear;
357 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
358 pp_character (&pp, 'U');
359 goto mangle_linear;
360 mangle_linear:
361 gcc_assert (arg.linear_step != 0);
362 if (arg.linear_step > 1)
363 pp_unsigned_wide_integer (&pp, arg.linear_step);
364 else if (arg.linear_step < 0)
365 {
366 pp_character (&pp, 'n');
367 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
368 arg.linear_step));
369 }
370 break;
371 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
372 pp_string (&pp, "ls");
373 pp_unsigned_wide_integer (&pp, arg.linear_step);
374 break;
375 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
376 pp_string (&pp, "Rs");
377 pp_unsigned_wide_integer (&pp, arg.linear_step);
378 break;
379 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
380 pp_string (&pp, "Ls");
381 pp_unsigned_wide_integer (&pp, arg.linear_step);
382 break;
383 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
384 pp_string (&pp, "Us");
385 pp_unsigned_wide_integer (&pp, arg.linear_step);
386 break;
387 default:
388 pp_character (&pp, 'v');
389 }
390 if (arg.alignment)
391 {
392 pp_character (&pp, 'a');
393 pp_decimal_int (&pp, arg.alignment);
394 }
395 }
396
397 pp_underscore (&pp);
398 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
399 if (*str == '*')
400 ++str;
401 pp_string (&pp, str);
402 str = pp_formatted_text (&pp);
403
404 /* If there already is a SIMD clone with the same mangled name, don't
405 add another one. This can happen e.g. for
406 #pragma omp declare simd
407 #pragma omp declare simd simdlen(8)
408 int foo (int, int);
409 if the simdlen is assumed to be 8 for the first one, etc. */
410 for (struct cgraph_node *clone = node->simd_clones; clone;
411 clone = clone->simdclone->next_clone)
412 if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str))
413 return NULL_TREE;
414
415 return get_identifier (str);
416}
417
418/* Create a simd clone of OLD_NODE and return it. */
419
420static struct cgraph_node *
421simd_clone_create (struct cgraph_node *old_node)
422{
423 struct cgraph_node *new_node;
424 if (old_node->definition)
425 {
426 if (!old_node->has_gimple_body_p ())
427 return NULL;
428 old_node->get_body ();
429 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
430 false, NULL, NULL,
431 "simdclone");
432 }
433 else
434 {
435 tree old_decl = old_node->decl;
436 tree new_decl = copy_node (old_node->decl);
437 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
438 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
439 SET_DECL_RTL (new_decl, NULL);
440 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
441 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
442 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
443 if (old_node->in_other_partition)
444 new_node->in_other_partition = 1;
445 }
446 if (new_node == NULL)
447 return new_node;
448
449 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
450 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl);
451 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl);
452 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl);
453 DECL_VISIBILITY_SPECIFIED (new_node->decl)
454 = DECL_VISIBILITY_SPECIFIED (old_node->decl);
455 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl);
456 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl);
457 if (DECL_ONE_ONLY (old_node->decl))
458 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl));
459
460 /* The method cgraph_version_clone_with_body () will force the new
461 symbol local. Undo this, and inherit external visibility from
462 the old node. */
463 new_node->local.local = old_node->local.local;
464 new_node->externally_visible = old_node->externally_visible;
465
466 return new_node;
467}
468
469/* Adjust the return type of the given function to its appropriate
470 vector counterpart. Returns a simd array to be used throughout the
471 function as a return value. */
472
473static tree
474simd_clone_adjust_return_type (struct cgraph_node *node)
475{
476 tree fndecl = node->decl;
477 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
478 unsigned int veclen;
479 tree t;
480
481 /* Adjust the function return type. */
482 if (orig_rettype == void_type_node)
483 return NULL_TREE;
484 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
485 t = TREE_TYPE (TREE_TYPE (fndecl));
486 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
487 veclen = node->simdclone->vecsize_int;
488 else
489 veclen = node->simdclone->vecsize_float;
490 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t));
491 if (veclen > node->simdclone->simdlen)
492 veclen = node->simdclone->simdlen;
493 if (POINTER_TYPE_P (t))
494 t = pointer_sized_int_node;
495 if (veclen == node->simdclone->simdlen)
496 t = build_vector_type (t, node->simdclone->simdlen);
497 else
498 {
499 t = build_vector_type (t, veclen);
500 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
501 }
502 TREE_TYPE (TREE_TYPE (fndecl)) = t;
503 if (!node->definition)
504 return NULL_TREE;
505
506 t = DECL_RESULT (fndecl);
507 /* Adjust the DECL_RESULT. */
508 gcc_assert (TREE_TYPE (t) != void_type_node);
509 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
510 relayout_decl (t);
511
512 tree atype = build_array_type_nelts (orig_rettype,
513 node->simdclone->simdlen);
514 if (veclen != node->simdclone->simdlen)
515 return build1 (VIEW_CONVERT_EXPR, atype, t);
516
517 /* Set up a SIMD array to use as the return value. */
518 tree retval = create_tmp_var_raw (atype, "retval");
519 gimple_add_tmp_var (retval);
520 return retval;
521}
522
523/* Each vector argument has a corresponding array to be used locally
524 as part of the eventual loop. Create such temporary array and
525 return it.
526
527 PREFIX is the prefix to be used for the temporary.
528
529 TYPE is the inner element type.
530
531 SIMDLEN is the number of elements. */
532
533static tree
534create_tmp_simd_array (const char *prefix, tree type, int simdlen)
535{
536 tree atype = build_array_type_nelts (type, simdlen);
537 tree avar = create_tmp_var_raw (atype, prefix);
538 gimple_add_tmp_var (avar);
539 return avar;
540}
541
542/* Modify the function argument types to their corresponding vector
543 counterparts if appropriate. Also, create one array for each simd
544 argument to be used locally when using the function arguments as
545 part of the loop.
546
547 NODE is the function whose arguments are to be adjusted.
548
549 Returns an adjustment vector that will be filled describing how the
550 argument types will be adjusted. */
551
552static ipa_parm_adjustment_vec
553simd_clone_adjust_argument_types (struct cgraph_node *node)
554{
555 vec<tree> args;
556 ipa_parm_adjustment_vec adjustments;
557
558 if (node->definition)
559 args = ipa_get_vector_of_formal_parms (node->decl);
560 else
561 args = simd_clone_vector_of_formal_parm_types (node->decl);
562 adjustments.create (args.length ());
563 unsigned i, j, veclen;
564 struct ipa_parm_adjustment adj;
565 struct cgraph_simd_clone *sc = node->simdclone;
566
567 for (i = 0; i < sc->nargs; ++i)
568 {
569 memset (&adj, 0, sizeof (adj));
570 tree parm = args[i];
571 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
572 adj.base_index = i;
573 adj.base = parm;
574
575 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
576 sc->args[i].orig_type = parm_type;
577
578 switch (sc->args[i].arg_type)
579 {
580 default:
581 /* No adjustment necessary for scalar arguments. */
582 adj.op = IPA_PARM_OP_COPY;
583 break;
584 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
585 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
586 if (node->definition)
587 sc->args[i].simd_array
588 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
589 TREE_TYPE (parm_type),
590 sc->simdlen);
591 adj.op = IPA_PARM_OP_COPY;
592 break;
593 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
594 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
595 case SIMD_CLONE_ARG_TYPE_VECTOR:
596 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
597 veclen = sc->vecsize_int;
598 else
599 veclen = sc->vecsize_float;
600 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type));
601 if (veclen > sc->simdlen)
602 veclen = sc->simdlen;
603 adj.arg_prefix = "simd";
604 if (POINTER_TYPE_P (parm_type))
605 adj.type = build_vector_type (pointer_sized_int_node, veclen);
606 else
607 adj.type = build_vector_type (parm_type, veclen);
608 sc->args[i].vector_type = adj.type;
609 for (j = veclen; j < sc->simdlen; j += veclen)
610 {
611 adjustments.safe_push (adj);
612 if (j == veclen)
613 {
614 memset (&adj, 0, sizeof (adj));
615 adj.op = IPA_PARM_OP_NEW;
616 adj.arg_prefix = "simd";
617 adj.base_index = i;
618 adj.type = sc->args[i].vector_type;
619 }
620 }
621
622 if (node->definition)
623 sc->args[i].simd_array
624 = create_tmp_simd_array (DECL_NAME (parm)
625 ? IDENTIFIER_POINTER (DECL_NAME (parm))
626 : NULL, parm_type, sc->simdlen);
627 }
628 adjustments.safe_push (adj);
629 }
630
631 if (sc->inbranch)
632 {
633 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
634
635 memset (&adj, 0, sizeof (adj));
636 adj.op = IPA_PARM_OP_NEW;
637 adj.arg_prefix = "mask";
638
639 adj.base_index = i;
640 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
641 veclen = sc->vecsize_int;
642 else
643 veclen = sc->vecsize_float;
644 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type));
645 if (veclen > sc->simdlen)
646 veclen = sc->simdlen;
647 if (sc->mask_mode != VOIDmode)
648 adj.type
649 = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
650 else if (POINTER_TYPE_P (base_type))
651 adj.type = build_vector_type (pointer_sized_int_node, veclen);
652 else
653 adj.type = build_vector_type (base_type, veclen);
654 adjustments.safe_push (adj);
655
656 for (j = veclen; j < sc->simdlen; j += veclen)
657 adjustments.safe_push (adj);
658
659 /* We have previously allocated one extra entry for the mask. Use
660 it and fill it. */
661 sc->nargs++;
662 if (sc->mask_mode != VOIDmode)
663 base_type = boolean_type_node;
664 if (node->definition)
665 {
666 sc->args[i].orig_arg
667 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
668 if (sc->mask_mode == VOIDmode)
669 sc->args[i].simd_array
670 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
671 else if (veclen < sc->simdlen)
672 sc->args[i].simd_array
673 = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen);
674 else
675 sc->args[i].simd_array = NULL_TREE;
676 }
677 sc->args[i].orig_type = base_type;
678 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
679 }
680
681 if (node->definition)
682 ipa_modify_formal_parameters (node->decl, adjustments);
683 else
684 {
685 tree new_arg_types = NULL_TREE, new_reversed;
686 bool last_parm_void = false;
687 if (args.length () > 0 && args.last () == void_type_node)
688 last_parm_void = true;
689
690 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
691 j = adjustments.length ();
692 for (i = 0; i < j; i++)
693 {
694 struct ipa_parm_adjustment *adj = &adjustments[i];
695 tree ptype;
696 if (adj->op == IPA_PARM_OP_COPY)
697 ptype = args[adj->base_index];
698 else
699 ptype = adj->type;
700 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
701 }
702 new_reversed = nreverse (new_arg_types);
703 if (last_parm_void)
704 {
705 if (new_reversed)
706 TREE_CHAIN (new_arg_types) = void_list_node;
707 else
708 new_reversed = void_list_node;
709 }
710
711 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
712 TYPE_ARG_TYPES (new_type) = new_reversed;
713 TREE_TYPE (node->decl) = new_type;
714
715 adjustments.release ();
716 }
717 args.release ();
718 return adjustments;
719}
720
721/* Initialize and copy the function arguments in NODE to their
722 corresponding local simd arrays. Returns a fresh gimple_seq with
723 the instruction sequence generated. */
724
725static gimple_seq
726simd_clone_init_simd_arrays (struct cgraph_node *node,
727 ipa_parm_adjustment_vec adjustments)
728{
729 gimple_seq seq = NULL;
730 unsigned i = 0, j = 0, k;
731
732 for (tree arg = DECL_ARGUMENTS (node->decl);
733 arg;
734 arg = DECL_CHAIN (arg), i++, j++)
735 {
736 if (adjustments[j].op == IPA_PARM_OP_COPY
737 || POINTER_TYPE_P (TREE_TYPE (arg)))
738 continue;
739
740 node->simdclone->args[i].vector_arg = arg;
741
742 tree array = node->simdclone->args[i].simd_array;
743 if (node->simdclone->mask_mode != VOIDmode
744 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
745 {
746 if (array == NULL_TREE)
747 continue;
748 unsigned int l
749 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
750 for (k = 0; k <= l; k++)
751 {
752 if (k)
753 {
754 arg = DECL_CHAIN (arg);
755 j++;
756 }
757 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
758 array, size_int (k), NULL, NULL);
759 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
760 gimplify_and_add (t, &seq);
761 }
762 continue;
763 }
764 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen)
765 {
766 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
767 tree ptr = build_fold_addr_expr (array);
768 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
769 build_int_cst (ptype, 0));
770 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
771 gimplify_and_add (t, &seq);
772 }
773 else
774 {
775 unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
776 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
777 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
778 {
779 tree ptr = build_fold_addr_expr (array);
780 int elemsize;
781 if (k)
782 {
783 arg = DECL_CHAIN (arg);
784 j++;
785 }
786 tree elemtype = TREE_TYPE (TREE_TYPE (arg));
787 elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype));
788 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
789 build_int_cst (ptype, k * elemsize));
790 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
791 gimplify_and_add (t, &seq);
792 }
793 }
794 }
795 return seq;
796}
797
798/* Callback info for ipa_simd_modify_stmt_ops below. */
799
800struct modify_stmt_info {
801 ipa_parm_adjustment_vec adjustments;
802 gimple *stmt;
803 /* True if the parent statement was modified by
804 ipa_simd_modify_stmt_ops. */
805 bool modified;
806};
807
808/* Callback for walk_gimple_op.
809
810 Adjust operands from a given statement as specified in the
811 adjustments vector in the callback data. */
812
813static tree
814ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
815{
816 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
817 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
818 tree *orig_tp = tp;
819 if (TREE_CODE (*tp) == ADDR_EXPR)
820 tp = &TREE_OPERAND (*tp, 0);
821 struct ipa_parm_adjustment *cand = NULL;
822 if (TREE_CODE (*tp) == PARM_DECL)
823 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
824 else
825 {
826 if (TYPE_P (*tp))
827 *walk_subtrees = 0;
828 }
829
830 tree repl = NULL_TREE;
831 if (cand)
832 repl = unshare_expr (cand->new_decl);
833 else
834 {
835 if (tp != orig_tp)
836 {
837 *walk_subtrees = 0;
838 bool modified = info->modified;
839 info->modified = false;
840 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
841 if (!info->modified)
842 {
843 info->modified = modified;
844 return NULL_TREE;
845 }
846 info->modified = modified;
847 repl = *tp;
848 }
849 else
850 return NULL_TREE;
851 }
852
853 if (tp != orig_tp)
854 {
855 repl = build_fold_addr_expr (repl);
856 gimple *stmt;
857 if (is_gimple_debug (info->stmt))
858 {
859 tree vexpr = make_node (DEBUG_EXPR_DECL);
860 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
861 DECL_ARTIFICIAL (vexpr) = 1;
862 TREE_TYPE (vexpr) = TREE_TYPE (repl);
863 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl)));
864 repl = vexpr;
865 }
866 else
867 {
868 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
869 repl = gimple_assign_lhs (stmt);
870 }
871 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
872 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
873 *orig_tp = repl;
874 }
875 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
876 {
877 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
878 *tp = vce;
879 }
880 else
881 *tp = repl;
882
883 info->modified = true;
884 return NULL_TREE;
885}
886
887/* Traverse the function body and perform all modifications as
888 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
889 modified such that the replacement/reduction value will now be an
890 offset into the corresponding simd_array.
891
892 This function will replace all function argument uses with their
893 corresponding simd array elements, and ajust the return values
894 accordingly. */
895
896static void
897ipa_simd_modify_function_body (struct cgraph_node *node,
898 ipa_parm_adjustment_vec adjustments,
899 tree retval_array, tree iter)
900{
901 basic_block bb;
902 unsigned int i, j, l;
903
904 /* Re-use the adjustments array, but this time use it to replace
905 every function argument use to an offset into the corresponding
906 simd_array. */
907 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
908 {
909 if (!node->simdclone->args[i].vector_arg)
910 continue;
911
912 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
913 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
914 adjustments[j].new_decl
915 = build4 (ARRAY_REF,
916 basetype,
917 node->simdclone->args[i].simd_array,
918 iter,
919 NULL_TREE, NULL_TREE);
920 if (adjustments[j].op == IPA_PARM_OP_NONE
921 && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen)
922 j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1;
923 }
924
925 l = adjustments.length ();
926 tree name;
927
928 FOR_EACH_SSA_NAME (i, name, cfun)
929 {
930 if (SSA_NAME_VAR (name)
931 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
932 {
933 for (j = 0; j < l; j++)
934 if (SSA_NAME_VAR (name) == adjustments[j].base
935 && adjustments[j].new_decl)
936 {
937 tree base_var;
938 if (adjustments[j].new_ssa_base == NULL_TREE)
939 {
940 base_var
941 = copy_var_decl (adjustments[j].base,
942 DECL_NAME (adjustments[j].base),
943 TREE_TYPE (adjustments[j].base));
944 adjustments[j].new_ssa_base = base_var;
945 }
946 else
947 base_var = adjustments[j].new_ssa_base;
948 if (SSA_NAME_IS_DEFAULT_DEF (name))
949 {
950 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
951 gimple_stmt_iterator gsi = gsi_after_labels (bb);
952 tree new_decl = unshare_expr (adjustments[j].new_decl);
953 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
954 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
955 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
956 gimple *stmt = gimple_build_assign (name, new_decl);
957 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
958 }
959 else
960 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
961 }
962 }
963 }
964
965 struct modify_stmt_info info;
966 info.adjustments = adjustments;
967
968 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
969 {
970 gimple_stmt_iterator gsi;
971
972 gsi = gsi_start_bb (bb);
973 while (!gsi_end_p (gsi))
974 {
975 gimple *stmt = gsi_stmt (gsi);
976 info.stmt = stmt;
977 struct walk_stmt_info wi;
978
979 memset (&wi, 0, sizeof (wi));
980 info.modified = false;
981 wi.info = &info;
982 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
983
984 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
985 {
986 tree retval = gimple_return_retval (return_stmt);
987 if (!retval)
988 {
989 gsi_remove (&gsi, true);
990 continue;
991 }
992
993 /* Replace `return foo' with `retval_array[iter] = foo'. */
994 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
995 retval_array, iter, NULL, NULL);
996 stmt = gimple_build_assign (ref, retval);
997 gsi_replace (&gsi, stmt, true);
998 info.modified = true;
999 }
1000
1001 if (info.modified)
1002 {
1003 update_stmt (stmt);
1004 if (maybe_clean_eh_stmt (stmt))
1005 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1006 }
1007 gsi_next (&gsi);
1008 }
1009 }
1010}
1011
1012/* Helper function of simd_clone_adjust, return linear step addend
1013 of Ith argument. */
1014
1015static tree
1016simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
1017 tree addtype, basic_block entry_bb)
1018{
1019 tree ptype = NULL_TREE;
1020 switch (node->simdclone->args[i].arg_type)
1021 {
1022 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
1023 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
1024 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1025 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
1026 return build_int_cst (addtype, node->simdclone->args[i].linear_step);
1027 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
1028 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
1029 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1030 break;
1031 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1032 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1033 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
1034 break;
1035 default:
1036 gcc_unreachable ();
1037 }
1038
1039 unsigned int idx = node->simdclone->args[i].linear_step;
1040 tree arg = node->simdclone->args[idx].orig_arg;
1041 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
1042 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1043 gimple *g;
1044 tree ret;
1045 if (is_gimple_reg (arg))
1046 ret = get_or_create_ssa_default_def (cfun, arg);
1047 else
1048 {
1049 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
1050 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1051 ret = gimple_assign_lhs (g);
1052 }
1053 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
1054 {
1055 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
1056 build_simple_mem_ref (ret));
1057 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1058 ret = gimple_assign_lhs (g);
1059 }
1060 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
1061 {
1062 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
1063 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1064 ret = gimple_assign_lhs (g);
1065 }
1066 if (POINTER_TYPE_P (ptype))
1067 {
1068 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
1069 if (size && TREE_CODE (size) == INTEGER_CST)
1070 {
1071 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
1072 ret, fold_convert (addtype, size));
1073 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1074 ret = gimple_assign_lhs (g);
1075 }
1076 }
1077 return ret;
1078}
1079
1080/* Adjust the argument types in NODE to their appropriate vector
1081 counterparts. */
1082
1083static void
1084simd_clone_adjust (struct cgraph_node *node)
1085{
1086 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
1087
1088 targetm.simd_clone.adjust (node);
1089
1090 tree retval = simd_clone_adjust_return_type (node);
1091 ipa_parm_adjustment_vec adjustments
1092 = simd_clone_adjust_argument_types (node);
1093
1094 push_gimplify_context ();
1095
1096 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
1097
1098 /* Adjust all uses of vector arguments accordingly. Adjust all
1099 return values accordingly. */
1100 tree iter = create_tmp_var (unsigned_type_node, "iter");
1101 tree iter1 = make_ssa_name (iter);
1102 tree iter2 = NULL_TREE;
1103 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
1104 adjustments.release ();
1105
1106 /* Initialize the iteration variable. */
1107 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1108 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
1109 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1110 /* Insert the SIMD array and iv initialization at function
1111 entry. */
1112 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
1113
1114 pop_gimplify_context (NULL);
1115
1116 gimple *g;
1117 basic_block incr_bb = NULL;
1118 struct loop *loop = NULL;
1119
1120 /* Create a new BB right before the original exit BB, to hold the
1121 iteration increment and the condition/branch. */
1122 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1123 {
1124 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
1125 incr_bb = create_empty_bb (orig_exit);
1126 incr_bb->count = profile_count::zero ();
1127 add_bb_to_loop (incr_bb, body_bb->loop_father);
1128 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
1129 flag. Set it now to be a FALLTHRU_EDGE. */
1130 gcc_assert (EDGE_COUNT (orig_exit->succs) == 1);
1131 EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU;
1132 for (unsigned i = 0;
1133 i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i)
1134 {
1135 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
1136 redirect_edge_succ (e, incr_bb);
1137 incr_bb->count += e->count ();
1138 }
1139 }
1140 else if (node->simdclone->inbranch)
1141 {
1142 incr_bb = create_empty_bb (entry_bb);
1143 incr_bb->count = profile_count::zero ();
1144 add_bb_to_loop (incr_bb, body_bb->loop_father);
1145 }
1146
1147 if (incr_bb)
1148 {
1149 make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1150 gsi = gsi_last_bb (incr_bb);
1151 iter2 = make_ssa_name (iter);
1152 g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
1153 build_int_cst (unsigned_type_node, 1));
1154 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1155
1156 /* Mostly annotate the loop for the vectorizer (the rest is done
1157 below). */
1158 loop = alloc_loop ();
1159 cfun->has_force_vectorize_loops = true;
1160 loop->safelen = node->simdclone->simdlen;
1161 loop->force_vectorize = true;
1162 loop->header = body_bb;
1163 }
1164
1165 /* Branch around the body if the mask applies. */
1166 if (node->simdclone->inbranch)
1167 {
1168 gsi = gsi_last_bb (loop->header);
1169 tree mask_array
1170 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
1171 tree mask;
1172 if (node->simdclone->mask_mode != VOIDmode)
1173 {
1174 tree shift_cnt;
1175 if (mask_array == NULL_TREE)
1176 {
1177 tree arg = node->simdclone->args[node->simdclone->nargs
1178 - 1].vector_arg;
1179 mask = get_or_create_ssa_default_def (cfun, arg);
1180 shift_cnt = iter1;
1181 }
1182 else
1183 {
1184 tree maskt = TREE_TYPE (mask_array);
1185 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
1186 c = node->simdclone->simdlen / (c + 1);
1187 int s = exact_log2 (c);
1188 gcc_assert (s > 0);
1189 c--;
1190 tree idx = make_ssa_name (TREE_TYPE (iter1));
1191 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
1192 build_int_cst (NULL_TREE, s));
1193 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1194 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1195 tree aref = build4 (ARRAY_REF,
1196 TREE_TYPE (TREE_TYPE (mask_array)),
1197 mask_array, idx, NULL, NULL);
1198 g = gimple_build_assign (mask, aref);
1199 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1200 shift_cnt = make_ssa_name (TREE_TYPE (iter1));
1201 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
1202 build_int_cst (TREE_TYPE (iter1), c));
1203 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1204 }
1205 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1206 RSHIFT_EXPR, mask, shift_cnt);
1207 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1208 mask = gimple_assign_lhs (g);
1209 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1210 BIT_AND_EXPR, mask,
1211 build_int_cst (TREE_TYPE (mask), 1));
1212 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1213 mask = gimple_assign_lhs (g);
1214 }
1215 else
1216 {
1217 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1218 tree aref = build4 (ARRAY_REF,
1219 TREE_TYPE (TREE_TYPE (mask_array)),
1220 mask_array, iter1, NULL, NULL);
1221 g = gimple_build_assign (mask, aref);
1222 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1223 int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref)));
1224 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
1225 {
1226 aref = build1 (VIEW_CONVERT_EXPR,
1227 build_nonstandard_integer_type (bitsize, 0),
1228 mask);
1229 mask = make_ssa_name (TREE_TYPE (aref));
1230 g = gimple_build_assign (mask, aref);
1231 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1232 }
1233 }
1234
1235 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
1236 NULL, NULL);
1237 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1238 edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
1239 e->probability = profile_probability::unlikely ().guessed ();
1240 incr_bb->count += e->count ();
1241 edge fallthru = FALLTHRU_EDGE (loop->header);
1242 fallthru->flags = EDGE_FALSE_VALUE;
1243 fallthru->probability = profile_probability::likely ().guessed ();
1244 }
1245
1246 basic_block latch_bb = NULL;
1247 basic_block new_exit_bb = NULL;
1248
1249 /* Generate the condition. */
1250 if (incr_bb)
1251 {
1252 gsi = gsi_last_bb (incr_bb);
1253 g = gimple_build_cond (LT_EXPR, iter2,
1254 build_int_cst (unsigned_type_node,
1255 node->simdclone->simdlen),
1256 NULL, NULL);
1257 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1258 edge e = split_block (incr_bb, gsi_stmt (gsi));
1259 latch_bb = e->dest;
1260 new_exit_bb = split_block_after_labels (latch_bb)->dest;
1261 loop->latch = latch_bb;
1262
1263 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
1264
1265 edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
1266
1267 /* FIXME: Do we need to distribute probabilities for the conditional? */
1268 new_e->probability = profile_probability::guessed_never ();
1269 /* The successor of incr_bb is already pointing to latch_bb; just
1270 change the flags.
1271 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1272 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
1273 }
1274
1275 gphi *phi = create_phi_node (iter1, body_bb);
1276 edge preheader_edge = find_edge (entry_bb, body_bb);
1277 edge latch_edge = NULL;
1278 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
1279 UNKNOWN_LOCATION);
1280 if (incr_bb)
1281 {
1282 latch_edge = single_succ_edge (latch_bb);
1283 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1284
1285 /* Generate the new return. */
1286 gsi = gsi_last_bb (new_exit_bb);
1287 if (retval
1288 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
1289 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
1290 retval = TREE_OPERAND (retval, 0);
1291 else if (retval)
1292 {
1293 retval = build1 (VIEW_CONVERT_EXPR,
1294 TREE_TYPE (TREE_TYPE (node->decl)),
1295 retval);
1296 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
1297 false, GSI_CONTINUE_LINKING);
1298 }
1299 g = gimple_build_return (retval);
1300 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1301 }
1302
1303 /* Handle aligned clauses by replacing default defs of the aligned
1304 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1305 lhs. Handle linear by adding PHIs. */
1306 for (unsigned i = 0; i < node->simdclone->nargs; i++)
1307 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1308 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
1309 || !is_gimple_reg_type
1310 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1311 {
1312 tree orig_arg = node->simdclone->args[i].orig_arg;
1313 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
1314 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1315 else
1316 {
1317 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
1318 gimple_add_tmp_var (iter1);
1319 }
1320 gsi = gsi_after_labels (entry_bb);
1321 g = gimple_build_assign (iter1, orig_arg);
1322 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1323 gsi = gsi_after_labels (body_bb);
1324 g = gimple_build_assign (orig_arg, iter1);
1325 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1326 }
1327 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1328 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
1329 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1330 == REFERENCE_TYPE
1331 && TREE_ADDRESSABLE
1332 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1333 {
1334 tree orig_arg = node->simdclone->args[i].orig_arg;
1335 tree def = ssa_default_def (cfun, orig_arg);
1336 if (def && !has_zero_uses (def))
1337 {
1338 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
1339 gimple_add_tmp_var (iter1);
1340 gsi = gsi_after_labels (entry_bb);
1341 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
1342 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1343 gsi = gsi_after_labels (body_bb);
1344 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
1345 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1346 }
1347 }
1348 else if (node->simdclone->args[i].alignment
1349 && node->simdclone->args[i].arg_type
1350 == SIMD_CLONE_ARG_TYPE_UNIFORM
1351 && (node->simdclone->args[i].alignment
1352 & (node->simdclone->args[i].alignment - 1)) == 0
1353 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1354 == POINTER_TYPE)
1355 {
1356 unsigned int alignment = node->simdclone->args[i].alignment;
1357 tree orig_arg = node->simdclone->args[i].orig_arg;
1358 tree def = ssa_default_def (cfun, orig_arg);
1359 if (def && !has_zero_uses (def))
1360 {
1361 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
1362 gimple_seq seq = NULL;
1363 bool need_cvt = false;
1364 gcall *call
1365 = gimple_build_call (fn, 2, def, size_int (alignment));
1366 g = call;
1367 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
1368 ptr_type_node))
1369 need_cvt = true;
1370 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
1371 gimple_call_set_lhs (g, t);
1372 gimple_seq_add_stmt_without_update (&seq, g);
1373 if (need_cvt)
1374 {
1375 t = make_ssa_name (orig_arg);
1376 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
1377 gimple_seq_add_stmt_without_update (&seq, g);
1378 }
1379 gsi_insert_seq_on_edge_immediate
1380 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
1381
1382 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1383 node->create_edge (cgraph_node::get_create (fn),
1384 call, entry_bb->count);
1385
1386 imm_use_iterator iter;
1387 use_operand_p use_p;
1388 gimple *use_stmt;
1389 tree repl = gimple_get_lhs (g);
1390 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1391 if (is_gimple_debug (use_stmt) || use_stmt == call)
1392 continue;
1393 else
1394 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1395 SET_USE (use_p, repl);
1396 }
1397 }
1398 else if ((node->simdclone->args[i].arg_type
1399 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
1400 || (node->simdclone->args[i].arg_type
1401 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
1402 || (node->simdclone->args[i].arg_type
1403 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
1404 || (node->simdclone->args[i].arg_type
1405 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
1406 {
1407 tree orig_arg = node->simdclone->args[i].orig_arg;
1408 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1409 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
1410 tree def = NULL_TREE;
1411 if (TREE_ADDRESSABLE (orig_arg))
1412 {
1413 def = make_ssa_name (TREE_TYPE (orig_arg));
1414 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1415 if (incr_bb)
1416 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
1417 gsi = gsi_after_labels (entry_bb);
1418 g = gimple_build_assign (def, orig_arg);
1419 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1420 }
1421 else
1422 {
1423 def = ssa_default_def (cfun, orig_arg);
1424 if (!def || has_zero_uses (def))
1425 def = NULL_TREE;
1426 else
1427 {
1428 iter1 = make_ssa_name (orig_arg);
1429 if (incr_bb)
1430 iter2 = make_ssa_name (orig_arg);
1431 }
1432 }
1433 if (def)
1434 {
1435 phi = create_phi_node (iter1, body_bb);
1436 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
1437 if (incr_bb)
1438 {
1439 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1440 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1441 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1442 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1443 ? TREE_TYPE (orig_arg) : sizetype;
1444 tree addcst = simd_clone_linear_addend (node, i, addtype,
1445 entry_bb);
1446 gsi = gsi_last_bb (incr_bb);
1447 g = gimple_build_assign (iter2, code, iter1, addcst);
1448 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1449 }
1450
1451 imm_use_iterator iter;
1452 use_operand_p use_p;
1453 gimple *use_stmt;
1454 if (TREE_ADDRESSABLE (orig_arg))
1455 {
1456 gsi = gsi_after_labels (body_bb);
1457 g = gimple_build_assign (orig_arg, iter1);
1458 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1459 }
1460 else
1461 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1462 if (use_stmt == phi)
1463 continue;
1464 else
1465 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1466 SET_USE (use_p, iter1);
1467 }
1468 }
1469 else if (node->simdclone->args[i].arg_type
1470 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1471 || (node->simdclone->args[i].arg_type
1472 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
1473 {
1474 tree orig_arg = node->simdclone->args[i].orig_arg;
1475 tree def = ssa_default_def (cfun, orig_arg);
1476 gcc_assert (!TREE_ADDRESSABLE (orig_arg)
1477 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
1478 if (def && !has_zero_uses (def))
1479 {
1480 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
1481 iter1 = make_ssa_name (orig_arg);
1482 if (incr_bb)
1483 iter2 = make_ssa_name (orig_arg);
1484 tree iter3 = make_ssa_name (rtype);
1485 tree iter4 = make_ssa_name (rtype);
1486 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
1487 gsi = gsi_after_labels (entry_bb);
1488 gimple *load
1489 = gimple_build_assign (iter3, build_simple_mem_ref (def));
1490 gsi_insert_before (&gsi, load, GSI_NEW_STMT);
1491
1492 tree array = node->simdclone->args[i].simd_array;
1493 TREE_ADDRESSABLE (array) = 1;
1494 tree ptr = build_fold_addr_expr (array);
1495 phi = create_phi_node (iter1, body_bb);
1496 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
1497 if (incr_bb)
1498 {
1499 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1500 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
1501 TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
1502 gsi = gsi_last_bb (incr_bb);
1503 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1504 }
1505
1506 phi = create_phi_node (iter4, body_bb);
1507 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
1508 if (incr_bb)
1509 {
1510 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
1511 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1512 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1513 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1514 ? TREE_TYPE (iter3) : sizetype;
1515 tree addcst = simd_clone_linear_addend (node, i, addtype,
1516 entry_bb);
1517 g = gimple_build_assign (iter5, code, iter4, addcst);
1518 gsi = gsi_last_bb (incr_bb);
1519 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1520 }
1521
1522 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
1523 gsi = gsi_after_labels (body_bb);
1524 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1525
1526 imm_use_iterator iter;
1527 use_operand_p use_p;
1528 gimple *use_stmt;
1529 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1530 if (use_stmt == load)
1531 continue;
1532 else
1533 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1534 SET_USE (use_p, iter1);
1535
1536 if (!TYPE_READONLY (rtype) && incr_bb)
1537 {
1538 tree v = make_ssa_name (rtype);
1539 tree aref = build4 (ARRAY_REF, rtype, array,
1540 size_zero_node, NULL_TREE,
1541 NULL_TREE);
1542 gsi = gsi_after_labels (new_exit_bb);
1543 g = gimple_build_assign (v, aref);
1544 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1545 g = gimple_build_assign (build_simple_mem_ref (def), v);
1546 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1547 }
1548 }
1549 }
1550
1551 calculate_dominance_info (CDI_DOMINATORS);
1552 if (loop)
1553 add_loop (loop, loop->header->loop_father);
1554 update_ssa (TODO_update_ssa);
1555
1556 pop_cfun ();
1557}
1558
1559/* If the function in NODE is tagged as an elemental SIMD function,
1560 create the appropriate SIMD clones. */
1561
1562static void
1563expand_simd_clones (struct cgraph_node *node)
1564{
1565 tree attr = lookup_attribute ("omp declare simd",
1566 DECL_ATTRIBUTES (node->decl));
1567 if (attr == NULL_TREE
1568 || node->global.inlined_to
1569 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
1570 return;
1571
1572 /* Ignore
1573 #pragma omp declare simd
1574 extern int foo ();
1575 in C, there we don't know the argument types at all. */
1576 if (!node->definition
1577 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
1578 return;
1579
1580 /* Call this before creating clone_info, as it might ggc_collect. */
1581 if (node->definition && node->has_gimple_body_p ())
1582 node->get_body ();
1583
1584 do
1585 {
1586 /* Start with parsing the "omp declare simd" attribute(s). */
1587 bool inbranch_clause_specified;
1588 struct cgraph_simd_clone *clone_info
1589 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
1590 &inbranch_clause_specified);
1591 if (clone_info == NULL)
1592 continue;
1593
1594 int orig_simdlen = clone_info->simdlen;
1595 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
1596 /* The target can return 0 (no simd clones should be created),
1597 1 (just one ISA of simd clones should be created) or higher
1598 count of ISA variants. In that case, clone_info is initialized
1599 for the first ISA variant. */
1600 int count
1601 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
1602 base_type, 0);
1603 if (count == 0)
1604 continue;
1605
1606 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1607 also create one inbranch and one !inbranch clone of it. */
1608 for (int i = 0; i < count * 2; i++)
1609 {
1610 struct cgraph_simd_clone *clone = clone_info;
1611 if (inbranch_clause_specified && (i & 1) != 0)
1612 continue;
1613
1614 if (i != 0)
1615 {
1616 clone = simd_clone_struct_alloc (clone_info->nargs
1617 + ((i & 1) != 0));
1618 simd_clone_struct_copy (clone, clone_info);
1619 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1620 and simd_clone_adjust_argument_types did to the first
1621 clone's info. */
1622 clone->nargs -= clone_info->inbranch;
1623 clone->simdlen = orig_simdlen;
1624 /* And call the target hook again to get the right ISA. */
1625 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
1626 base_type,
1627 i / 2);
1628 if ((i & 1) != 0)
1629 clone->inbranch = 1;
1630 }
1631
1632 /* simd_clone_mangle might fail if such a clone has been created
1633 already. */
1634 tree id = simd_clone_mangle (node, clone);
1635 if (id == NULL_TREE)
1636 continue;
1637
1638 /* Only when we are sure we want to create the clone actually
1639 clone the function (or definitions) or create another
1640 extern FUNCTION_DECL (for prototypes without definitions). */
1641 struct cgraph_node *n = simd_clone_create (node);
1642 if (n == NULL)
1643 continue;
1644
1645 n->simdclone = clone;
1646 clone->origin = node;
1647 clone->next_clone = NULL;
1648 if (node->simd_clones == NULL)
1649 {
1650 clone->prev_clone = n;
1651 node->simd_clones = n;
1652 }
1653 else
1654 {
1655 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
1656 clone->prev_clone->simdclone->next_clone = n;
1657 node->simd_clones->simdclone->prev_clone = n;
1658 }
1659 symtab->change_decl_assembler_name (n->decl, id);
1660 /* And finally adjust the return type, parameters and for
1661 definitions also function body. */
1662 if (node->definition)
1663 simd_clone_adjust (n);
1664 else
1665 {
1666 simd_clone_adjust_return_type (n);
1667 simd_clone_adjust_argument_types (n);
1668 }
1669 }
1670 }
1671 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
1672}
1673
1674/* Entry point for IPA simd clone creation pass. */
1675
1676static unsigned int
1677ipa_omp_simd_clone (void)
1678{
1679 struct cgraph_node *node;
1680 FOR_EACH_FUNCTION (node)
1681 expand_simd_clones (node);
1682 return 0;
1683}
1684
1685namespace {
1686
1687const pass_data pass_data_omp_simd_clone =
1688{
1689 SIMPLE_IPA_PASS, /* type */
1690 "simdclone", /* name */
1691 OPTGROUP_OMP, /* optinfo_flags */
1692 TV_NONE, /* tv_id */
1693 ( PROP_ssa | PROP_cfg ), /* properties_required */
1694 0, /* properties_provided */
1695 0, /* properties_destroyed */
1696 0, /* todo_flags_start */
1697 0, /* todo_flags_finish */
1698};
1699
1700class pass_omp_simd_clone : public simple_ipa_opt_pass
1701{
1702public:
1703 pass_omp_simd_clone(gcc::context *ctxt)
1704 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
1705 {}
1706
1707 /* opt_pass methods: */
1708 virtual bool gate (function *);
1709 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
1710};
1711
1712bool
1713pass_omp_simd_clone::gate (function *)
1714{
1715 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
1716}
1717
1718} // anon namespace
1719
1720simple_ipa_opt_pass *
1721make_pass_omp_simd_clone (gcc::context *ctxt)
1722{
1723 return new pass_omp_simd_clone (ctxt);
1724}
1725