1 | /* Lowering pass for OMP directives. Converts OMP directives into explicit |
2 | calls to the runtime library (libgomp), data marshalling to implement data |
3 | sharing and copying clauses, offloading to accelerators, and more. |
4 | |
5 | Contributed by Diego Novillo <dnovillo@redhat.com> |
6 | |
7 | Copyright (C) 2005-2024 Free Software Foundation, Inc. |
8 | |
9 | This file is part of GCC. |
10 | |
11 | GCC is free software; you can redistribute it and/or modify it under |
12 | the terms of the GNU General Public License as published by the Free |
13 | Software Foundation; either version 3, or (at your option) any later |
14 | version. |
15 | |
16 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
17 | WARRANTY; without even the implied warranty of MERCHANTABILITY or |
18 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
19 | for more details. |
20 | |
21 | You should have received a copy of the GNU General Public License |
22 | along with GCC; see the file COPYING3. If not see |
23 | <http://www.gnu.org/licenses/>. */ |
24 | |
25 | #include "config.h" |
26 | #include "system.h" |
27 | #include "coretypes.h" |
28 | #include "backend.h" |
29 | #include "target.h" |
30 | #include "tree.h" |
31 | #include "gimple.h" |
32 | #include "tree-pass.h" |
33 | #include "ssa.h" |
34 | #include "cgraph.h" |
35 | #include "pretty-print.h" |
36 | #include "diagnostic-core.h" |
37 | #include "fold-const.h" |
38 | #include "stor-layout.h" |
39 | #include "internal-fn.h" |
40 | #include "gimple-iterator.h" |
41 | #include "gimple-fold.h" |
42 | #include "gimplify.h" |
43 | #include "gimplify-me.h" |
44 | #include "gimple-walk.h" |
45 | #include "tree-iterator.h" |
46 | #include "tree-inline.h" |
47 | #include "langhooks.h" |
48 | #include "tree-dfa.h" |
49 | #include "tree-ssa.h" |
50 | #include "splay-tree.h" |
51 | #include "omp-general.h" |
52 | #include "omp-low.h" |
53 | #include "gimple-low.h" |
54 | #include "alloc-pool.h" |
55 | #include "symbol-summary.h" |
56 | #include "tree-nested.h" |
57 | #include "context.h" |
58 | #include "gomp-constants.h" |
59 | #include "gimple-pretty-print.h" |
60 | #include "stringpool.h" |
61 | #include "attribs.h" |
62 | #include "omp-offload.h" |
63 | |
64 | /* Lowering of OMP parallel and workshare constructs proceeds in two |
65 | phases. The first phase scans the function looking for OMP statements |
66 | and then for variables that must be replaced to satisfy data sharing |
67 | clauses. The second phase expands code for the constructs, as well as |
68 | re-gimplifying things when variables have been replaced with complex |
69 | expressions. |
70 | |
71 | Final code generation is done by pass_expand_omp. The flowgraph is |
72 | scanned for regions which are then moved to a new |
73 | function, to be invoked by the thread library, or offloaded. */ |
74 | |
75 | /* Context structure. Used to store information about each parallel |
76 | directive in the code. */ |
77 | |
78 | struct omp_context |
79 | { |
80 | /* This field must be at the beginning, as we do "inheritance": Some |
81 | callback functions for tree-inline.cc (e.g., omp_copy_decl) |
82 | receive a copy_body_data pointer that is up-casted to an |
83 | omp_context pointer. */ |
84 | copy_body_data cb; |
85 | |
86 | /* The tree of contexts corresponding to the encountered constructs. */ |
87 | struct omp_context *outer; |
88 | gimple *stmt; |
89 | |
90 | /* Map variables to fields in a structure that allows communication |
91 | between sending and receiving threads. */ |
92 | splay_tree field_map; |
93 | tree record_type; |
94 | tree sender_decl; |
95 | tree receiver_decl; |
96 | |
97 | /* These are used just by task contexts, if task firstprivate fn is |
98 | needed. srecord_type is used to communicate from the thread |
99 | that encountered the task construct to task firstprivate fn, |
100 | record_type is allocated by GOMP_task, initialized by task firstprivate |
101 | fn and passed to the task body fn. */ |
102 | splay_tree sfield_map; |
103 | tree srecord_type; |
104 | |
105 | /* A chain of variables to add to the top-level block surrounding the |
106 | construct. In the case of a parallel, this is in the child function. */ |
107 | tree block_vars; |
108 | |
109 | /* Label to which GOMP_cancel{,llation_point} and explicit and implicit |
110 | barriers should jump to during omplower pass. */ |
111 | tree cancel_label; |
112 | |
113 | /* The sibling GIMPLE_OMP_FOR simd with _simt_ clause or NULL |
114 | otherwise. */ |
115 | gimple *simt_stmt; |
116 | |
117 | /* For task reductions registered in this context, a vector containing |
118 | the length of the private copies block (if constant, otherwise NULL) |
119 | and then offsets (if constant, otherwise NULL) for each entry. */ |
120 | vec<tree> task_reductions; |
121 | |
122 | /* A hash map from the reduction clauses to the registered array |
123 | elts. */ |
124 | hash_map<tree, unsigned> *task_reduction_map; |
125 | |
126 | /* And a hash map from the lastprivate(conditional:) variables to their |
127 | corresponding tracking loop iteration variables. */ |
128 | hash_map<tree, tree> *lastprivate_conditional_map; |
129 | |
130 | /* And a hash map from the allocate variables to their corresponding |
131 | allocators. */ |
132 | hash_map<tree, tree> *allocate_map; |
133 | |
134 | /* A tree_list of the reduction clauses in this context. This is |
135 | only used for checking the consistency of OpenACC reduction |
136 | clauses in scan_omp_for and is not guaranteed to contain a valid |
137 | value outside of this function. */ |
138 | tree local_reduction_clauses; |
139 | |
140 | /* A tree_list of the reduction clauses in outer contexts. This is |
141 | only used for checking the consistency of OpenACC reduction |
142 | clauses in scan_omp_for and is not guaranteed to contain a valid |
143 | value outside of this function. */ |
144 | tree outer_reduction_clauses; |
145 | |
146 | /* Nesting depth of this context. Used to beautify error messages re |
147 | invalid gotos. The outermost ctx is depth 1, with depth 0 being |
148 | reserved for the main body of the function. */ |
149 | int depth; |
150 | |
151 | /* True if this parallel directive is nested within another. */ |
152 | bool is_nested; |
153 | |
154 | /* True if this construct can be cancelled. */ |
155 | bool cancellable; |
156 | |
157 | /* True if lower_omp_1 should look up lastprivate conditional in parent |
158 | context. */ |
159 | bool combined_into_simd_safelen1; |
160 | |
161 | /* True if there is nested scan context with inclusive clause. */ |
162 | bool scan_inclusive; |
163 | |
164 | /* True if there is nested scan context with exclusive clause. */ |
165 | bool scan_exclusive; |
166 | |
167 | /* True in the second simd loop of for simd with inscan reductions. */ |
168 | bool for_simd_scan_phase; |
169 | |
170 | /* True if there is order(concurrent) clause on the construct. */ |
171 | bool order_concurrent; |
172 | |
173 | /* True if there is bind clause on the construct (i.e. a loop construct). */ |
174 | bool loop_p; |
175 | |
176 | /* Only used for omp target contexts. True if a teams construct is |
177 | strictly nested in it. */ |
178 | bool teams_nested_p; |
179 | |
180 | /* Only used for omp target contexts. True if an OpenMP construct other |
181 | than teams is strictly nested in it. */ |
182 | bool nonteams_nested_p; |
183 | |
184 | /* Candidates for adjusting OpenACC privatization level. */ |
185 | vec<tree> oacc_privatization_candidates; |
186 | }; |
187 | |
188 | static splay_tree all_contexts; |
189 | static int taskreg_nesting_level; |
190 | static int target_nesting_level; |
191 | static bitmap make_addressable_vars; |
192 | static bitmap global_nonaddressable_vars; |
193 | static vec<omp_context *> taskreg_contexts; |
194 | static vec<gomp_task *> task_cpyfns; |
195 | |
196 | static void scan_omp (gimple_seq *, omp_context *); |
197 | static tree scan_omp_1_op (tree *, int *, void *); |
198 | static bool omp_maybe_offloaded_ctx (omp_context *ctx); |
199 | |
200 | #define WALK_SUBSTMTS \ |
201 | case GIMPLE_BIND: \ |
202 | case GIMPLE_TRY: \ |
203 | case GIMPLE_CATCH: \ |
204 | case GIMPLE_EH_FILTER: \ |
205 | case GIMPLE_ASSUME: \ |
206 | case GIMPLE_TRANSACTION: \ |
207 | /* The sub-statements for these should be walked. */ \ |
208 | *handled_ops_p = false; \ |
209 | break; |
210 | |
211 | /* Return whether CTX represents an OpenACC 'parallel' or 'serial' construct. |
212 | (This doesn't include OpenACC 'kernels' decomposed parts.) */ |
213 | |
214 | static bool |
215 | is_oacc_parallel_or_serial (omp_context *ctx) |
216 | { |
217 | enum gimple_code outer_type = gimple_code (g: ctx->stmt); |
218 | return ((outer_type == GIMPLE_OMP_TARGET) |
219 | && ((gimple_omp_target_kind (g: ctx->stmt) |
220 | == GF_OMP_TARGET_KIND_OACC_PARALLEL) |
221 | || (gimple_omp_target_kind (g: ctx->stmt) |
222 | == GF_OMP_TARGET_KIND_OACC_SERIAL))); |
223 | } |
224 | |
225 | /* Return whether CTX represents an OpenACC 'kernels' construct. |
226 | (This doesn't include OpenACC 'kernels' decomposed parts.) */ |
227 | |
228 | static bool |
229 | is_oacc_kernels (omp_context *ctx) |
230 | { |
231 | enum gimple_code outer_type = gimple_code (g: ctx->stmt); |
232 | return ((outer_type == GIMPLE_OMP_TARGET) |
233 | && (gimple_omp_target_kind (g: ctx->stmt) |
234 | == GF_OMP_TARGET_KIND_OACC_KERNELS)); |
235 | } |
236 | |
237 | /* Return whether CTX represents an OpenACC 'kernels' decomposed part. */ |
238 | |
239 | static bool |
240 | is_oacc_kernels_decomposed_part (omp_context *ctx) |
241 | { |
242 | enum gimple_code outer_type = gimple_code (g: ctx->stmt); |
243 | return ((outer_type == GIMPLE_OMP_TARGET) |
244 | && ((gimple_omp_target_kind (g: ctx->stmt) |
245 | == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED) |
246 | || (gimple_omp_target_kind (g: ctx->stmt) |
247 | == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE) |
248 | || (gimple_omp_target_kind (g: ctx->stmt) |
249 | == GF_OMP_TARGET_KIND_OACC_DATA_KERNELS))); |
250 | } |
251 | |
252 | /* Return true if STMT corresponds to an OpenMP target region. */ |
253 | static bool |
254 | is_omp_target (gimple *stmt) |
255 | { |
256 | if (gimple_code (g: stmt) == GIMPLE_OMP_TARGET) |
257 | { |
258 | int kind = gimple_omp_target_kind (g: stmt); |
259 | return (kind == GF_OMP_TARGET_KIND_REGION |
260 | || kind == GF_OMP_TARGET_KIND_DATA |
261 | || kind == GF_OMP_TARGET_KIND_ENTER_DATA |
262 | || kind == GF_OMP_TARGET_KIND_EXIT_DATA); |
263 | } |
264 | return false; |
265 | } |
266 | |
267 | /* If DECL is the artificial dummy VAR_DECL created for non-static |
268 | data member privatization, return the underlying "this" parameter, |
269 | otherwise return NULL. */ |
270 | |
271 | tree |
272 | omp_member_access_dummy_var (tree decl) |
273 | { |
274 | if (!VAR_P (decl) |
275 | || !DECL_ARTIFICIAL (decl) |
276 | || !DECL_IGNORED_P (decl) |
277 | || !DECL_HAS_VALUE_EXPR_P (decl) |
278 | || !lang_hooks.decls.omp_disregard_value_expr (decl, false)) |
279 | return NULL_TREE; |
280 | |
281 | tree v = DECL_VALUE_EXPR (decl); |
282 | if (TREE_CODE (v) != COMPONENT_REF) |
283 | return NULL_TREE; |
284 | |
285 | while (1) |
286 | switch (TREE_CODE (v)) |
287 | { |
288 | case COMPONENT_REF: |
289 | case MEM_REF: |
290 | case INDIRECT_REF: |
291 | CASE_CONVERT: |
292 | case POINTER_PLUS_EXPR: |
293 | v = TREE_OPERAND (v, 0); |
294 | continue; |
295 | case PARM_DECL: |
296 | if (DECL_CONTEXT (v) == current_function_decl |
297 | && DECL_ARTIFICIAL (v) |
298 | && TREE_CODE (TREE_TYPE (v)) == POINTER_TYPE) |
299 | return v; |
300 | return NULL_TREE; |
301 | default: |
302 | return NULL_TREE; |
303 | } |
304 | } |
305 | |
306 | /* Helper for unshare_and_remap, called through walk_tree. */ |
307 | |
308 | static tree |
309 | unshare_and_remap_1 (tree *tp, int *walk_subtrees, void *data) |
310 | { |
311 | tree *pair = (tree *) data; |
312 | if (*tp == pair[0]) |
313 | { |
314 | *tp = unshare_expr (pair[1]); |
315 | *walk_subtrees = 0; |
316 | } |
317 | else if (IS_TYPE_OR_DECL_P (*tp)) |
318 | *walk_subtrees = 0; |
319 | return NULL_TREE; |
320 | } |
321 | |
322 | /* Return unshare_expr (X) with all occurrences of FROM |
323 | replaced with TO. */ |
324 | |
325 | static tree |
326 | unshare_and_remap (tree x, tree from, tree to) |
327 | { |
328 | tree pair[2] = { from, to }; |
329 | x = unshare_expr (x); |
330 | walk_tree (&x, unshare_and_remap_1, pair, NULL); |
331 | return x; |
332 | } |
333 | |
334 | /* Convenience function for calling scan_omp_1_op on tree operands. */ |
335 | |
336 | static inline tree |
337 | scan_omp_op (tree *tp, omp_context *ctx) |
338 | { |
339 | struct walk_stmt_info wi; |
340 | |
341 | memset (s: &wi, c: 0, n: sizeof (wi)); |
342 | wi.info = ctx; |
343 | wi.want_locations = true; |
344 | |
345 | return walk_tree (tp, scan_omp_1_op, &wi, NULL); |
346 | } |
347 | |
348 | static void lower_omp (gimple_seq *, omp_context *); |
349 | static tree lookup_decl_in_outer_ctx (tree, omp_context *); |
350 | static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *); |
351 | |
352 | /* Return true if CTX is for an omp parallel. */ |
353 | |
354 | static inline bool |
355 | is_parallel_ctx (omp_context *ctx) |
356 | { |
357 | return gimple_code (g: ctx->stmt) == GIMPLE_OMP_PARALLEL; |
358 | } |
359 | |
360 | |
361 | /* Return true if CTX is for an omp task. */ |
362 | |
363 | static inline bool |
364 | is_task_ctx (omp_context *ctx) |
365 | { |
366 | return gimple_code (g: ctx->stmt) == GIMPLE_OMP_TASK; |
367 | } |
368 | |
369 | |
370 | /* Return true if CTX is for an omp taskloop. */ |
371 | |
372 | static inline bool |
373 | is_taskloop_ctx (omp_context *ctx) |
374 | { |
375 | return gimple_code (g: ctx->stmt) == GIMPLE_OMP_FOR |
376 | && gimple_omp_for_kind (g: ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP; |
377 | } |
378 | |
379 | |
380 | /* Return true if CTX is for a host omp teams. */ |
381 | |
382 | static inline bool |
383 | is_host_teams_ctx (omp_context *ctx) |
384 | { |
385 | return gimple_code (g: ctx->stmt) == GIMPLE_OMP_TEAMS |
386 | && gimple_omp_teams_host (omp_teams_stmt: as_a <gomp_teams *> (p: ctx->stmt)); |
387 | } |
388 | |
389 | /* Return true if CTX is for an omp parallel or omp task or host omp teams |
390 | (the last one is strictly not a task region in OpenMP speak, but we |
391 | need to treat it similarly). */ |
392 | |
393 | static inline bool |
394 | is_taskreg_ctx (omp_context *ctx) |
395 | { |
396 | return is_parallel_ctx (ctx) || is_task_ctx (ctx) || is_host_teams_ctx (ctx); |
397 | } |
398 | |
399 | /* Return true if EXPR is variable sized. */ |
400 | |
401 | static inline bool |
402 | is_variable_sized (const_tree expr) |
403 | { |
404 | return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr))); |
405 | } |
406 | |
407 | /* Lookup variables. The "maybe" form |
408 | allows for the variable form to not have been entered, otherwise we |
409 | assert that the variable must have been entered. */ |
410 | |
411 | static inline tree |
412 | lookup_decl (tree var, omp_context *ctx) |
413 | { |
414 | tree *n = ctx->cb.decl_map->get (k: var); |
415 | return *n; |
416 | } |
417 | |
418 | static inline tree |
419 | maybe_lookup_decl (const_tree var, omp_context *ctx) |
420 | { |
421 | tree *n = ctx->cb.decl_map->get (k: const_cast<tree> (var)); |
422 | return n ? *n : NULL_TREE; |
423 | } |
424 | |
425 | static inline tree |
426 | lookup_field (tree var, omp_context *ctx) |
427 | { |
428 | splay_tree_node n; |
429 | n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var); |
430 | return (tree) n->value; |
431 | } |
432 | |
433 | static inline tree |
434 | lookup_sfield (splay_tree_key key, omp_context *ctx) |
435 | { |
436 | splay_tree_node n; |
437 | n = splay_tree_lookup (ctx->sfield_map |
438 | ? ctx->sfield_map : ctx->field_map, key); |
439 | return (tree) n->value; |
440 | } |
441 | |
442 | static inline tree |
443 | lookup_sfield (tree var, omp_context *ctx) |
444 | { |
445 | return lookup_sfield (key: (splay_tree_key) var, ctx); |
446 | } |
447 | |
448 | static inline tree |
449 | maybe_lookup_field (splay_tree_key key, omp_context *ctx) |
450 | { |
451 | splay_tree_node n; |
452 | n = splay_tree_lookup (ctx->field_map, key); |
453 | return n ? (tree) n->value : NULL_TREE; |
454 | } |
455 | |
456 | static inline tree |
457 | maybe_lookup_field (tree var, omp_context *ctx) |
458 | { |
459 | return maybe_lookup_field (key: (splay_tree_key) var, ctx); |
460 | } |
461 | |
462 | /* Return true if DECL should be copied by pointer. SHARED_CTX is |
463 | the parallel context if DECL is to be shared. */ |
464 | |
465 | static bool |
466 | use_pointer_for_field (tree decl, omp_context *shared_ctx) |
467 | { |
468 | if (AGGREGATE_TYPE_P (TREE_TYPE (decl)) |
469 | || TYPE_ATOMIC (TREE_TYPE (decl))) |
470 | return true; |
471 | |
472 | /* We can only use copy-in/copy-out semantics for shared variables |
473 | when we know the value is not accessible from an outer scope. */ |
474 | if (shared_ctx) |
475 | { |
476 | gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt)); |
477 | |
478 | /* ??? Trivially accessible from anywhere. But why would we even |
479 | be passing an address in this case? Should we simply assert |
480 | this to be false, or should we have a cleanup pass that removes |
481 | these from the list of mappings? */ |
482 | if (is_global_var (t: maybe_lookup_decl_in_outer_ctx (decl, shared_ctx))) |
483 | return true; |
484 | |
485 | /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell |
486 | without analyzing the expression whether or not its location |
487 | is accessible to anyone else. In the case of nested parallel |
488 | regions it certainly may be. */ |
489 | if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl)) |
490 | return true; |
491 | |
492 | /* Do not use copy-in/copy-out for variables that have their |
493 | address taken. */ |
494 | if (is_global_var (t: decl)) |
495 | { |
496 | /* For file scope vars, track whether we've seen them as |
497 | non-addressable initially and in that case, keep the same |
498 | answer for the duration of the pass, even when they are made |
499 | addressable later on e.g. through reduction expansion. Global |
500 | variables which weren't addressable before the pass will not |
501 | have their privatized copies address taken. See PR91216. */ |
502 | if (!TREE_ADDRESSABLE (decl)) |
503 | { |
504 | if (!global_nonaddressable_vars) |
505 | global_nonaddressable_vars = BITMAP_ALLOC (NULL); |
506 | bitmap_set_bit (global_nonaddressable_vars, DECL_UID (decl)); |
507 | } |
508 | else if (!global_nonaddressable_vars |
509 | || !bitmap_bit_p (global_nonaddressable_vars, |
510 | DECL_UID (decl))) |
511 | return true; |
512 | } |
513 | else if (TREE_ADDRESSABLE (decl)) |
514 | return true; |
515 | |
516 | /* lower_send_shared_vars only uses copy-in, but not copy-out |
517 | for these. */ |
518 | if (TREE_READONLY (decl) |
519 | || ((TREE_CODE (decl) == RESULT_DECL |
520 | || TREE_CODE (decl) == PARM_DECL) |
521 | && DECL_BY_REFERENCE (decl))) |
522 | return false; |
523 | |
524 | /* Disallow copy-in/out in nested parallel if |
525 | decl is shared in outer parallel, otherwise |
526 | each thread could store the shared variable |
527 | in its own copy-in location, making the |
528 | variable no longer really shared. */ |
529 | if (shared_ctx->is_nested) |
530 | { |
531 | omp_context *up; |
532 | |
533 | for (up = shared_ctx->outer; up; up = up->outer) |
534 | if ((is_taskreg_ctx (ctx: up) |
535 | || (gimple_code (g: up->stmt) == GIMPLE_OMP_TARGET |
536 | && is_gimple_omp_offloaded (stmt: up->stmt))) |
537 | && maybe_lookup_decl (var: decl, ctx: up)) |
538 | break; |
539 | |
540 | if (up) |
541 | { |
542 | tree c; |
543 | |
544 | if (gimple_code (g: up->stmt) == GIMPLE_OMP_TARGET) |
545 | { |
546 | for (c = gimple_omp_target_clauses (gs: up->stmt); |
547 | c; c = OMP_CLAUSE_CHAIN (c)) |
548 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
549 | && OMP_CLAUSE_DECL (c) == decl) |
550 | break; |
551 | } |
552 | else |
553 | for (c = gimple_omp_taskreg_clauses (gs: up->stmt); |
554 | c; c = OMP_CLAUSE_CHAIN (c)) |
555 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED |
556 | && OMP_CLAUSE_DECL (c) == decl) |
557 | break; |
558 | |
559 | if (c) |
560 | goto maybe_mark_addressable_and_ret; |
561 | } |
562 | } |
563 | |
564 | /* For tasks avoid using copy-in/out. As tasks can be |
565 | deferred or executed in different thread, when GOMP_task |
566 | returns, the task hasn't necessarily terminated. */ |
567 | if (is_task_ctx (ctx: shared_ctx)) |
568 | { |
569 | tree outer; |
570 | maybe_mark_addressable_and_ret: |
571 | outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx); |
572 | if (is_gimple_reg (outer) && !omp_member_access_dummy_var (decl: outer)) |
573 | { |
574 | /* Taking address of OUTER in lower_send_shared_vars |
575 | might need regimplification of everything that uses the |
576 | variable. */ |
577 | if (!make_addressable_vars) |
578 | make_addressable_vars = BITMAP_ALLOC (NULL); |
579 | bitmap_set_bit (make_addressable_vars, DECL_UID (outer)); |
580 | TREE_ADDRESSABLE (outer) = 1; |
581 | } |
582 | return true; |
583 | } |
584 | } |
585 | |
586 | return false; |
587 | } |
588 | |
589 | /* Construct a new automatic decl similar to VAR. */ |
590 | |
591 | static tree |
592 | omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx) |
593 | { |
594 | tree copy = copy_var_decl (var, name, type); |
595 | |
596 | DECL_CONTEXT (copy) = current_function_decl; |
597 | |
598 | if (ctx) |
599 | { |
600 | DECL_CHAIN (copy) = ctx->block_vars; |
601 | ctx->block_vars = copy; |
602 | } |
603 | else |
604 | record_vars (copy); |
605 | |
606 | /* If VAR is listed in make_addressable_vars, it wasn't |
607 | originally addressable, but was only later made so. |
608 | We don't need to take address of privatizations |
609 | from that var. */ |
610 | if (TREE_ADDRESSABLE (var) |
611 | && ((make_addressable_vars |
612 | && bitmap_bit_p (make_addressable_vars, DECL_UID (var))) |
613 | || (global_nonaddressable_vars |
614 | && bitmap_bit_p (global_nonaddressable_vars, DECL_UID (var))))) |
615 | TREE_ADDRESSABLE (copy) = 0; |
616 | |
617 | return copy; |
618 | } |
619 | |
620 | static tree |
621 | omp_copy_decl_1 (tree var, omp_context *ctx) |
622 | { |
623 | return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx); |
624 | } |
625 | |
626 | /* Build tree nodes to access the field for VAR on the receiver side. */ |
627 | |
628 | static tree |
629 | build_receiver_ref (tree var, bool by_ref, omp_context *ctx) |
630 | { |
631 | tree x, field = lookup_field (var, ctx); |
632 | |
633 | /* If the receiver record type was remapped in the child function, |
634 | remap the field into the new record type. */ |
635 | x = maybe_lookup_field (var: field, ctx); |
636 | if (x != NULL) |
637 | field = x; |
638 | |
639 | x = build_simple_mem_ref (ctx->receiver_decl); |
640 | TREE_THIS_NOTRAP (x) = 1; |
641 | x = omp_build_component_ref (obj: x, field); |
642 | if (by_ref) |
643 | { |
644 | x = build_simple_mem_ref (x); |
645 | TREE_THIS_NOTRAP (x) = 1; |
646 | } |
647 | |
648 | return x; |
649 | } |
650 | |
651 | /* Build tree nodes to access VAR in the scope outer to CTX. In the case |
652 | of a parallel, this is a component reference; for workshare constructs |
653 | this is some variable. */ |
654 | |
655 | static tree |
656 | build_outer_var_ref (tree var, omp_context *ctx, |
657 | enum omp_clause_code code = OMP_CLAUSE_ERROR) |
658 | { |
659 | tree x; |
660 | omp_context *outer = ctx->outer; |
661 | for (; outer; outer = outer->outer) |
662 | { |
663 | if (gimple_code (g: outer->stmt) == GIMPLE_OMP_TASKGROUP) |
664 | continue; |
665 | if (gimple_code (g: outer->stmt) == GIMPLE_OMP_SCOPE |
666 | && !maybe_lookup_decl (var, ctx: outer)) |
667 | continue; |
668 | break; |
669 | } |
670 | |
671 | if (is_global_var (t: maybe_lookup_decl_in_outer_ctx (var, ctx))) |
672 | x = var; |
673 | else if (is_variable_sized (expr: var)) |
674 | { |
675 | x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0); |
676 | x = build_outer_var_ref (var: x, ctx, code); |
677 | x = build_simple_mem_ref (x); |
678 | } |
679 | else if (is_taskreg_ctx (ctx)) |
680 | { |
681 | bool by_ref = use_pointer_for_field (decl: var, NULL); |
682 | x = build_receiver_ref (var, by_ref, ctx); |
683 | } |
684 | else if ((gimple_code (g: ctx->stmt) == GIMPLE_OMP_FOR |
685 | && gimple_omp_for_kind (g: ctx->stmt) == GF_OMP_FOR_KIND_SIMD) |
686 | || ctx->loop_p |
687 | || code == OMP_CLAUSE_ALLOCATE |
688 | || (code == OMP_CLAUSE_PRIVATE |
689 | && (gimple_code (g: ctx->stmt) == GIMPLE_OMP_FOR |
690 | || gimple_code (g: ctx->stmt) == GIMPLE_OMP_SECTIONS |
691 | || gimple_code (g: ctx->stmt) == GIMPLE_OMP_SINGLE))) |
692 | { |
693 | /* #pragma omp simd isn't a worksharing construct, and can reference |
694 | even private vars in its linear etc. clauses. |
695 | Similarly for OMP_CLAUSE_PRIVATE with outer ref, that can refer |
696 | to private vars in all worksharing constructs. */ |
697 | x = NULL_TREE; |
698 | if (outer && is_taskreg_ctx (ctx: outer)) |
699 | x = lookup_decl (var, ctx: outer); |
700 | else if (outer) |
701 | x = maybe_lookup_decl_in_outer_ctx (var, ctx); |
702 | if (x == NULL_TREE) |
703 | x = var; |
704 | } |
705 | else if (code == OMP_CLAUSE_LASTPRIVATE && is_taskloop_ctx (ctx)) |
706 | { |
707 | gcc_assert (outer); |
708 | splay_tree_node n |
709 | = splay_tree_lookup (outer->field_map, |
710 | (splay_tree_key) &DECL_UID (var)); |
711 | if (n == NULL) |
712 | { |
713 | if (is_global_var (t: maybe_lookup_decl_in_outer_ctx (var, outer))) |
714 | x = var; |
715 | else |
716 | x = lookup_decl (var, ctx: outer); |
717 | } |
718 | else |
719 | { |
720 | tree field = (tree) n->value; |
721 | /* If the receiver record type was remapped in the child function, |
722 | remap the field into the new record type. */ |
723 | x = maybe_lookup_field (var: field, ctx: outer); |
724 | if (x != NULL) |
725 | field = x; |
726 | |
727 | x = build_simple_mem_ref (outer->receiver_decl); |
728 | x = omp_build_component_ref (obj: x, field); |
729 | if (use_pointer_for_field (decl: var, shared_ctx: outer)) |
730 | x = build_simple_mem_ref (x); |
731 | } |
732 | } |
733 | else if (outer) |
734 | x = lookup_decl (var, ctx: outer); |
735 | else if (omp_privatize_by_reference (decl: var)) |
736 | /* This can happen with orphaned constructs. If var is reference, it is |
737 | possible it is shared and as such valid. */ |
738 | x = var; |
739 | else if (omp_member_access_dummy_var (decl: var)) |
740 | x = var; |
741 | else |
742 | gcc_unreachable (); |
743 | |
744 | if (x == var) |
745 | { |
746 | tree t = omp_member_access_dummy_var (decl: var); |
747 | if (t) |
748 | { |
749 | x = DECL_VALUE_EXPR (var); |
750 | tree o = maybe_lookup_decl_in_outer_ctx (t, ctx); |
751 | if (o != t) |
752 | x = unshare_and_remap (x, from: t, to: o); |
753 | else |
754 | x = unshare_expr (x); |
755 | } |
756 | } |
757 | |
758 | if (omp_privatize_by_reference (decl: var)) |
759 | x = build_simple_mem_ref (x); |
760 | |
761 | return x; |
762 | } |
763 | |
764 | /* Build tree nodes to access the field for VAR on the sender side. */ |
765 | |
766 | static tree |
767 | build_sender_ref (splay_tree_key key, omp_context *ctx) |
768 | { |
769 | tree field = lookup_sfield (key, ctx); |
770 | return omp_build_component_ref (obj: ctx->sender_decl, field); |
771 | } |
772 | |
773 | static tree |
774 | build_sender_ref (tree var, omp_context *ctx) |
775 | { |
776 | return build_sender_ref (key: (splay_tree_key) var, ctx); |
777 | } |
778 | |
779 | /* Add a new field for VAR inside the structure CTX->SENDER_DECL. If |
780 | BASE_POINTERS_RESTRICT, declare the field with restrict. */ |
781 | |
782 | static void |
783 | install_var_field (tree var, bool by_ref, int mask, omp_context *ctx) |
784 | { |
785 | tree field, type, sfield = NULL_TREE; |
786 | splay_tree_key key = (splay_tree_key) var; |
787 | |
788 | if ((mask & 16) != 0) |
789 | { |
790 | key = (splay_tree_key) &DECL_NAME (var); |
791 | gcc_checking_assert (key != (splay_tree_key) var); |
792 | } |
793 | if ((mask & 8) != 0) |
794 | { |
795 | key = (splay_tree_key) &DECL_UID (var); |
796 | gcc_checking_assert (key != (splay_tree_key) var); |
797 | } |
798 | gcc_assert ((mask & 1) == 0 |
799 | || !splay_tree_lookup (ctx->field_map, key)); |
800 | gcc_assert ((mask & 2) == 0 || !ctx->sfield_map |
801 | || !splay_tree_lookup (ctx->sfield_map, key)); |
802 | gcc_assert ((mask & 3) == 3 |
803 | || !is_gimple_omp_oacc (ctx->stmt)); |
804 | |
805 | type = TREE_TYPE (var); |
806 | if ((mask & 16) != 0) |
807 | type = lang_hooks.decls.omp_array_data (var, true); |
808 | |
809 | /* Prevent redeclaring the var in the split-off function with a restrict |
810 | pointer type. Note that we only clear type itself, restrict qualifiers in |
811 | the pointed-to type will be ignored by points-to analysis. */ |
812 | if (POINTER_TYPE_P (type) |
813 | && TYPE_RESTRICT (type)) |
814 | type = build_qualified_type (type, TYPE_QUALS (type) & ~TYPE_QUAL_RESTRICT); |
815 | |
816 | if (mask & 4) |
817 | { |
818 | gcc_assert (TREE_CODE (type) == ARRAY_TYPE); |
819 | type = build_pointer_type (build_pointer_type (type)); |
820 | } |
821 | else if (by_ref) |
822 | type = build_pointer_type (type); |
823 | else if ((mask & (32 | 3)) == 1 |
824 | && omp_privatize_by_reference (decl: var)) |
825 | type = TREE_TYPE (type); |
826 | |
827 | field = build_decl (DECL_SOURCE_LOCATION (var), |
828 | FIELD_DECL, DECL_NAME (var), type); |
829 | |
830 | /* Remember what variable this field was created for. This does have a |
831 | side effect of making dwarf2out ignore this member, so for helpful |
832 | debugging we clear it later in delete_omp_context. */ |
833 | DECL_ABSTRACT_ORIGIN (field) = var; |
834 | if ((mask & 16) == 0 && type == TREE_TYPE (var)) |
835 | { |
836 | SET_DECL_ALIGN (field, DECL_ALIGN (var)); |
837 | DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var); |
838 | TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var); |
839 | } |
840 | else |
841 | SET_DECL_ALIGN (field, TYPE_ALIGN (type)); |
842 | |
843 | if ((mask & 3) == 3) |
844 | { |
845 | insert_field_into_struct (ctx->record_type, field); |
846 | if (ctx->srecord_type) |
847 | { |
848 | sfield = build_decl (DECL_SOURCE_LOCATION (var), |
849 | FIELD_DECL, DECL_NAME (var), type); |
850 | DECL_ABSTRACT_ORIGIN (sfield) = var; |
851 | SET_DECL_ALIGN (sfield, DECL_ALIGN (field)); |
852 | DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field); |
853 | TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field); |
854 | insert_field_into_struct (ctx->srecord_type, sfield); |
855 | } |
856 | } |
857 | else |
858 | { |
859 | if (ctx->srecord_type == NULL_TREE) |
860 | { |
861 | tree t; |
862 | |
863 | ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE); |
864 | ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
865 | for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t)) |
866 | { |
867 | sfield = build_decl (DECL_SOURCE_LOCATION (t), |
868 | FIELD_DECL, DECL_NAME (t), TREE_TYPE (t)); |
869 | DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t); |
870 | insert_field_into_struct (ctx->srecord_type, sfield); |
871 | splay_tree_insert (ctx->sfield_map, |
872 | (splay_tree_key) DECL_ABSTRACT_ORIGIN (t), |
873 | (splay_tree_value) sfield); |
874 | } |
875 | } |
876 | sfield = field; |
877 | insert_field_into_struct ((mask & 1) ? ctx->record_type |
878 | : ctx->srecord_type, field); |
879 | } |
880 | |
881 | if (mask & 1) |
882 | splay_tree_insert (ctx->field_map, key, (splay_tree_value) field); |
883 | if ((mask & 2) && ctx->sfield_map) |
884 | splay_tree_insert (ctx->sfield_map, key, (splay_tree_value) sfield); |
885 | } |
886 | |
887 | static tree |
888 | install_var_local (tree var, omp_context *ctx) |
889 | { |
890 | tree new_var = omp_copy_decl_1 (var, ctx); |
891 | insert_decl_map (&ctx->cb, var, new_var); |
892 | return new_var; |
893 | } |
894 | |
895 | /* Adjust the replacement for DECL in CTX for the new context. This means |
896 | copying the DECL_VALUE_EXPR, and fixing up the type. */ |
897 | |
898 | static void |
899 | fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug) |
900 | { |
901 | tree new_decl, size; |
902 | |
903 | new_decl = lookup_decl (var: decl, ctx); |
904 | |
905 | TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), id: &ctx->cb); |
906 | |
907 | if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug) |
908 | && DECL_HAS_VALUE_EXPR_P (decl)) |
909 | { |
910 | tree ve = DECL_VALUE_EXPR (decl); |
911 | walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL); |
912 | SET_DECL_VALUE_EXPR (new_decl, ve); |
913 | DECL_HAS_VALUE_EXPR_P (new_decl) = 1; |
914 | } |
915 | |
916 | if (!TREE_CONSTANT (DECL_SIZE (new_decl))) |
917 | { |
918 | size = remap_decl (DECL_SIZE (decl), id: &ctx->cb); |
919 | if (size == error_mark_node) |
920 | size = TYPE_SIZE (TREE_TYPE (new_decl)); |
921 | DECL_SIZE (new_decl) = size; |
922 | |
923 | size = remap_decl (DECL_SIZE_UNIT (decl), id: &ctx->cb); |
924 | if (size == error_mark_node) |
925 | size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl)); |
926 | DECL_SIZE_UNIT (new_decl) = size; |
927 | } |
928 | } |
929 | |
930 | /* The callback for remap_decl. Search all containing contexts for a |
931 | mapping of the variable; this avoids having to duplicate the splay |
932 | tree ahead of time. We know a mapping doesn't already exist in the |
933 | given context. Create new mappings to implement default semantics. */ |
934 | |
935 | static tree |
936 | omp_copy_decl (tree var, copy_body_data *cb) |
937 | { |
938 | omp_context *ctx = (omp_context *) cb; |
939 | tree new_var; |
940 | |
941 | if (TREE_CODE (var) == LABEL_DECL) |
942 | { |
943 | if (FORCED_LABEL (var) || DECL_NONLOCAL (var)) |
944 | return var; |
945 | new_var = create_artificial_label (DECL_SOURCE_LOCATION (var)); |
946 | DECL_CONTEXT (new_var) = current_function_decl; |
947 | insert_decl_map (&ctx->cb, var, new_var); |
948 | return new_var; |
949 | } |
950 | |
951 | while (!is_taskreg_ctx (ctx)) |
952 | { |
953 | ctx = ctx->outer; |
954 | if (ctx == NULL) |
955 | return var; |
956 | new_var = maybe_lookup_decl (var, ctx); |
957 | if (new_var) |
958 | return new_var; |
959 | } |
960 | |
961 | if (is_global_var (t: var) || decl_function_context (var) != ctx->cb.src_fn) |
962 | return var; |
963 | |
964 | return error_mark_node; |
965 | } |
966 | |
967 | /* Create a new context, with OUTER_CTX being the surrounding context. */ |
968 | |
969 | static omp_context * |
970 | new_omp_context (gimple *stmt, omp_context *outer_ctx) |
971 | { |
972 | omp_context *ctx = XCNEW (omp_context); |
973 | |
974 | splay_tree_insert (all_contexts, (splay_tree_key) stmt, |
975 | (splay_tree_value) ctx); |
976 | ctx->stmt = stmt; |
977 | |
978 | if (outer_ctx) |
979 | { |
980 | ctx->outer = outer_ctx; |
981 | ctx->cb = outer_ctx->cb; |
982 | ctx->cb.block = NULL; |
983 | ctx->depth = outer_ctx->depth + 1; |
984 | } |
985 | else |
986 | { |
987 | ctx->cb.src_fn = current_function_decl; |
988 | ctx->cb.dst_fn = current_function_decl; |
989 | ctx->cb.src_node = cgraph_node::get (decl: current_function_decl); |
990 | gcc_checking_assert (ctx->cb.src_node); |
991 | ctx->cb.dst_node = ctx->cb.src_node; |
992 | ctx->cb.src_cfun = cfun; |
993 | ctx->cb.copy_decl = omp_copy_decl; |
994 | ctx->cb.eh_lp_nr = 0; |
995 | ctx->cb.transform_call_graph_edges = CB_CGE_MOVE; |
996 | ctx->cb.adjust_array_error_bounds = true; |
997 | ctx->cb.dont_remap_vla_if_no_change = true; |
998 | ctx->depth = 1; |
999 | } |
1000 | |
1001 | ctx->cb.decl_map = new hash_map<tree, tree>; |
1002 | |
1003 | return ctx; |
1004 | } |
1005 | |
1006 | static gimple_seq maybe_catch_exception (gimple_seq); |
1007 | |
1008 | /* Finalize task copyfn. */ |
1009 | |
1010 | static void |
1011 | finalize_task_copyfn (gomp_task *task_stmt) |
1012 | { |
1013 | struct function *child_cfun; |
1014 | tree child_fn; |
1015 | gimple_seq seq = NULL, new_seq; |
1016 | gbind *bind; |
1017 | |
1018 | child_fn = gimple_omp_task_copy_fn (gs: task_stmt); |
1019 | if (child_fn == NULL_TREE) |
1020 | return; |
1021 | |
1022 | child_cfun = DECL_STRUCT_FUNCTION (child_fn); |
1023 | DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties; |
1024 | |
1025 | push_cfun (new_cfun: child_cfun); |
1026 | bind = gimplify_body (child_fn, false); |
1027 | gimple_seq_add_stmt (&seq, bind); |
1028 | new_seq = maybe_catch_exception (seq); |
1029 | if (new_seq != seq) |
1030 | { |
1031 | bind = gimple_build_bind (NULL, new_seq, NULL); |
1032 | seq = NULL; |
1033 | gimple_seq_add_stmt (&seq, bind); |
1034 | } |
1035 | gimple_set_body (child_fn, seq); |
1036 | pop_cfun (); |
1037 | |
1038 | /* Inform the callgraph about the new function. */ |
1039 | cgraph_node *node = cgraph_node::get_create (child_fn); |
1040 | node->parallelized_function = 1; |
1041 | cgraph_node::add_new_function (fndecl: child_fn, lowered: false); |
1042 | } |
1043 | |
1044 | /* Destroy a omp_context data structures. Called through the splay tree |
1045 | value delete callback. */ |
1046 | |
1047 | static void |
1048 | delete_omp_context (splay_tree_value value) |
1049 | { |
1050 | omp_context *ctx = (omp_context *) value; |
1051 | |
1052 | delete ctx->cb.decl_map; |
1053 | |
1054 | if (ctx->field_map) |
1055 | splay_tree_delete (ctx->field_map); |
1056 | if (ctx->sfield_map) |
1057 | splay_tree_delete (ctx->sfield_map); |
1058 | |
1059 | /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before |
1060 | it produces corrupt debug information. */ |
1061 | if (ctx->record_type) |
1062 | { |
1063 | tree t; |
1064 | for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t)) |
1065 | DECL_ABSTRACT_ORIGIN (t) = NULL; |
1066 | } |
1067 | if (ctx->srecord_type) |
1068 | { |
1069 | tree t; |
1070 | for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t)) |
1071 | DECL_ABSTRACT_ORIGIN (t) = NULL; |
1072 | } |
1073 | |
1074 | if (ctx->task_reduction_map) |
1075 | { |
1076 | ctx->task_reductions.release (); |
1077 | delete ctx->task_reduction_map; |
1078 | } |
1079 | |
1080 | delete ctx->lastprivate_conditional_map; |
1081 | delete ctx->allocate_map; |
1082 | |
1083 | XDELETE (ctx); |
1084 | } |
1085 | |
1086 | /* Fix up RECEIVER_DECL with a type that has been remapped to the child |
1087 | context. */ |
1088 | |
1089 | static void |
1090 | fixup_child_record_type (omp_context *ctx) |
1091 | { |
1092 | tree f, type = ctx->record_type; |
1093 | |
1094 | if (!ctx->receiver_decl) |
1095 | return; |
1096 | /* ??? It isn't sufficient to just call remap_type here, because |
1097 | variably_modified_type_p doesn't work the way we expect for |
1098 | record types. Testing each field for whether it needs remapping |
1099 | and creating a new record by hand works, however. */ |
1100 | for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f)) |
1101 | if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn)) |
1102 | break; |
1103 | if (f) |
1104 | { |
1105 | tree name, new_fields = NULL; |
1106 | |
1107 | type = lang_hooks.types.make_type (RECORD_TYPE); |
1108 | name = DECL_NAME (TYPE_NAME (ctx->record_type)); |
1109 | name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl), |
1110 | TYPE_DECL, name, type); |
1111 | TYPE_NAME (type) = name; |
1112 | |
1113 | for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f)) |
1114 | { |
1115 | tree new_f = copy_node (f); |
1116 | DECL_CONTEXT (new_f) = type; |
1117 | TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), id: &ctx->cb); |
1118 | DECL_CHAIN (new_f) = new_fields; |
1119 | walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL); |
1120 | walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, |
1121 | &ctx->cb, NULL); |
1122 | walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r, |
1123 | &ctx->cb, NULL); |
1124 | new_fields = new_f; |
1125 | |
1126 | /* Arrange to be able to look up the receiver field |
1127 | given the sender field. */ |
1128 | splay_tree_insert (ctx->field_map, (splay_tree_key) f, |
1129 | (splay_tree_value) new_f); |
1130 | } |
1131 | TYPE_FIELDS (type) = nreverse (new_fields); |
1132 | layout_type (type); |
1133 | } |
1134 | |
1135 | /* In a target region we never modify any of the pointers in *.omp_data_i, |
1136 | so attempt to help the optimizers. */ |
1137 | if (is_gimple_omp_offloaded (stmt: ctx->stmt)) |
1138 | type = build_qualified_type (type, TYPE_QUAL_CONST); |
1139 | |
1140 | TREE_TYPE (ctx->receiver_decl) |
1141 | = build_qualified_type (build_reference_type (type), TYPE_QUAL_RESTRICT); |
1142 | } |
1143 | |
1144 | /* Instantiate decls as necessary in CTX to satisfy the data sharing |
1145 | specified by CLAUSES. */ |
1146 | |
1147 | static void |
1148 | scan_sharing_clauses (tree clauses, omp_context *ctx) |
1149 | { |
1150 | tree c, decl; |
1151 | bool scan_array_reductions = false; |
1152 | |
1153 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
1154 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_ALLOCATE |
1155 | && (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE |
1156 | /* omp_default_mem_alloc is 1 */ |
1157 | || !integer_onep (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)) |
1158 | || OMP_CLAUSE_ALLOCATE_ALIGN (c) != NULL_TREE)) |
1159 | { |
1160 | /* The allocate clauses that appear on a target construct or on |
1161 | constructs in a target region must specify an allocator expression |
1162 | unless a requires directive with the dynamic_allocators clause |
1163 | is present in the same compilation unit. */ |
1164 | if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE |
1165 | && ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0) |
1166 | && omp_maybe_offloaded_ctx (ctx)) |
1167 | error_at (OMP_CLAUSE_LOCATION (c), "%<allocate%> clause must" |
1168 | " specify an allocator here" ); |
1169 | if (ctx->allocate_map == NULL) |
1170 | ctx->allocate_map = new hash_map<tree, tree>; |
1171 | tree val = integer_zero_node; |
1172 | if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)) |
1173 | val = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c); |
1174 | if (OMP_CLAUSE_ALLOCATE_ALIGN (c)) |
1175 | val = build_tree_list (val, OMP_CLAUSE_ALLOCATE_ALIGN (c)); |
1176 | ctx->allocate_map->put (OMP_CLAUSE_DECL (c), v: val); |
1177 | } |
1178 | |
1179 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
1180 | { |
1181 | bool by_ref; |
1182 | |
1183 | switch (OMP_CLAUSE_CODE (c)) |
1184 | { |
1185 | case OMP_CLAUSE_PRIVATE: |
1186 | decl = OMP_CLAUSE_DECL (c); |
1187 | if (OMP_CLAUSE_PRIVATE_OUTER_REF (c)) |
1188 | goto do_private; |
1189 | else if (!is_variable_sized (expr: decl)) |
1190 | install_var_local (var: decl, ctx); |
1191 | break; |
1192 | |
1193 | case OMP_CLAUSE_SHARED: |
1194 | decl = OMP_CLAUSE_DECL (c); |
1195 | if (ctx->allocate_map && ctx->allocate_map->get (k: decl)) |
1196 | ctx->allocate_map->remove (k: decl); |
1197 | /* Ignore shared directives in teams construct inside of |
1198 | target construct. */ |
1199 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_TEAMS |
1200 | && !is_host_teams_ctx (ctx)) |
1201 | { |
1202 | /* Global variables don't need to be copied, |
1203 | the receiver side will use them directly. */ |
1204 | tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx); |
1205 | if (is_global_var (t: odecl)) |
1206 | break; |
1207 | insert_decl_map (&ctx->cb, decl, odecl); |
1208 | break; |
1209 | } |
1210 | gcc_assert (is_taskreg_ctx (ctx)); |
1211 | gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl)) |
1212 | || !is_variable_sized (decl)); |
1213 | /* Global variables don't need to be copied, |
1214 | the receiver side will use them directly. */ |
1215 | if (is_global_var (t: maybe_lookup_decl_in_outer_ctx (decl, ctx))) |
1216 | break; |
1217 | if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)) |
1218 | { |
1219 | use_pointer_for_field (decl, shared_ctx: ctx); |
1220 | break; |
1221 | } |
1222 | by_ref = use_pointer_for_field (decl, NULL); |
1223 | if ((! TREE_READONLY (decl) && !OMP_CLAUSE_SHARED_READONLY (c)) |
1224 | || TREE_ADDRESSABLE (decl) |
1225 | || by_ref |
1226 | || omp_privatize_by_reference (decl)) |
1227 | { |
1228 | by_ref = use_pointer_for_field (decl, shared_ctx: ctx); |
1229 | install_var_field (var: decl, by_ref, mask: 3, ctx); |
1230 | install_var_local (var: decl, ctx); |
1231 | break; |
1232 | } |
1233 | /* We don't need to copy const scalar vars back. */ |
1234 | OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE); |
1235 | goto do_private; |
1236 | |
1237 | case OMP_CLAUSE_REDUCTION: |
1238 | /* Collect 'reduction' clauses on OpenACC compute construct. */ |
1239 | if (is_gimple_omp_oacc (stmt: ctx->stmt) |
1240 | && is_gimple_omp_offloaded (stmt: ctx->stmt)) |
1241 | { |
1242 | /* No 'reduction' clauses on OpenACC 'kernels'. */ |
1243 | gcc_checking_assert (!is_oacc_kernels (ctx)); |
1244 | /* Likewise, on OpenACC 'kernels' decomposed parts. */ |
1245 | gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx)); |
1246 | |
1247 | ctx->local_reduction_clauses |
1248 | = tree_cons (NULL, c, ctx->local_reduction_clauses); |
1249 | } |
1250 | /* FALLTHRU */ |
1251 | |
1252 | case OMP_CLAUSE_IN_REDUCTION: |
1253 | decl = OMP_CLAUSE_DECL (c); |
1254 | if (ctx->allocate_map |
1255 | && ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
1256 | && (OMP_CLAUSE_REDUCTION_INSCAN (c) |
1257 | || OMP_CLAUSE_REDUCTION_TASK (c))) |
1258 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION |
1259 | || is_task_ctx (ctx))) |
1260 | { |
1261 | /* For now. */ |
1262 | if (ctx->allocate_map->get (k: decl)) |
1263 | ctx->allocate_map->remove (k: decl); |
1264 | } |
1265 | if (TREE_CODE (decl) == MEM_REF) |
1266 | { |
1267 | tree t = TREE_OPERAND (decl, 0); |
1268 | if (TREE_CODE (t) == POINTER_PLUS_EXPR) |
1269 | t = TREE_OPERAND (t, 0); |
1270 | if (INDIRECT_REF_P (t) |
1271 | || TREE_CODE (t) == ADDR_EXPR) |
1272 | t = TREE_OPERAND (t, 0); |
1273 | if (is_omp_target (stmt: ctx->stmt)) |
1274 | { |
1275 | if (is_variable_sized (expr: t)) |
1276 | { |
1277 | gcc_assert (DECL_HAS_VALUE_EXPR_P (t)); |
1278 | t = DECL_VALUE_EXPR (t); |
1279 | gcc_assert (INDIRECT_REF_P (t)); |
1280 | t = TREE_OPERAND (t, 0); |
1281 | gcc_assert (DECL_P (t)); |
1282 | } |
1283 | tree at = t; |
1284 | if (ctx->outer) |
1285 | scan_omp_op (tp: &at, ctx: ctx->outer); |
1286 | tree nt = omp_copy_decl_1 (var: at, ctx: ctx->outer); |
1287 | splay_tree_insert (ctx->field_map, |
1288 | (splay_tree_key) &DECL_CONTEXT (t), |
1289 | (splay_tree_value) nt); |
1290 | if (at != t) |
1291 | splay_tree_insert (ctx->field_map, |
1292 | (splay_tree_key) &DECL_CONTEXT (at), |
1293 | (splay_tree_value) nt); |
1294 | break; |
1295 | } |
1296 | install_var_local (var: t, ctx); |
1297 | if (is_taskreg_ctx (ctx) |
1298 | && (!is_global_var (t: maybe_lookup_decl_in_outer_ctx (t, ctx)) |
1299 | || (is_task_ctx (ctx) |
1300 | && (TREE_CODE (TREE_TYPE (t)) == POINTER_TYPE |
1301 | || (TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE |
1302 | && (TREE_CODE (TREE_TYPE (TREE_TYPE (t))) |
1303 | == POINTER_TYPE))))) |
1304 | && !is_variable_sized (expr: t) |
1305 | && (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION |
1306 | || (!OMP_CLAUSE_REDUCTION_TASK (c) |
1307 | && !is_task_ctx (ctx)))) |
1308 | { |
1309 | by_ref = use_pointer_for_field (decl: t, NULL); |
1310 | if (is_task_ctx (ctx) |
1311 | && TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE |
1312 | && TREE_CODE (TREE_TYPE (TREE_TYPE (t))) == POINTER_TYPE) |
1313 | { |
1314 | install_var_field (var: t, by_ref: false, mask: 1, ctx); |
1315 | install_var_field (var: t, by_ref, mask: 2, ctx); |
1316 | } |
1317 | else |
1318 | install_var_field (var: t, by_ref, mask: 3, ctx); |
1319 | } |
1320 | break; |
1321 | } |
1322 | if (is_omp_target (stmt: ctx->stmt)) |
1323 | { |
1324 | tree at = decl; |
1325 | if (ctx->outer) |
1326 | scan_omp_op (tp: &at, ctx: ctx->outer); |
1327 | tree nt = omp_copy_decl_1 (var: at, ctx: ctx->outer); |
1328 | splay_tree_insert (ctx->field_map, |
1329 | (splay_tree_key) &DECL_CONTEXT (decl), |
1330 | (splay_tree_value) nt); |
1331 | if (at != decl) |
1332 | splay_tree_insert (ctx->field_map, |
1333 | (splay_tree_key) &DECL_CONTEXT (at), |
1334 | (splay_tree_value) nt); |
1335 | break; |
1336 | } |
1337 | if (is_task_ctx (ctx) |
1338 | || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
1339 | && OMP_CLAUSE_REDUCTION_TASK (c) |
1340 | && is_parallel_ctx (ctx))) |
1341 | { |
1342 | /* Global variables don't need to be copied, |
1343 | the receiver side will use them directly. */ |
1344 | if (!is_global_var (t: maybe_lookup_decl_in_outer_ctx (decl, ctx))) |
1345 | { |
1346 | by_ref = use_pointer_for_field (decl, shared_ctx: ctx); |
1347 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION) |
1348 | install_var_field (var: decl, by_ref, mask: 3, ctx); |
1349 | } |
1350 | install_var_local (var: decl, ctx); |
1351 | break; |
1352 | } |
1353 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
1354 | && OMP_CLAUSE_REDUCTION_TASK (c)) |
1355 | { |
1356 | install_var_local (var: decl, ctx); |
1357 | break; |
1358 | } |
1359 | goto do_private; |
1360 | |
1361 | case OMP_CLAUSE_LASTPRIVATE: |
1362 | /* Let the corresponding firstprivate clause create |
1363 | the variable. */ |
1364 | if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) |
1365 | break; |
1366 | /* FALLTHRU */ |
1367 | |
1368 | case OMP_CLAUSE_FIRSTPRIVATE: |
1369 | case OMP_CLAUSE_LINEAR: |
1370 | decl = OMP_CLAUSE_DECL (c); |
1371 | do_private: |
1372 | if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE |
1373 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR |
1374 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR) |
1375 | && is_gimple_omp_offloaded (stmt: ctx->stmt)) |
1376 | { |
1377 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE |
1378 | || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR |
1379 | && lang_hooks.decls.omp_array_data (decl, true))) |
1380 | { |
1381 | by_ref = !omp_privatize_by_reference (decl); |
1382 | install_var_field (var: decl, by_ref, mask: 3, ctx); |
1383 | } |
1384 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR) |
1385 | { |
1386 | if (INDIRECT_REF_P (decl)) |
1387 | decl = TREE_OPERAND (decl, 0); |
1388 | install_var_field (var: decl, by_ref: true, mask: 3, ctx); |
1389 | } |
1390 | else if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE) |
1391 | install_var_field (var: decl, by_ref: true, mask: 3, ctx); |
1392 | else |
1393 | install_var_field (var: decl, by_ref: false, mask: 3, ctx); |
1394 | } |
1395 | if (is_variable_sized (expr: decl)) |
1396 | { |
1397 | if (is_task_ctx (ctx)) |
1398 | { |
1399 | if (ctx->allocate_map |
1400 | && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE) |
1401 | { |
1402 | /* For now. */ |
1403 | if (ctx->allocate_map->get (k: decl)) |
1404 | ctx->allocate_map->remove (k: decl); |
1405 | } |
1406 | install_var_field (var: decl, by_ref: false, mask: 1, ctx); |
1407 | } |
1408 | break; |
1409 | } |
1410 | else if (is_taskreg_ctx (ctx)) |
1411 | { |
1412 | bool global |
1413 | = is_global_var (t: maybe_lookup_decl_in_outer_ctx (decl, ctx)); |
1414 | by_ref = use_pointer_for_field (decl, NULL); |
1415 | |
1416 | if (is_task_ctx (ctx) |
1417 | && (global || by_ref || omp_privatize_by_reference (decl))) |
1418 | { |
1419 | if (ctx->allocate_map |
1420 | && ctx->allocate_map->get (k: decl)) |
1421 | install_var_field (var: decl, by_ref, mask: 32 | 1, ctx); |
1422 | else |
1423 | install_var_field (var: decl, by_ref: false, mask: 1, ctx); |
1424 | if (!global) |
1425 | install_var_field (var: decl, by_ref, mask: 2, ctx); |
1426 | } |
1427 | else if (!global) |
1428 | install_var_field (var: decl, by_ref, mask: 3, ctx); |
1429 | } |
1430 | install_var_local (var: decl, ctx); |
1431 | /* For descr arrays on target: firstprivatize data + attach ptr. */ |
1432 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE |
1433 | && is_gimple_omp_offloaded (stmt: ctx->stmt) |
1434 | && !is_gimple_omp_oacc (stmt: ctx->stmt) |
1435 | && lang_hooks.decls.omp_array_data (decl, true)) |
1436 | { |
1437 | install_var_field (var: decl, by_ref: false, mask: 16 | 3, ctx); |
1438 | install_var_field (var: decl, by_ref: true, mask: 8 | 3, ctx); |
1439 | } |
1440 | break; |
1441 | |
1442 | case OMP_CLAUSE_USE_DEVICE_PTR: |
1443 | case OMP_CLAUSE_USE_DEVICE_ADDR: |
1444 | decl = OMP_CLAUSE_DECL (c); |
1445 | |
1446 | /* Fortran array descriptors. */ |
1447 | if (lang_hooks.decls.omp_array_data (decl, true)) |
1448 | install_var_field (var: decl, by_ref: false, mask: 19, ctx); |
1449 | else if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR |
1450 | && !omp_privatize_by_reference (decl) |
1451 | && !omp_is_allocatable_or_ptr (decl)) |
1452 | || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE) |
1453 | install_var_field (var: decl, by_ref: true, mask: 11, ctx); |
1454 | else |
1455 | install_var_field (var: decl, by_ref: false, mask: 11, ctx); |
1456 | if (DECL_SIZE (decl) |
1457 | && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST) |
1458 | { |
1459 | tree decl2 = DECL_VALUE_EXPR (decl); |
1460 | gcc_assert (INDIRECT_REF_P (decl2)); |
1461 | decl2 = TREE_OPERAND (decl2, 0); |
1462 | gcc_assert (DECL_P (decl2)); |
1463 | install_var_local (var: decl2, ctx); |
1464 | } |
1465 | install_var_local (var: decl, ctx); |
1466 | break; |
1467 | |
1468 | case OMP_CLAUSE_HAS_DEVICE_ADDR: |
1469 | decl = OMP_CLAUSE_DECL (c); |
1470 | while (INDIRECT_REF_P (decl) |
1471 | || TREE_CODE (decl) == ARRAY_REF) |
1472 | decl = TREE_OPERAND (decl, 0); |
1473 | goto do_private; |
1474 | |
1475 | case OMP_CLAUSE_IS_DEVICE_PTR: |
1476 | decl = OMP_CLAUSE_DECL (c); |
1477 | goto do_private; |
1478 | |
1479 | case OMP_CLAUSE__LOOPTEMP_: |
1480 | case OMP_CLAUSE__REDUCTEMP_: |
1481 | gcc_assert (is_taskreg_ctx (ctx)); |
1482 | decl = OMP_CLAUSE_DECL (c); |
1483 | install_var_field (var: decl, by_ref: false, mask: 3, ctx); |
1484 | install_var_local (var: decl, ctx); |
1485 | break; |
1486 | |
1487 | case OMP_CLAUSE_COPYPRIVATE: |
1488 | case OMP_CLAUSE_COPYIN: |
1489 | decl = OMP_CLAUSE_DECL (c); |
1490 | by_ref = use_pointer_for_field (decl, NULL); |
1491 | install_var_field (var: decl, by_ref, mask: 3, ctx); |
1492 | break; |
1493 | |
1494 | case OMP_CLAUSE_FINAL: |
1495 | case OMP_CLAUSE_IF: |
1496 | case OMP_CLAUSE_SELF: |
1497 | case OMP_CLAUSE_NUM_THREADS: |
1498 | case OMP_CLAUSE_NUM_TEAMS: |
1499 | case OMP_CLAUSE_THREAD_LIMIT: |
1500 | case OMP_CLAUSE_DEVICE: |
1501 | case OMP_CLAUSE_SCHEDULE: |
1502 | case OMP_CLAUSE_DIST_SCHEDULE: |
1503 | case OMP_CLAUSE_DEPEND: |
1504 | case OMP_CLAUSE_PRIORITY: |
1505 | case OMP_CLAUSE_GRAINSIZE: |
1506 | case OMP_CLAUSE_NUM_TASKS: |
1507 | case OMP_CLAUSE_NUM_GANGS: |
1508 | case OMP_CLAUSE_NUM_WORKERS: |
1509 | case OMP_CLAUSE_VECTOR_LENGTH: |
1510 | case OMP_CLAUSE_DETACH: |
1511 | case OMP_CLAUSE_FILTER: |
1512 | if (ctx->outer) |
1513 | scan_omp_op (tp: &OMP_CLAUSE_OPERAND (c, 0), ctx: ctx->outer); |
1514 | break; |
1515 | |
1516 | case OMP_CLAUSE_TO: |
1517 | case OMP_CLAUSE_FROM: |
1518 | case OMP_CLAUSE_MAP: |
1519 | if (ctx->outer) |
1520 | scan_omp_op (tp: &OMP_CLAUSE_SIZE (c), ctx: ctx->outer); |
1521 | decl = OMP_CLAUSE_DECL (c); |
1522 | /* If requested, make 'decl' addressable. */ |
1523 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
1524 | && OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c)) |
1525 | { |
1526 | gcc_checking_assert (DECL_P (decl)); |
1527 | |
1528 | bool decl_addressable = TREE_ADDRESSABLE (decl); |
1529 | if (!decl_addressable) |
1530 | { |
1531 | if (!make_addressable_vars) |
1532 | make_addressable_vars = BITMAP_ALLOC (NULL); |
1533 | bitmap_set_bit (make_addressable_vars, DECL_UID (decl)); |
1534 | TREE_ADDRESSABLE (decl) = 1; |
1535 | } |
1536 | |
1537 | if (dump_enabled_p ()) |
1538 | { |
1539 | location_t loc = OMP_CLAUSE_LOCATION (c); |
1540 | const dump_user_location_t d_u_loc |
1541 | = dump_user_location_t::from_location_t (loc); |
1542 | /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */ |
1543 | #if __GNUC__ >= 10 |
1544 | # pragma GCC diagnostic push |
1545 | # pragma GCC diagnostic ignored "-Wformat" |
1546 | #endif |
1547 | if (!decl_addressable) |
1548 | dump_printf_loc (MSG_NOTE, d_u_loc, |
1549 | "variable %<%T%>" |
1550 | " made addressable\n" , |
1551 | decl); |
1552 | else |
1553 | dump_printf_loc (MSG_NOTE, d_u_loc, |
1554 | "variable %<%T%>" |
1555 | " already made addressable\n" , |
1556 | decl); |
1557 | #if __GNUC__ >= 10 |
1558 | # pragma GCC diagnostic pop |
1559 | #endif |
1560 | } |
1561 | |
1562 | /* Done. */ |
1563 | OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c) = 0; |
1564 | } |
1565 | /* Global variables with "omp declare target" attribute |
1566 | don't need to be copied, the receiver side will use them |
1567 | directly. However, global variables with "omp declare target link" |
1568 | attribute need to be copied. Or when ALWAYS modifier is used. */ |
1569 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
1570 | && DECL_P (decl) |
1571 | && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER |
1572 | && (OMP_CLAUSE_MAP_KIND (c) |
1573 | != GOMP_MAP_FIRSTPRIVATE_REFERENCE) |
1574 | && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH |
1575 | && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_DETACH) |
1576 | || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE) |
1577 | && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TO |
1578 | && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_FROM |
1579 | && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TOFROM |
1580 | && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TO |
1581 | && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_FROM |
1582 | && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TOFROM |
1583 | && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_TO_PSET |
1584 | && is_global_var (t: maybe_lookup_decl_in_outer_ctx (decl, ctx)) |
1585 | && varpool_node::get_create (decl)->offloadable |
1586 | && !lookup_attribute (attr_name: "omp declare target link" , |
1587 | DECL_ATTRIBUTES (decl))) |
1588 | break; |
1589 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
1590 | && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER) |
1591 | { |
1592 | /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are |
1593 | not offloaded; there is nothing to map for those. */ |
1594 | if (!is_gimple_omp_offloaded (stmt: ctx->stmt) |
1595 | && !POINTER_TYPE_P (TREE_TYPE (decl)) |
1596 | && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)) |
1597 | break; |
1598 | } |
1599 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
1600 | && DECL_P (decl) |
1601 | && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH |
1602 | || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH) |
1603 | && is_omp_target (stmt: ctx->stmt)) |
1604 | { |
1605 | /* If this is an offloaded region, an attach operation should |
1606 | only exist when the pointer variable is mapped in a prior |
1607 | clause. An exception is if we have a reference (to pointer): |
1608 | in that case we should have mapped "*decl" in a previous |
1609 | mapping instead of "decl". Skip the assertion in that case. |
1610 | If we had an error, we may not have attempted to sort clauses |
1611 | properly, so avoid the test. */ |
1612 | if (TREE_CODE (TREE_TYPE (decl)) != REFERENCE_TYPE |
1613 | && is_gimple_omp_offloaded (stmt: ctx->stmt) |
1614 | && !seen_error ()) |
1615 | gcc_assert |
1616 | (maybe_lookup_decl (decl, ctx) |
1617 | || (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)) |
1618 | && lookup_attribute ("omp declare target" , |
1619 | DECL_ATTRIBUTES (decl)))); |
1620 | |
1621 | /* By itself, attach/detach is generated as part of pointer |
1622 | variable mapping and should not create new variables in the |
1623 | offloaded region, however sender refs for it must be created |
1624 | for its address to be passed to the runtime. */ |
1625 | tree field |
1626 | = build_decl (OMP_CLAUSE_LOCATION (c), |
1627 | FIELD_DECL, NULL_TREE, ptr_type_node); |
1628 | SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node)); |
1629 | insert_field_into_struct (ctx->record_type, field); |
1630 | /* To not clash with a map of the pointer variable itself, |
1631 | attach/detach maps have their field looked up by the *clause* |
1632 | tree expression, not the decl. */ |
1633 | gcc_assert (!splay_tree_lookup (ctx->field_map, |
1634 | (splay_tree_key) c)); |
1635 | splay_tree_insert (ctx->field_map, (splay_tree_key) c, |
1636 | (splay_tree_value) field); |
1637 | break; |
1638 | } |
1639 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
1640 | && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER |
1641 | || (OMP_CLAUSE_MAP_KIND (c) |
1642 | == GOMP_MAP_FIRSTPRIVATE_REFERENCE))) |
1643 | { |
1644 | if (TREE_CODE (decl) == COMPONENT_REF |
1645 | || (INDIRECT_REF_P (decl) |
1646 | && TREE_CODE (TREE_OPERAND (decl, 0)) == COMPONENT_REF |
1647 | && (((TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0))) |
1648 | == REFERENCE_TYPE) |
1649 | || (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0))) |
1650 | == POINTER_TYPE))))) |
1651 | break; |
1652 | if (DECL_SIZE (decl) |
1653 | && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST) |
1654 | { |
1655 | tree decl2 = DECL_VALUE_EXPR (decl); |
1656 | gcc_assert (INDIRECT_REF_P (decl2)); |
1657 | decl2 = TREE_OPERAND (decl2, 0); |
1658 | gcc_assert (DECL_P (decl2)); |
1659 | install_var_local (var: decl2, ctx); |
1660 | } |
1661 | install_var_local (var: decl, ctx); |
1662 | break; |
1663 | } |
1664 | if (DECL_P (decl)) |
1665 | { |
1666 | if (DECL_SIZE (decl) |
1667 | && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST) |
1668 | { |
1669 | tree decl2 = DECL_VALUE_EXPR (decl); |
1670 | gcc_assert (INDIRECT_REF_P (decl2)); |
1671 | decl2 = TREE_OPERAND (decl2, 0); |
1672 | gcc_assert (DECL_P (decl2)); |
1673 | install_var_field (var: decl2, by_ref: true, mask: 3, ctx); |
1674 | install_var_local (var: decl2, ctx); |
1675 | install_var_local (var: decl, ctx); |
1676 | } |
1677 | else |
1678 | { |
1679 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
1680 | && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER |
1681 | && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) |
1682 | && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE) |
1683 | install_var_field (var: decl, by_ref: true, mask: 7, ctx); |
1684 | else |
1685 | install_var_field (var: decl, by_ref: true, mask: 3, ctx); |
1686 | if (is_gimple_omp_offloaded (stmt: ctx->stmt) |
1687 | && !(is_gimple_omp_oacc (stmt: ctx->stmt) |
1688 | && OMP_CLAUSE_MAP_IN_REDUCTION (c))) |
1689 | install_var_local (var: decl, ctx); |
1690 | } |
1691 | } |
1692 | else |
1693 | { |
1694 | tree base = get_base_address (t: decl); |
1695 | tree nc = OMP_CLAUSE_CHAIN (c); |
1696 | if (DECL_P (base) |
1697 | && nc != NULL_TREE |
1698 | && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP |
1699 | && OMP_CLAUSE_DECL (nc) == base |
1700 | && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER |
1701 | && integer_zerop (OMP_CLAUSE_SIZE (nc))) |
1702 | { |
1703 | OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1; |
1704 | OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1; |
1705 | } |
1706 | else |
1707 | { |
1708 | if (ctx->outer) |
1709 | { |
1710 | scan_omp_op (tp: &OMP_CLAUSE_DECL (c), ctx: ctx->outer); |
1711 | decl = OMP_CLAUSE_DECL (c); |
1712 | } |
1713 | gcc_assert (!splay_tree_lookup (ctx->field_map, |
1714 | (splay_tree_key) decl)); |
1715 | tree field |
1716 | = build_decl (OMP_CLAUSE_LOCATION (c), |
1717 | FIELD_DECL, NULL_TREE, ptr_type_node); |
1718 | SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node)); |
1719 | insert_field_into_struct (ctx->record_type, field); |
1720 | splay_tree_insert (ctx->field_map, (splay_tree_key) decl, |
1721 | (splay_tree_value) field); |
1722 | } |
1723 | } |
1724 | break; |
1725 | |
1726 | case OMP_CLAUSE_ORDER: |
1727 | ctx->order_concurrent = true; |
1728 | break; |
1729 | |
1730 | case OMP_CLAUSE_BIND: |
1731 | ctx->loop_p = true; |
1732 | break; |
1733 | |
1734 | case OMP_CLAUSE_NOWAIT: |
1735 | case OMP_CLAUSE_ORDERED: |
1736 | case OMP_CLAUSE_COLLAPSE: |
1737 | case OMP_CLAUSE_UNTIED: |
1738 | case OMP_CLAUSE_MERGEABLE: |
1739 | case OMP_CLAUSE_PROC_BIND: |
1740 | case OMP_CLAUSE_SAFELEN: |
1741 | case OMP_CLAUSE_SIMDLEN: |
1742 | case OMP_CLAUSE_THREADS: |
1743 | case OMP_CLAUSE_SIMD: |
1744 | case OMP_CLAUSE_NOGROUP: |
1745 | case OMP_CLAUSE_DEFAULTMAP: |
1746 | case OMP_CLAUSE_ASYNC: |
1747 | case OMP_CLAUSE_WAIT: |
1748 | case OMP_CLAUSE_GANG: |
1749 | case OMP_CLAUSE_WORKER: |
1750 | case OMP_CLAUSE_VECTOR: |
1751 | case OMP_CLAUSE_INDEPENDENT: |
1752 | case OMP_CLAUSE_AUTO: |
1753 | case OMP_CLAUSE_SEQ: |
1754 | case OMP_CLAUSE_TILE: |
1755 | case OMP_CLAUSE__SIMT_: |
1756 | case OMP_CLAUSE_DEFAULT: |
1757 | case OMP_CLAUSE_NONTEMPORAL: |
1758 | case OMP_CLAUSE_IF_PRESENT: |
1759 | case OMP_CLAUSE_FINALIZE: |
1760 | case OMP_CLAUSE_TASK_REDUCTION: |
1761 | case OMP_CLAUSE_ALLOCATE: |
1762 | break; |
1763 | |
1764 | case OMP_CLAUSE_ALIGNED: |
1765 | decl = OMP_CLAUSE_DECL (c); |
1766 | if (is_global_var (t: decl) |
1767 | && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE) |
1768 | install_var_local (var: decl, ctx); |
1769 | break; |
1770 | |
1771 | case OMP_CLAUSE__CONDTEMP_: |
1772 | decl = OMP_CLAUSE_DECL (c); |
1773 | if (is_parallel_ctx (ctx)) |
1774 | { |
1775 | install_var_field (var: decl, by_ref: false, mask: 3, ctx); |
1776 | install_var_local (var: decl, ctx); |
1777 | } |
1778 | else if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_FOR |
1779 | && gimple_omp_for_kind (g: ctx->stmt) == GF_OMP_FOR_KIND_SIMD |
1780 | && !OMP_CLAUSE__CONDTEMP__ITER (c)) |
1781 | install_var_local (var: decl, ctx); |
1782 | break; |
1783 | |
1784 | case OMP_CLAUSE__CACHE_: |
1785 | case OMP_CLAUSE_NOHOST: |
1786 | default: |
1787 | gcc_unreachable (); |
1788 | } |
1789 | } |
1790 | |
1791 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
1792 | { |
1793 | switch (OMP_CLAUSE_CODE (c)) |
1794 | { |
1795 | case OMP_CLAUSE_LASTPRIVATE: |
1796 | /* Let the corresponding firstprivate clause create |
1797 | the variable. */ |
1798 | if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)) |
1799 | scan_array_reductions = true; |
1800 | if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) |
1801 | break; |
1802 | /* FALLTHRU */ |
1803 | |
1804 | case OMP_CLAUSE_FIRSTPRIVATE: |
1805 | case OMP_CLAUSE_PRIVATE: |
1806 | case OMP_CLAUSE_LINEAR: |
1807 | case OMP_CLAUSE_HAS_DEVICE_ADDR: |
1808 | case OMP_CLAUSE_IS_DEVICE_PTR: |
1809 | decl = OMP_CLAUSE_DECL (c); |
1810 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR) |
1811 | { |
1812 | while (INDIRECT_REF_P (decl) |
1813 | || TREE_CODE (decl) == ARRAY_REF) |
1814 | decl = TREE_OPERAND (decl, 0); |
1815 | } |
1816 | |
1817 | if (is_variable_sized (expr: decl)) |
1818 | { |
1819 | if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE |
1820 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR |
1821 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR) |
1822 | && is_gimple_omp_offloaded (stmt: ctx->stmt)) |
1823 | { |
1824 | tree decl2 = DECL_VALUE_EXPR (decl); |
1825 | gcc_assert (INDIRECT_REF_P (decl2)); |
1826 | decl2 = TREE_OPERAND (decl2, 0); |
1827 | gcc_assert (DECL_P (decl2)); |
1828 | install_var_local (var: decl2, ctx); |
1829 | fixup_remapped_decl (decl: decl2, ctx, private_debug: false); |
1830 | } |
1831 | install_var_local (var: decl, ctx); |
1832 | } |
1833 | fixup_remapped_decl (decl, ctx, |
1834 | OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE |
1835 | && OMP_CLAUSE_PRIVATE_DEBUG (c)); |
1836 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR |
1837 | && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)) |
1838 | scan_array_reductions = true; |
1839 | break; |
1840 | |
1841 | case OMP_CLAUSE_REDUCTION: |
1842 | case OMP_CLAUSE_IN_REDUCTION: |
1843 | decl = OMP_CLAUSE_DECL (c); |
1844 | if (TREE_CODE (decl) != MEM_REF && !is_omp_target (stmt: ctx->stmt)) |
1845 | { |
1846 | if (is_variable_sized (expr: decl)) |
1847 | install_var_local (var: decl, ctx); |
1848 | fixup_remapped_decl (decl, ctx, private_debug: false); |
1849 | } |
1850 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
1851 | scan_array_reductions = true; |
1852 | break; |
1853 | |
1854 | case OMP_CLAUSE_TASK_REDUCTION: |
1855 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
1856 | scan_array_reductions = true; |
1857 | break; |
1858 | |
1859 | case OMP_CLAUSE_SHARED: |
1860 | /* Ignore shared directives in teams construct inside of |
1861 | target construct. */ |
1862 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_TEAMS |
1863 | && !is_host_teams_ctx (ctx)) |
1864 | break; |
1865 | decl = OMP_CLAUSE_DECL (c); |
1866 | if (is_global_var (t: maybe_lookup_decl_in_outer_ctx (decl, ctx))) |
1867 | break; |
1868 | if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)) |
1869 | { |
1870 | if (is_global_var (t: maybe_lookup_decl_in_outer_ctx (decl, |
1871 | ctx->outer))) |
1872 | break; |
1873 | bool by_ref = use_pointer_for_field (decl, shared_ctx: ctx); |
1874 | install_var_field (var: decl, by_ref, mask: 11, ctx); |
1875 | break; |
1876 | } |
1877 | fixup_remapped_decl (decl, ctx, private_debug: false); |
1878 | break; |
1879 | |
1880 | case OMP_CLAUSE_MAP: |
1881 | if (!is_gimple_omp_offloaded (stmt: ctx->stmt)) |
1882 | break; |
1883 | decl = OMP_CLAUSE_DECL (c); |
1884 | if (DECL_P (decl) |
1885 | && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER |
1886 | && (OMP_CLAUSE_MAP_KIND (c) |
1887 | != GOMP_MAP_FIRSTPRIVATE_REFERENCE)) |
1888 | || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE) |
1889 | && is_global_var (t: maybe_lookup_decl_in_outer_ctx (decl, ctx)) |
1890 | && varpool_node::get_create (decl)->offloadable) |
1891 | break; |
1892 | if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH |
1893 | || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH) |
1894 | && is_omp_target (stmt: ctx->stmt) |
1895 | && !is_gimple_omp_offloaded (stmt: ctx->stmt)) |
1896 | break; |
1897 | if (DECL_P (decl)) |
1898 | { |
1899 | if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER |
1900 | || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER) |
1901 | && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE |
1902 | && !COMPLETE_TYPE_P (TREE_TYPE (decl))) |
1903 | { |
1904 | tree new_decl = lookup_decl (var: decl, ctx); |
1905 | TREE_TYPE (new_decl) |
1906 | = remap_type (TREE_TYPE (decl), id: &ctx->cb); |
1907 | } |
1908 | else if (DECL_SIZE (decl) |
1909 | && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST) |
1910 | { |
1911 | tree decl2 = DECL_VALUE_EXPR (decl); |
1912 | gcc_assert (INDIRECT_REF_P (decl2)); |
1913 | decl2 = TREE_OPERAND (decl2, 0); |
1914 | gcc_assert (DECL_P (decl2)); |
1915 | fixup_remapped_decl (decl: decl2, ctx, private_debug: false); |
1916 | fixup_remapped_decl (decl, ctx, private_debug: true); |
1917 | } |
1918 | else |
1919 | fixup_remapped_decl (decl, ctx, private_debug: false); |
1920 | } |
1921 | break; |
1922 | |
1923 | case OMP_CLAUSE_COPYPRIVATE: |
1924 | case OMP_CLAUSE_COPYIN: |
1925 | case OMP_CLAUSE_DEFAULT: |
1926 | case OMP_CLAUSE_IF: |
1927 | case OMP_CLAUSE_SELF: |
1928 | case OMP_CLAUSE_NUM_THREADS: |
1929 | case OMP_CLAUSE_NUM_TEAMS: |
1930 | case OMP_CLAUSE_THREAD_LIMIT: |
1931 | case OMP_CLAUSE_DEVICE: |
1932 | case OMP_CLAUSE_SCHEDULE: |
1933 | case OMP_CLAUSE_DIST_SCHEDULE: |
1934 | case OMP_CLAUSE_NOWAIT: |
1935 | case OMP_CLAUSE_ORDERED: |
1936 | case OMP_CLAUSE_COLLAPSE: |
1937 | case OMP_CLAUSE_UNTIED: |
1938 | case OMP_CLAUSE_FINAL: |
1939 | case OMP_CLAUSE_MERGEABLE: |
1940 | case OMP_CLAUSE_PROC_BIND: |
1941 | case OMP_CLAUSE_SAFELEN: |
1942 | case OMP_CLAUSE_SIMDLEN: |
1943 | case OMP_CLAUSE_ALIGNED: |
1944 | case OMP_CLAUSE_DEPEND: |
1945 | case OMP_CLAUSE_DETACH: |
1946 | case OMP_CLAUSE_ALLOCATE: |
1947 | case OMP_CLAUSE__LOOPTEMP_: |
1948 | case OMP_CLAUSE__REDUCTEMP_: |
1949 | case OMP_CLAUSE_TO: |
1950 | case OMP_CLAUSE_FROM: |
1951 | case OMP_CLAUSE_PRIORITY: |
1952 | case OMP_CLAUSE_GRAINSIZE: |
1953 | case OMP_CLAUSE_NUM_TASKS: |
1954 | case OMP_CLAUSE_THREADS: |
1955 | case OMP_CLAUSE_SIMD: |
1956 | case OMP_CLAUSE_NOGROUP: |
1957 | case OMP_CLAUSE_DEFAULTMAP: |
1958 | case OMP_CLAUSE_ORDER: |
1959 | case OMP_CLAUSE_BIND: |
1960 | case OMP_CLAUSE_USE_DEVICE_PTR: |
1961 | case OMP_CLAUSE_USE_DEVICE_ADDR: |
1962 | case OMP_CLAUSE_NONTEMPORAL: |
1963 | case OMP_CLAUSE_ASYNC: |
1964 | case OMP_CLAUSE_WAIT: |
1965 | case OMP_CLAUSE_NUM_GANGS: |
1966 | case OMP_CLAUSE_NUM_WORKERS: |
1967 | case OMP_CLAUSE_VECTOR_LENGTH: |
1968 | case OMP_CLAUSE_GANG: |
1969 | case OMP_CLAUSE_WORKER: |
1970 | case OMP_CLAUSE_VECTOR: |
1971 | case OMP_CLAUSE_INDEPENDENT: |
1972 | case OMP_CLAUSE_AUTO: |
1973 | case OMP_CLAUSE_SEQ: |
1974 | case OMP_CLAUSE_TILE: |
1975 | case OMP_CLAUSE__SIMT_: |
1976 | case OMP_CLAUSE_IF_PRESENT: |
1977 | case OMP_CLAUSE_FINALIZE: |
1978 | case OMP_CLAUSE_FILTER: |
1979 | case OMP_CLAUSE__CONDTEMP_: |
1980 | break; |
1981 | |
1982 | case OMP_CLAUSE__CACHE_: |
1983 | case OMP_CLAUSE_NOHOST: |
1984 | default: |
1985 | gcc_unreachable (); |
1986 | } |
1987 | } |
1988 | |
1989 | gcc_checking_assert (!scan_array_reductions |
1990 | || !is_gimple_omp_oacc (ctx->stmt)); |
1991 | if (scan_array_reductions) |
1992 | { |
1993 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
1994 | if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
1995 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION |
1996 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION) |
1997 | && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
1998 | { |
1999 | omp_context *rctx = ctx; |
2000 | if (is_omp_target (stmt: ctx->stmt)) |
2001 | rctx = ctx->outer; |
2002 | scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), rctx); |
2003 | scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), rctx); |
2004 | } |
2005 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
2006 | && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)) |
2007 | scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx); |
2008 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR |
2009 | && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)) |
2010 | scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx); |
2011 | } |
2012 | } |
2013 | |
2014 | /* Create a new name for omp child function. Returns an identifier. */ |
2015 | |
2016 | static tree |
2017 | create_omp_child_function_name (bool task_copy) |
2018 | { |
2019 | return clone_function_name_numbered (decl: current_function_decl, |
2020 | suffix: task_copy ? "_omp_cpyfn" : "_omp_fn" ); |
2021 | } |
2022 | |
2023 | /* Return true if CTX may belong to offloaded code: either if current function |
2024 | is offloaded, or any enclosing context corresponds to a target region. */ |
2025 | |
2026 | static bool |
2027 | omp_maybe_offloaded_ctx (omp_context *ctx) |
2028 | { |
2029 | if (cgraph_node::get (decl: current_function_decl)->offloadable) |
2030 | return true; |
2031 | for (; ctx; ctx = ctx->outer) |
2032 | if (is_gimple_omp_offloaded (stmt: ctx->stmt)) |
2033 | return true; |
2034 | return false; |
2035 | } |
2036 | |
2037 | /* Build a decl for the omp child function. It'll not contain a body |
2038 | yet, just the bare decl. */ |
2039 | |
2040 | static void |
2041 | create_omp_child_function (omp_context *ctx, bool task_copy) |
2042 | { |
2043 | tree decl, type, name, t; |
2044 | |
2045 | name = create_omp_child_function_name (task_copy); |
2046 | if (task_copy) |
2047 | type = build_function_type_list (void_type_node, ptr_type_node, |
2048 | ptr_type_node, NULL_TREE); |
2049 | else |
2050 | type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE); |
2051 | |
2052 | decl = build_decl (gimple_location (g: ctx->stmt), FUNCTION_DECL, name, type); |
2053 | |
2054 | gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt) |
2055 | || !task_copy); |
2056 | if (!task_copy) |
2057 | ctx->cb.dst_fn = decl; |
2058 | else |
2059 | gimple_omp_task_set_copy_fn (gs: ctx->stmt, copy_fn: decl); |
2060 | |
2061 | TREE_STATIC (decl) = 1; |
2062 | TREE_USED (decl) = 1; |
2063 | DECL_ARTIFICIAL (decl) = 1; |
2064 | DECL_IGNORED_P (decl) = 0; |
2065 | TREE_PUBLIC (decl) = 0; |
2066 | DECL_UNINLINABLE (decl) = 1; |
2067 | DECL_EXTERNAL (decl) = 0; |
2068 | DECL_CONTEXT (decl) = NULL_TREE; |
2069 | DECL_INITIAL (decl) = make_node (BLOCK); |
2070 | BLOCK_SUPERCONTEXT (DECL_INITIAL (decl)) = decl; |
2071 | DECL_ATTRIBUTES (decl) = DECL_ATTRIBUTES (current_function_decl); |
2072 | /* Remove omp declare simd attribute from the new attributes. */ |
2073 | if (tree a = lookup_attribute (attr_name: "omp declare simd" , DECL_ATTRIBUTES (decl))) |
2074 | { |
2075 | while (tree a2 = lookup_attribute (attr_name: "omp declare simd" , TREE_CHAIN (a))) |
2076 | a = a2; |
2077 | a = TREE_CHAIN (a); |
2078 | for (tree *p = &DECL_ATTRIBUTES (decl); *p != a;) |
2079 | if (is_attribute_p (attr_name: "omp declare simd" , ident: get_attribute_name (*p))) |
2080 | *p = TREE_CHAIN (*p); |
2081 | else |
2082 | { |
2083 | tree chain = TREE_CHAIN (*p); |
2084 | *p = copy_node (*p); |
2085 | p = &TREE_CHAIN (*p); |
2086 | *p = chain; |
2087 | } |
2088 | } |
2089 | DECL_FUNCTION_SPECIFIC_OPTIMIZATION (decl) |
2090 | = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (current_function_decl); |
2091 | DECL_FUNCTION_SPECIFIC_TARGET (decl) |
2092 | = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl); |
2093 | DECL_FUNCTION_VERSIONED (decl) |
2094 | = DECL_FUNCTION_VERSIONED (current_function_decl); |
2095 | |
2096 | if (omp_maybe_offloaded_ctx (ctx)) |
2097 | { |
2098 | cgraph_node::get_create (decl)->offloadable = 1; |
2099 | if (ENABLE_OFFLOADING) |
2100 | g->have_offload = true; |
2101 | } |
2102 | |
2103 | if (cgraph_node::get_create (decl)->offloadable) |
2104 | { |
2105 | const char *target_attr = (is_gimple_omp_offloaded (stmt: ctx->stmt) |
2106 | ? "omp target entrypoint" |
2107 | : "omp declare target" ); |
2108 | if (lookup_attribute (attr_name: "omp declare target" , |
2109 | DECL_ATTRIBUTES (current_function_decl))) |
2110 | { |
2111 | if (is_gimple_omp_offloaded (stmt: ctx->stmt)) |
2112 | DECL_ATTRIBUTES (decl) |
2113 | = remove_attribute ("omp declare target" , |
2114 | copy_list (DECL_ATTRIBUTES (decl))); |
2115 | else |
2116 | target_attr = NULL; |
2117 | } |
2118 | if (target_attr |
2119 | && is_gimple_omp_offloaded (stmt: ctx->stmt) |
2120 | && lookup_attribute (attr_name: "noclone" , DECL_ATTRIBUTES (decl)) == NULL_TREE) |
2121 | DECL_ATTRIBUTES (decl) = tree_cons (get_identifier ("noclone" ), |
2122 | NULL_TREE, DECL_ATTRIBUTES (decl)); |
2123 | if (target_attr) |
2124 | DECL_ATTRIBUTES (decl) |
2125 | = tree_cons (get_identifier (target_attr), |
2126 | NULL_TREE, DECL_ATTRIBUTES (decl)); |
2127 | } |
2128 | |
2129 | t = build_decl (DECL_SOURCE_LOCATION (decl), |
2130 | RESULT_DECL, NULL_TREE, void_type_node); |
2131 | DECL_ARTIFICIAL (t) = 1; |
2132 | DECL_IGNORED_P (t) = 1; |
2133 | DECL_CONTEXT (t) = decl; |
2134 | DECL_RESULT (decl) = t; |
2135 | |
2136 | tree data_name = get_identifier (".omp_data_i" ); |
2137 | t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name, |
2138 | ptr_type_node); |
2139 | DECL_ARTIFICIAL (t) = 1; |
2140 | DECL_NAMELESS (t) = 1; |
2141 | DECL_ARG_TYPE (t) = ptr_type_node; |
2142 | DECL_CONTEXT (t) = current_function_decl; |
2143 | TREE_USED (t) = 1; |
2144 | TREE_READONLY (t) = 1; |
2145 | DECL_ARGUMENTS (decl) = t; |
2146 | if (!task_copy) |
2147 | ctx->receiver_decl = t; |
2148 | else |
2149 | { |
2150 | t = build_decl (DECL_SOURCE_LOCATION (decl), |
2151 | PARM_DECL, get_identifier (".omp_data_o" ), |
2152 | ptr_type_node); |
2153 | DECL_ARTIFICIAL (t) = 1; |
2154 | DECL_NAMELESS (t) = 1; |
2155 | DECL_ARG_TYPE (t) = ptr_type_node; |
2156 | DECL_CONTEXT (t) = current_function_decl; |
2157 | TREE_USED (t) = 1; |
2158 | TREE_ADDRESSABLE (t) = 1; |
2159 | DECL_CHAIN (t) = DECL_ARGUMENTS (decl); |
2160 | DECL_ARGUMENTS (decl) = t; |
2161 | } |
2162 | |
2163 | /* Allocate memory for the function structure. The call to |
2164 | allocate_struct_function clobbers CFUN, so we need to restore |
2165 | it afterward. */ |
2166 | push_struct_function (fndecl: decl); |
2167 | cfun->function_end_locus = gimple_location (g: ctx->stmt); |
2168 | init_tree_ssa (cfun); |
2169 | pop_cfun (); |
2170 | } |
2171 | |
2172 | /* Callback for walk_gimple_seq. Check if combined parallel |
2173 | contains gimple_omp_for_combined_into_p OMP_FOR. */ |
2174 | |
2175 | tree |
2176 | omp_find_combined_for (gimple_stmt_iterator *gsi_p, |
2177 | bool *handled_ops_p, |
2178 | struct walk_stmt_info *wi) |
2179 | { |
2180 | gimple *stmt = gsi_stmt (i: *gsi_p); |
2181 | |
2182 | *handled_ops_p = true; |
2183 | switch (gimple_code (g: stmt)) |
2184 | { |
2185 | WALK_SUBSTMTS; |
2186 | |
2187 | case GIMPLE_OMP_FOR: |
2188 | if (gimple_omp_for_combined_into_p (g: stmt) |
2189 | && gimple_omp_for_kind (g: stmt) |
2190 | == *(const enum gf_mask *) (wi->info)) |
2191 | { |
2192 | wi->info = stmt; |
2193 | return integer_zero_node; |
2194 | } |
2195 | break; |
2196 | default: |
2197 | break; |
2198 | } |
2199 | return NULL; |
2200 | } |
2201 | |
2202 | /* Add _LOOPTEMP_/_REDUCTEMP_ clauses on OpenMP parallel or task. */ |
2203 | |
2204 | static void |
2205 | add_taskreg_looptemp_clauses (enum gf_mask msk, gimple *stmt, |
2206 | omp_context *outer_ctx) |
2207 | { |
2208 | struct walk_stmt_info wi; |
2209 | |
2210 | memset (s: &wi, c: 0, n: sizeof (wi)); |
2211 | wi.val_only = true; |
2212 | wi.info = (void *) &msk; |
2213 | walk_gimple_seq (gimple_omp_body (gs: stmt), omp_find_combined_for, NULL, &wi); |
2214 | if (wi.info != (void *) &msk) |
2215 | { |
2216 | gomp_for *for_stmt = as_a <gomp_for *> (p: (gimple *) wi.info); |
2217 | struct omp_for_data fd; |
2218 | omp_extract_for_data (for_stmt, fd: &fd, NULL); |
2219 | /* We need two temporaries with fd.loop.v type (istart/iend) |
2220 | and then (fd.collapse - 1) temporaries with the same |
2221 | type for count2 ... countN-1 vars if not constant. */ |
2222 | size_t count = 2, i; |
2223 | tree type = fd.iter_type; |
2224 | if (fd.collapse > 1 |
2225 | && TREE_CODE (fd.loop.n2) != INTEGER_CST) |
2226 | { |
2227 | count += fd.collapse - 1; |
2228 | /* If there are lastprivate clauses on the inner |
2229 | GIMPLE_OMP_FOR, add one more temporaries for the total number |
2230 | of iterations (product of count1 ... countN-1). */ |
2231 | if (omp_find_clause (clauses: gimple_omp_for_clauses (gs: for_stmt), |
2232 | kind: OMP_CLAUSE_LASTPRIVATE) |
2233 | || (msk == GF_OMP_FOR_KIND_FOR |
2234 | && omp_find_clause (clauses: gimple_omp_parallel_clauses (gs: stmt), |
2235 | kind: OMP_CLAUSE_LASTPRIVATE))) |
2236 | { |
2237 | tree temp = create_tmp_var (type); |
2238 | tree c = build_omp_clause (UNKNOWN_LOCATION, |
2239 | OMP_CLAUSE__LOOPTEMP_); |
2240 | insert_decl_map (&outer_ctx->cb, temp, temp); |
2241 | OMP_CLAUSE_DECL (c) = temp; |
2242 | OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (gs: stmt); |
2243 | gimple_omp_taskreg_set_clauses (gs: stmt, clauses: c); |
2244 | } |
2245 | if (fd.non_rect |
2246 | && fd.last_nonrect == fd.first_nonrect + 1) |
2247 | if (tree v = gimple_omp_for_index (gs: for_stmt, i: fd.last_nonrect)) |
2248 | if (!TYPE_UNSIGNED (TREE_TYPE (v))) |
2249 | { |
2250 | v = gimple_omp_for_index (gs: for_stmt, i: fd.first_nonrect); |
2251 | tree type2 = TREE_TYPE (v); |
2252 | count++; |
2253 | for (i = 0; i < 3; i++) |
2254 | { |
2255 | tree temp = create_tmp_var (type2); |
2256 | tree c = build_omp_clause (UNKNOWN_LOCATION, |
2257 | OMP_CLAUSE__LOOPTEMP_); |
2258 | insert_decl_map (&outer_ctx->cb, temp, temp); |
2259 | OMP_CLAUSE_DECL (c) = temp; |
2260 | OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (gs: stmt); |
2261 | gimple_omp_taskreg_set_clauses (gs: stmt, clauses: c); |
2262 | } |
2263 | } |
2264 | } |
2265 | for (i = 0; i < count; i++) |
2266 | { |
2267 | tree temp = create_tmp_var (type); |
2268 | tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_); |
2269 | insert_decl_map (&outer_ctx->cb, temp, temp); |
2270 | OMP_CLAUSE_DECL (c) = temp; |
2271 | OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (gs: stmt); |
2272 | gimple_omp_taskreg_set_clauses (gs: stmt, clauses: c); |
2273 | } |
2274 | } |
2275 | if (msk == GF_OMP_FOR_KIND_TASKLOOP |
2276 | && omp_find_clause (clauses: gimple_omp_task_clauses (gs: stmt), |
2277 | kind: OMP_CLAUSE_REDUCTION)) |
2278 | { |
2279 | tree type = build_pointer_type (pointer_sized_int_node); |
2280 | tree temp = create_tmp_var (type); |
2281 | tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_); |
2282 | insert_decl_map (&outer_ctx->cb, temp, temp); |
2283 | OMP_CLAUSE_DECL (c) = temp; |
2284 | OMP_CLAUSE_CHAIN (c) = gimple_omp_task_clauses (gs: stmt); |
2285 | gimple_omp_task_set_clauses (gs: stmt, clauses: c); |
2286 | } |
2287 | } |
2288 | |
2289 | /* Scan an OpenMP parallel directive. */ |
2290 | |
2291 | static void |
2292 | scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx) |
2293 | { |
2294 | omp_context *ctx; |
2295 | tree name; |
2296 | gomp_parallel *stmt = as_a <gomp_parallel *> (p: gsi_stmt (i: *gsi)); |
2297 | |
2298 | /* Ignore parallel directives with empty bodies, unless there |
2299 | are copyin clauses. */ |
2300 | if (optimize > 0 |
2301 | && empty_body_p (gimple_omp_body (gs: stmt)) |
2302 | && omp_find_clause (clauses: gimple_omp_parallel_clauses (gs: stmt), |
2303 | kind: OMP_CLAUSE_COPYIN) == NULL) |
2304 | { |
2305 | gsi_replace (gsi, gimple_build_nop (), false); |
2306 | return; |
2307 | } |
2308 | |
2309 | if (gimple_omp_parallel_combined_p (g: stmt)) |
2310 | add_taskreg_looptemp_clauses (msk: GF_OMP_FOR_KIND_FOR, stmt, outer_ctx); |
2311 | for (tree c = omp_find_clause (clauses: gimple_omp_parallel_clauses (gs: stmt), |
2312 | kind: OMP_CLAUSE_REDUCTION); |
2313 | c; c = omp_find_clause (OMP_CLAUSE_CHAIN (c), kind: OMP_CLAUSE_REDUCTION)) |
2314 | if (OMP_CLAUSE_REDUCTION_TASK (c)) |
2315 | { |
2316 | tree type = build_pointer_type (pointer_sized_int_node); |
2317 | tree temp = create_tmp_var (type); |
2318 | tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_); |
2319 | if (outer_ctx) |
2320 | insert_decl_map (&outer_ctx->cb, temp, temp); |
2321 | OMP_CLAUSE_DECL (c) = temp; |
2322 | OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (gs: stmt); |
2323 | gimple_omp_parallel_set_clauses (omp_parallel_stmt: stmt, clauses: c); |
2324 | break; |
2325 | } |
2326 | else if (OMP_CLAUSE_CHAIN (c) == NULL_TREE) |
2327 | break; |
2328 | |
2329 | ctx = new_omp_context (stmt, outer_ctx); |
2330 | taskreg_contexts.safe_push (obj: ctx); |
2331 | if (taskreg_nesting_level > 1) |
2332 | ctx->is_nested = true; |
2333 | ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
2334 | ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); |
2335 | name = create_tmp_var_name (".omp_data_s" ); |
2336 | name = build_decl (gimple_location (g: stmt), |
2337 | TYPE_DECL, name, ctx->record_type); |
2338 | DECL_ARTIFICIAL (name) = 1; |
2339 | DECL_NAMELESS (name) = 1; |
2340 | TYPE_NAME (ctx->record_type) = name; |
2341 | TYPE_ARTIFICIAL (ctx->record_type) = 1; |
2342 | create_omp_child_function (ctx, task_copy: false); |
2343 | gimple_omp_parallel_set_child_fn (omp_parallel_stmt: stmt, child_fn: ctx->cb.dst_fn); |
2344 | |
2345 | scan_sharing_clauses (clauses: gimple_omp_parallel_clauses (gs: stmt), ctx); |
2346 | scan_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
2347 | |
2348 | if (TYPE_FIELDS (ctx->record_type) == NULL) |
2349 | ctx->record_type = ctx->receiver_decl = NULL; |
2350 | } |
2351 | |
2352 | /* Scan an OpenMP task directive. */ |
2353 | |
2354 | static void |
2355 | scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx) |
2356 | { |
2357 | omp_context *ctx; |
2358 | tree name, t; |
2359 | gomp_task *stmt = as_a <gomp_task *> (p: gsi_stmt (i: *gsi)); |
2360 | |
2361 | /* Ignore task directives with empty bodies, unless they have depend |
2362 | clause. */ |
2363 | if (optimize > 0 |
2364 | && gimple_omp_body (gs: stmt) |
2365 | && empty_body_p (gimple_omp_body (gs: stmt)) |
2366 | && !omp_find_clause (clauses: gimple_omp_task_clauses (gs: stmt), kind: OMP_CLAUSE_DEPEND)) |
2367 | { |
2368 | gsi_replace (gsi, gimple_build_nop (), false); |
2369 | return; |
2370 | } |
2371 | |
2372 | if (gimple_omp_task_taskloop_p (g: stmt)) |
2373 | add_taskreg_looptemp_clauses (msk: GF_OMP_FOR_KIND_TASKLOOP, stmt, outer_ctx); |
2374 | |
2375 | ctx = new_omp_context (stmt, outer_ctx); |
2376 | |
2377 | if (gimple_omp_task_taskwait_p (g: stmt)) |
2378 | { |
2379 | scan_sharing_clauses (clauses: gimple_omp_task_clauses (gs: stmt), ctx); |
2380 | return; |
2381 | } |
2382 | |
2383 | taskreg_contexts.safe_push (obj: ctx); |
2384 | if (taskreg_nesting_level > 1) |
2385 | ctx->is_nested = true; |
2386 | ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
2387 | ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); |
2388 | name = create_tmp_var_name (".omp_data_s" ); |
2389 | name = build_decl (gimple_location (g: stmt), |
2390 | TYPE_DECL, name, ctx->record_type); |
2391 | DECL_ARTIFICIAL (name) = 1; |
2392 | DECL_NAMELESS (name) = 1; |
2393 | TYPE_NAME (ctx->record_type) = name; |
2394 | TYPE_ARTIFICIAL (ctx->record_type) = 1; |
2395 | create_omp_child_function (ctx, task_copy: false); |
2396 | gimple_omp_task_set_child_fn (gs: stmt, child_fn: ctx->cb.dst_fn); |
2397 | |
2398 | scan_sharing_clauses (clauses: gimple_omp_task_clauses (gs: stmt), ctx); |
2399 | |
2400 | if (ctx->srecord_type) |
2401 | { |
2402 | name = create_tmp_var_name (".omp_data_a" ); |
2403 | name = build_decl (gimple_location (g: stmt), |
2404 | TYPE_DECL, name, ctx->srecord_type); |
2405 | DECL_ARTIFICIAL (name) = 1; |
2406 | DECL_NAMELESS (name) = 1; |
2407 | TYPE_NAME (ctx->srecord_type) = name; |
2408 | TYPE_ARTIFICIAL (ctx->srecord_type) = 1; |
2409 | create_omp_child_function (ctx, task_copy: true); |
2410 | } |
2411 | |
2412 | scan_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
2413 | |
2414 | if (TYPE_FIELDS (ctx->record_type) == NULL) |
2415 | { |
2416 | ctx->record_type = ctx->receiver_decl = NULL; |
2417 | t = build_int_cst (long_integer_type_node, 0); |
2418 | gimple_omp_task_set_arg_size (gs: stmt, arg_size: t); |
2419 | t = build_int_cst (long_integer_type_node, 1); |
2420 | gimple_omp_task_set_arg_align (gs: stmt, arg_align: t); |
2421 | } |
2422 | } |
2423 | |
2424 | /* Helper function for finish_taskreg_scan, called through walk_tree. |
2425 | If maybe_lookup_decl_in_outer_context returns non-NULL for some |
2426 | tree, replace it in the expression. */ |
2427 | |
2428 | static tree |
2429 | finish_taskreg_remap (tree *tp, int *walk_subtrees, void *data) |
2430 | { |
2431 | if (VAR_P (*tp)) |
2432 | { |
2433 | omp_context *ctx = (omp_context *) data; |
2434 | tree t = maybe_lookup_decl_in_outer_ctx (*tp, ctx); |
2435 | if (t != *tp) |
2436 | { |
2437 | if (DECL_HAS_VALUE_EXPR_P (t)) |
2438 | t = unshare_expr (DECL_VALUE_EXPR (t)); |
2439 | *tp = t; |
2440 | } |
2441 | *walk_subtrees = 0; |
2442 | } |
2443 | else if (IS_TYPE_OR_DECL_P (*tp)) |
2444 | *walk_subtrees = 0; |
2445 | return NULL_TREE; |
2446 | } |
2447 | |
2448 | /* If any decls have been made addressable during scan_omp, |
2449 | adjust their fields if needed, and layout record types |
2450 | of parallel/task constructs. */ |
2451 | |
2452 | static void |
2453 | finish_taskreg_scan (omp_context *ctx) |
2454 | { |
2455 | if (ctx->record_type == NULL_TREE) |
2456 | return; |
2457 | |
2458 | /* If any make_addressable_vars were needed, verify all |
2459 | OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK,TEAMS} |
2460 | statements if use_pointer_for_field hasn't changed |
2461 | because of that. If it did, update field types now. */ |
2462 | if (make_addressable_vars) |
2463 | { |
2464 | tree c; |
2465 | |
2466 | for (c = gimple_omp_taskreg_clauses (gs: ctx->stmt); |
2467 | c; c = OMP_CLAUSE_CHAIN (c)) |
2468 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED |
2469 | && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)) |
2470 | { |
2471 | tree decl = OMP_CLAUSE_DECL (c); |
2472 | |
2473 | /* Global variables don't need to be copied, |
2474 | the receiver side will use them directly. */ |
2475 | if (is_global_var (t: maybe_lookup_decl_in_outer_ctx (decl, ctx))) |
2476 | continue; |
2477 | if (!bitmap_bit_p (make_addressable_vars, DECL_UID (decl)) |
2478 | || !use_pointer_for_field (decl, shared_ctx: ctx)) |
2479 | continue; |
2480 | tree field = lookup_field (var: decl, ctx); |
2481 | if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE |
2482 | && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl)) |
2483 | continue; |
2484 | TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl)); |
2485 | TREE_THIS_VOLATILE (field) = 0; |
2486 | DECL_USER_ALIGN (field) = 0; |
2487 | SET_DECL_ALIGN (field, TYPE_ALIGN (TREE_TYPE (field))); |
2488 | if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field)) |
2489 | SET_TYPE_ALIGN (ctx->record_type, DECL_ALIGN (field)); |
2490 | if (ctx->srecord_type) |
2491 | { |
2492 | tree sfield = lookup_sfield (var: decl, ctx); |
2493 | TREE_TYPE (sfield) = TREE_TYPE (field); |
2494 | TREE_THIS_VOLATILE (sfield) = 0; |
2495 | DECL_USER_ALIGN (sfield) = 0; |
2496 | SET_DECL_ALIGN (sfield, DECL_ALIGN (field)); |
2497 | if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield)) |
2498 | SET_TYPE_ALIGN (ctx->srecord_type, DECL_ALIGN (sfield)); |
2499 | } |
2500 | } |
2501 | } |
2502 | |
2503 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_PARALLEL) |
2504 | { |
2505 | tree clauses = gimple_omp_parallel_clauses (gs: ctx->stmt); |
2506 | tree c = omp_find_clause (clauses, kind: OMP_CLAUSE__REDUCTEMP_); |
2507 | if (c) |
2508 | { |
2509 | /* Move the _reductemp_ clause first. GOMP_parallel_reductions |
2510 | expects to find it at the start of data. */ |
2511 | tree f = lookup_field (OMP_CLAUSE_DECL (c), ctx); |
2512 | tree *p = &TYPE_FIELDS (ctx->record_type); |
2513 | while (*p) |
2514 | if (*p == f) |
2515 | { |
2516 | *p = DECL_CHAIN (*p); |
2517 | break; |
2518 | } |
2519 | else |
2520 | p = &DECL_CHAIN (*p); |
2521 | DECL_CHAIN (f) = TYPE_FIELDS (ctx->record_type); |
2522 | TYPE_FIELDS (ctx->record_type) = f; |
2523 | } |
2524 | layout_type (ctx->record_type); |
2525 | fixup_child_record_type (ctx); |
2526 | } |
2527 | else if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_TEAMS) |
2528 | { |
2529 | layout_type (ctx->record_type); |
2530 | fixup_child_record_type (ctx); |
2531 | } |
2532 | else |
2533 | { |
2534 | location_t loc = gimple_location (g: ctx->stmt); |
2535 | tree *p, vla_fields = NULL_TREE, *q = &vla_fields; |
2536 | tree detach_clause |
2537 | = omp_find_clause (clauses: gimple_omp_task_clauses (gs: ctx->stmt), |
2538 | kind: OMP_CLAUSE_DETACH); |
2539 | /* Move VLA fields to the end. */ |
2540 | p = &TYPE_FIELDS (ctx->record_type); |
2541 | while (*p) |
2542 | if (!TYPE_SIZE_UNIT (TREE_TYPE (*p)) |
2543 | || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p)))) |
2544 | { |
2545 | *q = *p; |
2546 | *p = TREE_CHAIN (*p); |
2547 | TREE_CHAIN (*q) = NULL_TREE; |
2548 | q = &TREE_CHAIN (*q); |
2549 | } |
2550 | else |
2551 | p = &DECL_CHAIN (*p); |
2552 | *p = vla_fields; |
2553 | if (gimple_omp_task_taskloop_p (g: ctx->stmt)) |
2554 | { |
2555 | /* Move fields corresponding to first and second _looptemp_ |
2556 | clause first. There are filled by GOMP_taskloop |
2557 | and thus need to be in specific positions. */ |
2558 | tree clauses = gimple_omp_task_clauses (gs: ctx->stmt); |
2559 | tree c1 = omp_find_clause (clauses, kind: OMP_CLAUSE__LOOPTEMP_); |
2560 | tree c2 = omp_find_clause (OMP_CLAUSE_CHAIN (c1), |
2561 | kind: OMP_CLAUSE__LOOPTEMP_); |
2562 | tree c3 = omp_find_clause (clauses, kind: OMP_CLAUSE__REDUCTEMP_); |
2563 | tree f1 = lookup_field (OMP_CLAUSE_DECL (c1), ctx); |
2564 | tree f2 = lookup_field (OMP_CLAUSE_DECL (c2), ctx); |
2565 | tree f3 = c3 ? lookup_field (OMP_CLAUSE_DECL (c3), ctx) : NULL_TREE; |
2566 | p = &TYPE_FIELDS (ctx->record_type); |
2567 | while (*p) |
2568 | if (*p == f1 || *p == f2 || *p == f3) |
2569 | *p = DECL_CHAIN (*p); |
2570 | else |
2571 | p = &DECL_CHAIN (*p); |
2572 | DECL_CHAIN (f1) = f2; |
2573 | if (c3) |
2574 | { |
2575 | DECL_CHAIN (f2) = f3; |
2576 | DECL_CHAIN (f3) = TYPE_FIELDS (ctx->record_type); |
2577 | } |
2578 | else |
2579 | DECL_CHAIN (f2) = TYPE_FIELDS (ctx->record_type); |
2580 | TYPE_FIELDS (ctx->record_type) = f1; |
2581 | if (ctx->srecord_type) |
2582 | { |
2583 | f1 = lookup_sfield (OMP_CLAUSE_DECL (c1), ctx); |
2584 | f2 = lookup_sfield (OMP_CLAUSE_DECL (c2), ctx); |
2585 | if (c3) |
2586 | f3 = lookup_sfield (OMP_CLAUSE_DECL (c3), ctx); |
2587 | p = &TYPE_FIELDS (ctx->srecord_type); |
2588 | while (*p) |
2589 | if (*p == f1 || *p == f2 || *p == f3) |
2590 | *p = DECL_CHAIN (*p); |
2591 | else |
2592 | p = &DECL_CHAIN (*p); |
2593 | DECL_CHAIN (f1) = f2; |
2594 | DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type); |
2595 | if (c3) |
2596 | { |
2597 | DECL_CHAIN (f2) = f3; |
2598 | DECL_CHAIN (f3) = TYPE_FIELDS (ctx->srecord_type); |
2599 | } |
2600 | else |
2601 | DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type); |
2602 | TYPE_FIELDS (ctx->srecord_type) = f1; |
2603 | } |
2604 | } |
2605 | if (detach_clause) |
2606 | { |
2607 | tree c, field; |
2608 | |
2609 | /* Look for a firstprivate clause with the detach event handle. */ |
2610 | for (c = gimple_omp_taskreg_clauses (gs: ctx->stmt); |
2611 | c; c = OMP_CLAUSE_CHAIN (c)) |
2612 | { |
2613 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE) |
2614 | continue; |
2615 | if (maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c), ctx) |
2616 | == OMP_CLAUSE_DECL (detach_clause)) |
2617 | break; |
2618 | } |
2619 | |
2620 | gcc_assert (c); |
2621 | field = lookup_field (OMP_CLAUSE_DECL (c), ctx); |
2622 | |
2623 | /* Move field corresponding to the detach clause first. |
2624 | This is filled by GOMP_task and needs to be in a |
2625 | specific position. */ |
2626 | p = &TYPE_FIELDS (ctx->record_type); |
2627 | while (*p) |
2628 | if (*p == field) |
2629 | *p = DECL_CHAIN (*p); |
2630 | else |
2631 | p = &DECL_CHAIN (*p); |
2632 | DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type); |
2633 | TYPE_FIELDS (ctx->record_type) = field; |
2634 | if (ctx->srecord_type) |
2635 | { |
2636 | field = lookup_sfield (OMP_CLAUSE_DECL (c), ctx); |
2637 | p = &TYPE_FIELDS (ctx->srecord_type); |
2638 | while (*p) |
2639 | if (*p == field) |
2640 | *p = DECL_CHAIN (*p); |
2641 | else |
2642 | p = &DECL_CHAIN (*p); |
2643 | DECL_CHAIN (field) = TYPE_FIELDS (ctx->srecord_type); |
2644 | TYPE_FIELDS (ctx->srecord_type) = field; |
2645 | } |
2646 | } |
2647 | layout_type (ctx->record_type); |
2648 | fixup_child_record_type (ctx); |
2649 | if (ctx->srecord_type) |
2650 | layout_type (ctx->srecord_type); |
2651 | tree t = fold_convert_loc (loc, long_integer_type_node, |
2652 | TYPE_SIZE_UNIT (ctx->record_type)); |
2653 | if (TREE_CODE (t) != INTEGER_CST) |
2654 | { |
2655 | t = unshare_expr (t); |
2656 | walk_tree (&t, finish_taskreg_remap, ctx, NULL); |
2657 | } |
2658 | gimple_omp_task_set_arg_size (gs: ctx->stmt, arg_size: t); |
2659 | t = build_int_cst (long_integer_type_node, |
2660 | TYPE_ALIGN_UNIT (ctx->record_type)); |
2661 | gimple_omp_task_set_arg_align (gs: ctx->stmt, arg_align: t); |
2662 | } |
2663 | } |
2664 | |
2665 | /* Find the enclosing offload context. */ |
2666 | |
2667 | static omp_context * |
2668 | enclosing_target_ctx (omp_context *ctx) |
2669 | { |
2670 | for (; ctx; ctx = ctx->outer) |
2671 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_TARGET) |
2672 | break; |
2673 | |
2674 | return ctx; |
2675 | } |
2676 | |
2677 | /* Return whether CTX's parent compute construct is an OpenACC 'kernels' |
2678 | construct. |
2679 | (This doesn't include OpenACC 'kernels' decomposed parts.) */ |
2680 | |
2681 | static bool |
2682 | ctx_in_oacc_kernels_region (omp_context *ctx) |
2683 | { |
2684 | for (;ctx != NULL; ctx = ctx->outer) |
2685 | { |
2686 | gimple *stmt = ctx->stmt; |
2687 | if (gimple_code (g: stmt) == GIMPLE_OMP_TARGET |
2688 | && gimple_omp_target_kind (g: stmt) == GF_OMP_TARGET_KIND_OACC_KERNELS) |
2689 | return true; |
2690 | } |
2691 | |
2692 | return false; |
2693 | } |
2694 | |
2695 | /* Check the parallelism clauses inside a OpenACC 'kernels' region. |
2696 | (This doesn't include OpenACC 'kernels' decomposed parts.) |
2697 | Until kernels handling moves to use the same loop indirection |
2698 | scheme as parallel, we need to do this checking early. */ |
2699 | |
2700 | static unsigned |
2701 | check_oacc_kernel_gwv (gomp_for *stmt, omp_context *ctx) |
2702 | { |
2703 | bool checking = true; |
2704 | unsigned outer_mask = 0; |
2705 | unsigned this_mask = 0; |
2706 | bool has_seq = false, has_auto = false; |
2707 | |
2708 | if (ctx->outer) |
2709 | outer_mask = check_oacc_kernel_gwv (NULL, ctx: ctx->outer); |
2710 | if (!stmt) |
2711 | { |
2712 | checking = false; |
2713 | if (gimple_code (g: ctx->stmt) != GIMPLE_OMP_FOR) |
2714 | return outer_mask; |
2715 | stmt = as_a <gomp_for *> (p: ctx->stmt); |
2716 | } |
2717 | |
2718 | for (tree c = gimple_omp_for_clauses (gs: stmt); c; c = OMP_CLAUSE_CHAIN (c)) |
2719 | { |
2720 | switch (OMP_CLAUSE_CODE (c)) |
2721 | { |
2722 | case OMP_CLAUSE_GANG: |
2723 | this_mask |= GOMP_DIM_MASK (GOMP_DIM_GANG); |
2724 | break; |
2725 | case OMP_CLAUSE_WORKER: |
2726 | this_mask |= GOMP_DIM_MASK (GOMP_DIM_WORKER); |
2727 | break; |
2728 | case OMP_CLAUSE_VECTOR: |
2729 | this_mask |= GOMP_DIM_MASK (GOMP_DIM_VECTOR); |
2730 | break; |
2731 | case OMP_CLAUSE_SEQ: |
2732 | has_seq = true; |
2733 | break; |
2734 | case OMP_CLAUSE_AUTO: |
2735 | has_auto = true; |
2736 | break; |
2737 | default: |
2738 | break; |
2739 | } |
2740 | } |
2741 | |
2742 | if (checking) |
2743 | { |
2744 | if (has_seq && (this_mask || has_auto)) |
2745 | error_at (gimple_location (g: stmt), "%<seq%> overrides other" |
2746 | " OpenACC loop specifiers" ); |
2747 | else if (has_auto && this_mask) |
2748 | error_at (gimple_location (g: stmt), "%<auto%> conflicts with other" |
2749 | " OpenACC loop specifiers" ); |
2750 | |
2751 | if (this_mask & outer_mask) |
2752 | error_at (gimple_location (g: stmt), "inner loop uses same" |
2753 | " OpenACC parallelism as containing loop" ); |
2754 | } |
2755 | |
2756 | return outer_mask | this_mask; |
2757 | } |
2758 | |
2759 | /* Scan a GIMPLE_OMP_FOR. */ |
2760 | |
2761 | static omp_context * |
2762 | scan_omp_for (gomp_for *stmt, omp_context *outer_ctx) |
2763 | { |
2764 | omp_context *ctx; |
2765 | size_t i; |
2766 | tree clauses = gimple_omp_for_clauses (gs: stmt); |
2767 | |
2768 | ctx = new_omp_context (stmt, outer_ctx); |
2769 | |
2770 | if (is_gimple_omp_oacc (stmt)) |
2771 | { |
2772 | omp_context *tgt = enclosing_target_ctx (ctx: outer_ctx); |
2773 | |
2774 | if (!(tgt && is_oacc_kernels (ctx: tgt))) |
2775 | for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
2776 | { |
2777 | tree c_op0; |
2778 | switch (OMP_CLAUSE_CODE (c)) |
2779 | { |
2780 | case OMP_CLAUSE_GANG: |
2781 | c_op0 = OMP_CLAUSE_GANG_EXPR (c); |
2782 | break; |
2783 | |
2784 | case OMP_CLAUSE_WORKER: |
2785 | c_op0 = OMP_CLAUSE_WORKER_EXPR (c); |
2786 | break; |
2787 | |
2788 | case OMP_CLAUSE_VECTOR: |
2789 | c_op0 = OMP_CLAUSE_VECTOR_EXPR (c); |
2790 | break; |
2791 | |
2792 | default: |
2793 | continue; |
2794 | } |
2795 | |
2796 | if (c_op0) |
2797 | { |
2798 | /* By construction, this is impossible for OpenACC 'kernels' |
2799 | decomposed parts. */ |
2800 | gcc_assert (!(tgt && is_oacc_kernels_decomposed_part (tgt))); |
2801 | |
2802 | error_at (OMP_CLAUSE_LOCATION (c), |
2803 | "argument not permitted on %qs clause" , |
2804 | omp_clause_code_name[OMP_CLAUSE_CODE (c)]); |
2805 | if (tgt) |
2806 | inform (gimple_location (g: tgt->stmt), |
2807 | "enclosing parent compute construct" ); |
2808 | else if (oacc_get_fn_attrib (fn: current_function_decl)) |
2809 | inform (DECL_SOURCE_LOCATION (current_function_decl), |
2810 | "enclosing routine" ); |
2811 | else |
2812 | gcc_unreachable (); |
2813 | } |
2814 | } |
2815 | |
2816 | if (tgt && is_oacc_kernels (ctx: tgt)) |
2817 | check_oacc_kernel_gwv (stmt, ctx); |
2818 | |
2819 | /* Collect all variables named in reductions on this loop. Ensure |
2820 | that, if this loop has a reduction on some variable v, and there is |
2821 | a reduction on v somewhere in an outer context, then there is a |
2822 | reduction on v on all intervening loops as well. */ |
2823 | tree local_reduction_clauses = NULL; |
2824 | for (tree c = gimple_omp_for_clauses (gs: stmt); c; c = OMP_CLAUSE_CHAIN (c)) |
2825 | { |
2826 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION) |
2827 | local_reduction_clauses |
2828 | = tree_cons (NULL, c, local_reduction_clauses); |
2829 | } |
2830 | if (ctx->outer_reduction_clauses == NULL && ctx->outer != NULL) |
2831 | ctx->outer_reduction_clauses |
2832 | = chainon (unshare_expr (ctx->outer->local_reduction_clauses), |
2833 | ctx->outer->outer_reduction_clauses); |
2834 | tree outer_reduction_clauses = ctx->outer_reduction_clauses; |
2835 | tree local_iter = local_reduction_clauses; |
2836 | for (; local_iter; local_iter = TREE_CHAIN (local_iter)) |
2837 | { |
2838 | tree local_clause = TREE_VALUE (local_iter); |
2839 | tree local_var = OMP_CLAUSE_DECL (local_clause); |
2840 | tree_code local_op = OMP_CLAUSE_REDUCTION_CODE (local_clause); |
2841 | bool have_outer_reduction = false; |
2842 | tree ctx_iter = outer_reduction_clauses; |
2843 | for (; ctx_iter; ctx_iter = TREE_CHAIN (ctx_iter)) |
2844 | { |
2845 | tree outer_clause = TREE_VALUE (ctx_iter); |
2846 | tree outer_var = OMP_CLAUSE_DECL (outer_clause); |
2847 | tree_code outer_op = OMP_CLAUSE_REDUCTION_CODE (outer_clause); |
2848 | if (outer_var == local_var && outer_op != local_op) |
2849 | { |
2850 | if (warning_at (OMP_CLAUSE_LOCATION (local_clause), |
2851 | OPT_Wopenmp, "conflicting reduction " |
2852 | "operations for %qE" , |
2853 | local_var)) |
2854 | inform (OMP_CLAUSE_LOCATION (outer_clause), |
2855 | "location of the previous reduction for %qE" , |
2856 | outer_var); |
2857 | } |
2858 | if (outer_var == local_var) |
2859 | { |
2860 | have_outer_reduction = true; |
2861 | break; |
2862 | } |
2863 | } |
2864 | if (have_outer_reduction) |
2865 | { |
2866 | /* There is a reduction on outer_var both on this loop and on |
2867 | some enclosing loop. Walk up the context tree until such a |
2868 | loop with a reduction on outer_var is found, and complain |
2869 | about all intervening loops that do not have such a |
2870 | reduction. */ |
2871 | struct omp_context *curr_loop = ctx->outer; |
2872 | bool found = false; |
2873 | while (curr_loop != NULL) |
2874 | { |
2875 | tree curr_iter = curr_loop->local_reduction_clauses; |
2876 | for (; curr_iter; curr_iter = TREE_CHAIN (curr_iter)) |
2877 | { |
2878 | tree curr_clause = TREE_VALUE (curr_iter); |
2879 | tree curr_var = OMP_CLAUSE_DECL (curr_clause); |
2880 | if (curr_var == local_var) |
2881 | { |
2882 | found = true; |
2883 | break; |
2884 | } |
2885 | } |
2886 | if (!found) |
2887 | warning_at (gimple_location (g: curr_loop->stmt), OPT_Wopenmp, |
2888 | "nested loop in reduction needs " |
2889 | "reduction clause for %qE" , |
2890 | local_var); |
2891 | else |
2892 | break; |
2893 | curr_loop = curr_loop->outer; |
2894 | } |
2895 | } |
2896 | } |
2897 | ctx->local_reduction_clauses = local_reduction_clauses; |
2898 | ctx->outer_reduction_clauses |
2899 | = chainon (unshare_expr (ctx->local_reduction_clauses), |
2900 | ctx->outer_reduction_clauses); |
2901 | |
2902 | if (tgt && is_oacc_kernels (ctx: tgt)) |
2903 | { |
2904 | /* Strip out reductions, as they are not handled yet. */ |
2905 | tree *prev_ptr = &clauses; |
2906 | |
2907 | while (tree probe = *prev_ptr) |
2908 | { |
2909 | tree *next_ptr = &OMP_CLAUSE_CHAIN (probe); |
2910 | |
2911 | if (OMP_CLAUSE_CODE (probe) == OMP_CLAUSE_REDUCTION) |
2912 | *prev_ptr = *next_ptr; |
2913 | else |
2914 | prev_ptr = next_ptr; |
2915 | } |
2916 | |
2917 | gimple_omp_for_set_clauses (gs: stmt, clauses); |
2918 | } |
2919 | } |
2920 | |
2921 | scan_sharing_clauses (clauses, ctx); |
2922 | |
2923 | scan_omp (gimple_omp_for_pre_body_ptr (gs: stmt), ctx); |
2924 | for (i = 0; i < gimple_omp_for_collapse (gs: stmt); i++) |
2925 | { |
2926 | scan_omp_op (tp: gimple_omp_for_index_ptr (gs: stmt, i), ctx); |
2927 | scan_omp_op (tp: gimple_omp_for_initial_ptr (gs: stmt, i), ctx); |
2928 | scan_omp_op (tp: gimple_omp_for_final_ptr (gs: stmt, i), ctx); |
2929 | scan_omp_op (tp: gimple_omp_for_incr_ptr (gs: stmt, i), ctx); |
2930 | } |
2931 | scan_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
2932 | return ctx; |
2933 | } |
2934 | |
2935 | /* Duplicate #pragma omp simd, one for SIMT, another one for SIMD. */ |
2936 | |
2937 | static void |
2938 | scan_omp_simd (gimple_stmt_iterator *gsi, gomp_for *stmt, |
2939 | omp_context *outer_ctx) |
2940 | { |
2941 | gbind *bind = gimple_build_bind (NULL, NULL, NULL); |
2942 | gsi_replace (gsi, bind, false); |
2943 | gimple_seq seq = NULL; |
2944 | gimple *g = gimple_build_call_internal (IFN_GOMP_USE_SIMT, 0); |
2945 | tree cond = create_tmp_var_raw (integer_type_node); |
2946 | DECL_CONTEXT (cond) = current_function_decl; |
2947 | DECL_SEEN_IN_BIND_EXPR_P (cond) = 1; |
2948 | gimple_bind_set_vars (bind_stmt: bind, vars: cond); |
2949 | gimple_call_set_lhs (gs: g, lhs: cond); |
2950 | gimple_seq_add_stmt (&seq, g); |
2951 | tree lab1 = create_artificial_label (UNKNOWN_LOCATION); |
2952 | tree lab2 = create_artificial_label (UNKNOWN_LOCATION); |
2953 | tree lab3 = create_artificial_label (UNKNOWN_LOCATION); |
2954 | g = gimple_build_cond (NE_EXPR, cond, integer_zero_node, lab1, lab2); |
2955 | gimple_seq_add_stmt (&seq, g); |
2956 | g = gimple_build_label (label: lab1); |
2957 | gimple_seq_add_stmt (&seq, g); |
2958 | gimple_seq new_seq = copy_gimple_seq_and_replace_locals (seq: stmt); |
2959 | gomp_for *new_stmt = as_a <gomp_for *> (p: new_seq); |
2960 | tree clause = build_omp_clause (gimple_location (g: stmt), OMP_CLAUSE__SIMT_); |
2961 | OMP_CLAUSE_CHAIN (clause) = gimple_omp_for_clauses (gs: new_stmt); |
2962 | gimple_omp_for_set_clauses (gs: new_stmt, clauses: clause); |
2963 | gimple_seq_add_stmt (&seq, new_stmt); |
2964 | g = gimple_build_goto (dest: lab3); |
2965 | gimple_seq_add_stmt (&seq, g); |
2966 | g = gimple_build_label (label: lab2); |
2967 | gimple_seq_add_stmt (&seq, g); |
2968 | gimple_seq_add_stmt (&seq, stmt); |
2969 | g = gimple_build_label (label: lab3); |
2970 | gimple_seq_add_stmt (&seq, g); |
2971 | gimple_bind_set_body (bind_stmt: bind, seq); |
2972 | update_stmt (s: bind); |
2973 | scan_omp_for (stmt: new_stmt, outer_ctx); |
2974 | scan_omp_for (stmt, outer_ctx)->simt_stmt = new_stmt; |
2975 | } |
2976 | |
2977 | static tree omp_find_scan (gimple_stmt_iterator *, bool *, |
2978 | struct walk_stmt_info *); |
2979 | static omp_context *maybe_lookup_ctx (gimple *); |
2980 | |
2981 | /* Duplicate #pragma omp simd, one for the scan input phase loop and one |
2982 | for scan phase loop. */ |
2983 | |
2984 | static void |
2985 | scan_omp_simd_scan (gimple_stmt_iterator *gsi, gomp_for *stmt, |
2986 | omp_context *outer_ctx) |
2987 | { |
2988 | /* The only change between inclusive and exclusive scan will be |
2989 | within the first simd loop, so just use inclusive in the |
2990 | worksharing loop. */ |
2991 | outer_ctx->scan_inclusive = true; |
2992 | tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_INCLUSIVE); |
2993 | OMP_CLAUSE_DECL (c) = integer_zero_node; |
2994 | |
2995 | gomp_scan *input_stmt = gimple_build_omp_scan (NULL, NULL_TREE); |
2996 | gomp_scan *scan_stmt = gimple_build_omp_scan (NULL, c); |
2997 | gsi_replace (gsi, input_stmt, false); |
2998 | gimple_seq input_body = NULL; |
2999 | gimple_seq_add_stmt (&input_body, stmt); |
3000 | gsi_insert_after (gsi, scan_stmt, GSI_NEW_STMT); |
3001 | |
3002 | gimple_stmt_iterator input1_gsi = gsi_none (); |
3003 | struct walk_stmt_info wi; |
3004 | memset (s: &wi, c: 0, n: sizeof (wi)); |
3005 | wi.val_only = true; |
3006 | wi.info = (void *) &input1_gsi; |
3007 | walk_gimple_seq_mod (gimple_omp_body_ptr (gs: stmt), omp_find_scan, NULL, &wi); |
3008 | gcc_assert (!gsi_end_p (input1_gsi)); |
3009 | |
3010 | gimple *input_stmt1 = gsi_stmt (i: input1_gsi); |
3011 | gsi_next (i: &input1_gsi); |
3012 | gimple *scan_stmt1 = gsi_stmt (i: input1_gsi); |
3013 | gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN); |
3014 | c = gimple_omp_scan_clauses (scan_stmt: as_a <gomp_scan *> (p: scan_stmt1)); |
3015 | if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE) |
3016 | std::swap (a&: input_stmt1, b&: scan_stmt1); |
3017 | |
3018 | gimple_seq input_body1 = gimple_omp_body (gs: input_stmt1); |
3019 | gimple_omp_set_body (gs: input_stmt1, NULL); |
3020 | |
3021 | gimple_seq scan_body = copy_gimple_seq_and_replace_locals (seq: stmt); |
3022 | gomp_for *new_stmt = as_a <gomp_for *> (p: scan_body); |
3023 | |
3024 | gimple_omp_set_body (gs: input_stmt1, body: input_body1); |
3025 | gimple_omp_set_body (gs: scan_stmt1, NULL); |
3026 | |
3027 | gimple_stmt_iterator input2_gsi = gsi_none (); |
3028 | memset (s: &wi, c: 0, n: sizeof (wi)); |
3029 | wi.val_only = true; |
3030 | wi.info = (void *) &input2_gsi; |
3031 | walk_gimple_seq_mod (gimple_omp_body_ptr (gs: new_stmt), omp_find_scan, |
3032 | NULL, &wi); |
3033 | gcc_assert (!gsi_end_p (input2_gsi)); |
3034 | |
3035 | gimple *input_stmt2 = gsi_stmt (i: input2_gsi); |
3036 | gsi_next (i: &input2_gsi); |
3037 | gimple *scan_stmt2 = gsi_stmt (i: input2_gsi); |
3038 | gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN); |
3039 | if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE) |
3040 | std::swap (a&: input_stmt2, b&: scan_stmt2); |
3041 | |
3042 | gimple_omp_set_body (gs: input_stmt2, NULL); |
3043 | |
3044 | gimple_omp_set_body (gs: input_stmt, body: input_body); |
3045 | gimple_omp_set_body (gs: scan_stmt, body: scan_body); |
3046 | |
3047 | omp_context *ctx = new_omp_context (stmt: input_stmt, outer_ctx); |
3048 | scan_omp (gimple_omp_body_ptr (gs: input_stmt), ctx); |
3049 | |
3050 | ctx = new_omp_context (stmt: scan_stmt, outer_ctx); |
3051 | scan_omp (gimple_omp_body_ptr (gs: scan_stmt), ctx); |
3052 | |
3053 | maybe_lookup_ctx (new_stmt)->for_simd_scan_phase = true; |
3054 | } |
3055 | |
3056 | /* Scan an OpenMP sections directive. */ |
3057 | |
3058 | static void |
3059 | scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx) |
3060 | { |
3061 | omp_context *ctx; |
3062 | |
3063 | ctx = new_omp_context (stmt, outer_ctx); |
3064 | scan_sharing_clauses (clauses: gimple_omp_sections_clauses (gs: stmt), ctx); |
3065 | scan_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
3066 | } |
3067 | |
3068 | /* Scan an OpenMP single directive. */ |
3069 | |
3070 | static void |
3071 | scan_omp_single (gomp_single *stmt, omp_context *outer_ctx) |
3072 | { |
3073 | omp_context *ctx; |
3074 | tree name; |
3075 | |
3076 | ctx = new_omp_context (stmt, outer_ctx); |
3077 | ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
3078 | ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); |
3079 | name = create_tmp_var_name (".omp_copy_s" ); |
3080 | name = build_decl (gimple_location (g: stmt), |
3081 | TYPE_DECL, name, ctx->record_type); |
3082 | TYPE_NAME (ctx->record_type) = name; |
3083 | |
3084 | scan_sharing_clauses (clauses: gimple_omp_single_clauses (gs: stmt), ctx); |
3085 | scan_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
3086 | |
3087 | if (TYPE_FIELDS (ctx->record_type) == NULL) |
3088 | ctx->record_type = NULL; |
3089 | else |
3090 | layout_type (ctx->record_type); |
3091 | } |
3092 | |
3093 | /* Scan a GIMPLE_OMP_TARGET. */ |
3094 | |
3095 | static void |
3096 | scan_omp_target (gomp_target *stmt, omp_context *outer_ctx) |
3097 | { |
3098 | omp_context *ctx; |
3099 | tree name; |
3100 | bool offloaded = is_gimple_omp_offloaded (stmt); |
3101 | tree clauses = gimple_omp_target_clauses (gs: stmt); |
3102 | |
3103 | ctx = new_omp_context (stmt, outer_ctx); |
3104 | ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
3105 | ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); |
3106 | name = create_tmp_var_name (".omp_data_t" ); |
3107 | name = build_decl (gimple_location (g: stmt), |
3108 | TYPE_DECL, name, ctx->record_type); |
3109 | DECL_ARTIFICIAL (name) = 1; |
3110 | DECL_NAMELESS (name) = 1; |
3111 | TYPE_NAME (ctx->record_type) = name; |
3112 | TYPE_ARTIFICIAL (ctx->record_type) = 1; |
3113 | |
3114 | if (offloaded) |
3115 | { |
3116 | create_omp_child_function (ctx, task_copy: false); |
3117 | gimple_omp_target_set_child_fn (omp_target_stmt: stmt, child_fn: ctx->cb.dst_fn); |
3118 | } |
3119 | |
3120 | scan_sharing_clauses (clauses, ctx); |
3121 | scan_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
3122 | |
3123 | if (TYPE_FIELDS (ctx->record_type) == NULL) |
3124 | ctx->record_type = ctx->receiver_decl = NULL; |
3125 | else |
3126 | { |
3127 | TYPE_FIELDS (ctx->record_type) |
3128 | = nreverse (TYPE_FIELDS (ctx->record_type)); |
3129 | if (flag_checking) |
3130 | { |
3131 | unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type)); |
3132 | for (tree field = TYPE_FIELDS (ctx->record_type); |
3133 | field; |
3134 | field = DECL_CHAIN (field)) |
3135 | gcc_assert (DECL_ALIGN (field) == align); |
3136 | } |
3137 | layout_type (ctx->record_type); |
3138 | if (offloaded) |
3139 | fixup_child_record_type (ctx); |
3140 | } |
3141 | |
3142 | if (ctx->teams_nested_p && ctx->nonteams_nested_p) |
3143 | { |
3144 | error_at (gimple_location (g: stmt), |
3145 | "%<target%> construct with nested %<teams%> construct " |
3146 | "contains directives outside of the %<teams%> construct" ); |
3147 | gimple_omp_set_body (gs: stmt, body: gimple_build_bind (NULL, NULL, NULL)); |
3148 | } |
3149 | } |
3150 | |
3151 | /* Scan an OpenMP teams directive. */ |
3152 | |
3153 | static void |
3154 | scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx) |
3155 | { |
3156 | omp_context *ctx = new_omp_context (stmt, outer_ctx); |
3157 | |
3158 | if (!gimple_omp_teams_host (omp_teams_stmt: stmt)) |
3159 | { |
3160 | scan_sharing_clauses (clauses: gimple_omp_teams_clauses (gs: stmt), ctx); |
3161 | scan_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
3162 | return; |
3163 | } |
3164 | taskreg_contexts.safe_push (obj: ctx); |
3165 | gcc_assert (taskreg_nesting_level == 1); |
3166 | ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
3167 | ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); |
3168 | tree name = create_tmp_var_name (".omp_data_s" ); |
3169 | name = build_decl (gimple_location (g: stmt), |
3170 | TYPE_DECL, name, ctx->record_type); |
3171 | DECL_ARTIFICIAL (name) = 1; |
3172 | DECL_NAMELESS (name) = 1; |
3173 | TYPE_NAME (ctx->record_type) = name; |
3174 | TYPE_ARTIFICIAL (ctx->record_type) = 1; |
3175 | create_omp_child_function (ctx, task_copy: false); |
3176 | gimple_omp_teams_set_child_fn (omp_teams_stmt: stmt, child_fn: ctx->cb.dst_fn); |
3177 | |
3178 | scan_sharing_clauses (clauses: gimple_omp_teams_clauses (gs: stmt), ctx); |
3179 | scan_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
3180 | |
3181 | if (TYPE_FIELDS (ctx->record_type) == NULL) |
3182 | ctx->record_type = ctx->receiver_decl = NULL; |
3183 | } |
3184 | |
3185 | /* Check nesting restrictions. */ |
3186 | static bool |
3187 | check_omp_nesting_restrictions (gimple *stmt, omp_context *ctx) |
3188 | { |
3189 | tree c; |
3190 | |
3191 | /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin) |
3192 | inside an OpenACC CTX. */ |
3193 | if (gimple_code (g: stmt) == GIMPLE_OMP_ATOMIC_LOAD |
3194 | || gimple_code (g: stmt) == GIMPLE_OMP_ATOMIC_STORE) |
3195 | /* ..., except for the atomic codes that OpenACC shares with OpenMP. */ |
3196 | ; |
3197 | else if (!(is_gimple_omp (stmt) |
3198 | && is_gimple_omp_oacc (stmt))) |
3199 | { |
3200 | if (oacc_get_fn_attrib (cfun->decl) != NULL) |
3201 | { |
3202 | error_at (gimple_location (g: stmt), |
3203 | "non-OpenACC construct inside of OpenACC routine" ); |
3204 | return false; |
3205 | } |
3206 | else |
3207 | for (omp_context *octx = ctx; octx != NULL; octx = octx->outer) |
3208 | if (is_gimple_omp (stmt: octx->stmt) |
3209 | && is_gimple_omp_oacc (stmt: octx->stmt)) |
3210 | { |
3211 | error_at (gimple_location (g: stmt), |
3212 | "non-OpenACC construct inside of OpenACC region" ); |
3213 | return false; |
3214 | } |
3215 | } |
3216 | |
3217 | if (ctx != NULL) |
3218 | { |
3219 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_TARGET |
3220 | && gimple_omp_target_kind (g: ctx->stmt) == GF_OMP_TARGET_KIND_REGION) |
3221 | { |
3222 | c = omp_find_clause (clauses: gimple_omp_target_clauses (gs: ctx->stmt), |
3223 | kind: OMP_CLAUSE_DEVICE); |
3224 | if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c)) |
3225 | { |
3226 | error_at (gimple_location (g: stmt), |
3227 | "OpenMP constructs are not allowed in target region " |
3228 | "with %<ancestor%>" ); |
3229 | return false; |
3230 | } |
3231 | |
3232 | if (gimple_code (g: stmt) == GIMPLE_OMP_TEAMS && !ctx->teams_nested_p) |
3233 | ctx->teams_nested_p = true; |
3234 | else |
3235 | ctx->nonteams_nested_p = true; |
3236 | } |
3237 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_SCAN |
3238 | && ctx->outer |
3239 | && gimple_code (g: ctx->outer->stmt) == GIMPLE_OMP_FOR) |
3240 | ctx = ctx->outer; |
3241 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_FOR |
3242 | && gimple_omp_for_kind (g: ctx->stmt) == GF_OMP_FOR_KIND_SIMD |
3243 | && !ctx->loop_p) |
3244 | { |
3245 | c = NULL_TREE; |
3246 | if (ctx->order_concurrent |
3247 | && (gimple_code (g: stmt) == GIMPLE_OMP_ORDERED |
3248 | || gimple_code (g: stmt) == GIMPLE_OMP_ATOMIC_LOAD |
3249 | || gimple_code (g: stmt) == GIMPLE_OMP_ATOMIC_STORE)) |
3250 | { |
3251 | error_at (gimple_location (g: stmt), |
3252 | "OpenMP constructs other than %<parallel%>, %<loop%>" |
3253 | " or %<simd%> may not be nested inside a region with" |
3254 | " the %<order(concurrent)%> clause" ); |
3255 | return false; |
3256 | } |
3257 | if (gimple_code (g: stmt) == GIMPLE_OMP_ORDERED) |
3258 | { |
3259 | c = gimple_omp_ordered_clauses (ord_stmt: as_a <gomp_ordered *> (p: stmt)); |
3260 | if (omp_find_clause (clauses: c, kind: OMP_CLAUSE_SIMD)) |
3261 | { |
3262 | if (omp_find_clause (clauses: c, kind: OMP_CLAUSE_THREADS) |
3263 | && (ctx->outer == NULL |
3264 | || !gimple_omp_for_combined_into_p (g: ctx->stmt) |
3265 | || gimple_code (g: ctx->outer->stmt) != GIMPLE_OMP_FOR |
3266 | || (gimple_omp_for_kind (g: ctx->outer->stmt) |
3267 | != GF_OMP_FOR_KIND_FOR) |
3268 | || !gimple_omp_for_combined_p (g: ctx->outer->stmt))) |
3269 | { |
3270 | error_at (gimple_location (g: stmt), |
3271 | "%<ordered simd threads%> must be closely " |
3272 | "nested inside of %<%s simd%> region" , |
3273 | lang_GNU_Fortran () ? "do" : "for" ); |
3274 | return false; |
3275 | } |
3276 | return true; |
3277 | } |
3278 | } |
3279 | else if (gimple_code (g: stmt) == GIMPLE_OMP_ATOMIC_LOAD |
3280 | || gimple_code (g: stmt) == GIMPLE_OMP_ATOMIC_STORE |
3281 | || gimple_code (g: stmt) == GIMPLE_OMP_SCAN) |
3282 | return true; |
3283 | else if (gimple_code (g: stmt) == GIMPLE_OMP_FOR |
3284 | && gimple_omp_for_kind (g: ctx->stmt) == GF_OMP_FOR_KIND_SIMD) |
3285 | return true; |
3286 | error_at (gimple_location (g: stmt), |
3287 | "OpenMP constructs other than " |
3288 | "%<ordered simd%>, %<simd%>, %<loop%> or %<atomic%> may " |
3289 | "not be nested inside %<simd%> region" ); |
3290 | return false; |
3291 | } |
3292 | else if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_TEAMS) |
3293 | { |
3294 | if ((gimple_code (g: stmt) != GIMPLE_OMP_FOR |
3295 | || (gimple_omp_for_kind (g: stmt) != GF_OMP_FOR_KIND_DISTRIBUTE |
3296 | && omp_find_clause (clauses: gimple_omp_for_clauses (gs: stmt), |
3297 | kind: OMP_CLAUSE_BIND) == NULL_TREE)) |
3298 | && gimple_code (g: stmt) != GIMPLE_OMP_PARALLEL) |
3299 | { |
3300 | error_at (gimple_location (g: stmt), |
3301 | "only %<distribute%>, %<parallel%> or %<loop%> " |
3302 | "regions are allowed to be strictly nested inside " |
3303 | "%<teams%> region" ); |
3304 | return false; |
3305 | } |
3306 | } |
3307 | else if (ctx->order_concurrent |
3308 | && gimple_code (g: stmt) != GIMPLE_OMP_PARALLEL |
3309 | && (gimple_code (g: stmt) != GIMPLE_OMP_FOR |
3310 | || gimple_omp_for_kind (g: stmt) != GF_OMP_FOR_KIND_SIMD) |
3311 | && gimple_code (g: stmt) != GIMPLE_OMP_SCAN) |
3312 | { |
3313 | if (ctx->loop_p) |
3314 | error_at (gimple_location (g: stmt), |
3315 | "OpenMP constructs other than %<parallel%>, %<loop%> or " |
3316 | "%<simd%> may not be nested inside a %<loop%> region" ); |
3317 | else |
3318 | error_at (gimple_location (g: stmt), |
3319 | "OpenMP constructs other than %<parallel%>, %<loop%> or " |
3320 | "%<simd%> may not be nested inside a region with " |
3321 | "the %<order(concurrent)%> clause" ); |
3322 | return false; |
3323 | } |
3324 | } |
3325 | switch (gimple_code (g: stmt)) |
3326 | { |
3327 | case GIMPLE_OMP_FOR: |
3328 | if (gimple_omp_for_kind (g: stmt) == GF_OMP_FOR_KIND_SIMD) |
3329 | return true; |
3330 | if (gimple_omp_for_kind (g: stmt) == GF_OMP_FOR_KIND_DISTRIBUTE) |
3331 | { |
3332 | if (ctx != NULL && gimple_code (g: ctx->stmt) != GIMPLE_OMP_TEAMS) |
3333 | { |
3334 | error_at (gimple_location (g: stmt), |
3335 | "%<distribute%> region must be strictly nested " |
3336 | "inside %<teams%> construct" ); |
3337 | return false; |
3338 | } |
3339 | return true; |
3340 | } |
3341 | /* We split taskloop into task and nested taskloop in it. */ |
3342 | if (gimple_omp_for_kind (g: stmt) == GF_OMP_FOR_KIND_TASKLOOP) |
3343 | return true; |
3344 | /* For now, hope this will change and loop bind(parallel) will not |
3345 | be allowed in lots of contexts. */ |
3346 | if (gimple_omp_for_kind (g: stmt) == GF_OMP_FOR_KIND_FOR |
3347 | && omp_find_clause (clauses: gimple_omp_for_clauses (gs: stmt), kind: OMP_CLAUSE_BIND)) |
3348 | return true; |
3349 | if (gimple_omp_for_kind (g: stmt) == GF_OMP_FOR_KIND_OACC_LOOP) |
3350 | { |
3351 | bool ok = false; |
3352 | |
3353 | if (ctx) |
3354 | switch (gimple_code (g: ctx->stmt)) |
3355 | { |
3356 | case GIMPLE_OMP_FOR: |
3357 | ok = (gimple_omp_for_kind (g: ctx->stmt) |
3358 | == GF_OMP_FOR_KIND_OACC_LOOP); |
3359 | break; |
3360 | |
3361 | case GIMPLE_OMP_TARGET: |
3362 | switch (gimple_omp_target_kind (g: ctx->stmt)) |
3363 | { |
3364 | case GF_OMP_TARGET_KIND_OACC_PARALLEL: |
3365 | case GF_OMP_TARGET_KIND_OACC_KERNELS: |
3366 | case GF_OMP_TARGET_KIND_OACC_SERIAL: |
3367 | case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED: |
3368 | case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE: |
3369 | ok = true; |
3370 | break; |
3371 | |
3372 | default: |
3373 | break; |
3374 | } |
3375 | |
3376 | default: |
3377 | break; |
3378 | } |
3379 | else if (oacc_get_fn_attrib (fn: current_function_decl)) |
3380 | ok = true; |
3381 | if (!ok) |
3382 | { |
3383 | error_at (gimple_location (g: stmt), |
3384 | "OpenACC loop directive must be associated with" |
3385 | " an OpenACC compute region" ); |
3386 | return false; |
3387 | } |
3388 | } |
3389 | /* FALLTHRU */ |
3390 | case GIMPLE_CALL: |
3391 | if (is_gimple_call (gs: stmt) |
3392 | && (DECL_FUNCTION_CODE (decl: gimple_call_fndecl (gs: stmt)) |
3393 | == BUILT_IN_GOMP_CANCEL |
3394 | || DECL_FUNCTION_CODE (decl: gimple_call_fndecl (gs: stmt)) |
3395 | == BUILT_IN_GOMP_CANCELLATION_POINT)) |
3396 | { |
3397 | const char *bad = NULL; |
3398 | const char *kind = NULL; |
3399 | const char *construct |
3400 | = (DECL_FUNCTION_CODE (decl: gimple_call_fndecl (gs: stmt)) |
3401 | == BUILT_IN_GOMP_CANCEL) |
3402 | ? "cancel" |
3403 | : "cancellation point" ; |
3404 | if (ctx == NULL) |
3405 | { |
3406 | error_at (gimple_location (g: stmt), "orphaned %qs construct" , |
3407 | construct); |
3408 | return false; |
3409 | } |
3410 | switch (tree_fits_shwi_p (gimple_call_arg (gs: stmt, index: 0)) |
3411 | ? tree_to_shwi (gimple_call_arg (gs: stmt, index: 0)) |
3412 | : 0) |
3413 | { |
3414 | case 1: |
3415 | if (gimple_code (g: ctx->stmt) != GIMPLE_OMP_PARALLEL) |
3416 | bad = "parallel" ; |
3417 | else if (DECL_FUNCTION_CODE (decl: gimple_call_fndecl (gs: stmt)) |
3418 | == BUILT_IN_GOMP_CANCEL |
3419 | && !integer_zerop (gimple_call_arg (gs: stmt, index: 1))) |
3420 | ctx->cancellable = true; |
3421 | kind = "parallel" ; |
3422 | break; |
3423 | case 2: |
3424 | if (gimple_code (g: ctx->stmt) != GIMPLE_OMP_FOR |
3425 | || gimple_omp_for_kind (g: ctx->stmt) != GF_OMP_FOR_KIND_FOR) |
3426 | bad = "for" ; |
3427 | else if (DECL_FUNCTION_CODE (decl: gimple_call_fndecl (gs: stmt)) |
3428 | == BUILT_IN_GOMP_CANCEL |
3429 | && !integer_zerop (gimple_call_arg (gs: stmt, index: 1))) |
3430 | { |
3431 | ctx->cancellable = true; |
3432 | if (omp_find_clause (clauses: gimple_omp_for_clauses (gs: ctx->stmt), |
3433 | kind: OMP_CLAUSE_NOWAIT)) |
3434 | warning_at (gimple_location (g: stmt), OPT_Wopenmp, |
3435 | "%<cancel for%> inside " |
3436 | "%<nowait%> for construct" ); |
3437 | if (omp_find_clause (clauses: gimple_omp_for_clauses (gs: ctx->stmt), |
3438 | kind: OMP_CLAUSE_ORDERED)) |
3439 | warning_at (gimple_location (g: stmt), OPT_Wopenmp, |
3440 | "%<cancel for%> inside " |
3441 | "%<ordered%> for construct" ); |
3442 | } |
3443 | kind = "for" ; |
3444 | break; |
3445 | case 4: |
3446 | if (gimple_code (g: ctx->stmt) != GIMPLE_OMP_SECTIONS |
3447 | && gimple_code (g: ctx->stmt) != GIMPLE_OMP_SECTION) |
3448 | bad = "sections" ; |
3449 | else if (DECL_FUNCTION_CODE (decl: gimple_call_fndecl (gs: stmt)) |
3450 | == BUILT_IN_GOMP_CANCEL |
3451 | && !integer_zerop (gimple_call_arg (gs: stmt, index: 1))) |
3452 | { |
3453 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_SECTIONS) |
3454 | { |
3455 | ctx->cancellable = true; |
3456 | if (omp_find_clause (clauses: gimple_omp_sections_clauses |
3457 | (gs: ctx->stmt), |
3458 | kind: OMP_CLAUSE_NOWAIT)) |
3459 | warning_at (gimple_location (g: stmt), OPT_Wopenmp, |
3460 | "%<cancel sections%> inside " |
3461 | "%<nowait%> sections construct" ); |
3462 | } |
3463 | else |
3464 | { |
3465 | gcc_assert (ctx->outer |
3466 | && gimple_code (ctx->outer->stmt) |
3467 | == GIMPLE_OMP_SECTIONS); |
3468 | ctx->outer->cancellable = true; |
3469 | if (omp_find_clause (clauses: gimple_omp_sections_clauses |
3470 | (gs: ctx->outer->stmt), |
3471 | kind: OMP_CLAUSE_NOWAIT)) |
3472 | warning_at (gimple_location (g: stmt), OPT_Wopenmp, |
3473 | "%<cancel sections%> inside " |
3474 | "%<nowait%> sections construct" ); |
3475 | } |
3476 | } |
3477 | kind = "sections" ; |
3478 | break; |
3479 | case 8: |
3480 | if (!is_task_ctx (ctx) |
3481 | && (!is_taskloop_ctx (ctx) |
3482 | || ctx->outer == NULL |
3483 | || !is_task_ctx (ctx: ctx->outer))) |
3484 | bad = "task" ; |
3485 | else |
3486 | { |
3487 | for (omp_context *octx = ctx->outer; |
3488 | octx; octx = octx->outer) |
3489 | { |
3490 | switch (gimple_code (g: octx->stmt)) |
3491 | { |
3492 | case GIMPLE_OMP_TASKGROUP: |
3493 | break; |
3494 | case GIMPLE_OMP_TARGET: |
3495 | if (gimple_omp_target_kind (g: octx->stmt) |
3496 | != GF_OMP_TARGET_KIND_REGION) |
3497 | continue; |
3498 | /* FALLTHRU */ |
3499 | case GIMPLE_OMP_PARALLEL: |
3500 | case GIMPLE_OMP_TEAMS: |
3501 | error_at (gimple_location (g: stmt), |
3502 | "%<%s taskgroup%> construct not closely " |
3503 | "nested inside of %<taskgroup%> region" , |
3504 | construct); |
3505 | return false; |
3506 | case GIMPLE_OMP_TASK: |
3507 | if (gimple_omp_task_taskloop_p (g: octx->stmt) |
3508 | && octx->outer |
3509 | && is_taskloop_ctx (ctx: octx->outer)) |
3510 | { |
3511 | tree clauses |
3512 | = gimple_omp_for_clauses (gs: octx->outer->stmt); |
3513 | if (!omp_find_clause (clauses, kind: OMP_CLAUSE_NOGROUP)) |
3514 | break; |
3515 | } |
3516 | continue; |
3517 | default: |
3518 | continue; |
3519 | } |
3520 | break; |
3521 | } |
3522 | ctx->cancellable = true; |
3523 | } |
3524 | kind = "taskgroup" ; |
3525 | break; |
3526 | default: |
3527 | error_at (gimple_location (g: stmt), "invalid arguments" ); |
3528 | return false; |
3529 | } |
3530 | if (bad) |
3531 | { |
3532 | error_at (gimple_location (g: stmt), |
3533 | "%<%s %s%> construct not closely nested inside of %qs" , |
3534 | construct, kind, bad); |
3535 | return false; |
3536 | } |
3537 | } |
3538 | /* FALLTHRU */ |
3539 | case GIMPLE_OMP_SECTIONS: |
3540 | case GIMPLE_OMP_SINGLE: |
3541 | for (; ctx != NULL; ctx = ctx->outer) |
3542 | switch (gimple_code (g: ctx->stmt)) |
3543 | { |
3544 | case GIMPLE_OMP_FOR: |
3545 | if (gimple_omp_for_kind (g: ctx->stmt) != GF_OMP_FOR_KIND_FOR |
3546 | && gimple_omp_for_kind (g: ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP) |
3547 | break; |
3548 | /* FALLTHRU */ |
3549 | case GIMPLE_OMP_SECTIONS: |
3550 | case GIMPLE_OMP_SINGLE: |
3551 | case GIMPLE_OMP_ORDERED: |
3552 | case GIMPLE_OMP_MASTER: |
3553 | case GIMPLE_OMP_MASKED: |
3554 | case GIMPLE_OMP_TASK: |
3555 | case GIMPLE_OMP_CRITICAL: |
3556 | if (is_gimple_call (gs: stmt)) |
3557 | { |
3558 | if (DECL_FUNCTION_CODE (decl: gimple_call_fndecl (gs: stmt)) |
3559 | != BUILT_IN_GOMP_BARRIER) |
3560 | return true; |
3561 | error_at (gimple_location (g: stmt), |
3562 | "barrier region may not be closely nested inside " |
3563 | "of work-sharing, %<loop%>, %<critical%>, " |
3564 | "%<ordered%>, %<master%>, %<masked%>, explicit " |
3565 | "%<task%> or %<taskloop%> region" ); |
3566 | return false; |
3567 | } |
3568 | error_at (gimple_location (g: stmt), |
3569 | "work-sharing region may not be closely nested inside " |
3570 | "of work-sharing, %<loop%>, %<critical%>, %<ordered%>, " |
3571 | "%<master%>, %<masked%>, explicit %<task%> or " |
3572 | "%<taskloop%> region" ); |
3573 | return false; |
3574 | case GIMPLE_OMP_PARALLEL: |
3575 | case GIMPLE_OMP_TEAMS: |
3576 | return true; |
3577 | case GIMPLE_OMP_TARGET: |
3578 | if (gimple_omp_target_kind (g: ctx->stmt) |
3579 | == GF_OMP_TARGET_KIND_REGION) |
3580 | return true; |
3581 | break; |
3582 | default: |
3583 | break; |
3584 | } |
3585 | break; |
3586 | case GIMPLE_OMP_MASTER: |
3587 | case GIMPLE_OMP_MASKED: |
3588 | for (; ctx != NULL; ctx = ctx->outer) |
3589 | switch (gimple_code (g: ctx->stmt)) |
3590 | { |
3591 | case GIMPLE_OMP_FOR: |
3592 | if (gimple_omp_for_kind (g: ctx->stmt) != GF_OMP_FOR_KIND_FOR |
3593 | && gimple_omp_for_kind (g: ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP) |
3594 | break; |
3595 | /* FALLTHRU */ |
3596 | case GIMPLE_OMP_SECTIONS: |
3597 | case GIMPLE_OMP_SINGLE: |
3598 | case GIMPLE_OMP_TASK: |
3599 | error_at (gimple_location (g: stmt), |
3600 | "%qs region may not be closely nested inside " |
3601 | "of work-sharing, %<loop%>, explicit %<task%> or " |
3602 | "%<taskloop%> region" , |
3603 | gimple_code (g: stmt) == GIMPLE_OMP_MASTER |
3604 | ? "master" : "masked" ); |
3605 | return false; |
3606 | case GIMPLE_OMP_PARALLEL: |
3607 | case GIMPLE_OMP_TEAMS: |
3608 | return true; |
3609 | case GIMPLE_OMP_TARGET: |
3610 | if (gimple_omp_target_kind (g: ctx->stmt) |
3611 | == GF_OMP_TARGET_KIND_REGION) |
3612 | return true; |
3613 | break; |
3614 | default: |
3615 | break; |
3616 | } |
3617 | break; |
3618 | case GIMPLE_OMP_SCOPE: |
3619 | for (; ctx != NULL; ctx = ctx->outer) |
3620 | switch (gimple_code (g: ctx->stmt)) |
3621 | { |
3622 | case GIMPLE_OMP_FOR: |
3623 | if (gimple_omp_for_kind (g: ctx->stmt) != GF_OMP_FOR_KIND_FOR |
3624 | && gimple_omp_for_kind (g: ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP) |
3625 | break; |
3626 | /* FALLTHRU */ |
3627 | case GIMPLE_OMP_SECTIONS: |
3628 | case GIMPLE_OMP_SINGLE: |
3629 | case GIMPLE_OMP_TASK: |
3630 | case GIMPLE_OMP_CRITICAL: |
3631 | case GIMPLE_OMP_ORDERED: |
3632 | case GIMPLE_OMP_MASTER: |
3633 | case GIMPLE_OMP_MASKED: |
3634 | error_at (gimple_location (g: stmt), |
3635 | "%<scope%> region may not be closely nested inside " |
3636 | "of work-sharing, %<loop%>, explicit %<task%>, " |
3637 | "%<taskloop%>, %<critical%>, %<ordered%>, %<master%>, " |
3638 | "or %<masked%> region" ); |
3639 | return false; |
3640 | case GIMPLE_OMP_PARALLEL: |
3641 | case GIMPLE_OMP_TEAMS: |
3642 | return true; |
3643 | case GIMPLE_OMP_TARGET: |
3644 | if (gimple_omp_target_kind (g: ctx->stmt) |
3645 | == GF_OMP_TARGET_KIND_REGION) |
3646 | return true; |
3647 | break; |
3648 | default: |
3649 | break; |
3650 | } |
3651 | break; |
3652 | case GIMPLE_OMP_TASK: |
3653 | for (c = gimple_omp_task_clauses (gs: stmt); c; c = OMP_CLAUSE_CHAIN (c)) |
3654 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS) |
3655 | { |
3656 | enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c); |
3657 | error_at (OMP_CLAUSE_LOCATION (c), |
3658 | "%<%s(%s)%> is only allowed in %<omp ordered%>" , |
3659 | OMP_CLAUSE_DOACROSS_DEPEND (c) ? "depend" : "doacross" , |
3660 | kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink" ); |
3661 | return false; |
3662 | } |
3663 | break; |
3664 | case GIMPLE_OMP_ORDERED: |
3665 | for (c = gimple_omp_ordered_clauses (ord_stmt: as_a <gomp_ordered *> (p: stmt)); |
3666 | c; c = OMP_CLAUSE_CHAIN (c)) |
3667 | { |
3668 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS) |
3669 | { |
3670 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND) |
3671 | { |
3672 | error_at (OMP_CLAUSE_LOCATION (c), |
3673 | "invalid depend kind in omp %<ordered%> %<depend%>" ); |
3674 | return false; |
3675 | } |
3676 | gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREADS |
3677 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SIMD); |
3678 | continue; |
3679 | } |
3680 | |
3681 | tree oclause; |
3682 | /* Look for containing ordered(N) loop. */ |
3683 | if (ctx == NULL |
3684 | || gimple_code (g: ctx->stmt) != GIMPLE_OMP_FOR |
3685 | || (oclause |
3686 | = omp_find_clause (clauses: gimple_omp_for_clauses (gs: ctx->stmt), |
3687 | kind: OMP_CLAUSE_ORDERED)) == NULL_TREE) |
3688 | { |
3689 | error_at (OMP_CLAUSE_LOCATION (c), |
3690 | "%<ordered%> construct with %<depend%> clause " |
3691 | "must be closely nested inside an %<ordered%> loop" ); |
3692 | return false; |
3693 | } |
3694 | } |
3695 | c = gimple_omp_ordered_clauses (ord_stmt: as_a <gomp_ordered *> (p: stmt)); |
3696 | if (omp_find_clause (clauses: c, kind: OMP_CLAUSE_SIMD)) |
3697 | { |
3698 | /* ordered simd must be closely nested inside of simd region, |
3699 | and simd region must not encounter constructs other than |
3700 | ordered simd, therefore ordered simd may be either orphaned, |
3701 | or ctx->stmt must be simd. The latter case is handled already |
3702 | earlier. */ |
3703 | if (ctx != NULL) |
3704 | { |
3705 | error_at (gimple_location (g: stmt), |
3706 | "%<ordered%> %<simd%> must be closely nested inside " |
3707 | "%<simd%> region" ); |
3708 | return false; |
3709 | } |
3710 | } |
3711 | for (; ctx != NULL; ctx = ctx->outer) |
3712 | switch (gimple_code (g: ctx->stmt)) |
3713 | { |
3714 | case GIMPLE_OMP_CRITICAL: |
3715 | case GIMPLE_OMP_TASK: |
3716 | case GIMPLE_OMP_ORDERED: |
3717 | ordered_in_taskloop: |
3718 | error_at (gimple_location (g: stmt), |
3719 | "%<ordered%> region may not be closely nested inside " |
3720 | "of %<critical%>, %<ordered%>, explicit %<task%> or " |
3721 | "%<taskloop%> region" ); |
3722 | return false; |
3723 | case GIMPLE_OMP_FOR: |
3724 | if (gimple_omp_for_kind (g: ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP) |
3725 | goto ordered_in_taskloop; |
3726 | tree o; |
3727 | o = omp_find_clause (clauses: gimple_omp_for_clauses (gs: ctx->stmt), |
3728 | kind: OMP_CLAUSE_ORDERED); |
3729 | if (o == NULL) |
3730 | { |
3731 | error_at (gimple_location (g: stmt), |
3732 | "%<ordered%> region must be closely nested inside " |
3733 | "a loop region with an %<ordered%> clause" ); |
3734 | return false; |
3735 | } |
3736 | if (!gimple_omp_ordered_standalone_p (g: stmt)) |
3737 | { |
3738 | if (OMP_CLAUSE_ORDERED_DOACROSS (o)) |
3739 | { |
3740 | error_at (gimple_location (g: stmt), |
3741 | "%<ordered%> construct without %<doacross%> or " |
3742 | "%<depend%> clauses must not have the same " |
3743 | "binding region as %<ordered%> construct with " |
3744 | "those clauses" ); |
3745 | return false; |
3746 | } |
3747 | else if (OMP_CLAUSE_ORDERED_EXPR (o)) |
3748 | { |
3749 | tree co |
3750 | = omp_find_clause (clauses: gimple_omp_for_clauses (gs: ctx->stmt), |
3751 | kind: OMP_CLAUSE_COLLAPSE); |
3752 | HOST_WIDE_INT |
3753 | o_n = tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (o)); |
3754 | HOST_WIDE_INT c_n = 1; |
3755 | if (co) |
3756 | c_n = tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (co)); |
3757 | if (o_n != c_n) |
3758 | { |
3759 | error_at (gimple_location (g: stmt), |
3760 | "%<ordered%> construct without %<doacross%> " |
3761 | "or %<depend%> clauses binds to loop where " |
3762 | "%<collapse%> argument %wd is different from " |
3763 | "%<ordered%> argument %wd" , c_n, o_n); |
3764 | return false; |
3765 | } |
3766 | } |
3767 | } |
3768 | return true; |
3769 | case GIMPLE_OMP_TARGET: |
3770 | if (gimple_omp_target_kind (g: ctx->stmt) |
3771 | != GF_OMP_TARGET_KIND_REGION) |
3772 | break; |
3773 | /* FALLTHRU */ |
3774 | case GIMPLE_OMP_PARALLEL: |
3775 | case GIMPLE_OMP_TEAMS: |
3776 | error_at (gimple_location (g: stmt), |
3777 | "%<ordered%> region must be closely nested inside " |
3778 | "a loop region with an %<ordered%> clause" ); |
3779 | return false; |
3780 | default: |
3781 | break; |
3782 | } |
3783 | break; |
3784 | case GIMPLE_OMP_CRITICAL: |
3785 | { |
3786 | tree this_stmt_name |
3787 | = gimple_omp_critical_name (crit_stmt: as_a <gomp_critical *> (p: stmt)); |
3788 | for (; ctx != NULL; ctx = ctx->outer) |
3789 | if (gomp_critical *other_crit |
3790 | = dyn_cast <gomp_critical *> (p: ctx->stmt)) |
3791 | if (this_stmt_name == gimple_omp_critical_name (crit_stmt: other_crit)) |
3792 | { |
3793 | error_at (gimple_location (g: stmt), |
3794 | "%<critical%> region may not be nested inside " |
3795 | "a %<critical%> region with the same name" ); |
3796 | return false; |
3797 | } |
3798 | } |
3799 | break; |
3800 | case GIMPLE_OMP_TEAMS: |
3801 | if (ctx == NULL) |
3802 | break; |
3803 | else if (gimple_code (g: ctx->stmt) != GIMPLE_OMP_TARGET |
3804 | || (gimple_omp_target_kind (g: ctx->stmt) |
3805 | != GF_OMP_TARGET_KIND_REGION)) |
3806 | { |
3807 | /* Teams construct can appear either strictly nested inside of |
3808 | target construct with no intervening stmts, or can be encountered |
3809 | only by initial task (so must not appear inside any OpenMP |
3810 | construct. */ |
3811 | error_at (gimple_location (g: stmt), |
3812 | "%<teams%> construct must be closely nested inside of " |
3813 | "%<target%> construct or not nested in any OpenMP " |
3814 | "construct" ); |
3815 | return false; |
3816 | } |
3817 | break; |
3818 | case GIMPLE_OMP_TARGET: |
3819 | for (c = gimple_omp_target_clauses (gs: stmt); c; c = OMP_CLAUSE_CHAIN (c)) |
3820 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS) |
3821 | { |
3822 | enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c); |
3823 | error_at (OMP_CLAUSE_LOCATION (c), |
3824 | "%<depend(%s)%> is only allowed in %<omp ordered%>" , |
3825 | kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink" ); |
3826 | return false; |
3827 | } |
3828 | if (is_gimple_omp_offloaded (stmt) |
3829 | && oacc_get_fn_attrib (cfun->decl) != NULL) |
3830 | { |
3831 | error_at (gimple_location (g: stmt), |
3832 | "OpenACC region inside of OpenACC routine, nested " |
3833 | "parallelism not supported yet" ); |
3834 | return false; |
3835 | } |
3836 | for (; ctx != NULL; ctx = ctx->outer) |
3837 | { |
3838 | if (gimple_code (g: ctx->stmt) != GIMPLE_OMP_TARGET) |
3839 | { |
3840 | if (is_gimple_omp (stmt) |
3841 | && is_gimple_omp_oacc (stmt) |
3842 | && is_gimple_omp (stmt: ctx->stmt)) |
3843 | { |
3844 | error_at (gimple_location (g: stmt), |
3845 | "OpenACC construct inside of non-OpenACC region" ); |
3846 | return false; |
3847 | } |
3848 | continue; |
3849 | } |
3850 | |
3851 | const char *stmt_name, *ctx_stmt_name; |
3852 | switch (gimple_omp_target_kind (g: stmt)) |
3853 | { |
3854 | case GF_OMP_TARGET_KIND_REGION: stmt_name = "target" ; break; |
3855 | case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data" ; break; |
3856 | case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update" ; break; |
3857 | case GF_OMP_TARGET_KIND_ENTER_DATA: |
3858 | stmt_name = "target enter data" ; break; |
3859 | case GF_OMP_TARGET_KIND_EXIT_DATA: |
3860 | stmt_name = "target exit data" ; break; |
3861 | case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel" ; break; |
3862 | case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels" ; break; |
3863 | case GF_OMP_TARGET_KIND_OACC_SERIAL: stmt_name = "serial" ; break; |
3864 | case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data" ; break; |
3865 | case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update" ; break; |
3866 | case GF_OMP_TARGET_KIND_OACC_ENTER_DATA: |
3867 | stmt_name = "enter data" ; break; |
3868 | case GF_OMP_TARGET_KIND_OACC_EXIT_DATA: |
3869 | stmt_name = "exit data" ; break; |
3870 | case GF_OMP_TARGET_KIND_OACC_DECLARE: stmt_name = "declare" ; break; |
3871 | case GF_OMP_TARGET_KIND_OACC_HOST_DATA: stmt_name = "host_data" ; |
3872 | break; |
3873 | case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED: |
3874 | case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE: |
3875 | case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS: |
3876 | /* OpenACC 'kernels' decomposed parts. */ |
3877 | stmt_name = "kernels" ; break; |
3878 | default: gcc_unreachable (); |
3879 | } |
3880 | switch (gimple_omp_target_kind (g: ctx->stmt)) |
3881 | { |
3882 | case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target" ; break; |
3883 | case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data" ; break; |
3884 | case GF_OMP_TARGET_KIND_OACC_PARALLEL: |
3885 | ctx_stmt_name = "parallel" ; break; |
3886 | case GF_OMP_TARGET_KIND_OACC_KERNELS: |
3887 | ctx_stmt_name = "kernels" ; break; |
3888 | case GF_OMP_TARGET_KIND_OACC_SERIAL: |
3889 | ctx_stmt_name = "serial" ; break; |
3890 | case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data" ; break; |
3891 | case GF_OMP_TARGET_KIND_OACC_HOST_DATA: |
3892 | ctx_stmt_name = "host_data" ; break; |
3893 | case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED: |
3894 | case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE: |
3895 | case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS: |
3896 | /* OpenACC 'kernels' decomposed parts. */ |
3897 | ctx_stmt_name = "kernels" ; break; |
3898 | default: gcc_unreachable (); |
3899 | } |
3900 | |
3901 | /* OpenACC/OpenMP mismatch? */ |
3902 | if (is_gimple_omp_oacc (stmt) |
3903 | != is_gimple_omp_oacc (stmt: ctx->stmt)) |
3904 | { |
3905 | error_at (gimple_location (g: stmt), |
3906 | "%s %qs construct inside of %s %qs region" , |
3907 | (is_gimple_omp_oacc (stmt) |
3908 | ? "OpenACC" : "OpenMP" ), stmt_name, |
3909 | (is_gimple_omp_oacc (stmt: ctx->stmt) |
3910 | ? "OpenACC" : "OpenMP" ), ctx_stmt_name); |
3911 | return false; |
3912 | } |
3913 | if (is_gimple_omp_offloaded (stmt: ctx->stmt)) |
3914 | { |
3915 | /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */ |
3916 | if (is_gimple_omp_oacc (stmt: ctx->stmt)) |
3917 | { |
3918 | error_at (gimple_location (g: stmt), |
3919 | "%qs construct inside of %qs region" , |
3920 | stmt_name, ctx_stmt_name); |
3921 | return false; |
3922 | } |
3923 | else |
3924 | { |
3925 | if ((gimple_omp_target_kind (g: ctx->stmt) |
3926 | == GF_OMP_TARGET_KIND_REGION) |
3927 | && (gimple_omp_target_kind (g: stmt) |
3928 | == GF_OMP_TARGET_KIND_REGION)) |
3929 | { |
3930 | c = omp_find_clause (clauses: gimple_omp_target_clauses (gs: stmt), |
3931 | kind: OMP_CLAUSE_DEVICE); |
3932 | if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c)) |
3933 | break; |
3934 | } |
3935 | warning_at (gimple_location (g: stmt), OPT_Wopenmp, |
3936 | "%qs construct inside of %qs region" , |
3937 | stmt_name, ctx_stmt_name); |
3938 | } |
3939 | } |
3940 | } |
3941 | break; |
3942 | default: |
3943 | break; |
3944 | } |
3945 | return true; |
3946 | } |
3947 | |
3948 | |
3949 | /* Helper function scan_omp. |
3950 | |
3951 | Callback for walk_tree or operators in walk_gimple_stmt used to |
3952 | scan for OMP directives in TP. */ |
3953 | |
3954 | static tree |
3955 | scan_omp_1_op (tree *tp, int *walk_subtrees, void *data) |
3956 | { |
3957 | struct walk_stmt_info *wi = (struct walk_stmt_info *) data; |
3958 | omp_context *ctx = (omp_context *) wi->info; |
3959 | tree t = *tp; |
3960 | tree tmp; |
3961 | |
3962 | switch (TREE_CODE (t)) |
3963 | { |
3964 | case VAR_DECL: |
3965 | case PARM_DECL: |
3966 | case LABEL_DECL: |
3967 | case RESULT_DECL: |
3968 | if (ctx) |
3969 | { |
3970 | tmp = NULL_TREE; |
3971 | if (TREE_CODE (t) == VAR_DECL |
3972 | && (tmp = lookup_attribute (attr_name: "omp allocate var" , |
3973 | DECL_ATTRIBUTES (t))) != NULL_TREE) |
3974 | t = TREE_VALUE (TREE_VALUE (tmp)); |
3975 | tree repl = remap_decl (decl: t, id: &ctx->cb); |
3976 | gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK); |
3977 | if (tmp != NULL_TREE && t != repl) |
3978 | *tp = build_fold_addr_expr (repl); |
3979 | else if (tmp == NULL_TREE) |
3980 | *tp = repl; |
3981 | } |
3982 | break; |
3983 | |
3984 | case INDIRECT_REF: |
3985 | case MEM_REF: |
3986 | if (ctx |
3987 | && TREE_CODE (TREE_OPERAND (t, 0)) == VAR_DECL |
3988 | && ((tmp = lookup_attribute (attr_name: "omp allocate var" , |
3989 | DECL_ATTRIBUTES (TREE_OPERAND (t, 0)))) |
3990 | != NULL_TREE)) |
3991 | { |
3992 | tmp = TREE_VALUE (TREE_VALUE (tmp)); |
3993 | tree repl = remap_decl (decl: tmp, id: &ctx->cb); |
3994 | gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK); |
3995 | if (tmp != repl) |
3996 | *tp = repl; |
3997 | break; |
3998 | } |
3999 | gcc_fallthrough (); |
4000 | |
4001 | default: |
4002 | if (ctx && TYPE_P (t)) |
4003 | *tp = remap_type (type: t, id: &ctx->cb); |
4004 | else if (!DECL_P (t)) |
4005 | { |
4006 | *walk_subtrees = 1; |
4007 | if (ctx) |
4008 | { |
4009 | tree tem = remap_type (TREE_TYPE (t), id: &ctx->cb); |
4010 | if (tem != TREE_TYPE (t)) |
4011 | { |
4012 | if (TREE_CODE (t) == INTEGER_CST) |
4013 | *tp = wide_int_to_tree (type: tem, cst: wi::to_wide (t)); |
4014 | else |
4015 | TREE_TYPE (t) = tem; |
4016 | } |
4017 | } |
4018 | } |
4019 | break; |
4020 | } |
4021 | |
4022 | return NULL_TREE; |
4023 | } |
4024 | |
4025 | /* Return true if FNDECL is a setjmp or a longjmp. */ |
4026 | |
4027 | static bool |
4028 | setjmp_or_longjmp_p (const_tree fndecl) |
4029 | { |
4030 | if (fndecl_built_in_p (node: fndecl, name1: BUILT_IN_SETJMP, names: BUILT_IN_LONGJMP)) |
4031 | return true; |
4032 | |
4033 | tree declname = DECL_NAME (fndecl); |
4034 | if (!declname |
4035 | || (DECL_CONTEXT (fndecl) != NULL_TREE |
4036 | && TREE_CODE (DECL_CONTEXT (fndecl)) != TRANSLATION_UNIT_DECL) |
4037 | || !TREE_PUBLIC (fndecl)) |
4038 | return false; |
4039 | |
4040 | const char *name = IDENTIFIER_POINTER (declname); |
4041 | return !strcmp (s1: name, s2: "setjmp" ) || !strcmp (s1: name, s2: "longjmp" ); |
4042 | } |
4043 | |
4044 | /* Helper function for scan_omp. |
4045 | |
4046 | Callback for walk_gimple_stmt used to scan for OMP directives in |
4047 | the current statement in GSI. */ |
4048 | |
4049 | static tree |
4050 | scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p, |
4051 | struct walk_stmt_info *wi) |
4052 | { |
4053 | gimple *stmt = gsi_stmt (i: *gsi); |
4054 | omp_context *ctx = (omp_context *) wi->info; |
4055 | |
4056 | if (gimple_has_location (g: stmt)) |
4057 | input_location = gimple_location (g: stmt); |
4058 | |
4059 | /* Check the nesting restrictions. */ |
4060 | bool remove = false; |
4061 | if (is_gimple_omp (stmt)) |
4062 | remove = !check_omp_nesting_restrictions (stmt, ctx); |
4063 | else if (is_gimple_call (gs: stmt)) |
4064 | { |
4065 | tree fndecl = gimple_call_fndecl (gs: stmt); |
4066 | if (fndecl) |
4067 | { |
4068 | if (ctx |
4069 | && gimple_code (g: ctx->stmt) == GIMPLE_OMP_FOR |
4070 | && gimple_omp_for_kind (g: ctx->stmt) == GF_OMP_FOR_KIND_SIMD |
4071 | && setjmp_or_longjmp_p (fndecl) |
4072 | && !ctx->loop_p) |
4073 | { |
4074 | remove = true; |
4075 | error_at (gimple_location (g: stmt), |
4076 | "setjmp/longjmp inside %<simd%> construct" ); |
4077 | } |
4078 | else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) |
4079 | switch (DECL_FUNCTION_CODE (decl: fndecl)) |
4080 | { |
4081 | case BUILT_IN_GOMP_BARRIER: |
4082 | case BUILT_IN_GOMP_CANCEL: |
4083 | case BUILT_IN_GOMP_CANCELLATION_POINT: |
4084 | case BUILT_IN_GOMP_TASKYIELD: |
4085 | case BUILT_IN_GOMP_TASKWAIT: |
4086 | case BUILT_IN_GOMP_TASKGROUP_START: |
4087 | case BUILT_IN_GOMP_TASKGROUP_END: |
4088 | remove = !check_omp_nesting_restrictions (stmt, ctx); |
4089 | break; |
4090 | default: |
4091 | break; |
4092 | } |
4093 | else if (ctx) |
4094 | { |
4095 | omp_context *octx = ctx; |
4096 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_SCAN && ctx->outer) |
4097 | octx = ctx->outer; |
4098 | if (octx->order_concurrent && omp_runtime_api_call (fndecl)) |
4099 | { |
4100 | remove = true; |
4101 | error_at (gimple_location (g: stmt), |
4102 | "OpenMP runtime API call %qD in a region with " |
4103 | "%<order(concurrent)%> clause" , fndecl); |
4104 | } |
4105 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_TEAMS |
4106 | && omp_runtime_api_call (fndecl) |
4107 | && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl)) |
4108 | != strlen (s: "omp_get_num_teams" )) |
4109 | || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)), |
4110 | s2: "omp_get_num_teams" ) != 0) |
4111 | && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl)) |
4112 | != strlen (s: "omp_get_team_num" )) |
4113 | || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)), |
4114 | s2: "omp_get_team_num" ) != 0)) |
4115 | { |
4116 | remove = true; |
4117 | error_at (gimple_location (g: stmt), |
4118 | "OpenMP runtime API call %qD strictly nested in a " |
4119 | "%<teams%> region" , fndecl); |
4120 | } |
4121 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_TARGET |
4122 | && (gimple_omp_target_kind (g: ctx->stmt) |
4123 | == GF_OMP_TARGET_KIND_REGION) |
4124 | && omp_runtime_api_call (fndecl)) |
4125 | { |
4126 | tree tgt_clauses = gimple_omp_target_clauses (gs: ctx->stmt); |
4127 | tree c = omp_find_clause (clauses: tgt_clauses, kind: OMP_CLAUSE_DEVICE); |
4128 | if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c)) |
4129 | error_at (gimple_location (g: stmt), |
4130 | "OpenMP runtime API call %qD in a region with " |
4131 | "%<device(ancestor)%> clause" , fndecl); |
4132 | } |
4133 | } |
4134 | } |
4135 | } |
4136 | if (remove) |
4137 | { |
4138 | stmt = gimple_build_nop (); |
4139 | gsi_replace (gsi, stmt, false); |
4140 | } |
4141 | |
4142 | *handled_ops_p = true; |
4143 | |
4144 | switch (gimple_code (g: stmt)) |
4145 | { |
4146 | case GIMPLE_OMP_PARALLEL: |
4147 | taskreg_nesting_level++; |
4148 | scan_omp_parallel (gsi, outer_ctx: ctx); |
4149 | taskreg_nesting_level--; |
4150 | break; |
4151 | |
4152 | case GIMPLE_OMP_TASK: |
4153 | taskreg_nesting_level++; |
4154 | scan_omp_task (gsi, outer_ctx: ctx); |
4155 | taskreg_nesting_level--; |
4156 | break; |
4157 | |
4158 | case GIMPLE_OMP_FOR: |
4159 | if ((gimple_omp_for_kind (g: as_a <gomp_for *> (p: stmt)) |
4160 | == GF_OMP_FOR_KIND_SIMD) |
4161 | && gimple_omp_for_combined_into_p (g: stmt) |
4162 | && gimple_code (g: ctx->stmt) != GIMPLE_OMP_SCAN) |
4163 | { |
4164 | tree clauses = gimple_omp_for_clauses (gs: as_a <gomp_for *> (p: stmt)); |
4165 | tree c = omp_find_clause (clauses, kind: OMP_CLAUSE_REDUCTION); |
4166 | if (c && OMP_CLAUSE_REDUCTION_INSCAN (c) && !seen_error ()) |
4167 | { |
4168 | scan_omp_simd_scan (gsi, stmt: as_a <gomp_for *> (p: stmt), outer_ctx: ctx); |
4169 | break; |
4170 | } |
4171 | } |
4172 | if ((gimple_omp_for_kind (g: as_a <gomp_for *> (p: stmt)) |
4173 | == GF_OMP_FOR_KIND_SIMD) |
4174 | && omp_maybe_offloaded_ctx (ctx) |
4175 | && omp_max_simt_vf () |
4176 | && gimple_omp_for_collapse (gs: stmt) == 1) |
4177 | scan_omp_simd (gsi, stmt: as_a <gomp_for *> (p: stmt), outer_ctx: ctx); |
4178 | else |
4179 | scan_omp_for (stmt: as_a <gomp_for *> (p: stmt), outer_ctx: ctx); |
4180 | break; |
4181 | |
4182 | case GIMPLE_OMP_SCOPE: |
4183 | ctx = new_omp_context (stmt, outer_ctx: ctx); |
4184 | scan_sharing_clauses (clauses: gimple_omp_scope_clauses (gs: stmt), ctx); |
4185 | scan_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
4186 | break; |
4187 | |
4188 | case GIMPLE_OMP_SECTIONS: |
4189 | scan_omp_sections (stmt: as_a <gomp_sections *> (p: stmt), outer_ctx: ctx); |
4190 | break; |
4191 | |
4192 | case GIMPLE_OMP_SINGLE: |
4193 | scan_omp_single (stmt: as_a <gomp_single *> (p: stmt), outer_ctx: ctx); |
4194 | break; |
4195 | |
4196 | case GIMPLE_OMP_SCAN: |
4197 | if (tree clauses = gimple_omp_scan_clauses (scan_stmt: as_a <gomp_scan *> (p: stmt))) |
4198 | { |
4199 | if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_INCLUSIVE) |
4200 | ctx->scan_inclusive = true; |
4201 | else if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_EXCLUSIVE) |
4202 | ctx->scan_exclusive = true; |
4203 | } |
4204 | /* FALLTHRU */ |
4205 | case GIMPLE_OMP_SECTION: |
4206 | case GIMPLE_OMP_STRUCTURED_BLOCK: |
4207 | case GIMPLE_OMP_MASTER: |
4208 | case GIMPLE_OMP_ORDERED: |
4209 | case GIMPLE_OMP_CRITICAL: |
4210 | ctx = new_omp_context (stmt, outer_ctx: ctx); |
4211 | scan_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
4212 | break; |
4213 | |
4214 | case GIMPLE_OMP_MASKED: |
4215 | ctx = new_omp_context (stmt, outer_ctx: ctx); |
4216 | scan_sharing_clauses (clauses: gimple_omp_masked_clauses (gs: stmt), ctx); |
4217 | scan_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
4218 | break; |
4219 | |
4220 | case GIMPLE_OMP_TASKGROUP: |
4221 | ctx = new_omp_context (stmt, outer_ctx: ctx); |
4222 | scan_sharing_clauses (clauses: gimple_omp_taskgroup_clauses (gs: stmt), ctx); |
4223 | scan_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
4224 | break; |
4225 | |
4226 | case GIMPLE_OMP_TARGET: |
4227 | if (is_gimple_omp_offloaded (stmt)) |
4228 | { |
4229 | taskreg_nesting_level++; |
4230 | scan_omp_target (stmt: as_a <gomp_target *> (p: stmt), outer_ctx: ctx); |
4231 | taskreg_nesting_level--; |
4232 | } |
4233 | else |
4234 | scan_omp_target (stmt: as_a <gomp_target *> (p: stmt), outer_ctx: ctx); |
4235 | break; |
4236 | |
4237 | case GIMPLE_OMP_TEAMS: |
4238 | if (gimple_omp_teams_host (omp_teams_stmt: as_a <gomp_teams *> (p: stmt))) |
4239 | { |
4240 | taskreg_nesting_level++; |
4241 | scan_omp_teams (stmt: as_a <gomp_teams *> (p: stmt), outer_ctx: ctx); |
4242 | taskreg_nesting_level--; |
4243 | } |
4244 | else |
4245 | scan_omp_teams (stmt: as_a <gomp_teams *> (p: stmt), outer_ctx: ctx); |
4246 | break; |
4247 | |
4248 | case GIMPLE_BIND: |
4249 | { |
4250 | tree var; |
4251 | |
4252 | *handled_ops_p = false; |
4253 | if (ctx) |
4254 | for (var = gimple_bind_vars (bind_stmt: as_a <gbind *> (p: stmt)); |
4255 | var ; |
4256 | var = DECL_CHAIN (var)) |
4257 | insert_decl_map (&ctx->cb, var, var); |
4258 | } |
4259 | break; |
4260 | default: |
4261 | *handled_ops_p = false; |
4262 | break; |
4263 | } |
4264 | |
4265 | return NULL_TREE; |
4266 | } |
4267 | |
4268 | |
4269 | /* Scan all the statements starting at the current statement. CTX |
4270 | contains context information about the OMP directives and |
4271 | clauses found during the scan. */ |
4272 | |
4273 | static void |
4274 | scan_omp (gimple_seq *body_p, omp_context *ctx) |
4275 | { |
4276 | location_t saved_location; |
4277 | struct walk_stmt_info wi; |
4278 | |
4279 | memset (s: &wi, c: 0, n: sizeof (wi)); |
4280 | wi.info = ctx; |
4281 | wi.want_locations = true; |
4282 | |
4283 | saved_location = input_location; |
4284 | walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi); |
4285 | input_location = saved_location; |
4286 | } |
4287 | |
4288 | /* Re-gimplification and code generation routines. */ |
4289 | |
4290 | /* Remove omp_member_access_dummy_var variables from gimple_bind_vars |
4291 | of BIND if in a method. */ |
4292 | |
4293 | static void |
4294 | maybe_remove_omp_member_access_dummy_vars (gbind *bind) |
4295 | { |
4296 | if (DECL_ARGUMENTS (current_function_decl) |
4297 | && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl)) |
4298 | && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl))) |
4299 | == POINTER_TYPE)) |
4300 | { |
4301 | tree vars = gimple_bind_vars (bind_stmt: bind); |
4302 | for (tree *pvar = &vars; *pvar; ) |
4303 | if (omp_member_access_dummy_var (decl: *pvar)) |
4304 | *pvar = DECL_CHAIN (*pvar); |
4305 | else |
4306 | pvar = &DECL_CHAIN (*pvar); |
4307 | gimple_bind_set_vars (bind_stmt: bind, vars); |
4308 | } |
4309 | } |
4310 | |
4311 | /* Remove omp_member_access_dummy_var variables from BLOCK_VARS of |
4312 | block and its subblocks. */ |
4313 | |
4314 | static void |
4315 | remove_member_access_dummy_vars (tree block) |
4316 | { |
4317 | for (tree *pvar = &BLOCK_VARS (block); *pvar; ) |
4318 | if (omp_member_access_dummy_var (decl: *pvar)) |
4319 | *pvar = DECL_CHAIN (*pvar); |
4320 | else |
4321 | pvar = &DECL_CHAIN (*pvar); |
4322 | |
4323 | for (block = BLOCK_SUBBLOCKS (block); block; block = BLOCK_CHAIN (block)) |
4324 | remove_member_access_dummy_vars (block); |
4325 | } |
4326 | |
4327 | /* If a context was created for STMT when it was scanned, return it. */ |
4328 | |
4329 | static omp_context * |
4330 | maybe_lookup_ctx (gimple *stmt) |
4331 | { |
4332 | splay_tree_node n; |
4333 | n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt); |
4334 | return n ? (omp_context *) n->value : NULL; |
4335 | } |
4336 | |
4337 | |
4338 | /* Find the mapping for DECL in CTX or the immediately enclosing |
4339 | context that has a mapping for DECL. |
4340 | |
4341 | If CTX is a nested parallel directive, we may have to use the decl |
4342 | mappings created in CTX's parent context. Suppose that we have the |
4343 | following parallel nesting (variable UIDs showed for clarity): |
4344 | |
4345 | iD.1562 = 0; |
4346 | #omp parallel shared(iD.1562) -> outer parallel |
4347 | iD.1562 = iD.1562 + 1; |
4348 | |
4349 | #omp parallel shared (iD.1562) -> inner parallel |
4350 | iD.1562 = iD.1562 - 1; |
4351 | |
4352 | Each parallel structure will create a distinct .omp_data_s structure |
4353 | for copying iD.1562 in/out of the directive: |
4354 | |
4355 | outer parallel .omp_data_s.1.i -> iD.1562 |
4356 | inner parallel .omp_data_s.2.i -> iD.1562 |
4357 | |
4358 | A shared variable mapping will produce a copy-out operation before |
4359 | the parallel directive and a copy-in operation after it. So, in |
4360 | this case we would have: |
4361 | |
4362 | iD.1562 = 0; |
4363 | .omp_data_o.1.i = iD.1562; |
4364 | #omp parallel shared(iD.1562) -> outer parallel |
4365 | .omp_data_i.1 = &.omp_data_o.1 |
4366 | .omp_data_i.1->i = .omp_data_i.1->i + 1; |
4367 | |
4368 | .omp_data_o.2.i = iD.1562; -> ** |
4369 | #omp parallel shared(iD.1562) -> inner parallel |
4370 | .omp_data_i.2 = &.omp_data_o.2 |
4371 | .omp_data_i.2->i = .omp_data_i.2->i - 1; |
4372 | |
4373 | |
4374 | ** This is a problem. The symbol iD.1562 cannot be referenced |
4375 | inside the body of the outer parallel region. But since we are |
4376 | emitting this copy operation while expanding the inner parallel |
4377 | directive, we need to access the CTX structure of the outer |
4378 | parallel directive to get the correct mapping: |
4379 | |
4380 | .omp_data_o.2.i = .omp_data_i.1->i |
4381 | |
4382 | Since there may be other workshare or parallel directives enclosing |
4383 | the parallel directive, it may be necessary to walk up the context |
4384 | parent chain. This is not a problem in general because nested |
4385 | parallelism happens only rarely. */ |
4386 | |
4387 | static tree |
4388 | lookup_decl_in_outer_ctx (tree decl, omp_context *ctx) |
4389 | { |
4390 | tree t; |
4391 | omp_context *up; |
4392 | |
4393 | for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer) |
4394 | t = maybe_lookup_decl (var: decl, ctx: up); |
4395 | |
4396 | gcc_assert (!ctx->is_nested || t || is_global_var (decl)); |
4397 | |
4398 | return t ? t : decl; |
4399 | } |
4400 | |
4401 | |
4402 | /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found |
4403 | in outer contexts. */ |
4404 | |
4405 | static tree |
4406 | maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx) |
4407 | { |
4408 | tree t = NULL; |
4409 | omp_context *up; |
4410 | |
4411 | for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer) |
4412 | t = maybe_lookup_decl (var: decl, ctx: up); |
4413 | |
4414 | return t ? t : decl; |
4415 | } |
4416 | |
4417 | |
4418 | /* Construct the initialization value for reduction operation OP. */ |
4419 | |
4420 | tree |
4421 | omp_reduction_init_op (location_t loc, enum tree_code op, tree type) |
4422 | { |
4423 | switch (op) |
4424 | { |
4425 | case PLUS_EXPR: |
4426 | case MINUS_EXPR: |
4427 | case BIT_IOR_EXPR: |
4428 | case BIT_XOR_EXPR: |
4429 | case TRUTH_OR_EXPR: |
4430 | case TRUTH_ORIF_EXPR: |
4431 | case TRUTH_XOR_EXPR: |
4432 | case NE_EXPR: |
4433 | return build_zero_cst (type); |
4434 | |
4435 | case MULT_EXPR: |
4436 | case TRUTH_AND_EXPR: |
4437 | case TRUTH_ANDIF_EXPR: |
4438 | case EQ_EXPR: |
4439 | return fold_convert_loc (loc, type, integer_one_node); |
4440 | |
4441 | case BIT_AND_EXPR: |
4442 | return fold_convert_loc (loc, type, integer_minus_one_node); |
4443 | |
4444 | case MAX_EXPR: |
4445 | if (SCALAR_FLOAT_TYPE_P (type)) |
4446 | { |
4447 | REAL_VALUE_TYPE min; |
4448 | if (HONOR_INFINITIES (type)) |
4449 | real_arithmetic (&min, NEGATE_EXPR, &dconstinf, NULL); |
4450 | else |
4451 | real_maxval (&min, 1, TYPE_MODE (type)); |
4452 | return build_real (type, min); |
4453 | } |
4454 | else if (POINTER_TYPE_P (type)) |
4455 | { |
4456 | wide_int min |
4457 | = wi::min_value (TYPE_PRECISION (type), TYPE_SIGN (type)); |
4458 | return wide_int_to_tree (type, cst: min); |
4459 | } |
4460 | else |
4461 | { |
4462 | gcc_assert (INTEGRAL_TYPE_P (type)); |
4463 | return TYPE_MIN_VALUE (type); |
4464 | } |
4465 | |
4466 | case MIN_EXPR: |
4467 | if (SCALAR_FLOAT_TYPE_P (type)) |
4468 | { |
4469 | REAL_VALUE_TYPE max; |
4470 | if (HONOR_INFINITIES (type)) |
4471 | max = dconstinf; |
4472 | else |
4473 | real_maxval (&max, 0, TYPE_MODE (type)); |
4474 | return build_real (type, max); |
4475 | } |
4476 | else if (POINTER_TYPE_P (type)) |
4477 | { |
4478 | wide_int max |
4479 | = wi::max_value (TYPE_PRECISION (type), TYPE_SIGN (type)); |
4480 | return wide_int_to_tree (type, cst: max); |
4481 | } |
4482 | else |
4483 | { |
4484 | gcc_assert (INTEGRAL_TYPE_P (type)); |
4485 | return TYPE_MAX_VALUE (type); |
4486 | } |
4487 | |
4488 | default: |
4489 | gcc_unreachable (); |
4490 | } |
4491 | } |
4492 | |
4493 | /* Construct the initialization value for reduction CLAUSE. */ |
4494 | |
4495 | tree |
4496 | omp_reduction_init (tree clause, tree type) |
4497 | { |
4498 | return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause), |
4499 | OMP_CLAUSE_REDUCTION_CODE (clause), type); |
4500 | } |
4501 | |
4502 | /* Return alignment to be assumed for var in CLAUSE, which should be |
4503 | OMP_CLAUSE_ALIGNED. */ |
4504 | |
4505 | static tree |
4506 | omp_clause_aligned_alignment (tree clause) |
4507 | { |
4508 | if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause)) |
4509 | return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause); |
4510 | |
4511 | /* Otherwise return implementation defined alignment. */ |
4512 | unsigned int al = 1; |
4513 | opt_scalar_mode mode_iter; |
4514 | auto_vector_modes modes; |
4515 | targetm.vectorize.autovectorize_vector_modes (&modes, true); |
4516 | static enum mode_class classes[] |
4517 | = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT }; |
4518 | for (int i = 0; i < 4; i += 2) |
4519 | /* The for loop above dictates that we only walk through scalar classes. */ |
4520 | FOR_EACH_MODE_IN_CLASS (mode_iter, classes[i]) |
4521 | { |
4522 | scalar_mode mode = mode_iter.require (); |
4523 | machine_mode vmode = targetm.vectorize.preferred_simd_mode (mode); |
4524 | if (GET_MODE_CLASS (vmode) != classes[i + 1]) |
4525 | continue; |
4526 | machine_mode alt_vmode; |
4527 | for (unsigned int j = 0; j < modes.length (); ++j) |
4528 | if (related_vector_mode (modes[j], mode).exists (mode: &alt_vmode) |
4529 | && known_ge (GET_MODE_SIZE (alt_vmode), GET_MODE_SIZE (vmode))) |
4530 | vmode = alt_vmode; |
4531 | |
4532 | tree type = lang_hooks.types.type_for_mode (mode, 1); |
4533 | if (type == NULL_TREE || TYPE_MODE (type) != mode) |
4534 | continue; |
4535 | type = build_vector_type_for_mode (type, vmode); |
4536 | if (TYPE_MODE (type) != vmode) |
4537 | continue; |
4538 | if (TYPE_ALIGN_UNIT (type) > al) |
4539 | al = TYPE_ALIGN_UNIT (type); |
4540 | } |
4541 | return build_int_cst (integer_type_node, al); |
4542 | } |
4543 | |
4544 | |
4545 | /* This structure is part of the interface between lower_rec_simd_input_clauses |
4546 | and lower_rec_input_clauses. */ |
4547 | |
4548 | class omplow_simd_context { |
4549 | public: |
4550 | omplow_simd_context () { memset (s: this, c: 0, n: sizeof (*this)); } |
4551 | tree idx; |
4552 | tree lane; |
4553 | tree lastlane; |
4554 | vec<tree, va_heap> simt_eargs; |
4555 | gimple_seq simt_dlist; |
4556 | poly_uint64 max_vf; |
4557 | bool is_simt; |
4558 | }; |
4559 | |
4560 | /* Helper function of lower_rec_input_clauses, used for #pragma omp simd |
4561 | privatization. */ |
4562 | |
4563 | static bool |
4564 | lower_rec_simd_input_clauses (tree new_var, omp_context *ctx, |
4565 | omplow_simd_context *sctx, tree &ivar, |
4566 | tree &lvar, tree *rvar = NULL, |
4567 | tree *rvar2 = NULL) |
4568 | { |
4569 | if (known_eq (sctx->max_vf, 0U)) |
4570 | { |
4571 | sctx->max_vf = sctx->is_simt ? omp_max_simt_vf () : omp_max_vf (); |
4572 | if (maybe_gt (sctx->max_vf, 1U)) |
4573 | { |
4574 | tree c = omp_find_clause (clauses: gimple_omp_for_clauses (gs: ctx->stmt), |
4575 | kind: OMP_CLAUSE_SAFELEN); |
4576 | if (c) |
4577 | { |
4578 | poly_uint64 safe_len; |
4579 | if (!poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), value: &safe_len) |
4580 | || maybe_lt (a: safe_len, b: 1U)) |
4581 | sctx->max_vf = 1; |
4582 | else |
4583 | sctx->max_vf = lower_bound (a: sctx->max_vf, b: safe_len); |
4584 | } |
4585 | } |
4586 | if (sctx->is_simt && !known_eq (sctx->max_vf, 1U)) |
4587 | { |
4588 | for (tree c = gimple_omp_for_clauses (gs: ctx->stmt); c; |
4589 | c = OMP_CLAUSE_CHAIN (c)) |
4590 | { |
4591 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) |
4592 | continue; |
4593 | |
4594 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
4595 | { |
4596 | /* UDR reductions are not supported yet for SIMT, disable |
4597 | SIMT. */ |
4598 | sctx->max_vf = 1; |
4599 | break; |
4600 | } |
4601 | |
4602 | if (truth_value_p (OMP_CLAUSE_REDUCTION_CODE (c)) |
4603 | && !INTEGRAL_TYPE_P (TREE_TYPE (new_var))) |
4604 | { |
4605 | /* Doing boolean operations on non-integral types is |
4606 | for conformance only, it's not worth supporting this |
4607 | for SIMT. */ |
4608 | sctx->max_vf = 1; |
4609 | break; |
4610 | } |
4611 | } |
4612 | } |
4613 | if (maybe_gt (sctx->max_vf, 1U)) |
4614 | { |
4615 | sctx->idx = create_tmp_var (unsigned_type_node); |
4616 | sctx->lane = create_tmp_var (unsigned_type_node); |
4617 | } |
4618 | } |
4619 | if (known_eq (sctx->max_vf, 1U)) |
4620 | return false; |
4621 | |
4622 | if (sctx->is_simt) |
4623 | { |
4624 | if (is_gimple_reg (new_var)) |
4625 | { |
4626 | ivar = lvar = new_var; |
4627 | return true; |
4628 | } |
4629 | tree type = TREE_TYPE (new_var), ptype = build_pointer_type (type); |
4630 | ivar = lvar = create_tmp_var (type); |
4631 | TREE_ADDRESSABLE (ivar) = 1; |
4632 | DECL_ATTRIBUTES (ivar) = tree_cons (get_identifier ("omp simt private" ), |
4633 | NULL, DECL_ATTRIBUTES (ivar)); |
4634 | sctx->simt_eargs.safe_push (obj: build1 (ADDR_EXPR, ptype, ivar)); |
4635 | tree clobber = build_clobber (type); |
4636 | gimple *g = gimple_build_assign (ivar, clobber); |
4637 | gimple_seq_add_stmt (&sctx->simt_dlist, g); |
4638 | } |
4639 | else |
4640 | { |
4641 | tree atype = build_array_type_nelts (TREE_TYPE (new_var), sctx->max_vf); |
4642 | tree avar = create_tmp_var_raw (atype); |
4643 | if (TREE_ADDRESSABLE (new_var)) |
4644 | TREE_ADDRESSABLE (avar) = 1; |
4645 | DECL_ATTRIBUTES (avar) |
4646 | = tree_cons (get_identifier ("omp simd array" ), NULL, |
4647 | DECL_ATTRIBUTES (avar)); |
4648 | gimple_add_tmp_var (avar); |
4649 | tree iavar = avar; |
4650 | if (rvar && !ctx->for_simd_scan_phase) |
4651 | { |
4652 | /* For inscan reductions, create another array temporary, |
4653 | which will hold the reduced value. */ |
4654 | iavar = create_tmp_var_raw (atype); |
4655 | if (TREE_ADDRESSABLE (new_var)) |
4656 | TREE_ADDRESSABLE (iavar) = 1; |
4657 | DECL_ATTRIBUTES (iavar) |
4658 | = tree_cons (get_identifier ("omp simd array" ), NULL, |
4659 | tree_cons (get_identifier ("omp simd inscan" ), NULL, |
4660 | DECL_ATTRIBUTES (iavar))); |
4661 | gimple_add_tmp_var (iavar); |
4662 | ctx->cb.decl_map->put (k: avar, v: iavar); |
4663 | if (sctx->lastlane == NULL_TREE) |
4664 | sctx->lastlane = create_tmp_var (unsigned_type_node); |
4665 | *rvar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar, |
4666 | sctx->lastlane, NULL_TREE, NULL_TREE); |
4667 | TREE_THIS_NOTRAP (*rvar) = 1; |
4668 | |
4669 | if (ctx->scan_exclusive) |
4670 | { |
4671 | /* And for exclusive scan yet another one, which will |
4672 | hold the value during the scan phase. */ |
4673 | tree savar = create_tmp_var_raw (atype); |
4674 | if (TREE_ADDRESSABLE (new_var)) |
4675 | TREE_ADDRESSABLE (savar) = 1; |
4676 | DECL_ATTRIBUTES (savar) |
4677 | = tree_cons (get_identifier ("omp simd array" ), NULL, |
4678 | tree_cons (get_identifier ("omp simd inscan " |
4679 | "exclusive" ), NULL, |
4680 | DECL_ATTRIBUTES (savar))); |
4681 | gimple_add_tmp_var (savar); |
4682 | ctx->cb.decl_map->put (k: iavar, v: savar); |
4683 | *rvar2 = build4 (ARRAY_REF, TREE_TYPE (new_var), savar, |
4684 | sctx->idx, NULL_TREE, NULL_TREE); |
4685 | TREE_THIS_NOTRAP (*rvar2) = 1; |
4686 | } |
4687 | } |
4688 | ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar, sctx->idx, |
4689 | NULL_TREE, NULL_TREE); |
4690 | lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, sctx->lane, |
4691 | NULL_TREE, NULL_TREE); |
4692 | TREE_THIS_NOTRAP (ivar) = 1; |
4693 | TREE_THIS_NOTRAP (lvar) = 1; |
4694 | } |
4695 | if (DECL_P (new_var)) |
4696 | { |
4697 | SET_DECL_VALUE_EXPR (new_var, lvar); |
4698 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
4699 | } |
4700 | return true; |
4701 | } |
4702 | |
4703 | /* Helper function of lower_rec_input_clauses. For a reference |
4704 | in simd reduction, add an underlying variable it will reference. */ |
4705 | |
4706 | static void |
4707 | handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist) |
4708 | { |
4709 | tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard))); |
4710 | if (TREE_CONSTANT (z)) |
4711 | { |
4712 | z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)), |
4713 | get_name (new_vard)); |
4714 | gimple_add_tmp_var (z); |
4715 | TREE_ADDRESSABLE (z) = 1; |
4716 | z = build_fold_addr_expr_loc (loc, z); |
4717 | gimplify_assign (new_vard, z, ilist); |
4718 | } |
4719 | } |
4720 | |
4721 | /* Helper function for lower_rec_input_clauses. Emit into ilist sequence |
4722 | code to emit (type) (tskred_temp[idx]). */ |
4723 | |
4724 | static tree |
4725 | task_reduction_read (gimple_seq *ilist, tree tskred_temp, tree type, |
4726 | unsigned idx) |
4727 | { |
4728 | unsigned HOST_WIDE_INT sz |
4729 | = tree_to_uhwi (TYPE_SIZE_UNIT (pointer_sized_int_node)); |
4730 | tree r = build2 (MEM_REF, pointer_sized_int_node, |
4731 | tskred_temp, build_int_cst (TREE_TYPE (tskred_temp), |
4732 | idx * sz)); |
4733 | tree v = create_tmp_var (pointer_sized_int_node); |
4734 | gimple *g = gimple_build_assign (v, r); |
4735 | gimple_seq_add_stmt (ilist, g); |
4736 | if (!useless_type_conversion_p (type, pointer_sized_int_node)) |
4737 | { |
4738 | v = create_tmp_var (type); |
4739 | g = gimple_build_assign (v, NOP_EXPR, gimple_assign_lhs (gs: g)); |
4740 | gimple_seq_add_stmt (ilist, g); |
4741 | } |
4742 | return v; |
4743 | } |
4744 | |
4745 | /* Lower early initialization of privatized variable NEW_VAR |
4746 | if it needs an allocator (has allocate clause). */ |
4747 | |
4748 | static bool |
4749 | lower_private_allocate (tree var, tree new_var, tree &allocator, |
4750 | tree &allocate_ptr, gimple_seq *ilist, |
4751 | omp_context *ctx, bool is_ref, tree size) |
4752 | { |
4753 | if (allocator) |
4754 | return false; |
4755 | gcc_assert (allocate_ptr == NULL_TREE); |
4756 | if (ctx->allocate_map |
4757 | && (DECL_P (new_var) || (TYPE_P (new_var) && size))) |
4758 | if (tree *allocatorp = ctx->allocate_map->get (k: var)) |
4759 | allocator = *allocatorp; |
4760 | if (allocator == NULL_TREE) |
4761 | return false; |
4762 | if (!is_ref && omp_privatize_by_reference (decl: var)) |
4763 | { |
4764 | allocator = NULL_TREE; |
4765 | return false; |
4766 | } |
4767 | |
4768 | unsigned HOST_WIDE_INT ialign = 0; |
4769 | if (TREE_CODE (allocator) == TREE_LIST) |
4770 | { |
4771 | ialign = tree_to_uhwi (TREE_VALUE (allocator)); |
4772 | allocator = TREE_PURPOSE (allocator); |
4773 | } |
4774 | if (TREE_CODE (allocator) != INTEGER_CST) |
4775 | allocator = build_outer_var_ref (var: allocator, ctx, code: OMP_CLAUSE_ALLOCATE); |
4776 | allocator = fold_convert (pointer_sized_int_node, allocator); |
4777 | if (TREE_CODE (allocator) != INTEGER_CST) |
4778 | { |
4779 | tree var = create_tmp_var (TREE_TYPE (allocator)); |
4780 | gimplify_assign (var, allocator, ilist); |
4781 | allocator = var; |
4782 | } |
4783 | |
4784 | tree ptr_type, align, sz = size; |
4785 | if (TYPE_P (new_var)) |
4786 | { |
4787 | ptr_type = build_pointer_type (new_var); |
4788 | ialign = MAX (ialign, TYPE_ALIGN_UNIT (new_var)); |
4789 | } |
4790 | else if (is_ref) |
4791 | { |
4792 | ptr_type = build_pointer_type (TREE_TYPE (TREE_TYPE (new_var))); |
4793 | ialign = MAX (ialign, TYPE_ALIGN_UNIT (TREE_TYPE (ptr_type))); |
4794 | } |
4795 | else |
4796 | { |
4797 | ptr_type = build_pointer_type (TREE_TYPE (new_var)); |
4798 | ialign = MAX (ialign, DECL_ALIGN_UNIT (new_var)); |
4799 | if (sz == NULL_TREE) |
4800 | sz = fold_convert (size_type_node, DECL_SIZE_UNIT (new_var)); |
4801 | } |
4802 | align = build_int_cst (size_type_node, ialign); |
4803 | if (TREE_CODE (sz) != INTEGER_CST) |
4804 | { |
4805 | tree szvar = create_tmp_var (size_type_node); |
4806 | gimplify_assign (szvar, sz, ilist); |
4807 | sz = szvar; |
4808 | } |
4809 | allocate_ptr = create_tmp_var (ptr_type); |
4810 | tree a = builtin_decl_explicit (fncode: BUILT_IN_GOMP_ALLOC); |
4811 | gimple *g = gimple_build_call (a, 3, align, sz, allocator); |
4812 | gimple_call_set_lhs (gs: g, lhs: allocate_ptr); |
4813 | gimple_seq_add_stmt (ilist, g); |
4814 | if (!is_ref) |
4815 | { |
4816 | tree x = build_simple_mem_ref (allocate_ptr); |
4817 | TREE_THIS_NOTRAP (x) = 1; |
4818 | SET_DECL_VALUE_EXPR (new_var, x); |
4819 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
4820 | } |
4821 | return true; |
4822 | } |
4823 | |
4824 | /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN, |
4825 | from the receiver (aka child) side and initializers for REFERENCE_TYPE |
4826 | private variables. Initialization statements go in ILIST, while calls |
4827 | to destructors go in DLIST. */ |
4828 | |
4829 | static void |
4830 | lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist, |
4831 | omp_context *ctx, struct omp_for_data *fd) |
4832 | { |
4833 | tree c, copyin_seq, x, ptr; |
4834 | bool copyin_by_ref = false; |
4835 | bool lastprivate_firstprivate = false; |
4836 | bool reduction_omp_orig_ref = false; |
4837 | int pass; |
4838 | bool is_simd = (gimple_code (g: ctx->stmt) == GIMPLE_OMP_FOR |
4839 | && gimple_omp_for_kind (g: ctx->stmt) == GF_OMP_FOR_KIND_SIMD); |
4840 | omplow_simd_context sctx = omplow_simd_context (); |
4841 | tree simt_lane = NULL_TREE, simtrec = NULL_TREE; |
4842 | tree ivar = NULL_TREE, lvar = NULL_TREE, uid = NULL_TREE; |
4843 | gimple_seq llist[4] = { }; |
4844 | tree nonconst_simd_if = NULL_TREE; |
4845 | |
4846 | copyin_seq = NULL; |
4847 | sctx.is_simt = is_simd && omp_find_clause (clauses, kind: OMP_CLAUSE__SIMT_); |
4848 | |
4849 | /* Set max_vf=1 (which will later enforce safelen=1) in simd loops |
4850 | with data sharing clauses referencing variable sized vars. That |
4851 | is unnecessarily hard to support and very unlikely to result in |
4852 | vectorized code anyway. */ |
4853 | if (is_simd) |
4854 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
4855 | switch (OMP_CLAUSE_CODE (c)) |
4856 | { |
4857 | case OMP_CLAUSE_LINEAR: |
4858 | if (OMP_CLAUSE_LINEAR_ARRAY (c)) |
4859 | sctx.max_vf = 1; |
4860 | /* FALLTHRU */ |
4861 | case OMP_CLAUSE_PRIVATE: |
4862 | case OMP_CLAUSE_FIRSTPRIVATE: |
4863 | case OMP_CLAUSE_LASTPRIVATE: |
4864 | if (is_variable_sized (OMP_CLAUSE_DECL (c))) |
4865 | sctx.max_vf = 1; |
4866 | else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c))) |
4867 | { |
4868 | tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c))); |
4869 | if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype))) |
4870 | sctx.max_vf = 1; |
4871 | } |
4872 | break; |
4873 | case OMP_CLAUSE_REDUCTION: |
4874 | case OMP_CLAUSE_IN_REDUCTION: |
4875 | if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF |
4876 | || is_variable_sized (OMP_CLAUSE_DECL (c))) |
4877 | sctx.max_vf = 1; |
4878 | else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c))) |
4879 | { |
4880 | tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c))); |
4881 | if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype))) |
4882 | sctx.max_vf = 1; |
4883 | } |
4884 | break; |
4885 | case OMP_CLAUSE_IF: |
4886 | if (integer_zerop (OMP_CLAUSE_IF_EXPR (c))) |
4887 | sctx.max_vf = 1; |
4888 | else if (TREE_CODE (OMP_CLAUSE_IF_EXPR (c)) != INTEGER_CST) |
4889 | nonconst_simd_if = OMP_CLAUSE_IF_EXPR (c); |
4890 | break; |
4891 | case OMP_CLAUSE_SIMDLEN: |
4892 | if (integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (c))) |
4893 | sctx.max_vf = 1; |
4894 | break; |
4895 | case OMP_CLAUSE__CONDTEMP_: |
4896 | /* FIXME: lastprivate(conditional:) not handled for SIMT yet. */ |
4897 | if (sctx.is_simt) |
4898 | sctx.max_vf = 1; |
4899 | break; |
4900 | default: |
4901 | continue; |
4902 | } |
4903 | |
4904 | /* Add a placeholder for simduid. */ |
4905 | if (sctx.is_simt && maybe_ne (a: sctx.max_vf, b: 1U)) |
4906 | sctx.simt_eargs.safe_push (NULL_TREE); |
4907 | |
4908 | unsigned task_reduction_cnt = 0; |
4909 | unsigned task_reduction_cntorig = 0; |
4910 | unsigned task_reduction_cnt_full = 0; |
4911 | unsigned task_reduction_cntorig_full = 0; |
4912 | unsigned task_reduction_other_cnt = 0; |
4913 | tree tskred_atype = NULL_TREE, tskred_avar = NULL_TREE; |
4914 | tree tskred_base = NULL_TREE, tskred_temp = NULL_TREE; |
4915 | /* Do all the fixed sized types in the first pass, and the variable sized |
4916 | types in the second pass. This makes sure that the scalar arguments to |
4917 | the variable sized types are processed before we use them in the |
4918 | variable sized operations. For task reductions we use 4 passes, in the |
4919 | first two we ignore them, in the third one gather arguments for |
4920 | GOMP_task_reduction_remap call and in the last pass actually handle |
4921 | the task reductions. */ |
4922 | for (pass = 0; pass < ((task_reduction_cnt || task_reduction_other_cnt) |
4923 | ? 4 : 2); ++pass) |
4924 | { |
4925 | if (pass == 2 && task_reduction_cnt) |
4926 | { |
4927 | tskred_atype |
4928 | = build_array_type_nelts (ptr_type_node, task_reduction_cnt |
4929 | + task_reduction_cntorig); |
4930 | tskred_avar = create_tmp_var_raw (tskred_atype); |
4931 | gimple_add_tmp_var (tskred_avar); |
4932 | TREE_ADDRESSABLE (tskred_avar) = 1; |
4933 | task_reduction_cnt_full = task_reduction_cnt; |
4934 | task_reduction_cntorig_full = task_reduction_cntorig; |
4935 | } |
4936 | else if (pass == 3 && task_reduction_cnt) |
4937 | { |
4938 | x = builtin_decl_explicit (fncode: BUILT_IN_GOMP_TASK_REDUCTION_REMAP); |
4939 | gimple *g |
4940 | = gimple_build_call (x, 3, size_int (task_reduction_cnt), |
4941 | size_int (task_reduction_cntorig), |
4942 | build_fold_addr_expr (tskred_avar)); |
4943 | gimple_seq_add_stmt (ilist, g); |
4944 | } |
4945 | if (pass == 3 && task_reduction_other_cnt) |
4946 | { |
4947 | /* For reduction clauses, build |
4948 | tskred_base = (void *) tskred_temp[2] |
4949 | + omp_get_thread_num () * tskred_temp[1] |
4950 | or if tskred_temp[1] is known to be constant, that constant |
4951 | directly. This is the start of the private reduction copy block |
4952 | for the current thread. */ |
4953 | tree v = create_tmp_var (integer_type_node); |
4954 | x = builtin_decl_explicit (fncode: BUILT_IN_OMP_GET_THREAD_NUM); |
4955 | gimple *g = gimple_build_call (x, 0); |
4956 | gimple_call_set_lhs (gs: g, lhs: v); |
4957 | gimple_seq_add_stmt (ilist, g); |
4958 | c = omp_find_clause (clauses, kind: OMP_CLAUSE__REDUCTEMP_); |
4959 | tskred_temp = OMP_CLAUSE_DECL (c); |
4960 | if (is_taskreg_ctx (ctx)) |
4961 | tskred_temp = lookup_decl (var: tskred_temp, ctx); |
4962 | tree v2 = create_tmp_var (sizetype); |
4963 | g = gimple_build_assign (v2, NOP_EXPR, v); |
4964 | gimple_seq_add_stmt (ilist, g); |
4965 | if (ctx->task_reductions[0]) |
4966 | v = fold_convert (sizetype, ctx->task_reductions[0]); |
4967 | else |
4968 | v = task_reduction_read (ilist, tskred_temp, sizetype, idx: 1); |
4969 | tree v3 = create_tmp_var (sizetype); |
4970 | g = gimple_build_assign (v3, MULT_EXPR, v2, v); |
4971 | gimple_seq_add_stmt (ilist, g); |
4972 | v = task_reduction_read (ilist, tskred_temp, ptr_type_node, idx: 2); |
4973 | tskred_base = create_tmp_var (ptr_type_node); |
4974 | g = gimple_build_assign (tskred_base, POINTER_PLUS_EXPR, v, v3); |
4975 | gimple_seq_add_stmt (ilist, g); |
4976 | } |
4977 | task_reduction_cnt = 0; |
4978 | task_reduction_cntorig = 0; |
4979 | task_reduction_other_cnt = 0; |
4980 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
4981 | { |
4982 | enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c); |
4983 | tree var, new_var; |
4984 | bool by_ref; |
4985 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
4986 | bool task_reduction_p = false; |
4987 | bool task_reduction_needs_orig_p = false; |
4988 | tree cond = NULL_TREE; |
4989 | tree allocator, allocate_ptr; |
4990 | |
4991 | switch (c_kind) |
4992 | { |
4993 | case OMP_CLAUSE_PRIVATE: |
4994 | if (OMP_CLAUSE_PRIVATE_DEBUG (c)) |
4995 | continue; |
4996 | break; |
4997 | case OMP_CLAUSE_SHARED: |
4998 | /* Ignore shared directives in teams construct inside |
4999 | of target construct. */ |
5000 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_TEAMS |
5001 | && !is_host_teams_ctx (ctx)) |
5002 | continue; |
5003 | if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL) |
5004 | { |
5005 | gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c) |
5006 | || is_global_var (OMP_CLAUSE_DECL (c))); |
5007 | continue; |
5008 | } |
5009 | case OMP_CLAUSE_FIRSTPRIVATE: |
5010 | case OMP_CLAUSE_COPYIN: |
5011 | break; |
5012 | case OMP_CLAUSE_LINEAR: |
5013 | if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c) |
5014 | && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)) |
5015 | lastprivate_firstprivate = true; |
5016 | break; |
5017 | case OMP_CLAUSE_REDUCTION: |
5018 | case OMP_CLAUSE_IN_REDUCTION: |
5019 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION |
5020 | || is_task_ctx (ctx) |
5021 | || OMP_CLAUSE_REDUCTION_TASK (c)) |
5022 | { |
5023 | task_reduction_p = true; |
5024 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION) |
5025 | { |
5026 | task_reduction_other_cnt++; |
5027 | if (pass == 2) |
5028 | continue; |
5029 | } |
5030 | else |
5031 | task_reduction_cnt++; |
5032 | if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)) |
5033 | { |
5034 | var = OMP_CLAUSE_DECL (c); |
5035 | /* If var is a global variable that isn't privatized |
5036 | in outer contexts, we don't need to look up the |
5037 | original address, it is always the address of the |
5038 | global variable itself. */ |
5039 | if (!DECL_P (var) |
5040 | || omp_privatize_by_reference (decl: var) |
5041 | || !is_global_var |
5042 | (t: maybe_lookup_decl_in_outer_ctx (decl: var, ctx))) |
5043 | { |
5044 | task_reduction_needs_orig_p = true; |
5045 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) |
5046 | task_reduction_cntorig++; |
5047 | } |
5048 | } |
5049 | } |
5050 | else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)) |
5051 | reduction_omp_orig_ref = true; |
5052 | break; |
5053 | case OMP_CLAUSE__REDUCTEMP_: |
5054 | if (!is_taskreg_ctx (ctx)) |
5055 | continue; |
5056 | /* FALLTHRU */ |
5057 | case OMP_CLAUSE__LOOPTEMP_: |
5058 | /* Handle _looptemp_/_reductemp_ clauses only on |
5059 | parallel/task. */ |
5060 | if (fd) |
5061 | continue; |
5062 | break; |
5063 | case OMP_CLAUSE_LASTPRIVATE: |
5064 | if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) |
5065 | { |
5066 | lastprivate_firstprivate = true; |
5067 | if (pass != 0 || is_taskloop_ctx (ctx)) |
5068 | continue; |
5069 | } |
5070 | /* Even without corresponding firstprivate, if |
5071 | decl is Fortran allocatable, it needs outer var |
5072 | reference. */ |
5073 | else if (pass == 0 |
5074 | && lang_hooks.decls.omp_private_outer_ref |
5075 | (OMP_CLAUSE_DECL (c))) |
5076 | lastprivate_firstprivate = true; |
5077 | break; |
5078 | case OMP_CLAUSE_ALIGNED: |
5079 | if (pass != 1) |
5080 | continue; |
5081 | var = OMP_CLAUSE_DECL (c); |
5082 | if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE |
5083 | && !is_global_var (t: var)) |
5084 | { |
5085 | new_var = maybe_lookup_decl (var, ctx); |
5086 | if (new_var == NULL_TREE) |
5087 | new_var = maybe_lookup_decl_in_outer_ctx (decl: var, ctx); |
5088 | x = builtin_decl_explicit (fncode: BUILT_IN_ASSUME_ALIGNED); |
5089 | tree alarg = omp_clause_aligned_alignment (clause: c); |
5090 | alarg = fold_convert_loc (clause_loc, size_type_node, alarg); |
5091 | x = build_call_expr_loc (clause_loc, x, 2, new_var, alarg); |
5092 | x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x); |
5093 | x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x); |
5094 | gimplify_and_add (x, ilist); |
5095 | } |
5096 | else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE |
5097 | && is_global_var (t: var)) |
5098 | { |
5099 | tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2; |
5100 | new_var = lookup_decl (var, ctx); |
5101 | t = maybe_lookup_decl_in_outer_ctx (decl: var, ctx); |
5102 | t = build_fold_addr_expr_loc (clause_loc, t); |
5103 | t2 = builtin_decl_explicit (fncode: BUILT_IN_ASSUME_ALIGNED); |
5104 | tree alarg = omp_clause_aligned_alignment (clause: c); |
5105 | alarg = fold_convert_loc (clause_loc, size_type_node, alarg); |
5106 | t = build_call_expr_loc (clause_loc, t2, 2, t, alarg); |
5107 | t = fold_convert_loc (clause_loc, ptype, t); |
5108 | x = create_tmp_var (ptype); |
5109 | t = build2 (MODIFY_EXPR, ptype, x, t); |
5110 | gimplify_and_add (t, ilist); |
5111 | t = build_simple_mem_ref_loc (clause_loc, x); |
5112 | SET_DECL_VALUE_EXPR (new_var, t); |
5113 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
5114 | } |
5115 | continue; |
5116 | case OMP_CLAUSE__CONDTEMP_: |
5117 | if (is_parallel_ctx (ctx) |
5118 | || (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c))) |
5119 | break; |
5120 | continue; |
5121 | default: |
5122 | continue; |
5123 | } |
5124 | |
5125 | if (task_reduction_p != (pass >= 2)) |
5126 | continue; |
5127 | |
5128 | allocator = NULL_TREE; |
5129 | allocate_ptr = NULL_TREE; |
5130 | new_var = var = OMP_CLAUSE_DECL (c); |
5131 | if ((c_kind == OMP_CLAUSE_REDUCTION |
5132 | || c_kind == OMP_CLAUSE_IN_REDUCTION) |
5133 | && TREE_CODE (var) == MEM_REF) |
5134 | { |
5135 | var = TREE_OPERAND (var, 0); |
5136 | if (TREE_CODE (var) == POINTER_PLUS_EXPR) |
5137 | var = TREE_OPERAND (var, 0); |
5138 | if (TREE_CODE (var) == INDIRECT_REF |
5139 | || TREE_CODE (var) == ADDR_EXPR) |
5140 | var = TREE_OPERAND (var, 0); |
5141 | if (is_variable_sized (expr: var)) |
5142 | { |
5143 | gcc_assert (DECL_HAS_VALUE_EXPR_P (var)); |
5144 | var = DECL_VALUE_EXPR (var); |
5145 | gcc_assert (TREE_CODE (var) == INDIRECT_REF); |
5146 | var = TREE_OPERAND (var, 0); |
5147 | gcc_assert (DECL_P (var)); |
5148 | } |
5149 | new_var = var; |
5150 | } |
5151 | if (c_kind == OMP_CLAUSE_IN_REDUCTION && is_omp_target (stmt: ctx->stmt)) |
5152 | { |
5153 | splay_tree_key key = (splay_tree_key) &DECL_CONTEXT (var); |
5154 | new_var = (tree) splay_tree_lookup (ctx->field_map, key)->value; |
5155 | } |
5156 | else if (c_kind != OMP_CLAUSE_COPYIN) |
5157 | new_var = lookup_decl (var, ctx); |
5158 | |
5159 | if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN) |
5160 | { |
5161 | if (pass != 0) |
5162 | continue; |
5163 | } |
5164 | /* C/C++ array section reductions. */ |
5165 | else if ((c_kind == OMP_CLAUSE_REDUCTION |
5166 | || c_kind == OMP_CLAUSE_IN_REDUCTION) |
5167 | && var != OMP_CLAUSE_DECL (c)) |
5168 | { |
5169 | if (pass == 0) |
5170 | continue; |
5171 | |
5172 | tree bias = TREE_OPERAND (OMP_CLAUSE_DECL (c), 1); |
5173 | tree orig_var = TREE_OPERAND (OMP_CLAUSE_DECL (c), 0); |
5174 | |
5175 | if (TREE_CODE (orig_var) == POINTER_PLUS_EXPR) |
5176 | { |
5177 | tree b = TREE_OPERAND (orig_var, 1); |
5178 | if (is_omp_target (stmt: ctx->stmt)) |
5179 | b = NULL_TREE; |
5180 | else |
5181 | b = maybe_lookup_decl (var: b, ctx); |
5182 | if (b == NULL) |
5183 | { |
5184 | b = TREE_OPERAND (orig_var, 1); |
5185 | b = maybe_lookup_decl_in_outer_ctx (decl: b, ctx); |
5186 | } |
5187 | if (integer_zerop (bias)) |
5188 | bias = b; |
5189 | else |
5190 | { |
5191 | bias = fold_convert_loc (clause_loc, |
5192 | TREE_TYPE (b), bias); |
5193 | bias = fold_build2_loc (clause_loc, PLUS_EXPR, |
5194 | TREE_TYPE (b), b, bias); |
5195 | } |
5196 | orig_var = TREE_OPERAND (orig_var, 0); |
5197 | } |
5198 | if (pass == 2) |
5199 | { |
5200 | tree out = maybe_lookup_decl_in_outer_ctx (decl: var, ctx); |
5201 | if (is_global_var (t: out) |
5202 | && TREE_CODE (TREE_TYPE (out)) != POINTER_TYPE |
5203 | && (TREE_CODE (TREE_TYPE (out)) != REFERENCE_TYPE |
5204 | || (TREE_CODE (TREE_TYPE (TREE_TYPE (out))) |
5205 | != POINTER_TYPE))) |
5206 | x = var; |
5207 | else if (is_omp_target (stmt: ctx->stmt)) |
5208 | x = out; |
5209 | else |
5210 | { |
5211 | bool by_ref = use_pointer_for_field (decl: var, NULL); |
5212 | x = build_receiver_ref (var, by_ref, ctx); |
5213 | if (TREE_CODE (TREE_TYPE (var)) == REFERENCE_TYPE |
5214 | && (TREE_CODE (TREE_TYPE (TREE_TYPE (var))) |
5215 | == POINTER_TYPE)) |
5216 | x = build_fold_addr_expr (x); |
5217 | } |
5218 | if (TREE_CODE (orig_var) == INDIRECT_REF) |
5219 | x = build_simple_mem_ref (x); |
5220 | else if (TREE_CODE (orig_var) == ADDR_EXPR) |
5221 | { |
5222 | if (var == TREE_OPERAND (orig_var, 0)) |
5223 | x = build_fold_addr_expr (x); |
5224 | } |
5225 | bias = fold_convert (sizetype, bias); |
5226 | x = fold_convert (ptr_type_node, x); |
5227 | x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR, |
5228 | TREE_TYPE (x), x, bias); |
5229 | unsigned cnt = task_reduction_cnt - 1; |
5230 | if (!task_reduction_needs_orig_p) |
5231 | cnt += (task_reduction_cntorig_full |
5232 | - task_reduction_cntorig); |
5233 | else |
5234 | cnt = task_reduction_cntorig - 1; |
5235 | tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar, |
5236 | size_int (cnt), NULL_TREE, NULL_TREE); |
5237 | gimplify_assign (r, x, ilist); |
5238 | continue; |
5239 | } |
5240 | |
5241 | if (TREE_CODE (orig_var) == INDIRECT_REF |
5242 | || TREE_CODE (orig_var) == ADDR_EXPR) |
5243 | orig_var = TREE_OPERAND (orig_var, 0); |
5244 | tree d = OMP_CLAUSE_DECL (c); |
5245 | tree type = TREE_TYPE (d); |
5246 | gcc_assert (TREE_CODE (type) == ARRAY_TYPE); |
5247 | tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type)); |
5248 | tree sz = v; |
5249 | const char *name = get_name (orig_var); |
5250 | if (pass != 3 && !TREE_CONSTANT (v)) |
5251 | { |
5252 | tree t; |
5253 | if (is_omp_target (stmt: ctx->stmt)) |
5254 | t = NULL_TREE; |
5255 | else |
5256 | t = maybe_lookup_decl (var: v, ctx); |
5257 | if (t) |
5258 | v = t; |
5259 | else |
5260 | v = maybe_lookup_decl_in_outer_ctx (decl: v, ctx); |
5261 | gimplify_expr (&v, ilist, NULL, is_gimple_val, fb_rvalue); |
5262 | t = fold_build2_loc (clause_loc, PLUS_EXPR, |
5263 | TREE_TYPE (v), v, |
5264 | build_int_cst (TREE_TYPE (v), 1)); |
5265 | sz = fold_build2_loc (clause_loc, MULT_EXPR, |
5266 | TREE_TYPE (v), t, |
5267 | TYPE_SIZE_UNIT (TREE_TYPE (type))); |
5268 | } |
5269 | if (pass == 3) |
5270 | { |
5271 | tree xv = create_tmp_var (ptr_type_node); |
5272 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) |
5273 | { |
5274 | unsigned cnt = task_reduction_cnt - 1; |
5275 | if (!task_reduction_needs_orig_p) |
5276 | cnt += (task_reduction_cntorig_full |
5277 | - task_reduction_cntorig); |
5278 | else |
5279 | cnt = task_reduction_cntorig - 1; |
5280 | x = build4 (ARRAY_REF, ptr_type_node, tskred_avar, |
5281 | size_int (cnt), NULL_TREE, NULL_TREE); |
5282 | |
5283 | gimple *g = gimple_build_assign (xv, x); |
5284 | gimple_seq_add_stmt (ilist, g); |
5285 | } |
5286 | else |
5287 | { |
5288 | unsigned int idx = *ctx->task_reduction_map->get (k: c); |
5289 | tree off; |
5290 | if (ctx->task_reductions[1 + idx]) |
5291 | off = fold_convert (sizetype, |
5292 | ctx->task_reductions[1 + idx]); |
5293 | else |
5294 | off = task_reduction_read (ilist, tskred_temp, sizetype, |
5295 | idx: 7 + 3 * idx + 1); |
5296 | gimple *g = gimple_build_assign (xv, POINTER_PLUS_EXPR, |
5297 | tskred_base, off); |
5298 | gimple_seq_add_stmt (ilist, g); |
5299 | } |
5300 | x = fold_convert (build_pointer_type (boolean_type_node), |
5301 | xv); |
5302 | if (TREE_CONSTANT (v)) |
5303 | x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x, |
5304 | TYPE_SIZE_UNIT (type)); |
5305 | else |
5306 | { |
5307 | tree t; |
5308 | if (is_omp_target (stmt: ctx->stmt)) |
5309 | t = NULL_TREE; |
5310 | else |
5311 | t = maybe_lookup_decl (var: v, ctx); |
5312 | if (t) |
5313 | v = t; |
5314 | else |
5315 | v = maybe_lookup_decl_in_outer_ctx (decl: v, ctx); |
5316 | gimplify_expr (&v, ilist, NULL, is_gimple_val, |
5317 | fb_rvalue); |
5318 | t = fold_build2_loc (clause_loc, PLUS_EXPR, |
5319 | TREE_TYPE (v), v, |
5320 | build_int_cst (TREE_TYPE (v), 1)); |
5321 | t = fold_build2_loc (clause_loc, MULT_EXPR, |
5322 | TREE_TYPE (v), t, |
5323 | TYPE_SIZE_UNIT (TREE_TYPE (type))); |
5324 | x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x, t); |
5325 | } |
5326 | cond = create_tmp_var (TREE_TYPE (x)); |
5327 | gimplify_assign (cond, x, ilist); |
5328 | x = xv; |
5329 | } |
5330 | else if (lower_private_allocate (var, new_var: type, allocator, |
5331 | allocate_ptr, ilist, ctx, |
5332 | is_ref: true, |
5333 | TREE_CONSTANT (v) |
5334 | ? TYPE_SIZE_UNIT (type) |
5335 | : sz)) |
5336 | x = allocate_ptr; |
5337 | else if (TREE_CONSTANT (v)) |
5338 | { |
5339 | x = create_tmp_var_raw (type, name); |
5340 | gimple_add_tmp_var (x); |
5341 | TREE_ADDRESSABLE (x) = 1; |
5342 | x = build_fold_addr_expr_loc (clause_loc, x); |
5343 | } |
5344 | else |
5345 | { |
5346 | tree atmp |
5347 | = builtin_decl_explicit (fncode: BUILT_IN_ALLOCA_WITH_ALIGN); |
5348 | tree al = size_int (TYPE_ALIGN (TREE_TYPE (type))); |
5349 | x = build_call_expr_loc (clause_loc, atmp, 2, sz, al); |
5350 | } |
5351 | |
5352 | tree ptype = build_pointer_type (TREE_TYPE (type)); |
5353 | x = fold_convert_loc (clause_loc, ptype, x); |
5354 | tree y = create_tmp_var (ptype, name); |
5355 | gimplify_assign (y, x, ilist); |
5356 | x = y; |
5357 | tree yb = y; |
5358 | |
5359 | if (!integer_zerop (bias)) |
5360 | { |
5361 | bias = fold_convert_loc (clause_loc, pointer_sized_int_node, |
5362 | bias); |
5363 | yb = fold_convert_loc (clause_loc, pointer_sized_int_node, |
5364 | x); |
5365 | yb = fold_build2_loc (clause_loc, MINUS_EXPR, |
5366 | pointer_sized_int_node, yb, bias); |
5367 | x = fold_convert_loc (clause_loc, TREE_TYPE (x), yb); |
5368 | yb = create_tmp_var (ptype, name); |
5369 | gimplify_assign (yb, x, ilist); |
5370 | x = yb; |
5371 | } |
5372 | |
5373 | d = TREE_OPERAND (d, 0); |
5374 | if (TREE_CODE (d) == POINTER_PLUS_EXPR) |
5375 | d = TREE_OPERAND (d, 0); |
5376 | if (TREE_CODE (d) == ADDR_EXPR) |
5377 | { |
5378 | if (orig_var != var) |
5379 | { |
5380 | gcc_assert (is_variable_sized (orig_var)); |
5381 | x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), |
5382 | x); |
5383 | gimplify_assign (new_var, x, ilist); |
5384 | tree new_orig_var = lookup_decl (var: orig_var, ctx); |
5385 | tree t = build_fold_indirect_ref (new_var); |
5386 | DECL_IGNORED_P (new_var) = 0; |
5387 | TREE_THIS_NOTRAP (t) = 1; |
5388 | SET_DECL_VALUE_EXPR (new_orig_var, t); |
5389 | DECL_HAS_VALUE_EXPR_P (new_orig_var) = 1; |
5390 | } |
5391 | else |
5392 | { |
5393 | x = build2 (MEM_REF, TREE_TYPE (new_var), x, |
5394 | build_int_cst (ptype, 0)); |
5395 | SET_DECL_VALUE_EXPR (new_var, x); |
5396 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
5397 | } |
5398 | } |
5399 | else |
5400 | { |
5401 | gcc_assert (orig_var == var); |
5402 | if (TREE_CODE (d) == INDIRECT_REF) |
5403 | { |
5404 | x = create_tmp_var (ptype, name); |
5405 | TREE_ADDRESSABLE (x) = 1; |
5406 | gimplify_assign (x, yb, ilist); |
5407 | x = build_fold_addr_expr_loc (clause_loc, x); |
5408 | } |
5409 | x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x); |
5410 | gimplify_assign (new_var, x, ilist); |
5411 | } |
5412 | /* GOMP_taskgroup_reduction_register memsets the whole |
5413 | array to zero. If the initializer is zero, we don't |
5414 | need to initialize it again, just mark it as ever |
5415 | used unconditionally, i.e. cond = true. */ |
5416 | if (cond |
5417 | && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE |
5418 | && initializer_zerop (omp_reduction_init (clause: c, |
5419 | TREE_TYPE (type)))) |
5420 | { |
5421 | gimple *g = gimple_build_assign (build_simple_mem_ref (cond), |
5422 | boolean_true_node); |
5423 | gimple_seq_add_stmt (ilist, g); |
5424 | continue; |
5425 | } |
5426 | tree end = create_artificial_label (UNKNOWN_LOCATION); |
5427 | if (cond) |
5428 | { |
5429 | gimple *g; |
5430 | if (!is_parallel_ctx (ctx)) |
5431 | { |
5432 | tree condv = create_tmp_var (boolean_type_node); |
5433 | g = gimple_build_assign (condv, |
5434 | build_simple_mem_ref (cond)); |
5435 | gimple_seq_add_stmt (ilist, g); |
5436 | tree lab1 = create_artificial_label (UNKNOWN_LOCATION); |
5437 | g = gimple_build_cond (NE_EXPR, condv, |
5438 | boolean_false_node, end, lab1); |
5439 | gimple_seq_add_stmt (ilist, g); |
5440 | gimple_seq_add_stmt (ilist, gimple_build_label (label: lab1)); |
5441 | } |
5442 | g = gimple_build_assign (build_simple_mem_ref (cond), |
5443 | boolean_true_node); |
5444 | gimple_seq_add_stmt (ilist, g); |
5445 | } |
5446 | |
5447 | tree y1 = create_tmp_var (ptype); |
5448 | gimplify_assign (y1, y, ilist); |
5449 | tree i2 = NULL_TREE, y2 = NULL_TREE; |
5450 | tree body2 = NULL_TREE, end2 = NULL_TREE; |
5451 | tree y3 = NULL_TREE, y4 = NULL_TREE; |
5452 | if (task_reduction_needs_orig_p) |
5453 | { |
5454 | y3 = create_tmp_var (ptype); |
5455 | tree ref; |
5456 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) |
5457 | ref = build4 (ARRAY_REF, ptr_type_node, tskred_avar, |
5458 | size_int (task_reduction_cnt_full |
5459 | + task_reduction_cntorig - 1), |
5460 | NULL_TREE, NULL_TREE); |
5461 | else |
5462 | { |
5463 | unsigned int idx = *ctx->task_reduction_map->get (k: c); |
5464 | ref = task_reduction_read (ilist, tskred_temp, type: ptype, |
5465 | idx: 7 + 3 * idx); |
5466 | } |
5467 | gimplify_assign (y3, ref, ilist); |
5468 | } |
5469 | else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) || is_simd) |
5470 | { |
5471 | if (pass != 3) |
5472 | { |
5473 | y2 = create_tmp_var (ptype); |
5474 | gimplify_assign (y2, y, ilist); |
5475 | } |
5476 | if (is_simd || OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)) |
5477 | { |
5478 | tree ref = build_outer_var_ref (var, ctx); |
5479 | /* For ref build_outer_var_ref already performs this. */ |
5480 | if (TREE_CODE (d) == INDIRECT_REF) |
5481 | gcc_assert (omp_privatize_by_reference (var)); |
5482 | else if (TREE_CODE (d) == ADDR_EXPR) |
5483 | ref = build_fold_addr_expr (ref); |
5484 | else if (omp_privatize_by_reference (decl: var)) |
5485 | ref = build_fold_addr_expr (ref); |
5486 | ref = fold_convert_loc (clause_loc, ptype, ref); |
5487 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) |
5488 | && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)) |
5489 | { |
5490 | y3 = create_tmp_var (ptype); |
5491 | gimplify_assign (y3, unshare_expr (ref), ilist); |
5492 | } |
5493 | if (is_simd) |
5494 | { |
5495 | y4 = create_tmp_var (ptype); |
5496 | gimplify_assign (y4, ref, dlist); |
5497 | } |
5498 | } |
5499 | } |
5500 | tree i = create_tmp_var (TREE_TYPE (v)); |
5501 | gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), ilist); |
5502 | tree body = create_artificial_label (UNKNOWN_LOCATION); |
5503 | gimple_seq_add_stmt (ilist, gimple_build_label (label: body)); |
5504 | if (y2) |
5505 | { |
5506 | i2 = create_tmp_var (TREE_TYPE (v)); |
5507 | gimplify_assign (i2, build_int_cst (TREE_TYPE (v), 0), dlist); |
5508 | body2 = create_artificial_label (UNKNOWN_LOCATION); |
5509 | end2 = create_artificial_label (UNKNOWN_LOCATION); |
5510 | gimple_seq_add_stmt (dlist, gimple_build_label (label: body2)); |
5511 | } |
5512 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
5513 | { |
5514 | tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); |
5515 | tree decl_placeholder |
5516 | = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c); |
5517 | SET_DECL_VALUE_EXPR (decl_placeholder, |
5518 | build_simple_mem_ref (y1)); |
5519 | DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1; |
5520 | SET_DECL_VALUE_EXPR (placeholder, |
5521 | y3 ? build_simple_mem_ref (y3) |
5522 | : error_mark_node); |
5523 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
5524 | x = lang_hooks.decls.omp_clause_default_ctor |
5525 | (c, build_simple_mem_ref (y1), |
5526 | y3 ? build_simple_mem_ref (y3) : NULL_TREE); |
5527 | if (x) |
5528 | gimplify_and_add (x, ilist); |
5529 | if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) |
5530 | { |
5531 | gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); |
5532 | lower_omp (&tseq, ctx); |
5533 | gimple_seq_add_seq (ilist, tseq); |
5534 | } |
5535 | OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; |
5536 | if (is_simd) |
5537 | { |
5538 | SET_DECL_VALUE_EXPR (decl_placeholder, |
5539 | build_simple_mem_ref (y2)); |
5540 | SET_DECL_VALUE_EXPR (placeholder, |
5541 | build_simple_mem_ref (y4)); |
5542 | gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); |
5543 | lower_omp (&tseq, ctx); |
5544 | gimple_seq_add_seq (dlist, tseq); |
5545 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; |
5546 | } |
5547 | DECL_HAS_VALUE_EXPR_P (placeholder) = 0; |
5548 | DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 0; |
5549 | if (y2) |
5550 | { |
5551 | x = lang_hooks.decls.omp_clause_dtor |
5552 | (c, build_simple_mem_ref (y2)); |
5553 | if (x) |
5554 | gimplify_and_add (x, dlist); |
5555 | } |
5556 | } |
5557 | else |
5558 | { |
5559 | x = omp_reduction_init (clause: c, TREE_TYPE (type)); |
5560 | enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c); |
5561 | |
5562 | /* reduction(-:var) sums up the partial results, so it |
5563 | acts identically to reduction(+:var). */ |
5564 | if (code == MINUS_EXPR) |
5565 | code = PLUS_EXPR; |
5566 | |
5567 | gimplify_assign (build_simple_mem_ref (y1), x, ilist); |
5568 | if (is_simd) |
5569 | { |
5570 | x = build2 (code, TREE_TYPE (type), |
5571 | build_simple_mem_ref (y4), |
5572 | build_simple_mem_ref (y2)); |
5573 | gimplify_assign (build_simple_mem_ref (y4), x, dlist); |
5574 | } |
5575 | } |
5576 | gimple *g |
5577 | = gimple_build_assign (y1, POINTER_PLUS_EXPR, y1, |
5578 | TYPE_SIZE_UNIT (TREE_TYPE (type))); |
5579 | gimple_seq_add_stmt (ilist, g); |
5580 | if (y3) |
5581 | { |
5582 | g = gimple_build_assign (y3, POINTER_PLUS_EXPR, y3, |
5583 | TYPE_SIZE_UNIT (TREE_TYPE (type))); |
5584 | gimple_seq_add_stmt (ilist, g); |
5585 | } |
5586 | g = gimple_build_assign (i, PLUS_EXPR, i, |
5587 | build_int_cst (TREE_TYPE (i), 1)); |
5588 | gimple_seq_add_stmt (ilist, g); |
5589 | g = gimple_build_cond (LE_EXPR, i, v, body, end); |
5590 | gimple_seq_add_stmt (ilist, g); |
5591 | gimple_seq_add_stmt (ilist, gimple_build_label (label: end)); |
5592 | if (y2) |
5593 | { |
5594 | g = gimple_build_assign (y2, POINTER_PLUS_EXPR, y2, |
5595 | TYPE_SIZE_UNIT (TREE_TYPE (type))); |
5596 | gimple_seq_add_stmt (dlist, g); |
5597 | if (y4) |
5598 | { |
5599 | g = gimple_build_assign |
5600 | (y4, POINTER_PLUS_EXPR, y4, |
5601 | TYPE_SIZE_UNIT (TREE_TYPE (type))); |
5602 | gimple_seq_add_stmt (dlist, g); |
5603 | } |
5604 | g = gimple_build_assign (i2, PLUS_EXPR, i2, |
5605 | build_int_cst (TREE_TYPE (i2), 1)); |
5606 | gimple_seq_add_stmt (dlist, g); |
5607 | g = gimple_build_cond (LE_EXPR, i2, v, body2, end2); |
5608 | gimple_seq_add_stmt (dlist, g); |
5609 | gimple_seq_add_stmt (dlist, gimple_build_label (label: end2)); |
5610 | } |
5611 | if (allocator) |
5612 | { |
5613 | tree f = builtin_decl_explicit (fncode: BUILT_IN_GOMP_FREE); |
5614 | g = gimple_build_call (f, 2, allocate_ptr, allocator); |
5615 | gimple_seq_add_stmt (dlist, g); |
5616 | } |
5617 | continue; |
5618 | } |
5619 | else if (pass == 2) |
5620 | { |
5621 | tree out = maybe_lookup_decl_in_outer_ctx (decl: var, ctx); |
5622 | if (is_global_var (t: out)) |
5623 | x = var; |
5624 | else if (is_omp_target (stmt: ctx->stmt)) |
5625 | x = out; |
5626 | else |
5627 | { |
5628 | bool by_ref = use_pointer_for_field (decl: var, shared_ctx: ctx); |
5629 | x = build_receiver_ref (var, by_ref, ctx); |
5630 | } |
5631 | if (!omp_privatize_by_reference (decl: var)) |
5632 | x = build_fold_addr_expr (x); |
5633 | x = fold_convert (ptr_type_node, x); |
5634 | unsigned cnt = task_reduction_cnt - 1; |
5635 | if (!task_reduction_needs_orig_p) |
5636 | cnt += task_reduction_cntorig_full - task_reduction_cntorig; |
5637 | else |
5638 | cnt = task_reduction_cntorig - 1; |
5639 | tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar, |
5640 | size_int (cnt), NULL_TREE, NULL_TREE); |
5641 | gimplify_assign (r, x, ilist); |
5642 | continue; |
5643 | } |
5644 | else if (pass == 3) |
5645 | { |
5646 | tree type = TREE_TYPE (new_var); |
5647 | if (!omp_privatize_by_reference (decl: var)) |
5648 | type = build_pointer_type (type); |
5649 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) |
5650 | { |
5651 | unsigned cnt = task_reduction_cnt - 1; |
5652 | if (!task_reduction_needs_orig_p) |
5653 | cnt += (task_reduction_cntorig_full |
5654 | - task_reduction_cntorig); |
5655 | else |
5656 | cnt = task_reduction_cntorig - 1; |
5657 | x = build4 (ARRAY_REF, ptr_type_node, tskred_avar, |
5658 | size_int (cnt), NULL_TREE, NULL_TREE); |
5659 | } |
5660 | else |
5661 | { |
5662 | unsigned int idx = *ctx->task_reduction_map->get (k: c); |
5663 | tree off; |
5664 | if (ctx->task_reductions[1 + idx]) |
5665 | off = fold_convert (sizetype, |
5666 | ctx->task_reductions[1 + idx]); |
5667 | else |
5668 | off = task_reduction_read (ilist, tskred_temp, sizetype, |
5669 | idx: 7 + 3 * idx + 1); |
5670 | x = fold_build2 (POINTER_PLUS_EXPR, ptr_type_node, |
5671 | tskred_base, off); |
5672 | } |
5673 | x = fold_convert (type, x); |
5674 | tree t; |
5675 | if (omp_privatize_by_reference (decl: var)) |
5676 | { |
5677 | gimplify_assign (new_var, x, ilist); |
5678 | t = new_var; |
5679 | new_var = build_simple_mem_ref (new_var); |
5680 | } |
5681 | else |
5682 | { |
5683 | t = create_tmp_var (type); |
5684 | gimplify_assign (t, x, ilist); |
5685 | SET_DECL_VALUE_EXPR (new_var, build_simple_mem_ref (t)); |
5686 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
5687 | } |
5688 | t = fold_convert (build_pointer_type (boolean_type_node), t); |
5689 | t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t, |
5690 | TYPE_SIZE_UNIT (TREE_TYPE (type))); |
5691 | cond = create_tmp_var (TREE_TYPE (t)); |
5692 | gimplify_assign (cond, t, ilist); |
5693 | } |
5694 | else if (is_variable_sized (expr: var)) |
5695 | { |
5696 | /* For variable sized types, we need to allocate the |
5697 | actual storage here. Call alloca and store the |
5698 | result in the pointer decl that we created elsewhere. */ |
5699 | if (pass == 0) |
5700 | continue; |
5701 | |
5702 | if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx)) |
5703 | { |
5704 | tree tmp; |
5705 | |
5706 | ptr = DECL_VALUE_EXPR (new_var); |
5707 | gcc_assert (TREE_CODE (ptr) == INDIRECT_REF); |
5708 | ptr = TREE_OPERAND (ptr, 0); |
5709 | gcc_assert (DECL_P (ptr)); |
5710 | x = TYPE_SIZE_UNIT (TREE_TYPE (new_var)); |
5711 | |
5712 | if (lower_private_allocate (var, new_var, allocator, |
5713 | allocate_ptr, ilist, ctx, |
5714 | is_ref: false, size: x)) |
5715 | tmp = allocate_ptr; |
5716 | else |
5717 | { |
5718 | /* void *tmp = __builtin_alloca */ |
5719 | tree atmp |
5720 | = builtin_decl_explicit (fncode: BUILT_IN_ALLOCA_WITH_ALIGN); |
5721 | gcall *stmt |
5722 | = gimple_build_call (atmp, 2, x, |
5723 | size_int (DECL_ALIGN (var))); |
5724 | cfun->calls_alloca = 1; |
5725 | tmp = create_tmp_var_raw (ptr_type_node); |
5726 | gimple_add_tmp_var (tmp); |
5727 | gimple_call_set_lhs (gs: stmt, lhs: tmp); |
5728 | |
5729 | gimple_seq_add_stmt (ilist, stmt); |
5730 | } |
5731 | |
5732 | x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp); |
5733 | gimplify_assign (ptr, x, ilist); |
5734 | } |
5735 | } |
5736 | else if (omp_privatize_by_reference (decl: var) |
5737 | && (c_kind != OMP_CLAUSE_FIRSTPRIVATE |
5738 | || !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c))) |
5739 | { |
5740 | /* For references that are being privatized for Fortran, |
5741 | allocate new backing storage for the new pointer |
5742 | variable. This allows us to avoid changing all the |
5743 | code that expects a pointer to something that expects |
5744 | a direct variable. */ |
5745 | if (pass == 0) |
5746 | continue; |
5747 | |
5748 | x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var))); |
5749 | if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx)) |
5750 | { |
5751 | x = build_receiver_ref (var, by_ref: false, ctx); |
5752 | if (ctx->allocate_map) |
5753 | if (tree *allocatep = ctx->allocate_map->get (k: var)) |
5754 | { |
5755 | allocator = *allocatep; |
5756 | if (TREE_CODE (allocator) == TREE_LIST) |
5757 | allocator = TREE_PURPOSE (allocator); |
5758 | if (TREE_CODE (allocator) != INTEGER_CST) |
5759 | allocator = build_outer_var_ref (var: allocator, ctx); |
5760 | allocator = fold_convert (pointer_sized_int_node, |
5761 | allocator); |
5762 | allocate_ptr = unshare_expr (x); |
5763 | } |
5764 | if (allocator == NULL_TREE) |
5765 | x = build_fold_addr_expr_loc (clause_loc, x); |
5766 | } |
5767 | else if (lower_private_allocate (var, new_var, allocator, |
5768 | allocate_ptr, |
5769 | ilist, ctx, is_ref: true, size: x)) |
5770 | x = allocate_ptr; |
5771 | else if (TREE_CONSTANT (x)) |
5772 | { |
5773 | /* For reduction in SIMD loop, defer adding the |
5774 | initialization of the reference, because if we decide |
5775 | to use SIMD array for it, the initilization could cause |
5776 | expansion ICE. Ditto for other privatization clauses. */ |
5777 | if (is_simd) |
5778 | x = NULL_TREE; |
5779 | else |
5780 | { |
5781 | x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)), |
5782 | get_name (var)); |
5783 | gimple_add_tmp_var (x); |
5784 | TREE_ADDRESSABLE (x) = 1; |
5785 | x = build_fold_addr_expr_loc (clause_loc, x); |
5786 | } |
5787 | } |
5788 | else |
5789 | { |
5790 | tree atmp |
5791 | = builtin_decl_explicit (fncode: BUILT_IN_ALLOCA_WITH_ALIGN); |
5792 | tree rtype = TREE_TYPE (TREE_TYPE (new_var)); |
5793 | tree al = size_int (TYPE_ALIGN (rtype)); |
5794 | x = build_call_expr_loc (clause_loc, atmp, 2, x, al); |
5795 | } |
5796 | |
5797 | if (x) |
5798 | { |
5799 | x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x); |
5800 | gimplify_assign (new_var, x, ilist); |
5801 | } |
5802 | |
5803 | new_var = build_simple_mem_ref_loc (clause_loc, new_var); |
5804 | } |
5805 | else if ((c_kind == OMP_CLAUSE_REDUCTION |
5806 | || c_kind == OMP_CLAUSE_IN_REDUCTION) |
5807 | && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
5808 | { |
5809 | if (pass == 0) |
5810 | continue; |
5811 | } |
5812 | else if (pass != 0) |
5813 | continue; |
5814 | |
5815 | switch (OMP_CLAUSE_CODE (c)) |
5816 | { |
5817 | case OMP_CLAUSE_SHARED: |
5818 | /* Ignore shared directives in teams construct inside |
5819 | target construct. */ |
5820 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_TEAMS |
5821 | && !is_host_teams_ctx (ctx)) |
5822 | continue; |
5823 | /* Shared global vars are just accessed directly. */ |
5824 | if (is_global_var (t: new_var)) |
5825 | break; |
5826 | /* For taskloop firstprivate/lastprivate, represented |
5827 | as firstprivate and shared clause on the task, new_var |
5828 | is the firstprivate var. */ |
5829 | if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)) |
5830 | break; |
5831 | /* Set up the DECL_VALUE_EXPR for shared variables now. This |
5832 | needs to be delayed until after fixup_child_record_type so |
5833 | that we get the correct type during the dereference. */ |
5834 | by_ref = use_pointer_for_field (decl: var, shared_ctx: ctx); |
5835 | x = build_receiver_ref (var, by_ref, ctx); |
5836 | SET_DECL_VALUE_EXPR (new_var, x); |
5837 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
5838 | |
5839 | /* ??? If VAR is not passed by reference, and the variable |
5840 | hasn't been initialized yet, then we'll get a warning for |
5841 | the store into the omp_data_s structure. Ideally, we'd be |
5842 | able to notice this and not store anything at all, but |
5843 | we're generating code too early. Suppress the warning. */ |
5844 | if (!by_ref) |
5845 | suppress_warning (var, OPT_Wuninitialized); |
5846 | break; |
5847 | |
5848 | case OMP_CLAUSE__CONDTEMP_: |
5849 | if (is_parallel_ctx (ctx)) |
5850 | { |
5851 | x = build_receiver_ref (var, by_ref: false, ctx); |
5852 | SET_DECL_VALUE_EXPR (new_var, x); |
5853 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
5854 | } |
5855 | else if (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c)) |
5856 | { |
5857 | x = build_zero_cst (TREE_TYPE (var)); |
5858 | goto do_private; |
5859 | } |
5860 | break; |
5861 | |
5862 | case OMP_CLAUSE_LASTPRIVATE: |
5863 | if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) |
5864 | break; |
5865 | /* FALLTHRU */ |
5866 | |
5867 | case OMP_CLAUSE_PRIVATE: |
5868 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE) |
5869 | x = build_outer_var_ref (var, ctx); |
5870 | else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c)) |
5871 | { |
5872 | if (is_task_ctx (ctx)) |
5873 | x = build_receiver_ref (var, by_ref: false, ctx); |
5874 | else |
5875 | x = build_outer_var_ref (var, ctx, code: OMP_CLAUSE_PRIVATE); |
5876 | } |
5877 | else |
5878 | x = NULL; |
5879 | do_private: |
5880 | tree nx; |
5881 | bool copy_ctor; |
5882 | copy_ctor = false; |
5883 | lower_private_allocate (var, new_var, allocator, allocate_ptr, |
5884 | ilist, ctx, is_ref: false, NULL_TREE); |
5885 | nx = unshare_expr (new_var); |
5886 | if (is_simd |
5887 | && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
5888 | && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c)) |
5889 | copy_ctor = true; |
5890 | if (copy_ctor) |
5891 | nx = lang_hooks.decls.omp_clause_copy_ctor (c, nx, x); |
5892 | else |
5893 | nx = lang_hooks.decls.omp_clause_default_ctor (c, nx, x); |
5894 | if (is_simd) |
5895 | { |
5896 | tree y = lang_hooks.decls.omp_clause_dtor (c, new_var); |
5897 | if ((TREE_ADDRESSABLE (new_var) || nx || y |
5898 | || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
5899 | && (gimple_omp_for_collapse (gs: ctx->stmt) != 1 |
5900 | || (gimple_omp_for_index (gs: ctx->stmt, i: 0) |
5901 | != new_var))) |
5902 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_ |
5903 | || omp_privatize_by_reference (decl: var)) |
5904 | && lower_rec_simd_input_clauses (new_var, ctx, sctx: &sctx, |
5905 | ivar, lvar)) |
5906 | { |
5907 | if (omp_privatize_by_reference (decl: var)) |
5908 | { |
5909 | gcc_assert (TREE_CODE (new_var) == MEM_REF); |
5910 | tree new_vard = TREE_OPERAND (new_var, 0); |
5911 | gcc_assert (DECL_P (new_vard)); |
5912 | SET_DECL_VALUE_EXPR (new_vard, |
5913 | build_fold_addr_expr (lvar)); |
5914 | DECL_HAS_VALUE_EXPR_P (new_vard) = 1; |
5915 | } |
5916 | |
5917 | if (nx) |
5918 | { |
5919 | tree iv = unshare_expr (ivar); |
5920 | if (copy_ctor) |
5921 | x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, |
5922 | x); |
5923 | else |
5924 | x = lang_hooks.decls.omp_clause_default_ctor (c, |
5925 | iv, |
5926 | x); |
5927 | } |
5928 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_) |
5929 | { |
5930 | x = build2 (MODIFY_EXPR, TREE_TYPE (ivar), |
5931 | unshare_expr (ivar), x); |
5932 | nx = x; |
5933 | } |
5934 | if (nx && x) |
5935 | gimplify_and_add (x, &llist[0]); |
5936 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
5937 | && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)) |
5938 | { |
5939 | tree v = new_var; |
5940 | if (!DECL_P (v)) |
5941 | { |
5942 | gcc_assert (TREE_CODE (v) == MEM_REF); |
5943 | v = TREE_OPERAND (v, 0); |
5944 | gcc_assert (DECL_P (v)); |
5945 | } |
5946 | v = *ctx->lastprivate_conditional_map->get (k: v); |
5947 | tree t = create_tmp_var (TREE_TYPE (v)); |
5948 | tree z = build_zero_cst (TREE_TYPE (v)); |
5949 | tree orig_v |
5950 | = build_outer_var_ref (var, ctx, |
5951 | code: OMP_CLAUSE_LASTPRIVATE); |
5952 | gimple_seq_add_stmt (dlist, |
5953 | gimple_build_assign (t, z)); |
5954 | gcc_assert (DECL_HAS_VALUE_EXPR_P (v)); |
5955 | tree civar = DECL_VALUE_EXPR (v); |
5956 | gcc_assert (TREE_CODE (civar) == ARRAY_REF); |
5957 | civar = unshare_expr (civar); |
5958 | TREE_OPERAND (civar, 1) = sctx.idx; |
5959 | x = build2 (MODIFY_EXPR, TREE_TYPE (t), t, |
5960 | unshare_expr (civar)); |
5961 | x = build2 (COMPOUND_EXPR, TREE_TYPE (orig_v), x, |
5962 | build2 (MODIFY_EXPR, TREE_TYPE (orig_v), |
5963 | orig_v, unshare_expr (ivar))); |
5964 | tree cond = build2 (LT_EXPR, boolean_type_node, t, |
5965 | civar); |
5966 | x = build3 (COND_EXPR, void_type_node, cond, x, |
5967 | void_node); |
5968 | gimple_seq tseq = NULL; |
5969 | gimplify_and_add (x, &tseq); |
5970 | if (ctx->outer) |
5971 | lower_omp (&tseq, ctx->outer); |
5972 | gimple_seq_add_seq (&llist[1], tseq); |
5973 | } |
5974 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
5975 | && ctx->for_simd_scan_phase) |
5976 | { |
5977 | x = unshare_expr (ivar); |
5978 | tree orig_v |
5979 | = build_outer_var_ref (var, ctx, |
5980 | code: OMP_CLAUSE_LASTPRIVATE); |
5981 | x = lang_hooks.decls.omp_clause_assign_op (c, x, |
5982 | orig_v); |
5983 | gimplify_and_add (x, &llist[0]); |
5984 | } |
5985 | if (y) |
5986 | { |
5987 | y = lang_hooks.decls.omp_clause_dtor (c, ivar); |
5988 | if (y) |
5989 | gimplify_and_add (y, &llist[1]); |
5990 | } |
5991 | break; |
5992 | } |
5993 | if (omp_privatize_by_reference (decl: var)) |
5994 | { |
5995 | gcc_assert (TREE_CODE (new_var) == MEM_REF); |
5996 | tree new_vard = TREE_OPERAND (new_var, 0); |
5997 | gcc_assert (DECL_P (new_vard)); |
5998 | tree type = TREE_TYPE (TREE_TYPE (new_vard)); |
5999 | x = TYPE_SIZE_UNIT (type); |
6000 | if (TREE_CONSTANT (x)) |
6001 | { |
6002 | x = create_tmp_var_raw (type, get_name (var)); |
6003 | gimple_add_tmp_var (x); |
6004 | TREE_ADDRESSABLE (x) = 1; |
6005 | x = build_fold_addr_expr_loc (clause_loc, x); |
6006 | x = fold_convert_loc (clause_loc, |
6007 | TREE_TYPE (new_vard), x); |
6008 | gimplify_assign (new_vard, x, ilist); |
6009 | } |
6010 | } |
6011 | } |
6012 | if (nx) |
6013 | gimplify_and_add (nx, ilist); |
6014 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
6015 | && is_simd |
6016 | && ctx->for_simd_scan_phase) |
6017 | { |
6018 | tree orig_v = build_outer_var_ref (var, ctx, |
6019 | code: OMP_CLAUSE_LASTPRIVATE); |
6020 | x = lang_hooks.decls.omp_clause_assign_op (c, new_var, |
6021 | orig_v); |
6022 | gimplify_and_add (x, ilist); |
6023 | } |
6024 | /* FALLTHRU */ |
6025 | |
6026 | do_dtor: |
6027 | x = lang_hooks.decls.omp_clause_dtor (c, new_var); |
6028 | if (x) |
6029 | gimplify_and_add (x, dlist); |
6030 | if (allocator) |
6031 | { |
6032 | if (!is_gimple_val (allocator)) |
6033 | { |
6034 | tree avar = create_tmp_var (TREE_TYPE (allocator)); |
6035 | gimplify_assign (avar, allocator, dlist); |
6036 | allocator = avar; |
6037 | } |
6038 | if (!is_gimple_val (allocate_ptr)) |
6039 | { |
6040 | tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr)); |
6041 | gimplify_assign (apvar, allocate_ptr, dlist); |
6042 | allocate_ptr = apvar; |
6043 | } |
6044 | tree f = builtin_decl_explicit (fncode: BUILT_IN_GOMP_FREE); |
6045 | gimple *g |
6046 | = gimple_build_call (f, 2, allocate_ptr, allocator); |
6047 | gimple_seq_add_stmt (dlist, g); |
6048 | } |
6049 | break; |
6050 | |
6051 | case OMP_CLAUSE_LINEAR: |
6052 | if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c)) |
6053 | goto do_firstprivate; |
6054 | if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c)) |
6055 | x = NULL; |
6056 | else |
6057 | x = build_outer_var_ref (var, ctx); |
6058 | goto do_private; |
6059 | |
6060 | case OMP_CLAUSE_FIRSTPRIVATE: |
6061 | if (is_task_ctx (ctx)) |
6062 | { |
6063 | if ((omp_privatize_by_reference (decl: var) |
6064 | && !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)) |
6065 | || is_variable_sized (expr: var)) |
6066 | goto do_dtor; |
6067 | else if (is_global_var (t: maybe_lookup_decl_in_outer_ctx (decl: var, |
6068 | ctx)) |
6069 | || use_pointer_for_field (decl: var, NULL)) |
6070 | { |
6071 | x = build_receiver_ref (var, by_ref: false, ctx); |
6072 | if (ctx->allocate_map) |
6073 | if (tree *allocatep = ctx->allocate_map->get (k: var)) |
6074 | { |
6075 | allocator = *allocatep; |
6076 | if (TREE_CODE (allocator) == TREE_LIST) |
6077 | allocator = TREE_PURPOSE (allocator); |
6078 | if (TREE_CODE (allocator) != INTEGER_CST) |
6079 | allocator = build_outer_var_ref (var: allocator, ctx); |
6080 | allocator = fold_convert (pointer_sized_int_node, |
6081 | allocator); |
6082 | allocate_ptr = unshare_expr (x); |
6083 | x = build_simple_mem_ref (x); |
6084 | TREE_THIS_NOTRAP (x) = 1; |
6085 | } |
6086 | SET_DECL_VALUE_EXPR (new_var, x); |
6087 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
6088 | goto do_dtor; |
6089 | } |
6090 | } |
6091 | if (OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c) |
6092 | && omp_privatize_by_reference (decl: var)) |
6093 | { |
6094 | x = build_outer_var_ref (var, ctx); |
6095 | gcc_assert (TREE_CODE (x) == MEM_REF |
6096 | && integer_zerop (TREE_OPERAND (x, 1))); |
6097 | x = TREE_OPERAND (x, 0); |
6098 | x = lang_hooks.decls.omp_clause_copy_ctor |
6099 | (c, unshare_expr (new_var), x); |
6100 | gimplify_and_add (x, ilist); |
6101 | goto do_dtor; |
6102 | } |
6103 | do_firstprivate: |
6104 | lower_private_allocate (var, new_var, allocator, allocate_ptr, |
6105 | ilist, ctx, is_ref: false, NULL_TREE); |
6106 | x = build_outer_var_ref (var, ctx); |
6107 | if (is_simd) |
6108 | { |
6109 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR |
6110 | && gimple_omp_for_combined_into_p (g: ctx->stmt)) |
6111 | { |
6112 | tree t = OMP_CLAUSE_LINEAR_STEP (c); |
6113 | if (DECL_P (t)) |
6114 | t = build_outer_var_ref (var: t, ctx); |
6115 | tree stept = TREE_TYPE (t); |
6116 | tree ct = omp_find_clause (clauses, |
6117 | kind: OMP_CLAUSE__LOOPTEMP_); |
6118 | gcc_assert (ct); |
6119 | tree l = OMP_CLAUSE_DECL (ct); |
6120 | tree n1 = fd->loop.n1; |
6121 | tree step = fd->loop.step; |
6122 | tree itype = TREE_TYPE (l); |
6123 | if (POINTER_TYPE_P (itype)) |
6124 | itype = signed_type_for (itype); |
6125 | l = fold_build2 (MINUS_EXPR, itype, l, n1); |
6126 | if (TYPE_UNSIGNED (itype) |
6127 | && fd->loop.cond_code == GT_EXPR) |
6128 | l = fold_build2 (TRUNC_DIV_EXPR, itype, |
6129 | fold_build1 (NEGATE_EXPR, itype, l), |
6130 | fold_build1 (NEGATE_EXPR, |
6131 | itype, step)); |
6132 | else |
6133 | l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step); |
6134 | t = fold_build2 (MULT_EXPR, stept, |
6135 | fold_convert (stept, l), t); |
6136 | |
6137 | if (OMP_CLAUSE_LINEAR_ARRAY (c)) |
6138 | { |
6139 | if (omp_privatize_by_reference (decl: var)) |
6140 | { |
6141 | gcc_assert (TREE_CODE (new_var) == MEM_REF); |
6142 | tree new_vard = TREE_OPERAND (new_var, 0); |
6143 | gcc_assert (DECL_P (new_vard)); |
6144 | tree type = TREE_TYPE (TREE_TYPE (new_vard)); |
6145 | nx = TYPE_SIZE_UNIT (type); |
6146 | if (TREE_CONSTANT (nx)) |
6147 | { |
6148 | nx = create_tmp_var_raw (type, |
6149 | get_name (var)); |
6150 | gimple_add_tmp_var (nx); |
6151 | TREE_ADDRESSABLE (nx) = 1; |
6152 | nx = build_fold_addr_expr_loc (clause_loc, |
6153 | nx); |
6154 | nx = fold_convert_loc (clause_loc, |
6155 | TREE_TYPE (new_vard), |
6156 | nx); |
6157 | gimplify_assign (new_vard, nx, ilist); |
6158 | } |
6159 | } |
6160 | |
6161 | x = lang_hooks.decls.omp_clause_linear_ctor |
6162 | (c, new_var, x, t); |
6163 | gimplify_and_add (x, ilist); |
6164 | goto do_dtor; |
6165 | } |
6166 | |
6167 | if (POINTER_TYPE_P (TREE_TYPE (x))) |
6168 | x = fold_build_pointer_plus (x, t); |
6169 | else |
6170 | x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x, |
6171 | fold_convert (TREE_TYPE (x), t)); |
6172 | } |
6173 | |
6174 | if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR |
6175 | || TREE_ADDRESSABLE (new_var) |
6176 | || omp_privatize_by_reference (decl: var)) |
6177 | && lower_rec_simd_input_clauses (new_var, ctx, sctx: &sctx, |
6178 | ivar, lvar)) |
6179 | { |
6180 | if (omp_privatize_by_reference (decl: var)) |
6181 | { |
6182 | gcc_assert (TREE_CODE (new_var) == MEM_REF); |
6183 | tree new_vard = TREE_OPERAND (new_var, 0); |
6184 | gcc_assert (DECL_P (new_vard)); |
6185 | SET_DECL_VALUE_EXPR (new_vard, |
6186 | build_fold_addr_expr (lvar)); |
6187 | DECL_HAS_VALUE_EXPR_P (new_vard) = 1; |
6188 | } |
6189 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR) |
6190 | { |
6191 | tree iv = create_tmp_var (TREE_TYPE (new_var)); |
6192 | x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x); |
6193 | gimplify_and_add (x, ilist); |
6194 | gimple_stmt_iterator gsi |
6195 | = gsi_start (seq&: *gimple_omp_body_ptr (gs: ctx->stmt)); |
6196 | gassign *g |
6197 | = gimple_build_assign (unshare_expr (lvar), iv); |
6198 | gsi_insert_before_without_update (&gsi, g, |
6199 | GSI_SAME_STMT); |
6200 | tree t = OMP_CLAUSE_LINEAR_STEP (c); |
6201 | enum tree_code code = PLUS_EXPR; |
6202 | if (POINTER_TYPE_P (TREE_TYPE (new_var))) |
6203 | code = POINTER_PLUS_EXPR; |
6204 | g = gimple_build_assign (iv, code, iv, t); |
6205 | gsi_insert_before_without_update (&gsi, g, |
6206 | GSI_SAME_STMT); |
6207 | break; |
6208 | } |
6209 | x = lang_hooks.decls.omp_clause_copy_ctor |
6210 | (c, unshare_expr (ivar), x); |
6211 | gimplify_and_add (x, &llist[0]); |
6212 | x = lang_hooks.decls.omp_clause_dtor (c, ivar); |
6213 | if (x) |
6214 | gimplify_and_add (x, &llist[1]); |
6215 | break; |
6216 | } |
6217 | if (omp_privatize_by_reference (decl: var)) |
6218 | { |
6219 | gcc_assert (TREE_CODE (new_var) == MEM_REF); |
6220 | tree new_vard = TREE_OPERAND (new_var, 0); |
6221 | gcc_assert (DECL_P (new_vard)); |
6222 | tree type = TREE_TYPE (TREE_TYPE (new_vard)); |
6223 | nx = TYPE_SIZE_UNIT (type); |
6224 | if (TREE_CONSTANT (nx)) |
6225 | { |
6226 | nx = create_tmp_var_raw (type, get_name (var)); |
6227 | gimple_add_tmp_var (nx); |
6228 | TREE_ADDRESSABLE (nx) = 1; |
6229 | nx = build_fold_addr_expr_loc (clause_loc, nx); |
6230 | nx = fold_convert_loc (clause_loc, |
6231 | TREE_TYPE (new_vard), nx); |
6232 | gimplify_assign (new_vard, nx, ilist); |
6233 | } |
6234 | } |
6235 | } |
6236 | x = lang_hooks.decls.omp_clause_copy_ctor |
6237 | (c, unshare_expr (new_var), x); |
6238 | gimplify_and_add (x, ilist); |
6239 | goto do_dtor; |
6240 | |
6241 | case OMP_CLAUSE__LOOPTEMP_: |
6242 | case OMP_CLAUSE__REDUCTEMP_: |
6243 | gcc_assert (is_taskreg_ctx (ctx)); |
6244 | x = build_outer_var_ref (var, ctx); |
6245 | x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x); |
6246 | gimplify_and_add (x, ilist); |
6247 | break; |
6248 | |
6249 | case OMP_CLAUSE_COPYIN: |
6250 | by_ref = use_pointer_for_field (decl: var, NULL); |
6251 | x = build_receiver_ref (var, by_ref, ctx); |
6252 | x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x); |
6253 | append_to_statement_list (x, ©in_seq); |
6254 | copyin_by_ref |= by_ref; |
6255 | break; |
6256 | |
6257 | case OMP_CLAUSE_REDUCTION: |
6258 | case OMP_CLAUSE_IN_REDUCTION: |
6259 | /* OpenACC reductions are initialized using the |
6260 | GOACC_REDUCTION internal function. */ |
6261 | if (is_gimple_omp_oacc (stmt: ctx->stmt)) |
6262 | break; |
6263 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
6264 | { |
6265 | tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); |
6266 | gimple *tseq; |
6267 | tree ptype = TREE_TYPE (placeholder); |
6268 | if (cond) |
6269 | { |
6270 | x = error_mark_node; |
6271 | if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c) |
6272 | && !task_reduction_needs_orig_p) |
6273 | x = var; |
6274 | else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)) |
6275 | { |
6276 | tree pptype = build_pointer_type (ptype); |
6277 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) |
6278 | x = build4 (ARRAY_REF, ptr_type_node, tskred_avar, |
6279 | size_int (task_reduction_cnt_full |
6280 | + task_reduction_cntorig - 1), |
6281 | NULL_TREE, NULL_TREE); |
6282 | else |
6283 | { |
6284 | unsigned int idx |
6285 | = *ctx->task_reduction_map->get (k: c); |
6286 | x = task_reduction_read (ilist, tskred_temp, |
6287 | type: pptype, idx: 7 + 3 * idx); |
6288 | } |
6289 | x = fold_convert (pptype, x); |
6290 | x = build_simple_mem_ref (x); |
6291 | } |
6292 | } |
6293 | else |
6294 | { |
6295 | lower_private_allocate (var, new_var, allocator, |
6296 | allocate_ptr, ilist, ctx, is_ref: false, |
6297 | NULL_TREE); |
6298 | x = build_outer_var_ref (var, ctx); |
6299 | |
6300 | if (omp_privatize_by_reference (decl: var) |
6301 | && !useless_type_conversion_p (ptype, TREE_TYPE (x))) |
6302 | x = build_fold_addr_expr_loc (clause_loc, x); |
6303 | } |
6304 | SET_DECL_VALUE_EXPR (placeholder, x); |
6305 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
6306 | tree new_vard = new_var; |
6307 | if (omp_privatize_by_reference (decl: var)) |
6308 | { |
6309 | gcc_assert (TREE_CODE (new_var) == MEM_REF); |
6310 | new_vard = TREE_OPERAND (new_var, 0); |
6311 | gcc_assert (DECL_P (new_vard)); |
6312 | } |
6313 | tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE; |
6314 | if (is_simd |
6315 | && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
6316 | && OMP_CLAUSE_REDUCTION_INSCAN (c)) |
6317 | rvarp = &rvar; |
6318 | if (is_simd |
6319 | && lower_rec_simd_input_clauses (new_var, ctx, sctx: &sctx, |
6320 | ivar, lvar, rvar: rvarp, |
6321 | rvar2: &rvar2)) |
6322 | { |
6323 | if (new_vard == new_var) |
6324 | { |
6325 | gcc_assert (DECL_VALUE_EXPR (new_var) == lvar); |
6326 | SET_DECL_VALUE_EXPR (new_var, ivar); |
6327 | } |
6328 | else |
6329 | { |
6330 | SET_DECL_VALUE_EXPR (new_vard, |
6331 | build_fold_addr_expr (ivar)); |
6332 | DECL_HAS_VALUE_EXPR_P (new_vard) = 1; |
6333 | } |
6334 | x = lang_hooks.decls.omp_clause_default_ctor |
6335 | (c, unshare_expr (ivar), |
6336 | build_outer_var_ref (var, ctx)); |
6337 | if (rvarp && ctx->for_simd_scan_phase) |
6338 | { |
6339 | if (x) |
6340 | gimplify_and_add (x, &llist[0]); |
6341 | x = lang_hooks.decls.omp_clause_dtor (c, ivar); |
6342 | if (x) |
6343 | gimplify_and_add (x, &llist[1]); |
6344 | break; |
6345 | } |
6346 | else if (rvarp) |
6347 | { |
6348 | if (x) |
6349 | { |
6350 | gimplify_and_add (x, &llist[0]); |
6351 | |
6352 | tree ivar2 = unshare_expr (lvar); |
6353 | TREE_OPERAND (ivar2, 1) = sctx.idx; |
6354 | x = lang_hooks.decls.omp_clause_default_ctor |
6355 | (c, ivar2, build_outer_var_ref (var, ctx)); |
6356 | gimplify_and_add (x, &llist[0]); |
6357 | |
6358 | if (rvar2) |
6359 | { |
6360 | x = lang_hooks.decls.omp_clause_default_ctor |
6361 | (c, unshare_expr (rvar2), |
6362 | build_outer_var_ref (var, ctx)); |
6363 | gimplify_and_add (x, &llist[0]); |
6364 | } |
6365 | |
6366 | /* For types that need construction, add another |
6367 | private var which will be default constructed |
6368 | and optionally initialized with |
6369 | OMP_CLAUSE_REDUCTION_GIMPLE_INIT, as in the |
6370 | loop we want to assign this value instead of |
6371 | constructing and destructing it in each |
6372 | iteration. */ |
6373 | tree nv = create_tmp_var_raw (TREE_TYPE (ivar)); |
6374 | gimple_add_tmp_var (nv); |
6375 | ctx->cb.decl_map->put (TREE_OPERAND (rvar2 |
6376 | ? rvar2 |
6377 | : ivar, 0), |
6378 | v: nv); |
6379 | x = lang_hooks.decls.omp_clause_default_ctor |
6380 | (c, nv, build_outer_var_ref (var, ctx)); |
6381 | gimplify_and_add (x, ilist); |
6382 | |
6383 | if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) |
6384 | { |
6385 | tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); |
6386 | x = DECL_VALUE_EXPR (new_vard); |
6387 | tree vexpr = nv; |
6388 | if (new_vard != new_var) |
6389 | vexpr = build_fold_addr_expr (nv); |
6390 | SET_DECL_VALUE_EXPR (new_vard, vexpr); |
6391 | lower_omp (&tseq, ctx); |
6392 | SET_DECL_VALUE_EXPR (new_vard, x); |
6393 | gimple_seq_add_seq (ilist, tseq); |
6394 | OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; |
6395 | } |
6396 | |
6397 | x = lang_hooks.decls.omp_clause_dtor (c, nv); |
6398 | if (x) |
6399 | gimplify_and_add (x, dlist); |
6400 | } |
6401 | |
6402 | tree ref = build_outer_var_ref (var, ctx); |
6403 | x = unshare_expr (ivar); |
6404 | x = lang_hooks.decls.omp_clause_assign_op (c, x, |
6405 | ref); |
6406 | gimplify_and_add (x, &llist[0]); |
6407 | |
6408 | ref = build_outer_var_ref (var, ctx); |
6409 | x = lang_hooks.decls.omp_clause_assign_op (c, ref, |
6410 | rvar); |
6411 | gimplify_and_add (x, &llist[3]); |
6412 | |
6413 | DECL_HAS_VALUE_EXPR_P (placeholder) = 0; |
6414 | if (new_vard == new_var) |
6415 | SET_DECL_VALUE_EXPR (new_var, lvar); |
6416 | else |
6417 | SET_DECL_VALUE_EXPR (new_vard, |
6418 | build_fold_addr_expr (lvar)); |
6419 | |
6420 | x = lang_hooks.decls.omp_clause_dtor (c, ivar); |
6421 | if (x) |
6422 | gimplify_and_add (x, &llist[1]); |
6423 | |
6424 | tree ivar2 = unshare_expr (lvar); |
6425 | TREE_OPERAND (ivar2, 1) = sctx.idx; |
6426 | x = lang_hooks.decls.omp_clause_dtor (c, ivar2); |
6427 | if (x) |
6428 | gimplify_and_add (x, &llist[1]); |
6429 | |
6430 | if (rvar2) |
6431 | { |
6432 | x = lang_hooks.decls.omp_clause_dtor (c, rvar2); |
6433 | if (x) |
6434 | gimplify_and_add (x, &llist[1]); |
6435 | } |
6436 | break; |
6437 | } |
6438 | if (x) |
6439 | gimplify_and_add (x, &llist[0]); |
6440 | if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) |
6441 | { |
6442 | tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); |
6443 | lower_omp (&tseq, ctx); |
6444 | gimple_seq_add_seq (&llist[0], tseq); |
6445 | } |
6446 | OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; |
6447 | tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); |
6448 | lower_omp (&tseq, ctx); |
6449 | gimple_seq_add_seq (&llist[1], tseq); |
6450 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; |
6451 | DECL_HAS_VALUE_EXPR_P (placeholder) = 0; |
6452 | if (new_vard == new_var) |
6453 | SET_DECL_VALUE_EXPR (new_var, lvar); |
6454 | else |
6455 | SET_DECL_VALUE_EXPR (new_vard, |
6456 | build_fold_addr_expr (lvar)); |
6457 | x = lang_hooks.decls.omp_clause_dtor (c, ivar); |
6458 | if (x) |
6459 | gimplify_and_add (x, &llist[1]); |
6460 | break; |
6461 | } |
6462 | /* If this is a reference to constant size reduction var |
6463 | with placeholder, we haven't emitted the initializer |
6464 | for it because it is undesirable if SIMD arrays are used. |
6465 | But if they aren't used, we need to emit the deferred |
6466 | initialization now. */ |
6467 | else if (omp_privatize_by_reference (decl: var) && is_simd) |
6468 | handle_simd_reference (loc: clause_loc, new_vard, ilist); |
6469 | |
6470 | tree lab2 = NULL_TREE; |
6471 | if (cond) |
6472 | { |
6473 | gimple *g; |
6474 | if (!is_parallel_ctx (ctx)) |
6475 | { |
6476 | tree condv = create_tmp_var (boolean_type_node); |
6477 | tree m = build_simple_mem_ref (cond); |
6478 | g = gimple_build_assign (condv, m); |
6479 | gimple_seq_add_stmt (ilist, g); |
6480 | tree lab1 |
6481 | = create_artificial_label (UNKNOWN_LOCATION); |
6482 | lab2 = create_artificial_label (UNKNOWN_LOCATION); |
6483 | g = gimple_build_cond (NE_EXPR, condv, |
6484 | boolean_false_node, |
6485 | lab2, lab1); |
6486 | gimple_seq_add_stmt (ilist, g); |
6487 | gimple_seq_add_stmt (ilist, |
6488 | gimple_build_label (label: lab1)); |
6489 | } |
6490 | g = gimple_build_assign (build_simple_mem_ref (cond), |
6491 | boolean_true_node); |
6492 | gimple_seq_add_stmt (ilist, g); |
6493 | } |
6494 | x = lang_hooks.decls.omp_clause_default_ctor |
6495 | (c, unshare_expr (new_var), |
6496 | cond ? NULL_TREE |
6497 | : build_outer_var_ref (var, ctx)); |
6498 | if (x) |
6499 | gimplify_and_add (x, ilist); |
6500 | |
6501 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
6502 | && OMP_CLAUSE_REDUCTION_INSCAN (c)) |
6503 | { |
6504 | if (ctx->for_simd_scan_phase) |
6505 | goto do_dtor; |
6506 | if (x || (!is_simd |
6507 | && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))) |
6508 | { |
6509 | tree nv = create_tmp_var_raw (TREE_TYPE (new_var)); |
6510 | gimple_add_tmp_var (nv); |
6511 | ctx->cb.decl_map->put (k: new_vard, v: nv); |
6512 | x = lang_hooks.decls.omp_clause_default_ctor |
6513 | (c, nv, build_outer_var_ref (var, ctx)); |
6514 | if (x) |
6515 | gimplify_and_add (x, ilist); |
6516 | if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) |
6517 | { |
6518 | tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); |
6519 | tree vexpr = nv; |
6520 | if (new_vard != new_var) |
6521 | vexpr = build_fold_addr_expr (nv); |
6522 | SET_DECL_VALUE_EXPR (new_vard, vexpr); |
6523 | DECL_HAS_VALUE_EXPR_P (new_vard) = 1; |
6524 | lower_omp (&tseq, ctx); |
6525 | SET_DECL_VALUE_EXPR (new_vard, NULL_TREE); |
6526 | DECL_HAS_VALUE_EXPR_P (new_vard) = 0; |
6527 | gimple_seq_add_seq (ilist, tseq); |
6528 | } |
6529 | OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; |
6530 | if (is_simd && ctx->scan_exclusive) |
6531 | { |
6532 | tree nv2 |
6533 | = create_tmp_var_raw (TREE_TYPE (new_var)); |
6534 | gimple_add_tmp_var (nv2); |
6535 | ctx->cb.decl_map->put (k: nv, v: nv2); |
6536 | x = lang_hooks.decls.omp_clause_default_ctor |
6537 | (c, nv2, build_outer_var_ref (var, ctx)); |
6538 | gimplify_and_add (x, ilist); |
6539 | x = lang_hooks.decls.omp_clause_dtor (c, nv2); |
6540 | if (x) |
6541 | gimplify_and_add (x, dlist); |
6542 | } |
6543 | x = lang_hooks.decls.omp_clause_dtor (c, nv); |
6544 | if (x) |
6545 | gimplify_and_add (x, dlist); |
6546 | } |
6547 | else if (is_simd |
6548 | && ctx->scan_exclusive |
6549 | && TREE_ADDRESSABLE (TREE_TYPE (new_var))) |
6550 | { |
6551 | tree nv2 = create_tmp_var_raw (TREE_TYPE (new_var)); |
6552 | gimple_add_tmp_var (nv2); |
6553 | ctx->cb.decl_map->put (k: new_vard, v: nv2); |
6554 | x = lang_hooks.decls.omp_clause_dtor (c, nv2); |
6555 | if (x) |
6556 | gimplify_and_add (x, dlist); |
6557 | } |
6558 | DECL_HAS_VALUE_EXPR_P (placeholder) = 0; |
6559 | goto do_dtor; |
6560 | } |
6561 | |
6562 | if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) |
6563 | { |
6564 | tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); |
6565 | if (c_kind == OMP_CLAUSE_IN_REDUCTION |
6566 | && is_omp_target (stmt: ctx->stmt)) |
6567 | { |
6568 | tree d = maybe_lookup_decl_in_outer_ctx (decl: var, ctx); |
6569 | tree oldv = NULL_TREE; |
6570 | gcc_assert (d); |
6571 | if (DECL_HAS_VALUE_EXPR_P (d)) |
6572 | oldv = DECL_VALUE_EXPR (d); |
6573 | SET_DECL_VALUE_EXPR (d, new_vard); |
6574 | DECL_HAS_VALUE_EXPR_P (d) = 1; |
6575 | lower_omp (&tseq, ctx); |
6576 | if (oldv) |
6577 | SET_DECL_VALUE_EXPR (d, oldv); |
6578 | else |
6579 | { |
6580 | SET_DECL_VALUE_EXPR (d, NULL_TREE); |
6581 | DECL_HAS_VALUE_EXPR_P (d) = 0; |
6582 | } |
6583 | } |
6584 | else |
6585 | lower_omp (&tseq, ctx); |
6586 | gimple_seq_add_seq (ilist, tseq); |
6587 | } |
6588 | OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; |
6589 | if (is_simd) |
6590 | { |
6591 | tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); |
6592 | lower_omp (&tseq, ctx); |
6593 | gimple_seq_add_seq (dlist, tseq); |
6594 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; |
6595 | } |
6596 | DECL_HAS_VALUE_EXPR_P (placeholder) = 0; |
6597 | if (cond) |
6598 | { |
6599 | if (lab2) |
6600 | gimple_seq_add_stmt (ilist, gimple_build_label (label: lab2)); |
6601 | break; |
6602 | } |
6603 | goto do_dtor; |
6604 | } |
6605 | else |
6606 | { |
6607 | x = omp_reduction_init (clause: c, TREE_TYPE (new_var)); |
6608 | gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE); |
6609 | enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c); |
6610 | |
6611 | if (cond) |
6612 | { |
6613 | gimple *g; |
6614 | tree lab2 = NULL_TREE; |
6615 | /* GOMP_taskgroup_reduction_register memsets the whole |
6616 | array to zero. If the initializer is zero, we don't |
6617 | need to initialize it again, just mark it as ever |
6618 | used unconditionally, i.e. cond = true. */ |
6619 | if (initializer_zerop (x)) |
6620 | { |
6621 | g = gimple_build_assign (build_simple_mem_ref (cond), |
6622 | boolean_true_node); |
6623 | gimple_seq_add_stmt (ilist, g); |
6624 | break; |
6625 | } |
6626 | |
6627 | /* Otherwise, emit |
6628 | if (!cond) { cond = true; new_var = x; } */ |
6629 | if (!is_parallel_ctx (ctx)) |
6630 | { |
6631 | tree condv = create_tmp_var (boolean_type_node); |
6632 | tree m = build_simple_mem_ref (cond); |
6633 | g = gimple_build_assign (condv, m); |
6634 | gimple_seq_add_stmt (ilist, g); |
6635 | tree lab1 |
6636 | = create_artificial_label (UNKNOWN_LOCATION); |
6637 | lab2 = create_artificial_label (UNKNOWN_LOCATION); |
6638 | g = gimple_build_cond (NE_EXPR, condv, |
6639 | boolean_false_node, |
6640 | lab2, lab1); |
6641 | gimple_seq_add_stmt (ilist, g); |
6642 | gimple_seq_add_stmt (ilist, |
6643 | gimple_build_label (label: lab1)); |
6644 | } |
6645 | g = gimple_build_assign (build_simple_mem_ref (cond), |
6646 | boolean_true_node); |
6647 | gimple_seq_add_stmt (ilist, g); |
6648 | gimplify_assign (new_var, x, ilist); |
6649 | if (lab2) |
6650 | gimple_seq_add_stmt (ilist, gimple_build_label (label: lab2)); |
6651 | break; |
6652 | } |
6653 | |
6654 | /* reduction(-:var) sums up the partial results, so it |
6655 | acts identically to reduction(+:var). */ |
6656 | if (code == MINUS_EXPR) |
6657 | code = PLUS_EXPR; |
6658 | |
6659 | bool is_truth_op |
6660 | = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR); |
6661 | tree new_vard = new_var; |
6662 | if (is_simd && omp_privatize_by_reference (decl: var)) |
6663 | { |
6664 | gcc_assert (TREE_CODE (new_var) == MEM_REF); |
6665 | new_vard = TREE_OPERAND (new_var, 0); |
6666 | gcc_assert (DECL_P (new_vard)); |
6667 | } |
6668 | tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE; |
6669 | if (is_simd |
6670 | && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
6671 | && OMP_CLAUSE_REDUCTION_INSCAN (c)) |
6672 | rvarp = &rvar; |
6673 | if (is_simd |
6674 | && lower_rec_simd_input_clauses (new_var, ctx, sctx: &sctx, |
6675 | ivar, lvar, rvar: rvarp, |
6676 | rvar2: &rvar2)) |
6677 | { |
6678 | if (new_vard != new_var) |
6679 | { |
6680 | SET_DECL_VALUE_EXPR (new_vard, |
6681 | build_fold_addr_expr (lvar)); |
6682 | DECL_HAS_VALUE_EXPR_P (new_vard) = 1; |
6683 | } |
6684 | |
6685 | tree ref = build_outer_var_ref (var, ctx); |
6686 | |
6687 | if (rvarp) |
6688 | { |
6689 | if (ctx->for_simd_scan_phase) |
6690 | break; |
6691 | gimplify_assign (ivar, ref, &llist[0]); |
6692 | ref = build_outer_var_ref (var, ctx); |
6693 | gimplify_assign (ref, rvar, &llist[3]); |
6694 | break; |
6695 | } |
6696 | |
6697 | gimplify_assign (unshare_expr (ivar), x, &llist[0]); |
6698 | |
6699 | if (sctx.is_simt) |
6700 | { |
6701 | if (!simt_lane) |
6702 | simt_lane = create_tmp_var (unsigned_type_node); |
6703 | x = build_call_expr_internal_loc |
6704 | (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_BFLY, |
6705 | TREE_TYPE (ivar), 2, ivar, simt_lane); |
6706 | /* Make sure x is evaluated unconditionally. */ |
6707 | tree bfly_var = create_tmp_var (TREE_TYPE (ivar)); |
6708 | gimplify_assign (bfly_var, x, &llist[2]); |
6709 | x = build2 (code, TREE_TYPE (ivar), ivar, bfly_var); |
6710 | gimplify_assign (ivar, x, &llist[2]); |
6711 | } |
6712 | tree ivar2 = ivar; |
6713 | tree ref2 = ref; |
6714 | if (is_truth_op) |
6715 | { |
6716 | tree zero = build_zero_cst (TREE_TYPE (ivar)); |
6717 | ivar2 = fold_build2_loc (clause_loc, NE_EXPR, |
6718 | boolean_type_node, ivar, |
6719 | zero); |
6720 | ref2 = fold_build2_loc (clause_loc, NE_EXPR, |
6721 | boolean_type_node, ref, |
6722 | zero); |
6723 | } |
6724 | x = build2 (code, TREE_TYPE (ref), ref2, ivar2); |
6725 | if (is_truth_op) |
6726 | x = fold_convert (TREE_TYPE (ref), x); |
6727 | ref = build_outer_var_ref (var, ctx); |
6728 | gimplify_assign (ref, x, &llist[1]); |
6729 | |
6730 | } |
6731 | else |
6732 | { |
6733 | lower_private_allocate (var, new_var, allocator, |
6734 | allocate_ptr, ilist, ctx, |
6735 | is_ref: false, NULL_TREE); |
6736 | if (omp_privatize_by_reference (decl: var) && is_simd) |
6737 | handle_simd_reference (loc: clause_loc, new_vard, ilist); |
6738 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
6739 | && OMP_CLAUSE_REDUCTION_INSCAN (c)) |
6740 | break; |
6741 | gimplify_assign (new_var, x, ilist); |
6742 | if (is_simd) |
6743 | { |
6744 | tree ref = build_outer_var_ref (var, ctx); |
6745 | tree new_var2 = new_var; |
6746 | tree ref2 = ref; |
6747 | if (is_truth_op) |
6748 | { |
6749 | tree zero = build_zero_cst (TREE_TYPE (new_var)); |
6750 | new_var2 |
6751 | = fold_build2_loc (clause_loc, NE_EXPR, |
6752 | boolean_type_node, new_var, |
6753 | zero); |
6754 | ref2 = fold_build2_loc (clause_loc, NE_EXPR, |
6755 | boolean_type_node, ref, |
6756 | zero); |
6757 | } |
6758 | x = build2 (code, TREE_TYPE (ref2), ref2, new_var2); |
6759 | if (is_truth_op) |
6760 | x = fold_convert (TREE_TYPE (new_var), x); |
6761 | ref = build_outer_var_ref (var, ctx); |
6762 | gimplify_assign (ref, x, dlist); |
6763 | } |
6764 | if (allocator) |
6765 | goto do_dtor; |
6766 | } |
6767 | } |
6768 | break; |
6769 | |
6770 | default: |
6771 | gcc_unreachable (); |
6772 | } |
6773 | } |
6774 | } |
6775 | if (tskred_avar) |
6776 | { |
6777 | tree clobber = build_clobber (TREE_TYPE (tskred_avar)); |
6778 | gimple_seq_add_stmt (ilist, gimple_build_assign (tskred_avar, clobber)); |
6779 | } |
6780 | |
6781 | if (known_eq (sctx.max_vf, 1U)) |
6782 | { |
6783 | sctx.is_simt = false; |
6784 | if (ctx->lastprivate_conditional_map) |
6785 | { |
6786 | if (gimple_omp_for_combined_into_p (g: ctx->stmt)) |
6787 | { |
6788 | /* Signal to lower_omp_1 that it should use parent context. */ |
6789 | ctx->combined_into_simd_safelen1 = true; |
6790 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
6791 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
6792 | && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)) |
6793 | { |
6794 | tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx); |
6795 | omp_context *outer = ctx->outer; |
6796 | if (gimple_code (g: outer->stmt) == GIMPLE_OMP_SCAN) |
6797 | outer = outer->outer; |
6798 | tree *v = ctx->lastprivate_conditional_map->get (k: o); |
6799 | tree po = lookup_decl (OMP_CLAUSE_DECL (c), ctx: outer); |
6800 | tree *pv = outer->lastprivate_conditional_map->get (k: po); |
6801 | *v = *pv; |
6802 | } |
6803 | } |
6804 | else |
6805 | { |
6806 | /* When not vectorized, treat lastprivate(conditional:) like |
6807 | normal lastprivate, as there will be just one simd lane |
6808 | writing the privatized variable. */ |
6809 | delete ctx->lastprivate_conditional_map; |
6810 | ctx->lastprivate_conditional_map = NULL; |
6811 | } |
6812 | } |
6813 | } |
6814 | |
6815 | if (nonconst_simd_if) |
6816 | { |
6817 | if (sctx.lane == NULL_TREE) |
6818 | { |
6819 | sctx.idx = create_tmp_var (unsigned_type_node); |
6820 | sctx.lane = create_tmp_var (unsigned_type_node); |
6821 | } |
6822 | /* FIXME: For now. */ |
6823 | sctx.is_simt = false; |
6824 | } |
6825 | |
6826 | if (sctx.lane || sctx.is_simt) |
6827 | { |
6828 | uid = create_tmp_var (ptr_type_node, "simduid" ); |
6829 | /* Don't want uninit warnings on simduid, it is always uninitialized, |
6830 | but we use it not for the value, but for the DECL_UID only. */ |
6831 | suppress_warning (uid, OPT_Wuninitialized); |
6832 | c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_); |
6833 | OMP_CLAUSE__SIMDUID__DECL (c) = uid; |
6834 | OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (gs: ctx->stmt); |
6835 | gimple_omp_for_set_clauses (gs: ctx->stmt, clauses: c); |
6836 | } |
6837 | /* Emit calls denoting privatized variables and initializing a pointer to |
6838 | structure that holds private variables as fields after ompdevlow pass. */ |
6839 | if (sctx.is_simt) |
6840 | { |
6841 | sctx.simt_eargs[0] = uid; |
6842 | gimple *g |
6843 | = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, sctx.simt_eargs); |
6844 | gimple_call_set_lhs (gs: g, lhs: uid); |
6845 | gimple_seq_add_stmt (ilist, g); |
6846 | sctx.simt_eargs.release (); |
6847 | |
6848 | simtrec = create_tmp_var (ptr_type_node, ".omp_simt" ); |
6849 | g = gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC, 1, uid); |
6850 | gimple_call_set_lhs (gs: g, lhs: simtrec); |
6851 | gimple_seq_add_stmt (ilist, g); |
6852 | } |
6853 | if (sctx.lane) |
6854 | { |
6855 | gimple *g = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, |
6856 | 2 + (nonconst_simd_if != NULL), |
6857 | uid, integer_zero_node, |
6858 | nonconst_simd_if); |
6859 | gimple_call_set_lhs (gs: g, lhs: sctx.lane); |
6860 | gimple_stmt_iterator gsi = gsi_start (seq&: *gimple_omp_body_ptr (gs: ctx->stmt)); |
6861 | gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT); |
6862 | g = gimple_build_assign (sctx.lane, INTEGER_CST, |
6863 | build_int_cst (unsigned_type_node, 0)); |
6864 | gimple_seq_add_stmt (ilist, g); |
6865 | if (sctx.lastlane) |
6866 | { |
6867 | g = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE, |
6868 | 2, uid, sctx.lane); |
6869 | gimple_call_set_lhs (gs: g, lhs: sctx.lastlane); |
6870 | gimple_seq_add_stmt (dlist, g); |
6871 | gimple_seq_add_seq (dlist, llist[3]); |
6872 | } |
6873 | /* Emit reductions across SIMT lanes in log_2(simt_vf) steps. */ |
6874 | if (llist[2]) |
6875 | { |
6876 | tree simt_vf = create_tmp_var (unsigned_type_node); |
6877 | g = gimple_build_call_internal (IFN_GOMP_SIMT_VF, 0); |
6878 | gimple_call_set_lhs (gs: g, lhs: simt_vf); |
6879 | gimple_seq_add_stmt (dlist, g); |
6880 | |
6881 | tree t = build_int_cst (unsigned_type_node, 1); |
6882 | g = gimple_build_assign (simt_lane, INTEGER_CST, t); |
6883 | gimple_seq_add_stmt (dlist, g); |
6884 | |
6885 | t = build_int_cst (unsigned_type_node, 0); |
6886 | g = gimple_build_assign (sctx.idx, INTEGER_CST, t); |
6887 | gimple_seq_add_stmt (dlist, g); |
6888 | |
6889 | tree body = create_artificial_label (UNKNOWN_LOCATION); |
6890 | tree = create_artificial_label (UNKNOWN_LOCATION); |
6891 | tree end = create_artificial_label (UNKNOWN_LOCATION); |
6892 | gimple_seq_add_stmt (dlist, gimple_build_goto (dest: header)); |
6893 | gimple_seq_add_stmt (dlist, gimple_build_label (label: body)); |
6894 | |
6895 | gimple_seq_add_seq (dlist, llist[2]); |
6896 | |
6897 | g = gimple_build_assign (simt_lane, LSHIFT_EXPR, simt_lane, integer_one_node); |
6898 | gimple_seq_add_stmt (dlist, g); |
6899 | |
6900 | gimple_seq_add_stmt (dlist, gimple_build_label (label: header)); |
6901 | g = gimple_build_cond (LT_EXPR, simt_lane, simt_vf, body, end); |
6902 | gimple_seq_add_stmt (dlist, g); |
6903 | |
6904 | gimple_seq_add_stmt (dlist, gimple_build_label (label: end)); |
6905 | } |
6906 | for (int i = 0; i < 2; i++) |
6907 | if (llist[i]) |
6908 | { |
6909 | tree vf = create_tmp_var (unsigned_type_node); |
6910 | g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid); |
6911 | gimple_call_set_lhs (gs: g, lhs: vf); |
6912 | gimple_seq *seq = i == 0 ? ilist : dlist; |
6913 | gimple_seq_add_stmt (seq, g); |
6914 | tree t = build_int_cst (unsigned_type_node, 0); |
6915 | g = gimple_build_assign (sctx.idx, INTEGER_CST, t); |
6916 | gimple_seq_add_stmt (seq, g); |
6917 | tree body = create_artificial_label (UNKNOWN_LOCATION); |
6918 | tree = create_artificial_label (UNKNOWN_LOCATION); |
6919 | tree end = create_artificial_label (UNKNOWN_LOCATION); |
6920 | gimple_seq_add_stmt (seq, gimple_build_goto (dest: header)); |
6921 | gimple_seq_add_stmt (seq, gimple_build_label (label: body)); |
6922 | gimple_seq_add_seq (seq, llist[i]); |
6923 | t = build_int_cst (unsigned_type_node, 1); |
6924 | g = gimple_build_assign (sctx.idx, PLUS_EXPR, sctx.idx, t); |
6925 | gimple_seq_add_stmt (seq, g); |
6926 | gimple_seq_add_stmt (seq, gimple_build_label (label: header)); |
6927 | g = gimple_build_cond (LT_EXPR, sctx.idx, vf, body, end); |
6928 | gimple_seq_add_stmt (seq, g); |
6929 | gimple_seq_add_stmt (seq, gimple_build_label (label: end)); |
6930 | } |
6931 | } |
6932 | if (sctx.is_simt) |
6933 | { |
6934 | gimple_seq_add_seq (dlist, sctx.simt_dlist); |
6935 | gimple *g |
6936 | = gimple_build_call_internal (IFN_GOMP_SIMT_EXIT, 1, simtrec); |
6937 | gimple_seq_add_stmt (dlist, g); |
6938 | } |
6939 | |
6940 | /* The copyin sequence is not to be executed by the main thread, since |
6941 | that would result in self-copies. Perhaps not visible to scalars, |
6942 | but it certainly is to C++ operator=. */ |
6943 | if (copyin_seq) |
6944 | { |
6945 | x = build_call_expr (builtin_decl_explicit (fncode: BUILT_IN_OMP_GET_THREAD_NUM), |
6946 | 0); |
6947 | x = build2 (NE_EXPR, boolean_type_node, x, |
6948 | build_int_cst (TREE_TYPE (x), 0)); |
6949 | x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL); |
6950 | gimplify_and_add (x, ilist); |
6951 | } |
6952 | |
6953 | /* If any copyin variable is passed by reference, we must ensure the |
6954 | master thread doesn't modify it before it is copied over in all |
6955 | threads. Similarly for variables in both firstprivate and |
6956 | lastprivate clauses we need to ensure the lastprivate copying |
6957 | happens after firstprivate copying in all threads. And similarly |
6958 | for UDRs if initializer expression refers to omp_orig. */ |
6959 | if (copyin_by_ref || lastprivate_firstprivate |
6960 | || (reduction_omp_orig_ref |
6961 | && !ctx->scan_inclusive |
6962 | && !ctx->scan_exclusive)) |
6963 | { |
6964 | /* Don't add any barrier for #pragma omp simd or |
6965 | #pragma omp distribute. */ |
6966 | if (!is_task_ctx (ctx) |
6967 | && (gimple_code (g: ctx->stmt) != GIMPLE_OMP_FOR |
6968 | || gimple_omp_for_kind (g: ctx->stmt) == GF_OMP_FOR_KIND_FOR)) |
6969 | gimple_seq_add_stmt (ilist, omp_build_barrier (NULL_TREE)); |
6970 | } |
6971 | |
6972 | /* If max_vf is non-zero, then we can use only a vectorization factor |
6973 | up to the max_vf we chose. So stick it into the safelen clause. */ |
6974 | if (maybe_ne (a: sctx.max_vf, b: 0U)) |
6975 | { |
6976 | tree c = omp_find_clause (clauses: gimple_omp_for_clauses (gs: ctx->stmt), |
6977 | kind: OMP_CLAUSE_SAFELEN); |
6978 | poly_uint64 safe_len; |
6979 | if (c == NULL_TREE |
6980 | || (poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), value: &safe_len) |
6981 | && maybe_gt (safe_len, sctx.max_vf))) |
6982 | { |
6983 | c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN); |
6984 | OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node, |
6985 | sctx.max_vf); |
6986 | OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (gs: ctx->stmt); |
6987 | gimple_omp_for_set_clauses (gs: ctx->stmt, clauses: c); |
6988 | } |
6989 | } |
6990 | } |
6991 | |
6992 | /* Create temporary variables for lastprivate(conditional:) implementation |
6993 | in context CTX with CLAUSES. */ |
6994 | |
6995 | static void |
6996 | lower_lastprivate_conditional_clauses (tree *clauses, omp_context *ctx) |
6997 | { |
6998 | tree iter_type = NULL_TREE; |
6999 | tree cond_ptr = NULL_TREE; |
7000 | tree iter_var = NULL_TREE; |
7001 | bool is_simd = (gimple_code (g: ctx->stmt) == GIMPLE_OMP_FOR |
7002 | && gimple_omp_for_kind (g: ctx->stmt) == GF_OMP_FOR_KIND_SIMD); |
7003 | tree next = *clauses; |
7004 | for (tree c = *clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
7005 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
7006 | && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)) |
7007 | { |
7008 | if (is_simd) |
7009 | { |
7010 | tree cc = omp_find_clause (clauses: next, kind: OMP_CLAUSE__CONDTEMP_); |
7011 | gcc_assert (cc); |
7012 | if (iter_type == NULL_TREE) |
7013 | { |
7014 | iter_type = TREE_TYPE (OMP_CLAUSE_DECL (cc)); |
7015 | iter_var = create_tmp_var_raw (iter_type); |
7016 | DECL_CONTEXT (iter_var) = current_function_decl; |
7017 | DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1; |
7018 | DECL_CHAIN (iter_var) = ctx->block_vars; |
7019 | ctx->block_vars = iter_var; |
7020 | tree c3 |
7021 | = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_); |
7022 | OMP_CLAUSE__CONDTEMP__ITER (c3) = 1; |
7023 | OMP_CLAUSE_DECL (c3) = iter_var; |
7024 | OMP_CLAUSE_CHAIN (c3) = *clauses; |
7025 | *clauses = c3; |
7026 | ctx->lastprivate_conditional_map = new hash_map<tree, tree>; |
7027 | } |
7028 | next = OMP_CLAUSE_CHAIN (cc); |
7029 | tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx); |
7030 | tree v = lookup_decl (OMP_CLAUSE_DECL (cc), ctx); |
7031 | ctx->lastprivate_conditional_map->put (k: o, v); |
7032 | continue; |
7033 | } |
7034 | if (iter_type == NULL) |
7035 | { |
7036 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_FOR) |
7037 | { |
7038 | struct omp_for_data fd; |
7039 | omp_extract_for_data (for_stmt: as_a <gomp_for *> (p: ctx->stmt), fd: &fd, |
7040 | NULL); |
7041 | iter_type = unsigned_type_for (fd.iter_type); |
7042 | } |
7043 | else if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_SECTIONS) |
7044 | iter_type = unsigned_type_node; |
7045 | tree c2 = omp_find_clause (clauses: *clauses, kind: OMP_CLAUSE__CONDTEMP_); |
7046 | if (c2) |
7047 | { |
7048 | cond_ptr |
7049 | = lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c2), ctx); |
7050 | OMP_CLAUSE_DECL (c2) = cond_ptr; |
7051 | } |
7052 | else |
7053 | { |
7054 | cond_ptr = create_tmp_var_raw (build_pointer_type (iter_type)); |
7055 | DECL_CONTEXT (cond_ptr) = current_function_decl; |
7056 | DECL_SEEN_IN_BIND_EXPR_P (cond_ptr) = 1; |
7057 | DECL_CHAIN (cond_ptr) = ctx->block_vars; |
7058 | ctx->block_vars = cond_ptr; |
7059 | c2 = build_omp_clause (UNKNOWN_LOCATION, |
7060 | OMP_CLAUSE__CONDTEMP_); |
7061 | OMP_CLAUSE_DECL (c2) = cond_ptr; |
7062 | OMP_CLAUSE_CHAIN (c2) = *clauses; |
7063 | *clauses = c2; |
7064 | } |
7065 | iter_var = create_tmp_var_raw (iter_type); |
7066 | DECL_CONTEXT (iter_var) = current_function_decl; |
7067 | DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1; |
7068 | DECL_CHAIN (iter_var) = ctx->block_vars; |
7069 | ctx->block_vars = iter_var; |
7070 | tree c3 |
7071 | = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_); |
7072 | OMP_CLAUSE__CONDTEMP__ITER (c3) = 1; |
7073 | OMP_CLAUSE_DECL (c3) = iter_var; |
7074 | OMP_CLAUSE_CHAIN (c3) = OMP_CLAUSE_CHAIN (c2); |
7075 | OMP_CLAUSE_CHAIN (c2) = c3; |
7076 | ctx->lastprivate_conditional_map = new hash_map<tree, tree>; |
7077 | } |
7078 | tree v = create_tmp_var_raw (iter_type); |
7079 | DECL_CONTEXT (v) = current_function_decl; |
7080 | DECL_SEEN_IN_BIND_EXPR_P (v) = 1; |
7081 | DECL_CHAIN (v) = ctx->block_vars; |
7082 | ctx->block_vars = v; |
7083 | tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx); |
7084 | ctx->lastprivate_conditional_map->put (k: o, v); |
7085 | } |
7086 | } |
7087 | |
7088 | |
7089 | /* Generate code to implement the LASTPRIVATE clauses. This is used for |
7090 | both parallel and workshare constructs. PREDICATE may be NULL if it's |
7091 | always true. BODY_P is the sequence to insert early initialization |
7092 | if needed, STMT_LIST is where the non-conditional lastprivate handling |
7093 | goes into and CSTMT_LIST is a sequence that needs to be run in a critical |
7094 | section. */ |
7095 | |
7096 | static void |
7097 | lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *body_p, |
7098 | gimple_seq *stmt_list, gimple_seq *cstmt_list, |
7099 | omp_context *ctx) |
7100 | { |
7101 | tree x, c, label = NULL, orig_clauses = clauses; |
7102 | bool par_clauses = false; |
7103 | tree simduid = NULL, lastlane = NULL, simtcond = NULL, simtlast = NULL; |
7104 | unsigned HOST_WIDE_INT conditional_off = 0; |
7105 | gimple_seq post_stmt_list = NULL; |
7106 | |
7107 | /* Early exit if there are no lastprivate or linear clauses. */ |
7108 | for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses)) |
7109 | if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE |
7110 | || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR |
7111 | && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses))) |
7112 | break; |
7113 | if (clauses == NULL) |
7114 | { |
7115 | /* If this was a workshare clause, see if it had been combined |
7116 | with its parallel. In that case, look for the clauses on the |
7117 | parallel statement itself. */ |
7118 | if (is_parallel_ctx (ctx)) |
7119 | return; |
7120 | |
7121 | ctx = ctx->outer; |
7122 | if (ctx == NULL || !is_parallel_ctx (ctx)) |
7123 | return; |
7124 | |
7125 | clauses = omp_find_clause (clauses: gimple_omp_parallel_clauses (gs: ctx->stmt), |
7126 | kind: OMP_CLAUSE_LASTPRIVATE); |
7127 | if (clauses == NULL) |
7128 | return; |
7129 | par_clauses = true; |
7130 | } |
7131 | |
7132 | bool maybe_simt = false; |
7133 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_FOR |
7134 | && gimple_omp_for_kind (g: ctx->stmt) == GF_OMP_FOR_KIND_SIMD) |
7135 | { |
7136 | maybe_simt = omp_find_clause (clauses: orig_clauses, kind: OMP_CLAUSE__SIMT_); |
7137 | simduid = omp_find_clause (clauses: orig_clauses, kind: OMP_CLAUSE__SIMDUID_); |
7138 | if (simduid) |
7139 | simduid = OMP_CLAUSE__SIMDUID__DECL (simduid); |
7140 | } |
7141 | |
7142 | if (predicate) |
7143 | { |
7144 | gcond *stmt; |
7145 | tree label_true, arm1, arm2; |
7146 | enum tree_code pred_code = TREE_CODE (predicate); |
7147 | |
7148 | label = create_artificial_label (UNKNOWN_LOCATION); |
7149 | label_true = create_artificial_label (UNKNOWN_LOCATION); |
7150 | if (TREE_CODE_CLASS (pred_code) == tcc_comparison) |
7151 | { |
7152 | arm1 = TREE_OPERAND (predicate, 0); |
7153 | arm2 = TREE_OPERAND (predicate, 1); |
7154 | gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue); |
7155 | gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue); |
7156 | } |
7157 | else |
7158 | { |
7159 | arm1 = predicate; |
7160 | gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue); |
7161 | arm2 = boolean_false_node; |
7162 | pred_code = NE_EXPR; |
7163 | } |
7164 | if (maybe_simt) |
7165 | { |
7166 | c = build2 (pred_code, boolean_type_node, arm1, arm2); |
7167 | c = fold_convert (integer_type_node, c); |
7168 | simtcond = create_tmp_var (integer_type_node); |
7169 | gimplify_assign (simtcond, c, stmt_list); |
7170 | gcall *g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY, |
7171 | 1, simtcond); |
7172 | c = create_tmp_var (integer_type_node); |
7173 | gimple_call_set_lhs (gs: g, lhs: c); |
7174 | gimple_seq_add_stmt (stmt_list, g); |
7175 | stmt = gimple_build_cond (NE_EXPR, c, integer_zero_node, |
7176 | label_true, label); |
7177 | } |
7178 | else |
7179 | stmt = gimple_build_cond (pred_code, arm1, arm2, label_true, label); |
7180 | gimple_seq_add_stmt (stmt_list, stmt); |
7181 | gimple_seq_add_stmt (stmt_list, gimple_build_label (label: label_true)); |
7182 | } |
7183 | |
7184 | tree cond_ptr = NULL_TREE; |
7185 | for (c = clauses; c ;) |
7186 | { |
7187 | tree var, new_var; |
7188 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
7189 | gimple_seq *this_stmt_list = stmt_list; |
7190 | tree lab2 = NULL_TREE; |
7191 | |
7192 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
7193 | && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c) |
7194 | && ctx->lastprivate_conditional_map |
7195 | && !ctx->combined_into_simd_safelen1) |
7196 | { |
7197 | gcc_assert (body_p); |
7198 | if (simduid) |
7199 | goto next; |
7200 | if (cond_ptr == NULL_TREE) |
7201 | { |
7202 | cond_ptr = omp_find_clause (clauses: orig_clauses, kind: OMP_CLAUSE__CONDTEMP_); |
7203 | cond_ptr = OMP_CLAUSE_DECL (cond_ptr); |
7204 | } |
7205 | tree type = TREE_TYPE (TREE_TYPE (cond_ptr)); |
7206 | tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx); |
7207 | tree v = *ctx->lastprivate_conditional_map->get (k: o); |
7208 | gimplify_assign (v, build_zero_cst (type), body_p); |
7209 | this_stmt_list = cstmt_list; |
7210 | tree mem; |
7211 | if (POINTER_TYPE_P (TREE_TYPE (cond_ptr))) |
7212 | { |
7213 | mem = build2 (MEM_REF, type, cond_ptr, |
7214 | build_int_cst (TREE_TYPE (cond_ptr), |
7215 | conditional_off)); |
7216 | conditional_off += tree_to_uhwi (TYPE_SIZE_UNIT (type)); |
7217 | } |
7218 | else |
7219 | mem = build4 (ARRAY_REF, type, cond_ptr, |
7220 | size_int (conditional_off++), NULL_TREE, NULL_TREE); |
7221 | tree mem2 = copy_node (mem); |
7222 | gimple_seq seq = NULL; |
7223 | mem = force_gimple_operand (mem, &seq, true, NULL_TREE); |
7224 | gimple_seq_add_seq (this_stmt_list, seq); |
7225 | tree lab1 = create_artificial_label (UNKNOWN_LOCATION); |
7226 | lab2 = create_artificial_label (UNKNOWN_LOCATION); |
7227 | gimple *g = gimple_build_cond (GT_EXPR, v, mem, lab1, lab2); |
7228 | gimple_seq_add_stmt (this_stmt_list, g); |
7229 | gimple_seq_add_stmt (this_stmt_list, gimple_build_label (label: lab1)); |
7230 | gimplify_assign (mem2, v, this_stmt_list); |
7231 | } |
7232 | else if (predicate |
7233 | && ctx->combined_into_simd_safelen1 |
7234 | && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
7235 | && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c) |
7236 | && ctx->lastprivate_conditional_map) |
7237 | this_stmt_list = &post_stmt_list; |
7238 | |
7239 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
7240 | || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR |
7241 | && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))) |
7242 | { |
7243 | var = OMP_CLAUSE_DECL (c); |
7244 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
7245 | && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c) |
7246 | && is_taskloop_ctx (ctx)) |
7247 | { |
7248 | gcc_checking_assert (ctx->outer && is_task_ctx (ctx->outer)); |
7249 | new_var = lookup_decl (var, ctx: ctx->outer); |
7250 | } |
7251 | else |
7252 | { |
7253 | new_var = lookup_decl (var, ctx); |
7254 | /* Avoid uninitialized warnings for lastprivate and |
7255 | for linear iterators. */ |
7256 | if (predicate |
7257 | && (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
7258 | || OMP_CLAUSE_LINEAR_NO_COPYIN (c))) |
7259 | suppress_warning (new_var, OPT_Wuninitialized); |
7260 | } |
7261 | |
7262 | if (!maybe_simt && simduid && DECL_HAS_VALUE_EXPR_P (new_var)) |
7263 | { |
7264 | tree val = DECL_VALUE_EXPR (new_var); |
7265 | if (TREE_CODE (val) == ARRAY_REF |
7266 | && VAR_P (TREE_OPERAND (val, 0)) |
7267 | && lookup_attribute (attr_name: "omp simd array" , |
7268 | DECL_ATTRIBUTES (TREE_OPERAND (val, |
7269 | 0)))) |
7270 | { |
7271 | if (lastlane == NULL) |
7272 | { |
7273 | lastlane = create_tmp_var (unsigned_type_node); |
7274 | gcall *g |
7275 | = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE, |
7276 | 2, simduid, |
7277 | TREE_OPERAND (val, 1)); |
7278 | gimple_call_set_lhs (gs: g, lhs: lastlane); |
7279 | gimple_seq_add_stmt (this_stmt_list, g); |
7280 | } |
7281 | new_var = build4 (ARRAY_REF, TREE_TYPE (val), |
7282 | TREE_OPERAND (val, 0), lastlane, |
7283 | NULL_TREE, NULL_TREE); |
7284 | TREE_THIS_NOTRAP (new_var) = 1; |
7285 | } |
7286 | } |
7287 | else if (maybe_simt) |
7288 | { |
7289 | tree val = (DECL_HAS_VALUE_EXPR_P (new_var) |
7290 | ? DECL_VALUE_EXPR (new_var) |
7291 | : new_var); |
7292 | if (simtlast == NULL) |
7293 | { |
7294 | simtlast = create_tmp_var (unsigned_type_node); |
7295 | gcall *g = gimple_build_call_internal |
7296 | (IFN_GOMP_SIMT_LAST_LANE, 1, simtcond); |
7297 | gimple_call_set_lhs (gs: g, lhs: simtlast); |
7298 | gimple_seq_add_stmt (this_stmt_list, g); |
7299 | } |
7300 | x = build_call_expr_internal_loc |
7301 | (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_IDX, |
7302 | TREE_TYPE (val), 2, val, simtlast); |
7303 | new_var = unshare_expr (new_var); |
7304 | gimplify_assign (new_var, x, this_stmt_list); |
7305 | new_var = unshare_expr (new_var); |
7306 | } |
7307 | |
7308 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
7309 | && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)) |
7310 | { |
7311 | lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx); |
7312 | gimple_seq_add_seq (this_stmt_list, |
7313 | OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)); |
7314 | OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL; |
7315 | } |
7316 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR |
7317 | && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)) |
7318 | { |
7319 | lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx); |
7320 | gimple_seq_add_seq (this_stmt_list, |
7321 | OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)); |
7322 | OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL; |
7323 | } |
7324 | |
7325 | x = NULL_TREE; |
7326 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
7327 | && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c) |
7328 | && is_taskloop_ctx (ctx)) |
7329 | { |
7330 | tree ovar = maybe_lookup_decl_in_outer_ctx (decl: var, |
7331 | ctx: ctx->outer->outer); |
7332 | if (is_global_var (t: ovar)) |
7333 | x = ovar; |
7334 | } |
7335 | if (!x) |
7336 | x = build_outer_var_ref (var, ctx, code: OMP_CLAUSE_LASTPRIVATE); |
7337 | if (omp_privatize_by_reference (decl: var)) |
7338 | new_var = build_simple_mem_ref_loc (clause_loc, new_var); |
7339 | x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var); |
7340 | gimplify_and_add (x, this_stmt_list); |
7341 | |
7342 | if (lab2) |
7343 | gimple_seq_add_stmt (this_stmt_list, gimple_build_label (label: lab2)); |
7344 | } |
7345 | |
7346 | next: |
7347 | c = OMP_CLAUSE_CHAIN (c); |
7348 | if (c == NULL && !par_clauses) |
7349 | { |
7350 | /* If this was a workshare clause, see if it had been combined |
7351 | with its parallel. In that case, continue looking for the |
7352 | clauses also on the parallel statement itself. */ |
7353 | if (is_parallel_ctx (ctx)) |
7354 | break; |
7355 | |
7356 | ctx = ctx->outer; |
7357 | if (ctx == NULL || !is_parallel_ctx (ctx)) |
7358 | break; |
7359 | |
7360 | c = omp_find_clause (clauses: gimple_omp_parallel_clauses (gs: ctx->stmt), |
7361 | kind: OMP_CLAUSE_LASTPRIVATE); |
7362 | par_clauses = true; |
7363 | } |
7364 | } |
7365 | |
7366 | if (label) |
7367 | gimple_seq_add_stmt (stmt_list, gimple_build_label (label)); |
7368 | gimple_seq_add_seq (stmt_list, post_stmt_list); |
7369 | } |
7370 | |
7371 | /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL |
7372 | (which might be a placeholder). INNER is true if this is an inner |
7373 | axis of a multi-axis loop. FORK and JOIN are (optional) fork and |
7374 | join markers. Generate the before-loop forking sequence in |
7375 | FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The |
7376 | general form of these sequences is |
7377 | |
7378 | GOACC_REDUCTION_SETUP |
7379 | GOACC_FORK |
7380 | GOACC_REDUCTION_INIT |
7381 | ... |
7382 | GOACC_REDUCTION_FINI |
7383 | GOACC_JOIN |
7384 | GOACC_REDUCTION_TEARDOWN. */ |
7385 | |
7386 | static void |
7387 | lower_oacc_reductions (location_t loc, tree clauses, tree level, bool inner, |
7388 | gcall *fork, gcall *private_marker, gcall *join, |
7389 | gimple_seq *fork_seq, gimple_seq *join_seq, |
7390 | omp_context *ctx) |
7391 | { |
7392 | gimple_seq before_fork = NULL; |
7393 | gimple_seq after_fork = NULL; |
7394 | gimple_seq before_join = NULL; |
7395 | gimple_seq after_join = NULL; |
7396 | tree init_code = NULL_TREE, fini_code = NULL_TREE, |
7397 | setup_code = NULL_TREE, teardown_code = NULL_TREE; |
7398 | unsigned offset = 0; |
7399 | |
7400 | for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
7401 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION) |
7402 | { |
7403 | /* No 'reduction' clauses on OpenACC 'kernels'. */ |
7404 | gcc_checking_assert (!is_oacc_kernels (ctx)); |
7405 | /* Likewise, on OpenACC 'kernels' decomposed parts. */ |
7406 | gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx)); |
7407 | |
7408 | tree orig = OMP_CLAUSE_DECL (c); |
7409 | tree var = maybe_lookup_decl (var: orig, ctx); |
7410 | tree ref_to_res = NULL_TREE; |
7411 | tree incoming, outgoing, v1, v2, v3; |
7412 | bool is_private = false; |
7413 | |
7414 | enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c); |
7415 | if (rcode == MINUS_EXPR) |
7416 | rcode = PLUS_EXPR; |
7417 | else if (rcode == TRUTH_ANDIF_EXPR) |
7418 | rcode = BIT_AND_EXPR; |
7419 | else if (rcode == TRUTH_ORIF_EXPR) |
7420 | rcode = BIT_IOR_EXPR; |
7421 | tree op = build_int_cst (unsigned_type_node, rcode); |
7422 | |
7423 | if (!var) |
7424 | var = orig; |
7425 | |
7426 | incoming = outgoing = var; |
7427 | |
7428 | if (!inner) |
7429 | { |
7430 | /* See if an outer construct also reduces this variable. */ |
7431 | omp_context *outer = ctx; |
7432 | |
7433 | while (omp_context *probe = outer->outer) |
7434 | { |
7435 | enum gimple_code type = gimple_code (g: probe->stmt); |
7436 | tree cls; |
7437 | |
7438 | switch (type) |
7439 | { |
7440 | case GIMPLE_OMP_FOR: |
7441 | cls = gimple_omp_for_clauses (gs: probe->stmt); |
7442 | break; |
7443 | |
7444 | case GIMPLE_OMP_TARGET: |
7445 | /* No 'reduction' clauses inside OpenACC 'kernels' |
7446 | regions. */ |
7447 | gcc_checking_assert (!is_oacc_kernels (probe)); |
7448 | |
7449 | if (!is_gimple_omp_offloaded (stmt: probe->stmt)) |
7450 | goto do_lookup; |
7451 | |
7452 | cls = gimple_omp_target_clauses (gs: probe->stmt); |
7453 | break; |
7454 | |
7455 | default: |
7456 | goto do_lookup; |
7457 | } |
7458 | |
7459 | outer = probe; |
7460 | for (; cls; cls = OMP_CLAUSE_CHAIN (cls)) |
7461 | if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_REDUCTION |
7462 | && orig == OMP_CLAUSE_DECL (cls)) |
7463 | { |
7464 | incoming = outgoing = lookup_decl (var: orig, ctx: probe); |
7465 | goto has_outer_reduction; |
7466 | } |
7467 | else if ((OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_FIRSTPRIVATE |
7468 | || OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_PRIVATE) |
7469 | && orig == OMP_CLAUSE_DECL (cls)) |
7470 | { |
7471 | is_private = true; |
7472 | goto do_lookup; |
7473 | } |
7474 | } |
7475 | |
7476 | do_lookup: |
7477 | /* This is the outermost construct with this reduction, |
7478 | see if there's a mapping for it. */ |
7479 | if (gimple_code (g: outer->stmt) == GIMPLE_OMP_TARGET |
7480 | && maybe_lookup_field (var: orig, ctx: outer) && !is_private) |
7481 | { |
7482 | ref_to_res = build_receiver_ref (var: orig, by_ref: false, ctx: outer); |
7483 | if (omp_privatize_by_reference (decl: orig)) |
7484 | ref_to_res = build_simple_mem_ref (ref_to_res); |
7485 | |
7486 | tree type = TREE_TYPE (var); |
7487 | if (POINTER_TYPE_P (type)) |
7488 | type = TREE_TYPE (type); |
7489 | |
7490 | outgoing = var; |
7491 | incoming = omp_reduction_init_op (loc, op: rcode, type); |
7492 | } |
7493 | else |
7494 | { |
7495 | /* Try to look at enclosing contexts for reduction var, |
7496 | use original if no mapping found. */ |
7497 | tree t = NULL_TREE; |
7498 | omp_context *c = ctx->outer; |
7499 | while (c && !t) |
7500 | { |
7501 | t = maybe_lookup_decl (var: orig, ctx: c); |
7502 | c = c->outer; |
7503 | } |
7504 | incoming = outgoing = (t ? t : orig); |
7505 | } |
7506 | |
7507 | has_outer_reduction:; |
7508 | } |
7509 | |
7510 | if (!ref_to_res) |
7511 | ref_to_res = integer_zero_node; |
7512 | |
7513 | if (omp_privatize_by_reference (decl: orig)) |
7514 | { |
7515 | tree type = TREE_TYPE (var); |
7516 | const char *id = IDENTIFIER_POINTER (DECL_NAME (var)); |
7517 | |
7518 | if (!inner) |
7519 | { |
7520 | tree x = create_tmp_var (TREE_TYPE (type), id); |
7521 | gimplify_assign (var, build_fold_addr_expr (x), fork_seq); |
7522 | } |
7523 | |
7524 | v1 = create_tmp_var (type, id); |
7525 | v2 = create_tmp_var (type, id); |
7526 | v3 = create_tmp_var (type, id); |
7527 | |
7528 | gimplify_assign (v1, var, fork_seq); |
7529 | gimplify_assign (v2, var, fork_seq); |
7530 | gimplify_assign (v3, var, fork_seq); |
7531 | |
7532 | var = build_simple_mem_ref (var); |
7533 | v1 = build_simple_mem_ref (v1); |
7534 | v2 = build_simple_mem_ref (v2); |
7535 | v3 = build_simple_mem_ref (v3); |
7536 | outgoing = build_simple_mem_ref (outgoing); |
7537 | |
7538 | if (!TREE_CONSTANT (incoming)) |
7539 | incoming = build_simple_mem_ref (incoming); |
7540 | } |
7541 | else |
7542 | /* Note that 'var' might be a mem ref. */ |
7543 | v1 = v2 = v3 = var; |
7544 | |
7545 | /* Determine position in reduction buffer, which may be used |
7546 | by target. The parser has ensured that this is not a |
7547 | variable-sized type. */ |
7548 | fixed_size_mode mode |
7549 | = as_a <fixed_size_mode> (TYPE_MODE (TREE_TYPE (var))); |
7550 | unsigned align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT; |
7551 | offset = (offset + align - 1) & ~(align - 1); |
7552 | tree off = build_int_cst (sizetype, offset); |
7553 | offset += GET_MODE_SIZE (mode); |
7554 | |
7555 | if (!init_code) |
7556 | { |
7557 | init_code = build_int_cst (integer_type_node, |
7558 | IFN_GOACC_REDUCTION_INIT); |
7559 | fini_code = build_int_cst (integer_type_node, |
7560 | IFN_GOACC_REDUCTION_FINI); |
7561 | setup_code = build_int_cst (integer_type_node, |
7562 | IFN_GOACC_REDUCTION_SETUP); |
7563 | teardown_code = build_int_cst (integer_type_node, |
7564 | IFN_GOACC_REDUCTION_TEARDOWN); |
7565 | } |
7566 | |
7567 | tree setup_call |
7568 | = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION, |
7569 | TREE_TYPE (var), 6, setup_code, |
7570 | unshare_expr (ref_to_res), |
7571 | unshare_expr (incoming), |
7572 | level, op, off); |
7573 | tree init_call |
7574 | = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION, |
7575 | TREE_TYPE (var), 6, init_code, |
7576 | unshare_expr (ref_to_res), |
7577 | unshare_expr (v1), level, op, off); |
7578 | tree fini_call |
7579 | = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION, |
7580 | TREE_TYPE (var), 6, fini_code, |
7581 | unshare_expr (ref_to_res), |
7582 | unshare_expr (v2), level, op, off); |
7583 | tree teardown_call |
7584 | = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION, |
7585 | TREE_TYPE (var), 6, teardown_code, |
7586 | ref_to_res, unshare_expr (v3), |
7587 | level, op, off); |
7588 | |
7589 | gimplify_assign (unshare_expr (v1), setup_call, &before_fork); |
7590 | gimplify_assign (unshare_expr (v2), init_call, &after_fork); |
7591 | gimplify_assign (unshare_expr (v3), fini_call, &before_join); |
7592 | gimplify_assign (unshare_expr (outgoing), teardown_call, &after_join); |
7593 | } |
7594 | |
7595 | /* Now stitch things together. */ |
7596 | gimple_seq_add_seq (fork_seq, before_fork); |
7597 | if (private_marker) |
7598 | gimple_seq_add_stmt (fork_seq, private_marker); |
7599 | if (fork) |
7600 | gimple_seq_add_stmt (fork_seq, fork); |
7601 | gimple_seq_add_seq (fork_seq, after_fork); |
7602 | |
7603 | gimple_seq_add_seq (join_seq, before_join); |
7604 | if (join) |
7605 | gimple_seq_add_stmt (join_seq, join); |
7606 | gimple_seq_add_seq (join_seq, after_join); |
7607 | } |
7608 | |
7609 | /* Generate code to implement the REDUCTION clauses, append it |
7610 | to STMT_SEQP. CLIST if non-NULL is a pointer to a sequence |
7611 | that should be emitted also inside of the critical section, |
7612 | in that case clear *CLIST afterwards, otherwise leave it as is |
7613 | and let the caller emit it itself. */ |
7614 | |
7615 | static void |
7616 | lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, |
7617 | gimple_seq *clist, omp_context *ctx) |
7618 | { |
7619 | gimple_seq sub_seq = NULL; |
7620 | gimple *stmt; |
7621 | tree x, c; |
7622 | int count = 0; |
7623 | |
7624 | /* OpenACC loop reductions are handled elsewhere. */ |
7625 | if (is_gimple_omp_oacc (stmt: ctx->stmt)) |
7626 | return; |
7627 | |
7628 | /* SIMD reductions are handled in lower_rec_input_clauses. */ |
7629 | if (gimple_code (g: ctx->stmt) == GIMPLE_OMP_FOR |
7630 | && gimple_omp_for_kind (g: ctx->stmt) == GF_OMP_FOR_KIND_SIMD) |
7631 | return; |
7632 | |
7633 | /* inscan reductions are handled elsewhere. */ |
7634 | if (ctx->scan_inclusive || ctx->scan_exclusive) |
7635 | return; |
7636 | |
7637 | /* First see if there is exactly one reduction clause. Use OMP_ATOMIC |
7638 | update in that case, otherwise use a lock. */ |
7639 | for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c)) |
7640 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
7641 | && !OMP_CLAUSE_REDUCTION_TASK (c)) |
7642 | { |
7643 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) |
7644 | || TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF) |
7645 | { |
7646 | /* Never use OMP_ATOMIC for array reductions or UDRs. */ |
7647 | count = -1; |
7648 | break; |
7649 | } |
7650 | count++; |
7651 | } |
7652 | |
7653 | if (count == 0) |
7654 | return; |
7655 | |
7656 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
7657 | { |
7658 | tree var, ref, new_var, orig_var; |
7659 | enum tree_code code; |
7660 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
7661 | |
7662 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION |
7663 | || OMP_CLAUSE_REDUCTION_TASK (c)) |
7664 | continue; |
7665 | |
7666 | enum omp_clause_code ccode = OMP_CLAUSE_REDUCTION; |
7667 | orig_var = var = OMP_CLAUSE_DECL (c); |
7668 | if (TREE_CODE (var) == MEM_REF) |
7669 | { |
7670 | var = TREE_OPERAND (var, 0); |
7671 | if (TREE_CODE (var) == POINTER_PLUS_EXPR) |
7672 | var = TREE_OPERAND (var, 0); |
7673 | if (TREE_CODE (var) == ADDR_EXPR) |
7674 | var = TREE_OPERAND (var, 0); |
7675 | else |
7676 | { |
7677 | /* If this is a pointer or referenced based array |
7678 | section, the var could be private in the outer |
7679 | context e.g. on orphaned loop construct. Pretend this |
7680 | is private variable's outer reference. */ |
7681 | ccode = OMP_CLAUSE_PRIVATE; |
7682 | if (INDIRECT_REF_P (var)) |
7683 | var = TREE_OPERAND (var, 0); |
7684 | } |
7685 | orig_var = var; |
7686 | if (is_variable_sized (expr: var)) |
7687 | { |
7688 | gcc_assert (DECL_HAS_VALUE_EXPR_P (var)); |
7689 | var = DECL_VALUE_EXPR (var); |
7690 | gcc_assert (INDIRECT_REF_P (var)); |
7691 | var = TREE_OPERAND (var, 0); |
7692 | gcc_assert (DECL_P (var)); |
7693 | } |
7694 | } |
7695 | new_var = lookup_decl (var, ctx); |
7696 | if (var == OMP_CLAUSE_DECL (c) |
7697 | && omp_privatize_by_reference (decl: var)) |
7698 | new_var = build_simple_mem_ref_loc (clause_loc, new_var); |
7699 | ref = build_outer_var_ref (var, ctx, code: ccode); |
7700 | code = OMP_CLAUSE_REDUCTION_CODE (c); |
7701 | |
7702 | /* reduction(-:var) sums up the partial results, so it acts |
7703 | identically to reduction(+:var). */ |
7704 | if (code == MINUS_EXPR) |
7705 | code = PLUS_EXPR; |
7706 | |
7707 | bool is_truth_op = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR); |
7708 | if (count == 1) |
7709 | { |
7710 | tree addr = build_fold_addr_expr_loc (clause_loc, ref); |
7711 | |
7712 | addr = save_expr (addr); |
7713 | ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr); |
7714 | tree new_var2 = new_var; |
7715 | tree ref2 = ref; |
7716 | if (is_truth_op) |
7717 | { |
7718 | tree zero = build_zero_cst (TREE_TYPE (new_var)); |
7719 | new_var2 = fold_build2_loc (clause_loc, NE_EXPR, |
7720 | boolean_type_node, new_var, zero); |
7721 | ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node, |
7722 | ref, zero); |
7723 | } |
7724 | x = fold_build2_loc (clause_loc, code, TREE_TYPE (new_var2), ref2, |
7725 | new_var2); |
7726 | if (is_truth_op) |
7727 | x = fold_convert (TREE_TYPE (new_var), x); |
7728 | x = build2 (OMP_ATOMIC, void_type_node, addr, x); |
7729 | OMP_ATOMIC_MEMORY_ORDER (x) = OMP_MEMORY_ORDER_RELAXED; |
7730 | gimplify_and_add (x, stmt_seqp); |
7731 | return; |
7732 | } |
7733 | else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF) |
7734 | { |
7735 | tree d = OMP_CLAUSE_DECL (c); |
7736 | tree type = TREE_TYPE (d); |
7737 | tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type)); |
7738 | tree i = create_tmp_var (TREE_TYPE (v)); |
7739 | tree ptype = build_pointer_type (TREE_TYPE (type)); |
7740 | tree bias = TREE_OPERAND (d, 1); |
7741 | d = TREE_OPERAND (d, 0); |
7742 | if (TREE_CODE (d) == POINTER_PLUS_EXPR) |
7743 | { |
7744 | tree b = TREE_OPERAND (d, 1); |
7745 | b = maybe_lookup_decl (var: b, ctx); |
7746 | if (b == NULL) |
7747 | { |
7748 | b = TREE_OPERAND (d, 1); |
7749 | b = maybe_lookup_decl_in_outer_ctx (decl: b, ctx); |
7750 | } |
7751 | if (integer_zerop (bias)) |
7752 | bias = b; |
7753 | else |
7754 | { |
7755 | bias = fold_convert_loc (clause_loc, TREE_TYPE (b), bias); |
7756 | bias = fold_build2_loc (clause_loc, PLUS_EXPR, |
7757 | TREE_TYPE (b), b, bias); |
7758 | } |
7759 | d = TREE_OPERAND (d, 0); |
7760 | } |
7761 | /* For ref build_outer_var_ref already performs this, so |
7762 | only new_var needs a dereference. */ |
7763 | if (INDIRECT_REF_P (d)) |
7764 | { |
7765 | new_var = build_simple_mem_ref_loc (clause_loc, new_var); |
7766 | gcc_assert (omp_privatize_by_reference (var) |
7767 | && var == orig_var); |
7768 | } |
7769 | else if (TREE_CODE (d) == ADDR_EXPR) |
7770 | { |
7771 | if (orig_var == var) |
7772 | { |
7773 | new_var = build_fold_addr_expr (new_var); |
7774 | ref = build_fold_addr_expr (ref); |
7775 | } |
7776 | } |
7777 | else |
7778 | { |
7779 | gcc_assert (orig_var == var); |
7780 | if (omp_privatize_by_reference (decl: var)) |
7781 | ref = build_fold_addr_expr (ref); |
7782 | } |
7783 | if (DECL_P (v)) |
7784 | { |
7785 | tree t = maybe_lookup_decl (var: v, ctx); |
7786 | if (t) |
7787 | v = t; |
7788 | else |
7789 | v = maybe_lookup_decl_in_outer_ctx (decl: v, ctx); |
7790 | gimplify_expr (&v, stmt_seqp, NULL, is_gimple_val, fb_rvalue); |
7791 | } |
7792 | if (!integer_zerop (bias)) |
7793 | { |
7794 | bias = fold_convert_loc (clause_loc, sizetype, bias); |
7795 | new_var = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR, |
7796 | TREE_TYPE (new_var), new_var, |
7797 | unshare_expr (bias)); |
7798 | ref = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR, |
7799 | TREE_TYPE (ref), ref, bias); |
7800 | } |
7801 | new_var = fold_convert_loc (clause_loc, ptype, new_var); |
7802 | ref = fold_convert_loc (clause_loc, ptype, ref); |
7803 | tree m = create_tmp_var (ptype); |
7804 | gimplify_assign (m, new_var, stmt_seqp); |
7805 | new_var = m; |
7806 | m = create_tmp_var (ptype); |
7807 | gimplify_assign (m, ref, stmt_seqp); |
7808 | ref = m; |
7809 | gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), stmt_seqp); |
7810 | tree body = create_artificial_label (UNKNOWN_LOCATION); |
7811 | tree end = create_artificial_label (UNKNOWN_LOCATION); |
7812 | gimple_seq_add_stmt (&sub_seq, gimple_build_label (label: body)); |
7813 | tree priv = build_simple_mem_ref_loc (clause_loc, new_var); |
7814 | tree out = build_simple_mem_ref_loc (clause_loc, ref); |
7815 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
7816 | { |
7817 | tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); |
7818 | tree decl_placeholder |
7819 | = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c); |
7820 | SET_DECL_VALUE_EXPR (placeholder, out); |
7821 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
7822 | SET_DECL_VALUE_EXPR (decl_placeholder, priv); |
7823 | DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1; |
7824 | lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx); |
7825 | gimple_seq_add_seq (&sub_seq, |
7826 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c)); |
7827 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; |
7828 | OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL; |
7829 | OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL; |
7830 | } |
7831 | else |
7832 | { |
7833 | tree out2 = out; |
7834 | tree priv2 = priv; |
7835 | if (is_truth_op) |
7836 | { |
7837 | tree zero = build_zero_cst (TREE_TYPE (out)); |
7838 | out2 = fold_build2_loc (clause_loc, NE_EXPR, |
7839 | boolean_type_node, out, zero); |
7840 | priv2 = fold_build2_loc (clause_loc, NE_EXPR, |
7841 | boolean_type_node, priv, zero); |
7842 | } |
7843 | x = build2 (code, TREE_TYPE (out2), out2, priv2); |
7844 | if (is_truth_op) |
7845 | x = fold_convert (TREE_TYPE (out), x); |
7846 | out = unshare_expr (out); |
7847 | gimplify_assign (out, x, &sub_seq); |
7848 | } |
7849 | gimple *g = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var, |
7850 | TYPE_SIZE_UNIT (TREE_TYPE (type))); |
7851 | gimple_seq_add_stmt (&sub_seq, g); |
7852 | g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref, |
7853 | TYPE_SIZE_UNIT (TREE_TYPE (type))); |
7854 | gimple_seq_add_stmt (&sub_seq, g); |
7855 | g = gimple_build_assign (i, PLUS_EXPR, i, |
7856 | build_int_cst (TREE_TYPE (i), 1)); |
7857 | gimple_seq_add_stmt (&sub_seq, g); |
7858 | g = gimple_build_cond (LE_EXPR, i, v, body, end); |
7859 | gimple_seq_add_stmt (&sub_seq, g); |
7860 | gimple_seq_add_stmt (&sub_seq, gimple_build_label (label: end)); |
7861 | } |
7862 | else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
7863 | { |
7864 | tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); |
7865 | |
7866 | if (omp_privatize_by_reference (decl: var) |
7867 | && !useless_type_conversion_p (TREE_TYPE (placeholder), |
7868 | TREE_TYPE (ref))) |
7869 | ref = build_fold_addr_expr_loc (clause_loc, ref); |
7870 | SET_DECL_VALUE_EXPR (placeholder, ref); |
7871 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
7872 | lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx); |
7873 | gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c)); |
7874 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; |
7875 | OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL; |
7876 | } |
7877 | else |
7878 | { |
7879 | tree new_var2 = new_var; |
7880 | tree ref2 = ref; |
7881 | if (is_truth_op) |
7882 | { |
7883 | tree zero = build_zero_cst (TREE_TYPE (new_var)); |
7884 | new_var2 = fold_build2_loc (clause_loc, NE_EXPR, |
7885 | boolean_type_node, new_var, zero); |
7886 | ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node, |
7887 | ref, zero); |
7888 | } |
7889 | x = build2 (code, TREE_TYPE (ref), ref2, new_var2); |
7890 | if (is_truth_op) |
7891 | x = fold_convert (TREE_TYPE (new_var), x); |
7892 | ref = build_outer_var_ref (var, ctx); |
7893 | gimplify_assign (ref, x, &sub_seq); |
7894 | } |
7895 | } |
7896 | |
7897 | stmt = gimple_build_call (builtin_decl_explicit (fncode: BUILT_IN_GOMP_ATOMIC_START), |
7898 | 0); |
7899 | gimple_seq_add_stmt (stmt_seqp, stmt); |
7900 | |
7901 | gimple_seq_add_seq (stmt_seqp, sub_seq); |
7902 | |
7903 | if (clist) |
7904 | { |
7905 | gimple_seq_add_seq (stmt_seqp, *clist); |
7906 | *clist = NULL; |
7907 | } |
7908 | |
7909 | stmt = gimple_build_call (builtin_decl_explicit (fncode: BUILT_IN_GOMP_ATOMIC_END), |
7910 | 0); |
7911 | gimple_seq_add_stmt (stmt_seqp, stmt); |
7912 | } |
7913 | |
7914 | |
7915 | /* Generate code to implement the COPYPRIVATE clauses. */ |
7916 | |
7917 | static void |
7918 | lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist, |
7919 | omp_context *ctx) |
7920 | { |
7921 | tree c; |
7922 | |
7923 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
7924 | { |
7925 | tree var, new_var, ref, x; |
7926 | bool by_ref; |
7927 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
7928 | |
7929 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE) |
7930 | continue; |
7931 | |
7932 | var = OMP_CLAUSE_DECL (c); |
7933 | by_ref = use_pointer_for_field (decl: var, NULL); |
7934 | |
7935 | ref = build_sender_ref (var, ctx); |
7936 | x = new_var = lookup_decl_in_outer_ctx (decl: var, ctx); |
7937 | if (by_ref) |
7938 | { |
7939 | x = build_fold_addr_expr_loc (clause_loc, new_var); |
7940 | x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x); |
7941 | } |
7942 | gimplify_assign (ref, x, slist); |
7943 | |
7944 | ref = build_receiver_ref (var, by_ref: false, ctx); |
7945 | if (by_ref) |
7946 | { |
7947 | ref = fold_convert_loc (clause_loc, |
7948 | build_pointer_type (TREE_TYPE (new_var)), |
7949 | ref); |
7950 | ref = build_fold_indirect_ref_loc (clause_loc, ref); |
7951 | } |
7952 | if (omp_privatize_by_reference (decl: var)) |
7953 | { |
7954 | ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref); |
7955 | ref = build_simple_mem_ref_loc (clause_loc, ref); |
7956 | new_var = build_simple_mem_ref_loc (clause_loc, new_var); |
7957 | } |
7958 | x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref); |
7959 | gimplify_and_add (x, rlist); |
7960 | } |
7961 | } |
7962 | |
7963 | |
7964 | /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE, |
7965 | and REDUCTION from the sender (aka parent) side. */ |
7966 | |
7967 | static void |
7968 | lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist, |
7969 | omp_context *ctx) |
7970 | { |
7971 | tree c, t; |
7972 | int ignored_looptemp = 0; |
7973 | bool is_taskloop = false; |
7974 | |
7975 | /* For taskloop, ignore first two _looptemp_ clauses, those are initialized |
7976 | by GOMP_taskloop. */ |
7977 | if (is_task_ctx (ctx) && gimple_omp_task_taskloop_p (g: ctx->stmt)) |
7978 | { |
7979 | ignored_looptemp = 2; |
7980 | is_taskloop = true; |
7981 | } |
7982 | |
7983 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
7984 | { |
7985 | tree val, ref, x, var; |
7986 | bool by_ref, do_in = false, do_out = false; |
7987 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
7988 | |
7989 | switch (OMP_CLAUSE_CODE (c)) |
7990 | { |
7991 | case OMP_CLAUSE_PRIVATE: |
7992 | if (OMP_CLAUSE_PRIVATE_OUTER_REF (c)) |
7993 | break; |
7994 | continue; |
7995 | case OMP_CLAUSE_FIRSTPRIVATE: |
7996 | case OMP_CLAUSE_COPYIN: |
7997 | case OMP_CLAUSE_LASTPRIVATE: |
7998 | case OMP_CLAUSE_IN_REDUCTION: |
7999 | case OMP_CLAUSE__REDUCTEMP_: |
8000 | break; |
8001 | case OMP_CLAUSE_REDUCTION: |
8002 | if (is_task_ctx (ctx) || OMP_CLAUSE_REDUCTION_TASK (c)) |
8003 | continue; |
8004 | break; |
8005 | case OMP_CLAUSE_SHARED: |
8006 | if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)) |
8007 | break; |
8008 | continue; |
8009 | case OMP_CLAUSE__LOOPTEMP_: |
8010 | if (ignored_looptemp) |
8011 | { |
8012 | ignored_looptemp--; |
8013 | continue; |
8014 | } |
8015 | break; |
8016 | default: |
8017 | continue; |
8018 | } |
8019 | |
8020 | val = OMP_CLAUSE_DECL (c); |
8021 | if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
8022 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION) |
8023 | && TREE_CODE (val) == MEM_REF) |
8024 | { |
8025 | val = TREE_OPERAND (val, 0); |
8026 | if (TREE_CODE (val) == POINTER_PLUS_EXPR) |
8027 | val = TREE_OPERAND (val, 0); |
8028 | if (INDIRECT_REF_P (val) |
8029 | || TREE_CODE (val) == ADDR_EXPR) |
8030 | val = TREE_OPERAND (val, 0); |
8031 | if (is_variable_sized (expr: val)) |
8032 | continue; |
8033 | } |
8034 | |
8035 | /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the |
8036 | outer taskloop region. */ |
8037 | omp_context *ctx_for_o = ctx; |
8038 | if (is_taskloop |
8039 | && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED |
8040 | && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)) |
8041 | ctx_for_o = ctx->outer; |
8042 | |
8043 | var = lookup_decl_in_outer_ctx (decl: val, ctx: ctx_for_o); |
8044 | |
8045 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN |
8046 | && is_global_var (t: var) |
8047 | && (val == OMP_CLAUSE_DECL (c) |
8048 | || !is_task_ctx (ctx) |
8049 | || (TREE_CODE (TREE_TYPE (val)) != POINTER_TYPE |
8050 | && (TREE_CODE (TREE_TYPE (val)) != REFERENCE_TYPE |
8051 | || (TREE_CODE (TREE_TYPE (TREE_TYPE (val))) |
8052 | != POINTER_TYPE))))) |
8053 | continue; |
8054 | |
8055 | t = omp_member_access_dummy_var (decl: var); |
8056 | if (t) |
8057 | { |
8058 | var = DECL_VALUE_EXPR (var); |
8059 | tree o = maybe_lookup_decl_in_outer_ctx (decl: t, ctx: ctx_for_o); |
8060 | if (o != t) |
8061 | var = unshare_and_remap (x: var, from: t, to: o); |
8062 | else |
8063 | var = unshare_expr (var); |
8064 | } |
8065 | |
8066 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED) |
8067 | { |
8068 | /* Handle taskloop firstprivate/lastprivate, where the |
8069 | lastprivate on GIMPLE_OMP_TASK is represented as |
8070 | OMP_CLAUSE_SHARED_FIRSTPRIVATE. */ |
8071 | tree f = lookup_sfield (key: (splay_tree_key) &DECL_UID (val), ctx); |
8072 | x = omp_build_component_ref (obj: ctx->sender_decl, field: f); |
8073 | if (use_pointer_for_field (decl: val, shared_ctx: ctx)) |
8074 | var = build_fold_addr_expr (var); |
8075 | gimplify_assign (x, var, ilist); |
8076 | DECL_ABSTRACT_ORIGIN (f) = NULL; |
8077 | continue; |
8078 | } |
8079 | |
8080 | if (((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION |
8081 | && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IN_REDUCTION) |
8082 | || val == OMP_CLAUSE_DECL (c)) |
8083 | && is_variable_sized (expr: val)) |
8084 | continue; |
8085 | by_ref = use_pointer_for_field (decl: val, NULL); |
8086 | |
8087 | switch (OMP_CLAUSE_CODE (c)) |
8088 | { |
8089 | case OMP_CLAUSE_FIRSTPRIVATE: |
8090 | if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c) |
8091 | && !by_ref |
8092 | && is_task_ctx (ctx)) |
8093 | suppress_warning (var); |
8094 | do_in = true; |
8095 | break; |
8096 | |
8097 | case OMP_CLAUSE_PRIVATE: |
8098 | case OMP_CLAUSE_COPYIN: |
8099 | case OMP_CLAUSE__LOOPTEMP_: |
8100 | case OMP_CLAUSE__REDUCTEMP_: |
8101 | do_in = true; |
8102 | break; |
8103 | |
8104 | case OMP_CLAUSE_LASTPRIVATE: |
8105 | if (by_ref || omp_privatize_by_reference (decl: val)) |
8106 | { |
8107 | if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) |
8108 | continue; |
8109 | do_in = true; |
8110 | } |
8111 | else |
8112 | { |
8113 | do_out = true; |
8114 | if (lang_hooks.decls.omp_private_outer_ref (val)) |
8115 | do_in = true; |
8116 | } |
8117 | break; |
8118 | |
8119 | case OMP_CLAUSE_REDUCTION: |
8120 | case OMP_CLAUSE_IN_REDUCTION: |
8121 | do_in = true; |
8122 | if (val == OMP_CLAUSE_DECL (c)) |
8123 | { |
8124 | if (is_task_ctx (ctx)) |
8125 | by_ref = use_pointer_for_field (decl: val, shared_ctx: ctx); |
8126 | else |
8127 | do_out = !(by_ref || omp_privatize_by_reference (decl: val)); |
8128 | } |
8129 | else |
8130 | by_ref = TREE_CODE (TREE_TYPE (val)) == ARRAY_TYPE; |
8131 | break; |
8132 | |
8133 | default: |
8134 | gcc_unreachable (); |
8135 | } |
8136 | |
8137 | if (do_in) |
8138 | { |
8139 | ref = build_sender_ref (var: val, ctx); |
8140 | x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var; |
8141 | gimplify_assign (ref, x, ilist); |
8142 | if (is_task_ctx (ctx)) |
8143 | DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL; |
8144 | } |
8145 | |
8146 | if (do_out) |
8147 | { |
8148 | ref = build_sender_ref (var: val, ctx); |
8149 | gimplify_assign (var, ref, olist); |
8150 | } |
8151 | } |
8152 | } |
8153 | |
8154 | /* Generate code to implement SHARED from the sender (aka parent) |
8155 | side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't |
8156 | list things that got automatically shared. */ |
8157 | |
8158 | static void |
8159 | lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx) |
8160 | { |
8161 | tree var, ovar, nvar, t, f, x, record_type; |
8162 | |
8163 | if (ctx->record_type == NULL) |
8164 | return; |
8165 | |
8166 | record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type; |
8167 | for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f)) |
8168 | { |
8169 | ovar = DECL_ABSTRACT_ORIGIN (f); |
8170 | if (!ovar || TREE_CODE (ovar) == FIELD_DECL) |
8171 | continue; |
8172 | |
8173 | nvar = maybe_lookup_decl (var: ovar, ctx); |
8174 | if (!nvar |
8175 | || !DECL_HAS_VALUE_EXPR_P (nvar) |
8176 | || (ctx->allocate_map |
8177 | && ctx->allocate_map->get (k: ovar))) |
8178 | continue; |
8179 | |
8180 | /* If CTX is a nested parallel directive. Find the immediately |
8181 | enclosing parallel or workshare construct that contains a |
8182 | mapping for OVAR. */ |
8183 | var = lookup_decl_in_outer_ctx (decl: ovar, ctx); |
8184 | |
8185 | t = omp_member_access_dummy_var (decl: var); |
8186 | if (t) |
8187 | { |
8188 | var = DECL_VALUE_EXPR (var); |
8189 | tree o = maybe_lookup_decl_in_outer_ctx (decl: t, ctx); |
8190 | if (o != t) |
8191 | var = unshare_and_remap (x: var, from: t, to: o); |
8192 | else |
8193 | var = unshare_expr (var); |
8194 | } |
8195 | |
8196 | if (use_pointer_for_field (decl: ovar, shared_ctx: ctx)) |
8197 | { |
8198 | x = build_sender_ref (var: ovar, ctx); |
8199 | if (TREE_CODE (TREE_TYPE (f)) == ARRAY_TYPE |
8200 | && TREE_TYPE (f) == TREE_TYPE (ovar)) |
8201 | { |
8202 | gcc_assert (is_parallel_ctx (ctx) |
8203 | && DECL_ARTIFICIAL (ovar)); |
8204 | /* _condtemp_ clause. */ |
8205 | var = build_constructor (TREE_TYPE (x), NULL); |
8206 | } |
8207 | else |
8208 | var = build_fold_addr_expr (var); |
8209 | gimplify_assign (x, var, ilist); |
8210 | } |
8211 | else |
8212 | { |
8213 | x = build_sender_ref (var: ovar, ctx); |
8214 | gimplify_assign (x, var, ilist); |
8215 | |
8216 | if (!TREE_READONLY (var) |
8217 | /* We don't need to receive a new reference to a result |
8218 | or parm decl. In fact we may not store to it as we will |
8219 | invalidate any pending RSO and generate wrong gimple |
8220 | during inlining. */ |
8221 | && !((TREE_CODE (var) == RESULT_DECL |
8222 | || TREE_CODE (var) == PARM_DECL) |
8223 | && DECL_BY_REFERENCE (var))) |
8224 | { |
8225 | x = build_sender_ref (var: ovar, ctx); |
8226 | gimplify_assign (var, x, olist); |
8227 | } |
8228 | } |
8229 | } |
8230 | } |
8231 | |
8232 | /* Emit an OpenACC head marker call, encapulating the partitioning and |
8233 | other information that must be processed by the target compiler. |
8234 | Return the maximum number of dimensions the associated loop might |
8235 | be partitioned over. */ |
8236 | |
8237 | static unsigned |
8238 | lower_oacc_head_mark (location_t loc, tree ddvar, tree clauses, |
8239 | gimple_seq *seq, omp_context *ctx) |
8240 | { |
8241 | unsigned levels = 0; |
8242 | unsigned tag = 0; |
8243 | tree gang_static = NULL_TREE; |
8244 | auto_vec<tree, 5> args; |
8245 | |
8246 | args.quick_push (obj: build_int_cst |
8247 | (integer_type_node, IFN_UNIQUE_OACC_HEAD_MARK)); |
8248 | args.quick_push (obj: ddvar); |
8249 | for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
8250 | { |
8251 | switch (OMP_CLAUSE_CODE (c)) |
8252 | { |
8253 | case OMP_CLAUSE_GANG: |
8254 | tag |= OLF_DIM_GANG; |
8255 | gang_static = OMP_CLAUSE_GANG_STATIC_EXPR (c); |
8256 | /* static:* is represented by -1, and we can ignore it, as |
8257 | scheduling is always static. */ |
8258 | if (gang_static && integer_minus_onep (gang_static)) |
8259 | gang_static = NULL_TREE; |
8260 | levels++; |
8261 | break; |
8262 | |
8263 | case OMP_CLAUSE_WORKER: |
8264 | tag |= OLF_DIM_WORKER; |
8265 | levels++; |
8266 | break; |
8267 | |
8268 | case OMP_CLAUSE_VECTOR: |
8269 | tag |= OLF_DIM_VECTOR; |
8270 | levels++; |
8271 | break; |
8272 | |
8273 | case OMP_CLAUSE_SEQ: |
8274 | tag |= OLF_SEQ; |
8275 | break; |
8276 | |
8277 | case OMP_CLAUSE_AUTO: |
8278 | tag |= OLF_AUTO; |
8279 | break; |
8280 | |
8281 | case OMP_CLAUSE_INDEPENDENT: |
8282 | tag |= OLF_INDEPENDENT; |
8283 | break; |
8284 | |
8285 | case OMP_CLAUSE_TILE: |
8286 | tag |= OLF_TILE; |
8287 | break; |
8288 | |
8289 | case OMP_CLAUSE_REDUCTION: |
8290 | tag |= OLF_REDUCTION; |
8291 | break; |
8292 | |
8293 | default: |
8294 | continue; |
8295 | } |
8296 | } |
8297 | |
8298 | if (gang_static) |
8299 | { |
8300 | if (DECL_P (gang_static)) |
8301 | gang_static = build_outer_var_ref (var: gang_static, ctx); |
8302 | tag |= OLF_GANG_STATIC; |
8303 | } |
8304 | |
8305 | omp_context *tgt = enclosing_target_ctx (ctx); |
8306 | if (!tgt || is_oacc_parallel_or_serial (ctx: tgt)) |
8307 | ; |
8308 | else if (is_oacc_kernels (ctx: tgt)) |
8309 | /* Not using this loops handling inside OpenACC 'kernels' regions. */ |
8310 | gcc_unreachable (); |
8311 | else if (is_oacc_kernels_decomposed_part (ctx: tgt)) |
8312 | ; |
8313 | else |
8314 | gcc_unreachable (); |
8315 | |
8316 | /* In a parallel region, loops are implicitly INDEPENDENT. */ |
8317 | if (!tgt || is_oacc_parallel_or_serial (ctx: tgt)) |
8318 | tag |= OLF_INDEPENDENT; |
8319 | |
8320 | /* Loops inside OpenACC 'kernels' decomposed parts' regions are expected to |
8321 | have an explicit 'seq' or 'independent' clause, and no 'auto' clause. */ |
8322 | if (tgt && is_oacc_kernels_decomposed_part (ctx: tgt)) |
8323 | { |
8324 | gcc_assert (tag & (OLF_SEQ | OLF_INDEPENDENT)); |
8325 | gcc_assert (!(tag & OLF_AUTO)); |
8326 | } |
8327 | |
8328 | if (tag & OLF_TILE) |
8329 | /* Tiling could use all 3 levels. */ |
8330 | levels = 3; |
8331 | else |
8332 | { |
8333 | /* A loop lacking SEQ, GANG, WORKER and/or VECTOR could be AUTO. |
8334 | Ensure at least one level, or 2 for possible auto |
8335 | partitioning */ |
8336 | bool maybe_auto = !(tag & (((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1) |
8337 | << OLF_DIM_BASE) | OLF_SEQ)); |
8338 | |
8339 | if (levels < 1u + maybe_auto) |
8340 | levels = 1u + maybe_auto; |
8341 | } |
8342 | |
8343 | args.quick_push (obj: build_int_cst (integer_type_node, levels)); |
8344 | args.quick_push (obj: build_int_cst (integer_type_node, tag)); |
8345 | if (gang_static) |
8346 | args.quick_push (obj: gang_static); |
8347 | |
8348 | gcall *call = gimple_build_call_internal_vec (IFN_UNIQUE, args); |
8349 | gimple_set_location (g: call, location: loc); |
8350 | gimple_set_lhs (call, ddvar); |
8351 | gimple_seq_add_stmt (seq, call); |
8352 | |
8353 | return levels; |
8354 | } |
8355 | |
8356 | /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the |
8357 | partitioning level of the enclosed region. */ |
8358 | |
8359 | static void |
8360 | lower_oacc_loop_marker (location_t loc, tree ddvar, bool head, |
8361 | tree tofollow, gimple_seq *seq) |
8362 | { |
8363 | int marker_kind = (head ? IFN_UNIQUE_OACC_HEAD_MARK |
8364 | : IFN_UNIQUE_OACC_TAIL_MARK); |
8365 | tree marker = build_int_cst (integer_type_node, marker_kind); |
8366 | int nargs = 2 + (tofollow != NULL_TREE); |
8367 | gcall *call = gimple_build_call_internal (IFN_UNIQUE, nargs, |
8368 | marker, ddvar, tofollow); |
8369 | gimple_set_location (g: call, location: loc); |
8370 | gimple_set_lhs (call, ddvar); |
8371 | gimple_seq_add_stmt (seq, call); |
8372 | } |
8373 | |
8374 | /* Generate the before and after OpenACC loop sequences. CLAUSES are |
8375 | the loop clauses, from which we extract reductions. Initialize |
8376 | HEAD and TAIL. */ |
8377 | |
8378 | static void |
8379 | lower_oacc_head_tail (location_t loc, tree clauses, gcall *private_marker, |
8380 | gimple_seq *head, gimple_seq *tail, omp_context *ctx) |
8381 | { |
8382 | bool inner = false; |
8383 | tree ddvar = create_tmp_var (integer_type_node, ".data_dep" ); |
8384 | gimple_seq_add_stmt (head, gimple_build_assign (ddvar, integer_zero_node)); |
8385 | |
8386 | unsigned count = lower_oacc_head_mark (loc, ddvar, clauses, seq: head, ctx); |
8387 | |
8388 | if (private_marker) |
8389 | { |
8390 | gimple_set_location (g: private_marker, location: loc); |
8391 | gimple_call_set_lhs (gs: private_marker, lhs: ddvar); |
8392 | gimple_call_set_arg (gs: private_marker, index: 1, arg: ddvar); |
8393 | } |
8394 | |
8395 | tree fork_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_FORK); |
8396 | tree join_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_JOIN); |
8397 | |
8398 | gcc_assert (count); |
8399 | for (unsigned done = 1; count; count--, done++) |
8400 | { |
8401 | gimple_seq fork_seq = NULL; |
8402 | gimple_seq join_seq = NULL; |
8403 | |
8404 | tree place = build_int_cst (integer_type_node, -1); |
8405 | gcall *fork = gimple_build_call_internal (IFN_UNIQUE, 3, |
8406 | fork_kind, ddvar, place); |
8407 | gimple_set_location (g: fork, location: loc); |
8408 | gimple_set_lhs (fork, ddvar); |
8409 | |
8410 | gcall *join = gimple_build_call_internal (IFN_UNIQUE, 3, |
8411 | join_kind, ddvar, place); |
8412 | gimple_set_location (g: join, location: loc); |
8413 | gimple_set_lhs (join, ddvar); |
8414 | |
8415 | /* Mark the beginning of this level sequence. */ |
8416 | if (inner) |
8417 | lower_oacc_loop_marker (loc, ddvar, head: true, |
8418 | tofollow: build_int_cst (integer_type_node, count), |
8419 | seq: &fork_seq); |
8420 | lower_oacc_loop_marker (loc, ddvar, head: false, |
8421 | tofollow: build_int_cst (integer_type_node, done), |
8422 | seq: &join_seq); |
8423 | |
8424 | lower_oacc_reductions (loc, clauses, level: place, inner, |
8425 | fork, private_marker: (count == 1) ? private_marker : NULL, |
8426 | join, fork_seq: &fork_seq, join_seq: &join_seq, ctx); |
8427 | |
8428 | /* Append this level to head. */ |
8429 | gimple_seq_add_seq (head, fork_seq); |
8430 | /* Prepend it to tail. */ |
8431 | gimple_seq_add_seq (&join_seq, *tail); |
8432 | *tail = join_seq; |
8433 | |
8434 | inner = true; |
8435 | } |
8436 | |
8437 | /* Mark the end of the sequence. */ |
8438 | lower_oacc_loop_marker (loc, ddvar, head: true, NULL_TREE, seq: head); |
8439 | lower_oacc_loop_marker (loc, ddvar, head: false, NULL_TREE, seq: tail); |
8440 | } |
8441 | |
8442 | /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW |
8443 | catch handler and return it. This prevents programs from violating the |
8444 | structured block semantics with throws. */ |
8445 | |
8446 | static gimple_seq |
8447 | maybe_catch_exception (gimple_seq body) |
8448 | { |
8449 | gimple *g; |
8450 | tree decl; |
8451 | |
8452 | if (!flag_exceptions) |
8453 | return body; |
8454 | |
8455 | if (lang_hooks.eh_protect_cleanup_actions != NULL) |
8456 | decl = lang_hooks.eh_protect_cleanup_actions (); |
8457 | else |
8458 | decl = builtin_decl_explicit (fncode: BUILT_IN_TRAP); |
8459 | |
8460 | g = gimple_build_eh_must_not_throw (decl); |
8461 | g = gimple_build_try (body, gimple_seq_alloc_with_stmt (stmt: g), |
8462 | GIMPLE_TRY_CATCH); |
8463 | |
8464 | return gimple_seq_alloc_with_stmt (stmt: g); |
8465 | } |
8466 | |
8467 | |
8468 | /* Routines to lower OMP directives into OMP-GIMPLE. */ |
8469 | |
8470 | /* If ctx is a worksharing context inside of a cancellable parallel |
8471 | region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN |
8472 | and conditional branch to parallel's cancel_label to handle |
8473 | cancellation in the implicit barrier. */ |
8474 | |
8475 | static void |
8476 | maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple *omp_return, |
8477 | gimple_seq *body) |
8478 | { |
8479 | gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN); |
8480 | if (gimple_omp_return_nowait_p (g: omp_return)) |
8481 | return; |
8482 | for (omp_context *outer = ctx->outer; outer; outer = outer->outer) |
8483 | if (gimple_code (g: outer->stmt) == GIMPLE_OMP_PARALLEL |
8484 | && outer->cancellable) |
8485 | { |
8486 | tree fndecl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_CANCEL); |
8487 | tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl)); |
8488 | tree lhs = create_tmp_var (c_bool_type); |
8489 | gimple_omp_return_set_lhs (g: omp_return, lhs); |
8490 | tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION); |
8491 | gimple *g = gimple_build_cond (NE_EXPR, lhs, |
8492 | fold_convert (c_bool_type, |
8493 | boolean_false_node), |
8494 | outer->cancel_label, fallthru_label); |
8495 | gimple_seq_add_stmt (body, g); |
8496 | gimple_seq_add_stmt (body, gimple_build_label (label: fallthru_label)); |
8497 | } |
8498 | else if (gimple_code (g: outer->stmt) != GIMPLE_OMP_TASKGROUP |
8499 | && gimple_code (g: outer->stmt) != GIMPLE_OMP_SCOPE) |
8500 | return; |
8501 | } |
8502 | |
8503 | /* Find the first task_reduction or reduction clause or return NULL |
8504 | if there are none. */ |
8505 | |
8506 | static inline tree |
8507 | omp_task_reductions_find_first (tree clauses, enum tree_code code, |
8508 | enum omp_clause_code ccode) |
8509 | { |
8510 | while (1) |
8511 | { |
8512 | clauses = omp_find_clause (clauses, kind: ccode); |
8513 | if (clauses == NULL_TREE) |
8514 | return NULL_TREE; |
8515 | if (ccode != OMP_CLAUSE_REDUCTION |
8516 | || code == OMP_TASKLOOP |
8517 | || OMP_CLAUSE_REDUCTION_TASK (clauses)) |
8518 | return clauses; |
8519 | clauses = OMP_CLAUSE_CHAIN (clauses); |
8520 | } |
8521 | } |
8522 | |
8523 | static void lower_omp_task_reductions (omp_context *, enum tree_code, tree, |
8524 | gimple_seq *, gimple_seq *); |
8525 | |
8526 | /* Lower the OpenMP sections directive in the current statement in GSI_P. |
8527 | CTX is the enclosing OMP context for the current statement. */ |
8528 | |
8529 | static void |
8530 | lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
8531 | { |
8532 | tree block, control; |
8533 | gimple_stmt_iterator tgsi; |
8534 | gomp_sections *stmt; |
8535 | gimple *t; |
8536 | gbind *new_stmt, *bind; |
8537 | gimple_seq ilist, dlist, olist, tred_dlist = NULL, clist = NULL, new_body; |
8538 | |
8539 | stmt = as_a <gomp_sections *> (p: gsi_stmt (i: *gsi_p)); |
8540 | |
8541 | push_gimplify_context (); |
8542 | |
8543 | dlist = NULL; |
8544 | ilist = NULL; |
8545 | |
8546 | tree rclauses |
8547 | = omp_task_reductions_find_first (clauses: gimple_omp_sections_clauses (gs: stmt), |
8548 | code: OMP_SECTIONS, ccode: OMP_CLAUSE_REDUCTION); |
8549 | tree rtmp = NULL_TREE; |
8550 | if (rclauses) |
8551 | { |
8552 | tree type = build_pointer_type (pointer_sized_int_node); |
8553 | tree temp = create_tmp_var (type); |
8554 | tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_); |
8555 | OMP_CLAUSE_DECL (c) = temp; |
8556 | OMP_CLAUSE_CHAIN (c) = gimple_omp_sections_clauses (gs: stmt); |
8557 | gimple_omp_sections_set_clauses (gs: stmt, clauses: c); |
8558 | lower_omp_task_reductions (ctx, OMP_SECTIONS, |
8559 | gimple_omp_sections_clauses (gs: stmt), |
8560 | &ilist, &tred_dlist); |
8561 | rclauses = c; |
8562 | rtmp = make_ssa_name (var: type); |
8563 | gimple_seq_add_stmt (&ilist, gimple_build_assign (rtmp, temp)); |
8564 | } |
8565 | |
8566 | tree *clauses_ptr = gimple_omp_sections_clauses_ptr (gs: stmt); |
8567 | lower_lastprivate_conditional_clauses (clauses: clauses_ptr, ctx); |
8568 | |
8569 | lower_rec_input_clauses (clauses: gimple_omp_sections_clauses (gs: stmt), |
8570 | ilist: &ilist, dlist: &dlist, ctx, NULL); |
8571 | |
8572 | control = create_tmp_var (unsigned_type_node, ".section" ); |
8573 | gimple_omp_sections_set_control (gs: stmt, control); |
8574 | |
8575 | new_body = gimple_omp_body (gs: stmt); |
8576 | gimple_omp_set_body (gs: stmt, NULL); |
8577 | tgsi = gsi_start (seq&: new_body); |
8578 | for (; !gsi_end_p (i: tgsi); gsi_next (i: &tgsi)) |
8579 | { |
8580 | omp_context *sctx; |
8581 | gimple *sec_start; |
8582 | |
8583 | sec_start = gsi_stmt (i: tgsi); |
8584 | sctx = maybe_lookup_ctx (stmt: sec_start); |
8585 | gcc_assert (sctx); |
8586 | |
8587 | lower_omp (gimple_omp_body_ptr (gs: sec_start), sctx); |
8588 | gsi_insert_seq_after (&tgsi, gimple_omp_body (gs: sec_start), |
8589 | GSI_CONTINUE_LINKING); |
8590 | gimple_omp_set_body (gs: sec_start, NULL); |
8591 | |
8592 | if (gsi_one_before_end_p (i: tgsi)) |
8593 | { |
8594 | gimple_seq l = NULL; |
8595 | lower_lastprivate_clauses (clauses: gimple_omp_sections_clauses (gs: stmt), NULL, |
8596 | body_p: &ilist, stmt_list: &l, cstmt_list: &clist, ctx); |
8597 | gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING); |
8598 | gimple_omp_section_set_last (g: sec_start); |
8599 | } |
8600 | |
8601 | gsi_insert_after (&tgsi, gimple_build_omp_return (false), |
8602 | GSI_CONTINUE_LINKING); |
8603 | } |
8604 | |
8605 | block = make_node (BLOCK); |
8606 | bind = gimple_build_bind (NULL, new_body, block); |
8607 | |
8608 | olist = NULL; |
8609 | lower_reduction_clauses (clauses: gimple_omp_sections_clauses (gs: stmt), stmt_seqp: &olist, |
8610 | clist: &clist, ctx); |
8611 | if (clist) |
8612 | { |
8613 | tree fndecl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_ATOMIC_START); |
8614 | gcall *g = gimple_build_call (fndecl, 0); |
8615 | gimple_seq_add_stmt (&olist, g); |
8616 | gimple_seq_add_seq (&olist, clist); |
8617 | fndecl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_ATOMIC_END); |
8618 | g = gimple_build_call (fndecl, 0); |
8619 | gimple_seq_add_stmt (&olist, g); |
8620 | } |
8621 | |
8622 | block = make_node (BLOCK); |
8623 | new_stmt = gimple_build_bind (NULL, NULL, block); |
8624 | gsi_replace (gsi_p, new_stmt, true); |
8625 | |
8626 | pop_gimplify_context (new_stmt); |
8627 | gimple_bind_append_vars (bind_stmt: new_stmt, vars: ctx->block_vars); |
8628 | BLOCK_VARS (block) = gimple_bind_vars (bind_stmt: bind); |
8629 | if (BLOCK_VARS (block)) |
8630 | TREE_USED (block) = 1; |
8631 | |
8632 | new_body = NULL; |
8633 | gimple_seq_add_seq (&new_body, ilist); |
8634 | gimple_seq_add_stmt (&new_body, stmt); |
8635 | gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ()); |
8636 | gimple_seq_add_stmt (&new_body, bind); |
8637 | |
8638 | t = gimple_build_omp_continue (control, control); |
8639 | gimple_seq_add_stmt (&new_body, t); |
8640 | |
8641 | gimple_seq_add_seq (&new_body, olist); |
8642 | if (ctx->cancellable) |
8643 | gimple_seq_add_stmt (&new_body, gimple_build_label (label: ctx->cancel_label)); |
8644 | gimple_seq_add_seq (&new_body, dlist); |
8645 | |
8646 | new_body = maybe_catch_exception (body: new_body); |
8647 | |
8648 | bool nowait = omp_find_clause (clauses: gimple_omp_sections_clauses (gs: stmt), |
8649 | kind: OMP_CLAUSE_NOWAIT) != NULL_TREE; |
8650 | t = gimple_build_omp_return (nowait); |
8651 | gimple_seq_add_stmt (&new_body, t); |
8652 | gimple_seq_add_seq (&new_body, tred_dlist); |
8653 | maybe_add_implicit_barrier_cancel (ctx, omp_return: t, body: &new_body); |
8654 | |
8655 | if (rclauses) |
8656 | OMP_CLAUSE_DECL (rclauses) = rtmp; |
8657 | |
8658 | gimple_bind_set_body (bind_stmt: new_stmt, seq: new_body); |
8659 | } |
8660 | |
8661 | |
8662 | /* A subroutine of lower_omp_single. Expand the simple form of |
8663 | a GIMPLE_OMP_SINGLE, without a copyprivate clause: |
8664 | |
8665 | if (GOMP_single_start ()) |
8666 | BODY; |
8667 | [ GOMP_barrier (); ] -> unless 'nowait' is present. |
8668 | |
8669 | FIXME. It may be better to delay expanding the logic of this until |
8670 | pass_expand_omp. The expanded logic may make the job more difficult |
8671 | to a synchronization analysis pass. */ |
8672 | |
8673 | static void |
8674 | lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p) |
8675 | { |
8676 | location_t loc = gimple_location (g: single_stmt); |
8677 | tree tlabel = create_artificial_label (loc); |
8678 | tree flabel = create_artificial_label (loc); |
8679 | gimple *call, *cond; |
8680 | tree lhs, decl; |
8681 | |
8682 | decl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_SINGLE_START); |
8683 | lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl))); |
8684 | call = gimple_build_call (decl, 0); |
8685 | gimple_call_set_lhs (gs: call, lhs); |
8686 | gimple_seq_add_stmt (pre_p, call); |
8687 | |
8688 | cond = gimple_build_cond (EQ_EXPR, lhs, |
8689 | fold_convert_loc (loc, TREE_TYPE (lhs), |
8690 | boolean_true_node), |
8691 | tlabel, flabel); |
8692 | gimple_seq_add_stmt (pre_p, cond); |
8693 | gimple_seq_add_stmt (pre_p, gimple_build_label (label: tlabel)); |
8694 | gimple_seq_add_seq (pre_p, gimple_omp_body (gs: single_stmt)); |
8695 | gimple_seq_add_stmt (pre_p, gimple_build_label (label: flabel)); |
8696 | } |
8697 | |
8698 | |
8699 | /* A subroutine of lower_omp_single. Expand the simple form of |
8700 | a GIMPLE_OMP_SINGLE, with a copyprivate clause: |
8701 | |
8702 | #pragma omp single copyprivate (a, b, c) |
8703 | |
8704 | Create a new structure to hold copies of 'a', 'b' and 'c' and emit: |
8705 | |
8706 | { |
8707 | if ((copyout_p = GOMP_single_copy_start ()) == NULL) |
8708 | { |
8709 | BODY; |
8710 | copyout.a = a; |
8711 | copyout.b = b; |
8712 | copyout.c = c; |
8713 | GOMP_single_copy_end (©out); |
8714 | } |
8715 | else |
8716 | { |
8717 | a = copyout_p->a; |
8718 | b = copyout_p->b; |
8719 | c = copyout_p->c; |
8720 | } |
8721 | GOMP_barrier (); |
8722 | } |
8723 | |
8724 | FIXME. It may be better to delay expanding the logic of this until |
8725 | pass_expand_omp. The expanded logic may make the job more difficult |
8726 | to a synchronization analysis pass. */ |
8727 | |
8728 | static void |
8729 | lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p, |
8730 | omp_context *ctx) |
8731 | { |
8732 | tree ptr_type, t, l0, l1, l2, bfn_decl; |
8733 | gimple_seq copyin_seq; |
8734 | location_t loc = gimple_location (g: single_stmt); |
8735 | |
8736 | ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o" ); |
8737 | |
8738 | ptr_type = build_pointer_type (ctx->record_type); |
8739 | ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i" ); |
8740 | |
8741 | l0 = create_artificial_label (loc); |
8742 | l1 = create_artificial_label (loc); |
8743 | l2 = create_artificial_label (loc); |
8744 | |
8745 | bfn_decl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_SINGLE_COPY_START); |
8746 | t = build_call_expr_loc (loc, bfn_decl, 0); |
8747 | t = fold_convert_loc (loc, ptr_type, t); |
8748 | gimplify_assign (ctx->receiver_decl, t, pre_p); |
8749 | |
8750 | t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl, |
8751 | build_int_cst (ptr_type, 0)); |
8752 | t = build3 (COND_EXPR, void_type_node, t, |
8753 | build_and_jump (&l0), build_and_jump (&l1)); |
8754 | gimplify_and_add (t, pre_p); |
8755 | |
8756 | gimple_seq_add_stmt (pre_p, gimple_build_label (label: l0)); |
8757 | |
8758 | gimple_seq_add_seq (pre_p, gimple_omp_body (gs: single_stmt)); |
8759 | |
8760 | copyin_seq = NULL; |
8761 | lower_copyprivate_clauses (clauses: gimple_omp_single_clauses (gs: single_stmt), slist: pre_p, |
8762 | rlist: ©in_seq, ctx); |
8763 | |
8764 | t = build_fold_addr_expr_loc (loc, ctx->sender_decl); |
8765 | bfn_decl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_SINGLE_COPY_END); |
8766 | t = build_call_expr_loc (loc, bfn_decl, 1, t); |
8767 | gimplify_and_add (t, pre_p); |
8768 | |
8769 | t = build_and_jump (&l2); |
8770 | gimplify_and_add (t, pre_p); |
8771 | |
8772 | gimple_seq_add_stmt (pre_p, gimple_build_label (label: l1)); |
8773 | |
8774 | gimple_seq_add_seq (pre_p, copyin_seq); |
8775 | |
8776 | gimple_seq_add_stmt (pre_p, gimple_build_label (label: l2)); |
8777 | } |
8778 | |
8779 | |
8780 | /* Expand code for an OpenMP single directive. */ |
8781 | |
8782 | static void |
8783 | lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
8784 | { |
8785 | tree block; |
8786 | gomp_single *single_stmt = as_a <gomp_single *> (p: gsi_stmt (i: *gsi_p)); |
8787 | gbind *bind; |
8788 | gimple_seq bind_body, bind_body_tail = NULL, dlist; |
8789 | |
8790 | push_gimplify_context (); |
8791 | |
8792 | block = make_node (BLOCK); |
8793 | bind = gimple_build_bind (NULL, NULL, block); |
8794 | gsi_replace (gsi_p, bind, true); |
8795 | bind_body = NULL; |
8796 | dlist = NULL; |
8797 | lower_rec_input_clauses (clauses: gimple_omp_single_clauses (gs: single_stmt), |
8798 | ilist: &bind_body, dlist: &dlist, ctx, NULL); |
8799 | lower_omp (gimple_omp_body_ptr (gs: single_stmt), ctx); |
8800 | |
8801 | gimple_seq_add_stmt (&bind_body, single_stmt); |
8802 | |
8803 | if (ctx->record_type) |
8804 | lower_omp_single_copy (single_stmt, pre_p: &bind_body, ctx); |
8805 | else |
8806 | lower_omp_single_simple (single_stmt, pre_p: &bind_body); |
8807 | |
8808 | gimple_omp_set_body (gs: single_stmt, NULL); |
8809 | |
8810 | gimple_seq_add_seq (&bind_body, dlist); |
8811 | |
8812 | bind_body = maybe_catch_exception (body: bind_body); |
8813 | |
8814 | bool nowait = omp_find_clause (clauses: gimple_omp_single_clauses (gs: single_stmt), |
8815 | kind: OMP_CLAUSE_NOWAIT) != NULL_TREE; |
8816 | gimple *g = gimple_build_omp_return (nowait); |
8817 | gimple_seq_add_stmt (&bind_body_tail, g); |
8818 | maybe_add_implicit_barrier_cancel (ctx, omp_return: g, body: &bind_body_tail); |
8819 | if (ctx->record_type) |
8820 | { |
8821 | gimple_stmt_iterator gsi = gsi_start (seq&: bind_body_tail); |
8822 | tree clobber = build_clobber (ctx->record_type); |
8823 | gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl, |
8824 | clobber), GSI_SAME_STMT); |
8825 | } |
8826 | gimple_seq_add_seq (&bind_body, bind_body_tail); |
8827 | gimple_bind_set_body (bind_stmt: bind, seq: bind_body); |
8828 | |
8829 | pop_gimplify_context (bind); |
8830 | |
8831 | gimple_bind_append_vars (bind_stmt: bind, vars: ctx->block_vars); |
8832 | BLOCK_VARS (block) = ctx->block_vars; |
8833 | if (BLOCK_VARS (block)) |
8834 | TREE_USED (block) = 1; |
8835 | } |
8836 | |
8837 | |
8838 | /* Lower code for an OMP scope directive. */ |
8839 | |
8840 | static void |
8841 | lower_omp_scope (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
8842 | { |
8843 | tree block; |
8844 | gimple *scope_stmt = gsi_stmt (i: *gsi_p); |
8845 | gbind *bind; |
8846 | gimple_seq bind_body, bind_body_tail = NULL, dlist; |
8847 | gimple_seq tred_dlist = NULL; |
8848 | |
8849 | push_gimplify_context (); |
8850 | |
8851 | block = make_node (BLOCK); |
8852 | bind = gimple_build_bind (NULL, NULL, block); |
8853 | gsi_replace (gsi_p, bind, true); |
8854 | bind_body = NULL; |
8855 | dlist = NULL; |
8856 | |
8857 | tree rclauses |
8858 | = omp_task_reductions_find_first (clauses: gimple_omp_scope_clauses (gs: scope_stmt), |
8859 | code: OMP_SCOPE, ccode: OMP_CLAUSE_REDUCTION); |
8860 | if (rclauses) |
8861 | { |
8862 | tree type = build_pointer_type (pointer_sized_int_node); |
8863 | tree temp = create_tmp_var (type); |
8864 | tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_); |
8865 | OMP_CLAUSE_DECL (c) = temp; |
8866 | OMP_CLAUSE_CHAIN (c) = gimple_omp_scope_clauses (gs: scope_stmt); |
8867 | gimple_omp_scope_set_clauses (gs: scope_stmt, clauses: c); |
8868 | lower_omp_task_reductions (ctx, OMP_SCOPE, |
8869 | gimple_omp_scope_clauses (gs: scope_stmt), |
8870 | &bind_body, &tred_dlist); |
8871 | rclauses = c; |
8872 | tree fndecl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_SCOPE_START); |
8873 | gimple *stmt = gimple_build_call (fndecl, 1, temp); |
8874 | gimple_seq_add_stmt (&bind_body, stmt); |
8875 | } |
8876 | |
8877 | lower_rec_input_clauses (clauses: gimple_omp_scope_clauses (gs: scope_stmt), |
8878 | ilist: &bind_body, dlist: &dlist, ctx, NULL); |
8879 | lower_omp (gimple_omp_body_ptr (gs: scope_stmt), ctx); |
8880 | |
8881 | gimple_seq_add_stmt (&bind_body, scope_stmt); |
8882 | |
8883 | gimple_seq_add_seq (&bind_body, gimple_omp_body (gs: scope_stmt)); |
8884 | |
8885 | gimple_omp_set_body (gs: scope_stmt, NULL); |
8886 | |
8887 | gimple_seq clist = NULL; |
8888 | lower_reduction_clauses (clauses: gimple_omp_scope_clauses (gs: scope_stmt), |
8889 | stmt_seqp: &bind_body, clist: &clist, ctx); |
8890 | if (clist) |
8891 | { |
8892 | tree fndecl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_ATOMIC_START); |
8893 | gcall *g = gimple_build_call (fndecl, 0); |
8894 | gimple_seq_add_stmt (&bind_body, g); |
8895 | gimple_seq_add_seq (&bind_body, clist); |
8896 | fndecl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_ATOMIC_END); |
8897 | g = gimple_build_call (fndecl, 0); |
8898 | gimple_seq_add_stmt (&bind_body, g); |
8899 | } |
8900 | |
8901 | gimple_seq_add_seq (&bind_body, dlist); |
8902 | |
8903 | bind_body = maybe_catch_exception (body: bind_body); |
8904 | |
8905 | bool nowait = omp_find_clause (clauses: gimple_omp_scope_clauses (gs: scope_stmt), |
8906 | kind: OMP_CLAUSE_NOWAIT) != NULL_TREE; |
8907 | gimple *g = gimple_build_omp_return (nowait); |
8908 | gimple_seq_add_stmt (&bind_body_tail, g); |
8909 | gimple_seq_add_seq (&bind_body_tail, tred_dlist); |
8910 | maybe_add_implicit_barrier_cancel (ctx, omp_return: g, body: &bind_body_tail); |
8911 | if (ctx->record_type) |
8912 | { |
8913 | gimple_stmt_iterator gsi = gsi_start (seq&: bind_body_tail); |
8914 | tree clobber = build_clobber (ctx->record_type); |
8915 | gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl, |
8916 | clobber), GSI_SAME_STMT); |
8917 | } |
8918 | gimple_seq_add_seq (&bind_body, bind_body_tail); |
8919 | |
8920 | gimple_bind_set_body (bind_stmt: bind, seq: bind_body); |
8921 | |
8922 | pop_gimplify_context (bind); |
8923 | |
8924 | gimple_bind_append_vars (bind_stmt: bind, vars: ctx->block_vars); |
8925 | BLOCK_VARS (block) = ctx->block_vars; |
8926 | if (BLOCK_VARS (block)) |
8927 | TREE_USED (block) = 1; |
8928 | } |
8929 | /* Expand code for an OpenMP master or masked directive. */ |
8930 | |
8931 | static void |
8932 | lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
8933 | { |
8934 | tree block, lab = NULL, x, bfn_decl; |
8935 | gimple *stmt = gsi_stmt (i: *gsi_p); |
8936 | gbind *bind; |
8937 | location_t loc = gimple_location (g: stmt); |
8938 | gimple_seq tseq; |
8939 | tree filter = integer_zero_node; |
8940 | |
8941 | push_gimplify_context (); |
8942 | |
8943 | if (gimple_code (g: stmt) == GIMPLE_OMP_MASKED) |
8944 | { |
8945 | filter = omp_find_clause (clauses: gimple_omp_masked_clauses (gs: stmt), |
8946 | kind: OMP_CLAUSE_FILTER); |
8947 | if (filter) |
8948 | filter = fold_convert (integer_type_node, |
8949 | OMP_CLAUSE_FILTER_EXPR (filter)); |
8950 | else |
8951 | filter = integer_zero_node; |
8952 | } |
8953 | block = make_node (BLOCK); |
8954 | bind = gimple_build_bind (NULL, NULL, block); |
8955 | gsi_replace (gsi_p, bind, true); |
8956 | gimple_bind_add_stmt (bind_stmt: bind, stmt); |
8957 | |
8958 | bfn_decl = builtin_decl_explicit (fncode: BUILT_IN_OMP_GET_THREAD_NUM); |
8959 | x = build_call_expr_loc (loc, bfn_decl, 0); |
8960 | x = build2 (EQ_EXPR, boolean_type_node, x, filter); |
8961 | x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab)); |
8962 | tseq = NULL; |
8963 | gimplify_and_add (x, &tseq); |
8964 | gimple_bind_add_seq (bind_stmt: bind, seq: tseq); |
8965 | |
8966 | lower_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
8967 | gimple_omp_set_body (gs: stmt, body: maybe_catch_exception (body: gimple_omp_body (gs: stmt))); |
8968 | gimple_bind_add_seq (bind_stmt: bind, seq: gimple_omp_body (gs: stmt)); |
8969 | gimple_omp_set_body (gs: stmt, NULL); |
8970 | |
8971 | gimple_bind_add_stmt (bind_stmt: bind, stmt: gimple_build_label (label: lab)); |
8972 | |
8973 | gimple_bind_add_stmt (bind_stmt: bind, stmt: gimple_build_omp_return (true)); |
8974 | |
8975 | pop_gimplify_context (bind); |
8976 | |
8977 | gimple_bind_append_vars (bind_stmt: bind, vars: ctx->block_vars); |
8978 | BLOCK_VARS (block) = ctx->block_vars; |
8979 | } |
8980 | |
8981 | /* Helper function for lower_omp_task_reductions. For a specific PASS |
8982 | find out the current clause it should be processed, or return false |
8983 | if all have been processed already. */ |
8984 | |
8985 | static inline bool |
8986 | omp_task_reduction_iterate (int pass, enum tree_code code, |
8987 | enum omp_clause_code ccode, tree *c, tree *decl, |
8988 | tree *type, tree *next) |
8989 | { |
8990 | for (; *c; *c = omp_find_clause (OMP_CLAUSE_CHAIN (*c), kind: ccode)) |
8991 | { |
8992 | if (ccode == OMP_CLAUSE_REDUCTION |
8993 | && code != OMP_TASKLOOP |
8994 | && !OMP_CLAUSE_REDUCTION_TASK (*c)) |
8995 | continue; |
8996 | *decl = OMP_CLAUSE_DECL (*c); |
8997 | *type = TREE_TYPE (*decl); |
8998 | if (TREE_CODE (*decl) == MEM_REF) |
8999 | { |
9000 | if (pass != 1) |
9001 | continue; |
9002 | } |
9003 | else |
9004 | { |
9005 | if (omp_privatize_by_reference (decl: *decl)) |
9006 | *type = TREE_TYPE (*type); |
9007 | if (pass != (!TREE_CONSTANT (TYPE_SIZE_UNIT (*type)))) |
9008 | continue; |
9009 | } |
9010 | *next = omp_find_clause (OMP_CLAUSE_CHAIN (*c), kind: ccode); |
9011 | return true; |
9012 | } |
9013 | *decl = NULL_TREE; |
9014 | *type = NULL_TREE; |
9015 | *next = NULL_TREE; |
9016 | return false; |
9017 | } |
9018 | |
9019 | /* Lower task_reduction and reduction clauses (the latter unless CODE is |
9020 | OMP_TASKGROUP only with task modifier). Register mapping of those in |
9021 | START sequence and reducing them and unregister them in the END sequence. */ |
9022 | |
9023 | static void |
9024 | lower_omp_task_reductions (omp_context *ctx, enum tree_code code, tree clauses, |
9025 | gimple_seq *start, gimple_seq *end) |
9026 | { |
9027 | enum omp_clause_code ccode |
9028 | = (code == OMP_TASKGROUP |
9029 | ? OMP_CLAUSE_TASK_REDUCTION : OMP_CLAUSE_REDUCTION); |
9030 | tree cancellable = NULL_TREE; |
9031 | clauses = omp_task_reductions_find_first (clauses, code, ccode); |
9032 | if (clauses == NULL_TREE) |
9033 | return; |
9034 | if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE) |
9035 | { |
9036 | for (omp_context *outer = ctx->outer; outer; outer = outer->outer) |
9037 | if (gimple_code (g: outer->stmt) == GIMPLE_OMP_PARALLEL |
9038 | && outer->cancellable) |
9039 | { |
9040 | cancellable = error_mark_node; |
9041 | break; |
9042 | } |
9043 | else if (gimple_code (g: outer->stmt) != GIMPLE_OMP_TASKGROUP |
9044 | && gimple_code (g: outer->stmt) != GIMPLE_OMP_SCOPE) |
9045 | break; |
9046 | } |
9047 | tree record_type = lang_hooks.types.make_type (RECORD_TYPE); |
9048 | tree *last = &TYPE_FIELDS (record_type); |
9049 | unsigned cnt = 0; |
9050 | if (cancellable) |
9051 | { |
9052 | tree field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE, |
9053 | ptr_type_node); |
9054 | tree ifield = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE, |
9055 | integer_type_node); |
9056 | *last = field; |
9057 | DECL_CHAIN (field) = ifield; |
9058 | last = &DECL_CHAIN (ifield); |
9059 | DECL_CONTEXT (field) = record_type; |
9060 | if (TYPE_ALIGN (record_type) < DECL_ALIGN (field)) |
9061 | SET_TYPE_ALIGN (record_type, DECL_ALIGN (field)); |
9062 | DECL_CONTEXT (ifield) = record_type; |
9063 | if (TYPE_ALIGN (record_type) < DECL_ALIGN (ifield)) |
9064 | SET_TYPE_ALIGN (record_type, DECL_ALIGN (ifield)); |
9065 | } |
9066 | for (int pass = 0; pass < 2; pass++) |
9067 | { |
9068 | tree decl, type, next; |
9069 | for (tree c = clauses; |
9070 | omp_task_reduction_iterate (pass, code, ccode, |
9071 | c: &c, decl: &decl, type: &type, next: &next); c = next) |
9072 | { |
9073 | ++cnt; |
9074 | tree new_type = type; |
9075 | if (ctx->outer) |
9076 | new_type = remap_type (type, id: &ctx->outer->cb); |
9077 | tree field |
9078 | = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL, |
9079 | DECL_P (decl) ? DECL_NAME (decl) : NULL_TREE, |
9080 | new_type); |
9081 | if (DECL_P (decl) && type == TREE_TYPE (decl)) |
9082 | { |
9083 | SET_DECL_ALIGN (field, DECL_ALIGN (decl)); |
9084 | DECL_USER_ALIGN (field) = DECL_USER_ALIGN (decl); |
9085 | TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (decl); |
9086 | } |
9087 | else |
9088 | SET_DECL_ALIGN (field, TYPE_ALIGN (type)); |
9089 | DECL_CONTEXT (field) = record_type; |
9090 | if (TYPE_ALIGN (record_type) < DECL_ALIGN (field)) |
9091 | SET_TYPE_ALIGN (record_type, DECL_ALIGN (field)); |
9092 | *last = field; |
9093 | last = &DECL_CHAIN (field); |
9094 | tree bfield |
9095 | = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL, NULL_TREE, |
9096 | boolean_type_node); |
9097 | DECL_CONTEXT (bfield) = record_type; |
9098 | if (TYPE_ALIGN (record_type) < DECL_ALIGN (bfield)) |
9099 | SET_TYPE_ALIGN (record_type, DECL_ALIGN (bfield)); |
9100 | *last = bfield; |
9101 | last = &DECL_CHAIN (bfield); |
9102 | } |
9103 | } |
9104 | *last = NULL_TREE; |
9105 | layout_type (record_type); |
9106 | |
9107 | /* Build up an array which registers with the runtime all the reductions |
9108 | and deregisters them at the end. Format documented in libgomp/task.c. */ |
9109 | tree atype = build_array_type_nelts (pointer_sized_int_node, 7 + cnt * 3); |
9110 | tree avar = create_tmp_var_raw (atype); |
9111 | gimple_add_tmp_var (avar); |
9112 | TREE_ADDRESSABLE (avar) = 1; |
9113 | tree r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_zero_node, |
9114 | NULL_TREE, NULL_TREE); |
9115 | tree t = build_int_cst (pointer_sized_int_node, cnt); |
9116 | gimple_seq_add_stmt (start, gimple_build_assign (r, t)); |
9117 | gimple_seq seq = NULL; |
9118 | tree sz = fold_convert (pointer_sized_int_node, |
9119 | TYPE_SIZE_UNIT (record_type)); |
9120 | int cachesz = 64; |
9121 | sz = fold_build2 (PLUS_EXPR, pointer_sized_int_node, sz, |
9122 | build_int_cst (pointer_sized_int_node, cachesz - 1)); |
9123 | sz = fold_build2 (BIT_AND_EXPR, pointer_sized_int_node, sz, |
9124 | build_int_cst (pointer_sized_int_node, ~(cachesz - 1))); |
9125 | ctx->task_reductions.create (nelems: 1 + cnt); |
9126 | ctx->task_reduction_map = new hash_map<tree, unsigned>; |
9127 | ctx->task_reductions.quick_push (TREE_CODE (sz) == INTEGER_CST |
9128 | ? sz : NULL_TREE); |
9129 | sz = force_gimple_operand (sz, &seq, true, NULL_TREE); |
9130 | gimple_seq_add_seq (start, seq); |
9131 | r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_one_node, |
9132 | NULL_TREE, NULL_TREE); |
9133 | gimple_seq_add_stmt (start, gimple_build_assign (r, sz)); |
9134 | r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2), |
9135 | NULL_TREE, NULL_TREE); |
9136 | t = build_int_cst (pointer_sized_int_node, |
9137 | MAX (TYPE_ALIGN_UNIT (record_type), (unsigned) cachesz)); |
9138 | gimple_seq_add_stmt (start, gimple_build_assign (r, t)); |
9139 | r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (3), |
9140 | NULL_TREE, NULL_TREE); |
9141 | t = build_int_cst (pointer_sized_int_node, -1); |
9142 | gimple_seq_add_stmt (start, gimple_build_assign (r, t)); |
9143 | r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (4), |
9144 | NULL_TREE, NULL_TREE); |
9145 | t = build_int_cst (pointer_sized_int_node, 0); |
9146 | gimple_seq_add_stmt (start, gimple_build_assign (r, t)); |
9147 | |
9148 | /* In end, build a loop that iterates from 0 to < omp_get_num_threads () |
9149 | and for each task reduction checks a bool right after the private variable |
9150 | within that thread's chunk; if the bool is clear, it hasn't been |
9151 | initialized and thus isn't going to be reduced nor destructed, otherwise |
9152 | reduce and destruct it. */ |
9153 | tree idx = create_tmp_var (size_type_node); |
9154 | gimple_seq_add_stmt (end, gimple_build_assign (idx, size_zero_node)); |
9155 | tree num_thr_sz = create_tmp_var (size_type_node); |
9156 | tree lab1 = create_artificial_label (UNKNOWN_LOCATION); |
9157 | tree lab2 = create_artificial_label (UNKNOWN_LOCATION); |
9158 | tree lab3 = NULL_TREE, lab7 = NULL_TREE; |
9159 | gimple *g; |
9160 | if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE) |
9161 | { |
9162 | /* For worksharing constructs or scope, only perform it in the master |
9163 | thread, with the exception of cancelled implicit barriers - then only |
9164 | handle the current thread. */ |
9165 | tree lab4 = create_artificial_label (UNKNOWN_LOCATION); |
9166 | t = builtin_decl_explicit (fncode: BUILT_IN_OMP_GET_THREAD_NUM); |
9167 | tree thr_num = create_tmp_var (integer_type_node); |
9168 | g = gimple_build_call (t, 0); |
9169 | gimple_call_set_lhs (gs: g, lhs: thr_num); |
9170 | gimple_seq_add_stmt (end, g); |
9171 | if (cancellable) |
9172 | { |
9173 | tree c; |
9174 | tree lab5 = create_artificial_label (UNKNOWN_LOCATION); |
9175 | tree lab6 = create_artificial_label (UNKNOWN_LOCATION); |
9176 | lab3 = create_artificial_label (UNKNOWN_LOCATION); |
9177 | if (code == OMP_FOR) |
9178 | c = gimple_omp_for_clauses (gs: ctx->stmt); |
9179 | else if (code == OMP_SECTIONS) |
9180 | c = gimple_omp_sections_clauses (gs: ctx->stmt); |
9181 | else /* if (code == OMP_SCOPE) */ |
9182 | c = gimple_omp_scope_clauses (gs: ctx->stmt); |
9183 | c = OMP_CLAUSE_DECL (omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_)); |
9184 | cancellable = c; |
9185 | g = gimple_build_cond (NE_EXPR, c, build_zero_cst (TREE_TYPE (c)), |
9186 | lab5, lab6); |
9187 | gimple_seq_add_stmt (end, g); |
9188 | gimple_seq_add_stmt (end, gimple_build_label (label: lab5)); |
9189 | g = gimple_build_assign (idx, NOP_EXPR, thr_num); |
9190 | gimple_seq_add_stmt (end, g); |
9191 | g = gimple_build_assign (num_thr_sz, PLUS_EXPR, idx, |
9192 | build_one_cst (TREE_TYPE (idx))); |
9193 | gimple_seq_add_stmt (end, g); |
9194 | gimple_seq_add_stmt (end, gimple_build_goto (dest: lab3)); |
9195 | gimple_seq_add_stmt (end, gimple_build_label (label: lab6)); |
9196 | } |
9197 | g = gimple_build_cond (NE_EXPR, thr_num, integer_zero_node, lab2, lab4); |
9198 | gimple_seq_add_stmt (end, g); |
9199 | gimple_seq_add_stmt (end, gimple_build_label (label: lab4)); |
9200 | } |
9201 | if (code != OMP_PARALLEL) |
9202 | { |
9203 | t = builtin_decl_explicit (fncode: BUILT_IN_OMP_GET_NUM_THREADS); |
9204 | tree num_thr = create_tmp_var (integer_type_node); |
9205 | g = gimple_build_call (t, 0); |
9206 | gimple_call_set_lhs (gs: g, lhs: num_thr); |
9207 | gimple_seq_add_stmt (end, g); |
9208 | g = gimple_build_assign (num_thr_sz, NOP_EXPR, num_thr); |
9209 | gimple_seq_add_stmt (end, g); |
9210 | if (cancellable) |
9211 | gimple_seq_add_stmt (end, gimple_build_label (label: lab3)); |
9212 | } |
9213 | else |
9214 | { |
9215 | tree c = omp_find_clause (clauses: gimple_omp_parallel_clauses (gs: ctx->stmt), |
9216 | kind: OMP_CLAUSE__REDUCTEMP_); |
9217 | t = fold_convert (pointer_sized_int_node, OMP_CLAUSE_DECL (c)); |
9218 | t = fold_convert (size_type_node, t); |
9219 | gimplify_assign (num_thr_sz, t, end); |
9220 | } |
9221 | t = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2), |
9222 | NULL_TREE, NULL_TREE); |
9223 | tree data = create_tmp_var (pointer_sized_int_node); |
9224 | gimple_seq_add_stmt (end, gimple_build_assign (data, t)); |
9225 | if (code == OMP_TASKLOOP) |
9226 | { |
9227 | lab7 = create_artificial_label (UNKNOWN_LOCATION); |
9228 | g = gimple_build_cond (NE_EXPR, data, |
9229 | build_zero_cst (pointer_sized_int_node), |
9230 | lab1, lab7); |
9231 | gimple_seq_add_stmt (end, g); |
9232 | } |
9233 | gimple_seq_add_stmt (end, gimple_build_label (label: lab1)); |
9234 | tree ptr; |
9235 | if (TREE_CODE (TYPE_SIZE_UNIT (record_type)) == INTEGER_CST) |
9236 | ptr = create_tmp_var (build_pointer_type (record_type)); |
9237 | else |
9238 | ptr = create_tmp_var (ptr_type_node); |
9239 | gimple_seq_add_stmt (end, gimple_build_assign (ptr, NOP_EXPR, data)); |
9240 | |
9241 | tree field = TYPE_FIELDS (record_type); |
9242 | cnt = 0; |
9243 | if (cancellable) |
9244 | field = DECL_CHAIN (DECL_CHAIN (field)); |
9245 | for (int pass = 0; pass < 2; pass++) |
9246 | { |
9247 | tree decl, type, next; |
9248 | for (tree c = clauses; |
9249 | omp_task_reduction_iterate (pass, code, ccode, |
9250 | c: &c, decl: &decl, type: &type, next: &next); c = next) |
9251 | { |
9252 | tree var = decl, ref; |
9253 | if (TREE_CODE (decl) == MEM_REF) |
9254 | { |
9255 | var = TREE_OPERAND (var, 0); |
9256 | if (TREE_CODE (var) == POINTER_PLUS_EXPR) |
9257 | var = TREE_OPERAND (var, 0); |
9258 | tree v = var; |
9259 | if (TREE_CODE (var) == ADDR_EXPR) |
9260 | var = TREE_OPERAND (var, 0); |
9261 | else if (INDIRECT_REF_P (var)) |
9262 | var = TREE_OPERAND (var, 0); |
9263 | tree orig_var = var; |
9264 | if (is_variable_sized (expr: var)) |
9265 | { |
9266 | gcc_assert (DECL_HAS_VALUE_EXPR_P (var)); |
9267 | var = DECL_VALUE_EXPR (var); |
9268 | gcc_assert (INDIRECT_REF_P (var)); |
9269 | var = TREE_OPERAND (var, 0); |
9270 | gcc_assert (DECL_P (var)); |
9271 | } |
9272 | t = ref = maybe_lookup_decl_in_outer_ctx (decl: var, ctx); |
9273 | if (orig_var != var) |
9274 | gcc_assert (TREE_CODE (v) == ADDR_EXPR); |
9275 | else if (TREE_CODE (v) == ADDR_EXPR) |
9276 | t = build_fold_addr_expr (t); |
9277 | else if (INDIRECT_REF_P (v)) |
9278 | t = build_fold_indirect_ref (t); |
9279 | if (TREE_CODE (TREE_OPERAND (decl, 0)) == POINTER_PLUS_EXPR) |
9280 | { |
9281 | tree b = TREE_OPERAND (TREE_OPERAND (decl, 0), 1); |
9282 | b = maybe_lookup_decl_in_outer_ctx (decl: b, ctx); |
9283 | t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t, b); |
9284 | } |
9285 | if (!integer_zerop (TREE_OPERAND (decl, 1))) |
9286 | t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t, |
9287 | fold_convert (size_type_node, |
9288 | TREE_OPERAND (decl, 1))); |
9289 | } |
9290 | else |
9291 | { |
9292 | t = ref = maybe_lookup_decl_in_outer_ctx (decl: var, ctx); |
9293 | if (!omp_privatize_by_reference (decl)) |
9294 | t = build_fold_addr_expr (t); |
9295 | } |
9296 | t = fold_convert (pointer_sized_int_node, t); |
9297 | seq = NULL; |
9298 | t = force_gimple_operand (t, &seq, true, NULL_TREE); |
9299 | gimple_seq_add_seq (start, seq); |
9300 | r = build4 (ARRAY_REF, pointer_sized_int_node, avar, |
9301 | size_int (7 + cnt * 3), NULL_TREE, NULL_TREE); |
9302 | gimple_seq_add_stmt (start, gimple_build_assign (r, t)); |
9303 | t = unshare_expr (byte_position (field)); |
9304 | t = fold_convert (pointer_sized_int_node, t); |
9305 | ctx->task_reduction_map->put (k: c, v: cnt); |
9306 | ctx->task_reductions.quick_push (TREE_CODE (t) == INTEGER_CST |
9307 | ? t : NULL_TREE); |
9308 | seq = NULL; |
9309 | t = force_gimple_operand (t, &seq, true, NULL_TREE); |
9310 | gimple_seq_add_seq (start, seq); |
9311 | r = build4 (ARRAY_REF, pointer_sized_int_node, avar, |
9312 | size_int (7 + cnt * 3 + 1), NULL_TREE, NULL_TREE); |
9313 | gimple_seq_add_stmt (start, gimple_build_assign (r, t)); |
9314 | |
9315 | tree bfield = DECL_CHAIN (field); |
9316 | tree cond; |
9317 | if (code == OMP_PARALLEL |
9318 | || code == OMP_FOR |
9319 | || code == OMP_SECTIONS |
9320 | || code == OMP_SCOPE) |
9321 | /* In parallel, worksharing or scope all threads unconditionally |
9322 | initialize all their task reduction private variables. */ |
9323 | cond = boolean_true_node; |
9324 | else if (TREE_TYPE (ptr) == ptr_type_node) |
9325 | { |
9326 | cond = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr, |
9327 | unshare_expr (byte_position (bfield))); |
9328 | seq = NULL; |
9329 | cond = force_gimple_operand (cond, &seq, true, NULL_TREE); |
9330 | gimple_seq_add_seq (end, seq); |
9331 | tree pbool = build_pointer_type (TREE_TYPE (bfield)); |
9332 | cond = build2 (MEM_REF, TREE_TYPE (bfield), cond, |
9333 | build_int_cst (pbool, 0)); |
9334 | } |
9335 | else |
9336 | cond = build3 (COMPONENT_REF, TREE_TYPE (bfield), |
9337 | build_simple_mem_ref (ptr), bfield, NULL_TREE); |
9338 | tree lab3 = create_artificial_label (UNKNOWN_LOCATION); |
9339 | tree lab4 = create_artificial_label (UNKNOWN_LOCATION); |
9340 | tree condv = create_tmp_var (boolean_type_node); |
9341 | gimple_seq_add_stmt (end, gimple_build_assign (condv, cond)); |
9342 | g = gimple_build_cond (NE_EXPR, condv, boolean_false_node, |
9343 | lab3, lab4); |
9344 | gimple_seq_add_stmt (end, g); |
9345 | gimple_seq_add_stmt (end, gimple_build_label (label: lab3)); |
9346 | if (cancellable && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE) |
9347 | { |
9348 | /* If this reduction doesn't need destruction and parallel |
9349 | has been cancelled, there is nothing to do for this |
9350 | reduction, so jump around the merge operation. */ |
9351 | tree lab5 = create_artificial_label (UNKNOWN_LOCATION); |
9352 | g = gimple_build_cond (NE_EXPR, cancellable, |
9353 | build_zero_cst (TREE_TYPE (cancellable)), |
9354 | lab4, lab5); |
9355 | gimple_seq_add_stmt (end, g); |
9356 | gimple_seq_add_stmt (end, gimple_build_label (label: lab5)); |
9357 | } |
9358 | |
9359 | tree new_var; |
9360 | if (TREE_TYPE (ptr) == ptr_type_node) |
9361 | { |
9362 | new_var = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr, |
9363 | unshare_expr (byte_position (field))); |
9364 | seq = NULL; |
9365 | new_var = force_gimple_operand (new_var, &seq, true, NULL_TREE); |
9366 | gimple_seq_add_seq (end, seq); |
9367 | tree pbool = build_pointer_type (TREE_TYPE (field)); |
9368 | new_var = build2 (MEM_REF, TREE_TYPE (field), new_var, |
9369 | build_int_cst (pbool, 0)); |
9370 | } |
9371 | else |
9372 | new_var = build3 (COMPONENT_REF, TREE_TYPE (field), |
9373 | build_simple_mem_ref (ptr), field, NULL_TREE); |
9374 | |
9375 | enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c); |
9376 | if (TREE_CODE (decl) != MEM_REF |
9377 | && omp_privatize_by_reference (decl)) |
9378 | ref = build_simple_mem_ref (ref); |
9379 | /* reduction(-:var) sums up the partial results, so it acts |
9380 | identically to reduction(+:var). */ |
9381 | if (rcode == MINUS_EXPR) |
9382 | rcode = PLUS_EXPR; |
9383 | if (TREE_CODE (decl) == MEM_REF) |
9384 | { |
9385 | tree type = TREE_TYPE (new_var); |
9386 | tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type)); |
9387 | tree i = create_tmp_var (TREE_TYPE (v)); |
9388 | tree ptype = build_pointer_type (TREE_TYPE (type)); |
9389 | if (DECL_P (v)) |
9390 | { |
9391 | v = maybe_lookup_decl_in_outer_ctx (decl: v, ctx); |
9392 | tree vv = create_tmp_var (TREE_TYPE (v)); |
9393 | gimplify_assign (vv, v, start); |
9394 | v = vv; |
9395 | } |
9396 | ref = build4 (ARRAY_REF, pointer_sized_int_node, avar, |
9397 | size_int (7 + cnt * 3), NULL_TREE, NULL_TREE); |
9398 | new_var = build_fold_addr_expr (new_var); |
9399 | new_var = fold_convert (ptype, new_var); |
9400 | ref = fold_convert (ptype, ref); |
9401 | tree m = create_tmp_var (ptype); |
9402 | gimplify_assign (m, new_var, end); |
9403 | new_var = m; |
9404 | m = create_tmp_var (ptype); |
9405 | gimplify_assign (m, ref, end); |
9406 | ref = m; |
9407 | gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), end); |
9408 | tree body = create_artificial_label (UNKNOWN_LOCATION); |
9409 | tree endl = create_artificial_label (UNKNOWN_LOCATION); |
9410 | gimple_seq_add_stmt (end, gimple_build_label (label: body)); |
9411 | tree priv = build_simple_mem_ref (new_var); |
9412 | tree out = build_simple_mem_ref (ref); |
9413 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
9414 | { |
9415 | tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); |
9416 | tree decl_placeholder |
9417 | = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c); |
9418 | tree lab6 = NULL_TREE; |
9419 | if (cancellable) |
9420 | { |
9421 | /* If this reduction needs destruction and parallel |
9422 | has been cancelled, jump around the merge operation |
9423 | to the destruction. */ |
9424 | tree lab5 = create_artificial_label (UNKNOWN_LOCATION); |
9425 | lab6 = create_artificial_label (UNKNOWN_LOCATION); |
9426 | tree zero = build_zero_cst (TREE_TYPE (cancellable)); |
9427 | g = gimple_build_cond (NE_EXPR, cancellable, zero, |
9428 | lab6, lab5); |
9429 | gimple_seq_add_stmt (end, g); |
9430 | gimple_seq_add_stmt (end, gimple_build_label (label: lab5)); |
9431 | } |
9432 | SET_DECL_VALUE_EXPR (placeholder, out); |
9433 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
9434 | SET_DECL_VALUE_EXPR (decl_placeholder, priv); |
9435 | DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1; |
9436 | lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx); |
9437 | gimple_seq_add_seq (end, |
9438 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c)); |
9439 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; |
9440 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION) |
9441 | { |
9442 | OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL; |
9443 | OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL; |
9444 | } |
9445 | if (cancellable) |
9446 | gimple_seq_add_stmt (end, gimple_build_label (label: lab6)); |
9447 | tree x = lang_hooks.decls.omp_clause_dtor (c, priv); |
9448 | if (x) |
9449 | { |
9450 | gimple_seq tseq = NULL; |
9451 | gimplify_stmt (&x, &tseq); |
9452 | gimple_seq_add_seq (end, tseq); |
9453 | } |
9454 | } |
9455 | else |
9456 | { |
9457 | tree x = build2 (rcode, TREE_TYPE (out), out, priv); |
9458 | out = unshare_expr (out); |
9459 | gimplify_assign (out, x, end); |
9460 | } |
9461 | gimple *g |
9462 | = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var, |
9463 | TYPE_SIZE_UNIT (TREE_TYPE (type))); |
9464 | gimple_seq_add_stmt (end, g); |
9465 | g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref, |
9466 | TYPE_SIZE_UNIT (TREE_TYPE (type))); |
9467 | gimple_seq_add_stmt (end, g); |
9468 | g = gimple_build_assign (i, PLUS_EXPR, i, |
9469 | build_int_cst (TREE_TYPE (i), 1)); |
9470 | gimple_seq_add_stmt (end, g); |
9471 | g = gimple_build_cond (LE_EXPR, i, v, body, endl); |
9472 | gimple_seq_add_stmt (end, g); |
9473 | gimple_seq_add_stmt (end, gimple_build_label (label: endl)); |
9474 | } |
9475 | else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
9476 | { |
9477 | tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); |
9478 | tree oldv = NULL_TREE; |
9479 | tree lab6 = NULL_TREE; |
9480 | if (cancellable) |
9481 | { |
9482 | /* If this reduction needs destruction and parallel |
9483 | has been cancelled, jump around the merge operation |
9484 | to the destruction. */ |
9485 | tree lab5 = create_artificial_label (UNKNOWN_LOCATION); |
9486 | lab6 = create_artificial_label (UNKNOWN_LOCATION); |
9487 | tree zero = build_zero_cst (TREE_TYPE (cancellable)); |
9488 | g = gimple_build_cond (NE_EXPR, cancellable, zero, |
9489 | lab6, lab5); |
9490 | gimple_seq_add_stmt (end, g); |
9491 | gimple_seq_add_stmt (end, gimple_build_label (label: lab5)); |
9492 | } |
9493 | if (omp_privatize_by_reference (decl) |
9494 | && !useless_type_conversion_p (TREE_TYPE (placeholder), |
9495 | TREE_TYPE (ref))) |
9496 | ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref); |
9497 | ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref); |
9498 | tree refv = create_tmp_var (TREE_TYPE (ref)); |
9499 | gimplify_assign (refv, ref, end); |
9500 | ref = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), refv); |
9501 | SET_DECL_VALUE_EXPR (placeholder, ref); |
9502 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
9503 | tree d = maybe_lookup_decl (var: decl, ctx); |
9504 | gcc_assert (d); |
9505 | if (DECL_HAS_VALUE_EXPR_P (d)) |
9506 | oldv = DECL_VALUE_EXPR (d); |
9507 | if (omp_privatize_by_reference (decl: var)) |
9508 | { |
9509 | tree v = fold_convert (TREE_TYPE (d), |
9510 | build_fold_addr_expr (new_var)); |
9511 | SET_DECL_VALUE_EXPR (d, v); |
9512 | } |
9513 | else |
9514 | SET_DECL_VALUE_EXPR (d, new_var); |
9515 | DECL_HAS_VALUE_EXPR_P (d) = 1; |
9516 | lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx); |
9517 | if (oldv) |
9518 | SET_DECL_VALUE_EXPR (d, oldv); |
9519 | else |
9520 | { |
9521 | SET_DECL_VALUE_EXPR (d, NULL_TREE); |
9522 | DECL_HAS_VALUE_EXPR_P (d) = 0; |
9523 | } |
9524 | gimple_seq_add_seq (end, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c)); |
9525 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; |
9526 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION) |
9527 | OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL; |
9528 | if (cancellable) |
9529 | gimple_seq_add_stmt (end, gimple_build_label (label: lab6)); |
9530 | tree x = lang_hooks.decls.omp_clause_dtor (c, new_var); |
9531 | if (x) |
9532 | { |
9533 | gimple_seq tseq = NULL; |
9534 | gimplify_stmt (&x, &tseq); |
9535 | gimple_seq_add_seq (end, tseq); |
9536 | } |
9537 | } |
9538 | else |
9539 | { |
9540 | tree x = build2 (rcode, TREE_TYPE (ref), ref, new_var); |
9541 | ref = unshare_expr (ref); |
9542 | gimplify_assign (ref, x, end); |
9543 | } |
9544 | gimple_seq_add_stmt (end, gimple_build_label (label: lab4)); |
9545 | ++cnt; |
9546 | field = DECL_CHAIN (bfield); |
9547 | } |
9548 | } |
9549 | |
9550 | if (code == OMP_TASKGROUP) |
9551 | { |
9552 | t = builtin_decl_explicit (fncode: BUILT_IN_GOMP_TASKGROUP_REDUCTION_REGISTER); |
9553 | g = gimple_build_call (t, 1, build_fold_addr_expr (avar)); |
9554 | gimple_seq_add_stmt (start, g); |
9555 | } |
9556 | else |
9557 | { |
9558 | tree c; |
9559 | if (code == OMP_FOR) |
9560 | c = gimple_omp_for_clauses (gs: ctx->stmt); |
9561 | else if (code == OMP_SECTIONS) |
9562 | c = gimple_omp_sections_clauses (gs: ctx->stmt); |
9563 | else if (code == OMP_SCOPE) |
9564 | c = gimple_omp_scope_clauses (gs: ctx->stmt); |
9565 | else |
9566 | c = gimple_omp_taskreg_clauses (gs: ctx->stmt); |
9567 | c = omp_find_clause (clauses: c, kind: OMP_CLAUSE__REDUCTEMP_); |
9568 | t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (c)), |
9569 | build_fold_addr_expr (avar)); |
9570 | gimplify_assign (OMP_CLAUSE_DECL (c), t, start); |
9571 | } |
9572 | |
9573 | gimple_seq_add_stmt (end, gimple_build_assign (data, PLUS_EXPR, data, sz)); |
9574 | gimple_seq_add_stmt (end, gimple_build_assign (idx, PLUS_EXPR, idx, |
9575 | size_one_node)); |
9576 | g = gimple_build_cond (NE_EXPR, idx, num_thr_sz, lab1, lab2); |
9577 | gimple_seq_add_stmt (end, g); |
9578 | gimple_seq_add_stmt (end, gimple_build_label (label: lab2)); |
9579 | if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE) |
9580 | { |
9581 | enum built_in_function bfn |
9582 | = BUILT_IN_GOMP_WORKSHARE_TASK_REDUCTION_UNREGISTER; |
9583 | t = builtin_decl_explicit (fncode: bfn); |
9584 | tree c_bool_type = TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (t))); |
9585 | tree arg; |
9586 | if (cancellable) |
9587 | { |
9588 | arg = create_tmp_var (c_bool_type); |
9589 | gimple_seq_add_stmt (end, gimple_build_assign (arg, NOP_EXPR, |
9590 | cancellable)); |
9591 | } |
9592 | else |
9593 | arg = build_int_cst (c_bool_type, 0); |
9594 | g = gimple_build_call (t, 1, arg); |
9595 | } |
9596 | else |
9597 | { |
9598 | t = builtin_decl_explicit (fncode: BUILT_IN_GOMP_TASKGROUP_REDUCTION_UNREGISTER); |
9599 | g = gimple_build_call (t, 1, build_fold_addr_expr (avar)); |
9600 | } |
9601 | gimple_seq_add_stmt (end, g); |
9602 | if (lab7) |
9603 | gimple_seq_add_stmt (end, gimple_build_label (label: lab7)); |
9604 | t = build_constructor (atype, NULL); |
9605 | TREE_THIS_VOLATILE (t) = 1; |
9606 | gimple_seq_add_stmt (end, gimple_build_assign (avar, t)); |
9607 | } |
9608 | |
9609 | /* Expand code for an OpenMP taskgroup directive. */ |
9610 | |
9611 | static void |
9612 | lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
9613 | { |
9614 | gimple *stmt = gsi_stmt (i: *gsi_p); |
9615 | gcall *x; |
9616 | gbind *bind; |
9617 | gimple_seq dseq = NULL; |
9618 | tree block = make_node (BLOCK); |
9619 | |
9620 | bind = gimple_build_bind (NULL, NULL, block); |
9621 | gsi_replace (gsi_p, bind, true); |
9622 | gimple_bind_add_stmt (bind_stmt: bind, stmt); |
9623 | |
9624 | push_gimplify_context (); |
9625 | |
9626 | x = gimple_build_call (builtin_decl_explicit (fncode: BUILT_IN_GOMP_TASKGROUP_START), |
9627 | 0); |
9628 | gimple_bind_add_stmt (bind_stmt: bind, stmt: x); |
9629 | |
9630 | lower_omp_task_reductions (ctx, code: OMP_TASKGROUP, |
9631 | clauses: gimple_omp_taskgroup_clauses (gs: stmt), |
9632 | start: gimple_bind_body_ptr (bind_stmt: bind), end: &dseq); |
9633 | |
9634 | lower_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
9635 | gimple_bind_add_seq (bind_stmt: bind, seq: gimple_omp_body (gs: stmt)); |
9636 | gimple_omp_set_body (gs: stmt, NULL); |
9637 | |
9638 | gimple_bind_add_seq (bind_stmt: bind, seq: dseq); |
9639 | |
9640 | pop_gimplify_context (bind); |
9641 | |
9642 | gimple_bind_append_vars (bind_stmt: bind, vars: ctx->block_vars); |
9643 | BLOCK_VARS (block) = ctx->block_vars; |
9644 | } |
9645 | |
9646 | |
9647 | /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */ |
9648 | |
9649 | static void |
9650 | lower_omp_ordered_clauses (gimple_stmt_iterator *gsi_p, gomp_ordered *ord_stmt, |
9651 | omp_context *ctx) |
9652 | { |
9653 | struct omp_for_data fd; |
9654 | if (!ctx->outer || gimple_code (g: ctx->outer->stmt) != GIMPLE_OMP_FOR) |
9655 | return; |
9656 | |
9657 | unsigned int len = gimple_omp_for_collapse (gs: ctx->outer->stmt); |
9658 | struct omp_for_data_loop *loops = XALLOCAVEC (struct omp_for_data_loop, len); |
9659 | omp_extract_for_data (for_stmt: as_a <gomp_for *> (p: ctx->outer->stmt), fd: &fd, loops); |
9660 | if (!fd.ordered) |
9661 | return; |
9662 | |
9663 | tree *list_p = gimple_omp_ordered_clauses_ptr (ord_stmt); |
9664 | tree c = gimple_omp_ordered_clauses (ord_stmt); |
9665 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS |
9666 | && OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SINK) |
9667 | { |
9668 | /* Merge depend clauses from multiple adjacent |
9669 | #pragma omp ordered depend(sink:...) constructs |
9670 | into one #pragma omp ordered depend(sink:...), so that |
9671 | we can optimize them together. */ |
9672 | gimple_stmt_iterator gsi = *gsi_p; |
9673 | gsi_next (i: &gsi); |
9674 | while (!gsi_end_p (i: gsi)) |
9675 | { |
9676 | gimple *stmt = gsi_stmt (i: gsi); |
9677 | if (is_gimple_debug (gs: stmt) |
9678 | || gimple_code (g: stmt) == GIMPLE_NOP) |
9679 | { |
9680 | gsi_next (i: &gsi); |
9681 | continue; |
9682 | } |
9683 | if (gimple_code (g: stmt) != GIMPLE_OMP_ORDERED) |
9684 | break; |
9685 | gomp_ordered *ord_stmt2 = as_a <gomp_ordered *> (p: stmt); |
9686 | c = gimple_omp_ordered_clauses (ord_stmt: ord_stmt2); |
9687 | if (c == NULL_TREE |
9688 | || OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS |
9689 | || OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK) |
9690 | break; |
9691 | while (*list_p) |
9692 | list_p = &OMP_CLAUSE_CHAIN (*list_p); |
9693 | *list_p = c; |
9694 | gsi_remove (&gsi, true); |
9695 | } |
9696 | } |
9697 | |
9698 | /* Canonicalize sink dependence clauses into one folded clause if |
9699 | possible. |
9700 | |
9701 | The basic algorithm is to create a sink vector whose first |
9702 | element is the GCD of all the first elements, and whose remaining |
9703 | elements are the minimum of the subsequent columns. |
9704 | |
9705 | We ignore dependence vectors whose first element is zero because |
9706 | such dependencies are known to be executed by the same thread. |
9707 | |
9708 | We take into account the direction of the loop, so a minimum |
9709 | becomes a maximum if the loop is iterating forwards. We also |
9710 | ignore sink clauses where the loop direction is unknown, or where |
9711 | the offsets are clearly invalid because they are not a multiple |
9712 | of the loop increment. |
9713 | |
9714 | For example: |
9715 | |
9716 | #pragma omp for ordered(2) |
9717 | for (i=0; i < N; ++i) |
9718 | for (j=0; j < M; ++j) |
9719 | { |
9720 | #pragma omp ordered \ |
9721 | depend(sink:i-8,j-2) \ |
9722 | depend(sink:i,j-1) \ // Completely ignored because i+0. |
9723 | depend(sink:i-4,j-3) \ |
9724 | depend(sink:i-6,j-4) |
9725 | #pragma omp ordered depend(source) |
9726 | } |
9727 | |
9728 | Folded clause is: |
9729 | |
9730 | depend(sink:-gcd(8,4,6),-min(2,3,4)) |
9731 | -or- |
9732 | depend(sink:-2,-2) |
9733 | */ |
9734 | |
9735 | /* FIXME: Computing GCD's where the first element is zero is |
9736 | non-trivial in the presence of collapsed loops. Do this later. */ |
9737 | if (fd.collapse > 1) |
9738 | return; |
9739 | |
9740 | wide_int *folded_deps = XALLOCAVEC (wide_int, 2 * len - 1); |
9741 | |
9742 | /* wide_int is not a POD so it must be default-constructed. */ |
9743 | for (unsigned i = 0; i != 2 * len - 1; ++i) |
9744 | new (static_cast<void*>(folded_deps + i)) wide_int (); |
9745 | |
9746 | tree folded_dep = NULL_TREE; |
9747 | /* TRUE if the first dimension's offset is negative. */ |
9748 | bool neg_offset_p = false; |
9749 | |
9750 | list_p = gimple_omp_ordered_clauses_ptr (ord_stmt); |
9751 | unsigned int i; |
9752 | while ((c = *list_p) != NULL) |
9753 | { |
9754 | bool remove = false; |
9755 | |
9756 | gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS); |
9757 | if (OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK) |
9758 | goto next_ordered_clause; |
9759 | |
9760 | tree vec; |
9761 | for (vec = OMP_CLAUSE_DECL (c), i = 0; |
9762 | vec && TREE_CODE (vec) == TREE_LIST; |
9763 | vec = TREE_CHAIN (vec), ++i) |
9764 | { |
9765 | gcc_assert (i < len); |
9766 | |
9767 | /* omp_extract_for_data has canonicalized the condition. */ |
9768 | gcc_assert (fd.loops[i].cond_code == LT_EXPR |
9769 | || fd.loops[i].cond_code == GT_EXPR); |
9770 | bool forward = fd.loops[i].cond_code == LT_EXPR; |
9771 | bool maybe_lexically_later = true; |
9772 | |
9773 | /* While the committee makes up its mind, bail if we have any |
9774 | non-constant steps. */ |
9775 | if (TREE_CODE (fd.loops[i].step) != INTEGER_CST) |
9776 | goto lower_omp_ordered_ret; |
9777 | |
9778 | tree itype = TREE_TYPE (TREE_VALUE (vec)); |
9779 | if (POINTER_TYPE_P (itype)) |
9780 | itype = sizetype; |
9781 | wide_int offset = wide_int::from (x: wi::to_wide (TREE_PURPOSE (vec)), |
9782 | TYPE_PRECISION (itype), |
9783 | TYPE_SIGN (itype)); |
9784 | |
9785 | /* Ignore invalid offsets that are not multiples of the step. */ |
9786 | if (!wi::multiple_of_p (x: wi::abs (x: offset), |
9787 | y: wi::abs (x: wi::to_wide (t: fd.loops[i].step)), |
9788 | sgn: UNSIGNED)) |
9789 | { |
9790 | warning_at (OMP_CLAUSE_LOCATION (c), OPT_Wopenmp, |
9791 | "ignoring %<sink%> clause with offset that is not " |
9792 | "a multiple of the loop step" ); |
9793 | remove = true; |
9794 | goto next_ordered_clause; |
9795 | } |
9796 | |
9797 | /* Calculate the first dimension. The first dimension of |
9798 | the folded dependency vector is the GCD of the first |
9799 | elements, while ignoring any first elements whose offset |
9800 | is 0. */ |
9801 | if (i == 0) |
9802 | { |
9803 | /* Ignore dependence vectors whose first dimension is 0. */ |
9804 | if (offset == 0) |
9805 | { |
9806 | remove = true; |
9807 | goto next_ordered_clause; |
9808 | } |
9809 | else |
9810 | { |
9811 | if (!TYPE_UNSIGNED (itype) && (forward ^ wi::neg_p (x: offset))) |
9812 | { |
9813 | error_at (OMP_CLAUSE_LOCATION (c), |
9814 | "first offset must be in opposite direction " |
9815 | "of loop iterations" ); |
9816 | goto lower_omp_ordered_ret; |
9817 | } |
9818 | if (forward) |
9819 | offset = -offset; |
9820 | neg_offset_p = forward; |
9821 | /* Initialize the first time around. */ |
9822 | if (folded_dep == NULL_TREE) |
9823 | { |
9824 | folded_dep = c; |
9825 | folded_deps[0] = offset; |
9826 | } |
9827 | else |
9828 | folded_deps[0] = wi::gcd (a: folded_deps[0], |
9829 | b: offset, sgn: UNSIGNED); |
9830 | } |
9831 | } |
9832 | /* Calculate minimum for the remaining dimensions. */ |
9833 | else |
9834 | { |
9835 | folded_deps[len + i - 1] = offset; |
9836 | if (folded_dep == c) |
9837 | folded_deps[i] = offset; |
9838 | else if (maybe_lexically_later |
9839 | && !wi::eq_p (x: folded_deps[i], y: offset)) |
9840 | { |
9841 | if (forward ^ wi::gts_p (x: folded_deps[i], y: offset)) |
9842 | { |
9843 | unsigned int j; |
9844 | folded_dep = c; |
9845 | for (j = 1; j <= i; j++) |
9846 | folded_deps[j] = folded_deps[len + j - 1]; |
9847 | } |
9848 | else |
9849 | maybe_lexically_later = false; |
9850 | } |
9851 | } |
9852 | } |
9853 | gcc_assert (i == len); |
9854 | |
9855 | remove = true; |
9856 | |
9857 | next_ordered_clause: |
9858 | if (remove) |
9859 | *list_p = OMP_CLAUSE_CHAIN (c); |
9860 | else |
9861 | list_p = &OMP_CLAUSE_CHAIN (c); |
9862 | } |
9863 | |
9864 | if (folded_dep) |
9865 | { |
9866 | if (neg_offset_p) |
9867 | folded_deps[0] = -folded_deps[0]; |
9868 | |
9869 | tree itype = TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep))); |
9870 | if (POINTER_TYPE_P (itype)) |
9871 | itype = sizetype; |
9872 | |
9873 | TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep)) |
9874 | = wide_int_to_tree (type: itype, cst: folded_deps[0]); |
9875 | OMP_CLAUSE_CHAIN (folded_dep) = gimple_omp_ordered_clauses (ord_stmt); |
9876 | *gimple_omp_ordered_clauses_ptr (ord_stmt) = folded_dep; |
9877 | } |
9878 | |
9879 | lower_omp_ordered_ret: |
9880 | |
9881 | /* Ordered without clauses is #pragma omp threads, while we want |
9882 | a nop instead if we remove all clauses. */ |
9883 | if (gimple_omp_ordered_clauses (ord_stmt) == NULL_TREE) |
9884 | gsi_replace (gsi_p, gimple_build_nop (), true); |
9885 | } |
9886 | |
9887 | |
9888 | /* Expand code for an OpenMP ordered directive. */ |
9889 | |
9890 | static void |
9891 | lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
9892 | { |
9893 | tree block; |
9894 | gimple *stmt = gsi_stmt (i: *gsi_p), *g; |
9895 | gomp_ordered *ord_stmt = as_a <gomp_ordered *> (p: stmt); |
9896 | gcall *x; |
9897 | gbind *bind; |
9898 | bool simd = omp_find_clause (clauses: gimple_omp_ordered_clauses (ord_stmt), |
9899 | kind: OMP_CLAUSE_SIMD); |
9900 | /* FIXME: this should check presence of OMP_CLAUSE__SIMT_ on the enclosing |
9901 | loop. */ |
9902 | bool maybe_simt |
9903 | = simd && omp_maybe_offloaded_ctx (ctx) && omp_max_simt_vf () > 1; |
9904 | bool threads = omp_find_clause (clauses: gimple_omp_ordered_clauses (ord_stmt), |
9905 | kind: OMP_CLAUSE_THREADS); |
9906 | |
9907 | if (gimple_omp_ordered_standalone_p (g: ord_stmt)) |
9908 | { |
9909 | /* FIXME: This is needs to be moved to the expansion to verify various |
9910 | conditions only testable on cfg with dominators computed, and also |
9911 | all the depend clauses to be merged still might need to be available |
9912 | for the runtime checks. */ |
9913 | if (0) |
9914 | lower_omp_ordered_clauses (gsi_p, ord_stmt, ctx); |
9915 | return; |
9916 | } |
9917 | |
9918 | push_gimplify_context (); |
9919 | |
9920 | block = make_node (BLOCK); |
9921 | bind = gimple_build_bind (NULL, NULL, block); |
9922 | gsi_replace (gsi_p, bind, true); |
9923 | gimple_bind_add_stmt (bind_stmt: bind, stmt); |
9924 | |
9925 | if (simd) |
9926 | { |
9927 | x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START, 1, |
9928 | build_int_cst (NULL_TREE, threads)); |
9929 | cfun->has_simduid_loops = true; |
9930 | } |
9931 | else |
9932 | x = gimple_build_call (builtin_decl_explicit (fncode: BUILT_IN_GOMP_ORDERED_START), |
9933 | 0); |
9934 | gimple_bind_add_stmt (bind_stmt: bind, stmt: x); |
9935 | |
9936 | tree counter = NULL_TREE, test = NULL_TREE, body = NULL_TREE; |
9937 | if (maybe_simt) |
9938 | { |
9939 | counter = create_tmp_var (integer_type_node); |
9940 | g = gimple_build_call_internal (IFN_GOMP_SIMT_LANE, 0); |
9941 | gimple_call_set_lhs (gs: g, lhs: counter); |
9942 | gimple_bind_add_stmt (bind_stmt: bind, stmt: g); |
9943 | |
9944 | body = create_artificial_label (UNKNOWN_LOCATION); |
9945 | test = create_artificial_label (UNKNOWN_LOCATION); |
9946 | gimple_bind_add_stmt (bind_stmt: bind, stmt: gimple_build_label (label: body)); |
9947 | |
9948 | tree simt_pred = create_tmp_var (integer_type_node); |
9949 | g = gimple_build_call_internal (IFN_GOMP_SIMT_ORDERED_PRED, 1, counter); |
9950 | gimple_call_set_lhs (gs: g, lhs: simt_pred); |
9951 | gimple_bind_add_stmt (bind_stmt: bind, stmt: g); |
9952 | |
9953 | tree t = create_artificial_label (UNKNOWN_LOCATION); |
9954 | g = gimple_build_cond (EQ_EXPR, simt_pred, integer_zero_node, t, test); |
9955 | gimple_bind_add_stmt (bind_stmt: bind, stmt: g); |
9956 | |
9957 | gimple_bind_add_stmt (bind_stmt: bind, stmt: gimple_build_label (label: t)); |
9958 | } |
9959 | lower_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
9960 | gimple_omp_set_body (gs: stmt, body: maybe_catch_exception (body: gimple_omp_body (gs: stmt))); |
9961 | gimple_bind_add_seq (bind_stmt: bind, seq: gimple_omp_body (gs: stmt)); |
9962 | gimple_omp_set_body (gs: stmt, NULL); |
9963 | |
9964 | if (maybe_simt) |
9965 | { |
9966 | gimple_bind_add_stmt (bind_stmt: bind, stmt: gimple_build_label (label: test)); |
9967 | g = gimple_build_assign (counter, MINUS_EXPR, counter, integer_one_node); |
9968 | gimple_bind_add_stmt (bind_stmt: bind, stmt: g); |
9969 | |
9970 | tree c = build2 (GE_EXPR, boolean_type_node, counter, integer_zero_node); |
9971 | tree nonneg = create_tmp_var (integer_type_node); |
9972 | gimple_seq tseq = NULL; |
9973 | gimplify_assign (nonneg, fold_convert (integer_type_node, c), &tseq); |
9974 | gimple_bind_add_seq (bind_stmt: bind, seq: tseq); |
9975 | |
9976 | g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY, 1, nonneg); |
9977 | gimple_call_set_lhs (gs: g, lhs: nonneg); |
9978 | gimple_bind_add_stmt (bind_stmt: bind, stmt: g); |
9979 | |
9980 | tree end = create_artificial_label (UNKNOWN_LOCATION); |
9981 | g = gimple_build_cond (NE_EXPR, nonneg, integer_zero_node, body, end); |
9982 | gimple_bind_add_stmt (bind_stmt: bind, stmt: g); |
9983 | |
9984 | gimple_bind_add_stmt (bind_stmt: bind, stmt: gimple_build_label (label: end)); |
9985 | } |
9986 | if (simd) |
9987 | x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END, 1, |
9988 | build_int_cst (NULL_TREE, threads)); |
9989 | else |
9990 | x = gimple_build_call (builtin_decl_explicit (fncode: BUILT_IN_GOMP_ORDERED_END), |
9991 | 0); |
9992 | gimple_bind_add_stmt (bind_stmt: bind, stmt: x); |
9993 | |
9994 | gimple_bind_add_stmt (bind_stmt: bind, stmt: gimple_build_omp_return (true)); |
9995 | |
9996 | pop_gimplify_context (bind); |
9997 | |
9998 | gimple_bind_append_vars (bind_stmt: bind, vars: ctx->block_vars); |
9999 | BLOCK_VARS (block) = gimple_bind_vars (bind_stmt: bind); |
10000 | } |
10001 | |
10002 | |
10003 | /* Expand code for an OpenMP scan directive and the structured block |
10004 | before the scan directive. */ |
10005 | |
10006 | static void |
10007 | lower_omp_scan (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
10008 | { |
10009 | gimple *stmt = gsi_stmt (i: *gsi_p); |
10010 | bool has_clauses |
10011 | = gimple_omp_scan_clauses (scan_stmt: as_a <gomp_scan *> (p: stmt)) != NULL; |
10012 | tree lane = NULL_TREE; |
10013 | gimple_seq before = NULL; |
10014 | omp_context *octx = ctx->outer; |
10015 | gcc_assert (octx); |
10016 | if (octx->scan_exclusive && !has_clauses) |
10017 | { |
10018 | gimple_stmt_iterator gsi2 = *gsi_p; |
10019 | gsi_next (i: &gsi2); |
10020 | gimple *stmt2 = gsi_stmt (i: gsi2); |
10021 | /* For exclusive scan, swap GIMPLE_OMP_SCAN without clauses |
10022 | with following GIMPLE_OMP_SCAN with clauses, so that input_phase, |
10023 | the one with exclusive clause(s), comes first. */ |
10024 | if (stmt2 |
10025 | && gimple_code (g: stmt2) == GIMPLE_OMP_SCAN |
10026 | && gimple_omp_scan_clauses (scan_stmt: as_a <gomp_scan *> (p: stmt2)) != NULL) |
10027 | { |
10028 | gsi_remove (gsi_p, false); |
10029 | gsi_insert_after (gsi_p, stmt, GSI_SAME_STMT); |
10030 | ctx = maybe_lookup_ctx (stmt: stmt2); |
10031 | gcc_assert (ctx); |
10032 | lower_omp_scan (gsi_p, ctx); |
10033 | return; |
10034 | } |
10035 | } |
10036 | |
10037 | bool input_phase = has_clauses ^ octx->scan_inclusive; |
10038 | bool is_simd = (gimple_code (g: octx->stmt) == GIMPLE_OMP_FOR |
10039 | && gimple_omp_for_kind (g: octx->stmt) == GF_OMP_FOR_KIND_SIMD); |
10040 | bool is_for = (gimple_code (g: octx->stmt) == GIMPLE_OMP_FOR |
10041 | && gimple_omp_for_kind (g: octx->stmt) == GF_OMP_FOR_KIND_FOR |
10042 | && !gimple_omp_for_combined_p (g: octx->stmt)); |
10043 | bool is_for_simd = is_simd && gimple_omp_for_combined_into_p (g: octx->stmt); |
10044 | if (is_for_simd && octx->for_simd_scan_phase) |
10045 | is_simd = false; |
10046 | if (is_simd) |
10047 | if (tree c = omp_find_clause (clauses: gimple_omp_for_clauses (gs: octx->stmt), |
10048 | kind: OMP_CLAUSE__SIMDUID_)) |
10049 | { |
10050 | tree uid = OMP_CLAUSE__SIMDUID__DECL (c); |
10051 | lane = create_tmp_var (unsigned_type_node); |
10052 | tree t = build_int_cst (integer_type_node, |
10053 | input_phase ? 1 |
10054 | : octx->scan_inclusive ? 2 : 3); |
10055 | gimple *g |
10056 | = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 2, uid, t); |
10057 | gimple_call_set_lhs (gs: g, lhs: lane); |
10058 | gimple_seq_add_stmt (&before, g); |
10059 | } |
10060 | |
10061 | if (is_simd || is_for) |
10062 | { |
10063 | for (tree c = gimple_omp_for_clauses (gs: octx->stmt); |
10064 | c; c = OMP_CLAUSE_CHAIN (c)) |
10065 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
10066 | && OMP_CLAUSE_REDUCTION_INSCAN (c)) |
10067 | { |
10068 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
10069 | tree var = OMP_CLAUSE_DECL (c); |
10070 | tree new_var = lookup_decl (var, ctx: octx); |
10071 | tree val = new_var; |
10072 | tree var2 = NULL_TREE; |
10073 | tree var3 = NULL_TREE; |
10074 | tree var4 = NULL_TREE; |
10075 | tree lane0 = NULL_TREE; |
10076 | tree new_vard = new_var; |
10077 | if (omp_privatize_by_reference (decl: var)) |
10078 | { |
10079 | new_var = build_simple_mem_ref_loc (clause_loc, new_var); |
10080 | val = new_var; |
10081 | } |
10082 | if (DECL_HAS_VALUE_EXPR_P (new_vard)) |
10083 | { |
10084 | val = DECL_VALUE_EXPR (new_vard); |
10085 | if (new_vard != new_var) |
10086 | { |
10087 | gcc_assert (TREE_CODE (val) == ADDR_EXPR); |
10088 | val = TREE_OPERAND (val, 0); |
10089 | } |
10090 | if (TREE_CODE (val) == ARRAY_REF |
10091 | && VAR_P (TREE_OPERAND (val, 0))) |
10092 | { |
10093 | tree v = TREE_OPERAND (val, 0); |
10094 | if (lookup_attribute (attr_name: "omp simd array" , |
10095 | DECL_ATTRIBUTES (v))) |
10096 | { |
10097 | val = unshare_expr (val); |
10098 | lane0 = TREE_OPERAND (val, 1); |
10099 | TREE_OPERAND (val, 1) = lane; |
10100 | var2 = lookup_decl (var: v, ctx: octx); |
10101 | if (octx->scan_exclusive) |
10102 | var4 = lookup_decl (var: var2, ctx: octx); |
10103 | if (input_phase |
10104 | && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
10105 | var3 = maybe_lookup_decl (var: var4 ? var4 : var2, ctx: octx); |
10106 | if (!input_phase) |
10107 | { |
10108 | var2 = build4 (ARRAY_REF, TREE_TYPE (val), |
10109 | var2, lane, NULL_TREE, NULL_TREE); |
10110 | TREE_THIS_NOTRAP (var2) = 1; |
10111 | if (octx->scan_exclusive) |
10112 | { |
10113 | var4 = build4 (ARRAY_REF, TREE_TYPE (val), |
10114 | var4, lane, NULL_TREE, |
10115 | NULL_TREE); |
10116 | TREE_THIS_NOTRAP (var4) = 1; |
10117 | } |
10118 | } |
10119 | else |
10120 | var2 = val; |
10121 | } |
10122 | } |
10123 | gcc_assert (var2); |
10124 | } |
10125 | else |
10126 | { |
10127 | var2 = build_outer_var_ref (var, ctx: octx); |
10128 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
10129 | { |
10130 | var3 = maybe_lookup_decl (var: new_vard, ctx: octx); |
10131 | if (var3 == new_vard || var3 == NULL_TREE) |
10132 | var3 = NULL_TREE; |
10133 | else if (is_simd && octx->scan_exclusive && !input_phase) |
10134 | { |
10135 | var4 = maybe_lookup_decl (var: var3, ctx: octx); |
10136 | if (var4 == var3 || var4 == NULL_TREE) |
10137 | { |
10138 | if (TREE_ADDRESSABLE (TREE_TYPE (new_var))) |
10139 | { |
10140 | var4 = var3; |
10141 | var3 = NULL_TREE; |
10142 | } |
10143 | else |
10144 | var4 = NULL_TREE; |
10145 | } |
10146 | } |
10147 | } |
10148 | if (is_simd |
10149 | && octx->scan_exclusive |
10150 | && !input_phase |
10151 | && var4 == NULL_TREE) |
10152 | var4 = create_tmp_var (TREE_TYPE (val)); |
10153 | } |
10154 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
10155 | { |
10156 | tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); |
10157 | if (input_phase) |
10158 | { |
10159 | if (var3) |
10160 | { |
10161 | /* If we've added a separate identity element |
10162 | variable, copy it over into val. */ |
10163 | tree x = lang_hooks.decls.omp_clause_assign_op (c, val, |
10164 | var3); |
10165 | gimplify_and_add (x, &before); |
10166 | } |
10167 | else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) |
10168 | { |
10169 | /* Otherwise, assign to it the identity element. */ |
10170 | gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); |
10171 | if (is_for) |
10172 | tseq = copy_gimple_seq_and_replace_locals (seq: tseq); |
10173 | tree ref = build_outer_var_ref (var, ctx: octx); |
10174 | tree x = (DECL_HAS_VALUE_EXPR_P (new_vard) |
10175 | ? DECL_VALUE_EXPR (new_vard) : NULL_TREE); |
10176 | if (x) |
10177 | { |
10178 | if (new_vard != new_var) |
10179 | val = build_fold_addr_expr_loc (clause_loc, val); |
10180 | SET_DECL_VALUE_EXPR (new_vard, val); |
10181 | } |
10182 | SET_DECL_VALUE_EXPR (placeholder, ref); |
10183 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
10184 | lower_omp (&tseq, octx); |
10185 | if (x) |
10186 | SET_DECL_VALUE_EXPR (new_vard, x); |
10187 | SET_DECL_VALUE_EXPR (placeholder, NULL_TREE); |
10188 | DECL_HAS_VALUE_EXPR_P (placeholder) = 0; |
10189 | gimple_seq_add_seq (&before, tseq); |
10190 | if (is_simd) |
10191 | OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; |
10192 | } |
10193 | } |
10194 | else if (is_simd) |
10195 | { |
10196 | tree x; |
10197 | if (octx->scan_exclusive) |
10198 | { |
10199 | tree v4 = unshare_expr (var4); |
10200 | tree v2 = unshare_expr (var2); |
10201 | x = lang_hooks.decls.omp_clause_assign_op (c, v4, v2); |
10202 | gimplify_and_add (x, &before); |
10203 | } |
10204 | gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); |
10205 | x = (DECL_HAS_VALUE_EXPR_P (new_vard) |
10206 | ? DECL_VALUE_EXPR (new_vard) : NULL_TREE); |
10207 | tree vexpr = val; |
10208 | if (x && new_vard != new_var) |
10209 | vexpr = build_fold_addr_expr_loc (clause_loc, val); |
10210 | if (x) |
10211 | SET_DECL_VALUE_EXPR (new_vard, vexpr); |
10212 | SET_DECL_VALUE_EXPR (placeholder, var2); |
10213 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
10214 | lower_omp (&tseq, octx); |
10215 | gimple_seq_add_seq (&before, tseq); |
10216 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; |
10217 | if (x) |
10218 | SET_DECL_VALUE_EXPR (new_vard, x); |
10219 | SET_DECL_VALUE_EXPR (placeholder, NULL_TREE); |
10220 | DECL_HAS_VALUE_EXPR_P (placeholder) = 0; |
10221 | if (octx->scan_inclusive) |
10222 | { |
10223 | x = lang_hooks.decls.omp_clause_assign_op (c, val, |
10224 | var2); |
10225 | gimplify_and_add (x, &before); |
10226 | } |
10227 | else if (lane0 == NULL_TREE) |
10228 | { |
10229 | x = lang_hooks.decls.omp_clause_assign_op (c, val, |
10230 | var4); |
10231 | gimplify_and_add (x, &before); |
10232 | } |
10233 | } |
10234 | } |
10235 | else |
10236 | { |
10237 | if (input_phase) |
10238 | { |
10239 | /* input phase. Set val to initializer before |
10240 | the body. */ |
10241 | tree x = omp_reduction_init (clause: c, TREE_TYPE (new_var)); |
10242 | gimplify_assign (val, x, &before); |
10243 | } |
10244 | else if (is_simd) |
10245 | { |
10246 | /* scan phase. */ |
10247 | enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c); |
10248 | if (code == MINUS_EXPR) |
10249 | code = PLUS_EXPR; |
10250 | |
10251 | tree x = build2 (code, TREE_TYPE (var2), |
10252 | unshare_expr (var2), unshare_expr (val)); |
10253 | if (octx->scan_inclusive) |
10254 | { |
10255 | gimplify_assign (unshare_expr (var2), x, &before); |
10256 | gimplify_assign (val, var2, &before); |
10257 | } |
10258 | else |
10259 | { |
10260 | gimplify_assign (unshare_expr (var4), |
10261 | unshare_expr (var2), &before); |
10262 | gimplify_assign (var2, x, &before); |
10263 | if (lane0 == NULL_TREE) |
10264 | gimplify_assign (val, var4, &before); |
10265 | } |
10266 | } |
10267 | } |
10268 | if (octx->scan_exclusive && !input_phase && lane0) |
10269 | { |
10270 | tree vexpr = unshare_expr (var4); |
10271 | TREE_OPERAND (vexpr, 1) = lane0; |
10272 | if (new_vard != new_var) |
10273 | vexpr = build_fold_addr_expr_loc (clause_loc, vexpr); |
10274 | SET_DECL_VALUE_EXPR (new_vard, vexpr); |
10275 | } |
10276 | } |
10277 | } |
10278 | if (is_simd && !is_for_simd) |
10279 | { |
10280 | gsi_insert_seq_after (gsi_p, gimple_omp_body (gs: stmt), GSI_SAME_STMT); |
10281 | gsi_insert_seq_after (gsi_p, before, GSI_SAME_STMT); |
10282 | gsi_replace (gsi_p, gimple_build_nop (), true); |
10283 | return; |
10284 | } |
10285 | lower_omp (gimple_omp_body_ptr (gs: stmt), octx); |
10286 | if (before) |
10287 | { |
10288 | gimple_stmt_iterator gsi = gsi_start (seq&: *gimple_omp_body_ptr (gs: stmt)); |
10289 | gsi_insert_seq_before (&gsi, before, GSI_SAME_STMT); |
10290 | } |
10291 | } |
10292 | |
10293 | |
10294 | /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple |
10295 | substitution of a couple of function calls. But in the NAMED case, |
10296 | requires that languages coordinate a symbol name. It is therefore |
10297 | best put here in common code. */ |
10298 | |
10299 | static GTY(()) hash_map<tree, tree> *critical_name_mutexes; |
10300 | |
10301 | static void |
10302 | lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
10303 | { |
10304 | tree block; |
10305 | tree name, lock, unlock; |
10306 | gomp_critical *stmt = as_a <gomp_critical *> (p: gsi_stmt (i: *gsi_p)); |
10307 | gbind *bind; |
10308 | location_t loc = gimple_location (g: stmt); |
10309 | gimple_seq tbody; |
10310 | |
10311 | name = gimple_omp_critical_name (crit_stmt: stmt); |
10312 | if (name) |
10313 | { |
10314 | tree decl; |
10315 | |
10316 | if (!critical_name_mutexes) |
10317 | critical_name_mutexes = hash_map<tree, tree>::create_ggc (size: 10); |
10318 | |
10319 | tree *n = critical_name_mutexes->get (k: name); |
10320 | if (n == NULL) |
10321 | { |
10322 | char *new_str; |
10323 | |
10324 | decl = create_tmp_var_raw (ptr_type_node); |
10325 | |
10326 | new_str = ACONCAT ((".gomp_critical_user_" , |
10327 | IDENTIFIER_POINTER (name), NULL)); |
10328 | DECL_NAME (decl) = get_identifier (new_str); |
10329 | TREE_PUBLIC (decl) = 1; |
10330 | TREE_STATIC (decl) = 1; |
10331 | DECL_COMMON (decl) = 1; |
10332 | DECL_ARTIFICIAL (decl) = 1; |
10333 | DECL_IGNORED_P (decl) = 1; |
10334 | |
10335 | varpool_node::finalize_decl (decl); |
10336 | |
10337 | critical_name_mutexes->put (k: name, v: decl); |
10338 | } |
10339 | else |
10340 | decl = *n; |
10341 | |
10342 | /* If '#pragma omp critical' is inside offloaded region or |
10343 | inside function marked as offloadable, the symbol must be |
10344 | marked as offloadable too. */ |
10345 | omp_context *octx; |
10346 | if (cgraph_node::get (decl: current_function_decl)->offloadable) |
10347 | varpool_node::get_create (decl)->offloadable = 1; |
10348 | else |
10349 | for (octx = ctx->outer; octx; octx = octx->outer) |
10350 | if (is_gimple_omp_offloaded (stmt: octx->stmt)) |
10351 | { |
10352 | varpool_node::get_create (decl)->offloadable = 1; |
10353 | break; |
10354 | } |
10355 | |
10356 | lock = builtin_decl_explicit (fncode: BUILT_IN_GOMP_CRITICAL_NAME_START); |
10357 | lock = build_call_expr_loc (loc, lock, 1, |
10358 | build_fold_addr_expr_loc (loc, decl)); |
10359 | |
10360 | unlock = builtin_decl_explicit (fncode: BUILT_IN_GOMP_CRITICAL_NAME_END); |
10361 | unlock = build_call_expr_loc (loc, unlock, 1, |
10362 | build_fold_addr_expr_loc (loc, decl)); |
10363 | } |
10364 | else |
10365 | { |
10366 | lock = builtin_decl_explicit (fncode: BUILT_IN_GOMP_CRITICAL_START); |
10367 | lock = build_call_expr_loc (loc, lock, 0); |
10368 | |
10369 | unlock = builtin_decl_explicit (fncode: BUILT_IN_GOMP_CRITICAL_END); |
10370 | unlock = build_call_expr_loc (loc, unlock, 0); |
10371 | } |
10372 | |
10373 | push_gimplify_context (); |
10374 | |
10375 | block = make_node (BLOCK); |
10376 | bind = gimple_build_bind (NULL, NULL, block); |
10377 | gsi_replace (gsi_p, bind, true); |
10378 | gimple_bind_add_stmt (bind_stmt: bind, stmt); |
10379 | |
10380 | tbody = gimple_bind_body (gs: bind); |
10381 | gimplify_and_add (lock, &tbody); |
10382 | gimple_bind_set_body (bind_stmt: bind, seq: tbody); |
10383 | |
10384 | lower_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
10385 | gimple_omp_set_body (gs: stmt, body: maybe_catch_exception (body: gimple_omp_body (gs: stmt))); |
10386 | gimple_bind_add_seq (bind_stmt: bind, seq: gimple_omp_body (gs: stmt)); |
10387 | gimple_omp_set_body (gs: stmt, NULL); |
10388 | |
10389 | tbody = gimple_bind_body (gs: bind); |
10390 | gimplify_and_add (unlock, &tbody); |
10391 | gimple_bind_set_body (bind_stmt: bind, seq: tbody); |
10392 | |
10393 | gimple_bind_add_stmt (bind_stmt: bind, stmt: gimple_build_omp_return (true)); |
10394 | |
10395 | pop_gimplify_context (bind); |
10396 | gimple_bind_append_vars (bind_stmt: bind, vars: ctx->block_vars); |
10397 | BLOCK_VARS (block) = gimple_bind_vars (bind_stmt: bind); |
10398 | } |
10399 | |
10400 | /* A subroutine of lower_omp_for. Generate code to emit the predicate |
10401 | for a lastprivate clause. Given a loop control predicate of (V |
10402 | cond N2), we gate the clause on (!(V cond N2)). The lowered form |
10403 | is appended to *DLIST, iterator initialization is appended to |
10404 | *BODY_P. *CLIST is for lastprivate(conditional:) code that needs |
10405 | to be emitted in a critical section. */ |
10406 | |
10407 | static void |
10408 | lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p, |
10409 | gimple_seq *dlist, gimple_seq *clist, |
10410 | struct omp_context *ctx) |
10411 | { |
10412 | tree clauses, cond, vinit; |
10413 | enum tree_code cond_code; |
10414 | gimple_seq stmts; |
10415 | |
10416 | cond_code = fd->loop.cond_code; |
10417 | cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR; |
10418 | |
10419 | /* When possible, use a strict equality expression. This can let VRP |
10420 | type optimizations deduce the value and remove a copy. */ |
10421 | if (tree_fits_shwi_p (fd->loop.step)) |
10422 | { |
10423 | HOST_WIDE_INT step = tree_to_shwi (fd->loop.step); |
10424 | if (step == 1 || step == -1) |
10425 | cond_code = EQ_EXPR; |
10426 | } |
10427 | |
10428 | tree n2 = fd->loop.n2; |
10429 | if (fd->collapse > 1 |
10430 | && TREE_CODE (n2) != INTEGER_CST |
10431 | && gimple_omp_for_combined_into_p (g: fd->for_stmt)) |
10432 | { |
10433 | struct omp_context *taskreg_ctx = NULL; |
10434 | if (gimple_code (g: ctx->outer->stmt) == GIMPLE_OMP_FOR) |
10435 | { |
10436 | gomp_for *gfor = as_a <gomp_for *> (p: ctx->outer->stmt); |
10437 | if (gimple_omp_for_kind (g: gfor) == GF_OMP_FOR_KIND_FOR |
10438 | || gimple_omp_for_kind (g: gfor) == GF_OMP_FOR_KIND_DISTRIBUTE) |
10439 | { |
10440 | if (gimple_omp_for_combined_into_p (g: gfor)) |
10441 | { |
10442 | gcc_assert (ctx->outer->outer |
10443 | && is_parallel_ctx (ctx->outer->outer)); |
10444 | taskreg_ctx = ctx->outer->outer; |
10445 | } |
10446 | else |
10447 | { |
10448 | struct omp_for_data outer_fd; |
10449 | omp_extract_for_data (for_stmt: gfor, fd: &outer_fd, NULL); |
10450 | n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2); |
10451 | } |
10452 | } |
10453 | else if (gimple_omp_for_kind (g: gfor) == GF_OMP_FOR_KIND_TASKLOOP) |
10454 | taskreg_ctx = ctx->outer->outer; |
10455 | } |
10456 | else if (is_taskreg_ctx (ctx: ctx->outer)) |
10457 | taskreg_ctx = ctx->outer; |
10458 | if (taskreg_ctx) |
10459 | { |
10460 | int i; |
10461 | tree taskreg_clauses |
10462 | = gimple_omp_taskreg_clauses (gs: taskreg_ctx->stmt); |
10463 | tree innerc = omp_find_clause (clauses: taskreg_clauses, |
10464 | kind: OMP_CLAUSE__LOOPTEMP_); |
10465 | gcc_assert (innerc); |
10466 | int count = fd->collapse; |
10467 | if (fd->non_rect |
10468 | && fd->last_nonrect == fd->first_nonrect + 1) |
10469 | if (tree v = gimple_omp_for_index (gs: fd->for_stmt, i: fd->last_nonrect)) |
10470 | if (!TYPE_UNSIGNED (TREE_TYPE (v))) |
10471 | count += 4; |
10472 | for (i = 0; i < count; i++) |
10473 | { |
10474 | innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc), |
10475 | kind: OMP_CLAUSE__LOOPTEMP_); |
10476 | gcc_assert (innerc); |
10477 | } |
10478 | innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc), |
10479 | kind: OMP_CLAUSE__LOOPTEMP_); |
10480 | if (innerc) |
10481 | n2 = fold_convert (TREE_TYPE (n2), |
10482 | lookup_decl (OMP_CLAUSE_DECL (innerc), |
10483 | taskreg_ctx)); |
10484 | } |
10485 | } |
10486 | cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2); |
10487 | |
10488 | clauses = gimple_omp_for_clauses (gs: fd->for_stmt); |
10489 | stmts = NULL; |
10490 | lower_lastprivate_clauses (clauses, predicate: cond, body_p, stmt_list: &stmts, cstmt_list: clist, ctx); |
10491 | if (!gimple_seq_empty_p (s: stmts)) |
10492 | { |
10493 | gimple_seq_add_seq (&stmts, *dlist); |
10494 | *dlist = stmts; |
10495 | |
10496 | /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */ |
10497 | vinit = fd->loop.n1; |
10498 | if (cond_code == EQ_EXPR |
10499 | && tree_fits_shwi_p (fd->loop.n2) |
10500 | && ! integer_zerop (fd->loop.n2)) |
10501 | vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0); |
10502 | else |
10503 | vinit = unshare_expr (vinit); |
10504 | |
10505 | /* Initialize the iterator variable, so that threads that don't execute |
10506 | any iterations don't execute the lastprivate clauses by accident. */ |
10507 | gimplify_assign (fd->loop.v, vinit, body_p); |
10508 | } |
10509 | } |
10510 | |
10511 | /* OpenACC privatization. |
10512 | |
10513 | Or, in other words, *sharing* at the respective OpenACC level of |
10514 | parallelism. |
10515 | |
10516 | From a correctness perspective, a non-addressable variable can't be accessed |
10517 | outside the current thread, so it can go in a (faster than shared memory) |
10518 | register -- though that register may need to be broadcast in some |
10519 | circumstances. A variable can only meaningfully be "shared" across workers |
10520 | or vector lanes if its address is taken, e.g. by a call to an atomic |
10521 | builtin. |
10522 | |
10523 | From an optimisation perspective, the answer might be fuzzier: maybe |
10524 | sometimes, using shared memory directly would be faster than |
10525 | broadcasting. */ |
10526 | |
10527 | static void |
10528 | oacc_privatization_begin_diagnose_var (const dump_flags_t l_dump_flags, |
10529 | const location_t loc, const tree c, |
10530 | const tree decl) |
10531 | { |
10532 | const dump_user_location_t d_u_loc |
10533 | = dump_user_location_t::from_location_t (loc); |
10534 | /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */ |
10535 | #if __GNUC__ >= 10 |
10536 | # pragma GCC diagnostic push |
10537 | # pragma GCC diagnostic ignored "-Wformat" |
10538 | #endif |
10539 | dump_printf_loc (l_dump_flags, d_u_loc, |
10540 | "variable %<%T%> " , decl); |
10541 | #if __GNUC__ >= 10 |
10542 | # pragma GCC diagnostic pop |
10543 | #endif |
10544 | if (c) |
10545 | dump_printf (l_dump_flags, |
10546 | "in %qs clause " , |
10547 | omp_clause_code_name[OMP_CLAUSE_CODE (c)]); |
10548 | else |
10549 | dump_printf (l_dump_flags, |
10550 | "declared in block " ); |
10551 | } |
10552 | |
10553 | static bool |
10554 | oacc_privatization_candidate_p (const location_t loc, const tree c, |
10555 | const tree decl) |
10556 | { |
10557 | dump_flags_t l_dump_flags = get_openacc_privatization_dump_flags (); |
10558 | |
10559 | /* There is some differentiation depending on block vs. clause. */ |
10560 | bool block = !c; |
10561 | |
10562 | bool res = true; |
10563 | |
10564 | if (res && !VAR_P (decl)) |
10565 | { |
10566 | /* A PARM_DECL (appearing in a 'private' clause) is expected to have been |
10567 | privatized into a new VAR_DECL. */ |
10568 | gcc_checking_assert (TREE_CODE (decl) != PARM_DECL); |
10569 | |
10570 | res = false; |
10571 | |
10572 | if (dump_enabled_p ()) |
10573 | { |
10574 | oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl); |
10575 | dump_printf (l_dump_flags, |
10576 | "potentially has improper OpenACC privatization level: %qs\n" , |
10577 | get_tree_code_name (TREE_CODE (decl))); |
10578 | } |
10579 | } |
10580 | |
10581 | if (res && block && TREE_STATIC (decl)) |
10582 | { |
10583 | res = false; |
10584 | |
10585 | if (dump_enabled_p ()) |
10586 | { |
10587 | oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl); |
10588 | dump_printf (l_dump_flags, |
10589 | "isn%'t candidate for adjusting OpenACC privatization level: %s\n" , |
10590 | "static" ); |
10591 | } |
10592 | } |
10593 | |
10594 | if (res && block && DECL_EXTERNAL (decl)) |
10595 | { |
10596 | res = false; |
10597 | |
10598 | if (dump_enabled_p ()) |
10599 | { |
10600 | oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl); |
10601 | dump_printf (l_dump_flags, |
10602 | "isn%'t candidate for adjusting OpenACC privatization level: %s\n" , |
10603 | "external" ); |
10604 | } |
10605 | } |
10606 | |
10607 | if (res && !TREE_ADDRESSABLE (decl)) |
10608 | { |
10609 | res = false; |
10610 | |
10611 | if (dump_enabled_p ()) |
10612 | { |
10613 | oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl); |
10614 | dump_printf (l_dump_flags, |
10615 | "isn%'t candidate for adjusting OpenACC privatization level: %s\n" , |
10616 | "not addressable" ); |
10617 | } |
10618 | } |
10619 | |
10620 | /* If an artificial variable has been added to a bind, e.g. |
10621 | a compiler-generated temporary structure used by the Fortran front-end, do |
10622 | not consider it as a privatization candidate. Note that variables on |
10623 | the stack are private per-thread by default: making them "gang-private" |
10624 | for OpenACC actually means to share a single instance of a variable |
10625 | amongst all workers and threads spawned within each gang. |
10626 | At present, no compiler-generated artificial variables require such |
10627 | sharing semantics, so this is safe. */ |
10628 | |
10629 | if (res && block && DECL_ARTIFICIAL (decl)) |
10630 | { |
10631 | res = false; |
10632 | |
10633 | if (dump_enabled_p ()) |
10634 | { |
10635 | oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl); |
10636 | dump_printf (l_dump_flags, |
10637 | "isn%'t candidate for adjusting OpenACC privatization " |
10638 | "level: %s\n" , "artificial" ); |
10639 | } |
10640 | } |
10641 | |
10642 | if (res) |
10643 | { |
10644 | if (dump_enabled_p ()) |
10645 | { |
10646 | oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl); |
10647 | dump_printf (l_dump_flags, |
10648 | "is candidate for adjusting OpenACC privatization level\n" ); |
10649 | } |
10650 | } |
10651 | |
10652 | if (dump_file && (dump_flags & TDF_DETAILS)) |
10653 | { |
10654 | print_generic_decl (dump_file, decl, dump_flags); |
10655 | fprintf (stream: dump_file, format: "\n" ); |
10656 | } |
10657 | |
10658 | return res; |
10659 | } |
10660 | |
10661 | /* Scan CLAUSES for candidates for adjusting OpenACC privatization level in |
10662 | CTX. */ |
10663 | |
10664 | static void |
10665 | oacc_privatization_scan_clause_chain (omp_context *ctx, tree clauses) |
10666 | { |
10667 | for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
10668 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE) |
10669 | { |
10670 | tree decl = OMP_CLAUSE_DECL (c); |
10671 | |
10672 | tree new_decl = lookup_decl (var: decl, ctx); |
10673 | |
10674 | if (!oacc_privatization_candidate_p (OMP_CLAUSE_LOCATION (c), c, |
10675 | decl: new_decl)) |
10676 | continue; |
10677 | |
10678 | gcc_checking_assert |
10679 | (!ctx->oacc_privatization_candidates.contains (new_decl)); |
10680 | ctx->oacc_privatization_candidates.safe_push (obj: new_decl); |
10681 | } |
10682 | } |
10683 | |
10684 | /* Scan DECLS for candidates for adjusting OpenACC privatization level in |
10685 | CTX. */ |
10686 | |
10687 | static void |
10688 | oacc_privatization_scan_decl_chain (omp_context *ctx, tree decls) |
10689 | { |
10690 | for (tree decl = decls; decl; decl = DECL_CHAIN (decl)) |
10691 | { |
10692 | tree new_decl = lookup_decl (var: decl, ctx); |
10693 | gcc_checking_assert (new_decl == decl); |
10694 | |
10695 | if (!oacc_privatization_candidate_p (loc: gimple_location (g: ctx->stmt), NULL, |
10696 | decl: new_decl)) |
10697 | continue; |
10698 | |
10699 | gcc_checking_assert |
10700 | (!ctx->oacc_privatization_candidates.contains (new_decl)); |
10701 | ctx->oacc_privatization_candidates.safe_push (obj: new_decl); |
10702 | } |
10703 | } |
10704 | |
10705 | /* Callback for walk_gimple_seq. Find #pragma omp scan statement. */ |
10706 | |
10707 | static tree |
10708 | omp_find_scan (gimple_stmt_iterator *gsi_p, bool *handled_ops_p, |
10709 | struct walk_stmt_info *wi) |
10710 | { |
10711 | gimple *stmt = gsi_stmt (i: *gsi_p); |
10712 | |
10713 | *handled_ops_p = true; |
10714 | switch (gimple_code (g: stmt)) |
10715 | { |
10716 | WALK_SUBSTMTS; |
10717 | |
10718 | case GIMPLE_OMP_FOR: |
10719 | if (gimple_omp_for_kind (g: stmt) == GF_OMP_FOR_KIND_SIMD |
10720 | && gimple_omp_for_combined_into_p (g: stmt)) |
10721 | *handled_ops_p = false; |
10722 | break; |
10723 | |
10724 | case GIMPLE_OMP_SCAN: |
10725 | *(gimple_stmt_iterator *) (wi->info) = *gsi_p; |
10726 | return integer_zero_node; |
10727 | default: |
10728 | break; |
10729 | } |
10730 | return NULL; |
10731 | } |
10732 | |
10733 | /* Helper function for lower_omp_for, add transformations for a worksharing |
10734 | loop with scan directives inside of it. |
10735 | For worksharing loop not combined with simd, transform: |
10736 | #pragma omp for reduction(inscan,+:r) private(i) |
10737 | for (i = 0; i < n; i = i + 1) |
10738 | { |
10739 | { |
10740 | update (r); |
10741 | } |
10742 | #pragma omp scan inclusive(r) |
10743 | { |
10744 | use (r); |
10745 | } |
10746 | } |
10747 | |
10748 | into two worksharing loops + code to merge results: |
10749 | |
10750 | num_threads = omp_get_num_threads (); |
10751 | thread_num = omp_get_thread_num (); |
10752 | if (thread_num == 0) goto <D.2099>; else goto <D.2100>; |
10753 | <D.2099>: |
10754 | var2 = r; |
10755 | goto <D.2101>; |
10756 | <D.2100>: |
10757 | // For UDRs this is UDR init, or if ctors are needed, copy from |
10758 | // var3 that has been constructed to contain the neutral element. |
10759 | var2 = 0; |
10760 | <D.2101>: |
10761 | ivar = 0; |
10762 | // The _scantemp_ clauses will arrange for rpriva to be initialized to |
10763 | // a shared array with num_threads elements and rprivb to a local array |
10764 | // number of elements equal to the number of (contiguous) iterations the |
10765 | // current thread will perform. controlb and controlp variables are |
10766 | // temporaries to handle deallocation of rprivb at the end of second |
10767 | // GOMP_FOR. |
10768 | #pragma omp for _scantemp_(rpriva) _scantemp_(rprivb) _scantemp_(controlb) \ |
10769 | _scantemp_(controlp) reduction(inscan,+:r) private(i) nowait |
10770 | for (i = 0; i < n; i = i + 1) |
10771 | { |
10772 | { |
10773 | // For UDRs this is UDR init or copy from var3. |
10774 | r = 0; |
10775 | // This is the input phase from user code. |
10776 | update (r); |
10777 | } |
10778 | { |
10779 | // For UDRs this is UDR merge. |
10780 | var2 = var2 + r; |
10781 | // Rather than handing it over to the user, save to local thread's |
10782 | // array. |
10783 | rprivb[ivar] = var2; |
10784 | // For exclusive scan, the above two statements are swapped. |
10785 | ivar = ivar + 1; |
10786 | } |
10787 | } |
10788 | // And remember the final value from this thread's into the shared |
10789 | // rpriva array. |
10790 | rpriva[(sizetype) thread_num] = var2; |
10791 | // If more than one thread, compute using Work-Efficient prefix sum |
10792 | // the inclusive parallel scan of the rpriva array. |
10793 | if (num_threads > 1) goto <D.2102>; else goto <D.2103>; |
10794 | <D.2102>: |
10795 | GOMP_barrier (); |
10796 | down = 0; |
10797 | k = 1; |
10798 | num_threadsu = (unsigned int) num_threads; |
10799 | thread_numup1 = (unsigned int) thread_num + 1; |
10800 | <D.2108>: |
10801 | twok = k << 1; |
10802 | if (twok > num_threadsu) goto <D.2110>; else goto <D.2111>; |
10803 | <D.2110>: |
10804 | down = 4294967295; |
10805 | k = k >> 1; |
10806 | if (k == num_threadsu) goto <D.2112>; else goto <D.2111>; |
10807 | <D.2112>: |
10808 | k = k >> 1; |
10809 | <D.2111>: |
10810 | twok = k << 1; |
10811 | cplx = .MUL_OVERFLOW (thread_nump1, twok); |
10812 | mul = REALPART_EXPR <cplx>; |
10813 | ovf = IMAGPART_EXPR <cplx>; |
10814 | if (ovf == 0) goto <D.2116>; else goto <D.2117>; |
10815 | <D.2116>: |
10816 | andv = k & down; |
10817 | andvm1 = andv + 4294967295; |
10818 | l = mul + andvm1; |
10819 | if (l < num_threadsu) goto <D.2120>; else goto <D.2117>; |
10820 | <D.2120>: |
10821 | // For UDRs this is UDR merge, performed using var2 variable as temporary, |
10822 | // i.e. var2 = rpriva[l - k]; UDR merge (var2, rpriva[l]); rpriva[l] = var2; |
10823 | rpriva[l] = rpriva[l - k] + rpriva[l]; |
10824 | <D.2117>: |
10825 | if (down == 0) goto <D.2121>; else goto <D.2122>; |
10826 | <D.2121>: |
10827 | k = k << 1; |
10828 | goto <D.2123>; |
10829 | <D.2122>: |
10830 | k = k >> 1; |
10831 | <D.2123>: |
10832 | GOMP_barrier (); |
10833 | if (k != 0) goto <D.2108>; else goto <D.2103>; |
10834 | <D.2103>: |
10835 | if (thread_num == 0) goto <D.2124>; else goto <D.2125>; |
10836 | <D.2124>: |
10837 | // For UDRs this is UDR init or copy from var3. |
10838 | var2 = 0; |
10839 | goto <D.2126>; |
10840 | <D.2125>: |
10841 | var2 = rpriva[thread_num - 1]; |
10842 | <D.2126>: |
10843 | ivar = 0; |
10844 | #pragma omp for _scantemp_(controlb) _scantemp_(controlp) \ |
10845 | reduction(inscan,+:r) private(i) |
10846 | for (i = 0; i < n; i = i + 1) |
10847 | { |
10848 | { |
10849 | // For UDRs, this is r = var2; UDR merge (r, rprivb[ivar]); |
10850 | r = var2 + rprivb[ivar]; |
10851 | } |
10852 | { |
10853 | // This is the scan phase from user code. |
10854 | use (r); |
10855 | // Plus a bump of the iterator. |
10856 | ivar = ivar + 1; |
10857 | } |
10858 | } */ |
10859 | |
10860 | static void |
10861 | lower_omp_for_scan (gimple_seq *body_p, gimple_seq *dlist, gomp_for *stmt, |
10862 | struct omp_for_data *fd, omp_context *ctx) |
10863 | { |
10864 | bool is_for_simd = gimple_omp_for_combined_p (g: stmt); |
10865 | gcc_assert (ctx->scan_inclusive || ctx->scan_exclusive); |
10866 | |
10867 | gimple_seq body = gimple_omp_body (gs: stmt); |
10868 | gimple_stmt_iterator input1_gsi = gsi_none (); |
10869 | struct walk_stmt_info wi; |
10870 | memset (s: &wi, c: 0, n: sizeof (wi)); |
10871 | wi.val_only = true; |
10872 | wi.info = (void *) &input1_gsi; |
10873 | walk_gimple_seq_mod (&body, omp_find_scan, NULL, &wi); |
10874 | gcc_assert (!gsi_end_p (input1_gsi)); |
10875 | |
10876 | gimple *input_stmt1 = gsi_stmt (i: input1_gsi); |
10877 | gimple_stmt_iterator gsi = input1_gsi; |
10878 | gsi_next (i: &gsi); |
10879 | gimple_stmt_iterator scan1_gsi = gsi; |
10880 | gimple *scan_stmt1 = gsi_stmt (i: gsi); |
10881 | gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN); |
10882 | |
10883 | gimple_seq input_body = gimple_omp_body (gs: input_stmt1); |
10884 | gimple_seq scan_body = gimple_omp_body (gs: scan_stmt1); |
10885 | gimple_omp_set_body (gs: input_stmt1, NULL); |
10886 | gimple_omp_set_body (gs: scan_stmt1, NULL); |
10887 | gimple_omp_set_body (gs: stmt, NULL); |
10888 | |
10889 | gomp_for *new_stmt = as_a <gomp_for *> (p: gimple_copy (stmt)); |
10890 | gimple_seq new_body = copy_gimple_seq_and_replace_locals (seq: body); |
10891 | gimple_omp_set_body (gs: stmt, body); |
10892 | gimple_omp_set_body (gs: input_stmt1, body: input_body); |
10893 | |
10894 | gimple_stmt_iterator input2_gsi = gsi_none (); |
10895 | memset (s: &wi, c: 0, n: sizeof (wi)); |
10896 | wi.val_only = true; |
10897 | wi.info = (void *) &input2_gsi; |
10898 | walk_gimple_seq_mod (&new_body, omp_find_scan, NULL, &wi); |
10899 | gcc_assert (!gsi_end_p (input2_gsi)); |
10900 | |
10901 | gimple *input_stmt2 = gsi_stmt (i: input2_gsi); |
10902 | gsi = input2_gsi; |
10903 | gsi_next (i: &gsi); |
10904 | gimple_stmt_iterator scan2_gsi = gsi; |
10905 | gimple *scan_stmt2 = gsi_stmt (i: gsi); |
10906 | gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN); |
10907 | gimple_omp_set_body (gs: scan_stmt2, body: scan_body); |
10908 | |
10909 | gimple_stmt_iterator input3_gsi = gsi_none (); |
10910 | gimple_stmt_iterator scan3_gsi = gsi_none (); |
10911 | gimple_stmt_iterator input4_gsi = gsi_none (); |
10912 | gimple_stmt_iterator scan4_gsi = gsi_none (); |
10913 | gimple *input_stmt3 = NULL, *scan_stmt3 = NULL; |
10914 | gimple *input_stmt4 = NULL, *scan_stmt4 = NULL; |
10915 | omp_context *input_simd_ctx = NULL, *scan_simd_ctx = NULL; |
10916 | if (is_for_simd) |
10917 | { |
10918 | memset (s: &wi, c: 0, n: sizeof (wi)); |
10919 | wi.val_only = true; |
10920 | wi.info = (void *) &input3_gsi; |
10921 | walk_gimple_seq_mod (&input_body, omp_find_scan, NULL, &wi); |
10922 | gcc_assert (!gsi_end_p (input3_gsi)); |
10923 | |
10924 | input_stmt3 = gsi_stmt (i: input3_gsi); |
10925 | gsi = input3_gsi; |
10926 | gsi_next (i: &gsi); |
10927 | scan3_gsi = gsi; |
10928 | scan_stmt3 = gsi_stmt (i: gsi); |
10929 | gcc_assert (scan_stmt3 && gimple_code (scan_stmt3) == GIMPLE_OMP_SCAN); |
10930 | |
10931 | memset (s: &wi, c: 0, n: sizeof (wi)); |
10932 | wi.val_only = true; |
10933 | wi.info = (void *) &input4_gsi; |
10934 | walk_gimple_seq_mod (&scan_body, omp_find_scan, NULL, &wi); |
10935 | gcc_assert (!gsi_end_p (input4_gsi)); |
10936 | |
10937 | input_stmt4 = gsi_stmt (i: input4_gsi); |
10938 | gsi = input4_gsi; |
10939 | gsi_next (i: &gsi); |
10940 | scan4_gsi = gsi; |
10941 | scan_stmt4 = gsi_stmt (i: gsi); |
10942 | gcc_assert (scan_stmt4 && gimple_code (scan_stmt4) == GIMPLE_OMP_SCAN); |
10943 | |
10944 | input_simd_ctx = maybe_lookup_ctx (stmt: input_stmt3)->outer; |
10945 | scan_simd_ctx = maybe_lookup_ctx (stmt: input_stmt4)->outer; |
10946 | } |
10947 | |
10948 | tree num_threads = create_tmp_var (integer_type_node); |
10949 | tree thread_num = create_tmp_var (integer_type_node); |
10950 | tree nthreads_decl = builtin_decl_explicit (fncode: BUILT_IN_OMP_GET_NUM_THREADS); |
10951 | tree threadnum_decl = builtin_decl_explicit (fncode: BUILT_IN_OMP_GET_THREAD_NUM); |
10952 | gimple *g = gimple_build_call (nthreads_decl, 0); |
10953 | gimple_call_set_lhs (gs: g, lhs: num_threads); |
10954 | gimple_seq_add_stmt (body_p, g); |
10955 | g = gimple_build_call (threadnum_decl, 0); |
10956 | gimple_call_set_lhs (gs: g, lhs: thread_num); |
10957 | gimple_seq_add_stmt (body_p, g); |
10958 | |
10959 | tree ivar = create_tmp_var (sizetype); |
10960 | tree new_clauses1 = NULL_TREE, new_clauses2 = NULL_TREE; |
10961 | tree *cp1 = &new_clauses1, *cp2 = &new_clauses2; |
10962 | tree k = create_tmp_var (unsigned_type_node); |
10963 | tree l = create_tmp_var (unsigned_type_node); |
10964 | |
10965 | gimple_seq clist = NULL, mdlist = NULL; |
10966 | gimple_seq thr01_list = NULL, thrn1_list = NULL; |
10967 | gimple_seq thr02_list = NULL, thrn2_list = NULL; |
10968 | gimple_seq scan1_list = NULL, input2_list = NULL; |
10969 | gimple_seq last_list = NULL, reduc_list = NULL; |
10970 | for (tree c = gimple_omp_for_clauses (gs: stmt); c; c = OMP_CLAUSE_CHAIN (c)) |
10971 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
10972 | && OMP_CLAUSE_REDUCTION_INSCAN (c)) |
10973 | { |
10974 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
10975 | tree var = OMP_CLAUSE_DECL (c); |
10976 | tree new_var = lookup_decl (var, ctx); |
10977 | tree var3 = NULL_TREE; |
10978 | tree new_vard = new_var; |
10979 | if (omp_privatize_by_reference (decl: var)) |
10980 | new_var = build_simple_mem_ref_loc (clause_loc, new_var); |
10981 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
10982 | { |
10983 | var3 = maybe_lookup_decl (var: new_vard, ctx); |
10984 | if (var3 == new_vard) |
10985 | var3 = NULL_TREE; |
10986 | } |
10987 | |
10988 | tree ptype = build_pointer_type (TREE_TYPE (new_var)); |
10989 | tree rpriva = create_tmp_var (ptype); |
10990 | tree nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_); |
10991 | OMP_CLAUSE_DECL (nc) = rpriva; |
10992 | *cp1 = nc; |
10993 | cp1 = &OMP_CLAUSE_CHAIN (nc); |
10994 | |
10995 | tree rprivb = create_tmp_var (ptype); |
10996 | nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_); |
10997 | OMP_CLAUSE_DECL (nc) = rprivb; |
10998 | OMP_CLAUSE__SCANTEMP__ALLOC (nc) = 1; |
10999 | *cp1 = nc; |
11000 | cp1 = &OMP_CLAUSE_CHAIN (nc); |
11001 | |
11002 | tree var2 = create_tmp_var_raw (TREE_TYPE (new_var)); |
11003 | if (new_vard != new_var) |
11004 | TREE_ADDRESSABLE (var2) = 1; |
11005 | gimple_add_tmp_var (var2); |
11006 | |
11007 | tree x = fold_convert_loc (clause_loc, sizetype, thread_num); |
11008 | x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x, |
11009 | TYPE_SIZE_UNIT (TREE_TYPE (ptype))); |
11010 | x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x); |
11011 | tree rpriva_ref = build_simple_mem_ref_loc (clause_loc, x); |
11012 | |
11013 | x = fold_build2_loc (clause_loc, PLUS_EXPR, integer_type_node, |
11014 | thread_num, integer_minus_one_node); |
11015 | x = fold_convert_loc (clause_loc, sizetype, x); |
11016 | x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x, |
11017 | TYPE_SIZE_UNIT (TREE_TYPE (ptype))); |
11018 | x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x); |
11019 | tree rprivam1_ref = build_simple_mem_ref_loc (clause_loc, x); |
11020 | |
11021 | x = fold_convert_loc (clause_loc, sizetype, l); |
11022 | x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x, |
11023 | TYPE_SIZE_UNIT (TREE_TYPE (ptype))); |
11024 | x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x); |
11025 | tree rprival_ref = build_simple_mem_ref_loc (clause_loc, x); |
11026 | |
11027 | x = fold_build2_loc (clause_loc, MINUS_EXPR, unsigned_type_node, l, k); |
11028 | x = fold_convert_loc (clause_loc, sizetype, x); |
11029 | x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x, |
11030 | TYPE_SIZE_UNIT (TREE_TYPE (ptype))); |
11031 | x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x); |
11032 | tree rprivalmk_ref = build_simple_mem_ref_loc (clause_loc, x); |
11033 | |
11034 | x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, ivar, |
11035 | TYPE_SIZE_UNIT (TREE_TYPE (ptype))); |
11036 | x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rprivb), rprivb, x); |
11037 | tree rprivb_ref = build_simple_mem_ref_loc (clause_loc, x); |
11038 | |
11039 | tree var4 = is_for_simd ? new_var : var2; |
11040 | tree var5 = NULL_TREE, var6 = NULL_TREE; |
11041 | if (is_for_simd) |
11042 | { |
11043 | var5 = lookup_decl (var, ctx: input_simd_ctx); |
11044 | var6 = lookup_decl (var, ctx: scan_simd_ctx); |
11045 | if (new_vard != new_var) |
11046 | { |
11047 | var5 = build_simple_mem_ref_loc (clause_loc, var5); |
11048 | var6 = build_simple_mem_ref_loc (clause_loc, var6); |
11049 | } |
11050 | } |
11051 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
11052 | { |
11053 | tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); |
11054 | tree val = var2; |
11055 | |
11056 | x = lang_hooks.decls.omp_clause_default_ctor |
11057 | (c, var2, build_outer_var_ref (var, ctx)); |
11058 | if (x) |
11059 | gimplify_and_add (x, &clist); |
11060 | |
11061 | x = build_outer_var_ref (var, ctx); |
11062 | x = lang_hooks.decls.omp_clause_assign_op (c, unshare_expr (var4), |
11063 | x); |
11064 | gimplify_and_add (x, &thr01_list); |
11065 | |
11066 | tree y = (DECL_HAS_VALUE_EXPR_P (new_vard) |
11067 | ? DECL_VALUE_EXPR (new_vard) : NULL_TREE); |
11068 | if (var3) |
11069 | { |
11070 | x = unshare_expr (var4); |
11071 | x = lang_hooks.decls.omp_clause_assign_op (c, x, var3); |
11072 | gimplify_and_add (x, &thrn1_list); |
11073 | x = unshare_expr (var4); |
11074 | x = lang_hooks.decls.omp_clause_assign_op (c, x, var3); |
11075 | gimplify_and_add (x, &thr02_list); |
11076 | } |
11077 | else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) |
11078 | { |
11079 | /* Otherwise, assign to it the identity element. */ |
11080 | gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); |
11081 | tseq = copy_gimple_seq_and_replace_locals (seq: tseq); |
11082 | if (!is_for_simd) |
11083 | { |
11084 | if (new_vard != new_var) |
11085 | val = build_fold_addr_expr_loc (clause_loc, val); |
11086 | SET_DECL_VALUE_EXPR (new_vard, val); |
11087 | DECL_HAS_VALUE_EXPR_P (new_vard) = 1; |
11088 | } |
11089 | SET_DECL_VALUE_EXPR (placeholder, error_mark_node); |
11090 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
11091 | lower_omp (&tseq, ctx); |
11092 | gimple_seq_add_seq (&thrn1_list, tseq); |
11093 | tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); |
11094 | lower_omp (&tseq, ctx); |
11095 | gimple_seq_add_seq (&thr02_list, tseq); |
11096 | SET_DECL_VALUE_EXPR (placeholder, NULL_TREE); |
11097 | DECL_HAS_VALUE_EXPR_P (placeholder) = 0; |
11098 | OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; |
11099 | if (y) |
11100 | SET_DECL_VALUE_EXPR (new_vard, y); |
11101 | else |
11102 | { |
11103 | DECL_HAS_VALUE_EXPR_P (new_vard) = 0; |
11104 | SET_DECL_VALUE_EXPR (new_vard, NULL_TREE); |
11105 | } |
11106 | } |
11107 | |
11108 | x = unshare_expr (var4); |
11109 | x = lang_hooks.decls.omp_clause_assign_op (c, x, rprivam1_ref); |
11110 | gimplify_and_add (x, &thrn2_list); |
11111 | |
11112 | if (is_for_simd) |
11113 | { |
11114 | x = unshare_expr (rprivb_ref); |
11115 | x = lang_hooks.decls.omp_clause_assign_op (c, x, var5); |
11116 | gimplify_and_add (x, &scan1_list); |
11117 | } |
11118 | else |
11119 | { |
11120 | if (ctx->scan_exclusive) |
11121 | { |
11122 | x = unshare_expr (rprivb_ref); |
11123 | x = lang_hooks.decls.omp_clause_assign_op (c, x, var2); |
11124 | gimplify_and_add (x, &scan1_list); |
11125 | } |
11126 | |
11127 | gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); |
11128 | tseq = copy_gimple_seq_and_replace_locals (seq: tseq); |
11129 | SET_DECL_VALUE_EXPR (placeholder, var2); |
11130 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
11131 | lower_omp (&tseq, ctx); |
11132 | gimple_seq_add_seq (&scan1_list, tseq); |
11133 | |
11134 | if (ctx->scan_inclusive) |
11135 | { |
11136 | x = unshare_expr (rprivb_ref); |
11137 | x = lang_hooks.decls.omp_clause_assign_op (c, x, var2); |
11138 | gimplify_and_add (x, &scan1_list); |
11139 | } |
11140 | } |
11141 | |
11142 | x = unshare_expr (rpriva_ref); |
11143 | x = lang_hooks.decls.omp_clause_assign_op (c, x, |
11144 | unshare_expr (var4)); |
11145 | gimplify_and_add (x, &mdlist); |
11146 | |
11147 | x = unshare_expr (is_for_simd ? var6 : new_var); |
11148 | x = lang_hooks.decls.omp_clause_assign_op (c, x, var4); |
11149 | gimplify_and_add (x, &input2_list); |
11150 | |
11151 | val = rprivb_ref; |
11152 | if (new_vard != new_var) |
11153 | val = build_fold_addr_expr_loc (clause_loc, val); |
11154 | |
11155 | gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); |
11156 | tseq = copy_gimple_seq_and_replace_locals (seq: tseq); |
11157 | SET_DECL_VALUE_EXPR (new_vard, val); |
11158 | DECL_HAS_VALUE_EXPR_P (new_vard) = 1; |
11159 | if (is_for_simd) |
11160 | { |
11161 | SET_DECL_VALUE_EXPR (placeholder, var6); |
11162 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
11163 | } |
11164 | else |
11165 | DECL_HAS_VALUE_EXPR_P (placeholder) = 0; |
11166 | lower_omp (&tseq, ctx); |
11167 | if (y) |
11168 | SET_DECL_VALUE_EXPR (new_vard, y); |
11169 | else |
11170 | { |
11171 | DECL_HAS_VALUE_EXPR_P (new_vard) = 0; |
11172 | SET_DECL_VALUE_EXPR (new_vard, NULL_TREE); |
11173 | } |
11174 | if (!is_for_simd) |
11175 | { |
11176 | SET_DECL_VALUE_EXPR (placeholder, new_var); |
11177 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
11178 | lower_omp (&tseq, ctx); |
11179 | } |
11180 | gimple_seq_add_seq (&input2_list, tseq); |
11181 | |
11182 | x = build_outer_var_ref (var, ctx); |
11183 | x = lang_hooks.decls.omp_clause_assign_op (c, x, rpriva_ref); |
11184 | gimplify_and_add (x, &last_list); |
11185 | |
11186 | x = lang_hooks.decls.omp_clause_assign_op (c, var2, rprivalmk_ref); |
11187 | gimplify_and_add (x, &reduc_list); |
11188 | tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); |
11189 | tseq = copy_gimple_seq_and_replace_locals (seq: tseq); |
11190 | val = rprival_ref; |
11191 | if (new_vard != new_var) |
11192 | val = build_fold_addr_expr_loc (clause_loc, val); |
11193 | SET_DECL_VALUE_EXPR (new_vard, val); |
11194 | DECL_HAS_VALUE_EXPR_P (new_vard) = 1; |
11195 | SET_DECL_VALUE_EXPR (placeholder, var2); |
11196 | lower_omp (&tseq, ctx); |
11197 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; |
11198 | SET_DECL_VALUE_EXPR (placeholder, NULL_TREE); |
11199 | DECL_HAS_VALUE_EXPR_P (placeholder) = 0; |
11200 | if (y) |
11201 | SET_DECL_VALUE_EXPR (new_vard, y); |
11202 | else |
11203 | { |
11204 | DECL_HAS_VALUE_EXPR_P (new_vard) = 0; |
11205 | SET_DECL_VALUE_EXPR (new_vard, NULL_TREE); |
11206 | } |
11207 | gimple_seq_add_seq (&reduc_list, tseq); |
11208 | x = lang_hooks.decls.omp_clause_assign_op (c, rprival_ref, var2); |
11209 | gimplify_and_add (x, &reduc_list); |
11210 | |
11211 | x = lang_hooks.decls.omp_clause_dtor (c, var2); |
11212 | if (x) |
11213 | gimplify_and_add (x, dlist); |
11214 | } |
11215 | else |
11216 | { |
11217 | x = build_outer_var_ref (var, ctx); |
11218 | gimplify_assign (unshare_expr (var4), x, &thr01_list); |
11219 | |
11220 | x = omp_reduction_init (clause: c, TREE_TYPE (new_var)); |
11221 | gimplify_assign (unshare_expr (var4), unshare_expr (x), |
11222 | &thrn1_list); |
11223 | gimplify_assign (unshare_expr (var4), x, &thr02_list); |
11224 | |
11225 | gimplify_assign (unshare_expr (var4), rprivam1_ref, &thrn2_list); |
11226 | |
11227 | enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c); |
11228 | if (code == MINUS_EXPR) |
11229 | code = PLUS_EXPR; |
11230 | |
11231 | if (is_for_simd) |
11232 | gimplify_assign (unshare_expr (rprivb_ref), var5, &scan1_list); |
11233 | else |
11234 | { |
11235 | if (ctx->scan_exclusive) |
11236 | gimplify_assign (unshare_expr (rprivb_ref), var2, |
11237 | &scan1_list); |
11238 | x = build2 (code, TREE_TYPE (new_var), var2, new_var); |
11239 | gimplify_assign (var2, x, &scan1_list); |
11240 | if (ctx->scan_inclusive) |
11241 | gimplify_assign (unshare_expr (rprivb_ref), var2, |
11242 | &scan1_list); |
11243 | } |
11244 | |
11245 | gimplify_assign (unshare_expr (rpriva_ref), unshare_expr (var4), |
11246 | &mdlist); |
11247 | |
11248 | x = build2 (code, TREE_TYPE (new_var), var4, rprivb_ref); |
11249 | gimplify_assign (is_for_simd ? var6 : new_var, x, &input2_list); |
11250 | |
11251 | gimplify_assign (build_outer_var_ref (var, ctx), rpriva_ref, |
11252 | &last_list); |
11253 | |
11254 | x = build2 (code, TREE_TYPE (new_var), rprivalmk_ref, |
11255 | unshare_expr (rprival_ref)); |
11256 | gimplify_assign (rprival_ref, x, &reduc_list); |
11257 | } |
11258 | } |
11259 | |
11260 | g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node); |
11261 | gimple_seq_add_stmt (&scan1_list, g); |
11262 | g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node); |
11263 | gimple_seq_add_stmt (gimple_omp_body_ptr (gs: is_for_simd |
11264 | ? scan_stmt4 : scan_stmt2), g); |
11265 | |
11266 | tree controlb = create_tmp_var (boolean_type_node); |
11267 | tree controlp = create_tmp_var (ptr_type_node); |
11268 | tree nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_); |
11269 | OMP_CLAUSE_DECL (nc) = controlb; |
11270 | OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1; |
11271 | *cp1 = nc; |
11272 | cp1 = &OMP_CLAUSE_CHAIN (nc); |
11273 | nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_); |
11274 | OMP_CLAUSE_DECL (nc) = controlp; |
11275 | OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1; |
11276 | *cp1 = nc; |
11277 | cp1 = &OMP_CLAUSE_CHAIN (nc); |
11278 | nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_); |
11279 | OMP_CLAUSE_DECL (nc) = controlb; |
11280 | OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1; |
11281 | *cp2 = nc; |
11282 | cp2 = &OMP_CLAUSE_CHAIN (nc); |
11283 | nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_); |
11284 | OMP_CLAUSE_DECL (nc) = controlp; |
11285 | OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1; |
11286 | *cp2 = nc; |
11287 | cp2 = &OMP_CLAUSE_CHAIN (nc); |
11288 | |
11289 | *cp1 = gimple_omp_for_clauses (gs: stmt); |
11290 | gimple_omp_for_set_clauses (gs: stmt, clauses: new_clauses1); |
11291 | *cp2 = gimple_omp_for_clauses (gs: new_stmt); |
11292 | gimple_omp_for_set_clauses (gs: new_stmt, clauses: new_clauses2); |
11293 | |
11294 | if (is_for_simd) |
11295 | { |
11296 | gimple_seq_add_seq (gimple_omp_body_ptr (gs: scan_stmt3), scan1_list); |
11297 | gimple_seq_add_seq (gimple_omp_body_ptr (gs: input_stmt4), input2_list); |
11298 | |
11299 | gsi_insert_seq_after (&input3_gsi, gimple_omp_body (gs: input_stmt3), |
11300 | GSI_SAME_STMT); |
11301 | gsi_remove (&input3_gsi, true); |
11302 | gsi_insert_seq_after (&scan3_gsi, gimple_omp_body (gs: scan_stmt3), |
11303 | GSI_SAME_STMT); |
11304 | gsi_remove (&scan3_gsi, true); |
11305 | gsi_insert_seq_after (&input4_gsi, gimple_omp_body (gs: input_stmt4), |
11306 | GSI_SAME_STMT); |
11307 | gsi_remove (&input4_gsi, true); |
11308 | gsi_insert_seq_after (&scan4_gsi, gimple_omp_body (gs: scan_stmt4), |
11309 | GSI_SAME_STMT); |
11310 | gsi_remove (&scan4_gsi, true); |
11311 | } |
11312 | else |
11313 | { |
11314 | gimple_omp_set_body (gs: scan_stmt1, body: scan1_list); |
11315 | gimple_omp_set_body (gs: input_stmt2, body: input2_list); |
11316 | } |
11317 | |
11318 | gsi_insert_seq_after (&input1_gsi, gimple_omp_body (gs: input_stmt1), |
11319 | GSI_SAME_STMT); |
11320 | gsi_remove (&input1_gsi, true); |
11321 | gsi_insert_seq_after (&scan1_gsi, gimple_omp_body (gs: scan_stmt1), |
11322 | GSI_SAME_STMT); |
11323 | gsi_remove (&scan1_gsi, true); |
11324 | gsi_insert_seq_after (&input2_gsi, gimple_omp_body (gs: input_stmt2), |
11325 | GSI_SAME_STMT); |
11326 | gsi_remove (&input2_gsi, true); |
11327 | gsi_insert_seq_after (&scan2_gsi, gimple_omp_body (gs: scan_stmt2), |
11328 | GSI_SAME_STMT); |
11329 | gsi_remove (&scan2_gsi, true); |
11330 | |
11331 | gimple_seq_add_seq (body_p, clist); |
11332 | |
11333 | tree lab1 = create_artificial_label (UNKNOWN_LOCATION); |
11334 | tree lab2 = create_artificial_label (UNKNOWN_LOCATION); |
11335 | tree lab3 = create_artificial_label (UNKNOWN_LOCATION); |
11336 | g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2); |
11337 | gimple_seq_add_stmt (body_p, g); |
11338 | g = gimple_build_label (label: lab1); |
11339 | gimple_seq_add_stmt (body_p, g); |
11340 | gimple_seq_add_seq (body_p, thr01_list); |
11341 | g = gimple_build_goto (dest: lab3); |
11342 | gimple_seq_add_stmt (body_p, g); |
11343 | g = gimple_build_label (label: lab2); |
11344 | gimple_seq_add_stmt (body_p, g); |
11345 | gimple_seq_add_seq (body_p, thrn1_list); |
11346 | g = gimple_build_label (label: lab3); |
11347 | gimple_seq_add_stmt (body_p, g); |
11348 | |
11349 | g = gimple_build_assign (ivar, size_zero_node); |
11350 | gimple_seq_add_stmt (body_p, g); |
11351 | |
11352 | gimple_seq_add_stmt (body_p, stmt); |
11353 | gimple_seq_add_seq (body_p, body); |
11354 | gimple_seq_add_stmt (body_p, gimple_build_omp_continue (fd->loop.v, |
11355 | fd->loop.v)); |
11356 | |
11357 | g = gimple_build_omp_return (true); |
11358 | gimple_seq_add_stmt (body_p, g); |
11359 | gimple_seq_add_seq (body_p, mdlist); |
11360 | |
11361 | lab1 = create_artificial_label (UNKNOWN_LOCATION); |
11362 | lab2 = create_artificial_label (UNKNOWN_LOCATION); |
11363 | g = gimple_build_cond (GT_EXPR, num_threads, integer_one_node, lab1, lab2); |
11364 | gimple_seq_add_stmt (body_p, g); |
11365 | g = gimple_build_label (label: lab1); |
11366 | gimple_seq_add_stmt (body_p, g); |
11367 | |
11368 | g = omp_build_barrier (NULL); |
11369 | gimple_seq_add_stmt (body_p, g); |
11370 | |
11371 | tree down = create_tmp_var (unsigned_type_node); |
11372 | g = gimple_build_assign (down, build_zero_cst (unsigned_type_node)); |
11373 | gimple_seq_add_stmt (body_p, g); |
11374 | |
11375 | g = gimple_build_assign (k, build_one_cst (unsigned_type_node)); |
11376 | gimple_seq_add_stmt (body_p, g); |
11377 | |
11378 | tree num_threadsu = create_tmp_var (unsigned_type_node); |
11379 | g = gimple_build_assign (num_threadsu, NOP_EXPR, num_threads); |
11380 | gimple_seq_add_stmt (body_p, g); |
11381 | |
11382 | tree thread_numu = create_tmp_var (unsigned_type_node); |
11383 | g = gimple_build_assign (thread_numu, NOP_EXPR, thread_num); |
11384 | gimple_seq_add_stmt (body_p, g); |
11385 | |
11386 | tree thread_nump1 = create_tmp_var (unsigned_type_node); |
11387 | g = gimple_build_assign (thread_nump1, PLUS_EXPR, thread_numu, |
11388 | build_int_cst (unsigned_type_node, 1)); |
11389 | gimple_seq_add_stmt (body_p, g); |
11390 | |
11391 | lab3 = create_artificial_label (UNKNOWN_LOCATION); |
11392 | g = gimple_build_label (label: lab3); |
11393 | gimple_seq_add_stmt (body_p, g); |
11394 | |
11395 | tree twok = create_tmp_var (unsigned_type_node); |
11396 | g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node); |
11397 | gimple_seq_add_stmt (body_p, g); |
11398 | |
11399 | tree lab4 = create_artificial_label (UNKNOWN_LOCATION); |
11400 | tree lab5 = create_artificial_label (UNKNOWN_LOCATION); |
11401 | tree lab6 = create_artificial_label (UNKNOWN_LOCATION); |
11402 | g = gimple_build_cond (GT_EXPR, twok, num_threadsu, lab4, lab5); |
11403 | gimple_seq_add_stmt (body_p, g); |
11404 | g = gimple_build_label (label: lab4); |
11405 | gimple_seq_add_stmt (body_p, g); |
11406 | g = gimple_build_assign (down, build_all_ones_cst (unsigned_type_node)); |
11407 | gimple_seq_add_stmt (body_p, g); |
11408 | g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node); |
11409 | gimple_seq_add_stmt (body_p, g); |
11410 | |
11411 | g = gimple_build_cond (EQ_EXPR, k, num_threadsu, lab6, lab5); |
11412 | gimple_seq_add_stmt (body_p, g); |
11413 | g = gimple_build_label (label: lab6); |
11414 | gimple_seq_add_stmt (body_p, g); |
11415 | |
11416 | g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node); |
11417 | gimple_seq_add_stmt (body_p, g); |
11418 | |
11419 | g = gimple_build_label (label: lab5); |
11420 | gimple_seq_add_stmt (body_p, g); |
11421 | |
11422 | g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node); |
11423 | gimple_seq_add_stmt (body_p, g); |
11424 | |
11425 | tree cplx = create_tmp_var (build_complex_type (unsigned_type_node, named: false)); |
11426 | g = gimple_build_call_internal (IFN_MUL_OVERFLOW, 2, thread_nump1, twok); |
11427 | gimple_call_set_lhs (gs: g, lhs: cplx); |
11428 | gimple_seq_add_stmt (body_p, g); |
11429 | tree mul = create_tmp_var (unsigned_type_node); |
11430 | g = gimple_build_assign (mul, REALPART_EXPR, |
11431 | build1 (REALPART_EXPR, unsigned_type_node, cplx)); |
11432 | gimple_seq_add_stmt (body_p, g); |
11433 | tree ovf = create_tmp_var (unsigned_type_node); |
11434 | g = gimple_build_assign (ovf, IMAGPART_EXPR, |
11435 | build1 (IMAGPART_EXPR, unsigned_type_node, cplx)); |
11436 | gimple_seq_add_stmt (body_p, g); |
11437 | |
11438 | tree lab7 = create_artificial_label (UNKNOWN_LOCATION); |
11439 | tree lab8 = create_artificial_label (UNKNOWN_LOCATION); |
11440 | g = gimple_build_cond (EQ_EXPR, ovf, build_zero_cst (unsigned_type_node), |
11441 | lab7, lab8); |
11442 | gimple_seq_add_stmt (body_p, g); |
11443 | g = gimple_build_label (label: lab7); |
11444 | gimple_seq_add_stmt (body_p, g); |
11445 | |
11446 | tree andv = create_tmp_var (unsigned_type_node); |
11447 | g = gimple_build_assign (andv, BIT_AND_EXPR, k, down); |
11448 | gimple_seq_add_stmt (body_p, g); |
11449 | tree andvm1 = create_tmp_var (unsigned_type_node); |
11450 | g = gimple_build_assign (andvm1, PLUS_EXPR, andv, |
11451 | build_minus_one_cst (unsigned_type_node)); |
11452 | gimple_seq_add_stmt (body_p, g); |
11453 | |
11454 | g = gimple_build_assign (l, PLUS_EXPR, mul, andvm1); |
11455 | gimple_seq_add_stmt (body_p, g); |
11456 | |
11457 | tree lab9 = create_artificial_label (UNKNOWN_LOCATION); |
11458 | g = gimple_build_cond (LT_EXPR, l, num_threadsu, lab9, lab8); |
11459 | gimple_seq_add_stmt (body_p, g); |
11460 | g = gimple_build_label (label: lab9); |
11461 | gimple_seq_add_stmt (body_p, g); |
11462 | gimple_seq_add_seq (body_p, reduc_list); |
11463 | g = gimple_build_label (label: lab8); |
11464 | gimple_seq_add_stmt (body_p, g); |
11465 | |
11466 | tree lab10 = create_artificial_label (UNKNOWN_LOCATION); |
11467 | tree lab11 = create_artificial_label (UNKNOWN_LOCATION); |
11468 | tree lab12 = create_artificial_label (UNKNOWN_LOCATION); |
11469 | g = gimple_build_cond (EQ_EXPR, down, build_zero_cst (unsigned_type_node), |
11470 | lab10, lab11); |
11471 | gimple_seq_add_stmt (body_p, g); |
11472 | g = gimple_build_label (label: lab10); |
11473 | gimple_seq_add_stmt (body_p, g); |
11474 | g = gimple_build_assign (k, LSHIFT_EXPR, k, integer_one_node); |
11475 | gimple_seq_add_stmt (body_p, g); |
11476 | g = gimple_build_goto (dest: lab12); |
11477 | gimple_seq_add_stmt (body_p, g); |
11478 | g = gimple_build_label (label: lab11); |
11479 | gimple_seq_add_stmt (body_p, g); |
11480 | g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node); |
11481 | gimple_seq_add_stmt (body_p, g); |
11482 | g = gimple_build_label (label: lab12); |
11483 | gimple_seq_add_stmt (body_p, g); |
11484 | |
11485 | g = omp_build_barrier (NULL); |
11486 | gimple_seq_add_stmt (body_p, g); |
11487 | |
11488 | g = gimple_build_cond (NE_EXPR, k, build_zero_cst (unsigned_type_node), |
11489 | lab3, lab2); |
11490 | gimple_seq_add_stmt (body_p, g); |
11491 | |
11492 | g = gimple_build_label (label: lab2); |
11493 | gimple_seq_add_stmt (body_p, g); |
11494 | |
11495 | lab1 = create_artificial_label (UNKNOWN_LOCATION); |
11496 | lab2 = create_artificial_label (UNKNOWN_LOCATION); |
11497 | lab3 = create_artificial_label (UNKNOWN_LOCATION); |
11498 | g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2); |
11499 | gimple_seq_add_stmt (body_p, g); |
11500 | g = gimple_build_label (label: lab1); |
11501 | gimple_seq_add_stmt (body_p, g); |
11502 | gimple_seq_add_seq (body_p, thr02_list); |
11503 | g = gimple_build_goto (dest: lab3); |
11504 | gimple_seq_add_stmt (body_p, g); |
11505 | g = gimple_build_label (label: lab2); |
11506 | gimple_seq_add_stmt (body_p, g); |
11507 | gimple_seq_add_seq (body_p, thrn2_list); |
11508 | g = gimple_build_label (label: lab3); |
11509 | gimple_seq_add_stmt (body_p, g); |
11510 | |
11511 | g = gimple_build_assign (ivar, size_zero_node); |
11512 | gimple_seq_add_stmt (body_p, g); |
11513 | gimple_seq_add_stmt (body_p, new_stmt); |
11514 | gimple_seq_add_seq (body_p, new_body); |
11515 | |
11516 | gimple_seq new_dlist = NULL; |
11517 | lab1 = create_artificial_label (UNKNOWN_LOCATION); |
11518 | lab2 = create_artificial_label (UNKNOWN_LOCATION); |
11519 | tree num_threadsm1 = create_tmp_var (integer_type_node); |
11520 | g = gimple_build_assign (num_threadsm1, PLUS_EXPR, num_threads, |
11521 | integer_minus_one_node); |
11522 | gimple_seq_add_stmt (&new_dlist, g); |
11523 | g = gimple_build_cond (EQ_EXPR, thread_num, num_threadsm1, lab1, lab2); |
11524 | gimple_seq_add_stmt (&new_dlist, g); |
11525 | g = gimple_build_label (label: lab1); |
11526 | gimple_seq_add_stmt (&new_dlist, g); |
11527 | gimple_seq_add_seq (&new_dlist, last_list); |
11528 | g = gimple_build_label (label: lab2); |
11529 | gimple_seq_add_stmt (&new_dlist, g); |
11530 | gimple_seq_add_seq (&new_dlist, *dlist); |
11531 | *dlist = new_dlist; |
11532 | } |
11533 | |
11534 | /* Build an internal UNIQUE function with type IFN_UNIQUE_OACC_PRIVATE listing |
11535 | the addresses of variables to be made private at the surrounding |
11536 | parallelism level. Such functions appear in the gimple code stream in two |
11537 | forms, e.g. for a partitioned loop: |
11538 | |
11539 | .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6, 1, 68); |
11540 | .data_dep.6 = .UNIQUE (OACC_PRIVATE, .data_dep.6, -1, &w); |
11541 | .data_dep.6 = .UNIQUE (OACC_FORK, .data_dep.6, -1); |
11542 | .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6); |
11543 | |
11544 | or alternatively, OACC_PRIVATE can appear at the top level of a parallel, |
11545 | not as part of a HEAD_MARK sequence: |
11546 | |
11547 | .UNIQUE (OACC_PRIVATE, 0, 0, &w); |
11548 | |
11549 | For such stand-alone appearances, the 3rd argument is always 0, denoting |
11550 | gang partitioning. */ |
11551 | |
11552 | static gcall * |
11553 | lower_oacc_private_marker (omp_context *ctx) |
11554 | { |
11555 | if (ctx->oacc_privatization_candidates.length () == 0) |
11556 | return NULL; |
11557 | |
11558 | auto_vec<tree, 5> args; |
11559 | |
11560 | args.quick_push (obj: build_int_cst (integer_type_node, IFN_UNIQUE_OACC_PRIVATE)); |
11561 | args.quick_push (integer_zero_node); |
11562 | args.quick_push (integer_minus_one_node); |
11563 | |
11564 | int i; |
11565 | tree decl; |
11566 | FOR_EACH_VEC_ELT (ctx->oacc_privatization_candidates, i, decl) |
11567 | { |
11568 | gcc_checking_assert (TREE_ADDRESSABLE (decl)); |
11569 | tree addr = build_fold_addr_expr (decl); |
11570 | args.safe_push (obj: addr); |
11571 | } |
11572 | |
11573 | return gimple_build_call_internal_vec (IFN_UNIQUE, args); |
11574 | } |
11575 | |
11576 | /* Lower code for an OMP loop directive. */ |
11577 | |
11578 | static void |
11579 | lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
11580 | { |
11581 | tree *rhs_p, block; |
11582 | struct omp_for_data fd, *fdp = NULL; |
11583 | gomp_for *stmt = as_a <gomp_for *> (p: gsi_stmt (i: *gsi_p)); |
11584 | gbind *new_stmt; |
11585 | gimple_seq omp_for_body, body, dlist, tred_ilist = NULL, tred_dlist = NULL; |
11586 | gimple_seq cnt_list = NULL, clist = NULL; |
11587 | gimple_seq oacc_head = NULL, oacc_tail = NULL; |
11588 | size_t i; |
11589 | |
11590 | push_gimplify_context (); |
11591 | |
11592 | if (is_gimple_omp_oacc (stmt: ctx->stmt)) |
11593 | oacc_privatization_scan_clause_chain (ctx, clauses: gimple_omp_for_clauses (gs: stmt)); |
11594 | |
11595 | lower_omp (gimple_omp_for_pre_body_ptr (gs: stmt), ctx); |
11596 | |
11597 | block = make_node (BLOCK); |
11598 | new_stmt = gimple_build_bind (NULL, NULL, block); |
11599 | /* Replace at gsi right away, so that 'stmt' is no member |
11600 | of a sequence anymore as we're going to add to a different |
11601 | one below. */ |
11602 | gsi_replace (gsi_p, new_stmt, true); |
11603 | |
11604 | /* Move declaration of temporaries in the loop body before we make |
11605 | it go away. */ |
11606 | omp_for_body = gimple_omp_body (gs: stmt); |
11607 | if (!gimple_seq_empty_p (s: omp_for_body) |
11608 | && gimple_code (g: gimple_seq_first_stmt (s: omp_for_body)) == GIMPLE_BIND) |
11609 | { |
11610 | gbind *inner_bind |
11611 | = as_a <gbind *> (p: gimple_seq_first_stmt (s: omp_for_body)); |
11612 | tree vars = gimple_bind_vars (bind_stmt: inner_bind); |
11613 | if (is_gimple_omp_oacc (stmt: ctx->stmt)) |
11614 | oacc_privatization_scan_decl_chain (ctx, decls: vars); |
11615 | gimple_bind_append_vars (bind_stmt: new_stmt, vars); |
11616 | /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't |
11617 | keep them on the inner_bind and it's block. */ |
11618 | gimple_bind_set_vars (bind_stmt: inner_bind, NULL_TREE); |
11619 | if (gimple_bind_block (bind_stmt: inner_bind)) |
11620 | BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE; |
11621 | } |
11622 | |
11623 | if (gimple_omp_for_combined_into_p (g: stmt)) |
11624 | { |
11625 | omp_extract_for_data (for_stmt: stmt, fd: &fd, NULL); |
11626 | fdp = &fd; |
11627 | |
11628 | /* We need two temporaries with fd.loop.v type (istart/iend) |
11629 | and then (fd.collapse - 1) temporaries with the same |
11630 | type for count2 ... countN-1 vars if not constant. */ |
11631 | size_t count = 2; |
11632 | tree type = fd.iter_type; |
11633 | if (fd.collapse > 1 |
11634 | && TREE_CODE (fd.loop.n2) != INTEGER_CST) |
11635 | count += fd.collapse - 1; |
11636 | size_t count2 = 0; |
11637 | tree type2 = NULL_TREE; |
11638 | bool taskreg_for |
11639 | = (gimple_omp_for_kind (g: stmt) == GF_OMP_FOR_KIND_FOR |
11640 | || gimple_omp_for_kind (g: stmt) == GF_OMP_FOR_KIND_TASKLOOP); |
11641 | tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (gs: stmt); |
11642 | tree simtc = NULL; |
11643 | tree clauses = *pc; |
11644 | if (fd.collapse > 1 |
11645 | && fd.non_rect |
11646 | && fd.last_nonrect == fd.first_nonrect + 1 |
11647 | && TREE_CODE (fd.loop.n2) != INTEGER_CST) |
11648 | if (tree v = gimple_omp_for_index (gs: stmt, i: fd.last_nonrect)) |
11649 | if (!TYPE_UNSIGNED (TREE_TYPE (v))) |
11650 | { |
11651 | v = gimple_omp_for_index (gs: stmt, i: fd.first_nonrect); |
11652 | type2 = TREE_TYPE (v); |
11653 | count++; |
11654 | count2 = 3; |
11655 | } |
11656 | if (taskreg_for) |
11657 | outerc |
11658 | = omp_find_clause (clauses: gimple_omp_taskreg_clauses (gs: ctx->outer->stmt), |
11659 | kind: OMP_CLAUSE__LOOPTEMP_); |
11660 | if (ctx->simt_stmt) |
11661 | simtc = omp_find_clause (clauses: gimple_omp_for_clauses (gs: ctx->simt_stmt), |
11662 | kind: OMP_CLAUSE__LOOPTEMP_); |
11663 | for (i = 0; i < count + count2; i++) |
11664 | { |
11665 | tree temp; |
11666 | if (taskreg_for) |
11667 | { |
11668 | gcc_assert (outerc); |
11669 | temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx: ctx->outer); |
11670 | outerc = omp_find_clause (OMP_CLAUSE_CHAIN (outerc), |
11671 | kind: OMP_CLAUSE__LOOPTEMP_); |
11672 | } |
11673 | else |
11674 | { |
11675 | /* If there are 2 adjacent SIMD stmts, one with _simt_ |
11676 | clause, another without, make sure they have the same |
11677 | decls in _looptemp_ clauses, because the outer stmt |
11678 | they are combined into will look up just one inner_stmt. */ |
11679 | if (ctx->simt_stmt) |
11680 | temp = OMP_CLAUSE_DECL (simtc); |
11681 | else |
11682 | temp = create_tmp_var (i >= count ? type2 : type); |
11683 | insert_decl_map (&ctx->outer->cb, temp, temp); |
11684 | } |
11685 | *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_); |
11686 | OMP_CLAUSE_DECL (*pc) = temp; |
11687 | pc = &OMP_CLAUSE_CHAIN (*pc); |
11688 | if (ctx->simt_stmt) |
11689 | simtc = omp_find_clause (OMP_CLAUSE_CHAIN (simtc), |
11690 | kind: OMP_CLAUSE__LOOPTEMP_); |
11691 | } |
11692 | *pc = clauses; |
11693 | } |
11694 | |
11695 | /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */ |
11696 | dlist = NULL; |
11697 | body = NULL; |
11698 | tree rclauses |
11699 | = omp_task_reductions_find_first (clauses: gimple_omp_for_clauses (gs: stmt), code: OMP_FOR, |
11700 | ccode: OMP_CLAUSE_REDUCTION); |
11701 | tree rtmp = NULL_TREE; |
11702 | if (rclauses) |
11703 | { |
11704 | tree type = build_pointer_type (pointer_sized_int_node); |
11705 | tree temp = create_tmp_var (type); |
11706 | tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_); |
11707 | OMP_CLAUSE_DECL (c) = temp; |
11708 | OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (gs: stmt); |
11709 | gimple_omp_for_set_clauses (gs: stmt, clauses: c); |
11710 | lower_omp_task_reductions (ctx, code: OMP_FOR, |
11711 | clauses: gimple_omp_for_clauses (gs: stmt), |
11712 | start: &tred_ilist, end: &tred_dlist); |
11713 | rclauses = c; |
11714 | rtmp = make_ssa_name (var: type); |
11715 | gimple_seq_add_stmt (&body, gimple_build_assign (rtmp, temp)); |
11716 | } |
11717 | |
11718 | lower_lastprivate_conditional_clauses (clauses: gimple_omp_for_clauses_ptr (gs: stmt), |
11719 | ctx); |
11720 | |
11721 | lower_rec_input_clauses (clauses: gimple_omp_for_clauses (gs: stmt), ilist: &body, dlist: &dlist, ctx, |
11722 | fd: fdp); |
11723 | gimple_seq_add_seq (rclauses ? &tred_ilist : &body, |
11724 | gimple_omp_for_pre_body (gs: stmt)); |
11725 | |
11726 | lower_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
11727 | |
11728 | gcall *private_marker = NULL; |
11729 | if (is_gimple_omp_oacc (stmt: ctx->stmt) |
11730 | && !gimple_seq_empty_p (s: omp_for_body)) |
11731 | private_marker = lower_oacc_private_marker (ctx); |
11732 | |
11733 | /* Lower the header expressions. At this point, we can assume that |
11734 | the header is of the form: |
11735 | |
11736 | #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3) |
11737 | |
11738 | We just need to make sure that VAL1, VAL2 and VAL3 are lowered |
11739 | using the .omp_data_s mapping, if needed. */ |
11740 | for (i = 0; i < gimple_omp_for_collapse (gs: stmt); i++) |
11741 | { |
11742 | rhs_p = gimple_omp_for_initial_ptr (gs: stmt, i); |
11743 | if (TREE_CODE (*rhs_p) == TREE_VEC) |
11744 | { |
11745 | if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1))) |
11746 | TREE_VEC_ELT (*rhs_p, 1) |
11747 | = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list); |
11748 | if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2))) |
11749 | TREE_VEC_ELT (*rhs_p, 2) |
11750 | = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list); |
11751 | } |
11752 | else if (!is_gimple_min_invariant (*rhs_p)) |
11753 | *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list); |
11754 | else if (TREE_CODE (*rhs_p) == ADDR_EXPR) |
11755 | recompute_tree_invariant_for_addr_expr (*rhs_p); |
11756 | |
11757 | rhs_p = gimple_omp_for_final_ptr (gs: stmt, i); |
11758 | if (TREE_CODE (*rhs_p) == TREE_VEC) |
11759 | { |
11760 | if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1))) |
11761 | TREE_VEC_ELT (*rhs_p, 1) |
11762 | = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list); |
11763 | if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2))) |
11764 | TREE_VEC_ELT (*rhs_p, 2) |
11765 | = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list); |
11766 | } |
11767 | else if (!is_gimple_min_invariant (*rhs_p)) |
11768 | *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list); |
11769 | else if (TREE_CODE (*rhs_p) == ADDR_EXPR) |
11770 | recompute_tree_invariant_for_addr_expr (*rhs_p); |
11771 | |
11772 | rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1); |
11773 | if (!is_gimple_min_invariant (*rhs_p)) |
11774 | *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list); |
11775 | } |
11776 | if (rclauses) |
11777 | gimple_seq_add_seq (&tred_ilist, cnt_list); |
11778 | else |
11779 | gimple_seq_add_seq (&body, cnt_list); |
11780 | |
11781 | /* Once lowered, extract the bounds and clauses. */ |
11782 | omp_extract_for_data (for_stmt: stmt, fd: &fd, NULL); |
11783 | |
11784 | if (is_gimple_omp_oacc (stmt: ctx->stmt) |
11785 | && !ctx_in_oacc_kernels_region (ctx)) |
11786 | lower_oacc_head_tail (loc: gimple_location (g: stmt), |
11787 | clauses: gimple_omp_for_clauses (gs: stmt), private_marker, |
11788 | head: &oacc_head, tail: &oacc_tail, ctx); |
11789 | |
11790 | /* Add OpenACC partitioning and reduction markers just before the loop. */ |
11791 | if (oacc_head) |
11792 | gimple_seq_add_seq (&body, oacc_head); |
11793 | |
11794 | lower_omp_for_lastprivate (fd: &fd, body_p: &body, dlist: &dlist, clist: &clist, ctx); |
11795 | |
11796 | if (gimple_omp_for_kind (g: stmt) == GF_OMP_FOR_KIND_FOR) |
11797 | for (tree c = gimple_omp_for_clauses (gs: stmt); c; c = OMP_CLAUSE_CHAIN (c)) |
11798 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR |
11799 | && !OMP_CLAUSE_LINEAR_NO_COPYIN (c)) |
11800 | { |
11801 | OMP_CLAUSE_DECL (c) = lookup_decl (OMP_CLAUSE_DECL (c), ctx); |
11802 | if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c))) |
11803 | OMP_CLAUSE_LINEAR_STEP (c) |
11804 | = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c), |
11805 | ctx); |
11806 | } |
11807 | |
11808 | if ((ctx->scan_inclusive || ctx->scan_exclusive) |
11809 | && gimple_omp_for_kind (g: stmt) == GF_OMP_FOR_KIND_FOR) |
11810 | lower_omp_for_scan (body_p: &body, dlist: &dlist, stmt, fd: &fd, ctx); |
11811 | else |
11812 | { |
11813 | gimple_seq_add_stmt (&body, stmt); |
11814 | gimple_seq_add_seq (&body, gimple_omp_body (gs: stmt)); |
11815 | } |
11816 | |
11817 | gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v, |
11818 | fd.loop.v)); |
11819 | |
11820 | /* After the loop, add exit clauses. */ |
11821 | lower_reduction_clauses (clauses: gimple_omp_for_clauses (gs: stmt), stmt_seqp: &body, clist: &clist, ctx); |
11822 | |
11823 | if (clist) |
11824 | { |
11825 | tree fndecl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_ATOMIC_START); |
11826 | gcall *g = gimple_build_call (fndecl, 0); |
11827 | gimple_seq_add_stmt (&body, g); |
11828 | gimple_seq_add_seq (&body, clist); |
11829 | fndecl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_ATOMIC_END); |
11830 | g = gimple_build_call (fndecl, 0); |
11831 | gimple_seq_add_stmt (&body, g); |
11832 | } |
11833 | |
11834 | if (ctx->cancellable) |
11835 | gimple_seq_add_stmt (&body, gimple_build_label (label: ctx->cancel_label)); |
11836 | |
11837 | gimple_seq_add_seq (&body, dlist); |
11838 | |
11839 | if (rclauses) |
11840 | { |
11841 | gimple_seq_add_seq (&tred_ilist, body); |
11842 | body = tred_ilist; |
11843 | } |
11844 | |
11845 | body = maybe_catch_exception (body); |
11846 | |
11847 | /* Region exit marker goes at the end of the loop body. */ |
11848 | gimple *g = gimple_build_omp_return (fd.have_nowait); |
11849 | gimple_seq_add_stmt (&body, g); |
11850 | |
11851 | gimple_seq_add_seq (&body, tred_dlist); |
11852 | |
11853 | maybe_add_implicit_barrier_cancel (ctx, omp_return: g, body: &body); |
11854 | |
11855 | if (rclauses) |
11856 | OMP_CLAUSE_DECL (rclauses) = rtmp; |
11857 | |
11858 | /* Add OpenACC joining and reduction markers just after the loop. */ |
11859 | if (oacc_tail) |
11860 | gimple_seq_add_seq (&body, oacc_tail); |
11861 | |
11862 | pop_gimplify_context (new_stmt); |
11863 | |
11864 | gimple_bind_append_vars (bind_stmt: new_stmt, vars: ctx->block_vars); |
11865 | maybe_remove_omp_member_access_dummy_vars (bind: new_stmt); |
11866 | BLOCK_VARS (block) = gimple_bind_vars (bind_stmt: new_stmt); |
11867 | if (BLOCK_VARS (block)) |
11868 | TREE_USED (block) = 1; |
11869 | |
11870 | gimple_bind_set_body (bind_stmt: new_stmt, seq: body); |
11871 | gimple_omp_set_body (gs: stmt, NULL); |
11872 | gimple_omp_for_set_pre_body (gs: stmt, NULL); |
11873 | } |
11874 | |
11875 | /* Callback for walk_stmts. Check if the current statement only contains |
11876 | GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */ |
11877 | |
11878 | static tree |
11879 | check_combined_parallel (gimple_stmt_iterator *gsi_p, |
11880 | bool *handled_ops_p, |
11881 | struct walk_stmt_info *wi) |
11882 | { |
11883 | int *info = (int *) wi->info; |
11884 | gimple *stmt = gsi_stmt (i: *gsi_p); |
11885 | |
11886 | *handled_ops_p = true; |
11887 | switch (gimple_code (g: stmt)) |
11888 | { |
11889 | WALK_SUBSTMTS; |
11890 | |
11891 | case GIMPLE_DEBUG: |
11892 | break; |
11893 | case GIMPLE_OMP_FOR: |
11894 | case GIMPLE_OMP_SECTIONS: |
11895 | *info = *info == 0 ? 1 : -1; |
11896 | break; |
11897 | default: |
11898 | *info = -1; |
11899 | break; |
11900 | } |
11901 | return NULL; |
11902 | } |
11903 | |
11904 | struct omp_taskcopy_context |
11905 | { |
11906 | /* This field must be at the beginning, as we do "inheritance": Some |
11907 | callback functions for tree-inline.cc (e.g., omp_copy_decl) |
11908 | receive a copy_body_data pointer that is up-casted to an |
11909 | omp_context pointer. */ |
11910 | copy_body_data cb; |
11911 | omp_context *ctx; |
11912 | }; |
11913 | |
11914 | static tree |
11915 | task_copyfn_copy_decl (tree var, copy_body_data *cb) |
11916 | { |
11917 | struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb; |
11918 | |
11919 | if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var)) |
11920 | return create_tmp_var (TREE_TYPE (var)); |
11921 | |
11922 | return var; |
11923 | } |
11924 | |
11925 | static tree |
11926 | task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type) |
11927 | { |
11928 | tree name, new_fields = NULL, type, f; |
11929 | |
11930 | type = lang_hooks.types.make_type (RECORD_TYPE); |
11931 | name = DECL_NAME (TYPE_NAME (orig_type)); |
11932 | name = build_decl (gimple_location (g: tcctx->ctx->stmt), |
11933 | TYPE_DECL, name, type); |
11934 | TYPE_NAME (type) = name; |
11935 | |
11936 | for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f)) |
11937 | { |
11938 | tree new_f = copy_node (f); |
11939 | DECL_CONTEXT (new_f) = type; |
11940 | TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), id: &tcctx->cb); |
11941 | TREE_CHAIN (new_f) = new_fields; |
11942 | walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL); |
11943 | walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL); |
11944 | walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r, |
11945 | &tcctx->cb, NULL); |
11946 | new_fields = new_f; |
11947 | tcctx->cb.decl_map->put (k: f, v: new_f); |
11948 | } |
11949 | TYPE_FIELDS (type) = nreverse (new_fields); |
11950 | layout_type (type); |
11951 | return type; |
11952 | } |
11953 | |
11954 | /* Create task copyfn. */ |
11955 | |
11956 | static void |
11957 | create_task_copyfn (gomp_task *task_stmt, omp_context *ctx) |
11958 | { |
11959 | struct function *child_cfun; |
11960 | tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl; |
11961 | tree record_type, srecord_type, bind, list; |
11962 | bool record_needs_remap = false, srecord_needs_remap = false; |
11963 | splay_tree_node n; |
11964 | struct omp_taskcopy_context tcctx; |
11965 | location_t loc = gimple_location (g: task_stmt); |
11966 | size_t looptempno = 0; |
11967 | |
11968 | child_fn = gimple_omp_task_copy_fn (gs: task_stmt); |
11969 | task_cpyfns.safe_push (obj: task_stmt); |
11970 | child_cfun = DECL_STRUCT_FUNCTION (child_fn); |
11971 | gcc_assert (child_cfun->cfg == NULL); |
11972 | DECL_SAVED_TREE (child_fn) = alloc_stmt_list (); |
11973 | |
11974 | /* Reset DECL_CONTEXT on function arguments. */ |
11975 | for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t)) |
11976 | DECL_CONTEXT (t) = child_fn; |
11977 | |
11978 | /* Populate the function. */ |
11979 | push_gimplify_context (); |
11980 | push_cfun (new_cfun: child_cfun); |
11981 | |
11982 | bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL); |
11983 | TREE_SIDE_EFFECTS (bind) = 1; |
11984 | list = NULL; |
11985 | DECL_SAVED_TREE (child_fn) = bind; |
11986 | DECL_SOURCE_LOCATION (child_fn) = gimple_location (g: task_stmt); |
11987 | |
11988 | /* Remap src and dst argument types if needed. */ |
11989 | record_type = ctx->record_type; |
11990 | srecord_type = ctx->srecord_type; |
11991 | for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f)) |
11992 | if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn)) |
11993 | { |
11994 | record_needs_remap = true; |
11995 | break; |
11996 | } |
11997 | for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f)) |
11998 | if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn)) |
11999 | { |
12000 | srecord_needs_remap = true; |
12001 | break; |
12002 | } |
12003 | |
12004 | if (record_needs_remap || srecord_needs_remap) |
12005 | { |
12006 | memset (s: &tcctx, c: '\0', n: sizeof (tcctx)); |
12007 | tcctx.cb.src_fn = ctx->cb.src_fn; |
12008 | tcctx.cb.dst_fn = child_fn; |
12009 | tcctx.cb.src_node = cgraph_node::get (decl: tcctx.cb.src_fn); |
12010 | gcc_checking_assert (tcctx.cb.src_node); |
12011 | tcctx.cb.dst_node = tcctx.cb.src_node; |
12012 | tcctx.cb.src_cfun = ctx->cb.src_cfun; |
12013 | tcctx.cb.copy_decl = task_copyfn_copy_decl; |
12014 | tcctx.cb.eh_lp_nr = 0; |
12015 | tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE; |
12016 | tcctx.cb.decl_map = new hash_map<tree, tree>; |
12017 | tcctx.ctx = ctx; |
12018 | |
12019 | if (record_needs_remap) |
12020 | record_type = task_copyfn_remap_type (tcctx: &tcctx, orig_type: record_type); |
12021 | if (srecord_needs_remap) |
12022 | srecord_type = task_copyfn_remap_type (tcctx: &tcctx, orig_type: srecord_type); |
12023 | } |
12024 | else |
12025 | tcctx.cb.decl_map = NULL; |
12026 | |
12027 | arg = DECL_ARGUMENTS (child_fn); |
12028 | TREE_TYPE (arg) = build_pointer_type (record_type); |
12029 | sarg = DECL_CHAIN (arg); |
12030 | TREE_TYPE (sarg) = build_pointer_type (srecord_type); |
12031 | |
12032 | /* First pass: initialize temporaries used in record_type and srecord_type |
12033 | sizes and field offsets. */ |
12034 | if (tcctx.cb.decl_map) |
12035 | for (c = gimple_omp_task_clauses (gs: task_stmt); c; c = OMP_CLAUSE_CHAIN (c)) |
12036 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE) |
12037 | { |
12038 | tree *p; |
12039 | |
12040 | decl = OMP_CLAUSE_DECL (c); |
12041 | p = tcctx.cb.decl_map->get (k: decl); |
12042 | if (p == NULL) |
12043 | continue; |
12044 | n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); |
12045 | sf = (tree) n->value; |
12046 | sf = *tcctx.cb.decl_map->get (k: sf); |
12047 | src = build_simple_mem_ref_loc (loc, sarg); |
12048 | src = omp_build_component_ref (obj: src, field: sf); |
12049 | t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src); |
12050 | append_to_statement_list (t, &list); |
12051 | } |
12052 | |
12053 | /* Second pass: copy shared var pointers and copy construct non-VLA |
12054 | firstprivate vars. */ |
12055 | for (c = gimple_omp_task_clauses (gs: task_stmt); c; c = OMP_CLAUSE_CHAIN (c)) |
12056 | switch (OMP_CLAUSE_CODE (c)) |
12057 | { |
12058 | splay_tree_key key; |
12059 | case OMP_CLAUSE_SHARED: |
12060 | decl = OMP_CLAUSE_DECL (c); |
12061 | key = (splay_tree_key) decl; |
12062 | if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)) |
12063 | key = (splay_tree_key) &DECL_UID (decl); |
12064 | n = splay_tree_lookup (ctx->field_map, key); |
12065 | if (n == NULL) |
12066 | break; |
12067 | f = (tree) n->value; |
12068 | if (tcctx.cb.decl_map) |
12069 | f = *tcctx.cb.decl_map->get (k: f); |
12070 | n = splay_tree_lookup (ctx->sfield_map, key); |
12071 | sf = (tree) n->value; |
12072 | if (tcctx.cb.decl_map) |
12073 | sf = *tcctx.cb.decl_map->get (k: sf); |
12074 | src = build_simple_mem_ref_loc (loc, sarg); |
12075 | src = omp_build_component_ref (obj: src, field: sf); |
12076 | dst = build_simple_mem_ref_loc (loc, arg); |
12077 | dst = omp_build_component_ref (obj: dst, field: f); |
12078 | t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src); |
12079 | append_to_statement_list (t, &list); |
12080 | break; |
12081 | case OMP_CLAUSE_REDUCTION: |
12082 | case OMP_CLAUSE_IN_REDUCTION: |
12083 | decl = OMP_CLAUSE_DECL (c); |
12084 | if (TREE_CODE (decl) == MEM_REF) |
12085 | { |
12086 | decl = TREE_OPERAND (decl, 0); |
12087 | if (TREE_CODE (decl) == POINTER_PLUS_EXPR) |
12088 | decl = TREE_OPERAND (decl, 0); |
12089 | if (TREE_CODE (decl) == INDIRECT_REF |
12090 | || TREE_CODE (decl) == ADDR_EXPR) |
12091 | decl = TREE_OPERAND (decl, 0); |
12092 | } |
12093 | key = (splay_tree_key) decl; |
12094 | n = splay_tree_lookup (ctx->field_map, key); |
12095 | if (n == NULL) |
12096 | break; |
12097 | f = (tree) n->value; |
12098 | if (tcctx.cb.decl_map) |
12099 | f = *tcctx.cb.decl_map->get (k: f); |
12100 | n = splay_tree_lookup (ctx->sfield_map, key); |
12101 | sf = (tree) n->value; |
12102 | if (tcctx.cb.decl_map) |
12103 | sf = *tcctx.cb.decl_map->get (k: sf); |
12104 | src = build_simple_mem_ref_loc (loc, sarg); |
12105 | src = omp_build_component_ref (obj: src, field: sf); |
12106 | if (decl != OMP_CLAUSE_DECL (c) |
12107 | && TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE |
12108 | && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == POINTER_TYPE) |
12109 | src = build_simple_mem_ref_loc (loc, src); |
12110 | dst = build_simple_mem_ref_loc (loc, arg); |
12111 | dst = omp_build_component_ref (obj: dst, field: f); |
12112 | t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src); |
12113 | append_to_statement_list (t, &list); |
12114 | break; |
12115 | case OMP_CLAUSE__LOOPTEMP_: |
12116 | /* Fields for first two _looptemp_ clauses are initialized by |
12117 | GOMP_taskloop*, the rest are handled like firstprivate. */ |
12118 | if (looptempno < 2) |
12119 | { |
12120 | looptempno++; |
12121 | break; |
12122 | } |
12123 | /* FALLTHRU */ |
12124 | case OMP_CLAUSE__REDUCTEMP_: |
12125 | case OMP_CLAUSE_FIRSTPRIVATE: |
12126 | decl = OMP_CLAUSE_DECL (c); |
12127 | if (is_variable_sized (expr: decl)) |
12128 | break; |
12129 | n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); |
12130 | if (n == NULL) |
12131 | break; |
12132 | f = (tree) n->value; |
12133 | if (tcctx.cb.decl_map) |
12134 | f = *tcctx.cb.decl_map->get (k: f); |
12135 | n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); |
12136 | if (n != NULL) |
12137 | { |
12138 | sf = (tree) n->value; |
12139 | if (tcctx.cb.decl_map) |
12140 | sf = *tcctx.cb.decl_map->get (k: sf); |
12141 | src = build_simple_mem_ref_loc (loc, sarg); |
12142 | src = omp_build_component_ref (obj: src, field: sf); |
12143 | if (use_pointer_for_field (decl, NULL) |
12144 | || omp_privatize_by_reference (decl)) |
12145 | src = build_simple_mem_ref_loc (loc, src); |
12146 | } |
12147 | else |
12148 | src = decl; |
12149 | dst = build_simple_mem_ref_loc (loc, arg); |
12150 | dst = omp_build_component_ref (obj: dst, field: f); |
12151 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE) |
12152 | t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src); |
12153 | else |
12154 | { |
12155 | if (ctx->allocate_map) |
12156 | if (tree *allocatorp = ctx->allocate_map->get (k: decl)) |
12157 | { |
12158 | tree allocator = *allocatorp; |
12159 | HOST_WIDE_INT ialign = 0; |
12160 | if (TREE_CODE (allocator) == TREE_LIST) |
12161 | { |
12162 | ialign = tree_to_uhwi (TREE_VALUE (allocator)); |
12163 | allocator = TREE_PURPOSE (allocator); |
12164 | } |
12165 | if (TREE_CODE (allocator) != INTEGER_CST) |
12166 | { |
12167 | n = splay_tree_lookup (ctx->sfield_map, |
12168 | (splay_tree_key) allocator); |
12169 | allocator = (tree) n->value; |
12170 | if (tcctx.cb.decl_map) |
12171 | allocator = *tcctx.cb.decl_map->get (k: allocator); |
12172 | tree a = build_simple_mem_ref_loc (loc, sarg); |
12173 | allocator = omp_build_component_ref (obj: a, field: allocator); |
12174 | } |
12175 | allocator = fold_convert (pointer_sized_int_node, allocator); |
12176 | tree a = builtin_decl_explicit (fncode: BUILT_IN_GOMP_ALLOC); |
12177 | tree align = build_int_cst (size_type_node, |
12178 | MAX (ialign, |
12179 | DECL_ALIGN_UNIT (decl))); |
12180 | tree sz = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (dst))); |
12181 | tree ptr = build_call_expr_loc (loc, a, 3, align, sz, |
12182 | allocator); |
12183 | ptr = fold_convert (TREE_TYPE (dst), ptr); |
12184 | t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, ptr); |
12185 | append_to_statement_list (t, &list); |
12186 | dst = build_simple_mem_ref_loc (loc, dst); |
12187 | } |
12188 | t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src); |
12189 | } |
12190 | append_to_statement_list (t, &list); |
12191 | break; |
12192 | case OMP_CLAUSE_PRIVATE: |
12193 | if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c)) |
12194 | break; |
12195 | decl = OMP_CLAUSE_DECL (c); |
12196 | n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); |
12197 | f = (tree) n->value; |
12198 | if (tcctx.cb.decl_map) |
12199 | f = *tcctx.cb.decl_map->get (k: f); |
12200 | n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); |
12201 | if (n != NULL) |
12202 | { |
12203 | sf = (tree) n->value; |
12204 | if (tcctx.cb.decl_map) |
12205 | sf = *tcctx.cb.decl_map->get (k: sf); |
12206 | src = build_simple_mem_ref_loc (loc, sarg); |
12207 | src = omp_build_component_ref (obj: src, field: sf); |
12208 | if (use_pointer_for_field (decl, NULL)) |
12209 | src = build_simple_mem_ref_loc (loc, src); |
12210 | } |
12211 | else |
12212 | src = decl; |
12213 | dst = build_simple_mem_ref_loc (loc, arg); |
12214 | dst = omp_build_component_ref (obj: dst, field: f); |
12215 | t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src); |
12216 | append_to_statement_list (t, &list); |
12217 | break; |
12218 | default: |
12219 | break; |
12220 | } |
12221 | |
12222 | /* Last pass: handle VLA firstprivates. */ |
12223 | if (tcctx.cb.decl_map) |
12224 | for (c = gimple_omp_task_clauses (gs: task_stmt); c; c = OMP_CLAUSE_CHAIN (c)) |
12225 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE) |
12226 | { |
12227 | tree ind, ptr, df; |
12228 | |
12229 | decl = OMP_CLAUSE_DECL (c); |
12230 | if (!is_variable_sized (expr: decl)) |
12231 | continue; |
12232 | n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); |
12233 | if (n == NULL) |
12234 | continue; |
12235 | f = (tree) n->value; |
12236 | f = *tcctx.cb.decl_map->get (k: f); |
12237 | gcc_assert (DECL_HAS_VALUE_EXPR_P (decl)); |
12238 | ind = DECL_VALUE_EXPR (decl); |
12239 | gcc_assert (TREE_CODE (ind) == INDIRECT_REF); |
12240 | gcc_assert (DECL_P (TREE_OPERAND (ind, 0))); |
12241 | n = splay_tree_lookup (ctx->sfield_map, |
12242 | (splay_tree_key) TREE_OPERAND (ind, 0)); |
12243 | sf = (tree) n->value; |
12244 | sf = *tcctx.cb.decl_map->get (k: sf); |
12245 | src = build_simple_mem_ref_loc (loc, sarg); |
12246 | src = omp_build_component_ref (obj: src, field: sf); |
12247 | src = build_simple_mem_ref_loc (loc, src); |
12248 | dst = build_simple_mem_ref_loc (loc, arg); |
12249 | dst = omp_build_component_ref (obj: dst, field: f); |
12250 | t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src); |
12251 | append_to_statement_list (t, &list); |
12252 | n = splay_tree_lookup (ctx->field_map, |
12253 | (splay_tree_key) TREE_OPERAND (ind, 0)); |
12254 | df = (tree) n->value; |
12255 | df = *tcctx.cb.decl_map->get (k: df); |
12256 | ptr = build_simple_mem_ref_loc (loc, arg); |
12257 | ptr = omp_build_component_ref (obj: ptr, field: df); |
12258 | t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr, |
12259 | build_fold_addr_expr_loc (loc, dst)); |
12260 | append_to_statement_list (t, &list); |
12261 | } |
12262 | |
12263 | t = build1 (RETURN_EXPR, void_type_node, NULL); |
12264 | append_to_statement_list (t, &list); |
12265 | |
12266 | if (tcctx.cb.decl_map) |
12267 | delete tcctx.cb.decl_map; |
12268 | pop_gimplify_context (NULL); |
12269 | BIND_EXPR_BODY (bind) = list; |
12270 | pop_cfun (); |
12271 | } |
12272 | |
12273 | static void |
12274 | lower_depend_clauses (tree *pclauses, gimple_seq *iseq, gimple_seq *oseq) |
12275 | { |
12276 | tree c, clauses; |
12277 | gimple *g; |
12278 | size_t cnt[5] = { 0, 0, 0, 0, 0 }, idx = 2, i; |
12279 | |
12280 | clauses = omp_find_clause (clauses: *pclauses, kind: OMP_CLAUSE_DEPEND); |
12281 | gcc_assert (clauses); |
12282 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
12283 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND) |
12284 | switch (OMP_CLAUSE_DEPEND_KIND (c)) |
12285 | { |
12286 | case OMP_CLAUSE_DEPEND_LAST: |
12287 | /* Lowering already done at gimplification. */ |
12288 | return; |
12289 | case OMP_CLAUSE_DEPEND_IN: |
12290 | cnt[2]++; |
12291 | break; |
12292 | case OMP_CLAUSE_DEPEND_OUT: |
12293 | case OMP_CLAUSE_DEPEND_INOUT: |
12294 | cnt[0]++; |
12295 | break; |
12296 | case OMP_CLAUSE_DEPEND_MUTEXINOUTSET: |
12297 | cnt[1]++; |
12298 | break; |
12299 | case OMP_CLAUSE_DEPEND_DEPOBJ: |
12300 | cnt[3]++; |
12301 | break; |
12302 | case OMP_CLAUSE_DEPEND_INOUTSET: |
12303 | cnt[4]++; |
12304 | break; |
12305 | default: |
12306 | gcc_unreachable (); |
12307 | } |
12308 | if (cnt[1] || cnt[3] || cnt[4]) |
12309 | idx = 5; |
12310 | size_t total = cnt[0] + cnt[1] + cnt[2] + cnt[3] + cnt[4]; |
12311 | size_t inoutidx = total + idx; |
12312 | tree type = build_array_type_nelts (ptr_type_node, total + idx + 2 * cnt[4]); |
12313 | tree array = create_tmp_var (type); |
12314 | TREE_ADDRESSABLE (array) = 1; |
12315 | tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE, |
12316 | NULL_TREE); |
12317 | if (idx == 5) |
12318 | { |
12319 | g = gimple_build_assign (r, build_int_cst (ptr_type_node, 0)); |
12320 | gimple_seq_add_stmt (iseq, g); |
12321 | r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE, |
12322 | NULL_TREE); |
12323 | } |
12324 | g = gimple_build_assign (r, build_int_cst (ptr_type_node, total)); |
12325 | gimple_seq_add_stmt (iseq, g); |
12326 | for (i = 0; i < (idx == 5 ? 3 : 1); i++) |
12327 | { |
12328 | r = build4 (ARRAY_REF, ptr_type_node, array, |
12329 | size_int (i + 1 + (idx == 5)), NULL_TREE, NULL_TREE); |
12330 | g = gimple_build_assign (r, build_int_cst (ptr_type_node, cnt[i])); |
12331 | gimple_seq_add_stmt (iseq, g); |
12332 | } |
12333 | for (i = 0; i < 5; i++) |
12334 | { |
12335 | if (cnt[i] == 0) |
12336 | continue; |
12337 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
12338 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DEPEND) |
12339 | continue; |
12340 | else |
12341 | { |
12342 | switch (OMP_CLAUSE_DEPEND_KIND (c)) |
12343 | { |
12344 | case OMP_CLAUSE_DEPEND_IN: |
12345 | if (i != 2) |
12346 | continue; |
12347 | break; |
12348 | case OMP_CLAUSE_DEPEND_OUT: |
12349 | case OMP_CLAUSE_DEPEND_INOUT: |
12350 | if (i != 0) |
12351 | continue; |
12352 | break; |
12353 | case OMP_CLAUSE_DEPEND_MUTEXINOUTSET: |
12354 | if (i != 1) |
12355 | continue; |
12356 | break; |
12357 | case OMP_CLAUSE_DEPEND_DEPOBJ: |
12358 | if (i != 3) |
12359 | continue; |
12360 | break; |
12361 | case OMP_CLAUSE_DEPEND_INOUTSET: |
12362 | if (i != 4) |
12363 | continue; |
12364 | break; |
12365 | default: |
12366 | gcc_unreachable (); |
12367 | } |
12368 | tree t = OMP_CLAUSE_DECL (c); |
12369 | if (i == 4) |
12370 | { |
12371 | t = build4 (ARRAY_REF, ptr_type_node, array, |
12372 | size_int (inoutidx), NULL_TREE, NULL_TREE); |
12373 | t = build_fold_addr_expr (t); |
12374 | inoutidx += 2; |
12375 | } |
12376 | t = fold_convert (ptr_type_node, t); |
12377 | gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue); |
12378 | r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++), |
12379 | NULL_TREE, NULL_TREE); |
12380 | g = gimple_build_assign (r, t); |
12381 | gimple_seq_add_stmt (iseq, g); |
12382 | } |
12383 | } |
12384 | if (cnt[4]) |
12385 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
12386 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND |
12387 | && OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_INOUTSET) |
12388 | { |
12389 | tree t = OMP_CLAUSE_DECL (c); |
12390 | t = fold_convert (ptr_type_node, t); |
12391 | gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue); |
12392 | r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++), |
12393 | NULL_TREE, NULL_TREE); |
12394 | g = gimple_build_assign (r, t); |
12395 | gimple_seq_add_stmt (iseq, g); |
12396 | t = build_int_cst (ptr_type_node, GOMP_DEPEND_INOUTSET); |
12397 | r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++), |
12398 | NULL_TREE, NULL_TREE); |
12399 | g = gimple_build_assign (r, t); |
12400 | gimple_seq_add_stmt (iseq, g); |
12401 | } |
12402 | |
12403 | c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND); |
12404 | OMP_CLAUSE_DEPEND_KIND (c) = OMP_CLAUSE_DEPEND_LAST; |
12405 | OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array); |
12406 | OMP_CLAUSE_CHAIN (c) = *pclauses; |
12407 | *pclauses = c; |
12408 | tree clobber = build_clobber (type); |
12409 | g = gimple_build_assign (array, clobber); |
12410 | gimple_seq_add_stmt (oseq, g); |
12411 | } |
12412 | |
12413 | /* Lower the OpenMP parallel or task directive in the current statement |
12414 | in GSI_P. CTX holds context information for the directive. */ |
12415 | |
12416 | static void |
12417 | lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
12418 | { |
12419 | tree clauses; |
12420 | tree child_fn, t; |
12421 | gimple *stmt = gsi_stmt (i: *gsi_p); |
12422 | gbind *par_bind, *bind, *dep_bind = NULL; |
12423 | gimple_seq par_body; |
12424 | location_t loc = gimple_location (g: stmt); |
12425 | |
12426 | clauses = gimple_omp_taskreg_clauses (gs: stmt); |
12427 | if (gimple_code (g: stmt) == GIMPLE_OMP_TASK |
12428 | && gimple_omp_task_taskwait_p (g: stmt)) |
12429 | { |
12430 | par_bind = NULL; |
12431 | par_body = NULL; |
12432 | } |
12433 | else |
12434 | { |
12435 | par_bind |
12436 | = as_a <gbind *> (p: gimple_seq_first_stmt (s: gimple_omp_body (gs: stmt))); |
12437 | par_body = gimple_bind_body (gs: par_bind); |
12438 | } |
12439 | child_fn = ctx->cb.dst_fn; |
12440 | if (gimple_code (g: stmt) == GIMPLE_OMP_PARALLEL |
12441 | && !gimple_omp_parallel_combined_p (g: stmt)) |
12442 | { |
12443 | struct walk_stmt_info wi; |
12444 | int ws_num = 0; |
12445 | |
12446 | memset (s: &wi, c: 0, n: sizeof (wi)); |
12447 | wi.info = &ws_num; |
12448 | wi.val_only = true; |
12449 | walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi); |
12450 | if (ws_num == 1) |
12451 | gimple_omp_parallel_set_combined_p (g: stmt, combined_p: true); |
12452 | } |
12453 | gimple_seq dep_ilist = NULL; |
12454 | gimple_seq dep_olist = NULL; |
12455 | if (gimple_code (g: stmt) == GIMPLE_OMP_TASK |
12456 | && omp_find_clause (clauses, kind: OMP_CLAUSE_DEPEND)) |
12457 | { |
12458 | push_gimplify_context (); |
12459 | dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK)); |
12460 | lower_depend_clauses (pclauses: gimple_omp_task_clauses_ptr (gs: stmt), |
12461 | iseq: &dep_ilist, oseq: &dep_olist); |
12462 | } |
12463 | |
12464 | if (gimple_code (g: stmt) == GIMPLE_OMP_TASK |
12465 | && gimple_omp_task_taskwait_p (g: stmt)) |
12466 | { |
12467 | if (dep_bind) |
12468 | { |
12469 | gsi_replace (gsi_p, dep_bind, true); |
12470 | gimple_bind_add_seq (bind_stmt: dep_bind, seq: dep_ilist); |
12471 | gimple_bind_add_stmt (bind_stmt: dep_bind, stmt); |
12472 | gimple_bind_add_seq (bind_stmt: dep_bind, seq: dep_olist); |
12473 | pop_gimplify_context (dep_bind); |
12474 | } |
12475 | return; |
12476 | } |
12477 | |
12478 | if (ctx->srecord_type) |
12479 | create_task_copyfn (task_stmt: as_a <gomp_task *> (p: stmt), ctx); |
12480 | |
12481 | gimple_seq tskred_ilist = NULL; |
12482 | gimple_seq tskred_olist = NULL; |
12483 | if ((is_task_ctx (ctx) |
12484 | && gimple_omp_task_taskloop_p (g: ctx->stmt) |
12485 | && omp_find_clause (clauses: gimple_omp_task_clauses (gs: ctx->stmt), |
12486 | kind: OMP_CLAUSE_REDUCTION)) |
12487 | || (is_parallel_ctx (ctx) |
12488 | && omp_find_clause (clauses: gimple_omp_parallel_clauses (gs: stmt), |
12489 | kind: OMP_CLAUSE__REDUCTEMP_))) |
12490 | { |
12491 | if (dep_bind == NULL) |
12492 | { |
12493 | push_gimplify_context (); |
12494 | dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK)); |
12495 | } |
12496 | lower_omp_task_reductions (ctx, code: is_task_ctx (ctx) ? OMP_TASKLOOP |
12497 | : OMP_PARALLEL, |
12498 | clauses: gimple_omp_taskreg_clauses (gs: ctx->stmt), |
12499 | start: &tskred_ilist, end: &tskred_olist); |
12500 | } |
12501 | |
12502 | push_gimplify_context (); |
12503 | |
12504 | gimple_seq par_olist = NULL; |
12505 | gimple_seq par_ilist = NULL; |
12506 | gimple_seq par_rlist = NULL; |
12507 | lower_rec_input_clauses (clauses, ilist: &par_ilist, dlist: &par_olist, ctx, NULL); |
12508 | lower_omp (&par_body, ctx); |
12509 | if (gimple_code (g: stmt) != GIMPLE_OMP_TASK) |
12510 | lower_reduction_clauses (clauses, stmt_seqp: &par_rlist, NULL, ctx); |
12511 | |
12512 | /* Declare all the variables created by mapping and the variables |
12513 | declared in the scope of the parallel body. */ |
12514 | record_vars_into (ctx->block_vars, child_fn); |
12515 | maybe_remove_omp_member_access_dummy_vars (bind: par_bind); |
12516 | record_vars_into (gimple_bind_vars (bind_stmt: par_bind), child_fn); |
12517 | |
12518 | if (ctx->record_type) |
12519 | { |
12520 | ctx->sender_decl |
12521 | = create_tmp_var (ctx->srecord_type ? ctx->srecord_type |
12522 | : ctx->record_type, ".omp_data_o" ); |
12523 | DECL_NAMELESS (ctx->sender_decl) = 1; |
12524 | TREE_ADDRESSABLE (ctx->sender_decl) = 1; |
12525 | gimple_omp_taskreg_set_data_arg (gs: stmt, data_arg: ctx->sender_decl); |
12526 | } |
12527 | |
12528 | gimple_seq olist = NULL; |
12529 | gimple_seq ilist = NULL; |
12530 | lower_send_clauses (clauses, ilist: &ilist, olist: &olist, ctx); |
12531 | lower_send_shared_vars (ilist: &ilist, olist: &olist, ctx); |
12532 | |
12533 | if (ctx->record_type) |
12534 | { |
12535 | tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl)); |
12536 | gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl, |
12537 | clobber)); |
12538 | } |
12539 | |
12540 | /* Once all the expansions are done, sequence all the different |
12541 | fragments inside gimple_omp_body. */ |
12542 | |
12543 | gimple_seq new_body = NULL; |
12544 | |
12545 | if (ctx->record_type) |
12546 | { |
12547 | t = build_fold_addr_expr_loc (loc, ctx->sender_decl); |
12548 | /* fixup_child_record_type might have changed receiver_decl's type. */ |
12549 | t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t); |
12550 | gimple_seq_add_stmt (&new_body, |
12551 | gimple_build_assign (ctx->receiver_decl, t)); |
12552 | } |
12553 | |
12554 | gimple_seq_add_seq (&new_body, par_ilist); |
12555 | gimple_seq_add_seq (&new_body, par_body); |
12556 | gimple_seq_add_seq (&new_body, par_rlist); |
12557 | if (ctx->cancellable) |
12558 | gimple_seq_add_stmt (&new_body, gimple_build_label (label: ctx->cancel_label)); |
12559 | gimple_seq_add_seq (&new_body, par_olist); |
12560 | new_body = maybe_catch_exception (body: new_body); |
12561 | if (gimple_code (g: stmt) == GIMPLE_OMP_TASK) |
12562 | gimple_seq_add_stmt (&new_body, |
12563 | gimple_build_omp_continue (integer_zero_node, |
12564 | integer_zero_node)); |
12565 | gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false)); |
12566 | gimple_omp_set_body (gs: stmt, body: new_body); |
12567 | |
12568 | if (dep_bind && gimple_bind_block (bind_stmt: par_bind) == NULL_TREE) |
12569 | bind = gimple_build_bind (NULL, NULL, make_node (BLOCK)); |
12570 | else |
12571 | bind = gimple_build_bind (NULL, NULL, gimple_bind_block (bind_stmt: par_bind)); |
12572 | gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true); |
12573 | gimple_bind_add_seq (bind_stmt: bind, seq: ilist); |
12574 | gimple_bind_add_stmt (bind_stmt: bind, stmt); |
12575 | gimple_bind_add_seq (bind_stmt: bind, seq: olist); |
12576 | |
12577 | pop_gimplify_context (NULL); |
12578 | |
12579 | if (dep_bind) |
12580 | { |
12581 | gimple_bind_add_seq (bind_stmt: dep_bind, seq: dep_ilist); |
12582 | gimple_bind_add_seq (bind_stmt: dep_bind, seq: tskred_ilist); |
12583 | gimple_bind_add_stmt (bind_stmt: dep_bind, stmt: bind); |
12584 | gimple_bind_add_seq (bind_stmt: dep_bind, seq: tskred_olist); |
12585 | gimple_bind_add_seq (bind_stmt: dep_bind, seq: dep_olist); |
12586 | pop_gimplify_context (dep_bind); |
12587 | } |
12588 | } |
12589 | |
12590 | /* Lower the GIMPLE_OMP_TARGET in the current statement |
12591 | in GSI_P. CTX holds context information for the directive. */ |
12592 | |
12593 | static void |
12594 | lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
12595 | { |
12596 | tree clauses; |
12597 | tree child_fn, t, c; |
12598 | gomp_target *stmt = as_a <gomp_target *> (p: gsi_stmt (i: *gsi_p)); |
12599 | gbind *tgt_bind, *bind, *dep_bind = NULL; |
12600 | gimple_seq tgt_body, olist, ilist, fplist, new_body; |
12601 | location_t loc = gimple_location (g: stmt); |
12602 | bool offloaded, data_region; |
12603 | unsigned int map_cnt = 0; |
12604 | tree in_reduction_clauses = NULL_TREE; |
12605 | |
12606 | offloaded = is_gimple_omp_offloaded (stmt); |
12607 | switch (gimple_omp_target_kind (g: stmt)) |
12608 | { |
12609 | case GF_OMP_TARGET_KIND_REGION: |
12610 | tree *p, *q; |
12611 | q = &in_reduction_clauses; |
12612 | for (p = gimple_omp_target_clauses_ptr (gs: stmt); *p; ) |
12613 | if (OMP_CLAUSE_CODE (*p) == OMP_CLAUSE_IN_REDUCTION) |
12614 | { |
12615 | *q = *p; |
12616 | q = &OMP_CLAUSE_CHAIN (*q); |
12617 | *p = OMP_CLAUSE_CHAIN (*p); |
12618 | } |
12619 | else |
12620 | p = &OMP_CLAUSE_CHAIN (*p); |
12621 | *q = NULL_TREE; |
12622 | *p = in_reduction_clauses; |
12623 | /* FALLTHRU */ |
12624 | case GF_OMP_TARGET_KIND_UPDATE: |
12625 | case GF_OMP_TARGET_KIND_ENTER_DATA: |
12626 | case GF_OMP_TARGET_KIND_EXIT_DATA: |
12627 | case GF_OMP_TARGET_KIND_OACC_PARALLEL: |
12628 | case GF_OMP_TARGET_KIND_OACC_KERNELS: |
12629 | case GF_OMP_TARGET_KIND_OACC_SERIAL: |
12630 | case GF_OMP_TARGET_KIND_OACC_UPDATE: |
12631 | case GF_OMP_TARGET_KIND_OACC_ENTER_DATA: |
12632 | case GF_OMP_TARGET_KIND_OACC_EXIT_DATA: |
12633 | case GF_OMP_TARGET_KIND_OACC_DECLARE: |
12634 | case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED: |
12635 | case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE: |
12636 | data_region = false; |
12637 | break; |
12638 | case GF_OMP_TARGET_KIND_DATA: |
12639 | case GF_OMP_TARGET_KIND_OACC_DATA: |
12640 | case GF_OMP_TARGET_KIND_OACC_HOST_DATA: |
12641 | case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS: |
12642 | data_region = true; |
12643 | break; |
12644 | default: |
12645 | gcc_unreachable (); |
12646 | } |
12647 | |
12648 | /* Ensure that requires map is written via output_offload_tables, even if only |
12649 | 'target (enter/exit) data' is used in the translation unit. */ |
12650 | if (ENABLE_OFFLOADING && (omp_requires_mask & OMP_REQUIRES_TARGET_USED)) |
12651 | g->have_offload = true; |
12652 | |
12653 | clauses = gimple_omp_target_clauses (gs: stmt); |
12654 | |
12655 | gimple_seq dep_ilist = NULL; |
12656 | gimple_seq dep_olist = NULL; |
12657 | bool has_depend = omp_find_clause (clauses, kind: OMP_CLAUSE_DEPEND) != NULL_TREE; |
12658 | if (has_depend || in_reduction_clauses) |
12659 | { |
12660 | push_gimplify_context (); |
12661 | dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK)); |
12662 | if (has_depend) |
12663 | lower_depend_clauses (pclauses: gimple_omp_target_clauses_ptr (gs: stmt), |
12664 | iseq: &dep_ilist, oseq: &dep_olist); |
12665 | if (in_reduction_clauses) |
12666 | lower_rec_input_clauses (clauses: in_reduction_clauses, ilist: &dep_ilist, dlist: &dep_olist, |
12667 | ctx, NULL); |
12668 | } |
12669 | |
12670 | tgt_bind = NULL; |
12671 | tgt_body = NULL; |
12672 | if (offloaded) |
12673 | { |
12674 | tgt_bind = gimple_seq_first_stmt_as_a_bind (s: gimple_omp_body (gs: stmt)); |
12675 | tgt_body = gimple_bind_body (gs: tgt_bind); |
12676 | } |
12677 | else if (data_region) |
12678 | tgt_body = gimple_omp_body (gs: stmt); |
12679 | child_fn = ctx->cb.dst_fn; |
12680 | |
12681 | push_gimplify_context (); |
12682 | fplist = NULL; |
12683 | |
12684 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
12685 | switch (OMP_CLAUSE_CODE (c)) |
12686 | { |
12687 | tree var, x; |
12688 | |
12689 | default: |
12690 | break; |
12691 | case OMP_CLAUSE_MAP: |
12692 | #if CHECKING_P |
12693 | /* First check what we're prepared to handle in the following. */ |
12694 | switch (OMP_CLAUSE_MAP_KIND (c)) |
12695 | { |
12696 | case GOMP_MAP_ALLOC: |
12697 | case GOMP_MAP_TO: |
12698 | case GOMP_MAP_FROM: |
12699 | case GOMP_MAP_TOFROM: |
12700 | case GOMP_MAP_POINTER: |
12701 | case GOMP_MAP_TO_PSET: |
12702 | case GOMP_MAP_DELETE: |
12703 | case GOMP_MAP_RELEASE: |
12704 | case GOMP_MAP_ALWAYS_TO: |
12705 | case GOMP_MAP_ALWAYS_FROM: |
12706 | case GOMP_MAP_ALWAYS_TOFROM: |
12707 | case GOMP_MAP_FORCE_PRESENT: |
12708 | case GOMP_MAP_ALWAYS_PRESENT_FROM: |
12709 | case GOMP_MAP_ALWAYS_PRESENT_TO: |
12710 | case GOMP_MAP_ALWAYS_PRESENT_TOFROM: |
12711 | |
12712 | case GOMP_MAP_FIRSTPRIVATE_POINTER: |
12713 | case GOMP_MAP_FIRSTPRIVATE_REFERENCE: |
12714 | case GOMP_MAP_STRUCT: |
12715 | case GOMP_MAP_STRUCT_UNORD: |
12716 | case GOMP_MAP_ALWAYS_POINTER: |
12717 | case GOMP_MAP_ATTACH: |
12718 | case GOMP_MAP_DETACH: |
12719 | case GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION: |
12720 | case GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION: |
12721 | break; |
12722 | case GOMP_MAP_IF_PRESENT: |
12723 | case GOMP_MAP_FORCE_ALLOC: |
12724 | case GOMP_MAP_FORCE_TO: |
12725 | case GOMP_MAP_FORCE_FROM: |
12726 | case GOMP_MAP_FORCE_TOFROM: |
12727 | case GOMP_MAP_FORCE_DEVICEPTR: |
12728 | case GOMP_MAP_DEVICE_RESIDENT: |
12729 | case GOMP_MAP_LINK: |
12730 | case GOMP_MAP_FORCE_DETACH: |
12731 | gcc_assert (is_gimple_omp_oacc (stmt)); |
12732 | break; |
12733 | default: |
12734 | gcc_unreachable (); |
12735 | } |
12736 | #endif |
12737 | /* FALLTHRU */ |
12738 | case OMP_CLAUSE_TO: |
12739 | case OMP_CLAUSE_FROM: |
12740 | oacc_firstprivate: |
12741 | var = OMP_CLAUSE_DECL (c); |
12742 | if (!DECL_P (var)) |
12743 | { |
12744 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP |
12745 | || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) |
12746 | && (OMP_CLAUSE_MAP_KIND (c) |
12747 | != GOMP_MAP_FIRSTPRIVATE_POINTER))) |
12748 | map_cnt++; |
12749 | continue; |
12750 | } |
12751 | |
12752 | if (DECL_SIZE (var) |
12753 | && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST) |
12754 | { |
12755 | tree var2 = DECL_VALUE_EXPR (var); |
12756 | gcc_assert (TREE_CODE (var2) == INDIRECT_REF); |
12757 | var2 = TREE_OPERAND (var2, 0); |
12758 | gcc_assert (DECL_P (var2)); |
12759 | var = var2; |
12760 | } |
12761 | |
12762 | if (offloaded |
12763 | && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
12764 | && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER |
12765 | || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)) |
12766 | { |
12767 | if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE) |
12768 | { |
12769 | if (is_global_var (t: maybe_lookup_decl_in_outer_ctx (decl: var, ctx)) |
12770 | && varpool_node::get_create (decl: var)->offloadable) |
12771 | continue; |
12772 | |
12773 | tree type = build_pointer_type (TREE_TYPE (var)); |
12774 | tree new_var = lookup_decl (var, ctx); |
12775 | x = create_tmp_var_raw (type, get_name (new_var)); |
12776 | gimple_add_tmp_var (x); |
12777 | x = build_simple_mem_ref (x); |
12778 | SET_DECL_VALUE_EXPR (new_var, x); |
12779 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
12780 | } |
12781 | continue; |
12782 | } |
12783 | |
12784 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
12785 | && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH |
12786 | || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH) |
12787 | && is_omp_target (stmt)) |
12788 | { |
12789 | gcc_assert (maybe_lookup_field (c, ctx)); |
12790 | map_cnt++; |
12791 | continue; |
12792 | } |
12793 | |
12794 | if (!maybe_lookup_field (var, ctx)) |
12795 | continue; |
12796 | |
12797 | /* Don't remap compute constructs' reduction variables, because the |
12798 | intermediate result must be local to each gang. */ |
12799 | if (offloaded && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
12800 | && is_gimple_omp_oacc (stmt: ctx->stmt) |
12801 | && OMP_CLAUSE_MAP_IN_REDUCTION (c))) |
12802 | { |
12803 | x = build_receiver_ref (var, by_ref: true, ctx); |
12804 | tree new_var = lookup_decl (var, ctx); |
12805 | |
12806 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
12807 | && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER |
12808 | && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) |
12809 | && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE) |
12810 | x = build_simple_mem_ref (x); |
12811 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE) |
12812 | { |
12813 | gcc_assert (is_gimple_omp_oacc (ctx->stmt)); |
12814 | if (omp_privatize_by_reference (decl: new_var) |
12815 | && (TREE_CODE (TREE_TYPE (new_var)) != POINTER_TYPE |
12816 | || DECL_BY_REFERENCE (var))) |
12817 | { |
12818 | /* Create a local object to hold the instance |
12819 | value. */ |
12820 | tree type = TREE_TYPE (TREE_TYPE (new_var)); |
12821 | const char *id = IDENTIFIER_POINTER (DECL_NAME (new_var)); |
12822 | tree inst = create_tmp_var (type, id); |
12823 | gimplify_assign (inst, fold_indirect_ref (x), &fplist); |
12824 | x = build_fold_addr_expr (inst); |
12825 | } |
12826 | gimplify_assign (new_var, x, &fplist); |
12827 | } |
12828 | else if (DECL_P (new_var)) |
12829 | { |
12830 | SET_DECL_VALUE_EXPR (new_var, x); |
12831 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
12832 | } |
12833 | else |
12834 | gcc_unreachable (); |
12835 | } |
12836 | map_cnt++; |
12837 | break; |
12838 | |
12839 | case OMP_CLAUSE_FIRSTPRIVATE: |
12840 | omp_firstprivate_recv: |
12841 | gcc_checking_assert (offloaded); |
12842 | if (is_gimple_omp_oacc (stmt: ctx->stmt)) |
12843 | { |
12844 | /* No 'firstprivate' clauses on OpenACC 'kernels'. */ |
12845 | gcc_checking_assert (!is_oacc_kernels (ctx)); |
12846 | /* Likewise, on OpenACC 'kernels' decomposed parts. */ |
12847 | gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx)); |
12848 | |
12849 | goto oacc_firstprivate; |
12850 | } |
12851 | map_cnt++; |
12852 | var = OMP_CLAUSE_DECL (c); |
12853 | if (!omp_privatize_by_reference (decl: var) |
12854 | && !is_gimple_reg_type (TREE_TYPE (var))) |
12855 | { |
12856 | tree new_var = lookup_decl (var, ctx); |
12857 | if (is_variable_sized (expr: var)) |
12858 | { |
12859 | tree pvar = DECL_VALUE_EXPR (var); |
12860 | gcc_assert (TREE_CODE (pvar) == INDIRECT_REF); |
12861 | pvar = TREE_OPERAND (pvar, 0); |
12862 | gcc_assert (DECL_P (pvar)); |
12863 | tree new_pvar = lookup_decl (var: pvar, ctx); |
12864 | x = build_fold_indirect_ref (new_pvar); |
12865 | TREE_THIS_NOTRAP (x) = 1; |
12866 | } |
12867 | else |
12868 | x = build_receiver_ref (var, by_ref: true, ctx); |
12869 | SET_DECL_VALUE_EXPR (new_var, x); |
12870 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
12871 | } |
12872 | /* Fortran array descriptors: firstprivate of data + attach. */ |
12873 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR |
12874 | && lang_hooks.decls.omp_array_data (var, true)) |
12875 | map_cnt += 2; |
12876 | break; |
12877 | |
12878 | case OMP_CLAUSE_PRIVATE: |
12879 | gcc_checking_assert (offloaded); |
12880 | if (is_gimple_omp_oacc (stmt: ctx->stmt)) |
12881 | { |
12882 | /* No 'private' clauses on OpenACC 'kernels'. */ |
12883 | gcc_checking_assert (!is_oacc_kernels (ctx)); |
12884 | /* Likewise, on OpenACC 'kernels' decomposed parts. */ |
12885 | gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx)); |
12886 | |
12887 | break; |
12888 | } |
12889 | var = OMP_CLAUSE_DECL (c); |
12890 | if (is_variable_sized (expr: var)) |
12891 | { |
12892 | tree new_var = lookup_decl (var, ctx); |
12893 | tree pvar = DECL_VALUE_EXPR (var); |
12894 | gcc_assert (TREE_CODE (pvar) == INDIRECT_REF); |
12895 | pvar = TREE_OPERAND (pvar, 0); |
12896 | gcc_assert (DECL_P (pvar)); |
12897 | tree new_pvar = lookup_decl (var: pvar, ctx); |
12898 | x = build_fold_indirect_ref (new_pvar); |
12899 | TREE_THIS_NOTRAP (x) = 1; |
12900 | SET_DECL_VALUE_EXPR (new_var, x); |
12901 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
12902 | } |
12903 | break; |
12904 | |
12905 | case OMP_CLAUSE_USE_DEVICE_PTR: |
12906 | case OMP_CLAUSE_USE_DEVICE_ADDR: |
12907 | case OMP_CLAUSE_HAS_DEVICE_ADDR: |
12908 | case OMP_CLAUSE_IS_DEVICE_PTR: |
12909 | var = OMP_CLAUSE_DECL (c); |
12910 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR) |
12911 | { |
12912 | while (TREE_CODE (var) == INDIRECT_REF |
12913 | || TREE_CODE (var) == ARRAY_REF) |
12914 | var = TREE_OPERAND (var, 0); |
12915 | if (lang_hooks.decls.omp_array_data (var, true)) |
12916 | goto omp_firstprivate_recv; |
12917 | } |
12918 | map_cnt++; |
12919 | if (is_variable_sized (expr: var)) |
12920 | { |
12921 | tree new_var = lookup_decl (var, ctx); |
12922 | tree pvar = DECL_VALUE_EXPR (var); |
12923 | gcc_assert (TREE_CODE (pvar) == INDIRECT_REF); |
12924 | pvar = TREE_OPERAND (pvar, 0); |
12925 | gcc_assert (DECL_P (pvar)); |
12926 | tree new_pvar = lookup_decl (var: pvar, ctx); |
12927 | x = build_fold_indirect_ref (new_pvar); |
12928 | TREE_THIS_NOTRAP (x) = 1; |
12929 | SET_DECL_VALUE_EXPR (new_var, x); |
12930 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
12931 | } |
12932 | else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR |
12933 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR) |
12934 | && !omp_privatize_by_reference (decl: var) |
12935 | && !omp_is_allocatable_or_ptr (decl: var) |
12936 | && !lang_hooks.decls.omp_array_data (var, true)) |
12937 | || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE) |
12938 | { |
12939 | tree new_var = lookup_decl (var, ctx); |
12940 | tree type = build_pointer_type (TREE_TYPE (var)); |
12941 | x = create_tmp_var_raw (type, get_name (new_var)); |
12942 | gimple_add_tmp_var (x); |
12943 | x = build_simple_mem_ref (x); |
12944 | SET_DECL_VALUE_EXPR (new_var, x); |
12945 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
12946 | } |
12947 | else |
12948 | { |
12949 | tree new_var = lookup_decl (var, ctx); |
12950 | x = create_tmp_var_raw (TREE_TYPE (new_var), get_name (new_var)); |
12951 | gimple_add_tmp_var (x); |
12952 | SET_DECL_VALUE_EXPR (new_var, x); |
12953 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
12954 | } |
12955 | break; |
12956 | } |
12957 | |
12958 | if (offloaded) |
12959 | { |
12960 | target_nesting_level++; |
12961 | lower_omp (&tgt_body, ctx); |
12962 | target_nesting_level--; |
12963 | } |
12964 | else if (data_region) |
12965 | lower_omp (&tgt_body, ctx); |
12966 | |
12967 | if (offloaded) |
12968 | { |
12969 | /* Declare all the variables created by mapping and the variables |
12970 | declared in the scope of the target body. */ |
12971 | record_vars_into (ctx->block_vars, child_fn); |
12972 | maybe_remove_omp_member_access_dummy_vars (bind: tgt_bind); |
12973 | record_vars_into (gimple_bind_vars (bind_stmt: tgt_bind), child_fn); |
12974 | } |
12975 | |
12976 | olist = NULL; |
12977 | ilist = NULL; |
12978 | if (ctx->record_type) |
12979 | { |
12980 | ctx->sender_decl |
12981 | = create_tmp_var (ctx->record_type, ".omp_data_arr" ); |
12982 | DECL_NAMELESS (ctx->sender_decl) = 1; |
12983 | TREE_ADDRESSABLE (ctx->sender_decl) = 1; |
12984 | t = make_tree_vec (3); |
12985 | TREE_VEC_ELT (t, 0) = ctx->sender_decl; |
12986 | TREE_VEC_ELT (t, 1) |
12987 | = create_tmp_var (build_array_type_nelts (size_type_node, map_cnt), |
12988 | ".omp_data_sizes" ); |
12989 | DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1; |
12990 | TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1; |
12991 | TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1; |
12992 | tree tkind_type = short_unsigned_type_node; |
12993 | int talign_shift = 8; |
12994 | TREE_VEC_ELT (t, 2) |
12995 | = create_tmp_var (build_array_type_nelts (tkind_type, map_cnt), |
12996 | ".omp_data_kinds" ); |
12997 | DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1; |
12998 | TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1; |
12999 | TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1; |
13000 | gimple_omp_target_set_data_arg (omp_target_stmt: stmt, data_arg: t); |
13001 | |
13002 | vec<constructor_elt, va_gc> *vsize; |
13003 | vec<constructor_elt, va_gc> *vkind; |
13004 | vec_alloc (v&: vsize, nelems: map_cnt); |
13005 | vec_alloc (v&: vkind, nelems: map_cnt); |
13006 | unsigned int map_idx = 0; |
13007 | |
13008 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
13009 | switch (OMP_CLAUSE_CODE (c)) |
13010 | { |
13011 | tree ovar, nc, s, purpose, var, x, type; |
13012 | unsigned int talign; |
13013 | |
13014 | default: |
13015 | break; |
13016 | |
13017 | case OMP_CLAUSE_MAP: |
13018 | case OMP_CLAUSE_TO: |
13019 | case OMP_CLAUSE_FROM: |
13020 | oacc_firstprivate_map: |
13021 | nc = c; |
13022 | ovar = OMP_CLAUSE_DECL (c); |
13023 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
13024 | && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER |
13025 | || (OMP_CLAUSE_MAP_KIND (c) |
13026 | == GOMP_MAP_FIRSTPRIVATE_REFERENCE))) |
13027 | break; |
13028 | if (!DECL_P (ovar)) |
13029 | { |
13030 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
13031 | && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)) |
13032 | { |
13033 | nc = OMP_CLAUSE_CHAIN (c); |
13034 | gcc_checking_assert (OMP_CLAUSE_DECL (nc) |
13035 | == get_base_address (ovar)); |
13036 | ovar = OMP_CLAUSE_DECL (nc); |
13037 | } |
13038 | else |
13039 | { |
13040 | tree x = build_sender_ref (var: ovar, ctx); |
13041 | tree v = ovar; |
13042 | if (in_reduction_clauses |
13043 | && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
13044 | && OMP_CLAUSE_MAP_IN_REDUCTION (c)) |
13045 | { |
13046 | v = unshare_expr (v); |
13047 | tree *p = &v; |
13048 | while (handled_component_p (t: *p) |
13049 | || TREE_CODE (*p) == INDIRECT_REF |
13050 | || TREE_CODE (*p) == ADDR_EXPR |
13051 | || TREE_CODE (*p) == MEM_REF |
13052 | || TREE_CODE (*p) == NON_LVALUE_EXPR) |
13053 | p = &TREE_OPERAND (*p, 0); |
13054 | tree d = *p; |
13055 | if (is_variable_sized (expr: d)) |
13056 | { |
13057 | gcc_assert (DECL_HAS_VALUE_EXPR_P (d)); |
13058 | d = DECL_VALUE_EXPR (d); |
13059 | gcc_assert (TREE_CODE (d) == INDIRECT_REF); |
13060 | d = TREE_OPERAND (d, 0); |
13061 | gcc_assert (DECL_P (d)); |
13062 | } |
13063 | splay_tree_key key |
13064 | = (splay_tree_key) &DECL_CONTEXT (d); |
13065 | tree nd = (tree) splay_tree_lookup (ctx->field_map, |
13066 | key)->value; |
13067 | if (d == *p) |
13068 | *p = nd; |
13069 | else |
13070 | *p = build_fold_indirect_ref (nd); |
13071 | } |
13072 | v = build_fold_addr_expr_with_type (v, ptr_type_node); |
13073 | gimplify_assign (x, v, &ilist); |
13074 | nc = NULL_TREE; |
13075 | } |
13076 | } |
13077 | else |
13078 | { |
13079 | if (DECL_SIZE (ovar) |
13080 | && TREE_CODE (DECL_SIZE (ovar)) != INTEGER_CST) |
13081 | { |
13082 | tree ovar2 = DECL_VALUE_EXPR (ovar); |
13083 | gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF); |
13084 | ovar2 = TREE_OPERAND (ovar2, 0); |
13085 | gcc_assert (DECL_P (ovar2)); |
13086 | ovar = ovar2; |
13087 | } |
13088 | if (!maybe_lookup_field (var: ovar, ctx) |
13089 | && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
13090 | && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH |
13091 | || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH))) |
13092 | continue; |
13093 | } |
13094 | |
13095 | talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar)); |
13096 | if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign) |
13097 | talign = DECL_ALIGN_UNIT (ovar); |
13098 | |
13099 | var = NULL_TREE; |
13100 | if (nc) |
13101 | { |
13102 | if (in_reduction_clauses |
13103 | && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
13104 | && OMP_CLAUSE_MAP_IN_REDUCTION (c)) |
13105 | { |
13106 | tree d = ovar; |
13107 | if (is_variable_sized (expr: d)) |
13108 | { |
13109 | gcc_assert (DECL_HAS_VALUE_EXPR_P (d)); |
13110 | d = DECL_VALUE_EXPR (d); |
13111 | gcc_assert (TREE_CODE (d) == INDIRECT_REF); |
13112 | d = TREE_OPERAND (d, 0); |
13113 | gcc_assert (DECL_P (d)); |
13114 | } |
13115 | splay_tree_key key |
13116 | = (splay_tree_key) &DECL_CONTEXT (d); |
13117 | tree nd = (tree) splay_tree_lookup (ctx->field_map, |
13118 | key)->value; |
13119 | if (d == ovar) |
13120 | var = nd; |
13121 | else |
13122 | var = build_fold_indirect_ref (nd); |
13123 | } |
13124 | else |
13125 | var = lookup_decl_in_outer_ctx (decl: ovar, ctx); |
13126 | } |
13127 | if (nc |
13128 | && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
13129 | && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH |
13130 | || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH) |
13131 | && is_omp_target (stmt)) |
13132 | { |
13133 | x = build_sender_ref (var: c, ctx); |
13134 | gimplify_assign (x, build_fold_addr_expr (var), &ilist); |
13135 | } |
13136 | else if (nc) |
13137 | { |
13138 | x = build_sender_ref (var: ovar, ctx); |
13139 | |
13140 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
13141 | && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER |
13142 | && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) |
13143 | && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE) |
13144 | { |
13145 | gcc_assert (offloaded); |
13146 | tree avar |
13147 | = create_tmp_var (TREE_TYPE (TREE_TYPE (x))); |
13148 | mark_addressable (avar); |
13149 | gimplify_assign (avar, build_fold_addr_expr (var), &ilist); |
13150 | talign = DECL_ALIGN_UNIT (avar); |
13151 | avar = build_fold_addr_expr (avar); |
13152 | gimplify_assign (x, avar, &ilist); |
13153 | } |
13154 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE) |
13155 | { |
13156 | gcc_assert (is_gimple_omp_oacc (ctx->stmt)); |
13157 | if (!omp_privatize_by_reference (decl: var)) |
13158 | { |
13159 | if (is_gimple_reg (var) |
13160 | && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c)) |
13161 | suppress_warning (var); |
13162 | var = build_fold_addr_expr (var); |
13163 | } |
13164 | else |
13165 | talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar))); |
13166 | gimplify_assign (x, var, &ilist); |
13167 | } |
13168 | else if (is_gimple_reg (var)) |
13169 | { |
13170 | gcc_assert (offloaded); |
13171 | tree avar = create_tmp_var (TREE_TYPE (var)); |
13172 | mark_addressable (avar); |
13173 | enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c); |
13174 | if (GOMP_MAP_COPY_TO_P (map_kind) |
13175 | || map_kind == GOMP_MAP_POINTER |
13176 | || map_kind == GOMP_MAP_TO_PSET |
13177 | || map_kind == GOMP_MAP_FORCE_DEVICEPTR) |
13178 | { |
13179 | /* If we need to initialize a temporary |
13180 | with VAR because it is not addressable, and |
13181 | the variable hasn't been initialized yet, then |
13182 | we'll get a warning for the store to avar. |
13183 | Don't warn in that case, the mapping might |
13184 | be implicit. */ |
13185 | suppress_warning (var, OPT_Wuninitialized); |
13186 | gimplify_assign (avar, var, &ilist); |
13187 | } |
13188 | avar = build_fold_addr_expr (avar); |
13189 | gimplify_assign (x, avar, &ilist); |
13190 | if ((GOMP_MAP_COPY_FROM_P (map_kind) |
13191 | || map_kind == GOMP_MAP_FORCE_DEVICEPTR) |
13192 | && !TYPE_READONLY (TREE_TYPE (var))) |
13193 | { |
13194 | x = unshare_expr (x); |
13195 | x = build_simple_mem_ref (x); |
13196 | gimplify_assign (var, x, &olist); |
13197 | } |
13198 | } |
13199 | else |
13200 | { |
13201 | /* While MAP is handled explicitly by the FE, |
13202 | for 'target update', only the identified is passed. */ |
13203 | if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FROM |
13204 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TO) |
13205 | && (omp_is_allocatable_or_ptr (decl: var) |
13206 | && omp_check_optional_argument (decl: var, for_present_check: false))) |
13207 | var = build_fold_indirect_ref (var); |
13208 | else if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FROM |
13209 | && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_TO) |
13210 | || (!omp_is_allocatable_or_ptr (decl: var) |
13211 | && !omp_check_optional_argument (decl: var, for_present_check: false))) |
13212 | var = build_fold_addr_expr (var); |
13213 | gimplify_assign (x, var, &ilist); |
13214 | } |
13215 | } |
13216 | s = NULL_TREE; |
13217 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE) |
13218 | { |
13219 | gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt)); |
13220 | s = TREE_TYPE (ovar); |
13221 | if (TREE_CODE (s) == REFERENCE_TYPE |
13222 | || omp_check_optional_argument (decl: ovar, for_present_check: false)) |
13223 | s = TREE_TYPE (s); |
13224 | s = TYPE_SIZE_UNIT (s); |
13225 | } |
13226 | else |
13227 | s = OMP_CLAUSE_SIZE (c); |
13228 | if (s == NULL_TREE) |
13229 | s = TYPE_SIZE_UNIT (TREE_TYPE (ovar)); |
13230 | s = fold_convert (size_type_node, s); |
13231 | purpose = size_int (map_idx++); |
13232 | CONSTRUCTOR_APPEND_ELT (vsize, purpose, s); |
13233 | if (TREE_CODE (s) != INTEGER_CST) |
13234 | TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0; |
13235 | |
13236 | unsigned HOST_WIDE_INT tkind, tkind_zero; |
13237 | switch (OMP_CLAUSE_CODE (c)) |
13238 | { |
13239 | case OMP_CLAUSE_MAP: |
13240 | tkind = OMP_CLAUSE_MAP_KIND (c); |
13241 | tkind_zero = tkind; |
13242 | if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c)) |
13243 | switch (tkind) |
13244 | { |
13245 | case GOMP_MAP_ALLOC: |
13246 | case GOMP_MAP_IF_PRESENT: |
13247 | case GOMP_MAP_TO: |
13248 | case GOMP_MAP_FROM: |
13249 | case GOMP_MAP_TOFROM: |
13250 | case GOMP_MAP_ALWAYS_TO: |
13251 | case GOMP_MAP_ALWAYS_FROM: |
13252 | case GOMP_MAP_ALWAYS_TOFROM: |
13253 | case GOMP_MAP_ALWAYS_PRESENT_TO: |
13254 | case GOMP_MAP_ALWAYS_PRESENT_FROM: |
13255 | case GOMP_MAP_ALWAYS_PRESENT_TOFROM: |
13256 | case GOMP_MAP_RELEASE: |
13257 | case GOMP_MAP_FORCE_TO: |
13258 | case GOMP_MAP_FORCE_FROM: |
13259 | case GOMP_MAP_FORCE_TOFROM: |
13260 | case GOMP_MAP_FORCE_PRESENT: |
13261 | tkind_zero = GOMP_MAP_ZERO_LEN_ARRAY_SECTION; |
13262 | break; |
13263 | case GOMP_MAP_DELETE: |
13264 | tkind_zero = GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION; |
13265 | default: |
13266 | break; |
13267 | } |
13268 | if (tkind_zero != tkind) |
13269 | { |
13270 | if (integer_zerop (s)) |
13271 | tkind = tkind_zero; |
13272 | else if (integer_nonzerop (s)) |
13273 | tkind_zero = tkind; |
13274 | } |
13275 | if (tkind_zero == tkind |
13276 | && OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c) |
13277 | && (((tkind & GOMP_MAP_FLAG_SPECIAL_BITS) |
13278 | & ~GOMP_MAP_IMPLICIT) |
13279 | == 0)) |
13280 | { |
13281 | /* If this is an implicit map, and the GOMP_MAP_IMPLICIT |
13282 | bits are not interfered by other special bit encodings, |
13283 | then turn the GOMP_IMPLICIT_BIT flag on for the runtime |
13284 | to see. */ |
13285 | tkind |= GOMP_MAP_IMPLICIT; |
13286 | tkind_zero = tkind; |
13287 | } |
13288 | break; |
13289 | case OMP_CLAUSE_FIRSTPRIVATE: |
13290 | gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt)); |
13291 | tkind = GOMP_MAP_TO; |
13292 | tkind_zero = tkind; |
13293 | break; |
13294 | case OMP_CLAUSE_TO: |
13295 | tkind |
13296 | = (OMP_CLAUSE_MOTION_PRESENT (c) |
13297 | ? GOMP_MAP_ALWAYS_PRESENT_TO : GOMP_MAP_TO); |
13298 | tkind_zero = tkind; |
13299 | break; |
13300 | case OMP_CLAUSE_FROM: |
13301 | tkind |
13302 | = (OMP_CLAUSE_MOTION_PRESENT (c) |
13303 | ? GOMP_MAP_ALWAYS_PRESENT_FROM : GOMP_MAP_FROM); |
13304 | tkind_zero = tkind; |
13305 | break; |
13306 | default: |
13307 | gcc_unreachable (); |
13308 | } |
13309 | gcc_checking_assert (tkind |
13310 | < (HOST_WIDE_INT_C (1U) << talign_shift)); |
13311 | gcc_checking_assert (tkind_zero |
13312 | < (HOST_WIDE_INT_C (1U) << talign_shift)); |
13313 | talign = ceil_log2 (x: talign); |
13314 | tkind |= talign << talign_shift; |
13315 | tkind_zero |= talign << talign_shift; |
13316 | gcc_checking_assert (tkind |
13317 | <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type))); |
13318 | gcc_checking_assert (tkind_zero |
13319 | <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type))); |
13320 | if (tkind == tkind_zero) |
13321 | x = build_int_cstu (type: tkind_type, tkind); |
13322 | else |
13323 | { |
13324 | TREE_STATIC (TREE_VEC_ELT (t, 2)) = 0; |
13325 | x = build3 (COND_EXPR, tkind_type, |
13326 | fold_build2 (EQ_EXPR, boolean_type_node, |
13327 | unshare_expr (s), size_zero_node), |
13328 | build_int_cstu (type: tkind_type, tkind_zero), |
13329 | build_int_cstu (type: tkind_type, tkind)); |
13330 | } |
13331 | CONSTRUCTOR_APPEND_ELT (vkind, purpose, x); |
13332 | if (nc && nc != c) |
13333 | c = nc; |
13334 | break; |
13335 | |
13336 | case OMP_CLAUSE_FIRSTPRIVATE: |
13337 | omp_has_device_addr_descr: |
13338 | if (is_gimple_omp_oacc (stmt: ctx->stmt)) |
13339 | goto oacc_firstprivate_map; |
13340 | ovar = OMP_CLAUSE_DECL (c); |
13341 | if (omp_privatize_by_reference (decl: ovar)) |
13342 | talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar))); |
13343 | else |
13344 | talign = DECL_ALIGN_UNIT (ovar); |
13345 | var = lookup_decl_in_outer_ctx (decl: ovar, ctx); |
13346 | x = build_sender_ref (var: ovar, ctx); |
13347 | tkind = GOMP_MAP_FIRSTPRIVATE; |
13348 | type = TREE_TYPE (ovar); |
13349 | if (omp_privatize_by_reference (decl: ovar)) |
13350 | type = TREE_TYPE (type); |
13351 | if ((INTEGRAL_TYPE_P (type) |
13352 | && TYPE_PRECISION (type) <= POINTER_SIZE) |
13353 | || TREE_CODE (type) == POINTER_TYPE) |
13354 | { |
13355 | tkind = GOMP_MAP_FIRSTPRIVATE_INT; |
13356 | tree t = var; |
13357 | if (omp_privatize_by_reference (decl: var)) |
13358 | t = build_simple_mem_ref (var); |
13359 | else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c)) |
13360 | suppress_warning (var); |
13361 | if (TREE_CODE (type) != POINTER_TYPE) |
13362 | t = fold_convert (pointer_sized_int_node, t); |
13363 | t = fold_convert (TREE_TYPE (x), t); |
13364 | gimplify_assign (x, t, &ilist); |
13365 | } |
13366 | else if (omp_privatize_by_reference (decl: var)) |
13367 | gimplify_assign (x, var, &ilist); |
13368 | else if (is_gimple_reg (var)) |
13369 | { |
13370 | tree avar = create_tmp_var (TREE_TYPE (var)); |
13371 | mark_addressable (avar); |
13372 | if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c)) |
13373 | suppress_warning (var); |
13374 | gimplify_assign (avar, var, &ilist); |
13375 | avar = build_fold_addr_expr (avar); |
13376 | gimplify_assign (x, avar, &ilist); |
13377 | } |
13378 | else |
13379 | { |
13380 | var = build_fold_addr_expr (var); |
13381 | gimplify_assign (x, var, &ilist); |
13382 | } |
13383 | if (tkind == GOMP_MAP_FIRSTPRIVATE_INT) |
13384 | s = size_int (0); |
13385 | else if (omp_privatize_by_reference (decl: ovar)) |
13386 | s = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar))); |
13387 | else |
13388 | s = TYPE_SIZE_UNIT (TREE_TYPE (ovar)); |
13389 | s = fold_convert (size_type_node, s); |
13390 | purpose = size_int (map_idx++); |
13391 | CONSTRUCTOR_APPEND_ELT (vsize, purpose, s); |
13392 | if (TREE_CODE (s) != INTEGER_CST) |
13393 | TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0; |
13394 | |
13395 | gcc_checking_assert (tkind |
13396 | < (HOST_WIDE_INT_C (1U) << talign_shift)); |
13397 | talign = ceil_log2 (x: talign); |
13398 | tkind |= talign << talign_shift; |
13399 | gcc_checking_assert (tkind |
13400 | <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type))); |
13401 | CONSTRUCTOR_APPEND_ELT (vkind, purpose, |
13402 | build_int_cstu (tkind_type, tkind)); |
13403 | /* Fortran array descriptors: firstprivate of data + attach. */ |
13404 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR |
13405 | && lang_hooks.decls.omp_array_data (ovar, true)) |
13406 | { |
13407 | tree not_null_lb, null_lb, after_lb; |
13408 | tree var1, var2, size1, size2; |
13409 | tree present = omp_check_optional_argument (decl: ovar, for_present_check: true); |
13410 | if (present) |
13411 | { |
13412 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
13413 | not_null_lb = create_artificial_label (clause_loc); |
13414 | null_lb = create_artificial_label (clause_loc); |
13415 | after_lb = create_artificial_label (clause_loc); |
13416 | gimple_seq seq = NULL; |
13417 | present = force_gimple_operand (present, &seq, true, |
13418 | NULL_TREE); |
13419 | gimple_seq_add_seq (&ilist, seq); |
13420 | gimple_seq_add_stmt (&ilist, |
13421 | gimple_build_cond_from_tree (present, |
13422 | not_null_lb, null_lb)); |
13423 | gimple_seq_add_stmt (&ilist, |
13424 | gimple_build_label (label: not_null_lb)); |
13425 | } |
13426 | var1 = lang_hooks.decls.omp_array_data (var, false); |
13427 | size1 = lang_hooks.decls.omp_array_size (var, &ilist); |
13428 | var2 = build_fold_addr_expr (x); |
13429 | if (!POINTER_TYPE_P (TREE_TYPE (var))) |
13430 | var = build_fold_addr_expr (var); |
13431 | size2 = fold_build2 (POINTER_DIFF_EXPR, ssizetype, |
13432 | build_fold_addr_expr (var1), var); |
13433 | size2 = fold_convert (sizetype, size2); |
13434 | if (present) |
13435 | { |
13436 | tree tmp = create_tmp_var (TREE_TYPE (var1)); |
13437 | gimplify_assign (tmp, var1, &ilist); |
13438 | var1 = tmp; |
13439 | tmp = create_tmp_var (TREE_TYPE (var2)); |
13440 | gimplify_assign (tmp, var2, &ilist); |
13441 | var2 = tmp; |
13442 | tmp = create_tmp_var (TREE_TYPE (size1)); |
13443 | gimplify_assign (tmp, size1, &ilist); |
13444 | size1 = tmp; |
13445 | tmp = create_tmp_var (TREE_TYPE (size2)); |
13446 | gimplify_assign (tmp, size2, &ilist); |
13447 | size2 = tmp; |
13448 | gimple_seq_add_stmt (&ilist, gimple_build_goto (dest: after_lb)); |
13449 | gimple_seq_add_stmt (&ilist, gimple_build_label (label: null_lb)); |
13450 | gimplify_assign (var1, null_pointer_node, &ilist); |
13451 | gimplify_assign (var2, null_pointer_node, &ilist); |
13452 | gimplify_assign (size1, size_zero_node, &ilist); |
13453 | gimplify_assign (size2, size_zero_node, &ilist); |
13454 | gimple_seq_add_stmt (&ilist, gimple_build_label (label: after_lb)); |
13455 | } |
13456 | x = build_sender_ref (key: (splay_tree_key) &DECL_NAME (ovar), ctx); |
13457 | gimplify_assign (x, var1, &ilist); |
13458 | tkind = GOMP_MAP_FIRSTPRIVATE; |
13459 | talign = DECL_ALIGN_UNIT (ovar); |
13460 | talign = ceil_log2 (x: talign); |
13461 | tkind |= talign << talign_shift; |
13462 | gcc_checking_assert (tkind |
13463 | <= tree_to_uhwi ( |
13464 | TYPE_MAX_VALUE (tkind_type))); |
13465 | purpose = size_int (map_idx++); |
13466 | CONSTRUCTOR_APPEND_ELT (vsize, purpose, size1); |
13467 | if (TREE_CODE (size1) != INTEGER_CST) |
13468 | TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0; |
13469 | CONSTRUCTOR_APPEND_ELT (vkind, purpose, |
13470 | build_int_cstu (tkind_type, tkind)); |
13471 | x = build_sender_ref (key: (splay_tree_key) &DECL_UID (ovar), ctx); |
13472 | gimplify_assign (x, var2, &ilist); |
13473 | tkind = GOMP_MAP_ATTACH; |
13474 | purpose = size_int (map_idx++); |
13475 | CONSTRUCTOR_APPEND_ELT (vsize, purpose, size2); |
13476 | CONSTRUCTOR_APPEND_ELT (vkind, purpose, |
13477 | build_int_cstu (tkind_type, tkind)); |
13478 | } |
13479 | break; |
13480 | |
13481 | case OMP_CLAUSE_USE_DEVICE_PTR: |
13482 | case OMP_CLAUSE_USE_DEVICE_ADDR: |
13483 | case OMP_CLAUSE_HAS_DEVICE_ADDR: |
13484 | case OMP_CLAUSE_IS_DEVICE_PTR: |
13485 | ovar = OMP_CLAUSE_DECL (c); |
13486 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR) |
13487 | { |
13488 | if (lang_hooks.decls.omp_array_data (ovar, true)) |
13489 | goto omp_has_device_addr_descr; |
13490 | while (TREE_CODE (ovar) == INDIRECT_REF |
13491 | || TREE_CODE (ovar) == ARRAY_REF) |
13492 | ovar = TREE_OPERAND (ovar, 0); |
13493 | } |
13494 | var = lookup_decl_in_outer_ctx (decl: ovar, ctx); |
13495 | |
13496 | if (lang_hooks.decls.omp_array_data (ovar, true)) |
13497 | { |
13498 | tkind = ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR |
13499 | && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR) |
13500 | ? GOMP_MAP_USE_DEVICE_PTR : GOMP_MAP_FIRSTPRIVATE_INT); |
13501 | x = build_sender_ref (key: (splay_tree_key) &DECL_NAME (ovar), ctx); |
13502 | } |
13503 | else if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR |
13504 | && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR) |
13505 | { |
13506 | tkind = GOMP_MAP_USE_DEVICE_PTR; |
13507 | x = build_sender_ref (key: (splay_tree_key) &DECL_UID (ovar), ctx); |
13508 | } |
13509 | else |
13510 | { |
13511 | tkind = GOMP_MAP_FIRSTPRIVATE_INT; |
13512 | x = build_sender_ref (var: ovar, ctx); |
13513 | } |
13514 | |
13515 | if (is_gimple_omp_oacc (stmt: ctx->stmt)) |
13516 | { |
13517 | gcc_assert (tkind == GOMP_MAP_USE_DEVICE_PTR); |
13518 | |
13519 | if (OMP_CLAUSE_USE_DEVICE_PTR_IF_PRESENT (c)) |
13520 | tkind = GOMP_MAP_USE_DEVICE_PTR_IF_PRESENT; |
13521 | } |
13522 | |
13523 | type = TREE_TYPE (ovar); |
13524 | if (lang_hooks.decls.omp_array_data (ovar, true)) |
13525 | var = lang_hooks.decls.omp_array_data (var, false); |
13526 | else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR |
13527 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR) |
13528 | && !omp_privatize_by_reference (decl: ovar) |
13529 | && !omp_is_allocatable_or_ptr (decl: ovar)) |
13530 | || TREE_CODE (type) == ARRAY_TYPE) |
13531 | var = build_fold_addr_expr (var); |
13532 | else |
13533 | { |
13534 | if (omp_privatize_by_reference (decl: ovar) |
13535 | || omp_check_optional_argument (decl: ovar, for_present_check: false) |
13536 | || omp_is_allocatable_or_ptr (decl: ovar)) |
13537 | { |
13538 | type = TREE_TYPE (type); |
13539 | if (POINTER_TYPE_P (type) |
13540 | && TREE_CODE (type) != ARRAY_TYPE |
13541 | && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR |
13542 | && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR |
13543 | && !omp_is_allocatable_or_ptr (decl: ovar)) |
13544 | || (omp_privatize_by_reference (decl: ovar) |
13545 | && omp_is_allocatable_or_ptr (decl: ovar)))) |
13546 | var = build_simple_mem_ref (var); |
13547 | var = fold_convert (TREE_TYPE (x), var); |
13548 | } |
13549 | } |
13550 | tree present; |
13551 | present = omp_check_optional_argument (decl: ovar, for_present_check: true); |
13552 | if (present) |
13553 | { |
13554 | tree null_label = create_artificial_label (UNKNOWN_LOCATION); |
13555 | tree notnull_label = create_artificial_label (UNKNOWN_LOCATION); |
13556 | tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION); |
13557 | tree new_x = unshare_expr (x); |
13558 | gimplify_expr (&present, &ilist, NULL, is_gimple_val, |
13559 | fb_rvalue); |
13560 | gcond *cond = gimple_build_cond_from_tree (present, |
13561 | notnull_label, |
13562 | null_label); |
13563 | gimple_seq_add_stmt (&ilist, cond); |
13564 | gimple_seq_add_stmt (&ilist, gimple_build_label (label: null_label)); |
13565 | gimplify_assign (new_x, null_pointer_node, &ilist); |
13566 | gimple_seq_add_stmt (&ilist, gimple_build_goto (dest: opt_arg_label)); |
13567 | gimple_seq_add_stmt (&ilist, |
13568 | gimple_build_label (label: notnull_label)); |
13569 | gimplify_assign (x, var, &ilist); |
13570 | gimple_seq_add_stmt (&ilist, |
13571 | gimple_build_label (label: opt_arg_label)); |
13572 | } |
13573 | else |
13574 | gimplify_assign (x, var, &ilist); |
13575 | s = size_int (0); |
13576 | purpose = size_int (map_idx++); |
13577 | CONSTRUCTOR_APPEND_ELT (vsize, purpose, s); |
13578 | gcc_checking_assert (tkind |
13579 | < (HOST_WIDE_INT_C (1U) << talign_shift)); |
13580 | gcc_checking_assert (tkind |
13581 | <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type))); |
13582 | CONSTRUCTOR_APPEND_ELT (vkind, purpose, |
13583 | build_int_cstu (tkind_type, tkind)); |
13584 | break; |
13585 | } |
13586 | |
13587 | gcc_assert (map_idx == map_cnt); |
13588 | |
13589 | DECL_INITIAL (TREE_VEC_ELT (t, 1)) |
13590 | = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize); |
13591 | DECL_INITIAL (TREE_VEC_ELT (t, 2)) |
13592 | = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind); |
13593 | for (int i = 1; i <= 2; i++) |
13594 | if (!TREE_STATIC (TREE_VEC_ELT (t, i))) |
13595 | { |
13596 | gimple_seq initlist = NULL; |
13597 | force_gimple_operand (build1 (DECL_EXPR, void_type_node, |
13598 | TREE_VEC_ELT (t, i)), |
13599 | &initlist, true, NULL_TREE); |
13600 | gimple_seq_add_seq (&ilist, initlist); |
13601 | |
13602 | tree clobber = build_clobber (TREE_TYPE (TREE_VEC_ELT (t, i))); |
13603 | gimple_seq_add_stmt (&olist, |
13604 | gimple_build_assign (TREE_VEC_ELT (t, i), |
13605 | clobber)); |
13606 | } |
13607 | else if (omp_maybe_offloaded_ctx (ctx: ctx->outer)) |
13608 | { |
13609 | tree id = get_identifier ("omp declare target" ); |
13610 | tree decl = TREE_VEC_ELT (t, i); |
13611 | DECL_ATTRIBUTES (decl) |
13612 | = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (decl)); |
13613 | varpool_node *node = varpool_node::get (decl); |
13614 | if (node) |
13615 | { |
13616 | node->offloadable = 1; |
13617 | if (ENABLE_OFFLOADING) |
13618 | { |
13619 | g->have_offload = true; |
13620 | vec_safe_push (v&: offload_vars, obj: t); |
13621 | } |
13622 | } |
13623 | } |
13624 | |
13625 | tree clobber = build_clobber (ctx->record_type); |
13626 | gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl, |
13627 | clobber)); |
13628 | } |
13629 | |
13630 | /* Once all the expansions are done, sequence all the different |
13631 | fragments inside gimple_omp_body. */ |
13632 | |
13633 | new_body = NULL; |
13634 | |
13635 | if (offloaded |
13636 | && ctx->record_type) |
13637 | { |
13638 | t = build_fold_addr_expr_loc (loc, ctx->sender_decl); |
13639 | /* fixup_child_record_type might have changed receiver_decl's type. */ |
13640 | t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t); |
13641 | gimple_seq_add_stmt (&new_body, |
13642 | gimple_build_assign (ctx->receiver_decl, t)); |
13643 | } |
13644 | gimple_seq_add_seq (&new_body, fplist); |
13645 | |
13646 | if (offloaded || data_region) |
13647 | { |
13648 | tree prev = NULL_TREE; |
13649 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
13650 | switch (OMP_CLAUSE_CODE (c)) |
13651 | { |
13652 | tree var, x; |
13653 | default: |
13654 | break; |
13655 | case OMP_CLAUSE_FIRSTPRIVATE: |
13656 | omp_firstprivatize_data_region: |
13657 | if (is_gimple_omp_oacc (stmt: ctx->stmt)) |
13658 | break; |
13659 | var = OMP_CLAUSE_DECL (c); |
13660 | if (omp_privatize_by_reference (decl: var) |
13661 | || is_gimple_reg_type (TREE_TYPE (var))) |
13662 | { |
13663 | tree new_var = lookup_decl (var, ctx); |
13664 | tree type; |
13665 | type = TREE_TYPE (var); |
13666 | if (omp_privatize_by_reference (decl: var)) |
13667 | type = TREE_TYPE (type); |
13668 | if ((INTEGRAL_TYPE_P (type) |
13669 | && TYPE_PRECISION (type) <= POINTER_SIZE) |
13670 | || TREE_CODE (type) == POINTER_TYPE) |
13671 | { |
13672 | x = build_receiver_ref (var, by_ref: false, ctx); |
13673 | if (TREE_CODE (type) != POINTER_TYPE) |
13674 | x = fold_convert (pointer_sized_int_node, x); |
13675 | x = fold_convert (type, x); |
13676 | gimplify_expr (&x, &new_body, NULL, is_gimple_val, |
13677 | fb_rvalue); |
13678 | if (omp_privatize_by_reference (decl: var)) |
13679 | { |
13680 | tree v = create_tmp_var_raw (type, get_name (var)); |
13681 | gimple_add_tmp_var (v); |
13682 | TREE_ADDRESSABLE (v) = 1; |
13683 | gimple_seq_add_stmt (&new_body, |
13684 | gimple_build_assign (v, x)); |
13685 | x = build_fold_addr_expr (v); |
13686 | } |
13687 | gimple_seq_add_stmt (&new_body, |
13688 | gimple_build_assign (new_var, x)); |
13689 | } |
13690 | else |
13691 | { |
13692 | bool by_ref = !omp_privatize_by_reference (decl: var); |
13693 | x = build_receiver_ref (var, by_ref, ctx); |
13694 | gimplify_expr (&x, &new_body, NULL, is_gimple_val, |
13695 | fb_rvalue); |
13696 | gimple_seq_add_stmt (&new_body, |
13697 | gimple_build_assign (new_var, x)); |
13698 | } |
13699 | } |
13700 | else if (is_variable_sized (expr: var)) |
13701 | { |
13702 | tree pvar = DECL_VALUE_EXPR (var); |
13703 | gcc_assert (TREE_CODE (pvar) == INDIRECT_REF); |
13704 | pvar = TREE_OPERAND (pvar, 0); |
13705 | gcc_assert (DECL_P (pvar)); |
13706 | tree new_var = lookup_decl (var: pvar, ctx); |
13707 | x = build_receiver_ref (var, by_ref: false, ctx); |
13708 | gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue); |
13709 | gimple_seq_add_stmt (&new_body, |
13710 | gimple_build_assign (new_var, x)); |
13711 | } |
13712 | break; |
13713 | case OMP_CLAUSE_PRIVATE: |
13714 | if (is_gimple_omp_oacc (stmt: ctx->stmt)) |
13715 | break; |
13716 | var = OMP_CLAUSE_DECL (c); |
13717 | if (omp_privatize_by_reference (decl: var)) |
13718 | { |
13719 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
13720 | tree new_var = lookup_decl (var, ctx); |
13721 | x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var))); |
13722 | if (TREE_CONSTANT (x)) |
13723 | { |
13724 | x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)), |
13725 | get_name (var)); |
13726 | gimple_add_tmp_var (x); |
13727 | TREE_ADDRESSABLE (x) = 1; |
13728 | x = build_fold_addr_expr_loc (clause_loc, x); |
13729 | } |
13730 | else |
13731 | break; |
13732 | |
13733 | x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x); |
13734 | gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue); |
13735 | gimple_seq_add_stmt (&new_body, |
13736 | gimple_build_assign (new_var, x)); |
13737 | } |
13738 | break; |
13739 | case OMP_CLAUSE_USE_DEVICE_PTR: |
13740 | case OMP_CLAUSE_USE_DEVICE_ADDR: |
13741 | case OMP_CLAUSE_HAS_DEVICE_ADDR: |
13742 | case OMP_CLAUSE_IS_DEVICE_PTR: |
13743 | tree new_var; |
13744 | gimple_seq assign_body; |
13745 | bool is_array_data; |
13746 | bool do_optional_check; |
13747 | assign_body = NULL; |
13748 | do_optional_check = false; |
13749 | var = OMP_CLAUSE_DECL (c); |
13750 | is_array_data = lang_hooks.decls.omp_array_data (var, true) != NULL; |
13751 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR && is_array_data) |
13752 | goto omp_firstprivatize_data_region; |
13753 | |
13754 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR |
13755 | && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR) |
13756 | x = build_sender_ref (key: is_array_data |
13757 | ? (splay_tree_key) &DECL_NAME (var) |
13758 | : (splay_tree_key) &DECL_UID (var), ctx); |
13759 | else |
13760 | { |
13761 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR) |
13762 | { |
13763 | while (TREE_CODE (var) == INDIRECT_REF |
13764 | || TREE_CODE (var) == ARRAY_REF) |
13765 | var = TREE_OPERAND (var, 0); |
13766 | } |
13767 | x = build_receiver_ref (var, by_ref: false, ctx); |
13768 | } |
13769 | |
13770 | if (is_array_data) |
13771 | { |
13772 | bool is_ref = omp_privatize_by_reference (decl: var); |
13773 | do_optional_check = true; |
13774 | /* First, we copy the descriptor data from the host; then |
13775 | we update its data to point to the target address. */ |
13776 | new_var = lookup_decl (var, ctx); |
13777 | new_var = DECL_VALUE_EXPR (new_var); |
13778 | tree v = new_var; |
13779 | tree v2 = var; |
13780 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR |
13781 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR) |
13782 | v2 = maybe_lookup_decl_in_outer_ctx (decl: var, ctx); |
13783 | |
13784 | if (is_ref) |
13785 | { |
13786 | v2 = build_fold_indirect_ref (v2); |
13787 | v = create_tmp_var_raw (TREE_TYPE (v2), get_name (var)); |
13788 | gimple_add_tmp_var (v); |
13789 | TREE_ADDRESSABLE (v) = 1; |
13790 | gimplify_assign (v, v2, &assign_body); |
13791 | tree rhs = build_fold_addr_expr (v); |
13792 | gimple_seq_add_stmt (&assign_body, |
13793 | gimple_build_assign (new_var, rhs)); |
13794 | } |
13795 | else |
13796 | gimplify_assign (new_var, v2, &assign_body); |
13797 | |
13798 | v2 = lang_hooks.decls.omp_array_data (unshare_expr (v), false); |
13799 | gcc_assert (v2); |
13800 | gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue); |
13801 | gimple_seq_add_stmt (&assign_body, |
13802 | gimple_build_assign (v2, x)); |
13803 | } |
13804 | else if (is_variable_sized (expr: var)) |
13805 | { |
13806 | tree pvar = DECL_VALUE_EXPR (var); |
13807 | gcc_assert (TREE_CODE (pvar) == INDIRECT_REF); |
13808 | pvar = TREE_OPERAND (pvar, 0); |
13809 | gcc_assert (DECL_P (pvar)); |
13810 | new_var = lookup_decl (var: pvar, ctx); |
13811 | gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue); |
13812 | gimple_seq_add_stmt (&assign_body, |
13813 | gimple_build_assign (new_var, x)); |
13814 | } |
13815 | else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR |
13816 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR) |
13817 | && !omp_privatize_by_reference (decl: var) |
13818 | && !omp_is_allocatable_or_ptr (decl: var)) |
13819 | || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE) |
13820 | { |
13821 | new_var = lookup_decl (var, ctx); |
13822 | new_var = DECL_VALUE_EXPR (new_var); |
13823 | gcc_assert (TREE_CODE (new_var) == MEM_REF); |
13824 | new_var = TREE_OPERAND (new_var, 0); |
13825 | gcc_assert (DECL_P (new_var)); |
13826 | gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue); |
13827 | gimple_seq_add_stmt (&assign_body, |
13828 | gimple_build_assign (new_var, x)); |
13829 | } |
13830 | else |
13831 | { |
13832 | tree type = TREE_TYPE (var); |
13833 | new_var = lookup_decl (var, ctx); |
13834 | if (omp_privatize_by_reference (decl: var)) |
13835 | { |
13836 | type = TREE_TYPE (type); |
13837 | if (POINTER_TYPE_P (type) |
13838 | && TREE_CODE (type) != ARRAY_TYPE |
13839 | && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR |
13840 | && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR) |
13841 | || (omp_privatize_by_reference (decl: var) |
13842 | && omp_is_allocatable_or_ptr (decl: var)))) |
13843 | { |
13844 | tree v = create_tmp_var_raw (type, get_name (var)); |
13845 | gimple_add_tmp_var (v); |
13846 | TREE_ADDRESSABLE (v) = 1; |
13847 | x = fold_convert (type, x); |
13848 | gimplify_expr (&x, &assign_body, NULL, is_gimple_val, |
13849 | fb_rvalue); |
13850 | gimple_seq_add_stmt (&assign_body, |
13851 | gimple_build_assign (v, x)); |
13852 | x = build_fold_addr_expr (v); |
13853 | do_optional_check = true; |
13854 | } |
13855 | } |
13856 | new_var = DECL_VALUE_EXPR (new_var); |
13857 | x = fold_convert (TREE_TYPE (new_var), x); |
13858 | gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue); |
13859 | gimple_seq_add_stmt (&assign_body, |
13860 | gimple_build_assign (new_var, x)); |
13861 | } |
13862 | tree present; |
13863 | present = ((do_optional_check |
13864 | && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR |
13865 | && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR) |
13866 | ? omp_check_optional_argument (OMP_CLAUSE_DECL (c), for_present_check: true) |
13867 | : NULL_TREE); |
13868 | if (present) |
13869 | { |
13870 | tree null_label = create_artificial_label (UNKNOWN_LOCATION); |
13871 | tree notnull_label = create_artificial_label (UNKNOWN_LOCATION); |
13872 | tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION); |
13873 | glabel *null_glabel = gimple_build_label (label: null_label); |
13874 | glabel *notnull_glabel = gimple_build_label (label: notnull_label); |
13875 | ggoto *opt_arg_ggoto = gimple_build_goto (dest: opt_arg_label); |
13876 | gimplify_expr (&x, &new_body, NULL, is_gimple_val, |
13877 | fb_rvalue); |
13878 | gimplify_expr (&present, &new_body, NULL, is_gimple_val, |
13879 | fb_rvalue); |
13880 | gcond *cond = gimple_build_cond_from_tree (present, |
13881 | notnull_label, |
13882 | null_label); |
13883 | gimple_seq_add_stmt (&new_body, cond); |
13884 | gimple_seq_add_stmt (&new_body, null_glabel); |
13885 | gimplify_assign (new_var, null_pointer_node, &new_body); |
13886 | gimple_seq_add_stmt (&new_body, opt_arg_ggoto); |
13887 | gimple_seq_add_stmt (&new_body, notnull_glabel); |
13888 | gimple_seq_add_seq (&new_body, assign_body); |
13889 | gimple_seq_add_stmt (&new_body, |
13890 | gimple_build_label (label: opt_arg_label)); |
13891 | } |
13892 | else |
13893 | gimple_seq_add_seq (&new_body, assign_body); |
13894 | break; |
13895 | } |
13896 | /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass, |
13897 | so that firstprivate vars holding OMP_CLAUSE_SIZE if needed |
13898 | are already handled. Similarly OMP_CLAUSE_PRIVATE for VLAs |
13899 | or references to VLAs. */ |
13900 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
13901 | switch (OMP_CLAUSE_CODE (c)) |
13902 | { |
13903 | tree var; |
13904 | default: |
13905 | break; |
13906 | case OMP_CLAUSE_MAP: |
13907 | if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER |
13908 | || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE) |
13909 | { |
13910 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
13911 | poly_int64 offset = 0; |
13912 | gcc_assert (prev); |
13913 | var = OMP_CLAUSE_DECL (c); |
13914 | if (DECL_P (var) |
13915 | && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE |
13916 | && is_global_var (t: maybe_lookup_decl_in_outer_ctx (decl: var, |
13917 | ctx)) |
13918 | && varpool_node::get_create (decl: var)->offloadable) |
13919 | break; |
13920 | if (TREE_CODE (var) == INDIRECT_REF |
13921 | && TREE_CODE (TREE_OPERAND (var, 0)) == COMPONENT_REF) |
13922 | var = TREE_OPERAND (var, 0); |
13923 | if (TREE_CODE (var) == COMPONENT_REF) |
13924 | { |
13925 | var = get_addr_base_and_unit_offset (var, &offset); |
13926 | gcc_assert (var != NULL_TREE && DECL_P (var)); |
13927 | } |
13928 | else if (DECL_SIZE (var) |
13929 | && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST) |
13930 | { |
13931 | tree var2 = DECL_VALUE_EXPR (var); |
13932 | gcc_assert (TREE_CODE (var2) == INDIRECT_REF); |
13933 | var2 = TREE_OPERAND (var2, 0); |
13934 | gcc_assert (DECL_P (var2)); |
13935 | var = var2; |
13936 | } |
13937 | tree new_var = lookup_decl (var, ctx), x; |
13938 | tree type = TREE_TYPE (new_var); |
13939 | bool is_ref; |
13940 | if (TREE_CODE (OMP_CLAUSE_DECL (c)) == INDIRECT_REF |
13941 | && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0)) |
13942 | == COMPONENT_REF)) |
13943 | { |
13944 | type = TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0)); |
13945 | is_ref = true; |
13946 | new_var = build2 (MEM_REF, type, |
13947 | build_fold_addr_expr (new_var), |
13948 | build_int_cst (build_pointer_type (type), |
13949 | offset)); |
13950 | } |
13951 | else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPONENT_REF) |
13952 | { |
13953 | type = TREE_TYPE (OMP_CLAUSE_DECL (c)); |
13954 | is_ref = TREE_CODE (type) == REFERENCE_TYPE; |
13955 | new_var = build2 (MEM_REF, type, |
13956 | build_fold_addr_expr (new_var), |
13957 | build_int_cst (build_pointer_type (type), |
13958 | offset)); |
13959 | } |
13960 | else |
13961 | is_ref = omp_privatize_by_reference (decl: var); |
13962 | if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE) |
13963 | is_ref = false; |
13964 | bool ref_to_array = false; |
13965 | bool ref_to_ptr = false; |
13966 | if (is_ref) |
13967 | { |
13968 | type = TREE_TYPE (type); |
13969 | if (TREE_CODE (type) == ARRAY_TYPE) |
13970 | { |
13971 | type = build_pointer_type (type); |
13972 | ref_to_array = true; |
13973 | } |
13974 | } |
13975 | else if (TREE_CODE (type) == ARRAY_TYPE) |
13976 | { |
13977 | tree decl2 = DECL_VALUE_EXPR (new_var); |
13978 | gcc_assert (TREE_CODE (decl2) == MEM_REF); |
13979 | decl2 = TREE_OPERAND (decl2, 0); |
13980 | gcc_assert (DECL_P (decl2)); |
13981 | new_var = decl2; |
13982 | type = TREE_TYPE (new_var); |
13983 | } |
13984 | else if (TREE_CODE (type) == REFERENCE_TYPE |
13985 | && TREE_CODE (TREE_TYPE (type)) == POINTER_TYPE) |
13986 | { |
13987 | type = TREE_TYPE (type); |
13988 | ref_to_ptr = true; |
13989 | } |
13990 | x = build_receiver_ref (OMP_CLAUSE_DECL (prev), by_ref: false, ctx); |
13991 | x = fold_convert_loc (clause_loc, type, x); |
13992 | if (!integer_zerop (OMP_CLAUSE_SIZE (c))) |
13993 | { |
13994 | tree bias = OMP_CLAUSE_SIZE (c); |
13995 | if (DECL_P (bias)) |
13996 | bias = lookup_decl (var: bias, ctx); |
13997 | bias = fold_convert_loc (clause_loc, sizetype, bias); |
13998 | bias = fold_build1_loc (clause_loc, NEGATE_EXPR, sizetype, |
13999 | bias); |
14000 | x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR, |
14001 | TREE_TYPE (x), x, bias); |
14002 | } |
14003 | if (ref_to_array) |
14004 | x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x); |
14005 | gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue); |
14006 | if ((is_ref && !ref_to_array) |
14007 | || ref_to_ptr) |
14008 | { |
14009 | tree t = create_tmp_var_raw (type, get_name (var)); |
14010 | gimple_add_tmp_var (t); |
14011 | TREE_ADDRESSABLE (t) = 1; |
14012 | gimple_seq_add_stmt (&new_body, |
14013 | gimple_build_assign (t, x)); |
14014 | x = build_fold_addr_expr_loc (clause_loc, t); |
14015 | } |
14016 | gimple_seq_add_stmt (&new_body, |
14017 | gimple_build_assign (new_var, x)); |
14018 | prev = NULL_TREE; |
14019 | } |
14020 | else if (OMP_CLAUSE_CHAIN (c) |
14021 | && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c)) |
14022 | == OMP_CLAUSE_MAP |
14023 | && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c)) |
14024 | == GOMP_MAP_FIRSTPRIVATE_POINTER |
14025 | || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c)) |
14026 | == GOMP_MAP_FIRSTPRIVATE_REFERENCE))) |
14027 | prev = c; |
14028 | break; |
14029 | case OMP_CLAUSE_PRIVATE: |
14030 | var = OMP_CLAUSE_DECL (c); |
14031 | if (is_variable_sized (expr: var)) |
14032 | { |
14033 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
14034 | tree new_var = lookup_decl (var, ctx); |
14035 | tree pvar = DECL_VALUE_EXPR (var); |
14036 | gcc_assert (TREE_CODE (pvar) == INDIRECT_REF); |
14037 | pvar = TREE_OPERAND (pvar, 0); |
14038 | gcc_assert (DECL_P (pvar)); |
14039 | tree new_pvar = lookup_decl (var: pvar, ctx); |
14040 | tree atmp = builtin_decl_explicit (fncode: BUILT_IN_ALLOCA_WITH_ALIGN); |
14041 | tree al = size_int (DECL_ALIGN (var)); |
14042 | tree x = TYPE_SIZE_UNIT (TREE_TYPE (new_var)); |
14043 | x = build_call_expr_loc (clause_loc, atmp, 2, x, al); |
14044 | x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar), x); |
14045 | gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue); |
14046 | gimple_seq_add_stmt (&new_body, |
14047 | gimple_build_assign (new_pvar, x)); |
14048 | } |
14049 | else if (omp_privatize_by_reference (decl: var) |
14050 | && !is_gimple_omp_oacc (stmt: ctx->stmt)) |
14051 | { |
14052 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
14053 | tree new_var = lookup_decl (var, ctx); |
14054 | tree x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var))); |
14055 | if (TREE_CONSTANT (x)) |
14056 | break; |
14057 | else |
14058 | { |
14059 | tree atmp |
14060 | = builtin_decl_explicit (fncode: BUILT_IN_ALLOCA_WITH_ALIGN); |
14061 | tree rtype = TREE_TYPE (TREE_TYPE (new_var)); |
14062 | tree al = size_int (TYPE_ALIGN (rtype)); |
14063 | x = build_call_expr_loc (clause_loc, atmp, 2, x, al); |
14064 | } |
14065 | |
14066 | x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x); |
14067 | gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue); |
14068 | gimple_seq_add_stmt (&new_body, |
14069 | gimple_build_assign (new_var, x)); |
14070 | } |
14071 | break; |
14072 | } |
14073 | |
14074 | gimple_seq fork_seq = NULL; |
14075 | gimple_seq join_seq = NULL; |
14076 | |
14077 | if (offloaded && is_gimple_omp_oacc (stmt: ctx->stmt)) |
14078 | { |
14079 | /* If there are reductions on the offloaded region itself, treat |
14080 | them as a dummy GANG loop. */ |
14081 | tree level = build_int_cst (integer_type_node, GOMP_DIM_GANG); |
14082 | |
14083 | gcall *private_marker = lower_oacc_private_marker (ctx); |
14084 | |
14085 | if (private_marker) |
14086 | gimple_call_set_arg (gs: private_marker, index: 2, arg: level); |
14087 | |
14088 | lower_oacc_reductions (loc: gimple_location (g: ctx->stmt), clauses, level, |
14089 | inner: false, NULL, private_marker, NULL, fork_seq: &fork_seq, |
14090 | join_seq: &join_seq, ctx); |
14091 | } |
14092 | |
14093 | gimple_seq_add_seq (&new_body, fork_seq); |
14094 | gimple_seq_add_seq (&new_body, tgt_body); |
14095 | gimple_seq_add_seq (&new_body, join_seq); |
14096 | |
14097 | if (offloaded) |
14098 | { |
14099 | new_body = maybe_catch_exception (body: new_body); |
14100 | gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false)); |
14101 | } |
14102 | gimple_omp_set_body (gs: stmt, body: new_body); |
14103 | } |
14104 | |
14105 | bind = gimple_build_bind (NULL, NULL, |
14106 | tgt_bind ? gimple_bind_block (bind_stmt: tgt_bind) |
14107 | : NULL_TREE); |
14108 | gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true); |
14109 | gimple_bind_add_seq (bind_stmt: bind, seq: ilist); |
14110 | gimple_bind_add_stmt (bind_stmt: bind, stmt); |
14111 | gimple_bind_add_seq (bind_stmt: bind, seq: olist); |
14112 | |
14113 | pop_gimplify_context (NULL); |
14114 | |
14115 | if (dep_bind) |
14116 | { |
14117 | gimple_bind_add_seq (bind_stmt: dep_bind, seq: dep_ilist); |
14118 | gimple_bind_add_stmt (bind_stmt: dep_bind, stmt: bind); |
14119 | gimple_bind_add_seq (bind_stmt: dep_bind, seq: dep_olist); |
14120 | pop_gimplify_context (dep_bind); |
14121 | } |
14122 | } |
14123 | |
14124 | /* Expand code for an OpenMP teams directive. */ |
14125 | |
14126 | static void |
14127 | lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
14128 | { |
14129 | gomp_teams *teams_stmt = as_a <gomp_teams *> (p: gsi_stmt (i: *gsi_p)); |
14130 | push_gimplify_context (); |
14131 | |
14132 | tree block = make_node (BLOCK); |
14133 | gbind *bind = gimple_build_bind (NULL, NULL, block); |
14134 | gsi_replace (gsi_p, bind, true); |
14135 | gimple_seq bind_body = NULL; |
14136 | gimple_seq dlist = NULL; |
14137 | gimple_seq olist = NULL; |
14138 | |
14139 | tree num_teams = omp_find_clause (clauses: gimple_omp_teams_clauses (gs: teams_stmt), |
14140 | kind: OMP_CLAUSE_NUM_TEAMS); |
14141 | tree num_teams_lower = NULL_TREE; |
14142 | if (num_teams == NULL_TREE) |
14143 | num_teams = build_int_cst (unsigned_type_node, 0); |
14144 | else |
14145 | { |
14146 | num_teams_lower = OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (num_teams); |
14147 | if (num_teams_lower) |
14148 | { |
14149 | num_teams_lower = fold_convert (unsigned_type_node, num_teams_lower); |
14150 | gimplify_expr (&num_teams_lower, &bind_body, NULL, is_gimple_val, |
14151 | fb_rvalue); |
14152 | } |
14153 | num_teams = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (num_teams); |
14154 | num_teams = fold_convert (unsigned_type_node, num_teams); |
14155 | gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue); |
14156 | } |
14157 | if (num_teams_lower == NULL_TREE) |
14158 | num_teams_lower = num_teams; |
14159 | tree thread_limit = omp_find_clause (clauses: gimple_omp_teams_clauses (gs: teams_stmt), |
14160 | kind: OMP_CLAUSE_THREAD_LIMIT); |
14161 | if (thread_limit == NULL_TREE) |
14162 | thread_limit = build_int_cst (unsigned_type_node, 0); |
14163 | else |
14164 | { |
14165 | thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit); |
14166 | thread_limit = fold_convert (unsigned_type_node, thread_limit); |
14167 | gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val, |
14168 | fb_rvalue); |
14169 | } |
14170 | location_t loc = gimple_location (g: teams_stmt); |
14171 | tree decl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_TEAMS4); |
14172 | tree rettype = TREE_TYPE (TREE_TYPE (decl)); |
14173 | tree first = create_tmp_var (rettype); |
14174 | gimple_seq_add_stmt (&bind_body, |
14175 | gimple_build_assign (first, build_one_cst (rettype))); |
14176 | tree llabel = create_artificial_label (loc); |
14177 | gimple_seq_add_stmt (&bind_body, gimple_build_label (label: llabel)); |
14178 | gimple *call |
14179 | = gimple_build_call (decl, 4, num_teams_lower, num_teams, thread_limit, |
14180 | first); |
14181 | gimple_set_location (g: call, location: loc); |
14182 | tree temp = create_tmp_var (rettype); |
14183 | gimple_call_set_lhs (gs: call, lhs: temp); |
14184 | gimple_seq_add_stmt (&bind_body, call); |
14185 | |
14186 | tree tlabel = create_artificial_label (loc); |
14187 | tree flabel = create_artificial_label (loc); |
14188 | gimple *cond = gimple_build_cond (NE_EXPR, temp, build_zero_cst (rettype), |
14189 | tlabel, flabel); |
14190 | gimple_seq_add_stmt (&bind_body, cond); |
14191 | gimple_seq_add_stmt (&bind_body, gimple_build_label (label: tlabel)); |
14192 | gimple_seq_add_stmt (&bind_body, |
14193 | gimple_build_assign (first, build_zero_cst (rettype))); |
14194 | |
14195 | lower_rec_input_clauses (clauses: gimple_omp_teams_clauses (gs: teams_stmt), |
14196 | ilist: &bind_body, dlist: &dlist, ctx, NULL); |
14197 | lower_omp (gimple_omp_body_ptr (gs: teams_stmt), ctx); |
14198 | lower_reduction_clauses (clauses: gimple_omp_teams_clauses (gs: teams_stmt), stmt_seqp: &olist, |
14199 | NULL, ctx); |
14200 | gimple_seq_add_stmt (&bind_body, teams_stmt); |
14201 | |
14202 | gimple_seq_add_seq (&bind_body, gimple_omp_body (gs: teams_stmt)); |
14203 | gimple_omp_set_body (gs: teams_stmt, NULL); |
14204 | gimple_seq_add_seq (&bind_body, olist); |
14205 | gimple_seq_add_seq (&bind_body, dlist); |
14206 | gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true)); |
14207 | gimple_seq_add_stmt (&bind_body, gimple_build_goto (dest: llabel)); |
14208 | gimple_seq_add_stmt (&bind_body, gimple_build_label (label: flabel)); |
14209 | gimple_bind_set_body (bind_stmt: bind, seq: bind_body); |
14210 | |
14211 | pop_gimplify_context (bind); |
14212 | |
14213 | gimple_bind_append_vars (bind_stmt: bind, vars: ctx->block_vars); |
14214 | BLOCK_VARS (block) = ctx->block_vars; |
14215 | if (BLOCK_VARS (block)) |
14216 | TREE_USED (block) = 1; |
14217 | } |
14218 | |
14219 | /* Callback for lower_omp_1. Return non-NULL if *tp needs to be |
14220 | regimplified. If DATA is non-NULL, lower_omp_1 is outside |
14221 | of OMP context, but with make_addressable_vars set. */ |
14222 | |
14223 | static tree |
14224 | lower_omp_regimplify_p (tree *tp, int *walk_subtrees, |
14225 | void *data) |
14226 | { |
14227 | tree t = *tp; |
14228 | |
14229 | /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */ |
14230 | if ((VAR_P (t) || TREE_CODE (t) == PARM_DECL || TREE_CODE (t) == RESULT_DECL) |
14231 | && data == NULL |
14232 | && DECL_HAS_VALUE_EXPR_P (t)) |
14233 | return t; |
14234 | |
14235 | if (make_addressable_vars |
14236 | && DECL_P (t) |
14237 | && bitmap_bit_p (make_addressable_vars, DECL_UID (t))) |
14238 | return t; |
14239 | |
14240 | /* If a global variable has been privatized, TREE_CONSTANT on |
14241 | ADDR_EXPR might be wrong. */ |
14242 | if (data == NULL && TREE_CODE (t) == ADDR_EXPR) |
14243 | recompute_tree_invariant_for_addr_expr (t); |
14244 | |
14245 | *walk_subtrees = !IS_TYPE_OR_DECL_P (t); |
14246 | return NULL_TREE; |
14247 | } |
14248 | |
14249 | /* Data to be communicated between lower_omp_regimplify_operands and |
14250 | lower_omp_regimplify_operands_p. */ |
14251 | |
14252 | struct lower_omp_regimplify_operands_data |
14253 | { |
14254 | omp_context *ctx; |
14255 | vec<tree> *decls; |
14256 | }; |
14257 | |
14258 | /* Helper function for lower_omp_regimplify_operands. Find |
14259 | omp_member_access_dummy_var vars and adjust temporarily their |
14260 | DECL_VALUE_EXPRs if needed. */ |
14261 | |
14262 | static tree |
14263 | lower_omp_regimplify_operands_p (tree *tp, int *walk_subtrees, |
14264 | void *data) |
14265 | { |
14266 | tree t = omp_member_access_dummy_var (decl: *tp); |
14267 | if (t) |
14268 | { |
14269 | struct walk_stmt_info *wi = (struct walk_stmt_info *) data; |
14270 | lower_omp_regimplify_operands_data *ldata |
14271 | = (lower_omp_regimplify_operands_data *) wi->info; |
14272 | tree o = maybe_lookup_decl (var: t, ctx: ldata->ctx); |
14273 | if (o != t) |
14274 | { |
14275 | ldata->decls->safe_push (DECL_VALUE_EXPR (*tp)); |
14276 | ldata->decls->safe_push (obj: *tp); |
14277 | tree v = unshare_and_remap (DECL_VALUE_EXPR (*tp), from: t, to: o); |
14278 | SET_DECL_VALUE_EXPR (*tp, v); |
14279 | } |
14280 | } |
14281 | *walk_subtrees = !IS_TYPE_OR_DECL_P (*tp); |
14282 | return NULL_TREE; |
14283 | } |
14284 | |
14285 | /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs |
14286 | of omp_member_access_dummy_var vars during regimplification. */ |
14287 | |
14288 | static void |
14289 | lower_omp_regimplify_operands (omp_context *ctx, gimple *stmt, |
14290 | gimple_stmt_iterator *gsi_p) |
14291 | { |
14292 | auto_vec<tree, 10> decls; |
14293 | if (ctx) |
14294 | { |
14295 | struct walk_stmt_info wi; |
14296 | memset (s: &wi, c: '\0', n: sizeof (wi)); |
14297 | struct lower_omp_regimplify_operands_data data; |
14298 | data.ctx = ctx; |
14299 | data.decls = &decls; |
14300 | wi.info = &data; |
14301 | walk_gimple_op (stmt, lower_omp_regimplify_operands_p, &wi); |
14302 | } |
14303 | gimple_regimplify_operands (stmt, gsi_p); |
14304 | while (!decls.is_empty ()) |
14305 | { |
14306 | tree t = decls.pop (); |
14307 | tree v = decls.pop (); |
14308 | SET_DECL_VALUE_EXPR (t, v); |
14309 | } |
14310 | } |
14311 | |
14312 | static void |
14313 | lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
14314 | { |
14315 | gimple *stmt = gsi_stmt (i: *gsi_p); |
14316 | struct walk_stmt_info wi; |
14317 | gcall *call_stmt; |
14318 | |
14319 | if (gimple_has_location (g: stmt)) |
14320 | input_location = gimple_location (g: stmt); |
14321 | |
14322 | if (make_addressable_vars) |
14323 | memset (s: &wi, c: '\0', n: sizeof (wi)); |
14324 | |
14325 | /* If we have issued syntax errors, avoid doing any heavy lifting. |
14326 | Just replace the OMP directives with a NOP to avoid |
14327 | confusing RTL expansion. */ |
14328 | if (seen_error () && is_gimple_omp (stmt)) |
14329 | { |
14330 | gsi_replace (gsi_p, gimple_build_nop (), true); |
14331 | return; |
14332 | } |
14333 | |
14334 | switch (gimple_code (g: stmt)) |
14335 | { |
14336 | case GIMPLE_COND: |
14337 | { |
14338 | gcond *cond_stmt = as_a <gcond *> (p: stmt); |
14339 | if ((ctx || make_addressable_vars) |
14340 | && (walk_tree (gimple_cond_lhs_ptr (cond_stmt), |
14341 | lower_omp_regimplify_p, |
14342 | ctx ? NULL : &wi, NULL) |
14343 | || walk_tree (gimple_cond_rhs_ptr (cond_stmt), |
14344 | lower_omp_regimplify_p, |
14345 | ctx ? NULL : &wi, NULL))) |
14346 | lower_omp_regimplify_operands (ctx, stmt: cond_stmt, gsi_p); |
14347 | } |
14348 | break; |
14349 | case GIMPLE_CATCH: |
14350 | lower_omp (gimple_catch_handler_ptr (catch_stmt: as_a <gcatch *> (p: stmt)), ctx); |
14351 | break; |
14352 | case GIMPLE_EH_FILTER: |
14353 | lower_omp (gimple_eh_filter_failure_ptr (gs: stmt), ctx); |
14354 | break; |
14355 | case GIMPLE_TRY: |
14356 | lower_omp (gimple_try_eval_ptr (gs: stmt), ctx); |
14357 | lower_omp (gimple_try_cleanup_ptr (gs: stmt), ctx); |
14358 | break; |
14359 | case GIMPLE_ASSUME: |
14360 | lower_omp (gimple_assume_body_ptr (gs: stmt), ctx); |
14361 | break; |
14362 | case GIMPLE_TRANSACTION: |
14363 | lower_omp (gimple_transaction_body_ptr (transaction_stmt: as_a <gtransaction *> (p: stmt)), |
14364 | ctx); |
14365 | break; |
14366 | case GIMPLE_BIND: |
14367 | if (ctx && is_gimple_omp_oacc (stmt: ctx->stmt)) |
14368 | { |
14369 | tree vars = gimple_bind_vars (bind_stmt: as_a <gbind *> (p: stmt)); |
14370 | oacc_privatization_scan_decl_chain (ctx, decls: vars); |
14371 | } |
14372 | lower_omp (gimple_bind_body_ptr (bind_stmt: as_a <gbind *> (p: stmt)), ctx); |
14373 | maybe_remove_omp_member_access_dummy_vars (bind: as_a <gbind *> (p: stmt)); |
14374 | break; |
14375 | case GIMPLE_OMP_PARALLEL: |
14376 | case GIMPLE_OMP_TASK: |
14377 | ctx = maybe_lookup_ctx (stmt); |
14378 | gcc_assert (ctx); |
14379 | if (ctx->cancellable) |
14380 | ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION); |
14381 | lower_omp_taskreg (gsi_p, ctx); |
14382 | break; |
14383 | case GIMPLE_OMP_FOR: |
14384 | ctx = maybe_lookup_ctx (stmt); |
14385 | gcc_assert (ctx); |
14386 | if (ctx->cancellable) |
14387 | ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION); |
14388 | lower_omp_for (gsi_p, ctx); |
14389 | break; |
14390 | case GIMPLE_OMP_SECTIONS: |
14391 | ctx = maybe_lookup_ctx (stmt); |
14392 | gcc_assert (ctx); |
14393 | if (ctx->cancellable) |
14394 | ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION); |
14395 | lower_omp_sections (gsi_p, ctx); |
14396 | break; |
14397 | case GIMPLE_OMP_SCOPE: |
14398 | ctx = maybe_lookup_ctx (stmt); |
14399 | gcc_assert (ctx); |
14400 | lower_omp_scope (gsi_p, ctx); |
14401 | break; |
14402 | case GIMPLE_OMP_SINGLE: |
14403 | ctx = maybe_lookup_ctx (stmt); |
14404 | gcc_assert (ctx); |
14405 | lower_omp_single (gsi_p, ctx); |
14406 | break; |
14407 | case GIMPLE_OMP_STRUCTURED_BLOCK: |
14408 | /* We have already done error checking at this point, so these nodes |
14409 | can be completely removed and replaced with their body. */ |
14410 | ctx = maybe_lookup_ctx (stmt); |
14411 | gcc_assert (ctx); |
14412 | lower_omp (gimple_omp_body_ptr (gs: stmt), ctx); |
14413 | gsi_replace_with_seq (gsi_p, gimple_omp_body (gs: stmt), true); |
14414 | break; |
14415 | case GIMPLE_OMP_MASTER: |
14416 | case GIMPLE_OMP_MASKED: |
14417 | ctx = maybe_lookup_ctx (stmt); |
14418 | gcc_assert (ctx); |
14419 | lower_omp_master (gsi_p, ctx); |
14420 | break; |
14421 | case GIMPLE_OMP_TASKGROUP: |
14422 | ctx = maybe_lookup_ctx (stmt); |
14423 | gcc_assert (ctx); |
14424 | lower_omp_taskgroup (gsi_p, ctx); |
14425 | break; |
14426 | case GIMPLE_OMP_ORDERED: |
14427 | ctx = maybe_lookup_ctx (stmt); |
14428 | gcc_assert (ctx); |
14429 | lower_omp_ordered (gsi_p, ctx); |
14430 | break; |
14431 | case GIMPLE_OMP_SCAN: |
14432 | ctx = maybe_lookup_ctx (stmt); |
14433 | gcc_assert (ctx); |
14434 | lower_omp_scan (gsi_p, ctx); |
14435 | break; |
14436 | case GIMPLE_OMP_CRITICAL: |
14437 | ctx = maybe_lookup_ctx (stmt); |
14438 | gcc_assert (ctx); |
14439 | lower_omp_critical (gsi_p, ctx); |
14440 | break; |
14441 | case GIMPLE_OMP_ATOMIC_LOAD: |
14442 | if ((ctx || make_addressable_vars) |
14443 | && walk_tree (gimple_omp_atomic_load_rhs_ptr ( |
14444 | as_a <gomp_atomic_load *> (stmt)), |
14445 | lower_omp_regimplify_p, ctx ? NULL : &wi, NULL)) |
14446 | lower_omp_regimplify_operands (ctx, stmt, gsi_p); |
14447 | break; |
14448 | case GIMPLE_OMP_TARGET: |
14449 | ctx = maybe_lookup_ctx (stmt); |
14450 | gcc_assert (ctx); |
14451 | lower_omp_target (gsi_p, ctx); |
14452 | break; |
14453 | case GIMPLE_OMP_TEAMS: |
14454 | ctx = maybe_lookup_ctx (stmt); |
14455 | gcc_assert (ctx); |
14456 | if (gimple_omp_teams_host (omp_teams_stmt: as_a <gomp_teams *> (p: stmt))) |
14457 | lower_omp_taskreg (gsi_p, ctx); |
14458 | else |
14459 | lower_omp_teams (gsi_p, ctx); |
14460 | break; |
14461 | case GIMPLE_CALL: |
14462 | tree fndecl; |
14463 | call_stmt = as_a <gcall *> (p: stmt); |
14464 | fndecl = gimple_call_fndecl (gs: call_stmt); |
14465 | if (fndecl |
14466 | && fndecl_built_in_p (node: fndecl, klass: BUILT_IN_NORMAL)) |
14467 | switch (DECL_FUNCTION_CODE (decl: fndecl)) |
14468 | { |
14469 | case BUILT_IN_GOMP_BARRIER: |
14470 | if (ctx == NULL) |
14471 | break; |
14472 | /* FALLTHRU */ |
14473 | case BUILT_IN_GOMP_CANCEL: |
14474 | case BUILT_IN_GOMP_CANCELLATION_POINT: |
14475 | omp_context *cctx; |
14476 | cctx = ctx; |
14477 | if (gimple_code (g: cctx->stmt) == GIMPLE_OMP_SECTION) |
14478 | cctx = cctx->outer; |
14479 | gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE); |
14480 | if (!cctx->cancellable) |
14481 | { |
14482 | if (DECL_FUNCTION_CODE (decl: fndecl) |
14483 | == BUILT_IN_GOMP_CANCELLATION_POINT) |
14484 | { |
14485 | stmt = gimple_build_nop (); |
14486 | gsi_replace (gsi_p, stmt, false); |
14487 | } |
14488 | break; |
14489 | } |
14490 | if (DECL_FUNCTION_CODE (decl: fndecl) == BUILT_IN_GOMP_BARRIER) |
14491 | { |
14492 | fndecl = builtin_decl_explicit (fncode: BUILT_IN_GOMP_BARRIER_CANCEL); |
14493 | gimple_call_set_fndecl (gs: call_stmt, decl: fndecl); |
14494 | gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl)); |
14495 | } |
14496 | tree lhs; |
14497 | lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl))); |
14498 | gimple_call_set_lhs (gs: call_stmt, lhs); |
14499 | tree fallthru_label; |
14500 | fallthru_label = create_artificial_label (UNKNOWN_LOCATION); |
14501 | gimple *g; |
14502 | g = gimple_build_label (label: fallthru_label); |
14503 | gsi_insert_after (gsi_p, g, GSI_SAME_STMT); |
14504 | g = gimple_build_cond (NE_EXPR, lhs, |
14505 | fold_convert (TREE_TYPE (lhs), |
14506 | boolean_false_node), |
14507 | cctx->cancel_label, fallthru_label); |
14508 | gsi_insert_after (gsi_p, g, GSI_SAME_STMT); |
14509 | break; |
14510 | default: |
14511 | break; |
14512 | } |
14513 | goto regimplify; |
14514 | |
14515 | case GIMPLE_ASSIGN: |
14516 | for (omp_context *up = ctx; up; up = up->outer) |
14517 | { |
14518 | if (gimple_code (g: up->stmt) == GIMPLE_OMP_ORDERED |
14519 | || gimple_code (g: up->stmt) == GIMPLE_OMP_CRITICAL |
14520 | || gimple_code (g: up->stmt) == GIMPLE_OMP_TASKGROUP |
14521 | || gimple_code (g: up->stmt) == GIMPLE_OMP_SCOPE |
14522 | || gimple_code (g: up->stmt) == GIMPLE_OMP_SECTION |
14523 | || gimple_code (g: up->stmt) == GIMPLE_OMP_SCAN |
14524 | || (gimple_code (g: up->stmt) == GIMPLE_OMP_TARGET |
14525 | && (gimple_omp_target_kind (g: up->stmt) |
14526 | == GF_OMP_TARGET_KIND_DATA))) |
14527 | continue; |
14528 | else if (!up->lastprivate_conditional_map) |
14529 | break; |
14530 | tree lhs = get_base_address (t: gimple_assign_lhs (gs: stmt)); |
14531 | if (TREE_CODE (lhs) == MEM_REF |
14532 | && DECL_P (TREE_OPERAND (lhs, 0)) |
14533 | && TREE_CODE (TREE_TYPE (TREE_OPERAND (lhs, |
14534 | 0))) == REFERENCE_TYPE) |
14535 | lhs = TREE_OPERAND (lhs, 0); |
14536 | if (DECL_P (lhs)) |
14537 | if (tree *v = up->lastprivate_conditional_map->get (k: lhs)) |
14538 | { |
14539 | tree clauses; |
14540 | if (up->combined_into_simd_safelen1) |
14541 | { |
14542 | up = up->outer; |
14543 | if (gimple_code (g: up->stmt) == GIMPLE_OMP_SCAN) |
14544 | up = up->outer; |
14545 | } |
14546 | if (gimple_code (g: up->stmt) == GIMPLE_OMP_FOR) |
14547 | clauses = gimple_omp_for_clauses (gs: up->stmt); |
14548 | else |
14549 | clauses = gimple_omp_sections_clauses (gs: up->stmt); |
14550 | tree c = omp_find_clause (clauses, kind: OMP_CLAUSE__CONDTEMP_); |
14551 | if (!OMP_CLAUSE__CONDTEMP__ITER (c)) |
14552 | c = omp_find_clause (OMP_CLAUSE_CHAIN (c), |
14553 | kind: OMP_CLAUSE__CONDTEMP_); |
14554 | gcc_assert (OMP_CLAUSE__CONDTEMP__ITER (c)); |
14555 | gimple *g = gimple_build_assign (*v, OMP_CLAUSE_DECL (c)); |
14556 | gsi_insert_after (gsi_p, g, GSI_SAME_STMT); |
14557 | } |
14558 | } |
14559 | /* FALLTHRU */ |
14560 | |
14561 | default: |
14562 | regimplify: |
14563 | if ((ctx || make_addressable_vars) |
14564 | && walk_gimple_op (stmt, lower_omp_regimplify_p, |
14565 | ctx ? NULL : &wi)) |
14566 | { |
14567 | /* Just remove clobbers, this should happen only if we have |
14568 | "privatized" local addressable variables in SIMD regions, |
14569 | the clobber isn't needed in that case and gimplifying address |
14570 | of the ARRAY_REF into a pointer and creating MEM_REF based |
14571 | clobber would create worse code than we get with the clobber |
14572 | dropped. */ |
14573 | if (gimple_clobber_p (s: stmt)) |
14574 | { |
14575 | gsi_replace (gsi_p, gimple_build_nop (), true); |
14576 | break; |
14577 | } |
14578 | lower_omp_regimplify_operands (ctx, stmt, gsi_p); |
14579 | } |
14580 | break; |
14581 | } |
14582 | } |
14583 | |
14584 | static void |
14585 | lower_omp (gimple_seq *body, omp_context *ctx) |
14586 | { |
14587 | location_t saved_location = input_location; |
14588 | gimple_stmt_iterator gsi; |
14589 | for (gsi = gsi_start (seq&: *body); !gsi_end_p (i: gsi); gsi_next (i: &gsi)) |
14590 | lower_omp_1 (gsi_p: &gsi, ctx); |
14591 | /* During gimplification, we haven't folded statments inside offloading |
14592 | or taskreg regions (gimplify.cc:maybe_fold_stmt); do that now. */ |
14593 | if (target_nesting_level || taskreg_nesting_level) |
14594 | for (gsi = gsi_start (seq&: *body); !gsi_end_p (i: gsi); gsi_next (i: &gsi)) |
14595 | fold_stmt (&gsi); |
14596 | input_location = saved_location; |
14597 | } |
14598 | |
14599 | /* Main entry point. */ |
14600 | |
14601 | static unsigned int |
14602 | execute_lower_omp (void) |
14603 | { |
14604 | gimple_seq body; |
14605 | int i; |
14606 | omp_context *ctx; |
14607 | |
14608 | /* This pass always runs, to provide PROP_gimple_lomp. |
14609 | But often, there is nothing to do. */ |
14610 | if (flag_openacc == 0 && flag_openmp == 0 |
14611 | && flag_openmp_simd == 0) |
14612 | return 0; |
14613 | |
14614 | all_contexts = splay_tree_new (splay_tree_compare_pointers, 0, |
14615 | delete_omp_context); |
14616 | |
14617 | body = gimple_body (current_function_decl); |
14618 | |
14619 | scan_omp (body_p: &body, NULL); |
14620 | gcc_assert (taskreg_nesting_level == 0); |
14621 | FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx) |
14622 | finish_taskreg_scan (ctx); |
14623 | taskreg_contexts.release (); |
14624 | |
14625 | if (all_contexts->root) |
14626 | { |
14627 | if (make_addressable_vars) |
14628 | push_gimplify_context (); |
14629 | lower_omp (body: &body, NULL); |
14630 | if (make_addressable_vars) |
14631 | pop_gimplify_context (NULL); |
14632 | } |
14633 | |
14634 | if (all_contexts) |
14635 | { |
14636 | splay_tree_delete (all_contexts); |
14637 | all_contexts = NULL; |
14638 | } |
14639 | BITMAP_FREE (make_addressable_vars); |
14640 | BITMAP_FREE (global_nonaddressable_vars); |
14641 | |
14642 | /* If current function is a method, remove artificial dummy VAR_DECL created |
14643 | for non-static data member privatization, they aren't needed for |
14644 | debuginfo nor anything else, have been already replaced everywhere in the |
14645 | IL and cause problems with LTO. */ |
14646 | if (DECL_ARGUMENTS (current_function_decl) |
14647 | && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl)) |
14648 | && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl))) |
14649 | == POINTER_TYPE)) |
14650 | remove_member_access_dummy_vars (DECL_INITIAL (current_function_decl)); |
14651 | |
14652 | for (auto task_stmt : task_cpyfns) |
14653 | finalize_task_copyfn (task_stmt); |
14654 | task_cpyfns.release (); |
14655 | return 0; |
14656 | } |
14657 | |
14658 | namespace { |
14659 | |
14660 | const pass_data pass_data_lower_omp = |
14661 | { |
14662 | .type: GIMPLE_PASS, /* type */ |
14663 | .name: "omplower" , /* name */ |
14664 | .optinfo_flags: OPTGROUP_OMP, /* optinfo_flags */ |
14665 | .tv_id: TV_NONE, /* tv_id */ |
14666 | PROP_gimple_any, /* properties_required */ |
14667 | PROP_gimple_lomp | PROP_gimple_lomp_dev, /* properties_provided */ |
14668 | .properties_destroyed: 0, /* properties_destroyed */ |
14669 | .todo_flags_start: 0, /* todo_flags_start */ |
14670 | .todo_flags_finish: 0, /* todo_flags_finish */ |
14671 | }; |
14672 | |
14673 | class pass_lower_omp : public gimple_opt_pass |
14674 | { |
14675 | public: |
14676 | pass_lower_omp (gcc::context *ctxt) |
14677 | : gimple_opt_pass (pass_data_lower_omp, ctxt) |
14678 | {} |
14679 | |
14680 | /* opt_pass methods: */ |
14681 | unsigned int execute (function *) final override |
14682 | { |
14683 | return execute_lower_omp (); |
14684 | } |
14685 | |
14686 | }; // class pass_lower_omp |
14687 | |
14688 | } // anon namespace |
14689 | |
14690 | gimple_opt_pass * |
14691 | make_pass_lower_omp (gcc::context *ctxt) |
14692 | { |
14693 | return new pass_lower_omp (ctxt); |
14694 | } |
14695 | |
14696 | /* The following is a utility to diagnose structured block violations. |
14697 | It is not part of the "omplower" pass, as that's invoked too late. It |
14698 | should be invoked by the respective front ends after gimplification. */ |
14699 | |
14700 | static splay_tree all_labels; |
14701 | |
14702 | /* Check for mismatched contexts and generate an error if needed. Return |
14703 | true if an error is detected. */ |
14704 | |
14705 | static bool |
14706 | diagnose_sb_0 (gimple_stmt_iterator *gsi_p, |
14707 | gimple *branch_ctx, gimple *label_ctx) |
14708 | { |
14709 | gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx)); |
14710 | gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx)); |
14711 | |
14712 | if (label_ctx == branch_ctx) |
14713 | return false; |
14714 | |
14715 | const char* kind = NULL; |
14716 | |
14717 | if (flag_openacc) |
14718 | { |
14719 | if ((branch_ctx && is_gimple_omp_oacc (stmt: branch_ctx)) |
14720 | || (label_ctx && is_gimple_omp_oacc (stmt: label_ctx))) |
14721 | { |
14722 | gcc_checking_assert (kind == NULL); |
14723 | kind = "OpenACC" ; |
14724 | } |
14725 | } |
14726 | if (kind == NULL) |
14727 | { |
14728 | gcc_checking_assert (flag_openmp || flag_openmp_simd); |
14729 | kind = "OpenMP" ; |
14730 | } |
14731 | |
14732 | /* Previously we kept track of the label's entire context in diagnose_sb_[12] |
14733 | so we could traverse it and issue a correct "exit" or "enter" error |
14734 | message upon a structured block violation. |
14735 | |
14736 | We built the context by building a list with tree_cons'ing, but there is |
14737 | no easy counterpart in gimple tuples. It seems like far too much work |
14738 | for issuing exit/enter error messages. If someone really misses the |
14739 | distinct error message... patches welcome. */ |
14740 | |
14741 | #if 0 |
14742 | /* Try to avoid confusing the user by producing and error message |
14743 | with correct "exit" or "enter" verbiage. We prefer "exit" |
14744 | unless we can show that LABEL_CTX is nested within BRANCH_CTX. */ |
14745 | if (branch_ctx == NULL) |
14746 | exit_p = false; |
14747 | else |
14748 | { |
14749 | while (label_ctx) |
14750 | { |
14751 | if (TREE_VALUE (label_ctx) == branch_ctx) |
14752 | { |
14753 | exit_p = false; |
14754 | break; |
14755 | } |
14756 | label_ctx = TREE_CHAIN (label_ctx); |
14757 | } |
14758 | } |
14759 | |
14760 | if (exit_p) |
14761 | error ("invalid exit from %s structured block" , kind); |
14762 | else |
14763 | error ("invalid entry to %s structured block" , kind); |
14764 | #endif |
14765 | |
14766 | /* If it's obvious we have an invalid entry, be specific about the error. */ |
14767 | if (branch_ctx == NULL) |
14768 | error ("invalid entry to %s structured block" , kind); |
14769 | else |
14770 | { |
14771 | /* Otherwise, be vague and lazy, but efficient. */ |
14772 | error ("invalid branch to/from %s structured block" , kind); |
14773 | } |
14774 | |
14775 | gsi_replace (gsi_p, gimple_build_nop (), false); |
14776 | return true; |
14777 | } |
14778 | |
14779 | /* Pass 1: Create a minimal tree of structured blocks, and record |
14780 | where each label is found. */ |
14781 | |
14782 | static tree |
14783 | diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p, |
14784 | struct walk_stmt_info *wi) |
14785 | { |
14786 | gimple *context = (gimple *) wi->info; |
14787 | gimple *inner_context; |
14788 | gimple *stmt = gsi_stmt (i: *gsi_p); |
14789 | |
14790 | *handled_ops_p = true; |
14791 | |
14792 | switch (gimple_code (g: stmt)) |
14793 | { |
14794 | WALK_SUBSTMTS; |
14795 | |
14796 | case GIMPLE_OMP_PARALLEL: |
14797 | case GIMPLE_OMP_TASK: |
14798 | case GIMPLE_OMP_SCOPE: |
14799 | case GIMPLE_OMP_SECTIONS: |
14800 | case GIMPLE_OMP_SINGLE: |
14801 | case GIMPLE_OMP_SECTION: |
14802 | case GIMPLE_OMP_STRUCTURED_BLOCK: |
14803 | case GIMPLE_OMP_MASTER: |
14804 | case GIMPLE_OMP_MASKED: |
14805 | case GIMPLE_OMP_ORDERED: |
14806 | case GIMPLE_OMP_SCAN: |
14807 | case GIMPLE_OMP_CRITICAL: |
14808 | case GIMPLE_OMP_TARGET: |
14809 | case GIMPLE_OMP_TEAMS: |
14810 | case GIMPLE_OMP_TASKGROUP: |
14811 | /* The minimal context here is just the current OMP construct. */ |
14812 | inner_context = stmt; |
14813 | wi->info = inner_context; |
14814 | walk_gimple_seq (gimple_omp_body (gs: stmt), diagnose_sb_1, NULL, wi); |
14815 | wi->info = context; |
14816 | break; |
14817 | |
14818 | case GIMPLE_OMP_FOR: |
14819 | inner_context = stmt; |
14820 | wi->info = inner_context; |
14821 | /* gimple_omp_for_{index,initial,final} are all DECLs; no need to |
14822 | walk them. */ |
14823 | walk_gimple_seq (gimple_omp_for_pre_body (gs: stmt), |
14824 | diagnose_sb_1, NULL, wi); |
14825 | walk_gimple_seq (gimple_omp_body (gs: stmt), diagnose_sb_1, NULL, wi); |
14826 | wi->info = context; |
14827 | break; |
14828 | |
14829 | case GIMPLE_LABEL: |
14830 | splay_tree_insert (all_labels, |
14831 | (splay_tree_key) gimple_label_label ( |
14832 | gs: as_a <glabel *> (p: stmt)), |
14833 | (splay_tree_value) context); |
14834 | break; |
14835 | |
14836 | default: |
14837 | break; |
14838 | } |
14839 | |
14840 | return NULL_TREE; |
14841 | } |
14842 | |
14843 | /* Pass 2: Check each branch and see if its context differs from that of |
14844 | the destination label's context. */ |
14845 | |
14846 | static tree |
14847 | diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p, |
14848 | struct walk_stmt_info *wi) |
14849 | { |
14850 | gimple *context = (gimple *) wi->info; |
14851 | splay_tree_node n; |
14852 | gimple *stmt = gsi_stmt (i: *gsi_p); |
14853 | |
14854 | *handled_ops_p = true; |
14855 | |
14856 | switch (gimple_code (g: stmt)) |
14857 | { |
14858 | WALK_SUBSTMTS; |
14859 | |
14860 | case GIMPLE_OMP_PARALLEL: |
14861 | case GIMPLE_OMP_TASK: |
14862 | case GIMPLE_OMP_SCOPE: |
14863 | case GIMPLE_OMP_SECTIONS: |
14864 | case GIMPLE_OMP_SINGLE: |
14865 | case GIMPLE_OMP_SECTION: |
14866 | case GIMPLE_OMP_STRUCTURED_BLOCK: |
14867 | case GIMPLE_OMP_MASTER: |
14868 | case GIMPLE_OMP_MASKED: |
14869 | case GIMPLE_OMP_ORDERED: |
14870 | case GIMPLE_OMP_SCAN: |
14871 | case GIMPLE_OMP_CRITICAL: |
14872 | case GIMPLE_OMP_TARGET: |
14873 | case GIMPLE_OMP_TEAMS: |
14874 | case GIMPLE_OMP_TASKGROUP: |
14875 | wi->info = stmt; |
14876 | walk_gimple_seq_mod (gimple_omp_body_ptr (gs: stmt), diagnose_sb_2, NULL, wi); |
14877 | wi->info = context; |
14878 | break; |
14879 | |
14880 | case GIMPLE_OMP_FOR: |
14881 | wi->info = stmt; |
14882 | /* gimple_omp_for_{index,initial,final} are all DECLs; no need to |
14883 | walk them. */ |
14884 | walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (gs: stmt), |
14885 | diagnose_sb_2, NULL, wi); |
14886 | walk_gimple_seq_mod (gimple_omp_body_ptr (gs: stmt), diagnose_sb_2, NULL, wi); |
14887 | wi->info = context; |
14888 | break; |
14889 | |
14890 | case GIMPLE_COND: |
14891 | { |
14892 | gcond *cond_stmt = as_a <gcond *> (p: stmt); |
14893 | tree lab = gimple_cond_true_label (gs: cond_stmt); |
14894 | if (lab) |
14895 | { |
14896 | n = splay_tree_lookup (all_labels, |
14897 | (splay_tree_key) lab); |
14898 | diagnose_sb_0 (gsi_p, branch_ctx: context, |
14899 | label_ctx: n ? (gimple *) n->value : NULL); |
14900 | } |
14901 | lab = gimple_cond_false_label (gs: cond_stmt); |
14902 | if (lab) |
14903 | { |
14904 | n = splay_tree_lookup (all_labels, |
14905 | (splay_tree_key) lab); |
14906 | diagnose_sb_0 (gsi_p, branch_ctx: context, |
14907 | label_ctx: n ? (gimple *) n->value : NULL); |
14908 | } |
14909 | } |
14910 | break; |
14911 | |
14912 | case GIMPLE_GOTO: |
14913 | { |
14914 | tree lab = gimple_goto_dest (gs: stmt); |
14915 | if (TREE_CODE (lab) != LABEL_DECL) |
14916 | break; |
14917 | |
14918 | n = splay_tree_lookup (all_labels, (splay_tree_key) lab); |
14919 | diagnose_sb_0 (gsi_p, branch_ctx: context, label_ctx: n ? (gimple *) n->value : NULL); |
14920 | } |
14921 | break; |
14922 | |
14923 | case GIMPLE_SWITCH: |
14924 | { |
14925 | gswitch *switch_stmt = as_a <gswitch *> (p: stmt); |
14926 | unsigned int i; |
14927 | for (i = 0; i < gimple_switch_num_labels (gs: switch_stmt); ++i) |
14928 | { |
14929 | tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i)); |
14930 | n = splay_tree_lookup (all_labels, (splay_tree_key) lab); |
14931 | if (n && diagnose_sb_0 (gsi_p, branch_ctx: context, label_ctx: (gimple *) n->value)) |
14932 | break; |
14933 | } |
14934 | } |
14935 | break; |
14936 | |
14937 | case GIMPLE_RETURN: |
14938 | diagnose_sb_0 (gsi_p, branch_ctx: context, NULL); |
14939 | break; |
14940 | |
14941 | default: |
14942 | break; |
14943 | } |
14944 | |
14945 | return NULL_TREE; |
14946 | } |
14947 | |
14948 | static unsigned int |
14949 | diagnose_omp_structured_block_errors (void) |
14950 | { |
14951 | struct walk_stmt_info wi; |
14952 | gimple_seq body = gimple_body (current_function_decl); |
14953 | |
14954 | all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
14955 | |
14956 | memset (s: &wi, c: 0, n: sizeof (wi)); |
14957 | walk_gimple_seq (body, diagnose_sb_1, NULL, &wi); |
14958 | |
14959 | memset (s: &wi, c: 0, n: sizeof (wi)); |
14960 | wi.want_locations = true; |
14961 | walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi); |
14962 | |
14963 | gimple_set_body (current_function_decl, body); |
14964 | |
14965 | splay_tree_delete (all_labels); |
14966 | all_labels = NULL; |
14967 | |
14968 | return 0; |
14969 | } |
14970 | |
14971 | namespace { |
14972 | |
14973 | const pass_data pass_data_diagnose_omp_blocks = |
14974 | { |
14975 | .type: GIMPLE_PASS, /* type */ |
14976 | .name: "*diagnose_omp_blocks" , /* name */ |
14977 | .optinfo_flags: OPTGROUP_OMP, /* optinfo_flags */ |
14978 | .tv_id: TV_NONE, /* tv_id */ |
14979 | PROP_gimple_any, /* properties_required */ |
14980 | .properties_provided: 0, /* properties_provided */ |
14981 | .properties_destroyed: 0, /* properties_destroyed */ |
14982 | .todo_flags_start: 0, /* todo_flags_start */ |
14983 | .todo_flags_finish: 0, /* todo_flags_finish */ |
14984 | }; |
14985 | |
14986 | class pass_diagnose_omp_blocks : public gimple_opt_pass |
14987 | { |
14988 | public: |
14989 | pass_diagnose_omp_blocks (gcc::context *ctxt) |
14990 | : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt) |
14991 | {} |
14992 | |
14993 | /* opt_pass methods: */ |
14994 | bool gate (function *) final override |
14995 | { |
14996 | return flag_openacc || flag_openmp || flag_openmp_simd; |
14997 | } |
14998 | unsigned int execute (function *) final override |
14999 | { |
15000 | return diagnose_omp_structured_block_errors (); |
15001 | } |
15002 | |
15003 | }; // class pass_diagnose_omp_blocks |
15004 | |
15005 | } // anon namespace |
15006 | |
15007 | gimple_opt_pass * |
15008 | make_pass_diagnose_omp_blocks (gcc::context *ctxt) |
15009 | { |
15010 | return new pass_diagnose_omp_blocks (ctxt); |
15011 | } |
15012 | |
15013 | |
15014 | #include "gt-omp-low.h" |
15015 | |