1 | /* Copyright (C) 2005-2017 Free Software Foundation, Inc. |
2 | Contributed by Richard Henderson <rth@redhat.com>. |
3 | |
4 | This file is part of the GNU Offloading and Multi Processing Library |
5 | (libgomp). |
6 | |
7 | Libgomp is free software; you can redistribute it and/or modify it |
8 | under the terms of the GNU General Public License as published by |
9 | the Free Software Foundation; either version 3, or (at your option) |
10 | any later version. |
11 | |
12 | Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY |
13 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
14 | FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
15 | more details. |
16 | |
17 | Under Section 7 of GPL version 3, you are granted additional |
18 | permissions described in the GCC Runtime Library Exception, version |
19 | 3.1, as published by the Free Software Foundation. |
20 | |
21 | You should have received a copy of the GNU General Public License and |
22 | a copy of the GCC Runtime Library Exception along with this program; |
23 | see the files COPYING3 and COPYING.RUNTIME respectively. If not, see |
24 | <http://www.gnu.org/licenses/>. */ |
25 | |
26 | /* This file contains data types and function declarations that are not |
27 | part of the official OpenACC or OpenMP user interfaces. There are |
28 | declarations in here that are part of the GNU Offloading and Multi |
29 | Processing ABI, in that the compiler is required to know about them |
30 | and use them. |
31 | |
32 | The convention is that the all caps prefix "GOMP" is used group items |
33 | that are part of the external ABI, and the lower case prefix "gomp" |
34 | is used group items that are completely private to the library. */ |
35 | |
36 | #ifndef LIBGOMP_H |
37 | #define LIBGOMP_H 1 |
38 | |
39 | #ifndef _LIBGOMP_CHECKING_ |
40 | /* Define to 1 to perform internal sanity checks. */ |
41 | #define _LIBGOMP_CHECKING_ 0 |
42 | #endif |
43 | |
44 | #include "config.h" |
45 | #include "gstdint.h" |
46 | #include "libgomp-plugin.h" |
47 | |
48 | #ifdef HAVE_PTHREAD_H |
49 | #include <pthread.h> |
50 | #endif |
51 | #include <stdbool.h> |
52 | #include <stdlib.h> |
53 | #include <stdarg.h> |
54 | |
55 | /* Needed for memset in priority_queue.c. */ |
56 | #if _LIBGOMP_CHECKING_ |
57 | # ifdef STRING_WITH_STRINGS |
58 | # include <string.h> |
59 | # include <strings.h> |
60 | # else |
61 | # ifdef HAVE_STRING_H |
62 | # include <string.h> |
63 | # else |
64 | # ifdef HAVE_STRINGS_H |
65 | # include <strings.h> |
66 | # endif |
67 | # endif |
68 | # endif |
69 | #endif |
70 | |
71 | #ifdef HAVE_ATTRIBUTE_VISIBILITY |
72 | # pragma GCC visibility push(hidden) |
73 | #endif |
74 | |
75 | /* If we were a C++ library, we'd get this from <std/atomic>. */ |
76 | enum memmodel |
77 | { |
78 | MEMMODEL_RELAXED = 0, |
79 | MEMMODEL_CONSUME = 1, |
80 | MEMMODEL_ACQUIRE = 2, |
81 | MEMMODEL_RELEASE = 3, |
82 | MEMMODEL_ACQ_REL = 4, |
83 | MEMMODEL_SEQ_CST = 5 |
84 | }; |
85 | |
86 | /* alloc.c */ |
87 | |
88 | extern void *gomp_malloc (size_t) __attribute__((malloc)); |
89 | extern void *gomp_malloc_cleared (size_t) __attribute__((malloc)); |
90 | extern void *gomp_realloc (void *, size_t); |
91 | |
92 | /* Avoid conflicting prototypes of alloca() in system headers by using |
93 | GCC's builtin alloca(). */ |
94 | #define gomp_alloca(x) __builtin_alloca(x) |
95 | |
96 | /* error.c */ |
97 | |
98 | extern void gomp_vdebug (int, const char *, va_list); |
99 | extern void gomp_debug (int, const char *, ...) |
100 | __attribute__ ((format (printf, 2, 3))); |
101 | #define gomp_vdebug(KIND, FMT, VALIST) \ |
102 | do { \ |
103 | if (__builtin_expect (gomp_debug_var, 0)) \ |
104 | (gomp_vdebug) ((KIND), (FMT), (VALIST)); \ |
105 | } while (0) |
106 | #define gomp_debug(KIND, ...) \ |
107 | do { \ |
108 | if (__builtin_expect (gomp_debug_var, 0)) \ |
109 | (gomp_debug) ((KIND), __VA_ARGS__); \ |
110 | } while (0) |
111 | extern void gomp_verror (const char *, va_list); |
112 | extern void gomp_error (const char *, ...) |
113 | __attribute__ ((format (printf, 1, 2))); |
114 | extern void gomp_vfatal (const char *, va_list) |
115 | __attribute__ ((noreturn)); |
116 | extern void gomp_fatal (const char *, ...) |
117 | __attribute__ ((noreturn, format (printf, 1, 2))); |
118 | |
119 | struct gomp_task; |
120 | struct gomp_taskgroup; |
121 | struct htab; |
122 | |
123 | #include "priority_queue.h" |
124 | #include "sem.h" |
125 | #include "mutex.h" |
126 | #include "bar.h" |
127 | #include "simple-bar.h" |
128 | #include "ptrlock.h" |
129 | |
130 | |
131 | /* This structure contains the data to control one work-sharing construct, |
132 | either a LOOP (FOR/DO) or a SECTIONS. */ |
133 | |
134 | enum gomp_schedule_type |
135 | { |
136 | GFS_RUNTIME, |
137 | GFS_STATIC, |
138 | GFS_DYNAMIC, |
139 | GFS_GUIDED, |
140 | GFS_AUTO |
141 | }; |
142 | |
143 | struct gomp_doacross_work_share |
144 | { |
145 | union { |
146 | /* chunk_size copy, as ws->chunk_size is multiplied by incr for |
147 | GFS_DYNAMIC. */ |
148 | long chunk_size; |
149 | /* Likewise, but for ull implementation. */ |
150 | unsigned long long chunk_size_ull; |
151 | /* For schedule(static,0) this is the number |
152 | of iterations assigned to the last thread, i.e. number of |
153 | iterations / number of threads. */ |
154 | long q; |
155 | /* Likewise, but for ull implementation. */ |
156 | unsigned long long q_ull; |
157 | }; |
158 | /* Size of each array entry (padded to cache line size). */ |
159 | unsigned long elt_sz; |
160 | /* Number of dimensions in sink vectors. */ |
161 | unsigned int ncounts; |
162 | /* True if the iterations can be flattened. */ |
163 | bool flattened; |
164 | /* Actual array (of elt_sz sized units), aligned to cache line size. |
165 | This is indexed by team_id for GFS_STATIC and outermost iteration |
166 | / chunk_size for other schedules. */ |
167 | unsigned char *array; |
168 | /* These two are only used for schedule(static,0). */ |
169 | /* This one is number of iterations % number of threads. */ |
170 | long t; |
171 | union { |
172 | /* And this one is cached t * (q + 1). */ |
173 | long boundary; |
174 | /* Likewise, but for the ull implementation. */ |
175 | unsigned long long boundary_ull; |
176 | }; |
177 | /* Array of shift counts for each dimension if they can be flattened. */ |
178 | unsigned int shift_counts[]; |
179 | }; |
180 | |
181 | struct gomp_work_share |
182 | { |
183 | /* This member records the SCHEDULE clause to be used for this construct. |
184 | The user specification of "runtime" will already have been resolved. |
185 | If this is a SECTIONS construct, this value will always be DYNAMIC. */ |
186 | enum gomp_schedule_type sched; |
187 | |
188 | int mode; |
189 | |
190 | union { |
191 | struct { |
192 | /* This is the chunk_size argument to the SCHEDULE clause. */ |
193 | long chunk_size; |
194 | |
195 | /* This is the iteration end point. If this is a SECTIONS construct, |
196 | this is the number of contained sections. */ |
197 | long end; |
198 | |
199 | /* This is the iteration step. If this is a SECTIONS construct, this |
200 | is always 1. */ |
201 | long incr; |
202 | }; |
203 | |
204 | struct { |
205 | /* The same as above, but for the unsigned long long loop variants. */ |
206 | unsigned long long chunk_size_ull; |
207 | unsigned long long end_ull; |
208 | unsigned long long incr_ull; |
209 | }; |
210 | }; |
211 | |
212 | union { |
213 | /* This is a circular queue that details which threads will be allowed |
214 | into the ordered region and in which order. When a thread allocates |
215 | iterations on which it is going to work, it also registers itself at |
216 | the end of the array. When a thread reaches the ordered region, it |
217 | checks to see if it is the one at the head of the queue. If not, it |
218 | blocks on its RELEASE semaphore. */ |
219 | unsigned *ordered_team_ids; |
220 | |
221 | /* This is a pointer to DOACROSS work share data. */ |
222 | struct gomp_doacross_work_share *doacross; |
223 | }; |
224 | |
225 | /* This is the number of threads that have registered themselves in |
226 | the circular queue ordered_team_ids. */ |
227 | unsigned ordered_num_used; |
228 | |
229 | /* This is the team_id of the currently acknowledged owner of the ordered |
230 | section, or -1u if the ordered section has not been acknowledged by |
231 | any thread. This is distinguished from the thread that is *allowed* |
232 | to take the section next. */ |
233 | unsigned ordered_owner; |
234 | |
235 | /* This is the index into the circular queue ordered_team_ids of the |
236 | current thread that's allowed into the ordered reason. */ |
237 | unsigned ordered_cur; |
238 | |
239 | /* This is a chain of allocated gomp_work_share blocks, valid only |
240 | in the first gomp_work_share struct in the block. */ |
241 | struct gomp_work_share *next_alloc; |
242 | |
243 | /* The above fields are written once during workshare initialization, |
244 | or related to ordered worksharing. Make sure the following fields |
245 | are in a different cache line. */ |
246 | |
247 | /* This lock protects the update of the following members. */ |
248 | gomp_mutex_t lock __attribute__((aligned (64))); |
249 | |
250 | /* This is the count of the number of threads that have exited the work |
251 | share construct. If the construct was marked nowait, they have moved on |
252 | to other work; otherwise they're blocked on a barrier. The last member |
253 | of the team to exit the work share construct must deallocate it. */ |
254 | unsigned threads_completed; |
255 | |
256 | union { |
257 | /* This is the next iteration value to be allocated. In the case of |
258 | GFS_STATIC loops, this the iteration start point and never changes. */ |
259 | long next; |
260 | |
261 | /* The same, but with unsigned long long type. */ |
262 | unsigned long long next_ull; |
263 | |
264 | /* This is the returned data structure for SINGLE COPYPRIVATE. */ |
265 | void *copyprivate; |
266 | }; |
267 | |
268 | union { |
269 | /* Link to gomp_work_share struct for next work sharing construct |
270 | encountered after this one. */ |
271 | gomp_ptrlock_t next_ws; |
272 | |
273 | /* gomp_work_share structs are chained in the free work share cache |
274 | through this. */ |
275 | struct gomp_work_share *next_free; |
276 | }; |
277 | |
278 | /* If only few threads are in the team, ordered_team_ids can point |
279 | to this array which fills the padding at the end of this struct. */ |
280 | unsigned inline_ordered_team_ids[0]; |
281 | }; |
282 | |
283 | /* This structure contains all of the thread-local data associated with |
284 | a thread team. This is the data that must be saved when a thread |
285 | encounters a nested PARALLEL construct. */ |
286 | |
287 | struct gomp_team_state |
288 | { |
289 | /* This is the team of which the thread is currently a member. */ |
290 | struct gomp_team *team; |
291 | |
292 | /* This is the work share construct which this thread is currently |
293 | processing. Recall that with NOWAIT, not all threads may be |
294 | processing the same construct. */ |
295 | struct gomp_work_share *work_share; |
296 | |
297 | /* This is the previous work share construct or NULL if there wasn't any. |
298 | When all threads are done with the current work sharing construct, |
299 | the previous one can be freed. The current one can't, as its |
300 | next_ws field is used. */ |
301 | struct gomp_work_share *last_work_share; |
302 | |
303 | /* This is the ID of this thread within the team. This value is |
304 | guaranteed to be between 0 and N-1, where N is the number of |
305 | threads in the team. */ |
306 | unsigned team_id; |
307 | |
308 | /* Nesting level. */ |
309 | unsigned level; |
310 | |
311 | /* Active nesting level. Only active parallel regions are counted. */ |
312 | unsigned active_level; |
313 | |
314 | /* Place-partition-var, offset and length into gomp_places_list array. */ |
315 | unsigned place_partition_off; |
316 | unsigned place_partition_len; |
317 | |
318 | #ifdef HAVE_SYNC_BUILTINS |
319 | /* Number of single stmts encountered. */ |
320 | unsigned long single_count; |
321 | #endif |
322 | |
323 | /* For GFS_RUNTIME loops that resolved to GFS_STATIC, this is the |
324 | trip number through the loop. So first time a particular loop |
325 | is encountered this number is 0, the second time through the loop |
326 | is 1, etc. This is unused when the compiler knows in advance that |
327 | the loop is statically scheduled. */ |
328 | unsigned long static_trip; |
329 | }; |
330 | |
331 | struct target_mem_desc; |
332 | |
333 | /* These are the OpenMP 4.0 Internal Control Variables described in |
334 | section 2.3.1. Those described as having one copy per task are |
335 | stored within the structure; those described as having one copy |
336 | for the whole program are (naturally) global variables. */ |
337 | |
338 | struct gomp_task_icv |
339 | { |
340 | unsigned long nthreads_var; |
341 | enum gomp_schedule_type run_sched_var; |
342 | int run_sched_chunk_size; |
343 | int default_device_var; |
344 | unsigned int thread_limit_var; |
345 | bool dyn_var; |
346 | bool nest_var; |
347 | char bind_var; |
348 | /* Internal ICV. */ |
349 | struct target_mem_desc *target_data; |
350 | }; |
351 | |
352 | extern struct gomp_task_icv gomp_global_icv; |
353 | #ifndef HAVE_SYNC_BUILTINS |
354 | extern gomp_mutex_t gomp_managed_threads_lock; |
355 | #endif |
356 | extern unsigned long gomp_max_active_levels_var; |
357 | extern bool gomp_cancel_var; |
358 | extern int gomp_max_task_priority_var; |
359 | extern unsigned long long gomp_spin_count_var, gomp_throttled_spin_count_var; |
360 | extern unsigned long gomp_available_cpus, gomp_managed_threads; |
361 | extern unsigned long *gomp_nthreads_var_list, gomp_nthreads_var_list_len; |
362 | extern char *gomp_bind_var_list; |
363 | extern unsigned long gomp_bind_var_list_len; |
364 | extern void **gomp_places_list; |
365 | extern unsigned long gomp_places_list_len; |
366 | extern unsigned int gomp_num_teams_var; |
367 | extern int gomp_debug_var; |
368 | extern int goacc_device_num; |
369 | extern char *goacc_device_type; |
370 | |
371 | enum gomp_task_kind |
372 | { |
373 | /* Implicit task. */ |
374 | GOMP_TASK_IMPLICIT, |
375 | /* Undeferred task. */ |
376 | GOMP_TASK_UNDEFERRED, |
377 | /* Task created by GOMP_task and waiting to be run. */ |
378 | GOMP_TASK_WAITING, |
379 | /* Task currently executing or scheduled and about to execute. */ |
380 | GOMP_TASK_TIED, |
381 | /* Used for target tasks that have vars mapped and async run started, |
382 | but not yet completed. Once that completes, they will be readded |
383 | into the queues as GOMP_TASK_WAITING in order to perform the var |
384 | unmapping. */ |
385 | GOMP_TASK_ASYNC_RUNNING |
386 | }; |
387 | |
388 | struct gomp_task_depend_entry |
389 | { |
390 | /* Address of dependency. */ |
391 | void *addr; |
392 | struct gomp_task_depend_entry *next; |
393 | struct gomp_task_depend_entry *prev; |
394 | /* Task that provides the dependency in ADDR. */ |
395 | struct gomp_task *task; |
396 | /* Depend entry is of type "IN". */ |
397 | bool is_in; |
398 | bool redundant; |
399 | bool redundant_out; |
400 | }; |
401 | |
402 | struct gomp_dependers_vec |
403 | { |
404 | size_t n_elem; |
405 | size_t allocated; |
406 | struct gomp_task *elem[]; |
407 | }; |
408 | |
409 | /* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies. */ |
410 | |
411 | struct gomp_taskwait |
412 | { |
413 | bool in_taskwait; |
414 | bool in_depend_wait; |
415 | /* Number of tasks we are waiting for. */ |
416 | size_t n_depend; |
417 | gomp_sem_t taskwait_sem; |
418 | }; |
419 | |
420 | /* This structure describes a "task" to be run by a thread. */ |
421 | |
422 | struct gomp_task |
423 | { |
424 | /* Parent of this task. */ |
425 | struct gomp_task *parent; |
426 | /* Children of this task. */ |
427 | struct priority_queue children_queue; |
428 | /* Taskgroup this task belongs in. */ |
429 | struct gomp_taskgroup *taskgroup; |
430 | /* Tasks that depend on this task. */ |
431 | struct gomp_dependers_vec *dependers; |
432 | struct htab *depend_hash; |
433 | struct gomp_taskwait *taskwait; |
434 | /* Number of items in DEPEND. */ |
435 | size_t depend_count; |
436 | /* Number of tasks this task depends on. Once this counter reaches |
437 | 0, we have no unsatisfied dependencies, and this task can be put |
438 | into the various queues to be scheduled. */ |
439 | size_t num_dependees; |
440 | |
441 | /* Priority of this task. */ |
442 | int priority; |
443 | /* The priority node for this task in each of the different queues. |
444 | We put this here to avoid allocating space for each priority |
445 | node. Then we play offsetof() games to convert between pnode[] |
446 | entries and the gomp_task in which they reside. */ |
447 | struct priority_node pnode[3]; |
448 | |
449 | struct gomp_task_icv icv; |
450 | void (*fn) (void *); |
451 | void *fn_data; |
452 | enum gomp_task_kind kind; |
453 | bool in_tied_task; |
454 | bool final_task; |
455 | bool copy_ctors_done; |
456 | /* Set for undeferred tasks with unsatisfied dependencies which |
457 | block further execution of their parent until the dependencies |
458 | are satisfied. */ |
459 | bool parent_depends_on; |
460 | /* Dependencies provided and/or needed for this task. DEPEND_COUNT |
461 | is the number of items available. */ |
462 | struct gomp_task_depend_entry depend[]; |
463 | }; |
464 | |
465 | /* This structure describes a single #pragma omp taskgroup. */ |
466 | |
467 | struct gomp_taskgroup |
468 | { |
469 | struct gomp_taskgroup *prev; |
470 | /* Queue of tasks that belong in this taskgroup. */ |
471 | struct priority_queue taskgroup_queue; |
472 | bool in_taskgroup_wait; |
473 | bool cancelled; |
474 | gomp_sem_t taskgroup_sem; |
475 | size_t num_children; |
476 | }; |
477 | |
478 | /* Various state of OpenMP async offloading tasks. */ |
479 | enum gomp_target_task_state |
480 | { |
481 | GOMP_TARGET_TASK_DATA, |
482 | GOMP_TARGET_TASK_BEFORE_MAP, |
483 | GOMP_TARGET_TASK_FALLBACK, |
484 | GOMP_TARGET_TASK_READY_TO_RUN, |
485 | GOMP_TARGET_TASK_RUNNING, |
486 | GOMP_TARGET_TASK_FINISHED |
487 | }; |
488 | |
489 | /* This structure describes a target task. */ |
490 | |
491 | struct gomp_target_task |
492 | { |
493 | struct gomp_device_descr *devicep; |
494 | void (*fn) (void *); |
495 | size_t mapnum; |
496 | size_t *sizes; |
497 | unsigned short *kinds; |
498 | unsigned int flags; |
499 | enum gomp_target_task_state state; |
500 | struct target_mem_desc *tgt; |
501 | struct gomp_task *task; |
502 | struct gomp_team *team; |
503 | /* Device-specific target arguments. */ |
504 | void **args; |
505 | void *hostaddrs[]; |
506 | }; |
507 | |
508 | /* This structure describes a "team" of threads. These are the threads |
509 | that are spawned by a PARALLEL constructs, as well as the work sharing |
510 | constructs that the team encounters. */ |
511 | |
512 | struct gomp_team |
513 | { |
514 | /* This is the number of threads in the current team. */ |
515 | unsigned nthreads; |
516 | |
517 | /* This is number of gomp_work_share structs that have been allocated |
518 | as a block last time. */ |
519 | unsigned work_share_chunk; |
520 | |
521 | /* This is the saved team state that applied to a master thread before |
522 | the current thread was created. */ |
523 | struct gomp_team_state prev_ts; |
524 | |
525 | /* This semaphore should be used by the master thread instead of its |
526 | "native" semaphore in the thread structure. Required for nested |
527 | parallels, as the master is a member of two teams. */ |
528 | gomp_sem_t master_release; |
529 | |
530 | /* This points to an array with pointers to the release semaphore |
531 | of the threads in the team. */ |
532 | gomp_sem_t **ordered_release; |
533 | |
534 | /* List of work shares on which gomp_fini_work_share hasn't been |
535 | called yet. If the team hasn't been cancelled, this should be |
536 | equal to each thr->ts.work_share, but otherwise it can be a possibly |
537 | long list of workshares. */ |
538 | struct gomp_work_share *work_shares_to_free; |
539 | |
540 | /* List of gomp_work_share structs chained through next_free fields. |
541 | This is populated and taken off only by the first thread in the |
542 | team encountering a new work sharing construct, in a critical |
543 | section. */ |
544 | struct gomp_work_share *work_share_list_alloc; |
545 | |
546 | /* List of gomp_work_share structs freed by free_work_share. New |
547 | entries are atomically added to the start of the list, and |
548 | alloc_work_share can safely only move all but the first entry |
549 | to work_share_list alloc, as free_work_share can happen concurrently |
550 | with alloc_work_share. */ |
551 | struct gomp_work_share *work_share_list_free; |
552 | |
553 | #ifdef HAVE_SYNC_BUILTINS |
554 | /* Number of simple single regions encountered by threads in this |
555 | team. */ |
556 | unsigned long single_count; |
557 | #else |
558 | /* Mutex protecting addition of workshares to work_share_list_free. */ |
559 | gomp_mutex_t work_share_list_free_lock; |
560 | #endif |
561 | |
562 | /* This barrier is used for most synchronization of the team. */ |
563 | gomp_barrier_t barrier; |
564 | |
565 | /* Initial work shares, to avoid allocating any gomp_work_share |
566 | structs in the common case. */ |
567 | struct gomp_work_share work_shares[8]; |
568 | |
569 | gomp_mutex_t task_lock; |
570 | /* Scheduled tasks. */ |
571 | struct priority_queue task_queue; |
572 | /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team. */ |
573 | unsigned int task_count; |
574 | /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled. */ |
575 | unsigned int task_queued_count; |
576 | /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running |
577 | directly in gomp_barrier_handle_tasks; tasks spawned |
578 | from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when |
579 | that is called from a task run from gomp_barrier_handle_tasks. |
580 | task_running_count should be always <= team->nthreads, |
581 | and if current task isn't in_tied_task, then it will be |
582 | even < team->nthreads. */ |
583 | unsigned int task_running_count; |
584 | int work_share_cancelled; |
585 | int team_cancelled; |
586 | |
587 | /* This array contains structures for implicit tasks. */ |
588 | struct gomp_task implicit_task[]; |
589 | }; |
590 | |
591 | /* This structure contains all data that is private to libgomp and is |
592 | allocated per thread. */ |
593 | |
594 | struct gomp_thread |
595 | { |
596 | /* This is the function that the thread should run upon launch. */ |
597 | void (*fn) (void *data); |
598 | void *data; |
599 | |
600 | /* This is the current team state for this thread. The ts.team member |
601 | is NULL only if the thread is idle. */ |
602 | struct gomp_team_state ts; |
603 | |
604 | /* This is the task that the thread is currently executing. */ |
605 | struct gomp_task *task; |
606 | |
607 | /* This semaphore is used for ordered loops. */ |
608 | gomp_sem_t release; |
609 | |
610 | /* Place this thread is bound to plus one, or zero if not bound |
611 | to any place. */ |
612 | unsigned int place; |
613 | |
614 | /* User pthread thread pool */ |
615 | struct gomp_thread_pool *thread_pool; |
616 | }; |
617 | |
618 | |
619 | struct gomp_thread_pool |
620 | { |
621 | /* This array manages threads spawned from the top level, which will |
622 | return to the idle loop once the current PARALLEL construct ends. */ |
623 | struct gomp_thread **threads; |
624 | unsigned threads_size; |
625 | unsigned threads_used; |
626 | /* The last team is used for non-nested teams to delay their destruction to |
627 | make sure all the threads in the team move on to the pool's barrier before |
628 | the team's barrier is destroyed. */ |
629 | struct gomp_team *last_team; |
630 | /* Number of threads running in this contention group. */ |
631 | unsigned long threads_busy; |
632 | |
633 | /* This barrier holds and releases threads waiting in thread pools. */ |
634 | gomp_simple_barrier_t threads_dock; |
635 | }; |
636 | |
637 | enum gomp_cancel_kind |
638 | { |
639 | GOMP_CANCEL_PARALLEL = 1, |
640 | GOMP_CANCEL_LOOP = 2, |
641 | GOMP_CANCEL_FOR = GOMP_CANCEL_LOOP, |
642 | GOMP_CANCEL_DO = GOMP_CANCEL_LOOP, |
643 | GOMP_CANCEL_SECTIONS = 4, |
644 | GOMP_CANCEL_TASKGROUP = 8 |
645 | }; |
646 | |
647 | /* ... and here is that TLS data. */ |
648 | |
649 | #if defined __nvptx__ |
650 | extern struct gomp_thread *nvptx_thrs __attribute__((shared)); |
651 | static inline struct gomp_thread *gomp_thread (void) |
652 | { |
653 | int tid; |
654 | asm ("mov.u32 %0, %%tid.y;" : "=r" (tid)); |
655 | return nvptx_thrs + tid; |
656 | } |
657 | #elif defined HAVE_TLS || defined USE_EMUTLS |
658 | extern __thread struct gomp_thread gomp_tls_data; |
659 | static inline struct gomp_thread *gomp_thread (void) |
660 | { |
661 | return &gomp_tls_data; |
662 | } |
663 | #else |
664 | extern pthread_key_t gomp_tls_key; |
665 | static inline struct gomp_thread *gomp_thread (void) |
666 | { |
667 | return pthread_getspecific (gomp_tls_key); |
668 | } |
669 | #endif |
670 | |
671 | extern struct gomp_task_icv *gomp_new_icv (void); |
672 | |
673 | /* Here's how to access the current copy of the ICVs. */ |
674 | |
675 | static inline struct gomp_task_icv *gomp_icv (bool write) |
676 | { |
677 | struct gomp_task *task = gomp_thread ()->task; |
678 | if (task) |
679 | return &task->icv; |
680 | else if (write) |
681 | return gomp_new_icv (); |
682 | else |
683 | return &gomp_global_icv; |
684 | } |
685 | |
686 | #ifdef LIBGOMP_USE_PTHREADS |
687 | /* The attributes to be used during thread creation. */ |
688 | extern pthread_attr_t gomp_thread_attr; |
689 | |
690 | extern pthread_key_t gomp_thread_destructor; |
691 | #endif |
692 | |
693 | /* Function prototypes. */ |
694 | |
695 | /* affinity.c */ |
696 | |
697 | extern void gomp_init_affinity (void); |
698 | #ifdef LIBGOMP_USE_PTHREADS |
699 | extern void gomp_init_thread_affinity (pthread_attr_t *, unsigned int); |
700 | #endif |
701 | extern void **gomp_affinity_alloc (unsigned long, bool); |
702 | extern void gomp_affinity_init_place (void *); |
703 | extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long, |
704 | long, bool); |
705 | extern bool gomp_affinity_remove_cpu (void *, unsigned long); |
706 | extern bool gomp_affinity_copy_place (void *, void *, long); |
707 | extern bool gomp_affinity_same_place (void *, void *); |
708 | extern bool gomp_affinity_finalize_place_list (bool); |
709 | extern bool gomp_affinity_init_level (int, unsigned long, bool); |
710 | extern void gomp_affinity_print_place (void *); |
711 | extern void gomp_get_place_proc_ids_8 (int, int64_t *); |
712 | |
713 | /* iter.c */ |
714 | |
715 | extern int gomp_iter_static_next (long *, long *); |
716 | extern bool gomp_iter_dynamic_next_locked (long *, long *); |
717 | extern bool gomp_iter_guided_next_locked (long *, long *); |
718 | |
719 | #ifdef HAVE_SYNC_BUILTINS |
720 | extern bool gomp_iter_dynamic_next (long *, long *); |
721 | extern bool gomp_iter_guided_next (long *, long *); |
722 | #endif |
723 | |
724 | /* iter_ull.c */ |
725 | |
726 | extern int gomp_iter_ull_static_next (unsigned long long *, |
727 | unsigned long long *); |
728 | extern bool gomp_iter_ull_dynamic_next_locked (unsigned long long *, |
729 | unsigned long long *); |
730 | extern bool gomp_iter_ull_guided_next_locked (unsigned long long *, |
731 | unsigned long long *); |
732 | |
733 | #if defined HAVE_SYNC_BUILTINS && defined __LP64__ |
734 | extern bool gomp_iter_ull_dynamic_next (unsigned long long *, |
735 | unsigned long long *); |
736 | extern bool gomp_iter_ull_guided_next (unsigned long long *, |
737 | unsigned long long *); |
738 | #endif |
739 | |
740 | /* ordered.c */ |
741 | |
742 | extern void gomp_ordered_first (void); |
743 | extern void gomp_ordered_last (void); |
744 | extern void gomp_ordered_next (void); |
745 | extern void gomp_ordered_static_init (void); |
746 | extern void gomp_ordered_static_next (void); |
747 | extern void gomp_ordered_sync (void); |
748 | extern void gomp_doacross_init (unsigned, long *, long); |
749 | extern void gomp_doacross_ull_init (unsigned, unsigned long long *, |
750 | unsigned long long); |
751 | |
752 | /* parallel.c */ |
753 | |
754 | extern unsigned gomp_resolve_num_threads (unsigned, unsigned); |
755 | |
756 | /* proc.c (in config/) */ |
757 | |
758 | extern void gomp_init_num_threads (void); |
759 | extern unsigned gomp_dynamic_max_threads (void); |
760 | |
761 | /* task.c */ |
762 | |
763 | extern void gomp_init_task (struct gomp_task *, struct gomp_task *, |
764 | struct gomp_task_icv *); |
765 | extern void gomp_end_task (void); |
766 | extern void gomp_barrier_handle_tasks (gomp_barrier_state_t); |
767 | extern void gomp_task_maybe_wait_for_dependencies (void **); |
768 | extern bool gomp_create_target_task (struct gomp_device_descr *, |
769 | void (*) (void *), size_t, void **, |
770 | size_t *, unsigned short *, unsigned int, |
771 | void **, void **, |
772 | enum gomp_target_task_state); |
773 | |
774 | static void inline |
775 | gomp_finish_task (struct gomp_task *task) |
776 | { |
777 | if (__builtin_expect (task->depend_hash != NULL, 0)) |
778 | free (task->depend_hash); |
779 | } |
780 | |
781 | /* team.c */ |
782 | |
783 | extern struct gomp_team *gomp_new_team (unsigned); |
784 | extern void gomp_team_start (void (*) (void *), void *, unsigned, |
785 | unsigned, struct gomp_team *); |
786 | extern void gomp_team_end (void); |
787 | extern void gomp_free_thread (void *); |
788 | |
789 | /* target.c */ |
790 | |
791 | extern void gomp_init_targets_once (void); |
792 | extern int gomp_get_num_devices (void); |
793 | extern bool gomp_target_task_fn (void *); |
794 | |
795 | /* Splay tree definitions. */ |
796 | typedef struct splay_tree_node_s *splay_tree_node; |
797 | typedef struct splay_tree_s *splay_tree; |
798 | typedef struct splay_tree_key_s *splay_tree_key; |
799 | |
800 | struct target_var_desc { |
801 | /* Splay key. */ |
802 | splay_tree_key key; |
803 | /* True if data should be copied from device to host at the end. */ |
804 | bool copy_from; |
805 | /* True if data always should be copied from device to host at the end. */ |
806 | bool always_copy_from; |
807 | /* Relative offset against key host_start. */ |
808 | uintptr_t offset; |
809 | /* Actual length. */ |
810 | uintptr_t length; |
811 | }; |
812 | |
813 | struct target_mem_desc { |
814 | /* Reference count. */ |
815 | uintptr_t refcount; |
816 | /* All the splay nodes allocated together. */ |
817 | splay_tree_node array; |
818 | /* Start of the target region. */ |
819 | uintptr_t tgt_start; |
820 | /* End of the targer region. */ |
821 | uintptr_t tgt_end; |
822 | /* Handle to free. */ |
823 | void *to_free; |
824 | /* Previous target_mem_desc. */ |
825 | struct target_mem_desc *prev; |
826 | /* Number of items in following list. */ |
827 | size_t list_count; |
828 | |
829 | /* Corresponding target device descriptor. */ |
830 | struct gomp_device_descr *device_descr; |
831 | |
832 | /* List of target items to remove (or decrease refcount) |
833 | at the end of region. */ |
834 | struct target_var_desc list[]; |
835 | }; |
836 | |
837 | /* Special value for refcount - infinity. */ |
838 | #define REFCOUNT_INFINITY (~(uintptr_t) 0) |
839 | /* Special value for refcount - tgt_offset contains target address of the |
840 | artificial pointer to "omp declare target link" object. */ |
841 | #define REFCOUNT_LINK (~(uintptr_t) 1) |
842 | |
843 | struct splay_tree_key_s { |
844 | /* Address of the host object. */ |
845 | uintptr_t host_start; |
846 | /* Address immediately after the host object. */ |
847 | uintptr_t host_end; |
848 | /* Descriptor of the target memory. */ |
849 | struct target_mem_desc *tgt; |
850 | /* Offset from tgt->tgt_start to the start of the target object. */ |
851 | uintptr_t tgt_offset; |
852 | /* Reference count. */ |
853 | uintptr_t refcount; |
854 | /* Pointer to the original mapping of "omp declare target link" object. */ |
855 | splay_tree_key link_key; |
856 | }; |
857 | |
858 | /* The comparison function. */ |
859 | |
860 | static inline int |
861 | splay_compare (splay_tree_key x, splay_tree_key y) |
862 | { |
863 | if (x->host_start == x->host_end |
864 | && y->host_start == y->host_end) |
865 | return 0; |
866 | if (x->host_end <= y->host_start) |
867 | return -1; |
868 | if (x->host_start >= y->host_end) |
869 | return 1; |
870 | return 0; |
871 | } |
872 | |
873 | #include "splay-tree.h" |
874 | |
875 | typedef struct acc_dispatch_t |
876 | { |
877 | /* This is a linked list of data mapped using the |
878 | acc_map_data/acc_unmap_data or "acc enter data"/"acc exit data" pragmas. |
879 | Unlike mapped_data in the goacc_thread struct, unmapping can |
880 | happen out-of-order with respect to mapping. */ |
881 | /* This is guarded by the lock in the "outer" struct gomp_device_descr. */ |
882 | struct target_mem_desc *data_environ; |
883 | |
884 | /* Execute. */ |
885 | __typeof (GOMP_OFFLOAD_openacc_exec) *exec_func; |
886 | |
887 | /* Async cleanup callback registration. */ |
888 | __typeof (GOMP_OFFLOAD_openacc_register_async_cleanup) |
889 | *register_async_cleanup_func; |
890 | |
891 | /* Asynchronous routines. */ |
892 | __typeof (GOMP_OFFLOAD_openacc_async_test) *async_test_func; |
893 | __typeof (GOMP_OFFLOAD_openacc_async_test_all) *async_test_all_func; |
894 | __typeof (GOMP_OFFLOAD_openacc_async_wait) *async_wait_func; |
895 | __typeof (GOMP_OFFLOAD_openacc_async_wait_async) *async_wait_async_func; |
896 | __typeof (GOMP_OFFLOAD_openacc_async_wait_all) *async_wait_all_func; |
897 | __typeof (GOMP_OFFLOAD_openacc_async_wait_all_async) |
898 | *async_wait_all_async_func; |
899 | __typeof (GOMP_OFFLOAD_openacc_async_set_async) *async_set_async_func; |
900 | |
901 | /* Create/destroy TLS data. */ |
902 | __typeof (GOMP_OFFLOAD_openacc_create_thread_data) *create_thread_data_func; |
903 | __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data) |
904 | *destroy_thread_data_func; |
905 | |
906 | /* NVIDIA target specific routines. */ |
907 | struct { |
908 | __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_device) |
909 | *get_current_device_func; |
910 | __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_context) |
911 | *get_current_context_func; |
912 | __typeof (GOMP_OFFLOAD_openacc_cuda_get_stream) *get_stream_func; |
913 | __typeof (GOMP_OFFLOAD_openacc_cuda_set_stream) *set_stream_func; |
914 | } cuda; |
915 | } acc_dispatch_t; |
916 | |
917 | /* Various state of the accelerator device. */ |
918 | enum gomp_device_state |
919 | { |
920 | GOMP_DEVICE_UNINITIALIZED, |
921 | GOMP_DEVICE_INITIALIZED, |
922 | GOMP_DEVICE_FINALIZED |
923 | }; |
924 | |
925 | /* This structure describes accelerator device. |
926 | It contains name of the corresponding libgomp plugin, function handlers for |
927 | interaction with the device, ID-number of the device, and information about |
928 | mapped memory. */ |
929 | struct gomp_device_descr |
930 | { |
931 | /* Immutable data, which is only set during initialization, and which is not |
932 | guarded by the lock. */ |
933 | |
934 | /* The name of the device. */ |
935 | const char *name; |
936 | |
937 | /* Capabilities of device (supports OpenACC, OpenMP). */ |
938 | unsigned int capabilities; |
939 | |
940 | /* This is the ID number of device among devices of the same type. */ |
941 | int target_id; |
942 | |
943 | /* This is the TYPE of device. */ |
944 | enum offload_target_type type; |
945 | |
946 | /* Function handlers. */ |
947 | __typeof (GOMP_OFFLOAD_get_name) *get_name_func; |
948 | __typeof (GOMP_OFFLOAD_get_caps) *get_caps_func; |
949 | __typeof (GOMP_OFFLOAD_get_type) *get_type_func; |
950 | __typeof (GOMP_OFFLOAD_get_num_devices) *get_num_devices_func; |
951 | __typeof (GOMP_OFFLOAD_init_device) *init_device_func; |
952 | __typeof (GOMP_OFFLOAD_fini_device) *fini_device_func; |
953 | __typeof (GOMP_OFFLOAD_version) *version_func; |
954 | __typeof (GOMP_OFFLOAD_load_image) *load_image_func; |
955 | __typeof (GOMP_OFFLOAD_unload_image) *unload_image_func; |
956 | __typeof (GOMP_OFFLOAD_alloc) *alloc_func; |
957 | __typeof (GOMP_OFFLOAD_free) *free_func; |
958 | __typeof (GOMP_OFFLOAD_dev2host) *dev2host_func; |
959 | __typeof (GOMP_OFFLOAD_host2dev) *host2dev_func; |
960 | __typeof (GOMP_OFFLOAD_dev2dev) *dev2dev_func; |
961 | __typeof (GOMP_OFFLOAD_can_run) *can_run_func; |
962 | __typeof (GOMP_OFFLOAD_run) *run_func; |
963 | __typeof (GOMP_OFFLOAD_async_run) *async_run_func; |
964 | |
965 | /* Splay tree containing information about mapped memory regions. */ |
966 | struct splay_tree_s mem_map; |
967 | |
968 | /* Mutex for the mutable data. */ |
969 | gomp_mutex_t lock; |
970 | |
971 | /* Current state of the device. OpenACC allows to move from INITIALIZED state |
972 | back to UNINITIALIZED state. OpenMP allows only to move from INITIALIZED |
973 | to FINALIZED state (at program shutdown). */ |
974 | enum gomp_device_state state; |
975 | |
976 | /* OpenACC-specific data and functions. */ |
977 | /* This is mutable because of its mutable data_environ and target_data |
978 | members. */ |
979 | acc_dispatch_t openacc; |
980 | }; |
981 | |
982 | /* Kind of the pragma, for which gomp_map_vars () is called. */ |
983 | enum gomp_map_vars_kind |
984 | { |
985 | GOMP_MAP_VARS_OPENACC, |
986 | GOMP_MAP_VARS_TARGET, |
987 | GOMP_MAP_VARS_DATA, |
988 | GOMP_MAP_VARS_ENTER_DATA |
989 | }; |
990 | |
991 | extern void gomp_acc_insert_pointer (size_t, void **, size_t *, void *); |
992 | extern void gomp_acc_remove_pointer (void *, bool, int, int); |
993 | |
994 | extern struct target_mem_desc *gomp_map_vars (struct gomp_device_descr *, |
995 | size_t, void **, void **, |
996 | size_t *, void *, bool, |
997 | enum gomp_map_vars_kind); |
998 | extern void gomp_unmap_vars (struct target_mem_desc *, bool); |
999 | extern void gomp_init_device (struct gomp_device_descr *); |
1000 | extern void gomp_free_memmap (struct splay_tree_s *); |
1001 | extern void gomp_unload_device (struct gomp_device_descr *); |
1002 | |
1003 | /* work.c */ |
1004 | |
1005 | extern void gomp_init_work_share (struct gomp_work_share *, bool, unsigned); |
1006 | extern void gomp_fini_work_share (struct gomp_work_share *); |
1007 | extern bool gomp_work_share_start (bool); |
1008 | extern void gomp_work_share_end (void); |
1009 | extern bool gomp_work_share_end_cancel (void); |
1010 | extern void gomp_work_share_end_nowait (void); |
1011 | |
1012 | static inline void |
1013 | gomp_work_share_init_done (void) |
1014 | { |
1015 | struct gomp_thread *thr = gomp_thread (); |
1016 | if (__builtin_expect (thr->ts.last_work_share != NULL, 1)) |
1017 | gomp_ptrlock_set (&thr->ts.last_work_share->next_ws, thr->ts.work_share); |
1018 | } |
1019 | |
1020 | #ifdef HAVE_ATTRIBUTE_VISIBILITY |
1021 | # pragma GCC visibility pop |
1022 | #endif |
1023 | |
1024 | /* Now that we're back to default visibility, include the globals. */ |
1025 | #include "libgomp_g.h" |
1026 | |
1027 | /* Include omp.h by parts. */ |
1028 | #include "omp-lock.h" |
1029 | #define _LIBGOMP_OMP_LOCK_DEFINED 1 |
1030 | #include "omp.h.in" |
1031 | |
1032 | #if !defined (HAVE_ATTRIBUTE_VISIBILITY) \ |
1033 | || !defined (HAVE_ATTRIBUTE_ALIAS) \ |
1034 | || !defined (HAVE_AS_SYMVER_DIRECTIVE) \ |
1035 | || !defined (PIC) \ |
1036 | || !defined (HAVE_SYMVER_SYMBOL_RENAMING_RUNTIME_SUPPORT) |
1037 | # undef LIBGOMP_GNU_SYMBOL_VERSIONING |
1038 | #endif |
1039 | |
1040 | #ifdef LIBGOMP_GNU_SYMBOL_VERSIONING |
1041 | extern void gomp_init_lock_30 (omp_lock_t *) __GOMP_NOTHROW; |
1042 | extern void gomp_destroy_lock_30 (omp_lock_t *) __GOMP_NOTHROW; |
1043 | extern void gomp_set_lock_30 (omp_lock_t *) __GOMP_NOTHROW; |
1044 | extern void gomp_unset_lock_30 (omp_lock_t *) __GOMP_NOTHROW; |
1045 | extern int gomp_test_lock_30 (omp_lock_t *) __GOMP_NOTHROW; |
1046 | extern void gomp_init_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; |
1047 | extern void gomp_destroy_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; |
1048 | extern void gomp_set_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; |
1049 | extern void gomp_unset_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; |
1050 | extern int gomp_test_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; |
1051 | |
1052 | extern void gomp_init_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; |
1053 | extern void gomp_destroy_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; |
1054 | extern void gomp_set_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; |
1055 | extern void gomp_unset_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; |
1056 | extern int gomp_test_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; |
1057 | extern void gomp_init_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; |
1058 | extern void gomp_destroy_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; |
1059 | extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; |
1060 | extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; |
1061 | extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; |
1062 | |
1063 | # define omp_lock_symver(fn) \ |
1064 | __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \ |
1065 | __asm (".symver g" #fn "_25, " #fn "@OMP_1.0"); |
1066 | #else |
1067 | # define gomp_init_lock_30 omp_init_lock |
1068 | # define gomp_destroy_lock_30 omp_destroy_lock |
1069 | # define gomp_set_lock_30 omp_set_lock |
1070 | # define gomp_unset_lock_30 omp_unset_lock |
1071 | # define gomp_test_lock_30 omp_test_lock |
1072 | # define gomp_init_nest_lock_30 omp_init_nest_lock |
1073 | # define gomp_destroy_nest_lock_30 omp_destroy_nest_lock |
1074 | # define gomp_set_nest_lock_30 omp_set_nest_lock |
1075 | # define gomp_unset_nest_lock_30 omp_unset_nest_lock |
1076 | # define gomp_test_nest_lock_30 omp_test_nest_lock |
1077 | #endif |
1078 | |
1079 | #ifdef HAVE_ATTRIBUTE_VISIBILITY |
1080 | # define attribute_hidden __attribute__ ((visibility ("hidden"))) |
1081 | #else |
1082 | # define attribute_hidden |
1083 | #endif |
1084 | |
1085 | #ifdef HAVE_ATTRIBUTE_ALIAS |
1086 | # define strong_alias(fn, al) \ |
1087 | extern __typeof (fn) al __attribute__ ((alias (#fn))); |
1088 | |
1089 | # define ialias_ulp ialias_str1(__USER_LABEL_PREFIX__) |
1090 | # define ialias_str1(x) ialias_str2(x) |
1091 | # define ialias_str2(x) #x |
1092 | # define ialias(fn) \ |
1093 | extern __typeof (fn) gomp_ialias_##fn \ |
1094 | __attribute__ ((alias (#fn))) attribute_hidden; |
1095 | # define ialias_redirect(fn) \ |
1096 | extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden; |
1097 | # define ialias_call(fn) gomp_ialias_ ## fn |
1098 | #else |
1099 | # define ialias(fn) |
1100 | # define ialias_redirect(fn) |
1101 | # define ialias_call(fn) fn |
1102 | #endif |
1103 | |
1104 | /* Helper function for priority_node_to_task() and |
1105 | task_to_priority_node(). |
1106 | |
1107 | Return the offset from a task to its priority_node entry. The |
1108 | priority_node entry is has a type of TYPE. */ |
1109 | |
1110 | static inline size_t |
1111 | priority_queue_offset (enum priority_queue_type type) |
1112 | { |
1113 | return offsetof (struct gomp_task, pnode[(int) type]); |
1114 | } |
1115 | |
1116 | /* Return the task associated with a priority NODE of type TYPE. */ |
1117 | |
1118 | static inline struct gomp_task * |
1119 | priority_node_to_task (enum priority_queue_type type, |
1120 | struct priority_node *node) |
1121 | { |
1122 | return (struct gomp_task *) ((char *) node - priority_queue_offset (type)); |
1123 | } |
1124 | |
1125 | /* Return the priority node of type TYPE for a given TASK. */ |
1126 | |
1127 | static inline struct priority_node * |
1128 | task_to_priority_node (enum priority_queue_type type, |
1129 | struct gomp_task *task) |
1130 | { |
1131 | return (struct priority_node *) ((char *) task |
1132 | + priority_queue_offset (type)); |
1133 | } |
1134 | #endif /* LIBGOMP_H */ |
1135 | |