1	/*
2	* z_Linux_util.cpp -- platform specific routines.
3	*/
4
5	//===----------------------------------------------------------------------===//
6	//
7	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8	// See https://llvm.org/LICENSE.txt for license information.
9	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "kmp.h"
14	#include "kmp_affinity.h"
15	#include "kmp_i18n.h"
16	#include "kmp_io.h"
17	#include "kmp_itt.h"
18	#include "kmp_lock.h"
19	#include "kmp_stats.h"
20	#include "kmp_str.h"
21	#include "kmp_wait_release.h"
22	#include "kmp_wrapper_getpid.h"
23
24	#if !KMP_OS_DRAGONFLY && !KMP_OS_FREEBSD && !KMP_OS_NETBSD && !KMP_OS_OPENBSD
25	#include <alloca.h>
26	#endif
27	#include <math.h> // HUGE_VAL.
28	#if KMP_OS_LINUX
29	#include <semaphore.h>
30	#endif // KMP_OS_LINUX
31	#include <sys/resource.h>
32	#if !KMP_OS_AIX
33	#include <sys/syscall.h>
34	#endif
35	#include <sys/time.h>
36	#include <sys/times.h>
37	#include <unistd.h>
38
39	#if KMP_OS_LINUX
40	#include <sys/sysinfo.h>
41	#if KMP_USE_FUTEX
42	// We should really include <futex.h>, but that causes compatibility problems on
43	// different Linux* OS distributions that either require that you include (or
44	// break when you try to include) <pci/types.h>. Since all we need is the two
45	// macros below (which are part of the kernel ABI, so can't change) we just
46	// define the constants here and don't include <futex.h>
47	#ifndef FUTEX_WAIT
48	#define FUTEX_WAIT 0
49	#endif
50	#ifndef FUTEX_WAKE
51	#define FUTEX_WAKE 1
52	#endif
53	#endif
54	#elif KMP_OS_DARWIN
55	#include <mach/mach.h>
56	#include <sys/sysctl.h>
57	#elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD
58	#include <sys/types.h>
59	#include <sys/sysctl.h>
60	#include <sys/user.h>
61	#include <pthread_np.h>
62	#if KMP_OS_DRAGONFLY
63	#include <kvm.h>
64	#endif
65	#elif KMP_OS_NETBSD || KMP_OS_OPENBSD
66	#include <sys/types.h>
67	#include <sys/sysctl.h>
68	#if KMP_OS_NETBSD
69	#include <sched.h>
70	#endif
71	#elif KMP_OS_SOLARIS
72	#include <libproc.h>
73	#include <procfs.h>
74	#include <thread.h>
75	#include <sys/loadavg.h>
76	#endif
77
78	#include <ctype.h>
79	#include <dirent.h>
80	#include <fcntl.h>
81
82	struct kmp_sys_timer {
83	struct timespec start;
84	};
85
86	#ifndef TIMEVAL_TO_TIMESPEC
87	// Convert timeval to timespec.
88	#define TIMEVAL_TO_TIMESPEC(tv, ts) \
89	do { \
90	(ts)->tv_sec = (tv)->tv_sec; \
91	(ts)->tv_nsec = (tv)->tv_usec * 1000; \
92	} while (0)
93	#endif
94
95	// Convert timespec to nanoseconds.
96	#define TS2NS(timespec) \
97	(((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec)
98
99	static struct kmp_sys_timer __kmp_sys_timer_data;
100
101	#if KMP_HANDLE_SIGNALS
102	typedef void (*sig_func_t)(int);
103	STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[NSIG];
104	static sigset_t __kmp_sigset;
105	#endif
106
107	static int __kmp_init_runtime = FALSE;
108
109	static int __kmp_fork_count = 0;
110
111	static pthread_condattr_t __kmp_suspend_cond_attr;
112	static pthread_mutexattr_t __kmp_suspend_mutex_attr;
113
114	static kmp_cond_align_t __kmp_wait_cv;
115	static kmp_mutex_align_t __kmp_wait_mx;
116
117	kmp_uint64 __kmp_ticks_per_msec = 1000000;
118	kmp_uint64 __kmp_ticks_per_usec = 1000;
119
120	#ifdef DEBUG_SUSPEND
121	static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) {
122	KMP_SNPRINTF(buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
123	cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
124	cond->c_cond.__c_waiting);
125	}
126	#endif
127
128	#if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
129	KMP_OS_AIX) && \
130	KMP_AFFINITY_SUPPORTED)
131
132	/* Affinity support */
133
134	void __kmp_affinity_bind_thread(int which) {
135	KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
136	"Illegal set affinity operation when not capable");
137
138	kmp_affin_mask_t *mask;
139	KMP_CPU_ALLOC_ON_STACK(mask);
140	KMP_CPU_ZERO(mask);
141	KMP_CPU_SET(which, mask);
142	__kmp_set_system_affinity(mask, TRUE);
143	KMP_CPU_FREE_FROM_STACK(mask);
144	}
145
146	#if KMP_OS_AIX
147	void __kmp_affinity_determine_capable(const char *env_var) {
148	// All versions of AIX support bindprocessor().
149
150	size_t mask_size = __kmp_xproc / CHAR_BIT;
151	// Round up to byte boundary.
152	if (__kmp_xproc % CHAR_BIT)
153	++mask_size;
154
155	// Round up to the mask_size_type boundary.
156	if (mask_size % sizeof(__kmp_affin_mask_size))
157	mask_size += sizeof(__kmp_affin_mask_size) -
158	mask_size % sizeof(__kmp_affin_mask_size);
159	KMP_AFFINITY_ENABLE(mask_size);
160	KA_TRACE(10,
161	("__kmp_affinity_determine_capable: "
162	"AIX OS affinity interface bindprocessor functional (mask size = "
163	"%" KMP_SIZE_T_SPEC ").\n",
164	__kmp_affin_mask_size));
165	}
166
167	#else // !KMP_OS_AIX
168
169	/* Determine if we can access affinity functionality on this version of
170	* Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
171	* __kmp_affin_mask_size to the appropriate value (0 means not capable). */
172	void __kmp_affinity_determine_capable(const char *env_var) {
173	// Check and see if the OS supports thread affinity.
174
175	#if KMP_OS_LINUX
176	#define KMP_CPU_SET_SIZE_LIMIT (1024 * 1024)
177	#define KMP_CPU_SET_TRY_SIZE CACHE_LINE
178	#elif KMP_OS_FREEBSD || KMP_OS_DRAGONFLY
179	#define KMP_CPU_SET_SIZE_LIMIT (sizeof(cpuset_t))
180	#elif KMP_OS_NETBSD
181	#define KMP_CPU_SET_SIZE_LIMIT (256)
182	#endif
183
184	int verbose = __kmp_affinity.flags.verbose;
185	int warnings = __kmp_affinity.flags.warnings;
186	enum affinity_type type = __kmp_affinity.type;
187
188	#if KMP_OS_LINUX
189	long gCode;
190	unsigned char *buf;
191	buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
192
193	// If the syscall returns a suggestion for the size,
194	// then we don't have to search for an appropriate size.
195	gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_TRY_SIZE, buf);
196	KA_TRACE(30, ("__kmp_affinity_determine_capable: "
197	"initial getaffinity call returned %ld errno = %d\n",
198	gCode, errno));
199
200	if (gCode < 0 && errno != EINVAL) {
201	// System call not supported
202	if (verbose ||
203	(warnings && (type != affinity_none) && (type != affinity_default) &&
204	(type != affinity_disabled))) {
205	int error = errno;
206	kmp_msg_t err_code = KMP_ERR(error);
207	__kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
208	err_code, __kmp_msg_null);
209	if (__kmp_generate_warnings == kmp_warnings_off) {
210	__kmp_str_free(str: &err_code.str);
211	}
212	}
213	KMP_AFFINITY_DISABLE();
214	KMP_INTERNAL_FREE(buf);
215	return;
216	} else if (gCode > 0) {
217	// The optimal situation: the OS returns the size of the buffer it expects.
218	KMP_AFFINITY_ENABLE(gCode);
219	KA_TRACE(10, ("__kmp_affinity_determine_capable: "
220	"affinity supported (mask size %d)\n",
221	(int)__kmp_affin_mask_size));
222	KMP_INTERNAL_FREE(buf);
223	return;
224	}
225
226	// Call the getaffinity system call repeatedly with increasing set sizes
227	// until we succeed, or reach an upper bound on the search.
228	KA_TRACE(30, ("__kmp_affinity_determine_capable: "
229	"searching for proper set size\n"));
230	int size;
231	for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
232	gCode = syscall(__NR_sched_getaffinity, 0, size, buf);
233	KA_TRACE(30, ("__kmp_affinity_determine_capable: "
234	"getaffinity for mask size %ld returned %ld errno = %d\n",
235	size, gCode, errno));
236
237	if (gCode < 0) {
238	if (errno == ENOSYS) {
239	// We shouldn't get here
240	KA_TRACE(30, ("__kmp_affinity_determine_capable: "
241	"inconsistent OS call behavior: errno == ENOSYS for mask "
242	"size %d\n",
243	size));
244	if (verbose ||
245	(warnings && (type != affinity_none) &&
246	(type != affinity_default) && (type != affinity_disabled))) {
247	int error = errno;
248	kmp_msg_t err_code = KMP_ERR(error);
249	__kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
250	err_code, __kmp_msg_null);
251	if (__kmp_generate_warnings == kmp_warnings_off) {
252	__kmp_str_free(str: &err_code.str);
253	}
254	}
255	KMP_AFFINITY_DISABLE();
256	KMP_INTERNAL_FREE(buf);
257	return;
258	}
259	continue;
260	}
261
262	KMP_AFFINITY_ENABLE(gCode);
263	KA_TRACE(10, ("__kmp_affinity_determine_capable: "
264	"affinity supported (mask size %d)\n",
265	(int)__kmp_affin_mask_size));
266	KMP_INTERNAL_FREE(buf);
267	return;
268	}
269	#elif KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY
270	long gCode;
271	unsigned char *buf;
272	buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
273	gCode = pthread_getaffinity_np(pthread_self(), KMP_CPU_SET_SIZE_LIMIT,
274	reinterpret_cast<cpuset_t *>(buf));
275	KA_TRACE(30, ("__kmp_affinity_determine_capable: "
276	"initial getaffinity call returned %d errno = %d\n",
277	gCode, errno));
278	if (gCode == 0) {
279	KMP_AFFINITY_ENABLE(KMP_CPU_SET_SIZE_LIMIT);
280	KA_TRACE(10, ("__kmp_affinity_determine_capable: "
281	"affinity supported (mask size %d)\n",
282	(int)__kmp_affin_mask_size));
283	KMP_INTERNAL_FREE(buf);
284	return;
285	}
286	#endif
287	KMP_INTERNAL_FREE(buf);
288
289	// Affinity is not supported
290	KMP_AFFINITY_DISABLE();
291	KA_TRACE(10, ("__kmp_affinity_determine_capable: "
292	"cannot determine mask size - affinity not supported\n"));
293	if (verbose || (warnings && (type != affinity_none) &&
294	(type != affinity_default) && (type != affinity_disabled))) {
295	KMP_WARNING(AffCantGetMaskSize, env_var);
296	}
297	}
298	#endif // KMP_OS_AIX
299	#endif // (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
300	KMP_OS_DRAGONFLY || KMP_OS_AIX) && KMP_AFFINITY_SUPPORTED
301
302	#if KMP_USE_FUTEX
303
304	int __kmp_futex_determine_capable() {
305	int loc = 0;
306	long rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0);
307	int retval = (rc == 0) || (errno != ENOSYS);
308
309	KA_TRACE(10,
310	("__kmp_futex_determine_capable: rc = %d errno = %d\n", rc, errno));
311	KA_TRACE(10, ("__kmp_futex_determine_capable: futex syscall%s supported\n",
312	retval ? "" : " not"));
313
314	return retval;
315	}
316
317	#endif // KMP_USE_FUTEX
318
319	#if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM) && (!KMP_ASM_INTRINS)
320	/* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
321	use compare_and_store for these routines */
322
323	kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 d) {
324	kmp_int8 old_value, new_value;
325
326	old_value = TCR_1(*p);
327	new_value = old_value | d;
328
329	while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
330	KMP_CPU_PAUSE();
331	old_value = TCR_1(*p);
332	new_value = old_value | d;
333	}
334	return old_value;
335	}
336
337	kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 d) {
338	kmp_int8 old_value, new_value;
339
340	old_value = TCR_1(*p);
341	new_value = old_value & d;
342
343	while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
344	KMP_CPU_PAUSE();
345	old_value = TCR_1(*p);
346	new_value = old_value & d;
347	}
348	return old_value;
349	}
350
351	kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 d) {
352	kmp_uint32 old_value, new_value;
353
354	old_value = TCR_4(*p);
355	new_value = old_value | d;
356
357	while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
358	KMP_CPU_PAUSE();
359	old_value = TCR_4(*p);
360	new_value = old_value | d;
361	}
362	return old_value;
363	}
364
365	kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) {
366	kmp_uint32 old_value, new_value;
367
368	old_value = TCR_4(*p);
369	new_value = old_value & d;
370
371	while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
372	KMP_CPU_PAUSE();
373	old_value = TCR_4(*p);
374	new_value = old_value & d;
375	}
376	return old_value;
377	}
378
379	#if KMP_ARCH_X86 || KMP_ARCH_WASM
380	kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) {
381	kmp_int8 old_value, new_value;
382
383	old_value = TCR_1(*p);
384	new_value = old_value + d;
385
386	while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
387	KMP_CPU_PAUSE();
388	old_value = TCR_1(*p);
389	new_value = old_value + d;
390	}
391	return old_value;
392	}
393
394	kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 d) {
395	kmp_int64 old_value, new_value;
396
397	old_value = TCR_8(*p);
398	new_value = old_value + d;
399
400	while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
401	KMP_CPU_PAUSE();
402	old_value = TCR_8(*p);
403	new_value = old_value + d;
404	}
405	return old_value;
406	}
407	#endif /* KMP_ARCH_X86 */
408
409	kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 d) {
410	kmp_uint64 old_value, new_value;
411
412	old_value = TCR_8(*p);
413	new_value = old_value | d;
414	while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
415	KMP_CPU_PAUSE();
416	old_value = TCR_8(*p);
417	new_value = old_value | d;
418	}
419	return old_value;
420	}
421
422	kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 d) {
423	kmp_uint64 old_value, new_value;
424
425	old_value = TCR_8(*p);
426	new_value = old_value & d;
427	while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
428	KMP_CPU_PAUSE();
429	old_value = TCR_8(*p);
430	new_value = old_value & d;
431	}
432	return old_value;
433	}
434
435	#endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
436
437	void __kmp_terminate_thread(int gtid) {
438	int status;
439	kmp_info_t *th = __kmp_threads[gtid];
440
441	if (!th)
442	return;
443
444	#ifdef KMP_CANCEL_THREADS
445	KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
446	status = pthread_cancel(th: th->th.th_info.ds.ds_thread);
447	if (status != 0 && status != ESRCH) {
448	__kmp_fatal(KMP_MSG(CantTerminateWorkerThread), KMP_ERR(status),
449	__kmp_msg_null);
450	}
451	#endif
452	KMP_YIELD(TRUE);
453	} //
454
455	/* Set thread stack info.
456	If values are unreasonable, assume call failed and use incremental stack
457	refinement method instead. Returns TRUE if the stack parameters could be
458	determined exactly, FALSE if incremental refinement is necessary. */
459	static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
460	int stack_data;
461	#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
462	KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
463	int status;
464	size_t size = 0;
465	void *addr = 0;
466
467	/* Always do incremental stack refinement for ubermaster threads since the
468	initial thread stack range can be reduced by sibling thread creation so
469	pthread_attr_getstack may cause thread gtid aliasing */
470	if (!KMP_UBER_GTID(gtid)) {
471
472	#if KMP_OS_SOLARIS
473	stack_t s;
474	if ((status = thr_stksegment(&s)) < 0) {
475	KMP_CHECK_SYSFAIL("thr_stksegment", status);
476	}
477
478	addr = s.ss_sp;
479	size = s.ss_size;
480	KA_TRACE(60, ("__kmp_set_stack_info: T#%d thr_stksegment returned size:"
481	" %lu, low addr: %p\n",
482	gtid, size, addr));
483	#else
484	pthread_attr_t attr;
485	/* Fetch the real thread attributes */
486	status = pthread_attr_init(attr: &attr);
487	KMP_CHECK_SYSFAIL("pthread_attr_init", status);
488	#if KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD
489	status = pthread_attr_get_np(pthread_self(), &attr);
490	KMP_CHECK_SYSFAIL("pthread_attr_get_np", status);
491	#else
492	status = pthread_getattr_np(th: pthread_self(), attr: &attr);
493	KMP_CHECK_SYSFAIL("pthread_getattr_np", status);
494	#endif
495	status = pthread_attr_getstack(attr: &attr, stackaddr: &addr, stacksize: &size);
496	KMP_CHECK_SYSFAIL("pthread_attr_getstack", status);
497	KA_TRACE(60,
498	("__kmp_set_stack_info: T#%d pthread_attr_getstack returned size:"
499	" %lu, low addr: %p\n",
500	gtid, size, addr));
501	status = pthread_attr_destroy(attr: &attr);
502	KMP_CHECK_SYSFAIL("pthread_attr_destroy", status);
503	#endif
504	}
505
506	if (size != 0 && addr != 0) { // was stack parameter determination successful?
507	/* Store the correct base and size */
508	TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
509	TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
510	TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
511	return TRUE;
512	}
513	#endif /* KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD \
514	|| KMP_OS_HURD || KMP_OS_SOLARIS */
515	/* Use incremental refinement starting from initial conservative estimate */
516	TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
517	TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
518	TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
519	return FALSE;
520	}
521
522	static void *__kmp_launch_worker(void *thr) {
523	int status, old_type, old_state;
524	#ifdef KMP_BLOCK_SIGNALS
525	sigset_t new_set, old_set;
526	#endif /* KMP_BLOCK_SIGNALS */
527	void *exit_val;
528	#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
529	KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
530	void *volatile padding = 0;
531	#endif
532	int gtid;
533
534	gtid = ((kmp_info_t *)thr)->th.th_info.ds.ds_gtid;
535	__kmp_gtid_set_specific(gtid);
536	#ifdef KMP_TDATA_GTID
537	__kmp_gtid = gtid;
538	#endif
539	#if KMP_STATS_ENABLED
540	// set thread local index to point to thread-specific stats
541	__kmp_stats_thread_ptr = ((kmp_info_t *)thr)->th.th_stats;
542	__kmp_stats_thread_ptr->startLife();
543	KMP_SET_THREAD_STATE(IDLE);
544	KMP_INIT_PARTITIONED_TIMERS(OMP_idle);
545	#endif
546
547	#if USE_ITT_BUILD
548	__kmp_itt_thread_name(gtid);
549	#endif /* USE_ITT_BUILD */
550
551	#if KMP_AFFINITY_SUPPORTED
552	__kmp_affinity_bind_init_mask(gtid);
553	#endif
554
555	#ifdef KMP_CANCEL_THREADS
556	status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, oldtype: &old_type);
557	KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
558	// josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
559	status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, oldstate: &old_state);
560	KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
561	#endif
562
563	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
564	// Set FP control regs to be a copy of the parallel initialization thread's.
565	__kmp_clear_x87_fpu_status_word();
566	__kmp_load_x87_fpu_control_word(p: &__kmp_init_x87_fpu_control_word);
567	__kmp_load_mxcsr(p: &__kmp_init_mxcsr);
568	#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
569
570	#ifdef KMP_BLOCK_SIGNALS
571	status = sigfillset(&new_set);
572	KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
573	status = pthread_sigmask(SIG_BLOCK, &new_set, &old_set);
574	KMP_CHECK_SYSFAIL("pthread_sigmask", status);
575	#endif /* KMP_BLOCK_SIGNALS */
576
577	#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
578	KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
579	if (__kmp_stkoffset > 0 && gtid > 0) {
580	padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
581	(void)padding;
582	}
583	#endif
584
585	KMP_MB();
586	__kmp_set_stack_info(gtid, th: (kmp_info_t *)thr);
587
588	__kmp_check_stack_overlap(thr: (kmp_info_t *)thr);
589
590	exit_val = __kmp_launch_thread(thr: (kmp_info_t *)thr);
591
592	#ifdef KMP_BLOCK_SIGNALS
593	status = pthread_sigmask(SIG_SETMASK, &old_set, NULL);
594	KMP_CHECK_SYSFAIL("pthread_sigmask", status);
595	#endif /* KMP_BLOCK_SIGNALS */
596
597	return exit_val;
598	}
599
600	#if KMP_USE_MONITOR
601	/* The monitor thread controls all of the threads in the complex */
602
603	static void *__kmp_launch_monitor(void *thr) {
604	int status, old_type, old_state;
605	#ifdef KMP_BLOCK_SIGNALS
606	sigset_t new_set;
607	#endif /* KMP_BLOCK_SIGNALS */
608	struct timespec interval;
609
610	KMP_MB(); /* Flush all pending memory write invalidates. */
611
612	KA_TRACE(10, ("__kmp_launch_monitor: #1 launched\n"));
613
614	/* register us as the monitor thread */
615	__kmp_gtid_set_specific(KMP_GTID_MONITOR);
616	#ifdef KMP_TDATA_GTID
617	__kmp_gtid = KMP_GTID_MONITOR;
618	#endif
619
620	KMP_MB();
621
622	#if USE_ITT_BUILD
623	// Instruct Intel(R) Threading Tools to ignore monitor thread.
624	__kmp_itt_thread_ignore();
625	#endif /* USE_ITT_BUILD */
626
627	__kmp_set_stack_info(((kmp_info_t *)thr)->th.th_info.ds.ds_gtid,
628	(kmp_info_t *)thr);
629
630	__kmp_check_stack_overlap((kmp_info_t *)thr);
631
632	#ifdef KMP_CANCEL_THREADS
633	status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
634	KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
635	// josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
636	status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
637	KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
638	#endif
639
640	#if KMP_REAL_TIME_FIX
641	// This is a potential fix which allows application with real-time scheduling
642	// policy work. However, decision about the fix is not made yet, so it is
643	// disabled by default.
644	{ // Are program started with real-time scheduling policy?
645	int sched = sched_getscheduler(0);
646	if (sched == SCHED_FIFO || sched == SCHED_RR) {
647	// Yes, we are a part of real-time application. Try to increase the
648	// priority of the monitor.
649	struct sched_param param;
650	int max_priority = sched_get_priority_max(sched);
651	int rc;
652	KMP_WARNING(RealTimeSchedNotSupported);
653	sched_getparam(0, &param);
654	if (param.sched_priority < max_priority) {
655	param.sched_priority += 1;
656	rc = sched_setscheduler(0, sched, &param);
657	if (rc != 0) {
658	int error = errno;
659	kmp_msg_t err_code = KMP_ERR(error);
660	__kmp_msg(kmp_ms_warning, KMP_MSG(CantChangeMonitorPriority),
661	err_code, KMP_MSG(MonitorWillStarve), __kmp_msg_null);
662	if (__kmp_generate_warnings == kmp_warnings_off) {
663	__kmp_str_free(&err_code.str);
664	}
665	}
666	} else {
667	// We cannot abort here, because number of CPUs may be enough for all
668	// the threads, including the monitor thread, so application could
669	// potentially work...
670	__kmp_msg(kmp_ms_warning, KMP_MSG(RunningAtMaxPriority),
671	KMP_MSG(MonitorWillStarve), KMP_HNT(RunningAtMaxPriority),
672	__kmp_msg_null);
673	}
674	}
675	// AC: free thread that waits for monitor started
676	TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
677	}
678	#endif // KMP_REAL_TIME_FIX
679
680	KMP_MB(); /* Flush all pending memory write invalidates. */
681
682	if (__kmp_monitor_wakeups == 1) {
683	interval.tv_sec = 1;
684	interval.tv_nsec = 0;
685	} else {
686	interval.tv_sec = 0;
687	interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
688	}
689
690	KA_TRACE(10, ("__kmp_launch_monitor: #2 monitor\n"));
691
692	while (!TCR_4(__kmp_global.g.g_done)) {
693	struct timespec now;
694	struct timeval tval;
695
696	/* This thread monitors the state of the system */
697
698	KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
699
700	status = gettimeofday(&tval, NULL);
701	KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
702	TIMEVAL_TO_TIMESPEC(&tval, &now);
703
704	now.tv_sec += interval.tv_sec;
705	now.tv_nsec += interval.tv_nsec;
706
707	if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
708	now.tv_sec += 1;
709	now.tv_nsec -= KMP_NSEC_PER_SEC;
710	}
711
712	status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
713	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
714	// AC: the monitor should not fall asleep if g_done has been set
715	if (!TCR_4(__kmp_global.g.g_done)) { // check once more under mutex
716	status = pthread_cond_timedwait(&__kmp_wait_cv.c_cond,
717	&__kmp_wait_mx.m_mutex, &now);
718	if (status != 0) {
719	if (status != ETIMEDOUT && status != EINTR) {
720	KMP_SYSFAIL("pthread_cond_timedwait", status);
721	}
722	}
723	}
724	status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
725	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
726
727	TCW_4(__kmp_global.g.g_time.dt.t_value,
728	TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
729
730	KMP_MB(); /* Flush all pending memory write invalidates. */
731	}
732
733	KA_TRACE(10, ("__kmp_launch_monitor: #3 cleanup\n"));
734
735	#ifdef KMP_BLOCK_SIGNALS
736	status = sigfillset(&new_set);
737	KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
738	status = pthread_sigmask(SIG_UNBLOCK, &new_set, NULL);
739	KMP_CHECK_SYSFAIL("pthread_sigmask", status);
740	#endif /* KMP_BLOCK_SIGNALS */
741
742	KA_TRACE(10, ("__kmp_launch_monitor: #4 finished\n"));
743
744	if (__kmp_global.g.g_abort != 0) {
745	/* now we need to terminate the worker threads */
746	/* the value of t_abort is the signal we caught */
747
748	int gtid;
749
750	KA_TRACE(10, ("__kmp_launch_monitor: #5 terminate sig=%d\n",
751	__kmp_global.g.g_abort));
752
753	/* terminate the OpenMP worker threads */
754	/* TODO this is not valid for sibling threads!!
755	* the uber master might not be 0 anymore.. */
756	for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
757	__kmp_terminate_thread(gtid);
758
759	__kmp_cleanup();
760
761	KA_TRACE(10, ("__kmp_launch_monitor: #6 raise sig=%d\n",
762	__kmp_global.g.g_abort));
763
764	if (__kmp_global.g.g_abort > 0)
765	raise(__kmp_global.g.g_abort);
766	}
767
768	KA_TRACE(10, ("__kmp_launch_monitor: #7 exit\n"));
769
770	return thr;
771	}
772	#endif // KMP_USE_MONITOR
773
774	void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
775	pthread_t handle;
776	pthread_attr_t thread_attr;
777	int status;
778
779	th->th.th_info.ds.ds_gtid = gtid;
780
781	#if KMP_STATS_ENABLED
782	// sets up worker thread stats
783	__kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
784
785	// th->th.th_stats is used to transfer thread-specific stats-pointer to
786	// __kmp_launch_worker. So when thread is created (goes into
787	// __kmp_launch_worker) it will set its thread local pointer to
788	// th->th.th_stats
789	if (!KMP_UBER_GTID(gtid)) {
790	th->th.th_stats = __kmp_stats_list->push_back(gtid);
791	} else {
792	// For root threads, __kmp_stats_thread_ptr is set in __kmp_register_root(),
793	// so set the th->th.th_stats field to it.
794	th->th.th_stats = __kmp_stats_thread_ptr;
795	}
796	__kmp_release_tas_lock(&__kmp_stats_lock, gtid);
797
798	#endif // KMP_STATS_ENABLED
799
800	if (KMP_UBER_GTID(gtid)) {
801	KA_TRACE(10, ("__kmp_create_worker: uber thread (%d)\n", gtid));
802	th->th.th_info.ds.ds_thread = pthread_self();
803	__kmp_set_stack_info(gtid, th);
804	__kmp_check_stack_overlap(thr: th);
805	return;
806	}
807
808	KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
809
810	KMP_MB(); /* Flush all pending memory write invalidates. */
811
812	#ifdef KMP_THREAD_ATTR
813	status = pthread_attr_init(attr: &thread_attr);
814	if (status != 0) {
815	__kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
816	}
817	status = pthread_attr_setdetachstate(attr: &thread_attr, PTHREAD_CREATE_JOINABLE);
818	if (status != 0) {
819	__kmp_fatal(KMP_MSG(CantSetWorkerState), KMP_ERR(status), __kmp_msg_null);
820	}
821
822	/* Set stack size for this thread now.
823	The multiple of 2 is there because on some machines, requesting an unusual
824	stacksize causes the thread to have an offset before the dummy alloca()
825	takes place to create the offset. Since we want the user to have a
826	sufficient stacksize AND support a stack offset, we alloca() twice the
827	offset so that the upcoming alloca() does not eliminate any premade offset,
828	and also gives the user the stack space they requested for all threads */
829	stack_size += gtid * __kmp_stkoffset * 2;
830
831	KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
832	"__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
833	gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
834
835	#ifdef _POSIX_THREAD_ATTR_STACKSIZE
836	status = pthread_attr_setstacksize(attr: &thread_attr, stacksize: stack_size);
837	#ifdef KMP_BACKUP_STKSIZE
838	if (status != 0) {
839	if (!__kmp_env_stksize) {
840	stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
841	__kmp_stksize = KMP_BACKUP_STKSIZE;
842	KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
843	"__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
844	"bytes\n",
845	gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
846	status = pthread_attr_setstacksize(attr: &thread_attr, stacksize: stack_size);
847	}
848	}
849	#endif /* KMP_BACKUP_STKSIZE */
850	if (status != 0) {
851	__kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
852	KMP_HNT(ChangeWorkerStackSize), __kmp_msg_null);
853	}
854	#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
855
856	#endif /* KMP_THREAD_ATTR */
857
858	status =
859	pthread_create(newthread: &handle, attr: &thread_attr, start_routine: __kmp_launch_worker, arg: (void *)th);
860	if (status != 0 || !handle) { // ??? Why do we check handle??
861	#ifdef _POSIX_THREAD_ATTR_STACKSIZE
862	if (status == EINVAL) {
863	__kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
864	KMP_HNT(IncreaseWorkerStackSize), __kmp_msg_null);
865	}
866	if (status == ENOMEM) {
867	__kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
868	KMP_HNT(DecreaseWorkerStackSize), __kmp_msg_null);
869	}
870	#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
871	if (status == EAGAIN) {
872	__kmp_fatal(KMP_MSG(NoResourcesForWorkerThread), KMP_ERR(status),
873	KMP_HNT(Decrease_NUM_THREADS), __kmp_msg_null);
874	}
875	KMP_SYSFAIL("pthread_create", status);
876	}
877
878	th->th.th_info.ds.ds_thread = handle;
879
880	#ifdef KMP_THREAD_ATTR
881	status = pthread_attr_destroy(attr: &thread_attr);
882	if (status) {
883	kmp_msg_t err_code = KMP_ERR(status);
884	__kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
885	__kmp_msg_null);
886	if (__kmp_generate_warnings == kmp_warnings_off) {
887	__kmp_str_free(str: &err_code.str);
888	}
889	}
890	#endif /* KMP_THREAD_ATTR */
891
892	KMP_MB(); /* Flush all pending memory write invalidates. */
893
894	KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
895
896	} // __kmp_create_worker
897
898	#if KMP_USE_MONITOR
899	void __kmp_create_monitor(kmp_info_t *th) {
900	pthread_t handle;
901	pthread_attr_t thread_attr;
902	size_t size;
903	int status;
904	int auto_adj_size = FALSE;
905
906	if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
907	// We don't need monitor thread in case of MAX_BLOCKTIME
908	KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
909	"MAX blocktime\n"));
910	th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
911	th->th.th_info.ds.ds_gtid = 0;
912	return;
913	}
914	KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
915
916	KMP_MB(); /* Flush all pending memory write invalidates. */
917
918	th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
919	th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
920	#if KMP_REAL_TIME_FIX
921	TCW_4(__kmp_global.g.g_time.dt.t_value,
922	-1); // Will use it for synchronization a bit later.
923	#else
924	TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
925	#endif // KMP_REAL_TIME_FIX
926
927	#ifdef KMP_THREAD_ATTR
928	if (__kmp_monitor_stksize == 0) {
929	__kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
930	auto_adj_size = TRUE;
931	}
932	status = pthread_attr_init(&thread_attr);
933	if (status != 0) {
934	__kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
935	}
936	status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
937	if (status != 0) {
938	__kmp_fatal(KMP_MSG(CantSetMonitorState), KMP_ERR(status), __kmp_msg_null);
939	}
940
941	#ifdef _POSIX_THREAD_ATTR_STACKSIZE
942	status = pthread_attr_getstacksize(&thread_attr, &size);
943	KMP_CHECK_SYSFAIL("pthread_attr_getstacksize", status);
944	#else
945	size = __kmp_sys_min_stksize;
946	#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
947	#endif /* KMP_THREAD_ATTR */
948
949	if (__kmp_monitor_stksize == 0) {
950	__kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
951	}
952	if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
953	__kmp_monitor_stksize = __kmp_sys_min_stksize;
954	}
955
956	KA_TRACE(10, ("__kmp_create_monitor: default stacksize = %lu bytes,"
957	"requested stacksize = %lu bytes\n",
958	size, __kmp_monitor_stksize));
959
960	retry:
961
962	/* Set stack size for this thread now. */
963	#ifdef _POSIX_THREAD_ATTR_STACKSIZE
964	KA_TRACE(10, ("__kmp_create_monitor: setting stacksize = %lu bytes,",
965	__kmp_monitor_stksize));
966	status = pthread_attr_setstacksize(&thread_attr, __kmp_monitor_stksize);
967	if (status != 0) {
968	if (auto_adj_size) {
969	__kmp_monitor_stksize *= 2;
970	goto retry;
971	}
972	kmp_msg_t err_code = KMP_ERR(status);
973	__kmp_msg(kmp_ms_warning, // should this be fatal? BB
974	KMP_MSG(CantSetMonitorStackSize, (long int)__kmp_monitor_stksize),
975	err_code, KMP_HNT(ChangeMonitorStackSize), __kmp_msg_null);
976	if (__kmp_generate_warnings == kmp_warnings_off) {
977	__kmp_str_free(&err_code.str);
978	}
979	}
980	#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
981
982	status =
983	pthread_create(&handle, &thread_attr, __kmp_launch_monitor, (void *)th);
984
985	if (status != 0) {
986	#ifdef _POSIX_THREAD_ATTR_STACKSIZE
987	if (status == EINVAL) {
988	if (auto_adj_size && (__kmp_monitor_stksize < (size_t)0x40000000)) {
989	__kmp_monitor_stksize *= 2;
990	goto retry;
991	}
992	__kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
993	KMP_ERR(status), KMP_HNT(IncreaseMonitorStackSize),
994	__kmp_msg_null);
995	}
996	if (status == ENOMEM) {
997	__kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
998	KMP_ERR(status), KMP_HNT(DecreaseMonitorStackSize),
999	__kmp_msg_null);
1000	}
1001	#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1002	if (status == EAGAIN) {
1003	__kmp_fatal(KMP_MSG(NoResourcesForMonitorThread), KMP_ERR(status),
1004	KMP_HNT(DecreaseNumberOfThreadsInUse), __kmp_msg_null);
1005	}
1006	KMP_SYSFAIL("pthread_create", status);
1007	}
1008
1009	th->th.th_info.ds.ds_thread = handle;
1010
1011	#if KMP_REAL_TIME_FIX
1012	// Wait for the monitor thread is really started and set its *priority*.
1013	KMP_DEBUG_ASSERT(sizeof(kmp_uint32) ==
1014	sizeof(__kmp_global.g.g_time.dt.t_value));
1015	__kmp_wait_4((kmp_uint32 volatile *)&__kmp_global.g.g_time.dt.t_value, -1,
1016	&__kmp_neq_4, NULL);
1017	#endif // KMP_REAL_TIME_FIX
1018
1019	#ifdef KMP_THREAD_ATTR
1020	status = pthread_attr_destroy(&thread_attr);
1021	if (status != 0) {
1022	kmp_msg_t err_code = KMP_ERR(status);
1023	__kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
1024	__kmp_msg_null);
1025	if (__kmp_generate_warnings == kmp_warnings_off) {
1026	__kmp_str_free(&err_code.str);
1027	}
1028	}
1029	#endif
1030
1031	KMP_MB(); /* Flush all pending memory write invalidates. */
1032
1033	KA_TRACE(10, ("__kmp_create_monitor: monitor created %#.8lx\n",
1034	th->th.th_info.ds.ds_thread));
1035
1036	} // __kmp_create_monitor
1037	#endif // KMP_USE_MONITOR
1038
1039	void __kmp_exit_thread(int exit_status) {
1040	#if KMP_OS_WASI
1041	// TODO: the wasm32-wasi-threads target does not yet support pthread_exit.
1042	#else
1043	pthread_exit(retval: (void *)(intptr_t)exit_status);
1044	#endif
1045	} // __kmp_exit_thread
1046
1047	#if KMP_USE_MONITOR
1048	void __kmp_resume_monitor();
1049
1050	extern "C" void __kmp_reap_monitor(kmp_info_t *th) {
1051	int status;
1052	void *exit_val;
1053
1054	KA_TRACE(10, ("__kmp_reap_monitor: try to reap monitor thread with handle"
1055	" %#.8lx\n",
1056	th->th.th_info.ds.ds_thread));
1057
1058	// If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1059	// If both tid and gtid are 0, it means the monitor did not ever start.
1060	// If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1061	KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1062	if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1063	KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1064	return;
1065	}
1066
1067	KMP_MB(); /* Flush all pending memory write invalidates. */
1068
1069	/* First, check to see whether the monitor thread exists to wake it up. This
1070	is to avoid performance problem when the monitor sleeps during
1071	blocktime-size interval */
1072
1073	status = pthread_kill(th->th.th_info.ds.ds_thread, 0);
1074	if (status != ESRCH) {
1075	__kmp_resume_monitor(); // Wake up the monitor thread
1076	}
1077	KA_TRACE(10, ("__kmp_reap_monitor: try to join with monitor\n"));
1078	status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1079	if (exit_val != th) {
1080	__kmp_fatal(KMP_MSG(ReapMonitorError), KMP_ERR(status), __kmp_msg_null);
1081	}
1082
1083	th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1084	th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1085
1086	KA_TRACE(10, ("__kmp_reap_monitor: done reaping monitor thread with handle"
1087	" %#.8lx\n",
1088	th->th.th_info.ds.ds_thread));
1089
1090	KMP_MB(); /* Flush all pending memory write invalidates. */
1091	}
1092	#else
1093	// Empty symbol to export (see exports_so.txt) when
1094	// monitor thread feature is disabled
1095	extern "C" void __kmp_reap_monitor(kmp_info_t *th) { (void)th; }
1096	#endif // KMP_USE_MONITOR
1097
1098	void __kmp_reap_worker(kmp_info_t *th) {
1099	int status;
1100	void *exit_val;
1101
1102	KMP_MB(); /* Flush all pending memory write invalidates. */
1103
1104	KA_TRACE(
1105	10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid));
1106
1107	status = pthread_join(th: th->th.th_info.ds.ds_thread, thread_return: &exit_val);
1108	#ifdef KMP_DEBUG
1109	/* Don't expose these to the user until we understand when they trigger */
1110	if (status != 0) {
1111	__kmp_fatal(KMP_MSG(ReapWorkerError), KMP_ERR(status), __kmp_msg_null);
1112	}
1113	if (exit_val != th) {
1114	KA_TRACE(10, ("__kmp_reap_worker: worker T#%d did not reap properly, "
1115	"exit_val = %p\n",
1116	th->th.th_info.ds.ds_gtid, exit_val));
1117	}
1118	#else
1119	(void)status; // unused variable
1120	#endif /* KMP_DEBUG */
1121
1122	KA_TRACE(10, ("__kmp_reap_worker: done reaping T#%d\n",
1123	th->th.th_info.ds.ds_gtid));
1124
1125	KMP_MB(); /* Flush all pending memory write invalidates. */
1126	}
1127
1128	#if KMP_HANDLE_SIGNALS
1129
1130	static void __kmp_null_handler(int signo) {
1131	// Do nothing, for doing SIG_IGN-type actions.
1132	} // __kmp_null_handler
1133
1134	static void __kmp_team_handler(int signo) {
1135	if (__kmp_global.g.g_abort == 0) {
1136	/* Stage 1 signal handler, let's shut down all of the threads */
1137	#ifdef KMP_DEBUG
1138	__kmp_debug_printf(format: "__kmp_team_handler: caught signal = %d\n", signo);
1139	#endif
1140	switch (signo) {
1141	case SIGHUP:
1142	case SIGINT:
1143	case SIGQUIT:
1144	case SIGILL:
1145	case SIGABRT:
1146	case SIGFPE:
1147	case SIGBUS:
1148	case SIGSEGV:
1149	#ifdef SIGSYS
1150	case SIGSYS:
1151	#endif
1152	case SIGTERM:
1153	if (__kmp_debug_buf) {
1154	__kmp_dump_debug_buffer();
1155	}
1156	__kmp_unregister_library(); // cleanup shared memory
1157	KMP_MB(); // Flush all pending memory write invalidates.
1158	TCW_4(__kmp_global.g.g_abort, signo);
1159	KMP_MB(); // Flush all pending memory write invalidates.
1160	TCW_4(__kmp_global.g.g_done, TRUE);
1161	KMP_MB(); // Flush all pending memory write invalidates.
1162	break;
1163	default:
1164	#ifdef KMP_DEBUG
1165	__kmp_debug_printf(format: "__kmp_team_handler: unknown signal type");
1166	#endif
1167	break;
1168	}
1169	}
1170	} // __kmp_team_handler
1171
1172	static void __kmp_sigaction(int signum, const struct sigaction *act,
1173	struct sigaction *oldact) {
1174	int rc = sigaction(sig: signum, act: act, oact: oldact);
1175	KMP_CHECK_SYSFAIL_ERRNO("sigaction", rc);
1176	}
1177
1178	static void __kmp_install_one_handler(int sig, sig_func_t handler_func,
1179	int parallel_init) {
1180	KMP_MB(); // Flush all pending memory write invalidates.
1181	KB_TRACE(60,
1182	("__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init));
1183	if (parallel_init) {
1184	struct sigaction new_action;
1185	struct sigaction old_action;
1186	new_action.sa_handler = handler_func;
1187	new_action.sa_flags = 0;
1188	sigfillset(set: &new_action.sa_mask);
1189	__kmp_sigaction(signum: sig, act: &new_action, oldact: &old_action);
1190	if (old_action.sa_handler == __kmp_sighldrs[sig].sa_handler) {
1191	sigaddset(set: &__kmp_sigset, signo: sig);
1192	} else {
1193	// Restore/keep user's handler if one previously installed.
1194	__kmp_sigaction(signum: sig, act: &old_action, NULL);
1195	}
1196	} else {
1197	// Save initial/system signal handlers to see if user handlers installed.
1198	__kmp_sigaction(signum: sig, NULL, oldact: &__kmp_sighldrs[sig]);
1199	}
1200	KMP_MB(); // Flush all pending memory write invalidates.
1201	} // __kmp_install_one_handler
1202
1203	static void __kmp_remove_one_handler(int sig) {
1204	KB_TRACE(60, ("__kmp_remove_one_handler( %d )\n", sig));
1205	if (sigismember(set: &__kmp_sigset, signo: sig)) {
1206	struct sigaction old;
1207	KMP_MB(); // Flush all pending memory write invalidates.
1208	__kmp_sigaction(signum: sig, act: &__kmp_sighldrs[sig], oldact: &old);
1209	if ((old.sa_handler != __kmp_team_handler) &&
1210	(old.sa_handler != __kmp_null_handler)) {
1211	// Restore the users signal handler.
1212	KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1213	"restoring: sig=%d\n",
1214	sig));
1215	__kmp_sigaction(signum: sig, act: &old, NULL);
1216	}
1217	sigdelset(set: &__kmp_sigset, signo: sig);
1218	KMP_MB(); // Flush all pending memory write invalidates.
1219	}
1220	} // __kmp_remove_one_handler
1221
1222	void __kmp_install_signals(int parallel_init) {
1223	KB_TRACE(10, ("__kmp_install_signals( %d )\n", parallel_init));
1224	if (__kmp_handle_signals || !parallel_init) {
1225	// If ! parallel_init, we do not install handlers, just save original
1226	// handlers. Let us do it even __handle_signals is 0.
1227	sigemptyset(set: &__kmp_sigset);
1228	__kmp_install_one_handler(SIGHUP, handler_func: __kmp_team_handler, parallel_init);
1229	__kmp_install_one_handler(SIGINT, handler_func: __kmp_team_handler, parallel_init);
1230	__kmp_install_one_handler(SIGQUIT, handler_func: __kmp_team_handler, parallel_init);
1231	__kmp_install_one_handler(SIGILL, handler_func: __kmp_team_handler, parallel_init);
1232	__kmp_install_one_handler(SIGABRT, handler_func: __kmp_team_handler, parallel_init);
1233	__kmp_install_one_handler(SIGFPE, handler_func: __kmp_team_handler, parallel_init);
1234	__kmp_install_one_handler(SIGBUS, handler_func: __kmp_team_handler, parallel_init);
1235	__kmp_install_one_handler(SIGSEGV, handler_func: __kmp_team_handler, parallel_init);
1236	#ifdef SIGSYS
1237	__kmp_install_one_handler(SIGSYS, handler_func: __kmp_team_handler, parallel_init);
1238	#endif // SIGSYS
1239	__kmp_install_one_handler(SIGTERM, handler_func: __kmp_team_handler, parallel_init);
1240	#ifdef SIGPIPE
1241	__kmp_install_one_handler(SIGPIPE, handler_func: __kmp_team_handler, parallel_init);
1242	#endif // SIGPIPE
1243	}
1244	} // __kmp_install_signals
1245
1246	void __kmp_remove_signals(void) {
1247	int sig;
1248	KB_TRACE(10, ("__kmp_remove_signals()\n"));
1249	for (sig = 1; sig < NSIG; ++sig) {
1250	__kmp_remove_one_handler(sig);
1251	}
1252	} // __kmp_remove_signals
1253
1254	#endif // KMP_HANDLE_SIGNALS
1255
1256	void __kmp_enable(int new_state) {
1257	#ifdef KMP_CANCEL_THREADS
1258	int status, old_state;
1259	status = pthread_setcancelstate(state: new_state, oldstate: &old_state);
1260	KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1261	KMP_DEBUG_ASSERT(old_state == PTHREAD_CANCEL_DISABLE);
1262	#endif
1263	}
1264
1265	void __kmp_disable(int *old_state) {
1266	#ifdef KMP_CANCEL_THREADS
1267	int status;
1268	status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, oldstate: old_state);
1269	KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1270	#endif
1271	}
1272
1273	static void __kmp_atfork_prepare(void) {
1274	__kmp_acquire_bootstrap_lock(lck: &__kmp_initz_lock);
1275	__kmp_acquire_bootstrap_lock(lck: &__kmp_forkjoin_lock);
1276	}
1277
1278	static void __kmp_atfork_parent(void) {
1279	__kmp_release_bootstrap_lock(lck: &__kmp_forkjoin_lock);
1280	__kmp_release_bootstrap_lock(lck: &__kmp_initz_lock);
1281	}
1282
1283	/* Reset the library so execution in the child starts "all over again" with
1284	clean data structures in initial states. Don't worry about freeing memory
1285	allocated by parent, just abandon it to be safe. */
1286	static void __kmp_atfork_child(void) {
1287	__kmp_release_bootstrap_lock(lck: &__kmp_forkjoin_lock);
1288	__kmp_release_bootstrap_lock(lck: &__kmp_initz_lock);
1289	/* TODO make sure this is done right for nested/sibling */
1290	// ATT: Memory leaks are here? TODO: Check it and fix.
1291	/* KMP_ASSERT( 0 ); */
1292
1293	++__kmp_fork_count;
1294
1295	#if KMP_AFFINITY_SUPPORTED
1296	#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
1297	KMP_OS_AIX
1298	// reset the affinity in the child to the initial thread
1299	// affinity in the parent
1300	kmp_set_thread_affinity_mask_initial();
1301	#endif
1302	// Set default not to bind threads tightly in the child (we're expecting
1303	// over-subscription after the fork and this can improve things for
1304	// scripting languages that use OpenMP inside process-parallel code).
1305	if (__kmp_nested_proc_bind.bind_types != NULL) {
1306	__kmp_nested_proc_bind.bind_types[0] = proc_bind_false;
1307	}
1308	for (kmp_affinity_t *affinity : __kmp_affinities)
1309	*affinity = KMP_AFFINITY_INIT(affinity->env_var);
1310	__kmp_affin_fullMask = nullptr;
1311	__kmp_affin_origMask = nullptr;
1312	__kmp_topology = nullptr;
1313	#endif // KMP_AFFINITY_SUPPORTED
1314
1315	#if KMP_USE_MONITOR
1316	__kmp_init_monitor = 0;
1317	#endif
1318	__kmp_init_parallel = FALSE;
1319	__kmp_init_middle = FALSE;
1320	__kmp_init_serial = FALSE;
1321	TCW_4(__kmp_init_gtid, FALSE);
1322	__kmp_init_common = FALSE;
1323
1324	TCW_4(__kmp_init_user_locks, FALSE);
1325	#if !KMP_USE_DYNAMIC_LOCK
1326	__kmp_user_lock_table.used = 1;
1327	__kmp_user_lock_table.allocated = 0;
1328	__kmp_user_lock_table.table = NULL;
1329	__kmp_lock_blocks = NULL;
1330	#endif
1331
1332	__kmp_all_nth = 0;
1333	TCW_4(__kmp_nth, 0);
1334
1335	__kmp_thread_pool = NULL;
1336	__kmp_thread_pool_insert_pt = NULL;
1337	__kmp_team_pool = NULL;
1338
1339	/* Must actually zero all the *cache arguments passed to __kmpc_threadprivate
1340	here so threadprivate doesn't use stale data */
1341	KA_TRACE(10, ("__kmp_atfork_child: checking cache address list %p\n",
1342	__kmp_threadpriv_cache_list));
1343
1344	while (__kmp_threadpriv_cache_list != NULL) {
1345
1346	if (*__kmp_threadpriv_cache_list->addr != NULL) {
1347	KC_TRACE(50, ("__kmp_atfork_child: zeroing cache at address %p\n",
1348	&(*__kmp_threadpriv_cache_list->addr)));
1349
1350	*__kmp_threadpriv_cache_list->addr = NULL;
1351	}
1352	__kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next;
1353	}
1354
1355	__kmp_init_runtime = FALSE;
1356
1357	/* reset statically initialized locks */
1358	__kmp_init_bootstrap_lock(lck: &__kmp_initz_lock);
1359	__kmp_init_bootstrap_lock(lck: &__kmp_stdio_lock);
1360	__kmp_init_bootstrap_lock(lck: &__kmp_console_lock);
1361	__kmp_init_bootstrap_lock(lck: &__kmp_task_team_lock);
1362
1363	#if USE_ITT_BUILD
1364	__kmp_itt_reset(); // reset ITT's global state
1365	#endif /* USE_ITT_BUILD */
1366
1367	{
1368	// Child process often get terminated without any use of OpenMP. That might
1369	// cause mapped shared memory file to be left unattended. Thus we postpone
1370	// library registration till middle initialization in the child process.
1371	__kmp_need_register_serial = FALSE;
1372	__kmp_serial_initialize();
1373	}
1374
1375	/* This is necessary to make sure no stale data is left around */
1376	/* AC: customers complain that we use unsafe routines in the atfork
1377	handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1378	in dynamic_link when check the presence of shared tbbmalloc library.
1379	Suggestion is to make the library initialization lazier, similar
1380	to what done for __kmpc_begin(). */
1381	// TODO: synchronize all static initializations with regular library
1382	// startup; look at kmp_global.cpp and etc.
1383	//__kmp_internal_begin ();
1384	}
1385
1386	void __kmp_register_atfork(void) {
1387	if (__kmp_need_register_atfork) {
1388	#if !KMP_OS_WASI
1389	int status = pthread_atfork(prepare: __kmp_atfork_prepare, parent: __kmp_atfork_parent,
1390	child: __kmp_atfork_child);
1391	KMP_CHECK_SYSFAIL("pthread_atfork", status);
1392	#endif
1393	__kmp_need_register_atfork = FALSE;
1394	}
1395	}
1396
1397	void __kmp_suspend_initialize(void) {
1398	int status;
1399	status = pthread_mutexattr_init(attr: &__kmp_suspend_mutex_attr);
1400	KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
1401	status = pthread_condattr_init(attr: &__kmp_suspend_cond_attr);
1402	KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
1403	}
1404
1405	void __kmp_suspend_initialize_thread(kmp_info_t *th) {
1406	int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count);
1407	int new_value = __kmp_fork_count + 1;
1408	// Return if already initialized
1409	if (old_value == new_value)
1410	return;
1411	// Wait, then return if being initialized
1412	if (old_value == -1 || !__kmp_atomic_compare_store(
1413	p: &th->th.th_suspend_init_count, expected: old_value, desired: -1)) {
1414	while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) != new_value) {
1415	KMP_CPU_PAUSE();
1416	}
1417	} else {
1418	// Claim to be the initializer and do initializations
1419	int status;
1420	status = pthread_cond_init(cond: &th->th.th_suspend_cv.c_cond,
1421	cond_attr: &__kmp_suspend_cond_attr);
1422	KMP_CHECK_SYSFAIL("pthread_cond_init", status);
1423	status = pthread_mutex_init(mutex: &th->th.th_suspend_mx.m_mutex,
1424	mutexattr: &__kmp_suspend_mutex_attr);
1425	KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
1426	KMP_ATOMIC_ST_REL(&th->th.th_suspend_init_count, new_value);
1427	}
1428	}
1429
1430	void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
1431	if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) > __kmp_fork_count) {
1432	/* this means we have initialize the suspension pthread objects for this
1433	thread in this instance of the process */
1434	int status;
1435
1436	status = pthread_cond_destroy(cond: &th->th.th_suspend_cv.c_cond);
1437	if (status != 0 && status != EBUSY) {
1438	KMP_SYSFAIL("pthread_cond_destroy", status);
1439	}
1440	status = pthread_mutex_destroy(mutex: &th->th.th_suspend_mx.m_mutex);
1441	if (status != 0 && status != EBUSY) {
1442	KMP_SYSFAIL("pthread_mutex_destroy", status);
1443	}
1444	--th->th.th_suspend_init_count;
1445	KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count) ==
1446	__kmp_fork_count);
1447	}
1448	}
1449
1450	// return true if lock obtained, false otherwise
1451	int __kmp_try_suspend_mx(kmp_info_t *th) {
1452	return (pthread_mutex_trylock(mutex: &th->th.th_suspend_mx.m_mutex) == 0);
1453	}
1454
1455	void __kmp_lock_suspend_mx(kmp_info_t *th) {
1456	int status = pthread_mutex_lock(mutex: &th->th.th_suspend_mx.m_mutex);
1457	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1458	}
1459
1460	void __kmp_unlock_suspend_mx(kmp_info_t *th) {
1461	int status = pthread_mutex_unlock(mutex: &th->th.th_suspend_mx.m_mutex);
1462	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1463	}
1464
1465	/* This routine puts the calling thread to sleep after setting the
1466	sleep bit for the indicated flag variable to true. */
1467	template <class C>
1468	static inline void __kmp_suspend_template(int th_gtid, C *flag) {
1469	KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
1470	kmp_info_t *th = __kmp_threads[th_gtid];
1471	int status;
1472	typename C::flag_t old_spin;
1473
1474	KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid,
1475	flag->get()));
1476
1477	__kmp_suspend_initialize_thread(th);
1478
1479	__kmp_lock_suspend_mx(th);
1480
1481	KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1482	th_gtid, flag->get()));
1483
1484	/* TODO: shouldn't this use release semantics to ensure that
1485	__kmp_suspend_initialize_thread gets called first? */
1486	old_spin = flag->set_sleeping();
1487	TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1488	th->th.th_sleep_loc_type = flag->get_type();
1489	if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
1490	__kmp_pause_status != kmp_soft_paused) {
1491	flag->unset_sleeping();
1492	TCW_PTR(th->th.th_sleep_loc, NULL);
1493	th->th.th_sleep_loc_type = flag_unset;
1494	__kmp_unlock_suspend_mx(th);
1495	return;
1496	}
1497	KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x,"
1498	" was %x\n",
1499	th_gtid, flag->get(), flag->load(), old_spin));
1500
1501	if (flag->done_check_val(old_spin) || flag->done_check()) {
1502	flag->unset_sleeping();
1503	TCW_PTR(th->th.th_sleep_loc, NULL);
1504	th->th.th_sleep_loc_type = flag_unset;
1505	KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
1506	"for spin(%p)\n",
1507	th_gtid, flag->get()));
1508	} else {
1509	/* Encapsulate in a loop as the documentation states that this may
1510	"with low probability" return when the condition variable has
1511	not been signaled or broadcast */
1512	int deactivated = FALSE;
1513
1514	while (flag->is_sleeping()) {
1515	#ifdef DEBUG_SUSPEND
1516	char buffer[128];
1517	__kmp_suspend_count++;
1518	__kmp_print_cond(buffer, &th->th.th_suspend_cv);
1519	__kmp_printf("__kmp_suspend_template: suspending T#%d: %s\n", th_gtid,
1520	buffer);
1521	#endif
1522	// Mark the thread as no longer active (only in the first iteration of the
1523	// loop).
1524	if (!deactivated) {
1525	th->th.th_active = FALSE;
1526	if (th->th.th_active_in_pool) {
1527	th->th.th_active_in_pool = FALSE;
1528	KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
1529	KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
1530	}
1531	deactivated = TRUE;
1532	}
1533
1534	KMP_DEBUG_ASSERT(th->th.th_sleep_loc);
1535	KMP_DEBUG_ASSERT(flag->get_type() == th->th.th_sleep_loc_type);
1536
1537	#if USE_SUSPEND_TIMEOUT
1538	struct timespec now;
1539	struct timeval tval;
1540	int msecs;
1541
1542	status = gettimeofday(&tval, NULL);
1543	KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1544	TIMEVAL_TO_TIMESPEC(&tval, &now);
1545
1546	msecs = (4 * __kmp_dflt_blocktime) + 200;
1547	now.tv_sec += msecs / 1000;
1548	now.tv_nsec += (msecs % 1000) * 1000;
1549
1550	KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
1551	"pthread_cond_timedwait\n",
1552	th_gtid));
1553	status = pthread_cond_timedwait(&th->th.th_suspend_cv.c_cond,
1554	&th->th.th_suspend_mx.m_mutex, &now);
1555	#else
1556	KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform"
1557	" pthread_cond_wait\n",
1558	th_gtid));
1559	status = pthread_cond_wait(cond: &th->th.th_suspend_cv.c_cond,
1560	mutex: &th->th.th_suspend_mx.m_mutex);
1561	#endif // USE_SUSPEND_TIMEOUT
1562
1563	if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) {
1564	KMP_SYSFAIL("pthread_cond_wait", status);
1565	}
1566
1567	KMP_DEBUG_ASSERT(flag->get_type() == flag->get_ptr_type());
1568
1569	if (!flag->is_sleeping() &&
1570	((status == EINTR) || (status == ETIMEDOUT))) {
1571	// if interrupt or timeout, and thread is no longer sleeping, we need to
1572	// make sure sleep_loc gets reset; however, this shouldn't be needed if
1573	// we woke up with resume
1574	flag->unset_sleeping();
1575	TCW_PTR(th->th.th_sleep_loc, NULL);
1576	th->th.th_sleep_loc_type = flag_unset;
1577	}
1578	#ifdef KMP_DEBUG
1579	if (status == ETIMEDOUT) {
1580	if (flag->is_sleeping()) {
1581	KF_TRACE(100,
1582	("__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid));
1583	} else {
1584	KF_TRACE(2, ("__kmp_suspend_template: T#%d timeout wakeup, sleep bit "
1585	"not set!\n",
1586	th_gtid));
1587	TCW_PTR(th->th.th_sleep_loc, NULL);
1588	th->th.th_sleep_loc_type = flag_unset;
1589	}
1590	} else if (flag->is_sleeping()) {
1591	KF_TRACE(100,
1592	("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
1593	}
1594	#endif
1595	} // while
1596
1597	// Mark the thread as active again (if it was previous marked as inactive)
1598	if (deactivated) {
1599	th->th.th_active = TRUE;
1600	if (TCR_4(th->th.th_in_pool)) {
1601	KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
1602	th->th.th_active_in_pool = TRUE;
1603	}
1604	}
1605	}
1606	// We may have had the loop variable set before entering the loop body;
1607	// so we need to reset sleep_loc.
1608	TCW_PTR(th->th.th_sleep_loc, NULL);
1609	th->th.th_sleep_loc_type = flag_unset;
1610
1611	KMP_DEBUG_ASSERT(!flag->is_sleeping());
1612	KMP_DEBUG_ASSERT(!th->th.th_sleep_loc);
1613	#ifdef DEBUG_SUSPEND
1614	{
1615	char buffer[128];
1616	__kmp_print_cond(buffer, &th->th.th_suspend_cv);
1617	__kmp_printf("__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid,
1618	buffer);
1619	}
1620	#endif
1621
1622	__kmp_unlock_suspend_mx(th);
1623	KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
1624	}
1625
1626	template <bool C, bool S>
1627	void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) {
1628	__kmp_suspend_template(th_gtid, flag);
1629	}
1630	template <bool C, bool S>
1631	void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) {
1632	__kmp_suspend_template(th_gtid, flag);
1633	}
1634	template <bool C, bool S>
1635	void __kmp_atomic_suspend_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag) {
1636	__kmp_suspend_template(th_gtid, flag);
1637	}
1638	void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1639	__kmp_suspend_template(th_gtid, flag);
1640	}
1641
1642	template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *);
1643	template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *);
1644	template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *);
1645	template void
1646	__kmp_atomic_suspend_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1647	template void
1648	__kmp_atomic_suspend_64<true, false>(int, kmp_atomic_flag_64<true, false> *);
1649
1650	/* This routine signals the thread specified by target_gtid to wake up
1651	after setting the sleep bit indicated by the flag argument to FALSE.
1652	The target thread must already have called __kmp_suspend_template() */
1653	template <class C>
1654	static inline void __kmp_resume_template(int target_gtid, C *flag) {
1655	KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1656	kmp_info_t *th = __kmp_threads[target_gtid];
1657	int status;
1658
1659	#ifdef KMP_DEBUG
1660	int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1661	#endif
1662
1663	KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
1664	gtid, target_gtid));
1665	KMP_DEBUG_ASSERT(gtid != target_gtid);
1666
1667	__kmp_suspend_initialize_thread(th);
1668
1669	__kmp_lock_suspend_mx(th);
1670
1671	if (!flag || flag != th->th.th_sleep_loc) {
1672	// coming from __kmp_null_resume_wrapper, or thread is now sleeping on a
1673	// different location; wake up at new location
1674	flag = (C *)CCAST(void *, th->th.th_sleep_loc);
1675	}
1676
1677	// First, check if the flag is null or its type has changed. If so, someone
1678	// else woke it up.
1679	if (!flag) { // Thread doesn't appear to be sleeping on anything
1680	KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1681	"awake: flag(%p)\n",
1682	gtid, target_gtid, (void *)NULL));
1683	__kmp_unlock_suspend_mx(th);
1684	return;
1685	} else if (flag->get_type() != th->th.th_sleep_loc_type) {
1686	// Flag type does not appear to match this function template; possibly the
1687	// thread is sleeping on something else. Try null resume again.
1688	KF_TRACE(
1689	5,
1690	("__kmp_resume_template: T#%d retrying, thread T#%d Mismatch flag(%p), "
1691	"spin(%p) type=%d ptr_type=%d\n",
1692	gtid, target_gtid, flag, flag->get(), flag->get_type(),
1693	th->th.th_sleep_loc_type));
1694	__kmp_unlock_suspend_mx(th);
1695	__kmp_null_resume_wrapper(thr: th);
1696	return;
1697	} else { // if multiple threads are sleeping, flag should be internally
1698	// referring to a specific thread here
1699	if (!flag->is_sleeping()) {
1700	KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1701	"awake: flag(%p): %u\n",
1702	gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1703	__kmp_unlock_suspend_mx(th);
1704	return;
1705	}
1706	}
1707	KMP_DEBUG_ASSERT(flag);
1708	flag->unset_sleeping();
1709	TCW_PTR(th->th.th_sleep_loc, NULL);
1710	th->th.th_sleep_loc_type = flag_unset;
1711
1712	KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset "
1713	"sleep bit for flag's loc(%p): %u\n",
1714	gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1715
1716	#ifdef DEBUG_SUSPEND
1717	{
1718	char buffer[128];
1719	__kmp_print_cond(buffer, &th->th.th_suspend_cv);
1720	__kmp_printf("__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid,
1721	target_gtid, buffer);
1722	}
1723	#endif
1724	status = pthread_cond_signal(cond: &th->th.th_suspend_cv.c_cond);
1725	KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1726	__kmp_unlock_suspend_mx(th);
1727	KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
1728	" for T#%d\n",
1729	gtid, target_gtid));
1730	}
1731
1732	template <bool C, bool S>
1733	void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) {
1734	__kmp_resume_template(target_gtid, flag);
1735	}
1736	template <bool C, bool S>
1737	void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) {
1738	__kmp_resume_template(target_gtid, flag);
1739	}
1740	template <bool C, bool S>
1741	void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64<C, S> *flag) {
1742	__kmp_resume_template(target_gtid, flag);
1743	}
1744	void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1745	__kmp_resume_template(target_gtid, flag);
1746	}
1747
1748	template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *);
1749	template void __kmp_resume_32<false, false>(int, kmp_flag_32<false, false> *);
1750	template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *);
1751	template void
1752	__kmp_atomic_resume_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1753
1754	#if KMP_USE_MONITOR
1755	void __kmp_resume_monitor() {
1756	KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1757	int status;
1758	#ifdef KMP_DEBUG
1759	int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1760	KF_TRACE(30, ("__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n", gtid,
1761	KMP_GTID_MONITOR));
1762	KMP_DEBUG_ASSERT(gtid != KMP_GTID_MONITOR);
1763	#endif
1764	status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
1765	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1766	#ifdef DEBUG_SUSPEND
1767	{
1768	char buffer[128];
1769	__kmp_print_cond(buffer, &__kmp_wait_cv.c_cond);
1770	__kmp_printf("__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid,
1771	KMP_GTID_MONITOR, buffer);
1772	}
1773	#endif
1774	status = pthread_cond_signal(&__kmp_wait_cv.c_cond);
1775	KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1776	status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
1777	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1778	KF_TRACE(30, ("__kmp_resume_monitor: T#%d exiting after signaling wake up"
1779	" for T#%d\n",
1780	gtid, KMP_GTID_MONITOR));
1781	}
1782	#endif // KMP_USE_MONITOR
1783
1784	void __kmp_yield() { sched_yield(); }
1785
1786	void __kmp_gtid_set_specific(int gtid) {
1787	if (__kmp_init_gtid) {
1788	int status;
1789	status = pthread_setspecific(key: __kmp_gtid_threadprivate_key,
1790	pointer: (void *)(intptr_t)(gtid + 1));
1791	KMP_CHECK_SYSFAIL("pthread_setspecific", status);
1792	} else {
1793	KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
1794	}
1795	}
1796
1797	int __kmp_gtid_get_specific() {
1798	int gtid;
1799	if (!__kmp_init_gtid) {
1800	KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
1801	"KMP_GTID_SHUTDOWN\n"));
1802	return KMP_GTID_SHUTDOWN;
1803	}
1804	gtid = (int)(size_t)pthread_getspecific(key: __kmp_gtid_threadprivate_key);
1805	if (gtid == 0) {
1806	gtid = KMP_GTID_DNE;
1807	} else {
1808	gtid--;
1809	}
1810	KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
1811	__kmp_gtid_threadprivate_key, gtid));
1812	return gtid;
1813	}
1814
1815	double __kmp_read_cpu_time(void) {
1816	/*clock_t t;*/
1817	struct tms buffer;
1818
1819	/*t =*/times(buffer: &buffer);
1820
1821	return (double)(buffer.tms_utime + buffer.tms_cutime) /
1822	(double)CLOCKS_PER_SEC;
1823	}
1824
1825	int __kmp_read_system_info(struct kmp_sys_info *info) {
1826	int status;
1827	struct rusage r_usage;
1828
1829	memset(s: info, c: 0, n: sizeof(*info));
1830
1831	status = getrusage(RUSAGE_SELF, usage: &r_usage);
1832	KMP_CHECK_SYSFAIL_ERRNO("getrusage", status);
1833
1834	#if !KMP_OS_WASI
1835	// The maximum resident set size utilized (in kilobytes)
1836	info->maxrss = r_usage.ru_maxrss;
1837	// The number of page faults serviced without any I/O
1838	info->minflt = r_usage.ru_minflt;
1839	// The number of page faults serviced that required I/O
1840	info->majflt = r_usage.ru_majflt;
1841	// The number of times a process was "swapped" out of memory
1842	info->nswap = r_usage.ru_nswap;
1843	// The number of times the file system had to perform input
1844	info->inblock = r_usage.ru_inblock;
1845	// The number of times the file system had to perform output
1846	info->oublock = r_usage.ru_oublock;
1847	// The number of times a context switch was voluntarily
1848	info->nvcsw = r_usage.ru_nvcsw;
1849	// The number of times a context switch was forced
1850	info->nivcsw = r_usage.ru_nivcsw;
1851	#endif
1852
1853	return (status != 0);
1854	}
1855
1856	void __kmp_read_system_time(double *delta) {
1857	double t_ns;
1858	struct timeval tval;
1859	struct timespec stop;
1860	int status;
1861
1862	status = gettimeofday(tv: &tval, NULL);
1863	KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1864	TIMEVAL_TO_TIMESPEC(&tval, &stop);
1865	t_ns = (double)(TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start));
1866	*delta = (t_ns * 1e-9);
1867	}
1868
1869	void __kmp_clear_system_time(void) {
1870	struct timeval tval;
1871	int status;
1872	status = gettimeofday(tv: &tval, NULL);
1873	KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1874	TIMEVAL_TO_TIMESPEC(&tval, &__kmp_sys_timer_data.start);
1875	}
1876
1877	static int __kmp_get_xproc(void) {
1878
1879	int r = 0;
1880
1881	#if KMP_OS_LINUX
1882
1883	__kmp_type_convert(src: sysconf(_SC_NPROCESSORS_CONF), dest: &(r));
1884
1885	#elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || \
1886	KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX
1887
1888	__kmp_type_convert(sysconf(_SC_NPROCESSORS_ONLN), &(r));
1889
1890	#elif KMP_OS_DARWIN
1891
1892	size_t len = sizeof(r);
1893	sysctlbyname("hw.logicalcpu", &r, &len, NULL, 0);
1894
1895	#else
1896
1897	#error "Unknown or unsupported OS."
1898
1899	#endif
1900
1901	return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
1902
1903	} // __kmp_get_xproc
1904
1905	int __kmp_read_from_file(char const *path, char const *format, ...) {
1906	int result;
1907	va_list args;
1908
1909	va_start(args, format);
1910	FILE *f = fopen(filename: path, modes: "rb");
1911	if (f == NULL) {
1912	va_end(args);
1913	return 0;
1914	}
1915	result = vfscanf(s: f, format: format, arg: args);
1916	fclose(stream: f);
1917	va_end(args);
1918
1919	return result;
1920	}
1921
1922	void __kmp_runtime_initialize(void) {
1923	int status;
1924	pthread_mutexattr_t mutex_attr;
1925	pthread_condattr_t cond_attr;
1926
1927	if (__kmp_init_runtime) {
1928	return;
1929	}
1930
1931	#if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1932	if (!__kmp_cpuinfo.initialized) {
1933	__kmp_query_cpuid(p: &__kmp_cpuinfo);
1934	}
1935	#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1936
1937	__kmp_xproc = __kmp_get_xproc();
1938
1939	#if !KMP_32_BIT_ARCH
1940	struct rlimit rlim;
1941	// read stack size of calling thread, save it as default for worker threads;
1942	// this should be done before reading environment variables
1943	status = getrlimit(RLIMIT_STACK, rlimits: &rlim);
1944	if (status == 0) { // success?
1945	__kmp_stksize = rlim.rlim_cur;
1946	__kmp_check_stksize(val: &__kmp_stksize); // check value and adjust if needed
1947	}
1948	#endif /* KMP_32_BIT_ARCH */
1949
1950	if (sysconf(_SC_THREADS)) {
1951
1952	/* Query the maximum number of threads */
1953	__kmp_type_convert(src: sysconf(_SC_THREAD_THREADS_MAX), dest: &(__kmp_sys_max_nth));
1954	#ifdef __ve__
1955	if (__kmp_sys_max_nth == -1) {
1956	// VE's pthread supports only up to 64 threads per a VE process.
1957	// So we use that KMP_MAX_NTH (predefined as 64) here.
1958	__kmp_sys_max_nth = KMP_MAX_NTH;
1959	}
1960	#else
1961	if (__kmp_sys_max_nth == -1) {
1962	/* Unlimited threads for NPTL */
1963	__kmp_sys_max_nth = INT_MAX;
1964	} else if (__kmp_sys_max_nth <= 1) {
1965	/* Can't tell, just use PTHREAD_THREADS_MAX */
1966	__kmp_sys_max_nth = KMP_MAX_NTH;
1967	}
1968	#endif
1969
1970	/* Query the minimum stack size */
1971	__kmp_sys_min_stksize = sysconf(_SC_THREAD_STACK_MIN);
1972	if (__kmp_sys_min_stksize <= 1) {
1973	__kmp_sys_min_stksize = KMP_MIN_STKSIZE;
1974	}
1975	}
1976
1977	/* Set up minimum number of threads to switch to TLS gtid */
1978	__kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
1979
1980	status = pthread_key_create(key: &__kmp_gtid_threadprivate_key,
1981	destr_function: __kmp_internal_end_dest);
1982	KMP_CHECK_SYSFAIL("pthread_key_create", status);
1983	status = pthread_mutexattr_init(attr: &mutex_attr);
1984	KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
1985	status = pthread_mutex_init(mutex: &__kmp_wait_mx.m_mutex, mutexattr: &mutex_attr);
1986	KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
1987	status = pthread_mutexattr_destroy(attr: &mutex_attr);
1988	KMP_CHECK_SYSFAIL("pthread_mutexattr_destroy", status);
1989	status = pthread_condattr_init(attr: &cond_attr);
1990	KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
1991	status = pthread_cond_init(cond: &__kmp_wait_cv.c_cond, cond_attr: &cond_attr);
1992	KMP_CHECK_SYSFAIL("pthread_cond_init", status);
1993	status = pthread_condattr_destroy(attr: &cond_attr);
1994	KMP_CHECK_SYSFAIL("pthread_condattr_destroy", status);
1995	#if USE_ITT_BUILD
1996	__kmp_itt_initialize();
1997	#endif /* USE_ITT_BUILD */
1998
1999	__kmp_init_runtime = TRUE;
2000	}
2001
2002	void __kmp_runtime_destroy(void) {
2003	int status;
2004
2005	if (!__kmp_init_runtime) {
2006	return; // Nothing to do.
2007	}
2008
2009	#if USE_ITT_BUILD
2010	__kmp_itt_destroy();
2011	#endif /* USE_ITT_BUILD */
2012
2013	status = pthread_key_delete(key: __kmp_gtid_threadprivate_key);
2014	KMP_CHECK_SYSFAIL("pthread_key_delete", status);
2015
2016	status = pthread_mutex_destroy(mutex: &__kmp_wait_mx.m_mutex);
2017	if (status != 0 && status != EBUSY) {
2018	KMP_SYSFAIL("pthread_mutex_destroy", status);
2019	}
2020	status = pthread_cond_destroy(cond: &__kmp_wait_cv.c_cond);
2021	if (status != 0 && status != EBUSY) {
2022	KMP_SYSFAIL("pthread_cond_destroy", status);
2023	}
2024	#if KMP_AFFINITY_SUPPORTED
2025	__kmp_affinity_uninitialize();
2026	#endif
2027
2028	__kmp_init_runtime = FALSE;
2029	}
2030
2031	/* Put the thread to sleep for a time period */
2032	/* NOTE: not currently used anywhere */
2033	void __kmp_thread_sleep(int millis) { sleep(seconds: (millis + 500) / 1000); }
2034
2035	/* Calculate the elapsed wall clock time for the user */
2036	void __kmp_elapsed(double *t) {
2037	int status;
2038	#ifdef FIX_SGI_CLOCK
2039	struct timespec ts;
2040
2041	status = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
2042	KMP_CHECK_SYSFAIL_ERRNO("clock_gettime", status);
2043	*t =
2044	(double)ts.tv_nsec * (1.0 / (double)KMP_NSEC_PER_SEC) + (double)ts.tv_sec;
2045	#else
2046	struct timeval tv;
2047
2048	status = gettimeofday(tv: &tv, NULL);
2049	KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
2050	*t =
2051	(double)tv.tv_usec * (1.0 / (double)KMP_USEC_PER_SEC) + (double)tv.tv_sec;
2052	#endif
2053	}
2054
2055	/* Calculate the elapsed wall clock tick for the user */
2056	void __kmp_elapsed_tick(double *t) { *t = 1 / (double)CLOCKS_PER_SEC; }
2057
2058	/* Return the current time stamp in nsec */
2059	kmp_uint64 __kmp_now_nsec() {
2060	struct timeval t;
2061	gettimeofday(tv: &t, NULL);
2062	kmp_uint64 nsec = (kmp_uint64)KMP_NSEC_PER_SEC * (kmp_uint64)t.tv_sec +
2063	(kmp_uint64)1000 * (kmp_uint64)t.tv_usec;
2064	return nsec;
2065	}
2066
2067	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
2068	/* Measure clock ticks per millisecond */
2069	void __kmp_initialize_system_tick() {
2070	kmp_uint64 now, nsec2, diff;
2071	kmp_uint64 delay = 1000000; // ~450 usec on most machines.
2072	kmp_uint64 nsec = __kmp_now_nsec();
2073	kmp_uint64 goal = __kmp_hardware_timestamp() + delay;
2074	while ((now = __kmp_hardware_timestamp()) < goal)
2075	;
2076	nsec2 = __kmp_now_nsec();
2077	diff = nsec2 - nsec;
2078	if (diff > 0) {
2079	double tpus = 1000.0 * (double)(delay + (now - goal)) / (double)diff;
2080	if (tpus > 0.0) {
2081	__kmp_ticks_per_msec = (kmp_uint64)(tpus * 1000.0);
2082	__kmp_ticks_per_usec = (kmp_uint64)tpus;
2083	}
2084	}
2085	}
2086	#endif
2087
2088	/* Determine whether the given address is mapped into the current address
2089	space. */
2090
2091	int __kmp_is_address_mapped(void *addr) {
2092
2093	int found = 0;
2094	int rc;
2095
2096	#if KMP_OS_LINUX || KMP_OS_HURD
2097
2098	/* On GNUish OSes, read the /proc/<pid>/maps pseudo-file to get all the
2099	address ranges mapped into the address space. */
2100
2101	char *name = __kmp_str_format(format: "/proc/%d/maps", getpid());
2102	FILE *file = NULL;
2103
2104	file = fopen(filename: name, modes: "r");
2105	KMP_ASSERT(file != NULL);
2106
2107	for (;;) {
2108
2109	void *beginning = NULL;
2110	void *ending = NULL;
2111	char perms[5];
2112
2113	rc = fscanf(stream: file, format: "%p-%p %4s %*[^\n]\n", &beginning, &ending, perms);
2114	if (rc == EOF) {
2115	break;
2116	}
2117	KMP_ASSERT(rc == 3 &&
2118	KMP_STRLEN(perms) == 4); // Make sure all fields are read.
2119
2120	// Ending address is not included in the region, but beginning is.
2121	if ((addr >= beginning) && (addr < ending)) {
2122	perms[2] = 0; // 3th and 4th character does not matter.
2123	if (strcmp(s1: perms, s2: "rw") == 0) {
2124	// Memory we are looking for should be readable and writable.
2125	found = 1;
2126	}
2127	break;
2128	}
2129	}
2130
2131	// Free resources.
2132	fclose(stream: file);
2133	KMP_INTERNAL_FREE(name);
2134	#elif KMP_OS_FREEBSD
2135	char *buf;
2136	size_t lstsz;
2137	int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
2138	rc = sysctl(mib, 4, NULL, &lstsz, NULL, 0);
2139	if (rc < 0)
2140	return 0;
2141	// We pass from number of vm entry's semantic
2142	// to size of whole entry map list.
2143	lstsz = lstsz * 4 / 3;
2144	buf = reinterpret_cast<char *>(KMP_INTERNAL_MALLOC(lstsz));
2145	rc = sysctl(mib, 4, buf, &lstsz, NULL, 0);
2146	if (rc < 0) {
2147	KMP_INTERNAL_FREE(buf);
2148	return 0;
2149	}
2150
2151	char *lw = buf;
2152	char *up = buf + lstsz;
2153
2154	while (lw < up) {
2155	struct kinfo_vmentry *cur = reinterpret_cast<struct kinfo_vmentry *>(lw);
2156	size_t cursz = cur->kve_structsize;
2157	if (cursz == 0)
2158	break;
2159	void *start = reinterpret_cast<void *>(cur->kve_start);
2160	void *end = reinterpret_cast<void *>(cur->kve_end);
2161	// Readable/Writable addresses within current map entry
2162	if ((addr >= start) && (addr < end)) {
2163	if ((cur->kve_protection & KVME_PROT_READ) != 0 &&
2164	(cur->kve_protection & KVME_PROT_WRITE) != 0) {
2165	found = 1;
2166	break;
2167	}
2168	}
2169	lw += cursz;
2170	}
2171	KMP_INTERNAL_FREE(buf);
2172	#elif KMP_OS_DRAGONFLY
2173	char err[_POSIX2_LINE_MAX];
2174	kinfo_proc *proc;
2175	vmspace sp;
2176	vm_map *cur;
2177	vm_map_entry entry, *c;
2178	struct proc p;
2179	kvm_t *fd;
2180	uintptr_t uaddr;
2181	int num;
2182
2183	fd = kvm_openfiles(nullptr, nullptr, nullptr, O_RDONLY, err);
2184	if (!fd) {
2185	return 0;
2186	}
2187
2188	proc = kvm_getprocs(fd, KERN_PROC_PID, getpid(), &num);
2189
2190	if (kvm_read(fd, static_cast<uintptr_t>(proc->kp_paddr), &p, sizeof(p)) !=
2191	sizeof(p) ||
2192	kvm_read(fd, reinterpret_cast<uintptr_t>(p.p_vmspace), &sp, sizeof(sp)) !=
2193	sizeof(sp)) {
2194	kvm_close(fd);
2195	return 0;
2196	}
2197
2198	(void)rc;
2199	cur = &sp.vm_map;
2200	uaddr = reinterpret_cast<uintptr_t>(addr);
2201	for (c = kvm_vm_map_entry_first(fd, cur, &entry); c;
2202	c = kvm_vm_map_entry_next(fd, c, &entry)) {
2203	if ((uaddr >= entry.ba.start) && (uaddr <= entry.ba.end)) {
2204	if ((entry.protection & VM_PROT_READ) != 0 &&
2205	(entry.protection & VM_PROT_WRITE) != 0) {
2206	found = 1;
2207	break;
2208	}
2209	}
2210	}
2211
2212	kvm_close(fd);
2213	#elif KMP_OS_SOLARIS
2214	prmap_t *cur, *map;
2215	void *buf;
2216	uintptr_t uaddr;
2217	ssize_t rd;
2218	int err;
2219	int file;
2220
2221	pid_t pid = getpid();
2222	struct ps_prochandle *fd = Pgrab(pid, PGRAB_RDONLY, &err);
2223	;
2224
2225	if (!fd) {
2226	return 0;
2227	}
2228
2229	char *name = __kmp_str_format("/proc/%d/map", pid);
2230	size_t sz = (1 << 20);
2231	file = open(name, O_RDONLY);
2232	if (file == -1) {
2233	KMP_INTERNAL_FREE(name);
2234	return 0;
2235	}
2236
2237	buf = KMP_INTERNAL_MALLOC(sz);
2238
2239	while (sz > 0 && (rd = pread(file, buf, sz, 0)) == sz) {
2240	void *newbuf;
2241	sz <<= 1;
2242	newbuf = KMP_INTERNAL_REALLOC(buf, sz);
2243	buf = newbuf;
2244	}
2245
2246	map = reinterpret_cast<prmap_t *>(buf);
2247	uaddr = reinterpret_cast<uintptr_t>(addr);
2248
2249	for (cur = map; rd > 0; cur++, rd = -sizeof(*map)) {
2250	if ((uaddr >= cur->pr_vaddr) && (uaddr < cur->pr_vaddr)) {
2251	if ((cur->pr_mflags & MA_READ) != 0 && (cur->pr_mflags & MA_WRITE) != 0) {
2252	found = 1;
2253	break;
2254	}
2255	}
2256	}
2257
2258	KMP_INTERNAL_FREE(map);
2259	close(file);
2260	KMP_INTERNAL_FREE(name);
2261	#elif KMP_OS_DARWIN
2262
2263	/* On OS X*, /proc pseudo filesystem is not available. Try to read memory
2264	using vm interface. */
2265
2266	int buffer;
2267	vm_size_t count;
2268	rc = vm_read_overwrite(
2269	mach_task_self(), // Task to read memory of.
2270	(vm_address_t)(addr), // Address to read from.
2271	1, // Number of bytes to be read.
2272	(vm_address_t)(&buffer), // Address of buffer to save read bytes in.
2273	&count // Address of var to save number of read bytes in.
2274	);
2275	if (rc == 0) {
2276	// Memory successfully read.
2277	found = 1;
2278	}
2279
2280	#elif KMP_OS_NETBSD
2281
2282	int mib[5];
2283	mib[0] = CTL_VM;
2284	mib[1] = VM_PROC;
2285	mib[2] = VM_PROC_MAP;
2286	mib[3] = getpid();
2287	mib[4] = sizeof(struct kinfo_vmentry);
2288
2289	size_t size;
2290	rc = sysctl(mib, __arraycount(mib), NULL, &size, NULL, 0);
2291	KMP_ASSERT(!rc);
2292	KMP_ASSERT(size);
2293
2294	size = size * 4 / 3;
2295	struct kinfo_vmentry *kiv = (struct kinfo_vmentry *)KMP_INTERNAL_MALLOC(size);
2296	KMP_ASSERT(kiv);
2297
2298	rc = sysctl(mib, __arraycount(mib), kiv, &size, NULL, 0);
2299	KMP_ASSERT(!rc);
2300	KMP_ASSERT(size);
2301
2302	for (size_t i = 0; i < size; i++) {
2303	if (kiv[i].kve_start >= (uint64_t)addr &&
2304	kiv[i].kve_end <= (uint64_t)addr) {
2305	found = 1;
2306	break;
2307	}
2308	}
2309	KMP_INTERNAL_FREE(kiv);
2310	#elif KMP_OS_OPENBSD
2311
2312	int mib[3];
2313	mib[0] = CTL_KERN;
2314	mib[1] = KERN_PROC_VMMAP;
2315	mib[2] = getpid();
2316
2317	size_t size;
2318	uint64_t end;
2319	rc = sysctl(mib, 3, NULL, &size, NULL, 0);
2320	KMP_ASSERT(!rc);
2321	KMP_ASSERT(size);
2322	end = size;
2323
2324	struct kinfo_vmentry kiv = {.kve_start = 0};
2325
2326	while ((rc = sysctl(mib, 3, &kiv, &size, NULL, 0)) == 0) {
2327	KMP_ASSERT(size);
2328	if (kiv.kve_end == end)
2329	break;
2330
2331	if (kiv.kve_start >= (uint64_t)addr && kiv.kve_end <= (uint64_t)addr) {
2332	found = 1;
2333	break;
2334	}
2335	kiv.kve_start += 1;
2336	}
2337	#elif KMP_OS_WASI
2338	found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE);
2339	#elif KMP_OS_AIX
2340
2341	(void)rc;
2342	// FIXME(AIX): Implement this
2343	found = 1;
2344
2345	#else
2346
2347	#error "Unknown or unsupported OS"
2348
2349	#endif
2350
2351	return found;
2352
2353	} // __kmp_is_address_mapped
2354
2355	#ifdef USE_LOAD_BALANCE
2356
2357	#if KMP_OS_DARWIN || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
2358	KMP_OS_OPENBSD || KMP_OS_SOLARIS
2359
2360	// The function returns the rounded value of the system load average
2361	// during given time interval which depends on the value of
2362	// __kmp_load_balance_interval variable (default is 60 sec, other values
2363	// may be 300 sec or 900 sec).
2364	// It returns -1 in case of error.
2365	int __kmp_get_load_balance(int max) {
2366	double averages[3];
2367	int ret_avg = 0;
2368
2369	int res = getloadavg(averages, 3);
2370
2371	// Check __kmp_load_balance_interval to determine which of averages to use.
2372	// getloadavg() may return the number of samples less than requested that is
2373	// less than 3.
2374	if (__kmp_load_balance_interval < 180 && (res >= 1)) {
2375	ret_avg = (int)averages[0]; // 1 min
2376	} else if ((__kmp_load_balance_interval >= 180 &&
2377	__kmp_load_balance_interval < 600) &&
2378	(res >= 2)) {
2379	ret_avg = (int)averages[1]; // 5 min
2380	} else if ((__kmp_load_balance_interval >= 600) && (res == 3)) {
2381	ret_avg = (int)averages[2]; // 15 min
2382	} else { // Error occurred
2383	return -1;
2384	}
2385
2386	return ret_avg;
2387	}
2388
2389	#else // Linux* OS
2390
2391	// The function returns number of running (not sleeping) threads, or -1 in case
2392	// of error. Error could be reported if Linux* OS kernel too old (without
2393	// "/proc" support). Counting running threads stops if max running threads
2394	// encountered.
2395	int __kmp_get_load_balance(int max) {
2396	static int permanent_error = 0;
2397	static int glb_running_threads = 0; // Saved count of the running threads for
2398	// the thread balance algorithm
2399	static double glb_call_time = 0; /* Thread balance algorithm call time */
2400
2401	int running_threads = 0; // Number of running threads in the system.
2402
2403	DIR *proc_dir = NULL; // Handle of "/proc/" directory.
2404	struct dirent *proc_entry = NULL;
2405
2406	kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2407	DIR *task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2408	struct dirent *task_entry = NULL;
2409	int task_path_fixed_len;
2410
2411	kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2412	int stat_file = -1;
2413	int stat_path_fixed_len;
2414
2415	#ifdef KMP_DEBUG
2416	int total_processes = 0; // Total number of processes in system.
2417	#endif
2418
2419	double call_time = 0.0;
2420
2421	__kmp_str_buf_init(&task_path);
2422	__kmp_str_buf_init(&stat_path);
2423
2424	__kmp_elapsed(t: &call_time);
2425
2426	if (glb_call_time &&
2427	(call_time - glb_call_time < __kmp_load_balance_interval)) {
2428	running_threads = glb_running_threads;
2429	goto finish;
2430	}
2431
2432	glb_call_time = call_time;
2433
2434	// Do not spend time on scanning "/proc/" if we have a permanent error.
2435	if (permanent_error) {
2436	running_threads = -1;
2437	goto finish;
2438	}
2439
2440	if (max <= 0) {
2441	max = INT_MAX;
2442	}
2443
2444	// Open "/proc/" directory.
2445	proc_dir = opendir(name: "/proc");
2446	if (proc_dir == NULL) {
2447	// Cannot open "/proc/". Probably the kernel does not support it. Return an
2448	// error now and in subsequent calls.
2449	running_threads = -1;
2450	permanent_error = 1;
2451	goto finish;
2452	}
2453
2454	// Initialize fixed part of task_path. This part will not change.
2455	__kmp_str_buf_cat(buffer: &task_path, str: "/proc/", len: 6);
2456	task_path_fixed_len = task_path.used; // Remember number of used characters.
2457
2458	proc_entry = readdir(dirp: proc_dir);
2459	while (proc_entry != NULL) {
2460	#if KMP_OS_AIX
2461	// Proc entry name starts with a digit. Assume it is a process' directory.
2462	if (isdigit(proc_entry->d_name[0])) {
2463	#else
2464	// Proc entry is a directory and name starts with a digit. Assume it is a
2465	// process' directory.
2466	if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) {
2467	#endif
2468
2469	#ifdef KMP_DEBUG
2470	++total_processes;
2471	#endif
2472	// Make sure init process is the very first in "/proc", so we can replace
2473	// strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes ==
2474	// 1. We are going to check that total_processes == 1 => d_name == "1" is
2475	// true (where "=>" is implication). Since C++ does not have => operator,
2476	// let us replace it with its equivalent: a => b == ! a || b.
2477	KMP_DEBUG_ASSERT(total_processes != 1 ||
2478	strcmp(proc_entry->d_name, "1") == 0);
2479
2480	// Construct task_path.
2481	task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2482	__kmp_str_buf_cat(buffer: &task_path, str: proc_entry->d_name,
2483	KMP_STRLEN(s: proc_entry->d_name));
2484	__kmp_str_buf_cat(buffer: &task_path, str: "/task", len: 5);
2485
2486	task_dir = opendir(name: task_path.str);
2487	if (task_dir == NULL) {
2488	// Process can finish between reading "/proc/" directory entry and
2489	// opening process' "task/" directory. So, in general case we should not
2490	// complain, but have to skip this process and read the next one. But on
2491	// systems with no "task/" support we will spend lot of time to scan
2492	// "/proc/" tree again and again without any benefit. "init" process
2493	// (its pid is 1) should exist always, so, if we cannot open
2494	// "/proc/1/task/" directory, it means "task/" is not supported by
2495	// kernel. Report an error now and in the future.
2496	if (strcmp(s1: proc_entry->d_name, s2: "1") == 0) {
2497	running_threads = -1;
2498	permanent_error = 1;
2499	goto finish;
2500	}
2501	} else {
2502	// Construct fixed part of stat file path.
2503	__kmp_str_buf_clear(buffer: &stat_path);
2504	__kmp_str_buf_cat(buffer: &stat_path, str: task_path.str, len: task_path.used);
2505	__kmp_str_buf_cat(buffer: &stat_path, str: "/", len: 1);
2506	stat_path_fixed_len = stat_path.used;
2507
2508	task_entry = readdir(dirp: task_dir);
2509	while (task_entry != NULL) {
2510	// It is a directory and name starts with a digit.
2511	#if KMP_OS_AIX
2512	if (isdigit(task_entry->d_name[0])) {
2513	#else
2514	if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) {
2515	#endif
2516
2517	// Construct complete stat file path. Easiest way would be:
2518	// __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str,
2519	// task_entry->d_name );
2520	// but seriae of __kmp_str_buf_cat works a bit faster.
2521	stat_path.used =
2522	stat_path_fixed_len; // Reset stat path to its fixed part.
2523	__kmp_str_buf_cat(buffer: &stat_path, str: task_entry->d_name,
2524	KMP_STRLEN(s: task_entry->d_name));
2525	__kmp_str_buf_cat(buffer: &stat_path, str: "/stat", len: 5);
2526
2527	// Note: Low-level API (open/read/close) is used. High-level API
2528	// (fopen/fclose) works ~ 30 % slower.
2529	stat_file = open(file: stat_path.str, O_RDONLY);
2530	if (stat_file == -1) {
2531	// We cannot report an error because task (thread) can terminate
2532	// just before reading this file.
2533	} else {
2534	/* Content of "stat" file looks like:
2535	24285 (program) S ...
2536
2537	It is a single line (if program name does not include funny
2538	symbols). First number is a thread id, then name of executable
2539	file name in paretheses, then state of the thread. We need just
2540	thread state.
2541
2542	Good news: Length of program name is 15 characters max. Longer
2543	names are truncated.
2544
2545	Thus, we need rather short buffer: 15 chars for program name +
2546	2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2547
2548	Bad news: Program name may contain special symbols like space,
2549	closing parenthesis, or even new line. This makes parsing
2550	"stat" file not 100 % reliable. In case of fanny program names
2551	parsing may fail (report incorrect thread state).
2552
2553	Parsing "status" file looks more promissing (due to different
2554	file structure and escaping special symbols) but reading and
2555	parsing of "status" file works slower.
2556	-- ln
2557	*/
2558	char buffer[65];
2559	ssize_t len;
2560	len = read(fd: stat_file, buf: buffer, nbytes: sizeof(buffer) - 1);
2561	if (len >= 0) {
2562	buffer[len] = 0;
2563	// Using scanf:
2564	// sscanf( buffer, "%*d (%*s) %c ", & state );
2565	// looks very nice, but searching for a closing parenthesis
2566	// works a bit faster.
2567	char *close_parent = strstr(haystack: buffer, needle: ") ");
2568	if (close_parent != NULL) {
2569	char state = *(close_parent + 2);
2570	if (state == 'R') {
2571	++running_threads;
2572	if (running_threads >= max) {
2573	goto finish;
2574	}
2575	}
2576	}
2577	}
2578	close(fd: stat_file);
2579	stat_file = -1;
2580	}
2581	}
2582	task_entry = readdir(dirp: task_dir);
2583	}
2584	closedir(dirp: task_dir);
2585	task_dir = NULL;
2586	}
2587	}
2588	proc_entry = readdir(dirp: proc_dir);
2589	}
2590
2591	// There _might_ be a timing hole where the thread executing this
2592	// code get skipped in the load balance, and running_threads is 0.
2593	// Assert in the debug builds only!!!
2594	KMP_DEBUG_ASSERT(running_threads > 0);
2595	if (running_threads <= 0) {
2596	running_threads = 1;
2597	}
2598
2599	finish: // Clean up and exit.
2600	if (proc_dir != NULL) {
2601	closedir(dirp: proc_dir);
2602	}
2603	__kmp_str_buf_free(buffer: &task_path);
2604	if (task_dir != NULL) {
2605	closedir(dirp: task_dir);
2606	}
2607	__kmp_str_buf_free(buffer: &stat_path);
2608	if (stat_file != -1) {
2609	close(fd: stat_file);
2610	}
2611
2612	glb_running_threads = running_threads;
2613
2614	return running_threads;
2615
2616	} // __kmp_get_load_balance
2617
2618	#endif // KMP_OS_DARWIN
2619
2620	#endif // USE_LOAD_BALANCE
2621
2622	#if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || \
2623	((KMP_OS_LINUX || KMP_OS_DARWIN) && KMP_ARCH_AARCH64) || \
2624	KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || \
2625	KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF || \
2626	KMP_ARCH_AARCH64_32)
2627
2628	// Because WebAssembly will use `call_indirect` to invoke the microtask and
2629	// WebAssembly indirect calls check that the called signature is a precise
2630	// match, we need to cast each microtask function pointer back from `void *` to
2631	// its original type.
2632	typedef void (*microtask_t0)(int *, int *);
2633	typedef void (*microtask_t1)(int *, int *, void *);
2634	typedef void (*microtask_t2)(int *, int *, void *, void *);
2635	typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
2636	typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
2637	typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
2638	void *);
2639	typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
2640	void *, void *);
2641	typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
2642	void *, void *, void *);
2643	typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
2644	void *, void *, void *, void *);
2645	typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
2646	void *, void *, void *, void *, void *);
2647	typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
2648	void *, void *, void *, void *, void *, void *);
2649	typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
2650	void *, void *, void *, void *, void *, void *,
2651	void *);
2652	typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
2653	void *, void *, void *, void *, void *, void *,
2654	void *, void *);
2655	typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
2656	void *, void *, void *, void *, void *, void *,
2657	void *, void *, void *);
2658	typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
2659	void *, void *, void *, void *, void *, void *,
2660	void *, void *, void *, void *);
2661	typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
2662	void *, void *, void *, void *, void *, void *,
2663	void *, void *, void *, void *, void *);
2664
2665	// we really only need the case with 1 argument, because CLANG always build
2666	// a struct of pointers to shared variables referenced in the outlined function
2667	int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
2668	void *p_argv[]
2669	#if OMPT_SUPPORT
2670	,
2671	void **exit_frame_ptr
2672	#endif
2673	) {
2674	#if OMPT_SUPPORT
2675	*exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
2676	#endif
2677
2678	switch (argc) {
2679	default:
2680	fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2681	fflush(stderr);
2682	exit(-1);
2683	case 0:
2684	(*(microtask_t0)pkfn)(&gtid, &tid);
2685	break;
2686	case 1:
2687	(*(microtask_t1)pkfn)(&gtid, &tid, p_argv[0]);
2688	break;
2689	case 2:
2690	(*(microtask_t2)pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
2691	break;
2692	case 3:
2693	(*(microtask_t3)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
2694	break;
2695	case 4:
2696	(*(microtask_t4)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2697	p_argv[3]);
2698	break;
2699	case 5:
2700	(*(microtask_t5)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2701	p_argv[3], p_argv[4]);
2702	break;
2703	case 6:
2704	(*(microtask_t6)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2705	p_argv[3], p_argv[4], p_argv[5]);
2706	break;
2707	case 7:
2708	(*(microtask_t7)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2709	p_argv[3], p_argv[4], p_argv[5], p_argv[6]);
2710	break;
2711	case 8:
2712	(*(microtask_t8)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2713	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2714	p_argv[7]);
2715	break;
2716	case 9:
2717	(*(microtask_t9)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2718	p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7],
2719	p_argv[8]);
2720	break;
2721	case 10:
2722	(*(microtask_t10)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2723	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2724	p_argv[7], p_argv[8], p_argv[9]);
2725	break;
2726	case 11:
2727	(*(microtask_t11)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2728	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2729	p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2730	break;
2731	case 12:
2732	(*(microtask_t12)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2733	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2734	p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2735	p_argv[11]);
2736	break;
2737	case 13:
2738	(*(microtask_t13)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2739	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2740	p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2741	p_argv[11], p_argv[12]);
2742	break;
2743	case 14:
2744	(*(microtask_t14)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2745	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2746	p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2747	p_argv[11], p_argv[12], p_argv[13]);
2748	break;
2749	case 15:
2750	(*(microtask_t15)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2751	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2752	p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2753	p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2754	break;
2755	}
2756
2757	return 1;
2758	}
2759
2760	#endif
2761
2762	#if KMP_OS_LINUX
2763	// Functions for hidden helper task
2764	namespace {
2765	// Condition variable for initializing hidden helper team
2766	pthread_cond_t hidden_helper_threads_initz_cond_var;
2767	pthread_mutex_t hidden_helper_threads_initz_lock;
2768	volatile int hidden_helper_initz_signaled = FALSE;
2769
2770	// Condition variable for deinitializing hidden helper team
2771	pthread_cond_t hidden_helper_threads_deinitz_cond_var;
2772	pthread_mutex_t hidden_helper_threads_deinitz_lock;
2773	volatile int hidden_helper_deinitz_signaled = FALSE;
2774
2775	// Condition variable for the wrapper function of main thread
2776	pthread_cond_t hidden_helper_main_thread_cond_var;
2777	pthread_mutex_t hidden_helper_main_thread_lock;
2778	volatile int hidden_helper_main_thread_signaled = FALSE;
2779
2780	// Semaphore for worker threads. We don't use condition variable here in case
2781	// that when multiple signals are sent at the same time, only one thread might
2782	// be waken.
2783	sem_t hidden_helper_task_sem;
2784	} // namespace
2785
2786	void __kmp_hidden_helper_worker_thread_wait() {
2787	int status = sem_wait(sem: &hidden_helper_task_sem);
2788	KMP_CHECK_SYSFAIL("sem_wait", status);
2789	}
2790
2791	void __kmp_do_initialize_hidden_helper_threads() {
2792	// Initialize condition variable
2793	int status =
2794	pthread_cond_init(cond: &hidden_helper_threads_initz_cond_var, cond_attr: nullptr);
2795	KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2796
2797	status = pthread_cond_init(cond: &hidden_helper_threads_deinitz_cond_var, cond_attr: nullptr);
2798	KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2799
2800	status = pthread_cond_init(cond: &hidden_helper_main_thread_cond_var, cond_attr: nullptr);
2801	KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2802
2803	status = pthread_mutex_init(mutex: &hidden_helper_threads_initz_lock, mutexattr: nullptr);
2804	KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2805
2806	status = pthread_mutex_init(mutex: &hidden_helper_threads_deinitz_lock, mutexattr: nullptr);
2807	KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2808
2809	status = pthread_mutex_init(mutex: &hidden_helper_main_thread_lock, mutexattr: nullptr);
2810	KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2811
2812	// Initialize the semaphore
2813	status = sem_init(sem: &hidden_helper_task_sem, pshared: 0, value: 0);
2814	KMP_CHECK_SYSFAIL("sem_init", status);
2815
2816	// Create a new thread to finish initialization
2817	pthread_t handle;
2818	status = pthread_create(
2819	newthread: &handle, attr: nullptr,
2820	start_routine: [](void *) -> void * {
2821	__kmp_hidden_helper_threads_initz_routine();
2822	return nullptr;
2823	},
2824	arg: nullptr);
2825	KMP_CHECK_SYSFAIL("pthread_create", status);
2826	}
2827
2828	void __kmp_hidden_helper_threads_initz_wait() {
2829	// Initial thread waits here for the completion of the initialization. The
2830	// condition variable will be notified by main thread of hidden helper teams.
2831	int status = pthread_mutex_lock(mutex: &hidden_helper_threads_initz_lock);
2832	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2833
2834	if (!TCR_4(hidden_helper_initz_signaled)) {
2835	status = pthread_cond_wait(cond: &hidden_helper_threads_initz_cond_var,
2836	mutex: &hidden_helper_threads_initz_lock);
2837	KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2838	}
2839
2840	status = pthread_mutex_unlock(mutex: &hidden_helper_threads_initz_lock);
2841	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2842	}
2843
2844	void __kmp_hidden_helper_initz_release() {
2845	// After all initialization, reset __kmp_init_hidden_helper_threads to false.
2846	int status = pthread_mutex_lock(mutex: &hidden_helper_threads_initz_lock);
2847	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2848
2849	status = pthread_cond_signal(cond: &hidden_helper_threads_initz_cond_var);
2850	KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2851
2852	TCW_SYNC_4(hidden_helper_initz_signaled, TRUE);
2853
2854	status = pthread_mutex_unlock(mutex: &hidden_helper_threads_initz_lock);
2855	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2856	}
2857
2858	void __kmp_hidden_helper_main_thread_wait() {
2859	// The main thread of hidden helper team will be blocked here. The
2860	// condition variable can only be signal in the destructor of RTL.
2861	int status = pthread_mutex_lock(mutex: &hidden_helper_main_thread_lock);
2862	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2863
2864	if (!TCR_4(hidden_helper_main_thread_signaled)) {
2865	status = pthread_cond_wait(cond: &hidden_helper_main_thread_cond_var,
2866	mutex: &hidden_helper_main_thread_lock);
2867	KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2868	}
2869
2870	status = pthread_mutex_unlock(mutex: &hidden_helper_main_thread_lock);
2871	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2872	}
2873
2874	void __kmp_hidden_helper_main_thread_release() {
2875	// The initial thread of OpenMP RTL should call this function to wake up the
2876	// main thread of hidden helper team.
2877	int status = pthread_mutex_lock(mutex: &hidden_helper_main_thread_lock);
2878	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2879
2880	status = pthread_cond_signal(cond: &hidden_helper_main_thread_cond_var);
2881	KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
2882
2883	// The hidden helper team is done here
2884	TCW_SYNC_4(hidden_helper_main_thread_signaled, TRUE);
2885
2886	status = pthread_mutex_unlock(mutex: &hidden_helper_main_thread_lock);
2887	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2888	}
2889
2890	void __kmp_hidden_helper_worker_thread_signal() {
2891	int status = sem_post(sem: &hidden_helper_task_sem);
2892	KMP_CHECK_SYSFAIL("sem_post", status);
2893	}
2894
2895	void __kmp_hidden_helper_threads_deinitz_wait() {
2896	// Initial thread waits here for the completion of the deinitialization. The
2897	// condition variable will be notified by main thread of hidden helper teams.
2898	int status = pthread_mutex_lock(mutex: &hidden_helper_threads_deinitz_lock);
2899	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2900
2901	if (!TCR_4(hidden_helper_deinitz_signaled)) {
2902	status = pthread_cond_wait(cond: &hidden_helper_threads_deinitz_cond_var,
2903	mutex: &hidden_helper_threads_deinitz_lock);
2904	KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2905	}
2906
2907	status = pthread_mutex_unlock(mutex: &hidden_helper_threads_deinitz_lock);
2908	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2909	}
2910
2911	void __kmp_hidden_helper_threads_deinitz_release() {
2912	int status = pthread_mutex_lock(mutex: &hidden_helper_threads_deinitz_lock);
2913	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2914
2915	status = pthread_cond_signal(cond: &hidden_helper_threads_deinitz_cond_var);
2916	KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2917
2918	TCW_SYNC_4(hidden_helper_deinitz_signaled, TRUE);
2919
2920	status = pthread_mutex_unlock(mutex: &hidden_helper_threads_deinitz_lock);
2921	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2922	}
2923	#else // KMP_OS_LINUX
2924	void __kmp_hidden_helper_worker_thread_wait() {
2925	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2926	}
2927
2928	void __kmp_do_initialize_hidden_helper_threads() {
2929	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2930	}
2931
2932	void __kmp_hidden_helper_threads_initz_wait() {
2933	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2934	}
2935
2936	void __kmp_hidden_helper_initz_release() {
2937	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2938	}
2939
2940	void __kmp_hidden_helper_main_thread_wait() {
2941	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2942	}
2943
2944	void __kmp_hidden_helper_main_thread_release() {
2945	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2946	}
2947
2948	void __kmp_hidden_helper_worker_thread_signal() {
2949	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2950	}
2951
2952	void __kmp_hidden_helper_threads_deinitz_wait() {
2953	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2954	}
2955
2956	void __kmp_hidden_helper_threads_deinitz_release() {
2957	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2958	}
2959	#endif // KMP_OS_LINUX
2960
2961	bool __kmp_detect_shm() {
2962	DIR *dir = opendir(name: "/dev/shm");
2963	if (dir) { // /dev/shm exists
2964	closedir(dirp: dir);
2965	return true;
2966	} else if (ENOENT == errno) { // /dev/shm does not exist
2967	return false;
2968	} else { // opendir() failed
2969	return false;
2970	}
2971	}
2972
2973	bool __kmp_detect_tmp() {
2974	DIR *dir = opendir(name: "/tmp");
2975	if (dir) { // /tmp exists
2976	closedir(dirp: dir);
2977	return true;
2978	} else if (ENOENT == errno) { // /tmp does not exist
2979	return false;
2980	} else { // opendir() failed
2981	return false;
2982	}
2983	}
2984
2985	// end of file //
2986