LLVM OpenMP* Runtime Library
z_Linux_util.c
1 /*
2  * z_Linux_util.c -- platform specific routines.
3  */
4 
5 
6 //===----------------------------------------------------------------------===//
7 //
8 // The LLVM Compiler Infrastructure
9 //
10 // This file is dual licensed under the MIT and the University of Illinois Open
11 // Source Licenses. See LICENSE.txt for details.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 
16 #include "kmp.h"
17 #include "kmp_wrapper_getpid.h"
18 #include "kmp_itt.h"
19 #include "kmp_str.h"
20 #include "kmp_i18n.h"
21 #include "kmp_io.h"
22 #include "kmp_stats.h"
23 #include "kmp_wait_release.h"
24 
25 #if !KMP_OS_FREEBSD && !KMP_OS_NETBSD
26 # include <alloca.h>
27 #endif
28 #include <unistd.h>
29 #include <math.h> // HUGE_VAL.
30 #include <sys/time.h>
31 #include <sys/times.h>
32 #include <sys/resource.h>
33 #include <sys/syscall.h>
34 
35 #if KMP_OS_LINUX && !KMP_OS_CNK
36 # include <sys/sysinfo.h>
37 # if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
38 // We should really include <futex.h>, but that causes compatibility problems on different
39 // Linux* OS distributions that either require that you include (or break when you try to include)
40 // <pci/types.h>.
41 // Since all we need is the two macros below (which are part of the kernel ABI, so can't change)
42 // we just define the constants here and don't include <futex.h>
43 # ifndef FUTEX_WAIT
44 # define FUTEX_WAIT 0
45 # endif
46 # ifndef FUTEX_WAKE
47 # define FUTEX_WAKE 1
48 # endif
49 # endif
50 #elif KMP_OS_DARWIN
51 # include <sys/sysctl.h>
52 # include <mach/mach.h>
53 #elif KMP_OS_FREEBSD
54 # include <pthread_np.h>
55 #endif
56 
57 
58 #include <dirent.h>
59 #include <ctype.h>
60 #include <fcntl.h>
61 
62 /* ------------------------------------------------------------------------ */
63 /* ------------------------------------------------------------------------ */
64 
65 struct kmp_sys_timer {
66  struct timespec start;
67 };
68 
69 // Convert timespec to nanoseconds.
70 #define TS2NS(timespec) (((timespec).tv_sec * 1e9) + (timespec).tv_nsec)
71 
72 static struct kmp_sys_timer __kmp_sys_timer_data;
73 
74 #if KMP_HANDLE_SIGNALS
75  typedef void (* sig_func_t )( int );
76  STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[ NSIG ];
77  static sigset_t __kmp_sigset;
78 #endif
79 
80 static int __kmp_init_runtime = FALSE;
81 
82 static int __kmp_fork_count = 0;
83 
84 static pthread_condattr_t __kmp_suspend_cond_attr;
85 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
86 
87 static kmp_cond_align_t __kmp_wait_cv;
88 static kmp_mutex_align_t __kmp_wait_mx;
89 
90 /* ------------------------------------------------------------------------ */
91 /* ------------------------------------------------------------------------ */
92 
93 #ifdef DEBUG_SUSPEND
94 static void
95 __kmp_print_cond( char *buffer, kmp_cond_align_t *cond )
96 {
97  KMP_SNPRINTF( buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
98  cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
99  cond->c_cond.__c_waiting );
100 }
101 #endif
102 
103 /* ------------------------------------------------------------------------ */
104 /* ------------------------------------------------------------------------ */
105 
106 #if ( KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED)
107 
108 /*
109  * Affinity support
110  */
111 
112 /*
113  * On some of the older OS's that we build on, these constants aren't present
114  * in <asm/unistd.h> #included from <sys.syscall.h>. They must be the same on
115  * all systems of the same arch where they are defined, and they cannot change.
116  * stone forever.
117  */
118 
119 # if KMP_ARCH_X86 || KMP_ARCH_ARM
120 # ifndef __NR_sched_setaffinity
121 # define __NR_sched_setaffinity 241
122 # elif __NR_sched_setaffinity != 241
123 # error Wrong code for setaffinity system call.
124 # endif /* __NR_sched_setaffinity */
125 # ifndef __NR_sched_getaffinity
126 # define __NR_sched_getaffinity 242
127 # elif __NR_sched_getaffinity != 242
128 # error Wrong code for getaffinity system call.
129 # endif /* __NR_sched_getaffinity */
130 
131 # elif KMP_ARCH_AARCH64
132 # ifndef __NR_sched_setaffinity
133 # define __NR_sched_setaffinity 122
134 # elif __NR_sched_setaffinity != 122
135 # error Wrong code for setaffinity system call.
136 # endif /* __NR_sched_setaffinity */
137 # ifndef __NR_sched_getaffinity
138 # define __NR_sched_getaffinity 123
139 # elif __NR_sched_getaffinity != 123
140 # error Wrong code for getaffinity system call.
141 # endif /* __NR_sched_getaffinity */
142 
143 # elif KMP_ARCH_X86_64
144 # ifndef __NR_sched_setaffinity
145 # define __NR_sched_setaffinity 203
146 # elif __NR_sched_setaffinity != 203
147 # error Wrong code for setaffinity system call.
148 # endif /* __NR_sched_setaffinity */
149 # ifndef __NR_sched_getaffinity
150 # define __NR_sched_getaffinity 204
151 # elif __NR_sched_getaffinity != 204
152 # error Wrong code for getaffinity system call.
153 # endif /* __NR_sched_getaffinity */
154 
155 # elif KMP_ARCH_PPC64
156 # ifndef __NR_sched_setaffinity
157 # define __NR_sched_setaffinity 222
158 # elif __NR_sched_setaffinity != 222
159 # error Wrong code for setaffinity system call.
160 # endif /* __NR_sched_setaffinity */
161 # ifndef __NR_sched_getaffinity
162 # define __NR_sched_getaffinity 223
163 # elif __NR_sched_getaffinity != 223
164 # error Wrong code for getaffinity system call.
165 # endif /* __NR_sched_getaffinity */
166 
167 # elif KMP_ARCH_MIPS
168 # ifndef __NR_sched_setaffinity
169 # define __NR_sched_setaffinity 4239
170 # elif __NR_sched_setaffinity != 4239
171 # error Wrong code for setaffinity system call.
172 # endif /* __NR_sched_setaffinity */
173 # ifndef __NR_sched_getaffinity
174 # define __NR_sched_getaffinity 4240
175 # elif __NR_sched_getaffinity != 4240
176 # error Wrong code for getaffinity system call.
177 # endif /* __NR_sched_getaffinity */
178 
179 # elif KMP_ARCH_MIPS64
180 # ifndef __NR_sched_setaffinity
181 # define __NR_sched_setaffinity 5195
182 # elif __NR_sched_setaffinity != 5195
183 # error Wrong code for setaffinity system call.
184 # endif /* __NR_sched_setaffinity */
185 # ifndef __NR_sched_getaffinity
186 # define __NR_sched_getaffinity 5196
187 # elif __NR_sched_getaffinity != 5196
188 # error Wrong code for getaffinity system call.
189 # endif /* __NR_sched_getaffinity */
190 
191 # else
192 # error Unknown or unsupported architecture
193 
194 # endif /* KMP_ARCH_* */
195 
196 int
197 __kmp_set_system_affinity( kmp_affin_mask_t const *mask, int abort_on_error )
198 {
199  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
200  "Illegal set affinity operation when not capable");
201 #if KMP_USE_HWLOC
202  int retval = hwloc_set_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
203 #else
204  int retval = syscall( __NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask );
205 #endif
206  if (retval >= 0) {
207  return 0;
208  }
209  int error = errno;
210  if (abort_on_error) {
211  __kmp_msg(
212  kmp_ms_fatal,
213  KMP_MSG( FatalSysError ),
214  KMP_ERR( error ),
215  __kmp_msg_null
216  );
217  }
218  return error;
219 }
220 
221 int
222 __kmp_get_system_affinity( kmp_affin_mask_t *mask, int abort_on_error )
223 {
224  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
225  "Illegal get affinity operation when not capable");
226 
227 #if KMP_USE_HWLOC
228  int retval = hwloc_get_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
229 #else
230  int retval = syscall( __NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask );
231 #endif
232  if (retval >= 0) {
233  return 0;
234  }
235  int error = errno;
236  if (abort_on_error) {
237  __kmp_msg(
238  kmp_ms_fatal,
239  KMP_MSG( FatalSysError ),
240  KMP_ERR( error ),
241  __kmp_msg_null
242  );
243  }
244  return error;
245 }
246 
247 void
248 __kmp_affinity_bind_thread( int which )
249 {
250  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
251  "Illegal set affinity operation when not capable");
252 
253  kmp_affin_mask_t *mask;
254  KMP_CPU_ALLOC_ON_STACK(mask);
255  KMP_CPU_ZERO(mask);
256  KMP_CPU_SET(which, mask);
257  __kmp_set_system_affinity(mask, TRUE);
258  KMP_CPU_FREE_FROM_STACK(mask);
259 }
260 
261 /*
262  * Determine if we can access affinity functionality on this version of
263  * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
264  * __kmp_affin_mask_size to the appropriate value (0 means not capable).
265  */
266 void
267 __kmp_affinity_determine_capable(const char *env_var)
268 {
269  //
270  // Check and see if the OS supports thread affinity.
271  //
272 
273 # define KMP_CPU_SET_SIZE_LIMIT (1024*1024)
274 
275  int gCode;
276  int sCode;
277  kmp_affin_mask_t *buf;
278  buf = ( kmp_affin_mask_t * ) KMP_INTERNAL_MALLOC( KMP_CPU_SET_SIZE_LIMIT );
279 
280  // If Linux* OS:
281  // If the syscall fails or returns a suggestion for the size,
282  // then we don't have to search for an appropriate size.
283  gCode = syscall( __NR_sched_getaffinity, 0, KMP_CPU_SET_SIZE_LIMIT, buf );
284  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
285  "initial getaffinity call returned %d errno = %d\n",
286  gCode, errno));
287 
288  //if ((gCode < 0) && (errno == ENOSYS))
289  if (gCode < 0) {
290  //
291  // System call not supported
292  //
293  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
294  && (__kmp_affinity_type != affinity_none)
295  && (__kmp_affinity_type != affinity_default)
296  && (__kmp_affinity_type != affinity_disabled))) {
297  int error = errno;
298  __kmp_msg(
299  kmp_ms_warning,
300  KMP_MSG( GetAffSysCallNotSupported, env_var ),
301  KMP_ERR( error ),
302  __kmp_msg_null
303  );
304  }
305  KMP_AFFINITY_DISABLE();
306  KMP_INTERNAL_FREE(buf);
307  return;
308  }
309  if (gCode > 0) { // Linux* OS only
310  // The optimal situation: the OS returns the size of the buffer
311  // it expects.
312  //
313  // A verification of correct behavior is that Isetaffinity on a NULL
314  // buffer with the same size fails with errno set to EFAULT.
315  sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
316  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
317  "setaffinity for mask size %d returned %d errno = %d\n",
318  gCode, sCode, errno));
319  if (sCode < 0) {
320  if (errno == ENOSYS) {
321  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
322  && (__kmp_affinity_type != affinity_none)
323  && (__kmp_affinity_type != affinity_default)
324  && (__kmp_affinity_type != affinity_disabled))) {
325  int error = errno;
326  __kmp_msg(
327  kmp_ms_warning,
328  KMP_MSG( SetAffSysCallNotSupported, env_var ),
329  KMP_ERR( error ),
330  __kmp_msg_null
331  );
332  }
333  KMP_AFFINITY_DISABLE();
334  KMP_INTERNAL_FREE(buf);
335  }
336  if (errno == EFAULT) {
337  KMP_AFFINITY_ENABLE(gCode);
338  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
339  "affinity supported (mask size %d)\n",
340  (int)__kmp_affin_mask_size));
341  KMP_INTERNAL_FREE(buf);
342  return;
343  }
344  }
345  }
346 
347  //
348  // Call the getaffinity system call repeatedly with increasing set sizes
349  // until we succeed, or reach an upper bound on the search.
350  //
351  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
352  "searching for proper set size\n"));
353  int size;
354  for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
355  gCode = syscall( __NR_sched_getaffinity, 0, size, buf );
356  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
357  "getaffinity for mask size %d returned %d errno = %d\n", size,
358  gCode, errno));
359 
360  if (gCode < 0) {
361  if ( errno == ENOSYS )
362  {
363  //
364  // We shouldn't get here
365  //
366  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
367  "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
368  size));
369  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
370  && (__kmp_affinity_type != affinity_none)
371  && (__kmp_affinity_type != affinity_default)
372  && (__kmp_affinity_type != affinity_disabled))) {
373  int error = errno;
374  __kmp_msg(
375  kmp_ms_warning,
376  KMP_MSG( GetAffSysCallNotSupported, env_var ),
377  KMP_ERR( error ),
378  __kmp_msg_null
379  );
380  }
381  KMP_AFFINITY_DISABLE();
382  KMP_INTERNAL_FREE(buf);
383  return;
384  }
385  continue;
386  }
387 
388  sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
389  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
390  "setaffinity for mask size %d returned %d errno = %d\n",
391  gCode, sCode, errno));
392  if (sCode < 0) {
393  if (errno == ENOSYS) { // Linux* OS only
394  //
395  // We shouldn't get here
396  //
397  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
398  "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
399  size));
400  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
401  && (__kmp_affinity_type != affinity_none)
402  && (__kmp_affinity_type != affinity_default)
403  && (__kmp_affinity_type != affinity_disabled))) {
404  int error = errno;
405  __kmp_msg(
406  kmp_ms_warning,
407  KMP_MSG( SetAffSysCallNotSupported, env_var ),
408  KMP_ERR( error ),
409  __kmp_msg_null
410  );
411  }
412  KMP_AFFINITY_DISABLE();
413  KMP_INTERNAL_FREE(buf);
414  return;
415  }
416  if (errno == EFAULT) {
417  KMP_AFFINITY_ENABLE(gCode);
418  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
419  "affinity supported (mask size %d)\n",
420  (int)__kmp_affin_mask_size));
421  KMP_INTERNAL_FREE(buf);
422  return;
423  }
424  }
425  }
426  //int error = errno; // save uncaught error code
427  KMP_INTERNAL_FREE(buf);
428  // errno = error; // restore uncaught error code, will be printed at the next KMP_WARNING below
429 
430  //
431  // Affinity is not supported
432  //
433  KMP_AFFINITY_DISABLE();
434  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
435  "cannot determine mask size - affinity not supported\n"));
436  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
437  && (__kmp_affinity_type != affinity_none)
438  && (__kmp_affinity_type != affinity_default)
439  && (__kmp_affinity_type != affinity_disabled))) {
440  KMP_WARNING( AffCantGetMaskSize, env_var );
441  }
442 }
443 
444 #endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
445 
446 /* ------------------------------------------------------------------------ */
447 /* ------------------------------------------------------------------------ */
448 
449 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && !KMP_OS_CNK
450 
451 int
452 __kmp_futex_determine_capable()
453 {
454  int loc = 0;
455  int rc = syscall( __NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0 );
456  int retval = ( rc == 0 ) || ( errno != ENOSYS );
457 
458  KA_TRACE(10, ( "__kmp_futex_determine_capable: rc = %d errno = %d\n", rc,
459  errno ) );
460  KA_TRACE(10, ( "__kmp_futex_determine_capable: futex syscall%s supported\n",
461  retval ? "" : " not" ) );
462 
463  return retval;
464 }
465 
466 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) && !KMP_OS_CNK
467 
468 /* ------------------------------------------------------------------------ */
469 /* ------------------------------------------------------------------------ */
470 
471 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS)
472 /*
473  * Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
474  * use compare_and_store for these routines
475  */
476 
477 kmp_int8
478 __kmp_test_then_or8( volatile kmp_int8 *p, kmp_int8 d )
479 {
480  kmp_int8 old_value, new_value;
481 
482  old_value = TCR_1( *p );
483  new_value = old_value | d;
484 
485  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
486  {
487  KMP_CPU_PAUSE();
488  old_value = TCR_1( *p );
489  new_value = old_value | d;
490  }
491  return old_value;
492 }
493 
494 kmp_int8
495 __kmp_test_then_and8( volatile kmp_int8 *p, kmp_int8 d )
496 {
497  kmp_int8 old_value, new_value;
498 
499  old_value = TCR_1( *p );
500  new_value = old_value & d;
501 
502  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
503  {
504  KMP_CPU_PAUSE();
505  old_value = TCR_1( *p );
506  new_value = old_value & d;
507  }
508  return old_value;
509 }
510 
511 kmp_int32
512 __kmp_test_then_or32( volatile kmp_int32 *p, kmp_int32 d )
513 {
514  kmp_int32 old_value, new_value;
515 
516  old_value = TCR_4( *p );
517  new_value = old_value | d;
518 
519  while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
520  {
521  KMP_CPU_PAUSE();
522  old_value = TCR_4( *p );
523  new_value = old_value | d;
524  }
525  return old_value;
526 }
527 
528 kmp_int32
529 __kmp_test_then_and32( volatile kmp_int32 *p, kmp_int32 d )
530 {
531  kmp_int32 old_value, new_value;
532 
533  old_value = TCR_4( *p );
534  new_value = old_value & d;
535 
536  while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
537  {
538  KMP_CPU_PAUSE();
539  old_value = TCR_4( *p );
540  new_value = old_value & d;
541  }
542  return old_value;
543 }
544 
545 # if KMP_ARCH_X86 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64
546 kmp_int8
547 __kmp_test_then_add8( volatile kmp_int8 *p, kmp_int8 d )
548 {
549  kmp_int8 old_value, new_value;
550 
551  old_value = TCR_1( *p );
552  new_value = old_value + d;
553 
554  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
555  {
556  KMP_CPU_PAUSE();
557  old_value = TCR_1( *p );
558  new_value = old_value + d;
559  }
560  return old_value;
561 }
562 
563 kmp_int64
564 __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 d )
565 {
566  kmp_int64 old_value, new_value;
567 
568  old_value = TCR_8( *p );
569  new_value = old_value + d;
570 
571  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
572  {
573  KMP_CPU_PAUSE();
574  old_value = TCR_8( *p );
575  new_value = old_value + d;
576  }
577  return old_value;
578 }
579 # endif /* KMP_ARCH_X86 */
580 
581 kmp_int64
582 __kmp_test_then_or64( volatile kmp_int64 *p, kmp_int64 d )
583 {
584  kmp_int64 old_value, new_value;
585 
586  old_value = TCR_8( *p );
587  new_value = old_value | d;
588  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
589  {
590  KMP_CPU_PAUSE();
591  old_value = TCR_8( *p );
592  new_value = old_value | d;
593  }
594  return old_value;
595 }
596 
597 kmp_int64
598 __kmp_test_then_and64( volatile kmp_int64 *p, kmp_int64 d )
599 {
600  kmp_int64 old_value, new_value;
601 
602  old_value = TCR_8( *p );
603  new_value = old_value & d;
604  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
605  {
606  KMP_CPU_PAUSE();
607  old_value = TCR_8( *p );
608  new_value = old_value & d;
609  }
610  return old_value;
611 }
612 
613 #endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
614 
615 void
616 __kmp_terminate_thread( int gtid )
617 {
618  int status;
619  kmp_info_t *th = __kmp_threads[ gtid ];
620 
621  if ( !th ) return;
622 
623  #ifdef KMP_CANCEL_THREADS
624  KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) );
625  status = pthread_cancel( th->th.th_info.ds.ds_thread );
626  if ( status != 0 && status != ESRCH ) {
627  __kmp_msg(
628  kmp_ms_fatal,
629  KMP_MSG( CantTerminateWorkerThread ),
630  KMP_ERR( status ),
631  __kmp_msg_null
632  );
633  }; // if
634  #endif
635  __kmp_yield( TRUE );
636 } //
637 
638 /* ------------------------------------------------------------------------ */
639 /* ------------------------------------------------------------------------ */
640 
641 /* ------------------------------------------------------------------------ */
642 /* ------------------------------------------------------------------------ */
643 
644 /*
645  * Set thread stack info according to values returned by
646  * pthread_getattr_np().
647  * If values are unreasonable, assume call failed and use
648  * incremental stack refinement method instead.
649  * Returns TRUE if the stack parameters could be determined exactly,
650  * FALSE if incremental refinement is necessary.
651  */
652 static kmp_int32
653 __kmp_set_stack_info( int gtid, kmp_info_t *th )
654 {
655  int stack_data;
656 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD
657  /* Linux* OS only -- no pthread_getattr_np support on OS X* */
658  pthread_attr_t attr;
659  int status;
660  size_t size = 0;
661  void * addr = 0;
662 
663  /* Always do incremental stack refinement for ubermaster threads since the initial
664  thread stack range can be reduced by sibling thread creation so pthread_attr_getstack
665  may cause thread gtid aliasing */
666  if ( ! KMP_UBER_GTID(gtid) ) {
667 
668  /* Fetch the real thread attributes */
669  status = pthread_attr_init( &attr );
670  KMP_CHECK_SYSFAIL( "pthread_attr_init", status );
671 #if KMP_OS_FREEBSD || KMP_OS_NETBSD
672  status = pthread_attr_get_np( pthread_self(), &attr );
673  KMP_CHECK_SYSFAIL( "pthread_attr_get_np", status );
674 #else
675  status = pthread_getattr_np( pthread_self(), &attr );
676  KMP_CHECK_SYSFAIL( "pthread_getattr_np", status );
677 #endif
678  status = pthread_attr_getstack( &attr, &addr, &size );
679  KMP_CHECK_SYSFAIL( "pthread_attr_getstack", status );
680  KA_TRACE( 60, ( "__kmp_set_stack_info: T#%d pthread_attr_getstack returned size: %lu, "
681  "low addr: %p\n",
682  gtid, size, addr ));
683 
684  status = pthread_attr_destroy( &attr );
685  KMP_CHECK_SYSFAIL( "pthread_attr_destroy", status );
686  }
687 
688  if ( size != 0 && addr != 0 ) { /* was stack parameter determination successful? */
689  /* Store the correct base and size */
690  TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
691  TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
692  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
693  return TRUE;
694  }
695 #endif /* KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD */
696  /* Use incremental refinement starting from initial conservative estimate */
697  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
698  TCW_PTR(th -> th.th_info.ds.ds_stackbase, &stack_data);
699  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
700  return FALSE;
701 }
702 
703 static void*
704 __kmp_launch_worker( void *thr )
705 {
706  int status, old_type, old_state;
707 #ifdef KMP_BLOCK_SIGNALS
708  sigset_t new_set, old_set;
709 #endif /* KMP_BLOCK_SIGNALS */
710  void *exit_val;
711 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD
712  void * volatile padding = 0;
713 #endif
714  int gtid;
715 
716  gtid = ((kmp_info_t*)thr) -> th.th_info.ds.ds_gtid;
717  __kmp_gtid_set_specific( gtid );
718 #ifdef KMP_TDATA_GTID
719  __kmp_gtid = gtid;
720 #endif
721 #if KMP_STATS_ENABLED
722  // set __thread local index to point to thread-specific stats
723  __kmp_stats_thread_ptr = ((kmp_info_t*)thr)->th.th_stats;
724 #endif
725 
726 #if USE_ITT_BUILD
727  __kmp_itt_thread_name( gtid );
728 #endif /* USE_ITT_BUILD */
729 
730 #if KMP_AFFINITY_SUPPORTED
731  __kmp_affinity_set_init_mask( gtid, FALSE );
732 #endif
733 
734 #ifdef KMP_CANCEL_THREADS
735  status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
736  KMP_CHECK_SYSFAIL( "pthread_setcanceltype", status );
737  /* josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? */
738  status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
739  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
740 #endif
741 
742 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
743  //
744  // Set the FP control regs to be a copy of
745  // the parallel initialization thread's.
746  //
747  __kmp_clear_x87_fpu_status_word();
748  __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
749  __kmp_load_mxcsr( &__kmp_init_mxcsr );
750 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
751 
752 #ifdef KMP_BLOCK_SIGNALS
753  status = sigfillset( & new_set );
754  KMP_CHECK_SYSFAIL_ERRNO( "sigfillset", status );
755  status = pthread_sigmask( SIG_BLOCK, & new_set, & old_set );
756  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
757 #endif /* KMP_BLOCK_SIGNALS */
758 
759 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD
760  if ( __kmp_stkoffset > 0 && gtid > 0 ) {
761  padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
762  }
763 #endif
764 
765  KMP_MB();
766  __kmp_set_stack_info( gtid, (kmp_info_t*)thr );
767 
768  __kmp_check_stack_overlap( (kmp_info_t*)thr );
769 
770  exit_val = __kmp_launch_thread( (kmp_info_t *) thr );
771 
772 #ifdef KMP_BLOCK_SIGNALS
773  status = pthread_sigmask( SIG_SETMASK, & old_set, NULL );
774  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
775 #endif /* KMP_BLOCK_SIGNALS */
776 
777  return exit_val;
778 }
779 
780 
781 /* The monitor thread controls all of the threads in the complex */
782 
783 static void*
784 __kmp_launch_monitor( void *thr )
785 {
786  int status, old_type, old_state;
787 #ifdef KMP_BLOCK_SIGNALS
788  sigset_t new_set;
789 #endif /* KMP_BLOCK_SIGNALS */
790  struct timespec interval;
791  int yield_count;
792  int yield_cycles = 0;
793 
794  KMP_MB(); /* Flush all pending memory write invalidates. */
795 
796  KA_TRACE( 10, ("__kmp_launch_monitor: #1 launched\n" ) );
797 
798  /* register us as the monitor thread */
799  __kmp_gtid_set_specific( KMP_GTID_MONITOR );
800 #ifdef KMP_TDATA_GTID
801  __kmp_gtid = KMP_GTID_MONITOR;
802 #endif
803 
804  KMP_MB();
805 
806 #if USE_ITT_BUILD
807  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore monitor thread.
808 #endif /* USE_ITT_BUILD */
809 
810  __kmp_set_stack_info( ((kmp_info_t*)thr)->th.th_info.ds.ds_gtid, (kmp_info_t*)thr );
811 
812  __kmp_check_stack_overlap( (kmp_info_t*)thr );
813 
814 #ifdef KMP_CANCEL_THREADS
815  status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
816  KMP_CHECK_SYSFAIL( "pthread_setcanceltype", status );
817  /* josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? */
818  status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
819  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
820 #endif
821 
822  #if KMP_REAL_TIME_FIX
823  // This is a potential fix which allows application with real-time scheduling policy work.
824  // However, decision about the fix is not made yet, so it is disabled by default.
825  { // Are program started with real-time scheduling policy?
826  int sched = sched_getscheduler( 0 );
827  if ( sched == SCHED_FIFO || sched == SCHED_RR ) {
828  // Yes, we are a part of real-time application. Try to increase the priority of the
829  // monitor.
830  struct sched_param param;
831  int max_priority = sched_get_priority_max( sched );
832  int rc;
833  KMP_WARNING( RealTimeSchedNotSupported );
834  sched_getparam( 0, & param );
835  if ( param.sched_priority < max_priority ) {
836  param.sched_priority += 1;
837  rc = sched_setscheduler( 0, sched, & param );
838  if ( rc != 0 ) {
839  int error = errno;
840  __kmp_msg(
841  kmp_ms_warning,
842  KMP_MSG( CantChangeMonitorPriority ),
843  KMP_ERR( error ),
844  KMP_MSG( MonitorWillStarve ),
845  __kmp_msg_null
846  );
847  }; // if
848  } else {
849  // We cannot abort here, because number of CPUs may be enough for all the threads,
850  // including the monitor thread, so application could potentially work...
851  __kmp_msg(
852  kmp_ms_warning,
853  KMP_MSG( RunningAtMaxPriority ),
854  KMP_MSG( MonitorWillStarve ),
855  KMP_HNT( RunningAtMaxPriority ),
856  __kmp_msg_null
857  );
858  }; // if
859  }; // if
860  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 ); // AC: free thread that waits for monitor started
861  }
862  #endif // KMP_REAL_TIME_FIX
863 
864  KMP_MB(); /* Flush all pending memory write invalidates. */
865 
866  if ( __kmp_monitor_wakeups == 1 ) {
867  interval.tv_sec = 1;
868  interval.tv_nsec = 0;
869  } else {
870  interval.tv_sec = 0;
871  interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
872  }
873 
874  KA_TRACE( 10, ("__kmp_launch_monitor: #2 monitor\n" ) );
875 
876  if (__kmp_yield_cycle) {
877  __kmp_yielding_on = 0; /* Start out with yielding shut off */
878  yield_count = __kmp_yield_off_count;
879  } else {
880  __kmp_yielding_on = 1; /* Yielding is on permanently */
881  }
882 
883  while( ! TCR_4( __kmp_global.g.g_done ) ) {
884  struct timespec now;
885  struct timeval tval;
886 
887  /* This thread monitors the state of the system */
888 
889  KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) );
890 
891  status = gettimeofday( &tval, NULL );
892  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
893  TIMEVAL_TO_TIMESPEC( &tval, &now );
894 
895  now.tv_sec += interval.tv_sec;
896  now.tv_nsec += interval.tv_nsec;
897 
898  if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
899  now.tv_sec += 1;
900  now.tv_nsec -= KMP_NSEC_PER_SEC;
901  }
902 
903  status = pthread_mutex_lock( & __kmp_wait_mx.m_mutex );
904  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
905  // AC: the monitor should not fall asleep if g_done has been set
906  if ( !TCR_4(__kmp_global.g.g_done) ) { // check once more under mutex
907  status = pthread_cond_timedwait( &__kmp_wait_cv.c_cond, &__kmp_wait_mx.m_mutex, &now );
908  if ( status != 0 ) {
909  if ( status != ETIMEDOUT && status != EINTR ) {
910  KMP_SYSFAIL( "pthread_cond_timedwait", status );
911  };
912  };
913  };
914  status = pthread_mutex_unlock( & __kmp_wait_mx.m_mutex );
915  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
916 
917  if (__kmp_yield_cycle) {
918  yield_cycles++;
919  if ( (yield_cycles % yield_count) == 0 ) {
920  if (__kmp_yielding_on) {
921  __kmp_yielding_on = 0; /* Turn it off now */
922  yield_count = __kmp_yield_off_count;
923  } else {
924  __kmp_yielding_on = 1; /* Turn it on now */
925  yield_count = __kmp_yield_on_count;
926  }
927  yield_cycles = 0;
928  }
929  } else {
930  __kmp_yielding_on = 1;
931  }
932 
933  TCW_4( __kmp_global.g.g_time.dt.t_value,
934  TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
935 
936  KMP_MB(); /* Flush all pending memory write invalidates. */
937  }
938 
939  KA_TRACE( 10, ("__kmp_launch_monitor: #3 cleanup\n" ) );
940 
941 #ifdef KMP_BLOCK_SIGNALS
942  status = sigfillset( & new_set );
943  KMP_CHECK_SYSFAIL_ERRNO( "sigfillset", status );
944  status = pthread_sigmask( SIG_UNBLOCK, & new_set, NULL );
945  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
946 #endif /* KMP_BLOCK_SIGNALS */
947 
948  KA_TRACE( 10, ("__kmp_launch_monitor: #4 finished\n" ) );
949 
950  if( __kmp_global.g.g_abort != 0 ) {
951  /* now we need to terminate the worker threads */
952  /* the value of t_abort is the signal we caught */
953 
954  int gtid;
955 
956  KA_TRACE( 10, ("__kmp_launch_monitor: #5 terminate sig=%d\n", __kmp_global.g.g_abort ) );
957 
958  /* terminate the OpenMP worker threads */
959  /* TODO this is not valid for sibling threads!!
960  * the uber master might not be 0 anymore.. */
961  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
962  __kmp_terminate_thread( gtid );
963 
964  __kmp_cleanup();
965 
966  KA_TRACE( 10, ("__kmp_launch_monitor: #6 raise sig=%d\n", __kmp_global.g.g_abort ) );
967 
968  if (__kmp_global.g.g_abort > 0)
969  raise( __kmp_global.g.g_abort );
970 
971  }
972 
973  KA_TRACE( 10, ("__kmp_launch_monitor: #7 exit\n" ) );
974 
975  return thr;
976 }
977 
978 void
979 __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
980 {
981  pthread_t handle;
982  pthread_attr_t thread_attr;
983  int status;
984 
985 
986  th->th.th_info.ds.ds_gtid = gtid;
987 
988 #if KMP_STATS_ENABLED
989  // sets up worker thread stats
990  __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
991 
992  // th->th.th_stats is used to transfer thread specific stats-pointer to __kmp_launch_worker
993  // So when thread is created (goes into __kmp_launch_worker) it will
994  // set it's __thread local pointer to th->th.th_stats
995  th->th.th_stats = __kmp_stats_list.push_back(gtid);
996  if(KMP_UBER_GTID(gtid)) {
997  __kmp_stats_start_time = tsc_tick_count::now();
998  __kmp_stats_thread_ptr = th->th.th_stats;
999  __kmp_stats_init();
1000  KMP_START_EXPLICIT_TIMER(OMP_serial);
1001  KMP_START_EXPLICIT_TIMER(OMP_start_end);
1002  }
1003  __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
1004 
1005 #endif // KMP_STATS_ENABLED
1006 
1007  if ( KMP_UBER_GTID(gtid) ) {
1008  KA_TRACE( 10, ("__kmp_create_worker: uber thread (%d)\n", gtid ) );
1009  th -> th.th_info.ds.ds_thread = pthread_self();
1010  __kmp_set_stack_info( gtid, th );
1011  __kmp_check_stack_overlap( th );
1012  return;
1013  }; // if
1014 
1015  KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1016 
1017  KMP_MB(); /* Flush all pending memory write invalidates. */
1018 
1019 #ifdef KMP_THREAD_ATTR
1020  {
1021  status = pthread_attr_init( &thread_attr );
1022  if ( status != 0 ) {
1023  __kmp_msg(
1024  kmp_ms_fatal,
1025  KMP_MSG( CantInitThreadAttrs ),
1026  KMP_ERR( status ),
1027  __kmp_msg_null
1028  );
1029  }; // if
1030  status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1031  if ( status != 0 ) {
1032  __kmp_msg(
1033  kmp_ms_fatal,
1034  KMP_MSG( CantSetWorkerState ),
1035  KMP_ERR( status ),
1036  __kmp_msg_null
1037  );
1038  }; // if
1039 
1040  /* Set stack size for this thread now.
1041  * The multiple of 2 is there because on some machines, requesting an unusual stacksize
1042  * causes the thread to have an offset before the dummy alloca() takes place to create the
1043  * offset. Since we want the user to have a sufficient stacksize AND support a stack offset, we
1044  * alloca() twice the offset so that the upcoming alloca() does not eliminate any premade
1045  * offset, and also gives the user the stack space they requested for all threads */
1046  stack_size += gtid * __kmp_stkoffset * 2;
1047 
1048  KA_TRACE( 10, ( "__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1049  "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
1050  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size ) );
1051 
1052 # ifdef _POSIX_THREAD_ATTR_STACKSIZE
1053  status = pthread_attr_setstacksize( & thread_attr, stack_size );
1054 # ifdef KMP_BACKUP_STKSIZE
1055  if ( status != 0 ) {
1056  if ( ! __kmp_env_stksize ) {
1057  stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
1058  __kmp_stksize = KMP_BACKUP_STKSIZE;
1059  KA_TRACE( 10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1060  "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
1061  "bytes\n",
1062  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size )
1063  );
1064  status = pthread_attr_setstacksize( &thread_attr, stack_size );
1065  }; // if
1066  }; // if
1067 # endif /* KMP_BACKUP_STKSIZE */
1068  if ( status != 0 ) {
1069  __kmp_msg(
1070  kmp_ms_fatal,
1071  KMP_MSG( CantSetWorkerStackSize, stack_size ),
1072  KMP_ERR( status ),
1073  KMP_HNT( ChangeWorkerStackSize ),
1074  __kmp_msg_null
1075  );
1076  }; // if
1077 # endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1078  }
1079 #endif /* KMP_THREAD_ATTR */
1080 
1081  {
1082  status = pthread_create( & handle, & thread_attr, __kmp_launch_worker, (void *) th );
1083  if ( status != 0 || ! handle ) { // ??? Why do we check handle??
1084 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1085  if ( status == EINVAL ) {
1086  __kmp_msg(
1087  kmp_ms_fatal,
1088  KMP_MSG( CantSetWorkerStackSize, stack_size ),
1089  KMP_ERR( status ),
1090  KMP_HNT( IncreaseWorkerStackSize ),
1091  __kmp_msg_null
1092  );
1093  };
1094  if ( status == ENOMEM ) {
1095  __kmp_msg(
1096  kmp_ms_fatal,
1097  KMP_MSG( CantSetWorkerStackSize, stack_size ),
1098  KMP_ERR( status ),
1099  KMP_HNT( DecreaseWorkerStackSize ),
1100  __kmp_msg_null
1101  );
1102  };
1103 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1104  if ( status == EAGAIN ) {
1105  __kmp_msg(
1106  kmp_ms_fatal,
1107  KMP_MSG( NoResourcesForWorkerThread ),
1108  KMP_ERR( status ),
1109  KMP_HNT( Decrease_NUM_THREADS ),
1110  __kmp_msg_null
1111  );
1112  }; // if
1113  KMP_SYSFAIL( "pthread_create", status );
1114  }; // if
1115 
1116  th->th.th_info.ds.ds_thread = handle;
1117  }
1118 
1119 #ifdef KMP_THREAD_ATTR
1120  {
1121  status = pthread_attr_destroy( & thread_attr );
1122  if ( status ) {
1123  __kmp_msg(
1124  kmp_ms_warning,
1125  KMP_MSG( CantDestroyThreadAttrs ),
1126  KMP_ERR( status ),
1127  __kmp_msg_null
1128  );
1129  }; // if
1130  }
1131 #endif /* KMP_THREAD_ATTR */
1132 
1133  KMP_MB(); /* Flush all pending memory write invalidates. */
1134 
1135  KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1136 
1137 } // __kmp_create_worker
1138 
1139 
1140 void
1141 __kmp_create_monitor( kmp_info_t *th )
1142 {
1143  pthread_t handle;
1144  pthread_attr_t thread_attr;
1145  size_t size;
1146  int status;
1147  int auto_adj_size = FALSE;
1148 
1149  if( __kmp_dflt_blocktime == KMP_MAX_BLOCKTIME ) {
1150  // We don't need monitor thread in case of MAX_BLOCKTIME
1151  KA_TRACE( 10, ("__kmp_create_monitor: skipping monitor thread because of MAX blocktime\n" ) );
1152  th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
1153  th->th.th_info.ds.ds_gtid = 0;
1154  return;
1155  }
1156  KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) );
1157 
1158  KMP_MB(); /* Flush all pending memory write invalidates. */
1159 
1160  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1161  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1162  #if KMP_REAL_TIME_FIX
1163  TCW_4( __kmp_global.g.g_time.dt.t_value, -1 ); // Will use it for synchronization a bit later.
1164  #else
1165  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1166  #endif // KMP_REAL_TIME_FIX
1167 
1168  #ifdef KMP_THREAD_ATTR
1169  if ( __kmp_monitor_stksize == 0 ) {
1170  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1171  auto_adj_size = TRUE;
1172  }
1173  status = pthread_attr_init( &thread_attr );
1174  if ( status != 0 ) {
1175  __kmp_msg(
1176  kmp_ms_fatal,
1177  KMP_MSG( CantInitThreadAttrs ),
1178  KMP_ERR( status ),
1179  __kmp_msg_null
1180  );
1181  }; // if
1182  status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1183  if ( status != 0 ) {
1184  __kmp_msg(
1185  kmp_ms_fatal,
1186  KMP_MSG( CantSetMonitorState ),
1187  KMP_ERR( status ),
1188  __kmp_msg_null
1189  );
1190  }; // if
1191 
1192  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1193  status = pthread_attr_getstacksize( & thread_attr, & size );
1194  KMP_CHECK_SYSFAIL( "pthread_attr_getstacksize", status );
1195  #else
1196  size = __kmp_sys_min_stksize;
1197  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1198  #endif /* KMP_THREAD_ATTR */
1199 
1200  if ( __kmp_monitor_stksize == 0 ) {
1201  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1202  }
1203  if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1204  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1205  }
1206 
1207  KA_TRACE( 10, ( "__kmp_create_monitor: default stacksize = %lu bytes,"
1208  "requested stacksize = %lu bytes\n",
1209  size, __kmp_monitor_stksize ) );
1210 
1211  retry:
1212 
1213  /* Set stack size for this thread now. */
1214 
1215  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1216  KA_TRACE( 10, ( "__kmp_create_monitor: setting stacksize = %lu bytes,",
1217  __kmp_monitor_stksize ) );
1218  status = pthread_attr_setstacksize( & thread_attr, __kmp_monitor_stksize );
1219  if ( status != 0 ) {
1220  if ( auto_adj_size ) {
1221  __kmp_monitor_stksize *= 2;
1222  goto retry;
1223  }
1224  __kmp_msg(
1225  kmp_ms_warning, // should this be fatal? BB
1226  KMP_MSG( CantSetMonitorStackSize, (long int) __kmp_monitor_stksize ),
1227  KMP_ERR( status ),
1228  KMP_HNT( ChangeMonitorStackSize ),
1229  __kmp_msg_null
1230  );
1231  }; // if
1232  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1233 
1234  status = pthread_create( &handle, & thread_attr, __kmp_launch_monitor, (void *) th );
1235 
1236  if ( status != 0 ) {
1237  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1238  if ( status == EINVAL ) {
1239  if ( auto_adj_size && ( __kmp_monitor_stksize < (size_t)0x40000000 ) ) {
1240  __kmp_monitor_stksize *= 2;
1241  goto retry;
1242  }
1243  __kmp_msg(
1244  kmp_ms_fatal,
1245  KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1246  KMP_ERR( status ),
1247  KMP_HNT( IncreaseMonitorStackSize ),
1248  __kmp_msg_null
1249  );
1250  }; // if
1251  if ( status == ENOMEM ) {
1252  __kmp_msg(
1253  kmp_ms_fatal,
1254  KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1255  KMP_ERR( status ),
1256  KMP_HNT( DecreaseMonitorStackSize ),
1257  __kmp_msg_null
1258  );
1259  }; // if
1260  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1261  if ( status == EAGAIN ) {
1262  __kmp_msg(
1263  kmp_ms_fatal,
1264  KMP_MSG( NoResourcesForMonitorThread ),
1265  KMP_ERR( status ),
1266  KMP_HNT( DecreaseNumberOfThreadsInUse ),
1267  __kmp_msg_null
1268  );
1269  }; // if
1270  KMP_SYSFAIL( "pthread_create", status );
1271  }; // if
1272 
1273  th->th.th_info.ds.ds_thread = handle;
1274 
1275  #if KMP_REAL_TIME_FIX
1276  // Wait for the monitor thread is really started and set its *priority*.
1277  KMP_DEBUG_ASSERT( sizeof( kmp_uint32 ) == sizeof( __kmp_global.g.g_time.dt.t_value ) );
1278  __kmp_wait_yield_4(
1279  (kmp_uint32 volatile *) & __kmp_global.g.g_time.dt.t_value, -1, & __kmp_neq_4, NULL
1280  );
1281  #endif // KMP_REAL_TIME_FIX
1282 
1283  #ifdef KMP_THREAD_ATTR
1284  status = pthread_attr_destroy( & thread_attr );
1285  if ( status != 0 ) {
1286  __kmp_msg( //
1287  kmp_ms_warning,
1288  KMP_MSG( CantDestroyThreadAttrs ),
1289  KMP_ERR( status ),
1290  __kmp_msg_null
1291  );
1292  }; // if
1293  #endif
1294 
1295  KMP_MB(); /* Flush all pending memory write invalidates. */
1296 
1297  KA_TRACE( 10, ( "__kmp_create_monitor: monitor created %#.8lx\n", th->th.th_info.ds.ds_thread ) );
1298 
1299 } // __kmp_create_monitor
1300 
1301 void
1302 __kmp_exit_thread(
1303  int exit_status
1304 ) {
1305  pthread_exit( (void *)(intptr_t) exit_status );
1306 } // __kmp_exit_thread
1307 
1308 void __kmp_resume_monitor();
1309 
1310 void
1311 __kmp_reap_monitor( kmp_info_t *th )
1312 {
1313  int status;
1314  void *exit_val;
1315 
1316  KA_TRACE( 10, ("__kmp_reap_monitor: try to reap monitor thread with handle %#.8lx\n",
1317  th->th.th_info.ds.ds_thread ) );
1318 
1319  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1320  // If both tid and gtid are 0, it means the monitor did not ever start.
1321  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1322  KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1323  if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1324  KA_TRACE( 10, ("__kmp_reap_monitor: monitor did not start, returning\n") );
1325  return;
1326  }; // if
1327 
1328  KMP_MB(); /* Flush all pending memory write invalidates. */
1329 
1330 
1331  /* First, check to see whether the monitor thread exists. This could prevent a hang,
1332  but if the monitor dies after the pthread_kill call and before the pthread_join
1333  call, it will still hang. */
1334 
1335  status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1336  if (status == ESRCH) {
1337 
1338  KA_TRACE( 10, ("__kmp_reap_monitor: monitor does not exist, returning\n") );
1339 
1340  } else
1341  {
1342  __kmp_resume_monitor(); // Wake up the monitor thread
1343  status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1344  if (exit_val != th) {
1345  __kmp_msg(
1346  kmp_ms_fatal,
1347  KMP_MSG( ReapMonitorError ),
1348  KMP_ERR( status ),
1349  __kmp_msg_null
1350  );
1351  }
1352  }
1353 
1354  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1355  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1356 
1357  KA_TRACE( 10, ("__kmp_reap_monitor: done reaping monitor thread with handle %#.8lx\n",
1358  th->th.th_info.ds.ds_thread ) );
1359 
1360  KMP_MB(); /* Flush all pending memory write invalidates. */
1361 
1362 }
1363 
1364 void
1365 __kmp_reap_worker( kmp_info_t *th )
1366 {
1367  int status;
1368  void *exit_val;
1369 
1370  KMP_MB(); /* Flush all pending memory write invalidates. */
1371 
1372  KA_TRACE( 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid ) );
1373 
1374  /* First, check to see whether the worker thread exists. This could prevent a hang,
1375  but if the worker dies after the pthread_kill call and before the pthread_join
1376  call, it will still hang. */
1377 
1378  {
1379  status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1380  if (status == ESRCH) {
1381  KA_TRACE( 10, ("__kmp_reap_worker: worker T#%d does not exist, returning\n",
1382  th->th.th_info.ds.ds_gtid ) );
1383  }
1384  else {
1385  KA_TRACE( 10, ("__kmp_reap_worker: try to join with worker T#%d\n",
1386  th->th.th_info.ds.ds_gtid ) );
1387 
1388  status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1389 #ifdef KMP_DEBUG
1390  /* Don't expose these to the user until we understand when they trigger */
1391  if ( status != 0 ) {
1392  __kmp_msg(
1393  kmp_ms_fatal,
1394  KMP_MSG( ReapWorkerError ),
1395  KMP_ERR( status ),
1396  __kmp_msg_null
1397  );
1398  }
1399  if ( exit_val != th ) {
1400  KA_TRACE( 10, ( "__kmp_reap_worker: worker T#%d did not reap properly, "
1401  "exit_val = %p\n",
1402  th->th.th_info.ds.ds_gtid, exit_val ) );
1403  }
1404 #endif /* KMP_DEBUG */
1405  }
1406  }
1407 
1408  KA_TRACE( 10, ("__kmp_reap_worker: done reaping T#%d\n", th->th.th_info.ds.ds_gtid ) );
1409 
1410  KMP_MB(); /* Flush all pending memory write invalidates. */
1411 }
1412 
1413 
1414 /* ------------------------------------------------------------------------ */
1415 /* ------------------------------------------------------------------------ */
1416 
1417 #if KMP_HANDLE_SIGNALS
1418 
1419 
1420 static void
1421 __kmp_null_handler( int signo )
1422 {
1423  // Do nothing, for doing SIG_IGN-type actions.
1424 } // __kmp_null_handler
1425 
1426 
1427 static void
1428 __kmp_team_handler( int signo )
1429 {
1430  if ( __kmp_global.g.g_abort == 0 ) {
1431  /* Stage 1 signal handler, let's shut down all of the threads */
1432  #ifdef KMP_DEBUG
1433  __kmp_debug_printf( "__kmp_team_handler: caught signal = %d\n", signo );
1434  #endif
1435  switch ( signo ) {
1436  case SIGHUP :
1437  case SIGINT :
1438  case SIGQUIT :
1439  case SIGILL :
1440  case SIGABRT :
1441  case SIGFPE :
1442  case SIGBUS :
1443  case SIGSEGV :
1444  #ifdef SIGSYS
1445  case SIGSYS :
1446  #endif
1447  case SIGTERM :
1448  if ( __kmp_debug_buf ) {
1449  __kmp_dump_debug_buffer( );
1450  }; // if
1451  KMP_MB(); // Flush all pending memory write invalidates.
1452  TCW_4( __kmp_global.g.g_abort, signo );
1453  KMP_MB(); // Flush all pending memory write invalidates.
1454  TCW_4( __kmp_global.g.g_done, TRUE );
1455  KMP_MB(); // Flush all pending memory write invalidates.
1456  break;
1457  default:
1458  #ifdef KMP_DEBUG
1459  __kmp_debug_printf( "__kmp_team_handler: unknown signal type" );
1460  #endif
1461  break;
1462  }; // switch
1463  }; // if
1464 } // __kmp_team_handler
1465 
1466 
1467 static
1468 void __kmp_sigaction( int signum, const struct sigaction * act, struct sigaction * oldact ) {
1469  int rc = sigaction( signum, act, oldact );
1470  KMP_CHECK_SYSFAIL_ERRNO( "sigaction", rc );
1471 }
1472 
1473 
1474 static void
1475 __kmp_install_one_handler( int sig, sig_func_t handler_func, int parallel_init )
1476 {
1477  KMP_MB(); // Flush all pending memory write invalidates.
1478  KB_TRACE( 60, ( "__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init ) );
1479  if ( parallel_init ) {
1480  struct sigaction new_action;
1481  struct sigaction old_action;
1482  new_action.sa_handler = handler_func;
1483  new_action.sa_flags = 0;
1484  sigfillset( & new_action.sa_mask );
1485  __kmp_sigaction( sig, & new_action, & old_action );
1486  if ( old_action.sa_handler == __kmp_sighldrs[ sig ].sa_handler ) {
1487  sigaddset( & __kmp_sigset, sig );
1488  } else {
1489  // Restore/keep user's handler if one previously installed.
1490  __kmp_sigaction( sig, & old_action, NULL );
1491  }; // if
1492  } else {
1493  // Save initial/system signal handlers to see if user handlers installed.
1494  __kmp_sigaction( sig, NULL, & __kmp_sighldrs[ sig ] );
1495  }; // if
1496  KMP_MB(); // Flush all pending memory write invalidates.
1497 } // __kmp_install_one_handler
1498 
1499 
1500 static void
1501 __kmp_remove_one_handler( int sig )
1502 {
1503  KB_TRACE( 60, ( "__kmp_remove_one_handler( %d )\n", sig ) );
1504  if ( sigismember( & __kmp_sigset, sig ) ) {
1505  struct sigaction old;
1506  KMP_MB(); // Flush all pending memory write invalidates.
1507  __kmp_sigaction( sig, & __kmp_sighldrs[ sig ], & old );
1508  if ( ( old.sa_handler != __kmp_team_handler ) && ( old.sa_handler != __kmp_null_handler ) ) {
1509  // Restore the users signal handler.
1510  KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1511  __kmp_sigaction( sig, & old, NULL );
1512  }; // if
1513  sigdelset( & __kmp_sigset, sig );
1514  KMP_MB(); // Flush all pending memory write invalidates.
1515  }; // if
1516 } // __kmp_remove_one_handler
1517 
1518 
1519 void
1520 __kmp_install_signals( int parallel_init )
1521 {
1522  KB_TRACE( 10, ( "__kmp_install_signals( %d )\n", parallel_init ) );
1523  if ( __kmp_handle_signals || ! parallel_init ) {
1524  // If ! parallel_init, we do not install handlers, just save original handlers.
1525  // Let us do it even __handle_signals is 0.
1526  sigemptyset( & __kmp_sigset );
1527  __kmp_install_one_handler( SIGHUP, __kmp_team_handler, parallel_init );
1528  __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1529  __kmp_install_one_handler( SIGQUIT, __kmp_team_handler, parallel_init );
1530  __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1531  __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1532  __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1533  __kmp_install_one_handler( SIGBUS, __kmp_team_handler, parallel_init );
1534  __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1535  #ifdef SIGSYS
1536  __kmp_install_one_handler( SIGSYS, __kmp_team_handler, parallel_init );
1537  #endif // SIGSYS
1538  __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1539  #ifdef SIGPIPE
1540  __kmp_install_one_handler( SIGPIPE, __kmp_team_handler, parallel_init );
1541  #endif // SIGPIPE
1542  }; // if
1543 } // __kmp_install_signals
1544 
1545 
1546 void
1547 __kmp_remove_signals( void )
1548 {
1549  int sig;
1550  KB_TRACE( 10, ( "__kmp_remove_signals()\n" ) );
1551  for ( sig = 1; sig < NSIG; ++ sig ) {
1552  __kmp_remove_one_handler( sig );
1553  }; // for sig
1554 } // __kmp_remove_signals
1555 
1556 
1557 #endif // KMP_HANDLE_SIGNALS
1558 
1559 /* ------------------------------------------------------------------------ */
1560 /* ------------------------------------------------------------------------ */
1561 
1562 void
1563 __kmp_enable( int new_state )
1564 {
1565  #ifdef KMP_CANCEL_THREADS
1566  int status, old_state;
1567  status = pthread_setcancelstate( new_state, & old_state );
1568  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
1569  KMP_DEBUG_ASSERT( old_state == PTHREAD_CANCEL_DISABLE );
1570  #endif
1571 }
1572 
1573 void
1574 __kmp_disable( int * old_state )
1575 {
1576  #ifdef KMP_CANCEL_THREADS
1577  int status;
1578  status = pthread_setcancelstate( PTHREAD_CANCEL_DISABLE, old_state );
1579  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
1580  #endif
1581 }
1582 
1583 /* ------------------------------------------------------------------------ */
1584 /* ------------------------------------------------------------------------ */
1585 
1586 static void
1587 __kmp_atfork_prepare (void)
1588 {
1589  /* nothing to do */
1590 }
1591 
1592 static void
1593 __kmp_atfork_parent (void)
1594 {
1595  /* nothing to do */
1596 }
1597 
1598 /*
1599  Reset the library so execution in the child starts "all over again" with
1600  clean data structures in initial states. Don't worry about freeing memory
1601  allocated by parent, just abandon it to be safe.
1602 */
1603 static void
1604 __kmp_atfork_child (void)
1605 {
1606  /* TODO make sure this is done right for nested/sibling */
1607  // ATT: Memory leaks are here? TODO: Check it and fix.
1608  /* KMP_ASSERT( 0 ); */
1609 
1610  ++__kmp_fork_count;
1611 
1612  __kmp_init_runtime = FALSE;
1613  __kmp_init_monitor = 0;
1614  __kmp_init_parallel = FALSE;
1615  __kmp_init_middle = FALSE;
1616  __kmp_init_serial = FALSE;
1617  TCW_4(__kmp_init_gtid, FALSE);
1618  __kmp_init_common = FALSE;
1619 
1620  TCW_4(__kmp_init_user_locks, FALSE);
1621 #if ! KMP_USE_DYNAMIC_LOCK
1622  __kmp_user_lock_table.used = 1;
1623  __kmp_user_lock_table.allocated = 0;
1624  __kmp_user_lock_table.table = NULL;
1625  __kmp_lock_blocks = NULL;
1626 #endif
1627 
1628  __kmp_all_nth = 0;
1629  TCW_4(__kmp_nth, 0);
1630 
1631  /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate here
1632  so threadprivate doesn't use stale data */
1633  KA_TRACE( 10, ( "__kmp_atfork_child: checking cache address list %p\n",
1634  __kmp_threadpriv_cache_list ) );
1635 
1636  while ( __kmp_threadpriv_cache_list != NULL ) {
1637 
1638  if ( *__kmp_threadpriv_cache_list -> addr != NULL ) {
1639  KC_TRACE( 50, ( "__kmp_atfork_child: zeroing cache at address %p\n",
1640  &(*__kmp_threadpriv_cache_list -> addr) ) );
1641 
1642  *__kmp_threadpriv_cache_list -> addr = NULL;
1643  }
1644  __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list -> next;
1645  }
1646 
1647  __kmp_init_runtime = FALSE;
1648 
1649  /* reset statically initialized locks */
1650  __kmp_init_bootstrap_lock( &__kmp_initz_lock );
1651  __kmp_init_bootstrap_lock( &__kmp_stdio_lock );
1652  __kmp_init_bootstrap_lock( &__kmp_console_lock );
1653 
1654  /* This is necessary to make sure no stale data is left around */
1655  /* AC: customers complain that we use unsafe routines in the atfork
1656  handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1657  in dynamic_link when check the presence of shared tbbmalloc library.
1658  Suggestion is to make the library initialization lazier, similar
1659  to what done for __kmpc_begin(). */
1660  // TODO: synchronize all static initializations with regular library
1661  // startup; look at kmp_global.c and etc.
1662  //__kmp_internal_begin ();
1663 
1664 }
1665 
1666 void
1667 __kmp_register_atfork(void) {
1668  if ( __kmp_need_register_atfork ) {
1669  int status = pthread_atfork( __kmp_atfork_prepare, __kmp_atfork_parent, __kmp_atfork_child );
1670  KMP_CHECK_SYSFAIL( "pthread_atfork", status );
1671  __kmp_need_register_atfork = FALSE;
1672  }
1673 }
1674 
1675 void
1676 __kmp_suspend_initialize( void )
1677 {
1678  int status;
1679  status = pthread_mutexattr_init( &__kmp_suspend_mutex_attr );
1680  KMP_CHECK_SYSFAIL( "pthread_mutexattr_init", status );
1681  status = pthread_condattr_init( &__kmp_suspend_cond_attr );
1682  KMP_CHECK_SYSFAIL( "pthread_condattr_init", status );
1683 }
1684 
1685 static void
1686 __kmp_suspend_initialize_thread( kmp_info_t *th )
1687 {
1688  if ( th->th.th_suspend_init_count <= __kmp_fork_count ) {
1689  /* this means we haven't initialized the suspension pthread objects for this thread
1690  in this instance of the process */
1691  int status;
1692  status = pthread_cond_init( &th->th.th_suspend_cv.c_cond, &__kmp_suspend_cond_attr );
1693  KMP_CHECK_SYSFAIL( "pthread_cond_init", status );
1694  status = pthread_mutex_init( &th->th.th_suspend_mx.m_mutex, & __kmp_suspend_mutex_attr );
1695  KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
1696  *(volatile int*)&th->th.th_suspend_init_count = __kmp_fork_count + 1;
1697  };
1698 }
1699 
1700 void
1701 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
1702 {
1703  if(th->th.th_suspend_init_count > __kmp_fork_count) {
1704  /* this means we have initialize the suspension pthread objects for this thread
1705  in this instance of the process */
1706  int status;
1707 
1708  status = pthread_cond_destroy( &th->th.th_suspend_cv.c_cond );
1709  if ( status != 0 && status != EBUSY ) {
1710  KMP_SYSFAIL( "pthread_cond_destroy", status );
1711  };
1712  status = pthread_mutex_destroy( &th->th.th_suspend_mx.m_mutex );
1713  if ( status != 0 && status != EBUSY ) {
1714  KMP_SYSFAIL( "pthread_mutex_destroy", status );
1715  };
1716  --th->th.th_suspend_init_count;
1717  KMP_DEBUG_ASSERT(th->th.th_suspend_init_count == __kmp_fork_count);
1718  }
1719 }
1720 
1721 /* This routine puts the calling thread to sleep after setting the
1722  * sleep bit for the indicated flag variable to true.
1723  */
1724 template <class C>
1725 static inline void __kmp_suspend_template( int th_gtid, C *flag )
1726 {
1727  KMP_TIME_DEVELOPER_BLOCK(USER_suspend);
1728  kmp_info_t *th = __kmp_threads[th_gtid];
1729  int status;
1730  typename C::flag_t old_spin;
1731 
1732  KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, flag->get() ) );
1733 
1734  __kmp_suspend_initialize_thread( th );
1735 
1736  status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1737  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1738 
1739  KF_TRACE( 10, ( "__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1740  th_gtid, flag->get() ) );
1741 
1742  /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread
1743  gets called first?
1744  */
1745  old_spin = flag->set_sleeping();
1746 
1747  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x, was %x\n",
1748  th_gtid, flag->get(), *(flag->get()), old_spin ) );
1749 
1750  if ( flag->done_check_val(old_spin) ) {
1751  old_spin = flag->unset_sleeping();
1752  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d false alarm, reset sleep bit for spin(%p)\n",
1753  th_gtid, flag->get()) );
1754  } else {
1755  /* Encapsulate in a loop as the documentation states that this may
1756  * "with low probability" return when the condition variable has
1757  * not been signaled or broadcast
1758  */
1759  int deactivated = FALSE;
1760  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1761  while ( flag->is_sleeping() ) {
1762 #ifdef DEBUG_SUSPEND
1763  char buffer[128];
1764  __kmp_suspend_count++;
1765  __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1766  __kmp_printf( "__kmp_suspend_template: suspending T#%d: %s\n", th_gtid, buffer );
1767 #endif
1768  // Mark the thread as no longer active (only in the first iteration of the loop).
1769  if ( ! deactivated ) {
1770  th->th.th_active = FALSE;
1771  if ( th->th.th_active_in_pool ) {
1772  th->th.th_active_in_pool = FALSE;
1773  KMP_TEST_THEN_DEC32(
1774  (kmp_int32 *) &__kmp_thread_pool_active_nth );
1775  KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
1776  }
1777  deactivated = TRUE;
1778 
1779 
1780  }
1781 
1782 #if USE_SUSPEND_TIMEOUT
1783  struct timespec now;
1784  struct timeval tval;
1785  int msecs;
1786 
1787  status = gettimeofday( &tval, NULL );
1788  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
1789  TIMEVAL_TO_TIMESPEC( &tval, &now );
1790 
1791  msecs = (4*__kmp_dflt_blocktime) + 200;
1792  now.tv_sec += msecs / 1000;
1793  now.tv_nsec += (msecs % 1000)*1000;
1794 
1795  KF_TRACE( 15, ( "__kmp_suspend_template: T#%d about to perform pthread_cond_timedwait\n",
1796  th_gtid ) );
1797  status = pthread_cond_timedwait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex, & now );
1798 #else
1799  KF_TRACE( 15, ( "__kmp_suspend_template: T#%d about to perform pthread_cond_wait\n",
1800  th_gtid ) );
1801  status = pthread_cond_wait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex );
1802 #endif
1803 
1804  if ( (status != 0) && (status != EINTR) && (status != ETIMEDOUT) ) {
1805  KMP_SYSFAIL( "pthread_cond_wait", status );
1806  }
1807 #ifdef KMP_DEBUG
1808  if (status == ETIMEDOUT) {
1809  if ( flag->is_sleeping() ) {
1810  KF_TRACE( 100, ( "__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid ) );
1811  } else {
1812  KF_TRACE( 2, ( "__kmp_suspend_template: T#%d timeout wakeup, sleep bit not set!\n",
1813  th_gtid ) );
1814  }
1815  } else if ( flag->is_sleeping() ) {
1816  KF_TRACE( 100, ( "__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ) );
1817  }
1818 #endif
1819  } // while
1820 
1821  // Mark the thread as active again (if it was previous marked as inactive)
1822  if ( deactivated ) {
1823  th->th.th_active = TRUE;
1824  if ( TCR_4(th->th.th_in_pool) ) {
1825  KMP_TEST_THEN_INC32( (kmp_int32 *) &__kmp_thread_pool_active_nth );
1826  th->th.th_active_in_pool = TRUE;
1827  }
1828  }
1829  }
1830 
1831 #ifdef DEBUG_SUSPEND
1832  {
1833  char buffer[128];
1834  __kmp_print_cond( buffer, &th->th.th_suspend_cv);
1835  __kmp_printf( "__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid, buffer );
1836  }
1837 #endif
1838 
1839 
1840  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1841  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1842 
1843  KF_TRACE( 30, ("__kmp_suspend_template: T#%d exit\n", th_gtid ) );
1844 }
1845 
1846 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
1847  __kmp_suspend_template(th_gtid, flag);
1848 }
1849 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
1850  __kmp_suspend_template(th_gtid, flag);
1851 }
1852 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1853  __kmp_suspend_template(th_gtid, flag);
1854 }
1855 
1856 
1857 /* This routine signals the thread specified by target_gtid to wake up
1858  * after setting the sleep bit indicated by the flag argument to FALSE.
1859  * The target thread must already have called __kmp_suspend_template()
1860  */
1861 template <class C>
1862 static inline void __kmp_resume_template( int target_gtid, C *flag )
1863 {
1864  KMP_TIME_DEVELOPER_BLOCK(USER_resume);
1865  kmp_info_t *th = __kmp_threads[target_gtid];
1866  int status;
1867 
1868 #ifdef KMP_DEBUG
1869  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1870 #endif
1871 
1872  KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
1873  KMP_DEBUG_ASSERT( gtid != target_gtid );
1874 
1875  __kmp_suspend_initialize_thread( th );
1876 
1877  status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1878  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1879 
1880  if (!flag) { // coming from __kmp_null_resume_wrapper
1881  flag = (C *)th->th.th_sleep_loc;
1882  }
1883 
1884  // First, check if the flag is null or its type has changed. If so, someone else woke it up.
1885  if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type simply shows what flag was cast to
1886  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p)\n",
1887  gtid, target_gtid, NULL ) );
1888  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1889  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1890  return;
1891  }
1892  else { // if multiple threads are sleeping, flag should be internally referring to a specific thread here
1893  typename C::flag_t old_spin = flag->unset_sleeping();
1894  if ( ! flag->is_sleeping_val(old_spin) ) {
1895  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p): "
1896  "%u => %u\n",
1897  gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1898 
1899  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1900  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1901  return;
1902  }
1903  KF_TRACE( 5, ( "__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p): "
1904  "%u => %u\n",
1905  gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1906  }
1907  TCW_PTR(th->th.th_sleep_loc, NULL);
1908 
1909 
1910 #ifdef DEBUG_SUSPEND
1911  {
1912  char buffer[128];
1913  __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1914  __kmp_printf( "__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid, target_gtid, buffer );
1915  }
1916 #endif
1917 
1918 
1919  status = pthread_cond_signal( &th->th.th_suspend_cv.c_cond );
1920  KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
1921  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1922  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1923  KF_TRACE( 30, ( "__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
1924  gtid, target_gtid ) );
1925 }
1926 
1927 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
1928  __kmp_resume_template(target_gtid, flag);
1929 }
1930 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
1931  __kmp_resume_template(target_gtid, flag);
1932 }
1933 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1934  __kmp_resume_template(target_gtid, flag);
1935 }
1936 
1937 void
1938 __kmp_resume_monitor()
1939 {
1940  int status;
1941 #ifdef KMP_DEBUG
1942  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1943  KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n",
1944  gtid, KMP_GTID_MONITOR ) );
1945  KMP_DEBUG_ASSERT( gtid != KMP_GTID_MONITOR );
1946 #endif
1947  status = pthread_mutex_lock( &__kmp_wait_mx.m_mutex );
1948  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1949 #ifdef DEBUG_SUSPEND
1950  {
1951  char buffer[128];
1952  __kmp_print_cond( buffer, &__kmp_wait_cv.c_cond );
1953  __kmp_printf( "__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid, KMP_GTID_MONITOR, buffer );
1954  }
1955 #endif
1956  status = pthread_cond_signal( &__kmp_wait_cv.c_cond );
1957  KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
1958  status = pthread_mutex_unlock( &__kmp_wait_mx.m_mutex );
1959  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1960  KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d exiting after signaling wake up for T#%d\n",
1961  gtid, KMP_GTID_MONITOR ) );
1962 }
1963 
1964 /* ------------------------------------------------------------------------ */
1965 /* ------------------------------------------------------------------------ */
1966 
1967 void
1968 __kmp_yield( int cond )
1969 {
1970  if (cond && __kmp_yielding_on) {
1971  sched_yield();
1972  }
1973 }
1974 
1975 /* ------------------------------------------------------------------------ */
1976 /* ------------------------------------------------------------------------ */
1977 
1978 void
1979 __kmp_gtid_set_specific( int gtid )
1980 {
1981  int status;
1982  KMP_ASSERT( __kmp_init_runtime );
1983  status = pthread_setspecific( __kmp_gtid_threadprivate_key, (void*)(intptr_t)(gtid+1) );
1984  KMP_CHECK_SYSFAIL( "pthread_setspecific", status );
1985 }
1986 
1987 int
1988 __kmp_gtid_get_specific()
1989 {
1990  int gtid;
1991  if ( !__kmp_init_runtime ) {
1992  KA_TRACE( 50, ("__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
1993  return KMP_GTID_SHUTDOWN;
1994  }
1995  gtid = (int)(size_t)pthread_getspecific( __kmp_gtid_threadprivate_key );
1996  if ( gtid == 0 ) {
1997  gtid = KMP_GTID_DNE;
1998  }
1999  else {
2000  gtid--;
2001  }
2002  KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
2003  __kmp_gtid_threadprivate_key, gtid ));
2004  return gtid;
2005 }
2006 
2007 /* ------------------------------------------------------------------------ */
2008 /* ------------------------------------------------------------------------ */
2009 
2010 double
2011 __kmp_read_cpu_time( void )
2012 {
2013  /*clock_t t;*/
2014  struct tms buffer;
2015 
2016  /*t =*/ times( & buffer );
2017 
2018  return (buffer.tms_utime + buffer.tms_cutime) / (double) CLOCKS_PER_SEC;
2019 }
2020 
2021 int
2022 __kmp_read_system_info( struct kmp_sys_info *info )
2023 {
2024  int status;
2025  struct rusage r_usage;
2026 
2027  memset( info, 0, sizeof( *info ) );
2028 
2029  status = getrusage( RUSAGE_SELF, &r_usage);
2030  KMP_CHECK_SYSFAIL_ERRNO( "getrusage", status );
2031 
2032  info->maxrss = r_usage.ru_maxrss; /* the maximum resident set size utilized (in kilobytes) */
2033  info->minflt = r_usage.ru_minflt; /* the number of page faults serviced without any I/O */
2034  info->majflt = r_usage.ru_majflt; /* the number of page faults serviced that required I/O */
2035  info->nswap = r_usage.ru_nswap; /* the number of times a process was "swapped" out of memory */
2036  info->inblock = r_usage.ru_inblock; /* the number of times the file system had to perform input */
2037  info->oublock = r_usage.ru_oublock; /* the number of times the file system had to perform output */
2038  info->nvcsw = r_usage.ru_nvcsw; /* the number of times a context switch was voluntarily */
2039  info->nivcsw = r_usage.ru_nivcsw; /* the number of times a context switch was forced */
2040 
2041  return (status != 0);
2042 }
2043 
2044 /* ------------------------------------------------------------------------ */
2045 /* ------------------------------------------------------------------------ */
2046 
2047 void
2048 __kmp_read_system_time( double *delta )
2049 {
2050  double t_ns;
2051  struct timeval tval;
2052  struct timespec stop;
2053  int status;
2054 
2055  status = gettimeofday( &tval, NULL );
2056  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2057  TIMEVAL_TO_TIMESPEC( &tval, &stop );
2058  t_ns = TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start);
2059  *delta = (t_ns * 1e-9);
2060 }
2061 
2062 void
2063 __kmp_clear_system_time( void )
2064 {
2065  struct timeval tval;
2066  int status;
2067  status = gettimeofday( &tval, NULL );
2068  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2069  TIMEVAL_TO_TIMESPEC( &tval, &__kmp_sys_timer_data.start );
2070 }
2071 
2072 /* ------------------------------------------------------------------------ */
2073 /* ------------------------------------------------------------------------ */
2074 
2075 #ifdef BUILD_TV
2076 
2077 void
2078 __kmp_tv_threadprivate_store( kmp_info_t *th, void *global_addr, void *thread_addr )
2079 {
2080  struct tv_data *p;
2081 
2082  p = (struct tv_data *) __kmp_allocate( sizeof( *p ) );
2083 
2084  p->u.tp.global_addr = global_addr;
2085  p->u.tp.thread_addr = thread_addr;
2086 
2087  p->type = (void *) 1;
2088 
2089  p->next = th->th.th_local.tv_data;
2090  th->th.th_local.tv_data = p;
2091 
2092  if ( p->next == 0 ) {
2093  int rc = pthread_setspecific( __kmp_tv_key, p );
2094  KMP_CHECK_SYSFAIL( "pthread_setspecific", rc );
2095  }
2096 }
2097 
2098 #endif /* BUILD_TV */
2099 
2100 /* ------------------------------------------------------------------------ */
2101 /* ------------------------------------------------------------------------ */
2102 
2103 static int
2104 __kmp_get_xproc( void ) {
2105 
2106  int r = 0;
2107 
2108  #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD
2109 
2110  r = sysconf( _SC_NPROCESSORS_ONLN );
2111 
2112  #elif KMP_OS_DARWIN
2113 
2114  // Bug C77011 High "OpenMP Threads and number of active cores".
2115 
2116  // Find the number of available CPUs.
2117  kern_return_t rc;
2118  host_basic_info_data_t info;
2119  mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT;
2120  rc = host_info( mach_host_self(), HOST_BASIC_INFO, (host_info_t) & info, & num );
2121  if ( rc == 0 && num == HOST_BASIC_INFO_COUNT ) {
2122  // Cannot use KA_TRACE() here because this code works before trace support is
2123  // initialized.
2124  r = info.avail_cpus;
2125  } else {
2126  KMP_WARNING( CantGetNumAvailCPU );
2127  KMP_INFORM( AssumedNumCPU );
2128  }; // if
2129 
2130  #else
2131 
2132  #error "Unknown or unsupported OS."
2133 
2134  #endif
2135 
2136  return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
2137 
2138 } // __kmp_get_xproc
2139 
2140 int
2141 __kmp_read_from_file( char const *path, char const *format, ... )
2142 {
2143  int result;
2144  va_list args;
2145 
2146  va_start(args, format);
2147  FILE *f = fopen(path, "rb");
2148  if ( f == NULL )
2149  return 0;
2150  result = vfscanf(f, format, args);
2151  fclose(f);
2152 
2153  return result;
2154 }
2155 
2156 void
2157 __kmp_runtime_initialize( void )
2158 {
2159  int status;
2160  pthread_mutexattr_t mutex_attr;
2161  pthread_condattr_t cond_attr;
2162 
2163  if ( __kmp_init_runtime ) {
2164  return;
2165  }; // if
2166 
2167  #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
2168  if ( ! __kmp_cpuinfo.initialized ) {
2169  __kmp_query_cpuid( &__kmp_cpuinfo );
2170  }; // if
2171  #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
2172 
2173  __kmp_xproc = __kmp_get_xproc();
2174 
2175  if ( sysconf( _SC_THREADS ) ) {
2176 
2177  /* Query the maximum number of threads */
2178  __kmp_sys_max_nth = sysconf( _SC_THREAD_THREADS_MAX );
2179  if ( __kmp_sys_max_nth == -1 ) {
2180  /* Unlimited threads for NPTL */
2181  __kmp_sys_max_nth = INT_MAX;
2182  }
2183  else if ( __kmp_sys_max_nth <= 1 ) {
2184  /* Can't tell, just use PTHREAD_THREADS_MAX */
2185  __kmp_sys_max_nth = KMP_MAX_NTH;
2186  }
2187 
2188  /* Query the minimum stack size */
2189  __kmp_sys_min_stksize = sysconf( _SC_THREAD_STACK_MIN );
2190  if ( __kmp_sys_min_stksize <= 1 ) {
2191  __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
2192  }
2193  }
2194 
2195  /* Set up minimum number of threads to switch to TLS gtid */
2196  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
2197 
2198  #ifdef BUILD_TV
2199  {
2200  int rc = pthread_key_create( & __kmp_tv_key, 0 );
2201  KMP_CHECK_SYSFAIL( "pthread_key_create", rc );
2202  }
2203  #endif
2204 
2205  status = pthread_key_create( &__kmp_gtid_threadprivate_key, __kmp_internal_end_dest );
2206  KMP_CHECK_SYSFAIL( "pthread_key_create", status );
2207  status = pthread_mutexattr_init( & mutex_attr );
2208  KMP_CHECK_SYSFAIL( "pthread_mutexattr_init", status );
2209  status = pthread_mutex_init( & __kmp_wait_mx.m_mutex, & mutex_attr );
2210  KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
2211  status = pthread_condattr_init( & cond_attr );
2212  KMP_CHECK_SYSFAIL( "pthread_condattr_init", status );
2213  status = pthread_cond_init( & __kmp_wait_cv.c_cond, & cond_attr );
2214  KMP_CHECK_SYSFAIL( "pthread_cond_init", status );
2215 #if USE_ITT_BUILD
2216  __kmp_itt_initialize();
2217 #endif /* USE_ITT_BUILD */
2218 
2219  __kmp_init_runtime = TRUE;
2220 }
2221 
2222 void
2223 __kmp_runtime_destroy( void )
2224 {
2225  int status;
2226 
2227  if ( ! __kmp_init_runtime ) {
2228  return; // Nothing to do.
2229  };
2230 
2231 #if USE_ITT_BUILD
2232  __kmp_itt_destroy();
2233 #endif /* USE_ITT_BUILD */
2234 
2235  status = pthread_key_delete( __kmp_gtid_threadprivate_key );
2236  KMP_CHECK_SYSFAIL( "pthread_key_delete", status );
2237  #ifdef BUILD_TV
2238  status = pthread_key_delete( __kmp_tv_key );
2239  KMP_CHECK_SYSFAIL( "pthread_key_delete", status );
2240  #endif
2241 
2242  status = pthread_mutex_destroy( & __kmp_wait_mx.m_mutex );
2243  if ( status != 0 && status != EBUSY ) {
2244  KMP_SYSFAIL( "pthread_mutex_destroy", status );
2245  }
2246  status = pthread_cond_destroy( & __kmp_wait_cv.c_cond );
2247  if ( status != 0 && status != EBUSY ) {
2248  KMP_SYSFAIL( "pthread_cond_destroy", status );
2249  }
2250  #if KMP_AFFINITY_SUPPORTED
2251  __kmp_affinity_uninitialize();
2252  #endif
2253 
2254  __kmp_init_runtime = FALSE;
2255 }
2256 
2257 
2258 /* Put the thread to sleep for a time period */
2259 /* NOTE: not currently used anywhere */
2260 void
2261 __kmp_thread_sleep( int millis )
2262 {
2263  sleep( ( millis + 500 ) / 1000 );
2264 }
2265 
2266 /* Calculate the elapsed wall clock time for the user */
2267 void
2268 __kmp_elapsed( double *t )
2269 {
2270  int status;
2271 # ifdef FIX_SGI_CLOCK
2272  struct timespec ts;
2273 
2274  status = clock_gettime( CLOCK_PROCESS_CPUTIME_ID, &ts );
2275  KMP_CHECK_SYSFAIL_ERRNO( "clock_gettime", status );
2276  *t = (double) ts.tv_nsec * (1.0 / (double) KMP_NSEC_PER_SEC) +
2277  (double) ts.tv_sec;
2278 # else
2279  struct timeval tv;
2280 
2281  status = gettimeofday( & tv, NULL );
2282  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2283  *t = (double) tv.tv_usec * (1.0 / (double) KMP_USEC_PER_SEC) +
2284  (double) tv.tv_sec;
2285 # endif
2286 }
2287 
2288 /* Calculate the elapsed wall clock tick for the user */
2289 void
2290 __kmp_elapsed_tick( double *t )
2291 {
2292  *t = 1 / (double) CLOCKS_PER_SEC;
2293 }
2294 
2295 /*
2296  Determine whether the given address is mapped into the current address space.
2297 */
2298 
2299 int
2300 __kmp_is_address_mapped( void * addr ) {
2301 
2302  int found = 0;
2303  int rc;
2304 
2305  #if KMP_OS_LINUX || KMP_OS_FREEBSD
2306 
2307  /*
2308  On Linux* OS, read the /proc/<pid>/maps pseudo-file to get all the address ranges mapped
2309  into the address space.
2310  */
2311 
2312  char * name = __kmp_str_format( "/proc/%d/maps", getpid() );
2313  FILE * file = NULL;
2314 
2315  file = fopen( name, "r" );
2316  KMP_ASSERT( file != NULL );
2317 
2318  for ( ; ; ) {
2319 
2320  void * beginning = NULL;
2321  void * ending = NULL;
2322  char perms[ 5 ];
2323 
2324  rc = fscanf( file, "%p-%p %4s %*[^\n]\n", & beginning, & ending, perms );
2325  if ( rc == EOF ) {
2326  break;
2327  }; // if
2328  KMP_ASSERT( rc == 3 && KMP_STRLEN( perms ) == 4 ); // Make sure all fields are read.
2329 
2330  // Ending address is not included in the region, but beginning is.
2331  if ( ( addr >= beginning ) && ( addr < ending ) ) {
2332  perms[ 2 ] = 0; // 3th and 4th character does not matter.
2333  if ( strcmp( perms, "rw" ) == 0 ) {
2334  // Memory we are looking for should be readable and writable.
2335  found = 1;
2336  }; // if
2337  break;
2338  }; // if
2339 
2340  }; // forever
2341 
2342  // Free resources.
2343  fclose( file );
2344  KMP_INTERNAL_FREE( name );
2345 
2346  #elif KMP_OS_DARWIN
2347 
2348  /*
2349  On OS X*, /proc pseudo filesystem is not available. Try to read memory using vm
2350  interface.
2351  */
2352 
2353  int buffer;
2354  vm_size_t count;
2355  rc =
2356  vm_read_overwrite(
2357  mach_task_self(), // Task to read memory of.
2358  (vm_address_t)( addr ), // Address to read from.
2359  1, // Number of bytes to be read.
2360  (vm_address_t)( & buffer ), // Address of buffer to save read bytes in.
2361  & count // Address of var to save number of read bytes in.
2362  );
2363  if ( rc == 0 ) {
2364  // Memory successfully read.
2365  found = 1;
2366  }; // if
2367 
2368  #elif KMP_OS_FREEBSD || KMP_OS_NETBSD
2369 
2370  // FIXME(FreeBSD, NetBSD): Implement this
2371  found = 1;
2372 
2373  #else
2374 
2375  #error "Unknown or unsupported OS"
2376 
2377  #endif
2378 
2379  return found;
2380 
2381 } // __kmp_is_address_mapped
2382 
2383 #ifdef USE_LOAD_BALANCE
2384 
2385 
2386 # if KMP_OS_DARWIN
2387 
2388 // The function returns the rounded value of the system load average
2389 // during given time interval which depends on the value of
2390 // __kmp_load_balance_interval variable (default is 60 sec, other values
2391 // may be 300 sec or 900 sec).
2392 // It returns -1 in case of error.
2393 int
2394 __kmp_get_load_balance( int max )
2395 {
2396  double averages[3];
2397  int ret_avg = 0;
2398 
2399  int res = getloadavg( averages, 3 );
2400 
2401  //Check __kmp_load_balance_interval to determine which of averages to use.
2402  // getloadavg() may return the number of samples less than requested that is
2403  // less than 3.
2404  if ( __kmp_load_balance_interval < 180 && ( res >= 1 ) ) {
2405  ret_avg = averages[0];// 1 min
2406  } else if ( ( __kmp_load_balance_interval >= 180
2407  && __kmp_load_balance_interval < 600 ) && ( res >= 2 ) ) {
2408  ret_avg = averages[1];// 5 min
2409  } else if ( ( __kmp_load_balance_interval >= 600 ) && ( res == 3 ) ) {
2410  ret_avg = averages[2];// 15 min
2411  } else {// Error occurred
2412  return -1;
2413  }
2414 
2415  return ret_avg;
2416 }
2417 
2418 # else // Linux* OS
2419 
2420 // The fuction returns number of running (not sleeping) threads, or -1 in case of error.
2421 // Error could be reported if Linux* OS kernel too old (without "/proc" support).
2422 // Counting running threads stops if max running threads encountered.
2423 int
2424 __kmp_get_load_balance( int max )
2425 {
2426  static int permanent_error = 0;
2427 
2428  static int glb_running_threads = 0; /* Saved count of the running threads for the thread balance algortihm */
2429  static double glb_call_time = 0; /* Thread balance algorithm call time */
2430 
2431  int running_threads = 0; // Number of running threads in the system.
2432 
2433  DIR * proc_dir = NULL; // Handle of "/proc/" directory.
2434  struct dirent * proc_entry = NULL;
2435 
2436  kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2437  DIR * task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2438  struct dirent * task_entry = NULL;
2439  int task_path_fixed_len;
2440 
2441  kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2442  int stat_file = -1;
2443  int stat_path_fixed_len;
2444 
2445  int total_processes = 0; // Total number of processes in system.
2446  int total_threads = 0; // Total number of threads in system.
2447 
2448  double call_time = 0.0;
2449 
2450  __kmp_str_buf_init( & task_path );
2451  __kmp_str_buf_init( & stat_path );
2452 
2453  __kmp_elapsed( & call_time );
2454 
2455  if ( glb_call_time &&
2456  ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
2457  running_threads = glb_running_threads;
2458  goto finish;
2459  }
2460 
2461  glb_call_time = call_time;
2462 
2463  // Do not spend time on scanning "/proc/" if we have a permanent error.
2464  if ( permanent_error ) {
2465  running_threads = -1;
2466  goto finish;
2467  }; // if
2468 
2469  if ( max <= 0 ) {
2470  max = INT_MAX;
2471  }; // if
2472 
2473  // Open "/proc/" directory.
2474  proc_dir = opendir( "/proc" );
2475  if ( proc_dir == NULL ) {
2476  // Cannot open "/prroc/". Probably the kernel does not support it. Return an error now and
2477  // in subsequent calls.
2478  running_threads = -1;
2479  permanent_error = 1;
2480  goto finish;
2481  }; // if
2482 
2483  // Initialize fixed part of task_path. This part will not change.
2484  __kmp_str_buf_cat( & task_path, "/proc/", 6 );
2485  task_path_fixed_len = task_path.used; // Remember number of used characters.
2486 
2487  proc_entry = readdir( proc_dir );
2488  while ( proc_entry != NULL ) {
2489  // Proc entry is a directory and name starts with a digit. Assume it is a process'
2490  // directory.
2491  if ( proc_entry->d_type == DT_DIR && isdigit( proc_entry->d_name[ 0 ] ) ) {
2492 
2493  ++ total_processes;
2494  // Make sure init process is the very first in "/proc", so we can replace
2495  // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes == 1.
2496  // We are going to check that total_processes == 1 => d_name == "1" is true (where
2497  // "=>" is implication). Since C++ does not have => operator, let us replace it with its
2498  // equivalent: a => b == ! a || b.
2499  KMP_DEBUG_ASSERT( total_processes != 1 || strcmp( proc_entry->d_name, "1" ) == 0 );
2500 
2501  // Construct task_path.
2502  task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2503  __kmp_str_buf_cat( & task_path, proc_entry->d_name, KMP_STRLEN( proc_entry->d_name ) );
2504  __kmp_str_buf_cat( & task_path, "/task", 5 );
2505 
2506  task_dir = opendir( task_path.str );
2507  if ( task_dir == NULL ) {
2508  // Process can finish between reading "/proc/" directory entry and opening process'
2509  // "task/" directory. So, in general case we should not complain, but have to skip
2510  // this process and read the next one.
2511  // But on systems with no "task/" support we will spend lot of time to scan "/proc/"
2512  // tree again and again without any benefit. "init" process (its pid is 1) should
2513  // exist always, so, if we cannot open "/proc/1/task/" directory, it means "task/"
2514  // is not supported by kernel. Report an error now and in the future.
2515  if ( strcmp( proc_entry->d_name, "1" ) == 0 ) {
2516  running_threads = -1;
2517  permanent_error = 1;
2518  goto finish;
2519  }; // if
2520  } else {
2521  // Construct fixed part of stat file path.
2522  __kmp_str_buf_clear( & stat_path );
2523  __kmp_str_buf_cat( & stat_path, task_path.str, task_path.used );
2524  __kmp_str_buf_cat( & stat_path, "/", 1 );
2525  stat_path_fixed_len = stat_path.used;
2526 
2527  task_entry = readdir( task_dir );
2528  while ( task_entry != NULL ) {
2529  // It is a directory and name starts with a digit.
2530  if ( proc_entry->d_type == DT_DIR && isdigit( task_entry->d_name[ 0 ] ) ) {
2531 
2532  ++ total_threads;
2533 
2534  // Consruct complete stat file path. Easiest way would be:
2535  // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str, task_entry->d_name );
2536  // but seriae of __kmp_str_buf_cat works a bit faster.
2537  stat_path.used = stat_path_fixed_len; // Reset stat path to its fixed part.
2538  __kmp_str_buf_cat( & stat_path, task_entry->d_name, KMP_STRLEN( task_entry->d_name ) );
2539  __kmp_str_buf_cat( & stat_path, "/stat", 5 );
2540 
2541  // Note: Low-level API (open/read/close) is used. High-level API
2542  // (fopen/fclose) works ~ 30 % slower.
2543  stat_file = open( stat_path.str, O_RDONLY );
2544  if ( stat_file == -1 ) {
2545  // We cannot report an error because task (thread) can terminate just
2546  // before reading this file.
2547  } else {
2548  /*
2549  Content of "stat" file looks like:
2550 
2551  24285 (program) S ...
2552 
2553  It is a single line (if program name does not include fanny
2554  symbols). First number is a thread id, then name of executable file
2555  name in paretheses, then state of the thread. We need just thread
2556  state.
2557 
2558  Good news: Length of program name is 15 characters max. Longer
2559  names are truncated.
2560 
2561  Thus, we need rather short buffer: 15 chars for program name +
2562  2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2563 
2564  Bad news: Program name may contain special symbols like space,
2565  closing parenthesis, or even new line. This makes parsing "stat"
2566  file not 100 % reliable. In case of fanny program names parsing
2567  may fail (report incorrect thread state).
2568 
2569  Parsing "status" file looks more promissing (due to different
2570  file structure and escaping special symbols) but reading and
2571  parsing of "status" file works slower.
2572 
2573  -- ln
2574  */
2575  char buffer[ 65 ];
2576  int len;
2577  len = read( stat_file, buffer, sizeof( buffer ) - 1 );
2578  if ( len >= 0 ) {
2579  buffer[ len ] = 0;
2580  // Using scanf:
2581  // sscanf( buffer, "%*d (%*s) %c ", & state );
2582  // looks very nice, but searching for a closing parenthesis works a
2583  // bit faster.
2584  char * close_parent = strstr( buffer, ") " );
2585  if ( close_parent != NULL ) {
2586  char state = * ( close_parent + 2 );
2587  if ( state == 'R' ) {
2588  ++ running_threads;
2589  if ( running_threads >= max ) {
2590  goto finish;
2591  }; // if
2592  }; // if
2593  }; // if
2594  }; // if
2595  close( stat_file );
2596  stat_file = -1;
2597  }; // if
2598  }; // if
2599  task_entry = readdir( task_dir );
2600  }; // while
2601  closedir( task_dir );
2602  task_dir = NULL;
2603  }; // if
2604  }; // if
2605  proc_entry = readdir( proc_dir );
2606  }; // while
2607 
2608  //
2609  // There _might_ be a timing hole where the thread executing this
2610  // code get skipped in the load balance, and running_threads is 0.
2611  // Assert in the debug builds only!!!
2612  //
2613  KMP_DEBUG_ASSERT( running_threads > 0 );
2614  if ( running_threads <= 0 ) {
2615  running_threads = 1;
2616  }
2617 
2618  finish: // Clean up and exit.
2619  if ( proc_dir != NULL ) {
2620  closedir( proc_dir );
2621  }; // if
2622  __kmp_str_buf_free( & task_path );
2623  if ( task_dir != NULL ) {
2624  closedir( task_dir );
2625  }; // if
2626  __kmp_str_buf_free( & stat_path );
2627  if ( stat_file != -1 ) {
2628  close( stat_file );
2629  }; // if
2630 
2631  glb_running_threads = running_threads;
2632 
2633  return running_threads;
2634 
2635 } // __kmp_get_load_balance
2636 
2637 # endif // KMP_OS_DARWIN
2638 
2639 #endif // USE_LOAD_BALANCE
2640 
2641 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC)
2642 
2643 // we really only need the case with 1 argument, because CLANG always build
2644 // a struct of pointers to shared variables referenced in the outlined function
2645 int
2646 __kmp_invoke_microtask( microtask_t pkfn,
2647  int gtid, int tid,
2648  int argc, void *p_argv[]
2649 #if OMPT_SUPPORT
2650  , void **exit_frame_ptr
2651 #endif
2652 )
2653 {
2654 #if OMPT_SUPPORT
2655  *exit_frame_ptr = __builtin_frame_address(0);
2656 #endif
2657 
2658  switch (argc) {
2659  default:
2660  fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2661  fflush(stderr);
2662  exit(-1);
2663  case 0:
2664  (*pkfn)(&gtid, &tid);
2665  break;
2666  case 1:
2667  (*pkfn)(&gtid, &tid, p_argv[0]);
2668  break;
2669  case 2:
2670  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
2671  break;
2672  case 3:
2673  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
2674  break;
2675  case 4:
2676  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]);
2677  break;
2678  case 5:
2679  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]);
2680  break;
2681  case 6:
2682  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2683  p_argv[5]);
2684  break;
2685  case 7:
2686  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2687  p_argv[5], p_argv[6]);
2688  break;
2689  case 8:
2690  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2691  p_argv[5], p_argv[6], p_argv[7]);
2692  break;
2693  case 9:
2694  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2695  p_argv[5], p_argv[6], p_argv[7], p_argv[8]);
2696  break;
2697  case 10:
2698  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2699  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]);
2700  break;
2701  case 11:
2702  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2703  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2704  break;
2705  case 12:
2706  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2707  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2708  p_argv[11]);
2709  break;
2710  case 13:
2711  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2712  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2713  p_argv[11], p_argv[12]);
2714  break;
2715  case 14:
2716  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2717  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2718  p_argv[11], p_argv[12], p_argv[13]);
2719  break;
2720  case 15:
2721  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2722  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2723  p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2724  break;
2725  }
2726 
2727 #if OMPT_SUPPORT
2728  *exit_frame_ptr = 0;
2729 #endif
2730 
2731  return 1;
2732 }
2733 
2734 #endif
2735 
2736 // end of file //
2737 
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
Definition: kmp_stats.h:671