LLVM OpenMP* Runtime Library
z_Linux_util.c
1 /*
2  * z_Linux_util.c -- platform specific routines.
3  */
4 
5 
6 //===----------------------------------------------------------------------===//
7 //
8 // The LLVM Compiler Infrastructure
9 //
10 // This file is dual licensed under the MIT and the University of Illinois Open
11 // Source Licenses. See LICENSE.txt for details.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 
16 #include "kmp.h"
17 #include "kmp_wrapper_getpid.h"
18 #include "kmp_itt.h"
19 #include "kmp_str.h"
20 #include "kmp_i18n.h"
21 #include "kmp_lock.h"
22 #include "kmp_io.h"
23 #include "kmp_stats.h"
24 #include "kmp_wait_release.h"
25 
26 #if !KMP_OS_FREEBSD && !KMP_OS_NETBSD
27 # include <alloca.h>
28 #endif
29 #include <unistd.h>
30 #include <math.h> // HUGE_VAL.
31 #include <sys/time.h>
32 #include <sys/times.h>
33 #include <sys/resource.h>
34 #include <sys/syscall.h>
35 
36 #if KMP_OS_LINUX && !KMP_OS_CNK
37 # include <sys/sysinfo.h>
38 # if KMP_USE_FUTEX
39 // We should really include <futex.h>, but that causes compatibility problems on different
40 // Linux* OS distributions that either require that you include (or break when you try to include)
41 // <pci/types.h>.
42 // Since all we need is the two macros below (which are part of the kernel ABI, so can't change)
43 // we just define the constants here and don't include <futex.h>
44 # ifndef FUTEX_WAIT
45 # define FUTEX_WAIT 0
46 # endif
47 # ifndef FUTEX_WAKE
48 # define FUTEX_WAKE 1
49 # endif
50 # endif
51 #elif KMP_OS_DARWIN
52 # include <sys/sysctl.h>
53 # include <mach/mach.h>
54 #elif KMP_OS_FREEBSD
55 # include <pthread_np.h>
56 #endif
57 
58 #include <dirent.h>
59 #include <ctype.h>
60 #include <fcntl.h>
61 
62 /* ------------------------------------------------------------------------ */
63 /* ------------------------------------------------------------------------ */
64 
65 struct kmp_sys_timer {
66  struct timespec start;
67 };
68 
69 // Convert timespec to nanoseconds.
70 #define TS2NS(timespec) (((timespec).tv_sec * 1e9) + (timespec).tv_nsec)
71 
72 static struct kmp_sys_timer __kmp_sys_timer_data;
73 
74 #if KMP_HANDLE_SIGNALS
75  typedef void (* sig_func_t )( int );
76  STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[ NSIG ];
77  static sigset_t __kmp_sigset;
78 #endif
79 
80 static int __kmp_init_runtime = FALSE;
81 
82 static int __kmp_fork_count = 0;
83 
84 static pthread_condattr_t __kmp_suspend_cond_attr;
85 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
86 
87 static kmp_cond_align_t __kmp_wait_cv;
88 static kmp_mutex_align_t __kmp_wait_mx;
89 
90 /* ------------------------------------------------------------------------ */
91 /* ------------------------------------------------------------------------ */
92 
93 #ifdef DEBUG_SUSPEND
94 static void
95 __kmp_print_cond( char *buffer, kmp_cond_align_t *cond )
96 {
97  KMP_SNPRINTF( buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
98  cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
99  cond->c_cond.__c_waiting );
100 }
101 #endif
102 
103 /* ------------------------------------------------------------------------ */
104 /* ------------------------------------------------------------------------ */
105 
106 #if ( KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED)
107 
108 /*
109  * Affinity support
110  */
111 
112 /*
113  * On some of the older OS's that we build on, these constants aren't present
114  * in <asm/unistd.h> #included from <sys.syscall.h>. They must be the same on
115  * all systems of the same arch where they are defined, and they cannot change.
116  * stone forever.
117  */
118 
119 # if KMP_ARCH_X86 || KMP_ARCH_ARM
120 # ifndef __NR_sched_setaffinity
121 # define __NR_sched_setaffinity 241
122 # elif __NR_sched_setaffinity != 241
123 # error Wrong code for setaffinity system call.
124 # endif /* __NR_sched_setaffinity */
125 # ifndef __NR_sched_getaffinity
126 # define __NR_sched_getaffinity 242
127 # elif __NR_sched_getaffinity != 242
128 # error Wrong code for getaffinity system call.
129 # endif /* __NR_sched_getaffinity */
130 
131 # elif KMP_ARCH_AARCH64
132 # ifndef __NR_sched_setaffinity
133 # define __NR_sched_setaffinity 122
134 # elif __NR_sched_setaffinity != 122
135 # error Wrong code for setaffinity system call.
136 # endif /* __NR_sched_setaffinity */
137 # ifndef __NR_sched_getaffinity
138 # define __NR_sched_getaffinity 123
139 # elif __NR_sched_getaffinity != 123
140 # error Wrong code for getaffinity system call.
141 # endif /* __NR_sched_getaffinity */
142 
143 # elif KMP_ARCH_X86_64
144 # ifndef __NR_sched_setaffinity
145 # define __NR_sched_setaffinity 203
146 # elif __NR_sched_setaffinity != 203
147 # error Wrong code for setaffinity system call.
148 # endif /* __NR_sched_setaffinity */
149 # ifndef __NR_sched_getaffinity
150 # define __NR_sched_getaffinity 204
151 # elif __NR_sched_getaffinity != 204
152 # error Wrong code for getaffinity system call.
153 # endif /* __NR_sched_getaffinity */
154 
155 # elif KMP_ARCH_PPC64
156 # ifndef __NR_sched_setaffinity
157 # define __NR_sched_setaffinity 222
158 # elif __NR_sched_setaffinity != 222
159 # error Wrong code for setaffinity system call.
160 # endif /* __NR_sched_setaffinity */
161 # ifndef __NR_sched_getaffinity
162 # define __NR_sched_getaffinity 223
163 # elif __NR_sched_getaffinity != 223
164 # error Wrong code for getaffinity system call.
165 # endif /* __NR_sched_getaffinity */
166 
167 # elif KMP_ARCH_MIPS
168 # ifndef __NR_sched_setaffinity
169 # define __NR_sched_setaffinity 4239
170 # elif __NR_sched_setaffinity != 4239
171 # error Wrong code for setaffinity system call.
172 # endif /* __NR_sched_setaffinity */
173 # ifndef __NR_sched_getaffinity
174 # define __NR_sched_getaffinity 4240
175 # elif __NR_sched_getaffinity != 4240
176 # error Wrong code for getaffinity system call.
177 # endif /* __NR_sched_getaffinity */
178 
179 # elif KMP_ARCH_MIPS64
180 # ifndef __NR_sched_setaffinity
181 # define __NR_sched_setaffinity 5195
182 # elif __NR_sched_setaffinity != 5195
183 # error Wrong code for setaffinity system call.
184 # endif /* __NR_sched_setaffinity */
185 # ifndef __NR_sched_getaffinity
186 # define __NR_sched_getaffinity 5196
187 # elif __NR_sched_getaffinity != 5196
188 # error Wrong code for getaffinity system call.
189 # endif /* __NR_sched_getaffinity */
190 
191 # else
192 # error Unknown or unsupported architecture
193 
194 # endif /* KMP_ARCH_* */
195 
196 int
197 __kmp_set_system_affinity( kmp_affin_mask_t const *mask, int abort_on_error )
198 {
199  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
200  "Illegal set affinity operation when not capable");
201 #if KMP_USE_HWLOC
202  int retval = hwloc_set_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
203 #else
204  int retval = syscall( __NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask );
205 #endif
206  if (retval >= 0) {
207  return 0;
208  }
209  int error = errno;
210  if (abort_on_error) {
211  __kmp_msg(
212  kmp_ms_fatal,
213  KMP_MSG( FatalSysError ),
214  KMP_ERR( error ),
215  __kmp_msg_null
216  );
217  }
218  return error;
219 }
220 
221 int
222 __kmp_get_system_affinity( kmp_affin_mask_t *mask, int abort_on_error )
223 {
224  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
225  "Illegal get affinity operation when not capable");
226 
227 #if KMP_USE_HWLOC
228  int retval = hwloc_get_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
229 #else
230  int retval = syscall( __NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask );
231 #endif
232  if (retval >= 0) {
233  return 0;
234  }
235  int error = errno;
236  if (abort_on_error) {
237  __kmp_msg(
238  kmp_ms_fatal,
239  KMP_MSG( FatalSysError ),
240  KMP_ERR( error ),
241  __kmp_msg_null
242  );
243  }
244  return error;
245 }
246 
247 void
248 __kmp_affinity_bind_thread( int which )
249 {
250  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
251  "Illegal set affinity operation when not capable");
252 
253  kmp_affin_mask_t *mask;
254  KMP_CPU_ALLOC_ON_STACK(mask);
255  KMP_CPU_ZERO(mask);
256  KMP_CPU_SET(which, mask);
257  __kmp_set_system_affinity(mask, TRUE);
258  KMP_CPU_FREE_FROM_STACK(mask);
259 }
260 
261 /*
262  * Determine if we can access affinity functionality on this version of
263  * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
264  * __kmp_affin_mask_size to the appropriate value (0 means not capable).
265  */
266 void
267 __kmp_affinity_determine_capable(const char *env_var)
268 {
269  //
270  // Check and see if the OS supports thread affinity.
271  //
272 
273 # define KMP_CPU_SET_SIZE_LIMIT (1024*1024)
274 
275  int gCode;
276  int sCode;
277  kmp_affin_mask_t *buf;
278  buf = ( kmp_affin_mask_t * ) KMP_INTERNAL_MALLOC( KMP_CPU_SET_SIZE_LIMIT );
279 
280  // If Linux* OS:
281  // If the syscall fails or returns a suggestion for the size,
282  // then we don't have to search for an appropriate size.
283  gCode = syscall( __NR_sched_getaffinity, 0, KMP_CPU_SET_SIZE_LIMIT, buf );
284  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
285  "initial getaffinity call returned %d errno = %d\n",
286  gCode, errno));
287 
288  //if ((gCode < 0) && (errno == ENOSYS))
289  if (gCode < 0) {
290  //
291  // System call not supported
292  //
293  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
294  && (__kmp_affinity_type != affinity_none)
295  && (__kmp_affinity_type != affinity_default)
296  && (__kmp_affinity_type != affinity_disabled))) {
297  int error = errno;
298  __kmp_msg(
299  kmp_ms_warning,
300  KMP_MSG( GetAffSysCallNotSupported, env_var ),
301  KMP_ERR( error ),
302  __kmp_msg_null
303  );
304  }
305  KMP_AFFINITY_DISABLE();
306  KMP_INTERNAL_FREE(buf);
307  return;
308  }
309  if (gCode > 0) { // Linux* OS only
310  // The optimal situation: the OS returns the size of the buffer
311  // it expects.
312  //
313  // A verification of correct behavior is that Isetaffinity on a NULL
314  // buffer with the same size fails with errno set to EFAULT.
315  sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
316  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
317  "setaffinity for mask size %d returned %d errno = %d\n",
318  gCode, sCode, errno));
319  if (sCode < 0) {
320  if (errno == ENOSYS) {
321  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
322  && (__kmp_affinity_type != affinity_none)
323  && (__kmp_affinity_type != affinity_default)
324  && (__kmp_affinity_type != affinity_disabled))) {
325  int error = errno;
326  __kmp_msg(
327  kmp_ms_warning,
328  KMP_MSG( SetAffSysCallNotSupported, env_var ),
329  KMP_ERR( error ),
330  __kmp_msg_null
331  );
332  }
333  KMP_AFFINITY_DISABLE();
334  KMP_INTERNAL_FREE(buf);
335  }
336  if (errno == EFAULT) {
337  KMP_AFFINITY_ENABLE(gCode);
338  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
339  "affinity supported (mask size %d)\n",
340  (int)__kmp_affin_mask_size));
341  KMP_INTERNAL_FREE(buf);
342  return;
343  }
344  }
345  }
346 
347  //
348  // Call the getaffinity system call repeatedly with increasing set sizes
349  // until we succeed, or reach an upper bound on the search.
350  //
351  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
352  "searching for proper set size\n"));
353  int size;
354  for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
355  gCode = syscall( __NR_sched_getaffinity, 0, size, buf );
356  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
357  "getaffinity for mask size %d returned %d errno = %d\n", size,
358  gCode, errno));
359 
360  if (gCode < 0) {
361  if ( errno == ENOSYS )
362  {
363  //
364  // We shouldn't get here
365  //
366  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
367  "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
368  size));
369  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
370  && (__kmp_affinity_type != affinity_none)
371  && (__kmp_affinity_type != affinity_default)
372  && (__kmp_affinity_type != affinity_disabled))) {
373  int error = errno;
374  __kmp_msg(
375  kmp_ms_warning,
376  KMP_MSG( GetAffSysCallNotSupported, env_var ),
377  KMP_ERR( error ),
378  __kmp_msg_null
379  );
380  }
381  KMP_AFFINITY_DISABLE();
382  KMP_INTERNAL_FREE(buf);
383  return;
384  }
385  continue;
386  }
387 
388  sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
389  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
390  "setaffinity for mask size %d returned %d errno = %d\n",
391  gCode, sCode, errno));
392  if (sCode < 0) {
393  if (errno == ENOSYS) { // Linux* OS only
394  //
395  // We shouldn't get here
396  //
397  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
398  "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
399  size));
400  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
401  && (__kmp_affinity_type != affinity_none)
402  && (__kmp_affinity_type != affinity_default)
403  && (__kmp_affinity_type != affinity_disabled))) {
404  int error = errno;
405  __kmp_msg(
406  kmp_ms_warning,
407  KMP_MSG( SetAffSysCallNotSupported, env_var ),
408  KMP_ERR( error ),
409  __kmp_msg_null
410  );
411  }
412  KMP_AFFINITY_DISABLE();
413  KMP_INTERNAL_FREE(buf);
414  return;
415  }
416  if (errno == EFAULT) {
417  KMP_AFFINITY_ENABLE(gCode);
418  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
419  "affinity supported (mask size %d)\n",
420  (int)__kmp_affin_mask_size));
421  KMP_INTERNAL_FREE(buf);
422  return;
423  }
424  }
425  }
426  //int error = errno; // save uncaught error code
427  KMP_INTERNAL_FREE(buf);
428  // errno = error; // restore uncaught error code, will be printed at the next KMP_WARNING below
429 
430  //
431  // Affinity is not supported
432  //
433  KMP_AFFINITY_DISABLE();
434  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
435  "cannot determine mask size - affinity not supported\n"));
436  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
437  && (__kmp_affinity_type != affinity_none)
438  && (__kmp_affinity_type != affinity_default)
439  && (__kmp_affinity_type != affinity_disabled))) {
440  KMP_WARNING( AffCantGetMaskSize, env_var );
441  }
442 }
443 
444 #endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
445 
446 /* ------------------------------------------------------------------------ */
447 /* ------------------------------------------------------------------------ */
448 
449 #if KMP_USE_FUTEX
450 
451 int
452 __kmp_futex_determine_capable()
453 {
454  int loc = 0;
455  int rc = syscall( __NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0 );
456  int retval = ( rc == 0 ) || ( errno != ENOSYS );
457 
458  KA_TRACE(10, ( "__kmp_futex_determine_capable: rc = %d errno = %d\n", rc,
459  errno ) );
460  KA_TRACE(10, ( "__kmp_futex_determine_capable: futex syscall%s supported\n",
461  retval ? "" : " not" ) );
462 
463  return retval;
464 }
465 
466 #endif // KMP_USE_FUTEX
467 
468 /* ------------------------------------------------------------------------ */
469 /* ------------------------------------------------------------------------ */
470 
471 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS)
472 /*
473  * Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
474  * use compare_and_store for these routines
475  */
476 
477 kmp_int8
478 __kmp_test_then_or8( volatile kmp_int8 *p, kmp_int8 d )
479 {
480  kmp_int8 old_value, new_value;
481 
482  old_value = TCR_1( *p );
483  new_value = old_value | d;
484 
485  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
486  {
487  KMP_CPU_PAUSE();
488  old_value = TCR_1( *p );
489  new_value = old_value | d;
490  }
491  return old_value;
492 }
493 
494 kmp_int8
495 __kmp_test_then_and8( volatile kmp_int8 *p, kmp_int8 d )
496 {
497  kmp_int8 old_value, new_value;
498 
499  old_value = TCR_1( *p );
500  new_value = old_value & d;
501 
502  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
503  {
504  KMP_CPU_PAUSE();
505  old_value = TCR_1( *p );
506  new_value = old_value & d;
507  }
508  return old_value;
509 }
510 
511 kmp_int32
512 __kmp_test_then_or32( volatile kmp_int32 *p, kmp_int32 d )
513 {
514  kmp_int32 old_value, new_value;
515 
516  old_value = TCR_4( *p );
517  new_value = old_value | d;
518 
519  while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
520  {
521  KMP_CPU_PAUSE();
522  old_value = TCR_4( *p );
523  new_value = old_value | d;
524  }
525  return old_value;
526 }
527 
528 kmp_int32
529 __kmp_test_then_and32( volatile kmp_int32 *p, kmp_int32 d )
530 {
531  kmp_int32 old_value, new_value;
532 
533  old_value = TCR_4( *p );
534  new_value = old_value & d;
535 
536  while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
537  {
538  KMP_CPU_PAUSE();
539  old_value = TCR_4( *p );
540  new_value = old_value & d;
541  }
542  return old_value;
543 }
544 
545 # if KMP_ARCH_X86 || KMP_ARCH_PPC64 || (KMP_OS_LINUX && KMP_ARCH_AARCH64)
546 kmp_int8
547 __kmp_test_then_add8( volatile kmp_int8 *p, kmp_int8 d )
548 {
549  kmp_int8 old_value, new_value;
550 
551  old_value = TCR_1( *p );
552  new_value = old_value + d;
553 
554  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
555  {
556  KMP_CPU_PAUSE();
557  old_value = TCR_1( *p );
558  new_value = old_value + d;
559  }
560  return old_value;
561 }
562 
563 kmp_int64
564 __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 d )
565 {
566  kmp_int64 old_value, new_value;
567 
568  old_value = TCR_8( *p );
569  new_value = old_value + d;
570 
571  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
572  {
573  KMP_CPU_PAUSE();
574  old_value = TCR_8( *p );
575  new_value = old_value + d;
576  }
577  return old_value;
578 }
579 # endif /* KMP_ARCH_X86 || KMP_ARCH_PPC64 || (KMP_OS_LINUX && KMP_ARCH_AARCH64) */
580 
581 kmp_int64
582 __kmp_test_then_or64( volatile kmp_int64 *p, kmp_int64 d )
583 {
584  kmp_int64 old_value, new_value;
585 
586  old_value = TCR_8( *p );
587  new_value = old_value | d;
588  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
589  {
590  KMP_CPU_PAUSE();
591  old_value = TCR_8( *p );
592  new_value = old_value | d;
593  }
594  return old_value;
595 }
596 
597 kmp_int64
598 __kmp_test_then_and64( volatile kmp_int64 *p, kmp_int64 d )
599 {
600  kmp_int64 old_value, new_value;
601 
602  old_value = TCR_8( *p );
603  new_value = old_value & d;
604  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
605  {
606  KMP_CPU_PAUSE();
607  old_value = TCR_8( *p );
608  new_value = old_value & d;
609  }
610  return old_value;
611 }
612 
613 #endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
614 
615 void
616 __kmp_terminate_thread( int gtid )
617 {
618  int status;
619  kmp_info_t *th = __kmp_threads[ gtid ];
620 
621  if ( !th ) return;
622 
623  #ifdef KMP_CANCEL_THREADS
624  KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) );
625  status = pthread_cancel( th->th.th_info.ds.ds_thread );
626  if ( status != 0 && status != ESRCH ) {
627  __kmp_msg(
628  kmp_ms_fatal,
629  KMP_MSG( CantTerminateWorkerThread ),
630  KMP_ERR( status ),
631  __kmp_msg_null
632  );
633  }; // if
634  #endif
635  __kmp_yield( TRUE );
636 } //
637 
638 /* ------------------------------------------------------------------------ */
639 /* ------------------------------------------------------------------------ */
640 
641 /* ------------------------------------------------------------------------ */
642 /* ------------------------------------------------------------------------ */
643 
644 /*
645  * Set thread stack info according to values returned by
646  * pthread_getattr_np().
647  * If values are unreasonable, assume call failed and use
648  * incremental stack refinement method instead.
649  * Returns TRUE if the stack parameters could be determined exactly,
650  * FALSE if incremental refinement is necessary.
651  */
652 static kmp_int32
653 __kmp_set_stack_info( int gtid, kmp_info_t *th )
654 {
655  int stack_data;
656 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD
657  /* Linux* OS only -- no pthread_getattr_np support on OS X* */
658  pthread_attr_t attr;
659  int status;
660  size_t size = 0;
661  void * addr = 0;
662 
663  /* Always do incremental stack refinement for ubermaster threads since the initial
664  thread stack range can be reduced by sibling thread creation so pthread_attr_getstack
665  may cause thread gtid aliasing */
666  if ( ! KMP_UBER_GTID(gtid) ) {
667 
668  /* Fetch the real thread attributes */
669  status = pthread_attr_init( &attr );
670  KMP_CHECK_SYSFAIL( "pthread_attr_init", status );
671 #if KMP_OS_FREEBSD || KMP_OS_NETBSD
672  status = pthread_attr_get_np( pthread_self(), &attr );
673  KMP_CHECK_SYSFAIL( "pthread_attr_get_np", status );
674 #else
675  status = pthread_getattr_np( pthread_self(), &attr );
676  KMP_CHECK_SYSFAIL( "pthread_getattr_np", status );
677 #endif
678  status = pthread_attr_getstack( &attr, &addr, &size );
679  KMP_CHECK_SYSFAIL( "pthread_attr_getstack", status );
680  KA_TRACE( 60, ( "__kmp_set_stack_info: T#%d pthread_attr_getstack returned size: %lu, "
681  "low addr: %p\n",
682  gtid, size, addr ));
683 
684  status = pthread_attr_destroy( &attr );
685  KMP_CHECK_SYSFAIL( "pthread_attr_destroy", status );
686  }
687 
688  if ( size != 0 && addr != 0 ) { /* was stack parameter determination successful? */
689  /* Store the correct base and size */
690  TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
691  TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
692  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
693  return TRUE;
694  }
695 #endif /* KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD */
696  /* Use incremental refinement starting from initial conservative estimate */
697  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
698  TCW_PTR(th -> th.th_info.ds.ds_stackbase, &stack_data);
699  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
700  return FALSE;
701 }
702 
703 static void*
704 __kmp_launch_worker( void *thr )
705 {
706  int status, old_type, old_state;
707 #ifdef KMP_BLOCK_SIGNALS
708  sigset_t new_set, old_set;
709 #endif /* KMP_BLOCK_SIGNALS */
710  void *exit_val;
711 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD
712  void * volatile padding = 0;
713 #endif
714  int gtid;
715 
716  gtid = ((kmp_info_t*)thr) -> th.th_info.ds.ds_gtid;
717  __kmp_gtid_set_specific( gtid );
718 #ifdef KMP_TDATA_GTID
719  __kmp_gtid = gtid;
720 #endif
721 #if KMP_STATS_ENABLED
722  // set __thread local index to point to thread-specific stats
723  __kmp_stats_thread_ptr = ((kmp_info_t*)thr)->th.th_stats;
724  KMP_START_EXPLICIT_TIMER(OMP_worker_thread_life);
725  KMP_SET_THREAD_STATE(IDLE);
726  KMP_INIT_PARTITIONED_TIMERS(OMP_idle);
727 #endif
728 
729 #if USE_ITT_BUILD
730  __kmp_itt_thread_name( gtid );
731 #endif /* USE_ITT_BUILD */
732 
733 #if KMP_AFFINITY_SUPPORTED
734  __kmp_affinity_set_init_mask( gtid, FALSE );
735 #endif
736 
737 #ifdef KMP_CANCEL_THREADS
738  status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
739  KMP_CHECK_SYSFAIL( "pthread_setcanceltype", status );
740  /* josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? */
741  status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
742  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
743 #endif
744 
745 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
746  //
747  // Set the FP control regs to be a copy of
748  // the parallel initialization thread's.
749  //
750  __kmp_clear_x87_fpu_status_word();
751  __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
752  __kmp_load_mxcsr( &__kmp_init_mxcsr );
753 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
754 
755 #ifdef KMP_BLOCK_SIGNALS
756  status = sigfillset( & new_set );
757  KMP_CHECK_SYSFAIL_ERRNO( "sigfillset", status );
758  status = pthread_sigmask( SIG_BLOCK, & new_set, & old_set );
759  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
760 #endif /* KMP_BLOCK_SIGNALS */
761 
762 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD
763  if ( __kmp_stkoffset > 0 && gtid > 0 ) {
764  padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
765  }
766 #endif
767 
768  KMP_MB();
769  __kmp_set_stack_info( gtid, (kmp_info_t*)thr );
770 
771  __kmp_check_stack_overlap( (kmp_info_t*)thr );
772 
773  exit_val = __kmp_launch_thread( (kmp_info_t *) thr );
774 
775 #ifdef KMP_BLOCK_SIGNALS
776  status = pthread_sigmask( SIG_SETMASK, & old_set, NULL );
777  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
778 #endif /* KMP_BLOCK_SIGNALS */
779 
780  return exit_val;
781 }
782 
783 /* The monitor thread controls all of the threads in the complex */
784 
785 static void*
786 __kmp_launch_monitor( void *thr )
787 {
788  int status, old_type, old_state;
789 #ifdef KMP_BLOCK_SIGNALS
790  sigset_t new_set;
791 #endif /* KMP_BLOCK_SIGNALS */
792  struct timespec interval;
793  int yield_count;
794  int yield_cycles = 0;
795 
796  KMP_MB(); /* Flush all pending memory write invalidates. */
797 
798  KA_TRACE( 10, ("__kmp_launch_monitor: #1 launched\n" ) );
799 
800  /* register us as the monitor thread */
801  __kmp_gtid_set_specific( KMP_GTID_MONITOR );
802 #ifdef KMP_TDATA_GTID
803  __kmp_gtid = KMP_GTID_MONITOR;
804 #endif
805 
806  KMP_MB();
807 
808 #if USE_ITT_BUILD
809  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore monitor thread.
810 #endif /* USE_ITT_BUILD */
811 
812  __kmp_set_stack_info( ((kmp_info_t*)thr)->th.th_info.ds.ds_gtid, (kmp_info_t*)thr );
813 
814  __kmp_check_stack_overlap( (kmp_info_t*)thr );
815 
816 #ifdef KMP_CANCEL_THREADS
817  status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
818  KMP_CHECK_SYSFAIL( "pthread_setcanceltype", status );
819  /* josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? */
820  status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
821  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
822 #endif
823 
824  #if KMP_REAL_TIME_FIX
825  // This is a potential fix which allows application with real-time scheduling policy work.
826  // However, decision about the fix is not made yet, so it is disabled by default.
827  { // Are program started with real-time scheduling policy?
828  int sched = sched_getscheduler( 0 );
829  if ( sched == SCHED_FIFO || sched == SCHED_RR ) {
830  // Yes, we are a part of real-time application. Try to increase the priority of the
831  // monitor.
832  struct sched_param param;
833  int max_priority = sched_get_priority_max( sched );
834  int rc;
835  KMP_WARNING( RealTimeSchedNotSupported );
836  sched_getparam( 0, & param );
837  if ( param.sched_priority < max_priority ) {
838  param.sched_priority += 1;
839  rc = sched_setscheduler( 0, sched, & param );
840  if ( rc != 0 ) {
841  int error = errno;
842  __kmp_msg(
843  kmp_ms_warning,
844  KMP_MSG( CantChangeMonitorPriority ),
845  KMP_ERR( error ),
846  KMP_MSG( MonitorWillStarve ),
847  __kmp_msg_null
848  );
849  }; // if
850  } else {
851  // We cannot abort here, because number of CPUs may be enough for all the threads,
852  // including the monitor thread, so application could potentially work...
853  __kmp_msg(
854  kmp_ms_warning,
855  KMP_MSG( RunningAtMaxPriority ),
856  KMP_MSG( MonitorWillStarve ),
857  KMP_HNT( RunningAtMaxPriority ),
858  __kmp_msg_null
859  );
860  }; // if
861  }; // if
862  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 ); // AC: free thread that waits for monitor started
863  }
864  #endif // KMP_REAL_TIME_FIX
865 
866  KMP_MB(); /* Flush all pending memory write invalidates. */
867 
868  if ( __kmp_monitor_wakeups == 1 ) {
869  interval.tv_sec = 1;
870  interval.tv_nsec = 0;
871  } else {
872  interval.tv_sec = 0;
873  interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
874  }
875 
876  KA_TRACE( 10, ("__kmp_launch_monitor: #2 monitor\n" ) );
877 
878  if (__kmp_yield_cycle) {
879  __kmp_yielding_on = 0; /* Start out with yielding shut off */
880  yield_count = __kmp_yield_off_count;
881  } else {
882  __kmp_yielding_on = 1; /* Yielding is on permanently */
883  }
884 
885  while( ! TCR_4( __kmp_global.g.g_done ) ) {
886  struct timespec now;
887  struct timeval tval;
888 
889  /* This thread monitors the state of the system */
890 
891  KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) );
892 
893  status = gettimeofday( &tval, NULL );
894  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
895  TIMEVAL_TO_TIMESPEC( &tval, &now );
896 
897  now.tv_sec += interval.tv_sec;
898  now.tv_nsec += interval.tv_nsec;
899 
900  if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
901  now.tv_sec += 1;
902  now.tv_nsec -= KMP_NSEC_PER_SEC;
903  }
904 
905  status = pthread_mutex_lock( & __kmp_wait_mx.m_mutex );
906  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
907  // AC: the monitor should not fall asleep if g_done has been set
908  if ( !TCR_4(__kmp_global.g.g_done) ) { // check once more under mutex
909  status = pthread_cond_timedwait( &__kmp_wait_cv.c_cond, &__kmp_wait_mx.m_mutex, &now );
910  if ( status != 0 ) {
911  if ( status != ETIMEDOUT && status != EINTR ) {
912  KMP_SYSFAIL( "pthread_cond_timedwait", status );
913  };
914  };
915  };
916  status = pthread_mutex_unlock( & __kmp_wait_mx.m_mutex );
917  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
918 
919  if (__kmp_yield_cycle) {
920  yield_cycles++;
921  if ( (yield_cycles % yield_count) == 0 ) {
922  if (__kmp_yielding_on) {
923  __kmp_yielding_on = 0; /* Turn it off now */
924  yield_count = __kmp_yield_off_count;
925  } else {
926  __kmp_yielding_on = 1; /* Turn it on now */
927  yield_count = __kmp_yield_on_count;
928  }
929  yield_cycles = 0;
930  }
931  } else {
932  __kmp_yielding_on = 1;
933  }
934 
935  TCW_4( __kmp_global.g.g_time.dt.t_value,
936  TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
937 
938  KMP_MB(); /* Flush all pending memory write invalidates. */
939  }
940 
941  KA_TRACE( 10, ("__kmp_launch_monitor: #3 cleanup\n" ) );
942 
943 #ifdef KMP_BLOCK_SIGNALS
944  status = sigfillset( & new_set );
945  KMP_CHECK_SYSFAIL_ERRNO( "sigfillset", status );
946  status = pthread_sigmask( SIG_UNBLOCK, & new_set, NULL );
947  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
948 #endif /* KMP_BLOCK_SIGNALS */
949 
950  KA_TRACE( 10, ("__kmp_launch_monitor: #4 finished\n" ) );
951 
952  if( __kmp_global.g.g_abort != 0 ) {
953  /* now we need to terminate the worker threads */
954  /* the value of t_abort is the signal we caught */
955 
956  int gtid;
957 
958  KA_TRACE( 10, ("__kmp_launch_monitor: #5 terminate sig=%d\n", __kmp_global.g.g_abort ) );
959 
960  /* terminate the OpenMP worker threads */
961  /* TODO this is not valid for sibling threads!!
962  * the uber master might not be 0 anymore.. */
963  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
964  __kmp_terminate_thread( gtid );
965 
966  __kmp_cleanup();
967 
968  KA_TRACE( 10, ("__kmp_launch_monitor: #6 raise sig=%d\n", __kmp_global.g.g_abort ) );
969 
970  if (__kmp_global.g.g_abort > 0)
971  raise( __kmp_global.g.g_abort );
972 
973  }
974 
975  KA_TRACE( 10, ("__kmp_launch_monitor: #7 exit\n" ) );
976 
977  return thr;
978 }
979 
980 void
981 __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
982 {
983  pthread_t handle;
984  pthread_attr_t thread_attr;
985  int status;
986 
987 
988  th->th.th_info.ds.ds_gtid = gtid;
989 
990 #if KMP_STATS_ENABLED
991  // sets up worker thread stats
992  __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
993 
994  // th->th.th_stats is used to transfer thread specific stats-pointer to __kmp_launch_worker
995  // So when thread is created (goes into __kmp_launch_worker) it will
996  // set it's __thread local pointer to th->th.th_stats
997  th->th.th_stats = __kmp_stats_list.push_back(gtid);
998  if(KMP_UBER_GTID(gtid)) {
999  __kmp_stats_start_time = tsc_tick_count::now();
1000  __kmp_stats_thread_ptr = th->th.th_stats;
1001  __kmp_stats_init();
1002  KMP_START_EXPLICIT_TIMER(OMP_worker_thread_life);
1003  KMP_SET_THREAD_STATE(SERIAL_REGION);
1004  KMP_INIT_PARTITIONED_TIMERS(OMP_serial);
1005  }
1006  __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
1007 
1008 #endif // KMP_STATS_ENABLED
1009 
1010  if ( KMP_UBER_GTID(gtid) ) {
1011  KA_TRACE( 10, ("__kmp_create_worker: uber thread (%d)\n", gtid ) );
1012  th -> th.th_info.ds.ds_thread = pthread_self();
1013  __kmp_set_stack_info( gtid, th );
1014  __kmp_check_stack_overlap( th );
1015  return;
1016  }; // if
1017 
1018  KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1019 
1020  KMP_MB(); /* Flush all pending memory write invalidates. */
1021 
1022 #ifdef KMP_THREAD_ATTR
1023  status = pthread_attr_init( &thread_attr );
1024  if ( status != 0 ) {
1025  __kmp_msg(kmp_ms_fatal, KMP_MSG( CantInitThreadAttrs ), KMP_ERR( status ), __kmp_msg_null);
1026  }; // if
1027  status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1028  if ( status != 0 ) {
1029  __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerState ), KMP_ERR( status ), __kmp_msg_null);
1030  }; // if
1031 
1032  /* Set stack size for this thread now.
1033  * The multiple of 2 is there because on some machines, requesting an unusual stacksize
1034  * causes the thread to have an offset before the dummy alloca() takes place to create the
1035  * offset. Since we want the user to have a sufficient stacksize AND support a stack offset, we
1036  * alloca() twice the offset so that the upcoming alloca() does not eliminate any premade
1037  * offset, and also gives the user the stack space they requested for all threads */
1038  stack_size += gtid * __kmp_stkoffset * 2;
1039 
1040  KA_TRACE( 10, ( "__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1041  "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
1042  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size ) );
1043 
1044 # ifdef _POSIX_THREAD_ATTR_STACKSIZE
1045  status = pthread_attr_setstacksize( & thread_attr, stack_size );
1046 # ifdef KMP_BACKUP_STKSIZE
1047  if ( status != 0 ) {
1048  if ( ! __kmp_env_stksize ) {
1049  stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
1050  __kmp_stksize = KMP_BACKUP_STKSIZE;
1051  KA_TRACE( 10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1052  "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
1053  "bytes\n",
1054  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size )
1055  );
1056  status = pthread_attr_setstacksize( &thread_attr, stack_size );
1057  }; // if
1058  }; // if
1059 # endif /* KMP_BACKUP_STKSIZE */
1060  if ( status != 0 ) {
1061  __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerStackSize, stack_size ), KMP_ERR( status ),
1062  KMP_HNT( ChangeWorkerStackSize ), __kmp_msg_null);
1063  }; // if
1064 # endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1065 
1066 #endif /* KMP_THREAD_ATTR */
1067 
1068  status = pthread_create( & handle, & thread_attr, __kmp_launch_worker, (void *) th );
1069  if ( status != 0 || ! handle ) { // ??? Why do we check handle??
1070 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1071  if ( status == EINVAL ) {
1072  __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerStackSize, stack_size ), KMP_ERR( status ),
1073  KMP_HNT( IncreaseWorkerStackSize ), __kmp_msg_null);
1074  };
1075  if ( status == ENOMEM ) {
1076  __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerStackSize, stack_size ), KMP_ERR( status ),
1077  KMP_HNT( DecreaseWorkerStackSize ), __kmp_msg_null);
1078  };
1079 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1080  if ( status == EAGAIN ) {
1081  __kmp_msg(kmp_ms_fatal, KMP_MSG( NoResourcesForWorkerThread ), KMP_ERR( status ),
1082  KMP_HNT( Decrease_NUM_THREADS ), __kmp_msg_null);
1083  }; // if
1084  KMP_SYSFAIL( "pthread_create", status );
1085  }; // if
1086 
1087  th->th.th_info.ds.ds_thread = handle;
1088 
1089 #ifdef KMP_THREAD_ATTR
1090  status = pthread_attr_destroy( & thread_attr );
1091  if ( status ) {
1092  __kmp_msg(kmp_ms_warning, KMP_MSG( CantDestroyThreadAttrs ), KMP_ERR( status ), __kmp_msg_null);
1093  }; // if
1094 #endif /* KMP_THREAD_ATTR */
1095 
1096  KMP_MB(); /* Flush all pending memory write invalidates. */
1097 
1098  KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1099 
1100 } // __kmp_create_worker
1101 
1102 
1103 void
1104 __kmp_create_monitor( kmp_info_t *th )
1105 {
1106  pthread_t handle;
1107  pthread_attr_t thread_attr;
1108  size_t size;
1109  int status;
1110  int auto_adj_size = FALSE;
1111 
1112  if( __kmp_dflt_blocktime == KMP_MAX_BLOCKTIME ) {
1113  // We don't need monitor thread in case of MAX_BLOCKTIME
1114  KA_TRACE( 10, ("__kmp_create_monitor: skipping monitor thread because of MAX blocktime\n" ) );
1115  th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
1116  th->th.th_info.ds.ds_gtid = 0;
1117  return;
1118  }
1119  KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) );
1120 
1121  KMP_MB(); /* Flush all pending memory write invalidates. */
1122 
1123  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1124  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1125  #if KMP_REAL_TIME_FIX
1126  TCW_4( __kmp_global.g.g_time.dt.t_value, -1 ); // Will use it for synchronization a bit later.
1127  #else
1128  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1129  #endif // KMP_REAL_TIME_FIX
1130 
1131  #ifdef KMP_THREAD_ATTR
1132  if ( __kmp_monitor_stksize == 0 ) {
1133  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1134  auto_adj_size = TRUE;
1135  }
1136  status = pthread_attr_init( &thread_attr );
1137  if ( status != 0 ) {
1138  __kmp_msg(
1139  kmp_ms_fatal,
1140  KMP_MSG( CantInitThreadAttrs ),
1141  KMP_ERR( status ),
1142  __kmp_msg_null
1143  );
1144  }; // if
1145  status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1146  if ( status != 0 ) {
1147  __kmp_msg(
1148  kmp_ms_fatal,
1149  KMP_MSG( CantSetMonitorState ),
1150  KMP_ERR( status ),
1151  __kmp_msg_null
1152  );
1153  }; // if
1154 
1155  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1156  status = pthread_attr_getstacksize( & thread_attr, & size );
1157  KMP_CHECK_SYSFAIL( "pthread_attr_getstacksize", status );
1158  #else
1159  size = __kmp_sys_min_stksize;
1160  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1161  #endif /* KMP_THREAD_ATTR */
1162 
1163  if ( __kmp_monitor_stksize == 0 ) {
1164  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1165  }
1166  if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1167  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1168  }
1169 
1170  KA_TRACE( 10, ( "__kmp_create_monitor: default stacksize = %lu bytes,"
1171  "requested stacksize = %lu bytes\n",
1172  size, __kmp_monitor_stksize ) );
1173 
1174  retry:
1175 
1176  /* Set stack size for this thread now. */
1177 
1178  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1179  KA_TRACE( 10, ( "__kmp_create_monitor: setting stacksize = %lu bytes,",
1180  __kmp_monitor_stksize ) );
1181  status = pthread_attr_setstacksize( & thread_attr, __kmp_monitor_stksize );
1182  if ( status != 0 ) {
1183  if ( auto_adj_size ) {
1184  __kmp_monitor_stksize *= 2;
1185  goto retry;
1186  }
1187  __kmp_msg(
1188  kmp_ms_warning, // should this be fatal? BB
1189  KMP_MSG( CantSetMonitorStackSize, (long int) __kmp_monitor_stksize ),
1190  KMP_ERR( status ),
1191  KMP_HNT( ChangeMonitorStackSize ),
1192  __kmp_msg_null
1193  );
1194  }; // if
1195  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1196 
1197  status = pthread_create( &handle, & thread_attr, __kmp_launch_monitor, (void *) th );
1198 
1199  if ( status != 0 ) {
1200  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1201  if ( status == EINVAL ) {
1202  if ( auto_adj_size && ( __kmp_monitor_stksize < (size_t)0x40000000 ) ) {
1203  __kmp_monitor_stksize *= 2;
1204  goto retry;
1205  }
1206  __kmp_msg(
1207  kmp_ms_fatal,
1208  KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1209  KMP_ERR( status ),
1210  KMP_HNT( IncreaseMonitorStackSize ),
1211  __kmp_msg_null
1212  );
1213  }; // if
1214  if ( status == ENOMEM ) {
1215  __kmp_msg(
1216  kmp_ms_fatal,
1217  KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1218  KMP_ERR( status ),
1219  KMP_HNT( DecreaseMonitorStackSize ),
1220  __kmp_msg_null
1221  );
1222  }; // if
1223  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1224  if ( status == EAGAIN ) {
1225  __kmp_msg(
1226  kmp_ms_fatal,
1227  KMP_MSG( NoResourcesForMonitorThread ),
1228  KMP_ERR( status ),
1229  KMP_HNT( DecreaseNumberOfThreadsInUse ),
1230  __kmp_msg_null
1231  );
1232  }; // if
1233  KMP_SYSFAIL( "pthread_create", status );
1234  }; // if
1235 
1236  th->th.th_info.ds.ds_thread = handle;
1237 
1238  #if KMP_REAL_TIME_FIX
1239  // Wait for the monitor thread is really started and set its *priority*.
1240  KMP_DEBUG_ASSERT( sizeof( kmp_uint32 ) == sizeof( __kmp_global.g.g_time.dt.t_value ) );
1241  __kmp_wait_yield_4(
1242  (kmp_uint32 volatile *) & __kmp_global.g.g_time.dt.t_value, -1, & __kmp_neq_4, NULL
1243  );
1244  #endif // KMP_REAL_TIME_FIX
1245 
1246  #ifdef KMP_THREAD_ATTR
1247  status = pthread_attr_destroy( & thread_attr );
1248  if ( status != 0 ) {
1249  __kmp_msg( //
1250  kmp_ms_warning,
1251  KMP_MSG( CantDestroyThreadAttrs ),
1252  KMP_ERR( status ),
1253  __kmp_msg_null
1254  );
1255  }; // if
1256  #endif
1257 
1258  KMP_MB(); /* Flush all pending memory write invalidates. */
1259 
1260  KA_TRACE( 10, ( "__kmp_create_monitor: monitor created %#.8lx\n", th->th.th_info.ds.ds_thread ) );
1261 
1262 } // __kmp_create_monitor
1263 
1264 void
1265 __kmp_exit_thread(
1266  int exit_status
1267 ) {
1268  pthread_exit( (void *)(intptr_t) exit_status );
1269 } // __kmp_exit_thread
1270 
1271 void __kmp_resume_monitor();
1272 
1273 void
1274 __kmp_reap_monitor( kmp_info_t *th )
1275 {
1276  int status;
1277  void *exit_val;
1278 
1279  KA_TRACE( 10, ("__kmp_reap_monitor: try to reap monitor thread with handle %#.8lx\n",
1280  th->th.th_info.ds.ds_thread ) );
1281 
1282  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1283  // If both tid and gtid are 0, it means the monitor did not ever start.
1284  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1285  KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1286  if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1287  KA_TRACE( 10, ("__kmp_reap_monitor: monitor did not start, returning\n") );
1288  return;
1289  }; // if
1290 
1291  KMP_MB(); /* Flush all pending memory write invalidates. */
1292 
1293 
1294  /* First, check to see whether the monitor thread exists to wake it up. This is
1295  to avoid performance problem when the monitor sleeps during blocktime-size
1296  interval */
1297 
1298  status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1299  if (status != ESRCH) {
1300  __kmp_resume_monitor(); // Wake up the monitor thread
1301  }
1302  KA_TRACE( 10, ("__kmp_reap_monitor: try to join with monitor\n") );
1303  status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1304  if (exit_val != th) {
1305  __kmp_msg(
1306  kmp_ms_fatal,
1307  KMP_MSG( ReapMonitorError ),
1308  KMP_ERR( status ),
1309  __kmp_msg_null
1310  );
1311  }
1312 
1313  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1314  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1315 
1316  KA_TRACE( 10, ("__kmp_reap_monitor: done reaping monitor thread with handle %#.8lx\n",
1317  th->th.th_info.ds.ds_thread ) );
1318 
1319  KMP_MB(); /* Flush all pending memory write invalidates. */
1320 
1321 }
1322 
1323 void
1324 __kmp_reap_worker( kmp_info_t *th )
1325 {
1326  int status;
1327  void *exit_val;
1328 
1329  KMP_MB(); /* Flush all pending memory write invalidates. */
1330 
1331  KA_TRACE( 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid ) );
1332 
1333  status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1334 #ifdef KMP_DEBUG
1335  /* Don't expose these to the user until we understand when they trigger */
1336  if ( status != 0 ) {
1337  __kmp_msg(kmp_ms_fatal, KMP_MSG( ReapWorkerError ), KMP_ERR( status ), __kmp_msg_null);
1338  }
1339  if ( exit_val != th ) {
1340  KA_TRACE( 10, ( "__kmp_reap_worker: worker T#%d did not reap properly, exit_val = %p\n",
1341  th->th.th_info.ds.ds_gtid, exit_val ) );
1342  }
1343 #endif /* KMP_DEBUG */
1344 
1345  KA_TRACE( 10, ("__kmp_reap_worker: done reaping T#%d\n", th->th.th_info.ds.ds_gtid ) );
1346 
1347  KMP_MB(); /* Flush all pending memory write invalidates. */
1348 }
1349 
1350 
1351 /* ------------------------------------------------------------------------ */
1352 /* ------------------------------------------------------------------------ */
1353 
1354 #if KMP_HANDLE_SIGNALS
1355 
1356 
1357 static void
1358 __kmp_null_handler( int signo )
1359 {
1360  // Do nothing, for doing SIG_IGN-type actions.
1361 } // __kmp_null_handler
1362 
1363 
1364 static void
1365 __kmp_team_handler( int signo )
1366 {
1367  if ( __kmp_global.g.g_abort == 0 ) {
1368  /* Stage 1 signal handler, let's shut down all of the threads */
1369  #ifdef KMP_DEBUG
1370  __kmp_debug_printf( "__kmp_team_handler: caught signal = %d\n", signo );
1371  #endif
1372  switch ( signo ) {
1373  case SIGHUP :
1374  case SIGINT :
1375  case SIGQUIT :
1376  case SIGILL :
1377  case SIGABRT :
1378  case SIGFPE :
1379  case SIGBUS :
1380  case SIGSEGV :
1381  #ifdef SIGSYS
1382  case SIGSYS :
1383  #endif
1384  case SIGTERM :
1385  if ( __kmp_debug_buf ) {
1386  __kmp_dump_debug_buffer( );
1387  }; // if
1388  KMP_MB(); // Flush all pending memory write invalidates.
1389  TCW_4( __kmp_global.g.g_abort, signo );
1390  KMP_MB(); // Flush all pending memory write invalidates.
1391  TCW_4( __kmp_global.g.g_done, TRUE );
1392  KMP_MB(); // Flush all pending memory write invalidates.
1393  break;
1394  default:
1395  #ifdef KMP_DEBUG
1396  __kmp_debug_printf( "__kmp_team_handler: unknown signal type" );
1397  #endif
1398  break;
1399  }; // switch
1400  }; // if
1401 } // __kmp_team_handler
1402 
1403 
1404 static
1405 void __kmp_sigaction( int signum, const struct sigaction * act, struct sigaction * oldact ) {
1406  int rc = sigaction( signum, act, oldact );
1407  KMP_CHECK_SYSFAIL_ERRNO( "sigaction", rc );
1408 }
1409 
1410 
1411 static void
1412 __kmp_install_one_handler( int sig, sig_func_t handler_func, int parallel_init )
1413 {
1414  KMP_MB(); // Flush all pending memory write invalidates.
1415  KB_TRACE( 60, ( "__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init ) );
1416  if ( parallel_init ) {
1417  struct sigaction new_action;
1418  struct sigaction old_action;
1419  new_action.sa_handler = handler_func;
1420  new_action.sa_flags = 0;
1421  sigfillset( & new_action.sa_mask );
1422  __kmp_sigaction( sig, & new_action, & old_action );
1423  if ( old_action.sa_handler == __kmp_sighldrs[ sig ].sa_handler ) {
1424  sigaddset( & __kmp_sigset, sig );
1425  } else {
1426  // Restore/keep user's handler if one previously installed.
1427  __kmp_sigaction( sig, & old_action, NULL );
1428  }; // if
1429  } else {
1430  // Save initial/system signal handlers to see if user handlers installed.
1431  __kmp_sigaction( sig, NULL, & __kmp_sighldrs[ sig ] );
1432  }; // if
1433  KMP_MB(); // Flush all pending memory write invalidates.
1434 } // __kmp_install_one_handler
1435 
1436 
1437 static void
1438 __kmp_remove_one_handler( int sig )
1439 {
1440  KB_TRACE( 60, ( "__kmp_remove_one_handler( %d )\n", sig ) );
1441  if ( sigismember( & __kmp_sigset, sig ) ) {
1442  struct sigaction old;
1443  KMP_MB(); // Flush all pending memory write invalidates.
1444  __kmp_sigaction( sig, & __kmp_sighldrs[ sig ], & old );
1445  if ( ( old.sa_handler != __kmp_team_handler ) && ( old.sa_handler != __kmp_null_handler ) ) {
1446  // Restore the users signal handler.
1447  KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1448  __kmp_sigaction( sig, & old, NULL );
1449  }; // if
1450  sigdelset( & __kmp_sigset, sig );
1451  KMP_MB(); // Flush all pending memory write invalidates.
1452  }; // if
1453 } // __kmp_remove_one_handler
1454 
1455 
1456 void
1457 __kmp_install_signals( int parallel_init )
1458 {
1459  KB_TRACE( 10, ( "__kmp_install_signals( %d )\n", parallel_init ) );
1460  if ( __kmp_handle_signals || ! parallel_init ) {
1461  // If ! parallel_init, we do not install handlers, just save original handlers.
1462  // Let us do it even __handle_signals is 0.
1463  sigemptyset( & __kmp_sigset );
1464  __kmp_install_one_handler( SIGHUP, __kmp_team_handler, parallel_init );
1465  __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1466  __kmp_install_one_handler( SIGQUIT, __kmp_team_handler, parallel_init );
1467  __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1468  __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1469  __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1470  __kmp_install_one_handler( SIGBUS, __kmp_team_handler, parallel_init );
1471  __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1472  #ifdef SIGSYS
1473  __kmp_install_one_handler( SIGSYS, __kmp_team_handler, parallel_init );
1474  #endif // SIGSYS
1475  __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1476  #ifdef SIGPIPE
1477  __kmp_install_one_handler( SIGPIPE, __kmp_team_handler, parallel_init );
1478  #endif // SIGPIPE
1479  }; // if
1480 } // __kmp_install_signals
1481 
1482 
1483 void
1484 __kmp_remove_signals( void )
1485 {
1486  int sig;
1487  KB_TRACE( 10, ( "__kmp_remove_signals()\n" ) );
1488  for ( sig = 1; sig < NSIG; ++ sig ) {
1489  __kmp_remove_one_handler( sig );
1490  }; // for sig
1491 } // __kmp_remove_signals
1492 
1493 
1494 #endif // KMP_HANDLE_SIGNALS
1495 
1496 /* ------------------------------------------------------------------------ */
1497 /* ------------------------------------------------------------------------ */
1498 
1499 void
1500 __kmp_enable( int new_state )
1501 {
1502  #ifdef KMP_CANCEL_THREADS
1503  int status, old_state;
1504  status = pthread_setcancelstate( new_state, & old_state );
1505  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
1506  KMP_DEBUG_ASSERT( old_state == PTHREAD_CANCEL_DISABLE );
1507  #endif
1508 }
1509 
1510 void
1511 __kmp_disable( int * old_state )
1512 {
1513  #ifdef KMP_CANCEL_THREADS
1514  int status;
1515  status = pthread_setcancelstate( PTHREAD_CANCEL_DISABLE, old_state );
1516  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
1517  #endif
1518 }
1519 
1520 /* ------------------------------------------------------------------------ */
1521 /* ------------------------------------------------------------------------ */
1522 
1523 static void
1524 __kmp_atfork_prepare (void)
1525 {
1526  /* nothing to do */
1527 }
1528 
1529 static void
1530 __kmp_atfork_parent (void)
1531 {
1532  /* nothing to do */
1533 }
1534 
1535 /*
1536  Reset the library so execution in the child starts "all over again" with
1537  clean data structures in initial states. Don't worry about freeing memory
1538  allocated by parent, just abandon it to be safe.
1539 */
1540 static void
1541 __kmp_atfork_child (void)
1542 {
1543  /* TODO make sure this is done right for nested/sibling */
1544  // ATT: Memory leaks are here? TODO: Check it and fix.
1545  /* KMP_ASSERT( 0 ); */
1546 
1547  ++__kmp_fork_count;
1548 
1549  __kmp_init_runtime = FALSE;
1550  __kmp_init_monitor = 0;
1551  __kmp_init_parallel = FALSE;
1552  __kmp_init_middle = FALSE;
1553  __kmp_init_serial = FALSE;
1554  TCW_4(__kmp_init_gtid, FALSE);
1555  __kmp_init_common = FALSE;
1556 
1557  TCW_4(__kmp_init_user_locks, FALSE);
1558 #if ! KMP_USE_DYNAMIC_LOCK
1559  __kmp_user_lock_table.used = 1;
1560  __kmp_user_lock_table.allocated = 0;
1561  __kmp_user_lock_table.table = NULL;
1562  __kmp_lock_blocks = NULL;
1563 #endif
1564 
1565  __kmp_all_nth = 0;
1566  TCW_4(__kmp_nth, 0);
1567 
1568  /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate here
1569  so threadprivate doesn't use stale data */
1570  KA_TRACE( 10, ( "__kmp_atfork_child: checking cache address list %p\n",
1571  __kmp_threadpriv_cache_list ) );
1572 
1573  while ( __kmp_threadpriv_cache_list != NULL ) {
1574 
1575  if ( *__kmp_threadpriv_cache_list -> addr != NULL ) {
1576  KC_TRACE( 50, ( "__kmp_atfork_child: zeroing cache at address %p\n",
1577  &(*__kmp_threadpriv_cache_list -> addr) ) );
1578 
1579  *__kmp_threadpriv_cache_list -> addr = NULL;
1580  }
1581  __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list -> next;
1582  }
1583 
1584  __kmp_init_runtime = FALSE;
1585 
1586  /* reset statically initialized locks */
1587  __kmp_init_bootstrap_lock( &__kmp_initz_lock );
1588  __kmp_init_bootstrap_lock( &__kmp_stdio_lock );
1589  __kmp_init_bootstrap_lock( &__kmp_console_lock );
1590 
1591  /* This is necessary to make sure no stale data is left around */
1592  /* AC: customers complain that we use unsafe routines in the atfork
1593  handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1594  in dynamic_link when check the presence of shared tbbmalloc library.
1595  Suggestion is to make the library initialization lazier, similar
1596  to what done for __kmpc_begin(). */
1597  // TODO: synchronize all static initializations with regular library
1598  // startup; look at kmp_global.c and etc.
1599  //__kmp_internal_begin ();
1600 
1601 }
1602 
1603 void
1604 __kmp_register_atfork(void) {
1605  if ( __kmp_need_register_atfork ) {
1606  int status = pthread_atfork( __kmp_atfork_prepare, __kmp_atfork_parent, __kmp_atfork_child );
1607  KMP_CHECK_SYSFAIL( "pthread_atfork", status );
1608  __kmp_need_register_atfork = FALSE;
1609  }
1610 }
1611 
1612 void
1613 __kmp_suspend_initialize( void )
1614 {
1615  int status;
1616  status = pthread_mutexattr_init( &__kmp_suspend_mutex_attr );
1617  KMP_CHECK_SYSFAIL( "pthread_mutexattr_init", status );
1618  status = pthread_condattr_init( &__kmp_suspend_cond_attr );
1619  KMP_CHECK_SYSFAIL( "pthread_condattr_init", status );
1620 }
1621 
1622 static void
1623 __kmp_suspend_initialize_thread( kmp_info_t *th )
1624 {
1625  if ( th->th.th_suspend_init_count <= __kmp_fork_count ) {
1626  /* this means we haven't initialized the suspension pthread objects for this thread
1627  in this instance of the process */
1628  int status;
1629  status = pthread_cond_init( &th->th.th_suspend_cv.c_cond, &__kmp_suspend_cond_attr );
1630  KMP_CHECK_SYSFAIL( "pthread_cond_init", status );
1631  status = pthread_mutex_init( &th->th.th_suspend_mx.m_mutex, & __kmp_suspend_mutex_attr );
1632  KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
1633  *(volatile int*)&th->th.th_suspend_init_count = __kmp_fork_count + 1;
1634  };
1635 }
1636 
1637 void
1638 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
1639 {
1640  if(th->th.th_suspend_init_count > __kmp_fork_count) {
1641  /* this means we have initialize the suspension pthread objects for this thread
1642  in this instance of the process */
1643  int status;
1644 
1645  status = pthread_cond_destroy( &th->th.th_suspend_cv.c_cond );
1646  if ( status != 0 && status != EBUSY ) {
1647  KMP_SYSFAIL( "pthread_cond_destroy", status );
1648  };
1649  status = pthread_mutex_destroy( &th->th.th_suspend_mx.m_mutex );
1650  if ( status != 0 && status != EBUSY ) {
1651  KMP_SYSFAIL( "pthread_mutex_destroy", status );
1652  };
1653  --th->th.th_suspend_init_count;
1654  KMP_DEBUG_ASSERT(th->th.th_suspend_init_count == __kmp_fork_count);
1655  }
1656 }
1657 
1658 /* This routine puts the calling thread to sleep after setting the
1659  * sleep bit for the indicated flag variable to true.
1660  */
1661 template <class C>
1662 static inline void __kmp_suspend_template( int th_gtid, C *flag )
1663 {
1664  KMP_TIME_DEVELOPER_BLOCK(USER_suspend);
1665  kmp_info_t *th = __kmp_threads[th_gtid];
1666  int status;
1667  typename C::flag_t old_spin;
1668 
1669  KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, flag->get() ) );
1670 
1671  __kmp_suspend_initialize_thread( th );
1672 
1673  status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1674  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1675 
1676  KF_TRACE( 10, ( "__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1677  th_gtid, flag->get() ) );
1678 
1679  /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread
1680  gets called first?
1681  */
1682  old_spin = flag->set_sleeping();
1683 
1684  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x, was %x\n",
1685  th_gtid, flag->get(), *(flag->get()), old_spin ) );
1686 
1687  if ( flag->done_check_val(old_spin) ) {
1688  old_spin = flag->unset_sleeping();
1689  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d false alarm, reset sleep bit for spin(%p)\n",
1690  th_gtid, flag->get()) );
1691  } else {
1692  /* Encapsulate in a loop as the documentation states that this may
1693  * "with low probability" return when the condition variable has
1694  * not been signaled or broadcast
1695  */
1696  int deactivated = FALSE;
1697  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1698  while ( flag->is_sleeping() ) {
1699 #ifdef DEBUG_SUSPEND
1700  char buffer[128];
1701  __kmp_suspend_count++;
1702  __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1703  __kmp_printf( "__kmp_suspend_template: suspending T#%d: %s\n", th_gtid, buffer );
1704 #endif
1705  // Mark the thread as no longer active (only in the first iteration of the loop).
1706  if ( ! deactivated ) {
1707  th->th.th_active = FALSE;
1708  if ( th->th.th_active_in_pool ) {
1709  th->th.th_active_in_pool = FALSE;
1710  KMP_TEST_THEN_DEC32(
1711  (kmp_int32 *) &__kmp_thread_pool_active_nth );
1712  KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
1713  }
1714  deactivated = TRUE;
1715  }
1716 
1717 #if USE_SUSPEND_TIMEOUT
1718  struct timespec now;
1719  struct timeval tval;
1720  int msecs;
1721 
1722  status = gettimeofday( &tval, NULL );
1723  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
1724  TIMEVAL_TO_TIMESPEC( &tval, &now );
1725 
1726  msecs = (4*__kmp_dflt_blocktime) + 200;
1727  now.tv_sec += msecs / 1000;
1728  now.tv_nsec += (msecs % 1000)*1000;
1729 
1730  KF_TRACE( 15, ( "__kmp_suspend_template: T#%d about to perform pthread_cond_timedwait\n",
1731  th_gtid ) );
1732  status = pthread_cond_timedwait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex, & now );
1733 #else
1734  KF_TRACE( 15, ( "__kmp_suspend_template: T#%d about to perform pthread_cond_wait\n",
1735  th_gtid ) );
1736  status = pthread_cond_wait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex );
1737 #endif
1738 
1739  if ( (status != 0) && (status != EINTR) && (status != ETIMEDOUT) ) {
1740  KMP_SYSFAIL( "pthread_cond_wait", status );
1741  }
1742 #ifdef KMP_DEBUG
1743  if (status == ETIMEDOUT) {
1744  if ( flag->is_sleeping() ) {
1745  KF_TRACE( 100, ( "__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid ) );
1746  } else {
1747  KF_TRACE( 2, ( "__kmp_suspend_template: T#%d timeout wakeup, sleep bit not set!\n",
1748  th_gtid ) );
1749  }
1750  } else if ( flag->is_sleeping() ) {
1751  KF_TRACE( 100, ( "__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ) );
1752  }
1753 #endif
1754  } // while
1755 
1756  // Mark the thread as active again (if it was previous marked as inactive)
1757  if ( deactivated ) {
1758  th->th.th_active = TRUE;
1759  if ( TCR_4(th->th.th_in_pool) ) {
1760  KMP_TEST_THEN_INC32( (kmp_int32 *) &__kmp_thread_pool_active_nth );
1761  th->th.th_active_in_pool = TRUE;
1762  }
1763  }
1764  }
1765 
1766 #ifdef DEBUG_SUSPEND
1767  {
1768  char buffer[128];
1769  __kmp_print_cond( buffer, &th->th.th_suspend_cv);
1770  __kmp_printf( "__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid, buffer );
1771  }
1772 #endif
1773 
1774  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1775  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1776 
1777  KF_TRACE( 30, ("__kmp_suspend_template: T#%d exit\n", th_gtid ) );
1778 }
1779 
1780 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
1781  __kmp_suspend_template(th_gtid, flag);
1782 }
1783 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
1784  __kmp_suspend_template(th_gtid, flag);
1785 }
1786 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1787  __kmp_suspend_template(th_gtid, flag);
1788 }
1789 
1790 
1791 /* This routine signals the thread specified by target_gtid to wake up
1792  * after setting the sleep bit indicated by the flag argument to FALSE.
1793  * The target thread must already have called __kmp_suspend_template()
1794  */
1795 template <class C>
1796 static inline void __kmp_resume_template( int target_gtid, C *flag )
1797 {
1798  KMP_TIME_DEVELOPER_BLOCK(USER_resume);
1799  kmp_info_t *th = __kmp_threads[target_gtid];
1800  int status;
1801 
1802 #ifdef KMP_DEBUG
1803  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1804 #endif
1805 
1806  KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
1807  KMP_DEBUG_ASSERT( gtid != target_gtid );
1808 
1809  __kmp_suspend_initialize_thread( th );
1810 
1811  status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1812  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1813 
1814  if (!flag) { // coming from __kmp_null_resume_wrapper
1815  flag = (C *)th->th.th_sleep_loc;
1816  }
1817 
1818  // First, check if the flag is null or its type has changed. If so, someone else woke it up.
1819  if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type simply shows what flag was cast to
1820  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p)\n",
1821  gtid, target_gtid, NULL ) );
1822  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1823  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1824  return;
1825  }
1826  else { // if multiple threads are sleeping, flag should be internally referring to a specific thread here
1827  typename C::flag_t old_spin = flag->unset_sleeping();
1828  if ( ! flag->is_sleeping_val(old_spin) ) {
1829  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p): "
1830  "%u => %u\n",
1831  gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1832  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1833  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1834  return;
1835  }
1836  KF_TRACE( 5, ( "__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p): "
1837  "%u => %u\n",
1838  gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1839  }
1840  TCW_PTR(th->th.th_sleep_loc, NULL);
1841 
1842 
1843 #ifdef DEBUG_SUSPEND
1844  {
1845  char buffer[128];
1846  __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1847  __kmp_printf( "__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid, target_gtid, buffer );
1848  }
1849 #endif
1850 
1851  status = pthread_cond_signal( &th->th.th_suspend_cv.c_cond );
1852  KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
1853  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1854  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1855  KF_TRACE( 30, ( "__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
1856  gtid, target_gtid ) );
1857 }
1858 
1859 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
1860  __kmp_resume_template(target_gtid, flag);
1861 }
1862 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
1863  __kmp_resume_template(target_gtid, flag);
1864 }
1865 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1866  __kmp_resume_template(target_gtid, flag);
1867 }
1868 
1869 void
1870 __kmp_resume_monitor()
1871 {
1872  KMP_TIME_DEVELOPER_BLOCK(USER_resume);
1873  int status;
1874 #ifdef KMP_DEBUG
1875  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1876  KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n",
1877  gtid, KMP_GTID_MONITOR ) );
1878  KMP_DEBUG_ASSERT( gtid != KMP_GTID_MONITOR );
1879 #endif
1880  status = pthread_mutex_lock( &__kmp_wait_mx.m_mutex );
1881  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1882 #ifdef DEBUG_SUSPEND
1883  {
1884  char buffer[128];
1885  __kmp_print_cond( buffer, &__kmp_wait_cv.c_cond );
1886  __kmp_printf( "__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid, KMP_GTID_MONITOR, buffer );
1887  }
1888 #endif
1889  status = pthread_cond_signal( &__kmp_wait_cv.c_cond );
1890  KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
1891  status = pthread_mutex_unlock( &__kmp_wait_mx.m_mutex );
1892  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1893  KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d exiting after signaling wake up for T#%d\n",
1894  gtid, KMP_GTID_MONITOR ) );
1895 }
1896 
1897 /* ------------------------------------------------------------------------ */
1898 /* ------------------------------------------------------------------------ */
1899 
1900 void
1901 __kmp_yield( int cond )
1902 {
1903  if (cond && __kmp_yielding_on) {
1904  sched_yield();
1905  }
1906 }
1907 
1908 /* ------------------------------------------------------------------------ */
1909 /* ------------------------------------------------------------------------ */
1910 
1911 void
1912 __kmp_gtid_set_specific( int gtid )
1913 {
1914  if( __kmp_init_gtid ) {
1915  int status;
1916  status = pthread_setspecific( __kmp_gtid_threadprivate_key, (void*)(intptr_t)(gtid+1) );
1917  KMP_CHECK_SYSFAIL( "pthread_setspecific", status );
1918  } else {
1919  KA_TRACE( 50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n" ) );
1920  }
1921 }
1922 
1923 int
1924 __kmp_gtid_get_specific()
1925 {
1926  int gtid;
1927  if ( !__kmp_init_gtid ) {
1928  KA_TRACE( 50, ("__kmp_gtid_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
1929  return KMP_GTID_SHUTDOWN;
1930  }
1931  gtid = (int)(size_t)pthread_getspecific( __kmp_gtid_threadprivate_key );
1932  if ( gtid == 0 ) {
1933  gtid = KMP_GTID_DNE;
1934  }
1935  else {
1936  gtid--;
1937  }
1938  KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
1939  __kmp_gtid_threadprivate_key, gtid ));
1940  return gtid;
1941 }
1942 
1943 /* ------------------------------------------------------------------------ */
1944 /* ------------------------------------------------------------------------ */
1945 
1946 double
1947 __kmp_read_cpu_time( void )
1948 {
1949  /*clock_t t;*/
1950  struct tms buffer;
1951 
1952  /*t =*/ times( & buffer );
1953 
1954  return (buffer.tms_utime + buffer.tms_cutime) / (double) CLOCKS_PER_SEC;
1955 }
1956 
1957 int
1958 __kmp_read_system_info( struct kmp_sys_info *info )
1959 {
1960  int status;
1961  struct rusage r_usage;
1962 
1963  memset( info, 0, sizeof( *info ) );
1964 
1965  status = getrusage( RUSAGE_SELF, &r_usage);
1966  KMP_CHECK_SYSFAIL_ERRNO( "getrusage", status );
1967 
1968  info->maxrss = r_usage.ru_maxrss; /* the maximum resident set size utilized (in kilobytes) */
1969  info->minflt = r_usage.ru_minflt; /* the number of page faults serviced without any I/O */
1970  info->majflt = r_usage.ru_majflt; /* the number of page faults serviced that required I/O */
1971  info->nswap = r_usage.ru_nswap; /* the number of times a process was "swapped" out of memory */
1972  info->inblock = r_usage.ru_inblock; /* the number of times the file system had to perform input */
1973  info->oublock = r_usage.ru_oublock; /* the number of times the file system had to perform output */
1974  info->nvcsw = r_usage.ru_nvcsw; /* the number of times a context switch was voluntarily */
1975  info->nivcsw = r_usage.ru_nivcsw; /* the number of times a context switch was forced */
1976 
1977  return (status != 0);
1978 }
1979 
1980 /* ------------------------------------------------------------------------ */
1981 /* ------------------------------------------------------------------------ */
1982 
1983 void
1984 __kmp_read_system_time( double *delta )
1985 {
1986  double t_ns;
1987  struct timeval tval;
1988  struct timespec stop;
1989  int status;
1990 
1991  status = gettimeofday( &tval, NULL );
1992  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
1993  TIMEVAL_TO_TIMESPEC( &tval, &stop );
1994  t_ns = TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start);
1995  *delta = (t_ns * 1e-9);
1996 }
1997 
1998 void
1999 __kmp_clear_system_time( void )
2000 {
2001  struct timeval tval;
2002  int status;
2003  status = gettimeofday( &tval, NULL );
2004  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2005  TIMEVAL_TO_TIMESPEC( &tval, &__kmp_sys_timer_data.start );
2006 }
2007 
2008 /* ------------------------------------------------------------------------ */
2009 /* ------------------------------------------------------------------------ */
2010 
2011 #ifdef BUILD_TV
2012 
2013 void
2014 __kmp_tv_threadprivate_store( kmp_info_t *th, void *global_addr, void *thread_addr )
2015 {
2016  struct tv_data *p;
2017 
2018  p = (struct tv_data *) __kmp_allocate( sizeof( *p ) );
2019 
2020  p->u.tp.global_addr = global_addr;
2021  p->u.tp.thread_addr = thread_addr;
2022 
2023  p->type = (void *) 1;
2024 
2025  p->next = th->th.th_local.tv_data;
2026  th->th.th_local.tv_data = p;
2027 
2028  if ( p->next == 0 ) {
2029  int rc = pthread_setspecific( __kmp_tv_key, p );
2030  KMP_CHECK_SYSFAIL( "pthread_setspecific", rc );
2031  }
2032 }
2033 
2034 #endif /* BUILD_TV */
2035 
2036 /* ------------------------------------------------------------------------ */
2037 /* ------------------------------------------------------------------------ */
2038 
2039 static int
2040 __kmp_get_xproc( void ) {
2041 
2042  int r = 0;
2043 
2044  #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD
2045 
2046  r = sysconf( _SC_NPROCESSORS_ONLN );
2047 
2048  #elif KMP_OS_DARWIN
2049 
2050  // Bug C77011 High "OpenMP Threads and number of active cores".
2051 
2052  // Find the number of available CPUs.
2053  kern_return_t rc;
2054  host_basic_info_data_t info;
2055  mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT;
2056  rc = host_info( mach_host_self(), HOST_BASIC_INFO, (host_info_t) & info, & num );
2057  if ( rc == 0 && num == HOST_BASIC_INFO_COUNT ) {
2058  // Cannot use KA_TRACE() here because this code works before trace support is
2059  // initialized.
2060  r = info.avail_cpus;
2061  } else {
2062  KMP_WARNING( CantGetNumAvailCPU );
2063  KMP_INFORM( AssumedNumCPU );
2064  }; // if
2065 
2066  #else
2067 
2068  #error "Unknown or unsupported OS."
2069 
2070  #endif
2071 
2072  return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
2073 
2074 } // __kmp_get_xproc
2075 
2076 int
2077 __kmp_read_from_file( char const *path, char const *format, ... )
2078 {
2079  int result;
2080  va_list args;
2081 
2082  va_start(args, format);
2083  FILE *f = fopen(path, "rb");
2084  if ( f == NULL )
2085  return 0;
2086  result = vfscanf(f, format, args);
2087  fclose(f);
2088 
2089  return result;
2090 }
2091 
2092 void
2093 __kmp_runtime_initialize( void )
2094 {
2095  int status;
2096  pthread_mutexattr_t mutex_attr;
2097  pthread_condattr_t cond_attr;
2098 
2099  if ( __kmp_init_runtime ) {
2100  return;
2101  }; // if
2102 
2103  #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
2104  if ( ! __kmp_cpuinfo.initialized ) {
2105  __kmp_query_cpuid( &__kmp_cpuinfo );
2106  }; // if
2107  #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
2108 
2109  __kmp_xproc = __kmp_get_xproc();
2110 
2111  if ( sysconf( _SC_THREADS ) ) {
2112 
2113  /* Query the maximum number of threads */
2114  __kmp_sys_max_nth = sysconf( _SC_THREAD_THREADS_MAX );
2115  if ( __kmp_sys_max_nth == -1 ) {
2116  /* Unlimited threads for NPTL */
2117  __kmp_sys_max_nth = INT_MAX;
2118  }
2119  else if ( __kmp_sys_max_nth <= 1 ) {
2120  /* Can't tell, just use PTHREAD_THREADS_MAX */
2121  __kmp_sys_max_nth = KMP_MAX_NTH;
2122  }
2123 
2124  /* Query the minimum stack size */
2125  __kmp_sys_min_stksize = sysconf( _SC_THREAD_STACK_MIN );
2126  if ( __kmp_sys_min_stksize <= 1 ) {
2127  __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
2128  }
2129  }
2130 
2131  /* Set up minimum number of threads to switch to TLS gtid */
2132  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
2133 
2134  #ifdef BUILD_TV
2135  {
2136  int rc = pthread_key_create( & __kmp_tv_key, 0 );
2137  KMP_CHECK_SYSFAIL( "pthread_key_create", rc );
2138  }
2139  #endif
2140 
2141  status = pthread_key_create( &__kmp_gtid_threadprivate_key, __kmp_internal_end_dest );
2142  KMP_CHECK_SYSFAIL( "pthread_key_create", status );
2143  status = pthread_mutexattr_init( & mutex_attr );
2144  KMP_CHECK_SYSFAIL( "pthread_mutexattr_init", status );
2145  status = pthread_mutex_init( & __kmp_wait_mx.m_mutex, & mutex_attr );
2146  KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
2147  status = pthread_condattr_init( & cond_attr );
2148  KMP_CHECK_SYSFAIL( "pthread_condattr_init", status );
2149  status = pthread_cond_init( & __kmp_wait_cv.c_cond, & cond_attr );
2150  KMP_CHECK_SYSFAIL( "pthread_cond_init", status );
2151 #if USE_ITT_BUILD
2152  __kmp_itt_initialize();
2153 #endif /* USE_ITT_BUILD */
2154 
2155  __kmp_init_runtime = TRUE;
2156 }
2157 
2158 void
2159 __kmp_runtime_destroy( void )
2160 {
2161  int status;
2162 
2163  if ( ! __kmp_init_runtime ) {
2164  return; // Nothing to do.
2165  };
2166 
2167 #if USE_ITT_BUILD
2168  __kmp_itt_destroy();
2169 #endif /* USE_ITT_BUILD */
2170 
2171  status = pthread_key_delete( __kmp_gtid_threadprivate_key );
2172  KMP_CHECK_SYSFAIL( "pthread_key_delete", status );
2173  #ifdef BUILD_TV
2174  status = pthread_key_delete( __kmp_tv_key );
2175  KMP_CHECK_SYSFAIL( "pthread_key_delete", status );
2176  #endif
2177 
2178  status = pthread_mutex_destroy( & __kmp_wait_mx.m_mutex );
2179  if ( status != 0 && status != EBUSY ) {
2180  KMP_SYSFAIL( "pthread_mutex_destroy", status );
2181  }
2182  status = pthread_cond_destroy( & __kmp_wait_cv.c_cond );
2183  if ( status != 0 && status != EBUSY ) {
2184  KMP_SYSFAIL( "pthread_cond_destroy", status );
2185  }
2186  #if KMP_AFFINITY_SUPPORTED
2187  __kmp_affinity_uninitialize();
2188  #endif
2189 
2190  __kmp_init_runtime = FALSE;
2191 }
2192 
2193 
2194 /* Put the thread to sleep for a time period */
2195 /* NOTE: not currently used anywhere */
2196 void
2197 __kmp_thread_sleep( int millis )
2198 {
2199  sleep( ( millis + 500 ) / 1000 );
2200 }
2201 
2202 /* Calculate the elapsed wall clock time for the user */
2203 void
2204 __kmp_elapsed( double *t )
2205 {
2206  int status;
2207 # ifdef FIX_SGI_CLOCK
2208  struct timespec ts;
2209 
2210  status = clock_gettime( CLOCK_PROCESS_CPUTIME_ID, &ts );
2211  KMP_CHECK_SYSFAIL_ERRNO( "clock_gettime", status );
2212  *t = (double) ts.tv_nsec * (1.0 / (double) KMP_NSEC_PER_SEC) +
2213  (double) ts.tv_sec;
2214 # else
2215  struct timeval tv;
2216 
2217  status = gettimeofday( & tv, NULL );
2218  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2219  *t = (double) tv.tv_usec * (1.0 / (double) KMP_USEC_PER_SEC) +
2220  (double) tv.tv_sec;
2221 # endif
2222 }
2223 
2224 /* Calculate the elapsed wall clock tick for the user */
2225 void
2226 __kmp_elapsed_tick( double *t )
2227 {
2228  *t = 1 / (double) CLOCKS_PER_SEC;
2229 }
2230 
2231 /* Return the current time stamp in nsec */
2232 kmp_uint64
2233 __kmp_now_nsec()
2234 {
2235  struct timeval t;
2236  gettimeofday(&t, NULL);
2237  return KMP_NSEC_PER_SEC*t.tv_sec + 1000*t.tv_usec;
2238 }
2239 
2240 /*
2241  Determine whether the given address is mapped into the current address space.
2242 */
2243 
2244 int
2245 __kmp_is_address_mapped( void * addr ) {
2246 
2247  int found = 0;
2248  int rc;
2249 
2250  #if KMP_OS_LINUX || KMP_OS_FREEBSD
2251 
2252  /*
2253  On Linux* OS, read the /proc/<pid>/maps pseudo-file to get all the address ranges mapped
2254  into the address space.
2255  */
2256 
2257  char * name = __kmp_str_format( "/proc/%d/maps", getpid() );
2258  FILE * file = NULL;
2259 
2260  file = fopen( name, "r" );
2261  KMP_ASSERT( file != NULL );
2262 
2263  for ( ; ; ) {
2264 
2265  void * beginning = NULL;
2266  void * ending = NULL;
2267  char perms[ 5 ];
2268 
2269  rc = fscanf( file, "%p-%p %4s %*[^\n]\n", & beginning, & ending, perms );
2270  if ( rc == EOF ) {
2271  break;
2272  }; // if
2273  KMP_ASSERT( rc == 3 && KMP_STRLEN( perms ) == 4 ); // Make sure all fields are read.
2274 
2275  // Ending address is not included in the region, but beginning is.
2276  if ( ( addr >= beginning ) && ( addr < ending ) ) {
2277  perms[ 2 ] = 0; // 3th and 4th character does not matter.
2278  if ( strcmp( perms, "rw" ) == 0 ) {
2279  // Memory we are looking for should be readable and writable.
2280  found = 1;
2281  }; // if
2282  break;
2283  }; // if
2284 
2285  }; // forever
2286 
2287  // Free resources.
2288  fclose( file );
2289  KMP_INTERNAL_FREE( name );
2290 
2291  #elif KMP_OS_DARWIN
2292 
2293  /*
2294  On OS X*, /proc pseudo filesystem is not available. Try to read memory using vm
2295  interface.
2296  */
2297 
2298  int buffer;
2299  vm_size_t count;
2300  rc =
2301  vm_read_overwrite(
2302  mach_task_self(), // Task to read memory of.
2303  (vm_address_t)( addr ), // Address to read from.
2304  1, // Number of bytes to be read.
2305  (vm_address_t)( & buffer ), // Address of buffer to save read bytes in.
2306  & count // Address of var to save number of read bytes in.
2307  );
2308  if ( rc == 0 ) {
2309  // Memory successfully read.
2310  found = 1;
2311  }; // if
2312 
2313  #elif KMP_OS_FREEBSD || KMP_OS_NETBSD
2314 
2315  // FIXME(FreeBSD, NetBSD): Implement this
2316  found = 1;
2317 
2318  #else
2319 
2320  #error "Unknown or unsupported OS"
2321 
2322  #endif
2323 
2324  return found;
2325 
2326 } // __kmp_is_address_mapped
2327 
2328 #ifdef USE_LOAD_BALANCE
2329 
2330 
2331 # if KMP_OS_DARWIN
2332 
2333 // The function returns the rounded value of the system load average
2334 // during given time interval which depends on the value of
2335 // __kmp_load_balance_interval variable (default is 60 sec, other values
2336 // may be 300 sec or 900 sec).
2337 // It returns -1 in case of error.
2338 int
2339 __kmp_get_load_balance( int max )
2340 {
2341  double averages[3];
2342  int ret_avg = 0;
2343 
2344  int res = getloadavg( averages, 3 );
2345 
2346  //Check __kmp_load_balance_interval to determine which of averages to use.
2347  // getloadavg() may return the number of samples less than requested that is
2348  // less than 3.
2349  if ( __kmp_load_balance_interval < 180 && ( res >= 1 ) ) {
2350  ret_avg = averages[0];// 1 min
2351  } else if ( ( __kmp_load_balance_interval >= 180
2352  && __kmp_load_balance_interval < 600 ) && ( res >= 2 ) ) {
2353  ret_avg = averages[1];// 5 min
2354  } else if ( ( __kmp_load_balance_interval >= 600 ) && ( res == 3 ) ) {
2355  ret_avg = averages[2];// 15 min
2356  } else {// Error occurred
2357  return -1;
2358  }
2359 
2360  return ret_avg;
2361 }
2362 
2363 # else // Linux* OS
2364 
2365 // The fuction returns number of running (not sleeping) threads, or -1 in case of error.
2366 // Error could be reported if Linux* OS kernel too old (without "/proc" support).
2367 // Counting running threads stops if max running threads encountered.
2368 int
2369 __kmp_get_load_balance( int max )
2370 {
2371  static int permanent_error = 0;
2372 
2373  static int glb_running_threads = 0; /* Saved count of the running threads for the thread balance algortihm */
2374  static double glb_call_time = 0; /* Thread balance algorithm call time */
2375 
2376  int running_threads = 0; // Number of running threads in the system.
2377 
2378  DIR * proc_dir = NULL; // Handle of "/proc/" directory.
2379  struct dirent * proc_entry = NULL;
2380 
2381  kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2382  DIR * task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2383  struct dirent * task_entry = NULL;
2384  int task_path_fixed_len;
2385 
2386  kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2387  int stat_file = -1;
2388  int stat_path_fixed_len;
2389 
2390  int total_processes = 0; // Total number of processes in system.
2391  int total_threads = 0; // Total number of threads in system.
2392 
2393  double call_time = 0.0;
2394 
2395  __kmp_str_buf_init( & task_path );
2396  __kmp_str_buf_init( & stat_path );
2397 
2398  __kmp_elapsed( & call_time );
2399 
2400  if ( glb_call_time &&
2401  ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
2402  running_threads = glb_running_threads;
2403  goto finish;
2404  }
2405 
2406  glb_call_time = call_time;
2407 
2408  // Do not spend time on scanning "/proc/" if we have a permanent error.
2409  if ( permanent_error ) {
2410  running_threads = -1;
2411  goto finish;
2412  }; // if
2413 
2414  if ( max <= 0 ) {
2415  max = INT_MAX;
2416  }; // if
2417 
2418  // Open "/proc/" directory.
2419  proc_dir = opendir( "/proc" );
2420  if ( proc_dir == NULL ) {
2421  // Cannot open "/prroc/". Probably the kernel does not support it. Return an error now and
2422  // in subsequent calls.
2423  running_threads = -1;
2424  permanent_error = 1;
2425  goto finish;
2426  }; // if
2427 
2428  // Initialize fixed part of task_path. This part will not change.
2429  __kmp_str_buf_cat( & task_path, "/proc/", 6 );
2430  task_path_fixed_len = task_path.used; // Remember number of used characters.
2431 
2432  proc_entry = readdir( proc_dir );
2433  while ( proc_entry != NULL ) {
2434  // Proc entry is a directory and name starts with a digit. Assume it is a process'
2435  // directory.
2436  if ( proc_entry->d_type == DT_DIR && isdigit( proc_entry->d_name[ 0 ] ) ) {
2437 
2438  ++ total_processes;
2439  // Make sure init process is the very first in "/proc", so we can replace
2440  // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes == 1.
2441  // We are going to check that total_processes == 1 => d_name == "1" is true (where
2442  // "=>" is implication). Since C++ does not have => operator, let us replace it with its
2443  // equivalent: a => b == ! a || b.
2444  KMP_DEBUG_ASSERT( total_processes != 1 || strcmp( proc_entry->d_name, "1" ) == 0 );
2445 
2446  // Construct task_path.
2447  task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2448  __kmp_str_buf_cat( & task_path, proc_entry->d_name, KMP_STRLEN( proc_entry->d_name ) );
2449  __kmp_str_buf_cat( & task_path, "/task", 5 );
2450 
2451  task_dir = opendir( task_path.str );
2452  if ( task_dir == NULL ) {
2453  // Process can finish between reading "/proc/" directory entry and opening process'
2454  // "task/" directory. So, in general case we should not complain, but have to skip
2455  // this process and read the next one.
2456  // But on systems with no "task/" support we will spend lot of time to scan "/proc/"
2457  // tree again and again without any benefit. "init" process (its pid is 1) should
2458  // exist always, so, if we cannot open "/proc/1/task/" directory, it means "task/"
2459  // is not supported by kernel. Report an error now and in the future.
2460  if ( strcmp( proc_entry->d_name, "1" ) == 0 ) {
2461  running_threads = -1;
2462  permanent_error = 1;
2463  goto finish;
2464  }; // if
2465  } else {
2466  // Construct fixed part of stat file path.
2467  __kmp_str_buf_clear( & stat_path );
2468  __kmp_str_buf_cat( & stat_path, task_path.str, task_path.used );
2469  __kmp_str_buf_cat( & stat_path, "/", 1 );
2470  stat_path_fixed_len = stat_path.used;
2471 
2472  task_entry = readdir( task_dir );
2473  while ( task_entry != NULL ) {
2474  // It is a directory and name starts with a digit.
2475  if ( proc_entry->d_type == DT_DIR && isdigit( task_entry->d_name[ 0 ] ) ) {
2476 
2477  ++ total_threads;
2478 
2479  // Consruct complete stat file path. Easiest way would be:
2480  // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str, task_entry->d_name );
2481  // but seriae of __kmp_str_buf_cat works a bit faster.
2482  stat_path.used = stat_path_fixed_len; // Reset stat path to its fixed part.
2483  __kmp_str_buf_cat( & stat_path, task_entry->d_name, KMP_STRLEN( task_entry->d_name ) );
2484  __kmp_str_buf_cat( & stat_path, "/stat", 5 );
2485 
2486  // Note: Low-level API (open/read/close) is used. High-level API
2487  // (fopen/fclose) works ~ 30 % slower.
2488  stat_file = open( stat_path.str, O_RDONLY );
2489  if ( stat_file == -1 ) {
2490  // We cannot report an error because task (thread) can terminate just
2491  // before reading this file.
2492  } else {
2493  /*
2494  Content of "stat" file looks like:
2495 
2496  24285 (program) S ...
2497 
2498  It is a single line (if program name does not include fanny
2499  symbols). First number is a thread id, then name of executable file
2500  name in paretheses, then state of the thread. We need just thread
2501  state.
2502 
2503  Good news: Length of program name is 15 characters max. Longer
2504  names are truncated.
2505 
2506  Thus, we need rather short buffer: 15 chars for program name +
2507  2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2508 
2509  Bad news: Program name may contain special symbols like space,
2510  closing parenthesis, or even new line. This makes parsing "stat"
2511  file not 100 % reliable. In case of fanny program names parsing
2512  may fail (report incorrect thread state).
2513 
2514  Parsing "status" file looks more promissing (due to different
2515  file structure and escaping special symbols) but reading and
2516  parsing of "status" file works slower.
2517 
2518  -- ln
2519  */
2520  char buffer[ 65 ];
2521  int len;
2522  len = read( stat_file, buffer, sizeof( buffer ) - 1 );
2523  if ( len >= 0 ) {
2524  buffer[ len ] = 0;
2525  // Using scanf:
2526  // sscanf( buffer, "%*d (%*s) %c ", & state );
2527  // looks very nice, but searching for a closing parenthesis works a
2528  // bit faster.
2529  char * close_parent = strstr( buffer, ") " );
2530  if ( close_parent != NULL ) {
2531  char state = * ( close_parent + 2 );
2532  if ( state == 'R' ) {
2533  ++ running_threads;
2534  if ( running_threads >= max ) {
2535  goto finish;
2536  }; // if
2537  }; // if
2538  }; // if
2539  }; // if
2540  close( stat_file );
2541  stat_file = -1;
2542  }; // if
2543  }; // if
2544  task_entry = readdir( task_dir );
2545  }; // while
2546  closedir( task_dir );
2547  task_dir = NULL;
2548  }; // if
2549  }; // if
2550  proc_entry = readdir( proc_dir );
2551  }; // while
2552 
2553  //
2554  // There _might_ be a timing hole where the thread executing this
2555  // code get skipped in the load balance, and running_threads is 0.
2556  // Assert in the debug builds only!!!
2557  //
2558  KMP_DEBUG_ASSERT( running_threads > 0 );
2559  if ( running_threads <= 0 ) {
2560  running_threads = 1;
2561  }
2562 
2563  finish: // Clean up and exit.
2564  if ( proc_dir != NULL ) {
2565  closedir( proc_dir );
2566  }; // if
2567  __kmp_str_buf_free( & task_path );
2568  if ( task_dir != NULL ) {
2569  closedir( task_dir );
2570  }; // if
2571  __kmp_str_buf_free( & stat_path );
2572  if ( stat_file != -1 ) {
2573  close( stat_file );
2574  }; // if
2575 
2576  glb_running_threads = running_threads;
2577 
2578  return running_threads;
2579 
2580 } // __kmp_get_load_balance
2581 
2582 # endif // KMP_OS_DARWIN
2583 
2584 #endif // USE_LOAD_BALANCE
2585 
2586 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || (KMP_OS_LINUX && KMP_ARCH_AARCH64) || KMP_ARCH_PPC64)
2587 
2588 // we really only need the case with 1 argument, because CLANG always build
2589 // a struct of pointers to shared variables referenced in the outlined function
2590 int
2591 __kmp_invoke_microtask( microtask_t pkfn,
2592  int gtid, int tid,
2593  int argc, void *p_argv[]
2594 #if OMPT_SUPPORT
2595  , void **exit_frame_ptr
2596 #endif
2597 )
2598 {
2599 #if OMPT_SUPPORT
2600  *exit_frame_ptr = __builtin_frame_address(0);
2601 #endif
2602 
2603  switch (argc) {
2604  default:
2605  fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2606  fflush(stderr);
2607  exit(-1);
2608  case 0:
2609  (*pkfn)(&gtid, &tid);
2610  break;
2611  case 1:
2612  (*pkfn)(&gtid, &tid, p_argv[0]);
2613  break;
2614  case 2:
2615  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
2616  break;
2617  case 3:
2618  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
2619  break;
2620  case 4:
2621  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]);
2622  break;
2623  case 5:
2624  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]);
2625  break;
2626  case 6:
2627  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2628  p_argv[5]);
2629  break;
2630  case 7:
2631  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2632  p_argv[5], p_argv[6]);
2633  break;
2634  case 8:
2635  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2636  p_argv[5], p_argv[6], p_argv[7]);
2637  break;
2638  case 9:
2639  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2640  p_argv[5], p_argv[6], p_argv[7], p_argv[8]);
2641  break;
2642  case 10:
2643  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2644  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]);
2645  break;
2646  case 11:
2647  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2648  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2649  break;
2650  case 12:
2651  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2652  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2653  p_argv[11]);
2654  break;
2655  case 13:
2656  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2657  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2658  p_argv[11], p_argv[12]);
2659  break;
2660  case 14:
2661  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2662  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2663  p_argv[11], p_argv[12], p_argv[13]);
2664  break;
2665  case 15:
2666  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2667  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2668  p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2669  break;
2670  }
2671 
2672 #if OMPT_SUPPORT
2673  *exit_frame_ptr = 0;
2674 #endif
2675 
2676  return 1;
2677 }
2678 
2679 #endif
2680 
2681 // end of file //
2682 
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
Definition: kmp_stats.h:791
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the paritioned timers to begin with name.
Definition: kmp_stats.h:832