LLVM OpenMP* Runtime Library
z_Windows_NT_util.cpp
1 /*
2  * z_Windows_NT_util.cpp -- platform specific routines.
3  */
4 
5 
6 //===----------------------------------------------------------------------===//
7 //
8 // The LLVM Compiler Infrastructure
9 //
10 // This file is dual licensed under the MIT and the University of Illinois Open
11 // Source Licenses. See LICENSE.txt for details.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 
16 #include "kmp.h"
17 #include "kmp_itt.h"
18 #include "kmp_i18n.h"
19 #include "kmp_io.h"
20 #include "kmp_wait_release.h"
21 #include "kmp_affinity.h"
22 
23 /* This code is related to NtQuerySystemInformation() function. This function
24  is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
25  number of running threads in the system. */
26 
27 #include <ntstatus.h>
28 #include <ntsecapi.h> // UNICODE_STRING
29 
30 enum SYSTEM_INFORMATION_CLASS {
31  SystemProcessInformation = 5
32 }; // SYSTEM_INFORMATION_CLASS
33 
34 struct CLIENT_ID {
35  HANDLE UniqueProcess;
36  HANDLE UniqueThread;
37 }; // struct CLIENT_ID
38 
39 enum THREAD_STATE {
40  StateInitialized,
41  StateReady,
42  StateRunning,
43  StateStandby,
44  StateTerminated,
45  StateWait,
46  StateTransition,
47  StateUnknown
48 }; // enum THREAD_STATE
49 
50 struct VM_COUNTERS {
51  SIZE_T PeakVirtualSize;
52  SIZE_T VirtualSize;
53  ULONG PageFaultCount;
54  SIZE_T PeakWorkingSetSize;
55  SIZE_T WorkingSetSize;
56  SIZE_T QuotaPeakPagedPoolUsage;
57  SIZE_T QuotaPagedPoolUsage;
58  SIZE_T QuotaPeakNonPagedPoolUsage;
59  SIZE_T QuotaNonPagedPoolUsage;
60  SIZE_T PagefileUsage;
61  SIZE_T PeakPagefileUsage;
62  SIZE_T PrivatePageCount;
63 }; // struct VM_COUNTERS
64 
65 struct SYSTEM_THREAD {
66  LARGE_INTEGER KernelTime;
67  LARGE_INTEGER UserTime;
68  LARGE_INTEGER CreateTime;
69  ULONG WaitTime;
70  LPVOID StartAddress;
71  CLIENT_ID ClientId;
72  DWORD Priority;
73  LONG BasePriority;
74  ULONG ContextSwitchCount;
75  THREAD_STATE State;
76  ULONG WaitReason;
77 }; // SYSTEM_THREAD
78 
79 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, KernelTime ) == 0 );
80 #if KMP_ARCH_X86
81  KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 28 );
82  KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 52 );
83 #else
84  KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 32 );
85  KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 68 );
86 #endif
87 
88 struct SYSTEM_PROCESS_INFORMATION {
89  ULONG NextEntryOffset;
90  ULONG NumberOfThreads;
91  LARGE_INTEGER Reserved[ 3 ];
92  LARGE_INTEGER CreateTime;
93  LARGE_INTEGER UserTime;
94  LARGE_INTEGER KernelTime;
95  UNICODE_STRING ImageName;
96  DWORD BasePriority;
97  HANDLE ProcessId;
98  HANDLE ParentProcessId;
99  ULONG HandleCount;
100  ULONG Reserved2[ 2 ];
101  VM_COUNTERS VMCounters;
102  IO_COUNTERS IOCounters;
103  SYSTEM_THREAD Threads[ 1 ];
104 }; // SYSTEM_PROCESS_INFORMATION
105 typedef SYSTEM_PROCESS_INFORMATION * PSYSTEM_PROCESS_INFORMATION;
106 
107 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, NextEntryOffset ) == 0 );
108 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, CreateTime ) == 32 );
109 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ImageName ) == 56 );
110 #if KMP_ARCH_X86
111  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 68 );
112  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 76 );
113  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 88 );
114  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 136 );
115  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 184 );
116 #else
117  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 80 );
118  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 96 );
119  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 112 );
120  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 208 );
121  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 256 );
122 #endif
123 
124 typedef NTSTATUS (NTAPI *NtQuerySystemInformation_t)( SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG );
125 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
126 
127 HMODULE ntdll = NULL;
128 
129 /* End of NtQuerySystemInformation()-related code */
130 
131 static HMODULE kernel32 = NULL;
132 
133 /* ----------------------------------------------------------------------------------- */
134 /* ----------------------------------------------------------------------------------- */
135 
136 #if KMP_HANDLE_SIGNALS
137  typedef void (* sig_func_t )( int );
138  static sig_func_t __kmp_sighldrs[ NSIG ];
139  static int __kmp_siginstalled[ NSIG ];
140 #endif
141 
142 #if KMP_USE_MONITOR
143 static HANDLE __kmp_monitor_ev;
144 #endif
145 static kmp_int64 __kmp_win32_time;
146 double __kmp_win32_tick;
147 
148 int __kmp_init_runtime = FALSE;
149 CRITICAL_SECTION __kmp_win32_section;
150 
151 void
152 __kmp_win32_mutex_init( kmp_win32_mutex_t *mx )
153 {
154  InitializeCriticalSection( & mx->cs );
155 #if USE_ITT_BUILD
156  __kmp_itt_system_object_created( & mx->cs, "Critical Section" );
157 #endif /* USE_ITT_BUILD */
158 }
159 
160 void
161 __kmp_win32_mutex_destroy( kmp_win32_mutex_t *mx )
162 {
163  DeleteCriticalSection( & mx->cs );
164 }
165 
166 void
167 __kmp_win32_mutex_lock( kmp_win32_mutex_t *mx )
168 {
169  EnterCriticalSection( & mx->cs );
170 }
171 
172 void
173 __kmp_win32_mutex_unlock( kmp_win32_mutex_t *mx )
174 {
175  LeaveCriticalSection( & mx->cs );
176 }
177 
178 void
179 __kmp_win32_cond_init( kmp_win32_cond_t *cv )
180 {
181  cv->waiters_count_ = 0;
182  cv->wait_generation_count_ = 0;
183  cv->release_count_ = 0;
184 
185  /* Initialize the critical section */
186  __kmp_win32_mutex_init( & cv->waiters_count_lock_ );
187 
188  /* Create a manual-reset event. */
189  cv->event_ = CreateEvent( NULL, // no security
190  TRUE, // manual-reset
191  FALSE, // non-signaled initially
192  NULL ); // unnamed
193 #if USE_ITT_BUILD
194  __kmp_itt_system_object_created( cv->event_, "Event" );
195 #endif /* USE_ITT_BUILD */
196 }
197 
198 void
199 __kmp_win32_cond_destroy( kmp_win32_cond_t *cv )
200 {
201  __kmp_win32_mutex_destroy( & cv->waiters_count_lock_ );
202  __kmp_free_handle( cv->event_ );
203  memset( cv, '\0', sizeof( *cv ) );
204 }
205 
206 /* TODO associate cv with a team instead of a thread so as to optimize
207  * the case where we wake up a whole team */
208 
209 void
210 __kmp_win32_cond_wait( kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx, kmp_info_t *th, int need_decrease_load )
211 {
212  int my_generation;
213  int last_waiter;
214 
215  /* Avoid race conditions */
216  __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
217 
218  /* Increment count of waiters */
219  cv->waiters_count_++;
220 
221  /* Store current generation in our activation record. */
222  my_generation = cv->wait_generation_count_;
223 
224  __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
225  __kmp_win32_mutex_unlock( mx );
226 
227  for (;;) {
228  int wait_done;
229 
230  /* Wait until the event is signaled */
231  WaitForSingleObject( cv->event_, INFINITE );
232 
233  __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
234 
235  /* Exit the loop when the <cv->event_> is signaled and
236  * there are still waiting threads from this <wait_generation>
237  * that haven't been released from this wait yet. */
238  wait_done = ( cv->release_count_ > 0 ) &&
239  ( cv->wait_generation_count_ != my_generation );
240 
241  __kmp_win32_mutex_unlock( &cv->waiters_count_lock_);
242 
243  /* there used to be a semicolon after the if statement,
244  * it looked like a bug, so i removed it */
245  if( wait_done )
246  break;
247  }
248 
249  __kmp_win32_mutex_lock( mx );
250  __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
251 
252  cv->waiters_count_--;
253  cv->release_count_--;
254 
255  last_waiter = ( cv->release_count_ == 0 );
256 
257  __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
258 
259  if( last_waiter ) {
260  /* We're the last waiter to be notified, so reset the manual event. */
261  ResetEvent( cv->event_ );
262  }
263 }
264 
265 void
266 __kmp_win32_cond_broadcast( kmp_win32_cond_t *cv )
267 {
268  __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
269 
270  if( cv->waiters_count_ > 0 ) {
271  SetEvent( cv->event_ );
272  /* Release all the threads in this generation. */
273 
274  cv->release_count_ = cv->waiters_count_;
275 
276  /* Start a new generation. */
277  cv->wait_generation_count_++;
278  }
279 
280  __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
281 }
282 
283 void
284 __kmp_win32_cond_signal( kmp_win32_cond_t *cv )
285 {
286  __kmp_win32_cond_broadcast( cv );
287 }
288 
289 /* ------------------------------------------------------------------------ */
290 /* ------------------------------------------------------------------------ */
291 
292 void
293 __kmp_enable( int new_state )
294 {
295  if (__kmp_init_runtime)
296  LeaveCriticalSection( & __kmp_win32_section );
297 }
298 
299 void
300 __kmp_disable( int *old_state )
301 {
302  *old_state = 0;
303 
304  if (__kmp_init_runtime)
305  EnterCriticalSection( & __kmp_win32_section );
306 }
307 
308 void
309 __kmp_suspend_initialize( void )
310 {
311  /* do nothing */
312 }
313 
314 static void
315 __kmp_suspend_initialize_thread( kmp_info_t *th )
316 {
317  if ( ! TCR_4( th->th.th_suspend_init ) ) {
318  /* this means we haven't initialized the suspension pthread objects for this thread
319  in this instance of the process */
320  __kmp_win32_cond_init( &th->th.th_suspend_cv );
321  __kmp_win32_mutex_init( &th->th.th_suspend_mx );
322  TCW_4( th->th.th_suspend_init, TRUE );
323  }
324 }
325 
326 void
327 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
328 {
329  if ( TCR_4( th->th.th_suspend_init ) ) {
330  /* this means we have initialize the suspension pthread objects for this thread
331  in this instance of the process */
332  __kmp_win32_cond_destroy( & th->th.th_suspend_cv );
333  __kmp_win32_mutex_destroy( & th->th.th_suspend_mx );
334  TCW_4( th->th.th_suspend_init, FALSE );
335  }
336 }
337 
338 /* This routine puts the calling thread to sleep after setting the
339  * sleep bit for the indicated flag variable to true.
340  */
341 template <class C>
342 static inline void __kmp_suspend_template( int th_gtid, C *flag )
343 {
344  kmp_info_t *th = __kmp_threads[th_gtid];
345  int status;
346  typename C::flag_t old_spin;
347 
348  KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n", th_gtid, flag->get() ) );
349 
350  __kmp_suspend_initialize_thread( th );
351  __kmp_win32_mutex_lock( &th->th.th_suspend_mx );
352 
353  KF_TRACE( 10, ( "__kmp_suspend_template: T#%d setting sleep bit for flag's loc(%p)\n",
354  th_gtid, flag->get() ) );
355 
356  /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread
357  gets called first?
358  */
359  old_spin = flag->set_sleeping();
360 
361  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d set sleep bit for flag's loc(%p)==%d\n",
362  th_gtid, flag->get(), *(flag->get()) ) );
363 
364  if ( flag->done_check_val(old_spin) ) {
365  old_spin = flag->unset_sleeping();
366  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d false alarm, reset sleep bit for flag's loc(%p)\n",
367  th_gtid, flag->get()) );
368  } else {
369 #ifdef DEBUG_SUSPEND
370  __kmp_suspend_count++;
371 #endif
372  /* Encapsulate in a loop as the documentation states that this may
373  * "with low probability" return when the condition variable has
374  * not been signaled or broadcast
375  */
376  int deactivated = FALSE;
377  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
378  while ( flag->is_sleeping() ) {
379  KF_TRACE( 15, ("__kmp_suspend_template: T#%d about to perform kmp_win32_cond_wait()\n",
380  th_gtid ) );
381  // Mark the thread as no longer active (only in the first iteration of the loop).
382  if ( ! deactivated ) {
383  th->th.th_active = FALSE;
384  if ( th->th.th_active_in_pool ) {
385  th->th.th_active_in_pool = FALSE;
386  KMP_TEST_THEN_DEC32(
387  (kmp_int32 *) &__kmp_thread_pool_active_nth );
388  KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
389  }
390  deactivated = TRUE;
391 
392  __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
393  }
394  else {
395  __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
396  }
397 
398 #ifdef KMP_DEBUG
399  if( flag->is_sleeping() ) {
400  KF_TRACE( 100, ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ));
401  }
402 #endif /* KMP_DEBUG */
403 
404  } // while
405 
406  // Mark the thread as active again (if it was previous marked as inactive)
407  if ( deactivated ) {
408  th->th.th_active = TRUE;
409  if ( TCR_4(th->th.th_in_pool) ) {
410  KMP_TEST_THEN_INC32(
411  (kmp_int32 *) &__kmp_thread_pool_active_nth );
412  th->th.th_active_in_pool = TRUE;
413  }
414  }
415  }
416 
417  __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
418 
419  KF_TRACE( 30, ("__kmp_suspend_template: T#%d exit\n", th_gtid ) );
420 }
421 
422 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
423  __kmp_suspend_template(th_gtid, flag);
424 }
425 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
426  __kmp_suspend_template(th_gtid, flag);
427 }
428 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
429  __kmp_suspend_template(th_gtid, flag);
430 }
431 
432 
433 /* This routine signals the thread specified by target_gtid to wake up
434  * after setting the sleep bit indicated by the flag argument to FALSE
435  */
436 template <class C>
437 static inline void __kmp_resume_template( int target_gtid, C *flag )
438 {
439  kmp_info_t *th = __kmp_threads[target_gtid];
440  int status;
441 
442 #ifdef KMP_DEBUG
443  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
444 #endif
445 
446  KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
447 
448  __kmp_suspend_initialize_thread( th );
449  __kmp_win32_mutex_lock( &th->th.th_suspend_mx );
450 
451  if (!flag) { // coming from __kmp_null_resume_wrapper
452  flag = (C *)th->th.th_sleep_loc;
453  }
454 
455  // First, check if the flag is null or its type has changed. If so, someone else woke it up.
456  if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type simply shows what flag was cast to
457  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p)\n",
458  gtid, target_gtid, NULL ) );
459  __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
460  return;
461  }
462  else {
463  typename C::flag_t old_spin = flag->unset_sleeping();
464  if ( !flag->is_sleeping_val(old_spin) ) {
465  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p): "
466  "%u => %u\n",
467  gtid, target_gtid, flag->get(), old_spin, *(flag->get()) ) );
468  __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
469  return;
470  }
471  }
472  TCW_PTR(th->th.th_sleep_loc, NULL);
473 
474  KF_TRACE( 5, ( "__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p)\n",
475  gtid, target_gtid, flag->get() ) );
476 
477  __kmp_win32_cond_signal( &th->th.th_suspend_cv );
478  __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
479 
480  KF_TRACE( 30, ( "__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
481  gtid, target_gtid ) );
482 }
483 
484 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
485  __kmp_resume_template(target_gtid, flag);
486 }
487 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
488  __kmp_resume_template(target_gtid, flag);
489 }
490 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
491  __kmp_resume_template(target_gtid, flag);
492 }
493 
494 
495 /* ------------------------------------------------------------------------ */
496 /* ------------------------------------------------------------------------ */
497 
498 void
499 __kmp_yield( int cond )
500 {
501  if (cond)
502  Sleep(0);
503 }
504 
505 /* ------------------------------------------------------------------------ */
506 /* ------------------------------------------------------------------------ */
507 
508 void
509 __kmp_gtid_set_specific( int gtid )
510 {
511  if( __kmp_init_gtid ) {
512  KA_TRACE( 50, ("__kmp_gtid_set_specific: T#%d key:%d\n",
513  gtid, __kmp_gtid_threadprivate_key ));
514  if( ! TlsSetValue( __kmp_gtid_threadprivate_key, (LPVOID)(gtid+1)) )
515  KMP_FATAL( TLSSetValueFailed );
516  } else {
517  KA_TRACE( 50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n" ) );
518  }
519 }
520 
521 int
522 __kmp_gtid_get_specific()
523 {
524  int gtid;
525  if( !__kmp_init_gtid ) {
526  KA_TRACE( 50, ("__kmp_gtid_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
527  return KMP_GTID_SHUTDOWN;
528  }
529  gtid = (int)(kmp_intptr_t)TlsGetValue( __kmp_gtid_threadprivate_key );
530  if ( gtid == 0 ) {
531  gtid = KMP_GTID_DNE;
532  }
533  else {
534  gtid--;
535  }
536  KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
537  __kmp_gtid_threadprivate_key, gtid ));
538  return gtid;
539 }
540 
541 /* ------------------------------------------------------------------------ */
542 /* ------------------------------------------------------------------------ */
543 
544 void
545 __kmp_affinity_bind_thread( int proc )
546 {
547  if (__kmp_num_proc_groups > 1) {
548  //
549  // Form the GROUP_AFFINITY struct directly, rather than filling
550  // out a bit vector and calling __kmp_set_system_affinity().
551  //
552  GROUP_AFFINITY ga;
553  KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups
554  * CHAR_BIT * sizeof(DWORD_PTR))));
555  ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
556  ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
557  ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
558 
559  KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
560  if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
561  DWORD error = GetLastError();
562  if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
563  kmp_msg_t err_code = KMP_ERR( error );
564  __kmp_msg(
565  kmp_ms_warning,
566  KMP_MSG( CantSetThreadAffMask ),
567  err_code,
568  __kmp_msg_null
569  );
570  if (__kmp_generate_warnings == kmp_warnings_off) {
571  __kmp_str_free(&err_code.str);
572  }
573  }
574  }
575  } else {
576  kmp_affin_mask_t *mask;
577  KMP_CPU_ALLOC_ON_STACK(mask);
578  KMP_CPU_ZERO(mask);
579  KMP_CPU_SET(proc, mask);
580  __kmp_set_system_affinity(mask, TRUE);
581  KMP_CPU_FREE_FROM_STACK(mask);
582  }
583 }
584 
585 void
586 __kmp_affinity_determine_capable( const char *env_var )
587 {
588  //
589  // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask().
590  //
591 
592 #if KMP_GROUP_AFFINITY
593  KMP_AFFINITY_ENABLE(__kmp_num_proc_groups*sizeof(DWORD_PTR));
594 #else
595  KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR));
596 #endif
597 
598  KA_TRACE( 10, (
599  "__kmp_affinity_determine_capable: "
600  "Windows* OS affinity interface functional (mask size = %" KMP_SIZE_T_SPEC ").\n",
601  __kmp_affin_mask_size
602  ) );
603 }
604 
605 double
606 __kmp_read_cpu_time( void )
607 {
608  FILETIME CreationTime, ExitTime, KernelTime, UserTime;
609  int status;
610  double cpu_time;
611 
612  cpu_time = 0;
613 
614  status = GetProcessTimes( GetCurrentProcess(), &CreationTime,
615  &ExitTime, &KernelTime, &UserTime );
616 
617  if (status) {
618  double sec = 0;
619 
620  sec += KernelTime.dwHighDateTime;
621  sec += UserTime.dwHighDateTime;
622 
623  /* Shift left by 32 bits */
624  sec *= (double) (1 << 16) * (double) (1 << 16);
625 
626  sec += KernelTime.dwLowDateTime;
627  sec += UserTime.dwLowDateTime;
628 
629  cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
630  }
631 
632  return cpu_time;
633 }
634 
635 int
636 __kmp_read_system_info( struct kmp_sys_info *info )
637 {
638  info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
639  info->minflt = 0; /* the number of page faults serviced without any I/O */
640  info->majflt = 0; /* the number of page faults serviced that required I/O */
641  info->nswap = 0; /* the number of times a process was "swapped" out of memory */
642  info->inblock = 0; /* the number of times the file system had to perform input */
643  info->oublock = 0; /* the number of times the file system had to perform output */
644  info->nvcsw = 0; /* the number of times a context switch was voluntarily */
645  info->nivcsw = 0; /* the number of times a context switch was forced */
646 
647  return 1;
648 }
649 
650 /* ------------------------------------------------------------------------ */
651 /* ------------------------------------------------------------------------ */
652 
653 
654 void
655 __kmp_runtime_initialize( void )
656 {
657  SYSTEM_INFO info;
658  kmp_str_buf_t path;
659  UINT path_size;
660 
661  if ( __kmp_init_runtime ) {
662  return;
663  };
664 
665 #if KMP_DYNAMIC_LIB
666  /* Pin dynamic library for the lifetime of application */
667  {
668  // First, turn off error message boxes
669  UINT err_mode = SetErrorMode (SEM_FAILCRITICALERRORS);
670  HMODULE h;
671  BOOL ret = GetModuleHandleEx( GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS
672  |GET_MODULE_HANDLE_EX_FLAG_PIN,
673  (LPCTSTR)&__kmp_serial_initialize, &h);
674  KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
675  SetErrorMode (err_mode); // Restore error mode
676  KA_TRACE( 10, ("__kmp_runtime_initialize: dynamic library pinned\n") );
677  }
678 #endif
679 
680  InitializeCriticalSection( & __kmp_win32_section );
681 #if USE_ITT_BUILD
682  __kmp_itt_system_object_created( & __kmp_win32_section, "Critical Section" );
683 #endif /* USE_ITT_BUILD */
684  __kmp_initialize_system_tick();
685 
686  #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
687  if ( ! __kmp_cpuinfo.initialized ) {
688  __kmp_query_cpuid( & __kmp_cpuinfo );
689  }; // if
690  #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
691 
692  /* Set up minimum number of threads to switch to TLS gtid */
693  #if KMP_OS_WINDOWS && ! defined KMP_DYNAMIC_LIB
694  // Windows* OS, static library.
695  /*
696  New thread may use stack space previously used by another thread, currently terminated.
697  On Windows* OS, in case of static linking, we do not know the moment of thread termination,
698  and our structures (__kmp_threads and __kmp_root arrays) are still keep info about dead
699  threads. This leads to problem in __kmp_get_global_thread_id() function: it wrongly
700  finds gtid (by searching through stack addresses of all known threads) for unregistered
701  foreign tread.
702 
703  Setting __kmp_tls_gtid_min to 0 workarounds this problem: __kmp_get_global_thread_id()
704  does not search through stacks, but get gtid from TLS immediately.
705 
706  --ln
707  */
708  __kmp_tls_gtid_min = 0;
709  #else
710  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
711  #endif
712 
713  /* for the static library */
714  if ( !__kmp_gtid_threadprivate_key ) {
715  __kmp_gtid_threadprivate_key = TlsAlloc();
716  if( __kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES ) {
717  KMP_FATAL( TLSOutOfIndexes );
718  }
719  }
720 
721 
722  //
723  // Load ntdll.dll.
724  //
725  /*
726  Simple
727  GetModuleHandle( "ntdll.dl" )
728  is not suitable due to security issue (see
729  http://www.microsoft.com/technet/security/advisory/2269637.mspx). We have to specify full
730  path to the library.
731  */
732  __kmp_str_buf_init( & path );
733  path_size = GetSystemDirectory( path.str, path.size );
734  KMP_DEBUG_ASSERT( path_size > 0 );
735  if ( path_size >= path.size ) {
736  //
737  // Buffer is too short. Expand the buffer and try again.
738  //
739  __kmp_str_buf_reserve( & path, path_size );
740  path_size = GetSystemDirectory( path.str, path.size );
741  KMP_DEBUG_ASSERT( path_size > 0 );
742  }; // if
743  if ( path_size > 0 && path_size < path.size ) {
744  //
745  // Now we have system directory name in the buffer.
746  // Append backslash and name of dll to form full path,
747  //
748  path.used = path_size;
749  __kmp_str_buf_print( & path, "\\%s", "ntdll.dll" );
750 
751  //
752  // Now load ntdll using full path.
753  //
754  ntdll = GetModuleHandle( path.str );
755  }
756 
757  KMP_DEBUG_ASSERT( ntdll != NULL );
758  if ( ntdll != NULL ) {
759  NtQuerySystemInformation = (NtQuerySystemInformation_t) GetProcAddress( ntdll, "NtQuerySystemInformation" );
760  }
761  KMP_DEBUG_ASSERT( NtQuerySystemInformation != NULL );
762 
763 #if KMP_GROUP_AFFINITY
764  //
765  // Load kernel32.dll.
766  // Same caveat - must use full system path name.
767  //
768  if ( path_size > 0 && path_size < path.size ) {
769  //
770  // Truncate the buffer back to just the system path length,
771  // discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
772  //
773  path.used = path_size;
774  __kmp_str_buf_print( & path, "\\%s", "kernel32.dll" );
775 
776  //
777  // Load kernel32.dll using full path.
778  //
779  kernel32 = GetModuleHandle( path.str );
780  KA_TRACE( 10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str ) );
781 
782  //
783  // Load the function pointers to kernel32.dll routines
784  // that may or may not exist on this system.
785  //
786  if ( kernel32 != NULL ) {
787  __kmp_GetActiveProcessorCount = (kmp_GetActiveProcessorCount_t) GetProcAddress( kernel32, "GetActiveProcessorCount" );
788  __kmp_GetActiveProcessorGroupCount = (kmp_GetActiveProcessorGroupCount_t) GetProcAddress( kernel32, "GetActiveProcessorGroupCount" );
789  __kmp_GetThreadGroupAffinity = (kmp_GetThreadGroupAffinity_t) GetProcAddress( kernel32, "GetThreadGroupAffinity" );
790  __kmp_SetThreadGroupAffinity = (kmp_SetThreadGroupAffinity_t) GetProcAddress( kernel32, "SetThreadGroupAffinity" );
791 
792  KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount = %p\n", __kmp_GetActiveProcessorCount ) );
793  KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorGroupCount = %p\n", __kmp_GetActiveProcessorGroupCount ) );
794  KA_TRACE( 10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity = %p\n", __kmp_GetThreadGroupAffinity ) );
795  KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity = %p\n", __kmp_SetThreadGroupAffinity ) );
796  KA_TRACE( 10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n", sizeof(kmp_affin_mask_t) ) );
797 
798  //
799  // See if group affinity is supported on this system.
800  // If so, calculate the #groups and #procs.
801  //
802  // Group affinity was introduced with Windows* 7 OS and
803  // Windows* Server 2008 R2 OS.
804  //
805  if ( ( __kmp_GetActiveProcessorCount != NULL )
806  && ( __kmp_GetActiveProcessorGroupCount != NULL )
807  && ( __kmp_GetThreadGroupAffinity != NULL )
808  && ( __kmp_SetThreadGroupAffinity != NULL )
809  && ( ( __kmp_num_proc_groups
810  = __kmp_GetActiveProcessorGroupCount() ) > 1 ) ) {
811  //
812  // Calculate the total number of active OS procs.
813  //
814  int i;
815 
816  KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );
817 
818  __kmp_xproc = 0;
819 
820  for ( i = 0; i < __kmp_num_proc_groups; i++ ) {
821  DWORD size = __kmp_GetActiveProcessorCount( i );
822  __kmp_xproc += size;
823  KA_TRACE( 10, ("__kmp_runtime_initialize: proc group %d size = %d\n", i, size ) );
824  }
825  }
826  else {
827  KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );
828  }
829  }
830  }
831  if ( __kmp_num_proc_groups <= 1 ) {
832  GetSystemInfo( & info );
833  __kmp_xproc = info.dwNumberOfProcessors;
834  }
835 #else
836  GetSystemInfo( & info );
837  __kmp_xproc = info.dwNumberOfProcessors;
838 #endif /* KMP_GROUP_AFFINITY */
839 
840  //
841  // If the OS said there were 0 procs, take a guess and use a value of 2.
842  // This is done for Linux* OS, also. Do we need error / warning?
843  //
844  if ( __kmp_xproc <= 0 ) {
845  __kmp_xproc = 2;
846  }
847 
848  KA_TRACE( 5, ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc) );
849 
850  __kmp_str_buf_free( & path );
851 
852 #if USE_ITT_BUILD
853  __kmp_itt_initialize();
854 #endif /* USE_ITT_BUILD */
855 
856  __kmp_init_runtime = TRUE;
857 } // __kmp_runtime_initialize
858 
859 void
860 __kmp_runtime_destroy( void )
861 {
862  if ( ! __kmp_init_runtime ) {
863  return;
864  }
865 
866 #if USE_ITT_BUILD
867  __kmp_itt_destroy();
868 #endif /* USE_ITT_BUILD */
869 
870  /* we can't DeleteCriticalsection( & __kmp_win32_section ); */
871  /* due to the KX_TRACE() commands */
872  KA_TRACE( 40, ("__kmp_runtime_destroy\n" ));
873 
874  if( __kmp_gtid_threadprivate_key ) {
875  TlsFree( __kmp_gtid_threadprivate_key );
876  __kmp_gtid_threadprivate_key = 0;
877  }
878 
879  __kmp_affinity_uninitialize();
880  DeleteCriticalSection( & __kmp_win32_section );
881 
882  ntdll = NULL;
883  NtQuerySystemInformation = NULL;
884 
885 #if KMP_ARCH_X86_64
886  kernel32 = NULL;
887  __kmp_GetActiveProcessorCount = NULL;
888  __kmp_GetActiveProcessorGroupCount = NULL;
889  __kmp_GetThreadGroupAffinity = NULL;
890  __kmp_SetThreadGroupAffinity = NULL;
891 #endif // KMP_ARCH_X86_64
892 
893  __kmp_init_runtime = FALSE;
894 }
895 
896 
897 void
898 __kmp_terminate_thread( int gtid )
899 {
900  kmp_info_t *th = __kmp_threads[ gtid ];
901 
902  if( !th ) return;
903 
904  KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) );
905 
906  if (TerminateThread( th->th.th_info.ds.ds_thread, (DWORD) -1) == FALSE) {
907  /* It's OK, the thread may have exited already */
908  }
909  __kmp_free_handle( th->th.th_info.ds.ds_thread );
910 }
911 
912 /* ------------------------------------------------------------------------ */
913 /* ------------------------------------------------------------------------ */
914 
915 void
916 __kmp_clear_system_time( void )
917 {
918  BOOL status;
919  LARGE_INTEGER time;
920  status = QueryPerformanceCounter( & time );
921  __kmp_win32_time = (kmp_int64) time.QuadPart;
922 }
923 
924 void
925 __kmp_initialize_system_tick( void )
926 {
927  {
928  BOOL status;
929  LARGE_INTEGER freq;
930 
931  status = QueryPerformanceFrequency( & freq );
932  if (! status) {
933  DWORD error = GetLastError();
934  __kmp_msg(
935  kmp_ms_fatal,
936  KMP_MSG( FunctionError, "QueryPerformanceFrequency()" ),
937  KMP_ERR( error ),
938  __kmp_msg_null
939  );
940 
941  }
942  else {
943  __kmp_win32_tick = ((double) 1.0) / (double) freq.QuadPart;
944  }
945  }
946 }
947 
948 /* Calculate the elapsed wall clock time for the user */
949 
950 void
951 __kmp_elapsed( double *t )
952 {
953  BOOL status;
954  LARGE_INTEGER now;
955  status = QueryPerformanceCounter( & now );
956  *t = ((double) now.QuadPart) * __kmp_win32_tick;
957 }
958 
959 /* Calculate the elapsed wall clock tick for the user */
960 
961 void
962 __kmp_elapsed_tick( double *t )
963 {
964  *t = __kmp_win32_tick;
965 }
966 
967 void
968 __kmp_read_system_time( double *delta )
969 {
970  if (delta != NULL) {
971  BOOL status;
972  LARGE_INTEGER now;
973 
974  status = QueryPerformanceCounter( & now );
975 
976  *delta = ((double) (((kmp_int64) now.QuadPart) - __kmp_win32_time))
977  * __kmp_win32_tick;
978  }
979 }
980 
981 /* Return the current time stamp in nsec */
982 kmp_uint64
983 __kmp_now_nsec()
984 {
985  LARGE_INTEGER now;
986  QueryPerformanceCounter(&now);
987  return 1e9 * __kmp_win32_tick * now.QuadPart;
988 }
989 
990 /* ------------------------------------------------------------------------ */
991 /* ------------------------------------------------------------------------ */
992 
993 void * __stdcall
994 __kmp_launch_worker( void *arg )
995 {
996  volatile void *stack_data;
997  void *exit_val;
998  void *padding = 0;
999  kmp_info_t *this_thr = (kmp_info_t *) arg;
1000  int gtid;
1001 
1002  gtid = this_thr->th.th_info.ds.ds_gtid;
1003  __kmp_gtid_set_specific( gtid );
1004 #ifdef KMP_TDATA_GTID
1005  #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1006  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1007  "reference: http://support.microsoft.com/kb/118816"
1008  //__kmp_gtid = gtid;
1009 #endif
1010 
1011 #if USE_ITT_BUILD
1012  __kmp_itt_thread_name( gtid );
1013 #endif /* USE_ITT_BUILD */
1014 
1015  __kmp_affinity_set_init_mask( gtid, FALSE );
1016 
1017 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1018  //
1019  // Set the FP control regs to be a copy of
1020  // the parallel initialization thread's.
1021  //
1022  __kmp_clear_x87_fpu_status_word();
1023  __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
1024  __kmp_load_mxcsr( &__kmp_init_mxcsr );
1025 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1026 
1027  if ( __kmp_stkoffset > 0 && gtid > 0 ) {
1028  padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
1029  }
1030 
1031  KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
1032  this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1033  TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
1034 
1035  if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid
1036  TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
1037  KMP_ASSERT( this_thr -> th.th_info.ds.ds_stackgrow == FALSE );
1038  __kmp_check_stack_overlap( this_thr );
1039  }
1040  KMP_MB();
1041  exit_val = __kmp_launch_thread( this_thr );
1042  KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
1043  TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
1044  KMP_MB();
1045  return exit_val;
1046 }
1047 
1048 #if KMP_USE_MONITOR
1049 /* The monitor thread controls all of the threads in the complex */
1050 
1051 void * __stdcall
1052 __kmp_launch_monitor( void *arg )
1053 {
1054  DWORD wait_status;
1055  kmp_thread_t monitor;
1056  int status;
1057  int interval;
1058  kmp_info_t *this_thr = (kmp_info_t *) arg;
1059 
1060  KMP_DEBUG_ASSERT(__kmp_init_monitor);
1061  TCW_4( __kmp_init_monitor, 2 ); // AC: Signal the library that monitor has started
1062  // TODO: hide "2" in enum (like {true,false,started})
1063  this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1064  TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
1065 
1066  KMP_MB(); /* Flush all pending memory write invalidates. */
1067  KA_TRACE( 10, ("__kmp_launch_monitor: launched\n" ) );
1068 
1069  monitor = GetCurrentThread();
1070 
1071  /* set thread priority */
1072  status = SetThreadPriority( monitor, THREAD_PRIORITY_HIGHEST );
1073  if (! status) {
1074  DWORD error = GetLastError();
1075  __kmp_msg(
1076  kmp_ms_fatal,
1077  KMP_MSG( CantSetThreadPriority ),
1078  KMP_ERR( error ),
1079  __kmp_msg_null
1080  );
1081  }
1082 
1083  /* register us as monitor */
1084  __kmp_gtid_set_specific( KMP_GTID_MONITOR );
1085 #ifdef KMP_TDATA_GTID
1086  #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1087  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1088  "reference: http://support.microsoft.com/kb/118816"
1089  //__kmp_gtid = KMP_GTID_MONITOR;
1090 #endif
1091 
1092 #if USE_ITT_BUILD
1093  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore monitor thread.
1094 #endif /* USE_ITT_BUILD */
1095 
1096  KMP_MB(); /* Flush all pending memory write invalidates. */
1097 
1098  interval = ( 1000 / __kmp_monitor_wakeups ); /* in milliseconds */
1099 
1100  while (! TCR_4(__kmp_global.g.g_done)) {
1101  /* This thread monitors the state of the system */
1102 
1103  KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) );
1104 
1105  wait_status = WaitForSingleObject( __kmp_monitor_ev, interval );
1106 
1107  if (wait_status == WAIT_TIMEOUT) {
1108  TCW_4( __kmp_global.g.g_time.dt.t_value,
1109  TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
1110  }
1111 
1112  KMP_MB(); /* Flush all pending memory write invalidates. */
1113  }
1114 
1115  KA_TRACE( 10, ("__kmp_launch_monitor: finished\n" ) );
1116 
1117  status = SetThreadPriority( monitor, THREAD_PRIORITY_NORMAL );
1118  if (! status) {
1119  DWORD error = GetLastError();
1120  __kmp_msg(
1121  kmp_ms_fatal,
1122  KMP_MSG( CantSetThreadPriority ),
1123  KMP_ERR( error ),
1124  __kmp_msg_null
1125  );
1126  }
1127 
1128  if (__kmp_global.g.g_abort != 0) {
1129  /* now we need to terminate the worker threads */
1130  /* the value of t_abort is the signal we caught */
1131 
1132  int gtid;
1133 
1134  KA_TRACE( 10, ("__kmp_launch_monitor: terminate sig=%d\n", (__kmp_global.g.g_abort) ) );
1135 
1136  /* terminate the OpenMP worker threads */
1137  /* TODO this is not valid for sibling threads!!
1138  * the uber master might not be 0 anymore.. */
1139  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1140  __kmp_terminate_thread( gtid );
1141 
1142  __kmp_cleanup();
1143 
1144  Sleep( 0 );
1145 
1146  KA_TRACE( 10, ("__kmp_launch_monitor: raise sig=%d\n", (__kmp_global.g.g_abort) ) );
1147 
1148  if (__kmp_global.g.g_abort > 0) {
1149  raise( __kmp_global.g.g_abort );
1150  }
1151  }
1152 
1153  TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
1154 
1155  KMP_MB();
1156  return arg;
1157 }
1158 #endif
1159 
1160 void
1161 __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
1162 {
1163  kmp_thread_t handle;
1164  DWORD idThread;
1165 
1166  KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1167 
1168  th->th.th_info.ds.ds_gtid = gtid;
1169 
1170  if ( KMP_UBER_GTID(gtid) ) {
1171  int stack_data;
1172 
1173  /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for other threads to use.
1174  Is it appropriate to just use GetCurrentThread? When should we close this handle? When
1175  unregistering the root?
1176  */
1177  {
1178  BOOL rc;
1179  rc = DuplicateHandle(
1180  GetCurrentProcess(),
1181  GetCurrentThread(),
1182  GetCurrentProcess(),
1183  &th->th.th_info.ds.ds_thread,
1184  0,
1185  FALSE,
1186  DUPLICATE_SAME_ACCESS
1187  );
1188  KMP_ASSERT( rc );
1189  KA_TRACE( 10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, handle = %" KMP_UINTPTR_SPEC "\n",
1190  (LPVOID)th,
1191  th->th.th_info.ds.ds_thread ) );
1192  th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1193  }
1194  if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid
1195  /* we will dynamically update the stack range if gtid_mode == 1 */
1196  TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1197  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1198  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1199  __kmp_check_stack_overlap( th );
1200  }
1201  }
1202  else {
1203  KMP_MB(); /* Flush all pending memory write invalidates. */
1204 
1205  /* Set stack size for this thread now. */
1206  KA_TRACE( 10, ( "__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC
1207  " bytes\n", stack_size ) );
1208 
1209  stack_size += gtid * __kmp_stkoffset;
1210 
1211  TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1212  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1213 
1214  KA_TRACE( 10, ( "__kmp_create_worker: (before) stack_size = %"
1215  KMP_SIZE_T_SPEC
1216  " bytes, &__kmp_launch_worker = %p, th = %p, "
1217  "&idThread = %p\n",
1218  (SIZE_T) stack_size,
1219  (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
1220  (LPVOID) th, &idThread ) );
1221 
1222  handle = CreateThread( NULL, (SIZE_T) stack_size,
1223  (LPTHREAD_START_ROUTINE) __kmp_launch_worker,
1224  (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
1225 
1226  KA_TRACE( 10, ( "__kmp_create_worker: (after) stack_size = %"
1227  KMP_SIZE_T_SPEC
1228  " bytes, &__kmp_launch_worker = %p, th = %p, "
1229  "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
1230  (SIZE_T) stack_size,
1231  (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
1232  (LPVOID) th, idThread, handle ) );
1233 
1234  if ( handle == 0 ) {
1235  DWORD error = GetLastError();
1236  __kmp_msg(kmp_ms_fatal, KMP_MSG( CantCreateThread ), KMP_ERR( error ), __kmp_msg_null);
1237  } else {
1238  th->th.th_info.ds.ds_thread = handle;
1239  }
1240 
1241  KMP_MB(); /* Flush all pending memory write invalidates. */
1242  }
1243 
1244  KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1245 }
1246 
1247 int
1248 __kmp_still_running(kmp_info_t *th) {
1249  return (WAIT_TIMEOUT == WaitForSingleObject( th->th.th_info.ds.ds_thread, 0));
1250 }
1251 
1252 #if KMP_USE_MONITOR
1253 void
1254 __kmp_create_monitor( kmp_info_t *th )
1255 {
1256  kmp_thread_t handle;
1257  DWORD idThread;
1258  int ideal, new_ideal;
1259 
1260  if( __kmp_dflt_blocktime == KMP_MAX_BLOCKTIME ) {
1261  // We don't need monitor thread in case of MAX_BLOCKTIME
1262  KA_TRACE( 10, ("__kmp_create_monitor: skipping monitor thread because of MAX blocktime\n" ) );
1263  th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
1264  th->th.th_info.ds.ds_gtid = 0;
1265  TCW_4( __kmp_init_monitor, 2 ); // Signal to stop waiting for monitor creation
1266  return;
1267  }
1268  KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) );
1269 
1270  KMP_MB(); /* Flush all pending memory write invalidates. */
1271 
1272  __kmp_monitor_ev = CreateEvent( NULL, TRUE, FALSE, NULL );
1273  if ( __kmp_monitor_ev == NULL ) {
1274  DWORD error = GetLastError();
1275  __kmp_msg(
1276  kmp_ms_fatal,
1277  KMP_MSG( CantCreateEvent ),
1278  KMP_ERR( error ),
1279  __kmp_msg_null
1280  );
1281  }; // if
1282 #if USE_ITT_BUILD
1283  __kmp_itt_system_object_created( __kmp_monitor_ev, "Event" );
1284 #endif /* USE_ITT_BUILD */
1285 
1286  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1287  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1288 
1289  // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
1290  // to automatically expand stacksize based on CreateThread error code.
1291  if ( __kmp_monitor_stksize == 0 ) {
1292  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1293  }
1294  if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1295  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1296  }
1297 
1298  KA_TRACE( 10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
1299  (int) __kmp_monitor_stksize ) );
1300 
1301  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1302 
1303  handle = CreateThread( NULL, (SIZE_T) __kmp_monitor_stksize,
1304  (LPTHREAD_START_ROUTINE) __kmp_launch_monitor,
1305  (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
1306  if (handle == 0) {
1307  DWORD error = GetLastError();
1308  __kmp_msg(
1309  kmp_ms_fatal,
1310  KMP_MSG( CantCreateThread ),
1311  KMP_ERR( error ),
1312  __kmp_msg_null
1313  );
1314  }
1315  else
1316  th->th.th_info.ds.ds_thread = handle;
1317 
1318  KMP_MB(); /* Flush all pending memory write invalidates. */
1319 
1320  KA_TRACE( 10, ("__kmp_create_monitor: monitor created %p\n",
1321  (void *) th->th.th_info.ds.ds_thread ) );
1322 }
1323 #endif
1324 
1325 /*
1326  Check to see if thread is still alive.
1327 
1328  NOTE: The ExitProcess(code) system call causes all threads to Terminate
1329  with a exit_val = code. Because of this we can not rely on
1330  exit_val having any particular value. So this routine may
1331  return STILL_ALIVE in exit_val even after the thread is dead.
1332 */
1333 
1334 int
1335 __kmp_is_thread_alive( kmp_info_t * th, DWORD *exit_val )
1336 {
1337  DWORD rc;
1338  rc = GetExitCodeThread( th->th.th_info.ds.ds_thread, exit_val );
1339  if ( rc == 0 ) {
1340  DWORD error = GetLastError();
1341  __kmp_msg(
1342  kmp_ms_fatal,
1343  KMP_MSG( FunctionError, "GetExitCodeThread()" ),
1344  KMP_ERR( error ),
1345  __kmp_msg_null
1346  );
1347  }; // if
1348  return ( *exit_val == STILL_ACTIVE );
1349 }
1350 
1351 
1352 void
1353 __kmp_exit_thread(
1354  int exit_status
1355 ) {
1356  ExitThread( exit_status );
1357 } // __kmp_exit_thread
1358 
1359 /*
1360  This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
1361 */
1362 static void
1363 __kmp_reap_common( kmp_info_t * th )
1364 {
1365  DWORD exit_val;
1366 
1367  KMP_MB(); /* Flush all pending memory write invalidates. */
1368 
1369  KA_TRACE( 10, ( "__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid ) );
1370 
1371  /*
1372  2006-10-19:
1373 
1374  There are two opposite situations:
1375 
1376  1. Windows* OS keep thread alive after it resets ds_alive flag and exits from thread
1377  function. (For example, see C70770/Q394281 "unloading of dll based on OMP is very
1378  slow".)
1379  2. Windows* OS may kill thread before it resets ds_alive flag.
1380 
1381  Right solution seems to be waiting for *either* thread termination *or* ds_alive resetting.
1382 
1383  */
1384 
1385  {
1386  // TODO: This code is very similar to KMP_WAIT_YIELD. Need to generalize KMP_WAIT_YIELD to
1387  // cover this usage also.
1388  void * obj = NULL;
1389  register kmp_uint32 spins;
1390 #if USE_ITT_BUILD
1391  KMP_FSYNC_SPIN_INIT( obj, (void*) & th->th.th_info.ds.ds_alive );
1392 #endif /* USE_ITT_BUILD */
1393  KMP_INIT_YIELD( spins );
1394  do {
1395 #if USE_ITT_BUILD
1396  KMP_FSYNC_SPIN_PREPARE( obj );
1397 #endif /* USE_ITT_BUILD */
1398  __kmp_is_thread_alive( th, &exit_val );
1399  KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc );
1400  KMP_YIELD_SPIN( spins );
1401  } while ( exit_val == STILL_ACTIVE && TCR_4( th->th.th_info.ds.ds_alive ) );
1402 #if USE_ITT_BUILD
1403  if ( exit_val == STILL_ACTIVE ) {
1404  KMP_FSYNC_CANCEL( obj );
1405  } else {
1406  KMP_FSYNC_SPIN_ACQUIRED( obj );
1407  }; // if
1408 #endif /* USE_ITT_BUILD */
1409  }
1410 
1411  __kmp_free_handle( th->th.th_info.ds.ds_thread );
1412 
1413  /*
1414  * NOTE: The ExitProcess(code) system call causes all threads to Terminate
1415  * with a exit_val = code. Because of this we can not rely on
1416  * exit_val having any particular value.
1417  */
1418  if ( exit_val == STILL_ACTIVE ) {
1419  KA_TRACE( 1, ( "__kmp_reap_common: thread still active.\n" ) );
1420  } else if ( (void *) exit_val != (void *) th) {
1421  KA_TRACE( 1, ( "__kmp_reap_common: ExitProcess / TerminateThread used?\n" ) );
1422  }; // if
1423 
1424  KA_TRACE( 10,
1425  (
1426  "__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC "\n",
1427  th->th.th_info.ds.ds_gtid,
1428  th->th.th_info.ds.ds_thread
1429  )
1430  );
1431 
1432  th->th.th_info.ds.ds_thread = 0;
1433  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1434  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1435  th->th.th_info.ds.ds_thread_id = 0;
1436 
1437  KMP_MB(); /* Flush all pending memory write invalidates. */
1438 }
1439 
1440 #if KMP_USE_MONITOR
1441 void
1442 __kmp_reap_monitor( kmp_info_t *th )
1443 {
1444  int status;
1445 
1446  KA_TRACE( 10, ("__kmp_reap_monitor: try to reap %p\n",
1447  (void *) th->th.th_info.ds.ds_thread ) );
1448 
1449  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1450  // If both tid and gtid are 0, it means the monitor did not ever start.
1451  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1452  KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1453  if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1454  KA_TRACE( 10, ("__kmp_reap_monitor: monitor did not start, returning\n") );
1455  return;
1456  }; // if
1457 
1458  KMP_MB(); /* Flush all pending memory write invalidates. */
1459 
1460  status = SetEvent( __kmp_monitor_ev );
1461  if ( status == FALSE ) {
1462  DWORD error = GetLastError();
1463  __kmp_msg(
1464  kmp_ms_fatal,
1465  KMP_MSG( CantSetEvent ),
1466  KMP_ERR( error ),
1467  __kmp_msg_null
1468  );
1469  }
1470  KA_TRACE( 10, ( "__kmp_reap_monitor: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
1471  __kmp_reap_common( th );
1472 
1473  __kmp_free_handle( __kmp_monitor_ev );
1474 
1475  KMP_MB(); /* Flush all pending memory write invalidates. */
1476 }
1477 #endif
1478 
1479 void
1480 __kmp_reap_worker( kmp_info_t * th )
1481 {
1482  KA_TRACE( 10, ( "__kmp_reap_worker: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
1483  __kmp_reap_common( th );
1484 }
1485 
1486 /* ------------------------------------------------------------------------ */
1487 /* ------------------------------------------------------------------------ */
1488 
1489 #if KMP_HANDLE_SIGNALS
1490 
1491 
1492 static void
1493 __kmp_team_handler( int signo )
1494 {
1495  if ( __kmp_global.g.g_abort == 0 ) {
1496  // Stage 1 signal handler, let's shut down all of the threads.
1497  if ( __kmp_debug_buf ) {
1498  __kmp_dump_debug_buffer();
1499  }; // if
1500  KMP_MB(); // Flush all pending memory write invalidates.
1501  TCW_4( __kmp_global.g.g_abort, signo );
1502  KMP_MB(); // Flush all pending memory write invalidates.
1503  TCW_4( __kmp_global.g.g_done, TRUE );
1504  KMP_MB(); // Flush all pending memory write invalidates.
1505  }
1506 } // __kmp_team_handler
1507 
1508 
1509 
1510 static
1511 sig_func_t __kmp_signal( int signum, sig_func_t handler ) {
1512  sig_func_t old = signal( signum, handler );
1513  if ( old == SIG_ERR ) {
1514  int error = errno;
1515  __kmp_msg( kmp_ms_fatal, KMP_MSG( FunctionError, "signal" ), KMP_ERR( error ), __kmp_msg_null );
1516  }; // if
1517  return old;
1518 }
1519 
1520 static void
1521 __kmp_install_one_handler(
1522  int sig,
1523  sig_func_t handler,
1524  int parallel_init
1525 ) {
1526  sig_func_t old;
1527  KMP_MB(); /* Flush all pending memory write invalidates. */
1528  KB_TRACE( 60, ("__kmp_install_one_handler: called: sig=%d\n", sig ) );
1529  if ( parallel_init ) {
1530  old = __kmp_signal( sig, handler );
1531  // SIG_DFL on Windows* OS in NULL or 0.
1532  if ( old == __kmp_sighldrs[ sig ] ) {
1533  __kmp_siginstalled[ sig ] = 1;
1534  } else {
1535  // Restore/keep user's handler if one previously installed.
1536  old = __kmp_signal( sig, old );
1537  }; // if
1538  } else {
1539  // Save initial/system signal handlers to see if user handlers installed.
1540  // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals called once with
1541  // parallel_init == TRUE.
1542  old = __kmp_signal( sig, SIG_DFL );
1543  __kmp_sighldrs[ sig ] = old;
1544  __kmp_signal( sig, old );
1545  }; // if
1546  KMP_MB(); /* Flush all pending memory write invalidates. */
1547 } // __kmp_install_one_handler
1548 
1549 static void
1550 __kmp_remove_one_handler( int sig ) {
1551  if ( __kmp_siginstalled[ sig ] ) {
1552  sig_func_t old;
1553  KMP_MB(); // Flush all pending memory write invalidates.
1554  KB_TRACE( 60, ( "__kmp_remove_one_handler: called: sig=%d\n", sig ) );
1555  old = __kmp_signal( sig, __kmp_sighldrs[ sig ] );
1556  if ( old != __kmp_team_handler ) {
1557  KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1558  old = __kmp_signal( sig, old );
1559  }; // if
1560  __kmp_sighldrs[ sig ] = NULL;
1561  __kmp_siginstalled[ sig ] = 0;
1562  KMP_MB(); // Flush all pending memory write invalidates.
1563  }; // if
1564 } // __kmp_remove_one_handler
1565 
1566 
1567 void
1568 __kmp_install_signals( int parallel_init )
1569 {
1570  KB_TRACE( 10, ( "__kmp_install_signals: called\n" ) );
1571  if ( ! __kmp_handle_signals ) {
1572  KB_TRACE( 10, ( "__kmp_install_signals: KMP_HANDLE_SIGNALS is false - handlers not installed\n" ) );
1573  return;
1574  }; // if
1575  __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1576  __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1577  __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1578  __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1579  __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1580  __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1581 } // __kmp_install_signals
1582 
1583 
1584 void
1585 __kmp_remove_signals( void )
1586 {
1587  int sig;
1588  KB_TRACE( 10, ("__kmp_remove_signals: called\n" ) );
1589  for ( sig = 1; sig < NSIG; ++ sig ) {
1590  __kmp_remove_one_handler( sig );
1591  }; // for sig
1592 } // __kmp_remove_signals
1593 
1594 
1595 #endif // KMP_HANDLE_SIGNALS
1596 
1597 /* Put the thread to sleep for a time period */
1598 void
1599 __kmp_thread_sleep( int millis )
1600 {
1601  DWORD status;
1602 
1603  status = SleepEx( (DWORD) millis, FALSE );
1604  if ( status ) {
1605  DWORD error = GetLastError();
1606  __kmp_msg(
1607  kmp_ms_fatal,
1608  KMP_MSG( FunctionError, "SleepEx()" ),
1609  KMP_ERR( error ),
1610  __kmp_msg_null
1611  );
1612  }
1613 }
1614 
1615 /* Determine whether the given address is mapped into the current address space. */
1616 int
1617 __kmp_is_address_mapped( void * addr )
1618 {
1619  DWORD status;
1620  MEMORY_BASIC_INFORMATION lpBuffer;
1621  SIZE_T dwLength;
1622 
1623  dwLength = sizeof(MEMORY_BASIC_INFORMATION);
1624 
1625  status = VirtualQuery( addr, &lpBuffer, dwLength );
1626 
1627  return !((( lpBuffer.State == MEM_RESERVE) || ( lpBuffer.State == MEM_FREE )) ||
1628  (( lpBuffer.Protect == PAGE_NOACCESS ) || ( lpBuffer.Protect == PAGE_EXECUTE )));
1629 }
1630 
1631 kmp_uint64
1632 __kmp_hardware_timestamp(void)
1633 {
1634  kmp_uint64 r = 0;
1635 
1636  QueryPerformanceCounter((LARGE_INTEGER*) &r);
1637  return r;
1638 }
1639 
1640 /* Free handle and check the error code */
1641 void
1642 __kmp_free_handle( kmp_thread_t tHandle )
1643 {
1644 /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined as HANDLE */
1645  BOOL rc;
1646  rc = CloseHandle( tHandle );
1647  if ( !rc ) {
1648  DWORD error = GetLastError();
1649  __kmp_msg(
1650  kmp_ms_fatal,
1651  KMP_MSG( CantCloseHandle ),
1652  KMP_ERR( error ),
1653  __kmp_msg_null
1654  );
1655  }
1656 }
1657 
1658 int
1659 __kmp_get_load_balance( int max ) {
1660 
1661  static ULONG glb_buff_size = 100 * 1024;
1662 
1663  static int glb_running_threads = 0; /* Saved count of the running threads for the thread balance algortihm */
1664  static double glb_call_time = 0; /* Thread balance algorithm call time */
1665 
1666  int running_threads = 0; // Number of running threads in the system.
1667  NTSTATUS status = 0;
1668  ULONG buff_size = 0;
1669  ULONG info_size = 0;
1670  void * buffer = NULL;
1671  PSYSTEM_PROCESS_INFORMATION spi = NULL;
1672  int first_time = 1;
1673 
1674  double call_time = 0.0; //start, finish;
1675 
1676  __kmp_elapsed( & call_time );
1677 
1678  if ( glb_call_time &&
1679  ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
1680  running_threads = glb_running_threads;
1681  goto finish;
1682  }
1683  glb_call_time = call_time;
1684 
1685  // Do not spend time on running algorithm if we have a permanent error.
1686  if ( NtQuerySystemInformation == NULL ) {
1687  running_threads = -1;
1688  goto finish;
1689  }; // if
1690 
1691  if ( max <= 0 ) {
1692  max = INT_MAX;
1693  }; // if
1694 
1695  do {
1696 
1697  if ( first_time ) {
1698  buff_size = glb_buff_size;
1699  } else {
1700  buff_size = 2 * buff_size;
1701  }
1702 
1703  buffer = KMP_INTERNAL_REALLOC( buffer, buff_size );
1704  if ( buffer == NULL ) {
1705  running_threads = -1;
1706  goto finish;
1707  }; // if
1708  status = NtQuerySystemInformation( SystemProcessInformation, buffer, buff_size, & info_size );
1709  first_time = 0;
1710 
1711  } while ( status == STATUS_INFO_LENGTH_MISMATCH );
1712  glb_buff_size = buff_size;
1713 
1714  #define CHECK( cond ) \
1715  { \
1716  KMP_DEBUG_ASSERT( cond ); \
1717  if ( ! ( cond ) ) { \
1718  running_threads = -1; \
1719  goto finish; \
1720  } \
1721  }
1722 
1723  CHECK( buff_size >= info_size );
1724  spi = PSYSTEM_PROCESS_INFORMATION( buffer );
1725  for ( ; ; ) {
1726  ptrdiff_t offset = uintptr_t( spi ) - uintptr_t( buffer );
1727  CHECK( 0 <= offset && offset + sizeof( SYSTEM_PROCESS_INFORMATION ) < info_size );
1728  HANDLE pid = spi->ProcessId;
1729  ULONG num = spi->NumberOfThreads;
1730  CHECK( num >= 1 );
1731  size_t spi_size = sizeof( SYSTEM_PROCESS_INFORMATION ) + sizeof( SYSTEM_THREAD ) * ( num - 1 );
1732  CHECK( offset + spi_size < info_size ); // Make sure process info record fits the buffer.
1733  if ( spi->NextEntryOffset != 0 ) {
1734  CHECK( spi_size <= spi->NextEntryOffset ); // And do not overlap with the next record.
1735  }; // if
1736  // pid == 0 corresponds to the System Idle Process. It always has running threads
1737  // on all cores. So, we don't consider the running threads of this process.
1738  if ( pid != 0 ) {
1739  for ( int i = 0; i < num; ++ i ) {
1740  THREAD_STATE state = spi->Threads[ i ].State;
1741  // Count threads that have Ready or Running state.
1742  // !!! TODO: Why comment does not match the code???
1743  if ( state == StateRunning ) {
1744  ++ running_threads;
1745  // Stop counting running threads if the number is already greater than
1746  // the number of available cores
1747  if ( running_threads >= max ) {
1748  goto finish;
1749  }
1750  } // if
1751  }; // for i
1752  } // if
1753  if ( spi->NextEntryOffset == 0 ) {
1754  break;
1755  }; // if
1756  spi = PSYSTEM_PROCESS_INFORMATION( uintptr_t( spi ) + spi->NextEntryOffset );
1757  }; // forever
1758 
1759  #undef CHECK
1760 
1761  finish: // Clean up and exit.
1762 
1763  if ( buffer != NULL ) {
1764  KMP_INTERNAL_FREE( buffer );
1765  }; // if
1766 
1767  glb_running_threads = running_threads;
1768 
1769  return running_threads;
1770 
1771 } //__kmp_get_load_balance()
1772