LLVM OpenMP* Runtime Library
z_Windows_NT_util.cpp
1 /*
2  * z_Windows_NT_util.cpp -- platform specific routines.
3  */
4 
5 //===----------------------------------------------------------------------===//
6 //
7 // The LLVM Compiler Infrastructure
8 //
9 // This file is dual licensed under the MIT and the University of Illinois Open
10 // Source Licenses. See LICENSE.txt for details.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "kmp.h"
15 #include "kmp_affinity.h"
16 #include "kmp_i18n.h"
17 #include "kmp_io.h"
18 #include "kmp_itt.h"
19 #include "kmp_wait_release.h"
20 
21 /* This code is related to NtQuerySystemInformation() function. This function
22  is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
23  number of running threads in the system. */
24 
25 #include <ntsecapi.h> // UNICODE_STRING
26 #include <ntstatus.h>
27 
28 enum SYSTEM_INFORMATION_CLASS {
29  SystemProcessInformation = 5
30 }; // SYSTEM_INFORMATION_CLASS
31 
32 struct CLIENT_ID {
33  HANDLE UniqueProcess;
34  HANDLE UniqueThread;
35 }; // struct CLIENT_ID
36 
37 enum THREAD_STATE {
38  StateInitialized,
39  StateReady,
40  StateRunning,
41  StateStandby,
42  StateTerminated,
43  StateWait,
44  StateTransition,
45  StateUnknown
46 }; // enum THREAD_STATE
47 
48 struct VM_COUNTERS {
49  SIZE_T PeakVirtualSize;
50  SIZE_T VirtualSize;
51  ULONG PageFaultCount;
52  SIZE_T PeakWorkingSetSize;
53  SIZE_T WorkingSetSize;
54  SIZE_T QuotaPeakPagedPoolUsage;
55  SIZE_T QuotaPagedPoolUsage;
56  SIZE_T QuotaPeakNonPagedPoolUsage;
57  SIZE_T QuotaNonPagedPoolUsage;
58  SIZE_T PagefileUsage;
59  SIZE_T PeakPagefileUsage;
60  SIZE_T PrivatePageCount;
61 }; // struct VM_COUNTERS
62 
63 struct SYSTEM_THREAD {
64  LARGE_INTEGER KernelTime;
65  LARGE_INTEGER UserTime;
66  LARGE_INTEGER CreateTime;
67  ULONG WaitTime;
68  LPVOID StartAddress;
69  CLIENT_ID ClientId;
70  DWORD Priority;
71  LONG BasePriority;
72  ULONG ContextSwitchCount;
73  THREAD_STATE State;
74  ULONG WaitReason;
75 }; // SYSTEM_THREAD
76 
77 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, KernelTime) == 0);
78 #if KMP_ARCH_X86
79 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 28);
80 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 52);
81 #else
82 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 32);
83 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 68);
84 #endif
85 
86 struct SYSTEM_PROCESS_INFORMATION {
87  ULONG NextEntryOffset;
88  ULONG NumberOfThreads;
89  LARGE_INTEGER Reserved[3];
90  LARGE_INTEGER CreateTime;
91  LARGE_INTEGER UserTime;
92  LARGE_INTEGER KernelTime;
93  UNICODE_STRING ImageName;
94  DWORD BasePriority;
95  HANDLE ProcessId;
96  HANDLE ParentProcessId;
97  ULONG HandleCount;
98  ULONG Reserved2[2];
99  VM_COUNTERS VMCounters;
100  IO_COUNTERS IOCounters;
101  SYSTEM_THREAD Threads[1];
102 }; // SYSTEM_PROCESS_INFORMATION
103 typedef SYSTEM_PROCESS_INFORMATION *PSYSTEM_PROCESS_INFORMATION;
104 
105 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, NextEntryOffset) == 0);
106 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, CreateTime) == 32);
107 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ImageName) == 56);
108 #if KMP_ARCH_X86
109 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 68);
110 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 76);
111 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 88);
112 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 136);
113 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 184);
114 #else
115 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 80);
116 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 96);
117 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 112);
118 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 208);
119 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 256);
120 #endif
121 
122 typedef NTSTATUS(NTAPI *NtQuerySystemInformation_t)(SYSTEM_INFORMATION_CLASS,
123  PVOID, ULONG, PULONG);
124 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
125 
126 HMODULE ntdll = NULL;
127 
128 /* End of NtQuerySystemInformation()-related code */
129 
130 static HMODULE kernel32 = NULL;
131 
132 #if KMP_HANDLE_SIGNALS
133 typedef void (*sig_func_t)(int);
134 static sig_func_t __kmp_sighldrs[NSIG];
135 static int __kmp_siginstalled[NSIG];
136 #endif
137 
138 #if KMP_USE_MONITOR
139 static HANDLE __kmp_monitor_ev;
140 #endif
141 static kmp_int64 __kmp_win32_time;
142 double __kmp_win32_tick;
143 
144 int __kmp_init_runtime = FALSE;
145 CRITICAL_SECTION __kmp_win32_section;
146 
147 void __kmp_win32_mutex_init(kmp_win32_mutex_t *mx) {
148  InitializeCriticalSection(&mx->cs);
149 #if USE_ITT_BUILD
150  __kmp_itt_system_object_created(&mx->cs, "Critical Section");
151 #endif /* USE_ITT_BUILD */
152 }
153 
154 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
155  DeleteCriticalSection(&mx->cs);
156 }
157 
158 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
159  EnterCriticalSection(&mx->cs);
160 }
161 
162 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
163  LeaveCriticalSection(&mx->cs);
164 }
165 
166 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
167  cv->waiters_count_ = 0;
168  cv->wait_generation_count_ = 0;
169  cv->release_count_ = 0;
170 
171  /* Initialize the critical section */
172  __kmp_win32_mutex_init(&cv->waiters_count_lock_);
173 
174  /* Create a manual-reset event. */
175  cv->event_ = CreateEvent(NULL, // no security
176  TRUE, // manual-reset
177  FALSE, // non-signaled initially
178  NULL); // unnamed
179 #if USE_ITT_BUILD
180  __kmp_itt_system_object_created(cv->event_, "Event");
181 #endif /* USE_ITT_BUILD */
182 }
183 
184 void __kmp_win32_cond_destroy(kmp_win32_cond_t *cv) {
185  __kmp_win32_mutex_destroy(&cv->waiters_count_lock_);
186  __kmp_free_handle(cv->event_);
187  memset(cv, '\0', sizeof(*cv));
188 }
189 
190 /* TODO associate cv with a team instead of a thread so as to optimize
191  the case where we wake up a whole team */
192 
193 void __kmp_win32_cond_wait(kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx,
194  kmp_info_t *th, int need_decrease_load) {
195  int my_generation;
196  int last_waiter;
197 
198  /* Avoid race conditions */
199  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
200 
201  /* Increment count of waiters */
202  cv->waiters_count_++;
203 
204  /* Store current generation in our activation record. */
205  my_generation = cv->wait_generation_count_;
206 
207  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
208  __kmp_win32_mutex_unlock(mx);
209 
210  for (;;) {
211  int wait_done;
212 
213  /* Wait until the event is signaled */
214  WaitForSingleObject(cv->event_, INFINITE);
215 
216  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
217 
218  /* Exit the loop when the <cv->event_> is signaled and there are still
219  waiting threads from this <wait_generation> that haven't been released
220  from this wait yet. */
221  wait_done = (cv->release_count_ > 0) &&
222  (cv->wait_generation_count_ != my_generation);
223 
224  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
225 
226  /* there used to be a semicolon after the if statement, it looked like a
227  bug, so i removed it */
228  if (wait_done)
229  break;
230  }
231 
232  __kmp_win32_mutex_lock(mx);
233  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
234 
235  cv->waiters_count_--;
236  cv->release_count_--;
237 
238  last_waiter = (cv->release_count_ == 0);
239 
240  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
241 
242  if (last_waiter) {
243  /* We're the last waiter to be notified, so reset the manual event. */
244  ResetEvent(cv->event_);
245  }
246 }
247 
248 void __kmp_win32_cond_broadcast(kmp_win32_cond_t *cv) {
249  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
250 
251  if (cv->waiters_count_ > 0) {
252  SetEvent(cv->event_);
253  /* Release all the threads in this generation. */
254 
255  cv->release_count_ = cv->waiters_count_;
256 
257  /* Start a new generation. */
258  cv->wait_generation_count_++;
259  }
260 
261  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
262 }
263 
264 void __kmp_win32_cond_signal(kmp_win32_cond_t *cv) {
265  __kmp_win32_cond_broadcast(cv);
266 }
267 
268 void __kmp_enable(int new_state) {
269  if (__kmp_init_runtime)
270  LeaveCriticalSection(&__kmp_win32_section);
271 }
272 
273 void __kmp_disable(int *old_state) {
274  *old_state = 0;
275 
276  if (__kmp_init_runtime)
277  EnterCriticalSection(&__kmp_win32_section);
278 }
279 
280 void __kmp_suspend_initialize(void) { /* do nothing */
281 }
282 
283 static void __kmp_suspend_initialize_thread(kmp_info_t *th) {
284  if (!TCR_4(th->th.th_suspend_init)) {
285  /* this means we haven't initialized the suspension pthread objects for this
286  thread in this instance of the process */
287  __kmp_win32_cond_init(&th->th.th_suspend_cv);
288  __kmp_win32_mutex_init(&th->th.th_suspend_mx);
289  TCW_4(th->th.th_suspend_init, TRUE);
290  }
291 }
292 
293 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
294  if (TCR_4(th->th.th_suspend_init)) {
295  /* this means we have initialize the suspension pthread objects for this
296  thread in this instance of the process */
297  __kmp_win32_cond_destroy(&th->th.th_suspend_cv);
298  __kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
299  TCW_4(th->th.th_suspend_init, FALSE);
300  }
301 }
302 
303 /* This routine puts the calling thread to sleep after setting the
304  sleep bit for the indicated flag variable to true. */
305 template <class C>
306 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
307  kmp_info_t *th = __kmp_threads[th_gtid];
308  int status;
309  typename C::flag_t old_spin;
310 
311  KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
312  th_gtid, flag->get()));
313 
314  __kmp_suspend_initialize_thread(th);
315  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
316 
317  KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's"
318  " loc(%p)\n",
319  th_gtid, flag->get()));
320 
321  /* TODO: shouldn't this use release semantics to ensure that
322  __kmp_suspend_initialize_thread gets called first? */
323  old_spin = flag->set_sleeping();
324 
325  KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's"
326  " loc(%p)==%d\n",
327  th_gtid, flag->get(), *(flag->get())));
328 
329  if (flag->done_check_val(old_spin)) {
330  old_spin = flag->unset_sleeping();
331  KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
332  "for flag's loc(%p)\n",
333  th_gtid, flag->get()));
334  } else {
335 #ifdef DEBUG_SUSPEND
336  __kmp_suspend_count++;
337 #endif
338  /* Encapsulate in a loop as the documentation states that this may "with
339  low probability" return when the condition variable has not been signaled
340  or broadcast */
341  int deactivated = FALSE;
342  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
343  while (flag->is_sleeping()) {
344  KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
345  "kmp_win32_cond_wait()\n",
346  th_gtid));
347  // Mark the thread as no longer active (only in the first iteration of the
348  // loop).
349  if (!deactivated) {
350  th->th.th_active = FALSE;
351  if (th->th.th_active_in_pool) {
352  th->th.th_active_in_pool = FALSE;
353  KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
354  KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
355  }
356  deactivated = TRUE;
357 
358  __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, 0,
359  0);
360  } else {
361  __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, 0,
362  0);
363  }
364 
365 #ifdef KMP_DEBUG
366  if (flag->is_sleeping()) {
367  KF_TRACE(100,
368  ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
369  }
370 #endif /* KMP_DEBUG */
371 
372  } // while
373 
374  // Mark the thread as active again (if it was previous marked as inactive)
375  if (deactivated) {
376  th->th.th_active = TRUE;
377  if (TCR_4(th->th.th_in_pool)) {
378  KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
379  th->th.th_active_in_pool = TRUE;
380  }
381  }
382  }
383 
384  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
385 
386  KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
387 }
388 
389 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
390  __kmp_suspend_template(th_gtid, flag);
391 }
392 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
393  __kmp_suspend_template(th_gtid, flag);
394 }
395 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
396  __kmp_suspend_template(th_gtid, flag);
397 }
398 
399 /* This routine signals the thread specified by target_gtid to wake up
400  after setting the sleep bit indicated by the flag argument to FALSE */
401 template <class C>
402 static inline void __kmp_resume_template(int target_gtid, C *flag) {
403  kmp_info_t *th = __kmp_threads[target_gtid];
404  int status;
405 
406 #ifdef KMP_DEBUG
407  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
408 #endif
409 
410  KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
411  gtid, target_gtid));
412 
413  __kmp_suspend_initialize_thread(th);
414  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
415 
416  if (!flag) { // coming from __kmp_null_resume_wrapper
417  flag = (C *)th->th.th_sleep_loc;
418  }
419 
420  // First, check if the flag is null or its type has changed. If so, someone
421  // else woke it up.
422  if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type
423  // simply shows what
424  // flag was cast to
425  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
426  "awake: flag's loc(%p)\n",
427  gtid, target_gtid, NULL));
428  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
429  return;
430  } else {
431  typename C::flag_t old_spin = flag->unset_sleeping();
432  if (!flag->is_sleeping_val(old_spin)) {
433  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
434  "awake: flag's loc(%p): %u => %u\n",
435  gtid, target_gtid, flag->get(), old_spin, *(flag->get())));
436  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
437  return;
438  }
439  }
440  TCW_PTR(th->th.th_sleep_loc, NULL);
441  KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep "
442  "bit for flag's loc(%p)\n",
443  gtid, target_gtid, flag->get()));
444 
445  __kmp_win32_cond_signal(&th->th.th_suspend_cv);
446  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
447 
448  KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
449  " for T#%d\n",
450  gtid, target_gtid));
451 }
452 
453 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
454  __kmp_resume_template(target_gtid, flag);
455 }
456 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
457  __kmp_resume_template(target_gtid, flag);
458 }
459 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
460  __kmp_resume_template(target_gtid, flag);
461 }
462 
463 void __kmp_yield(int cond) {
464  if (cond)
465  Sleep(0);
466 }
467 
468 void __kmp_gtid_set_specific(int gtid) {
469  if (__kmp_init_gtid) {
470  KA_TRACE(50, ("__kmp_gtid_set_specific: T#%d key:%d\n", gtid,
471  __kmp_gtid_threadprivate_key));
472  if (!TlsSetValue(__kmp_gtid_threadprivate_key, (LPVOID)(gtid + 1)))
473  KMP_FATAL(TLSSetValueFailed);
474  } else {
475  KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
476  }
477 }
478 
479 int __kmp_gtid_get_specific() {
480  int gtid;
481  if (!__kmp_init_gtid) {
482  KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
483  "KMP_GTID_SHUTDOWN\n"));
484  return KMP_GTID_SHUTDOWN;
485  }
486  gtid = (int)(kmp_intptr_t)TlsGetValue(__kmp_gtid_threadprivate_key);
487  if (gtid == 0) {
488  gtid = KMP_GTID_DNE;
489  } else {
490  gtid--;
491  }
492  KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
493  __kmp_gtid_threadprivate_key, gtid));
494  return gtid;
495 }
496 
497 void __kmp_affinity_bind_thread(int proc) {
498  if (__kmp_num_proc_groups > 1) {
499  // Form the GROUP_AFFINITY struct directly, rather than filling
500  // out a bit vector and calling __kmp_set_system_affinity().
501  GROUP_AFFINITY ga;
502  KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups * CHAR_BIT *
503  sizeof(DWORD_PTR))));
504  ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
505  ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
506  ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
507 
508  KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
509  if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
510  DWORD error = GetLastError();
511  if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
512  kmp_msg_t err_code = KMP_ERR(error);
513  __kmp_msg(kmp_ms_warning, KMP_MSG(CantSetThreadAffMask), err_code,
514  __kmp_msg_null);
515  if (__kmp_generate_warnings == kmp_warnings_off) {
516  __kmp_str_free(&err_code.str);
517  }
518  }
519  }
520  } else {
521  kmp_affin_mask_t *mask;
522  KMP_CPU_ALLOC_ON_STACK(mask);
523  KMP_CPU_ZERO(mask);
524  KMP_CPU_SET(proc, mask);
525  __kmp_set_system_affinity(mask, TRUE);
526  KMP_CPU_FREE_FROM_STACK(mask);
527  }
528 }
529 
530 void __kmp_affinity_determine_capable(const char *env_var) {
531 // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask().
532 
533 #if KMP_GROUP_AFFINITY
534  KMP_AFFINITY_ENABLE(__kmp_num_proc_groups * sizeof(DWORD_PTR));
535 #else
536  KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR));
537 #endif
538 
539  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
540  "Windows* OS affinity interface functional (mask size = "
541  "%" KMP_SIZE_T_SPEC ").\n",
542  __kmp_affin_mask_size));
543 }
544 
545 double __kmp_read_cpu_time(void) {
546  FILETIME CreationTime, ExitTime, KernelTime, UserTime;
547  int status;
548  double cpu_time;
549 
550  cpu_time = 0;
551 
552  status = GetProcessTimes(GetCurrentProcess(), &CreationTime, &ExitTime,
553  &KernelTime, &UserTime);
554 
555  if (status) {
556  double sec = 0;
557 
558  sec += KernelTime.dwHighDateTime;
559  sec += UserTime.dwHighDateTime;
560 
561  /* Shift left by 32 bits */
562  sec *= (double)(1 << 16) * (double)(1 << 16);
563 
564  sec += KernelTime.dwLowDateTime;
565  sec += UserTime.dwLowDateTime;
566 
567  cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
568  }
569 
570  return cpu_time;
571 }
572 
573 int __kmp_read_system_info(struct kmp_sys_info *info) {
574  info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
575  info->minflt = 0; /* the number of page faults serviced without any I/O */
576  info->majflt = 0; /* the number of page faults serviced that required I/O */
577  info->nswap = 0; // the number of times a process was "swapped" out of memory
578  info->inblock = 0; // the number of times the file system had to perform input
579  info->oublock = 0; // number of times the file system had to perform output
580  info->nvcsw = 0; /* the number of times a context switch was voluntarily */
581  info->nivcsw = 0; /* the number of times a context switch was forced */
582 
583  return 1;
584 }
585 
586 void __kmp_runtime_initialize(void) {
587  SYSTEM_INFO info;
588  kmp_str_buf_t path;
589  UINT path_size;
590 
591  if (__kmp_init_runtime) {
592  return;
593  }
594 
595 #if KMP_DYNAMIC_LIB
596  /* Pin dynamic library for the lifetime of application */
597  {
598  // First, turn off error message boxes
599  UINT err_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
600  HMODULE h;
601  BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
602  GET_MODULE_HANDLE_EX_FLAG_PIN,
603  (LPCTSTR)&__kmp_serial_initialize, &h);
604  KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
605  SetErrorMode(err_mode); // Restore error mode
606  KA_TRACE(10, ("__kmp_runtime_initialize: dynamic library pinned\n"));
607  }
608 #endif
609 
610  InitializeCriticalSection(&__kmp_win32_section);
611 #if USE_ITT_BUILD
612  __kmp_itt_system_object_created(&__kmp_win32_section, "Critical Section");
613 #endif /* USE_ITT_BUILD */
614  __kmp_initialize_system_tick();
615 
616 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
617  if (!__kmp_cpuinfo.initialized) {
618  __kmp_query_cpuid(&__kmp_cpuinfo);
619  }
620 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
621 
622 /* Set up minimum number of threads to switch to TLS gtid */
623 #if KMP_OS_WINDOWS && !defined KMP_DYNAMIC_LIB
624  // Windows* OS, static library.
625  /* New thread may use stack space previously used by another thread,
626  currently terminated. On Windows* OS, in case of static linking, we do not
627  know the moment of thread termination, and our structures (__kmp_threads
628  and __kmp_root arrays) are still keep info about dead threads. This leads
629  to problem in __kmp_get_global_thread_id() function: it wrongly finds gtid
630  (by searching through stack addresses of all known threads) for
631  unregistered foreign tread.
632 
633  Setting __kmp_tls_gtid_min to 0 workarounds this problem:
634  __kmp_get_global_thread_id() does not search through stacks, but get gtid
635  from TLS immediately.
636  --ln
637  */
638  __kmp_tls_gtid_min = 0;
639 #else
640  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
641 #endif
642 
643  /* for the static library */
644  if (!__kmp_gtid_threadprivate_key) {
645  __kmp_gtid_threadprivate_key = TlsAlloc();
646  if (__kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES) {
647  KMP_FATAL(TLSOutOfIndexes);
648  }
649  }
650 
651  // Load ntdll.dll.
652  /* Simple GetModuleHandle( "ntdll.dl" ) is not suitable due to security issue
653  (see http://www.microsoft.com/technet/security/advisory/2269637.mspx). We
654  have to specify full path to the library. */
655  __kmp_str_buf_init(&path);
656  path_size = GetSystemDirectory(path.str, path.size);
657  KMP_DEBUG_ASSERT(path_size > 0);
658  if (path_size >= path.size) {
659  // Buffer is too short. Expand the buffer and try again.
660  __kmp_str_buf_reserve(&path, path_size);
661  path_size = GetSystemDirectory(path.str, path.size);
662  KMP_DEBUG_ASSERT(path_size > 0);
663  }
664  if (path_size > 0 && path_size < path.size) {
665  // Now we have system directory name in the buffer.
666  // Append backslash and name of dll to form full path,
667  path.used = path_size;
668  __kmp_str_buf_print(&path, "\\%s", "ntdll.dll");
669 
670  // Now load ntdll using full path.
671  ntdll = GetModuleHandle(path.str);
672  }
673 
674  KMP_DEBUG_ASSERT(ntdll != NULL);
675  if (ntdll != NULL) {
676  NtQuerySystemInformation = (NtQuerySystemInformation_t)GetProcAddress(
677  ntdll, "NtQuerySystemInformation");
678  }
679  KMP_DEBUG_ASSERT(NtQuerySystemInformation != NULL);
680 
681 #if KMP_GROUP_AFFINITY
682  // Load kernel32.dll.
683  // Same caveat - must use full system path name.
684  if (path_size > 0 && path_size < path.size) {
685  // Truncate the buffer back to just the system path length,
686  // discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
687  path.used = path_size;
688  __kmp_str_buf_print(&path, "\\%s", "kernel32.dll");
689 
690  // Load kernel32.dll using full path.
691  kernel32 = GetModuleHandle(path.str);
692  KA_TRACE(10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str));
693 
694  // Load the function pointers to kernel32.dll routines
695  // that may or may not exist on this system.
696  if (kernel32 != NULL) {
697  __kmp_GetActiveProcessorCount =
698  (kmp_GetActiveProcessorCount_t)GetProcAddress(
699  kernel32, "GetActiveProcessorCount");
700  __kmp_GetActiveProcessorGroupCount =
701  (kmp_GetActiveProcessorGroupCount_t)GetProcAddress(
702  kernel32, "GetActiveProcessorGroupCount");
703  __kmp_GetThreadGroupAffinity =
704  (kmp_GetThreadGroupAffinity_t)GetProcAddress(
705  kernel32, "GetThreadGroupAffinity");
706  __kmp_SetThreadGroupAffinity =
707  (kmp_SetThreadGroupAffinity_t)GetProcAddress(
708  kernel32, "SetThreadGroupAffinity");
709 
710  KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount"
711  " = %p\n",
712  __kmp_GetActiveProcessorCount));
713  KA_TRACE(10, ("__kmp_runtime_initialize: "
714  "__kmp_GetActiveProcessorGroupCount = %p\n",
715  __kmp_GetActiveProcessorGroupCount));
716  KA_TRACE(10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity"
717  " = %p\n",
718  __kmp_GetThreadGroupAffinity));
719  KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity"
720  " = %p\n",
721  __kmp_SetThreadGroupAffinity));
722  KA_TRACE(10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n",
723  sizeof(kmp_affin_mask_t)));
724 
725  // See if group affinity is supported on this system.
726  // If so, calculate the #groups and #procs.
727  //
728  // Group affinity was introduced with Windows* 7 OS and
729  // Windows* Server 2008 R2 OS.
730  if ((__kmp_GetActiveProcessorCount != NULL) &&
731  (__kmp_GetActiveProcessorGroupCount != NULL) &&
732  (__kmp_GetThreadGroupAffinity != NULL) &&
733  (__kmp_SetThreadGroupAffinity != NULL) &&
734  ((__kmp_num_proc_groups = __kmp_GetActiveProcessorGroupCount()) >
735  1)) {
736  // Calculate the total number of active OS procs.
737  int i;
738 
739  KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
740  " detected\n",
741  __kmp_num_proc_groups));
742 
743  __kmp_xproc = 0;
744 
745  for (i = 0; i < __kmp_num_proc_groups; i++) {
746  DWORD size = __kmp_GetActiveProcessorCount(i);
747  __kmp_xproc += size;
748  KA_TRACE(10, ("__kmp_runtime_initialize: proc group %d size = %d\n",
749  i, size));
750  }
751  } else {
752  KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
753  " detected\n",
754  __kmp_num_proc_groups));
755  }
756  }
757  }
758  if (__kmp_num_proc_groups <= 1) {
759  GetSystemInfo(&info);
760  __kmp_xproc = info.dwNumberOfProcessors;
761  }
762 #else
763  GetSystemInfo(&info);
764  __kmp_xproc = info.dwNumberOfProcessors;
765 #endif /* KMP_GROUP_AFFINITY */
766 
767  // If the OS said there were 0 procs, take a guess and use a value of 2.
768  // This is done for Linux* OS, also. Do we need error / warning?
769  if (__kmp_xproc <= 0) {
770  __kmp_xproc = 2;
771  }
772 
773  KA_TRACE(5,
774  ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc));
775 
776  __kmp_str_buf_free(&path);
777 
778 #if USE_ITT_BUILD
779  __kmp_itt_initialize();
780 #endif /* USE_ITT_BUILD */
781 
782  __kmp_init_runtime = TRUE;
783 } // __kmp_runtime_initialize
784 
785 void __kmp_runtime_destroy(void) {
786  if (!__kmp_init_runtime) {
787  return;
788  }
789 
790 #if USE_ITT_BUILD
791  __kmp_itt_destroy();
792 #endif /* USE_ITT_BUILD */
793 
794  /* we can't DeleteCriticalsection( & __kmp_win32_section ); */
795  /* due to the KX_TRACE() commands */
796  KA_TRACE(40, ("__kmp_runtime_destroy\n"));
797 
798  if (__kmp_gtid_threadprivate_key) {
799  TlsFree(__kmp_gtid_threadprivate_key);
800  __kmp_gtid_threadprivate_key = 0;
801  }
802 
803  __kmp_affinity_uninitialize();
804  DeleteCriticalSection(&__kmp_win32_section);
805 
806  ntdll = NULL;
807  NtQuerySystemInformation = NULL;
808 
809 #if KMP_ARCH_X86_64
810  kernel32 = NULL;
811  __kmp_GetActiveProcessorCount = NULL;
812  __kmp_GetActiveProcessorGroupCount = NULL;
813  __kmp_GetThreadGroupAffinity = NULL;
814  __kmp_SetThreadGroupAffinity = NULL;
815 #endif // KMP_ARCH_X86_64
816 
817  __kmp_init_runtime = FALSE;
818 }
819 
820 void __kmp_terminate_thread(int gtid) {
821  kmp_info_t *th = __kmp_threads[gtid];
822 
823  if (!th)
824  return;
825 
826  KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
827 
828  if (TerminateThread(th->th.th_info.ds.ds_thread, (DWORD)-1) == FALSE) {
829  /* It's OK, the thread may have exited already */
830  }
831  __kmp_free_handle(th->th.th_info.ds.ds_thread);
832 }
833 
834 void __kmp_clear_system_time(void) {
835  BOOL status;
836  LARGE_INTEGER time;
837  status = QueryPerformanceCounter(&time);
838  __kmp_win32_time = (kmp_int64)time.QuadPart;
839 }
840 
841 void __kmp_initialize_system_tick(void) {
842  {
843  BOOL status;
844  LARGE_INTEGER freq;
845 
846  status = QueryPerformanceFrequency(&freq);
847  if (!status) {
848  DWORD error = GetLastError();
849  __kmp_fatal(KMP_MSG(FunctionError, "QueryPerformanceFrequency()"),
850  KMP_ERR(error), __kmp_msg_null);
851 
852  } else {
853  __kmp_win32_tick = ((double)1.0) / (double)freq.QuadPart;
854  }
855  }
856 }
857 
858 /* Calculate the elapsed wall clock time for the user */
859 
860 void __kmp_elapsed(double *t) {
861  BOOL status;
862  LARGE_INTEGER now;
863  status = QueryPerformanceCounter(&now);
864  *t = ((double)now.QuadPart) * __kmp_win32_tick;
865 }
866 
867 /* Calculate the elapsed wall clock tick for the user */
868 
869 void __kmp_elapsed_tick(double *t) { *t = __kmp_win32_tick; }
870 
871 void __kmp_read_system_time(double *delta) {
872  if (delta != NULL) {
873  BOOL status;
874  LARGE_INTEGER now;
875 
876  status = QueryPerformanceCounter(&now);
877 
878  *delta = ((double)(((kmp_int64)now.QuadPart) - __kmp_win32_time)) *
879  __kmp_win32_tick;
880  }
881 }
882 
883 /* Return the current time stamp in nsec */
884 kmp_uint64 __kmp_now_nsec() {
885  LARGE_INTEGER now;
886  QueryPerformanceCounter(&now);
887  return 1e9 * __kmp_win32_tick * now.QuadPart;
888 }
889 
890 void *__stdcall __kmp_launch_worker(void *arg) {
891  volatile void *stack_data;
892  void *exit_val;
893  void *padding = 0;
894  kmp_info_t *this_thr = (kmp_info_t *)arg;
895  int gtid;
896 
897  gtid = this_thr->th.th_info.ds.ds_gtid;
898  __kmp_gtid_set_specific(gtid);
899 #ifdef KMP_TDATA_GTID
900 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
901  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
902  "reference: http://support.microsoft.com/kb/118816"
903 //__kmp_gtid = gtid;
904 #endif
905 
906 #if USE_ITT_BUILD
907  __kmp_itt_thread_name(gtid);
908 #endif /* USE_ITT_BUILD */
909 
910  __kmp_affinity_set_init_mask(gtid, FALSE);
911 
912 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
913  // Set FP control regs to be a copy of the parallel initialization thread's.
914  __kmp_clear_x87_fpu_status_word();
915  __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
916  __kmp_load_mxcsr(&__kmp_init_mxcsr);
917 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
918 
919  if (__kmp_stkoffset > 0 && gtid > 0) {
920  padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
921  }
922 
923  KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
924  this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
925  TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
926 
927  if (TCR_4(__kmp_gtid_mode) <
928  2) { // check stack only if it is used to get gtid
929  TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
930  KMP_ASSERT(this_thr->th.th_info.ds.ds_stackgrow == FALSE);
931  __kmp_check_stack_overlap(this_thr);
932  }
933  KMP_MB();
934  exit_val = __kmp_launch_thread(this_thr);
935  KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
936  TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
937  KMP_MB();
938  return exit_val;
939 }
940 
941 #if KMP_USE_MONITOR
942 /* The monitor thread controls all of the threads in the complex */
943 
944 void *__stdcall __kmp_launch_monitor(void *arg) {
945  DWORD wait_status;
946  kmp_thread_t monitor;
947  int status;
948  int interval;
949  kmp_info_t *this_thr = (kmp_info_t *)arg;
950 
951  KMP_DEBUG_ASSERT(__kmp_init_monitor);
952  TCW_4(__kmp_init_monitor, 2); // AC: Signal library that monitor has started
953  // TODO: hide "2" in enum (like {true,false,started})
954  this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
955  TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
956 
957  KMP_MB(); /* Flush all pending memory write invalidates. */
958  KA_TRACE(10, ("__kmp_launch_monitor: launched\n"));
959 
960  monitor = GetCurrentThread();
961 
962  /* set thread priority */
963  status = SetThreadPriority(monitor, THREAD_PRIORITY_HIGHEST);
964  if (!status) {
965  DWORD error = GetLastError();
966  __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
967  }
968 
969  /* register us as monitor */
970  __kmp_gtid_set_specific(KMP_GTID_MONITOR);
971 #ifdef KMP_TDATA_GTID
972 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
973  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
974  "reference: http://support.microsoft.com/kb/118816"
975 //__kmp_gtid = KMP_GTID_MONITOR;
976 #endif
977 
978 #if USE_ITT_BUILD
979  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore
980 // monitor thread.
981 #endif /* USE_ITT_BUILD */
982 
983  KMP_MB(); /* Flush all pending memory write invalidates. */
984 
985  interval = (1000 / __kmp_monitor_wakeups); /* in milliseconds */
986 
987  while (!TCR_4(__kmp_global.g.g_done)) {
988  /* This thread monitors the state of the system */
989 
990  KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
991 
992  wait_status = WaitForSingleObject(__kmp_monitor_ev, interval);
993 
994  if (wait_status == WAIT_TIMEOUT) {
995  TCW_4(__kmp_global.g.g_time.dt.t_value,
996  TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
997  }
998 
999  KMP_MB(); /* Flush all pending memory write invalidates. */
1000  }
1001 
1002  KA_TRACE(10, ("__kmp_launch_monitor: finished\n"));
1003 
1004  status = SetThreadPriority(monitor, THREAD_PRIORITY_NORMAL);
1005  if (!status) {
1006  DWORD error = GetLastError();
1007  __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1008  }
1009 
1010  if (__kmp_global.g.g_abort != 0) {
1011  /* now we need to terminate the worker threads */
1012  /* the value of t_abort is the signal we caught */
1013  int gtid;
1014 
1015  KA_TRACE(10, ("__kmp_launch_monitor: terminate sig=%d\n",
1016  (__kmp_global.g.g_abort)));
1017 
1018  /* terminate the OpenMP worker threads */
1019  /* TODO this is not valid for sibling threads!!
1020  * the uber master might not be 0 anymore.. */
1021  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1022  __kmp_terminate_thread(gtid);
1023 
1024  __kmp_cleanup();
1025 
1026  Sleep(0);
1027 
1028  KA_TRACE(10,
1029  ("__kmp_launch_monitor: raise sig=%d\n", __kmp_global.g.g_abort));
1030 
1031  if (__kmp_global.g.g_abort > 0) {
1032  raise(__kmp_global.g.g_abort);
1033  }
1034  }
1035 
1036  TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
1037 
1038  KMP_MB();
1039  return arg;
1040 }
1041 #endif
1042 
1043 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
1044  kmp_thread_t handle;
1045  DWORD idThread;
1046 
1047  KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
1048 
1049  th->th.th_info.ds.ds_gtid = gtid;
1050 
1051  if (KMP_UBER_GTID(gtid)) {
1052  int stack_data;
1053 
1054  /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for
1055  other threads to use. Is it appropriate to just use GetCurrentThread?
1056  When should we close this handle? When unregistering the root? */
1057  {
1058  BOOL rc;
1059  rc = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
1060  GetCurrentProcess(), &th->th.th_info.ds.ds_thread, 0,
1061  FALSE, DUPLICATE_SAME_ACCESS);
1062  KMP_ASSERT(rc);
1063  KA_TRACE(10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, "
1064  "handle = %" KMP_UINTPTR_SPEC "\n",
1065  (LPVOID)th, th->th.th_info.ds.ds_thread));
1066  th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1067  }
1068  if (TCR_4(__kmp_gtid_mode) < 2) { // check stack only if used to get gtid
1069  /* we will dynamically update the stack range if gtid_mode == 1 */
1070  TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1071  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1072  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1073  __kmp_check_stack_overlap(th);
1074  }
1075  } else {
1076  KMP_MB(); /* Flush all pending memory write invalidates. */
1077 
1078  /* Set stack size for this thread now. */
1079  KA_TRACE(10,
1080  ("__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC " bytes\n",
1081  stack_size));
1082 
1083  stack_size += gtid * __kmp_stkoffset;
1084 
1085  TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1086  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1087 
1088  KA_TRACE(10,
1089  ("__kmp_create_worker: (before) stack_size = %" KMP_SIZE_T_SPEC
1090  " bytes, &__kmp_launch_worker = %p, th = %p, &idThread = %p\n",
1091  (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1092  (LPVOID)th, &idThread));
1093 
1094  handle = CreateThread(
1095  NULL, (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)__kmp_launch_worker,
1096  (LPVOID)th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1097 
1098  KA_TRACE(10,
1099  ("__kmp_create_worker: (after) stack_size = %" KMP_SIZE_T_SPEC
1100  " bytes, &__kmp_launch_worker = %p, th = %p, "
1101  "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
1102  (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1103  (LPVOID)th, idThread, handle));
1104 
1105  if (handle == 0) {
1106  DWORD error = GetLastError();
1107  __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1108  } else {
1109  th->th.th_info.ds.ds_thread = handle;
1110  }
1111 
1112  KMP_MB(); /* Flush all pending memory write invalidates. */
1113  }
1114 
1115  KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
1116 }
1117 
1118 int __kmp_still_running(kmp_info_t *th) {
1119  return (WAIT_TIMEOUT == WaitForSingleObject(th->th.th_info.ds.ds_thread, 0));
1120 }
1121 
1122 #if KMP_USE_MONITOR
1123 void __kmp_create_monitor(kmp_info_t *th) {
1124  kmp_thread_t handle;
1125  DWORD idThread;
1126  int ideal, new_ideal;
1127 
1128  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
1129  // We don't need monitor thread in case of MAX_BLOCKTIME
1130  KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
1131  "MAX blocktime\n"));
1132  th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
1133  th->th.th_info.ds.ds_gtid = 0;
1134  TCW_4(__kmp_init_monitor, 2); // Signal to stop waiting for monitor creation
1135  return;
1136  }
1137  KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
1138 
1139  KMP_MB(); /* Flush all pending memory write invalidates. */
1140 
1141  __kmp_monitor_ev = CreateEvent(NULL, TRUE, FALSE, NULL);
1142  if (__kmp_monitor_ev == NULL) {
1143  DWORD error = GetLastError();
1144  __kmp_fatal(KMP_MSG(CantCreateEvent), KMP_ERR(error), __kmp_msg_null);
1145  }
1146 #if USE_ITT_BUILD
1147  __kmp_itt_system_object_created(__kmp_monitor_ev, "Event");
1148 #endif /* USE_ITT_BUILD */
1149 
1150  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1151  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1152 
1153  // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
1154  // to automatically expand stacksize based on CreateThread error code.
1155  if (__kmp_monitor_stksize == 0) {
1156  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1157  }
1158  if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
1159  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1160  }
1161 
1162  KA_TRACE(10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
1163  (int)__kmp_monitor_stksize));
1164 
1165  TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
1166 
1167  handle =
1168  CreateThread(NULL, (SIZE_T)__kmp_monitor_stksize,
1169  (LPTHREAD_START_ROUTINE)__kmp_launch_monitor, (LPVOID)th,
1170  STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1171  if (handle == 0) {
1172  DWORD error = GetLastError();
1173  __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1174  } else
1175  th->th.th_info.ds.ds_thread = handle;
1176 
1177  KMP_MB(); /* Flush all pending memory write invalidates. */
1178 
1179  KA_TRACE(10, ("__kmp_create_monitor: monitor created %p\n",
1180  (void *)th->th.th_info.ds.ds_thread));
1181 }
1182 #endif
1183 
1184 /* Check to see if thread is still alive.
1185  NOTE: The ExitProcess(code) system call causes all threads to Terminate
1186  with a exit_val = code. Because of this we can not rely on exit_val having
1187  any particular value. So this routine may return STILL_ALIVE in exit_val
1188  even after the thread is dead. */
1189 
1190 int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val) {
1191  DWORD rc;
1192  rc = GetExitCodeThread(th->th.th_info.ds.ds_thread, exit_val);
1193  if (rc == 0) {
1194  DWORD error = GetLastError();
1195  __kmp_fatal(KMP_MSG(FunctionError, "GetExitCodeThread()"), KMP_ERR(error),
1196  __kmp_msg_null);
1197  }
1198  return (*exit_val == STILL_ACTIVE);
1199 }
1200 
1201 void __kmp_exit_thread(int exit_status) {
1202  ExitThread(exit_status);
1203 } // __kmp_exit_thread
1204 
1205 // This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
1206 static void __kmp_reap_common(kmp_info_t *th) {
1207  DWORD exit_val;
1208 
1209  KMP_MB(); /* Flush all pending memory write invalidates. */
1210 
1211  KA_TRACE(
1212  10, ("__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid));
1213 
1214  /* 2006-10-19:
1215  There are two opposite situations:
1216  1. Windows* OS keep thread alive after it resets ds_alive flag and
1217  exits from thread function. (For example, see C70770/Q394281 "unloading of
1218  dll based on OMP is very slow".)
1219  2. Windows* OS may kill thread before it resets ds_alive flag.
1220 
1221  Right solution seems to be waiting for *either* thread termination *or*
1222  ds_alive resetting. */
1223  {
1224  // TODO: This code is very similar to KMP_WAIT_YIELD. Need to generalize
1225  // KMP_WAIT_YIELD to cover this usage also.
1226  void *obj = NULL;
1227  kmp_uint32 spins;
1228 #if USE_ITT_BUILD
1229  KMP_FSYNC_SPIN_INIT(obj, (void *)&th->th.th_info.ds.ds_alive);
1230 #endif /* USE_ITT_BUILD */
1231  KMP_INIT_YIELD(spins);
1232  do {
1233 #if USE_ITT_BUILD
1234  KMP_FSYNC_SPIN_PREPARE(obj);
1235 #endif /* USE_ITT_BUILD */
1236  __kmp_is_thread_alive(th, &exit_val);
1237  KMP_YIELD(TCR_4(__kmp_nth) > __kmp_avail_proc);
1238  KMP_YIELD_SPIN(spins);
1239  } while (exit_val == STILL_ACTIVE && TCR_4(th->th.th_info.ds.ds_alive));
1240 #if USE_ITT_BUILD
1241  if (exit_val == STILL_ACTIVE) {
1242  KMP_FSYNC_CANCEL(obj);
1243  } else {
1244  KMP_FSYNC_SPIN_ACQUIRED(obj);
1245  }
1246 #endif /* USE_ITT_BUILD */
1247  }
1248 
1249  __kmp_free_handle(th->th.th_info.ds.ds_thread);
1250 
1251  /* NOTE: The ExitProcess(code) system call causes all threads to Terminate
1252  with a exit_val = code. Because of this we can not rely on exit_val having
1253  any particular value. */
1254  if (exit_val == STILL_ACTIVE) {
1255  KA_TRACE(1, ("__kmp_reap_common: thread still active.\n"));
1256  } else if ((void *)exit_val != (void *)th) {
1257  KA_TRACE(1, ("__kmp_reap_common: ExitProcess / TerminateThread used?\n"));
1258  }
1259 
1260  KA_TRACE(10,
1261  ("__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
1262  "\n",
1263  th->th.th_info.ds.ds_gtid, th->th.th_info.ds.ds_thread));
1264 
1265  th->th.th_info.ds.ds_thread = 0;
1266  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1267  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1268  th->th.th_info.ds.ds_thread_id = 0;
1269 
1270  KMP_MB(); /* Flush all pending memory write invalidates. */
1271 }
1272 
1273 #if KMP_USE_MONITOR
1274 void __kmp_reap_monitor(kmp_info_t *th) {
1275  int status;
1276 
1277  KA_TRACE(10, ("__kmp_reap_monitor: try to reap %p\n",
1278  (void *)th->th.th_info.ds.ds_thread));
1279 
1280  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1281  // If both tid and gtid are 0, it means the monitor did not ever start.
1282  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1283  KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1284  if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1285  KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1286  return;
1287  }
1288 
1289  KMP_MB(); /* Flush all pending memory write invalidates. */
1290 
1291  status = SetEvent(__kmp_monitor_ev);
1292  if (status == FALSE) {
1293  DWORD error = GetLastError();
1294  __kmp_fatal(KMP_MSG(CantSetEvent), KMP_ERR(error), __kmp_msg_null);
1295  }
1296  KA_TRACE(10, ("__kmp_reap_monitor: reaping thread (%d)\n",
1297  th->th.th_info.ds.ds_gtid));
1298  __kmp_reap_common(th);
1299 
1300  __kmp_free_handle(__kmp_monitor_ev);
1301 
1302  KMP_MB(); /* Flush all pending memory write invalidates. */
1303 }
1304 #endif
1305 
1306 void __kmp_reap_worker(kmp_info_t *th) {
1307  KA_TRACE(10, ("__kmp_reap_worker: reaping thread (%d)\n",
1308  th->th.th_info.ds.ds_gtid));
1309  __kmp_reap_common(th);
1310 }
1311 
1312 #if KMP_HANDLE_SIGNALS
1313 
1314 static void __kmp_team_handler(int signo) {
1315  if (__kmp_global.g.g_abort == 0) {
1316  // Stage 1 signal handler, let's shut down all of the threads.
1317  if (__kmp_debug_buf) {
1318  __kmp_dump_debug_buffer();
1319  }
1320  KMP_MB(); // Flush all pending memory write invalidates.
1321  TCW_4(__kmp_global.g.g_abort, signo);
1322  KMP_MB(); // Flush all pending memory write invalidates.
1323  TCW_4(__kmp_global.g.g_done, TRUE);
1324  KMP_MB(); // Flush all pending memory write invalidates.
1325  }
1326 } // __kmp_team_handler
1327 
1328 static sig_func_t __kmp_signal(int signum, sig_func_t handler) {
1329  sig_func_t old = signal(signum, handler);
1330  if (old == SIG_ERR) {
1331  int error = errno;
1332  __kmp_fatal(KMP_MSG(FunctionError, "signal"), KMP_ERR(error),
1333  __kmp_msg_null);
1334  }
1335  return old;
1336 }
1337 
1338 static void __kmp_install_one_handler(int sig, sig_func_t handler,
1339  int parallel_init) {
1340  sig_func_t old;
1341  KMP_MB(); /* Flush all pending memory write invalidates. */
1342  KB_TRACE(60, ("__kmp_install_one_handler: called: sig=%d\n", sig));
1343  if (parallel_init) {
1344  old = __kmp_signal(sig, handler);
1345  // SIG_DFL on Windows* OS in NULL or 0.
1346  if (old == __kmp_sighldrs[sig]) {
1347  __kmp_siginstalled[sig] = 1;
1348  } else { // Restore/keep user's handler if one previously installed.
1349  old = __kmp_signal(sig, old);
1350  }
1351  } else {
1352  // Save initial/system signal handlers to see if user handlers installed.
1353  // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals
1354  // called once with parallel_init == TRUE.
1355  old = __kmp_signal(sig, SIG_DFL);
1356  __kmp_sighldrs[sig] = old;
1357  __kmp_signal(sig, old);
1358  }
1359  KMP_MB(); /* Flush all pending memory write invalidates. */
1360 } // __kmp_install_one_handler
1361 
1362 static void __kmp_remove_one_handler(int sig) {
1363  if (__kmp_siginstalled[sig]) {
1364  sig_func_t old;
1365  KMP_MB(); // Flush all pending memory write invalidates.
1366  KB_TRACE(60, ("__kmp_remove_one_handler: called: sig=%d\n", sig));
1367  old = __kmp_signal(sig, __kmp_sighldrs[sig]);
1368  if (old != __kmp_team_handler) {
1369  KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1370  "restoring: sig=%d\n",
1371  sig));
1372  old = __kmp_signal(sig, old);
1373  }
1374  __kmp_sighldrs[sig] = NULL;
1375  __kmp_siginstalled[sig] = 0;
1376  KMP_MB(); // Flush all pending memory write invalidates.
1377  }
1378 } // __kmp_remove_one_handler
1379 
1380 void __kmp_install_signals(int parallel_init) {
1381  KB_TRACE(10, ("__kmp_install_signals: called\n"));
1382  if (!__kmp_handle_signals) {
1383  KB_TRACE(10, ("__kmp_install_signals: KMP_HANDLE_SIGNALS is false - "
1384  "handlers not installed\n"));
1385  return;
1386  }
1387  __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1388  __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1389  __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1390  __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1391  __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1392  __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1393 } // __kmp_install_signals
1394 
1395 void __kmp_remove_signals(void) {
1396  int sig;
1397  KB_TRACE(10, ("__kmp_remove_signals: called\n"));
1398  for (sig = 1; sig < NSIG; ++sig) {
1399  __kmp_remove_one_handler(sig);
1400  }
1401 } // __kmp_remove_signals
1402 
1403 #endif // KMP_HANDLE_SIGNALS
1404 
1405 /* Put the thread to sleep for a time period */
1406 void __kmp_thread_sleep(int millis) {
1407  DWORD status;
1408 
1409  status = SleepEx((DWORD)millis, FALSE);
1410  if (status) {
1411  DWORD error = GetLastError();
1412  __kmp_fatal(KMP_MSG(FunctionError, "SleepEx()"), KMP_ERR(error),
1413  __kmp_msg_null);
1414  }
1415 }
1416 
1417 // Determine whether the given address is mapped into the current address space.
1418 int __kmp_is_address_mapped(void *addr) {
1419  DWORD status;
1420  MEMORY_BASIC_INFORMATION lpBuffer;
1421  SIZE_T dwLength;
1422 
1423  dwLength = sizeof(MEMORY_BASIC_INFORMATION);
1424 
1425  status = VirtualQuery(addr, &lpBuffer, dwLength);
1426 
1427  return !(((lpBuffer.State == MEM_RESERVE) || (lpBuffer.State == MEM_FREE)) ||
1428  ((lpBuffer.Protect == PAGE_NOACCESS) ||
1429  (lpBuffer.Protect == PAGE_EXECUTE)));
1430 }
1431 
1432 kmp_uint64 __kmp_hardware_timestamp(void) {
1433  kmp_uint64 r = 0;
1434 
1435  QueryPerformanceCounter((LARGE_INTEGER *)&r);
1436  return r;
1437 }
1438 
1439 /* Free handle and check the error code */
1440 void __kmp_free_handle(kmp_thread_t tHandle) {
1441  /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined
1442  * as HANDLE */
1443  BOOL rc;
1444  rc = CloseHandle(tHandle);
1445  if (!rc) {
1446  DWORD error = GetLastError();
1447  __kmp_fatal(KMP_MSG(CantCloseHandle), KMP_ERR(error), __kmp_msg_null);
1448  }
1449 }
1450 
1451 int __kmp_get_load_balance(int max) {
1452  static ULONG glb_buff_size = 100 * 1024;
1453 
1454  // Saved count of the running threads for the thread balance algortihm
1455  static int glb_running_threads = 0;
1456  static double glb_call_time = 0; /* Thread balance algorithm call time */
1457 
1458  int running_threads = 0; // Number of running threads in the system.
1459  NTSTATUS status = 0;
1460  ULONG buff_size = 0;
1461  ULONG info_size = 0;
1462  void *buffer = NULL;
1463  PSYSTEM_PROCESS_INFORMATION spi = NULL;
1464  int first_time = 1;
1465 
1466  double call_time = 0.0; // start, finish;
1467 
1468  __kmp_elapsed(&call_time);
1469 
1470  if (glb_call_time &&
1471  (call_time - glb_call_time < __kmp_load_balance_interval)) {
1472  running_threads = glb_running_threads;
1473  goto finish;
1474  }
1475  glb_call_time = call_time;
1476 
1477  // Do not spend time on running algorithm if we have a permanent error.
1478  if (NtQuerySystemInformation == NULL) {
1479  running_threads = -1;
1480  goto finish;
1481  }
1482 
1483  if (max <= 0) {
1484  max = INT_MAX;
1485  }
1486 
1487  do {
1488 
1489  if (first_time) {
1490  buff_size = glb_buff_size;
1491  } else {
1492  buff_size = 2 * buff_size;
1493  }
1494 
1495  buffer = KMP_INTERNAL_REALLOC(buffer, buff_size);
1496  if (buffer == NULL) {
1497  running_threads = -1;
1498  goto finish;
1499  }
1500  status = NtQuerySystemInformation(SystemProcessInformation, buffer,
1501  buff_size, &info_size);
1502  first_time = 0;
1503 
1504  } while (status == STATUS_INFO_LENGTH_MISMATCH);
1505  glb_buff_size = buff_size;
1506 
1507 #define CHECK(cond) \
1508  { \
1509  KMP_DEBUG_ASSERT(cond); \
1510  if (!(cond)) { \
1511  running_threads = -1; \
1512  goto finish; \
1513  } \
1514  }
1515 
1516  CHECK(buff_size >= info_size);
1517  spi = PSYSTEM_PROCESS_INFORMATION(buffer);
1518  for (;;) {
1519  ptrdiff_t offset = uintptr_t(spi) - uintptr_t(buffer);
1520  CHECK(0 <= offset &&
1521  offset + sizeof(SYSTEM_PROCESS_INFORMATION) < info_size);
1522  HANDLE pid = spi->ProcessId;
1523  ULONG num = spi->NumberOfThreads;
1524  CHECK(num >= 1);
1525  size_t spi_size =
1526  sizeof(SYSTEM_PROCESS_INFORMATION) + sizeof(SYSTEM_THREAD) * (num - 1);
1527  CHECK(offset + spi_size <
1528  info_size); // Make sure process info record fits the buffer.
1529  if (spi->NextEntryOffset != 0) {
1530  CHECK(spi_size <=
1531  spi->NextEntryOffset); // And do not overlap with the next record.
1532  }
1533  // pid == 0 corresponds to the System Idle Process. It always has running
1534  // threads on all cores. So, we don't consider the running threads of this
1535  // process.
1536  if (pid != 0) {
1537  for (int i = 0; i < num; ++i) {
1538  THREAD_STATE state = spi->Threads[i].State;
1539  // Count threads that have Ready or Running state.
1540  // !!! TODO: Why comment does not match the code???
1541  if (state == StateRunning) {
1542  ++running_threads;
1543  // Stop counting running threads if the number is already greater than
1544  // the number of available cores
1545  if (running_threads >= max) {
1546  goto finish;
1547  }
1548  }
1549  }
1550  }
1551  if (spi->NextEntryOffset == 0) {
1552  break;
1553  }
1554  spi = PSYSTEM_PROCESS_INFORMATION(uintptr_t(spi) + spi->NextEntryOffset);
1555  }
1556 
1557 #undef CHECK
1558 
1559 finish: // Clean up and exit.
1560 
1561  if (buffer != NULL) {
1562  KMP_INTERNAL_FREE(buffer);
1563  }
1564 
1565  glb_running_threads = running_threads;
1566 
1567  return running_threads;
1568 } //__kmp_get_load_balance()