17 #include "kmp_error.h" 21 #include "kmp_stats.h" 24 #include "ompt-specific.h" 27 #define MAX_MESSAGE 512 43 if ((env = getenv(
"KMP_INITIAL_THREAD_BIND")) != NULL &&
44 __kmp_str_match_true(env)) {
45 __kmp_middle_initialize();
46 KC_TRACE(10, (
"__kmpc_begin: middle initialization called\n"));
47 }
else if (__kmp_ignore_mppbeg() == FALSE) {
49 __kmp_internal_begin();
50 KC_TRACE(10, (
"__kmpc_begin: called\n"));
68 if (__kmp_ignore_mppend() == FALSE) {
69 KC_TRACE(10, (
"__kmpc_end: called\n"));
70 KA_TRACE(30, (
"__kmpc_end\n"));
72 __kmp_internal_end_thread(-1);
95 kmp_int32 gtid = __kmp_entry_gtid();
97 KC_TRACE(10, (
"__kmpc_global_thread_num: T#%d\n", gtid));
118 (
"__kmpc_global_num_threads: num_threads = %d\n", __kmp_all_nth));
120 return TCR_4(__kmp_all_nth);
130 KC_TRACE(10, (
"__kmpc_bound_thread_num: called\n"));
131 return __kmp_tid_from_gtid(__kmp_entry_gtid());
140 KC_TRACE(10, (
"__kmpc_bound_num_threads: called\n"));
142 return __kmp_entry_thread()->th.th_team->t.t_nproc;
162 if (__kmp_par_range == 0) {
169 semi2 = strchr(semi2,
';');
173 semi2 = strchr(semi2 + 1,
';');
177 if (__kmp_par_range_filename[0]) {
178 const char *name = semi2 - 1;
179 while ((name > loc->
psource) && (*name !=
'/') && (*name !=
';')) {
182 if ((*name ==
'/') || (*name ==
';')) {
185 if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
186 return __kmp_par_range < 0;
189 semi3 = strchr(semi2 + 1,
';');
190 if (__kmp_par_range_routine[0]) {
191 if ((semi3 != NULL) && (semi3 > semi2) &&
192 (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
193 return __kmp_par_range < 0;
196 if (KMP_SSCANF(semi3 + 1,
"%d", &line_no) == 1) {
197 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
198 return __kmp_par_range > 0;
200 return __kmp_par_range < 0;
214 return __kmp_entry_thread()->th.th_root->r.r_active;
227 kmp_int32 num_threads) {
228 KA_TRACE(20, (
"__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
229 global_tid, num_threads));
231 __kmp_push_num_threads(loc, global_tid, num_threads);
234 void __kmpc_pop_num_threads(
ident_t *loc, kmp_int32 global_tid) {
235 KA_TRACE(20, (
"__kmpc_pop_num_threads: enter\n"));
242 void __kmpc_push_proc_bind(
ident_t *loc, kmp_int32 global_tid,
243 kmp_int32 proc_bind) {
244 KA_TRACE(20, (
"__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n", global_tid,
247 __kmp_push_proc_bind(loc, global_tid, (kmp_proc_bind_t)proc_bind);
263 int gtid = __kmp_entry_gtid();
265 #if (KMP_STATS_ENABLED) 269 if (previous_state == stats_state_e::SERIAL_REGION) {
270 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_parallel_overhead);
272 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel_overhead);
285 va_start(ap, microtask);
288 omp_frame_t *ompt_frame;
289 if (ompt_enabled.enabled) {
290 kmp_info_t *master_th = __kmp_threads[gtid];
291 kmp_team_t *parent_team = master_th->th.th_team;
292 ompt_lw_taskteam_t *lwt = parent_team->t.ompt_serialized_team_info;
294 ompt_frame = &(lwt->ompt_task_info.frame);
296 int tid = __kmp_tid_from_gtid(gtid);
298 parent_team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame);
300 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
301 OMPT_STORE_RETURN_ADDRESS(gtid);
305 #if INCLUDE_SSC_MARKS 308 __kmp_fork_call(loc, gtid, fork_context_intel, argc,
309 VOLATILE_CAST(microtask_t) microtask,
310 VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
312 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
318 #if INCLUDE_SSC_MARKS 321 __kmp_join_call(loc, gtid
331 #if KMP_STATS_ENABLED 332 if (previous_state == stats_state_e::SERIAL_REGION) {
333 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
335 KMP_POP_PARTITIONED_TIMER();
337 #endif // KMP_STATS_ENABLED 353 kmp_int32 num_teams, kmp_int32 num_threads) {
355 (
"__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
356 global_tid, num_teams, num_threads));
358 __kmp_push_num_teams(loc, global_tid, num_teams, num_threads);
373 int gtid = __kmp_entry_gtid();
374 kmp_info_t *this_thr = __kmp_threads[gtid];
376 va_start(ap, microtask);
381 this_thr->th.th_teams_microtask = microtask;
382 this_thr->th.th_teams_level =
383 this_thr->th.th_team->t.t_level;
386 kmp_team_t *parent_team = this_thr->th.th_team;
387 int tid = __kmp_tid_from_gtid(gtid);
388 if (ompt_enabled.enabled) {
389 parent_team->t.t_implicit_task_taskdata[tid]
390 .ompt_task_info.frame.enter_frame = OMPT_GET_FRAME_ADDRESS(1);
392 OMPT_STORE_RETURN_ADDRESS(gtid);
397 if (this_thr->th.th_teams_size.nteams == 0) {
398 __kmp_push_num_teams(loc, gtid, 0, 0);
400 KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
401 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
402 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
404 __kmp_fork_call(loc, gtid, fork_context_intel, argc,
405 VOLATILE_CAST(microtask_t)
407 VOLATILE_CAST(launch_t) __kmp_invoke_teams_master,
408 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
414 __kmp_join_call(loc, gtid
421 this_thr->th.th_teams_microtask = NULL;
422 this_thr->th.th_teams_level = 0;
423 *(kmp_int64 *)(&this_thr->th.th_teams_size) = 0L;
432 int __kmpc_invoke_task_func(
int gtid) {
return __kmp_invoke_task_func(gtid); }
451 OMPT_STORE_RETURN_ADDRESS(global_tid);
453 __kmp_serialized_parallel(loc, global_tid);
464 kmp_internal_control_t *top;
465 kmp_info_t *this_thr;
466 kmp_team_t *serial_team;
469 (
"__kmpc_end_serialized_parallel: called by T#%d\n", global_tid));
477 if (!TCR_4(__kmp_init_parallel))
478 __kmp_parallel_initialize();
480 this_thr = __kmp_threads[global_tid];
481 serial_team = this_thr->th.th_serial_team;
484 kmp_task_team_t *task_team = this_thr->th.th_task_team;
487 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks)
488 __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL));
492 KMP_DEBUG_ASSERT(serial_team);
493 KMP_ASSERT(serial_team->t.t_serialized);
494 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
495 KMP_DEBUG_ASSERT(serial_team != this_thr->th.th_root->r.r_root_team);
496 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
497 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
500 if (ompt_enabled.enabled &&
501 this_thr->th.ompt_thread_info.state != omp_state_overhead) {
502 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = NULL;
503 if (ompt_enabled.ompt_callback_implicit_task) {
504 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
505 ompt_scope_end, NULL, OMPT_CUR_TASK_DATA(this_thr), 1,
506 OMPT_CUR_TASK_INFO(this_thr)->thread_num);
510 ompt_data_t *parent_task_data;
511 __ompt_get_task_info_internal(1, NULL, &parent_task_data, NULL, NULL, NULL);
513 if (ompt_enabled.ompt_callback_parallel_end) {
514 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
515 &(serial_team->t.ompt_team_info.parallel_data), parent_task_data,
516 ompt_parallel_invoker_program, OMPT_LOAD_RETURN_ADDRESS(global_tid));
518 __ompt_lw_taskteam_unlink(this_thr);
519 this_thr->th.ompt_thread_info.state = omp_state_overhead;
525 top = serial_team->t.t_control_stack_top;
526 if (top && top->serial_nesting_level == serial_team->t.t_serialized) {
527 copy_icvs(&serial_team->t.t_threads[0]->th.th_current_task->td_icvs, top);
528 serial_team->t.t_control_stack_top = top->next;
533 serial_team->t.t_level--;
536 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
538 dispatch_private_info_t *disp_buffer =
539 serial_team->t.t_dispatch->th_disp_buffer;
540 serial_team->t.t_dispatch->th_disp_buffer =
541 serial_team->t.t_dispatch->th_disp_buffer->next;
542 __kmp_free(disp_buffer);
545 this_thr->th.th_def_allocator = serial_team->t.t_def_allocator;
548 --serial_team->t.t_serialized;
549 if (serial_team->t.t_serialized == 0) {
553 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 554 if (__kmp_inherit_fp_control && serial_team->t.t_fp_control_saved) {
555 __kmp_clear_x87_fpu_status_word();
556 __kmp_load_x87_fpu_control_word(&serial_team->t.t_x87_fpu_control_word);
557 __kmp_load_mxcsr(&serial_team->t.t_mxcsr);
561 this_thr->th.th_team = serial_team->t.t_parent;
562 this_thr->th.th_info.ds.ds_tid = serial_team->t.t_master_tid;
565 this_thr->th.th_team_nproc = serial_team->t.t_parent->t.t_nproc;
566 this_thr->th.th_team_master =
567 serial_team->t.t_parent->t.t_threads[0];
568 this_thr->th.th_team_serialized = this_thr->th.th_team->t.t_serialized;
571 this_thr->th.th_dispatch =
572 &this_thr->th.th_team->t.t_dispatch[serial_team->t.t_master_tid];
574 __kmp_pop_current_task_from_thread(this_thr);
576 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 0);
577 this_thr->th.th_current_task->td_flags.executing = 1;
579 if (__kmp_tasking_mode != tskm_immediate_exec) {
581 this_thr->th.th_task_team =
582 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
584 (
"__kmpc_end_serialized_parallel: T#%d restoring task_team %p / " 586 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
589 if (__kmp_tasking_mode != tskm_immediate_exec) {
590 KA_TRACE(20, (
"__kmpc_end_serialized_parallel: T#%d decreasing nesting " 591 "depth of serial team %p to %d\n",
592 global_tid, serial_team, serial_team->t.t_serialized));
596 if (__kmp_env_consistency_check)
597 __kmp_pop_parallel(global_tid, NULL);
599 if (ompt_enabled.enabled)
600 this_thr->th.ompt_thread_info.state =
601 ((this_thr->th.th_team_serialized) ? omp_state_work_serial
602 : omp_state_work_parallel);
615 KC_TRACE(10, (
"__kmpc_flush: called\n"));
620 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 634 if (!__kmp_cpuinfo.initialized) {
635 __kmp_query_cpuid(&__kmp_cpuinfo);
637 if (!__kmp_cpuinfo.sse2) {
642 #elif KMP_COMPILER_MSVC 645 __sync_synchronize();
646 #endif // KMP_COMPILER_ICC 649 #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64) 665 #error Unknown or unsupported architecture 668 #if OMPT_SUPPORT && OMPT_OPTIONAL 669 if (ompt_enabled.ompt_callback_flush) {
670 ompt_callbacks.ompt_callback(ompt_callback_flush)(
671 __ompt_get_thread_data_internal(), OMPT_GET_RETURN_ADDRESS(0));
686 KC_TRACE(10, (
"__kmpc_barrier: called T#%d\n", global_tid));
688 if (!TCR_4(__kmp_init_parallel))
689 __kmp_parallel_initialize();
691 if (__kmp_env_consistency_check) {
693 KMP_WARNING(ConstructIdentInvalid);
696 __kmp_check_barrier(global_tid, ct_barrier, loc);
700 omp_frame_t *ompt_frame;
701 if (ompt_enabled.enabled) {
702 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
703 if (ompt_frame->enter_frame == NULL)
704 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
705 OMPT_STORE_RETURN_ADDRESS(global_tid);
708 __kmp_threads[global_tid]->th.th_ident = loc;
716 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
717 #if OMPT_SUPPORT && OMPT_OPTIONAL 718 if (ompt_enabled.enabled) {
719 ompt_frame->enter_frame = NULL;
734 KC_TRACE(10, (
"__kmpc_master: called T#%d\n", global_tid));
736 if (!TCR_4(__kmp_init_parallel))
737 __kmp_parallel_initialize();
739 if (KMP_MASTER_GTID(global_tid)) {
741 KMP_PUSH_PARTITIONED_TIMER(OMP_master);
745 #if OMPT_SUPPORT && OMPT_OPTIONAL 747 if (ompt_enabled.ompt_callback_master) {
748 kmp_info_t *this_thr = __kmp_threads[global_tid];
749 kmp_team_t *team = this_thr->th.th_team;
751 int tid = __kmp_tid_from_gtid(global_tid);
752 ompt_callbacks.ompt_callback(ompt_callback_master)(
753 ompt_scope_begin, &(team->t.ompt_team_info.parallel_data),
754 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
755 OMPT_GET_RETURN_ADDRESS(0));
760 if (__kmp_env_consistency_check) {
761 #if KMP_USE_DYNAMIC_LOCK 763 __kmp_push_sync(global_tid, ct_master, loc, NULL, 0);
765 __kmp_check_sync(global_tid, ct_master, loc, NULL, 0);
768 __kmp_push_sync(global_tid, ct_master, loc, NULL);
770 __kmp_check_sync(global_tid, ct_master, loc, NULL);
786 KC_TRACE(10, (
"__kmpc_end_master: called T#%d\n", global_tid));
788 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(global_tid));
789 KMP_POP_PARTITIONED_TIMER();
791 #if OMPT_SUPPORT && OMPT_OPTIONAL 792 kmp_info_t *this_thr = __kmp_threads[global_tid];
793 kmp_team_t *team = this_thr->th.th_team;
794 if (ompt_enabled.ompt_callback_master) {
795 int tid = __kmp_tid_from_gtid(global_tid);
796 ompt_callbacks.ompt_callback(ompt_callback_master)(
797 ompt_scope_end, &(team->t.ompt_team_info.parallel_data),
798 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
799 OMPT_GET_RETURN_ADDRESS(0));
803 if (__kmp_env_consistency_check) {
805 KMP_WARNING(ThreadIdentInvalid);
807 if (KMP_MASTER_GTID(global_tid))
808 __kmp_pop_sync(global_tid, ct_master, loc);
822 KMP_DEBUG_ASSERT(__kmp_init_serial);
824 KC_TRACE(10, (
"__kmpc_ordered: called T#%d\n", gtid));
826 if (!TCR_4(__kmp_init_parallel))
827 __kmp_parallel_initialize();
830 __kmp_itt_ordered_prep(gtid);
834 th = __kmp_threads[gtid];
836 #if OMPT_SUPPORT && OMPT_OPTIONAL 840 if (ompt_enabled.enabled) {
841 OMPT_STORE_RETURN_ADDRESS(gtid);
842 team = __kmp_team_from_gtid(gtid);
843 lck = (omp_wait_id_t)&team->t.t_ordered.dt.t_value;
845 th->th.ompt_thread_info.wait_id = lck;
846 th->th.ompt_thread_info.state = omp_state_wait_ordered;
849 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
850 if (ompt_enabled.ompt_callback_mutex_acquire) {
851 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
852 ompt_mutex_ordered, omp_lock_hint_none, kmp_mutex_impl_spin,
853 (omp_wait_id_t)lck, codeptr_ra);
858 if (th->th.th_dispatch->th_deo_fcn != 0)
859 (*th->th.th_dispatch->th_deo_fcn)(>id, &cid, loc);
861 __kmp_parallel_deo(>id, &cid, loc);
863 #if OMPT_SUPPORT && OMPT_OPTIONAL 864 if (ompt_enabled.enabled) {
866 th->th.ompt_thread_info.state = omp_state_work_parallel;
867 th->th.ompt_thread_info.wait_id = 0;
870 if (ompt_enabled.ompt_callback_mutex_acquired) {
871 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
872 ompt_mutex_ordered, (omp_wait_id_t)lck, codeptr_ra);
878 __kmp_itt_ordered_start(gtid);
893 KC_TRACE(10, (
"__kmpc_end_ordered: called T#%d\n", gtid));
896 __kmp_itt_ordered_end(gtid);
900 th = __kmp_threads[gtid];
902 if (th->th.th_dispatch->th_dxo_fcn != 0)
903 (*th->th.th_dispatch->th_dxo_fcn)(>id, &cid, loc);
905 __kmp_parallel_dxo(>id, &cid, loc);
907 #if OMPT_SUPPORT && OMPT_OPTIONAL 908 OMPT_STORE_RETURN_ADDRESS(gtid);
909 if (ompt_enabled.ompt_callback_mutex_released) {
910 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
912 (omp_wait_id_t)&__kmp_team_from_gtid(gtid)->t.t_ordered.dt.t_value,
913 OMPT_LOAD_RETURN_ADDRESS(gtid));
918 #if KMP_USE_DYNAMIC_LOCK 920 static __forceinline
void 921 __kmp_init_indirect_csptr(kmp_critical_name *crit,
ident_t const *loc,
922 kmp_int32 gtid, kmp_indirect_locktag_t tag) {
926 kmp_indirect_lock_t **lck;
927 lck = (kmp_indirect_lock_t **)crit;
928 kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
929 KMP_I_LOCK_FUNC(ilk, init)(ilk->lock);
930 KMP_SET_I_LOCK_LOCATION(ilk, loc);
931 KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
933 (
"__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag));
935 __kmp_itt_critical_creating(ilk->lock, loc);
937 int status = KMP_COMPARE_AND_STORE_PTR(lck,
nullptr, ilk);
940 __kmp_itt_critical_destroyed(ilk->lock);
946 KMP_DEBUG_ASSERT(*lck != NULL);
950 #define KMP_ACQUIRE_TAS_LOCK(lock, gtid) \ 952 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ 953 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \ 954 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \ 955 if (KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \ 956 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \ 958 KMP_FSYNC_PREPARE(l); \ 959 KMP_INIT_YIELD(spins); \ 960 if (TCR_4(__kmp_nth) > \ 961 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 964 KMP_YIELD_SPIN(spins); \ 966 kmp_backoff_t backoff = __kmp_spin_backoff_params; \ 968 KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \ 969 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \ 970 __kmp_spin_backoff(&backoff); \ 971 if (TCR_4(__kmp_nth) > \ 972 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 975 KMP_YIELD_SPIN(spins); \ 979 KMP_FSYNC_ACQUIRED(l); \ 983 #define KMP_TEST_TAS_LOCK(lock, gtid, rc) \ 985 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ 986 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \ 987 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \ 988 rc = KMP_ATOMIC_LD_RLX(&l->lk.poll) == tas_free && \ 989 __kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy); \ 993 #define KMP_RELEASE_TAS_LOCK(lock, gtid) \ 994 { KMP_ATOMIC_ST_REL(&((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); } 998 #include <sys/syscall.h> 1001 #define FUTEX_WAIT 0 1004 #define FUTEX_WAKE 1 1008 #define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) \ 1010 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1011 kmp_int32 gtid_code = (gtid + 1) << 1; \ 1013 KMP_FSYNC_PREPARE(ftx); \ 1014 kmp_int32 poll_val; \ 1015 while ((poll_val = KMP_COMPARE_AND_STORE_RET32( \ 1016 &(ftx->lk.poll), KMP_LOCK_FREE(futex), \ 1017 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \ 1018 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \ 1020 if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, \ 1022 KMP_LOCK_BUSY(1, futex))) { \ 1025 poll_val |= KMP_LOCK_BUSY(1, futex); \ 1028 if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, \ 1029 NULL, NULL, 0)) != 0) { \ 1034 KMP_FSYNC_ACQUIRED(ftx); \ 1038 #define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) \ 1040 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1041 if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \ 1042 KMP_LOCK_BUSY(gtid + 1 << 1, futex))) { \ 1043 KMP_FSYNC_ACQUIRED(ftx); \ 1051 #define KMP_RELEASE_FUTEX_LOCK(lock, gtid) \ 1053 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1055 KMP_FSYNC_RELEASING(ftx); \ 1056 kmp_int32 poll_val = \ 1057 KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \ 1058 if (KMP_LOCK_STRIP(poll_val) & 1) { \ 1059 syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, \ 1060 KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \ 1063 KMP_YIELD(TCR_4(__kmp_nth) > \ 1064 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)); \ 1067 #endif // KMP_USE_FUTEX 1069 #else // KMP_USE_DYNAMIC_LOCK 1071 static kmp_user_lock_p __kmp_get_critical_section_ptr(kmp_critical_name *crit,
1074 kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1077 kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1084 lck = __kmp_user_lock_allocate(&idx, gtid, kmp_lf_critical_section);
1085 __kmp_init_user_lock_with_checks(lck);
1086 __kmp_set_user_lock_location(lck, loc);
1088 __kmp_itt_critical_creating(lck);
1099 int status = KMP_COMPARE_AND_STORE_PTR(lck_pp, 0, lck);
1104 __kmp_itt_critical_destroyed(lck);
1108 __kmp_destroy_user_lock_with_checks(lck);
1109 __kmp_user_lock_free(&idx, gtid, lck);
1110 lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1111 KMP_DEBUG_ASSERT(lck != NULL);
1117 #endif // KMP_USE_DYNAMIC_LOCK 1130 kmp_critical_name *crit) {
1131 #if KMP_USE_DYNAMIC_LOCK 1132 #if OMPT_SUPPORT && OMPT_OPTIONAL 1133 OMPT_STORE_RETURN_ADDRESS(global_tid);
1134 #endif // OMPT_SUPPORT 1135 __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
1138 #if OMPT_SUPPORT && OMPT_OPTIONAL 1139 omp_state_t prev_state = omp_state_undefined;
1140 ompt_thread_info_t ti;
1142 kmp_user_lock_p lck;
1144 KC_TRACE(10, (
"__kmpc_critical: called T#%d\n", global_tid));
1148 KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1149 KMP_CHECK_USER_LOCK_INIT();
1151 if ((__kmp_user_lock_kind == lk_tas) &&
1152 (
sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1153 lck = (kmp_user_lock_p)crit;
1156 else if ((__kmp_user_lock_kind == lk_futex) &&
1157 (
sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1158 lck = (kmp_user_lock_p)crit;
1162 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
1165 if (__kmp_env_consistency_check)
1166 __kmp_push_sync(global_tid, ct_critical, loc, lck);
1174 __kmp_itt_critical_acquiring(lck);
1176 #if OMPT_SUPPORT && OMPT_OPTIONAL 1177 OMPT_STORE_RETURN_ADDRESS(gtid);
1178 void *codeptr_ra = NULL;
1179 if (ompt_enabled.enabled) {
1180 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1182 prev_state = ti.state;
1183 ti.wait_id = (omp_wait_id_t)lck;
1184 ti.state = omp_state_wait_critical;
1187 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
1188 if (ompt_enabled.ompt_callback_mutex_acquire) {
1189 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1190 ompt_mutex_critical, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
1191 (omp_wait_id_t)crit, codeptr_ra);
1197 __kmp_acquire_user_lock_with_checks(lck, global_tid);
1200 __kmp_itt_critical_acquired(lck);
1202 #if OMPT_SUPPORT && OMPT_OPTIONAL 1203 if (ompt_enabled.enabled) {
1205 ti.state = prev_state;
1209 if (ompt_enabled.ompt_callback_mutex_acquired) {
1210 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1211 ompt_mutex_critical, (omp_wait_id_t)crit, codeptr_ra);
1215 KMP_POP_PARTITIONED_TIMER();
1217 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1218 KA_TRACE(15, (
"__kmpc_critical: done T#%d\n", global_tid));
1219 #endif // KMP_USE_DYNAMIC_LOCK 1222 #if KMP_USE_DYNAMIC_LOCK 1225 static __forceinline kmp_dyna_lockseq_t __kmp_map_hint_to_lock(uintptr_t hint) {
1227 #define KMP_TSX_LOCK(seq) lockseq_##seq 1229 #define KMP_TSX_LOCK(seq) __kmp_user_lock_seq 1232 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1233 #define KMP_CPUINFO_RTM (__kmp_cpuinfo.rtm) 1235 #define KMP_CPUINFO_RTM 0 1239 if (hint & kmp_lock_hint_hle)
1240 return KMP_TSX_LOCK(hle);
1241 if (hint & kmp_lock_hint_rtm)
1242 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm) : __kmp_user_lock_seq;
1243 if (hint & kmp_lock_hint_adaptive)
1244 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive) : __kmp_user_lock_seq;
1247 if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended))
1248 return __kmp_user_lock_seq;
1249 if ((hint & omp_lock_hint_speculative) &&
1250 (hint & omp_lock_hint_nonspeculative))
1251 return __kmp_user_lock_seq;
1254 if (hint & omp_lock_hint_contended)
1255 return lockseq_queuing;
1258 if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative))
1262 if (hint & omp_lock_hint_speculative)
1263 return KMP_TSX_LOCK(hle);
1265 return __kmp_user_lock_seq;
1268 #if OMPT_SUPPORT && OMPT_OPTIONAL 1269 #if KMP_USE_DYNAMIC_LOCK 1270 static kmp_mutex_impl_t
1271 __ompt_get_mutex_impl_type(
void *user_lock, kmp_indirect_lock_t *ilock = 0) {
1273 switch (KMP_EXTRACT_D_TAG(user_lock)) {
1278 return kmp_mutex_impl_queuing;
1281 return kmp_mutex_impl_spin;
1284 return kmp_mutex_impl_speculative;
1287 return ompt_mutex_impl_unknown;
1289 ilock = KMP_LOOKUP_I_LOCK(user_lock);
1292 switch (ilock->type) {
1294 case locktag_adaptive:
1296 return kmp_mutex_impl_speculative;
1298 case locktag_nested_tas:
1299 return kmp_mutex_impl_spin;
1301 case locktag_nested_futex:
1303 case locktag_ticket:
1304 case locktag_queuing:
1306 case locktag_nested_ticket:
1307 case locktag_nested_queuing:
1308 case locktag_nested_drdpa:
1309 return kmp_mutex_impl_queuing;
1311 return ompt_mutex_impl_unknown;
1316 static kmp_mutex_impl_t __ompt_get_mutex_impl_type() {
1317 switch (__kmp_user_lock_kind) {
1319 return kmp_mutex_impl_spin;
1326 return kmp_mutex_impl_queuing;
1331 return kmp_mutex_impl_speculative;
1334 return ompt_mutex_impl_unknown;
1337 #endif // KMP_USE_DYNAMIC_LOCK 1338 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1353 void __kmpc_critical_with_hint(
ident_t *loc, kmp_int32 global_tid,
1354 kmp_critical_name *crit, uint32_t hint) {
1356 kmp_user_lock_p lck;
1357 #if OMPT_SUPPORT && OMPT_OPTIONAL 1358 omp_state_t prev_state = omp_state_undefined;
1359 ompt_thread_info_t ti;
1361 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1363 codeptr = OMPT_GET_RETURN_ADDRESS(0);
1366 KC_TRACE(10, (
"__kmpc_critical: called T#%d\n", global_tid));
1368 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
1370 KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1372 kmp_dyna_lockseq_t lckseq = __kmp_map_hint_to_lock(hint);
1373 if (KMP_IS_D_LOCK(lckseq)) {
1374 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)crit, 0,
1375 KMP_GET_D_TAG(lckseq));
1377 __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(lckseq));
1383 if (KMP_EXTRACT_D_TAG(lk) != 0) {
1384 lck = (kmp_user_lock_p)lk;
1385 if (__kmp_env_consistency_check) {
1386 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1387 __kmp_map_hint_to_lock(hint));
1390 __kmp_itt_critical_acquiring(lck);
1392 #if OMPT_SUPPORT && OMPT_OPTIONAL 1393 if (ompt_enabled.enabled) {
1394 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1396 prev_state = ti.state;
1397 ti.wait_id = (omp_wait_id_t)lck;
1398 ti.state = omp_state_wait_critical;
1401 if (ompt_enabled.ompt_callback_mutex_acquire) {
1402 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1403 ompt_mutex_critical, (
unsigned int)hint,
1404 __ompt_get_mutex_impl_type(crit), (omp_wait_id_t)crit, codeptr);
1408 #if KMP_USE_INLINED_TAS 1409 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1410 KMP_ACQUIRE_TAS_LOCK(lck, global_tid);
1412 #elif KMP_USE_INLINED_FUTEX 1413 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1414 KMP_ACQUIRE_FUTEX_LOCK(lck, global_tid);
1418 KMP_D_LOCK_FUNC(lk,
set)(lk, global_tid);
1421 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
1423 if (__kmp_env_consistency_check) {
1424 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1425 __kmp_map_hint_to_lock(hint));
1428 __kmp_itt_critical_acquiring(lck);
1430 #if OMPT_SUPPORT && OMPT_OPTIONAL 1431 if (ompt_enabled.enabled) {
1432 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1434 prev_state = ti.state;
1435 ti.wait_id = (omp_wait_id_t)lck;
1436 ti.state = omp_state_wait_critical;
1439 if (ompt_enabled.ompt_callback_mutex_acquire) {
1440 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1441 ompt_mutex_critical, (
unsigned int)hint,
1442 __ompt_get_mutex_impl_type(0, ilk), (omp_wait_id_t)crit, codeptr);
1446 KMP_I_LOCK_FUNC(ilk,
set)(lck, global_tid);
1448 KMP_POP_PARTITIONED_TIMER();
1451 __kmp_itt_critical_acquired(lck);
1453 #if OMPT_SUPPORT && OMPT_OPTIONAL 1454 if (ompt_enabled.enabled) {
1456 ti.state = prev_state;
1460 if (ompt_enabled.ompt_callback_mutex_acquired) {
1461 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1462 ompt_mutex_critical, (omp_wait_id_t)crit, codeptr);
1467 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1468 KA_TRACE(15, (
"__kmpc_critical: done T#%d\n", global_tid));
1471 #endif // KMP_USE_DYNAMIC_LOCK 1483 kmp_critical_name *crit) {
1484 kmp_user_lock_p lck;
1486 KC_TRACE(10, (
"__kmpc_end_critical: called T#%d\n", global_tid));
1488 #if KMP_USE_DYNAMIC_LOCK 1489 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
1490 lck = (kmp_user_lock_p)crit;
1491 KMP_ASSERT(lck != NULL);
1492 if (__kmp_env_consistency_check) {
1493 __kmp_pop_sync(global_tid, ct_critical, loc);
1496 __kmp_itt_critical_releasing(lck);
1498 #if KMP_USE_INLINED_TAS 1499 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1500 KMP_RELEASE_TAS_LOCK(lck, global_tid);
1502 #elif KMP_USE_INLINED_FUTEX 1503 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1504 KMP_RELEASE_FUTEX_LOCK(lck, global_tid);
1508 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
1511 kmp_indirect_lock_t *ilk =
1512 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
1513 KMP_ASSERT(ilk != NULL);
1515 if (__kmp_env_consistency_check) {
1516 __kmp_pop_sync(global_tid, ct_critical, loc);
1519 __kmp_itt_critical_releasing(lck);
1521 KMP_I_LOCK_FUNC(ilk, unset)(lck, global_tid);
1524 #else // KMP_USE_DYNAMIC_LOCK 1526 if ((__kmp_user_lock_kind == lk_tas) &&
1527 (
sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1528 lck = (kmp_user_lock_p)crit;
1531 else if ((__kmp_user_lock_kind == lk_futex) &&
1532 (
sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1533 lck = (kmp_user_lock_p)crit;
1537 lck = (kmp_user_lock_p)TCR_PTR(*((kmp_user_lock_p *)crit));
1540 KMP_ASSERT(lck != NULL);
1542 if (__kmp_env_consistency_check)
1543 __kmp_pop_sync(global_tid, ct_critical, loc);
1546 __kmp_itt_critical_releasing(lck);
1550 __kmp_release_user_lock_with_checks(lck, global_tid);
1552 #endif // KMP_USE_DYNAMIC_LOCK 1554 #if OMPT_SUPPORT && OMPT_OPTIONAL 1557 OMPT_STORE_RETURN_ADDRESS(global_tid);
1558 if (ompt_enabled.ompt_callback_mutex_released) {
1559 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
1560 ompt_mutex_critical, (omp_wait_id_t)crit, OMPT_LOAD_RETURN_ADDRESS(0));
1564 KMP_POP_PARTITIONED_TIMER();
1565 KA_TRACE(15, (
"__kmpc_end_critical: done T#%d\n", global_tid));
1580 KC_TRACE(10, (
"__kmpc_barrier_master: called T#%d\n", global_tid));
1582 if (!TCR_4(__kmp_init_parallel))
1583 __kmp_parallel_initialize();
1585 if (__kmp_env_consistency_check)
1586 __kmp_check_barrier(global_tid, ct_barrier, loc);
1589 omp_frame_t *ompt_frame;
1590 if (ompt_enabled.enabled) {
1591 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1592 if (ompt_frame->enter_frame == NULL)
1593 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
1594 OMPT_STORE_RETURN_ADDRESS(global_tid);
1598 __kmp_threads[global_tid]->th.th_ident = loc;
1600 status = __kmp_barrier(bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL);
1601 #if OMPT_SUPPORT && OMPT_OPTIONAL 1602 if (ompt_enabled.enabled) {
1603 ompt_frame->enter_frame = NULL;
1607 return (status != 0) ? 0 : 1;
1620 KC_TRACE(10, (
"__kmpc_end_barrier_master: called T#%d\n", global_tid));
1622 __kmp_end_split_barrier(bs_plain_barrier, global_tid);
1638 KC_TRACE(10, (
"__kmpc_barrier_master_nowait: called T#%d\n", global_tid));
1640 if (!TCR_4(__kmp_init_parallel))
1641 __kmp_parallel_initialize();
1643 if (__kmp_env_consistency_check) {
1645 KMP_WARNING(ConstructIdentInvalid);
1647 __kmp_check_barrier(global_tid, ct_barrier, loc);
1651 omp_frame_t *ompt_frame;
1652 if (ompt_enabled.enabled) {
1653 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1654 if (ompt_frame->enter_frame == NULL)
1655 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
1656 OMPT_STORE_RETURN_ADDRESS(global_tid);
1660 __kmp_threads[global_tid]->th.th_ident = loc;
1662 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
1663 #if OMPT_SUPPORT && OMPT_OPTIONAL 1664 if (ompt_enabled.enabled) {
1665 ompt_frame->enter_frame = NULL;
1671 if (__kmp_env_consistency_check) {
1675 if (global_tid < 0) {
1676 KMP_WARNING(ThreadIdentInvalid);
1682 __kmp_pop_sync(global_tid, ct_master, loc);
1702 kmp_int32 rc = __kmp_enter_single(global_tid, loc, TRUE);
1707 KMP_PUSH_PARTITIONED_TIMER(OMP_single);
1710 #if OMPT_SUPPORT && OMPT_OPTIONAL 1711 kmp_info_t *this_thr = __kmp_threads[global_tid];
1712 kmp_team_t *team = this_thr->th.th_team;
1713 int tid = __kmp_tid_from_gtid(global_tid);
1715 if (ompt_enabled.enabled) {
1717 if (ompt_enabled.ompt_callback_work) {
1718 ompt_callbacks.ompt_callback(ompt_callback_work)(
1719 ompt_work_single_executor, ompt_scope_begin,
1720 &(team->t.ompt_team_info.parallel_data),
1721 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1722 1, OMPT_GET_RETURN_ADDRESS(0));
1725 if (ompt_enabled.ompt_callback_work) {
1726 ompt_callbacks.ompt_callback(ompt_callback_work)(
1727 ompt_work_single_other, ompt_scope_begin,
1728 &(team->t.ompt_team_info.parallel_data),
1729 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1730 1, OMPT_GET_RETURN_ADDRESS(0));
1731 ompt_callbacks.ompt_callback(ompt_callback_work)(
1732 ompt_work_single_other, ompt_scope_end,
1733 &(team->t.ompt_team_info.parallel_data),
1734 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1735 1, OMPT_GET_RETURN_ADDRESS(0));
1754 __kmp_exit_single(global_tid);
1755 KMP_POP_PARTITIONED_TIMER();
1757 #if OMPT_SUPPORT && OMPT_OPTIONAL 1758 kmp_info_t *this_thr = __kmp_threads[global_tid];
1759 kmp_team_t *team = this_thr->th.th_team;
1760 int tid = __kmp_tid_from_gtid(global_tid);
1762 if (ompt_enabled.ompt_callback_work) {
1763 ompt_callbacks.ompt_callback(ompt_callback_work)(
1764 ompt_work_single_executor, ompt_scope_end,
1765 &(team->t.ompt_team_info.parallel_data),
1766 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
1767 OMPT_GET_RETURN_ADDRESS(0));
1780 KMP_POP_PARTITIONED_TIMER();
1781 KE_TRACE(10, (
"__kmpc_for_static_fini called T#%d\n", global_tid));
1783 #if OMPT_SUPPORT && OMPT_OPTIONAL 1784 if (ompt_enabled.ompt_callback_work) {
1785 ompt_work_t ompt_work_type = ompt_work_loop;
1786 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
1787 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
1791 ompt_work_type = ompt_work_loop;
1793 ompt_work_type = ompt_work_sections;
1795 ompt_work_type = ompt_work_distribute;
1800 KMP_DEBUG_ASSERT(ompt_work_type);
1802 ompt_callbacks.ompt_callback(ompt_callback_work)(
1803 ompt_work_type, ompt_scope_end, &(team_info->parallel_data),
1804 &(task_info->task_data), 0, OMPT_GET_RETURN_ADDRESS(0));
1807 if (__kmp_env_consistency_check)
1808 __kmp_pop_workshare(global_tid, ct_pdo, loc);
1814 void ompc_set_num_threads(
int arg) {
1816 __kmp_set_num_threads(arg, __kmp_entry_gtid());
1819 void ompc_set_dynamic(
int flag) {
1823 thread = __kmp_entry_thread();
1825 __kmp_save_internal_controls(thread);
1827 set__dynamic(thread, flag ? TRUE : FALSE);
1830 void ompc_set_nested(
int flag) {
1834 thread = __kmp_entry_thread();
1836 __kmp_save_internal_controls(thread);
1838 set__nested(thread, flag ? TRUE : FALSE);
1841 void ompc_set_max_active_levels(
int max_active_levels) {
1846 __kmp_set_max_active_levels(__kmp_entry_gtid(), max_active_levels);
1849 void ompc_set_schedule(omp_sched_t kind,
int modifier) {
1851 __kmp_set_schedule(__kmp_entry_gtid(), (kmp_sched_t)kind, modifier);
1854 int ompc_get_ancestor_thread_num(
int level) {
1855 return __kmp_get_ancestor_thread_num(__kmp_entry_gtid(), level);
1858 int ompc_get_team_size(
int level) {
1859 return __kmp_get_team_size(__kmp_entry_gtid(), level);
1862 void kmpc_set_stacksize(
int arg) {
1864 __kmp_aux_set_stacksize(arg);
1867 void kmpc_set_stacksize_s(
size_t arg) {
1869 __kmp_aux_set_stacksize(arg);
1872 void kmpc_set_blocktime(
int arg) {
1876 gtid = __kmp_entry_gtid();
1877 tid = __kmp_tid_from_gtid(gtid);
1878 thread = __kmp_thread_from_gtid(gtid);
1880 __kmp_aux_set_blocktime(arg, thread, tid);
1883 void kmpc_set_library(
int arg) {
1885 __kmp_user_set_library((
enum library_type)arg);
1888 void kmpc_set_defaults(
char const *str) {
1890 __kmp_aux_set_defaults(str, KMP_STRLEN(str));
1893 void kmpc_set_disp_num_buffers(
int arg) {
1896 if (__kmp_init_serial == 0 && arg > 0)
1897 __kmp_dispatch_num_buffers = arg;
1900 int kmpc_set_affinity_mask_proc(
int proc,
void **mask) {
1901 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1904 if (!TCR_4(__kmp_init_middle)) {
1905 __kmp_middle_initialize();
1907 return __kmp_aux_set_affinity_mask_proc(proc, mask);
1911 int kmpc_unset_affinity_mask_proc(
int proc,
void **mask) {
1912 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1915 if (!TCR_4(__kmp_init_middle)) {
1916 __kmp_middle_initialize();
1918 return __kmp_aux_unset_affinity_mask_proc(proc, mask);
1922 int kmpc_get_affinity_mask_proc(
int proc,
void **mask) {
1923 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1926 if (!TCR_4(__kmp_init_middle)) {
1927 __kmp_middle_initialize();
1929 return __kmp_aux_get_affinity_mask_proc(proc, mask);
1979 void *cpy_data,
void (*cpy_func)(
void *,
void *),
1983 KC_TRACE(10, (
"__kmpc_copyprivate: called T#%d\n", gtid));
1987 data_ptr = &__kmp_team_from_gtid(gtid)->t.t_copypriv_data;
1989 if (__kmp_env_consistency_check) {
1991 KMP_WARNING(ConstructIdentInvalid);
1998 *data_ptr = cpy_data;
2001 omp_frame_t *ompt_frame;
2002 if (ompt_enabled.enabled) {
2003 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
2004 if (ompt_frame->enter_frame == NULL)
2005 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
2006 OMPT_STORE_RETURN_ADDRESS(gtid);
2011 __kmp_threads[gtid]->th.th_ident = loc;
2013 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2016 (*cpy_func)(cpy_data, *data_ptr);
2022 if (ompt_enabled.enabled) {
2023 OMPT_STORE_RETURN_ADDRESS(gtid);
2027 __kmp_threads[gtid]->th.th_ident = loc;
2030 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2031 #if OMPT_SUPPORT && OMPT_OPTIONAL 2032 if (ompt_enabled.enabled) {
2033 ompt_frame->enter_frame = NULL;
2040 #define INIT_LOCK __kmp_init_user_lock_with_checks 2041 #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks 2042 #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks 2043 #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed 2044 #define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks 2045 #define ACQUIRE_NESTED_LOCK_TIMED \ 2046 __kmp_acquire_nested_user_lock_with_checks_timed 2047 #define RELEASE_LOCK __kmp_release_user_lock_with_checks 2048 #define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks 2049 #define TEST_LOCK __kmp_test_user_lock_with_checks 2050 #define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks 2051 #define DESTROY_LOCK __kmp_destroy_user_lock_with_checks 2052 #define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks 2057 #if KMP_USE_DYNAMIC_LOCK 2060 static __forceinline
void __kmp_init_lock_with_hint(
ident_t *loc,
void **lock,
2061 kmp_dyna_lockseq_t seq) {
2062 if (KMP_IS_D_LOCK(seq)) {
2063 KMP_INIT_D_LOCK(lock, seq);
2065 __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
2068 KMP_INIT_I_LOCK(lock, seq);
2070 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2071 __kmp_itt_lock_creating(ilk->lock, loc);
2077 static __forceinline
void 2078 __kmp_init_nest_lock_with_hint(
ident_t *loc,
void **lock,
2079 kmp_dyna_lockseq_t seq) {
2082 if (seq == lockseq_hle || seq == lockseq_rtm || seq == lockseq_adaptive)
2083 seq = __kmp_user_lock_seq;
2087 seq = lockseq_nested_tas;
2091 seq = lockseq_nested_futex;
2094 case lockseq_ticket:
2095 seq = lockseq_nested_ticket;
2097 case lockseq_queuing:
2098 seq = lockseq_nested_queuing;
2101 seq = lockseq_nested_drdpa;
2104 seq = lockseq_nested_queuing;
2106 KMP_INIT_I_LOCK(lock, seq);
2108 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2109 __kmp_itt_lock_creating(ilk->lock, loc);
2114 void __kmpc_init_lock_with_hint(
ident_t *loc, kmp_int32 gtid,
void **user_lock,
2116 KMP_DEBUG_ASSERT(__kmp_init_serial);
2117 if (__kmp_env_consistency_check && user_lock == NULL) {
2118 KMP_FATAL(LockIsUninitialized,
"omp_init_lock_with_hint");
2121 __kmp_init_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2123 #if OMPT_SUPPORT && OMPT_OPTIONAL 2125 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2127 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2128 if (ompt_enabled.ompt_callback_lock_init) {
2129 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2130 ompt_mutex_lock, (omp_lock_hint_t)hint,
2131 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2138 void __kmpc_init_nest_lock_with_hint(
ident_t *loc, kmp_int32 gtid,
2139 void **user_lock, uintptr_t hint) {
2140 KMP_DEBUG_ASSERT(__kmp_init_serial);
2141 if (__kmp_env_consistency_check && user_lock == NULL) {
2142 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock_with_hint");
2145 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2147 #if OMPT_SUPPORT && OMPT_OPTIONAL 2149 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2151 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2152 if (ompt_enabled.ompt_callback_lock_init) {
2153 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2154 ompt_mutex_nest_lock, (omp_lock_hint_t)hint,
2155 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2161 #endif // KMP_USE_DYNAMIC_LOCK 2164 void __kmpc_init_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2165 #if KMP_USE_DYNAMIC_LOCK 2167 KMP_DEBUG_ASSERT(__kmp_init_serial);
2168 if (__kmp_env_consistency_check && user_lock == NULL) {
2169 KMP_FATAL(LockIsUninitialized,
"omp_init_lock");
2171 __kmp_init_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2173 #if OMPT_SUPPORT && OMPT_OPTIONAL 2175 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2177 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2178 if (ompt_enabled.ompt_callback_lock_init) {
2179 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2180 ompt_mutex_lock, omp_lock_hint_none,
2181 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2186 #else // KMP_USE_DYNAMIC_LOCK 2188 static char const *
const func =
"omp_init_lock";
2189 kmp_user_lock_p lck;
2190 KMP_DEBUG_ASSERT(__kmp_init_serial);
2192 if (__kmp_env_consistency_check) {
2193 if (user_lock == NULL) {
2194 KMP_FATAL(LockIsUninitialized, func);
2198 KMP_CHECK_USER_LOCK_INIT();
2200 if ((__kmp_user_lock_kind == lk_tas) &&
2201 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2202 lck = (kmp_user_lock_p)user_lock;
2205 else if ((__kmp_user_lock_kind == lk_futex) &&
2206 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2207 lck = (kmp_user_lock_p)user_lock;
2211 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2214 __kmp_set_user_lock_location(lck, loc);
2216 #if OMPT_SUPPORT && OMPT_OPTIONAL 2218 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2220 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2221 if (ompt_enabled.ompt_callback_lock_init) {
2222 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2223 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2224 (omp_wait_id_t)user_lock, codeptr);
2229 __kmp_itt_lock_creating(lck);
2232 #endif // KMP_USE_DYNAMIC_LOCK 2236 void __kmpc_init_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2237 #if KMP_USE_DYNAMIC_LOCK 2239 KMP_DEBUG_ASSERT(__kmp_init_serial);
2240 if (__kmp_env_consistency_check && user_lock == NULL) {
2241 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock");
2243 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2245 #if OMPT_SUPPORT && OMPT_OPTIONAL 2247 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2249 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2250 if (ompt_enabled.ompt_callback_lock_init) {
2251 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2252 ompt_mutex_nest_lock, omp_lock_hint_none,
2253 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2258 #else // KMP_USE_DYNAMIC_LOCK 2260 static char const *
const func =
"omp_init_nest_lock";
2261 kmp_user_lock_p lck;
2262 KMP_DEBUG_ASSERT(__kmp_init_serial);
2264 if (__kmp_env_consistency_check) {
2265 if (user_lock == NULL) {
2266 KMP_FATAL(LockIsUninitialized, func);
2270 KMP_CHECK_USER_LOCK_INIT();
2272 if ((__kmp_user_lock_kind == lk_tas) &&
2273 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2274 OMP_NEST_LOCK_T_SIZE)) {
2275 lck = (kmp_user_lock_p)user_lock;
2278 else if ((__kmp_user_lock_kind == lk_futex) &&
2279 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2280 OMP_NEST_LOCK_T_SIZE)) {
2281 lck = (kmp_user_lock_p)user_lock;
2285 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2288 INIT_NESTED_LOCK(lck);
2289 __kmp_set_user_lock_location(lck, loc);
2291 #if OMPT_SUPPORT && OMPT_OPTIONAL 2293 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2295 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2296 if (ompt_enabled.ompt_callback_lock_init) {
2297 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2298 ompt_mutex_nest_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2299 (omp_wait_id_t)user_lock, codeptr);
2304 __kmp_itt_lock_creating(lck);
2307 #endif // KMP_USE_DYNAMIC_LOCK 2310 void __kmpc_destroy_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2311 #if KMP_USE_DYNAMIC_LOCK 2314 kmp_user_lock_p lck;
2315 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2316 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2318 lck = (kmp_user_lock_p)user_lock;
2320 __kmp_itt_lock_destroyed(lck);
2322 #if OMPT_SUPPORT && OMPT_OPTIONAL 2324 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2326 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2327 if (ompt_enabled.ompt_callback_lock_destroy) {
2328 kmp_user_lock_p lck;
2329 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2330 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2332 lck = (kmp_user_lock_p)user_lock;
2334 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2335 ompt_mutex_lock, (omp_wait_id_t)user_lock, codeptr);
2338 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2340 kmp_user_lock_p lck;
2342 if ((__kmp_user_lock_kind == lk_tas) &&
2343 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2344 lck = (kmp_user_lock_p)user_lock;
2347 else if ((__kmp_user_lock_kind == lk_futex) &&
2348 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2349 lck = (kmp_user_lock_p)user_lock;
2353 lck = __kmp_lookup_user_lock(user_lock,
"omp_destroy_lock");
2356 #if OMPT_SUPPORT && OMPT_OPTIONAL 2358 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2360 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2361 if (ompt_enabled.ompt_callback_lock_destroy) {
2362 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2363 ompt_mutex_lock, (omp_wait_id_t)user_lock, codeptr);
2368 __kmp_itt_lock_destroyed(lck);
2372 if ((__kmp_user_lock_kind == lk_tas) &&
2373 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2377 else if ((__kmp_user_lock_kind == lk_futex) &&
2378 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2383 __kmp_user_lock_free(user_lock, gtid, lck);
2385 #endif // KMP_USE_DYNAMIC_LOCK 2389 void __kmpc_destroy_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2390 #if KMP_USE_DYNAMIC_LOCK 2393 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock);
2394 __kmp_itt_lock_destroyed(ilk->lock);
2396 #if OMPT_SUPPORT && OMPT_OPTIONAL 2398 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2400 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2401 if (ompt_enabled.ompt_callback_lock_destroy) {
2402 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2403 ompt_mutex_nest_lock, (omp_wait_id_t)user_lock, codeptr);
2406 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2408 #else // KMP_USE_DYNAMIC_LOCK 2410 kmp_user_lock_p lck;
2412 if ((__kmp_user_lock_kind == lk_tas) &&
2413 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2414 OMP_NEST_LOCK_T_SIZE)) {
2415 lck = (kmp_user_lock_p)user_lock;
2418 else if ((__kmp_user_lock_kind == lk_futex) &&
2419 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2420 OMP_NEST_LOCK_T_SIZE)) {
2421 lck = (kmp_user_lock_p)user_lock;
2425 lck = __kmp_lookup_user_lock(user_lock,
"omp_destroy_nest_lock");
2428 #if OMPT_SUPPORT && OMPT_OPTIONAL 2430 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2432 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2433 if (ompt_enabled.ompt_callback_lock_destroy) {
2434 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2435 ompt_mutex_nest_lock, (omp_wait_id_t)user_lock, codeptr);
2440 __kmp_itt_lock_destroyed(lck);
2443 DESTROY_NESTED_LOCK(lck);
2445 if ((__kmp_user_lock_kind == lk_tas) &&
2446 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2447 OMP_NEST_LOCK_T_SIZE)) {
2451 else if ((__kmp_user_lock_kind == lk_futex) &&
2452 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2453 OMP_NEST_LOCK_T_SIZE)) {
2458 __kmp_user_lock_free(user_lock, gtid, lck);
2460 #endif // KMP_USE_DYNAMIC_LOCK 2463 void __kmpc_set_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2465 #if KMP_USE_DYNAMIC_LOCK 2466 int tag = KMP_EXTRACT_D_TAG(user_lock);
2468 __kmp_itt_lock_acquiring(
2472 #if OMPT_SUPPORT && OMPT_OPTIONAL 2474 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2476 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2477 if (ompt_enabled.ompt_callback_mutex_acquire) {
2478 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2479 ompt_mutex_lock, omp_lock_hint_none,
2480 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2484 #if KMP_USE_INLINED_TAS 2485 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2486 KMP_ACQUIRE_TAS_LOCK(user_lock, gtid);
2488 #elif KMP_USE_INLINED_FUTEX 2489 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2490 KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
2494 __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2497 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2499 #if OMPT_SUPPORT && OMPT_OPTIONAL 2500 if (ompt_enabled.ompt_callback_mutex_acquired) {
2501 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2502 ompt_mutex_lock, (omp_wait_id_t)user_lock, codeptr);
2506 #else // KMP_USE_DYNAMIC_LOCK 2508 kmp_user_lock_p lck;
2510 if ((__kmp_user_lock_kind == lk_tas) &&
2511 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2512 lck = (kmp_user_lock_p)user_lock;
2515 else if ((__kmp_user_lock_kind == lk_futex) &&
2516 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2517 lck = (kmp_user_lock_p)user_lock;
2521 lck = __kmp_lookup_user_lock(user_lock,
"omp_set_lock");
2525 __kmp_itt_lock_acquiring(lck);
2527 #if OMPT_SUPPORT && OMPT_OPTIONAL 2529 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2531 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2532 if (ompt_enabled.ompt_callback_mutex_acquire) {
2533 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2534 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2535 (omp_wait_id_t)lck, codeptr);
2539 ACQUIRE_LOCK(lck, gtid);
2542 __kmp_itt_lock_acquired(lck);
2545 #if OMPT_SUPPORT && OMPT_OPTIONAL 2546 if (ompt_enabled.ompt_callback_mutex_acquired) {
2547 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2548 ompt_mutex_lock, (omp_wait_id_t)lck, codeptr);
2552 #endif // KMP_USE_DYNAMIC_LOCK 2555 void __kmpc_set_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2556 #if KMP_USE_DYNAMIC_LOCK 2559 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2561 #if OMPT_SUPPORT && OMPT_OPTIONAL 2563 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2565 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2566 if (ompt_enabled.enabled) {
2567 if (ompt_enabled.ompt_callback_mutex_acquire) {
2568 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2569 ompt_mutex_nest_lock, omp_lock_hint_none,
2570 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2575 int acquire_status =
2576 KMP_D_LOCK_FUNC(user_lock,
set)((kmp_dyna_lock_t *)user_lock, gtid);
2577 (void) acquire_status;
2579 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2582 #if OMPT_SUPPORT && OMPT_OPTIONAL 2583 if (ompt_enabled.enabled) {
2584 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2585 if (ompt_enabled.ompt_callback_mutex_acquired) {
2587 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2588 ompt_mutex_nest_lock, (omp_wait_id_t)user_lock, codeptr);
2591 if (ompt_enabled.ompt_callback_nest_lock) {
2593 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2594 ompt_scope_begin, (omp_wait_id_t)user_lock, codeptr);
2600 #else // KMP_USE_DYNAMIC_LOCK 2602 kmp_user_lock_p lck;
2604 if ((__kmp_user_lock_kind == lk_tas) &&
2605 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2606 OMP_NEST_LOCK_T_SIZE)) {
2607 lck = (kmp_user_lock_p)user_lock;
2610 else if ((__kmp_user_lock_kind == lk_futex) &&
2611 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2612 OMP_NEST_LOCK_T_SIZE)) {
2613 lck = (kmp_user_lock_p)user_lock;
2617 lck = __kmp_lookup_user_lock(user_lock,
"omp_set_nest_lock");
2621 __kmp_itt_lock_acquiring(lck);
2623 #if OMPT_SUPPORT && OMPT_OPTIONAL 2625 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2627 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2628 if (ompt_enabled.enabled) {
2629 if (ompt_enabled.ompt_callback_mutex_acquire) {
2630 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2631 ompt_mutex_nest_lock, omp_lock_hint_none,
2632 __ompt_get_mutex_impl_type(), (omp_wait_id_t)lck, codeptr);
2637 ACQUIRE_NESTED_LOCK(lck, gtid, &acquire_status);
2640 __kmp_itt_lock_acquired(lck);
2643 #if OMPT_SUPPORT && OMPT_OPTIONAL 2644 if (ompt_enabled.enabled) {
2645 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2646 if (ompt_enabled.ompt_callback_mutex_acquired) {
2648 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2649 ompt_mutex_nest_lock, (omp_wait_id_t)lck, codeptr);
2652 if (ompt_enabled.ompt_callback_nest_lock) {
2654 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2655 ompt_scope_begin, (omp_wait_id_t)lck, codeptr);
2661 #endif // KMP_USE_DYNAMIC_LOCK 2664 void __kmpc_unset_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2665 #if KMP_USE_DYNAMIC_LOCK 2667 int tag = KMP_EXTRACT_D_TAG(user_lock);
2669 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2671 #if KMP_USE_INLINED_TAS 2672 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2673 KMP_RELEASE_TAS_LOCK(user_lock, gtid);
2675 #elif KMP_USE_INLINED_FUTEX 2676 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2677 KMP_RELEASE_FUTEX_LOCK(user_lock, gtid);
2681 __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2684 #if OMPT_SUPPORT && OMPT_OPTIONAL 2686 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2688 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2689 if (ompt_enabled.ompt_callback_mutex_released) {
2690 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2691 ompt_mutex_lock, (omp_wait_id_t)user_lock, codeptr);
2695 #else // KMP_USE_DYNAMIC_LOCK 2697 kmp_user_lock_p lck;
2702 if ((__kmp_user_lock_kind == lk_tas) &&
2703 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2704 #if KMP_OS_LINUX && \ 2705 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2708 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2710 TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
2713 #if OMPT_SUPPORT && OMPT_OPTIONAL 2715 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2717 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2718 if (ompt_enabled.ompt_callback_mutex_released) {
2719 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2720 ompt_mutex_lock, (omp_wait_id_t)lck, codeptr);
2726 lck = (kmp_user_lock_p)user_lock;
2730 else if ((__kmp_user_lock_kind == lk_futex) &&
2731 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2732 lck = (kmp_user_lock_p)user_lock;
2736 lck = __kmp_lookup_user_lock(user_lock,
"omp_unset_lock");
2740 __kmp_itt_lock_releasing(lck);
2743 RELEASE_LOCK(lck, gtid);
2745 #if OMPT_SUPPORT && OMPT_OPTIONAL 2747 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2749 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2750 if (ompt_enabled.ompt_callback_mutex_released) {
2751 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2752 ompt_mutex_lock, (omp_wait_id_t)lck, codeptr);
2756 #endif // KMP_USE_DYNAMIC_LOCK 2760 void __kmpc_unset_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2761 #if KMP_USE_DYNAMIC_LOCK 2764 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2766 int release_status =
2767 KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
2768 (void) release_status;
2770 #if OMPT_SUPPORT && OMPT_OPTIONAL 2772 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2774 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2775 if (ompt_enabled.enabled) {
2776 if (release_status == KMP_LOCK_RELEASED) {
2777 if (ompt_enabled.ompt_callback_mutex_released) {
2779 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2780 ompt_mutex_nest_lock, (omp_wait_id_t)user_lock, codeptr);
2782 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2784 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2785 ompt_scope_end, (omp_wait_id_t)user_lock, codeptr);
2790 #else // KMP_USE_DYNAMIC_LOCK 2792 kmp_user_lock_p lck;
2796 if ((__kmp_user_lock_kind == lk_tas) &&
2797 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2798 OMP_NEST_LOCK_T_SIZE)) {
2799 #if KMP_OS_LINUX && \ 2800 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2802 kmp_tas_lock_t *tl = (kmp_tas_lock_t *)user_lock;
2804 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2807 #if OMPT_SUPPORT && OMPT_OPTIONAL 2808 int release_status = KMP_LOCK_STILL_HELD;
2811 if (--(tl->lk.depth_locked) == 0) {
2812 TCW_4(tl->lk.poll, 0);
2813 #if OMPT_SUPPORT && OMPT_OPTIONAL 2814 release_status = KMP_LOCK_RELEASED;
2819 #if OMPT_SUPPORT && OMPT_OPTIONAL 2821 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2823 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2824 if (ompt_enabled.enabled) {
2825 if (release_status == KMP_LOCK_RELEASED) {
2826 if (ompt_enabled.ompt_callback_mutex_released) {
2828 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2829 ompt_mutex_nest_lock, (omp_wait_id_t)lck, codeptr);
2831 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2833 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2834 ompt_mutex_scope_end, (omp_wait_id_t)lck, codeptr);
2841 lck = (kmp_user_lock_p)user_lock;
2845 else if ((__kmp_user_lock_kind == lk_futex) &&
2846 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2847 OMP_NEST_LOCK_T_SIZE)) {
2848 lck = (kmp_user_lock_p)user_lock;
2852 lck = __kmp_lookup_user_lock(user_lock,
"omp_unset_nest_lock");
2856 __kmp_itt_lock_releasing(lck);
2860 release_status = RELEASE_NESTED_LOCK(lck, gtid);
2861 #if OMPT_SUPPORT && OMPT_OPTIONAL 2863 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2865 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2866 if (ompt_enabled.enabled) {
2867 if (release_status == KMP_LOCK_RELEASED) {
2868 if (ompt_enabled.ompt_callback_mutex_released) {
2870 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2871 ompt_mutex_nest_lock, (omp_wait_id_t)lck, codeptr);
2873 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2875 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2876 ompt_mutex_scope_end, (omp_wait_id_t)lck, codeptr);
2881 #endif // KMP_USE_DYNAMIC_LOCK 2885 int __kmpc_test_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2888 #if KMP_USE_DYNAMIC_LOCK 2890 int tag = KMP_EXTRACT_D_TAG(user_lock);
2892 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2894 #if OMPT_SUPPORT && OMPT_OPTIONAL 2896 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2898 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2899 if (ompt_enabled.ompt_callback_mutex_acquire) {
2900 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2901 ompt_mutex_lock, omp_lock_hint_none,
2902 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2906 #if KMP_USE_INLINED_TAS 2907 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2908 KMP_TEST_TAS_LOCK(user_lock, gtid, rc);
2910 #elif KMP_USE_INLINED_FUTEX 2911 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2912 KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc);
2916 rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2920 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2922 #if OMPT_SUPPORT && OMPT_OPTIONAL 2923 if (ompt_enabled.ompt_callback_mutex_acquired) {
2924 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2925 ompt_mutex_lock, (omp_wait_id_t)user_lock, codeptr);
2931 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
2936 #else // KMP_USE_DYNAMIC_LOCK 2938 kmp_user_lock_p lck;
2941 if ((__kmp_user_lock_kind == lk_tas) &&
2942 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2943 lck = (kmp_user_lock_p)user_lock;
2946 else if ((__kmp_user_lock_kind == lk_futex) &&
2947 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2948 lck = (kmp_user_lock_p)user_lock;
2952 lck = __kmp_lookup_user_lock(user_lock,
"omp_test_lock");
2956 __kmp_itt_lock_acquiring(lck);
2958 #if OMPT_SUPPORT && OMPT_OPTIONAL 2960 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2962 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2963 if (ompt_enabled.ompt_callback_mutex_acquire) {
2964 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2965 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2966 (omp_wait_id_t)lck, codeptr);
2970 rc = TEST_LOCK(lck, gtid);
2973 __kmp_itt_lock_acquired(lck);
2975 __kmp_itt_lock_cancelled(lck);
2978 #if OMPT_SUPPORT && OMPT_OPTIONAL 2979 if (rc && ompt_enabled.ompt_callback_mutex_acquired) {
2980 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2981 ompt_mutex_lock, (omp_wait_id_t)lck, codeptr);
2985 return (rc ? FTN_TRUE : FTN_FALSE);
2989 #endif // KMP_USE_DYNAMIC_LOCK 2993 int __kmpc_test_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2994 #if KMP_USE_DYNAMIC_LOCK 2997 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2999 #if OMPT_SUPPORT && OMPT_OPTIONAL 3001 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3003 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3004 if (ompt_enabled.ompt_callback_mutex_acquire) {
3005 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3006 ompt_mutex_nest_lock, omp_lock_hint_none,
3007 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
3011 rc = KMP_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid);
3014 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
3016 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
3019 #if OMPT_SUPPORT && OMPT_OPTIONAL 3020 if (ompt_enabled.enabled && rc) {
3022 if (ompt_enabled.ompt_callback_mutex_acquired) {
3024 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3025 ompt_mutex_nest_lock, (omp_wait_id_t)user_lock, codeptr);
3028 if (ompt_enabled.ompt_callback_nest_lock) {
3030 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3031 ompt_scope_begin, (omp_wait_id_t)user_lock, codeptr);
3038 #else // KMP_USE_DYNAMIC_LOCK 3040 kmp_user_lock_p lck;
3043 if ((__kmp_user_lock_kind == lk_tas) &&
3044 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
3045 OMP_NEST_LOCK_T_SIZE)) {
3046 lck = (kmp_user_lock_p)user_lock;
3049 else if ((__kmp_user_lock_kind == lk_futex) &&
3050 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
3051 OMP_NEST_LOCK_T_SIZE)) {
3052 lck = (kmp_user_lock_p)user_lock;
3056 lck = __kmp_lookup_user_lock(user_lock,
"omp_test_nest_lock");
3060 __kmp_itt_lock_acquiring(lck);
3063 #if OMPT_SUPPORT && OMPT_OPTIONAL 3065 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3067 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3068 if (ompt_enabled.enabled) &&
3069 ompt_enabled.ompt_callback_mutex_acquire) {
3070 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3071 ompt_mutex_nest_lock, omp_lock_hint_none,
3072 __ompt_get_mutex_impl_type(), (omp_wait_id_t)lck, codeptr);
3076 rc = TEST_NESTED_LOCK(lck, gtid);
3079 __kmp_itt_lock_acquired(lck);
3081 __kmp_itt_lock_cancelled(lck);
3084 #if OMPT_SUPPORT && OMPT_OPTIONAL 3085 if (ompt_enabled.enabled && rc) {
3087 if (ompt_enabled.ompt_callback_mutex_acquired) {
3089 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3090 ompt_mutex_nest_lock, (omp_wait_id_t)lck, codeptr);
3093 if (ompt_enabled.ompt_callback_nest_lock) {
3095 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3096 ompt_mutex_scope_begin, (omp_wait_id_t)lck, codeptr);
3105 #endif // KMP_USE_DYNAMIC_LOCK 3115 #define __KMP_SET_REDUCTION_METHOD(gtid, rmethod) \ 3116 ((__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) = (rmethod)) 3118 #define __KMP_GET_REDUCTION_METHOD(gtid) \ 3119 (__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) 3125 static __forceinline
void 3126 __kmp_enter_critical_section_reduce_block(
ident_t *loc, kmp_int32 global_tid,
3127 kmp_critical_name *crit) {
3133 kmp_user_lock_p lck;
3135 #if KMP_USE_DYNAMIC_LOCK 3137 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
3140 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3141 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)crit, 0,
3142 KMP_GET_D_TAG(__kmp_user_lock_seq));
3144 __kmp_init_indirect_csptr(crit, loc, global_tid,
3145 KMP_GET_I_TAG(__kmp_user_lock_seq));
3151 if (KMP_EXTRACT_D_TAG(lk) != 0) {
3152 lck = (kmp_user_lock_p)lk;
3153 KMP_DEBUG_ASSERT(lck != NULL);
3154 if (__kmp_env_consistency_check) {
3155 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3157 KMP_D_LOCK_FUNC(lk,
set)(lk, global_tid);
3159 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
3161 KMP_DEBUG_ASSERT(lck != NULL);
3162 if (__kmp_env_consistency_check) {
3163 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3165 KMP_I_LOCK_FUNC(ilk,
set)(lck, global_tid);
3168 #else // KMP_USE_DYNAMIC_LOCK 3173 if (__kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE) {
3174 lck = (kmp_user_lock_p)crit;
3176 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
3178 KMP_DEBUG_ASSERT(lck != NULL);
3180 if (__kmp_env_consistency_check)
3181 __kmp_push_sync(global_tid, ct_critical, loc, lck);
3183 __kmp_acquire_user_lock_with_checks(lck, global_tid);
3185 #endif // KMP_USE_DYNAMIC_LOCK 3189 static __forceinline
void 3190 __kmp_end_critical_section_reduce_block(
ident_t *loc, kmp_int32 global_tid,
3191 kmp_critical_name *crit) {
3193 kmp_user_lock_p lck;
3195 #if KMP_USE_DYNAMIC_LOCK 3197 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3198 lck = (kmp_user_lock_p)crit;
3199 if (__kmp_env_consistency_check)
3200 __kmp_pop_sync(global_tid, ct_critical, loc);
3201 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
3203 kmp_indirect_lock_t *ilk =
3204 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
3205 if (__kmp_env_consistency_check)
3206 __kmp_pop_sync(global_tid, ct_critical, loc);
3207 KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
3210 #else // KMP_USE_DYNAMIC_LOCK 3215 if (__kmp_base_user_lock_size > 32) {
3216 lck = *((kmp_user_lock_p *)crit);
3217 KMP_ASSERT(lck != NULL);
3219 lck = (kmp_user_lock_p)crit;
3222 if (__kmp_env_consistency_check)
3223 __kmp_pop_sync(global_tid, ct_critical, loc);
3225 __kmp_release_user_lock_with_checks(lck, global_tid);
3227 #endif // KMP_USE_DYNAMIC_LOCK 3231 static __forceinline
int 3232 __kmp_swap_teams_for_teams_reduction(kmp_info_t *th, kmp_team_t **team_p,
3237 if (th->th.th_teams_microtask) {
3238 *team_p = team = th->th.th_team;
3239 if (team->t.t_level == th->th.th_teams_level) {
3241 KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid);
3243 th->th.th_info.ds.ds_tid = team->t.t_master_tid;
3244 th->th.th_team = team->t.t_parent;
3245 th->th.th_team_nproc = th->th.th_team->t.t_nproc;
3246 th->th.th_task_team = th->th.th_team->t.t_task_team[0];
3247 *task_state = th->th.th_task_state;
3248 th->th.th_task_state = 0;
3256 static __forceinline
void 3257 __kmp_restore_swapped_teams(kmp_info_t *th, kmp_team_t *team,
int task_state) {
3259 th->th.th_info.ds.ds_tid = 0;
3260 th->th.th_team = team;
3261 th->th.th_team_nproc = team->t.t_nproc;
3262 th->th.th_task_team = team->t.t_task_team[task_state];
3263 th->th.th_task_state = task_state;
3285 size_t reduce_size,
void *reduce_data,
3286 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
3287 kmp_critical_name *lck) {
3291 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3295 int teams_swapped = 0, task_state;
3297 KA_TRACE(10, (
"__kmpc_reduce_nowait() enter: called T#%d\n", global_tid));
3305 if (!TCR_4(__kmp_init_parallel))
3306 __kmp_parallel_initialize();
3309 #if KMP_USE_DYNAMIC_LOCK 3310 if (__kmp_env_consistency_check)
3311 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3313 if (__kmp_env_consistency_check)
3314 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3318 th = __kmp_thread_from_gtid(global_tid);
3319 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3320 #endif // OMP_40_ENABLED 3338 packed_reduction_method = __kmp_determine_reduction_method(
3339 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3340 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3342 if (packed_reduction_method == critical_reduce_block) {
3344 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3347 }
else if (packed_reduction_method == empty_reduce_block) {
3353 }
else if (packed_reduction_method == atomic_reduce_block) {
3363 if (__kmp_env_consistency_check)
3364 __kmp_pop_sync(global_tid, ct_reduce, loc);
3366 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3367 tree_reduce_block)) {
3387 omp_frame_t *ompt_frame;
3388 if (ompt_enabled.enabled) {
3389 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3390 if (ompt_frame->enter_frame == NULL)
3391 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3392 OMPT_STORE_RETURN_ADDRESS(global_tid);
3396 __kmp_threads[global_tid]->th.th_ident = loc;
3399 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3400 global_tid, FALSE, reduce_size, reduce_data, reduce_func);
3401 retval = (retval != 0) ? (0) : (1);
3402 #if OMPT_SUPPORT && OMPT_OPTIONAL 3403 if (ompt_enabled.enabled) {
3404 ompt_frame->enter_frame = NULL;
3410 if (__kmp_env_consistency_check) {
3412 __kmp_pop_sync(global_tid, ct_reduce, loc);
3422 if (teams_swapped) {
3423 __kmp_restore_swapped_teams(th, team, task_state);
3428 (
"__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n",
3429 global_tid, packed_reduction_method, retval));
3443 kmp_critical_name *lck) {
3445 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3447 KA_TRACE(10, (
"__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid));
3449 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
3451 if (packed_reduction_method == critical_reduce_block) {
3453 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
3455 }
else if (packed_reduction_method == empty_reduce_block) {
3460 }
else if (packed_reduction_method == atomic_reduce_block) {
3467 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3468 tree_reduce_block)) {
3478 if (__kmp_env_consistency_check)
3479 __kmp_pop_sync(global_tid, ct_reduce, loc);
3481 KA_TRACE(10, (
"__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n",
3482 global_tid, packed_reduction_method));
3505 size_t reduce_size,
void *reduce_data,
3506 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
3507 kmp_critical_name *lck) {
3510 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3514 int teams_swapped = 0, task_state;
3517 KA_TRACE(10, (
"__kmpc_reduce() enter: called T#%d\n", global_tid));
3525 if (!TCR_4(__kmp_init_parallel))
3526 __kmp_parallel_initialize();
3529 #if KMP_USE_DYNAMIC_LOCK 3530 if (__kmp_env_consistency_check)
3531 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3533 if (__kmp_env_consistency_check)
3534 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3538 th = __kmp_thread_from_gtid(global_tid);
3539 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3540 #endif // OMP_40_ENABLED 3542 packed_reduction_method = __kmp_determine_reduction_method(
3543 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3544 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3546 if (packed_reduction_method == critical_reduce_block) {
3548 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3551 }
else if (packed_reduction_method == empty_reduce_block) {
3557 }
else if (packed_reduction_method == atomic_reduce_block) {
3561 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3562 tree_reduce_block)) {
3568 omp_frame_t *ompt_frame;
3569 if (ompt_enabled.enabled) {
3570 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3571 if (ompt_frame->enter_frame == NULL)
3572 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3573 OMPT_STORE_RETURN_ADDRESS(global_tid);
3577 __kmp_threads[global_tid]->th.th_ident =
3581 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3582 global_tid, TRUE, reduce_size, reduce_data, reduce_func);
3583 retval = (retval != 0) ? (0) : (1);
3584 #if OMPT_SUPPORT && OMPT_OPTIONAL 3585 if (ompt_enabled.enabled) {
3586 ompt_frame->enter_frame = NULL;
3592 if (__kmp_env_consistency_check) {
3594 __kmp_pop_sync(global_tid, ct_reduce, loc);
3604 if (teams_swapped) {
3605 __kmp_restore_swapped_teams(th, team, task_state);
3610 (
"__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n",
3611 global_tid, packed_reduction_method, retval));
3627 kmp_critical_name *lck) {
3629 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3633 int teams_swapped = 0, task_state;
3636 KA_TRACE(10, (
"__kmpc_end_reduce() enter: called T#%d\n", global_tid));
3639 th = __kmp_thread_from_gtid(global_tid);
3640 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3641 #endif // OMP_40_ENABLED 3643 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
3648 if (packed_reduction_method == critical_reduce_block) {
3650 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
3654 omp_frame_t *ompt_frame;
3655 if (ompt_enabled.enabled) {
3656 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3657 if (ompt_frame->enter_frame == NULL)
3658 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3659 OMPT_STORE_RETURN_ADDRESS(global_tid);
3663 __kmp_threads[global_tid]->th.th_ident = loc;
3665 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3666 #if OMPT_SUPPORT && OMPT_OPTIONAL 3667 if (ompt_enabled.enabled) {
3668 ompt_frame->enter_frame = NULL;
3672 }
else if (packed_reduction_method == empty_reduce_block) {
3678 omp_frame_t *ompt_frame;
3679 if (ompt_enabled.enabled) {
3680 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3681 if (ompt_frame->enter_frame == NULL)
3682 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3683 OMPT_STORE_RETURN_ADDRESS(global_tid);
3687 __kmp_threads[global_tid]->th.th_ident = loc;
3689 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3690 #if OMPT_SUPPORT && OMPT_OPTIONAL 3691 if (ompt_enabled.enabled) {
3692 ompt_frame->enter_frame = NULL;
3696 }
else if (packed_reduction_method == atomic_reduce_block) {
3699 omp_frame_t *ompt_frame;
3700 if (ompt_enabled.enabled) {
3701 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3702 if (ompt_frame->enter_frame == NULL)
3703 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3704 OMPT_STORE_RETURN_ADDRESS(global_tid);
3709 __kmp_threads[global_tid]->th.th_ident = loc;
3711 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3712 #if OMPT_SUPPORT && OMPT_OPTIONAL 3713 if (ompt_enabled.enabled) {
3714 ompt_frame->enter_frame = NULL;
3718 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3719 tree_reduce_block)) {
3722 __kmp_end_split_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3731 if (teams_swapped) {
3732 __kmp_restore_swapped_teams(th, team, task_state);
3736 if (__kmp_env_consistency_check)
3737 __kmp_pop_sync(global_tid, ct_reduce, loc);
3739 KA_TRACE(10, (
"__kmpc_end_reduce() exit: called T#%d: method %08x\n",
3740 global_tid, packed_reduction_method));
3745 #undef __KMP_GET_REDUCTION_METHOD 3746 #undef __KMP_SET_REDUCTION_METHOD 3750 kmp_uint64 __kmpc_get_taskid() {
3755 gtid = __kmp_get_gtid();
3759 thread = __kmp_thread_from_gtid(gtid);
3760 return thread->th.th_current_task->td_task_id;
3764 kmp_uint64 __kmpc_get_parent_taskid() {
3768 kmp_taskdata_t *parent_task;
3770 gtid = __kmp_get_gtid();
3774 thread = __kmp_thread_from_gtid(gtid);
3775 parent_task = thread->th.th_current_task->td_parent;
3776 return (parent_task == NULL ? 0 : parent_task->td_task_id);
3792 void __kmpc_doacross_init(
ident_t *loc,
int gtid,
int num_dims,
3793 const struct kmp_dim *dims) {
3795 kmp_int64 last, trace_count;
3796 kmp_info_t *th = __kmp_threads[gtid];
3797 kmp_team_t *team = th->th.th_team;
3799 kmp_disp_t *pr_buf = th->th.th_dispatch;
3800 dispatch_shared_info_t *sh_buf;
3804 (
"__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n",
3805 gtid, num_dims, !team->t.t_serialized));
3806 KMP_DEBUG_ASSERT(dims != NULL);
3807 KMP_DEBUG_ASSERT(num_dims > 0);
3809 if (team->t.t_serialized) {
3810 KA_TRACE(20, (
"__kmpc_doacross_init() exit: serialized team\n"));
3813 KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
3814 idx = pr_buf->th_doacross_buf_idx++;
3816 sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
3819 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL);
3820 pr_buf->th_doacross_info = (kmp_int64 *)__kmp_thread_malloc(
3821 th,
sizeof(kmp_int64) * (4 * num_dims + 1));
3822 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
3823 pr_buf->th_doacross_info[0] =
3824 (kmp_int64)num_dims;
3827 pr_buf->th_doacross_info[1] = (kmp_int64)&sh_buf->doacross_num_done;
3828 pr_buf->th_doacross_info[2] = dims[0].lo;
3829 pr_buf->th_doacross_info[3] = dims[0].up;
3830 pr_buf->th_doacross_info[4] = dims[0].st;
3832 for (j = 1; j < num_dims; ++j) {
3835 if (dims[j].st == 1) {
3837 range_length = dims[j].up - dims[j].lo + 1;
3839 if (dims[j].st > 0) {
3840 KMP_DEBUG_ASSERT(dims[j].up > dims[j].lo);
3841 range_length = (kmp_uint64)(dims[j].up - dims[j].lo) / dims[j].st + 1;
3843 KMP_DEBUG_ASSERT(dims[j].lo > dims[j].up);
3845 (kmp_uint64)(dims[j].lo - dims[j].up) / (-dims[j].st) + 1;
3848 pr_buf->th_doacross_info[last++] = range_length;
3849 pr_buf->th_doacross_info[last++] = dims[j].lo;
3850 pr_buf->th_doacross_info[last++] = dims[j].up;
3851 pr_buf->th_doacross_info[last++] = dims[j].st;
3856 if (dims[0].st == 1) {
3857 trace_count = dims[0].up - dims[0].lo + 1;
3858 }
else if (dims[0].st > 0) {
3859 KMP_DEBUG_ASSERT(dims[0].up > dims[0].lo);
3860 trace_count = (kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].st + 1;
3862 KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up);
3863 trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1;
3865 for (j = 1; j < num_dims; ++j) {
3866 trace_count *= pr_buf->th_doacross_info[4 * j + 1];
3868 KMP_DEBUG_ASSERT(trace_count > 0);
3872 if (idx != sh_buf->doacross_buf_idx) {
3874 __kmp_wait_yield_4((
volatile kmp_uint32 *)&sh_buf->doacross_buf_idx, idx,
3881 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET32(
3882 (
volatile kmp_int32 *)&sh_buf->doacross_flags, NULL, 1);
3884 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET64(
3885 (
volatile kmp_int64 *)&sh_buf->doacross_flags, NULL, 1LL);
3887 if (flags == NULL) {
3889 size_t size = trace_count / 8 + 8;
3890 flags = (kmp_uint32 *)__kmp_thread_calloc(th, size, 1);
3892 sh_buf->doacross_flags = flags;
3893 }
else if (flags == (kmp_uint32 *)1) {
3896 while (*(
volatile kmp_int32 *)&sh_buf->doacross_flags == 1)
3898 while (*(
volatile kmp_int64 *)&sh_buf->doacross_flags == 1LL)
3905 KMP_DEBUG_ASSERT(sh_buf->doacross_flags > (kmp_uint32 *)1);
3906 pr_buf->th_doacross_flags =
3907 sh_buf->doacross_flags;
3909 KA_TRACE(20, (
"__kmpc_doacross_init() exit: T#%d\n", gtid));
3912 void __kmpc_doacross_wait(
ident_t *loc,
int gtid,
const kmp_int64 *vec) {
3913 kmp_int32 shft, num_dims, i;
3915 kmp_int64 iter_number;
3916 kmp_info_t *th = __kmp_threads[gtid];
3917 kmp_team_t *team = th->th.th_team;
3919 kmp_int64 lo, up, st;
3921 KA_TRACE(20, (
"__kmpc_doacross_wait() enter: called T#%d\n", gtid));
3922 if (team->t.t_serialized) {
3923 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: serialized team\n"));
3928 pr_buf = th->th.th_dispatch;
3929 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
3930 num_dims = pr_buf->th_doacross_info[0];
3931 lo = pr_buf->th_doacross_info[2];
3932 up = pr_buf->th_doacross_info[3];
3933 st = pr_buf->th_doacross_info[4];
3935 if (vec[0] < lo || vec[0] > up) {
3936 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3937 "bounds [%lld,%lld]\n",
3938 gtid, vec[0], lo, up));
3941 iter_number = vec[0] - lo;
3942 }
else if (st > 0) {
3943 if (vec[0] < lo || vec[0] > up) {
3944 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3945 "bounds [%lld,%lld]\n",
3946 gtid, vec[0], lo, up));
3949 iter_number = (kmp_uint64)(vec[0] - lo) / st;
3951 if (vec[0] > lo || vec[0] < up) {
3952 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3953 "bounds [%lld,%lld]\n",
3954 gtid, vec[0], lo, up));
3957 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
3959 for (i = 1; i < num_dims; ++i) {
3961 kmp_int32 j = i * 4;
3962 ln = pr_buf->th_doacross_info[j + 1];
3963 lo = pr_buf->th_doacross_info[j + 2];
3964 up = pr_buf->th_doacross_info[j + 3];
3965 st = pr_buf->th_doacross_info[j + 4];
3967 if (vec[i] < lo || vec[i] > up) {
3968 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3969 "bounds [%lld,%lld]\n",
3970 gtid, vec[i], lo, up));
3974 }
else if (st > 0) {
3975 if (vec[i] < lo || vec[i] > up) {
3976 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3977 "bounds [%lld,%lld]\n",
3978 gtid, vec[i], lo, up));
3981 iter = (kmp_uint64)(vec[i] - lo) / st;
3983 if (vec[i] > lo || vec[i] < up) {
3984 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3985 "bounds [%lld,%lld]\n",
3986 gtid, vec[i], lo, up));
3989 iter = (kmp_uint64)(lo - vec[i]) / (-st);
3991 iter_number = iter + ln * iter_number;
3993 shft = iter_number % 32;
3996 while ((flag & pr_buf->th_doacross_flags[iter_number]) == 0) {
4001 (
"__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
4002 gtid, (iter_number << 5) + shft));
4005 void __kmpc_doacross_post(
ident_t *loc,
int gtid,
const kmp_int64 *vec) {
4006 kmp_int32 shft, num_dims, i;
4008 kmp_int64 iter_number;
4009 kmp_info_t *th = __kmp_threads[gtid];
4010 kmp_team_t *team = th->th.th_team;
4014 KA_TRACE(20, (
"__kmpc_doacross_post() enter: called T#%d\n", gtid));
4015 if (team->t.t_serialized) {
4016 KA_TRACE(20, (
"__kmpc_doacross_post() exit: serialized team\n"));
4022 pr_buf = th->th.th_dispatch;
4023 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
4024 num_dims = pr_buf->th_doacross_info[0];
4025 lo = pr_buf->th_doacross_info[2];
4026 st = pr_buf->th_doacross_info[4];
4028 iter_number = vec[0] - lo;
4029 }
else if (st > 0) {
4030 iter_number = (kmp_uint64)(vec[0] - lo) / st;
4032 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
4034 for (i = 1; i < num_dims; ++i) {
4036 kmp_int32 j = i * 4;
4037 ln = pr_buf->th_doacross_info[j + 1];
4038 lo = pr_buf->th_doacross_info[j + 2];
4039 st = pr_buf->th_doacross_info[j + 4];
4042 }
else if (st > 0) {
4043 iter = (kmp_uint64)(vec[i] - lo) / st;
4045 iter = (kmp_uint64)(lo - vec[i]) / (-st);
4047 iter_number = iter + ln * iter_number;
4049 shft = iter_number % 32;
4053 if ((flag & pr_buf->th_doacross_flags[iter_number]) == 0)
4054 KMP_TEST_THEN_OR32(&pr_buf->th_doacross_flags[iter_number], flag);
4055 KA_TRACE(20, (
"__kmpc_doacross_post() exit: T#%d iter %lld posted\n", gtid,
4056 (iter_number << 5) + shft));
4059 void __kmpc_doacross_fini(
ident_t *loc,
int gtid) {
4061 kmp_info_t *th = __kmp_threads[gtid];
4062 kmp_team_t *team = th->th.th_team;
4063 kmp_disp_t *pr_buf = th->th.th_dispatch;
4065 KA_TRACE(20, (
"__kmpc_doacross_fini() enter: called T#%d\n", gtid));
4066 if (team->t.t_serialized) {
4067 KA_TRACE(20, (
"__kmpc_doacross_fini() exit: serialized team %p\n", team));
4070 num_done = KMP_TEST_THEN_INC32((kmp_int32 *)pr_buf->th_doacross_info[1]) + 1;
4071 if (num_done == th->th.th_team_nproc) {
4073 int idx = pr_buf->th_doacross_buf_idx - 1;
4074 dispatch_shared_info_t *sh_buf =
4075 &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
4076 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] ==
4077 (kmp_int64)&sh_buf->doacross_num_done);
4078 KMP_DEBUG_ASSERT(num_done == sh_buf->doacross_num_done);
4079 KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx);
4080 __kmp_thread_free(th, CCAST(kmp_uint32 *, sh_buf->doacross_flags));
4081 sh_buf->doacross_flags = NULL;
4082 sh_buf->doacross_num_done = 0;
4083 sh_buf->doacross_buf_idx +=
4084 __kmp_dispatch_num_buffers;
4087 pr_buf->th_doacross_flags = NULL;
4088 __kmp_thread_free(th, (
void *)pr_buf->th_doacross_info);
4089 pr_buf->th_doacross_info = NULL;
4090 KA_TRACE(20, (
"__kmpc_doacross_fini() exit: T#%d\n", gtid));
4095 int __kmpc_get_target_offload(
void) {
4096 if (!__kmp_init_serial) {
4097 __kmp_serial_initialize();
4099 return __kmp_target_offload;
4101 #endif // OMP_50_ENABLED kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
kmp_int32 __kmpc_global_thread_num(ident_t *loc)
void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
void __kmpc_flush(ident_t *loc)
kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end(ident_t *loc)
void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid)
void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
void __kmpc_begin(ident_t *loc, kmp_int32 flags)
kmp_int32 __kmpc_bound_thread_num(ident_t *loc)
kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid)
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
kmp_int32 __kmpc_in_parallel(ident_t *loc)
kmp_int32 __kmpc_ok_to_fork(ident_t *loc)
kmp_int32 __kmpc_global_num_threads(ident_t *loc)
kmp_int32 __kmpc_bound_num_threads(ident_t *loc)
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
void(* kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid)
void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
stats_state_e
the states which a thread can be in
void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)